From 62255231b38489be68ed3724390f811a39982355 Mon Sep 17 00:00:00 2001 From: Uday Kishore Resu Date: Sat, 10 Oct 2026 00:27:10 -0400 Subject: [PATCH] Implement load operand support for carry and borrow operations in AMD64 backend - Added new operations for ADCQload, ADDQcarryload, SBBQload, and SUBQborrowload in the SSA rewrite rules. - Enhanced the flag allocation logic to handle cases where carry and borrow values are read from memory. - Introduced new test cases to validate the behavior of carry and borrow operations when operands are loaded from memory. - Ensured that the recomputation of flags after function calls does not read memory again, preserving the correctness of carry values. --- src/cmd/compile/internal/amd64/ssa.go | 8 + src/cmd/compile/internal/ssa/_gen/AMD64.rules | 19 + src/cmd/compile/internal/ssa/_gen/AMD64Ops.go | 10 + .../internal/ssa/_gen/AMD64splitload.rules | 7 + .../ssa/rewrite/amd64/rewriteAMD64.go | 404 ++++++++++++++++++ .../amd64splitload/rewriteAMD64splitload.go | 100 +++++ src/cmd/compile/internal/ssa/ssaop/opGen.go | 84 ++++ .../compile/internal/ssacompile/flagalloc.go | 69 ++- test/codegen/mathbits.go | 35 ++ test/fixedbugs/issue80400.go | 78 ++++ 10 files changed, 800 insertions(+), 14 deletions(-) create mode 100644 test/fixedbugs/issue80400.go diff --git a/src/cmd/compile/internal/amd64/ssa.go b/src/cmd/compile/internal/amd64/ssa.go index c4833d5a2c3461..fe4b39602cb813 100644 --- a/src/cmd/compile/internal/amd64/ssa.go +++ b/src/cmd/compile/internal/amd64/ssa.go @@ -564,6 +564,14 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) { p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg0() + case ssaop.OpAMD64ADDQcarryload, ssaop.OpAMD64ADCQload, ssaop.OpAMD64SUBQborrowload, ssaop.OpAMD64SBBQload: + p := s.Prog(v.Op.Asm()) + p.From.Type = obj.TYPE_MEM + p.From.Reg = v.Args[1].Reg() + ssagen.AddAux(&p.From, v) + p.To.Type = obj.TYPE_REG + p.To.Reg = v.Reg0() + case ssaop.OpAMD64ADDQconstcarry, ssaop.OpAMD64ADCQconst, ssaop.OpAMD64SUBQconstborrow, ssaop.OpAMD64SBBQconst: p := s.Prog(v.Op.Asm()) p.From.Type = obj.TYPE_CONST diff --git a/src/cmd/compile/internal/ssa/_gen/AMD64.rules b/src/cmd/compile/internal/ssa/_gen/AMD64.rules index 118002ffd5ed17..989942a9ad564e 100644 --- a/src/cmd/compile/internal/ssa/_gen/AMD64.rules +++ b/src/cmd/compile/internal/ssa/_gen/AMD64.rules @@ -48,6 +48,15 @@ (ADCQconst x [c] (InvertFlags f)) => (ADCQconst x [c] (Select1 (NEGLflags (MOVBQZX (SETA f))))) (SBBQ x y (InvertFlags f)) => (SBBQ x y (Select1 (NEGLflags (MOVBQZX (SETA f))))) (SBBQconst x [c] (InvertFlags f)) => (SBBQconst x [c] (Select1 (NEGLflags (MOVBQZX (SETA f))))) +(ADCQload x [off] {sym} ptr (InvertFlags f) mem) => (ADCQload x [off] {sym} ptr (Select1 (NEGLflags (MOVBQZX (SETA f)))) mem) +(SBBQload x [off] {sym} ptr (InvertFlags f) mem) => (SBBQload x [off] {sym} ptr (Select1 (NEGLflags (MOVBQZX (SETA f)))) mem) +// Merge a load into the second operand of ADCQ and friends. +(ADDQcarry x l:(MOVQload [off] {sym} ptr mem)) && ssa.CanMergeLoadClobber(v, l, x) && ssa.Clobber(l) => (ADDQcarryload x [off] {sym} ptr mem) +(ADCQ x l:(MOVQload [off] {sym} ptr mem) carry) && ssa.CanMergeLoadClobber(v, l, x) && ssa.Clobber(l) => (ADCQload x [off] {sym} ptr carry mem) +(SUBQborrow x l:(MOVQload [off] {sym} ptr mem)) && ssa.CanMergeLoadClobber(v, l, x) && ssa.Clobber(l) => (SUBQborrowload x [off] {sym} ptr mem) +(SBBQ x l:(MOVQload [off] {sym} ptr mem) borrow) && ssa.CanMergeLoadClobber(v, l, x) && ssa.Clobber(l) => (SBBQload x [off] {sym} ptr borrow mem) +(ADCQload x [off] {sym} ptr (FlagEQ) mem) => (ADDQcarryload x [off] {sym} ptr mem) +(SBBQload x [off] {sym} ptr (FlagEQ) mem) => (SUBQborrowload x [off] {sym} ptr mem) // ADDQ/SUBQ an from a carry flag into ADCQ/SBBQ // TODO: maybe add ADCL and SBBL ? (ADDQ x (MOVBQZX (SETB flags))) => (Select0 (ADCQconst [0] x flags)) @@ -1030,6 +1039,10 @@ ((ADD|SUB|AND|OR|XOR)Qload [off1+off2] {sym} val base mem) ((ADD|SUB|AND|OR|XOR)Lload [off1] {sym} val (ADDQconst [off2] base) mem) && ssa.Is32Bit(int64(off1)+int64(off2)) => ((ADD|SUB|AND|OR|XOR)Lload [off1+off2] {sym} val base mem) +((ADDQcarry|SUBQborrow)load [off1] {sym} val (ADDQconst [off2] base) mem) && ssa.Is32Bit(int64(off1)+int64(off2)) => + ((ADDQcarry|SUBQborrow)load [off1+off2] {sym} val base mem) +((ADC|SBB)Qload [off1] {sym} val (ADDQconst [off2] base) carry mem) && ssa.Is32Bit(int64(off1)+int64(off2)) => + ((ADC|SBB)Qload [off1+off2] {sym} val base carry mem) (CMP(Q|L|W|B)load [off1] {sym} (ADDQconst [off2] base) val mem) && ssa.Is32Bit(int64(off1)+int64(off2)) => (CMP(Q|L|W|B)load [off1+off2] {sym} base val mem) (CMP(Q|L|W|B)constload [valoff1] {sym} (ADDQconst [off2] base) mem) && ssa.ValAndOff(valoff1).CanAdd32(off2) => @@ -1085,6 +1098,12 @@ ((ADD|SUB|AND|OR|XOR)Lload [off1] {sym1} val (LEAQ [off2] {sym2} base) mem) && ssa.Is32Bit(int64(off1)+int64(off2)) && ssa.CanMergeSym(sym1, sym2) => ((ADD|SUB|AND|OR|XOR)Lload [off1+off2] {ssa.MergeSym(sym1,sym2)} val base mem) +((ADDQcarry|SUBQborrow)load [off1] {sym1} val (LEAQ [off2] {sym2} base) mem) + && ssa.Is32Bit(int64(off1)+int64(off2)) && ssa.CanMergeSym(sym1, sym2) => + ((ADDQcarry|SUBQborrow)load [off1+off2] {ssa.MergeSym(sym1,sym2)} val base mem) +((ADC|SBB)Qload [off1] {sym1} val (LEAQ [off2] {sym2} base) carry mem) + && ssa.Is32Bit(int64(off1)+int64(off2)) && ssa.CanMergeSym(sym1, sym2) => + ((ADC|SBB)Qload [off1+off2] {ssa.MergeSym(sym1,sym2)} val base carry mem) (CMP(Q|L|W|B)load [off1] {sym1} (LEAQ [off2] {sym2} base) val mem) && ssa.Is32Bit(int64(off1)+int64(off2)) && ssa.CanMergeSym(sym1, sym2) => (CMP(Q|L|W|B)load [off1+off2] {ssa.MergeSym(sym1,sym2)} base val mem) diff --git a/src/cmd/compile/internal/ssa/_gen/AMD64Ops.go b/src/cmd/compile/internal/ssa/_gen/AMD64Ops.go index abeb5d13f6ea91..a9eab728baf156 100644 --- a/src/cmd/compile/internal/ssa/_gen/AMD64Ops.go +++ b/src/cmd/compile/internal/ssa/_gen/AMD64Ops.go @@ -165,6 +165,9 @@ func init() { gp21flags = regInfo{inputs: []regMask{gp, gp}, outputs: []regMask{gp, regMask{}}} gp2flags1flags = regInfo{inputs: []regMask{gp, gp, regMask{}}, outputs: []regMask{gp, regMask{}}} + gp21loadflags = regInfo{inputs: []regMask{gp, gpspsbg, regMask{}}, outputs: []regMask{gp, regMask{}}} + gp2flags1loadflags = regInfo{inputs: []regMask{gp, gpspsbg, regMask{}, regMask{}}, outputs: []regMask{gp, regMask{}}} + gp2flags = regInfo{inputs: []regMask{gpsp, gpsp}} gp1flags = regInfo{inputs: []regMask{gpsp}} gp0flagsLoad = regInfo{inputs: []regMask{gpspsbg, regMask{}}} @@ -429,6 +432,13 @@ func init() { {name: "SUBQconstborrow", argLength: 1, reg: gp11flags, typ: "(UInt64,Flags)", asm: "SUBQ", aux: "Int32", resultInArg0: true}, // r = arg0-auxint {name: "SBBQconst", argLength: 2, reg: gp1flags1flags, typ: "(UInt64,Flags)", asm: "SBBQ", aux: "Int32", resultInArg0: true}, // r = arg0-(auxint+carry(arg1)) + // Load/op combos of the carry and borrow ops above, with the second operand + // read from memory. The memory argument comes last. + {name: "ADDQcarryload", argLength: 3, reg: gp21loadflags, typ: "(UInt64,Flags)", asm: "ADDQ", aux: "SymOff", resultInArg0: true, faultOnNilArg1: true, symEffect: "Read", addrSinkArg1: true}, // r = arg0+*(arg1+auxint+aux), arg2=mem + {name: "ADCQload", argLength: 4, reg: gp2flags1loadflags, typ: "(UInt64,Flags)", asm: "ADCQ", aux: "SymOff", resultInArg0: true, faultOnNilArg1: true, symEffect: "Read", addrSinkArg1: true}, // r = arg0+*(arg1+auxint+aux)+carry(arg2), arg3=mem + {name: "SUBQborrowload", argLength: 3, reg: gp21loadflags, typ: "(UInt64,Flags)", asm: "SUBQ", aux: "SymOff", resultInArg0: true, faultOnNilArg1: true, symEffect: "Read", addrSinkArg1: true}, // r = arg0-*(arg1+auxint+aux), arg2=mem + {name: "SBBQload", argLength: 4, reg: gp2flags1loadflags, typ: "(UInt64,Flags)", asm: "SBBQ", aux: "SymOff", resultInArg0: true, faultOnNilArg1: true, symEffect: "Read", addrSinkArg1: true}, // r = arg0-(*(arg1+auxint+aux)+carry(arg2)), arg3=mem + {name: "MULQU2", argLength: 2, reg: regInfo{inputs: []regMask{ax, gpsp}, outputs: []regMask{dx, ax}}, commutative: true, asm: "MULQ", clobberFlags: true, earlyOk: true}, // arg0 * arg1, returns (hi, lo) // MULXQ is the BMI2 unsigned 64x64->128 multiply. arg0 must be in DX // (the implicit operand); arg1 is any register or memory. Outputs are diff --git a/src/cmd/compile/internal/ssa/_gen/AMD64splitload.rules b/src/cmd/compile/internal/ssa/_gen/AMD64splitload.rules index ad106b53cf3132..4cac4987d729a0 100644 --- a/src/cmd/compile/internal/ssa/_gen/AMD64splitload.rules +++ b/src/cmd/compile/internal/ssa/_gen/AMD64splitload.rules @@ -43,3 +43,10 @@ (CMPQconstloadidx8 {sym} [vo] ptr idx mem) && vo.Val() != 0 => (CMPQconst (MOVQloadidx8 {sym} [vo.Off()] ptr idx mem) [vo.Val()]) (CMPLconstloadidx4 {sym} [vo] ptr idx mem) && vo.Val() != 0 => (CMPLconst (MOVLloadidx4 {sym} [vo.Off()] ptr idx mem) [vo.Val()]) (CMPWconstloadidx2 {sym} [vo] ptr idx mem) && vo.Val() != 0 => (CMPWconst (MOVWloadidx2 {sym} [vo.Off()] ptr idx mem) [vo.Val16()]) + +// Load/op combos of the carry chain ops. These are tuple-valued, so flagalloc +// splits them when it needs to recompute their flags result elsewhere. +(ADDQcarryload {sym} [off] x ptr mem) => (ADDQcarry x (MOVQload {sym} [off] ptr mem)) +(ADCQload {sym} [off] x ptr carry mem) => (ADCQ x (MOVQload {sym} [off] ptr mem) carry) +(SUBQborrowload {sym} [off] x ptr mem) => (SUBQborrow x (MOVQload {sym} [off] ptr mem)) +(SBBQload {sym} [off] x ptr borrow mem) => (SBBQ x (MOVQload {sym} [off] ptr mem) borrow) diff --git a/src/cmd/compile/internal/ssa/rewrite/amd64/rewriteAMD64.go b/src/cmd/compile/internal/ssa/rewrite/amd64/rewriteAMD64.go index 873fd3746bad9a..e44501f0203c95 100644 --- a/src/cmd/compile/internal/ssa/rewrite/amd64/rewriteAMD64.go +++ b/src/cmd/compile/internal/ssa/rewrite/amd64/rewriteAMD64.go @@ -59,6 +59,8 @@ func RewriteValue(v *ssa.Value) bool { return rewriteValue_OpAMD64ADCQ(v) case ssaop.OpAMD64ADCQconst: return rewriteValue_OpAMD64ADCQconst(v) + case ssaop.OpAMD64ADCQload: + return rewriteValue_OpAMD64ADCQload(v) case ssaop.OpAMD64ADDBconstmodify: return rewriteValue_OpAMD64ADDBconstmodify(v) case ssaop.OpAMD64ADDL: @@ -77,6 +79,8 @@ func RewriteValue(v *ssa.Value) bool { return rewriteValue_OpAMD64ADDQ(v) case ssaop.OpAMD64ADDQcarry: return rewriteValue_OpAMD64ADDQcarry(v) + case ssaop.OpAMD64ADDQcarryload: + return rewriteValue_OpAMD64ADDQcarryload(v) case ssaop.OpAMD64ADDQconst: return rewriteValue_OpAMD64ADDQconst(v) case ssaop.OpAMD64ADDQconstmodify: @@ -473,6 +477,8 @@ func RewriteValue(v *ssa.Value) bool { return rewriteValue_OpAMD64SBBQcarrymask(v) case ssaop.OpAMD64SBBQconst: return rewriteValue_OpAMD64SBBQconst(v) + case ssaop.OpAMD64SBBQload: + return rewriteValue_OpAMD64SBBQload(v) case ssaop.OpAMD64SETA: return rewriteValue_OpAMD64SETA(v) case ssaop.OpAMD64SETAE: @@ -557,6 +563,8 @@ func RewriteValue(v *ssa.Value) bool { return rewriteValue_OpAMD64SUBQ(v) case ssaop.OpAMD64SUBQborrow: return rewriteValue_OpAMD64SUBQborrow(v) + case ssaop.OpAMD64SUBQborrowload: + return rewriteValue_OpAMD64SUBQborrowload(v) case ssaop.OpAMD64SUBQconst: return rewriteValue_OpAMD64SUBQconst(v) case ssaop.OpAMD64SUBQload: @@ -7610,6 +7618,32 @@ func rewriteValue_OpAMD64ADCQ(v *ssa.Value) bool { v.AddArg3(x, y, v0) return true } + // match: (ADCQ x l:(MOVQload [off] {sym} ptr mem) carry) + // cond: ssa.CanMergeLoadClobber(v, l, x) && ssa.Clobber(l) + // result: (ADCQload x [off] {sym} ptr carry mem) + for { + for _i0 := 0; _i0 <= 1; _i0, v_0, v_1 = _i0+1, v_1, v_0 { + x := v_0 + l := v_1 + if l.Op != ssaop.OpAMD64MOVQload { + continue + } + off := ssa.AuxIntToInt32(l.AuxInt) + sym := ssa.AuxToSym(l.Aux) + mem := l.Args[1] + ptr := l.Args[0] + carry := v_2 + if !(ssa.CanMergeLoadClobber(v, l, x) && ssa.Clobber(l)) { + continue + } + v.Reset(ssaop.OpAMD64ADCQload) + v.AuxInt = ssa.Int32ToAuxInt(off) + v.Aux = ssa.SymToAux(sym) + v.AddArg4(x, ptr, carry, mem) + return true + } + break + } return false } func rewriteValue_OpAMD64ADCQconst(v *ssa.Value) bool { @@ -7654,6 +7688,105 @@ func rewriteValue_OpAMD64ADCQconst(v *ssa.Value) bool { } return false } +func rewriteValue_OpAMD64ADCQload(v *ssa.Value) bool { + v_3 := v.Args[3] + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + b := v.Block + typ := &b.Func.Config.Types + // match: (ADCQload x [off] {sym} ptr (InvertFlags f) mem) + // result: (ADCQload x [off] {sym} ptr (Select1 (NEGLflags (MOVBQZX (SETA f)))) mem) + for { + off := ssa.AuxIntToInt32(v.AuxInt) + sym := ssa.AuxToSym(v.Aux) + x := v_0 + ptr := v_1 + if v_2.Op != ssaop.OpAMD64InvertFlags { + break + } + f := v_2.Args[0] + mem := v_3 + v.Reset(ssaop.OpAMD64ADCQload) + v.AuxInt = ssa.Int32ToAuxInt(off) + v.Aux = ssa.SymToAux(sym) + v0 := b.NewValue0(v.Pos, ssaop.OpSelect1, types.TypeFlags) + v1 := b.NewValue0(v.Pos, ssaop.OpAMD64NEGLflags, types.NewTuple(typ.UInt32, types.TypeFlags)) + v2 := b.NewValue0(v.Pos, ssaop.OpAMD64MOVBQZX, types.Types[types.TUINT32]) + v3 := b.NewValue0(v.Pos, ssaop.OpAMD64SETA, types.Types[types.TUINT8]) + v3.AddArg(f) + v2.AddArg(v3) + v1.AddArg(v2) + v0.AddArg(v1) + v.AddArg4(x, ptr, v0, mem) + return true + } + // match: (ADCQload x [off] {sym} ptr (FlagEQ) mem) + // result: (ADDQcarryload x [off] {sym} ptr mem) + for { + off := ssa.AuxIntToInt32(v.AuxInt) + sym := ssa.AuxToSym(v.Aux) + x := v_0 + ptr := v_1 + if v_2.Op != ssaop.OpAMD64FlagEQ { + break + } + mem := v_3 + v.Reset(ssaop.OpAMD64ADDQcarryload) + v.AuxInt = ssa.Int32ToAuxInt(off) + v.Aux = ssa.SymToAux(sym) + v.AddArg3(x, ptr, mem) + return true + } + // match: (ADCQload [off1] {sym} val (ADDQconst [off2] base) carry mem) + // cond: ssa.Is32Bit(int64(off1)+int64(off2)) + // result: (ADCQload [off1+off2] {sym} val base carry mem) + for { + off1 := ssa.AuxIntToInt32(v.AuxInt) + sym := ssa.AuxToSym(v.Aux) + val := v_0 + if v_1.Op != ssaop.OpAMD64ADDQconst { + break + } + off2 := ssa.AuxIntToInt32(v_1.AuxInt) + base := v_1.Args[0] + carry := v_2 + mem := v_3 + if !(ssa.Is32Bit(int64(off1) + int64(off2))) { + break + } + v.Reset(ssaop.OpAMD64ADCQload) + v.AuxInt = ssa.Int32ToAuxInt(off1 + off2) + v.Aux = ssa.SymToAux(sym) + v.AddArg4(val, base, carry, mem) + return true + } + // match: (ADCQload [off1] {sym1} val (LEAQ [off2] {sym2} base) carry mem) + // cond: ssa.Is32Bit(int64(off1)+int64(off2)) && ssa.CanMergeSym(sym1, sym2) + // result: (ADCQload [off1+off2] {ssa.MergeSym(sym1,sym2)} val base carry mem) + for { + off1 := ssa.AuxIntToInt32(v.AuxInt) + sym1 := ssa.AuxToSym(v.Aux) + val := v_0 + if v_1.Op != ssaop.OpAMD64LEAQ { + break + } + off2 := ssa.AuxIntToInt32(v_1.AuxInt) + sym2 := ssa.AuxToSym(v_1.Aux) + base := v_1.Args[0] + carry := v_2 + mem := v_3 + if !(ssa.Is32Bit(int64(off1)+int64(off2)) && ssa.CanMergeSym(sym1, sym2)) { + break + } + v.Reset(ssaop.OpAMD64ADCQload) + v.AuxInt = ssa.Int32ToAuxInt(off1 + off2) + v.Aux = ssa.SymToAux(ssa.MergeSym(sym1, sym2)) + v.AddArg4(val, base, carry, mem) + return true + } + return false +} func rewriteValue_OpAMD64ADDBconstmodify(v *ssa.Value) bool { v_1 := v.Args[1] v_0 := v.Args[0] @@ -8563,6 +8696,82 @@ func rewriteValue_OpAMD64ADDQcarry(v *ssa.Value) bool { } break } + // match: (ADDQcarry x l:(MOVQload [off] {sym} ptr mem)) + // cond: ssa.CanMergeLoadClobber(v, l, x) && ssa.Clobber(l) + // result: (ADDQcarryload x [off] {sym} ptr mem) + for { + for _i0 := 0; _i0 <= 1; _i0, v_0, v_1 = _i0+1, v_1, v_0 { + x := v_0 + l := v_1 + if l.Op != ssaop.OpAMD64MOVQload { + continue + } + off := ssa.AuxIntToInt32(l.AuxInt) + sym := ssa.AuxToSym(l.Aux) + mem := l.Args[1] + ptr := l.Args[0] + if !(ssa.CanMergeLoadClobber(v, l, x) && ssa.Clobber(l)) { + continue + } + v.Reset(ssaop.OpAMD64ADDQcarryload) + v.AuxInt = ssa.Int32ToAuxInt(off) + v.Aux = ssa.SymToAux(sym) + v.AddArg3(x, ptr, mem) + return true + } + break + } + return false +} +func rewriteValue_OpAMD64ADDQcarryload(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (ADDQcarryload [off1] {sym} val (ADDQconst [off2] base) mem) + // cond: ssa.Is32Bit(int64(off1)+int64(off2)) + // result: (ADDQcarryload [off1+off2] {sym} val base mem) + for { + off1 := ssa.AuxIntToInt32(v.AuxInt) + sym := ssa.AuxToSym(v.Aux) + val := v_0 + if v_1.Op != ssaop.OpAMD64ADDQconst { + break + } + off2 := ssa.AuxIntToInt32(v_1.AuxInt) + base := v_1.Args[0] + mem := v_2 + if !(ssa.Is32Bit(int64(off1) + int64(off2))) { + break + } + v.Reset(ssaop.OpAMD64ADDQcarryload) + v.AuxInt = ssa.Int32ToAuxInt(off1 + off2) + v.Aux = ssa.SymToAux(sym) + v.AddArg3(val, base, mem) + return true + } + // match: (ADDQcarryload [off1] {sym1} val (LEAQ [off2] {sym2} base) mem) + // cond: ssa.Is32Bit(int64(off1)+int64(off2)) && ssa.CanMergeSym(sym1, sym2) + // result: (ADDQcarryload [off1+off2] {ssa.MergeSym(sym1,sym2)} val base mem) + for { + off1 := ssa.AuxIntToInt32(v.AuxInt) + sym1 := ssa.AuxToSym(v.Aux) + val := v_0 + if v_1.Op != ssaop.OpAMD64LEAQ { + break + } + off2 := ssa.AuxIntToInt32(v_1.AuxInt) + sym2 := ssa.AuxToSym(v_1.Aux) + base := v_1.Args[0] + mem := v_2 + if !(ssa.Is32Bit(int64(off1)+int64(off2)) && ssa.CanMergeSym(sym1, sym2)) { + break + } + v.Reset(ssaop.OpAMD64ADDQcarryload) + v.AuxInt = ssa.Int32ToAuxInt(off1 + off2) + v.Aux = ssa.SymToAux(ssa.MergeSym(sym1, sym2)) + v.AddArg3(val, base, mem) + return true + } return false } func rewriteValue_OpAMD64ADDQconst(v *ssa.Value) bool { @@ -32281,6 +32490,29 @@ func rewriteValue_OpAMD64SBBQ(v *ssa.Value) bool { v.AddArg3(x, y, v0) return true } + // match: (SBBQ x l:(MOVQload [off] {sym} ptr mem) borrow) + // cond: ssa.CanMergeLoadClobber(v, l, x) && ssa.Clobber(l) + // result: (SBBQload x [off] {sym} ptr borrow mem) + for { + x := v_0 + l := v_1 + if l.Op != ssaop.OpAMD64MOVQload { + break + } + off := ssa.AuxIntToInt32(l.AuxInt) + sym := ssa.AuxToSym(l.Aux) + mem := l.Args[1] + ptr := l.Args[0] + borrow := v_2 + if !(ssa.CanMergeLoadClobber(v, l, x) && ssa.Clobber(l)) { + break + } + v.Reset(ssaop.OpAMD64SBBQload) + v.AuxInt = ssa.Int32ToAuxInt(off) + v.Aux = ssa.SymToAux(sym) + v.AddArg4(x, ptr, borrow, mem) + return true + } return false } func rewriteValue_OpAMD64SBBQcarrymask(v *ssa.Value) bool { @@ -32379,6 +32611,105 @@ func rewriteValue_OpAMD64SBBQconst(v *ssa.Value) bool { } return false } +func rewriteValue_OpAMD64SBBQload(v *ssa.Value) bool { + v_3 := v.Args[3] + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + b := v.Block + typ := &b.Func.Config.Types + // match: (SBBQload x [off] {sym} ptr (InvertFlags f) mem) + // result: (SBBQload x [off] {sym} ptr (Select1 (NEGLflags (MOVBQZX (SETA f)))) mem) + for { + off := ssa.AuxIntToInt32(v.AuxInt) + sym := ssa.AuxToSym(v.Aux) + x := v_0 + ptr := v_1 + if v_2.Op != ssaop.OpAMD64InvertFlags { + break + } + f := v_2.Args[0] + mem := v_3 + v.Reset(ssaop.OpAMD64SBBQload) + v.AuxInt = ssa.Int32ToAuxInt(off) + v.Aux = ssa.SymToAux(sym) + v0 := b.NewValue0(v.Pos, ssaop.OpSelect1, types.TypeFlags) + v1 := b.NewValue0(v.Pos, ssaop.OpAMD64NEGLflags, types.NewTuple(typ.UInt32, types.TypeFlags)) + v2 := b.NewValue0(v.Pos, ssaop.OpAMD64MOVBQZX, types.Types[types.TUINT32]) + v3 := b.NewValue0(v.Pos, ssaop.OpAMD64SETA, types.Types[types.TUINT8]) + v3.AddArg(f) + v2.AddArg(v3) + v1.AddArg(v2) + v0.AddArg(v1) + v.AddArg4(x, ptr, v0, mem) + return true + } + // match: (SBBQload x [off] {sym} ptr (FlagEQ) mem) + // result: (SUBQborrowload x [off] {sym} ptr mem) + for { + off := ssa.AuxIntToInt32(v.AuxInt) + sym := ssa.AuxToSym(v.Aux) + x := v_0 + ptr := v_1 + if v_2.Op != ssaop.OpAMD64FlagEQ { + break + } + mem := v_3 + v.Reset(ssaop.OpAMD64SUBQborrowload) + v.AuxInt = ssa.Int32ToAuxInt(off) + v.Aux = ssa.SymToAux(sym) + v.AddArg3(x, ptr, mem) + return true + } + // match: (SBBQload [off1] {sym} val (ADDQconst [off2] base) carry mem) + // cond: ssa.Is32Bit(int64(off1)+int64(off2)) + // result: (SBBQload [off1+off2] {sym} val base carry mem) + for { + off1 := ssa.AuxIntToInt32(v.AuxInt) + sym := ssa.AuxToSym(v.Aux) + val := v_0 + if v_1.Op != ssaop.OpAMD64ADDQconst { + break + } + off2 := ssa.AuxIntToInt32(v_1.AuxInt) + base := v_1.Args[0] + carry := v_2 + mem := v_3 + if !(ssa.Is32Bit(int64(off1) + int64(off2))) { + break + } + v.Reset(ssaop.OpAMD64SBBQload) + v.AuxInt = ssa.Int32ToAuxInt(off1 + off2) + v.Aux = ssa.SymToAux(sym) + v.AddArg4(val, base, carry, mem) + return true + } + // match: (SBBQload [off1] {sym1} val (LEAQ [off2] {sym2} base) carry mem) + // cond: ssa.Is32Bit(int64(off1)+int64(off2)) && ssa.CanMergeSym(sym1, sym2) + // result: (SBBQload [off1+off2] {ssa.MergeSym(sym1,sym2)} val base carry mem) + for { + off1 := ssa.AuxIntToInt32(v.AuxInt) + sym1 := ssa.AuxToSym(v.Aux) + val := v_0 + if v_1.Op != ssaop.OpAMD64LEAQ { + break + } + off2 := ssa.AuxIntToInt32(v_1.AuxInt) + sym2 := ssa.AuxToSym(v_1.Aux) + base := v_1.Args[0] + carry := v_2 + mem := v_3 + if !(ssa.Is32Bit(int64(off1)+int64(off2)) && ssa.CanMergeSym(sym1, sym2)) { + break + } + v.Reset(ssaop.OpAMD64SBBQload) + v.AuxInt = ssa.Int32ToAuxInt(off1 + off2) + v.Aux = ssa.SymToAux(ssa.MergeSym(sym1, sym2)) + v.AddArg4(val, base, carry, mem) + return true + } + return false +} func rewriteValue_OpAMD64SETA(v *ssa.Value) bool { v_0 := v.Args[0] // match: (SETA (InvertFlags x)) @@ -41167,6 +41498,79 @@ func rewriteValue_OpAMD64SUBQborrow(v *ssa.Value) bool { v.AddArg(x) return true } + // match: (SUBQborrow x l:(MOVQload [off] {sym} ptr mem)) + // cond: ssa.CanMergeLoadClobber(v, l, x) && ssa.Clobber(l) + // result: (SUBQborrowload x [off] {sym} ptr mem) + for { + x := v_0 + l := v_1 + if l.Op != ssaop.OpAMD64MOVQload { + break + } + off := ssa.AuxIntToInt32(l.AuxInt) + sym := ssa.AuxToSym(l.Aux) + mem := l.Args[1] + ptr := l.Args[0] + if !(ssa.CanMergeLoadClobber(v, l, x) && ssa.Clobber(l)) { + break + } + v.Reset(ssaop.OpAMD64SUBQborrowload) + v.AuxInt = ssa.Int32ToAuxInt(off) + v.Aux = ssa.SymToAux(sym) + v.AddArg3(x, ptr, mem) + return true + } + return false +} +func rewriteValue_OpAMD64SUBQborrowload(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (SUBQborrowload [off1] {sym} val (ADDQconst [off2] base) mem) + // cond: ssa.Is32Bit(int64(off1)+int64(off2)) + // result: (SUBQborrowload [off1+off2] {sym} val base mem) + for { + off1 := ssa.AuxIntToInt32(v.AuxInt) + sym := ssa.AuxToSym(v.Aux) + val := v_0 + if v_1.Op != ssaop.OpAMD64ADDQconst { + break + } + off2 := ssa.AuxIntToInt32(v_1.AuxInt) + base := v_1.Args[0] + mem := v_2 + if !(ssa.Is32Bit(int64(off1) + int64(off2))) { + break + } + v.Reset(ssaop.OpAMD64SUBQborrowload) + v.AuxInt = ssa.Int32ToAuxInt(off1 + off2) + v.Aux = ssa.SymToAux(sym) + v.AddArg3(val, base, mem) + return true + } + // match: (SUBQborrowload [off1] {sym1} val (LEAQ [off2] {sym2} base) mem) + // cond: ssa.Is32Bit(int64(off1)+int64(off2)) && ssa.CanMergeSym(sym1, sym2) + // result: (SUBQborrowload [off1+off2] {ssa.MergeSym(sym1,sym2)} val base mem) + for { + off1 := ssa.AuxIntToInt32(v.AuxInt) + sym1 := ssa.AuxToSym(v.Aux) + val := v_0 + if v_1.Op != ssaop.OpAMD64LEAQ { + break + } + off2 := ssa.AuxIntToInt32(v_1.AuxInt) + sym2 := ssa.AuxToSym(v_1.Aux) + base := v_1.Args[0] + mem := v_2 + if !(ssa.Is32Bit(int64(off1)+int64(off2)) && ssa.CanMergeSym(sym1, sym2)) { + break + } + v.Reset(ssaop.OpAMD64SUBQborrowload) + v.AuxInt = ssa.Int32ToAuxInt(off1 + off2) + v.Aux = ssa.SymToAux(ssa.MergeSym(sym1, sym2)) + v.AddArg3(val, base, mem) + return true + } return false } func rewriteValue_OpAMD64SUBQconst(v *ssa.Value) bool { diff --git a/src/cmd/compile/internal/ssa/rewrite/amd64splitload/rewriteAMD64splitload.go b/src/cmd/compile/internal/ssa/rewrite/amd64splitload/rewriteAMD64splitload.go index 50cbc150dbdec2..06eeda83c2bdd5 100644 --- a/src/cmd/compile/internal/ssa/rewrite/amd64splitload/rewriteAMD64splitload.go +++ b/src/cmd/compile/internal/ssa/rewrite/amd64splitload/rewriteAMD64splitload.go @@ -7,6 +7,10 @@ import "cmd/compile/internal/ssa" func RewriteValue(v *ssa.Value) bool { switch v.Op { + case ssaop.OpAMD64ADCQload: + return rewriteValue_OpAMD64ADCQload(v) + case ssaop.OpAMD64ADDQcarryload: + return rewriteValue_OpAMD64ADDQcarryload(v) case ssaop.OpAMD64CMPBconstload: return rewriteValue_OpAMD64CMPBconstload(v) case ssaop.OpAMD64CMPBconstloadidx1: @@ -51,9 +55,59 @@ func RewriteValue(v *ssa.Value) bool { return rewriteValue_OpAMD64CMPWloadidx1(v) case ssaop.OpAMD64CMPWloadidx2: return rewriteValue_OpAMD64CMPWloadidx2(v) + case ssaop.OpAMD64SBBQload: + return rewriteValue_OpAMD64SBBQload(v) + case ssaop.OpAMD64SUBQborrowload: + return rewriteValue_OpAMD64SUBQborrowload(v) } return false } +func rewriteValue_OpAMD64ADCQload(v *ssa.Value) bool { + v_3 := v.Args[3] + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + b := v.Block + // match: (ADCQload {sym} [off] x ptr carry mem) + // result: (ADCQ x (MOVQload {sym} [off] ptr mem) carry) + for { + off := ssa.AuxIntToInt32(v.AuxInt) + sym := ssa.AuxToSym(v.Aux) + x := v_0 + ptr := v_1 + carry := v_2 + mem := v_3 + v.Reset(ssaop.OpAMD64ADCQ) + v0 := b.NewValue0(v.Pos, ssaop.OpAMD64MOVQload, x.Type) + v0.AuxInt = ssa.Int32ToAuxInt(off) + v0.Aux = ssa.SymToAux(sym) + v0.AddArg2(ptr, mem) + v.AddArg3(x, v0, carry) + return true + } +} +func rewriteValue_OpAMD64ADDQcarryload(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + b := v.Block + // match: (ADDQcarryload {sym} [off] x ptr mem) + // result: (ADDQcarry x (MOVQload {sym} [off] ptr mem)) + for { + off := ssa.AuxIntToInt32(v.AuxInt) + sym := ssa.AuxToSym(v.Aux) + x := v_0 + ptr := v_1 + mem := v_2 + v.Reset(ssaop.OpAMD64ADDQcarry) + v0 := b.NewValue0(v.Pos, ssaop.OpAMD64MOVQload, x.Type) + v0.AuxInt = ssa.Int32ToAuxInt(off) + v0.Aux = ssa.SymToAux(sym) + v0.AddArg2(ptr, mem) + v.AddArg2(x, v0) + return true + } +} func rewriteValue_OpAMD64CMPBconstload(v *ssa.Value) bool { v_1 := v.Args[1] v_0 := v.Args[0] @@ -837,6 +891,52 @@ func rewriteValue_OpAMD64CMPWloadidx2(v *ssa.Value) bool { return true } } +func rewriteValue_OpAMD64SBBQload(v *ssa.Value) bool { + v_3 := v.Args[3] + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + b := v.Block + // match: (SBBQload {sym} [off] x ptr borrow mem) + // result: (SBBQ x (MOVQload {sym} [off] ptr mem) borrow) + for { + off := ssa.AuxIntToInt32(v.AuxInt) + sym := ssa.AuxToSym(v.Aux) + x := v_0 + ptr := v_1 + borrow := v_2 + mem := v_3 + v.Reset(ssaop.OpAMD64SBBQ) + v0 := b.NewValue0(v.Pos, ssaop.OpAMD64MOVQload, x.Type) + v0.AuxInt = ssa.Int32ToAuxInt(off) + v0.Aux = ssa.SymToAux(sym) + v0.AddArg2(ptr, mem) + v.AddArg3(x, v0, borrow) + return true + } +} +func rewriteValue_OpAMD64SUBQborrowload(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + b := v.Block + // match: (SUBQborrowload {sym} [off] x ptr mem) + // result: (SUBQborrow x (MOVQload {sym} [off] ptr mem)) + for { + off := ssa.AuxIntToInt32(v.AuxInt) + sym := ssa.AuxToSym(v.Aux) + x := v_0 + ptr := v_1 + mem := v_2 + v.Reset(ssaop.OpAMD64SUBQborrow) + v0 := b.NewValue0(v.Pos, ssaop.OpAMD64MOVQload, x.Type) + v0.AuxInt = ssa.Int32ToAuxInt(off) + v0.Aux = ssa.SymToAux(sym) + v0.AddArg2(ptr, mem) + v.AddArg2(x, v0) + return true + } +} func RewriteBlock(b *ssa.Block) bool { return false } diff --git a/src/cmd/compile/internal/ssa/ssaop/opGen.go b/src/cmd/compile/internal/ssa/ssaop/opGen.go index 54dc21a2896dcd..e64cb08b40ee27 100644 --- a/src/cmd/compile/internal/ssa/ssaop/opGen.go +++ b/src/cmd/compile/internal/ssa/ssaop/opGen.go @@ -342,6 +342,10 @@ const ( OpAMD64SBBQ OpAMD64SUBQconstborrow OpAMD64SBBQconst + OpAMD64ADDQcarryload + OpAMD64ADCQload + OpAMD64SUBQborrowload + OpAMD64SBBQload OpAMD64MULQU2 OpAMD64MULXQ OpAMD64DIVQU2 @@ -14144,6 +14148,86 @@ var OpcodeTable = [...]OpInfo{ }, }, }, + { + Name: "ADDQcarryload", + AuxType: AuxTypeSymOff, + ArgLen: 3, + ResultInArg0: true, + FaultOnNilArg1: true, + AddrSinkArg1: true, + symEffect: SymRead, + asm: x86.AADDQ, + Reg: RegInfo{ + Inputs: []InputInfo{ + {0, RegMask{V1: 49135, V2: 0}}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R15 + {1, RegMask{V1: 72057594037993471, V2: 0}}, // AX CX DX BX SP BP SI DI R8 R9 R10 R11 R12 R13 g R15 SB + }, + Outputs: []OutputInfo{ + {1, RegMask{V1: 0, V2: 0}}, + {0, RegMask{V1: 49135, V2: 0}}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R15 + }, + }, + }, + { + Name: "ADCQload", + AuxType: AuxTypeSymOff, + ArgLen: 4, + ResultInArg0: true, + FaultOnNilArg1: true, + AddrSinkArg1: true, + symEffect: SymRead, + asm: x86.AADCQ, + Reg: RegInfo{ + Inputs: []InputInfo{ + {0, RegMask{V1: 49135, V2: 0}}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R15 + {1, RegMask{V1: 72057594037993471, V2: 0}}, // AX CX DX BX SP BP SI DI R8 R9 R10 R11 R12 R13 g R15 SB + }, + Outputs: []OutputInfo{ + {1, RegMask{V1: 0, V2: 0}}, + {0, RegMask{V1: 49135, V2: 0}}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R15 + }, + }, + }, + { + Name: "SUBQborrowload", + AuxType: AuxTypeSymOff, + ArgLen: 3, + ResultInArg0: true, + FaultOnNilArg1: true, + AddrSinkArg1: true, + symEffect: SymRead, + asm: x86.ASUBQ, + Reg: RegInfo{ + Inputs: []InputInfo{ + {0, RegMask{V1: 49135, V2: 0}}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R15 + {1, RegMask{V1: 72057594037993471, V2: 0}}, // AX CX DX BX SP BP SI DI R8 R9 R10 R11 R12 R13 g R15 SB + }, + Outputs: []OutputInfo{ + {1, RegMask{V1: 0, V2: 0}}, + {0, RegMask{V1: 49135, V2: 0}}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R15 + }, + }, + }, + { + Name: "SBBQload", + AuxType: AuxTypeSymOff, + ArgLen: 4, + ResultInArg0: true, + FaultOnNilArg1: true, + AddrSinkArg1: true, + symEffect: SymRead, + asm: x86.ASBBQ, + Reg: RegInfo{ + Inputs: []InputInfo{ + {0, RegMask{V1: 49135, V2: 0}}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R15 + {1, RegMask{V1: 72057594037993471, V2: 0}}, // AX CX DX BX SP BP SI DI R8 R9 R10 R11 R12 R13 g R15 SB + }, + Outputs: []OutputInfo{ + {1, RegMask{V1: 0, V2: 0}}, + {0, RegMask{V1: 49135, V2: 0}}, // AX CX DX BX BP SI DI R8 R9 R10 R11 R12 R13 R15 + }, + }, + }, { Name: "MULQU2", ArgLen: 2, diff --git a/src/cmd/compile/internal/ssacompile/flagalloc.go b/src/cmd/compile/internal/ssacompile/flagalloc.go index 2fe98018b98379..01f9b5dcdcf0e0 100644 --- a/src/cmd/compile/internal/ssacompile/flagalloc.go +++ b/src/cmd/compile/internal/ssacompile/flagalloc.go @@ -77,7 +77,24 @@ func flagalloc(f *ssa.Func) { } // Compute which flags values will need to be spilled. - spill := map[ssa.ID]bool{} + // copyFlags restores a spilled flag value by copying its generator + // and, recursively, the generators of that generator's flag + // inputs, such as the earlier links of an ADCQ carry chain. + // copied records the spilled values and every value that may be + // copied along with them. + copied := map[ssa.ID]bool{} + var markCopied func(v *ssa.Value) + markCopied = func(v *ssa.Value) { + if copied[v.ID] { + return + } + copied[v.ID] = true + for _, a := range v.Args { + if a.Type.IsFlags() || a.Type.IsTuple() { + markCopied(a) + } + } + } for _, b := range f.Blocks { var flag *ssa.Value if len(b.Preds) > 0 { @@ -92,7 +109,7 @@ func flagalloc(f *ssa.Func) { continue } // a will need to be restored here. - spill[a.ID] = true + markCopied(a) flag = a } if v.ClobbersFlags() { @@ -104,17 +121,50 @@ func flagalloc(f *ssa.Func) { } for _, v := range b.ControlValues() { if v != flag && v.Type.IsFlags() { - spill[v.ID] = true + markCopied(v) } } if v := end[b.ID]; v != nil && v != flag { - spill[v.ID] = true + markCopied(v) } } - // Add flag spill and recomputation where they are needed. + // A copy executes later than the original, when the memory that + // the original read may have been overwritten. So split every + // generator that may be copied and that reads memory into a load + // and a flag generator, leaving the load in the original's place. + // This must happen before any copying, because the copies of a + // generator may be made while processing a block that comes + // before the generator's own block. var remove []*ssa.Value // values that should be checked for possible removal var oldSched []*ssa.Value + for _, b := range f.Blocks { + split := false + for _, v := range b.Values { + if copied[v.ID] && v.MemoryArg() != nil { + split = true + break + } + } + if !split { + continue + } + oldSched = append(oldSched[:0], b.Values...) + b.Values = b.Values[:0] + for _, v := range oldSched { + if copied[v.ID] && v.MemoryArg() != nil { + remove = append(remove, v) + // SplitLoad appends the new load to b.Values, + // so it lands just before v. + if !f.Config.SplitLoad(v) { + f.Fatalf("can't split flag generator: %s", v.LongString()) + } + } + b.Values = append(b.Values, v) + } + } + + // Add flag spill and recomputation where they are needed. for _, b := range f.Blocks { oldSched = append(oldSched[:0], b.Values...) b.Values = b.Values[:0] @@ -135,15 +185,6 @@ func flagalloc(f *ssa.Func) { f.Fatalf("phi of flags not supported: %s", v.LongString()) } - // If v will be spilled, and v uses memory, then we must split it - // into a load + a flag generator. - if spill[v.ID] && v.MemoryArg() != nil { - remove = append(remove, v) - if !f.Config.SplitLoad(v) { - f.Fatalf("can't split flag generator: %s", v.LongString()) - } - } - // Make sure any flag arg of v is in the flags register. // If not, recompute it. for i, a := range v.Args { diff --git a/test/codegen/mathbits.go b/test/codegen/mathbits.go index 034019283d6580..e2c8934030baaf 100644 --- a/test/codegen/mathbits.go +++ b/test/codegen/mathbits.go @@ -981,6 +981,41 @@ func Sub64MSaveC(p, q, r, c *[2]uint64) { r[1], c[1] = bits.Sub64(p[1], q[1], c[0]) } +// The second operand of a carry chain link can be read +// straight from memory; see issue 80400. + +func Add64LoadOperand(a, b [2]uint64, s uint64) uint64 { + // amd64:`ADDQ [^,]*\(SP\), [A-Z]+` + _, c := bits.Add64(a[0], b[0], 0) + // amd64:`ADCQ [^,]*\(SP\), [A-Z]+` + _, c = bits.Add64(a[1], b[1], c) + return s + c +} + +func Add128LoadOperand(x, y *[2]uint64) (lo, hi uint64) { + var c uint64 + lo, c = bits.Add64(x[0], y[0], 0) + // amd64:`ADCQ 8\([A-Z]+\), [A-Z]+` + hi, _ = bits.Add64(x[1], y[1], c) + return +} + +func Sub64LoadOperand(a, b [2]uint64, s uint64) uint64 { + // amd64:`SUBQ [^,]*\(SP\), [A-Z]+` + _, c := bits.Sub64(a[0], b[0], 0) + // amd64:`SBBQ [^,]*\(SP\), [A-Z]+` + _, c = bits.Sub64(a[1], b[1], c) + return s + c +} + +func Sub128LoadOperand(x, y *[2]uint64) (lo, hi uint64) { + var b uint64 + lo, b = bits.Sub64(x[0], y[0], 0) + // amd64:`SBBQ 8\([A-Z]+\), [A-Z]+` + hi, _ = bits.Sub64(x[1], y[1], b) + return +} + func Sub64PanicOnOverflowEQ(a, b uint64) uint64 { r, b := bits.Sub64(a, b, 0) // s390x:"BRC [$]12," -"ADDE" -"SUBE" diff --git a/test/fixedbugs/issue80400.go b/test/fixedbugs/issue80400.go new file mode 100644 index 00000000000000..88616fed251a16 --- /dev/null +++ b/test/fixedbugs/issue80400.go @@ -0,0 +1,78 @@ +// run + +// Copyright 2026 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// When the carry of a link that reads its operand from memory has to +// survive a call, flagalloc recomputes it after the call. The +// recomputation must not read memory again, because a store in +// between may have changed it. + +package main + +import "math/bits" + +//go:noinline +func sink() {} + +//go:noinline +func add(x, y *[2]uint64, p *uint64, v uint64) uint64 { + _, c := bits.Add64(x[0], y[0], 0) + *p = v + if v > 100 { + sink() + } + hi, _ := bits.Add64(x[1], y[1], c) + return hi +} + +//go:noinline +func sub(x, y *[2]uint64, p *uint64, v uint64) uint64 { + _, b := bits.Sub64(x[0], y[0], 0) + *p = v + if v > 100 { + sink() + } + hi, _ := bits.Sub64(x[1], y[1], b) + return hi +} + +//go:noinline +func add3(x, y *[3]uint64, p *uint64, v uint64) uint64 { + _, c := bits.Add64(x[0], y[0], 0) + _, c = bits.Add64(x[1], y[1], c) + *p = v + if v > 100 { + sink() + } + hi, _ := bits.Add64(x[2], y[2], c) + return hi +} + +func main() { + // The store through p overwrites y[0] after the first link has + // read it. The carry must come from the old value. + x := [2]uint64{1 << 63, 0} + y := [2]uint64{1 << 63, 0} + if got := add(&x, &y, &y[0], 1000); got != 1 { + panic(got) + } + + x = [2]uint64{0, 5} + y = [2]uint64{1, 0} + if got := sub(&x, &y, &y[0], 1000); got != 4 { + panic(got) + } + + x3 := [3]uint64{^uint64(0), ^uint64(0), 0} + y3 := [3]uint64{1, 0, 0} + if got := add3(&x3, &y3, &y3[1], 1000); got != 1 { + panic(got) + } + x3 = [3]uint64{^uint64(0), ^uint64(0), 0} + y3 = [3]uint64{1, 0, 0} + if got := add3(&x3, &y3, &y3[0], 1000); got != 1 { + panic(got) + } +}