authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-05-04 23:00:41+02:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-05-05 21:43:36+02:00
logd112cd52f36cbb00e18009417044ab1e4496dd80
treec9e34928674e6434c5a75171f6cb97d375293add
parentf4421c01e8a8793a7c02fce870bacbb710bb1e12

aarch64: fix mul_with_overflow for ints <= 32bits

4 files changed, 224 insertions(+), 37 deletions(-)

src/arch/aarch64/CodeGen.zig+46-32
...@@ -1294,29 +1294,23 @@ fn binOpRegister(...@@ -1294,29 +1294,23 @@ fn binOpRegister(
1294 };1294 };
1295 defer self.register_manager.unfreezeRegs(&.{rhs_reg});1295 defer self.register_manager.unfreezeRegs(&.{rhs_reg});
12961296
1297 const dest_reg: Register = reg: {1297 const dest_reg = switch (mir_tag) {
1298 const dest_reg = switch (mir_tag) {1298 .cmp_shifted_register => undefined, // cmp has no destination register
1299 .cmp_shifted_register => undefined, // cmp has no destination register1299 else => if (maybe_inst) |inst| blk: {
1300 else => if (maybe_inst) |inst| blk: {1300 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1301 const bin_op = self.air.instructions.items(.data)[inst].bin_op;1301
13021302 if (lhs_is_register and self.reuseOperand(inst, bin_op.lhs, 0, lhs)) {
1303 if (lhs_is_register and self.reuseOperand(inst, bin_op.lhs, 0, lhs)) {1303 break :blk lhs_reg;
1304 break :blk lhs_reg;1304 } else if (rhs_is_register and self.reuseOperand(inst, bin_op.rhs, 1, rhs)) {
1305 } else if (rhs_is_register and self.reuseOperand(inst, bin_op.rhs, 1, rhs)) {1305 break :blk rhs_reg;
1306 break :blk rhs_reg;1306 } else {
1307 } else {1307 const raw_reg = try self.register_manager.allocReg(inst);
1308 const raw_reg = try self.register_manager.allocReg(inst);
1309 break :blk registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
1310 }
1311 } else blk: {
1312 const raw_reg = try self.register_manager.allocReg(null);
1313 break :blk registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));1308 break :blk registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
1314 },1309 }
1315 };1310 } else blk: {
1316 break :reg switch (mir_tag) {1311 const raw_reg = try self.register_manager.allocReg(null);
1317 .smull, .umull => dest_reg.to64(),1312 break :blk registerAlias(raw_reg, lhs_ty.abiSize(self.target.*));
1318 else => dest_reg,1313 },
1319 };
1320 };1314 };
13211315
1322 if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs);1316 if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs);
...@@ -1341,9 +1335,7 @@ fn binOpRegister(...@@ -1341,9 +1335,7 @@ fn binOpRegister(
1341 .shift = .lsl,1335 .shift = .lsl,
1342 } },1336 } },
1343 .mul,1337 .mul,
1344 .smulh,
1345 .smull,1338 .smull,
1346 .umulh,
1347 .umull,1339 .umull,
1348 .lsl_register,1340 .lsl_register,
1349 .asr_register,1341 .asr_register,
...@@ -1932,16 +1924,38 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -1932,16 +1924,38 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1932 self.register_manager.freezeRegs(&.{truncated_reg});1924 self.register_manager.freezeRegs(&.{truncated_reg});
1933 defer self.register_manager.unfreezeRegs(&.{truncated_reg});1925 defer self.register_manager.unfreezeRegs(&.{truncated_reg});
19341926
1935 try self.truncRegister(dest_reg, truncated_reg, int_info.signedness, int_info.bits);1927 try self.truncRegister(
1936 _ = try self.binOp(1928 dest_reg.to32(),
1937 .cmp_eq,1929 truncated_reg.to32(),
1938 null,1930 int_info.signedness,
1939 dest,1931 int_info.bits,
1940 .{ .register = truncated_reg },
1941 Type.usize,
1942 Type.usize,
1943 );1932 );
19441933
1934 switch (int_info.signedness) {
1935 .signed => {
1936 _ = try self.addInst(.{
1937 .tag = .cmp_extended_register,
1938 .data = .{ .rr_extend_shift = .{
1939 .rn = dest_reg.to64(),
1940 .rm = truncated_reg.to32(),
1941 .ext_type = .sxtw,
1942 .imm3 = 0,
1943 } },
1944 });
1945 },
1946 .unsigned => {
1947 _ = try self.addInst(.{
1948 .tag = .cmp_extended_register,
1949 .data = .{ .rr_extend_shift = .{
1950 .rn = dest_reg.to64(),
1951 .rm = truncated_reg.to32(),
1952 .ext_type = .uxtw,
1953 .imm3 = 0,
1954 } },
1955 });
1956 },
1957 }
1958
1945 try self.genSetStack(lhs_ty, stack_offset, .{ .register = truncated_reg });1959 try self.genSetStack(lhs_ty, stack_offset, .{ .register = truncated_reg });
1946 try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{1960 try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{
1947 .compare_flags_unsigned = .neq,1961 .compare_flags_unsigned = .neq,
src/arch/aarch64/Emit.zig+51-4
...@@ -114,6 +114,12 @@ pub fn emitMir(...@@ -114,6 +114,12 @@ pub fn emitMir(
114 .sub_shifted_register => try emit.mirAddSubtractShiftedRegister(inst),114 .sub_shifted_register => try emit.mirAddSubtractShiftedRegister(inst),
115 .subs_shifted_register => try emit.mirAddSubtractShiftedRegister(inst),115 .subs_shifted_register => try emit.mirAddSubtractShiftedRegister(inst),
116116
117 .add_extended_register => try emit.mirAddSubtractExtendedRegister(inst),
118 .adds_extended_register => try emit.mirAddSubtractExtendedRegister(inst),
119 .sub_extended_register => try emit.mirAddSubtractExtendedRegister(inst),
120 .subs_extended_register => try emit.mirAddSubtractExtendedRegister(inst),
121 .cmp_extended_register => try emit.mirAddSubtractExtendedRegister(inst),
122
117 .cset => try emit.mirConditionalSelect(inst),123 .cset => try emit.mirConditionalSelect(inst),
118124
119 .dbg_line => try emit.mirDbgLine(inst),125 .dbg_line => try emit.mirDbgLine(inst),
...@@ -732,6 +738,47 @@ fn mirAddSubtractShiftedRegister(emit: *Emit, inst: Mir.Inst.Index) !void {...@@ -732,6 +738,47 @@ fn mirAddSubtractShiftedRegister(emit: *Emit, inst: Mir.Inst.Index) !void {
732 }738 }
733}739}
734740
741fn mirAddSubtractExtendedRegister(emit: *Emit, inst: Mir.Inst.Index) !void {
742 const tag = emit.mir.instructions.items(.tag)[inst];
743 switch (tag) {
744 .add_extended_register,
745 .adds_extended_register,
746 .sub_extended_register,
747 .subs_extended_register,
748 => {
749 const rrr_extend_shift = emit.mir.instructions.items(.data)[inst].rrr_extend_shift;
750 const rd = rrr_extend_shift.rd;
751 const rn = rrr_extend_shift.rn;
752 const rm = rrr_extend_shift.rm;
753 const ext_type = rrr_extend_shift.ext_type;
754 const imm3 = rrr_extend_shift.imm3;
755
756 switch (tag) {
757 .add_extended_register => try emit.writeInstruction(Instruction.addExtendedRegister(rd, rn, rm, ext_type, imm3)),
758 .adds_extended_register => try emit.writeInstruction(Instruction.addsExtendedRegister(rd, rn, rm, ext_type, imm3)),
759 .sub_extended_register => try emit.writeInstruction(Instruction.subExtendedRegister(rd, rn, rm, ext_type, imm3)),
760 .subs_extended_register => try emit.writeInstruction(Instruction.subsExtendedRegister(rd, rn, rm, ext_type, imm3)),
761 else => unreachable,
762 }
763 },
764 .cmp_extended_register => {
765 const rr_extend_shift = emit.mir.instructions.items(.data)[inst].rr_extend_shift;
766 const rn = rr_extend_shift.rn;
767 const rm = rr_extend_shift.rm;
768 const ext_type = rr_extend_shift.ext_type;
769 const imm3 = rr_extend_shift.imm3;
770 const zr: Register = switch (rn.size()) {
771 32 => .wzr,
772 64 => .xzr,
773 else => unreachable,
774 };
775
776 try emit.writeInstruction(Instruction.subsExtendedRegister(zr, rn, rm, ext_type, imm3));
777 },
778 else => unreachable,
779 }
780}
781
735fn mirConditionalSelect(emit: *Emit, inst: Mir.Inst.Index) !void {782fn mirConditionalSelect(emit: *Emit, inst: Mir.Inst.Index) !void {
736 const tag = emit.mir.instructions.items(.tag)[inst];783 const tag = emit.mir.instructions.items(.tag)[inst];
737 switch (tag) {784 switch (tag) {
...@@ -1013,10 +1060,10 @@ fn mirDataProcessing3Source(emit: *Emit, inst: Mir.Inst.Index) !void {...@@ -1013,10 +1060,10 @@ fn mirDataProcessing3Source(emit: *Emit, inst: Mir.Inst.Index) !void {
10131060
1014 switch (tag) {1061 switch (tag) {
1015 .mul => try emit.writeInstruction(Instruction.mul(rrr.rd, rrr.rn, rrr.rm)),1062 .mul => try emit.writeInstruction(Instruction.mul(rrr.rd, rrr.rn, rrr.rm)),
1016 .smulh => try emit.writeInstruction(Instruction.smulh(rrr.rd, rrr.rn, rrr.rm)),1063 .smulh => try emit.writeInstruction(Instruction.smulh(rrr.rd.to64(), rrr.rn.to64(), rrr.rm.to64())),
1017 .smull => try emit.writeInstruction(Instruction.smull(rrr.rd, rrr.rn, rrr.rm)),1064 .smull => try emit.writeInstruction(Instruction.smull(rrr.rd.to64(), rrr.rn.to32(), rrr.rm.to32())),
1018 .umulh => try emit.writeInstruction(Instruction.umulh(rrr.rd, rrr.rn, rrr.rm)),1065 .umulh => try emit.writeInstruction(Instruction.umulh(rrr.rd.to64(), rrr.rn.to64(), rrr.rm.to64())),
1019 .umull => try emit.writeInstruction(Instruction.umull(rrr.rd, rrr.rn, rrr.rm)),1066 .umull => try emit.writeInstruction(Instruction.umull(rrr.rd.to64(), rrr.rn.to32(), rrr.rm.to32())),
1020 else => unreachable,1067 else => unreachable,
1021 }1068 }
1022}1069}
src/arch/aarch64/Mir.zig+29
...@@ -32,6 +32,10 @@ pub const Inst = struct {...@@ -32,6 +32,10 @@ pub const Inst = struct {
32 add_shifted_register,32 add_shifted_register,
33 /// Add, update condition flags (shifted register)33 /// Add, update condition flags (shifted register)
34 adds_shifted_register,34 adds_shifted_register,
35 /// Add (extended register)
36 add_extended_register,
37 /// Add, update condition flags (extended register)
38 adds_extended_register,
35 /// Bitwise AND (shifted register)39 /// Bitwise AND (shifted register)
36 and_shifted_register,40 and_shifted_register,
37 /// Arithmetic Shift Right (immediate)41 /// Arithmetic Shift Right (immediate)
...@@ -56,6 +60,8 @@ pub const Inst = struct {...@@ -56,6 +60,8 @@ pub const Inst = struct {
56 cmp_immediate,60 cmp_immediate,
57 /// Compare (shifted register)61 /// Compare (shifted register)
58 cmp_shifted_register,62 cmp_shifted_register,
63 /// Compare (extended register)
64 cmp_extended_register,
59 /// Conditional set65 /// Conditional set
60 cset,66 cset,
61 /// Pseudo-instruction: End of prologue67 /// Pseudo-instruction: End of prologue
...@@ -184,6 +190,10 @@ pub const Inst = struct {...@@ -184,6 +190,10 @@ pub const Inst = struct {
184 sub_shifted_register,190 sub_shifted_register,
185 /// Subtract, update condition flags (shifted register)191 /// Subtract, update condition flags (shifted register)
186 subs_shifted_register,192 subs_shifted_register,
193 /// Subtract (extended register)
194 sub_extended_register,
195 /// Subtract, update condition flags (extended register)
196 subs_extended_register,
187 /// Supervisor Call197 /// Supervisor Call
188 svc,198 svc,
189 /// Test bits (immediate)199 /// Test bits (immediate)
...@@ -300,6 +310,15 @@ pub const Inst = struct {...@@ -300,6 +310,15 @@ pub const Inst = struct {
300 imm6: u6,310 imm6: u6,
301 shift: bits.Instruction.AddSubtractShiftedRegisterShift,311 shift: bits.Instruction.AddSubtractShiftedRegisterShift,
302 },312 },
313 /// Two registers with sign-extension (extension type and 3-bit shift amount)
314 ///
315 /// Used by e.g. cmp_extended_register
316 rr_extend_shift: struct {
317 rn: Register,
318 rm: Register,
319 ext_type: bits.Instruction.AddSubtractExtendedRegisterOption,
320 imm3: u3,
321 },
303 /// Two registers and a shift (logical instruction version)322 /// Two registers and a shift (logical instruction version)
304 /// (shift type and 6-bit amount)323 /// (shift type and 6-bit amount)
305 ///324 ///
...@@ -356,6 +375,16 @@ pub const Inst = struct {...@@ -356,6 +375,16 @@ pub const Inst = struct {
356 imm6: u6,375 imm6: u6,
357 shift: bits.Instruction.AddSubtractShiftedRegisterShift,376 shift: bits.Instruction.AddSubtractShiftedRegisterShift,
358 },377 },
378 /// Three registers with sign-extension (extension type and 3-bit shift amount)
379 ///
380 /// Used by e.g. add_extended_register
381 rrr_extend_shift: struct {
382 rd: Register,
383 rn: Register,
384 rm: Register,
385 ext_type: bits.Instruction.AddSubtractExtendedRegisterOption,
386 imm3: u3,
387 },
359 /// Three registers and a shift (logical instruction version)388 /// Three registers and a shift (logical instruction version)
360 /// (shift type and 6-bit amount)389 /// (shift type and 6-bit amount)
361 ///390 ///
src/arch/aarch64/bits.zig+98-1
...@@ -330,6 +330,17 @@ pub const Instruction = union(enum) {...@@ -330,6 +330,17 @@ pub const Instruction = union(enum) {
330 op: u1,330 op: u1,
331 sf: u1,331 sf: u1,
332 },332 },
333 add_subtract_extended_register: packed struct {
334 rd: u5,
335 rn: u5,
336 imm3: u3,
337 option: u3,
338 rm: u5,
339 fixed: u8 = 0b01011_00_1,
340 s: u1,
341 op: u1,
342 sf: u1,
343 },
333 conditional_branch: struct {344 conditional_branch: struct {
334 cond: u4,345 cond: u4,
335 o0: u1,346 o0: u1,
...@@ -495,6 +506,7 @@ pub const Instruction = union(enum) {...@@ -495,6 +506,7 @@ pub const Instruction = union(enum) {
495 .logical_immediate => |v| @bitCast(u32, v),506 .logical_immediate => |v| @bitCast(u32, v),
496 .bitfield => |v| @bitCast(u32, v),507 .bitfield => |v| @bitCast(u32, v),
497 .add_subtract_shifted_register => |v| @bitCast(u32, v),508 .add_subtract_shifted_register => |v| @bitCast(u32, v),
509 .add_subtract_extended_register => |v| @bitCast(u32, v),
498 // TODO once packed structs work, this can be refactored510 // TODO once packed structs work, this can be refactored
499 .conditional_branch => |v| @as(u32, v.cond) | (@as(u32, v.o0) << 4) | (@as(u32, v.imm19) << 5) | (@as(u32, v.o1) << 24) | (@as(u32, v.fixed) << 25),511 .conditional_branch => |v| @as(u32, v.cond) | (@as(u32, v.o0) << 4) | (@as(u32, v.imm19) << 5) | (@as(u32, v.o1) << 24) | (@as(u32, v.fixed) << 25),
500 .compare_and_branch => |v| @as(u32, v.rt) | (@as(u32, v.imm19) << 5) | (@as(u32, v.op) << 24) | (@as(u32, v.fixed) << 25) | (@as(u32, v.sf) << 31),512 .compare_and_branch => |v| @as(u32, v.rt) | (@as(u32, v.imm19) << 5) | (@as(u32, v.op) << 24) | (@as(u32, v.fixed) << 25) | (@as(u32, v.sf) << 31),
...@@ -1006,6 +1018,44 @@ pub const Instruction = union(enum) {...@@ -1006,6 +1018,44 @@ pub const Instruction = union(enum) {
1006 };1018 };
1007 }1019 }
10081020
1021 pub const AddSubtractExtendedRegisterOption = enum(u3) {
1022 uxtb,
1023 uxth,
1024 uxtw,
1025 uxtx, // serves also as lsl
1026 sxtb,
1027 sxth,
1028 sxtw,
1029 sxtx,
1030 };
1031
1032 fn addSubtractExtendedRegister(
1033 op: u1,
1034 s: u1,
1035 rd: Register,
1036 rn: Register,
1037 rm: Register,
1038 extend: AddSubtractExtendedRegisterOption,
1039 imm3: u3,
1040 ) Instruction {
1041 return Instruction{
1042 .add_subtract_extended_register = .{
1043 .rd = rd.enc(),
1044 .rn = rn.enc(),
1045 .imm3 = imm3,
1046 .option = @enumToInt(extend),
1047 .rm = rm.enc(),
1048 .s = s,
1049 .op = op,
1050 .sf = switch (rd.size()) {
1051 32 => 0b0,
1052 64 => 0b1,
1053 else => unreachable, // unexpected register size
1054 },
1055 },
1056 };
1057 }
1058
1009 fn conditionalBranch(1059 fn conditionalBranch(
1010 o0: u1,1060 o0: u1,
1011 o1: u1,1061 o1: u1,
...@@ -1524,6 +1574,48 @@ pub const Instruction = union(enum) {...@@ -1524,6 +1574,48 @@ pub const Instruction = union(enum) {
1524 return addSubtractShiftedRegister(0b1, 0b1, shift, rd, rn, rm, imm6);1574 return addSubtractShiftedRegister(0b1, 0b1, shift, rd, rn, rm, imm6);
1525 }1575 }
15261576
1577 // Add/subtract (extended register)
1578
1579 pub fn addExtendedRegister(
1580 rd: Register,
1581 rn: Register,
1582 rm: Register,
1583 extend: AddSubtractExtendedRegisterOption,
1584 imm3: u3,
1585 ) Instruction {
1586 return addSubtractExtendedRegister(0b0, 0b0, rd, rn, rm, extend, imm3);
1587 }
1588
1589 pub fn addsExtendedRegister(
1590 rd: Register,
1591 rn: Register,
1592 rm: Register,
1593 extend: AddSubtractExtendedRegisterOption,
1594 imm3: u3,
1595 ) Instruction {
1596 return addSubtractExtendedRegister(0b0, 0b1, rd, rn, rm, extend, imm3);
1597 }
1598
1599 pub fn subExtendedRegister(
1600 rd: Register,
1601 rn: Register,
1602 rm: Register,
1603 extend: AddSubtractExtendedRegisterOption,
1604 imm3: u3,
1605 ) Instruction {
1606 return addSubtractExtendedRegister(0b1, 0b0, rd, rn, rm, extend, imm3);
1607 }
1608
1609 pub fn subsExtendedRegister(
1610 rd: Register,
1611 rn: Register,
1612 rm: Register,
1613 extend: AddSubtractExtendedRegisterOption,
1614 imm3: u3,
1615 ) Instruction {
1616 return addSubtractExtendedRegister(0b1, 0b1, rd, rn, rm, extend, imm3);
1617 }
1618
1527 // Conditional branch1619 // Conditional branch
15281620
1529 pub fn bCond(cond: Condition, offset: i21) Instruction {1621 pub fn bCond(cond: Condition, offset: i21) Instruction {
...@@ -1565,11 +1657,12 @@ pub const Instruction = union(enum) {...@@ -1565,11 +1657,12 @@ pub const Instruction = union(enum) {
1565 }1657 }
15661658
1567 pub fn smaddl(rd: Register, rn: Register, rm: Register, ra: Register) Instruction {1659 pub fn smaddl(rd: Register, rn: Register, rm: Register, ra: Register) Instruction {
1660 assert(rd.size() == 64 and rn.size() == 32 and rm.size() == 32 and ra.size() == 64);
1568 return dataProcessing3Source(0b00, 0b001, 0b0, rd, rn, rm, ra);1661 return dataProcessing3Source(0b00, 0b001, 0b0, rd, rn, rm, ra);
1569 }1662 }
15701663
1571 pub fn umaddl(rd: Register, rn: Register, rm: Register, ra: Register) Instruction {1664 pub fn umaddl(rd: Register, rn: Register, rm: Register, ra: Register) Instruction {
1572 assert(rd.size() == 64);1665 assert(rd.size() == 64 and rn.size() == 32 and rm.size() == 32 and ra.size() == 64);
1573 return dataProcessing3Source(0b00, 0b101, 0b0, rd, rn, rm, ra);1666 return dataProcessing3Source(0b00, 0b101, 0b0, rd, rn, rm, ra);
1574 }1667 }
15751668
...@@ -1837,6 +1930,10 @@ test "serialize instructions" {...@@ -1837,6 +1930,10 @@ test "serialize instructions" {
1837 .inst = Instruction.smulh(.x0, .x1, .x2),1930 .inst = Instruction.smulh(.x0, .x1, .x2),
1838 .expected = 0b1_00_11011_0_10_00010_0_11111_00001_00000,1931 .expected = 0b1_00_11011_0_10_00010_0_11111_00001_00000,
1839 },1932 },
1933 .{ // adds x0, x1, x2, sxtx
1934 .inst = Instruction.addsExtendedRegister(.x0, .x1, .x2, .sxtx, 0),
1935 .expected = 0b1_0_1_01011_00_1_00010_111_000_00001_00000,
1936 },
1840 };1937 };
18411938
1842 for (testcases) |case| {1939 for (testcases) |case| {