authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-03-30 11:22:35+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-03-30 11:22:35+02:00
logb153e156b1d431995e7f89150c93ec05aa3966f8
tree9311b768bbf2547767df8a3dc8c58ececdac190f
parentc21f046a8b019c42aa0dbb0fb9c592b590edf977
parent376d0878ec2e366633e2dbc7c91ab7ae2a6ae5b7
signature Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #11342 from ziglang/stage2-x64-mul-div

x64: use all available registers (caller and callee), and fix how we multiply (fixes @mulWithOverflow for byte-size operands)

5 files changed, 198 insertions(+), 145 deletions(-)

src/arch/x86_64/CodeGen.zig+181-135
......@@ -31,6 +31,8 @@ const bits = @import("bits.zig");
3131const abi = @import("abi.zig");
3232const Register = bits.Register;
3333const callee_preserved_regs = abi.callee_preserved_regs;
34const caller_preserved_regs = abi.caller_preserved_regs;
35const allocatable_registers = abi.allocatable_registers;
3436const c_abi_int_param_regs = abi.c_abi_int_param_regs;
3537const c_abi_int_return_regs = abi.c_abi_int_return_regs;
3638
......@@ -40,7 +42,7 @@ const InnerError = error{
4042 OutOfRegisters,
4143};
4244
43const RegisterManager = RegisterManagerFn(Self, Register, &callee_preserved_regs);
45const RegisterManager = RegisterManagerFn(Self, Register, &allocatable_registers);
4446
4547gpa: Allocator,
4648air: Air,
......@@ -56,7 +58,6 @@ arg_index: u32,
5658src_loc: Module.SrcLoc,
5759stack_align: u32,
5860
59ret_backpatches: std.ArrayListUnmanaged(Mir.Inst.Index) = .{},
6061compare_flags_inst: ?Air.Inst.Index = null,
6162
6263/// MIR Instructions
......@@ -351,7 +352,6 @@ pub fn generate(
351352 std.AutoHashMap(Mir.Inst.Index, Air.Inst.Index).init(bin_file.allocator)
352353 else {},
353354 };
354 defer function.ret_backpatches.deinit(bin_file.allocator);
355355 defer function.stack.deinit(bin_file.allocator);
356356 defer function.blocks.deinit(bin_file.allocator);
357357 defer function.exitlude_jump_relocs.deinit(bin_file.allocator);
......@@ -480,6 +480,21 @@ fn gen(self: *Self) InnerError!void {
480480 .data = undefined,
481481 });
482482
483 if (self.ret_mcv == .stack_offset) {
484 // The address where to store the return value for the caller is in `.rdi`
485 // register which the callee is free to clobber. Therefore, we purposely
486 // spill it to stack immediately.
487 const ptr_ty = Type.usize;
488 const abi_size = @intCast(u32, ptr_ty.abiSize(self.target.*));
489 const abi_align = ptr_ty.abiAlignment(self.target.*);
490 const stack_offset = mem.alignForwardGeneric(u32, self.next_stack_offset + abi_size, abi_align);
491 self.next_stack_offset = stack_offset;
492 self.max_end_stack = @maximum(self.max_end_stack, self.next_stack_offset);
493 try self.genSetStack(ptr_ty, @intCast(i32, stack_offset), MCValue{ .register = .rdi }, .{});
494 self.ret_mcv = MCValue{ .stack_offset = @intCast(i32, stack_offset) };
495 log.debug("gen: spilling .rdi to stack at offset {}", .{stack_offset});
496 }
497
483498 _ = try self.addInst(.{
484499 .tag = .dbg_prologue_end,
485500 .ops = undefined,
......@@ -517,23 +532,6 @@ fn gen(self: *Self) InnerError!void {
517532 var disp = data.disp + 8;
518533 inline for (callee_preserved_regs) |reg, i| {
519534 if (self.register_manager.isRegAllocated(reg)) {
520 if (reg.to64() == .rdi) {
521 for (self.ret_backpatches.items) |inst| {
522 log.debug(".rdi was spilled, backpatching with mov from stack at offset {}", .{
523 -@intCast(i32, disp),
524 });
525 const ops = Mir.Ops.decode(self.mir_instructions.items(.ops)[inst]);
526 self.mir_instructions.set(inst, Mir.Inst{
527 .tag = .mov,
528 .ops = (Mir.Ops{
529 .reg1 = ops.reg1,
530 .reg2 = .rbp,
531 .flags = 0b01,
532 }).encode(),
533 .data = .{ .imm = @bitCast(u32, -@intCast(i32, disp)) },
534 });
535 }
536 }
537535 data.regs |= 1 << @intCast(u5, i);
538536 self.max_end_stack += 8;
539537 disp += 8;
......@@ -912,8 +910,7 @@ fn allocMem(self: *Self, inst: Air.Inst.Index, abi_size: u32, abi_align: u32) !u
912910 // TODO find a free slot instead of always appending
913911 const offset = mem.alignForwardGeneric(u32, self.next_stack_offset + abi_size, abi_align);
914912 self.next_stack_offset = offset;
915 if (self.next_stack_offset > self.max_end_stack)
916 self.max_end_stack = self.next_stack_offset;
913 self.max_end_stack = @maximum(self.max_end_stack, self.next_stack_offset);
917914 try self.stack.putNoClobber(self.gpa, offset, .{
918915 .inst = inst,
919916 .size = abi_size,
......@@ -1256,7 +1253,7 @@ fn genPtrBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_r
12561253 offset_mcv.freezeIfRegister(&self.register_manager);
12571254 defer offset_mcv.unfreezeIfRegister(&self.register_manager);
12581255
1259 try self.genIMulOpMir(offset_ty, offset_mcv, .{ .immediate = elem_size });
1256 try self.genIntMulComplexOpMir(offset_ty, offset_mcv, .{ .immediate = elem_size });
12601257
12611258 const tag = self.air.instructions.items(.tag)[inst];
12621259 switch (tag) {
......@@ -1398,10 +1395,29 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
13981395
13991396fn airMul(self: *Self, inst: Air.Inst.Index) !void {
14001397 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1401 const result: MCValue = if (self.liveness.isUnused(inst))
1402 .dead
1403 else
1404 try self.genBinMathOp(inst, bin_op.lhs, bin_op.rhs);
1398 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1399 const ty = self.air.typeOfIndex(inst);
1400
1401 if (ty.zigTypeTag() != .Int) {
1402 return self.fail("TODO implement 'mul' for operands of dst type {}", .{ty.zigTypeTag()});
1403 }
1404
1405 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
1406 try self.register_manager.getReg(.rax, inst);
1407 try self.register_manager.getReg(.rdx, null);
1408 self.register_manager.freezeRegs(&.{ .rax, .rdx });
1409 defer self.register_manager.unfreezeRegs(&.{ .rax, .rdx });
1410
1411 const lhs = try self.resolveInst(bin_op.lhs);
1412 const rhs = try self.resolveInst(bin_op.rhs);
1413
1414 const signedness = ty.intInfo(self.target.*).signedness;
1415 try self.genIntMulDivOpMir(switch (signedness) {
1416 .signed => .imul,
1417 .unsigned => .mul,
1418 }, ty, signedness, lhs, rhs);
1419 break :result MCValue{ .register = .rax };
1420 };
14051421 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
14061422}
14071423
......@@ -1476,23 +1492,33 @@ fn airSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
14761492fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
14771493 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
14781494 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
1495 const result = if (self.liveness.isUnused(inst)) .dead else result: {
1496 const ty = self.air.typeOf(bin_op.lhs);
1497 const signedness: std.builtin.Signedness = blk: {
1498 if (ty.zigTypeTag() != .Int) {
1499 return self.fail("TODO implement airMulWithOverflow for type {}", .{ty.fmtDebug()});
1500 }
1501 break :blk ty.intInfo(self.target.*).signedness;
1502 };
14791503
1480 if (self.liveness.isUnused(inst)) {
1481 return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
1482 }
1504 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
1505 try self.register_manager.getReg(.rax, inst);
1506 try self.register_manager.getReg(.rdx, null);
1507 self.register_manager.freezeRegs(&.{ .rax, .rdx });
1508 defer self.register_manager.unfreezeRegs(&.{ .rax, .rdx });
14831509
1484 const ty = self.air.typeOf(bin_op.lhs);
1485 const signedness: std.builtin.Signedness = blk: {
1486 if (ty.zigTypeTag() != .Int) {
1487 return self.fail("TODO implement airMulWithOverflow for type {}", .{ty.fmtDebug()});
1488 }
1489 break :blk ty.intInfo(self.target.*).signedness;
1490 };
1510 const lhs = try self.resolveInst(bin_op.lhs);
1511 const rhs = try self.resolveInst(bin_op.rhs);
14911512
1492 const partial = try self.genBinMathOp(inst, bin_op.lhs, bin_op.rhs);
1493 const result: MCValue = switch (signedness) {
1494 .signed => .{ .register_overflow_signed = partial.register },
1495 .unsigned => .{ .register_overflow_unsigned = partial.register },
1513 try self.genIntMulDivOpMir(switch (signedness) {
1514 .signed => .imul,
1515 .unsigned => .mul,
1516 }, ty, signedness, lhs, rhs);
1517
1518 switch (signedness) {
1519 .signed => break :result MCValue{ .register_overflow_signed = .rax },
1520 .unsigned => break :result MCValue{ .register_overflow_unsigned = .rax },
1521 }
14961522 };
14971523
14981524 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
......@@ -1503,11 +1529,12 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
15031529 return self.fail("TODO implement airShlWithOverflow for {}", .{self.target.cpu.arch});
15041530}
15051531
1506/// Generates signed or unsigned integer division.
1507/// Requires use of .rax and .rdx registers. Spills them if necessary.
1532/// Generates signed or unsigned integer multiplication/division.
1533/// Clobbers .rax and .rdx registers.
15081534/// Quotient is saved in .rax and remainder in .rdx.
1509fn genIntDivOpMir(
1535fn genIntMulDivOpMir(
15101536 self: *Self,
1537 tag: Mir.Inst.Tag,
15111538 ty: Type,
15121539 signedness: std.builtin.Signedness,
15131540 lhs: MCValue,
......@@ -1515,25 +1542,10 @@ fn genIntDivOpMir(
15151542) !void {
15161543 const abi_size = @intCast(u32, ty.abiSize(self.target.*));
15171544 if (abi_size > 8) {
1518 return self.fail("TODO implement genIntDivOpMir for ABI size larger than 8", .{});
1545 return self.fail("TODO implement genIntMulDivOpMir for ABI size larger than 8", .{});
15191546 }
15201547
1521 try self.register_manager.getReg(.rax, null);
1522 try self.register_manager.getReg(.rdx, null);
1523 self.register_manager.freezeRegs(&.{ .rax, .rdx });
1524 defer self.register_manager.unfreezeRegs(&.{ .rax, .rdx });
1525
1526 const dividend = switch (lhs) {
1527 .register => lhs,
1528 else => blk: {
1529 const reg = try self.copyToTmpRegister(ty, lhs);
1530 break :blk MCValue{ .register = reg };
1531 },
1532 };
1533 try self.genSetReg(ty, .rax, dividend);
1534
1535 self.register_manager.freezeRegs(&.{dividend.register});
1536 defer self.register_manager.unfreezeRegs(&.{dividend.register});
1548 try self.genSetReg(ty, .rax, lhs);
15371549
15381550 switch (signedness) {
15391551 .signed => {
......@@ -1557,22 +1569,19 @@ fn genIntDivOpMir(
15571569 },
15581570 }
15591571
1560 const divisor = switch (rhs) {
1572 const factor = switch (rhs) {
15611573 .register => rhs,
1574 .stack_offset => rhs,
15621575 else => blk: {
15631576 const reg = try self.copyToTmpRegister(ty, rhs);
15641577 break :blk MCValue{ .register = reg };
15651578 },
15661579 };
1567 const op_tag: Mir.Inst.Tag = switch (signedness) {
1568 .signed => .idiv,
1569 .unsigned => .div,
1570 };
15711580
1572 switch (divisor) {
1581 switch (factor) {
15731582 .register => |reg| {
15741583 _ = try self.addInst(.{
1575 .tag = op_tag,
1584 .tag = tag,
15761585 .ops = (Mir.Ops{
15771586 .reg1 = reg,
15781587 }).encode(),
......@@ -1581,7 +1590,7 @@ fn genIntDivOpMir(
15811590 },
15821591 .stack_offset => |off| {
15831592 _ = try self.addInst(.{
1584 .tag = op_tag,
1593 .tag = tag,
15851594 .ops = (Mir.Ops{
15861595 .reg2 = .rbp,
15871596 .flags = switch (abi_size) {
......@@ -1599,6 +1608,7 @@ fn genIntDivOpMir(
15991608 }
16001609}
16011610
1611/// Clobbers .rax and .rdx registers.
16021612fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCValue {
16031613 const signedness = ty.intInfo(self.target.*).signedness;
16041614 const dividend = switch (lhs) {
......@@ -1614,7 +1624,10 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
16141624 self.register_manager.freezeRegs(&.{divisor});
16151625 defer self.register_manager.unfreezeRegs(&.{ dividend, divisor });
16161626
1617 try self.genIntDivOpMir(Type.isize, signedness, .{ .register = dividend }, .{ .register = divisor });
1627 try self.genIntMulDivOpMir(switch (signedness) {
1628 .signed => .idiv,
1629 .unsigned => .div,
1630 }, Type.isize, signedness, .{ .register = dividend }, .{ .register = divisor });
16181631
16191632 _ = try self.addInst(.{
16201633 .tag = .xor,
......@@ -1666,22 +1679,53 @@ fn airDiv(self: *Self, inst: Air.Inst.Index) !void {
16661679 return self.fail("TODO implement {}", .{tag});
16671680 }
16681681
1682 const signedness = ty.intInfo(self.target.*).signedness;
1683
16691684 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
1670 try self.register_manager.getReg(.rax, null);
1685 const track_rax: ?Air.Inst.Index = blk: {
1686 if (signedness == .unsigned) break :blk inst;
1687 switch (tag) {
1688 .div_exact, .div_trunc => break :blk inst,
1689 else => break :blk null,
1690 }
1691 };
1692 try self.register_manager.getReg(.rax, track_rax);
16711693 try self.register_manager.getReg(.rdx, null);
1694 self.register_manager.freezeRegs(&.{ .rax, .rdx });
1695 defer self.register_manager.unfreezeRegs(&.{ .rax, .rdx });
16721696
16731697 const lhs = try self.resolveInst(bin_op.lhs);
1674 const rhs = try self.resolveInst(bin_op.rhs);
1698 lhs.freezeIfRegister(&self.register_manager);
1699 defer lhs.unfreezeIfRegister(&self.register_manager);
1700
1701 const rhs = blk: {
1702 const rhs = try self.resolveInst(bin_op.rhs);
1703 if (signedness == .signed) {
1704 switch (tag) {
1705 .div_floor => {
1706 rhs.freezeIfRegister(&self.register_manager);
1707 defer rhs.unfreezeIfRegister(&self.register_manager);
1708 break :blk try self.copyToRegisterWithInstTracking(inst, ty, rhs);
1709 },
1710 else => {},
1711 }
1712 }
1713 break :blk rhs;
1714 };
1715 rhs.freezeIfRegister(&self.register_manager);
1716 defer rhs.unfreezeIfRegister(&self.register_manager);
16751717
1676 const signedness = ty.intInfo(self.target.*).signedness;
16771718 if (signedness == .unsigned) {
1678 try self.genIntDivOpMir(ty, signedness, lhs, rhs);
1719 try self.genIntMulDivOpMir(.div, ty, signedness, lhs, rhs);
16791720 break :result MCValue{ .register = .rax };
16801721 }
16811722
16821723 switch (tag) {
16831724 .div_exact, .div_trunc => {
1684 try self.genIntDivOpMir(ty, signedness, lhs, rhs);
1725 try self.genIntMulDivOpMir(switch (signedness) {
1726 .signed => .idiv,
1727 .unsigned => .div,
1728 }, ty, signedness, lhs, rhs);
16851729 break :result MCValue{ .register = .rax };
16861730 },
16871731 .div_floor => {
......@@ -1702,11 +1746,18 @@ fn airRem(self: *Self, inst: Air.Inst.Index) !void {
17021746 }
17031747 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
17041748 try self.register_manager.getReg(.rax, null);
1705 try self.register_manager.getReg(.rdx, null);
1749 try self.register_manager.getReg(.rdx, inst);
1750 self.register_manager.freezeRegs(&.{ .rax, .rdx });
1751 defer self.register_manager.unfreezeRegs(&.{ .rax, .rdx });
1752
17061753 const lhs = try self.resolveInst(bin_op.lhs);
17071754 const rhs = try self.resolveInst(bin_op.rhs);
1755
17081756 const signedness = ty.intInfo(self.target.*).signedness;
1709 try self.genIntDivOpMir(ty, signedness, lhs, rhs);
1757 try self.genIntMulDivOpMir(switch (signedness) {
1758 .signed => .idiv,
1759 .unsigned => .div,
1760 }, ty, signedness, lhs, rhs);
17101761 break :result MCValue{ .register = .rdx };
17111762 };
17121763 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
......@@ -1719,25 +1770,33 @@ fn airMod(self: *Self, inst: Air.Inst.Index) !void {
17191770 if (ty.zigTypeTag() != .Int) {
17201771 return self.fail("TODO implement .mod for operands of dst type {}", .{ty.zigTypeTag()});
17211772 }
1773 const signedness = ty.intInfo(self.target.*).signedness;
1774
17221775 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
17231776 try self.register_manager.getReg(.rax, null);
1724 try self.register_manager.getReg(.rdx, null);
1777 try self.register_manager.getReg(.rdx, if (signedness == .unsigned) inst else null);
1778 self.register_manager.freezeRegs(&.{ .rax, .rdx });
1779 defer self.register_manager.unfreezeRegs(&.{ .rax, .rdx });
1780
17251781 const lhs = try self.resolveInst(bin_op.lhs);
17261782 const rhs = try self.resolveInst(bin_op.rhs);
1727 const signedness = ty.intInfo(self.target.*).signedness;
1783
17281784 switch (signedness) {
17291785 .unsigned => {
1730 try self.genIntDivOpMir(ty, signedness, lhs, rhs);
1786 try self.genIntMulDivOpMir(switch (signedness) {
1787 .signed => .idiv,
1788 .unsigned => .div,
1789 }, ty, signedness, lhs, rhs);
17311790 break :result MCValue{ .register = .rdx };
17321791 },
17331792 .signed => {
17341793 const div_floor = try self.genInlineIntDivFloor(ty, lhs, rhs);
1735 try self.genIMulOpMir(ty, div_floor, rhs);
1794 try self.genIntMulComplexOpMir(ty, div_floor, rhs);
17361795
1737 const reg = try self.copyToTmpRegister(ty, lhs);
1738 try self.genBinMathOpMir(.sub, ty, .{ .register = reg }, div_floor);
1796 const result = try self.copyToRegisterWithInstTracking(inst, ty, lhs);
1797 try self.genBinMathOpMir(.sub, ty, result, div_floor);
17391798
1740 break :result MCValue{ .register = reg };
1799 break :result result;
17411800 },
17421801 }
17431802 };
......@@ -2134,7 +2193,7 @@ fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void {
21342193
21352194fn elemOffset(self: *Self, index_ty: Type, index: MCValue, elem_size: u64) !Register {
21362195 const reg = try self.copyToTmpRegister(index_ty, index);
2137 try self.genIMulOpMir(index_ty, .{ .register = reg }, .{ .immediate = elem_size });
2196 try self.genIntMulComplexOpMir(index_ty, .{ .register = reg }, .{ .immediate = elem_size });
21382197 return reg;
21392198}
21402199
......@@ -3098,7 +3157,6 @@ fn genBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs:
30983157 .bool_or, .bit_or => try self.genBinMathOpMir(.@"or", dst_ty, dst_mcv, src_mcv),
30993158 .bool_and, .bit_and => try self.genBinMathOpMir(.@"and", dst_ty, dst_mcv, src_mcv),
31003159 .xor, .not => try self.genBinMathOpMir(.xor, dst_ty, dst_mcv, src_mcv),
3101 .mul, .mulwrap, .mul_with_overflow => try self.genIMulOpMir(dst_ty, dst_mcv, src_mcv),
31023160 else => unreachable,
31033161 }
31043162 return dst_mcv;
......@@ -3254,8 +3312,9 @@ fn genBinMathOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MC
32543312 }
32553313}
32563314
3257// Performs integer multiplication between dst_mcv and src_mcv, storing the result in dst_mcv.
3258fn genIMulOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void {
3315/// Performs multi-operand integer multiplication between dst_mcv and src_mcv, storing the result in dst_mcv.
3316/// Does not support byte-size operands.
3317fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void {
32593318 const abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
32603319 switch (dst_mcv) {
32613320 .none => unreachable,
......@@ -3301,7 +3360,7 @@ fn genIMulOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !
33013360 } else {
33023361 // TODO verify we don't spill and assign to the same register as dst_mcv
33033362 const src_reg = try self.copyToTmpRegister(dst_ty, src_mcv);
3304 return self.genIMulOpMir(dst_ty, dst_mcv, MCValue{ .register = src_reg });
3363 return self.genIntMulComplexOpMir(dst_ty, dst_mcv, MCValue{ .register = src_reg });
33053364 }
33063365 },
33073366 .stack_offset => |off| {
......@@ -3458,15 +3517,20 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
34583517
34593518 try self.spillCompareFlagsIfOccupied();
34603519
3520 for (caller_preserved_regs) |reg| {
3521 try self.register_manager.getReg(reg, null);
3522 }
3523
34613524 if (info.return_value == .stack_offset) {
34623525 const ret_ty = fn_ty.fnReturnType();
34633526 const ret_abi_size = @intCast(u32, ret_ty.abiSize(self.target.*));
34643527 const ret_abi_align = @intCast(u32, ret_ty.abiAlignment(self.target.*));
34653528 const stack_offset = @intCast(i32, try self.allocMem(inst, ret_abi_size, ret_abi_align));
3529 log.debug("airCall: return value on stack at offset {}", .{stack_offset});
34663530
34673531 try self.register_manager.getReg(.rdi, null);
3468 self.register_manager.freezeRegs(&.{.rdi});
34693532 try self.genSetReg(Type.usize, .rdi, .{ .ptr_stack_offset = stack_offset });
3533 self.register_manager.freezeRegs(&.{.rdi});
34703534
34713535 info.return_value.stack_offset = stack_offset;
34723536 }
......@@ -3653,15 +3717,13 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions.
36533717
36543718 const result: MCValue = result: {
36553719 switch (info.return_value) {
3656 .register => |reg| {
3657 if (RegisterManager.indexOfReg(&callee_preserved_regs, reg) == null) {
3658 // Save function return value in a callee saved register
3659 break :result try self.copyToRegisterWithInstTracking(
3660 inst,
3661 self.air.typeOfIndex(inst),
3662 info.return_value,
3663 );
3664 }
3720 .register => {
3721 // Save function return value in a new register
3722 break :result try self.copyToRegisterWithInstTracking(
3723 inst,
3724 self.air.typeOfIndex(inst),
3725 info.return_value,
3726 );
36653727 },
36663728 else => {},
36673729 }
......@@ -3688,19 +3750,11 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {
36883750 const ret_ty = self.fn_type.fnReturnType();
36893751 switch (self.ret_mcv) {
36903752 .stack_offset => {
3691 // TODO audit register allocation!
3692 self.register_manager.freezeRegs(&.{ .rax, .rcx, .rdi });
3693 defer self.register_manager.unfreezeRegs(&.{ .rax, .rcx, .rdi });
3694 const reg = try self.register_manager.allocReg(null);
3695 const backpatch = try self.addInst(.{
3696 .tag = .mov,
3697 .ops = (Mir.Ops{
3698 .reg1 = reg,
3699 .reg2 = .rdi,
3700 }).encode(),
3701 .data = undefined,
3702 });
3703 try self.ret_backpatches.append(self.gpa, backpatch);
3753 self.register_manager.freezeRegs(&.{ .rax, .rcx });
3754 defer self.register_manager.unfreezeRegs(&.{ .rax, .rcx });
3755 const reg = try self.copyToTmpRegister(Type.usize, self.ret_mcv);
3756 self.register_manager.freezeRegs(&.{reg});
3757 defer self.register_manager.unfreezeRegs(&.{reg});
37043758 try self.genSetStack(ret_ty, 0, operand, .{
37053759 .source_stack_base = .rbp,
37063760 .dest_stack_base = reg,
......@@ -3731,19 +3785,11 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
37313785 const elem_ty = ptr_ty.elemType();
37323786 switch (self.ret_mcv) {
37333787 .stack_offset => {
3734 // TODO audit register allocation!
3735 self.register_manager.freezeRegs(&.{ .rax, .rcx, .rdi });
3736 defer self.register_manager.unfreezeRegs(&.{ .rax, .rcx, .rdi });
3737 const reg = try self.register_manager.allocReg(null);
3738 const backpatch = try self.addInst(.{
3739 .tag = .mov,
3740 .ops = (Mir.Ops{
3741 .reg1 = reg,
3742 .reg2 = .rdi,
3743 }).encode(),
3744 .data = undefined,
3745 });
3746 try self.ret_backpatches.append(self.gpa, backpatch);
3788 self.register_manager.freezeRegs(&.{ .rax, .rcx });
3789 defer self.register_manager.unfreezeRegs(&.{ .rax, .rcx });
3790 const reg = try self.copyToTmpRegister(Type.usize, self.ret_mcv);
3791 self.register_manager.freezeRegs(&.{reg});
3792 defer self.register_manager.unfreezeRegs(&.{reg});
37473793 try self.genInlineMemcpy(.{ .stack_offset = 0 }, ptr, .{ .immediate = elem_ty.abiSize(self.target.*) }, .{
37483794 .source_stack_base = .rbp,
37493795 .dest_stack_base = reg,
......@@ -5033,6 +5079,7 @@ const InlineMemcpyOpts = struct {
50335079 dest_stack_base: ?Register = null,
50345080};
50355081
5082/// Spills .rax and .rcx.
50365083fn genInlineMemcpy(
50375084 self: *Self,
50385085 dst_ptr: MCValue,
......@@ -5040,13 +5087,7 @@ fn genInlineMemcpy(
50405087 len: MCValue,
50415088 opts: InlineMemcpyOpts,
50425089) InnerError!void {
5043 // TODO this is wrong. We should check first if any of the operands is in `.rax` or `.rcx` before
5044 // spilling. Consolidate with other TODOs regarding register allocation mechanics.
5045 try self.register_manager.getReg(.rax, null);
5046 try self.register_manager.getReg(.rcx, null);
5047
50485090 self.register_manager.freezeRegs(&.{ .rax, .rcx });
5049 defer self.register_manager.unfreezeRegs(&.{ .rax, .rcx });
50505091
50515092 if (opts.source_stack_base) |reg| self.register_manager.freezeRegs(&.{reg});
50525093 defer if (opts.source_stack_base) |reg| self.register_manager.unfreezeRegs(&.{reg});
......@@ -5086,7 +5127,6 @@ fn genInlineMemcpy(
50865127 return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr});
50875128 },
50885129 }
5089
50905130 self.register_manager.freezeRegs(&.{dst_addr_reg});
50915131 defer self.register_manager.unfreezeRegs(&.{dst_addr_reg});
50925132
......@@ -5122,7 +5162,6 @@ fn genInlineMemcpy(
51225162 return self.fail("TODO implement memcpy for setting stack when src is {}", .{src_ptr});
51235163 },
51245164 }
5125
51265165 self.register_manager.freezeRegs(&.{src_addr_reg});
51275166 defer self.register_manager.unfreezeRegs(&.{src_addr_reg});
51285167
......@@ -5130,6 +5169,11 @@ fn genInlineMemcpy(
51305169 const count_reg = regs[0].to64();
51315170 const tmp_reg = regs[1].to8();
51325171
5172 self.register_manager.unfreezeRegs(&.{ .rax, .rcx });
5173
5174 try self.register_manager.getReg(.rax, null);
5175 try self.register_manager.getReg(.rcx, null);
5176
51335177 try self.genSetReg(Type.usize, count_reg, len);
51345178
51355179 // mov rcx, 0
......@@ -5225,6 +5269,7 @@ fn genInlineMemcpy(
52255269 try self.performReloc(loop_reloc);
52265270}
52275271
5272/// Spills .rax register.
52285273fn genInlineMemset(
52295274 self: *Self,
52305275 dst_ptr: MCValue,
......@@ -5232,9 +5277,7 @@ fn genInlineMemset(
52325277 len: MCValue,
52335278 opts: InlineMemcpyOpts,
52345279) InnerError!void {
5235 try self.register_manager.getReg(.rax, null);
52365280 self.register_manager.freezeRegs(&.{.rax});
5237 defer self.register_manager.unfreezeRegs(&.{.rax});
52385281
52395282 const addr_reg = try self.register_manager.allocReg(null);
52405283 switch (dst_ptr) {
......@@ -5271,6 +5314,9 @@ fn genInlineMemset(
52715314 self.register_manager.freezeRegs(&.{addr_reg});
52725315 defer self.register_manager.unfreezeRegs(&.{addr_reg});
52735316
5317 self.register_manager.unfreezeRegs(&.{.rax});
5318 try self.register_manager.getReg(.rax, null);
5319
52745320 try self.genSetReg(Type.usize, .rax, len);
52755321 try self.genBinMathOpMir(.sub, Type.usize, .{ .register = .rax }, .{ .immediate = 1 });
52765322
src/arch/x86_64/Emit.zig+5-2
......@@ -145,6 +145,7 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
145145 .sar => try emit.mirShift(.sar, inst),
146146
147147 .imul => try emit.mirMulDiv(.imul, inst),
148 .mul => try emit.mirMulDiv(.mul, inst),
148149 .idiv => try emit.mirMulDiv(.idiv, inst),
149150 .div => try emit.mirMulDiv(.div, inst),
150151 .imul_complex => try emit.mirIMulComplex(inst),
......@@ -1164,6 +1165,7 @@ const Tag = enum {
11641165 brk,
11651166 nop,
11661167 imul,
1168 mul,
11671169 idiv,
11681170 div,
11691171 syscall,
......@@ -1411,7 +1413,7 @@ inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode {
14111413 .setnl, .setge => OpCode.twoByte(0x0f, 0x9d),
14121414 .setle, .setng => OpCode.twoByte(0x0f, 0x9e),
14131415 .setnle, .setg => OpCode.twoByte(0x0f, 0x9f),
1414 .idiv, .div, .imul => OpCode.oneByte(if (is_one_byte) 0xf6 else 0xf7),
1416 .idiv, .div, .imul, .mul => OpCode.oneByte(if (is_one_byte) 0xf6 else 0xf7),
14151417 .fisttp16 => OpCode.oneByte(0xdf),
14161418 .fisttp32 => OpCode.oneByte(0xdb),
14171419 .fisttp64 => OpCode.oneByte(0xdd),
......@@ -1554,9 +1556,10 @@ inline fn getModRmExt(tag: Tag) ?u3 {
15541556 => 0x4,
15551557 .shr => 0x5,
15561558 .sar => 0x7,
1559 .mul => 0x4,
15571560 .imul => 0x5,
1558 .idiv => 0x7,
15591561 .div => 0x6,
1562 .idiv => 0x7,
15601563 .fisttp16 => 0x1,
15611564 .fisttp32 => 0x1,
15621565 .fisttp64 => 0x1,
src/arch/x86_64/Mir.zig+1
......@@ -227,6 +227,7 @@ pub const Inst = struct {
227227 /// 0b11 qword ptr [reg2 + imm32]
228228 imul,
229229 idiv,
230 mul,
230231 div,
231232
232233 /// ops flags: form:
src/arch/x86_64/abi.zig+8-3
......@@ -370,8 +370,13 @@ pub fn classifySystemV(ty: Type, target: Target) [8]Class {
370370 }
371371}
372372
373/// These registers need to be preserved (saved on the stack) and restored by the callee before getting clobbered
374/// and when the callee returns.
375pub const callee_preserved_regs = [_]Register{ .rcx, .rsi, .rdi, .r8, .r9, .r10, .r11 };
373/// Note that .rsp and .rbp also belong to this set, however, we never expect to use them
374/// for anything else but stack offset tracking therefore we exclude them from this set.
375pub const callee_preserved_regs = [_]Register{ .rbx, .r12, .r13, .r14, .r15 };
376/// These registers need to be preserved (saved on the stack) and restored by the caller before
377/// the caller relinquishes control to a subroutine via call instruction (or similar).
378/// In other words, these registers are free to use by the callee.
379pub const caller_preserved_regs = [_]Register{ .rax, .rcx, .rdx, .rsi, .rdi, .r8, .r9, .r10, .r11 };
380pub const allocatable_registers = callee_preserved_regs ++ caller_preserved_regs;
376381pub const c_abi_int_param_regs = [_]Register{ .rdi, .rsi, .rdx, .rcx, .r8, .r9 };
377382pub const c_abi_int_return_regs = [_]Register{ .rax, .rdx };
test/behavior/math.zig+3-5
......@@ -697,11 +697,9 @@ test "@mulWithOverflow" {
697697 try expect(!@mulWithOverflow(u8, a, b, &result));
698698 try expect(result == 246);
699699
700 if (builtin.zig_backend != .stage2_x86_64) { // TODO fix mul/imul on x86_64
701 b = 4;
702 try expect(@mulWithOverflow(u8, a, b, &result));
703 try expect(result == 236);
704 }
700 b = 4;
701 try expect(@mulWithOverflow(u8, a, b, &result));
702 try expect(result == 236);
705703}
706704
707705test "@subWithOverflow" {