authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-06-07 23:22:20+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-06-07 23:22:20+02:00
log3cb387338234620e00645417565dc234dc5105c2
treeb8a9785e51adadbdf0bc9f44e9c501d2c441f050
parent6c59aa9e029c27c6da03287e907e72ac2c735b7b
parent27dad11ef166d5834be1166c86289c8e37aa0471
signature Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #11814 from ziglang/x64-stack-handling

x64: improves stack handling, fixes a heisenbug, adds micro-optimisations

5 files changed, 242 insertions(+), 199 deletions(-)

src/arch/x86_64/CodeGen.zig+166-157
...@@ -409,13 +409,11 @@ fn gen(self: *Self) InnerError!void {...@@ -409,13 +409,11 @@ fn gen(self: *Self) InnerError!void {
409 // The address where to store the return value for the caller is in `.rdi`409 // The address where to store the return value for the caller is in `.rdi`
410 // register which the callee is free to clobber. Therefore, we purposely410 // register which the callee is free to clobber. Therefore, we purposely
411 // spill it to stack immediately.411 // spill it to stack immediately.
412 const ptr_ty = Type.usize;412 const stack_offset = mem.alignForwardGeneric(u32, self.next_stack_offset + 8, 8);
413 const abi_size = @intCast(u32, ptr_ty.abiSize(self.target.*));
414 const abi_align = ptr_ty.abiAlignment(self.target.*);
415 const stack_offset = mem.alignForwardGeneric(u32, self.next_stack_offset + abi_size, abi_align);
416 self.next_stack_offset = stack_offset;413 self.next_stack_offset = stack_offset;
417 self.max_end_stack = @maximum(self.max_end_stack, self.next_stack_offset);414 self.max_end_stack = @maximum(self.max_end_stack, self.next_stack_offset);
418 try self.genSetStack(ptr_ty, @intCast(i32, stack_offset), MCValue{ .register = .rdi }, .{});415
416 try self.genSetStack(Type.usize, @intCast(i32, stack_offset), MCValue{ .register = .rdi }, .{});
419 self.ret_mcv = MCValue{ .stack_offset = @intCast(i32, stack_offset) };417 self.ret_mcv = MCValue{ .stack_offset = @intCast(i32, stack_offset) };
420 log.debug("gen: spilling .rdi to stack at offset {}", .{stack_offset});418 log.debug("gen: spilling .rdi to stack at offset {}", .{stack_offset});
421 }419 }
...@@ -426,11 +424,11 @@ fn gen(self: *Self) InnerError!void {...@@ -426,11 +424,11 @@ fn gen(self: *Self) InnerError!void {
426 .data = undefined,424 .data = undefined,
427 });425 });
428426
429 // push the callee_preserved_regs that were used427 // Push callee-preserved regs that were used actually in use.
430 const backpatch_push_callee_preserved_regs_i = try self.addInst(.{428 const backpatch_push_callee_preserved_regs = try self.addInst(.{
431 .tag = .push_regs_from_callee_preserved_regs,429 .tag = .nop,
432 .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),430 .ops = undefined,
433 .data = .{ .payload = undefined }, // to be backpatched431 .data = undefined,
434 });432 });
435433
436 try self.genBody(self.air.getMainBody());434 try self.genBody(self.air.getMainBody());
...@@ -446,31 +444,21 @@ fn gen(self: *Self) InnerError!void {...@@ -446,31 +444,21 @@ fn gen(self: *Self) InnerError!void {
446 self.mir_instructions.items(.data)[jmp_reloc].inst = @intCast(u32, self.mir_instructions.len);444 self.mir_instructions.items(.data)[jmp_reloc].inst = @intCast(u32, self.mir_instructions.len);
447 }445 }
448446
449 // calculate the data for callee_preserved_regs to be pushed and popped447 // Create list of registers to save in the prologue.
450 const callee_preserved_regs_payload = blk: {448 // TODO handle register classes
451 var data = Mir.RegsToPushOrPop{449 var reg_list: Mir.RegisterList(Register, &callee_preserved_regs) = .{};
452 .regs = 0,450 inline for (callee_preserved_regs) |reg| {
453 .disp = mem.alignForwardGeneric(u32, self.next_stack_offset, 8),451 if (self.register_manager.isRegAllocated(reg)) {
454 };452 reg_list.push(reg);
455 var disp = data.disp + 8;
456 inline for (callee_preserved_regs) |reg, i| {
457 if (self.register_manager.isRegAllocated(reg)) {
458 data.regs |= 1 << @intCast(u5, i);
459 self.max_end_stack += 8;
460 disp += 8;
461 }
462 }453 }
463 break :blk try self.addExtra(data);454 }
464 };455 const saved_regs_stack_space: u32 = reg_list.count() * 8;
465456
466 const data = self.mir_instructions.items(.data);457 // Pop saved callee-preserved regs.
467 // backpatch the push instruction458 const backpatch_pop_callee_preserved_regs = try self.addInst(.{
468 data[backpatch_push_callee_preserved_regs_i].payload = callee_preserved_regs_payload;459 .tag = .nop,
469 // pop the callee_preserved_regs460 .ops = undefined,
470 _ = try self.addInst(.{461 .data = undefined,
471 .tag = .pop_regs_from_callee_preserved_regs,
472 .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),
473 .data = .{ .payload = callee_preserved_regs_payload },
474 });462 });
475463
476 _ = try self.addInst(.{464 _ = try self.addInst(.{
...@@ -502,9 +490,11 @@ fn gen(self: *Self) InnerError!void {...@@ -502,9 +490,11 @@ fn gen(self: *Self) InnerError!void {
502 if (self.max_end_stack > math.maxInt(i32)) {490 if (self.max_end_stack > math.maxInt(i32)) {
503 return self.failSymbol("too much stack used in call parameters", .{});491 return self.failSymbol("too much stack used in call parameters", .{});
504 }492 }
505 // TODO we should reuse this mechanism to align the stack when calling any function even if493
506 // we do not pass any args on the stack BUT we still push regs to stack with `push` inst.494 const aligned_stack_end = @intCast(
507 const aligned_stack_end = @intCast(u32, mem.alignForward(self.max_end_stack, self.stack_align));495 u32,
496 mem.alignForward(self.max_end_stack + saved_regs_stack_space, self.stack_align),
497 );
508 if (aligned_stack_end > 0) {498 if (aligned_stack_end > 0) {
509 self.mir_instructions.set(backpatch_stack_sub, .{499 self.mir_instructions.set(backpatch_stack_sub, .{
510 .tag = .sub,500 .tag = .sub,
...@@ -516,6 +506,21 @@ fn gen(self: *Self) InnerError!void {...@@ -516,6 +506,21 @@ fn gen(self: *Self) InnerError!void {
516 .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }),506 .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }),
517 .data = .{ .imm = aligned_stack_end },507 .data = .{ .imm = aligned_stack_end },
518 });508 });
509
510 const save_reg_list = try self.addExtra(Mir.SaveRegisterList{
511 .register_list = reg_list.asInt(),
512 .stack_end = aligned_stack_end,
513 });
514 self.mir_instructions.set(backpatch_push_callee_preserved_regs, .{
515 .tag = .push_regs,
516 .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),
517 .data = .{ .payload = save_reg_list },
518 });
519 self.mir_instructions.set(backpatch_pop_callee_preserved_regs, .{
520 .tag = .pop_regs,
521 .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),
522 .data = .{ .payload = save_reg_list },
523 });
519 }524 }
520 } else {525 } else {
521 _ = try self.addInst(.{526 _ = try self.addInst(.{
...@@ -907,6 +912,39 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {...@@ -907,6 +912,39 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
907 return MCValue{ .stack_offset = @intCast(i32, stack_offset) };912 return MCValue{ .stack_offset = @intCast(i32, stack_offset) };
908}913}
909914
915const State = struct {
916 next_stack_offset: u32,
917 registers: abi.RegisterManager.TrackedRegisters,
918 free_registers: abi.RegisterManager.RegisterBitSet,
919 eflags_inst: ?Air.Inst.Index,
920 stack: std.AutoHashMapUnmanaged(u32, StackAllocation),
921
922 fn deinit(state: *State, gpa: Allocator) void {
923 state.stack.deinit(gpa);
924 }
925};
926
927fn captureState(self: *Self) !State {
928 return State{
929 .next_stack_offset = self.next_stack_offset,
930 .registers = self.register_manager.registers,
931 .free_registers = self.register_manager.free_registers,
932 .eflags_inst = self.eflags_inst,
933 .stack = try self.stack.clone(self.gpa),
934 };
935}
936
937fn revertState(self: *Self, state: State) void {
938 self.register_manager.registers = state.registers;
939 self.eflags_inst = state.eflags_inst;
940
941 self.stack.deinit(self.gpa);
942 self.stack = state.stack;
943
944 self.next_stack_offset = state.next_stack_offset;
945 self.register_manager.free_registers = state.free_registers;
946}
947
910pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {948pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {
911 const stack_mcv = try self.allocRegOrMem(inst, false);949 const stack_mcv = try self.allocRegOrMem(inst, false);
912 log.debug("spilling {d} to stack mcv {any}", .{ inst, stack_mcv });950 log.debug("spilling {d} to stack mcv {any}", .{ inst, stack_mcv });
...@@ -2062,8 +2100,22 @@ fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -2062,8 +2100,22 @@ fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void {
2062}2100}
20632101
2064fn elemOffset(self: *Self, index_ty: Type, index: MCValue, elem_size: u64) !Register {2102fn elemOffset(self: *Self, index_ty: Type, index: MCValue, elem_size: u64) !Register {
2065 const reg = try self.copyToTmpRegister(index_ty, index);2103 const reg: Register = blk: {
2066 try self.genIntMulComplexOpMir(index_ty, .{ .register = reg }, .{ .immediate = elem_size });2104 switch (index) {
2105 .immediate => |imm| {
2106 // Optimisation: if index MCValue is an immediate, we can multiply in `comptime`
2107 // and set the register directly to the scaled offset as an immediate.
2108 const reg = try self.register_manager.allocReg(null, gp);
2109 try self.genSetReg(index_ty, reg, .{ .immediate = imm * elem_size });
2110 break :blk reg;
2111 },
2112 else => {
2113 const reg = try self.copyToTmpRegister(index_ty, index);
2114 try self.genIntMulComplexOpMir(index_ty, .{ .register = reg }, .{ .immediate = elem_size });
2115 break :blk reg;
2116 },
2117 }
2118 };
2067 return reg;2119 return reg;
2068}2120}
20692121
...@@ -2678,15 +2730,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type...@@ -2678,15 +2730,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2678 // movabs does not support indirect register addressing2730 // movabs does not support indirect register addressing
2679 // so we need an extra register and an extra mov.2731 // so we need an extra register and an extra mov.
2680 const tmp_reg = try self.copyToTmpRegister(value_ty, value);2732 const tmp_reg = try self.copyToTmpRegister(value_ty, value);
2681 _ = try self.addInst(.{2733 return self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
2682 .tag = .mov,
2683 .ops = Mir.Inst.Ops.encode(.{
2684 .reg1 = reg.to64(),
2685 .reg2 = tmp_reg.to64(),
2686 .flags = 0b10,
2687 }),
2688 .data = .{ .imm = 0 },
2689 });
2690 },2734 },
2691 else => {2735 else => {
2692 return self.fail("TODO implement set pointee with immediate of ABI size {d}", .{abi_size});2736 return self.fail("TODO implement set pointee with immediate of ABI size {d}", .{abi_size});
...@@ -2694,15 +2738,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type...@@ -2694,15 +2738,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2694 }2738 }
2695 },2739 },
2696 .register => |src_reg| {2740 .register => |src_reg| {
2697 _ = try self.addInst(.{2741 try self.genInlineMemcpyRegisterRegister(value_ty, reg, src_reg, 0);
2698 .tag = .mov,
2699 .ops = Mir.Inst.Ops.encode(.{
2700 .reg1 = reg.to64(),
2701 .reg2 = registerAlias(src_reg, @intCast(u32, abi_size)),
2702 .flags = 0b10,
2703 }),
2704 .data = .{ .imm = 0 },
2705 });
2706 },2742 },
2707 .got_load,2743 .got_load,
2708 .direct_load,2744 .direct_load,
...@@ -2752,6 +2788,8 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type...@@ -2752,6 +2788,8 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2752 .data = .{ .imm = 0 },2788 .data = .{ .imm = 0 },
2753 });2789 });
27542790
2791 const new_ptr = MCValue{ .register = addr_reg.to64() };
2792
2755 switch (value) {2793 switch (value) {
2756 .immediate => |imm| {2794 .immediate => |imm| {
2757 if (abi_size > 8) {2795 if (abi_size > 8) {
...@@ -2790,16 +2828,8 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type...@@ -2790,16 +2828,8 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2790 .data = .{ .payload = payload },2828 .data = .{ .payload = payload },
2791 });2829 });
2792 },2830 },
2793 .register => |reg| {2831 .register => {
2794 _ = try self.addInst(.{2832 return self.store(new_ptr, value, ptr_ty, value_ty);
2795 .tag = .mov,
2796 .ops = Mir.Inst.Ops.encode(.{
2797 .reg1 = addr_reg.to64(),
2798 .reg2 = reg,
2799 .flags = 0b10,
2800 }),
2801 .data = .{ .imm = 0 },
2802 });
2803 },2833 },
2804 .got_load,2834 .got_load,
2805 .direct_load,2835 .direct_load,
...@@ -2821,37 +2851,18 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type...@@ -2821,37 +2851,18 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2821 }),2851 }),
2822 .data = .{ .imm = 0 },2852 .data = .{ .imm = 0 },
2823 });2853 });
2824 _ = try self.addInst(.{2854 return self.store(new_ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
2825 .tag = .mov,
2826 .ops = Mir.Inst.Ops.encode(.{
2827 .reg1 = addr_reg.to64(),
2828 .reg2 = tmp_reg,
2829 .flags = 0b10,
2830 }),
2831 .data = .{ .imm = 0 },
2832 });
2833 return;
2834 }2855 }
28352856
2836 try self.genInlineMemcpy(.{ .register = addr_reg.to64() }, value, .{ .immediate = abi_size }, .{});2857 try self.genInlineMemcpy(new_ptr, value, .{ .immediate = abi_size }, .{});
2837 },2858 },
2838 .stack_offset => {2859 .stack_offset => {
2839 if (abi_size <= 8) {2860 if (abi_size <= 8) {
2840 // TODO this should really be a recursive call
2841 const tmp_reg = try self.copyToTmpRegister(value_ty, value);2861 const tmp_reg = try self.copyToTmpRegister(value_ty, value);
2842 _ = try self.addInst(.{2862 return self.store(new_ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
2843 .tag = .mov,
2844 .ops = Mir.Inst.Ops.encode(.{
2845 .reg1 = addr_reg.to64(),
2846 .reg2 = tmp_reg,
2847 .flags = 0b10,
2848 }),
2849 .data = .{ .imm = 0 },
2850 });
2851 return;
2852 }2863 }
28532864
2854 try self.genInlineMemcpy(.{ .register = addr_reg.to64() }, value, .{ .immediate = abi_size }, .{});2865 try self.genInlineMemcpy(new_ptr, value, .{ .immediate = abi_size }, .{});
2855 },2866 },
2856 else => return self.fail("TODO implement storing {} to MCValue.memory", .{value}),2867 else => return self.fail("TODO implement storing {} to MCValue.memory", .{value}),
2857 }2868 }
...@@ -4503,12 +4514,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {...@@ -4503,12 +4514,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
4503 }4514 }
45044515
4505 // Capture the state of register and stack allocation state so that we can revert to it.4516 // Capture the state of register and stack allocation state so that we can revert to it.
4506 const parent_next_stack_offset = self.next_stack_offset;4517 const saved_state = try self.captureState();
4507 const parent_free_registers = self.register_manager.free_registers;
4508 const parent_eflags_inst = self.eflags_inst;
4509 var parent_stack = try self.stack.clone(self.gpa);
4510 defer parent_stack.deinit(self.gpa);
4511 const parent_registers = self.register_manager.registers;
45124518
4513 try self.branch_stack.append(.{});4519 try self.branch_stack.append(.{});
4514 errdefer {4520 errdefer {
...@@ -4526,17 +4532,10 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {...@@ -4526,17 +4532,10 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
4526 var saved_then_branch = self.branch_stack.pop();4532 var saved_then_branch = self.branch_stack.pop();
4527 defer saved_then_branch.deinit(self.gpa);4533 defer saved_then_branch.deinit(self.gpa);
45284534
4529 self.register_manager.registers = parent_registers;4535 self.revertState(saved_state);
4530 self.eflags_inst = parent_eflags_inst;
4531
4532 self.stack.deinit(self.gpa);
4533 self.stack = parent_stack;
4534 parent_stack = .{};
4535
4536 self.next_stack_offset = parent_next_stack_offset;
4537 self.register_manager.free_registers = parent_free_registers;
45384536
4539 try self.performReloc(reloc);4537 try self.performReloc(reloc);
4538
4540 const else_branch = self.branch_stack.addOneAssumeCapacity();4539 const else_branch = self.branch_stack.addOneAssumeCapacity();
4541 else_branch.* = .{};4540 else_branch.* = .{};
45424541
...@@ -5021,12 +5020,7 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {...@@ -5021,12 +5020,7 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
5021 }5020 }
50225021
5023 // Capture the state of register and stack allocation state so that we can revert to it.5022 // Capture the state of register and stack allocation state so that we can revert to it.
5024 const parent_next_stack_offset = self.next_stack_offset;5023 const saved_state = try self.captureState();
5025 const parent_free_registers = self.register_manager.free_registers;
5026 const parent_eflags_inst = self.eflags_inst;
5027 var parent_stack = try self.stack.clone(self.gpa);
5028 defer parent_stack.deinit(self.gpa);
5029 const parent_registers = self.register_manager.registers;
50305024
5031 try self.branch_stack.append(.{});5025 try self.branch_stack.append(.{});
5032 errdefer {5026 errdefer {
...@@ -5044,14 +5038,7 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {...@@ -5044,14 +5038,7 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
5044 var saved_case_branch = self.branch_stack.pop();5038 var saved_case_branch = self.branch_stack.pop();
5045 defer saved_case_branch.deinit(self.gpa);5039 defer saved_case_branch.deinit(self.gpa);
50465040
5047 self.register_manager.registers = parent_registers;5041 self.revertState(saved_state);
5048 self.eflags_inst = parent_eflags_inst;
5049 self.stack.deinit(self.gpa);
5050 self.stack = parent_stack;
5051 parent_stack = .{};
5052
5053 self.next_stack_offset = parent_next_stack_offset;
5054 self.register_manager.free_registers = parent_free_registers;
50555042
5056 for (relocs) |reloc| {5043 for (relocs) |reloc| {
5057 try self.performReloc(reloc);5044 try self.performReloc(reloc);
...@@ -5612,45 +5599,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl...@@ -5612,45 +5599,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5612 return self.fail("TODO genSetStack for register for type float with no intrinsics", .{});5599 return self.fail("TODO genSetStack for register for type float with no intrinsics", .{});
5613 },5600 },
5614 else => {5601 else => {
5615 if (!math.isPowerOfTwo(abi_size)) {5602 try self.genInlineMemcpyRegisterRegister(ty, base_reg, reg, stack_offset);
5616 const reg_lock = self.register_manager.lockReg(reg);
5617 defer if (reg_lock) |lock| self.register_manager.unlockReg(lock);
5618
5619 const tmp_reg = try self.copyToTmpRegister(ty, mcv);
5620
5621 var next_offset = stack_offset;
5622 var remainder = abi_size;
5623 while (remainder > 0) {
5624 const nearest_power_of_two = @as(u6, 1) << math.log2_int(u3, @intCast(u3, remainder));
5625
5626 _ = try self.addInst(.{
5627 .tag = .mov,
5628 .ops = Mir.Inst.Ops.encode(.{
5629 .reg1 = base_reg,
5630 .reg2 = registerAlias(tmp_reg, nearest_power_of_two),
5631 .flags = 0b10,
5632 }),
5633 .data = .{ .imm = @bitCast(u32, -next_offset) },
5634 });
5635
5636 if (nearest_power_of_two > 1) {
5637 try self.genShiftBinOpMir(.shr, ty, tmp_reg, .{ .immediate = nearest_power_of_two * 8 });
5638 }
5639
5640 remainder -= nearest_power_of_two;
5641 next_offset -= nearest_power_of_two;
5642 }
5643 } else {
5644 _ = try self.addInst(.{
5645 .tag = .mov,
5646 .ops = Mir.Inst.Ops.encode(.{
5647 .reg1 = base_reg,
5648 .reg2 = registerAlias(reg, @intCast(u32, abi_size)),
5649 .flags = 0b10,
5650 }),
5651 .data = .{ .imm = @bitCast(u32, -stack_offset) },
5652 });
5653 }
5654 },5603 },
5655 }5604 }
5656 },5605 },
...@@ -5685,6 +5634,66 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl...@@ -5685,6 +5634,66 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5685 }5634 }
5686}5635}
56875636
5637/// Like `genInlineMemcpy` but copies value from a register to an address via dereferencing
5638/// of destination register.
5639/// Boils down to MOV r/m64, r64.
5640fn genInlineMemcpyRegisterRegister(
5641 self: *Self,
5642 ty: Type,
5643 dst_reg: Register,
5644 src_reg: Register,
5645 offset: i32,
5646) InnerError!void {
5647 assert(dst_reg.size() == 64);
5648
5649 const dst_reg_lock = self.register_manager.lockReg(dst_reg);
5650 defer if (dst_reg_lock) |lock| self.register_manager.unlockReg(lock);
5651
5652 const src_reg_lock = self.register_manager.lockReg(src_reg);
5653 defer if (src_reg_lock) |lock| self.register_manager.unlockReg(lock);
5654
5655 const abi_size = @intCast(u32, ty.abiSize(self.target.*));
5656
5657 if (!math.isPowerOfTwo(abi_size)) {
5658 const tmp_reg = try self.copyToTmpRegister(ty, .{ .register = src_reg });
5659
5660 var next_offset = offset;
5661 var remainder = abi_size;
5662 while (remainder > 0) {
5663 const nearest_power_of_two = @as(u6, 1) << math.log2_int(u3, @intCast(u3, remainder));
5664
5665 _ = try self.addInst(.{
5666 .tag = .mov,
5667 .ops = Mir.Inst.Ops.encode(.{
5668 .reg1 = dst_reg,
5669 .reg2 = registerAlias(tmp_reg, nearest_power_of_two),
5670 .flags = 0b10,
5671 }),
5672 .data = .{ .imm = @bitCast(u32, -next_offset) },
5673 });
5674
5675 if (nearest_power_of_two > 1) {
5676 try self.genShiftBinOpMir(.shr, ty, tmp_reg, .{
5677 .immediate = nearest_power_of_two * 8,
5678 });
5679 }
5680
5681 remainder -= nearest_power_of_two;
5682 next_offset -= nearest_power_of_two;
5683 }
5684 } else {
5685 _ = try self.addInst(.{
5686 .tag = .mov,
5687 .ops = Mir.Inst.Ops.encode(.{
5688 .reg1 = dst_reg,
5689 .reg2 = registerAlias(src_reg, @intCast(u32, abi_size)),
5690 .flags = 0b10,
5691 }),
5692 .data = .{ .imm = @bitCast(u32, -offset) },
5693 });
5694 }
5695}
5696
5688const InlineMemcpyOpts = struct {5697const InlineMemcpyOpts = struct {
5689 source_stack_base: ?Register = null,5698 source_stack_base: ?Register = null,
5690 dest_stack_base: ?Register = null,5699 dest_stack_base: ?Register = null,
src/arch/x86_64/Emit.zig+21-24
...@@ -169,7 +169,7 @@ pub fn lowerMir(emit: *Emit) InnerError!void {...@@ -169,7 +169,7 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
169 .@"test" => try emit.mirTest(inst),169 .@"test" => try emit.mirTest(inst),
170170
171 .interrupt => try emit.mirInterrupt(inst),171 .interrupt => try emit.mirInterrupt(inst),
172 .nop => try emit.mirNop(),172 .nop => {}, // just skip it
173173
174 // SSE instructions174 // SSE instructions
175 .mov_f64_sse => try emit.mirMovFloatSse(.movsd, inst),175 .mov_f64_sse => try emit.mirMovFloatSse(.movsd, inst),
...@@ -198,8 +198,8 @@ pub fn lowerMir(emit: *Emit) InnerError!void {...@@ -198,8 +198,8 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
198 .dbg_prologue_end => try emit.mirDbgPrologueEnd(inst),198 .dbg_prologue_end => try emit.mirDbgPrologueEnd(inst),
199 .dbg_epilogue_begin => try emit.mirDbgEpilogueBegin(inst),199 .dbg_epilogue_begin => try emit.mirDbgEpilogueBegin(inst),
200200
201 .push_regs_from_callee_preserved_regs => try emit.mirPushPopRegsFromCalleePreservedRegs(.push, inst),201 .push_regs => try emit.mirPushPopRegisterList(.push, inst),
202 .pop_regs_from_callee_preserved_regs => try emit.mirPushPopRegsFromCalleePreservedRegs(.pop, inst),202 .pop_regs => try emit.mirPushPopRegisterList(.pop, inst),
203203
204 else => {204 else => {
205 return emit.fail("Implement MIR->Emit lowering for x86_64 for pseudo-inst: {s}", .{tag});205 return emit.fail("Implement MIR->Emit lowering for x86_64 for pseudo-inst: {s}", .{tag});
...@@ -246,10 +246,6 @@ fn mirInterrupt(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {...@@ -246,10 +246,6 @@ fn mirInterrupt(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
246 }246 }
247}247}
248248
249fn mirNop(emit: *Emit) InnerError!void {
250 return lowerToZoEnc(.nop, emit.code);
251}
252
253fn mirSyscall(emit: *Emit) InnerError!void {249fn mirSyscall(emit: *Emit) InnerError!void {
254 return lowerToZoEnc(.syscall, emit.code);250 return lowerToZoEnc(.syscall, emit.code);
255}251}
...@@ -283,26 +279,27 @@ fn mirPushPop(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {...@@ -283,26 +279,27 @@ fn mirPushPop(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
283 }279 }
284}280}
285281
286fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {282fn mirPushPopRegisterList(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
287 const ops = emit.mir.instructions.items(.ops)[inst].decode();283 const ops = emit.mir.instructions.items(.ops)[inst].decode();
288 const payload = emit.mir.instructions.items(.data)[inst].payload;284 const payload = emit.mir.instructions.items(.data)[inst].payload;
289 const data = emit.mir.extraData(Mir.RegsToPushOrPop, payload).data;285 const save_reg_list = emit.mir.extraData(Mir.SaveRegisterList, payload).data;
290 const regs = data.regs;286 const reg_list = Mir.RegisterList(Register, &abi.callee_preserved_regs).fromInt(save_reg_list.register_list);
291 var disp: u32 = data.disp + 8;287 var disp: i32 = -@intCast(i32, save_reg_list.stack_end);
292 for (abi.callee_preserved_regs) |reg, i| {288 inline for (abi.callee_preserved_regs) |reg| {
293 if ((regs >> @intCast(u5, i)) & 1 == 0) continue;289 if (reg_list.isSet(reg)) {
294 if (tag == .push) {290 switch (tag) {
295 try lowerToMrEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{291 .push => try lowerToMrEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
296 .disp = @bitCast(u32, -@intCast(i32, disp)),292 .disp = @bitCast(u32, disp),
297 .base = ops.reg1,293 .base = ops.reg1,
298 }), reg.to64(), emit.code);294 }), reg, emit.code),
299 } else {295 .pop => try lowerToRmEnc(.mov, reg, RegisterOrMemory.mem(.qword_ptr, .{
300 try lowerToRmEnc(.mov, reg.to64(), RegisterOrMemory.mem(.qword_ptr, .{296 .disp = @bitCast(u32, disp),
301 .disp = @bitCast(u32, -@intCast(i32, disp)),297 .base = ops.reg1,
302 .base = ops.reg1,298 }), emit.code),
303 }), emit.code);299 else => unreachable,
300 }
301 disp += 8;
304 }302 }
305 disp += 8;
306 }303 }
307}304}
308305
src/arch/x86_64/Mir.zig+53-16
...@@ -14,6 +14,7 @@ const assert = std.debug.assert;...@@ -14,6 +14,7 @@ const assert = std.debug.assert;
14const bits = @import("bits.zig");14const bits = @import("bits.zig");
15const Air = @import("../../Air.zig");15const Air = @import("../../Air.zig");
16const CodeGen = @import("CodeGen.zig");16const CodeGen = @import("CodeGen.zig");
17const IntegerBitSet = std.bit_set.IntegerBitSet;
17const Register = bits.Register;18const Register = bits.Register;
1819
19instructions: std.MultiArrayList(Inst).Slice,20instructions: std.MultiArrayList(Inst).Slice,
...@@ -379,19 +380,13 @@ pub const Inst = struct {...@@ -379,19 +380,13 @@ pub const Inst = struct {
379 /// update debug line380 /// update debug line
380 dbg_line,381 dbg_line,
381382
382 /// push registers from the callee_preserved_regs383 /// push registers
383 /// data is the bitfield of which regs to push384 /// Uses `payload` field with `SaveRegisterList` as payload.
384 /// for example on x86_64, the callee_preserved_regs are [_]Register{ .rcx, .rsi, .rdi, .r8, .r9, .r10, .r11 }; };385 push_regs,
385 /// so to push rcx and r8 one would make data 0b00000000_00000000_00000000_00001001 (the first and fourth bits are set)386
386 /// ops is unused387 /// pop registers
387 push_regs_from_callee_preserved_regs,388 /// Uses `payload` field with `SaveRegisterList` as payload.
388389 pop_regs,
389 /// pop registers from the callee_preserved_regs
390 /// data is the bitfield of which regs to pop
391 /// for example on x86_64, the callee_preserved_regs are [_]Register{ .rcx, .rsi, .rdi, .r8, .r9, .r10, .r11 }; };
392 /// so to pop rcx and r8 one would make data 0b00000000_00000000_00000000_00001001 (the first and fourth bits are set)
393 /// ops is unused
394 pop_regs_from_callee_preserved_regs,
395 };390 };
396 /// The position of an MIR instruction within the `Mir` instructions array.391 /// The position of an MIR instruction within the `Mir` instructions array.
397 pub const Index = u32;392 pub const Index = u32;
...@@ -471,9 +466,51 @@ pub const Inst = struct {...@@ -471,9 +466,51 @@ pub const Inst = struct {
471 }466 }
472};467};
473468
474pub const RegsToPushOrPop = struct {469pub fn RegisterList(comptime Reg: type, comptime registers: []const Reg) type {
475 regs: u32,470 assert(registers.len <= @bitSizeOf(u32));
476 disp: u32,471 return struct {
472 bitset: RegBitSet = RegBitSet.initEmpty(),
473
474 const RegBitSet = IntegerBitSet(registers.len);
475 const Self = @This();
476
477 fn getIndexForReg(reg: Reg) RegBitSet.MaskInt {
478 inline for (registers) |cpreg, i| {
479 if (reg.id() == cpreg.id()) return i;
480 }
481 unreachable; // register not in input register list!
482 }
483
484 pub fn push(self: *Self, reg: Reg) void {
485 const index = getIndexForReg(reg);
486 self.bitset.set(index);
487 }
488
489 pub fn isSet(self: Self, reg: Reg) bool {
490 const index = getIndexForReg(reg);
491 return self.bitset.isSet(index);
492 }
493
494 pub fn asInt(self: Self) u32 {
495 return self.bitset.mask;
496 }
497
498 pub fn fromInt(mask: u32) Self {
499 return .{
500 .bitset = RegBitSet{ .mask = @intCast(RegBitSet.MaskInt, mask) },
501 };
502 }
503
504 pub fn count(self: Self) u32 {
505 return @intCast(u32, self.bitset.count());
506 }
507 };
508}
509
510pub const SaveRegisterList = struct {
511 /// Use `RegisterList` to populate.
512 register_list: u32,
513 stack_end: u32,
477};514};
478515
479pub const ImmPair = struct {516pub const ImmPair = struct {
src/register_manager.zig+2-1
...@@ -39,7 +39,7 @@ pub fn RegisterManager(...@@ -39,7 +39,7 @@ pub fn RegisterManager(
39 /// register is free), the value in that slot is undefined.39 /// register is free), the value in that slot is undefined.
40 ///40 ///
41 /// The key must be canonical register.41 /// The key must be canonical register.
42 registers: [tracked_registers.len]Air.Inst.Index = undefined,42 registers: TrackedRegisters = undefined,
43 /// Tracks which registers are free (in which case the43 /// Tracks which registers are free (in which case the
44 /// corresponding bit is set to 1)44 /// corresponding bit is set to 1)
45 free_registers: RegisterBitSet = RegisterBitSet.initFull(),45 free_registers: RegisterBitSet = RegisterBitSet.initFull(),
...@@ -51,6 +51,7 @@ pub fn RegisterManager(...@@ -51,6 +51,7 @@ pub fn RegisterManager(
5151
52 const Self = @This();52 const Self = @This();
5353
54 pub const TrackedRegisters = [tracked_registers.len]Air.Inst.Index;
54 pub const RegisterBitSet = StaticBitSet(tracked_registers.len);55 pub const RegisterBitSet = StaticBitSet(tracked_registers.len);
5556
56 fn getFunction(self: *Self) *Function {57 fn getFunction(self: *Self) *Function {
test/behavior/bugs/1381.zig-1
...@@ -14,7 +14,6 @@ const A = union(enum) {...@@ -14,7 +14,6 @@ const A = union(enum) {
14test "union that needs padding bytes inside an array" {14test "union that needs padding bytes inside an array" {
15 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;15 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
16 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;16 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
17 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
1817
19 var as = [_]A{18 var as = [_]A{
20 A{ .B = B{ .D = 1 } },19 A{ .B = B{ .D = 1 } },