authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-03-23 02:03:43-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-03-24 17:57:58-04:00
log12c07fcf20aba9e986b5b2131515b33e3d27176a
tree1dceb6741c25a00f4742359acdaf4f5e9d4ef98a
parentdbe1b4a7e5731e4fb17d42b754faf1052aa78f32

x86_64: fix more value tracking bugs

5 files changed, 85 insertions(+), 95 deletions(-)

src/arch/x86_64/CodeGen.zig+81-89
......@@ -214,11 +214,6 @@ const StackAllocation = struct {
214214
215215const BlockData = struct {
216216 relocs: std.ArrayListUnmanaged(Mir.Inst.Index),
217 /// The first break instruction encounters `null` here and chooses a
218 /// machine code value for the block result, populating this field.
219 /// Following break instructions encounter that value and use it for
220 /// the location to store their block results.
221 mcv: MCValue,
222217};
223218
224219const BigTomb = struct {
......@@ -1078,16 +1073,12 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
10781073 var it = self.register_manager.free_registers.iterator(.{ .kind = .unset });
10791074 while (it.next()) |index| {
10801075 const tracked_inst = self.register_manager.registers[index];
1081 switch (air_tags[tracked_inst]) {
1082 .block => {},
1083 else => assert(RegisterManager.indexOfRegIntoTracked(
1084 switch (self.getResolvedInstValue(tracked_inst).?) {
1085 .register => |reg| reg,
1086 .register_overflow => |ro| ro.reg,
1087 else => unreachable,
1088 },
1089 ).? == index),
1090 }
1076 const tracked_mcv = self.getResolvedInstValue(tracked_inst).?.*;
1077 assert(RegisterManager.indexOfRegIntoTracked(switch (tracked_mcv) {
1078 .register => |reg| reg,
1079 .register_overflow => |ro| ro.reg,
1080 else => unreachable,
1081 }).? == index);
10911082 }
10921083 }
10931084 }
......@@ -1114,7 +1105,7 @@ fn freeValue(self: *Self, value: MCValue) void {
11141105fn processDeath(self: *Self, inst: Air.Inst.Index) void {
11151106 const air_tags = self.air.instructions.items(.tag);
11161107 if (air_tags[inst] == .constant) return; // Constants are immortal.
1117 const prev_value = self.getResolvedInstValue(inst) orelse return;
1108 const prev_value = (self.getResolvedInstValue(inst) orelse return).*;
11181109 log.debug("%{d} => {}", .{ inst, MCValue.dead });
11191110 // When editing this function, note that the logic must synchronize with `reuseOperand`.
11201111 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
......@@ -1259,45 +1250,29 @@ fn allocRegOrMemAdvanced(self: *Self, elem_ty: Type, inst: ?Air.Inst.Index, reg_
12591250}
12601251
12611252const State = struct {
1262 next_stack_offset: u32,
12631253 registers: abi.RegisterManager.TrackedRegisters,
12641254 free_registers: abi.RegisterManager.RegisterBitSet,
12651255 eflags_inst: ?Air.Inst.Index,
1266 stack: std.AutoHashMapUnmanaged(u32, StackAllocation),
1267
1268 fn deinit(state: *State, gpa: Allocator) void {
1269 state.stack.deinit(gpa);
1270 }
12711256};
12721257
1273fn captureState(self: *Self) !State {
1258fn captureState(self: *Self) State {
12741259 return State{
1275 .next_stack_offset = self.next_stack_offset,
12761260 .registers = self.register_manager.registers,
12771261 .free_registers = self.register_manager.free_registers,
12781262 .eflags_inst = self.eflags_inst,
1279 .stack = try self.stack.clone(self.gpa),
12801263 };
12811264}
12821265
1283fn revertState(self: *Self, state: State) !void {
1284 var stack = try state.stack.clone(self.gpa);
1285 errdefer stack.deinit(self.gpa);
1286
1287 self.register_manager.registers = state.registers;
1266fn revertState(self: *Self, state: State) void {
12881267 self.eflags_inst = state.eflags_inst;
1289
1290 self.stack.deinit(self.gpa);
1291 self.stack = stack;
1292
1293 self.next_stack_offset = state.next_stack_offset;
12941268 self.register_manager.free_registers = state.free_registers;
1269 self.register_manager.registers = state.registers;
12951270}
12961271
12971272pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {
12981273 const stack_mcv = try self.allocRegOrMem(inst, false);
12991274 log.debug("spilling %{d} to stack mcv {any}", .{ inst, stack_mcv });
1300 const reg_mcv = self.getResolvedInstValue(inst).?;
1275 const reg_mcv = self.getResolvedInstValue(inst).?.*;
13011276 switch (reg_mcv) {
13021277 .register => |other| {
13031278 assert(reg.to64() == other.to64());
......@@ -1314,7 +1289,7 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void
13141289
13151290pub fn spillEflagsIfOccupied(self: *Self) !void {
13161291 if (self.eflags_inst) |inst_to_save| {
1317 const mcv = self.getResolvedInstValue(inst_to_save).?;
1292 const mcv = self.getResolvedInstValue(inst_to_save).?.*;
13181293 const new_mcv = switch (mcv) {
13191294 .register_overflow => try self.allocRegOrMem(inst_to_save, false),
13201295 .eflags => try self.allocRegOrMem(inst_to_save, true),
......@@ -2607,7 +2582,7 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
26072582 const index_ty = self.air.typeOf(bin_op.rhs);
26082583 const index = try self.resolveInst(bin_op.rhs);
26092584 const index_lock: ?RegisterLock = switch (index) {
2610 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2585 .register => |reg| self.register_manager.lockReg(reg),
26112586 else => null,
26122587 };
26132588 defer if (index_lock) |lock| self.register_manager.unlockReg(lock);
......@@ -2656,7 +2631,7 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
26562631 const index_ty = self.air.typeOf(extra.rhs);
26572632 const index = try self.resolveInst(extra.rhs);
26582633 const index_lock: ?RegisterLock = switch (index) {
2659 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2634 .register => |reg| self.register_manager.lockReg(reg),
26602635 else => null,
26612636 };
26622637 defer if (index_lock) |lock| self.register_manager.unlockReg(lock);
......@@ -3202,8 +3177,8 @@ fn reuseOperand(
32023177 .register => |reg| {
32033178 // If it's in the registers table, need to associate the register with the
32043179 // new instruction.
3205 if (RegisterManager.indexOfRegIntoTracked(reg)) |index| {
3206 if (!self.register_manager.isRegFree(reg)) {
3180 if (!self.register_manager.isRegFree(reg)) {
3181 if (RegisterManager.indexOfRegIntoTracked(reg)) |index| {
32073182 self.register_manager.registers[index] = inst;
32083183 }
32093184 }
......@@ -3542,7 +3517,7 @@ fn airStore(self: *Self, inst: Air.Inst.Index) !void {
35423517 const value_ty = self.air.typeOf(bin_op.rhs);
35433518 log.debug("airStore(%{d}): {} <- {}", .{ inst, ptr, value });
35443519 try self.store(ptr, value, ptr_ty, value_ty);
3545 return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
3520 return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });
35463521}
35473522
35483523fn airStructFieldPtr(self: *Self, inst: Air.Inst.Index) !void {
......@@ -5270,8 +5245,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
52705245 }
52715246
52725247 // Capture the state of register and stack allocation state so that we can revert to it.
5273 var saved_state = try self.captureState();
5274 defer saved_state.deinit(self.gpa);
5248 const saved_state = self.captureState();
52755249
52765250 {
52775251 try self.branch_stack.append(.{});
......@@ -5289,7 +5263,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
52895263 var then_branch = self.branch_stack.pop();
52905264 defer then_branch.deinit(self.gpa);
52915265
5292 try self.revertState(saved_state);
5266 self.revertState(saved_state);
52935267
52945268 try self.performReloc(reloc);
52955269
......@@ -5632,24 +5606,28 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
56325606 try self.blocks.putNoClobber(self.gpa, inst, .{
56335607 // A block is a setup to be able to jump to the end.
56345608 .relocs = .{},
5635 // It also acts as a receptacle for break operands.
5636 // Here we use `MCValue.none` to represent a null value so that the first
5637 // break instruction will choose a MCValue for the block result and overwrite
5638 // this field. Following break instructions will use that MCValue to put their
5639 // block results.
5640 .mcv = if (self.liveness.isUnused(inst)) .dead else .none,
56415609 });
56425610 defer self.blocks.getPtr(inst).?.relocs.deinit(self.gpa);
56435611
5612 {
5613 // Here we use `.none` to represent a null value so that the first break
5614 // instruction will choose a MCValue for the block result and overwrite
5615 // this field. Following break instructions will use that MCValue to put
5616 // their block results.
5617 const ty = self.air.typeOfIndex(inst);
5618 const result: MCValue =
5619 if (!ty.hasRuntimeBitsIgnoreComptime() or self.liveness.isUnused(inst)) .dead else .none;
5620 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
5621 branch.inst_table.putAssumeCapacityNoClobber(inst, result);
5622 }
5623
56445624 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
56455625 const extra = self.air.extraData(Air.Block, ty_pl.payload);
56465626 const body = self.air.extra[extra.end..][0..extra.data.body_len];
56475627 try self.genBody(body);
56485628
56495629 for (self.blocks.getPtr(inst).?.relocs.items) |reloc| try self.performReloc(reloc);
5650
5651 const result = self.blocks.getPtr(inst).?.mcv;
5652 return self.finishAir(inst, result, .{ .none, .none, .none });
5630 self.finishAirBookkeeping();
56535631}
56545632
56555633fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
......@@ -5687,8 +5665,7 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
56875665 defer if (prev_branch) |*branch| branch.deinit(self.gpa);
56885666
56895667 // Capture the state of register and stack allocation state so that we can revert to it.
5690 var saved_state = try self.captureState();
5691 defer saved_state.deinit(self.gpa);
5668 const saved_state = self.captureState();
56925669
56935670 const cases_len = switch_br.data.cases_len + @boolToInt(switch_br.data.else_body_len > 0);
56945671 while (case_i < switch_br.data.cases_len) : (case_i += 1) {
......@@ -5698,7 +5675,7 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
56985675 extra_index = case.end + items.len + case_body.len;
56995676
57005677 // Revert to the previous register and stack allocation state.
5701 if (prev_branch) |_| try self.revertState(saved_state);
5678 if (prev_branch) |_| self.revertState(saved_state);
57025679
57035680 var relocs = try self.gpa.alloc(u32, items.len);
57045681 defer self.gpa.free(relocs);
......@@ -5742,7 +5719,7 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
57425719 const else_body = self.air.extra[extra_index..][0..switch_br.data.else_body_len];
57435720
57445721 // Revert to the previous register and stack allocation state.
5745 if (prev_branch) |_| try self.revertState(saved_state);
5722 if (prev_branch) |_| self.revertState(saved_state);
57465723
57475724 {
57485725 if (cases_len > 1) try self.branch_stack.append(.{});
......@@ -5801,7 +5778,7 @@ fn canonicaliseBranches(
58015778 if (target_value == .dead) continue;
58025779 // The instruction is only overridden in the else branch.
58035780 // If integer overflows occurs, the question is: why wasn't the instruction marked dead?
5804 break :blk self.getResolvedInstValue(target_key).?;
5781 break :blk self.getResolvedInstValue(target_key).?.*;
58055782 };
58065783 log.debug("consolidating target_entry {d} {}=>{}", .{ target_key, target_value, canon_mcv });
58075784 // TODO make sure the destination stack offset / register does not already have something
......@@ -5816,17 +5793,18 @@ fn canonicaliseBranches(
58165793 // We already deleted the items from this table that matched the target_branch.
58175794 // So these are all instructions that are only overridden in the canon branch.
58185795 const parent_mcv =
5819 if (canon_value != .dead) self.getResolvedInstValue(canon_key).? else undefined;
5796 if (canon_value != .dead) self.getResolvedInstValue(canon_key).?.* else undefined;
5797 if (canon_value != .dead) {
5798 log.debug("consolidating canon_entry {d} {}=>{}", .{ canon_key, parent_mcv, canon_value });
5799 // TODO make sure the destination stack offset / register does not already have something
5800 // going on there.
5801 try self.setRegOrMem(self.air.typeOfIndex(canon_key), canon_value, parent_mcv);
5802 self.freeValue(parent_mcv);
5803 // TODO track the new register / stack allocation
5804 }
58205805 if (update_parent) {
58215806 parent_branch.inst_table.putAssumeCapacity(canon_key, canon_value);
58225807 }
5823 if (canon_value == .dead) continue;
5824 log.debug("consolidating canon_entry {d} {}=>{}", .{ canon_key, parent_mcv, canon_value });
5825 // TODO make sure the destination stack offset / register does not already have something
5826 // going on there.
5827 try self.setRegOrMem(self.air.typeOfIndex(canon_key), canon_value, parent_mcv);
5828 self.freeValue(parent_mcv);
5829 // TODO track the new register / stack allocation
58305808 }
58315809}
58325810
......@@ -5845,27 +5823,41 @@ fn performReloc(self: *Self, reloc: Mir.Inst.Index) !void {
58455823
58465824fn airBr(self: *Self, inst: Air.Inst.Index) !void {
58475825 const branch = self.air.instructions.items(.data)[inst].br;
5848 try self.br(branch.block_inst, branch.operand);
5826 try self.br(inst, branch.block_inst, branch.operand);
58495827 return self.finishAir(inst, .dead, .{ branch.operand, .none, .none });
58505828}
58515829
5852fn br(self: *Self, block: Air.Inst.Index, operand: Air.Inst.Ref) !void {
5853 const block_data = self.blocks.getPtr(block).?;
5854 if (block_data.mcv != .dead and self.air.typeOf(operand).hasRuntimeBits()) {
5855 const operand_mcv = try self.resolveInst(operand);
5856 if (block_data.mcv == .none) {
5857 block_data.mcv = switch (operand_mcv) {
5858 .none, .dead, .unreach => unreachable,
5859 .register, .stack_offset, .memory => operand_mcv,
5860 .eflags, .immediate, .ptr_stack_offset => blk: {
5861 const new_mcv = try self.allocRegOrMem(block, true);
5862 try self.setRegOrMem(self.air.typeOfIndex(block), new_mcv, operand_mcv);
5863 break :blk new_mcv;
5864 },
5865 else => return self.fail("TODO implement block_data.mcv = operand_mcv for {}", .{operand_mcv}),
5866 };
5867 } else {
5868 try self.setRegOrMem(self.air.typeOfIndex(block), block_data.mcv, operand_mcv);
5830fn br(self: *Self, inst: Air.Inst.Index, block: Air.Inst.Index, operand: Air.Inst.Ref) !void {
5831 // The first break instruction encounters `.none` here and chooses a
5832 // machine code value for the block result, populating this field.
5833 // Following break instructions encounter that value and use it for
5834 // the location to store their block results.
5835 if (self.getResolvedInstValue(block)) |dst_mcv| {
5836 const src_mcv = try self.resolveInst(operand);
5837 switch (dst_mcv.*) {
5838 .none => {
5839 const result = result: {
5840 if (!self.reuseOperand(inst, operand, 0, src_mcv)) {
5841 const new_mcv = try self.allocRegOrMem(block, true);
5842 try self.setRegOrMem(self.air.typeOfIndex(block), new_mcv, src_mcv);
5843 break :result new_mcv;
5844 }
5845
5846 // the value is actually tracked with block, not inst
5847 switch (src_mcv) {
5848 .register => |reg| if (!self.register_manager.isRegFree(reg)) {
5849 if (RegisterManager.indexOfRegIntoTracked(reg)) |index| {
5850 self.register_manager.registers[index] = block;
5851 }
5852 },
5853 .stack_offset => {},
5854 else => unreachable,
5855 }
5856 break :result src_mcv;
5857 };
5858 dst_mcv.* = result;
5859 },
5860 else => try self.setRegOrMem(self.air.typeOfIndex(block), dst_mcv.*, src_mcv),
58695861 }
58705862 }
58715863 return self.brVoid(block);
......@@ -7243,17 +7235,17 @@ fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {
72437235 return gop.value_ptr.*;
72447236 },
72457237 .const_ty => unreachable,
7246 else => return self.getResolvedInstValue(inst_index).?,
7238 else => return self.getResolvedInstValue(inst_index).?.*,
72477239 }
72487240}
72497241
7250fn getResolvedInstValue(self: *Self, inst: Air.Inst.Index) ?MCValue {
7242fn getResolvedInstValue(self: *Self, inst: Air.Inst.Index) ?*MCValue {
72517243 // Treat each stack item as a "layer" on top of the previous one.
72527244 var i: usize = self.branch_stack.items.len;
72537245 while (true) {
72547246 i -= 1;
7255 if (self.branch_stack.items[i].inst_table.get(inst)) |mcv| {
7256 return if (mcv != .dead) mcv else null;
7247 if (self.branch_stack.items[i].inst_table.getPtr(inst)) |mcv| {
7248 return if (mcv.* != .dead) mcv else null;
72577249 }
72587250 }
72597251}
test/behavior/cast.zig+2-5
......@@ -38,6 +38,8 @@ fn peerTypeTAndOptionalT(c: bool, b: bool) ?usize {
3838}
3939
4040test "resolve undefined with integer" {
41 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
42
4143 try testResolveUndefWithInt(true, 1234);
4244 comptime try testResolveUndefWithInt(true, 1234);
4345}
......@@ -419,7 +421,6 @@ fn testCastIntToErr(err: anyerror) !void {
419421test "peer resolve array and const slice" {
420422 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
421423 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
422 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
423424 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
424425
425426 try testPeerResolveArrayConstSlice(true);
......@@ -818,7 +819,6 @@ test "peer type resolution: error union after non-error" {
818819test "peer cast *[0]T to E![]const T" {
819820 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
820821 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
821 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
822822 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
823823
824824 var buffer: [5]u8 = "abcde".*;
......@@ -833,7 +833,6 @@ test "peer cast *[0]T to E![]const T" {
833833test "peer cast *[0]T to []const T" {
834834 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
835835 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
836 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
837836 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
838837
839838 var buffer: [5]u8 = "abcde".*;
......@@ -855,7 +854,6 @@ test "peer cast *[N]T to [*]T" {
855854test "peer resolution of string literals" {
856855 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
857856 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
858 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
859857 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
860858
861859 const S = struct {
......@@ -1360,7 +1358,6 @@ test "cast f128 to narrower types" {
13601358test "peer type resolution: unreachable, null, slice" {
13611359 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
13621360 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1363 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
13641361 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
13651362
13661363 const S = struct {
test/behavior/enum.zig+1
......@@ -904,6 +904,7 @@ test "enum literal casting to tagged union" {
904904 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
905905 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
906906 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
907 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
907908
908909 const Arch = union(enum) {
909910 x86_64,
test/behavior/if.zig+1
......@@ -115,6 +115,7 @@ test "if peer expressions inferred optional type" {
115115 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
116116 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
117117 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
118 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
118119
119120 var self: []const u8 = "abcdef";
120121 var index: usize = 0;
test/behavior/switch.zig-1
......@@ -509,7 +509,6 @@ test "return result loc and then switch with range implicit casted to error unio
509509}
510510
511511test "switch with null and T peer types and inferred result location type" {
512 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
513512 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
514513 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
515514 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO