authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-04-10 03:29:32-04:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2023-04-20 20:49:35+01:00
log07d57623b30991ac00160a0ac0e3c44d85c73931
tree7ab17d800e48fc6554e756240bebde8699f804cd
parentf18ee1e2a2d4fb7ece2741efbfe4bda8b056068e
signature Commit is signed but in an unrecognized format.

x86_64: instruction tracking cleanup

1 files changed, 109 insertions(+), 84 deletions(-)

src/arch/x86_64/CodeGen.zig+109-84
......@@ -337,6 +337,8 @@ pub fn generate(
337337 };
338338 defer {
339339 function.stack.deinit(gpa);
340 var block_it = function.blocks.valueIterator();
341 while (block_it.next()) |block| block.deinit(gpa);
340342 function.blocks.deinit(gpa);
341343 function.inst_tracking.deinit(gpa);
342344 function.const_tracking.deinit(gpa);
......@@ -1184,7 +1186,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
11841186 var it = self.register_manager.free_registers.iterator(.{ .kind = .unset });
11851187 while (it.next()) |index| {
11861188 const tracked_inst = self.register_manager.registers[index];
1187 const tracking = self.getResolvedInstValue(tracked_inst).?;
1189 const tracking = self.getResolvedInstValue(tracked_inst);
11881190 assert(RegisterManager.indexOfRegIntoTracked(switch (tracking.short) {
11891191 .register => |reg| reg,
11901192 .register_overflow => |ro| ro.reg,
......@@ -1232,7 +1234,7 @@ fn processDeath(self: *Self, inst: Air.Inst.Index) void {
12321234 const air_tags = self.air.instructions.items(.tag);
12331235 if (air_tags[inst] == .constant) return;
12341236 log.debug("%{d} => {}", .{ inst, MCValue.dead });
1235 if (self.getResolvedInstValue(inst)) |tracking| tracking.die(self);
1237 self.inst_tracking.getPtr(inst).?.die(self);
12361238}
12371239
12381240/// Called when there are no operands, and the instruction is always unreferenced.
......@@ -1378,7 +1380,7 @@ fn saveState(self: *Self) !State {
13781380 return state;
13791381}
13801382
1381fn restoreState(self: *Self, state: State, comptime opts: struct {
1383fn restoreState(self: *Self, state: State, deaths: []u32, comptime opts: struct {
13821384 emit_instructions: bool,
13831385 update_tracking: bool,
13841386 resurrect: bool,
......@@ -1389,8 +1391,16 @@ fn restoreState(self: *Self, state: State, comptime opts: struct {
13891391 self.inst_tracking.shrinkRetainingCapacity(state.inst_tracking_len);
13901392 }
13911393
1392 if (opts.resurrect) for (self.inst_tracking.values()) |*tracking|
1393 tracking.resurrect(state.scope_generation);
1394 if (opts.resurrect) {
1395 var death_i: usize = 0;
1396 for (self.inst_tracking.values()[0..state.inst_tracking_len], 0..) |*tracking, tracking_i| {
1397 if (death_i < deaths.len and deaths[death_i] == tracking_i) {
1398 // oops, it was actually a death instead
1399 death_i += 1;
1400 tracking.die(self);
1401 } else tracking.resurrect(state.scope_generation);
1402 }
1403 } else assert(deaths.len == 0);
13941404
13951405 for (0..state.registers.len) |index| {
13961406 const current_maybe_inst = if (self.register_manager.free_registers.isSet(index))
......@@ -3616,7 +3626,7 @@ fn reuseOperand(
36163626
36173627 // Prevent the operand deaths processing code from deallocating it.
36183628 self.liveness.clearOperandDeath(inst, op_index);
3619 if (self.getResolvedInstValue(Air.refToIndex(operand).?)) |tracking| tracking.reuse(self);
3629 self.getResolvedInstValue(Air.refToIndex(operand).?).reuse(self);
36203630
36213631 return true;
36223632}
......@@ -6009,9 +6019,8 @@ fn airTry(self: *Self, inst: Air.Inst.Index) !void {
60096019 const extra = self.air.extraData(Air.Try, pl_op.payload);
60106020 const body = self.air.extra[extra.end..][0..extra.data.body_len];
60116021 const err_union_ty = self.air.typeOf(pl_op.operand);
6012 const err_union = try self.resolveInst(pl_op.operand);
6013 const result = try self.genTry(inst, err_union, body, err_union_ty, false);
6014 return self.finishAir(inst, result, .{ pl_op.operand, .none, .none });
6022 const result = try self.genTry(inst, pl_op.operand, body, err_union_ty, false);
6023 return self.finishAir(inst, result, .{ .none, .none, .none });
60156024}
60166025
60176026fn airTryPtr(self: *Self, inst: Air.Inst.Index) !void {
......@@ -6019,15 +6028,14 @@ fn airTryPtr(self: *Self, inst: Air.Inst.Index) !void {
60196028 const extra = self.air.extraData(Air.TryPtr, ty_pl.payload);
60206029 const body = self.air.extra[extra.end..][0..extra.data.body_len];
60216030 const err_union_ty = self.air.typeOf(extra.data.ptr).childType();
6022 const err_union_ptr = try self.resolveInst(extra.data.ptr);
6023 const result = try self.genTry(inst, err_union_ptr, body, err_union_ty, true);
6024 return self.finishAir(inst, result, .{ extra.data.ptr, .none, .none });
6031 const result = try self.genTry(inst, extra.data.ptr, body, err_union_ty, true);
6032 return self.finishAir(inst, result, .{ .none, .none, .none });
60256033}
60266034
60276035fn genTry(
60286036 self: *Self,
60296037 inst: Air.Inst.Index,
6030 err_union: MCValue,
6038 err_union: Air.Inst.Ref,
60316039 body: []const Air.Inst.Index,
60326040 err_union_ty: Type,
60336041 operand_is_ptr: bool,
......@@ -6035,14 +6043,37 @@ fn genTry(
60356043 if (operand_is_ptr) {
60366044 return self.fail("TODO genTry for pointers", .{});
60376045 }
6038 const is_err_mcv = try self.isErr(null, err_union_ty, err_union);
6046 const liveness_cond_br = self.liveness.getCondBr(inst);
6047
6048 const err_union_mcv = try self.resolveInst(err_union);
6049 const is_err_mcv = try self.isErr(null, err_union_ty, err_union_mcv);
6050
60396051 const reloc = try self.genCondBrMir(Type.anyerror, is_err_mcv);
6052
6053 if (self.liveness.operandDies(inst, 0)) {
6054 if (Air.refToIndex(err_union)) |err_union_inst| self.processDeath(err_union_inst);
6055 }
6056
6057 self.scope_generation += 1;
6058 const state = try self.saveState();
6059
6060 for (liveness_cond_br.else_deaths) |operand| self.processDeath(operand);
60406061 try self.genBody(body);
6062 try self.restoreState(state, &.{}, .{
6063 .emit_instructions = false,
6064 .update_tracking = true,
6065 .resurrect = true,
6066 .close_scope = true,
6067 });
6068
60416069 try self.performReloc(reloc);
6070
6071 for (liveness_cond_br.then_deaths) |operand| self.processDeath(operand);
6072
60426073 const result = if (self.liveness.isUnused(inst))
60436074 .unreach
60446075 else
6045 try self.genUnwrapErrorUnionPayloadMir(inst, err_union_ty, err_union);
6076 try self.genUnwrapErrorUnionPayloadMir(inst, err_union_ty, err_union_mcv);
60466077 return result;
60476078}
60486079
......@@ -6123,7 +6154,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
61236154 const extra = self.air.extraData(Air.CondBr, pl_op.payload);
61246155 const then_body = self.air.extra[extra.end..][0..extra.data.then_body_len];
61256156 const else_body = self.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len];
6126 const liveness_condbr = self.liveness.getCondBr(inst);
6157 const liveness_cond_br = self.liveness.getCondBr(inst);
61276158
61286159 const reloc = try self.genCondBrMir(cond_ty, cond);
61296160
......@@ -6139,9 +6170,9 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
61396170 self.scope_generation += 1;
61406171 const inner_state = try self.saveState();
61416172
6142 for (liveness_condbr.then_deaths) |operand| self.processDeath(operand);
6173 for (liveness_cond_br.then_deaths) |operand| self.processDeath(operand);
61436174 try self.genBody(then_body);
6144 try self.restoreState(inner_state, .{
6175 try self.restoreState(inner_state, &.{}, .{
61456176 .emit_instructions = false,
61466177 .update_tracking = true,
61476178 .resurrect = true,
......@@ -6150,16 +6181,16 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
61506181
61516182 try self.performReloc(reloc);
61526183
6153 for (liveness_condbr.else_deaths) |operand| self.processDeath(operand);
6184 for (liveness_cond_br.else_deaths) |operand| self.processDeath(operand);
61546185 try self.genBody(else_body);
6155 try self.restoreState(inner_state, .{
6186 try self.restoreState(inner_state, &.{}, .{
61566187 .emit_instructions = false,
61576188 .update_tracking = true,
61586189 .resurrect = true,
61596190 .close_scope = true,
61606191 });
61616192 }
6162 try self.restoreState(outer_state, .{
6193 try self.restoreState(outer_state, &.{}, .{
61636194 .emit_instructions = false,
61646195 .update_tracking = false,
61656196 .resurrect = false,
......@@ -6473,7 +6504,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
64736504 const state = try self.saveState();
64746505
64756506 try self.genBody(body);
6476 try self.restoreState(state, .{
6507 try self.restoreState(state, &.{}, .{
64776508 .emit_instructions = true,
64786509 .update_tracking = false,
64796510 .resurrect = false,
......@@ -6486,40 +6517,29 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
64866517
64876518fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
64886519 // A block is a setup to be able to jump to the end.
6489 const ty = self.air.typeOfIndex(inst);
6490
6491 // Here we use .{ .long = .unreach } to represent a null value so that the
6492 // first break instruction will choose a MCValue for the block result and
6493 // overwrite this field. Following break instructions will use that MCValue
6494 // to put their block results.
6495 self.inst_tracking.putAssumeCapacityNoClobber(inst, .{
6496 .long = .unreach,
6497 .short = if (ty.isNoReturn()) .unreach else .none,
6498 });
6520 self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(.unreach));
64996521
65006522 self.scope_generation += 1;
65016523 try self.blocks.putNoClobber(self.gpa, inst, .{ .state = self.initRetroactiveState() });
6502 defer {
6503 var block_data = self.blocks.fetchRemove(inst).?.value;
6504 block_data.deinit(self.gpa);
6505 }
65066524
65076525 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
65086526 const extra = self.air.extraData(Air.Block, ty_pl.payload);
65096527 const body = self.air.extra[extra.end..][0..extra.data.body_len];
65106528 try self.genBody(body);
65116529
6512 const tracking = self.inst_tracking.getPtr(inst).?;
6513 const block_data = self.blocks.getPtr(inst).?;
6514 for (block_data.deaths.items) |tracking_index| self.inst_tracking.values()[tracking_index].die(self);
6515 if (tracking.short != .unreach) try self.restoreState(block_data.state, .{
6516 .emit_instructions = false,
6517 .update_tracking = true,
6518 .resurrect = false,
6519 .close_scope = true,
6520 });
6521 for (block_data.relocs.items) |reloc| try self.performReloc(reloc);
6530 var block_data = self.blocks.fetchRemove(inst).?;
6531 defer block_data.value.deinit(self.gpa);
6532 if (block_data.value.relocs.items.len > 0) {
6533 try self.restoreState(block_data.value.state, block_data.value.deaths.items, .{
6534 .emit_instructions = false,
6535 .update_tracking = true,
6536 .resurrect = true,
6537 .close_scope = true,
6538 });
6539 for (block_data.value.relocs.items) |reloc| try self.performReloc(reloc);
6540 }
65226541
6542 const tracking = self.inst_tracking.getPtr(inst).?;
65236543 if (self.liveness.isUnused(inst)) tracking.die(self);
65246544 self.getValue(tracking.short, inst);
65256545 self.finishAirBookkeeping();
......@@ -6569,7 +6589,7 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
65696589 for (liveness.deaths[case_i]) |operand| self.processDeath(operand);
65706590
65716591 try self.genBody(case_body);
6572 try self.restoreState(inner_state, .{
6592 try self.restoreState(inner_state, &.{}, .{
65736593 .emit_instructions = false,
65746594 .update_tracking = true,
65756595 .resurrect = true,
......@@ -6586,7 +6606,7 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
65866606 for (liveness.deaths[else_deaths]) |operand| self.processDeath(operand);
65876607
65886608 try self.genBody(else_body);
6589 try self.restoreState(inner_state, .{
6609 try self.restoreState(inner_state, &.{}, .{
65906610 .emit_instructions = false,
65916611 .update_tracking = true,
65926612 .resurrect = true,
......@@ -6594,7 +6614,7 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
65946614 });
65956615 }
65966616 }
6597 try self.restoreState(outer_state, .{
6617 try self.restoreState(outer_state, &.{}, .{
65986618 .emit_instructions = false,
65996619 .update_tracking = false,
66006620 .resurrect = false,
......@@ -6620,58 +6640,64 @@ fn performReloc(self: *Self, reloc: Mir.Inst.Index) !void {
66206640
66216641fn airBr(self: *Self, inst: Air.Inst.Index) !void {
66226642 const br = self.air.instructions.items(.data)[inst].br;
6643 const src_mcv = try self.resolveInst(br.operand);
6644
66236645 const block_ty = self.air.typeOfIndex(br.block_inst);
66246646 const block_unused =
66256647 !block_ty.hasRuntimeBitsIgnoreComptime() or self.liveness.isUnused(br.block_inst);
6626
6627 // Process operand death early so that it is properly accounted for in the State below.
6628 const src_mcv = try self.resolveInst(br.operand);
6629 if (self.liveness.operandDies(inst, 0)) {
6630 if (Air.refToIndex(br.operand)) |op_inst| self.processDeath(op_inst);
6631 }
6632
66336648 const block_tracking = self.inst_tracking.getPtr(br.block_inst).?;
66346649 const block_data = self.blocks.getPtr(br.block_inst).?;
6635 if (block_tracking.long == .unreach) {
6636 // .unreach is used to mean that we are the first branch
66376650
6651 if (block_data.relocs.items.len == 0) {
66386652 // We need to compute a list of deaths for later. This list needs to include
66396653 // instructions that was born before, and has died since, the target block.
6640 for (self.inst_tracking.values()[0..block_data.state.inst_tracking_len], 0..) |
6641 *tracking,
6642 tracked_index,
6643 | switch (tracking.short) {
6654 for (
6655 self.inst_tracking.values()[0..block_data.state.inst_tracking_len],
6656 0..,
6657 ) |*tracking, tracked_index| switch (tracking.short) {
66446658 .dead => |die_generation| if (die_generation >= block_data.state.scope_generation)
66456659 try block_data.deaths.append(self.gpa, @intCast(u32, tracked_index)),
66466660 else => {},
66476661 };
66486662
6649 const result = result: {
6663 block_tracking.* = InstTracking.init(result: {
66506664 if (block_unused) break :result .none;
6651 if (self.reuseOperand(inst, br.operand, 0, src_mcv)) break :result src_mcv;
6665 if (self.reuseOperand(inst, br.operand, 0, src_mcv)) {
6666 // Fix instruction tracking
6667 switch (src_mcv) {
6668 .register => |reg| if (RegisterManager.indexOfRegIntoTracked(reg)) |index| {
6669 self.register_manager.registers[index] = br.block_inst;
6670 },
6671 else => {},
6672 }
6673 break :result src_mcv;
6674 }
66526675
66536676 const new_mcv = try self.allocRegOrMem(br.block_inst, true);
66546677 try self.setRegOrMem(block_ty, new_mcv, src_mcv);
66556678 break :result new_mcv;
6656 };
6657 block_tracking.* = InstTracking.init(result);
6658 try self.saveRetroactiveState(&block_data.state);
6659 self.freeValue(result);
6660 } else {
6661 if (!block_unused) try self.setRegOrMem(block_ty, block_tracking.short, src_mcv);
6662 try self.restoreState(block_data.state, .{
6663 .emit_instructions = true,
6664 .update_tracking = false,
6665 .resurrect = false,
6666 .close_scope = false,
66676679 });
6680 } else if (!block_unused) try self.setRegOrMem(block_ty, block_tracking.short, src_mcv);
6681
6682 // Process operand death so that it is properly accounted for in the State below.
6683 if (self.liveness.operandDies(inst, 0)) {
6684 if (Air.refToIndex(br.operand)) |op_inst| self.processDeath(op_inst);
66686685 }
66696686
6687 if (block_data.relocs.items.len == 0) {
6688 try self.saveRetroactiveState(&block_data.state);
6689 block_tracking.die(self);
6690 } else try self.restoreState(block_data.state, &.{}, .{
6691 .emit_instructions = true,
6692 .update_tracking = false,
6693 .resurrect = false,
6694 .close_scope = false,
6695 });
6696
66706697 // Emit a jump with a relocation. It will be patched up after the block ends.
6671 try block_data.relocs.ensureUnusedCapacity(self.gpa, 1);
66726698 // Leave the jump offset undefined
66736699 const jmp_reloc = try self.asmJmpReloc(undefined);
6674 block_data.relocs.appendAssumeCapacity(jmp_reloc);
6700 try block_data.relocs.append(self.gpa, jmp_reloc);
66756701
66766702 self.finishAirBookkeeping();
66776703}
......@@ -8636,7 +8662,7 @@ fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {
86368662 else => self.inst_tracking.getPtr(inst).?,
86378663 }.short;
86388664 switch (mcv) {
8639 .none, .unreach => unreachable,
8665 .none, .unreach, .dead => unreachable,
86408666 else => return mcv,
86418667 }
86428668 }
......@@ -8644,15 +8670,14 @@ fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {
86448670 return self.genTypedValue(.{ .ty = ty, .val = self.air.value(ref).? });
86458671}
86468672
8647fn getResolvedInstValue(self: *Self, inst: Air.Inst.Index) ?*InstTracking {
8673fn getResolvedInstValue(self: *Self, inst: Air.Inst.Index) *InstTracking {
86488674 const tracking = switch (self.air.instructions.items(.tag)[inst]) {
8649 .constant => self.const_tracking.getPtr(inst) orelse return null,
8675 .constant => &self.const_tracking,
86508676 .const_ty => unreachable,
8651 else => self.inst_tracking.getPtr(inst).?,
8652 };
8677 else => &self.inst_tracking,
8678 }.getPtr(inst).?;
86538679 return switch (tracking.short) {
8654 .unreach => unreachable,
8655 .dead => null,
8680 .none, .unreach, .dead => unreachable,
86568681 else => tracking,
86578682 };
86588683}