| ... | ... | @@ -105,7 +105,7 @@ frame_allocs: std.MultiArrayList(FrameAlloc) = .{}, |
| 105 | 105 | free_frame_indices: std.AutoArrayHashMapUnmanaged(FrameIndex, void) = .{}, |
| 106 | 106 | frame_locs: std.MultiArrayList(Mir.FrameLoc) = .{}, |
| 107 | 107 | |
| 108 | | loop_repeat_info: std.AutoHashMapUnmanaged(Air.Inst.Index, struct { |
| 108 | loops: std.AutoHashMapUnmanaged(Air.Inst.Index, struct { |
| 109 | 109 | /// The state to restore before branching. |
| 110 | 110 | state: State, |
| 111 | 111 | /// The branch target. |
| ... | ... | @@ -219,6 +219,38 @@ pub const MCValue = union(enum) { |
| 219 | 219 | reserved_frame: FrameIndex, |
| 220 | 220 | air_ref: Air.Inst.Ref, |
| 221 | 221 | |
| 222 | fn isModifiable(mcv: MCValue) bool { |
| 223 | return switch (mcv) { |
| 224 | .none, |
| 225 | .unreach, |
| 226 | .dead, |
| 227 | .undef, |
| 228 | .immediate, |
| 229 | .register_offset, |
| 230 | .eflags, |
| 231 | .register_overflow, |
| 232 | .lea_symbol, |
| 233 | .lea_direct, |
| 234 | .lea_got, |
| 235 | .lea_tlv, |
| 236 | .lea_frame, |
| 237 | .elementwise_regs_then_frame, |
| 238 | .reserved_frame, |
| 239 | .air_ref, |
| 240 | => false, |
| 241 | .register, |
| 242 | .register_pair, |
| 243 | .memory, |
| 244 | .load_symbol, |
| 245 | .load_got, |
| 246 | .load_direct, |
| 247 | .load_tlv, |
| 248 | .indirect, |
| 249 | => true, |
| 250 | .load_frame => |frame_addr| !frame_addr.index.isNamed(), |
| 251 | }; |
| 252 | } |
| 253 | |
| 222 | 254 | fn isMemory(mcv: MCValue) bool { |
| 223 | 255 | return switch (mcv) { |
| 224 | 256 | .memory, .indirect, .load_frame => true, |
| ... | ... | @@ -822,7 +854,7 @@ pub fn generate( |
| 822 | 854 | function.frame_allocs.deinit(gpa); |
| 823 | 855 | function.free_frame_indices.deinit(gpa); |
| 824 | 856 | function.frame_locs.deinit(gpa); |
| 825 | | function.loop_repeat_info.deinit(gpa); |
| 857 | function.loops.deinit(gpa); |
| 826 | 858 | var block_it = function.blocks.valueIterator(); |
| 827 | 859 | while (block_it.next()) |block| block.deinit(gpa); |
| 828 | 860 | function.blocks.deinit(gpa); |
| ... | ... | @@ -2156,18 +2188,20 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { |
| 2156 | 2188 | const air_tags = self.air.instructions.items(.tag); |
| 2157 | 2189 | |
| 2158 | 2190 | self.arg_index = 0; |
| 2159 | | for (body) |inst| { |
| 2160 | | wip_mir_log.debug("{}", .{self.fmtAir(inst)}); |
| 2161 | | verbose_tracking_log.debug("{}", .{self.fmtTracking()}); |
| 2191 | for (body) |inst| switch (air_tags[@intFromEnum(inst)]) { |
| 2192 | .arg => { |
| 2193 | wip_mir_log.debug("{}", .{self.fmtAir(inst)}); |
| 2194 | verbose_tracking_log.debug("{}", .{self.fmtTracking()}); |
| 2162 | 2195 | |
| 2163 | | const old_air_bookkeeping = self.air_bookkeeping; |
| 2164 | | try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1); |
| 2165 | | switch (air_tags[@intFromEnum(inst)]) { |
| 2166 | | .arg => try self.airArg(inst), |
| 2167 | | else => break, |
| 2168 | | } |
| 2169 | | self.checkInvariantsAfterAirInst(inst, old_air_bookkeeping); |
| 2170 | | } |
| 2196 | const old_air_bookkeeping = self.air_bookkeeping; |
| 2197 | try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1); |
| 2198 | |
| 2199 | try self.airArg(inst); |
| 2200 | |
| 2201 | self.checkInvariantsAfterAirInst(inst, old_air_bookkeeping); |
| 2202 | }, |
| 2203 | else => break, |
| 2204 | }; |
| 2171 | 2205 | |
| 2172 | 2206 | if (self.arg_index == 0) try self.airDbgVarArgs(); |
| 2173 | 2207 | self.arg_index = 0; |
| ... | ... | @@ -2256,7 +2290,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { |
| 2256 | 2290 | .block => try self.airBlock(inst), |
| 2257 | 2291 | .br => try self.airBr(inst), |
| 2258 | 2292 | .repeat => try self.airRepeat(inst), |
| 2259 | | .switch_dispatch => return self.fail("TODO implement `switch_dispatch`", .{}), |
| 2293 | .switch_dispatch => try self.airSwitchDispatch(inst), |
| 2260 | 2294 | .trap => try self.airTrap(), |
| 2261 | 2295 | .breakpoint => try self.airBreakpoint(), |
| 2262 | 2296 | .ret_addr => try self.airRetAddr(inst), |
| ... | ... | @@ -2345,7 +2379,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { |
| 2345 | 2379 | .field_parent_ptr => try self.airFieldParentPtr(inst), |
| 2346 | 2380 | |
| 2347 | 2381 | .switch_br => try self.airSwitchBr(inst), |
| 2348 | | .loop_switch_br => return self.fail("TODO implement `loop_switch_br`", .{}), |
| 2382 | .loop_switch_br => try self.airLoopSwitchBr(inst), |
| 2349 | 2383 | .slice_ptr => try self.airSlicePtr(inst), |
| 2350 | 2384 | .slice_len => try self.airSliceLen(inst), |
| 2351 | 2385 | |
| ... | ... | @@ -13637,14 +13671,13 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void { |
| 13637 | 13671 | self.scope_generation += 1; |
| 13638 | 13672 | const state = try self.saveState(); |
| 13639 | 13673 | |
| 13640 | | try self.loop_repeat_info.putNoClobber(self.gpa, inst, .{ |
| 13674 | try self.loops.putNoClobber(self.gpa, inst, .{ |
| 13641 | 13675 | .state = state, |
| 13642 | 13676 | .jmp_target = @intCast(self.mir_instructions.len), |
| 13643 | 13677 | }); |
| 13644 | | defer assert(self.loop_repeat_info.remove(inst)); |
| 13678 | defer assert(self.loops.remove(inst)); |
| 13645 | 13679 | |
| 13646 | 13680 | try self.genBody(body); |
| 13647 | | |
| 13648 | 13681 | self.finishAirBookkeeping(); |
| 13649 | 13682 | } |
| 13650 | 13683 | |
| ... | ... | @@ -13685,10 +13718,8 @@ fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) ! |
| 13685 | 13718 | self.finishAirBookkeeping(); |
| 13686 | 13719 | } |
| 13687 | 13720 | |
| 13688 | | fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void { |
| 13721 | fn lowerSwitchBr(self: *Self, inst: Air.Inst.Index, switch_br: Air.UnwrappedSwitch, condition: MCValue) !void { |
| 13689 | 13722 | const zcu = self.pt.zcu; |
| 13690 | | const switch_br = self.air.unwrapSwitch(inst); |
| 13691 | | const condition = try self.resolveInst(switch_br.operand); |
| 13692 | 13723 | const condition_ty = self.typeOf(switch_br.operand); |
| 13693 | 13724 | const liveness = try self.liveness.getSwitchBr(self.gpa, inst, switch_br.cases_len + 1); |
| 13694 | 13725 | defer self.gpa.free(liveness.deaths); |
| ... | ... | @@ -13699,13 +13730,6 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void { |
| 13699 | 13730 | else => unreachable, |
| 13700 | 13731 | }; |
| 13701 | 13732 | |
| 13702 | | // If the condition dies here in this switch instruction, process |
| 13703 | | // that death now instead of later as this has an effect on |
| 13704 | | // whether it needs to be spilled in the branches |
| 13705 | | if (self.liveness.operandDies(inst, 0)) { |
| 13706 | | if (switch_br.operand.toIndex()) |op_inst| try self.processDeath(op_inst); |
| 13707 | | } |
| 13708 | | |
| 13709 | 13733 | self.scope_generation += 1; |
| 13710 | 13734 | const state = try self.saveState(); |
| 13711 | 13735 | |
| ... | ... | @@ -13810,11 +13834,111 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void { |
| 13810 | 13834 | .close_scope = true, |
| 13811 | 13835 | }); |
| 13812 | 13836 | } |
| 13837 | } |
| 13838 | |
| 13839 | fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void { |
| 13840 | const switch_br = self.air.unwrapSwitch(inst); |
| 13841 | const condition = try self.resolveInst(switch_br.operand); |
| 13842 | |
| 13843 | // If the condition dies here in this switch instruction, process |
| 13844 | // that death now instead of later as this has an effect on |
| 13845 | // whether it needs to be spilled in the branches |
| 13846 | if (self.liveness.operandDies(inst, 0)) { |
| 13847 | if (switch_br.operand.toIndex()) |op_inst| try self.processDeath(op_inst); |
| 13848 | } |
| 13849 | |
| 13850 | try self.lowerSwitchBr(inst, switch_br, condition); |
| 13813 | 13851 | |
| 13814 | 13852 | // We already took care of pl_op.operand earlier, so there's nothing left to do |
| 13815 | 13853 | self.finishAirBookkeeping(); |
| 13816 | 13854 | } |
| 13817 | 13855 | |
| 13856 | fn airLoopSwitchBr(self: *Self, inst: Air.Inst.Index) !void { |
| 13857 | const switch_br = self.air.unwrapSwitch(inst); |
| 13858 | const condition = try self.resolveInst(switch_br.operand); |
| 13859 | |
| 13860 | const mat_cond = if (condition.isModifiable() and |
| 13861 | self.reuseOperand(inst, switch_br.operand, 0, condition)) |
| 13862 | condition |
| 13863 | else mat_cond: { |
| 13864 | const mat_cond = try self.allocRegOrMem(inst, true); |
| 13865 | try self.genCopy(self.typeOf(switch_br.operand), mat_cond, condition, .{}); |
| 13866 | break :mat_cond mat_cond; |
| 13867 | }; |
| 13868 | self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(mat_cond)); |
| 13869 | |
| 13870 | // If the condition dies here in this switch instruction, process |
| 13871 | // that death now instead of later as this has an effect on |
| 13872 | // whether it needs to be spilled in the branches |
| 13873 | if (self.liveness.operandDies(inst, 0)) { |
| 13874 | if (switch_br.operand.toIndex()) |op_inst| try self.processDeath(op_inst); |
| 13875 | } |
| 13876 | |
| 13877 | self.scope_generation += 1; |
| 13878 | const state = try self.saveState(); |
| 13879 | |
| 13880 | try self.loops.putNoClobber(self.gpa, inst, .{ |
| 13881 | .state = state, |
| 13882 | .jmp_target = @intCast(self.mir_instructions.len), |
| 13883 | }); |
| 13884 | defer assert(self.loops.remove(inst)); |
| 13885 | |
| 13886 | // Stop tracking block result without forgetting tracking info |
| 13887 | try self.freeValue(mat_cond); |
| 13888 | |
| 13889 | try self.lowerSwitchBr(inst, switch_br, mat_cond); |
| 13890 | |
| 13891 | try self.processDeath(inst); |
| 13892 | self.finishAirBookkeeping(); |
| 13893 | } |
| 13894 | |
| 13895 | fn airSwitchDispatch(self: *Self, inst: Air.Inst.Index) !void { |
| 13896 | const br = self.air.instructions.items(.data)[@intFromEnum(inst)].br; |
| 13897 | |
| 13898 | const block_ty = self.typeOfIndex(br.block_inst); |
| 13899 | const block_tracking = self.inst_tracking.getPtr(br.block_inst).?; |
| 13900 | const loop_data = self.loops.getPtr(br.block_inst).?; |
| 13901 | done: { |
| 13902 | try self.getValue(block_tracking.short, null); |
| 13903 | const src_mcv = try self.resolveInst(br.operand); |
| 13904 | |
| 13905 | if (self.reuseOperandAdvanced(inst, br.operand, 0, src_mcv, br.block_inst)) { |
| 13906 | try self.getValue(block_tracking.short, br.block_inst); |
| 13907 | // .long = .none to avoid merging operand and block result stack frames. |
| 13908 | const current_tracking: InstTracking = .{ .long = .none, .short = src_mcv }; |
| 13909 | try current_tracking.materializeUnsafe(self, br.block_inst, block_tracking.*); |
| 13910 | for (current_tracking.getRegs()) |src_reg| self.register_manager.freeReg(src_reg); |
| 13911 | break :done; |
| 13912 | } |
| 13913 | |
| 13914 | try self.getValue(block_tracking.short, br.block_inst); |
| 13915 | const dst_mcv = block_tracking.short; |
| 13916 | try self.genCopy(block_ty, dst_mcv, try self.resolveInst(br.operand), .{}); |
| 13917 | break :done; |
| 13918 | } |
| 13919 | |
| 13920 | // Process operand death so that it is properly accounted for in the State below. |
| 13921 | if (self.liveness.operandDies(inst, 0)) { |
| 13922 | if (br.operand.toIndex()) |op_inst| try self.processDeath(op_inst); |
| 13923 | } |
| 13924 | |
| 13925 | try self.restoreState(loop_data.state, &.{}, .{ |
| 13926 | .emit_instructions = true, |
| 13927 | .update_tracking = false, |
| 13928 | .resurrect = false, |
| 13929 | .close_scope = false, |
| 13930 | }); |
| 13931 | |
| 13932 | // Emit a jump with a relocation. It will be patched up after the block ends. |
| 13933 | // Leave the jump offset undefined |
| 13934 | _ = try self.asmJmpReloc(loop_data.jmp_target); |
| 13935 | |
| 13936 | // Stop tracking block result without forgetting tracking info |
| 13937 | try self.freeValue(block_tracking.short); |
| 13938 | |
| 13939 | self.finishAirBookkeeping(); |
| 13940 | } |
| 13941 | |
| 13818 | 13942 | fn performReloc(self: *Self, reloc: Mir.Inst.Index) void { |
| 13819 | 13943 | const next_inst: u32 = @intCast(self.mir_instructions.len); |
| 13820 | 13944 | switch (self.mir_instructions.items(.tag)[reloc]) { |
| ... | ... | @@ -13891,7 +14015,7 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void { |
| 13891 | 14015 | |
| 13892 | 14016 | fn airRepeat(self: *Self, inst: Air.Inst.Index) !void { |
| 13893 | 14017 | const loop_inst = self.air.instructions.items(.data)[@intFromEnum(inst)].repeat.loop_inst; |
| 13894 | | const repeat_info = self.loop_repeat_info.get(loop_inst).?; |
| 14018 | const repeat_info = self.loops.get(loop_inst).?; |
| 13895 | 14019 | try self.restoreState(repeat_info.state, &.{}, .{ |
| 13896 | 14020 | .emit_instructions = true, |
| 13897 | 14021 | .update_tracking = false, |
| ... | ... | @@ -19578,7 +19702,10 @@ fn typeOf(self: *Self, inst: Air.Inst.Ref) Type { |
| 19578 | 19702 | fn typeOfIndex(self: *Self, inst: Air.Inst.Index) Type { |
| 19579 | 19703 | const pt = self.pt; |
| 19580 | 19704 | const zcu = pt.zcu; |
| 19581 | | return self.air.typeOfIndex(inst, &zcu.intern_pool); |
| 19705 | return switch (self.air.instructions.items(.tag)[@intFromEnum(inst)]) { |
| 19706 | .loop_switch_br => self.typeOf(self.air.unwrapSwitch(inst).operand), |
| 19707 | else => self.air.typeOfIndex(inst, &zcu.intern_pool), |
| 19708 | }; |
| 19582 | 19709 | } |
| 19583 | 19710 | |
| 19584 | 19711 | fn intCompilerRtAbiName(int_bits: u32) u8 { |
Jacob Young <jacobly@ziglang.org> 2024-08-31 22:18:30-04:00
Matthew Lugg <mlugg@mlugg.co.uk> 2024-09-01 18:31:01+01:00