authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-05-28 22:45:19-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-05-29 03:57:48-04:00
logc04be630d996a0e01fd3cf05e6cade006e4226ce
treeedd9d55ad05435b91bd3cb95208a76ead2001094
parentf25212a479c4f26566b6b851e245e49c6f508b96

Legalize: introduce a new pass before liveness

Each target can opt into different sets of legalize features. By performing these transformations before liveness, instructions that become unreferenced will have up-to-date liveness information.

36 files changed, 3225 insertions(+), 3086 deletions(-)

CMakeLists.txt+4-2
......@@ -512,13 +512,15 @@ set(ZIG_STAGE2_SOURCES
512512 lib/std/zig/llvm/bitcode_writer.zig
513513 lib/std/zig/llvm/ir.zig
514514 src/Air.zig
515 src/Air/Legalize.zig
516 src/Air/Liveness.zig
517 src/Air/Liveness/Verify.zig
518 src/Air/types_resolved.zig
515519 src/Builtin.zig
516520 src/Compilation.zig
517521 src/Compilation/Config.zig
518522 src/DarwinPosixSpawn.zig
519523 src/InternPool.zig
520 src/Liveness.zig
521 src/Liveness/Verify.zig
522524 src/Package.zig
523525 src/Package/Fetch.zig
524526 src/Package/Fetch/git.zig
src/Air.zig+24-15
......@@ -9,16 +9,19 @@ const builtin = @import("builtin");
99const assert = std.debug.assert;
1010
1111const Air = @This();
12const Value = @import("Value.zig");
13const Type = @import("Type.zig");
1412const InternPool = @import("InternPool.zig");
13const Type = @import("Type.zig");
14const Value = @import("Value.zig");
1515const Zcu = @import("Zcu.zig");
1616const types_resolved = @import("Air/types_resolved.zig");
1717
18pub const Legalize = @import("Air/Legalize.zig");
19pub const Liveness = @import("Air/Liveness.zig");
20
1821instructions: std.MultiArrayList(Inst).Slice,
1922/// The meaning of this data is determined by `Inst.Tag` value.
2023/// The first few indexes are reserved. See `ExtraIndex` for the values.
21extra: []const u32,
24extra: std.ArrayListUnmanaged(u32),
2225
2326pub const ExtraIndex = enum(u32) {
2427 /// Payload index of the main `Block` in the `extra` array.
......@@ -244,22 +247,27 @@ pub const Inst = struct {
244247 /// Uses the `bin_op` field.
245248 bit_or,
246249 /// Shift right. `>>`
250 /// The rhs type may be a scalar version of the lhs type.
247251 /// Uses the `bin_op` field.
248252 shr,
249253 /// Shift right. The shift produces a poison value if it shifts out any non-zero bits.
254 /// The rhs type may be a scalar version of the lhs type.
250255 /// Uses the `bin_op` field.
251256 shr_exact,
252257 /// Shift left. `<<`
258 /// The rhs type may be a scalar version of the lhs type.
253259 /// Uses the `bin_op` field.
254260 shl,
255261 /// Shift left; For unsigned integers, the shift produces a poison value if it shifts
256262 /// out any non-zero bits. For signed integers, the shift produces a poison value if
257263 /// it shifts out any bits that disagree with the resultant sign bit.
264 /// The rhs type may be a scalar version of the lhs type.
258265 /// Uses the `bin_op` field.
259266 shl_exact,
260267 /// Saturating integer shift left. `<<|`. The result is the same type as the `lhs`.
261268 /// The `rhs` must have the same vector shape as the `lhs`, but with any unsigned
262269 /// integer as the scalar type.
270 /// The rhs type may be a scalar version of the lhs type.
263271 /// Uses the `bin_op` field.
264272 shl_sat,
265273 /// Bitwise XOR. `^`
......@@ -1378,9 +1386,9 @@ pub const UnionInit = struct {
13781386};
13791387
13801388pub fn getMainBody(air: Air) []const Air.Inst.Index {
1381 const body_index = air.extra[@intFromEnum(ExtraIndex.main_block)];
1389 const body_index = air.extra.items[@intFromEnum(ExtraIndex.main_block)];
13821390 const extra = air.extraData(Block, body_index);
1383 return @ptrCast(air.extra[extra.end..][0..extra.data.body_len]);
1391 return @ptrCast(air.extra.items[extra.end..][0..extra.data.body_len]);
13841392}
13851393
13861394pub fn typeOf(air: *const Air, inst: Air.Inst.Ref, ip: *const InternPool) Type {
......@@ -1656,9 +1664,9 @@ pub fn extraData(air: Air, comptime T: type, index: usize) struct { data: T, end
16561664 var result: T = undefined;
16571665 inline for (fields) |field| {
16581666 @field(result, field.name) = switch (field.type) {
1659 u32 => air.extra[i],
1660 InternPool.Index, Inst.Ref => @enumFromInt(air.extra[i]),
1661 i32, CondBr.BranchHints => @bitCast(air.extra[i]),
1667 u32 => air.extra.items[i],
1668 InternPool.Index, Inst.Ref => @enumFromInt(air.extra.items[i]),
1669 i32, CondBr.BranchHints => @bitCast(air.extra.items[i]),
16621670 else => @compileError("bad field type: " ++ @typeName(field.type)),
16631671 };
16641672 i += 1;
......@@ -1671,7 +1679,7 @@ pub fn extraData(air: Air, comptime T: type, index: usize) struct { data: T, end
16711679
16721680pub fn deinit(air: *Air, gpa: std.mem.Allocator) void {
16731681 air.instructions.deinit(gpa);
1674 gpa.free(air.extra);
1682 air.extra.deinit(gpa);
16751683 air.* = undefined;
16761684}
16771685
......@@ -1700,7 +1708,7 @@ pub const NullTerminatedString = enum(u32) {
17001708
17011709 pub fn toSlice(nts: NullTerminatedString, air: Air) [:0]const u8 {
17021710 if (nts == .none) return "";
1703 const bytes = std.mem.sliceAsBytes(air.extra[@intFromEnum(nts)..]);
1711 const bytes = std.mem.sliceAsBytes(air.extra.items[@intFromEnum(nts)..]);
17041712 return bytes[0..std.mem.indexOfScalar(u8, bytes, 0).? :0];
17051713 }
17061714};
......@@ -1943,7 +1951,7 @@ pub const UnwrappedSwitch = struct {
19431951 return us.getHintInner(us.cases_len);
19441952 }
19451953 fn getHintInner(us: UnwrappedSwitch, idx: u32) std.builtin.BranchHint {
1946 const bag = us.air.extra[us.branch_hints_start..][idx / 10];
1954 const bag = us.air.extra.items[us.branch_hints_start..][idx / 10];
19471955 const bits: u3 = @truncate(bag >> @intCast(3 * (idx % 10)));
19481956 return @enumFromInt(bits);
19491957 }
......@@ -1971,13 +1979,13 @@ pub const UnwrappedSwitch = struct {
19711979
19721980 const extra = it.air.extraData(SwitchBr.Case, it.extra_index);
19731981 var extra_index = extra.end;
1974 const items: []const Inst.Ref = @ptrCast(it.air.extra[extra_index..][0..extra.data.items_len]);
1982 const items: []const Inst.Ref = @ptrCast(it.air.extra.items[extra_index..][0..extra.data.items_len]);
19751983 extra_index += items.len;
19761984 // TODO: ptrcast from []const Inst.Ref to []const [2]Inst.Ref when supported
1977 const ranges_ptr: [*]const [2]Inst.Ref = @ptrCast(it.air.extra[extra_index..]);
1985 const ranges_ptr: [*]const [2]Inst.Ref = @ptrCast(it.air.extra.items[extra_index..]);
19781986 const ranges: []const [2]Inst.Ref = ranges_ptr[0..extra.data.ranges_len];
19791987 extra_index += ranges.len * 2;
1980 const body: []const Inst.Index = @ptrCast(it.air.extra[extra_index..][0..extra.data.body_len]);
1988 const body: []const Inst.Index = @ptrCast(it.air.extra.items[extra_index..][0..extra.data.body_len]);
19811989 extra_index += body.len;
19821990 it.extra_index = @intCast(extra_index);
19831991
......@@ -1992,7 +2000,7 @@ pub const UnwrappedSwitch = struct {
19922000 /// Returns the body of the "default" (`else`) case.
19932001 pub fn elseBody(it: *CaseIterator) []const Inst.Index {
19942002 assert(it.next_case == it.cases_len);
1995 return @ptrCast(it.air.extra[it.extra_index..][0..it.else_body_len]);
2003 return @ptrCast(it.air.extra.items[it.extra_index..][0..it.else_body_len]);
19962004 }
19972005 pub const Case = struct {
19982006 idx: u32,
......@@ -2025,6 +2033,7 @@ pub fn unwrapSwitch(air: *const Air, switch_inst: Inst.Index) UnwrappedSwitch {
20252033pub const typesFullyResolved = types_resolved.typesFullyResolved;
20262034pub const typeFullyResolved = types_resolved.checkType;
20272035pub const valFullyResolved = types_resolved.checkVal;
2036pub const legalize = Legalize.legalize;
20282037
20292038pub const CoveragePoint = enum(u1) {
20302039 /// Indicates the block is not a place of interest corresponding to
src/Air/Legalize.zig created+147
......@@ -0,0 +1,147 @@
1zcu: *const Zcu,
2air: Air,
3features: std.enums.EnumSet(Feature),
4
5pub const Feature = enum {
6 /// Legalize (shift lhs, (splat rhs)) -> (shift lhs, rhs)
7 remove_shift_vector_rhs_splat,
8 /// Legalize reduce of a one element vector to a bitcast
9 reduce_one_elem_to_bitcast,
10};
11
12pub const Features = std.enums.EnumFieldStruct(Feature, bool, false);
13
14pub fn legalize(air: *Air, backend: std.builtin.CompilerBackend, zcu: *const Zcu) std.mem.Allocator.Error!void {
15 var l: Legalize = .{
16 .zcu = zcu,
17 .air = air.*,
18 .features = features: switch (backend) {
19 .other, .stage1 => unreachable,
20 inline .stage2_llvm,
21 .stage2_c,
22 .stage2_wasm,
23 .stage2_arm,
24 .stage2_x86_64,
25 .stage2_aarch64,
26 .stage2_x86,
27 .stage2_riscv64,
28 .stage2_sparc64,
29 .stage2_spirv64,
30 .stage2_powerpc,
31 => |ct_backend| {
32 const Backend = codegen.importBackend(ct_backend) orelse break :features .initEmpty();
33 break :features if (@hasDecl(Backend, "legalize_features"))
34 .init(Backend.legalize_features)
35 else
36 .initEmpty();
37 },
38 _ => unreachable,
39 },
40 };
41 defer air.* = l.air;
42 if (!l.features.bits.eql(.initEmpty())) try l.legalizeBody(l.air.getMainBody());
43}
44
45fn legalizeBody(l: *Legalize, body: []const Air.Inst.Index) std.mem.Allocator.Error!void {
46 const zcu = l.zcu;
47 const ip = &zcu.intern_pool;
48 const tags = l.air.instructions.items(.tag);
49 const data = l.air.instructions.items(.data);
50 for (body) |inst| inst: switch (tags[@intFromEnum(inst)]) {
51 else => {},
52
53 .shl,
54 .shl_exact,
55 .shl_sat,
56 .shr,
57 .shr_exact,
58 => |air_tag| if (l.features.contains(.remove_shift_vector_rhs_splat)) done: {
59 const bin_op = data[@intFromEnum(inst)].bin_op;
60 const ty = l.air.typeOf(bin_op.rhs, ip);
61 if (!ty.isVector(zcu)) break :done;
62 if (bin_op.rhs.toInterned()) |rhs_ip_index| switch (ip.indexToKey(rhs_ip_index)) {
63 else => {},
64 .aggregate => |aggregate| switch (aggregate.storage) {
65 else => {},
66 .repeated_elem => |splat| continue :inst l.replaceInst(inst, air_tag, .{ .bin_op = .{
67 .lhs = bin_op.lhs,
68 .rhs = Air.internedToRef(splat),
69 } }),
70 },
71 } else {
72 const rhs_inst = bin_op.rhs.toIndex().?;
73 switch (tags[@intFromEnum(rhs_inst)]) {
74 else => {},
75 .splat => continue :inst l.replaceInst(inst, air_tag, .{ .bin_op = .{
76 .lhs = bin_op.lhs,
77 .rhs = data[@intFromEnum(rhs_inst)].ty_op.operand,
78 } }),
79 }
80 }
81 },
82
83 .reduce,
84 .reduce_optimized,
85 => if (l.features.contains(.reduce_one_elem_to_bitcast)) done: {
86 const reduce = data[@intFromEnum(inst)].reduce;
87 const vector_ty = l.air.typeOf(reduce.operand, ip);
88 switch (vector_ty.vectorLen(zcu)) {
89 0 => unreachable,
90 1 => continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{
91 .ty = Air.internedToRef(vector_ty.scalarType(zcu).toIntern()),
92 .operand = reduce.operand,
93 } }),
94 else => break :done,
95 }
96 },
97
98 .@"try", .try_cold => {
99 const pl_op = data[@intFromEnum(inst)].pl_op;
100 const extra = l.air.extraData(Air.Try, pl_op.payload);
101 try l.legalizeBody(@ptrCast(l.air.extra.items[extra.end..][0..extra.data.body_len]));
102 },
103 .try_ptr, .try_ptr_cold => {
104 const ty_pl = data[@intFromEnum(inst)].ty_pl;
105 const extra = l.air.extraData(Air.TryPtr, ty_pl.payload);
106 try l.legalizeBody(@ptrCast(l.air.extra.items[extra.end..][0..extra.data.body_len]));
107 },
108 .block, .loop => {
109 const ty_pl = data[@intFromEnum(inst)].ty_pl;
110 const extra = l.air.extraData(Air.Block, ty_pl.payload);
111 try l.legalizeBody(@ptrCast(l.air.extra.items[extra.end..][0..extra.data.body_len]));
112 },
113 .dbg_inline_block => {
114 const ty_pl = data[@intFromEnum(inst)].ty_pl;
115 const extra = l.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
116 try l.legalizeBody(@ptrCast(l.air.extra.items[extra.end..][0..extra.data.body_len]));
117 },
118 .cond_br => {
119 const pl_op = data[@intFromEnum(inst)].pl_op;
120 const extra = l.air.extraData(Air.CondBr, pl_op.payload);
121 try l.legalizeBody(@ptrCast(l.air.extra.items[extra.end..][0..extra.data.then_body_len]));
122 try l.legalizeBody(@ptrCast(l.air.extra.items[extra.end + extra.data.then_body_len ..][0..extra.data.else_body_len]));
123 },
124 .switch_br, .loop_switch_br => {
125 const switch_br = l.air.unwrapSwitch(inst);
126 var it = switch_br.iterateCases();
127 while (it.next()) |case| try l.legalizeBody(case.body);
128 try l.legalizeBody(it.elseBody());
129 },
130 };
131}
132
133// inline to propagate comptime `tag`s
134inline fn replaceInst(l: *Legalize, inst: Air.Inst.Index, tag: Air.Inst.Tag, data: Air.Inst.Data) Air.Inst.Tag {
135 const ip = &l.zcu.intern_pool;
136 const orig_ty = if (std.debug.runtime_safety) l.air.typeOfIndex(inst, ip) else {};
137 l.air.instructions.items(.tag)[@intFromEnum(inst)] = tag;
138 l.air.instructions.items(.data)[@intFromEnum(inst)] = data;
139 if (std.debug.runtime_safety) std.debug.assert(l.air.typeOfIndex(inst, ip).toIntern() == orig_ty.toIntern());
140 return tag;
141}
142
143const Air = @import("../Air.zig");
144const codegen = @import("../codegen.zig");
145const Legalize = @This();
146const std = @import("std");
147const Zcu = @import("../Zcu.zig");
src/Air/Liveness.zig created+2050
......@@ -0,0 +1,2050 @@
1//! For each AIR instruction, we want to know:
2//! * Is the instruction unreferenced (e.g. dies immediately)?
3//! * For each of its operands, does the operand die with this instruction (e.g. is
4//! this the last reference to it)?
5//! Some instructions are special, such as:
6//! * Conditional Branches
7//! * Switch Branches
8const std = @import("std");
9const log = std.log.scoped(.liveness);
10const assert = std.debug.assert;
11const Allocator = std.mem.Allocator;
12const Log2Int = std.math.Log2Int;
13
14const Liveness = @This();
15const trace = @import("../tracy.zig").trace;
16const Air = @import("../Air.zig");
17const InternPool = @import("../InternPool.zig");
18
19pub const Verify = @import("Liveness/Verify.zig");
20
21/// This array is split into sets of 4 bits per AIR instruction.
22/// The MSB (0bX000) is whether the instruction is unreferenced.
23/// The LSB (0b000X) is the first operand, and so on, up to 3 operands. A set bit means the
24/// operand dies after this instruction.
25/// Instructions which need more data to track liveness have special handling via the
26/// `special` table.
27tomb_bits: []usize,
28/// Sparse table of specially handled instructions. The value is an index into the `extra`
29/// array. The meaning of the data depends on the AIR tag.
30/// * `cond_br` - points to a `CondBr` in `extra` at this index.
31/// * `try`, `try_ptr` - points to a `CondBr` in `extra` at this index. The error path (the block
32/// in the instruction) is considered the "else" path, and the rest of the block the "then".
33/// * `switch_br` - points to a `SwitchBr` in `extra` at this index.
34/// * `loop_switch_br` - points to a `SwitchBr` in `extra` at this index.
35/// * `block` - points to a `Block` in `extra` at this index.
36/// * `asm`, `call`, `aggregate_init` - the value is a set of bits which are the extra tomb
37/// bits of operands.
38/// The main tomb bits are still used and the extra ones are starting with the lsb of the
39/// value here.
40special: std.AutoHashMapUnmanaged(Air.Inst.Index, u32),
41/// Auxiliary data. The way this data is interpreted is determined contextually.
42extra: []const u32,
43
44/// Trailing is the set of instructions whose lifetimes end at the start of the then branch,
45/// followed by the set of instructions whose lifetimes end at the start of the else branch.
46pub const CondBr = struct {
47 then_death_count: u32,
48 else_death_count: u32,
49};
50
51/// Trailing is:
52/// * For each case in the same order as in the AIR:
53/// - case_death_count: u32
54/// - Air.Inst.Index for each `case_death_count`: set of instructions whose lifetimes
55/// end at the start of this case.
56/// * Air.Inst.Index for each `else_death_count`: set of instructions whose lifetimes
57/// end at the start of the else case.
58pub const SwitchBr = struct {
59 else_death_count: u32,
60};
61
62/// Trailing is the set of instructions which die in the block. Note that these are not additional
63/// deaths (they are all recorded as normal within the block), but backends may use this information
64/// as a more efficient way to track which instructions are still alive after a block.
65pub const Block = struct {
66 death_count: u32,
67};
68
69/// Liveness analysis runs in several passes. Each pass iterates backwards over instructions in
70/// bodies, and recurses into bodies.
71const LivenessPass = enum {
72 /// In this pass, we perform some basic analysis of loops to gain information the main pass needs.
73 /// In particular, for every `loop` and `loop_switch_br`, we track the following information:
74 /// * Every outer block which the loop body contains a `br` to.
75 /// * Every outer loop which the loop body contains a `repeat` to.
76 /// * Every operand referenced within the loop body but created outside the loop.
77 /// This gives the main analysis pass enough information to determine the full set of
78 /// instructions which need to be alive when a loop repeats. This data is TEMPORARILY stored in
79 /// `a.extra`. It is not re-added to `extra` by the main pass, since it is not useful to
80 /// backends.
81 loop_analysis,
82
83 /// This pass performs the main liveness analysis, setting up tombs and extra data while
84 /// considering control flow etc.
85 main_analysis,
86};
87
88/// Each analysis pass may wish to pass data through calls. A pointer to a `LivenessPassData(pass)`
89/// stored on the stack is passed through calls to `analyzeInst` etc.
90fn LivenessPassData(comptime pass: LivenessPass) type {
91 return switch (pass) {
92 .loop_analysis => struct {
93 /// The set of blocks which are exited with a `br` instruction at some point within this
94 /// body and which we are currently within. Also includes `loop`s which are the target
95 /// of a `repeat` instruction, and `loop_switch_br`s which are the target of a
96 /// `switch_dispatch` instruction.
97 breaks: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .empty,
98
99 /// The set of operands for which we have seen at least one usage but not their birth.
100 live_set: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .empty,
101
102 fn deinit(self: *@This(), gpa: Allocator) void {
103 self.breaks.deinit(gpa);
104 self.live_set.deinit(gpa);
105 }
106 },
107
108 .main_analysis => struct {
109 /// Every `block` and `loop` currently under analysis.
110 block_scopes: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockScope) = .empty,
111
112 /// The set of instructions currently alive in the current control
113 /// flow branch.
114 live_set: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .empty,
115
116 /// The extra data initialized by the `loop_analysis` pass for this pass to consume.
117 /// Owned by this struct during this pass.
118 old_extra: std.ArrayListUnmanaged(u32) = .empty,
119
120 const BlockScope = struct {
121 /// If this is a `block`, these instructions are alive upon a `br` to this block.
122 /// If this is a `loop`, these instructions are alive upon a `repeat` to this block.
123 live_set: std.AutoHashMapUnmanaged(Air.Inst.Index, void),
124 };
125
126 fn deinit(self: *@This(), gpa: Allocator) void {
127 var it = self.block_scopes.valueIterator();
128 while (it.next()) |block| {
129 block.live_set.deinit(gpa);
130 }
131 self.block_scopes.deinit(gpa);
132 self.live_set.deinit(gpa);
133 self.old_extra.deinit(gpa);
134 }
135 },
136 };
137}
138
139pub fn analyze(gpa: Allocator, air: Air, intern_pool: *InternPool) Allocator.Error!Liveness {
140 const tracy = trace(@src());
141 defer tracy.end();
142
143 var a: Analysis = .{
144 .gpa = gpa,
145 .air = air,
146 .tomb_bits = try gpa.alloc(
147 usize,
148 (air.instructions.len * bpi + @bitSizeOf(usize) - 1) / @bitSizeOf(usize),
149 ),
150 .extra = .{},
151 .special = .{},
152 .intern_pool = intern_pool,
153 };
154 errdefer gpa.free(a.tomb_bits);
155 errdefer a.special.deinit(gpa);
156 defer a.extra.deinit(gpa);
157
158 @memset(a.tomb_bits, 0);
159
160 const main_body = air.getMainBody();
161
162 {
163 var data: LivenessPassData(.loop_analysis) = .{};
164 defer data.deinit(gpa);
165 try analyzeBody(&a, .loop_analysis, &data, main_body);
166 }
167
168 {
169 var data: LivenessPassData(.main_analysis) = .{};
170 defer data.deinit(gpa);
171 data.old_extra = a.extra;
172 a.extra = .{};
173 try analyzeBody(&a, .main_analysis, &data, main_body);
174 assert(data.live_set.count() == 0);
175 }
176
177 return .{
178 .tomb_bits = a.tomb_bits,
179 .special = a.special,
180 .extra = try a.extra.toOwnedSlice(gpa),
181 };
182}
183
184pub fn getTombBits(l: Liveness, inst: Air.Inst.Index) Bpi {
185 const usize_index = (@intFromEnum(inst) * bpi) / @bitSizeOf(usize);
186 return @as(Bpi, @truncate(l.tomb_bits[usize_index] >>
187 @as(Log2Int(usize), @intCast((@intFromEnum(inst) % (@bitSizeOf(usize) / bpi)) * bpi))));
188}
189
190pub fn isUnused(l: Liveness, inst: Air.Inst.Index) bool {
191 const usize_index = (@intFromEnum(inst) * bpi) / @bitSizeOf(usize);
192 const mask = @as(usize, 1) <<
193 @as(Log2Int(usize), @intCast((@intFromEnum(inst) % (@bitSizeOf(usize) / bpi)) * bpi + (bpi - 1)));
194 return (l.tomb_bits[usize_index] & mask) != 0;
195}
196
197pub fn operandDies(l: Liveness, inst: Air.Inst.Index, operand: OperandInt) bool {
198 assert(operand < bpi - 1);
199 const usize_index = (@intFromEnum(inst) * bpi) / @bitSizeOf(usize);
200 const mask = @as(usize, 1) <<
201 @as(Log2Int(usize), @intCast((@intFromEnum(inst) % (@bitSizeOf(usize) / bpi)) * bpi + operand));
202 return (l.tomb_bits[usize_index] & mask) != 0;
203}
204
205const OperandCategory = enum {
206 /// The operand lives on, but this instruction cannot possibly mutate memory.
207 none,
208 /// The operand lives on and this instruction can mutate memory.
209 write,
210 /// The operand dies at this instruction.
211 tomb,
212 /// The operand lives on, and this instruction is noreturn.
213 noret,
214 /// This instruction is too complicated for analysis, no information is available.
215 complex,
216};
217
218/// Given an instruction that we are examining, and an operand that we are looking for,
219/// returns a classification.
220pub fn categorizeOperand(
221 l: Liveness,
222 air: Air,
223 inst: Air.Inst.Index,
224 operand: Air.Inst.Index,
225 ip: *const InternPool,
226) OperandCategory {
227 const air_tags = air.instructions.items(.tag);
228 const air_datas = air.instructions.items(.data);
229 const operand_ref = operand.toRef();
230 switch (air_tags[@intFromEnum(inst)]) {
231 .add,
232 .add_safe,
233 .add_wrap,
234 .add_sat,
235 .add_optimized,
236 .sub,
237 .sub_safe,
238 .sub_wrap,
239 .sub_sat,
240 .sub_optimized,
241 .mul,
242 .mul_safe,
243 .mul_wrap,
244 .mul_sat,
245 .mul_optimized,
246 .div_float,
247 .div_trunc,
248 .div_floor,
249 .div_exact,
250 .rem,
251 .mod,
252 .bit_and,
253 .bit_or,
254 .xor,
255 .cmp_lt,
256 .cmp_lte,
257 .cmp_eq,
258 .cmp_gte,
259 .cmp_gt,
260 .cmp_neq,
261 .bool_and,
262 .bool_or,
263 .array_elem_val,
264 .slice_elem_val,
265 .ptr_elem_val,
266 .shl,
267 .shl_exact,
268 .shl_sat,
269 .shr,
270 .shr_exact,
271 .min,
272 .max,
273 .div_float_optimized,
274 .div_trunc_optimized,
275 .div_floor_optimized,
276 .div_exact_optimized,
277 .rem_optimized,
278 .mod_optimized,
279 .neg_optimized,
280 .cmp_lt_optimized,
281 .cmp_lte_optimized,
282 .cmp_eq_optimized,
283 .cmp_gte_optimized,
284 .cmp_gt_optimized,
285 .cmp_neq_optimized,
286 => {
287 const o = air_datas[@intFromEnum(inst)].bin_op;
288 if (o.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
289 if (o.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
290 return .none;
291 },
292
293 .store,
294 .store_safe,
295 .atomic_store_unordered,
296 .atomic_store_monotonic,
297 .atomic_store_release,
298 .atomic_store_seq_cst,
299 .set_union_tag,
300 .memset,
301 .memset_safe,
302 .memcpy,
303 .memmove,
304 => {
305 const o = air_datas[@intFromEnum(inst)].bin_op;
306 if (o.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
307 if (o.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .write);
308 return .write;
309 },
310
311 .vector_store_elem => {
312 const o = air_datas[@intFromEnum(inst)].vector_store_elem;
313 const extra = air.extraData(Air.Bin, o.payload).data;
314 if (o.vector_ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
315 if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
316 if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 2, .none);
317 return .write;
318 },
319
320 .arg,
321 .alloc,
322 .inferred_alloc,
323 .inferred_alloc_comptime,
324 .ret_ptr,
325 .trap,
326 .breakpoint,
327 .repeat,
328 .switch_dispatch,
329 .dbg_stmt,
330 .dbg_empty_stmt,
331 .unreach,
332 .ret_addr,
333 .frame_addr,
334 .wasm_memory_size,
335 .err_return_trace,
336 .save_err_return_trace_index,
337 .tlv_dllimport_ptr,
338 .c_va_start,
339 .work_item_id,
340 .work_group_size,
341 .work_group_id,
342 => return .none,
343
344 .not,
345 .bitcast,
346 .load,
347 .fpext,
348 .fptrunc,
349 .intcast,
350 .intcast_safe,
351 .trunc,
352 .optional_payload,
353 .optional_payload_ptr,
354 .wrap_optional,
355 .unwrap_errunion_payload,
356 .unwrap_errunion_err,
357 .unwrap_errunion_payload_ptr,
358 .unwrap_errunion_err_ptr,
359 .wrap_errunion_payload,
360 .wrap_errunion_err,
361 .slice_ptr,
362 .slice_len,
363 .ptr_slice_len_ptr,
364 .ptr_slice_ptr_ptr,
365 .struct_field_ptr_index_0,
366 .struct_field_ptr_index_1,
367 .struct_field_ptr_index_2,
368 .struct_field_ptr_index_3,
369 .array_to_slice,
370 .int_from_float,
371 .int_from_float_optimized,
372 .float_from_int,
373 .get_union_tag,
374 .clz,
375 .ctz,
376 .popcount,
377 .byte_swap,
378 .bit_reverse,
379 .splat,
380 .error_set_has_value,
381 .addrspace_cast,
382 .c_va_arg,
383 .c_va_copy,
384 .abs,
385 => {
386 const o = air_datas[@intFromEnum(inst)].ty_op;
387 if (o.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
388 return .none;
389 },
390
391 .optional_payload_ptr_set,
392 .errunion_payload_ptr_set,
393 => {
394 const o = air_datas[@intFromEnum(inst)].ty_op;
395 if (o.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
396 return .write;
397 },
398
399 .is_null,
400 .is_non_null,
401 .is_null_ptr,
402 .is_non_null_ptr,
403 .is_err,
404 .is_non_err,
405 .is_err_ptr,
406 .is_non_err_ptr,
407 .is_named_enum_value,
408 .tag_name,
409 .error_name,
410 .sqrt,
411 .sin,
412 .cos,
413 .tan,
414 .exp,
415 .exp2,
416 .log,
417 .log2,
418 .log10,
419 .floor,
420 .ceil,
421 .round,
422 .trunc_float,
423 .neg,
424 .cmp_lt_errors_len,
425 .c_va_end,
426 => {
427 const o = air_datas[@intFromEnum(inst)].un_op;
428 if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
429 return .none;
430 },
431
432 .ret,
433 .ret_safe,
434 .ret_load,
435 => {
436 const o = air_datas[@intFromEnum(inst)].un_op;
437 if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .noret);
438 return .noret;
439 },
440
441 .set_err_return_trace => {
442 const o = air_datas[@intFromEnum(inst)].un_op;
443 if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
444 return .write;
445 },
446
447 .add_with_overflow,
448 .sub_with_overflow,
449 .mul_with_overflow,
450 .shl_with_overflow,
451 .ptr_add,
452 .ptr_sub,
453 .ptr_elem_ptr,
454 .slice_elem_ptr,
455 .slice,
456 => {
457 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
458 const extra = air.extraData(Air.Bin, ty_pl.payload).data;
459 if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
460 if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
461 return .none;
462 },
463
464 .dbg_var_ptr,
465 .dbg_var_val,
466 .dbg_arg_inline,
467 => {
468 const o = air_datas[@intFromEnum(inst)].pl_op.operand;
469 if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
470 return .none;
471 },
472
473 .prefetch => {
474 const prefetch = air_datas[@intFromEnum(inst)].prefetch;
475 if (prefetch.ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
476 return .none;
477 },
478
479 .call, .call_always_tail, .call_never_tail, .call_never_inline => {
480 const inst_data = air_datas[@intFromEnum(inst)].pl_op;
481 const callee = inst_data.operand;
482 const extra = air.extraData(Air.Call, inst_data.payload);
483 const args = @as([]const Air.Inst.Ref, @ptrCast(air.extra.items[extra.end..][0..extra.data.args_len]));
484 if (args.len + 1 <= bpi - 1) {
485 if (callee == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
486 for (args, 0..) |arg, i| {
487 if (arg == operand_ref) return matchOperandSmallIndex(l, inst, @as(OperandInt, @intCast(i + 1)), .write);
488 }
489 return .write;
490 }
491 var bt = l.iterateBigTomb(inst);
492 if (bt.feed()) {
493 if (callee == operand_ref) return .tomb;
494 } else {
495 if (callee == operand_ref) return .write;
496 }
497 for (args) |arg| {
498 if (bt.feed()) {
499 if (arg == operand_ref) return .tomb;
500 } else {
501 if (arg == operand_ref) return .write;
502 }
503 }
504 return .write;
505 },
506 .select => {
507 const pl_op = air_datas[@intFromEnum(inst)].pl_op;
508 const extra = air.extraData(Air.Bin, pl_op.payload).data;
509 if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
510 if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
511 if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 2, .none);
512 return .none;
513 },
514 .shuffle => {
515 const extra = air.extraData(Air.Shuffle, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
516 if (extra.a == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
517 if (extra.b == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
518 return .none;
519 },
520 .reduce, .reduce_optimized => {
521 const reduce = air_datas[@intFromEnum(inst)].reduce;
522 if (reduce.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
523 return .none;
524 },
525 .cmp_vector, .cmp_vector_optimized => {
526 const extra = air.extraData(Air.VectorCmp, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
527 if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
528 if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
529 return .none;
530 },
531 .aggregate_init => {
532 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
533 const aggregate_ty = ty_pl.ty.toType();
534 const len = @as(usize, @intCast(aggregate_ty.arrayLenIp(ip)));
535 const elements = @as([]const Air.Inst.Ref, @ptrCast(air.extra.items[ty_pl.payload..][0..len]));
536
537 if (elements.len <= bpi - 1) {
538 for (elements, 0..) |elem, i| {
539 if (elem == operand_ref) return matchOperandSmallIndex(l, inst, @as(OperandInt, @intCast(i)), .none);
540 }
541 return .none;
542 }
543
544 var bt = l.iterateBigTomb(inst);
545 for (elements) |elem| {
546 if (bt.feed()) {
547 if (elem == operand_ref) return .tomb;
548 } else {
549 if (elem == operand_ref) return .write;
550 }
551 }
552 return .write;
553 },
554 .union_init => {
555 const extra = air.extraData(Air.UnionInit, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
556 if (extra.init == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
557 return .none;
558 },
559 .struct_field_ptr, .struct_field_val => {
560 const extra = air.extraData(Air.StructField, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
561 if (extra.struct_operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
562 return .none;
563 },
564 .field_parent_ptr => {
565 const extra = air.extraData(Air.FieldParentPtr, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
566 if (extra.field_ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
567 return .none;
568 },
569 .cmpxchg_strong, .cmpxchg_weak => {
570 const extra = air.extraData(Air.Cmpxchg, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
571 if (extra.ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
572 if (extra.expected_value == operand_ref) return matchOperandSmallIndex(l, inst, 1, .write);
573 if (extra.new_value == operand_ref) return matchOperandSmallIndex(l, inst, 2, .write);
574 return .write;
575 },
576 .mul_add => {
577 const pl_op = air_datas[@intFromEnum(inst)].pl_op;
578 const extra = air.extraData(Air.Bin, pl_op.payload).data;
579 if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
580 if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
581 if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 2, .none);
582 return .none;
583 },
584 .atomic_load => {
585 const ptr = air_datas[@intFromEnum(inst)].atomic_load.ptr;
586 if (ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
587 return .none;
588 },
589 .atomic_rmw => {
590 const pl_op = air_datas[@intFromEnum(inst)].pl_op;
591 const extra = air.extraData(Air.AtomicRmw, pl_op.payload).data;
592 if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
593 if (extra.operand == operand_ref) return matchOperandSmallIndex(l, inst, 1, .write);
594 return .write;
595 },
596
597 .br => {
598 const br = air_datas[@intFromEnum(inst)].br;
599 if (br.operand == operand_ref) return matchOperandSmallIndex(l, operand, 0, .noret);
600 return .noret;
601 },
602 .assembly => {
603 return .complex;
604 },
605 .block, .dbg_inline_block => |tag| {
606 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
607 const body: []const Air.Inst.Index = @ptrCast(switch (tag) {
608 inline .block, .dbg_inline_block => |comptime_tag| body: {
609 const extra = air.extraData(switch (comptime_tag) {
610 .block => Air.Block,
611 .dbg_inline_block => Air.DbgInlineBlock,
612 else => unreachable,
613 }, ty_pl.payload);
614 break :body air.extra.items[extra.end..][0..extra.data.body_len];
615 },
616 else => unreachable,
617 });
618
619 if (body.len == 1 and air_tags[@intFromEnum(body[0])] == .cond_br) {
620 // Peephole optimization for "panic-like" conditionals, which have
621 // one empty branch and another which calls a `noreturn` function.
622 // This allows us to infer that safety checks do not modify memory,
623 // as far as control flow successors are concerned.
624
625 const inst_data = air_datas[@intFromEnum(body[0])].pl_op;
626 const cond_extra = air.extraData(Air.CondBr, inst_data.payload);
627 if (inst_data.operand == operand_ref and operandDies(l, body[0], 0))
628 return .tomb;
629
630 if (cond_extra.data.then_body_len > 2 or cond_extra.data.else_body_len > 2)
631 return .complex;
632
633 const then_body: []const Air.Inst.Index = @ptrCast(air.extra.items[cond_extra.end..][0..cond_extra.data.then_body_len]);
634 const else_body: []const Air.Inst.Index = @ptrCast(air.extra.items[cond_extra.end + cond_extra.data.then_body_len ..][0..cond_extra.data.else_body_len]);
635 if (then_body.len > 1 and air_tags[@intFromEnum(then_body[1])] != .unreach)
636 return .complex;
637 if (else_body.len > 1 and air_tags[@intFromEnum(else_body[1])] != .unreach)
638 return .complex;
639
640 var operand_live: bool = true;
641 for (&[_]Air.Inst.Index{ then_body[0], else_body[0] }) |cond_inst| {
642 if (l.categorizeOperand(air, cond_inst, operand, ip) == .tomb)
643 operand_live = false;
644
645 switch (air_tags[@intFromEnum(cond_inst)]) {
646 .br => { // Breaks immediately back to block
647 const br = air_datas[@intFromEnum(cond_inst)].br;
648 if (br.block_inst != inst)
649 return .complex;
650 },
651 .call => {}, // Calls a noreturn function
652 else => return .complex,
653 }
654 }
655 return if (operand_live) .none else .tomb;
656 }
657
658 return .complex;
659 },
660
661 .@"try",
662 .try_cold,
663 .try_ptr,
664 .try_ptr_cold,
665 .loop,
666 .cond_br,
667 .switch_br,
668 .loop_switch_br,
669 => return .complex,
670
671 .wasm_memory_grow => {
672 const pl_op = air_datas[@intFromEnum(inst)].pl_op;
673 if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
674 return .none;
675 },
676 }
677}
678
679fn matchOperandSmallIndex(
680 l: Liveness,
681 inst: Air.Inst.Index,
682 operand: OperandInt,
683 default: OperandCategory,
684) OperandCategory {
685 if (operandDies(l, inst, operand)) {
686 return .tomb;
687 } else {
688 return default;
689 }
690}
691
692/// Higher level API.
693pub const CondBrSlices = struct {
694 then_deaths: []const Air.Inst.Index,
695 else_deaths: []const Air.Inst.Index,
696};
697
698pub fn getCondBr(l: Liveness, inst: Air.Inst.Index) CondBrSlices {
699 var index: usize = l.special.get(inst) orelse return .{
700 .then_deaths = &.{},
701 .else_deaths = &.{},
702 };
703 const then_death_count = l.extra[index];
704 index += 1;
705 const else_death_count = l.extra[index];
706 index += 1;
707 const then_deaths: []const Air.Inst.Index = @ptrCast(l.extra[index..][0..then_death_count]);
708 index += then_death_count;
709 return .{
710 .then_deaths = then_deaths,
711 .else_deaths = @ptrCast(l.extra[index..][0..else_death_count]),
712 };
713}
714
715/// Indexed by case number as they appear in AIR.
716/// Else is the last element.
717pub const SwitchBrTable = struct {
718 deaths: []const []const Air.Inst.Index,
719};
720
721/// Caller owns the memory.
722pub fn getSwitchBr(l: Liveness, gpa: Allocator, inst: Air.Inst.Index, cases_len: u32) Allocator.Error!SwitchBrTable {
723 var index: usize = l.special.get(inst) orelse return .{ .deaths = &.{} };
724 const else_death_count = l.extra[index];
725 index += 1;
726
727 var deaths = try gpa.alloc([]const Air.Inst.Index, cases_len);
728 errdefer gpa.free(deaths);
729
730 var case_i: u32 = 0;
731 while (case_i < cases_len - 1) : (case_i += 1) {
732 const case_death_count: u32 = l.extra[index];
733 index += 1;
734 deaths[case_i] = @ptrCast(l.extra[index..][0..case_death_count]);
735 index += case_death_count;
736 }
737 {
738 // Else
739 deaths[case_i] = @ptrCast(l.extra[index..][0..else_death_count]);
740 }
741 return .{ .deaths = deaths };
742}
743
744/// Note that this information is technically redundant, but is useful for
745/// backends nonetheless: see `Block`.
746pub const BlockSlices = struct {
747 deaths: []const Air.Inst.Index,
748};
749
750pub fn getBlock(l: Liveness, inst: Air.Inst.Index) BlockSlices {
751 const index: usize = l.special.get(inst) orelse return .{
752 .deaths = &.{},
753 };
754 const death_count = l.extra[index];
755 const deaths: []const Air.Inst.Index = @ptrCast(l.extra[index + 1 ..][0..death_count]);
756 return .{
757 .deaths = deaths,
758 };
759}
760
761pub const LoopSlice = struct {
762 deaths: []const Air.Inst.Index,
763};
764
765pub fn deinit(l: *Liveness, gpa: Allocator) void {
766 gpa.free(l.tomb_bits);
767 gpa.free(l.extra);
768 l.special.deinit(gpa);
769 l.* = undefined;
770}
771
772pub fn iterateBigTomb(l: Liveness, inst: Air.Inst.Index) BigTomb {
773 return .{
774 .tomb_bits = l.getTombBits(inst),
775 .extra_start = l.special.get(inst) orelse 0,
776 .extra_offset = 0,
777 .extra = l.extra,
778 .bit_index = 0,
779 .reached_end = false,
780 };
781}
782
783/// How many tomb bits per AIR instruction.
784pub const bpi = 4;
785pub const Bpi = std.meta.Int(.unsigned, bpi);
786pub const OperandInt = std.math.Log2Int(Bpi);
787
788/// Useful for decoders of Liveness information.
789pub const BigTomb = struct {
790 tomb_bits: Liveness.Bpi,
791 bit_index: u32,
792 extra_start: u32,
793 extra_offset: u32,
794 extra: []const u32,
795 reached_end: bool,
796
797 /// Returns whether the next operand dies.
798 pub fn feed(bt: *BigTomb) bool {
799 if (bt.reached_end) return false;
800
801 const this_bit_index = bt.bit_index;
802 bt.bit_index += 1;
803
804 const small_tombs = bpi - 1;
805 if (this_bit_index < small_tombs) {
806 const dies = @as(u1, @truncate(bt.tomb_bits >> @as(Liveness.OperandInt, @intCast(this_bit_index)))) != 0;
807 return dies;
808 }
809
810 const big_bit_index = this_bit_index - small_tombs;
811 while (big_bit_index - bt.extra_offset * 31 >= 31) {
812 if (@as(u1, @truncate(bt.extra[bt.extra_start + bt.extra_offset] >> 31)) != 0) {
813 bt.reached_end = true;
814 return false;
815 }
816 bt.extra_offset += 1;
817 }
818 const dies = @as(u1, @truncate(bt.extra[bt.extra_start + bt.extra_offset] >>
819 @as(u5, @intCast(big_bit_index - bt.extra_offset * 31)))) != 0;
820 return dies;
821 }
822};
823
824/// In-progress data; on successful analysis converted into `Liveness`.
825const Analysis = struct {
826 gpa: Allocator,
827 air: Air,
828 intern_pool: *InternPool,
829 tomb_bits: []usize,
830 special: std.AutoHashMapUnmanaged(Air.Inst.Index, u32),
831 extra: std.ArrayListUnmanaged(u32),
832
833 fn addExtra(a: *Analysis, extra: anytype) Allocator.Error!u32 {
834 const fields = std.meta.fields(@TypeOf(extra));
835 try a.extra.ensureUnusedCapacity(a.gpa, fields.len);
836 return addExtraAssumeCapacity(a, extra);
837 }
838
839 fn addExtraAssumeCapacity(a: *Analysis, extra: anytype) u32 {
840 const fields = std.meta.fields(@TypeOf(extra));
841 const result = @as(u32, @intCast(a.extra.items.len));
842 inline for (fields) |field| {
843 a.extra.appendAssumeCapacity(switch (field.type) {
844 u32 => @field(extra, field.name),
845 else => @compileError("bad field type"),
846 });
847 }
848 return result;
849 }
850};
851
852fn analyzeBody(
853 a: *Analysis,
854 comptime pass: LivenessPass,
855 data: *LivenessPassData(pass),
856 body: []const Air.Inst.Index,
857) Allocator.Error!void {
858 var i: usize = body.len;
859 while (i != 0) {
860 i -= 1;
861 const inst = body[i];
862 try analyzeInst(a, pass, data, inst);
863 }
864}
865
866fn analyzeInst(
867 a: *Analysis,
868 comptime pass: LivenessPass,
869 data: *LivenessPassData(pass),
870 inst: Air.Inst.Index,
871) Allocator.Error!void {
872 const ip = a.intern_pool;
873 const inst_tags = a.air.instructions.items(.tag);
874 const inst_datas = a.air.instructions.items(.data);
875
876 switch (inst_tags[@intFromEnum(inst)]) {
877 .add,
878 .add_safe,
879 .add_optimized,
880 .add_wrap,
881 .add_sat,
882 .sub,
883 .sub_safe,
884 .sub_optimized,
885 .sub_wrap,
886 .sub_sat,
887 .mul,
888 .mul_safe,
889 .mul_optimized,
890 .mul_wrap,
891 .mul_sat,
892 .div_float,
893 .div_float_optimized,
894 .div_trunc,
895 .div_trunc_optimized,
896 .div_floor,
897 .div_floor_optimized,
898 .div_exact,
899 .div_exact_optimized,
900 .rem,
901 .rem_optimized,
902 .mod,
903 .mod_optimized,
904 .bit_and,
905 .bit_or,
906 .xor,
907 .cmp_lt,
908 .cmp_lt_optimized,
909 .cmp_lte,
910 .cmp_lte_optimized,
911 .cmp_eq,
912 .cmp_eq_optimized,
913 .cmp_gte,
914 .cmp_gte_optimized,
915 .cmp_gt,
916 .cmp_gt_optimized,
917 .cmp_neq,
918 .cmp_neq_optimized,
919 .bool_and,
920 .bool_or,
921 .store,
922 .store_safe,
923 .array_elem_val,
924 .slice_elem_val,
925 .ptr_elem_val,
926 .shl,
927 .shl_exact,
928 .shl_sat,
929 .shr,
930 .shr_exact,
931 .atomic_store_unordered,
932 .atomic_store_monotonic,
933 .atomic_store_release,
934 .atomic_store_seq_cst,
935 .set_union_tag,
936 .min,
937 .max,
938 .memset,
939 .memset_safe,
940 .memcpy,
941 .memmove,
942 => {
943 const o = inst_datas[@intFromEnum(inst)].bin_op;
944 return analyzeOperands(a, pass, data, inst, .{ o.lhs, o.rhs, .none });
945 },
946
947 .vector_store_elem => {
948 const o = inst_datas[@intFromEnum(inst)].vector_store_elem;
949 const extra = a.air.extraData(Air.Bin, o.payload).data;
950 return analyzeOperands(a, pass, data, inst, .{ o.vector_ptr, extra.lhs, extra.rhs });
951 },
952
953 .arg,
954 .alloc,
955 .ret_ptr,
956 .breakpoint,
957 .dbg_stmt,
958 .dbg_empty_stmt,
959 .ret_addr,
960 .frame_addr,
961 .wasm_memory_size,
962 .err_return_trace,
963 .save_err_return_trace_index,
964 .tlv_dllimport_ptr,
965 .c_va_start,
966 .work_item_id,
967 .work_group_size,
968 .work_group_id,
969 => return analyzeOperands(a, pass, data, inst, .{ .none, .none, .none }),
970
971 .inferred_alloc, .inferred_alloc_comptime => unreachable,
972
973 .trap,
974 .unreach,
975 => return analyzeFuncEnd(a, pass, data, inst, .{ .none, .none, .none }),
976
977 .not,
978 .bitcast,
979 .load,
980 .fpext,
981 .fptrunc,
982 .intcast,
983 .intcast_safe,
984 .trunc,
985 .optional_payload,
986 .optional_payload_ptr,
987 .optional_payload_ptr_set,
988 .errunion_payload_ptr_set,
989 .wrap_optional,
990 .unwrap_errunion_payload,
991 .unwrap_errunion_err,
992 .unwrap_errunion_payload_ptr,
993 .unwrap_errunion_err_ptr,
994 .wrap_errunion_payload,
995 .wrap_errunion_err,
996 .slice_ptr,
997 .slice_len,
998 .ptr_slice_len_ptr,
999 .ptr_slice_ptr_ptr,
1000 .struct_field_ptr_index_0,
1001 .struct_field_ptr_index_1,
1002 .struct_field_ptr_index_2,
1003 .struct_field_ptr_index_3,
1004 .array_to_slice,
1005 .int_from_float,
1006 .int_from_float_optimized,
1007 .float_from_int,
1008 .get_union_tag,
1009 .clz,
1010 .ctz,
1011 .popcount,
1012 .byte_swap,
1013 .bit_reverse,
1014 .splat,
1015 .error_set_has_value,
1016 .addrspace_cast,
1017 .c_va_arg,
1018 .c_va_copy,
1019 .abs,
1020 => {
1021 const o = inst_datas[@intFromEnum(inst)].ty_op;
1022 return analyzeOperands(a, pass, data, inst, .{ o.operand, .none, .none });
1023 },
1024
1025 .is_null,
1026 .is_non_null,
1027 .is_null_ptr,
1028 .is_non_null_ptr,
1029 .is_err,
1030 .is_non_err,
1031 .is_err_ptr,
1032 .is_non_err_ptr,
1033 .is_named_enum_value,
1034 .tag_name,
1035 .error_name,
1036 .sqrt,
1037 .sin,
1038 .cos,
1039 .tan,
1040 .exp,
1041 .exp2,
1042 .log,
1043 .log2,
1044 .log10,
1045 .floor,
1046 .ceil,
1047 .round,
1048 .trunc_float,
1049 .neg,
1050 .neg_optimized,
1051 .cmp_lt_errors_len,
1052 .set_err_return_trace,
1053 .c_va_end,
1054 => {
1055 const operand = inst_datas[@intFromEnum(inst)].un_op;
1056 return analyzeOperands(a, pass, data, inst, .{ operand, .none, .none });
1057 },
1058
1059 .ret,
1060 .ret_safe,
1061 .ret_load,
1062 => {
1063 const operand = inst_datas[@intFromEnum(inst)].un_op;
1064 return analyzeFuncEnd(a, pass, data, inst, .{ operand, .none, .none });
1065 },
1066
1067 .add_with_overflow,
1068 .sub_with_overflow,
1069 .mul_with_overflow,
1070 .shl_with_overflow,
1071 .ptr_add,
1072 .ptr_sub,
1073 .ptr_elem_ptr,
1074 .slice_elem_ptr,
1075 .slice,
1076 => {
1077 const ty_pl = inst_datas[@intFromEnum(inst)].ty_pl;
1078 const extra = a.air.extraData(Air.Bin, ty_pl.payload).data;
1079 return analyzeOperands(a, pass, data, inst, .{ extra.lhs, extra.rhs, .none });
1080 },
1081
1082 .dbg_var_ptr,
1083 .dbg_var_val,
1084 .dbg_arg_inline,
1085 => {
1086 const operand = inst_datas[@intFromEnum(inst)].pl_op.operand;
1087 return analyzeOperands(a, pass, data, inst, .{ operand, .none, .none });
1088 },
1089
1090 .prefetch => {
1091 const prefetch = inst_datas[@intFromEnum(inst)].prefetch;
1092 return analyzeOperands(a, pass, data, inst, .{ prefetch.ptr, .none, .none });
1093 },
1094
1095 .call, .call_always_tail, .call_never_tail, .call_never_inline => {
1096 const inst_data = inst_datas[@intFromEnum(inst)].pl_op;
1097 const callee = inst_data.operand;
1098 const extra = a.air.extraData(Air.Call, inst_data.payload);
1099 const args = @as([]const Air.Inst.Ref, @ptrCast(a.air.extra.items[extra.end..][0..extra.data.args_len]));
1100 if (args.len + 1 <= bpi - 1) {
1101 var buf = [1]Air.Inst.Ref{.none} ** (bpi - 1);
1102 buf[0] = callee;
1103 @memcpy(buf[1..][0..args.len], args);
1104 return analyzeOperands(a, pass, data, inst, buf);
1105 }
1106
1107 var big = try AnalyzeBigOperands(pass).init(a, data, inst, args.len + 1);
1108 defer big.deinit();
1109 var i: usize = args.len;
1110 while (i > 0) {
1111 i -= 1;
1112 try big.feed(args[i]);
1113 }
1114 try big.feed(callee);
1115 return big.finish();
1116 },
1117 .select => {
1118 const pl_op = inst_datas[@intFromEnum(inst)].pl_op;
1119 const extra = a.air.extraData(Air.Bin, pl_op.payload).data;
1120 return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, extra.lhs, extra.rhs });
1121 },
1122 .shuffle => {
1123 const extra = a.air.extraData(Air.Shuffle, inst_datas[@intFromEnum(inst)].ty_pl.payload).data;
1124 return analyzeOperands(a, pass, data, inst, .{ extra.a, extra.b, .none });
1125 },
1126 .reduce, .reduce_optimized => {
1127 const reduce = inst_datas[@intFromEnum(inst)].reduce;
1128 return analyzeOperands(a, pass, data, inst, .{ reduce.operand, .none, .none });
1129 },
1130 .cmp_vector, .cmp_vector_optimized => {
1131 const extra = a.air.extraData(Air.VectorCmp, inst_datas[@intFromEnum(inst)].ty_pl.payload).data;
1132 return analyzeOperands(a, pass, data, inst, .{ extra.lhs, extra.rhs, .none });
1133 },
1134 .aggregate_init => {
1135 const ty_pl = inst_datas[@intFromEnum(inst)].ty_pl;
1136 const aggregate_ty = ty_pl.ty.toType();
1137 const len = @as(usize, @intCast(aggregate_ty.arrayLenIp(ip)));
1138 const elements = @as([]const Air.Inst.Ref, @ptrCast(a.air.extra.items[ty_pl.payload..][0..len]));
1139
1140 if (elements.len <= bpi - 1) {
1141 var buf = [1]Air.Inst.Ref{.none} ** (bpi - 1);
1142 @memcpy(buf[0..elements.len], elements);
1143 return analyzeOperands(a, pass, data, inst, buf);
1144 }
1145
1146 var big = try AnalyzeBigOperands(pass).init(a, data, inst, elements.len);
1147 defer big.deinit();
1148 var i: usize = elements.len;
1149 while (i > 0) {
1150 i -= 1;
1151 try big.feed(elements[i]);
1152 }
1153 return big.finish();
1154 },
1155 .union_init => {
1156 const extra = a.air.extraData(Air.UnionInit, inst_datas[@intFromEnum(inst)].ty_pl.payload).data;
1157 return analyzeOperands(a, pass, data, inst, .{ extra.init, .none, .none });
1158 },
1159 .struct_field_ptr, .struct_field_val => {
1160 const extra = a.air.extraData(Air.StructField, inst_datas[@intFromEnum(inst)].ty_pl.payload).data;
1161 return analyzeOperands(a, pass, data, inst, .{ extra.struct_operand, .none, .none });
1162 },
1163 .field_parent_ptr => {
1164 const extra = a.air.extraData(Air.FieldParentPtr, inst_datas[@intFromEnum(inst)].ty_pl.payload).data;
1165 return analyzeOperands(a, pass, data, inst, .{ extra.field_ptr, .none, .none });
1166 },
1167 .cmpxchg_strong, .cmpxchg_weak => {
1168 const extra = a.air.extraData(Air.Cmpxchg, inst_datas[@intFromEnum(inst)].ty_pl.payload).data;
1169 return analyzeOperands(a, pass, data, inst, .{ extra.ptr, extra.expected_value, extra.new_value });
1170 },
1171 .mul_add => {
1172 const pl_op = inst_datas[@intFromEnum(inst)].pl_op;
1173 const extra = a.air.extraData(Air.Bin, pl_op.payload).data;
1174 return analyzeOperands(a, pass, data, inst, .{ extra.lhs, extra.rhs, pl_op.operand });
1175 },
1176 .atomic_load => {
1177 const ptr = inst_datas[@intFromEnum(inst)].atomic_load.ptr;
1178 return analyzeOperands(a, pass, data, inst, .{ ptr, .none, .none });
1179 },
1180 .atomic_rmw => {
1181 const pl_op = inst_datas[@intFromEnum(inst)].pl_op;
1182 const extra = a.air.extraData(Air.AtomicRmw, pl_op.payload).data;
1183 return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, extra.operand, .none });
1184 },
1185
1186 .br => return analyzeInstBr(a, pass, data, inst),
1187 .repeat => return analyzeInstRepeat(a, pass, data, inst),
1188 .switch_dispatch => return analyzeInstSwitchDispatch(a, pass, data, inst),
1189
1190 .assembly => {
1191 const extra = a.air.extraData(Air.Asm, inst_datas[@intFromEnum(inst)].ty_pl.payload);
1192 var extra_i: usize = extra.end;
1193 const outputs = @as([]const Air.Inst.Ref, @ptrCast(a.air.extra.items[extra_i..][0..extra.data.outputs_len]));
1194 extra_i += outputs.len;
1195 const inputs = @as([]const Air.Inst.Ref, @ptrCast(a.air.extra.items[extra_i..][0..extra.data.inputs_len]));
1196 extra_i += inputs.len;
1197
1198 const num_operands = simple: {
1199 var buf = [1]Air.Inst.Ref{.none} ** (bpi - 1);
1200 var buf_index: usize = 0;
1201 for (outputs) |output| {
1202 if (output != .none) {
1203 if (buf_index < buf.len) buf[buf_index] = output;
1204 buf_index += 1;
1205 }
1206 }
1207 if (buf_index + inputs.len > buf.len) {
1208 break :simple buf_index + inputs.len;
1209 }
1210 @memcpy(buf[buf_index..][0..inputs.len], inputs);
1211 return analyzeOperands(a, pass, data, inst, buf);
1212 };
1213
1214 var big = try AnalyzeBigOperands(pass).init(a, data, inst, num_operands);
1215 defer big.deinit();
1216 var i: usize = inputs.len;
1217 while (i > 0) {
1218 i -= 1;
1219 try big.feed(inputs[i]);
1220 }
1221 i = outputs.len;
1222 while (i > 0) {
1223 i -= 1;
1224 if (outputs[i] != .none) {
1225 try big.feed(outputs[i]);
1226 }
1227 }
1228 return big.finish();
1229 },
1230
1231 inline .block, .dbg_inline_block => |comptime_tag| {
1232 const ty_pl = inst_datas[@intFromEnum(inst)].ty_pl;
1233 const extra = a.air.extraData(switch (comptime_tag) {
1234 .block => Air.Block,
1235 .dbg_inline_block => Air.DbgInlineBlock,
1236 else => unreachable,
1237 }, ty_pl.payload);
1238 return analyzeInstBlock(a, pass, data, inst, ty_pl.ty, @ptrCast(a.air.extra.items[extra.end..][0..extra.data.body_len]));
1239 },
1240 .loop => return analyzeInstLoop(a, pass, data, inst),
1241
1242 .@"try", .try_cold => return analyzeInstCondBr(a, pass, data, inst, .@"try"),
1243 .try_ptr, .try_ptr_cold => return analyzeInstCondBr(a, pass, data, inst, .try_ptr),
1244 .cond_br => return analyzeInstCondBr(a, pass, data, inst, .cond_br),
1245 .switch_br => return analyzeInstSwitchBr(a, pass, data, inst, false),
1246 .loop_switch_br => return analyzeInstSwitchBr(a, pass, data, inst, true),
1247
1248 .wasm_memory_grow => {
1249 const pl_op = inst_datas[@intFromEnum(inst)].pl_op;
1250 return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, .none, .none });
1251 },
1252 }
1253}
1254
1255/// Every instruction should hit this (after handling any nested bodies), in every pass. In the
1256/// initial pass, it is responsible for marking deaths of the (first three) operands and noticing
1257/// immediate deaths.
1258fn analyzeOperands(
1259 a: *Analysis,
1260 comptime pass: LivenessPass,
1261 data: *LivenessPassData(pass),
1262 inst: Air.Inst.Index,
1263 operands: [bpi - 1]Air.Inst.Ref,
1264) Allocator.Error!void {
1265 const gpa = a.gpa;
1266 const ip = a.intern_pool;
1267
1268 switch (pass) {
1269 .loop_analysis => {
1270 _ = data.live_set.remove(inst);
1271
1272 for (operands) |op_ref| {
1273 const operand = op_ref.toIndexAllowNone() orelse continue;
1274 _ = try data.live_set.put(gpa, operand, {});
1275 }
1276 },
1277
1278 .main_analysis => {
1279 const usize_index = (@intFromEnum(inst) * bpi) / @bitSizeOf(usize);
1280
1281 // This logic must synchronize with `will_die_immediately` in `AnalyzeBigOperands.init`.
1282 const immediate_death = if (data.live_set.remove(inst)) blk: {
1283 log.debug("[{}] %{}: removed from live set", .{ pass, @intFromEnum(inst) });
1284 break :blk false;
1285 } else blk: {
1286 log.debug("[{}] %{}: immediate death", .{ pass, @intFromEnum(inst) });
1287 break :blk true;
1288 };
1289
1290 var tomb_bits: Bpi = @as(Bpi, @intFromBool(immediate_death)) << (bpi - 1);
1291
1292 // If our result is unused and the instruction doesn't need to be lowered, backends will
1293 // skip the lowering of this instruction, so we don't want to record uses of operands.
1294 // That way, we can mark as many instructions as possible unused.
1295 if (!immediate_death or a.air.mustLower(inst, ip)) {
1296 // Note that it's important we iterate over the operands backwards, so that if a dying
1297 // operand is used multiple times we mark its last use as its death.
1298 var i = operands.len;
1299 while (i > 0) {
1300 i -= 1;
1301 const op_ref = operands[i];
1302 const operand = op_ref.toIndexAllowNone() orelse continue;
1303
1304 const mask = @as(Bpi, 1) << @as(OperandInt, @intCast(i));
1305
1306 if ((try data.live_set.fetchPut(gpa, operand, {})) == null) {
1307 log.debug("[{}] %{}: added %{} to live set (operand dies here)", .{ pass, @intFromEnum(inst), operand });
1308 tomb_bits |= mask;
1309 }
1310 }
1311 }
1312
1313 a.tomb_bits[usize_index] |= @as(usize, tomb_bits) <<
1314 @as(Log2Int(usize), @intCast((@intFromEnum(inst) % (@bitSizeOf(usize) / bpi)) * bpi));
1315 },
1316 }
1317}
1318
1319/// Like `analyzeOperands`, but for an instruction which returns from a function, so should
1320/// effectively kill every remaining live value other than its operands.
1321fn analyzeFuncEnd(
1322 a: *Analysis,
1323 comptime pass: LivenessPass,
1324 data: *LivenessPassData(pass),
1325 inst: Air.Inst.Index,
1326 operands: [bpi - 1]Air.Inst.Ref,
1327) Allocator.Error!void {
1328 switch (pass) {
1329 .loop_analysis => {
1330 // No operands need to be alive if we're returning from the function, so we don't need
1331 // to touch `breaks` here even though this is sort of like a break to the top level.
1332 },
1333
1334 .main_analysis => {
1335 data.live_set.clearRetainingCapacity();
1336 },
1337 }
1338
1339 return analyzeOperands(a, pass, data, inst, operands);
1340}
1341
1342fn analyzeInstBr(
1343 a: *Analysis,
1344 comptime pass: LivenessPass,
1345 data: *LivenessPassData(pass),
1346 inst: Air.Inst.Index,
1347) !void {
1348 const inst_datas = a.air.instructions.items(.data);
1349 const br = inst_datas[@intFromEnum(inst)].br;
1350 const gpa = a.gpa;
1351
1352 switch (pass) {
1353 .loop_analysis => {
1354 try data.breaks.put(gpa, br.block_inst, {});
1355 },
1356
1357 .main_analysis => {
1358 const block_scope = data.block_scopes.get(br.block_inst).?; // we should always be breaking from an enclosing block
1359
1360 const new_live_set = try block_scope.live_set.clone(gpa);
1361 data.live_set.deinit(gpa);
1362 data.live_set = new_live_set;
1363 },
1364 }
1365
1366 return analyzeOperands(a, pass, data, inst, .{ br.operand, .none, .none });
1367}
1368
1369fn analyzeInstRepeat(
1370 a: *Analysis,
1371 comptime pass: LivenessPass,
1372 data: *LivenessPassData(pass),
1373 inst: Air.Inst.Index,
1374) !void {
1375 const inst_datas = a.air.instructions.items(.data);
1376 const repeat = inst_datas[@intFromEnum(inst)].repeat;
1377 const gpa = a.gpa;
1378
1379 switch (pass) {
1380 .loop_analysis => {
1381 try data.breaks.put(gpa, repeat.loop_inst, {});
1382 },
1383
1384 .main_analysis => {
1385 const block_scope = data.block_scopes.get(repeat.loop_inst).?; // we should always be repeating an enclosing loop
1386
1387 const new_live_set = try block_scope.live_set.clone(gpa);
1388 data.live_set.deinit(gpa);
1389 data.live_set = new_live_set;
1390 },
1391 }
1392
1393 return analyzeOperands(a, pass, data, inst, .{ .none, .none, .none });
1394}
1395
1396fn analyzeInstSwitchDispatch(
1397 a: *Analysis,
1398 comptime pass: LivenessPass,
1399 data: *LivenessPassData(pass),
1400 inst: Air.Inst.Index,
1401) !void {
1402 // This happens to be identical to `analyzeInstBr`, but is separated anyway for clarity.
1403
1404 const inst_datas = a.air.instructions.items(.data);
1405 const br = inst_datas[@intFromEnum(inst)].br;
1406 const gpa = a.gpa;
1407
1408 switch (pass) {
1409 .loop_analysis => {
1410 try data.breaks.put(gpa, br.block_inst, {});
1411 },
1412
1413 .main_analysis => {
1414 const block_scope = data.block_scopes.get(br.block_inst).?; // we should always be repeating an enclosing loop
1415
1416 const new_live_set = try block_scope.live_set.clone(gpa);
1417 data.live_set.deinit(gpa);
1418 data.live_set = new_live_set;
1419 },
1420 }
1421
1422 return analyzeOperands(a, pass, data, inst, .{ br.operand, .none, .none });
1423}
1424
1425fn analyzeInstBlock(
1426 a: *Analysis,
1427 comptime pass: LivenessPass,
1428 data: *LivenessPassData(pass),
1429 inst: Air.Inst.Index,
1430 ty: Air.Inst.Ref,
1431 body: []const Air.Inst.Index,
1432) !void {
1433 const gpa = a.gpa;
1434
1435 // We actually want to do `analyzeOperands` *first*, since our result logically doesn't
1436 // exist until the block body ends (and we're iterating backwards)
1437 try analyzeOperands(a, pass, data, inst, .{ .none, .none, .none });
1438
1439 switch (pass) {
1440 .loop_analysis => {
1441 try analyzeBody(a, pass, data, body);
1442 _ = data.breaks.remove(inst);
1443 },
1444
1445 .main_analysis => {
1446 log.debug("[{}] %{}: block live set is {}", .{ pass, inst, fmtInstSet(&data.live_set) });
1447 // We can move the live set because the body should have a noreturn
1448 // instruction which overrides the set.
1449 try data.block_scopes.put(gpa, inst, .{
1450 .live_set = data.live_set.move(),
1451 });
1452 defer {
1453 log.debug("[{}] %{}: popped block scope", .{ pass, inst });
1454 var scope = data.block_scopes.fetchRemove(inst).?.value;
1455 scope.live_set.deinit(gpa);
1456 }
1457
1458 log.debug("[{}] %{}: pushed new block scope", .{ pass, inst });
1459 try analyzeBody(a, pass, data, body);
1460
1461 // If the block is noreturn, block deaths not only aren't useful, they're impossible to
1462 // find: there could be more stuff alive after the block than before it!
1463 if (!a.intern_pool.isNoReturn(ty.toType().toIntern())) {
1464 // The block kills the difference in the live sets
1465 const block_scope = data.block_scopes.get(inst).?;
1466 const num_deaths = data.live_set.count() - block_scope.live_set.count();
1467
1468 try a.extra.ensureUnusedCapacity(gpa, num_deaths + std.meta.fields(Block).len);
1469 const extra_index = a.addExtraAssumeCapacity(Block{
1470 .death_count = num_deaths,
1471 });
1472
1473 var measured_num: u32 = 0;
1474 var it = data.live_set.keyIterator();
1475 while (it.next()) |key| {
1476 const alive = key.*;
1477 if (!block_scope.live_set.contains(alive)) {
1478 // Dies in block
1479 a.extra.appendAssumeCapacity(@intFromEnum(alive));
1480 measured_num += 1;
1481 }
1482 }
1483 assert(measured_num == num_deaths); // post-live-set should be a subset of pre-live-set
1484 try a.special.put(gpa, inst, extra_index);
1485 log.debug("[{}] %{}: block deaths are {}", .{
1486 pass,
1487 inst,
1488 fmtInstList(@ptrCast(a.extra.items[extra_index + 1 ..][0..num_deaths])),
1489 });
1490 }
1491 },
1492 }
1493}
1494
1495fn writeLoopInfo(
1496 a: *Analysis,
1497 data: *LivenessPassData(.loop_analysis),
1498 inst: Air.Inst.Index,
1499 old_breaks: std.AutoHashMapUnmanaged(Air.Inst.Index, void),
1500 old_live: std.AutoHashMapUnmanaged(Air.Inst.Index, void),
1501) !void {
1502 const gpa = a.gpa;
1503
1504 // `loop`s are guaranteed to have at least one matching `repeat`.
1505 // Similarly, `loop_switch_br`s have a matching `switch_dispatch`.
1506 // However, we no longer care about repeats of this loop for resolving
1507 // which operands must live within it.
1508 assert(data.breaks.remove(inst));
1509
1510 const extra_index: u32 = @intCast(a.extra.items.len);
1511
1512 const num_breaks = data.breaks.count();
1513 try a.extra.ensureUnusedCapacity(gpa, 1 + num_breaks);
1514
1515 a.extra.appendAssumeCapacity(num_breaks);
1516
1517 var it = data.breaks.keyIterator();
1518 while (it.next()) |key| {
1519 const block_inst = key.*;
1520 a.extra.appendAssumeCapacity(@intFromEnum(block_inst));
1521 }
1522 log.debug("[{}] %{}: includes breaks to {}", .{ LivenessPass.loop_analysis, inst, fmtInstSet(&data.breaks) });
1523
1524 // Now we put the live operands from the loop body in too
1525 const num_live = data.live_set.count();
1526 try a.extra.ensureUnusedCapacity(gpa, 1 + num_live);
1527
1528 a.extra.appendAssumeCapacity(num_live);
1529 it = data.live_set.keyIterator();
1530 while (it.next()) |key| {
1531 const alive = key.*;
1532 a.extra.appendAssumeCapacity(@intFromEnum(alive));
1533 }
1534 log.debug("[{}] %{}: maintain liveness of {}", .{ LivenessPass.loop_analysis, inst, fmtInstSet(&data.live_set) });
1535
1536 try a.special.put(gpa, inst, extra_index);
1537
1538 // Add back operands which were previously alive
1539 it = old_live.keyIterator();
1540 while (it.next()) |key| {
1541 const alive = key.*;
1542 try data.live_set.put(gpa, alive, {});
1543 }
1544
1545 // And the same for breaks
1546 it = old_breaks.keyIterator();
1547 while (it.next()) |key| {
1548 const block_inst = key.*;
1549 try data.breaks.put(gpa, block_inst, {});
1550 }
1551}
1552
1553/// When analyzing a loop in the main pass, sets up `data.live_set` to be the set
1554/// of operands known to be alive when the loop repeats.
1555fn resolveLoopLiveSet(
1556 a: *Analysis,
1557 data: *LivenessPassData(.main_analysis),
1558 inst: Air.Inst.Index,
1559) !void {
1560 const gpa = a.gpa;
1561
1562 const extra_idx = a.special.fetchRemove(inst).?.value;
1563 const num_breaks = data.old_extra.items[extra_idx];
1564 const breaks: []const Air.Inst.Index = @ptrCast(data.old_extra.items[extra_idx + 1 ..][0..num_breaks]);
1565
1566 const num_loop_live = data.old_extra.items[extra_idx + num_breaks + 1];
1567 const loop_live: []const Air.Inst.Index = @ptrCast(data.old_extra.items[extra_idx + num_breaks + 2 ..][0..num_loop_live]);
1568
1569 // This is necessarily not in the same control flow branch, because loops are noreturn
1570 data.live_set.clearRetainingCapacity();
1571
1572 try data.live_set.ensureUnusedCapacity(gpa, @intCast(loop_live.len));
1573 for (loop_live) |alive| data.live_set.putAssumeCapacity(alive, {});
1574
1575 log.debug("[{}] %{}: block live set is {}", .{ LivenessPass.main_analysis, inst, fmtInstSet(&data.live_set) });
1576
1577 for (breaks) |block_inst| {
1578 // We might break to this block, so include every operand that the block needs alive
1579 const block_scope = data.block_scopes.get(block_inst).?;
1580
1581 var it = block_scope.live_set.keyIterator();
1582 while (it.next()) |key| {
1583 const alive = key.*;
1584 try data.live_set.put(gpa, alive, {});
1585 }
1586 }
1587
1588 log.debug("[{}] %{}: loop live set is {}", .{ LivenessPass.main_analysis, inst, fmtInstSet(&data.live_set) });
1589}
1590
1591fn analyzeInstLoop(
1592 a: *Analysis,
1593 comptime pass: LivenessPass,
1594 data: *LivenessPassData(pass),
1595 inst: Air.Inst.Index,
1596) !void {
1597 const inst_datas = a.air.instructions.items(.data);
1598 const extra = a.air.extraData(Air.Block, inst_datas[@intFromEnum(inst)].ty_pl.payload);
1599 const body: []const Air.Inst.Index = @ptrCast(a.air.extra.items[extra.end..][0..extra.data.body_len]);
1600 const gpa = a.gpa;
1601
1602 try analyzeOperands(a, pass, data, inst, .{ .none, .none, .none });
1603
1604 switch (pass) {
1605 .loop_analysis => {
1606 var old_breaks = data.breaks.move();
1607 defer old_breaks.deinit(gpa);
1608
1609 var old_live = data.live_set.move();
1610 defer old_live.deinit(gpa);
1611
1612 try analyzeBody(a, pass, data, body);
1613
1614 try writeLoopInfo(a, data, inst, old_breaks, old_live);
1615 },
1616
1617 .main_analysis => {
1618 try resolveLoopLiveSet(a, data, inst);
1619
1620 // Now, `data.live_set` is the operands which must be alive when the loop repeats.
1621 // Move them into a block scope for corresponding `repeat` instructions to notice.
1622 try data.block_scopes.putNoClobber(gpa, inst, .{
1623 .live_set = data.live_set.move(),
1624 });
1625 defer {
1626 log.debug("[{}] %{}: popped loop block scop", .{ pass, inst });
1627 var scope = data.block_scopes.fetchRemove(inst).?.value;
1628 scope.live_set.deinit(gpa);
1629 }
1630 try analyzeBody(a, pass, data, body);
1631 },
1632 }
1633}
1634
1635/// Despite its name, this function is used for analysis of not only `cond_br` instructions, but
1636/// also `try` and `try_ptr`, which are highly related. The `inst_type` parameter indicates which
1637/// type of instruction `inst` points to.
1638fn analyzeInstCondBr(
1639 a: *Analysis,
1640 comptime pass: LivenessPass,
1641 data: *LivenessPassData(pass),
1642 inst: Air.Inst.Index,
1643 comptime inst_type: enum { cond_br, @"try", try_ptr },
1644) !void {
1645 const inst_datas = a.air.instructions.items(.data);
1646 const gpa = a.gpa;
1647
1648 const extra = switch (inst_type) {
1649 .cond_br => a.air.extraData(Air.CondBr, inst_datas[@intFromEnum(inst)].pl_op.payload),
1650 .@"try" => a.air.extraData(Air.Try, inst_datas[@intFromEnum(inst)].pl_op.payload),
1651 .try_ptr => a.air.extraData(Air.TryPtr, inst_datas[@intFromEnum(inst)].ty_pl.payload),
1652 };
1653
1654 const condition = switch (inst_type) {
1655 .cond_br, .@"try" => inst_datas[@intFromEnum(inst)].pl_op.operand,
1656 .try_ptr => extra.data.ptr,
1657 };
1658
1659 const then_body: []const Air.Inst.Index = switch (inst_type) {
1660 .cond_br => @ptrCast(a.air.extra.items[extra.end..][0..extra.data.then_body_len]),
1661 else => &.{}, // we won't use this
1662 };
1663
1664 const else_body: []const Air.Inst.Index = @ptrCast(switch (inst_type) {
1665 .cond_br => a.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len],
1666 .@"try", .try_ptr => a.air.extra.items[extra.end..][0..extra.data.body_len],
1667 });
1668
1669 switch (pass) {
1670 .loop_analysis => {
1671 switch (inst_type) {
1672 .cond_br => try analyzeBody(a, pass, data, then_body),
1673 .@"try", .try_ptr => {},
1674 }
1675 try analyzeBody(a, pass, data, else_body);
1676 },
1677
1678 .main_analysis => {
1679 switch (inst_type) {
1680 .cond_br => try analyzeBody(a, pass, data, then_body),
1681 .@"try", .try_ptr => {}, // The "then body" is just the remainder of this block
1682 }
1683 var then_live = data.live_set.move();
1684 defer then_live.deinit(gpa);
1685
1686 try analyzeBody(a, pass, data, else_body);
1687 var else_live = data.live_set.move();
1688 defer else_live.deinit(gpa);
1689
1690 // Operands which are alive in one branch but not the other need to die at the start of
1691 // the peer branch.
1692
1693 var then_mirrored_deaths: std.ArrayListUnmanaged(Air.Inst.Index) = .empty;
1694 defer then_mirrored_deaths.deinit(gpa);
1695
1696 var else_mirrored_deaths: std.ArrayListUnmanaged(Air.Inst.Index) = .empty;
1697 defer else_mirrored_deaths.deinit(gpa);
1698
1699 // Note: this invalidates `else_live`, but expands `then_live` to be their union
1700 {
1701 var it = then_live.keyIterator();
1702 while (it.next()) |key| {
1703 const death = key.*;
1704 if (else_live.remove(death)) continue; // removing makes the loop below faster
1705
1706 // If this is a `try`, the "then body" (rest of the branch) might have
1707 // referenced our result. We want to avoid killing this value in the else branch
1708 // if that's the case, since it only exists in the (fake) then branch.
1709 switch (inst_type) {
1710 .cond_br => {},
1711 .@"try", .try_ptr => if (death == inst) continue,
1712 }
1713
1714 try else_mirrored_deaths.append(gpa, death);
1715 }
1716 // Since we removed common stuff above, `else_live` is now only operands
1717 // which are *only* alive in the else branch
1718 it = else_live.keyIterator();
1719 while (it.next()) |key| {
1720 const death = key.*;
1721 try then_mirrored_deaths.append(gpa, death);
1722 // Make `then_live` contain the full live set (i.e. union of both)
1723 try then_live.put(gpa, death, {});
1724 }
1725 }
1726
1727 log.debug("[{}] %{}: 'then' branch mirrored deaths are {}", .{ pass, inst, fmtInstList(then_mirrored_deaths.items) });
1728 log.debug("[{}] %{}: 'else' branch mirrored deaths are {}", .{ pass, inst, fmtInstList(else_mirrored_deaths.items) });
1729
1730 data.live_set.deinit(gpa);
1731 data.live_set = then_live.move(); // Really the union of both live sets
1732
1733 log.debug("[{}] %{}: new live set is {}", .{ pass, inst, fmtInstSet(&data.live_set) });
1734
1735 // Write the mirrored deaths to `extra`
1736 const then_death_count = @as(u32, @intCast(then_mirrored_deaths.items.len));
1737 const else_death_count = @as(u32, @intCast(else_mirrored_deaths.items.len));
1738 try a.extra.ensureUnusedCapacity(gpa, std.meta.fields(CondBr).len + then_death_count + else_death_count);
1739 const extra_index = a.addExtraAssumeCapacity(CondBr{
1740 .then_death_count = then_death_count,
1741 .else_death_count = else_death_count,
1742 });
1743 a.extra.appendSliceAssumeCapacity(@ptrCast(then_mirrored_deaths.items));
1744 a.extra.appendSliceAssumeCapacity(@ptrCast(else_mirrored_deaths.items));
1745 try a.special.put(gpa, inst, extra_index);
1746 },
1747 }
1748
1749 try analyzeOperands(a, pass, data, inst, .{ condition, .none, .none });
1750}
1751
1752fn analyzeInstSwitchBr(
1753 a: *Analysis,
1754 comptime pass: LivenessPass,
1755 data: *LivenessPassData(pass),
1756 inst: Air.Inst.Index,
1757 is_dispatch_loop: bool,
1758) !void {
1759 const inst_datas = a.air.instructions.items(.data);
1760 const pl_op = inst_datas[@intFromEnum(inst)].pl_op;
1761 const condition = pl_op.operand;
1762 const switch_br = a.air.unwrapSwitch(inst);
1763 const gpa = a.gpa;
1764 const ncases = switch_br.cases_len;
1765
1766 switch (pass) {
1767 .loop_analysis => {
1768 var old_breaks: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .empty;
1769 defer old_breaks.deinit(gpa);
1770
1771 var old_live: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .empty;
1772 defer old_live.deinit(gpa);
1773
1774 if (is_dispatch_loop) {
1775 old_breaks = data.breaks.move();
1776 old_live = data.live_set.move();
1777 }
1778
1779 var it = switch_br.iterateCases();
1780 while (it.next()) |case| {
1781 try analyzeBody(a, pass, data, case.body);
1782 }
1783 { // else
1784 const else_body = it.elseBody();
1785 try analyzeBody(a, pass, data, else_body);
1786 }
1787
1788 if (is_dispatch_loop) {
1789 try writeLoopInfo(a, data, inst, old_breaks, old_live);
1790 }
1791 },
1792
1793 .main_analysis => {
1794 if (is_dispatch_loop) {
1795 try resolveLoopLiveSet(a, data, inst);
1796 try data.block_scopes.putNoClobber(gpa, inst, .{
1797 .live_set = data.live_set.move(),
1798 });
1799 }
1800 defer if (is_dispatch_loop) {
1801 log.debug("[{}] %{}: popped loop block scop", .{ pass, inst });
1802 var scope = data.block_scopes.fetchRemove(inst).?.value;
1803 scope.live_set.deinit(gpa);
1804 };
1805 // This is, all in all, just a messier version of the `cond_br` logic. If you're trying
1806 // to understand it, I encourage looking at `analyzeInstCondBr` first.
1807
1808 const DeathSet = std.AutoHashMapUnmanaged(Air.Inst.Index, void);
1809 const DeathList = std.ArrayListUnmanaged(Air.Inst.Index);
1810
1811 var case_live_sets = try gpa.alloc(std.AutoHashMapUnmanaged(Air.Inst.Index, void), ncases + 1); // +1 for else
1812 defer gpa.free(case_live_sets);
1813
1814 @memset(case_live_sets, .{});
1815 defer for (case_live_sets) |*live_set| live_set.deinit(gpa);
1816
1817 var case_it = switch_br.iterateCases();
1818 while (case_it.next()) |case| {
1819 try analyzeBody(a, pass, data, case.body);
1820 case_live_sets[case.idx] = data.live_set.move();
1821 }
1822 { // else
1823 const else_body = case_it.elseBody();
1824 try analyzeBody(a, pass, data, else_body);
1825 case_live_sets[ncases] = data.live_set.move();
1826 }
1827
1828 const mirrored_deaths = try gpa.alloc(DeathList, ncases + 1);
1829 defer gpa.free(mirrored_deaths);
1830
1831 @memset(mirrored_deaths, .{});
1832 defer for (mirrored_deaths) |*md| md.deinit(gpa);
1833
1834 {
1835 var all_alive: DeathSet = .{};
1836 defer all_alive.deinit(gpa);
1837
1838 for (case_live_sets) |*live_set| {
1839 try all_alive.ensureUnusedCapacity(gpa, live_set.count());
1840 var it = live_set.keyIterator();
1841 while (it.next()) |key| {
1842 const alive = key.*;
1843 all_alive.putAssumeCapacity(alive, {});
1844 }
1845 }
1846
1847 for (mirrored_deaths, case_live_sets) |*mirrored, *live_set| {
1848 var it = all_alive.keyIterator();
1849 while (it.next()) |key| {
1850 const alive = key.*;
1851 if (!live_set.contains(alive)) {
1852 // Should die at the start of this branch
1853 try mirrored.append(gpa, alive);
1854 }
1855 }
1856 }
1857
1858 for (mirrored_deaths, 0..) |mirrored, i| {
1859 log.debug("[{}] %{}: case {} mirrored deaths are {}", .{ pass, inst, i, fmtInstList(mirrored.items) });
1860 }
1861
1862 data.live_set.deinit(gpa);
1863 data.live_set = all_alive.move();
1864
1865 log.debug("[{}] %{}: new live set is {}", .{ pass, inst, fmtInstSet(&data.live_set) });
1866 }
1867
1868 const else_death_count = @as(u32, @intCast(mirrored_deaths[ncases].items.len));
1869 const extra_index = try a.addExtra(SwitchBr{
1870 .else_death_count = else_death_count,
1871 });
1872 for (mirrored_deaths[0..ncases]) |mirrored| {
1873 const num = @as(u32, @intCast(mirrored.items.len));
1874 try a.extra.ensureUnusedCapacity(gpa, num + 1);
1875 a.extra.appendAssumeCapacity(num);
1876 a.extra.appendSliceAssumeCapacity(@ptrCast(mirrored.items));
1877 }
1878 try a.extra.ensureUnusedCapacity(gpa, else_death_count);
1879 a.extra.appendSliceAssumeCapacity(@ptrCast(mirrored_deaths[ncases].items));
1880 try a.special.put(gpa, inst, extra_index);
1881 },
1882 }
1883
1884 try analyzeOperands(a, pass, data, inst, .{ condition, .none, .none });
1885}
1886
1887fn AnalyzeBigOperands(comptime pass: LivenessPass) type {
1888 return struct {
1889 a: *Analysis,
1890 data: *LivenessPassData(pass),
1891 inst: Air.Inst.Index,
1892
1893 operands_remaining: u32,
1894 small: [bpi - 1]Air.Inst.Ref = .{.none} ** (bpi - 1),
1895 extra_tombs: []u32,
1896
1897 // Only used in `LivenessPass.main_analysis`
1898 will_die_immediately: bool,
1899
1900 const Self = @This();
1901
1902 fn init(
1903 a: *Analysis,
1904 data: *LivenessPassData(pass),
1905 inst: Air.Inst.Index,
1906 total_operands: usize,
1907 ) !Self {
1908 const extra_operands = @as(u32, @intCast(total_operands)) -| (bpi - 1);
1909 const max_extra_tombs = (extra_operands + 30) / 31;
1910
1911 const extra_tombs: []u32 = switch (pass) {
1912 .loop_analysis => &.{},
1913 .main_analysis => try a.gpa.alloc(u32, max_extra_tombs),
1914 };
1915 errdefer a.gpa.free(extra_tombs);
1916
1917 @memset(extra_tombs, 0);
1918
1919 const will_die_immediately: bool = switch (pass) {
1920 .loop_analysis => false, // track everything, since we don't have full liveness information yet
1921 .main_analysis => !data.live_set.contains(inst),
1922 };
1923
1924 return .{
1925 .a = a,
1926 .data = data,
1927 .inst = inst,
1928 .operands_remaining = @as(u32, @intCast(total_operands)),
1929 .extra_tombs = extra_tombs,
1930 .will_die_immediately = will_die_immediately,
1931 };
1932 }
1933
1934 /// Must be called with operands in reverse order.
1935 fn feed(big: *Self, op_ref: Air.Inst.Ref) !void {
1936 const ip = big.a.intern_pool;
1937 // Note that after this, `operands_remaining` becomes the index of the current operand
1938 big.operands_remaining -= 1;
1939
1940 if (big.operands_remaining < bpi - 1) {
1941 big.small[big.operands_remaining] = op_ref;
1942 return;
1943 }
1944
1945 const operand = op_ref.toIndex() orelse return;
1946
1947 // If our result is unused and the instruction doesn't need to be lowered, backends will
1948 // skip the lowering of this instruction, so we don't want to record uses of operands.
1949 // That way, we can mark as many instructions as possible unused.
1950 if (big.will_die_immediately and !big.a.air.mustLower(big.inst, ip)) return;
1951
1952 const extra_byte = (big.operands_remaining - (bpi - 1)) / 31;
1953 const extra_bit = @as(u5, @intCast(big.operands_remaining - (bpi - 1) - extra_byte * 31));
1954
1955 const gpa = big.a.gpa;
1956
1957 switch (pass) {
1958 .loop_analysis => {
1959 _ = try big.data.live_set.put(gpa, operand, {});
1960 },
1961
1962 .main_analysis => {
1963 if ((try big.data.live_set.fetchPut(gpa, operand, {})) == null) {
1964 log.debug("[{}] %{}: added %{} to live set (operand dies here)", .{ pass, big.inst, operand });
1965 big.extra_tombs[extra_byte] |= @as(u32, 1) << extra_bit;
1966 }
1967 },
1968 }
1969 }
1970
1971 fn finish(big: *Self) !void {
1972 const gpa = big.a.gpa;
1973
1974 std.debug.assert(big.operands_remaining == 0);
1975
1976 switch (pass) {
1977 .loop_analysis => {},
1978
1979 .main_analysis => {
1980 // Note that the MSB is set on the final tomb to indicate the terminal element. This
1981 // allows for an optimisation where we only add as many extra tombs as are needed to
1982 // represent the dying operands. Each pass modifies operand bits and so needs to write
1983 // back, so let's figure out how many extra tombs we really need. Note that we always
1984 // keep at least one.
1985 var num: usize = big.extra_tombs.len;
1986 while (num > 1) {
1987 if (@as(u31, @truncate(big.extra_tombs[num - 1])) != 0) {
1988 // Some operand dies here
1989 break;
1990 }
1991 num -= 1;
1992 }
1993 // Mark final tomb
1994 big.extra_tombs[num - 1] |= @as(u32, 1) << 31;
1995
1996 const extra_tombs = big.extra_tombs[0..num];
1997
1998 const extra_index = @as(u32, @intCast(big.a.extra.items.len));
1999 try big.a.extra.appendSlice(gpa, extra_tombs);
2000 try big.a.special.put(gpa, big.inst, extra_index);
2001 },
2002 }
2003
2004 try analyzeOperands(big.a, pass, big.data, big.inst, big.small);
2005 }
2006
2007 fn deinit(big: *Self) void {
2008 big.a.gpa.free(big.extra_tombs);
2009 }
2010 };
2011}
2012
2013fn fmtInstSet(set: *const std.AutoHashMapUnmanaged(Air.Inst.Index, void)) FmtInstSet {
2014 return .{ .set = set };
2015}
2016
2017const FmtInstSet = struct {
2018 set: *const std.AutoHashMapUnmanaged(Air.Inst.Index, void),
2019
2020 pub fn format(val: FmtInstSet, comptime _: []const u8, _: std.fmt.FormatOptions, w: anytype) !void {
2021 if (val.set.count() == 0) {
2022 try w.writeAll("[no instructions]");
2023 return;
2024 }
2025 var it = val.set.keyIterator();
2026 try w.print("%{}", .{it.next().?.*});
2027 while (it.next()) |key| {
2028 try w.print(" %{}", .{key.*});
2029 }
2030 }
2031};
2032
2033fn fmtInstList(list: []const Air.Inst.Index) FmtInstList {
2034 return .{ .list = list };
2035}
2036
2037const FmtInstList = struct {
2038 list: []const Air.Inst.Index,
2039
2040 pub fn format(val: FmtInstList, comptime _: []const u8, _: std.fmt.FormatOptions, w: anytype) !void {
2041 if (val.list.len == 0) {
2042 try w.writeAll("[no instructions]");
2043 return;
2044 }
2045 try w.print("%{}", .{val.list[0]});
2046 for (val.list[1..]) |inst| {
2047 try w.print(" %{}", .{inst});
2048 }
2049 }
2050};
src/Air/Liveness/Verify.zig created+642
......@@ -0,0 +1,642 @@
1//! Verifies that Liveness information is valid.
2
3gpa: std.mem.Allocator,
4air: Air,
5liveness: Liveness,
6live: LiveMap = .{},
7blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, LiveMap) = .empty,
8loops: std.AutoHashMapUnmanaged(Air.Inst.Index, LiveMap) = .empty,
9intern_pool: *const InternPool,
10
11pub const Error = error{ LivenessInvalid, OutOfMemory };
12
13pub fn deinit(self: *Verify) void {
14 self.live.deinit(self.gpa);
15 {
16 var it = self.blocks.valueIterator();
17 while (it.next()) |block| block.deinit(self.gpa);
18 self.blocks.deinit(self.gpa);
19 }
20 {
21 var it = self.loops.valueIterator();
22 while (it.next()) |block| block.deinit(self.gpa);
23 self.loops.deinit(self.gpa);
24 }
25 self.* = undefined;
26}
27
28pub fn verify(self: *Verify) Error!void {
29 self.live.clearRetainingCapacity();
30 self.blocks.clearRetainingCapacity();
31 self.loops.clearRetainingCapacity();
32 try self.verifyBody(self.air.getMainBody());
33 // We don't care about `self.live` now, because the loop body was noreturn - everything being dead was checked on `ret` etc
34 assert(self.blocks.count() == 0);
35 assert(self.loops.count() == 0);
36}
37
38const LiveMap = std.AutoHashMapUnmanaged(Air.Inst.Index, void);
39
40fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
41 const ip = self.intern_pool;
42 const tags = self.air.instructions.items(.tag);
43 const data = self.air.instructions.items(.data);
44 for (body) |inst| {
45 if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip)) {
46 // This instruction will not be lowered and should be ignored.
47 continue;
48 }
49
50 switch (tags[@intFromEnum(inst)]) {
51 // no operands
52 .arg,
53 .alloc,
54 .inferred_alloc,
55 .inferred_alloc_comptime,
56 .ret_ptr,
57 .breakpoint,
58 .dbg_stmt,
59 .dbg_empty_stmt,
60 .ret_addr,
61 .frame_addr,
62 .wasm_memory_size,
63 .err_return_trace,
64 .save_err_return_trace_index,
65 .tlv_dllimport_ptr,
66 .c_va_start,
67 .work_item_id,
68 .work_group_size,
69 .work_group_id,
70 => try self.verifyInstOperands(inst, .{ .none, .none, .none }),
71
72 .trap, .unreach => {
73 try self.verifyInstOperands(inst, .{ .none, .none, .none });
74 // This instruction terminates the function, so everything should be dead
75 if (self.live.count() > 0) return invalid("%{}: instructions still alive", .{inst});
76 },
77
78 // unary
79 .not,
80 .bitcast,
81 .load,
82 .fpext,
83 .fptrunc,
84 .intcast,
85 .intcast_safe,
86 .trunc,
87 .optional_payload,
88 .optional_payload_ptr,
89 .optional_payload_ptr_set,
90 .errunion_payload_ptr_set,
91 .wrap_optional,
92 .unwrap_errunion_payload,
93 .unwrap_errunion_err,
94 .unwrap_errunion_payload_ptr,
95 .unwrap_errunion_err_ptr,
96 .wrap_errunion_payload,
97 .wrap_errunion_err,
98 .slice_ptr,
99 .slice_len,
100 .ptr_slice_len_ptr,
101 .ptr_slice_ptr_ptr,
102 .struct_field_ptr_index_0,
103 .struct_field_ptr_index_1,
104 .struct_field_ptr_index_2,
105 .struct_field_ptr_index_3,
106 .array_to_slice,
107 .int_from_float,
108 .int_from_float_optimized,
109 .float_from_int,
110 .get_union_tag,
111 .clz,
112 .ctz,
113 .popcount,
114 .byte_swap,
115 .bit_reverse,
116 .splat,
117 .error_set_has_value,
118 .addrspace_cast,
119 .c_va_arg,
120 .c_va_copy,
121 .abs,
122 => {
123 const ty_op = data[@intFromEnum(inst)].ty_op;
124 try self.verifyInstOperands(inst, .{ ty_op.operand, .none, .none });
125 },
126 .is_null,
127 .is_non_null,
128 .is_null_ptr,
129 .is_non_null_ptr,
130 .is_err,
131 .is_non_err,
132 .is_err_ptr,
133 .is_non_err_ptr,
134 .is_named_enum_value,
135 .tag_name,
136 .error_name,
137 .sqrt,
138 .sin,
139 .cos,
140 .tan,
141 .exp,
142 .exp2,
143 .log,
144 .log2,
145 .log10,
146 .floor,
147 .ceil,
148 .round,
149 .trunc_float,
150 .neg,
151 .neg_optimized,
152 .cmp_lt_errors_len,
153 .set_err_return_trace,
154 .c_va_end,
155 => {
156 const un_op = data[@intFromEnum(inst)].un_op;
157 try self.verifyInstOperands(inst, .{ un_op, .none, .none });
158 },
159 .ret,
160 .ret_safe,
161 .ret_load,
162 => {
163 const un_op = data[@intFromEnum(inst)].un_op;
164 try self.verifyInstOperands(inst, .{ un_op, .none, .none });
165 // This instruction terminates the function, so everything should be dead
166 if (self.live.count() > 0) return invalid("%{}: instructions still alive", .{inst});
167 },
168 .dbg_var_ptr,
169 .dbg_var_val,
170 .dbg_arg_inline,
171 .wasm_memory_grow,
172 => {
173 const pl_op = data[@intFromEnum(inst)].pl_op;
174 try self.verifyInstOperands(inst, .{ pl_op.operand, .none, .none });
175 },
176 .prefetch => {
177 const prefetch = data[@intFromEnum(inst)].prefetch;
178 try self.verifyInstOperands(inst, .{ prefetch.ptr, .none, .none });
179 },
180 .reduce,
181 .reduce_optimized,
182 => {
183 const reduce = data[@intFromEnum(inst)].reduce;
184 try self.verifyInstOperands(inst, .{ reduce.operand, .none, .none });
185 },
186 .union_init => {
187 const ty_pl = data[@intFromEnum(inst)].ty_pl;
188 const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data;
189 try self.verifyInstOperands(inst, .{ extra.init, .none, .none });
190 },
191 .struct_field_ptr, .struct_field_val => {
192 const ty_pl = data[@intFromEnum(inst)].ty_pl;
193 const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
194 try self.verifyInstOperands(inst, .{ extra.struct_operand, .none, .none });
195 },
196 .field_parent_ptr => {
197 const ty_pl = data[@intFromEnum(inst)].ty_pl;
198 const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
199 try self.verifyInstOperands(inst, .{ extra.field_ptr, .none, .none });
200 },
201 .atomic_load => {
202 const atomic_load = data[@intFromEnum(inst)].atomic_load;
203 try self.verifyInstOperands(inst, .{ atomic_load.ptr, .none, .none });
204 },
205
206 // binary
207 .add,
208 .add_safe,
209 .add_optimized,
210 .add_wrap,
211 .add_sat,
212 .sub,
213 .sub_safe,
214 .sub_optimized,
215 .sub_wrap,
216 .sub_sat,
217 .mul,
218 .mul_safe,
219 .mul_optimized,
220 .mul_wrap,
221 .mul_sat,
222 .div_float,
223 .div_float_optimized,
224 .div_trunc,
225 .div_trunc_optimized,
226 .div_floor,
227 .div_floor_optimized,
228 .div_exact,
229 .div_exact_optimized,
230 .rem,
231 .rem_optimized,
232 .mod,
233 .mod_optimized,
234 .bit_and,
235 .bit_or,
236 .xor,
237 .cmp_lt,
238 .cmp_lt_optimized,
239 .cmp_lte,
240 .cmp_lte_optimized,
241 .cmp_eq,
242 .cmp_eq_optimized,
243 .cmp_gte,
244 .cmp_gte_optimized,
245 .cmp_gt,
246 .cmp_gt_optimized,
247 .cmp_neq,
248 .cmp_neq_optimized,
249 .bool_and,
250 .bool_or,
251 .store,
252 .store_safe,
253 .array_elem_val,
254 .slice_elem_val,
255 .ptr_elem_val,
256 .shl,
257 .shl_exact,
258 .shl_sat,
259 .shr,
260 .shr_exact,
261 .atomic_store_unordered,
262 .atomic_store_monotonic,
263 .atomic_store_release,
264 .atomic_store_seq_cst,
265 .set_union_tag,
266 .min,
267 .max,
268 .memset,
269 .memset_safe,
270 .memcpy,
271 .memmove,
272 => {
273 const bin_op = data[@intFromEnum(inst)].bin_op;
274 try self.verifyInstOperands(inst, .{ bin_op.lhs, bin_op.rhs, .none });
275 },
276 .add_with_overflow,
277 .sub_with_overflow,
278 .mul_with_overflow,
279 .shl_with_overflow,
280 .ptr_add,
281 .ptr_sub,
282 .ptr_elem_ptr,
283 .slice_elem_ptr,
284 .slice,
285 => {
286 const ty_pl = data[@intFromEnum(inst)].ty_pl;
287 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
288 try self.verifyInstOperands(inst, .{ extra.lhs, extra.rhs, .none });
289 },
290 .shuffle => {
291 const ty_pl = data[@intFromEnum(inst)].ty_pl;
292 const extra = self.air.extraData(Air.Shuffle, ty_pl.payload).data;
293 try self.verifyInstOperands(inst, .{ extra.a, extra.b, .none });
294 },
295 .cmp_vector,
296 .cmp_vector_optimized,
297 => {
298 const ty_pl = data[@intFromEnum(inst)].ty_pl;
299 const extra = self.air.extraData(Air.VectorCmp, ty_pl.payload).data;
300 try self.verifyInstOperands(inst, .{ extra.lhs, extra.rhs, .none });
301 },
302 .atomic_rmw => {
303 const pl_op = data[@intFromEnum(inst)].pl_op;
304 const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data;
305 try self.verifyInstOperands(inst, .{ pl_op.operand, extra.operand, .none });
306 },
307
308 // ternary
309 .select => {
310 const pl_op = data[@intFromEnum(inst)].pl_op;
311 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
312 try self.verifyInstOperands(inst, .{ pl_op.operand, extra.lhs, extra.rhs });
313 },
314 .mul_add => {
315 const pl_op = data[@intFromEnum(inst)].pl_op;
316 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
317 try self.verifyInstOperands(inst, .{ extra.lhs, extra.rhs, pl_op.operand });
318 },
319 .vector_store_elem => {
320 const vector_store_elem = data[@intFromEnum(inst)].vector_store_elem;
321 const extra = self.air.extraData(Air.Bin, vector_store_elem.payload).data;
322 try self.verifyInstOperands(inst, .{ vector_store_elem.vector_ptr, extra.lhs, extra.rhs });
323 },
324 .cmpxchg_strong,
325 .cmpxchg_weak,
326 => {
327 const ty_pl = data[@intFromEnum(inst)].ty_pl;
328 const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
329 try self.verifyInstOperands(inst, .{ extra.ptr, extra.expected_value, extra.new_value });
330 },
331
332 // big tombs
333 .aggregate_init => {
334 const ty_pl = data[@intFromEnum(inst)].ty_pl;
335 const aggregate_ty = ty_pl.ty.toType();
336 const len = @as(usize, @intCast(aggregate_ty.arrayLenIp(ip)));
337 const elements = @as([]const Air.Inst.Ref, @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]));
338
339 var bt = self.liveness.iterateBigTomb(inst);
340 for (elements) |element| {
341 try self.verifyOperand(inst, element, bt.feed());
342 }
343 try self.verifyInst(inst);
344 },
345 .call, .call_always_tail, .call_never_tail, .call_never_inline => {
346 const pl_op = data[@intFromEnum(inst)].pl_op;
347 const extra = self.air.extraData(Air.Call, pl_op.payload);
348 const args = @as(
349 []const Air.Inst.Ref,
350 @ptrCast(self.air.extra.items[extra.end..][0..extra.data.args_len]),
351 );
352
353 var bt = self.liveness.iterateBigTomb(inst);
354 try self.verifyOperand(inst, pl_op.operand, bt.feed());
355 for (args) |arg| {
356 try self.verifyOperand(inst, arg, bt.feed());
357 }
358 try self.verifyInst(inst);
359 },
360 .assembly => {
361 const ty_pl = data[@intFromEnum(inst)].ty_pl;
362 const extra = self.air.extraData(Air.Asm, ty_pl.payload);
363 var extra_i = extra.end;
364 const outputs = @as(
365 []const Air.Inst.Ref,
366 @ptrCast(self.air.extra.items[extra_i..][0..extra.data.outputs_len]),
367 );
368 extra_i += outputs.len;
369 const inputs = @as(
370 []const Air.Inst.Ref,
371 @ptrCast(self.air.extra.items[extra_i..][0..extra.data.inputs_len]),
372 );
373 extra_i += inputs.len;
374
375 var bt = self.liveness.iterateBigTomb(inst);
376 for (outputs) |output| {
377 if (output != .none) {
378 try self.verifyOperand(inst, output, bt.feed());
379 }
380 }
381 for (inputs) |input| {
382 try self.verifyOperand(inst, input, bt.feed());
383 }
384 try self.verifyInst(inst);
385 },
386
387 // control flow
388 .@"try", .try_cold => {
389 const pl_op = data[@intFromEnum(inst)].pl_op;
390 const extra = self.air.extraData(Air.Try, pl_op.payload);
391 const try_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
392
393 const cond_br_liveness = self.liveness.getCondBr(inst);
394
395 try self.verifyOperand(inst, pl_op.operand, self.liveness.operandDies(inst, 0));
396
397 var live = try self.live.clone(self.gpa);
398 defer live.deinit(self.gpa);
399
400 for (cond_br_liveness.else_deaths) |death| try self.verifyDeath(inst, death);
401 try self.verifyBody(try_body);
402
403 self.live.deinit(self.gpa);
404 self.live = live.move();
405
406 for (cond_br_liveness.then_deaths) |death| try self.verifyDeath(inst, death);
407
408 try self.verifyInst(inst);
409 },
410 .try_ptr, .try_ptr_cold => {
411 const ty_pl = data[@intFromEnum(inst)].ty_pl;
412 const extra = self.air.extraData(Air.TryPtr, ty_pl.payload);
413 const try_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
414
415 const cond_br_liveness = self.liveness.getCondBr(inst);
416
417 try self.verifyOperand(inst, extra.data.ptr, self.liveness.operandDies(inst, 0));
418
419 var live = try self.live.clone(self.gpa);
420 defer live.deinit(self.gpa);
421
422 for (cond_br_liveness.else_deaths) |death| try self.verifyDeath(inst, death);
423 try self.verifyBody(try_body);
424
425 self.live.deinit(self.gpa);
426 self.live = live.move();
427
428 for (cond_br_liveness.then_deaths) |death| try self.verifyDeath(inst, death);
429
430 try self.verifyInst(inst);
431 },
432 .br => {
433 const br = data[@intFromEnum(inst)].br;
434 const gop = try self.blocks.getOrPut(self.gpa, br.block_inst);
435
436 try self.verifyOperand(inst, br.operand, self.liveness.operandDies(inst, 0));
437 if (gop.found_existing) {
438 try self.verifyMatchingLiveness(br.block_inst, gop.value_ptr.*);
439 } else {
440 gop.value_ptr.* = try self.live.clone(self.gpa);
441 }
442 try self.verifyInst(inst);
443 },
444 .repeat => {
445 const repeat = data[@intFromEnum(inst)].repeat;
446 const expected_live = self.loops.get(repeat.loop_inst) orelse
447 return invalid("%{}: loop %{} not in scope", .{ @intFromEnum(inst), @intFromEnum(repeat.loop_inst) });
448
449 try self.verifyMatchingLiveness(repeat.loop_inst, expected_live);
450 },
451 .switch_dispatch => {
452 const br = data[@intFromEnum(inst)].br;
453
454 try self.verifyOperand(inst, br.operand, self.liveness.operandDies(inst, 0));
455
456 const expected_live = self.loops.get(br.block_inst) orelse
457 return invalid("%{}: loop %{} not in scope", .{ @intFromEnum(inst), @intFromEnum(br.block_inst) });
458
459 try self.verifyMatchingLiveness(br.block_inst, expected_live);
460 },
461 .block, .dbg_inline_block => |tag| {
462 const ty_pl = data[@intFromEnum(inst)].ty_pl;
463 const block_ty = ty_pl.ty.toType();
464 const block_body: []const Air.Inst.Index = @ptrCast(switch (tag) {
465 inline .block, .dbg_inline_block => |comptime_tag| body: {
466 const extra = self.air.extraData(switch (comptime_tag) {
467 .block => Air.Block,
468 .dbg_inline_block => Air.DbgInlineBlock,
469 else => unreachable,
470 }, ty_pl.payload);
471 break :body self.air.extra.items[extra.end..][0..extra.data.body_len];
472 },
473 else => unreachable,
474 });
475 const block_liveness = self.liveness.getBlock(inst);
476
477 var orig_live = try self.live.clone(self.gpa);
478 defer orig_live.deinit(self.gpa);
479
480 assert(!self.blocks.contains(inst));
481 try self.verifyBody(block_body);
482
483 // Liveness data after the block body is garbage, but we want to
484 // restore it to verify deaths
485 self.live.deinit(self.gpa);
486 self.live = orig_live.move();
487
488 for (block_liveness.deaths) |death| try self.verifyDeath(inst, death);
489
490 if (ip.isNoReturn(block_ty.toIntern())) {
491 assert(!self.blocks.contains(inst));
492 } else {
493 var live = self.blocks.fetchRemove(inst).?.value;
494 defer live.deinit(self.gpa);
495
496 try self.verifyMatchingLiveness(inst, live);
497 }
498
499 try self.verifyInstOperands(inst, .{ .none, .none, .none });
500 },
501 .loop => {
502 const ty_pl = data[@intFromEnum(inst)].ty_pl;
503 const extra = self.air.extraData(Air.Block, ty_pl.payload);
504 const loop_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
505
506 // The same stuff should be alive after the loop as before it.
507 const gop = try self.loops.getOrPut(self.gpa, inst);
508 if (gop.found_existing) return invalid("%{}: loop already exists", .{@intFromEnum(inst)});
509 defer {
510 var live = self.loops.fetchRemove(inst).?;
511 live.value.deinit(self.gpa);
512 }
513 gop.value_ptr.* = try self.live.clone(self.gpa);
514
515 try self.verifyBody(loop_body);
516
517 try self.verifyInstOperands(inst, .{ .none, .none, .none });
518 },
519 .cond_br => {
520 const pl_op = data[@intFromEnum(inst)].pl_op;
521 const extra = self.air.extraData(Air.CondBr, pl_op.payload);
522 const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.then_body_len]);
523 const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]);
524 const cond_br_liveness = self.liveness.getCondBr(inst);
525
526 try self.verifyOperand(inst, pl_op.operand, self.liveness.operandDies(inst, 0));
527
528 var live = try self.live.clone(self.gpa);
529 defer live.deinit(self.gpa);
530
531 for (cond_br_liveness.then_deaths) |death| try self.verifyDeath(inst, death);
532 try self.verifyBody(then_body);
533
534 self.live.deinit(self.gpa);
535 self.live = live.move();
536
537 for (cond_br_liveness.else_deaths) |death| try self.verifyDeath(inst, death);
538 try self.verifyBody(else_body);
539
540 try self.verifyInst(inst);
541 },
542 .switch_br, .loop_switch_br => {
543 const switch_br = self.air.unwrapSwitch(inst);
544 const switch_br_liveness = try self.liveness.getSwitchBr(
545 self.gpa,
546 inst,
547 switch_br.cases_len + 1,
548 );
549 defer self.gpa.free(switch_br_liveness.deaths);
550
551 try self.verifyOperand(inst, switch_br.operand, self.liveness.operandDies(inst, 0));
552
553 // Excluding the operand (which we just handled), the same stuff should be alive
554 // after the loop as before it.
555 {
556 const gop = try self.loops.getOrPut(self.gpa, inst);
557 if (gop.found_existing) return invalid("%{}: loop already exists", .{@intFromEnum(inst)});
558 gop.value_ptr.* = self.live.move();
559 }
560 defer {
561 var live = self.loops.fetchRemove(inst).?;
562 live.value.deinit(self.gpa);
563 }
564
565 var it = switch_br.iterateCases();
566 while (it.next()) |case| {
567 self.live.deinit(self.gpa);
568 self.live = try self.loops.get(inst).?.clone(self.gpa);
569
570 for (switch_br_liveness.deaths[case.idx]) |death| try self.verifyDeath(inst, death);
571 try self.verifyBody(case.body);
572 }
573
574 const else_body = it.elseBody();
575 if (else_body.len > 0) {
576 self.live.deinit(self.gpa);
577 self.live = try self.loops.get(inst).?.clone(self.gpa);
578 for (switch_br_liveness.deaths[switch_br.cases_len]) |death| try self.verifyDeath(inst, death);
579 try self.verifyBody(else_body);
580 }
581
582 try self.verifyInst(inst);
583 },
584 }
585 }
586}
587
588fn verifyDeath(self: *Verify, inst: Air.Inst.Index, operand: Air.Inst.Index) Error!void {
589 try self.verifyOperand(inst, operand.toRef(), true);
590}
591
592fn verifyOperand(self: *Verify, inst: Air.Inst.Index, op_ref: Air.Inst.Ref, dies: bool) Error!void {
593 const operand = op_ref.toIndexAllowNone() orelse {
594 assert(!dies);
595 return;
596 };
597 if (dies) {
598 if (!self.live.remove(operand)) return invalid("%{}: dead operand %{} reused and killed again", .{ inst, operand });
599 } else {
600 if (!self.live.contains(operand)) return invalid("%{}: dead operand %{} reused", .{ inst, operand });
601 }
602}
603
604fn verifyInstOperands(
605 self: *Verify,
606 inst: Air.Inst.Index,
607 operands: [Liveness.bpi - 1]Air.Inst.Ref,
608) Error!void {
609 for (operands, 0..) |operand, operand_index| {
610 const dies = self.liveness.operandDies(inst, @as(Liveness.OperandInt, @intCast(operand_index)));
611 try self.verifyOperand(inst, operand, dies);
612 }
613 try self.verifyInst(inst);
614}
615
616fn verifyInst(self: *Verify, inst: Air.Inst.Index) Error!void {
617 if (self.liveness.isUnused(inst)) {
618 assert(!self.live.contains(inst));
619 } else {
620 try self.live.putNoClobber(self.gpa, inst, {});
621 }
622}
623
624fn verifyMatchingLiveness(self: *Verify, block: Air.Inst.Index, live: LiveMap) Error!void {
625 if (self.live.count() != live.count()) return invalid("%{}: different deaths across branches", .{block});
626 var live_it = self.live.keyIterator();
627 while (live_it.next()) |live_inst| if (!live.contains(live_inst.*)) return invalid("%{}: different deaths across branches", .{block});
628}
629
630fn invalid(comptime fmt: []const u8, args: anytype) error{LivenessInvalid} {
631 log.err(fmt, args);
632 return error.LivenessInvalid;
633}
634
635const std = @import("std");
636const assert = std.debug.assert;
637const log = std.log.scoped(.liveness_verify);
638
639const Air = @import("../../Air.zig");
640const Liveness = @import("../Liveness.zig");
641const InternPool = @import("../../InternPool.zig");
642const Verify = @This();
src/Air/types_resolved.zig+10-10
......@@ -171,7 +171,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
171171 if (!checkType(data.ty_pl.ty.toType(), zcu)) return false;
172172 if (!checkBody(
173173 air,
174 @ptrCast(air.extra[extra.end..][0..extra.data.body_len]),
174 @ptrCast(air.extra.items[extra.end..][0..extra.data.body_len]),
175175 zcu,
176176 )) return false;
177177 },
......@@ -181,7 +181,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
181181 if (!checkType(data.ty_pl.ty.toType(), zcu)) return false;
182182 if (!checkBody(
183183 air,
184 @ptrCast(air.extra[extra.end..][0..extra.data.body_len]),
184 @ptrCast(air.extra.items[extra.end..][0..extra.data.body_len]),
185185 zcu,
186186 )) return false;
187187 },
......@@ -270,7 +270,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
270270 .aggregate_init => {
271271 const ty = data.ty_pl.ty.toType();
272272 const elems_len: usize = @intCast(ty.arrayLen(zcu));
273 const elems: []const Air.Inst.Ref = @ptrCast(air.extra[data.ty_pl.payload..][0..elems_len]);
273 const elems: []const Air.Inst.Ref = @ptrCast(air.extra.items[data.ty_pl.payload..][0..elems_len]);
274274 if (!checkType(ty, zcu)) return false;
275275 if (ty.zigTypeTag(zcu) == .@"struct") {
276276 for (elems, 0..) |elem, elem_idx| {
......@@ -336,7 +336,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
336336 .call_never_inline,
337337 => {
338338 const extra = air.extraData(Air.Call, data.pl_op.payload);
339 const args: []const Air.Inst.Ref = @ptrCast(air.extra[extra.end..][0..extra.data.args_len]);
339 const args: []const Air.Inst.Ref = @ptrCast(air.extra.items[extra.end..][0..extra.data.args_len]);
340340 if (!checkRef(data.pl_op.operand, zcu)) return false;
341341 for (args) |arg| if (!checkRef(arg, zcu)) return false;
342342 },
......@@ -353,7 +353,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
353353 if (!checkRef(data.pl_op.operand, zcu)) return false;
354354 if (!checkBody(
355355 air,
356 @ptrCast(air.extra[extra.end..][0..extra.data.body_len]),
356 @ptrCast(air.extra.items[extra.end..][0..extra.data.body_len]),
357357 zcu,
358358 )) return false;
359359 },
......@@ -364,7 +364,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
364364 if (!checkRef(extra.data.ptr, zcu)) return false;
365365 if (!checkBody(
366366 air,
367 @ptrCast(air.extra[extra.end..][0..extra.data.body_len]),
367 @ptrCast(air.extra.items[extra.end..][0..extra.data.body_len]),
368368 zcu,
369369 )) return false;
370370 },
......@@ -374,12 +374,12 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
374374 if (!checkRef(data.pl_op.operand, zcu)) return false;
375375 if (!checkBody(
376376 air,
377 @ptrCast(air.extra[extra.end..][0..extra.data.then_body_len]),
377 @ptrCast(air.extra.items[extra.end..][0..extra.data.then_body_len]),
378378 zcu,
379379 )) return false;
380380 if (!checkBody(
381381 air,
382 @ptrCast(air.extra[extra.end + extra.data.then_body_len ..][0..extra.data.else_body_len]),
382 @ptrCast(air.extra.items[extra.end + extra.data.then_body_len ..][0..extra.data.else_body_len]),
383383 zcu,
384384 )) return false;
385385 },
......@@ -404,8 +404,8 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
404404 if (!checkType(data.ty_pl.ty.toType(), zcu)) return false;
405405 // Luckily, we only care about the inputs and outputs, so we don't have to do
406406 // the whole null-terminated string dance.
407 const outputs: []const Air.Inst.Ref = @ptrCast(air.extra[extra.end..][0..extra.data.outputs_len]);
408 const inputs: []const Air.Inst.Ref = @ptrCast(air.extra[extra.end + extra.data.outputs_len ..][0..extra.data.inputs_len]);
407 const outputs: []const Air.Inst.Ref = @ptrCast(air.extra.items[extra.end..][0..extra.data.outputs_len]);
408 const inputs: []const Air.Inst.Ref = @ptrCast(air.extra.items[extra.end + extra.data.outputs_len ..][0..extra.data.inputs_len]);
409409 for (outputs) |output| if (output != .none and !checkRef(output, zcu)) return false;
410410 for (inputs) |input| if (input != .none and !checkRef(input, zcu)) return false;
411411 },
src/Liveness.zig deleted-2050
......@@ -1,2050 +0,0 @@
1//! For each AIR instruction, we want to know:
2//! * Is the instruction unreferenced (e.g. dies immediately)?
3//! * For each of its operands, does the operand die with this instruction (e.g. is
4//! this the last reference to it)?
5//! Some instructions are special, such as:
6//! * Conditional Branches
7//! * Switch Branches
8const std = @import("std");
9const log = std.log.scoped(.liveness);
10const assert = std.debug.assert;
11const Allocator = std.mem.Allocator;
12const Log2Int = std.math.Log2Int;
13
14const Liveness = @This();
15const trace = @import("tracy.zig").trace;
16const Air = @import("Air.zig");
17const InternPool = @import("InternPool.zig");
18
19pub const Verify = @import("Liveness/Verify.zig");
20
21/// This array is split into sets of 4 bits per AIR instruction.
22/// The MSB (0bX000) is whether the instruction is unreferenced.
23/// The LSB (0b000X) is the first operand, and so on, up to 3 operands. A set bit means the
24/// operand dies after this instruction.
25/// Instructions which need more data to track liveness have special handling via the
26/// `special` table.
27tomb_bits: []usize,
28/// Sparse table of specially handled instructions. The value is an index into the `extra`
29/// array. The meaning of the data depends on the AIR tag.
30/// * `cond_br` - points to a `CondBr` in `extra` at this index.
31/// * `try`, `try_ptr` - points to a `CondBr` in `extra` at this index. The error path (the block
32/// in the instruction) is considered the "else" path, and the rest of the block the "then".
33/// * `switch_br` - points to a `SwitchBr` in `extra` at this index.
34/// * `loop_switch_br` - points to a `SwitchBr` in `extra` at this index.
35/// * `block` - points to a `Block` in `extra` at this index.
36/// * `asm`, `call`, `aggregate_init` - the value is a set of bits which are the extra tomb
37/// bits of operands.
38/// The main tomb bits are still used and the extra ones are starting with the lsb of the
39/// value here.
40special: std.AutoHashMapUnmanaged(Air.Inst.Index, u32),
41/// Auxiliary data. The way this data is interpreted is determined contextually.
42extra: []const u32,
43
44/// Trailing is the set of instructions whose lifetimes end at the start of the then branch,
45/// followed by the set of instructions whose lifetimes end at the start of the else branch.
46pub const CondBr = struct {
47 then_death_count: u32,
48 else_death_count: u32,
49};
50
51/// Trailing is:
52/// * For each case in the same order as in the AIR:
53/// - case_death_count: u32
54/// - Air.Inst.Index for each `case_death_count`: set of instructions whose lifetimes
55/// end at the start of this case.
56/// * Air.Inst.Index for each `else_death_count`: set of instructions whose lifetimes
57/// end at the start of the else case.
58pub const SwitchBr = struct {
59 else_death_count: u32,
60};
61
62/// Trailing is the set of instructions which die in the block. Note that these are not additional
63/// deaths (they are all recorded as normal within the block), but backends may use this information
64/// as a more efficient way to track which instructions are still alive after a block.
65pub const Block = struct {
66 death_count: u32,
67};
68
69/// Liveness analysis runs in several passes. Each pass iterates backwards over instructions in
70/// bodies, and recurses into bodies.
71const LivenessPass = enum {
72 /// In this pass, we perform some basic analysis of loops to gain information the main pass needs.
73 /// In particular, for every `loop` and `loop_switch_br`, we track the following information:
74 /// * Every outer block which the loop body contains a `br` to.
75 /// * Every outer loop which the loop body contains a `repeat` to.
76 /// * Every operand referenced within the loop body but created outside the loop.
77 /// This gives the main analysis pass enough information to determine the full set of
78 /// instructions which need to be alive when a loop repeats. This data is TEMPORARILY stored in
79 /// `a.extra`. It is not re-added to `extra` by the main pass, since it is not useful to
80 /// backends.
81 loop_analysis,
82
83 /// This pass performs the main liveness analysis, setting up tombs and extra data while
84 /// considering control flow etc.
85 main_analysis,
86};
87
88/// Each analysis pass may wish to pass data through calls. A pointer to a `LivenessPassData(pass)`
89/// stored on the stack is passed through calls to `analyzeInst` etc.
90fn LivenessPassData(comptime pass: LivenessPass) type {
91 return switch (pass) {
92 .loop_analysis => struct {
93 /// The set of blocks which are exited with a `br` instruction at some point within this
94 /// body and which we are currently within. Also includes `loop`s which are the target
95 /// of a `repeat` instruction, and `loop_switch_br`s which are the target of a
96 /// `switch_dispatch` instruction.
97 breaks: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .empty,
98
99 /// The set of operands for which we have seen at least one usage but not their birth.
100 live_set: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .empty,
101
102 fn deinit(self: *@This(), gpa: Allocator) void {
103 self.breaks.deinit(gpa);
104 self.live_set.deinit(gpa);
105 }
106 },
107
108 .main_analysis => struct {
109 /// Every `block` and `loop` currently under analysis.
110 block_scopes: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockScope) = .empty,
111
112 /// The set of instructions currently alive in the current control
113 /// flow branch.
114 live_set: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .empty,
115
116 /// The extra data initialized by the `loop_analysis` pass for this pass to consume.
117 /// Owned by this struct during this pass.
118 old_extra: std.ArrayListUnmanaged(u32) = .empty,
119
120 const BlockScope = struct {
121 /// If this is a `block`, these instructions are alive upon a `br` to this block.
122 /// If this is a `loop`, these instructions are alive upon a `repeat` to this block.
123 live_set: std.AutoHashMapUnmanaged(Air.Inst.Index, void),
124 };
125
126 fn deinit(self: *@This(), gpa: Allocator) void {
127 var it = self.block_scopes.valueIterator();
128 while (it.next()) |block| {
129 block.live_set.deinit(gpa);
130 }
131 self.block_scopes.deinit(gpa);
132 self.live_set.deinit(gpa);
133 self.old_extra.deinit(gpa);
134 }
135 },
136 };
137}
138
139pub fn analyze(gpa: Allocator, air: Air, intern_pool: *InternPool) Allocator.Error!Liveness {
140 const tracy = trace(@src());
141 defer tracy.end();
142
143 var a: Analysis = .{
144 .gpa = gpa,
145 .air = air,
146 .tomb_bits = try gpa.alloc(
147 usize,
148 (air.instructions.len * bpi + @bitSizeOf(usize) - 1) / @bitSizeOf(usize),
149 ),
150 .extra = .{},
151 .special = .{},
152 .intern_pool = intern_pool,
153 };
154 errdefer gpa.free(a.tomb_bits);
155 errdefer a.special.deinit(gpa);
156 defer a.extra.deinit(gpa);
157
158 @memset(a.tomb_bits, 0);
159
160 const main_body = air.getMainBody();
161
162 {
163 var data: LivenessPassData(.loop_analysis) = .{};
164 defer data.deinit(gpa);
165 try analyzeBody(&a, .loop_analysis, &data, main_body);
166 }
167
168 {
169 var data: LivenessPassData(.main_analysis) = .{};
170 defer data.deinit(gpa);
171 data.old_extra = a.extra;
172 a.extra = .{};
173 try analyzeBody(&a, .main_analysis, &data, main_body);
174 assert(data.live_set.count() == 0);
175 }
176
177 return .{
178 .tomb_bits = a.tomb_bits,
179 .special = a.special,
180 .extra = try a.extra.toOwnedSlice(gpa),
181 };
182}
183
184pub fn getTombBits(l: Liveness, inst: Air.Inst.Index) Bpi {
185 const usize_index = (@intFromEnum(inst) * bpi) / @bitSizeOf(usize);
186 return @as(Bpi, @truncate(l.tomb_bits[usize_index] >>
187 @as(Log2Int(usize), @intCast((@intFromEnum(inst) % (@bitSizeOf(usize) / bpi)) * bpi))));
188}
189
190pub fn isUnused(l: Liveness, inst: Air.Inst.Index) bool {
191 const usize_index = (@intFromEnum(inst) * bpi) / @bitSizeOf(usize);
192 const mask = @as(usize, 1) <<
193 @as(Log2Int(usize), @intCast((@intFromEnum(inst) % (@bitSizeOf(usize) / bpi)) * bpi + (bpi - 1)));
194 return (l.tomb_bits[usize_index] & mask) != 0;
195}
196
197pub fn operandDies(l: Liveness, inst: Air.Inst.Index, operand: OperandInt) bool {
198 assert(operand < bpi - 1);
199 const usize_index = (@intFromEnum(inst) * bpi) / @bitSizeOf(usize);
200 const mask = @as(usize, 1) <<
201 @as(Log2Int(usize), @intCast((@intFromEnum(inst) % (@bitSizeOf(usize) / bpi)) * bpi + operand));
202 return (l.tomb_bits[usize_index] & mask) != 0;
203}
204
205const OperandCategory = enum {
206 /// The operand lives on, but this instruction cannot possibly mutate memory.
207 none,
208 /// The operand lives on and this instruction can mutate memory.
209 write,
210 /// The operand dies at this instruction.
211 tomb,
212 /// The operand lives on, and this instruction is noreturn.
213 noret,
214 /// This instruction is too complicated for analysis, no information is available.
215 complex,
216};
217
218/// Given an instruction that we are examining, and an operand that we are looking for,
219/// returns a classification.
220pub fn categorizeOperand(
221 l: Liveness,
222 air: Air,
223 inst: Air.Inst.Index,
224 operand: Air.Inst.Index,
225 ip: *const InternPool,
226) OperandCategory {
227 const air_tags = air.instructions.items(.tag);
228 const air_datas = air.instructions.items(.data);
229 const operand_ref = operand.toRef();
230 switch (air_tags[@intFromEnum(inst)]) {
231 .add,
232 .add_safe,
233 .add_wrap,
234 .add_sat,
235 .add_optimized,
236 .sub,
237 .sub_safe,
238 .sub_wrap,
239 .sub_sat,
240 .sub_optimized,
241 .mul,
242 .mul_safe,
243 .mul_wrap,
244 .mul_sat,
245 .mul_optimized,
246 .div_float,
247 .div_trunc,
248 .div_floor,
249 .div_exact,
250 .rem,
251 .mod,
252 .bit_and,
253 .bit_or,
254 .xor,
255 .cmp_lt,
256 .cmp_lte,
257 .cmp_eq,
258 .cmp_gte,
259 .cmp_gt,
260 .cmp_neq,
261 .bool_and,
262 .bool_or,
263 .array_elem_val,
264 .slice_elem_val,
265 .ptr_elem_val,
266 .shl,
267 .shl_exact,
268 .shl_sat,
269 .shr,
270 .shr_exact,
271 .min,
272 .max,
273 .div_float_optimized,
274 .div_trunc_optimized,
275 .div_floor_optimized,
276 .div_exact_optimized,
277 .rem_optimized,
278 .mod_optimized,
279 .neg_optimized,
280 .cmp_lt_optimized,
281 .cmp_lte_optimized,
282 .cmp_eq_optimized,
283 .cmp_gte_optimized,
284 .cmp_gt_optimized,
285 .cmp_neq_optimized,
286 => {
287 const o = air_datas[@intFromEnum(inst)].bin_op;
288 if (o.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
289 if (o.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
290 return .none;
291 },
292
293 .store,
294 .store_safe,
295 .atomic_store_unordered,
296 .atomic_store_monotonic,
297 .atomic_store_release,
298 .atomic_store_seq_cst,
299 .set_union_tag,
300 .memset,
301 .memset_safe,
302 .memcpy,
303 .memmove,
304 => {
305 const o = air_datas[@intFromEnum(inst)].bin_op;
306 if (o.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
307 if (o.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .write);
308 return .write;
309 },
310
311 .vector_store_elem => {
312 const o = air_datas[@intFromEnum(inst)].vector_store_elem;
313 const extra = air.extraData(Air.Bin, o.payload).data;
314 if (o.vector_ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
315 if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
316 if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 2, .none);
317 return .write;
318 },
319
320 .arg,
321 .alloc,
322 .inferred_alloc,
323 .inferred_alloc_comptime,
324 .ret_ptr,
325 .trap,
326 .breakpoint,
327 .repeat,
328 .switch_dispatch,
329 .dbg_stmt,
330 .dbg_empty_stmt,
331 .unreach,
332 .ret_addr,
333 .frame_addr,
334 .wasm_memory_size,
335 .err_return_trace,
336 .save_err_return_trace_index,
337 .tlv_dllimport_ptr,
338 .c_va_start,
339 .work_item_id,
340 .work_group_size,
341 .work_group_id,
342 => return .none,
343
344 .not,
345 .bitcast,
346 .load,
347 .fpext,
348 .fptrunc,
349 .intcast,
350 .intcast_safe,
351 .trunc,
352 .optional_payload,
353 .optional_payload_ptr,
354 .wrap_optional,
355 .unwrap_errunion_payload,
356 .unwrap_errunion_err,
357 .unwrap_errunion_payload_ptr,
358 .unwrap_errunion_err_ptr,
359 .wrap_errunion_payload,
360 .wrap_errunion_err,
361 .slice_ptr,
362 .slice_len,
363 .ptr_slice_len_ptr,
364 .ptr_slice_ptr_ptr,
365 .struct_field_ptr_index_0,
366 .struct_field_ptr_index_1,
367 .struct_field_ptr_index_2,
368 .struct_field_ptr_index_3,
369 .array_to_slice,
370 .int_from_float,
371 .int_from_float_optimized,
372 .float_from_int,
373 .get_union_tag,
374 .clz,
375 .ctz,
376 .popcount,
377 .byte_swap,
378 .bit_reverse,
379 .splat,
380 .error_set_has_value,
381 .addrspace_cast,
382 .c_va_arg,
383 .c_va_copy,
384 .abs,
385 => {
386 const o = air_datas[@intFromEnum(inst)].ty_op;
387 if (o.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
388 return .none;
389 },
390
391 .optional_payload_ptr_set,
392 .errunion_payload_ptr_set,
393 => {
394 const o = air_datas[@intFromEnum(inst)].ty_op;
395 if (o.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
396 return .write;
397 },
398
399 .is_null,
400 .is_non_null,
401 .is_null_ptr,
402 .is_non_null_ptr,
403 .is_err,
404 .is_non_err,
405 .is_err_ptr,
406 .is_non_err_ptr,
407 .is_named_enum_value,
408 .tag_name,
409 .error_name,
410 .sqrt,
411 .sin,
412 .cos,
413 .tan,
414 .exp,
415 .exp2,
416 .log,
417 .log2,
418 .log10,
419 .floor,
420 .ceil,
421 .round,
422 .trunc_float,
423 .neg,
424 .cmp_lt_errors_len,
425 .c_va_end,
426 => {
427 const o = air_datas[@intFromEnum(inst)].un_op;
428 if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
429 return .none;
430 },
431
432 .ret,
433 .ret_safe,
434 .ret_load,
435 => {
436 const o = air_datas[@intFromEnum(inst)].un_op;
437 if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .noret);
438 return .noret;
439 },
440
441 .set_err_return_trace => {
442 const o = air_datas[@intFromEnum(inst)].un_op;
443 if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
444 return .write;
445 },
446
447 .add_with_overflow,
448 .sub_with_overflow,
449 .mul_with_overflow,
450 .shl_with_overflow,
451 .ptr_add,
452 .ptr_sub,
453 .ptr_elem_ptr,
454 .slice_elem_ptr,
455 .slice,
456 => {
457 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
458 const extra = air.extraData(Air.Bin, ty_pl.payload).data;
459 if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
460 if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
461 return .none;
462 },
463
464 .dbg_var_ptr,
465 .dbg_var_val,
466 .dbg_arg_inline,
467 => {
468 const o = air_datas[@intFromEnum(inst)].pl_op.operand;
469 if (o == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
470 return .none;
471 },
472
473 .prefetch => {
474 const prefetch = air_datas[@intFromEnum(inst)].prefetch;
475 if (prefetch.ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
476 return .none;
477 },
478
479 .call, .call_always_tail, .call_never_tail, .call_never_inline => {
480 const inst_data = air_datas[@intFromEnum(inst)].pl_op;
481 const callee = inst_data.operand;
482 const extra = air.extraData(Air.Call, inst_data.payload);
483 const args = @as([]const Air.Inst.Ref, @ptrCast(air.extra[extra.end..][0..extra.data.args_len]));
484 if (args.len + 1 <= bpi - 1) {
485 if (callee == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
486 for (args, 0..) |arg, i| {
487 if (arg == operand_ref) return matchOperandSmallIndex(l, inst, @as(OperandInt, @intCast(i + 1)), .write);
488 }
489 return .write;
490 }
491 var bt = l.iterateBigTomb(inst);
492 if (bt.feed()) {
493 if (callee == operand_ref) return .tomb;
494 } else {
495 if (callee == operand_ref) return .write;
496 }
497 for (args) |arg| {
498 if (bt.feed()) {
499 if (arg == operand_ref) return .tomb;
500 } else {
501 if (arg == operand_ref) return .write;
502 }
503 }
504 return .write;
505 },
506 .select => {
507 const pl_op = air_datas[@intFromEnum(inst)].pl_op;
508 const extra = air.extraData(Air.Bin, pl_op.payload).data;
509 if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
510 if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
511 if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 2, .none);
512 return .none;
513 },
514 .shuffle => {
515 const extra = air.extraData(Air.Shuffle, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
516 if (extra.a == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
517 if (extra.b == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
518 return .none;
519 },
520 .reduce, .reduce_optimized => {
521 const reduce = air_datas[@intFromEnum(inst)].reduce;
522 if (reduce.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
523 return .none;
524 },
525 .cmp_vector, .cmp_vector_optimized => {
526 const extra = air.extraData(Air.VectorCmp, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
527 if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
528 if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
529 return .none;
530 },
531 .aggregate_init => {
532 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
533 const aggregate_ty = ty_pl.ty.toType();
534 const len = @as(usize, @intCast(aggregate_ty.arrayLenIp(ip)));
535 const elements = @as([]const Air.Inst.Ref, @ptrCast(air.extra[ty_pl.payload..][0..len]));
536
537 if (elements.len <= bpi - 1) {
538 for (elements, 0..) |elem, i| {
539 if (elem == operand_ref) return matchOperandSmallIndex(l, inst, @as(OperandInt, @intCast(i)), .none);
540 }
541 return .none;
542 }
543
544 var bt = l.iterateBigTomb(inst);
545 for (elements) |elem| {
546 if (bt.feed()) {
547 if (elem == operand_ref) return .tomb;
548 } else {
549 if (elem == operand_ref) return .write;
550 }
551 }
552 return .write;
553 },
554 .union_init => {
555 const extra = air.extraData(Air.UnionInit, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
556 if (extra.init == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
557 return .none;
558 },
559 .struct_field_ptr, .struct_field_val => {
560 const extra = air.extraData(Air.StructField, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
561 if (extra.struct_operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
562 return .none;
563 },
564 .field_parent_ptr => {
565 const extra = air.extraData(Air.FieldParentPtr, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
566 if (extra.field_ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
567 return .none;
568 },
569 .cmpxchg_strong, .cmpxchg_weak => {
570 const extra = air.extraData(Air.Cmpxchg, air_datas[@intFromEnum(inst)].ty_pl.payload).data;
571 if (extra.ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
572 if (extra.expected_value == operand_ref) return matchOperandSmallIndex(l, inst, 1, .write);
573 if (extra.new_value == operand_ref) return matchOperandSmallIndex(l, inst, 2, .write);
574 return .write;
575 },
576 .mul_add => {
577 const pl_op = air_datas[@intFromEnum(inst)].pl_op;
578 const extra = air.extraData(Air.Bin, pl_op.payload).data;
579 if (extra.lhs == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
580 if (extra.rhs == operand_ref) return matchOperandSmallIndex(l, inst, 1, .none);
581 if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 2, .none);
582 return .none;
583 },
584 .atomic_load => {
585 const ptr = air_datas[@intFromEnum(inst)].atomic_load.ptr;
586 if (ptr == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
587 return .none;
588 },
589 .atomic_rmw => {
590 const pl_op = air_datas[@intFromEnum(inst)].pl_op;
591 const extra = air.extraData(Air.AtomicRmw, pl_op.payload).data;
592 if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .write);
593 if (extra.operand == operand_ref) return matchOperandSmallIndex(l, inst, 1, .write);
594 return .write;
595 },
596
597 .br => {
598 const br = air_datas[@intFromEnum(inst)].br;
599 if (br.operand == operand_ref) return matchOperandSmallIndex(l, operand, 0, .noret);
600 return .noret;
601 },
602 .assembly => {
603 return .complex;
604 },
605 .block, .dbg_inline_block => |tag| {
606 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
607 const body: []const Air.Inst.Index = @ptrCast(switch (tag) {
608 inline .block, .dbg_inline_block => |comptime_tag| body: {
609 const extra = air.extraData(switch (comptime_tag) {
610 .block => Air.Block,
611 .dbg_inline_block => Air.DbgInlineBlock,
612 else => unreachable,
613 }, ty_pl.payload);
614 break :body air.extra[extra.end..][0..extra.data.body_len];
615 },
616 else => unreachable,
617 });
618
619 if (body.len == 1 and air_tags[@intFromEnum(body[0])] == .cond_br) {
620 // Peephole optimization for "panic-like" conditionals, which have
621 // one empty branch and another which calls a `noreturn` function.
622 // This allows us to infer that safety checks do not modify memory,
623 // as far as control flow successors are concerned.
624
625 const inst_data = air_datas[@intFromEnum(body[0])].pl_op;
626 const cond_extra = air.extraData(Air.CondBr, inst_data.payload);
627 if (inst_data.operand == operand_ref and operandDies(l, body[0], 0))
628 return .tomb;
629
630 if (cond_extra.data.then_body_len > 2 or cond_extra.data.else_body_len > 2)
631 return .complex;
632
633 const then_body: []const Air.Inst.Index = @ptrCast(air.extra[cond_extra.end..][0..cond_extra.data.then_body_len]);
634 const else_body: []const Air.Inst.Index = @ptrCast(air.extra[cond_extra.end + cond_extra.data.then_body_len ..][0..cond_extra.data.else_body_len]);
635 if (then_body.len > 1 and air_tags[@intFromEnum(then_body[1])] != .unreach)
636 return .complex;
637 if (else_body.len > 1 and air_tags[@intFromEnum(else_body[1])] != .unreach)
638 return .complex;
639
640 var operand_live: bool = true;
641 for (&[_]Air.Inst.Index{ then_body[0], else_body[0] }) |cond_inst| {
642 if (l.categorizeOperand(air, cond_inst, operand, ip) == .tomb)
643 operand_live = false;
644
645 switch (air_tags[@intFromEnum(cond_inst)]) {
646 .br => { // Breaks immediately back to block
647 const br = air_datas[@intFromEnum(cond_inst)].br;
648 if (br.block_inst != inst)
649 return .complex;
650 },
651 .call => {}, // Calls a noreturn function
652 else => return .complex,
653 }
654 }
655 return if (operand_live) .none else .tomb;
656 }
657
658 return .complex;
659 },
660
661 .@"try",
662 .try_cold,
663 .try_ptr,
664 .try_ptr_cold,
665 .loop,
666 .cond_br,
667 .switch_br,
668 .loop_switch_br,
669 => return .complex,
670
671 .wasm_memory_grow => {
672 const pl_op = air_datas[@intFromEnum(inst)].pl_op;
673 if (pl_op.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
674 return .none;
675 },
676 }
677}
678
679fn matchOperandSmallIndex(
680 l: Liveness,
681 inst: Air.Inst.Index,
682 operand: OperandInt,
683 default: OperandCategory,
684) OperandCategory {
685 if (operandDies(l, inst, operand)) {
686 return .tomb;
687 } else {
688 return default;
689 }
690}
691
692/// Higher level API.
693pub const CondBrSlices = struct {
694 then_deaths: []const Air.Inst.Index,
695 else_deaths: []const Air.Inst.Index,
696};
697
698pub fn getCondBr(l: Liveness, inst: Air.Inst.Index) CondBrSlices {
699 var index: usize = l.special.get(inst) orelse return .{
700 .then_deaths = &.{},
701 .else_deaths = &.{},
702 };
703 const then_death_count = l.extra[index];
704 index += 1;
705 const else_death_count = l.extra[index];
706 index += 1;
707 const then_deaths: []const Air.Inst.Index = @ptrCast(l.extra[index..][0..then_death_count]);
708 index += then_death_count;
709 return .{
710 .then_deaths = then_deaths,
711 .else_deaths = @ptrCast(l.extra[index..][0..else_death_count]),
712 };
713}
714
715/// Indexed by case number as they appear in AIR.
716/// Else is the last element.
717pub const SwitchBrTable = struct {
718 deaths: []const []const Air.Inst.Index,
719};
720
721/// Caller owns the memory.
722pub fn getSwitchBr(l: Liveness, gpa: Allocator, inst: Air.Inst.Index, cases_len: u32) Allocator.Error!SwitchBrTable {
723 var index: usize = l.special.get(inst) orelse return .{ .deaths = &.{} };
724 const else_death_count = l.extra[index];
725 index += 1;
726
727 var deaths = try gpa.alloc([]const Air.Inst.Index, cases_len);
728 errdefer gpa.free(deaths);
729
730 var case_i: u32 = 0;
731 while (case_i < cases_len - 1) : (case_i += 1) {
732 const case_death_count: u32 = l.extra[index];
733 index += 1;
734 deaths[case_i] = @ptrCast(l.extra[index..][0..case_death_count]);
735 index += case_death_count;
736 }
737 {
738 // Else
739 deaths[case_i] = @ptrCast(l.extra[index..][0..else_death_count]);
740 }
741 return .{ .deaths = deaths };
742}
743
744/// Note that this information is technically redundant, but is useful for
745/// backends nonetheless: see `Block`.
746pub const BlockSlices = struct {
747 deaths: []const Air.Inst.Index,
748};
749
750pub fn getBlock(l: Liveness, inst: Air.Inst.Index) BlockSlices {
751 const index: usize = l.special.get(inst) orelse return .{
752 .deaths = &.{},
753 };
754 const death_count = l.extra[index];
755 const deaths: []const Air.Inst.Index = @ptrCast(l.extra[index + 1 ..][0..death_count]);
756 return .{
757 .deaths = deaths,
758 };
759}
760
761pub const LoopSlice = struct {
762 deaths: []const Air.Inst.Index,
763};
764
765pub fn deinit(l: *Liveness, gpa: Allocator) void {
766 gpa.free(l.tomb_bits);
767 gpa.free(l.extra);
768 l.special.deinit(gpa);
769 l.* = undefined;
770}
771
772pub fn iterateBigTomb(l: Liveness, inst: Air.Inst.Index) BigTomb {
773 return .{
774 .tomb_bits = l.getTombBits(inst),
775 .extra_start = l.special.get(inst) orelse 0,
776 .extra_offset = 0,
777 .extra = l.extra,
778 .bit_index = 0,
779 .reached_end = false,
780 };
781}
782
783/// How many tomb bits per AIR instruction.
784pub const bpi = 4;
785pub const Bpi = std.meta.Int(.unsigned, bpi);
786pub const OperandInt = std.math.Log2Int(Bpi);
787
788/// Useful for decoders of Liveness information.
789pub const BigTomb = struct {
790 tomb_bits: Liveness.Bpi,
791 bit_index: u32,
792 extra_start: u32,
793 extra_offset: u32,
794 extra: []const u32,
795 reached_end: bool,
796
797 /// Returns whether the next operand dies.
798 pub fn feed(bt: *BigTomb) bool {
799 if (bt.reached_end) return false;
800
801 const this_bit_index = bt.bit_index;
802 bt.bit_index += 1;
803
804 const small_tombs = bpi - 1;
805 if (this_bit_index < small_tombs) {
806 const dies = @as(u1, @truncate(bt.tomb_bits >> @as(Liveness.OperandInt, @intCast(this_bit_index)))) != 0;
807 return dies;
808 }
809
810 const big_bit_index = this_bit_index - small_tombs;
811 while (big_bit_index - bt.extra_offset * 31 >= 31) {
812 if (@as(u1, @truncate(bt.extra[bt.extra_start + bt.extra_offset] >> 31)) != 0) {
813 bt.reached_end = true;
814 return false;
815 }
816 bt.extra_offset += 1;
817 }
818 const dies = @as(u1, @truncate(bt.extra[bt.extra_start + bt.extra_offset] >>
819 @as(u5, @intCast(big_bit_index - bt.extra_offset * 31)))) != 0;
820 return dies;
821 }
822};
823
824/// In-progress data; on successful analysis converted into `Liveness`.
825const Analysis = struct {
826 gpa: Allocator,
827 air: Air,
828 intern_pool: *InternPool,
829 tomb_bits: []usize,
830 special: std.AutoHashMapUnmanaged(Air.Inst.Index, u32),
831 extra: std.ArrayListUnmanaged(u32),
832
833 fn addExtra(a: *Analysis, extra: anytype) Allocator.Error!u32 {
834 const fields = std.meta.fields(@TypeOf(extra));
835 try a.extra.ensureUnusedCapacity(a.gpa, fields.len);
836 return addExtraAssumeCapacity(a, extra);
837 }
838
839 fn addExtraAssumeCapacity(a: *Analysis, extra: anytype) u32 {
840 const fields = std.meta.fields(@TypeOf(extra));
841 const result = @as(u32, @intCast(a.extra.items.len));
842 inline for (fields) |field| {
843 a.extra.appendAssumeCapacity(switch (field.type) {
844 u32 => @field(extra, field.name),
845 else => @compileError("bad field type"),
846 });
847 }
848 return result;
849 }
850};
851
852fn analyzeBody(
853 a: *Analysis,
854 comptime pass: LivenessPass,
855 data: *LivenessPassData(pass),
856 body: []const Air.Inst.Index,
857) Allocator.Error!void {
858 var i: usize = body.len;
859 while (i != 0) {
860 i -= 1;
861 const inst = body[i];
862 try analyzeInst(a, pass, data, inst);
863 }
864}
865
866fn analyzeInst(
867 a: *Analysis,
868 comptime pass: LivenessPass,
869 data: *LivenessPassData(pass),
870 inst: Air.Inst.Index,
871) Allocator.Error!void {
872 const ip = a.intern_pool;
873 const inst_tags = a.air.instructions.items(.tag);
874 const inst_datas = a.air.instructions.items(.data);
875
876 switch (inst_tags[@intFromEnum(inst)]) {
877 .add,
878 .add_safe,
879 .add_optimized,
880 .add_wrap,
881 .add_sat,
882 .sub,
883 .sub_safe,
884 .sub_optimized,
885 .sub_wrap,
886 .sub_sat,
887 .mul,
888 .mul_safe,
889 .mul_optimized,
890 .mul_wrap,
891 .mul_sat,
892 .div_float,
893 .div_float_optimized,
894 .div_trunc,
895 .div_trunc_optimized,
896 .div_floor,
897 .div_floor_optimized,
898 .div_exact,
899 .div_exact_optimized,
900 .rem,
901 .rem_optimized,
902 .mod,
903 .mod_optimized,
904 .bit_and,
905 .bit_or,
906 .xor,
907 .cmp_lt,
908 .cmp_lt_optimized,
909 .cmp_lte,
910 .cmp_lte_optimized,
911 .cmp_eq,
912 .cmp_eq_optimized,
913 .cmp_gte,
914 .cmp_gte_optimized,
915 .cmp_gt,
916 .cmp_gt_optimized,
917 .cmp_neq,
918 .cmp_neq_optimized,
919 .bool_and,
920 .bool_or,
921 .store,
922 .store_safe,
923 .array_elem_val,
924 .slice_elem_val,
925 .ptr_elem_val,
926 .shl,
927 .shl_exact,
928 .shl_sat,
929 .shr,
930 .shr_exact,
931 .atomic_store_unordered,
932 .atomic_store_monotonic,
933 .atomic_store_release,
934 .atomic_store_seq_cst,
935 .set_union_tag,
936 .min,
937 .max,
938 .memset,
939 .memset_safe,
940 .memcpy,
941 .memmove,
942 => {
943 const o = inst_datas[@intFromEnum(inst)].bin_op;
944 return analyzeOperands(a, pass, data, inst, .{ o.lhs, o.rhs, .none });
945 },
946
947 .vector_store_elem => {
948 const o = inst_datas[@intFromEnum(inst)].vector_store_elem;
949 const extra = a.air.extraData(Air.Bin, o.payload).data;
950 return analyzeOperands(a, pass, data, inst, .{ o.vector_ptr, extra.lhs, extra.rhs });
951 },
952
953 .arg,
954 .alloc,
955 .ret_ptr,
956 .breakpoint,
957 .dbg_stmt,
958 .dbg_empty_stmt,
959 .ret_addr,
960 .frame_addr,
961 .wasm_memory_size,
962 .err_return_trace,
963 .save_err_return_trace_index,
964 .tlv_dllimport_ptr,
965 .c_va_start,
966 .work_item_id,
967 .work_group_size,
968 .work_group_id,
969 => return analyzeOperands(a, pass, data, inst, .{ .none, .none, .none }),
970
971 .inferred_alloc, .inferred_alloc_comptime => unreachable,
972
973 .trap,
974 .unreach,
975 => return analyzeFuncEnd(a, pass, data, inst, .{ .none, .none, .none }),
976
977 .not,
978 .bitcast,
979 .load,
980 .fpext,
981 .fptrunc,
982 .intcast,
983 .intcast_safe,
984 .trunc,
985 .optional_payload,
986 .optional_payload_ptr,
987 .optional_payload_ptr_set,
988 .errunion_payload_ptr_set,
989 .wrap_optional,
990 .unwrap_errunion_payload,
991 .unwrap_errunion_err,
992 .unwrap_errunion_payload_ptr,
993 .unwrap_errunion_err_ptr,
994 .wrap_errunion_payload,
995 .wrap_errunion_err,
996 .slice_ptr,
997 .slice_len,
998 .ptr_slice_len_ptr,
999 .ptr_slice_ptr_ptr,
1000 .struct_field_ptr_index_0,
1001 .struct_field_ptr_index_1,
1002 .struct_field_ptr_index_2,
1003 .struct_field_ptr_index_3,
1004 .array_to_slice,
1005 .int_from_float,
1006 .int_from_float_optimized,
1007 .float_from_int,
1008 .get_union_tag,
1009 .clz,
1010 .ctz,
1011 .popcount,
1012 .byte_swap,
1013 .bit_reverse,
1014 .splat,
1015 .error_set_has_value,
1016 .addrspace_cast,
1017 .c_va_arg,
1018 .c_va_copy,
1019 .abs,
1020 => {
1021 const o = inst_datas[@intFromEnum(inst)].ty_op;
1022 return analyzeOperands(a, pass, data, inst, .{ o.operand, .none, .none });
1023 },
1024
1025 .is_null,
1026 .is_non_null,
1027 .is_null_ptr,
1028 .is_non_null_ptr,
1029 .is_err,
1030 .is_non_err,
1031 .is_err_ptr,
1032 .is_non_err_ptr,
1033 .is_named_enum_value,
1034 .tag_name,
1035 .error_name,
1036 .sqrt,
1037 .sin,
1038 .cos,
1039 .tan,
1040 .exp,
1041 .exp2,
1042 .log,
1043 .log2,
1044 .log10,
1045 .floor,
1046 .ceil,
1047 .round,
1048 .trunc_float,
1049 .neg,
1050 .neg_optimized,
1051 .cmp_lt_errors_len,
1052 .set_err_return_trace,
1053 .c_va_end,
1054 => {
1055 const operand = inst_datas[@intFromEnum(inst)].un_op;
1056 return analyzeOperands(a, pass, data, inst, .{ operand, .none, .none });
1057 },
1058
1059 .ret,
1060 .ret_safe,
1061 .ret_load,
1062 => {
1063 const operand = inst_datas[@intFromEnum(inst)].un_op;
1064 return analyzeFuncEnd(a, pass, data, inst, .{ operand, .none, .none });
1065 },
1066
1067 .add_with_overflow,
1068 .sub_with_overflow,
1069 .mul_with_overflow,
1070 .shl_with_overflow,
1071 .ptr_add,
1072 .ptr_sub,
1073 .ptr_elem_ptr,
1074 .slice_elem_ptr,
1075 .slice,
1076 => {
1077 const ty_pl = inst_datas[@intFromEnum(inst)].ty_pl;
1078 const extra = a.air.extraData(Air.Bin, ty_pl.payload).data;
1079 return analyzeOperands(a, pass, data, inst, .{ extra.lhs, extra.rhs, .none });
1080 },
1081
1082 .dbg_var_ptr,
1083 .dbg_var_val,
1084 .dbg_arg_inline,
1085 => {
1086 const operand = inst_datas[@intFromEnum(inst)].pl_op.operand;
1087 return analyzeOperands(a, pass, data, inst, .{ operand, .none, .none });
1088 },
1089
1090 .prefetch => {
1091 const prefetch = inst_datas[@intFromEnum(inst)].prefetch;
1092 return analyzeOperands(a, pass, data, inst, .{ prefetch.ptr, .none, .none });
1093 },
1094
1095 .call, .call_always_tail, .call_never_tail, .call_never_inline => {
1096 const inst_data = inst_datas[@intFromEnum(inst)].pl_op;
1097 const callee = inst_data.operand;
1098 const extra = a.air.extraData(Air.Call, inst_data.payload);
1099 const args = @as([]const Air.Inst.Ref, @ptrCast(a.air.extra[extra.end..][0..extra.data.args_len]));
1100 if (args.len + 1 <= bpi - 1) {
1101 var buf = [1]Air.Inst.Ref{.none} ** (bpi - 1);
1102 buf[0] = callee;
1103 @memcpy(buf[1..][0..args.len], args);
1104 return analyzeOperands(a, pass, data, inst, buf);
1105 }
1106
1107 var big = try AnalyzeBigOperands(pass).init(a, data, inst, args.len + 1);
1108 defer big.deinit();
1109 var i: usize = args.len;
1110 while (i > 0) {
1111 i -= 1;
1112 try big.feed(args[i]);
1113 }
1114 try big.feed(callee);
1115 return big.finish();
1116 },
1117 .select => {
1118 const pl_op = inst_datas[@intFromEnum(inst)].pl_op;
1119 const extra = a.air.extraData(Air.Bin, pl_op.payload).data;
1120 return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, extra.lhs, extra.rhs });
1121 },
1122 .shuffle => {
1123 const extra = a.air.extraData(Air.Shuffle, inst_datas[@intFromEnum(inst)].ty_pl.payload).data;
1124 return analyzeOperands(a, pass, data, inst, .{ extra.a, extra.b, .none });
1125 },
1126 .reduce, .reduce_optimized => {
1127 const reduce = inst_datas[@intFromEnum(inst)].reduce;
1128 return analyzeOperands(a, pass, data, inst, .{ reduce.operand, .none, .none });
1129 },
1130 .cmp_vector, .cmp_vector_optimized => {
1131 const extra = a.air.extraData(Air.VectorCmp, inst_datas[@intFromEnum(inst)].ty_pl.payload).data;
1132 return analyzeOperands(a, pass, data, inst, .{ extra.lhs, extra.rhs, .none });
1133 },
1134 .aggregate_init => {
1135 const ty_pl = inst_datas[@intFromEnum(inst)].ty_pl;
1136 const aggregate_ty = ty_pl.ty.toType();
1137 const len = @as(usize, @intCast(aggregate_ty.arrayLenIp(ip)));
1138 const elements = @as([]const Air.Inst.Ref, @ptrCast(a.air.extra[ty_pl.payload..][0..len]));
1139
1140 if (elements.len <= bpi - 1) {
1141 var buf = [1]Air.Inst.Ref{.none} ** (bpi - 1);
1142 @memcpy(buf[0..elements.len], elements);
1143 return analyzeOperands(a, pass, data, inst, buf);
1144 }
1145
1146 var big = try AnalyzeBigOperands(pass).init(a, data, inst, elements.len);
1147 defer big.deinit();
1148 var i: usize = elements.len;
1149 while (i > 0) {
1150 i -= 1;
1151 try big.feed(elements[i]);
1152 }
1153 return big.finish();
1154 },
1155 .union_init => {
1156 const extra = a.air.extraData(Air.UnionInit, inst_datas[@intFromEnum(inst)].ty_pl.payload).data;
1157 return analyzeOperands(a, pass, data, inst, .{ extra.init, .none, .none });
1158 },
1159 .struct_field_ptr, .struct_field_val => {
1160 const extra = a.air.extraData(Air.StructField, inst_datas[@intFromEnum(inst)].ty_pl.payload).data;
1161 return analyzeOperands(a, pass, data, inst, .{ extra.struct_operand, .none, .none });
1162 },
1163 .field_parent_ptr => {
1164 const extra = a.air.extraData(Air.FieldParentPtr, inst_datas[@intFromEnum(inst)].ty_pl.payload).data;
1165 return analyzeOperands(a, pass, data, inst, .{ extra.field_ptr, .none, .none });
1166 },
1167 .cmpxchg_strong, .cmpxchg_weak => {
1168 const extra = a.air.extraData(Air.Cmpxchg, inst_datas[@intFromEnum(inst)].ty_pl.payload).data;
1169 return analyzeOperands(a, pass, data, inst, .{ extra.ptr, extra.expected_value, extra.new_value });
1170 },
1171 .mul_add => {
1172 const pl_op = inst_datas[@intFromEnum(inst)].pl_op;
1173 const extra = a.air.extraData(Air.Bin, pl_op.payload).data;
1174 return analyzeOperands(a, pass, data, inst, .{ extra.lhs, extra.rhs, pl_op.operand });
1175 },
1176 .atomic_load => {
1177 const ptr = inst_datas[@intFromEnum(inst)].atomic_load.ptr;
1178 return analyzeOperands(a, pass, data, inst, .{ ptr, .none, .none });
1179 },
1180 .atomic_rmw => {
1181 const pl_op = inst_datas[@intFromEnum(inst)].pl_op;
1182 const extra = a.air.extraData(Air.AtomicRmw, pl_op.payload).data;
1183 return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, extra.operand, .none });
1184 },
1185
1186 .br => return analyzeInstBr(a, pass, data, inst),
1187 .repeat => return analyzeInstRepeat(a, pass, data, inst),
1188 .switch_dispatch => return analyzeInstSwitchDispatch(a, pass, data, inst),
1189
1190 .assembly => {
1191 const extra = a.air.extraData(Air.Asm, inst_datas[@intFromEnum(inst)].ty_pl.payload);
1192 var extra_i: usize = extra.end;
1193 const outputs = @as([]const Air.Inst.Ref, @ptrCast(a.air.extra[extra_i..][0..extra.data.outputs_len]));
1194 extra_i += outputs.len;
1195 const inputs = @as([]const Air.Inst.Ref, @ptrCast(a.air.extra[extra_i..][0..extra.data.inputs_len]));
1196 extra_i += inputs.len;
1197
1198 const num_operands = simple: {
1199 var buf = [1]Air.Inst.Ref{.none} ** (bpi - 1);
1200 var buf_index: usize = 0;
1201 for (outputs) |output| {
1202 if (output != .none) {
1203 if (buf_index < buf.len) buf[buf_index] = output;
1204 buf_index += 1;
1205 }
1206 }
1207 if (buf_index + inputs.len > buf.len) {
1208 break :simple buf_index + inputs.len;
1209 }
1210 @memcpy(buf[buf_index..][0..inputs.len], inputs);
1211 return analyzeOperands(a, pass, data, inst, buf);
1212 };
1213
1214 var big = try AnalyzeBigOperands(pass).init(a, data, inst, num_operands);
1215 defer big.deinit();
1216 var i: usize = inputs.len;
1217 while (i > 0) {
1218 i -= 1;
1219 try big.feed(inputs[i]);
1220 }
1221 i = outputs.len;
1222 while (i > 0) {
1223 i -= 1;
1224 if (outputs[i] != .none) {
1225 try big.feed(outputs[i]);
1226 }
1227 }
1228 return big.finish();
1229 },
1230
1231 inline .block, .dbg_inline_block => |comptime_tag| {
1232 const ty_pl = inst_datas[@intFromEnum(inst)].ty_pl;
1233 const extra = a.air.extraData(switch (comptime_tag) {
1234 .block => Air.Block,
1235 .dbg_inline_block => Air.DbgInlineBlock,
1236 else => unreachable,
1237 }, ty_pl.payload);
1238 return analyzeInstBlock(a, pass, data, inst, ty_pl.ty, @ptrCast(a.air.extra[extra.end..][0..extra.data.body_len]));
1239 },
1240 .loop => return analyzeInstLoop(a, pass, data, inst),
1241
1242 .@"try", .try_cold => return analyzeInstCondBr(a, pass, data, inst, .@"try"),
1243 .try_ptr, .try_ptr_cold => return analyzeInstCondBr(a, pass, data, inst, .try_ptr),
1244 .cond_br => return analyzeInstCondBr(a, pass, data, inst, .cond_br),
1245 .switch_br => return analyzeInstSwitchBr(a, pass, data, inst, false),
1246 .loop_switch_br => return analyzeInstSwitchBr(a, pass, data, inst, true),
1247
1248 .wasm_memory_grow => {
1249 const pl_op = inst_datas[@intFromEnum(inst)].pl_op;
1250 return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, .none, .none });
1251 },
1252 }
1253}
1254
1255/// Every instruction should hit this (after handling any nested bodies), in every pass. In the
1256/// initial pass, it is responsible for marking deaths of the (first three) operands and noticing
1257/// immediate deaths.
1258fn analyzeOperands(
1259 a: *Analysis,
1260 comptime pass: LivenessPass,
1261 data: *LivenessPassData(pass),
1262 inst: Air.Inst.Index,
1263 operands: [bpi - 1]Air.Inst.Ref,
1264) Allocator.Error!void {
1265 const gpa = a.gpa;
1266 const ip = a.intern_pool;
1267
1268 switch (pass) {
1269 .loop_analysis => {
1270 _ = data.live_set.remove(inst);
1271
1272 for (operands) |op_ref| {
1273 const operand = op_ref.toIndexAllowNone() orelse continue;
1274 _ = try data.live_set.put(gpa, operand, {});
1275 }
1276 },
1277
1278 .main_analysis => {
1279 const usize_index = (@intFromEnum(inst) * bpi) / @bitSizeOf(usize);
1280
1281 // This logic must synchronize with `will_die_immediately` in `AnalyzeBigOperands.init`.
1282 const immediate_death = if (data.live_set.remove(inst)) blk: {
1283 log.debug("[{}] %{}: removed from live set", .{ pass, @intFromEnum(inst) });
1284 break :blk false;
1285 } else blk: {
1286 log.debug("[{}] %{}: immediate death", .{ pass, @intFromEnum(inst) });
1287 break :blk true;
1288 };
1289
1290 var tomb_bits: Bpi = @as(Bpi, @intFromBool(immediate_death)) << (bpi - 1);
1291
1292 // If our result is unused and the instruction doesn't need to be lowered, backends will
1293 // skip the lowering of this instruction, so we don't want to record uses of operands.
1294 // That way, we can mark as many instructions as possible unused.
1295 if (!immediate_death or a.air.mustLower(inst, ip)) {
1296 // Note that it's important we iterate over the operands backwards, so that if a dying
1297 // operand is used multiple times we mark its last use as its death.
1298 var i = operands.len;
1299 while (i > 0) {
1300 i -= 1;
1301 const op_ref = operands[i];
1302 const operand = op_ref.toIndexAllowNone() orelse continue;
1303
1304 const mask = @as(Bpi, 1) << @as(OperandInt, @intCast(i));
1305
1306 if ((try data.live_set.fetchPut(gpa, operand, {})) == null) {
1307 log.debug("[{}] %{}: added %{} to live set (operand dies here)", .{ pass, @intFromEnum(inst), operand });
1308 tomb_bits |= mask;
1309 }
1310 }
1311 }
1312
1313 a.tomb_bits[usize_index] |= @as(usize, tomb_bits) <<
1314 @as(Log2Int(usize), @intCast((@intFromEnum(inst) % (@bitSizeOf(usize) / bpi)) * bpi));
1315 },
1316 }
1317}
1318
1319/// Like `analyzeOperands`, but for an instruction which returns from a function, so should
1320/// effectively kill every remaining live value other than its operands.
1321fn analyzeFuncEnd(
1322 a: *Analysis,
1323 comptime pass: LivenessPass,
1324 data: *LivenessPassData(pass),
1325 inst: Air.Inst.Index,
1326 operands: [bpi - 1]Air.Inst.Ref,
1327) Allocator.Error!void {
1328 switch (pass) {
1329 .loop_analysis => {
1330 // No operands need to be alive if we're returning from the function, so we don't need
1331 // to touch `breaks` here even though this is sort of like a break to the top level.
1332 },
1333
1334 .main_analysis => {
1335 data.live_set.clearRetainingCapacity();
1336 },
1337 }
1338
1339 return analyzeOperands(a, pass, data, inst, operands);
1340}
1341
1342fn analyzeInstBr(
1343 a: *Analysis,
1344 comptime pass: LivenessPass,
1345 data: *LivenessPassData(pass),
1346 inst: Air.Inst.Index,
1347) !void {
1348 const inst_datas = a.air.instructions.items(.data);
1349 const br = inst_datas[@intFromEnum(inst)].br;
1350 const gpa = a.gpa;
1351
1352 switch (pass) {
1353 .loop_analysis => {
1354 try data.breaks.put(gpa, br.block_inst, {});
1355 },
1356
1357 .main_analysis => {
1358 const block_scope = data.block_scopes.get(br.block_inst).?; // we should always be breaking from an enclosing block
1359
1360 const new_live_set = try block_scope.live_set.clone(gpa);
1361 data.live_set.deinit(gpa);
1362 data.live_set = new_live_set;
1363 },
1364 }
1365
1366 return analyzeOperands(a, pass, data, inst, .{ br.operand, .none, .none });
1367}
1368
1369fn analyzeInstRepeat(
1370 a: *Analysis,
1371 comptime pass: LivenessPass,
1372 data: *LivenessPassData(pass),
1373 inst: Air.Inst.Index,
1374) !void {
1375 const inst_datas = a.air.instructions.items(.data);
1376 const repeat = inst_datas[@intFromEnum(inst)].repeat;
1377 const gpa = a.gpa;
1378
1379 switch (pass) {
1380 .loop_analysis => {
1381 try data.breaks.put(gpa, repeat.loop_inst, {});
1382 },
1383
1384 .main_analysis => {
1385 const block_scope = data.block_scopes.get(repeat.loop_inst).?; // we should always be repeating an enclosing loop
1386
1387 const new_live_set = try block_scope.live_set.clone(gpa);
1388 data.live_set.deinit(gpa);
1389 data.live_set = new_live_set;
1390 },
1391 }
1392
1393 return analyzeOperands(a, pass, data, inst, .{ .none, .none, .none });
1394}
1395
1396fn analyzeInstSwitchDispatch(
1397 a: *Analysis,
1398 comptime pass: LivenessPass,
1399 data: *LivenessPassData(pass),
1400 inst: Air.Inst.Index,
1401) !void {
1402 // This happens to be identical to `analyzeInstBr`, but is separated anyway for clarity.
1403
1404 const inst_datas = a.air.instructions.items(.data);
1405 const br = inst_datas[@intFromEnum(inst)].br;
1406 const gpa = a.gpa;
1407
1408 switch (pass) {
1409 .loop_analysis => {
1410 try data.breaks.put(gpa, br.block_inst, {});
1411 },
1412
1413 .main_analysis => {
1414 const block_scope = data.block_scopes.get(br.block_inst).?; // we should always be repeating an enclosing loop
1415
1416 const new_live_set = try block_scope.live_set.clone(gpa);
1417 data.live_set.deinit(gpa);
1418 data.live_set = new_live_set;
1419 },
1420 }
1421
1422 return analyzeOperands(a, pass, data, inst, .{ br.operand, .none, .none });
1423}
1424
1425fn analyzeInstBlock(
1426 a: *Analysis,
1427 comptime pass: LivenessPass,
1428 data: *LivenessPassData(pass),
1429 inst: Air.Inst.Index,
1430 ty: Air.Inst.Ref,
1431 body: []const Air.Inst.Index,
1432) !void {
1433 const gpa = a.gpa;
1434
1435 // We actually want to do `analyzeOperands` *first*, since our result logically doesn't
1436 // exist until the block body ends (and we're iterating backwards)
1437 try analyzeOperands(a, pass, data, inst, .{ .none, .none, .none });
1438
1439 switch (pass) {
1440 .loop_analysis => {
1441 try analyzeBody(a, pass, data, body);
1442 _ = data.breaks.remove(inst);
1443 },
1444
1445 .main_analysis => {
1446 log.debug("[{}] %{}: block live set is {}", .{ pass, inst, fmtInstSet(&data.live_set) });
1447 // We can move the live set because the body should have a noreturn
1448 // instruction which overrides the set.
1449 try data.block_scopes.put(gpa, inst, .{
1450 .live_set = data.live_set.move(),
1451 });
1452 defer {
1453 log.debug("[{}] %{}: popped block scope", .{ pass, inst });
1454 var scope = data.block_scopes.fetchRemove(inst).?.value;
1455 scope.live_set.deinit(gpa);
1456 }
1457
1458 log.debug("[{}] %{}: pushed new block scope", .{ pass, inst });
1459 try analyzeBody(a, pass, data, body);
1460
1461 // If the block is noreturn, block deaths not only aren't useful, they're impossible to
1462 // find: there could be more stuff alive after the block than before it!
1463 if (!a.intern_pool.isNoReturn(ty.toType().toIntern())) {
1464 // The block kills the difference in the live sets
1465 const block_scope = data.block_scopes.get(inst).?;
1466 const num_deaths = data.live_set.count() - block_scope.live_set.count();
1467
1468 try a.extra.ensureUnusedCapacity(gpa, num_deaths + std.meta.fields(Block).len);
1469 const extra_index = a.addExtraAssumeCapacity(Block{
1470 .death_count = num_deaths,
1471 });
1472
1473 var measured_num: u32 = 0;
1474 var it = data.live_set.keyIterator();
1475 while (it.next()) |key| {
1476 const alive = key.*;
1477 if (!block_scope.live_set.contains(alive)) {
1478 // Dies in block
1479 a.extra.appendAssumeCapacity(@intFromEnum(alive));
1480 measured_num += 1;
1481 }
1482 }
1483 assert(measured_num == num_deaths); // post-live-set should be a subset of pre-live-set
1484 try a.special.put(gpa, inst, extra_index);
1485 log.debug("[{}] %{}: block deaths are {}", .{
1486 pass,
1487 inst,
1488 fmtInstList(@ptrCast(a.extra.items[extra_index + 1 ..][0..num_deaths])),
1489 });
1490 }
1491 },
1492 }
1493}
1494
1495fn writeLoopInfo(
1496 a: *Analysis,
1497 data: *LivenessPassData(.loop_analysis),
1498 inst: Air.Inst.Index,
1499 old_breaks: std.AutoHashMapUnmanaged(Air.Inst.Index, void),
1500 old_live: std.AutoHashMapUnmanaged(Air.Inst.Index, void),
1501) !void {
1502 const gpa = a.gpa;
1503
1504 // `loop`s are guaranteed to have at least one matching `repeat`.
1505 // Similarly, `loop_switch_br`s have a matching `switch_dispatch`.
1506 // However, we no longer care about repeats of this loop for resolving
1507 // which operands must live within it.
1508 assert(data.breaks.remove(inst));
1509
1510 const extra_index: u32 = @intCast(a.extra.items.len);
1511
1512 const num_breaks = data.breaks.count();
1513 try a.extra.ensureUnusedCapacity(gpa, 1 + num_breaks);
1514
1515 a.extra.appendAssumeCapacity(num_breaks);
1516
1517 var it = data.breaks.keyIterator();
1518 while (it.next()) |key| {
1519 const block_inst = key.*;
1520 a.extra.appendAssumeCapacity(@intFromEnum(block_inst));
1521 }
1522 log.debug("[{}] %{}: includes breaks to {}", .{ LivenessPass.loop_analysis, inst, fmtInstSet(&data.breaks) });
1523
1524 // Now we put the live operands from the loop body in too
1525 const num_live = data.live_set.count();
1526 try a.extra.ensureUnusedCapacity(gpa, 1 + num_live);
1527
1528 a.extra.appendAssumeCapacity(num_live);
1529 it = data.live_set.keyIterator();
1530 while (it.next()) |key| {
1531 const alive = key.*;
1532 a.extra.appendAssumeCapacity(@intFromEnum(alive));
1533 }
1534 log.debug("[{}] %{}: maintain liveness of {}", .{ LivenessPass.loop_analysis, inst, fmtInstSet(&data.live_set) });
1535
1536 try a.special.put(gpa, inst, extra_index);
1537
1538 // Add back operands which were previously alive
1539 it = old_live.keyIterator();
1540 while (it.next()) |key| {
1541 const alive = key.*;
1542 try data.live_set.put(gpa, alive, {});
1543 }
1544
1545 // And the same for breaks
1546 it = old_breaks.keyIterator();
1547 while (it.next()) |key| {
1548 const block_inst = key.*;
1549 try data.breaks.put(gpa, block_inst, {});
1550 }
1551}
1552
1553/// When analyzing a loop in the main pass, sets up `data.live_set` to be the set
1554/// of operands known to be alive when the loop repeats.
1555fn resolveLoopLiveSet(
1556 a: *Analysis,
1557 data: *LivenessPassData(.main_analysis),
1558 inst: Air.Inst.Index,
1559) !void {
1560 const gpa = a.gpa;
1561
1562 const extra_idx = a.special.fetchRemove(inst).?.value;
1563 const num_breaks = data.old_extra.items[extra_idx];
1564 const breaks: []const Air.Inst.Index = @ptrCast(data.old_extra.items[extra_idx + 1 ..][0..num_breaks]);
1565
1566 const num_loop_live = data.old_extra.items[extra_idx + num_breaks + 1];
1567 const loop_live: []const Air.Inst.Index = @ptrCast(data.old_extra.items[extra_idx + num_breaks + 2 ..][0..num_loop_live]);
1568
1569 // This is necessarily not in the same control flow branch, because loops are noreturn
1570 data.live_set.clearRetainingCapacity();
1571
1572 try data.live_set.ensureUnusedCapacity(gpa, @intCast(loop_live.len));
1573 for (loop_live) |alive| data.live_set.putAssumeCapacity(alive, {});
1574
1575 log.debug("[{}] %{}: block live set is {}", .{ LivenessPass.main_analysis, inst, fmtInstSet(&data.live_set) });
1576
1577 for (breaks) |block_inst| {
1578 // We might break to this block, so include every operand that the block needs alive
1579 const block_scope = data.block_scopes.get(block_inst).?;
1580
1581 var it = block_scope.live_set.keyIterator();
1582 while (it.next()) |key| {
1583 const alive = key.*;
1584 try data.live_set.put(gpa, alive, {});
1585 }
1586 }
1587
1588 log.debug("[{}] %{}: loop live set is {}", .{ LivenessPass.main_analysis, inst, fmtInstSet(&data.live_set) });
1589}
1590
1591fn analyzeInstLoop(
1592 a: *Analysis,
1593 comptime pass: LivenessPass,
1594 data: *LivenessPassData(pass),
1595 inst: Air.Inst.Index,
1596) !void {
1597 const inst_datas = a.air.instructions.items(.data);
1598 const extra = a.air.extraData(Air.Block, inst_datas[@intFromEnum(inst)].ty_pl.payload);
1599 const body: []const Air.Inst.Index = @ptrCast(a.air.extra[extra.end..][0..extra.data.body_len]);
1600 const gpa = a.gpa;
1601
1602 try analyzeOperands(a, pass, data, inst, .{ .none, .none, .none });
1603
1604 switch (pass) {
1605 .loop_analysis => {
1606 var old_breaks = data.breaks.move();
1607 defer old_breaks.deinit(gpa);
1608
1609 var old_live = data.live_set.move();
1610 defer old_live.deinit(gpa);
1611
1612 try analyzeBody(a, pass, data, body);
1613
1614 try writeLoopInfo(a, data, inst, old_breaks, old_live);
1615 },
1616
1617 .main_analysis => {
1618 try resolveLoopLiveSet(a, data, inst);
1619
1620 // Now, `data.live_set` is the operands which must be alive when the loop repeats.
1621 // Move them into a block scope for corresponding `repeat` instructions to notice.
1622 try data.block_scopes.putNoClobber(gpa, inst, .{
1623 .live_set = data.live_set.move(),
1624 });
1625 defer {
1626 log.debug("[{}] %{}: popped loop block scop", .{ pass, inst });
1627 var scope = data.block_scopes.fetchRemove(inst).?.value;
1628 scope.live_set.deinit(gpa);
1629 }
1630 try analyzeBody(a, pass, data, body);
1631 },
1632 }
1633}
1634
1635/// Despite its name, this function is used for analysis of not only `cond_br` instructions, but
1636/// also `try` and `try_ptr`, which are highly related. The `inst_type` parameter indicates which
1637/// type of instruction `inst` points to.
1638fn analyzeInstCondBr(
1639 a: *Analysis,
1640 comptime pass: LivenessPass,
1641 data: *LivenessPassData(pass),
1642 inst: Air.Inst.Index,
1643 comptime inst_type: enum { cond_br, @"try", try_ptr },
1644) !void {
1645 const inst_datas = a.air.instructions.items(.data);
1646 const gpa = a.gpa;
1647
1648 const extra = switch (inst_type) {
1649 .cond_br => a.air.extraData(Air.CondBr, inst_datas[@intFromEnum(inst)].pl_op.payload),
1650 .@"try" => a.air.extraData(Air.Try, inst_datas[@intFromEnum(inst)].pl_op.payload),
1651 .try_ptr => a.air.extraData(Air.TryPtr, inst_datas[@intFromEnum(inst)].ty_pl.payload),
1652 };
1653
1654 const condition = switch (inst_type) {
1655 .cond_br, .@"try" => inst_datas[@intFromEnum(inst)].pl_op.operand,
1656 .try_ptr => extra.data.ptr,
1657 };
1658
1659 const then_body: []const Air.Inst.Index = switch (inst_type) {
1660 .cond_br => @ptrCast(a.air.extra[extra.end..][0..extra.data.then_body_len]),
1661 else => &.{}, // we won't use this
1662 };
1663
1664 const else_body: []const Air.Inst.Index = @ptrCast(switch (inst_type) {
1665 .cond_br => a.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len],
1666 .@"try", .try_ptr => a.air.extra[extra.end..][0..extra.data.body_len],
1667 });
1668
1669 switch (pass) {
1670 .loop_analysis => {
1671 switch (inst_type) {
1672 .cond_br => try analyzeBody(a, pass, data, then_body),
1673 .@"try", .try_ptr => {},
1674 }
1675 try analyzeBody(a, pass, data, else_body);
1676 },
1677
1678 .main_analysis => {
1679 switch (inst_type) {
1680 .cond_br => try analyzeBody(a, pass, data, then_body),
1681 .@"try", .try_ptr => {}, // The "then body" is just the remainder of this block
1682 }
1683 var then_live = data.live_set.move();
1684 defer then_live.deinit(gpa);
1685
1686 try analyzeBody(a, pass, data, else_body);
1687 var else_live = data.live_set.move();
1688 defer else_live.deinit(gpa);
1689
1690 // Operands which are alive in one branch but not the other need to die at the start of
1691 // the peer branch.
1692
1693 var then_mirrored_deaths: std.ArrayListUnmanaged(Air.Inst.Index) = .empty;
1694 defer then_mirrored_deaths.deinit(gpa);
1695
1696 var else_mirrored_deaths: std.ArrayListUnmanaged(Air.Inst.Index) = .empty;
1697 defer else_mirrored_deaths.deinit(gpa);
1698
1699 // Note: this invalidates `else_live`, but expands `then_live` to be their union
1700 {
1701 var it = then_live.keyIterator();
1702 while (it.next()) |key| {
1703 const death = key.*;
1704 if (else_live.remove(death)) continue; // removing makes the loop below faster
1705
1706 // If this is a `try`, the "then body" (rest of the branch) might have
1707 // referenced our result. We want to avoid killing this value in the else branch
1708 // if that's the case, since it only exists in the (fake) then branch.
1709 switch (inst_type) {
1710 .cond_br => {},
1711 .@"try", .try_ptr => if (death == inst) continue,
1712 }
1713
1714 try else_mirrored_deaths.append(gpa, death);
1715 }
1716 // Since we removed common stuff above, `else_live` is now only operands
1717 // which are *only* alive in the else branch
1718 it = else_live.keyIterator();
1719 while (it.next()) |key| {
1720 const death = key.*;
1721 try then_mirrored_deaths.append(gpa, death);
1722 // Make `then_live` contain the full live set (i.e. union of both)
1723 try then_live.put(gpa, death, {});
1724 }
1725 }
1726
1727 log.debug("[{}] %{}: 'then' branch mirrored deaths are {}", .{ pass, inst, fmtInstList(then_mirrored_deaths.items) });
1728 log.debug("[{}] %{}: 'else' branch mirrored deaths are {}", .{ pass, inst, fmtInstList(else_mirrored_deaths.items) });
1729
1730 data.live_set.deinit(gpa);
1731 data.live_set = then_live.move(); // Really the union of both live sets
1732
1733 log.debug("[{}] %{}: new live set is {}", .{ pass, inst, fmtInstSet(&data.live_set) });
1734
1735 // Write the mirrored deaths to `extra`
1736 const then_death_count = @as(u32, @intCast(then_mirrored_deaths.items.len));
1737 const else_death_count = @as(u32, @intCast(else_mirrored_deaths.items.len));
1738 try a.extra.ensureUnusedCapacity(gpa, std.meta.fields(CondBr).len + then_death_count + else_death_count);
1739 const extra_index = a.addExtraAssumeCapacity(CondBr{
1740 .then_death_count = then_death_count,
1741 .else_death_count = else_death_count,
1742 });
1743 a.extra.appendSliceAssumeCapacity(@ptrCast(then_mirrored_deaths.items));
1744 a.extra.appendSliceAssumeCapacity(@ptrCast(else_mirrored_deaths.items));
1745 try a.special.put(gpa, inst, extra_index);
1746 },
1747 }
1748
1749 try analyzeOperands(a, pass, data, inst, .{ condition, .none, .none });
1750}
1751
1752fn analyzeInstSwitchBr(
1753 a: *Analysis,
1754 comptime pass: LivenessPass,
1755 data: *LivenessPassData(pass),
1756 inst: Air.Inst.Index,
1757 is_dispatch_loop: bool,
1758) !void {
1759 const inst_datas = a.air.instructions.items(.data);
1760 const pl_op = inst_datas[@intFromEnum(inst)].pl_op;
1761 const condition = pl_op.operand;
1762 const switch_br = a.air.unwrapSwitch(inst);
1763 const gpa = a.gpa;
1764 const ncases = switch_br.cases_len;
1765
1766 switch (pass) {
1767 .loop_analysis => {
1768 var old_breaks: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .empty;
1769 defer old_breaks.deinit(gpa);
1770
1771 var old_live: std.AutoHashMapUnmanaged(Air.Inst.Index, void) = .empty;
1772 defer old_live.deinit(gpa);
1773
1774 if (is_dispatch_loop) {
1775 old_breaks = data.breaks.move();
1776 old_live = data.live_set.move();
1777 }
1778
1779 var it = switch_br.iterateCases();
1780 while (it.next()) |case| {
1781 try analyzeBody(a, pass, data, case.body);
1782 }
1783 { // else
1784 const else_body = it.elseBody();
1785 try analyzeBody(a, pass, data, else_body);
1786 }
1787
1788 if (is_dispatch_loop) {
1789 try writeLoopInfo(a, data, inst, old_breaks, old_live);
1790 }
1791 },
1792
1793 .main_analysis => {
1794 if (is_dispatch_loop) {
1795 try resolveLoopLiveSet(a, data, inst);
1796 try data.block_scopes.putNoClobber(gpa, inst, .{
1797 .live_set = data.live_set.move(),
1798 });
1799 }
1800 defer if (is_dispatch_loop) {
1801 log.debug("[{}] %{}: popped loop block scop", .{ pass, inst });
1802 var scope = data.block_scopes.fetchRemove(inst).?.value;
1803 scope.live_set.deinit(gpa);
1804 };
1805 // This is, all in all, just a messier version of the `cond_br` logic. If you're trying
1806 // to understand it, I encourage looking at `analyzeInstCondBr` first.
1807
1808 const DeathSet = std.AutoHashMapUnmanaged(Air.Inst.Index, void);
1809 const DeathList = std.ArrayListUnmanaged(Air.Inst.Index);
1810
1811 var case_live_sets = try gpa.alloc(std.AutoHashMapUnmanaged(Air.Inst.Index, void), ncases + 1); // +1 for else
1812 defer gpa.free(case_live_sets);
1813
1814 @memset(case_live_sets, .{});
1815 defer for (case_live_sets) |*live_set| live_set.deinit(gpa);
1816
1817 var case_it = switch_br.iterateCases();
1818 while (case_it.next()) |case| {
1819 try analyzeBody(a, pass, data, case.body);
1820 case_live_sets[case.idx] = data.live_set.move();
1821 }
1822 { // else
1823 const else_body = case_it.elseBody();
1824 try analyzeBody(a, pass, data, else_body);
1825 case_live_sets[ncases] = data.live_set.move();
1826 }
1827
1828 const mirrored_deaths = try gpa.alloc(DeathList, ncases + 1);
1829 defer gpa.free(mirrored_deaths);
1830
1831 @memset(mirrored_deaths, .{});
1832 defer for (mirrored_deaths) |*md| md.deinit(gpa);
1833
1834 {
1835 var all_alive: DeathSet = .{};
1836 defer all_alive.deinit(gpa);
1837
1838 for (case_live_sets) |*live_set| {
1839 try all_alive.ensureUnusedCapacity(gpa, live_set.count());
1840 var it = live_set.keyIterator();
1841 while (it.next()) |key| {
1842 const alive = key.*;
1843 all_alive.putAssumeCapacity(alive, {});
1844 }
1845 }
1846
1847 for (mirrored_deaths, case_live_sets) |*mirrored, *live_set| {
1848 var it = all_alive.keyIterator();
1849 while (it.next()) |key| {
1850 const alive = key.*;
1851 if (!live_set.contains(alive)) {
1852 // Should die at the start of this branch
1853 try mirrored.append(gpa, alive);
1854 }
1855 }
1856 }
1857
1858 for (mirrored_deaths, 0..) |mirrored, i| {
1859 log.debug("[{}] %{}: case {} mirrored deaths are {}", .{ pass, inst, i, fmtInstList(mirrored.items) });
1860 }
1861
1862 data.live_set.deinit(gpa);
1863 data.live_set = all_alive.move();
1864
1865 log.debug("[{}] %{}: new live set is {}", .{ pass, inst, fmtInstSet(&data.live_set) });
1866 }
1867
1868 const else_death_count = @as(u32, @intCast(mirrored_deaths[ncases].items.len));
1869 const extra_index = try a.addExtra(SwitchBr{
1870 .else_death_count = else_death_count,
1871 });
1872 for (mirrored_deaths[0..ncases]) |mirrored| {
1873 const num = @as(u32, @intCast(mirrored.items.len));
1874 try a.extra.ensureUnusedCapacity(gpa, num + 1);
1875 a.extra.appendAssumeCapacity(num);
1876 a.extra.appendSliceAssumeCapacity(@ptrCast(mirrored.items));
1877 }
1878 try a.extra.ensureUnusedCapacity(gpa, else_death_count);
1879 a.extra.appendSliceAssumeCapacity(@ptrCast(mirrored_deaths[ncases].items));
1880 try a.special.put(gpa, inst, extra_index);
1881 },
1882 }
1883
1884 try analyzeOperands(a, pass, data, inst, .{ condition, .none, .none });
1885}
1886
1887fn AnalyzeBigOperands(comptime pass: LivenessPass) type {
1888 return struct {
1889 a: *Analysis,
1890 data: *LivenessPassData(pass),
1891 inst: Air.Inst.Index,
1892
1893 operands_remaining: u32,
1894 small: [bpi - 1]Air.Inst.Ref = .{.none} ** (bpi - 1),
1895 extra_tombs: []u32,
1896
1897 // Only used in `LivenessPass.main_analysis`
1898 will_die_immediately: bool,
1899
1900 const Self = @This();
1901
1902 fn init(
1903 a: *Analysis,
1904 data: *LivenessPassData(pass),
1905 inst: Air.Inst.Index,
1906 total_operands: usize,
1907 ) !Self {
1908 const extra_operands = @as(u32, @intCast(total_operands)) -| (bpi - 1);
1909 const max_extra_tombs = (extra_operands + 30) / 31;
1910
1911 const extra_tombs: []u32 = switch (pass) {
1912 .loop_analysis => &.{},
1913 .main_analysis => try a.gpa.alloc(u32, max_extra_tombs),
1914 };
1915 errdefer a.gpa.free(extra_tombs);
1916
1917 @memset(extra_tombs, 0);
1918
1919 const will_die_immediately: bool = switch (pass) {
1920 .loop_analysis => false, // track everything, since we don't have full liveness information yet
1921 .main_analysis => !data.live_set.contains(inst),
1922 };
1923
1924 return .{
1925 .a = a,
1926 .data = data,
1927 .inst = inst,
1928 .operands_remaining = @as(u32, @intCast(total_operands)),
1929 .extra_tombs = extra_tombs,
1930 .will_die_immediately = will_die_immediately,
1931 };
1932 }
1933
1934 /// Must be called with operands in reverse order.
1935 fn feed(big: *Self, op_ref: Air.Inst.Ref) !void {
1936 const ip = big.a.intern_pool;
1937 // Note that after this, `operands_remaining` becomes the index of the current operand
1938 big.operands_remaining -= 1;
1939
1940 if (big.operands_remaining < bpi - 1) {
1941 big.small[big.operands_remaining] = op_ref;
1942 return;
1943 }
1944
1945 const operand = op_ref.toIndex() orelse return;
1946
1947 // If our result is unused and the instruction doesn't need to be lowered, backends will
1948 // skip the lowering of this instruction, so we don't want to record uses of operands.
1949 // That way, we can mark as many instructions as possible unused.
1950 if (big.will_die_immediately and !big.a.air.mustLower(big.inst, ip)) return;
1951
1952 const extra_byte = (big.operands_remaining - (bpi - 1)) / 31;
1953 const extra_bit = @as(u5, @intCast(big.operands_remaining - (bpi - 1) - extra_byte * 31));
1954
1955 const gpa = big.a.gpa;
1956
1957 switch (pass) {
1958 .loop_analysis => {
1959 _ = try big.data.live_set.put(gpa, operand, {});
1960 },
1961
1962 .main_analysis => {
1963 if ((try big.data.live_set.fetchPut(gpa, operand, {})) == null) {
1964 log.debug("[{}] %{}: added %{} to live set (operand dies here)", .{ pass, big.inst, operand });
1965 big.extra_tombs[extra_byte] |= @as(u32, 1) << extra_bit;
1966 }
1967 },
1968 }
1969 }
1970
1971 fn finish(big: *Self) !void {
1972 const gpa = big.a.gpa;
1973
1974 std.debug.assert(big.operands_remaining == 0);
1975
1976 switch (pass) {
1977 .loop_analysis => {},
1978
1979 .main_analysis => {
1980 // Note that the MSB is set on the final tomb to indicate the terminal element. This
1981 // allows for an optimisation where we only add as many extra tombs as are needed to
1982 // represent the dying operands. Each pass modifies operand bits and so needs to write
1983 // back, so let's figure out how many extra tombs we really need. Note that we always
1984 // keep at least one.
1985 var num: usize = big.extra_tombs.len;
1986 while (num > 1) {
1987 if (@as(u31, @truncate(big.extra_tombs[num - 1])) != 0) {
1988 // Some operand dies here
1989 break;
1990 }
1991 num -= 1;
1992 }
1993 // Mark final tomb
1994 big.extra_tombs[num - 1] |= @as(u32, 1) << 31;
1995
1996 const extra_tombs = big.extra_tombs[0..num];
1997
1998 const extra_index = @as(u32, @intCast(big.a.extra.items.len));
1999 try big.a.extra.appendSlice(gpa, extra_tombs);
2000 try big.a.special.put(gpa, big.inst, extra_index);
2001 },
2002 }
2003
2004 try analyzeOperands(big.a, pass, big.data, big.inst, big.small);
2005 }
2006
2007 fn deinit(big: *Self) void {
2008 big.a.gpa.free(big.extra_tombs);
2009 }
2010 };
2011}
2012
2013fn fmtInstSet(set: *const std.AutoHashMapUnmanaged(Air.Inst.Index, void)) FmtInstSet {
2014 return .{ .set = set };
2015}
2016
2017const FmtInstSet = struct {
2018 set: *const std.AutoHashMapUnmanaged(Air.Inst.Index, void),
2019
2020 pub fn format(val: FmtInstSet, comptime _: []const u8, _: std.fmt.FormatOptions, w: anytype) !void {
2021 if (val.set.count() == 0) {
2022 try w.writeAll("[no instructions]");
2023 return;
2024 }
2025 var it = val.set.keyIterator();
2026 try w.print("%{}", .{it.next().?.*});
2027 while (it.next()) |key| {
2028 try w.print(" %{}", .{key.*});
2029 }
2030 }
2031};
2032
2033fn fmtInstList(list: []const Air.Inst.Index) FmtInstList {
2034 return .{ .list = list };
2035}
2036
2037const FmtInstList = struct {
2038 list: []const Air.Inst.Index,
2039
2040 pub fn format(val: FmtInstList, comptime _: []const u8, _: std.fmt.FormatOptions, w: anytype) !void {
2041 if (val.list.len == 0) {
2042 try w.writeAll("[no instructions]");
2043 return;
2044 }
2045 try w.print("%{}", .{val.list[0]});
2046 for (val.list[1..]) |inst| {
2047 try w.print(" %{}", .{inst});
2048 }
2049 }
2050};
src/Liveness/Verify.zig deleted-642
......@@ -1,642 +0,0 @@
1//! Verifies that Liveness information is valid.
2
3gpa: std.mem.Allocator,
4air: Air,
5liveness: Liveness,
6live: LiveMap = .{},
7blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, LiveMap) = .empty,
8loops: std.AutoHashMapUnmanaged(Air.Inst.Index, LiveMap) = .empty,
9intern_pool: *const InternPool,
10
11pub const Error = error{ LivenessInvalid, OutOfMemory };
12
13pub fn deinit(self: *Verify) void {
14 self.live.deinit(self.gpa);
15 {
16 var it = self.blocks.valueIterator();
17 while (it.next()) |block| block.deinit(self.gpa);
18 self.blocks.deinit(self.gpa);
19 }
20 {
21 var it = self.loops.valueIterator();
22 while (it.next()) |block| block.deinit(self.gpa);
23 self.loops.deinit(self.gpa);
24 }
25 self.* = undefined;
26}
27
28pub fn verify(self: *Verify) Error!void {
29 self.live.clearRetainingCapacity();
30 self.blocks.clearRetainingCapacity();
31 self.loops.clearRetainingCapacity();
32 try self.verifyBody(self.air.getMainBody());
33 // We don't care about `self.live` now, because the loop body was noreturn - everything being dead was checked on `ret` etc
34 assert(self.blocks.count() == 0);
35 assert(self.loops.count() == 0);
36}
37
38const LiveMap = std.AutoHashMapUnmanaged(Air.Inst.Index, void);
39
40fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
41 const ip = self.intern_pool;
42 const tags = self.air.instructions.items(.tag);
43 const data = self.air.instructions.items(.data);
44 for (body) |inst| {
45 if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip)) {
46 // This instruction will not be lowered and should be ignored.
47 continue;
48 }
49
50 switch (tags[@intFromEnum(inst)]) {
51 // no operands
52 .arg,
53 .alloc,
54 .inferred_alloc,
55 .inferred_alloc_comptime,
56 .ret_ptr,
57 .breakpoint,
58 .dbg_stmt,
59 .dbg_empty_stmt,
60 .ret_addr,
61 .frame_addr,
62 .wasm_memory_size,
63 .err_return_trace,
64 .save_err_return_trace_index,
65 .tlv_dllimport_ptr,
66 .c_va_start,
67 .work_item_id,
68 .work_group_size,
69 .work_group_id,
70 => try self.verifyInstOperands(inst, .{ .none, .none, .none }),
71
72 .trap, .unreach => {
73 try self.verifyInstOperands(inst, .{ .none, .none, .none });
74 // This instruction terminates the function, so everything should be dead
75 if (self.live.count() > 0) return invalid("%{}: instructions still alive", .{inst});
76 },
77
78 // unary
79 .not,
80 .bitcast,
81 .load,
82 .fpext,
83 .fptrunc,
84 .intcast,
85 .intcast_safe,
86 .trunc,
87 .optional_payload,
88 .optional_payload_ptr,
89 .optional_payload_ptr_set,
90 .errunion_payload_ptr_set,
91 .wrap_optional,
92 .unwrap_errunion_payload,
93 .unwrap_errunion_err,
94 .unwrap_errunion_payload_ptr,
95 .unwrap_errunion_err_ptr,
96 .wrap_errunion_payload,
97 .wrap_errunion_err,
98 .slice_ptr,
99 .slice_len,
100 .ptr_slice_len_ptr,
101 .ptr_slice_ptr_ptr,
102 .struct_field_ptr_index_0,
103 .struct_field_ptr_index_1,
104 .struct_field_ptr_index_2,
105 .struct_field_ptr_index_3,
106 .array_to_slice,
107 .int_from_float,
108 .int_from_float_optimized,
109 .float_from_int,
110 .get_union_tag,
111 .clz,
112 .ctz,
113 .popcount,
114 .byte_swap,
115 .bit_reverse,
116 .splat,
117 .error_set_has_value,
118 .addrspace_cast,
119 .c_va_arg,
120 .c_va_copy,
121 .abs,
122 => {
123 const ty_op = data[@intFromEnum(inst)].ty_op;
124 try self.verifyInstOperands(inst, .{ ty_op.operand, .none, .none });
125 },
126 .is_null,
127 .is_non_null,
128 .is_null_ptr,
129 .is_non_null_ptr,
130 .is_err,
131 .is_non_err,
132 .is_err_ptr,
133 .is_non_err_ptr,
134 .is_named_enum_value,
135 .tag_name,
136 .error_name,
137 .sqrt,
138 .sin,
139 .cos,
140 .tan,
141 .exp,
142 .exp2,
143 .log,
144 .log2,
145 .log10,
146 .floor,
147 .ceil,
148 .round,
149 .trunc_float,
150 .neg,
151 .neg_optimized,
152 .cmp_lt_errors_len,
153 .set_err_return_trace,
154 .c_va_end,
155 => {
156 const un_op = data[@intFromEnum(inst)].un_op;
157 try self.verifyInstOperands(inst, .{ un_op, .none, .none });
158 },
159 .ret,
160 .ret_safe,
161 .ret_load,
162 => {
163 const un_op = data[@intFromEnum(inst)].un_op;
164 try self.verifyInstOperands(inst, .{ un_op, .none, .none });
165 // This instruction terminates the function, so everything should be dead
166 if (self.live.count() > 0) return invalid("%{}: instructions still alive", .{inst});
167 },
168 .dbg_var_ptr,
169 .dbg_var_val,
170 .dbg_arg_inline,
171 .wasm_memory_grow,
172 => {
173 const pl_op = data[@intFromEnum(inst)].pl_op;
174 try self.verifyInstOperands(inst, .{ pl_op.operand, .none, .none });
175 },
176 .prefetch => {
177 const prefetch = data[@intFromEnum(inst)].prefetch;
178 try self.verifyInstOperands(inst, .{ prefetch.ptr, .none, .none });
179 },
180 .reduce,
181 .reduce_optimized,
182 => {
183 const reduce = data[@intFromEnum(inst)].reduce;
184 try self.verifyInstOperands(inst, .{ reduce.operand, .none, .none });
185 },
186 .union_init => {
187 const ty_pl = data[@intFromEnum(inst)].ty_pl;
188 const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data;
189 try self.verifyInstOperands(inst, .{ extra.init, .none, .none });
190 },
191 .struct_field_ptr, .struct_field_val => {
192 const ty_pl = data[@intFromEnum(inst)].ty_pl;
193 const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
194 try self.verifyInstOperands(inst, .{ extra.struct_operand, .none, .none });
195 },
196 .field_parent_ptr => {
197 const ty_pl = data[@intFromEnum(inst)].ty_pl;
198 const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
199 try self.verifyInstOperands(inst, .{ extra.field_ptr, .none, .none });
200 },
201 .atomic_load => {
202 const atomic_load = data[@intFromEnum(inst)].atomic_load;
203 try self.verifyInstOperands(inst, .{ atomic_load.ptr, .none, .none });
204 },
205
206 // binary
207 .add,
208 .add_safe,
209 .add_optimized,
210 .add_wrap,
211 .add_sat,
212 .sub,
213 .sub_safe,
214 .sub_optimized,
215 .sub_wrap,
216 .sub_sat,
217 .mul,
218 .mul_safe,
219 .mul_optimized,
220 .mul_wrap,
221 .mul_sat,
222 .div_float,
223 .div_float_optimized,
224 .div_trunc,
225 .div_trunc_optimized,
226 .div_floor,
227 .div_floor_optimized,
228 .div_exact,
229 .div_exact_optimized,
230 .rem,
231 .rem_optimized,
232 .mod,
233 .mod_optimized,
234 .bit_and,
235 .bit_or,
236 .xor,
237 .cmp_lt,
238 .cmp_lt_optimized,
239 .cmp_lte,
240 .cmp_lte_optimized,
241 .cmp_eq,
242 .cmp_eq_optimized,
243 .cmp_gte,
244 .cmp_gte_optimized,
245 .cmp_gt,
246 .cmp_gt_optimized,
247 .cmp_neq,
248 .cmp_neq_optimized,
249 .bool_and,
250 .bool_or,
251 .store,
252 .store_safe,
253 .array_elem_val,
254 .slice_elem_val,
255 .ptr_elem_val,
256 .shl,
257 .shl_exact,
258 .shl_sat,
259 .shr,
260 .shr_exact,
261 .atomic_store_unordered,
262 .atomic_store_monotonic,
263 .atomic_store_release,
264 .atomic_store_seq_cst,
265 .set_union_tag,
266 .min,
267 .max,
268 .memset,
269 .memset_safe,
270 .memcpy,
271 .memmove,
272 => {
273 const bin_op = data[@intFromEnum(inst)].bin_op;
274 try self.verifyInstOperands(inst, .{ bin_op.lhs, bin_op.rhs, .none });
275 },
276 .add_with_overflow,
277 .sub_with_overflow,
278 .mul_with_overflow,
279 .shl_with_overflow,
280 .ptr_add,
281 .ptr_sub,
282 .ptr_elem_ptr,
283 .slice_elem_ptr,
284 .slice,
285 => {
286 const ty_pl = data[@intFromEnum(inst)].ty_pl;
287 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
288 try self.verifyInstOperands(inst, .{ extra.lhs, extra.rhs, .none });
289 },
290 .shuffle => {
291 const ty_pl = data[@intFromEnum(inst)].ty_pl;
292 const extra = self.air.extraData(Air.Shuffle, ty_pl.payload).data;
293 try self.verifyInstOperands(inst, .{ extra.a, extra.b, .none });
294 },
295 .cmp_vector,
296 .cmp_vector_optimized,
297 => {
298 const ty_pl = data[@intFromEnum(inst)].ty_pl;
299 const extra = self.air.extraData(Air.VectorCmp, ty_pl.payload).data;
300 try self.verifyInstOperands(inst, .{ extra.lhs, extra.rhs, .none });
301 },
302 .atomic_rmw => {
303 const pl_op = data[@intFromEnum(inst)].pl_op;
304 const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data;
305 try self.verifyInstOperands(inst, .{ pl_op.operand, extra.operand, .none });
306 },
307
308 // ternary
309 .select => {
310 const pl_op = data[@intFromEnum(inst)].pl_op;
311 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
312 try self.verifyInstOperands(inst, .{ pl_op.operand, extra.lhs, extra.rhs });
313 },
314 .mul_add => {
315 const pl_op = data[@intFromEnum(inst)].pl_op;
316 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
317 try self.verifyInstOperands(inst, .{ extra.lhs, extra.rhs, pl_op.operand });
318 },
319 .vector_store_elem => {
320 const vector_store_elem = data[@intFromEnum(inst)].vector_store_elem;
321 const extra = self.air.extraData(Air.Bin, vector_store_elem.payload).data;
322 try self.verifyInstOperands(inst, .{ vector_store_elem.vector_ptr, extra.lhs, extra.rhs });
323 },
324 .cmpxchg_strong,
325 .cmpxchg_weak,
326 => {
327 const ty_pl = data[@intFromEnum(inst)].ty_pl;
328 const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
329 try self.verifyInstOperands(inst, .{ extra.ptr, extra.expected_value, extra.new_value });
330 },
331
332 // big tombs
333 .aggregate_init => {
334 const ty_pl = data[@intFromEnum(inst)].ty_pl;
335 const aggregate_ty = ty_pl.ty.toType();
336 const len = @as(usize, @intCast(aggregate_ty.arrayLenIp(ip)));
337 const elements = @as([]const Air.Inst.Ref, @ptrCast(self.air.extra[ty_pl.payload..][0..len]));
338
339 var bt = self.liveness.iterateBigTomb(inst);
340 for (elements) |element| {
341 try self.verifyOperand(inst, element, bt.feed());
342 }
343 try self.verifyInst(inst);
344 },
345 .call, .call_always_tail, .call_never_tail, .call_never_inline => {
346 const pl_op = data[@intFromEnum(inst)].pl_op;
347 const extra = self.air.extraData(Air.Call, pl_op.payload);
348 const args = @as(
349 []const Air.Inst.Ref,
350 @ptrCast(self.air.extra[extra.end..][0..extra.data.args_len]),
351 );
352
353 var bt = self.liveness.iterateBigTomb(inst);
354 try self.verifyOperand(inst, pl_op.operand, bt.feed());
355 for (args) |arg| {
356 try self.verifyOperand(inst, arg, bt.feed());
357 }
358 try self.verifyInst(inst);
359 },
360 .assembly => {
361 const ty_pl = data[@intFromEnum(inst)].ty_pl;
362 const extra = self.air.extraData(Air.Asm, ty_pl.payload);
363 var extra_i = extra.end;
364 const outputs = @as(
365 []const Air.Inst.Ref,
366 @ptrCast(self.air.extra[extra_i..][0..extra.data.outputs_len]),
367 );
368 extra_i += outputs.len;
369 const inputs = @as(
370 []const Air.Inst.Ref,
371 @ptrCast(self.air.extra[extra_i..][0..extra.data.inputs_len]),
372 );
373 extra_i += inputs.len;
374
375 var bt = self.liveness.iterateBigTomb(inst);
376 for (outputs) |output| {
377 if (output != .none) {
378 try self.verifyOperand(inst, output, bt.feed());
379 }
380 }
381 for (inputs) |input| {
382 try self.verifyOperand(inst, input, bt.feed());
383 }
384 try self.verifyInst(inst);
385 },
386
387 // control flow
388 .@"try", .try_cold => {
389 const pl_op = data[@intFromEnum(inst)].pl_op;
390 const extra = self.air.extraData(Air.Try, pl_op.payload);
391 const try_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
392
393 const cond_br_liveness = self.liveness.getCondBr(inst);
394
395 try self.verifyOperand(inst, pl_op.operand, self.liveness.operandDies(inst, 0));
396
397 var live = try self.live.clone(self.gpa);
398 defer live.deinit(self.gpa);
399
400 for (cond_br_liveness.else_deaths) |death| try self.verifyDeath(inst, death);
401 try self.verifyBody(try_body);
402
403 self.live.deinit(self.gpa);
404 self.live = live.move();
405
406 for (cond_br_liveness.then_deaths) |death| try self.verifyDeath(inst, death);
407
408 try self.verifyInst(inst);
409 },
410 .try_ptr, .try_ptr_cold => {
411 const ty_pl = data[@intFromEnum(inst)].ty_pl;
412 const extra = self.air.extraData(Air.TryPtr, ty_pl.payload);
413 const try_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
414
415 const cond_br_liveness = self.liveness.getCondBr(inst);
416
417 try self.verifyOperand(inst, extra.data.ptr, self.liveness.operandDies(inst, 0));
418
419 var live = try self.live.clone(self.gpa);
420 defer live.deinit(self.gpa);
421
422 for (cond_br_liveness.else_deaths) |death| try self.verifyDeath(inst, death);
423 try self.verifyBody(try_body);
424
425 self.live.deinit(self.gpa);
426 self.live = live.move();
427
428 for (cond_br_liveness.then_deaths) |death| try self.verifyDeath(inst, death);
429
430 try self.verifyInst(inst);
431 },
432 .br => {
433 const br = data[@intFromEnum(inst)].br;
434 const gop = try self.blocks.getOrPut(self.gpa, br.block_inst);
435
436 try self.verifyOperand(inst, br.operand, self.liveness.operandDies(inst, 0));
437 if (gop.found_existing) {
438 try self.verifyMatchingLiveness(br.block_inst, gop.value_ptr.*);
439 } else {
440 gop.value_ptr.* = try self.live.clone(self.gpa);
441 }
442 try self.verifyInst(inst);
443 },
444 .repeat => {
445 const repeat = data[@intFromEnum(inst)].repeat;
446 const expected_live = self.loops.get(repeat.loop_inst) orelse
447 return invalid("%{}: loop %{} not in scope", .{ @intFromEnum(inst), @intFromEnum(repeat.loop_inst) });
448
449 try self.verifyMatchingLiveness(repeat.loop_inst, expected_live);
450 },
451 .switch_dispatch => {
452 const br = data[@intFromEnum(inst)].br;
453
454 try self.verifyOperand(inst, br.operand, self.liveness.operandDies(inst, 0));
455
456 const expected_live = self.loops.get(br.block_inst) orelse
457 return invalid("%{}: loop %{} not in scope", .{ @intFromEnum(inst), @intFromEnum(br.block_inst) });
458
459 try self.verifyMatchingLiveness(br.block_inst, expected_live);
460 },
461 .block, .dbg_inline_block => |tag| {
462 const ty_pl = data[@intFromEnum(inst)].ty_pl;
463 const block_ty = ty_pl.ty.toType();
464 const block_body: []const Air.Inst.Index = @ptrCast(switch (tag) {
465 inline .block, .dbg_inline_block => |comptime_tag| body: {
466 const extra = self.air.extraData(switch (comptime_tag) {
467 .block => Air.Block,
468 .dbg_inline_block => Air.DbgInlineBlock,
469 else => unreachable,
470 }, ty_pl.payload);
471 break :body self.air.extra[extra.end..][0..extra.data.body_len];
472 },
473 else => unreachable,
474 });
475 const block_liveness = self.liveness.getBlock(inst);
476
477 var orig_live = try self.live.clone(self.gpa);
478 defer orig_live.deinit(self.gpa);
479
480 assert(!self.blocks.contains(inst));
481 try self.verifyBody(block_body);
482
483 // Liveness data after the block body is garbage, but we want to
484 // restore it to verify deaths
485 self.live.deinit(self.gpa);
486 self.live = orig_live.move();
487
488 for (block_liveness.deaths) |death| try self.verifyDeath(inst, death);
489
490 if (ip.isNoReturn(block_ty.toIntern())) {
491 assert(!self.blocks.contains(inst));
492 } else {
493 var live = self.blocks.fetchRemove(inst).?.value;
494 defer live.deinit(self.gpa);
495
496 try self.verifyMatchingLiveness(inst, live);
497 }
498
499 try self.verifyInstOperands(inst, .{ .none, .none, .none });
500 },
501 .loop => {
502 const ty_pl = data[@intFromEnum(inst)].ty_pl;
503 const extra = self.air.extraData(Air.Block, ty_pl.payload);
504 const loop_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
505
506 // The same stuff should be alive after the loop as before it.
507 const gop = try self.loops.getOrPut(self.gpa, inst);
508 if (gop.found_existing) return invalid("%{}: loop already exists", .{@intFromEnum(inst)});
509 defer {
510 var live = self.loops.fetchRemove(inst).?;
511 live.value.deinit(self.gpa);
512 }
513 gop.value_ptr.* = try self.live.clone(self.gpa);
514
515 try self.verifyBody(loop_body);
516
517 try self.verifyInstOperands(inst, .{ .none, .none, .none });
518 },
519 .cond_br => {
520 const pl_op = data[@intFromEnum(inst)].pl_op;
521 const extra = self.air.extraData(Air.CondBr, pl_op.payload);
522 const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.then_body_len]);
523 const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]);
524 const cond_br_liveness = self.liveness.getCondBr(inst);
525
526 try self.verifyOperand(inst, pl_op.operand, self.liveness.operandDies(inst, 0));
527
528 var live = try self.live.clone(self.gpa);
529 defer live.deinit(self.gpa);
530
531 for (cond_br_liveness.then_deaths) |death| try self.verifyDeath(inst, death);
532 try self.verifyBody(then_body);
533
534 self.live.deinit(self.gpa);
535 self.live = live.move();
536
537 for (cond_br_liveness.else_deaths) |death| try self.verifyDeath(inst, death);
538 try self.verifyBody(else_body);
539
540 try self.verifyInst(inst);
541 },
542 .switch_br, .loop_switch_br => {
543 const switch_br = self.air.unwrapSwitch(inst);
544 const switch_br_liveness = try self.liveness.getSwitchBr(
545 self.gpa,
546 inst,
547 switch_br.cases_len + 1,
548 );
549 defer self.gpa.free(switch_br_liveness.deaths);
550
551 try self.verifyOperand(inst, switch_br.operand, self.liveness.operandDies(inst, 0));
552
553 // Excluding the operand (which we just handled), the same stuff should be alive
554 // after the loop as before it.
555 {
556 const gop = try self.loops.getOrPut(self.gpa, inst);
557 if (gop.found_existing) return invalid("%{}: loop already exists", .{@intFromEnum(inst)});
558 gop.value_ptr.* = self.live.move();
559 }
560 defer {
561 var live = self.loops.fetchRemove(inst).?;
562 live.value.deinit(self.gpa);
563 }
564
565 var it = switch_br.iterateCases();
566 while (it.next()) |case| {
567 self.live.deinit(self.gpa);
568 self.live = try self.loops.get(inst).?.clone(self.gpa);
569
570 for (switch_br_liveness.deaths[case.idx]) |death| try self.verifyDeath(inst, death);
571 try self.verifyBody(case.body);
572 }
573
574 const else_body = it.elseBody();
575 if (else_body.len > 0) {
576 self.live.deinit(self.gpa);
577 self.live = try self.loops.get(inst).?.clone(self.gpa);
578 for (switch_br_liveness.deaths[switch_br.cases_len]) |death| try self.verifyDeath(inst, death);
579 try self.verifyBody(else_body);
580 }
581
582 try self.verifyInst(inst);
583 },
584 }
585 }
586}
587
588fn verifyDeath(self: *Verify, inst: Air.Inst.Index, operand: Air.Inst.Index) Error!void {
589 try self.verifyOperand(inst, operand.toRef(), true);
590}
591
592fn verifyOperand(self: *Verify, inst: Air.Inst.Index, op_ref: Air.Inst.Ref, dies: bool) Error!void {
593 const operand = op_ref.toIndexAllowNone() orelse {
594 assert(!dies);
595 return;
596 };
597 if (dies) {
598 if (!self.live.remove(operand)) return invalid("%{}: dead operand %{} reused and killed again", .{ inst, operand });
599 } else {
600 if (!self.live.contains(operand)) return invalid("%{}: dead operand %{} reused", .{ inst, operand });
601 }
602}
603
604fn verifyInstOperands(
605 self: *Verify,
606 inst: Air.Inst.Index,
607 operands: [Liveness.bpi - 1]Air.Inst.Ref,
608) Error!void {
609 for (operands, 0..) |operand, operand_index| {
610 const dies = self.liveness.operandDies(inst, @as(Liveness.OperandInt, @intCast(operand_index)));
611 try self.verifyOperand(inst, operand, dies);
612 }
613 try self.verifyInst(inst);
614}
615
616fn verifyInst(self: *Verify, inst: Air.Inst.Index) Error!void {
617 if (self.liveness.isUnused(inst)) {
618 assert(!self.live.contains(inst));
619 } else {
620 try self.live.putNoClobber(self.gpa, inst, {});
621 }
622}
623
624fn verifyMatchingLiveness(self: *Verify, block: Air.Inst.Index, live: LiveMap) Error!void {
625 if (self.live.count() != live.count()) return invalid("%{}: different deaths across branches", .{block});
626 var live_it = self.live.keyIterator();
627 while (live_it.next()) |live_inst| if (!live.contains(live_inst.*)) return invalid("%{}: different deaths across branches", .{block});
628}
629
630fn invalid(comptime fmt: []const u8, args: anytype) error{LivenessInvalid} {
631 log.err(fmt, args);
632 return error.LivenessInvalid;
633}
634
635const std = @import("std");
636const assert = std.debug.assert;
637const log = std.log.scoped(.liveness_verify);
638
639const Air = @import("../Air.zig");
640const Liveness = @import("../Liveness.zig");
641const InternPool = @import("../InternPool.zig");
642const Verify = @This();
src/Sema.zig+2-8
......@@ -756,13 +756,7 @@ pub const Block = struct {
756756 fn addReduce(block: *Block, operand: Air.Inst.Ref, operation: std.builtin.ReduceOp) !Air.Inst.Ref {
757757 const sema = block.sema;
758758 const zcu = sema.pt.zcu;
759 const vector_ty = sema.typeOf(operand);
760 switch (vector_ty.vectorLen(zcu)) {
761 0 => unreachable,
762 1 => return block.addBinOp(.array_elem_val, operand, .zero_usize),
763 else => {},
764 }
765 const allow_optimized = switch (vector_ty.childType(zcu).zigTypeTag(zcu)) {
759 const allow_optimized = switch (sema.typeOf(operand).childType(zcu).zigTypeTag(zcu)) {
766760 .float => true,
767761 .bool, .int => false,
768762 else => unreachable,
......@@ -36849,7 +36843,7 @@ fn typeOf(sema: *Sema, inst: Air.Inst.Ref) Type {
3684936843pub fn getTmpAir(sema: Sema) Air {
3685036844 return .{
3685136845 .instructions = sema.air_instructions.slice(),
36852 .extra = sema.air_extra.items,
36846 .extra = sema.air_extra,
3685336847 };
3685436848}
3685536849
src/Zcu.zig-1
......@@ -30,7 +30,6 @@ const AstGen = std.zig.AstGen;
3030const Sema = @import("Sema.zig");
3131const target_util = @import("target.zig");
3232const build_options = @import("build_options");
33const Liveness = @import("Liveness.zig");
3433const isUpDir = @import("introspect.zig").isUpDir;
3534const clang = @import("clang.zig");
3635const InternPool = @import("InternPool.zig");
src/Zcu/PerThread.zig+28-24
......@@ -16,7 +16,6 @@ const dev = @import("../dev.zig");
1616const InternPool = @import("../InternPool.zig");
1717const AnalUnit = InternPool.AnalUnit;
1818const introspect = @import("../introspect.zig");
19const Liveness = @import("../Liveness.zig");
2019const log = std.log.scoped(.zcu);
2120const Module = @import("../Package.zig").Module;
2221const Sema = @import("../Sema.zig");
......@@ -1721,34 +1720,43 @@ fn analyzeFuncBody(
17211720
17221721/// Takes ownership of `air`, even on error.
17231722/// If any types referenced by `air` are unresolved, marks the codegen as failed.
1724pub fn linkerUpdateFunc(pt: Zcu.PerThread, func_index: InternPool.Index, air: Air) Allocator.Error!void {
1723pub fn linkerUpdateFunc(pt: Zcu.PerThread, func_index: InternPool.Index, air: *Air) Allocator.Error!void {
17251724 const zcu = pt.zcu;
17261725 const gpa = zcu.gpa;
17271726 const ip = &zcu.intern_pool;
17281727 const comp = zcu.comp;
17291728
1730 defer {
1731 var air_mut = air;
1732 air_mut.deinit(gpa);
1733 }
1734
17351729 const func = zcu.funcInfo(func_index);
17361730 const nav_index = func.owner_nav;
17371731 const nav = ip.getNav(nav_index);
17381732
1739 var liveness = try Liveness.analyze(gpa, air, ip);
1733 const codegen_prog_node = zcu.codegen_prog_node.start(nav.fqn.toSlice(ip), 0);
1734 defer codegen_prog_node.end();
1735
1736 if (!air.typesFullyResolved(zcu)) {
1737 // A type we depend on failed to resolve. This is a transitive failure.
1738 // Correcting this failure will involve changing a type this function
1739 // depends on, hence triggering re-analysis of this function, so this
1740 // interacts correctly with incremental compilation.
1741 return;
1742 }
1743
1744 const backend = target_util.zigBackend(zcu.root_mod.resolved_target.result, zcu.comp.config.use_llvm);
1745 try air.legalize(backend, zcu);
1746
1747 var liveness = try Air.Liveness.analyze(gpa, air.*, ip);
17401748 defer liveness.deinit(gpa);
17411749
17421750 if (build_options.enable_debug_extensions and comp.verbose_air) {
17431751 std.debug.print("# Begin Function AIR: {}:\n", .{nav.fqn.fmt(ip)});
1744 @import("../print_air.zig").dump(pt, air, liveness);
1752 @import("../print_air.zig").dump(pt, air.*, liveness);
17451753 std.debug.print("# End Function AIR: {}\n\n", .{nav.fqn.fmt(ip)});
17461754 }
17471755
17481756 if (std.debug.runtime_safety) {
1749 var verify: Liveness.Verify = .{
1757 var verify: Air.Liveness.Verify = .{
17501758 .gpa = gpa,
1751 .air = air,
1759 .air = air.*,
17521760 .liveness = liveness,
17531761 .intern_pool = ip,
17541762 };
......@@ -1768,16 +1776,8 @@ pub fn linkerUpdateFunc(pt: Zcu.PerThread, func_index: InternPool.Index, air: Ai
17681776 };
17691777 }
17701778
1771 const codegen_prog_node = zcu.codegen_prog_node.start(nav.fqn.toSlice(ip), 0);
1772 defer codegen_prog_node.end();
1773
1774 if (!air.typesFullyResolved(zcu)) {
1775 // A type we depend on failed to resolve. This is a transitive failure.
1776 // Correcting this failure will involve changing a type this function
1777 // depends on, hence triggering re-analysis of this function, so this
1778 // interacts correctly with incremental compilation.
1779 } else if (comp.bin_file) |lf| {
1780 lf.updateFunc(pt, func_index, air, liveness) catch |err| switch (err) {
1779 if (comp.bin_file) |lf| {
1780 lf.updateFunc(pt, func_index, air.*, liveness) catch |err| switch (err) {
17811781 error.OutOfMemory => return error.OutOfMemory,
17821782 error.CodegenFail => assert(zcu.failed_codegen.contains(nav_index)),
17831783 error.Overflow, error.RelocationNotByteAligned => {
......@@ -1791,7 +1791,7 @@ pub fn linkerUpdateFunc(pt: Zcu.PerThread, func_index: InternPool.Index, air: Ai
17911791 },
17921792 };
17931793 } else if (zcu.llvm_object) |llvm_object| {
1794 llvm_object.updateFunc(pt, func_index, air, liveness) catch |err| switch (err) {
1794 llvm_object.updateFunc(pt, func_index, air.*, liveness) catch |err| switch (err) {
17951795 error.OutOfMemory => return error.OutOfMemory,
17961796 };
17971797 }
......@@ -3080,9 +3080,13 @@ fn analyzeFnBodyInner(pt: Zcu.PerThread, func_index: InternPool.Index) Zcu.SemaE
30803080
30813081 try sema.flushExports();
30823082
3083 defer {
3084 sema.air_instructions = .empty;
3085 sema.air_extra = .empty;
3086 }
30833087 return .{
3084 .instructions = sema.air_instructions.toOwnedSlice(),
3085 .extra = try sema.air_extra.toOwnedSlice(gpa),
3088 .instructions = sema.air_instructions.slice(),
3089 .extra = sema.air_extra,
30863090 };
30873091}
30883092
src/arch/aarch64/CodeGen.zig+32-33
......@@ -7,7 +7,6 @@ const codegen = @import("../../codegen.zig");
77const Air = @import("../../Air.zig");
88const Mir = @import("Mir.zig");
99const Emit = @import("Emit.zig");
10const Liveness = @import("../../Liveness.zig");
1110const Type = @import("../../Type.zig");
1211const Value = @import("../../Value.zig");
1312const link = @import("../../link.zig");
......@@ -44,7 +43,7 @@ const InnerError = CodeGenError || error{OutOfRegisters};
4443gpa: Allocator,
4544pt: Zcu.PerThread,
4645air: Air,
47liveness: Liveness,
46liveness: Air.Liveness,
4847bin_file: *link.File,
4948debug_output: link.File.DebugInfoOutput,
5049target: *const std.Target,
......@@ -71,7 +70,7 @@ end_di_column: u32,
7170/// which is a relative jump, based on the address following the reloc.
7271exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty,
7372
74reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined,
73reused_operands: std.StaticBitSet(Air.Liveness.bpi - 1) = undefined,
7574
7675/// We postpone the creation of debug info for function args and locals
7776/// until after all Mir instructions have been generated. Only then we
......@@ -273,7 +272,7 @@ const BlockData = struct {
273272const BigTomb = struct {
274273 function: *Self,
275274 inst: Air.Inst.Index,
276 lbt: Liveness.BigTomb,
275 lbt: Air.Liveness.BigTomb,
277276
278277 fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void {
279278 const dies = bt.lbt.feed();
......@@ -324,7 +323,7 @@ pub fn generate(
324323 src_loc: Zcu.LazySrcLoc,
325324 func_index: InternPool.Index,
326325 air: Air,
327 liveness: Liveness,
326 liveness: Air.Liveness,
328327 code: *std.ArrayListUnmanaged(u8),
329328 debug_output: link.File.DebugInfoOutput,
330329) CodeGenError!void {
......@@ -646,7 +645,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
646645 continue;
647646
648647 const old_air_bookkeeping = self.air_bookkeeping;
649 try self.ensureProcessDeathCapacity(Liveness.bpi);
648 try self.ensureProcessDeathCapacity(Air.Liveness.bpi);
650649
651650 self.reused_operands = @TypeOf(self.reused_operands).initEmpty();
652651 switch (air_tags[@intFromEnum(inst)]) {
......@@ -930,14 +929,14 @@ fn finishAirBookkeeping(self: *Self) void {
930929 }
931930}
932931
933fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref) void {
932fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Air.Liveness.bpi - 1]Air.Inst.Ref) void {
934933 const tomb_bits = self.liveness.getTombBits(inst);
935934 for (0.., operands) |op_index, op| {
936 if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
935 if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
937936 if (self.reused_operands.isSet(op_index)) continue;
938937 self.processDeath(op.toIndexAllowNone() orelse continue);
939938 }
940 if (tomb_bits & 1 << (Liveness.bpi - 1) == 0) {
939 if (tomb_bits & 1 << (Air.Liveness.bpi - 1) == 0) {
941940 log.debug("%{d} => {}", .{ inst, result });
942941 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
943942 branch.inst_table.putAssumeCapacityNoClobber(inst, result);
......@@ -1568,7 +1567,7 @@ const ReuseMetadata = struct {
15681567 /// inputs to the Air instruction are omitted (e.g. when they can
15691568 /// be represented as immediates to the Mir instruction),
15701569 /// operand_mapping should reflect that fact.
1571 operand_mapping: []const Liveness.OperandInt,
1570 operand_mapping: []const Air.Liveness.OperandInt,
15721571};
15731572
15741573/// Allocate a set of registers for use as arguments for a Mir
......@@ -1835,7 +1834,7 @@ fn binOpImmediate(
18351834 const write_args = [_]WriteArg{
18361835 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &dest_reg },
18371836 };
1838 const operand_mapping: []const Liveness.OperandInt = if (lhs_and_rhs_swapped) &.{1} else &.{0};
1837 const operand_mapping: []const Air.Liveness.OperandInt = if (lhs_and_rhs_swapped) &.{1} else &.{0};
18391838 try self.allocRegs(
18401839 &read_args,
18411840 &write_args,
......@@ -3584,7 +3583,7 @@ fn reuseOperand(
35843583 self: *Self,
35853584 inst: Air.Inst.Index,
35863585 operand: Air.Inst.Ref,
3587 op_index: Liveness.OperandInt,
3586 op_index: Air.Liveness.OperandInt,
35883587 mcv: MCValue,
35893588) bool {
35903589 if (!self.liveness.operandDies(inst, op_index))
......@@ -4250,7 +4249,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
42504249 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
42514250 const callee = pl_op.operand;
42524251 const extra = self.air.extraData(Air.Call, pl_op.payload);
4253 const args = @as([]const Air.Inst.Ref, @ptrCast(self.air.extra[extra.end..][0..extra.data.args_len]));
4252 const args: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.args_len]);
42544253 const ty = self.typeOf(callee);
42554254 const pt = self.pt;
42564255 const zcu = pt.zcu;
......@@ -4389,8 +4388,8 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
43894388 break :result info.return_value;
43904389 };
43914390
4392 if (args.len + 1 <= Liveness.bpi - 1) {
4393 var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
4391 if (args.len + 1 <= Air.Liveness.bpi - 1) {
4392 var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1);
43944393 buf[0] = callee;
43954394 @memcpy(buf[1..][0..args.len], args);
43964395 return self.finishAir(inst, result, buf);
......@@ -4613,7 +4612,7 @@ fn airDbgInlineBlock(self: *Self, inst: Air.Inst.Index) InnerError!void {
46134612 const func = zcu.funcInfo(extra.data.func);
46144613 // TODO emit debug info for function change
46154614 _ = func;
4616 try self.lowerBlock(inst, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]));
4615 try self.lowerBlock(inst, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]));
46174616}
46184617
46194618fn airDbgVar(self: *Self, inst: Air.Inst.Index) InnerError!void {
......@@ -4671,8 +4670,8 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) InnerError!void {
46714670 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
46724671 const cond = try self.resolveInst(pl_op.operand);
46734672 const extra = self.air.extraData(Air.CondBr, pl_op.payload);
4674 const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.then_body_len]);
4675 const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]);
4673 const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.then_body_len]);
4674 const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]);
46764675 const liveness_condbr = self.liveness.getCondBr(inst);
46774676
46784677 const reloc = try self.condBr(cond);
......@@ -5016,7 +5015,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) InnerError!void {
50165015 // A loop is a setup to be able to jump back to the beginning.
50175016 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
50185017 const loop = self.air.extraData(Air.Block, ty_pl.payload);
5019 const body: []const Air.Inst.Index = @ptrCast(self.air.extra[loop.end..][0..loop.data.body_len]);
5018 const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[loop.end..][0..loop.data.body_len]);
50205019 const start_index = @as(u32, @intCast(self.mir_instructions.len));
50215020
50225021 try self.genBody(body);
......@@ -5036,7 +5035,7 @@ fn jump(self: *Self, inst: Mir.Inst.Index) !void {
50365035fn airBlock(self: *Self, inst: Air.Inst.Index) InnerError!void {
50375036 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
50385037 const extra = self.air.extraData(Air.Block, ty_pl.payload);
5039 try self.lowerBlock(inst, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]));
5038 try self.lowerBlock(inst, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]));
50405039}
50415040
50425041fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void {
......@@ -5255,9 +5254,9 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) InnerError!void {
52555254 const is_volatile = @as(u1, @truncate(extra.data.flags >> 31)) != 0;
52565255 const clobbers_len = @as(u31, @truncate(extra.data.flags));
52575256 var extra_i: usize = extra.end;
5258 const outputs = @as([]const Air.Inst.Ref, @ptrCast(self.air.extra[extra_i..][0..extra.data.outputs_len]));
5257 const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.outputs_len]);
52595258 extra_i += outputs.len;
5260 const inputs = @as([]const Air.Inst.Ref, @ptrCast(self.air.extra[extra_i..][0..extra.data.inputs_len]));
5259 const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.inputs_len]);
52615260 extra_i += inputs.len;
52625261
52635262 const dead = !is_volatile and self.liveness.isUnused(inst);
......@@ -5270,8 +5269,8 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) InnerError!void {
52705269 if (output != .none) {
52715270 return self.fail("TODO implement codegen for non-expr asm", .{});
52725271 }
5273 const extra_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]);
5274 const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0);
5272 const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]);
5273 const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0);
52755274 const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
52765275 // This equation accounts for the fact that even if we have exactly 4 bytes
52775276 // for the string, we still use the next u32 for the null terminator.
......@@ -5281,7 +5280,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) InnerError!void {
52815280 } else null;
52825281
52835282 for (inputs) |input| {
5284 const input_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]);
5283 const input_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]);
52855284 const constraint = std.mem.sliceTo(input_bytes, 0);
52865285 const name = std.mem.sliceTo(input_bytes[constraint.len + 1 ..], 0);
52875286 // This equation accounts for the fact that even if we have exactly 4 bytes
......@@ -5303,7 +5302,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) InnerError!void {
53035302 {
53045303 var clobber_i: u32 = 0;
53055304 while (clobber_i < clobbers_len) : (clobber_i += 1) {
5306 const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0);
5305 const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0);
53075306 // This equation accounts for the fact that even if we have exactly 4 bytes
53085307 // for the string, we still use the next u32 for the null terminator.
53095308 extra_i += clobber.len / 4 + 1;
......@@ -5312,7 +5311,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) InnerError!void {
53125311 }
53135312 }
53145313
5315 const asm_source = std.mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len];
5314 const asm_source = std.mem.sliceAsBytes(self.air.extra.items[extra_i..])[0..extra.data.source_len];
53165315
53175316 if (mem.eql(u8, asm_source, "svc #0")) {
53185317 _ = try self.addInst(.{
......@@ -5342,7 +5341,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) InnerError!void {
53425341 };
53435342
53445343 simple: {
5345 var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
5344 var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1);
53465345 var buf_index: usize = 0;
53475346 for (outputs) |output| {
53485347 if (output == .none) continue;
......@@ -6052,14 +6051,14 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) InnerError!void {
60526051 const vector_ty = self.typeOfIndex(inst);
60536052 const len = vector_ty.vectorLen(zcu);
60546053 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6055 const elements = @as([]const Air.Inst.Ref, @ptrCast(self.air.extra[ty_pl.payload..][0..len]));
6054 const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]);
60566055 const result: MCValue = res: {
60576056 if (self.liveness.isUnused(inst)) break :res MCValue.dead;
60586057 return self.fail("TODO implement airAggregateInit for {}", .{self.target.cpu.arch});
60596058 };
60606059
6061 if (elements.len <= Liveness.bpi - 1) {
6062 var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
6060 if (elements.len <= Air.Liveness.bpi - 1) {
6061 var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1);
60636062 @memcpy(buf[0..elements.len], elements);
60646063 return self.finishAir(inst, result, buf);
60656064 }
......@@ -6095,7 +6094,7 @@ fn airTry(self: *Self, inst: Air.Inst.Index) InnerError!void {
60956094 const pt = self.pt;
60966095 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
60976096 const extra = self.air.extraData(Air.Try, pl_op.payload);
6098 const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
6097 const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
60996098 const result: MCValue = result: {
61006099 const error_union_bind: ReadArg.Bind = .{ .inst = pl_op.operand };
61016100 const error_union_ty = self.typeOf(pl_op.operand);
......@@ -6122,7 +6121,7 @@ fn airTry(self: *Self, inst: Air.Inst.Index) InnerError!void {
61226121fn airTryPtr(self: *Self, inst: Air.Inst.Index) InnerError!void {
61236122 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
61246123 const extra = self.air.extraData(Air.TryPtr, ty_pl.payload);
6125 const body = self.air.extra[extra.end..][0..extra.data.body_len];
6124 const body = self.air.extra.items[extra.end..][0..extra.data.body_len];
61266125 _ = body;
61276126 return self.fail("TODO implement airTryPtr for arm", .{});
61286127 // return self.finishAir(inst, result, .{ extra.data.ptr, .none, .none });
src/arch/arm/CodeGen.zig+32-33
......@@ -7,7 +7,6 @@ const codegen = @import("../../codegen.zig");
77const Air = @import("../../Air.zig");
88const Mir = @import("Mir.zig");
99const Emit = @import("Emit.zig");
10const Liveness = @import("../../Liveness.zig");
1110const Type = @import("../../Type.zig");
1211const Value = @import("../../Value.zig");
1312const link = @import("../../link.zig");
......@@ -45,7 +44,7 @@ const InnerError = CodeGenError || error{OutOfRegisters};
4544gpa: Allocator,
4645pt: Zcu.PerThread,
4746air: Air,
48liveness: Liveness,
47liveness: Air.Liveness,
4948bin_file: *link.File,
5049debug_output: link.File.DebugInfoOutput,
5150target: *const std.Target,
......@@ -72,7 +71,7 @@ end_di_column: u32,
7271/// which is a relative jump, based on the address following the reloc.
7372exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty,
7473
75reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined,
74reused_operands: std.StaticBitSet(Air.Liveness.bpi - 1) = undefined,
7675
7776/// We postpone the creation of debug info for function args and locals
7877/// until after all Mir instructions have been generated. Only then we
......@@ -195,7 +194,7 @@ const BlockData = struct {
195194const BigTomb = struct {
196195 function: *Self,
197196 inst: Air.Inst.Index,
198 lbt: Liveness.BigTomb,
197 lbt: Air.Liveness.BigTomb,
199198
200199 fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void {
201200 const dies = bt.lbt.feed();
......@@ -333,7 +332,7 @@ pub fn generate(
333332 src_loc: Zcu.LazySrcLoc,
334333 func_index: InternPool.Index,
335334 air: Air,
336 liveness: Liveness,
335 liveness: Air.Liveness,
337336 code: *std.ArrayListUnmanaged(u8),
338337 debug_output: link.File.DebugInfoOutput,
339338) CodeGenError!void {
......@@ -635,7 +634,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
635634 continue;
636635
637636 const old_air_bookkeeping = self.air_bookkeeping;
638 try self.ensureProcessDeathCapacity(Liveness.bpi);
637 try self.ensureProcessDeathCapacity(Air.Liveness.bpi);
639638
640639 self.reused_operands = @TypeOf(self.reused_operands).initEmpty();
641640 switch (air_tags[@intFromEnum(inst)]) {
......@@ -921,14 +920,14 @@ fn finishAirBookkeeping(self: *Self) void {
921920 }
922921}
923922
924fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref) void {
923fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Air.Liveness.bpi - 1]Air.Inst.Ref) void {
925924 const tomb_bits = self.liveness.getTombBits(inst);
926925 for (0.., operands) |op_index, op| {
927 if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
926 if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
928927 if (self.reused_operands.isSet(op_index)) continue;
929928 self.processDeath(op.toIndexAllowNone() orelse continue);
930929 }
931 if (tomb_bits & 1 << (Liveness.bpi - 1) == 0) {
930 if (tomb_bits & 1 << (Air.Liveness.bpi - 1) == 0) {
932931 log.debug("%{d} => {}", .{ inst, result });
933932 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
934933 branch.inst_table.putAssumeCapacityNoClobber(inst, result);
......@@ -2617,7 +2616,7 @@ fn reuseOperand(
26172616 self: *Self,
26182617 inst: Air.Inst.Index,
26192618 operand: Air.Inst.Ref,
2620 op_index: Liveness.OperandInt,
2619 op_index: Air.Liveness.OperandInt,
26212620 mcv: MCValue,
26222621) bool {
26232622 if (!self.liveness.operandDies(inst, op_index))
......@@ -3094,7 +3093,7 @@ const ReuseMetadata = struct {
30943093 /// inputs to the Air instruction are omitted (e.g. when they can
30953094 /// be represented as immediates to the Mir instruction),
30963095 /// operand_mapping should reflect that fact.
3097 operand_mapping: []const Liveness.OperandInt,
3096 operand_mapping: []const Air.Liveness.OperandInt,
30983097};
30993098
31003099/// Allocate a set of registers for use as arguments for a Mir
......@@ -3342,7 +3341,7 @@ fn binOpImmediate(
33423341 const write_args = [_]WriteArg{
33433342 .{ .ty = lhs_ty, .bind = .none, .class = gp, .reg = &dest_reg },
33443343 };
3345 const operand_mapping: []const Liveness.OperandInt = if (lhs_and_rhs_swapped) &.{1} else &.{0};
3344 const operand_mapping: []const Air.Liveness.OperandInt = if (lhs_and_rhs_swapped) &.{1} else &.{0};
33463345 try self.allocRegs(
33473346 &read_args,
33483347 &write_args,
......@@ -4232,7 +4231,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
42324231 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
42334232 const callee = pl_op.operand;
42344233 const extra = self.air.extraData(Air.Call, pl_op.payload);
4235 const args: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra.end..][0..extra.data.args_len]);
4234 const args: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.args_len]);
42364235 const ty = self.typeOf(callee);
42374236 const pt = self.pt;
42384237 const zcu = pt.zcu;
......@@ -4361,8 +4360,8 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
43614360 break :result info.return_value;
43624361 };
43634362
4364 if (args.len <= Liveness.bpi - 2) {
4365 var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
4363 if (args.len <= Air.Liveness.bpi - 2) {
4364 var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1);
43664365 buf[0] = callee;
43674366 @memcpy(buf[1..][0..args.len], args);
43684367 return self.finishAir(inst, result, buf);
......@@ -4585,7 +4584,7 @@ fn airDbgInlineBlock(self: *Self, inst: Air.Inst.Index) !void {
45854584 const func = zcu.funcInfo(extra.data.func);
45864585 // TODO emit debug info for function change
45874586 _ = func;
4588 try self.lowerBlock(inst, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]));
4587 try self.lowerBlock(inst, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]));
45894588}
45904589
45914590fn airDbgVar(self: *Self, inst: Air.Inst.Index) !void {
......@@ -4646,8 +4645,8 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
46464645 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
46474646 const cond_inst = try self.resolveInst(pl_op.operand);
46484647 const extra = self.air.extraData(Air.CondBr, pl_op.payload);
4649 const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.then_body_len]);
4650 const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]);
4648 const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.then_body_len]);
4649 const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]);
46514650 const liveness_condbr = self.liveness.getCondBr(inst);
46524651
46534652 const reloc: Mir.Inst.Index = try self.condBr(cond_inst);
......@@ -4966,7 +4965,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
49664965 // A loop is a setup to be able to jump back to the beginning.
49674966 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
49684967 const loop = self.air.extraData(Air.Block, ty_pl.payload);
4969 const body: []const Air.Inst.Index = @ptrCast(self.air.extra[loop.end..][0..loop.data.body_len]);
4968 const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[loop.end..][0..loop.data.body_len]);
49704969 const start_index: Mir.Inst.Index = @intCast(self.mir_instructions.len);
49714970
49724971 try self.genBody(body);
......@@ -4986,7 +4985,7 @@ fn jump(self: *Self, inst: Mir.Inst.Index) !void {
49864985fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
49874986 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
49884987 const extra = self.air.extraData(Air.Block, ty_pl.payload);
4989 try self.lowerBlock(inst, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]));
4988 try self.lowerBlock(inst, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]));
49904989}
49914990
49924991fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void {
......@@ -5199,9 +5198,9 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
51995198 const is_volatile = @as(u1, @truncate(extra.data.flags >> 31)) != 0;
52005199 const clobbers_len: u31 = @truncate(extra.data.flags);
52015200 var extra_i: usize = extra.end;
5202 const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra_i..][0..extra.data.outputs_len]);
5201 const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.outputs_len]);
52035202 extra_i += outputs.len;
5204 const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra_i..][0..extra.data.inputs_len]);
5203 const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.inputs_len]);
52055204 extra_i += inputs.len;
52065205
52075206 const dead = !is_volatile and self.liveness.isUnused(inst);
......@@ -5214,8 +5213,8 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
52145213 if (output != .none) {
52155214 return self.fail("TODO implement codegen for non-expr asm", .{});
52165215 }
5217 const extra_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]);
5218 const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0);
5216 const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]);
5217 const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0);
52195218 const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
52205219 // This equation accounts for the fact that even if we have exactly 4 bytes
52215220 // for the string, we still use the next u32 for the null terminator.
......@@ -5225,7 +5224,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
52255224 } else null;
52265225
52275226 for (inputs) |input| {
5228 const input_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]);
5227 const input_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]);
52295228 const constraint = std.mem.sliceTo(input_bytes, 0);
52305229 const name = std.mem.sliceTo(input_bytes[constraint.len + 1 ..], 0);
52315230 // This equation accounts for the fact that even if we have exactly 4 bytes
......@@ -5247,7 +5246,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
52475246 {
52485247 var clobber_i: u32 = 0;
52495248 while (clobber_i < clobbers_len) : (clobber_i += 1) {
5250 const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0);
5249 const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0);
52515250 // This equation accounts for the fact that even if we have exactly 4 bytes
52525251 // for the string, we still use the next u32 for the null terminator.
52535252 extra_i += clobber.len / 4 + 1;
......@@ -5256,7 +5255,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
52565255 }
52575256 }
52585257
5259 const asm_source = std.mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len];
5258 const asm_source = std.mem.sliceAsBytes(self.air.extra.items[extra_i..])[0..extra.data.source_len];
52605259
52615260 if (mem.eql(u8, asm_source, "svc #0")) {
52625261 _ = try self.addInst(.{
......@@ -5282,7 +5281,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
52825281 };
52835282
52845283 simple: {
5285 var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
5284 var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1);
52865285 var buf_index: usize = 0;
52875286 for (outputs) |output| {
52885287 if (output == .none) continue;
......@@ -6021,14 +6020,14 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
60216020 const vector_ty = self.typeOfIndex(inst);
60226021 const len = vector_ty.vectorLen(zcu);
60236022 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6024 const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra[ty_pl.payload..][0..len]);
6023 const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]);
60256024 const result: MCValue = res: {
60266025 if (self.liveness.isUnused(inst)) break :res MCValue.dead;
60276026 return self.fail("TODO implement airAggregateInit for arm", .{});
60286027 };
60296028
6030 if (elements.len <= Liveness.bpi - 1) {
6031 var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
6029 if (elements.len <= Air.Liveness.bpi - 1) {
6030 var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1);
60326031 @memcpy(buf[0..elements.len], elements);
60336032 return self.finishAir(inst, result, buf);
60346033 }
......@@ -6065,7 +6064,7 @@ fn airTry(self: *Self, inst: Air.Inst.Index) !void {
60656064 const pt = self.pt;
60666065 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
60676066 const extra = self.air.extraData(Air.Try, pl_op.payload);
6068 const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
6067 const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
60696068 const result: MCValue = result: {
60706069 const error_union_bind: ReadArg.Bind = .{ .inst = pl_op.operand };
60716070 const error_union_ty = self.typeOf(pl_op.operand);
......@@ -6092,7 +6091,7 @@ fn airTry(self: *Self, inst: Air.Inst.Index) !void {
60926091fn airTryPtr(self: *Self, inst: Air.Inst.Index) !void {
60936092 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
60946093 const extra = self.air.extraData(Air.TryPtr, ty_pl.payload);
6095 const body = self.air.extra[extra.end..][0..extra.data.body_len];
6094 const body = self.air.extra.items[extra.end..][0..extra.data.body_len];
60966095 _ = body;
60976096 return self.fail("TODO implement airTryPtr for arm", .{});
60986097 // return self.finishAir(inst, result, .{ extra.data.ptr, .none, .none });
src/arch/powerpc/CodeGen.zig+1-2
......@@ -5,7 +5,6 @@ const Air = @import("../../Air.zig");
55const codegen = @import("../../codegen.zig");
66const InternPool = @import("../../InternPool.zig");
77const link = @import("../../link.zig");
8const Liveness = @import("../../Liveness.zig");
98const Zcu = @import("../../Zcu.zig");
109
1110const assert = std.debug.assert;
......@@ -17,7 +16,7 @@ pub fn generate(
1716 src_loc: Zcu.LazySrcLoc,
1817 func_index: InternPool.Index,
1918 air: Air,
20 liveness: Liveness,
19 liveness: Air.Liveness,
2120 code: *std.ArrayListUnmanaged(u8),
2221 debug_output: link.File.DebugInfoOutput,
2322) codegen.CodeGenError!void {
src/arch/riscv64/CodeGen.zig+29-30
......@@ -10,7 +10,6 @@ const Allocator = mem.Allocator;
1010const Air = @import("../../Air.zig");
1111const Mir = @import("Mir.zig");
1212const Emit = @import("Emit.zig");
13const Liveness = @import("../../Liveness.zig");
1413const Type = @import("../../Type.zig");
1514const Value = @import("../../Value.zig");
1615const link = @import("../../link.zig");
......@@ -54,7 +53,7 @@ const InnerError = CodeGenError || error{OutOfRegisters};
5453
5554pt: Zcu.PerThread,
5655air: Air,
57liveness: Liveness,
56liveness: Air.Liveness,
5857bin_file: *link.File,
5958gpa: Allocator,
6059
......@@ -82,7 +81,7 @@ scope_generation: u32,
8281/// which is a relative jump, based on the address following the reloc.
8382exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty,
8483
85reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined,
84reused_operands: std.StaticBitSet(Air.Liveness.bpi - 1) = undefined,
8685
8786/// Whenever there is a runtime branch, we push a Branch onto this stack,
8887/// and pop it off when the runtime branch joins. This provides an "overlay"
......@@ -739,7 +738,7 @@ pub fn generate(
739738 src_loc: Zcu.LazySrcLoc,
740739 func_index: InternPool.Index,
741740 air: Air,
742 liveness: Liveness,
741 liveness: Air.Liveness,
743742 code: *std.ArrayListUnmanaged(u8),
744743 debug_output: link.File.DebugInfoOutput,
745744) CodeGenError!void {
......@@ -1426,7 +1425,7 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
14261425 verbose_tracking_log.debug("{}", .{func.fmtTracking()});
14271426
14281427 const old_air_bookkeeping = func.air_bookkeeping;
1429 try func.ensureProcessDeathCapacity(Liveness.bpi);
1428 try func.ensureProcessDeathCapacity(Air.Liveness.bpi);
14301429
14311430 func.reused_operands = @TypeOf(func.reused_operands).initEmpty();
14321431 try func.inst_tracking.ensureUnusedCapacity(func.gpa, 1);
......@@ -1731,7 +1730,7 @@ fn freeValue(func: *Func, value: MCValue) !void {
17311730 }
17321731}
17331732
1734fn feed(func: *Func, bt: *Liveness.BigTomb, operand: Air.Inst.Ref) !void {
1733fn feed(func: *Func, bt: *Air.Liveness.BigTomb, operand: Air.Inst.Ref) !void {
17351734 if (bt.feed()) if (operand.toIndex()) |inst| {
17361735 log.debug("feed inst: %{}", .{inst});
17371736 try func.processDeath(inst);
......@@ -1776,11 +1775,11 @@ fn finishAir(
17761775 func: *Func,
17771776 inst: Air.Inst.Index,
17781777 result: MCValue,
1779 operands: [Liveness.bpi - 1]Air.Inst.Ref,
1778 operands: [Air.Liveness.bpi - 1]Air.Inst.Ref,
17801779) !void {
17811780 const tomb_bits = func.liveness.getTombBits(inst);
17821781 for (0.., operands) |op_index, op| {
1783 if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
1782 if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
17841783 if (func.reused_operands.isSet(op_index)) continue;
17851784 try func.processDeath(op.toIndexAllowNone() orelse continue);
17861785 }
......@@ -3651,7 +3650,7 @@ fn airTlvDllimportPtr(func: *Func, inst: Air.Inst.Index) !void {
36513650fn airTry(func: *Func, inst: Air.Inst.Index) !void {
36523651 const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
36533652 const extra = func.air.extraData(Air.Try, pl_op.payload);
3654 const body: []const Air.Inst.Index = @ptrCast(func.air.extra[extra.end..][0..extra.data.body_len]);
3653 const body: []const Air.Inst.Index = @ptrCast(func.air.extra.items[extra.end..][0..extra.data.body_len]);
36553654 const operand_ty = func.typeOf(pl_op.operand);
36563655 const result = try func.genTry(inst, pl_op.operand, body, operand_ty, false);
36573656 return func.finishAir(inst, result, .{ .none, .none, .none });
......@@ -4419,7 +4418,7 @@ fn reuseOperand(
44194418 func: *Func,
44204419 inst: Air.Inst.Index,
44214420 operand: Air.Inst.Ref,
4422 op_index: Liveness.OperandInt,
4421 op_index: Air.Liveness.OperandInt,
44234422 mcv: MCValue,
44244423) bool {
44254424 return func.reuseOperandAdvanced(inst, operand, op_index, mcv, inst);
......@@ -4429,7 +4428,7 @@ fn reuseOperandAdvanced(
44294428 func: *Func,
44304429 inst: Air.Inst.Index,
44314430 operand: Air.Inst.Ref,
4432 op_index: Liveness.OperandInt,
4431 op_index: Air.Liveness.OperandInt,
44334432 mcv: MCValue,
44344433 maybe_tracked_inst: ?Air.Inst.Index,
44354434) bool {
......@@ -4816,7 +4815,7 @@ fn airCall(func: *Func, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
48164815 const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
48174816 const callee = pl_op.operand;
48184817 const extra = func.air.extraData(Air.Call, pl_op.payload);
4819 const arg_refs: []const Air.Inst.Ref = @ptrCast(func.air.extra[extra.end..][0..extra.data.args_len]);
4818 const arg_refs: []const Air.Inst.Ref = @ptrCast(func.air.extra.items[extra.end..][0..extra.data.args_len]);
48204819
48214820 const expected_num_args = 8;
48224821 const ExpectedContents = extern struct {
......@@ -5232,7 +5231,7 @@ fn airDbgStmt(func: *Func, inst: Air.Inst.Index) !void {
52325231fn airDbgInlineBlock(func: *Func, inst: Air.Inst.Index) !void {
52335232 const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
52345233 const extra = func.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
5235 try func.lowerBlock(inst, @ptrCast(func.air.extra[extra.end..][0..extra.data.body_len]));
5234 try func.lowerBlock(inst, @ptrCast(func.air.extra.items[extra.end..][0..extra.data.body_len]));
52365235}
52375236
52385237fn airDbgVar(func: *Func, inst: Air.Inst.Index) InnerError!void {
......@@ -5284,8 +5283,8 @@ fn airCondBr(func: *Func, inst: Air.Inst.Index) !void {
52845283 const cond = try func.resolveInst(pl_op.operand);
52855284 const cond_ty = func.typeOf(pl_op.operand);
52865285 const extra = func.air.extraData(Air.CondBr, pl_op.payload);
5287 const then_body: []const Air.Inst.Index = @ptrCast(func.air.extra[extra.end..][0..extra.data.then_body_len]);
5288 const else_body: []const Air.Inst.Index = @ptrCast(func.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]);
5286 const then_body: []const Air.Inst.Index = @ptrCast(func.air.extra.items[extra.end..][0..extra.data.then_body_len]);
5287 const else_body: []const Air.Inst.Index = @ptrCast(func.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]);
52895288 const liveness_cond_br = func.liveness.getCondBr(inst);
52905289
52915290 // If the condition dies here in this condbr instruction, process
......@@ -5644,7 +5643,7 @@ fn airLoop(func: *Func, inst: Air.Inst.Index) !void {
56445643 // A loop is a setup to be able to jump back to the beginning.
56455644 const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
56465645 const loop = func.air.extraData(Air.Block, ty_pl.payload);
5647 const body: []const Air.Inst.Index = @ptrCast(func.air.extra[loop.end..][0..loop.data.body_len]);
5646 const body: []const Air.Inst.Index = @ptrCast(func.air.extra.items[loop.end..][0..loop.data.body_len]);
56485647
56495648 func.scope_generation += 1;
56505649 const state = try func.saveState();
......@@ -5674,7 +5673,7 @@ fn jump(func: *Func, index: Mir.Inst.Index) !Mir.Inst.Index {
56745673fn airBlock(func: *Func, inst: Air.Inst.Index) !void {
56755674 const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
56765675 const extra = func.air.extraData(Air.Block, ty_pl.payload);
5677 try func.lowerBlock(inst, @ptrCast(func.air.extra[extra.end..][0..extra.data.body_len]));
5676 try func.lowerBlock(inst, @ptrCast(func.air.extra.items[extra.end..][0..extra.data.body_len]));
56785677}
56795678
56805679fn lowerBlock(func: *Func, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void {
......@@ -6063,9 +6062,9 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {
60636062 const clobbers_len: u31 = @truncate(extra.data.flags);
60646063 var extra_i: usize = extra.end;
60656064 const outputs: []const Air.Inst.Ref =
6066 @ptrCast(func.air.extra[extra_i..][0..extra.data.outputs_len]);
6065 @ptrCast(func.air.extra.items[extra_i..][0..extra.data.outputs_len]);
60676066 extra_i += outputs.len;
6068 const inputs: []const Air.Inst.Ref = @ptrCast(func.air.extra[extra_i..][0..extra.data.inputs_len]);
6067 const inputs: []const Air.Inst.Ref = @ptrCast(func.air.extra.items[extra_i..][0..extra.data.inputs_len]);
60696068 extra_i += inputs.len;
60706069
60716070 var result: MCValue = .none;
......@@ -6083,8 +6082,8 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {
60836082
60846083 var outputs_extra_i = extra_i;
60856084 for (outputs) |output| {
6086 const extra_bytes = mem.sliceAsBytes(func.air.extra[extra_i..]);
6087 const constraint = mem.sliceTo(mem.sliceAsBytes(func.air.extra[extra_i..]), 0);
6085 const extra_bytes = mem.sliceAsBytes(func.air.extra.items[extra_i..]);
6086 const constraint = mem.sliceTo(mem.sliceAsBytes(func.air.extra.items[extra_i..]), 0);
60886087 const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
60896088 // This equation accounts for the fact that even if we have exactly 4 bytes
60906089 // for the string, we still use the next u32 for the null terminator.
......@@ -6141,7 +6140,7 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {
61416140 }
61426141
61436142 for (inputs) |input| {
6144 const input_bytes = mem.sliceAsBytes(func.air.extra[extra_i..]);
6143 const input_bytes = mem.sliceAsBytes(func.air.extra.items[extra_i..]);
61456144 const constraint = mem.sliceTo(input_bytes, 0);
61466145 const name = mem.sliceTo(input_bytes[constraint.len + 1 ..], 0);
61476146 // This equation accounts for the fact that even if we have exactly 4 bytes
......@@ -6177,7 +6176,7 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {
61776176 {
61786177 var clobber_i: u32 = 0;
61796178 while (clobber_i < clobbers_len) : (clobber_i += 1) {
6180 const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(func.air.extra[extra_i..]), 0);
6179 const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(func.air.extra.items[extra_i..]), 0);
61816180 // This equation accounts for the fact that even if we have exactly 4 bytes
61826181 // for the string, we still use the next u32 for the null terminator.
61836182 extra_i += clobber.len / 4 + 1;
......@@ -6224,7 +6223,7 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {
62246223 labels.deinit(func.gpa);
62256224 }
62266225
6227 const asm_source = std.mem.sliceAsBytes(func.air.extra[extra_i..])[0..extra.data.source_len];
6226 const asm_source = std.mem.sliceAsBytes(func.air.extra.items[extra_i..])[0..extra.data.source_len];
62286227 var line_it = mem.tokenizeAny(u8, asm_source, "\n\r;");
62296228 next_line: while (line_it.next()) |line| {
62306229 var mnem_it = mem.tokenizeAny(u8, line, " \t");
......@@ -6493,9 +6492,9 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {
64936492 return func.fail("undefined label: '{s}'", .{label.key_ptr.*});
64946493
64956494 for (outputs, args.items[0..outputs.len]) |output, arg_mcv| {
6496 const extra_bytes = mem.sliceAsBytes(func.air.extra[outputs_extra_i..]);
6495 const extra_bytes = mem.sliceAsBytes(func.air.extra.items[outputs_extra_i..]);
64976496 const constraint =
6498 mem.sliceTo(mem.sliceAsBytes(func.air.extra[outputs_extra_i..]), 0);
6497 mem.sliceTo(mem.sliceAsBytes(func.air.extra.items[outputs_extra_i..]), 0);
64996498 const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
65006499 // This equation accounts for the fact that even if we have exactly 4 bytes
65016500 // for the string, we still use the next u32 for the null terminator.
......@@ -6508,7 +6507,7 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void {
65086507 }
65096508
65106509 simple: {
6511 var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
6510 var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1);
65126511 var buf_index: usize = 0;
65136512 for (outputs) |output| {
65146513 if (output == .none) continue;
......@@ -8027,7 +8026,7 @@ fn airAggregateInit(func: *Func, inst: Air.Inst.Index) !void {
80278026 const result_ty = func.typeOfIndex(inst);
80288027 const len: usize = @intCast(result_ty.arrayLen(zcu));
80298028 const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
8030 const elements: []const Air.Inst.Ref = @ptrCast(func.air.extra[ty_pl.payload..][0..len]);
8029 const elements: []const Air.Inst.Ref = @ptrCast(func.air.extra.items[ty_pl.payload..][0..len]);
80318030
80328031 const result: MCValue = result: {
80338032 switch (result_ty.zigTypeTag(zcu)) {
......@@ -8113,8 +8112,8 @@ fn airAggregateInit(func: *Func, inst: Air.Inst.Index) !void {
81138112 }
81148113 };
81158114
8116 if (elements.len <= Liveness.bpi - 1) {
8117 var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
8115 if (elements.len <= Air.Liveness.bpi - 1) {
8116 var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1);
81188117 @memcpy(buf[0..elements.len], elements);
81198118 return func.finishAir(inst, result, buf);
81208119 }
src/arch/sparc64/CodeGen.zig+37-38
......@@ -18,7 +18,6 @@ const codegen = @import("../../codegen.zig");
1818const Air = @import("../../Air.zig");
1919const Mir = @import("Mir.zig");
2020const Emit = @import("Emit.zig");
21const Liveness = @import("../../Liveness.zig");
2221const Type = @import("../../Type.zig");
2322const CodeGenError = codegen.CodeGenError;
2423const Endian = std.builtin.Endian;
......@@ -50,7 +49,7 @@ const RegisterView = enum(u1) {
5049gpa: Allocator,
5150pt: Zcu.PerThread,
5251air: Air,
53liveness: Liveness,
52liveness: Air.Liveness,
5453bin_file: *link.File,
5554target: *const std.Target,
5655func_index: InternPool.Index,
......@@ -78,7 +77,7 @@ end_di_column: u32,
7877/// which is a relative jump, based on the address following the reloc.
7978exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty,
8079
81reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined,
80reused_operands: std.StaticBitSet(Air.Liveness.bpi - 1) = undefined,
8281
8382/// Whenever there is a runtime branch, we push a Branch onto this stack,
8483/// and pop it off when the runtime branch joins. This provides an "overlay"
......@@ -240,7 +239,7 @@ const CallMCValues = struct {
240239const BigTomb = struct {
241240 function: *Self,
242241 inst: Air.Inst.Index,
243 lbt: Liveness.BigTomb,
242 lbt: Air.Liveness.BigTomb,
244243
245244 fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void {
246245 const dies = bt.lbt.feed();
......@@ -266,7 +265,7 @@ pub fn generate(
266265 src_loc: Zcu.LazySrcLoc,
267266 func_index: InternPool.Index,
268267 air: Air,
269 liveness: Liveness,
268 liveness: Air.Liveness,
270269 code: *std.ArrayListUnmanaged(u8),
271270 debug_output: link.File.DebugInfoOutput,
272271) CodeGenError!void {
......@@ -493,7 +492,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
493492 continue;
494493
495494 const old_air_bookkeeping = self.air_bookkeeping;
496 try self.ensureProcessDeathCapacity(Liveness.bpi);
495 try self.ensureProcessDeathCapacity(Air.Liveness.bpi);
497496
498497 self.reused_operands = @TypeOf(self.reused_operands).initEmpty();
499498 switch (air_tags[@intFromEnum(inst)]) {
......@@ -839,14 +838,14 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
839838 const vector_ty = self.typeOfIndex(inst);
840839 const len = vector_ty.vectorLen(zcu);
841840 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
842 const elements = @as([]const Air.Inst.Ref, @ptrCast(self.air.extra[ty_pl.payload..][0..len]));
841 const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]);
843842 const result: MCValue = res: {
844843 if (self.liveness.isUnused(inst)) break :res MCValue.dead;
845844 return self.fail("TODO implement airAggregateInit for {}", .{self.target.cpu.arch});
846845 };
847846
848 if (elements.len <= Liveness.bpi - 1) {
849 var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
847 if (elements.len <= Air.Liveness.bpi - 1) {
848 var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1);
850849 @memcpy(buf[0..elements.len], elements);
851850 return self.finishAir(inst, result, buf);
852851 }
......@@ -876,7 +875,7 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
876875 const ptr_ty = self.typeOf(ty_op.operand);
877876 const ptr = try self.resolveInst(ty_op.operand);
878877 const array_ty = ptr_ty.childType(zcu);
879 const array_len = @as(u32, @intCast(array_ty.arrayLen(zcu)));
878 const array_len: u32 = @intCast(array_ty.arrayLen(zcu));
880879 const ptr_bytes = 8;
881880 const stack_offset = try self.allocMem(inst, ptr_bytes * 2, .@"8");
882881 try self.genSetStack(ptr_ty, stack_offset, ptr);
......@@ -890,11 +889,11 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
890889 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
891890 const extra = self.air.extraData(Air.Asm, ty_pl.payload);
892891 const is_volatile = (extra.data.flags & 0x80000000) != 0;
893 const clobbers_len = @as(u31, @truncate(extra.data.flags));
892 const clobbers_len: u31 = @truncate(extra.data.flags);
894893 var extra_i: usize = extra.end;
895 const outputs = @as([]const Air.Inst.Ref, @ptrCast(self.air.extra[extra_i .. extra_i + extra.data.outputs_len]));
894 const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i .. extra_i + extra.data.outputs_len]);
896895 extra_i += outputs.len;
897 const inputs = @as([]const Air.Inst.Ref, @ptrCast(self.air.extra[extra_i .. extra_i + extra.data.inputs_len]));
896 const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i .. extra_i + extra.data.inputs_len]);
898897 extra_i += inputs.len;
899898
900899 const dead = !is_volatile and self.liveness.isUnused(inst);
......@@ -907,8 +906,8 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
907906 if (output != .none) {
908907 return self.fail("TODO implement codegen for non-expr asm", .{});
909908 }
910 const extra_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]);
911 const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0);
909 const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]);
910 const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0);
912911 const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
913912 // This equation accounts for the fact that even if we have exactly 4 bytes
914913 // for the string, we still use the next u32 for the null terminator.
......@@ -918,7 +917,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
918917 } else null;
919918
920919 for (inputs) |input| {
921 const input_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]);
920 const input_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]);
922921 const constraint = std.mem.sliceTo(input_bytes, 0);
923922 const name = std.mem.sliceTo(input_bytes[constraint.len + 1 ..], 0);
924923 // This equation accounts for the fact that even if we have exactly 4 bytes
......@@ -940,7 +939,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
940939 {
941940 var clobber_i: u32 = 0;
942941 while (clobber_i < clobbers_len) : (clobber_i += 1) {
943 const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0);
942 const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0);
944943 // This equation accounts for the fact that even if we have exactly 4 bytes
945944 // for the string, we still use the next u32 for the null terminator.
946945 extra_i += clobber.len / 4 + 1;
......@@ -949,7 +948,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
949948 }
950949 }
951950
952 const asm_source = std.mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len];
951 const asm_source = std.mem.sliceAsBytes(self.air.extra.items[extra_i..])[0..extra.data.source_len];
953952
954953 if (mem.eql(u8, asm_source, "ta 0x6d")) {
955954 _ = try self.addInst(.{
......@@ -980,7 +979,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
980979 };
981980
982981 simple: {
983 var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
982 var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1);
984983 var buf_index: usize = 0;
985984 for (outputs) |output| {
986985 if (output == .none) continue;
......@@ -1124,7 +1123,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
11241123fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
11251124 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
11261125 const extra = self.air.extraData(Air.Block, ty_pl.payload);
1127 try self.lowerBlock(inst, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]));
1126 try self.lowerBlock(inst, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]));
11281127}
11291128
11301129fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void {
......@@ -1292,7 +1291,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
12921291 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
12931292 const callee = pl_op.operand;
12941293 const extra = self.air.extraData(Air.Call, pl_op.payload);
1295 const args = @as([]const Air.Inst.Ref, @ptrCast(self.air.extra[extra.end .. extra.end + extra.data.args_len]));
1294 const args: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra.end .. extra.end + extra.data.args_len]);
12961295 const ty = self.typeOf(callee);
12971296 const pt = self.pt;
12981297 const zcu = pt.zcu;
......@@ -1376,8 +1375,8 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
13761375
13771376 const result = info.return_value;
13781377
1379 if (args.len + 1 <= Liveness.bpi - 1) {
1380 var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
1378 if (args.len + 1 <= Air.Liveness.bpi - 1) {
1379 var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1);
13811380 buf[0] = callee;
13821381 @memcpy(buf[1..][0..args.len], args);
13831382 return self.finishAir(inst, result, buf);
......@@ -1477,8 +1476,8 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
14771476 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
14781477 const condition = try self.resolveInst(pl_op.operand);
14791478 const extra = self.air.extraData(Air.CondBr, pl_op.payload);
1480 const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.then_body_len]);
1481 const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]);
1479 const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.then_body_len]);
1480 const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]);
14821481 const liveness_condbr = self.liveness.getCondBr(inst);
14831482
14841483 // Here we emit a branch to the false section.
......@@ -1629,7 +1628,7 @@ fn airDbgInlineBlock(self: *Self, inst: Air.Inst.Index) !void {
16291628 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
16301629 const extra = self.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
16311630 // TODO emit debug info for function change
1632 try self.lowerBlock(inst, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]));
1631 try self.lowerBlock(inst, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]));
16331632}
16341633
16351634fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
......@@ -1795,8 +1794,8 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
17951794 // A loop is a setup to be able to jump back to the beginning.
17961795 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
17971796 const loop = self.air.extraData(Air.Block, ty_pl.payload);
1798 const body: []const Air.Inst.Index = @ptrCast(self.air.extra[loop.end .. loop.end + loop.data.body_len]);
1799 const start = @as(u32, @intCast(self.mir_instructions.len));
1797 const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[loop.end .. loop.end + loop.data.body_len]);
1798 const start: u32 = @intCast(self.mir_instructions.len);
18001799
18011800 try self.genBody(body);
18021801 try self.jump(start);
......@@ -2514,7 +2513,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
25142513 const zcu = self.pt.zcu;
25152514 const mcv = try self.resolveInst(operand);
25162515 const struct_ty = self.typeOf(operand);
2517 const struct_field_offset = @as(u32, @intCast(struct_ty.structFieldOffset(index, zcu)));
2516 const struct_field_offset: u32 = @intCast(struct_ty.structFieldOffset(index, zcu));
25182517
25192518 switch (mcv) {
25202519 .dead, .unreach => unreachable,
......@@ -2612,7 +2611,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
26122611fn airTry(self: *Self, inst: Air.Inst.Index) !void {
26132612 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
26142613 const extra = self.air.extraData(Air.Try, pl_op.payload);
2615 const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
2614 const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
26162615 const result: MCValue = result: {
26172616 const error_union_ty = self.typeOf(pl_op.operand);
26182617 const error_union = try self.resolveInst(pl_op.operand);
......@@ -3478,7 +3477,7 @@ fn errUnionPayload(self: *Self, error_union_mcv: MCValue, error_union_ty: Type)
34783477 return MCValue.none;
34793478 }
34803479
3481 const payload_offset = @as(u32, @intCast(errUnionPayloadOffset(payload_ty, zcu)));
3480 const payload_offset: u32 = @intCast(errUnionPayloadOffset(payload_ty, zcu));
34823481 switch (error_union_mcv) {
34833482 .register => return self.fail("TODO errUnionPayload for registers", .{}),
34843483 .stack_offset => |off| {
......@@ -3513,14 +3512,14 @@ fn finishAirBookkeeping(self: *Self) void {
35133512 }
35143513}
35153514
3516fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref) void {
3515fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Air.Liveness.bpi - 1]Air.Inst.Ref) void {
35173516 const tomb_bits = self.liveness.getTombBits(inst);
35183517 for (0.., operands) |op_index, op| {
3519 if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
3518 if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
35203519 if (self.reused_operands.isSet(op_index)) continue;
35213520 self.processDeath(op.toIndexAllowNone() orelse continue);
35223521 }
3523 if (tomb_bits & 1 << (Liveness.bpi - 1) == 0) {
3522 if (tomb_bits & 1 << (Air.Liveness.bpi - 1) == 0) {
35243523 log.debug("%{d} => {}", .{ inst, result });
35253524 const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
35263525 branch.inst_table.putAssumeCapacityNoClobber(inst, result);
......@@ -3944,7 +3943,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
39443943 try self.genSetStack(wrapped_ty, stack_offset, .{ .register = rwo.reg });
39453944
39463945 const overflow_bit_ty = ty.fieldType(1, zcu);
3947 const overflow_bit_offset = @as(u32, @intCast(ty.structFieldOffset(1, zcu)));
3946 const overflow_bit_offset: u32 = @intCast(ty.structFieldOffset(1, zcu));
39483947 const cond_reg = try self.register_manager.allocReg(null, gp);
39493948
39503949 // TODO handle floating point CCRs
......@@ -4449,7 +4448,7 @@ fn resolveCallingConventionValues(self: *Self, fn_ty: Type, role: RegisterView)
44494448 };
44504449
44514450 for (fn_info.param_types.get(ip), result.args) |ty, *result_arg| {
4452 const param_size = @as(u32, @intCast(Type.fromInterned(ty).abiSize(zcu)));
4451 const param_size: u32 = @intCast(Type.fromInterned(ty).abiSize(zcu));
44534452 if (param_size <= 8) {
44544453 if (next_register < argument_registers.len) {
44554454 result_arg.* = .{ .register = argument_registers[next_register] };
......@@ -4534,7 +4533,7 @@ fn ret(self: *Self, mcv: MCValue) !void {
45344533 try self.exitlude_jump_relocs.append(self.gpa, index);
45354534}
45364535
4537fn reuseOperand(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, op_index: Liveness.OperandInt, mcv: MCValue) bool {
4536fn reuseOperand(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, op_index: Air.Liveness.OperandInt, mcv: MCValue) bool {
45384537 if (!self.liveness.operandDies(inst, op_index))
45394538 return false;
45404539
......@@ -4664,7 +4663,7 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde
46644663 const mcv = try self.resolveInst(operand);
46654664 const ptr_ty = self.typeOf(operand);
46664665 const struct_ty = ptr_ty.childType(zcu);
4667 const struct_field_offset = @as(u32, @intCast(struct_ty.structFieldOffset(index, zcu)));
4666 const struct_field_offset: u32 = @intCast(struct_ty.structFieldOffset(index, zcu));
46684667 switch (mcv) {
46694668 .ptr_stack_offset => |off| {
46704669 break :result MCValue{ .ptr_stack_offset = off - struct_field_offset };
src/arch/sparc64/Emit.zig-1
......@@ -7,7 +7,6 @@ const assert = std.debug.assert;
77const link = @import("../../link.zig");
88const Zcu = @import("../../Zcu.zig");
99const ErrorMsg = Zcu.ErrorMsg;
10const Liveness = @import("../../Liveness.zig");
1110const log = std.log.scoped(.sparcv9_emit);
1211
1312const Emit = @This();
src/arch/wasm/CodeGen.zig+16-17
......@@ -17,7 +17,6 @@ const Value = @import("../../Value.zig");
1717const Compilation = @import("../../Compilation.zig");
1818const link = @import("../../link.zig");
1919const Air = @import("../../Air.zig");
20const Liveness = @import("../../Liveness.zig");
2120const Mir = @import("Mir.zig");
2221const Emit = @import("Emit.zig");
2322const abi = @import("abi.zig");
......@@ -39,7 +38,7 @@ owner_nav: InternPool.Nav.Index,
3938/// and block
4039block_depth: u32 = 0,
4140air: Air,
42liveness: Liveness,
41liveness: Air.Liveness,
4342gpa: mem.Allocator,
4443func_index: InternPool.Index,
4544/// Contains a list of current branches.
......@@ -771,7 +770,7 @@ fn resolveValue(cg: *CodeGen, val: Value) InnerError!WValue {
771770
772771/// NOTE: if result == .stack, it will be stored in .local
773772fn finishAir(cg: *CodeGen, inst: Air.Inst.Index, result: WValue, operands: []const Air.Inst.Ref) InnerError!void {
774 assert(operands.len <= Liveness.bpi - 1);
773 assert(operands.len <= Air.Liveness.bpi - 1);
775774 var tomb_bits = cg.liveness.getTombBits(inst);
776775 for (operands) |operand| {
777776 const dies = @as(u1, @truncate(tomb_bits)) != 0;
......@@ -811,7 +810,7 @@ inline fn currentBranch(cg: *CodeGen) *Branch {
811810const BigTomb = struct {
812811 gen: *CodeGen,
813812 inst: Air.Inst.Index,
814 lbt: Liveness.BigTomb,
813 lbt: Air.Liveness.BigTomb,
815814
816815 fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void {
817816 const dies = bt.lbt.feed();
......@@ -1262,7 +1261,7 @@ pub fn function(
12621261 pt: Zcu.PerThread,
12631262 func_index: InternPool.Index,
12641263 air: Air,
1265 liveness: Liveness,
1264 liveness: Air.Liveness,
12661265) Error!Function {
12671266 const zcu = pt.zcu;
12681267 const gpa = zcu.gpa;
......@@ -2123,7 +2122,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
21232122 continue;
21242123 }
21252124 const old_bookkeeping_value = cg.air_bookkeeping;
2126 try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, Liveness.bpi);
2125 try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, Air.Liveness.bpi);
21272126 try cg.genInst(inst);
21282127
21292128 if (std.debug.runtime_safety and cg.air_bookkeeping < old_bookkeeping_value + 1) {
......@@ -2217,7 +2216,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifie
22172216 if (modifier == .always_tail) return cg.fail("TODO implement tail calls for wasm", .{});
22182217 const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
22192218 const extra = cg.air.extraData(Air.Call, pl_op.payload);
2220 const args: []const Air.Inst.Ref = @ptrCast(cg.air.extra[extra.end..][0..extra.data.args_len]);
2219 const args: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.args_len]);
22212220 const ty = cg.typeOf(pl_op.operand);
22222221
22232222 const pt = cg.pt;
......@@ -3410,7 +3409,7 @@ fn emitUndefined(cg: *CodeGen, ty: Type) InnerError!WValue {
34103409fn airBlock(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
34113410 const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
34123411 const extra = cg.air.extraData(Air.Block, ty_pl.payload);
3413 try cg.lowerBlock(inst, ty_pl.ty.toType(), @ptrCast(cg.air.extra[extra.end..][0..extra.data.body_len]));
3412 try cg.lowerBlock(inst, ty_pl.ty.toType(), @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len]));
34143413}
34153414
34163415fn lowerBlock(cg: *CodeGen, inst: Air.Inst.Index, block_ty: Type, body: []const Air.Inst.Index) InnerError!void {
......@@ -3456,7 +3455,7 @@ fn endBlock(cg: *CodeGen) !void {
34563455fn airLoop(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
34573456 const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
34583457 const loop = cg.air.extraData(Air.Block, ty_pl.payload);
3459 const body: []const Air.Inst.Index = @ptrCast(cg.air.extra[loop.end..][0..loop.data.body_len]);
3458 const body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[loop.end..][0..loop.data.body_len]);
34603459
34613460 // result type of loop is always 'noreturn', meaning we can always
34623461 // emit the wasm type 'block_empty'.
......@@ -3475,8 +3474,8 @@ fn airCondBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
34753474 const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
34763475 const condition = try cg.resolveInst(pl_op.operand);
34773476 const extra = cg.air.extraData(Air.CondBr, pl_op.payload);
3478 const then_body: []const Air.Inst.Index = @ptrCast(cg.air.extra[extra.end..][0..extra.data.then_body_len]);
3479 const else_body: []const Air.Inst.Index = @ptrCast(cg.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]);
3477 const then_body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.then_body_len]);
3478 const else_body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]);
34803479 const liveness_condbr = cg.liveness.getCondBr(inst);
34813480
34823481 // result type is always noreturn, so use `block_empty` as type.
......@@ -5238,7 +5237,7 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
52385237 const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
52395238 const result_ty = cg.typeOfIndex(inst);
52405239 const len = @as(usize, @intCast(result_ty.arrayLen(zcu)));
5241 const elements = @as([]const Air.Inst.Ref, @ptrCast(cg.air.extra[ty_pl.payload..][0..len]));
5240 const elements: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[ty_pl.payload..][0..len]);
52425241
52435242 const result: WValue = result_value: {
52445243 switch (result_ty.zigTypeTag(zcu)) {
......@@ -5352,8 +5351,8 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
53525351 }
53535352 };
53545353
5355 if (elements.len <= Liveness.bpi - 1) {
5356 var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
5354 if (elements.len <= Air.Liveness.bpi - 1) {
5355 var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1);
53575356 @memcpy(buf[0..elements.len], elements);
53585357 return cg.finishAir(inst, result, &buf);
53595358 }
......@@ -6454,7 +6453,7 @@ fn airDbgInlineBlock(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
64546453 const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
64556454 const extra = cg.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
64566455 // TODO
6457 try cg.lowerBlock(inst, ty_pl.ty.toType(), @ptrCast(cg.air.extra[extra.end..][0..extra.data.body_len]));
6456 try cg.lowerBlock(inst, ty_pl.ty.toType(), @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len]));
64586457}
64596458
64606459fn airDbgVar(
......@@ -6472,7 +6471,7 @@ fn airTry(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
64726471 const pl_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
64736472 const err_union = try cg.resolveInst(pl_op.operand);
64746473 const extra = cg.air.extraData(Air.Try, pl_op.payload);
6475 const body: []const Air.Inst.Index = @ptrCast(cg.air.extra[extra.end..][0..extra.data.body_len]);
6474 const body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len]);
64766475 const err_union_ty = cg.typeOf(pl_op.operand);
64776476 const result = try lowerTry(cg, inst, err_union, body, err_union_ty, false);
64786477 return cg.finishAir(inst, result, &.{pl_op.operand});
......@@ -6483,7 +6482,7 @@ fn airTryPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
64836482 const ty_pl = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
64846483 const extra = cg.air.extraData(Air.TryPtr, ty_pl.payload);
64856484 const err_union_ptr = try cg.resolveInst(extra.data.ptr);
6486 const body: []const Air.Inst.Index = @ptrCast(cg.air.extra[extra.end..][0..extra.data.body_len]);
6485 const body: []const Air.Inst.Index = @ptrCast(cg.air.extra.items[extra.end..][0..extra.data.body_len]);
64876486 const err_union_ty = cg.typeOf(extra.data.ptr).childType(zcu);
64886487 const result = try lowerTry(cg, inst, err_union_ptr, body, err_union_ty, true);
64896488 return cg.finishAir(inst, result, &.{extra.data.ptr});
src/arch/x86_64/CodeGen.zig+40-36
......@@ -10,7 +10,6 @@ const wip_mir_log = std.log.scoped(.wip_mir);
1010const Air = @import("../../Air.zig");
1111const Allocator = std.mem.Allocator;
1212const Emit = @import("Emit.zig");
13const Liveness = @import("../../Liveness.zig");
1413const Lower = @import("Lower.zig");
1514const Mir = @import("Mir.zig");
1615const Zcu = @import("../../Zcu.zig");
......@@ -33,6 +32,11 @@ const FrameIndex = bits.FrameIndex;
3332
3433const InnerError = codegen.CodeGenError || error{OutOfRegisters};
3534
35pub const legalize_features: Air.Legalize.Features = .{
36 .remove_shift_vector_rhs_splat = false,
37 .reduce_one_elem_to_bitcast = true,
38};
39
3640/// Set this to `false` to uncover Sema OPV bugs.
3741/// https://github.com/ziglang/zig/issues/22419
3842const hack_around_sema_opv_bugs = true;
......@@ -42,7 +46,7 @@ const err_ret_trace_index: Air.Inst.Index = @enumFromInt(std.math.maxInt(u32));
4246gpa: Allocator,
4347pt: Zcu.PerThread,
4448air: Air,
45liveness: Liveness,
49liveness: Air.Liveness,
4650bin_file: *link.File,
4751debug_output: link.File.DebugInfoOutput,
4852target: *const std.Target,
......@@ -78,7 +82,7 @@ mir_table: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty,
7882/// which is a relative jump, based on the address following the reloc.
7983epilogue_relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty,
8084
81reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined,
85reused_operands: std.StaticBitSet(Air.Liveness.bpi - 1) = undefined,
8286inst_tracking: InstTrackingMap = .empty,
8387
8488// Key is the block instruction
......@@ -859,7 +863,7 @@ pub fn generate(
859863 src_loc: Zcu.LazySrcLoc,
860864 func_index: InternPool.Index,
861865 air: Air,
862 liveness: Liveness,
866 liveness: Air.Liveness,
863867 code: *std.ArrayListUnmanaged(u8),
864868 debug_output: link.File.DebugInfoOutput,
865869) codegen.CodeGenError!void {
......@@ -63335,7 +63339,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6333563339 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
6333663340 const block = cg.air.extraData(Air.Block, ty_pl.payload);
6333763341 if (cg.debug_output != .none) try cg.asmPseudo(.pseudo_dbg_enter_block_none);
63338 try cg.lowerBlock(inst, @ptrCast(cg.air.extra[block.end..][0..block.data.body_len]));
63342 try cg.lowerBlock(inst, @ptrCast(cg.air.extra.items[block.end..][0..block.data.body_len]));
6333963343 if (cg.debug_output != .none) try cg.asmPseudo(.pseudo_dbg_leave_block_none);
6334063344 },
6334163345 .loop => if (use_old) try cg.airLoop(inst) else {
......@@ -63346,7 +63350,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
6334663350 .target = @intCast(cg.mir_instructions.len),
6334763351 });
6334863352 defer assert(cg.loops.remove(inst));
63349 try cg.genBodyBlock(@ptrCast(cg.air.extra[block.end..][0..block.data.body_len]));
63353 try cg.genBodyBlock(@ptrCast(cg.air.extra.items[block.end..][0..block.data.body_len]));
6335063354 },
6335163355 .repeat => if (use_old) try cg.airRepeat(inst) else {
6335263356 const repeat = air_datas[@intFromEnum(inst)].repeat;
......@@ -84360,7 +84364,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8436084364 .ops = .pseudo_dbg_enter_inline_func,
8436184365 .data = .{ .func = dbg_inline_block.data.func },
8436284366 });
84363 try cg.lowerBlock(inst, @ptrCast(cg.air.extra[dbg_inline_block.end..][0..dbg_inline_block.data.body_len]));
84367 try cg.lowerBlock(inst, @ptrCast(cg.air.extra.items[dbg_inline_block.end..][0..dbg_inline_block.data.body_len]));
8436484368 if (cg.debug_output != .none) _ = try cg.addInst(.{
8436584369 .tag = .pseudo,
8436684370 .ops = .pseudo_dbg_leave_inline_func,
......@@ -160620,7 +160624,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
160620160624 var bt = cg.liveness.iterateBigTomb(inst);
160621160625 switch (ip.indexToKey(agg_ty.toIntern())) {
160622160626 inline .array_type, .vector_type => |sequence_type| {
160623 const elems: []const Air.Inst.Ref = @ptrCast(cg.air.extra[ty_pl.payload..][0..@intCast(sequence_type.len)]);
160627 const elems: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[ty_pl.payload..][0..@intCast(sequence_type.len)]);
160624160628 const elem_size = Type.fromInterned(sequence_type.child).abiSize(zcu);
160625160629 var elem_disp: u31 = 0;
160626160630 for (elems) |elem_ref| {
......@@ -160638,7 +160642,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
160638160642 },
160639160643 .struct_type => {
160640160644 const loaded_struct = ip.loadStructType(agg_ty.toIntern());
160641 const elems: []const Air.Inst.Ref = @ptrCast(cg.air.extra[ty_pl.payload..][0..loaded_struct.field_types.len]);
160645 const elems: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[ty_pl.payload..][0..loaded_struct.field_types.len]);
160642160646 switch (loaded_struct.layout) {
160643160647 .auto, .@"extern" => {
160644160648 for (elems, 0..) |elem_ref, field_index| {
......@@ -160657,7 +160661,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
160657160661 }
160658160662 },
160659160663 .tuple_type => |tuple_type| {
160660 const elems: []const Air.Inst.Ref = @ptrCast(cg.air.extra[ty_pl.payload..][0..tuple_type.types.len]);
160664 const elems: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[ty_pl.payload..][0..tuple_type.types.len]);
160661160665 var elem_disp: u31 = 0;
160662160666 for (elems, 0..) |elem_ref, field_index| {
160663160667 const elem_dies = bt.feed();
......@@ -162630,7 +162634,7 @@ fn freeValue(self: *CodeGen, value: MCValue) !void {
162630162634 }
162631162635}
162632162636
162633fn feed(self: *CodeGen, bt: *Liveness.BigTomb, operand: Air.Inst.Ref) !void {
162637fn feed(self: *CodeGen, bt: *Air.Liveness.BigTomb, operand: Air.Inst.Ref) !void {
162634162638 if (bt.feed()) if (operand.toIndex()) |inst| try self.processDeath(inst);
162635162639}
162636162640
......@@ -162657,11 +162661,11 @@ fn finishAir(
162657162661 self: *CodeGen,
162658162662 inst: Air.Inst.Index,
162659162663 result: MCValue,
162660 operands: [Liveness.bpi - 1]Air.Inst.Ref,
162664 operands: [Air.Liveness.bpi - 1]Air.Inst.Ref,
162661162665) !void {
162662162666 const tomb_bits = self.liveness.getTombBits(inst);
162663162667 for (0.., operands) |op_index, op| {
162664 if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
162668 if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
162665162669 if (self.reused_operands.isSet(op_index)) continue;
162666162670 try self.processDeath(op.toIndexAllowNone() orelse continue);
162667162671 }
......@@ -167965,7 +167969,7 @@ fn reuseOperand(
167965167969 self: *CodeGen,
167966167970 inst: Air.Inst.Index,
167967167971 operand: Air.Inst.Ref,
167968 op_index: Liveness.OperandInt,
167972 op_index: Air.Liveness.OperandInt,
167969167973 mcv: MCValue,
167970167974) bool {
167971167975 return self.reuseOperandAdvanced(inst, operand, op_index, mcv, inst);
......@@ -167975,7 +167979,7 @@ fn reuseOperandAdvanced(
167975167979 self: *CodeGen,
167976167980 inst: Air.Inst.Index,
167977167981 operand: Air.Inst.Ref,
167978 op_index: Liveness.OperandInt,
167982 op_index: Air.Liveness.OperandInt,
167979167983 mcv: MCValue,
167980167984 maybe_tracked_inst: ?Air.Inst.Index,
167981167985) bool {
......@@ -172435,7 +172439,7 @@ fn airCall(self: *CodeGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif
172435172439 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
172436172440 const extra = self.air.extraData(Air.Call, pl_op.payload);
172437172441 const arg_refs: []const Air.Inst.Ref =
172438 @ptrCast(self.air.extra[extra.end..][0..extra.data.args_len]);
172442 @ptrCast(self.air.extra.items[extra.end..][0..extra.data.args_len]);
172439172443
172440172444 const ExpectedContents = extern struct {
172441172445 tys: [16][@sizeOf(Type)]u8 align(@alignOf(Type)),
......@@ -173349,7 +173353,7 @@ fn airCmpLtErrorsLen(self: *CodeGen, inst: Air.Inst.Index) !void {
173349173353fn airTry(self: *CodeGen, inst: Air.Inst.Index) !void {
173350173354 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
173351173355 const extra = self.air.extraData(Air.Try, pl_op.payload);
173352 const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
173356 const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
173353173357 const operand_ty = self.typeOf(pl_op.operand);
173354173358 const result = try self.genTry(inst, pl_op.operand, body, operand_ty, false);
173355173359 return self.finishAir(inst, result, .{ .none, .none, .none });
......@@ -173358,7 +173362,7 @@ fn airTry(self: *CodeGen, inst: Air.Inst.Index) !void {
173358173362fn airTryPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
173359173363 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
173360173364 const extra = self.air.extraData(Air.TryPtr, ty_pl.payload);
173361 const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
173365 const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
173362173366 const operand_ty = self.typeOf(extra.data.ptr);
173363173367 const result = try self.genTry(inst, extra.data.ptr, body, operand_ty, true);
173364173368 return self.finishAir(inst, result, .{ .none, .none, .none });
......@@ -173449,9 +173453,9 @@ fn airCondBr(self: *CodeGen, inst: Air.Inst.Index) !void {
173449173453 const cond_ty = self.typeOf(pl_op.operand);
173450173454 const extra = self.air.extraData(Air.CondBr, pl_op.payload);
173451173455 const then_body: []const Air.Inst.Index =
173452 @ptrCast(self.air.extra[extra.end..][0..extra.data.then_body_len]);
173456 @ptrCast(self.air.extra.items[extra.end..][0..extra.data.then_body_len]);
173453173457 const else_body: []const Air.Inst.Index =
173454 @ptrCast(self.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]);
173458 @ptrCast(self.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]);
173455173459 const liveness_cond_br = self.liveness.getCondBr(inst);
173456173460
173457173461 // If the condition dies here in this condbr instruction, process
......@@ -173838,7 +173842,7 @@ fn airLoop(self: *CodeGen, inst: Air.Inst.Index) !void {
173838173842 // A loop is a setup to be able to jump back to the beginning.
173839173843 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
173840173844 const loop = self.air.extraData(Air.Block, ty_pl.payload);
173841 const body: []const Air.Inst.Index = @ptrCast(self.air.extra[loop.end..][0..loop.data.body_len]);
173845 const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[loop.end..][0..loop.data.body_len]);
173842173846
173843173847 const state = try self.saveState();
173844173848
......@@ -174469,9 +174473,9 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
174469174473 const extra = self.air.extraData(Air.Asm, ty_pl.payload);
174470174474 const clobbers_len: u31 = @truncate(extra.data.flags);
174471174475 var extra_i: usize = extra.end;
174472 const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra_i..][0..extra.data.outputs_len]);
174476 const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.outputs_len]);
174473174477 extra_i += outputs.len;
174474 const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra_i..][0..extra.data.inputs_len]);
174478 const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.inputs_len]);
174475174479 extra_i += inputs.len;
174476174480
174477174481 var result: MCValue = .none;
......@@ -174489,8 +174493,8 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
174489174493
174490174494 var outputs_extra_i = extra_i;
174491174495 for (outputs) |output| {
174492 const extra_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]);
174493 const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0);
174496 const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]);
174497 const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0);
174494174498 const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
174495174499 // This equation accounts for the fact that even if we have exactly 4 bytes
174496174500 // for the string, we still use the next u32 for the null terminator.
......@@ -174575,7 +174579,7 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
174575174579 }
174576174580
174577174581 for (inputs) |input| {
174578 const input_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]);
174582 const input_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]);
174579174583 const constraint = std.mem.sliceTo(input_bytes, 0);
174580174584 const name = std.mem.sliceTo(input_bytes[constraint.len + 1 ..], 0);
174581174585 // This equation accounts for the fact that even if we have exactly 4 bytes
......@@ -174663,7 +174667,7 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
174663174667 {
174664174668 var clobber_i: u32 = 0;
174665174669 while (clobber_i < clobbers_len) : (clobber_i += 1) {
174666 const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0);
174670 const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0);
174667174671 // This equation accounts for the fact that even if we have exactly 4 bytes
174668174672 // for the string, we still use the next u32 for the null terminator.
174669174673 extra_i += clobber.len / 4 + 1;
......@@ -174719,7 +174723,7 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
174719174723 labels.deinit(self.gpa);
174720174724 }
174721174725
174722 const asm_source = std.mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len];
174726 const asm_source = std.mem.sliceAsBytes(self.air.extra.items[extra_i..])[0..extra.data.source_len];
174723174727 var line_it = std.mem.tokenizeAny(u8, asm_source, "\n\r;");
174724174728 next_line: while (line_it.next()) |line| {
174725174729 var mnem_it = std.mem.tokenizeAny(u8, line, " \t");
......@@ -175131,9 +175135,9 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
175131175135 return self.fail("undefined label: '{s}'", .{label.key_ptr.*});
175132175136
175133175137 for (outputs, args.items[0..outputs.len]) |output, arg_mcv| {
175134 const extra_bytes = std.mem.sliceAsBytes(self.air.extra[outputs_extra_i..]);
175138 const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[outputs_extra_i..]);
175135175139 const constraint =
175136 std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[outputs_extra_i..]), 0);
175140 std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[outputs_extra_i..]), 0);
175137175141 const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
175138175142 // This equation accounts for the fact that even if we have exactly 4 bytes
175139175143 // for the string, we still use the next u32 for the null terminator.
......@@ -175146,7 +175150,7 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
175146175150 }
175147175151
175148175152 simple: {
175149 var buf: [Liveness.bpi - 1]Air.Inst.Ref = @splat(.none);
175153 var buf: [Air.Liveness.bpi - 1]Air.Inst.Ref = @splat(.none);
175150175154 var buf_index: usize = 0;
175151175155 for (outputs) |output| {
175152175156 if (output == .none) continue;
......@@ -179659,7 +179663,7 @@ fn airAggregateInit(self: *CodeGen, inst: Air.Inst.Index) !void {
179659179663 const result_ty = self.typeOfIndex(inst);
179660179664 const len: usize = @intCast(result_ty.arrayLen(zcu));
179661179665 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
179662 const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra[ty_pl.payload..][0..len]);
179666 const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]);
179663179667 const result: MCValue = result: {
179664179668 switch (result_ty.zigTypeTag(zcu)) {
179665179669 .@"struct" => {
......@@ -179823,8 +179827,8 @@ fn airAggregateInit(self: *CodeGen, inst: Air.Inst.Index) !void {
179823179827 }
179824179828 };
179825179829
179826 if (elements.len <= Liveness.bpi - 1) {
179827 var buf: [Liveness.bpi - 1]Air.Inst.Ref = @splat(.none);
179830 if (elements.len <= Air.Liveness.bpi - 1) {
179831 var buf: [Air.Liveness.bpi - 1]Air.Inst.Ref = @splat(.none);
179828179832 @memcpy(buf[0..elements.len], elements);
179829179833 return self.finishAir(inst, result, buf);
179830179834 }
......@@ -186387,7 +186391,7 @@ const Temp = struct {
186387186391 for (0.., op_refs, op_temps) |op_index, op_ref, op_temp| {
186388186392 if (op_temp.index == temp.index) continue;
186389186393 if (op_temp.tracking(cg).short != .dead) try op_temp.die(cg);
186390 if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
186394 if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
186391186395 if (cg.reused_operands.isSet(op_index)) continue;
186392186396 try cg.processDeath(op_ref.toIndexAllowNone() orelse continue);
186393186397 }
......@@ -186407,7 +186411,7 @@ const Temp = struct {
186407186411 }
186408186412 for (0.., op_refs, op_temps) |op_index, op_ref, op_temp| {
186409186413 if (op_temp.index != temp.index) continue;
186410 if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
186414 if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
186411186415 if (cg.reused_operands.isSet(op_index)) continue;
186412186416 try cg.processDeath(op_ref.toIndexAllowNone() orelse continue);
186413186417 }
src/codegen.zig+8-6
......@@ -14,7 +14,6 @@ const Allocator = mem.Allocator;
1414const Compilation = @import("Compilation.zig");
1515const ErrorMsg = Zcu.ErrorMsg;
1616const InternPool = @import("InternPool.zig");
17const Liveness = @import("Liveness.zig");
1817const Zcu = @import("Zcu.zig");
1918
2019const Type = @import("Type.zig");
......@@ -33,15 +32,18 @@ fn devFeatureForBackend(comptime backend: std.builtin.CompilerBackend) dev.Featu
3332 return @field(dev.Feature, @tagName(backend)["stage2_".len..] ++ "_backend");
3433}
3534
36fn importBackend(comptime backend: std.builtin.CompilerBackend) type {
35pub fn importBackend(comptime backend: std.builtin.CompilerBackend) ?type {
3736 return switch (backend) {
3837 .stage2_aarch64 => @import("arch/aarch64/CodeGen.zig"),
3938 .stage2_arm => @import("arch/arm/CodeGen.zig"),
39 .stage2_c => @import("codegen/c.zig"),
40 .stage2_llvm => @import("codegen/llvm.zig"),
4041 .stage2_powerpc => @import("arch/powerpc/CodeGen.zig"),
4142 .stage2_riscv64 => @import("arch/riscv64/CodeGen.zig"),
4243 .stage2_sparc64 => @import("arch/sparc64/CodeGen.zig"),
44 .stage2_spirv64 => @import("codegen/spirv.zig"),
4345 .stage2_x86_64 => @import("arch/x86_64/CodeGen.zig"),
44 else => unreachable,
46 else => null,
4547 };
4648}
4749
......@@ -51,7 +53,7 @@ pub fn generateFunction(
5153 src_loc: Zcu.LazySrcLoc,
5254 func_index: InternPool.Index,
5355 air: Air,
54 liveness: Liveness,
56 liveness: Air.Liveness,
5557 code: *std.ArrayListUnmanaged(u8),
5658 debug_output: link.File.DebugInfoOutput,
5759) CodeGenError!void {
......@@ -68,7 +70,7 @@ pub fn generateFunction(
6870 .stage2_x86_64,
6971 => |backend| {
7072 dev.check(devFeatureForBackend(backend));
71 return importBackend(backend).generate(lf, pt, src_loc, func_index, air, liveness, code, debug_output);
73 return importBackend(backend).?.generate(lf, pt, src_loc, func_index, air, liveness, code, debug_output);
7274 },
7375 }
7476}
......@@ -93,7 +95,7 @@ pub fn generateLazyFunction(
9395 .stage2_x86_64,
9496 => |backend| {
9597 dev.check(devFeatureForBackend(backend));
96 return importBackend(backend).generateLazy(lf, pt, src_loc, lazy_sym, code, debug_output);
98 return importBackend(backend).?.generateLazy(lf, pt, src_loc, lazy_sym, code, debug_output);
9799 },
98100 }
99101}
src/codegen/c.zig+34-35
......@@ -14,7 +14,6 @@ const C = link.File.C;
1414const Decl = Zcu.Decl;
1515const trace = @import("../tracy.zig").trace;
1616const Air = @import("../Air.zig");
17const Liveness = @import("../Liveness.zig");
1817const InternPool = @import("../InternPool.zig");
1918const Alignment = InternPool.Alignment;
2019
......@@ -356,7 +355,7 @@ pub fn isMangledIdent(ident: []const u8, solo: bool) bool {
356355/// It is not available when generating .h file.
357356pub const Function = struct {
358357 air: Air,
359 liveness: Liveness,
358 liveness: Air.Liveness,
360359 value_map: CValueMap,
361360 blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .empty,
362361 next_arg_index: u32 = 0,
......@@ -2323,9 +2322,9 @@ pub const DeclGen = struct {
23232322
23242323 const pt = dg.pt;
23252324 const zcu = pt.zcu;
2326 const int_info = if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else std.builtin.Type.Int{
2325 const int_info: std.builtin.Type.Int = if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else .{
23272326 .signedness = .unsigned,
2328 .bits = @as(u16, @intCast(ty.bitSize(zcu))),
2327 .bits = @intCast(ty.bitSize(zcu)),
23292328 };
23302329
23312330 if (is_big) try writer.print(", {}", .{int_info.signedness == .signed});
......@@ -3179,7 +3178,7 @@ fn genBodyResolveState(f: *Function, inst: Air.Inst.Index, leading_deaths: []con
31793178 // Remember how many locals there were before entering the body so that we can free any that
31803179 // were newly introduced. Any new locals must necessarily be logically free after the then
31813180 // branch is complete.
3182 const pre_locals_len = @as(LocalIndex, @intCast(f.locals.items.len));
3181 const pre_locals_len: LocalIndex = @intCast(f.locals.items.len);
31833182
31843183 for (leading_deaths) |death| {
31853184 try die(f, inst, death.toRef());
......@@ -4540,7 +4539,7 @@ fn airCall(
45404539
45414540 const pl_op = f.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
45424541 const extra = f.air.extraData(Air.Call, pl_op.payload);
4543 const args = @as([]const Air.Inst.Ref, @ptrCast(f.air.extra[extra.end..][0..extra.data.args_len]));
4542 const args: []const Air.Inst.Ref = @ptrCast(f.air.extra.items[extra.end..][0..extra.data.args_len]);
45444543
45454544 const resolved_args = try gpa.alloc(CValue, args.len);
45464545 defer gpa.free(resolved_args);
......@@ -4708,7 +4707,7 @@ fn airDbgInlineBlock(f: *Function, inst: Air.Inst.Index) !CValue {
47084707 const owner_nav = ip.getNav(zcu.funcInfo(extra.data.func).owner_nav);
47094708 const writer = f.object.writer();
47104709 try writer.print("/* inline:{} */\n", .{owner_nav.fqn.fmt(&zcu.intern_pool)});
4711 return lowerBlock(f, inst, @ptrCast(f.air.extra[extra.end..][0..extra.data.body_len]));
4710 return lowerBlock(f, inst, @ptrCast(f.air.extra.items[extra.end..][0..extra.data.body_len]));
47124711}
47134712
47144713fn airDbgVar(f: *Function, inst: Air.Inst.Index) !CValue {
......@@ -4729,7 +4728,7 @@ fn airDbgVar(f: *Function, inst: Air.Inst.Index) !CValue {
47294728fn airBlock(f: *Function, inst: Air.Inst.Index) !CValue {
47304729 const ty_pl = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
47314730 const extra = f.air.extraData(Air.Block, ty_pl.payload);
4732 return lowerBlock(f, inst, @ptrCast(f.air.extra[extra.end..][0..extra.data.body_len]));
4731 return lowerBlock(f, inst, @ptrCast(f.air.extra.items[extra.end..][0..extra.data.body_len]));
47334732}
47344733
47354734fn lowerBlock(f: *Function, inst: Air.Inst.Index, body: []const Air.Inst.Index) !CValue {
......@@ -4781,7 +4780,7 @@ fn lowerBlock(f: *Function, inst: Air.Inst.Index, body: []const Air.Inst.Index)
47814780fn airTry(f: *Function, inst: Air.Inst.Index) !CValue {
47824781 const pl_op = f.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
47834782 const extra = f.air.extraData(Air.Try, pl_op.payload);
4784 const body: []const Air.Inst.Index = @ptrCast(f.air.extra[extra.end..][0..extra.data.body_len]);
4783 const body: []const Air.Inst.Index = @ptrCast(f.air.extra.items[extra.end..][0..extra.data.body_len]);
47854784 const err_union_ty = f.typeOf(pl_op.operand);
47864785 return lowerTry(f, inst, pl_op.operand, body, err_union_ty, false);
47874786}
......@@ -4791,7 +4790,7 @@ fn airTryPtr(f: *Function, inst: Air.Inst.Index) !CValue {
47914790 const zcu = pt.zcu;
47924791 const ty_pl = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
47934792 const extra = f.air.extraData(Air.TryPtr, ty_pl.payload);
4794 const body: []const Air.Inst.Index = @ptrCast(f.air.extra[extra.end..][0..extra.data.body_len]);
4793 const body: []const Air.Inst.Index = @ptrCast(f.air.extra.items[extra.end..][0..extra.data.body_len]);
47954794 const err_union_ty = f.typeOf(extra.data.ptr).childType(zcu);
47964795 return lowerTry(f, inst, extra.data.ptr, body, err_union_ty, true);
47974796}
......@@ -5100,7 +5099,7 @@ fn airUnreach(f: *Function) !void {
51005099fn airLoop(f: *Function, inst: Air.Inst.Index) !void {
51015100 const ty_pl = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
51025101 const loop = f.air.extraData(Air.Block, ty_pl.payload);
5103 const body: []const Air.Inst.Index = @ptrCast(f.air.extra[loop.end..][0..loop.data.body_len]);
5102 const body: []const Air.Inst.Index = @ptrCast(f.air.extra.items[loop.end..][0..loop.data.body_len]);
51045103 const writer = f.object.writer();
51055104
51065105 // `repeat` instructions matching this loop will branch to
......@@ -5116,8 +5115,8 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !void {
51165115 const cond = try f.resolveInst(pl_op.operand);
51175116 try reap(f, inst, &.{pl_op.operand});
51185117 const extra = f.air.extraData(Air.CondBr, pl_op.payload);
5119 const then_body: []const Air.Inst.Index = @ptrCast(f.air.extra[extra.end..][0..extra.data.then_body_len]);
5120 const else_body: []const Air.Inst.Index = @ptrCast(f.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]);
5118 const then_body: []const Air.Inst.Index = @ptrCast(f.air.extra.items[extra.end..][0..extra.data.then_body_len]);
5119 const else_body: []const Air.Inst.Index = @ptrCast(f.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]);
51215120 const liveness_condbr = f.liveness.getCondBr(inst);
51225121 const writer = f.object.writer();
51235122
......@@ -5322,12 +5321,12 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue {
53225321 const ty_pl = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
53235322 const extra = f.air.extraData(Air.Asm, ty_pl.payload);
53245323 const is_volatile = @as(u1, @truncate(extra.data.flags >> 31)) != 0;
5325 const clobbers_len = @as(u31, @truncate(extra.data.flags));
5324 const clobbers_len: u31 = @truncate(extra.data.flags);
53265325 const gpa = f.object.dg.gpa;
53275326 var extra_i: usize = extra.end;
5328 const outputs = @as([]const Air.Inst.Ref, @ptrCast(f.air.extra[extra_i..][0..extra.data.outputs_len]));
5327 const outputs: []const Air.Inst.Ref = @ptrCast(f.air.extra.items[extra_i..][0..extra.data.outputs_len]);
53295328 extra_i += outputs.len;
5330 const inputs = @as([]const Air.Inst.Ref, @ptrCast(f.air.extra[extra_i..][0..extra.data.inputs_len]));
5329 const inputs: []const Air.Inst.Ref = @ptrCast(f.air.extra.items[extra_i..][0..extra.data.inputs_len]);
53315330 extra_i += inputs.len;
53325331
53335332 const result = result: {
......@@ -5347,10 +5346,10 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue {
53475346 break :local inst_local;
53485347 } else .none;
53495348
5350 const locals_begin = @as(LocalIndex, @intCast(f.locals.items.len));
5349 const locals_begin: LocalIndex = @intCast(f.locals.items.len);
53515350 const constraints_extra_begin = extra_i;
53525351 for (outputs) |output| {
5353 const extra_bytes = mem.sliceAsBytes(f.air.extra[extra_i..]);
5352 const extra_bytes = mem.sliceAsBytes(f.air.extra.items[extra_i..]);
53545353 const constraint = mem.sliceTo(extra_bytes, 0);
53555354 const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
53565355 // This equation accounts for the fact that even if we have exactly 4 bytes
......@@ -5384,7 +5383,7 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue {
53845383 }
53855384 }
53865385 for (inputs) |input| {
5387 const extra_bytes = mem.sliceAsBytes(f.air.extra[extra_i..]);
5386 const extra_bytes = mem.sliceAsBytes(f.air.extra.items[extra_i..]);
53885387 const constraint = mem.sliceTo(extra_bytes, 0);
53895388 const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
53905389 // This equation accounts for the fact that even if we have exactly 4 bytes
......@@ -5419,14 +5418,14 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue {
54195418 }
54205419 }
54215420 for (0..clobbers_len) |_| {
5422 const clobber = mem.sliceTo(mem.sliceAsBytes(f.air.extra[extra_i..]), 0);
5421 const clobber = mem.sliceTo(mem.sliceAsBytes(f.air.extra.items[extra_i..]), 0);
54235422 // This equation accounts for the fact that even if we have exactly 4 bytes
54245423 // for the string, we still use the next u32 for the null terminator.
54255424 extra_i += clobber.len / 4 + 1;
54265425 }
54275426
54285427 {
5429 const asm_source = mem.sliceAsBytes(f.air.extra[extra_i..])[0..extra.data.source_len];
5428 const asm_source = mem.sliceAsBytes(f.air.extra.items[extra_i..])[0..extra.data.source_len];
54305429
54315430 var stack = std.heap.stackFallback(256, f.object.dg.gpa);
54325431 const allocator = stack.get();
......@@ -5484,7 +5483,7 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue {
54845483 var locals_index = locals_begin;
54855484 try writer.writeByte(':');
54865485 for (outputs, 0..) |output, index| {
5487 const extra_bytes = mem.sliceAsBytes(f.air.extra[extra_i..]);
5486 const extra_bytes = mem.sliceAsBytes(f.air.extra.items[extra_i..]);
54885487 const constraint = mem.sliceTo(extra_bytes, 0);
54895488 const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
54905489 // This equation accounts for the fact that even if we have exactly 4 bytes
......@@ -5508,7 +5507,7 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue {
55085507 }
55095508 try writer.writeByte(':');
55105509 for (inputs, 0..) |input, index| {
5511 const extra_bytes = mem.sliceAsBytes(f.air.extra[extra_i..]);
5510 const extra_bytes = mem.sliceAsBytes(f.air.extra.items[extra_i..]);
55125511 const constraint = mem.sliceTo(extra_bytes, 0);
55135512 const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
55145513 // This equation accounts for the fact that even if we have exactly 4 bytes
......@@ -5531,7 +5530,7 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue {
55315530 }
55325531 try writer.writeByte(':');
55335532 for (0..clobbers_len) |clobber_i| {
5534 const clobber = mem.sliceTo(mem.sliceAsBytes(f.air.extra[extra_i..]), 0);
5533 const clobber = mem.sliceTo(mem.sliceAsBytes(f.air.extra.items[extra_i..]), 0);
55355534 // This equation accounts for the fact that even if we have exactly 4 bytes
55365535 // for the string, we still use the next u32 for the null terminator.
55375536 extra_i += clobber.len / 4 + 1;
......@@ -5546,7 +5545,7 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue {
55465545 extra_i = constraints_extra_begin;
55475546 locals_index = locals_begin;
55485547 for (outputs) |output| {
5549 const extra_bytes = mem.sliceAsBytes(f.air.extra[extra_i..]);
5548 const extra_bytes = mem.sliceAsBytes(f.air.extra.items[extra_i..]);
55505549 const constraint = mem.sliceTo(extra_bytes, 0);
55515550 const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
55525551 // This equation accounts for the fact that even if we have exactly 4 bytes
......@@ -6725,7 +6724,7 @@ fn airAtomicRmw(f: *Function, inst: Air.Inst.Index) !CValue {
67256724 const operand_mat = try Materialize.start(f, inst, ty, operand);
67266725 try reap(f, inst, &.{ pl_op.operand, extra.operand });
67276726
6728 const repr_bits = @as(u16, @intCast(ty.abiSize(zcu) * 8));
6727 const repr_bits: u16 = @intCast(ty.abiSize(zcu) * 8);
67296728 const is_float = ty.isRuntimeFloat();
67306729 const is_128 = repr_bits == 128;
67316730 const repr_ty = if (is_float) pt.intType(.unsigned, repr_bits) catch unreachable else ty;
......@@ -7325,8 +7324,8 @@ fn airAggregateInit(f: *Function, inst: Air.Inst.Index) !CValue {
73257324 const ip = &zcu.intern_pool;
73267325 const ty_pl = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
73277326 const inst_ty = f.typeOfIndex(inst);
7328 const len = @as(usize, @intCast(inst_ty.arrayLen(zcu)));
7329 const elements = @as([]const Air.Inst.Ref, @ptrCast(f.air.extra[ty_pl.payload..][0..len]));
7327 const len: usize = @intCast(inst_ty.arrayLen(zcu));
7328 const elements: []const Air.Inst.Ref = @ptrCast(f.air.extra.items[ty_pl.payload..][0..len]);
73307329 const gpa = f.object.dg.gpa;
73317330 const resolved_elements = try gpa.alloc(CValue, elements.len);
73327331 defer gpa.free(resolved_elements);
......@@ -7830,7 +7829,7 @@ fn IndentWriter(comptime UnderlyingWriter: type) type {
78307829 }
78317830
78327831 pub fn write(self: *Self, bytes: []const u8) Error!usize {
7833 if (bytes.len == 0) return @as(usize, 0);
7832 if (bytes.len == 0) return 0;
78347833
78357834 const current_indent = self.indent_count * Self.indent_delta;
78367835 if (self.current_line_empty and current_indent > 0) {
......@@ -7860,7 +7859,7 @@ fn IndentWriter(comptime UnderlyingWriter: type) type {
78607859 }
78617860
78627861 fn writeNoIndent(self: *Self, bytes: []const u8) Error!usize {
7863 if (bytes.len == 0) return @as(usize, 0);
7862 if (bytes.len == 0) return 0;
78647863
78657864 try self.underlying_writer.writeAll(bytes);
78667865 if (bytes[bytes.len - 1] == '\n') {
......@@ -8048,7 +8047,7 @@ fn fmtStringLiteral(str: []const u8, sentinel: ?u8) std.fmt.Formatter(formatStri
80488047fn undefPattern(comptime IntType: type) IntType {
80498048 const int_info = @typeInfo(IntType).int;
80508049 const UnsignedType = std.meta.Int(.unsigned, int_info.bits);
8051 return @as(IntType, @bitCast(@as(UnsignedType, (1 << (int_info.bits | 1)) / 3)));
8050 return @bitCast(@as(UnsignedType, (1 << (int_info.bits | 1)) / 3));
80528051}
80538052
80548053const FormatIntLiteralContext = struct {
......@@ -8188,9 +8187,9 @@ fn formatIntLiteral(
81888187 wrap.len = wrap.limbs.len;
81898188 const limbs_per_c_limb = @divExact(wrap.len, c_limb_info.count);
81908189
8191 var c_limb_int_info = std.builtin.Type.Int{
8190 var c_limb_int_info: std.builtin.Type.Int = .{
81928191 .signedness = undefined,
8193 .bits = @as(u16, @intCast(@divExact(c_bits, c_limb_info.count))),
8192 .bits = @intCast(@divExact(c_bits, c_limb_info.count)),
81948193 };
81958194 var c_limb_ctype: CType = undefined;
81968195
......@@ -8349,7 +8348,7 @@ fn lowersToArray(ty: Type, pt: Zcu.PerThread) bool {
83498348}
83508349
83518350fn reap(f: *Function, inst: Air.Inst.Index, operands: []const Air.Inst.Ref) !void {
8352 assert(operands.len <= Liveness.bpi - 1);
8351 assert(operands.len <= Air.Liveness.bpi - 1);
83538352 var tomb_bits = f.liveness.getTombBits(inst);
83548353 for (operands) |operand| {
83558354 const dies = @as(u1, @truncate(tomb_bits)) != 0;
......@@ -8400,7 +8399,7 @@ fn freeLocal(f: *Function, inst: ?Air.Inst.Index, local_index: LocalIndex, ref_i
84008399const BigTomb = struct {
84018400 f: *Function,
84028401 inst: Air.Inst.Index,
8403 lbt: Liveness.BigTomb,
8402 lbt: Air.Liveness.BigTomb,
84048403
84058404 fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) !void {
84068405 const dies = bt.lbt.feed();
src/codegen/llvm.zig+20-21
......@@ -18,7 +18,6 @@ const Zcu = @import("../Zcu.zig");
1818const InternPool = @import("../InternPool.zig");
1919const Package = @import("../Package.zig");
2020const Air = @import("../Air.zig");
21const Liveness = @import("../Liveness.zig");
2221const Value = @import("../Value.zig");
2322const Type = @import("../Type.zig");
2423const x86_64_abi = @import("../arch/x86_64/abi.zig");
......@@ -1121,7 +1120,7 @@ pub const Object = struct {
11211120 pt: Zcu.PerThread,
11221121 func_index: InternPool.Index,
11231122 air: Air,
1124 liveness: Liveness,
1123 liveness: Air.Liveness,
11251124 ) !void {
11261125 assert(std.meta.eql(pt, o.pt));
11271126 const zcu = pt.zcu;
......@@ -4616,7 +4615,7 @@ pub const FuncGen = struct {
46164615 gpa: Allocator,
46174616 ng: *NavGen,
46184617 air: Air,
4619 liveness: Liveness,
4618 liveness: Air.Liveness,
46204619 wip: Builder.WipFunction,
46214620 is_naked: bool,
46224621 fuzz: ?Fuzz,
......@@ -5183,7 +5182,7 @@ pub const FuncGen = struct {
51835182 fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !Builder.Value {
51845183 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
51855184 const extra = self.air.extraData(Air.Call, pl_op.payload);
5186 const args: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra.end..][0..extra.data.args_len]);
5185 const args: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.args_len]);
51875186 const o = self.ng.object;
51885187 const pt = o.pt;
51895188 const zcu = pt.zcu;
......@@ -5856,7 +5855,7 @@ pub const FuncGen = struct {
58565855 fn airBlock(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
58575856 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
58585857 const extra = self.air.extraData(Air.Block, ty_pl.payload);
5859 return self.lowerBlock(inst, null, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]));
5858 return self.lowerBlock(inst, null, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]));
58605859 }
58615860
58625861 fn lowerBlock(
......@@ -6140,8 +6139,8 @@ pub const FuncGen = struct {
61406139 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
61416140 const cond = try self.resolveInst(pl_op.operand);
61426141 const extra = self.air.extraData(Air.CondBr, pl_op.payload);
6143 const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.then_body_len]);
6144 const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]);
6142 const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.then_body_len]);
6143 const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]);
61456144
61466145 const Hint = enum {
61476146 none,
......@@ -6205,7 +6204,7 @@ pub const FuncGen = struct {
62056204 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
62066205 const err_union = try self.resolveInst(pl_op.operand);
62076206 const extra = self.air.extraData(Air.Try, pl_op.payload);
6208 const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
6207 const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
62096208 const err_union_ty = self.typeOf(pl_op.operand);
62106209 const payload_ty = self.typeOfIndex(inst);
62116210 const can_elide_load = if (isByRef(payload_ty, zcu)) self.canElideLoad(body_tail) else false;
......@@ -6219,7 +6218,7 @@ pub const FuncGen = struct {
62196218 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
62206219 const extra = self.air.extraData(Air.TryPtr, ty_pl.payload);
62216220 const err_union_ptr = try self.resolveInst(extra.data.ptr);
6222 const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
6221 const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
62236222 const err_union_ty = self.typeOf(extra.data.ptr).childType(zcu);
62246223 const is_unused = self.liveness.isUnused(inst);
62256224
......@@ -6550,7 +6549,7 @@ pub const FuncGen = struct {
65506549 fn airLoop(self: *FuncGen, inst: Air.Inst.Index) !void {
65516550 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
65526551 const loop = self.air.extraData(Air.Block, ty_pl.payload);
6553 const body: []const Air.Inst.Index = @ptrCast(self.air.extra[loop.end..][0..loop.data.body_len]);
6552 const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[loop.end..][0..loop.data.body_len]);
65546553 const loop_block = try self.wip.block(1, "Loop"); // `airRepeat` will increment incoming each time
65556554 _ = try self.wip.br(loop_block);
65566555
......@@ -7076,7 +7075,7 @@ pub const FuncGen = struct {
70767075 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
70777076 const extra = self.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
70787077 self.arg_inline_index = 0;
7079 return self.lowerBlock(inst, extra.data.func, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]));
7078 return self.lowerBlock(inst, extra.data.func, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]));
70807079 }
70817080
70827081 fn airDbgVarPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
......@@ -7201,9 +7200,9 @@ pub const FuncGen = struct {
72017200 const clobbers_len: u31 = @truncate(extra.data.flags);
72027201 var extra_i: usize = extra.end;
72037202
7204 const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra_i..][0..extra.data.outputs_len]);
7203 const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.outputs_len]);
72057204 extra_i += outputs.len;
7206 const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra_i..][0..extra.data.inputs_len]);
7205 const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.inputs_len]);
72077206 extra_i += inputs.len;
72087207
72097208 var llvm_constraints: std.ArrayListUnmanaged(u8) = .empty;
......@@ -7239,8 +7238,8 @@ pub const FuncGen = struct {
72397238
72407239 var rw_extra_i = extra_i;
72417240 for (outputs, llvm_ret_indirect, llvm_rw_vals) |output, *is_indirect, *llvm_rw_val| {
7242 const extra_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]);
7243 const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0);
7241 const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]);
7242 const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0);
72447243 const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
72457244 // This equation accounts for the fact that even if we have exactly 4 bytes
72467245 // for the string, we still use the next u32 for the null terminator.
......@@ -7320,7 +7319,7 @@ pub const FuncGen = struct {
73207319 }
73217320
73227321 for (inputs) |input| {
7323 const extra_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]);
7322 const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]);
73247323 const constraint = std.mem.sliceTo(extra_bytes, 0);
73257324 const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
73267325 // This equation accounts for the fact that even if we have exactly 4 bytes
......@@ -7385,8 +7384,8 @@ pub const FuncGen = struct {
73857384 }
73867385
73877386 for (outputs, llvm_ret_indirect, llvm_rw_vals, 0..) |output, is_indirect, llvm_rw_val, output_index| {
7388 const extra_bytes = std.mem.sliceAsBytes(self.air.extra[rw_extra_i..]);
7389 const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[rw_extra_i..]), 0);
7387 const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[rw_extra_i..]);
7388 const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[rw_extra_i..]), 0);
73907389 const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
73917390 // This equation accounts for the fact that even if we have exactly 4 bytes
73927391 // for the string, we still use the next u32 for the null terminator.
......@@ -7425,7 +7424,7 @@ pub const FuncGen = struct {
74257424 {
74267425 var clobber_i: u32 = 0;
74277426 while (clobber_i < clobbers_len) : (clobber_i += 1) {
7428 const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0);
7427 const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0);
74297428 // This equation accounts for the fact that even if we have exactly 4 bytes
74307429 // for the string, we still use the next u32 for the null terminator.
74317430 extra_i += clobber.len / 4 + 1;
......@@ -7465,7 +7464,7 @@ pub const FuncGen = struct {
74657464 else => {},
74667465 }
74677466
7468 const asm_source = std.mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len];
7467 const asm_source = std.mem.sliceAsBytes(self.air.extra.items[extra_i..])[0..extra.data.source_len];
74697468
74707469 // hackety hacks until stage2 has proper inline asm in the frontend.
74717470 var rendered_template = std.ArrayList(u8).init(self.gpa);
......@@ -10628,7 +10627,7 @@ pub const FuncGen = struct {
1062810627 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
1062910628 const result_ty = self.typeOfIndex(inst);
1063010629 const len: usize = @intCast(result_ty.arrayLen(zcu));
10631 const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra[ty_pl.payload..][0..len]);
10630 const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]);
1063210631 const llvm_result_ty = try o.lowerType(result_ty);
1063310632
1063410633 switch (result_ty.zigTypeTag(zcu)) {
src/codegen/spirv.zig+20-21
......@@ -10,7 +10,6 @@ const Decl = Zcu.Decl;
1010const Type = @import("../Type.zig");
1111const Value = @import("../Value.zig");
1212const Air = @import("../Air.zig");
13const Liveness = @import("../Liveness.zig");
1413const InternPool = @import("../InternPool.zig");
1514
1615const spec = @import("spirv/spec.zig");
......@@ -195,7 +194,7 @@ pub const Object = struct {
195194 pt: Zcu.PerThread,
196195 nav_index: InternPool.Nav.Index,
197196 air: Air,
198 liveness: Liveness,
197 liveness: Air.Liveness,
199198 do_codegen: bool,
200199 ) !void {
201200 const zcu = pt.zcu;
......@@ -242,7 +241,7 @@ pub const Object = struct {
242241 pt: Zcu.PerThread,
243242 func_index: InternPool.Index,
244243 air: Air,
245 liveness: Liveness,
244 liveness: Air.Liveness,
246245 ) !void {
247246 const nav = pt.zcu.funcInfo(func_index).owner_nav;
248247 // TODO: Separate types for generating decls and functions?
......@@ -303,7 +302,7 @@ const NavGen = struct {
303302
304303 /// The liveness analysis of the intermediate code for the declaration we are currently generating.
305304 /// Note: If the declaration is not a function, this value will be undefined!
306 liveness: Liveness,
305 liveness: Air.Liveness,
307306
308307 /// An array of function argument result-ids. Each index corresponds with the
309308 /// function argument of the same index.
......@@ -4627,7 +4626,7 @@ const NavGen = struct {
46274626 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
46284627 const result_ty = self.typeOfIndex(inst);
46294628 const len: usize = @intCast(result_ty.arrayLen(zcu));
4630 const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra[ty_pl.payload..][0..len]);
4629 const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]);
46314630
46324631 switch (result_ty.zigTypeTag(zcu)) {
46334632 .@"struct" => {
......@@ -5474,7 +5473,7 @@ const NavGen = struct {
54745473 fn airBlock(self: *NavGen, inst: Air.Inst.Index) !?IdRef {
54755474 const inst_datas = self.air.instructions.items(.data);
54765475 const extra = self.air.extraData(Air.Block, inst_datas[@intFromEnum(inst)].ty_pl.payload);
5477 return self.lowerBlock(inst, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]));
5476 return self.lowerBlock(inst, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]));
54785477 }
54795478
54805479 fn lowerBlock(self: *NavGen, inst: Air.Inst.Index, body: []const Air.Inst.Index) !?IdRef {
......@@ -5657,8 +5656,8 @@ const NavGen = struct {
56575656 fn airCondBr(self: *NavGen, inst: Air.Inst.Index) !void {
56585657 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
56595658 const cond_br = self.air.extraData(Air.CondBr, pl_op.payload);
5660 const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra[cond_br.end..][0..cond_br.data.then_body_len]);
5661 const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[cond_br.end + then_body.len ..][0..cond_br.data.else_body_len]);
5659 const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[cond_br.end..][0..cond_br.data.then_body_len]);
5660 const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[cond_br.end + then_body.len ..][0..cond_br.data.else_body_len]);
56625661 const condition_id = try self.resolve(pl_op.operand);
56635662
56645663 const then_label = self.spv.allocId();
......@@ -5717,7 +5716,7 @@ const NavGen = struct {
57175716 fn airLoop(self: *NavGen, inst: Air.Inst.Index) !void {
57185717 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
57195718 const loop = self.air.extraData(Air.Block, ty_pl.payload);
5720 const body: []const Air.Inst.Index = @ptrCast(self.air.extra[loop.end..][0..loop.data.body_len]);
5719 const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[loop.end..][0..loop.data.body_len]);
57215720
57225721 const body_label = self.spv.allocId();
57235722
......@@ -5837,7 +5836,7 @@ const NavGen = struct {
58375836 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
58385837 const err_union_id = try self.resolve(pl_op.operand);
58395838 const extra = self.air.extraData(Air.Try, pl_op.payload);
5840 const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
5839 const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
58415840
58425841 const err_union_ty = self.typeOf(pl_op.operand);
58435842 const payload_ty = self.typeOfIndex(inst);
......@@ -6344,7 +6343,7 @@ const NavGen = struct {
63446343 const old_base_line = self.base_line;
63456344 defer self.base_line = old_base_line;
63466345 self.base_line = zcu.navSrcLine(zcu.funcInfo(extra.data.func).owner_nav);
6347 return self.lowerBlock(inst, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]));
6346 return self.lowerBlock(inst, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]));
63486347 }
63496348
63506349 fn airDbgVar(self: *NavGen, inst: Air.Inst.Index) !void {
......@@ -6365,9 +6364,9 @@ const NavGen = struct {
63656364 if (!is_volatile and self.liveness.isUnused(inst)) return null;
63666365
63676366 var extra_i: usize = extra.end;
6368 const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra_i..][0..extra.data.outputs_len]);
6367 const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.outputs_len]);
63696368 extra_i += outputs.len;
6370 const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra_i..][0..extra.data.inputs_len]);
6369 const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.inputs_len]);
63716370 extra_i += inputs.len;
63726371
63736372 if (outputs.len > 1) {
......@@ -6386,15 +6385,15 @@ const NavGen = struct {
63866385 if (output != .none) {
63876386 return self.todo("implement inline asm with non-returned output", .{});
63886387 }
6389 const extra_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]);
6390 const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0);
6388 const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]);
6389 const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0);
63916390 const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
63926391 extra_i += (constraint.len + name.len + (2 + 3)) / 4;
63936392 // TODO: Record output and use it somewhere.
63946393 }
63956394
63966395 for (inputs) |input| {
6397 const extra_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]);
6396 const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]);
63986397 const constraint = std.mem.sliceTo(extra_bytes, 0);
63996398 const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
64006399 // This equation accounts for the fact that even if we have exactly 4 bytes
......@@ -6461,13 +6460,13 @@ const NavGen = struct {
64616460 {
64626461 var clobber_i: u32 = 0;
64636462 while (clobber_i < clobbers_len) : (clobber_i += 1) {
6464 const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0);
6463 const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0);
64656464 extra_i += clobber.len / 4 + 1;
64666465 // TODO: Record clobber and use it somewhere.
64676466 }
64686467 }
64696468
6470 const asm_source = std.mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len];
6469 const asm_source = std.mem.sliceAsBytes(self.air.extra.items[extra_i..])[0..extra.data.source_len];
64716470
64726471 as.assemble(asm_source) catch |err| switch (err) {
64736472 error.AssembleFail => {
......@@ -6501,8 +6500,8 @@ const NavGen = struct {
65016500
65026501 for (outputs) |output| {
65036502 _ = output;
6504 const extra_bytes = std.mem.sliceAsBytes(self.air.extra[output_extra_i..]);
6505 const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[output_extra_i..]), 0);
6503 const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[output_extra_i..]);
6504 const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[output_extra_i..]), 0);
65066505 const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
65076506 output_extra_i += (constraint.len + name.len + (2 + 3)) / 4;
65086507
......@@ -6531,7 +6530,7 @@ const NavGen = struct {
65316530 const zcu = pt.zcu;
65326531 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
65336532 const extra = self.air.extraData(Air.Call, pl_op.payload);
6534 const args: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra.end..][0..extra.data.args_len]);
6533 const args: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.args_len]);
65356534 const callee_ty = self.typeOf(pl_op.operand);
65366535 const zig_fn_ty = switch (callee_ty.zigTypeTag(zcu)) {
65376536 .@"fn" => callee_ty,
src/link.zig+4-4
......@@ -15,7 +15,6 @@ const Path = std.Build.Cache.Path;
1515const Directory = std.Build.Cache.Directory;
1616const Compilation = @import("Compilation.zig");
1717const LibCInstallation = std.zig.LibCInstallation;
18const Liveness = @import("Liveness.zig");
1918const Zcu = @import("Zcu.zig");
2019const InternPool = @import("InternPool.zig");
2120const Type = @import("Type.zig");
......@@ -738,7 +737,7 @@ pub const File = struct {
738737 pt: Zcu.PerThread,
739738 func_index: InternPool.Index,
740739 air: Air,
741 liveness: Liveness,
740 liveness: Air.Liveness,
742741 ) UpdateNavError!void {
743742 switch (base.tag) {
744743 inline else => |tag| {
......@@ -1601,8 +1600,9 @@ pub fn doTask(comp: *Compilation, tid: usize, task: Task) void {
16011600 if (comp.remaining_prelink_tasks == 0) {
16021601 const pt: Zcu.PerThread = .activate(comp.zcu.?, @enumFromInt(tid));
16031602 defer pt.deactivate();
1604 // This call takes ownership of `func.air`.
1605 pt.linkerUpdateFunc(func.func, func.air) catch |err| switch (err) {
1603 var air = func.air;
1604 defer air.deinit(comp.gpa);
1605 pt.linkerUpdateFunc(func.func, &air) catch |err| switch (err) {
16061606 error.OutOfMemory => diags.setAllocFailure(),
16071607 };
16081608 } else {
src/link/C.zig+1-2
......@@ -18,7 +18,6 @@ const trace = @import("../tracy.zig").trace;
1818const Type = @import("../Type.zig");
1919const Value = @import("../Value.zig");
2020const Air = @import("../Air.zig");
21const Liveness = @import("../Liveness.zig");
2221
2322pub const zig_h = "#include \"zig.h\"\n";
2423
......@@ -180,7 +179,7 @@ pub fn updateFunc(
180179 pt: Zcu.PerThread,
181180 func_index: InternPool.Index,
182181 air: Air,
183 liveness: Liveness,
182 liveness: Air.Liveness,
184183) link.File.UpdateNavError!void {
185184 const zcu = pt.zcu;
186185 const gpa = zcu.gpa;
src/link/Coff.zig+1-2
......@@ -1098,7 +1098,7 @@ pub fn updateFunc(
10981098 pt: Zcu.PerThread,
10991099 func_index: InternPool.Index,
11001100 air: Air,
1101 liveness: Liveness,
1101 liveness: Air.Liveness,
11021102) link.File.UpdateNavError!void {
11031103 if (build_options.skip_non_native and builtin.object_format != .coff) {
11041104 @panic("Attempted to compile for object format that was disabled by build configuration");
......@@ -3802,7 +3802,6 @@ const trace = @import("../tracy.zig").trace;
38023802
38033803const Air = @import("../Air.zig");
38043804const Compilation = @import("../Compilation.zig");
3805const Liveness = @import("../Liveness.zig");
38063805const LlvmObject = @import("../codegen/llvm.zig").Object;
38073806const Zcu = @import("../Zcu.zig");
38083807const InternPool = @import("../InternPool.zig");
src/link/Elf.zig+1-2
......@@ -2385,7 +2385,7 @@ pub fn updateFunc(
23852385 pt: Zcu.PerThread,
23862386 func_index: InternPool.Index,
23872387 air: Air,
2388 liveness: Liveness,
2388 liveness: Air.Liveness,
23892389) link.File.UpdateNavError!void {
23902390 if (build_options.skip_non_native and builtin.object_format != .elf) {
23912391 @panic("Attempted to compile for object format that was disabled by build configuration");
......@@ -5323,7 +5323,6 @@ const GotSection = synthetic_sections.GotSection;
53235323const GotPltSection = synthetic_sections.GotPltSection;
53245324const HashSection = synthetic_sections.HashSection;
53255325const LinkerDefined = @import("Elf/LinkerDefined.zig");
5326const Liveness = @import("../Liveness.zig");
53275326const LlvmObject = @import("../codegen/llvm.zig").Object;
53285327const Zcu = @import("../Zcu.zig");
53295328const Object = @import("Elf/Object.zig");
src/link/Elf/ZigObject.zig+1-2
......@@ -1416,7 +1416,7 @@ pub fn updateFunc(
14161416 pt: Zcu.PerThread,
14171417 func_index: InternPool.Index,
14181418 air: Air,
1419 liveness: Liveness,
1419 liveness: Air.Liveness,
14201420) link.File.UpdateNavError!void {
14211421 const tracy = trace(@src());
14221422 defer tracy.end();
......@@ -2367,7 +2367,6 @@ const Dwarf = @import("../Dwarf.zig");
23672367const Elf = @import("../Elf.zig");
23682368const File = @import("file.zig").File;
23692369const InternPool = @import("../../InternPool.zig");
2370const Liveness = @import("../../Liveness.zig");
23712370const Zcu = @import("../../Zcu.zig");
23722371const Object = @import("Object.zig");
23732372const Symbol = @import("Symbol.zig");
src/link/Goff.zig+1-2
......@@ -17,7 +17,6 @@ const link = @import("../link.zig");
1717const trace = @import("../tracy.zig").trace;
1818const build_options = @import("build_options");
1919const Air = @import("../Air.zig");
20const Liveness = @import("../Liveness.zig");
2120const LlvmObject = @import("../codegen/llvm.zig").Object;
2221
2322base: link.File,
......@@ -79,7 +78,7 @@ pub fn updateFunc(
7978 pt: Zcu.PerThread,
8079 func_index: InternPool.Index,
8180 air: Air,
82 liveness: Liveness,
81 liveness: Air.Liveness,
8382) link.File.UpdateNavError!void {
8483 if (build_options.skip_non_native and builtin.object_format != .goff)
8584 @panic("Attempted to compile for object format that was disabled by build configuration");
src/link/MachO.zig+1-2
......@@ -3074,7 +3074,7 @@ pub fn updateFunc(
30743074 pt: Zcu.PerThread,
30753075 func_index: InternPool.Index,
30763076 air: Air,
3077 liveness: Liveness,
3077 liveness: Air.Liveness,
30783078) link.File.UpdateNavError!void {
30793079 if (build_options.skip_non_native and builtin.object_format != .macho) {
30803080 @panic("Attempted to compile for object format that was disabled by build configuration");
......@@ -5496,7 +5496,6 @@ const ObjcStubsSection = synthetic.ObjcStubsSection;
54965496const Object = @import("MachO/Object.zig");
54975497const LazyBind = bind.LazyBind;
54985498const LaSymbolPtrSection = synthetic.LaSymbolPtrSection;
5499const Liveness = @import("../Liveness.zig");
55005499const LlvmObject = @import("../codegen/llvm.zig").Object;
55015500const Md5 = std.crypto.hash.Md5;
55025501const Zcu = @import("../Zcu.zig");
src/link/MachO/ZigObject.zig+1-2
......@@ -778,7 +778,7 @@ pub fn updateFunc(
778778 pt: Zcu.PerThread,
779779 func_index: InternPool.Index,
780780 air: Air,
781 liveness: Liveness,
781 liveness: Air.Liveness,
782782) link.File.UpdateNavError!void {
783783 const tracy = trace(@src());
784784 defer tracy.end();
......@@ -1820,7 +1820,6 @@ const Atom = @import("Atom.zig");
18201820const Dwarf = @import("../Dwarf.zig");
18211821const File = @import("file.zig").File;
18221822const InternPool = @import("../../InternPool.zig");
1823const Liveness = @import("../../Liveness.zig");
18241823const MachO = @import("../MachO.zig");
18251824const Nlist = Object.Nlist;
18261825const Zcu = @import("../../Zcu.zig");
src/link/Plan9.zig+1-2
......@@ -12,7 +12,6 @@ const trace = @import("../tracy.zig").trace;
1212const File = link.File;
1313const build_options = @import("build_options");
1414const Air = @import("../Air.zig");
15const Liveness = @import("../Liveness.zig");
1615const Type = @import("../Type.zig");
1716const Value = @import("../Value.zig");
1817const AnalUnit = InternPool.AnalUnit;
......@@ -389,7 +388,7 @@ pub fn updateFunc(
389388 pt: Zcu.PerThread,
390389 func_index: InternPool.Index,
391390 air: Air,
392 liveness: Liveness,
391 liveness: Air.Liveness,
393392) link.File.UpdateNavError!void {
394393 if (build_options.skip_non_native and builtin.object_format != .plan9) {
395394 @panic("Attempted to compile for object format that was disabled by build configuration");
src/link/SpirV.zig+1-2
......@@ -36,7 +36,6 @@ const codegen = @import("../codegen/spirv.zig");
3636const trace = @import("../tracy.zig").trace;
3737const build_options = @import("build_options");
3838const Air = @import("../Air.zig");
39const Liveness = @import("../Liveness.zig");
4039const Type = @import("../Type.zig");
4140const Value = @import("../Value.zig");
4241
......@@ -118,7 +117,7 @@ pub fn updateFunc(
118117 pt: Zcu.PerThread,
119118 func_index: InternPool.Index,
120119 air: Air,
121 liveness: Liveness,
120 liveness: Air.Liveness,
122121) link.File.UpdateNavError!void {
123122 if (build_options.skip_non_native) {
124123 @panic("Attempted to compile for architecture that was disabled by build configuration");
src/link/Wasm.zig+1-2
......@@ -36,7 +36,6 @@ const abi = @import("../arch/wasm/abi.zig");
3636const Compilation = @import("../Compilation.zig");
3737const Dwarf = @import("Dwarf.zig");
3838const InternPool = @import("../InternPool.zig");
39const Liveness = @import("../Liveness.zig");
4039const LlvmObject = @import("../codegen/llvm.zig").Object;
4140const Zcu = @import("../Zcu.zig");
4241const codegen = @import("../codegen.zig");
......@@ -3193,7 +3192,7 @@ pub fn deinit(wasm: *Wasm) void {
31933192 wasm.missing_exports.deinit(gpa);
31943193}
31953194
3196pub fn updateFunc(wasm: *Wasm, pt: Zcu.PerThread, func_index: InternPool.Index, air: Air, liveness: Liveness) !void {
3195pub fn updateFunc(wasm: *Wasm, pt: Zcu.PerThread, func_index: InternPool.Index, air: Air, liveness: Air.Liveness) !void {
31973196 if (build_options.skip_non_native and builtin.object_format != .wasm) {
31983197 @panic("Attempted to compile for object format that was disabled by build configuration");
31993198 }
src/link/Xcoff.zig+1-2
......@@ -17,7 +17,6 @@ const link = @import("../link.zig");
1717const trace = @import("../tracy.zig").trace;
1818const build_options = @import("build_options");
1919const Air = @import("../Air.zig");
20const Liveness = @import("../Liveness.zig");
2120const LlvmObject = @import("../codegen/llvm.zig").Object;
2221
2322base: link.File,
......@@ -79,7 +78,7 @@ pub fn updateFunc(
7978 pt: Zcu.PerThread,
8079 func_index: InternPool.Index,
8180 air: Air,
82 liveness: Liveness,
81 liveness: Air.Liveness,
8382) link.File.UpdateNavError!void {
8483 if (build_options.skip_non_native and builtin.object_format != .xcoff)
8584 @panic("Attempted to compile for object format that was disabled by build configuration");
src/print_air.zig+34-35
......@@ -6,20 +6,19 @@ const Zcu = @import("Zcu.zig");
66const Value = @import("Value.zig");
77const Type = @import("Type.zig");
88const Air = @import("Air.zig");
9const Liveness = @import("Liveness.zig");
109const InternPool = @import("InternPool.zig");
1110
12pub fn write(stream: anytype, pt: Zcu.PerThread, air: Air, liveness: ?Liveness) void {
11pub fn write(stream: anytype, pt: Zcu.PerThread, air: Air, liveness: ?Air.Liveness) void {
1312 const instruction_bytes = air.instructions.len *
1413 // Here we don't use @sizeOf(Air.Inst.Data) because it would include
1514 // the debug safety tag but we want to measure release size.
1615 (@sizeOf(Air.Inst.Tag) + 8);
17 const extra_bytes = air.extra.len * @sizeOf(u32);
16 const extra_bytes = air.extra.items.len * @sizeOf(u32);
1817 const tomb_bytes = if (liveness) |l| l.tomb_bits.len * @sizeOf(usize) else 0;
1918 const liveness_extra_bytes = if (liveness) |l| l.extra.len * @sizeOf(u32) else 0;
2019 const liveness_special_bytes = if (liveness) |l| l.special.count() * 8 else 0;
2120 const total_bytes = @sizeOf(Air) + instruction_bytes + extra_bytes +
22 @sizeOf(Liveness) + liveness_extra_bytes +
21 @sizeOf(Air.Liveness) + liveness_extra_bytes +
2322 liveness_special_bytes + tomb_bytes;
2423
2524 // zig fmt: off
......@@ -34,7 +33,7 @@ pub fn write(stream: anytype, pt: Zcu.PerThread, air: Air, liveness: ?Liveness)
3433 , .{
3534 fmtIntSizeBin(total_bytes),
3635 air.instructions.len, fmtIntSizeBin(instruction_bytes),
37 air.extra.len, fmtIntSizeBin(extra_bytes),
36 air.extra.items.len, fmtIntSizeBin(extra_bytes),
3837 fmtIntSizeBin(tomb_bytes),
3938 if (liveness) |l| l.extra.len else 0, fmtIntSizeBin(liveness_extra_bytes),
4039 if (liveness) |l| l.special.count() else 0, fmtIntSizeBin(liveness_special_bytes),
......@@ -57,7 +56,7 @@ pub fn writeInst(
5756 inst: Air.Inst.Index,
5857 pt: Zcu.PerThread,
5958 air: Air,
60 liveness: ?Liveness,
59 liveness: ?Air.Liveness,
6160) void {
6261 var writer: Writer = .{
6362 .pt = pt,
......@@ -70,11 +69,11 @@ pub fn writeInst(
7069 writer.writeInst(stream, inst) catch return;
7170}
7271
73pub fn dump(pt: Zcu.PerThread, air: Air, liveness: ?Liveness) void {
72pub fn dump(pt: Zcu.PerThread, air: Air, liveness: ?Air.Liveness) void {
7473 write(std.io.getStdErr().writer(), pt, air, liveness);
7574}
7675
77pub fn dumpInst(inst: Air.Inst.Index, pt: Zcu.PerThread, air: Air, liveness: ?Liveness) void {
76pub fn dumpInst(inst: Air.Inst.Index, pt: Zcu.PerThread, air: Air, liveness: ?Air.Liveness) void {
7877 writeInst(std.io.getStdErr().writer(), inst, pt, air, liveness);
7978}
8079
......@@ -82,7 +81,7 @@ const Writer = struct {
8281 pt: Zcu.PerThread,
8382 gpa: Allocator,
8483 air: Air,
85 liveness: ?Liveness,
84 liveness: ?Air.Liveness,
8685 indent: usize,
8786 skip_body: bool,
8887
......@@ -391,15 +390,15 @@ const Writer = struct {
391390 },
392391 else => unreachable,
393392 }
394 break :body w.air.extra[extra.end..][0..extra.data.body_len];
393 break :body w.air.extra.items[extra.end..][0..extra.data.body_len];
395394 },
396395 else => unreachable,
397396 });
398397 if (w.skip_body) return s.writeAll(", ...");
399 const liveness_block = if (w.liveness) |liveness|
398 const liveness_block: Air.Liveness.BlockSlices = if (w.liveness) |liveness|
400399 liveness.getBlock(inst)
401400 else
402 Liveness.BlockSlices{ .deaths = &.{} };
401 .{ .deaths = &.{} };
403402
404403 try s.writeAll(", {\n");
405404 const old_indent = w.indent;
......@@ -417,7 +416,7 @@ const Writer = struct {
417416 fn writeLoop(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
418417 const ty_pl = w.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
419418 const extra = w.air.extraData(Air.Block, ty_pl.payload);
420 const body: []const Air.Inst.Index = @ptrCast(w.air.extra[extra.end..][0..extra.data.body_len]);
419 const body: []const Air.Inst.Index = @ptrCast(w.air.extra.items[extra.end..][0..extra.data.body_len]);
421420
422421 try w.writeType(s, ty_pl.ty.toType());
423422 if (w.skip_body) return s.writeAll(", ...");
......@@ -435,7 +434,7 @@ const Writer = struct {
435434 const ty_pl = w.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
436435 const vector_ty = ty_pl.ty.toType();
437436 const len = @as(usize, @intCast(vector_ty.arrayLen(zcu)));
438 const elements = @as([]const Air.Inst.Ref, @ptrCast(w.air.extra[ty_pl.payload..][0..len]));
437 const elements = @as([]const Air.Inst.Ref, @ptrCast(w.air.extra.items[ty_pl.payload..][0..len]));
439438
440439 try w.writeType(s, vector_ty);
441440 try s.writeAll(", [");
......@@ -622,13 +621,13 @@ const Writer = struct {
622621 try s.writeAll(", volatile");
623622 }
624623
625 const outputs = @as([]const Air.Inst.Ref, @ptrCast(w.air.extra[extra_i..][0..extra.data.outputs_len]));
624 const outputs = @as([]const Air.Inst.Ref, @ptrCast(w.air.extra.items[extra_i..][0..extra.data.outputs_len]));
626625 extra_i += outputs.len;
627 const inputs = @as([]const Air.Inst.Ref, @ptrCast(w.air.extra[extra_i..][0..extra.data.inputs_len]));
626 const inputs = @as([]const Air.Inst.Ref, @ptrCast(w.air.extra.items[extra_i..][0..extra.data.inputs_len]));
628627 extra_i += inputs.len;
629628
630629 for (outputs) |output| {
631 const extra_bytes = std.mem.sliceAsBytes(w.air.extra[extra_i..]);
630 const extra_bytes = std.mem.sliceAsBytes(w.air.extra.items[extra_i..]);
632631 const constraint = std.mem.sliceTo(extra_bytes, 0);
633632 const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
634633
......@@ -648,7 +647,7 @@ const Writer = struct {
648647 }
649648
650649 for (inputs) |input| {
651 const extra_bytes = std.mem.sliceAsBytes(w.air.extra[extra_i..]);
650 const extra_bytes = std.mem.sliceAsBytes(w.air.extra.items[extra_i..]);
652651 const constraint = std.mem.sliceTo(extra_bytes, 0);
653652 const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
654653 // This equation accounts for the fact that even if we have exactly 4 bytes
......@@ -665,7 +664,7 @@ const Writer = struct {
665664 {
666665 var clobber_i: u32 = 0;
667666 while (clobber_i < clobbers_len) : (clobber_i += 1) {
668 const extra_bytes = std.mem.sliceAsBytes(w.air.extra[extra_i..]);
667 const extra_bytes = std.mem.sliceAsBytes(w.air.extra.items[extra_i..]);
669668 const clobber = std.mem.sliceTo(extra_bytes, 0);
670669 // This equation accounts for the fact that even if we have exactly 4 bytes
671670 // for the string, we still use the next u32 for the null terminator.
......@@ -676,7 +675,7 @@ const Writer = struct {
676675 try s.writeAll("}");
677676 }
678677 }
679 const asm_source = std.mem.sliceAsBytes(w.air.extra[extra_i..])[0..extra.data.source_len];
678 const asm_source = std.mem.sliceAsBytes(w.air.extra.items[extra_i..])[0..extra.data.source_len];
680679 try s.print(", \"{}\"", .{std.zig.fmtEscapes(asm_source)});
681680 }
682681
......@@ -695,7 +694,7 @@ const Writer = struct {
695694 fn writeCall(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
696695 const pl_op = w.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
697696 const extra = w.air.extraData(Air.Call, pl_op.payload);
698 const args = @as([]const Air.Inst.Ref, @ptrCast(w.air.extra[extra.end..][0..extra.data.args_len]));
697 const args = @as([]const Air.Inst.Ref, @ptrCast(w.air.extra.items[extra.end..][0..extra.data.args_len]));
699698 try w.writeOperand(s, inst, 0, pl_op.operand);
700699 try s.writeAll(", [");
701700 for (args, 0..) |arg, i| {
......@@ -720,11 +719,11 @@ const Writer = struct {
720719 fn writeTry(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
721720 const pl_op = w.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
722721 const extra = w.air.extraData(Air.Try, pl_op.payload);
723 const body: []const Air.Inst.Index = @ptrCast(w.air.extra[extra.end..][0..extra.data.body_len]);
724 const liveness_condbr = if (w.liveness) |liveness|
722 const body: []const Air.Inst.Index = @ptrCast(w.air.extra.items[extra.end..][0..extra.data.body_len]);
723 const liveness_condbr: Air.Liveness.CondBrSlices = if (w.liveness) |liveness|
725724 liveness.getCondBr(inst)
726725 else
727 Liveness.CondBrSlices{ .then_deaths = &.{}, .else_deaths = &.{} };
726 .{ .then_deaths = &.{}, .else_deaths = &.{} };
728727
729728 try w.writeOperand(s, inst, 0, pl_op.operand);
730729 if (w.skip_body) return s.writeAll(", ...");
......@@ -754,11 +753,11 @@ const Writer = struct {
754753 fn writeTryPtr(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
755754 const ty_pl = w.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
756755 const extra = w.air.extraData(Air.TryPtr, ty_pl.payload);
757 const body: []const Air.Inst.Index = @ptrCast(w.air.extra[extra.end..][0..extra.data.body_len]);
758 const liveness_condbr = if (w.liveness) |liveness|
756 const body: []const Air.Inst.Index = @ptrCast(w.air.extra.items[extra.end..][0..extra.data.body_len]);
757 const liveness_condbr: Air.Liveness.CondBrSlices = if (w.liveness) |liveness|
759758 liveness.getCondBr(inst)
760759 else
761 Liveness.CondBrSlices{ .then_deaths = &.{}, .else_deaths = &.{} };
760 .{ .then_deaths = &.{}, .else_deaths = &.{} };
762761
763762 try w.writeOperand(s, inst, 0, extra.data.ptr);
764763
......@@ -791,12 +790,12 @@ const Writer = struct {
791790 fn writeCondBr(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
792791 const pl_op = w.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
793792 const extra = w.air.extraData(Air.CondBr, pl_op.payload);
794 const then_body: []const Air.Inst.Index = @ptrCast(w.air.extra[extra.end..][0..extra.data.then_body_len]);
795 const else_body: []const Air.Inst.Index = @ptrCast(w.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]);
796 const liveness_condbr = if (w.liveness) |liveness|
793 const then_body: []const Air.Inst.Index = @ptrCast(w.air.extra.items[extra.end..][0..extra.data.then_body_len]);
794 const else_body: []const Air.Inst.Index = @ptrCast(w.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]);
795 const liveness_condbr: Air.Liveness.CondBrSlices = if (w.liveness) |liveness|
797796 liveness.getCondBr(inst)
798797 else
799 Liveness.CondBrSlices{ .then_deaths = &.{}, .else_deaths = &.{} };
798 .{ .then_deaths = &.{}, .else_deaths = &.{} };
800799
801800 try w.writeOperand(s, inst, 0, pl_op.operand);
802801 if (w.skip_body) return s.writeAll(", ...");
......@@ -850,14 +849,14 @@ const Writer = struct {
850849 fn writeSwitchBr(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
851850 const switch_br = w.air.unwrapSwitch(inst);
852851
853 const liveness = if (w.liveness) |liveness|
852 const liveness: Air.Liveness.SwitchBrTable = if (w.liveness) |liveness|
854853 liveness.getSwitchBr(w.gpa, inst, switch_br.cases_len + 1) catch
855854 @panic("out of memory")
856855 else blk: {
857856 const slice = w.gpa.alloc([]const Air.Inst.Index, switch_br.cases_len + 1) catch
858857 @panic("out of memory");
859858 @memset(slice, &.{});
860 break :blk Liveness.SwitchBrTable{ .deaths = slice };
859 break :blk .{ .deaths = slice };
861860 };
862861 defer w.gpa.free(liveness.deaths);
863862
......@@ -956,10 +955,10 @@ const Writer = struct {
956955 op_index: usize,
957956 operand: Air.Inst.Ref,
958957 ) @TypeOf(s).Error!void {
959 const small_tomb_bits = Liveness.bpi - 1;
958 const small_tomb_bits = Air.Liveness.bpi - 1;
960959 const dies = if (w.liveness) |liveness| blk: {
961960 if (op_index < small_tomb_bits)
962 break :blk liveness.operandDies(inst, @as(Liveness.OperandInt, @intCast(op_index)));
961 break :blk liveness.operandDies(inst, @intCast(op_index));
963962 var extra_index = liveness.special.get(inst).?;
964963 var tomb_op_index: usize = small_tomb_bits;
965964 while (true) {