Commit b116063e02bf2bb1975f5ae862fcd25f8fbeda09 | zig

author	Andrew Kelley <andrew@ziglang.org> 2024-02-26 21:51:19-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2024-02-26 21:51:19-07:00
log	b116063e02bf2bb1975f5ae862fcd25f8fbeda09
tree	e6d0c0c0099bef2402ab5ffb770ac6a145a47878
parent	a2e87aba664c622fe368ce7fcbcdc499b9fd9cf9

move AstGen to std.zig.AstGen

Part of an effort to ship more of the compiler in source form.

8 files changed, 13667 insertions(+), 13666 deletions(-)

CMakeLists.txt
lib
- std
  - zig.zig
  - zig
    AstGen.zig
src

CMakeLists.txt+1-1

lib/std/zig.zig+1

...	@@ -10,6 +10,7 @@ pub const string_literal = @import("zig/string_literal.zig");	...	@@ -10,6 +10,7 @@ pub const string_literal = @import("zig/string_literal.zig");
10	pub const number_literal = @import("zig/number_literal.zig");	10	pub const number_literal = @import("zig/number_literal.zig");
11	pub const primitives = @import("zig/primitives.zig");	11	pub const primitives = @import("zig/primitives.zig");
12	pub const Ast = @import("zig/Ast.zig");	12	pub const Ast = @import("zig/Ast.zig");
		13	pub const AstGen = @import("zig/AstGen.zig");
13	pub const Zir = @import("zig/Zir.zig");	14	pub const Zir = @import("zig/Zir.zig");
14	pub const system = @import("zig/system.zig");	15	pub const system = @import("zig/system.zig");
15	/// Deprecated: use `std.Target.Query`.	16	/// Deprecated: use `std.Target.Query`.

lib/std/zig/AstGen.zig created+13661

...	@@ -0,0 +1,13661 @@
1	//! Ingests an AST and produces ZIR code.
2	const AstGen = @This();
3
4	const std = @import("std");
5	const Ast = std.zig.Ast;
6	const mem = std.mem;
7	const Allocator = std.mem.Allocator;
8	const assert = std.debug.assert;
9	const ArrayListUnmanaged = std.ArrayListUnmanaged;
10	const StringIndexAdapter = std.hash_map.StringIndexAdapter;
11	const StringIndexContext = std.hash_map.StringIndexContext;
12
13	const isPrimitive = std.zig.primitives.isPrimitive;
14
15	const Zir = std.zig.Zir;
16	const BuiltinFn = std.zig.BuiltinFn;
17	const AstRlAnnotate = std.zig.AstRlAnnotate;
18
19	gpa: Allocator,
20	tree: *const Ast,
21	/// The set of nodes which, given the choice, must expose a result pointer to
22	/// sub-expressions. See `AstRlAnnotate` for details.
23	nodes_need_rl: *const AstRlAnnotate.RlNeededSet,
24	instructions: std.MultiArrayList(Zir.Inst) = .{},
25	extra: ArrayListUnmanaged(u32) = .{},
26	string_bytes: ArrayListUnmanaged(u8) = .{},
27	/// Tracks the current byte offset within the source file.
28	/// Used to populate line deltas in the ZIR. AstGen maintains
29	/// this "cursor" throughout the entire AST lowering process in order
30	/// to avoid starting over the line/column scan for every declaration, which
31	/// would be O(N^2).
32	source_offset: u32 = 0,
33	/// Tracks the corresponding line of `source_offset`.
34	/// This value is absolute.
35	source_line: u32 = 0,
36	/// Tracks the corresponding column of `source_offset`.
37	/// This value is absolute.
38	source_column: u32 = 0,
39	/// Used for temporary allocations; freed after AstGen is complete.
40	/// The resulting ZIR code has no references to anything in this arena.
41	arena: Allocator,
42	string_table: std.HashMapUnmanaged(u32, void, StringIndexContext, std.hash_map.default_max_load_percentage) = .{},
43	compile_errors: ArrayListUnmanaged(Zir.Inst.CompileErrors.Item) = .{},
44	/// The topmost block of the current function.
45	fn_block: ?*GenZir = null,
46	fn_var_args: bool = false,
47	/// The return type of the current function. This may be a trivial `Ref`, or
48	/// otherwise it refers to a `ret_type` instruction.
49	fn_ret_ty: Zir.Inst.Ref = .none,
50	/// Maps string table indexes to the first `@import` ZIR instruction
51	/// that uses this string as the operand.
52	imports: std.AutoArrayHashMapUnmanaged(Zir.NullTerminatedString, Ast.TokenIndex) = .{},
53	/// Used for temporary storage when building payloads.
54	scratch: std.ArrayListUnmanaged(u32) = .{},
55	/// Whenever a `ref` instruction is needed, it is created and saved in this
56	/// table instead of being immediately appended to the current block body.
57	/// Then, when the instruction is being added to the parent block (typically from
58	/// setBlockBody), if it has a ref_table entry, then the ref instruction is added
59	/// there. This makes sure two properties are upheld:
60	/// 1. All pointers to the same locals return the same address. This is required
61	/// to be compliant with the language specification.
62	/// 2. `ref` instructions will dominate their uses. This is a required property
63	/// of ZIR.
64	/// The key is the ref operand; the value is the ref instruction.
65	ref_table: std.AutoHashMapUnmanaged(Zir.Inst.Index, Zir.Inst.Index) = .{},
66
67	const InnerError = error{ OutOfMemory, AnalysisFail };
68
69	fn addExtra(astgen: *AstGen, extra: anytype) Allocator.Error!u32 {
70	const fields = std.meta.fields(@TypeOf(extra));
71	try astgen.extra.ensureUnusedCapacity(astgen.gpa, fields.len);
72	return addExtraAssumeCapacity(astgen, extra);
73	}
74
75	fn addExtraAssumeCapacity(astgen: *AstGen, extra: anytype) u32 {
76	const fields = std.meta.fields(@TypeOf(extra));
77	const extra_index: u32 = @intCast(astgen.extra.items.len);
78	astgen.extra.items.len += fields.len;
79	setExtra(astgen, extra_index, extra);
80	return extra_index;
81	}
82
83	fn setExtra(astgen: *AstGen, index: usize, extra: anytype) void {
84	const fields = std.meta.fields(@TypeOf(extra));
85	var i = index;
86	inline for (fields) \|field\| {
87	astgen.extra.items[i] = switch (field.type) {
88	u32 => @field(extra, field.name),
89
90	Zir.Inst.Ref,
91	Zir.Inst.Index,
92	Zir.Inst.Declaration.Name,
93	Zir.NullTerminatedString,
94	=> @intFromEnum(@field(extra, field.name)),
95
96	i32,
97	Zir.Inst.Call.Flags,
98	Zir.Inst.BuiltinCall.Flags,
99	Zir.Inst.SwitchBlock.Bits,
100	Zir.Inst.SwitchBlockErrUnion.Bits,
101	Zir.Inst.FuncFancy.Bits,
102	Zir.Inst.Declaration.Flags,
103	=> @bitCast(@field(extra, field.name)),
104
105	else => @compileError("bad field type"),
106	};
107	i += 1;
108	}
109	}
110
111	fn reserveExtra(astgen: *AstGen, size: usize) Allocator.Error!u32 {
112	const extra_index: u32 = @intCast(astgen.extra.items.len);
113	try astgen.extra.resize(astgen.gpa, extra_index + size);
114	return extra_index;
115	}
116
117	fn appendRefs(astgen: *AstGen, refs: []const Zir.Inst.Ref) !void {
118	return astgen.extra.appendSlice(astgen.gpa, @ptrCast(refs));
119	}
120
121	fn appendRefsAssumeCapacity(astgen: *AstGen, refs: []const Zir.Inst.Ref) void {
122	astgen.extra.appendSliceAssumeCapacity(@ptrCast(refs));
123	}
124
125	pub fn generate(gpa: Allocator, tree: Ast) Allocator.Error!Zir {
126	var arena = std.heap.ArenaAllocator.init(gpa);
127	defer arena.deinit();
128
129	var nodes_need_rl = try AstRlAnnotate.annotate(gpa, arena.allocator(), tree);
130	defer nodes_need_rl.deinit(gpa);
131
132	var astgen: AstGen = .{
133	.gpa = gpa,
134	.arena = arena.allocator(),
135	.tree = &tree,
136	.nodes_need_rl = &nodes_need_rl,
137	};
138	defer astgen.deinit(gpa);
139
140	// String table index 0 is reserved for `NullTerminatedString.empty`.
141	try astgen.string_bytes.append(gpa, 0);
142
143	// We expect at least as many ZIR instructions and extra data items
144	// as AST nodes.
145	try astgen.instructions.ensureTotalCapacity(gpa, tree.nodes.len);
146
147	// First few indexes of extra are reserved and set at the end.
148	const reserved_count = @typeInfo(Zir.ExtraIndex).Enum.fields.len;
149	try astgen.extra.ensureTotalCapacity(gpa, tree.nodes.len + reserved_count);
150	astgen.extra.items.len += reserved_count;
151
152	var top_scope: Scope.Top = .{};
153
154	var gz_instructions: std.ArrayListUnmanaged(Zir.Inst.Index) = .{};
155	var gen_scope: GenZir = .{
156	.is_comptime = true,
157	.parent = &top_scope.base,
158	.anon_name_strategy = .parent,
159	.decl_node_index = 0,
160	.decl_line = 0,
161	.astgen = &astgen,
162	.instructions = &gz_instructions,
163	.instructions_top = 0,
164	};
165	defer gz_instructions.deinit(gpa);
166
167	// The AST -> ZIR lowering process assumes an AST that does not have any
168	// parse errors.
169	if (tree.errors.len == 0) {
170	if (AstGen.structDeclInner(
171	&gen_scope,
172	&gen_scope.base,
173	0,
174	tree.containerDeclRoot(),
175	.Auto,
176	0,
177	)) \|struct_decl_ref\| {
178	assert(struct_decl_ref.toIndex().? == .main_struct_inst);
179	} else \|err\| switch (err) {
180	error.OutOfMemory => return error.OutOfMemory,
181	error.AnalysisFail => {}, // Handled via compile_errors below.
182	}
183	} else {
184	try lowerAstErrors(&astgen);
185	}
186
187	const err_index = @intFromEnum(Zir.ExtraIndex.compile_errors);
188	if (astgen.compile_errors.items.len == 0) {
189	astgen.extra.items[err_index] = 0;
190	} else {
191	try astgen.extra.ensureUnusedCapacity(gpa, 1 + astgen.compile_errors.items.len *
192	@typeInfo(Zir.Inst.CompileErrors.Item).Struct.fields.len);
193
194	astgen.extra.items[err_index] = astgen.addExtraAssumeCapacity(Zir.Inst.CompileErrors{
195	.items_len = @intCast(astgen.compile_errors.items.len),
196	});
197
198	for (astgen.compile_errors.items) \|item\| {
199	_ = astgen.addExtraAssumeCapacity(item);
200	}
201	}
202
203	const imports_index = @intFromEnum(Zir.ExtraIndex.imports);
204	if (astgen.imports.count() == 0) {
205	astgen.extra.items[imports_index] = 0;
206	} else {
207	try astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.Imports).Struct.fields.len +
208	astgen.imports.count() * @typeInfo(Zir.Inst.Imports.Item).Struct.fields.len);
209
210	astgen.extra.items[imports_index] = astgen.addExtraAssumeCapacity(Zir.Inst.Imports{
211	.imports_len = @intCast(astgen.imports.count()),
212	});
213
214	var it = astgen.imports.iterator();
215	while (it.next()) \|entry\| {
216	_ = astgen.addExtraAssumeCapacity(Zir.Inst.Imports.Item{
217	.name = entry.key_ptr.*,
218	.token = entry.value_ptr.*,
219	});
220	}
221	}
222
223	return Zir{
224	.instructions = astgen.instructions.toOwnedSlice(),
225	.string_bytes = try astgen.string_bytes.toOwnedSlice(gpa),
226	.extra = try astgen.extra.toOwnedSlice(gpa),
227	};
228	}
229
230	fn deinit(astgen: *AstGen, gpa: Allocator) void {
231	astgen.instructions.deinit(gpa);
232	astgen.extra.deinit(gpa);
233	astgen.string_table.deinit(gpa);
234	astgen.string_bytes.deinit(gpa);
235	astgen.compile_errors.deinit(gpa);
236	astgen.imports.deinit(gpa);
237	astgen.scratch.deinit(gpa);
238	astgen.ref_table.deinit(gpa);
239	}
240
241	const ResultInfo = struct {
242	/// The semantics requested for the result location
243	rl: Loc,
244
245	/// The "operator" consuming the result location
246	ctx: Context = .none,
247
248	/// Turns a `coerced_ty` back into a `ty`. Should be called at branch points
249	/// such as if and switch expressions.
250	fn br(ri: ResultInfo) ResultInfo {
251	return switch (ri.rl) {
252	.coerced_ty => \|ty\| .{
253	.rl = .{ .ty = ty },
254	.ctx = ri.ctx,
255	},
256	else => ri,
257	};
258	}
259
260	fn zirTag(ri: ResultInfo) Zir.Inst.Tag {
261	switch (ri.rl) {
262	.ty => return switch (ri.ctx) {
263	.shift_op => .as_shift_operand,
264	else => .as_node,
265	},
266	else => unreachable,
267	}
268	}
269
270	const Loc = union(enum) {
271	/// The expression is the right-hand side of assignment to `_`. Only the side-effects of the
272	/// expression should be generated. The result instruction from the expression must
273	/// be ignored.
274	discard,
275	/// The expression has an inferred type, and it will be evaluated as an rvalue.
276	none,
277	/// The expression will be coerced into this type, but it will be evaluated as an rvalue.
278	ty: Zir.Inst.Ref,
279	/// Same as `ty` but it is guaranteed that Sema will additionally perform the coercion,
280	/// so no `as` instruction needs to be emitted.
281	coerced_ty: Zir.Inst.Ref,
282	/// The expression must generate a pointer rather than a value. For example, the left hand side
283	/// of an assignment uses this kind of result location.
284	ref,
285	/// The expression must generate a pointer rather than a value, and the pointer will be coerced
286	/// by other code to this type, which is guaranteed by earlier instructions to be a pointer type.
287	ref_coerced_ty: Zir.Inst.Ref,
288	/// The expression must store its result into this typed pointer. The result instruction
289	/// from the expression must be ignored.
290	ptr: PtrResultLoc,
291	/// The expression must store its result into this allocation, which has an inferred type.
292	/// The result instruction from the expression must be ignored.
293	/// Always an instruction with tag `alloc_inferred`.
294	inferred_ptr: Zir.Inst.Ref,
295	/// The expression has a sequence of pointers to store its results into due to a destructure
296	/// operation. Each of these pointers may or may not have an inferred type.
297	destructure: struct {
298	/// The AST node of the destructure operation itself.
299	src_node: Ast.Node.Index,
300	/// The pointers to store results into.
301	components: []const DestructureComponent,
302	},
303
304	const DestructureComponent = union(enum) {
305	typed_ptr: PtrResultLoc,
306	inferred_ptr: Zir.Inst.Ref,
307	discard,
308	};
309
310	const PtrResultLoc = struct {
311	inst: Zir.Inst.Ref,
312	src_node: ?Ast.Node.Index = null,
313	};
314
315	/// Find the result type for a cast builtin given the result location.
316	/// If the location does not have a known result type, emits an error on
317	/// the given node.
318	fn resultType(rl: Loc, gz: *GenZir, node: Ast.Node.Index) !?Zir.Inst.Ref {
319	return switch (rl) {
320	.discard, .none, .ref, .inferred_ptr, .destructure => null,
321	.ty, .coerced_ty => \|ty_ref\| ty_ref,
322	.ref_coerced_ty => \|ptr_ty\| try gz.addUnNode(.elem_type, ptr_ty, node),
323	.ptr => \|ptr\| {
324	const ptr_ty = try gz.addUnNode(.typeof, ptr.inst, node);
325	return try gz.addUnNode(.elem_type, ptr_ty, node);
326	},
327	};
328	}
329
330	fn resultTypeForCast(rl: Loc, gz: *GenZir, node: Ast.Node.Index, builtin_name: []const u8) !Zir.Inst.Ref {
331	const astgen = gz.astgen;
332	if (try rl.resultType(gz, node)) \|ty\| return ty;
333	switch (rl) {
334	.destructure => \|destructure\| return astgen.failNodeNotes(node, "{s} must have a known result type", .{builtin_name}, &.{
335	try astgen.errNoteNode(destructure.src_node, "destructure expressions do not provide a single result type", .{}),
336	try astgen.errNoteNode(node, "use @as to provide explicit result type", .{}),
337	}),
338	else => return astgen.failNodeNotes(node, "{s} must have a known result type", .{builtin_name}, &.{
339	try astgen.errNoteNode(node, "use @as to provide explicit result type", .{}),
340	}),
341	}
342	}
343	};
344
345	const Context = enum {
346	/// The expression is the operand to a return expression.
347	@"return",
348	/// The expression is the input to an error-handling operator (if-else, try, or catch).
349	error_handling_expr,
350	/// The expression is the right-hand side of a shift operation.
351	shift_op,
352	/// The expression is an argument in a function call.
353	fn_arg,
354	/// The expression is the right-hand side of an initializer for a `const` variable
355	const_init,
356	/// The expression is the right-hand side of an assignment expression.
357	assignment,
358	/// No specific operator in particular.
359	none,
360	};
361	};
362
363	const coerced_align_ri: ResultInfo = .{ .rl = .{ .coerced_ty = .u29_type } };
364	const coerced_addrspace_ri: ResultInfo = .{ .rl = .{ .coerced_ty = .address_space_type } };
365	const coerced_linksection_ri: ResultInfo = .{ .rl = .{ .coerced_ty = .slice_const_u8_type } };
366	const coerced_type_ri: ResultInfo = .{ .rl = .{ .coerced_ty = .type_type } };
367	const coerced_bool_ri: ResultInfo = .{ .rl = .{ .coerced_ty = .bool_type } };
368
369	fn typeExpr(gz: GenZir, scope: Scope, type_node: Ast.Node.Index) InnerError!Zir.Inst.Ref {
370	return comptimeExpr(gz, scope, coerced_type_ri, type_node);
371	}
372
373	fn reachableTypeExpr(
374	gz: *GenZir,
375	scope: *Scope,
376	type_node: Ast.Node.Index,
377	reachable_node: Ast.Node.Index,
378	) InnerError!Zir.Inst.Ref {
379	return reachableExprComptime(gz, scope, coerced_type_ri, type_node, reachable_node, true);
380	}
381
382	/// Same as `expr` but fails with a compile error if the result type is `noreturn`.
383	fn reachableExpr(
384	gz: *GenZir,
385	scope: *Scope,
386	ri: ResultInfo,
387	node: Ast.Node.Index,
388	reachable_node: Ast.Node.Index,
389	) InnerError!Zir.Inst.Ref {
390	return reachableExprComptime(gz, scope, ri, node, reachable_node, false);
391	}
392
393	fn reachableExprComptime(
394	gz: *GenZir,
395	scope: *Scope,
396	ri: ResultInfo,
397	node: Ast.Node.Index,
398	reachable_node: Ast.Node.Index,
399	force_comptime: bool,
400	) InnerError!Zir.Inst.Ref {
401	const result_inst = if (force_comptime)
402	try comptimeExpr(gz, scope, ri, node)
403	else
404	try expr(gz, scope, ri, node);
405
406	if (gz.refIsNoReturn(result_inst)) {
407	try gz.astgen.appendErrorNodeNotes(reachable_node, "unreachable code", .{}, &[_]u32{
408	try gz.astgen.errNoteNode(node, "control flow is diverted here", .{}),
409	});
410	}
411	return result_inst;
412	}
413
414	fn lvalExpr(gz: GenZir, scope: Scope, node: Ast.Node.Index) InnerError!Zir.Inst.Ref {
415	const astgen = gz.astgen;
416	const tree = astgen.tree;
417	const node_tags = tree.nodes.items(.tag);
418	const main_tokens = tree.nodes.items(.main_token);
419	switch (node_tags[node]) {
420	.root => unreachable,
421	.@"usingnamespace" => unreachable,
422	.test_decl => unreachable,
423	.global_var_decl => unreachable,
424	.local_var_decl => unreachable,
425	.simple_var_decl => unreachable,
426	.aligned_var_decl => unreachable,
427	.switch_case => unreachable,
428	.switch_case_inline => unreachable,
429	.switch_case_one => unreachable,
430	.switch_case_inline_one => unreachable,
431	.container_field_init => unreachable,
432	.container_field_align => unreachable,
433	.container_field => unreachable,
434	.asm_output => unreachable,
435	.asm_input => unreachable,
436
437	.assign,
438	.assign_destructure,
439	.assign_bit_and,
440	.assign_bit_or,
441	.assign_shl,
442	.assign_shl_sat,
443	.assign_shr,
444	.assign_bit_xor,
445	.assign_div,
446	.assign_sub,
447	.assign_sub_wrap,
448	.assign_sub_sat,
449	.assign_mod,
450	.assign_add,
451	.assign_add_wrap,
452	.assign_add_sat,
453	.assign_mul,
454	.assign_mul_wrap,
455	.assign_mul_sat,
456	.add,
457	.add_wrap,
458	.add_sat,
459	.sub,
460	.sub_wrap,
461	.sub_sat,
462	.mul,
463	.mul_wrap,
464	.mul_sat,
465	.div,
466	.mod,
467	.bit_and,
468	.bit_or,
469	.shl,
470	.shl_sat,
471	.shr,
472	.bit_xor,
473	.bang_equal,
474	.equal_equal,
475	.greater_than,
476	.greater_or_equal,
477	.less_than,
478	.less_or_equal,
479	.array_cat,
480	.array_mult,
481	.bool_and,
482	.bool_or,
483	.@"asm",
484	.asm_simple,
485	.string_literal,
486	.number_literal,
487	.call,
488	.call_comma,
489	.async_call,
490	.async_call_comma,
491	.call_one,
492	.call_one_comma,
493	.async_call_one,
494	.async_call_one_comma,
495	.unreachable_literal,
496	.@"return",
497	.@"if",
498	.if_simple,
499	.@"while",
500	.while_simple,
501	.while_cont,
502	.bool_not,
503	.address_of,
504	.optional_type,
505	.block,
506	.block_semicolon,
507	.block_two,
508	.block_two_semicolon,
509	.@"break",
510	.ptr_type_aligned,
511	.ptr_type_sentinel,
512	.ptr_type,
513	.ptr_type_bit_range,
514	.array_type,
515	.array_type_sentinel,
516	.enum_literal,
517	.multiline_string_literal,
518	.char_literal,
519	.@"defer",
520	.@"errdefer",
521	.@"catch",
522	.error_union,
523	.merge_error_sets,
524	.switch_range,
525	.for_range,
526	.@"await",
527	.bit_not,
528	.negation,
529	.negation_wrap,
530	.@"resume",
531	.@"try",
532	.slice,
533	.slice_open,
534	.slice_sentinel,
535	.array_init_one,
536	.array_init_one_comma,
537	.array_init_dot_two,
538	.array_init_dot_two_comma,
539	.array_init_dot,
540	.array_init_dot_comma,
541	.array_init,
542	.array_init_comma,
543	.struct_init_one,
544	.struct_init_one_comma,
545	.struct_init_dot_two,
546	.struct_init_dot_two_comma,
547	.struct_init_dot,
548	.struct_init_dot_comma,
549	.struct_init,
550	.struct_init_comma,
551	.@"switch",
552	.switch_comma,
553	.@"for",
554	.for_simple,
555	.@"suspend",
556	.@"continue",
557	.fn_proto_simple,
558	.fn_proto_multi,
559	.fn_proto_one,
560	.fn_proto,
561	.fn_decl,
562	.anyframe_type,
563	.anyframe_literal,
564	.error_set_decl,
565	.container_decl,
566	.container_decl_trailing,
567	.container_decl_two,
568	.container_decl_two_trailing,
569	.container_decl_arg,
570	.container_decl_arg_trailing,
571	.tagged_union,
572	.tagged_union_trailing,
573	.tagged_union_two,
574	.tagged_union_two_trailing,
575	.tagged_union_enum_tag,
576	.tagged_union_enum_tag_trailing,
577	.@"comptime",
578	.@"nosuspend",
579	.error_value,
580	=> return astgen.failNode(node, "invalid left-hand side to assignment", .{}),
581
582	.builtin_call,
583	.builtin_call_comma,
584	.builtin_call_two,
585	.builtin_call_two_comma,
586	=> {
587	const builtin_token = main_tokens[node];
588	const builtin_name = tree.tokenSlice(builtin_token);
589	// If the builtin is an invalid name, we don't cause an error here; instead
590	// let it pass, and the error will be "invalid builtin function" later.
591	if (BuiltinFn.list.get(builtin_name)) \|info\| {
592	if (!info.allows_lvalue) {
593	return astgen.failNode(node, "invalid left-hand side to assignment", .{});
594	}
595	}
596	},
597
598	// These can be assigned to.
599	.unwrap_optional,
600	.deref,
601	.field_access,
602	.array_access,
603	.identifier,
604	.grouped_expression,
605	.@"orelse",
606	=> {},
607	}
608	return expr(gz, scope, .{ .rl = .ref }, node);
609	}
610
611	/// Turn Zig AST into untyped ZIR instructions.
612	/// When `rl` is discard, ptr, inferred_ptr, or inferred_ptr, the
613	/// result instruction can be used to inspect whether it is isNoReturn() but that is it,
614	/// it must otherwise not be used.
615	fn expr(gz: GenZir, scope: Scope, ri: ResultInfo, node: Ast.Node.Index) InnerError!Zir.Inst.Ref {
616	const astgen = gz.astgen;
617	const tree = astgen.tree;
618	const main_tokens = tree.nodes.items(.main_token);
619	const token_tags = tree.tokens.items(.tag);
620	const node_datas = tree.nodes.items(.data);
621	const node_tags = tree.nodes.items(.tag);
622
623	const prev_anon_name_strategy = gz.anon_name_strategy;
624	defer gz.anon_name_strategy = prev_anon_name_strategy;
625	if (!nodeUsesAnonNameStrategy(tree, node)) {
626	gz.anon_name_strategy = .anon;
627	}
628
629	switch (node_tags[node]) {
630	.root => unreachable, // Top-level declaration.
631	.@"usingnamespace" => unreachable, // Top-level declaration.
632	.test_decl => unreachable, // Top-level declaration.
633	.container_field_init => unreachable, // Top-level declaration.
634	.container_field_align => unreachable, // Top-level declaration.
635	.container_field => unreachable, // Top-level declaration.
636	.fn_decl => unreachable, // Top-level declaration.
637
638	.global_var_decl => unreachable, // Handled in `blockExpr`.
639	.local_var_decl => unreachable, // Handled in `blockExpr`.
640	.simple_var_decl => unreachable, // Handled in `blockExpr`.
641	.aligned_var_decl => unreachable, // Handled in `blockExpr`.
642	.@"defer" => unreachable, // Handled in `blockExpr`.
643	.@"errdefer" => unreachable, // Handled in `blockExpr`.
644
645	.switch_case => unreachable, // Handled in `switchExpr`.
646	.switch_case_inline => unreachable, // Handled in `switchExpr`.
647	.switch_case_one => unreachable, // Handled in `switchExpr`.
648	.switch_case_inline_one => unreachable, // Handled in `switchExpr`.
649	.switch_range => unreachable, // Handled in `switchExpr`.
650
651	.asm_output => unreachable, // Handled in `asmExpr`.
652	.asm_input => unreachable, // Handled in `asmExpr`.
653
654	.for_range => unreachable, // Handled in `forExpr`.
655
656	.assign => {
657	try assign(gz, scope, node);
658	return rvalue(gz, ri, .void_value, node);
659	},
660
661	.assign_destructure => {
662	// Note that this variant does not declare any new var/const: that
663	// variant is handled by `blockExprStmts`.
664	try assignDestructure(gz, scope, node);
665	return rvalue(gz, ri, .void_value, node);
666	},
667
668	.assign_shl => {
669	try assignShift(gz, scope, node, .shl);
670	return rvalue(gz, ri, .void_value, node);
671	},
672	.assign_shl_sat => {
673	try assignShiftSat(gz, scope, node);
674	return rvalue(gz, ri, .void_value, node);
675	},
676	.assign_shr => {
677	try assignShift(gz, scope, node, .shr);
678	return rvalue(gz, ri, .void_value, node);
679	},
680
681	.assign_bit_and => {
682	try assignOp(gz, scope, node, .bit_and);
683	return rvalue(gz, ri, .void_value, node);
684	},
685	.assign_bit_or => {
686	try assignOp(gz, scope, node, .bit_or);
687	return rvalue(gz, ri, .void_value, node);
688	},
689	.assign_bit_xor => {
690	try assignOp(gz, scope, node, .xor);
691	return rvalue(gz, ri, .void_value, node);
692	},
693	.assign_div => {
694	try assignOp(gz, scope, node, .div);
695	return rvalue(gz, ri, .void_value, node);
696	},
697	.assign_sub => {
698	try assignOp(gz, scope, node, .sub);
699	return rvalue(gz, ri, .void_value, node);
700	},
701	.assign_sub_wrap => {
702	try assignOp(gz, scope, node, .subwrap);
703	return rvalue(gz, ri, .void_value, node);
704	},
705	.assign_sub_sat => {
706	try assignOp(gz, scope, node, .sub_sat);
707	return rvalue(gz, ri, .void_value, node);
708	},
709	.assign_mod => {
710	try assignOp(gz, scope, node, .mod_rem);
711	return rvalue(gz, ri, .void_value, node);
712	},
713	.assign_add => {
714	try assignOp(gz, scope, node, .add);
715	return rvalue(gz, ri, .void_value, node);
716	},
717	.assign_add_wrap => {
718	try assignOp(gz, scope, node, .addwrap);
719	return rvalue(gz, ri, .void_value, node);
720	},
721	.assign_add_sat => {
722	try assignOp(gz, scope, node, .add_sat);
723	return rvalue(gz, ri, .void_value, node);
724	},
725	.assign_mul => {
726	try assignOp(gz, scope, node, .mul);
727	return rvalue(gz, ri, .void_value, node);
728	},
729	.assign_mul_wrap => {
730	try assignOp(gz, scope, node, .mulwrap);
731	return rvalue(gz, ri, .void_value, node);
732	},
733	.assign_mul_sat => {
734	try assignOp(gz, scope, node, .mul_sat);
735	return rvalue(gz, ri, .void_value, node);
736	},
737
738	// zig fmt: off
739	.shl => return shiftOp(gz, scope, ri, node, node_datas[node].lhs, node_datas[node].rhs, .shl),
740	.shr => return shiftOp(gz, scope, ri, node, node_datas[node].lhs, node_datas[node].rhs, .shr),
741
742	.add => return simpleBinOp(gz, scope, ri, node, .add),
743	.add_wrap => return simpleBinOp(gz, scope, ri, node, .addwrap),
744	.add_sat => return simpleBinOp(gz, scope, ri, node, .add_sat),
745	.sub => return simpleBinOp(gz, scope, ri, node, .sub),
746	.sub_wrap => return simpleBinOp(gz, scope, ri, node, .subwrap),
747	.sub_sat => return simpleBinOp(gz, scope, ri, node, .sub_sat),
748	.mul => return simpleBinOp(gz, scope, ri, node, .mul),
749	.mul_wrap => return simpleBinOp(gz, scope, ri, node, .mulwrap),
750	.mul_sat => return simpleBinOp(gz, scope, ri, node, .mul_sat),
751	.div => return simpleBinOp(gz, scope, ri, node, .div),
752	.mod => return simpleBinOp(gz, scope, ri, node, .mod_rem),
753	.shl_sat => return simpleBinOp(gz, scope, ri, node, .shl_sat),
754
755	.bit_and => return simpleBinOp(gz, scope, ri, node, .bit_and),
756	.bit_or => return simpleBinOp(gz, scope, ri, node, .bit_or),
757	.bit_xor => return simpleBinOp(gz, scope, ri, node, .xor),
758	.bang_equal => return simpleBinOp(gz, scope, ri, node, .cmp_neq),
759	.equal_equal => return simpleBinOp(gz, scope, ri, node, .cmp_eq),
760	.greater_than => return simpleBinOp(gz, scope, ri, node, .cmp_gt),
761	.greater_or_equal => return simpleBinOp(gz, scope, ri, node, .cmp_gte),
762	.less_than => return simpleBinOp(gz, scope, ri, node, .cmp_lt),
763	.less_or_equal => return simpleBinOp(gz, scope, ri, node, .cmp_lte),
764	.array_cat => return simpleBinOp(gz, scope, ri, node, .array_cat),
765
766	.array_mult => {
767	// This syntax form does not currently use the result type in the language specification.
768	// However, the result type can be used to emit more optimal code for large multiplications by
769	// having Sema perform a coercion before the multiplication operation.
770	const result = try gz.addPlNode(.array_mul, node, Zir.Inst.ArrayMul{
771	.res_ty = if (try ri.rl.resultType(gz, node)) \|t\| t else .none,
772	.lhs = try expr(gz, scope, .{ .rl = .none }, node_datas[node].lhs),
773	.rhs = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, node_datas[node].rhs),
774	});
775	return rvalue(gz, ri, result, node);
776	},
777
778	.error_union => return simpleBinOp(gz, scope, ri, node, .error_union_type),
779	.merge_error_sets => return simpleBinOp(gz, scope, ri, node, .merge_error_sets),
780
781	.bool_and => return boolBinOp(gz, scope, ri, node, .bool_br_and),
782	.bool_or => return boolBinOp(gz, scope, ri, node, .bool_br_or),
783
784	.bool_not => return simpleUnOp(gz, scope, ri, node, coerced_bool_ri, node_datas[node].lhs, .bool_not),
785	.bit_not => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, node_datas[node].lhs, .bit_not),
786
787	.negation => return negation(gz, scope, ri, node),
788	.negation_wrap => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, node_datas[node].lhs, .negate_wrap),
789
790	.identifier => return identifier(gz, scope, ri, node),
791
792	.asm_simple,
793	.@"asm",
794	=> return asmExpr(gz, scope, ri, node, tree.fullAsm(node).?),
795
796	.string_literal => return stringLiteral(gz, ri, node),
797	.multiline_string_literal => return multilineStringLiteral(gz, ri, node),
798
799	.number_literal => return numberLiteral(gz, ri, node, node, .positive),
800	// zig fmt: on
801
802	.builtin_call_two, .builtin_call_two_comma => {
803	if (node_datas[node].lhs == 0) {
804	const params = [_]Ast.Node.Index{};
805	return builtinCall(gz, scope, ri, node, &params);
806	} else if (node_datas[node].rhs == 0) {
807	const params = [_]Ast.Node.Index{node_datas[node].lhs};
808	return builtinCall(gz, scope, ri, node, &params);
809	} else {
810	const params = [_]Ast.Node.Index{ node_datas[node].lhs, node_datas[node].rhs };
811	return builtinCall(gz, scope, ri, node, &params);
812	}
813	},
814	.builtin_call, .builtin_call_comma => {
815	const params = tree.extra_data[node_datas[node].lhs..node_datas[node].rhs];
816	return builtinCall(gz, scope, ri, node, params);
817	},
818
819	.call_one,
820	.call_one_comma,
821	.async_call_one,
822	.async_call_one_comma,
823	.call,
824	.call_comma,
825	.async_call,
826	.async_call_comma,
827	=> {
828	var buf: [1]Ast.Node.Index = undefined;
829	return callExpr(gz, scope, ri, node, tree.fullCall(&buf, node).?);
830	},
831
832	.unreachable_literal => {
833	try emitDbgNode(gz, node);
834	_ = try gz.addAsIndex(.{
835	.tag = .@"unreachable",
836	.data = .{ .@"unreachable" = .{
837	.src_node = gz.nodeIndexToRelative(node),
838	} },
839	});
840	return Zir.Inst.Ref.unreachable_value;
841	},
842	.@"return" => return ret(gz, scope, node),
843	.field_access => return fieldAccess(gz, scope, ri, node),
844
845	.if_simple,
846	.@"if",
847	=> {
848	const if_full = tree.fullIf(node).?;
849	no_switch_on_err: {
850	const error_token = if_full.error_token orelse break :no_switch_on_err;
851	switch (node_tags[if_full.ast.else_expr]) {
852	.@"switch", .switch_comma => {},
853	else => break :no_switch_on_err,
854	}
855	const switch_operand = node_datas[if_full.ast.else_expr].lhs;
856	if (node_tags[switch_operand] != .identifier) break :no_switch_on_err;
857	if (!mem.eql(u8, tree.tokenSlice(error_token), tree.tokenSlice(main_tokens[switch_operand]))) break :no_switch_on_err;
858	return switchExprErrUnion(gz, scope, ri.br(), node, .@"if");
859	}
860	return ifExpr(gz, scope, ri.br(), node, if_full);
861	},
862
863	.while_simple,
864	.while_cont,
865	.@"while",
866	=> return whileExpr(gz, scope, ri.br(), node, tree.fullWhile(node).?, false),
867
868	.for_simple, .@"for" => return forExpr(gz, scope, ri.br(), node, tree.fullFor(node).?, false),
869
870	.slice_open => {
871	const lhs = try expr(gz, scope, .{ .rl = .ref }, node_datas[node].lhs);
872
873	const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
874	const start = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, node_datas[node].rhs);
875	try emitDbgStmt(gz, cursor);
876	const result = try gz.addPlNode(.slice_start, node, Zir.Inst.SliceStart{
877	.lhs = lhs,
878	.start = start,
879	});
880	return rvalue(gz, ri, result, node);
881	},
882	.slice => {
883	const extra = tree.extraData(node_datas[node].rhs, Ast.Node.Slice);
884	const lhs_node = node_datas[node].lhs;
885	const lhs_tag = node_tags[lhs_node];
886	const lhs_is_slice_sentinel = lhs_tag == .slice_sentinel;
887	const lhs_is_open_slice = lhs_tag == .slice_open or
888	(lhs_is_slice_sentinel and tree.extraData(node_datas[lhs_node].rhs, Ast.Node.SliceSentinel).end == 0);
889	if (lhs_is_open_slice and nodeIsTriviallyZero(tree, extra.start)) {
890	const lhs = try expr(gz, scope, .{ .rl = .ref }, node_datas[lhs_node].lhs);
891
892	const start = if (lhs_is_slice_sentinel) start: {
893	const lhs_extra = tree.extraData(node_datas[lhs_node].rhs, Ast.Node.SliceSentinel);
894	break :start try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, lhs_extra.start);
895	} else try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, node_datas[lhs_node].rhs);
896
897	const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
898	const len = if (extra.end != 0) try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, extra.end) else .none;
899	try emitDbgStmt(gz, cursor);
900	const result = try gz.addPlNode(.slice_length, node, Zir.Inst.SliceLength{
901	.lhs = lhs,
902	.start = start,
903	.len = len,
904	.start_src_node_offset = gz.nodeIndexToRelative(lhs_node),
905	.sentinel = .none,
906	});
907	return rvalue(gz, ri, result, node);
908	}
909	const lhs = try expr(gz, scope, .{ .rl = .ref }, node_datas[node].lhs);
910
911	const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
912	const start = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, extra.start);
913	const end = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, extra.end);
914	try emitDbgStmt(gz, cursor);
915	const result = try gz.addPlNode(.slice_end, node, Zir.Inst.SliceEnd{
916	.lhs = lhs,
917	.start = start,
918	.end = end,
919	});
920	return rvalue(gz, ri, result, node);
921	},
922	.slice_sentinel => {
923	const extra = tree.extraData(node_datas[node].rhs, Ast.Node.SliceSentinel);
924	const lhs_node = node_datas[node].lhs;
925	const lhs_tag = node_tags[lhs_node];
926	const lhs_is_slice_sentinel = lhs_tag == .slice_sentinel;
927	const lhs_is_open_slice = lhs_tag == .slice_open or
928	(lhs_is_slice_sentinel and tree.extraData(node_datas[lhs_node].rhs, Ast.Node.SliceSentinel).end == 0);
929	if (lhs_is_open_slice and nodeIsTriviallyZero(tree, extra.start)) {
930	const lhs = try expr(gz, scope, .{ .rl = .ref }, node_datas[lhs_node].lhs);
931
932	const start = if (lhs_is_slice_sentinel) start: {
933	const lhs_extra = tree.extraData(node_datas[lhs_node].rhs, Ast.Node.SliceSentinel);
934	break :start try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, lhs_extra.start);
935	} else try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, node_datas[lhs_node].rhs);
936
937	const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
938	const len = if (extra.end != 0) try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, extra.end) else .none;
939	const sentinel = try expr(gz, scope, .{ .rl = .none }, extra.sentinel);
940	try emitDbgStmt(gz, cursor);
941	const result = try gz.addPlNode(.slice_length, node, Zir.Inst.SliceLength{
942	.lhs = lhs,
943	.start = start,
944	.len = len,
945	.start_src_node_offset = gz.nodeIndexToRelative(lhs_node),
946	.sentinel = sentinel,
947	});
948	return rvalue(gz, ri, result, node);
949	}
950	const lhs = try expr(gz, scope, .{ .rl = .ref }, node_datas[node].lhs);
951
952	const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
953	const start = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, extra.start);
954	const end = if (extra.end != 0) try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, extra.end) else .none;
955	const sentinel = try expr(gz, scope, .{ .rl = .none }, extra.sentinel);
956	try emitDbgStmt(gz, cursor);
957	const result = try gz.addPlNode(.slice_sentinel, node, Zir.Inst.SliceSentinel{
958	.lhs = lhs,
959	.start = start,
960	.end = end,
961	.sentinel = sentinel,
962	});
963	return rvalue(gz, ri, result, node);
964	},
965
966	.deref => {
967	const lhs = try expr(gz, scope, .{ .rl = .none }, node_datas[node].lhs);
968	_ = try gz.addUnNode(.validate_deref, lhs, node);
969	switch (ri.rl) {
970	.ref, .ref_coerced_ty => return lhs,
971	else => {
972	const result = try gz.addUnNode(.load, lhs, node);
973	return rvalue(gz, ri, result, node);
974	},
975	}
976	},
977	.address_of => {
978	const operand_rl: ResultInfo.Loc = if (try ri.rl.resultType(gz, node)) \|res_ty_inst\| rl: {
979	_ = try gz.addUnTok(.validate_ref_ty, res_ty_inst, tree.firstToken(node));
980	break :rl .{ .ref_coerced_ty = res_ty_inst };
981	} else .ref;
982	const result = try expr(gz, scope, .{ .rl = operand_rl }, node_datas[node].lhs);
983	return rvalue(gz, ri, result, node);
984	},
985	.optional_type => {
986	const operand = try typeExpr(gz, scope, node_datas[node].lhs);
987	const result = try gz.addUnNode(.optional_type, operand, node);
988	return rvalue(gz, ri, result, node);
989	},
990	.unwrap_optional => switch (ri.rl) {
991	.ref, .ref_coerced_ty => {
992	const lhs = try expr(gz, scope, .{ .rl = .ref }, node_datas[node].lhs);
993
994	const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
995	try emitDbgStmt(gz, cursor);
996
997	return gz.addUnNode(.optional_payload_safe_ptr, lhs, node);
998	},
999	else => {
1000	const lhs = try expr(gz, scope, .{ .rl = .none }, node_datas[node].lhs);
1001
1002	const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
1003	try emitDbgStmt(gz, cursor);
1004
1005	return rvalue(gz, ri, try gz.addUnNode(.optional_payload_safe, lhs, node), node);
1006	},
1007	},
1008	.block_two, .block_two_semicolon => {
1009	const statements = [2]Ast.Node.Index{ node_datas[node].lhs, node_datas[node].rhs };
1010	if (node_datas[node].lhs == 0) {
1011	return blockExpr(gz, scope, ri, node, statements[0..0]);
1012	} else if (node_datas[node].rhs == 0) {
1013	return blockExpr(gz, scope, ri, node, statements[0..1]);
1014	} else {
1015	return blockExpr(gz, scope, ri, node, statements[0..2]);
1016	}
1017	},
1018	.block, .block_semicolon => {
1019	const statements = tree.extra_data[node_datas[node].lhs..node_datas[node].rhs];
1020	return blockExpr(gz, scope, ri, node, statements);
1021	},
1022	.enum_literal => return simpleStrTok(gz, ri, main_tokens[node], node, .enum_literal),
1023	.error_value => return simpleStrTok(gz, ri, node_datas[node].rhs, node, .error_value),
1024	// TODO restore this when implementing https://github.com/ziglang/zig/issues/6025
1025	// .anyframe_literal => return rvalue(gz, ri, .anyframe_type, node),
1026	.anyframe_literal => {
1027	const result = try gz.addUnNode(.anyframe_type, .void_type, node);
1028	return rvalue(gz, ri, result, node);
1029	},
1030	.anyframe_type => {
1031	const return_type = try typeExpr(gz, scope, node_datas[node].rhs);
1032	const result = try gz.addUnNode(.anyframe_type, return_type, node);
1033	return rvalue(gz, ri, result, node);
1034	},
1035	.@"catch" => {
1036	const catch_token = main_tokens[node];
1037	const payload_token: ?Ast.TokenIndex = if (token_tags[catch_token + 1] == .pipe)
1038	catch_token + 2
1039	else
1040	null;
1041	no_switch_on_err: {
1042	const capture_token = payload_token orelse break :no_switch_on_err;
1043	switch (node_tags[node_datas[node].rhs]) {
1044	.@"switch", .switch_comma => {},
1045	else => break :no_switch_on_err,
1046	}
1047	const switch_operand = node_datas[node_datas[node].rhs].lhs;
1048	if (node_tags[switch_operand] != .identifier) break :no_switch_on_err;
1049	if (!mem.eql(u8, tree.tokenSlice(capture_token), tree.tokenSlice(main_tokens[switch_operand]))) break :no_switch_on_err;
1050	return switchExprErrUnion(gz, scope, ri.br(), node, .@"catch");
1051	}
1052	switch (ri.rl) {
1053	.ref, .ref_coerced_ty => return orelseCatchExpr(
1054	gz,
1055	scope,
1056	ri,
1057	node,
1058	node_datas[node].lhs,
1059	.is_non_err_ptr,
1060	.err_union_payload_unsafe_ptr,
1061	.err_union_code_ptr,
1062	node_datas[node].rhs,
1063	payload_token,
1064	),
1065	else => return orelseCatchExpr(
1066	gz,
1067	scope,
1068	ri,
1069	node,
1070	node_datas[node].lhs,
1071	.is_non_err,
1072	.err_union_payload_unsafe,
1073	.err_union_code,
1074	node_datas[node].rhs,
1075	payload_token,
1076	),
1077	}
1078	},
1079	.@"orelse" => switch (ri.rl) {
1080	.ref, .ref_coerced_ty => return orelseCatchExpr(
1081	gz,
1082	scope,
1083	ri,
1084	node,
1085	node_datas[node].lhs,
1086	.is_non_null_ptr,
1087	.optional_payload_unsafe_ptr,
1088	undefined,
1089	node_datas[node].rhs,
1090	null,
1091	),
1092	else => return orelseCatchExpr(
1093	gz,
1094	scope,
1095	ri,
1096	node,
1097	node_datas[node].lhs,
1098	.is_non_null,
1099	.optional_payload_unsafe,
1100	undefined,
1101	node_datas[node].rhs,
1102	null,
1103	),
1104	},
1105
1106	.ptr_type_aligned,
1107	.ptr_type_sentinel,
1108	.ptr_type,
1109	.ptr_type_bit_range,
1110	=> return ptrType(gz, scope, ri, node, tree.fullPtrType(node).?),
1111
1112	.container_decl,
1113	.container_decl_trailing,
1114	.container_decl_arg,
1115	.container_decl_arg_trailing,
1116	.container_decl_two,
1117	.container_decl_two_trailing,
1118	.tagged_union,
1119	.tagged_union_trailing,
1120	.tagged_union_enum_tag,
1121	.tagged_union_enum_tag_trailing,
1122	.tagged_union_two,
1123	.tagged_union_two_trailing,
1124	=> {
1125	var buf: [2]Ast.Node.Index = undefined;
1126	return containerDecl(gz, scope, ri, node, tree.fullContainerDecl(&buf, node).?);
1127	},
1128
1129	.@"break" => return breakExpr(gz, scope, node),
1130	.@"continue" => return continueExpr(gz, scope, node),
1131	.grouped_expression => return expr(gz, scope, ri, node_datas[node].lhs),
1132	.array_type => return arrayType(gz, scope, ri, node),
1133	.array_type_sentinel => return arrayTypeSentinel(gz, scope, ri, node),
1134	.char_literal => return charLiteral(gz, ri, node),
1135	.error_set_decl => return errorSetDecl(gz, ri, node),
1136	.array_access => return arrayAccess(gz, scope, ri, node),
1137	.@"comptime" => return comptimeExprAst(gz, scope, ri, node),
1138	.@"switch", .switch_comma => return switchExpr(gz, scope, ri.br(), node),
1139
1140	.@"nosuspend" => return nosuspendExpr(gz, scope, ri, node),
1141	.@"suspend" => return suspendExpr(gz, scope, node),
1142	.@"await" => return awaitExpr(gz, scope, ri, node),
1143	.@"resume" => return resumeExpr(gz, scope, ri, node),
1144
1145	.@"try" => return tryExpr(gz, scope, ri, node, node_datas[node].lhs),
1146
1147	.array_init_one,
1148	.array_init_one_comma,
1149	.array_init_dot_two,
1150	.array_init_dot_two_comma,
1151	.array_init_dot,
1152	.array_init_dot_comma,
1153	.array_init,
1154	.array_init_comma,
1155	=> {
1156	var buf: [2]Ast.Node.Index = undefined;
1157	return arrayInitExpr(gz, scope, ri, node, tree.fullArrayInit(&buf, node).?);
1158	},
1159
1160	.struct_init_one,
1161	.struct_init_one_comma,
1162	.struct_init_dot_two,
1163	.struct_init_dot_two_comma,
1164	.struct_init_dot,
1165	.struct_init_dot_comma,
1166	.struct_init,
1167	.struct_init_comma,
1168	=> {
1169	var buf: [2]Ast.Node.Index = undefined;
1170	return structInitExpr(gz, scope, ri, node, tree.fullStructInit(&buf, node).?);
1171	},
1172
1173	.fn_proto_simple,
1174	.fn_proto_multi,
1175	.fn_proto_one,
1176	.fn_proto,
1177	=> {
1178	var buf: [1]Ast.Node.Index = undefined;
1179	return fnProtoExpr(gz, scope, ri, node, tree.fullFnProto(&buf, node).?);
1180	},
1181	}
1182	}
1183
1184	fn nosuspendExpr(
1185	gz: *GenZir,
1186	scope: *Scope,
1187	ri: ResultInfo,
1188	node: Ast.Node.Index,
1189	) InnerError!Zir.Inst.Ref {
1190	const astgen = gz.astgen;
1191	const tree = astgen.tree;
1192	const node_datas = tree.nodes.items(.data);
1193	const body_node = node_datas[node].lhs;
1194	assert(body_node != 0);
1195	if (gz.nosuspend_node != 0) {
1196	try astgen.appendErrorNodeNotes(node, "redundant nosuspend block", .{}, &[_]u32{
1197	try astgen.errNoteNode(gz.nosuspend_node, "other nosuspend block here", .{}),
1198	});
1199	}
1200	gz.nosuspend_node = node;
1201	defer gz.nosuspend_node = 0;
1202	return expr(gz, scope, ri, body_node);
1203	}
1204
1205	fn suspendExpr(
1206	gz: *GenZir,
1207	scope: *Scope,
1208	node: Ast.Node.Index,
1209	) InnerError!Zir.Inst.Ref {
1210	const astgen = gz.astgen;
1211	const gpa = astgen.gpa;
1212	const tree = astgen.tree;
1213	const node_datas = tree.nodes.items(.data);
1214	const body_node = node_datas[node].lhs;
1215
1216	if (gz.nosuspend_node != 0) {
1217	return astgen.failNodeNotes(node, "suspend inside nosuspend block", .{}, &[_]u32{
1218	try astgen.errNoteNode(gz.nosuspend_node, "nosuspend block here", .{}),
1219	});
1220	}
1221	if (gz.suspend_node != 0) {
1222	return astgen.failNodeNotes(node, "cannot suspend inside suspend block", .{}, &[_]u32{
1223	try astgen.errNoteNode(gz.suspend_node, "other suspend block here", .{}),
1224	});
1225	}
1226	assert(body_node != 0);
1227
1228	const suspend_inst = try gz.makeBlockInst(.suspend_block, node);
1229	try gz.instructions.append(gpa, suspend_inst);
1230
1231	var suspend_scope = gz.makeSubBlock(scope);
1232	suspend_scope.suspend_node = node;
1233	defer suspend_scope.unstack();
1234
1235	const body_result = try expr(&suspend_scope, &suspend_scope.base, .{ .rl = .none }, body_node);
1236	if (!gz.refIsNoReturn(body_result)) {
1237	_ = try suspend_scope.addBreak(.break_inline, suspend_inst, .void_value);
1238	}
1239	try suspend_scope.setBlockBody(suspend_inst);
1240
1241	return suspend_inst.toRef();
1242	}
1243
1244	fn awaitExpr(
1245	gz: *GenZir,
1246	scope: *Scope,
1247	ri: ResultInfo,
1248	node: Ast.Node.Index,
1249	) InnerError!Zir.Inst.Ref {
1250	const astgen = gz.astgen;
1251	const tree = astgen.tree;
1252	const node_datas = tree.nodes.items(.data);
1253	const rhs_node = node_datas[node].lhs;
1254
1255	if (gz.suspend_node != 0) {
1256	return astgen.failNodeNotes(node, "cannot await inside suspend block", .{}, &[_]u32{
1257	try astgen.errNoteNode(gz.suspend_node, "suspend block here", .{}),
1258	});
1259	}
1260	const operand = try expr(gz, scope, .{ .rl = .ref }, rhs_node);
1261	const result = if (gz.nosuspend_node != 0)
1262	try gz.addExtendedPayload(.await_nosuspend, Zir.Inst.UnNode{
1263	.node = gz.nodeIndexToRelative(node),
1264	.operand = operand,
1265	})
1266	else
1267	try gz.addUnNode(.@"await", operand, node);
1268
1269	return rvalue(gz, ri, result, node);
1270	}
1271
1272	fn resumeExpr(
1273	gz: *GenZir,
1274	scope: *Scope,
1275	ri: ResultInfo,
1276	node: Ast.Node.Index,
1277	) InnerError!Zir.Inst.Ref {
1278	const astgen = gz.astgen;
1279	const tree = astgen.tree;
1280	const node_datas = tree.nodes.items(.data);
1281	const rhs_node = node_datas[node].lhs;
1282	const operand = try expr(gz, scope, .{ .rl = .ref }, rhs_node);
1283	const result = try gz.addUnNode(.@"resume", operand, node);
1284	return rvalue(gz, ri, result, node);
1285	}
1286
1287	fn fnProtoExpr(
1288	gz: *GenZir,
1289	scope: *Scope,
1290	ri: ResultInfo,
1291	node: Ast.Node.Index,
1292	fn_proto: Ast.full.FnProto,
1293	) InnerError!Zir.Inst.Ref {
1294	const astgen = gz.astgen;
1295	const tree = astgen.tree;
1296	const token_tags = tree.tokens.items(.tag);
1297
1298	if (fn_proto.name_token) \|some\| {
1299	return astgen.failTok(some, "function type cannot have a name", .{});
1300	}
1301
1302	const is_extern = blk: {
1303	const maybe_extern_token = fn_proto.extern_export_inline_token orelse break :blk false;
1304	break :blk token_tags[maybe_extern_token] == .keyword_extern;
1305	};
1306	assert(!is_extern);
1307
1308	var block_scope = gz.makeSubBlock(scope);
1309	defer block_scope.unstack();
1310
1311	const block_inst = try gz.makeBlockInst(.block_inline, node);
1312
1313	var noalias_bits: u32 = 0;
1314	const is_var_args = is_var_args: {
1315	var param_type_i: usize = 0;
1316	var it = fn_proto.iterate(tree);
1317	while (it.next()) \|param\| : (param_type_i += 1) {
1318	const is_comptime = if (param.comptime_noalias) \|token\| switch (token_tags[token]) {
1319	.keyword_noalias => is_comptime: {
1320	noalias_bits \|= @as(u32, 1) << (std.math.cast(u5, param_type_i) orelse
1321	return astgen.failTok(token, "this compiler implementation only supports 'noalias' on the first 32 parameters", .{}));
1322	break :is_comptime false;
1323	},
1324	.keyword_comptime => true,
1325	else => false,
1326	} else false;
1327
1328	const is_anytype = if (param.anytype_ellipsis3) \|token\| blk: {
1329	switch (token_tags[token]) {
1330	.keyword_anytype => break :blk true,
1331	.ellipsis3 => break :is_var_args true,
1332	else => unreachable,
1333	}
1334	} else false;
1335
1336	const param_name = if (param.name_token) \|name_token\| blk: {
1337	if (mem.eql(u8, "_", tree.tokenSlice(name_token)))
1338	break :blk .empty;
1339
1340	break :blk try astgen.identAsString(name_token);
1341	} else .empty;
1342
1343	if (is_anytype) {
1344	const name_token = param.name_token orelse param.anytype_ellipsis3.?;
1345
1346	const tag: Zir.Inst.Tag = if (is_comptime)
1347	.param_anytype_comptime
1348	else
1349	.param_anytype;
1350	_ = try block_scope.addStrTok(tag, param_name, name_token);
1351	} else {
1352	const param_type_node = param.type_expr;
1353	assert(param_type_node != 0);
1354	var param_gz = block_scope.makeSubBlock(scope);
1355	defer param_gz.unstack();
1356	const param_type = try expr(&param_gz, scope, coerced_type_ri, param_type_node);
1357	const param_inst_expected: Zir.Inst.Index = @enumFromInt(astgen.instructions.len + 1);
1358	_ = try param_gz.addBreakWithSrcNode(.break_inline, param_inst_expected, param_type, param_type_node);
1359	const main_tokens = tree.nodes.items(.main_token);
1360	const name_token = param.name_token orelse main_tokens[param_type_node];
1361	const tag: Zir.Inst.Tag = if (is_comptime) .param_comptime else .param;
1362	const param_inst = try block_scope.addParam(&param_gz, tag, name_token, param_name, param.first_doc_comment);
1363	assert(param_inst_expected == param_inst);
1364	}
1365	}
1366	break :is_var_args false;
1367	};
1368
1369	const align_ref: Zir.Inst.Ref = if (fn_proto.ast.align_expr == 0) .none else inst: {
1370	break :inst try expr(&block_scope, scope, coerced_align_ri, fn_proto.ast.align_expr);
1371	};
1372
1373	if (fn_proto.ast.addrspace_expr != 0) {
1374	return astgen.failNode(fn_proto.ast.addrspace_expr, "addrspace not allowed on function prototypes", .{});
1375	}
1376
1377	if (fn_proto.ast.section_expr != 0) {
1378	return astgen.failNode(fn_proto.ast.section_expr, "linksection not allowed on function prototypes", .{});
1379	}
1380
1381	const cc: Zir.Inst.Ref = if (fn_proto.ast.callconv_expr != 0)
1382	try expr(
1383	&block_scope,
1384	scope,
1385	.{ .rl = .{ .coerced_ty = .calling_convention_type } },
1386	fn_proto.ast.callconv_expr,
1387	)
1388	else
1389	Zir.Inst.Ref.none;
1390
1391	const maybe_bang = tree.firstToken(fn_proto.ast.return_type) - 1;
1392	const is_inferred_error = token_tags[maybe_bang] == .bang;
1393	if (is_inferred_error) {
1394	return astgen.failTok(maybe_bang, "function prototype may not have inferred error set", .{});
1395	}
1396	const ret_ty = try expr(&block_scope, scope, coerced_type_ri, fn_proto.ast.return_type);
1397
1398	const result = try block_scope.addFunc(.{
1399	.src_node = fn_proto.ast.proto_node,
1400
1401	.cc_ref = cc,
1402	.cc_gz = null,
1403	.align_ref = align_ref,
1404	.align_gz = null,
1405	.ret_ref = ret_ty,
1406	.ret_gz = null,
1407	.section_ref = .none,
1408	.section_gz = null,
1409	.addrspace_ref = .none,
1410	.addrspace_gz = null,
1411
1412	.param_block = block_inst,
1413	.body_gz = null,
1414	.lib_name = .empty,
1415	.is_var_args = is_var_args,
1416	.is_inferred_error = false,
1417	.is_test = false,
1418	.is_extern = false,
1419	.is_noinline = false,
1420	.noalias_bits = noalias_bits,
1421	});
1422
1423	_ = try block_scope.addBreak(.break_inline, block_inst, result);
1424	try block_scope.setBlockBody(block_inst);
1425	try gz.instructions.append(astgen.gpa, block_inst);
1426
1427	return rvalue(gz, ri, block_inst.toRef(), fn_proto.ast.proto_node);
1428	}
1429
1430	fn arrayInitExpr(
1431	gz: *GenZir,
1432	scope: *Scope,
1433	ri: ResultInfo,
1434	node: Ast.Node.Index,
1435	array_init: Ast.full.ArrayInit,
1436	) InnerError!Zir.Inst.Ref {
1437	const astgen = gz.astgen;
1438	const tree = astgen.tree;
1439	const node_tags = tree.nodes.items(.tag);
1440	const main_tokens = tree.nodes.items(.main_token);
1441
1442	assert(array_init.ast.elements.len != 0); // Otherwise it would be struct init.
1443
1444	const array_ty: Zir.Inst.Ref, const elem_ty: Zir.Inst.Ref = inst: {
1445	if (array_init.ast.type_expr == 0) break :inst .{ .none, .none };
1446
1447	infer: {
1448	const array_type: Ast.full.ArrayType = tree.fullArrayType(array_init.ast.type_expr) orelse break :infer;
1449	// This intentionally does not support `@"_"` syntax.
1450	if (node_tags[array_type.ast.elem_count] == .identifier and
1451	mem.eql(u8, tree.tokenSlice(main_tokens[array_type.ast.elem_count]), "_"))
1452	{
1453	const len_inst = try gz.addInt(array_init.ast.elements.len);
1454	const elem_type = try typeExpr(gz, scope, array_type.ast.elem_type);
1455	if (array_type.ast.sentinel == 0) {
1456	const array_type_inst = try gz.addPlNode(.array_type, array_init.ast.type_expr, Zir.Inst.Bin{
1457	.lhs = len_inst,
1458	.rhs = elem_type,
1459	});
1460	break :inst .{ array_type_inst, elem_type };
1461	} else {
1462	const sentinel = try comptimeExpr(gz, scope, .{ .rl = .{ .ty = elem_type } }, array_type.ast.sentinel);
1463	const array_type_inst = try gz.addPlNode(
1464	.array_type_sentinel,
1465	array_init.ast.type_expr,
1466	Zir.Inst.ArrayTypeSentinel{
1467	.len = len_inst,
1468	.elem_type = elem_type,
1469	.sentinel = sentinel,
1470	},
1471	);
1472	break :inst .{ array_type_inst, elem_type };
1473	}
1474	}
1475	}
1476	const array_type_inst = try typeExpr(gz, scope, array_init.ast.type_expr);
1477	_ = try gz.addPlNode(.validate_array_init_ty, node, Zir.Inst.ArrayInit{
1478	.ty = array_type_inst,
1479	.init_count = @intCast(array_init.ast.elements.len),
1480	});
1481	break :inst .{ array_type_inst, .none };
1482	};
1483
1484	if (array_ty != .none) {
1485	// Typed inits do not use RLS for language simplicity.
1486	switch (ri.rl) {
1487	.discard => {
1488	if (elem_ty != .none) {
1489	const elem_ri: ResultInfo = .{ .rl = .{ .ty = elem_ty } };
1490	for (array_init.ast.elements) \|elem_init\| {
1491	_ = try expr(gz, scope, elem_ri, elem_init);
1492	}
1493	} else {
1494	for (array_init.ast.elements, 0..) \|elem_init, i\| {
1495	const this_elem_ty = try gz.add(.{
1496	.tag = .array_init_elem_type,
1497	.data = .{ .bin = .{
1498	.lhs = array_ty,
1499	.rhs = @enumFromInt(i),
1500	} },
1501	});
1502	_ = try expr(gz, scope, .{ .rl = .{ .ty = this_elem_ty } }, elem_init);
1503	}
1504	}
1505	return .void_value;
1506	},
1507	.ref => return arrayInitExprTyped(gz, scope, node, array_init.ast.elements, array_ty, elem_ty, true),
1508	else => {
1509	const array_inst = try arrayInitExprTyped(gz, scope, node, array_init.ast.elements, array_ty, elem_ty, false);
1510	return rvalue(gz, ri, array_inst, node);
1511	},
1512	}
1513	}
1514
1515	switch (ri.rl) {
1516	.none => return arrayInitExprAnon(gz, scope, node, array_init.ast.elements),
1517	.discard => {
1518	for (array_init.ast.elements) \|elem_init\| {
1519	_ = try expr(gz, scope, .{ .rl = .discard }, elem_init);
1520	}
1521	return Zir.Inst.Ref.void_value;
1522	},
1523	.ref => {
1524	const result = try arrayInitExprAnon(gz, scope, node, array_init.ast.elements);
1525	return gz.addUnTok(.ref, result, tree.firstToken(node));
1526	},
1527	.ref_coerced_ty => \|ptr_ty_inst\| {
1528	const dest_arr_ty_inst = try gz.addPlNode(.validate_array_init_ref_ty, node, Zir.Inst.ArrayInitRefTy{
1529	.ptr_ty = ptr_ty_inst,
1530	.elem_count = @intCast(array_init.ast.elements.len),
1531	});
1532	return arrayInitExprTyped(gz, scope, node, array_init.ast.elements, dest_arr_ty_inst, .none, true);
1533	},
1534	.ty, .coerced_ty => \|result_ty_inst\| {
1535	_ = try gz.addPlNode(.validate_array_init_result_ty, node, Zir.Inst.ArrayInit{
1536	.ty = result_ty_inst,
1537	.init_count = @intCast(array_init.ast.elements.len),
1538	});
1539	return arrayInitExprTyped(gz, scope, node, array_init.ast.elements, result_ty_inst, .none, false);
1540	},
1541	.ptr => \|ptr\| {
1542	try arrayInitExprPtr(gz, scope, node, array_init.ast.elements, ptr.inst);
1543	return .void_value;
1544	},
1545	.inferred_ptr => {
1546	// We can't get elem pointers of an untyped inferred alloc, so must perform a
1547	// standard anonymous initialization followed by an rvalue store.
1548	// See corresponding logic in structInitExpr.
1549	const result = try arrayInitExprAnon(gz, scope, node, array_init.ast.elements);
1550	return rvalue(gz, ri, result, node);
1551	},
1552	.destructure => \|destructure\| {
1553	// Untyped init - destructure directly into result pointers
1554	if (array_init.ast.elements.len != destructure.components.len) {
1555	return astgen.failNodeNotes(node, "expected {} elements for destructure, found {}", .{
1556	destructure.components.len,
1557	array_init.ast.elements.len,
1558	}, &.{
1559	try astgen.errNoteNode(destructure.src_node, "result destructured here", .{}),
1560	});
1561	}
1562	for (array_init.ast.elements, destructure.components) \|elem_init, ds_comp\| {
1563	const elem_ri: ResultInfo = .{ .rl = switch (ds_comp) {
1564	.typed_ptr => \|ptr_rl\| .{ .ptr = ptr_rl },
1565	.inferred_ptr => \|ptr_inst\| .{ .inferred_ptr = ptr_inst },
1566	.discard => .discard,
1567	} };
1568	_ = try expr(gz, scope, elem_ri, elem_init);
1569	}
1570	return .void_value;
1571	},
1572	}
1573	}
1574
1575	/// An array initialization expression using an `array_init_anon` instruction.
1576	fn arrayInitExprAnon(
1577	gz: *GenZir,
1578	scope: *Scope,
1579	node: Ast.Node.Index,
1580	elements: []const Ast.Node.Index,
1581	) InnerError!Zir.Inst.Ref {
1582	const astgen = gz.astgen;
1583
1584	const payload_index = try addExtra(astgen, Zir.Inst.MultiOp{
1585	.operands_len = @intCast(elements.len),
1586	});
1587	var extra_index = try reserveExtra(astgen, elements.len);
1588
1589	for (elements) \|elem_init\| {
1590	const elem_ref = try expr(gz, scope, .{ .rl = .none }, elem_init);
1591	astgen.extra.items[extra_index] = @intFromEnum(elem_ref);
1592	extra_index += 1;
1593	}
1594	return try gz.addPlNodePayloadIndex(.array_init_anon, node, payload_index);
1595	}
1596
1597	/// An array initialization expression using an `array_init` or `array_init_ref` instruction.
1598	fn arrayInitExprTyped(
1599	gz: *GenZir,
1600	scope: *Scope,
1601	node: Ast.Node.Index,
1602	elements: []const Ast.Node.Index,
1603	ty_inst: Zir.Inst.Ref,
1604	maybe_elem_ty_inst: Zir.Inst.Ref,
1605	is_ref: bool,
1606	) InnerError!Zir.Inst.Ref {
1607	const astgen = gz.astgen;
1608
1609	const len = elements.len + 1; // +1 for type
1610	const payload_index = try addExtra(astgen, Zir.Inst.MultiOp{
1611	.operands_len = @intCast(len),
1612	});
1613	var extra_index = try reserveExtra(astgen, len);
1614	astgen.extra.items[extra_index] = @intFromEnum(ty_inst);
1615	extra_index += 1;
1616
1617	if (maybe_elem_ty_inst != .none) {
1618	const elem_ri: ResultInfo = .{ .rl = .{ .coerced_ty = maybe_elem_ty_inst } };
1619	for (elements) \|elem_init\| {
1620	const elem_inst = try expr(gz, scope, elem_ri, elem_init);
1621	astgen.extra.items[extra_index] = @intFromEnum(elem_inst);
1622	extra_index += 1;
1623	}
1624	} else {
1625	for (elements, 0..) \|elem_init, i\| {
1626	const ri: ResultInfo = .{ .rl = .{ .coerced_ty = try gz.add(.{
1627	.tag = .array_init_elem_type,
1628	.data = .{ .bin = .{
1629	.lhs = ty_inst,
1630	.rhs = @enumFromInt(i),
1631	} },
1632	}) } };
1633
1634	const elem_inst = try expr(gz, scope, ri, elem_init);
1635	astgen.extra.items[extra_index] = @intFromEnum(elem_inst);
1636	extra_index += 1;
1637	}
1638	}
1639
1640	const tag: Zir.Inst.Tag = if (is_ref) .array_init_ref else .array_init;
1641	return try gz.addPlNodePayloadIndex(tag, node, payload_index);
1642	}
1643
1644	/// An array initialization expression using element pointers.
1645	fn arrayInitExprPtr(
1646	gz: *GenZir,
1647	scope: *Scope,
1648	node: Ast.Node.Index,
1649	elements: []const Ast.Node.Index,
1650	ptr_inst: Zir.Inst.Ref,
1651	) InnerError!void {
1652	const astgen = gz.astgen;
1653
1654	const array_ptr_inst = try gz.addUnNode(.opt_eu_base_ptr_init, ptr_inst, node);
1655
1656	const payload_index = try addExtra(astgen, Zir.Inst.Block{
1657	.body_len = @intCast(elements.len),
1658	});
1659	var extra_index = try reserveExtra(astgen, elements.len);
1660
1661	for (elements, 0..) \|elem_init, i\| {
1662	const elem_ptr_inst = try gz.addPlNode(.array_init_elem_ptr, elem_init, Zir.Inst.ElemPtrImm{
1663	.ptr = array_ptr_inst,
1664	.index = @intCast(i),
1665	});
1666	astgen.extra.items[extra_index] = @intFromEnum(elem_ptr_inst.toIndex().?);
1667	extra_index += 1;
1668	_ = try expr(gz, scope, .{ .rl = .{ .ptr = .{ .inst = elem_ptr_inst } } }, elem_init);
1669	}
1670
1671	_ = try gz.addPlNodePayloadIndex(.validate_ptr_array_init, node, payload_index);
1672	}
1673
1674	fn structInitExpr(
1675	gz: *GenZir,
1676	scope: *Scope,
1677	ri: ResultInfo,
1678	node: Ast.Node.Index,
1679	struct_init: Ast.full.StructInit,
1680	) InnerError!Zir.Inst.Ref {
1681	const astgen = gz.astgen;
1682	const tree = astgen.tree;
1683
1684	if (struct_init.ast.type_expr == 0) {
1685	if (struct_init.ast.fields.len == 0) {
1686	// Anonymous init with no fields.
1687	switch (ri.rl) {
1688	.discard => return .void_value,
1689	.ref_coerced_ty => \|ptr_ty_inst\| return gz.addUnNode(.struct_init_empty_ref_result, ptr_ty_inst, node),
1690	.ty, .coerced_ty => \|ty_inst\| return gz.addUnNode(.struct_init_empty_result, ty_inst, node),
1691	.ptr => {
1692	// TODO: should we modify this to use RLS for the field stores here?
1693	const ty_inst = (try ri.rl.resultType(gz, node)).?;
1694	const val = try gz.addUnNode(.struct_init_empty_result, ty_inst, node);
1695	return rvalue(gz, ri, val, node);
1696	},
1697	.none, .ref, .inferred_ptr => {
1698	return rvalue(gz, ri, .empty_struct, node);
1699	},
1700	.destructure => \|destructure\| {
1701	return astgen.failNodeNotes(node, "empty initializer cannot be destructured", .{}, &.{
1702	try astgen.errNoteNode(destructure.src_node, "result destructured here", .{}),
1703	});
1704	},
1705	}
1706	}
1707	} else array: {
1708	const node_tags = tree.nodes.items(.tag);
1709	const main_tokens = tree.nodes.items(.main_token);
1710	const array_type: Ast.full.ArrayType = tree.fullArrayType(struct_init.ast.type_expr) orelse {
1711	if (struct_init.ast.fields.len == 0) {
1712	const ty_inst = try typeExpr(gz, scope, struct_init.ast.type_expr);
1713	const result = try gz.addUnNode(.struct_init_empty, ty_inst, node);
1714	return rvalue(gz, ri, result, node);
1715	}
1716	break :array;
1717	};
1718	const is_inferred_array_len = node_tags[array_type.ast.elem_count] == .identifier and
1719	// This intentionally does not support `@"_"` syntax.
1720	mem.eql(u8, tree.tokenSlice(main_tokens[array_type.ast.elem_count]), "_");
1721	if (struct_init.ast.fields.len == 0) {
1722	if (is_inferred_array_len) {
1723	const elem_type = try typeExpr(gz, scope, array_type.ast.elem_type);
1724	const array_type_inst = if (array_type.ast.sentinel == 0) blk: {
1725	break :blk try gz.addPlNode(.array_type, struct_init.ast.type_expr, Zir.Inst.Bin{
1726	.lhs = .zero_usize,
1727	.rhs = elem_type,
1728	});
1729	} else blk: {
1730	const sentinel = try comptimeExpr(gz, scope, .{ .rl = .{ .ty = elem_type } }, array_type.ast.sentinel);
1731	break :blk try gz.addPlNode(
1732	.array_type_sentinel,
1733	struct_init.ast.type_expr,
1734	Zir.Inst.ArrayTypeSentinel{
1735	.len = .zero_usize,
1736	.elem_type = elem_type,
1737	.sentinel = sentinel,
1738	},
1739	);
1740	};
1741	const result = try gz.addUnNode(.struct_init_empty, array_type_inst, node);
1742	return rvalue(gz, ri, result, node);
1743	}
1744	const ty_inst = try typeExpr(gz, scope, struct_init.ast.type_expr);
1745	const result = try gz.addUnNode(.struct_init_empty, ty_inst, node);
1746	return rvalue(gz, ri, result, node);
1747	} else {
1748	return astgen.failNode(
1749	struct_init.ast.type_expr,
1750	"initializing array with struct syntax",
1751	.{},
1752	);
1753	}
1754	}
1755
1756	{
1757	var sfba = std.heap.stackFallback(256, astgen.arena);
1758	const sfba_allocator = sfba.get();
1759
1760	var duplicate_names = std.AutoArrayHashMap(Zir.NullTerminatedString, ArrayListUnmanaged(Ast.TokenIndex)).init(sfba_allocator);
1761	try duplicate_names.ensureTotalCapacity(@intCast(struct_init.ast.fields.len));
1762
1763	// When there aren't errors, use this to avoid a second iteration.
1764	var any_duplicate = false;
1765
1766	for (struct_init.ast.fields) \|field\| {
1767	const name_token = tree.firstToken(field) - 2;
1768	const name_index = try astgen.identAsString(name_token);
1769
1770	const gop = try duplicate_names.getOrPut(name_index);
1771
1772	if (gop.found_existing) {
1773	try gop.value_ptr.append(sfba_allocator, name_token);
1774	any_duplicate = true;
1775	} else {
1776	gop.value_ptr.* = .{};
1777	try gop.value_ptr.append(sfba_allocator, name_token);
1778	}
1779	}
1780
1781	if (any_duplicate) {
1782	var it = duplicate_names.iterator();
1783
1784	while (it.next()) \|entry\| {
1785	const record = entry.value_ptr.*;
1786	if (record.items.len > 1) {
1787	var error_notes = std.ArrayList(u32).init(astgen.arena);
1788
1789	for (record.items[1..]) \|duplicate\| {
1790	try error_notes.append(try astgen.errNoteTok(duplicate, "duplicate name here", .{}));
1791	}
1792
1793	try error_notes.append(try astgen.errNoteNode(node, "struct declared here", .{}));
1794
1795	try astgen.appendErrorTokNotes(
1796	record.items[0],
1797	"duplicate struct field name",
1798	.{},
1799	error_notes.items,
1800	);
1801	}
1802	}
1803
1804	return error.AnalysisFail;
1805	}
1806	}
1807
1808	if (struct_init.ast.type_expr != 0) {
1809	// Typed inits do not use RLS for language simplicity.
1810	const ty_inst = try typeExpr(gz, scope, struct_init.ast.type_expr);
1811	_ = try gz.addUnNode(.validate_struct_init_ty, ty_inst, node);
1812	switch (ri.rl) {
1813	.ref => return structInitExprTyped(gz, scope, node, struct_init, ty_inst, true),
1814	else => {
1815	const struct_inst = try structInitExprTyped(gz, scope, node, struct_init, ty_inst, false);
1816	return rvalue(gz, ri, struct_inst, node);
1817	},
1818	}
1819	}
1820
1821	switch (ri.rl) {
1822	.none => return structInitExprAnon(gz, scope, node, struct_init),
1823	.discard => {
1824	// Even if discarding we must perform side-effects.
1825	for (struct_init.ast.fields) \|field_init\| {
1826	_ = try expr(gz, scope, .{ .rl = .discard }, field_init);
1827	}
1828	return .void_value;
1829	},
1830	.ref => {
1831	const result = try structInitExprAnon(gz, scope, node, struct_init);
1832	return gz.addUnTok(.ref, result, tree.firstToken(node));
1833	},
1834	.ref_coerced_ty => \|ptr_ty_inst\| {
1835	const result_ty_inst = try gz.addUnNode(.elem_type, ptr_ty_inst, node);
1836	_ = try gz.addUnNode(.validate_struct_init_result_ty, result_ty_inst, node);
1837	return structInitExprTyped(gz, scope, node, struct_init, result_ty_inst, true);
1838	},
1839	.ty, .coerced_ty => \|result_ty_inst\| {
1840	_ = try gz.addUnNode(.validate_struct_init_result_ty, result_ty_inst, node);
1841	return structInitExprTyped(gz, scope, node, struct_init, result_ty_inst, false);
1842	},
1843	.ptr => \|ptr\| {
1844	try structInitExprPtr(gz, scope, node, struct_init, ptr.inst);
1845	return .void_value;
1846	},
1847	.inferred_ptr => {
1848	// We can't get field pointers of an untyped inferred alloc, so must perform a
1849	// standard anonymous initialization followed by an rvalue store.
1850	// See corresponding logic in arrayInitExpr.
1851	const struct_inst = try structInitExprAnon(gz, scope, node, struct_init);
1852	return rvalue(gz, ri, struct_inst, node);
1853	},
1854	.destructure => \|destructure\| {
1855	// This is an untyped init, so is an actual struct, which does
1856	// not support destructuring.
1857	return astgen.failNodeNotes(node, "struct value cannot be destructured", .{}, &.{
1858	try astgen.errNoteNode(destructure.src_node, "result destructured here", .{}),
1859	});
1860	},
1861	}
1862	}
1863
1864	/// A struct initialization expression using a `struct_init_anon` instruction.
1865	fn structInitExprAnon(
1866	gz: *GenZir,
1867	scope: *Scope,
1868	node: Ast.Node.Index,
1869	struct_init: Ast.full.StructInit,
1870	) InnerError!Zir.Inst.Ref {
1871	const astgen = gz.astgen;
1872	const tree = astgen.tree;
1873
1874	const payload_index = try addExtra(astgen, Zir.Inst.StructInitAnon{
1875	.fields_len = @intCast(struct_init.ast.fields.len),
1876	});
1877	const field_size = @typeInfo(Zir.Inst.StructInitAnon.Item).Struct.fields.len;
1878	var extra_index: usize = try reserveExtra(astgen, struct_init.ast.fields.len * field_size);
1879
1880	for (struct_init.ast.fields) \|field_init\| {
1881	const name_token = tree.firstToken(field_init) - 2;
1882	const str_index = try astgen.identAsString(name_token);
1883	setExtra(astgen, extra_index, Zir.Inst.StructInitAnon.Item{
1884	.field_name = str_index,
1885	.init = try expr(gz, scope, .{ .rl = .none }, field_init),
1886	});
1887	extra_index += field_size;
1888	}
1889
1890	return gz.addPlNodePayloadIndex(.struct_init_anon, node, payload_index);
1891	}
1892
1893	/// A struct initialization expression using a `struct_init` or `struct_init_ref` instruction.
1894	fn structInitExprTyped(
1895	gz: *GenZir,
1896	scope: *Scope,
1897	node: Ast.Node.Index,
1898	struct_init: Ast.full.StructInit,
1899	ty_inst: Zir.Inst.Ref,
1900	is_ref: bool,
1901	) InnerError!Zir.Inst.Ref {
1902	const astgen = gz.astgen;
1903	const tree = astgen.tree;
1904
1905	const payload_index = try addExtra(astgen, Zir.Inst.StructInit{
1906	.fields_len = @intCast(struct_init.ast.fields.len),
1907	});
1908	const field_size = @typeInfo(Zir.Inst.StructInit.Item).Struct.fields.len;
1909	var extra_index: usize = try reserveExtra(astgen, struct_init.ast.fields.len * field_size);
1910
1911	for (struct_init.ast.fields) \|field_init\| {
1912	const name_token = tree.firstToken(field_init) - 2;
1913	const str_index = try astgen.identAsString(name_token);
1914	const field_ty_inst = try gz.addPlNode(.struct_init_field_type, field_init, Zir.Inst.FieldType{
1915	.container_type = ty_inst,
1916	.name_start = str_index,
1917	});
1918	setExtra(astgen, extra_index, Zir.Inst.StructInit.Item{
1919	.field_type = field_ty_inst.toIndex().?,
1920	.init = try expr(gz, scope, .{ .rl = .{ .coerced_ty = field_ty_inst } }, field_init),
1921	});
1922	extra_index += field_size;
1923	}
1924
1925	const tag: Zir.Inst.Tag = if (is_ref) .struct_init_ref else .struct_init;
1926	return gz.addPlNodePayloadIndex(tag, node, payload_index);
1927	}
1928
1929	/// A struct initialization expression using field pointers.
1930	fn structInitExprPtr(
1931	gz: *GenZir,
1932	scope: *Scope,
1933	node: Ast.Node.Index,
1934	struct_init: Ast.full.StructInit,
1935	ptr_inst: Zir.Inst.Ref,
1936	) InnerError!void {
1937	const astgen = gz.astgen;
1938	const tree = astgen.tree;
1939
1940	const struct_ptr_inst = try gz.addUnNode(.opt_eu_base_ptr_init, ptr_inst, node);
1941
1942	const payload_index = try addExtra(astgen, Zir.Inst.Block{
1943	.body_len = @intCast(struct_init.ast.fields.len),
1944	});
1945	var extra_index = try reserveExtra(astgen, struct_init.ast.fields.len);
1946
1947	for (struct_init.ast.fields) \|field_init\| {
1948	const name_token = tree.firstToken(field_init) - 2;
1949	const str_index = try astgen.identAsString(name_token);
1950	const field_ptr = try gz.addPlNode(.struct_init_field_ptr, field_init, Zir.Inst.Field{
1951	.lhs = struct_ptr_inst,
1952	.field_name_start = str_index,
1953	});
1954	astgen.extra.items[extra_index] = @intFromEnum(field_ptr.toIndex().?);
1955	extra_index += 1;
1956	_ = try expr(gz, scope, .{ .rl = .{ .ptr = .{ .inst = field_ptr } } }, field_init);
1957	}
1958
1959	_ = try gz.addPlNodePayloadIndex(.validate_ptr_struct_init, node, payload_index);
1960	}
1961
1962	/// This explicitly calls expr in a comptime scope by wrapping it in a `block_comptime` if
1963	/// necessary. It should be used whenever we need to force compile-time evaluation of something,
1964	/// such as a type.
1965	/// The function corresponding to `comptime` expression syntax is `comptimeExprAst`.
1966	fn comptimeExpr(
1967	gz: *GenZir,
1968	scope: *Scope,
1969	ri: ResultInfo,
1970	node: Ast.Node.Index,
1971	) InnerError!Zir.Inst.Ref {
1972	if (gz.is_comptime) {
1973	// No need to change anything!
1974	return expr(gz, scope, ri, node);
1975	}
1976
1977	// There's an optimization here: if the body will be evaluated at comptime regardless, there's
1978	// no need to wrap it in a block. This is hard to determine in general, but we can identify a
1979	// common subset of trivially comptime expressions to take down the size of the ZIR a bit.
1980	const tree = gz.astgen.tree;
1981	const main_tokens = tree.nodes.items(.main_token);
1982	const node_tags = tree.nodes.items(.tag);
1983	switch (node_tags[node]) {
1984	// Any identifier in `primitive_instrs` is trivially comptime. In particular, this includes
1985	// some common types, so we can elide `block_comptime` for a few common type annotations.
1986	.identifier => {
1987	const ident_token = main_tokens[node];
1988	const ident_name_raw = tree.tokenSlice(ident_token);
1989	if (primitive_instrs.get(ident_name_raw)) \|zir_const_ref\| {
1990	// No need to worry about result location here, we're not creating a comptime block!
1991	return rvalue(gz, ri, zir_const_ref, node);
1992	}
1993	},
1994
1995	// We can also avoid the block for a few trivial AST tags which are always comptime-known.
1996	.number_literal, .string_literal, .multiline_string_literal, .enum_literal, .error_value => {
1997	// No need to worry about result location here, we're not creating a comptime block!
1998	return expr(gz, scope, ri, node);
1999	},
2000
2001	// Lastly, for labelled blocks, avoid emitting a labelled block directly inside this
2002	// comptime block, because that would be silly! Note that we don't bother doing this for
2003	// unlabelled blocks, since they don't generate blocks at comptime anyway (see `blockExpr`).
2004	.block_two, .block_two_semicolon, .block, .block_semicolon => {
2005	const token_tags = tree.tokens.items(.tag);
2006	const lbrace = main_tokens[node];
2007	// Careful! We can't pass in the real result location here, since it may
2008	// refer to runtime memory. A runtime-to-comptime boundary has to remove
2009	// result location information, compute the result, and copy it to the true
2010	// result location at runtime. We do this below as well.
2011	const ty_only_ri: ResultInfo = .{
2012	.ctx = ri.ctx,
2013	.rl = if (try ri.rl.resultType(gz, node)) \|res_ty\|
2014	.{ .coerced_ty = res_ty }
2015	else
2016	.none,
2017	};
2018	if (token_tags[lbrace - 1] == .colon and
2019	token_tags[lbrace - 2] == .identifier)
2020	{
2021	const node_datas = tree.nodes.items(.data);
2022	switch (node_tags[node]) {
2023	.block_two, .block_two_semicolon => {
2024	const stmts: [2]Ast.Node.Index = .{ node_datas[node].lhs, node_datas[node].rhs };
2025	const stmt_slice = if (stmts[0] == 0)
2026	stmts[0..0]
2027	else if (stmts[1] == 0)
2028	stmts[0..1]
2029	else
2030	stmts[0..2];
2031
2032	const block_ref = try labeledBlockExpr(gz, scope, ty_only_ri, node, stmt_slice, true);
2033	return rvalue(gz, ri, block_ref, node);
2034	},
2035	.block, .block_semicolon => {
2036	const stmts = tree.extra_data[node_datas[node].lhs..node_datas[node].rhs];
2037	// Replace result location and copy back later - see above.
2038	const block_ref = try labeledBlockExpr(gz, scope, ty_only_ri, node, stmts, true);
2039	return rvalue(gz, ri, block_ref, node);
2040	},
2041	else => unreachable,
2042	}
2043	}
2044	},
2045
2046	// In other cases, we don't optimize anything - we need a wrapper comptime block.
2047	else => {},
2048	}
2049
2050	var block_scope = gz.makeSubBlock(scope);
2051	block_scope.is_comptime = true;
2052	defer block_scope.unstack();
2053
2054	const block_inst = try gz.makeBlockInst(.block_comptime, node);
2055	// Replace result location and copy back later - see above.
2056	const ty_only_ri: ResultInfo = .{
2057	.ctx = ri.ctx,
2058	.rl = if (try ri.rl.resultType(gz, node)) \|res_ty\|
2059	.{ .coerced_ty = res_ty }
2060	else
2061	.none,
2062	};
2063	const block_result = try expr(&block_scope, scope, ty_only_ri, node);
2064	if (!gz.refIsNoReturn(block_result)) {
2065	_ = try block_scope.addBreak(.@"break", block_inst, block_result);
2066	}
2067	try block_scope.setBlockBody(block_inst);
2068	try gz.instructions.append(gz.astgen.gpa, block_inst);
2069
2070	return rvalue(gz, ri, block_inst.toRef(), node);
2071	}
2072
2073	/// This one is for an actual `comptime` syntax, and will emit a compile error if
2074	/// the scope is already known to be comptime-evaluated.
2075	/// See `comptimeExpr` for the helper function for calling expr in a comptime scope.
2076	fn comptimeExprAst(
2077	gz: *GenZir,
2078	scope: *Scope,
2079	ri: ResultInfo,
2080	node: Ast.Node.Index,
2081	) InnerError!Zir.Inst.Ref {
2082	const astgen = gz.astgen;
2083	if (gz.is_comptime) {
2084	return astgen.failNode(node, "redundant comptime keyword in already comptime scope", .{});
2085	}
2086	const tree = astgen.tree;
2087	const node_datas = tree.nodes.items(.data);
2088	const body_node = node_datas[node].lhs;
2089	return comptimeExpr(gz, scope, ri, body_node);
2090	}
2091
2092	/// Restore the error return trace index. Performs the restore only if the result is a non-error or
2093	/// if the result location is a non-error-handling expression.
2094	fn restoreErrRetIndex(
2095	gz: *GenZir,
2096	bt: GenZir.BranchTarget,
2097	ri: ResultInfo,
2098	node: Ast.Node.Index,
2099	result: Zir.Inst.Ref,
2100	) !void {
2101	const op = switch (nodeMayEvalToError(gz.astgen.tree, node)) {
2102	.always => return, // never restore/pop
2103	.never => .none, // always restore/pop
2104	.maybe => switch (ri.ctx) {
2105	.error_handling_expr, .@"return", .fn_arg, .const_init => switch (ri.rl) {
2106	.ptr => \|ptr_res\| try gz.addUnNode(.load, ptr_res.inst, node),
2107	.inferred_ptr => blk: {
2108	// This is a terrible workaround for Sema's inability to load from a .alloc_inferred ptr
2109	// before its type has been resolved. There is no valid operand to use here, so error
2110	// traces will be popped prematurely.
2111	// TODO: Update this to do a proper load from the rl_ptr, once Sema can support it.
2112	break :blk .none;
2113	},
2114	.destructure => return, // value must be a tuple or array, so never restore/pop
2115	else => result,
2116	},
2117	else => .none, // always restore/pop
2118	},
2119	};
2120	_ = try gz.addRestoreErrRetIndex(bt, .{ .if_non_error = op }, node);
2121	}
2122
2123	fn breakExpr(parent_gz: GenZir, parent_scope: Scope, node: Ast.Node.Index) InnerError!Zir.Inst.Ref {
2124	const astgen = parent_gz.astgen;
2125	const tree = astgen.tree;
2126	const node_datas = tree.nodes.items(.data);
2127	const break_label = node_datas[node].lhs;
2128	const rhs = node_datas[node].rhs;
2129
2130	// Look for the label in the scope.
2131	var scope = parent_scope;
2132	while (true) {
2133	switch (scope.tag) {
2134	.gen_zir => {
2135	const block_gz = scope.cast(GenZir).?;
2136
2137	if (block_gz.cur_defer_node != 0) {
2138	// We are breaking out of a `defer` block.
2139	return astgen.failNodeNotes(node, "cannot break out of defer expression", .{}, &.{
2140	try astgen.errNoteNode(
2141	block_gz.cur_defer_node,
2142	"defer expression here",
2143	.{},
2144	),
2145	});
2146	}
2147
2148	const block_inst = blk: {
2149	if (break_label != 0) {
2150	if (block_gz.label) \|*label\| {
2151	if (try astgen.tokenIdentEql(label.token, break_label)) {
2152	label.used = true;
2153	break :blk label.block_inst;
2154	}
2155	}
2156	} else if (block_gz.break_block.unwrap()) \|i\| {
2157	break :blk i;
2158	}
2159	// If not the target, start over with the parent
2160	scope = block_gz.parent;
2161	continue;
2162	};
2163	// If we made it here, this block is the target of the break expr
2164
2165	const break_tag: Zir.Inst.Tag = if (block_gz.is_inline)
2166	.break_inline
2167	else
2168	.@"break";
2169
2170	if (rhs == 0) {
2171	_ = try rvalue(parent_gz, block_gz.break_result_info, .void_value, node);
2172
2173	try genDefers(parent_gz, scope, parent_scope, .normal_only);
2174
2175	// As our last action before the break, "pop" the error trace if needed
2176	if (!block_gz.is_comptime)
2177	_ = try parent_gz.addRestoreErrRetIndex(.{ .block = block_inst }, .always, node);
2178
2179	_ = try parent_gz.addBreak(break_tag, block_inst, .void_value);
2180	return Zir.Inst.Ref.unreachable_value;
2181	}
2182
2183	const operand = try reachableExpr(parent_gz, parent_scope, block_gz.break_result_info, rhs, node);
2184
2185	try genDefers(parent_gz, scope, parent_scope, .normal_only);
2186
2187	// As our last action before the break, "pop" the error trace if needed
2188	if (!block_gz.is_comptime)
2189	try restoreErrRetIndex(parent_gz, .{ .block = block_inst }, block_gz.break_result_info, rhs, operand);
2190
2191	switch (block_gz.break_result_info.rl) {
2192	.ptr => {
2193	// In this case we don't have any mechanism to intercept it;
2194	// we assume the result location is written, and we break with void.
2195	_ = try parent_gz.addBreak(break_tag, block_inst, .void_value);
2196	},
2197	.discard => {
2198	_ = try parent_gz.addBreak(break_tag, block_inst, .void_value);
2199	},
2200	else => {
2201	_ = try parent_gz.addBreakWithSrcNode(break_tag, block_inst, operand, rhs);
2202	},
2203	}
2204	return Zir.Inst.Ref.unreachable_value;
2205	},
2206	.local_val => scope = scope.cast(Scope.LocalVal).?.parent,
2207	.local_ptr => scope = scope.cast(Scope.LocalPtr).?.parent,
2208	.namespace, .enum_namespace => break,
2209	.defer_normal, .defer_error => scope = scope.cast(Scope.Defer).?.parent,
2210	.top => unreachable,
2211	}
2212	}
2213	if (break_label != 0) {
2214	const label_name = try astgen.identifierTokenString(break_label);
2215	return astgen.failTok(break_label, "label not found: '{s}'", .{label_name});
2216	} else {
2217	return astgen.failNode(node, "break expression outside loop", .{});
2218	}
2219	}
2220
2221	fn continueExpr(parent_gz: GenZir, parent_scope: Scope, node: Ast.Node.Index) InnerError!Zir.Inst.Ref {
2222	const astgen = parent_gz.astgen;
2223	const tree = astgen.tree;
2224	const node_datas = tree.nodes.items(.data);
2225	const break_label = node_datas[node].lhs;
2226
2227	// Look for the label in the scope.
2228	var scope = parent_scope;
2229	while (true) {
2230	switch (scope.tag) {
2231	.gen_zir => {
2232	const gen_zir = scope.cast(GenZir).?;
2233
2234	if (gen_zir.cur_defer_node != 0) {
2235	return astgen.failNodeNotes(node, "cannot continue out of defer expression", .{}, &.{
2236	try astgen.errNoteNode(
2237	gen_zir.cur_defer_node,
2238	"defer expression here",
2239	.{},
2240	),
2241	});
2242	}
2243	const continue_block = gen_zir.continue_block.unwrap() orelse {
2244	scope = gen_zir.parent;
2245	continue;
2246	};
2247	if (break_label != 0) blk: {
2248	if (gen_zir.label) \|*label\| {
2249	if (try astgen.tokenIdentEql(label.token, break_label)) {
2250	label.used = true;
2251	break :blk;
2252	}
2253	}
2254	// found continue but either it has a different label, or no label
2255	scope = gen_zir.parent;
2256	continue;
2257	}
2258
2259	const break_tag: Zir.Inst.Tag = if (gen_zir.is_inline)
2260	.break_inline
2261	else
2262	.@"break";
2263	if (break_tag == .break_inline) {
2264	_ = try parent_gz.addUnNode(.check_comptime_control_flow, continue_block.toRef(), node);
2265	}
2266
2267	// As our last action before the continue, "pop" the error trace if needed
2268	if (!gen_zir.is_comptime)
2269	_ = try parent_gz.addRestoreErrRetIndex(.{ .block = continue_block }, .always, node);
2270
2271	_ = try parent_gz.addBreak(break_tag, continue_block, .void_value);
2272	return Zir.Inst.Ref.unreachable_value;
2273	},
2274	.local_val => scope = scope.cast(Scope.LocalVal).?.parent,
2275	.local_ptr => scope = scope.cast(Scope.LocalPtr).?.parent,
2276	.defer_normal => {
2277	const defer_scope = scope.cast(Scope.Defer).?;
2278	scope = defer_scope.parent;
2279	try parent_gz.addDefer(defer_scope.index, defer_scope.len);
2280	},
2281	.defer_error => scope = scope.cast(Scope.Defer).?.parent,
2282	.namespace, .enum_namespace => break,
2283	.top => unreachable,
2284	}
2285	}
2286	if (break_label != 0) {
2287	const label_name = try astgen.identifierTokenString(break_label);
2288	return astgen.failTok(break_label, "label not found: '{s}'", .{label_name});
2289	} else {
2290	return astgen.failNode(node, "continue expression outside loop", .{});
2291	}
2292	}
2293
2294	fn blockExpr(
2295	gz: *GenZir,
2296	scope: *Scope,
2297	ri: ResultInfo,
2298	block_node: Ast.Node.Index,
2299	statements: []const Ast.Node.Index,
2300	) InnerError!Zir.Inst.Ref {
2301	const astgen = gz.astgen;
2302	const tree = astgen.tree;
2303	const main_tokens = tree.nodes.items(.main_token);
2304	const token_tags = tree.tokens.items(.tag);
2305
2306	const lbrace = main_tokens[block_node];
2307	if (token_tags[lbrace - 1] == .colon and
2308	token_tags[lbrace - 2] == .identifier)
2309	{
2310	return labeledBlockExpr(gz, scope, ri, block_node, statements, false);
2311	}
2312
2313	if (!gz.is_comptime) {
2314	// Since this block is unlabeled, its control flow is effectively linear and we
2315	// can almost get away with inlining the block here. However, we actually need
2316	// to preserve the .block for Sema, to properly pop the error return trace.
2317
2318	const block_tag: Zir.Inst.Tag = .block;
2319	const block_inst = try gz.makeBlockInst(block_tag, block_node);
2320	try gz.instructions.append(astgen.gpa, block_inst);
2321
2322	var block_scope = gz.makeSubBlock(scope);
2323	defer block_scope.unstack();
2324
2325	try blockExprStmts(&block_scope, &block_scope.base, statements);
2326
2327	if (!block_scope.endsWithNoReturn()) {
2328	// As our last action before the break, "pop" the error trace if needed
2329	_ = try gz.addRestoreErrRetIndex(.{ .block = block_inst }, .always, block_node);
2330	_ = try block_scope.addBreak(.@"break", block_inst, .void_value);
2331	}
2332
2333	try block_scope.setBlockBody(block_inst);
2334	} else {
2335	var sub_gz = gz.makeSubBlock(scope);
2336	try blockExprStmts(&sub_gz, &sub_gz.base, statements);
2337	}
2338
2339	return rvalue(gz, ri, .void_value, block_node);
2340	}
2341
2342	fn checkLabelRedefinition(astgen: AstGen, parent_scope: Scope, label: Ast.TokenIndex) !void {
2343	// Look for the label in the scope.
2344	var scope = parent_scope;
2345	while (true) {
2346	switch (scope.tag) {
2347	.gen_zir => {
2348	const gen_zir = scope.cast(GenZir).?;
2349	if (gen_zir.label) \|prev_label\| {
2350	if (try astgen.tokenIdentEql(label, prev_label.token)) {
2351	const label_name = try astgen.identifierTokenString(label);
2352	return astgen.failTokNotes(label, "redefinition of label '{s}'", .{
2353	label_name,
2354	}, &[_]u32{
2355	try astgen.errNoteTok(
2356	prev_label.token,
2357	"previous definition here",
2358	.{},
2359	),
2360	});
2361	}
2362	}
2363	scope = gen_zir.parent;
2364	},
2365	.local_val => scope = scope.cast(Scope.LocalVal).?.parent,
2366	.local_ptr => scope = scope.cast(Scope.LocalPtr).?.parent,
2367	.defer_normal, .defer_error => scope = scope.cast(Scope.Defer).?.parent,
2368	.namespace, .enum_namespace => break,
2369	.top => unreachable,
2370	}
2371	}
2372	}
2373
2374	fn labeledBlockExpr(
2375	gz: *GenZir,
2376	parent_scope: *Scope,
2377	ri: ResultInfo,
2378	block_node: Ast.Node.Index,
2379	statements: []const Ast.Node.Index,
2380	force_comptime: bool,
2381	) InnerError!Zir.Inst.Ref {
2382	const astgen = gz.astgen;
2383	const tree = astgen.tree;
2384	const main_tokens = tree.nodes.items(.main_token);
2385	const token_tags = tree.tokens.items(.tag);
2386
2387	const lbrace = main_tokens[block_node];
2388	const label_token = lbrace - 2;
2389	assert(token_tags[label_token] == .identifier);
2390
2391	try astgen.checkLabelRedefinition(parent_scope, label_token);
2392
2393	const need_rl = astgen.nodes_need_rl.contains(block_node);
2394	const block_ri: ResultInfo = if (need_rl) ri else .{
2395	.rl = switch (ri.rl) {
2396	.ptr => .{ .ty = (try ri.rl.resultType(gz, block_node)).? },
2397	.inferred_ptr => .none,
2398	else => ri.rl,
2399	},
2400	.ctx = ri.ctx,
2401	};
2402	// We need to call `rvalue` to write through to the pointer only if we had a
2403	// result pointer and aren't forwarding it.
2404	const LocTag = @typeInfo(ResultInfo.Loc).Union.tag_type.?;
2405	const need_result_rvalue = @as(LocTag, block_ri.rl) != @as(LocTag, ri.rl);
2406
2407	// Reserve the Block ZIR instruction index so that we can put it into the GenZir struct
2408	// so that break statements can reference it.
2409	const block_tag: Zir.Inst.Tag = if (force_comptime) .block_comptime else .block;
2410	const block_inst = try gz.makeBlockInst(block_tag, block_node);
2411	try gz.instructions.append(astgen.gpa, block_inst);
2412	var block_scope = gz.makeSubBlock(parent_scope);
2413	block_scope.label = GenZir.Label{
2414	.token = label_token,
2415	.block_inst = block_inst,
2416	};
2417	block_scope.setBreakResultInfo(block_ri);
2418	if (force_comptime) block_scope.is_comptime = true;
2419	defer block_scope.unstack();
2420
2421	try blockExprStmts(&block_scope, &block_scope.base, statements);
2422	if (!block_scope.endsWithNoReturn()) {
2423	// As our last action before the return, "pop" the error trace if needed
2424	_ = try gz.addRestoreErrRetIndex(.{ .block = block_inst }, .always, block_node);
2425	_ = try block_scope.addBreak(.@"break", block_inst, .void_value);
2426	}
2427
2428	if (!block_scope.label.?.used) {
2429	try astgen.appendErrorTok(label_token, "unused block label", .{});
2430	}
2431
2432	try block_scope.setBlockBody(block_inst);
2433	if (need_result_rvalue) {
2434	return rvalue(gz, ri, block_inst.toRef(), block_node);
2435	} else {
2436	return block_inst.toRef();
2437	}
2438	}
2439
2440	fn blockExprStmts(gz: GenZir, parent_scope: Scope, statements: []const Ast.Node.Index) !void {
2441	const astgen = gz.astgen;
2442	const tree = astgen.tree;
2443	const node_tags = tree.nodes.items(.tag);
2444	const node_data = tree.nodes.items(.data);
2445
2446	if (statements.len == 0) return;
2447
2448	var block_arena = std.heap.ArenaAllocator.init(gz.astgen.gpa);
2449	defer block_arena.deinit();
2450	const block_arena_allocator = block_arena.allocator();
2451
2452	var noreturn_src_node: Ast.Node.Index = 0;
2453	var scope = parent_scope;
2454	for (statements) \|statement\| {
2455	if (noreturn_src_node != 0) {
2456	try astgen.appendErrorNodeNotes(
2457	statement,
2458	"unreachable code",
2459	.{},
2460	&[_]u32{
2461	try astgen.errNoteNode(
2462	noreturn_src_node,
2463	"control flow is diverted here",
2464	.{},
2465	),
2466	},
2467	);
2468	}
2469	var inner_node = statement;
2470	while (true) {
2471	switch (node_tags[inner_node]) {
2472	// zig fmt: off
2473	.global_var_decl,
2474	.local_var_decl,
2475	.simple_var_decl,
2476	.aligned_var_decl, => scope = try varDecl(gz, scope, statement, block_arena_allocator, tree.fullVarDecl(statement).?),
2477
2478	.assign_destructure => scope = try assignDestructureMaybeDecls(gz, scope, statement, block_arena_allocator),
2479
2480	.@"defer" => scope = try deferStmt(gz, scope, statement, block_arena_allocator, .defer_normal),
2481	.@"errdefer" => scope = try deferStmt(gz, scope, statement, block_arena_allocator, .defer_error),
2482
2483	.assign => try assign(gz, scope, statement),
2484
2485	.assign_shl => try assignShift(gz, scope, statement, .shl),
2486	.assign_shr => try assignShift(gz, scope, statement, .shr),
2487
2488	.assign_bit_and => try assignOp(gz, scope, statement, .bit_and),
2489	.assign_bit_or => try assignOp(gz, scope, statement, .bit_or),
2490	.assign_bit_xor => try assignOp(gz, scope, statement, .xor),
2491	.assign_div => try assignOp(gz, scope, statement, .div),
2492	.assign_sub => try assignOp(gz, scope, statement, .sub),
2493	.assign_sub_wrap => try assignOp(gz, scope, statement, .subwrap),
2494	.assign_mod => try assignOp(gz, scope, statement, .mod_rem),
2495	.assign_add => try assignOp(gz, scope, statement, .add),
2496	.assign_add_wrap => try assignOp(gz, scope, statement, .addwrap),
2497	.assign_mul => try assignOp(gz, scope, statement, .mul),
2498	.assign_mul_wrap => try assignOp(gz, scope, statement, .mulwrap),
2499
2500	.grouped_expression => {
2501	inner_node = node_data[statement].lhs;
2502	continue;
2503	},
2504
2505	.while_simple,
2506	.while_cont,
2507	.@"while", => _ = try whileExpr(gz, scope, .{ .rl = .none }, inner_node, tree.fullWhile(inner_node).?, true),
2508
2509	.for_simple,
2510	.@"for", => _ = try forExpr(gz, scope, .{ .rl = .none }, inner_node, tree.fullFor(inner_node).?, true),
2511
2512	else => noreturn_src_node = try unusedResultExpr(gz, scope, inner_node),
2513	// zig fmt: on
2514	}
2515	break;
2516	}
2517	}
2518
2519	try genDefers(gz, parent_scope, scope, .normal_only);
2520	try checkUsed(gz, parent_scope, scope);
2521	}
2522
2523	/// Returns AST source node of the thing that is noreturn if the statement is
2524	/// definitely `noreturn`. Otherwise returns 0.
2525	fn unusedResultExpr(gz: GenZir, scope: Scope, statement: Ast.Node.Index) InnerError!Ast.Node.Index {
2526	try emitDbgNode(gz, statement);
2527	// We need to emit an error if the result is not `noreturn` or `void`, but
2528	// we want to avoid adding the ZIR instruction if possible for performance.
2529	const maybe_unused_result = try expr(gz, scope, .{ .rl = .none }, statement);
2530	return addEnsureResult(gz, maybe_unused_result, statement);
2531	}
2532
2533	fn addEnsureResult(gz: *GenZir, maybe_unused_result: Zir.Inst.Ref, statement: Ast.Node.Index) InnerError!Ast.Node.Index {
2534	var noreturn_src_node: Ast.Node.Index = 0;
2535	const elide_check = if (maybe_unused_result.toIndex()) \|inst\| b: {
2536	// Note that this array becomes invalid after appending more items to it
2537	// in the above while loop.
2538	const zir_tags = gz.astgen.instructions.items(.tag);
2539	switch (zir_tags[@intFromEnum(inst)]) {
2540	// For some instructions, modify the zir data
2541	// so we can avoid a separate ensure_result_used instruction.
2542	.call, .field_call => {
2543	const break_extra = gz.astgen.instructions.items(.data)[@intFromEnum(inst)].pl_node.payload_index;
2544	comptime assert(std.meta.fieldIndex(Zir.Inst.Call, "flags") ==
2545	std.meta.fieldIndex(Zir.Inst.FieldCall, "flags"));
2546	const flags: *Zir.Inst.Call.Flags = @ptrCast(&gz.astgen.extra.items[
2547	break_extra + std.meta.fieldIndex(Zir.Inst.Call, "flags").?
2548	]);
2549	flags.ensure_result_used = true;
2550	break :b true;
2551	},
2552	.builtin_call => {
2553	const break_extra = gz.astgen.instructions.items(.data)[@intFromEnum(inst)].pl_node.payload_index;
2554	const flags: *Zir.Inst.BuiltinCall.Flags = @ptrCast(&gz.astgen.extra.items[
2555	break_extra + std.meta.fieldIndex(Zir.Inst.BuiltinCall, "flags").?
2556	]);
2557	flags.ensure_result_used = true;
2558	break :b true;
2559	},
2560
2561	// ZIR instructions that might be a type other than `noreturn` or `void`.
2562	.add,
2563	.addwrap,
2564	.add_sat,
2565	.add_unsafe,
2566	.param,
2567	.param_comptime,
2568	.param_anytype,
2569	.param_anytype_comptime,
2570	.alloc,
2571	.alloc_mut,
2572	.alloc_comptime_mut,
2573	.alloc_inferred,
2574	.alloc_inferred_mut,
2575	.alloc_inferred_comptime,
2576	.alloc_inferred_comptime_mut,
2577	.make_ptr_const,
2578	.array_cat,
2579	.array_mul,
2580	.array_type,
2581	.array_type_sentinel,
2582	.elem_type,
2583	.indexable_ptr_elem_type,
2584	.vector_elem_type,
2585	.vector_type,
2586	.indexable_ptr_len,
2587	.anyframe_type,
2588	.as_node,
2589	.as_shift_operand,
2590	.bit_and,
2591	.bitcast,
2592	.bit_or,
2593	.block,
2594	.block_comptime,
2595	.block_inline,
2596	.declaration,
2597	.suspend_block,
2598	.loop,
2599	.bool_br_and,
2600	.bool_br_or,
2601	.bool_not,
2602	.cmp_lt,
2603	.cmp_lte,
2604	.cmp_eq,
2605	.cmp_gte,
2606	.cmp_gt,
2607	.cmp_neq,
2608	.decl_ref,
2609	.decl_val,
2610	.load,
2611	.div,
2612	.elem_ptr,
2613	.elem_val,
2614	.elem_ptr_node,
2615	.elem_val_node,
2616	.elem_val_imm,
2617	.field_ptr,
2618	.field_val,
2619	.field_ptr_named,
2620	.field_val_named,
2621	.func,
2622	.func_inferred,
2623	.func_fancy,
2624	.int,
2625	.int_big,
2626	.float,
2627	.float128,
2628	.int_type,
2629	.is_non_null,
2630	.is_non_null_ptr,
2631	.is_non_err,
2632	.is_non_err_ptr,
2633	.ret_is_non_err,
2634	.mod_rem,
2635	.mul,
2636	.mulwrap,
2637	.mul_sat,
2638	.ref,
2639	.shl,
2640	.shl_sat,
2641	.shr,
2642	.str,
2643	.sub,
2644	.subwrap,
2645	.sub_sat,
2646	.negate,
2647	.negate_wrap,
2648	.typeof,
2649	.typeof_builtin,
2650	.xor,
2651	.optional_type,
2652	.optional_payload_safe,
2653	.optional_payload_unsafe,
2654	.optional_payload_safe_ptr,
2655	.optional_payload_unsafe_ptr,
2656	.err_union_payload_unsafe,
2657	.err_union_payload_unsafe_ptr,
2658	.err_union_code,
2659	.err_union_code_ptr,
2660	.ptr_type,
2661	.enum_literal,
2662	.merge_error_sets,
2663	.error_union_type,
2664	.bit_not,
2665	.error_value,
2666	.slice_start,
2667	.slice_end,
2668	.slice_sentinel,
2669	.slice_length,
2670	.import,
2671	.switch_block,
2672	.switch_block_ref,
2673	.switch_block_err_union,
2674	.union_init,
2675	.field_type_ref,
2676	.error_set_decl,
2677	.error_set_decl_anon,
2678	.error_set_decl_func,
2679	.enum_from_int,
2680	.int_from_enum,
2681	.type_info,
2682	.size_of,
2683	.bit_size_of,
2684	.typeof_log2_int_type,
2685	.int_from_ptr,
2686	.align_of,
2687	.int_from_bool,
2688	.embed_file,
2689	.error_name,
2690	.sqrt,
2691	.sin,
2692	.cos,
2693	.tan,
2694	.exp,
2695	.exp2,
2696	.log,
2697	.log2,
2698	.log10,
2699	.abs,
2700	.floor,
2701	.ceil,
2702	.trunc,
2703	.round,
2704	.tag_name,
2705	.type_name,
2706	.frame_type,
2707	.frame_size,
2708	.int_from_float,
2709	.float_from_int,
2710	.ptr_from_int,
2711	.float_cast,
2712	.int_cast,
2713	.ptr_cast,
2714	.truncate,
2715	.has_decl,
2716	.has_field,
2717	.clz,
2718	.ctz,
2719	.pop_count,
2720	.byte_swap,
2721	.bit_reverse,
2722	.div_exact,
2723	.div_floor,
2724	.div_trunc,
2725	.mod,
2726	.rem,
2727	.shl_exact,
2728	.shr_exact,
2729	.bit_offset_of,
2730	.offset_of,
2731	.splat,
2732	.reduce,
2733	.shuffle,
2734	.atomic_load,
2735	.atomic_rmw,
2736	.mul_add,
2737	.field_parent_ptr,
2738	.max,
2739	.min,
2740	.c_import,
2741	.@"resume",
2742	.@"await",
2743	.ret_err_value_code,
2744	.closure_get,
2745	.ret_ptr,
2746	.ret_type,
2747	.for_len,
2748	.@"try",
2749	.try_ptr,
2750	.opt_eu_base_ptr_init,
2751	.coerce_ptr_elem_ty,
2752	.struct_init_empty,
2753	.struct_init_empty_result,
2754	.struct_init_empty_ref_result,
2755	.struct_init_anon,
2756	.struct_init,
2757	.struct_init_ref,
2758	.struct_init_field_type,
2759	.struct_init_field_ptr,
2760	.array_init_anon,
2761	.array_init,
2762	.array_init_ref,
2763	.validate_array_init_ref_ty,
2764	.array_init_elem_type,
2765	.array_init_elem_ptr,
2766	=> break :b false,
2767
2768	.extended => switch (gz.astgen.instructions.items(.data)[@intFromEnum(inst)].extended.opcode) {
2769	.breakpoint,
2770	.fence,
2771	.set_float_mode,
2772	.set_align_stack,
2773	.set_cold,
2774	=> break :b true,
2775	else => break :b false,
2776	},
2777
2778	// ZIR instructions that are always `noreturn`.
2779	.@"break",
2780	.break_inline,
2781	.condbr,
2782	.condbr_inline,
2783	.compile_error,
2784	.ret_node,
2785	.ret_load,
2786	.ret_implicit,
2787	.ret_err_value,
2788	.@"unreachable",
2789	.repeat,
2790	.repeat_inline,
2791	.panic,
2792	.trap,
2793	.check_comptime_control_flow,
2794	=> {
2795	noreturn_src_node = statement;
2796	break :b true;
2797	},
2798
2799	// ZIR instructions that are always `void`.
2800	.dbg_stmt,
2801	.dbg_var_ptr,
2802	.dbg_var_val,
2803	.ensure_result_used,
2804	.ensure_result_non_error,
2805	.ensure_err_union_payload_void,
2806	.@"export",
2807	.export_value,
2808	.set_eval_branch_quota,
2809	.atomic_store,
2810	.store_node,
2811	.store_to_inferred_ptr,
2812	.resolve_inferred_alloc,
2813	.set_runtime_safety,
2814	.closure_capture,
2815	.memcpy,
2816	.memset,
2817	.validate_deref,
2818	.validate_destructure,
2819	.save_err_ret_index,
2820	.restore_err_ret_index_unconditional,
2821	.restore_err_ret_index_fn_entry,
2822	.validate_struct_init_ty,
2823	.validate_struct_init_result_ty,
2824	.validate_ptr_struct_init,
2825	.validate_array_init_ty,
2826	.validate_array_init_result_ty,
2827	.validate_ptr_array_init,
2828	.validate_ref_ty,
2829	=> break :b true,
2830
2831	.@"defer" => unreachable,
2832	.defer_err_code => unreachable,
2833	}
2834	} else switch (maybe_unused_result) {
2835	.none => unreachable,
2836
2837	.unreachable_value => b: {
2838	noreturn_src_node = statement;
2839	break :b true;
2840	},
2841
2842	.void_value => true,
2843
2844	else => false,
2845	};
2846	if (!elide_check) {
2847	_ = try gz.addUnNode(.ensure_result_used, maybe_unused_result, statement);
2848	}
2849	return noreturn_src_node;
2850	}
2851
2852	fn countDefers(outer_scope: Scope, inner_scope: Scope) struct {
2853	have_any: bool,
2854	have_normal: bool,
2855	have_err: bool,
2856	need_err_code: bool,
2857	} {
2858	var have_normal = false;
2859	var have_err = false;
2860	var need_err_code = false;
2861	var scope = inner_scope;
2862	while (scope != outer_scope) {
2863	switch (scope.tag) {
2864	.gen_zir => scope = scope.cast(GenZir).?.parent,
2865	.local_val => scope = scope.cast(Scope.LocalVal).?.parent,
2866	.local_ptr => scope = scope.cast(Scope.LocalPtr).?.parent,
2867	.defer_normal => {
2868	const defer_scope = scope.cast(Scope.Defer).?;
2869	scope = defer_scope.parent;
2870
2871	have_normal = true;
2872	},
2873	.defer_error => {
2874	const defer_scope = scope.cast(Scope.Defer).?;
2875	scope = defer_scope.parent;
2876
2877	have_err = true;
2878
2879	const have_err_payload = defer_scope.remapped_err_code != .none;
2880	need_err_code = need_err_code or have_err_payload;
2881	},
2882	.namespace, .enum_namespace => unreachable,
2883	.top => unreachable,
2884	}
2885	}
2886	return .{
2887	.have_any = have_normal or have_err,
2888	.have_normal = have_normal,
2889	.have_err = have_err,
2890	.need_err_code = need_err_code,
2891	};
2892	}
2893
2894	const DefersToEmit = union(enum) {
2895	both: Zir.Inst.Ref, // err code
2896	both_sans_err,
2897	normal_only,
2898	};
2899
2900	fn genDefers(
2901	gz: *GenZir,
2902	outer_scope: *Scope,
2903	inner_scope: *Scope,
2904	which_ones: DefersToEmit,
2905	) InnerError!void {
2906	const gpa = gz.astgen.gpa;
2907
2908	var scope = inner_scope;
2909	while (scope != outer_scope) {
2910	switch (scope.tag) {
2911	.gen_zir => scope = scope.cast(GenZir).?.parent,
2912	.local_val => scope = scope.cast(Scope.LocalVal).?.parent,
2913	.local_ptr => scope = scope.cast(Scope.LocalPtr).?.parent,
2914	.defer_normal => {
2915	const defer_scope = scope.cast(Scope.Defer).?;
2916	scope = defer_scope.parent;
2917	try gz.addDefer(defer_scope.index, defer_scope.len);
2918	},
2919	.defer_error => {
2920	const defer_scope = scope.cast(Scope.Defer).?;
2921	scope = defer_scope.parent;
2922	switch (which_ones) {
2923	.both_sans_err => {
2924	try gz.addDefer(defer_scope.index, defer_scope.len);
2925	},
2926	.both => \|err_code\| {
2927	if (defer_scope.remapped_err_code.unwrap()) \|remapped_err_code\| {
2928	try gz.instructions.ensureUnusedCapacity(gpa, 1);
2929	try gz.astgen.instructions.ensureUnusedCapacity(gpa, 1);
2930
2931	const payload_index = try gz.astgen.addExtra(Zir.Inst.DeferErrCode{
2932	.remapped_err_code = remapped_err_code,
2933	.index = defer_scope.index,
2934	.len = defer_scope.len,
2935	});
2936	const new_index: Zir.Inst.Index = @enumFromInt(gz.astgen.instructions.len);
2937	gz.astgen.instructions.appendAssumeCapacity(.{
2938	.tag = .defer_err_code,
2939	.data = .{ .defer_err_code = .{
2940	.err_code = err_code,
2941	.payload_index = payload_index,
2942	} },
2943	});
2944	gz.instructions.appendAssumeCapacity(new_index);
2945	} else {
2946	try gz.addDefer(defer_scope.index, defer_scope.len);
2947	}
2948	},
2949	.normal_only => continue,
2950	}
2951	},
2952	.namespace, .enum_namespace => unreachable,
2953	.top => unreachable,
2954	}
2955	}
2956	}
2957
2958	fn checkUsed(gz: GenZir, outer_scope: Scope, inner_scope: *Scope) InnerError!void {
2959	const astgen = gz.astgen;
2960
2961	var scope = inner_scope;
2962	while (scope != outer_scope) {
2963	switch (scope.tag) {
2964	.gen_zir => scope = scope.cast(GenZir).?.parent,
2965	.local_val => {
2966	const s = scope.cast(Scope.LocalVal).?;
2967	if (s.used == 0 and s.discarded == 0) {
2968	try astgen.appendErrorTok(s.token_src, "unused {s}", .{@tagName(s.id_cat)});
2969	} else if (s.used != 0 and s.discarded != 0) {
2970	try astgen.appendErrorTokNotes(s.discarded, "pointless discard of {s}", .{@tagName(s.id_cat)}, &[_]u32{
2971	try gz.astgen.errNoteTok(s.used, "used here", .{}),
2972	});
2973	}
2974	scope = s.parent;
2975	},
2976	.local_ptr => {
2977	const s = scope.cast(Scope.LocalPtr).?;
2978	if (s.used == 0 and s.discarded == 0) {
2979	try astgen.appendErrorTok(s.token_src, "unused {s}", .{@tagName(s.id_cat)});
2980	} else {
2981	if (s.used != 0 and s.discarded != 0) {
2982	try astgen.appendErrorTokNotes(s.discarded, "pointless discard of {s}", .{@tagName(s.id_cat)}, &[_]u32{
2983	try astgen.errNoteTok(s.used, "used here", .{}),
2984	});
2985	}
2986	if (s.id_cat == .@"local variable" and !s.used_as_lvalue) {
2987	try astgen.appendErrorTokNotes(s.token_src, "local variable is never mutated", .{}, &.{
2988	try astgen.errNoteTok(s.token_src, "consider using 'const'", .{}),
2989	});
2990	}
2991	}
2992
2993	scope = s.parent;
2994	},
2995	.defer_normal, .defer_error => scope = scope.cast(Scope.Defer).?.parent,
2996	.namespace, .enum_namespace => unreachable,
2997	.top => unreachable,
2998	}
2999	}
3000	}
3001
3002	fn deferStmt(
3003	gz: *GenZir,
3004	scope: *Scope,
3005	node: Ast.Node.Index,
3006	block_arena: Allocator,
3007	scope_tag: Scope.Tag,
3008	) InnerError!*Scope {
3009	var defer_gen = gz.makeSubBlock(scope);
3010	defer_gen.cur_defer_node = node;
3011	defer_gen.any_defer_node = node;
3012	defer defer_gen.unstack();
3013
3014	const tree = gz.astgen.tree;
3015	const node_datas = tree.nodes.items(.data);
3016	const expr_node = node_datas[node].rhs;
3017
3018	const payload_token = node_datas[node].lhs;
3019	var local_val_scope: Scope.LocalVal = undefined;
3020	var opt_remapped_err_code: Zir.Inst.OptionalIndex = .none;
3021	const have_err_code = scope_tag == .defer_error and payload_token != 0;
3022	const sub_scope = if (!have_err_code) &defer_gen.base else blk: {
3023	const ident_name = try gz.astgen.identAsString(payload_token);
3024	const remapped_err_code: Zir.Inst.Index = @enumFromInt(gz.astgen.instructions.len);
3025	opt_remapped_err_code = remapped_err_code.toOptional();
3026	try gz.astgen.instructions.append(gz.astgen.gpa, .{
3027	.tag = .extended,
3028	.data = .{ .extended = .{
3029	.opcode = .value_placeholder,
3030	.small = undefined,
3031	.operand = undefined,
3032	} },
3033	});
3034	const remapped_err_code_ref = remapped_err_code.toRef();
3035	local_val_scope = .{
3036	.parent = &defer_gen.base,
3037	.gen_zir = gz,
3038	.name = ident_name,
3039	.inst = remapped_err_code_ref,
3040	.token_src = payload_token,
3041	.id_cat = .capture,
3042	};
3043	try gz.addDbgVar(.dbg_var_val, ident_name, remapped_err_code_ref);
3044	break :blk &local_val_scope.base;
3045	};
3046	_ = try unusedResultExpr(&defer_gen, sub_scope, expr_node);
3047	try checkUsed(gz, scope, sub_scope);
3048	_ = try defer_gen.addBreak(.break_inline, @enumFromInt(0), .void_value);
3049
3050	// We must handle ref_table for remapped_err_code manually.
3051	const body = defer_gen.instructionsSlice();
3052	const body_len = blk: {
3053	var refs: u32 = 0;
3054	if (opt_remapped_err_code.unwrap()) \|remapped_err_code\| {
3055	var cur_inst = remapped_err_code;
3056	while (gz.astgen.ref_table.get(cur_inst)) \|ref_inst\| {
3057	refs += 1;
3058	cur_inst = ref_inst;
3059	}
3060	}
3061	break :blk gz.astgen.countBodyLenAfterFixups(body) + refs;
3062	};
3063
3064	const index: u32 = @intCast(gz.astgen.extra.items.len);
3065	try gz.astgen.extra.ensureUnusedCapacity(gz.astgen.gpa, body_len);
3066	if (opt_remapped_err_code.unwrap()) \|remapped_err_code\| {
3067	if (gz.astgen.ref_table.fetchRemove(remapped_err_code)) \|kv\| {
3068	gz.astgen.appendPossiblyRefdBodyInst(&gz.astgen.extra, kv.value);
3069	}
3070	}
3071	gz.astgen.appendBodyWithFixups(body);
3072
3073	const defer_scope = try block_arena.create(Scope.Defer);
3074
3075	defer_scope.* = .{
3076	.base = .{ .tag = scope_tag },
3077	.parent = scope,
3078	.index = index,
3079	.len = body_len,
3080	.remapped_err_code = opt_remapped_err_code,
3081	};
3082	return &defer_scope.base;
3083	}
3084
3085	fn varDecl(
3086	gz: *GenZir,
3087	scope: *Scope,
3088	node: Ast.Node.Index,
3089	block_arena: Allocator,
3090	var_decl: Ast.full.VarDecl,
3091	) InnerError!*Scope {
3092	try emitDbgNode(gz, node);
3093	const astgen = gz.astgen;
3094	const tree = astgen.tree;
3095	const token_tags = tree.tokens.items(.tag);
3096	const main_tokens = tree.nodes.items(.main_token);
3097
3098	const name_token = var_decl.ast.mut_token + 1;
3099	const ident_name_raw = tree.tokenSlice(name_token);
3100	if (mem.eql(u8, ident_name_raw, "_")) {
3101	return astgen.failTok(name_token, "'_' used as an identifier without @\"_\" syntax", .{});
3102	}
3103	const ident_name = try astgen.identAsString(name_token);
3104
3105	try astgen.detectLocalShadowing(
3106	scope,
3107	ident_name,
3108	name_token,
3109	ident_name_raw,
3110	if (token_tags[var_decl.ast.mut_token] == .keyword_const) .@"local constant" else .@"local variable",
3111	);
3112
3113	if (var_decl.ast.init_node == 0) {
3114	return astgen.failNode(node, "variables must be initialized", .{});
3115	}
3116
3117	if (var_decl.ast.addrspace_node != 0) {
3118	return astgen.failTok(main_tokens[var_decl.ast.addrspace_node], "cannot set address space of local variable '{s}'", .{ident_name_raw});
3119	}
3120
3121	if (var_decl.ast.section_node != 0) {
3122	return astgen.failTok(main_tokens[var_decl.ast.section_node], "cannot set section of local variable '{s}'", .{ident_name_raw});
3123	}
3124
3125	const align_inst: Zir.Inst.Ref = if (var_decl.ast.align_node != 0)
3126	try expr(gz, scope, coerced_align_ri, var_decl.ast.align_node)
3127	else
3128	.none;
3129
3130	switch (token_tags[var_decl.ast.mut_token]) {
3131	.keyword_const => {
3132	if (var_decl.comptime_token) \|comptime_token\| {
3133	try astgen.appendErrorTok(comptime_token, "'comptime const' is redundant; instead wrap the initialization expression with 'comptime'", .{});
3134	}
3135
3136	// Depending on the type of AST the initialization expression is, we may need an lvalue
3137	// or an rvalue as a result location. If it is an rvalue, we can use the instruction as
3138	// the variable, no memory location needed.
3139	const type_node = var_decl.ast.type_node;
3140	if (align_inst == .none and
3141	!astgen.nodes_need_rl.contains(node))
3142	{
3143	const result_info: ResultInfo = if (type_node != 0) .{
3144	.rl = .{ .ty = try typeExpr(gz, scope, type_node) },
3145	.ctx = .const_init,
3146	} else .{ .rl = .none, .ctx = .const_init };
3147	const prev_anon_name_strategy = gz.anon_name_strategy;
3148	gz.anon_name_strategy = .dbg_var;
3149	const init_inst = try reachableExpr(gz, scope, result_info, var_decl.ast.init_node, node);
3150	gz.anon_name_strategy = prev_anon_name_strategy;
3151
3152	try gz.addDbgVar(.dbg_var_val, ident_name, init_inst);
3153
3154	// The const init expression may have modified the error return trace, so signal
3155	// to Sema that it should save the new index for restoring later.
3156	if (nodeMayAppendToErrorTrace(tree, var_decl.ast.init_node))
3157	_ = try gz.addSaveErrRetIndex(.{ .if_of_error_type = init_inst });
3158
3159	const sub_scope = try block_arena.create(Scope.LocalVal);
3160	sub_scope.* = .{
3161	.parent = scope,
3162	.gen_zir = gz,
3163	.name = ident_name,
3164	.inst = init_inst,
3165	.token_src = name_token,
3166	.id_cat = .@"local constant",
3167	};
3168	return &sub_scope.base;
3169	}
3170
3171	const is_comptime = gz.is_comptime or
3172	tree.nodes.items(.tag)[var_decl.ast.init_node] == .@"comptime";
3173
3174	var resolve_inferred_alloc: Zir.Inst.Ref = .none;
3175	var opt_type_inst: Zir.Inst.Ref = .none;
3176	const init_rl: ResultInfo.Loc = if (type_node != 0) init_rl: {
3177	const type_inst = try typeExpr(gz, scope, type_node);
3178	opt_type_inst = type_inst;
3179	if (align_inst == .none) {
3180	break :init_rl .{ .ptr = .{ .inst = try gz.addUnNode(.alloc, type_inst, node) } };
3181	} else {
3182	break :init_rl .{ .ptr = .{ .inst = try gz.addAllocExtended(.{
3183	.node = node,
3184	.type_inst = type_inst,
3185	.align_inst = align_inst,
3186	.is_const = true,
3187	.is_comptime = is_comptime,
3188	}) } };
3189	}
3190	} else init_rl: {
3191	const alloc_inst = if (align_inst == .none) ptr: {
3192	const tag: Zir.Inst.Tag = if (is_comptime)
3193	.alloc_inferred_comptime
3194	else
3195	.alloc_inferred;
3196	break :ptr try gz.addNode(tag, node);
3197	} else ptr: {
3198	break :ptr try gz.addAllocExtended(.{
3199	.node = node,
3200	.type_inst = .none,
3201	.align_inst = align_inst,
3202	.is_const = true,
3203	.is_comptime = is_comptime,
3204	});
3205	};
3206	resolve_inferred_alloc = alloc_inst;
3207	break :init_rl .{ .inferred_ptr = alloc_inst };
3208	};
3209	const var_ptr = switch (init_rl) {
3210	.ptr => \|ptr\| ptr.inst,
3211	.inferred_ptr => \|inst\| inst,
3212	else => unreachable,
3213	};
3214	const init_result_info: ResultInfo = .{ .rl = init_rl, .ctx = .const_init };
3215
3216	const prev_anon_name_strategy = gz.anon_name_strategy;
3217	gz.anon_name_strategy = .dbg_var;
3218	defer gz.anon_name_strategy = prev_anon_name_strategy;
3219	const init_inst = try reachableExpr(gz, scope, init_result_info, var_decl.ast.init_node, node);
3220
3221	// The const init expression may have modified the error return trace, so signal
3222	// to Sema that it should save the new index for restoring later.
3223	if (nodeMayAppendToErrorTrace(tree, var_decl.ast.init_node))
3224	_ = try gz.addSaveErrRetIndex(.{ .if_of_error_type = init_inst });
3225
3226	const const_ptr = if (resolve_inferred_alloc != .none) p: {
3227	_ = try gz.addUnNode(.resolve_inferred_alloc, resolve_inferred_alloc, node);
3228	break :p var_ptr;
3229	} else try gz.addUnNode(.make_ptr_const, var_ptr, node);
3230
3231	try gz.addDbgVar(.dbg_var_ptr, ident_name, const_ptr);
3232
3233	const sub_scope = try block_arena.create(Scope.LocalPtr);
3234	sub_scope.* = .{
3235	.parent = scope,
3236	.gen_zir = gz,
3237	.name = ident_name,
3238	.ptr = const_ptr,
3239	.token_src = name_token,
3240	.maybe_comptime = true,
3241	.id_cat = .@"local constant",
3242	};
3243	return &sub_scope.base;
3244	},
3245	.keyword_var => {
3246	if (var_decl.comptime_token != null and gz.is_comptime)
3247	return astgen.failTok(var_decl.comptime_token.?, "'comptime var' is redundant in comptime scope", .{});
3248	const is_comptime = var_decl.comptime_token != null or gz.is_comptime;
3249	var resolve_inferred_alloc: Zir.Inst.Ref = .none;
3250	const alloc: Zir.Inst.Ref, const result_info: ResultInfo = if (var_decl.ast.type_node != 0) a: {
3251	const type_inst = try typeExpr(gz, scope, var_decl.ast.type_node);
3252	const alloc = alloc: {
3253	if (align_inst == .none) {
3254	const tag: Zir.Inst.Tag = if (is_comptime)
3255	.alloc_comptime_mut
3256	else
3257	.alloc_mut;
3258	break :alloc try gz.addUnNode(tag, type_inst, node);
3259	} else {
3260	break :alloc try gz.addAllocExtended(.{
3261	.node = node,
3262	.type_inst = type_inst,
3263	.align_inst = align_inst,
3264	.is_const = false,
3265	.is_comptime = is_comptime,
3266	});
3267	}
3268	};
3269	break :a .{ alloc, .{ .rl = .{ .ptr = .{ .inst = alloc } } } };
3270	} else a: {
3271	const alloc = alloc: {
3272	if (align_inst == .none) {
3273	const tag: Zir.Inst.Tag = if (is_comptime)
3274	.alloc_inferred_comptime_mut
3275	else
3276	.alloc_inferred_mut;
3277	break :alloc try gz.addNode(tag, node);
3278	} else {
3279	break :alloc try gz.addAllocExtended(.{
3280	.node = node,
3281	.type_inst = .none,
3282	.align_inst = align_inst,
3283	.is_const = false,
3284	.is_comptime = is_comptime,
3285	});
3286	}
3287	};
3288	resolve_inferred_alloc = alloc;
3289	break :a .{ alloc, .{ .rl = .{ .inferred_ptr = alloc } } };
3290	};
3291	const prev_anon_name_strategy = gz.anon_name_strategy;
3292	gz.anon_name_strategy = .dbg_var;
3293	_ = try reachableExprComptime(gz, scope, result_info, var_decl.ast.init_node, node, is_comptime);
3294	gz.anon_name_strategy = prev_anon_name_strategy;
3295	if (resolve_inferred_alloc != .none) {
3296	_ = try gz.addUnNode(.resolve_inferred_alloc, resolve_inferred_alloc, node);
3297	}
3298
3299	try gz.addDbgVar(.dbg_var_ptr, ident_name, alloc);
3300
3301	const sub_scope = try block_arena.create(Scope.LocalPtr);
3302	sub_scope.* = .{
3303	.parent = scope,
3304	.gen_zir = gz,
3305	.name = ident_name,
3306	.ptr = alloc,
3307	.token_src = name_token,
3308	.maybe_comptime = is_comptime,
3309	.id_cat = .@"local variable",
3310	};
3311	return &sub_scope.base;
3312	},
3313	else => unreachable,
3314	}
3315	}
3316
3317	fn emitDbgNode(gz: *GenZir, node: Ast.Node.Index) !void {
3318	// The instruction emitted here is for debugging runtime code.
3319	// If the current block will be evaluated only during semantic analysis
3320	// then no dbg_stmt ZIR instruction is needed.
3321	if (gz.is_comptime) return;
3322	const astgen = gz.astgen;
3323	astgen.advanceSourceCursorToNode(node);
3324	const line = astgen.source_line - gz.decl_line;
3325	const column = astgen.source_column;
3326	try emitDbgStmt(gz, .{ line, column });
3327	}
3328
3329	fn assign(gz: GenZir, scope: Scope, infix_node: Ast.Node.Index) InnerError!void {
3330	try emitDbgNode(gz, infix_node);
3331	const astgen = gz.astgen;
3332	const tree = astgen.tree;
3333	const node_datas = tree.nodes.items(.data);
3334	const main_tokens = tree.nodes.items(.main_token);
3335	const node_tags = tree.nodes.items(.tag);
3336
3337	const lhs = node_datas[infix_node].lhs;
3338	const rhs = node_datas[infix_node].rhs;
3339	if (node_tags[lhs] == .identifier) {
3340	// This intentionally does not support `@"_"` syntax.
3341	const ident_name = tree.tokenSlice(main_tokens[lhs]);
3342	if (mem.eql(u8, ident_name, "_")) {
3343	_ = try expr(gz, scope, .{ .rl = .discard, .ctx = .assignment }, rhs);
3344	return;
3345	}
3346	}
3347	const lvalue = try lvalExpr(gz, scope, lhs);
3348	_ = try expr(gz, scope, .{ .rl = .{ .ptr = .{
3349	.inst = lvalue,
3350	.src_node = infix_node,
3351	} } }, rhs);
3352	}
3353
3354	/// Handles destructure assignments where no LHS is a `const` or `var` decl.
3355	fn assignDestructure(gz: GenZir, scope: Scope, node: Ast.Node.Index) InnerError!void {
3356	try emitDbgNode(gz, node);
3357	const astgen = gz.astgen;
3358	const tree = astgen.tree;
3359	const token_tags = tree.tokens.items(.tag);
3360	const node_datas = tree.nodes.items(.data);
3361	const main_tokens = tree.nodes.items(.main_token);
3362	const node_tags = tree.nodes.items(.tag);
3363
3364	const extra_index = node_datas[node].lhs;
3365	const lhs_count = tree.extra_data[extra_index];
3366	const lhs_nodes: []const Ast.Node.Index = @ptrCast(tree.extra_data[extra_index + 1 ..][0..lhs_count]);
3367	const rhs = node_datas[node].rhs;
3368
3369	const maybe_comptime_token = tree.firstToken(node) - 1;
3370	const declared_comptime = token_tags[maybe_comptime_token] == .keyword_comptime;
3371
3372	if (declared_comptime and gz.is_comptime) {
3373	return astgen.failNode(node, "redundant comptime keyword in already comptime scope", .{});
3374	}
3375
3376	// If this expression is marked comptime, we must wrap the whole thing in a comptime block.
3377	var gz_buf: GenZir = undefined;
3378	const inner_gz = if (declared_comptime) bs: {
3379	gz_buf = gz.makeSubBlock(scope);
3380	gz_buf.is_comptime = true;
3381	break :bs &gz_buf;
3382	} else gz;
3383	defer if (declared_comptime) inner_gz.unstack();
3384
3385	const rl_components = try astgen.arena.alloc(ResultInfo.Loc.DestructureComponent, lhs_nodes.len);
3386	for (rl_components, lhs_nodes) \|*lhs_rl, lhs_node\| {
3387	if (node_tags[lhs_node] == .identifier) {
3388	// This intentionally does not support `@"_"` syntax.
3389	const ident_name = tree.tokenSlice(main_tokens[lhs_node]);
3390	if (mem.eql(u8, ident_name, "_")) {
3391	lhs_rl.* = .discard;
3392	continue;
3393	}
3394	}
3395	lhs_rl.* = .{ .typed_ptr = .{
3396	.inst = try lvalExpr(inner_gz, scope, lhs_node),
3397	.src_node = lhs_node,
3398	} };
3399	}
3400
3401	const ri: ResultInfo = .{ .rl = .{ .destructure = .{
3402	.src_node = node,
3403	.components = rl_components,
3404	} } };
3405
3406	_ = try expr(inner_gz, scope, ri, rhs);
3407
3408	if (declared_comptime) {
3409	const comptime_block_inst = try gz.makeBlockInst(.block_comptime, node);
3410	_ = try inner_gz.addBreak(.@"break", comptime_block_inst, .void_value);
3411	try inner_gz.setBlockBody(comptime_block_inst);
3412	try gz.instructions.append(gz.astgen.gpa, comptime_block_inst);
3413	}
3414	}
3415
3416	/// Handles destructure assignments where the LHS may contain `const` or `var` decls.
3417	fn assignDestructureMaybeDecls(
3418	gz: *GenZir,
3419	scope: *Scope,
3420	node: Ast.Node.Index,
3421	block_arena: Allocator,
3422	) InnerError!*Scope {
3423	try emitDbgNode(gz, node);
3424	const astgen = gz.astgen;
3425	const tree = astgen.tree;
3426	const token_tags = tree.tokens.items(.tag);
3427	const node_datas = tree.nodes.items(.data);
3428	const main_tokens = tree.nodes.items(.main_token);
3429	const node_tags = tree.nodes.items(.tag);
3430
3431	const extra_index = node_datas[node].lhs;
3432	const lhs_count = tree.extra_data[extra_index];
3433	const lhs_nodes: []const Ast.Node.Index = @ptrCast(tree.extra_data[extra_index + 1 ..][0..lhs_count]);
3434	const rhs = node_datas[node].rhs;
3435
3436	const maybe_comptime_token = tree.firstToken(node) - 1;
3437	const declared_comptime = token_tags[maybe_comptime_token] == .keyword_comptime;
3438	if (declared_comptime and gz.is_comptime) {
3439	return astgen.failNode(node, "redundant comptime keyword in already comptime scope", .{});
3440	}
3441
3442	const is_comptime = declared_comptime or gz.is_comptime;
3443	const rhs_is_comptime = tree.nodes.items(.tag)[rhs] == .@"comptime";
3444
3445	// When declaring consts via a destructure, we always use a result pointer.
3446	// This avoids the need to create tuple types, and is also likely easier to
3447	// optimize, since it's a bit tricky for the optimizer to "split up" the
3448	// value into individual pointer writes down the line.
3449
3450	// We know this rl information won't live past the evaluation of this
3451	// expression, so it may as well go in the block arena.
3452	const rl_components = try block_arena.alloc(ResultInfo.Loc.DestructureComponent, lhs_nodes.len);
3453	var any_non_const_lhs = false;
3454	var any_lvalue_expr = false;
3455	for (rl_components, lhs_nodes) \|*lhs_rl, lhs_node\| {
3456	switch (node_tags[lhs_node]) {
3457	.identifier => {
3458	// This intentionally does not support `@"_"` syntax.
3459	const ident_name = tree.tokenSlice(main_tokens[lhs_node]);
3460	if (mem.eql(u8, ident_name, "_")) {
3461	any_non_const_lhs = true;
3462	lhs_rl.* = .discard;
3463	continue;
3464	}
3465	},
3466	.global_var_decl, .local_var_decl, .simple_var_decl, .aligned_var_decl => {
3467	const full = tree.fullVarDecl(lhs_node).?;
3468
3469	const name_token = full.ast.mut_token + 1;
3470	const ident_name_raw = tree.tokenSlice(name_token);
3471	if (mem.eql(u8, ident_name_raw, "_")) {
3472	return astgen.failTok(name_token, "'_' used as an identifier without @\"_\" syntax", .{});
3473	}
3474
3475	// We detect shadowing in the second pass over these, while we're creating scopes.
3476
3477	if (full.ast.addrspace_node != 0) {
3478	return astgen.failTok(main_tokens[full.ast.addrspace_node], "cannot set address space of local variable '{s}'", .{ident_name_raw});
3479	}
3480	if (full.ast.section_node != 0) {
3481	return astgen.failTok(main_tokens[full.ast.section_node], "cannot set section of local variable '{s}'", .{ident_name_raw});
3482	}
3483
3484	const is_const = switch (token_tags[full.ast.mut_token]) {
3485	.keyword_var => false,
3486	.keyword_const => true,
3487	else => unreachable,
3488	};
3489	if (!is_const) any_non_const_lhs = true;
3490
3491	// We also mark `const`s as comptime if the RHS is definitely comptime-known.
3492	const this_lhs_comptime = is_comptime or (is_const and rhs_is_comptime);
3493
3494	const align_inst: Zir.Inst.Ref = if (full.ast.align_node != 0)
3495	try expr(gz, scope, coerced_align_ri, full.ast.align_node)
3496	else
3497	.none;
3498
3499	if (full.ast.type_node != 0) {
3500	// Typed alloc
3501	const type_inst = try typeExpr(gz, scope, full.ast.type_node);
3502	const ptr = if (align_inst == .none) ptr: {
3503	const tag: Zir.Inst.Tag = if (is_const)
3504	.alloc
3505	else if (this_lhs_comptime)
3506	.alloc_comptime_mut
3507	else
3508	.alloc_mut;
3509	break :ptr try gz.addUnNode(tag, type_inst, node);
3510	} else try gz.addAllocExtended(.{
3511	.node = node,
3512	.type_inst = type_inst,
3513	.align_inst = align_inst,
3514	.is_const = is_const,
3515	.is_comptime = this_lhs_comptime,
3516	});
3517	lhs_rl.* = .{ .typed_ptr = .{ .inst = ptr } };
3518	} else {
3519	// Inferred alloc
3520	const ptr = if (align_inst == .none) ptr: {
3521	const tag: Zir.Inst.Tag = if (is_const) tag: {
3522	break :tag if (this_lhs_comptime) .alloc_inferred_comptime else .alloc_inferred;
3523	} else tag: {
3524	break :tag if (this_lhs_comptime) .alloc_inferred_comptime_mut else .alloc_inferred_mut;
3525	};
3526	break :ptr try gz.addNode(tag, node);
3527	} else try gz.addAllocExtended(.{
3528	.node = node,
3529	.type_inst = .none,
3530	.align_inst = align_inst,
3531	.is_const = is_const,
3532	.is_comptime = this_lhs_comptime,
3533	});
3534	lhs_rl.* = .{ .inferred_ptr = ptr };
3535	}
3536
3537	continue;
3538	},
3539	else => {},
3540	}
3541	// This LHS is just an lvalue expression.
3542	// We will fill in its result pointer later, inside a comptime block.
3543	any_non_const_lhs = true;
3544	any_lvalue_expr = true;
3545	lhs_rl.* = .{ .typed_ptr = .{
3546	.inst = undefined,
3547	.src_node = lhs_node,
3548	} };
3549	}
3550
3551	if (declared_comptime and !any_non_const_lhs) {
3552	try astgen.appendErrorTok(maybe_comptime_token, "'comptime const' is redundant; instead wrap the initialization expression with 'comptime'", .{});
3553	}
3554
3555	// If this expression is marked comptime, we must wrap it in a comptime block.
3556	var gz_buf: GenZir = undefined;
3557	const inner_gz = if (declared_comptime) bs: {
3558	gz_buf = gz.makeSubBlock(scope);
3559	gz_buf.is_comptime = true;
3560	break :bs &gz_buf;
3561	} else gz;
3562	defer if (declared_comptime) inner_gz.unstack();
3563
3564	if (any_lvalue_expr) {
3565	// At least one LHS was an lvalue expr. Iterate again in order to
3566	// evaluate the lvalues from within the possible block_comptime.
3567	for (rl_components, lhs_nodes) \|*lhs_rl, lhs_node\| {
3568	if (lhs_rl.* != .typed_ptr) continue;
3569	switch (node_tags[lhs_node]) {
3570	.global_var_decl, .local_var_decl, .simple_var_decl, .aligned_var_decl => continue,
3571	else => {},
3572	}
3573	lhs_rl.typed_ptr.inst = try lvalExpr(inner_gz, scope, lhs_node);
3574	}
3575	}
3576
3577	// We can't give a reasonable anon name strategy for destructured inits, so
3578	// leave it at its default of `.anon`.
3579	_ = try reachableExpr(inner_gz, scope, .{ .rl = .{ .destructure = .{
3580	.src_node = node,
3581	.components = rl_components,
3582	} } }, rhs, node);
3583
3584	if (declared_comptime) {
3585	// Finish the block_comptime. Inferred alloc resolution etc will occur
3586	// in the parent block.
3587	const comptime_block_inst = try gz.makeBlockInst(.block_comptime, node);
3588	_ = try inner_gz.addBreak(.@"break", comptime_block_inst, .void_value);
3589	try inner_gz.setBlockBody(comptime_block_inst);
3590	try gz.instructions.append(gz.astgen.gpa, comptime_block_inst);
3591	}
3592
3593	// Now, iterate over the LHS exprs to construct any new scopes.
3594	// If there were any inferred allocations, resolve them.
3595	// If there were any `const` decls, make the pointer constant.
3596	var cur_scope = scope;
3597	for (rl_components, lhs_nodes) \|lhs_rl, lhs_node\| {
3598	switch (node_tags[lhs_node]) {
3599	.local_var_decl, .simple_var_decl, .aligned_var_decl => {},
3600	else => continue, // We were mutating an existing lvalue - nothing to do
3601	}
3602	const full = tree.fullVarDecl(lhs_node).?;
3603	const raw_ptr = switch (lhs_rl) {
3604	.discard => unreachable,
3605	.typed_ptr => \|typed_ptr\| typed_ptr.inst,
3606	.inferred_ptr => \|ptr_inst\| ptr_inst,
3607	};
3608	// If the alloc was inferred, resolve it.
3609	if (full.ast.type_node == 0) {
3610	_ = try gz.addUnNode(.resolve_inferred_alloc, raw_ptr, lhs_node);
3611	}
3612	const is_const = switch (token_tags[full.ast.mut_token]) {
3613	.keyword_var => false,
3614	.keyword_const => true,
3615	else => unreachable,
3616	};
3617	// If the alloc was const, make it const.
3618	const var_ptr = if (is_const and full.ast.type_node != 0) make_const: {
3619	// Note that we don't do this if type_node == 0 since `resolve_inferred_alloc`
3620	// handles it for us.
3621	break :make_const try gz.addUnNode(.make_ptr_const, raw_ptr, node);
3622	} else raw_ptr;
3623	const name_token = full.ast.mut_token + 1;
3624	const ident_name_raw = tree.tokenSlice(name_token);
3625	const ident_name = try astgen.identAsString(name_token);
3626	try astgen.detectLocalShadowing(
3627	cur_scope,
3628	ident_name,
3629	name_token,
3630	ident_name_raw,
3631	if (is_const) .@"local constant" else .@"local variable",
3632	);
3633	try gz.addDbgVar(.dbg_var_ptr, ident_name, var_ptr);
3634	// Finally, create the scope.
3635	const sub_scope = try block_arena.create(Scope.LocalPtr);
3636	sub_scope.* = .{
3637	.parent = cur_scope,
3638	.gen_zir = gz,
3639	.name = ident_name,
3640	.ptr = var_ptr,
3641	.token_src = name_token,
3642	.maybe_comptime = is_const or is_comptime,
3643	.id_cat = if (is_const) .@"local constant" else .@"local variable",
3644	};
3645	cur_scope = &sub_scope.base;
3646	}
3647
3648	return cur_scope;
3649	}
3650
3651	fn assignOp(
3652	gz: *GenZir,
3653	scope: *Scope,
3654	infix_node: Ast.Node.Index,
3655	op_inst_tag: Zir.Inst.Tag,
3656	) InnerError!void {
3657	try emitDbgNode(gz, infix_node);
3658	const astgen = gz.astgen;
3659	const tree = astgen.tree;
3660	const node_datas = tree.nodes.items(.data);
3661
3662	const lhs_ptr = try lvalExpr(gz, scope, node_datas[infix_node].lhs);
3663
3664	const cursor = switch (op_inst_tag) {
3665	.add, .sub, .mul, .div, .mod_rem => maybeAdvanceSourceCursorToMainToken(gz, infix_node),
3666	else => undefined,
3667	};
3668	const lhs = try gz.addUnNode(.load, lhs_ptr, infix_node);
3669	const lhs_type = try gz.addUnNode(.typeof, lhs, infix_node);
3670	const rhs = try expr(gz, scope, .{ .rl = .{ .coerced_ty = lhs_type } }, node_datas[infix_node].rhs);
3671
3672	switch (op_inst_tag) {
3673	.add, .sub, .mul, .div, .mod_rem => {
3674	try emitDbgStmt(gz, cursor);
3675	},
3676	else => {},
3677	}
3678	const result = try gz.addPlNode(op_inst_tag, infix_node, Zir.Inst.Bin{
3679	.lhs = lhs,
3680	.rhs = rhs,
3681	});
3682	_ = try gz.addPlNode(.store_node, infix_node, Zir.Inst.Bin{
3683	.lhs = lhs_ptr,
3684	.rhs = result,
3685	});
3686	}
3687
3688	fn assignShift(
3689	gz: *GenZir,
3690	scope: *Scope,
3691	infix_node: Ast.Node.Index,
3692	op_inst_tag: Zir.Inst.Tag,
3693	) InnerError!void {
3694	try emitDbgNode(gz, infix_node);
3695	const astgen = gz.astgen;
3696	const tree = astgen.tree;
3697	const node_datas = tree.nodes.items(.data);
3698
3699	const lhs_ptr = try lvalExpr(gz, scope, node_datas[infix_node].lhs);
3700	const lhs = try gz.addUnNode(.load, lhs_ptr, infix_node);
3701	const rhs_type = try gz.addUnNode(.typeof_log2_int_type, lhs, infix_node);
3702	const rhs = try expr(gz, scope, .{ .rl = .{ .ty = rhs_type } }, node_datas[infix_node].rhs);
3703
3704	const result = try gz.addPlNode(op_inst_tag, infix_node, Zir.Inst.Bin{
3705	.lhs = lhs,
3706	.rhs = rhs,
3707	});
3708	_ = try gz.addPlNode(.store_node, infix_node, Zir.Inst.Bin{
3709	.lhs = lhs_ptr,
3710	.rhs = result,
3711	});
3712	}
3713
3714	fn assignShiftSat(gz: GenZir, scope: Scope, infix_node: Ast.Node.Index) InnerError!void {
3715	try emitDbgNode(gz, infix_node);
3716	const astgen = gz.astgen;
3717	const tree = astgen.tree;
3718	const node_datas = tree.nodes.items(.data);
3719
3720	const lhs_ptr = try lvalExpr(gz, scope, node_datas[infix_node].lhs);
3721	const lhs = try gz.addUnNode(.load, lhs_ptr, infix_node);
3722	// Saturating shift-left allows any integer type for both the LHS and RHS.
3723	const rhs = try expr(gz, scope, .{ .rl = .none }, node_datas[infix_node].rhs);
3724
3725	const result = try gz.addPlNode(.shl_sat, infix_node, Zir.Inst.Bin{
3726	.lhs = lhs,
3727	.rhs = rhs,
3728	});
3729	_ = try gz.addPlNode(.store_node, infix_node, Zir.Inst.Bin{
3730	.lhs = lhs_ptr,
3731	.rhs = result,
3732	});
3733	}
3734
3735	fn ptrType(
3736	gz: *GenZir,
3737	scope: *Scope,
3738	ri: ResultInfo,
3739	node: Ast.Node.Index,
3740	ptr_info: Ast.full.PtrType,
3741	) InnerError!Zir.Inst.Ref {
3742	if (ptr_info.size == .C and ptr_info.allowzero_token != null) {
3743	return gz.astgen.failTok(ptr_info.allowzero_token.?, "C pointers always allow address zero", .{});
3744	}
3745
3746	const source_offset = gz.astgen.source_offset;
3747	const source_line = gz.astgen.source_line;
3748	const source_column = gz.astgen.source_column;
3749	const elem_type = try typeExpr(gz, scope, ptr_info.ast.child_type);
3750
3751	var sentinel_ref: Zir.Inst.Ref = .none;
3752	var align_ref: Zir.Inst.Ref = .none;
3753	var addrspace_ref: Zir.Inst.Ref = .none;
3754	var bit_start_ref: Zir.Inst.Ref = .none;
3755	var bit_end_ref: Zir.Inst.Ref = .none;
3756	var trailing_count: u32 = 0;
3757
3758	if (ptr_info.ast.sentinel != 0) {
3759	// These attributes can appear in any order and they all come before the
3760	// element type so we need to reset the source cursor before generating them.
3761	gz.astgen.source_offset = source_offset;
3762	gz.astgen.source_line = source_line;
3763	gz.astgen.source_column = source_column;
3764
3765	sentinel_ref = try comptimeExpr(gz, scope, .{ .rl = .{ .ty = elem_type } }, ptr_info.ast.sentinel);
3766	trailing_count += 1;
3767	}
3768	if (ptr_info.ast.addrspace_node != 0) {
3769	gz.astgen.source_offset = source_offset;
3770	gz.astgen.source_line = source_line;
3771	gz.astgen.source_column = source_column;
3772
3773	addrspace_ref = try expr(gz, scope, coerced_addrspace_ri, ptr_info.ast.addrspace_node);
3774	trailing_count += 1;
3775	}
3776	if (ptr_info.ast.align_node != 0) {
3777	gz.astgen.source_offset = source_offset;
3778	gz.astgen.source_line = source_line;
3779	gz.astgen.source_column = source_column;
3780
3781	align_ref = try expr(gz, scope, coerced_align_ri, ptr_info.ast.align_node);
3782	trailing_count += 1;
3783	}
3784	if (ptr_info.ast.bit_range_start != 0) {
3785	assert(ptr_info.ast.bit_range_end != 0);
3786	bit_start_ref = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .u16_type } }, ptr_info.ast.bit_range_start);
3787	bit_end_ref = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .u16_type } }, ptr_info.ast.bit_range_end);
3788	trailing_count += 2;
3789	}
3790
3791	const gpa = gz.astgen.gpa;
3792	try gz.instructions.ensureUnusedCapacity(gpa, 1);
3793	try gz.astgen.instructions.ensureUnusedCapacity(gpa, 1);
3794	try gz.astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.PtrType).Struct.fields.len +
3795	trailing_count);
3796
3797	const payload_index = gz.astgen.addExtraAssumeCapacity(Zir.Inst.PtrType{
3798	.elem_type = elem_type,
3799	.src_node = gz.nodeIndexToRelative(node),
3800	});
3801	if (sentinel_ref != .none) {
3802	gz.astgen.extra.appendAssumeCapacity(@intFromEnum(sentinel_ref));
3803	}
3804	if (align_ref != .none) {
3805	gz.astgen.extra.appendAssumeCapacity(@intFromEnum(align_ref));
3806	}
3807	if (addrspace_ref != .none) {
3808	gz.astgen.extra.appendAssumeCapacity(@intFromEnum(addrspace_ref));
3809	}
3810	if (bit_start_ref != .none) {
3811	gz.astgen.extra.appendAssumeCapacity(@intFromEnum(bit_start_ref));
3812	gz.astgen.extra.appendAssumeCapacity(@intFromEnum(bit_end_ref));
3813	}
3814
3815	const new_index: Zir.Inst.Index = @enumFromInt(gz.astgen.instructions.len);
3816	const result = new_index.toRef();
3817	gz.astgen.instructions.appendAssumeCapacity(.{ .tag = .ptr_type, .data = .{
3818	.ptr_type = .{
3819	.flags = .{
3820	.is_allowzero = ptr_info.allowzero_token != null,
3821	.is_mutable = ptr_info.const_token == null,
3822	.is_volatile = ptr_info.volatile_token != null,
3823	.has_sentinel = sentinel_ref != .none,
3824	.has_align = align_ref != .none,
3825	.has_addrspace = addrspace_ref != .none,
3826	.has_bit_range = bit_start_ref != .none,
3827	},
3828	.size = ptr_info.size,
3829	.payload_index = payload_index,
3830	},
3831	} });
3832	gz.instructions.appendAssumeCapacity(new_index);
3833
3834	return rvalue(gz, ri, result, node);
3835	}
3836
3837	fn arrayType(gz: GenZir, scope: Scope, ri: ResultInfo, node: Ast.Node.Index) !Zir.Inst.Ref {
3838	const astgen = gz.astgen;
3839	const tree = astgen.tree;
3840	const node_datas = tree.nodes.items(.data);
3841	const node_tags = tree.nodes.items(.tag);
3842	const main_tokens = tree.nodes.items(.main_token);
3843
3844	const len_node = node_datas[node].lhs;
3845	if (node_tags[len_node] == .identifier and
3846	mem.eql(u8, tree.tokenSlice(main_tokens[len_node]), "_"))
3847	{
3848	return astgen.failNode(len_node, "unable to infer array size", .{});
3849	}
3850	const len = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, len_node);
3851	const elem_type = try typeExpr(gz, scope, node_datas[node].rhs);
3852
3853	const result = try gz.addPlNode(.array_type, node, Zir.Inst.Bin{
3854	.lhs = len,
3855	.rhs = elem_type,
3856	});
3857	return rvalue(gz, ri, result, node);
3858	}
3859
3860	fn arrayTypeSentinel(gz: GenZir, scope: Scope, ri: ResultInfo, node: Ast.Node.Index) !Zir.Inst.Ref {
3861	const astgen = gz.astgen;
3862	const tree = astgen.tree;
3863	const node_datas = tree.nodes.items(.data);
3864	const node_tags = tree.nodes.items(.tag);
3865	const main_tokens = tree.nodes.items(.main_token);
3866	const extra = tree.extraData(node_datas[node].rhs, Ast.Node.ArrayTypeSentinel);
3867
3868	const len_node = node_datas[node].lhs;
3869	if (node_tags[len_node] == .identifier and
3870	mem.eql(u8, tree.tokenSlice(main_tokens[len_node]), "_"))
3871	{
3872	return astgen.failNode(len_node, "unable to infer array size", .{});
3873	}
3874	const len = try reachableExpr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, len_node, node);
3875	const elem_type = try typeExpr(gz, scope, extra.elem_type);
3876	const sentinel = try reachableExprComptime(gz, scope, .{ .rl = .{ .coerced_ty = elem_type } }, extra.sentinel, node, true);
3877
3878	const result = try gz.addPlNode(.array_type_sentinel, node, Zir.Inst.ArrayTypeSentinel{
3879	.len = len,
3880	.elem_type = elem_type,
3881	.sentinel = sentinel,
3882	});
3883	return rvalue(gz, ri, result, node);
3884	}
3885
3886	const WipMembers = struct {
3887	payload: *ArrayListUnmanaged(u32),
3888	payload_top: usize,
3889	field_bits_start: u32,
3890	fields_start: u32,
3891	fields_end: u32,
3892	decl_index: u32 = 0,
3893	field_index: u32 = 0,
3894
3895	const Self = @This();
3896
3897	fn init(gpa: Allocator, payload: *ArrayListUnmanaged(u32), decl_count: u32, field_count: u32, comptime bits_per_field: u32, comptime max_field_size: u32) Allocator.Error!Self {
3898	const payload_top: u32 = @intCast(payload.items.len);
3899	const field_bits_start = payload_top + decl_count;
3900	const fields_start = field_bits_start + if (bits_per_field > 0) blk: {
3901	const fields_per_u32 = 32 / bits_per_field;
3902	break :blk (field_count + fields_per_u32 - 1) / fields_per_u32;
3903	} else 0;
3904	const payload_end = fields_start + field_count * max_field_size;
3905	try payload.resize(gpa, payload_end);
3906	return .{
3907	.payload = payload,
3908	.payload_top = payload_top,
3909	.field_bits_start = field_bits_start,
3910	.fields_start = fields_start,
3911	.fields_end = fields_start,
3912	};
3913	}
3914
3915	fn nextDecl(self: *Self, decl_inst: Zir.Inst.Index) void {
3916	self.payload.items[self.payload_top + self.decl_index] = @intFromEnum(decl_inst);
3917	self.decl_index += 1;
3918	}
3919
3920	fn nextField(self: *Self, comptime bits_per_field: u32, bits: [bits_per_field]bool) void {
3921	const fields_per_u32 = 32 / bits_per_field;
3922	const index = self.field_bits_start + self.field_index / fields_per_u32;
3923	assert(index < self.fields_start);
3924	var bit_bag: u32 = if (self.field_index % fields_per_u32 == 0) 0 else self.payload.items[index];
3925	bit_bag >>= bits_per_field;
3926	comptime var i = 0;
3927	inline while (i < bits_per_field) : (i += 1) {
3928	bit_bag \|= @as(u32, @intFromBool(bits[i])) << (32 - bits_per_field + i);
3929	}
3930	self.payload.items[index] = bit_bag;
3931	self.field_index += 1;
3932	}
3933
3934	fn appendToField(self: *Self, data: u32) void {
3935	assert(self.fields_end < self.payload.items.len);
3936	self.payload.items[self.fields_end] = data;
3937	self.fields_end += 1;
3938	}
3939
3940	fn finishBits(self: *Self, comptime bits_per_field: u32) void {
3941	if (bits_per_field > 0) {
3942	const fields_per_u32 = 32 / bits_per_field;
3943	const empty_field_slots = fields_per_u32 - (self.field_index % fields_per_u32);
3944	if (self.field_index > 0 and empty_field_slots < fields_per_u32) {
3945	const index = self.field_bits_start + self.field_index / fields_per_u32;
3946	self.payload.items[index] >>= @intCast(empty_field_slots * bits_per_field);
3947	}
3948	}
3949	}
3950
3951	fn declsSlice(self: *Self) []u32 {
3952	return self.payload.items[self.payload_top..][0..self.decl_index];
3953	}
3954
3955	fn fieldsSlice(self: *Self) []u32 {
3956	return self.payload.items[self.field_bits_start..self.fields_end];
3957	}
3958
3959	fn deinit(self: *Self) void {
3960	self.payload.items.len = self.payload_top;
3961	}
3962	};
3963
3964	fn fnDecl(
3965	astgen: *AstGen,
3966	gz: *GenZir,
3967	scope: *Scope,
3968	wip_members: *WipMembers,
3969	decl_node: Ast.Node.Index,
3970	body_node: Ast.Node.Index,
3971	fn_proto: Ast.full.FnProto,
3972	) InnerError!void {
3973	const tree = astgen.tree;
3974	const token_tags = tree.tokens.items(.tag);
3975
3976	// missing function name already happened in scanDecls()
3977	const fn_name_token = fn_proto.name_token orelse return error.AnalysisFail;
3978
3979	// We insert this at the beginning so that its instruction index marks the
3980	// start of the top level declaration.
3981	const decl_inst = try gz.makeBlockInst(.declaration, fn_proto.ast.proto_node);
3982	astgen.advanceSourceCursorToNode(decl_node);
3983
3984	var decl_gz: GenZir = .{
3985	.is_comptime = true,
3986	.decl_node_index = fn_proto.ast.proto_node,
3987	.decl_line = astgen.source_line,
3988	.parent = scope,
3989	.astgen = astgen,
3990	.instructions = gz.instructions,
3991	.instructions_top = gz.instructions.items.len,
3992	};
3993	defer decl_gz.unstack();
3994
3995	var fn_gz: GenZir = .{
3996	.is_comptime = false,
3997	.decl_node_index = fn_proto.ast.proto_node,
3998	.decl_line = decl_gz.decl_line,
3999	.parent = &decl_gz.base,
4000	.astgen = astgen,
4001	.instructions = gz.instructions,
4002	.instructions_top = GenZir.unstacked_top,
4003	};
4004	defer fn_gz.unstack();
4005
4006	const is_pub = fn_proto.visib_token != null;
4007	const is_export = blk: {
4008	const maybe_export_token = fn_proto.extern_export_inline_token orelse break :blk false;
4009	break :blk token_tags[maybe_export_token] == .keyword_export;
4010	};
4011	const is_extern = blk: {
4012	const maybe_extern_token = fn_proto.extern_export_inline_token orelse break :blk false;
4013	break :blk token_tags[maybe_extern_token] == .keyword_extern;
4014	};
4015	const has_inline_keyword = blk: {
4016	const maybe_inline_token = fn_proto.extern_export_inline_token orelse break :blk false;
4017	break :blk token_tags[maybe_inline_token] == .keyword_inline;
4018	};
4019	const is_noinline = blk: {
4020	const maybe_noinline_token = fn_proto.extern_export_inline_token orelse break :blk false;
4021	break :blk token_tags[maybe_noinline_token] == .keyword_noinline;
4022	};
4023
4024	const doc_comment_index = try astgen.docCommentAsString(fn_proto.firstToken());
4025
4026	wip_members.nextDecl(decl_inst);
4027
4028	var noalias_bits: u32 = 0;
4029	var params_scope = &fn_gz.base;
4030	const is_var_args = is_var_args: {
4031	var param_type_i: usize = 0;
4032	var it = fn_proto.iterate(tree);
4033	while (it.next()) \|param\| : (param_type_i += 1) {
4034	const is_comptime = if (param.comptime_noalias) \|token\| switch (token_tags[token]) {
4035	.keyword_noalias => is_comptime: {
4036	noalias_bits \|= @as(u32, 1) << (std.math.cast(u5, param_type_i) orelse
4037	return astgen.failTok(token, "this compiler implementation only supports 'noalias' on the first 32 parameters", .{}));
4038	break :is_comptime false;
4039	},
4040	.keyword_comptime => true,
4041	else => false,
4042	} else false;
4043
4044	const is_anytype = if (param.anytype_ellipsis3) \|token\| blk: {
4045	switch (token_tags[token]) {
4046	.keyword_anytype => break :blk true,
4047	.ellipsis3 => break :is_var_args true,
4048	else => unreachable,
4049	}
4050	} else false;
4051
4052	const param_name: Zir.NullTerminatedString = if (param.name_token) \|name_token\| blk: {
4053	const name_bytes = tree.tokenSlice(name_token);
4054	if (mem.eql(u8, "_", name_bytes))
4055	break :blk .empty;
4056
4057	const param_name = try astgen.identAsString(name_token);
4058	if (!is_extern) {
4059	try astgen.detectLocalShadowing(params_scope, param_name, name_token, name_bytes, .@"function parameter");
4060	}
4061	break :blk param_name;
4062	} else if (!is_extern) {
4063	if (param.anytype_ellipsis3) \|tok\| {
4064	return astgen.failTok(tok, "missing parameter name", .{});
4065	} else {
4066	ambiguous: {
4067	if (tree.nodes.items(.tag)[param.type_expr] != .identifier) break :ambiguous;
4068	const main_token = tree.nodes.items(.main_token)[param.type_expr];
4069	const identifier_str = tree.tokenSlice(main_token);
4070	if (isPrimitive(identifier_str)) break :ambiguous;
4071	return astgen.failNodeNotes(
4072	param.type_expr,
4073	"missing parameter name or type",
4074	.{},
4075	&[_]u32{
4076	try astgen.errNoteNode(
4077	param.type_expr,
4078	"if this is a name, annotate its type '{s}: T'",
4079	.{identifier_str},
4080	),
4081	try astgen.errNoteNode(
4082	param.type_expr,
4083	"if this is a type, give it a name '<name>: {s}'",
4084	.{identifier_str},
4085	),
4086	},
4087	);
4088	}
4089	return astgen.failNode(param.type_expr, "missing parameter name", .{});
4090	}
4091	} else .empty;
4092
4093	const param_inst = if (is_anytype) param: {
4094	const name_token = param.name_token orelse param.anytype_ellipsis3.?;
4095	const tag: Zir.Inst.Tag = if (is_comptime)
4096	.param_anytype_comptime
4097	else
4098	.param_anytype;
4099	break :param try decl_gz.addStrTok(tag, param_name, name_token);
4100	} else param: {
4101	const param_type_node = param.type_expr;
4102	assert(param_type_node != 0);
4103	var param_gz = decl_gz.makeSubBlock(scope);
4104	defer param_gz.unstack();
4105	const param_type = try expr(&param_gz, params_scope, coerced_type_ri, param_type_node);
4106	const param_inst_expected: Zir.Inst.Index = @enumFromInt(astgen.instructions.len + 1);
4107	_ = try param_gz.addBreakWithSrcNode(.break_inline, param_inst_expected, param_type, param_type_node);
4108
4109	const main_tokens = tree.nodes.items(.main_token);
4110	const name_token = param.name_token orelse main_tokens[param_type_node];
4111	const tag: Zir.Inst.Tag = if (is_comptime) .param_comptime else .param;
4112	const param_inst = try decl_gz.addParam(&param_gz, tag, name_token, param_name, param.first_doc_comment);
4113	assert(param_inst_expected == param_inst);
4114	break :param param_inst.toRef();
4115	};
4116
4117	if (param_name == .empty or is_extern) continue;
4118
4119	const sub_scope = try astgen.arena.create(Scope.LocalVal);
4120	sub_scope.* = .{
4121	.parent = params_scope,
4122	.gen_zir = &decl_gz,
4123	.name = param_name,
4124	.inst = param_inst,
4125	.token_src = param.name_token.?,
4126	.id_cat = .@"function parameter",
4127	};
4128	params_scope = &sub_scope.base;
4129	}
4130	break :is_var_args false;
4131	};
4132
4133	const lib_name = if (fn_proto.lib_name) \|lib_name_token\| blk: {
4134	const lib_name_str = try astgen.strLitAsString(lib_name_token);
4135	const lib_name_slice = astgen.string_bytes.items[@intFromEnum(lib_name_str.index)..][0..lib_name_str.len];
4136	if (mem.indexOfScalar(u8, lib_name_slice, 0) != null) {
4137	return astgen.failTok(lib_name_token, "library name cannot contain null bytes", .{});
4138	} else if (lib_name_str.len == 0) {
4139	return astgen.failTok(lib_name_token, "library name cannot be empty", .{});
4140	}
4141	break :blk lib_name_str.index;
4142	} else .empty;
4143
4144	const maybe_bang = tree.firstToken(fn_proto.ast.return_type) - 1;
4145	const is_inferred_error = token_tags[maybe_bang] == .bang;
4146
4147	// After creating the function ZIR instruction, it will need to update the break
4148	// instructions inside the expression blocks for align, addrspace, cc, and ret_ty
4149	// to use the function instruction as the "block" to break from.
4150
4151	var align_gz = decl_gz.makeSubBlock(params_scope);
4152	defer align_gz.unstack();
4153	const align_ref: Zir.Inst.Ref = if (fn_proto.ast.align_expr == 0) .none else inst: {
4154	const inst = try expr(&decl_gz, params_scope, coerced_align_ri, fn_proto.ast.align_expr);
4155	if (align_gz.instructionsSlice().len == 0) {
4156	// In this case we will send a len=0 body which can be encoded more efficiently.
4157	break :inst inst;
4158	}
4159	_ = try align_gz.addBreak(.break_inline, @enumFromInt(0), inst);
4160	break :inst inst;
4161	};
4162
4163	var addrspace_gz = decl_gz.makeSubBlock(params_scope);
4164	defer addrspace_gz.unstack();
4165	const addrspace_ref: Zir.Inst.Ref = if (fn_proto.ast.addrspace_expr == 0) .none else inst: {
4166	const inst = try expr(&decl_gz, params_scope, coerced_addrspace_ri, fn_proto.ast.addrspace_expr);
4167	if (addrspace_gz.instructionsSlice().len == 0) {
4168	// In this case we will send a len=0 body which can be encoded more efficiently.
4169	break :inst inst;
4170	}
4171	_ = try addrspace_gz.addBreak(.break_inline, @enumFromInt(0), inst);
4172	break :inst inst;
4173	};
4174
4175	var section_gz = decl_gz.makeSubBlock(params_scope);
4176	defer section_gz.unstack();
4177	const section_ref: Zir.Inst.Ref = if (fn_proto.ast.section_expr == 0) .none else inst: {
4178	const inst = try expr(&decl_gz, params_scope, .{ .rl = .{ .coerced_ty = .slice_const_u8_type } }, fn_proto.ast.section_expr);
4179	if (section_gz.instructionsSlice().len == 0) {
4180	// In this case we will send a len=0 body which can be encoded more efficiently.
4181	break :inst inst;
4182	}
4183	_ = try section_gz.addBreak(.break_inline, @enumFromInt(0), inst);
4184	break :inst inst;
4185	};
4186
4187	var cc_gz = decl_gz.makeSubBlock(params_scope);
4188	defer cc_gz.unstack();
4189	const cc_ref: Zir.Inst.Ref = blk: {
4190	if (fn_proto.ast.callconv_expr != 0) {
4191	if (has_inline_keyword) {
4192	return astgen.failNode(
4193	fn_proto.ast.callconv_expr,
4194	"explicit callconv incompatible with inline keyword",
4195	.{},
4196	);
4197	}
4198	const inst = try expr(
4199	&decl_gz,
4200	params_scope,
4201	.{ .rl = .{ .coerced_ty = .calling_convention_type } },
4202	fn_proto.ast.callconv_expr,
4203	);
4204	if (cc_gz.instructionsSlice().len == 0) {
4205	// In this case we will send a len=0 body which can be encoded more efficiently.
4206	break :blk inst;
4207	}
4208	_ = try cc_gz.addBreak(.break_inline, @enumFromInt(0), inst);
4209	break :blk inst;
4210	} else if (is_extern) {
4211	// note: https://github.com/ziglang/zig/issues/5269
4212	break :blk .calling_convention_c;
4213	} else if (has_inline_keyword) {
4214	break :blk .calling_convention_inline;
4215	} else {
4216	break :blk .none;
4217	}
4218	};
4219
4220	var ret_gz = decl_gz.makeSubBlock(params_scope);
4221	defer ret_gz.unstack();
4222	const ret_ref: Zir.Inst.Ref = inst: {
4223	const inst = try expr(&ret_gz, params_scope, coerced_type_ri, fn_proto.ast.return_type);
4224	if (ret_gz.instructionsSlice().len == 0) {
4225	// In this case we will send a len=0 body which can be encoded more efficiently.
4226	break :inst inst;
4227	}
4228	_ = try ret_gz.addBreak(.break_inline, @enumFromInt(0), inst);
4229	break :inst inst;
4230	};
4231
4232	const func_inst: Zir.Inst.Ref = if (body_node == 0) func: {
4233	if (!is_extern) {
4234	return astgen.failTok(fn_proto.ast.fn_token, "non-extern function has no body", .{});
4235	}
4236	if (is_inferred_error) {
4237	return astgen.failTok(maybe_bang, "function prototype may not have inferred error set", .{});
4238	}
4239	break :func try decl_gz.addFunc(.{
4240	.src_node = decl_node,
4241	.cc_ref = cc_ref,
4242	.cc_gz = &cc_gz,
4243	.align_ref = align_ref,
4244	.align_gz = &align_gz,
4245	.ret_ref = ret_ref,
4246	.ret_gz = &ret_gz,
4247	.section_ref = section_ref,
4248	.section_gz = &section_gz,
4249	.addrspace_ref = addrspace_ref,
4250	.addrspace_gz = &addrspace_gz,
4251	.param_block = decl_inst,
4252	.body_gz = null,
4253	.lib_name = lib_name,
4254	.is_var_args = is_var_args,
4255	.is_inferred_error = false,
4256	.is_test = false,
4257	.is_extern = true,
4258	.is_noinline = is_noinline,
4259	.noalias_bits = noalias_bits,
4260	});
4261	} else func: {
4262	// as a scope, fn_gz encloses ret_gz, but for instruction list, fn_gz stacks on ret_gz
4263	fn_gz.instructions_top = ret_gz.instructions.items.len;
4264
4265	const prev_fn_block = astgen.fn_block;
4266	const prev_fn_ret_ty = astgen.fn_ret_ty;
4267	astgen.fn_block = &fn_gz;
4268	astgen.fn_ret_ty = if (is_inferred_error or ret_ref.toIndex() != null) r: {
4269	// We're essentially guaranteed to need the return type at some point,
4270	// since the return type is likely not `void` or `noreturn` so there
4271	// will probably be an explicit return requiring RLS. Fetch this
4272	// return type now so the rest of the function can use it.
4273	break :r try fn_gz.addNode(.ret_type, decl_node);
4274	} else ret_ref;
4275	defer {
4276	astgen.fn_block = prev_fn_block;
4277	astgen.fn_ret_ty = prev_fn_ret_ty;
4278	}
4279
4280	const prev_var_args = astgen.fn_var_args;
4281	astgen.fn_var_args = is_var_args;
4282	defer astgen.fn_var_args = prev_var_args;
4283
4284	astgen.advanceSourceCursorToNode(body_node);
4285	const lbrace_line = astgen.source_line - decl_gz.decl_line;
4286	const lbrace_column = astgen.source_column;
4287
4288	_ = try expr(&fn_gz, params_scope, .{ .rl = .none }, body_node);
4289	try checkUsed(gz, &fn_gz.base, params_scope);
4290
4291	if (!fn_gz.endsWithNoReturn()) {
4292	// As our last action before the return, "pop" the error trace if needed
4293	_ = try fn_gz.addRestoreErrRetIndex(.ret, .always, decl_node);
4294
4295	// Add implicit return at end of function.
4296	_ = try fn_gz.addUnTok(.ret_implicit, .void_value, tree.lastToken(body_node));
4297	}
4298
4299	break :func try decl_gz.addFunc(.{
4300	.src_node = decl_node,
4301	.cc_ref = cc_ref,
4302	.cc_gz = &cc_gz,
4303	.align_ref = align_ref,
4304	.align_gz = &align_gz,
4305	.ret_ref = ret_ref,
4306	.ret_gz = &ret_gz,
4307	.section_ref = section_ref,
4308	.section_gz = &section_gz,
4309	.addrspace_ref = addrspace_ref,
4310	.addrspace_gz = &addrspace_gz,
4311	.lbrace_line = lbrace_line,
4312	.lbrace_column = lbrace_column,
4313	.param_block = decl_inst,
4314	.body_gz = &fn_gz,
4315	.lib_name = lib_name,
4316	.is_var_args = is_var_args,
4317	.is_inferred_error = is_inferred_error,
4318	.is_test = false,
4319	.is_extern = false,
4320	.is_noinline = is_noinline,
4321	.noalias_bits = noalias_bits,
4322	});
4323	};
4324
4325	// We add this at the end so that its instruction index marks the end range
4326	// of the top level declaration. addFunc already unstacked fn_gz and ret_gz.
4327	_ = try decl_gz.addBreak(.break_inline, decl_inst, func_inst);
4328
4329	try setDeclaration(
4330	decl_inst,
4331	std.zig.hashSrc(tree.getNodeSource(decl_node)),
4332	.{ .named = fn_name_token },
4333	decl_gz.decl_line - gz.decl_line,
4334	is_pub,
4335	is_export,
4336	doc_comment_index,
4337	&decl_gz,
4338	// align, linksection, and addrspace are passed in the func instruction in this case.
4339	// TODO: move them from the function instruction to the declaration instruction?
4340	null,
4341	);
4342	}
4343
4344	fn globalVarDecl(
4345	astgen: *AstGen,
4346	gz: *GenZir,
4347	scope: *Scope,
4348	wip_members: *WipMembers,
4349	node: Ast.Node.Index,
4350	var_decl: Ast.full.VarDecl,
4351	) InnerError!void {
4352	const tree = astgen.tree;
4353	const token_tags = tree.tokens.items(.tag);
4354
4355	const is_mutable = token_tags[var_decl.ast.mut_token] == .keyword_var;
4356	// We do this at the beginning so that the instruction index marks the range start
4357	// of the top level declaration.
4358	const decl_inst = try gz.makeBlockInst(.declaration, node);
4359
4360	const name_token = var_decl.ast.mut_token + 1;
4361	astgen.advanceSourceCursorToNode(node);
4362
4363	var block_scope: GenZir = .{
4364	.parent = scope,
4365	.decl_node_index = node,
4366	.decl_line = astgen.source_line,
4367	.astgen = astgen,
4368	.is_comptime = true,
4369	.anon_name_strategy = .parent,
4370	.instructions = gz.instructions,
4371	.instructions_top = gz.instructions.items.len,
4372	};
4373	defer block_scope.unstack();
4374
4375	const is_pub = var_decl.visib_token != null;
4376	const is_export = blk: {
4377	const maybe_export_token = var_decl.extern_export_token orelse break :blk false;
4378	break :blk token_tags[maybe_export_token] == .keyword_export;
4379	};
4380	const is_extern = blk: {
4381	const maybe_extern_token = var_decl.extern_export_token orelse break :blk false;
4382	break :blk token_tags[maybe_extern_token] == .keyword_extern;
4383	};
4384	wip_members.nextDecl(decl_inst);
4385
4386	const is_threadlocal = if (var_decl.threadlocal_token) \|tok\| blk: {
4387	if (!is_mutable) {
4388	return astgen.failTok(tok, "threadlocal variable cannot be constant", .{});
4389	}
4390	break :blk true;
4391	} else false;
4392
4393	const lib_name = if (var_decl.lib_name) \|lib_name_token\| blk: {
4394	const lib_name_str = try astgen.strLitAsString(lib_name_token);
4395	const lib_name_slice = astgen.string_bytes.items[@intFromEnum(lib_name_str.index)..][0..lib_name_str.len];
4396	if (mem.indexOfScalar(u8, lib_name_slice, 0) != null) {
4397	return astgen.failTok(lib_name_token, "library name cannot contain null bytes", .{});
4398	} else if (lib_name_str.len == 0) {
4399	return astgen.failTok(lib_name_token, "library name cannot be empty", .{});
4400	}
4401	break :blk lib_name_str.index;
4402	} else .empty;
4403
4404	const doc_comment_index = try astgen.docCommentAsString(var_decl.firstToken());
4405
4406	assert(var_decl.comptime_token == null); // handled by parser
4407
4408	const var_inst: Zir.Inst.Ref = if (var_decl.ast.init_node != 0) vi: {
4409	if (is_extern) {
4410	return astgen.failNode(
4411	var_decl.ast.init_node,
4412	"extern variables have no initializers",
4413	.{},
4414	);
4415	}
4416
4417	const type_inst: Zir.Inst.Ref = if (var_decl.ast.type_node != 0)
4418	try expr(
4419	&block_scope,
4420	&block_scope.base,
4421	coerced_type_ri,
4422	var_decl.ast.type_node,
4423	)
4424	else
4425	.none;
4426
4427	const init_inst = try expr(
4428	&block_scope,
4429	&block_scope.base,
4430	if (type_inst != .none) .{ .rl = .{ .ty = type_inst } } else .{ .rl = .none },
4431	var_decl.ast.init_node,
4432	);
4433
4434	if (is_mutable) {
4435	const var_inst = try block_scope.addVar(.{
4436	.var_type = type_inst,
4437	.lib_name = .empty,
4438	.align_inst = .none, // passed via the decls data
4439	.init = init_inst,
4440	.is_extern = false,
4441	.is_const = !is_mutable,
4442	.is_threadlocal = is_threadlocal,
4443	});
4444	break :vi var_inst;
4445	} else {
4446	break :vi init_inst;
4447	}
4448	} else if (!is_extern) {
4449	return astgen.failNode(node, "variables must be initialized", .{});
4450	} else if (var_decl.ast.type_node != 0) vi: {
4451	// Extern variable which has an explicit type.
4452	const type_inst = try typeExpr(&block_scope, &block_scope.base, var_decl.ast.type_node);
4453
4454	const var_inst = try block_scope.addVar(.{
4455	.var_type = type_inst,
4456	.lib_name = lib_name,
4457	.align_inst = .none, // passed via the decls data
4458	.init = .none,
4459	.is_extern = true,
4460	.is_const = !is_mutable,
4461	.is_threadlocal = is_threadlocal,
4462	});
4463	break :vi var_inst;
4464	} else {
4465	return astgen.failNode(node, "unable to infer variable type", .{});
4466	};
4467
4468	// We do this at the end so that the instruction index marks the end
4469	// range of a top level declaration.
4470	_ = try block_scope.addBreakWithSrcNode(.break_inline, decl_inst, var_inst, node);
4471
4472	var align_gz = block_scope.makeSubBlock(scope);
4473	if (var_decl.ast.align_node != 0) {
4474	const align_inst = try expr(&align_gz, &align_gz.base, coerced_align_ri, var_decl.ast.align_node);
4475	_ = try align_gz.addBreakWithSrcNode(.break_inline, decl_inst, align_inst, node);
4476	}
4477
4478	var linksection_gz = align_gz.makeSubBlock(scope);
4479	if (var_decl.ast.section_node != 0) {
4480	const linksection_inst = try expr(&linksection_gz, &linksection_gz.base, coerced_linksection_ri, var_decl.ast.section_node);
4481	_ = try linksection_gz.addBreakWithSrcNode(.break_inline, decl_inst, linksection_inst, node);
4482	}
4483
4484	var addrspace_gz = linksection_gz.makeSubBlock(scope);
4485	if (var_decl.ast.addrspace_node != 0) {
4486	const addrspace_inst = try expr(&addrspace_gz, &addrspace_gz.base, coerced_addrspace_ri, var_decl.ast.addrspace_node);
4487	_ = try addrspace_gz.addBreakWithSrcNode(.break_inline, decl_inst, addrspace_inst, node);
4488	}
4489
4490	try setDeclaration(
4491	decl_inst,
4492	std.zig.hashSrc(tree.getNodeSource(node)),
4493	.{ .named = name_token },
4494	block_scope.decl_line - gz.decl_line,
4495	is_pub,
4496	is_export,
4497	doc_comment_index,
4498	&block_scope,
4499	.{
4500	.align_gz = &align_gz,
4501	.linksection_gz = &linksection_gz,
4502	.addrspace_gz = &addrspace_gz,
4503	},
4504	);
4505	}
4506
4507	fn comptimeDecl(
4508	astgen: *AstGen,
4509	gz: *GenZir,
4510	scope: *Scope,
4511	wip_members: *WipMembers,
4512	node: Ast.Node.Index,
4513	) InnerError!void {
4514	const tree = astgen.tree;
4515	const node_datas = tree.nodes.items(.data);
4516	const body_node = node_datas[node].lhs;
4517
4518	// Up top so the ZIR instruction index marks the start range of this
4519	// top-level declaration.
4520	const decl_inst = try gz.makeBlockInst(.declaration, node);
4521	wip_members.nextDecl(decl_inst);
4522	astgen.advanceSourceCursorToNode(node);
4523
4524	var decl_block: GenZir = .{
4525	.is_comptime = true,
4526	.decl_node_index = node,
4527	.decl_line = astgen.source_line,
4528	.parent = scope,
4529	.astgen = astgen,
4530	.instructions = gz.instructions,
4531	.instructions_top = gz.instructions.items.len,
4532	};
4533	defer decl_block.unstack();
4534
4535	const block_result = try expr(&decl_block, &decl_block.base, .{ .rl = .none }, body_node);
4536	if (decl_block.isEmpty() or !decl_block.refIsNoReturn(block_result)) {
4537	_ = try decl_block.addBreak(.break_inline, decl_inst, .void_value);
4538	}
4539
4540	try setDeclaration(
4541	decl_inst,
4542	std.zig.hashSrc(tree.getNodeSource(node)),
4543	.@"comptime",
4544	decl_block.decl_line - gz.decl_line,
4545	false,
4546	false,
4547	.empty,
4548	&decl_block,
4549	null,
4550	);
4551	}
4552
4553	fn usingnamespaceDecl(
4554	astgen: *AstGen,
4555	gz: *GenZir,
4556	scope: *Scope,
4557	wip_members: *WipMembers,
4558	node: Ast.Node.Index,
4559	) InnerError!void {
4560	const tree = astgen.tree;
4561	const node_datas = tree.nodes.items(.data);
4562
4563	const type_expr = node_datas[node].lhs;
4564	const is_pub = blk: {
4565	const main_tokens = tree.nodes.items(.main_token);
4566	const token_tags = tree.tokens.items(.tag);
4567	const main_token = main_tokens[node];
4568	break :blk (main_token > 0 and token_tags[main_token - 1] == .keyword_pub);
4569	};
4570	// Up top so the ZIR instruction index marks the start range of this
4571	// top-level declaration.
4572	const decl_inst = try gz.makeBlockInst(.declaration, node);
4573	wip_members.nextDecl(decl_inst);
4574	astgen.advanceSourceCursorToNode(node);
4575
4576	var decl_block: GenZir = .{
4577	.is_comptime = true,
4578	.decl_node_index = node,
4579	.decl_line = astgen.source_line,
4580	.parent = scope,
4581	.astgen = astgen,
4582	.instructions = gz.instructions,
4583	.instructions_top = gz.instructions.items.len,
4584	};
4585	defer decl_block.unstack();
4586
4587	const namespace_inst = try typeExpr(&decl_block, &decl_block.base, type_expr);
4588	_ = try decl_block.addBreak(.break_inline, decl_inst, namespace_inst);
4589
4590	try setDeclaration(
4591	decl_inst,
4592	std.zig.hashSrc(tree.getNodeSource(node)),
4593	.@"usingnamespace",
4594	decl_block.decl_line - gz.decl_line,
4595	is_pub,
4596	false,
4597	.empty,
4598	&decl_block,
4599	null,
4600	);
4601	}
4602
4603	fn testDecl(
4604	astgen: *AstGen,
4605	gz: *GenZir,
4606	scope: *Scope,
4607	wip_members: *WipMembers,
4608	node: Ast.Node.Index,
4609	) InnerError!void {
4610	const tree = astgen.tree;
4611	const node_datas = tree.nodes.items(.data);
4612	const body_node = node_datas[node].rhs;
4613
4614	// Up top so the ZIR instruction index marks the start range of this
4615	// top-level declaration.
4616	const decl_inst = try gz.makeBlockInst(.declaration, node);
4617
4618	wip_members.nextDecl(decl_inst);
4619	astgen.advanceSourceCursorToNode(node);
4620
4621	var decl_block: GenZir = .{
4622	.is_comptime = true,
4623	.decl_node_index = node,
4624	.decl_line = astgen.source_line,
4625	.parent = scope,
4626	.astgen = astgen,
4627	.instructions = gz.instructions,
4628	.instructions_top = gz.instructions.items.len,
4629	};
4630	defer decl_block.unstack();
4631
4632	const main_tokens = tree.nodes.items(.main_token);
4633	const token_tags = tree.tokens.items(.tag);
4634	const test_token = main_tokens[node];
4635	const test_name_token = test_token + 1;
4636	const test_name: DeclarationName = switch (token_tags[test_name_token]) {
4637	else => .unnamed_test,
4638	.string_literal => .{ .named_test = test_name_token },
4639	.identifier => blk: {
4640	const ident_name_raw = tree.tokenSlice(test_name_token);
4641
4642	if (mem.eql(u8, ident_name_raw, "_")) return astgen.failTok(test_name_token, "'_' used as an identifier without @\"_\" syntax", .{});
4643
4644	// if not @"" syntax, just use raw token slice
4645	if (ident_name_raw[0] != '@') {
4646	if (isPrimitive(ident_name_raw)) return astgen.failTok(test_name_token, "cannot test a primitive", .{});
4647	}
4648
4649	// Local variables, including function parameters.
4650	const name_str_index = try astgen.identAsString(test_name_token);
4651	var s = scope;
4652	var found_already: ?Ast.Node.Index = null; // we have found a decl with the same name already
4653	var num_namespaces_out: u32 = 0;
4654	var capturing_namespace: ?*Scope.Namespace = null;
4655	while (true) switch (s.tag) {
4656	.local_val => {
4657	const local_val = s.cast(Scope.LocalVal).?;
4658	if (local_val.name == name_str_index) {
4659	local_val.used = test_name_token;
4660	return astgen.failTokNotes(test_name_token, "cannot test a {s}", .{
4661	@tagName(local_val.id_cat),
4662	}, &[_]u32{
4663	try astgen.errNoteTok(local_val.token_src, "{s} declared here", .{
4664	@tagName(local_val.id_cat),
4665	}),
4666	});
4667	}
4668	s = local_val.parent;
4669	},
4670	.local_ptr => {
4671	const local_ptr = s.cast(Scope.LocalPtr).?;
4672	if (local_ptr.name == name_str_index) {
4673	local_ptr.used = test_name_token;
4674	return astgen.failTokNotes(test_name_token, "cannot test a {s}", .{
4675	@tagName(local_ptr.id_cat),
4676	}, &[_]u32{
4677	try astgen.errNoteTok(local_ptr.token_src, "{s} declared here", .{
4678	@tagName(local_ptr.id_cat),
4679	}),
4680	});
4681	}
4682	s = local_ptr.parent;
4683	},
4684	.gen_zir => s = s.cast(GenZir).?.parent,
4685	.defer_normal, .defer_error => s = s.cast(Scope.Defer).?.parent,
4686	.namespace, .enum_namespace => {
4687	const ns = s.cast(Scope.Namespace).?;
4688	if (ns.decls.get(name_str_index)) \|i\| {
4689	if (found_already) \|f\| {
4690	return astgen.failTokNotes(test_name_token, "ambiguous reference", .{}, &.{
4691	try astgen.errNoteNode(f, "declared here", .{}),
4692	try astgen.errNoteNode(i, "also declared here", .{}),
4693	});
4694	}
4695	// We found a match but must continue looking for ambiguous references to decls.
4696	found_already = i;
4697	}
4698	num_namespaces_out += 1;
4699	capturing_namespace = ns;
4700	s = ns.parent;
4701	},
4702	.top => break,
4703	};
4704	if (found_already == null) {
4705	const ident_name = try astgen.identifierTokenString(test_name_token);
4706	return astgen.failTok(test_name_token, "use of undeclared identifier '{s}'", .{ident_name});
4707	}
4708
4709	break :blk .{ .decltest = name_str_index };
4710	},
4711	};
4712
4713	var fn_block: GenZir = .{
4714	.is_comptime = false,
4715	.decl_node_index = node,
4716	.decl_line = decl_block.decl_line,
4717	.parent = &decl_block.base,
4718	.astgen = astgen,
4719	.instructions = decl_block.instructions,
4720	.instructions_top = decl_block.instructions.items.len,
4721	};
4722	defer fn_block.unstack();
4723
4724	const prev_fn_block = astgen.fn_block;
4725	const prev_fn_ret_ty = astgen.fn_ret_ty;
4726	astgen.fn_block = &fn_block;
4727	astgen.fn_ret_ty = .anyerror_void_error_union_type;
4728	defer {
4729	astgen.fn_block = prev_fn_block;
4730	astgen.fn_ret_ty = prev_fn_ret_ty;
4731	}
4732
4733	astgen.advanceSourceCursorToNode(body_node);
4734	const lbrace_line = astgen.source_line - decl_block.decl_line;
4735	const lbrace_column = astgen.source_column;
4736
4737	const block_result = try expr(&fn_block, &fn_block.base, .{ .rl = .none }, body_node);
4738	if (fn_block.isEmpty() or !fn_block.refIsNoReturn(block_result)) {
4739
4740	// As our last action before the return, "pop" the error trace if needed
4741	_ = try fn_block.addRestoreErrRetIndex(.ret, .always, node);
4742
4743	// Add implicit return at end of function.
4744	_ = try fn_block.addUnTok(.ret_implicit, .void_value, tree.lastToken(body_node));
4745	}
4746
4747	const func_inst = try decl_block.addFunc(.{
4748	.src_node = node,
4749
4750	.cc_ref = .none,
4751	.cc_gz = null,
4752	.align_ref = .none,
4753	.align_gz = null,
4754	.ret_ref = .anyerror_void_error_union_type,
4755	.ret_gz = null,
4756	.section_ref = .none,
4757	.section_gz = null,
4758	.addrspace_ref = .none,
4759	.addrspace_gz = null,
4760
4761	.lbrace_line = lbrace_line,
4762	.lbrace_column = lbrace_column,
4763	.param_block = decl_inst,
4764	.body_gz = &fn_block,
4765	.lib_name = .empty,
4766	.is_var_args = false,
4767	.is_inferred_error = false,
4768	.is_test = true,
4769	.is_extern = false,
4770	.is_noinline = false,
4771	.noalias_bits = 0,
4772	});
4773
4774	_ = try decl_block.addBreak(.break_inline, decl_inst, func_inst);
4775
4776	try setDeclaration(
4777	decl_inst,
4778	std.zig.hashSrc(tree.getNodeSource(node)),
4779	test_name,
4780	decl_block.decl_line - gz.decl_line,
4781	false,
4782	false,
4783	.empty,
4784	&decl_block,
4785	null,
4786	);
4787	}
4788
4789	fn structDeclInner(
4790	gz: *GenZir,
4791	scope: *Scope,
4792	node: Ast.Node.Index,
4793	container_decl: Ast.full.ContainerDecl,
4794	layout: std.builtin.Type.ContainerLayout,
4795	backing_int_node: Ast.Node.Index,
4796	) InnerError!Zir.Inst.Ref {
4797	const decl_inst = try gz.reserveInstructionIndex();
4798
4799	if (container_decl.ast.members.len == 0 and backing_int_node == 0) {
4800	try gz.setStruct(decl_inst, .{
4801	.src_node = node,
4802	.layout = layout,
4803	.fields_len = 0,
4804	.decls_len = 0,
4805	.backing_int_ref = .none,
4806	.backing_int_body_len = 0,
4807	.known_non_opv = false,
4808	.known_comptime_only = false,
4809	.is_tuple = false,
4810	.any_comptime_fields = false,
4811	.any_default_inits = false,
4812	.any_aligned_fields = false,
4813	.fields_hash = std.zig.hashSrc(@tagName(layout)),
4814	});
4815	return decl_inst.toRef();
4816	}
4817
4818	const astgen = gz.astgen;
4819	const gpa = astgen.gpa;
4820	const tree = astgen.tree;
4821
4822	var namespace: Scope.Namespace = .{
4823	.parent = scope,
4824	.node = node,
4825	.inst = decl_inst,
4826	.declaring_gz = gz,
4827	};
4828	defer namespace.deinit(gpa);
4829
4830	// The struct_decl instruction introduces a scope in which the decls of the struct
4831	// are in scope, so that field types, alignments, and default value expressions
4832	// can refer to decls within the struct itself.
4833	astgen.advanceSourceCursorToNode(node);
4834	var block_scope: GenZir = .{
4835	.parent = &namespace.base,
4836	.decl_node_index = node,
4837	.decl_line = gz.decl_line,
4838	.astgen = astgen,
4839	.is_comptime = true,
4840	.instructions = gz.instructions,
4841	.instructions_top = gz.instructions.items.len,
4842	};
4843	defer block_scope.unstack();
4844
4845	const scratch_top = astgen.scratch.items.len;
4846	defer astgen.scratch.items.len = scratch_top;
4847
4848	var backing_int_body_len: usize = 0;
4849	const backing_int_ref: Zir.Inst.Ref = blk: {
4850	if (backing_int_node != 0) {
4851	if (layout != .Packed) {
4852	return astgen.failNode(backing_int_node, "non-packed struct does not support backing integer type", .{});
4853	} else {
4854	const backing_int_ref = try typeExpr(&block_scope, &namespace.base, backing_int_node);
4855	if (!block_scope.isEmpty()) {
4856	if (!block_scope.endsWithNoReturn()) {
4857	_ = try block_scope.addBreak(.break_inline, decl_inst, backing_int_ref);
4858	}
4859
4860	const body = block_scope.instructionsSlice();
4861	const old_scratch_len = astgen.scratch.items.len;
4862	try astgen.scratch.ensureUnusedCapacity(gpa, countBodyLenAfterFixups(astgen, body));
4863	appendBodyWithFixupsArrayList(astgen, &astgen.scratch, body);
4864	backing_int_body_len = astgen.scratch.items.len - old_scratch_len;
4865	block_scope.instructions.items.len = block_scope.instructions_top;
4866	}
4867	break :blk backing_int_ref;
4868	}
4869	} else {
4870	break :blk .none;
4871	}
4872	};
4873
4874	const decl_count = try astgen.scanDecls(&namespace, container_decl.ast.members);
4875	const field_count: u32 = @intCast(container_decl.ast.members.len - decl_count);
4876
4877	const bits_per_field = 4;
4878	const max_field_size = 5;
4879	var wip_members = try WipMembers.init(gpa, &astgen.scratch, decl_count, field_count, bits_per_field, max_field_size);
4880	defer wip_members.deinit();
4881
4882	// We will use the scratch buffer, starting here, for the bodies:
4883	// bodies: { // for every fields_len
4884	// field_type_body_inst: Inst, // for each field_type_body_len
4885	// align_body_inst: Inst, // for each align_body_len
4886	// init_body_inst: Inst, // for each init_body_len
4887	// }
4888	// Note that the scratch buffer is simultaneously being used by WipMembers, however
4889	// it will not access any elements beyond this point in the ArrayList. It also
4890	// accesses via the ArrayList items field so it can handle the scratch buffer being
4891	// reallocated.
4892	// No defer needed here because it is handled by `wip_members.deinit()` above.
4893	const bodies_start = astgen.scratch.items.len;
4894
4895	const node_tags = tree.nodes.items(.tag);
4896	const is_tuple = for (container_decl.ast.members) \|member_node\| {
4897	const container_field = tree.fullContainerField(member_node) orelse continue;
4898	if (container_field.ast.tuple_like) break true;
4899	} else false;
4900
4901	if (is_tuple) switch (layout) {
4902	.Auto => {},
4903	.Extern => return astgen.failNode(node, "extern tuples are not supported", .{}),
4904	.Packed => return astgen.failNode(node, "packed tuples are not supported", .{}),
4905	};
4906
4907	if (is_tuple) for (container_decl.ast.members) \|member_node\| {
4908	switch (node_tags[member_node]) {
4909	.container_field_init,
4910	.container_field_align,
4911	.container_field,
4912	.@"comptime",
4913	.test_decl,
4914	=> continue,
4915	else => {
4916	const tuple_member = for (container_decl.ast.members) \|maybe_tuple\| switch (node_tags[maybe_tuple]) {
4917	.container_field_init,
4918	.container_field_align,
4919	.container_field,
4920	=> break maybe_tuple,
4921	else => {},
4922	} else unreachable;
4923	return astgen.failNodeNotes(
4924	member_node,
4925	"tuple declarations cannot contain declarations",
4926	.{},
4927	&[_]u32{
4928	try astgen.errNoteNode(tuple_member, "tuple field here", .{}),
4929	},
4930	);
4931	},
4932	}
4933	};
4934
4935	var fields_hasher = std.zig.SrcHasher.init(.{});
4936	fields_hasher.update(@tagName(layout));
4937	if (backing_int_node != 0) {
4938	fields_hasher.update(tree.getNodeSource(backing_int_node));
4939	}
4940
4941	var sfba = std.heap.stackFallback(256, astgen.arena);
4942	const sfba_allocator = sfba.get();
4943
4944	var duplicate_names = std.AutoArrayHashMap(Zir.NullTerminatedString, std.ArrayListUnmanaged(Ast.TokenIndex)).init(sfba_allocator);
4945	try duplicate_names.ensureTotalCapacity(field_count);
4946
4947	// When there aren't errors, use this to avoid a second iteration.
4948	var any_duplicate = false;
4949
4950	var known_non_opv = false;
4951	var known_comptime_only = false;
4952	var any_comptime_fields = false;
4953	var any_aligned_fields = false;
4954	var any_default_inits = false;
4955	for (container_decl.ast.members) \|member_node\| {
4956	var member = switch (try containerMember(&block_scope, &namespace.base, &wip_members, member_node)) {
4957	.decl => continue,
4958	.field => \|field\| field,
4959	};
4960
4961	fields_hasher.update(tree.getNodeSource(member_node));
4962
4963	if (!is_tuple) {
4964	const field_name = try astgen.identAsString(member.ast.main_token);
4965
4966	member.convertToNonTupleLike(astgen.tree.nodes);
4967	assert(!member.ast.tuple_like);
4968
4969	wip_members.appendToField(@intFromEnum(field_name));
4970
4971	const gop = try duplicate_names.getOrPut(field_name);
4972
4973	if (gop.found_existing) {
4974	try gop.value_ptr.append(sfba_allocator, member.ast.main_token);
4975	any_duplicate = true;
4976	} else {
4977	gop.value_ptr.* = .{};
4978	try gop.value_ptr.append(sfba_allocator, member.ast.main_token);
4979	}
4980	} else if (!member.ast.tuple_like) {
4981	return astgen.failTok(member.ast.main_token, "tuple field has a name", .{});
4982	}
4983
4984	const doc_comment_index = try astgen.docCommentAsString(member.firstToken());
4985	wip_members.appendToField(@intFromEnum(doc_comment_index));
4986
4987	if (member.ast.type_expr == 0) {
4988	return astgen.failTok(member.ast.main_token, "struct field missing type", .{});
4989	}
4990
4991	const field_type = try typeExpr(&block_scope, &namespace.base, member.ast.type_expr);
4992	const have_type_body = !block_scope.isEmpty();
4993	const have_align = member.ast.align_expr != 0;
4994	const have_value = member.ast.value_expr != 0;
4995	const is_comptime = member.comptime_token != null;
4996
4997	if (is_comptime) {
4998	switch (layout) {
4999	.Packed => return astgen.failTok(member.comptime_token.?, "packed struct fields cannot be marked comptime", .{}),
5000	.Extern => return astgen.failTok(member.comptime_token.?, "extern struct fields cannot be marked comptime", .{}),
5001	.Auto => any_comptime_fields = true,
5002	}
5003	} else {
5004	known_non_opv = known_non_opv or
5005	nodeImpliesMoreThanOnePossibleValue(tree, member.ast.type_expr);
5006	known_comptime_only = known_comptime_only or
5007	nodeImpliesComptimeOnly(tree, member.ast.type_expr);
5008	}
5009	wip_members.nextField(bits_per_field, .{ have_align, have_value, is_comptime, have_type_body });
5010
5011	if (have_type_body) {
5012	if (!block_scope.endsWithNoReturn()) {
5013	_ = try block_scope.addBreak(.break_inline, decl_inst, field_type);
5014	}
5015	const body = block_scope.instructionsSlice();
5016	const old_scratch_len = astgen.scratch.items.len;
5017	try astgen.scratch.ensureUnusedCapacity(gpa, countBodyLenAfterFixups(astgen, body));
5018	appendBodyWithFixupsArrayList(astgen, &astgen.scratch, body);
5019	wip_members.appendToField(@intCast(astgen.scratch.items.len - old_scratch_len));
5020	block_scope.instructions.items.len = block_scope.instructions_top;
5021	} else {
5022	wip_members.appendToField(@intFromEnum(field_type));
5023	}
5024
5025	if (have_align) {
5026	if (layout == .Packed) {
5027	try astgen.appendErrorNode(member.ast.align_expr, "unable to override alignment of packed struct fields", .{});
5028	}
5029	any_aligned_fields = true;
5030	const align_ref = try expr(&block_scope, &namespace.base, coerced_align_ri, member.ast.align_expr);
5031	if (!block_scope.endsWithNoReturn()) {
5032	_ = try block_scope.addBreak(.break_inline, decl_inst, align_ref);
5033	}
5034	const body = block_scope.instructionsSlice();
5035	const old_scratch_len = astgen.scratch.items.len;
5036	try astgen.scratch.ensureUnusedCapacity(gpa, countBodyLenAfterFixups(astgen, body));
5037	appendBodyWithFixupsArrayList(astgen, &astgen.scratch, body);
5038	wip_members.appendToField(@intCast(astgen.scratch.items.len - old_scratch_len));
5039	block_scope.instructions.items.len = block_scope.instructions_top;
5040	}
5041
5042	if (have_value) {
5043	any_default_inits = true;
5044
5045	// The decl_inst is used as here so that we can easily reconstruct a mapping
5046	// between it and the field type when the fields inits are analzyed.
5047	const ri: ResultInfo = .{ .rl = if (field_type == .none) .none else .{ .coerced_ty = decl_inst.toRef() } };
5048
5049	const default_inst = try expr(&block_scope, &namespace.base, ri, member.ast.value_expr);
5050	if (!block_scope.endsWithNoReturn()) {
5051	_ = try block_scope.addBreak(.break_inline, decl_inst, default_inst);
5052	}
5053	const body = block_scope.instructionsSlice();
5054	const old_scratch_len = astgen.scratch.items.len;
5055	try astgen.scratch.ensureUnusedCapacity(gpa, countBodyLenAfterFixups(astgen, body));
5056	appendBodyWithFixupsArrayList(astgen, &astgen.scratch, body);
5057	wip_members.appendToField(@intCast(astgen.scratch.items.len - old_scratch_len));
5058	block_scope.instructions.items.len = block_scope.instructions_top;
5059	} else if (member.comptime_token) \|comptime_token\| {
5060	return astgen.failTok(comptime_token, "comptime field without default initialization value", .{});
5061	}
5062	}
5063
5064	if (any_duplicate) {
5065	var it = duplicate_names.iterator();
5066
5067	while (it.next()) \|entry\| {
5068	const record = entry.value_ptr.*;
5069	if (record.items.len > 1) {
5070	var error_notes = std.ArrayList(u32).init(astgen.arena);
5071
5072	for (record.items[1..]) \|duplicate\| {
5073	try error_notes.append(try astgen.errNoteTok(duplicate, "duplicate field here", .{}));
5074	}
5075
5076	try error_notes.append(try astgen.errNoteNode(node, "struct declared here", .{}));
5077
5078	try astgen.appendErrorTokNotes(
5079	record.items[0],
5080	"duplicate struct field name",
5081	.{},
5082	error_notes.items,
5083	);
5084	}
5085	}
5086
5087	return error.AnalysisFail;
5088	}
5089
5090	var fields_hash: std.zig.SrcHash = undefined;
5091	fields_hasher.final(&fields_hash);
5092
5093	try gz.setStruct(decl_inst, .{
5094	.src_node = node,
5095	.layout = layout,
5096	.fields_len = field_count,
5097	.decls_len = decl_count,
5098	.backing_int_ref = backing_int_ref,
5099	.backing_int_body_len = @intCast(backing_int_body_len),
5100	.known_non_opv = known_non_opv,
5101	.known_comptime_only = known_comptime_only,
5102	.is_tuple = is_tuple,
5103	.any_comptime_fields = any_comptime_fields,
5104	.any_default_inits = any_default_inits,
5105	.any_aligned_fields = any_aligned_fields,
5106	.fields_hash = fields_hash,
5107	});
5108
5109	wip_members.finishBits(bits_per_field);
5110	const decls_slice = wip_members.declsSlice();
5111	const fields_slice = wip_members.fieldsSlice();
5112	const bodies_slice = astgen.scratch.items[bodies_start..];
5113	try astgen.extra.ensureUnusedCapacity(gpa, backing_int_body_len +
5114	decls_slice.len + fields_slice.len + bodies_slice.len);
5115	astgen.extra.appendSliceAssumeCapacity(astgen.scratch.items[scratch_top..][0..backing_int_body_len]);
5116	astgen.extra.appendSliceAssumeCapacity(decls_slice);
5117	astgen.extra.appendSliceAssumeCapacity(fields_slice);
5118	astgen.extra.appendSliceAssumeCapacity(bodies_slice);
5119
5120	block_scope.unstack();
5121	try gz.addNamespaceCaptures(&namespace);
5122	return decl_inst.toRef();
5123	}
5124
5125	fn unionDeclInner(
5126	gz: *GenZir,
5127	scope: *Scope,
5128	node: Ast.Node.Index,
5129	members: []const Ast.Node.Index,
5130	layout: std.builtin.Type.ContainerLayout,
5131	arg_node: Ast.Node.Index,
5132	auto_enum_tok: ?Ast.TokenIndex,
5133	) InnerError!Zir.Inst.Ref {
5134	const decl_inst = try gz.reserveInstructionIndex();
5135
5136	const astgen = gz.astgen;
5137	const gpa = astgen.gpa;
5138
5139	var namespace: Scope.Namespace = .{
5140	.parent = scope,
5141	.node = node,
5142	.inst = decl_inst,
5143	.declaring_gz = gz,
5144	};
5145	defer namespace.deinit(gpa);
5146
5147	// The union_decl instruction introduces a scope in which the decls of the union
5148	// are in scope, so that field types, alignments, and default value expressions
5149	// can refer to decls within the union itself.
5150	astgen.advanceSourceCursorToNode(node);
5151	var block_scope: GenZir = .{
5152	.parent = &namespace.base,
5153	.decl_node_index = node,
5154	.decl_line = gz.decl_line,
5155	.astgen = astgen,
5156	.is_comptime = true,
5157	.instructions = gz.instructions,
5158	.instructions_top = gz.instructions.items.len,
5159	};
5160	defer block_scope.unstack();
5161
5162	const decl_count = try astgen.scanDecls(&namespace, members);
5163	const field_count: u32 = @intCast(members.len - decl_count);
5164
5165	if (layout != .Auto and (auto_enum_tok != null or arg_node != 0)) {
5166	const layout_str = if (layout == .Extern) "extern" else "packed";
5167	if (arg_node != 0) {
5168	return astgen.failNode(arg_node, "{s} union does not support enum tag type", .{layout_str});
5169	} else {
5170	return astgen.failTok(auto_enum_tok.?, "{s} union does not support enum tag type", .{layout_str});
5171	}
5172	}
5173
5174	const arg_inst: Zir.Inst.Ref = if (arg_node != 0)
5175	try typeExpr(&block_scope, &namespace.base, arg_node)
5176	else
5177	.none;
5178
5179	const bits_per_field = 4;
5180	const max_field_size = 5;
5181	var any_aligned_fields = false;
5182	var wip_members = try WipMembers.init(gpa, &astgen.scratch, decl_count, field_count, bits_per_field, max_field_size);
5183	defer wip_members.deinit();
5184
5185	var fields_hasher = std.zig.SrcHasher.init(.{});
5186	fields_hasher.update(@tagName(layout));
5187	fields_hasher.update(&.{@intFromBool(auto_enum_tok != null)});
5188	if (arg_node != 0) {
5189	fields_hasher.update(astgen.tree.getNodeSource(arg_node));
5190	}
5191
5192	var sfba = std.heap.stackFallback(256, astgen.arena);
5193	const sfba_allocator = sfba.get();
5194
5195	var duplicate_names = std.AutoArrayHashMap(Zir.NullTerminatedString, std.ArrayListUnmanaged(Ast.TokenIndex)).init(sfba_allocator);
5196	try duplicate_names.ensureTotalCapacity(field_count);
5197
5198	// When there aren't errors, use this to avoid a second iteration.
5199	var any_duplicate = false;
5200
5201	for (members) \|member_node\| {
5202	var member = switch (try containerMember(&block_scope, &namespace.base, &wip_members, member_node)) {
5203	.decl => continue,
5204	.field => \|field\| field,
5205	};
5206	fields_hasher.update(astgen.tree.getNodeSource(member_node));
5207	member.convertToNonTupleLike(astgen.tree.nodes);
5208	if (member.ast.tuple_like) {
5209	return astgen.failTok(member.ast.main_token, "union field missing name", .{});
5210	}
5211	if (member.comptime_token) \|comptime_token\| {
5212	return astgen.failTok(comptime_token, "union fields cannot be marked comptime", .{});
5213	}
5214
5215	const field_name = try astgen.identAsString(member.ast.main_token);
5216	wip_members.appendToField(@intFromEnum(field_name));
5217
5218	const gop = try duplicate_names.getOrPut(field_name);
5219
5220	if (gop.found_existing) {
5221	try gop.value_ptr.append(sfba_allocator, member.ast.main_token);
5222	any_duplicate = true;
5223	} else {
5224	gop.value_ptr.* = .{};
5225	try gop.value_ptr.append(sfba_allocator, member.ast.main_token);
5226	}
5227
5228	const doc_comment_index = try astgen.docCommentAsString(member.firstToken());
5229	wip_members.appendToField(@intFromEnum(doc_comment_index));
5230
5231	const have_type = member.ast.type_expr != 0;
5232	const have_align = member.ast.align_expr != 0;
5233	const have_value = member.ast.value_expr != 0;
5234	const unused = false;
5235	wip_members.nextField(bits_per_field, .{ have_type, have_align, have_value, unused });
5236
5237	if (have_type) {
5238	const field_type = try typeExpr(&block_scope, &namespace.base, member.ast.type_expr);
5239	wip_members.appendToField(@intFromEnum(field_type));
5240	} else if (arg_inst == .none and auto_enum_tok == null) {
5241	return astgen.failNode(member_node, "union field missing type", .{});
5242	}
5243	if (have_align) {
5244	const align_inst = try expr(&block_scope, &block_scope.base, coerced_align_ri, member.ast.align_expr);
5245	wip_members.appendToField(@intFromEnum(align_inst));
5246	any_aligned_fields = true;
5247	}
5248	if (have_value) {
5249	if (arg_inst == .none) {
5250	return astgen.failNodeNotes(
5251	node,
5252	"explicitly valued tagged union missing integer tag type",
5253	.{},
5254	&[_]u32{
5255	try astgen.errNoteNode(
5256	member.ast.value_expr,
5257	"tag value specified here",
5258	.{},
5259	),
5260	},
5261	);
5262	}
5263	if (auto_enum_tok == null) {
5264	return astgen.failNodeNotes(
5265	node,
5266	"explicitly valued tagged union requires inferred enum tag type",
5267	.{},
5268	&[_]u32{
5269	try astgen.errNoteNode(
5270	member.ast.value_expr,
5271	"tag value specified here",
5272	.{},
5273	),
5274	},
5275	);
5276	}
5277	const tag_value = try expr(&block_scope, &block_scope.base, .{ .rl = .{ .ty = arg_inst } }, member.ast.value_expr);
5278	wip_members.appendToField(@intFromEnum(tag_value));
5279	}
5280	}
5281
5282	if (any_duplicate) {
5283	var it = duplicate_names.iterator();
5284
5285	while (it.next()) \|entry\| {
5286	const record = entry.value_ptr.*;
5287	if (record.items.len > 1) {
5288	var error_notes = std.ArrayList(u32).init(astgen.arena);
5289
5290	for (record.items[1..]) \|duplicate\| {
5291	try error_notes.append(try astgen.errNoteTok(duplicate, "duplicate field here", .{}));
5292	}
5293
5294	try error_notes.append(try astgen.errNoteNode(node, "union declared here", .{}));
5295
5296	try astgen.appendErrorTokNotes(
5297	record.items[0],
5298	"duplicate union field name",
5299	.{},
5300	error_notes.items,
5301	);
5302	}
5303	}
5304
5305	return error.AnalysisFail;
5306	}
5307
5308	var fields_hash: std.zig.SrcHash = undefined;
5309	fields_hasher.final(&fields_hash);
5310
5311	if (!block_scope.isEmpty()) {
5312	_ = try block_scope.addBreak(.break_inline, decl_inst, .void_value);
5313	}
5314
5315	const body = block_scope.instructionsSlice();
5316	const body_len = astgen.countBodyLenAfterFixups(body);
5317
5318	try gz.setUnion(decl_inst, .{
5319	.src_node = node,
5320	.layout = layout,
5321	.tag_type = arg_inst,
5322	.body_len = body_len,
5323	.fields_len = field_count,
5324	.decls_len = decl_count,
5325	.auto_enum_tag = auto_enum_tok != null,
5326	.any_aligned_fields = any_aligned_fields,
5327	.fields_hash = fields_hash,
5328	});
5329
5330	wip_members.finishBits(bits_per_field);
5331	const decls_slice = wip_members.declsSlice();
5332	const fields_slice = wip_members.fieldsSlice();
5333	try astgen.extra.ensureUnusedCapacity(gpa, decls_slice.len + body_len + fields_slice.len);
5334	astgen.extra.appendSliceAssumeCapacity(decls_slice);
5335	astgen.appendBodyWithFixups(body);
5336	astgen.extra.appendSliceAssumeCapacity(fields_slice);
5337
5338	block_scope.unstack();
5339	try gz.addNamespaceCaptures(&namespace);
5340	return decl_inst.toRef();
5341	}
5342
5343	fn containerDecl(
5344	gz: *GenZir,
5345	scope: *Scope,
5346	ri: ResultInfo,
5347	node: Ast.Node.Index,
5348	container_decl: Ast.full.ContainerDecl,
5349	) InnerError!Zir.Inst.Ref {
5350	const astgen = gz.astgen;
5351	const gpa = astgen.gpa;
5352	const tree = astgen.tree;
5353	const token_tags = tree.tokens.items(.tag);
5354
5355	const prev_fn_block = astgen.fn_block;
5356	astgen.fn_block = null;
5357	defer astgen.fn_block = prev_fn_block;
5358
5359	// We must not create any types until Sema. Here the goal is only to generate
5360	// ZIR for all the field types, alignments, and default value expressions.
5361
5362	switch (token_tags[container_decl.ast.main_token]) {
5363	.keyword_struct => {
5364	const layout = if (container_decl.layout_token) \|t\| switch (token_tags[t]) {
5365	.keyword_packed => std.builtin.Type.ContainerLayout.Packed,
5366	.keyword_extern => std.builtin.Type.ContainerLayout.Extern,
5367	else => unreachable,
5368	} else std.builtin.Type.ContainerLayout.Auto;
5369
5370	const result = try structDeclInner(gz, scope, node, container_decl, layout, container_decl.ast.arg);
5371	return rvalue(gz, ri, result, node);
5372	},
5373	.keyword_union => {
5374	const layout = if (container_decl.layout_token) \|t\| switch (token_tags[t]) {
5375	.keyword_packed => std.builtin.Type.ContainerLayout.Packed,
5376	.keyword_extern => std.builtin.Type.ContainerLayout.Extern,
5377	else => unreachable,
5378	} else std.builtin.Type.ContainerLayout.Auto;
5379
5380	const result = try unionDeclInner(gz, scope, node, container_decl.ast.members, layout, container_decl.ast.arg, container_decl.ast.enum_token);
5381	return rvalue(gz, ri, result, node);
5382	},
5383	.keyword_enum => {
5384	if (container_decl.layout_token) \|t\| {
5385	return astgen.failTok(t, "enums do not support 'packed' or 'extern'; instead provide an explicit integer tag type", .{});
5386	}
5387	// Count total fields as well as how many have explicitly provided tag values.
5388	const counts = blk: {
5389	var values: usize = 0;
5390	var total_fields: usize = 0;
5391	var decls: usize = 0;
5392	var nonexhaustive_node: Ast.Node.Index = 0;
5393	var nonfinal_nonexhaustive = false;
5394	for (container_decl.ast.members) \|member_node\| {
5395	var member = tree.fullContainerField(member_node) orelse {
5396	decls += 1;
5397	continue;
5398	};
5399	member.convertToNonTupleLike(astgen.tree.nodes);
5400	if (member.ast.tuple_like) {
5401	return astgen.failTok(member.ast.main_token, "enum field missing name", .{});
5402	}
5403	if (member.comptime_token) \|comptime_token\| {
5404	return astgen.failTok(comptime_token, "enum fields cannot be marked comptime", .{});
5405	}
5406	if (member.ast.type_expr != 0) {
5407	return astgen.failNodeNotes(
5408	member.ast.type_expr,
5409	"enum fields do not have types",
5410	.{},
5411	&[_]u32{
5412	try astgen.errNoteNode(
5413	node,
5414	"consider 'union(enum)' here to make it a tagged union",
5415	.{},
5416	),
5417	},
5418	);
5419	}
5420	if (member.ast.align_expr != 0) {
5421	return astgen.failNode(member.ast.align_expr, "enum fields cannot be aligned", .{});
5422	}
5423
5424	const name_token = member.ast.main_token;
5425	if (mem.eql(u8, tree.tokenSlice(name_token), "_")) {
5426	if (nonexhaustive_node != 0) {
5427	return astgen.failNodeNotes(
5428	member_node,
5429	"redundant non-exhaustive enum mark",
5430	.{},
5431	&[_]u32{
5432	try astgen.errNoteNode(
5433	nonexhaustive_node,
5434	"other mark here",
5435	.{},
5436	),
5437	},
5438	);
5439	}
5440	nonexhaustive_node = member_node;
5441	if (member.ast.value_expr != 0) {
5442	return astgen.failNode(member.ast.value_expr, "'_' is used to mark an enum as non-exhaustive and cannot be assigned a value", .{});
5443	}
5444	continue;
5445	} else if (nonexhaustive_node != 0) {
5446	nonfinal_nonexhaustive = true;
5447	}
5448	total_fields += 1;
5449	if (member.ast.value_expr != 0) {
5450	if (container_decl.ast.arg == 0) {
5451	return astgen.failNode(member.ast.value_expr, "value assigned to enum tag with inferred tag type", .{});
5452	}
5453	values += 1;
5454	}
5455	}
5456	if (nonfinal_nonexhaustive) {
5457	return astgen.failNode(nonexhaustive_node, "'_' field of non-exhaustive enum must be last", .{});
5458	}
5459	break :blk .{
5460	.total_fields = total_fields,
5461	.values = values,
5462	.decls = decls,
5463	.nonexhaustive_node = nonexhaustive_node,
5464	};
5465	};
5466	if (counts.nonexhaustive_node != 0 and container_decl.ast.arg == 0) {
5467	try astgen.appendErrorNodeNotes(
5468	node,
5469	"non-exhaustive enum missing integer tag type",
5470	.{},
5471	&[_]u32{
5472	try astgen.errNoteNode(
5473	counts.nonexhaustive_node,
5474	"marked non-exhaustive here",
5475	.{},
5476	),
5477	},
5478	);
5479	}
5480	// In this case we must generate ZIR code for the tag values, similar to
5481	// how structs are handled above.
5482	const nonexhaustive = counts.nonexhaustive_node != 0;
5483
5484	const decl_inst = try gz.reserveInstructionIndex();
5485
5486	var namespace: Scope.Namespace = .{
5487	.parent = scope,
5488	.node = node,
5489	.inst = decl_inst,
5490	.declaring_gz = gz,
5491	};
5492	defer namespace.deinit(gpa);
5493
5494	// The enum_decl instruction introduces a scope in which the decls of the enum
5495	// are in scope, so that tag values can refer to decls within the enum itself.
5496	astgen.advanceSourceCursorToNode(node);
5497	var block_scope: GenZir = .{
5498	.parent = &namespace.base,
5499	.decl_node_index = node,
5500	.decl_line = gz.decl_line,
5501	.astgen = astgen,
5502	.is_comptime = true,
5503	.instructions = gz.instructions,
5504	.instructions_top = gz.instructions.items.len,
5505	};
5506	defer block_scope.unstack();
5507
5508	_ = try astgen.scanDecls(&namespace, container_decl.ast.members);
5509	namespace.base.tag = .enum_namespace;
5510
5511	const arg_inst: Zir.Inst.Ref = if (container_decl.ast.arg != 0)
5512	try comptimeExpr(&block_scope, &namespace.base, coerced_type_ri, container_decl.ast.arg)
5513	else
5514	.none;
5515
5516	const bits_per_field = 1;
5517	const max_field_size = 3;
5518	var wip_members = try WipMembers.init(gpa, &astgen.scratch, @intCast(counts.decls), @intCast(counts.total_fields), bits_per_field, max_field_size);
5519	defer wip_members.deinit();
5520
5521	var fields_hasher = std.zig.SrcHasher.init(.{});
5522	if (container_decl.ast.arg != 0) {
5523	fields_hasher.update(tree.getNodeSource(container_decl.ast.arg));
5524	}
5525	fields_hasher.update(&.{@intFromBool(nonexhaustive)});
5526
5527	var sfba = std.heap.stackFallback(256, astgen.arena);
5528	const sfba_allocator = sfba.get();
5529
5530	var duplicate_names = std.AutoArrayHashMap(Zir.NullTerminatedString, std.ArrayListUnmanaged(Ast.TokenIndex)).init(sfba_allocator);
5531	try duplicate_names.ensureTotalCapacity(counts.total_fields);
5532
5533	// When there aren't errors, use this to avoid a second iteration.
5534	var any_duplicate = false;
5535
5536	for (container_decl.ast.members) \|member_node\| {
5537	if (member_node == counts.nonexhaustive_node)
5538	continue;
5539	fields_hasher.update(tree.getNodeSource(member_node));
5540	namespace.base.tag = .namespace;
5541	var member = switch (try containerMember(&block_scope, &namespace.base, &wip_members, member_node)) {
5542	.decl => continue,
5543	.field => \|field\| field,
5544	};
5545	member.convertToNonTupleLike(astgen.tree.nodes);
5546	assert(member.comptime_token == null);
5547	assert(member.ast.type_expr == 0);
5548	assert(member.ast.align_expr == 0);
5549
5550	const field_name = try astgen.identAsString(member.ast.main_token);
5551	wip_members.appendToField(@intFromEnum(field_name));
5552
5553	const gop = try duplicate_names.getOrPut(field_name);
5554
5555	if (gop.found_existing) {
5556	try gop.value_ptr.append(sfba_allocator, member.ast.main_token);
5557	any_duplicate = true;
5558	} else {
5559	gop.value_ptr.* = .{};
5560	try gop.value_ptr.append(sfba_allocator, member.ast.main_token);
5561	}
5562
5563	const doc_comment_index = try astgen.docCommentAsString(member.firstToken());
5564	wip_members.appendToField(@intFromEnum(doc_comment_index));
5565
5566	const have_value = member.ast.value_expr != 0;
5567	wip_members.nextField(bits_per_field, .{have_value});
5568
5569	if (have_value) {
5570	if (arg_inst == .none) {
5571	return astgen.failNodeNotes(
5572	node,
5573	"explicitly valued enum missing integer tag type",
5574	.{},
5575	&[_]u32{
5576	try astgen.errNoteNode(
5577	member.ast.value_expr,
5578	"tag value specified here",
5579	.{},
5580	),
5581	},
5582	);
5583	}
5584	namespace.base.tag = .enum_namespace;
5585	const tag_value_inst = try expr(&block_scope, &namespace.base, .{ .rl = .{ .ty = arg_inst } }, member.ast.value_expr);
5586	wip_members.appendToField(@intFromEnum(tag_value_inst));
5587	}
5588	}
5589
5590	if (any_duplicate) {
5591	var it = duplicate_names.iterator();
5592
5593	while (it.next()) \|entry\| {
5594	const record = entry.value_ptr.*;
5595	if (record.items.len > 1) {
5596	var error_notes = std.ArrayList(u32).init(astgen.arena);
5597
5598	for (record.items[1..]) \|duplicate\| {
5599	try error_notes.append(try astgen.errNoteTok(duplicate, "duplicate field here", .{}));
5600	}
5601
5602	try error_notes.append(try astgen.errNoteNode(node, "enum declared here", .{}));
5603
5604	try astgen.appendErrorTokNotes(
5605	record.items[0],
5606	"duplicate enum field name",
5607	.{},
5608	error_notes.items,
5609	);
5610	}
5611	}
5612
5613	return error.AnalysisFail;
5614	}
5615
5616	if (!block_scope.isEmpty()) {
5617	_ = try block_scope.addBreak(.break_inline, decl_inst, .void_value);
5618	}
5619
5620	var fields_hash: std.zig.SrcHash = undefined;
5621	fields_hasher.final(&fields_hash);
5622
5623	const body = block_scope.instructionsSlice();
5624	const body_len = astgen.countBodyLenAfterFixups(body);
5625
5626	try gz.setEnum(decl_inst, .{
5627	.src_node = node,
5628	.nonexhaustive = nonexhaustive,
5629	.tag_type = arg_inst,
5630	.body_len = body_len,
5631	.fields_len = @intCast(counts.total_fields),
5632	.decls_len = @intCast(counts.decls),
5633	.fields_hash = fields_hash,
5634	});
5635
5636	wip_members.finishBits(bits_per_field);
5637	const decls_slice = wip_members.declsSlice();
5638	const fields_slice = wip_members.fieldsSlice();
5639	try astgen.extra.ensureUnusedCapacity(gpa, decls_slice.len + body_len + fields_slice.len);
5640	astgen.extra.appendSliceAssumeCapacity(decls_slice);
5641	astgen.appendBodyWithFixups(body);
5642	astgen.extra.appendSliceAssumeCapacity(fields_slice);
5643
5644	block_scope.unstack();
5645	try gz.addNamespaceCaptures(&namespace);
5646	return rvalue(gz, ri, decl_inst.toRef(), node);
5647	},
5648	.keyword_opaque => {
5649	assert(container_decl.ast.arg == 0);
5650
5651	const decl_inst = try gz.reserveInstructionIndex();
5652
5653	var namespace: Scope.Namespace = .{
5654	.parent = scope,
5655	.node = node,
5656	.inst = decl_inst,
5657	.declaring_gz = gz,
5658	};
5659	defer namespace.deinit(gpa);
5660
5661	astgen.advanceSourceCursorToNode(node);
5662	var block_scope: GenZir = .{
5663	.parent = &namespace.base,
5664	.decl_node_index = node,
5665	.decl_line = gz.decl_line,
5666	.astgen = astgen,
5667	.is_comptime = true,
5668	.instructions = gz.instructions,
5669	.instructions_top = gz.instructions.items.len,
5670	};
5671	defer block_scope.unstack();
5672
5673	const decl_count = try astgen.scanDecls(&namespace, container_decl.ast.members);
5674
5675	var wip_members = try WipMembers.init(gpa, &astgen.scratch, decl_count, 0, 0, 0);
5676	defer wip_members.deinit();
5677
5678	for (container_decl.ast.members) \|member_node\| {
5679	const res = try containerMember(&block_scope, &namespace.base, &wip_members, member_node);
5680	if (res == .field) {
5681	return astgen.failNode(member_node, "opaque types cannot have fields", .{});
5682	}
5683	}
5684
5685	try gz.setOpaque(decl_inst, .{
5686	.src_node = node,
5687	.decls_len = decl_count,
5688	});
5689
5690	wip_members.finishBits(0);
5691	const decls_slice = wip_members.declsSlice();
5692	try astgen.extra.ensureUnusedCapacity(gpa, decls_slice.len);
5693	astgen.extra.appendSliceAssumeCapacity(decls_slice);
5694
5695	block_scope.unstack();
5696	try gz.addNamespaceCaptures(&namespace);
5697	return rvalue(gz, ri, decl_inst.toRef(), node);
5698	},
5699	else => unreachable,
5700	}
5701	}
5702
5703	const ContainerMemberResult = union(enum) { decl, field: Ast.full.ContainerField };
5704
5705	fn containerMember(
5706	gz: *GenZir,
5707	scope: *Scope,
5708	wip_members: *WipMembers,
5709	member_node: Ast.Node.Index,
5710	) InnerError!ContainerMemberResult {
5711	const astgen = gz.astgen;
5712	const tree = astgen.tree;
5713	const node_tags = tree.nodes.items(.tag);
5714	const node_datas = tree.nodes.items(.data);
5715	switch (node_tags[member_node]) {
5716	.container_field_init,
5717	.container_field_align,
5718	.container_field,
5719	=> return ContainerMemberResult{ .field = tree.fullContainerField(member_node).? },
5720
5721	.fn_proto,
5722	.fn_proto_multi,
5723	.fn_proto_one,
5724	.fn_proto_simple,
5725	.fn_decl,
5726	=> {
5727	var buf: [1]Ast.Node.Index = undefined;
5728	const full = tree.fullFnProto(&buf, member_node).?;
5729	const body = if (node_tags[member_node] == .fn_decl) node_datas[member_node].rhs else 0;
5730
5731	astgen.fnDecl(gz, scope, wip_members, member_node, body, full) catch \|err\| switch (err) {
5732	error.OutOfMemory => return error.OutOfMemory,
5733	error.AnalysisFail => {},
5734	};
5735	},
5736
5737	.global_var_decl,
5738	.local_var_decl,
5739	.simple_var_decl,
5740	.aligned_var_decl,
5741	=> {
5742	astgen.globalVarDecl(gz, scope, wip_members, member_node, tree.fullVarDecl(member_node).?) catch \|err\| switch (err) {
5743	error.OutOfMemory => return error.OutOfMemory,
5744	error.AnalysisFail => {},
5745	};
5746	},
5747
5748	.@"comptime" => {
5749	astgen.comptimeDecl(gz, scope, wip_members, member_node) catch \|err\| switch (err) {
5750	error.OutOfMemory => return error.OutOfMemory,
5751	error.AnalysisFail => {},
5752	};
5753	},
5754	.@"usingnamespace" => {
5755	astgen.usingnamespaceDecl(gz, scope, wip_members, member_node) catch \|err\| switch (err) {
5756	error.OutOfMemory => return error.OutOfMemory,
5757	error.AnalysisFail => {},
5758	};
5759	},
5760	.test_decl => {
5761	astgen.testDecl(gz, scope, wip_members, member_node) catch \|err\| switch (err) {
5762	error.OutOfMemory => return error.OutOfMemory,
5763	error.AnalysisFail => {},
5764	};
5765	},
5766	else => unreachable,
5767	}
5768	return .decl;
5769	}
5770
5771	fn errorSetDecl(gz: *GenZir, ri: ResultInfo, node: Ast.Node.Index) InnerError!Zir.Inst.Ref {
5772	const astgen = gz.astgen;
5773	const gpa = astgen.gpa;
5774	const tree = astgen.tree;
5775	const main_tokens = tree.nodes.items(.main_token);
5776	const token_tags = tree.tokens.items(.tag);
5777
5778	const payload_index = try reserveExtra(astgen, @typeInfo(Zir.Inst.ErrorSetDecl).Struct.fields.len);
5779	var fields_len: usize = 0;
5780	{
5781	var idents: std.AutoHashMapUnmanaged(Zir.NullTerminatedString, Ast.TokenIndex) = .{};
5782	defer idents.deinit(gpa);
5783
5784	const error_token = main_tokens[node];
5785	var tok_i = error_token + 2;
5786	while (true) : (tok_i += 1) {
5787	switch (token_tags[tok_i]) {
5788	.doc_comment, .comma => {},
5789	.identifier => {
5790	const str_index = try astgen.identAsString(tok_i);
5791	const gop = try idents.getOrPut(gpa, str_index);
5792	if (gop.found_existing) {
5793	const name = try gpa.dupe(u8, mem.span(astgen.nullTerminatedString(str_index)));
5794	defer gpa.free(name);
5795	return astgen.failTokNotes(
5796	tok_i,
5797	"duplicate error set field '{s}'",
5798	.{name},
5799	&[_]u32{
5800	try astgen.errNoteTok(
5801	gop.value_ptr.*,
5802	"previous declaration here",
5803	.{},
5804	),
5805	},
5806	);
5807	}
5808	gop.value_ptr.* = tok_i;
5809
5810	try astgen.extra.ensureUnusedCapacity(gpa, 2);
5811	astgen.extra.appendAssumeCapacity(@intFromEnum(str_index));
5812	const doc_comment_index = try astgen.docCommentAsString(tok_i);
5813	astgen.extra.appendAssumeCapacity(@intFromEnum(doc_comment_index));
5814	fields_len += 1;
5815	},
5816	.r_brace => break,
5817	else => unreachable,
5818	}
5819	}
5820	}
5821
5822	setExtra(astgen, payload_index, Zir.Inst.ErrorSetDecl{
5823	.fields_len = @intCast(fields_len),
5824	});
5825	const result = try gz.addPlNodePayloadIndex(.error_set_decl, node, payload_index);
5826	return rvalue(gz, ri, result, node);
5827	}
5828
5829	fn tryExpr(
5830	parent_gz: *GenZir,
5831	scope: *Scope,
5832	ri: ResultInfo,
5833	node: Ast.Node.Index,
5834	operand_node: Ast.Node.Index,
5835	) InnerError!Zir.Inst.Ref {
5836	const astgen = parent_gz.astgen;
5837
5838	const fn_block = astgen.fn_block orelse {
5839	return astgen.failNode(node, "'try' outside function scope", .{});
5840	};
5841
5842	if (parent_gz.any_defer_node != 0) {
5843	return astgen.failNodeNotes(node, "'try' not allowed inside defer expression", .{}, &.{
5844	try astgen.errNoteNode(
5845	parent_gz.any_defer_node,
5846	"defer expression here",
5847	.{},
5848	),
5849	});
5850	}
5851
5852	// Ensure debug line/column information is emitted for this try expression.
5853	// Then we will save the line/column so that we can emit another one that goes
5854	// "backwards" because we want to evaluate the operand, but then put the debug
5855	// info back at the try keyword for error return tracing.
5856	if (!parent_gz.is_comptime) {
5857	try emitDbgNode(parent_gz, node);
5858	}
5859	const try_lc = LineColumn{ astgen.source_line - parent_gz.decl_line, astgen.source_column };
5860
5861	const operand_ri: ResultInfo = switch (ri.rl) {
5862	.ref, .ref_coerced_ty => .{ .rl = .ref, .ctx = .error_handling_expr },
5863	else => .{ .rl = .none, .ctx = .error_handling_expr },
5864	};
5865	// This could be a pointer or value depending on the `ri` parameter.
5866	const operand = try reachableExpr(parent_gz, scope, operand_ri, operand_node, node);
5867	const block_tag: Zir.Inst.Tag = if (operand_ri.rl == .ref) .try_ptr else .@"try";
5868	const try_inst = try parent_gz.makeBlockInst(block_tag, node);
5869	try parent_gz.instructions.append(astgen.gpa, try_inst);
5870
5871	var else_scope = parent_gz.makeSubBlock(scope);
5872	defer else_scope.unstack();
5873
5874	const err_tag = switch (ri.rl) {
5875	.ref, .ref_coerced_ty => Zir.Inst.Tag.err_union_code_ptr,
5876	else => Zir.Inst.Tag.err_union_code,
5877	};
5878	const err_code = try else_scope.addUnNode(err_tag, operand, node);
5879	try genDefers(&else_scope, &fn_block.base, scope, .{ .both = err_code });
5880	try emitDbgStmt(&else_scope, try_lc);
5881	_ = try else_scope.addUnNode(.ret_node, err_code, node);
5882
5883	try else_scope.setTryBody(try_inst, operand);
5884	const result = try_inst.toRef();
5885	switch (ri.rl) {
5886	.ref, .ref_coerced_ty => return result,
5887	else => return rvalue(parent_gz, ri, result, node),
5888	}
5889	}
5890
5891	fn orelseCatchExpr(
5892	parent_gz: *GenZir,
5893	scope: *Scope,
5894	ri: ResultInfo,
5895	node: Ast.Node.Index,
5896	lhs: Ast.Node.Index,
5897	cond_op: Zir.Inst.Tag,
5898	unwrap_op: Zir.Inst.Tag,
5899	unwrap_code_op: Zir.Inst.Tag,
5900	rhs: Ast.Node.Index,
5901	payload_token: ?Ast.TokenIndex,
5902	) InnerError!Zir.Inst.Ref {
5903	const astgen = parent_gz.astgen;
5904	const tree = astgen.tree;
5905
5906	const need_rl = astgen.nodes_need_rl.contains(node);
5907	const block_ri: ResultInfo = if (need_rl) ri else .{
5908	.rl = switch (ri.rl) {
5909	.ptr => .{ .ty = (try ri.rl.resultType(parent_gz, node)).? },
5910	.inferred_ptr => .none,
5911	else => ri.rl,
5912	},
5913	.ctx = ri.ctx,
5914	};
5915	// We need to call `rvalue` to write through to the pointer only if we had a
5916	// result pointer and aren't forwarding it.
5917	const LocTag = @typeInfo(ResultInfo.Loc).Union.tag_type.?;
5918	const need_result_rvalue = @as(LocTag, block_ri.rl) != @as(LocTag, ri.rl);
5919
5920	const do_err_trace = astgen.fn_block != null and (cond_op == .is_non_err or cond_op == .is_non_err_ptr);
5921
5922	var block_scope = parent_gz.makeSubBlock(scope);
5923	block_scope.setBreakResultInfo(block_ri);
5924	defer block_scope.unstack();
5925
5926	const operand_ri: ResultInfo = switch (block_scope.break_result_info.rl) {
5927	.ref, .ref_coerced_ty => .{ .rl = .ref, .ctx = if (do_err_trace) .error_handling_expr else .none },
5928	else => .{ .rl = .none, .ctx = if (do_err_trace) .error_handling_expr else .none },
5929	};
5930	// This could be a pointer or value depending on the `operand_ri` parameter.
5931	// We cannot use `block_scope.break_result_info` because that has the bare
5932	// type, whereas this expression has the optional type. Later we make
5933	// up for this fact by calling rvalue on the else branch.
5934	const operand = try reachableExpr(&block_scope, &block_scope.base, operand_ri, lhs, rhs);
5935	const cond = try block_scope.addUnNode(cond_op, operand, node);
5936	const condbr = try block_scope.addCondBr(.condbr, node);
5937
5938	const block = try parent_gz.makeBlockInst(.block, node);
5939	try block_scope.setBlockBody(block);
5940	// block_scope unstacked now, can add new instructions to parent_gz
5941	try parent_gz.instructions.append(astgen.gpa, block);
5942
5943	var then_scope = block_scope.makeSubBlock(scope);
5944	defer then_scope.unstack();
5945
5946	// This could be a pointer or value depending on `unwrap_op`.
5947	const unwrapped_payload = try then_scope.addUnNode(unwrap_op, operand, node);
5948	const then_result = switch (ri.rl) {
5949	.ref, .ref_coerced_ty => unwrapped_payload,
5950	else => try rvalue(&then_scope, block_scope.break_result_info, unwrapped_payload, node),
5951	};
5952	_ = try then_scope.addBreakWithSrcNode(.@"break", block, then_result, node);
5953
5954	var else_scope = block_scope.makeSubBlock(scope);
5955	defer else_scope.unstack();
5956
5957	// We know that the operand (almost certainly) modified the error return trace,
5958	// so signal to Sema that it should save the new index for restoring later.
5959	if (do_err_trace and nodeMayAppendToErrorTrace(tree, lhs))
5960	_ = try else_scope.addSaveErrRetIndex(.always);
5961
5962	var err_val_scope: Scope.LocalVal = undefined;
5963	const else_sub_scope = blk: {
5964	const payload = payload_token orelse break :blk &else_scope.base;
5965	const err_str = tree.tokenSlice(payload);
5966	if (mem.eql(u8, err_str, "_")) {
5967	return astgen.failTok(payload, "discard of error capture; omit it instead", .{});
5968	}
5969	const err_name = try astgen.identAsString(payload);
5970
5971	try astgen.detectLocalShadowing(scope, err_name, payload, err_str, .capture);
5972
5973	err_val_scope = .{
5974	.parent = &else_scope.base,
5975	.gen_zir = &else_scope,
5976	.name = err_name,
5977	.inst = try else_scope.addUnNode(unwrap_code_op, operand, node),
5978	.token_src = payload,
5979	.id_cat = .capture,
5980	};
5981	break :blk &err_val_scope.base;
5982	};
5983
5984	const else_result = try expr(&else_scope, else_sub_scope, block_scope.break_result_info, rhs);
5985	if (!else_scope.endsWithNoReturn()) {
5986	// As our last action before the break, "pop" the error trace if needed
5987	if (do_err_trace)
5988	try restoreErrRetIndex(&else_scope, .{ .block = block }, block_scope.break_result_info, rhs, else_result);
5989
5990	_ = try else_scope.addBreakWithSrcNode(.@"break", block, else_result, rhs);
5991	}
5992	try checkUsed(parent_gz, &else_scope.base, else_sub_scope);
5993
5994	try setCondBrPayload(condbr, cond, &then_scope, &else_scope);
5995
5996	if (need_result_rvalue) {
5997	return rvalue(parent_gz, ri, block.toRef(), node);
5998	} else {
5999	return block.toRef();
6000	}
6001	}
6002
6003	/// Return whether the identifier names of two tokens are equal. Resolves @""
6004	/// tokens without allocating.
6005	/// OK in theory it could do it without allocating. This implementation
6006	/// allocates when the @"" form is used.
6007	fn tokenIdentEql(astgen: *AstGen, token1: Ast.TokenIndex, token2: Ast.TokenIndex) !bool {
6008	const ident_name_1 = try astgen.identifierTokenString(token1);
6009	const ident_name_2 = try astgen.identifierTokenString(token2);
6010	return mem.eql(u8, ident_name_1, ident_name_2);
6011	}
6012
6013	fn fieldAccess(
6014	gz: *GenZir,
6015	scope: *Scope,
6016	ri: ResultInfo,
6017	node: Ast.Node.Index,
6018	) InnerError!Zir.Inst.Ref {
6019	switch (ri.rl) {
6020	.ref, .ref_coerced_ty => return addFieldAccess(.field_ptr, gz, scope, .{ .rl = .ref }, node),
6021	else => {
6022	const access = try addFieldAccess(.field_val, gz, scope, .{ .rl = .none }, node);
6023	return rvalue(gz, ri, access, node);
6024	},
6025	}
6026	}
6027
6028	fn addFieldAccess(
6029	tag: Zir.Inst.Tag,
6030	gz: *GenZir,
6031	scope: *Scope,
6032	lhs_ri: ResultInfo,
6033	node: Ast.Node.Index,
6034	) InnerError!Zir.Inst.Ref {
6035	const astgen = gz.astgen;
6036	const tree = astgen.tree;
6037	const main_tokens = tree.nodes.items(.main_token);
6038	const node_datas = tree.nodes.items(.data);
6039
6040	const object_node = node_datas[node].lhs;
6041	const dot_token = main_tokens[node];
6042	const field_ident = dot_token + 1;
6043	const str_index = try astgen.identAsString(field_ident);
6044	const lhs = try expr(gz, scope, lhs_ri, object_node);
6045
6046	const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
6047	try emitDbgStmt(gz, cursor);
6048
6049	return gz.addPlNode(tag, node, Zir.Inst.Field{
6050	.lhs = lhs,
6051	.field_name_start = str_index,
6052	});
6053	}
6054
6055	fn arrayAccess(
6056	gz: *GenZir,
6057	scope: *Scope,
6058	ri: ResultInfo,
6059	node: Ast.Node.Index,
6060	) InnerError!Zir.Inst.Ref {
6061	const tree = gz.astgen.tree;
6062	const node_datas = tree.nodes.items(.data);
6063	switch (ri.rl) {
6064	.ref, .ref_coerced_ty => {
6065	const lhs = try expr(gz, scope, .{ .rl = .ref }, node_datas[node].lhs);
6066
6067	const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
6068
6069	const rhs = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, node_datas[node].rhs);
6070	try emitDbgStmt(gz, cursor);
6071
6072	return gz.addPlNode(.elem_ptr_node, node, Zir.Inst.Bin{ .lhs = lhs, .rhs = rhs });
6073	},
6074	else => {
6075	const lhs = try expr(gz, scope, .{ .rl = .none }, node_datas[node].lhs);
6076
6077	const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
6078
6079	const rhs = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, node_datas[node].rhs);
6080	try emitDbgStmt(gz, cursor);
6081
6082	return rvalue(gz, ri, try gz.addPlNode(.elem_val_node, node, Zir.Inst.Bin{ .lhs = lhs, .rhs = rhs }), node);
6083	},
6084	}
6085	}
6086
6087	fn simpleBinOp(
6088	gz: *GenZir,
6089	scope: *Scope,
6090	ri: ResultInfo,
6091	node: Ast.Node.Index,
6092	op_inst_tag: Zir.Inst.Tag,
6093	) InnerError!Zir.Inst.Ref {
6094	const astgen = gz.astgen;
6095	const tree = astgen.tree;
6096	const node_datas = tree.nodes.items(.data);
6097
6098	if (op_inst_tag == .cmp_neq or op_inst_tag == .cmp_eq) {
6099	const node_tags = tree.nodes.items(.tag);
6100	const str = if (op_inst_tag == .cmp_eq) "==" else "!=";
6101	if (node_tags[node_datas[node].lhs] == .string_literal or
6102	node_tags[node_datas[node].rhs] == .string_literal)
6103	return astgen.failNode(node, "cannot compare strings with {s}", .{str});
6104	}
6105
6106	const lhs = try reachableExpr(gz, scope, .{ .rl = .none }, node_datas[node].lhs, node);
6107	const cursor = switch (op_inst_tag) {
6108	.add, .sub, .mul, .div, .mod_rem => maybeAdvanceSourceCursorToMainToken(gz, node),
6109	else => undefined,
6110	};
6111	const rhs = try reachableExpr(gz, scope, .{ .rl = .none }, node_datas[node].rhs, node);
6112
6113	switch (op_inst_tag) {
6114	.add, .sub, .mul, .div, .mod_rem => {
6115	try emitDbgStmt(gz, cursor);
6116	},
6117	else => {},
6118	}
6119	const result = try gz.addPlNode(op_inst_tag, node, Zir.Inst.Bin{ .lhs = lhs, .rhs = rhs });
6120	return rvalue(gz, ri, result, node);
6121	}
6122
6123	fn simpleStrTok(
6124	gz: *GenZir,
6125	ri: ResultInfo,
6126	ident_token: Ast.TokenIndex,
6127	node: Ast.Node.Index,
6128	op_inst_tag: Zir.Inst.Tag,
6129	) InnerError!Zir.Inst.Ref {
6130	const astgen = gz.astgen;
6131	const str_index = try astgen.identAsString(ident_token);
6132	const result = try gz.addStrTok(op_inst_tag, str_index, ident_token);
6133	return rvalue(gz, ri, result, node);
6134	}
6135
6136	fn boolBinOp(
6137	gz: *GenZir,
6138	scope: *Scope,
6139	ri: ResultInfo,
6140	node: Ast.Node.Index,
6141	zir_tag: Zir.Inst.Tag,
6142	) InnerError!Zir.Inst.Ref {
6143	const astgen = gz.astgen;
6144	const tree = astgen.tree;
6145	const node_datas = tree.nodes.items(.data);
6146
6147	const lhs = try expr(gz, scope, coerced_bool_ri, node_datas[node].lhs);
6148	const bool_br = (try gz.addPlNodePayloadIndex(zir_tag, node, undefined)).toIndex().?;
6149
6150	var rhs_scope = gz.makeSubBlock(scope);
6151	defer rhs_scope.unstack();
6152	const rhs = try expr(&rhs_scope, &rhs_scope.base, coerced_bool_ri, node_datas[node].rhs);
6153	if (!gz.refIsNoReturn(rhs)) {
6154	_ = try rhs_scope.addBreakWithSrcNode(.break_inline, bool_br, rhs, node_datas[node].rhs);
6155	}
6156	try rhs_scope.setBoolBrBody(bool_br, lhs);
6157
6158	const block_ref = bool_br.toRef();
6159	return rvalue(gz, ri, block_ref, node);
6160	}
6161
6162	fn ifExpr(
6163	parent_gz: *GenZir,
6164	scope: *Scope,
6165	ri: ResultInfo,
6166	node: Ast.Node.Index,
6167	if_full: Ast.full.If,
6168	) InnerError!Zir.Inst.Ref {
6169	const astgen = parent_gz.astgen;
6170	const tree = astgen.tree;
6171	const token_tags = tree.tokens.items(.tag);
6172
6173	const do_err_trace = astgen.fn_block != null and if_full.error_token != null;
6174
6175	const need_rl = astgen.nodes_need_rl.contains(node);
6176	const block_ri: ResultInfo = if (need_rl) ri else .{
6177	.rl = switch (ri.rl) {
6178	.ptr => .{ .ty = (try ri.rl.resultType(parent_gz, node)).? },
6179	.inferred_ptr => .none,
6180	else => ri.rl,
6181	},
6182	.ctx = ri.ctx,
6183	};
6184	// We need to call `rvalue` to write through to the pointer only if we had a
6185	// result pointer and aren't forwarding it.
6186	const LocTag = @typeInfo(ResultInfo.Loc).Union.tag_type.?;
6187	const need_result_rvalue = @as(LocTag, block_ri.rl) != @as(LocTag, ri.rl);
6188
6189	var block_scope = parent_gz.makeSubBlock(scope);
6190	block_scope.setBreakResultInfo(block_ri);
6191	defer block_scope.unstack();
6192
6193	const payload_is_ref = if (if_full.payload_token) \|payload_token\|
6194	token_tags[payload_token] == .asterisk
6195	else
6196	false;
6197
6198	try emitDbgNode(parent_gz, if_full.ast.cond_expr);
6199	const cond: struct {
6200	inst: Zir.Inst.Ref,
6201	bool_bit: Zir.Inst.Ref,
6202	} = c: {
6203	if (if_full.error_token) \|_\| {
6204	const cond_ri: ResultInfo = .{ .rl = if (payload_is_ref) .ref else .none, .ctx = .error_handling_expr };
6205	const err_union = try expr(&block_scope, &block_scope.base, cond_ri, if_full.ast.cond_expr);
6206	const tag: Zir.Inst.Tag = if (payload_is_ref) .is_non_err_ptr else .is_non_err;
6207	break :c .{
6208	.inst = err_union,
6209	.bool_bit = try block_scope.addUnNode(tag, err_union, if_full.ast.cond_expr),
6210	};
6211	} else if (if_full.payload_token) \|_\| {
6212	const cond_ri: ResultInfo = .{ .rl = if (payload_is_ref) .ref else .none };
6213	const optional = try expr(&block_scope, &block_scope.base, cond_ri, if_full.ast.cond_expr);
6214	const tag: Zir.Inst.Tag = if (payload_is_ref) .is_non_null_ptr else .is_non_null;
6215	break :c .{
6216	.inst = optional,
6217	.bool_bit = try block_scope.addUnNode(tag, optional, if_full.ast.cond_expr),
6218	};
6219	} else {
6220	const cond = try expr(&block_scope, &block_scope.base, coerced_bool_ri, if_full.ast.cond_expr);
6221	break :c .{
6222	.inst = cond,
6223	.bool_bit = cond,
6224	};
6225	}
6226	};
6227
6228	const condbr = try block_scope.addCondBr(.condbr, node);
6229
6230	const block = try parent_gz.makeBlockInst(.block, node);
6231	try block_scope.setBlockBody(block);
6232	// block_scope unstacked now, can add new instructions to parent_gz
6233	try parent_gz.instructions.append(astgen.gpa, block);
6234
6235	var then_scope = parent_gz.makeSubBlock(scope);
6236	defer then_scope.unstack();
6237
6238	var payload_val_scope: Scope.LocalVal = undefined;
6239
6240	const then_node = if_full.ast.then_expr;
6241	const then_sub_scope = s: {
6242	if (if_full.error_token != null) {
6243	if (if_full.payload_token) \|payload_token\| {
6244	const tag: Zir.Inst.Tag = if (payload_is_ref)
6245	.err_union_payload_unsafe_ptr
6246	else
6247	.err_union_payload_unsafe;
6248	const payload_inst = try then_scope.addUnNode(tag, cond.inst, then_node);
6249	const token_name_index = payload_token + @intFromBool(payload_is_ref);
6250	const ident_name = try astgen.identAsString(token_name_index);
6251	const token_name_str = tree.tokenSlice(token_name_index);
6252	if (mem.eql(u8, "_", token_name_str))
6253	break :s &then_scope.base;
6254	try astgen.detectLocalShadowing(&then_scope.base, ident_name, token_name_index, token_name_str, .capture);
6255	payload_val_scope = .{
6256	.parent = &then_scope.base,
6257	.gen_zir = &then_scope,
6258	.name = ident_name,
6259	.inst = payload_inst,
6260	.token_src = token_name_index,
6261	.id_cat = .capture,
6262	};
6263	try then_scope.addDbgVar(.dbg_var_val, ident_name, payload_inst);
6264	break :s &payload_val_scope.base;
6265	} else {
6266	_ = try then_scope.addUnNode(.ensure_err_union_payload_void, cond.inst, node);
6267	break :s &then_scope.base;
6268	}
6269	} else if (if_full.payload_token) \|payload_token\| {
6270	const ident_token = if (payload_is_ref) payload_token + 1 else payload_token;
6271	const tag: Zir.Inst.Tag = if (payload_is_ref)
6272	.optional_payload_unsafe_ptr
6273	else
6274	.optional_payload_unsafe;
6275	const ident_bytes = tree.tokenSlice(ident_token);
6276	if (mem.eql(u8, "_", ident_bytes))
6277	break :s &then_scope.base;
6278	const payload_inst = try then_scope.addUnNode(tag, cond.inst, then_node);
6279	const ident_name = try astgen.identAsString(ident_token);
6280	try astgen.detectLocalShadowing(&then_scope.base, ident_name, ident_token, ident_bytes, .capture);
6281	payload_val_scope = .{
6282	.parent = &then_scope.base,
6283	.gen_zir = &then_scope,
6284	.name = ident_name,
6285	.inst = payload_inst,
6286	.token_src = ident_token,
6287	.id_cat = .capture,
6288	};
6289	try then_scope.addDbgVar(.dbg_var_val, ident_name, payload_inst);
6290	break :s &payload_val_scope.base;
6291	} else {
6292	break :s &then_scope.base;
6293	}
6294	};
6295
6296	const then_result = try expr(&then_scope, then_sub_scope, block_scope.break_result_info, then_node);
6297	try checkUsed(parent_gz, &then_scope.base, then_sub_scope);
6298	if (!then_scope.endsWithNoReturn()) {
6299	_ = try then_scope.addBreakWithSrcNode(.@"break", block, then_result, then_node);
6300	}
6301
6302	var else_scope = parent_gz.makeSubBlock(scope);
6303	defer else_scope.unstack();
6304
6305	// We know that the operand (almost certainly) modified the error return trace,
6306	// so signal to Sema that it should save the new index for restoring later.
6307	if (do_err_trace and nodeMayAppendToErrorTrace(tree, if_full.ast.cond_expr))
6308	_ = try else_scope.addSaveErrRetIndex(.always);
6309
6310	const else_node = if_full.ast.else_expr;
6311	if (else_node != 0) {
6312	const sub_scope = s: {
6313	if (if_full.error_token) \|error_token\| {
6314	const tag: Zir.Inst.Tag = if (payload_is_ref)
6315	.err_union_code_ptr
6316	else
6317	.err_union_code;
6318	const payload_inst = try else_scope.addUnNode(tag, cond.inst, if_full.ast.cond_expr);
6319	const ident_name = try astgen.identAsString(error_token);
6320	const error_token_str = tree.tokenSlice(error_token);
6321	if (mem.eql(u8, "_", error_token_str))
6322	break :s &else_scope.base;
6323	try astgen.detectLocalShadowing(&else_scope.base, ident_name, error_token, error_token_str, .capture);
6324	payload_val_scope = .{
6325	.parent = &else_scope.base,
6326	.gen_zir = &else_scope,
6327	.name = ident_name,
6328	.inst = payload_inst,
6329	.token_src = error_token,
6330	.id_cat = .capture,
6331	};
6332	try else_scope.addDbgVar(.dbg_var_val, ident_name, payload_inst);
6333	break :s &payload_val_scope.base;
6334	} else {
6335	break :s &else_scope.base;
6336	}
6337	};
6338	const else_result = try expr(&else_scope, sub_scope, block_scope.break_result_info, else_node);
6339	if (!else_scope.endsWithNoReturn()) {
6340	// As our last action before the break, "pop" the error trace if needed
6341	if (do_err_trace)
6342	try restoreErrRetIndex(&else_scope, .{ .block = block }, block_scope.break_result_info, else_node, else_result);
6343	_ = try else_scope.addBreakWithSrcNode(.@"break", block, else_result, else_node);
6344	}
6345	try checkUsed(parent_gz, &else_scope.base, sub_scope);
6346	} else {
6347	const result = try rvalue(&else_scope, ri, .void_value, node);
6348	_ = try else_scope.addBreak(.@"break", block, result);
6349	}
6350
6351	try setCondBrPayload(condbr, cond.bool_bit, &then_scope, &else_scope);
6352
6353	if (need_result_rvalue) {
6354	return rvalue(parent_gz, ri, block.toRef(), node);
6355	} else {
6356	return block.toRef();
6357	}
6358	}
6359
6360	/// Supports `else_scope` stacked on `then_scope`. Unstacks `else_scope` then `then_scope`.
6361	fn setCondBrPayload(
6362	condbr: Zir.Inst.Index,
6363	cond: Zir.Inst.Ref,
6364	then_scope: *GenZir,
6365	else_scope: *GenZir,
6366	) !void {
6367	defer then_scope.unstack();
6368	defer else_scope.unstack();
6369	const astgen = then_scope.astgen;
6370	const then_body = then_scope.instructionsSliceUpto(else_scope);
6371	const else_body = else_scope.instructionsSlice();
6372	const then_body_len = astgen.countBodyLenAfterFixups(then_body);
6373	const else_body_len = astgen.countBodyLenAfterFixups(else_body);
6374	try astgen.extra.ensureUnusedCapacity(
6375	astgen.gpa,
6376	@typeInfo(Zir.Inst.CondBr).Struct.fields.len + then_body_len + else_body_len,
6377	);
6378
6379	const zir_datas = astgen.instructions.items(.data);
6380	zir_datas[@intFromEnum(condbr)].pl_node.payload_index = astgen.addExtraAssumeCapacity(Zir.Inst.CondBr{
6381	.condition = cond,
6382	.then_body_len = then_body_len,
6383	.else_body_len = else_body_len,
6384	});
6385	astgen.appendBodyWithFixups(then_body);
6386	astgen.appendBodyWithFixups(else_body);
6387	}
6388
6389	fn whileExpr(
6390	parent_gz: *GenZir,
6391	scope: *Scope,
6392	ri: ResultInfo,
6393	node: Ast.Node.Index,
6394	while_full: Ast.full.While,
6395	is_statement: bool,
6396	) InnerError!Zir.Inst.Ref {
6397	const astgen = parent_gz.astgen;
6398	const tree = astgen.tree;
6399	const token_tags = tree.tokens.items(.tag);
6400
6401	const need_rl = astgen.nodes_need_rl.contains(node);
6402	const block_ri: ResultInfo = if (need_rl) ri else .{
6403	.rl = switch (ri.rl) {
6404	.ptr => .{ .ty = (try ri.rl.resultType(parent_gz, node)).? },
6405	.inferred_ptr => .none,
6406	else => ri.rl,
6407	},
6408	.ctx = ri.ctx,
6409	};
6410	// We need to call `rvalue` to write through to the pointer only if we had a
6411	// result pointer and aren't forwarding it.
6412	const LocTag = @typeInfo(ResultInfo.Loc).Union.tag_type.?;
6413	const need_result_rvalue = @as(LocTag, block_ri.rl) != @as(LocTag, ri.rl);
6414
6415	if (while_full.label_token) \|label_token\| {
6416	try astgen.checkLabelRedefinition(scope, label_token);
6417	}
6418
6419	const is_inline = while_full.inline_token != null;
6420	if (parent_gz.is_comptime and is_inline) {
6421	return astgen.failTok(while_full.inline_token.?, "redundant inline keyword in comptime scope", .{});
6422	}
6423	const loop_tag: Zir.Inst.Tag = if (is_inline) .block_inline else .loop;
6424	const loop_block = try parent_gz.makeBlockInst(loop_tag, node);
6425	try parent_gz.instructions.append(astgen.gpa, loop_block);
6426
6427	var loop_scope = parent_gz.makeSubBlock(scope);
6428	loop_scope.is_inline = is_inline;
6429	loop_scope.setBreakResultInfo(block_ri);
6430	defer loop_scope.unstack();
6431
6432	var cond_scope = parent_gz.makeSubBlock(&loop_scope.base);
6433	defer cond_scope.unstack();
6434
6435	const payload_is_ref = if (while_full.payload_token) \|payload_token\|
6436	token_tags[payload_token] == .asterisk
6437	else
6438	false;
6439
6440	try emitDbgNode(parent_gz, while_full.ast.cond_expr);
6441	const cond: struct {
6442	inst: Zir.Inst.Ref,
6443	bool_bit: Zir.Inst.Ref,
6444	} = c: {
6445	if (while_full.error_token) \|_\| {
6446	const cond_ri: ResultInfo = .{ .rl = if (payload_is_ref) .ref else .none };
6447	const err_union = try expr(&cond_scope, &cond_scope.base, cond_ri, while_full.ast.cond_expr);
6448	const tag: Zir.Inst.Tag = if (payload_is_ref) .is_non_err_ptr else .is_non_err;
6449	break :c .{
6450	.inst = err_union,
6451	.bool_bit = try cond_scope.addUnNode(tag, err_union, while_full.ast.cond_expr),
6452	};
6453	} else if (while_full.payload_token) \|_\| {
6454	const cond_ri: ResultInfo = .{ .rl = if (payload_is_ref) .ref else .none };
6455	const optional = try expr(&cond_scope, &cond_scope.base, cond_ri, while_full.ast.cond_expr);
6456	const tag: Zir.Inst.Tag = if (payload_is_ref) .is_non_null_ptr else .is_non_null;
6457	break :c .{
6458	.inst = optional,
6459	.bool_bit = try cond_scope.addUnNode(tag, optional, while_full.ast.cond_expr),
6460	};
6461	} else {
6462	const cond = try expr(&cond_scope, &cond_scope.base, coerced_bool_ri, while_full.ast.cond_expr);
6463	break :c .{
6464	.inst = cond,
6465	.bool_bit = cond,
6466	};
6467	}
6468	};
6469
6470	const condbr_tag: Zir.Inst.Tag = if (is_inline) .condbr_inline else .condbr;
6471	const condbr = try cond_scope.addCondBr(condbr_tag, node);
6472	const block_tag: Zir.Inst.Tag = if (is_inline) .block_inline else .block;
6473	const cond_block = try loop_scope.makeBlockInst(block_tag, node);
6474	try cond_scope.setBlockBody(cond_block);
6475	// cond_scope unstacked now, can add new instructions to loop_scope
6476	try loop_scope.instructions.append(astgen.gpa, cond_block);
6477
6478	// make scope now but don't stack on parent_gz until loop_scope
6479	// gets unstacked after cont_expr is emitted and added below
6480	var then_scope = parent_gz.makeSubBlock(&cond_scope.base);
6481	then_scope.instructions_top = GenZir.unstacked_top;
6482	defer then_scope.unstack();
6483
6484	var dbg_var_name: Zir.NullTerminatedString = .empty;
6485	var dbg_var_inst: Zir.Inst.Ref = undefined;
6486	var opt_payload_inst: Zir.Inst.OptionalIndex = .none;
6487	var payload_val_scope: Scope.LocalVal = undefined;
6488	const then_sub_scope = s: {
6489	if (while_full.error_token != null) {
6490	if (while_full.payload_token) \|payload_token\| {
6491	const tag: Zir.Inst.Tag = if (payload_is_ref)
6492	.err_union_payload_unsafe_ptr
6493	else
6494	.err_union_payload_unsafe;
6495	// will add this instruction to then_scope.instructions below
6496	const payload_inst = try then_scope.makeUnNode(tag, cond.inst, while_full.ast.cond_expr);
6497	opt_payload_inst = payload_inst.toOptional();
6498	const ident_token = payload_token + @intFromBool(payload_is_ref);
6499	const ident_bytes = tree.tokenSlice(ident_token);
6500	if (mem.eql(u8, "_", ident_bytes))
6501	break :s &then_scope.base;
6502	const ident_name = try astgen.identAsString(ident_token);
6503	try astgen.detectLocalShadowing(&then_scope.base, ident_name, ident_token, ident_bytes, .capture);
6504	payload_val_scope = .{
6505	.parent = &then_scope.base,
6506	.gen_zir = &then_scope,
6507	.name = ident_name,
6508	.inst = payload_inst.toRef(),
6509	.token_src = ident_token,
6510	.id_cat = .capture,
6511	};
6512	dbg_var_name = ident_name;
6513	dbg_var_inst = payload_inst.toRef();
6514	break :s &payload_val_scope.base;
6515	} else {
6516	_ = try then_scope.addUnNode(.ensure_err_union_payload_void, cond.inst, node);
6517	break :s &then_scope.base;
6518	}
6519	} else if (while_full.payload_token) \|payload_token\| {
6520	const ident_token = if (payload_is_ref) payload_token + 1 else payload_token;
6521	const tag: Zir.Inst.Tag = if (payload_is_ref)
6522	.optional_payload_unsafe_ptr
6523	else
6524	.optional_payload_unsafe;
6525	// will add this instruction to then_scope.instructions below
6526	const payload_inst = try then_scope.makeUnNode(tag, cond.inst, while_full.ast.cond_expr);
6527	opt_payload_inst = payload_inst.toOptional();
6528	const ident_name = try astgen.identAsString(ident_token);
6529	const ident_bytes = tree.tokenSlice(ident_token);
6530	if (mem.eql(u8, "_", ident_bytes))
6531	break :s &then_scope.base;
6532	try astgen.detectLocalShadowing(&then_scope.base, ident_name, ident_token, ident_bytes, .capture);
6533	payload_val_scope = .{
6534	.parent = &then_scope.base,
6535	.gen_zir = &then_scope,
6536	.name = ident_name,
6537	.inst = payload_inst.toRef(),
6538	.token_src = ident_token,
6539	.id_cat = .capture,
6540	};
6541	dbg_var_name = ident_name;
6542	dbg_var_inst = payload_inst.toRef();
6543	break :s &payload_val_scope.base;
6544	} else {
6545	break :s &then_scope.base;
6546	}
6547	};
6548
6549	var continue_scope = parent_gz.makeSubBlock(then_sub_scope);
6550	continue_scope.instructions_top = GenZir.unstacked_top;
6551	defer continue_scope.unstack();
6552	const continue_block = try then_scope.makeBlockInst(block_tag, node);
6553
6554	const repeat_tag: Zir.Inst.Tag = if (is_inline) .repeat_inline else .repeat;
6555	_ = try loop_scope.addNode(repeat_tag, node);
6556
6557	try loop_scope.setBlockBody(loop_block);
6558	loop_scope.break_block = loop_block.toOptional();
6559	loop_scope.continue_block = continue_block.toOptional();
6560	if (while_full.label_token) \|label_token\| {
6561	loop_scope.label = .{
6562	.token = label_token,
6563	.block_inst = loop_block,
6564	};
6565	}
6566
6567	// done adding instructions to loop_scope, can now stack then_scope
6568	then_scope.instructions_top = then_scope.instructions.items.len;
6569
6570	const then_node = while_full.ast.then_expr;
6571	if (opt_payload_inst.unwrap()) \|payload_inst\| {
6572	try then_scope.instructions.append(astgen.gpa, payload_inst);
6573	}
6574	if (dbg_var_name != .empty) try then_scope.addDbgVar(.dbg_var_val, dbg_var_name, dbg_var_inst);
6575	try then_scope.instructions.append(astgen.gpa, continue_block);
6576	// This code could be improved to avoid emitting the continue expr when there
6577	// are no jumps to it. This happens when the last statement of a while body is noreturn
6578	// and there are no `continue` statements.
6579	// Tracking issue: https://github.com/ziglang/zig/issues/9185
6580	if (while_full.ast.cont_expr != 0) {
6581	_ = try unusedResultExpr(&then_scope, then_sub_scope, while_full.ast.cont_expr);
6582	}
6583
6584	continue_scope.instructions_top = continue_scope.instructions.items.len;
6585	_ = try unusedResultExpr(&continue_scope, &continue_scope.base, then_node);
6586	try checkUsed(parent_gz, &then_scope.base, then_sub_scope);
6587	const break_tag: Zir.Inst.Tag = if (is_inline) .break_inline else .@"break";
6588	if (!continue_scope.endsWithNoReturn()) {
6589	_ = try continue_scope.addBreak(break_tag, continue_block, .void_value);
6590	}
6591	try continue_scope.setBlockBody(continue_block);
6592	_ = try then_scope.addBreak(break_tag, cond_block, .void_value);
6593
6594	var else_scope = parent_gz.makeSubBlock(&cond_scope.base);
6595	defer else_scope.unstack();
6596
6597	const else_node = while_full.ast.else_expr;
6598	if (else_node != 0) {
6599	const sub_scope = s: {
6600	if (while_full.error_token) \|error_token\| {
6601	const tag: Zir.Inst.Tag = if (payload_is_ref)
6602	.err_union_code_ptr
6603	else
6604	.err_union_code;
6605	const else_payload_inst = try else_scope.addUnNode(tag, cond.inst, while_full.ast.cond_expr);
6606	const ident_name = try astgen.identAsString(error_token);
6607	const ident_bytes = tree.tokenSlice(error_token);
6608	if (mem.eql(u8, ident_bytes, "_"))
6609	break :s &else_scope.base;
6610	try astgen.detectLocalShadowing(&else_scope.base, ident_name, error_token, ident_bytes, .capture);
6611	payload_val_scope = .{
6612	.parent = &else_scope.base,
6613	.gen_zir = &else_scope,
6614	.name = ident_name,
6615	.inst = else_payload_inst,
6616	.token_src = error_token,
6617	.id_cat = .capture,
6618	};
6619	try else_scope.addDbgVar(.dbg_var_val, ident_name, else_payload_inst);
6620	break :s &payload_val_scope.base;
6621	} else {
6622	break :s &else_scope.base;
6623	}
6624	};
6625	// Remove the continue block and break block so that `continue` and `break`
6626	// control flow apply to outer loops; not this one.
6627	loop_scope.continue_block = .none;
6628	loop_scope.break_block = .none;
6629	const else_result = try expr(&else_scope, sub_scope, loop_scope.break_result_info, else_node);
6630	if (is_statement) {
6631	_ = try addEnsureResult(&else_scope, else_result, else_node);
6632	}
6633
6634	try checkUsed(parent_gz, &else_scope.base, sub_scope);
6635	if (!else_scope.endsWithNoReturn()) {
6636	_ = try else_scope.addBreakWithSrcNode(break_tag, loop_block, else_result, else_node);
6637	}
6638	} else {
6639	const result = try rvalue(&else_scope, ri, .void_value, node);
6640	_ = try else_scope.addBreak(break_tag, loop_block, result);
6641	}
6642
6643	if (loop_scope.label) \|some\| {
6644	if (!some.used) {
6645	try astgen.appendErrorTok(some.token, "unused while loop label", .{});
6646	}
6647	}
6648
6649	try setCondBrPayload(condbr, cond.bool_bit, &then_scope, &else_scope);
6650
6651	const result = if (need_result_rvalue)
6652	try rvalue(parent_gz, ri, loop_block.toRef(), node)
6653	else
6654	loop_block.toRef();
6655
6656	if (is_statement) {
6657	_ = try parent_gz.addUnNode(.ensure_result_used, result, node);
6658	}
6659
6660	return result;
6661	}
6662
6663	fn forExpr(
6664	parent_gz: *GenZir,
6665	scope: *Scope,
6666	ri: ResultInfo,
6667	node: Ast.Node.Index,
6668	for_full: Ast.full.For,
6669	is_statement: bool,
6670	) InnerError!Zir.Inst.Ref {
6671	const astgen = parent_gz.astgen;
6672
6673	if (for_full.label_token) \|label_token\| {
6674	try astgen.checkLabelRedefinition(scope, label_token);
6675	}
6676
6677	const need_rl = astgen.nodes_need_rl.contains(node);
6678	const block_ri: ResultInfo = if (need_rl) ri else .{
6679	.rl = switch (ri.rl) {
6680	.ptr => .{ .ty = (try ri.rl.resultType(parent_gz, node)).? },
6681	.inferred_ptr => .none,
6682	else => ri.rl,
6683	},
6684	.ctx = ri.ctx,
6685	};
6686	// We need to call `rvalue` to write through to the pointer only if we had a
6687	// result pointer and aren't forwarding it.
6688	const LocTag = @typeInfo(ResultInfo.Loc).Union.tag_type.?;
6689	const need_result_rvalue = @as(LocTag, block_ri.rl) != @as(LocTag, ri.rl);
6690
6691	const is_inline = for_full.inline_token != null;
6692	if (parent_gz.is_comptime and is_inline) {
6693	return astgen.failTok(for_full.inline_token.?, "redundant inline keyword in comptime scope", .{});
6694	}
6695	const tree = astgen.tree;
6696	const token_tags = tree.tokens.items(.tag);
6697	const node_tags = tree.nodes.items(.tag);
6698	const node_data = tree.nodes.items(.data);
6699	const gpa = astgen.gpa;
6700
6701	// For counters, this is the start value; for indexables, this is the base
6702	// pointer that can be used with elem_ptr and similar instructions.
6703	// Special value `none` means that this is a counter and its start value is
6704	// zero, indicating that the main index counter can be used directly.
6705	const indexables = try gpa.alloc(Zir.Inst.Ref, for_full.ast.inputs.len);
6706	defer gpa.free(indexables);
6707	// elements of this array can be `none`, indicating no length check.
6708	const lens = try gpa.alloc(Zir.Inst.Ref, for_full.ast.inputs.len);
6709	defer gpa.free(lens);
6710
6711	// We will use a single zero-based counter no matter how many indexables there are.
6712	const index_ptr = blk: {
6713	const alloc_tag: Zir.Inst.Tag = if (is_inline) .alloc_comptime_mut else .alloc;
6714	const index_ptr = try parent_gz.addUnNode(alloc_tag, .usize_type, node);
6715	// initialize to zero
6716	_ = try parent_gz.addPlNode(.store_node, node, Zir.Inst.Bin{
6717	.lhs = index_ptr,
6718	.rhs = .zero_usize,
6719	});
6720	break :blk index_ptr;
6721	};
6722
6723	var any_len_checks = false;
6724
6725	{
6726	var capture_token = for_full.payload_token;
6727	for (for_full.ast.inputs, indexables, lens) \|input, indexable_ref, len_ref\| {
6728	const capture_is_ref = token_tags[capture_token] == .asterisk;
6729	const ident_tok = capture_token + @intFromBool(capture_is_ref);
6730	const is_discard = mem.eql(u8, tree.tokenSlice(ident_tok), "_");
6731
6732	if (is_discard and capture_is_ref) {
6733	return astgen.failTok(capture_token, "pointer modifier invalid on discard", .{});
6734	}
6735	// Skip over the comma, and on to the next capture (or the ending pipe character).
6736	capture_token = ident_tok + 2;
6737
6738	try emitDbgNode(parent_gz, input);
6739	if (node_tags[input] == .for_range) {
6740	if (capture_is_ref) {
6741	return astgen.failTok(ident_tok, "cannot capture reference to range", .{});
6742	}
6743	const start_node = node_data[input].lhs;
6744	const start_val = try expr(parent_gz, scope, .{ .rl = .{ .ty = .usize_type } }, start_node);
6745
6746	const end_node = node_data[input].rhs;
6747	const end_val = if (end_node != 0)
6748	try expr(parent_gz, scope, .{ .rl = .{ .ty = .usize_type } }, node_data[input].rhs)
6749	else
6750	.none;
6751
6752	if (end_val == .none and is_discard) {
6753	return astgen.failTok(ident_tok, "discard of unbounded counter", .{});
6754	}
6755
6756	const start_is_zero = nodeIsTriviallyZero(tree, start_node);
6757	const range_len = if (end_val == .none or start_is_zero)
6758	end_val
6759	else
6760	try parent_gz.addPlNode(.sub, input, Zir.Inst.Bin{
6761	.lhs = end_val,
6762	.rhs = start_val,
6763	});
6764
6765	any_len_checks = any_len_checks or range_len != .none;
6766	indexable_ref.* = if (start_is_zero) .none else start_val;
6767	len_ref.* = range_len;
6768	} else {
6769	const indexable = try expr(parent_gz, scope, .{ .rl = .none }, input);
6770
6771	any_len_checks = true;
6772	indexable_ref.* = indexable;
6773	len_ref.* = indexable;
6774	}
6775	}
6776	}
6777
6778	if (!any_len_checks) {
6779	return astgen.failNode(node, "unbounded for loop", .{});
6780	}
6781
6782	// We use a dedicated ZIR instruction to assert the lengths to assist with
6783	// nicer error reporting as well as fewer ZIR bytes emitted.
6784	const len: Zir.Inst.Ref = len: {
6785	const lens_len: u32 = @intCast(lens.len);
6786	try astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.MultiOp).Struct.fields.len + lens_len);
6787	const len = try parent_gz.addPlNode(.for_len, node, Zir.Inst.MultiOp{
6788	.operands_len = lens_len,
6789	});
6790	appendRefsAssumeCapacity(astgen, lens);
6791	break :len len;
6792	};
6793
6794	const loop_tag: Zir.Inst.Tag = if (is_inline) .block_inline else .loop;
6795	const loop_block = try parent_gz.makeBlockInst(loop_tag, node);
6796	try parent_gz.instructions.append(gpa, loop_block);
6797
6798	var loop_scope = parent_gz.makeSubBlock(scope);
6799	loop_scope.is_inline = is_inline;
6800	loop_scope.setBreakResultInfo(block_ri);
6801	defer loop_scope.unstack();
6802
6803	// We need to finish loop_scope later once we have the deferred refs from then_scope. However, the
6804	// load must be removed from instructions in the meantime or it appears to be part of parent_gz.
6805	const index = try loop_scope.addUnNode(.load, index_ptr, node);
6806	_ = loop_scope.instructions.pop();
6807
6808	var cond_scope = parent_gz.makeSubBlock(&loop_scope.base);
6809	defer cond_scope.unstack();
6810
6811	// Check the condition.
6812	const cond = try cond_scope.addPlNode(.cmp_lt, node, Zir.Inst.Bin{
6813	.lhs = index,
6814	.rhs = len,
6815	});
6816
6817	const condbr_tag: Zir.Inst.Tag = if (is_inline) .condbr_inline else .condbr;
6818	const condbr = try cond_scope.addCondBr(condbr_tag, node);
6819	const block_tag: Zir.Inst.Tag = if (is_inline) .block_inline else .block;
6820	const cond_block = try loop_scope.makeBlockInst(block_tag, node);
6821	try cond_scope.setBlockBody(cond_block);
6822
6823	loop_scope.break_block = loop_block.toOptional();
6824	loop_scope.continue_block = cond_block.toOptional();
6825	if (for_full.label_token) \|label_token\| {
6826	loop_scope.label = .{
6827	.token = label_token,
6828	.block_inst = loop_block,
6829	};
6830	}
6831
6832	const then_node = for_full.ast.then_expr;
6833	var then_scope = parent_gz.makeSubBlock(&cond_scope.base);
6834	defer then_scope.unstack();
6835
6836	const capture_scopes = try gpa.alloc(Scope.LocalVal, for_full.ast.inputs.len);
6837	defer gpa.free(capture_scopes);
6838
6839	const then_sub_scope = blk: {
6840	var capture_token = for_full.payload_token;
6841	var capture_sub_scope: *Scope = &then_scope.base;
6842	for (for_full.ast.inputs, indexables, capture_scopes) \|input, indexable_ref, *capture_scope\| {
6843	const capture_is_ref = token_tags[capture_token] == .asterisk;
6844	const ident_tok = capture_token + @intFromBool(capture_is_ref);
6845	const capture_name = tree.tokenSlice(ident_tok);
6846	// Skip over the comma, and on to the next capture (or the ending pipe character).
6847	capture_token = ident_tok + 2;
6848
6849	if (mem.eql(u8, capture_name, "_")) continue;
6850
6851	const name_str_index = try astgen.identAsString(ident_tok);
6852	try astgen.detectLocalShadowing(capture_sub_scope, name_str_index, ident_tok, capture_name, .capture);
6853
6854	const capture_inst = inst: {
6855	const is_counter = node_tags[input] == .for_range;
6856
6857	if (indexable_ref == .none) {
6858	// Special case: the main index can be used directly.
6859	assert(is_counter);
6860	assert(!capture_is_ref);
6861	break :inst index;
6862	}
6863
6864	// For counters, we add the index variable to the start value; for
6865	// indexables, we use it as an element index. This is so similar
6866	// that they can share the same code paths, branching only on the
6867	// ZIR tag.
6868	const switch_cond = (@as(u2, @intFromBool(capture_is_ref)) << 1) \| @intFromBool(is_counter);
6869	const tag: Zir.Inst.Tag = switch (switch_cond) {
6870	0b00 => .elem_val,
6871	0b01 => .add,
6872	0b10 => .elem_ptr,
6873	0b11 => unreachable, // compile error emitted already
6874	};
6875	break :inst try then_scope.addPlNode(tag, input, Zir.Inst.Bin{
6876	.lhs = indexable_ref,
6877	.rhs = index,
6878	});
6879	};
6880
6881	capture_scope.* = .{
6882	.parent = capture_sub_scope,
6883	.gen_zir = &then_scope,
6884	.name = name_str_index,
6885	.inst = capture_inst,
6886	.token_src = ident_tok,
6887	.id_cat = .capture,
6888	};
6889
6890	try then_scope.addDbgVar(.dbg_var_val, name_str_index, capture_inst);
6891	capture_sub_scope = &capture_scope.base;
6892	}
6893
6894	break :blk capture_sub_scope;
6895	};
6896
6897	const then_result = try expr(&then_scope, then_sub_scope, .{ .rl = .none }, then_node);
6898	_ = try addEnsureResult(&then_scope, then_result, then_node);
6899
6900	try checkUsed(parent_gz, &then_scope.base, then_sub_scope);
6901
6902	const break_tag: Zir.Inst.Tag = if (is_inline) .break_inline else .@"break";
6903
6904	_ = try then_scope.addBreak(break_tag, cond_block, .void_value);
6905
6906	var else_scope = parent_gz.makeSubBlock(&cond_scope.base);
6907	defer else_scope.unstack();
6908
6909	const else_node = for_full.ast.else_expr;
6910	if (else_node != 0) {
6911	const sub_scope = &else_scope.base;
6912	// Remove the continue block and break block so that `continue` and `break`
6913	// control flow apply to outer loops; not this one.
6914	loop_scope.continue_block = .none;
6915	loop_scope.break_block = .none;
6916	const else_result = try expr(&else_scope, sub_scope, loop_scope.break_result_info, else_node);
6917	if (is_statement) {
6918	_ = try addEnsureResult(&else_scope, else_result, else_node);
6919	}
6920	if (!else_scope.endsWithNoReturn()) {
6921	_ = try else_scope.addBreakWithSrcNode(break_tag, loop_block, else_result, else_node);
6922	}
6923	} else {
6924	const result = try rvalue(&else_scope, ri, .void_value, node);
6925	_ = try else_scope.addBreak(break_tag, loop_block, result);
6926	}
6927
6928	if (loop_scope.label) \|some\| {
6929	if (!some.used) {
6930	try astgen.appendErrorTok(some.token, "unused for loop label", .{});
6931	}
6932	}
6933
6934	try setCondBrPayload(condbr, cond, &then_scope, &else_scope);
6935
6936	// then_block and else_block unstacked now, can resurrect loop_scope to finally finish it
6937	{
6938	loop_scope.instructions_top = loop_scope.instructions.items.len;
6939	try loop_scope.instructions.appendSlice(gpa, &.{ index.toIndex().?, cond_block });
6940
6941	// Increment the index variable.
6942	const index_plus_one = try loop_scope.addPlNode(.add_unsafe, node, Zir.Inst.Bin{
6943	.lhs = index,
6944	.rhs = .one_usize,
6945	});
6946	_ = try loop_scope.addPlNode(.store_node, node, Zir.Inst.Bin{
6947	.lhs = index_ptr,
6948	.rhs = index_plus_one,
6949	});
6950	const repeat_tag: Zir.Inst.Tag = if (is_inline) .repeat_inline else .repeat;
6951	_ = try loop_scope.addNode(repeat_tag, node);
6952
6953	try loop_scope.setBlockBody(loop_block);
6954	}
6955
6956	const result = if (need_result_rvalue)
6957	try rvalue(parent_gz, ri, loop_block.toRef(), node)
6958	else
6959	loop_block.toRef();
6960
6961	if (is_statement) {
6962	_ = try parent_gz.addUnNode(.ensure_result_used, result, node);
6963	}
6964	return result;
6965	}
6966
6967	fn switchExprErrUnion(
6968	parent_gz: *GenZir,
6969	scope: *Scope,
6970	ri: ResultInfo,
6971	catch_or_if_node: Ast.Node.Index,
6972	node_ty: enum { @"catch", @"if" },
6973	) InnerError!Zir.Inst.Ref {
6974	const astgen = parent_gz.astgen;
6975	const gpa = astgen.gpa;
6976	const tree = astgen.tree;
6977	const node_datas = tree.nodes.items(.data);
6978	const node_tags = tree.nodes.items(.tag);
6979	const main_tokens = tree.nodes.items(.main_token);
6980	const token_tags = tree.tokens.items(.tag);
6981
6982	const if_full = switch (node_ty) {
6983	.@"catch" => undefined,
6984	.@"if" => tree.fullIf(catch_or_if_node).?,
6985	};
6986
6987	const switch_node, const operand_node, const error_payload = switch (node_ty) {
6988	.@"catch" => .{
6989	node_datas[catch_or_if_node].rhs,
6990	node_datas[catch_or_if_node].lhs,
6991	main_tokens[catch_or_if_node] + 2,
6992	},
6993	.@"if" => .{
6994	if_full.ast.else_expr,
6995	if_full.ast.cond_expr,
6996	if_full.error_token.?,
6997	},
6998	};
6999	assert(node_tags[switch_node] == .@"switch" or node_tags[switch_node] == .switch_comma);
7000
7001	const do_err_trace = astgen.fn_block != null;
7002
7003	const extra = tree.extraData(node_datas[switch_node].rhs, Ast.Node.SubRange);
7004	const case_nodes = tree.extra_data[extra.start..extra.end];
7005
7006	const need_rl = astgen.nodes_need_rl.contains(catch_or_if_node);
7007	const block_ri: ResultInfo = if (need_rl) ri else .{
7008	.rl = switch (ri.rl) {
7009	.ptr => .{ .ty = (try ri.rl.resultType(parent_gz, catch_or_if_node)).? },
7010	.inferred_ptr => .none,
7011	else => ri.rl,
7012	},
7013	.ctx = ri.ctx,
7014	};
7015
7016	const payload_is_ref = node_ty == .@"if" and
7017	if_full.payload_token != null and token_tags[if_full.payload_token.?] == .asterisk;
7018
7019	// We need to call `rvalue` to write through to the pointer only if we had a
7020	// result pointer and aren't forwarding it.
7021	const LocTag = @typeInfo(ResultInfo.Loc).Union.tag_type.?;
7022	const need_result_rvalue = @as(LocTag, block_ri.rl) != @as(LocTag, ri.rl);
7023	var scalar_cases_len: u32 = 0;
7024	var multi_cases_len: u32 = 0;
7025	var inline_cases_len: u32 = 0;
7026	var has_else = false;
7027	var else_node: Ast.Node.Index = 0;
7028	var else_src: ?Ast.TokenIndex = null;
7029	for (case_nodes) \|case_node\| {
7030	const case = tree.fullSwitchCase(case_node).?;
7031
7032	if (case.ast.values.len == 0) {
7033	const case_src = case.ast.arrow_token - 1;
7034	if (else_src) \|src\| {
7035	return astgen.failTokNotes(
7036	case_src,
7037	"multiple else prongs in switch expression",
7038	.{},
7039	&[_]u32{
7040	try astgen.errNoteTok(
7041	src,
7042	"previous else prong here",
7043	.{},
7044	),
7045	},
7046	);
7047	}
7048	has_else = true;
7049	else_node = case_node;
7050	else_src = case_src;
7051	continue;
7052	} else if (case.ast.values.len == 1 and
7053	node_tags[case.ast.values[0]] == .identifier and
7054	mem.eql(u8, tree.tokenSlice(main_tokens[case.ast.values[0]]), "_"))
7055	{
7056	const case_src = case.ast.arrow_token - 1;
7057	return astgen.failTokNotes(
7058	case_src,
7059	"'_' prong is not allowed when switching on errors",
7060	.{},
7061	&[_]u32{
7062	try astgen.errNoteTok(
7063	case_src,
7064	"consider using 'else'",
7065	.{},
7066	),
7067	},
7068	);
7069	}
7070
7071	for (case.ast.values) \|val\| {
7072	if (node_tags[val] == .string_literal)
7073	return astgen.failNode(val, "cannot switch on strings", .{});
7074	}
7075
7076	if (case.ast.values.len == 1 and node_tags[case.ast.values[0]] != .switch_range) {
7077	scalar_cases_len += 1;
7078	} else {
7079	multi_cases_len += 1;
7080	}
7081	if (case.inline_token != null) {
7082	inline_cases_len += 1;
7083	}
7084	}
7085
7086	const operand_ri: ResultInfo = .{
7087	.rl = if (payload_is_ref) .ref else .none,
7088	.ctx = .error_handling_expr,
7089	};
7090
7091	astgen.advanceSourceCursorToNode(operand_node);
7092	const operand_lc = LineColumn{ astgen.source_line - parent_gz.decl_line, astgen.source_column };
7093
7094	const raw_operand = try reachableExpr(parent_gz, scope, operand_ri, operand_node, switch_node);
7095	const item_ri: ResultInfo = .{ .rl = .none };
7096
7097	// This contains the data that goes into the `extra` array for the SwitchBlockErrUnion, except
7098	// the first cases_nodes.len slots are a table that indexes payloads later in the array,
7099	// with the non-error and else case indices coming first, then scalar_cases_len indexes, then
7100	// multi_cases_len indexes
7101	const payloads = &astgen.scratch;
7102	const scratch_top = astgen.scratch.items.len;
7103	const case_table_start = scratch_top;
7104	const scalar_case_table = case_table_start + 1 + @intFromBool(has_else);
7105	const multi_case_table = scalar_case_table + scalar_cases_len;
7106	const case_table_end = multi_case_table + multi_cases_len;
7107
7108	try astgen.scratch.resize(gpa, case_table_end);
7109	defer astgen.scratch.items.len = scratch_top;
7110
7111	var block_scope = parent_gz.makeSubBlock(scope);
7112	// block_scope not used for collecting instructions
7113	block_scope.instructions_top = GenZir.unstacked_top;
7114	block_scope.setBreakResultInfo(block_ri);
7115
7116	// Sema expects a dbg_stmt immediately before switch_block_err_union
7117	try emitDbgStmtForceCurrentIndex(parent_gz, operand_lc);
7118	// This gets added to the parent block later, after the item expressions.
7119	const switch_block = try parent_gz.makeBlockInst(.switch_block_err_union, switch_node);
7120
7121	// We re-use this same scope for all cases, including the special prong, if any.
7122	var case_scope = parent_gz.makeSubBlock(&block_scope.base);
7123	case_scope.instructions_top = GenZir.unstacked_top;
7124
7125	{
7126	const body_len_index: u32 = @intCast(payloads.items.len);
7127	payloads.items[case_table_start] = body_len_index;
7128	try payloads.resize(gpa, body_len_index + 1); // body_len
7129
7130	case_scope.instructions_top = parent_gz.instructions.items.len;
7131	defer case_scope.unstack();
7132
7133	const unwrap_payload_tag: Zir.Inst.Tag = if (payload_is_ref)
7134	.err_union_payload_unsafe_ptr
7135	else
7136	.err_union_payload_unsafe;
7137
7138	const unwrapped_payload = try case_scope.addUnNode(
7139	unwrap_payload_tag,
7140	raw_operand,
7141	catch_or_if_node,
7142	);
7143
7144	switch (node_ty) {
7145	.@"catch" => {
7146	const case_result = switch (ri.rl) {
7147	.ref, .ref_coerced_ty => unwrapped_payload,
7148	else => try rvalue(
7149	&case_scope,
7150	block_scope.break_result_info,
7151	unwrapped_payload,
7152	catch_or_if_node,
7153	),
7154	};
7155	_ = try case_scope.addBreakWithSrcNode(
7156	.@"break",
7157	switch_block,
7158	case_result,
7159	catch_or_if_node,
7160	);
7161	},
7162	.@"if" => {
7163	var payload_val_scope: Scope.LocalVal = undefined;
7164
7165	const then_node = if_full.ast.then_expr;
7166	const then_sub_scope = s: {
7167	assert(if_full.error_token != null);
7168	if (if_full.payload_token) \|payload_token\| {
7169	const token_name_index = payload_token + @intFromBool(payload_is_ref);
7170	const ident_name = try astgen.identAsString(token_name_index);
7171	const token_name_str = tree.tokenSlice(token_name_index);
7172	if (mem.eql(u8, "_", token_name_str))
7173	break :s &case_scope.base;
7174	try astgen.detectLocalShadowing(
7175	&case_scope.base,
7176	ident_name,
7177	token_name_index,
7178	token_name_str,
7179	.capture,
7180	);
7181	payload_val_scope = .{
7182	.parent = &case_scope.base,
7183	.gen_zir = &case_scope,
7184	.name = ident_name,
7185	.inst = unwrapped_payload,
7186	.token_src = token_name_index,
7187	.id_cat = .capture,
7188	};
7189	try case_scope.addDbgVar(.dbg_var_val, ident_name, unwrapped_payload);
7190	break :s &payload_val_scope.base;
7191	} else {
7192	_ = try case_scope.addUnNode(
7193	.ensure_err_union_payload_void,
7194	raw_operand,
7195	catch_or_if_node,
7196	);
7197	break :s &case_scope.base;
7198	}
7199	};
7200	const then_result = try expr(
7201	&case_scope,
7202	then_sub_scope,
7203	block_scope.break_result_info,
7204	then_node,
7205	);
7206	try checkUsed(parent_gz, &case_scope.base, then_sub_scope);
7207	if (!case_scope.endsWithNoReturn()) {
7208	_ = try case_scope.addBreakWithSrcNode(
7209	.@"break",
7210	switch_block,
7211	then_result,
7212	then_node,
7213	);
7214	}
7215	},
7216	}
7217
7218	const case_slice = case_scope.instructionsSlice();
7219	// Since we use the switch_block_err_union instruction itself to refer
7220	// to the capture, which will not be added to the child block, we need
7221	// to handle ref_table manually.
7222	const refs_len = refs: {
7223	var n: usize = 0;
7224	var check_inst = switch_block;
7225	while (astgen.ref_table.get(check_inst)) \|ref_inst\| {
7226	n += 1;
7227	check_inst = ref_inst;
7228	}
7229	break :refs n;
7230	};
7231	const body_len = refs_len + astgen.countBodyLenAfterFixups(case_slice);
7232	try payloads.ensureUnusedCapacity(gpa, body_len);
7233	const capture: Zir.Inst.SwitchBlock.ProngInfo.Capture = switch (node_ty) {
7234	.@"catch" => .none,
7235	.@"if" => if (if_full.payload_token == null)
7236	.none
7237	else if (payload_is_ref)
7238	.by_ref
7239	else
7240	.by_val,
7241	};
7242	payloads.items[body_len_index] = @bitCast(Zir.Inst.SwitchBlock.ProngInfo{
7243	.body_len = @intCast(body_len),
7244	.capture = capture,
7245	.is_inline = false,
7246	.has_tag_capture = false,
7247	});
7248	if (astgen.ref_table.fetchRemove(switch_block)) \|kv\| {
7249	appendPossiblyRefdBodyInst(astgen, payloads, kv.value);
7250	}
7251	appendBodyWithFixupsArrayList(astgen, payloads, case_slice);
7252	}
7253
7254	const err_name = blk: {
7255	const err_str = tree.tokenSlice(error_payload);
7256	if (mem.eql(u8, err_str, "_")) {
7257	return astgen.failTok(error_payload, "discard of error capture; omit it instead", .{});
7258	}
7259	const err_name = try astgen.identAsString(error_payload);
7260	try astgen.detectLocalShadowing(scope, err_name, error_payload, err_str, .capture);
7261
7262	break :blk err_name;
7263	};
7264
7265	// allocate a shared dummy instruction for the error capture
7266	const err_inst = err_inst: {
7267	const inst: Zir.Inst.Index = @enumFromInt(astgen.instructions.len);
7268	try astgen.instructions.append(astgen.gpa, .{
7269	.tag = .extended,
7270	.data = .{ .extended = .{
7271	.opcode = .value_placeholder,
7272	.small = undefined,
7273	.operand = undefined,
7274	} },
7275	});
7276	break :err_inst inst;
7277	};
7278
7279	// In this pass we generate all the item and prong expressions for error cases.
7280	var multi_case_index: u32 = 0;
7281	var scalar_case_index: u32 = 0;
7282	var any_uses_err_capture = false;
7283	for (case_nodes) \|case_node\| {
7284	const case = tree.fullSwitchCase(case_node).?;
7285
7286	const is_multi_case = case.ast.values.len > 1 or
7287	(case.ast.values.len == 1 and node_tags[case.ast.values[0]] == .switch_range);
7288
7289	var dbg_var_name: Zir.NullTerminatedString = .empty;
7290	var dbg_var_inst: Zir.Inst.Ref = undefined;
7291	var err_scope: Scope.LocalVal = undefined;
7292	var capture_scope: Scope.LocalVal = undefined;
7293
7294	const sub_scope = blk: {
7295	err_scope = .{
7296	.parent = &case_scope.base,
7297	.gen_zir = &case_scope,
7298	.name = err_name,
7299	.inst = err_inst.toRef(),
7300	.token_src = error_payload,
7301	.id_cat = .capture,
7302	};
7303
7304	const capture_token = case.payload_token orelse break :blk &err_scope.base;
7305	if (token_tags[capture_token] != .identifier) {
7306	return astgen.failTok(capture_token + 1, "error set cannot be captured by reference", .{});
7307	}
7308
7309	const capture_slice = tree.tokenSlice(capture_token);
7310	if (mem.eql(u8, capture_slice, "_")) {
7311	return astgen.failTok(capture_token, "discard of error capture; omit it instead", .{});
7312	}
7313	const tag_name = try astgen.identAsString(capture_token);
7314	try astgen.detectLocalShadowing(&case_scope.base, tag_name, capture_token, capture_slice, .capture);
7315
7316	capture_scope = .{
7317	.parent = &case_scope.base,
7318	.gen_zir = &case_scope,
7319	.name = tag_name,
7320	.inst = switch_block.toRef(),
7321	.token_src = capture_token,
7322	.id_cat = .capture,
7323	};
7324	dbg_var_name = tag_name;
7325	dbg_var_inst = switch_block.toRef();
7326
7327	err_scope.parent = &capture_scope.base;
7328
7329	break :blk &err_scope.base;
7330	};
7331
7332	const header_index: u32 = @intCast(payloads.items.len);
7333	const body_len_index = if (is_multi_case) blk: {
7334	payloads.items[multi_case_table + multi_case_index] = header_index;
7335	multi_case_index += 1;
7336	try payloads.resize(gpa, header_index + 3); // items_len, ranges_len, body_len
7337
7338	// items
7339	var items_len: u32 = 0;
7340	for (case.ast.values) \|item_node\| {
7341	if (node_tags[item_node] == .switch_range) continue;
7342	items_len += 1;
7343
7344	const item_inst = try comptimeExpr(parent_gz, scope, item_ri, item_node);
7345	try payloads.append(gpa, @intFromEnum(item_inst));
7346	}
7347
7348	// ranges
7349	var ranges_len: u32 = 0;
7350	for (case.ast.values) \|range\| {
7351	if (node_tags[range] != .switch_range) continue;
7352	ranges_len += 1;
7353
7354	const first = try comptimeExpr(parent_gz, scope, item_ri, node_datas[range].lhs);
7355	const last = try comptimeExpr(parent_gz, scope, item_ri, node_datas[range].rhs);
7356	try payloads.appendSlice(gpa, &[_]u32{
7357	@intFromEnum(first), @intFromEnum(last),
7358	});
7359	}
7360
7361	payloads.items[header_index] = items_len;
7362	payloads.items[header_index + 1] = ranges_len;
7363	break :blk header_index + 2;
7364	} else if (case_node == else_node) blk: {
7365	payloads.items[case_table_start + 1] = header_index;
7366	try payloads.resize(gpa, header_index + 1); // body_len
7367	break :blk header_index;
7368	} else blk: {
7369	payloads.items[scalar_case_table + scalar_case_index] = header_index;
7370	scalar_case_index += 1;
7371	try payloads.resize(gpa, header_index + 2); // item, body_len
7372	const item_node = case.ast.values[0];
7373	const item_inst = try comptimeExpr(parent_gz, scope, item_ri, item_node);
7374	payloads.items[header_index] = @intFromEnum(item_inst);
7375	break :blk header_index + 1;
7376	};
7377
7378	{
7379	// temporarily stack case_scope on parent_gz
7380	case_scope.instructions_top = parent_gz.instructions.items.len;
7381	defer case_scope.unstack();
7382
7383	if (do_err_trace and nodeMayAppendToErrorTrace(tree, operand_node))
7384	_ = try case_scope.addSaveErrRetIndex(.always);
7385
7386	if (dbg_var_name != .empty) {
7387	try case_scope.addDbgVar(.dbg_var_val, dbg_var_name, dbg_var_inst);
7388	}
7389
7390	const target_expr_node = case.ast.target_expr;
7391	const case_result = try expr(&case_scope, sub_scope, block_scope.break_result_info, target_expr_node);
7392	// check capture_scope, not err_scope to avoid false positive unused error capture
7393	try checkUsed(parent_gz, &case_scope.base, err_scope.parent);
7394	const uses_err = err_scope.used != 0 or err_scope.discarded != 0;
7395	if (uses_err) {
7396	try case_scope.addDbgVar(.dbg_var_val, err_name, err_inst.toRef());
7397	any_uses_err_capture = true;
7398	}
7399
7400	if (!parent_gz.refIsNoReturn(case_result)) {
7401	if (do_err_trace)
7402	try restoreErrRetIndex(
7403	&case_scope,
7404	.{ .block = switch_block },
7405	block_scope.break_result_info,
7406	target_expr_node,
7407	case_result,
7408	);
7409
7410	_ = try case_scope.addBreakWithSrcNode(.@"break", switch_block, case_result, target_expr_node);
7411	}
7412
7413	const case_slice = case_scope.instructionsSlice();
7414	// Since we use the switch_block_err_union instruction itself to refer
7415	// to the capture, which will not be added to the child block, we need
7416	// to handle ref_table manually.
7417	const refs_len = refs: {
7418	var n: usize = 0;
7419	var check_inst = switch_block;
7420	while (astgen.ref_table.get(check_inst)) \|ref_inst\| {
7421	n += 1;
7422	check_inst = ref_inst;
7423	}
7424	if (uses_err) {
7425	check_inst = err_inst;
7426	while (astgen.ref_table.get(check_inst)) \|ref_inst\| {
7427	n += 1;
7428	check_inst = ref_inst;
7429	}
7430	}
7431	break :refs n;
7432	};
7433	const body_len = refs_len + astgen.countBodyLenAfterFixups(case_slice);
7434	try payloads.ensureUnusedCapacity(gpa, body_len);
7435	payloads.items[body_len_index] = @bitCast(Zir.Inst.SwitchBlock.ProngInfo{
7436	.body_len = @intCast(body_len),
7437	.capture = if (case.payload_token != null) .by_val else .none,
7438	.is_inline = case.inline_token != null,
7439	.has_tag_capture = false,
7440	});
7441	if (astgen.ref_table.fetchRemove(switch_block)) \|kv\| {
7442	appendPossiblyRefdBodyInst(astgen, payloads, kv.value);
7443	}
7444	if (uses_err) {
7445	if (astgen.ref_table.fetchRemove(err_inst)) \|kv\| {
7446	appendPossiblyRefdBodyInst(astgen, payloads, kv.value);
7447	}
7448	}
7449	appendBodyWithFixupsArrayList(astgen, payloads, case_slice);
7450	}
7451	}
7452	// Now that the item expressions are generated we can add this.
7453	try parent_gz.instructions.append(gpa, switch_block);
7454
7455	try astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.SwitchBlockErrUnion).Struct.fields.len +
7456	@intFromBool(multi_cases_len != 0) +
7457	payloads.items.len - case_table_end +
7458	(case_table_end - case_table_start) * @typeInfo(Zir.Inst.As).Struct.fields.len);
7459
7460	const payload_index = astgen.addExtraAssumeCapacity(Zir.Inst.SwitchBlockErrUnion{
7461	.operand = raw_operand,
7462	.bits = Zir.Inst.SwitchBlockErrUnion.Bits{
7463	.has_multi_cases = multi_cases_len != 0,
7464	.has_else = has_else,
7465	.scalar_cases_len = @intCast(scalar_cases_len),
7466	.any_uses_err_capture = any_uses_err_capture,
7467	.payload_is_ref = payload_is_ref,
7468	},
7469	.main_src_node_offset = parent_gz.nodeIndexToRelative(catch_or_if_node),
7470	});
7471
7472	if (multi_cases_len != 0) {
7473	astgen.extra.appendAssumeCapacity(multi_cases_len);
7474	}
7475
7476	if (any_uses_err_capture) {
7477	astgen.extra.appendAssumeCapacity(@intFromEnum(err_inst));
7478	}
7479
7480	const zir_datas = astgen.instructions.items(.data);
7481	zir_datas[@intFromEnum(switch_block)].pl_node.payload_index = payload_index;
7482
7483	for (payloads.items[case_table_start..case_table_end], 0..) \|start_index, i\| {
7484	var body_len_index = start_index;
7485	var end_index = start_index;
7486	const table_index = case_table_start + i;
7487	if (table_index < scalar_case_table) {
7488	end_index += 1;
7489	} else if (table_index < multi_case_table) {
7490	body_len_index += 1;
7491	end_index += 2;
7492	} else {
7493	body_len_index += 2;
7494	const items_len = payloads.items[start_index];
7495	const ranges_len = payloads.items[start_index + 1];
7496	end_index += 3 + items_len + 2 * ranges_len;
7497	}
7498	const prong_info: Zir.Inst.SwitchBlock.ProngInfo = @bitCast(payloads.items[body_len_index]);
7499	end_index += prong_info.body_len;
7500	astgen.extra.appendSliceAssumeCapacity(payloads.items[start_index..end_index]);
7501	}
7502
7503	if (need_result_rvalue) {
7504	return rvalue(parent_gz, ri, switch_block.toRef(), switch_node);
7505	} else {
7506	return switch_block.toRef();
7507	}
7508	}
7509
7510	fn switchExpr(
7511	parent_gz: *GenZir,
7512	scope: *Scope,
7513	ri: ResultInfo,
7514	switch_node: Ast.Node.Index,
7515	) InnerError!Zir.Inst.Ref {
7516	const astgen = parent_gz.astgen;
7517	const gpa = astgen.gpa;
7518	const tree = astgen.tree;
7519	const node_datas = tree.nodes.items(.data);
7520	const node_tags = tree.nodes.items(.tag);
7521	const main_tokens = tree.nodes.items(.main_token);
7522	const token_tags = tree.tokens.items(.tag);
7523	const operand_node = node_datas[switch_node].lhs;
7524	const extra = tree.extraData(node_datas[switch_node].rhs, Ast.Node.SubRange);
7525	const case_nodes = tree.extra_data[extra.start..extra.end];
7526
7527	const need_rl = astgen.nodes_need_rl.contains(switch_node);
7528	const block_ri: ResultInfo = if (need_rl) ri else .{
7529	.rl = switch (ri.rl) {
7530	.ptr => .{ .ty = (try ri.rl.resultType(parent_gz, switch_node)).? },
7531	.inferred_ptr => .none,
7532	else => ri.rl,
7533	},
7534	.ctx = ri.ctx,
7535	};
7536	// We need to call `rvalue` to write through to the pointer only if we had a
7537	// result pointer and aren't forwarding it.
7538	const LocTag = @typeInfo(ResultInfo.Loc).Union.tag_type.?;
7539	const need_result_rvalue = @as(LocTag, block_ri.rl) != @as(LocTag, ri.rl);
7540
7541	// We perform two passes over the AST. This first pass is to collect information
7542	// for the following variables, make note of the special prong AST node index,
7543	// and bail out with a compile error if there are multiple special prongs present.
7544	var any_payload_is_ref = false;
7545	var any_has_tag_capture = false;
7546	var scalar_cases_len: u32 = 0;
7547	var multi_cases_len: u32 = 0;
7548	var inline_cases_len: u32 = 0;
7549	var special_prong: Zir.SpecialProng = .none;
7550	var special_node: Ast.Node.Index = 0;
7551	var else_src: ?Ast.TokenIndex = null;
7552	var underscore_src: ?Ast.TokenIndex = null;
7553	for (case_nodes) \|case_node\| {
7554	const case = tree.fullSwitchCase(case_node).?;
7555	if (case.payload_token) \|payload_token\| {
7556	const ident = if (token_tags[payload_token] == .asterisk) blk: {
7557	any_payload_is_ref = true;
7558	break :blk payload_token + 1;
7559	} else payload_token;
7560	if (token_tags[ident + 1] == .comma) {
7561	any_has_tag_capture = true;
7562	}
7563	}
7564	// Check for else/`_` prong.
7565	if (case.ast.values.len == 0) {
7566	const case_src = case.ast.arrow_token - 1;
7567	if (else_src) \|src\| {
7568	return astgen.failTokNotes(
7569	case_src,
7570	"multiple else prongs in switch expression",
7571	.{},
7572	&[_]u32{
7573	try astgen.errNoteTok(
7574	src,
7575	"previous else prong here",
7576	.{},
7577	),
7578	},
7579	);
7580	} else if (underscore_src) \|some_underscore\| {
7581	return astgen.failNodeNotes(
7582	switch_node,
7583	"else and '_' prong in switch expression",
7584	.{},
7585	&[_]u32{
7586	try astgen.errNoteTok(
7587	case_src,
7588	"else prong here",
7589	.{},
7590	),
7591	try astgen.errNoteTok(
7592	some_underscore,
7593	"'_' prong here",
7594	.{},
7595	),
7596	},
7597	);
7598	}
7599	special_node = case_node;
7600	special_prong = .@"else";
7601	else_src = case_src;
7602	continue;
7603	} else if (case.ast.values.len == 1 and
7604	node_tags[case.ast.values[0]] == .identifier and
7605	mem.eql(u8, tree.tokenSlice(main_tokens[case.ast.values[0]]), "_"))
7606	{
7607	const case_src = case.ast.arrow_token - 1;
7608	if (underscore_src) \|src\| {
7609	return astgen.failTokNotes(
7610	case_src,
7611	"multiple '_' prongs in switch expression",
7612	.{},
7613	&[_]u32{
7614	try astgen.errNoteTok(
7615	src,
7616	"previous '_' prong here",
7617	.{},
7618	),
7619	},
7620	);
7621	} else if (else_src) \|some_else\| {
7622	return astgen.failNodeNotes(
7623	switch_node,
7624	"else and '_' prong in switch expression",
7625	.{},
7626	&[_]u32{
7627	try astgen.errNoteTok(
7628	some_else,
7629	"else prong here",
7630	.{},
7631	),
7632	try astgen.errNoteTok(
7633	case_src,
7634	"'_' prong here",
7635	.{},
7636	),
7637	},
7638	);
7639	}
7640	if (case.inline_token != null) {
7641	return astgen.failTok(case_src, "cannot inline '_' prong", .{});
7642	}
7643	special_node = case_node;
7644	special_prong = .under;
7645	underscore_src = case_src;
7646	continue;
7647	}
7648
7649	for (case.ast.values) \|val\| {
7650	if (node_tags[val] == .string_literal)
7651	return astgen.failNode(val, "cannot switch on strings", .{});
7652	}
7653
7654	if (case.ast.values.len == 1 and node_tags[case.ast.values[0]] != .switch_range) {
7655	scalar_cases_len += 1;
7656	} else {
7657	multi_cases_len += 1;
7658	}
7659	if (case.inline_token != null) {
7660	inline_cases_len += 1;
7661	}
7662	}
7663
7664	const operand_ri: ResultInfo = .{ .rl = if (any_payload_is_ref) .ref else .none };
7665
7666	astgen.advanceSourceCursorToNode(operand_node);
7667	const operand_lc = LineColumn{ astgen.source_line - parent_gz.decl_line, astgen.source_column };
7668
7669	const raw_operand = try expr(parent_gz, scope, operand_ri, operand_node);
7670	const item_ri: ResultInfo = .{ .rl = .none };
7671
7672	// This contains the data that goes into the `extra` array for the SwitchBlock/SwitchBlockMulti,
7673	// except the first cases_nodes.len slots are a table that indexes payloads later in the array, with
7674	// the special case index coming first, then scalar_case_len indexes, then multi_cases_len indexes
7675	const payloads = &astgen.scratch;
7676	const scratch_top = astgen.scratch.items.len;
7677	const case_table_start = scratch_top;
7678	const scalar_case_table = case_table_start + @intFromBool(special_prong != .none);
7679	const multi_case_table = scalar_case_table + scalar_cases_len;
7680	const case_table_end = multi_case_table + multi_cases_len;
7681	try astgen.scratch.resize(gpa, case_table_end);
7682	defer astgen.scratch.items.len = scratch_top;
7683
7684	var block_scope = parent_gz.makeSubBlock(scope);
7685	// block_scope not used for collecting instructions
7686	block_scope.instructions_top = GenZir.unstacked_top;
7687	block_scope.setBreakResultInfo(block_ri);
7688
7689	// Sema expects a dbg_stmt immediately before switch_block(_ref)
7690	try emitDbgStmtForceCurrentIndex(parent_gz, operand_lc);
7691	// This gets added to the parent block later, after the item expressions.
7692	const switch_tag: Zir.Inst.Tag = if (any_payload_is_ref) .switch_block_ref else .switch_block;
7693	const switch_block = try parent_gz.makeBlockInst(switch_tag, switch_node);
7694
7695	// We re-use this same scope for all cases, including the special prong, if any.
7696	var case_scope = parent_gz.makeSubBlock(&block_scope.base);
7697	case_scope.instructions_top = GenZir.unstacked_top;
7698
7699	// If any prong has an inline tag capture, allocate a shared dummy instruction for it
7700	const tag_inst = if (any_has_tag_capture) tag_inst: {
7701	const inst: Zir.Inst.Index = @enumFromInt(astgen.instructions.len);
7702	try astgen.instructions.append(astgen.gpa, .{
7703	.tag = .extended,
7704	.data = .{ .extended = .{
7705	.opcode = .value_placeholder,
7706	.small = undefined,
7707	.operand = undefined,
7708	} },
7709	});
7710	break :tag_inst inst;
7711	} else undefined;
7712
7713	// In this pass we generate all the item and prong expressions.
7714	var multi_case_index: u32 = 0;
7715	var scalar_case_index: u32 = 0;
7716	for (case_nodes) \|case_node\| {
7717	const case = tree.fullSwitchCase(case_node).?;
7718
7719	const is_multi_case = case.ast.values.len > 1 or
7720	(case.ast.values.len == 1 and node_tags[case.ast.values[0]] == .switch_range);
7721
7722	var dbg_var_name: Zir.NullTerminatedString = .empty;
7723	var dbg_var_inst: Zir.Inst.Ref = undefined;
7724	var dbg_var_tag_name: Zir.NullTerminatedString = .empty;
7725	var dbg_var_tag_inst: Zir.Inst.Ref = undefined;
7726	var has_tag_capture = false;
7727	var capture_val_scope: Scope.LocalVal = undefined;
7728	var tag_scope: Scope.LocalVal = undefined;
7729
7730	var capture: Zir.Inst.SwitchBlock.ProngInfo.Capture = .none;
7731
7732	const sub_scope = blk: {
7733	const payload_token = case.payload_token orelse break :blk &case_scope.base;
7734	const ident = if (token_tags[payload_token] == .asterisk)
7735	payload_token + 1
7736	else
7737	payload_token;
7738
7739	const is_ptr = ident != payload_token;
7740	capture = if (is_ptr) .by_ref else .by_val;
7741
7742	const ident_slice = tree.tokenSlice(ident);
7743	var payload_sub_scope: *Scope = undefined;
7744	if (mem.eql(u8, ident_slice, "_")) {
7745	if (is_ptr) {
7746	return astgen.failTok(payload_token, "pointer modifier invalid on discard", .{});
7747	}
7748	payload_sub_scope = &case_scope.base;
7749	} else {
7750	const capture_name = try astgen.identAsString(ident);
7751	try astgen.detectLocalShadowing(&case_scope.base, capture_name, ident, ident_slice, .capture);
7752	capture_val_scope = .{
7753	.parent = &case_scope.base,
7754	.gen_zir = &case_scope,
7755	.name = capture_name,
7756	.inst = switch_block.toRef(),
7757	.token_src = ident,
7758	.id_cat = .capture,
7759	};
7760	dbg_var_name = capture_name;
7761	dbg_var_inst = switch_block.toRef();
7762	payload_sub_scope = &capture_val_scope.base;
7763	}
7764
7765	const tag_token = if (token_tags[ident + 1] == .comma)
7766	ident + 2
7767	else
7768	break :blk payload_sub_scope;
7769	const tag_slice = tree.tokenSlice(tag_token);
7770	if (mem.eql(u8, tag_slice, "_")) {
7771	return astgen.failTok(tag_token, "discard of tag capture; omit it instead", .{});
7772	} else if (case.inline_token == null) {
7773	return astgen.failTok(tag_token, "tag capture on non-inline prong", .{});
7774	}
7775	const tag_name = try astgen.identAsString(tag_token);
7776	try astgen.detectLocalShadowing(payload_sub_scope, tag_name, tag_token, tag_slice, .@"switch tag capture");
7777
7778	assert(any_has_tag_capture);
7779	has_tag_capture = true;
7780
7781	tag_scope = .{
7782	.parent = payload_sub_scope,
7783	.gen_zir = &case_scope,
7784	.name = tag_name,
7785	.inst = tag_inst.toRef(),
7786	.token_src = tag_token,
7787	.id_cat = .@"switch tag capture",
7788	};
7789	dbg_var_tag_name = tag_name;
7790	dbg_var_tag_inst = tag_inst.toRef();
7791	break :blk &tag_scope.base;
7792	};
7793
7794	const header_index: u32 = @intCast(payloads.items.len);
7795	const body_len_index = if (is_multi_case) blk: {
7796	payloads.items[multi_case_table + multi_case_index] = header_index;
7797	multi_case_index += 1;
7798	try payloads.resize(gpa, header_index + 3); // items_len, ranges_len, body_len
7799
7800	// items
7801	var items_len: u32 = 0;
7802	for (case.ast.values) \|item_node\| {
7803	if (node_tags[item_node] == .switch_range) continue;
7804	items_len += 1;
7805
7806	const item_inst = try comptimeExpr(parent_gz, scope, item_ri, item_node);
7807	try payloads.append(gpa, @intFromEnum(item_inst));
7808	}
7809
7810	// ranges
7811	var ranges_len: u32 = 0;
7812	for (case.ast.values) \|range\| {
7813	if (node_tags[range] != .switch_range) continue;
7814	ranges_len += 1;
7815
7816	const first = try comptimeExpr(parent_gz, scope, item_ri, node_datas[range].lhs);
7817	const last = try comptimeExpr(parent_gz, scope, item_ri, node_datas[range].rhs);
7818	try payloads.appendSlice(gpa, &[_]u32{
7819	@intFromEnum(first), @intFromEnum(last),
7820	});
7821	}
7822
7823	payloads.items[header_index] = items_len;
7824	payloads.items[header_index + 1] = ranges_len;
7825	break :blk header_index + 2;
7826	} else if (case_node == special_node) blk: {
7827	payloads.items[case_table_start] = header_index;
7828	try payloads.resize(gpa, header_index + 1); // body_len
7829	break :blk header_index;
7830	} else blk: {
7831	payloads.items[scalar_case_table + scalar_case_index] = header_index;
7832	scalar_case_index += 1;
7833	try payloads.resize(gpa, header_index + 2); // item, body_len
7834	const item_node = case.ast.values[0];
7835	const item_inst = try comptimeExpr(parent_gz, scope, item_ri, item_node);
7836	payloads.items[header_index] = @intFromEnum(item_inst);
7837	break :blk header_index + 1;
7838	};
7839
7840	{
7841	// temporarily stack case_scope on parent_gz
7842	case_scope.instructions_top = parent_gz.instructions.items.len;
7843	defer case_scope.unstack();
7844
7845	if (dbg_var_name != .empty) {
7846	try case_scope.addDbgVar(.dbg_var_val, dbg_var_name, dbg_var_inst);
7847	}
7848	if (dbg_var_tag_name != .empty) {
7849	try case_scope.addDbgVar(.dbg_var_val, dbg_var_tag_name, dbg_var_tag_inst);
7850	}
7851	const target_expr_node = case.ast.target_expr;
7852	const case_result = try expr(&case_scope, sub_scope, block_scope.break_result_info, target_expr_node);
7853	try checkUsed(parent_gz, &case_scope.base, sub_scope);
7854	if (!parent_gz.refIsNoReturn(case_result)) {
7855	_ = try case_scope.addBreakWithSrcNode(.@"break", switch_block, case_result, target_expr_node);
7856	}
7857
7858	const case_slice = case_scope.instructionsSlice();
7859	// Since we use the switch_block instruction itself to refer to the
7860	// capture, which will not be added to the child block, we need to
7861	// handle ref_table manually, and the same for the inline tag
7862	// capture instruction.
7863	const refs_len = refs: {
7864	var n: usize = 0;
7865	var check_inst = switch_block;
7866	while (astgen.ref_table.get(check_inst)) \|ref_inst\| {
7867	n += 1;
7868	check_inst = ref_inst;
7869	}
7870	if (has_tag_capture) {
7871	check_inst = tag_inst;
7872	while (astgen.ref_table.get(check_inst)) \|ref_inst\| {
7873	n += 1;
7874	check_inst = ref_inst;
7875	}
7876	}
7877	break :refs n;
7878	};
7879	const body_len = refs_len + astgen.countBodyLenAfterFixups(case_slice);
7880	try payloads.ensureUnusedCapacity(gpa, body_len);
7881	payloads.items[body_len_index] = @bitCast(Zir.Inst.SwitchBlock.ProngInfo{
7882	.body_len = @intCast(body_len),
7883	.capture = capture,
7884	.is_inline = case.inline_token != null,
7885	.has_tag_capture = has_tag_capture,
7886	});
7887	if (astgen.ref_table.fetchRemove(switch_block)) \|kv\| {
7888	appendPossiblyRefdBodyInst(astgen, payloads, kv.value);
7889	}
7890	if (has_tag_capture) {
7891	if (astgen.ref_table.fetchRemove(tag_inst)) \|kv\| {
7892	appendPossiblyRefdBodyInst(astgen, payloads, kv.value);
7893	}
7894	}
7895	appendBodyWithFixupsArrayList(astgen, payloads, case_slice);
7896	}
7897	}
7898	// Now that the item expressions are generated we can add this.
7899	try parent_gz.instructions.append(gpa, switch_block);
7900
7901	try astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.SwitchBlock).Struct.fields.len +
7902	@intFromBool(multi_cases_len != 0) +
7903	@intFromBool(any_has_tag_capture) +
7904	payloads.items.len - case_table_end +
7905	(case_table_end - case_table_start) * @typeInfo(Zir.Inst.As).Struct.fields.len);
7906
7907	const payload_index = astgen.addExtraAssumeCapacity(Zir.Inst.SwitchBlock{
7908	.operand = raw_operand,
7909	.bits = Zir.Inst.SwitchBlock.Bits{
7910	.has_multi_cases = multi_cases_len != 0,
7911	.has_else = special_prong == .@"else",
7912	.has_under = special_prong == .under,
7913	.any_has_tag_capture = any_has_tag_capture,
7914	.scalar_cases_len = @intCast(scalar_cases_len),
7915	},
7916	});
7917
7918	if (multi_cases_len != 0) {
7919	astgen.extra.appendAssumeCapacity(multi_cases_len);
7920	}
7921
7922	if (any_has_tag_capture) {
7923	astgen.extra.appendAssumeCapacity(@intFromEnum(tag_inst));
7924	}
7925
7926	const zir_datas = astgen.instructions.items(.data);
7927	zir_datas[@intFromEnum(switch_block)].pl_node.payload_index = payload_index;
7928
7929	for (payloads.items[case_table_start..case_table_end], 0..) \|start_index, i\| {
7930	var body_len_index = start_index;
7931	var end_index = start_index;
7932	const table_index = case_table_start + i;
7933	if (table_index < scalar_case_table) {
7934	end_index += 1;
7935	} else if (table_index < multi_case_table) {
7936	body_len_index += 1;
7937	end_index += 2;
7938	} else {
7939	body_len_index += 2;
7940	const items_len = payloads.items[start_index];
7941	const ranges_len = payloads.items[start_index + 1];
7942	end_index += 3 + items_len + 2 * ranges_len;
7943	}
7944	const prong_info: Zir.Inst.SwitchBlock.ProngInfo = @bitCast(payloads.items[body_len_index]);
7945	end_index += prong_info.body_len;
7946	astgen.extra.appendSliceAssumeCapacity(payloads.items[start_index..end_index]);
7947	}
7948
7949	if (need_result_rvalue) {
7950	return rvalue(parent_gz, ri, switch_block.toRef(), switch_node);
7951	} else {
7952	return switch_block.toRef();
7953	}
7954	}
7955
7956	fn ret(gz: GenZir, scope: Scope, node: Ast.Node.Index) InnerError!Zir.Inst.Ref {
7957	const astgen = gz.astgen;
7958	const tree = astgen.tree;
7959	const node_datas = tree.nodes.items(.data);
7960	const node_tags = tree.nodes.items(.tag);
7961
7962	if (astgen.fn_block == null) {
7963	return astgen.failNode(node, "'return' outside function scope", .{});
7964	}
7965
7966	if (gz.any_defer_node != 0) {
7967	return astgen.failNodeNotes(node, "cannot return from defer expression", .{}, &.{
7968	try astgen.errNoteNode(
7969	gz.any_defer_node,
7970	"defer expression here",
7971	.{},
7972	),
7973	});
7974	}
7975
7976	// Ensure debug line/column information is emitted for this return expression.
7977	// Then we will save the line/column so that we can emit another one that goes
7978	// "backwards" because we want to evaluate the operand, but then put the debug
7979	// info back at the return keyword for error return tracing.
7980	if (!gz.is_comptime) {
7981	try emitDbgNode(gz, node);
7982	}
7983	const ret_lc = LineColumn{ astgen.source_line - gz.decl_line, astgen.source_column };
7984
7985	const defer_outer = &astgen.fn_block.?.base;
7986
7987	const operand_node = node_datas[node].lhs;
7988	if (operand_node == 0) {
7989	// Returning a void value; skip error defers.
7990	try genDefers(gz, defer_outer, scope, .normal_only);
7991
7992	// As our last action before the return, "pop" the error trace if needed
7993	_ = try gz.addRestoreErrRetIndex(.ret, .always, node);
7994
7995	_ = try gz.addUnNode(.ret_node, .void_value, node);
7996	return Zir.Inst.Ref.unreachable_value;
7997	}
7998
7999	if (node_tags[operand_node] == .error_value) {
8000	// Hot path for `return error.Foo`. This bypasses result location logic as well as logic
8001	// for detecting whether to add something to the function's inferred error set.
8002	const ident_token = node_datas[operand_node].rhs;
8003	const err_name_str_index = try astgen.identAsString(ident_token);
8004	const defer_counts = countDefers(defer_outer, scope);
8005	if (!defer_counts.need_err_code) {
8006	try genDefers(gz, defer_outer, scope, .both_sans_err);
8007	try emitDbgStmt(gz, ret_lc);
8008	_ = try gz.addStrTok(.ret_err_value, err_name_str_index, ident_token);
8009	return Zir.Inst.Ref.unreachable_value;
8010	}
8011	const err_code = try gz.addStrTok(.ret_err_value_code, err_name_str_index, ident_token);
8012	try genDefers(gz, defer_outer, scope, .{ .both = err_code });
8013	try emitDbgStmt(gz, ret_lc);
8014	_ = try gz.addUnNode(.ret_node, err_code, node);
8015	return Zir.Inst.Ref.unreachable_value;
8016	}
8017
8018	const ri: ResultInfo = if (astgen.nodes_need_rl.contains(node)) .{
8019	.rl = .{ .ptr = .{ .inst = try gz.addNode(.ret_ptr, node) } },
8020	.ctx = .@"return",
8021	} else .{
8022	.rl = .{ .coerced_ty = astgen.fn_ret_ty },
8023	.ctx = .@"return",
8024	};
8025	const prev_anon_name_strategy = gz.anon_name_strategy;
8026	gz.anon_name_strategy = .func;
8027	const operand = try reachableExpr(gz, scope, ri, operand_node, node);
8028	gz.anon_name_strategy = prev_anon_name_strategy;
8029
8030	switch (nodeMayEvalToError(tree, operand_node)) {
8031	.never => {
8032	// Returning a value that cannot be an error; skip error defers.
8033	try genDefers(gz, defer_outer, scope, .normal_only);
8034
8035	// As our last action before the return, "pop" the error trace if needed
8036	_ = try gz.addRestoreErrRetIndex(.ret, .always, node);
8037
8038	try emitDbgStmt(gz, ret_lc);
8039	try gz.addRet(ri, operand, node);
8040	return Zir.Inst.Ref.unreachable_value;
8041	},
8042	.always => {
8043	// Value is always an error. Emit both error defers and regular defers.
8044	const err_code = if (ri.rl == .ptr) try gz.addUnNode(.load, ri.rl.ptr.inst, node) else operand;
8045	try genDefers(gz, defer_outer, scope, .{ .both = err_code });
8046	try emitDbgStmt(gz, ret_lc);
8047	try gz.addRet(ri, operand, node);
8048	return Zir.Inst.Ref.unreachable_value;
8049	},
8050	.maybe => {
8051	const defer_counts = countDefers(defer_outer, scope);
8052	if (!defer_counts.have_err) {
8053	// Only regular defers; no branch needed.
8054	try genDefers(gz, defer_outer, scope, .normal_only);
8055	try emitDbgStmt(gz, ret_lc);
8056
8057	// As our last action before the return, "pop" the error trace if needed
8058	const result = if (ri.rl == .ptr) try gz.addUnNode(.load, ri.rl.ptr.inst, node) else operand;
8059	_ = try gz.addRestoreErrRetIndex(.ret, .{ .if_non_error = result }, node);
8060
8061	try gz.addRet(ri, operand, node);
8062	return Zir.Inst.Ref.unreachable_value;
8063	}
8064
8065	// Emit conditional branch for generating errdefers.
8066	const result = if (ri.rl == .ptr) try gz.addUnNode(.load, ri.rl.ptr.inst, node) else operand;
8067	const is_non_err = try gz.addUnNode(.ret_is_non_err, result, node);
8068	const condbr = try gz.addCondBr(.condbr, node);
8069
8070	var then_scope = gz.makeSubBlock(scope);
8071	defer then_scope.unstack();
8072
8073	try genDefers(&then_scope, defer_outer, scope, .normal_only);
8074
8075	// As our last action before the return, "pop" the error trace if needed
8076	_ = try then_scope.addRestoreErrRetIndex(.ret, .always, node);
8077
8078	try emitDbgStmt(&then_scope, ret_lc);
8079	try then_scope.addRet(ri, operand, node);
8080
8081	var else_scope = gz.makeSubBlock(scope);
8082	defer else_scope.unstack();
8083
8084	const which_ones: DefersToEmit = if (!defer_counts.need_err_code) .both_sans_err else .{
8085	.both = try else_scope.addUnNode(.err_union_code, result, node),
8086	};
8087	try genDefers(&else_scope, defer_outer, scope, which_ones);
8088	try emitDbgStmt(&else_scope, ret_lc);
8089	try else_scope.addRet(ri, operand, node);
8090
8091	try setCondBrPayload(condbr, is_non_err, &then_scope, &else_scope);
8092
8093	return Zir.Inst.Ref.unreachable_value;
8094	},
8095	}
8096	}
8097
8098	/// Parses the string `buf` as a base 10 integer of type `u16`.
8099	///
8100	/// Unlike std.fmt.parseInt, does not allow the '_' character in `buf`.
8101	fn parseBitCount(buf: []const u8) std.fmt.ParseIntError!u16 {
8102	if (buf.len == 0) return error.InvalidCharacter;
8103
8104	var x: u16 = 0;
8105
8106	for (buf) \|c\| {
8107	const digit = switch (c) {
8108	'0'...'9' => c - '0',
8109	else => return error.InvalidCharacter,
8110	};
8111
8112	if (x != 0) x = try std.math.mul(u16, x, 10);
8113	x = try std.math.add(u16, x, digit);
8114	}
8115
8116	return x;
8117	}
8118
8119	fn identifier(
8120	gz: *GenZir,
8121	scope: *Scope,
8122	ri: ResultInfo,
8123	ident: Ast.Node.Index,
8124	) InnerError!Zir.Inst.Ref {
8125	const astgen = gz.astgen;
8126	const tree = astgen.tree;
8127	const main_tokens = tree.nodes.items(.main_token);
8128
8129	const ident_token = main_tokens[ident];
8130	const ident_name_raw = tree.tokenSlice(ident_token);
8131	if (mem.eql(u8, ident_name_raw, "_")) {
8132	return astgen.failNode(ident, "'_' used as an identifier without @\"_\" syntax", .{});
8133	}
8134
8135	// if not @"" syntax, just use raw token slice
8136	if (ident_name_raw[0] != '@') {
8137	if (primitive_instrs.get(ident_name_raw)) \|zir_const_ref\| {
8138	return rvalue(gz, ri, zir_const_ref, ident);
8139	}
8140
8141	if (ident_name_raw.len >= 2) integer: {
8142	const first_c = ident_name_raw[0];
8143	if (first_c == 'i' or first_c == 'u') {
8144	const signedness: std.builtin.Signedness = switch (first_c == 'i') {
8145	true => .signed,
8146	false => .unsigned,
8147	};
8148	if (ident_name_raw.len >= 3 and ident_name_raw[1] == '0') {
8149	return astgen.failNode(
8150	ident,
8151	"primitive integer type '{s}' has leading zero",
8152	.{ident_name_raw},
8153	);
8154	}
8155	const bit_count = parseBitCount(ident_name_raw[1..]) catch \|err\| switch (err) {
8156	error.Overflow => return astgen.failNode(
8157	ident,
8158	"primitive integer type '{s}' exceeds maximum bit width of 65535",
8159	.{ident_name_raw},
8160	),
8161	error.InvalidCharacter => break :integer,
8162	};
8163	const result = try gz.add(.{
8164	.tag = .int_type,
8165	.data = .{ .int_type = .{
8166	.src_node = gz.nodeIndexToRelative(ident),
8167	.signedness = signedness,
8168	.bit_count = bit_count,
8169	} },
8170	});
8171	return rvalue(gz, ri, result, ident);
8172	}
8173	}
8174	}
8175
8176	// Local variables, including function parameters.
8177	return localVarRef(gz, scope, ri, ident, ident_token);
8178	}
8179
8180	fn localVarRef(
8181	gz: *GenZir,
8182	scope: *Scope,
8183	ri: ResultInfo,
8184	ident: Ast.Node.Index,
8185	ident_token: Ast.TokenIndex,
8186	) InnerError!Zir.Inst.Ref {
8187	const astgen = gz.astgen;
8188	const gpa = astgen.gpa;
8189	const name_str_index = try astgen.identAsString(ident_token);
8190	var s = scope;
8191	var found_already: ?Ast.Node.Index = null; // we have found a decl with the same name already
8192	var num_namespaces_out: u32 = 0;
8193	var capturing_namespace: ?*Scope.Namespace = null;
8194	while (true) switch (s.tag) {
8195	.local_val => {
8196	const local_val = s.cast(Scope.LocalVal).?;
8197
8198	if (local_val.name == name_str_index) {
8199	// Locals cannot shadow anything, so we do not need to look for ambiguous
8200	// references in this case.
8201	if (ri.rl == .discard and ri.ctx == .assignment) {
8202	local_val.discarded = ident_token;
8203	} else {
8204	local_val.used = ident_token;
8205	}
8206
8207	const value_inst = try tunnelThroughClosure(
8208	gz,
8209	ident,
8210	num_namespaces_out,
8211	capturing_namespace,
8212	local_val.inst,
8213	local_val.token_src,
8214	gpa,
8215	);
8216
8217	return rvalueNoCoercePreRef(gz, ri, value_inst, ident);
8218	}
8219	s = local_val.parent;
8220	},
8221	.local_ptr => {
8222	const local_ptr = s.cast(Scope.LocalPtr).?;
8223	if (local_ptr.name == name_str_index) {
8224	if (ri.rl == .discard and ri.ctx == .assignment) {
8225	local_ptr.discarded = ident_token;
8226	} else {
8227	local_ptr.used = ident_token;
8228	}
8229
8230	// Can't close over a runtime variable
8231	if (num_namespaces_out != 0 and !local_ptr.maybe_comptime and !gz.is_typeof) {
8232	const ident_name = try astgen.identifierTokenString(ident_token);
8233	return astgen.failNodeNotes(ident, "mutable '{s}' not accessible from here", .{ident_name}, &.{
8234	try astgen.errNoteTok(local_ptr.token_src, "declared mutable here", .{}),
8235	try astgen.errNoteNode(capturing_namespace.?.node, "crosses namespace boundary here", .{}),
8236	});
8237	}
8238
8239	const ptr_inst = try tunnelThroughClosure(
8240	gz,
8241	ident,
8242	num_namespaces_out,
8243	capturing_namespace,
8244	local_ptr.ptr,
8245	local_ptr.token_src,
8246	gpa,
8247	);
8248
8249	switch (ri.rl) {
8250	.ref, .ref_coerced_ty => {
8251	local_ptr.used_as_lvalue = true;
8252	return ptr_inst;
8253	},
8254	else => {
8255	const loaded = try gz.addUnNode(.load, ptr_inst, ident);
8256	return rvalueNoCoercePreRef(gz, ri, loaded, ident);
8257	},
8258	}
8259	}
8260	s = local_ptr.parent;
8261	},
8262	.gen_zir => s = s.cast(GenZir).?.parent,
8263	.defer_normal, .defer_error => s = s.cast(Scope.Defer).?.parent,
8264	.namespace, .enum_namespace => {
8265	const ns = s.cast(Scope.Namespace).?;
8266	if (ns.decls.get(name_str_index)) \|i\| {
8267	if (found_already) \|f\| {
8268	return astgen.failNodeNotes(ident, "ambiguous reference", .{}, &.{
8269	try astgen.errNoteNode(f, "declared here", .{}),
8270	try astgen.errNoteNode(i, "also declared here", .{}),
8271	});
8272	}
8273	// We found a match but must continue looking for ambiguous references to decls.
8274	found_already = i;
8275	}
8276	if (s.tag == .namespace) num_namespaces_out += 1;
8277	capturing_namespace = ns;
8278	s = ns.parent;
8279	},
8280	.top => break,
8281	};
8282	if (found_already == null) {
8283	const ident_name = try astgen.identifierTokenString(ident_token);
8284	return astgen.failNode(ident, "use of undeclared identifier '{s}'", .{ident_name});
8285	}
8286
8287	// Decl references happen by name rather than ZIR index so that when unrelated
8288	// decls are modified, ZIR code containing references to them can be unmodified.
8289	switch (ri.rl) {
8290	.ref, .ref_coerced_ty => return gz.addStrTok(.decl_ref, name_str_index, ident_token),
8291	else => {
8292	const result = try gz.addStrTok(.decl_val, name_str_index, ident_token);
8293	return rvalueNoCoercePreRef(gz, ri, result, ident);
8294	},
8295	}
8296	}
8297
8298	/// Adds a capture to a namespace, if needed.
8299	/// Returns the index of the closure_capture instruction.
8300	fn tunnelThroughClosure(
8301	gz: *GenZir,
8302	inner_ref_node: Ast.Node.Index,
8303	num_tunnels: u32,
8304	ns: ?*Scope.Namespace,
8305	value: Zir.Inst.Ref,
8306	token: Ast.TokenIndex,
8307	gpa: Allocator,
8308	) !Zir.Inst.Ref {
8309	// For trivial values, we don't need a tunnel.
8310	// Just return the ref.
8311	if (num_tunnels == 0 or value.toIndex() == null) {
8312	return value;
8313	}
8314
8315	// Otherwise we need a tunnel. Check if this namespace
8316	// already has one for this value.
8317	const gop = try ns.?.captures.getOrPut(gpa, value.toIndex().?);
8318	if (!gop.found_existing) {
8319	// Make a new capture for this value but don't add it to the declaring_gz yet
8320	try gz.astgen.instructions.append(gz.astgen.gpa, .{
8321	.tag = .closure_capture,
8322	.data = .{ .un_tok = .{
8323	.operand = value,
8324	.src_tok = ns.?.declaring_gz.?.tokenIndexToRelative(token),
8325	} },
8326	});
8327	gop.value_ptr.* = @enumFromInt(gz.astgen.instructions.len - 1);
8328	}
8329
8330	// Add an instruction to get the value from the closure into
8331	// our current context
8332	return try gz.addInstNode(.closure_get, gop.value_ptr.*, inner_ref_node);
8333	}
8334
8335	fn stringLiteral(
8336	gz: *GenZir,
8337	ri: ResultInfo,
8338	node: Ast.Node.Index,
8339	) InnerError!Zir.Inst.Ref {
8340	const astgen = gz.astgen;
8341	const tree = astgen.tree;
8342	const main_tokens = tree.nodes.items(.main_token);
8343	const str_lit_token = main_tokens[node];
8344	const str = try astgen.strLitAsString(str_lit_token);
8345	const result = try gz.add(.{
8346	.tag = .str,
8347	.data = .{ .str = .{
8348	.start = str.index,
8349	.len = str.len,
8350	} },
8351	});
8352	return rvalue(gz, ri, result, node);
8353	}
8354
8355	fn multilineStringLiteral(
8356	gz: *GenZir,
8357	ri: ResultInfo,
8358	node: Ast.Node.Index,
8359	) InnerError!Zir.Inst.Ref {
8360	const astgen = gz.astgen;
8361	const str = try astgen.strLitNodeAsString(node);
8362	const result = try gz.add(.{
8363	.tag = .str,
8364	.data = .{ .str = .{
8365	.start = str.index,
8366	.len = str.len,
8367	} },
8368	});
8369	return rvalue(gz, ri, result, node);
8370	}
8371
8372	fn charLiteral(gz: *GenZir, ri: ResultInfo, node: Ast.Node.Index) InnerError!Zir.Inst.Ref {
8373	const astgen = gz.astgen;
8374	const tree = astgen.tree;
8375	const main_tokens = tree.nodes.items(.main_token);
8376	const main_token = main_tokens[node];
8377	const slice = tree.tokenSlice(main_token);
8378
8379	switch (std.zig.parseCharLiteral(slice)) {
8380	.success => \|codepoint\| {
8381	const result = try gz.addInt(codepoint);
8382	return rvalue(gz, ri, result, node);
8383	},
8384	.failure => \|err\| return astgen.failWithStrLitError(err, main_token, slice, 0),
8385	}
8386	}
8387
8388	const Sign = enum { negative, positive };
8389
8390	fn numberLiteral(gz: *GenZir, ri: ResultInfo, node: Ast.Node.Index, source_node: Ast.Node.Index, sign: Sign) InnerError!Zir.Inst.Ref {
8391	const astgen = gz.astgen;
8392	const tree = astgen.tree;
8393	const main_tokens = tree.nodes.items(.main_token);
8394	const num_token = main_tokens[node];
8395	const bytes = tree.tokenSlice(num_token);
8396
8397	const result: Zir.Inst.Ref = switch (std.zig.parseNumberLiteral(bytes)) {
8398	.int => \|num\| switch (num) {
8399	0 => if (sign == .positive) .zero else return astgen.failTokNotes(
8400	num_token,
8401	"integer literal '-0' is ambiguous",
8402	.{},
8403	&.{
8404	try astgen.errNoteTok(num_token, "use '0' for an integer zero", .{}),
8405	try astgen.errNoteTok(num_token, "use '-0.0' for a floating-point signed zero", .{}),
8406	},
8407	),
8408	1 => .one,
8409	else => try gz.addInt(num),
8410	},
8411	.big_int => \|base\| big: {
8412	const gpa = astgen.gpa;
8413	var big_int = try std.math.big.int.Managed.init(gpa);
8414	defer big_int.deinit();
8415	const prefix_offset: usize = if (base == .decimal) 0 else 2;
8416	big_int.setString(@intFromEnum(base), bytes[prefix_offset..]) catch \|err\| switch (err) {
8417	error.InvalidCharacter => unreachable, // caught in `parseNumberLiteral`
8418	error.InvalidBase => unreachable, // we only pass 16, 8, 2, see above
8419	error.OutOfMemory => return error.OutOfMemory,
8420	};
8421
8422	const limbs = big_int.limbs[0..big_int.len()];
8423	assert(big_int.isPositive());
8424	break :big try gz.addIntBig(limbs);
8425	},
8426	.float => {
8427	const unsigned_float_number = std.fmt.parseFloat(f128, bytes) catch \|err\| switch (err) {
8428	error.InvalidCharacter => unreachable, // validated by tokenizer
8429	};
8430	const float_number = switch (sign) {
8431	.negative => -unsigned_float_number,
8432	.positive => unsigned_float_number,
8433	};
8434	// If the value fits into a f64 without losing any precision, store it that way.
8435	@setFloatMode(.Strict);
8436	const smaller_float: f64 = @floatCast(float_number);
8437	const bigger_again: f128 = smaller_float;
8438	if (bigger_again == float_number) {
8439	const result = try gz.addFloat(smaller_float);
8440	return rvalue(gz, ri, result, source_node);
8441	}
8442	// We need to use 128 bits. Break the float into 4 u32 values so we can
8443	// put it into the `extra` array.
8444	const int_bits: u128 = @bitCast(float_number);
8445	const result = try gz.addPlNode(.float128, node, Zir.Inst.Float128{
8446	.piece0 = @truncate(int_bits),
8447	.piece1 = @truncate(int_bits >> 32),
8448	.piece2 = @truncate(int_bits >> 64),
8449	.piece3 = @truncate(int_bits >> 96),
8450	});
8451	return rvalue(gz, ri, result, source_node);
8452	},
8453	.failure => \|err\| return astgen.failWithNumberError(err, num_token, bytes),
8454	};
8455
8456	if (sign == .positive) {
8457	return rvalue(gz, ri, result, source_node);
8458	} else {
8459	const negated = try gz.addUnNode(.negate, result, source_node);
8460	return rvalue(gz, ri, negated, source_node);
8461	}
8462	}
8463
8464	fn failWithNumberError(astgen: *AstGen, err: std.zig.number_literal.Error, token: Ast.TokenIndex, bytes: []const u8) InnerError {
8465	const is_float = std.mem.indexOfScalar(u8, bytes, '.') != null;
8466	switch (err) {
8467	.leading_zero => if (is_float) {
8468	return astgen.failTok(token, "number '{s}' has leading zero", .{bytes});
8469	} else {
8470	return astgen.failTokNotes(token, "number '{s}' has leading zero", .{bytes}, &.{
8471	try astgen.errNoteTok(token, "use '0o' prefix for octal literals", .{}),
8472	});
8473	},
8474	.digit_after_base => return astgen.failTok(token, "expected a digit after base prefix", .{}),
8475	.upper_case_base => \|i\| return astgen.failOff(token, @intCast(i), "base prefix must be lowercase", .{}),
8476	.invalid_float_base => \|i\| return astgen.failOff(token, @intCast(i), "invalid base for float literal", .{}),
8477	.repeated_underscore => \|i\| return astgen.failOff(token, @intCast(i), "repeated digit separator", .{}),
8478	.invalid_underscore_after_special => \|i\| return astgen.failOff(token, @intCast(i), "expected digit before digit separator", .{}),
8479	.invalid_digit => \|info\| return astgen.failOff(token, @intCast(info.i), "invalid digit '{c}' for {s} base", .{ bytes[info.i], @tagName(info.base) }),
8480	.invalid_digit_exponent => \|i\| return astgen.failOff(token, @intCast(i), "invalid digit '{c}' in exponent", .{bytes[i]}),
8481	.duplicate_exponent => \|i\| return astgen.failOff(token, @intCast(i), "duplicate exponent", .{}),
8482	.exponent_after_underscore => \|i\| return astgen.failOff(token, @intCast(i), "expected digit before exponent", .{}),
8483	.special_after_underscore => \|i\| return astgen.failOff(token, @intCast(i), "expected digit before '{c}'", .{bytes[i]}),
8484	.trailing_special => \|i\| return astgen.failOff(token, @intCast(i), "expected digit after '{c}'", .{bytes[i - 1]}),
8485	.trailing_underscore => \|i\| return astgen.failOff(token, @intCast(i), "trailing digit separator", .{}),
8486	.duplicate_period => unreachable, // Validated by tokenizer
8487	.invalid_character => unreachable, // Validated by tokenizer
8488	.invalid_exponent_sign => \|i\| {
8489	assert(bytes.len >= 2 and bytes[0] == '0' and bytes[1] == 'x'); // Validated by tokenizer
8490	return astgen.failOff(token, @intCast(i), "sign '{c}' cannot follow digit '{c}' in hex base", .{ bytes[i], bytes[i - 1] });
8491	},
8492	}
8493	}
8494
8495	fn asmExpr(
8496	gz: *GenZir,
8497	scope: *Scope,
8498	ri: ResultInfo,
8499	node: Ast.Node.Index,
8500	full: Ast.full.Asm,
8501	) InnerError!Zir.Inst.Ref {
8502	const astgen = gz.astgen;
8503	const tree = astgen.tree;
8504	const main_tokens = tree.nodes.items(.main_token);
8505	const node_datas = tree.nodes.items(.data);
8506	const node_tags = tree.nodes.items(.tag);
8507	const token_tags = tree.tokens.items(.tag);
8508
8509	const TagAndTmpl = struct { tag: Zir.Inst.Extended, tmpl: Zir.NullTerminatedString };
8510	const tag_and_tmpl: TagAndTmpl = switch (node_tags[full.ast.template]) {
8511	.string_literal => .{
8512	.tag = .@"asm",
8513	.tmpl = (try astgen.strLitAsString(main_tokens[full.ast.template])).index,
8514	},
8515	.multiline_string_literal => .{
8516	.tag = .@"asm",
8517	.tmpl = (try astgen.strLitNodeAsString(full.ast.template)).index,
8518	},
8519	else => .{
8520	.tag = .asm_expr,
8521	.tmpl = @enumFromInt(@intFromEnum(try comptimeExpr(gz, scope, .{ .rl = .none }, full.ast.template))),
8522	},
8523	};
8524
8525	// See https://github.com/ziglang/zig/issues/215 and related issues discussing
8526	// possible inline assembly improvements. Until then here is status quo AstGen
8527	// for assembly syntax. It's used by std lib crypto aesni.zig.
8528	const is_container_asm = astgen.fn_block == null;
8529	if (is_container_asm) {
8530	if (full.volatile_token) \|t\|
8531	return astgen.failTok(t, "volatile is meaningless on global assembly", .{});
8532	if (full.outputs.len != 0 or full.inputs.len != 0 or full.first_clobber != null)
8533	return astgen.failNode(node, "global assembly cannot have inputs, outputs, or clobbers", .{});
8534	} else {
8535	if (full.outputs.len == 0 and full.volatile_token == null) {
8536	return astgen.failNode(node, "assembly expression with no output must be marked volatile", .{});
8537	}
8538	}
8539	if (full.outputs.len > 32) {
8540	return astgen.failNode(full.outputs[32], "too many asm outputs", .{});
8541	}
8542	var outputs_buffer: [32]Zir.Inst.Asm.Output = undefined;
8543	const outputs = outputs_buffer[0..full.outputs.len];
8544
8545	var output_type_bits: u32 = 0;
8546
8547	for (full.outputs, 0..) \|output_node, i\| {
8548	const symbolic_name = main_tokens[output_node];
8549	const name = try astgen.identAsString(symbolic_name);
8550	const constraint_token = symbolic_name + 2;
8551	const constraint = (try astgen.strLitAsString(constraint_token)).index;
8552	const has_arrow = token_tags[symbolic_name + 4] == .arrow;
8553	if (has_arrow) {
8554	if (output_type_bits != 0) {
8555	return astgen.failNode(output_node, "inline assembly allows up to one output value", .{});
8556	}
8557	output_type_bits \|= @as(u32, 1) << @intCast(i);
8558	const out_type_node = node_datas[output_node].lhs;
8559	const out_type_inst = try typeExpr(gz, scope, out_type_node);
8560	outputs[i] = .{
8561	.name = name,
8562	.constraint = constraint,
8563	.operand = out_type_inst,
8564	};
8565	} else {
8566	const ident_token = symbolic_name + 4;
8567	// TODO have a look at #215 and related issues and decide how to
8568	// handle outputs. Do we want this to be identifiers?
8569	// Or maybe we want to force this to be expressions with a pointer type.
8570	outputs[i] = .{
8571	.name = name,
8572	.constraint = constraint,
8573	.operand = try localVarRef(gz, scope, .{ .rl = .ref }, node, ident_token),
8574	};
8575	}
8576	}
8577
8578	if (full.inputs.len > 32) {
8579	return astgen.failNode(full.inputs[32], "too many asm inputs", .{});
8580	}
8581	var inputs_buffer: [32]Zir.Inst.Asm.Input = undefined;
8582	const inputs = inputs_buffer[0..full.inputs.len];
8583
8584	for (full.inputs, 0..) \|input_node, i\| {
8585	const symbolic_name = main_tokens[input_node];
8586	const name = try astgen.identAsString(symbolic_name);
8587	const constraint_token = symbolic_name + 2;
8588	const constraint = (try astgen.strLitAsString(constraint_token)).index;
8589	const operand = try expr(gz, scope, .{ .rl = .none }, node_datas[input_node].lhs);
8590	inputs[i] = .{
8591	.name = name,
8592	.constraint = constraint,
8593	.operand = operand,
8594	};
8595	}
8596
8597	var clobbers_buffer: [32]u32 = undefined;
8598	var clobber_i: usize = 0;
8599	if (full.first_clobber) \|first_clobber\| clobbers: {
8600	// asm ("foo" ::: "a", "b")
8601	// asm ("foo" ::: "a", "b",)
8602	var tok_i = first_clobber;
8603	while (true) : (tok_i += 1) {
8604	if (clobber_i >= clobbers_buffer.len) {
8605	return astgen.failTok(tok_i, "too many asm clobbers", .{});
8606	}
8607	clobbers_buffer[clobber_i] = @intFromEnum((try astgen.strLitAsString(tok_i)).index);
8608	clobber_i += 1;
8609	tok_i += 1;
8610	switch (token_tags[tok_i]) {
8611	.r_paren => break :clobbers,
8612	.comma => {
8613	if (token_tags[tok_i + 1] == .r_paren) {
8614	break :clobbers;
8615	} else {
8616	continue;
8617	}
8618	},
8619	else => unreachable,
8620	}
8621	}
8622	}
8623
8624	const result = try gz.addAsm(.{
8625	.tag = tag_and_tmpl.tag,
8626	.node = node,
8627	.asm_source = tag_and_tmpl.tmpl,
8628	.is_volatile = full.volatile_token != null,
8629	.output_type_bits = output_type_bits,
8630	.outputs = outputs,
8631	.inputs = inputs,
8632	.clobbers = clobbers_buffer[0..clobber_i],
8633	});
8634	return rvalue(gz, ri, result, node);
8635	}
8636
8637	fn as(
8638	gz: *GenZir,
8639	scope: *Scope,
8640	ri: ResultInfo,
8641	node: Ast.Node.Index,
8642	lhs: Ast.Node.Index,
8643	rhs: Ast.Node.Index,
8644	) InnerError!Zir.Inst.Ref {
8645	const dest_type = try typeExpr(gz, scope, lhs);
8646	const result = try reachableExpr(gz, scope, .{ .rl = .{ .ty = dest_type } }, rhs, node);
8647	return rvalue(gz, ri, result, node);
8648	}
8649
8650	fn unionInit(
8651	gz: *GenZir,
8652	scope: *Scope,
8653	ri: ResultInfo,
8654	node: Ast.Node.Index,
8655	params: []const Ast.Node.Index,
8656	) InnerError!Zir.Inst.Ref {
8657	const union_type = try typeExpr(gz, scope, params[0]);
8658	const field_name = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .slice_const_u8_type } }, params[1]);
8659	const field_type = try gz.addPlNode(.field_type_ref, node, Zir.Inst.FieldTypeRef{
8660	.container_type = union_type,
8661	.field_name = field_name,
8662	});
8663	const init = try reachableExpr(gz, scope, .{ .rl = .{ .ty = field_type } }, params[2], node);
8664	const result = try gz.addPlNode(.union_init, node, Zir.Inst.UnionInit{
8665	.union_type = union_type,
8666	.init = init,
8667	.field_name = field_name,
8668	});
8669	return rvalue(gz, ri, result, node);
8670	}
8671
8672	fn bitCast(
8673	gz: *GenZir,
8674	scope: *Scope,
8675	ri: ResultInfo,
8676	node: Ast.Node.Index,
8677	operand_node: Ast.Node.Index,
8678	) InnerError!Zir.Inst.Ref {
8679	const dest_type = try ri.rl.resultTypeForCast(gz, node, "@bitCast");
8680	const operand = try reachableExpr(gz, scope, .{ .rl = .none }, operand_node, node);
8681	const result = try gz.addPlNode(.bitcast, node, Zir.Inst.Bin{
8682	.lhs = dest_type,
8683	.rhs = operand,
8684	});
8685	return rvalue(gz, ri, result, node);
8686	}
8687
8688	/// Handle one or more nested pointer cast builtins:
8689	/// * @ptrCast
8690	/// * @alignCast
8691	/// * @addrSpaceCast
8692	/// * @constCast
8693	/// * @volatileCast
8694	/// Any sequence of such builtins is treated as a single operation. This allowed
8695	/// for sequences like `@ptrCast(@alignCast(ptr))` to work correctly despite the
8696	/// intermediate result type being unknown.
8697	fn ptrCast(
8698	gz: *GenZir,
8699	scope: *Scope,
8700	ri: ResultInfo,
8701	root_node: Ast.Node.Index,
8702	) InnerError!Zir.Inst.Ref {
8703	const astgen = gz.astgen;
8704	const tree = astgen.tree;
8705	const main_tokens = tree.nodes.items(.main_token);
8706	const node_datas = tree.nodes.items(.data);
8707	const node_tags = tree.nodes.items(.tag);
8708
8709	var flags: Zir.Inst.FullPtrCastFlags = .{};
8710
8711	// Note that all pointer cast builtins have one parameter, so we only need
8712	// to handle `builtin_call_two`.
8713	var node = root_node;
8714	while (true) {
8715	switch (node_tags[node]) {
8716	.builtin_call_two, .builtin_call_two_comma => {},
8717	.grouped_expression => {
8718	// Handle the chaining even with redundant parentheses
8719	node = node_datas[node].lhs;
8720	continue;
8721	},
8722	else => break,
8723	}
8724
8725	if (node_datas[node].lhs == 0) break; // 0 args
8726	if (node_datas[node].rhs != 0) break; // 2 args
8727
8728	const builtin_token = main_tokens[node];
8729	const builtin_name = tree.tokenSlice(builtin_token);
8730	const info = BuiltinFn.list.get(builtin_name) orelse break;
8731	if (info.param_count != 1) break;
8732
8733	switch (info.tag) {
8734	else => break,
8735	inline .ptr_cast,
8736	.align_cast,
8737	.addrspace_cast,
8738	.const_cast,
8739	.volatile_cast,
8740	=> \|tag\| {
8741	if (@field(flags, @tagName(tag))) {
8742	return astgen.failNode(node, "redundant {s}", .{builtin_name});
8743	}
8744	@field(flags, @tagName(tag)) = true;
8745	},
8746	}
8747
8748	node = node_datas[node].lhs;
8749	}
8750
8751	const flags_i: u5 = @bitCast(flags);
8752	assert(flags_i != 0);
8753
8754	const ptr_only: Zir.Inst.FullPtrCastFlags = .{ .ptr_cast = true };
8755	if (flags_i == @as(u5, @bitCast(ptr_only))) {
8756	// Special case: simpler representation
8757	return typeCast(gz, scope, ri, root_node, node, .ptr_cast, "@ptrCast");
8758	}
8759
8760	const no_result_ty_flags: Zir.Inst.FullPtrCastFlags = .{
8761	.const_cast = true,
8762	.volatile_cast = true,
8763	};
8764	if ((flags_i & ~@as(u5, @bitCast(no_result_ty_flags))) == 0) {
8765	// Result type not needed
8766	const cursor = maybeAdvanceSourceCursorToMainToken(gz, root_node);
8767	const operand = try expr(gz, scope, .{ .rl = .none }, node);
8768	try emitDbgStmt(gz, cursor);
8769	const result = try gz.addExtendedPayloadSmall(.ptr_cast_no_dest, flags_i, Zir.Inst.UnNode{
8770	.node = gz.nodeIndexToRelative(root_node),
8771	.operand = operand,
8772	});
8773	return rvalue(gz, ri, result, root_node);
8774	}
8775
8776	// Full cast including result type
8777
8778	const cursor = maybeAdvanceSourceCursorToMainToken(gz, root_node);
8779	const result_type = try ri.rl.resultTypeForCast(gz, root_node, flags.needResultTypeBuiltinName());
8780	const operand = try expr(gz, scope, .{ .rl = .none }, node);
8781	try emitDbgStmt(gz, cursor);
8782	const result = try gz.addExtendedPayloadSmall(.ptr_cast_full, flags_i, Zir.Inst.BinNode{
8783	.node = gz.nodeIndexToRelative(root_node),
8784	.lhs = result_type,
8785	.rhs = operand,
8786	});
8787	return rvalue(gz, ri, result, root_node);
8788	}
8789
8790	fn typeOf(
8791	gz: *GenZir,
8792	scope: *Scope,
8793	ri: ResultInfo,
8794	node: Ast.Node.Index,
8795	args: []const Ast.Node.Index,
8796	) InnerError!Zir.Inst.Ref {
8797	const astgen = gz.astgen;
8798	if (args.len < 1) {
8799	return astgen.failNode(node, "expected at least 1 argument, found 0", .{});
8800	}
8801	const gpa = astgen.gpa;
8802	if (args.len == 1) {
8803	const typeof_inst = try gz.makeBlockInst(.typeof_builtin, node);
8804
8805	var typeof_scope = gz.makeSubBlock(scope);
8806	typeof_scope.is_comptime = false;
8807	typeof_scope.is_typeof = true;
8808	typeof_scope.c_import = false;
8809	defer typeof_scope.unstack();
8810
8811	const ty_expr = try reachableExpr(&typeof_scope, &typeof_scope.base, .{ .rl = .none }, args[0], node);
8812	if (!gz.refIsNoReturn(ty_expr)) {
8813	_ = try typeof_scope.addBreak(.break_inline, typeof_inst, ty_expr);
8814	}
8815	try typeof_scope.setBlockBody(typeof_inst);
8816
8817	// typeof_scope unstacked now, can add new instructions to gz
8818	try gz.instructions.append(gpa, typeof_inst);
8819	return rvalue(gz, ri, typeof_inst.toRef(), node);
8820	}
8821	const payload_size: u32 = std.meta.fields(Zir.Inst.TypeOfPeer).len;
8822	const payload_index = try reserveExtra(astgen, payload_size + args.len);
8823	const args_index = payload_index + payload_size;
8824
8825	const typeof_inst = try gz.addExtendedMultiOpPayloadIndex(.typeof_peer, payload_index, args.len);
8826
8827	var typeof_scope = gz.makeSubBlock(scope);
8828	typeof_scope.is_comptime = false;
8829
8830	for (args, 0..) \|arg, i\| {
8831	const param_ref = try reachableExpr(&typeof_scope, &typeof_scope.base, .{ .rl = .none }, arg, node);
8832	astgen.extra.items[args_index + i] = @intFromEnum(param_ref);
8833	}
8834	_ = try typeof_scope.addBreak(.break_inline, typeof_inst.toIndex().?, .void_value);
8835
8836	const body = typeof_scope.instructionsSlice();
8837	const body_len = astgen.countBodyLenAfterFixups(body);
8838	astgen.setExtra(payload_index, Zir.Inst.TypeOfPeer{
8839	.body_len = @intCast(body_len),
8840	.body_index = @intCast(astgen.extra.items.len),
8841	.src_node = gz.nodeIndexToRelative(node),
8842	});
8843	try astgen.extra.ensureUnusedCapacity(gpa, body_len);
8844	astgen.appendBodyWithFixups(body);
8845	typeof_scope.unstack();
8846
8847	return rvalue(gz, ri, typeof_inst, node);
8848	}
8849
8850	fn minMax(
8851	gz: *GenZir,
8852	scope: *Scope,
8853	ri: ResultInfo,
8854	node: Ast.Node.Index,
8855	args: []const Ast.Node.Index,
8856	comptime op: enum { min, max },
8857	) InnerError!Zir.Inst.Ref {
8858	const astgen = gz.astgen;
8859	if (args.len < 2) {
8860	return astgen.failNode(node, "expected at least 2 arguments, found 0", .{});
8861	}
8862	if (args.len == 2) {
8863	const tag: Zir.Inst.Tag = switch (op) {
8864	.min => .min,
8865	.max => .max,
8866	};
8867	const a = try expr(gz, scope, .{ .rl = .none }, args[0]);
8868	const b = try expr(gz, scope, .{ .rl = .none }, args[1]);
8869	const result = try gz.addPlNode(tag, node, Zir.Inst.Bin{
8870	.lhs = a,
8871	.rhs = b,
8872	});
8873	return rvalue(gz, ri, result, node);
8874	}
8875	const payload_index = try addExtra(astgen, Zir.Inst.NodeMultiOp{
8876	.src_node = gz.nodeIndexToRelative(node),
8877	});
8878	var extra_index = try reserveExtra(gz.astgen, args.len);
8879	for (args) \|arg\| {
8880	const arg_ref = try expr(gz, scope, .{ .rl = .none }, arg);
8881	astgen.extra.items[extra_index] = @intFromEnum(arg_ref);
8882	extra_index += 1;
8883	}
8884	const tag: Zir.Inst.Extended = switch (op) {
8885	.min => .min_multi,
8886	.max => .max_multi,
8887	};
8888	const result = try gz.addExtendedMultiOpPayloadIndex(tag, payload_index, args.len);
8889	return rvalue(gz, ri, result, node);
8890	}
8891
8892	fn builtinCall(
8893	gz: *GenZir,
8894	scope: *Scope,
8895	ri: ResultInfo,
8896	node: Ast.Node.Index,
8897	params: []const Ast.Node.Index,
8898	) InnerError!Zir.Inst.Ref {
8899	const astgen = gz.astgen;
8900	const tree = astgen.tree;
8901	const main_tokens = tree.nodes.items(.main_token);
8902
8903	const builtin_token = main_tokens[node];
8904	const builtin_name = tree.tokenSlice(builtin_token);
8905
8906	// We handle the different builtins manually because they have different semantics depending
8907	// on the function. For example, `@as` and others participate in result location semantics,
8908	// and `@cImport` creates a special scope that collects a .c source code text buffer.
8909	// Also, some builtins have a variable number of parameters.
8910
8911	const info = BuiltinFn.list.get(builtin_name) orelse {
8912	return astgen.failNode(node, "invalid builtin function: '{s}'", .{
8913	builtin_name,
8914	});
8915	};
8916	if (info.param_count) \|expected\| {
8917	if (expected != params.len) {
8918	const s = if (expected == 1) "" else "s";
8919	return astgen.failNode(node, "expected {d} argument{s}, found {d}", .{
8920	expected, s, params.len,
8921	});
8922	}
8923	}
8924
8925	// Check function scope-only builtins
8926
8927	if (astgen.fn_block == null and info.illegal_outside_function)
8928	return astgen.failNode(node, "'{s}' outside function scope", .{builtin_name});
8929
8930	switch (info.tag) {
8931	.import => {
8932	const node_tags = tree.nodes.items(.tag);
8933	const operand_node = params[0];
8934
8935	if (node_tags[operand_node] != .string_literal) {
8936	// Spec reference: https://github.com/ziglang/zig/issues/2206
8937	return astgen.failNode(operand_node, "@import operand must be a string literal", .{});
8938	}
8939	const str_lit_token = main_tokens[operand_node];
8940	const str = try astgen.strLitAsString(str_lit_token);
8941	const str_slice = astgen.string_bytes.items[@intFromEnum(str.index)..][0..str.len];
8942	if (mem.indexOfScalar(u8, str_slice, 0) != null) {
8943	return astgen.failTok(str_lit_token, "import path cannot contain null bytes", .{});
8944	} else if (str.len == 0) {
8945	return astgen.failTok(str_lit_token, "import path cannot be empty", .{});
8946	}
8947	const result = try gz.addStrTok(.import, str.index, str_lit_token);
8948	const gop = try astgen.imports.getOrPut(astgen.gpa, str.index);
8949	if (!gop.found_existing) {
8950	gop.value_ptr.* = str_lit_token;
8951	}
8952	return rvalue(gz, ri, result, node);
8953	},
8954	.compile_log => {
8955	const payload_index = try addExtra(gz.astgen, Zir.Inst.NodeMultiOp{
8956	.src_node = gz.nodeIndexToRelative(node),
8957	});
8958	var extra_index = try reserveExtra(gz.astgen, params.len);
8959	for (params) \|param\| {
8960	const param_ref = try expr(gz, scope, .{ .rl = .none }, param);
8961	astgen.extra.items[extra_index] = @intFromEnum(param_ref);
8962	extra_index += 1;
8963	}
8964	const result = try gz.addExtendedMultiOpPayloadIndex(.compile_log, payload_index, params.len);
8965	return rvalue(gz, ri, result, node);
8966	},
8967	.field => {
8968	if (ri.rl == .ref or ri.rl == .ref_coerced_ty) {
8969	return gz.addPlNode(.field_ptr_named, node, Zir.Inst.FieldNamed{
8970	.lhs = try expr(gz, scope, .{ .rl = .ref }, params[0]),
8971	.field_name = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .slice_const_u8_type } }, params[1]),
8972	});
8973	}
8974	const result = try gz.addPlNode(.field_val_named, node, Zir.Inst.FieldNamed{
8975	.lhs = try expr(gz, scope, .{ .rl = .none }, params[0]),
8976	.field_name = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .slice_const_u8_type } }, params[1]),
8977	});
8978	return rvalue(gz, ri, result, node);
8979	},
8980
8981	// zig fmt: off
8982	.as => return as( gz, scope, ri, node, params[0], params[1]),
8983	.bit_cast => return bitCast( gz, scope, ri, node, params[0]),
8984	.TypeOf => return typeOf( gz, scope, ri, node, params),
8985	.union_init => return unionInit(gz, scope, ri, node, params),
8986	.c_import => return cImport( gz, scope, node, params[0]),
8987	.min => return minMax( gz, scope, ri, node, params, .min),
8988	.max => return minMax( gz, scope, ri, node, params, .max),
8989	// zig fmt: on
8990
8991	.@"export" => {
8992	const node_tags = tree.nodes.items(.tag);
8993	const node_datas = tree.nodes.items(.data);
8994	// This function causes a Decl to be exported. The first parameter is not an expression,
8995	// but an identifier of the Decl to be exported.
8996	var namespace: Zir.Inst.Ref = .none;
8997	var decl_name: Zir.NullTerminatedString = .empty;
8998	switch (node_tags[params[0]]) {
8999	.identifier => {
9000	const ident_token = main_tokens[params[0]];
9001	if (isPrimitive(tree.tokenSlice(ident_token))) {
9002	return astgen.failTok(ident_token, "unable to export primitive value", .{});
9003	}
9004	decl_name = try astgen.identAsString(ident_token);
9005
9006	var s = scope;
9007	var found_already: ?Ast.Node.Index = null; // we have found a decl with the same name already
9008	while (true) switch (s.tag) {
9009	.local_val => {
9010	const local_val = s.cast(Scope.LocalVal).?;
9011	if (local_val.name == decl_name) {
9012	local_val.used = ident_token;
9013	_ = try gz.addPlNode(.export_value, node, Zir.Inst.ExportValue{
9014	.operand = local_val.inst,
9015	.options = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .export_options_type } }, params[1]),
9016	});
9017	return rvalue(gz, ri, .void_value, node);
9018	}
9019	s = local_val.parent;
9020	},
9021	.local_ptr => {
9022	const local_ptr = s.cast(Scope.LocalPtr).?;
9023	if (local_ptr.name == decl_name) {
9024	if (!local_ptr.maybe_comptime)
9025	return astgen.failNode(params[0], "unable to export runtime-known value", .{});
9026	local_ptr.used = ident_token;
9027	const loaded = try gz.addUnNode(.load, local_ptr.ptr, node);
9028	_ = try gz.addPlNode(.export_value, node, Zir.Inst.ExportValue{
9029	.operand = loaded,
9030	.options = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .export_options_type } }, params[1]),
9031	});
9032	return rvalue(gz, ri, .void_value, node);
9033	}
9034	s = local_ptr.parent;
9035	},
9036	.gen_zir => s = s.cast(GenZir).?.parent,
9037	.defer_normal, .defer_error => s = s.cast(Scope.Defer).?.parent,
9038	.namespace, .enum_namespace => {
9039	const ns = s.cast(Scope.Namespace).?;
9040	if (ns.decls.get(decl_name)) \|i\| {
9041	if (found_already) \|f\| {
9042	return astgen.failNodeNotes(node, "ambiguous reference", .{}, &.{
9043	try astgen.errNoteNode(f, "declared here", .{}),
9044	try astgen.errNoteNode(i, "also declared here", .{}),
9045	});
9046	}
9047	// We found a match but must continue looking for ambiguous references to decls.
9048	found_already = i;
9049	}
9050	s = ns.parent;
9051	},
9052	.top => break,
9053	};
9054	if (found_already == null) {
9055	const ident_name = try astgen.identifierTokenString(ident_token);
9056	return astgen.failNode(params[0], "use of undeclared identifier '{s}'", .{ident_name});
9057	}
9058	},
9059	.field_access => {
9060	const namespace_node = node_datas[params[0]].lhs;
9061	namespace = try typeExpr(gz, scope, namespace_node);
9062	const dot_token = main_tokens[params[0]];
9063	const field_ident = dot_token + 1;
9064	decl_name = try astgen.identAsString(field_ident);
9065	},
9066	else => return astgen.failNode(params[0], "symbol to export must identify a declaration", .{}),
9067	}
9068	const options = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .export_options_type } }, params[1]);
9069	_ = try gz.addPlNode(.@"export", node, Zir.Inst.Export{
9070	.namespace = namespace,
9071	.decl_name = decl_name,
9072	.options = options,
9073	});
9074	return rvalue(gz, ri, .void_value, node);
9075	},
9076	.@"extern" => {
9077	const type_inst = try typeExpr(gz, scope, params[0]);
9078	const options = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .extern_options_type } }, params[1]);
9079	const result = try gz.addExtendedPayload(.builtin_extern, Zir.Inst.BinNode{
9080	.node = gz.nodeIndexToRelative(node),
9081	.lhs = type_inst,
9082	.rhs = options,
9083	});
9084	return rvalue(gz, ri, result, node);
9085	},
9086	.fence => {
9087	const order = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .atomic_order_type } }, params[0]);
9088	_ = try gz.addExtendedPayload(.fence, Zir.Inst.UnNode{
9089	.node = gz.nodeIndexToRelative(node),
9090	.operand = order,
9091	});
9092	return rvalue(gz, ri, .void_value, node);
9093	},
9094	.set_float_mode => {
9095	const order = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .float_mode_type } }, params[0]);
9096	_ = try gz.addExtendedPayload(.set_float_mode, Zir.Inst.UnNode{
9097	.node = gz.nodeIndexToRelative(node),
9098	.operand = order,
9099	});
9100	return rvalue(gz, ri, .void_value, node);
9101	},
9102	.set_align_stack => {
9103	const order = try expr(gz, scope, coerced_align_ri, params[0]);
9104	_ = try gz.addExtendedPayload(.set_align_stack, Zir.Inst.UnNode{
9105	.node = gz.nodeIndexToRelative(node),
9106	.operand = order,
9107	});
9108	return rvalue(gz, ri, .void_value, node);
9109	},
9110	.set_cold => {
9111	const order = try expr(gz, scope, ri, params[0]);
9112	_ = try gz.addExtendedPayload(.set_cold, Zir.Inst.UnNode{
9113	.node = gz.nodeIndexToRelative(node),
9114	.operand = order,
9115	});
9116	return rvalue(gz, ri, .void_value, node);
9117	},
9118
9119	.src => {
9120	const token_starts = tree.tokens.items(.start);
9121	const node_start = token_starts[tree.firstToken(node)];
9122	astgen.advanceSourceCursor(node_start);
9123	const result = try gz.addExtendedPayload(.builtin_src, Zir.Inst.Src{
9124	.node = gz.nodeIndexToRelative(node),
9125	.line = astgen.source_line,
9126	.column = astgen.source_column,
9127	});
9128	return rvalue(gz, ri, result, node);
9129	},
9130
9131	// zig fmt: off
9132	.This => return rvalue(gz, ri, try gz.addNodeExtended(.this, node), node),
9133	.return_address => return rvalue(gz, ri, try gz.addNodeExtended(.ret_addr, node), node),
9134	.error_return_trace => return rvalue(gz, ri, try gz.addNodeExtended(.error_return_trace, node), node),
9135	.frame => return rvalue(gz, ri, try gz.addNodeExtended(.frame, node), node),
9136	.frame_address => return rvalue(gz, ri, try gz.addNodeExtended(.frame_address, node), node),
9137	.breakpoint => return rvalue(gz, ri, try gz.addNodeExtended(.breakpoint, node), node),
9138	.in_comptime => return rvalue(gz, ri, try gz.addNodeExtended(.in_comptime, node), node),
9139
9140	.type_info => return simpleUnOpType(gz, scope, ri, node, params[0], .type_info),
9141	.size_of => return simpleUnOpType(gz, scope, ri, node, params[0], .size_of),
9142	.bit_size_of => return simpleUnOpType(gz, scope, ri, node, params[0], .bit_size_of),
9143	.align_of => return simpleUnOpType(gz, scope, ri, node, params[0], .align_of),
9144
9145	.int_from_ptr => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .int_from_ptr),
9146	.compile_error => return simpleUnOp(gz, scope, ri, node, .{ .rl = .{ .coerced_ty = .slice_const_u8_type } }, params[0], .compile_error),
9147	.set_eval_branch_quota => return simpleUnOp(gz, scope, ri, node, .{ .rl = .{ .coerced_ty = .u32_type } }, params[0], .set_eval_branch_quota),
9148	.int_from_enum => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .int_from_enum),
9149	.int_from_bool => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .int_from_bool),
9150	.embed_file => return simpleUnOp(gz, scope, ri, node, .{ .rl = .{ .coerced_ty = .slice_const_u8_type } }, params[0], .embed_file),
9151	.error_name => return simpleUnOp(gz, scope, ri, node, .{ .rl = .{ .coerced_ty = .anyerror_type } }, params[0], .error_name),
9152	.set_runtime_safety => return simpleUnOp(gz, scope, ri, node, coerced_bool_ri, params[0], .set_runtime_safety),
9153	.sqrt => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .sqrt),
9154	.sin => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .sin),
9155	.cos => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .cos),
9156	.tan => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .tan),
9157	.exp => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .exp),
9158	.exp2 => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .exp2),
9159	.log => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .log),
9160	.log2 => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .log2),
9161	.log10 => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .log10),
9162	.abs => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .abs),
9163	.floor => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .floor),
9164	.ceil => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .ceil),
9165	.trunc => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .trunc),
9166	.round => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .round),
9167	.tag_name => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .tag_name),
9168	.type_name => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .type_name),
9169	.Frame => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .frame_type),
9170	.frame_size => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .frame_size),
9171
9172	.int_from_float => return typeCast(gz, scope, ri, node, params[0], .int_from_float, builtin_name),
9173	.float_from_int => return typeCast(gz, scope, ri, node, params[0], .float_from_int, builtin_name),
9174	.ptr_from_int => return typeCast(gz, scope, ri, node, params[0], .ptr_from_int, builtin_name),
9175	.enum_from_int => return typeCast(gz, scope, ri, node, params[0], .enum_from_int, builtin_name),
9176	.float_cast => return typeCast(gz, scope, ri, node, params[0], .float_cast, builtin_name),
9177	.int_cast => return typeCast(gz, scope, ri, node, params[0], .int_cast, builtin_name),
9178	.truncate => return typeCast(gz, scope, ri, node, params[0], .truncate, builtin_name),
9179	// zig fmt: on
9180
9181	.Type => {
9182	const operand = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .type_info_type } }, params[0]);
9183
9184	const gpa = gz.astgen.gpa;
9185
9186	try gz.instructions.ensureUnusedCapacity(gpa, 1);
9187	try gz.astgen.instructions.ensureUnusedCapacity(gpa, 1);
9188
9189	const payload_index = try gz.astgen.addExtra(Zir.Inst.UnNode{
9190	.node = gz.nodeIndexToRelative(node),
9191	.operand = operand,
9192	});
9193	const new_index: Zir.Inst.Index = @enumFromInt(gz.astgen.instructions.len);
9194	gz.astgen.instructions.appendAssumeCapacity(.{
9195	.tag = .extended,
9196	.data = .{ .extended = .{
9197	.opcode = .reify,
9198	.small = @intFromEnum(gz.anon_name_strategy),
9199	.operand = payload_index,
9200	} },
9201	});
9202	gz.instructions.appendAssumeCapacity(new_index);
9203	const result = new_index.toRef();
9204	return rvalue(gz, ri, result, node);
9205	},
9206	.panic => {
9207	try emitDbgNode(gz, node);
9208	return simpleUnOp(gz, scope, ri, node, .{ .rl = .{ .coerced_ty = .slice_const_u8_type } }, params[0], .panic);
9209	},
9210	.trap => {
9211	try emitDbgNode(gz, node);
9212	_ = try gz.addNode(.trap, node);
9213	return rvalue(gz, ri, .unreachable_value, node);
9214	},
9215	.int_from_error => {
9216	const operand = try expr(gz, scope, .{ .rl = .none }, params[0]);
9217	const result = try gz.addExtendedPayload(.int_from_error, Zir.Inst.UnNode{
9218	.node = gz.nodeIndexToRelative(node),
9219	.operand = operand,
9220	});
9221	return rvalue(gz, ri, result, node);
9222	},
9223	.error_from_int => {
9224	const operand = try expr(gz, scope, .{ .rl = .none }, params[0]);
9225	const result = try gz.addExtendedPayload(.error_from_int, Zir.Inst.UnNode{
9226	.node = gz.nodeIndexToRelative(node),
9227	.operand = operand,
9228	});
9229	return rvalue(gz, ri, result, node);
9230	},
9231	.error_cast => {
9232	try emitDbgNode(gz, node);
9233
9234	const result = try gz.addExtendedPayload(.error_cast, Zir.Inst.BinNode{
9235	.lhs = try ri.rl.resultTypeForCast(gz, node, "@errorCast"),
9236	.rhs = try expr(gz, scope, .{ .rl = .none }, params[0]),
9237	.node = gz.nodeIndexToRelative(node),
9238	});
9239	return rvalue(gz, ri, result, node);
9240	},
9241	.ptr_cast,
9242	.align_cast,
9243	.addrspace_cast,
9244	.const_cast,
9245	.volatile_cast,
9246	=> return ptrCast(gz, scope, ri, node),
9247
9248	// zig fmt: off
9249	.has_decl => return hasDeclOrField(gz, scope, ri, node, params[0], params[1], .has_decl),
9250	.has_field => return hasDeclOrField(gz, scope, ri, node, params[0], params[1], .has_field),
9251
9252	.clz => return bitBuiltin(gz, scope, ri, node, params[0], .clz),
9253	.ctz => return bitBuiltin(gz, scope, ri, node, params[0], .ctz),
9254	.pop_count => return bitBuiltin(gz, scope, ri, node, params[0], .pop_count),
9255	.byte_swap => return bitBuiltin(gz, scope, ri, node, params[0], .byte_swap),
9256	.bit_reverse => return bitBuiltin(gz, scope, ri, node, params[0], .bit_reverse),
9257
9258	.div_exact => return divBuiltin(gz, scope, ri, node, params[0], params[1], .div_exact),
9259	.div_floor => return divBuiltin(gz, scope, ri, node, params[0], params[1], .div_floor),
9260	.div_trunc => return divBuiltin(gz, scope, ri, node, params[0], params[1], .div_trunc),
9261	.mod => return divBuiltin(gz, scope, ri, node, params[0], params[1], .mod),
9262	.rem => return divBuiltin(gz, scope, ri, node, params[0], params[1], .rem),
9263
9264	.shl_exact => return shiftOp(gz, scope, ri, node, params[0], params[1], .shl_exact),
9265	.shr_exact => return shiftOp(gz, scope, ri, node, params[0], params[1], .shr_exact),
9266
9267	.bit_offset_of => return offsetOf(gz, scope, ri, node, params[0], params[1], .bit_offset_of),
9268	.offset_of => return offsetOf(gz, scope, ri, node, params[0], params[1], .offset_of),
9269
9270	.c_undef => return simpleCBuiltin(gz, scope, ri, node, params[0], .c_undef),
9271	.c_include => return simpleCBuiltin(gz, scope, ri, node, params[0], .c_include),
9272
9273	.cmpxchg_strong => return cmpxchg(gz, scope, ri, node, params, 1),
9274	.cmpxchg_weak => return cmpxchg(gz, scope, ri, node, params, 0),
9275	// zig fmt: on
9276
9277	.wasm_memory_size => {
9278	const operand = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .u32_type } }, params[0]);
9279	const result = try gz.addExtendedPayload(.wasm_memory_size, Zir.Inst.UnNode{
9280	.node = gz.nodeIndexToRelative(node),
9281	.operand = operand,
9282	});
9283	return rvalue(gz, ri, result, node);
9284	},
9285	.wasm_memory_grow => {
9286	const index_arg = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .u32_type } }, params[0]);
9287	const delta_arg = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .u32_type } }, params[1]);
9288	const result = try gz.addExtendedPayload(.wasm_memory_grow, Zir.Inst.BinNode{
9289	.node = gz.nodeIndexToRelative(node),
9290	.lhs = index_arg,
9291	.rhs = delta_arg,
9292	});
9293	return rvalue(gz, ri, result, node);
9294	},
9295	.c_define => {
9296	if (!gz.c_import) return gz.astgen.failNode(node, "C define valid only inside C import block", .{});
9297	const name = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .slice_const_u8_type } }, params[0]);
9298	const value = try comptimeExpr(gz, scope, .{ .rl = .none }, params[1]);
9299	const result = try gz.addExtendedPayload(.c_define, Zir.Inst.BinNode{
9300	.node = gz.nodeIndexToRelative(node),
9301	.lhs = name,
9302	.rhs = value,
9303	});
9304	return rvalue(gz, ri, result, node);
9305	},
9306
9307	.splat => {
9308	const result_type = try ri.rl.resultTypeForCast(gz, node, "@splat");
9309	const elem_type = try gz.addUnNode(.vector_elem_type, result_type, node);
9310	const scalar = try expr(gz, scope, .{ .rl = .{ .ty = elem_type } }, params[0]);
9311	const result = try gz.addPlNode(.splat, node, Zir.Inst.Bin{
9312	.lhs = result_type,
9313	.rhs = scalar,
9314	});
9315	return rvalue(gz, ri, result, node);
9316	},
9317	.reduce => {
9318	const op = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .reduce_op_type } }, params[0]);
9319	const scalar = try expr(gz, scope, .{ .rl = .none }, params[1]);
9320	const result = try gz.addPlNode(.reduce, node, Zir.Inst.Bin{
9321	.lhs = op,
9322	.rhs = scalar,
9323	});
9324	return rvalue(gz, ri, result, node);
9325	},
9326
9327	.add_with_overflow => return overflowArithmetic(gz, scope, ri, node, params, .add_with_overflow),
9328	.sub_with_overflow => return overflowArithmetic(gz, scope, ri, node, params, .sub_with_overflow),
9329	.mul_with_overflow => return overflowArithmetic(gz, scope, ri, node, params, .mul_with_overflow),
9330	.shl_with_overflow => return overflowArithmetic(gz, scope, ri, node, params, .shl_with_overflow),
9331
9332	.atomic_load => {
9333	const result = try gz.addPlNode(.atomic_load, node, Zir.Inst.AtomicLoad{
9334	// zig fmt: off
9335	.elem_type = try typeExpr(gz, scope, params[0]),
9336	.ptr = try expr (gz, scope, .{ .rl = .none }, params[1]),
9337	.ordering = try expr (gz, scope, .{ .rl = .{ .coerced_ty = .atomic_order_type } }, params[2]),
9338	// zig fmt: on
9339	});
9340	return rvalue(gz, ri, result, node);
9341	},
9342	.atomic_rmw => {
9343	const int_type = try typeExpr(gz, scope, params[0]);
9344	const result = try gz.addPlNode(.atomic_rmw, node, Zir.Inst.AtomicRmw{
9345	// zig fmt: off
9346	.ptr = try expr(gz, scope, .{ .rl = .none }, params[1]),
9347	.operation = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .atomic_rmw_op_type } }, params[2]),
9348	.operand = try expr(gz, scope, .{ .rl = .{ .ty = int_type } }, params[3]),
9349	.ordering = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .atomic_order_type } }, params[4]),
9350	// zig fmt: on
9351	});
9352	return rvalue(gz, ri, result, node);
9353	},
9354	.atomic_store => {
9355	const int_type = try typeExpr(gz, scope, params[0]);
9356	_ = try gz.addPlNode(.atomic_store, node, Zir.Inst.AtomicStore{
9357	// zig fmt: off
9358	.ptr = try expr(gz, scope, .{ .rl = .none }, params[1]),
9359	.operand = try expr(gz, scope, .{ .rl = .{ .ty = int_type } }, params[2]),
9360	.ordering = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .atomic_order_type } }, params[3]),
9361	// zig fmt: on
9362	});
9363	return rvalue(gz, ri, .void_value, node);
9364	},
9365	.mul_add => {
9366	const float_type = try typeExpr(gz, scope, params[0]);
9367	const mulend1 = try expr(gz, scope, .{ .rl = .{ .coerced_ty = float_type } }, params[1]);
9368	const mulend2 = try expr(gz, scope, .{ .rl = .{ .coerced_ty = float_type } }, params[2]);
9369	const addend = try expr(gz, scope, .{ .rl = .{ .ty = float_type } }, params[3]);
9370	const result = try gz.addPlNode(.mul_add, node, Zir.Inst.MulAdd{
9371	.mulend1 = mulend1,
9372	.mulend2 = mulend2,
9373	.addend = addend,
9374	});
9375	return rvalue(gz, ri, result, node);
9376	},
9377	.call => {
9378	const modifier = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .call_modifier_type } }, params[0]);
9379	const callee = try expr(gz, scope, .{ .rl = .none }, params[1]);
9380	const args = try expr(gz, scope, .{ .rl = .none }, params[2]);
9381	const result = try gz.addPlNode(.builtin_call, node, Zir.Inst.BuiltinCall{
9382	.modifier = modifier,
9383	.callee = callee,
9384	.args = args,
9385	.flags = .{
9386	.is_nosuspend = gz.nosuspend_node != 0,
9387	.ensure_result_used = false,
9388	},
9389	});
9390	return rvalue(gz, ri, result, node);
9391	},
9392	.field_parent_ptr => {
9393	const parent_type = try typeExpr(gz, scope, params[0]);
9394	const field_name = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .slice_const_u8_type } }, params[1]);
9395	const result = try gz.addPlNode(.field_parent_ptr, node, Zir.Inst.FieldParentPtr{
9396	.parent_type = parent_type,
9397	.field_name = field_name,
9398	.field_ptr = try expr(gz, scope, .{ .rl = .none }, params[2]),
9399	});
9400	return rvalue(gz, ri, result, node);
9401	},
9402	.memcpy => {
9403	_ = try gz.addPlNode(.memcpy, node, Zir.Inst.Bin{
9404	.lhs = try expr(gz, scope, .{ .rl = .none }, params[0]),
9405	.rhs = try expr(gz, scope, .{ .rl = .none }, params[1]),
9406	});
9407	return rvalue(gz, ri, .void_value, node);
9408	},
9409	.memset => {
9410	const lhs = try expr(gz, scope, .{ .rl = .none }, params[0]);
9411	const lhs_ty = try gz.addUnNode(.typeof, lhs, params[0]);
9412	const elem_ty = try gz.addUnNode(.indexable_ptr_elem_type, lhs_ty, params[0]);
9413	_ = try gz.addPlNode(.memset, node, Zir.Inst.Bin{
9414	.lhs = lhs,
9415	.rhs = try expr(gz, scope, .{ .rl = .{ .coerced_ty = elem_ty } }, params[1]),
9416	});
9417	return rvalue(gz, ri, .void_value, node);
9418	},
9419	.shuffle => {
9420	const result = try gz.addPlNode(.shuffle, node, Zir.Inst.Shuffle{
9421	.elem_type = try typeExpr(gz, scope, params[0]),
9422	.a = try expr(gz, scope, .{ .rl = .none }, params[1]),
9423	.b = try expr(gz, scope, .{ .rl = .none }, params[2]),
9424	.mask = try comptimeExpr(gz, scope, .{ .rl = .none }, params[3]),
9425	});
9426	return rvalue(gz, ri, result, node);
9427	},
9428	.select => {
9429	const result = try gz.addExtendedPayload(.select, Zir.Inst.Select{
9430	.node = gz.nodeIndexToRelative(node),
9431	.elem_type = try typeExpr(gz, scope, params[0]),
9432	.pred = try expr(gz, scope, .{ .rl = .none }, params[1]),
9433	.a = try expr(gz, scope, .{ .rl = .none }, params[2]),
9434	.b = try expr(gz, scope, .{ .rl = .none }, params[3]),
9435	});
9436	return rvalue(gz, ri, result, node);
9437	},
9438	.async_call => {
9439	const result = try gz.addExtendedPayload(.builtin_async_call, Zir.Inst.AsyncCall{
9440	.node = gz.nodeIndexToRelative(node),
9441	.frame_buffer = try expr(gz, scope, .{ .rl = .none }, params[0]),
9442	.result_ptr = try expr(gz, scope, .{ .rl = .none }, params[1]),
9443	.fn_ptr = try expr(gz, scope, .{ .rl = .none }, params[2]),
9444	.args = try expr(gz, scope, .{ .rl = .none }, params[3]),
9445	});
9446	return rvalue(gz, ri, result, node);
9447	},
9448	.Vector => {
9449	const result = try gz.addPlNode(.vector_type, node, Zir.Inst.Bin{
9450	.lhs = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .u32_type } }, params[0]),
9451	.rhs = try typeExpr(gz, scope, params[1]),
9452	});
9453	return rvalue(gz, ri, result, node);
9454	},
9455	.prefetch => {
9456	const ptr = try expr(gz, scope, .{ .rl = .none }, params[0]);
9457	const options = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .prefetch_options_type } }, params[1]);
9458	_ = try gz.addExtendedPayload(.prefetch, Zir.Inst.BinNode{
9459	.node = gz.nodeIndexToRelative(node),
9460	.lhs = ptr,
9461	.rhs = options,
9462	});
9463	return rvalue(gz, ri, .void_value, node);
9464	},
9465	.c_va_arg => {
9466	const result = try gz.addExtendedPayload(.c_va_arg, Zir.Inst.BinNode{
9467	.node = gz.nodeIndexToRelative(node),
9468	.lhs = try expr(gz, scope, .{ .rl = .none }, params[0]),
9469	.rhs = try typeExpr(gz, scope, params[1]),
9470	});
9471	return rvalue(gz, ri, result, node);
9472	},
9473	.c_va_copy => {
9474	const result = try gz.addExtendedPayload(.c_va_copy, Zir.Inst.UnNode{
9475	.node = gz.nodeIndexToRelative(node),
9476	.operand = try expr(gz, scope, .{ .rl = .none }, params[0]),
9477	});
9478	return rvalue(gz, ri, result, node);
9479	},
9480	.c_va_end => {
9481	const result = try gz.addExtendedPayload(.c_va_end, Zir.Inst.UnNode{
9482	.node = gz.nodeIndexToRelative(node),
9483	.operand = try expr(gz, scope, .{ .rl = .none }, params[0]),
9484	});
9485	return rvalue(gz, ri, result, node);
9486	},
9487	.c_va_start => {
9488	if (!astgen.fn_var_args) {
9489	return astgen.failNode(node, "'@cVaStart' in a non-variadic function", .{});
9490	}
9491	return rvalue(gz, ri, try gz.addNodeExtended(.c_va_start, node), node);
9492	},
9493
9494	.work_item_id => {
9495	const operand = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .u32_type } }, params[0]);
9496	const result = try gz.addExtendedPayload(.work_item_id, Zir.Inst.UnNode{
9497	.node = gz.nodeIndexToRelative(node),
9498	.operand = operand,
9499	});
9500	return rvalue(gz, ri, result, node);
9501	},
9502	.work_group_size => {
9503	const operand = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .u32_type } }, params[0]);
9504	const result = try gz.addExtendedPayload(.work_group_size, Zir.Inst.UnNode{
9505	.node = gz.nodeIndexToRelative(node),
9506	.operand = operand,
9507	});
9508	return rvalue(gz, ri, result, node);
9509	},
9510	.work_group_id => {
9511	const operand = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .u32_type } }, params[0]);
9512	const result = try gz.addExtendedPayload(.work_group_id, Zir.Inst.UnNode{
9513	.node = gz.nodeIndexToRelative(node),
9514	.operand = operand,
9515	});
9516	return rvalue(gz, ri, result, node);
9517	},
9518	}
9519	}
9520
9521	fn hasDeclOrField(
9522	gz: *GenZir,
9523	scope: *Scope,
9524	ri: ResultInfo,
9525	node: Ast.Node.Index,
9526	lhs_node: Ast.Node.Index,
9527	rhs_node: Ast.Node.Index,
9528	tag: Zir.Inst.Tag,
9529	) InnerError!Zir.Inst.Ref {
9530	const container_type = try typeExpr(gz, scope, lhs_node);
9531	const name = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .slice_const_u8_type } }, rhs_node);
9532	const result = try gz.addPlNode(tag, node, Zir.Inst.Bin{
9533	.lhs = container_type,
9534	.rhs = name,
9535	});
9536	return rvalue(gz, ri, result, node);
9537	}
9538
9539	fn typeCast(
9540	gz: *GenZir,
9541	scope: *Scope,
9542	ri: ResultInfo,
9543	node: Ast.Node.Index,
9544	operand_node: Ast.Node.Index,
9545	tag: Zir.Inst.Tag,
9546	builtin_name: []const u8,
9547	) InnerError!Zir.Inst.Ref {
9548	const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
9549	const result_type = try ri.rl.resultTypeForCast(gz, node, builtin_name);
9550	const operand = try expr(gz, scope, .{ .rl = .none }, operand_node);
9551
9552	try emitDbgStmt(gz, cursor);
9553	const result = try gz.addPlNode(tag, node, Zir.Inst.Bin{
9554	.lhs = result_type,
9555	.rhs = operand,
9556	});
9557	return rvalue(gz, ri, result, node);
9558	}
9559
9560	fn simpleUnOpType(
9561	gz: *GenZir,
9562	scope: *Scope,
9563	ri: ResultInfo,
9564	node: Ast.Node.Index,
9565	operand_node: Ast.Node.Index,
9566	tag: Zir.Inst.Tag,
9567	) InnerError!Zir.Inst.Ref {
9568	const operand = try typeExpr(gz, scope, operand_node);
9569	const result = try gz.addUnNode(tag, operand, node);
9570	return rvalue(gz, ri, result, node);
9571	}
9572
9573	fn simpleUnOp(
9574	gz: *GenZir,
9575	scope: *Scope,
9576	ri: ResultInfo,
9577	node: Ast.Node.Index,
9578	operand_ri: ResultInfo,
9579	operand_node: Ast.Node.Index,
9580	tag: Zir.Inst.Tag,
9581	) InnerError!Zir.Inst.Ref {
9582	const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
9583	const operand = if (tag == .compile_error)
9584	try comptimeExpr(gz, scope, operand_ri, operand_node)
9585	else
9586	try expr(gz, scope, operand_ri, operand_node);
9587	switch (tag) {
9588	.tag_name, .error_name, .int_from_ptr => try emitDbgStmt(gz, cursor),
9589	else => {},
9590	}
9591	const result = try gz.addUnNode(tag, operand, node);
9592	return rvalue(gz, ri, result, node);
9593	}
9594
9595	fn negation(
9596	gz: *GenZir,
9597	scope: *Scope,
9598	ri: ResultInfo,
9599	node: Ast.Node.Index,
9600	) InnerError!Zir.Inst.Ref {
9601	const astgen = gz.astgen;
9602	const tree = astgen.tree;
9603	const node_tags = tree.nodes.items(.tag);
9604	const node_datas = tree.nodes.items(.data);
9605
9606	// Check for float literal as the sub-expression because we want to preserve
9607	// its negativity rather than having it go through comptime subtraction.
9608	const operand_node = node_datas[node].lhs;
9609	if (node_tags[operand_node] == .number_literal) {
9610	return numberLiteral(gz, ri, operand_node, node, .negative);
9611	}
9612
9613	const operand = try expr(gz, scope, .{ .rl = .none }, operand_node);
9614	const result = try gz.addUnNode(.negate, operand, node);
9615	return rvalue(gz, ri, result, node);
9616	}
9617
9618	fn cmpxchg(
9619	gz: *GenZir,
9620	scope: *Scope,
9621	ri: ResultInfo,
9622	node: Ast.Node.Index,
9623	params: []const Ast.Node.Index,
9624	small: u16,
9625	) InnerError!Zir.Inst.Ref {
9626	const int_type = try typeExpr(gz, scope, params[0]);
9627	const result = try gz.addExtendedPayloadSmall(.cmpxchg, small, Zir.Inst.Cmpxchg{
9628	// zig fmt: off
9629	.node = gz.nodeIndexToRelative(node),
9630	.ptr = try expr(gz, scope, .{ .rl = .none }, params[1]),
9631	.expected_value = try expr(gz, scope, .{ .rl = .{ .ty = int_type } }, params[2]),
9632	.new_value = try expr(gz, scope, .{ .rl = .{ .coerced_ty = int_type } }, params[3]),
9633	.success_order = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .atomic_order_type } }, params[4]),
9634	.failure_order = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .atomic_order_type } }, params[5]),
9635	// zig fmt: on
9636	});
9637	return rvalue(gz, ri, result, node);
9638	}
9639
9640	fn bitBuiltin(
9641	gz: *GenZir,
9642	scope: *Scope,
9643	ri: ResultInfo,
9644	node: Ast.Node.Index,
9645	operand_node: Ast.Node.Index,
9646	tag: Zir.Inst.Tag,
9647	) InnerError!Zir.Inst.Ref {
9648	const operand = try expr(gz, scope, .{ .rl = .none }, operand_node);
9649	const result = try gz.addUnNode(tag, operand, node);
9650	return rvalue(gz, ri, result, node);
9651	}
9652
9653	fn divBuiltin(
9654	gz: *GenZir,
9655	scope: *Scope,
9656	ri: ResultInfo,
9657	node: Ast.Node.Index,
9658	lhs_node: Ast.Node.Index,
9659	rhs_node: Ast.Node.Index,
9660	tag: Zir.Inst.Tag,
9661	) InnerError!Zir.Inst.Ref {
9662	const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
9663	const lhs = try expr(gz, scope, .{ .rl = .none }, lhs_node);
9664	const rhs = try expr(gz, scope, .{ .rl = .none }, rhs_node);
9665
9666	try emitDbgStmt(gz, cursor);
9667	const result = try gz.addPlNode(tag, node, Zir.Inst.Bin{ .lhs = lhs, .rhs = rhs });
9668	return rvalue(gz, ri, result, node);
9669	}
9670
9671	fn simpleCBuiltin(
9672	gz: *GenZir,
9673	scope: *Scope,
9674	ri: ResultInfo,
9675	node: Ast.Node.Index,
9676	operand_node: Ast.Node.Index,
9677	tag: Zir.Inst.Extended,
9678	) InnerError!Zir.Inst.Ref {
9679	const name: []const u8 = if (tag == .c_undef) "C undef" else "C include";
9680	if (!gz.c_import) return gz.astgen.failNode(node, "{s} valid only inside C import block", .{name});
9681	const operand = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .slice_const_u8_type } }, operand_node);
9682	_ = try gz.addExtendedPayload(tag, Zir.Inst.UnNode{
9683	.node = gz.nodeIndexToRelative(node),
9684	.operand = operand,
9685	});
9686	return rvalue(gz, ri, .void_value, node);
9687	}
9688
9689	fn offsetOf(
9690	gz: *GenZir,
9691	scope: *Scope,
9692	ri: ResultInfo,
9693	node: Ast.Node.Index,
9694	lhs_node: Ast.Node.Index,
9695	rhs_node: Ast.Node.Index,
9696	tag: Zir.Inst.Tag,
9697	) InnerError!Zir.Inst.Ref {
9698	const type_inst = try typeExpr(gz, scope, lhs_node);
9699	const field_name = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .slice_const_u8_type } }, rhs_node);
9700	const result = try gz.addPlNode(tag, node, Zir.Inst.Bin{
9701	.lhs = type_inst,
9702	.rhs = field_name,
9703	});
9704	return rvalue(gz, ri, result, node);
9705	}
9706
9707	fn shiftOp(
9708	gz: *GenZir,
9709	scope: *Scope,
9710	ri: ResultInfo,
9711	node: Ast.Node.Index,
9712	lhs_node: Ast.Node.Index,
9713	rhs_node: Ast.Node.Index,
9714	tag: Zir.Inst.Tag,
9715	) InnerError!Zir.Inst.Ref {
9716	const lhs = try expr(gz, scope, .{ .rl = .none }, lhs_node);
9717
9718	const cursor = switch (gz.astgen.tree.nodes.items(.tag)[node]) {
9719	.shl, .shr => maybeAdvanceSourceCursorToMainToken(gz, node),
9720	else => undefined,
9721	};
9722
9723	const log2_int_type = try gz.addUnNode(.typeof_log2_int_type, lhs, lhs_node);
9724	const rhs = try expr(gz, scope, .{ .rl = .{ .ty = log2_int_type }, .ctx = .shift_op }, rhs_node);
9725
9726	switch (gz.astgen.tree.nodes.items(.tag)[node]) {
9727	.shl, .shr => try emitDbgStmt(gz, cursor),
9728	else => undefined,
9729	}
9730
9731	const result = try gz.addPlNode(tag, node, Zir.Inst.Bin{
9732	.lhs = lhs,
9733	.rhs = rhs,
9734	});
9735	return rvalue(gz, ri, result, node);
9736	}
9737
9738	fn cImport(
9739	gz: *GenZir,
9740	scope: *Scope,
9741	node: Ast.Node.Index,
9742	body_node: Ast.Node.Index,
9743	) InnerError!Zir.Inst.Ref {
9744	const astgen = gz.astgen;
9745	const gpa = astgen.gpa;
9746
9747	if (gz.c_import) return gz.astgen.failNode(node, "cannot nest @cImport", .{});
9748
9749	var block_scope = gz.makeSubBlock(scope);
9750	block_scope.is_comptime = true;
9751	block_scope.c_import = true;
9752	defer block_scope.unstack();
9753
9754	const block_inst = try gz.makeBlockInst(.c_import, node);
9755	const block_result = try expr(&block_scope, &block_scope.base, .{ .rl = .none }, body_node);
9756	_ = try gz.addUnNode(.ensure_result_used, block_result, node);
9757	if (!gz.refIsNoReturn(block_result)) {
9758	_ = try block_scope.addBreak(.break_inline, block_inst, .void_value);
9759	}
9760	try block_scope.setBlockBody(block_inst);
9761	// block_scope unstacked now, can add new instructions to gz
9762	try gz.instructions.append(gpa, block_inst);
9763
9764	return block_inst.toRef();
9765	}
9766
9767	fn overflowArithmetic(
9768	gz: *GenZir,
9769	scope: *Scope,
9770	ri: ResultInfo,
9771	node: Ast.Node.Index,
9772	params: []const Ast.Node.Index,
9773	tag: Zir.Inst.Extended,
9774	) InnerError!Zir.Inst.Ref {
9775	const lhs = try expr(gz, scope, .{ .rl = .none }, params[0]);
9776	const rhs = try expr(gz, scope, .{ .rl = .none }, params[1]);
9777	const result = try gz.addExtendedPayload(tag, Zir.Inst.BinNode{
9778	.node = gz.nodeIndexToRelative(node),
9779	.lhs = lhs,
9780	.rhs = rhs,
9781	});
9782	return rvalue(gz, ri, result, node);
9783	}
9784
9785	fn callExpr(
9786	gz: *GenZir,
9787	scope: *Scope,
9788	ri: ResultInfo,
9789	node: Ast.Node.Index,
9790	call: Ast.full.Call,
9791	) InnerError!Zir.Inst.Ref {
9792	const astgen = gz.astgen;
9793
9794	const callee = try calleeExpr(gz, scope, call.ast.fn_expr);
9795	const modifier: std.builtin.CallModifier = blk: {
9796	if (gz.is_comptime) {
9797	break :blk .compile_time;
9798	}
9799	if (call.async_token != null) {
9800	break :blk .async_kw;
9801	}
9802	if (gz.nosuspend_node != 0) {
9803	break :blk .no_async;
9804	}
9805	break :blk .auto;
9806	};
9807
9808	{
9809	astgen.advanceSourceCursor(astgen.tree.tokens.items(.start)[call.ast.lparen]);
9810	const line = astgen.source_line - gz.decl_line;
9811	const column = astgen.source_column;
9812	// Sema expects a dbg_stmt immediately before call,
9813	try emitDbgStmtForceCurrentIndex(gz, .{ line, column });
9814	}
9815
9816	switch (callee) {
9817	.direct => \|obj\| assert(obj != .none),
9818	.field => \|field\| assert(field.obj_ptr != .none),
9819	}
9820	assert(node != 0);
9821
9822	const call_index: Zir.Inst.Index = @enumFromInt(astgen.instructions.len);
9823	const call_inst = call_index.toRef();
9824	try gz.astgen.instructions.append(astgen.gpa, undefined);
9825	try gz.instructions.append(astgen.gpa, call_index);
9826
9827	const scratch_top = astgen.scratch.items.len;
9828	defer astgen.scratch.items.len = scratch_top;
9829
9830	var scratch_index = scratch_top;
9831	try astgen.scratch.resize(astgen.gpa, scratch_top + call.ast.params.len);
9832
9833	for (call.ast.params) \|param_node\| {
9834	var arg_block = gz.makeSubBlock(scope);
9835	defer arg_block.unstack();
9836
9837	// `call_inst` is reused to provide the param type.
9838	const arg_ref = try expr(&arg_block, &arg_block.base, .{ .rl = .{ .coerced_ty = call_inst }, .ctx = .fn_arg }, param_node);
9839	_ = try arg_block.addBreakWithSrcNode(.break_inline, call_index, arg_ref, param_node);
9840
9841	const body = arg_block.instructionsSlice();
9842	try astgen.scratch.ensureUnusedCapacity(astgen.gpa, countBodyLenAfterFixups(astgen, body));
9843	appendBodyWithFixupsArrayList(astgen, &astgen.scratch, body);
9844
9845	astgen.scratch.items[scratch_index] = @intCast(astgen.scratch.items.len - scratch_top);
9846	scratch_index += 1;
9847	}
9848
9849	// If our result location is a try/catch/error-union-if/return, a function argument,
9850	// or an initializer for a `const` variable, the error trace propagates.
9851	// Otherwise, it should always be popped (handled in Sema).
9852	const propagate_error_trace = switch (ri.ctx) {
9853	.error_handling_expr, .@"return", .fn_arg, .const_init => true,
9854	else => false,
9855	};
9856
9857	switch (callee) {
9858	.direct => \|callee_obj\| {
9859	const payload_index = try addExtra(astgen, Zir.Inst.Call{
9860	.callee = callee_obj,
9861	.flags = .{
9862	.pop_error_return_trace = !propagate_error_trace,
9863	.packed_modifier = @intCast(@intFromEnum(modifier)),
9864	.args_len = @intCast(call.ast.params.len),
9865	},
9866	});
9867	if (call.ast.params.len != 0) {
9868	try astgen.extra.appendSlice(astgen.gpa, astgen.scratch.items[scratch_top..]);
9869	}
9870	gz.astgen.instructions.set(@intFromEnum(call_index), .{
9871	.tag = .call,
9872	.data = .{ .pl_node = .{
9873	.src_node = gz.nodeIndexToRelative(node),
9874	.payload_index = payload_index,
9875	} },
9876	});
9877	},
9878	.field => \|callee_field\| {
9879	const payload_index = try addExtra(astgen, Zir.Inst.FieldCall{
9880	.obj_ptr = callee_field.obj_ptr,
9881	.field_name_start = callee_field.field_name_start,
9882	.flags = .{
9883	.pop_error_return_trace = !propagate_error_trace,
9884	.packed_modifier = @intCast(@intFromEnum(modifier)),
9885	.args_len = @intCast(call.ast.params.len),
9886	},
9887	});
9888	if (call.ast.params.len != 0) {
9889	try astgen.extra.appendSlice(astgen.gpa, astgen.scratch.items[scratch_top..]);
9890	}
9891	gz.astgen.instructions.set(@intFromEnum(call_index), .{
9892	.tag = .field_call,
9893	.data = .{ .pl_node = .{
9894	.src_node = gz.nodeIndexToRelative(node),
9895	.payload_index = payload_index,
9896	} },
9897	});
9898	},
9899	}
9900	return rvalue(gz, ri, call_inst, node); // TODO function call with result location
9901	}
9902
9903	const Callee = union(enum) {
9904	field: struct {
9905	/// A pointer to the object the field is fetched on, so that we can
9906	/// promote the lvalue to an address if the first parameter requires it.
9907	obj_ptr: Zir.Inst.Ref,
9908	/// Offset into `string_bytes`.
9909	field_name_start: Zir.NullTerminatedString,
9910	},
9911	direct: Zir.Inst.Ref,
9912	};
9913
9914	/// calleeExpr generates the function part of a call expression (f in f(x)), but
9915	/// not the callee argument to the @call() builtin. Its purpose is to
9916	/// distinguish between standard calls and method call syntax `a.b()`. Thus, if
9917	/// the lhs is a field access, we return using the `field` union field;
9918	/// otherwise, we use the `direct` union field.
9919	fn calleeExpr(
9920	gz: *GenZir,
9921	scope: *Scope,
9922	node: Ast.Node.Index,
9923	) InnerError!Callee {
9924	const astgen = gz.astgen;
9925	const tree = astgen.tree;
9926
9927	const tag = tree.nodes.items(.tag)[node];
9928	switch (tag) {
9929	.field_access => {
9930	const main_tokens = tree.nodes.items(.main_token);
9931	const node_datas = tree.nodes.items(.data);
9932	const object_node = node_datas[node].lhs;
9933	const dot_token = main_tokens[node];
9934	const field_ident = dot_token + 1;
9935	const str_index = try astgen.identAsString(field_ident);
9936	// Capture the object by reference so we can promote it to an
9937	// address in Sema if needed.
9938	const lhs = try expr(gz, scope, .{ .rl = .ref }, object_node);
9939
9940	const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
9941	try emitDbgStmt(gz, cursor);
9942
9943	return .{ .field = .{
9944	.obj_ptr = lhs,
9945	.field_name_start = str_index,
9946	} };
9947	},
9948	else => return .{ .direct = try expr(gz, scope, .{ .rl = .none }, node) },
9949	}
9950	}
9951
9952	const primitive_instrs = std.ComptimeStringMap(Zir.Inst.Ref, .{
9953	.{ "anyerror", .anyerror_type },
9954	.{ "anyframe", .anyframe_type },
9955	.{ "anyopaque", .anyopaque_type },
9956	.{ "bool", .bool_type },
9957	.{ "c_int", .c_int_type },
9958	.{ "c_long", .c_long_type },
9959	.{ "c_longdouble", .c_longdouble_type },
9960	.{ "c_longlong", .c_longlong_type },
9961	.{ "c_char", .c_char_type },
9962	.{ "c_short", .c_short_type },
9963	.{ "c_uint", .c_uint_type },
9964	.{ "c_ulong", .c_ulong_type },
9965	.{ "c_ulonglong", .c_ulonglong_type },
9966	.{ "c_ushort", .c_ushort_type },
9967	.{ "comptime_float", .comptime_float_type },
9968	.{ "comptime_int", .comptime_int_type },
9969	.{ "f128", .f128_type },
9970	.{ "f16", .f16_type },
9971	.{ "f32", .f32_type },
9972	.{ "f64", .f64_type },
9973	.{ "f80", .f80_type },
9974	.{ "false", .bool_false },
9975	.{ "i16", .i16_type },
9976	.{ "i32", .i32_type },
9977	.{ "i64", .i64_type },
9978	.{ "i128", .i128_type },
9979	.{ "i8", .i8_type },
9980	.{ "isize", .isize_type },
9981	.{ "noreturn", .noreturn_type },
9982	.{ "null", .null_value },
9983	.{ "true", .bool_true },
9984	.{ "type", .type_type },
9985	.{ "u16", .u16_type },
9986	.{ "u29", .u29_type },
9987	.{ "u32", .u32_type },
9988	.{ "u64", .u64_type },
9989	.{ "u128", .u128_type },
9990	.{ "u1", .u1_type },
9991	.{ "u8", .u8_type },
9992	.{ "undefined", .undef },
9993	.{ "usize", .usize_type },
9994	.{ "void", .void_type },
9995	});
9996
9997	comptime {
9998	// These checks ensure that std.zig.primitives stays in sync with the primitive->Zir map.
9999	const primitives = std.zig.primitives;
10000	for (primitive_instrs.kvs) \|kv\| {
10001	if (!primitives.isPrimitive(kv.key)) {
10002	@compileError("std.zig.isPrimitive() is not aware of Zir instr '" ++ @tagName(kv.value) ++ "'");
10003	}
10004	}
10005	for (primitives.names.kvs) \|kv\| {
10006	if (primitive_instrs.get(kv.key) == null) {
10007	@compileError("std.zig.primitives entry '" ++ kv.key ++ "' does not have a corresponding Zir instr");
10008	}
10009	}
10010	}
10011
10012	fn nodeIsTriviallyZero(tree: *const Ast, node: Ast.Node.Index) bool {
10013	const node_tags = tree.nodes.items(.tag);
10014	const main_tokens = tree.nodes.items(.main_token);
10015
10016	switch (node_tags[node]) {
10017	.number_literal => {
10018	const ident = main_tokens[node];
10019	return switch (std.zig.parseNumberLiteral(tree.tokenSlice(ident))) {
10020	.int => \|number\| switch (number) {
10021	0 => true,
10022	else => false,
10023	},
10024	else => false,
10025	};
10026	},
10027	else => return false,
10028	}
10029	}
10030
10031	fn nodeMayAppendToErrorTrace(tree: *const Ast, start_node: Ast.Node.Index) bool {
10032	const node_tags = tree.nodes.items(.tag);
10033	const node_datas = tree.nodes.items(.data);
10034
10035	var node = start_node;
10036	while (true) {
10037	switch (node_tags[node]) {
10038	// These don't have the opportunity to call any runtime functions.
10039	.error_value,
10040	.identifier,
10041	.@"comptime",
10042	=> return false,
10043
10044	// Forward the question to the LHS sub-expression.
10045	.grouped_expression,
10046	.@"try",
10047	.@"nosuspend",
10048	.unwrap_optional,
10049	=> node = node_datas[node].lhs,
10050
10051	// Anything that does not eval to an error is guaranteed to pop any
10052	// additions to the error trace, so it effectively does not append.
10053	else => return nodeMayEvalToError(tree, start_node) != .never,
10054	}
10055	}
10056	}
10057
10058	fn nodeMayEvalToError(tree: *const Ast, start_node: Ast.Node.Index) BuiltinFn.EvalToError {
10059	const node_tags = tree.nodes.items(.tag);
10060	const node_datas = tree.nodes.items(.data);
10061	const main_tokens = tree.nodes.items(.main_token);
10062	const token_tags = tree.tokens.items(.tag);
10063
10064	var node = start_node;
10065	while (true) {
10066	switch (node_tags[node]) {
10067	.root,
10068	.@"usingnamespace",
10069	.test_decl,
10070	.switch_case,
10071	.switch_case_inline,
10072	.switch_case_one,
10073	.switch_case_inline_one,
10074	.container_field_init,
10075	.container_field_align,
10076	.container_field,
10077	.asm_output,
10078	.asm_input,
10079	=> unreachable,
10080
10081	.error_value => return .always,
10082
10083	.@"asm",
10084	.asm_simple,
10085	.identifier,
10086	.field_access,
10087	.deref,
10088	.array_access,
10089	.while_simple,
10090	.while_cont,
10091	.for_simple,
10092	.if_simple,
10093	.@"while",
10094	.@"if",
10095	.@"for",
10096	.@"switch",
10097	.switch_comma,
10098	.call_one,
10099	.call_one_comma,
10100	.async_call_one,
10101	.async_call_one_comma,
10102	.call,
10103	.call_comma,
10104	.async_call,
10105	.async_call_comma,
10106	=> return .maybe,
10107
10108	.@"return",
10109	.@"break",
10110	.@"continue",
10111	.bit_not,
10112	.bool_not,
10113	.global_var_decl,
10114	.local_var_decl,
10115	.simple_var_decl,
10116	.aligned_var_decl,
10117	.@"defer",
10118	.@"errdefer",
10119	.address_of,
10120	.optional_type,
10121	.negation,
10122	.negation_wrap,
10123	.@"resume",
10124	.array_type,
10125	.array_type_sentinel,
10126	.ptr_type_aligned,
10127	.ptr_type_sentinel,
10128	.ptr_type,
10129	.ptr_type_bit_range,
10130	.@"suspend",
10131	.fn_proto_simple,
10132	.fn_proto_multi,
10133	.fn_proto_one,
10134	.fn_proto,
10135	.fn_decl,
10136	.anyframe_type,
10137	.anyframe_literal,
10138	.number_literal,
10139	.enum_literal,
10140	.string_literal,
10141	.multiline_string_literal,
10142	.char_literal,
10143	.unreachable_literal,
10144	.error_set_decl,
10145	.container_decl,
10146	.container_decl_trailing,
10147	.container_decl_two,
10148	.container_decl_two_trailing,
10149	.container_decl_arg,
10150	.container_decl_arg_trailing,
10151	.tagged_union,
10152	.tagged_union_trailing,
10153	.tagged_union_two,
10154	.tagged_union_two_trailing,
10155	.tagged_union_enum_tag,
10156	.tagged_union_enum_tag_trailing,
10157	.add,
10158	.add_wrap,
10159	.add_sat,
10160	.array_cat,
10161	.array_mult,
10162	.assign,
10163	.assign_destructure,
10164	.assign_bit_and,
10165	.assign_bit_or,
10166	.assign_shl,
10167	.assign_shl_sat,
10168	.assign_shr,
10169	.assign_bit_xor,
10170	.assign_div,
10171	.assign_sub,
10172	.assign_sub_wrap,
10173	.assign_sub_sat,
10174	.assign_mod,
10175	.assign_add,
10176	.assign_add_wrap,
10177	.assign_add_sat,
10178	.assign_mul,
10179	.assign_mul_wrap,
10180	.assign_mul_sat,
10181	.bang_equal,
10182	.bit_and,
10183	.bit_or,
10184	.shl,
10185	.shl_sat,
10186	.shr,
10187	.bit_xor,
10188	.bool_and,
10189	.bool_or,
10190	.div,
10191	.equal_equal,
10192	.error_union,
10193	.greater_or_equal,
10194	.greater_than,
10195	.less_or_equal,
10196	.less_than,
10197	.merge_error_sets,
10198	.mod,
10199	.mul,
10200	.mul_wrap,
10201	.mul_sat,
10202	.switch_range,
10203	.for_range,
10204	.sub,
10205	.sub_wrap,
10206	.sub_sat,
10207	.slice,
10208	.slice_open,
10209	.slice_sentinel,
10210	.array_init_one,
10211	.array_init_one_comma,
10212	.array_init_dot_two,
10213	.array_init_dot_two_comma,
10214	.array_init_dot,
10215	.array_init_dot_comma,
10216	.array_init,
10217	.array_init_comma,
10218	.struct_init_one,
10219	.struct_init_one_comma,
10220	.struct_init_dot_two,
10221	.struct_init_dot_two_comma,
10222	.struct_init_dot,
10223	.struct_init_dot_comma,
10224	.struct_init,
10225	.struct_init_comma,
10226	=> return .never,
10227
10228	// Forward the question to the LHS sub-expression.
10229	.grouped_expression,
10230	.@"try",
10231	.@"await",
10232	.@"comptime",
10233	.@"nosuspend",
10234	.unwrap_optional,
10235	=> node = node_datas[node].lhs,
10236
10237	// LHS sub-expression may still be an error under the outer optional or error union
10238	.@"catch",
10239	.@"orelse",
10240	=> return .maybe,
10241
10242	.block_two,
10243	.block_two_semicolon,
10244	.block,
10245	.block_semicolon,
10246	=> {
10247	const lbrace = main_tokens[node];
10248	if (token_tags[lbrace - 1] == .colon) {
10249	// Labeled blocks may need a memory location to forward
10250	// to their break statements.
10251	return .maybe;
10252	} else {
10253	return .never;
10254	}
10255	},
10256
10257	.builtin_call,
10258	.builtin_call_comma,
10259	.builtin_call_two,
10260	.builtin_call_two_comma,
10261	=> {
10262	const builtin_token = main_tokens[node];
10263	const builtin_name = tree.tokenSlice(builtin_token);
10264	// If the builtin is an invalid name, we don't cause an error here; instead
10265	// let it pass, and the error will be "invalid builtin function" later.
10266	const builtin_info = BuiltinFn.list.get(builtin_name) orelse return .maybe;
10267	return builtin_info.eval_to_error;
10268	},
10269	}
10270	}
10271	}
10272
10273	/// Returns `true` if it is known the type expression has more than one possible value;
10274	/// `false` otherwise.
10275	fn nodeImpliesMoreThanOnePossibleValue(tree: *const Ast, start_node: Ast.Node.Index) bool {
10276	const node_tags = tree.nodes.items(.tag);
10277	const node_datas = tree.nodes.items(.data);
10278
10279	var node = start_node;
10280	while (true) {
10281	switch (node_tags[node]) {
10282	.root,
10283	.@"usingnamespace",
10284	.test_decl,
10285	.switch_case,
10286	.switch_case_inline,
10287	.switch_case_one,
10288	.switch_case_inline_one,
10289	.container_field_init,
10290	.container_field_align,
10291	.container_field,
10292	.asm_output,
10293	.asm_input,
10294	.global_var_decl,
10295	.local_var_decl,
10296	.simple_var_decl,
10297	.aligned_var_decl,
10298	=> unreachable,
10299
10300	.@"return",
10301	.@"break",
10302	.@"continue",
10303	.bit_not,
10304	.bool_not,
10305	.@"defer",
10306	.@"errdefer",
10307	.address_of,
10308	.negation,
10309	.negation_wrap,
10310	.@"resume",
10311	.array_type,
10312	.@"suspend",
10313	.fn_decl,
10314	.anyframe_literal,
10315	.number_literal,
10316	.enum_literal,
10317	.string_literal,
10318	.multiline_string_literal,
10319	.char_literal,
10320	.unreachable_literal,
10321	.error_set_decl,
10322	.container_decl,
10323	.container_decl_trailing,
10324	.container_decl_two,
10325	.container_decl_two_trailing,
10326	.container_decl_arg,
10327	.container_decl_arg_trailing,
10328	.tagged_union,
10329	.tagged_union_trailing,
10330	.tagged_union_two,
10331	.tagged_union_two_trailing,
10332	.tagged_union_enum_tag,
10333	.tagged_union_enum_tag_trailing,
10334	.@"asm",
10335	.asm_simple,
10336	.add,
10337	.add_wrap,
10338	.add_sat,
10339	.array_cat,
10340	.array_mult,
10341	.assign,
10342	.assign_destructure,
10343	.assign_bit_and,
10344	.assign_bit_or,
10345	.assign_shl,
10346	.assign_shl_sat,
10347	.assign_shr,
10348	.assign_bit_xor,
10349	.assign_div,
10350	.assign_sub,
10351	.assign_sub_wrap,
10352	.assign_sub_sat,
10353	.assign_mod,
10354	.assign_add,
10355	.assign_add_wrap,
10356	.assign_add_sat,
10357	.assign_mul,
10358	.assign_mul_wrap,
10359	.assign_mul_sat,
10360	.bang_equal,
10361	.bit_and,
10362	.bit_or,
10363	.shl,
10364	.shl_sat,
10365	.shr,
10366	.bit_xor,
10367	.bool_and,
10368	.bool_or,
10369	.div,
10370	.equal_equal,
10371	.error_union,
10372	.greater_or_equal,
10373	.greater_than,
10374	.less_or_equal,
10375	.less_than,
10376	.merge_error_sets,
10377	.mod,
10378	.mul,
10379	.mul_wrap,
10380	.mul_sat,
10381	.switch_range,
10382	.for_range,
10383	.field_access,
10384	.sub,
10385	.sub_wrap,
10386	.sub_sat,
10387	.slice,
10388	.slice_open,
10389	.slice_sentinel,
10390	.deref,
10391	.array_access,
10392	.error_value,
10393	.while_simple,
10394	.while_cont,
10395	.for_simple,
10396	.if_simple,
10397	.@"catch",
10398	.@"orelse",
10399	.array_init_one,
10400	.array_init_one_comma,
10401	.array_init_dot_two,
10402	.array_init_dot_two_comma,
10403	.array_init_dot,
10404	.array_init_dot_comma,
10405	.array_init,
10406	.array_init_comma,
10407	.struct_init_one,
10408	.struct_init_one_comma,
10409	.struct_init_dot_two,
10410	.struct_init_dot_two_comma,
10411	.struct_init_dot,
10412	.struct_init_dot_comma,
10413	.struct_init,
10414	.struct_init_comma,
10415	.@"while",
10416	.@"if",
10417	.@"for",
10418	.@"switch",
10419	.switch_comma,
10420	.call_one,
10421	.call_one_comma,
10422	.async_call_one,
10423	.async_call_one_comma,
10424	.call,
10425	.call_comma,
10426	.async_call,
10427	.async_call_comma,
10428	.block_two,
10429	.block_two_semicolon,
10430	.block,
10431	.block_semicolon,
10432	.builtin_call,
10433	.builtin_call_comma,
10434	.builtin_call_two,
10435	.builtin_call_two_comma,
10436	// these are function bodies, not pointers
10437	.fn_proto_simple,
10438	.fn_proto_multi,
10439	.fn_proto_one,
10440	.fn_proto,
10441	=> return false,
10442
10443	// Forward the question to the LHS sub-expression.
10444	.grouped_expression,
10445	.@"try",
10446	.@"await",
10447	.@"comptime",
10448	.@"nosuspend",
10449	.unwrap_optional,
10450	=> node = node_datas[node].lhs,
10451
10452	.ptr_type_aligned,
10453	.ptr_type_sentinel,
10454	.ptr_type,
10455	.ptr_type_bit_range,
10456	.optional_type,
10457	.anyframe_type,
10458	.array_type_sentinel,
10459	=> return true,
10460
10461	.identifier => {
10462	const main_tokens = tree.nodes.items(.main_token);
10463	const ident_bytes = tree.tokenSlice(main_tokens[node]);
10464	if (primitive_instrs.get(ident_bytes)) \|primitive\| switch (primitive) {
10465	.anyerror_type,
10466	.anyframe_type,
10467	.anyopaque_type,
10468	.bool_type,
10469	.c_int_type,
10470	.c_long_type,
10471	.c_longdouble_type,
10472	.c_longlong_type,
10473	.c_char_type,
10474	.c_short_type,
10475	.c_uint_type,
10476	.c_ulong_type,
10477	.c_ulonglong_type,
10478	.c_ushort_type,
10479	.comptime_float_type,
10480	.comptime_int_type,
10481	.f16_type,
10482	.f32_type,
10483	.f64_type,
10484	.f80_type,
10485	.f128_type,
10486	.i16_type,
10487	.i32_type,
10488	.i64_type,
10489	.i128_type,
10490	.i8_type,
10491	.isize_type,
10492	.type_type,
10493	.u16_type,
10494	.u29_type,
10495	.u32_type,
10496	.u64_type,
10497	.u128_type,
10498	.u1_type,
10499	.u8_type,
10500	.usize_type,
10501	=> return true,
10502
10503	.void_type,
10504	.bool_false,
10505	.bool_true,
10506	.null_value,
10507	.undef,
10508	.noreturn_type,
10509	=> return false,
10510
10511	else => unreachable, // that's all the values from `primitives`.
10512	} else {
10513	return false;
10514	}
10515	},
10516	}
10517	}
10518	}
10519
10520	/// Returns `true` if it is known the expression is a type that cannot be used at runtime;
10521	/// `false` otherwise.
10522	fn nodeImpliesComptimeOnly(tree: *const Ast, start_node: Ast.Node.Index) bool {
10523	const node_tags = tree.nodes.items(.tag);
10524	const node_datas = tree.nodes.items(.data);
10525
10526	var node = start_node;
10527	while (true) {
10528	switch (node_tags[node]) {
10529	.root,
10530	.@"usingnamespace",
10531	.test_decl,
10532	.switch_case,
10533	.switch_case_inline,
10534	.switch_case_one,
10535	.switch_case_inline_one,
10536	.container_field_init,
10537	.container_field_align,
10538	.container_field,
10539	.asm_output,
10540	.asm_input,
10541	.global_var_decl,
10542	.local_var_decl,
10543	.simple_var_decl,
10544	.aligned_var_decl,
10545	=> unreachable,
10546
10547	.@"return",
10548	.@"break",
10549	.@"continue",
10550	.bit_not,
10551	.bool_not,
10552	.@"defer",
10553	.@"errdefer",
10554	.address_of,
10555	.negation,
10556	.negation_wrap,
10557	.@"resume",
10558	.array_type,
10559	.@"suspend",
10560	.fn_decl,
10561	.anyframe_literal,
10562	.number_literal,
10563	.enum_literal,
10564	.string_literal,
10565	.multiline_string_literal,
10566	.char_literal,
10567	.unreachable_literal,
10568	.error_set_decl,
10569	.container_decl,
10570	.container_decl_trailing,
10571	.container_decl_two,
10572	.container_decl_two_trailing,
10573	.container_decl_arg,
10574	.container_decl_arg_trailing,
10575	.tagged_union,
10576	.tagged_union_trailing,
10577	.tagged_union_two,
10578	.tagged_union_two_trailing,
10579	.tagged_union_enum_tag,
10580	.tagged_union_enum_tag_trailing,
10581	.@"asm",
10582	.asm_simple,
10583	.add,
10584	.add_wrap,
10585	.add_sat,
10586	.array_cat,
10587	.array_mult,
10588	.assign,
10589	.assign_destructure,
10590	.assign_bit_and,
10591	.assign_bit_or,
10592	.assign_shl,
10593	.assign_shl_sat,
10594	.assign_shr,
10595	.assign_bit_xor,
10596	.assign_div,
10597	.assign_sub,
10598	.assign_sub_wrap,
10599	.assign_sub_sat,
10600	.assign_mod,
10601	.assign_add,
10602	.assign_add_wrap,
10603	.assign_add_sat,
10604	.assign_mul,
10605	.assign_mul_wrap,
10606	.assign_mul_sat,
10607	.bang_equal,
10608	.bit_and,
10609	.bit_or,
10610	.shl,
10611	.shl_sat,
10612	.shr,
10613	.bit_xor,
10614	.bool_and,
10615	.bool_or,
10616	.div,
10617	.equal_equal,
10618	.error_union,
10619	.greater_or_equal,
10620	.greater_than,
10621	.less_or_equal,
10622	.less_than,
10623	.merge_error_sets,
10624	.mod,
10625	.mul,
10626	.mul_wrap,
10627	.mul_sat,
10628	.switch_range,
10629	.for_range,
10630	.field_access,
10631	.sub,
10632	.sub_wrap,
10633	.sub_sat,
10634	.slice,
10635	.slice_open,
10636	.slice_sentinel,
10637	.deref,
10638	.array_access,
10639	.error_value,
10640	.while_simple,
10641	.while_cont,
10642	.for_simple,
10643	.if_simple,
10644	.@"catch",
10645	.@"orelse",
10646	.array_init_one,
10647	.array_init_one_comma,
10648	.array_init_dot_two,
10649	.array_init_dot_two_comma,
10650	.array_init_dot,
10651	.array_init_dot_comma,
10652	.array_init,
10653	.array_init_comma,
10654	.struct_init_one,
10655	.struct_init_one_comma,
10656	.struct_init_dot_two,
10657	.struct_init_dot_two_comma,
10658	.struct_init_dot,
10659	.struct_init_dot_comma,
10660	.struct_init,
10661	.struct_init_comma,
10662	.@"while",
10663	.@"if",
10664	.@"for",
10665	.@"switch",
10666	.switch_comma,
10667	.call_one,
10668	.call_one_comma,
10669	.async_call_one,
10670	.async_call_one_comma,
10671	.call,
10672	.call_comma,
10673	.async_call,
10674	.async_call_comma,
10675	.block_two,
10676	.block_two_semicolon,
10677	.block,
10678	.block_semicolon,
10679	.builtin_call,
10680	.builtin_call_comma,
10681	.builtin_call_two,
10682	.builtin_call_two_comma,
10683	.ptr_type_aligned,
10684	.ptr_type_sentinel,
10685	.ptr_type,
10686	.ptr_type_bit_range,
10687	.optional_type,
10688	.anyframe_type,
10689	.array_type_sentinel,
10690	=> return false,
10691
10692	// these are function bodies, not pointers
10693	.fn_proto_simple,
10694	.fn_proto_multi,
10695	.fn_proto_one,
10696	.fn_proto,
10697	=> return true,
10698
10699	// Forward the question to the LHS sub-expression.
10700	.grouped_expression,
10701	.@"try",
10702	.@"await",
10703	.@"comptime",
10704	.@"nosuspend",
10705	.unwrap_optional,
10706	=> node = node_datas[node].lhs,
10707
10708	.identifier => {
10709	const main_tokens = tree.nodes.items(.main_token);
10710	const ident_bytes = tree.tokenSlice(main_tokens[node]);
10711	if (primitive_instrs.get(ident_bytes)) \|primitive\| switch (primitive) {
10712	.anyerror_type,
10713	.anyframe_type,
10714	.anyopaque_type,
10715	.bool_type,
10716	.c_int_type,
10717	.c_long_type,
10718	.c_longdouble_type,
10719	.c_longlong_type,
10720	.c_char_type,
10721	.c_short_type,
10722	.c_uint_type,
10723	.c_ulong_type,
10724	.c_ulonglong_type,
10725	.c_ushort_type,
10726	.f16_type,
10727	.f32_type,
10728	.f64_type,
10729	.f80_type,
10730	.f128_type,
10731	.i16_type,
10732	.i32_type,
10733	.i64_type,
10734	.i128_type,
10735	.i8_type,
10736	.isize_type,
10737	.u16_type,
10738	.u29_type,
10739	.u32_type,
10740	.u64_type,
10741	.u128_type,
10742	.u1_type,
10743	.u8_type,
10744	.usize_type,
10745	.void_type,
10746	.bool_false,
10747	.bool_true,
10748	.null_value,
10749	.undef,
10750	.noreturn_type,
10751	=> return false,
10752
10753	.comptime_float_type,
10754	.comptime_int_type,
10755	.type_type,
10756	=> return true,
10757
10758	else => unreachable, // that's all the values from `primitives`.
10759	} else {
10760	return false;
10761	}
10762	},
10763	}
10764	}
10765	}
10766
10767	/// Returns `true` if the node uses `gz.anon_name_strategy`.
10768	fn nodeUsesAnonNameStrategy(tree: *const Ast, node: Ast.Node.Index) bool {
10769	const node_tags = tree.nodes.items(.tag);
10770	switch (node_tags[node]) {
10771	.container_decl,
10772	.container_decl_trailing,
10773	.container_decl_two,
10774	.container_decl_two_trailing,
10775	.container_decl_arg,
10776	.container_decl_arg_trailing,
10777	.tagged_union,
10778	.tagged_union_trailing,
10779	.tagged_union_two,
10780	.tagged_union_two_trailing,
10781	.tagged_union_enum_tag,
10782	.tagged_union_enum_tag_trailing,
10783	=> return true,
10784	.builtin_call_two, .builtin_call_two_comma, .builtin_call, .builtin_call_comma => {
10785	const builtin_token = tree.nodes.items(.main_token)[node];
10786	const builtin_name = tree.tokenSlice(builtin_token);
10787	return std.mem.eql(u8, builtin_name, "@Type");
10788	},
10789	else => return false,
10790	}
10791	}
10792
10793	/// Applies `rl` semantics to `result`. Expressions which do not do their own handling of
10794	/// result locations must call this function on their result.
10795	/// As an example, if `ri.rl` is `.ptr`, it will write the result to the pointer.
10796	/// If `ri.rl` is `.ty`, it will coerce the result to the type.
10797	/// Assumes nothing stacked on `gz`.
10798	fn rvalue(
10799	gz: *GenZir,
10800	ri: ResultInfo,
10801	raw_result: Zir.Inst.Ref,
10802	src_node: Ast.Node.Index,
10803	) InnerError!Zir.Inst.Ref {
10804	return rvalueInner(gz, ri, raw_result, src_node, true);
10805	}
10806
10807	/// Like `rvalue`, but refuses to perform coercions before taking references for
10808	/// the `ref_coerced_ty` result type. This is used for local variables which do
10809	/// not have `alloc`s, because we want variables to have consistent addresses,
10810	/// i.e. we want them to act like lvalues.
10811	fn rvalueNoCoercePreRef(
10812	gz: *GenZir,
10813	ri: ResultInfo,
10814	raw_result: Zir.Inst.Ref,
10815	src_node: Ast.Node.Index,
10816	) InnerError!Zir.Inst.Ref {
10817	return rvalueInner(gz, ri, raw_result, src_node, false);
10818	}
10819
10820	fn rvalueInner(
10821	gz: *GenZir,
10822	ri: ResultInfo,
10823	raw_result: Zir.Inst.Ref,
10824	src_node: Ast.Node.Index,
10825	allow_coerce_pre_ref: bool,
10826	) InnerError!Zir.Inst.Ref {
10827	const result = r: {
10828	if (raw_result.toIndex()) \|result_index\| {
10829	const zir_tags = gz.astgen.instructions.items(.tag);
10830	const data = gz.astgen.instructions.items(.data)[@intFromEnum(result_index)];
10831	if (zir_tags[@intFromEnum(result_index)].isAlwaysVoid(data)) {
10832	break :r Zir.Inst.Ref.void_value;
10833	}
10834	}
10835	break :r raw_result;
10836	};
10837	if (gz.endsWithNoReturn()) return result;
10838	switch (ri.rl) {
10839	.none, .coerced_ty => return result,
10840	.discard => {
10841	// Emit a compile error for discarding error values.
10842	_ = try gz.addUnNode(.ensure_result_non_error, result, src_node);
10843	return .void_value;
10844	},
10845	.ref, .ref_coerced_ty => {
10846	const coerced_result = if (allow_coerce_pre_ref and ri.rl == .ref_coerced_ty) res: {
10847	const ptr_ty = ri.rl.ref_coerced_ty;
10848	break :res try gz.addPlNode(.coerce_ptr_elem_ty, src_node, Zir.Inst.Bin{
10849	.lhs = ptr_ty,
10850	.rhs = result,
10851	});
10852	} else result;
10853	// We need a pointer but we have a value.
10854	// Unfortunately it's not quite as simple as directly emitting a ref
10855	// instruction here because we need subsequent address-of operator on
10856	// const locals to return the same address.
10857	const astgen = gz.astgen;
10858	const tree = astgen.tree;
10859	const src_token = tree.firstToken(src_node);
10860	const result_index = coerced_result.toIndex() orelse
10861	return gz.addUnTok(.ref, coerced_result, src_token);
10862	const zir_tags = gz.astgen.instructions.items(.tag);
10863	if (zir_tags[@intFromEnum(result_index)].isParam() or astgen.isInferred(coerced_result))
10864	return gz.addUnTok(.ref, coerced_result, src_token);
10865	const gop = try astgen.ref_table.getOrPut(astgen.gpa, result_index);
10866	if (!gop.found_existing) {
10867	gop.value_ptr.* = try gz.makeUnTok(.ref, coerced_result, src_token);
10868	}
10869	return gop.value_ptr.*.toRef();
10870	},
10871	.ty => \|ty_inst\| {
10872	// Quickly eliminate some common, unnecessary type coercion.
10873	const as_ty = @as(u64, @intFromEnum(Zir.Inst.Ref.type_type)) << 32;
10874	const as_comptime_int = @as(u64, @intFromEnum(Zir.Inst.Ref.comptime_int_type)) << 32;
10875	const as_bool = @as(u64, @intFromEnum(Zir.Inst.Ref.bool_type)) << 32;
10876	const as_usize = @as(u64, @intFromEnum(Zir.Inst.Ref.usize_type)) << 32;
10877	const as_void = @as(u64, @intFromEnum(Zir.Inst.Ref.void_type)) << 32;
10878	switch ((@as(u64, @intFromEnum(ty_inst)) << 32) \| @as(u64, @intFromEnum(result))) {
10879	as_ty \| @intFromEnum(Zir.Inst.Ref.u1_type),
10880	as_ty \| @intFromEnum(Zir.Inst.Ref.u8_type),
10881	as_ty \| @intFromEnum(Zir.Inst.Ref.i8_type),
10882	as_ty \| @intFromEnum(Zir.Inst.Ref.u16_type),
10883	as_ty \| @intFromEnum(Zir.Inst.Ref.u29_type),
10884	as_ty \| @intFromEnum(Zir.Inst.Ref.i16_type),
10885	as_ty \| @intFromEnum(Zir.Inst.Ref.u32_type),
10886	as_ty \| @intFromEnum(Zir.Inst.Ref.i32_type),
10887	as_ty \| @intFromEnum(Zir.Inst.Ref.u64_type),
10888	as_ty \| @intFromEnum(Zir.Inst.Ref.i64_type),
10889	as_ty \| @intFromEnum(Zir.Inst.Ref.u128_type),
10890	as_ty \| @intFromEnum(Zir.Inst.Ref.i128_type),
10891	as_ty \| @intFromEnum(Zir.Inst.Ref.usize_type),
10892	as_ty \| @intFromEnum(Zir.Inst.Ref.isize_type),
10893	as_ty \| @intFromEnum(Zir.Inst.Ref.c_char_type),
10894	as_ty \| @intFromEnum(Zir.Inst.Ref.c_short_type),
10895	as_ty \| @intFromEnum(Zir.Inst.Ref.c_ushort_type),
10896	as_ty \| @intFromEnum(Zir.Inst.Ref.c_int_type),
10897	as_ty \| @intFromEnum(Zir.Inst.Ref.c_uint_type),
10898	as_ty \| @intFromEnum(Zir.Inst.Ref.c_long_type),
10899	as_ty \| @intFromEnum(Zir.Inst.Ref.c_ulong_type),
10900	as_ty \| @intFromEnum(Zir.Inst.Ref.c_longlong_type),
10901	as_ty \| @intFromEnum(Zir.Inst.Ref.c_ulonglong_type),
10902	as_ty \| @intFromEnum(Zir.Inst.Ref.c_longdouble_type),
10903	as_ty \| @intFromEnum(Zir.Inst.Ref.f16_type),
10904	as_ty \| @intFromEnum(Zir.Inst.Ref.f32_type),
10905	as_ty \| @intFromEnum(Zir.Inst.Ref.f64_type),
10906	as_ty \| @intFromEnum(Zir.Inst.Ref.f80_type),
10907	as_ty \| @intFromEnum(Zir.Inst.Ref.f128_type),
10908	as_ty \| @intFromEnum(Zir.Inst.Ref.anyopaque_type),
10909	as_ty \| @intFromEnum(Zir.Inst.Ref.bool_type),
10910	as_ty \| @intFromEnum(Zir.Inst.Ref.void_type),
10911	as_ty \| @intFromEnum(Zir.Inst.Ref.type_type),
10912	as_ty \| @intFromEnum(Zir.Inst.Ref.anyerror_type),
10913	as_ty \| @intFromEnum(Zir.Inst.Ref.comptime_int_type),
10914	as_ty \| @intFromEnum(Zir.Inst.Ref.comptime_float_type),
10915	as_ty \| @intFromEnum(Zir.Inst.Ref.noreturn_type),
10916	as_ty \| @intFromEnum(Zir.Inst.Ref.anyframe_type),
10917	as_ty \| @intFromEnum(Zir.Inst.Ref.null_type),
10918	as_ty \| @intFromEnum(Zir.Inst.Ref.undefined_type),
10919	as_ty \| @intFromEnum(Zir.Inst.Ref.enum_literal_type),
10920	as_ty \| @intFromEnum(Zir.Inst.Ref.atomic_order_type),
10921	as_ty \| @intFromEnum(Zir.Inst.Ref.atomic_rmw_op_type),
10922	as_ty \| @intFromEnum(Zir.Inst.Ref.calling_convention_type),
10923	as_ty \| @intFromEnum(Zir.Inst.Ref.address_space_type),
10924	as_ty \| @intFromEnum(Zir.Inst.Ref.float_mode_type),
10925	as_ty \| @intFromEnum(Zir.Inst.Ref.reduce_op_type),
10926	as_ty \| @intFromEnum(Zir.Inst.Ref.call_modifier_type),
10927	as_ty \| @intFromEnum(Zir.Inst.Ref.prefetch_options_type),
10928	as_ty \| @intFromEnum(Zir.Inst.Ref.export_options_type),
10929	as_ty \| @intFromEnum(Zir.Inst.Ref.extern_options_type),
10930	as_ty \| @intFromEnum(Zir.Inst.Ref.type_info_type),
10931	as_ty \| @intFromEnum(Zir.Inst.Ref.manyptr_u8_type),
10932	as_ty \| @intFromEnum(Zir.Inst.Ref.manyptr_const_u8_type),
10933	as_ty \| @intFromEnum(Zir.Inst.Ref.manyptr_const_u8_sentinel_0_type),
10934	as_ty \| @intFromEnum(Zir.Inst.Ref.single_const_pointer_to_comptime_int_type),
10935	as_ty \| @intFromEnum(Zir.Inst.Ref.slice_const_u8_type),
10936	as_ty \| @intFromEnum(Zir.Inst.Ref.slice_const_u8_sentinel_0_type),
10937	as_ty \| @intFromEnum(Zir.Inst.Ref.anyerror_void_error_union_type),
10938	as_ty \| @intFromEnum(Zir.Inst.Ref.generic_poison_type),
10939	as_ty \| @intFromEnum(Zir.Inst.Ref.empty_struct_type),
10940	as_comptime_int \| @intFromEnum(Zir.Inst.Ref.zero),
10941	as_comptime_int \| @intFromEnum(Zir.Inst.Ref.one),
10942	as_bool \| @intFromEnum(Zir.Inst.Ref.bool_true),
10943	as_bool \| @intFromEnum(Zir.Inst.Ref.bool_false),
10944	as_usize \| @intFromEnum(Zir.Inst.Ref.zero_usize),
10945	as_usize \| @intFromEnum(Zir.Inst.Ref.one_usize),
10946	as_void \| @intFromEnum(Zir.Inst.Ref.void_value),
10947	=> return result, // type of result is already correct
10948
10949	// Need an explicit type coercion instruction.
10950	else => return gz.addPlNode(ri.zirTag(), src_node, Zir.Inst.As{
10951	.dest_type = ty_inst,
10952	.operand = result,
10953	}),
10954	}
10955	},
10956	.ptr => \|ptr_res\| {
10957	_ = try gz.addPlNode(.store_node, ptr_res.src_node orelse src_node, Zir.Inst.Bin{
10958	.lhs = ptr_res.inst,
10959	.rhs = result,
10960	});
10961	return .void_value;
10962	},
10963	.inferred_ptr => \|alloc\| {
10964	_ = try gz.addPlNode(.store_to_inferred_ptr, src_node, Zir.Inst.Bin{
10965	.lhs = alloc,
10966	.rhs = result,
10967	});
10968	return .void_value;
10969	},
10970	.destructure => \|destructure\| {
10971	const components = destructure.components;
10972	_ = try gz.addPlNode(.validate_destructure, src_node, Zir.Inst.ValidateDestructure{
10973	.operand = result,
10974	.destructure_node = gz.nodeIndexToRelative(destructure.src_node),
10975	.expect_len = @intCast(components.len),
10976	});
10977	for (components, 0..) \|component, i\| {
10978	if (component == .discard) continue;
10979	const elem_val = try gz.add(.{
10980	.tag = .elem_val_imm,
10981	.data = .{ .elem_val_imm = .{
10982	.operand = result,
10983	.idx = @intCast(i),
10984	} },
10985	});
10986	switch (component) {
10987	.typed_ptr => \|ptr_res\| {
10988	_ = try gz.addPlNode(.store_node, ptr_res.src_node orelse src_node, Zir.Inst.Bin{
10989	.lhs = ptr_res.inst,
10990	.rhs = elem_val,
10991	});
10992	},
10993	.inferred_ptr => \|ptr_inst\| {
10994	_ = try gz.addPlNode(.store_to_inferred_ptr, src_node, Zir.Inst.Bin{
10995	.lhs = ptr_inst,
10996	.rhs = elem_val,
10997	});
10998	},
10999	.discard => unreachable,
11000	}
11001	}
11002	return .void_value;
11003	},
11004	}
11005	}
11006
11007	/// Given an identifier token, obtain the string for it.
11008	/// If the token uses @"" syntax, parses as a string, reports errors if applicable,
11009	/// and allocates the result within `astgen.arena`.
11010	/// Otherwise, returns a reference to the source code bytes directly.
11011	/// See also `appendIdentStr` and `parseStrLit`.
11012	fn identifierTokenString(astgen: *AstGen, token: Ast.TokenIndex) InnerError![]const u8 {
11013	const tree = astgen.tree;
11014	const token_tags = tree.tokens.items(.tag);
11015	assert(token_tags[token] == .identifier);
11016	const ident_name = tree.tokenSlice(token);
11017	if (!mem.startsWith(u8, ident_name, "@")) {
11018	return ident_name;
11019	}
11020	var buf: ArrayListUnmanaged(u8) = .{};
11021	defer buf.deinit(astgen.gpa);
11022	try astgen.parseStrLit(token, &buf, ident_name, 1);
11023	if (mem.indexOfScalar(u8, buf.items, 0) != null) {
11024	return astgen.failTok(token, "identifier cannot contain null bytes", .{});
11025	} else if (buf.items.len == 0) {
11026	return astgen.failTok(token, "identifier cannot be empty", .{});
11027	}
11028	const duped = try astgen.arena.dupe(u8, buf.items);
11029	return duped;
11030	}
11031
11032	/// Given an identifier token, obtain the string for it (possibly parsing as a string
11033	/// literal if it is @"" syntax), and append the string to `buf`.
11034	/// See also `identifierTokenString` and `parseStrLit`.
11035	fn appendIdentStr(
11036	astgen: *AstGen,
11037	token: Ast.TokenIndex,
11038	buf: *ArrayListUnmanaged(u8),
11039	) InnerError!void {
11040	const tree = astgen.tree;
11041	const token_tags = tree.tokens.items(.tag);
11042	assert(token_tags[token] == .identifier);
11043	const ident_name = tree.tokenSlice(token);
11044	if (!mem.startsWith(u8, ident_name, "@")) {
11045	return buf.appendSlice(astgen.gpa, ident_name);
11046	} else {
11047	const start = buf.items.len;
11048	try astgen.parseStrLit(token, buf, ident_name, 1);
11049	const slice = buf.items[start..];
11050	if (mem.indexOfScalar(u8, slice, 0) != null) {
11051	return astgen.failTok(token, "identifier cannot contain null bytes", .{});
11052	} else if (slice.len == 0) {
11053	return astgen.failTok(token, "identifier cannot be empty", .{});
11054	}
11055	}
11056	}
11057
11058	/// Appends the result to `buf`.
11059	fn parseStrLit(
11060	astgen: *AstGen,
11061	token: Ast.TokenIndex,
11062	buf: *ArrayListUnmanaged(u8),
11063	bytes: []const u8,
11064	offset: u32,
11065	) InnerError!void {
11066	const raw_string = bytes[offset..];
11067	var buf_managed = buf.toManaged(astgen.gpa);
11068	const result = std.zig.string_literal.parseWrite(buf_managed.writer(), raw_string);
11069	buf.* = buf_managed.moveToUnmanaged();
11070	switch (try result) {
11071	.success => return,
11072	.failure => \|err\| return astgen.failWithStrLitError(err, token, bytes, offset),
11073	}
11074	}
11075
11076	fn failWithStrLitError(astgen: *AstGen, err: std.zig.string_literal.Error, token: Ast.TokenIndex, bytes: []const u8, offset: u32) InnerError {
11077	const raw_string = bytes[offset..];
11078	switch (err) {
11079	.invalid_escape_character => \|bad_index\| {
11080	return astgen.failOff(
11081	token,
11082	offset + @as(u32, @intCast(bad_index)),
11083	"invalid escape character: '{c}'",
11084	.{raw_string[bad_index]},
11085	);
11086	},
11087	.expected_hex_digit => \|bad_index\| {
11088	return astgen.failOff(
11089	token,
11090	offset + @as(u32, @intCast(bad_index)),
11091	"expected hex digit, found '{c}'",
11092	.{raw_string[bad_index]},
11093	);
11094	},
11095	.empty_unicode_escape_sequence => \|bad_index\| {
11096	return astgen.failOff(
11097	token,
11098	offset + @as(u32, @intCast(bad_index)),
11099	"empty unicode escape sequence",
11100	.{},
11101	);
11102	},
11103	.expected_hex_digit_or_rbrace => \|bad_index\| {
11104	return astgen.failOff(
11105	token,
11106	offset + @as(u32, @intCast(bad_index)),
11107	"expected hex digit or '}}', found '{c}'",
11108	.{raw_string[bad_index]},
11109	);
11110	},
11111	.invalid_unicode_codepoint => \|bad_index\| {
11112	return astgen.failOff(
11113	token,
11114	offset + @as(u32, @intCast(bad_index)),
11115	"unicode escape does not correspond to a valid codepoint",
11116	.{},
11117	);
11118	},
11119	.expected_lbrace => \|bad_index\| {
11120	return astgen.failOff(
11121	token,
11122	offset + @as(u32, @intCast(bad_index)),
11123	"expected '{{', found '{c}",
11124	.{raw_string[bad_index]},
11125	);
11126	},
11127	.expected_rbrace => \|bad_index\| {
11128	return astgen.failOff(
11129	token,
11130	offset + @as(u32, @intCast(bad_index)),
11131	"expected '}}', found '{c}",
11132	.{raw_string[bad_index]},
11133	);
11134	},
11135	.expected_single_quote => \|bad_index\| {
11136	return astgen.failOff(
11137	token,
11138	offset + @as(u32, @intCast(bad_index)),
11139	"expected single quote ('), found '{c}",
11140	.{raw_string[bad_index]},
11141	);
11142	},
11143	.invalid_character => \|bad_index\| {
11144	return astgen.failOff(
11145	token,
11146	offset + @as(u32, @intCast(bad_index)),
11147	"invalid byte in string or character literal: '{c}'",
11148	.{raw_string[bad_index]},
11149	);
11150	},
11151	}
11152	}
11153
11154	fn failNode(
11155	astgen: *AstGen,
11156	node: Ast.Node.Index,
11157	comptime format: []const u8,
11158	args: anytype,
11159	) InnerError {
11160	return astgen.failNodeNotes(node, format, args, &[0]u32{});
11161	}
11162
11163	fn appendErrorNode(
11164	astgen: *AstGen,
11165	node: Ast.Node.Index,
11166	comptime format: []const u8,
11167	args: anytype,
11168	) Allocator.Error!void {
11169	try astgen.appendErrorNodeNotes(node, format, args, &[0]u32{});
11170	}
11171
11172	fn appendErrorNodeNotes(
11173	astgen: *AstGen,
11174	node: Ast.Node.Index,
11175	comptime format: []const u8,
11176	args: anytype,
11177	notes: []const u32,
11178	) Allocator.Error!void {
11179	@setCold(true);
11180	const string_bytes = &astgen.string_bytes;
11181	const msg: Zir.NullTerminatedString = @enumFromInt(string_bytes.items.len);
11182	try string_bytes.writer(astgen.gpa).print(format ++ "\x00", args);
11183	const notes_index: u32 = if (notes.len != 0) blk: {
11184	const notes_start = astgen.extra.items.len;
11185	try astgen.extra.ensureTotalCapacity(astgen.gpa, notes_start + 1 + notes.len);
11186	astgen.extra.appendAssumeCapacity(@intCast(notes.len));
11187	astgen.extra.appendSliceAssumeCapacity(notes);
11188	break :blk @intCast(notes_start);
11189	} else 0;
11190	try astgen.compile_errors.append(astgen.gpa, .{
11191	.msg = msg,
11192	.node = node,
11193	.token = 0,
11194	.byte_offset = 0,
11195	.notes = notes_index,
11196	});
11197	}
11198
11199	fn failNodeNotes(
11200	astgen: *AstGen,
11201	node: Ast.Node.Index,
11202	comptime format: []const u8,
11203	args: anytype,
11204	notes: []const u32,
11205	) InnerError {
11206	try appendErrorNodeNotes(astgen, node, format, args, notes);
11207	return error.AnalysisFail;
11208	}
11209
11210	fn failTok(
11211	astgen: *AstGen,
11212	token: Ast.TokenIndex,
11213	comptime format: []const u8,
11214	args: anytype,
11215	) InnerError {
11216	return astgen.failTokNotes(token, format, args, &[0]u32{});
11217	}
11218
11219	fn appendErrorTok(
11220	astgen: *AstGen,
11221	token: Ast.TokenIndex,
11222	comptime format: []const u8,
11223	args: anytype,
11224	) !void {
11225	try astgen.appendErrorTokNotesOff(token, 0, format, args, &[0]u32{});
11226	}
11227
11228	fn failTokNotes(
11229	astgen: *AstGen,
11230	token: Ast.TokenIndex,
11231	comptime format: []const u8,
11232	args: anytype,
11233	notes: []const u32,
11234	) InnerError {
11235	try appendErrorTokNotesOff(astgen, token, 0, format, args, notes);
11236	return error.AnalysisFail;
11237	}
11238
11239	fn appendErrorTokNotes(
11240	astgen: *AstGen,
11241	token: Ast.TokenIndex,
11242	comptime format: []const u8,
11243	args: anytype,
11244	notes: []const u32,
11245	) !void {
11246	return appendErrorTokNotesOff(astgen, token, 0, format, args, notes);
11247	}
11248
11249	/// Same as `fail`, except given a token plus an offset from its starting byte
11250	/// offset.
11251	fn failOff(
11252	astgen: *AstGen,
11253	token: Ast.TokenIndex,
11254	byte_offset: u32,
11255	comptime format: []const u8,
11256	args: anytype,
11257	) InnerError {
11258	try appendErrorTokNotesOff(astgen, token, byte_offset, format, args, &.{});
11259	return error.AnalysisFail;
11260	}
11261
11262	fn appendErrorTokNotesOff(
11263	astgen: *AstGen,
11264	token: Ast.TokenIndex,
11265	byte_offset: u32,
11266	comptime format: []const u8,
11267	args: anytype,
11268	notes: []const u32,
11269	) !void {
11270	@setCold(true);
11271	const gpa = astgen.gpa;
11272	const string_bytes = &astgen.string_bytes;
11273	const msg: Zir.NullTerminatedString = @enumFromInt(string_bytes.items.len);
11274	try string_bytes.writer(gpa).print(format ++ "\x00", args);
11275	const notes_index: u32 = if (notes.len != 0) blk: {
11276	const notes_start = astgen.extra.items.len;
11277	try astgen.extra.ensureTotalCapacity(gpa, notes_start + 1 + notes.len);
11278	astgen.extra.appendAssumeCapacity(@intCast(notes.len));
11279	astgen.extra.appendSliceAssumeCapacity(notes);
11280	break :blk @intCast(notes_start);
11281	} else 0;
11282	try astgen.compile_errors.append(gpa, .{
11283	.msg = msg,
11284	.node = 0,
11285	.token = token,
11286	.byte_offset = byte_offset,
11287	.notes = notes_index,
11288	});
11289	}
11290
11291	fn errNoteTok(
11292	astgen: *AstGen,
11293	token: Ast.TokenIndex,
11294	comptime format: []const u8,
11295	args: anytype,
11296	) Allocator.Error!u32 {
11297	return errNoteTokOff(astgen, token, 0, format, args);
11298	}
11299
11300	fn errNoteTokOff(
11301	astgen: *AstGen,
11302	token: Ast.TokenIndex,
11303	byte_offset: u32,
11304	comptime format: []const u8,
11305	args: anytype,
11306	) Allocator.Error!u32 {
11307	@setCold(true);
11308	const string_bytes = &astgen.string_bytes;
11309	const msg: Zir.NullTerminatedString = @enumFromInt(string_bytes.items.len);
11310	try string_bytes.writer(astgen.gpa).print(format ++ "\x00", args);
11311	return astgen.addExtra(Zir.Inst.CompileErrors.Item{
11312	.msg = msg,
11313	.node = 0,
11314	.token = token,
11315	.byte_offset = byte_offset,
11316	.notes = 0,
11317	});
11318	}
11319
11320	fn errNoteNode(
11321	astgen: *AstGen,
11322	node: Ast.Node.Index,
11323	comptime format: []const u8,
11324	args: anytype,
11325	) Allocator.Error!u32 {
11326	@setCold(true);
11327	const string_bytes = &astgen.string_bytes;
11328	const msg: Zir.NullTerminatedString = @enumFromInt(string_bytes.items.len);
11329	try string_bytes.writer(astgen.gpa).print(format ++ "\x00", args);
11330	return astgen.addExtra(Zir.Inst.CompileErrors.Item{
11331	.msg = msg,
11332	.node = node,
11333	.token = 0,
11334	.byte_offset = 0,
11335	.notes = 0,
11336	});
11337	}
11338
11339	fn identAsString(astgen: *AstGen, ident_token: Ast.TokenIndex) !Zir.NullTerminatedString {
11340	const gpa = astgen.gpa;
11341	const string_bytes = &astgen.string_bytes;
11342	const str_index: u32 = @intCast(string_bytes.items.len);
11343	try astgen.appendIdentStr(ident_token, string_bytes);
11344	const key: []const u8 = string_bytes.items[str_index..];
11345	const gop = try astgen.string_table.getOrPutContextAdapted(gpa, key, StringIndexAdapter{
11346	.bytes = string_bytes,
11347	}, StringIndexContext{
11348	.bytes = string_bytes,
11349	});
11350	if (gop.found_existing) {
11351	string_bytes.shrinkRetainingCapacity(str_index);
11352	return @enumFromInt(gop.key_ptr.*);
11353	} else {
11354	gop.key_ptr.* = str_index;
11355	try string_bytes.append(gpa, 0);
11356	return @enumFromInt(str_index);
11357	}
11358	}
11359
11360	/// Adds a doc comment block to `string_bytes` by walking backwards from `end_token`.
11361	/// `end_token` must point at the first token after the last doc coment line.
11362	/// Returns 0 if no doc comment is present.
11363	fn docCommentAsString(astgen: *AstGen, end_token: Ast.TokenIndex) !Zir.NullTerminatedString {
11364	if (end_token == 0) return .empty;
11365
11366	const token_tags = astgen.tree.tokens.items(.tag);
11367
11368	var tok = end_token - 1;
11369	while (token_tags[tok] == .doc_comment) {
11370	if (tok == 0) break;
11371	tok -= 1;
11372	} else {
11373	tok += 1;
11374	}
11375
11376	return docCommentAsStringFromFirst(astgen, end_token, tok);
11377	}
11378
11379	/// end_token must be > the index of the last doc comment.
11380	fn docCommentAsStringFromFirst(
11381	astgen: *AstGen,
11382	end_token: Ast.TokenIndex,
11383	start_token: Ast.TokenIndex,
11384	) !Zir.NullTerminatedString {
11385	if (start_token == end_token) return .empty;
11386
11387	const gpa = astgen.gpa;
11388	const string_bytes = &astgen.string_bytes;
11389	const str_index: u32 = @intCast(string_bytes.items.len);
11390	const token_starts = astgen.tree.tokens.items(.start);
11391	const token_tags = astgen.tree.tokens.items(.tag);
11392
11393	const total_bytes = token_starts[end_token] - token_starts[start_token];
11394	try string_bytes.ensureUnusedCapacity(gpa, total_bytes);
11395
11396	var current_token = start_token;
11397	while (current_token < end_token) : (current_token += 1) {
11398	switch (token_tags[current_token]) {
11399	.doc_comment => {
11400	const tok_bytes = astgen.tree.tokenSlice(current_token)[3..];
11401	string_bytes.appendSliceAssumeCapacity(tok_bytes);
11402	if (current_token != end_token - 1) {
11403	string_bytes.appendAssumeCapacity('\n');
11404	}
11405	},
11406	else => break,
11407	}
11408	}
11409
11410	const key: []const u8 = string_bytes.items[str_index..];
11411	const gop = try astgen.string_table.getOrPutContextAdapted(gpa, key, StringIndexAdapter{
11412	.bytes = string_bytes,
11413	}, StringIndexContext{
11414	.bytes = string_bytes,
11415	});
11416
11417	if (gop.found_existing) {
11418	string_bytes.shrinkRetainingCapacity(str_index);
11419	return @enumFromInt(gop.key_ptr.*);
11420	} else {
11421	gop.key_ptr.* = str_index;
11422	try string_bytes.append(gpa, 0);
11423	return @enumFromInt(str_index);
11424	}
11425	}
11426
11427	const IndexSlice = struct { index: Zir.NullTerminatedString, len: u32 };
11428
11429	fn strLitAsString(astgen: *AstGen, str_lit_token: Ast.TokenIndex) !IndexSlice {
11430	const gpa = astgen.gpa;
11431	const string_bytes = &astgen.string_bytes;
11432	const str_index: u32 = @intCast(string_bytes.items.len);
11433	const token_bytes = astgen.tree.tokenSlice(str_lit_token);
11434	try astgen.parseStrLit(str_lit_token, string_bytes, token_bytes, 0);
11435	const key: []const u8 = string_bytes.items[str_index..];
11436	if (std.mem.indexOfScalar(u8, key, 0)) \|_\| return .{
11437	.index = @enumFromInt(str_index),
11438	.len = @intCast(key.len),
11439	};
11440	const gop = try astgen.string_table.getOrPutContextAdapted(gpa, key, StringIndexAdapter{
11441	.bytes = string_bytes,
11442	}, StringIndexContext{
11443	.bytes = string_bytes,
11444	});
11445	if (gop.found_existing) {
11446	string_bytes.shrinkRetainingCapacity(str_index);
11447	return .{
11448	.index = @enumFromInt(gop.key_ptr.*),
11449	.len = @intCast(key.len),
11450	};
11451	} else {
11452	gop.key_ptr.* = str_index;
11453	// Still need a null byte because we are using the same table
11454	// to lookup null terminated strings, so if we get a match, it has to
11455	// be null terminated for that to work.
11456	try string_bytes.append(gpa, 0);
11457	return .{
11458	.index = @enumFromInt(str_index),
11459	.len = @intCast(key.len),
11460	};
11461	}
11462	}
11463
11464	fn strLitNodeAsString(astgen: *AstGen, node: Ast.Node.Index) !IndexSlice {
11465	const tree = astgen.tree;
11466	const node_datas = tree.nodes.items(.data);
11467
11468	const start = node_datas[node].lhs;
11469	const end = node_datas[node].rhs;
11470
11471	const gpa = astgen.gpa;
11472	const string_bytes = &astgen.string_bytes;
11473	const str_index = string_bytes.items.len;
11474
11475	// First line: do not append a newline.
11476	var tok_i = start;
11477	{
11478	const slice = tree.tokenSlice(tok_i);
11479	const carriage_return_ending: usize = if (slice[slice.len - 2] == '\r') 2 else 1;
11480	const line_bytes = slice[2 .. slice.len - carriage_return_ending];
11481	try string_bytes.appendSlice(gpa, line_bytes);
11482	tok_i += 1;
11483	}
11484	// Following lines: each line prepends a newline.
11485	while (tok_i <= end) : (tok_i += 1) {
11486	const slice = tree.tokenSlice(tok_i);
11487	const carriage_return_ending: usize = if (slice[slice.len - 2] == '\r') 2 else 1;
11488	const line_bytes = slice[2 .. slice.len - carriage_return_ending];
11489	try string_bytes.ensureUnusedCapacity(gpa, line_bytes.len + 1);
11490	string_bytes.appendAssumeCapacity('\n');
11491	string_bytes.appendSliceAssumeCapacity(line_bytes);
11492	}
11493	const len = string_bytes.items.len - str_index;
11494	try string_bytes.append(gpa, 0);
11495	return IndexSlice{
11496	.index = @enumFromInt(str_index),
11497	.len = @intCast(len),
11498	};
11499	}
11500
11501	fn testNameString(astgen: *AstGen, str_lit_token: Ast.TokenIndex) !Zir.NullTerminatedString {
11502	const gpa = astgen.gpa;
11503	const string_bytes = &astgen.string_bytes;
11504	const str_index: u32 = @intCast(string_bytes.items.len);
11505	const token_bytes = astgen.tree.tokenSlice(str_lit_token);
11506	try string_bytes.append(gpa, 0); // Indicates this is a test.
11507	try astgen.parseStrLit(str_lit_token, string_bytes, token_bytes, 0);
11508	const slice = string_bytes.items[str_index + 1 ..];
11509	if (mem.indexOfScalar(u8, slice, 0) != null) {
11510	return astgen.failTok(str_lit_token, "test name cannot contain null bytes", .{});
11511	} else if (slice.len == 0) {
11512	return astgen.failTok(str_lit_token, "empty test name must be omitted", .{});
11513	}
11514	try string_bytes.append(gpa, 0);
11515	return @enumFromInt(str_index);
11516	}
11517
11518	const Scope = struct {
11519	tag: Tag,
11520
11521	fn cast(base: Scope, comptime T: type) ?T {
11522	if (T == Defer) {
11523	switch (base.tag) {
11524	.defer_normal, .defer_error => return @fieldParentPtr(T, "base", base),
11525	else => return null,
11526	}
11527	}
11528	if (T == Namespace) {
11529	switch (base.tag) {
11530	.namespace, .enum_namespace => return @fieldParentPtr(T, "base", base),
11531	else => return null,
11532	}
11533	}
11534	if (base.tag != T.base_tag)
11535	return null;
11536
11537	return @fieldParentPtr(T, "base", base);
11538	}
11539
11540	fn parent(base: Scope) ?Scope {
11541	return switch (base.tag) {
11542	.gen_zir => base.cast(GenZir).?.parent,
11543	.local_val => base.cast(LocalVal).?.parent,
11544	.local_ptr => base.cast(LocalPtr).?.parent,
11545	.defer_normal, .defer_error => base.cast(Defer).?.parent,
11546	.namespace, .enum_namespace => base.cast(Namespace).?.parent,
11547	.top => null,
11548	};
11549	}
11550
11551	const Tag = enum {
11552	gen_zir,
11553	local_val,
11554	local_ptr,
11555	defer_normal,
11556	defer_error,
11557	namespace,
11558	enum_namespace,
11559	top,
11560	};
11561
11562	/// The category of identifier. These tag names are user-visible in compile errors.
11563	const IdCat = enum {
11564	@"function parameter",
11565	@"local constant",
11566	@"local variable",
11567	@"switch tag capture",
11568	capture,
11569	};
11570
11571	/// This is always a `const` local and importantly the `inst` is a value type, not a pointer.
11572	/// This structure lives as long as the AST generation of the Block
11573	/// node that contains the variable.
11574	const LocalVal = struct {
11575	const base_tag: Tag = .local_val;
11576	base: Scope = Scope{ .tag = base_tag },
11577	/// Parents can be: `LocalVal`, `LocalPtr`, `GenZir`, `Defer`, `Namespace`.
11578	parent: *Scope,
11579	gen_zir: *GenZir,
11580	inst: Zir.Inst.Ref,
11581	/// Source location of the corresponding variable declaration.
11582	token_src: Ast.TokenIndex,
11583	/// Track the first identifer where it is referenced.
11584	/// 0 means never referenced.
11585	used: Ast.TokenIndex = 0,
11586	/// Track the identifier where it is discarded, like this `_ = foo;`.
11587	/// 0 means never discarded.
11588	discarded: Ast.TokenIndex = 0,
11589	/// String table index.
11590	name: Zir.NullTerminatedString,
11591	id_cat: IdCat,
11592	};
11593
11594	/// This could be a `const` or `var` local. It has a pointer instead of a value.
11595	/// This structure lives as long as the AST generation of the Block
11596	/// node that contains the variable.
11597	const LocalPtr = struct {
11598	const base_tag: Tag = .local_ptr;
11599	base: Scope = Scope{ .tag = base_tag },
11600	/// Parents can be: `LocalVal`, `LocalPtr`, `GenZir`, `Defer`, `Namespace`.
11601	parent: *Scope,
11602	gen_zir: *GenZir,
11603	ptr: Zir.Inst.Ref,
11604	/// Source location of the corresponding variable declaration.
11605	token_src: Ast.TokenIndex,
11606	/// Track the first identifer where it is referenced.
11607	/// 0 means never referenced.
11608	used: Ast.TokenIndex = 0,
11609	/// Track the identifier where it is discarded, like this `_ = foo;`.
11610	/// 0 means never discarded.
11611	discarded: Ast.TokenIndex = 0,
11612	/// Whether this value is used as an lvalue after inititialization.
11613	/// If not, we know it can be `const`, so will emit a compile error if it is `var`.
11614	used_as_lvalue: bool = false,
11615	/// String table index.
11616	name: Zir.NullTerminatedString,
11617	id_cat: IdCat,
11618	/// true means we find out during Sema whether the value is comptime.
11619	/// false means it is already known at AstGen the value is runtime-known.
11620	maybe_comptime: bool,
11621	};
11622
11623	const Defer = struct {
11624	base: Scope,
11625	/// Parents can be: `LocalVal`, `LocalPtr`, `GenZir`, `Defer`, `Namespace`.
11626	parent: *Scope,
11627	index: u32,
11628	len: u32,
11629	remapped_err_code: Zir.Inst.OptionalIndex = .none,
11630	};
11631
11632	/// Represents a global scope that has any number of declarations in it.
11633	/// Each declaration has this as the parent scope.
11634	const Namespace = struct {
11635	const base_tag: Tag = .namespace;
11636	base: Scope = Scope{ .tag = base_tag },
11637
11638	/// Parents can be: `LocalVal`, `LocalPtr`, `GenZir`, `Defer`, `Namespace`.
11639	parent: *Scope,
11640	/// Maps string table index to the source location of declaration,
11641	/// for the purposes of reporting name shadowing compile errors.
11642	decls: std.AutoHashMapUnmanaged(Zir.NullTerminatedString, Ast.Node.Index) = .{},
11643	node: Ast.Node.Index,
11644	inst: Zir.Inst.Index,
11645
11646	/// The astgen scope containing this namespace.
11647	/// Only valid during astgen.
11648	declaring_gz: ?*GenZir,
11649
11650	/// Map from the raw captured value to the instruction
11651	/// ref of the capture for decls in this namespace
11652	captures: std.AutoArrayHashMapUnmanaged(Zir.Inst.Index, Zir.Inst.Index) = .{},
11653
11654	fn deinit(self: *Namespace, gpa: Allocator) void {
11655	self.decls.deinit(gpa);
11656	self.captures.deinit(gpa);
11657	self.* = undefined;
11658	}
11659	};
11660
11661	const Top = struct {
11662	const base_tag: Scope.Tag = .top;
11663	base: Scope = Scope{ .tag = base_tag },
11664	};
11665	};
11666
11667	/// This is a temporary structure; references to it are valid only
11668	/// while constructing a `Zir`.
11669	const GenZir = struct {
11670	const base_tag: Scope.Tag = .gen_zir;
11671	base: Scope = Scope{ .tag = base_tag },
11672	/// Whether we're already in a scope known to be comptime. This is set
11673	/// whenever we know Sema will analyze the current block with `is_comptime`,
11674	/// for instance when we're within a `struct_decl` or a `block_comptime`.
11675	is_comptime: bool,
11676	/// Whether we're in an expression within a `@TypeOf` operand. In this case, closure of runtime
11677	/// variables is permitted where it is usually not.
11678	is_typeof: bool = false,
11679	/// This is set to true for inline loops; false otherwise.
11680	is_inline: bool = false,
11681	c_import: bool = false,
11682	/// How decls created in this scope should be named.
11683	anon_name_strategy: Zir.Inst.NameStrategy = .anon,
11684	/// The containing decl AST node.
11685	decl_node_index: Ast.Node.Index,
11686	/// The containing decl line index, absolute.
11687	decl_line: u32,
11688	/// Parents can be: `LocalVal`, `LocalPtr`, `GenZir`, `Defer`, `Namespace`.
11689	parent: *Scope,
11690	/// All `GenZir` scopes for the same ZIR share this.
11691	astgen: *AstGen,
11692	/// Keeps track of the list of instructions in this scope. Possibly shared.
11693	/// Indexes to instructions in `astgen`.
11694	instructions: *ArrayListUnmanaged(Zir.Inst.Index),
11695	/// A sub-block may share its instructions ArrayList with containing GenZir,
11696	/// if use is strictly nested. This saves prior size of list for unstacking.
11697	instructions_top: usize,
11698	label: ?Label = null,
11699	break_block: Zir.Inst.OptionalIndex = .none,
11700	continue_block: Zir.Inst.OptionalIndex = .none,
11701	/// Only valid when setBreakResultInfo is called.
11702	break_result_info: AstGen.ResultInfo = undefined,
11703
11704	suspend_node: Ast.Node.Index = 0,
11705	nosuspend_node: Ast.Node.Index = 0,
11706	/// Set if this GenZir is a defer.
11707	cur_defer_node: Ast.Node.Index = 0,
11708	// Set if this GenZir is a defer or it is inside a defer.
11709	any_defer_node: Ast.Node.Index = 0,
11710
11711	/// Namespace members are lazy. When executing a decl within a namespace,
11712	/// any references to external instructions need to be treated specially.
11713	/// This list tracks those references. See also .closure_capture and .closure_get.
11714	/// Keys are the raw instruction index, values are the closure_capture instruction.
11715	captures: std.AutoHashMapUnmanaged(Zir.Inst.Index, Zir.Inst.Index) = .{},
11716
11717	const unstacked_top = std.math.maxInt(usize);
11718	/// Call unstack before adding any new instructions to containing GenZir.
11719	fn unstack(self: *GenZir) void {
11720	if (self.instructions_top != unstacked_top) {
11721	self.instructions.items.len = self.instructions_top;
11722	self.instructions_top = unstacked_top;
11723	}
11724	}
11725
11726	fn isEmpty(self: *const GenZir) bool {
11727	return (self.instructions_top == unstacked_top) or
11728	(self.instructions.items.len == self.instructions_top);
11729	}
11730
11731	fn instructionsSlice(self: *const GenZir) []Zir.Inst.Index {
11732	return if (self.instructions_top == unstacked_top)
11733	&[0]Zir.Inst.Index{}
11734	else
11735	self.instructions.items[self.instructions_top..];
11736	}
11737
11738	fn instructionsSliceUpto(self: const GenZir, stacked_gz: GenZir) []Zir.Inst.Index {
11739	return if (self.instructions_top == unstacked_top)
11740	&[0]Zir.Inst.Index{}
11741	else if (self.instructions == stacked_gz.instructions and stacked_gz.instructions_top != unstacked_top)
11742	self.instructions.items[self.instructions_top..stacked_gz.instructions_top]
11743	else
11744	self.instructions.items[self.instructions_top..];
11745	}
11746
11747	fn makeSubBlock(gz: GenZir, scope: Scope) GenZir {
11748	return .{
11749	.is_comptime = gz.is_comptime,
11750	.is_typeof = gz.is_typeof,
11751	.c_import = gz.c_import,
11752	.decl_node_index = gz.decl_node_index,
11753	.decl_line = gz.decl_line,
11754	.parent = scope,
11755	.astgen = gz.astgen,
11756	.suspend_node = gz.suspend_node,
11757	.nosuspend_node = gz.nosuspend_node,
11758	.any_defer_node = gz.any_defer_node,
11759	.instructions = gz.instructions,
11760	.instructions_top = gz.instructions.items.len,
11761	};
11762	}
11763
11764	const Label = struct {
11765	token: Ast.TokenIndex,
11766	block_inst: Zir.Inst.Index,
11767	used: bool = false,
11768	};
11769
11770	/// Assumes nothing stacked on `gz`.
11771	fn endsWithNoReturn(gz: GenZir) bool {
11772	if (gz.isEmpty()) return false;
11773	const tags = gz.astgen.instructions.items(.tag);
11774	const last_inst = gz.instructions.items[gz.instructions.items.len - 1];
11775	return tags[@intFromEnum(last_inst)].isNoReturn();
11776	}
11777
11778	/// TODO all uses of this should be replaced with uses of `endsWithNoReturn`.
11779	fn refIsNoReturn(gz: GenZir, inst_ref: Zir.Inst.Ref) bool {
11780	if (inst_ref == .unreachable_value) return true;
11781	if (inst_ref.toIndex()) \|inst_index\| {
11782	return gz.astgen.instructions.items(.tag)[@intFromEnum(inst_index)].isNoReturn();
11783	}
11784	return false;
11785	}
11786
11787	fn nodeIndexToRelative(gz: GenZir, node_index: Ast.Node.Index) i32 {
11788	return @as(i32, @bitCast(node_index)) - @as(i32, @bitCast(gz.decl_node_index));
11789	}
11790
11791	fn tokenIndexToRelative(gz: GenZir, token: Ast.TokenIndex) u32 {
11792	return token - gz.srcToken();
11793	}
11794
11795	fn srcToken(gz: GenZir) Ast.TokenIndex {
11796	return gz.astgen.tree.firstToken(gz.decl_node_index);
11797	}
11798
11799	fn setBreakResultInfo(gz: *GenZir, parent_ri: AstGen.ResultInfo) void {
11800	// Depending on whether the result location is a pointer or value, different
11801	// ZIR needs to be generated. In the former case we rely on storing to the
11802	// pointer to communicate the result, and use breakvoid; in the latter case
11803	// the block break instructions will have the result values.
11804	switch (parent_ri.rl) {
11805	.coerced_ty => \|ty_inst\| {
11806	// Type coercion needs to happen before breaks.
11807	gz.break_result_info = .{ .rl = .{ .ty = ty_inst }, .ctx = parent_ri.ctx };
11808	},
11809	.discard => {
11810	// We don't forward the result context here. This prevents
11811	// "unnecessary discard" errors from being caused by expressions
11812	// far from the actual discard, such as a `break` from a
11813	// discarded block.
11814	gz.break_result_info = .{ .rl = .discard };
11815	},
11816	else => {
11817	gz.break_result_info = parent_ri;
11818	},
11819	}
11820	}
11821
11822	/// Assumes nothing stacked on `gz`. Unstacks `gz`.
11823	fn setBoolBrBody(gz: *GenZir, bool_br: Zir.Inst.Index, bool_br_lhs: Zir.Inst.Ref) !void {
11824	const astgen = gz.astgen;
11825	const gpa = astgen.gpa;
11826	const body = gz.instructionsSlice();
11827	const body_len = astgen.countBodyLenAfterFixups(body);
11828	try astgen.extra.ensureUnusedCapacity(
11829	gpa,
11830	@typeInfo(Zir.Inst.BoolBr).Struct.fields.len + body_len,
11831	);
11832	const zir_datas = astgen.instructions.items(.data);
11833	zir_datas[@intFromEnum(bool_br)].pl_node.payload_index = astgen.addExtraAssumeCapacity(Zir.Inst.BoolBr{
11834	.lhs = bool_br_lhs,
11835	.body_len = body_len,
11836	});
11837	astgen.appendBodyWithFixups(body);
11838	gz.unstack();
11839	}
11840
11841	/// Assumes nothing stacked on `gz`. Unstacks `gz`.
11842	fn setBlockBody(gz: *GenZir, inst: Zir.Inst.Index) !void {
11843	const astgen = gz.astgen;
11844	const gpa = astgen.gpa;
11845	const body = gz.instructionsSlice();
11846	const body_len = astgen.countBodyLenAfterFixups(body);
11847	try astgen.extra.ensureUnusedCapacity(
11848	gpa,
11849	@typeInfo(Zir.Inst.Block).Struct.fields.len + body_len,
11850	);
11851	const zir_datas = astgen.instructions.items(.data);
11852	zir_datas[@intFromEnum(inst)].pl_node.payload_index = astgen.addExtraAssumeCapacity(
11853	Zir.Inst.Block{ .body_len = body_len },
11854	);
11855	astgen.appendBodyWithFixups(body);
11856	gz.unstack();
11857	}
11858
11859	/// Assumes nothing stacked on `gz`. Unstacks `gz`.
11860	fn setTryBody(gz: *GenZir, inst: Zir.Inst.Index, operand: Zir.Inst.Ref) !void {
11861	const astgen = gz.astgen;
11862	const gpa = astgen.gpa;
11863	const body = gz.instructionsSlice();
11864	const body_len = astgen.countBodyLenAfterFixups(body);
11865	try astgen.extra.ensureUnusedCapacity(
11866	gpa,
11867	@typeInfo(Zir.Inst.Try).Struct.fields.len + body_len,
11868	);
11869	const zir_datas = astgen.instructions.items(.data);
11870	zir_datas[@intFromEnum(inst)].pl_node.payload_index = astgen.addExtraAssumeCapacity(
11871	Zir.Inst.Try{
11872	.operand = operand,
11873	.body_len = body_len,
11874	},
11875	);
11876	astgen.appendBodyWithFixups(body);
11877	gz.unstack();
11878	}
11879
11880	/// Must be called with the following stack set up:
11881	/// * gz (bottom)
11882	/// * align_gz
11883	/// * addrspace_gz
11884	/// * section_gz
11885	/// * cc_gz
11886	/// * ret_gz
11887	/// * body_gz (top)
11888	/// Unstacks all of those except for `gz`.
11889	fn addFunc(gz: *GenZir, args: struct {
11890	src_node: Ast.Node.Index,
11891	lbrace_line: u32 = 0,
11892	lbrace_column: u32 = 0,
11893	param_block: Zir.Inst.Index,
11894
11895	align_gz: ?*GenZir,
11896	addrspace_gz: ?*GenZir,
11897	section_gz: ?*GenZir,
11898	cc_gz: ?*GenZir,
11899	ret_gz: ?*GenZir,
11900	body_gz: ?*GenZir,
11901
11902	align_ref: Zir.Inst.Ref,
11903	addrspace_ref: Zir.Inst.Ref,
11904	section_ref: Zir.Inst.Ref,
11905	cc_ref: Zir.Inst.Ref,
11906	ret_ref: Zir.Inst.Ref,
11907
11908	lib_name: Zir.NullTerminatedString,
11909	noalias_bits: u32,
11910	is_var_args: bool,
11911	is_inferred_error: bool,
11912	is_test: bool,
11913	is_extern: bool,
11914	is_noinline: bool,
11915	}) !Zir.Inst.Ref {
11916	assert(args.src_node != 0);
11917	const astgen = gz.astgen;
11918	const gpa = astgen.gpa;
11919	const ret_ref = if (args.ret_ref == .void_type) .none else args.ret_ref;
11920	const new_index: Zir.Inst.Index = @enumFromInt(astgen.instructions.len);
11921
11922	try astgen.instructions.ensureUnusedCapacity(gpa, 1);
11923
11924	var body: []Zir.Inst.Index = &[0]Zir.Inst.Index{};
11925	var ret_body: []Zir.Inst.Index = &[0]Zir.Inst.Index{};
11926	var src_locs_and_hash_buffer: [7]u32 = undefined;
11927	var src_locs_and_hash: []u32 = src_locs_and_hash_buffer[0..0];
11928	if (args.body_gz) \|body_gz\| {
11929	const tree = astgen.tree;
11930	const node_tags = tree.nodes.items(.tag);
11931	const node_datas = tree.nodes.items(.data);
11932	const token_starts = tree.tokens.items(.start);
11933	const fn_decl = args.src_node;
11934	assert(node_tags[fn_decl] == .fn_decl or node_tags[fn_decl] == .test_decl);
11935	const block = node_datas[fn_decl].rhs;
11936	const rbrace_start = token_starts[tree.lastToken(block)];
11937	astgen.advanceSourceCursor(rbrace_start);
11938	const rbrace_line: u32 = @intCast(astgen.source_line - gz.decl_line);
11939	const rbrace_column: u32 = @intCast(astgen.source_column);
11940
11941	const columns = args.lbrace_column \| (rbrace_column << 16);
11942
11943	const proto_hash: std.zig.SrcHash = switch (node_tags[fn_decl]) {
11944	.fn_decl => sig_hash: {
11945	const proto_node = node_datas[fn_decl].lhs;
11946	break :sig_hash std.zig.hashSrc(tree.getNodeSource(proto_node));
11947	},
11948	.test_decl => std.zig.hashSrc(""), // tests don't have a prototype
11949	else => unreachable,
11950	};
11951	const proto_hash_arr: [4]u32 = @bitCast(proto_hash);
11952
11953	src_locs_and_hash_buffer = .{
11954	args.lbrace_line,
11955	rbrace_line,
11956	columns,
11957	proto_hash_arr[0],
11958	proto_hash_arr[1],
11959	proto_hash_arr[2],
11960	proto_hash_arr[3],
11961	};
11962	src_locs_and_hash = &src_locs_and_hash_buffer;
11963
11964	body = body_gz.instructionsSlice();
11965	if (args.ret_gz) \|ret_gz\|
11966	ret_body = ret_gz.instructionsSliceUpto(body_gz);
11967	} else {
11968	if (args.ret_gz) \|ret_gz\|
11969	ret_body = ret_gz.instructionsSlice();
11970	}
11971	const body_len = astgen.countBodyLenAfterFixups(body);
11972
11973	if (args.cc_ref != .none or args.lib_name != .empty or args.is_var_args or args.is_test or
11974	args.is_extern or args.align_ref != .none or args.section_ref != .none or
11975	args.addrspace_ref != .none or args.noalias_bits != 0 or args.is_noinline)
11976	{
11977	var align_body: []Zir.Inst.Index = &.{};
11978	var addrspace_body: []Zir.Inst.Index = &.{};
11979	var section_body: []Zir.Inst.Index = &.{};
11980	var cc_body: []Zir.Inst.Index = &.{};
11981	if (args.ret_gz != null) {
11982	align_body = args.align_gz.?.instructionsSliceUpto(args.addrspace_gz.?);
11983	addrspace_body = args.addrspace_gz.?.instructionsSliceUpto(args.section_gz.?);
11984	section_body = args.section_gz.?.instructionsSliceUpto(args.cc_gz.?);
11985	cc_body = args.cc_gz.?.instructionsSliceUpto(args.ret_gz.?);
11986	}
11987
11988	try astgen.extra.ensureUnusedCapacity(
11989	gpa,
11990	@typeInfo(Zir.Inst.FuncFancy).Struct.fields.len +
11991	fancyFnExprExtraLen(astgen, align_body, args.align_ref) +
11992	fancyFnExprExtraLen(astgen, addrspace_body, args.addrspace_ref) +
11993	fancyFnExprExtraLen(astgen, section_body, args.section_ref) +
11994	fancyFnExprExtraLen(astgen, cc_body, args.cc_ref) +
11995	fancyFnExprExtraLen(astgen, ret_body, ret_ref) +
11996	body_len + src_locs_and_hash.len +
11997	@intFromBool(args.lib_name != .empty) +
11998	@intFromBool(args.noalias_bits != 0),
11999	);
12000	const payload_index = astgen.addExtraAssumeCapacity(Zir.Inst.FuncFancy{
12001	.param_block = args.param_block,
12002	.body_len = body_len,
12003	.bits = .{
12004	.is_var_args = args.is_var_args,
12005	.is_inferred_error = args.is_inferred_error,
12006	.is_test = args.is_test,
12007	.is_extern = args.is_extern,
12008	.is_noinline = args.is_noinline,
12009	.has_lib_name = args.lib_name != .empty,
12010	.has_any_noalias = args.noalias_bits != 0,
12011
12012	.has_align_ref = args.align_ref != .none,
12013	.has_addrspace_ref = args.addrspace_ref != .none,
12014	.has_section_ref = args.section_ref != .none,
12015	.has_cc_ref = args.cc_ref != .none,
12016	.has_ret_ty_ref = ret_ref != .none,
12017
12018	.has_align_body = align_body.len != 0,
12019	.has_addrspace_body = addrspace_body.len != 0,
12020	.has_section_body = section_body.len != 0,
12021	.has_cc_body = cc_body.len != 0,
12022	.has_ret_ty_body = ret_body.len != 0,
12023	},
12024	});
12025	if (args.lib_name != .empty) {
12026	astgen.extra.appendAssumeCapacity(@intFromEnum(args.lib_name));
12027	}
12028
12029	const zir_datas = astgen.instructions.items(.data);
12030	if (align_body.len != 0) {
12031	astgen.extra.appendAssumeCapacity(countBodyLenAfterFixups(astgen, align_body));
12032	astgen.appendBodyWithFixups(align_body);
12033	const break_extra = zir_datas[@intFromEnum(align_body[align_body.len - 1])].@"break".payload_index;
12034	astgen.extra.items[break_extra + std.meta.fieldIndex(Zir.Inst.Break, "block_inst").?] =
12035	@intFromEnum(new_index);
12036	} else if (args.align_ref != .none) {
12037	astgen.extra.appendAssumeCapacity(@intFromEnum(args.align_ref));
12038	}
12039	if (addrspace_body.len != 0) {
12040	astgen.extra.appendAssumeCapacity(countBodyLenAfterFixups(astgen, addrspace_body));
12041	astgen.appendBodyWithFixups(addrspace_body);
12042	const break_extra =
12043	zir_datas[@intFromEnum(addrspace_body[addrspace_body.len - 1])].@"break".payload_index;
12044	astgen.extra.items[break_extra + std.meta.fieldIndex(Zir.Inst.Break, "block_inst").?] =
12045	@intFromEnum(new_index);
12046	} else if (args.addrspace_ref != .none) {
12047	astgen.extra.appendAssumeCapacity(@intFromEnum(args.addrspace_ref));
12048	}
12049	if (section_body.len != 0) {
12050	astgen.extra.appendAssumeCapacity(countBodyLenAfterFixups(astgen, section_body));
12051	astgen.appendBodyWithFixups(section_body);
12052	const break_extra =
12053	zir_datas[@intFromEnum(section_body[section_body.len - 1])].@"break".payload_index;
12054	astgen.extra.items[break_extra + std.meta.fieldIndex(Zir.Inst.Break, "block_inst").?] =
12055	@intFromEnum(new_index);
12056	} else if (args.section_ref != .none) {
12057	astgen.extra.appendAssumeCapacity(@intFromEnum(args.section_ref));
12058	}
12059	if (cc_body.len != 0) {
12060	astgen.extra.appendAssumeCapacity(countBodyLenAfterFixups(astgen, cc_body));
12061	astgen.appendBodyWithFixups(cc_body);
12062	const break_extra = zir_datas[@intFromEnum(cc_body[cc_body.len - 1])].@"break".payload_index;
12063	astgen.extra.items[break_extra + std.meta.fieldIndex(Zir.Inst.Break, "block_inst").?] =
12064	@intFromEnum(new_index);
12065	} else if (args.cc_ref != .none) {
12066	astgen.extra.appendAssumeCapacity(@intFromEnum(args.cc_ref));
12067	}
12068	if (ret_body.len != 0) {
12069	astgen.extra.appendAssumeCapacity(countBodyLenAfterFixups(astgen, ret_body));
12070	astgen.appendBodyWithFixups(ret_body);
12071	const break_extra = zir_datas[@intFromEnum(ret_body[ret_body.len - 1])].@"break".payload_index;
12072	astgen.extra.items[break_extra + std.meta.fieldIndex(Zir.Inst.Break, "block_inst").?] =
12073	@intFromEnum(new_index);
12074	} else if (ret_ref != .none) {
12075	astgen.extra.appendAssumeCapacity(@intFromEnum(ret_ref));
12076	}
12077
12078	if (args.noalias_bits != 0) {
12079	astgen.extra.appendAssumeCapacity(args.noalias_bits);
12080	}
12081
12082	astgen.appendBodyWithFixups(body);
12083	astgen.extra.appendSliceAssumeCapacity(src_locs_and_hash);
12084
12085	// Order is important when unstacking.
12086	if (args.body_gz) \|body_gz\| body_gz.unstack();
12087	if (args.ret_gz != null) {
12088	args.ret_gz.?.unstack();
12089	args.cc_gz.?.unstack();
12090	args.section_gz.?.unstack();
12091	args.addrspace_gz.?.unstack();
12092	args.align_gz.?.unstack();
12093	}
12094
12095	try gz.instructions.ensureUnusedCapacity(gpa, 1);
12096
12097	astgen.instructions.appendAssumeCapacity(.{
12098	.tag = .func_fancy,
12099	.data = .{ .pl_node = .{
12100	.src_node = gz.nodeIndexToRelative(args.src_node),
12101	.payload_index = payload_index,
12102	} },
12103	});
12104	gz.instructions.appendAssumeCapacity(new_index);
12105	return new_index.toRef();
12106	} else {
12107	try astgen.extra.ensureUnusedCapacity(
12108	gpa,
12109	@typeInfo(Zir.Inst.Func).Struct.fields.len + 1 +
12110	fancyFnExprExtraLen(astgen, ret_body, ret_ref) +
12111	body_len + src_locs_and_hash.len,
12112	);
12113
12114	const ret_body_len = if (ret_body.len != 0)
12115	countBodyLenAfterFixups(astgen, ret_body)
12116	else
12117	@intFromBool(ret_ref != .none);
12118
12119	const payload_index = astgen.addExtraAssumeCapacity(Zir.Inst.Func{
12120	.param_block = args.param_block,
12121	.ret_body_len = ret_body_len,
12122	.body_len = body_len,
12123	});
12124	const zir_datas = astgen.instructions.items(.data);
12125	if (ret_body.len != 0) {
12126	astgen.appendBodyWithFixups(ret_body);
12127
12128	const break_extra = zir_datas[@intFromEnum(ret_body[ret_body.len - 1])].@"break".payload_index;
12129	astgen.extra.items[break_extra + std.meta.fieldIndex(Zir.Inst.Break, "block_inst").?] =
12130	@intFromEnum(new_index);
12131	} else if (ret_ref != .none) {
12132	astgen.extra.appendAssumeCapacity(@intFromEnum(ret_ref));
12133	}
12134	astgen.appendBodyWithFixups(body);
12135	astgen.extra.appendSliceAssumeCapacity(src_locs_and_hash);
12136
12137	// Order is important when unstacking.
12138	if (args.body_gz) \|body_gz\| body_gz.unstack();
12139	if (args.ret_gz) \|ret_gz\| ret_gz.unstack();
12140	if (args.cc_gz) \|cc_gz\| cc_gz.unstack();
12141	if (args.section_gz) \|section_gz\| section_gz.unstack();
12142	if (args.addrspace_gz) \|addrspace_gz\| addrspace_gz.unstack();
12143	if (args.align_gz) \|align_gz\| align_gz.unstack();
12144
12145	try gz.instructions.ensureUnusedCapacity(gpa, 1);
12146
12147	const tag: Zir.Inst.Tag = if (args.is_inferred_error) .func_inferred else .func;
12148	astgen.instructions.appendAssumeCapacity(.{
12149	.tag = tag,
12150	.data = .{ .pl_node = .{
12151	.src_node = gz.nodeIndexToRelative(args.src_node),
12152	.payload_index = payload_index,
12153	} },
12154	});
12155	gz.instructions.appendAssumeCapacity(new_index);
12156	return new_index.toRef();
12157	}
12158	}
12159
12160	fn fancyFnExprExtraLen(astgen: *AstGen, body: []Zir.Inst.Index, ref: Zir.Inst.Ref) u32 {
12161	// In the case of non-empty body, there is one for the body length,
12162	// and then one for each instruction.
12163	return countBodyLenAfterFixups(astgen, body) + @intFromBool(ref != .none);
12164	}
12165
12166	fn addVar(gz: *GenZir, args: struct {
12167	align_inst: Zir.Inst.Ref,
12168	lib_name: Zir.NullTerminatedString,
12169	var_type: Zir.Inst.Ref,
12170	init: Zir.Inst.Ref,
12171	is_extern: bool,
12172	is_const: bool,
12173	is_threadlocal: bool,
12174	}) !Zir.Inst.Ref {
12175	const astgen = gz.astgen;
12176	const gpa = astgen.gpa;
12177
12178	try gz.instructions.ensureUnusedCapacity(gpa, 1);
12179	try astgen.instructions.ensureUnusedCapacity(gpa, 1);
12180
12181	try astgen.extra.ensureUnusedCapacity(
12182	gpa,
12183	@typeInfo(Zir.Inst.ExtendedVar).Struct.fields.len +
12184	@intFromBool(args.lib_name != .empty) +
12185	@intFromBool(args.align_inst != .none) +
12186	@intFromBool(args.init != .none),
12187	);
12188	const payload_index = astgen.addExtraAssumeCapacity(Zir.Inst.ExtendedVar{
12189	.var_type = args.var_type,
12190	});
12191	if (args.lib_name != .empty) {
12192	astgen.extra.appendAssumeCapacity(@intFromEnum(args.lib_name));
12193	}
12194	if (args.align_inst != .none) {
12195	astgen.extra.appendAssumeCapacity(@intFromEnum(args.align_inst));
12196	}
12197	if (args.init != .none) {
12198	astgen.extra.appendAssumeCapacity(@intFromEnum(args.init));
12199	}
12200
12201	const new_index: Zir.Inst.Index = @enumFromInt(astgen.instructions.len);
12202	astgen.instructions.appendAssumeCapacity(.{
12203	.tag = .extended,
12204	.data = .{ .extended = .{
12205	.opcode = .variable,
12206	.small = @bitCast(Zir.Inst.ExtendedVar.Small{
12207	.has_lib_name = args.lib_name != .empty,
12208	.has_align = args.align_inst != .none,
12209	.has_init = args.init != .none,
12210	.is_extern = args.is_extern,
12211	.is_const = args.is_const,
12212	.is_threadlocal = args.is_threadlocal,
12213	}),
12214	.operand = payload_index,
12215	} },
12216	});
12217	gz.instructions.appendAssumeCapacity(new_index);
12218	return new_index.toRef();
12219	}
12220
12221	fn addInt(gz: *GenZir, integer: u64) !Zir.Inst.Ref {
12222	return gz.add(.{
12223	.tag = .int,
12224	.data = .{ .int = integer },
12225	});
12226	}
12227
12228	fn addIntBig(gz: *GenZir, limbs: []const std.math.big.Limb) !Zir.Inst.Ref {
12229	const astgen = gz.astgen;
12230	const gpa = astgen.gpa;
12231	try gz.instructions.ensureUnusedCapacity(gpa, 1);
12232	try astgen.instructions.ensureUnusedCapacity(gpa, 1);
12233	try astgen.string_bytes.ensureUnusedCapacity(gpa, @sizeOf(std.math.big.Limb) * limbs.len);
12234
12235	const new_index: Zir.Inst.Index = @enumFromInt(astgen.instructions.len);
12236	astgen.instructions.appendAssumeCapacity(.{
12237	.tag = .int_big,
12238	.data = .{ .str = .{
12239	.start = @enumFromInt(astgen.string_bytes.items.len),
12240	.len = @intCast(limbs.len),
12241	} },
12242	});
12243	gz.instructions.appendAssumeCapacity(new_index);
12244	astgen.string_bytes.appendSliceAssumeCapacity(mem.sliceAsBytes(limbs));
12245	return new_index.toRef();
12246	}
12247
12248	fn addFloat(gz: *GenZir, number: f64) !Zir.Inst.Ref {
12249	return gz.add(.{
12250	.tag = .float,
12251	.data = .{ .float = number },
12252	});
12253	}
12254
12255	fn addUnNode(
12256	gz: *GenZir,
12257	tag: Zir.Inst.Tag,
12258	operand: Zir.Inst.Ref,
12259	/// Absolute node index. This function does the conversion to offset from Decl.
12260	src_node: Ast.Node.Index,
12261	) !Zir.Inst.Ref {
12262	assert(operand != .none);
12263	return gz.add(.{
12264	.tag = tag,
12265	.data = .{ .un_node = .{
12266	.operand = operand,
12267	.src_node = gz.nodeIndexToRelative(src_node),
12268	} },
12269	});
12270	}
12271
12272	fn makeUnNode(
12273	gz: *GenZir,
12274	tag: Zir.Inst.Tag,
12275	operand: Zir.Inst.Ref,
12276	/// Absolute node index. This function does the conversion to offset from Decl.
12277	src_node: Ast.Node.Index,
12278	) !Zir.Inst.Index {
12279	assert(operand != .none);
12280	const new_index: Zir.Inst.Index = @enumFromInt(gz.astgen.instructions.len);
12281	try gz.astgen.instructions.append(gz.astgen.gpa, .{
12282	.tag = tag,
12283	.data = .{ .un_node = .{
12284	.operand = operand,
12285	.src_node = gz.nodeIndexToRelative(src_node),
12286	} },
12287	});
12288	return new_index;
12289	}
12290
12291	fn addPlNode(
12292	gz: *GenZir,
12293	tag: Zir.Inst.Tag,
12294	/// Absolute node index. This function does the conversion to offset from Decl.
12295	src_node: Ast.Node.Index,
12296	extra: anytype,
12297	) !Zir.Inst.Ref {
12298	const gpa = gz.astgen.gpa;
12299	try gz.instructions.ensureUnusedCapacity(gpa, 1);
12300	try gz.astgen.instructions.ensureUnusedCapacity(gpa, 1);
12301
12302	const payload_index = try gz.astgen.addExtra(extra);
12303	const new_index: Zir.Inst.Index = @enumFromInt(gz.astgen.instructions.len);
12304	gz.astgen.instructions.appendAssumeCapacity(.{
12305	.tag = tag,
12306	.data = .{ .pl_node = .{
12307	.src_node = gz.nodeIndexToRelative(src_node),
12308	.payload_index = payload_index,
12309	} },
12310	});
12311	gz.instructions.appendAssumeCapacity(new_index);
12312	return new_index.toRef();
12313	}
12314
12315	fn addPlNodePayloadIndex(
12316	gz: *GenZir,
12317	tag: Zir.Inst.Tag,
12318	/// Absolute node index. This function does the conversion to offset from Decl.
12319	src_node: Ast.Node.Index,
12320	payload_index: u32,
12321	) !Zir.Inst.Ref {
12322	return try gz.add(.{
12323	.tag = tag,
12324	.data = .{ .pl_node = .{
12325	.src_node = gz.nodeIndexToRelative(src_node),
12326	.payload_index = payload_index,
12327	} },
12328	});
12329	}
12330
12331	/// Supports `param_gz` stacked on `gz`. Assumes nothing stacked on `param_gz`. Unstacks `param_gz`.
12332	fn addParam(
12333	gz: *GenZir,
12334	param_gz: *GenZir,
12335	tag: Zir.Inst.Tag,
12336	/// Absolute token index. This function does the conversion to Decl offset.
12337	abs_tok_index: Ast.TokenIndex,
12338	name: Zir.NullTerminatedString,
12339	first_doc_comment: ?Ast.TokenIndex,
12340	) !Zir.Inst.Index {
12341	const gpa = gz.astgen.gpa;
12342	const param_body = param_gz.instructionsSlice();
12343	const body_len = gz.astgen.countBodyLenAfterFixups(param_body);
12344	try gz.astgen.instructions.ensureUnusedCapacity(gpa, 1);
12345	try gz.astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.Param).Struct.fields.len + body_len);
12346
12347	const doc_comment_index = if (first_doc_comment) \|first\|
12348	try gz.astgen.docCommentAsStringFromFirst(abs_tok_index, first)
12349	else
12350	.empty;
12351
12352	const payload_index = gz.astgen.addExtraAssumeCapacity(Zir.Inst.Param{
12353	.name = name,
12354	.doc_comment = doc_comment_index,
12355	.body_len = @intCast(body_len),
12356	});
12357	gz.astgen.appendBodyWithFixups(param_body);
12358	param_gz.unstack();
12359
12360	const new_index: Zir.Inst.Index = @enumFromInt(gz.astgen.instructions.len);
12361	gz.astgen.instructions.appendAssumeCapacity(.{
12362	.tag = tag,
12363	.data = .{ .pl_tok = .{
12364	.src_tok = gz.tokenIndexToRelative(abs_tok_index),
12365	.payload_index = payload_index,
12366	} },
12367	});
12368	gz.instructions.appendAssumeCapacity(new_index);
12369	return new_index;
12370	}
12371
12372	fn addExtendedPayload(gz: *GenZir, opcode: Zir.Inst.Extended, extra: anytype) !Zir.Inst.Ref {
12373	return addExtendedPayloadSmall(gz, opcode, undefined, extra);
12374	}
12375
12376	fn addExtendedPayloadSmall(
12377	gz: *GenZir,
12378	opcode: Zir.Inst.Extended,
12379	small: u16,
12380	extra: anytype,
12381	) !Zir.Inst.Ref {
12382	const gpa = gz.astgen.gpa;
12383
12384	try gz.instructions.ensureUnusedCapacity(gpa, 1);
12385	try gz.astgen.instructions.ensureUnusedCapacity(gpa, 1);
12386
12387	const payload_index = try gz.astgen.addExtra(extra);
12388	const new_index: Zir.Inst.Index = @enumFromInt(gz.astgen.instructions.len);
12389	gz.astgen.instructions.appendAssumeCapacity(.{
12390	.tag = .extended,
12391	.data = .{ .extended = .{
12392	.opcode = opcode,
12393	.small = small,
12394	.operand = payload_index,
12395	} },
12396	});
12397	gz.instructions.appendAssumeCapacity(new_index);
12398	return new_index.toRef();
12399	}
12400
12401	fn addExtendedMultiOp(
12402	gz: *GenZir,
12403	opcode: Zir.Inst.Extended,
12404	node: Ast.Node.Index,
12405	operands: []const Zir.Inst.Ref,
12406	) !Zir.Inst.Ref {
12407	const astgen = gz.astgen;
12408	const gpa = astgen.gpa;
12409
12410	try gz.instructions.ensureUnusedCapacity(gpa, 1);
12411	try astgen.instructions.ensureUnusedCapacity(gpa, 1);
12412	try astgen.extra.ensureUnusedCapacity(
12413	gpa,
12414	@typeInfo(Zir.Inst.NodeMultiOp).Struct.fields.len + operands.len,
12415	);
12416
12417	const payload_index = astgen.addExtraAssumeCapacity(Zir.Inst.NodeMultiOp{
12418	.src_node = gz.nodeIndexToRelative(node),
12419	});
12420	const new_index: Zir.Inst.Index = @enumFromInt(astgen.instructions.len);
12421	astgen.instructions.appendAssumeCapacity(.{
12422	.tag = .extended,
12423	.data = .{ .extended = .{
12424	.opcode = opcode,
12425	.small = @intCast(operands.len),
12426	.operand = payload_index,
12427	} },
12428	});
12429	gz.instructions.appendAssumeCapacity(new_index);
12430	astgen.appendRefsAssumeCapacity(operands);
12431	return new_index.toRef();
12432	}
12433
12434	fn addExtendedMultiOpPayloadIndex(
12435	gz: *GenZir,
12436	opcode: Zir.Inst.Extended,
12437	payload_index: u32,
12438	trailing_len: usize,
12439	) !Zir.Inst.Ref {
12440	const astgen = gz.astgen;
12441	const gpa = astgen.gpa;
12442
12443	try gz.instructions.ensureUnusedCapacity(gpa, 1);
12444	try astgen.instructions.ensureUnusedCapacity(gpa, 1);
12445	const new_index: Zir.Inst.Index = @enumFromInt(astgen.instructions.len);
12446	astgen.instructions.appendAssumeCapacity(.{
12447	.tag = .extended,
12448	.data = .{ .extended = .{
12449	.opcode = opcode,
12450	.small = @intCast(trailing_len),
12451	.operand = payload_index,
12452	} },
12453	});
12454	gz.instructions.appendAssumeCapacity(new_index);
12455	return new_index.toRef();
12456	}
12457
12458	fn addUnTok(
12459	gz: *GenZir,
12460	tag: Zir.Inst.Tag,
12461	operand: Zir.Inst.Ref,
12462	/// Absolute token index. This function does the conversion to Decl offset.
12463	abs_tok_index: Ast.TokenIndex,
12464	) !Zir.Inst.Ref {
12465	assert(operand != .none);
12466	return gz.add(.{
12467	.tag = tag,
12468	.data = .{ .un_tok = .{
12469	.operand = operand,
12470	.src_tok = gz.tokenIndexToRelative(abs_tok_index),
12471	} },
12472	});
12473	}
12474
12475	fn makeUnTok(
12476	gz: *GenZir,
12477	tag: Zir.Inst.Tag,
12478	operand: Zir.Inst.Ref,
12479	/// Absolute token index. This function does the conversion to Decl offset.
12480	abs_tok_index: Ast.TokenIndex,
12481	) !Zir.Inst.Index {
12482	const astgen = gz.astgen;
12483	const new_index: Zir.Inst.Index = @enumFromInt(astgen.instructions.len);
12484	assert(operand != .none);
12485	try astgen.instructions.append(astgen.gpa, .{
12486	.tag = tag,
12487	.data = .{ .un_tok = .{
12488	.operand = operand,
12489	.src_tok = gz.tokenIndexToRelative(abs_tok_index),
12490	} },
12491	});
12492	return new_index;
12493	}
12494
12495	fn addStrTok(
12496	gz: *GenZir,
12497	tag: Zir.Inst.Tag,
12498	str_index: Zir.NullTerminatedString,
12499	/// Absolute token index. This function does the conversion to Decl offset.
12500	abs_tok_index: Ast.TokenIndex,
12501	) !Zir.Inst.Ref {
12502	return gz.add(.{
12503	.tag = tag,
12504	.data = .{ .str_tok = .{
12505	.start = str_index,
12506	.src_tok = gz.tokenIndexToRelative(abs_tok_index),
12507	} },
12508	});
12509	}
12510
12511	fn addSaveErrRetIndex(
12512	gz: *GenZir,
12513	cond: union(enum) {
12514	always: void,
12515	if_of_error_type: Zir.Inst.Ref,
12516	},
12517	) !Zir.Inst.Index {
12518	return gz.addAsIndex(.{
12519	.tag = .save_err_ret_index,
12520	.data = .{ .save_err_ret_index = .{
12521	.operand = switch (cond) {
12522	.if_of_error_type => \|x\| x,
12523	else => .none,
12524	},
12525	} },
12526	});
12527	}
12528
12529	const BranchTarget = union(enum) {
12530	ret,
12531	block: Zir.Inst.Index,
12532	};
12533
12534	fn addRestoreErrRetIndex(
12535	gz: *GenZir,
12536	bt: BranchTarget,
12537	cond: union(enum) {
12538	always: void,
12539	if_non_error: Zir.Inst.Ref,
12540	},
12541	src_node: Ast.Node.Index,
12542	) !Zir.Inst.Index {
12543	switch (cond) {
12544	.always => return gz.addAsIndex(.{
12545	.tag = .restore_err_ret_index_unconditional,
12546	.data = .{ .un_node = .{
12547	.operand = switch (bt) {
12548	.ret => .none,
12549	.block => \|b\| b.toRef(),
12550	},
12551	.src_node = gz.nodeIndexToRelative(src_node),
12552	} },
12553	}),
12554	.if_non_error => \|operand\| switch (bt) {
12555	.ret => return gz.addAsIndex(.{
12556	.tag = .restore_err_ret_index_fn_entry,
12557	.data = .{ .un_node = .{
12558	.operand = operand,
12559	.src_node = gz.nodeIndexToRelative(src_node),
12560	} },
12561	}),
12562	.block => \|block\| return (try gz.addExtendedPayload(
12563	.restore_err_ret_index,
12564	Zir.Inst.RestoreErrRetIndex{
12565	.src_node = gz.nodeIndexToRelative(src_node),
12566	.block = block.toRef(),
12567	.operand = operand,
12568	},
12569	)).toIndex().?,
12570	},
12571	}
12572	}
12573
12574	fn addBreak(
12575	gz: *GenZir,
12576	tag: Zir.Inst.Tag,
12577	block_inst: Zir.Inst.Index,
12578	operand: Zir.Inst.Ref,
12579	) !Zir.Inst.Index {
12580	const gpa = gz.astgen.gpa;
12581	try gz.instructions.ensureUnusedCapacity(gpa, 1);
12582
12583	const new_index = try gz.makeBreak(tag, block_inst, operand);
12584	gz.instructions.appendAssumeCapacity(new_index);
12585	return new_index;
12586	}
12587
12588	fn makeBreak(
12589	gz: *GenZir,
12590	tag: Zir.Inst.Tag,
12591	block_inst: Zir.Inst.Index,
12592	operand: Zir.Inst.Ref,
12593	) !Zir.Inst.Index {
12594	return gz.makeBreakCommon(tag, block_inst, operand, null);
12595	}
12596
12597	fn addBreakWithSrcNode(
12598	gz: *GenZir,
12599	tag: Zir.Inst.Tag,
12600	block_inst: Zir.Inst.Index,
12601	operand: Zir.Inst.Ref,
12602	operand_src_node: Ast.Node.Index,
12603	) !Zir.Inst.Index {
12604	const gpa = gz.astgen.gpa;
12605	try gz.instructions.ensureUnusedCapacity(gpa, 1);
12606
12607	const new_index = try gz.makeBreakWithSrcNode(tag, block_inst, operand, operand_src_node);
12608	gz.instructions.appendAssumeCapacity(new_index);
12609	return new_index;
12610	}
12611
12612	fn makeBreakWithSrcNode(
12613	gz: *GenZir,
12614	tag: Zir.Inst.Tag,
12615	block_inst: Zir.Inst.Index,
12616	operand: Zir.Inst.Ref,
12617	operand_src_node: Ast.Node.Index,
12618	) !Zir.Inst.Index {
12619	return gz.makeBreakCommon(tag, block_inst, operand, operand_src_node);
12620	}
12621
12622	fn makeBreakCommon(
12623	gz: *GenZir,
12624	tag: Zir.Inst.Tag,
12625	block_inst: Zir.Inst.Index,
12626	operand: Zir.Inst.Ref,
12627	operand_src_node: ?Ast.Node.Index,
12628	) !Zir.Inst.Index {
12629	const gpa = gz.astgen.gpa;
12630	try gz.astgen.instructions.ensureUnusedCapacity(gpa, 1);
12631	try gz.astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.Break).Struct.fields.len);
12632
12633	const new_index: Zir.Inst.Index = @enumFromInt(gz.astgen.instructions.len);
12634	gz.astgen.instructions.appendAssumeCapacity(.{
12635	.tag = tag,
12636	.data = .{ .@"break" = .{
12637	.operand = operand,
12638	.payload_index = gz.astgen.addExtraAssumeCapacity(Zir.Inst.Break{
12639	.operand_src_node = if (operand_src_node) \|src_node\|
12640	gz.nodeIndexToRelative(src_node)
12641	else
12642	Zir.Inst.Break.no_src_node,
12643	.block_inst = block_inst,
12644	}),
12645	} },
12646	});
12647	return new_index;
12648	}
12649
12650	fn addBin(
12651	gz: *GenZir,
12652	tag: Zir.Inst.Tag,
12653	lhs: Zir.Inst.Ref,
12654	rhs: Zir.Inst.Ref,
12655	) !Zir.Inst.Ref {
12656	assert(lhs != .none);
12657	assert(rhs != .none);
12658	return gz.add(.{
12659	.tag = tag,
12660	.data = .{ .bin = .{
12661	.lhs = lhs,
12662	.rhs = rhs,
12663	} },
12664	});
12665	}
12666
12667	fn addDefer(gz: *GenZir, index: u32, len: u32) !void {
12668	_ = try gz.add(.{
12669	.tag = .@"defer",
12670	.data = .{ .@"defer" = .{
12671	.index = index,
12672	.len = len,
12673	} },
12674	});
12675	}
12676
12677	fn addDecl(
12678	gz: *GenZir,
12679	tag: Zir.Inst.Tag,
12680	decl_index: u32,
12681	src_node: Ast.Node.Index,
12682	) !Zir.Inst.Ref {
12683	return gz.add(.{
12684	.tag = tag,
12685	.data = .{ .pl_node = .{
12686	.src_node = gz.nodeIndexToRelative(src_node),
12687	.payload_index = decl_index,
12688	} },
12689	});
12690	}
12691
12692	fn addNode(
12693	gz: *GenZir,
12694	tag: Zir.Inst.Tag,
12695	/// Absolute node index. This function does the conversion to offset from Decl.
12696	src_node: Ast.Node.Index,
12697	) !Zir.Inst.Ref {
12698	return gz.add(.{
12699	.tag = tag,
12700	.data = .{ .node = gz.nodeIndexToRelative(src_node) },
12701	});
12702	}
12703
12704	fn addInstNode(
12705	gz: *GenZir,
12706	tag: Zir.Inst.Tag,
12707	inst: Zir.Inst.Index,
12708	/// Absolute node index. This function does the conversion to offset from Decl.
12709	src_node: Ast.Node.Index,
12710	) !Zir.Inst.Ref {
12711	return gz.add(.{
12712	.tag = tag,
12713	.data = .{ .inst_node = .{
12714	.inst = inst,
12715	.src_node = gz.nodeIndexToRelative(src_node),
12716	} },
12717	});
12718	}
12719
12720	fn addNodeExtended(
12721	gz: *GenZir,
12722	opcode: Zir.Inst.Extended,
12723	/// Absolute node index. This function does the conversion to offset from Decl.
12724	src_node: Ast.Node.Index,
12725	) !Zir.Inst.Ref {
12726	return gz.add(.{
12727	.tag = .extended,
12728	.data = .{ .extended = .{
12729	.opcode = opcode,
12730	.small = undefined,
12731	.operand = @bitCast(gz.nodeIndexToRelative(src_node)),
12732	} },
12733	});
12734	}
12735
12736	fn addAllocExtended(
12737	gz: *GenZir,
12738	args: struct {
12739	/// Absolute node index. This function does the conversion to offset from Decl.
12740	node: Ast.Node.Index,
12741	type_inst: Zir.Inst.Ref,
12742	align_inst: Zir.Inst.Ref,
12743	is_const: bool,
12744	is_comptime: bool,
12745	},
12746	) !Zir.Inst.Ref {
12747	const astgen = gz.astgen;
12748	const gpa = astgen.gpa;
12749
12750	try gz.instructions.ensureUnusedCapacity(gpa, 1);
12751	try astgen.instructions.ensureUnusedCapacity(gpa, 1);
12752	try astgen.extra.ensureUnusedCapacity(
12753	gpa,
12754	@typeInfo(Zir.Inst.AllocExtended).Struct.fields.len +
12755	@intFromBool(args.type_inst != .none) +
12756	@intFromBool(args.align_inst != .none),
12757	);
12758	const payload_index = gz.astgen.addExtraAssumeCapacity(Zir.Inst.AllocExtended{
12759	.src_node = gz.nodeIndexToRelative(args.node),
12760	});
12761	if (args.type_inst != .none) {
12762	astgen.extra.appendAssumeCapacity(@intFromEnum(args.type_inst));
12763	}
12764	if (args.align_inst != .none) {
12765	astgen.extra.appendAssumeCapacity(@intFromEnum(args.align_inst));
12766	}
12767
12768	const has_type: u4 = @intFromBool(args.type_inst != .none);
12769	const has_align: u4 = @intFromBool(args.align_inst != .none);
12770	const is_const: u4 = @intFromBool(args.is_const);
12771	const is_comptime: u4 = @intFromBool(args.is_comptime);
12772	const small: u16 = has_type \| (has_align << 1) \| (is_const << 2) \| (is_comptime << 3);
12773
12774	const new_index: Zir.Inst.Index = @enumFromInt(astgen.instructions.len);
12775	astgen.instructions.appendAssumeCapacity(.{
12776	.tag = .extended,
12777	.data = .{ .extended = .{
12778	.opcode = .alloc,
12779	.small = small,
12780	.operand = payload_index,
12781	} },
12782	});
12783	gz.instructions.appendAssumeCapacity(new_index);
12784	return new_index.toRef();
12785	}
12786
12787	fn addAsm(
12788	gz: *GenZir,
12789	args: struct {
12790	tag: Zir.Inst.Extended,
12791	/// Absolute node index. This function does the conversion to offset from Decl.
12792	node: Ast.Node.Index,
12793	asm_source: Zir.NullTerminatedString,
12794	output_type_bits: u32,
12795	is_volatile: bool,
12796	outputs: []const Zir.Inst.Asm.Output,
12797	inputs: []const Zir.Inst.Asm.Input,
12798	clobbers: []const u32,
12799	},
12800	) !Zir.Inst.Ref {
12801	const astgen = gz.astgen;
12802	const gpa = astgen.gpa;
12803
12804	try gz.instructions.ensureUnusedCapacity(gpa, 1);
12805	try astgen.instructions.ensureUnusedCapacity(gpa, 1);
12806	try astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.Asm).Struct.fields.len +
12807	args.outputs.len * @typeInfo(Zir.Inst.Asm.Output).Struct.fields.len +
12808	args.inputs.len * @typeInfo(Zir.Inst.Asm.Input).Struct.fields.len +
12809	args.clobbers.len);
12810
12811	const payload_index = gz.astgen.addExtraAssumeCapacity(Zir.Inst.Asm{
12812	.src_node = gz.nodeIndexToRelative(args.node),
12813	.asm_source = args.asm_source,
12814	.output_type_bits = args.output_type_bits,
12815	});
12816	for (args.outputs) \|output\| {
12817	_ = gz.astgen.addExtraAssumeCapacity(output);
12818	}
12819	for (args.inputs) \|input\| {
12820	_ = gz.astgen.addExtraAssumeCapacity(input);
12821	}
12822	gz.astgen.extra.appendSliceAssumeCapacity(args.clobbers);
12823
12824	// * 0b00000000_000XXXXX - `outputs_len`.
12825	// * 0b000000XX_XXX00000 - `inputs_len`.
12826	// * 0b0XXXXX00_00000000 - `clobbers_len`.
12827	// * 0bX0000000_00000000 - is volatile
12828	const small: u16 = @as(u16, @intCast(args.outputs.len)) \|
12829	@as(u16, @intCast(args.inputs.len << 5)) \|
12830	@as(u16, @intCast(args.clobbers.len << 10)) \|
12831	(@as(u16, @intFromBool(args.is_volatile)) << 15);
12832
12833	const new_index: Zir.Inst.Index = @enumFromInt(astgen.instructions.len);
12834	astgen.instructions.appendAssumeCapacity(.{
12835	.tag = .extended,
12836	.data = .{ .extended = .{
12837	.opcode = args.tag,
12838	.small = small,
12839	.operand = payload_index,
12840	} },
12841	});
12842	gz.instructions.appendAssumeCapacity(new_index);
12843	return new_index.toRef();
12844	}
12845
12846	/// Note that this returns a `Zir.Inst.Index` not a ref.
12847	/// Does not append the block instruction to the scope.
12848	/// Leaves the `payload_index` field undefined.
12849	fn makeBlockInst(gz: *GenZir, tag: Zir.Inst.Tag, node: Ast.Node.Index) !Zir.Inst.Index {
12850	const new_index: Zir.Inst.Index = @enumFromInt(gz.astgen.instructions.len);
12851	const gpa = gz.astgen.gpa;
12852	try gz.astgen.instructions.append(gpa, .{
12853	.tag = tag,
12854	.data = .{ .pl_node = .{
12855	.src_node = gz.nodeIndexToRelative(node),
12856	.payload_index = undefined,
12857	} },
12858	});
12859	return new_index;
12860	}
12861
12862	/// Note that this returns a `Zir.Inst.Index` not a ref.
12863	/// Leaves the `payload_index` field undefined.
12864	fn addCondBr(gz: *GenZir, tag: Zir.Inst.Tag, node: Ast.Node.Index) !Zir.Inst.Index {
12865	const gpa = gz.astgen.gpa;
12866	try gz.instructions.ensureUnusedCapacity(gpa, 1);
12867	const new_index: Zir.Inst.Index = @enumFromInt(gz.astgen.instructions.len);
12868	try gz.astgen.instructions.append(gpa, .{
12869	.tag = tag,
12870	.data = .{ .pl_node = .{
12871	.src_node = gz.nodeIndexToRelative(node),
12872	.payload_index = undefined,
12873	} },
12874	});
12875	gz.instructions.appendAssumeCapacity(new_index);
12876	return new_index;
12877	}
12878
12879	fn setStruct(gz: *GenZir, inst: Zir.Inst.Index, args: struct {
12880	src_node: Ast.Node.Index,
12881	fields_len: u32,
12882	decls_len: u32,
12883	backing_int_ref: Zir.Inst.Ref,
12884	backing_int_body_len: u32,
12885	layout: std.builtin.Type.ContainerLayout,
12886	known_non_opv: bool,
12887	known_comptime_only: bool,
12888	is_tuple: bool,
12889	any_comptime_fields: bool,
12890	any_default_inits: bool,
12891	any_aligned_fields: bool,
12892	fields_hash: std.zig.SrcHash,
12893	}) !void {
12894	const astgen = gz.astgen;
12895	const gpa = astgen.gpa;
12896
12897	// Node 0 is valid for the root `struct_decl` of a file!
12898	assert(args.src_node != 0 or gz.parent.tag == .top);
12899
12900	const fields_hash_arr: [4]u32 = @bitCast(args.fields_hash);
12901
12902	try astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.StructDecl).Struct.fields.len + 4);
12903	const payload_index = astgen.addExtraAssumeCapacity(Zir.Inst.StructDecl{
12904	.fields_hash_0 = fields_hash_arr[0],
12905	.fields_hash_1 = fields_hash_arr[1],
12906	.fields_hash_2 = fields_hash_arr[2],
12907	.fields_hash_3 = fields_hash_arr[3],
12908	.src_node = gz.nodeIndexToRelative(args.src_node),
12909	});
12910
12911	if (args.fields_len != 0) {
12912	astgen.extra.appendAssumeCapacity(args.fields_len);
12913	}
12914	if (args.decls_len != 0) {
12915	astgen.extra.appendAssumeCapacity(args.decls_len);
12916	}
12917	if (args.backing_int_ref != .none) {
12918	astgen.extra.appendAssumeCapacity(args.backing_int_body_len);
12919	if (args.backing_int_body_len == 0) {
12920	astgen.extra.appendAssumeCapacity(@intFromEnum(args.backing_int_ref));
12921	}
12922	}
12923	astgen.instructions.set(@intFromEnum(inst), .{
12924	.tag = .extended,
12925	.data = .{ .extended = .{
12926	.opcode = .struct_decl,
12927	.small = @bitCast(Zir.Inst.StructDecl.Small{
12928	.has_fields_len = args.fields_len != 0,
12929	.has_decls_len = args.decls_len != 0,
12930	.has_backing_int = args.backing_int_ref != .none,
12931	.known_non_opv = args.known_non_opv,
12932	.known_comptime_only = args.known_comptime_only,
12933	.is_tuple = args.is_tuple,
12934	.name_strategy = gz.anon_name_strategy,
12935	.layout = args.layout,
12936	.any_comptime_fields = args.any_comptime_fields,
12937	.any_default_inits = args.any_default_inits,
12938	.any_aligned_fields = args.any_aligned_fields,
12939	}),
12940	.operand = payload_index,
12941	} },
12942	});
12943	}
12944
12945	fn setUnion(gz: *GenZir, inst: Zir.Inst.Index, args: struct {
12946	src_node: Ast.Node.Index,
12947	tag_type: Zir.Inst.Ref,
12948	body_len: u32,
12949	fields_len: u32,
12950	decls_len: u32,
12951	layout: std.builtin.Type.ContainerLayout,
12952	auto_enum_tag: bool,
12953	any_aligned_fields: bool,
12954	fields_hash: std.zig.SrcHash,
12955	}) !void {
12956	const astgen = gz.astgen;
12957	const gpa = astgen.gpa;
12958
12959	assert(args.src_node != 0);
12960
12961	const fields_hash_arr: [4]u32 = @bitCast(args.fields_hash);
12962
12963	try astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.UnionDecl).Struct.fields.len + 4);
12964	const payload_index = astgen.addExtraAssumeCapacity(Zir.Inst.UnionDecl{
12965	.fields_hash_0 = fields_hash_arr[0],
12966	.fields_hash_1 = fields_hash_arr[1],
12967	.fields_hash_2 = fields_hash_arr[2],
12968	.fields_hash_3 = fields_hash_arr[3],
12969	.src_node = gz.nodeIndexToRelative(args.src_node),
12970	});
12971
12972	if (args.tag_type != .none) {
12973	astgen.extra.appendAssumeCapacity(@intFromEnum(args.tag_type));
12974	}
12975	if (args.body_len != 0) {
12976	astgen.extra.appendAssumeCapacity(args.body_len);
12977	}
12978	if (args.fields_len != 0) {
12979	astgen.extra.appendAssumeCapacity(args.fields_len);
12980	}
12981	if (args.decls_len != 0) {
12982	astgen.extra.appendAssumeCapacity(args.decls_len);
12983	}
12984	astgen.instructions.set(@intFromEnum(inst), .{
12985	.tag = .extended,
12986	.data = .{ .extended = .{
12987	.opcode = .union_decl,
12988	.small = @bitCast(Zir.Inst.UnionDecl.Small{
12989	.has_tag_type = args.tag_type != .none,
12990	.has_body_len = args.body_len != 0,
12991	.has_fields_len = args.fields_len != 0,
12992	.has_decls_len = args.decls_len != 0,
12993	.name_strategy = gz.anon_name_strategy,
12994	.layout = args.layout,
12995	.auto_enum_tag = args.auto_enum_tag,
12996	.any_aligned_fields = args.any_aligned_fields,
12997	}),
12998	.operand = payload_index,
12999	} },
13000	});
13001	}
13002
13003	fn setEnum(gz: *GenZir, inst: Zir.Inst.Index, args: struct {
13004	src_node: Ast.Node.Index,
13005	tag_type: Zir.Inst.Ref,
13006	body_len: u32,
13007	fields_len: u32,
13008	decls_len: u32,
13009	nonexhaustive: bool,
13010	fields_hash: std.zig.SrcHash,
13011	}) !void {
13012	const astgen = gz.astgen;
13013	const gpa = astgen.gpa;
13014
13015	assert(args.src_node != 0);
13016
13017	const fields_hash_arr: [4]u32 = @bitCast(args.fields_hash);
13018
13019	try astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.EnumDecl).Struct.fields.len + 4);
13020	const payload_index = astgen.addExtraAssumeCapacity(Zir.Inst.EnumDecl{
13021	.fields_hash_0 = fields_hash_arr[0],
13022	.fields_hash_1 = fields_hash_arr[1],
13023	.fields_hash_2 = fields_hash_arr[2],
13024	.fields_hash_3 = fields_hash_arr[3],
13025	.src_node = gz.nodeIndexToRelative(args.src_node),
13026	});
13027
13028	if (args.tag_type != .none) {
13029	astgen.extra.appendAssumeCapacity(@intFromEnum(args.tag_type));
13030	}
13031	if (args.body_len != 0) {
13032	astgen.extra.appendAssumeCapacity(args.body_len);
13033	}
13034	if (args.fields_len != 0) {
13035	astgen.extra.appendAssumeCapacity(args.fields_len);
13036	}
13037	if (args.decls_len != 0) {
13038	astgen.extra.appendAssumeCapacity(args.decls_len);
13039	}
13040	astgen.instructions.set(@intFromEnum(inst), .{
13041	.tag = .extended,
13042	.data = .{ .extended = .{
13043	.opcode = .enum_decl,
13044	.small = @bitCast(Zir.Inst.EnumDecl.Small{
13045	.has_tag_type = args.tag_type != .none,
13046	.has_body_len = args.body_len != 0,
13047	.has_fields_len = args.fields_len != 0,
13048	.has_decls_len = args.decls_len != 0,
13049	.name_strategy = gz.anon_name_strategy,
13050	.nonexhaustive = args.nonexhaustive,
13051	}),
13052	.operand = payload_index,
13053	} },
13054	});
13055	}
13056
13057	fn setOpaque(gz: *GenZir, inst: Zir.Inst.Index, args: struct {
13058	src_node: Ast.Node.Index,
13059	decls_len: u32,
13060	}) !void {
13061	const astgen = gz.astgen;
13062	const gpa = astgen.gpa;
13063
13064	assert(args.src_node != 0);
13065
13066	try astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.OpaqueDecl).Struct.fields.len + 1);
13067	const payload_index = astgen.addExtraAssumeCapacity(Zir.Inst.OpaqueDecl{
13068	.src_node = gz.nodeIndexToRelative(args.src_node),
13069	});
13070
13071	if (args.decls_len != 0) {
13072	astgen.extra.appendAssumeCapacity(args.decls_len);
13073	}
13074	astgen.instructions.set(@intFromEnum(inst), .{
13075	.tag = .extended,
13076	.data = .{ .extended = .{
13077	.opcode = .opaque_decl,
13078	.small = @bitCast(Zir.Inst.OpaqueDecl.Small{
13079	.has_decls_len = args.decls_len != 0,
13080	.name_strategy = gz.anon_name_strategy,
13081	}),
13082	.operand = payload_index,
13083	} },
13084	});
13085	}
13086
13087	fn add(gz: *GenZir, inst: Zir.Inst) !Zir.Inst.Ref {
13088	return (try gz.addAsIndex(inst)).toRef();
13089	}
13090
13091	fn addAsIndex(gz: *GenZir, inst: Zir.Inst) !Zir.Inst.Index {
13092	const gpa = gz.astgen.gpa;
13093	try gz.instructions.ensureUnusedCapacity(gpa, 1);
13094	try gz.astgen.instructions.ensureUnusedCapacity(gpa, 1);
13095
13096	const new_index: Zir.Inst.Index = @enumFromInt(gz.astgen.instructions.len);
13097	gz.astgen.instructions.appendAssumeCapacity(inst);
13098	gz.instructions.appendAssumeCapacity(new_index);
13099	return new_index;
13100	}
13101
13102	fn reserveInstructionIndex(gz: *GenZir) !Zir.Inst.Index {
13103	const gpa = gz.astgen.gpa;
13104	try gz.instructions.ensureUnusedCapacity(gpa, 1);
13105	try gz.astgen.instructions.ensureUnusedCapacity(gpa, 1);
13106
13107	const new_index: Zir.Inst.Index = @enumFromInt(gz.astgen.instructions.len);
13108	gz.astgen.instructions.len += 1;
13109	gz.instructions.appendAssumeCapacity(new_index);
13110	return new_index;
13111	}
13112
13113	fn addRet(gz: *GenZir, ri: ResultInfo, operand: Zir.Inst.Ref, node: Ast.Node.Index) !void {
13114	switch (ri.rl) {
13115	.ptr => \|ptr_res\| _ = try gz.addUnNode(.ret_load, ptr_res.inst, node),
13116	.coerced_ty => _ = try gz.addUnNode(.ret_node, operand, node),
13117	else => unreachable,
13118	}
13119	}
13120
13121	fn addNamespaceCaptures(gz: GenZir, namespace: Scope.Namespace) !void {
13122	if (namespace.captures.count() > 0) {
13123	try gz.instructions.ensureUnusedCapacity(gz.astgen.gpa, namespace.captures.count());
13124	for (namespace.captures.values()) \|capture\| {
13125	gz.instructions.appendAssumeCapacity(capture);
13126	}
13127	}
13128	}
13129
13130	fn addDbgVar(gz: *GenZir, tag: Zir.Inst.Tag, name: Zir.NullTerminatedString, inst: Zir.Inst.Ref) !void {
13131	if (gz.is_comptime) return;
13132
13133	_ = try gz.add(.{ .tag = tag, .data = .{
13134	.str_op = .{
13135	.str = name,
13136	.operand = inst,
13137	},
13138	} });
13139	}
13140	};
13141
13142	/// This can only be for short-lived references; the memory becomes invalidated
13143	/// when another string is added.
13144	fn nullTerminatedString(astgen: AstGen, index: Zir.NullTerminatedString) [*:0]const u8 {
13145	return @ptrCast(astgen.string_bytes.items[@intFromEnum(index)..]);
13146	}
13147
13148	/// Local variables shadowing detection, including function parameters.
13149	fn detectLocalShadowing(
13150	astgen: *AstGen,
13151	scope: *Scope,
13152	ident_name: Zir.NullTerminatedString,
13153	name_token: Ast.TokenIndex,
13154	token_bytes: []const u8,
13155	id_cat: Scope.IdCat,
13156	) !void {
13157	const gpa = astgen.gpa;
13158	if (token_bytes[0] != '@' and isPrimitive(token_bytes)) {
13159	return astgen.failTokNotes(name_token, "name shadows primitive '{s}'", .{
13160	token_bytes,
13161	}, &[_]u32{
13162	try astgen.errNoteTok(name_token, "consider using @\"{s}\" to disambiguate", .{
13163	token_bytes,
13164	}),
13165	});
13166	}
13167
13168	var s = scope;
13169	var outer_scope = false;
13170	while (true) switch (s.tag) {
13171	.local_val => {
13172	const local_val = s.cast(Scope.LocalVal).?;
13173	if (local_val.name == ident_name) {
13174	const name_slice = mem.span(astgen.nullTerminatedString(ident_name));
13175	const name = try gpa.dupe(u8, name_slice);
13176	defer gpa.free(name);
13177	if (outer_scope) {
13178	return astgen.failTokNotes(name_token, "{s} '{s}' shadows {s} from outer scope", .{
13179	@tagName(id_cat), name, @tagName(local_val.id_cat),
13180	}, &[_]u32{
13181	try astgen.errNoteTok(
13182	local_val.token_src,
13183	"previous declaration here",
13184	.{},
13185	),
13186	});
13187	}
13188	return astgen.failTokNotes(name_token, "redeclaration of {s} '{s}'", .{
13189	@tagName(local_val.id_cat), name,
13190	}, &[_]u32{
13191	try astgen.errNoteTok(
13192	local_val.token_src,
13193	"previous declaration here",
13194	.{},
13195	),
13196	});
13197	}
13198	s = local_val.parent;
13199	},
13200	.local_ptr => {
13201	const local_ptr = s.cast(Scope.LocalPtr).?;
13202	if (local_ptr.name == ident_name) {
13203	const name_slice = mem.span(astgen.nullTerminatedString(ident_name));
13204	const name = try gpa.dupe(u8, name_slice);
13205	defer gpa.free(name);
13206	if (outer_scope) {
13207	return astgen.failTokNotes(name_token, "{s} '{s}' shadows {s} from outer scope", .{
13208	@tagName(id_cat), name, @tagName(local_ptr.id_cat),
13209	}, &[_]u32{
13210	try astgen.errNoteTok(
13211	local_ptr.token_src,
13212	"previous declaration here",
13213	.{},
13214	),
13215	});
13216	}
13217	return astgen.failTokNotes(name_token, "redeclaration of {s} '{s}'", .{
13218	@tagName(local_ptr.id_cat), name,
13219	}, &[_]u32{
13220	try astgen.errNoteTok(
13221	local_ptr.token_src,
13222	"previous declaration here",
13223	.{},
13224	),
13225	});
13226	}
13227	s = local_ptr.parent;
13228	},
13229	.namespace, .enum_namespace => {
13230	outer_scope = true;
13231	const ns = s.cast(Scope.Namespace).?;
13232	const decl_node = ns.decls.get(ident_name) orelse {
13233	s = ns.parent;
13234	continue;
13235	};
13236	const name_slice = mem.span(astgen.nullTerminatedString(ident_name));
13237	const name = try gpa.dupe(u8, name_slice);
13238	defer gpa.free(name);
13239	return astgen.failTokNotes(name_token, "{s} shadows declaration of '{s}'", .{
13240	@tagName(id_cat), name,
13241	}, &[_]u32{
13242	try astgen.errNoteNode(decl_node, "declared here", .{}),
13243	});
13244	},
13245	.gen_zir => {
13246	s = s.cast(GenZir).?.parent;
13247	outer_scope = true;
13248	},
13249	.defer_normal, .defer_error => s = s.cast(Scope.Defer).?.parent,
13250	.top => break,
13251	};
13252	}
13253
13254	const LineColumn = struct { u32, u32 };
13255
13256	/// Advances the source cursor to the main token of `node` if not in comptime scope.
13257	/// Usually paired with `emitDbgStmt`.
13258	fn maybeAdvanceSourceCursorToMainToken(gz: *GenZir, node: Ast.Node.Index) LineColumn {
13259	if (gz.is_comptime) return .{ gz.astgen.source_line - gz.decl_line, gz.astgen.source_column };
13260
13261	const tree = gz.astgen.tree;
13262	const token_starts = tree.tokens.items(.start);
13263	const main_tokens = tree.nodes.items(.main_token);
13264	const node_start = token_starts[main_tokens[node]];
13265	gz.astgen.advanceSourceCursor(node_start);
13266
13267	return .{ gz.astgen.source_line - gz.decl_line, gz.astgen.source_column };
13268	}
13269
13270	/// Advances the source cursor to the beginning of `node`.
13271	fn advanceSourceCursorToNode(astgen: *AstGen, node: Ast.Node.Index) void {
13272	const tree = astgen.tree;
13273	const token_starts = tree.tokens.items(.start);
13274	const node_start = token_starts[tree.firstToken(node)];
13275	astgen.advanceSourceCursor(node_start);
13276	}
13277
13278	/// Advances the source cursor to an absolute byte offset `end` in the file.
13279	fn advanceSourceCursor(astgen: *AstGen, end: usize) void {
13280	const source = astgen.tree.source;
13281	var i = astgen.source_offset;
13282	var line = astgen.source_line;
13283	var column = astgen.source_column;
13284	assert(i <= end);
13285	while (i < end) : (i += 1) {
13286	if (source[i] == '\n') {
13287	line += 1;
13288	column = 0;
13289	} else {
13290	column += 1;
13291	}
13292	}
13293	astgen.source_offset = i;
13294	astgen.source_line = line;
13295	astgen.source_column = column;
13296	}
13297
13298	fn scanDecls(astgen: AstGen, namespace: Scope.Namespace, members: []const Ast.Node.Index) !u32 {
13299	const gpa = astgen.gpa;
13300	const tree = astgen.tree;
13301	const node_tags = tree.nodes.items(.tag);
13302	const main_tokens = tree.nodes.items(.main_token);
13303	const token_tags = tree.tokens.items(.tag);
13304	var decl_count: u32 = 0;
13305	for (members) \|member_node\| {
13306	const name_token = switch (node_tags[member_node]) {
13307	.global_var_decl,
13308	.local_var_decl,
13309	.simple_var_decl,
13310	.aligned_var_decl,
13311	=> blk: {
13312	decl_count += 1;
13313	break :blk main_tokens[member_node] + 1;
13314	},
13315
13316	.fn_proto_simple,
13317	.fn_proto_multi,
13318	.fn_proto_one,
13319	.fn_proto,
13320	.fn_decl,
13321	=> blk: {
13322	decl_count += 1;
13323	const ident = main_tokens[member_node] + 1;
13324	if (token_tags[ident] != .identifier) {
13325	switch (astgen.failNode(member_node, "missing function name", .{})) {
13326	error.AnalysisFail => continue,
13327	error.OutOfMemory => return error.OutOfMemory,
13328	}
13329	}
13330	break :blk ident;
13331	},
13332
13333	.@"comptime", .@"usingnamespace", .test_decl => {
13334	decl_count += 1;
13335	continue;
13336	},
13337
13338	else => continue,
13339	};
13340
13341	const token_bytes = astgen.tree.tokenSlice(name_token);
13342	if (token_bytes[0] != '@' and isPrimitive(token_bytes)) {
13343	switch (astgen.failTokNotes(name_token, "name shadows primitive '{s}'", .{
13344	token_bytes,
13345	}, &[_]u32{
13346	try astgen.errNoteTok(name_token, "consider using @\"{s}\" to disambiguate", .{
13347	token_bytes,
13348	}),
13349	})) {
13350	error.AnalysisFail => continue,
13351	error.OutOfMemory => return error.OutOfMemory,
13352	}
13353	}
13354
13355	const name_str_index = try astgen.identAsString(name_token);
13356	const gop = try namespace.decls.getOrPut(gpa, name_str_index);
13357	if (gop.found_existing) {
13358	const name = try gpa.dupe(u8, mem.span(astgen.nullTerminatedString(name_str_index)));
13359	defer gpa.free(name);
13360	switch (astgen.failNodeNotes(member_node, "redeclaration of '{s}'", .{
13361	name,
13362	}, &[_]u32{
13363	try astgen.errNoteNode(gop.value_ptr.*, "other declaration here", .{}),
13364	})) {
13365	error.AnalysisFail => continue,
13366	error.OutOfMemory => return error.OutOfMemory,
13367	}
13368	}
13369
13370	var s = namespace.parent;
13371	while (true) switch (s.tag) {
13372	.local_val => {
13373	const local_val = s.cast(Scope.LocalVal).?;
13374	if (local_val.name == name_str_index) {
13375	return astgen.failTokNotes(name_token, "declaration '{s}' shadows {s} from outer scope", .{
13376	token_bytes, @tagName(local_val.id_cat),
13377	}, &[_]u32{
13378	try astgen.errNoteTok(
13379	local_val.token_src,
13380	"previous declaration here",
13381	.{},
13382	),
13383	});
13384	}
13385	s = local_val.parent;
13386	},
13387	.local_ptr => {
13388	const local_ptr = s.cast(Scope.LocalPtr).?;
13389	if (local_ptr.name == name_str_index) {
13390	return astgen.failTokNotes(name_token, "declaration '{s}' shadows {s} from outer scope", .{
13391	token_bytes, @tagName(local_ptr.id_cat),
13392	}, &[_]u32{
13393	try astgen.errNoteTok(
13394	local_ptr.token_src,
13395	"previous declaration here",
13396	.{},
13397	),
13398	});
13399	}
13400	s = local_ptr.parent;
13401	},
13402	.namespace, .enum_namespace => s = s.cast(Scope.Namespace).?.parent,
13403	.gen_zir => s = s.cast(GenZir).?.parent,
13404	.defer_normal, .defer_error => s = s.cast(Scope.Defer).?.parent,
13405	.top => break,
13406	};
13407	gop.value_ptr.* = member_node;
13408	}
13409	return decl_count;
13410	}
13411
13412	fn isInferred(astgen: *AstGen, ref: Zir.Inst.Ref) bool {
13413	const inst = ref.toIndex() orelse return false;
13414	const zir_tags = astgen.instructions.items(.tag);
13415	return switch (zir_tags[@intFromEnum(inst)]) {
13416	.alloc_inferred,
13417	.alloc_inferred_mut,
13418	.alloc_inferred_comptime,
13419	.alloc_inferred_comptime_mut,
13420	=> true,
13421
13422	.extended => {
13423	const zir_data = astgen.instructions.items(.data);
13424	if (zir_data[@intFromEnum(inst)].extended.opcode != .alloc) return false;
13425	const small: Zir.Inst.AllocExtended.Small = @bitCast(zir_data[@intFromEnum(inst)].extended.small);
13426	return !small.has_type;
13427	},
13428
13429	else => false,
13430	};
13431	}
13432
13433	/// Assumes capacity for body has already been added. Needed capacity taking into
13434	/// account fixups can be found with `countBodyLenAfterFixups`.
13435	fn appendBodyWithFixups(astgen: *AstGen, body: []const Zir.Inst.Index) void {
13436	return appendBodyWithFixupsArrayList(astgen, &astgen.extra, body);
13437	}
13438
13439	fn appendBodyWithFixupsArrayList(
13440	astgen: *AstGen,
13441	list: *std.ArrayListUnmanaged(u32),
13442	body: []const Zir.Inst.Index,
13443	) void {
13444	for (body) \|body_inst\| {
13445	appendPossiblyRefdBodyInst(astgen, list, body_inst);
13446	}
13447	}
13448
13449	fn appendPossiblyRefdBodyInst(
13450	astgen: *AstGen,
13451	list: *std.ArrayListUnmanaged(u32),
13452	body_inst: Zir.Inst.Index,
13453	) void {
13454	list.appendAssumeCapacity(@intFromEnum(body_inst));
13455	const kv = astgen.ref_table.fetchRemove(body_inst) orelse return;
13456	const ref_inst = kv.value;
13457	return appendPossiblyRefdBodyInst(astgen, list, ref_inst);
13458	}
13459
13460	fn countBodyLenAfterFixups(astgen: *AstGen, body: []const Zir.Inst.Index) u32 {
13461	var count = body.len;
13462	for (body) \|body_inst\| {
13463	var check_inst = body_inst;
13464	while (astgen.ref_table.get(check_inst)) \|ref_inst\| {
13465	count += 1;
13466	check_inst = ref_inst;
13467	}
13468	}
13469	return @intCast(count);
13470	}
13471
13472	fn emitDbgStmt(gz: *GenZir, lc: LineColumn) !void {
13473	if (gz.is_comptime) return;
13474	if (gz.instructions.items.len > 0) {
13475	const astgen = gz.astgen;
13476	const last = gz.instructions.items[gz.instructions.items.len - 1];
13477	if (astgen.instructions.items(.tag)[@intFromEnum(last)] == .dbg_stmt) {
13478	astgen.instructions.items(.data)[@intFromEnum(last)].dbg_stmt = .{
13479	.line = lc[0],
13480	.column = lc[1],
13481	};
13482	return;
13483	}
13484	}
13485
13486	_ = try gz.add(.{ .tag = .dbg_stmt, .data = .{
13487	.dbg_stmt = .{
13488	.line = lc[0],
13489	.column = lc[1],
13490	},
13491	} });
13492	}
13493
13494	/// In some cases, Sema expects us to generate a `dbg_stmt` at the instruction
13495	/// index directly preceding the next instruction (e.g. if a call is %10, it
13496	/// expects a dbg_stmt at %9). TODO: this logic may allow redundant dbg_stmt
13497	/// instructions; fix up Sema so we don't need it!
13498	fn emitDbgStmtForceCurrentIndex(gz: *GenZir, lc: LineColumn) !void {
13499	const astgen = gz.astgen;
13500	if (gz.instructions.items.len > 0 and
13501	@intFromEnum(gz.instructions.items[gz.instructions.items.len - 1]) == astgen.instructions.len - 1)
13502	{
13503	const last = astgen.instructions.len - 1;
13504	if (astgen.instructions.items(.tag)[last] == .dbg_stmt) {
13505	astgen.instructions.items(.data)[last].dbg_stmt = .{
13506	.line = lc[0],
13507	.column = lc[1],
13508	};
13509	return;
13510	}
13511	}
13512
13513	_ = try gz.add(.{ .tag = .dbg_stmt, .data = .{
13514	.dbg_stmt = .{
13515	.line = lc[0],
13516	.column = lc[1],
13517	},
13518	} });
13519	}
13520
13521	fn lowerAstErrors(astgen: *AstGen) !void {
13522	const tree = astgen.tree;
13523	assert(tree.errors.len > 0);
13524
13525	const gpa = astgen.gpa;
13526	const parse_err = tree.errors[0];
13527
13528	var msg: std.ArrayListUnmanaged(u8) = .{};
13529	defer msg.deinit(gpa);
13530
13531	const token_starts = tree.tokens.items(.start);
13532	const token_tags = tree.tokens.items(.tag);
13533
13534	var notes: std.ArrayListUnmanaged(u32) = .{};
13535	defer notes.deinit(gpa);
13536
13537	if (token_tags[parse_err.token + @intFromBool(parse_err.token_is_prev)] == .invalid) {
13538	const tok = parse_err.token + @intFromBool(parse_err.token_is_prev);
13539	const bad_off: u32 = @intCast(tree.tokenSlice(parse_err.token + @intFromBool(parse_err.token_is_prev)).len);
13540	const byte_abs = token_starts[parse_err.token + @intFromBool(parse_err.token_is_prev)] + bad_off;
13541	try notes.append(gpa, try astgen.errNoteTokOff(tok, bad_off, "invalid byte: '{'}'", .{
13542	std.zig.fmtEscapes(tree.source[byte_abs..][0..1]),
13543	}));
13544	}
13545
13546	for (tree.errors[1..]) \|note\| {
13547	if (!note.is_note) break;
13548
13549	msg.clearRetainingCapacity();
13550	try tree.renderError(note, msg.writer(gpa));
13551	try notes.append(gpa, try astgen.errNoteTok(note.token, "{s}", .{msg.items}));
13552	}
13553
13554	const extra_offset = tree.errorOffset(parse_err);
13555	msg.clearRetainingCapacity();
13556	try tree.renderError(parse_err, msg.writer(gpa));
13557	try astgen.appendErrorTokNotesOff(parse_err.token, extra_offset, "{s}", .{msg.items}, notes.items);
13558	}
13559
13560	const DeclarationName = union(enum) {
13561	named: Ast.TokenIndex,
13562	named_test: Ast.TokenIndex,
13563	unnamed_test,
13564	decltest: Zir.NullTerminatedString,
13565	@"comptime",
13566	@"usingnamespace",
13567	};
13568
13569	/// Sets all extra data for a `declaration` instruction.
13570	/// Unstacks `value_gz`, `align_gz`, `linksection_gz`, and `addrspace_gz`.
13571	fn setDeclaration(
13572	decl_inst: Zir.Inst.Index,
13573	src_hash: std.zig.SrcHash,
13574	name: DeclarationName,
13575	line_offset: u32,
13576	is_pub: bool,
13577	is_export: bool,
13578	doc_comment: Zir.NullTerminatedString,
13579	value_gz: *GenZir,
13580	/// May be `null` if all these blocks would be empty.
13581	/// If `null`, then `value_gz` must have nothing stacked on it.
13582	extra_gzs: ?struct {
13583	/// Must be stacked on `value_gz`.
13584	align_gz: *GenZir,
13585	/// Must be stacked on `align_gz`.
13586	linksection_gz: *GenZir,
13587	/// Must be stacked on `linksection_gz`, and have nothing stacked on it.
13588	addrspace_gz: *GenZir,
13589	},
13590	) !void {
13591	const astgen = value_gz.astgen;
13592	const gpa = astgen.gpa;
13593
13594	const empty_body: []Zir.Inst.Index = &.{};
13595	const value_body, const align_body, const linksection_body, const addrspace_body = if (extra_gzs) \|e\| .{
13596	value_gz.instructionsSliceUpto(e.align_gz),
13597	e.align_gz.instructionsSliceUpto(e.linksection_gz),
13598	e.linksection_gz.instructionsSliceUpto(e.addrspace_gz),
13599	e.addrspace_gz.instructionsSlice(),
13600	} else .{ value_gz.instructionsSlice(), empty_body, empty_body, empty_body };
13601
13602	const value_len = astgen.countBodyLenAfterFixups(value_body);
13603	const align_len = astgen.countBodyLenAfterFixups(align_body);
13604	const linksection_len = astgen.countBodyLenAfterFixups(linksection_body);
13605	const addrspace_len = astgen.countBodyLenAfterFixups(addrspace_body);
13606
13607	const true_doc_comment: Zir.NullTerminatedString = switch (name) {
13608	.decltest => \|test_name\| test_name,
13609	else => doc_comment,
13610	};
13611
13612	const src_hash_arr: [4]u32 = @bitCast(src_hash);
13613
13614	const extra: Zir.Inst.Declaration = .{
13615	.src_hash_0 = src_hash_arr[0],
13616	.src_hash_1 = src_hash_arr[1],
13617	.src_hash_2 = src_hash_arr[2],
13618	.src_hash_3 = src_hash_arr[3],
13619	.name = switch (name) {
13620	.named => \|tok\| @enumFromInt(@intFromEnum(try astgen.identAsString(tok))),
13621	.named_test => \|tok\| @enumFromInt(@intFromEnum(try astgen.testNameString(tok))),
13622	.unnamed_test => .unnamed_test,
13623	.decltest => .decltest,
13624	.@"comptime" => .@"comptime",
13625	.@"usingnamespace" => .@"usingnamespace",
13626	},
13627	.line_offset = line_offset,
13628	.flags = .{
13629	.value_body_len = @intCast(value_len),
13630	.is_pub = is_pub,
13631	.is_export = is_export,
13632	.has_doc_comment = true_doc_comment != .empty,
13633	.has_align_linksection_addrspace = align_len != 0 or linksection_len != 0 or addrspace_len != 0,
13634	},
13635	};
13636	astgen.instructions.items(.data)[@intFromEnum(decl_inst)].pl_node.payload_index = try astgen.addExtra(extra);
13637	if (extra.flags.has_doc_comment) {
13638	try astgen.extra.append(gpa, @intFromEnum(true_doc_comment));
13639	}
13640	if (extra.flags.has_align_linksection_addrspace) {
13641	try astgen.extra.appendSlice(gpa, &.{
13642	align_len,
13643	linksection_len,
13644	addrspace_len,
13645	});
13646	}
13647	try astgen.extra.ensureUnusedCapacity(gpa, value_len + align_len + linksection_len + addrspace_len);
13648	astgen.appendBodyWithFixups(value_body);
13649	if (extra.flags.has_align_linksection_addrspace) {
13650	astgen.appendBodyWithFixups(align_body);
13651	astgen.appendBodyWithFixups(linksection_body);
13652	astgen.appendBodyWithFixups(addrspace_body);
13653	}
13654
13655	if (extra_gzs) \|e\| {
13656	e.addrspace_gz.unstack();
13657	e.linksection_gz.unstack();
13658	e.align_gz.unstack();
13659	}
13660	value_gz.unstack();
13661	}

src/AstGen.zig deleted-13661

...	@@ -1,13661 +0,0 @@
1	//! Ingests an AST and produces ZIR code.
2	const AstGen = @This();
3
4	const std = @import("std");
5	const Ast = std.zig.Ast;
6	const mem = std.mem;
7	const Allocator = std.mem.Allocator;
8	const assert = std.debug.assert;
9	const ArrayListUnmanaged = std.ArrayListUnmanaged;
10	const StringIndexAdapter = std.hash_map.StringIndexAdapter;
11	const StringIndexContext = std.hash_map.StringIndexContext;
12
13	const isPrimitive = std.zig.primitives.isPrimitive;
14
15	const Zir = std.zig.Zir;
16	const BuiltinFn = std.zig.BuiltinFn;
17	const AstRlAnnotate = std.zig.AstRlAnnotate;
18
19	gpa: Allocator,
20	tree: *const Ast,
21	/// The set of nodes which, given the choice, must expose a result pointer to
22	/// sub-expressions. See `AstRlAnnotate` for details.
23	nodes_need_rl: *const AstRlAnnotate.RlNeededSet,
24	instructions: std.MultiArrayList(Zir.Inst) = .{},
25	extra: ArrayListUnmanaged(u32) = .{},
26	string_bytes: ArrayListUnmanaged(u8) = .{},
27	/// Tracks the current byte offset within the source file.
28	/// Used to populate line deltas in the ZIR. AstGen maintains
29	/// this "cursor" throughout the entire AST lowering process in order
30	/// to avoid starting over the line/column scan for every declaration, which
31	/// would be O(N^2).
32	source_offset: u32 = 0,
33	/// Tracks the corresponding line of `source_offset`.
34	/// This value is absolute.
35	source_line: u32 = 0,
36	/// Tracks the corresponding column of `source_offset`.
37	/// This value is absolute.
38	source_column: u32 = 0,
39	/// Used for temporary allocations; freed after AstGen is complete.
40	/// The resulting ZIR code has no references to anything in this arena.
41	arena: Allocator,
42	string_table: std.HashMapUnmanaged(u32, void, StringIndexContext, std.hash_map.default_max_load_percentage) = .{},
43	compile_errors: ArrayListUnmanaged(Zir.Inst.CompileErrors.Item) = .{},
44	/// The topmost block of the current function.
45	fn_block: ?*GenZir = null,
46	fn_var_args: bool = false,
47	/// The return type of the current function. This may be a trivial `Ref`, or
48	/// otherwise it refers to a `ret_type` instruction.
49	fn_ret_ty: Zir.Inst.Ref = .none,
50	/// Maps string table indexes to the first `@import` ZIR instruction
51	/// that uses this string as the operand.
52	imports: std.AutoArrayHashMapUnmanaged(Zir.NullTerminatedString, Ast.TokenIndex) = .{},
53	/// Used for temporary storage when building payloads.
54	scratch: std.ArrayListUnmanaged(u32) = .{},
55	/// Whenever a `ref` instruction is needed, it is created and saved in this
56	/// table instead of being immediately appended to the current block body.
57	/// Then, when the instruction is being added to the parent block (typically from
58	/// setBlockBody), if it has a ref_table entry, then the ref instruction is added
59	/// there. This makes sure two properties are upheld:
60	/// 1. All pointers to the same locals return the same address. This is required
61	/// to be compliant with the language specification.
62	/// 2. `ref` instructions will dominate their uses. This is a required property
63	/// of ZIR.
64	/// The key is the ref operand; the value is the ref instruction.
65	ref_table: std.AutoHashMapUnmanaged(Zir.Inst.Index, Zir.Inst.Index) = .{},
66
67	const InnerError = error{ OutOfMemory, AnalysisFail };
68
69	fn addExtra(astgen: *AstGen, extra: anytype) Allocator.Error!u32 {
70	const fields = std.meta.fields(@TypeOf(extra));
71	try astgen.extra.ensureUnusedCapacity(astgen.gpa, fields.len);
72	return addExtraAssumeCapacity(astgen, extra);
73	}
74
75	fn addExtraAssumeCapacity(astgen: *AstGen, extra: anytype) u32 {
76	const fields = std.meta.fields(@TypeOf(extra));
77	const extra_index: u32 = @intCast(astgen.extra.items.len);
78	astgen.extra.items.len += fields.len;
79	setExtra(astgen, extra_index, extra);
80	return extra_index;
81	}
82
83	fn setExtra(astgen: *AstGen, index: usize, extra: anytype) void {
84	const fields = std.meta.fields(@TypeOf(extra));
85	var i = index;
86	inline for (fields) \|field\| {
87	astgen.extra.items[i] = switch (field.type) {
88	u32 => @field(extra, field.name),
89
90	Zir.Inst.Ref,
91	Zir.Inst.Index,
92	Zir.Inst.Declaration.Name,
93	Zir.NullTerminatedString,
94	=> @intFromEnum(@field(extra, field.name)),
95
96	i32,
97	Zir.Inst.Call.Flags,
98	Zir.Inst.BuiltinCall.Flags,
99	Zir.Inst.SwitchBlock.Bits,
100	Zir.Inst.SwitchBlockErrUnion.Bits,
101	Zir.Inst.FuncFancy.Bits,
102	Zir.Inst.Declaration.Flags,
103	=> @bitCast(@field(extra, field.name)),
104
105	else => @compileError("bad field type"),
106	};
107	i += 1;
108	}
109	}
110
111	fn reserveExtra(astgen: *AstGen, size: usize) Allocator.Error!u32 {
112	const extra_index: u32 = @intCast(astgen.extra.items.len);
113	try astgen.extra.resize(astgen.gpa, extra_index + size);
114	return extra_index;
115	}
116
117	fn appendRefs(astgen: *AstGen, refs: []const Zir.Inst.Ref) !void {
118	return astgen.extra.appendSlice(astgen.gpa, @ptrCast(refs));
119	}
120
121	fn appendRefsAssumeCapacity(astgen: *AstGen, refs: []const Zir.Inst.Ref) void {
122	astgen.extra.appendSliceAssumeCapacity(@ptrCast(refs));
123	}
124
125	pub fn generate(gpa: Allocator, tree: Ast) Allocator.Error!Zir {
126	var arena = std.heap.ArenaAllocator.init(gpa);
127	defer arena.deinit();
128
129	var nodes_need_rl = try AstRlAnnotate.annotate(gpa, arena.allocator(), tree);
130	defer nodes_need_rl.deinit(gpa);
131
132	var astgen: AstGen = .{
133	.gpa = gpa,
134	.arena = arena.allocator(),
135	.tree = &tree,
136	.nodes_need_rl = &nodes_need_rl,
137	};
138	defer astgen.deinit(gpa);
139
140	// String table index 0 is reserved for `NullTerminatedString.empty`.
141	try astgen.string_bytes.append(gpa, 0);
142
143	// We expect at least as many ZIR instructions and extra data items
144	// as AST nodes.
145	try astgen.instructions.ensureTotalCapacity(gpa, tree.nodes.len);
146
147	// First few indexes of extra are reserved and set at the end.
148	const reserved_count = @typeInfo(Zir.ExtraIndex).Enum.fields.len;
149	try astgen.extra.ensureTotalCapacity(gpa, tree.nodes.len + reserved_count);
150	astgen.extra.items.len += reserved_count;
151
152	var top_scope: Scope.Top = .{};
153
154	var gz_instructions: std.ArrayListUnmanaged(Zir.Inst.Index) = .{};
155	var gen_scope: GenZir = .{
156	.is_comptime = true,
157	.parent = &top_scope.base,
158	.anon_name_strategy = .parent,
159	.decl_node_index = 0,
160	.decl_line = 0,
161	.astgen = &astgen,
162	.instructions = &gz_instructions,
163	.instructions_top = 0,
164	};
165	defer gz_instructions.deinit(gpa);
166
167	// The AST -> ZIR lowering process assumes an AST that does not have any
168	// parse errors.
169	if (tree.errors.len == 0) {
170	if (AstGen.structDeclInner(
171	&gen_scope,
172	&gen_scope.base,
173	0,
174	tree.containerDeclRoot(),
175	.Auto,
176	0,
177	)) \|struct_decl_ref\| {
178	assert(struct_decl_ref.toIndex().? == .main_struct_inst);
179	} else \|err\| switch (err) {
180	error.OutOfMemory => return error.OutOfMemory,
181	error.AnalysisFail => {}, // Handled via compile_errors below.
182	}
183	} else {
184	try lowerAstErrors(&astgen);
185	}
186
187	const err_index = @intFromEnum(Zir.ExtraIndex.compile_errors);
188	if (astgen.compile_errors.items.len == 0) {
189	astgen.extra.items[err_index] = 0;
190	} else {
191	try astgen.extra.ensureUnusedCapacity(gpa, 1 + astgen.compile_errors.items.len *
192	@typeInfo(Zir.Inst.CompileErrors.Item).Struct.fields.len);
193
194	astgen.extra.items[err_index] = astgen.addExtraAssumeCapacity(Zir.Inst.CompileErrors{
195	.items_len = @intCast(astgen.compile_errors.items.len),
196	});
197
198	for (astgen.compile_errors.items) \|item\| {
199	_ = astgen.addExtraAssumeCapacity(item);
200	}
201	}
202
203	const imports_index = @intFromEnum(Zir.ExtraIndex.imports);
204	if (astgen.imports.count() == 0) {
205	astgen.extra.items[imports_index] = 0;
206	} else {
207	try astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.Imports).Struct.fields.len +
208	astgen.imports.count() * @typeInfo(Zir.Inst.Imports.Item).Struct.fields.len);
209
210	astgen.extra.items[imports_index] = astgen.addExtraAssumeCapacity(Zir.Inst.Imports{
211	.imports_len = @intCast(astgen.imports.count()),
212	});
213
214	var it = astgen.imports.iterator();
215	while (it.next()) \|entry\| {
216	_ = astgen.addExtraAssumeCapacity(Zir.Inst.Imports.Item{
217	.name = entry.key_ptr.*,
218	.token = entry.value_ptr.*,
219	});
220	}
221	}
222
223	return Zir{
224	.instructions = astgen.instructions.toOwnedSlice(),
225	.string_bytes = try astgen.string_bytes.toOwnedSlice(gpa),
226	.extra = try astgen.extra.toOwnedSlice(gpa),
227	};
228	}
229
230	fn deinit(astgen: *AstGen, gpa: Allocator) void {
231	astgen.instructions.deinit(gpa);
232	astgen.extra.deinit(gpa);
233	astgen.string_table.deinit(gpa);
234	astgen.string_bytes.deinit(gpa);
235	astgen.compile_errors.deinit(gpa);
236	astgen.imports.deinit(gpa);
237	astgen.scratch.deinit(gpa);
238	astgen.ref_table.deinit(gpa);
239	}
240
241	const ResultInfo = struct {
242	/// The semantics requested for the result location
243	rl: Loc,
244
245	/// The "operator" consuming the result location
246	ctx: Context = .none,
247
248	/// Turns a `coerced_ty` back into a `ty`. Should be called at branch points
249	/// such as if and switch expressions.
250	fn br(ri: ResultInfo) ResultInfo {
251	return switch (ri.rl) {
252	.coerced_ty => \|ty\| .{
253	.rl = .{ .ty = ty },
254	.ctx = ri.ctx,
255	},
256	else => ri,
257	};
258	}
259
260	fn zirTag(ri: ResultInfo) Zir.Inst.Tag {
261	switch (ri.rl) {
262	.ty => return switch (ri.ctx) {
263	.shift_op => .as_shift_operand,
264	else => .as_node,
265	},
266	else => unreachable,
267	}
268	}
269
270	const Loc = union(enum) {
271	/// The expression is the right-hand side of assignment to `_`. Only the side-effects of the
272	/// expression should be generated. The result instruction from the expression must
273	/// be ignored.
274	discard,
275	/// The expression has an inferred type, and it will be evaluated as an rvalue.
276	none,
277	/// The expression will be coerced into this type, but it will be evaluated as an rvalue.
278	ty: Zir.Inst.Ref,
279	/// Same as `ty` but it is guaranteed that Sema will additionally perform the coercion,
280	/// so no `as` instruction needs to be emitted.
281	coerced_ty: Zir.Inst.Ref,
282	/// The expression must generate a pointer rather than a value. For example, the left hand side
283	/// of an assignment uses this kind of result location.
284	ref,
285	/// The expression must generate a pointer rather than a value, and the pointer will be coerced
286	/// by other code to this type, which is guaranteed by earlier instructions to be a pointer type.
287	ref_coerced_ty: Zir.Inst.Ref,
288	/// The expression must store its result into this typed pointer. The result instruction
289	/// from the expression must be ignored.
290	ptr: PtrResultLoc,
291	/// The expression must store its result into this allocation, which has an inferred type.
292	/// The result instruction from the expression must be ignored.
293	/// Always an instruction with tag `alloc_inferred`.
294	inferred_ptr: Zir.Inst.Ref,
295	/// The expression has a sequence of pointers to store its results into due to a destructure
296	/// operation. Each of these pointers may or may not have an inferred type.
297	destructure: struct {
298	/// The AST node of the destructure operation itself.
299	src_node: Ast.Node.Index,
300	/// The pointers to store results into.
301	components: []const DestructureComponent,
302	},
303
304	const DestructureComponent = union(enum) {
305	typed_ptr: PtrResultLoc,
306	inferred_ptr: Zir.Inst.Ref,
307	discard,
308	};
309
310	const PtrResultLoc = struct {
311	inst: Zir.Inst.Ref,
312	src_node: ?Ast.Node.Index = null,
313	};
314
315	/// Find the result type for a cast builtin given the result location.
316	/// If the location does not have a known result type, emits an error on
317	/// the given node.
318	fn resultType(rl: Loc, gz: *GenZir, node: Ast.Node.Index) !?Zir.Inst.Ref {
319	return switch (rl) {
320	.discard, .none, .ref, .inferred_ptr, .destructure => null,
321	.ty, .coerced_ty => \|ty_ref\| ty_ref,
322	.ref_coerced_ty => \|ptr_ty\| try gz.addUnNode(.elem_type, ptr_ty, node),
323	.ptr => \|ptr\| {
324	const ptr_ty = try gz.addUnNode(.typeof, ptr.inst, node);
325	return try gz.addUnNode(.elem_type, ptr_ty, node);
326	},
327	};
328	}
329
330	fn resultTypeForCast(rl: Loc, gz: *GenZir, node: Ast.Node.Index, builtin_name: []const u8) !Zir.Inst.Ref {
331	const astgen = gz.astgen;
332	if (try rl.resultType(gz, node)) \|ty\| return ty;
333	switch (rl) {
334	.destructure => \|destructure\| return astgen.failNodeNotes(node, "{s} must have a known result type", .{builtin_name}, &.{
335	try astgen.errNoteNode(destructure.src_node, "destructure expressions do not provide a single result type", .{}),
336	try astgen.errNoteNode(node, "use @as to provide explicit result type", .{}),
337	}),
338	else => return astgen.failNodeNotes(node, "{s} must have a known result type", .{builtin_name}, &.{
339	try astgen.errNoteNode(node, "use @as to provide explicit result type", .{}),
340	}),
341	}
342	}
343	};
344
345	const Context = enum {
346	/// The expression is the operand to a return expression.
347	@"return",
348	/// The expression is the input to an error-handling operator (if-else, try, or catch).
349	error_handling_expr,
350	/// The expression is the right-hand side of a shift operation.
351	shift_op,
352	/// The expression is an argument in a function call.
353	fn_arg,
354	/// The expression is the right-hand side of an initializer for a `const` variable
355	const_init,
356	/// The expression is the right-hand side of an assignment expression.
357	assignment,
358	/// No specific operator in particular.
359	none,
360	};
361	};
362
363	const coerced_align_ri: ResultInfo = .{ .rl = .{ .coerced_ty = .u29_type } };
364	const coerced_addrspace_ri: ResultInfo = .{ .rl = .{ .coerced_ty = .address_space_type } };
365	const coerced_linksection_ri: ResultInfo = .{ .rl = .{ .coerced_ty = .slice_const_u8_type } };
366	const coerced_type_ri: ResultInfo = .{ .rl = .{ .coerced_ty = .type_type } };
367	const coerced_bool_ri: ResultInfo = .{ .rl = .{ .coerced_ty = .bool_type } };
368
369	fn typeExpr(gz: GenZir, scope: Scope, type_node: Ast.Node.Index) InnerError!Zir.Inst.Ref {
370	return comptimeExpr(gz, scope, coerced_type_ri, type_node);
371	}
372
373	fn reachableTypeExpr(
374	gz: *GenZir,
375	scope: *Scope,
376	type_node: Ast.Node.Index,
377	reachable_node: Ast.Node.Index,
378	) InnerError!Zir.Inst.Ref {
379	return reachableExprComptime(gz, scope, coerced_type_ri, type_node, reachable_node, true);
380	}
381
382	/// Same as `expr` but fails with a compile error if the result type is `noreturn`.
383	fn reachableExpr(
384	gz: *GenZir,
385	scope: *Scope,
386	ri: ResultInfo,
387	node: Ast.Node.Index,
388	reachable_node: Ast.Node.Index,
389	) InnerError!Zir.Inst.Ref {
390	return reachableExprComptime(gz, scope, ri, node, reachable_node, false);
391	}
392
393	fn reachableExprComptime(
394	gz: *GenZir,
395	scope: *Scope,
396	ri: ResultInfo,
397	node: Ast.Node.Index,
398	reachable_node: Ast.Node.Index,
399	force_comptime: bool,
400	) InnerError!Zir.Inst.Ref {
401	const result_inst = if (force_comptime)
402	try comptimeExpr(gz, scope, ri, node)
403	else
404	try expr(gz, scope, ri, node);
405
406	if (gz.refIsNoReturn(result_inst)) {
407	try gz.astgen.appendErrorNodeNotes(reachable_node, "unreachable code", .{}, &[_]u32{
408	try gz.astgen.errNoteNode(node, "control flow is diverted here", .{}),
409	});
410	}
411	return result_inst;
412	}
413
414	fn lvalExpr(gz: GenZir, scope: Scope, node: Ast.Node.Index) InnerError!Zir.Inst.Ref {
415	const astgen = gz.astgen;
416	const tree = astgen.tree;
417	const node_tags = tree.nodes.items(.tag);
418	const main_tokens = tree.nodes.items(.main_token);
419	switch (node_tags[node]) {
420	.root => unreachable,
421	.@"usingnamespace" => unreachable,
422	.test_decl => unreachable,
423	.global_var_decl => unreachable,
424	.local_var_decl => unreachable,
425	.simple_var_decl => unreachable,
426	.aligned_var_decl => unreachable,
427	.switch_case => unreachable,
428	.switch_case_inline => unreachable,
429	.switch_case_one => unreachable,
430	.switch_case_inline_one => unreachable,
431	.container_field_init => unreachable,
432	.container_field_align => unreachable,
433	.container_field => unreachable,
434	.asm_output => unreachable,
435	.asm_input => unreachable,
436
437	.assign,
438	.assign_destructure,
439	.assign_bit_and,
440	.assign_bit_or,
441	.assign_shl,
442	.assign_shl_sat,
443	.assign_shr,
444	.assign_bit_xor,
445	.assign_div,
446	.assign_sub,
447	.assign_sub_wrap,
448	.assign_sub_sat,
449	.assign_mod,
450	.assign_add,
451	.assign_add_wrap,
452	.assign_add_sat,
453	.assign_mul,
454	.assign_mul_wrap,
455	.assign_mul_sat,
456	.add,
457	.add_wrap,
458	.add_sat,
459	.sub,
460	.sub_wrap,
461	.sub_sat,
462	.mul,
463	.mul_wrap,
464	.mul_sat,
465	.div,
466	.mod,
467	.bit_and,
468	.bit_or,
469	.shl,
470	.shl_sat,
471	.shr,
472	.bit_xor,
473	.bang_equal,
474	.equal_equal,
475	.greater_than,
476	.greater_or_equal,
477	.less_than,
478	.less_or_equal,
479	.array_cat,
480	.array_mult,
481	.bool_and,
482	.bool_or,
483	.@"asm",
484	.asm_simple,
485	.string_literal,
486	.number_literal,
487	.call,
488	.call_comma,
489	.async_call,
490	.async_call_comma,
491	.call_one,
492	.call_one_comma,
493	.async_call_one,
494	.async_call_one_comma,
495	.unreachable_literal,
496	.@"return",
497	.@"if",
498	.if_simple,
499	.@"while",
500	.while_simple,
501	.while_cont,
502	.bool_not,
503	.address_of,
504	.optional_type,
505	.block,
506	.block_semicolon,
507	.block_two,
508	.block_two_semicolon,
509	.@"break",
510	.ptr_type_aligned,
511	.ptr_type_sentinel,
512	.ptr_type,
513	.ptr_type_bit_range,
514	.array_type,
515	.array_type_sentinel,
516	.enum_literal,
517	.multiline_string_literal,
518	.char_literal,
519	.@"defer",
520	.@"errdefer",
521	.@"catch",
522	.error_union,
523	.merge_error_sets,
524	.switch_range,
525	.for_range,
526	.@"await",
527	.bit_not,
528	.negation,
529	.negation_wrap,
530	.@"resume",
531	.@"try",
532	.slice,
533	.slice_open,
534	.slice_sentinel,
535	.array_init_one,
536	.array_init_one_comma,
537	.array_init_dot_two,
538	.array_init_dot_two_comma,
539	.array_init_dot,
540	.array_init_dot_comma,
541	.array_init,
542	.array_init_comma,
543	.struct_init_one,
544	.struct_init_one_comma,
545	.struct_init_dot_two,
546	.struct_init_dot_two_comma,
547	.struct_init_dot,
548	.struct_init_dot_comma,
549	.struct_init,
550	.struct_init_comma,
551	.@"switch",
552	.switch_comma,
553	.@"for",
554	.for_simple,
555	.@"suspend",
556	.@"continue",
557	.fn_proto_simple,
558	.fn_proto_multi,
559	.fn_proto_one,
560	.fn_proto,
561	.fn_decl,
562	.anyframe_type,
563	.anyframe_literal,
564	.error_set_decl,
565	.container_decl,
566	.container_decl_trailing,
567	.container_decl_two,
568	.container_decl_two_trailing,
569	.container_decl_arg,
570	.container_decl_arg_trailing,
571	.tagged_union,
572	.tagged_union_trailing,
573	.tagged_union_two,
574	.tagged_union_two_trailing,
575	.tagged_union_enum_tag,
576	.tagged_union_enum_tag_trailing,
577	.@"comptime",
578	.@"nosuspend",
579	.error_value,
580	=> return astgen.failNode(node, "invalid left-hand side to assignment", .{}),
581
582	.builtin_call,
583	.builtin_call_comma,
584	.builtin_call_two,
585	.builtin_call_two_comma,
586	=> {
587	const builtin_token = main_tokens[node];
588	const builtin_name = tree.tokenSlice(builtin_token);
589	// If the builtin is an invalid name, we don't cause an error here; instead
590	// let it pass, and the error will be "invalid builtin function" later.
591	if (BuiltinFn.list.get(builtin_name)) \|info\| {
592	if (!info.allows_lvalue) {
593	return astgen.failNode(node, "invalid left-hand side to assignment", .{});
594	}
595	}
596	},
597
598	// These can be assigned to.
599	.unwrap_optional,
600	.deref,
601	.field_access,
602	.array_access,
603	.identifier,
604	.grouped_expression,
605	.@"orelse",
606	=> {},
607	}
608	return expr(gz, scope, .{ .rl = .ref }, node);
609	}
610
611	/// Turn Zig AST into untyped ZIR instructions.
612	/// When `rl` is discard, ptr, inferred_ptr, or inferred_ptr, the
613	/// result instruction can be used to inspect whether it is isNoReturn() but that is it,
614	/// it must otherwise not be used.
615	fn expr(gz: GenZir, scope: Scope, ri: ResultInfo, node: Ast.Node.Index) InnerError!Zir.Inst.Ref {
616	const astgen = gz.astgen;
617	const tree = astgen.tree;
618	const main_tokens = tree.nodes.items(.main_token);
619	const token_tags = tree.tokens.items(.tag);
620	const node_datas = tree.nodes.items(.data);
621	const node_tags = tree.nodes.items(.tag);
622
623	const prev_anon_name_strategy = gz.anon_name_strategy;
624	defer gz.anon_name_strategy = prev_anon_name_strategy;
625	if (!nodeUsesAnonNameStrategy(tree, node)) {
626	gz.anon_name_strategy = .anon;
627	}
628
629	switch (node_tags[node]) {
630	.root => unreachable, // Top-level declaration.
631	.@"usingnamespace" => unreachable, // Top-level declaration.
632	.test_decl => unreachable, // Top-level declaration.
633	.container_field_init => unreachable, // Top-level declaration.
634	.container_field_align => unreachable, // Top-level declaration.
635	.container_field => unreachable, // Top-level declaration.
636	.fn_decl => unreachable, // Top-level declaration.
637
638	.global_var_decl => unreachable, // Handled in `blockExpr`.
639	.local_var_decl => unreachable, // Handled in `blockExpr`.
640	.simple_var_decl => unreachable, // Handled in `blockExpr`.
641	.aligned_var_decl => unreachable, // Handled in `blockExpr`.
642	.@"defer" => unreachable, // Handled in `blockExpr`.
643	.@"errdefer" => unreachable, // Handled in `blockExpr`.
644
645	.switch_case => unreachable, // Handled in `switchExpr`.
646	.switch_case_inline => unreachable, // Handled in `switchExpr`.
647	.switch_case_one => unreachable, // Handled in `switchExpr`.
648	.switch_case_inline_one => unreachable, // Handled in `switchExpr`.
649	.switch_range => unreachable, // Handled in `switchExpr`.
650
651	.asm_output => unreachable, // Handled in `asmExpr`.
652	.asm_input => unreachable, // Handled in `asmExpr`.
653
654	.for_range => unreachable, // Handled in `forExpr`.
655
656	.assign => {
657	try assign(gz, scope, node);
658	return rvalue(gz, ri, .void_value, node);
659	},
660
661	.assign_destructure => {
662	// Note that this variant does not declare any new var/const: that
663	// variant is handled by `blockExprStmts`.
664	try assignDestructure(gz, scope, node);
665	return rvalue(gz, ri, .void_value, node);
666	},
667
668	.assign_shl => {
669	try assignShift(gz, scope, node, .shl);
670	return rvalue(gz, ri, .void_value, node);
671	},
672	.assign_shl_sat => {
673	try assignShiftSat(gz, scope, node);
674	return rvalue(gz, ri, .void_value, node);
675	},
676	.assign_shr => {
677	try assignShift(gz, scope, node, .shr);
678	return rvalue(gz, ri, .void_value, node);
679	},
680
681	.assign_bit_and => {
682	try assignOp(gz, scope, node, .bit_and);
683	return rvalue(gz, ri, .void_value, node);
684	},
685	.assign_bit_or => {
686	try assignOp(gz, scope, node, .bit_or);
687	return rvalue(gz, ri, .void_value, node);
688	},
689	.assign_bit_xor => {
690	try assignOp(gz, scope, node, .xor);
691	return rvalue(gz, ri, .void_value, node);
692	},
693	.assign_div => {
694	try assignOp(gz, scope, node, .div);
695	return rvalue(gz, ri, .void_value, node);
696	},
697	.assign_sub => {
698	try assignOp(gz, scope, node, .sub);
699	return rvalue(gz, ri, .void_value, node);
700	},
701	.assign_sub_wrap => {
702	try assignOp(gz, scope, node, .subwrap);
703	return rvalue(gz, ri, .void_value, node);
704	},
705	.assign_sub_sat => {
706	try assignOp(gz, scope, node, .sub_sat);
707	return rvalue(gz, ri, .void_value, node);
708	},
709	.assign_mod => {
710	try assignOp(gz, scope, node, .mod_rem);
711	return rvalue(gz, ri, .void_value, node);
712	},
713	.assign_add => {
714	try assignOp(gz, scope, node, .add);
715	return rvalue(gz, ri, .void_value, node);
716	},
717	.assign_add_wrap => {
718	try assignOp(gz, scope, node, .addwrap);
719	return rvalue(gz, ri, .void_value, node);
720	},
721	.assign_add_sat => {
722	try assignOp(gz, scope, node, .add_sat);
723	return rvalue(gz, ri, .void_value, node);
724	},
725	.assign_mul => {
726	try assignOp(gz, scope, node, .mul);
727	return rvalue(gz, ri, .void_value, node);
728	},
729	.assign_mul_wrap => {
730	try assignOp(gz, scope, node, .mulwrap);
731	return rvalue(gz, ri, .void_value, node);
732	},
733	.assign_mul_sat => {
734	try assignOp(gz, scope, node, .mul_sat);
735	return rvalue(gz, ri, .void_value, node);
736	},
737
738	// zig fmt: off
739	.shl => return shiftOp(gz, scope, ri, node, node_datas[node].lhs, node_datas[node].rhs, .shl),
740	.shr => return shiftOp(gz, scope, ri, node, node_datas[node].lhs, node_datas[node].rhs, .shr),
741
742	.add => return simpleBinOp(gz, scope, ri, node, .add),
743	.add_wrap => return simpleBinOp(gz, scope, ri, node, .addwrap),
744	.add_sat => return simpleBinOp(gz, scope, ri, node, .add_sat),
745	.sub => return simpleBinOp(gz, scope, ri, node, .sub),
746	.sub_wrap => return simpleBinOp(gz, scope, ri, node, .subwrap),
747	.sub_sat => return simpleBinOp(gz, scope, ri, node, .sub_sat),
748	.mul => return simpleBinOp(gz, scope, ri, node, .mul),
749	.mul_wrap => return simpleBinOp(gz, scope, ri, node, .mulwrap),
750	.mul_sat => return simpleBinOp(gz, scope, ri, node, .mul_sat),
751	.div => return simpleBinOp(gz, scope, ri, node, .div),
752	.mod => return simpleBinOp(gz, scope, ri, node, .mod_rem),
753	.shl_sat => return simpleBinOp(gz, scope, ri, node, .shl_sat),
754
755	.bit_and => return simpleBinOp(gz, scope, ri, node, .bit_and),
756	.bit_or => return simpleBinOp(gz, scope, ri, node, .bit_or),
757	.bit_xor => return simpleBinOp(gz, scope, ri, node, .xor),
758	.bang_equal => return simpleBinOp(gz, scope, ri, node, .cmp_neq),
759	.equal_equal => return simpleBinOp(gz, scope, ri, node, .cmp_eq),
760	.greater_than => return simpleBinOp(gz, scope, ri, node, .cmp_gt),
761	.greater_or_equal => return simpleBinOp(gz, scope, ri, node, .cmp_gte),
762	.less_than => return simpleBinOp(gz, scope, ri, node, .cmp_lt),
763	.less_or_equal => return simpleBinOp(gz, scope, ri, node, .cmp_lte),
764	.array_cat => return simpleBinOp(gz, scope, ri, node, .array_cat),
765
766	.array_mult => {
767	// This syntax form does not currently use the result type in the language specification.
768	// However, the result type can be used to emit more optimal code for large multiplications by
769	// having Sema perform a coercion before the multiplication operation.
770	const result = try gz.addPlNode(.array_mul, node, Zir.Inst.ArrayMul{
771	.res_ty = if (try ri.rl.resultType(gz, node)) \|t\| t else .none,
772	.lhs = try expr(gz, scope, .{ .rl = .none }, node_datas[node].lhs),
773	.rhs = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, node_datas[node].rhs),
774	});
775	return rvalue(gz, ri, result, node);
776	},
777
778	.error_union => return simpleBinOp(gz, scope, ri, node, .error_union_type),
779	.merge_error_sets => return simpleBinOp(gz, scope, ri, node, .merge_error_sets),
780
781	.bool_and => return boolBinOp(gz, scope, ri, node, .bool_br_and),
782	.bool_or => return boolBinOp(gz, scope, ri, node, .bool_br_or),
783
784	.bool_not => return simpleUnOp(gz, scope, ri, node, coerced_bool_ri, node_datas[node].lhs, .bool_not),
785	.bit_not => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, node_datas[node].lhs, .bit_not),
786
787	.negation => return negation(gz, scope, ri, node),
788	.negation_wrap => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, node_datas[node].lhs, .negate_wrap),
789
790	.identifier => return identifier(gz, scope, ri, node),
791
792	.asm_simple,
793	.@"asm",
794	=> return asmExpr(gz, scope, ri, node, tree.fullAsm(node).?),
795
796	.string_literal => return stringLiteral(gz, ri, node),
797	.multiline_string_literal => return multilineStringLiteral(gz, ri, node),
798
799	.number_literal => return numberLiteral(gz, ri, node, node, .positive),
800	// zig fmt: on
801
802	.builtin_call_two, .builtin_call_two_comma => {
803	if (node_datas[node].lhs == 0) {
804	const params = [_]Ast.Node.Index{};
805	return builtinCall(gz, scope, ri, node, &params);
806	} else if (node_datas[node].rhs == 0) {
807	const params = [_]Ast.Node.Index{node_datas[node].lhs};
808	return builtinCall(gz, scope, ri, node, &params);
809	} else {
810	const params = [_]Ast.Node.Index{ node_datas[node].lhs, node_datas[node].rhs };
811	return builtinCall(gz, scope, ri, node, &params);
812	}
813	},
814	.builtin_call, .builtin_call_comma => {
815	const params = tree.extra_data[node_datas[node].lhs..node_datas[node].rhs];
816	return builtinCall(gz, scope, ri, node, params);
817	},
818
819	.call_one,
820	.call_one_comma,
821	.async_call_one,
822	.async_call_one_comma,
823	.call,
824	.call_comma,
825	.async_call,
826	.async_call_comma,
827	=> {
828	var buf: [1]Ast.Node.Index = undefined;
829	return callExpr(gz, scope, ri, node, tree.fullCall(&buf, node).?);
830	},
831
832	.unreachable_literal => {
833	try emitDbgNode(gz, node);
834	_ = try gz.addAsIndex(.{
835	.tag = .@"unreachable",
836	.data = .{ .@"unreachable" = .{
837	.src_node = gz.nodeIndexToRelative(node),
838	} },
839	});
840	return Zir.Inst.Ref.unreachable_value;
841	},
842	.@"return" => return ret(gz, scope, node),
843	.field_access => return fieldAccess(gz, scope, ri, node),
844
845	.if_simple,
846	.@"if",
847	=> {
848	const if_full = tree.fullIf(node).?;
849	no_switch_on_err: {
850	const error_token = if_full.error_token orelse break :no_switch_on_err;
851	switch (node_tags[if_full.ast.else_expr]) {
852	.@"switch", .switch_comma => {},
853	else => break :no_switch_on_err,
854	}
855	const switch_operand = node_datas[if_full.ast.else_expr].lhs;
856	if (node_tags[switch_operand] != .identifier) break :no_switch_on_err;
857	if (!mem.eql(u8, tree.tokenSlice(error_token), tree.tokenSlice(main_tokens[switch_operand]))) break :no_switch_on_err;
858	return switchExprErrUnion(gz, scope, ri.br(), node, .@"if");
859	}
860	return ifExpr(gz, scope, ri.br(), node, if_full);
861	},
862
863	.while_simple,
864	.while_cont,
865	.@"while",
866	=> return whileExpr(gz, scope, ri.br(), node, tree.fullWhile(node).?, false),
867
868	.for_simple, .@"for" => return forExpr(gz, scope, ri.br(), node, tree.fullFor(node).?, false),
869
870	.slice_open => {
871	const lhs = try expr(gz, scope, .{ .rl = .ref }, node_datas[node].lhs);
872
873	const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
874	const start = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, node_datas[node].rhs);
875	try emitDbgStmt(gz, cursor);
876	const result = try gz.addPlNode(.slice_start, node, Zir.Inst.SliceStart{
877	.lhs = lhs,
878	.start = start,
879	});
880	return rvalue(gz, ri, result, node);
881	},
882	.slice => {
883	const extra = tree.extraData(node_datas[node].rhs, Ast.Node.Slice);
884	const lhs_node = node_datas[node].lhs;
885	const lhs_tag = node_tags[lhs_node];
886	const lhs_is_slice_sentinel = lhs_tag == .slice_sentinel;
887	const lhs_is_open_slice = lhs_tag == .slice_open or
888	(lhs_is_slice_sentinel and tree.extraData(node_datas[lhs_node].rhs, Ast.Node.SliceSentinel).end == 0);
889	if (lhs_is_open_slice and nodeIsTriviallyZero(tree, extra.start)) {
890	const lhs = try expr(gz, scope, .{ .rl = .ref }, node_datas[lhs_node].lhs);
891
892	const start = if (lhs_is_slice_sentinel) start: {
893	const lhs_extra = tree.extraData(node_datas[lhs_node].rhs, Ast.Node.SliceSentinel);
894	break :start try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, lhs_extra.start);
895	} else try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, node_datas[lhs_node].rhs);
896
897	const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
898	const len = if (extra.end != 0) try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, extra.end) else .none;
899	try emitDbgStmt(gz, cursor);
900	const result = try gz.addPlNode(.slice_length, node, Zir.Inst.SliceLength{
901	.lhs = lhs,
902	.start = start,
903	.len = len,
904	.start_src_node_offset = gz.nodeIndexToRelative(lhs_node),
905	.sentinel = .none,
906	});
907	return rvalue(gz, ri, result, node);
908	}
909	const lhs = try expr(gz, scope, .{ .rl = .ref }, node_datas[node].lhs);
910
911	const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
912	const start = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, extra.start);
913	const end = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, extra.end);
914	try emitDbgStmt(gz, cursor);
915	const result = try gz.addPlNode(.slice_end, node, Zir.Inst.SliceEnd{
916	.lhs = lhs,
917	.start = start,
918	.end = end,
919	});
920	return rvalue(gz, ri, result, node);
921	},
922	.slice_sentinel => {
923	const extra = tree.extraData(node_datas[node].rhs, Ast.Node.SliceSentinel);
924	const lhs_node = node_datas[node].lhs;
925	const lhs_tag = node_tags[lhs_node];
926	const lhs_is_slice_sentinel = lhs_tag == .slice_sentinel;
927	const lhs_is_open_slice = lhs_tag == .slice_open or
928	(lhs_is_slice_sentinel and tree.extraData(node_datas[lhs_node].rhs, Ast.Node.SliceSentinel).end == 0);
929	if (lhs_is_open_slice and nodeIsTriviallyZero(tree, extra.start)) {
930	const lhs = try expr(gz, scope, .{ .rl = .ref }, node_datas[lhs_node].lhs);
931
932	const start = if (lhs_is_slice_sentinel) start: {
933	const lhs_extra = tree.extraData(node_datas[lhs_node].rhs, Ast.Node.SliceSentinel);
934	break :start try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, lhs_extra.start);
935	} else try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, node_datas[lhs_node].rhs);
936
937	const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
938	const len = if (extra.end != 0) try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, extra.end) else .none;
939	const sentinel = try expr(gz, scope, .{ .rl = .none }, extra.sentinel);
940	try emitDbgStmt(gz, cursor);
941	const result = try gz.addPlNode(.slice_length, node, Zir.Inst.SliceLength{
942	.lhs = lhs,
943	.start = start,
944	.len = len,
945	.start_src_node_offset = gz.nodeIndexToRelative(lhs_node),
946	.sentinel = sentinel,
947	});
948	return rvalue(gz, ri, result, node);
949	}
950	const lhs = try expr(gz, scope, .{ .rl = .ref }, node_datas[node].lhs);
951
952	const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
953	const start = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, extra.start);
954	const end = if (extra.end != 0) try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, extra.end) else .none;
955	const sentinel = try expr(gz, scope, .{ .rl = .none }, extra.sentinel);
956	try emitDbgStmt(gz, cursor);
957	const result = try gz.addPlNode(.slice_sentinel, node, Zir.Inst.SliceSentinel{
958	.lhs = lhs,
959	.start = start,
960	.end = end,
961	.sentinel = sentinel,
962	});
963	return rvalue(gz, ri, result, node);
964	},
965
966	.deref => {
967	const lhs = try expr(gz, scope, .{ .rl = .none }, node_datas[node].lhs);
968	_ = try gz.addUnNode(.validate_deref, lhs, node);
969	switch (ri.rl) {
970	.ref, .ref_coerced_ty => return lhs,
971	else => {
972	const result = try gz.addUnNode(.load, lhs, node);
973	return rvalue(gz, ri, result, node);
974	},
975	}
976	},
977	.address_of => {
978	const operand_rl: ResultInfo.Loc = if (try ri.rl.resultType(gz, node)) \|res_ty_inst\| rl: {
979	_ = try gz.addUnTok(.validate_ref_ty, res_ty_inst, tree.firstToken(node));
980	break :rl .{ .ref_coerced_ty = res_ty_inst };
981	} else .ref;
982	const result = try expr(gz, scope, .{ .rl = operand_rl }, node_datas[node].lhs);
983	return rvalue(gz, ri, result, node);
984	},
985	.optional_type => {
986	const operand = try typeExpr(gz, scope, node_datas[node].lhs);
987	const result = try gz.addUnNode(.optional_type, operand, node);
988	return rvalue(gz, ri, result, node);
989	},
990	.unwrap_optional => switch (ri.rl) {
991	.ref, .ref_coerced_ty => {
992	const lhs = try expr(gz, scope, .{ .rl = .ref }, node_datas[node].lhs);
993
994	const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
995	try emitDbgStmt(gz, cursor);
996
997	return gz.addUnNode(.optional_payload_safe_ptr, lhs, node);
998	},
999	else => {
1000	const lhs = try expr(gz, scope, .{ .rl = .none }, node_datas[node].lhs);
1001
1002	const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
1003	try emitDbgStmt(gz, cursor);
1004
1005	return rvalue(gz, ri, try gz.addUnNode(.optional_payload_safe, lhs, node), node);
1006	},
1007	},
1008	.block_two, .block_two_semicolon => {
1009	const statements = [2]Ast.Node.Index{ node_datas[node].lhs, node_datas[node].rhs };
1010	if (node_datas[node].lhs == 0) {
1011	return blockExpr(gz, scope, ri, node, statements[0..0]);
1012	} else if (node_datas[node].rhs == 0) {
1013	return blockExpr(gz, scope, ri, node, statements[0..1]);
1014	} else {
1015	return blockExpr(gz, scope, ri, node, statements[0..2]);
1016	}
1017	},
1018	.block, .block_semicolon => {
1019	const statements = tree.extra_data[node_datas[node].lhs..node_datas[node].rhs];
1020	return blockExpr(gz, scope, ri, node, statements);
1021	},
1022	.enum_literal => return simpleStrTok(gz, ri, main_tokens[node], node, .enum_literal),
1023	.error_value => return simpleStrTok(gz, ri, node_datas[node].rhs, node, .error_value),
1024	// TODO restore this when implementing https://github.com/ziglang/zig/issues/6025
1025	// .anyframe_literal => return rvalue(gz, ri, .anyframe_type, node),
1026	.anyframe_literal => {
1027	const result = try gz.addUnNode(.anyframe_type, .void_type, node);
1028	return rvalue(gz, ri, result, node);
1029	},
1030	.anyframe_type => {
1031	const return_type = try typeExpr(gz, scope, node_datas[node].rhs);
1032	const result = try gz.addUnNode(.anyframe_type, return_type, node);
1033	return rvalue(gz, ri, result, node);
1034	},
1035	.@"catch" => {
1036	const catch_token = main_tokens[node];
1037	const payload_token: ?Ast.TokenIndex = if (token_tags[catch_token + 1] == .pipe)
1038	catch_token + 2
1039	else
1040	null;
1041	no_switch_on_err: {
1042	const capture_token = payload_token orelse break :no_switch_on_err;
1043	switch (node_tags[node_datas[node].rhs]) {
1044	.@"switch", .switch_comma => {},
1045	else => break :no_switch_on_err,
1046	}
1047	const switch_operand = node_datas[node_datas[node].rhs].lhs;
1048	if (node_tags[switch_operand] != .identifier) break :no_switch_on_err;
1049	if (!mem.eql(u8, tree.tokenSlice(capture_token), tree.tokenSlice(main_tokens[switch_operand]))) break :no_switch_on_err;
1050	return switchExprErrUnion(gz, scope, ri.br(), node, .@"catch");
1051	}
1052	switch (ri.rl) {
1053	.ref, .ref_coerced_ty => return orelseCatchExpr(
1054	gz,
1055	scope,
1056	ri,
1057	node,
1058	node_datas[node].lhs,
1059	.is_non_err_ptr,
1060	.err_union_payload_unsafe_ptr,
1061	.err_union_code_ptr,
1062	node_datas[node].rhs,
1063	payload_token,
1064	),
1065	else => return orelseCatchExpr(
1066	gz,
1067	scope,
1068	ri,
1069	node,
1070	node_datas[node].lhs,
1071	.is_non_err,
1072	.err_union_payload_unsafe,
1073	.err_union_code,
1074	node_datas[node].rhs,
1075	payload_token,
1076	),
1077	}
1078	},
1079	.@"orelse" => switch (ri.rl) {
1080	.ref, .ref_coerced_ty => return orelseCatchExpr(
1081	gz,
1082	scope,
1083	ri,
1084	node,
1085	node_datas[node].lhs,
1086	.is_non_null_ptr,
1087	.optional_payload_unsafe_ptr,
1088	undefined,
1089	node_datas[node].rhs,
1090	null,
1091	),
1092	else => return orelseCatchExpr(
1093	gz,
1094	scope,
1095	ri,
1096	node,
1097	node_datas[node].lhs,
1098	.is_non_null,
1099	.optional_payload_unsafe,
1100	undefined,
1101	node_datas[node].rhs,
1102	null,
1103	),
1104	},
1105
1106	.ptr_type_aligned,
1107	.ptr_type_sentinel,
1108	.ptr_type,
1109	.ptr_type_bit_range,
1110	=> return ptrType(gz, scope, ri, node, tree.fullPtrType(node).?),
1111
1112	.container_decl,
1113	.container_decl_trailing,
1114	.container_decl_arg,
1115	.container_decl_arg_trailing,
1116	.container_decl_two,
1117	.container_decl_two_trailing,
1118	.tagged_union,
1119	.tagged_union_trailing,
1120	.tagged_union_enum_tag,
1121	.tagged_union_enum_tag_trailing,
1122	.tagged_union_two,
1123	.tagged_union_two_trailing,
1124	=> {
1125	var buf: [2]Ast.Node.Index = undefined;
1126	return containerDecl(gz, scope, ri, node, tree.fullContainerDecl(&buf, node).?);
1127	},
1128
1129	.@"break" => return breakExpr(gz, scope, node),
1130	.@"continue" => return continueExpr(gz, scope, node),
1131	.grouped_expression => return expr(gz, scope, ri, node_datas[node].lhs),
1132	.array_type => return arrayType(gz, scope, ri, node),
1133	.array_type_sentinel => return arrayTypeSentinel(gz, scope, ri, node),
1134	.char_literal => return charLiteral(gz, ri, node),
1135	.error_set_decl => return errorSetDecl(gz, ri, node),
1136	.array_access => return arrayAccess(gz, scope, ri, node),
1137	.@"comptime" => return comptimeExprAst(gz, scope, ri, node),
1138	.@"switch", .switch_comma => return switchExpr(gz, scope, ri.br(), node),
1139
1140	.@"nosuspend" => return nosuspendExpr(gz, scope, ri, node),
1141	.@"suspend" => return suspendExpr(gz, scope, node),
1142	.@"await" => return awaitExpr(gz, scope, ri, node),
1143	.@"resume" => return resumeExpr(gz, scope, ri, node),
1144
1145	.@"try" => return tryExpr(gz, scope, ri, node, node_datas[node].lhs),
1146
1147	.array_init_one,
1148	.array_init_one_comma,
1149	.array_init_dot_two,
1150	.array_init_dot_two_comma,
1151	.array_init_dot,
1152	.array_init_dot_comma,
1153	.array_init,
1154	.array_init_comma,
1155	=> {
1156	var buf: [2]Ast.Node.Index = undefined;
1157	return arrayInitExpr(gz, scope, ri, node, tree.fullArrayInit(&buf, node).?);
1158	},
1159
1160	.struct_init_one,
1161	.struct_init_one_comma,
1162	.struct_init_dot_two,
1163	.struct_init_dot_two_comma,
1164	.struct_init_dot,
1165	.struct_init_dot_comma,
1166	.struct_init,
1167	.struct_init_comma,
1168	=> {
1169	var buf: [2]Ast.Node.Index = undefined;
1170	return structInitExpr(gz, scope, ri, node, tree.fullStructInit(&buf, node).?);
1171	},
1172
1173	.fn_proto_simple,
1174	.fn_proto_multi,
1175	.fn_proto_one,
1176	.fn_proto,
1177	=> {
1178	var buf: [1]Ast.Node.Index = undefined;
1179	return fnProtoExpr(gz, scope, ri, node, tree.fullFnProto(&buf, node).?);
1180	},
1181	}
1182	}
1183
1184	fn nosuspendExpr(
1185	gz: *GenZir,
1186	scope: *Scope,
1187	ri: ResultInfo,
1188	node: Ast.Node.Index,
1189	) InnerError!Zir.Inst.Ref {
1190	const astgen = gz.astgen;
1191	const tree = astgen.tree;
1192	const node_datas = tree.nodes.items(.data);
1193	const body_node = node_datas[node].lhs;
1194	assert(body_node != 0);
1195	if (gz.nosuspend_node != 0) {
1196	try astgen.appendErrorNodeNotes(node, "redundant nosuspend block", .{}, &[_]u32{
1197	try astgen.errNoteNode(gz.nosuspend_node, "other nosuspend block here", .{}),
1198	});
1199	}
1200	gz.nosuspend_node = node;
1201	defer gz.nosuspend_node = 0;
1202	return expr(gz, scope, ri, body_node);
1203	}
1204
1205	fn suspendExpr(
1206	gz: *GenZir,
1207	scope: *Scope,
1208	node: Ast.Node.Index,
1209	) InnerError!Zir.Inst.Ref {
1210	const astgen = gz.astgen;
1211	const gpa = astgen.gpa;
1212	const tree = astgen.tree;
1213	const node_datas = tree.nodes.items(.data);
1214	const body_node = node_datas[node].lhs;
1215
1216	if (gz.nosuspend_node != 0) {
1217	return astgen.failNodeNotes(node, "suspend inside nosuspend block", .{}, &[_]u32{
1218	try astgen.errNoteNode(gz.nosuspend_node, "nosuspend block here", .{}),
1219	});
1220	}
1221	if (gz.suspend_node != 0) {
1222	return astgen.failNodeNotes(node, "cannot suspend inside suspend block", .{}, &[_]u32{
1223	try astgen.errNoteNode(gz.suspend_node, "other suspend block here", .{}),
1224	});
1225	}
1226	assert(body_node != 0);
1227
1228	const suspend_inst = try gz.makeBlockInst(.suspend_block, node);
1229	try gz.instructions.append(gpa, suspend_inst);
1230
1231	var suspend_scope = gz.makeSubBlock(scope);
1232	suspend_scope.suspend_node = node;
1233	defer suspend_scope.unstack();
1234
1235	const body_result = try expr(&suspend_scope, &suspend_scope.base, .{ .rl = .none }, body_node);
1236	if (!gz.refIsNoReturn(body_result)) {
1237	_ = try suspend_scope.addBreak(.break_inline, suspend_inst, .void_value);
1238	}
1239	try suspend_scope.setBlockBody(suspend_inst);
1240
1241	return suspend_inst.toRef();
1242	}
1243
1244	fn awaitExpr(
1245	gz: *GenZir,
1246	scope: *Scope,
1247	ri: ResultInfo,
1248	node: Ast.Node.Index,
1249	) InnerError!Zir.Inst.Ref {
1250	const astgen = gz.astgen;
1251	const tree = astgen.tree;
1252	const node_datas = tree.nodes.items(.data);
1253	const rhs_node = node_datas[node].lhs;
1254
1255	if (gz.suspend_node != 0) {
1256	return astgen.failNodeNotes(node, "cannot await inside suspend block", .{}, &[_]u32{
1257	try astgen.errNoteNode(gz.suspend_node, "suspend block here", .{}),
1258	});
1259	}
1260	const operand = try expr(gz, scope, .{ .rl = .ref }, rhs_node);
1261	const result = if (gz.nosuspend_node != 0)
1262	try gz.addExtendedPayload(.await_nosuspend, Zir.Inst.UnNode{
1263	.node = gz.nodeIndexToRelative(node),
1264	.operand = operand,
1265	})
1266	else
1267	try gz.addUnNode(.@"await", operand, node);
1268
1269	return rvalue(gz, ri, result, node);
1270	}
1271
1272	fn resumeExpr(
1273	gz: *GenZir,
1274	scope: *Scope,
1275	ri: ResultInfo,
1276	node: Ast.Node.Index,
1277	) InnerError!Zir.Inst.Ref {
1278	const astgen = gz.astgen;
1279	const tree = astgen.tree;
1280	const node_datas = tree.nodes.items(.data);
1281	const rhs_node = node_datas[node].lhs;
1282	const operand = try expr(gz, scope, .{ .rl = .ref }, rhs_node);
1283	const result = try gz.addUnNode(.@"resume", operand, node);
1284	return rvalue(gz, ri, result, node);
1285	}
1286
1287	fn fnProtoExpr(
1288	gz: *GenZir,
1289	scope: *Scope,
1290	ri: ResultInfo,
1291	node: Ast.Node.Index,
1292	fn_proto: Ast.full.FnProto,
1293	) InnerError!Zir.Inst.Ref {
1294	const astgen = gz.astgen;
1295	const tree = astgen.tree;
1296	const token_tags = tree.tokens.items(.tag);
1297
1298	if (fn_proto.name_token) \|some\| {
1299	return astgen.failTok(some, "function type cannot have a name", .{});
1300	}
1301
1302	const is_extern = blk: {
1303	const maybe_extern_token = fn_proto.extern_export_inline_token orelse break :blk false;
1304	break :blk token_tags[maybe_extern_token] == .keyword_extern;
1305	};
1306	assert(!is_extern);
1307
1308	var block_scope = gz.makeSubBlock(scope);
1309	defer block_scope.unstack();
1310
1311	const block_inst = try gz.makeBlockInst(.block_inline, node);
1312
1313	var noalias_bits: u32 = 0;
1314	const is_var_args = is_var_args: {
1315	var param_type_i: usize = 0;
1316	var it = fn_proto.iterate(tree);
1317	while (it.next()) \|param\| : (param_type_i += 1) {
1318	const is_comptime = if (param.comptime_noalias) \|token\| switch (token_tags[token]) {
1319	.keyword_noalias => is_comptime: {
1320	noalias_bits \|= @as(u32, 1) << (std.math.cast(u5, param_type_i) orelse
1321	return astgen.failTok(token, "this compiler implementation only supports 'noalias' on the first 32 parameters", .{}));
1322	break :is_comptime false;
1323	},
1324	.keyword_comptime => true,
1325	else => false,
1326	} else false;
1327
1328	const is_anytype = if (param.anytype_ellipsis3) \|token\| blk: {
1329	switch (token_tags[token]) {
1330	.keyword_anytype => break :blk true,
1331	.ellipsis3 => break :is_var_args true,
1332	else => unreachable,
1333	}
1334	} else false;
1335
1336	const param_name = if (param.name_token) \|name_token\| blk: {
1337	if (mem.eql(u8, "_", tree.tokenSlice(name_token)))
1338	break :blk .empty;
1339
1340	break :blk try astgen.identAsString(name_token);
1341	} else .empty;
1342
1343	if (is_anytype) {
1344	const name_token = param.name_token orelse param.anytype_ellipsis3.?;
1345
1346	const tag: Zir.Inst.Tag = if (is_comptime)
1347	.param_anytype_comptime
1348	else
1349	.param_anytype;
1350	_ = try block_scope.addStrTok(tag, param_name, name_token);
1351	} else {
1352	const param_type_node = param.type_expr;
1353	assert(param_type_node != 0);
1354	var param_gz = block_scope.makeSubBlock(scope);
1355	defer param_gz.unstack();
1356	const param_type = try expr(&param_gz, scope, coerced_type_ri, param_type_node);
1357	const param_inst_expected: Zir.Inst.Index = @enumFromInt(astgen.instructions.len + 1);
1358	_ = try param_gz.addBreakWithSrcNode(.break_inline, param_inst_expected, param_type, param_type_node);
1359	const main_tokens = tree.nodes.items(.main_token);
1360	const name_token = param.name_token orelse main_tokens[param_type_node];
1361	const tag: Zir.Inst.Tag = if (is_comptime) .param_comptime else .param;
1362	const param_inst = try block_scope.addParam(&param_gz, tag, name_token, param_name, param.first_doc_comment);
1363	assert(param_inst_expected == param_inst);
1364	}
1365	}
1366	break :is_var_args false;
1367	};
1368
1369	const align_ref: Zir.Inst.Ref = if (fn_proto.ast.align_expr == 0) .none else inst: {
1370	break :inst try expr(&block_scope, scope, coerced_align_ri, fn_proto.ast.align_expr);
1371	};
1372
1373	if (fn_proto.ast.addrspace_expr != 0) {
1374	return astgen.failNode(fn_proto.ast.addrspace_expr, "addrspace not allowed on function prototypes", .{});
1375	}
1376
1377	if (fn_proto.ast.section_expr != 0) {
1378	return astgen.failNode(fn_proto.ast.section_expr, "linksection not allowed on function prototypes", .{});
1379	}
1380
1381	const cc: Zir.Inst.Ref = if (fn_proto.ast.callconv_expr != 0)
1382	try expr(
1383	&block_scope,
1384	scope,
1385	.{ .rl = .{ .coerced_ty = .calling_convention_type } },
1386	fn_proto.ast.callconv_expr,
1387	)
1388	else
1389	Zir.Inst.Ref.none;
1390
1391	const maybe_bang = tree.firstToken(fn_proto.ast.return_type) - 1;
1392	const is_inferred_error = token_tags[maybe_bang] == .bang;
1393	if (is_inferred_error) {
1394	return astgen.failTok(maybe_bang, "function prototype may not have inferred error set", .{});
1395	}
1396	const ret_ty = try expr(&block_scope, scope, coerced_type_ri, fn_proto.ast.return_type);
1397
1398	const result = try block_scope.addFunc(.{
1399	.src_node = fn_proto.ast.proto_node,
1400
1401	.cc_ref = cc,
1402	.cc_gz = null,
1403	.align_ref = align_ref,
1404	.align_gz = null,
1405	.ret_ref = ret_ty,
1406	.ret_gz = null,
1407	.section_ref = .none,
1408	.section_gz = null,
1409	.addrspace_ref = .none,
1410	.addrspace_gz = null,
1411
1412	.param_block = block_inst,
1413	.body_gz = null,
1414	.lib_name = .empty,
1415	.is_var_args = is_var_args,
1416	.is_inferred_error = false,
1417	.is_test = false,
1418	.is_extern = false,
1419	.is_noinline = false,
1420	.noalias_bits = noalias_bits,
1421	});
1422
1423	_ = try block_scope.addBreak(.break_inline, block_inst, result);
1424	try block_scope.setBlockBody(block_inst);
1425	try gz.instructions.append(astgen.gpa, block_inst);
1426
1427	return rvalue(gz, ri, block_inst.toRef(), fn_proto.ast.proto_node);
1428	}
1429
1430	fn arrayInitExpr(
1431	gz: *GenZir,
1432	scope: *Scope,
1433	ri: ResultInfo,
1434	node: Ast.Node.Index,
1435	array_init: Ast.full.ArrayInit,
1436	) InnerError!Zir.Inst.Ref {
1437	const astgen = gz.astgen;
1438	const tree = astgen.tree;
1439	const node_tags = tree.nodes.items(.tag);
1440	const main_tokens = tree.nodes.items(.main_token);
1441
1442	assert(array_init.ast.elements.len != 0); // Otherwise it would be struct init.
1443
1444	const array_ty: Zir.Inst.Ref, const elem_ty: Zir.Inst.Ref = inst: {
1445	if (array_init.ast.type_expr == 0) break :inst .{ .none, .none };
1446
1447	infer: {
1448	const array_type: Ast.full.ArrayType = tree.fullArrayType(array_init.ast.type_expr) orelse break :infer;
1449	// This intentionally does not support `@"_"` syntax.
1450	if (node_tags[array_type.ast.elem_count] == .identifier and
1451	mem.eql(u8, tree.tokenSlice(main_tokens[array_type.ast.elem_count]), "_"))
1452	{
1453	const len_inst = try gz.addInt(array_init.ast.elements.len);
1454	const elem_type = try typeExpr(gz, scope, array_type.ast.elem_type);
1455	if (array_type.ast.sentinel == 0) {
1456	const array_type_inst = try gz.addPlNode(.array_type, array_init.ast.type_expr, Zir.Inst.Bin{
1457	.lhs = len_inst,
1458	.rhs = elem_type,
1459	});
1460	break :inst .{ array_type_inst, elem_type };
1461	} else {
1462	const sentinel = try comptimeExpr(gz, scope, .{ .rl = .{ .ty = elem_type } }, array_type.ast.sentinel);
1463	const array_type_inst = try gz.addPlNode(
1464	.array_type_sentinel,
1465	array_init.ast.type_expr,
1466	Zir.Inst.ArrayTypeSentinel{
1467	.len = len_inst,
1468	.elem_type = elem_type,
1469	.sentinel = sentinel,
1470	},
1471	);
1472	break :inst .{ array_type_inst, elem_type };
1473	}
1474	}
1475	}
1476	const array_type_inst = try typeExpr(gz, scope, array_init.ast.type_expr);
1477	_ = try gz.addPlNode(.validate_array_init_ty, node, Zir.Inst.ArrayInit{
1478	.ty = array_type_inst,
1479	.init_count = @intCast(array_init.ast.elements.len),
1480	});
1481	break :inst .{ array_type_inst, .none };
1482	};
1483
1484	if (array_ty != .none) {
1485	// Typed inits do not use RLS for language simplicity.
1486	switch (ri.rl) {
1487	.discard => {
1488	if (elem_ty != .none) {
1489	const elem_ri: ResultInfo = .{ .rl = .{ .ty = elem_ty } };
1490	for (array_init.ast.elements) \|elem_init\| {
1491	_ = try expr(gz, scope, elem_ri, elem_init);
1492	}
1493	} else {
1494	for (array_init.ast.elements, 0..) \|elem_init, i\| {
1495	const this_elem_ty = try gz.add(.{
1496	.tag = .array_init_elem_type,
1497	.data = .{ .bin = .{
1498	.lhs = array_ty,
1499	.rhs = @enumFromInt(i),
1500	} },
1501	});
1502	_ = try expr(gz, scope, .{ .rl = .{ .ty = this_elem_ty } }, elem_init);
1503	}
1504	}
1505	return .void_value;
1506	},
1507	.ref => return arrayInitExprTyped(gz, scope, node, array_init.ast.elements, array_ty, elem_ty, true),
1508	else => {
1509	const array_inst = try arrayInitExprTyped(gz, scope, node, array_init.ast.elements, array_ty, elem_ty, false);
1510	return rvalue(gz, ri, array_inst, node);
1511	},
1512	}
1513	}
1514
1515	switch (ri.rl) {
1516	.none => return arrayInitExprAnon(gz, scope, node, array_init.ast.elements),
1517	.discard => {
1518	for (array_init.ast.elements) \|elem_init\| {
1519	_ = try expr(gz, scope, .{ .rl = .discard }, elem_init);
1520	}
1521	return Zir.Inst.Ref.void_value;
1522	},
1523	.ref => {
1524	const result = try arrayInitExprAnon(gz, scope, node, array_init.ast.elements);
1525	return gz.addUnTok(.ref, result, tree.firstToken(node));
1526	},
1527	.ref_coerced_ty => \|ptr_ty_inst\| {
1528	const dest_arr_ty_inst = try gz.addPlNode(.validate_array_init_ref_ty, node, Zir.Inst.ArrayInitRefTy{
1529	.ptr_ty = ptr_ty_inst,
1530	.elem_count = @intCast(array_init.ast.elements.len),
1531	});
1532	return arrayInitExprTyped(gz, scope, node, array_init.ast.elements, dest_arr_ty_inst, .none, true);
1533	},
1534	.ty, .coerced_ty => \|result_ty_inst\| {
1535	_ = try gz.addPlNode(.validate_array_init_result_ty, node, Zir.Inst.ArrayInit{
1536	.ty = result_ty_inst,
1537	.init_count = @intCast(array_init.ast.elements.len),
1538	});
1539	return arrayInitExprTyped(gz, scope, node, array_init.ast.elements, result_ty_inst, .none, false);
1540	},
1541	.ptr => \|ptr\| {
1542	try arrayInitExprPtr(gz, scope, node, array_init.ast.elements, ptr.inst);
1543	return .void_value;
1544	},
1545	.inferred_ptr => {
1546	// We can't get elem pointers of an untyped inferred alloc, so must perform a
1547	// standard anonymous initialization followed by an rvalue store.
1548	// See corresponding logic in structInitExpr.
1549	const result = try arrayInitExprAnon(gz, scope, node, array_init.ast.elements);
1550	return rvalue(gz, ri, result, node);
1551	},
1552	.destructure => \|destructure\| {
1553	// Untyped init - destructure directly into result pointers
1554	if (array_init.ast.elements.len != destructure.components.len) {
1555	return astgen.failNodeNotes(node, "expected {} elements for destructure, found {}", .{
1556	destructure.components.len,
1557	array_init.ast.elements.len,
1558	}, &.{
1559	try astgen.errNoteNode(destructure.src_node, "result destructured here", .{}),
1560	});
1561	}
1562	for (array_init.ast.elements, destructure.components) \|elem_init, ds_comp\| {
1563	const elem_ri: ResultInfo = .{ .rl = switch (ds_comp) {
1564	.typed_ptr => \|ptr_rl\| .{ .ptr = ptr_rl },
1565	.inferred_ptr => \|ptr_inst\| .{ .inferred_ptr = ptr_inst },
1566	.discard => .discard,
1567	} };
1568	_ = try expr(gz, scope, elem_ri, elem_init);
1569	}
1570	return .void_value;
1571	},
1572	}
1573	}
1574
1575	/// An array initialization expression using an `array_init_anon` instruction.
1576	fn arrayInitExprAnon(
1577	gz: *GenZir,
1578	scope: *Scope,
1579	node: Ast.Node.Index,
1580	elements: []const Ast.Node.Index,
1581	) InnerError!Zir.Inst.Ref {
1582	const astgen = gz.astgen;
1583
1584	const payload_index = try addExtra(astgen, Zir.Inst.MultiOp{
1585	.operands_len = @intCast(elements.len),
1586	});
1587	var extra_index = try reserveExtra(astgen, elements.len);
1588
1589	for (elements) \|elem_init\| {
1590	const elem_ref = try expr(gz, scope, .{ .rl = .none }, elem_init);
1591	astgen.extra.items[extra_index] = @intFromEnum(elem_ref);
1592	extra_index += 1;
1593	}
1594	return try gz.addPlNodePayloadIndex(.array_init_anon, node, payload_index);
1595	}
1596
1597	/// An array initialization expression using an `array_init` or `array_init_ref` instruction.
1598	fn arrayInitExprTyped(
1599	gz: *GenZir,
1600	scope: *Scope,
1601	node: Ast.Node.Index,
1602	elements: []const Ast.Node.Index,
1603	ty_inst: Zir.Inst.Ref,
1604	maybe_elem_ty_inst: Zir.Inst.Ref,
1605	is_ref: bool,
1606	) InnerError!Zir.Inst.Ref {
1607	const astgen = gz.astgen;
1608
1609	const len = elements.len + 1; // +1 for type
1610	const payload_index = try addExtra(astgen, Zir.Inst.MultiOp{
1611	.operands_len = @intCast(len),
1612	});
1613	var extra_index = try reserveExtra(astgen, len);
1614	astgen.extra.items[extra_index] = @intFromEnum(ty_inst);
1615	extra_index += 1;
1616
1617	if (maybe_elem_ty_inst != .none) {
1618	const elem_ri: ResultInfo = .{ .rl = .{ .coerced_ty = maybe_elem_ty_inst } };
1619	for (elements) \|elem_init\| {
1620	const elem_inst = try expr(gz, scope, elem_ri, elem_init);
1621	astgen.extra.items[extra_index] = @intFromEnum(elem_inst);
1622	extra_index += 1;
1623	}
1624	} else {
1625	for (elements, 0..) \|elem_init, i\| {
1626	const ri: ResultInfo = .{ .rl = .{ .coerced_ty = try gz.add(.{
1627	.tag = .array_init_elem_type,
1628	.data = .{ .bin = .{
1629	.lhs = ty_inst,
1630	.rhs = @enumFromInt(i),
1631	} },
1632	}) } };
1633
1634	const elem_inst = try expr(gz, scope, ri, elem_init);
1635	astgen.extra.items[extra_index] = @intFromEnum(elem_inst);
1636	extra_index += 1;
1637	}
1638	}
1639
1640	const tag: Zir.Inst.Tag = if (is_ref) .array_init_ref else .array_init;
1641	return try gz.addPlNodePayloadIndex(tag, node, payload_index);
1642	}
1643
1644	/// An array initialization expression using element pointers.
1645	fn arrayInitExprPtr(
1646	gz: *GenZir,
1647	scope: *Scope,
1648	node: Ast.Node.Index,
1649	elements: []const Ast.Node.Index,
1650	ptr_inst: Zir.Inst.Ref,
1651	) InnerError!void {
1652	const astgen = gz.astgen;
1653
1654	const array_ptr_inst = try gz.addUnNode(.opt_eu_base_ptr_init, ptr_inst, node);
1655
1656	const payload_index = try addExtra(astgen, Zir.Inst.Block{
1657	.body_len = @intCast(elements.len),
1658	});
1659	var extra_index = try reserveExtra(astgen, elements.len);
1660
1661	for (elements, 0..) \|elem_init, i\| {
1662	const elem_ptr_inst = try gz.addPlNode(.array_init_elem_ptr, elem_init, Zir.Inst.ElemPtrImm{
1663	.ptr = array_ptr_inst,
1664	.index = @intCast(i),
1665	});
1666	astgen.extra.items[extra_index] = @intFromEnum(elem_ptr_inst.toIndex().?);
1667	extra_index += 1;
1668	_ = try expr(gz, scope, .{ .rl = .{ .ptr = .{ .inst = elem_ptr_inst } } }, elem_init);
1669	}
1670
1671	_ = try gz.addPlNodePayloadIndex(.validate_ptr_array_init, node, payload_index);
1672	}
1673
1674	fn structInitExpr(
1675	gz: *GenZir,
1676	scope: *Scope,
1677	ri: ResultInfo,
1678	node: Ast.Node.Index,
1679	struct_init: Ast.full.StructInit,
1680	) InnerError!Zir.Inst.Ref {
1681	const astgen = gz.astgen;
1682	const tree = astgen.tree;
1683
1684	if (struct_init.ast.type_expr == 0) {
1685	if (struct_init.ast.fields.len == 0) {
1686	// Anonymous init with no fields.
1687	switch (ri.rl) {
1688	.discard => return .void_value,
1689	.ref_coerced_ty => \|ptr_ty_inst\| return gz.addUnNode(.struct_init_empty_ref_result, ptr_ty_inst, node),
1690	.ty, .coerced_ty => \|ty_inst\| return gz.addUnNode(.struct_init_empty_result, ty_inst, node),
1691	.ptr => {
1692	// TODO: should we modify this to use RLS for the field stores here?
1693	const ty_inst = (try ri.rl.resultType(gz, node)).?;
1694	const val = try gz.addUnNode(.struct_init_empty_result, ty_inst, node);
1695	return rvalue(gz, ri, val, node);
1696	},
1697	.none, .ref, .inferred_ptr => {
1698	return rvalue(gz, ri, .empty_struct, node);
1699	},
1700	.destructure => \|destructure\| {
1701	return astgen.failNodeNotes(node, "empty initializer cannot be destructured", .{}, &.{
1702	try astgen.errNoteNode(destructure.src_node, "result destructured here", .{}),
1703	});
1704	},
1705	}
1706	}
1707	} else array: {
1708	const node_tags = tree.nodes.items(.tag);
1709	const main_tokens = tree.nodes.items(.main_token);
1710	const array_type: Ast.full.ArrayType = tree.fullArrayType(struct_init.ast.type_expr) orelse {
1711	if (struct_init.ast.fields.len == 0) {
1712	const ty_inst = try typeExpr(gz, scope, struct_init.ast.type_expr);
1713	const result = try gz.addUnNode(.struct_init_empty, ty_inst, node);
1714	return rvalue(gz, ri, result, node);
1715	}
1716	break :array;
1717	};
1718	const is_inferred_array_len = node_tags[array_type.ast.elem_count] == .identifier and
1719	// This intentionally does not support `@"_"` syntax.
1720	mem.eql(u8, tree.tokenSlice(main_tokens[array_type.ast.elem_count]), "_");
1721	if (struct_init.ast.fields.len == 0) {
1722	if (is_inferred_array_len) {
1723	const elem_type = try typeExpr(gz, scope, array_type.ast.elem_type);
1724	const array_type_inst = if (array_type.ast.sentinel == 0) blk: {
1725	break :blk try gz.addPlNode(.array_type, struct_init.ast.type_expr, Zir.Inst.Bin{
1726	.lhs = .zero_usize,
1727	.rhs = elem_type,
1728	});
1729	} else blk: {
1730	const sentinel = try comptimeExpr(gz, scope, .{ .rl = .{ .ty = elem_type } }, array_type.ast.sentinel);
1731	break :blk try gz.addPlNode(
1732	.array_type_sentinel,
1733	struct_init.ast.type_expr,
1734	Zir.Inst.ArrayTypeSentinel{
1735	.len = .zero_usize,
1736	.elem_type = elem_type,
1737	.sentinel = sentinel,
1738	},
1739	);
1740	};
1741	const result = try gz.addUnNode(.struct_init_empty, array_type_inst, node);
1742	return rvalue(gz, ri, result, node);
1743	}
1744	const ty_inst = try typeExpr(gz, scope, struct_init.ast.type_expr);
1745	const result = try gz.addUnNode(.struct_init_empty, ty_inst, node);
1746	return rvalue(gz, ri, result, node);
1747	} else {
1748	return astgen.failNode(
1749	struct_init.ast.type_expr,
1750	"initializing array with struct syntax",
1751	.{},
1752	);
1753	}
1754	}
1755
1756	{
1757	var sfba = std.heap.stackFallback(256, astgen.arena);
1758	const sfba_allocator = sfba.get();
1759
1760	var duplicate_names = std.AutoArrayHashMap(Zir.NullTerminatedString, ArrayListUnmanaged(Ast.TokenIndex)).init(sfba_allocator);
1761	try duplicate_names.ensureTotalCapacity(@intCast(struct_init.ast.fields.len));
1762
1763	// When there aren't errors, use this to avoid a second iteration.
1764	var any_duplicate = false;
1765
1766	for (struct_init.ast.fields) \|field\| {
1767	const name_token = tree.firstToken(field) - 2;
1768	const name_index = try astgen.identAsString(name_token);
1769
1770	const gop = try duplicate_names.getOrPut(name_index);
1771
1772	if (gop.found_existing) {
1773	try gop.value_ptr.append(sfba_allocator, name_token);
1774	any_duplicate = true;
1775	} else {
1776	gop.value_ptr.* = .{};
1777	try gop.value_ptr.append(sfba_allocator, name_token);
1778	}
1779	}
1780
1781	if (any_duplicate) {
1782	var it = duplicate_names.iterator();
1783
1784	while (it.next()) \|entry\| {
1785	const record = entry.value_ptr.*;
1786	if (record.items.len > 1) {
1787	var error_notes = std.ArrayList(u32).init(astgen.arena);
1788
1789	for (record.items[1..]) \|duplicate\| {
1790	try error_notes.append(try astgen.errNoteTok(duplicate, "duplicate name here", .{}));
1791	}
1792
1793	try error_notes.append(try astgen.errNoteNode(node, "struct declared here", .{}));
1794
1795	try astgen.appendErrorTokNotes(
1796	record.items[0],
1797	"duplicate struct field name",
1798	.{},
1799	error_notes.items,
1800	);
1801	}
1802	}
1803
1804	return error.AnalysisFail;
1805	}
1806	}
1807
1808	if (struct_init.ast.type_expr != 0) {
1809	// Typed inits do not use RLS for language simplicity.
1810	const ty_inst = try typeExpr(gz, scope, struct_init.ast.type_expr);
1811	_ = try gz.addUnNode(.validate_struct_init_ty, ty_inst, node);
1812	switch (ri.rl) {
1813	.ref => return structInitExprTyped(gz, scope, node, struct_init, ty_inst, true),
1814	else => {
1815	const struct_inst = try structInitExprTyped(gz, scope, node, struct_init, ty_inst, false);
1816	return rvalue(gz, ri, struct_inst, node);
1817	},
1818	}
1819	}
1820
1821	switch (ri.rl) {
1822	.none => return structInitExprAnon(gz, scope, node, struct_init),
1823	.discard => {
1824	// Even if discarding we must perform side-effects.
1825	for (struct_init.ast.fields) \|field_init\| {
1826	_ = try expr(gz, scope, .{ .rl = .discard }, field_init);
1827	}
1828	return .void_value;
1829	},
1830	.ref => {
1831	const result = try structInitExprAnon(gz, scope, node, struct_init);
1832	return gz.addUnTok(.ref, result, tree.firstToken(node));
1833	},
1834	.ref_coerced_ty => \|ptr_ty_inst\| {
1835	const result_ty_inst = try gz.addUnNode(.elem_type, ptr_ty_inst, node);
1836	_ = try gz.addUnNode(.validate_struct_init_result_ty, result_ty_inst, node);
1837	return structInitExprTyped(gz, scope, node, struct_init, result_ty_inst, true);
1838	},
1839	.ty, .coerced_ty => \|result_ty_inst\| {
1840	_ = try gz.addUnNode(.validate_struct_init_result_ty, result_ty_inst, node);
1841	return structInitExprTyped(gz, scope, node, struct_init, result_ty_inst, false);
1842	},
1843	.ptr => \|ptr\| {
1844	try structInitExprPtr(gz, scope, node, struct_init, ptr.inst);
1845	return .void_value;
1846	},
1847	.inferred_ptr => {
1848	// We can't get field pointers of an untyped inferred alloc, so must perform a
1849	// standard anonymous initialization followed by an rvalue store.
1850	// See corresponding logic in arrayInitExpr.
1851	const struct_inst = try structInitExprAnon(gz, scope, node, struct_init);
1852	return rvalue(gz, ri, struct_inst, node);
1853	},
1854	.destructure => \|destructure\| {
1855	// This is an untyped init, so is an actual struct, which does
1856	// not support destructuring.
1857	return astgen.failNodeNotes(node, "struct value cannot be destructured", .{}, &.{
1858	try astgen.errNoteNode(destructure.src_node, "result destructured here", .{}),
1859	});
1860	},
1861	}
1862	}
1863
1864	/// A struct initialization expression using a `struct_init_anon` instruction.
1865	fn structInitExprAnon(
1866	gz: *GenZir,
1867	scope: *Scope,
1868	node: Ast.Node.Index,
1869	struct_init: Ast.full.StructInit,
1870	) InnerError!Zir.Inst.Ref {
1871	const astgen = gz.astgen;
1872	const tree = astgen.tree;
1873
1874	const payload_index = try addExtra(astgen, Zir.Inst.StructInitAnon{
1875	.fields_len = @intCast(struct_init.ast.fields.len),
1876	});
1877	const field_size = @typeInfo(Zir.Inst.StructInitAnon.Item).Struct.fields.len;
1878	var extra_index: usize = try reserveExtra(astgen, struct_init.ast.fields.len * field_size);
1879
1880	for (struct_init.ast.fields) \|field_init\| {
1881	const name_token = tree.firstToken(field_init) - 2;
1882	const str_index = try astgen.identAsString(name_token);
1883	setExtra(astgen, extra_index, Zir.Inst.StructInitAnon.Item{
1884	.field_name = str_index,
1885	.init = try expr(gz, scope, .{ .rl = .none }, field_init),
1886	});
1887	extra_index += field_size;
1888	}
1889
1890	return gz.addPlNodePayloadIndex(.struct_init_anon, node, payload_index);
1891	}
1892
1893	/// A struct initialization expression using a `struct_init` or `struct_init_ref` instruction.
1894	fn structInitExprTyped(
1895	gz: *GenZir,
1896	scope: *Scope,
1897	node: Ast.Node.Index,
1898	struct_init: Ast.full.StructInit,
1899	ty_inst: Zir.Inst.Ref,
1900	is_ref: bool,
1901	) InnerError!Zir.Inst.Ref {
1902	const astgen = gz.astgen;
1903	const tree = astgen.tree;
1904
1905	const payload_index = try addExtra(astgen, Zir.Inst.StructInit{
1906	.fields_len = @intCast(struct_init.ast.fields.len),
1907	});
1908	const field_size = @typeInfo(Zir.Inst.StructInit.Item).Struct.fields.len;
1909	var extra_index: usize = try reserveExtra(astgen, struct_init.ast.fields.len * field_size);
1910
1911	for (struct_init.ast.fields) \|field_init\| {
1912	const name_token = tree.firstToken(field_init) - 2;
1913	const str_index = try astgen.identAsString(name_token);
1914	const field_ty_inst = try gz.addPlNode(.struct_init_field_type, field_init, Zir.Inst.FieldType{
1915	.container_type = ty_inst,
1916	.name_start = str_index,
1917	});
1918	setExtra(astgen, extra_index, Zir.Inst.StructInit.Item{
1919	.field_type = field_ty_inst.toIndex().?,
1920	.init = try expr(gz, scope, .{ .rl = .{ .coerced_ty = field_ty_inst } }, field_init),
1921	});
1922	extra_index += field_size;
1923	}
1924
1925	const tag: Zir.Inst.Tag = if (is_ref) .struct_init_ref else .struct_init;
1926	return gz.addPlNodePayloadIndex(tag, node, payload_index);
1927	}
1928
1929	/// A struct initialization expression using field pointers.
1930	fn structInitExprPtr(
1931	gz: *GenZir,
1932	scope: *Scope,
1933	node: Ast.Node.Index,
1934	struct_init: Ast.full.StructInit,
1935	ptr_inst: Zir.Inst.Ref,
1936	) InnerError!void {
1937	const astgen = gz.astgen;
1938	const tree = astgen.tree;
1939
1940	const struct_ptr_inst = try gz.addUnNode(.opt_eu_base_ptr_init, ptr_inst, node);
1941
1942	const payload_index = try addExtra(astgen, Zir.Inst.Block{
1943	.body_len = @intCast(struct_init.ast.fields.len),
1944	});
1945	var extra_index = try reserveExtra(astgen, struct_init.ast.fields.len);
1946
1947	for (struct_init.ast.fields) \|field_init\| {
1948	const name_token = tree.firstToken(field_init) - 2;
1949	const str_index = try astgen.identAsString(name_token);
1950	const field_ptr = try gz.addPlNode(.struct_init_field_ptr, field_init, Zir.Inst.Field{
1951	.lhs = struct_ptr_inst,
1952	.field_name_start = str_index,
1953	});
1954	astgen.extra.items[extra_index] = @intFromEnum(field_ptr.toIndex().?);
1955	extra_index += 1;
1956	_ = try expr(gz, scope, .{ .rl = .{ .ptr = .{ .inst = field_ptr } } }, field_init);
1957	}
1958
1959	_ = try gz.addPlNodePayloadIndex(.validate_ptr_struct_init, node, payload_index);
1960	}
1961
1962	/// This explicitly calls expr in a comptime scope by wrapping it in a `block_comptime` if
1963	/// necessary. It should be used whenever we need to force compile-time evaluation of something,
1964	/// such as a type.
1965	/// The function corresponding to `comptime` expression syntax is `comptimeExprAst`.
1966	fn comptimeExpr(
1967	gz: *GenZir,
1968	scope: *Scope,
1969	ri: ResultInfo,
1970	node: Ast.Node.Index,
1971	) InnerError!Zir.Inst.Ref {
1972	if (gz.is_comptime) {
1973	// No need to change anything!
1974	return expr(gz, scope, ri, node);
1975	}
1976
1977	// There's an optimization here: if the body will be evaluated at comptime regardless, there's
1978	// no need to wrap it in a block. This is hard to determine in general, but we can identify a
1979	// common subset of trivially comptime expressions to take down the size of the ZIR a bit.
1980	const tree = gz.astgen.tree;
1981	const main_tokens = tree.nodes.items(.main_token);
1982	const node_tags = tree.nodes.items(.tag);
1983	switch (node_tags[node]) {
1984	// Any identifier in `primitive_instrs` is trivially comptime. In particular, this includes
1985	// some common types, so we can elide `block_comptime` for a few common type annotations.
1986	.identifier => {
1987	const ident_token = main_tokens[node];
1988	const ident_name_raw = tree.tokenSlice(ident_token);
1989	if (primitive_instrs.get(ident_name_raw)) \|zir_const_ref\| {
1990	// No need to worry about result location here, we're not creating a comptime block!
1991	return rvalue(gz, ri, zir_const_ref, node);
1992	}
1993	},
1994
1995	// We can also avoid the block for a few trivial AST tags which are always comptime-known.
1996	.number_literal, .string_literal, .multiline_string_literal, .enum_literal, .error_value => {
1997	// No need to worry about result location here, we're not creating a comptime block!
1998	return expr(gz, scope, ri, node);
1999	},
2000
2001	// Lastly, for labelled blocks, avoid emitting a labelled block directly inside this
2002	// comptime block, because that would be silly! Note that we don't bother doing this for
2003	// unlabelled blocks, since they don't generate blocks at comptime anyway (see `blockExpr`).
2004	.block_two, .block_two_semicolon, .block, .block_semicolon => {
2005	const token_tags = tree.tokens.items(.tag);
2006	const lbrace = main_tokens[node];
2007	// Careful! We can't pass in the real result location here, since it may
2008	// refer to runtime memory. A runtime-to-comptime boundary has to remove
2009	// result location information, compute the result, and copy it to the true
2010	// result location at runtime. We do this below as well.
2011	const ty_only_ri: ResultInfo = .{
2012	.ctx = ri.ctx,
2013	.rl = if (try ri.rl.resultType(gz, node)) \|res_ty\|
2014	.{ .coerced_ty = res_ty }
2015	else
2016	.none,
2017	};
2018	if (token_tags[lbrace - 1] == .colon and
2019	token_tags[lbrace - 2] == .identifier)
2020	{
2021	const node_datas = tree.nodes.items(.data);
2022	switch (node_tags[node]) {
2023	.block_two, .block_two_semicolon => {
2024	const stmts: [2]Ast.Node.Index = .{ node_datas[node].lhs, node_datas[node].rhs };
2025	const stmt_slice = if (stmts[0] == 0)
2026	stmts[0..0]
2027	else if (stmts[1] == 0)
2028	stmts[0..1]
2029	else
2030	stmts[0..2];
2031
2032	const block_ref = try labeledBlockExpr(gz, scope, ty_only_ri, node, stmt_slice, true);
2033	return rvalue(gz, ri, block_ref, node);
2034	},
2035	.block, .block_semicolon => {
2036	const stmts = tree.extra_data[node_datas[node].lhs..node_datas[node].rhs];
2037	// Replace result location and copy back later - see above.
2038	const block_ref = try labeledBlockExpr(gz, scope, ty_only_ri, node, stmts, true);
2039	return rvalue(gz, ri, block_ref, node);
2040	},
2041	else => unreachable,
2042	}
2043	}
2044	},
2045
2046	// In other cases, we don't optimize anything - we need a wrapper comptime block.
2047	else => {},
2048	}
2049
2050	var block_scope = gz.makeSubBlock(scope);
2051	block_scope.is_comptime = true;
2052	defer block_scope.unstack();
2053
2054	const block_inst = try gz.makeBlockInst(.block_comptime, node);
2055	// Replace result location and copy back later - see above.
2056	const ty_only_ri: ResultInfo = .{
2057	.ctx = ri.ctx,
2058	.rl = if (try ri.rl.resultType(gz, node)) \|res_ty\|
2059	.{ .coerced_ty = res_ty }
2060	else
2061	.none,
2062	};
2063	const block_result = try expr(&block_scope, scope, ty_only_ri, node);
2064	if (!gz.refIsNoReturn(block_result)) {
2065	_ = try block_scope.addBreak(.@"break", block_inst, block_result);
2066	}
2067	try block_scope.setBlockBody(block_inst);
2068	try gz.instructions.append(gz.astgen.gpa, block_inst);
2069
2070	return rvalue(gz, ri, block_inst.toRef(), node);
2071	}
2072
2073	/// This one is for an actual `comptime` syntax, and will emit a compile error if
2074	/// the scope is already known to be comptime-evaluated.
2075	/// See `comptimeExpr` for the helper function for calling expr in a comptime scope.
2076	fn comptimeExprAst(
2077	gz: *GenZir,
2078	scope: *Scope,
2079	ri: ResultInfo,
2080	node: Ast.Node.Index,
2081	) InnerError!Zir.Inst.Ref {
2082	const astgen = gz.astgen;
2083	if (gz.is_comptime) {
2084	return astgen.failNode(node, "redundant comptime keyword in already comptime scope", .{});
2085	}
2086	const tree = astgen.tree;
2087	const node_datas = tree.nodes.items(.data);
2088	const body_node = node_datas[node].lhs;
2089	return comptimeExpr(gz, scope, ri, body_node);
2090	}
2091
2092	/// Restore the error return trace index. Performs the restore only if the result is a non-error or
2093	/// if the result location is a non-error-handling expression.
2094	fn restoreErrRetIndex(
2095	gz: *GenZir,
2096	bt: GenZir.BranchTarget,
2097	ri: ResultInfo,
2098	node: Ast.Node.Index,
2099	result: Zir.Inst.Ref,
2100	) !void {
2101	const op = switch (nodeMayEvalToError(gz.astgen.tree, node)) {
2102	.always => return, // never restore/pop
2103	.never => .none, // always restore/pop
2104	.maybe => switch (ri.ctx) {
2105	.error_handling_expr, .@"return", .fn_arg, .const_init => switch (ri.rl) {
2106	.ptr => \|ptr_res\| try gz.addUnNode(.load, ptr_res.inst, node),
2107	.inferred_ptr => blk: {
2108	// This is a terrible workaround for Sema's inability to load from a .alloc_inferred ptr
2109	// before its type has been resolved. There is no valid operand to use here, so error
2110	// traces will be popped prematurely.
2111	// TODO: Update this to do a proper load from the rl_ptr, once Sema can support it.
2112	break :blk .none;
2113	},
2114	.destructure => return, // value must be a tuple or array, so never restore/pop
2115	else => result,
2116	},
2117	else => .none, // always restore/pop
2118	},
2119	};
2120	_ = try gz.addRestoreErrRetIndex(bt, .{ .if_non_error = op }, node);
2121	}
2122
2123	fn breakExpr(parent_gz: GenZir, parent_scope: Scope, node: Ast.Node.Index) InnerError!Zir.Inst.Ref {
2124	const astgen = parent_gz.astgen;
2125	const tree = astgen.tree;
2126	const node_datas = tree.nodes.items(.data);
2127	const break_label = node_datas[node].lhs;
2128	const rhs = node_datas[node].rhs;
2129
2130	// Look for the label in the scope.
2131	var scope = parent_scope;
2132	while (true) {
2133	switch (scope.tag) {
2134	.gen_zir => {
2135	const block_gz = scope.cast(GenZir).?;
2136
2137	if (block_gz.cur_defer_node != 0) {
2138	// We are breaking out of a `defer` block.
2139	return astgen.failNodeNotes(node, "cannot break out of defer expression", .{}, &.{
2140	try astgen.errNoteNode(
2141	block_gz.cur_defer_node,
2142	"defer expression here",
2143	.{},
2144	),
2145	});
2146	}
2147
2148	const block_inst = blk: {
2149	if (break_label != 0) {
2150	if (block_gz.label) \|*label\| {
2151	if (try astgen.tokenIdentEql(label.token, break_label)) {
2152	label.used = true;
2153	break :blk label.block_inst;
2154	}
2155	}
2156	} else if (block_gz.break_block.unwrap()) \|i\| {
2157	break :blk i;
2158	}
2159	// If not the target, start over with the parent
2160	scope = block_gz.parent;
2161	continue;
2162	};
2163	// If we made it here, this block is the target of the break expr
2164
2165	const break_tag: Zir.Inst.Tag = if (block_gz.is_inline)
2166	.break_inline
2167	else
2168	.@"break";
2169
2170	if (rhs == 0) {
2171	_ = try rvalue(parent_gz, block_gz.break_result_info, .void_value, node);
2172
2173	try genDefers(parent_gz, scope, parent_scope, .normal_only);
2174
2175	// As our last action before the break, "pop" the error trace if needed
2176	if (!block_gz.is_comptime)
2177	_ = try parent_gz.addRestoreErrRetIndex(.{ .block = block_inst }, .always, node);
2178
2179	_ = try parent_gz.addBreak(break_tag, block_inst, .void_value);
2180	return Zir.Inst.Ref.unreachable_value;
2181	}
2182
2183	const operand = try reachableExpr(parent_gz, parent_scope, block_gz.break_result_info, rhs, node);
2184
2185	try genDefers(parent_gz, scope, parent_scope, .normal_only);
2186
2187	// As our last action before the break, "pop" the error trace if needed
2188	if (!block_gz.is_comptime)
2189	try restoreErrRetIndex(parent_gz, .{ .block = block_inst }, block_gz.break_result_info, rhs, operand);
2190
2191	switch (block_gz.break_result_info.rl) {
2192	.ptr => {
2193	// In this case we don't have any mechanism to intercept it;
2194	// we assume the result location is written, and we break with void.
2195	_ = try parent_gz.addBreak(break_tag, block_inst, .void_value);
2196	},
2197	.discard => {
2198	_ = try parent_gz.addBreak(break_tag, block_inst, .void_value);
2199	},
2200	else => {
2201	_ = try parent_gz.addBreakWithSrcNode(break_tag, block_inst, operand, rhs);
2202	},
2203	}
2204	return Zir.Inst.Ref.unreachable_value;
2205	},
2206	.local_val => scope = scope.cast(Scope.LocalVal).?.parent,
2207	.local_ptr => scope = scope.cast(Scope.LocalPtr).?.parent,
2208	.namespace, .enum_namespace => break,
2209	.defer_normal, .defer_error => scope = scope.cast(Scope.Defer).?.parent,
2210	.top => unreachable,
2211	}
2212	}
2213	if (break_label != 0) {
2214	const label_name = try astgen.identifierTokenString(break_label);
2215	return astgen.failTok(break_label, "label not found: '{s}'", .{label_name});
2216	} else {
2217	return astgen.failNode(node, "break expression outside loop", .{});
2218	}
2219	}
2220
2221	fn continueExpr(parent_gz: GenZir, parent_scope: Scope, node: Ast.Node.Index) InnerError!Zir.Inst.Ref {
2222	const astgen = parent_gz.astgen;
2223	const tree = astgen.tree;
2224	const node_datas = tree.nodes.items(.data);
2225	const break_label = node_datas[node].lhs;
2226
2227	// Look for the label in the scope.
2228	var scope = parent_scope;
2229	while (true) {
2230	switch (scope.tag) {
2231	.gen_zir => {
2232	const gen_zir = scope.cast(GenZir).?;
2233
2234	if (gen_zir.cur_defer_node != 0) {
2235	return astgen.failNodeNotes(node, "cannot continue out of defer expression", .{}, &.{
2236	try astgen.errNoteNode(
2237	gen_zir.cur_defer_node,
2238	"defer expression here",
2239	.{},
2240	),
2241	});
2242	}
2243	const continue_block = gen_zir.continue_block.unwrap() orelse {
2244	scope = gen_zir.parent;
2245	continue;
2246	};
2247	if (break_label != 0) blk: {
2248	if (gen_zir.label) \|*label\| {
2249	if (try astgen.tokenIdentEql(label.token, break_label)) {
2250	label.used = true;
2251	break :blk;
2252	}
2253	}
2254	// found continue but either it has a different label, or no label
2255	scope = gen_zir.parent;
2256	continue;
2257	}
2258
2259	const break_tag: Zir.Inst.Tag = if (gen_zir.is_inline)
2260	.break_inline
2261	else
2262	.@"break";
2263	if (break_tag == .break_inline) {
2264	_ = try parent_gz.addUnNode(.check_comptime_control_flow, continue_block.toRef(), node);
2265	}
2266
2267	// As our last action before the continue, "pop" the error trace if needed
2268	if (!gen_zir.is_comptime)
2269	_ = try parent_gz.addRestoreErrRetIndex(.{ .block = continue_block }, .always, node);
2270
2271	_ = try parent_gz.addBreak(break_tag, continue_block, .void_value);
2272	return Zir.Inst.Ref.unreachable_value;
2273	},
2274	.local_val => scope = scope.cast(Scope.LocalVal).?.parent,
2275	.local_ptr => scope = scope.cast(Scope.LocalPtr).?.parent,
2276	.defer_normal => {
2277	const defer_scope = scope.cast(Scope.Defer).?;
2278	scope = defer_scope.parent;
2279	try parent_gz.addDefer(defer_scope.index, defer_scope.len);
2280	},
2281	.defer_error => scope = scope.cast(Scope.Defer).?.parent,
2282	.namespace, .enum_namespace => break,
2283	.top => unreachable,
2284	}
2285	}
2286	if (break_label != 0) {
2287	const label_name = try astgen.identifierTokenString(break_label);
2288	return astgen.failTok(break_label, "label not found: '{s}'", .{label_name});
2289	} else {
2290	return astgen.failNode(node, "continue expression outside loop", .{});
2291	}
2292	}
2293
2294	fn blockExpr(
2295	gz: *GenZir,
2296	scope: *Scope,
2297	ri: ResultInfo,
2298	block_node: Ast.Node.Index,
2299	statements: []const Ast.Node.Index,
2300	) InnerError!Zir.Inst.Ref {
2301	const astgen = gz.astgen;
2302	const tree = astgen.tree;
2303	const main_tokens = tree.nodes.items(.main_token);
2304	const token_tags = tree.tokens.items(.tag);
2305
2306	const lbrace = main_tokens[block_node];
2307	if (token_tags[lbrace - 1] == .colon and
2308	token_tags[lbrace - 2] == .identifier)
2309	{
2310	return labeledBlockExpr(gz, scope, ri, block_node, statements, false);
2311	}
2312
2313	if (!gz.is_comptime) {
2314	// Since this block is unlabeled, its control flow is effectively linear and we
2315	// can almost get away with inlining the block here. However, we actually need
2316	// to preserve the .block for Sema, to properly pop the error return trace.
2317
2318	const block_tag: Zir.Inst.Tag = .block;
2319	const block_inst = try gz.makeBlockInst(block_tag, block_node);
2320	try gz.instructions.append(astgen.gpa, block_inst);
2321
2322	var block_scope = gz.makeSubBlock(scope);
2323	defer block_scope.unstack();
2324
2325	try blockExprStmts(&block_scope, &block_scope.base, statements);
2326
2327	if (!block_scope.endsWithNoReturn()) {
2328	// As our last action before the break, "pop" the error trace if needed
2329	_ = try gz.addRestoreErrRetIndex(.{ .block = block_inst }, .always, block_node);
2330	_ = try block_scope.addBreak(.@"break", block_inst, .void_value);
2331	}
2332
2333	try block_scope.setBlockBody(block_inst);
2334	} else {
2335	var sub_gz = gz.makeSubBlock(scope);
2336	try blockExprStmts(&sub_gz, &sub_gz.base, statements);
2337	}
2338
2339	return rvalue(gz, ri, .void_value, block_node);
2340	}
2341
2342	fn checkLabelRedefinition(astgen: AstGen, parent_scope: Scope, label: Ast.TokenIndex) !void {
2343	// Look for the label in the scope.
2344	var scope = parent_scope;
2345	while (true) {
2346	switch (scope.tag) {
2347	.gen_zir => {
2348	const gen_zir = scope.cast(GenZir).?;
2349	if (gen_zir.label) \|prev_label\| {
2350	if (try astgen.tokenIdentEql(label, prev_label.token)) {
2351	const label_name = try astgen.identifierTokenString(label);
2352	return astgen.failTokNotes(label, "redefinition of label '{s}'", .{
2353	label_name,
2354	}, &[_]u32{
2355	try astgen.errNoteTok(
2356	prev_label.token,
2357	"previous definition here",
2358	.{},
2359	),
2360	});
2361	}
2362	}
2363	scope = gen_zir.parent;
2364	},
2365	.local_val => scope = scope.cast(Scope.LocalVal).?.parent,
2366	.local_ptr => scope = scope.cast(Scope.LocalPtr).?.parent,
2367	.defer_normal, .defer_error => scope = scope.cast(Scope.Defer).?.parent,
2368	.namespace, .enum_namespace => break,
2369	.top => unreachable,
2370	}
2371	}
2372	}
2373
2374	fn labeledBlockExpr(
2375	gz: *GenZir,
2376	parent_scope: *Scope,
2377	ri: ResultInfo,
2378	block_node: Ast.Node.Index,
2379	statements: []const Ast.Node.Index,
2380	force_comptime: bool,
2381	) InnerError!Zir.Inst.Ref {
2382	const astgen = gz.astgen;
2383	const tree = astgen.tree;
2384	const main_tokens = tree.nodes.items(.main_token);
2385	const token_tags = tree.tokens.items(.tag);
2386
2387	const lbrace = main_tokens[block_node];
2388	const label_token = lbrace - 2;
2389	assert(token_tags[label_token] == .identifier);
2390
2391	try astgen.checkLabelRedefinition(parent_scope, label_token);
2392
2393	const need_rl = astgen.nodes_need_rl.contains(block_node);
2394	const block_ri: ResultInfo = if (need_rl) ri else .{
2395	.rl = switch (ri.rl) {
2396	.ptr => .{ .ty = (try ri.rl.resultType(gz, block_node)).? },
2397	.inferred_ptr => .none,
2398	else => ri.rl,
2399	},
2400	.ctx = ri.ctx,
2401	};
2402	// We need to call `rvalue` to write through to the pointer only if we had a
2403	// result pointer and aren't forwarding it.
2404	const LocTag = @typeInfo(ResultInfo.Loc).Union.tag_type.?;
2405	const need_result_rvalue = @as(LocTag, block_ri.rl) != @as(LocTag, ri.rl);
2406
2407	// Reserve the Block ZIR instruction index so that we can put it into the GenZir struct
2408	// so that break statements can reference it.
2409	const block_tag: Zir.Inst.Tag = if (force_comptime) .block_comptime else .block;
2410	const block_inst = try gz.makeBlockInst(block_tag, block_node);
2411	try gz.instructions.append(astgen.gpa, block_inst);
2412	var block_scope = gz.makeSubBlock(parent_scope);
2413	block_scope.label = GenZir.Label{
2414	.token = label_token,
2415	.block_inst = block_inst,
2416	};
2417	block_scope.setBreakResultInfo(block_ri);
2418	if (force_comptime) block_scope.is_comptime = true;
2419	defer block_scope.unstack();
2420
2421	try blockExprStmts(&block_scope, &block_scope.base, statements);
2422	if (!block_scope.endsWithNoReturn()) {
2423	// As our last action before the return, "pop" the error trace if needed
2424	_ = try gz.addRestoreErrRetIndex(.{ .block = block_inst }, .always, block_node);
2425	_ = try block_scope.addBreak(.@"break", block_inst, .void_value);
2426	}
2427
2428	if (!block_scope.label.?.used) {
2429	try astgen.appendErrorTok(label_token, "unused block label", .{});
2430	}
2431
2432	try block_scope.setBlockBody(block_inst);
2433	if (need_result_rvalue) {
2434	return rvalue(gz, ri, block_inst.toRef(), block_node);
2435	} else {
2436	return block_inst.toRef();
2437	}
2438	}
2439
2440	fn blockExprStmts(gz: GenZir, parent_scope: Scope, statements: []const Ast.Node.Index) !void {
2441	const astgen = gz.astgen;
2442	const tree = astgen.tree;
2443	const node_tags = tree.nodes.items(.tag);
2444	const node_data = tree.nodes.items(.data);
2445
2446	if (statements.len == 0) return;
2447
2448	var block_arena = std.heap.ArenaAllocator.init(gz.astgen.gpa);
2449	defer block_arena.deinit();
2450	const block_arena_allocator = block_arena.allocator();
2451
2452	var noreturn_src_node: Ast.Node.Index = 0;
2453	var scope = parent_scope;
2454	for (statements) \|statement\| {
2455	if (noreturn_src_node != 0) {
2456	try astgen.appendErrorNodeNotes(
2457	statement,
2458	"unreachable code",
2459	.{},
2460	&[_]u32{
2461	try astgen.errNoteNode(
2462	noreturn_src_node,
2463	"control flow is diverted here",
2464	.{},
2465	),
2466	},
2467	);
2468	}
2469	var inner_node = statement;
2470	while (true) {
2471	switch (node_tags[inner_node]) {
2472	// zig fmt: off
2473	.global_var_decl,
2474	.local_var_decl,
2475	.simple_var_decl,
2476	.aligned_var_decl, => scope = try varDecl(gz, scope, statement, block_arena_allocator, tree.fullVarDecl(statement).?),
2477
2478	.assign_destructure => scope = try assignDestructureMaybeDecls(gz, scope, statement, block_arena_allocator),
2479
2480	.@"defer" => scope = try deferStmt(gz, scope, statement, block_arena_allocator, .defer_normal),
2481	.@"errdefer" => scope = try deferStmt(gz, scope, statement, block_arena_allocator, .defer_error),
2482
2483	.assign => try assign(gz, scope, statement),
2484
2485	.assign_shl => try assignShift(gz, scope, statement, .shl),
2486	.assign_shr => try assignShift(gz, scope, statement, .shr),
2487
2488	.assign_bit_and => try assignOp(gz, scope, statement, .bit_and),
2489	.assign_bit_or => try assignOp(gz, scope, statement, .bit_or),
2490	.assign_bit_xor => try assignOp(gz, scope, statement, .xor),
2491	.assign_div => try assignOp(gz, scope, statement, .div),
2492	.assign_sub => try assignOp(gz, scope, statement, .sub),
2493	.assign_sub_wrap => try assignOp(gz, scope, statement, .subwrap),
2494	.assign_mod => try assignOp(gz, scope, statement, .mod_rem),
2495	.assign_add => try assignOp(gz, scope, statement, .add),
2496	.assign_add_wrap => try assignOp(gz, scope, statement, .addwrap),
2497	.assign_mul => try assignOp(gz, scope, statement, .mul),
2498	.assign_mul_wrap => try assignOp(gz, scope, statement, .mulwrap),
2499
2500	.grouped_expression => {
2501	inner_node = node_data[statement].lhs;
2502	continue;
2503	},
2504
2505	.while_simple,
2506	.while_cont,
2507	.@"while", => _ = try whileExpr(gz, scope, .{ .rl = .none }, inner_node, tree.fullWhile(inner_node).?, true),
2508
2509	.for_simple,
2510	.@"for", => _ = try forExpr(gz, scope, .{ .rl = .none }, inner_node, tree.fullFor(inner_node).?, true),
2511
2512	else => noreturn_src_node = try unusedResultExpr(gz, scope, inner_node),
2513	// zig fmt: on
2514	}
2515	break;
2516	}
2517	}
2518
2519	try genDefers(gz, parent_scope, scope, .normal_only);
2520	try checkUsed(gz, parent_scope, scope);
2521	}
2522
2523	/// Returns AST source node of the thing that is noreturn if the statement is
2524	/// definitely `noreturn`. Otherwise returns 0.
2525	fn unusedResultExpr(gz: GenZir, scope: Scope, statement: Ast.Node.Index) InnerError!Ast.Node.Index {
2526	try emitDbgNode(gz, statement);
2527	// We need to emit an error if the result is not `noreturn` or `void`, but
2528	// we want to avoid adding the ZIR instruction if possible for performance.
2529	const maybe_unused_result = try expr(gz, scope, .{ .rl = .none }, statement);
2530	return addEnsureResult(gz, maybe_unused_result, statement);
2531	}
2532
2533	fn addEnsureResult(gz: *GenZir, maybe_unused_result: Zir.Inst.Ref, statement: Ast.Node.Index) InnerError!Ast.Node.Index {
2534	var noreturn_src_node: Ast.Node.Index = 0;
2535	const elide_check = if (maybe_unused_result.toIndex()) \|inst\| b: {
2536	// Note that this array becomes invalid after appending more items to it
2537	// in the above while loop.
2538	const zir_tags = gz.astgen.instructions.items(.tag);
2539	switch (zir_tags[@intFromEnum(inst)]) {
2540	// For some instructions, modify the zir data
2541	// so we can avoid a separate ensure_result_used instruction.
2542	.call, .field_call => {
2543	const break_extra = gz.astgen.instructions.items(.data)[@intFromEnum(inst)].pl_node.payload_index;
2544	comptime assert(std.meta.fieldIndex(Zir.Inst.Call, "flags") ==
2545	std.meta.fieldIndex(Zir.Inst.FieldCall, "flags"));
2546	const flags: *Zir.Inst.Call.Flags = @ptrCast(&gz.astgen.extra.items[
2547	break_extra + std.meta.fieldIndex(Zir.Inst.Call, "flags").?
2548	]);
2549	flags.ensure_result_used = true;
2550	break :b true;
2551	},
2552	.builtin_call => {
2553	const break_extra = gz.astgen.instructions.items(.data)[@intFromEnum(inst)].pl_node.payload_index;
2554	const flags: *Zir.Inst.BuiltinCall.Flags = @ptrCast(&gz.astgen.extra.items[
2555	break_extra + std.meta.fieldIndex(Zir.Inst.BuiltinCall, "flags").?
2556	]);
2557	flags.ensure_result_used = true;
2558	break :b true;
2559	},
2560
2561	// ZIR instructions that might be a type other than `noreturn` or `void`.
2562	.add,
2563	.addwrap,
2564	.add_sat,
2565	.add_unsafe,
2566	.param,
2567	.param_comptime,
2568	.param_anytype,
2569	.param_anytype_comptime,
2570	.alloc,
2571	.alloc_mut,
2572	.alloc_comptime_mut,
2573	.alloc_inferred,
2574	.alloc_inferred_mut,
2575	.alloc_inferred_comptime,
2576	.alloc_inferred_comptime_mut,
2577	.make_ptr_const,
2578	.array_cat,
2579	.array_mul,
2580	.array_type,
2581	.array_type_sentinel,
2582	.elem_type,
2583	.indexable_ptr_elem_type,
2584	.vector_elem_type,
2585	.vector_type,
2586	.indexable_ptr_len,
2587	.anyframe_type,
2588	.as_node,
2589	.as_shift_operand,
2590	.bit_and,
2591	.bitcast,
2592	.bit_or,
2593	.block,
2594	.block_comptime,
2595	.block_inline,
2596	.declaration,
2597	.suspend_block,
2598	.loop,
2599	.bool_br_and,
2600	.bool_br_or,
2601	.bool_not,
2602	.cmp_lt,
2603	.cmp_lte,
2604	.cmp_eq,
2605	.cmp_gte,
2606	.cmp_gt,
2607	.cmp_neq,
2608	.decl_ref,
2609	.decl_val,
2610	.load,
2611	.div,
2612	.elem_ptr,
2613	.elem_val,
2614	.elem_ptr_node,
2615	.elem_val_node,
2616	.elem_val_imm,
2617	.field_ptr,
2618	.field_val,
2619	.field_ptr_named,
2620	.field_val_named,
2621	.func,
2622	.func_inferred,
2623	.func_fancy,
2624	.int,
2625	.int_big,
2626	.float,
2627	.float128,
2628	.int_type,
2629	.is_non_null,
2630	.is_non_null_ptr,
2631	.is_non_err,
2632	.is_non_err_ptr,
2633	.ret_is_non_err,
2634	.mod_rem,
2635	.mul,
2636	.mulwrap,
2637	.mul_sat,
2638	.ref,
2639	.shl,
2640	.shl_sat,
2641	.shr,
2642	.str,
2643	.sub,
2644	.subwrap,
2645	.sub_sat,
2646	.negate,
2647	.negate_wrap,
2648	.typeof,
2649	.typeof_builtin,
2650	.xor,
2651	.optional_type,
2652	.optional_payload_safe,
2653	.optional_payload_unsafe,
2654	.optional_payload_safe_ptr,
2655	.optional_payload_unsafe_ptr,
2656	.err_union_payload_unsafe,
2657	.err_union_payload_unsafe_ptr,
2658	.err_union_code,
2659	.err_union_code_ptr,
2660	.ptr_type,
2661	.enum_literal,
2662	.merge_error_sets,
2663	.error_union_type,
2664	.bit_not,
2665	.error_value,
2666	.slice_start,
2667	.slice_end,
2668	.slice_sentinel,
2669	.slice_length,
2670	.import,
2671	.switch_block,
2672	.switch_block_ref,
2673	.switch_block_err_union,
2674	.union_init,
2675	.field_type_ref,
2676	.error_set_decl,
2677	.error_set_decl_anon,
2678	.error_set_decl_func,
2679	.enum_from_int,
2680	.int_from_enum,
2681	.type_info,
2682	.size_of,
2683	.bit_size_of,
2684	.typeof_log2_int_type,
2685	.int_from_ptr,
2686	.align_of,
2687	.int_from_bool,
2688	.embed_file,
2689	.error_name,
2690	.sqrt,
2691	.sin,
2692	.cos,
2693	.tan,
2694	.exp,
2695	.exp2,
2696	.log,
2697	.log2,
2698	.log10,
2699	.abs,
2700	.floor,
2701	.ceil,
2702	.trunc,
2703	.round,
2704	.tag_name,
2705	.type_name,
2706	.frame_type,
2707	.frame_size,
2708	.int_from_float,
2709	.float_from_int,
2710	.ptr_from_int,
2711	.float_cast,
2712	.int_cast,
2713	.ptr_cast,
2714	.truncate,
2715	.has_decl,
2716	.has_field,
2717	.clz,
2718	.ctz,
2719	.pop_count,
2720	.byte_swap,
2721	.bit_reverse,
2722	.div_exact,
2723	.div_floor,
2724	.div_trunc,
2725	.mod,
2726	.rem,
2727	.shl_exact,
2728	.shr_exact,
2729	.bit_offset_of,
2730	.offset_of,
2731	.splat,
2732	.reduce,
2733	.shuffle,
2734	.atomic_load,
2735	.atomic_rmw,
2736	.mul_add,
2737	.field_parent_ptr,
2738	.max,
2739	.min,
2740	.c_import,
2741	.@"resume",
2742	.@"await",
2743	.ret_err_value_code,
2744	.closure_get,
2745	.ret_ptr,
2746	.ret_type,
2747	.for_len,
2748	.@"try",
2749	.try_ptr,
2750	.opt_eu_base_ptr_init,
2751	.coerce_ptr_elem_ty,
2752	.struct_init_empty,
2753	.struct_init_empty_result,
2754	.struct_init_empty_ref_result,
2755	.struct_init_anon,
2756	.struct_init,
2757	.struct_init_ref,
2758	.struct_init_field_type,
2759	.struct_init_field_ptr,
2760	.array_init_anon,
2761	.array_init,
2762	.array_init_ref,
2763	.validate_array_init_ref_ty,
2764	.array_init_elem_type,
2765	.array_init_elem_ptr,
2766	=> break :b false,
2767
2768	.extended => switch (gz.astgen.instructions.items(.data)[@intFromEnum(inst)].extended.opcode) {
2769	.breakpoint,
2770	.fence,
2771	.set_float_mode,
2772	.set_align_stack,
2773	.set_cold,
2774	=> break :b true,
2775	else => break :b false,
2776	},
2777
2778	// ZIR instructions that are always `noreturn`.
2779	.@"break",
2780	.break_inline,
2781	.condbr,
2782	.condbr_inline,
2783	.compile_error,
2784	.ret_node,
2785	.ret_load,
2786	.ret_implicit,
2787	.ret_err_value,
2788	.@"unreachable",
2789	.repeat,
2790	.repeat_inline,
2791	.panic,
2792	.trap,
2793	.check_comptime_control_flow,
2794	=> {
2795	noreturn_src_node = statement;
2796	break :b true;
2797	},
2798
2799	// ZIR instructions that are always `void`.
2800	.dbg_stmt,
2801	.dbg_var_ptr,
2802	.dbg_var_val,
2803	.ensure_result_used,
2804	.ensure_result_non_error,
2805	.ensure_err_union_payload_void,
2806	.@"export",
2807	.export_value,
2808	.set_eval_branch_quota,
2809	.atomic_store,
2810	.store_node,
2811	.store_to_inferred_ptr,
2812	.resolve_inferred_alloc,
2813	.set_runtime_safety,
2814	.closure_capture,
2815	.memcpy,
2816	.memset,
2817	.validate_deref,
2818	.validate_destructure,
2819	.save_err_ret_index,
2820	.restore_err_ret_index_unconditional,
2821	.restore_err_ret_index_fn_entry,
2822	.validate_struct_init_ty,
2823	.validate_struct_init_result_ty,
2824	.validate_ptr_struct_init,
2825	.validate_array_init_ty,
2826	.validate_array_init_result_ty,
2827	.validate_ptr_array_init,
2828	.validate_ref_ty,
2829	=> break :b true,
2830
2831	.@"defer" => unreachable,
2832	.defer_err_code => unreachable,
2833	}
2834	} else switch (maybe_unused_result) {
2835	.none => unreachable,
2836
2837	.unreachable_value => b: {
2838	noreturn_src_node = statement;
2839	break :b true;
2840	},
2841
2842	.void_value => true,
2843
2844	else => false,
2845	};
2846	if (!elide_check) {
2847	_ = try gz.addUnNode(.ensure_result_used, maybe_unused_result, statement);
2848	}
2849	return noreturn_src_node;
2850	}
2851
2852	fn countDefers(outer_scope: Scope, inner_scope: Scope) struct {
2853	have_any: bool,
2854	have_normal: bool,
2855	have_err: bool,
2856	need_err_code: bool,
2857	} {
2858	var have_normal = false;
2859	var have_err = false;
2860	var need_err_code = false;
2861	var scope = inner_scope;
2862	while (scope != outer_scope) {
2863	switch (scope.tag) {
2864	.gen_zir => scope = scope.cast(GenZir).?.parent,
2865	.local_val => scope = scope.cast(Scope.LocalVal).?.parent,
2866	.local_ptr => scope = scope.cast(Scope.LocalPtr).?.parent,
2867	.defer_normal => {
2868	const defer_scope = scope.cast(Scope.Defer).?;
2869	scope = defer_scope.parent;
2870
2871	have_normal = true;
2872	},
2873	.defer_error => {
2874	const defer_scope = scope.cast(Scope.Defer).?;
2875	scope = defer_scope.parent;
2876
2877	have_err = true;
2878
2879	const have_err_payload = defer_scope.remapped_err_code != .none;
2880	need_err_code = need_err_code or have_err_payload;
2881	},
2882	.namespace, .enum_namespace => unreachable,
2883	.top => unreachable,
2884	}
2885	}
2886	return .{
2887	.have_any = have_normal or have_err,
2888	.have_normal = have_normal,
2889	.have_err = have_err,
2890	.need_err_code = need_err_code,
2891	};
2892	}
2893
2894	const DefersToEmit = union(enum) {
2895	both: Zir.Inst.Ref, // err code
2896	both_sans_err,
2897	normal_only,
2898	};
2899
2900	fn genDefers(
2901	gz: *GenZir,
2902	outer_scope: *Scope,
2903	inner_scope: *Scope,
2904	which_ones: DefersToEmit,
2905	) InnerError!void {
2906	const gpa = gz.astgen.gpa;
2907
2908	var scope = inner_scope;
2909	while (scope != outer_scope) {
2910	switch (scope.tag) {
2911	.gen_zir => scope = scope.cast(GenZir).?.parent,
2912	.local_val => scope = scope.cast(Scope.LocalVal).?.parent,
2913	.local_ptr => scope = scope.cast(Scope.LocalPtr).?.parent,
2914	.defer_normal => {
2915	const defer_scope = scope.cast(Scope.Defer).?;
2916	scope = defer_scope.parent;
2917	try gz.addDefer(defer_scope.index, defer_scope.len);
2918	},
2919	.defer_error => {
2920	const defer_scope = scope.cast(Scope.Defer).?;
2921	scope = defer_scope.parent;
2922	switch (which_ones) {
2923	.both_sans_err => {
2924	try gz.addDefer(defer_scope.index, defer_scope.len);
2925	},
2926	.both => \|err_code\| {
2927	if (defer_scope.remapped_err_code.unwrap()) \|remapped_err_code\| {
2928	try gz.instructions.ensureUnusedCapacity(gpa, 1);
2929	try gz.astgen.instructions.ensureUnusedCapacity(gpa, 1);
2930
2931	const payload_index = try gz.astgen.addExtra(Zir.Inst.DeferErrCode{
2932	.remapped_err_code = remapped_err_code,
2933	.index = defer_scope.index,
2934	.len = defer_scope.len,
2935	});
2936	const new_index: Zir.Inst.Index = @enumFromInt(gz.astgen.instructions.len);
2937	gz.astgen.instructions.appendAssumeCapacity(.{
2938	.tag = .defer_err_code,
2939	.data = .{ .defer_err_code = .{
2940	.err_code = err_code,
2941	.payload_index = payload_index,
2942	} },
2943	});
2944	gz.instructions.appendAssumeCapacity(new_index);
2945	} else {
2946	try gz.addDefer(defer_scope.index, defer_scope.len);
2947	}
2948	},
2949	.normal_only => continue,
2950	}
2951	},
2952	.namespace, .enum_namespace => unreachable,
2953	.top => unreachable,
2954	}
2955	}
2956	}
2957
2958	fn checkUsed(gz: GenZir, outer_scope: Scope, inner_scope: *Scope) InnerError!void {
2959	const astgen = gz.astgen;
2960
2961	var scope = inner_scope;
2962	while (scope != outer_scope) {
2963	switch (scope.tag) {
2964	.gen_zir => scope = scope.cast(GenZir).?.parent,
2965	.local_val => {
2966	const s = scope.cast(Scope.LocalVal).?;
2967	if (s.used == 0 and s.discarded == 0) {
2968	try astgen.appendErrorTok(s.token_src, "unused {s}", .{@tagName(s.id_cat)});
2969	} else if (s.used != 0 and s.discarded != 0) {
2970	try astgen.appendErrorTokNotes(s.discarded, "pointless discard of {s}", .{@tagName(s.id_cat)}, &[_]u32{
2971	try gz.astgen.errNoteTok(s.used, "used here", .{}),
2972	});
2973	}
2974	scope = s.parent;
2975	},
2976	.local_ptr => {
2977	const s = scope.cast(Scope.LocalPtr).?;
2978	if (s.used == 0 and s.discarded == 0) {
2979	try astgen.appendErrorTok(s.token_src, "unused {s}", .{@tagName(s.id_cat)});
2980	} else {
2981	if (s.used != 0 and s.discarded != 0) {
2982	try astgen.appendErrorTokNotes(s.discarded, "pointless discard of {s}", .{@tagName(s.id_cat)}, &[_]u32{
2983	try astgen.errNoteTok(s.used, "used here", .{}),
2984	});
2985	}
2986	if (s.id_cat == .@"local variable" and !s.used_as_lvalue) {
2987	try astgen.appendErrorTokNotes(s.token_src, "local variable is never mutated", .{}, &.{
2988	try astgen.errNoteTok(s.token_src, "consider using 'const'", .{}),
2989	});
2990	}
2991	}
2992
2993	scope = s.parent;
2994	},
2995	.defer_normal, .defer_error => scope = scope.cast(Scope.Defer).?.parent,
2996	.namespace, .enum_namespace => unreachable,
2997	.top => unreachable,
2998	}
2999	}
3000	}
3001
3002	fn deferStmt(
3003	gz: *GenZir,
3004	scope: *Scope,
3005	node: Ast.Node.Index,
3006	block_arena: Allocator,
3007	scope_tag: Scope.Tag,
3008	) InnerError!*Scope {
3009	var defer_gen = gz.makeSubBlock(scope);
3010	defer_gen.cur_defer_node = node;
3011	defer_gen.any_defer_node = node;
3012	defer defer_gen.unstack();
3013
3014	const tree = gz.astgen.tree;
3015	const node_datas = tree.nodes.items(.data);
3016	const expr_node = node_datas[node].rhs;
3017
3018	const payload_token = node_datas[node].lhs;
3019	var local_val_scope: Scope.LocalVal = undefined;
3020	var opt_remapped_err_code: Zir.Inst.OptionalIndex = .none;
3021	const have_err_code = scope_tag == .defer_error and payload_token != 0;
3022	const sub_scope = if (!have_err_code) &defer_gen.base else blk: {
3023	const ident_name = try gz.astgen.identAsString(payload_token);
3024	const remapped_err_code: Zir.Inst.Index = @enumFromInt(gz.astgen.instructions.len);
3025	opt_remapped_err_code = remapped_err_code.toOptional();
3026	try gz.astgen.instructions.append(gz.astgen.gpa, .{
3027	.tag = .extended,
3028	.data = .{ .extended = .{
3029	.opcode = .value_placeholder,
3030	.small = undefined,
3031	.operand = undefined,
3032	} },
3033	});
3034	const remapped_err_code_ref = remapped_err_code.toRef();
3035	local_val_scope = .{
3036	.parent = &defer_gen.base,
3037	.gen_zir = gz,
3038	.name = ident_name,
3039	.inst = remapped_err_code_ref,
3040	.token_src = payload_token,
3041	.id_cat = .capture,
3042	};
3043	try gz.addDbgVar(.dbg_var_val, ident_name, remapped_err_code_ref);
3044	break :blk &local_val_scope.base;
3045	};
3046	_ = try unusedResultExpr(&defer_gen, sub_scope, expr_node);
3047	try checkUsed(gz, scope, sub_scope);
3048	_ = try defer_gen.addBreak(.break_inline, @enumFromInt(0), .void_value);
3049
3050	// We must handle ref_table for remapped_err_code manually.
3051	const body = defer_gen.instructionsSlice();
3052	const body_len = blk: {
3053	var refs: u32 = 0;
3054	if (opt_remapped_err_code.unwrap()) \|remapped_err_code\| {
3055	var cur_inst = remapped_err_code;
3056	while (gz.astgen.ref_table.get(cur_inst)) \|ref_inst\| {
3057	refs += 1;
3058	cur_inst = ref_inst;
3059	}
3060	}
3061	break :blk gz.astgen.countBodyLenAfterFixups(body) + refs;
3062	};
3063
3064	const index: u32 = @intCast(gz.astgen.extra.items.len);
3065	try gz.astgen.extra.ensureUnusedCapacity(gz.astgen.gpa, body_len);
3066	if (opt_remapped_err_code.unwrap()) \|remapped_err_code\| {
3067	if (gz.astgen.ref_table.fetchRemove(remapped_err_code)) \|kv\| {
3068	gz.astgen.appendPossiblyRefdBodyInst(&gz.astgen.extra, kv.value);
3069	}
3070	}
3071	gz.astgen.appendBodyWithFixups(body);
3072
3073	const defer_scope = try block_arena.create(Scope.Defer);
3074
3075	defer_scope.* = .{
3076	.base = .{ .tag = scope_tag },
3077	.parent = scope,
3078	.index = index,
3079	.len = body_len,
3080	.remapped_err_code = opt_remapped_err_code,
3081	};
3082	return &defer_scope.base;
3083	}
3084
3085	fn varDecl(
3086	gz: *GenZir,
3087	scope: *Scope,
3088	node: Ast.Node.Index,
3089	block_arena: Allocator,
3090	var_decl: Ast.full.VarDecl,
3091	) InnerError!*Scope {
3092	try emitDbgNode(gz, node);
3093	const astgen = gz.astgen;
3094	const tree = astgen.tree;
3095	const token_tags = tree.tokens.items(.tag);
3096	const main_tokens = tree.nodes.items(.main_token);
3097
3098	const name_token = var_decl.ast.mut_token + 1;
3099	const ident_name_raw = tree.tokenSlice(name_token);
3100	if (mem.eql(u8, ident_name_raw, "_")) {
3101	return astgen.failTok(name_token, "'_' used as an identifier without @\"_\" syntax", .{});
3102	}
3103	const ident_name = try astgen.identAsString(name_token);
3104
3105	try astgen.detectLocalShadowing(
3106	scope,
3107	ident_name,
3108	name_token,
3109	ident_name_raw,
3110	if (token_tags[var_decl.ast.mut_token] == .keyword_const) .@"local constant" else .@"local variable",
3111	);
3112
3113	if (var_decl.ast.init_node == 0) {
3114	return astgen.failNode(node, "variables must be initialized", .{});
3115	}
3116
3117	if (var_decl.ast.addrspace_node != 0) {
3118	return astgen.failTok(main_tokens[var_decl.ast.addrspace_node], "cannot set address space of local variable '{s}'", .{ident_name_raw});
3119	}
3120
3121	if (var_decl.ast.section_node != 0) {
3122	return astgen.failTok(main_tokens[var_decl.ast.section_node], "cannot set section of local variable '{s}'", .{ident_name_raw});
3123	}
3124
3125	const align_inst: Zir.Inst.Ref = if (var_decl.ast.align_node != 0)
3126	try expr(gz, scope, coerced_align_ri, var_decl.ast.align_node)
3127	else
3128	.none;
3129
3130	switch (token_tags[var_decl.ast.mut_token]) {
3131	.keyword_const => {
3132	if (var_decl.comptime_token) \|comptime_token\| {
3133	try astgen.appendErrorTok(comptime_token, "'comptime const' is redundant; instead wrap the initialization expression with 'comptime'", .{});
3134	}
3135
3136	// Depending on the type of AST the initialization expression is, we may need an lvalue
3137	// or an rvalue as a result location. If it is an rvalue, we can use the instruction as
3138	// the variable, no memory location needed.
3139	const type_node = var_decl.ast.type_node;
3140	if (align_inst == .none and
3141	!astgen.nodes_need_rl.contains(node))
3142	{
3143	const result_info: ResultInfo = if (type_node != 0) .{
3144	.rl = .{ .ty = try typeExpr(gz, scope, type_node) },
3145	.ctx = .const_init,
3146	} else .{ .rl = .none, .ctx = .const_init };
3147	const prev_anon_name_strategy = gz.anon_name_strategy;
3148	gz.anon_name_strategy = .dbg_var;
3149	const init_inst = try reachableExpr(gz, scope, result_info, var_decl.ast.init_node, node);
3150	gz.anon_name_strategy = prev_anon_name_strategy;
3151
3152	try gz.addDbgVar(.dbg_var_val, ident_name, init_inst);
3153
3154	// The const init expression may have modified the error return trace, so signal
3155	// to Sema that it should save the new index for restoring later.
3156	if (nodeMayAppendToErrorTrace(tree, var_decl.ast.init_node))
3157	_ = try gz.addSaveErrRetIndex(.{ .if_of_error_type = init_inst });
3158
3159	const sub_scope = try block_arena.create(Scope.LocalVal);
3160	sub_scope.* = .{
3161	.parent = scope,
3162	.gen_zir = gz,
3163	.name = ident_name,
3164	.inst = init_inst,
3165	.token_src = name_token,
3166	.id_cat = .@"local constant",
3167	};
3168	return &sub_scope.base;
3169	}
3170
3171	const is_comptime = gz.is_comptime or
3172	tree.nodes.items(.tag)[var_decl.ast.init_node] == .@"comptime";
3173
3174	var resolve_inferred_alloc: Zir.Inst.Ref = .none;
3175	var opt_type_inst: Zir.Inst.Ref = .none;
3176	const init_rl: ResultInfo.Loc = if (type_node != 0) init_rl: {
3177	const type_inst = try typeExpr(gz, scope, type_node);
3178	opt_type_inst = type_inst;
3179	if (align_inst == .none) {
3180	break :init_rl .{ .ptr = .{ .inst = try gz.addUnNode(.alloc, type_inst, node) } };
3181	} else {
3182	break :init_rl .{ .ptr = .{ .inst = try gz.addAllocExtended(.{
3183	.node = node,
3184	.type_inst = type_inst,
3185	.align_inst = align_inst,
3186	.is_const = true,
3187	.is_comptime = is_comptime,
3188	}) } };
3189	}
3190	} else init_rl: {
3191	const alloc_inst = if (align_inst == .none) ptr: {
3192	const tag: Zir.Inst.Tag = if (is_comptime)
3193	.alloc_inferred_comptime
3194	else
3195	.alloc_inferred;
3196	break :ptr try gz.addNode(tag, node);
3197	} else ptr: {
3198	break :ptr try gz.addAllocExtended(.{
3199	.node = node,
3200	.type_inst = .none,
3201	.align_inst = align_inst,
3202	.is_const = true,
3203	.is_comptime = is_comptime,
3204	});
3205	};
3206	resolve_inferred_alloc = alloc_inst;
3207	break :init_rl .{ .inferred_ptr = alloc_inst };
3208	};
3209	const var_ptr = switch (init_rl) {
3210	.ptr => \|ptr\| ptr.inst,
3211	.inferred_ptr => \|inst\| inst,
3212	else => unreachable,
3213	};
3214	const init_result_info: ResultInfo = .{ .rl = init_rl, .ctx = .const_init };
3215
3216	const prev_anon_name_strategy = gz.anon_name_strategy;
3217	gz.anon_name_strategy = .dbg_var;
3218	defer gz.anon_name_strategy = prev_anon_name_strategy;
3219	const init_inst = try reachableExpr(gz, scope, init_result_info, var_decl.ast.init_node, node);
3220
3221	// The const init expression may have modified the error return trace, so signal
3222	// to Sema that it should save the new index for restoring later.
3223	if (nodeMayAppendToErrorTrace(tree, var_decl.ast.init_node))
3224	_ = try gz.addSaveErrRetIndex(.{ .if_of_error_type = init_inst });
3225
3226	const const_ptr = if (resolve_inferred_alloc != .none) p: {
3227	_ = try gz.addUnNode(.resolve_inferred_alloc, resolve_inferred_alloc, node);
3228	break :p var_ptr;
3229	} else try gz.addUnNode(.make_ptr_const, var_ptr, node);
3230
3231	try gz.addDbgVar(.dbg_var_ptr, ident_name, const_ptr);
3232
3233	const sub_scope = try block_arena.create(Scope.LocalPtr);
3234	sub_scope.* = .{
3235	.parent = scope,
3236	.gen_zir = gz,
3237	.name = ident_name,
3238	.ptr = const_ptr,
3239	.token_src = name_token,
3240	.maybe_comptime = true,
3241	.id_cat = .@"local constant",
3242	};
3243	return &sub_scope.base;
3244	},
3245	.keyword_var => {
3246	if (var_decl.comptime_token != null and gz.is_comptime)
3247	return astgen.failTok(var_decl.comptime_token.?, "'comptime var' is redundant in comptime scope", .{});
3248	const is_comptime = var_decl.comptime_token != null or gz.is_comptime;
3249	var resolve_inferred_alloc: Zir.Inst.Ref = .none;
3250	const alloc: Zir.Inst.Ref, const result_info: ResultInfo = if (var_decl.ast.type_node != 0) a: {
3251	const type_inst = try typeExpr(gz, scope, var_decl.ast.type_node);
3252	const alloc = alloc: {
3253	if (align_inst == .none) {
3254	const tag: Zir.Inst.Tag = if (is_comptime)
3255	.alloc_comptime_mut
3256	else
3257	.alloc_mut;
3258	break :alloc try gz.addUnNode(tag, type_inst, node);
3259	} else {
3260	break :alloc try gz.addAllocExtended(.{
3261	.node = node,
3262	.type_inst = type_inst,
3263	.align_inst = align_inst,
3264	.is_const = false,
3265	.is_comptime = is_comptime,
3266	});
3267	}
3268	};
3269	break :a .{ alloc, .{ .rl = .{ .ptr = .{ .inst = alloc } } } };
3270	} else a: {
3271	const alloc = alloc: {
3272	if (align_inst == .none) {
3273	const tag: Zir.Inst.Tag = if (is_comptime)
3274	.alloc_inferred_comptime_mut
3275	else
3276	.alloc_inferred_mut;
3277	break :alloc try gz.addNode(tag, node);
3278	} else {
3279	break :alloc try gz.addAllocExtended(.{
3280	.node = node,
3281	.type_inst = .none,
3282	.align_inst = align_inst,
3283	.is_const = false,
3284	.is_comptime = is_comptime,
3285	});
3286	}
3287	};
3288	resolve_inferred_alloc = alloc;
3289	break :a .{ alloc, .{ .rl = .{ .inferred_ptr = alloc } } };
3290	};
3291	const prev_anon_name_strategy = gz.anon_name_strategy;
3292	gz.anon_name_strategy = .dbg_var;
3293	_ = try reachableExprComptime(gz, scope, result_info, var_decl.ast.init_node, node, is_comptime);
3294	gz.anon_name_strategy = prev_anon_name_strategy;
3295	if (resolve_inferred_alloc != .none) {
3296	_ = try gz.addUnNode(.resolve_inferred_alloc, resolve_inferred_alloc, node);
3297	}
3298
3299	try gz.addDbgVar(.dbg_var_ptr, ident_name, alloc);
3300
3301	const sub_scope = try block_arena.create(Scope.LocalPtr);
3302	sub_scope.* = .{
3303	.parent = scope,
3304	.gen_zir = gz,
3305	.name = ident_name,
3306	.ptr = alloc,
3307	.token_src = name_token,
3308	.maybe_comptime = is_comptime,
3309	.id_cat = .@"local variable",
3310	};
3311	return &sub_scope.base;
3312	},
3313	else => unreachable,
3314	}
3315	}
3316
3317	fn emitDbgNode(gz: *GenZir, node: Ast.Node.Index) !void {
3318	// The instruction emitted here is for debugging runtime code.
3319	// If the current block will be evaluated only during semantic analysis
3320	// then no dbg_stmt ZIR instruction is needed.
3321	if (gz.is_comptime) return;
3322	const astgen = gz.astgen;
3323	astgen.advanceSourceCursorToNode(node);
3324	const line = astgen.source_line - gz.decl_line;
3325	const column = astgen.source_column;
3326	try emitDbgStmt(gz, .{ line, column });
3327	}
3328
3329	fn assign(gz: GenZir, scope: Scope, infix_node: Ast.Node.Index) InnerError!void {
3330	try emitDbgNode(gz, infix_node);
3331	const astgen = gz.astgen;
3332	const tree = astgen.tree;
3333	const node_datas = tree.nodes.items(.data);
3334	const main_tokens = tree.nodes.items(.main_token);
3335	const node_tags = tree.nodes.items(.tag);
3336
3337	const lhs = node_datas[infix_node].lhs;
3338	const rhs = node_datas[infix_node].rhs;
3339	if (node_tags[lhs] == .identifier) {
3340	// This intentionally does not support `@"_"` syntax.
3341	const ident_name = tree.tokenSlice(main_tokens[lhs]);
3342	if (mem.eql(u8, ident_name, "_")) {
3343	_ = try expr(gz, scope, .{ .rl = .discard, .ctx = .assignment }, rhs);
3344	return;
3345	}
3346	}
3347	const lvalue = try lvalExpr(gz, scope, lhs);
3348	_ = try expr(gz, scope, .{ .rl = .{ .ptr = .{
3349	.inst = lvalue,
3350	.src_node = infix_node,
3351	} } }, rhs);
3352	}
3353
3354	/// Handles destructure assignments where no LHS is a `const` or `var` decl.
3355	fn assignDestructure(gz: GenZir, scope: Scope, node: Ast.Node.Index) InnerError!void {
3356	try emitDbgNode(gz, node);
3357	const astgen = gz.astgen;
3358	const tree = astgen.tree;
3359	const token_tags = tree.tokens.items(.tag);
3360	const node_datas = tree.nodes.items(.data);
3361	const main_tokens = tree.nodes.items(.main_token);
3362	const node_tags = tree.nodes.items(.tag);
3363
3364	const extra_index = node_datas[node].lhs;
3365	const lhs_count = tree.extra_data[extra_index];
3366	const lhs_nodes: []const Ast.Node.Index = @ptrCast(tree.extra_data[extra_index + 1 ..][0..lhs_count]);
3367	const rhs = node_datas[node].rhs;
3368
3369	const maybe_comptime_token = tree.firstToken(node) - 1;
3370	const declared_comptime = token_tags[maybe_comptime_token] == .keyword_comptime;
3371
3372	if (declared_comptime and gz.is_comptime) {
3373	return astgen.failNode(node, "redundant comptime keyword in already comptime scope", .{});
3374	}
3375
3376	// If this expression is marked comptime, we must wrap the whole thing in a comptime block.
3377	var gz_buf: GenZir = undefined;
3378	const inner_gz = if (declared_comptime) bs: {
3379	gz_buf = gz.makeSubBlock(scope);
3380	gz_buf.is_comptime = true;
3381	break :bs &gz_buf;
3382	} else gz;
3383	defer if (declared_comptime) inner_gz.unstack();
3384
3385	const rl_components = try astgen.arena.alloc(ResultInfo.Loc.DestructureComponent, lhs_nodes.len);
3386	for (rl_components, lhs_nodes) \|*lhs_rl, lhs_node\| {
3387	if (node_tags[lhs_node] == .identifier) {
3388	// This intentionally does not support `@"_"` syntax.
3389	const ident_name = tree.tokenSlice(main_tokens[lhs_node]);
3390	if (mem.eql(u8, ident_name, "_")) {
3391	lhs_rl.* = .discard;
3392	continue;
3393	}
3394	}
3395	lhs_rl.* = .{ .typed_ptr = .{
3396	.inst = try lvalExpr(inner_gz, scope, lhs_node),
3397	.src_node = lhs_node,
3398	} };
3399	}
3400
3401	const ri: ResultInfo = .{ .rl = .{ .destructure = .{
3402	.src_node = node,
3403	.components = rl_components,
3404	} } };
3405
3406	_ = try expr(inner_gz, scope, ri, rhs);
3407
3408	if (declared_comptime) {
3409	const comptime_block_inst = try gz.makeBlockInst(.block_comptime, node);
3410	_ = try inner_gz.addBreak(.@"break", comptime_block_inst, .void_value);
3411	try inner_gz.setBlockBody(comptime_block_inst);
3412	try gz.instructions.append(gz.astgen.gpa, comptime_block_inst);
3413	}
3414	}
3415
3416	/// Handles destructure assignments where the LHS may contain `const` or `var` decls.
3417	fn assignDestructureMaybeDecls(
3418	gz: *GenZir,
3419	scope: *Scope,
3420	node: Ast.Node.Index,
3421	block_arena: Allocator,
3422	) InnerError!*Scope {
3423	try emitDbgNode(gz, node);
3424	const astgen = gz.astgen;
3425	const tree = astgen.tree;
3426	const token_tags = tree.tokens.items(.tag);
3427	const node_datas = tree.nodes.items(.data);
3428	const main_tokens = tree.nodes.items(.main_token);
3429	const node_tags = tree.nodes.items(.tag);
3430
3431	const extra_index = node_datas[node].lhs;
3432	const lhs_count = tree.extra_data[extra_index];
3433	const lhs_nodes: []const Ast.Node.Index = @ptrCast(tree.extra_data[extra_index + 1 ..][0..lhs_count]);
3434	const rhs = node_datas[node].rhs;
3435
3436	const maybe_comptime_token = tree.firstToken(node) - 1;
3437	const declared_comptime = token_tags[maybe_comptime_token] == .keyword_comptime;
3438	if (declared_comptime and gz.is_comptime) {
3439	return astgen.failNode(node, "redundant comptime keyword in already comptime scope", .{});
3440	}
3441
3442	const is_comptime = declared_comptime or gz.is_comptime;
3443	const rhs_is_comptime = tree.nodes.items(.tag)[rhs] == .@"comptime";
3444
3445	// When declaring consts via a destructure, we always use a result pointer.
3446	// This avoids the need to create tuple types, and is also likely easier to
3447	// optimize, since it's a bit tricky for the optimizer to "split up" the
3448	// value into individual pointer writes down the line.
3449
3450	// We know this rl information won't live past the evaluation of this
3451	// expression, so it may as well go in the block arena.
3452	const rl_components = try block_arena.alloc(ResultInfo.Loc.DestructureComponent, lhs_nodes.len);
3453	var any_non_const_lhs = false;
3454	var any_lvalue_expr = false;
3455	for (rl_components, lhs_nodes) \|*lhs_rl, lhs_node\| {
3456	switch (node_tags[lhs_node]) {
3457	.identifier => {
3458	// This intentionally does not support `@"_"` syntax.
3459	const ident_name = tree.tokenSlice(main_tokens[lhs_node]);
3460	if (mem.eql(u8, ident_name, "_")) {
3461	any_non_const_lhs = true;
3462	lhs_rl.* = .discard;
3463	continue;
3464	}
3465	},
3466	.global_var_decl, .local_var_decl, .simple_var_decl, .aligned_var_decl => {
3467	const full = tree.fullVarDecl(lhs_node).?;
3468
3469	const name_token = full.ast.mut_token + 1;
3470	const ident_name_raw = tree.tokenSlice(name_token);
3471	if (mem.eql(u8, ident_name_raw, "_")) {
3472	return astgen.failTok(name_token, "'_' used as an identifier without @\"_\" syntax", .{});
3473	}
3474
3475	// We detect shadowing in the second pass over these, while we're creating scopes.
3476
3477	if (full.ast.addrspace_node != 0) {
3478	return astgen.failTok(main_tokens[full.ast.addrspace_node], "cannot set address space of local variable '{s}'", .{ident_name_raw});
3479	}
3480	if (full.ast.section_node != 0) {
3481	return astgen.failTok(main_tokens[full.ast.section_node], "cannot set section of local variable '{s}'", .{ident_name_raw});
3482	}
3483
3484	const is_const = switch (token_tags[full.ast.mut_token]) {
3485	.keyword_var => false,
3486	.keyword_const => true,
3487	else => unreachable,
3488	};
3489	if (!is_const) any_non_const_lhs = true;
3490
3491	// We also mark `const`s as comptime if the RHS is definitely comptime-known.
3492	const this_lhs_comptime = is_comptime or (is_const and rhs_is_comptime);
3493
3494	const align_inst: Zir.Inst.Ref = if (full.ast.align_node != 0)
3495	try expr(gz, scope, coerced_align_ri, full.ast.align_node)
3496	else
3497	.none;
3498
3499	if (full.ast.type_node != 0) {
3500	// Typed alloc
3501	const type_inst = try typeExpr(gz, scope, full.ast.type_node);
3502	const ptr = if (align_inst == .none) ptr: {
3503	const tag: Zir.Inst.Tag = if (is_const)
3504	.alloc
3505	else if (this_lhs_comptime)
3506	.alloc_comptime_mut
3507	else
3508	.alloc_mut;
3509	break :ptr try gz.addUnNode(tag, type_inst, node);
3510	} else try gz.addAllocExtended(.{
3511	.node = node,
3512	.type_inst = type_inst,
3513	.align_inst = align_inst,
3514	.is_const = is_const,
3515	.is_comptime = this_lhs_comptime,
3516	});
3517	lhs_rl.* = .{ .typed_ptr = .{ .inst = ptr } };
3518	} else {
3519	// Inferred alloc
3520	const ptr = if (align_inst == .none) ptr: {
3521	const tag: Zir.Inst.Tag = if (is_const) tag: {
3522	break :tag if (this_lhs_comptime) .alloc_inferred_comptime else .alloc_inferred;
3523	} else tag: {
3524	break :tag if (this_lhs_comptime) .alloc_inferred_comptime_mut else .alloc_inferred_mut;
3525	};
3526	break :ptr try gz.addNode(tag, node);
3527	} else try gz.addAllocExtended(.{
3528	.node = node,
3529	.type_inst = .none,
3530	.align_inst = align_inst,
3531	.is_const = is_const,
3532	.is_comptime = this_lhs_comptime,
3533	});
3534	lhs_rl.* = .{ .inferred_ptr = ptr };
3535	}
3536
3537	continue;
3538	},
3539	else => {},
3540	}
3541	// This LHS is just an lvalue expression.
3542	// We will fill in its result pointer later, inside a comptime block.
3543	any_non_const_lhs = true;
3544	any_lvalue_expr = true;
3545	lhs_rl.* = .{ .typed_ptr = .{
3546	.inst = undefined,
3547	.src_node = lhs_node,
3548	} };
3549	}
3550
3551	if (declared_comptime and !any_non_const_lhs) {
3552	try astgen.appendErrorTok(maybe_comptime_token, "'comptime const' is redundant; instead wrap the initialization expression with 'comptime'", .{});
3553	}
3554
3555	// If this expression is marked comptime, we must wrap it in a comptime block.
3556	var gz_buf: GenZir = undefined;
3557	const inner_gz = if (declared_comptime) bs: {
3558	gz_buf = gz.makeSubBlock(scope);
3559	gz_buf.is_comptime = true;
3560	break :bs &gz_buf;
3561	} else gz;
3562	defer if (declared_comptime) inner_gz.unstack();
3563
3564	if (any_lvalue_expr) {
3565	// At least one LHS was an lvalue expr. Iterate again in order to
3566	// evaluate the lvalues from within the possible block_comptime.
3567	for (rl_components, lhs_nodes) \|*lhs_rl, lhs_node\| {
3568	if (lhs_rl.* != .typed_ptr) continue;
3569	switch (node_tags[lhs_node]) {
3570	.global_var_decl, .local_var_decl, .simple_var_decl, .aligned_var_decl => continue,
3571	else => {},
3572	}
3573	lhs_rl.typed_ptr.inst = try lvalExpr(inner_gz, scope, lhs_node);
3574	}
3575	}
3576
3577	// We can't give a reasonable anon name strategy for destructured inits, so
3578	// leave it at its default of `.anon`.
3579	_ = try reachableExpr(inner_gz, scope, .{ .rl = .{ .destructure = .{
3580	.src_node = node,
3581	.components = rl_components,
3582	} } }, rhs, node);
3583
3584	if (declared_comptime) {
3585	// Finish the block_comptime. Inferred alloc resolution etc will occur
3586	// in the parent block.
3587	const comptime_block_inst = try gz.makeBlockInst(.block_comptime, node);
3588	_ = try inner_gz.addBreak(.@"break", comptime_block_inst, .void_value);
3589	try inner_gz.setBlockBody(comptime_block_inst);
3590	try gz.instructions.append(gz.astgen.gpa, comptime_block_inst);
3591	}
3592
3593	// Now, iterate over the LHS exprs to construct any new scopes.
3594	// If there were any inferred allocations, resolve them.
3595	// If there were any `const` decls, make the pointer constant.
3596	var cur_scope = scope;
3597	for (rl_components, lhs_nodes) \|lhs_rl, lhs_node\| {
3598	switch (node_tags[lhs_node]) {
3599	.local_var_decl, .simple_var_decl, .aligned_var_decl => {},
3600	else => continue, // We were mutating an existing lvalue - nothing to do
3601	}
3602	const full = tree.fullVarDecl(lhs_node).?;
3603	const raw_ptr = switch (lhs_rl) {
3604	.discard => unreachable,
3605	.typed_ptr => \|typed_ptr\| typed_ptr.inst,
3606	.inferred_ptr => \|ptr_inst\| ptr_inst,
3607	};
3608	// If the alloc was inferred, resolve it.
3609	if (full.ast.type_node == 0) {
3610	_ = try gz.addUnNode(.resolve_inferred_alloc, raw_ptr, lhs_node);
3611	}
3612	const is_const = switch (token_tags[full.ast.mut_token]) {
3613	.keyword_var => false,
3614	.keyword_const => true,
3615	else => unreachable,
3616	};
3617	// If the alloc was const, make it const.
3618	const var_ptr = if (is_const and full.ast.type_node != 0) make_const: {
3619	// Note that we don't do this if type_node == 0 since `resolve_inferred_alloc`
3620	// handles it for us.
3621	break :make_const try gz.addUnNode(.make_ptr_const, raw_ptr, node);
3622	} else raw_ptr;
3623	const name_token = full.ast.mut_token + 1;
3624	const ident_name_raw = tree.tokenSlice(name_token);
3625	const ident_name = try astgen.identAsString(name_token);
3626	try astgen.detectLocalShadowing(
3627	cur_scope,
3628	ident_name,
3629	name_token,
3630	ident_name_raw,
3631	if (is_const) .@"local constant" else .@"local variable",
3632	);
3633	try gz.addDbgVar(.dbg_var_ptr, ident_name, var_ptr);
3634	// Finally, create the scope.
3635	const sub_scope = try block_arena.create(Scope.LocalPtr);
3636	sub_scope.* = .{
3637	.parent = cur_scope,
3638	.gen_zir = gz,
3639	.name = ident_name,
3640	.ptr = var_ptr,
3641	.token_src = name_token,
3642	.maybe_comptime = is_const or is_comptime,
3643	.id_cat = if (is_const) .@"local constant" else .@"local variable",
3644	};
3645	cur_scope = &sub_scope.base;
3646	}
3647
3648	return cur_scope;
3649	}
3650
3651	fn assignOp(
3652	gz: *GenZir,
3653	scope: *Scope,
3654	infix_node: Ast.Node.Index,
3655	op_inst_tag: Zir.Inst.Tag,
3656	) InnerError!void {
3657	try emitDbgNode(gz, infix_node);
3658	const astgen = gz.astgen;
3659	const tree = astgen.tree;
3660	const node_datas = tree.nodes.items(.data);
3661
3662	const lhs_ptr = try lvalExpr(gz, scope, node_datas[infix_node].lhs);
3663
3664	const cursor = switch (op_inst_tag) {
3665	.add, .sub, .mul, .div, .mod_rem => maybeAdvanceSourceCursorToMainToken(gz, infix_node),
3666	else => undefined,
3667	};
3668	const lhs = try gz.addUnNode(.load, lhs_ptr, infix_node);
3669	const lhs_type = try gz.addUnNode(.typeof, lhs, infix_node);
3670	const rhs = try expr(gz, scope, .{ .rl = .{ .coerced_ty = lhs_type } }, node_datas[infix_node].rhs);
3671
3672	switch (op_inst_tag) {
3673	.add, .sub, .mul, .div, .mod_rem => {
3674	try emitDbgStmt(gz, cursor);
3675	},
3676	else => {},
3677	}
3678	const result = try gz.addPlNode(op_inst_tag, infix_node, Zir.Inst.Bin{
3679	.lhs = lhs,
3680	.rhs = rhs,
3681	});
3682	_ = try gz.addPlNode(.store_node, infix_node, Zir.Inst.Bin{
3683	.lhs = lhs_ptr,
3684	.rhs = result,
3685	});
3686	}
3687
3688	fn assignShift(
3689	gz: *GenZir,
3690	scope: *Scope,
3691	infix_node: Ast.Node.Index,
3692	op_inst_tag: Zir.Inst.Tag,
3693	) InnerError!void {
3694	try emitDbgNode(gz, infix_node);
3695	const astgen = gz.astgen;
3696	const tree = astgen.tree;
3697	const node_datas = tree.nodes.items(.data);
3698
3699	const lhs_ptr = try lvalExpr(gz, scope, node_datas[infix_node].lhs);
3700	const lhs = try gz.addUnNode(.load, lhs_ptr, infix_node);
3701	const rhs_type = try gz.addUnNode(.typeof_log2_int_type, lhs, infix_node);
3702	const rhs = try expr(gz, scope, .{ .rl = .{ .ty = rhs_type } }, node_datas[infix_node].rhs);
3703
3704	const result = try gz.addPlNode(op_inst_tag, infix_node, Zir.Inst.Bin{
3705	.lhs = lhs,
3706	.rhs = rhs,
3707	});
3708	_ = try gz.addPlNode(.store_node, infix_node, Zir.Inst.Bin{
3709	.lhs = lhs_ptr,
3710	.rhs = result,
3711	});
3712	}
3713
3714	fn assignShiftSat(gz: GenZir, scope: Scope, infix_node: Ast.Node.Index) InnerError!void {
3715	try emitDbgNode(gz, infix_node);
3716	const astgen = gz.astgen;
3717	const tree = astgen.tree;
3718	const node_datas = tree.nodes.items(.data);
3719
3720	const lhs_ptr = try lvalExpr(gz, scope, node_datas[infix_node].lhs);
3721	const lhs = try gz.addUnNode(.load, lhs_ptr, infix_node);
3722	// Saturating shift-left allows any integer type for both the LHS and RHS.
3723	const rhs = try expr(gz, scope, .{ .rl = .none }, node_datas[infix_node].rhs);
3724
3725	const result = try gz.addPlNode(.shl_sat, infix_node, Zir.Inst.Bin{
3726	.lhs = lhs,
3727	.rhs = rhs,
3728	});
3729	_ = try gz.addPlNode(.store_node, infix_node, Zir.Inst.Bin{
3730	.lhs = lhs_ptr,
3731	.rhs = result,
3732	});
3733	}
3734
3735	fn ptrType(
3736	gz: *GenZir,
3737	scope: *Scope,
3738	ri: ResultInfo,
3739	node: Ast.Node.Index,
3740	ptr_info: Ast.full.PtrType,
3741	) InnerError!Zir.Inst.Ref {
3742	if (ptr_info.size == .C and ptr_info.allowzero_token != null) {
3743	return gz.astgen.failTok(ptr_info.allowzero_token.?, "C pointers always allow address zero", .{});
3744	}
3745
3746	const source_offset = gz.astgen.source_offset;
3747	const source_line = gz.astgen.source_line;
3748	const source_column = gz.astgen.source_column;
3749	const elem_type = try typeExpr(gz, scope, ptr_info.ast.child_type);
3750
3751	var sentinel_ref: Zir.Inst.Ref = .none;
3752	var align_ref: Zir.Inst.Ref = .none;
3753	var addrspace_ref: Zir.Inst.Ref = .none;
3754	var bit_start_ref: Zir.Inst.Ref = .none;
3755	var bit_end_ref: Zir.Inst.Ref = .none;
3756	var trailing_count: u32 = 0;
3757
3758	if (ptr_info.ast.sentinel != 0) {
3759	// These attributes can appear in any order and they all come before the
3760	// element type so we need to reset the source cursor before generating them.
3761	gz.astgen.source_offset = source_offset;
3762	gz.astgen.source_line = source_line;
3763	gz.astgen.source_column = source_column;
3764
3765	sentinel_ref = try comptimeExpr(gz, scope, .{ .rl = .{ .ty = elem_type } }, ptr_info.ast.sentinel);
3766	trailing_count += 1;
3767	}
3768	if (ptr_info.ast.addrspace_node != 0) {
3769	gz.astgen.source_offset = source_offset;
3770	gz.astgen.source_line = source_line;
3771	gz.astgen.source_column = source_column;
3772
3773	addrspace_ref = try expr(gz, scope, coerced_addrspace_ri, ptr_info.ast.addrspace_node);
3774	trailing_count += 1;
3775	}
3776	if (ptr_info.ast.align_node != 0) {
3777	gz.astgen.source_offset = source_offset;
3778	gz.astgen.source_line = source_line;
3779	gz.astgen.source_column = source_column;
3780
3781	align_ref = try expr(gz, scope, coerced_align_ri, ptr_info.ast.align_node);
3782	trailing_count += 1;
3783	}
3784	if (ptr_info.ast.bit_range_start != 0) {
3785	assert(ptr_info.ast.bit_range_end != 0);
3786	bit_start_ref = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .u16_type } }, ptr_info.ast.bit_range_start);
3787	bit_end_ref = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .u16_type } }, ptr_info.ast.bit_range_end);
3788	trailing_count += 2;
3789	}
3790
3791	const gpa = gz.astgen.gpa;
3792	try gz.instructions.ensureUnusedCapacity(gpa, 1);
3793	try gz.astgen.instructions.ensureUnusedCapacity(gpa, 1);
3794	try gz.astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.PtrType).Struct.fields.len +
3795	trailing_count);
3796
3797	const payload_index = gz.astgen.addExtraAssumeCapacity(Zir.Inst.PtrType{
3798	.elem_type = elem_type,
3799	.src_node = gz.nodeIndexToRelative(node),
3800	});
3801	if (sentinel_ref != .none) {
3802	gz.astgen.extra.appendAssumeCapacity(@intFromEnum(sentinel_ref));
3803	}
3804	if (align_ref != .none) {
3805	gz.astgen.extra.appendAssumeCapacity(@intFromEnum(align_ref));
3806	}
3807	if (addrspace_ref != .none) {
3808	gz.astgen.extra.appendAssumeCapacity(@intFromEnum(addrspace_ref));
3809	}
3810	if (bit_start_ref != .none) {
3811	gz.astgen.extra.appendAssumeCapacity(@intFromEnum(bit_start_ref));
3812	gz.astgen.extra.appendAssumeCapacity(@intFromEnum(bit_end_ref));
3813	}
3814
3815	const new_index: Zir.Inst.Index = @enumFromInt(gz.astgen.instructions.len);
3816	const result = new_index.toRef();
3817	gz.astgen.instructions.appendAssumeCapacity(.{ .tag = .ptr_type, .data = .{
3818	.ptr_type = .{
3819	.flags = .{
3820	.is_allowzero = ptr_info.allowzero_token != null,
3821	.is_mutable = ptr_info.const_token == null,
3822	.is_volatile = ptr_info.volatile_token != null,
3823	.has_sentinel = sentinel_ref != .none,
3824	.has_align = align_ref != .none,
3825	.has_addrspace = addrspace_ref != .none,
3826	.has_bit_range = bit_start_ref != .none,
3827	},
3828	.size = ptr_info.size,
3829	.payload_index = payload_index,
3830	},
3831	} });
3832	gz.instructions.appendAssumeCapacity(new_index);
3833
3834	return rvalue(gz, ri, result, node);
3835	}
3836
3837	fn arrayType(gz: GenZir, scope: Scope, ri: ResultInfo, node: Ast.Node.Index) !Zir.Inst.Ref {
3838	const astgen = gz.astgen;
3839	const tree = astgen.tree;
3840	const node_datas = tree.nodes.items(.data);
3841	const node_tags = tree.nodes.items(.tag);
3842	const main_tokens = tree.nodes.items(.main_token);
3843
3844	const len_node = node_datas[node].lhs;
3845	if (node_tags[len_node] == .identifier and
3846	mem.eql(u8, tree.tokenSlice(main_tokens[len_node]), "_"))
3847	{
3848	return astgen.failNode(len_node, "unable to infer array size", .{});
3849	}
3850	const len = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, len_node);
3851	const elem_type = try typeExpr(gz, scope, node_datas[node].rhs);
3852
3853	const result = try gz.addPlNode(.array_type, node, Zir.Inst.Bin{
3854	.lhs = len,
3855	.rhs = elem_type,
3856	});
3857	return rvalue(gz, ri, result, node);
3858	}
3859
3860	fn arrayTypeSentinel(gz: GenZir, scope: Scope, ri: ResultInfo, node: Ast.Node.Index) !Zir.Inst.Ref {
3861	const astgen = gz.astgen;
3862	const tree = astgen.tree;
3863	const node_datas = tree.nodes.items(.data);
3864	const node_tags = tree.nodes.items(.tag);
3865	const main_tokens = tree.nodes.items(.main_token);
3866	const extra = tree.extraData(node_datas[node].rhs, Ast.Node.ArrayTypeSentinel);
3867
3868	const len_node = node_datas[node].lhs;
3869	if (node_tags[len_node] == .identifier and
3870	mem.eql(u8, tree.tokenSlice(main_tokens[len_node]), "_"))
3871	{
3872	return astgen.failNode(len_node, "unable to infer array size", .{});
3873	}
3874	const len = try reachableExpr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, len_node, node);
3875	const elem_type = try typeExpr(gz, scope, extra.elem_type);
3876	const sentinel = try reachableExprComptime(gz, scope, .{ .rl = .{ .coerced_ty = elem_type } }, extra.sentinel, node, true);
3877
3878	const result = try gz.addPlNode(.array_type_sentinel, node, Zir.Inst.ArrayTypeSentinel{
3879	.len = len,
3880	.elem_type = elem_type,
3881	.sentinel = sentinel,
3882	});
3883	return rvalue(gz, ri, result, node);
3884	}
3885
3886	const WipMembers = struct {
3887	payload: *ArrayListUnmanaged(u32),
3888	payload_top: usize,
3889	field_bits_start: u32,
3890	fields_start: u32,
3891	fields_end: u32,
3892	decl_index: u32 = 0,
3893	field_index: u32 = 0,
3894
3895	const Self = @This();
3896
3897	fn init(gpa: Allocator, payload: *ArrayListUnmanaged(u32), decl_count: u32, field_count: u32, comptime bits_per_field: u32, comptime max_field_size: u32) Allocator.Error!Self {
3898	const payload_top: u32 = @intCast(payload.items.len);
3899	const field_bits_start = payload_top + decl_count;
3900	const fields_start = field_bits_start + if (bits_per_field > 0) blk: {
3901	const fields_per_u32 = 32 / bits_per_field;
3902	break :blk (field_count + fields_per_u32 - 1) / fields_per_u32;
3903	} else 0;
3904	const payload_end = fields_start + field_count * max_field_size;
3905	try payload.resize(gpa, payload_end);
3906	return .{
3907	.payload = payload,
3908	.payload_top = payload_top,
3909	.field_bits_start = field_bits_start,
3910	.fields_start = fields_start,
3911	.fields_end = fields_start,
3912	};
3913	}
3914
3915	fn nextDecl(self: *Self, decl_inst: Zir.Inst.Index) void {
3916	self.payload.items[self.payload_top + self.decl_index] = @intFromEnum(decl_inst);
3917	self.decl_index += 1;
3918	}
3919
3920	fn nextField(self: *Self, comptime bits_per_field: u32, bits: [bits_per_field]bool) void {
3921	const fields_per_u32 = 32 / bits_per_field;
3922	const index = self.field_bits_start + self.field_index / fields_per_u32;
3923	assert(index < self.fields_start);
3924	var bit_bag: u32 = if (self.field_index % fields_per_u32 == 0) 0 else self.payload.items[index];
3925	bit_bag >>= bits_per_field;
3926	comptime var i = 0;
3927	inline while (i < bits_per_field) : (i += 1) {
3928	bit_bag \|= @as(u32, @intFromBool(bits[i])) << (32 - bits_per_field + i);
3929	}
3930	self.payload.items[index] = bit_bag;
3931	self.field_index += 1;
3932	}
3933
3934	fn appendToField(self: *Self, data: u32) void {
3935	assert(self.fields_end < self.payload.items.len);
3936	self.payload.items[self.fields_end] = data;
3937	self.fields_end += 1;
3938	}
3939
3940	fn finishBits(self: *Self, comptime bits_per_field: u32) void {
3941	if (bits_per_field > 0) {
3942	const fields_per_u32 = 32 / bits_per_field;
3943	const empty_field_slots = fields_per_u32 - (self.field_index % fields_per_u32);
3944	if (self.field_index > 0 and empty_field_slots < fields_per_u32) {
3945	const index = self.field_bits_start + self.field_index / fields_per_u32;
3946	self.payload.items[index] >>= @intCast(empty_field_slots * bits_per_field);
3947	}
3948	}
3949	}
3950
3951	fn declsSlice(self: *Self) []u32 {
3952	return self.payload.items[self.payload_top..][0..self.decl_index];
3953	}
3954
3955	fn fieldsSlice(self: *Self) []u32 {
3956	return self.payload.items[self.field_bits_start..self.fields_end];
3957	}
3958
3959	fn deinit(self: *Self) void {
3960	self.payload.items.len = self.payload_top;
3961	}
3962	};
3963
3964	fn fnDecl(
3965	astgen: *AstGen,
3966	gz: *GenZir,
3967	scope: *Scope,
3968	wip_members: *WipMembers,
3969	decl_node: Ast.Node.Index,
3970	body_node: Ast.Node.Index,
3971	fn_proto: Ast.full.FnProto,
3972	) InnerError!void {
3973	const tree = astgen.tree;
3974	const token_tags = tree.tokens.items(.tag);
3975
3976	// missing function name already happened in scanDecls()
3977	const fn_name_token = fn_proto.name_token orelse return error.AnalysisFail;
3978
3979	// We insert this at the beginning so that its instruction index marks the
3980	// start of the top level declaration.
3981	const decl_inst = try gz.makeBlockInst(.declaration, fn_proto.ast.proto_node);
3982	astgen.advanceSourceCursorToNode(decl_node);
3983
3984	var decl_gz: GenZir = .{
3985	.is_comptime = true,
3986	.decl_node_index = fn_proto.ast.proto_node,
3987	.decl_line = astgen.source_line,
3988	.parent = scope,
3989	.astgen = astgen,
3990	.instructions = gz.instructions,
3991	.instructions_top = gz.instructions.items.len,
3992	};
3993	defer decl_gz.unstack();
3994
3995	var fn_gz: GenZir = .{
3996	.is_comptime = false,
3997	.decl_node_index = fn_proto.ast.proto_node,
3998	.decl_line = decl_gz.decl_line,
3999	.parent = &decl_gz.base,
4000	.astgen = astgen,
4001	.instructions = gz.instructions,
4002	.instructions_top = GenZir.unstacked_top,
4003	};
4004	defer fn_gz.unstack();
4005
4006	const is_pub = fn_proto.visib_token != null;
4007	const is_export = blk: {
4008	const maybe_export_token = fn_proto.extern_export_inline_token orelse break :blk false;
4009	break :blk token_tags[maybe_export_token] == .keyword_export;
4010	};
4011	const is_extern = blk: {
4012	const maybe_extern_token = fn_proto.extern_export_inline_token orelse break :blk false;
4013	break :blk token_tags[maybe_extern_token] == .keyword_extern;
4014	};
4015	const has_inline_keyword = blk: {
4016	const maybe_inline_token = fn_proto.extern_export_inline_token orelse break :blk false;
4017	break :blk token_tags[maybe_inline_token] == .keyword_inline;
4018	};
4019	const is_noinline = blk: {
4020	const maybe_noinline_token = fn_proto.extern_export_inline_token orelse break :blk false;
4021	break :blk token_tags[maybe_noinline_token] == .keyword_noinline;
4022	};
4023
4024	const doc_comment_index = try astgen.docCommentAsString(fn_proto.firstToken());
4025
4026	wip_members.nextDecl(decl_inst);
4027
4028	var noalias_bits: u32 = 0;
4029	var params_scope = &fn_gz.base;
4030	const is_var_args = is_var_args: {
4031	var param_type_i: usize = 0;
4032	var it = fn_proto.iterate(tree);
4033	while (it.next()) \|param\| : (param_type_i += 1) {
4034	const is_comptime = if (param.comptime_noalias) \|token\| switch (token_tags[token]) {
4035	.keyword_noalias => is_comptime: {
4036	noalias_bits \|= @as(u32, 1) << (std.math.cast(u5, param_type_i) orelse
4037	return astgen.failTok(token, "this compiler implementation only supports 'noalias' on the first 32 parameters", .{}));
4038	break :is_comptime false;
4039	},
4040	.keyword_comptime => true,
4041	else => false,
4042	} else false;
4043
4044	const is_anytype = if (param.anytype_ellipsis3) \|token\| blk: {
4045	switch (token_tags[token]) {
4046	.keyword_anytype => break :blk true,
4047	.ellipsis3 => break :is_var_args true,
4048	else => unreachable,
4049	}
4050	} else false;
4051
4052	const param_name: Zir.NullTerminatedString = if (param.name_token) \|name_token\| blk: {
4053	const name_bytes = tree.tokenSlice(name_token);
4054	if (mem.eql(u8, "_", name_bytes))
4055	break :blk .empty;
4056
4057	const param_name = try astgen.identAsString(name_token);
4058	if (!is_extern) {
4059	try astgen.detectLocalShadowing(params_scope, param_name, name_token, name_bytes, .@"function parameter");
4060	}
4061	break :blk param_name;
4062	} else if (!is_extern) {
4063	if (param.anytype_ellipsis3) \|tok\| {
4064	return astgen.failTok(tok, "missing parameter name", .{});
4065	} else {
4066	ambiguous: {
4067	if (tree.nodes.items(.tag)[param.type_expr] != .identifier) break :ambiguous;
4068	const main_token = tree.nodes.items(.main_token)[param.type_expr];
4069	const identifier_str = tree.tokenSlice(main_token);
4070	if (isPrimitive(identifier_str)) break :ambiguous;
4071	return astgen.failNodeNotes(
4072	param.type_expr,
4073	"missing parameter name or type",
4074	.{},
4075	&[_]u32{
4076	try astgen.errNoteNode(
4077	param.type_expr,
4078	"if this is a name, annotate its type '{s}: T'",
4079	.{identifier_str},
4080	),
4081	try astgen.errNoteNode(
4082	param.type_expr,
4083	"if this is a type, give it a name '<name>: {s}'",
4084	.{identifier_str},
4085	),
4086	},
4087	);
4088	}
4089	return astgen.failNode(param.type_expr, "missing parameter name", .{});
4090	}
4091	} else .empty;
4092
4093	const param_inst = if (is_anytype) param: {
4094	const name_token = param.name_token orelse param.anytype_ellipsis3.?;
4095	const tag: Zir.Inst.Tag = if (is_comptime)
4096	.param_anytype_comptime
4097	else
4098	.param_anytype;
4099	break :param try decl_gz.addStrTok(tag, param_name, name_token);
4100	} else param: {
4101	const param_type_node = param.type_expr;
4102	assert(param_type_node != 0);
4103	var param_gz = decl_gz.makeSubBlock(scope);
4104	defer param_gz.unstack();
4105	const param_type = try expr(&param_gz, params_scope, coerced_type_ri, param_type_node);
4106	const param_inst_expected: Zir.Inst.Index = @enumFromInt(astgen.instructions.len + 1);
4107	_ = try param_gz.addBreakWithSrcNode(.break_inline, param_inst_expected, param_type, param_type_node);
4108
4109	const main_tokens = tree.nodes.items(.main_token);
4110	const name_token = param.name_token orelse main_tokens[param_type_node];
4111	const tag: Zir.Inst.Tag = if (is_comptime) .param_comptime else .param;
4112	const param_inst = try decl_gz.addParam(&param_gz, tag, name_token, param_name, param.first_doc_comment);
4113	assert(param_inst_expected == param_inst);
4114	break :param param_inst.toRef();
4115	};
4116
4117	if (param_name == .empty or is_extern) continue;
4118
4119	const sub_scope = try astgen.arena.create(Scope.LocalVal);
4120	sub_scope.* = .{
4121	.parent = params_scope,
4122	.gen_zir = &decl_gz,
4123	.name = param_name,
4124	.inst = param_inst,
4125	.token_src = param.name_token.?,
4126	.id_cat = .@"function parameter",
4127	};
4128	params_scope = &sub_scope.base;
4129	}
4130	break :is_var_args false;
4131	};
4132
4133	const lib_name = if (fn_proto.lib_name) \|lib_name_token\| blk: {
4134	const lib_name_str = try astgen.strLitAsString(lib_name_token);
4135	const lib_name_slice = astgen.string_bytes.items[@intFromEnum(lib_name_str.index)..][0..lib_name_str.len];
4136	if (mem.indexOfScalar(u8, lib_name_slice, 0) != null) {
4137	return astgen.failTok(lib_name_token, "library name cannot contain null bytes", .{});
4138	} else if (lib_name_str.len == 0) {
4139	return astgen.failTok(lib_name_token, "library name cannot be empty", .{});
4140	}
4141	break :blk lib_name_str.index;
4142	} else .empty;
4143
4144	const maybe_bang = tree.firstToken(fn_proto.ast.return_type) - 1;
4145	const is_inferred_error = token_tags[maybe_bang] == .bang;
4146
4147	// After creating the function ZIR instruction, it will need to update the break
4148	// instructions inside the expression blocks for align, addrspace, cc, and ret_ty
4149	// to use the function instruction as the "block" to break from.
4150
4151	var align_gz = decl_gz.makeSubBlock(params_scope);
4152	defer align_gz.unstack();
4153	const align_ref: Zir.Inst.Ref = if (fn_proto.ast.align_expr == 0) .none else inst: {
4154	const inst = try expr(&decl_gz, params_scope, coerced_align_ri, fn_proto.ast.align_expr);
4155	if (align_gz.instructionsSlice().len == 0) {
4156	// In this case we will send a len=0 body which can be encoded more efficiently.
4157	break :inst inst;
4158	}
4159	_ = try align_gz.addBreak(.break_inline, @enumFromInt(0), inst);
4160	break :inst inst;
4161	};
4162
4163	var addrspace_gz = decl_gz.makeSubBlock(params_scope);
4164	defer addrspace_gz.unstack();
4165	const addrspace_ref: Zir.Inst.Ref = if (fn_proto.ast.addrspace_expr == 0) .none else inst: {
4166	const inst = try expr(&decl_gz, params_scope, coerced_addrspace_ri, fn_proto.ast.addrspace_expr);
4167	if (addrspace_gz.instructionsSlice().len == 0) {
4168	// In this case we will send a len=0 body which can be encoded more efficiently.
4169	break :inst inst;
4170	}
4171	_ = try addrspace_gz.addBreak(.break_inline, @enumFromInt(0), inst);
4172	break :inst inst;
4173	};
4174
4175	var section_gz = decl_gz.makeSubBlock(params_scope);
4176	defer section_gz.unstack();
4177	const section_ref: Zir.Inst.Ref = if (fn_proto.ast.section_expr == 0) .none else inst: {
4178	const inst = try expr(&decl_gz, params_scope, .{ .rl = .{ .coerced_ty = .slice_const_u8_type } }, fn_proto.ast.section_expr);
4179	if (section_gz.instructionsSlice().len == 0) {
4180	// In this case we will send a len=0 body which can be encoded more efficiently.
4181	break :inst inst;
4182	}
4183	_ = try section_gz.addBreak(.break_inline, @enumFromInt(0), inst);
4184	break :inst inst;
4185	};
4186
4187	var cc_gz = decl_gz.makeSubBlock(params_scope);
4188	defer cc_gz.unstack();
4189	const cc_ref: Zir.Inst.Ref = blk: {
4190	if (fn_proto.ast.callconv_expr != 0) {
4191	if (has_inline_keyword) {
4192	return astgen.failNode(
4193	fn_proto.ast.callconv_expr,
4194	"explicit callconv incompatible with inline keyword",
4195	.{},
4196	);
4197	}
4198	const inst = try expr(
4199	&decl_gz,
4200	params_scope,
4201	.{ .rl = .{ .coerced_ty = .calling_convention_type } },
4202	fn_proto.ast.callconv_expr,
4203	);
4204	if (cc_gz.instructionsSlice().len == 0) {
4205	// In this case we will send a len=0 body which can be encoded more efficiently.
4206	break :blk inst;
4207	}
4208	_ = try cc_gz.addBreak(.break_inline, @enumFromInt(0), inst);
4209	break :blk inst;
4210	} else if (is_extern) {
4211	// note: https://github.com/ziglang/zig/issues/5269
4212	break :blk .calling_convention_c;
4213	} else if (has_inline_keyword) {
4214	break :blk .calling_convention_inline;
4215	} else {
4216	break :blk .none;
4217	}
4218	};
4219
4220	var ret_gz = decl_gz.makeSubBlock(params_scope);
4221	defer ret_gz.unstack();
4222	const ret_ref: Zir.Inst.Ref = inst: {
4223	const inst = try expr(&ret_gz, params_scope, coerced_type_ri, fn_proto.ast.return_type);
4224	if (ret_gz.instructionsSlice().len == 0) {
4225	// In this case we will send a len=0 body which can be encoded more efficiently.
4226	break :inst inst;
4227	}
4228	_ = try ret_gz.addBreak(.break_inline, @enumFromInt(0), inst);
4229	break :inst inst;
4230	};
4231
4232	const func_inst: Zir.Inst.Ref = if (body_node == 0) func: {
4233	if (!is_extern) {
4234	return astgen.failTok(fn_proto.ast.fn_token, "non-extern function has no body", .{});
4235	}
4236	if (is_inferred_error) {
4237	return astgen.failTok(maybe_bang, "function prototype may not have inferred error set", .{});
4238	}
4239	break :func try decl_gz.addFunc(.{
4240	.src_node = decl_node,
4241	.cc_ref = cc_ref,
4242	.cc_gz = &cc_gz,
4243	.align_ref = align_ref,
4244	.align_gz = &align_gz,
4245	.ret_ref = ret_ref,
4246	.ret_gz = &ret_gz,
4247	.section_ref = section_ref,
4248	.section_gz = &section_gz,
4249	.addrspace_ref = addrspace_ref,
4250	.addrspace_gz = &addrspace_gz,
4251	.param_block = decl_inst,
4252	.body_gz = null,
4253	.lib_name = lib_name,
4254	.is_var_args = is_var_args,
4255	.is_inferred_error = false,
4256	.is_test = false,
4257	.is_extern = true,
4258	.is_noinline = is_noinline,
4259	.noalias_bits = noalias_bits,
4260	});
4261	} else func: {
4262	// as a scope, fn_gz encloses ret_gz, but for instruction list, fn_gz stacks on ret_gz
4263	fn_gz.instructions_top = ret_gz.instructions.items.len;
4264
4265	const prev_fn_block = astgen.fn_block;
4266	const prev_fn_ret_ty = astgen.fn_ret_ty;
4267	astgen.fn_block = &fn_gz;
4268	astgen.fn_ret_ty = if (is_inferred_error or ret_ref.toIndex() != null) r: {
4269	// We're essentially guaranteed to need the return type at some point,
4270	// since the return type is likely not `void` or `noreturn` so there
4271	// will probably be an explicit return requiring RLS. Fetch this
4272	// return type now so the rest of the function can use it.
4273	break :r try fn_gz.addNode(.ret_type, decl_node);
4274	} else ret_ref;
4275	defer {
4276	astgen.fn_block = prev_fn_block;
4277	astgen.fn_ret_ty = prev_fn_ret_ty;
4278	}
4279
4280	const prev_var_args = astgen.fn_var_args;
4281	astgen.fn_var_args = is_var_args;
4282	defer astgen.fn_var_args = prev_var_args;
4283
4284	astgen.advanceSourceCursorToNode(body_node);
4285	const lbrace_line = astgen.source_line - decl_gz.decl_line;
4286	const lbrace_column = astgen.source_column;
4287
4288	_ = try expr(&fn_gz, params_scope, .{ .rl = .none }, body_node);
4289	try checkUsed(gz, &fn_gz.base, params_scope);
4290
4291	if (!fn_gz.endsWithNoReturn()) {
4292	// As our last action before the return, "pop" the error trace if needed
4293	_ = try fn_gz.addRestoreErrRetIndex(.ret, .always, decl_node);
4294
4295	// Add implicit return at end of function.
4296	_ = try fn_gz.addUnTok(.ret_implicit, .void_value, tree.lastToken(body_node));
4297	}
4298
4299	break :func try decl_gz.addFunc(.{
4300	.src_node = decl_node,
4301	.cc_ref = cc_ref,
4302	.cc_gz = &cc_gz,
4303	.align_ref = align_ref,
4304	.align_gz = &align_gz,
4305	.ret_ref = ret_ref,
4306	.ret_gz = &ret_gz,
4307	.section_ref = section_ref,
4308	.section_gz = &section_gz,
4309	.addrspace_ref = addrspace_ref,
4310	.addrspace_gz = &addrspace_gz,
4311	.lbrace_line = lbrace_line,
4312	.lbrace_column = lbrace_column,
4313	.param_block = decl_inst,
4314	.body_gz = &fn_gz,
4315	.lib_name = lib_name,
4316	.is_var_args = is_var_args,
4317	.is_inferred_error = is_inferred_error,
4318	.is_test = false,
4319	.is_extern = false,
4320	.is_noinline = is_noinline,
4321	.noalias_bits = noalias_bits,
4322	});
4323	};
4324
4325	// We add this at the end so that its instruction index marks the end range
4326	// of the top level declaration. addFunc already unstacked fn_gz and ret_gz.
4327	_ = try decl_gz.addBreak(.break_inline, decl_inst, func_inst);
4328
4329	try setDeclaration(
4330	decl_inst,
4331	std.zig.hashSrc(tree.getNodeSource(decl_node)),
4332	.{ .named = fn_name_token },
4333	decl_gz.decl_line - gz.decl_line,
4334	is_pub,
4335	is_export,
4336	doc_comment_index,
4337	&decl_gz,
4338	// align, linksection, and addrspace are passed in the func instruction in this case.
4339	// TODO: move them from the function instruction to the declaration instruction?
4340	null,
4341	);
4342	}
4343
4344	fn globalVarDecl(
4345	astgen: *AstGen,
4346	gz: *GenZir,
4347	scope: *Scope,
4348	wip_members: *WipMembers,
4349	node: Ast.Node.Index,
4350	var_decl: Ast.full.VarDecl,
4351	) InnerError!void {
4352	const tree = astgen.tree;
4353	const token_tags = tree.tokens.items(.tag);
4354
4355	const is_mutable = token_tags[var_decl.ast.mut_token] == .keyword_var;
4356	// We do this at the beginning so that the instruction index marks the range start
4357	// of the top level declaration.
4358	const decl_inst = try gz.makeBlockInst(.declaration, node);
4359
4360	const name_token = var_decl.ast.mut_token + 1;
4361	astgen.advanceSourceCursorToNode(node);
4362
4363	var block_scope: GenZir = .{
4364	.parent = scope,
4365	.decl_node_index = node,
4366	.decl_line = astgen.source_line,
4367	.astgen = astgen,
4368	.is_comptime = true,
4369	.anon_name_strategy = .parent,
4370	.instructions = gz.instructions,
4371	.instructions_top = gz.instructions.items.len,
4372	};
4373	defer block_scope.unstack();
4374
4375	const is_pub = var_decl.visib_token != null;
4376	const is_export = blk: {
4377	const maybe_export_token = var_decl.extern_export_token orelse break :blk false;
4378	break :blk token_tags[maybe_export_token] == .keyword_export;
4379	};
4380	const is_extern = blk: {
4381	const maybe_extern_token = var_decl.extern_export_token orelse break :blk false;
4382	break :blk token_tags[maybe_extern_token] == .keyword_extern;
4383	};
4384	wip_members.nextDecl(decl_inst);
4385
4386	const is_threadlocal = if (var_decl.threadlocal_token) \|tok\| blk: {
4387	if (!is_mutable) {
4388	return astgen.failTok(tok, "threadlocal variable cannot be constant", .{});
4389	}
4390	break :blk true;
4391	} else false;
4392
4393	const lib_name = if (var_decl.lib_name) \|lib_name_token\| blk: {
4394	const lib_name_str = try astgen.strLitAsString(lib_name_token);
4395	const lib_name_slice = astgen.string_bytes.items[@intFromEnum(lib_name_str.index)..][0..lib_name_str.len];
4396	if (mem.indexOfScalar(u8, lib_name_slice, 0) != null) {
4397	return astgen.failTok(lib_name_token, "library name cannot contain null bytes", .{});
4398	} else if (lib_name_str.len == 0) {
4399	return astgen.failTok(lib_name_token, "library name cannot be empty", .{});
4400	}
4401	break :blk lib_name_str.index;
4402	} else .empty;
4403
4404	const doc_comment_index = try astgen.docCommentAsString(var_decl.firstToken());
4405
4406	assert(var_decl.comptime_token == null); // handled by parser
4407
4408	const var_inst: Zir.Inst.Ref = if (var_decl.ast.init_node != 0) vi: {
4409	if (is_extern) {
4410	return astgen.failNode(
4411	var_decl.ast.init_node,
4412	"extern variables have no initializers",
4413	.{},
4414	);
4415	}
4416
4417	const type_inst: Zir.Inst.Ref = if (var_decl.ast.type_node != 0)
4418	try expr(
4419	&block_scope,
4420	&block_scope.base,
4421	coerced_type_ri,
4422	var_decl.ast.type_node,
4423	)
4424	else
4425	.none;
4426
4427	const init_inst = try expr(
4428	&block_scope,
4429	&block_scope.base,
4430	if (type_inst != .none) .{ .rl = .{ .ty = type_inst } } else .{ .rl = .none },
4431	var_decl.ast.init_node,
4432	);
4433
4434	if (is_mutable) {
4435	const var_inst = try block_scope.addVar(.{
4436	.var_type = type_inst,
4437	.lib_name = .empty,
4438	.align_inst = .none, // passed via the decls data
4439	.init = init_inst,
4440	.is_extern = false,
4441	.is_const = !is_mutable,
4442	.is_threadlocal = is_threadlocal,
4443	});
4444	break :vi var_inst;
4445	} else {
4446	break :vi init_inst;
4447	}
4448	} else if (!is_extern) {
4449	return astgen.failNode(node, "variables must be initialized", .{});
4450	} else if (var_decl.ast.type_node != 0) vi: {
4451	// Extern variable which has an explicit type.
4452	const type_inst = try typeExpr(&block_scope, &block_scope.base, var_decl.ast.type_node);
4453
4454	const var_inst = try block_scope.addVar(.{
4455	.var_type = type_inst,
4456	.lib_name = lib_name,
4457	.align_inst = .none, // passed via the decls data
4458	.init = .none,
4459	.is_extern = true,
4460	.is_const = !is_mutable,
4461	.is_threadlocal = is_threadlocal,
4462	});
4463	break :vi var_inst;
4464	} else {
4465	return astgen.failNode(node, "unable to infer variable type", .{});
4466	};
4467
4468	// We do this at the end so that the instruction index marks the end
4469	// range of a top level declaration.
4470	_ = try block_scope.addBreakWithSrcNode(.break_inline, decl_inst, var_inst, node);
4471
4472	var align_gz = block_scope.makeSubBlock(scope);
4473	if (var_decl.ast.align_node != 0) {
4474	const align_inst = try expr(&align_gz, &align_gz.base, coerced_align_ri, var_decl.ast.align_node);
4475	_ = try align_gz.addBreakWithSrcNode(.break_inline, decl_inst, align_inst, node);
4476	}
4477
4478	var linksection_gz = align_gz.makeSubBlock(scope);
4479	if (var_decl.ast.section_node != 0) {
4480	const linksection_inst = try expr(&linksection_gz, &linksection_gz.base, coerced_linksection_ri, var_decl.ast.section_node);
4481	_ = try linksection_gz.addBreakWithSrcNode(.break_inline, decl_inst, linksection_inst, node);
4482	}
4483
4484	var addrspace_gz = linksection_gz.makeSubBlock(scope);
4485	if (var_decl.ast.addrspace_node != 0) {
4486	const addrspace_inst = try expr(&addrspace_gz, &addrspace_gz.base, coerced_addrspace_ri, var_decl.ast.addrspace_node);
4487	_ = try addrspace_gz.addBreakWithSrcNode(.break_inline, decl_inst, addrspace_inst, node);
4488	}
4489
4490	try setDeclaration(
4491	decl_inst,
4492	std.zig.hashSrc(tree.getNodeSource(node)),
4493	.{ .named = name_token },
4494	block_scope.decl_line - gz.decl_line,
4495	is_pub,
4496	is_export,
4497	doc_comment_index,
4498	&block_scope,
4499	.{
4500	.align_gz = &align_gz,
4501	.linksection_gz = &linksection_gz,
4502	.addrspace_gz = &addrspace_gz,
4503	},
4504	);
4505	}
4506
4507	fn comptimeDecl(
4508	astgen: *AstGen,
4509	gz: *GenZir,
4510	scope: *Scope,
4511	wip_members: *WipMembers,
4512	node: Ast.Node.Index,
4513	) InnerError!void {
4514	const tree = astgen.tree;
4515	const node_datas = tree.nodes.items(.data);
4516	const body_node = node_datas[node].lhs;
4517
4518	// Up top so the ZIR instruction index marks the start range of this
4519	// top-level declaration.
4520	const decl_inst = try gz.makeBlockInst(.declaration, node);
4521	wip_members.nextDecl(decl_inst);
4522	astgen.advanceSourceCursorToNode(node);
4523
4524	var decl_block: GenZir = .{
4525	.is_comptime = true,
4526	.decl_node_index = node,
4527	.decl_line = astgen.source_line,
4528	.parent = scope,
4529	.astgen = astgen,
4530	.instructions = gz.instructions,
4531	.instructions_top = gz.instructions.items.len,
4532	};
4533	defer decl_block.unstack();
4534
4535	const block_result = try expr(&decl_block, &decl_block.base, .{ .rl = .none }, body_node);
4536	if (decl_block.isEmpty() or !decl_block.refIsNoReturn(block_result)) {
4537	_ = try decl_block.addBreak(.break_inline, decl_inst, .void_value);
4538	}
4539
4540	try setDeclaration(
4541	decl_inst,
4542	std.zig.hashSrc(tree.getNodeSource(node)),
4543	.@"comptime",
4544	decl_block.decl_line - gz.decl_line,
4545	false,
4546	false,
4547	.empty,
4548	&decl_block,
4549	null,
4550	);
4551	}
4552
4553	fn usingnamespaceDecl(
4554	astgen: *AstGen,
4555	gz: *GenZir,
4556	scope: *Scope,
4557	wip_members: *WipMembers,
4558	node: Ast.Node.Index,
4559	) InnerError!void {
4560	const tree = astgen.tree;
4561	const node_datas = tree.nodes.items(.data);
4562
4563	const type_expr = node_datas[node].lhs;
4564	const is_pub = blk: {
4565	const main_tokens = tree.nodes.items(.main_token);
4566	const token_tags = tree.tokens.items(.tag);
4567	const main_token = main_tokens[node];
4568	break :blk (main_token > 0 and token_tags[main_token - 1] == .keyword_pub);
4569	};
4570	// Up top so the ZIR instruction index marks the start range of this
4571	// top-level declaration.
4572	const decl_inst = try gz.makeBlockInst(.declaration, node);
4573	wip_members.nextDecl(decl_inst);
4574	astgen.advanceSourceCursorToNode(node);
4575
4576	var decl_block: GenZir = .{
4577	.is_comptime = true,
4578	.decl_node_index = node,
4579	.decl_line = astgen.source_line,
4580	.parent = scope,
4581	.astgen = astgen,
4582	.instructions = gz.instructions,
4583	.instructions_top = gz.instructions.items.len,
4584	};
4585	defer decl_block.unstack();
4586
4587	const namespace_inst = try typeExpr(&decl_block, &decl_block.base, type_expr);
4588	_ = try decl_block.addBreak(.break_inline, decl_inst, namespace_inst);
4589
4590	try setDeclaration(
4591	decl_inst,
4592	std.zig.hashSrc(tree.getNodeSource(node)),
4593	.@"usingnamespace",
4594	decl_block.decl_line - gz.decl_line,
4595	is_pub,
4596	false,
4597	.empty,
4598	&decl_block,
4599	null,
4600	);
4601	}
4602
4603	fn testDecl(
4604	astgen: *AstGen,
4605	gz: *GenZir,
4606	scope: *Scope,
4607	wip_members: *WipMembers,
4608	node: Ast.Node.Index,
4609	) InnerError!void {
4610	const tree = astgen.tree;
4611	const node_datas = tree.nodes.items(.data);
4612	const body_node = node_datas[node].rhs;
4613
4614	// Up top so the ZIR instruction index marks the start range of this
4615	// top-level declaration.
4616	const decl_inst = try gz.makeBlockInst(.declaration, node);
4617
4618	wip_members.nextDecl(decl_inst);
4619	astgen.advanceSourceCursorToNode(node);
4620
4621	var decl_block: GenZir = .{
4622	.is_comptime = true,
4623	.decl_node_index = node,
4624	.decl_line = astgen.source_line,
4625	.parent = scope,
4626	.astgen = astgen,
4627	.instructions = gz.instructions,
4628	.instructions_top = gz.instructions.items.len,
4629	};
4630	defer decl_block.unstack();
4631
4632	const main_tokens = tree.nodes.items(.main_token);
4633	const token_tags = tree.tokens.items(.tag);
4634	const test_token = main_tokens[node];
4635	const test_name_token = test_token + 1;
4636	const test_name: DeclarationName = switch (token_tags[test_name_token]) {
4637	else => .unnamed_test,
4638	.string_literal => .{ .named_test = test_name_token },
4639	.identifier => blk: {
4640	const ident_name_raw = tree.tokenSlice(test_name_token);
4641
4642	if (mem.eql(u8, ident_name_raw, "_")) return astgen.failTok(test_name_token, "'_' used as an identifier without @\"_\" syntax", .{});
4643
4644	// if not @"" syntax, just use raw token slice
4645	if (ident_name_raw[0] != '@') {
4646	if (isPrimitive(ident_name_raw)) return astgen.failTok(test_name_token, "cannot test a primitive", .{});
4647	}
4648
4649	// Local variables, including function parameters.
4650	const name_str_index = try astgen.identAsString(test_name_token);
4651	var s = scope;
4652	var found_already: ?Ast.Node.Index = null; // we have found a decl with the same name already
4653	var num_namespaces_out: u32 = 0;
4654	var capturing_namespace: ?*Scope.Namespace = null;
4655	while (true) switch (s.tag) {
4656	.local_val => {
4657	const local_val = s.cast(Scope.LocalVal).?;
4658	if (local_val.name == name_str_index) {
4659	local_val.used = test_name_token;
4660	return astgen.failTokNotes(test_name_token, "cannot test a {s}", .{
4661	@tagName(local_val.id_cat),
4662	}, &[_]u32{
4663	try astgen.errNoteTok(local_val.token_src, "{s} declared here", .{
4664	@tagName(local_val.id_cat),
4665	}),
4666	});
4667	}
4668	s = local_val.parent;
4669	},
4670	.local_ptr => {
4671	const local_ptr = s.cast(Scope.LocalPtr).?;
4672	if (local_ptr.name == name_str_index) {
4673	local_ptr.used = test_name_token;
4674	return astgen.failTokNotes(test_name_token, "cannot test a {s}", .{
4675	@tagName(local_ptr.id_cat),
4676	}, &[_]u32{
4677	try astgen.errNoteTok(local_ptr.token_src, "{s} declared here", .{
4678	@tagName(local_ptr.id_cat),
4679	}),
4680	});
4681	}
4682	s = local_ptr.parent;
4683	},
4684	.gen_zir => s = s.cast(GenZir).?.parent,
4685	.defer_normal, .defer_error => s = s.cast(Scope.Defer).?.parent,
4686	.namespace, .enum_namespace => {
4687	const ns = s.cast(Scope.Namespace).?;
4688	if (ns.decls.get(name_str_index)) \|i\| {
4689	if (found_already) \|f\| {
4690	return astgen.failTokNotes(test_name_token, "ambiguous reference", .{}, &.{
4691	try astgen.errNoteNode(f, "declared here", .{}),
4692	try astgen.errNoteNode(i, "also declared here", .{}),
4693	});
4694	}
4695	// We found a match but must continue looking for ambiguous references to decls.
4696	found_already = i;
4697	}
4698	num_namespaces_out += 1;
4699	capturing_namespace = ns;
4700	s = ns.parent;
4701	},
4702	.top => break,
4703	};
4704	if (found_already == null) {
4705	const ident_name = try astgen.identifierTokenString(test_name_token);
4706	return astgen.failTok(test_name_token, "use of undeclared identifier '{s}'", .{ident_name});
4707	}
4708
4709	break :blk .{ .decltest = name_str_index };
4710	},
4711	};
4712
4713	var fn_block: GenZir = .{
4714	.is_comptime = false,
4715	.decl_node_index = node,
4716	.decl_line = decl_block.decl_line,
4717	.parent = &decl_block.base,
4718	.astgen = astgen,
4719	.instructions = decl_block.instructions,
4720	.instructions_top = decl_block.instructions.items.len,
4721	};
4722	defer fn_block.unstack();
4723
4724	const prev_fn_block = astgen.fn_block;
4725	const prev_fn_ret_ty = astgen.fn_ret_ty;
4726	astgen.fn_block = &fn_block;
4727	astgen.fn_ret_ty = .anyerror_void_error_union_type;
4728	defer {
4729	astgen.fn_block = prev_fn_block;
4730	astgen.fn_ret_ty = prev_fn_ret_ty;
4731	}
4732
4733	astgen.advanceSourceCursorToNode(body_node);
4734	const lbrace_line = astgen.source_line - decl_block.decl_line;
4735	const lbrace_column = astgen.source_column;
4736
4737	const block_result = try expr(&fn_block, &fn_block.base, .{ .rl = .none }, body_node);
4738	if (fn_block.isEmpty() or !fn_block.refIsNoReturn(block_result)) {
4739
4740	// As our last action before the return, "pop" the error trace if needed
4741	_ = try fn_block.addRestoreErrRetIndex(.ret, .always, node);
4742
4743	// Add implicit return at end of function.
4744	_ = try fn_block.addUnTok(.ret_implicit, .void_value, tree.lastToken(body_node));
4745	}
4746
4747	const func_inst = try decl_block.addFunc(.{
4748	.src_node = node,
4749
4750	.cc_ref = .none,
4751	.cc_gz = null,
4752	.align_ref = .none,
4753	.align_gz = null,
4754	.ret_ref = .anyerror_void_error_union_type,
4755	.ret_gz = null,
4756	.section_ref = .none,
4757	.section_gz = null,
4758	.addrspace_ref = .none,
4759	.addrspace_gz = null,
4760
4761	.lbrace_line = lbrace_line,
4762	.lbrace_column = lbrace_column,
4763	.param_block = decl_inst,
4764	.body_gz = &fn_block,
4765	.lib_name = .empty,
4766	.is_var_args = false,
4767	.is_inferred_error = false,
4768	.is_test = true,
4769	.is_extern = false,
4770	.is_noinline = false,
4771	.noalias_bits = 0,
4772	});
4773
4774	_ = try decl_block.addBreak(.break_inline, decl_inst, func_inst);
4775
4776	try setDeclaration(
4777	decl_inst,
4778	std.zig.hashSrc(tree.getNodeSource(node)),
4779	test_name,
4780	decl_block.decl_line - gz.decl_line,
4781	false,
4782	false,
4783	.empty,
4784	&decl_block,
4785	null,
4786	);
4787	}
4788
4789	fn structDeclInner(
4790	gz: *GenZir,
4791	scope: *Scope,
4792	node: Ast.Node.Index,
4793	container_decl: Ast.full.ContainerDecl,
4794	layout: std.builtin.Type.ContainerLayout,
4795	backing_int_node: Ast.Node.Index,
4796	) InnerError!Zir.Inst.Ref {
4797	const decl_inst = try gz.reserveInstructionIndex();
4798
4799	if (container_decl.ast.members.len == 0 and backing_int_node == 0) {
4800	try gz.setStruct(decl_inst, .{
4801	.src_node = node,
4802	.layout = layout,
4803	.fields_len = 0,
4804	.decls_len = 0,
4805	.backing_int_ref = .none,
4806	.backing_int_body_len = 0,
4807	.known_non_opv = false,
4808	.known_comptime_only = false,
4809	.is_tuple = false,
4810	.any_comptime_fields = false,
4811	.any_default_inits = false,
4812	.any_aligned_fields = false,
4813	.fields_hash = std.zig.hashSrc(@tagName(layout)),
4814	});
4815	return decl_inst.toRef();
4816	}
4817
4818	const astgen = gz.astgen;
4819	const gpa = astgen.gpa;
4820	const tree = astgen.tree;
4821
4822	var namespace: Scope.Namespace = .{
4823	.parent = scope,
4824	.node = node,
4825	.inst = decl_inst,
4826	.declaring_gz = gz,
4827	};
4828	defer namespace.deinit(gpa);
4829
4830	// The struct_decl instruction introduces a scope in which the decls of the struct
4831	// are in scope, so that field types, alignments, and default value expressions
4832	// can refer to decls within the struct itself.
4833	astgen.advanceSourceCursorToNode(node);
4834	var block_scope: GenZir = .{
4835	.parent = &namespace.base,
4836	.decl_node_index = node,
4837	.decl_line = gz.decl_line,
4838	.astgen = astgen,
4839	.is_comptime = true,
4840	.instructions = gz.instructions,
4841	.instructions_top = gz.instructions.items.len,
4842	};
4843	defer block_scope.unstack();
4844
4845	const scratch_top = astgen.scratch.items.len;
4846	defer astgen.scratch.items.len = scratch_top;
4847
4848	var backing_int_body_len: usize = 0;
4849	const backing_int_ref: Zir.Inst.Ref = blk: {
4850	if (backing_int_node != 0) {
4851	if (layout != .Packed) {
4852	return astgen.failNode(backing_int_node, "non-packed struct does not support backing integer type", .{});
4853	} else {
4854	const backing_int_ref = try typeExpr(&block_scope, &namespace.base, backing_int_node);
4855	if (!block_scope.isEmpty()) {
4856	if (!block_scope.endsWithNoReturn()) {
4857	_ = try block_scope.addBreak(.break_inline, decl_inst, backing_int_ref);
4858	}
4859
4860	const body = block_scope.instructionsSlice();
4861	const old_scratch_len = astgen.scratch.items.len;
4862	try astgen.scratch.ensureUnusedCapacity(gpa, countBodyLenAfterFixups(astgen, body));
4863	appendBodyWithFixupsArrayList(astgen, &astgen.scratch, body);
4864	backing_int_body_len = astgen.scratch.items.len - old_scratch_len;
4865	block_scope.instructions.items.len = block_scope.instructions_top;
4866	}
4867	break :blk backing_int_ref;
4868	}
4869	} else {
4870	break :blk .none;
4871	}
4872	};
4873
4874	const decl_count = try astgen.scanDecls(&namespace, container_decl.ast.members);
4875	const field_count: u32 = @intCast(container_decl.ast.members.len - decl_count);
4876
4877	const bits_per_field = 4;
4878	const max_field_size = 5;
4879	var wip_members = try WipMembers.init(gpa, &astgen.scratch, decl_count, field_count, bits_per_field, max_field_size);
4880	defer wip_members.deinit();
4881
4882	// We will use the scratch buffer, starting here, for the bodies:
4883	// bodies: { // for every fields_len
4884	// field_type_body_inst: Inst, // for each field_type_body_len
4885	// align_body_inst: Inst, // for each align_body_len
4886	// init_body_inst: Inst, // for each init_body_len
4887	// }
4888	// Note that the scratch buffer is simultaneously being used by WipMembers, however
4889	// it will not access any elements beyond this point in the ArrayList. It also
4890	// accesses via the ArrayList items field so it can handle the scratch buffer being
4891	// reallocated.
4892	// No defer needed here because it is handled by `wip_members.deinit()` above.
4893	const bodies_start = astgen.scratch.items.len;
4894
4895	const node_tags = tree.nodes.items(.tag);
4896	const is_tuple = for (container_decl.ast.members) \|member_node\| {
4897	const container_field = tree.fullContainerField(member_node) orelse continue;
4898	if (container_field.ast.tuple_like) break true;
4899	} else false;
4900
4901	if (is_tuple) switch (layout) {
4902	.Auto => {},
4903	.Extern => return astgen.failNode(node, "extern tuples are not supported", .{}),
4904	.Packed => return astgen.failNode(node, "packed tuples are not supported", .{}),
4905	};
4906
4907	if (is_tuple) for (container_decl.ast.members) \|member_node\| {
4908	switch (node_tags[member_node]) {
4909	.container_field_init,
4910	.container_field_align,
4911	.container_field,
4912	.@"comptime",
4913	.test_decl,
4914	=> continue,
4915	else => {
4916	const tuple_member = for (container_decl.ast.members) \|maybe_tuple\| switch (node_tags[maybe_tuple]) {
4917	.container_field_init,
4918	.container_field_align,
4919	.container_field,
4920	=> break maybe_tuple,
4921	else => {},
4922	} else unreachable;
4923	return astgen.failNodeNotes(
4924	member_node,
4925	"tuple declarations cannot contain declarations",
4926	.{},
4927	&[_]u32{
4928	try astgen.errNoteNode(tuple_member, "tuple field here", .{}),
4929	},
4930	);
4931	},
4932	}
4933	};
4934
4935	var fields_hasher = std.zig.SrcHasher.init(.{});
4936	fields_hasher.update(@tagName(layout));
4937	if (backing_int_node != 0) {
4938	fields_hasher.update(tree.getNodeSource(backing_int_node));
4939	}
4940
4941	var sfba = std.heap.stackFallback(256, astgen.arena);
4942	const sfba_allocator = sfba.get();
4943
4944	var duplicate_names = std.AutoArrayHashMap(Zir.NullTerminatedString, std.ArrayListUnmanaged(Ast.TokenIndex)).init(sfba_allocator);
4945	try duplicate_names.ensureTotalCapacity(field_count);
4946
4947	// When there aren't errors, use this to avoid a second iteration.
4948	var any_duplicate = false;
4949
4950	var known_non_opv = false;
4951	var known_comptime_only = false;
4952	var any_comptime_fields = false;
4953	var any_aligned_fields = false;
4954	var any_default_inits = false;
4955	for (container_decl.ast.members) \|member_node\| {
4956	var member = switch (try containerMember(&block_scope, &namespace.base, &wip_members, member_node)) {
4957	.decl => continue,
4958	.field => \|field\| field,
4959	};
4960
4961	fields_hasher.update(tree.getNodeSource(member_node));
4962
4963	if (!is_tuple) {
4964	const field_name = try astgen.identAsString(member.ast.main_token);
4965
4966	member.convertToNonTupleLike(astgen.tree.nodes);
4967	assert(!member.ast.tuple_like);
4968
4969	wip_members.appendToField(@intFromEnum(field_name));
4970
4971	const gop = try duplicate_names.getOrPut(field_name);
4972
4973	if (gop.found_existing) {
4974	try gop.value_ptr.append(sfba_allocator, member.ast.main_token);
4975	any_duplicate = true;
4976	} else {
4977	gop.value_ptr.* = .{};
4978	try gop.value_ptr.append(sfba_allocator, member.ast.main_token);
4979	}
4980	} else if (!member.ast.tuple_like) {
4981	return astgen.failTok(member.ast.main_token, "tuple field has a name", .{});
4982	}
4983
4984	const doc_comment_index = try astgen.docCommentAsString(member.firstToken());
4985	wip_members.appendToField(@intFromEnum(doc_comment_index));
4986
4987	if (member.ast.type_expr == 0) {
4988	return astgen.failTok(member.ast.main_token, "struct field missing type", .{});
4989	}
4990
4991	const field_type = try typeExpr(&block_scope, &namespace.base, member.ast.type_expr);
4992	const have_type_body = !block_scope.isEmpty();
4993	const have_align = member.ast.align_expr != 0;
4994	const have_value = member.ast.value_expr != 0;
4995	const is_comptime = member.comptime_token != null;
4996
4997	if (is_comptime) {
4998	switch (layout) {
4999	.Packed => return astgen.failTok(member.comptime_token.?, "packed struct fields cannot be marked comptime", .{}),
5000	.Extern => return astgen.failTok(member.comptime_token.?, "extern struct fields cannot be marked comptime", .{}),
5001	.Auto => any_comptime_fields = true,
5002	}
5003	} else {
5004	known_non_opv = known_non_opv or
5005	nodeImpliesMoreThanOnePossibleValue(tree, member.ast.type_expr);
5006	known_comptime_only = known_comptime_only or
5007	nodeImpliesComptimeOnly(tree, member.ast.type_expr);
5008	}
5009	wip_members.nextField(bits_per_field, .{ have_align, have_value, is_comptime, have_type_body });
5010
5011	if (have_type_body) {
5012	if (!block_scope.endsWithNoReturn()) {
5013	_ = try block_scope.addBreak(.break_inline, decl_inst, field_type);
5014	}
5015	const body = block_scope.instructionsSlice();
5016	const old_scratch_len = astgen.scratch.items.len;
5017	try astgen.scratch.ensureUnusedCapacity(gpa, countBodyLenAfterFixups(astgen, body));
5018	appendBodyWithFixupsArrayList(astgen, &astgen.scratch, body);
5019	wip_members.appendToField(@intCast(astgen.scratch.items.len - old_scratch_len));
5020	block_scope.instructions.items.len = block_scope.instructions_top;
5021	} else {
5022	wip_members.appendToField(@intFromEnum(field_type));
5023	}
5024
5025	if (have_align) {
5026	if (layout == .Packed) {
5027	try astgen.appendErrorNode(member.ast.align_expr, "unable to override alignment of packed struct fields", .{});
5028	}
5029	any_aligned_fields = true;
5030	const align_ref = try expr(&block_scope, &namespace.base, coerced_align_ri, member.ast.align_expr);
5031	if (!block_scope.endsWithNoReturn()) {
5032	_ = try block_scope.addBreak(.break_inline, decl_inst, align_ref);
5033	}
5034	const body = block_scope.instructionsSlice();
5035	const old_scratch_len = astgen.scratch.items.len;
5036	try astgen.scratch.ensureUnusedCapacity(gpa, countBodyLenAfterFixups(astgen, body));
5037	appendBodyWithFixupsArrayList(astgen, &astgen.scratch, body);
5038	wip_members.appendToField(@intCast(astgen.scratch.items.len - old_scratch_len));
5039	block_scope.instructions.items.len = block_scope.instructions_top;
5040	}
5041
5042	if (have_value) {
5043	any_default_inits = true;
5044
5045	// The decl_inst is used as here so that we can easily reconstruct a mapping
5046	// between it and the field type when the fields inits are analzyed.
5047	const ri: ResultInfo = .{ .rl = if (field_type == .none) .none else .{ .coerced_ty = decl_inst.toRef() } };
5048
5049	const default_inst = try expr(&block_scope, &namespace.base, ri, member.ast.value_expr);
5050	if (!block_scope.endsWithNoReturn()) {
5051	_ = try block_scope.addBreak(.break_inline, decl_inst, default_inst);
5052	}
5053	const body = block_scope.instructionsSlice();
5054	const old_scratch_len = astgen.scratch.items.len;
5055	try astgen.scratch.ensureUnusedCapacity(gpa, countBodyLenAfterFixups(astgen, body));
5056	appendBodyWithFixupsArrayList(astgen, &astgen.scratch, body);
5057	wip_members.appendToField(@intCast(astgen.scratch.items.len - old_scratch_len));
5058	block_scope.instructions.items.len = block_scope.instructions_top;
5059	} else if (member.comptime_token) \|comptime_token\| {
5060	return astgen.failTok(comptime_token, "comptime field without default initialization value", .{});
5061	}
5062	}
5063
5064	if (any_duplicate) {
5065	var it = duplicate_names.iterator();
5066
5067	while (it.next()) \|entry\| {
5068	const record = entry.value_ptr.*;
5069	if (record.items.len > 1) {
5070	var error_notes = std.ArrayList(u32).init(astgen.arena);
5071
5072	for (record.items[1..]) \|duplicate\| {
5073	try error_notes.append(try astgen.errNoteTok(duplicate, "duplicate field here", .{}));
5074	}
5075
5076	try error_notes.append(try astgen.errNoteNode(node, "struct declared here", .{}));
5077
5078	try astgen.appendErrorTokNotes(
5079	record.items[0],
5080	"duplicate struct field name",
5081	.{},
5082	error_notes.items,
5083	);
5084	}
5085	}
5086
5087	return error.AnalysisFail;
5088	}
5089
5090	var fields_hash: std.zig.SrcHash = undefined;
5091	fields_hasher.final(&fields_hash);
5092
5093	try gz.setStruct(decl_inst, .{
5094	.src_node = node,
5095	.layout = layout,
5096	.fields_len = field_count,
5097	.decls_len = decl_count,
5098	.backing_int_ref = backing_int_ref,
5099	.backing_int_body_len = @intCast(backing_int_body_len),
5100	.known_non_opv = known_non_opv,
5101	.known_comptime_only = known_comptime_only,
5102	.is_tuple = is_tuple,
5103	.any_comptime_fields = any_comptime_fields,
5104	.any_default_inits = any_default_inits,
5105	.any_aligned_fields = any_aligned_fields,
5106	.fields_hash = fields_hash,
5107	});
5108
5109	wip_members.finishBits(bits_per_field);
5110	const decls_slice = wip_members.declsSlice();
5111	const fields_slice = wip_members.fieldsSlice();
5112	const bodies_slice = astgen.scratch.items[bodies_start..];
5113	try astgen.extra.ensureUnusedCapacity(gpa, backing_int_body_len +
5114	decls_slice.len + fields_slice.len + bodies_slice.len);
5115	astgen.extra.appendSliceAssumeCapacity(astgen.scratch.items[scratch_top..][0..backing_int_body_len]);
5116	astgen.extra.appendSliceAssumeCapacity(decls_slice);
5117	astgen.extra.appendSliceAssumeCapacity(fields_slice);
5118	astgen.extra.appendSliceAssumeCapacity(bodies_slice);
5119
5120	block_scope.unstack();
5121	try gz.addNamespaceCaptures(&namespace);
5122	return decl_inst.toRef();
5123	}
5124
5125	fn unionDeclInner(
5126	gz: *GenZir,
5127	scope: *Scope,
5128	node: Ast.Node.Index,
5129	members: []const Ast.Node.Index,
5130	layout: std.builtin.Type.ContainerLayout,
5131	arg_node: Ast.Node.Index,
5132	auto_enum_tok: ?Ast.TokenIndex,
5133	) InnerError!Zir.Inst.Ref {
5134	const decl_inst = try gz.reserveInstructionIndex();
5135
5136	const astgen = gz.astgen;
5137	const gpa = astgen.gpa;
5138
5139	var namespace: Scope.Namespace = .{
5140	.parent = scope,
5141	.node = node,
5142	.inst = decl_inst,
5143	.declaring_gz = gz,
5144	};
5145	defer namespace.deinit(gpa);
5146
5147	// The union_decl instruction introduces a scope in which the decls of the union
5148	// are in scope, so that field types, alignments, and default value expressions
5149	// can refer to decls within the union itself.
5150	astgen.advanceSourceCursorToNode(node);
5151	var block_scope: GenZir = .{
5152	.parent = &namespace.base,
5153	.decl_node_index = node,
5154	.decl_line = gz.decl_line,
5155	.astgen = astgen,
5156	.is_comptime = true,
5157	.instructions = gz.instructions,
5158	.instructions_top = gz.instructions.items.len,
5159	};
5160	defer block_scope.unstack();
5161
5162	const decl_count = try astgen.scanDecls(&namespace, members);
5163	const field_count: u32 = @intCast(members.len - decl_count);
5164
5165	if (layout != .Auto and (auto_enum_tok != null or arg_node != 0)) {
5166	const layout_str = if (layout == .Extern) "extern" else "packed";
5167	if (arg_node != 0) {
5168	return astgen.failNode(arg_node, "{s} union does not support enum tag type", .{layout_str});
5169	} else {
5170	return astgen.failTok(auto_enum_tok.?, "{s} union does not support enum tag type", .{layout_str});
5171	}
5172	}
5173
5174	const arg_inst: Zir.Inst.Ref = if (arg_node != 0)
5175	try typeExpr(&block_scope, &namespace.base, arg_node)
5176	else
5177	.none;
5178
5179	const bits_per_field = 4;
5180	const max_field_size = 5;
5181	var any_aligned_fields = false;
5182	var wip_members = try WipMembers.init(gpa, &astgen.scratch, decl_count, field_count, bits_per_field, max_field_size);
5183	defer wip_members.deinit();
5184
5185	var fields_hasher = std.zig.SrcHasher.init(.{});
5186	fields_hasher.update(@tagName(layout));
5187	fields_hasher.update(&.{@intFromBool(auto_enum_tok != null)});
5188	if (arg_node != 0) {
5189	fields_hasher.update(astgen.tree.getNodeSource(arg_node));
5190	}
5191
5192	var sfba = std.heap.stackFallback(256, astgen.arena);
5193	const sfba_allocator = sfba.get();
5194
5195	var duplicate_names = std.AutoArrayHashMap(Zir.NullTerminatedString, std.ArrayListUnmanaged(Ast.TokenIndex)).init(sfba_allocator);
5196	try duplicate_names.ensureTotalCapacity(field_count);
5197
5198	// When there aren't errors, use this to avoid a second iteration.
5199	var any_duplicate = false;
5200
5201	for (members) \|member_node\| {
5202	var member = switch (try containerMember(&block_scope, &namespace.base, &wip_members, member_node)) {
5203	.decl => continue,
5204	.field => \|field\| field,
5205	};
5206	fields_hasher.update(astgen.tree.getNodeSource(member_node));
5207	member.convertToNonTupleLike(astgen.tree.nodes);
5208	if (member.ast.tuple_like) {
5209	return astgen.failTok(member.ast.main_token, "union field missing name", .{});
5210	}
5211	if (member.comptime_token) \|comptime_token\| {
5212	return astgen.failTok(comptime_token, "union fields cannot be marked comptime", .{});
5213	}
5214
5215	const field_name = try astgen.identAsString(member.ast.main_token);
5216	wip_members.appendToField(@intFromEnum(field_name));
5217
5218	const gop = try duplicate_names.getOrPut(field_name);
5219
5220	if (gop.found_existing) {
5221	try gop.value_ptr.append(sfba_allocator, member.ast.main_token);
5222	any_duplicate = true;
5223	} else {
5224	gop.value_ptr.* = .{};
5225	try gop.value_ptr.append(sfba_allocator, member.ast.main_token);
5226	}
5227
5228	const doc_comment_index = try astgen.docCommentAsString(member.firstToken());
5229	wip_members.appendToField(@intFromEnum(doc_comment_index));
5230
5231	const have_type = member.ast.type_expr != 0;
5232	const have_align = member.ast.align_expr != 0;
5233	const have_value = member.ast.value_expr != 0;
5234	const unused = false;
5235	wip_members.nextField(bits_per_field, .{ have_type, have_align, have_value, unused });
5236
5237	if (have_type) {
5238	const field_type = try typeExpr(&block_scope, &namespace.base, member.ast.type_expr);
5239	wip_members.appendToField(@intFromEnum(field_type));
5240	} else if (arg_inst == .none and auto_enum_tok == null) {
5241	return astgen.failNode(member_node, "union field missing type", .{});
5242	}
5243	if (have_align) {
5244	const align_inst = try expr(&block_scope, &block_scope.base, coerced_align_ri, member.ast.align_expr);
5245	wip_members.appendToField(@intFromEnum(align_inst));
5246	any_aligned_fields = true;
5247	}
5248	if (have_value) {
5249	if (arg_inst == .none) {
5250	return astgen.failNodeNotes(
5251	node,
5252	"explicitly valued tagged union missing integer tag type",
5253	.{},
5254	&[_]u32{
5255	try astgen.errNoteNode(
5256	member.ast.value_expr,
5257	"tag value specified here",
5258	.{},
5259	),
5260	},
5261	);
5262	}
5263	if (auto_enum_tok == null) {
5264	return astgen.failNodeNotes(
5265	node,
5266	"explicitly valued tagged union requires inferred enum tag type",
5267	.{},
5268	&[_]u32{
5269	try astgen.errNoteNode(
5270	member.ast.value_expr,
5271	"tag value specified here",
5272	.{},
5273	),
5274	},
5275	);
5276	}
5277	const tag_value = try expr(&block_scope, &block_scope.base, .{ .rl = .{ .ty = arg_inst } }, member.ast.value_expr);
5278	wip_members.appendToField(@intFromEnum(tag_value));
5279	}
5280	}
5281
5282	if (any_duplicate) {
5283	var it = duplicate_names.iterator();
5284
5285	while (it.next()) \|entry\| {
5286	const record = entry.value_ptr.*;
5287	if (record.items.len > 1) {
5288	var error_notes = std.ArrayList(u32).init(astgen.arena);
5289
5290	for (record.items[1..]) \|duplicate\| {
5291	try error_notes.append(try astgen.errNoteTok(duplicate, "duplicate field here", .{}));
5292	}
5293
5294	try error_notes.append(try astgen.errNoteNode(node, "union declared here", .{}));
5295
5296	try astgen.appendErrorTokNotes(
5297	record.items[0],
5298	"duplicate union field name",
5299	.{},
5300	error_notes.items,
5301	);
5302	}
5303	}
5304
5305	return error.AnalysisFail;
5306	}
5307
5308	var fields_hash: std.zig.SrcHash = undefined;
5309	fields_hasher.final(&fields_hash);
5310
5311	if (!block_scope.isEmpty()) {
5312	_ = try block_scope.addBreak(.break_inline, decl_inst, .void_value);
5313	}
5314
5315	const body = block_scope.instructionsSlice();
5316	const body_len = astgen.countBodyLenAfterFixups(body);
5317
5318	try gz.setUnion(decl_inst, .{
5319	.src_node = node,
5320	.layout = layout,
5321	.tag_type = arg_inst,
5322	.body_len = body_len,
5323	.fields_len = field_count,
5324	.decls_len = decl_count,
5325	.auto_enum_tag = auto_enum_tok != null,
5326	.any_aligned_fields = any_aligned_fields,
5327	.fields_hash = fields_hash,
5328	});
5329
5330	wip_members.finishBits(bits_per_field);
5331	const decls_slice = wip_members.declsSlice();
5332	const fields_slice = wip_members.fieldsSlice();
5333	try astgen.extra.ensureUnusedCapacity(gpa, decls_slice.len + body_len + fields_slice.len);
5334	astgen.extra.appendSliceAssumeCapacity(decls_slice);
5335	astgen.appendBodyWithFixups(body);
5336	astgen.extra.appendSliceAssumeCapacity(fields_slice);
5337
5338	block_scope.unstack();
5339	try gz.addNamespaceCaptures(&namespace);
5340	return decl_inst.toRef();
5341	}
5342
5343	fn containerDecl(
5344	gz: *GenZir,
5345	scope: *Scope,
5346	ri: ResultInfo,
5347	node: Ast.Node.Index,
5348	container_decl: Ast.full.ContainerDecl,
5349	) InnerError!Zir.Inst.Ref {
5350	const astgen = gz.astgen;
5351	const gpa = astgen.gpa;
5352	const tree = astgen.tree;
5353	const token_tags = tree.tokens.items(.tag);
5354
5355	const prev_fn_block = astgen.fn_block;
5356	astgen.fn_block = null;
5357	defer astgen.fn_block = prev_fn_block;
5358
5359	// We must not create any types until Sema. Here the goal is only to generate
5360	// ZIR for all the field types, alignments, and default value expressions.
5361
5362	switch (token_tags[container_decl.ast.main_token]) {
5363	.keyword_struct => {
5364	const layout = if (container_decl.layout_token) \|t\| switch (token_tags[t]) {
5365	.keyword_packed => std.builtin.Type.ContainerLayout.Packed,
5366	.keyword_extern => std.builtin.Type.ContainerLayout.Extern,
5367	else => unreachable,
5368	} else std.builtin.Type.ContainerLayout.Auto;
5369
5370	const result = try structDeclInner(gz, scope, node, container_decl, layout, container_decl.ast.arg);
5371	return rvalue(gz, ri, result, node);
5372	},
5373	.keyword_union => {
5374	const layout = if (container_decl.layout_token) \|t\| switch (token_tags[t]) {
5375	.keyword_packed => std.builtin.Type.ContainerLayout.Packed,
5376	.keyword_extern => std.builtin.Type.ContainerLayout.Extern,
5377	else => unreachable,
5378	} else std.builtin.Type.ContainerLayout.Auto;
5379
5380	const result = try unionDeclInner(gz, scope, node, container_decl.ast.members, layout, container_decl.ast.arg, container_decl.ast.enum_token);
5381	return rvalue(gz, ri, result, node);
5382	},
5383	.keyword_enum => {
5384	if (container_decl.layout_token) \|t\| {
5385	return astgen.failTok(t, "enums do not support 'packed' or 'extern'; instead provide an explicit integer tag type", .{});
5386	}
5387	// Count total fields as well as how many have explicitly provided tag values.
5388	const counts = blk: {
5389	var values: usize = 0;
5390	var total_fields: usize = 0;
5391	var decls: usize = 0;
5392	var nonexhaustive_node: Ast.Node.Index = 0;
5393	var nonfinal_nonexhaustive = false;
5394	for (container_decl.ast.members) \|member_node\| {
5395	var member = tree.fullContainerField(member_node) orelse {
5396	decls += 1;
5397	continue;
5398	};
5399	member.convertToNonTupleLike(astgen.tree.nodes);
5400	if (member.ast.tuple_like) {
5401	return astgen.failTok(member.ast.main_token, "enum field missing name", .{});
5402	}
5403	if (member.comptime_token) \|comptime_token\| {
5404	return astgen.failTok(comptime_token, "enum fields cannot be marked comptime", .{});
5405	}
5406	if (member.ast.type_expr != 0) {
5407	return astgen.failNodeNotes(
5408	member.ast.type_expr,
5409	"enum fields do not have types",
5410	.{},
5411	&[_]u32{
5412	try astgen.errNoteNode(
5413	node,
5414	"consider 'union(enum)' here to make it a tagged union",
5415	.{},
5416	),
5417	},
5418	);
5419	}
5420	if (member.ast.align_expr != 0) {
5421	return astgen.failNode(member.ast.align_expr, "enum fields cannot be aligned", .{});
5422	}
5423
5424	const name_token = member.ast.main_token;
5425	if (mem.eql(u8, tree.tokenSlice(name_token), "_")) {
5426	if (nonexhaustive_node != 0) {
5427	return astgen.failNodeNotes(
5428	member_node,
5429	"redundant non-exhaustive enum mark",
5430	.{},
5431	&[_]u32{
5432	try astgen.errNoteNode(
5433	nonexhaustive_node,
5434	"other mark here",
5435	.{},
5436	),
5437	},
5438	);
5439	}
5440	nonexhaustive_node = member_node;
5441	if (member.ast.value_expr != 0) {
5442	return astgen.failNode(member.ast.value_expr, "'_' is used to mark an enum as non-exhaustive and cannot be assigned a value", .{});
5443	}
5444	continue;
5445	} else if (nonexhaustive_node != 0) {
5446	nonfinal_nonexhaustive = true;
5447	}
5448	total_fields += 1;
5449	if (member.ast.value_expr != 0) {
5450	if (container_decl.ast.arg == 0) {
5451	return astgen.failNode(member.ast.value_expr, "value assigned to enum tag with inferred tag type", .{});
5452	}
5453	values += 1;
5454	}
5455	}
5456	if (nonfinal_nonexhaustive) {
5457	return astgen.failNode(nonexhaustive_node, "'_' field of non-exhaustive enum must be last", .{});
5458	}
5459	break :blk .{
5460	.total_fields = total_fields,
5461	.values = values,
5462	.decls = decls,
5463	.nonexhaustive_node = nonexhaustive_node,
5464	};
5465	};
5466	if (counts.nonexhaustive_node != 0 and container_decl.ast.arg == 0) {
5467	try astgen.appendErrorNodeNotes(
5468	node,
5469	"non-exhaustive enum missing integer tag type",
5470	.{},
5471	&[_]u32{
5472	try astgen.errNoteNode(
5473	counts.nonexhaustive_node,
5474	"marked non-exhaustive here",
5475	.{},
5476	),
5477	},
5478	);
5479	}
5480	// In this case we must generate ZIR code for the tag values, similar to
5481	// how structs are handled above.
5482	const nonexhaustive = counts.nonexhaustive_node != 0;
5483
5484	const decl_inst = try gz.reserveInstructionIndex();
5485
5486	var namespace: Scope.Namespace = .{
5487	.parent = scope,
5488	.node = node,
5489	.inst = decl_inst,
5490	.declaring_gz = gz,
5491	};
5492	defer namespace.deinit(gpa);
5493
5494	// The enum_decl instruction introduces a scope in which the decls of the enum
5495	// are in scope, so that tag values can refer to decls within the enum itself.
5496	astgen.advanceSourceCursorToNode(node);
5497	var block_scope: GenZir = .{
5498	.parent = &namespace.base,
5499	.decl_node_index = node,
5500	.decl_line = gz.decl_line,
5501	.astgen = astgen,
5502	.is_comptime = true,
5503	.instructions = gz.instructions,
5504	.instructions_top = gz.instructions.items.len,
5505	};
5506	defer block_scope.unstack();
5507
5508	_ = try astgen.scanDecls(&namespace, container_decl.ast.members);
5509	namespace.base.tag = .enum_namespace;
5510
5511	const arg_inst: Zir.Inst.Ref = if (container_decl.ast.arg != 0)
5512	try comptimeExpr(&block_scope, &namespace.base, coerced_type_ri, container_decl.ast.arg)
5513	else
5514	.none;
5515
5516	const bits_per_field = 1;
5517	const max_field_size = 3;
5518	var wip_members = try WipMembers.init(gpa, &astgen.scratch, @intCast(counts.decls), @intCast(counts.total_fields), bits_per_field, max_field_size);
5519	defer wip_members.deinit();
5520
5521	var fields_hasher = std.zig.SrcHasher.init(.{});
5522	if (container_decl.ast.arg != 0) {
5523	fields_hasher.update(tree.getNodeSource(container_decl.ast.arg));
5524	}
5525	fields_hasher.update(&.{@intFromBool(nonexhaustive)});
5526
5527	var sfba = std.heap.stackFallback(256, astgen.arena);
5528	const sfba_allocator = sfba.get();
5529
5530	var duplicate_names = std.AutoArrayHashMap(Zir.NullTerminatedString, std.ArrayListUnmanaged(Ast.TokenIndex)).init(sfba_allocator);
5531	try duplicate_names.ensureTotalCapacity(counts.total_fields);
5532
5533	// When there aren't errors, use this to avoid a second iteration.
5534	var any_duplicate = false;
5535
5536	for (container_decl.ast.members) \|member_node\| {
5537	if (member_node == counts.nonexhaustive_node)
5538	continue;
5539	fields_hasher.update(tree.getNodeSource(member_node));
5540	namespace.base.tag = .namespace;
5541	var member = switch (try containerMember(&block_scope, &namespace.base, &wip_members, member_node)) {
5542	.decl => continue,
5543	.field => \|field\| field,
5544	};
5545	member.convertToNonTupleLike(astgen.tree.nodes);
5546	assert(member.comptime_token == null);
5547	assert(member.ast.type_expr == 0);
5548	assert(member.ast.align_expr == 0);
5549
5550	const field_name = try astgen.identAsString(member.ast.main_token);
5551	wip_members.appendToField(@intFromEnum(field_name));
5552
5553	const gop = try duplicate_names.getOrPut(field_name);
5554
5555	if (gop.found_existing) {
5556	try gop.value_ptr.append(sfba_allocator, member.ast.main_token);
5557	any_duplicate = true;
5558	} else {
5559	gop.value_ptr.* = .{};
5560	try gop.value_ptr.append(sfba_allocator, member.ast.main_token);
5561	}
5562
5563	const doc_comment_index = try astgen.docCommentAsString(member.firstToken());
5564	wip_members.appendToField(@intFromEnum(doc_comment_index));
5565
5566	const have_value = member.ast.value_expr != 0;
5567	wip_members.nextField(bits_per_field, .{have_value});
5568
5569	if (have_value) {
5570	if (arg_inst == .none) {
5571	return astgen.failNodeNotes(
5572	node,
5573	"explicitly valued enum missing integer tag type",
5574	.{},
5575	&[_]u32{
5576	try astgen.errNoteNode(
5577	member.ast.value_expr,
5578	"tag value specified here",
5579	.{},
5580	),
5581	},
5582	);
5583	}
5584	namespace.base.tag = .enum_namespace;
5585	const tag_value_inst = try expr(&block_scope, &namespace.base, .{ .rl = .{ .ty = arg_inst } }, member.ast.value_expr);
5586	wip_members.appendToField(@intFromEnum(tag_value_inst));
5587	}
5588	}
5589
5590	if (any_duplicate) {
5591	var it = duplicate_names.iterator();
5592
5593	while (it.next()) \|entry\| {
5594	const record = entry.value_ptr.*;
5595	if (record.items.len > 1) {
5596	var error_notes = std.ArrayList(u32).init(astgen.arena);
5597
5598	for (record.items[1..]) \|duplicate\| {
5599	try error_notes.append(try astgen.errNoteTok(duplicate, "duplicate field here", .{}));
5600	}
5601
5602	try error_notes.append(try astgen.errNoteNode(node, "enum declared here", .{}));
5603
5604	try astgen.appendErrorTokNotes(
5605	record.items[0],
5606	"duplicate enum field name",
5607	.{},
5608	error_notes.items,
5609	);
5610	}
5611	}
5612
5613	return error.AnalysisFail;
5614	}
5615
5616	if (!block_scope.isEmpty()) {
5617	_ = try block_scope.addBreak(.break_inline, decl_inst, .void_value);
5618	}
5619
5620	var fields_hash: std.zig.SrcHash = undefined;
5621	fields_hasher.final(&fields_hash);
5622
5623	const body = block_scope.instructionsSlice();
5624	const body_len = astgen.countBodyLenAfterFixups(body);
5625
5626	try gz.setEnum(decl_inst, .{
5627	.src_node = node,
5628	.nonexhaustive = nonexhaustive,
5629	.tag_type = arg_inst,
5630	.body_len = body_len,
5631	.fields_len = @intCast(counts.total_fields),
5632	.decls_len = @intCast(counts.decls),
5633	.fields_hash = fields_hash,
5634	});
5635
5636	wip_members.finishBits(bits_per_field);
5637	const decls_slice = wip_members.declsSlice();
5638	const fields_slice = wip_members.fieldsSlice();
5639	try astgen.extra.ensureUnusedCapacity(gpa, decls_slice.len + body_len + fields_slice.len);
5640	astgen.extra.appendSliceAssumeCapacity(decls_slice);
5641	astgen.appendBodyWithFixups(body);
5642	astgen.extra.appendSliceAssumeCapacity(fields_slice);
5643
5644	block_scope.unstack();
5645	try gz.addNamespaceCaptures(&namespace);
5646	return rvalue(gz, ri, decl_inst.toRef(), node);
5647	},
5648	.keyword_opaque => {
5649	assert(container_decl.ast.arg == 0);
5650
5651	const decl_inst = try gz.reserveInstructionIndex();
5652
5653	var namespace: Scope.Namespace = .{
5654	.parent = scope,
5655	.node = node,
5656	.inst = decl_inst,
5657	.declaring_gz = gz,
5658	};
5659	defer namespace.deinit(gpa);
5660
5661	astgen.advanceSourceCursorToNode(node);
5662	var block_scope: GenZir = .{
5663	.parent = &namespace.base,
5664	.decl_node_index = node,
5665	.decl_line = gz.decl_line,
5666	.astgen = astgen,
5667	.is_comptime = true,
5668	.instructions = gz.instructions,
5669	.instructions_top = gz.instructions.items.len,
5670	};
5671	defer block_scope.unstack();
5672
5673	const decl_count = try astgen.scanDecls(&namespace, container_decl.ast.members);
5674
5675	var wip_members = try WipMembers.init(gpa, &astgen.scratch, decl_count, 0, 0, 0);
5676	defer wip_members.deinit();
5677
5678	for (container_decl.ast.members) \|member_node\| {
5679	const res = try containerMember(&block_scope, &namespace.base, &wip_members, member_node);
5680	if (res == .field) {
5681	return astgen.failNode(member_node, "opaque types cannot have fields", .{});
5682	}
5683	}
5684
5685	try gz.setOpaque(decl_inst, .{
5686	.src_node = node,
5687	.decls_len = decl_count,
5688	});
5689
5690	wip_members.finishBits(0);
5691	const decls_slice = wip_members.declsSlice();
5692	try astgen.extra.ensureUnusedCapacity(gpa, decls_slice.len);
5693	astgen.extra.appendSliceAssumeCapacity(decls_slice);
5694
5695	block_scope.unstack();
5696	try gz.addNamespaceCaptures(&namespace);
5697	return rvalue(gz, ri, decl_inst.toRef(), node);
5698	},
5699	else => unreachable,
5700	}
5701	}
5702
5703	const ContainerMemberResult = union(enum) { decl, field: Ast.full.ContainerField };
5704
5705	fn containerMember(
5706	gz: *GenZir,
5707	scope: *Scope,
5708	wip_members: *WipMembers,
5709	member_node: Ast.Node.Index,
5710	) InnerError!ContainerMemberResult {
5711	const astgen = gz.astgen;
5712	const tree = astgen.tree;
5713	const node_tags = tree.nodes.items(.tag);
5714	const node_datas = tree.nodes.items(.data);
5715	switch (node_tags[member_node]) {
5716	.container_field_init,
5717	.container_field_align,
5718	.container_field,
5719	=> return ContainerMemberResult{ .field = tree.fullContainerField(member_node).? },
5720
5721	.fn_proto,
5722	.fn_proto_multi,
5723	.fn_proto_one,
5724	.fn_proto_simple,
5725	.fn_decl,
5726	=> {
5727	var buf: [1]Ast.Node.Index = undefined;
5728	const full = tree.fullFnProto(&buf, member_node).?;
5729	const body = if (node_tags[member_node] == .fn_decl) node_datas[member_node].rhs else 0;
5730
5731	astgen.fnDecl(gz, scope, wip_members, member_node, body, full) catch \|err\| switch (err) {
5732	error.OutOfMemory => return error.OutOfMemory,
5733	error.AnalysisFail => {},
5734	};
5735	},
5736
5737	.global_var_decl,
5738	.local_var_decl,
5739	.simple_var_decl,
5740	.aligned_var_decl,
5741	=> {
5742	astgen.globalVarDecl(gz, scope, wip_members, member_node, tree.fullVarDecl(member_node).?) catch \|err\| switch (err) {
5743	error.OutOfMemory => return error.OutOfMemory,
5744	error.AnalysisFail => {},
5745	};
5746	},
5747
5748	.@"comptime" => {
5749	astgen.comptimeDecl(gz, scope, wip_members, member_node) catch \|err\| switch (err) {
5750	error.OutOfMemory => return error.OutOfMemory,
5751	error.AnalysisFail => {},
5752	};
5753	},
5754	.@"usingnamespace" => {
5755	astgen.usingnamespaceDecl(gz, scope, wip_members, member_node) catch \|err\| switch (err) {
5756	error.OutOfMemory => return error.OutOfMemory,
5757	error.AnalysisFail => {},
5758	};
5759	},
5760	.test_decl => {
5761	astgen.testDecl(gz, scope, wip_members, member_node) catch \|err\| switch (err) {
5762	error.OutOfMemory => return error.OutOfMemory,
5763	error.AnalysisFail => {},
5764	};
5765	},
5766	else => unreachable,
5767	}
5768	return .decl;
5769	}
5770
5771	fn errorSetDecl(gz: *GenZir, ri: ResultInfo, node: Ast.Node.Index) InnerError!Zir.Inst.Ref {
5772	const astgen = gz.astgen;
5773	const gpa = astgen.gpa;
5774	const tree = astgen.tree;
5775	const main_tokens = tree.nodes.items(.main_token);
5776	const token_tags = tree.tokens.items(.tag);
5777
5778	const payload_index = try reserveExtra(astgen, @typeInfo(Zir.Inst.ErrorSetDecl).Struct.fields.len);
5779	var fields_len: usize = 0;
5780	{
5781	var idents: std.AutoHashMapUnmanaged(Zir.NullTerminatedString, Ast.TokenIndex) = .{};
5782	defer idents.deinit(gpa);
5783
5784	const error_token = main_tokens[node];
5785	var tok_i = error_token + 2;
5786	while (true) : (tok_i += 1) {
5787	switch (token_tags[tok_i]) {
5788	.doc_comment, .comma => {},
5789	.identifier => {
5790	const str_index = try astgen.identAsString(tok_i);
5791	const gop = try idents.getOrPut(gpa, str_index);
5792	if (gop.found_existing) {
5793	const name = try gpa.dupe(u8, mem.span(astgen.nullTerminatedString(str_index)));
5794	defer gpa.free(name);
5795	return astgen.failTokNotes(
5796	tok_i,
5797	"duplicate error set field '{s}'",
5798	.{name},
5799	&[_]u32{
5800	try astgen.errNoteTok(
5801	gop.value_ptr.*,
5802	"previous declaration here",
5803	.{},
5804	),
5805	},
5806	);
5807	}
5808	gop.value_ptr.* = tok_i;
5809
5810	try astgen.extra.ensureUnusedCapacity(gpa, 2);
5811	astgen.extra.appendAssumeCapacity(@intFromEnum(str_index));
5812	const doc_comment_index = try astgen.docCommentAsString(tok_i);
5813	astgen.extra.appendAssumeCapacity(@intFromEnum(doc_comment_index));
5814	fields_len += 1;
5815	},
5816	.r_brace => break,
5817	else => unreachable,
5818	}
5819	}
5820	}
5821
5822	setExtra(astgen, payload_index, Zir.Inst.ErrorSetDecl{
5823	.fields_len = @intCast(fields_len),
5824	});
5825	const result = try gz.addPlNodePayloadIndex(.error_set_decl, node, payload_index);
5826	return rvalue(gz, ri, result, node);
5827	}
5828
5829	fn tryExpr(
5830	parent_gz: *GenZir,
5831	scope: *Scope,
5832	ri: ResultInfo,
5833	node: Ast.Node.Index,
5834	operand_node: Ast.Node.Index,
5835	) InnerError!Zir.Inst.Ref {
5836	const astgen = parent_gz.astgen;
5837
5838	const fn_block = astgen.fn_block orelse {
5839	return astgen.failNode(node, "'try' outside function scope", .{});
5840	};
5841
5842	if (parent_gz.any_defer_node != 0) {
5843	return astgen.failNodeNotes(node, "'try' not allowed inside defer expression", .{}, &.{
5844	try astgen.errNoteNode(
5845	parent_gz.any_defer_node,
5846	"defer expression here",
5847	.{},
5848	),
5849	});
5850	}
5851
5852	// Ensure debug line/column information is emitted for this try expression.
5853	// Then we will save the line/column so that we can emit another one that goes
5854	// "backwards" because we want to evaluate the operand, but then put the debug
5855	// info back at the try keyword for error return tracing.
5856	if (!parent_gz.is_comptime) {
5857	try emitDbgNode(parent_gz, node);
5858	}
5859	const try_lc = LineColumn{ astgen.source_line - parent_gz.decl_line, astgen.source_column };
5860
5861	const operand_ri: ResultInfo = switch (ri.rl) {
5862	.ref, .ref_coerced_ty => .{ .rl = .ref, .ctx = .error_handling_expr },
5863	else => .{ .rl = .none, .ctx = .error_handling_expr },
5864	};
5865	// This could be a pointer or value depending on the `ri` parameter.
5866	const operand = try reachableExpr(parent_gz, scope, operand_ri, operand_node, node);
5867	const block_tag: Zir.Inst.Tag = if (operand_ri.rl == .ref) .try_ptr else .@"try";
5868	const try_inst = try parent_gz.makeBlockInst(block_tag, node);
5869	try parent_gz.instructions.append(astgen.gpa, try_inst);
5870
5871	var else_scope = parent_gz.makeSubBlock(scope);
5872	defer else_scope.unstack();
5873
5874	const err_tag = switch (ri.rl) {
5875	.ref, .ref_coerced_ty => Zir.Inst.Tag.err_union_code_ptr,
5876	else => Zir.Inst.Tag.err_union_code,
5877	};
5878	const err_code = try else_scope.addUnNode(err_tag, operand, node);
5879	try genDefers(&else_scope, &fn_block.base, scope, .{ .both = err_code });
5880	try emitDbgStmt(&else_scope, try_lc);
5881	_ = try else_scope.addUnNode(.ret_node, err_code, node);
5882
5883	try else_scope.setTryBody(try_inst, operand);
5884	const result = try_inst.toRef();
5885	switch (ri.rl) {
5886	.ref, .ref_coerced_ty => return result,
5887	else => return rvalue(parent_gz, ri, result, node),
5888	}
5889	}
5890
5891	fn orelseCatchExpr(
5892	parent_gz: *GenZir,
5893	scope: *Scope,
5894	ri: ResultInfo,
5895	node: Ast.Node.Index,
5896	lhs: Ast.Node.Index,
5897	cond_op: Zir.Inst.Tag,
5898	unwrap_op: Zir.Inst.Tag,
5899	unwrap_code_op: Zir.Inst.Tag,
5900	rhs: Ast.Node.Index,
5901	payload_token: ?Ast.TokenIndex,
5902	) InnerError!Zir.Inst.Ref {
5903	const astgen = parent_gz.astgen;
5904	const tree = astgen.tree;
5905
5906	const need_rl = astgen.nodes_need_rl.contains(node);
5907	const block_ri: ResultInfo = if (need_rl) ri else .{
5908	.rl = switch (ri.rl) {
5909	.ptr => .{ .ty = (try ri.rl.resultType(parent_gz, node)).? },
5910	.inferred_ptr => .none,
5911	else => ri.rl,
5912	},
5913	.ctx = ri.ctx,
5914	};
5915	// We need to call `rvalue` to write through to the pointer only if we had a
5916	// result pointer and aren't forwarding it.
5917	const LocTag = @typeInfo(ResultInfo.Loc).Union.tag_type.?;
5918	const need_result_rvalue = @as(LocTag, block_ri.rl) != @as(LocTag, ri.rl);
5919
5920	const do_err_trace = astgen.fn_block != null and (cond_op == .is_non_err or cond_op == .is_non_err_ptr);
5921
5922	var block_scope = parent_gz.makeSubBlock(scope);
5923	block_scope.setBreakResultInfo(block_ri);
5924	defer block_scope.unstack();
5925
5926	const operand_ri: ResultInfo = switch (block_scope.break_result_info.rl) {
5927	.ref, .ref_coerced_ty => .{ .rl = .ref, .ctx = if (do_err_trace) .error_handling_expr else .none },
5928	else => .{ .rl = .none, .ctx = if (do_err_trace) .error_handling_expr else .none },
5929	};
5930	// This could be a pointer or value depending on the `operand_ri` parameter.
5931	// We cannot use `block_scope.break_result_info` because that has the bare
5932	// type, whereas this expression has the optional type. Later we make
5933	// up for this fact by calling rvalue on the else branch.
5934	const operand = try reachableExpr(&block_scope, &block_scope.base, operand_ri, lhs, rhs);
5935	const cond = try block_scope.addUnNode(cond_op, operand, node);
5936	const condbr = try block_scope.addCondBr(.condbr, node);
5937
5938	const block = try parent_gz.makeBlockInst(.block, node);
5939	try block_scope.setBlockBody(block);
5940	// block_scope unstacked now, can add new instructions to parent_gz
5941	try parent_gz.instructions.append(astgen.gpa, block);
5942
5943	var then_scope = block_scope.makeSubBlock(scope);
5944	defer then_scope.unstack();
5945
5946	// This could be a pointer or value depending on `unwrap_op`.
5947	const unwrapped_payload = try then_scope.addUnNode(unwrap_op, operand, node);
5948	const then_result = switch (ri.rl) {
5949	.ref, .ref_coerced_ty => unwrapped_payload,
5950	else => try rvalue(&then_scope, block_scope.break_result_info, unwrapped_payload, node),
5951	};
5952	_ = try then_scope.addBreakWithSrcNode(.@"break", block, then_result, node);
5953
5954	var else_scope = block_scope.makeSubBlock(scope);
5955	defer else_scope.unstack();
5956
5957	// We know that the operand (almost certainly) modified the error return trace,
5958	// so signal to Sema that it should save the new index for restoring later.
5959	if (do_err_trace and nodeMayAppendToErrorTrace(tree, lhs))
5960	_ = try else_scope.addSaveErrRetIndex(.always);
5961
5962	var err_val_scope: Scope.LocalVal = undefined;
5963	const else_sub_scope = blk: {
5964	const payload = payload_token orelse break :blk &else_scope.base;
5965	const err_str = tree.tokenSlice(payload);
5966	if (mem.eql(u8, err_str, "_")) {
5967	return astgen.failTok(payload, "discard of error capture; omit it instead", .{});
5968	}
5969	const err_name = try astgen.identAsString(payload);
5970
5971	try astgen.detectLocalShadowing(scope, err_name, payload, err_str, .capture);
5972
5973	err_val_scope = .{
5974	.parent = &else_scope.base,
5975	.gen_zir = &else_scope,
5976	.name = err_name,
5977	.inst = try else_scope.addUnNode(unwrap_code_op, operand, node),
5978	.token_src = payload,
5979	.id_cat = .capture,
5980	};
5981	break :blk &err_val_scope.base;
5982	};
5983
5984	const else_result = try expr(&else_scope, else_sub_scope, block_scope.break_result_info, rhs);
5985	if (!else_scope.endsWithNoReturn()) {
5986	// As our last action before the break, "pop" the error trace if needed
5987	if (do_err_trace)
5988	try restoreErrRetIndex(&else_scope, .{ .block = block }, block_scope.break_result_info, rhs, else_result);
5989
5990	_ = try else_scope.addBreakWithSrcNode(.@"break", block, else_result, rhs);
5991	}
5992	try checkUsed(parent_gz, &else_scope.base, else_sub_scope);
5993
5994	try setCondBrPayload(condbr, cond, &then_scope, &else_scope);
5995
5996	if (need_result_rvalue) {
5997	return rvalue(parent_gz, ri, block.toRef(), node);
5998	} else {
5999	return block.toRef();
6000	}
6001	}
6002
6003	/// Return whether the identifier names of two tokens are equal. Resolves @""
6004	/// tokens without allocating.
6005	/// OK in theory it could do it without allocating. This implementation
6006	/// allocates when the @"" form is used.
6007	fn tokenIdentEql(astgen: *AstGen, token1: Ast.TokenIndex, token2: Ast.TokenIndex) !bool {
6008	const ident_name_1 = try astgen.identifierTokenString(token1);
6009	const ident_name_2 = try astgen.identifierTokenString(token2);
6010	return mem.eql(u8, ident_name_1, ident_name_2);
6011	}
6012
6013	fn fieldAccess(
6014	gz: *GenZir,
6015	scope: *Scope,
6016	ri: ResultInfo,
6017	node: Ast.Node.Index,
6018	) InnerError!Zir.Inst.Ref {
6019	switch (ri.rl) {
6020	.ref, .ref_coerced_ty => return addFieldAccess(.field_ptr, gz, scope, .{ .rl = .ref }, node),
6021	else => {
6022	const access = try addFieldAccess(.field_val, gz, scope, .{ .rl = .none }, node);
6023	return rvalue(gz, ri, access, node);
6024	},
6025	}
6026	}
6027
6028	fn addFieldAccess(
6029	tag: Zir.Inst.Tag,
6030	gz: *GenZir,
6031	scope: *Scope,
6032	lhs_ri: ResultInfo,
6033	node: Ast.Node.Index,
6034	) InnerError!Zir.Inst.Ref {
6035	const astgen = gz.astgen;
6036	const tree = astgen.tree;
6037	const main_tokens = tree.nodes.items(.main_token);
6038	const node_datas = tree.nodes.items(.data);
6039
6040	const object_node = node_datas[node].lhs;
6041	const dot_token = main_tokens[node];
6042	const field_ident = dot_token + 1;
6043	const str_index = try astgen.identAsString(field_ident);
6044	const lhs = try expr(gz, scope, lhs_ri, object_node);
6045
6046	const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
6047	try emitDbgStmt(gz, cursor);
6048
6049	return gz.addPlNode(tag, node, Zir.Inst.Field{
6050	.lhs = lhs,
6051	.field_name_start = str_index,
6052	});
6053	}
6054
6055	fn arrayAccess(
6056	gz: *GenZir,
6057	scope: *Scope,
6058	ri: ResultInfo,
6059	node: Ast.Node.Index,
6060	) InnerError!Zir.Inst.Ref {
6061	const tree = gz.astgen.tree;
6062	const node_datas = tree.nodes.items(.data);
6063	switch (ri.rl) {
6064	.ref, .ref_coerced_ty => {
6065	const lhs = try expr(gz, scope, .{ .rl = .ref }, node_datas[node].lhs);
6066
6067	const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
6068
6069	const rhs = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, node_datas[node].rhs);
6070	try emitDbgStmt(gz, cursor);
6071
6072	return gz.addPlNode(.elem_ptr_node, node, Zir.Inst.Bin{ .lhs = lhs, .rhs = rhs });
6073	},
6074	else => {
6075	const lhs = try expr(gz, scope, .{ .rl = .none }, node_datas[node].lhs);
6076
6077	const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
6078
6079	const rhs = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, node_datas[node].rhs);
6080	try emitDbgStmt(gz, cursor);
6081
6082	return rvalue(gz, ri, try gz.addPlNode(.elem_val_node, node, Zir.Inst.Bin{ .lhs = lhs, .rhs = rhs }), node);
6083	},
6084	}
6085	}
6086
6087	fn simpleBinOp(
6088	gz: *GenZir,
6089	scope: *Scope,
6090	ri: ResultInfo,
6091	node: Ast.Node.Index,
6092	op_inst_tag: Zir.Inst.Tag,
6093	) InnerError!Zir.Inst.Ref {
6094	const astgen = gz.astgen;
6095	const tree = astgen.tree;
6096	const node_datas = tree.nodes.items(.data);
6097
6098	if (op_inst_tag == .cmp_neq or op_inst_tag == .cmp_eq) {
6099	const node_tags = tree.nodes.items(.tag);
6100	const str = if (op_inst_tag == .cmp_eq) "==" else "!=";
6101	if (node_tags[node_datas[node].lhs] == .string_literal or
6102	node_tags[node_datas[node].rhs] == .string_literal)
6103	return astgen.failNode(node, "cannot compare strings with {s}", .{str});
6104	}
6105
6106	const lhs = try reachableExpr(gz, scope, .{ .rl = .none }, node_datas[node].lhs, node);
6107	const cursor = switch (op_inst_tag) {
6108	.add, .sub, .mul, .div, .mod_rem => maybeAdvanceSourceCursorToMainToken(gz, node),
6109	else => undefined,
6110	};
6111	const rhs = try reachableExpr(gz, scope, .{ .rl = .none }, node_datas[node].rhs, node);
6112
6113	switch (op_inst_tag) {
6114	.add, .sub, .mul, .div, .mod_rem => {
6115	try emitDbgStmt(gz, cursor);
6116	},
6117	else => {},
6118	}
6119	const result = try gz.addPlNode(op_inst_tag, node, Zir.Inst.Bin{ .lhs = lhs, .rhs = rhs });
6120	return rvalue(gz, ri, result, node);
6121	}
6122
6123	fn simpleStrTok(
6124	gz: *GenZir,
6125	ri: ResultInfo,
6126	ident_token: Ast.TokenIndex,
6127	node: Ast.Node.Index,
6128	op_inst_tag: Zir.Inst.Tag,
6129	) InnerError!Zir.Inst.Ref {
6130	const astgen = gz.astgen;
6131	const str_index = try astgen.identAsString(ident_token);
6132	const result = try gz.addStrTok(op_inst_tag, str_index, ident_token);
6133	return rvalue(gz, ri, result, node);
6134	}
6135
6136	fn boolBinOp(
6137	gz: *GenZir,
6138	scope: *Scope,
6139	ri: ResultInfo,
6140	node: Ast.Node.Index,
6141	zir_tag: Zir.Inst.Tag,
6142	) InnerError!Zir.Inst.Ref {
6143	const astgen = gz.astgen;
6144	const tree = astgen.tree;
6145	const node_datas = tree.nodes.items(.data);
6146
6147	const lhs = try expr(gz, scope, coerced_bool_ri, node_datas[node].lhs);
6148	const bool_br = (try gz.addPlNodePayloadIndex(zir_tag, node, undefined)).toIndex().?;
6149
6150	var rhs_scope = gz.makeSubBlock(scope);
6151	defer rhs_scope.unstack();
6152	const rhs = try expr(&rhs_scope, &rhs_scope.base, coerced_bool_ri, node_datas[node].rhs);
6153	if (!gz.refIsNoReturn(rhs)) {
6154	_ = try rhs_scope.addBreakWithSrcNode(.break_inline, bool_br, rhs, node_datas[node].rhs);
6155	}
6156	try rhs_scope.setBoolBrBody(bool_br, lhs);
6157
6158	const block_ref = bool_br.toRef();
6159	return rvalue(gz, ri, block_ref, node);
6160	}
6161
6162	fn ifExpr(
6163	parent_gz: *GenZir,
6164	scope: *Scope,
6165	ri: ResultInfo,
6166	node: Ast.Node.Index,
6167	if_full: Ast.full.If,
6168	) InnerError!Zir.Inst.Ref {
6169	const astgen = parent_gz.astgen;
6170	const tree = astgen.tree;
6171	const token_tags = tree.tokens.items(.tag);
6172
6173	const do_err_trace = astgen.fn_block != null and if_full.error_token != null;
6174
6175	const need_rl = astgen.nodes_need_rl.contains(node);
6176	const block_ri: ResultInfo = if (need_rl) ri else .{
6177	.rl = switch (ri.rl) {
6178	.ptr => .{ .ty = (try ri.rl.resultType(parent_gz, node)).? },
6179	.inferred_ptr => .none,
6180	else => ri.rl,
6181	},
6182	.ctx = ri.ctx,
6183	};
6184	// We need to call `rvalue` to write through to the pointer only if we had a
6185	// result pointer and aren't forwarding it.
6186	const LocTag = @typeInfo(ResultInfo.Loc).Union.tag_type.?;
6187	const need_result_rvalue = @as(LocTag, block_ri.rl) != @as(LocTag, ri.rl);
6188
6189	var block_scope = parent_gz.makeSubBlock(scope);
6190	block_scope.setBreakResultInfo(block_ri);
6191	defer block_scope.unstack();
6192
6193	const payload_is_ref = if (if_full.payload_token) \|payload_token\|
6194	token_tags[payload_token] == .asterisk
6195	else
6196	false;
6197
6198	try emitDbgNode(parent_gz, if_full.ast.cond_expr);
6199	const cond: struct {
6200	inst: Zir.Inst.Ref,
6201	bool_bit: Zir.Inst.Ref,
6202	} = c: {
6203	if (if_full.error_token) \|_\| {
6204	const cond_ri: ResultInfo = .{ .rl = if (payload_is_ref) .ref else .none, .ctx = .error_handling_expr };
6205	const err_union = try expr(&block_scope, &block_scope.base, cond_ri, if_full.ast.cond_expr);
6206	const tag: Zir.Inst.Tag = if (payload_is_ref) .is_non_err_ptr else .is_non_err;
6207	break :c .{
6208	.inst = err_union,
6209	.bool_bit = try block_scope.addUnNode(tag, err_union, if_full.ast.cond_expr),
6210	};
6211	} else if (if_full.payload_token) \|_\| {
6212	const cond_ri: ResultInfo = .{ .rl = if (payload_is_ref) .ref else .none };
6213	const optional = try expr(&block_scope, &block_scope.base, cond_ri, if_full.ast.cond_expr);
6214	const tag: Zir.Inst.Tag = if (payload_is_ref) .is_non_null_ptr else .is_non_null;
6215	break :c .{
6216	.inst = optional,
6217	.bool_bit = try block_scope.addUnNode(tag, optional, if_full.ast.cond_expr),
6218	};
6219	} else {
6220	const cond = try expr(&block_scope, &block_scope.base, coerced_bool_ri, if_full.ast.cond_expr);
6221	break :c .{
6222	.inst = cond,
6223	.bool_bit = cond,
6224	};
6225	}
6226	};
6227
6228	const condbr = try block_scope.addCondBr(.condbr, node);
6229
6230	const block = try parent_gz.makeBlockInst(.block, node);
6231	try block_scope.setBlockBody(block);
6232	// block_scope unstacked now, can add new instructions to parent_gz
6233	try parent_gz.instructions.append(astgen.gpa, block);
6234
6235	var then_scope = parent_gz.makeSubBlock(scope);
6236	defer then_scope.unstack();
6237
6238	var payload_val_scope: Scope.LocalVal = undefined;
6239
6240	const then_node = if_full.ast.then_expr;
6241	const then_sub_scope = s: {
6242	if (if_full.error_token != null) {
6243	if (if_full.payload_token) \|payload_token\| {
6244	const tag: Zir.Inst.Tag = if (payload_is_ref)
6245	.err_union_payload_unsafe_ptr
6246	else
6247	.err_union_payload_unsafe;
6248	const payload_inst = try then_scope.addUnNode(tag, cond.inst, then_node);
6249	const token_name_index = payload_token + @intFromBool(payload_is_ref);
6250	const ident_name = try astgen.identAsString(token_name_index);
6251	const token_name_str = tree.tokenSlice(token_name_index);
6252	if (mem.eql(u8, "_", token_name_str))
6253	break :s &then_scope.base;
6254	try astgen.detectLocalShadowing(&then_scope.base, ident_name, token_name_index, token_name_str, .capture);
6255	payload_val_scope = .{
6256	.parent = &then_scope.base,
6257	.gen_zir = &then_scope,
6258	.name = ident_name,
6259	.inst = payload_inst,
6260	.token_src = token_name_index,
6261	.id_cat = .capture,
6262	};
6263	try then_scope.addDbgVar(.dbg_var_val, ident_name, payload_inst);
6264	break :s &payload_val_scope.base;
6265	} else {
6266	_ = try then_scope.addUnNode(.ensure_err_union_payload_void, cond.inst, node);
6267	break :s &then_scope.base;
6268	}
6269	} else if (if_full.payload_token) \|payload_token\| {
6270	const ident_token = if (payload_is_ref) payload_token + 1 else payload_token;
6271	const tag: Zir.Inst.Tag = if (payload_is_ref)
6272	.optional_payload_unsafe_ptr
6273	else
6274	.optional_payload_unsafe;
6275	const ident_bytes = tree.tokenSlice(ident_token);
6276	if (mem.eql(u8, "_", ident_bytes))
6277	break :s &then_scope.base;
6278	const payload_inst = try then_scope.addUnNode(tag, cond.inst, then_node);
6279	const ident_name = try astgen.identAsString(ident_token);
6280	try astgen.detectLocalShadowing(&then_scope.base, ident_name, ident_token, ident_bytes, .capture);
6281	payload_val_scope = .{
6282	.parent = &then_scope.base,
6283	.gen_zir = &then_scope,
6284	.name = ident_name,
6285	.inst = payload_inst,
6286	.token_src = ident_token,
6287	.id_cat = .capture,
6288	};
6289	try then_scope.addDbgVar(.dbg_var_val, ident_name, payload_inst);
6290	break :s &payload_val_scope.base;
6291	} else {
6292	break :s &then_scope.base;
6293	}
6294	};
6295
6296	const then_result = try expr(&then_scope, then_sub_scope, block_scope.break_result_info, then_node);
6297	try checkUsed(parent_gz, &then_scope.base, then_sub_scope);
6298	if (!then_scope.endsWithNoReturn()) {
6299	_ = try then_scope.addBreakWithSrcNode(.@"break", block, then_result, then_node);
6300	}
6301
6302	var else_scope = parent_gz.makeSubBlock(scope);
6303	defer else_scope.unstack();
6304
6305	// We know that the operand (almost certainly) modified the error return trace,
6306	// so signal to Sema that it should save the new index for restoring later.
6307	if (do_err_trace and nodeMayAppendToErrorTrace(tree, if_full.ast.cond_expr))
6308	_ = try else_scope.addSaveErrRetIndex(.always);
6309
6310	const else_node = if_full.ast.else_expr;
6311	if (else_node != 0) {
6312	const sub_scope = s: {
6313	if (if_full.error_token) \|error_token\| {
6314	const tag: Zir.Inst.Tag = if (payload_is_ref)
6315	.err_union_code_ptr
6316	else
6317	.err_union_code;
6318	const payload_inst = try else_scope.addUnNode(tag, cond.inst, if_full.ast.cond_expr);
6319	const ident_name = try astgen.identAsString(error_token);
6320	const error_token_str = tree.tokenSlice(error_token);
6321	if (mem.eql(u8, "_", error_token_str))
6322	break :s &else_scope.base;
6323	try astgen.detectLocalShadowing(&else_scope.base, ident_name, error_token, error_token_str, .capture);
6324	payload_val_scope = .{
6325	.parent = &else_scope.base,
6326	.gen_zir = &else_scope,
6327	.name = ident_name,
6328	.inst = payload_inst,
6329	.token_src = error_token,
6330	.id_cat = .capture,
6331	};
6332	try else_scope.addDbgVar(.dbg_var_val, ident_name, payload_inst);
6333	break :s &payload_val_scope.base;
6334	} else {
6335	break :s &else_scope.base;
6336	}
6337	};
6338	const else_result = try expr(&else_scope, sub_scope, block_scope.break_result_info, else_node);
6339	if (!else_scope.endsWithNoReturn()) {
6340	// As our last action before the break, "pop" the error trace if needed
6341	if (do_err_trace)
6342	try restoreErrRetIndex(&else_scope, .{ .block = block }, block_scope.break_result_info, else_node, else_result);
6343	_ = try else_scope.addBreakWithSrcNode(.@"break", block, else_result, else_node);
6344	}
6345	try checkUsed(parent_gz, &else_scope.base, sub_scope);
6346	} else {
6347	const result = try rvalue(&else_scope, ri, .void_value, node);
6348	_ = try else_scope.addBreak(.@"break", block, result);
6349	}
6350
6351	try setCondBrPayload(condbr, cond.bool_bit, &then_scope, &else_scope);
6352
6353	if (need_result_rvalue) {
6354	return rvalue(parent_gz, ri, block.toRef(), node);
6355	} else {
6356	return block.toRef();
6357	}
6358	}
6359
6360	/// Supports `else_scope` stacked on `then_scope`. Unstacks `else_scope` then `then_scope`.
6361	fn setCondBrPayload(
6362	condbr: Zir.Inst.Index,
6363	cond: Zir.Inst.Ref,
6364	then_scope: *GenZir,
6365	else_scope: *GenZir,
6366	) !void {
6367	defer then_scope.unstack();
6368	defer else_scope.unstack();
6369	const astgen = then_scope.astgen;
6370	const then_body = then_scope.instructionsSliceUpto(else_scope);
6371	const else_body = else_scope.instructionsSlice();
6372	const then_body_len = astgen.countBodyLenAfterFixups(then_body);
6373	const else_body_len = astgen.countBodyLenAfterFixups(else_body);
6374	try astgen.extra.ensureUnusedCapacity(
6375	astgen.gpa,
6376	@typeInfo(Zir.Inst.CondBr).Struct.fields.len + then_body_len + else_body_len,
6377	);
6378
6379	const zir_datas = astgen.instructions.items(.data);
6380	zir_datas[@intFromEnum(condbr)].pl_node.payload_index = astgen.addExtraAssumeCapacity(Zir.Inst.CondBr{
6381	.condition = cond,
6382	.then_body_len = then_body_len,
6383	.else_body_len = else_body_len,
6384	});
6385	astgen.appendBodyWithFixups(then_body);
6386	astgen.appendBodyWithFixups(else_body);
6387	}
6388
6389	fn whileExpr(
6390	parent_gz: *GenZir,
6391	scope: *Scope,
6392	ri: ResultInfo,
6393	node: Ast.Node.Index,
6394	while_full: Ast.full.While,
6395	is_statement: bool,
6396	) InnerError!Zir.Inst.Ref {
6397	const astgen = parent_gz.astgen;
6398	const tree = astgen.tree;
6399	const token_tags = tree.tokens.items(.tag);
6400
6401	const need_rl = astgen.nodes_need_rl.contains(node);
6402	const block_ri: ResultInfo = if (need_rl) ri else .{
6403	.rl = switch (ri.rl) {
6404	.ptr => .{ .ty = (try ri.rl.resultType(parent_gz, node)).? },
6405	.inferred_ptr => .none,
6406	else => ri.rl,
6407	},
6408	.ctx = ri.ctx,
6409	};
6410	// We need to call `rvalue` to write through to the pointer only if we had a
6411	// result pointer and aren't forwarding it.
6412	const LocTag = @typeInfo(ResultInfo.Loc).Union.tag_type.?;
6413	const need_result_rvalue = @as(LocTag, block_ri.rl) != @as(LocTag, ri.rl);
6414
6415	if (while_full.label_token) \|label_token\| {
6416	try astgen.checkLabelRedefinition(scope, label_token);
6417	}
6418
6419	const is_inline = while_full.inline_token != null;
6420	if (parent_gz.is_comptime and is_inline) {
6421	return astgen.failTok(while_full.inline_token.?, "redundant inline keyword in comptime scope", .{});
6422	}
6423	const loop_tag: Zir.Inst.Tag = if (is_inline) .block_inline else .loop;
6424	const loop_block = try parent_gz.makeBlockInst(loop_tag, node);
6425	try parent_gz.instructions.append(astgen.gpa, loop_block);
6426
6427	var loop_scope = parent_gz.makeSubBlock(scope);
6428	loop_scope.is_inline = is_inline;
6429	loop_scope.setBreakResultInfo(block_ri);
6430	defer loop_scope.unstack();
6431
6432	var cond_scope = parent_gz.makeSubBlock(&loop_scope.base);
6433	defer cond_scope.unstack();
6434
6435	const payload_is_ref = if (while_full.payload_token) \|payload_token\|
6436	token_tags[payload_token] == .asterisk
6437	else
6438	false;
6439
6440	try emitDbgNode(parent_gz, while_full.ast.cond_expr);
6441	const cond: struct {
6442	inst: Zir.Inst.Ref,
6443	bool_bit: Zir.Inst.Ref,
6444	} = c: {
6445	if (while_full.error_token) \|_\| {
6446	const cond_ri: ResultInfo = .{ .rl = if (payload_is_ref) .ref else .none };
6447	const err_union = try expr(&cond_scope, &cond_scope.base, cond_ri, while_full.ast.cond_expr);
6448	const tag: Zir.Inst.Tag = if (payload_is_ref) .is_non_err_ptr else .is_non_err;
6449	break :c .{
6450	.inst = err_union,
6451	.bool_bit = try cond_scope.addUnNode(tag, err_union, while_full.ast.cond_expr),
6452	};
6453	} else if (while_full.payload_token) \|_\| {
6454	const cond_ri: ResultInfo = .{ .rl = if (payload_is_ref) .ref else .none };
6455	const optional = try expr(&cond_scope, &cond_scope.base, cond_ri, while_full.ast.cond_expr);
6456	const tag: Zir.Inst.Tag = if (payload_is_ref) .is_non_null_ptr else .is_non_null;
6457	break :c .{
6458	.inst = optional,
6459	.bool_bit = try cond_scope.addUnNode(tag, optional, while_full.ast.cond_expr),
6460	};
6461	} else {
6462	const cond = try expr(&cond_scope, &cond_scope.base, coerced_bool_ri, while_full.ast.cond_expr);
6463	break :c .{
6464	.inst = cond,
6465	.bool_bit = cond,
6466	};
6467	}
6468	};
6469
6470	const condbr_tag: Zir.Inst.Tag = if (is_inline) .condbr_inline else .condbr;
6471	const condbr = try cond_scope.addCondBr(condbr_tag, node);
6472	const block_tag: Zir.Inst.Tag = if (is_inline) .block_inline else .block;
6473	const cond_block = try loop_scope.makeBlockInst(block_tag, node);
6474	try cond_scope.setBlockBody(cond_block);
6475	// cond_scope unstacked now, can add new instructions to loop_scope
6476	try loop_scope.instructions.append(astgen.gpa, cond_block);
6477
6478	// make scope now but don't stack on parent_gz until loop_scope
6479	// gets unstacked after cont_expr is emitted and added below
6480	var then_scope = parent_gz.makeSubBlock(&cond_scope.base);
6481	then_scope.instructions_top = GenZir.unstacked_top;
6482	defer then_scope.unstack();
6483
6484	var dbg_var_name: Zir.NullTerminatedString = .empty;
6485	var dbg_var_inst: Zir.Inst.Ref = undefined;
6486	var opt_payload_inst: Zir.Inst.OptionalIndex = .none;
6487	var payload_val_scope: Scope.LocalVal = undefined;
6488	const then_sub_scope = s: {
6489	if (while_full.error_token != null) {
6490	if (while_full.payload_token) \|payload_token\| {
6491	const tag: Zir.Inst.Tag = if (payload_is_ref)
6492	.err_union_payload_unsafe_ptr
6493	else
6494	.err_union_payload_unsafe;
6495	// will add this instruction to then_scope.instructions below
6496	const payload_inst = try then_scope.makeUnNode(tag, cond.inst, while_full.ast.cond_expr);
6497	opt_payload_inst = payload_inst.toOptional();
6498	const ident_token = payload_token + @intFromBool(payload_is_ref);
6499	const ident_bytes = tree.tokenSlice(ident_token);
6500	if (mem.eql(u8, "_", ident_bytes))
6501	break :s &then_scope.base;
6502	const ident_name = try astgen.identAsString(ident_token);
6503	try astgen.detectLocalShadowing(&then_scope.base, ident_name, ident_token, ident_bytes, .capture);
6504	payload_val_scope = .{
6505	.parent = &then_scope.base,
6506	.gen_zir = &then_scope,
6507	.name = ident_name,
6508	.inst = payload_inst.toRef(),
6509	.token_src = ident_token,
6510	.id_cat = .capture,
6511	};
6512	dbg_var_name = ident_name;
6513	dbg_var_inst = payload_inst.toRef();
6514	break :s &payload_val_scope.base;
6515	} else {
6516	_ = try then_scope.addUnNode(.ensure_err_union_payload_void, cond.inst, node);
6517	break :s &then_scope.base;
6518	}
6519	} else if (while_full.payload_token) \|payload_token\| {
6520	const ident_token = if (payload_is_ref) payload_token + 1 else payload_token;
6521	const tag: Zir.Inst.Tag = if (payload_is_ref)
6522	.optional_payload_unsafe_ptr
6523	else
6524	.optional_payload_unsafe;
6525	// will add this instruction to then_scope.instructions below
6526	const payload_inst = try then_scope.makeUnNode(tag, cond.inst, while_full.ast.cond_expr);
6527	opt_payload_inst = payload_inst.toOptional();
6528	const ident_name = try astgen.identAsString(ident_token);
6529	const ident_bytes = tree.tokenSlice(ident_token);
6530	if (mem.eql(u8, "_", ident_bytes))
6531	break :s &then_scope.base;
6532	try astgen.detectLocalShadowing(&then_scope.base, ident_name, ident_token, ident_bytes, .capture);
6533	payload_val_scope = .{
6534	.parent = &then_scope.base,
6535	.gen_zir = &then_scope,
6536	.name = ident_name,
6537	.inst = payload_inst.toRef(),
6538	.token_src = ident_token,
6539	.id_cat = .capture,
6540	};
6541	dbg_var_name = ident_name;
6542	dbg_var_inst = payload_inst.toRef();
6543	break :s &payload_val_scope.base;
6544	} else {
6545	break :s &then_scope.base;
6546	}
6547	};
6548
6549	var continue_scope = parent_gz.makeSubBlock(then_sub_scope);
6550	continue_scope.instructions_top = GenZir.unstacked_top;
6551	defer continue_scope.unstack();
6552	const continue_block = try then_scope.makeBlockInst(block_tag, node);
6553
6554	const repeat_tag: Zir.Inst.Tag = if (is_inline) .repeat_inline else .repeat;
6555	_ = try loop_scope.addNode(repeat_tag, node);
6556
6557	try loop_scope.setBlockBody(loop_block);
6558	loop_scope.break_block = loop_block.toOptional();
6559	loop_scope.continue_block = continue_block.toOptional();
6560	if (while_full.label_token) \|label_token\| {
6561	loop_scope.label = .{
6562	.token = label_token,
6563	.block_inst = loop_block,
6564	};
6565	}
6566
6567	// done adding instructions to loop_scope, can now stack then_scope
6568	then_scope.instructions_top = then_scope.instructions.items.len;
6569
6570	const then_node = while_full.ast.then_expr;
6571	if (opt_payload_inst.unwrap()) \|payload_inst\| {
6572	try then_scope.instructions.append(astgen.gpa, payload_inst);
6573	}
6574	if (dbg_var_name != .empty) try then_scope.addDbgVar(.dbg_var_val, dbg_var_name, dbg_var_inst);
6575	try then_scope.instructions.append(astgen.gpa, continue_block);
6576	// This code could be improved to avoid emitting the continue expr when there
6577	// are no jumps to it. This happens when the last statement of a while body is noreturn
6578	// and there are no `continue` statements.
6579	// Tracking issue: https://github.com/ziglang/zig/issues/9185
6580	if (while_full.ast.cont_expr != 0) {
6581	_ = try unusedResultExpr(&then_scope, then_sub_scope, while_full.ast.cont_expr);
6582	}
6583
6584	continue_scope.instructions_top = continue_scope.instructions.items.len;
6585	_ = try unusedResultExpr(&continue_scope, &continue_scope.base, then_node);
6586	try checkUsed(parent_gz, &then_scope.base, then_sub_scope);
6587	const break_tag: Zir.Inst.Tag = if (is_inline) .break_inline else .@"break";
6588	if (!continue_scope.endsWithNoReturn()) {
6589	_ = try continue_scope.addBreak(break_tag, continue_block, .void_value);
6590	}
6591	try continue_scope.setBlockBody(continue_block);
6592	_ = try then_scope.addBreak(break_tag, cond_block, .void_value);
6593
6594	var else_scope = parent_gz.makeSubBlock(&cond_scope.base);
6595	defer else_scope.unstack();
6596
6597	const else_node = while_full.ast.else_expr;
6598	if (else_node != 0) {
6599	const sub_scope = s: {
6600	if (while_full.error_token) \|error_token\| {
6601	const tag: Zir.Inst.Tag = if (payload_is_ref)
6602	.err_union_code_ptr
6603	else
6604	.err_union_code;
6605	const else_payload_inst = try else_scope.addUnNode(tag, cond.inst, while_full.ast.cond_expr);
6606	const ident_name = try astgen.identAsString(error_token);
6607	const ident_bytes = tree.tokenSlice(error_token);
6608	if (mem.eql(u8, ident_bytes, "_"))
6609	break :s &else_scope.base;
6610	try astgen.detectLocalShadowing(&else_scope.base, ident_name, error_token, ident_bytes, .capture);
6611	payload_val_scope = .{
6612	.parent = &else_scope.base,
6613	.gen_zir = &else_scope,
6614	.name = ident_name,
6615	.inst = else_payload_inst,
6616	.token_src = error_token,
6617	.id_cat = .capture,
6618	};
6619	try else_scope.addDbgVar(.dbg_var_val, ident_name, else_payload_inst);
6620	break :s &payload_val_scope.base;
6621	} else {
6622	break :s &else_scope.base;
6623	}
6624	};
6625	// Remove the continue block and break block so that `continue` and `break`
6626	// control flow apply to outer loops; not this one.
6627	loop_scope.continue_block = .none;
6628	loop_scope.break_block = .none;
6629	const else_result = try expr(&else_scope, sub_scope, loop_scope.break_result_info, else_node);
6630	if (is_statement) {
6631	_ = try addEnsureResult(&else_scope, else_result, else_node);
6632	}
6633
6634	try checkUsed(parent_gz, &else_scope.base, sub_scope);
6635	if (!else_scope.endsWithNoReturn()) {
6636	_ = try else_scope.addBreakWithSrcNode(break_tag, loop_block, else_result, else_node);
6637	}
6638	} else {
6639	const result = try rvalue(&else_scope, ri, .void_value, node);
6640	_ = try else_scope.addBreak(break_tag, loop_block, result);
6641	}
6642
6643	if (loop_scope.label) \|some\| {
6644	if (!some.used) {
6645	try astgen.appendErrorTok(some.token, "unused while loop label", .{});
6646	}
6647	}
6648
6649	try setCondBrPayload(condbr, cond.bool_bit, &then_scope, &else_scope);
6650
6651	const result = if (need_result_rvalue)
6652	try rvalue(parent_gz, ri, loop_block.toRef(), node)
6653	else
6654	loop_block.toRef();
6655
6656	if (is_statement) {
6657	_ = try parent_gz.addUnNode(.ensure_result_used, result, node);
6658	}
6659
6660	return result;
6661	}
6662
6663	fn forExpr(
6664	parent_gz: *GenZir,
6665	scope: *Scope,
6666	ri: ResultInfo,
6667	node: Ast.Node.Index,
6668	for_full: Ast.full.For,
6669	is_statement: bool,
6670	) InnerError!Zir.Inst.Ref {
6671	const astgen = parent_gz.astgen;
6672
6673	if (for_full.label_token) \|label_token\| {
6674	try astgen.checkLabelRedefinition(scope, label_token);
6675	}
6676
6677	const need_rl = astgen.nodes_need_rl.contains(node);
6678	const block_ri: ResultInfo = if (need_rl) ri else .{
6679	.rl = switch (ri.rl) {
6680	.ptr => .{ .ty = (try ri.rl.resultType(parent_gz, node)).? },
6681	.inferred_ptr => .none,
6682	else => ri.rl,
6683	},
6684	.ctx = ri.ctx,
6685	};
6686	// We need to call `rvalue` to write through to the pointer only if we had a
6687	// result pointer and aren't forwarding it.
6688	const LocTag = @typeInfo(ResultInfo.Loc).Union.tag_type.?;
6689	const need_result_rvalue = @as(LocTag, block_ri.rl) != @as(LocTag, ri.rl);
6690
6691	const is_inline = for_full.inline_token != null;
6692	if (parent_gz.is_comptime and is_inline) {
6693	return astgen.failTok(for_full.inline_token.?, "redundant inline keyword in comptime scope", .{});
6694	}
6695	const tree = astgen.tree;
6696	const token_tags = tree.tokens.items(.tag);
6697	const node_tags = tree.nodes.items(.tag);
6698	const node_data = tree.nodes.items(.data);
6699	const gpa = astgen.gpa;
6700
6701	// For counters, this is the start value; for indexables, this is the base
6702	// pointer that can be used with elem_ptr and similar instructions.
6703	// Special value `none` means that this is a counter and its start value is
6704	// zero, indicating that the main index counter can be used directly.
6705	const indexables = try gpa.alloc(Zir.Inst.Ref, for_full.ast.inputs.len);
6706	defer gpa.free(indexables);
6707	// elements of this array can be `none`, indicating no length check.
6708	const lens = try gpa.alloc(Zir.Inst.Ref, for_full.ast.inputs.len);
6709	defer gpa.free(lens);
6710
6711	// We will use a single zero-based counter no matter how many indexables there are.
6712	const index_ptr = blk: {
6713	const alloc_tag: Zir.Inst.Tag = if (is_inline) .alloc_comptime_mut else .alloc;
6714	const index_ptr = try parent_gz.addUnNode(alloc_tag, .usize_type, node);
6715	// initialize to zero
6716	_ = try parent_gz.addPlNode(.store_node, node, Zir.Inst.Bin{
6717	.lhs = index_ptr,
6718	.rhs = .zero_usize,
6719	});
6720	break :blk index_ptr;
6721	};
6722
6723	var any_len_checks = false;
6724
6725	{
6726	var capture_token = for_full.payload_token;
6727	for (for_full.ast.inputs, indexables, lens) \|input, indexable_ref, len_ref\| {
6728	const capture_is_ref = token_tags[capture_token] == .asterisk;
6729	const ident_tok = capture_token + @intFromBool(capture_is_ref);
6730	const is_discard = mem.eql(u8, tree.tokenSlice(ident_tok), "_");
6731
6732	if (is_discard and capture_is_ref) {
6733	return astgen.failTok(capture_token, "pointer modifier invalid on discard", .{});
6734	}
6735	// Skip over the comma, and on to the next capture (or the ending pipe character).
6736	capture_token = ident_tok + 2;
6737
6738	try emitDbgNode(parent_gz, input);
6739	if (node_tags[input] == .for_range) {
6740	if (capture_is_ref) {
6741	return astgen.failTok(ident_tok, "cannot capture reference to range", .{});
6742	}
6743	const start_node = node_data[input].lhs;
6744	const start_val = try expr(parent_gz, scope, .{ .rl = .{ .ty = .usize_type } }, start_node);
6745
6746	const end_node = node_data[input].rhs;
6747	const end_val = if (end_node != 0)
6748	try expr(parent_gz, scope, .{ .rl = .{ .ty = .usize_type } }, node_data[input].rhs)
6749	else
6750	.none;
6751
6752	if (end_val == .none and is_discard) {
6753	return astgen.failTok(ident_tok, "discard of unbounded counter", .{});
6754	}
6755
6756	const start_is_zero = nodeIsTriviallyZero(tree, start_node);
6757	const range_len = if (end_val == .none or start_is_zero)
6758	end_val
6759	else
6760	try parent_gz.addPlNode(.sub, input, Zir.Inst.Bin{
6761	.lhs = end_val,
6762	.rhs = start_val,
6763	});
6764
6765	any_len_checks = any_len_checks or range_len != .none;
6766	indexable_ref.* = if (start_is_zero) .none else start_val;
6767	len_ref.* = range_len;
6768	} else {
6769	const indexable = try expr(parent_gz, scope, .{ .rl = .none }, input);
6770
6771	any_len_checks = true;
6772	indexable_ref.* = indexable;
6773	len_ref.* = indexable;
6774	}
6775	}
6776	}
6777
6778	if (!any_len_checks) {
6779	return astgen.failNode(node, "unbounded for loop", .{});
6780	}
6781
6782	// We use a dedicated ZIR instruction to assert the lengths to assist with
6783	// nicer error reporting as well as fewer ZIR bytes emitted.
6784	const len: Zir.Inst.Ref = len: {
6785	const lens_len: u32 = @intCast(lens.len);
6786	try astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.MultiOp).Struct.fields.len + lens_len);
6787	const len = try parent_gz.addPlNode(.for_len, node, Zir.Inst.MultiOp{
6788	.operands_len = lens_len,
6789	});
6790	appendRefsAssumeCapacity(astgen, lens);
6791	break :len len;
6792	};
6793
6794	const loop_tag: Zir.Inst.Tag = if (is_inline) .block_inline else .loop;
6795	const loop_block = try parent_gz.makeBlockInst(loop_tag, node);
6796	try parent_gz.instructions.append(gpa, loop_block);
6797
6798	var loop_scope = parent_gz.makeSubBlock(scope);
6799	loop_scope.is_inline = is_inline;
6800	loop_scope.setBreakResultInfo(block_ri);
6801	defer loop_scope.unstack();
6802
6803	// We need to finish loop_scope later once we have the deferred refs from then_scope. However, the
6804	// load must be removed from instructions in the meantime or it appears to be part of parent_gz.
6805	const index = try loop_scope.addUnNode(.load, index_ptr, node);
6806	_ = loop_scope.instructions.pop();
6807
6808	var cond_scope = parent_gz.makeSubBlock(&loop_scope.base);
6809	defer cond_scope.unstack();
6810
6811	// Check the condition.
6812	const cond = try cond_scope.addPlNode(.cmp_lt, node, Zir.Inst.Bin{
6813	.lhs = index,
6814	.rhs = len,
6815	});
6816
6817	const condbr_tag: Zir.Inst.Tag = if (is_inline) .condbr_inline else .condbr;
6818	const condbr = try cond_scope.addCondBr(condbr_tag, node);
6819	const block_tag: Zir.Inst.Tag = if (is_inline) .block_inline else .block;
6820	const cond_block = try loop_scope.makeBlockInst(block_tag, node);
6821	try cond_scope.setBlockBody(cond_block);
6822
6823	loop_scope.break_block = loop_block.toOptional();
6824	loop_scope.continue_block = cond_block.toOptional();
6825	if (for_full.label_token) \|label_token\| {
6826	loop_scope.label = .{
6827	.token = label_token,
6828	.block_inst = loop_block,
6829	};
6830	}
6831
6832	const then_node = for_full.ast.then_expr;
6833	var then_scope = parent_gz.makeSubBlock(&cond_scope.base);
6834	defer then_scope.unstack();
6835
6836	const capture_scopes = try gpa.alloc(Scope.LocalVal, for_full.ast.inputs.len);
6837	defer gpa.free(capture_scopes);
6838
6839	const then_sub_scope = blk: {
6840	var capture_token = for_full.payload_token;
6841	var capture_sub_scope: *Scope = &then_scope.base;
6842	for (for_full.ast.inputs, indexables, capture_scopes) \|input, indexable_ref, *capture_scope\| {
6843	const capture_is_ref = token_tags[capture_token] == .asterisk;
6844	const ident_tok = capture_token + @intFromBool(capture_is_ref);
6845	const capture_name = tree.tokenSlice(ident_tok);
6846	// Skip over the comma, and on to the next capture (or the ending pipe character).
6847	capture_token = ident_tok + 2;
6848
6849	if (mem.eql(u8, capture_name, "_")) continue;
6850
6851	const name_str_index = try astgen.identAsString(ident_tok);
6852	try astgen.detectLocalShadowing(capture_sub_scope, name_str_index, ident_tok, capture_name, .capture);
6853
6854	const capture_inst = inst: {
6855	const is_counter = node_tags[input] == .for_range;
6856
6857	if (indexable_ref == .none) {
6858	// Special case: the main index can be used directly.
6859	assert(is_counter);
6860	assert(!capture_is_ref);
6861	break :inst index;
6862	}
6863
6864	// For counters, we add the index variable to the start value; for
6865	// indexables, we use it as an element index. This is so similar
6866	// that they can share the same code paths, branching only on the
6867	// ZIR tag.
6868	const switch_cond = (@as(u2, @intFromBool(capture_is_ref)) << 1) \| @intFromBool(is_counter);
6869	const tag: Zir.Inst.Tag = switch (switch_cond) {
6870	0b00 => .elem_val,
6871	0b01 => .add,
6872	0b10 => .elem_ptr,
6873	0b11 => unreachable, // compile error emitted already
6874	};
6875	break :inst try then_scope.addPlNode(tag, input, Zir.Inst.Bin{
6876	.lhs = indexable_ref,
6877	.rhs = index,
6878	});
6879	};
6880
6881	capture_scope.* = .{
6882	.parent = capture_sub_scope,
6883	.gen_zir = &then_scope,
6884	.name = name_str_index,
6885	.inst = capture_inst,
6886	.token_src = ident_tok,
6887	.id_cat = .capture,
6888	};
6889
6890	try then_scope.addDbgVar(.dbg_var_val, name_str_index, capture_inst);
6891	capture_sub_scope = &capture_scope.base;
6892	}
6893
6894	break :blk capture_sub_scope;
6895	};
6896
6897	const then_result = try expr(&then_scope, then_sub_scope, .{ .rl = .none }, then_node);
6898	_ = try addEnsureResult(&then_scope, then_result, then_node);
6899
6900	try checkUsed(parent_gz, &then_scope.base, then_sub_scope);
6901
6902	const break_tag: Zir.Inst.Tag = if (is_inline) .break_inline else .@"break";
6903
6904	_ = try then_scope.addBreak(break_tag, cond_block, .void_value);
6905
6906	var else_scope = parent_gz.makeSubBlock(&cond_scope.base);
6907	defer else_scope.unstack();
6908
6909	const else_node = for_full.ast.else_expr;
6910	if (else_node != 0) {
6911	const sub_scope = &else_scope.base;
6912	// Remove the continue block and break block so that `continue` and `break`
6913	// control flow apply to outer loops; not this one.
6914	loop_scope.continue_block = .none;
6915	loop_scope.break_block = .none;
6916	const else_result = try expr(&else_scope, sub_scope, loop_scope.break_result_info, else_node);
6917	if (is_statement) {
6918	_ = try addEnsureResult(&else_scope, else_result, else_node);
6919	}
6920	if (!else_scope.endsWithNoReturn()) {
6921	_ = try else_scope.addBreakWithSrcNode(break_tag, loop_block, else_result, else_node);
6922	}
6923	} else {
6924	const result = try rvalue(&else_scope, ri, .void_value, node);
6925	_ = try else_scope.addBreak(break_tag, loop_block, result);
6926	}
6927
6928	if (loop_scope.label) \|some\| {
6929	if (!some.used) {
6930	try astgen.appendErrorTok(some.token, "unused for loop label", .{});
6931	}
6932	}
6933
6934	try setCondBrPayload(condbr, cond, &then_scope, &else_scope);
6935
6936	// then_block and else_block unstacked now, can resurrect loop_scope to finally finish it
6937	{
6938	loop_scope.instructions_top = loop_scope.instructions.items.len;
6939	try loop_scope.instructions.appendSlice(gpa, &.{ index.toIndex().?, cond_block });
6940
6941	// Increment the index variable.
6942	const index_plus_one = try loop_scope.addPlNode(.add_unsafe, node, Zir.Inst.Bin{
6943	.lhs = index,
6944	.rhs = .one_usize,
6945	});
6946	_ = try loop_scope.addPlNode(.store_node, node, Zir.Inst.Bin{
6947	.lhs = index_ptr,
6948	.rhs = index_plus_one,
6949	});
6950	const repeat_tag: Zir.Inst.Tag = if (is_inline) .repeat_inline else .repeat;
6951	_ = try loop_scope.addNode(repeat_tag, node);
6952
6953	try loop_scope.setBlockBody(loop_block);
6954	}
6955
6956	const result = if (need_result_rvalue)
6957	try rvalue(parent_gz, ri, loop_block.toRef(), node)
6958	else
6959	loop_block.toRef();
6960
6961	if (is_statement) {
6962	_ = try parent_gz.addUnNode(.ensure_result_used, result, node);
6963	}
6964	return result;
6965	}
6966
6967	fn switchExprErrUnion(
6968	parent_gz: *GenZir,
6969	scope: *Scope,
6970	ri: ResultInfo,
6971	catch_or_if_node: Ast.Node.Index,
6972	node_ty: enum { @"catch", @"if" },
6973	) InnerError!Zir.Inst.Ref {
6974	const astgen = parent_gz.astgen;
6975	const gpa = astgen.gpa;
6976	const tree = astgen.tree;
6977	const node_datas = tree.nodes.items(.data);
6978	const node_tags = tree.nodes.items(.tag);
6979	const main_tokens = tree.nodes.items(.main_token);
6980	const token_tags = tree.tokens.items(.tag);
6981
6982	const if_full = switch (node_ty) {
6983	.@"catch" => undefined,
6984	.@"if" => tree.fullIf(catch_or_if_node).?,
6985	};
6986
6987	const switch_node, const operand_node, const error_payload = switch (node_ty) {
6988	.@"catch" => .{
6989	node_datas[catch_or_if_node].rhs,
6990	node_datas[catch_or_if_node].lhs,
6991	main_tokens[catch_or_if_node] + 2,
6992	},
6993	.@"if" => .{
6994	if_full.ast.else_expr,
6995	if_full.ast.cond_expr,
6996	if_full.error_token.?,
6997	},
6998	};
6999	assert(node_tags[switch_node] == .@"switch" or node_tags[switch_node] == .switch_comma);
7000
7001	const do_err_trace = astgen.fn_block != null;
7002
7003	const extra = tree.extraData(node_datas[switch_node].rhs, Ast.Node.SubRange);
7004	const case_nodes = tree.extra_data[extra.start..extra.end];
7005
7006	const need_rl = astgen.nodes_need_rl.contains(catch_or_if_node);
7007	const block_ri: ResultInfo = if (need_rl) ri else .{
7008	.rl = switch (ri.rl) {
7009	.ptr => .{ .ty = (try ri.rl.resultType(parent_gz, catch_or_if_node)).? },
7010	.inferred_ptr => .none,
7011	else => ri.rl,
7012	},
7013	.ctx = ri.ctx,
7014	};
7015
7016	const payload_is_ref = node_ty == .@"if" and
7017	if_full.payload_token != null and token_tags[if_full.payload_token.?] == .asterisk;
7018
7019	// We need to call `rvalue` to write through to the pointer only if we had a
7020	// result pointer and aren't forwarding it.
7021	const LocTag = @typeInfo(ResultInfo.Loc).Union.tag_type.?;
7022	const need_result_rvalue = @as(LocTag, block_ri.rl) != @as(LocTag, ri.rl);
7023	var scalar_cases_len: u32 = 0;
7024	var multi_cases_len: u32 = 0;
7025	var inline_cases_len: u32 = 0;
7026	var has_else = false;
7027	var else_node: Ast.Node.Index = 0;
7028	var else_src: ?Ast.TokenIndex = null;
7029	for (case_nodes) \|case_node\| {
7030	const case = tree.fullSwitchCase(case_node).?;
7031
7032	if (case.ast.values.len == 0) {
7033	const case_src = case.ast.arrow_token - 1;
7034	if (else_src) \|src\| {
7035	return astgen.failTokNotes(
7036	case_src,
7037	"multiple else prongs in switch expression",
7038	.{},
7039	&[_]u32{
7040	try astgen.errNoteTok(
7041	src,
7042	"previous else prong here",
7043	.{},
7044	),
7045	},
7046	);
7047	}
7048	has_else = true;
7049	else_node = case_node;
7050	else_src = case_src;
7051	continue;
7052	} else if (case.ast.values.len == 1 and
7053	node_tags[case.ast.values[0]] == .identifier and
7054	mem.eql(u8, tree.tokenSlice(main_tokens[case.ast.values[0]]), "_"))
7055	{
7056	const case_src = case.ast.arrow_token - 1;
7057	return astgen.failTokNotes(
7058	case_src,
7059	"'_' prong is not allowed when switching on errors",
7060	.{},
7061	&[_]u32{
7062	try astgen.errNoteTok(
7063	case_src,
7064	"consider using 'else'",
7065	.{},
7066	),
7067	},
7068	);
7069	}
7070
7071	for (case.ast.values) \|val\| {
7072	if (node_tags[val] == .string_literal)
7073	return astgen.failNode(val, "cannot switch on strings", .{});
7074	}
7075
7076	if (case.ast.values.len == 1 and node_tags[case.ast.values[0]] != .switch_range) {
7077	scalar_cases_len += 1;
7078	} else {
7079	multi_cases_len += 1;
7080	}
7081	if (case.inline_token != null) {
7082	inline_cases_len += 1;
7083	}
7084	}
7085
7086	const operand_ri: ResultInfo = .{
7087	.rl = if (payload_is_ref) .ref else .none,
7088	.ctx = .error_handling_expr,
7089	};
7090
7091	astgen.advanceSourceCursorToNode(operand_node);
7092	const operand_lc = LineColumn{ astgen.source_line - parent_gz.decl_line, astgen.source_column };
7093
7094	const raw_operand = try reachableExpr(parent_gz, scope, operand_ri, operand_node, switch_node);
7095	const item_ri: ResultInfo = .{ .rl = .none };
7096
7097	// This contains the data that goes into the `extra` array for the SwitchBlockErrUnion, except
7098	// the first cases_nodes.len slots are a table that indexes payloads later in the array,
7099	// with the non-error and else case indices coming first, then scalar_cases_len indexes, then
7100	// multi_cases_len indexes
7101	const payloads = &astgen.scratch;
7102	const scratch_top = astgen.scratch.items.len;
7103	const case_table_start = scratch_top;
7104	const scalar_case_table = case_table_start + 1 + @intFromBool(has_else);
7105	const multi_case_table = scalar_case_table + scalar_cases_len;
7106	const case_table_end = multi_case_table + multi_cases_len;
7107
7108	try astgen.scratch.resize(gpa, case_table_end);
7109	defer astgen.scratch.items.len = scratch_top;
7110
7111	var block_scope = parent_gz.makeSubBlock(scope);
7112	// block_scope not used for collecting instructions
7113	block_scope.instructions_top = GenZir.unstacked_top;
7114	block_scope.setBreakResultInfo(block_ri);
7115
7116	// Sema expects a dbg_stmt immediately before switch_block_err_union
7117	try emitDbgStmtForceCurrentIndex(parent_gz, operand_lc);
7118	// This gets added to the parent block later, after the item expressions.
7119	const switch_block = try parent_gz.makeBlockInst(.switch_block_err_union, switch_node);
7120
7121	// We re-use this same scope for all cases, including the special prong, if any.
7122	var case_scope = parent_gz.makeSubBlock(&block_scope.base);
7123	case_scope.instructions_top = GenZir.unstacked_top;
7124
7125	{
7126	const body_len_index: u32 = @intCast(payloads.items.len);
7127	payloads.items[case_table_start] = body_len_index;
7128	try payloads.resize(gpa, body_len_index + 1); // body_len
7129
7130	case_scope.instructions_top = parent_gz.instructions.items.len;
7131	defer case_scope.unstack();
7132
7133	const unwrap_payload_tag: Zir.Inst.Tag = if (payload_is_ref)
7134	.err_union_payload_unsafe_ptr
7135	else
7136	.err_union_payload_unsafe;
7137
7138	const unwrapped_payload = try case_scope.addUnNode(
7139	unwrap_payload_tag,
7140	raw_operand,
7141	catch_or_if_node,
7142	);
7143
7144	switch (node_ty) {
7145	.@"catch" => {
7146	const case_result = switch (ri.rl) {
7147	.ref, .ref_coerced_ty => unwrapped_payload,
7148	else => try rvalue(
7149	&case_scope,
7150	block_scope.break_result_info,
7151	unwrapped_payload,
7152	catch_or_if_node,
7153	),
7154	};
7155	_ = try case_scope.addBreakWithSrcNode(
7156	.@"break",
7157	switch_block,
7158	case_result,
7159	catch_or_if_node,
7160	);
7161	},
7162	.@"if" => {
7163	var payload_val_scope: Scope.LocalVal = undefined;
7164
7165	const then_node = if_full.ast.then_expr;
7166	const then_sub_scope = s: {
7167	assert(if_full.error_token != null);
7168	if (if_full.payload_token) \|payload_token\| {
7169	const token_name_index = payload_token + @intFromBool(payload_is_ref);
7170	const ident_name = try astgen.identAsString(token_name_index);
7171	const token_name_str = tree.tokenSlice(token_name_index);
7172	if (mem.eql(u8, "_", token_name_str))
7173	break :s &case_scope.base;
7174	try astgen.detectLocalShadowing(
7175	&case_scope.base,
7176	ident_name,
7177	token_name_index,
7178	token_name_str,
7179	.capture,
7180	);
7181	payload_val_scope = .{
7182	.parent = &case_scope.base,
7183	.gen_zir = &case_scope,
7184	.name = ident_name,
7185	.inst = unwrapped_payload,
7186	.token_src = token_name_index,
7187	.id_cat = .capture,
7188	};
7189	try case_scope.addDbgVar(.dbg_var_val, ident_name, unwrapped_payload);
7190	break :s &payload_val_scope.base;
7191	} else {
7192	_ = try case_scope.addUnNode(
7193	.ensure_err_union_payload_void,
7194	raw_operand,
7195	catch_or_if_node,
7196	);
7197	break :s &case_scope.base;
7198	}
7199	};
7200	const then_result = try expr(
7201	&case_scope,
7202	then_sub_scope,
7203	block_scope.break_result_info,
7204	then_node,
7205	);
7206	try checkUsed(parent_gz, &case_scope.base, then_sub_scope);
7207	if (!case_scope.endsWithNoReturn()) {
7208	_ = try case_scope.addBreakWithSrcNode(
7209	.@"break",
7210	switch_block,
7211	then_result,
7212	then_node,
7213	);
7214	}
7215	},
7216	}
7217
7218	const case_slice = case_scope.instructionsSlice();
7219	// Since we use the switch_block_err_union instruction itself to refer
7220	// to the capture, which will not be added to the child block, we need
7221	// to handle ref_table manually.
7222	const refs_len = refs: {
7223	var n: usize = 0;
7224	var check_inst = switch_block;
7225	while (astgen.ref_table.get(check_inst)) \|ref_inst\| {
7226	n += 1;
7227	check_inst = ref_inst;
7228	}
7229	break :refs n;
7230	};
7231	const body_len = refs_len + astgen.countBodyLenAfterFixups(case_slice);
7232	try payloads.ensureUnusedCapacity(gpa, body_len);
7233	const capture: Zir.Inst.SwitchBlock.ProngInfo.Capture = switch (node_ty) {
7234	.@"catch" => .none,
7235	.@"if" => if (if_full.payload_token == null)
7236	.none
7237	else if (payload_is_ref)
7238	.by_ref
7239	else
7240	.by_val,
7241	};
7242	payloads.items[body_len_index] = @bitCast(Zir.Inst.SwitchBlock.ProngInfo{
7243	.body_len = @intCast(body_len),
7244	.capture = capture,
7245	.is_inline = false,
7246	.has_tag_capture = false,
7247	});
7248	if (astgen.ref_table.fetchRemove(switch_block)) \|kv\| {
7249	appendPossiblyRefdBodyInst(astgen, payloads, kv.value);
7250	}
7251	appendBodyWithFixupsArrayList(astgen, payloads, case_slice);
7252	}
7253
7254	const err_name = blk: {
7255	const err_str = tree.tokenSlice(error_payload);
7256	if (mem.eql(u8, err_str, "_")) {
7257	return astgen.failTok(error_payload, "discard of error capture; omit it instead", .{});
7258	}
7259	const err_name = try astgen.identAsString(error_payload);
7260	try astgen.detectLocalShadowing(scope, err_name, error_payload, err_str, .capture);
7261
7262	break :blk err_name;
7263	};
7264
7265	// allocate a shared dummy instruction for the error capture
7266	const err_inst = err_inst: {
7267	const inst: Zir.Inst.Index = @enumFromInt(astgen.instructions.len);
7268	try astgen.instructions.append(astgen.gpa, .{
7269	.tag = .extended,
7270	.data = .{ .extended = .{
7271	.opcode = .value_placeholder,
7272	.small = undefined,
7273	.operand = undefined,
7274	} },
7275	});
7276	break :err_inst inst;
7277	};
7278
7279	// In this pass we generate all the item and prong expressions for error cases.
7280	var multi_case_index: u32 = 0;
7281	var scalar_case_index: u32 = 0;
7282	var any_uses_err_capture = false;
7283	for (case_nodes) \|case_node\| {
7284	const case = tree.fullSwitchCase(case_node).?;
7285
7286	const is_multi_case = case.ast.values.len > 1 or
7287	(case.ast.values.len == 1 and node_tags[case.ast.values[0]] == .switch_range);
7288
7289	var dbg_var_name: Zir.NullTerminatedString = .empty;
7290	var dbg_var_inst: Zir.Inst.Ref = undefined;
7291	var err_scope: Scope.LocalVal = undefined;
7292	var capture_scope: Scope.LocalVal = undefined;
7293
7294	const sub_scope = blk: {
7295	err_scope = .{
7296	.parent = &case_scope.base,
7297	.gen_zir = &case_scope,
7298	.name = err_name,
7299	.inst = err_inst.toRef(),
7300	.token_src = error_payload,
7301	.id_cat = .capture,
7302	};
7303
7304	const capture_token = case.payload_token orelse break :blk &err_scope.base;
7305	if (token_tags[capture_token] != .identifier) {
7306	return astgen.failTok(capture_token + 1, "error set cannot be captured by reference", .{});
7307	}
7308
7309	const capture_slice = tree.tokenSlice(capture_token);
7310	if (mem.eql(u8, capture_slice, "_")) {
7311	return astgen.failTok(capture_token, "discard of error capture; omit it instead", .{});
7312	}
7313	const tag_name = try astgen.identAsString(capture_token);
7314	try astgen.detectLocalShadowing(&case_scope.base, tag_name, capture_token, capture_slice, .capture);
7315
7316	capture_scope = .{
7317	.parent = &case_scope.base,
7318	.gen_zir = &case_scope,
7319	.name = tag_name,
7320	.inst = switch_block.toRef(),
7321	.token_src = capture_token,
7322	.id_cat = .capture,
7323	};
7324	dbg_var_name = tag_name;
7325	dbg_var_inst = switch_block.toRef();
7326
7327	err_scope.parent = &capture_scope.base;
7328
7329	break :blk &err_scope.base;
7330	};
7331
7332	const header_index: u32 = @intCast(payloads.items.len);
7333	const body_len_index = if (is_multi_case) blk: {
7334	payloads.items[multi_case_table + multi_case_index] = header_index;
7335	multi_case_index += 1;
7336	try payloads.resize(gpa, header_index + 3); // items_len, ranges_len, body_len
7337
7338	// items
7339	var items_len: u32 = 0;
7340	for (case.ast.values) \|item_node\| {
7341	if (node_tags[item_node] == .switch_range) continue;
7342	items_len += 1;
7343
7344	const item_inst = try comptimeExpr(parent_gz, scope, item_ri, item_node);
7345	try payloads.append(gpa, @intFromEnum(item_inst));
7346	}
7347
7348	// ranges
7349	var ranges_len: u32 = 0;
7350	for (case.ast.values) \|range\| {
7351	if (node_tags[range] != .switch_range) continue;
7352	ranges_len += 1;
7353
7354	const first = try comptimeExpr(parent_gz, scope, item_ri, node_datas[range].lhs);
7355	const last = try comptimeExpr(parent_gz, scope, item_ri, node_datas[range].rhs);
7356	try payloads.appendSlice(gpa, &[_]u32{
7357	@intFromEnum(first), @intFromEnum(last),
7358	});
7359	}
7360
7361	payloads.items[header_index] = items_len;
7362	payloads.items[header_index + 1] = ranges_len;
7363	break :blk header_index + 2;
7364	} else if (case_node == else_node) blk: {
7365	payloads.items[case_table_start + 1] = header_index;
7366	try payloads.resize(gpa, header_index + 1); // body_len
7367	break :blk header_index;
7368	} else blk: {
7369	payloads.items[scalar_case_table + scalar_case_index] = header_index;
7370	scalar_case_index += 1;
7371	try payloads.resize(gpa, header_index + 2); // item, body_len
7372	const item_node = case.ast.values[0];
7373	const item_inst = try comptimeExpr(parent_gz, scope, item_ri, item_node);
7374	payloads.items[header_index] = @intFromEnum(item_inst);
7375	break :blk header_index + 1;
7376	};
7377
7378	{
7379	// temporarily stack case_scope on parent_gz
7380	case_scope.instructions_top = parent_gz.instructions.items.len;
7381	defer case_scope.unstack();
7382
7383	if (do_err_trace and nodeMayAppendToErrorTrace(tree, operand_node))
7384	_ = try case_scope.addSaveErrRetIndex(.always);
7385
7386	if (dbg_var_name != .empty) {
7387	try case_scope.addDbgVar(.dbg_var_val, dbg_var_name, dbg_var_inst);
7388	}
7389
7390	const target_expr_node = case.ast.target_expr;
7391	const case_result = try expr(&case_scope, sub_scope, block_scope.break_result_info, target_expr_node);
7392	// check capture_scope, not err_scope to avoid false positive unused error capture
7393	try checkUsed(parent_gz, &case_scope.base, err_scope.parent);
7394	const uses_err = err_scope.used != 0 or err_scope.discarded != 0;
7395	if (uses_err) {
7396	try case_scope.addDbgVar(.dbg_var_val, err_name, err_inst.toRef());
7397	any_uses_err_capture = true;
7398	}
7399
7400	if (!parent_gz.refIsNoReturn(case_result)) {
7401	if (do_err_trace)
7402	try restoreErrRetIndex(
7403	&case_scope,
7404	.{ .block = switch_block },
7405	block_scope.break_result_info,
7406	target_expr_node,
7407	case_result,
7408	);
7409
7410	_ = try case_scope.addBreakWithSrcNode(.@"break", switch_block, case_result, target_expr_node);
7411	}
7412
7413	const case_slice = case_scope.instructionsSlice();
7414	// Since we use the switch_block_err_union instruction itself to refer
7415	// to the capture, which will not be added to the child block, we need
7416	// to handle ref_table manually.
7417	const refs_len = refs: {
7418	var n: usize = 0;
7419	var check_inst = switch_block;
7420	while (astgen.ref_table.get(check_inst)) \|ref_inst\| {
7421	n += 1;
7422	check_inst = ref_inst;
7423	}
7424	if (uses_err) {
7425	check_inst = err_inst;
7426	while (astgen.ref_table.get(check_inst)) \|ref_inst\| {
7427	n += 1;
7428	check_inst = ref_inst;
7429	}
7430	}
7431	break :refs n;
7432	};
7433	const body_len = refs_len + astgen.countBodyLenAfterFixups(case_slice);
7434	try payloads.ensureUnusedCapacity(gpa, body_len);
7435	payloads.items[body_len_index] = @bitCast(Zir.Inst.SwitchBlock.ProngInfo{
7436	.body_len = @intCast(body_len),
7437	.capture = if (case.payload_token != null) .by_val else .none,
7438	.is_inline = case.inline_token != null,
7439	.has_tag_capture = false,
7440	});
7441	if (astgen.ref_table.fetchRemove(switch_block)) \|kv\| {
7442	appendPossiblyRefdBodyInst(astgen, payloads, kv.value);
7443	}
7444	if (uses_err) {
7445	if (astgen.ref_table.fetchRemove(err_inst)) \|kv\| {
7446	appendPossiblyRefdBodyInst(astgen, payloads, kv.value);
7447	}
7448	}
7449	appendBodyWithFixupsArrayList(astgen, payloads, case_slice);
7450	}
7451	}
7452	// Now that the item expressions are generated we can add this.
7453	try parent_gz.instructions.append(gpa, switch_block);
7454
7455	try astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.SwitchBlockErrUnion).Struct.fields.len +
7456	@intFromBool(multi_cases_len != 0) +
7457	payloads.items.len - case_table_end +
7458	(case_table_end - case_table_start) * @typeInfo(Zir.Inst.As).Struct.fields.len);
7459
7460	const payload_index = astgen.addExtraAssumeCapacity(Zir.Inst.SwitchBlockErrUnion{
7461	.operand = raw_operand,
7462	.bits = Zir.Inst.SwitchBlockErrUnion.Bits{
7463	.has_multi_cases = multi_cases_len != 0,
7464	.has_else = has_else,
7465	.scalar_cases_len = @intCast(scalar_cases_len),
7466	.any_uses_err_capture = any_uses_err_capture,
7467	.payload_is_ref = payload_is_ref,
7468	},
7469	.main_src_node_offset = parent_gz.nodeIndexToRelative(catch_or_if_node),
7470	});
7471
7472	if (multi_cases_len != 0) {
7473	astgen.extra.appendAssumeCapacity(multi_cases_len);
7474	}
7475
7476	if (any_uses_err_capture) {
7477	astgen.extra.appendAssumeCapacity(@intFromEnum(err_inst));
7478	}
7479
7480	const zir_datas = astgen.instructions.items(.data);
7481	zir_datas[@intFromEnum(switch_block)].pl_node.payload_index = payload_index;
7482
7483	for (payloads.items[case_table_start..case_table_end], 0..) \|start_index, i\| {
7484	var body_len_index = start_index;
7485	var end_index = start_index;
7486	const table_index = case_table_start + i;
7487	if (table_index < scalar_case_table) {
7488	end_index += 1;
7489	} else if (table_index < multi_case_table) {
7490	body_len_index += 1;
7491	end_index += 2;
7492	} else {
7493	body_len_index += 2;
7494	const items_len = payloads.items[start_index];
7495	const ranges_len = payloads.items[start_index + 1];
7496	end_index += 3 + items_len + 2 * ranges_len;
7497	}
7498	const prong_info: Zir.Inst.SwitchBlock.ProngInfo = @bitCast(payloads.items[body_len_index]);
7499	end_index += prong_info.body_len;
7500	astgen.extra.appendSliceAssumeCapacity(payloads.items[start_index..end_index]);
7501	}
7502
7503	if (need_result_rvalue) {
7504	return rvalue(parent_gz, ri, switch_block.toRef(), switch_node);
7505	} else {
7506	return switch_block.toRef();
7507	}
7508	}
7509
7510	fn switchExpr(
7511	parent_gz: *GenZir,
7512	scope: *Scope,
7513	ri: ResultInfo,
7514	switch_node: Ast.Node.Index,
7515	) InnerError!Zir.Inst.Ref {
7516	const astgen = parent_gz.astgen;
7517	const gpa = astgen.gpa;
7518	const tree = astgen.tree;
7519	const node_datas = tree.nodes.items(.data);
7520	const node_tags = tree.nodes.items(.tag);
7521	const main_tokens = tree.nodes.items(.main_token);
7522	const token_tags = tree.tokens.items(.tag);
7523	const operand_node = node_datas[switch_node].lhs;
7524	const extra = tree.extraData(node_datas[switch_node].rhs, Ast.Node.SubRange);
7525	const case_nodes = tree.extra_data[extra.start..extra.end];
7526
7527	const need_rl = astgen.nodes_need_rl.contains(switch_node);
7528	const block_ri: ResultInfo = if (need_rl) ri else .{
7529	.rl = switch (ri.rl) {
7530	.ptr => .{ .ty = (try ri.rl.resultType(parent_gz, switch_node)).? },
7531	.inferred_ptr => .none,
7532	else => ri.rl,
7533	},
7534	.ctx = ri.ctx,
7535	};
7536	// We need to call `rvalue` to write through to the pointer only if we had a
7537	// result pointer and aren't forwarding it.
7538	const LocTag = @typeInfo(ResultInfo.Loc).Union.tag_type.?;
7539	const need_result_rvalue = @as(LocTag, block_ri.rl) != @as(LocTag, ri.rl);
7540
7541	// We perform two passes over the AST. This first pass is to collect information
7542	// for the following variables, make note of the special prong AST node index,
7543	// and bail out with a compile error if there are multiple special prongs present.
7544	var any_payload_is_ref = false;
7545	var any_has_tag_capture = false;
7546	var scalar_cases_len: u32 = 0;
7547	var multi_cases_len: u32 = 0;
7548	var inline_cases_len: u32 = 0;
7549	var special_prong: Zir.SpecialProng = .none;
7550	var special_node: Ast.Node.Index = 0;
7551	var else_src: ?Ast.TokenIndex = null;
7552	var underscore_src: ?Ast.TokenIndex = null;
7553	for (case_nodes) \|case_node\| {
7554	const case = tree.fullSwitchCase(case_node).?;
7555	if (case.payload_token) \|payload_token\| {
7556	const ident = if (token_tags[payload_token] == .asterisk) blk: {
7557	any_payload_is_ref = true;
7558	break :blk payload_token + 1;
7559	} else payload_token;
7560	if (token_tags[ident + 1] == .comma) {
7561	any_has_tag_capture = true;
7562	}
7563	}
7564	// Check for else/`_` prong.
7565	if (case.ast.values.len == 0) {
7566	const case_src = case.ast.arrow_token - 1;
7567	if (else_src) \|src\| {
7568	return astgen.failTokNotes(
7569	case_src,
7570	"multiple else prongs in switch expression",
7571	.{},
7572	&[_]u32{
7573	try astgen.errNoteTok(
7574	src,
7575	"previous else prong here",
7576	.{},
7577	),
7578	},
7579	);
7580	} else if (underscore_src) \|some_underscore\| {
7581	return astgen.failNodeNotes(
7582	switch_node,
7583	"else and '_' prong in switch expression",
7584	.{},
7585	&[_]u32{
7586	try astgen.errNoteTok(
7587	case_src,
7588	"else prong here",
7589	.{},
7590	),
7591	try astgen.errNoteTok(
7592	some_underscore,
7593	"'_' prong here",
7594	.{},
7595	),
7596	},
7597	);
7598	}
7599	special_node = case_node;
7600	special_prong = .@"else";
7601	else_src = case_src;
7602	continue;
7603	} else if (case.ast.values.len == 1 and
7604	node_tags[case.ast.values[0]] == .identifier and
7605	mem.eql(u8, tree.tokenSlice(main_tokens[case.ast.values[0]]), "_"))
7606	{
7607	const case_src = case.ast.arrow_token - 1;
7608	if (underscore_src) \|src\| {
7609	return astgen.failTokNotes(
7610	case_src,
7611	"multiple '_' prongs in switch expression",
7612	.{},
7613	&[_]u32{
7614	try astgen.errNoteTok(
7615	src,
7616	"previous '_' prong here",
7617	.{},
7618	),
7619	},
7620	);
7621	} else if (else_src) \|some_else\| {
7622	return astgen.failNodeNotes(
7623	switch_node,
7624	"else and '_' prong in switch expression",
7625	.{},
7626	&[_]u32{
7627	try astgen.errNoteTok(
7628	some_else,
7629	"else prong here",
7630	.{},
7631	),
7632	try astgen.errNoteTok(
7633	case_src,
7634	"'_' prong here",
7635	.{},
7636	),
7637	},
7638	);
7639	}
7640	if (case.inline_token != null) {
7641	return astgen.failTok(case_src, "cannot inline '_' prong", .{});
7642	}
7643	special_node = case_node;
7644	special_prong = .under;
7645	underscore_src = case_src;
7646	continue;
7647	}
7648
7649	for (case.ast.values) \|val\| {
7650	if (node_tags[val] == .string_literal)
7651	return astgen.failNode(val, "cannot switch on strings", .{});
7652	}
7653
7654	if (case.ast.values.len == 1 and node_tags[case.ast.values[0]] != .switch_range) {
7655	scalar_cases_len += 1;
7656	} else {
7657	multi_cases_len += 1;
7658	}
7659	if (case.inline_token != null) {
7660	inline_cases_len += 1;
7661	}
7662	}
7663
7664	const operand_ri: ResultInfo = .{ .rl = if (any_payload_is_ref) .ref else .none };
7665
7666	astgen.advanceSourceCursorToNode(operand_node);
7667	const operand_lc = LineColumn{ astgen.source_line - parent_gz.decl_line, astgen.source_column };
7668
7669	const raw_operand = try expr(parent_gz, scope, operand_ri, operand_node);
7670	const item_ri: ResultInfo = .{ .rl = .none };
7671
7672	// This contains the data that goes into the `extra` array for the SwitchBlock/SwitchBlockMulti,
7673	// except the first cases_nodes.len slots are a table that indexes payloads later in the array, with
7674	// the special case index coming first, then scalar_case_len indexes, then multi_cases_len indexes
7675	const payloads = &astgen.scratch;
7676	const scratch_top = astgen.scratch.items.len;
7677	const case_table_start = scratch_top;
7678	const scalar_case_table = case_table_start + @intFromBool(special_prong != .none);
7679	const multi_case_table = scalar_case_table + scalar_cases_len;
7680	const case_table_end = multi_case_table + multi_cases_len;
7681	try astgen.scratch.resize(gpa, case_table_end);
7682	defer astgen.scratch.items.len = scratch_top;
7683
7684	var block_scope = parent_gz.makeSubBlock(scope);
7685	// block_scope not used for collecting instructions
7686	block_scope.instructions_top = GenZir.unstacked_top;
7687	block_scope.setBreakResultInfo(block_ri);
7688
7689	// Sema expects a dbg_stmt immediately before switch_block(_ref)
7690	try emitDbgStmtForceCurrentIndex(parent_gz, operand_lc);
7691	// This gets added to the parent block later, after the item expressions.
7692	const switch_tag: Zir.Inst.Tag = if (any_payload_is_ref) .switch_block_ref else .switch_block;
7693	const switch_block = try parent_gz.makeBlockInst(switch_tag, switch_node);
7694
7695	// We re-use this same scope for all cases, including the special prong, if any.
7696	var case_scope = parent_gz.makeSubBlock(&block_scope.base);
7697	case_scope.instructions_top = GenZir.unstacked_top;
7698
7699	// If any prong has an inline tag capture, allocate a shared dummy instruction for it
7700	const tag_inst = if (any_has_tag_capture) tag_inst: {
7701	const inst: Zir.Inst.Index = @enumFromInt(astgen.instructions.len);
7702	try astgen.instructions.append(astgen.gpa, .{
7703	.tag = .extended,
7704	.data = .{ .extended = .{
7705	.opcode = .value_placeholder,
7706	.small = undefined,
7707	.operand = undefined,
7708	} },
7709	});
7710	break :tag_inst inst;
7711	} else undefined;
7712
7713	// In this pass we generate all the item and prong expressions.
7714	var multi_case_index: u32 = 0;
7715	var scalar_case_index: u32 = 0;
7716	for (case_nodes) \|case_node\| {
7717	const case = tree.fullSwitchCase(case_node).?;
7718
7719	const is_multi_case = case.ast.values.len > 1 or
7720	(case.ast.values.len == 1 and node_tags[case.ast.values[0]] == .switch_range);
7721
7722	var dbg_var_name: Zir.NullTerminatedString = .empty;
7723	var dbg_var_inst: Zir.Inst.Ref = undefined;
7724	var dbg_var_tag_name: Zir.NullTerminatedString = .empty;
7725	var dbg_var_tag_inst: Zir.Inst.Ref = undefined;
7726	var has_tag_capture = false;
7727	var capture_val_scope: Scope.LocalVal = undefined;
7728	var tag_scope: Scope.LocalVal = undefined;
7729
7730	var capture: Zir.Inst.SwitchBlock.ProngInfo.Capture = .none;
7731
7732	const sub_scope = blk: {
7733	const payload_token = case.payload_token orelse break :blk &case_scope.base;
7734	const ident = if (token_tags[payload_token] == .asterisk)
7735	payload_token + 1
7736	else
7737	payload_token;
7738
7739	const is_ptr = ident != payload_token;
7740	capture = if (is_ptr) .by_ref else .by_val;
7741
7742	const ident_slice = tree.tokenSlice(ident);
7743	var payload_sub_scope: *Scope = undefined;
7744	if (mem.eql(u8, ident_slice, "_")) {
7745	if (is_ptr) {
7746	return astgen.failTok(payload_token, "pointer modifier invalid on discard", .{});
7747	}
7748	payload_sub_scope = &case_scope.base;
7749	} else {
7750	const capture_name = try astgen.identAsString(ident);
7751	try astgen.detectLocalShadowing(&case_scope.base, capture_name, ident, ident_slice, .capture);
7752	capture_val_scope = .{
7753	.parent = &case_scope.base,
7754	.gen_zir = &case_scope,
7755	.name = capture_name,
7756	.inst = switch_block.toRef(),
7757	.token_src = ident,
7758	.id_cat = .capture,
7759	};
7760	dbg_var_name = capture_name;
7761	dbg_var_inst = switch_block.toRef();
7762	payload_sub_scope = &capture_val_scope.base;
7763	}
7764
7765	const tag_token = if (token_tags[ident + 1] == .comma)
7766	ident + 2
7767	else
7768	break :blk payload_sub_scope;
7769	const tag_slice = tree.tokenSlice(tag_token);
7770	if (mem.eql(u8, tag_slice, "_")) {
7771	return astgen.failTok(tag_token, "discard of tag capture; omit it instead", .{});
7772	} else if (case.inline_token == null) {
7773	return astgen.failTok(tag_token, "tag capture on non-inline prong", .{});
7774	}
7775	const tag_name = try astgen.identAsString(tag_token);
7776	try astgen.detectLocalShadowing(payload_sub_scope, tag_name, tag_token, tag_slice, .@"switch tag capture");
7777
7778	assert(any_has_tag_capture);
7779	has_tag_capture = true;
7780
7781	tag_scope = .{
7782	.parent = payload_sub_scope,
7783	.gen_zir = &case_scope,
7784	.name = tag_name,
7785	.inst = tag_inst.toRef(),
7786	.token_src = tag_token,
7787	.id_cat = .@"switch tag capture",
7788	};
7789	dbg_var_tag_name = tag_name;
7790	dbg_var_tag_inst = tag_inst.toRef();
7791	break :blk &tag_scope.base;
7792	};
7793
7794	const header_index: u32 = @intCast(payloads.items.len);
7795	const body_len_index = if (is_multi_case) blk: {
7796	payloads.items[multi_case_table + multi_case_index] = header_index;
7797	multi_case_index += 1;
7798	try payloads.resize(gpa, header_index + 3); // items_len, ranges_len, body_len
7799
7800	// items
7801	var items_len: u32 = 0;
7802	for (case.ast.values) \|item_node\| {
7803	if (node_tags[item_node] == .switch_range) continue;
7804	items_len += 1;
7805
7806	const item_inst = try comptimeExpr(parent_gz, scope, item_ri, item_node);
7807	try payloads.append(gpa, @intFromEnum(item_inst));
7808	}
7809
7810	// ranges
7811	var ranges_len: u32 = 0;
7812	for (case.ast.values) \|range\| {
7813	if (node_tags[range] != .switch_range) continue;
7814	ranges_len += 1;
7815
7816	const first = try comptimeExpr(parent_gz, scope, item_ri, node_datas[range].lhs);
7817	const last = try comptimeExpr(parent_gz, scope, item_ri, node_datas[range].rhs);
7818	try payloads.appendSlice(gpa, &[_]u32{
7819	@intFromEnum(first), @intFromEnum(last),
7820	});
7821	}
7822
7823	payloads.items[header_index] = items_len;
7824	payloads.items[header_index + 1] = ranges_len;
7825	break :blk header_index + 2;
7826	} else if (case_node == special_node) blk: {
7827	payloads.items[case_table_start] = header_index;
7828	try payloads.resize(gpa, header_index + 1); // body_len
7829	break :blk header_index;
7830	} else blk: {
7831	payloads.items[scalar_case_table + scalar_case_index] = header_index;
7832	scalar_case_index += 1;
7833	try payloads.resize(gpa, header_index + 2); // item, body_len
7834	const item_node = case.ast.values[0];
7835	const item_inst = try comptimeExpr(parent_gz, scope, item_ri, item_node);
7836	payloads.items[header_index] = @intFromEnum(item_inst);
7837	break :blk header_index + 1;
7838	};
7839
7840	{
7841	// temporarily stack case_scope on parent_gz
7842	case_scope.instructions_top = parent_gz.instructions.items.len;
7843	defer case_scope.unstack();
7844
7845	if (dbg_var_name != .empty) {
7846	try case_scope.addDbgVar(.dbg_var_val, dbg_var_name, dbg_var_inst);
7847	}
7848	if (dbg_var_tag_name != .empty) {
7849	try case_scope.addDbgVar(.dbg_var_val, dbg_var_tag_name, dbg_var_tag_inst);
7850	}
7851	const target_expr_node = case.ast.target_expr;
7852	const case_result = try expr(&case_scope, sub_scope, block_scope.break_result_info, target_expr_node);
7853	try checkUsed(parent_gz, &case_scope.base, sub_scope);
7854	if (!parent_gz.refIsNoReturn(case_result)) {
7855	_ = try case_scope.addBreakWithSrcNode(.@"break", switch_block, case_result, target_expr_node);
7856	}
7857
7858	const case_slice = case_scope.instructionsSlice();
7859	// Since we use the switch_block instruction itself to refer to the
7860	// capture, which will not be added to the child block, we need to
7861	// handle ref_table manually, and the same for the inline tag
7862	// capture instruction.
7863	const refs_len = refs: {
7864	var n: usize = 0;
7865	var check_inst = switch_block;
7866	while (astgen.ref_table.get(check_inst)) \|ref_inst\| {
7867	n += 1;
7868	check_inst = ref_inst;
7869	}
7870	if (has_tag_capture) {
7871	check_inst = tag_inst;
7872	while (astgen.ref_table.get(check_inst)) \|ref_inst\| {
7873	n += 1;
7874	check_inst = ref_inst;
7875	}
7876	}
7877	break :refs n;
7878	};
7879	const body_len = refs_len + astgen.countBodyLenAfterFixups(case_slice);
7880	try payloads.ensureUnusedCapacity(gpa, body_len);
7881	payloads.items[body_len_index] = @bitCast(Zir.Inst.SwitchBlock.ProngInfo{
7882	.body_len = @intCast(body_len),
7883	.capture = capture,
7884	.is_inline = case.inline_token != null,
7885	.has_tag_capture = has_tag_capture,
7886	});
7887	if (astgen.ref_table.fetchRemove(switch_block)) \|kv\| {
7888	appendPossiblyRefdBodyInst(astgen, payloads, kv.value);
7889	}
7890	if (has_tag_capture) {
7891	if (astgen.ref_table.fetchRemove(tag_inst)) \|kv\| {
7892	appendPossiblyRefdBodyInst(astgen, payloads, kv.value);
7893	}
7894	}
7895	appendBodyWithFixupsArrayList(astgen, payloads, case_slice);
7896	}
7897	}
7898	// Now that the item expressions are generated we can add this.
7899	try parent_gz.instructions.append(gpa, switch_block);
7900
7901	try astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.SwitchBlock).Struct.fields.len +
7902	@intFromBool(multi_cases_len != 0) +
7903	@intFromBool(any_has_tag_capture) +
7904	payloads.items.len - case_table_end +
7905	(case_table_end - case_table_start) * @typeInfo(Zir.Inst.As).Struct.fields.len);
7906
7907	const payload_index = astgen.addExtraAssumeCapacity(Zir.Inst.SwitchBlock{
7908	.operand = raw_operand,
7909	.bits = Zir.Inst.SwitchBlock.Bits{
7910	.has_multi_cases = multi_cases_len != 0,
7911	.has_else = special_prong == .@"else",
7912	.has_under = special_prong == .under,
7913	.any_has_tag_capture = any_has_tag_capture,
7914	.scalar_cases_len = @intCast(scalar_cases_len),
7915	},
7916	});
7917
7918	if (multi_cases_len != 0) {
7919	astgen.extra.appendAssumeCapacity(multi_cases_len);
7920	}
7921
7922	if (any_has_tag_capture) {
7923	astgen.extra.appendAssumeCapacity(@intFromEnum(tag_inst));
7924	}
7925
7926	const zir_datas = astgen.instructions.items(.data);
7927	zir_datas[@intFromEnum(switch_block)].pl_node.payload_index = payload_index;
7928
7929	for (payloads.items[case_table_start..case_table_end], 0..) \|start_index, i\| {
7930	var body_len_index = start_index;
7931	var end_index = start_index;
7932	const table_index = case_table_start + i;
7933	if (table_index < scalar_case_table) {
7934	end_index += 1;
7935	} else if (table_index < multi_case_table) {
7936	body_len_index += 1;
7937	end_index += 2;
7938	} else {
7939	body_len_index += 2;
7940	const items_len = payloads.items[start_index];
7941	const ranges_len = payloads.items[start_index + 1];
7942	end_index += 3 + items_len + 2 * ranges_len;
7943	}
7944	const prong_info: Zir.Inst.SwitchBlock.ProngInfo = @bitCast(payloads.items[body_len_index]);
7945	end_index += prong_info.body_len;
7946	astgen.extra.appendSliceAssumeCapacity(payloads.items[start_index..end_index]);
7947	}
7948
7949	if (need_result_rvalue) {
7950	return rvalue(parent_gz, ri, switch_block.toRef(), switch_node);
7951	} else {
7952	return switch_block.toRef();
7953	}
7954	}
7955
7956	fn ret(gz: GenZir, scope: Scope, node: Ast.Node.Index) InnerError!Zir.Inst.Ref {
7957	const astgen = gz.astgen;
7958	const tree = astgen.tree;
7959	const node_datas = tree.nodes.items(.data);
7960	const node_tags = tree.nodes.items(.tag);
7961
7962	if (astgen.fn_block == null) {
7963	return astgen.failNode(node, "'return' outside function scope", .{});
7964	}
7965
7966	if (gz.any_defer_node != 0) {
7967	return astgen.failNodeNotes(node, "cannot return from defer expression", .{}, &.{
7968	try astgen.errNoteNode(
7969	gz.any_defer_node,
7970	"defer expression here",
7971	.{},
7972	),
7973	});
7974	}
7975
7976	// Ensure debug line/column information is emitted for this return expression.
7977	// Then we will save the line/column so that we can emit another one that goes
7978	// "backwards" because we want to evaluate the operand, but then put the debug
7979	// info back at the return keyword for error return tracing.
7980	if (!gz.is_comptime) {
7981	try emitDbgNode(gz, node);
7982	}
7983	const ret_lc = LineColumn{ astgen.source_line - gz.decl_line, astgen.source_column };
7984
7985	const defer_outer = &astgen.fn_block.?.base;
7986
7987	const operand_node = node_datas[node].lhs;
7988	if (operand_node == 0) {
7989	// Returning a void value; skip error defers.
7990	try genDefers(gz, defer_outer, scope, .normal_only);
7991
7992	// As our last action before the return, "pop" the error trace if needed
7993	_ = try gz.addRestoreErrRetIndex(.ret, .always, node);
7994
7995	_ = try gz.addUnNode(.ret_node, .void_value, node);
7996	return Zir.Inst.Ref.unreachable_value;
7997	}
7998
7999	if (node_tags[operand_node] == .error_value) {
8000	// Hot path for `return error.Foo`. This bypasses result location logic as well as logic
8001	// for detecting whether to add something to the function's inferred error set.
8002	const ident_token = node_datas[operand_node].rhs;
8003	const err_name_str_index = try astgen.identAsString(ident_token);
8004	const defer_counts = countDefers(defer_outer, scope);
8005	if (!defer_counts.need_err_code) {
8006	try genDefers(gz, defer_outer, scope, .both_sans_err);
8007	try emitDbgStmt(gz, ret_lc);
8008	_ = try gz.addStrTok(.ret_err_value, err_name_str_index, ident_token);
8009	return Zir.Inst.Ref.unreachable_value;
8010	}
8011	const err_code = try gz.addStrTok(.ret_err_value_code, err_name_str_index, ident_token);
8012	try genDefers(gz, defer_outer, scope, .{ .both = err_code });
8013	try emitDbgStmt(gz, ret_lc);
8014	_ = try gz.addUnNode(.ret_node, err_code, node);
8015	return Zir.Inst.Ref.unreachable_value;
8016	}
8017
8018	const ri: ResultInfo = if (astgen.nodes_need_rl.contains(node)) .{
8019	.rl = .{ .ptr = .{ .inst = try gz.addNode(.ret_ptr, node) } },
8020	.ctx = .@"return",
8021	} else .{
8022	.rl = .{ .coerced_ty = astgen.fn_ret_ty },
8023	.ctx = .@"return",
8024	};
8025	const prev_anon_name_strategy = gz.anon_name_strategy;
8026	gz.anon_name_strategy = .func;
8027	const operand = try reachableExpr(gz, scope, ri, operand_node, node);
8028	gz.anon_name_strategy = prev_anon_name_strategy;
8029
8030	switch (nodeMayEvalToError(tree, operand_node)) {
8031	.never => {
8032	// Returning a value that cannot be an error; skip error defers.
8033	try genDefers(gz, defer_outer, scope, .normal_only);
8034
8035	// As our last action before the return, "pop" the error trace if needed
8036	_ = try gz.addRestoreErrRetIndex(.ret, .always, node);
8037
8038	try emitDbgStmt(gz, ret_lc);
8039	try gz.addRet(ri, operand, node);
8040	return Zir.Inst.Ref.unreachable_value;
8041	},
8042	.always => {
8043	// Value is always an error. Emit both error defers and regular defers.
8044	const err_code = if (ri.rl == .ptr) try gz.addUnNode(.load, ri.rl.ptr.inst, node) else operand;
8045	try genDefers(gz, defer_outer, scope, .{ .both = err_code });
8046	try emitDbgStmt(gz, ret_lc);
8047	try gz.addRet(ri, operand, node);
8048	return Zir.Inst.Ref.unreachable_value;
8049	},
8050	.maybe => {
8051	const defer_counts = countDefers(defer_outer, scope);
8052	if (!defer_counts.have_err) {
8053	// Only regular defers; no branch needed.
8054	try genDefers(gz, defer_outer, scope, .normal_only);
8055	try emitDbgStmt(gz, ret_lc);
8056
8057	// As our last action before the return, "pop" the error trace if needed
8058	const result = if (ri.rl == .ptr) try gz.addUnNode(.load, ri.rl.ptr.inst, node) else operand;
8059	_ = try gz.addRestoreErrRetIndex(.ret, .{ .if_non_error = result }, node);
8060
8061	try gz.addRet(ri, operand, node);
8062	return Zir.Inst.Ref.unreachable_value;
8063	}
8064
8065	// Emit conditional branch for generating errdefers.
8066	const result = if (ri.rl == .ptr) try gz.addUnNode(.load, ri.rl.ptr.inst, node) else operand;
8067	const is_non_err = try gz.addUnNode(.ret_is_non_err, result, node);
8068	const condbr = try gz.addCondBr(.condbr, node);
8069
8070	var then_scope = gz.makeSubBlock(scope);
8071	defer then_scope.unstack();
8072
8073	try genDefers(&then_scope, defer_outer, scope, .normal_only);
8074
8075	// As our last action before the return, "pop" the error trace if needed
8076	_ = try then_scope.addRestoreErrRetIndex(.ret, .always, node);
8077
8078	try emitDbgStmt(&then_scope, ret_lc);
8079	try then_scope.addRet(ri, operand, node);
8080
8081	var else_scope = gz.makeSubBlock(scope);
8082	defer else_scope.unstack();
8083
8084	const which_ones: DefersToEmit = if (!defer_counts.need_err_code) .both_sans_err else .{
8085	.both = try else_scope.addUnNode(.err_union_code, result, node),
8086	};
8087	try genDefers(&else_scope, defer_outer, scope, which_ones);
8088	try emitDbgStmt(&else_scope, ret_lc);
8089	try else_scope.addRet(ri, operand, node);
8090
8091	try setCondBrPayload(condbr, is_non_err, &then_scope, &else_scope);
8092
8093	return Zir.Inst.Ref.unreachable_value;
8094	},
8095	}
8096	}
8097
8098	/// Parses the string `buf` as a base 10 integer of type `u16`.
8099	///
8100	/// Unlike std.fmt.parseInt, does not allow the '_' character in `buf`.
8101	fn parseBitCount(buf: []const u8) std.fmt.ParseIntError!u16 {
8102	if (buf.len == 0) return error.InvalidCharacter;
8103
8104	var x: u16 = 0;
8105
8106	for (buf) \|c\| {
8107	const digit = switch (c) {
8108	'0'...'9' => c - '0',
8109	else => return error.InvalidCharacter,
8110	};
8111
8112	if (x != 0) x = try std.math.mul(u16, x, 10);
8113	x = try std.math.add(u16, x, digit);
8114	}
8115
8116	return x;
8117	}
8118
8119	fn identifier(
8120	gz: *GenZir,
8121	scope: *Scope,
8122	ri: ResultInfo,
8123	ident: Ast.Node.Index,
8124	) InnerError!Zir.Inst.Ref {
8125	const astgen = gz.astgen;
8126	const tree = astgen.tree;
8127	const main_tokens = tree.nodes.items(.main_token);
8128
8129	const ident_token = main_tokens[ident];
8130	const ident_name_raw = tree.tokenSlice(ident_token);
8131	if (mem.eql(u8, ident_name_raw, "_")) {
8132	return astgen.failNode(ident, "'_' used as an identifier without @\"_\" syntax", .{});
8133	}
8134
8135	// if not @"" syntax, just use raw token slice
8136	if (ident_name_raw[0] != '@') {
8137	if (primitive_instrs.get(ident_name_raw)) \|zir_const_ref\| {
8138	return rvalue(gz, ri, zir_const_ref, ident);
8139	}
8140
8141	if (ident_name_raw.len >= 2) integer: {
8142	const first_c = ident_name_raw[0];
8143	if (first_c == 'i' or first_c == 'u') {
8144	const signedness: std.builtin.Signedness = switch (first_c == 'i') {
8145	true => .signed,
8146	false => .unsigned,
8147	};
8148	if (ident_name_raw.len >= 3 and ident_name_raw[1] == '0') {
8149	return astgen.failNode(
8150	ident,
8151	"primitive integer type '{s}' has leading zero",
8152	.{ident_name_raw},
8153	);
8154	}
8155	const bit_count = parseBitCount(ident_name_raw[1..]) catch \|err\| switch (err) {
8156	error.Overflow => return astgen.failNode(
8157	ident,
8158	"primitive integer type '{s}' exceeds maximum bit width of 65535",
8159	.{ident_name_raw},
8160	),
8161	error.InvalidCharacter => break :integer,
8162	};
8163	const result = try gz.add(.{
8164	.tag = .int_type,
8165	.data = .{ .int_type = .{
8166	.src_node = gz.nodeIndexToRelative(ident),
8167	.signedness = signedness,
8168	.bit_count = bit_count,
8169	} },
8170	});
8171	return rvalue(gz, ri, result, ident);
8172	}
8173	}
8174	}
8175
8176	// Local variables, including function parameters.
8177	return localVarRef(gz, scope, ri, ident, ident_token);
8178	}
8179
8180	fn localVarRef(
8181	gz: *GenZir,
8182	scope: *Scope,
8183	ri: ResultInfo,
8184	ident: Ast.Node.Index,
8185	ident_token: Ast.TokenIndex,
8186	) InnerError!Zir.Inst.Ref {
8187	const astgen = gz.astgen;
8188	const gpa = astgen.gpa;
8189	const name_str_index = try astgen.identAsString(ident_token);
8190	var s = scope;
8191	var found_already: ?Ast.Node.Index = null; // we have found a decl with the same name already
8192	var num_namespaces_out: u32 = 0;
8193	var capturing_namespace: ?*Scope.Namespace = null;
8194	while (true) switch (s.tag) {
8195	.local_val => {
8196	const local_val = s.cast(Scope.LocalVal).?;
8197
8198	if (local_val.name == name_str_index) {
8199	// Locals cannot shadow anything, so we do not need to look for ambiguous
8200	// references in this case.
8201	if (ri.rl == .discard and ri.ctx == .assignment) {
8202	local_val.discarded = ident_token;
8203	} else {
8204	local_val.used = ident_token;
8205	}
8206
8207	const value_inst = try tunnelThroughClosure(
8208	gz,
8209	ident,
8210	num_namespaces_out,
8211	capturing_namespace,
8212	local_val.inst,
8213	local_val.token_src,
8214	gpa,
8215	);
8216
8217	return rvalueNoCoercePreRef(gz, ri, value_inst, ident);
8218	}
8219	s = local_val.parent;
8220	},
8221	.local_ptr => {
8222	const local_ptr = s.cast(Scope.LocalPtr).?;
8223	if (local_ptr.name == name_str_index) {
8224	if (ri.rl == .discard and ri.ctx == .assignment) {
8225	local_ptr.discarded = ident_token;
8226	} else {
8227	local_ptr.used = ident_token;
8228	}
8229
8230	// Can't close over a runtime variable
8231	if (num_namespaces_out != 0 and !local_ptr.maybe_comptime and !gz.is_typeof) {
8232	const ident_name = try astgen.identifierTokenString(ident_token);
8233	return astgen.failNodeNotes(ident, "mutable '{s}' not accessible from here", .{ident_name}, &.{
8234	try astgen.errNoteTok(local_ptr.token_src, "declared mutable here", .{}),
8235	try astgen.errNoteNode(capturing_namespace.?.node, "crosses namespace boundary here", .{}),
8236	});
8237	}
8238
8239	const ptr_inst = try tunnelThroughClosure(
8240	gz,
8241	ident,
8242	num_namespaces_out,
8243	capturing_namespace,
8244	local_ptr.ptr,
8245	local_ptr.token_src,
8246	gpa,
8247	);
8248
8249	switch (ri.rl) {
8250	.ref, .ref_coerced_ty => {
8251	local_ptr.used_as_lvalue = true;
8252	return ptr_inst;
8253	},
8254	else => {
8255	const loaded = try gz.addUnNode(.load, ptr_inst, ident);
8256	return rvalueNoCoercePreRef(gz, ri, loaded, ident);
8257	},
8258	}
8259	}
8260	s = local_ptr.parent;
8261	},
8262	.gen_zir => s = s.cast(GenZir).?.parent,
8263	.defer_normal, .defer_error => s = s.cast(Scope.Defer).?.parent,
8264	.namespace, .enum_namespace => {
8265	const ns = s.cast(Scope.Namespace).?;
8266	if (ns.decls.get(name_str_index)) \|i\| {
8267	if (found_already) \|f\| {
8268	return astgen.failNodeNotes(ident, "ambiguous reference", .{}, &.{
8269	try astgen.errNoteNode(f, "declared here", .{}),
8270	try astgen.errNoteNode(i, "also declared here", .{}),
8271	});
8272	}
8273	// We found a match but must continue looking for ambiguous references to decls.
8274	found_already = i;
8275	}
8276	if (s.tag == .namespace) num_namespaces_out += 1;
8277	capturing_namespace = ns;
8278	s = ns.parent;
8279	},
8280	.top => break,
8281	};
8282	if (found_already == null) {
8283	const ident_name = try astgen.identifierTokenString(ident_token);
8284	return astgen.failNode(ident, "use of undeclared identifier '{s}'", .{ident_name});
8285	}
8286
8287	// Decl references happen by name rather than ZIR index so that when unrelated
8288	// decls are modified, ZIR code containing references to them can be unmodified.
8289	switch (ri.rl) {
8290	.ref, .ref_coerced_ty => return gz.addStrTok(.decl_ref, name_str_index, ident_token),
8291	else => {
8292	const result = try gz.addStrTok(.decl_val, name_str_index, ident_token);
8293	return rvalueNoCoercePreRef(gz, ri, result, ident);
8294	},
8295	}
8296	}
8297
8298	/// Adds a capture to a namespace, if needed.
8299	/// Returns the index of the closure_capture instruction.
8300	fn tunnelThroughClosure(
8301	gz: *GenZir,
8302	inner_ref_node: Ast.Node.Index,
8303	num_tunnels: u32,
8304	ns: ?*Scope.Namespace,
8305	value: Zir.Inst.Ref,
8306	token: Ast.TokenIndex,
8307	gpa: Allocator,
8308	) !Zir.Inst.Ref {
8309	// For trivial values, we don't need a tunnel.
8310	// Just return the ref.
8311	if (num_tunnels == 0 or value.toIndex() == null) {
8312	return value;
8313	}
8314
8315	// Otherwise we need a tunnel. Check if this namespace
8316	// already has one for this value.
8317	const gop = try ns.?.captures.getOrPut(gpa, value.toIndex().?);
8318	if (!gop.found_existing) {
8319	// Make a new capture for this value but don't add it to the declaring_gz yet
8320	try gz.astgen.instructions.append(gz.astgen.gpa, .{
8321	.tag = .closure_capture,
8322	.data = .{ .un_tok = .{
8323	.operand = value,
8324	.src_tok = ns.?.declaring_gz.?.tokenIndexToRelative(token),
8325	} },
8326	});
8327	gop.value_ptr.* = @enumFromInt(gz.astgen.instructions.len - 1);
8328	}
8329
8330	// Add an instruction to get the value from the closure into
8331	// our current context
8332	return try gz.addInstNode(.closure_get, gop.value_ptr.*, inner_ref_node);
8333	}
8334
8335	fn stringLiteral(
8336	gz: *GenZir,
8337	ri: ResultInfo,
8338	node: Ast.Node.Index,
8339	) InnerError!Zir.Inst.Ref {
8340	const astgen = gz.astgen;
8341	const tree = astgen.tree;
8342	const main_tokens = tree.nodes.items(.main_token);
8343	const str_lit_token = main_tokens[node];
8344	const str = try astgen.strLitAsString(str_lit_token);
8345	const result = try gz.add(.{
8346	.tag = .str,
8347	.data = .{ .str = .{
8348	.start = str.index,
8349	.len = str.len,
8350	} },
8351	});
8352	return rvalue(gz, ri, result, node);
8353	}
8354
8355	fn multilineStringLiteral(
8356	gz: *GenZir,
8357	ri: ResultInfo,
8358	node: Ast.Node.Index,
8359	) InnerError!Zir.Inst.Ref {
8360	const astgen = gz.astgen;
8361	const str = try astgen.strLitNodeAsString(node);
8362	const result = try gz.add(.{
8363	.tag = .str,
8364	.data = .{ .str = .{
8365	.start = str.index,
8366	.len = str.len,
8367	} },
8368	});
8369	return rvalue(gz, ri, result, node);
8370	}
8371
8372	fn charLiteral(gz: *GenZir, ri: ResultInfo, node: Ast.Node.Index) InnerError!Zir.Inst.Ref {
8373	const astgen = gz.astgen;
8374	const tree = astgen.tree;
8375	const main_tokens = tree.nodes.items(.main_token);
8376	const main_token = main_tokens[node];
8377	const slice = tree.tokenSlice(main_token);
8378
8379	switch (std.zig.parseCharLiteral(slice)) {
8380	.success => \|codepoint\| {
8381	const result = try gz.addInt(codepoint);
8382	return rvalue(gz, ri, result, node);
8383	},
8384	.failure => \|err\| return astgen.failWithStrLitError(err, main_token, slice, 0),
8385	}
8386	}
8387
8388	const Sign = enum { negative, positive };
8389
8390	fn numberLiteral(gz: *GenZir, ri: ResultInfo, node: Ast.Node.Index, source_node: Ast.Node.Index, sign: Sign) InnerError!Zir.Inst.Ref {
8391	const astgen = gz.astgen;
8392	const tree = astgen.tree;
8393	const main_tokens = tree.nodes.items(.main_token);
8394	const num_token = main_tokens[node];
8395	const bytes = tree.tokenSlice(num_token);
8396
8397	const result: Zir.Inst.Ref = switch (std.zig.parseNumberLiteral(bytes)) {
8398	.int => \|num\| switch (num) {
8399	0 => if (sign == .positive) .zero else return astgen.failTokNotes(
8400	num_token,
8401	"integer literal '-0' is ambiguous",
8402	.{},
8403	&.{
8404	try astgen.errNoteTok(num_token, "use '0' for an integer zero", .{}),
8405	try astgen.errNoteTok(num_token, "use '-0.0' for a floating-point signed zero", .{}),
8406	},
8407	),
8408	1 => .one,
8409	else => try gz.addInt(num),
8410	},
8411	.big_int => \|base\| big: {
8412	const gpa = astgen.gpa;
8413	var big_int = try std.math.big.int.Managed.init(gpa);
8414	defer big_int.deinit();
8415	const prefix_offset: usize = if (base == .decimal) 0 else 2;
8416	big_int.setString(@intFromEnum(base), bytes[prefix_offset..]) catch \|err\| switch (err) {
8417	error.InvalidCharacter => unreachable, // caught in `parseNumberLiteral`
8418	error.InvalidBase => unreachable, // we only pass 16, 8, 2, see above
8419	error.OutOfMemory => return error.OutOfMemory,
8420	};
8421
8422	const limbs = big_int.limbs[0..big_int.len()];
8423	assert(big_int.isPositive());
8424	break :big try gz.addIntBig(limbs);
8425	},
8426	.float => {
8427	const unsigned_float_number = std.fmt.parseFloat(f128, bytes) catch \|err\| switch (err) {
8428	error.InvalidCharacter => unreachable, // validated by tokenizer
8429	};
8430	const float_number = switch (sign) {
8431	.negative => -unsigned_float_number,
8432	.positive => unsigned_float_number,
8433	};
8434	// If the value fits into a f64 without losing any precision, store it that way.
8435	@setFloatMode(.Strict);
8436	const smaller_float: f64 = @floatCast(float_number);
8437	const bigger_again: f128 = smaller_float;
8438	if (bigger_again == float_number) {
8439	const result = try gz.addFloat(smaller_float);
8440	return rvalue(gz, ri, result, source_node);
8441	}
8442	// We need to use 128 bits. Break the float into 4 u32 values so we can
8443	// put it into the `extra` array.
8444	const int_bits: u128 = @bitCast(float_number);
8445	const result = try gz.addPlNode(.float128, node, Zir.Inst.Float128{
8446	.piece0 = @truncate(int_bits),
8447	.piece1 = @truncate(int_bits >> 32),
8448	.piece2 = @truncate(int_bits >> 64),
8449	.piece3 = @truncate(int_bits >> 96),
8450	});
8451	return rvalue(gz, ri, result, source_node);
8452	},
8453	.failure => \|err\| return astgen.failWithNumberError(err, num_token, bytes),
8454	};
8455
8456	if (sign == .positive) {
8457	return rvalue(gz, ri, result, source_node);
8458	} else {
8459	const negated = try gz.addUnNode(.negate, result, source_node);
8460	return rvalue(gz, ri, negated, source_node);
8461	}
8462	}
8463
8464	fn failWithNumberError(astgen: *AstGen, err: std.zig.number_literal.Error, token: Ast.TokenIndex, bytes: []const u8) InnerError {
8465	const is_float = std.mem.indexOfScalar(u8, bytes, '.') != null;
8466	switch (err) {
8467	.leading_zero => if (is_float) {
8468	return astgen.failTok(token, "number '{s}' has leading zero", .{bytes});
8469	} else {
8470	return astgen.failTokNotes(token, "number '{s}' has leading zero", .{bytes}, &.{
8471	try astgen.errNoteTok(token, "use '0o' prefix for octal literals", .{}),
8472	});
8473	},
8474	.digit_after_base => return astgen.failTok(token, "expected a digit after base prefix", .{}),
8475	.upper_case_base => \|i\| return astgen.failOff(token, @intCast(i), "base prefix must be lowercase", .{}),
8476	.invalid_float_base => \|i\| return astgen.failOff(token, @intCast(i), "invalid base for float literal", .{}),
8477	.repeated_underscore => \|i\| return astgen.failOff(token, @intCast(i), "repeated digit separator", .{}),
8478	.invalid_underscore_after_special => \|i\| return astgen.failOff(token, @intCast(i), "expected digit before digit separator", .{}),
8479	.invalid_digit => \|info\| return astgen.failOff(token, @intCast(info.i), "invalid digit '{c}' for {s} base", .{ bytes[info.i], @tagName(info.base) }),
8480	.invalid_digit_exponent => \|i\| return astgen.failOff(token, @intCast(i), "invalid digit '{c}' in exponent", .{bytes[i]}),
8481	.duplicate_exponent => \|i\| return astgen.failOff(token, @intCast(i), "duplicate exponent", .{}),
8482	.exponent_after_underscore => \|i\| return astgen.failOff(token, @intCast(i), "expected digit before exponent", .{}),
8483	.special_after_underscore => \|i\| return astgen.failOff(token, @intCast(i), "expected digit before '{c}'", .{bytes[i]}),
8484	.trailing_special => \|i\| return astgen.failOff(token, @intCast(i), "expected digit after '{c}'", .{bytes[i - 1]}),
8485	.trailing_underscore => \|i\| return astgen.failOff(token, @intCast(i), "trailing digit separator", .{}),
8486	.duplicate_period => unreachable, // Validated by tokenizer
8487	.invalid_character => unreachable, // Validated by tokenizer
8488	.invalid_exponent_sign => \|i\| {
8489	assert(bytes.len >= 2 and bytes[0] == '0' and bytes[1] == 'x'); // Validated by tokenizer
8490	return astgen.failOff(token, @intCast(i), "sign '{c}' cannot follow digit '{c}' in hex base", .{ bytes[i], bytes[i - 1] });
8491	},
8492	}
8493	}
8494
8495	fn asmExpr(
8496	gz: *GenZir,
8497	scope: *Scope,
8498	ri: ResultInfo,
8499	node: Ast.Node.Index,
8500	full: Ast.full.Asm,
8501	) InnerError!Zir.Inst.Ref {
8502	const astgen = gz.astgen;
8503	const tree = astgen.tree;
8504	const main_tokens = tree.nodes.items(.main_token);
8505	const node_datas = tree.nodes.items(.data);
8506	const node_tags = tree.nodes.items(.tag);
8507	const token_tags = tree.tokens.items(.tag);
8508
8509	const TagAndTmpl = struct { tag: Zir.Inst.Extended, tmpl: Zir.NullTerminatedString };
8510	const tag_and_tmpl: TagAndTmpl = switch (node_tags[full.ast.template]) {
8511	.string_literal => .{
8512	.tag = .@"asm",
8513	.tmpl = (try astgen.strLitAsString(main_tokens[full.ast.template])).index,
8514	},
8515	.multiline_string_literal => .{
8516	.tag = .@"asm",
8517	.tmpl = (try astgen.strLitNodeAsString(full.ast.template)).index,
8518	},
8519	else => .{
8520	.tag = .asm_expr,
8521	.tmpl = @enumFromInt(@intFromEnum(try comptimeExpr(gz, scope, .{ .rl = .none }, full.ast.template))),
8522	},
8523	};
8524
8525	// See https://github.com/ziglang/zig/issues/215 and related issues discussing
8526	// possible inline assembly improvements. Until then here is status quo AstGen
8527	// for assembly syntax. It's used by std lib crypto aesni.zig.
8528	const is_container_asm = astgen.fn_block == null;
8529	if (is_container_asm) {
8530	if (full.volatile_token) \|t\|
8531	return astgen.failTok(t, "volatile is meaningless on global assembly", .{});
8532	if (full.outputs.len != 0 or full.inputs.len != 0 or full.first_clobber != null)
8533	return astgen.failNode(node, "global assembly cannot have inputs, outputs, or clobbers", .{});
8534	} else {
8535	if (full.outputs.len == 0 and full.volatile_token == null) {
8536	return astgen.failNode(node, "assembly expression with no output must be marked volatile", .{});
8537	}
8538	}
8539	if (full.outputs.len > 32) {
8540	return astgen.failNode(full.outputs[32], "too many asm outputs", .{});
8541	}
8542	var outputs_buffer: [32]Zir.Inst.Asm.Output = undefined;
8543	const outputs = outputs_buffer[0..full.outputs.len];
8544
8545	var output_type_bits: u32 = 0;
8546
8547	for (full.outputs, 0..) \|output_node, i\| {
8548	const symbolic_name = main_tokens[output_node];
8549	const name = try astgen.identAsString(symbolic_name);
8550	const constraint_token = symbolic_name + 2;
8551	const constraint = (try astgen.strLitAsString(constraint_token)).index;
8552	const has_arrow = token_tags[symbolic_name + 4] == .arrow;
8553	if (has_arrow) {
8554	if (output_type_bits != 0) {
8555	return astgen.failNode(output_node, "inline assembly allows up to one output value", .{});
8556	}
8557	output_type_bits \|= @as(u32, 1) << @intCast(i);
8558	const out_type_node = node_datas[output_node].lhs;
8559	const out_type_inst = try typeExpr(gz, scope, out_type_node);
8560	outputs[i] = .{
8561	.name = name,
8562	.constraint = constraint,
8563	.operand = out_type_inst,
8564	};
8565	} else {
8566	const ident_token = symbolic_name + 4;
8567	// TODO have a look at #215 and related issues and decide how to
8568	// handle outputs. Do we want this to be identifiers?
8569	// Or maybe we want to force this to be expressions with a pointer type.
8570	outputs[i] = .{
8571	.name = name,
8572	.constraint = constraint,
8573	.operand = try localVarRef(gz, scope, .{ .rl = .ref }, node, ident_token),
8574	};
8575	}
8576	}
8577
8578	if (full.inputs.len > 32) {
8579	return astgen.failNode(full.inputs[32], "too many asm inputs", .{});
8580	}
8581	var inputs_buffer: [32]Zir.Inst.Asm.Input = undefined;
8582	const inputs = inputs_buffer[0..full.inputs.len];
8583
8584	for (full.inputs, 0..) \|input_node, i\| {
8585	const symbolic_name = main_tokens[input_node];
8586	const name = try astgen.identAsString(symbolic_name);
8587	const constraint_token = symbolic_name + 2;
8588	const constraint = (try astgen.strLitAsString(constraint_token)).index;
8589	const operand = try expr(gz, scope, .{ .rl = .none }, node_datas[input_node].lhs);
8590	inputs[i] = .{
8591	.name = name,
8592	.constraint = constraint,
8593	.operand = operand,
8594	};
8595	}
8596
8597	var clobbers_buffer: [32]u32 = undefined;
8598	var clobber_i: usize = 0;
8599	if (full.first_clobber) \|first_clobber\| clobbers: {
8600	// asm ("foo" ::: "a", "b")
8601	// asm ("foo" ::: "a", "b",)
8602	var tok_i = first_clobber;
8603	while (true) : (tok_i += 1) {
8604	if (clobber_i >= clobbers_buffer.len) {
8605	return astgen.failTok(tok_i, "too many asm clobbers", .{});
8606	}
8607	clobbers_buffer[clobber_i] = @intFromEnum((try astgen.strLitAsString(tok_i)).index);
8608	clobber_i += 1;
8609	tok_i += 1;
8610	switch (token_tags[tok_i]) {
8611	.r_paren => break :clobbers,
8612	.comma => {
8613	if (token_tags[tok_i + 1] == .r_paren) {
8614	break :clobbers;
8615	} else {
8616	continue;
8617	}
8618	},
8619	else => unreachable,
8620	}
8621	}
8622	}
8623
8624	const result = try gz.addAsm(.{
8625	.tag = tag_and_tmpl.tag,
8626	.node = node,
8627	.asm_source = tag_and_tmpl.tmpl,
8628	.is_volatile = full.volatile_token != null,
8629	.output_type_bits = output_type_bits,
8630	.outputs = outputs,
8631	.inputs = inputs,
8632	.clobbers = clobbers_buffer[0..clobber_i],
8633	});
8634	return rvalue(gz, ri, result, node);
8635	}
8636
8637	fn as(
8638	gz: *GenZir,
8639	scope: *Scope,
8640	ri: ResultInfo,
8641	node: Ast.Node.Index,
8642	lhs: Ast.Node.Index,
8643	rhs: Ast.Node.Index,
8644	) InnerError!Zir.Inst.Ref {
8645	const dest_type = try typeExpr(gz, scope, lhs);
8646	const result = try reachableExpr(gz, scope, .{ .rl = .{ .ty = dest_type } }, rhs, node);
8647	return rvalue(gz, ri, result, node);
8648	}
8649
8650	fn unionInit(
8651	gz: *GenZir,
8652	scope: *Scope,
8653	ri: ResultInfo,
8654	node: Ast.Node.Index,
8655	params: []const Ast.Node.Index,
8656	) InnerError!Zir.Inst.Ref {
8657	const union_type = try typeExpr(gz, scope, params[0]);
8658	const field_name = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .slice_const_u8_type } }, params[1]);
8659	const field_type = try gz.addPlNode(.field_type_ref, node, Zir.Inst.FieldTypeRef{
8660	.container_type = union_type,
8661	.field_name = field_name,
8662	});
8663	const init = try reachableExpr(gz, scope, .{ .rl = .{ .ty = field_type } }, params[2], node);
8664	const result = try gz.addPlNode(.union_init, node, Zir.Inst.UnionInit{
8665	.union_type = union_type,
8666	.init = init,
8667	.field_name = field_name,
8668	});
8669	return rvalue(gz, ri, result, node);
8670	}
8671
8672	fn bitCast(
8673	gz: *GenZir,
8674	scope: *Scope,
8675	ri: ResultInfo,
8676	node: Ast.Node.Index,
8677	operand_node: Ast.Node.Index,
8678	) InnerError!Zir.Inst.Ref {
8679	const dest_type = try ri.rl.resultTypeForCast(gz, node, "@bitCast");
8680	const operand = try reachableExpr(gz, scope, .{ .rl = .none }, operand_node, node);
8681	const result = try gz.addPlNode(.bitcast, node, Zir.Inst.Bin{
8682	.lhs = dest_type,
8683	.rhs = operand,
8684	});
8685	return rvalue(gz, ri, result, node);
8686	}
8687
8688	/// Handle one or more nested pointer cast builtins:
8689	/// * @ptrCast
8690	/// * @alignCast
8691	/// * @addrSpaceCast
8692	/// * @constCast
8693	/// * @volatileCast
8694	/// Any sequence of such builtins is treated as a single operation. This allowed
8695	/// for sequences like `@ptrCast(@alignCast(ptr))` to work correctly despite the
8696	/// intermediate result type being unknown.
8697	fn ptrCast(
8698	gz: *GenZir,
8699	scope: *Scope,
8700	ri: ResultInfo,
8701	root_node: Ast.Node.Index,
8702	) InnerError!Zir.Inst.Ref {
8703	const astgen = gz.astgen;
8704	const tree = astgen.tree;
8705	const main_tokens = tree.nodes.items(.main_token);
8706	const node_datas = tree.nodes.items(.data);
8707	const node_tags = tree.nodes.items(.tag);
8708
8709	var flags: Zir.Inst.FullPtrCastFlags = .{};
8710
8711	// Note that all pointer cast builtins have one parameter, so we only need
8712	// to handle `builtin_call_two`.
8713	var node = root_node;
8714	while (true) {
8715	switch (node_tags[node]) {
8716	.builtin_call_two, .builtin_call_two_comma => {},
8717	.grouped_expression => {
8718	// Handle the chaining even with redundant parentheses
8719	node = node_datas[node].lhs;
8720	continue;
8721	},
8722	else => break,
8723	}
8724
8725	if (node_datas[node].lhs == 0) break; // 0 args
8726	if (node_datas[node].rhs != 0) break; // 2 args
8727
8728	const builtin_token = main_tokens[node];
8729	const builtin_name = tree.tokenSlice(builtin_token);
8730	const info = BuiltinFn.list.get(builtin_name) orelse break;
8731	if (info.param_count != 1) break;
8732
8733	switch (info.tag) {
8734	else => break,
8735	inline .ptr_cast,
8736	.align_cast,
8737	.addrspace_cast,
8738	.const_cast,
8739	.volatile_cast,
8740	=> \|tag\| {
8741	if (@field(flags, @tagName(tag))) {
8742	return astgen.failNode(node, "redundant {s}", .{builtin_name});
8743	}
8744	@field(flags, @tagName(tag)) = true;
8745	},
8746	}
8747
8748	node = node_datas[node].lhs;
8749	}
8750
8751	const flags_i: u5 = @bitCast(flags);
8752	assert(flags_i != 0);
8753
8754	const ptr_only: Zir.Inst.FullPtrCastFlags = .{ .ptr_cast = true };
8755	if (flags_i == @as(u5, @bitCast(ptr_only))) {
8756	// Special case: simpler representation
8757	return typeCast(gz, scope, ri, root_node, node, .ptr_cast, "@ptrCast");
8758	}
8759
8760	const no_result_ty_flags: Zir.Inst.FullPtrCastFlags = .{
8761	.const_cast = true,
8762	.volatile_cast = true,
8763	};
8764	if ((flags_i & ~@as(u5, @bitCast(no_result_ty_flags))) == 0) {
8765	// Result type not needed
8766	const cursor = maybeAdvanceSourceCursorToMainToken(gz, root_node);
8767	const operand = try expr(gz, scope, .{ .rl = .none }, node);
8768	try emitDbgStmt(gz, cursor);
8769	const result = try gz.addExtendedPayloadSmall(.ptr_cast_no_dest, flags_i, Zir.Inst.UnNode{
8770	.node = gz.nodeIndexToRelative(root_node),
8771	.operand = operand,
8772	});
8773	return rvalue(gz, ri, result, root_node);
8774	}
8775
8776	// Full cast including result type
8777
8778	const cursor = maybeAdvanceSourceCursorToMainToken(gz, root_node);
8779	const result_type = try ri.rl.resultTypeForCast(gz, root_node, flags.needResultTypeBuiltinName());
8780	const operand = try expr(gz, scope, .{ .rl = .none }, node);
8781	try emitDbgStmt(gz, cursor);
8782	const result = try gz.addExtendedPayloadSmall(.ptr_cast_full, flags_i, Zir.Inst.BinNode{
8783	.node = gz.nodeIndexToRelative(root_node),
8784	.lhs = result_type,
8785	.rhs = operand,
8786	});
8787	return rvalue(gz, ri, result, root_node);
8788	}
8789
8790	fn typeOf(
8791	gz: *GenZir,
8792	scope: *Scope,
8793	ri: ResultInfo,
8794	node: Ast.Node.Index,
8795	args: []const Ast.Node.Index,
8796	) InnerError!Zir.Inst.Ref {
8797	const astgen = gz.astgen;
8798	if (args.len < 1) {
8799	return astgen.failNode(node, "expected at least 1 argument, found 0", .{});
8800	}
8801	const gpa = astgen.gpa;
8802	if (args.len == 1) {
8803	const typeof_inst = try gz.makeBlockInst(.typeof_builtin, node);
8804
8805	var typeof_scope = gz.makeSubBlock(scope);
8806	typeof_scope.is_comptime = false;
8807	typeof_scope.is_typeof = true;
8808	typeof_scope.c_import = false;
8809	defer typeof_scope.unstack();
8810
8811	const ty_expr = try reachableExpr(&typeof_scope, &typeof_scope.base, .{ .rl = .none }, args[0], node);
8812	if (!gz.refIsNoReturn(ty_expr)) {
8813	_ = try typeof_scope.addBreak(.break_inline, typeof_inst, ty_expr);
8814	}
8815	try typeof_scope.setBlockBody(typeof_inst);
8816
8817	// typeof_scope unstacked now, can add new instructions to gz
8818	try gz.instructions.append(gpa, typeof_inst);
8819	return rvalue(gz, ri, typeof_inst.toRef(), node);
8820	}
8821	const payload_size: u32 = std.meta.fields(Zir.Inst.TypeOfPeer).len;
8822	const payload_index = try reserveExtra(astgen, payload_size + args.len);
8823	const args_index = payload_index + payload_size;
8824
8825	const typeof_inst = try gz.addExtendedMultiOpPayloadIndex(.typeof_peer, payload_index, args.len);
8826
8827	var typeof_scope = gz.makeSubBlock(scope);
8828	typeof_scope.is_comptime = false;
8829
8830	for (args, 0..) \|arg, i\| {
8831	const param_ref = try reachableExpr(&typeof_scope, &typeof_scope.base, .{ .rl = .none }, arg, node);
8832	astgen.extra.items[args_index + i] = @intFromEnum(param_ref);
8833	}
8834	_ = try typeof_scope.addBreak(.break_inline, typeof_inst.toIndex().?, .void_value);
8835
8836	const body = typeof_scope.instructionsSlice();
8837	const body_len = astgen.countBodyLenAfterFixups(body);
8838	astgen.setExtra(payload_index, Zir.Inst.TypeOfPeer{
8839	.body_len = @intCast(body_len),
8840	.body_index = @intCast(astgen.extra.items.len),
8841	.src_node = gz.nodeIndexToRelative(node),
8842	});
8843	try astgen.extra.ensureUnusedCapacity(gpa, body_len);
8844	astgen.appendBodyWithFixups(body);
8845	typeof_scope.unstack();
8846
8847	return rvalue(gz, ri, typeof_inst, node);
8848	}
8849
8850	fn minMax(
8851	gz: *GenZir,
8852	scope: *Scope,
8853	ri: ResultInfo,
8854	node: Ast.Node.Index,
8855	args: []const Ast.Node.Index,
8856	comptime op: enum { min, max },
8857	) InnerError!Zir.Inst.Ref {
8858	const astgen = gz.astgen;
8859	if (args.len < 2) {
8860	return astgen.failNode(node, "expected at least 2 arguments, found 0", .{});
8861	}
8862	if (args.len == 2) {
8863	const tag: Zir.Inst.Tag = switch (op) {
8864	.min => .min,
8865	.max => .max,
8866	};
8867	const a = try expr(gz, scope, .{ .rl = .none }, args[0]);
8868	const b = try expr(gz, scope, .{ .rl = .none }, args[1]);
8869	const result = try gz.addPlNode(tag, node, Zir.Inst.Bin{
8870	.lhs = a,
8871	.rhs = b,
8872	});
8873	return rvalue(gz, ri, result, node);
8874	}
8875	const payload_index = try addExtra(astgen, Zir.Inst.NodeMultiOp{
8876	.src_node = gz.nodeIndexToRelative(node),
8877	});
8878	var extra_index = try reserveExtra(gz.astgen, args.len);
8879	for (args) \|arg\| {
8880	const arg_ref = try expr(gz, scope, .{ .rl = .none }, arg);
8881	astgen.extra.items[extra_index] = @intFromEnum(arg_ref);
8882	extra_index += 1;
8883	}
8884	const tag: Zir.Inst.Extended = switch (op) {
8885	.min => .min_multi,
8886	.max => .max_multi,
8887	};
8888	const result = try gz.addExtendedMultiOpPayloadIndex(tag, payload_index, args.len);
8889	return rvalue(gz, ri, result, node);
8890	}
8891
8892	fn builtinCall(
8893	gz: *GenZir,
8894	scope: *Scope,
8895	ri: ResultInfo,
8896	node: Ast.Node.Index,
8897	params: []const Ast.Node.Index,
8898	) InnerError!Zir.Inst.Ref {
8899	const astgen = gz.astgen;
8900	const tree = astgen.tree;
8901	const main_tokens = tree.nodes.items(.main_token);
8902
8903	const builtin_token = main_tokens[node];
8904	const builtin_name = tree.tokenSlice(builtin_token);
8905
8906	// We handle the different builtins manually because they have different semantics depending
8907	// on the function. For example, `@as` and others participate in result location semantics,
8908	// and `@cImport` creates a special scope that collects a .c source code text buffer.
8909	// Also, some builtins have a variable number of parameters.
8910
8911	const info = BuiltinFn.list.get(builtin_name) orelse {
8912	return astgen.failNode(node, "invalid builtin function: '{s}'", .{
8913	builtin_name,
8914	});
8915	};
8916	if (info.param_count) \|expected\| {
8917	if (expected != params.len) {
8918	const s = if (expected == 1) "" else "s";
8919	return astgen.failNode(node, "expected {d} argument{s}, found {d}", .{
8920	expected, s, params.len,
8921	});
8922	}
8923	}
8924
8925	// Check function scope-only builtins
8926
8927	if (astgen.fn_block == null and info.illegal_outside_function)
8928	return astgen.failNode(node, "'{s}' outside function scope", .{builtin_name});
8929
8930	switch (info.tag) {
8931	.import => {
8932	const node_tags = tree.nodes.items(.tag);
8933	const operand_node = params[0];
8934
8935	if (node_tags[operand_node] != .string_literal) {
8936	// Spec reference: https://github.com/ziglang/zig/issues/2206
8937	return astgen.failNode(operand_node, "@import operand must be a string literal", .{});
8938	}
8939	const str_lit_token = main_tokens[operand_node];
8940	const str = try astgen.strLitAsString(str_lit_token);
8941	const str_slice = astgen.string_bytes.items[@intFromEnum(str.index)..][0..str.len];
8942	if (mem.indexOfScalar(u8, str_slice, 0) != null) {
8943	return astgen.failTok(str_lit_token, "import path cannot contain null bytes", .{});
8944	} else if (str.len == 0) {
8945	return astgen.failTok(str_lit_token, "import path cannot be empty", .{});
8946	}
8947	const result = try gz.addStrTok(.import, str.index, str_lit_token);
8948	const gop = try astgen.imports.getOrPut(astgen.gpa, str.index);
8949	if (!gop.found_existing) {
8950	gop.value_ptr.* = str_lit_token;
8951	}
8952	return rvalue(gz, ri, result, node);
8953	},
8954	.compile_log => {
8955	const payload_index = try addExtra(gz.astgen, Zir.Inst.NodeMultiOp{
8956	.src_node = gz.nodeIndexToRelative(node),
8957	});
8958	var extra_index = try reserveExtra(gz.astgen, params.len);
8959	for (params) \|param\| {
8960	const param_ref = try expr(gz, scope, .{ .rl = .none }, param);
8961	astgen.extra.items[extra_index] = @intFromEnum(param_ref);
8962	extra_index += 1;
8963	}
8964	const result = try gz.addExtendedMultiOpPayloadIndex(.compile_log, payload_index, params.len);
8965	return rvalue(gz, ri, result, node);
8966	},
8967	.field => {
8968	if (ri.rl == .ref or ri.rl == .ref_coerced_ty) {
8969	return gz.addPlNode(.field_ptr_named, node, Zir.Inst.FieldNamed{
8970	.lhs = try expr(gz, scope, .{ .rl = .ref }, params[0]),
8971	.field_name = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .slice_const_u8_type } }, params[1]),
8972	});
8973	}
8974	const result = try gz.addPlNode(.field_val_named, node, Zir.Inst.FieldNamed{
8975	.lhs = try expr(gz, scope, .{ .rl = .none }, params[0]),
8976	.field_name = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .slice_const_u8_type } }, params[1]),
8977	});
8978	return rvalue(gz, ri, result, node);
8979	},
8980
8981	// zig fmt: off
8982	.as => return as( gz, scope, ri, node, params[0], params[1]),
8983	.bit_cast => return bitCast( gz, scope, ri, node, params[0]),
8984	.TypeOf => return typeOf( gz, scope, ri, node, params),
8985	.union_init => return unionInit(gz, scope, ri, node, params),
8986	.c_import => return cImport( gz, scope, node, params[0]),
8987	.min => return minMax( gz, scope, ri, node, params, .min),
8988	.max => return minMax( gz, scope, ri, node, params, .max),
8989	// zig fmt: on
8990
8991	.@"export" => {
8992	const node_tags = tree.nodes.items(.tag);
8993	const node_datas = tree.nodes.items(.data);
8994	// This function causes a Decl to be exported. The first parameter is not an expression,
8995	// but an identifier of the Decl to be exported.
8996	var namespace: Zir.Inst.Ref = .none;
8997	var decl_name: Zir.NullTerminatedString = .empty;
8998	switch (node_tags[params[0]]) {
8999	.identifier => {
9000	const ident_token = main_tokens[params[0]];
9001	if (isPrimitive(tree.tokenSlice(ident_token))) {
9002	return astgen.failTok(ident_token, "unable to export primitive value", .{});
9003	}
9004	decl_name = try astgen.identAsString(ident_token);
9005
9006	var s = scope;
9007	var found_already: ?Ast.Node.Index = null; // we have found a decl with the same name already
9008	while (true) switch (s.tag) {
9009	.local_val => {
9010	const local_val = s.cast(Scope.LocalVal).?;
9011	if (local_val.name == decl_name) {
9012	local_val.used = ident_token;
9013	_ = try gz.addPlNode(.export_value, node, Zir.Inst.ExportValue{
9014	.operand = local_val.inst,
9015	.options = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .export_options_type } }, params[1]),
9016	});
9017	return rvalue(gz, ri, .void_value, node);
9018	}
9019	s = local_val.parent;
9020	},
9021	.local_ptr => {
9022	const local_ptr = s.cast(Scope.LocalPtr).?;
9023	if (local_ptr.name == decl_name) {
9024	if (!local_ptr.maybe_comptime)
9025	return astgen.failNode(params[0], "unable to export runtime-known value", .{});
9026	local_ptr.used = ident_token;
9027	const loaded = try gz.addUnNode(.load, local_ptr.ptr, node);
9028	_ = try gz.addPlNode(.export_value, node, Zir.Inst.ExportValue{
9029	.operand = loaded,
9030	.options = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .export_options_type } }, params[1]),
9031	});
9032	return rvalue(gz, ri, .void_value, node);
9033	}
9034	s = local_ptr.parent;
9035	},
9036	.gen_zir => s = s.cast(GenZir).?.parent,
9037	.defer_normal, .defer_error => s = s.cast(Scope.Defer).?.parent,
9038	.namespace, .enum_namespace => {
9039	const ns = s.cast(Scope.Namespace).?;
9040	if (ns.decls.get(decl_name)) \|i\| {
9041	if (found_already) \|f\| {
9042	return astgen.failNodeNotes(node, "ambiguous reference", .{}, &.{
9043	try astgen.errNoteNode(f, "declared here", .{}),
9044	try astgen.errNoteNode(i, "also declared here", .{}),
9045	});
9046	}
9047	// We found a match but must continue looking for ambiguous references to decls.
9048	found_already = i;
9049	}
9050	s = ns.parent;
9051	},
9052	.top => break,
9053	};
9054	if (found_already == null) {
9055	const ident_name = try astgen.identifierTokenString(ident_token);
9056	return astgen.failNode(params[0], "use of undeclared identifier '{s}'", .{ident_name});
9057	}
9058	},
9059	.field_access => {
9060	const namespace_node = node_datas[params[0]].lhs;
9061	namespace = try typeExpr(gz, scope, namespace_node);
9062	const dot_token = main_tokens[params[0]];
9063	const field_ident = dot_token + 1;
9064	decl_name = try astgen.identAsString(field_ident);
9065	},
9066	else => return astgen.failNode(params[0], "symbol to export must identify a declaration", .{}),
9067	}
9068	const options = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .export_options_type } }, params[1]);
9069	_ = try gz.addPlNode(.@"export", node, Zir.Inst.Export{
9070	.namespace = namespace,
9071	.decl_name = decl_name,
9072	.options = options,
9073	});
9074	return rvalue(gz, ri, .void_value, node);
9075	},
9076	.@"extern" => {
9077	const type_inst = try typeExpr(gz, scope, params[0]);
9078	const options = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .extern_options_type } }, params[1]);
9079	const result = try gz.addExtendedPayload(.builtin_extern, Zir.Inst.BinNode{
9080	.node = gz.nodeIndexToRelative(node),
9081	.lhs = type_inst,
9082	.rhs = options,
9083	});
9084	return rvalue(gz, ri, result, node);
9085	},
9086	.fence => {
9087	const order = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .atomic_order_type } }, params[0]);
9088	_ = try gz.addExtendedPayload(.fence, Zir.Inst.UnNode{
9089	.node = gz.nodeIndexToRelative(node),
9090	.operand = order,
9091	});
9092	return rvalue(gz, ri, .void_value, node);
9093	},
9094	.set_float_mode => {
9095	const order = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .float_mode_type } }, params[0]);
9096	_ = try gz.addExtendedPayload(.set_float_mode, Zir.Inst.UnNode{
9097	.node = gz.nodeIndexToRelative(node),
9098	.operand = order,
9099	});
9100	return rvalue(gz, ri, .void_value, node);
9101	},
9102	.set_align_stack => {
9103	const order = try expr(gz, scope, coerced_align_ri, params[0]);
9104	_ = try gz.addExtendedPayload(.set_align_stack, Zir.Inst.UnNode{
9105	.node = gz.nodeIndexToRelative(node),
9106	.operand = order,
9107	});
9108	return rvalue(gz, ri, .void_value, node);
9109	},
9110	.set_cold => {
9111	const order = try expr(gz, scope, ri, params[0]);
9112	_ = try gz.addExtendedPayload(.set_cold, Zir.Inst.UnNode{
9113	.node = gz.nodeIndexToRelative(node),
9114	.operand = order,
9115	});
9116	return rvalue(gz, ri, .void_value, node);
9117	},
9118
9119	.src => {
9120	const token_starts = tree.tokens.items(.start);
9121	const node_start = token_starts[tree.firstToken(node)];
9122	astgen.advanceSourceCursor(node_start);
9123	const result = try gz.addExtendedPayload(.builtin_src, Zir.Inst.Src{
9124	.node = gz.nodeIndexToRelative(node),
9125	.line = astgen.source_line,
9126	.column = astgen.source_column,
9127	});
9128	return rvalue(gz, ri, result, node);
9129	},
9130
9131	// zig fmt: off
9132	.This => return rvalue(gz, ri, try gz.addNodeExtended(.this, node), node),
9133	.return_address => return rvalue(gz, ri, try gz.addNodeExtended(.ret_addr, node), node),
9134	.error_return_trace => return rvalue(gz, ri, try gz.addNodeExtended(.error_return_trace, node), node),
9135	.frame => return rvalue(gz, ri, try gz.addNodeExtended(.frame, node), node),
9136	.frame_address => return rvalue(gz, ri, try gz.addNodeExtended(.frame_address, node), node),
9137	.breakpoint => return rvalue(gz, ri, try gz.addNodeExtended(.breakpoint, node), node),
9138	.in_comptime => return rvalue(gz, ri, try gz.addNodeExtended(.in_comptime, node), node),
9139
9140	.type_info => return simpleUnOpType(gz, scope, ri, node, params[0], .type_info),
9141	.size_of => return simpleUnOpType(gz, scope, ri, node, params[0], .size_of),
9142	.bit_size_of => return simpleUnOpType(gz, scope, ri, node, params[0], .bit_size_of),
9143	.align_of => return simpleUnOpType(gz, scope, ri, node, params[0], .align_of),
9144
9145	.int_from_ptr => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .int_from_ptr),
9146	.compile_error => return simpleUnOp(gz, scope, ri, node, .{ .rl = .{ .coerced_ty = .slice_const_u8_type } }, params[0], .compile_error),
9147	.set_eval_branch_quota => return simpleUnOp(gz, scope, ri, node, .{ .rl = .{ .coerced_ty = .u32_type } }, params[0], .set_eval_branch_quota),
9148	.int_from_enum => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .int_from_enum),
9149	.int_from_bool => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .int_from_bool),
9150	.embed_file => return simpleUnOp(gz, scope, ri, node, .{ .rl = .{ .coerced_ty = .slice_const_u8_type } }, params[0], .embed_file),
9151	.error_name => return simpleUnOp(gz, scope, ri, node, .{ .rl = .{ .coerced_ty = .anyerror_type } }, params[0], .error_name),
9152	.set_runtime_safety => return simpleUnOp(gz, scope, ri, node, coerced_bool_ri, params[0], .set_runtime_safety),
9153	.sqrt => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .sqrt),
9154	.sin => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .sin),
9155	.cos => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .cos),
9156	.tan => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .tan),
9157	.exp => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .exp),
9158	.exp2 => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .exp2),
9159	.log => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .log),
9160	.log2 => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .log2),
9161	.log10 => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .log10),
9162	.abs => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .abs),
9163	.floor => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .floor),
9164	.ceil => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .ceil),
9165	.trunc => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .trunc),
9166	.round => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .round),
9167	.tag_name => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .tag_name),
9168	.type_name => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .type_name),
9169	.Frame => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .frame_type),
9170	.frame_size => return simpleUnOp(gz, scope, ri, node, .{ .rl = .none }, params[0], .frame_size),
9171
9172	.int_from_float => return typeCast(gz, scope, ri, node, params[0], .int_from_float, builtin_name),
9173	.float_from_int => return typeCast(gz, scope, ri, node, params[0], .float_from_int, builtin_name),
9174	.ptr_from_int => return typeCast(gz, scope, ri, node, params[0], .ptr_from_int, builtin_name),
9175	.enum_from_int => return typeCast(gz, scope, ri, node, params[0], .enum_from_int, builtin_name),
9176	.float_cast => return typeCast(gz, scope, ri, node, params[0], .float_cast, builtin_name),
9177	.int_cast => return typeCast(gz, scope, ri, node, params[0], .int_cast, builtin_name),
9178	.truncate => return typeCast(gz, scope, ri, node, params[0], .truncate, builtin_name),
9179	// zig fmt: on
9180
9181	.Type => {
9182	const operand = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .type_info_type } }, params[0]);
9183
9184	const gpa = gz.astgen.gpa;
9185
9186	try gz.instructions.ensureUnusedCapacity(gpa, 1);
9187	try gz.astgen.instructions.ensureUnusedCapacity(gpa, 1);
9188
9189	const payload_index = try gz.astgen.addExtra(Zir.Inst.UnNode{
9190	.node = gz.nodeIndexToRelative(node),
9191	.operand = operand,
9192	});
9193	const new_index: Zir.Inst.Index = @enumFromInt(gz.astgen.instructions.len);
9194	gz.astgen.instructions.appendAssumeCapacity(.{
9195	.tag = .extended,
9196	.data = .{ .extended = .{
9197	.opcode = .reify,
9198	.small = @intFromEnum(gz.anon_name_strategy),
9199	.operand = payload_index,
9200	} },
9201	});
9202	gz.instructions.appendAssumeCapacity(new_index);
9203	const result = new_index.toRef();
9204	return rvalue(gz, ri, result, node);
9205	},
9206	.panic => {
9207	try emitDbgNode(gz, node);
9208	return simpleUnOp(gz, scope, ri, node, .{ .rl = .{ .coerced_ty = .slice_const_u8_type } }, params[0], .panic);
9209	},
9210	.trap => {
9211	try emitDbgNode(gz, node);
9212	_ = try gz.addNode(.trap, node);
9213	return rvalue(gz, ri, .unreachable_value, node);
9214	},
9215	.int_from_error => {
9216	const operand = try expr(gz, scope, .{ .rl = .none }, params[0]);
9217	const result = try gz.addExtendedPayload(.int_from_error, Zir.Inst.UnNode{
9218	.node = gz.nodeIndexToRelative(node),
9219	.operand = operand,
9220	});
9221	return rvalue(gz, ri, result, node);
9222	},
9223	.error_from_int => {
9224	const operand = try expr(gz, scope, .{ .rl = .none }, params[0]);
9225	const result = try gz.addExtendedPayload(.error_from_int, Zir.Inst.UnNode{
9226	.node = gz.nodeIndexToRelative(node),
9227	.operand = operand,
9228	});
9229	return rvalue(gz, ri, result, node);
9230	},
9231	.error_cast => {
9232	try emitDbgNode(gz, node);
9233
9234	const result = try gz.addExtendedPayload(.error_cast, Zir.Inst.BinNode{
9235	.lhs = try ri.rl.resultTypeForCast(gz, node, "@errorCast"),
9236	.rhs = try expr(gz, scope, .{ .rl = .none }, params[0]),
9237	.node = gz.nodeIndexToRelative(node),
9238	});
9239	return rvalue(gz, ri, result, node);
9240	},
9241	.ptr_cast,
9242	.align_cast,
9243	.addrspace_cast,
9244	.const_cast,
9245	.volatile_cast,
9246	=> return ptrCast(gz, scope, ri, node),
9247
9248	// zig fmt: off
9249	.has_decl => return hasDeclOrField(gz, scope, ri, node, params[0], params[1], .has_decl),
9250	.has_field => return hasDeclOrField(gz, scope, ri, node, params[0], params[1], .has_field),
9251
9252	.clz => return bitBuiltin(gz, scope, ri, node, params[0], .clz),
9253	.ctz => return bitBuiltin(gz, scope, ri, node, params[0], .ctz),
9254	.pop_count => return bitBuiltin(gz, scope, ri, node, params[0], .pop_count),
9255	.byte_swap => return bitBuiltin(gz, scope, ri, node, params[0], .byte_swap),
9256	.bit_reverse => return bitBuiltin(gz, scope, ri, node, params[0], .bit_reverse),
9257
9258	.div_exact => return divBuiltin(gz, scope, ri, node, params[0], params[1], .div_exact),
9259	.div_floor => return divBuiltin(gz, scope, ri, node, params[0], params[1], .div_floor),
9260	.div_trunc => return divBuiltin(gz, scope, ri, node, params[0], params[1], .div_trunc),
9261	.mod => return divBuiltin(gz, scope, ri, node, params[0], params[1], .mod),
9262	.rem => return divBuiltin(gz, scope, ri, node, params[0], params[1], .rem),
9263
9264	.shl_exact => return shiftOp(gz, scope, ri, node, params[0], params[1], .shl_exact),
9265	.shr_exact => return shiftOp(gz, scope, ri, node, params[0], params[1], .shr_exact),
9266
9267	.bit_offset_of => return offsetOf(gz, scope, ri, node, params[0], params[1], .bit_offset_of),
9268	.offset_of => return offsetOf(gz, scope, ri, node, params[0], params[1], .offset_of),
9269
9270	.c_undef => return simpleCBuiltin(gz, scope, ri, node, params[0], .c_undef),
9271	.c_include => return simpleCBuiltin(gz, scope, ri, node, params[0], .c_include),
9272
9273	.cmpxchg_strong => return cmpxchg(gz, scope, ri, node, params, 1),
9274	.cmpxchg_weak => return cmpxchg(gz, scope, ri, node, params, 0),
9275	// zig fmt: on
9276
9277	.wasm_memory_size => {
9278	const operand = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .u32_type } }, params[0]);
9279	const result = try gz.addExtendedPayload(.wasm_memory_size, Zir.Inst.UnNode{
9280	.node = gz.nodeIndexToRelative(node),
9281	.operand = operand,
9282	});
9283	return rvalue(gz, ri, result, node);
9284	},
9285	.wasm_memory_grow => {
9286	const index_arg = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .u32_type } }, params[0]);
9287	const delta_arg = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .u32_type } }, params[1]);
9288	const result = try gz.addExtendedPayload(.wasm_memory_grow, Zir.Inst.BinNode{
9289	.node = gz.nodeIndexToRelative(node),
9290	.lhs = index_arg,
9291	.rhs = delta_arg,
9292	});
9293	return rvalue(gz, ri, result, node);
9294	},
9295	.c_define => {
9296	if (!gz.c_import) return gz.astgen.failNode(node, "C define valid only inside C import block", .{});
9297	const name = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .slice_const_u8_type } }, params[0]);
9298	const value = try comptimeExpr(gz, scope, .{ .rl = .none }, params[1]);
9299	const result = try gz.addExtendedPayload(.c_define, Zir.Inst.BinNode{
9300	.node = gz.nodeIndexToRelative(node),
9301	.lhs = name,
9302	.rhs = value,
9303	});
9304	return rvalue(gz, ri, result, node);
9305	},
9306
9307	.splat => {
9308	const result_type = try ri.rl.resultTypeForCast(gz, node, "@splat");
9309	const elem_type = try gz.addUnNode(.vector_elem_type, result_type, node);
9310	const scalar = try expr(gz, scope, .{ .rl = .{ .ty = elem_type } }, params[0]);
9311	const result = try gz.addPlNode(.splat, node, Zir.Inst.Bin{
9312	.lhs = result_type,
9313	.rhs = scalar,
9314	});
9315	return rvalue(gz, ri, result, node);
9316	},
9317	.reduce => {
9318	const op = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .reduce_op_type } }, params[0]);
9319	const scalar = try expr(gz, scope, .{ .rl = .none }, params[1]);
9320	const result = try gz.addPlNode(.reduce, node, Zir.Inst.Bin{
9321	.lhs = op,
9322	.rhs = scalar,
9323	});
9324	return rvalue(gz, ri, result, node);
9325	},
9326
9327	.add_with_overflow => return overflowArithmetic(gz, scope, ri, node, params, .add_with_overflow),
9328	.sub_with_overflow => return overflowArithmetic(gz, scope, ri, node, params, .sub_with_overflow),
9329	.mul_with_overflow => return overflowArithmetic(gz, scope, ri, node, params, .mul_with_overflow),
9330	.shl_with_overflow => return overflowArithmetic(gz, scope, ri, node, params, .shl_with_overflow),
9331
9332	.atomic_load => {
9333	const result = try gz.addPlNode(.atomic_load, node, Zir.Inst.AtomicLoad{
9334	// zig fmt: off
9335	.elem_type = try typeExpr(gz, scope, params[0]),
9336	.ptr = try expr (gz, scope, .{ .rl = .none }, params[1]),
9337	.ordering = try expr (gz, scope, .{ .rl = .{ .coerced_ty = .atomic_order_type } }, params[2]),
9338	// zig fmt: on
9339	});
9340	return rvalue(gz, ri, result, node);
9341	},
9342	.atomic_rmw => {
9343	const int_type = try typeExpr(gz, scope, params[0]);
9344	const result = try gz.addPlNode(.atomic_rmw, node, Zir.Inst.AtomicRmw{
9345	// zig fmt: off
9346	.ptr = try expr(gz, scope, .{ .rl = .none }, params[1]),
9347	.operation = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .atomic_rmw_op_type } }, params[2]),
9348	.operand = try expr(gz, scope, .{ .rl = .{ .ty = int_type } }, params[3]),
9349	.ordering = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .atomic_order_type } }, params[4]),
9350	// zig fmt: on
9351	});
9352	return rvalue(gz, ri, result, node);
9353	},
9354	.atomic_store => {
9355	const int_type = try typeExpr(gz, scope, params[0]);
9356	_ = try gz.addPlNode(.atomic_store, node, Zir.Inst.AtomicStore{
9357	// zig fmt: off
9358	.ptr = try expr(gz, scope, .{ .rl = .none }, params[1]),
9359	.operand = try expr(gz, scope, .{ .rl = .{ .ty = int_type } }, params[2]),
9360	.ordering = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .atomic_order_type } }, params[3]),
9361	// zig fmt: on
9362	});
9363	return rvalue(gz, ri, .void_value, node);
9364	},
9365	.mul_add => {
9366	const float_type = try typeExpr(gz, scope, params[0]);
9367	const mulend1 = try expr(gz, scope, .{ .rl = .{ .coerced_ty = float_type } }, params[1]);
9368	const mulend2 = try expr(gz, scope, .{ .rl = .{ .coerced_ty = float_type } }, params[2]);
9369	const addend = try expr(gz, scope, .{ .rl = .{ .ty = float_type } }, params[3]);
9370	const result = try gz.addPlNode(.mul_add, node, Zir.Inst.MulAdd{
9371	.mulend1 = mulend1,
9372	.mulend2 = mulend2,
9373	.addend = addend,
9374	});
9375	return rvalue(gz, ri, result, node);
9376	},
9377	.call => {
9378	const modifier = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .call_modifier_type } }, params[0]);
9379	const callee = try expr(gz, scope, .{ .rl = .none }, params[1]);
9380	const args = try expr(gz, scope, .{ .rl = .none }, params[2]);
9381	const result = try gz.addPlNode(.builtin_call, node, Zir.Inst.BuiltinCall{
9382	.modifier = modifier,
9383	.callee = callee,
9384	.args = args,
9385	.flags = .{
9386	.is_nosuspend = gz.nosuspend_node != 0,
9387	.ensure_result_used = false,
9388	},
9389	});
9390	return rvalue(gz, ri, result, node);
9391	},
9392	.field_parent_ptr => {
9393	const parent_type = try typeExpr(gz, scope, params[0]);
9394	const field_name = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .slice_const_u8_type } }, params[1]);
9395	const result = try gz.addPlNode(.field_parent_ptr, node, Zir.Inst.FieldParentPtr{
9396	.parent_type = parent_type,
9397	.field_name = field_name,
9398	.field_ptr = try expr(gz, scope, .{ .rl = .none }, params[2]),
9399	});
9400	return rvalue(gz, ri, result, node);
9401	},
9402	.memcpy => {
9403	_ = try gz.addPlNode(.memcpy, node, Zir.Inst.Bin{
9404	.lhs = try expr(gz, scope, .{ .rl = .none }, params[0]),
9405	.rhs = try expr(gz, scope, .{ .rl = .none }, params[1]),
9406	});
9407	return rvalue(gz, ri, .void_value, node);
9408	},
9409	.memset => {
9410	const lhs = try expr(gz, scope, .{ .rl = .none }, params[0]);
9411	const lhs_ty = try gz.addUnNode(.typeof, lhs, params[0]);
9412	const elem_ty = try gz.addUnNode(.indexable_ptr_elem_type, lhs_ty, params[0]);
9413	_ = try gz.addPlNode(.memset, node, Zir.Inst.Bin{
9414	.lhs = lhs,
9415	.rhs = try expr(gz, scope, .{ .rl = .{ .coerced_ty = elem_ty } }, params[1]),
9416	});
9417	return rvalue(gz, ri, .void_value, node);
9418	},
9419	.shuffle => {
9420	const result = try gz.addPlNode(.shuffle, node, Zir.Inst.Shuffle{
9421	.elem_type = try typeExpr(gz, scope, params[0]),
9422	.a = try expr(gz, scope, .{ .rl = .none }, params[1]),
9423	.b = try expr(gz, scope, .{ .rl = .none }, params[2]),
9424	.mask = try comptimeExpr(gz, scope, .{ .rl = .none }, params[3]),
9425	});
9426	return rvalue(gz, ri, result, node);
9427	},
9428	.select => {
9429	const result = try gz.addExtendedPayload(.select, Zir.Inst.Select{
9430	.node = gz.nodeIndexToRelative(node),
9431	.elem_type = try typeExpr(gz, scope, params[0]),
9432	.pred = try expr(gz, scope, .{ .rl = .none }, params[1]),
9433	.a = try expr(gz, scope, .{ .rl = .none }, params[2]),
9434	.b = try expr(gz, scope, .{ .rl = .none }, params[3]),
9435	});
9436	return rvalue(gz, ri, result, node);
9437	},
9438	.async_call => {
9439	const result = try gz.addExtendedPayload(.builtin_async_call, Zir.Inst.AsyncCall{
9440	.node = gz.nodeIndexToRelative(node),
9441	.frame_buffer = try expr(gz, scope, .{ .rl = .none }, params[0]),
9442	.result_ptr = try expr(gz, scope, .{ .rl = .none }, params[1]),
9443	.fn_ptr = try expr(gz, scope, .{ .rl = .none }, params[2]),
9444	.args = try expr(gz, scope, .{ .rl = .none }, params[3]),
9445	});
9446	return rvalue(gz, ri, result, node);
9447	},
9448	.Vector => {
9449	const result = try gz.addPlNode(.vector_type, node, Zir.Inst.Bin{
9450	.lhs = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .u32_type } }, params[0]),
9451	.rhs = try typeExpr(gz, scope, params[1]),
9452	});
9453	return rvalue(gz, ri, result, node);
9454	},
9455	.prefetch => {
9456	const ptr = try expr(gz, scope, .{ .rl = .none }, params[0]);
9457	const options = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .prefetch_options_type } }, params[1]);
9458	_ = try gz.addExtendedPayload(.prefetch, Zir.Inst.BinNode{
9459	.node = gz.nodeIndexToRelative(node),
9460	.lhs = ptr,
9461	.rhs = options,
9462	});
9463	return rvalue(gz, ri, .void_value, node);
9464	},
9465	.c_va_arg => {
9466	const result = try gz.addExtendedPayload(.c_va_arg, Zir.Inst.BinNode{
9467	.node = gz.nodeIndexToRelative(node),
9468	.lhs = try expr(gz, scope, .{ .rl = .none }, params[0]),
9469	.rhs = try typeExpr(gz, scope, params[1]),
9470	});
9471	return rvalue(gz, ri, result, node);
9472	},
9473	.c_va_copy => {
9474	const result = try gz.addExtendedPayload(.c_va_copy, Zir.Inst.UnNode{
9475	.node = gz.nodeIndexToRelative(node),
9476	.operand = try expr(gz, scope, .{ .rl = .none }, params[0]),
9477	});
9478	return rvalue(gz, ri, result, node);
9479	},
9480	.c_va_end => {
9481	const result = try gz.addExtendedPayload(.c_va_end, Zir.Inst.UnNode{
9482	.node = gz.nodeIndexToRelative(node),
9483	.operand = try expr(gz, scope, .{ .rl = .none }, params[0]),
9484	});
9485	return rvalue(gz, ri, result, node);
9486	},
9487	.c_va_start => {
9488	if (!astgen.fn_var_args) {
9489	return astgen.failNode(node, "'@cVaStart' in a non-variadic function", .{});
9490	}
9491	return rvalue(gz, ri, try gz.addNodeExtended(.c_va_start, node), node);
9492	},
9493
9494	.work_item_id => {
9495	const operand = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .u32_type } }, params[0]);
9496	const result = try gz.addExtendedPayload(.work_item_id, Zir.Inst.UnNode{
9497	.node = gz.nodeIndexToRelative(node),
9498	.operand = operand,
9499	});
9500	return rvalue(gz, ri, result, node);
9501	},
9502	.work_group_size => {
9503	const operand = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .u32_type } }, params[0]);
9504	const result = try gz.addExtendedPayload(.work_group_size, Zir.Inst.UnNode{
9505	.node = gz.nodeIndexToRelative(node),
9506	.operand = operand,
9507	});
9508	return rvalue(gz, ri, result, node);
9509	},
9510	.work_group_id => {
9511	const operand = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .u32_type } }, params[0]);
9512	const result = try gz.addExtendedPayload(.work_group_id, Zir.Inst.UnNode{
9513	.node = gz.nodeIndexToRelative(node),
9514	.operand = operand,
9515	});
9516	return rvalue(gz, ri, result, node);
9517	},
9518	}
9519	}
9520
9521	fn hasDeclOrField(
9522	gz: *GenZir,
9523	scope: *Scope,
9524	ri: ResultInfo,
9525	node: Ast.Node.Index,
9526	lhs_node: Ast.Node.Index,
9527	rhs_node: Ast.Node.Index,
9528	tag: Zir.Inst.Tag,
9529	) InnerError!Zir.Inst.Ref {
9530	const container_type = try typeExpr(gz, scope, lhs_node);
9531	const name = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .slice_const_u8_type } }, rhs_node);
9532	const result = try gz.addPlNode(tag, node, Zir.Inst.Bin{
9533	.lhs = container_type,
9534	.rhs = name,
9535	});
9536	return rvalue(gz, ri, result, node);
9537	}
9538
9539	fn typeCast(
9540	gz: *GenZir,
9541	scope: *Scope,
9542	ri: ResultInfo,
9543	node: Ast.Node.Index,
9544	operand_node: Ast.Node.Index,
9545	tag: Zir.Inst.Tag,
9546	builtin_name: []const u8,
9547	) InnerError!Zir.Inst.Ref {
9548	const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
9549	const result_type = try ri.rl.resultTypeForCast(gz, node, builtin_name);
9550	const operand = try expr(gz, scope, .{ .rl = .none }, operand_node);
9551
9552	try emitDbgStmt(gz, cursor);
9553	const result = try gz.addPlNode(tag, node, Zir.Inst.Bin{
9554	.lhs = result_type,
9555	.rhs = operand,
9556	});
9557	return rvalue(gz, ri, result, node);
9558	}
9559
9560	fn simpleUnOpType(
9561	gz: *GenZir,
9562	scope: *Scope,
9563	ri: ResultInfo,
9564	node: Ast.Node.Index,
9565	operand_node: Ast.Node.Index,
9566	tag: Zir.Inst.Tag,
9567	) InnerError!Zir.Inst.Ref {
9568	const operand = try typeExpr(gz, scope, operand_node);
9569	const result = try gz.addUnNode(tag, operand, node);
9570	return rvalue(gz, ri, result, node);
9571	}
9572
9573	fn simpleUnOp(
9574	gz: *GenZir,
9575	scope: *Scope,
9576	ri: ResultInfo,
9577	node: Ast.Node.Index,
9578	operand_ri: ResultInfo,
9579	operand_node: Ast.Node.Index,
9580	tag: Zir.Inst.Tag,
9581	) InnerError!Zir.Inst.Ref {
9582	const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
9583	const operand = if (tag == .compile_error)
9584	try comptimeExpr(gz, scope, operand_ri, operand_node)
9585	else
9586	try expr(gz, scope, operand_ri, operand_node);
9587	switch (tag) {
9588	.tag_name, .error_name, .int_from_ptr => try emitDbgStmt(gz, cursor),
9589	else => {},
9590	}
9591	const result = try gz.addUnNode(tag, operand, node);
9592	return rvalue(gz, ri, result, node);
9593	}
9594
9595	fn negation(
9596	gz: *GenZir,
9597	scope: *Scope,
9598	ri: ResultInfo,
9599	node: Ast.Node.Index,
9600	) InnerError!Zir.Inst.Ref {
9601	const astgen = gz.astgen;
9602	const tree = astgen.tree;
9603	const node_tags = tree.nodes.items(.tag);
9604	const node_datas = tree.nodes.items(.data);
9605
9606	// Check for float literal as the sub-expression because we want to preserve
9607	// its negativity rather than having it go through comptime subtraction.
9608	const operand_node = node_datas[node].lhs;
9609	if (node_tags[operand_node] == .number_literal) {
9610	return numberLiteral(gz, ri, operand_node, node, .negative);
9611	}
9612
9613	const operand = try expr(gz, scope, .{ .rl = .none }, operand_node);
9614	const result = try gz.addUnNode(.negate, operand, node);
9615	return rvalue(gz, ri, result, node);
9616	}
9617
9618	fn cmpxchg(
9619	gz: *GenZir,
9620	scope: *Scope,
9621	ri: ResultInfo,
9622	node: Ast.Node.Index,
9623	params: []const Ast.Node.Index,
9624	small: u16,
9625	) InnerError!Zir.Inst.Ref {
9626	const int_type = try typeExpr(gz, scope, params[0]);
9627	const result = try gz.addExtendedPayloadSmall(.cmpxchg, small, Zir.Inst.Cmpxchg{
9628	// zig fmt: off
9629	.node = gz.nodeIndexToRelative(node),
9630	.ptr = try expr(gz, scope, .{ .rl = .none }, params[1]),
9631	.expected_value = try expr(gz, scope, .{ .rl = .{ .ty = int_type } }, params[2]),
9632	.new_value = try expr(gz, scope, .{ .rl = .{ .coerced_ty = int_type } }, params[3]),
9633	.success_order = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .atomic_order_type } }, params[4]),
9634	.failure_order = try expr(gz, scope, .{ .rl = .{ .coerced_ty = .atomic_order_type } }, params[5]),
9635	// zig fmt: on
9636	});
9637	return rvalue(gz, ri, result, node);
9638	}
9639
9640	fn bitBuiltin(
9641	gz: *GenZir,
9642	scope: *Scope,
9643	ri: ResultInfo,
9644	node: Ast.Node.Index,
9645	operand_node: Ast.Node.Index,
9646	tag: Zir.Inst.Tag,
9647	) InnerError!Zir.Inst.Ref {
9648	const operand = try expr(gz, scope, .{ .rl = .none }, operand_node);
9649	const result = try gz.addUnNode(tag, operand, node);
9650	return rvalue(gz, ri, result, node);
9651	}
9652
9653	fn divBuiltin(
9654	gz: *GenZir,
9655	scope: *Scope,
9656	ri: ResultInfo,
9657	node: Ast.Node.Index,
9658	lhs_node: Ast.Node.Index,
9659	rhs_node: Ast.Node.Index,
9660	tag: Zir.Inst.Tag,
9661	) InnerError!Zir.Inst.Ref {
9662	const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
9663	const lhs = try expr(gz, scope, .{ .rl = .none }, lhs_node);
9664	const rhs = try expr(gz, scope, .{ .rl = .none }, rhs_node);
9665
9666	try emitDbgStmt(gz, cursor);
9667	const result = try gz.addPlNode(tag, node, Zir.Inst.Bin{ .lhs = lhs, .rhs = rhs });
9668	return rvalue(gz, ri, result, node);
9669	}
9670
9671	fn simpleCBuiltin(
9672	gz: *GenZir,
9673	scope: *Scope,
9674	ri: ResultInfo,
9675	node: Ast.Node.Index,
9676	operand_node: Ast.Node.Index,
9677	tag: Zir.Inst.Extended,
9678	) InnerError!Zir.Inst.Ref {
9679	const name: []const u8 = if (tag == .c_undef) "C undef" else "C include";
9680	if (!gz.c_import) return gz.astgen.failNode(node, "{s} valid only inside C import block", .{name});
9681	const operand = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .slice_const_u8_type } }, operand_node);
9682	_ = try gz.addExtendedPayload(tag, Zir.Inst.UnNode{
9683	.node = gz.nodeIndexToRelative(node),
9684	.operand = operand,
9685	});
9686	return rvalue(gz, ri, .void_value, node);
9687	}
9688
9689	fn offsetOf(
9690	gz: *GenZir,
9691	scope: *Scope,
9692	ri: ResultInfo,
9693	node: Ast.Node.Index,
9694	lhs_node: Ast.Node.Index,
9695	rhs_node: Ast.Node.Index,
9696	tag: Zir.Inst.Tag,
9697	) InnerError!Zir.Inst.Ref {
9698	const type_inst = try typeExpr(gz, scope, lhs_node);
9699	const field_name = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .slice_const_u8_type } }, rhs_node);
9700	const result = try gz.addPlNode(tag, node, Zir.Inst.Bin{
9701	.lhs = type_inst,
9702	.rhs = field_name,
9703	});
9704	return rvalue(gz, ri, result, node);
9705	}
9706
9707	fn shiftOp(
9708	gz: *GenZir,
9709	scope: *Scope,
9710	ri: ResultInfo,
9711	node: Ast.Node.Index,
9712	lhs_node: Ast.Node.Index,
9713	rhs_node: Ast.Node.Index,
9714	tag: Zir.Inst.Tag,
9715	) InnerError!Zir.Inst.Ref {
9716	const lhs = try expr(gz, scope, .{ .rl = .none }, lhs_node);
9717
9718	const cursor = switch (gz.astgen.tree.nodes.items(.tag)[node]) {
9719	.shl, .shr => maybeAdvanceSourceCursorToMainToken(gz, node),
9720	else => undefined,
9721	};
9722
9723	const log2_int_type = try gz.addUnNode(.typeof_log2_int_type, lhs, lhs_node);
9724	const rhs = try expr(gz, scope, .{ .rl = .{ .ty = log2_int_type }, .ctx = .shift_op }, rhs_node);
9725
9726	switch (gz.astgen.tree.nodes.items(.tag)[node]) {
9727	.shl, .shr => try emitDbgStmt(gz, cursor),
9728	else => undefined,
9729	}
9730
9731	const result = try gz.addPlNode(tag, node, Zir.Inst.Bin{
9732	.lhs = lhs,
9733	.rhs = rhs,
9734	});
9735	return rvalue(gz, ri, result, node);
9736	}
9737
9738	fn cImport(
9739	gz: *GenZir,
9740	scope: *Scope,
9741	node: Ast.Node.Index,
9742	body_node: Ast.Node.Index,
9743	) InnerError!Zir.Inst.Ref {
9744	const astgen = gz.astgen;
9745	const gpa = astgen.gpa;
9746
9747	if (gz.c_import) return gz.astgen.failNode(node, "cannot nest @cImport", .{});
9748
9749	var block_scope = gz.makeSubBlock(scope);
9750	block_scope.is_comptime = true;
9751	block_scope.c_import = true;
9752	defer block_scope.unstack();
9753
9754	const block_inst = try gz.makeBlockInst(.c_import, node);
9755	const block_result = try expr(&block_scope, &block_scope.base, .{ .rl = .none }, body_node);
9756	_ = try gz.addUnNode(.ensure_result_used, block_result, node);
9757	if (!gz.refIsNoReturn(block_result)) {
9758	_ = try block_scope.addBreak(.break_inline, block_inst, .void_value);
9759	}
9760	try block_scope.setBlockBody(block_inst);
9761	// block_scope unstacked now, can add new instructions to gz
9762	try gz.instructions.append(gpa, block_inst);
9763
9764	return block_inst.toRef();
9765	}
9766
9767	fn overflowArithmetic(
9768	gz: *GenZir,
9769	scope: *Scope,
9770	ri: ResultInfo,
9771	node: Ast.Node.Index,
9772	params: []const Ast.Node.Index,
9773	tag: Zir.Inst.Extended,
9774	) InnerError!Zir.Inst.Ref {
9775	const lhs = try expr(gz, scope, .{ .rl = .none }, params[0]);
9776	const rhs = try expr(gz, scope, .{ .rl = .none }, params[1]);
9777	const result = try gz.addExtendedPayload(tag, Zir.Inst.BinNode{
9778	.node = gz.nodeIndexToRelative(node),
9779	.lhs = lhs,
9780	.rhs = rhs,
9781	});
9782	return rvalue(gz, ri, result, node);
9783	}
9784
9785	fn callExpr(
9786	gz: *GenZir,
9787	scope: *Scope,
9788	ri: ResultInfo,
9789	node: Ast.Node.Index,
9790	call: Ast.full.Call,
9791	) InnerError!Zir.Inst.Ref {
9792	const astgen = gz.astgen;
9793
9794	const callee = try calleeExpr(gz, scope, call.ast.fn_expr);
9795	const modifier: std.builtin.CallModifier = blk: {
9796	if (gz.is_comptime) {
9797	break :blk .compile_time;
9798	}
9799	if (call.async_token != null) {
9800	break :blk .async_kw;
9801	}
9802	if (gz.nosuspend_node != 0) {
9803	break :blk .no_async;
9804	}
9805	break :blk .auto;
9806	};
9807
9808	{
9809	astgen.advanceSourceCursor(astgen.tree.tokens.items(.start)[call.ast.lparen]);
9810	const line = astgen.source_line - gz.decl_line;
9811	const column = astgen.source_column;
9812	// Sema expects a dbg_stmt immediately before call,
9813	try emitDbgStmtForceCurrentIndex(gz, .{ line, column });
9814	}
9815
9816	switch (callee) {
9817	.direct => \|obj\| assert(obj != .none),
9818	.field => \|field\| assert(field.obj_ptr != .none),
9819	}
9820	assert(node != 0);
9821
9822	const call_index: Zir.Inst.Index = @enumFromInt(astgen.instructions.len);
9823	const call_inst = call_index.toRef();
9824	try gz.astgen.instructions.append(astgen.gpa, undefined);
9825	try gz.instructions.append(astgen.gpa, call_index);
9826
9827	const scratch_top = astgen.scratch.items.len;
9828	defer astgen.scratch.items.len = scratch_top;
9829
9830	var scratch_index = scratch_top;
9831	try astgen.scratch.resize(astgen.gpa, scratch_top + call.ast.params.len);
9832
9833	for (call.ast.params) \|param_node\| {
9834	var arg_block = gz.makeSubBlock(scope);
9835	defer arg_block.unstack();
9836
9837	// `call_inst` is reused to provide the param type.
9838	const arg_ref = try expr(&arg_block, &arg_block.base, .{ .rl = .{ .coerced_ty = call_inst }, .ctx = .fn_arg }, param_node);
9839	_ = try arg_block.addBreakWithSrcNode(.break_inline, call_index, arg_ref, param_node);
9840
9841	const body = arg_block.instructionsSlice();
9842	try astgen.scratch.ensureUnusedCapacity(astgen.gpa, countBodyLenAfterFixups(astgen, body));
9843	appendBodyWithFixupsArrayList(astgen, &astgen.scratch, body);
9844
9845	astgen.scratch.items[scratch_index] = @intCast(astgen.scratch.items.len - scratch_top);
9846	scratch_index += 1;
9847	}
9848
9849	// If our result location is a try/catch/error-union-if/return, a function argument,
9850	// or an initializer for a `const` variable, the error trace propagates.
9851	// Otherwise, it should always be popped (handled in Sema).
9852	const propagate_error_trace = switch (ri.ctx) {
9853	.error_handling_expr, .@"return", .fn_arg, .const_init => true,
9854	else => false,
9855	};
9856
9857	switch (callee) {
9858	.direct => \|callee_obj\| {
9859	const payload_index = try addExtra(astgen, Zir.Inst.Call{
9860	.callee = callee_obj,
9861	.flags = .{
9862	.pop_error_return_trace = !propagate_error_trace,
9863	.packed_modifier = @intCast(@intFromEnum(modifier)),
9864	.args_len = @intCast(call.ast.params.len),
9865	},
9866	});
9867	if (call.ast.params.len != 0) {
9868	try astgen.extra.appendSlice(astgen.gpa, astgen.scratch.items[scratch_top..]);
9869	}
9870	gz.astgen.instructions.set(@intFromEnum(call_index), .{
9871	.tag = .call,
9872	.data = .{ .pl_node = .{
9873	.src_node = gz.nodeIndexToRelative(node),
9874	.payload_index = payload_index,
9875	} },
9876	});
9877	},
9878	.field => \|callee_field\| {
9879	const payload_index = try addExtra(astgen, Zir.Inst.FieldCall{
9880	.obj_ptr = callee_field.obj_ptr,
9881	.field_name_start = callee_field.field_name_start,
9882	.flags = .{
9883	.pop_error_return_trace = !propagate_error_trace,
9884	.packed_modifier = @intCast(@intFromEnum(modifier)),
9885	.args_len = @intCast(call.ast.params.len),
9886	},
9887	});
9888	if (call.ast.params.len != 0) {
9889	try astgen.extra.appendSlice(astgen.gpa, astgen.scratch.items[scratch_top..]);
9890	}
9891	gz.astgen.instructions.set(@intFromEnum(call_index), .{
9892	.tag = .field_call,
9893	.data = .{ .pl_node = .{
9894	.src_node = gz.nodeIndexToRelative(node),
9895	.payload_index = payload_index,
9896	} },
9897	});
9898	},
9899	}
9900	return rvalue(gz, ri, call_inst, node); // TODO function call with result location
9901	}
9902
9903	const Callee = union(enum) {
9904	field: struct {
9905	/// A pointer to the object the field is fetched on, so that we can
9906	/// promote the lvalue to an address if the first parameter requires it.
9907	obj_ptr: Zir.Inst.Ref,
9908	/// Offset into `string_bytes`.
9909	field_name_start: Zir.NullTerminatedString,
9910	},
9911	direct: Zir.Inst.Ref,
9912	};
9913
9914	/// calleeExpr generates the function part of a call expression (f in f(x)), but
9915	/// not the callee argument to the @call() builtin. Its purpose is to
9916	/// distinguish between standard calls and method call syntax `a.b()`. Thus, if
9917	/// the lhs is a field access, we return using the `field` union field;
9918	/// otherwise, we use the `direct` union field.
9919	fn calleeExpr(
9920	gz: *GenZir,
9921	scope: *Scope,
9922	node: Ast.Node.Index,
9923	) InnerError!Callee {
9924	const astgen = gz.astgen;
9925	const tree = astgen.tree;
9926
9927	const tag = tree.nodes.items(.tag)[node];
9928	switch (tag) {
9929	.field_access => {
9930	const main_tokens = tree.nodes.items(.main_token);
9931	const node_datas = tree.nodes.items(.data);
9932	const object_node = node_datas[node].lhs;
9933	const dot_token = main_tokens[node];
9934	const field_ident = dot_token + 1;
9935	const str_index = try astgen.identAsString(field_ident);
9936	// Capture the object by reference so we can promote it to an
9937	// address in Sema if needed.
9938	const lhs = try expr(gz, scope, .{ .rl = .ref }, object_node);
9939
9940	const cursor = maybeAdvanceSourceCursorToMainToken(gz, node);
9941	try emitDbgStmt(gz, cursor);
9942
9943	return .{ .field = .{
9944	.obj_ptr = lhs,
9945	.field_name_start = str_index,
9946	} };
9947	},
9948	else => return .{ .direct = try expr(gz, scope, .{ .rl = .none }, node) },
9949	}
9950	}
9951
9952	const primitive_instrs = std.ComptimeStringMap(Zir.Inst.Ref, .{
9953	.{ "anyerror", .anyerror_type },
9954	.{ "anyframe", .anyframe_type },
9955	.{ "anyopaque", .anyopaque_type },
9956	.{ "bool", .bool_type },
9957	.{ "c_int", .c_int_type },
9958	.{ "c_long", .c_long_type },
9959	.{ "c_longdouble", .c_longdouble_type },
9960	.{ "c_longlong", .c_longlong_type },
9961	.{ "c_char", .c_char_type },
9962	.{ "c_short", .c_short_type },
9963	.{ "c_uint", .c_uint_type },
9964	.{ "c_ulong", .c_ulong_type },
9965	.{ "c_ulonglong", .c_ulonglong_type },
9966	.{ "c_ushort", .c_ushort_type },
9967	.{ "comptime_float", .comptime_float_type },
9968	.{ "comptime_int", .comptime_int_type },
9969	.{ "f128", .f128_type },
9970	.{ "f16", .f16_type },
9971	.{ "f32", .f32_type },
9972	.{ "f64", .f64_type },
9973	.{ "f80", .f80_type },
9974	.{ "false", .bool_false },
9975	.{ "i16", .i16_type },
9976	.{ "i32", .i32_type },
9977	.{ "i64", .i64_type },
9978	.{ "i128", .i128_type },
9979	.{ "i8", .i8_type },
9980	.{ "isize", .isize_type },
9981	.{ "noreturn", .noreturn_type },
9982	.{ "null", .null_value },
9983	.{ "true", .bool_true },
9984	.{ "type", .type_type },
9985	.{ "u16", .u16_type },
9986	.{ "u29", .u29_type },
9987	.{ "u32", .u32_type },
9988	.{ "u64", .u64_type },
9989	.{ "u128", .u128_type },
9990	.{ "u1", .u1_type },
9991	.{ "u8", .u8_type },
9992	.{ "undefined", .undef },
9993	.{ "usize", .usize_type },
9994	.{ "void", .void_type },
9995	});
9996
9997	comptime {
9998	// These checks ensure that std.zig.primitives stays in sync with the primitive->Zir map.
9999	const primitives = std.zig.primitives;
10000	for (primitive_instrs.kvs) \|kv\| {
10001	if (!primitives.isPrimitive(kv.key)) {
10002	@compileError("std.zig.isPrimitive() is not aware of Zir instr '" ++ @tagName(kv.value) ++ "'");
10003	}
10004	}
10005	for (primitives.names.kvs) \|kv\| {
10006	if (primitive_instrs.get(kv.key) == null) {
10007	@compileError("std.zig.primitives entry '" ++ kv.key ++ "' does not have a corresponding Zir instr");
10008	}
10009	}
10010	}
10011
10012	fn nodeIsTriviallyZero(tree: *const Ast, node: Ast.Node.Index) bool {
10013	const node_tags = tree.nodes.items(.tag);
10014	const main_tokens = tree.nodes.items(.main_token);
10015
10016	switch (node_tags[node]) {
10017	.number_literal => {
10018	const ident = main_tokens[node];
10019	return switch (std.zig.parseNumberLiteral(tree.tokenSlice(ident))) {
10020	.int => \|number\| switch (number) {
10021	0 => true,
10022	else => false,
10023	},
10024	else => false,
10025	};
10026	},
10027	else => return false,
10028	}
10029	}
10030
10031	fn nodeMayAppendToErrorTrace(tree: *const Ast, start_node: Ast.Node.Index) bool {
10032	const node_tags = tree.nodes.items(.tag);
10033	const node_datas = tree.nodes.items(.data);
10034
10035	var node = start_node;
10036	while (true) {
10037	switch (node_tags[node]) {
10038	// These don't have the opportunity to call any runtime functions.
10039	.error_value,
10040	.identifier,
10041	.@"comptime",
10042	=> return false,
10043
10044	// Forward the question to the LHS sub-expression.
10045	.grouped_expression,
10046	.@"try",
10047	.@"nosuspend",
10048	.unwrap_optional,
10049	=> node = node_datas[node].lhs,
10050
10051	// Anything that does not eval to an error is guaranteed to pop any
10052	// additions to the error trace, so it effectively does not append.
10053	else => return nodeMayEvalToError(tree, start_node) != .never,
10054	}
10055	}
10056	}
10057
10058	fn nodeMayEvalToError(tree: *const Ast, start_node: Ast.Node.Index) BuiltinFn.EvalToError {
10059	const node_tags = tree.nodes.items(.tag);
10060	const node_datas = tree.nodes.items(.data);
10061	const main_tokens = tree.nodes.items(.main_token);
10062	const token_tags = tree.tokens.items(.tag);
10063
10064	var node = start_node;
10065	while (true) {
10066	switch (node_tags[node]) {
10067	.root,
10068	.@"usingnamespace",
10069	.test_decl,
10070	.switch_case,
10071	.switch_case_inline,
10072	.switch_case_one,
10073	.switch_case_inline_one,
10074	.container_field_init,
10075	.container_field_align,
10076	.container_field,
10077	.asm_output,
10078	.asm_input,
10079	=> unreachable,
10080
10081	.error_value => return .always,
10082
10083	.@"asm",
10084	.asm_simple,
10085	.identifier,
10086	.field_access,
10087	.deref,
10088	.array_access,
10089	.while_simple,
10090	.while_cont,
10091	.for_simple,
10092	.if_simple,
10093	.@"while",
10094	.@"if",
10095	.@"for",
10096	.@"switch",
10097	.switch_comma,
10098	.call_one,
10099	.call_one_comma,
10100	.async_call_one,
10101	.async_call_one_comma,
10102	.call,
10103	.call_comma,
10104	.async_call,
10105	.async_call_comma,
10106	=> return .maybe,
10107
10108	.@"return",
10109	.@"break",
10110	.@"continue",
10111	.bit_not,
10112	.bool_not,
10113	.global_var_decl,
10114	.local_var_decl,
10115	.simple_var_decl,
10116	.aligned_var_decl,
10117	.@"defer",
10118	.@"errdefer",
10119	.address_of,
10120	.optional_type,
10121	.negation,
10122	.negation_wrap,
10123	.@"resume",
10124	.array_type,
10125	.array_type_sentinel,
10126	.ptr_type_aligned,
10127	.ptr_type_sentinel,
10128	.ptr_type,
10129	.ptr_type_bit_range,
10130	.@"suspend",
10131	.fn_proto_simple,
10132	.fn_proto_multi,
10133	.fn_proto_one,
10134	.fn_proto,
10135	.fn_decl,
10136	.anyframe_type,
10137	.anyframe_literal,
10138	.number_literal,
10139	.enum_literal,
10140	.string_literal,
10141	.multiline_string_literal,
10142	.char_literal,
10143	.unreachable_literal,
10144	.error_set_decl,
10145	.container_decl,
10146	.container_decl_trailing,
10147	.container_decl_two,
10148	.container_decl_two_trailing,
10149	.container_decl_arg,
10150	.container_decl_arg_trailing,
10151	.tagged_union,
10152	.tagged_union_trailing,
10153	.tagged_union_two,
10154	.tagged_union_two_trailing,
10155	.tagged_union_enum_tag,
10156	.tagged_union_enum_tag_trailing,
10157	.add,
10158	.add_wrap,
10159	.add_sat,
10160	.array_cat,
10161	.array_mult,
10162	.assign,
10163	.assign_destructure,
10164	.assign_bit_and,
10165	.assign_bit_or,
10166	.assign_shl,
10167	.assign_shl_sat,
10168	.assign_shr,
10169	.assign_bit_xor,
10170	.assign_div,
10171	.assign_sub,
10172	.assign_sub_wrap,
10173	.assign_sub_sat,
10174	.assign_mod,
10175	.assign_add,
10176	.assign_add_wrap,
10177	.assign_add_sat,
10178	.assign_mul,
10179	.assign_mul_wrap,
10180	.assign_mul_sat,
10181	.bang_equal,
10182	.bit_and,
10183	.bit_or,
10184	.shl,
10185	.shl_sat,
10186	.shr,
10187	.bit_xor,
10188	.bool_and,
10189	.bool_or,
10190	.div,
10191	.equal_equal,
10192	.error_union,
10193	.greater_or_equal,
10194	.greater_than,
10195	.less_or_equal,
10196	.less_than,
10197	.merge_error_sets,
10198	.mod,
10199	.mul,
10200	.mul_wrap,
10201	.mul_sat,
10202	.switch_range,
10203	.for_range,
10204	.sub,
10205	.sub_wrap,
10206	.sub_sat,
10207	.slice,
10208	.slice_open,
10209	.slice_sentinel,
10210	.array_init_one,
10211	.array_init_one_comma,
10212	.array_init_dot_two,
10213	.array_init_dot_two_comma,
10214	.array_init_dot,
10215	.array_init_dot_comma,
10216	.array_init,
10217	.array_init_comma,
10218	.struct_init_one,
10219	.struct_init_one_comma,
10220	.struct_init_dot_two,
10221	.struct_init_dot_two_comma,
10222	.struct_init_dot,
10223	.struct_init_dot_comma,
10224	.struct_init,
10225	.struct_init_comma,
10226	=> return .never,
10227
10228	// Forward the question to the LHS sub-expression.
10229	.grouped_expression,
10230	.@"try",
10231	.@"await",
10232	.@"comptime",
10233	.@"nosuspend",
10234	.unwrap_optional,
10235	=> node = node_datas[node].lhs,
10236
10237	// LHS sub-expression may still be an error under the outer optional or error union
10238	.@"catch",
10239	.@"orelse",
10240	=> return .maybe,
10241
10242	.block_two,
10243	.block_two_semicolon,
10244	.block,
10245	.block_semicolon,
10246	=> {
10247	const lbrace = main_tokens[node];
10248	if (token_tags[lbrace - 1] == .colon) {
10249	// Labeled blocks may need a memory location to forward
10250	// to their break statements.
10251	return .maybe;
10252	} else {
10253	return .never;
10254	}
10255	},
10256
10257	.builtin_call,
10258	.builtin_call_comma,
10259	.builtin_call_two,
10260	.builtin_call_two_comma,
10261	=> {
10262	const builtin_token = main_tokens[node];
10263	const builtin_name = tree.tokenSlice(builtin_token);
10264	// If the builtin is an invalid name, we don't cause an error here; instead
10265	// let it pass, and the error will be "invalid builtin function" later.
10266	const builtin_info = BuiltinFn.list.get(builtin_name) orelse return .maybe;
10267	return builtin_info.eval_to_error;
10268	},
10269	}
10270	}
10271	}
10272
10273	/// Returns `true` if it is known the type expression has more than one possible value;
10274	/// `false` otherwise.
10275	fn nodeImpliesMoreThanOnePossibleValue(tree: *const Ast, start_node: Ast.Node.Index) bool {
10276	const node_tags = tree.nodes.items(.tag);
10277	const node_datas = tree.nodes.items(.data);
10278
10279	var node = start_node;
10280	while (true) {
10281	switch (node_tags[node]) {
10282	.root,
10283	.@"usingnamespace",
10284	.test_decl,
10285	.switch_case,
10286	.switch_case_inline,
10287	.switch_case_one,
10288	.switch_case_inline_one,
10289	.container_field_init,
10290	.container_field_align,
10291	.container_field,
10292	.asm_output,
10293	.asm_input,
10294	.global_var_decl,
10295	.local_var_decl,
10296	.simple_var_decl,
10297	.aligned_var_decl,
10298	=> unreachable,
10299
10300	.@"return",
10301	.@"break",
10302	.@"continue",
10303	.bit_not,
10304	.bool_not,
10305	.@"defer",
10306	.@"errdefer",
10307	.address_of,
10308	.negation,
10309	.negation_wrap,
10310	.@"resume",
10311	.array_type,
10312	.@"suspend",
10313	.fn_decl,
10314	.anyframe_literal,
10315	.number_literal,
10316	.enum_literal,
10317	.string_literal,
10318	.multiline_string_literal,
10319	.char_literal,
10320	.unreachable_literal,
10321	.error_set_decl,
10322	.container_decl,
10323	.container_decl_trailing,
10324	.container_decl_two,
10325	.container_decl_two_trailing,
10326	.container_decl_arg,
10327	.container_decl_arg_trailing,
10328	.tagged_union,
10329	.tagged_union_trailing,
10330	.tagged_union_two,
10331	.tagged_union_two_trailing,
10332	.tagged_union_enum_tag,
10333	.tagged_union_enum_tag_trailing,
10334	.@"asm",
10335	.asm_simple,
10336	.add,
10337	.add_wrap,
10338	.add_sat,
10339	.array_cat,
10340	.array_mult,
10341	.assign,
10342	.assign_destructure,
10343	.assign_bit_and,
10344	.assign_bit_or,
10345	.assign_shl,
10346	.assign_shl_sat,
10347	.assign_shr,
10348	.assign_bit_xor,
10349	.assign_div,
10350	.assign_sub,
10351	.assign_sub_wrap,
10352	.assign_sub_sat,
10353	.assign_mod,
10354	.assign_add,
10355	.assign_add_wrap,
10356	.assign_add_sat,
10357	.assign_mul,
10358	.assign_mul_wrap,
10359	.assign_mul_sat,
10360	.bang_equal,
10361	.bit_and,
10362	.bit_or,
10363	.shl,
10364	.shl_sat,
10365	.shr,
10366	.bit_xor,
10367	.bool_and,
10368	.bool_or,
10369	.div,
10370	.equal_equal,
10371	.error_union,
10372	.greater_or_equal,
10373	.greater_than,
10374	.less_or_equal,
10375	.less_than,
10376	.merge_error_sets,
10377	.mod,
10378	.mul,
10379	.mul_wrap,
10380	.mul_sat,
10381	.switch_range,
10382	.for_range,
10383	.field_access,
10384	.sub,
10385	.sub_wrap,
10386	.sub_sat,
10387	.slice,
10388	.slice_open,
10389	.slice_sentinel,
10390	.deref,
10391	.array_access,
10392	.error_value,
10393	.while_simple,
10394	.while_cont,
10395	.for_simple,
10396	.if_simple,
10397	.@"catch",
10398	.@"orelse",
10399	.array_init_one,
10400	.array_init_one_comma,
10401	.array_init_dot_two,
10402	.array_init_dot_two_comma,
10403	.array_init_dot,
10404	.array_init_dot_comma,
10405	.array_init,
10406	.array_init_comma,
10407	.struct_init_one,
10408	.struct_init_one_comma,
10409	.struct_init_dot_two,
10410	.struct_init_dot_two_comma,
10411	.struct_init_dot,
10412	.struct_init_dot_comma,
10413	.struct_init,
10414	.struct_init_comma,
10415	.@"while",
10416	.@"if",
10417	.@"for",
10418	.@"switch",
10419	.switch_comma,
10420	.call_one,
10421	.call_one_comma,
10422	.async_call_one,
10423	.async_call_one_comma,
10424	.call,
10425	.call_comma,
10426	.async_call,
10427	.async_call_comma,
10428	.block_two,
10429	.block_two_semicolon,
10430	.block,
10431	.block_semicolon,
10432	.builtin_call,
10433	.builtin_call_comma,
10434	.builtin_call_two,
10435	.builtin_call_two_comma,
10436	// these are function bodies, not pointers
10437	.fn_proto_simple,
10438	.fn_proto_multi,
10439	.fn_proto_one,
10440	.fn_proto,
10441	=> return false,
10442
10443	// Forward the question to the LHS sub-expression.
10444	.grouped_expression,
10445	.@"try",
10446	.@"await",
10447	.@"comptime",
10448	.@"nosuspend",
10449	.unwrap_optional,
10450	=> node = node_datas[node].lhs,
10451
10452	.ptr_type_aligned,
10453	.ptr_type_sentinel,
10454	.ptr_type,
10455	.ptr_type_bit_range,
10456	.optional_type,
10457	.anyframe_type,
10458	.array_type_sentinel,
10459	=> return true,
10460
10461	.identifier => {
10462	const main_tokens = tree.nodes.items(.main_token);
10463	const ident_bytes = tree.tokenSlice(main_tokens[node]);
10464	if (primitive_instrs.get(ident_bytes)) \|primitive\| switch (primitive) {
10465	.anyerror_type,
10466	.anyframe_type,
10467	.anyopaque_type,
10468	.bool_type,
10469	.c_int_type,
10470	.c_long_type,
10471	.c_longdouble_type,
10472	.c_longlong_type,
10473	.c_char_type,
10474	.c_short_type,
10475	.c_uint_type,
10476	.c_ulong_type,
10477	.c_ulonglong_type,
10478	.c_ushort_type,
10479	.comptime_float_type,
10480	.comptime_int_type,
10481	.f16_type,
10482	.f32_type,
10483	.f64_type,
10484	.f80_type,
10485	.f128_type,
10486	.i16_type,
10487	.i32_type,
10488	.i64_type,
10489	.i128_type,
10490	.i8_type,
10491	.isize_type,
10492	.type_type,
10493	.u16_type,
10494	.u29_type,
10495	.u32_type,
10496	.u64_type,
10497	.u128_type,
10498	.u1_type,
10499	.u8_type,
10500	.usize_type,
10501	=> return true,
10502
10503	.void_type,
10504	.bool_false,
10505	.bool_true,
10506	.null_value,
10507	.undef,
10508	.noreturn_type,
10509	=> return false,
10510
10511	else => unreachable, // that's all the values from `primitives`.
10512	} else {
10513	return false;
10514	}
10515	},
10516	}
10517	}
10518	}
10519
10520	/// Returns `true` if it is known the expression is a type that cannot be used at runtime;
10521	/// `false` otherwise.
10522	fn nodeImpliesComptimeOnly(tree: *const Ast, start_node: Ast.Node.Index) bool {
10523	const node_tags = tree.nodes.items(.tag);
10524	const node_datas = tree.nodes.items(.data);
10525
10526	var node = start_node;
10527	while (true) {
10528	switch (node_tags[node]) {
10529	.root,
10530	.@"usingnamespace",
10531	.test_decl,
10532	.switch_case,
10533	.switch_case_inline,
10534	.switch_case_one,
10535	.switch_case_inline_one,
10536	.container_field_init,
10537	.container_field_align,
10538	.container_field,
10539	.asm_output,
10540	.asm_input,
10541	.global_var_decl,
10542	.local_var_decl,
10543	.simple_var_decl,
10544	.aligned_var_decl,
10545	=> unreachable,
10546
10547	.@"return",
10548	.@"break",
10549	.@"continue",
10550	.bit_not,
10551	.bool_not,
10552	.@"defer",
10553	.@"errdefer",
10554	.address_of,
10555	.negation,
10556	.negation_wrap,
10557	.@"resume",
10558	.array_type,
10559	.@"suspend",
10560	.fn_decl,
10561	.anyframe_literal,
10562	.number_literal,
10563	.enum_literal,
10564	.string_literal,
10565	.multiline_string_literal,
10566	.char_literal,
10567	.unreachable_literal,
10568	.error_set_decl,
10569	.container_decl,
10570	.container_decl_trailing,
10571	.container_decl_two,
10572	.container_decl_two_trailing,
10573	.container_decl_arg,
10574	.container_decl_arg_trailing,
10575	.tagged_union,
10576	.tagged_union_trailing,
10577	.tagged_union_two,
10578	.tagged_union_two_trailing,
10579	.tagged_union_enum_tag,
10580	.tagged_union_enum_tag_trailing,
10581	.@"asm",
10582	.asm_simple,
10583	.add,
10584	.add_wrap,
10585	.add_sat,
10586	.array_cat,
10587	.array_mult,
10588	.assign,
10589	.assign_destructure,
10590	.assign_bit_and,
10591	.assign_bit_or,
10592	.assign_shl,
10593	.assign_shl_sat,
10594	.assign_shr,
10595	.assign_bit_xor,
10596	.assign_div,
10597	.assign_sub,
10598	.assign_sub_wrap,
10599	.assign_sub_sat,
10600	.assign_mod,
10601	.assign_add,
10602	.assign_add_wrap,
10603	.assign_add_sat,
10604	.assign_mul,
10605	.assign_mul_wrap,
10606	.assign_mul_sat,
10607	.bang_equal,
10608	.bit_and,
10609	.bit_or,
10610	.shl,
10611	.shl_sat,
10612	.shr,
10613	.bit_xor,
10614	.bool_and,
10615	.bool_or,
10616	.div,
10617	.equal_equal,
10618	.error_union,
10619	.greater_or_equal,
10620	.greater_than,
10621	.less_or_equal,
10622	.less_than,
10623	.merge_error_sets,
10624	.mod,
10625	.mul,
10626	.mul_wrap,
10627	.mul_sat,
10628	.switch_range,
10629	.for_range,
10630	.field_access,
10631	.sub,
10632	.sub_wrap,
10633	.sub_sat,
10634	.slice,
10635	.slice_open,
10636	.slice_sentinel,
10637	.deref,
10638	.array_access,
10639	.error_value,
10640	.while_simple,
10641	.while_cont,
10642	.for_simple,
10643	.if_simple,
10644	.@"catch",
10645	.@"orelse",
10646	.array_init_one,
10647	.array_init_one_comma,
10648	.array_init_dot_two,
10649	.array_init_dot_two_comma,
10650	.array_init_dot,
10651	.array_init_dot_comma,
10652	.array_init,
10653	.array_init_comma,
10654	.struct_init_one,
10655	.struct_init_one_comma,
10656	.struct_init_dot_two,
10657	.struct_init_dot_two_comma,
10658	.struct_init_dot,
10659	.struct_init_dot_comma,
10660	.struct_init,
10661	.struct_init_comma,
10662	.@"while",
10663	.@"if",
10664	.@"for",
10665	.@"switch",
10666	.switch_comma,
10667	.call_one,
10668	.call_one_comma,
10669	.async_call_one,
10670	.async_call_one_comma,
10671	.call,
10672	.call_comma,
10673	.async_call,
10674	.async_call_comma,
10675	.block_two,
10676	.block_two_semicolon,
10677	.block,
10678	.block_semicolon,
10679	.builtin_call,
10680	.builtin_call_comma,
10681	.builtin_call_two,
10682	.builtin_call_two_comma,
10683	.ptr_type_aligned,
10684	.ptr_type_sentinel,
10685	.ptr_type,
10686	.ptr_type_bit_range,
10687	.optional_type,
10688	.anyframe_type,
10689	.array_type_sentinel,
10690	=> return false,
10691
10692	// these are function bodies, not pointers
10693	.fn_proto_simple,
10694	.fn_proto_multi,
10695	.fn_proto_one,
10696	.fn_proto,
10697	=> return true,
10698
10699	// Forward the question to the LHS sub-expression.
10700	.grouped_expression,
10701	.@"try",
10702	.@"await",
10703	.@"comptime",
10704	.@"nosuspend",
10705	.unwrap_optional,
10706	=> node = node_datas[node].lhs,
10707
10708	.identifier => {
10709	const main_tokens = tree.nodes.items(.main_token);
10710	const ident_bytes = tree.tokenSlice(main_tokens[node]);
10711	if (primitive_instrs.get(ident_bytes)) \|primitive\| switch (primitive) {
10712	.anyerror_type,
10713	.anyframe_type,
10714	.anyopaque_type,
10715	.bool_type,
10716	.c_int_type,
10717	.c_long_type,
10718	.c_longdouble_type,
10719	.c_longlong_type,
10720	.c_char_type,
10721	.c_short_type,
10722	.c_uint_type,
10723	.c_ulong_type,
10724	.c_ulonglong_type,
10725	.c_ushort_type,
10726	.f16_type,
10727	.f32_type,
10728	.f64_type,
10729	.f80_type,
10730	.f128_type,
10731	.i16_type,
10732	.i32_type,
10733	.i64_type,
10734	.i128_type,
10735	.i8_type,
10736	.isize_type,
10737	.u16_type,
10738	.u29_type,
10739	.u32_type,
10740	.u64_type,
10741	.u128_type,
10742	.u1_type,
10743	.u8_type,
10744	.usize_type,
10745	.void_type,
10746	.bool_false,
10747	.bool_true,
10748	.null_value,
10749	.undef,
10750	.noreturn_type,
10751	=> return false,
10752
10753	.comptime_float_type,
10754	.comptime_int_type,
10755	.type_type,
10756	=> return true,
10757
10758	else => unreachable, // that's all the values from `primitives`.
10759	} else {
10760	return false;
10761	}
10762	},
10763	}
10764	}
10765	}
10766
10767	/// Returns `true` if the node uses `gz.anon_name_strategy`.
10768	fn nodeUsesAnonNameStrategy(tree: *const Ast, node: Ast.Node.Index) bool {
10769	const node_tags = tree.nodes.items(.tag);
10770	switch (node_tags[node]) {
10771	.container_decl,
10772	.container_decl_trailing,
10773	.container_decl_two,
10774	.container_decl_two_trailing,
10775	.container_decl_arg,
10776	.container_decl_arg_trailing,
10777	.tagged_union,
10778	.tagged_union_trailing,
10779	.tagged_union_two,
10780	.tagged_union_two_trailing,
10781	.tagged_union_enum_tag,
10782	.tagged_union_enum_tag_trailing,
10783	=> return true,
10784	.builtin_call_two, .builtin_call_two_comma, .builtin_call, .builtin_call_comma => {
10785	const builtin_token = tree.nodes.items(.main_token)[node];
10786	const builtin_name = tree.tokenSlice(builtin_token);
10787	return std.mem.eql(u8, builtin_name, "@Type");
10788	},
10789	else => return false,
10790	}
10791	}
10792
10793	/// Applies `rl` semantics to `result`. Expressions which do not do their own handling of
10794	/// result locations must call this function on their result.
10795	/// As an example, if `ri.rl` is `.ptr`, it will write the result to the pointer.
10796	/// If `ri.rl` is `.ty`, it will coerce the result to the type.
10797	/// Assumes nothing stacked on `gz`.
10798	fn rvalue(
10799	gz: *GenZir,
10800	ri: ResultInfo,
10801	raw_result: Zir.Inst.Ref,
10802	src_node: Ast.Node.Index,
10803	) InnerError!Zir.Inst.Ref {
10804	return rvalueInner(gz, ri, raw_result, src_node, true);
10805	}
10806
10807	/// Like `rvalue`, but refuses to perform coercions before taking references for
10808	/// the `ref_coerced_ty` result type. This is used for local variables which do
10809	/// not have `alloc`s, because we want variables to have consistent addresses,
10810	/// i.e. we want them to act like lvalues.
10811	fn rvalueNoCoercePreRef(
10812	gz: *GenZir,
10813	ri: ResultInfo,
10814	raw_result: Zir.Inst.Ref,
10815	src_node: Ast.Node.Index,
10816	) InnerError!Zir.Inst.Ref {
10817	return rvalueInner(gz, ri, raw_result, src_node, false);
10818	}
10819
10820	fn rvalueInner(
10821	gz: *GenZir,
10822	ri: ResultInfo,
10823	raw_result: Zir.Inst.Ref,
10824	src_node: Ast.Node.Index,
10825	allow_coerce_pre_ref: bool,
10826	) InnerError!Zir.Inst.Ref {
10827	const result = r: {
10828	if (raw_result.toIndex()) \|result_index\| {
10829	const zir_tags = gz.astgen.instructions.items(.tag);
10830	const data = gz.astgen.instructions.items(.data)[@intFromEnum(result_index)];
10831	if (zir_tags[@intFromEnum(result_index)].isAlwaysVoid(data)) {
10832	break :r Zir.Inst.Ref.void_value;
10833	}
10834	}
10835	break :r raw_result;
10836	};
10837	if (gz.endsWithNoReturn()) return result;
10838	switch (ri.rl) {
10839	.none, .coerced_ty => return result,
10840	.discard => {
10841	// Emit a compile error for discarding error values.
10842	_ = try gz.addUnNode(.ensure_result_non_error, result, src_node);
10843	return .void_value;
10844	},
10845	.ref, .ref_coerced_ty => {
10846	const coerced_result = if (allow_coerce_pre_ref and ri.rl == .ref_coerced_ty) res: {
10847	const ptr_ty = ri.rl.ref_coerced_ty;
10848	break :res try gz.addPlNode(.coerce_ptr_elem_ty, src_node, Zir.Inst.Bin{
10849	.lhs = ptr_ty,
10850	.rhs = result,
10851	});
10852	} else result;
10853	// We need a pointer but we have a value.
10854	// Unfortunately it's not quite as simple as directly emitting a ref
10855	// instruction here because we need subsequent address-of operator on
10856	// const locals to return the same address.
10857	const astgen = gz.astgen;
10858	const tree = astgen.tree;
10859	const src_token = tree.firstToken(src_node);
10860	const result_index = coerced_result.toIndex() orelse
10861	return gz.addUnTok(.ref, coerced_result, src_token);
10862	const zir_tags = gz.astgen.instructions.items(.tag);
10863	if (zir_tags[@intFromEnum(result_index)].isParam() or astgen.isInferred(coerced_result))
10864	return gz.addUnTok(.ref, coerced_result, src_token);
10865	const gop = try astgen.ref_table.getOrPut(astgen.gpa, result_index);
10866	if (!gop.found_existing) {
10867	gop.value_ptr.* = try gz.makeUnTok(.ref, coerced_result, src_token);
10868	}
10869	return gop.value_ptr.*.toRef();
10870	},
10871	.ty => \|ty_inst\| {
10872	// Quickly eliminate some common, unnecessary type coercion.
10873	const as_ty = @as(u64, @intFromEnum(Zir.Inst.Ref.type_type)) << 32;
10874	const as_comptime_int = @as(u64, @intFromEnum(Zir.Inst.Ref.comptime_int_type)) << 32;
10875	const as_bool = @as(u64, @intFromEnum(Zir.Inst.Ref.bool_type)) << 32;
10876	const as_usize = @as(u64, @intFromEnum(Zir.Inst.Ref.usize_type)) << 32;
10877	const as_void = @as(u64, @intFromEnum(Zir.Inst.Ref.void_type)) << 32;
10878	switch ((@as(u64, @intFromEnum(ty_inst)) << 32) \| @as(u64, @intFromEnum(result))) {
10879	as_ty \| @intFromEnum(Zir.Inst.Ref.u1_type),
10880	as_ty \| @intFromEnum(Zir.Inst.Ref.u8_type),
10881	as_ty \| @intFromEnum(Zir.Inst.Ref.i8_type),
10882	as_ty \| @intFromEnum(Zir.Inst.Ref.u16_type),
10883	as_ty \| @intFromEnum(Zir.Inst.Ref.u29_type),
10884	as_ty \| @intFromEnum(Zir.Inst.Ref.i16_type),
10885	as_ty \| @intFromEnum(Zir.Inst.Ref.u32_type),
10886	as_ty \| @intFromEnum(Zir.Inst.Ref.i32_type),
10887	as_ty \| @intFromEnum(Zir.Inst.Ref.u64_type),
10888	as_ty \| @intFromEnum(Zir.Inst.Ref.i64_type),
10889	as_ty \| @intFromEnum(Zir.Inst.Ref.u128_type),
10890	as_ty \| @intFromEnum(Zir.Inst.Ref.i128_type),
10891	as_ty \| @intFromEnum(Zir.Inst.Ref.usize_type),
10892	as_ty \| @intFromEnum(Zir.Inst.Ref.isize_type),
10893	as_ty \| @intFromEnum(Zir.Inst.Ref.c_char_type),
10894	as_ty \| @intFromEnum(Zir.Inst.Ref.c_short_type),
10895	as_ty \| @intFromEnum(Zir.Inst.Ref.c_ushort_type),
10896	as_ty \| @intFromEnum(Zir.Inst.Ref.c_int_type),
10897	as_ty \| @intFromEnum(Zir.Inst.Ref.c_uint_type),
10898	as_ty \| @intFromEnum(Zir.Inst.Ref.c_long_type),
10899	as_ty \| @intFromEnum(Zir.Inst.Ref.c_ulong_type),
10900	as_ty \| @intFromEnum(Zir.Inst.Ref.c_longlong_type),
10901	as_ty \| @intFromEnum(Zir.Inst.Ref.c_ulonglong_type),
10902	as_ty \| @intFromEnum(Zir.Inst.Ref.c_longdouble_type),
10903	as_ty \| @intFromEnum(Zir.Inst.Ref.f16_type),
10904	as_ty \| @intFromEnum(Zir.Inst.Ref.f32_type),
10905	as_ty \| @intFromEnum(Zir.Inst.Ref.f64_type),
10906	as_ty \| @intFromEnum(Zir.Inst.Ref.f80_type),
10907	as_ty \| @intFromEnum(Zir.Inst.Ref.f128_type),
10908	as_ty \| @intFromEnum(Zir.Inst.Ref.anyopaque_type),
10909	as_ty \| @intFromEnum(Zir.Inst.Ref.bool_type),
10910	as_ty \| @intFromEnum(Zir.Inst.Ref.void_type),
10911	as_ty \| @intFromEnum(Zir.Inst.Ref.type_type),
10912	as_ty \| @intFromEnum(Zir.Inst.Ref.anyerror_type),
10913	as_ty \| @intFromEnum(Zir.Inst.Ref.comptime_int_type),
10914	as_ty \| @intFromEnum(Zir.Inst.Ref.comptime_float_type),
10915	as_ty \| @intFromEnum(Zir.Inst.Ref.noreturn_type),
10916	as_ty \| @intFromEnum(Zir.Inst.Ref.anyframe_type),
10917	as_ty \| @intFromEnum(Zir.Inst.Ref.null_type),
10918	as_ty \| @intFromEnum(Zir.Inst.Ref.undefined_type),
10919	as_ty \| @intFromEnum(Zir.Inst.Ref.enum_literal_type),
10920	as_ty \| @intFromEnum(Zir.Inst.Ref.atomic_order_type),
10921	as_ty \| @intFromEnum(Zir.Inst.Ref.atomic_rmw_op_type),
10922	as_ty \| @intFromEnum(Zir.Inst.Ref.calling_convention_type),
10923	as_ty \| @intFromEnum(Zir.Inst.Ref.address_space_type),
10924	as_ty \| @intFromEnum(Zir.Inst.Ref.float_mode_type),
10925	as_ty \| @intFromEnum(Zir.Inst.Ref.reduce_op_type),
10926	as_ty \| @intFromEnum(Zir.Inst.Ref.call_modifier_type),
10927	as_ty \| @intFromEnum(Zir.Inst.Ref.prefetch_options_type),
10928	as_ty \| @intFromEnum(Zir.Inst.Ref.export_options_type),
10929	as_ty \| @intFromEnum(Zir.Inst.Ref.extern_options_type),
10930	as_ty \| @intFromEnum(Zir.Inst.Ref.type_info_type),
10931	as_ty \| @intFromEnum(Zir.Inst.Ref.manyptr_u8_type),
10932	as_ty \| @intFromEnum(Zir.Inst.Ref.manyptr_const_u8_type),
10933	as_ty \| @intFromEnum(Zir.Inst.Ref.manyptr_const_u8_sentinel_0_type),
10934	as_ty \| @intFromEnum(Zir.Inst.Ref.single_const_pointer_to_comptime_int_type),
10935	as_ty \| @intFromEnum(Zir.Inst.Ref.slice_const_u8_type),
10936	as_ty \| @intFromEnum(Zir.Inst.Ref.slice_const_u8_sentinel_0_type),
10937	as_ty \| @intFromEnum(Zir.Inst.Ref.anyerror_void_error_union_type),
10938	as_ty \| @intFromEnum(Zir.Inst.Ref.generic_poison_type),
10939	as_ty \| @intFromEnum(Zir.Inst.Ref.empty_struct_type),
10940	as_comptime_int \| @intFromEnum(Zir.Inst.Ref.zero),
10941	as_comptime_int \| @intFromEnum(Zir.Inst.Ref.one),
10942	as_bool \| @intFromEnum(Zir.Inst.Ref.bool_true),
10943	as_bool \| @intFromEnum(Zir.Inst.Ref.bool_false),
10944	as_usize \| @intFromEnum(Zir.Inst.Ref.zero_usize),
10945	as_usize \| @intFromEnum(Zir.Inst.Ref.one_usize),
10946	as_void \| @intFromEnum(Zir.Inst.Ref.void_value),
10947	=> return result, // type of result is already correct
10948
10949	// Need an explicit type coercion instruction.
10950	else => return gz.addPlNode(ri.zirTag(), src_node, Zir.Inst.As{
10951	.dest_type = ty_inst,
10952	.operand = result,
10953	}),
10954	}
10955	},
10956	.ptr => \|ptr_res\| {
10957	_ = try gz.addPlNode(.store_node, ptr_res.src_node orelse src_node, Zir.Inst.Bin{
10958	.lhs = ptr_res.inst,
10959	.rhs = result,
10960	});
10961	return .void_value;
10962	},
10963	.inferred_ptr => \|alloc\| {
10964	_ = try gz.addPlNode(.store_to_inferred_ptr, src_node, Zir.Inst.Bin{
10965	.lhs = alloc,
10966	.rhs = result,
10967	});
10968	return .void_value;
10969	},
10970	.destructure => \|destructure\| {
10971	const components = destructure.components;
10972	_ = try gz.addPlNode(.validate_destructure, src_node, Zir.Inst.ValidateDestructure{
10973	.operand = result,
10974	.destructure_node = gz.nodeIndexToRelative(destructure.src_node),
10975	.expect_len = @intCast(components.len),
10976	});
10977	for (components, 0..) \|component, i\| {
10978	if (component == .discard) continue;
10979	const elem_val = try gz.add(.{
10980	.tag = .elem_val_imm,
10981	.data = .{ .elem_val_imm = .{
10982	.operand = result,
10983	.idx = @intCast(i),
10984	} },
10985	});
10986	switch (component) {
10987	.typed_ptr => \|ptr_res\| {
10988	_ = try gz.addPlNode(.store_node, ptr_res.src_node orelse src_node, Zir.Inst.Bin{
10989	.lhs = ptr_res.inst,
10990	.rhs = elem_val,
10991	});
10992	},
10993	.inferred_ptr => \|ptr_inst\| {
10994	_ = try gz.addPlNode(.store_to_inferred_ptr, src_node, Zir.Inst.Bin{
10995	.lhs = ptr_inst,
10996	.rhs = elem_val,
10997	});
10998	},
10999	.discard => unreachable,
11000	}
11001	}
11002	return .void_value;
11003	},
11004	}
11005	}
11006
11007	/// Given an identifier token, obtain the string for it.
11008	/// If the token uses @"" syntax, parses as a string, reports errors if applicable,
11009	/// and allocates the result within `astgen.arena`.
11010	/// Otherwise, returns a reference to the source code bytes directly.
11011	/// See also `appendIdentStr` and `parseStrLit`.
11012	fn identifierTokenString(astgen: *AstGen, token: Ast.TokenIndex) InnerError![]const u8 {
11013	const tree = astgen.tree;
11014	const token_tags = tree.tokens.items(.tag);
11015	assert(token_tags[token] == .identifier);
11016	const ident_name = tree.tokenSlice(token);
11017	if (!mem.startsWith(u8, ident_name, "@")) {
11018	return ident_name;
11019	}
11020	var buf: ArrayListUnmanaged(u8) = .{};
11021	defer buf.deinit(astgen.gpa);
11022	try astgen.parseStrLit(token, &buf, ident_name, 1);
11023	if (mem.indexOfScalar(u8, buf.items, 0) != null) {
11024	return astgen.failTok(token, "identifier cannot contain null bytes", .{});
11025	} else if (buf.items.len == 0) {
11026	return astgen.failTok(token, "identifier cannot be empty", .{});
11027	}
11028	const duped = try astgen.arena.dupe(u8, buf.items);
11029	return duped;
11030	}
11031
11032	/// Given an identifier token, obtain the string for it (possibly parsing as a string
11033	/// literal if it is @"" syntax), and append the string to `buf`.
11034	/// See also `identifierTokenString` and `parseStrLit`.
11035	fn appendIdentStr(
11036	astgen: *AstGen,
11037	token: Ast.TokenIndex,
11038	buf: *ArrayListUnmanaged(u8),
11039	) InnerError!void {
11040	const tree = astgen.tree;
11041	const token_tags = tree.tokens.items(.tag);
11042	assert(token_tags[token] == .identifier);
11043	const ident_name = tree.tokenSlice(token);
11044	if (!mem.startsWith(u8, ident_name, "@")) {
11045	return buf.appendSlice(astgen.gpa, ident_name);
11046	} else {
11047	const start = buf.items.len;
11048	try astgen.parseStrLit(token, buf, ident_name, 1);
11049	const slice = buf.items[start..];
11050	if (mem.indexOfScalar(u8, slice, 0) != null) {
11051	return astgen.failTok(token, "identifier cannot contain null bytes", .{});
11052	} else if (slice.len == 0) {
11053	return astgen.failTok(token, "identifier cannot be empty", .{});
11054	}
11055	}
11056	}
11057
11058	/// Appends the result to `buf`.
11059	fn parseStrLit(
11060	astgen: *AstGen,
11061	token: Ast.TokenIndex,
11062	buf: *ArrayListUnmanaged(u8),
11063	bytes: []const u8,
11064	offset: u32,
11065	) InnerError!void {
11066	const raw_string = bytes[offset..];
11067	var buf_managed = buf.toManaged(astgen.gpa);
11068	const result = std.zig.string_literal.parseWrite(buf_managed.writer(), raw_string);
11069	buf.* = buf_managed.moveToUnmanaged();
11070	switch (try result) {
11071	.success => return,
11072	.failure => \|err\| return astgen.failWithStrLitError(err, token, bytes, offset),
11073	}
11074	}
11075
11076	fn failWithStrLitError(astgen: *AstGen, err: std.zig.string_literal.Error, token: Ast.TokenIndex, bytes: []const u8, offset: u32) InnerError {
11077	const raw_string = bytes[offset..];
11078	switch (err) {
11079	.invalid_escape_character => \|bad_index\| {
11080	return astgen.failOff(
11081	token,
11082	offset + @as(u32, @intCast(bad_index)),
11083	"invalid escape character: '{c}'",
11084	.{raw_string[bad_index]},
11085	);
11086	},
11087	.expected_hex_digit => \|bad_index\| {
11088	return astgen.failOff(
11089	token,
11090	offset + @as(u32, @intCast(bad_index)),
11091	"expected hex digit, found '{c}'",
11092	.{raw_string[bad_index]},
11093	);
11094	},
11095	.empty_unicode_escape_sequence => \|bad_index\| {
11096	return astgen.failOff(
11097	token,
11098	offset + @as(u32, @intCast(bad_index)),
11099	"empty unicode escape sequence",
11100	.{},
11101	);
11102	},
11103	.expected_hex_digit_or_rbrace => \|bad_index\| {
11104	return astgen.failOff(
11105	token,
11106	offset + @as(u32, @intCast(bad_index)),
11107	"expected hex digit or '}}', found '{c}'",
11108	.{raw_string[bad_index]},
11109	);
11110	},
11111	.invalid_unicode_codepoint => \|bad_index\| {
11112	return astgen.failOff(
11113	token,
11114	offset + @as(u32, @intCast(bad_index)),
11115	"unicode escape does not correspond to a valid codepoint",
11116	.{},
11117	);
11118	},
11119	.expected_lbrace => \|bad_index\| {
11120	return astgen.failOff(
11121	token,
11122	offset + @as(u32, @intCast(bad_index)),
11123	"expected '{{', found '{c}",
11124	.{raw_string[bad_index]},
11125	);
11126	},
11127	.expected_rbrace => \|bad_index\| {
11128	return astgen.failOff(
11129	token,
11130	offset + @as(u32, @intCast(bad_index)),
11131	"expected '}}', found '{c}",
11132	.{raw_string[bad_index]},
11133	);
11134	},
11135	.expected_single_quote => \|bad_index\| {
11136	return astgen.failOff(
11137	token,
11138	offset + @as(u32, @intCast(bad_index)),
11139	"expected single quote ('), found '{c}",
11140	.{raw_string[bad_index]},
11141	);
11142	},
11143	.invalid_character => \|bad_index\| {
11144	return astgen.failOff(
11145	token,
11146	offset + @as(u32, @intCast(bad_index)),
11147	"invalid byte in string or character literal: '{c}'",
11148	.{raw_string[bad_index]},
11149	);
11150	},
11151	}
11152	}
11153
11154	fn failNode(
11155	astgen: *AstGen,
11156	node: Ast.Node.Index,
11157	comptime format: []const u8,
11158	args: anytype,
11159	) InnerError {
11160	return astgen.failNodeNotes(node, format, args, &[0]u32{});
11161	}
11162
11163	fn appendErrorNode(
11164	astgen: *AstGen,
11165	node: Ast.Node.Index,
11166	comptime format: []const u8,
11167	args: anytype,
11168	) Allocator.Error!void {
11169	try astgen.appendErrorNodeNotes(node, format, args, &[0]u32{});
11170	}
11171
11172	fn appendErrorNodeNotes(
11173	astgen: *AstGen,
11174	node: Ast.Node.Index,
11175	comptime format: []const u8,
11176	args: anytype,
11177	notes: []const u32,
11178	) Allocator.Error!void {
11179	@setCold(true);
11180	const string_bytes = &astgen.string_bytes;
11181	const msg: Zir.NullTerminatedString = @enumFromInt(string_bytes.items.len);
11182	try string_bytes.writer(astgen.gpa).print(format ++ "\x00", args);
11183	const notes_index: u32 = if (notes.len != 0) blk: {
11184	const notes_start = astgen.extra.items.len;
11185	try astgen.extra.ensureTotalCapacity(astgen.gpa, notes_start + 1 + notes.len);
11186	astgen.extra.appendAssumeCapacity(@intCast(notes.len));
11187	astgen.extra.appendSliceAssumeCapacity(notes);
11188	break :blk @intCast(notes_start);
11189	} else 0;
11190	try astgen.compile_errors.append(astgen.gpa, .{
11191	.msg = msg,
11192	.node = node,
11193	.token = 0,
11194	.byte_offset = 0,
11195	.notes = notes_index,
11196	});
11197	}
11198
11199	fn failNodeNotes(
11200	astgen: *AstGen,
11201	node: Ast.Node.Index,
11202	comptime format: []const u8,
11203	args: anytype,
11204	notes: []const u32,
11205	) InnerError {
11206	try appendErrorNodeNotes(astgen, node, format, args, notes);
11207	return error.AnalysisFail;
11208	}
11209
11210	fn failTok(
11211	astgen: *AstGen,
11212	token: Ast.TokenIndex,
11213	comptime format: []const u8,
11214	args: anytype,
11215	) InnerError {
11216	return astgen.failTokNotes(token, format, args, &[0]u32{});
11217	}
11218
11219	fn appendErrorTok(
11220	astgen: *AstGen,
11221	token: Ast.TokenIndex,
11222	comptime format: []const u8,
11223	args: anytype,
11224	) !void {
11225	try astgen.appendErrorTokNotesOff(token, 0, format, args, &[0]u32{});
11226	}
11227
11228	fn failTokNotes(
11229	astgen: *AstGen,
11230	token: Ast.TokenIndex,
11231	comptime format: []const u8,
11232	args: anytype,
11233	notes: []const u32,
11234	) InnerError {
11235	try appendErrorTokNotesOff(astgen, token, 0, format, args, notes);
11236	return error.AnalysisFail;
11237	}
11238
11239	fn appendErrorTokNotes(
11240	astgen: *AstGen,
11241	token: Ast.TokenIndex,
11242	comptime format: []const u8,
11243	args: anytype,
11244	notes: []const u32,
11245	) !void {
11246	return appendErrorTokNotesOff(astgen, token, 0, format, args, notes);
11247	}
11248
11249	/// Same as `fail`, except given a token plus an offset from its starting byte
11250	/// offset.
11251	fn failOff(
11252	astgen: *AstGen,
11253	token: Ast.TokenIndex,
11254	byte_offset: u32,
11255	comptime format: []const u8,
11256	args: anytype,
11257	) InnerError {
11258	try appendErrorTokNotesOff(astgen, token, byte_offset, format, args, &.{});
11259	return error.AnalysisFail;
11260	}
11261
11262	fn appendErrorTokNotesOff(
11263	astgen: *AstGen,
11264	token: Ast.TokenIndex,
11265	byte_offset: u32,
11266	comptime format: []const u8,
11267	args: anytype,
11268	notes: []const u32,
11269	) !void {
11270	@setCold(true);
11271	const gpa = astgen.gpa;
11272	const string_bytes = &astgen.string_bytes;
11273	const msg: Zir.NullTerminatedString = @enumFromInt(string_bytes.items.len);
11274	try string_bytes.writer(gpa).print(format ++ "\x00", args);
11275	const notes_index: u32 = if (notes.len != 0) blk: {
11276	const notes_start = astgen.extra.items.len;
11277	try astgen.extra.ensureTotalCapacity(gpa, notes_start + 1 + notes.len);
11278	astgen.extra.appendAssumeCapacity(@intCast(notes.len));
11279	astgen.extra.appendSliceAssumeCapacity(notes);
11280	break :blk @intCast(notes_start);
11281	} else 0;
11282	try astgen.compile_errors.append(gpa, .{
11283	.msg = msg,
11284	.node = 0,
11285	.token = token,
11286	.byte_offset = byte_offset,
11287	.notes = notes_index,
11288	});
11289	}
11290
11291	fn errNoteTok(
11292	astgen: *AstGen,
11293	token: Ast.TokenIndex,
11294	comptime format: []const u8,
11295	args: anytype,
11296	) Allocator.Error!u32 {
11297	return errNoteTokOff(astgen, token, 0, format, args);
11298	}
11299
11300	fn errNoteTokOff(
11301	astgen: *AstGen,
11302	token: Ast.TokenIndex,
11303	byte_offset: u32,
11304	comptime format: []const u8,
11305	args: anytype,
11306	) Allocator.Error!u32 {
11307	@setCold(true);
11308	const string_bytes = &astgen.string_bytes;
11309	const msg: Zir.NullTerminatedString = @enumFromInt(string_bytes.items.len);
11310	try string_bytes.writer(astgen.gpa).print(format ++ "\x00", args);
11311	return astgen.addExtra(Zir.Inst.CompileErrors.Item{
11312	.msg = msg,
11313	.node = 0,
11314	.token = token,
11315	.byte_offset = byte_offset,
11316	.notes = 0,
11317	});
11318	}
11319
11320	fn errNoteNode(
11321	astgen: *AstGen,
11322	node: Ast.Node.Index,
11323	comptime format: []const u8,
11324	args: anytype,
11325	) Allocator.Error!u32 {
11326	@setCold(true);
11327	const string_bytes = &astgen.string_bytes;
11328	const msg: Zir.NullTerminatedString = @enumFromInt(string_bytes.items.len);
11329	try string_bytes.writer(astgen.gpa).print(format ++ "\x00", args);
11330	return astgen.addExtra(Zir.Inst.CompileErrors.Item{
11331	.msg = msg,
11332	.node = node,
11333	.token = 0,
11334	.byte_offset = 0,
11335	.notes = 0,
11336	});
11337	}
11338
11339	fn identAsString(astgen: *AstGen, ident_token: Ast.TokenIndex) !Zir.NullTerminatedString {
11340	const gpa = astgen.gpa;
11341	const string_bytes = &astgen.string_bytes;
11342	const str_index: u32 = @intCast(string_bytes.items.len);
11343	try astgen.appendIdentStr(ident_token, string_bytes);
11344	const key: []const u8 = string_bytes.items[str_index..];
11345	const gop = try astgen.string_table.getOrPutContextAdapted(gpa, key, StringIndexAdapter{
11346	.bytes = string_bytes,
11347	}, StringIndexContext{
11348	.bytes = string_bytes,
11349	});
11350	if (gop.found_existing) {
11351	string_bytes.shrinkRetainingCapacity(str_index);
11352	return @enumFromInt(gop.key_ptr.*);
11353	} else {
11354	gop.key_ptr.* = str_index;
11355	try string_bytes.append(gpa, 0);
11356	return @enumFromInt(str_index);
11357	}
11358	}
11359
11360	/// Adds a doc comment block to `string_bytes` by walking backwards from `end_token`.
11361	/// `end_token` must point at the first token after the last doc coment line.
11362	/// Returns 0 if no doc comment is present.
11363	fn docCommentAsString(astgen: *AstGen, end_token: Ast.TokenIndex) !Zir.NullTerminatedString {
11364	if (end_token == 0) return .empty;
11365
11366	const token_tags = astgen.tree.tokens.items(.tag);
11367
11368	var tok = end_token - 1;
11369	while (token_tags[tok] == .doc_comment) {
11370	if (tok == 0) break;
11371	tok -= 1;
11372	} else {
11373	tok += 1;
11374	}
11375
11376	return docCommentAsStringFromFirst(astgen, end_token, tok);
11377	}
11378
11379	/// end_token must be > the index of the last doc comment.
11380	fn docCommentAsStringFromFirst(
11381	astgen: *AstGen,
11382	end_token: Ast.TokenIndex,
11383	start_token: Ast.TokenIndex,
11384	) !Zir.NullTerminatedString {
11385	if (start_token == end_token) return .empty;
11386
11387	const gpa = astgen.gpa;
11388	const string_bytes = &astgen.string_bytes;
11389	const str_index: u32 = @intCast(string_bytes.items.len);
11390	const token_starts = astgen.tree.tokens.items(.start);
11391	const token_tags = astgen.tree.tokens.items(.tag);
11392
11393	const total_bytes = token_starts[end_token] - token_starts[start_token];
11394	try string_bytes.ensureUnusedCapacity(gpa, total_bytes);
11395
11396	var current_token = start_token;
11397	while (current_token < end_token) : (current_token += 1) {
11398	switch (token_tags[current_token]) {
11399	.doc_comment => {
11400	const tok_bytes = astgen.tree.tokenSlice(current_token)[3..];
11401	string_bytes.appendSliceAssumeCapacity(tok_bytes);
11402	if (current_token != end_token - 1) {
11403	string_bytes.appendAssumeCapacity('\n');
11404	}
11405	},
11406	else => break,
11407	}
11408	}
11409
11410	const key: []const u8 = string_bytes.items[str_index..];
11411	const gop = try astgen.string_table.getOrPutContextAdapted(gpa, key, StringIndexAdapter{
11412	.bytes = string_bytes,
11413	}, StringIndexContext{
11414	.bytes = string_bytes,
11415	});
11416
11417	if (gop.found_existing) {
11418	string_bytes.shrinkRetainingCapacity(str_index);
11419	return @enumFromInt(gop.key_ptr.*);
11420	} else {
11421	gop.key_ptr.* = str_index;
11422	try string_bytes.append(gpa, 0);
11423	return @enumFromInt(str_index);
11424	}
11425	}
11426
11427	const IndexSlice = struct { index: Zir.NullTerminatedString, len: u32 };
11428
11429	fn strLitAsString(astgen: *AstGen, str_lit_token: Ast.TokenIndex) !IndexSlice {
11430	const gpa = astgen.gpa;
11431	const string_bytes = &astgen.string_bytes;
11432	const str_index: u32 = @intCast(string_bytes.items.len);
11433	const token_bytes = astgen.tree.tokenSlice(str_lit_token);
11434	try astgen.parseStrLit(str_lit_token, string_bytes, token_bytes, 0);
11435	const key: []const u8 = string_bytes.items[str_index..];
11436	if (std.mem.indexOfScalar(u8, key, 0)) \|_\| return .{
11437	.index = @enumFromInt(str_index),
11438	.len = @intCast(key.len),
11439	};
11440	const gop = try astgen.string_table.getOrPutContextAdapted(gpa, key, StringIndexAdapter{
11441	.bytes = string_bytes,
11442	}, StringIndexContext{
11443	.bytes = string_bytes,
11444	});
11445	if (gop.found_existing) {
11446	string_bytes.shrinkRetainingCapacity(str_index);
11447	return .{
11448	.index = @enumFromInt(gop.key_ptr.*),
11449	.len = @intCast(key.len),
11450	};
11451	} else {
11452	gop.key_ptr.* = str_index;
11453	// Still need a null byte because we are using the same table
11454	// to lookup null terminated strings, so if we get a match, it has to
11455	// be null terminated for that to work.
11456	try string_bytes.append(gpa, 0);
11457	return .{
11458	.index = @enumFromInt(str_index),
11459	.len = @intCast(key.len),
11460	};
11461	}
11462	}
11463
11464	fn strLitNodeAsString(astgen: *AstGen, node: Ast.Node.Index) !IndexSlice {
11465	const tree = astgen.tree;
11466	const node_datas = tree.nodes.items(.data);
11467
11468	const start = node_datas[node].lhs;
11469	const end = node_datas[node].rhs;
11470
11471	const gpa = astgen.gpa;
11472	const string_bytes = &astgen.string_bytes;
11473	const str_index = string_bytes.items.len;
11474
11475	// First line: do not append a newline.
11476	var tok_i = start;
11477	{
11478	const slice = tree.tokenSlice(tok_i);
11479	const carriage_return_ending: usize = if (slice[slice.len - 2] == '\r') 2 else 1;
11480	const line_bytes = slice[2 .. slice.len - carriage_return_ending];
11481	try string_bytes.appendSlice(gpa, line_bytes);
11482	tok_i += 1;
11483	}
11484	// Following lines: each line prepends a newline.
11485	while (tok_i <= end) : (tok_i += 1) {
11486	const slice = tree.tokenSlice(tok_i);
11487	const carriage_return_ending: usize = if (slice[slice.len - 2] == '\r') 2 else 1;
11488	const line_bytes = slice[2 .. slice.len - carriage_return_ending];
11489	try string_bytes.ensureUnusedCapacity(gpa, line_bytes.len + 1);
11490	string_bytes.appendAssumeCapacity('\n');
11491	string_bytes.appendSliceAssumeCapacity(line_bytes);
11492	}
11493	const len = string_bytes.items.len - str_index;
11494	try string_bytes.append(gpa, 0);
11495	return IndexSlice{
11496	.index = @enumFromInt(str_index),
11497	.len = @intCast(len),
11498	};
11499	}
11500
11501	fn testNameString(astgen: *AstGen, str_lit_token: Ast.TokenIndex) !Zir.NullTerminatedString {
11502	const gpa = astgen.gpa;
11503	const string_bytes = &astgen.string_bytes;
11504	const str_index: u32 = @intCast(string_bytes.items.len);
11505	const token_bytes = astgen.tree.tokenSlice(str_lit_token);
11506	try string_bytes.append(gpa, 0); // Indicates this is a test.
11507	try astgen.parseStrLit(str_lit_token, string_bytes, token_bytes, 0);
11508	const slice = string_bytes.items[str_index + 1 ..];
11509	if (mem.indexOfScalar(u8, slice, 0) != null) {
11510	return astgen.failTok(str_lit_token, "test name cannot contain null bytes", .{});
11511	} else if (slice.len == 0) {
11512	return astgen.failTok(str_lit_token, "empty test name must be omitted", .{});
11513	}
11514	try string_bytes.append(gpa, 0);
11515	return @enumFromInt(str_index);
11516	}
11517
11518	const Scope = struct {
11519	tag: Tag,
11520
11521	fn cast(base: Scope, comptime T: type) ?T {
11522	if (T == Defer) {
11523	switch (base.tag) {
11524	.defer_normal, .defer_error => return @fieldParentPtr(T, "base", base),
11525	else => return null,
11526	}
11527	}
11528	if (T == Namespace) {
11529	switch (base.tag) {
11530	.namespace, .enum_namespace => return @fieldParentPtr(T, "base", base),
11531	else => return null,
11532	}
11533	}
11534	if (base.tag != T.base_tag)
11535	return null;
11536
11537	return @fieldParentPtr(T, "base", base);
11538	}
11539
11540	fn parent(base: Scope) ?Scope {
11541	return switch (base.tag) {
11542	.gen_zir => base.cast(GenZir).?.parent,
11543	.local_val => base.cast(LocalVal).?.parent,
11544	.local_ptr => base.cast(LocalPtr).?.parent,
11545	.defer_normal, .defer_error => base.cast(Defer).?.parent,
11546	.namespace, .enum_namespace => base.cast(Namespace).?.parent,
11547	.top => null,
11548	};
11549	}
11550
11551	const Tag = enum {
11552	gen_zir,
11553	local_val,
11554	local_ptr,
11555	defer_normal,
11556	defer_error,
11557	namespace,
11558	enum_namespace,
11559	top,
11560	};
11561
11562	/// The category of identifier. These tag names are user-visible in compile errors.
11563	const IdCat = enum {
11564	@"function parameter",
11565	@"local constant",
11566	@"local variable",
11567	@"switch tag capture",
11568	capture,
11569	};
11570
11571	/// This is always a `const` local and importantly the `inst` is a value type, not a pointer.
11572	/// This structure lives as long as the AST generation of the Block
11573	/// node that contains the variable.
11574	const LocalVal = struct {
11575	const base_tag: Tag = .local_val;
11576	base: Scope = Scope{ .tag = base_tag },
11577	/// Parents can be: `LocalVal`, `LocalPtr`, `GenZir`, `Defer`, `Namespace`.
11578	parent: *Scope,
11579	gen_zir: *GenZir,
11580	inst: Zir.Inst.Ref,
11581	/// Source location of the corresponding variable declaration.
11582	token_src: Ast.TokenIndex,
11583	/// Track the first identifer where it is referenced.
11584	/// 0 means never referenced.
11585	used: Ast.TokenIndex = 0,
11586	/// Track the identifier where it is discarded, like this `_ = foo;`.
11587	/// 0 means never discarded.
11588	discarded: Ast.TokenIndex = 0,
11589	/// String table index.
11590	name: Zir.NullTerminatedString,
11591	id_cat: IdCat,
11592	};
11593
11594	/// This could be a `const` or `var` local. It has a pointer instead of a value.
11595	/// This structure lives as long as the AST generation of the Block
11596	/// node that contains the variable.
11597	const LocalPtr = struct {
11598	const base_tag: Tag = .local_ptr;
11599	base: Scope = Scope{ .tag = base_tag },
11600	/// Parents can be: `LocalVal`, `LocalPtr`, `GenZir`, `Defer`, `Namespace`.
11601	parent: *Scope,
11602	gen_zir: *GenZir,
11603	ptr: Zir.Inst.Ref,
11604	/// Source location of the corresponding variable declaration.
11605	token_src: Ast.TokenIndex,
11606	/// Track the first identifer where it is referenced.
11607	/// 0 means never referenced.
11608	used: Ast.TokenIndex = 0,
11609	/// Track the identifier where it is discarded, like this `_ = foo;`.
11610	/// 0 means never discarded.
11611	discarded: Ast.TokenIndex = 0,
11612	/// Whether this value is used as an lvalue after inititialization.
11613	/// If not, we know it can be `const`, so will emit a compile error if it is `var`.
11614	used_as_lvalue: bool = false,
11615	/// String table index.
11616	name: Zir.NullTerminatedString,
11617	id_cat: IdCat,
11618	/// true means we find out during Sema whether the value is comptime.
11619	/// false means it is already known at AstGen the value is runtime-known.
11620	maybe_comptime: bool,
11621	};
11622
11623	const Defer = struct {
11624	base: Scope,
11625	/// Parents can be: `LocalVal`, `LocalPtr`, `GenZir`, `Defer`, `Namespace`.
11626	parent: *Scope,
11627	index: u32,
11628	len: u32,
11629	remapped_err_code: Zir.Inst.OptionalIndex = .none,
11630	};
11631
11632	/// Represents a global scope that has any number of declarations in it.
11633	/// Each declaration has this as the parent scope.
11634	const Namespace = struct {
11635	const base_tag: Tag = .namespace;
11636	base: Scope = Scope{ .tag = base_tag },
11637
11638	/// Parents can be: `LocalVal`, `LocalPtr`, `GenZir`, `Defer`, `Namespace`.
11639	parent: *Scope,
11640	/// Maps string table index to the source location of declaration,
11641	/// for the purposes of reporting name shadowing compile errors.
11642	decls: std.AutoHashMapUnmanaged(Zir.NullTerminatedString, Ast.Node.Index) = .{},
11643	node: Ast.Node.Index,
11644	inst: Zir.Inst.Index,
11645
11646	/// The astgen scope containing this namespace.
11647	/// Only valid during astgen.
11648	declaring_gz: ?*GenZir,
11649
11650	/// Map from the raw captured value to the instruction
11651	/// ref of the capture for decls in this namespace
11652	captures: std.AutoArrayHashMapUnmanaged(Zir.Inst.Index, Zir.Inst.Index) = .{},
11653
11654	fn deinit(self: *Namespace, gpa: Allocator) void {
11655	self.decls.deinit(gpa);
11656	self.captures.deinit(gpa);
11657	self.* = undefined;
11658	}
11659	};
11660
11661	const Top = struct {
11662	const base_tag: Scope.Tag = .top;
11663	base: Scope = Scope{ .tag = base_tag },
11664	};
11665	};
11666
11667	/// This is a temporary structure; references to it are valid only
11668	/// while constructing a `Zir`.
11669	const GenZir = struct {
11670	const base_tag: Scope.Tag = .gen_zir;
11671	base: Scope = Scope{ .tag = base_tag },
11672	/// Whether we're already in a scope known to be comptime. This is set
11673	/// whenever we know Sema will analyze the current block with `is_comptime`,
11674	/// for instance when we're within a `struct_decl` or a `block_comptime`.
11675	is_comptime: bool,
11676	/// Whether we're in an expression within a `@TypeOf` operand. In this case, closure of runtime
11677	/// variables is permitted where it is usually not.
11678	is_typeof: bool = false,
11679	/// This is set to true for inline loops; false otherwise.
11680	is_inline: bool = false,
11681	c_import: bool = false,
11682	/// How decls created in this scope should be named.
11683	anon_name_strategy: Zir.Inst.NameStrategy = .anon,
11684	/// The containing decl AST node.
11685	decl_node_index: Ast.Node.Index,
11686	/// The containing decl line index, absolute.
11687	decl_line: u32,
11688	/// Parents can be: `LocalVal`, `LocalPtr`, `GenZir`, `Defer`, `Namespace`.
11689	parent: *Scope,
11690	/// All `GenZir` scopes for the same ZIR share this.
11691	astgen: *AstGen,
11692	/// Keeps track of the list of instructions in this scope. Possibly shared.
11693	/// Indexes to instructions in `astgen`.
11694	instructions: *ArrayListUnmanaged(Zir.Inst.Index),
11695	/// A sub-block may share its instructions ArrayList with containing GenZir,
11696	/// if use is strictly nested. This saves prior size of list for unstacking.
11697	instructions_top: usize,
11698	label: ?Label = null,
11699	break_block: Zir.Inst.OptionalIndex = .none,
11700	continue_block: Zir.Inst.OptionalIndex = .none,
11701	/// Only valid when setBreakResultInfo is called.
11702	break_result_info: AstGen.ResultInfo = undefined,
11703
11704	suspend_node: Ast.Node.Index = 0,
11705	nosuspend_node: Ast.Node.Index = 0,
11706	/// Set if this GenZir is a defer.
11707	cur_defer_node: Ast.Node.Index = 0,
11708	// Set if this GenZir is a defer or it is inside a defer.
11709	any_defer_node: Ast.Node.Index = 0,
11710
11711	/// Namespace members are lazy. When executing a decl within a namespace,
11712	/// any references to external instructions need to be treated specially.
11713	/// This list tracks those references. See also .closure_capture and .closure_get.
11714	/// Keys are the raw instruction index, values are the closure_capture instruction.
11715	captures: std.AutoHashMapUnmanaged(Zir.Inst.Index, Zir.Inst.Index) = .{},
11716
11717	const unstacked_top = std.math.maxInt(usize);
11718	/// Call unstack before adding any new instructions to containing GenZir.
11719	fn unstack(self: *GenZir) void {
11720	if (self.instructions_top != unstacked_top) {
11721	self.instructions.items.len = self.instructions_top;
11722	self.instructions_top = unstacked_top;
11723	}
11724	}
11725
11726	fn isEmpty(self: *const GenZir) bool {
11727	return (self.instructions_top == unstacked_top) or
11728	(self.instructions.items.len == self.instructions_top);
11729	}
11730
11731	fn instructionsSlice(self: *const GenZir) []Zir.Inst.Index {
11732	return if (self.instructions_top == unstacked_top)
11733	&[0]Zir.Inst.Index{}
11734	else
11735	self.instructions.items[self.instructions_top..];
11736	}
11737
11738	fn instructionsSliceUpto(self: const GenZir, stacked_gz: GenZir) []Zir.Inst.Index {
11739	return if (self.instructions_top == unstacked_top)
11740	&[0]Zir.Inst.Index{}
11741	else if (self.instructions == stacked_gz.instructions and stacked_gz.instructions_top != unstacked_top)
11742	self.instructions.items[self.instructions_top..stacked_gz.instructions_top]
11743	else
11744	self.instructions.items[self.instructions_top..];
11745	}
11746
11747	fn makeSubBlock(gz: GenZir, scope: Scope) GenZir {
11748	return .{
11749	.is_comptime = gz.is_comptime,
11750	.is_typeof = gz.is_typeof,
11751	.c_import = gz.c_import,
11752	.decl_node_index = gz.decl_node_index,
11753	.decl_line = gz.decl_line,
11754	.parent = scope,
11755	.astgen = gz.astgen,
11756	.suspend_node = gz.suspend_node,
11757	.nosuspend_node = gz.nosuspend_node,
11758	.any_defer_node = gz.any_defer_node,
11759	.instructions = gz.instructions,
11760	.instructions_top = gz.instructions.items.len,
11761	};
11762	}
11763
11764	const Label = struct {
11765	token: Ast.TokenIndex,
11766	block_inst: Zir.Inst.Index,
11767	used: bool = false,
11768	};
11769
11770	/// Assumes nothing stacked on `gz`.
11771	fn endsWithNoReturn(gz: GenZir) bool {
11772	if (gz.isEmpty()) return false;
11773	const tags = gz.astgen.instructions.items(.tag);
11774	const last_inst = gz.instructions.items[gz.instructions.items.len - 1];
11775	return tags[@intFromEnum(last_inst)].isNoReturn();
11776	}
11777
11778	/// TODO all uses of this should be replaced with uses of `endsWithNoReturn`.
11779	fn refIsNoReturn(gz: GenZir, inst_ref: Zir.Inst.Ref) bool {
11780	if (inst_ref == .unreachable_value) return true;
11781	if (inst_ref.toIndex()) \|inst_index\| {
11782	return gz.astgen.instructions.items(.tag)[@intFromEnum(inst_index)].isNoReturn();
11783	}
11784	return false;
11785	}
11786
11787	fn nodeIndexToRelative(gz: GenZir, node_index: Ast.Node.Index) i32 {
11788	return @as(i32, @bitCast(node_index)) - @as(i32, @bitCast(gz.decl_node_index));
11789	}
11790
11791	fn tokenIndexToRelative(gz: GenZir, token: Ast.TokenIndex) u32 {
11792	return token - gz.srcToken();
11793	}
11794
11795	fn srcToken(gz: GenZir) Ast.TokenIndex {
11796	return gz.astgen.tree.firstToken(gz.decl_node_index);
11797	}
11798
11799	fn setBreakResultInfo(gz: *GenZir, parent_ri: AstGen.ResultInfo) void {
11800	// Depending on whether the result location is a pointer or value, different
11801	// ZIR needs to be generated. In the former case we rely on storing to the
11802	// pointer to communicate the result, and use breakvoid; in the latter case
11803	// the block break instructions will have the result values.
11804	switch (parent_ri.rl) {
11805	.coerced_ty => \|ty_inst\| {
11806	// Type coercion needs to happen before breaks.
11807	gz.break_result_info = .{ .rl = .{ .ty = ty_inst }, .ctx = parent_ri.ctx };
11808	},
11809	.discard => {
11810	// We don't forward the result context here. This prevents
11811	// "unnecessary discard" errors from being caused by expressions
11812	// far from the actual discard, such as a `break` from a
11813	// discarded block.
11814	gz.break_result_info = .{ .rl = .discard };
11815	},
11816	else => {
11817	gz.break_result_info = parent_ri;
11818	},
11819	}
11820	}
11821
11822	/// Assumes nothing stacked on `gz`. Unstacks `gz`.
11823	fn setBoolBrBody(gz: *GenZir, bool_br: Zir.Inst.Index, bool_br_lhs: Zir.Inst.Ref) !void {
11824	const astgen = gz.astgen;
11825	const gpa = astgen.gpa;
11826	const body = gz.instructionsSlice();
11827	const body_len = astgen.countBodyLenAfterFixups(body);
11828	try astgen.extra.ensureUnusedCapacity(
11829	gpa,
11830	@typeInfo(Zir.Inst.BoolBr).Struct.fields.len + body_len,
11831	);
11832	const zir_datas = astgen.instructions.items(.data);
11833	zir_datas[@intFromEnum(bool_br)].pl_node.payload_index = astgen.addExtraAssumeCapacity(Zir.Inst.BoolBr{
11834	.lhs = bool_br_lhs,
11835	.body_len = body_len,
11836	});
11837	astgen.appendBodyWithFixups(body);
11838	gz.unstack();
11839	}
11840
11841	/// Assumes nothing stacked on `gz`. Unstacks `gz`.
11842	fn setBlockBody(gz: *GenZir, inst: Zir.Inst.Index) !void {
11843	const astgen = gz.astgen;
11844	const gpa = astgen.gpa;
11845	const body = gz.instructionsSlice();
11846	const body_len = astgen.countBodyLenAfterFixups(body);
11847	try astgen.extra.ensureUnusedCapacity(
11848	gpa,
11849	@typeInfo(Zir.Inst.Block).Struct.fields.len + body_len,
11850	);
11851	const zir_datas = astgen.instructions.items(.data);
11852	zir_datas[@intFromEnum(inst)].pl_node.payload_index = astgen.addExtraAssumeCapacity(
11853	Zir.Inst.Block{ .body_len = body_len },
11854	);
11855	astgen.appendBodyWithFixups(body);
11856	gz.unstack();
11857	}
11858
11859	/// Assumes nothing stacked on `gz`. Unstacks `gz`.
11860	fn setTryBody(gz: *GenZir, inst: Zir.Inst.Index, operand: Zir.Inst.Ref) !void {
11861	const astgen = gz.astgen;
11862	const gpa = astgen.gpa;
11863	const body = gz.instructionsSlice();
11864	const body_len = astgen.countBodyLenAfterFixups(body);
11865	try astgen.extra.ensureUnusedCapacity(
11866	gpa,
11867	@typeInfo(Zir.Inst.Try).Struct.fields.len + body_len,
11868	);
11869	const zir_datas = astgen.instructions.items(.data);
11870	zir_datas[@intFromEnum(inst)].pl_node.payload_index = astgen.addExtraAssumeCapacity(
11871	Zir.Inst.Try{
11872	.operand = operand,
11873	.body_len = body_len,
11874	},
11875	);
11876	astgen.appendBodyWithFixups(body);
11877	gz.unstack();
11878	}
11879
11880	/// Must be called with the following stack set up:
11881	/// * gz (bottom)
11882	/// * align_gz
11883	/// * addrspace_gz
11884	/// * section_gz
11885	/// * cc_gz
11886	/// * ret_gz
11887	/// * body_gz (top)
11888	/// Unstacks all of those except for `gz`.
11889	fn addFunc(gz: *GenZir, args: struct {
11890	src_node: Ast.Node.Index,
11891	lbrace_line: u32 = 0,
11892	lbrace_column: u32 = 0,
11893	param_block: Zir.Inst.Index,
11894
11895	align_gz: ?*GenZir,
11896	addrspace_gz: ?*GenZir,
11897	section_gz: ?*GenZir,
11898	cc_gz: ?*GenZir,
11899	ret_gz: ?*GenZir,
11900	body_gz: ?*GenZir,
11901
11902	align_ref: Zir.Inst.Ref,
11903	addrspace_ref: Zir.Inst.Ref,
11904	section_ref: Zir.Inst.Ref,
11905	cc_ref: Zir.Inst.Ref,
11906	ret_ref: Zir.Inst.Ref,
11907
11908	lib_name: Zir.NullTerminatedString,
11909	noalias_bits: u32,
11910	is_var_args: bool,
11911	is_inferred_error: bool,
11912	is_test: bool,
11913	is_extern: bool,
11914	is_noinline: bool,
11915	}) !Zir.Inst.Ref {
11916	assert(args.src_node != 0);
11917	const astgen = gz.astgen;
11918	const gpa = astgen.gpa;
11919	const ret_ref = if (args.ret_ref == .void_type) .none else args.ret_ref;
11920	const new_index: Zir.Inst.Index = @enumFromInt(astgen.instructions.len);
11921
11922	try astgen.instructions.ensureUnusedCapacity(gpa, 1);
11923
11924	var body: []Zir.Inst.Index = &[0]Zir.Inst.Index{};
11925	var ret_body: []Zir.Inst.Index = &[0]Zir.Inst.Index{};
11926	var src_locs_and_hash_buffer: [7]u32 = undefined;
11927	var src_locs_and_hash: []u32 = src_locs_and_hash_buffer[0..0];
11928	if (args.body_gz) \|body_gz\| {
11929	const tree = astgen.tree;
11930	const node_tags = tree.nodes.items(.tag);
11931	const node_datas = tree.nodes.items(.data);
11932	const token_starts = tree.tokens.items(.start);
11933	const fn_decl = args.src_node;
11934	assert(node_tags[fn_decl] == .fn_decl or node_tags[fn_decl] == .test_decl);
11935	const block = node_datas[fn_decl].rhs;
11936	const rbrace_start = token_starts[tree.lastToken(block)];
11937	astgen.advanceSourceCursor(rbrace_start);
11938	const rbrace_line: u32 = @intCast(astgen.source_line - gz.decl_line);
11939	const rbrace_column: u32 = @intCast(astgen.source_column);
11940
11941	const columns = args.lbrace_column \| (rbrace_column << 16);
11942
11943	const proto_hash: std.zig.SrcHash = switch (node_tags[fn_decl]) {
11944	.fn_decl => sig_hash: {
11945	const proto_node = node_datas[fn_decl].lhs;
11946	break :sig_hash std.zig.hashSrc(tree.getNodeSource(proto_node));
11947	},
11948	.test_decl => std.zig.hashSrc(""), // tests don't have a prototype
11949	else => unreachable,
11950	};
11951	const proto_hash_arr: [4]u32 = @bitCast(proto_hash);
11952
11953	src_locs_and_hash_buffer = .{
11954	args.lbrace_line,
11955	rbrace_line,
11956	columns,
11957	proto_hash_arr[0],
11958	proto_hash_arr[1],
11959	proto_hash_arr[2],
11960	proto_hash_arr[3],
11961	};
11962	src_locs_and_hash = &src_locs_and_hash_buffer;
11963
11964	body = body_gz.instructionsSlice();
11965	if (args.ret_gz) \|ret_gz\|
11966	ret_body = ret_gz.instructionsSliceUpto(body_gz);
11967	} else {
11968	if (args.ret_gz) \|ret_gz\|
11969	ret_body = ret_gz.instructionsSlice();
11970	}
11971	const body_len = astgen.countBodyLenAfterFixups(body);
11972
11973	if (args.cc_ref != .none or args.lib_name != .empty or args.is_var_args or args.is_test or
11974	args.is_extern or args.align_ref != .none or args.section_ref != .none or
11975	args.addrspace_ref != .none or args.noalias_bits != 0 or args.is_noinline)
11976	{
11977	var align_body: []Zir.Inst.Index = &.{};
11978	var addrspace_body: []Zir.Inst.Index = &.{};
11979	var section_body: []Zir.Inst.Index = &.{};
11980	var cc_body: []Zir.Inst.Index = &.{};
11981	if (args.ret_gz != null) {
11982	align_body = args.align_gz.?.instructionsSliceUpto(args.addrspace_gz.?);
11983	addrspace_body = args.addrspace_gz.?.instructionsSliceUpto(args.section_gz.?);
11984	section_body = args.section_gz.?.instructionsSliceUpto(args.cc_gz.?);
11985	cc_body = args.cc_gz.?.instructionsSliceUpto(args.ret_gz.?);
11986	}
11987
11988	try astgen.extra.ensureUnusedCapacity(
11989	gpa,
11990	@typeInfo(Zir.Inst.FuncFancy).Struct.fields.len +
11991	fancyFnExprExtraLen(astgen, align_body, args.align_ref) +
11992	fancyFnExprExtraLen(astgen, addrspace_body, args.addrspace_ref) +
11993	fancyFnExprExtraLen(astgen, section_body, args.section_ref) +
11994	fancyFnExprExtraLen(astgen, cc_body, args.cc_ref) +
11995	fancyFnExprExtraLen(astgen, ret_body, ret_ref) +
11996	body_len + src_locs_and_hash.len +
11997	@intFromBool(args.lib_name != .empty) +
11998	@intFromBool(args.noalias_bits != 0),
11999	);
12000	const payload_index = astgen.addExtraAssumeCapacity(Zir.Inst.FuncFancy{
12001	.param_block = args.param_block,
12002	.body_len = body_len,
12003	.bits = .{
12004	.is_var_args = args.is_var_args,
12005	.is_inferred_error = args.is_inferred_error,
12006	.is_test = args.is_test,
12007	.is_extern = args.is_extern,
12008	.is_noinline = args.is_noinline,
12009	.has_lib_name = args.lib_name != .empty,
12010	.has_any_noalias = args.noalias_bits != 0,
12011
12012	.has_align_ref = args.align_ref != .none,
12013	.has_addrspace_ref = args.addrspace_ref != .none,
12014	.has_section_ref = args.section_ref != .none,
12015	.has_cc_ref = args.cc_ref != .none,
12016	.has_ret_ty_ref = ret_ref != .none,
12017
12018	.has_align_body = align_body.len != 0,
12019	.has_addrspace_body = addrspace_body.len != 0,
12020	.has_section_body = section_body.len != 0,
12021	.has_cc_body = cc_body.len != 0,
12022	.has_ret_ty_body = ret_body.len != 0,
12023	},
12024	});
12025	if (args.lib_name != .empty) {
12026	astgen.extra.appendAssumeCapacity(@intFromEnum(args.lib_name));
12027	}
12028
12029	const zir_datas = astgen.instructions.items(.data);
12030	if (align_body.len != 0) {
12031	astgen.extra.appendAssumeCapacity(countBodyLenAfterFixups(astgen, align_body));
12032	astgen.appendBodyWithFixups(align_body);
12033	const break_extra = zir_datas[@intFromEnum(align_body[align_body.len - 1])].@"break".payload_index;
12034	astgen.extra.items[break_extra + std.meta.fieldIndex(Zir.Inst.Break, "block_inst").?] =
12035	@intFromEnum(new_index);
12036	} else if (args.align_ref != .none) {
12037	astgen.extra.appendAssumeCapacity(@intFromEnum(args.align_ref));
12038	}
12039	if (addrspace_body.len != 0) {
12040	astgen.extra.appendAssumeCapacity(countBodyLenAfterFixups(astgen, addrspace_body));
12041	astgen.appendBodyWithFixups(addrspace_body);
12042	const break_extra =
12043	zir_datas[@intFromEnum(addrspace_body[addrspace_body.len - 1])].@"break".payload_index;
12044	astgen.extra.items[break_extra + std.meta.fieldIndex(Zir.Inst.Break, "block_inst").?] =
12045	@intFromEnum(new_index);
12046	} else if (args.addrspace_ref != .none) {
12047	astgen.extra.appendAssumeCapacity(@intFromEnum(args.addrspace_ref));
12048	}
12049	if (section_body.len != 0) {
12050	astgen.extra.appendAssumeCapacity(countBodyLenAfterFixups(astgen, section_body));
12051	astgen.appendBodyWithFixups(section_body);
12052	const break_extra =
12053	zir_datas[@intFromEnum(section_body[section_body.len - 1])].@"break".payload_index;
12054	astgen.extra.items[break_extra + std.meta.fieldIndex(Zir.Inst.Break, "block_inst").?] =
12055	@intFromEnum(new_index);
12056	} else if (args.section_ref != .none) {
12057	astgen.extra.appendAssumeCapacity(@intFromEnum(args.section_ref));
12058	}
12059	if (cc_body.len != 0) {
12060	astgen.extra.appendAssumeCapacity(countBodyLenAfterFixups(astgen, cc_body));
12061	astgen.appendBodyWithFixups(cc_body);
12062	const break_extra = zir_datas[@intFromEnum(cc_body[cc_body.len - 1])].@"break".payload_index;
12063	astgen.extra.items[break_extra + std.meta.fieldIndex(Zir.Inst.Break, "block_inst").?] =
12064	@intFromEnum(new_index);
12065	} else if (args.cc_ref != .none) {
12066	astgen.extra.appendAssumeCapacity(@intFromEnum(args.cc_ref));
12067	}
12068	if (ret_body.len != 0) {
12069	astgen.extra.appendAssumeCapacity(countBodyLenAfterFixups(astgen, ret_body));
12070	astgen.appendBodyWithFixups(ret_body);
12071	const break_extra = zir_datas[@intFromEnum(ret_body[ret_body.len - 1])].@"break".payload_index;
12072	astgen.extra.items[break_extra + std.meta.fieldIndex(Zir.Inst.Break, "block_inst").?] =
12073	@intFromEnum(new_index);
12074	} else if (ret_ref != .none) {
12075	astgen.extra.appendAssumeCapacity(@intFromEnum(ret_ref));
12076	}
12077
12078	if (args.noalias_bits != 0) {
12079	astgen.extra.appendAssumeCapacity(args.noalias_bits);
12080	}
12081
12082	astgen.appendBodyWithFixups(body);
12083	astgen.extra.appendSliceAssumeCapacity(src_locs_and_hash);
12084
12085	// Order is important when unstacking.
12086	if (args.body_gz) \|body_gz\| body_gz.unstack();
12087	if (args.ret_gz != null) {
12088	args.ret_gz.?.unstack();
12089	args.cc_gz.?.unstack();
12090	args.section_gz.?.unstack();
12091	args.addrspace_gz.?.unstack();
12092	args.align_gz.?.unstack();
12093	}
12094
12095	try gz.instructions.ensureUnusedCapacity(gpa, 1);
12096
12097	astgen.instructions.appendAssumeCapacity(.{
12098	.tag = .func_fancy,
12099	.data = .{ .pl_node = .{
12100	.src_node = gz.nodeIndexToRelative(args.src_node),
12101	.payload_index = payload_index,
12102	} },
12103	});
12104	gz.instructions.appendAssumeCapacity(new_index);
12105	return new_index.toRef();
12106	} else {
12107	try astgen.extra.ensureUnusedCapacity(
12108	gpa,
12109	@typeInfo(Zir.Inst.Func).Struct.fields.len + 1 +
12110	fancyFnExprExtraLen(astgen, ret_body, ret_ref) +
12111	body_len + src_locs_and_hash.len,
12112	);
12113
12114	const ret_body_len = if (ret_body.len != 0)
12115	countBodyLenAfterFixups(astgen, ret_body)
12116	else
12117	@intFromBool(ret_ref != .none);
12118
12119	const payload_index = astgen.addExtraAssumeCapacity(Zir.Inst.Func{
12120	.param_block = args.param_block,
12121	.ret_body_len = ret_body_len,
12122	.body_len = body_len,
12123	});
12124	const zir_datas = astgen.instructions.items(.data);
12125	if (ret_body.len != 0) {
12126	astgen.appendBodyWithFixups(ret_body);
12127
12128	const break_extra = zir_datas[@intFromEnum(ret_body[ret_body.len - 1])].@"break".payload_index;
12129	astgen.extra.items[break_extra + std.meta.fieldIndex(Zir.Inst.Break, "block_inst").?] =
12130	@intFromEnum(new_index);
12131	} else if (ret_ref != .none) {
12132	astgen.extra.appendAssumeCapacity(@intFromEnum(ret_ref));
12133	}
12134	astgen.appendBodyWithFixups(body);
12135	astgen.extra.appendSliceAssumeCapacity(src_locs_and_hash);
12136
12137	// Order is important when unstacking.
12138	if (args.body_gz) \|body_gz\| body_gz.unstack();
12139	if (args.ret_gz) \|ret_gz\| ret_gz.unstack();
12140	if (args.cc_gz) \|cc_gz\| cc_gz.unstack();
12141	if (args.section_gz) \|section_gz\| section_gz.unstack();
12142	if (args.addrspace_gz) \|addrspace_gz\| addrspace_gz.unstack();
12143	if (args.align_gz) \|align_gz\| align_gz.unstack();
12144
12145	try gz.instructions.ensureUnusedCapacity(gpa, 1);
12146
12147	const tag: Zir.Inst.Tag = if (args.is_inferred_error) .func_inferred else .func;
12148	astgen.instructions.appendAssumeCapacity(.{
12149	.tag = tag,
12150	.data = .{ .pl_node = .{
12151	.src_node = gz.nodeIndexToRelative(args.src_node),
12152	.payload_index = payload_index,
12153	} },
12154	});
12155	gz.instructions.appendAssumeCapacity(new_index);
12156	return new_index.toRef();
12157	}
12158	}
12159
12160	fn fancyFnExprExtraLen(astgen: *AstGen, body: []Zir.Inst.Index, ref: Zir.Inst.Ref) u32 {
12161	// In the case of non-empty body, there is one for the body length,
12162	// and then one for each instruction.
12163	return countBodyLenAfterFixups(astgen, body) + @intFromBool(ref != .none);
12164	}
12165
12166	fn addVar(gz: *GenZir, args: struct {
12167	align_inst: Zir.Inst.Ref,
12168	lib_name: Zir.NullTerminatedString,
12169	var_type: Zir.Inst.Ref,
12170	init: Zir.Inst.Ref,
12171	is_extern: bool,
12172	is_const: bool,
12173	is_threadlocal: bool,
12174	}) !Zir.Inst.Ref {
12175	const astgen = gz.astgen;
12176	const gpa = astgen.gpa;
12177
12178	try gz.instructions.ensureUnusedCapacity(gpa, 1);
12179	try astgen.instructions.ensureUnusedCapacity(gpa, 1);
12180
12181	try astgen.extra.ensureUnusedCapacity(
12182	gpa,
12183	@typeInfo(Zir.Inst.ExtendedVar).Struct.fields.len +
12184	@intFromBool(args.lib_name != .empty) +
12185	@intFromBool(args.align_inst != .none) +
12186	@intFromBool(args.init != .none),
12187	);
12188	const payload_index = astgen.addExtraAssumeCapacity(Zir.Inst.ExtendedVar{
12189	.var_type = args.var_type,
12190	});
12191	if (args.lib_name != .empty) {
12192	astgen.extra.appendAssumeCapacity(@intFromEnum(args.lib_name));
12193	}
12194	if (args.align_inst != .none) {
12195	astgen.extra.appendAssumeCapacity(@intFromEnum(args.align_inst));
12196	}
12197	if (args.init != .none) {
12198	astgen.extra.appendAssumeCapacity(@intFromEnum(args.init));
12199	}
12200
12201	const new_index: Zir.Inst.Index = @enumFromInt(astgen.instructions.len);
12202	astgen.instructions.appendAssumeCapacity(.{
12203	.tag = .extended,
12204	.data = .{ .extended = .{
12205	.opcode = .variable,
12206	.small = @bitCast(Zir.Inst.ExtendedVar.Small{
12207	.has_lib_name = args.lib_name != .empty,
12208	.has_align = args.align_inst != .none,
12209	.has_init = args.init != .none,
12210	.is_extern = args.is_extern,
12211	.is_const = args.is_const,
12212	.is_threadlocal = args.is_threadlocal,
12213	}),
12214	.operand = payload_index,
12215	} },
12216	});
12217	gz.instructions.appendAssumeCapacity(new_index);
12218	return new_index.toRef();
12219	}
12220
12221	fn addInt(gz: *GenZir, integer: u64) !Zir.Inst.Ref {
12222	return gz.add(.{
12223	.tag = .int,
12224	.data = .{ .int = integer },
12225	});
12226	}
12227
12228	fn addIntBig(gz: *GenZir, limbs: []const std.math.big.Limb) !Zir.Inst.Ref {
12229	const astgen = gz.astgen;
12230	const gpa = astgen.gpa;
12231	try gz.instructions.ensureUnusedCapacity(gpa, 1);
12232	try astgen.instructions.ensureUnusedCapacity(gpa, 1);
12233	try astgen.string_bytes.ensureUnusedCapacity(gpa, @sizeOf(std.math.big.Limb) * limbs.len);
12234
12235	const new_index: Zir.Inst.Index = @enumFromInt(astgen.instructions.len);
12236	astgen.instructions.appendAssumeCapacity(.{
12237	.tag = .int_big,
12238	.data = .{ .str = .{
12239	.start = @enumFromInt(astgen.string_bytes.items.len),
12240	.len = @intCast(limbs.len),
12241	} },
12242	});
12243	gz.instructions.appendAssumeCapacity(new_index);
12244	astgen.string_bytes.appendSliceAssumeCapacity(mem.sliceAsBytes(limbs));
12245	return new_index.toRef();
12246	}
12247
12248	fn addFloat(gz: *GenZir, number: f64) !Zir.Inst.Ref {
12249	return gz.add(.{
12250	.tag = .float,
12251	.data = .{ .float = number },
12252	});
12253	}
12254
12255	fn addUnNode(
12256	gz: *GenZir,
12257	tag: Zir.Inst.Tag,
12258	operand: Zir.Inst.Ref,
12259	/// Absolute node index. This function does the conversion to offset from Decl.
12260	src_node: Ast.Node.Index,
12261	) !Zir.Inst.Ref {
12262	assert(operand != .none);
12263	return gz.add(.{
12264	.tag = tag,
12265	.data = .{ .un_node = .{
12266	.operand = operand,
12267	.src_node = gz.nodeIndexToRelative(src_node),
12268	} },
12269	});
12270	}
12271
12272	fn makeUnNode(
12273	gz: *GenZir,
12274	tag: Zir.Inst.Tag,
12275	operand: Zir.Inst.Ref,
12276	/// Absolute node index. This function does the conversion to offset from Decl.
12277	src_node: Ast.Node.Index,
12278	) !Zir.Inst.Index {
12279	assert(operand != .none);
12280	const new_index: Zir.Inst.Index = @enumFromInt(gz.astgen.instructions.len);
12281	try gz.astgen.instructions.append(gz.astgen.gpa, .{
12282	.tag = tag,
12283	.data = .{ .un_node = .{
12284	.operand = operand,
12285	.src_node = gz.nodeIndexToRelative(src_node),
12286	} },
12287	});
12288	return new_index;
12289	}
12290
12291	fn addPlNode(
12292	gz: *GenZir,
12293	tag: Zir.Inst.Tag,
12294	/// Absolute node index. This function does the conversion to offset from Decl.
12295	src_node: Ast.Node.Index,
12296	extra: anytype,
12297	) !Zir.Inst.Ref {
12298	const gpa = gz.astgen.gpa;
12299	try gz.instructions.ensureUnusedCapacity(gpa, 1);
12300	try gz.astgen.instructions.ensureUnusedCapacity(gpa, 1);
12301
12302	const payload_index = try gz.astgen.addExtra(extra);
12303	const new_index: Zir.Inst.Index = @enumFromInt(gz.astgen.instructions.len);
12304	gz.astgen.instructions.appendAssumeCapacity(.{
12305	.tag = tag,
12306	.data = .{ .pl_node = .{
12307	.src_node = gz.nodeIndexToRelative(src_node),
12308	.payload_index = payload_index,
12309	} },
12310	});
12311	gz.instructions.appendAssumeCapacity(new_index);
12312	return new_index.toRef();
12313	}
12314
12315	fn addPlNodePayloadIndex(
12316	gz: *GenZir,
12317	tag: Zir.Inst.Tag,
12318	/// Absolute node index. This function does the conversion to offset from Decl.
12319	src_node: Ast.Node.Index,
12320	payload_index: u32,
12321	) !Zir.Inst.Ref {
12322	return try gz.add(.{
12323	.tag = tag,
12324	.data = .{ .pl_node = .{
12325	.src_node = gz.nodeIndexToRelative(src_node),
12326	.payload_index = payload_index,
12327	} },
12328	});
12329	}
12330
12331	/// Supports `param_gz` stacked on `gz`. Assumes nothing stacked on `param_gz`. Unstacks `param_gz`.
12332	fn addParam(
12333	gz: *GenZir,
12334	param_gz: *GenZir,
12335	tag: Zir.Inst.Tag,
12336	/// Absolute token index. This function does the conversion to Decl offset.
12337	abs_tok_index: Ast.TokenIndex,
12338	name: Zir.NullTerminatedString,
12339	first_doc_comment: ?Ast.TokenIndex,
12340	) !Zir.Inst.Index {
12341	const gpa = gz.astgen.gpa;
12342	const param_body = param_gz.instructionsSlice();
12343	const body_len = gz.astgen.countBodyLenAfterFixups(param_body);
12344	try gz.astgen.instructions.ensureUnusedCapacity(gpa, 1);
12345	try gz.astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.Param).Struct.fields.len + body_len);
12346
12347	const doc_comment_index = if (first_doc_comment) \|first\|
12348	try gz.astgen.docCommentAsStringFromFirst(abs_tok_index, first)
12349	else
12350	.empty;
12351
12352	const payload_index = gz.astgen.addExtraAssumeCapacity(Zir.Inst.Param{
12353	.name = name,
12354	.doc_comment = doc_comment_index,
12355	.body_len = @intCast(body_len),
12356	});
12357	gz.astgen.appendBodyWithFixups(param_body);
12358	param_gz.unstack();
12359
12360	const new_index: Zir.Inst.Index = @enumFromInt(gz.astgen.instructions.len);
12361	gz.astgen.instructions.appendAssumeCapacity(.{
12362	.tag = tag,
12363	.data = .{ .pl_tok = .{
12364	.src_tok = gz.tokenIndexToRelative(abs_tok_index),
12365	.payload_index = payload_index,
12366	} },
12367	});
12368	gz.instructions.appendAssumeCapacity(new_index);
12369	return new_index;
12370	}
12371
12372	fn addExtendedPayload(gz: *GenZir, opcode: Zir.Inst.Extended, extra: anytype) !Zir.Inst.Ref {
12373	return addExtendedPayloadSmall(gz, opcode, undefined, extra);
12374	}
12375
12376	fn addExtendedPayloadSmall(
12377	gz: *GenZir,
12378	opcode: Zir.Inst.Extended,
12379	small: u16,
12380	extra: anytype,
12381	) !Zir.Inst.Ref {
12382	const gpa = gz.astgen.gpa;
12383
12384	try gz.instructions.ensureUnusedCapacity(gpa, 1);
12385	try gz.astgen.instructions.ensureUnusedCapacity(gpa, 1);
12386
12387	const payload_index = try gz.astgen.addExtra(extra);
12388	const new_index: Zir.Inst.Index = @enumFromInt(gz.astgen.instructions.len);
12389	gz.astgen.instructions.appendAssumeCapacity(.{
12390	.tag = .extended,
12391	.data = .{ .extended = .{
12392	.opcode = opcode,
12393	.small = small,
12394	.operand = payload_index,
12395	} },
12396	});
12397	gz.instructions.appendAssumeCapacity(new_index);
12398	return new_index.toRef();
12399	}
12400
12401	fn addExtendedMultiOp(
12402	gz: *GenZir,
12403	opcode: Zir.Inst.Extended,
12404	node: Ast.Node.Index,
12405	operands: []const Zir.Inst.Ref,
12406	) !Zir.Inst.Ref {
12407	const astgen = gz.astgen;
12408	const gpa = astgen.gpa;
12409
12410	try gz.instructions.ensureUnusedCapacity(gpa, 1);
12411	try astgen.instructions.ensureUnusedCapacity(gpa, 1);
12412	try astgen.extra.ensureUnusedCapacity(
12413	gpa,
12414	@typeInfo(Zir.Inst.NodeMultiOp).Struct.fields.len + operands.len,
12415	);
12416
12417	const payload_index = astgen.addExtraAssumeCapacity(Zir.Inst.NodeMultiOp{
12418	.src_node = gz.nodeIndexToRelative(node),
12419	});
12420	const new_index: Zir.Inst.Index = @enumFromInt(astgen.instructions.len);
12421	astgen.instructions.appendAssumeCapacity(.{
12422	.tag = .extended,
12423	.data = .{ .extended = .{
12424	.opcode = opcode,
12425	.small = @intCast(operands.len),
12426	.operand = payload_index,
12427	} },
12428	});
12429	gz.instructions.appendAssumeCapacity(new_index);
12430	astgen.appendRefsAssumeCapacity(operands);
12431	return new_index.toRef();
12432	}
12433
12434	fn addExtendedMultiOpPayloadIndex(
12435	gz: *GenZir,
12436	opcode: Zir.Inst.Extended,
12437	payload_index: u32,
12438	trailing_len: usize,
12439	) !Zir.Inst.Ref {
12440	const astgen = gz.astgen;
12441	const gpa = astgen.gpa;
12442
12443	try gz.instructions.ensureUnusedCapacity(gpa, 1);
12444	try astgen.instructions.ensureUnusedCapacity(gpa, 1);
12445	const new_index: Zir.Inst.Index = @enumFromInt(astgen.instructions.len);
12446	astgen.instructions.appendAssumeCapacity(.{
12447	.tag = .extended,
12448	.data = .{ .extended = .{
12449	.opcode = opcode,
12450	.small = @intCast(trailing_len),
12451	.operand = payload_index,
12452	} },
12453	});
12454	gz.instructions.appendAssumeCapacity(new_index);
12455	return new_index.toRef();
12456	}
12457
12458	fn addUnTok(
12459	gz: *GenZir,
12460	tag: Zir.Inst.Tag,
12461	operand: Zir.Inst.Ref,
12462	/// Absolute token index. This function does the conversion to Decl offset.
12463	abs_tok_index: Ast.TokenIndex,
12464	) !Zir.Inst.Ref {
12465	assert(operand != .none);
12466	return gz.add(.{
12467	.tag = tag,
12468	.data = .{ .un_tok = .{
12469	.operand = operand,
12470	.src_tok = gz.tokenIndexToRelative(abs_tok_index),
12471	} },
12472	});
12473	}
12474
12475	fn makeUnTok(
12476	gz: *GenZir,
12477	tag: Zir.Inst.Tag,
12478	operand: Zir.Inst.Ref,
12479	/// Absolute token index. This function does the conversion to Decl offset.
12480	abs_tok_index: Ast.TokenIndex,
12481	) !Zir.Inst.Index {
12482	const astgen = gz.astgen;
12483	const new_index: Zir.Inst.Index = @enumFromInt(astgen.instructions.len);
12484	assert(operand != .none);
12485	try astgen.instructions.append(astgen.gpa, .{
12486	.tag = tag,
12487	.data = .{ .un_tok = .{
12488	.operand = operand,
12489	.src_tok = gz.tokenIndexToRelative(abs_tok_index),
12490	} },
12491	});
12492	return new_index;
12493	}
12494
12495	fn addStrTok(
12496	gz: *GenZir,
12497	tag: Zir.Inst.Tag,
12498	str_index: Zir.NullTerminatedString,
12499	/// Absolute token index. This function does the conversion to Decl offset.
12500	abs_tok_index: Ast.TokenIndex,
12501	) !Zir.Inst.Ref {
12502	return gz.add(.{
12503	.tag = tag,
12504	.data = .{ .str_tok = .{
12505	.start = str_index,
12506	.src_tok = gz.tokenIndexToRelative(abs_tok_index),
12507	} },
12508	});
12509	}
12510
12511	fn addSaveErrRetIndex(
12512	gz: *GenZir,
12513	cond: union(enum) {
12514	always: void,
12515	if_of_error_type: Zir.Inst.Ref,
12516	},
12517	) !Zir.Inst.Index {
12518	return gz.addAsIndex(.{
12519	.tag = .save_err_ret_index,
12520	.data = .{ .save_err_ret_index = .{
12521	.operand = switch (cond) {
12522	.if_of_error_type => \|x\| x,
12523	else => .none,
12524	},
12525	} },
12526	});
12527	}
12528
12529	const BranchTarget = union(enum) {
12530	ret,
12531	block: Zir.Inst.Index,
12532	};
12533
12534	fn addRestoreErrRetIndex(
12535	gz: *GenZir,
12536	bt: BranchTarget,
12537	cond: union(enum) {
12538	always: void,
12539	if_non_error: Zir.Inst.Ref,
12540	},
12541	src_node: Ast.Node.Index,
12542	) !Zir.Inst.Index {
12543	switch (cond) {
12544	.always => return gz.addAsIndex(.{
12545	.tag = .restore_err_ret_index_unconditional,
12546	.data = .{ .un_node = .{
12547	.operand = switch (bt) {
12548	.ret => .none,
12549	.block => \|b\| b.toRef(),
12550	},
12551	.src_node = gz.nodeIndexToRelative(src_node),
12552	} },
12553	}),
12554	.if_non_error => \|operand\| switch (bt) {
12555	.ret => return gz.addAsIndex(.{
12556	.tag = .restore_err_ret_index_fn_entry,
12557	.data = .{ .un_node = .{
12558	.operand = operand,
12559	.src_node = gz.nodeIndexToRelative(src_node),
12560	} },
12561	}),
12562	.block => \|block\| return (try gz.addExtendedPayload(
12563	.restore_err_ret_index,
12564	Zir.Inst.RestoreErrRetIndex{
12565	.src_node = gz.nodeIndexToRelative(src_node),
12566	.block = block.toRef(),
12567	.operand = operand,
12568	},
12569	)).toIndex().?,
12570	},
12571	}
12572	}
12573
12574	fn addBreak(
12575	gz: *GenZir,
12576	tag: Zir.Inst.Tag,
12577	block_inst: Zir.Inst.Index,
12578	operand: Zir.Inst.Ref,
12579	) !Zir.Inst.Index {
12580	const gpa = gz.astgen.gpa;
12581	try gz.instructions.ensureUnusedCapacity(gpa, 1);
12582
12583	const new_index = try gz.makeBreak(tag, block_inst, operand);
12584	gz.instructions.appendAssumeCapacity(new_index);
12585	return new_index;
12586	}
12587
12588	fn makeBreak(
12589	gz: *GenZir,
12590	tag: Zir.Inst.Tag,
12591	block_inst: Zir.Inst.Index,
12592	operand: Zir.Inst.Ref,
12593	) !Zir.Inst.Index {
12594	return gz.makeBreakCommon(tag, block_inst, operand, null);
12595	}
12596
12597	fn addBreakWithSrcNode(
12598	gz: *GenZir,
12599	tag: Zir.Inst.Tag,
12600	block_inst: Zir.Inst.Index,
12601	operand: Zir.Inst.Ref,
12602	operand_src_node: Ast.Node.Index,
12603	) !Zir.Inst.Index {
12604	const gpa = gz.astgen.gpa;
12605	try gz.instructions.ensureUnusedCapacity(gpa, 1);
12606
12607	const new_index = try gz.makeBreakWithSrcNode(tag, block_inst, operand, operand_src_node);
12608	gz.instructions.appendAssumeCapacity(new_index);
12609	return new_index;
12610	}
12611
12612	fn makeBreakWithSrcNode(
12613	gz: *GenZir,
12614	tag: Zir.Inst.Tag,
12615	block_inst: Zir.Inst.Index,
12616	operand: Zir.Inst.Ref,
12617	operand_src_node: Ast.Node.Index,
12618	) !Zir.Inst.Index {
12619	return gz.makeBreakCommon(tag, block_inst, operand, operand_src_node);
12620	}
12621
12622	fn makeBreakCommon(
12623	gz: *GenZir,
12624	tag: Zir.Inst.Tag,
12625	block_inst: Zir.Inst.Index,
12626	operand: Zir.Inst.Ref,
12627	operand_src_node: ?Ast.Node.Index,
12628	) !Zir.Inst.Index {
12629	const gpa = gz.astgen.gpa;
12630	try gz.astgen.instructions.ensureUnusedCapacity(gpa, 1);
12631	try gz.astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.Break).Struct.fields.len);
12632
12633	const new_index: Zir.Inst.Index = @enumFromInt(gz.astgen.instructions.len);
12634	gz.astgen.instructions.appendAssumeCapacity(.{
12635	.tag = tag,
12636	.data = .{ .@"break" = .{
12637	.operand = operand,
12638	.payload_index = gz.astgen.addExtraAssumeCapacity(Zir.Inst.Break{
12639	.operand_src_node = if (operand_src_node) \|src_node\|
12640	gz.nodeIndexToRelative(src_node)
12641	else
12642	Zir.Inst.Break.no_src_node,
12643	.block_inst = block_inst,
12644	}),
12645	} },
12646	});
12647	return new_index;
12648	}
12649
12650	fn addBin(
12651	gz: *GenZir,
12652	tag: Zir.Inst.Tag,
12653	lhs: Zir.Inst.Ref,
12654	rhs: Zir.Inst.Ref,
12655	) !Zir.Inst.Ref {
12656	assert(lhs != .none);
12657	assert(rhs != .none);
12658	return gz.add(.{
12659	.tag = tag,
12660	.data = .{ .bin = .{
12661	.lhs = lhs,
12662	.rhs = rhs,
12663	} },
12664	});
12665	}
12666
12667	fn addDefer(gz: *GenZir, index: u32, len: u32) !void {
12668	_ = try gz.add(.{
12669	.tag = .@"defer",
12670	.data = .{ .@"defer" = .{
12671	.index = index,
12672	.len = len,
12673	} },
12674	});
12675	}
12676
12677	fn addDecl(
12678	gz: *GenZir,
12679	tag: Zir.Inst.Tag,
12680	decl_index: u32,
12681	src_node: Ast.Node.Index,
12682	) !Zir.Inst.Ref {
12683	return gz.add(.{
12684	.tag = tag,
12685	.data = .{ .pl_node = .{
12686	.src_node = gz.nodeIndexToRelative(src_node),
12687	.payload_index = decl_index,
12688	} },
12689	});
12690	}
12691
12692	fn addNode(
12693	gz: *GenZir,
12694	tag: Zir.Inst.Tag,
12695	/// Absolute node index. This function does the conversion to offset from Decl.
12696	src_node: Ast.Node.Index,
12697	) !Zir.Inst.Ref {
12698	return gz.add(.{
12699	.tag = tag,
12700	.data = .{ .node = gz.nodeIndexToRelative(src_node) },
12701	});
12702	}
12703
12704	fn addInstNode(
12705	gz: *GenZir,
12706	tag: Zir.Inst.Tag,
12707	inst: Zir.Inst.Index,
12708	/// Absolute node index. This function does the conversion to offset from Decl.
12709	src_node: Ast.Node.Index,
12710	) !Zir.Inst.Ref {
12711	return gz.add(.{
12712	.tag = tag,
12713	.data = .{ .inst_node = .{
12714	.inst = inst,
12715	.src_node = gz.nodeIndexToRelative(src_node),
12716	} },
12717	});
12718	}
12719
12720	fn addNodeExtended(
12721	gz: *GenZir,
12722	opcode: Zir.Inst.Extended,
12723	/// Absolute node index. This function does the conversion to offset from Decl.
12724	src_node: Ast.Node.Index,
12725	) !Zir.Inst.Ref {
12726	return gz.add(.{
12727	.tag = .extended,
12728	.data = .{ .extended = .{
12729	.opcode = opcode,
12730	.small = undefined,
12731	.operand = @bitCast(gz.nodeIndexToRelative(src_node)),
12732	} },
12733	});
12734	}
12735
12736	fn addAllocExtended(
12737	gz: *GenZir,
12738	args: struct {
12739	/// Absolute node index. This function does the conversion to offset from Decl.
12740	node: Ast.Node.Index,
12741	type_inst: Zir.Inst.Ref,
12742	align_inst: Zir.Inst.Ref,
12743	is_const: bool,
12744	is_comptime: bool,
12745	},
12746	) !Zir.Inst.Ref {
12747	const astgen = gz.astgen;
12748	const gpa = astgen.gpa;
12749
12750	try gz.instructions.ensureUnusedCapacity(gpa, 1);
12751	try astgen.instructions.ensureUnusedCapacity(gpa, 1);
12752	try astgen.extra.ensureUnusedCapacity(
12753	gpa,
12754	@typeInfo(Zir.Inst.AllocExtended).Struct.fields.len +
12755	@intFromBool(args.type_inst != .none) +
12756	@intFromBool(args.align_inst != .none),
12757	);
12758	const payload_index = gz.astgen.addExtraAssumeCapacity(Zir.Inst.AllocExtended{
12759	.src_node = gz.nodeIndexToRelative(args.node),
12760	});
12761	if (args.type_inst != .none) {
12762	astgen.extra.appendAssumeCapacity(@intFromEnum(args.type_inst));
12763	}
12764	if (args.align_inst != .none) {
12765	astgen.extra.appendAssumeCapacity(@intFromEnum(args.align_inst));
12766	}
12767
12768	const has_type: u4 = @intFromBool(args.type_inst != .none);
12769	const has_align: u4 = @intFromBool(args.align_inst != .none);
12770	const is_const: u4 = @intFromBool(args.is_const);
12771	const is_comptime: u4 = @intFromBool(args.is_comptime);
12772	const small: u16 = has_type \| (has_align << 1) \| (is_const << 2) \| (is_comptime << 3);
12773
12774	const new_index: Zir.Inst.Index = @enumFromInt(astgen.instructions.len);
12775	astgen.instructions.appendAssumeCapacity(.{
12776	.tag = .extended,
12777	.data = .{ .extended = .{
12778	.opcode = .alloc,
12779	.small = small,
12780	.operand = payload_index,
12781	} },
12782	});
12783	gz.instructions.appendAssumeCapacity(new_index);
12784	return new_index.toRef();
12785	}
12786
12787	fn addAsm(
12788	gz: *GenZir,
12789	args: struct {
12790	tag: Zir.Inst.Extended,
12791	/// Absolute node index. This function does the conversion to offset from Decl.
12792	node: Ast.Node.Index,
12793	asm_source: Zir.NullTerminatedString,
12794	output_type_bits: u32,
12795	is_volatile: bool,
12796	outputs: []const Zir.Inst.Asm.Output,
12797	inputs: []const Zir.Inst.Asm.Input,
12798	clobbers: []const u32,
12799	},
12800	) !Zir.Inst.Ref {
12801	const astgen = gz.astgen;
12802	const gpa = astgen.gpa;
12803
12804	try gz.instructions.ensureUnusedCapacity(gpa, 1);
12805	try astgen.instructions.ensureUnusedCapacity(gpa, 1);
12806	try astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.Asm).Struct.fields.len +
12807	args.outputs.len * @typeInfo(Zir.Inst.Asm.Output).Struct.fields.len +
12808	args.inputs.len * @typeInfo(Zir.Inst.Asm.Input).Struct.fields.len +
12809	args.clobbers.len);
12810
12811	const payload_index = gz.astgen.addExtraAssumeCapacity(Zir.Inst.Asm{
12812	.src_node = gz.nodeIndexToRelative(args.node),
12813	.asm_source = args.asm_source,
12814	.output_type_bits = args.output_type_bits,
12815	});
12816	for (args.outputs) \|output\| {
12817	_ = gz.astgen.addExtraAssumeCapacity(output);
12818	}
12819	for (args.inputs) \|input\| {
12820	_ = gz.astgen.addExtraAssumeCapacity(input);
12821	}
12822	gz.astgen.extra.appendSliceAssumeCapacity(args.clobbers);
12823
12824	// * 0b00000000_000XXXXX - `outputs_len`.
12825	// * 0b000000XX_XXX00000 - `inputs_len`.
12826	// * 0b0XXXXX00_00000000 - `clobbers_len`.
12827	// * 0bX0000000_00000000 - is volatile
12828	const small: u16 = @as(u16, @intCast(args.outputs.len)) \|
12829	@as(u16, @intCast(args.inputs.len << 5)) \|
12830	@as(u16, @intCast(args.clobbers.len << 10)) \|
12831	(@as(u16, @intFromBool(args.is_volatile)) << 15);
12832
12833	const new_index: Zir.Inst.Index = @enumFromInt(astgen.instructions.len);
12834	astgen.instructions.appendAssumeCapacity(.{
12835	.tag = .extended,
12836	.data = .{ .extended = .{
12837	.opcode = args.tag,
12838	.small = small,
12839	.operand = payload_index,
12840	} },
12841	});
12842	gz.instructions.appendAssumeCapacity(new_index);
12843	return new_index.toRef();
12844	}
12845
12846	/// Note that this returns a `Zir.Inst.Index` not a ref.
12847	/// Does not append the block instruction to the scope.
12848	/// Leaves the `payload_index` field undefined.
12849	fn makeBlockInst(gz: *GenZir, tag: Zir.Inst.Tag, node: Ast.Node.Index) !Zir.Inst.Index {
12850	const new_index: Zir.Inst.Index = @enumFromInt(gz.astgen.instructions.len);
12851	const gpa = gz.astgen.gpa;
12852	try gz.astgen.instructions.append(gpa, .{
12853	.tag = tag,
12854	.data = .{ .pl_node = .{
12855	.src_node = gz.nodeIndexToRelative(node),
12856	.payload_index = undefined,
12857	} },
12858	});
12859	return new_index;
12860	}
12861
12862	/// Note that this returns a `Zir.Inst.Index` not a ref.
12863	/// Leaves the `payload_index` field undefined.
12864	fn addCondBr(gz: *GenZir, tag: Zir.Inst.Tag, node: Ast.Node.Index) !Zir.Inst.Index {
12865	const gpa = gz.astgen.gpa;
12866	try gz.instructions.ensureUnusedCapacity(gpa, 1);
12867	const new_index: Zir.Inst.Index = @enumFromInt(gz.astgen.instructions.len);
12868	try gz.astgen.instructions.append(gpa, .{
12869	.tag = tag,
12870	.data = .{ .pl_node = .{
12871	.src_node = gz.nodeIndexToRelative(node),
12872	.payload_index = undefined,
12873	} },
12874	});
12875	gz.instructions.appendAssumeCapacity(new_index);
12876	return new_index;
12877	}
12878
12879	fn setStruct(gz: *GenZir, inst: Zir.Inst.Index, args: struct {
12880	src_node: Ast.Node.Index,
12881	fields_len: u32,
12882	decls_len: u32,
12883	backing_int_ref: Zir.Inst.Ref,
12884	backing_int_body_len: u32,
12885	layout: std.builtin.Type.ContainerLayout,
12886	known_non_opv: bool,
12887	known_comptime_only: bool,
12888	is_tuple: bool,
12889	any_comptime_fields: bool,
12890	any_default_inits: bool,
12891	any_aligned_fields: bool,
12892	fields_hash: std.zig.SrcHash,
12893	}) !void {
12894	const astgen = gz.astgen;
12895	const gpa = astgen.gpa;
12896
12897	// Node 0 is valid for the root `struct_decl` of a file!
12898	assert(args.src_node != 0 or gz.parent.tag == .top);
12899
12900	const fields_hash_arr: [4]u32 = @bitCast(args.fields_hash);
12901
12902	try astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.StructDecl).Struct.fields.len + 4);
12903	const payload_index = astgen.addExtraAssumeCapacity(Zir.Inst.StructDecl{
12904	.fields_hash_0 = fields_hash_arr[0],
12905	.fields_hash_1 = fields_hash_arr[1],
12906	.fields_hash_2 = fields_hash_arr[2],
12907	.fields_hash_3 = fields_hash_arr[3],
12908	.src_node = gz.nodeIndexToRelative(args.src_node),
12909	});
12910
12911	if (args.fields_len != 0) {
12912	astgen.extra.appendAssumeCapacity(args.fields_len);
12913	}
12914	if (args.decls_len != 0) {
12915	astgen.extra.appendAssumeCapacity(args.decls_len);
12916	}
12917	if (args.backing_int_ref != .none) {
12918	astgen.extra.appendAssumeCapacity(args.backing_int_body_len);
12919	if (args.backing_int_body_len == 0) {
12920	astgen.extra.appendAssumeCapacity(@intFromEnum(args.backing_int_ref));
12921	}
12922	}
12923	astgen.instructions.set(@intFromEnum(inst), .{
12924	.tag = .extended,
12925	.data = .{ .extended = .{
12926	.opcode = .struct_decl,
12927	.small = @bitCast(Zir.Inst.StructDecl.Small{
12928	.has_fields_len = args.fields_len != 0,
12929	.has_decls_len = args.decls_len != 0,
12930	.has_backing_int = args.backing_int_ref != .none,
12931	.known_non_opv = args.known_non_opv,
12932	.known_comptime_only = args.known_comptime_only,
12933	.is_tuple = args.is_tuple,
12934	.name_strategy = gz.anon_name_strategy,
12935	.layout = args.layout,
12936	.any_comptime_fields = args.any_comptime_fields,
12937	.any_default_inits = args.any_default_inits,
12938	.any_aligned_fields = args.any_aligned_fields,
12939	}),
12940	.operand = payload_index,
12941	} },
12942	});
12943	}
12944
12945	fn setUnion(gz: *GenZir, inst: Zir.Inst.Index, args: struct {
12946	src_node: Ast.Node.Index,
12947	tag_type: Zir.Inst.Ref,
12948	body_len: u32,
12949	fields_len: u32,
12950	decls_len: u32,
12951	layout: std.builtin.Type.ContainerLayout,
12952	auto_enum_tag: bool,
12953	any_aligned_fields: bool,
12954	fields_hash: std.zig.SrcHash,
12955	}) !void {
12956	const astgen = gz.astgen;
12957	const gpa = astgen.gpa;
12958
12959	assert(args.src_node != 0);
12960
12961	const fields_hash_arr: [4]u32 = @bitCast(args.fields_hash);
12962
12963	try astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.UnionDecl).Struct.fields.len + 4);
12964	const payload_index = astgen.addExtraAssumeCapacity(Zir.Inst.UnionDecl{
12965	.fields_hash_0 = fields_hash_arr[0],
12966	.fields_hash_1 = fields_hash_arr[1],
12967	.fields_hash_2 = fields_hash_arr[2],
12968	.fields_hash_3 = fields_hash_arr[3],
12969	.src_node = gz.nodeIndexToRelative(args.src_node),
12970	});
12971
12972	if (args.tag_type != .none) {
12973	astgen.extra.appendAssumeCapacity(@intFromEnum(args.tag_type));
12974	}
12975	if (args.body_len != 0) {
12976	astgen.extra.appendAssumeCapacity(args.body_len);
12977	}
12978	if (args.fields_len != 0) {
12979	astgen.extra.appendAssumeCapacity(args.fields_len);
12980	}
12981	if (args.decls_len != 0) {
12982	astgen.extra.appendAssumeCapacity(args.decls_len);
12983	}
12984	astgen.instructions.set(@intFromEnum(inst), .{
12985	.tag = .extended,
12986	.data = .{ .extended = .{
12987	.opcode = .union_decl,
12988	.small = @bitCast(Zir.Inst.UnionDecl.Small{
12989	.has_tag_type = args.tag_type != .none,
12990	.has_body_len = args.body_len != 0,
12991	.has_fields_len = args.fields_len != 0,
12992	.has_decls_len = args.decls_len != 0,
12993	.name_strategy = gz.anon_name_strategy,
12994	.layout = args.layout,
12995	.auto_enum_tag = args.auto_enum_tag,
12996	.any_aligned_fields = args.any_aligned_fields,
12997	}),
12998	.operand = payload_index,
12999	} },
13000	});
13001	}
13002
13003	fn setEnum(gz: *GenZir, inst: Zir.Inst.Index, args: struct {
13004	src_node: Ast.Node.Index,
13005	tag_type: Zir.Inst.Ref,
13006	body_len: u32,
13007	fields_len: u32,
13008	decls_len: u32,
13009	nonexhaustive: bool,
13010	fields_hash: std.zig.SrcHash,
13011	}) !void {
13012	const astgen = gz.astgen;
13013	const gpa = astgen.gpa;
13014
13015	assert(args.src_node != 0);
13016
13017	const fields_hash_arr: [4]u32 = @bitCast(args.fields_hash);
13018
13019	try astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.EnumDecl).Struct.fields.len + 4);
13020	const payload_index = astgen.addExtraAssumeCapacity(Zir.Inst.EnumDecl{
13021	.fields_hash_0 = fields_hash_arr[0],
13022	.fields_hash_1 = fields_hash_arr[1],
13023	.fields_hash_2 = fields_hash_arr[2],
13024	.fields_hash_3 = fields_hash_arr[3],
13025	.src_node = gz.nodeIndexToRelative(args.src_node),
13026	});
13027
13028	if (args.tag_type != .none) {
13029	astgen.extra.appendAssumeCapacity(@intFromEnum(args.tag_type));
13030	}
13031	if (args.body_len != 0) {
13032	astgen.extra.appendAssumeCapacity(args.body_len);
13033	}
13034	if (args.fields_len != 0) {
13035	astgen.extra.appendAssumeCapacity(args.fields_len);
13036	}
13037	if (args.decls_len != 0) {
13038	astgen.extra.appendAssumeCapacity(args.decls_len);
13039	}
13040	astgen.instructions.set(@intFromEnum(inst), .{
13041	.tag = .extended,
13042	.data = .{ .extended = .{
13043	.opcode = .enum_decl,
13044	.small = @bitCast(Zir.Inst.EnumDecl.Small{
13045	.has_tag_type = args.tag_type != .none,
13046	.has_body_len = args.body_len != 0,
13047	.has_fields_len = args.fields_len != 0,
13048	.has_decls_len = args.decls_len != 0,
13049	.name_strategy = gz.anon_name_strategy,
13050	.nonexhaustive = args.nonexhaustive,
13051	}),
13052	.operand = payload_index,
13053	} },
13054	});
13055	}
13056
13057	fn setOpaque(gz: *GenZir, inst: Zir.Inst.Index, args: struct {
13058	src_node: Ast.Node.Index,
13059	decls_len: u32,
13060	}) !void {
13061	const astgen = gz.astgen;
13062	const gpa = astgen.gpa;
13063
13064	assert(args.src_node != 0);
13065
13066	try astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.OpaqueDecl).Struct.fields.len + 1);
13067	const payload_index = astgen.addExtraAssumeCapacity(Zir.Inst.OpaqueDecl{
13068	.src_node = gz.nodeIndexToRelative(args.src_node),
13069	});
13070
13071	if (args.decls_len != 0) {
13072	astgen.extra.appendAssumeCapacity(args.decls_len);
13073	}
13074	astgen.instructions.set(@intFromEnum(inst), .{
13075	.tag = .extended,
13076	.data = .{ .extended = .{
13077	.opcode = .opaque_decl,
13078	.small = @bitCast(Zir.Inst.OpaqueDecl.Small{
13079	.has_decls_len = args.decls_len != 0,
13080	.name_strategy = gz.anon_name_strategy,
13081	}),
13082	.operand = payload_index,
13083	} },
13084	});
13085	}
13086
13087	fn add(gz: *GenZir, inst: Zir.Inst) !Zir.Inst.Ref {
13088	return (try gz.addAsIndex(inst)).toRef();
13089	}
13090
13091	fn addAsIndex(gz: *GenZir, inst: Zir.Inst) !Zir.Inst.Index {
13092	const gpa = gz.astgen.gpa;
13093	try gz.instructions.ensureUnusedCapacity(gpa, 1);
13094	try gz.astgen.instructions.ensureUnusedCapacity(gpa, 1);
13095
13096	const new_index: Zir.Inst.Index = @enumFromInt(gz.astgen.instructions.len);
13097	gz.astgen.instructions.appendAssumeCapacity(inst);
13098	gz.instructions.appendAssumeCapacity(new_index);
13099	return new_index;
13100	}
13101
13102	fn reserveInstructionIndex(gz: *GenZir) !Zir.Inst.Index {
13103	const gpa = gz.astgen.gpa;
13104	try gz.instructions.ensureUnusedCapacity(gpa, 1);
13105	try gz.astgen.instructions.ensureUnusedCapacity(gpa, 1);
13106
13107	const new_index: Zir.Inst.Index = @enumFromInt(gz.astgen.instructions.len);
13108	gz.astgen.instructions.len += 1;
13109	gz.instructions.appendAssumeCapacity(new_index);
13110	return new_index;
13111	}
13112
13113	fn addRet(gz: *GenZir, ri: ResultInfo, operand: Zir.Inst.Ref, node: Ast.Node.Index) !void {
13114	switch (ri.rl) {
13115	.ptr => \|ptr_res\| _ = try gz.addUnNode(.ret_load, ptr_res.inst, node),
13116	.coerced_ty => _ = try gz.addUnNode(.ret_node, operand, node),
13117	else => unreachable,
13118	}
13119	}
13120
13121	fn addNamespaceCaptures(gz: GenZir, namespace: Scope.Namespace) !void {
13122	if (namespace.captures.count() > 0) {
13123	try gz.instructions.ensureUnusedCapacity(gz.astgen.gpa, namespace.captures.count());
13124	for (namespace.captures.values()) \|capture\| {
13125	gz.instructions.appendAssumeCapacity(capture);
13126	}
13127	}
13128	}
13129
13130	fn addDbgVar(gz: *GenZir, tag: Zir.Inst.Tag, name: Zir.NullTerminatedString, inst: Zir.Inst.Ref) !void {
13131	if (gz.is_comptime) return;
13132
13133	_ = try gz.add(.{ .tag = tag, .data = .{
13134	.str_op = .{
13135	.str = name,
13136	.operand = inst,
13137	},
13138	} });
13139	}
13140	};
13141
13142	/// This can only be for short-lived references; the memory becomes invalidated
13143	/// when another string is added.
13144	fn nullTerminatedString(astgen: AstGen, index: Zir.NullTerminatedString) [*:0]const u8 {
13145	return @ptrCast(astgen.string_bytes.items[@intFromEnum(index)..]);
13146	}
13147
13148	/// Local variables shadowing detection, including function parameters.
13149	fn detectLocalShadowing(
13150	astgen: *AstGen,
13151	scope: *Scope,
13152	ident_name: Zir.NullTerminatedString,
13153	name_token: Ast.TokenIndex,
13154	token_bytes: []const u8,
13155	id_cat: Scope.IdCat,
13156	) !void {
13157	const gpa = astgen.gpa;
13158	if (token_bytes[0] != '@' and isPrimitive(token_bytes)) {
13159	return astgen.failTokNotes(name_token, "name shadows primitive '{s}'", .{
13160	token_bytes,
13161	}, &[_]u32{
13162	try astgen.errNoteTok(name_token, "consider using @\"{s}\" to disambiguate", .{
13163	token_bytes,
13164	}),
13165	});
13166	}
13167
13168	var s = scope;
13169	var outer_scope = false;
13170	while (true) switch (s.tag) {
13171	.local_val => {
13172	const local_val = s.cast(Scope.LocalVal).?;
13173	if (local_val.name == ident_name) {
13174	const name_slice = mem.span(astgen.nullTerminatedString(ident_name));
13175	const name = try gpa.dupe(u8, name_slice);
13176	defer gpa.free(name);
13177	if (outer_scope) {
13178	return astgen.failTokNotes(name_token, "{s} '{s}' shadows {s} from outer scope", .{
13179	@tagName(id_cat), name, @tagName(local_val.id_cat),
13180	}, &[_]u32{
13181	try astgen.errNoteTok(
13182	local_val.token_src,
13183	"previous declaration here",
13184	.{},
13185	),
13186	});
13187	}
13188	return astgen.failTokNotes(name_token, "redeclaration of {s} '{s}'", .{
13189	@tagName(local_val.id_cat), name,
13190	}, &[_]u32{
13191	try astgen.errNoteTok(
13192	local_val.token_src,
13193	"previous declaration here",
13194	.{},
13195	),
13196	});
13197	}
13198	s = local_val.parent;
13199	},
13200	.local_ptr => {
13201	const local_ptr = s.cast(Scope.LocalPtr).?;
13202	if (local_ptr.name == ident_name) {
13203	const name_slice = mem.span(astgen.nullTerminatedString(ident_name));
13204	const name = try gpa.dupe(u8, name_slice);
13205	defer gpa.free(name);
13206	if (outer_scope) {
13207	return astgen.failTokNotes(name_token, "{s} '{s}' shadows {s} from outer scope", .{
13208	@tagName(id_cat), name, @tagName(local_ptr.id_cat),
13209	}, &[_]u32{
13210	try astgen.errNoteTok(
13211	local_ptr.token_src,
13212	"previous declaration here",
13213	.{},
13214	),
13215	});
13216	}
13217	return astgen.failTokNotes(name_token, "redeclaration of {s} '{s}'", .{
13218	@tagName(local_ptr.id_cat), name,
13219	}, &[_]u32{
13220	try astgen.errNoteTok(
13221	local_ptr.token_src,
13222	"previous declaration here",
13223	.{},
13224	),
13225	});
13226	}
13227	s = local_ptr.parent;
13228	},
13229	.namespace, .enum_namespace => {
13230	outer_scope = true;
13231	const ns = s.cast(Scope.Namespace).?;
13232	const decl_node = ns.decls.get(ident_name) orelse {
13233	s = ns.parent;
13234	continue;
13235	};
13236	const name_slice = mem.span(astgen.nullTerminatedString(ident_name));
13237	const name = try gpa.dupe(u8, name_slice);
13238	defer gpa.free(name);
13239	return astgen.failTokNotes(name_token, "{s} shadows declaration of '{s}'", .{
13240	@tagName(id_cat), name,
13241	}, &[_]u32{
13242	try astgen.errNoteNode(decl_node, "declared here", .{}),
13243	});
13244	},
13245	.gen_zir => {
13246	s = s.cast(GenZir).?.parent;
13247	outer_scope = true;
13248	},
13249	.defer_normal, .defer_error => s = s.cast(Scope.Defer).?.parent,
13250	.top => break,
13251	};
13252	}
13253
13254	const LineColumn = struct { u32, u32 };
13255
13256	/// Advances the source cursor to the main token of `node` if not in comptime scope.
13257	/// Usually paired with `emitDbgStmt`.
13258	fn maybeAdvanceSourceCursorToMainToken(gz: *GenZir, node: Ast.Node.Index) LineColumn {
13259	if (gz.is_comptime) return .{ gz.astgen.source_line - gz.decl_line, gz.astgen.source_column };
13260
13261	const tree = gz.astgen.tree;
13262	const token_starts = tree.tokens.items(.start);
13263	const main_tokens = tree.nodes.items(.main_token);
13264	const node_start = token_starts[main_tokens[node]];
13265	gz.astgen.advanceSourceCursor(node_start);
13266
13267	return .{ gz.astgen.source_line - gz.decl_line, gz.astgen.source_column };
13268	}
13269
13270	/// Advances the source cursor to the beginning of `node`.
13271	fn advanceSourceCursorToNode(astgen: *AstGen, node: Ast.Node.Index) void {
13272	const tree = astgen.tree;
13273	const token_starts = tree.tokens.items(.start);
13274	const node_start = token_starts[tree.firstToken(node)];
13275	astgen.advanceSourceCursor(node_start);
13276	}
13277
13278	/// Advances the source cursor to an absolute byte offset `end` in the file.
13279	fn advanceSourceCursor(astgen: *AstGen, end: usize) void {
13280	const source = astgen.tree.source;
13281	var i = astgen.source_offset;
13282	var line = astgen.source_line;
13283	var column = astgen.source_column;
13284	assert(i <= end);
13285	while (i < end) : (i += 1) {
13286	if (source[i] == '\n') {
13287	line += 1;
13288	column = 0;
13289	} else {
13290	column += 1;
13291	}
13292	}
13293	astgen.source_offset = i;
13294	astgen.source_line = line;
13295	astgen.source_column = column;
13296	}
13297
13298	fn scanDecls(astgen: AstGen, namespace: Scope.Namespace, members: []const Ast.Node.Index) !u32 {
13299	const gpa = astgen.gpa;
13300	const tree = astgen.tree;
13301	const node_tags = tree.nodes.items(.tag);
13302	const main_tokens = tree.nodes.items(.main_token);
13303	const token_tags = tree.tokens.items(.tag);
13304	var decl_count: u32 = 0;
13305	for (members) \|member_node\| {
13306	const name_token = switch (node_tags[member_node]) {
13307	.global_var_decl,
13308	.local_var_decl,
13309	.simple_var_decl,
13310	.aligned_var_decl,
13311	=> blk: {
13312	decl_count += 1;
13313	break :blk main_tokens[member_node] + 1;
13314	},
13315
13316	.fn_proto_simple,
13317	.fn_proto_multi,
13318	.fn_proto_one,
13319	.fn_proto,
13320	.fn_decl,
13321	=> blk: {
13322	decl_count += 1;
13323	const ident = main_tokens[member_node] + 1;
13324	if (token_tags[ident] != .identifier) {
13325	switch (astgen.failNode(member_node, "missing function name", .{})) {
13326	error.AnalysisFail => continue,
13327	error.OutOfMemory => return error.OutOfMemory,
13328	}
13329	}
13330	break :blk ident;
13331	},
13332
13333	.@"comptime", .@"usingnamespace", .test_decl => {
13334	decl_count += 1;
13335	continue;
13336	},
13337
13338	else => continue,
13339	};
13340
13341	const token_bytes = astgen.tree.tokenSlice(name_token);
13342	if (token_bytes[0] != '@' and isPrimitive(token_bytes)) {
13343	switch (astgen.failTokNotes(name_token, "name shadows primitive '{s}'", .{
13344	token_bytes,
13345	}, &[_]u32{
13346	try astgen.errNoteTok(name_token, "consider using @\"{s}\" to disambiguate", .{
13347	token_bytes,
13348	}),
13349	})) {
13350	error.AnalysisFail => continue,
13351	error.OutOfMemory => return error.OutOfMemory,
13352	}
13353	}
13354
13355	const name_str_index = try astgen.identAsString(name_token);
13356	const gop = try namespace.decls.getOrPut(gpa, name_str_index);
13357	if (gop.found_existing) {
13358	const name = try gpa.dupe(u8, mem.span(astgen.nullTerminatedString(name_str_index)));
13359	defer gpa.free(name);
13360	switch (astgen.failNodeNotes(member_node, "redeclaration of '{s}'", .{
13361	name,
13362	}, &[_]u32{
13363	try astgen.errNoteNode(gop.value_ptr.*, "other declaration here", .{}),
13364	})) {
13365	error.AnalysisFail => continue,
13366	error.OutOfMemory => return error.OutOfMemory,
13367	}
13368	}
13369
13370	var s = namespace.parent;
13371	while (true) switch (s.tag) {
13372	.local_val => {
13373	const local_val = s.cast(Scope.LocalVal).?;
13374	if (local_val.name == name_str_index) {
13375	return astgen.failTokNotes(name_token, "declaration '{s}' shadows {s} from outer scope", .{
13376	token_bytes, @tagName(local_val.id_cat),
13377	}, &[_]u32{
13378	try astgen.errNoteTok(
13379	local_val.token_src,
13380	"previous declaration here",
13381	.{},
13382	),
13383	});
13384	}
13385	s = local_val.parent;
13386	},
13387	.local_ptr => {
13388	const local_ptr = s.cast(Scope.LocalPtr).?;
13389	if (local_ptr.name == name_str_index) {
13390	return astgen.failTokNotes(name_token, "declaration '{s}' shadows {s} from outer scope", .{
13391	token_bytes, @tagName(local_ptr.id_cat),
13392	}, &[_]u32{
13393	try astgen.errNoteTok(
13394	local_ptr.token_src,
13395	"previous declaration here",
13396	.{},
13397	),
13398	});
13399	}
13400	s = local_ptr.parent;
13401	},
13402	.namespace, .enum_namespace => s = s.cast(Scope.Namespace).?.parent,
13403	.gen_zir => s = s.cast(GenZir).?.parent,
13404	.defer_normal, .defer_error => s = s.cast(Scope.Defer).?.parent,
13405	.top => break,
13406	};
13407	gop.value_ptr.* = member_node;
13408	}
13409	return decl_count;
13410	}
13411
13412	fn isInferred(astgen: *AstGen, ref: Zir.Inst.Ref) bool {
13413	const inst = ref.toIndex() orelse return false;
13414	const zir_tags = astgen.instructions.items(.tag);
13415	return switch (zir_tags[@intFromEnum(inst)]) {
13416	.alloc_inferred,
13417	.alloc_inferred_mut,
13418	.alloc_inferred_comptime,
13419	.alloc_inferred_comptime_mut,
13420	=> true,
13421
13422	.extended => {
13423	const zir_data = astgen.instructions.items(.data);
13424	if (zir_data[@intFromEnum(inst)].extended.opcode != .alloc) return false;
13425	const small: Zir.Inst.AllocExtended.Small = @bitCast(zir_data[@intFromEnum(inst)].extended.small);
13426	return !small.has_type;
13427	},
13428
13429	else => false,
13430	};
13431	}
13432
13433	/// Assumes capacity for body has already been added. Needed capacity taking into
13434	/// account fixups can be found with `countBodyLenAfterFixups`.
13435	fn appendBodyWithFixups(astgen: *AstGen, body: []const Zir.Inst.Index) void {
13436	return appendBodyWithFixupsArrayList(astgen, &astgen.extra, body);
13437	}
13438
13439	fn appendBodyWithFixupsArrayList(
13440	astgen: *AstGen,
13441	list: *std.ArrayListUnmanaged(u32),
13442	body: []const Zir.Inst.Index,
13443	) void {
13444	for (body) \|body_inst\| {
13445	appendPossiblyRefdBodyInst(astgen, list, body_inst);
13446	}
13447	}
13448
13449	fn appendPossiblyRefdBodyInst(
13450	astgen: *AstGen,
13451	list: *std.ArrayListUnmanaged(u32),
13452	body_inst: Zir.Inst.Index,
13453	) void {
13454	list.appendAssumeCapacity(@intFromEnum(body_inst));
13455	const kv = astgen.ref_table.fetchRemove(body_inst) orelse return;
13456	const ref_inst = kv.value;
13457	return appendPossiblyRefdBodyInst(astgen, list, ref_inst);
13458	}
13459
13460	fn countBodyLenAfterFixups(astgen: *AstGen, body: []const Zir.Inst.Index) u32 {
13461	var count = body.len;
13462	for (body) \|body_inst\| {
13463	var check_inst = body_inst;
13464	while (astgen.ref_table.get(check_inst)) \|ref_inst\| {
13465	count += 1;
13466	check_inst = ref_inst;
13467	}
13468	}
13469	return @intCast(count);
13470	}
13471
13472	fn emitDbgStmt(gz: *GenZir, lc: LineColumn) !void {
13473	if (gz.is_comptime) return;
13474	if (gz.instructions.items.len > 0) {
13475	const astgen = gz.astgen;
13476	const last = gz.instructions.items[gz.instructions.items.len - 1];
13477	if (astgen.instructions.items(.tag)[@intFromEnum(last)] == .dbg_stmt) {
13478	astgen.instructions.items(.data)[@intFromEnum(last)].dbg_stmt = .{
13479	.line = lc[0],
13480	.column = lc[1],
13481	};
13482	return;
13483	}
13484	}
13485
13486	_ = try gz.add(.{ .tag = .dbg_stmt, .data = .{
13487	.dbg_stmt = .{
13488	.line = lc[0],
13489	.column = lc[1],
13490	},
13491	} });
13492	}
13493
13494	/// In some cases, Sema expects us to generate a `dbg_stmt` at the instruction
13495	/// index directly preceding the next instruction (e.g. if a call is %10, it
13496	/// expects a dbg_stmt at %9). TODO: this logic may allow redundant dbg_stmt
13497	/// instructions; fix up Sema so we don't need it!
13498	fn emitDbgStmtForceCurrentIndex(gz: *GenZir, lc: LineColumn) !void {
13499	const astgen = gz.astgen;
13500	if (gz.instructions.items.len > 0 and
13501	@intFromEnum(gz.instructions.items[gz.instructions.items.len - 1]) == astgen.instructions.len - 1)
13502	{
13503	const last = astgen.instructions.len - 1;
13504	if (astgen.instructions.items(.tag)[last] == .dbg_stmt) {
13505	astgen.instructions.items(.data)[last].dbg_stmt = .{
13506	.line = lc[0],
13507	.column = lc[1],
13508	};
13509	return;
13510	}
13511	}
13512
13513	_ = try gz.add(.{ .tag = .dbg_stmt, .data = .{
13514	.dbg_stmt = .{
13515	.line = lc[0],
13516	.column = lc[1],
13517	},
13518	} });
13519	}
13520
13521	fn lowerAstErrors(astgen: *AstGen) !void {
13522	const tree = astgen.tree;
13523	assert(tree.errors.len > 0);
13524
13525	const gpa = astgen.gpa;
13526	const parse_err = tree.errors[0];
13527
13528	var msg: std.ArrayListUnmanaged(u8) = .{};
13529	defer msg.deinit(gpa);
13530
13531	const token_starts = tree.tokens.items(.start);
13532	const token_tags = tree.tokens.items(.tag);
13533
13534	var notes: std.ArrayListUnmanaged(u32) = .{};
13535	defer notes.deinit(gpa);
13536
13537	if (token_tags[parse_err.token + @intFromBool(parse_err.token_is_prev)] == .invalid) {
13538	const tok = parse_err.token + @intFromBool(parse_err.token_is_prev);
13539	const bad_off: u32 = @intCast(tree.tokenSlice(parse_err.token + @intFromBool(parse_err.token_is_prev)).len);
13540	const byte_abs = token_starts[parse_err.token + @intFromBool(parse_err.token_is_prev)] + bad_off;
13541	try notes.append(gpa, try astgen.errNoteTokOff(tok, bad_off, "invalid byte: '{'}'", .{
13542	std.zig.fmtEscapes(tree.source[byte_abs..][0..1]),
13543	}));
13544	}
13545
13546	for (tree.errors[1..]) \|note\| {
13547	if (!note.is_note) break;
13548
13549	msg.clearRetainingCapacity();
13550	try tree.renderError(note, msg.writer(gpa));
13551	try notes.append(gpa, try astgen.errNoteTok(note.token, "{s}", .{msg.items}));
13552	}
13553
13554	const extra_offset = tree.errorOffset(parse_err);
13555	msg.clearRetainingCapacity();
13556	try tree.renderError(parse_err, msg.writer(gpa));
13557	try astgen.appendErrorTokNotesOff(parse_err.token, extra_offset, "{s}", .{msg.items}, notes.items);
13558	}
13559
13560	const DeclarationName = union(enum) {
13561	named: Ast.TokenIndex,
13562	named_test: Ast.TokenIndex,
13563	unnamed_test,
13564	decltest: Zir.NullTerminatedString,
13565	@"comptime",
13566	@"usingnamespace",
13567	};
13568
13569	/// Sets all extra data for a `declaration` instruction.
13570	/// Unstacks `value_gz`, `align_gz`, `linksection_gz`, and `addrspace_gz`.
13571	fn setDeclaration(
13572	decl_inst: Zir.Inst.Index,
13573	src_hash: std.zig.SrcHash,
13574	name: DeclarationName,
13575	line_offset: u32,
13576	is_pub: bool,
13577	is_export: bool,
13578	doc_comment: Zir.NullTerminatedString,
13579	value_gz: *GenZir,
13580	/// May be `null` if all these blocks would be empty.
13581	/// If `null`, then `value_gz` must have nothing stacked on it.
13582	extra_gzs: ?struct {
13583	/// Must be stacked on `value_gz`.
13584	align_gz: *GenZir,
13585	/// Must be stacked on `align_gz`.
13586	linksection_gz: *GenZir,
13587	/// Must be stacked on `linksection_gz`, and have nothing stacked on it.
13588	addrspace_gz: *GenZir,
13589	},
13590	) !void {
13591	const astgen = value_gz.astgen;
13592	const gpa = astgen.gpa;
13593
13594	const empty_body: []Zir.Inst.Index = &.{};
13595	const value_body, const align_body, const linksection_body, const addrspace_body = if (extra_gzs) \|e\| .{
13596	value_gz.instructionsSliceUpto(e.align_gz),
13597	e.align_gz.instructionsSliceUpto(e.linksection_gz),
13598	e.linksection_gz.instructionsSliceUpto(e.addrspace_gz),
13599	e.addrspace_gz.instructionsSlice(),
13600	} else .{ value_gz.instructionsSlice(), empty_body, empty_body, empty_body };
13601
13602	const value_len = astgen.countBodyLenAfterFixups(value_body);
13603	const align_len = astgen.countBodyLenAfterFixups(align_body);
13604	const linksection_len = astgen.countBodyLenAfterFixups(linksection_body);
13605	const addrspace_len = astgen.countBodyLenAfterFixups(addrspace_body);
13606
13607	const true_doc_comment: Zir.NullTerminatedString = switch (name) {
13608	.decltest => \|test_name\| test_name,
13609	else => doc_comment,
13610	};
13611
13612	const src_hash_arr: [4]u32 = @bitCast(src_hash);
13613
13614	const extra: Zir.Inst.Declaration = .{
13615	.src_hash_0 = src_hash_arr[0],
13616	.src_hash_1 = src_hash_arr[1],
13617	.src_hash_2 = src_hash_arr[2],
13618	.src_hash_3 = src_hash_arr[3],
13619	.name = switch (name) {
13620	.named => \|tok\| @enumFromInt(@intFromEnum(try astgen.identAsString(tok))),
13621	.named_test => \|tok\| @enumFromInt(@intFromEnum(try astgen.testNameString(tok))),
13622	.unnamed_test => .unnamed_test,
13623	.decltest => .decltest,
13624	.@"comptime" => .@"comptime",
13625	.@"usingnamespace" => .@"usingnamespace",
13626	},
13627	.line_offset = line_offset,
13628	.flags = .{
13629	.value_body_len = @intCast(value_len),
13630	.is_pub = is_pub,
13631	.is_export = is_export,
13632	.has_doc_comment = true_doc_comment != .empty,
13633	.has_align_linksection_addrspace = align_len != 0 or linksection_len != 0 or addrspace_len != 0,
13634	},
13635	};
13636	astgen.instructions.items(.data)[@intFromEnum(decl_inst)].pl_node.payload_index = try astgen.addExtra(extra);
13637	if (extra.flags.has_doc_comment) {
13638	try astgen.extra.append(gpa, @intFromEnum(true_doc_comment));
13639	}
13640	if (extra.flags.has_align_linksection_addrspace) {
13641	try astgen.extra.appendSlice(gpa, &.{
13642	align_len,
13643	linksection_len,
13644	addrspace_len,
13645	});
13646	}
13647	try astgen.extra.ensureUnusedCapacity(gpa, value_len + align_len + linksection_len + addrspace_len);
13648	astgen.appendBodyWithFixups(value_body);
13649	if (extra.flags.has_align_linksection_addrspace) {
13650	astgen.appendBodyWithFixups(align_body);
13651	astgen.appendBodyWithFixups(linksection_body);
13652	astgen.appendBodyWithFixups(addrspace_body);
13653	}
13654
13655	if (extra_gzs) \|e\| {
13656	e.addrspace_gz.unstack();
13657	e.linksection_gz.unstack();
13658	e.align_gz.unstack();
13659	}
13660	value_gz.unstack();
13661	}

src/Builtin.zig+1-1

...	@@ -296,7 +296,7 @@ const Allocator = std.mem.Allocator;	...	@@ -296,7 +296,7 @@ const Allocator = std.mem.Allocator;
296	const build_options = @import("build_options");	296	const build_options = @import("build_options");
297	const Module = @import("Package/Module.zig");	297	const Module = @import("Package/Module.zig");
298	const assert = std.debug.assert;	298	const assert = std.debug.assert;
299	const AstGen = @import("AstGen.zig");	299	const AstGen = std.zig.AstGen;
300	const File = @import("Module.zig").File;	300	const File = @import("Module.zig").File;
301	const Compilation = @import("Compilation.zig");	301	const Compilation = @import("Compilation.zig");
302	const log = std.log.scoped(.builtin);	302	const log = std.log.scoped(.builtin);

src/Module.zig+1-1

...	@@ -28,7 +28,7 @@ const link = @import("link.zig");	...	@@ -28,7 +28,7 @@ const link = @import("link.zig");
28	const Air = @import("Air.zig");	28	const Air = @import("Air.zig");
29	const Zir = std.zig.Zir;	29	const Zir = std.zig.Zir;
30	const trace = @import("tracy.zig").trace;	30	const trace = @import("tracy.zig").trace;
31	const AstGen = @import("AstGen.zig");	31	const AstGen = std.zig.AstGen;
32	const Sema = @import("Sema.zig");	32	const Sema = @import("Sema.zig");
33	const target_util = @import("target.zig");	33	const target_util = @import("target.zig");
34	const build_options = @import("build_options");	34	const build_options = @import("build_options");

src/main.zig+1-1

...	@@ -25,7 +25,7 @@ const Cache = std.Build.Cache;	...	@@ -25,7 +25,7 @@ const Cache = std.Build.Cache;
25	const target_util = @import("target.zig");	25	const target_util = @import("target.zig");
26	const crash_report = @import("crash_report.zig");	26	const crash_report = @import("crash_report.zig");
27	const Module = @import("Module.zig");	27	const Module = @import("Module.zig");
28	const AstGen = @import("AstGen.zig");	28	const AstGen = std.zig.AstGen;
29	const mingw = @import("mingw.zig");	29	const mingw = @import("mingw.zig");
30	const Server = std.zig.Server;	30	const Server = std.zig.Server;
31		31

src/reduce.zig+1-1

...	@@ -5,7 +5,7 @@ const assert = std.debug.assert;	...	@@ -5,7 +5,7 @@ const assert = std.debug.assert;
5	const fatal = @import("./main.zig").fatal;	5	const fatal = @import("./main.zig").fatal;
6	const Ast = std.zig.Ast;	6	const Ast = std.zig.Ast;
7	const Walk = @import("reduce/Walk.zig");	7	const Walk = @import("reduce/Walk.zig");
8	const AstGen = @import("AstGen.zig");	8	const AstGen = std.zig.AstGen;
9	const Zir = std.zig.Zir;	9	const Zir = std.zig.Zir;
10		10
11	const usage =	11	const usage =

...	@@ -505,6 +505,7 @@ set(ZIG_STAGE2_SOURCES	...	@@ -505,6 +505,7 @@ set(ZIG_STAGE2_SOURCES
505	"${CMAKE_SOURCE_DIR}/lib/std/unicode.zig"	505	"${CMAKE_SOURCE_DIR}/lib/std/unicode.zig"
506	"${CMAKE_SOURCE_DIR}/lib/std/zig.zig"	506	"${CMAKE_SOURCE_DIR}/lib/std/zig.zig"
507	"${CMAKE_SOURCE_DIR}/lib/std/zig/Ast.zig"	507	"${CMAKE_SOURCE_DIR}/lib/std/zig/Ast.zig"
		508	"${CMAKE_SOURCE_DIR}/lib/std/zig/AstGen.zig"
508	"${CMAKE_SOURCE_DIR}/lib/std/zig/AstRlAnnotate.zig"	509	"${CMAKE_SOURCE_DIR}/lib/std/zig/AstRlAnnotate.zig"
509	"${CMAKE_SOURCE_DIR}/lib/std/zig/c_builtins.zig"	510	"${CMAKE_SOURCE_DIR}/lib/std/zig/c_builtins.zig"
510	"${CMAKE_SOURCE_DIR}/lib/std/zig/Parse.zig"	511	"${CMAKE_SOURCE_DIR}/lib/std/zig/Parse.zig"
...	@@ -517,7 +518,6 @@ set(ZIG_STAGE2_SOURCES	...	@@ -517,7 +518,6 @@ set(ZIG_STAGE2_SOURCES
517	"${CMAKE_SOURCE_DIR}/lib/std/zig/tokenizer.zig"	518	"${CMAKE_SOURCE_DIR}/lib/std/zig/tokenizer.zig"
518	"${CMAKE_SOURCE_DIR}/lib/std/zig/Zir.zig"	519	"${CMAKE_SOURCE_DIR}/lib/std/zig/Zir.zig"
519	"${CMAKE_SOURCE_DIR}/src/Air.zig"	520	"${CMAKE_SOURCE_DIR}/src/Air.zig"
520	"${CMAKE_SOURCE_DIR}/src/AstGen.zig"
521	"${CMAKE_SOURCE_DIR}/src/Compilation.zig"	521	"${CMAKE_SOURCE_DIR}/src/Compilation.zig"
522	"${CMAKE_SOURCE_DIR}/src/Compilation/Config.zig"	522	"${CMAKE_SOURCE_DIR}/src/Compilation/Config.zig"
523	"${CMAKE_SOURCE_DIR}/src/Liveness.zig"	523	"${CMAKE_SOURCE_DIR}/src/Liveness.zig"