Meg's Mirrors

zig

1	// Compilation
2	pt: Zcu.PerThread,
3	zcu: *Zcu,
4	gpa: Allocator,
5	arena: Allocator,
6	air: Air,
7	liveness: Air.Liveness,
8	owner_nav: InternPool.Nav.Index,
9	base_line: u32,
10
11	// Module-level output (accumulated across the nav's codegen)
12	next_result_id: Word = 1,
13	decls: std.ArrayList(Decl) = .empty,
14	decl_deps: std.ArrayList(Decl.Index) = .empty,
15	nav_link: std.AutoHashMapUnmanaged(InternPool.Nav.Index, Decl.Index) = .empty,
16	uav_link: std.AutoHashMapUnmanaged(struct { InternPool.Index, spec.StorageClass }, Decl.Index) = .empty,
17	entry_points: std.array_hash_map.Auto(Id, EntryPoint) = .empty,
18	error_buffer: ?Decl.Index = null,
19	struct_types: std.array_hash_map.Custom(StructType, Id, StructType.HashContext, true) = .empty,
20	/// SPIR-V ids of OpVariables whose pointee is a Block struct
21	block_var_ids: std.AutoHashMapUnmanaged(Id, void) = .empty,
22	builtins: std.AutoHashMapUnmanaged(struct { spec.BuiltIn, spec.StorageClass }, Decl.Index) = .empty,
23	sections: struct {
24	// Module layout, according to SPIR-V Spec section 2.4, "Logical Layout of a Module".
25	extended_instruction_set: Section = .{},
26	memory_model: Section = .{},
27	execution_modes: Section = .{},
28	debug_strings: Section = .{},
29	debug_names: Section = .{},
30	annotations: Section = .{},
31	globals: Section = .{},
32	functions: Section = .{},
33	} = .{},
34
35	// Per-function state (reset between top-level genNav calls)
36	prologue: Section = .{},
37	body: Section = .{},
38	args: std.ArrayList(Id) = .empty,
39	next_arg_index: u32 = 0,
40	/// Caches the limb extractions for composite integer values so repeated
41	/// arithmetic on the same operand doesn't re-emit `OpCompositeExtract` per
42	/// limb per use. Slices are owned by `cg.arena`.
43	composite_limbs: std.AutoHashMapUnmanaged(Id, []const Id) = .empty,
44	block_stack: std.ArrayList(*Block) = .empty,
45	block_label: Id = .none,
46	/// Whether the current block has been terminated by a terminator
47	/// instruction (e.g. OpKill from inline assembly). When true, no further
48	/// branch instructions should be emitted for the current block.
49	block_terminated: bool = false,
50	block_results: std.AutoHashMapUnmanaged(Air.Inst.Index, Id) = .empty,
51	inst_results: std.AutoHashMapUnmanaged(Air.Inst.Index, Id) = .empty,
52	tracked_allocas: std.AutoHashMapUnmanaged(Id, ?Id) = .empty,
53	loop_switches: std.AutoHashMapUnmanaged(Air.Inst.Index, LoopSwitch) = .empty,
54	id_scratch: std.ArrayList(Id) = .empty,
55
56	fn hasInt64(target: *const std.Target) bool {
57	return target.cpu.arch == .spirv64 or target.cpu.has(.spirv, .int64);
58	}
59
60	fn bigIntBits(cg: *const CodeGen) u16 {
61	return if (hasInt64(cg.zcu.getTarget())) 64 else 32;
62	}
63
64	fn limbType(cg: *const CodeGen) Type {
65	return if (cg.bigIntBits() == 64) .u64 else .u32;
66	}
67
68	fn limbTypeId(cg: *CodeGen) !Id {
69	return cg.resolveType(cg.limbType(), .direct);
70	}
71
72	/// Data can be lowered into in two basic representations: indirect, which is when
73	/// a type is stored in memory, and direct, which is how a type is stored when its
74	/// a direct SPIR-V value.
75	pub const Repr = enum {
76	/// A SPIR-V value as it would be used in operations.
77	direct,
78	/// A SPIR-V value as it is stored in memory.
79	indirect,
80	};
81
82	/// A function or global, tracked here so the linker can order globals and build
83	/// per-entry-point interface lists.
84	pub const Decl = struct {
85	pub const Index = enum(u32) { _ };
86	pub const Kind = enum { func, global, invocation_global };
87
88	kind: Kind,
89	/// Result-id of the associated OpFunction / OpVariable / InvocationGlobal.
90	result_id: Id,
91	/// Range into `decl_deps` for this decl's dependencies.
92	begin_dep: usize = 0,
93	end_dep: usize = 0,
94	/// Whether an extern-function stub has been emitted.
95	has_extern_stub: bool = false,
96	};
97
98	pub const EntryPoint = struct {
99	decl_index: Decl.Index,
100	name: []const u8,
101	cc: std.builtin.CallingConvention,
102	};
103
104	const StructType = struct {
105	fields: []const Id,
106	ip_index: InternPool.Index,
107
108	const HashContext = struct {
109	pub fn hash(_: @This(), ty: StructType) u32 {
110	var hasher = std.hash.Wyhash.init(0);
111	hasher.update(std.mem.sliceAsBytes(ty.fields));
112	hasher.update(std.mem.asBytes(&ty.ip_index));
113	return @truncate(hasher.final());
114	}
115
116	pub fn eql(_: @This(), a: StructType, b: StructType, _: usize) bool {
117	return a.ip_index == b.ip_index and std.mem.eql(Id, a.fields, b.fields);
118	}
119	};
120	};
121
122	pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
123	return comptime &.initMany(&.{
124	.expand_bit_cast_safe,
125	.expand_int_cast_safe,
126	.expand_int_from_float_safe,
127	.expand_int_from_float_optimized_safe,
128	.expand_add_safe,
129	.expand_sub_safe,
130	.expand_mul_safe,
131
132	.expand_array_splat,
133	.expand_array_to_vector,
134	});
135	}
136
137	const LoopSwitch = struct { cond_var: Id, continue_label: Id };
138
139	/// Pointer-typed AIR refs should resolve through `resolvePtr` to handle the
140	/// `tracked_allocas` case explicitly at every use site.
141	const Ptr = union(enum) {
142	id: Id,
143	/// Function-local pointer whose value lives in `tracked_allocas` rather
144	/// than a real OpVariable. `slot` is the current pointee value.
145	tracked: struct { id: Id, slot: *?Id },
146	};
147
148	/// Tracks how control flow leaves a Zig `block` under SPIR-V's structured
149	/// control flow rules.
150	const Block = union(enum) {
151	const Incoming = struct {
152	src_label: Id,
153	/// Block index (u32) that control flow should jump to next.
154	next_block: Id,
155	};
156
157	const SelectionMerge = struct {
158	incoming: Incoming,
159	/// Label of the cond_br's merge block (undefined for top-of-stack).
160	merge_block: Id,
161	};
162
163	/// Selection blocks can't use early exits. Closing requires a "merge ladder"
164	/// of nested OpSelectionMerge instructions, one per pending merge.
165	selection: struct {
166	merge_stack: std.ArrayList(SelectionMerge) = .empty,
167	},
168	/// Loop blocks early-exit by jumping to the loop merge label.
169	loop: struct {
170	merges: std.ArrayList(Incoming) = .empty,
171	merge_block: Id,
172	},
173
174	fn deinit(block: *Block, gpa: Allocator) void {
175	switch (block.*) {
176	.selection => |*merge| merge.merge_stack.deinit(gpa),
177	.loop => |*merge| merge.merges.deinit(gpa),
178	}
179	block.* = undefined;
180	}
181	};
182
183	pub fn deinit(cg: *CodeGen) void {
184	const gpa = cg.gpa;
185	cg.block_stack.deinit(gpa);
186	cg.block_results.deinit(gpa);
187	cg.args.deinit(gpa);
188	cg.composite_limbs.deinit(gpa);
189	cg.tracked_allocas.deinit(gpa);
190	cg.inst_results.deinit(gpa);
191	cg.loop_switches.deinit(gpa);
192	cg.id_scratch.deinit(gpa);
193	cg.prologue.deinit(gpa);
194	cg.body.deinit(gpa);
195
196	cg.nav_link.deinit(gpa);
197	cg.uav_link.deinit(gpa);
198
199	cg.sections.extended_instruction_set.deinit(gpa);
200	cg.sections.memory_model.deinit(gpa);
201	cg.sections.execution_modes.deinit(gpa);
202	cg.sections.debug_strings.deinit(gpa);
203	cg.sections.debug_names.deinit(gpa);
204	cg.sections.annotations.deinit(gpa);
205	cg.sections.globals.deinit(gpa);
206	cg.sections.functions.deinit(gpa);
207
208	cg.struct_types.deinit(gpa);
209	cg.block_var_ids.deinit(gpa);
210	cg.builtins.deinit(gpa);
211
212	cg.decls.deinit(gpa);
213	cg.decl_deps.deinit(gpa);
214	cg.entry_points.deinit(gpa);
215	}
216
217	pub fn generate(
218	_: *link.File,
219	pt: Zcu.PerThread,
220	func_index: InternPool.Index,
221	air: *const Air,
222	liveness: *const ?Air.Liveness,
223	) codegen.Error!Mir {
224	const zcu = pt.zcu;
225	const gpa = zcu.gpa;
226	const nav = zcu.funcInfo(func_index).owner_nav;
227
228	var arena = std.heap.ArenaAllocator.init(gpa);
229	defer arena.deinit();
230
231	var cg: CodeGen = .{
232	.pt = pt,
233	.gpa = gpa,
234	.arena = arena.allocator(),
235	.zcu = zcu,
236	.air = air.*,
237	.liveness = liveness.*.?,
238	.owner_nav = nav,
239	.base_line = zcu.navSrcLine(nav),
240	};
241	defer cg.deinit();
242
243	cg.genNav(true) catch |err| switch (err) {
244	error.AlreadyReported => return error.AlreadyReported,
245	error.OutOfMemory => return error.OutOfMemory,
246	};
247
248	return cg.serializeToMir(gpa);
249	}
250
251	pub fn generateNav(
252	pt: Zcu.PerThread,
253	nav_index: InternPool.Nav.Index,
254	) codegen.Error!Mir {
255	const zcu = pt.zcu;
256	const gpa = zcu.gpa;
257
258	var arena = std.heap.ArenaAllocator.init(gpa);
259	defer arena.deinit();
260
261	var cg: CodeGen = .{
262	.pt = pt,
263	.gpa = gpa,
264	.arena = arena.allocator(),
265	.zcu = zcu,
266	.air = undefined,
267	.liveness = undefined,
268	.owner_nav = nav_index,
269	.base_line = zcu.navSrcLine(nav_index),
270	};
271	defer cg.deinit();
272
273	cg.genNav(false) catch |err| switch (err) {
274	error.AlreadyReported => return error.AlreadyReported,
275	error.OutOfMemory => return error.OutOfMemory,
276	};
277
278	return cg.serializeToMir(gpa);
279	}
280
281	fn serializeToMir(cg: *CodeGen, gpa: Allocator) codegen.Error!Mir {
282	const owner_entry = cg.nav_link.get(cg.owner_nav);
283	const owner_decl_index = owner_entry orelse return .{
284	.id_bound = cg.next_result_id,
285	.owner_nav = cg.owner_nav,
286	.kind = .func,
287	.decl_result_id = .none,
288	.extended_instruction_set = &.{},
289	.globals = &.{},
290	.functions = &.{},
291	.annotations = &.{},
292	.debug_names = &.{},
293	.debug_strings = &.{},
294	.execution_modes = &.{},
295	.nav_refs = &.{},
296	.uav_refs = &.{},
297	.decl_deps = &.{},
298	.internal_globals = &.{},
299	.entry_points = &.{},
300	};
301
302	const owner_decl = cg.declPtr(owner_decl_index);
303
304	var nav_refs: std.ArrayList(Mir.NavRef) = .empty;
305	defer nav_refs.deinit(gpa);
306	var nav_it = cg.nav_link.iterator();
307	while (nav_it.next()) |entry| {
308	if (entry.key_ptr.* == cg.owner_nav) continue;
309	const decl = cg.declPtr(entry.value_ptr.*);
310	try nav_refs.append(gpa, .{
311	.local_id = decl.result_id,
312	.nav = entry.key_ptr.*,
313	.kind = decl.kind,
314	});
315	}
316
317	var uav_refs: std.ArrayList(Mir.UavRef) = .empty;
318	defer uav_refs.deinit(gpa);
319	var uav_it = cg.uav_link.iterator();
320	while (uav_it.next()) |entry| {
321	const decl = cg.declPtr(entry.value_ptr.*);
322	try uav_refs.append(gpa, .{
323	.local_id = decl.result_id,
324	.val = entry.key_ptr.*[0],
325	.storage_class = entry.key_ptr.*[1],
326	.kind = decl.kind,
327	});
328	}
329
330	var decl_deps: std.ArrayList(Mir.DeclDep) = .empty;
331	defer decl_deps.deinit(gpa);
332	var internal_globals: std.ArrayList(Id) = .empty;
333	defer internal_globals.deinit(gpa);
334
335	const deps = cg.decl_deps.items[owner_decl.begin_dep..owner_decl.end_dep];
336	for (deps) |dep_index| {
337	const dep_decl = cg.declPtr(dep_index);
338	var found = false;
339	nav_it.index = 0;
340	while (nav_it.next()) |entry| {
341	if (entry.value_ptr.* == dep_index) {
342	try decl_deps.append(gpa, .{
343	.kind = dep_decl.kind,
344	.nav = entry.key_ptr.*,
345	});
346	found = true;
347	break;
348	}
349	}
350	if (!found and dep_decl.kind == .global) {
351	try internal_globals.append(gpa, dep_decl.result_id);
352	}
353	}
354
355	var ep_list: std.ArrayList(Mir.EntryPoint) = .empty;
356	defer ep_list.deinit(gpa);
357	var ep_it = cg.entry_points.iterator();
358	while (ep_it.next()) |entry| {
359	const ep = entry.value_ptr;
360	const ep_decl = cg.declPtr(ep.decl_index);
361	try ep_list.append(gpa, .{
362	.local_id = ep_decl.result_id,
363	.name = try gpa.dupe(u8, ep.name),
364	.cc = ep.cc,
365	});
366	}
367
368	return .{
369	.id_bound = cg.next_result_id,
370	.owner_nav = cg.owner_nav,
371	.kind = owner_decl.kind,
372	.decl_result_id = owner_decl.result_id,
373	.extended_instruction_set = try cg.sections.extended_instruction_set.instructions.toOwnedSlice(gpa),
374	.globals = try cg.sections.globals.instructions.toOwnedSlice(gpa),
375	.functions = try cg.sections.functions.instructions.toOwnedSlice(gpa),
376	.annotations = try cg.sections.annotations.instructions.toOwnedSlice(gpa),
377	.debug_names = try cg.sections.debug_names.instructions.toOwnedSlice(gpa),
378	.debug_strings = try cg.sections.debug_strings.instructions.toOwnedSlice(gpa),
379	.execution_modes = try cg.sections.execution_modes.instructions.toOwnedSlice(gpa),
380	.nav_refs = try nav_refs.toOwnedSlice(gpa),
381	.uav_refs = try uav_refs.toOwnedSlice(gpa),
382	.decl_deps = try decl_deps.toOwnedSlice(gpa),
383	.internal_globals = try internal_globals.toOwnedSlice(gpa),
384	.entry_points = try ep_list.toOwnedSlice(gpa),
385	};
386	}
387
388	fn typeOf(cg: *CodeGen, inst: Air.Inst.Ref) Type {
389	const zcu = cg.zcu;
390	return cg.air.typeOf(inst, &zcu.intern_pool);
391	}
392
393	fn typeOfIndex(cg: *CodeGen, inst: Air.Inst.Index) Type {
394	const zcu = cg.zcu;
395	return cg.air.typeOfIndex(inst, &zcu.intern_pool);
396	}
397
398	/// Does not generate the nav.
399	pub fn resolveNav(cg: *CodeGen, ip: *InternPool, nav_index: InternPool.Nav.Index) !Decl.Index {
400	const entry = try cg.nav_link.getOrPut(cg.gpa, nav_index);
401	if (!entry.found_existing) {
402	const nav = ip.getNav(nav_index);
403	// TODO: Extern fn?
404	const kind: Decl.Kind = if (ip.isFunctionType(nav.resolved.?.type))
405	.func
406	else switch (nav.resolved.?.@"addrspace") {
407	.generic => .invocation_global,
408	else => .global,
409	};
410	entry.value_ptr.* = try cg.allocDecl(kind);
411	}
412
413	return entry.value_ptr.*;
414	}
415
416	pub fn allocIds(cg: *CodeGen, n: u32) spec.IdRange {
417	defer cg.next_result_id += n;
418	return .{ .base = cg.next_result_id, .len = n };
419	}
420
421	pub fn allocId(cg: *CodeGen) Id {
422	return cg.allocIds(1).at(0);
423	}
424
425	pub fn idBound(cg: *const CodeGen) Word {
426	return cg.next_result_id;
427	}
428
429	pub fn addEntryPointDeps(
430	cg: *CodeGen,
431	decl_index: Decl.Index,
432	seen: *std.bit_set.Dynamic,
433	interface: *std.ArrayList(Id),
434	) !void {
435	const decl = cg.declPtr(decl_index);
436	const deps = cg.decl_deps.items[decl.begin_dep..decl.end_dep];
437
438	if (seen.isSet(@backingInt(decl_index))) {
439	return;
440	}
441
442	seen.set(@backingInt(decl_index));
443
444	if (decl.kind == .global) {
445	try interface.append(cg.gpa, decl.result_id);
446	}
447
448	for (deps) |dep| {
449	try cg.addEntryPointDeps(dep, seen, interface);
450	}
451	}
452
453	pub fn importInstructionSet(cg: *CodeGen, set: spec.InstructionSet) !Id {
454	assert(set != .core);
455	const result_id = cg.allocId();
456	try cg.sections.extended_instruction_set.emit(cg.gpa, .OpExtInstImport, .{
457	.id_result = result_id,
458	.name = @tagName(set),
459	});
460	return result_id;
461	}
462
463	pub fn boolType(cg: *CodeGen) !Id {
464	const result_id = cg.allocId();
465	try cg.sections.globals.emit(cg.gpa, .OpTypeBool, .{
466	.id_result = result_id,
467	});
468	return result_id;
469	}
470
471	pub fn voidType(cg: *CodeGen) !Id {
472	const result_id = cg.allocId();
473	try cg.sections.globals.emit(cg.gpa, .OpTypeVoid, .{
474	.id_result = result_id,
475	});
476	try cg.debugName(result_id, "void");
477	return result_id;
478	}
479
480	pub fn opaqueType(cg: *CodeGen, name: []const u8) !Id {
481	const result_id = cg.allocId();
482	try cg.sections.globals.emit(cg.gpa, .OpTypeOpaque, .{
483	.id_result = result_id,
484	.literal_string = name,
485	});
486	try cg.debugName(result_id, name);
487	return result_id;
488	}
489
490	pub fn backingIntBits(cg: *const CodeGen, bits: u16) struct { u16, bool } {
491	assert(bits != 0);
492	const target = cg.zcu.getTarget();
493	const ints = [_]struct { bits: u16, enabled: bool }{
494	.{ .bits = 8, .enabled = target.cpu.has(.spirv, .int8) },
495	.{ .bits = 16, .enabled = target.cpu.has(.spirv, .int16) },
496	.{ .bits = 32, .enabled = true },
497	.{ .bits = 64, .enabled = hasInt64(target) },
498	};
499
500	for (ints) |int| {
501	if (bits <= int.bits and int.enabled) return .{ int.bits, false };
502	}
503
504	return .{ std.mem.alignForward(u16, bits, cg.bigIntBits()), true };
505	}
506
507	pub fn intType(cg: *CodeGen, signedness: std.lang.Signedness, bits: u16) !Id {
508	assert(bits > 0);
509
510	const target = cg.zcu.getTarget();
511	const actual_signedness = switch (target.os.tag) {
512	// Kernel only supports unsigned ints.
513	.opencl, .amdhsa => .unsigned,
514	else => signedness,
515	};
516	const backing_bits, const big_int = cg.backingIntBits(bits);
517	if (big_int) {
518	const limb_bits = cg.bigIntBits();
519	const limb_ty = try cg.intType(.unsigned, limb_bits);
520	const len_ty = try cg.intType(.unsigned, 32);
521	const len_id = cg.allocId();
522	try cg.sections.globals.emit(cg.gpa, .OpConstant, .{
523	.id_result_type = len_ty,
524	.id_result = len_id,
525	.value = .{ .uint32 = backing_bits / limb_bits },
526	});
527	return cg.arrayType(len_id, limb_ty);
528	}
529
530	const result_id = cg.allocId();
531	try cg.sections.globals.emit(cg.gpa, .OpTypeInt, .{
532	.id_result = result_id,
533	.width = backing_bits,
534	.signedness = switch (actual_signedness) {
535	.signed => 1,
536	.unsigned => 0,
537	},
538	});
539	switch (actual_signedness) {
540	.signed => try cg.debugNameFmt(result_id, "i{}", .{backing_bits}),
541	.unsigned => try cg.debugNameFmt(result_id, "u{}", .{backing_bits}),
542	}
543	return result_id;
544	}
545
546	pub fn floatType(cg: *CodeGen, bits: u16) !Id {
547	assert(bits > 0);
548	const result_id = cg.allocId();
549	try cg.sections.globals.emit(cg.gpa, .OpTypeFloat, .{
550	.id_result = result_id,
551	.width = bits,
552	});
553	try cg.debugNameFmt(result_id, "f{}", .{bits});
554	return result_id;
555	}
556
557	pub fn vectorType(cg: *CodeGen, len: u32, child_ty_id: Id) !Id {
558	const result_id = cg.allocId();
559	try cg.sections.globals.emit(cg.gpa, .OpTypeVector, .{
560	.id_result = result_id,
561	.component_type = child_ty_id,
562	.component_count = len,
563	});
564	return result_id;
565	}
566
567	pub fn arrayType(cg: *CodeGen, len_id: Id, child_ty_id: Id) !Id {
568	const result_id = cg.allocId();
569	try cg.sections.globals.emit(cg.gpa, .OpTypeArray, .{
570	.id_result = result_id,
571	.element_type = child_ty_id,
572	.length = len_id,
573	});
574	return result_id;
575	}
576
577	pub fn ptrType(cg: *CodeGen, child_ty_id: Id, storage_class: spec.StorageClass) !Id {
578	const result_id = cg.allocId();
579	try cg.sections.globals.emit(cg.gpa, .OpTypePointer, .{
580	.id_result = result_id,
581	.storage_class = storage_class,
582	.type = child_ty_id,
583	});
584	return result_id;
585	}
586
587	pub fn structType(
588	cg: *CodeGen,
589	types: []const Id,
590	maybe_names: ?[]const []const u8,
591	ip_index: InternPool.Index,
592	) !Id {
593	const actual_ip_index = if (cg.zcu.comp.config.root_strip) .none else ip_index;
594
595	if (cg.struct_types.get(.{ .fields = types, .ip_index = actual_ip_index })) |id| return id;
596	const result_id = cg.allocId();
597	const types_dup = try cg.arena.dupe(Id, types);
598	try cg.sections.globals.emit(cg.gpa, .OpTypeStruct, .{
599	.id_result = result_id,
600	.id_ref = types_dup,
601	});
602
603	if (maybe_names) |names| {
604	assert(names.len == types.len);
605	for (names, 0..) |name, i| {
606	try cg.memberDebugName(result_id, @intCast(i), name);
607	}
608	}
609
610	try cg.struct_types.put(
611	cg.gpa,
612	.{ .fields = types_dup, .ip_index = actual_ip_index },
613	result_id,
614	);
615	return result_id;
616	}
617
618	/// Returns the layout-decorated variant of `ty` for use inside a Vulkan/OpenGL
619	/// interface block. Vulkan forbids nested Block decorations, so recursive calls
620	/// pass `false`, except through an array, whose elements are each a
621	/// block of their own.
622	///
623	/// This is distinct from `resolveType` because SPIR-V forbids such decorations
624	/// on the pointee of a Function-scope variable.
625	pub fn layoutType(cg: *CodeGen, ty: Type, is_block_root: bool) Error!Id {
626	const gpa = cg.gpa;
627	const zcu = cg.zcu;
628	const ip = &zcu.intern_pool;
629
630	const result_id: Id = switch (ty.zigTypeTag(zcu)) {
631	.@"struct" => id: {
632	const struct_type = ip.loadStructType(ty.toIntern());
633	if (struct_type.layout == .@"packed") return cg.resolveType(ty, .indirect);
634
635	var member_types: std.ArrayList(Id) = .empty;
636	defer member_types.deinit(gpa);
637	const id = cg.allocId();
638	if (is_block_root) try cg.decorate(id, .block);
639	var it = struct_type.iterateRuntimeOrder(ip);
640	while (it.next()) |field_index| {
641	const field_ty: Type = .fromInterned(struct_type.field_types.get(ip)[field_index]);
642	if (!field_ty.hasRuntimeBits(zcu)) continue;
643	try cg.decorateMember(id, @intCast(member_types.items.len), .{ .offset = .{
644	.byte_offset = @intCast(ty.structFieldOffset(field_index, zcu)),
645	} });
646	try member_types.append(gpa, try cg.layoutType(field_ty, false));
647	}
648	try cg.sections.globals.emit(gpa, .OpTypeStruct, .{
649	.id_result = id,
650	.id_ref = member_types.items,
651	});
652	break :id id;
653	},
654	.@"union" => id: {
655	const union_obj = zcu.typeToUnion(ty).?;
656	if (union_obj.layout == .@"packed") return cg.resolveType(ty, .indirect);
657
658	const layout = cg.unionLayout(ty);
659	if (!layout.has_payload) return cg.resolveType(ty, .indirect);
660
661	const id = cg.allocId();
662	if (is_block_root) try cg.decorate(id, .block);
663
664	var member_types: [4]Id = undefined;
665	const u8_id = try cg.resolveType(.u8, .direct);
666	if (layout.tag_size != 0) {
667	const tag_ty: Type = .fromInterned(union_obj.enum_tag_type);
668	try cg.decorateMember(id, layout.tag_index, .{ .offset = .{
669	.byte_offset = @intCast(ty.unionGetLayout(zcu).tagOffset()),
670	} });
671	member_types[layout.tag_index] = try cg.layoutType(tag_ty, false);
672	}
673	if (layout.payload_size != 0) {
674	try cg.decorateMember(id, layout.payload_index, .{ .offset = .{
675	.byte_offset = @intCast(ty.unionGetLayout(zcu).payloadOffset()),
676	} });
677	member_types[layout.payload_index] = try cg.layoutType(layout.payload_ty, false);
678	}
679	if (layout.payload_padding_size != 0) {
680	const len_id = try cg.constInt(.u32, layout.payload_padding_size);
681	const arr_id = try cg.arrayType(len_id, u8_id);
682	try cg.decorate(arr_id, .{ .array_stride = .{ .array_stride = 1 } });
683	member_types[layout.payload_padding_index] = arr_id;
684	}
685	if (layout.padding_size != 0) {
686	const len_id = try cg.constInt(.u32, layout.padding_size);
687	const arr_id = try cg.arrayType(len_id, u8_id);
688	try cg.decorate(arr_id, .{ .array_stride = .{ .array_stride = 1 } });
689	member_types[layout.padding_index] = arr_id;
690	}
691	try cg.sections.globals.emit(gpa, .OpTypeStruct, .{
692	.id_result = id,
693	.id_ref = member_types[0..layout.total_fields],
694	});
695	break :id id;
696	},
697	.array => id: {
698	const elem_ty = ty.childType(zcu);
699	const elem_ty_id = try cg.layoutType(elem_ty, is_block_root);
700	const total_len = std.math.cast(u32, ty.arrayLenIncludingSentinel(zcu)) orelse
701	return cg.fail("array type of {} elements is too large", .{ty.arrayLenIncludingSentinel(zcu)});
702	const id = try cg.arrayType(try cg.constInt(.u32, total_len), elem_ty_id);
703	if (!is_block_root and elem_ty.hasRuntimeBits(zcu)) {
704	try cg.decorate(id, .{
705	.array_stride = .{ .array_stride = @intCast(elem_ty.abiSize(zcu)) },
706	});
707	}
708	break :id id;
709	},
710	.spirv => if (ty.isSpirvRuntimeArray(zcu)) id: {
711	const elem_ty = ty.childType(zcu);
712	const elem_ty_id = try cg.layoutType(elem_ty, is_block_root);
713	const id = cg.allocId();
714	try cg.sections.globals.emit(gpa, .OpTypeRuntimeArray, .{
715	.id_result = id,
716	.element_type = elem_ty_id,
717	});
718	if (!is_block_root and elem_ty.hasRuntimeBits(zcu)) {
719	try cg.decorate(id, .{
720	.array_stride = .{ .array_stride = @intCast(elem_ty.abiSize(zcu)) },
721	});
722	}
723	break :id id;
724	} else return cg.resolveType(ty, .indirect),
725	else => return cg.resolveType(ty, .indirect),
726	};
727
728	return result_id;
729	}
730
731	pub fn functionType(cg: *CodeGen, return_ty_id: Id, param_type_ids: []const Id) !Id {
732	const result_id = cg.allocId();
733	try cg.sections.globals.emit(cg.gpa, .OpTypeFunction, .{
734	.id_result = result_id,
735	.return_type = return_ty_id,
736	.id_ref_2 = param_type_ids,
737	});
738	return result_id;
739	}
740
741	pub fn constUndef(cg: *CodeGen, ty_id: Id) !Id {
742	const result_id = cg.allocId();
743	try cg.sections.globals.emit(cg.gpa, .OpUndef, .{
744	.id_result_type = ty_id,
745	.id_result = result_id,
746	});
747	return result_id;
748	}
749
750	pub fn constNull(cg: *CodeGen, ty_id: Id) !Id {
751	const result_id = cg.allocId();
752	try cg.sections.globals.emit(cg.gpa, .OpConstantNull, .{
753	.id_result_type = ty_id,
754	.id_result = result_id,
755	});
756	return result_id;
757	}
758
759	pub fn decorate(
760	cg: *CodeGen,
761	target: Id,
762	decoration: spec.Decoration.Extended,
763	) !void {
764	try cg.sections.annotations.emit(cg.gpa, .OpDecorate, .{
765	.target = target,
766	.decoration = decoration,
767	});
768	}
769
770	pub fn decorateMember(
771	cg: *CodeGen,
772	structure_type: Id,
773	member: u32,
774	decoration: spec.Decoration.Extended,
775	) !void {
776	try cg.sections.annotations.emit(cg.gpa, .OpMemberDecorate, .{
777	.structure_type = structure_type,
778	.member = member,
779	.decoration = decoration,
780	});
781	}
782
783	pub fn allocDecl(cg: *CodeGen, kind: Decl.Kind) !Decl.Index {
784	try cg.decls.append(cg.gpa, .{
785	.kind = kind,
786	.result_id = cg.allocId(),
787	});
788
789	return @as(Decl.Index, @fromBackingInt(@intCast(@as(u32, @intCast(cg.decls.items.len - 1)))));
790	}
791
792	pub fn declPtr(cg: *CodeGen, index: Decl.Index) *Decl {
793	return &cg.decls.items[@backingInt(index)];
794	}
795
796	pub fn debugName(cg: *CodeGen, target: Id, name: []const u8) !void {
797	if (cg.zcu.comp.config.root_strip) return;
798	try cg.sections.debug_names.emit(cg.gpa, .OpName, .{
799	.target = target,
800	.name = name,
801	});
802	}
803
804	pub fn debugNameFmt(cg: *CodeGen, target: Id, comptime fmt: []const u8, args: anytype) !void {
805	if (cg.zcu.comp.config.root_strip) return;
806	const name = try std.fmt.allocPrint(cg.gpa, fmt, args);
807	defer cg.gpa.free(name);
808	try cg.debugName(target, name);
809	}
810
811	pub fn memberDebugName(cg: *CodeGen, target: Id, member: u32, name: []const u8) !void {
812	if (cg.zcu.comp.config.root_strip) return;
813	try cg.sections.debug_names.emit(cg.gpa, .OpMemberName, .{
814	.type = target,
815	.member = member,
816	.name = name,
817	});
818	}
819
820	pub fn storageClass(cg: *const CodeGen, as: std.lang.AddressSpace) spec.StorageClass {
821	const target = cg.zcu.getTarget();
822	return switch (as) {
823	.generic => .function,
824	.global => switch (target.os.tag) {
825	.opencl, .amdhsa => .cross_workgroup,
826	else => .storage_buffer,
827	},
828	.push_constant => .push_constant,
829	.output => .output,
830	.uniform => .uniform,
831	.storage_buffer => .storage_buffer,
832	.physical_storage_buffer => .physical_storage_buffer,
833	.constant => .uniform_constant,
834	.shared => .workgroup,
835	.local => .function,
836	.input => .input,
837	.gs,
838	.fs,
839	.ss,
840	.far,
841	.param,
842	.flash,
843	.flash1,
844	.flash2,
845	.flash3,
846	.flash4,
847	.flash5,
848	.cog,
849	.lut,
850	.hub,
851	.externref,
852	.funcref,
853	=> unreachable,
854	};
855	}
856
857	const Error = error{ AlreadyReported, OutOfMemory };
858
859	pub fn genNav(cg: *CodeGen, do_codegen: bool) Error!void {
860	const gpa = cg.gpa;
861	const zcu = cg.zcu;
862	const ip = &zcu.intern_pool;
863	const target = zcu.getTarget();
864
865	const nav = ip.getNav(cg.owner_nav);
866	const val = zcu.navValue(cg.owner_nav);
867	const ty = val.typeOf(zcu);
868
869	if (!do_codegen and !ty.hasRuntimeBits(zcu)) {
870	const child_ty = if (ty.zigTypeTag(zcu) == .pointer) ty.childType(zcu) else ty;
871	if (child_ty.zigTypeTag(zcu) != .spirv) return;
872	}
873
874	const spv_decl_index = try cg.resolveNav(ip, cg.owner_nav);
875	const decl = cg.declPtr(spv_decl_index);
876	const result_id = decl.result_id;
877	decl.begin_dep = cg.decl_deps.items.len;
878
879	switch (decl.kind) {
880	.func => {
881	if (nav.resolved.?.is_extern_decl) {
882	_ = try cg.resolveType(ty, .direct);
883	try emitExternFnStub(cg, nav, decl, ty);
884	decl.end_dep = cg.decl_deps.items.len;
885	return;
886	}
887
888	const fn_info = zcu.typeToFunc(ty).?;
889	const return_ty_id = try cg.resolveFnReturnType(.fromInterned(fn_info.return_type));
890	const is_test = zcu.test_functions.contains(cg.owner_nav);
891
892	const func_result_id = if (is_test) cg.allocId() else result_id;
893	const prototype_ty_id = try cg.resolveType(ty, .direct);
894	try cg.prologue.emit(gpa, .OpFunction, .{
895	.id_result_type = return_ty_id,
896	.id_result = func_result_id,
897	.function_type = prototype_ty_id,
898	// Note: the backend will never be asked to generate an inline function
899	// (this is handled in sema), so we don't need to set function_control here.
900	.function_control = .{},
901	});
902
903	try cg.args.ensureUnusedCapacity(gpa, fn_info.param_types.len);
904	for (fn_info.param_types.get(ip)) |param_ty_index| {
905	const param_ty: Type = .fromInterned(param_ty_index);
906	if (!param_ty.hasRuntimeBits(zcu)) continue;
907
908	const param_type_id = try cg.resolveType(param_ty, .direct);
909	const arg_result_id = cg.allocId();
910	try cg.prologue.emit(gpa, .OpFunctionParameter, .{
911	.id_result_type = param_type_id,
912	.id_result = arg_result_id,
913	});
914	cg.args.appendAssumeCapacity(arg_result_id);
915	}
916
917	// TODO: This could probably be done in a better way...
918	const root_block_id = cg.allocId();
919
920	// The root block of a function declaration should appear before OpVariable instructions,
921	// so it is generated into the function's prologue.
922	try cg.prologue.emit(gpa, .OpLabel, .{
923	.id_result = root_block_id,
924	});
925	cg.block_label = root_block_id;
926
927	const main_body = cg.air.getMainBody();
928	_ = try cg.genStructuredBody(.selection, main_body);
929	// We always expect paths to here to end, but we still need the block
930	// to act as a dummy merge block.
931	try cg.body.emit(gpa, .OpUnreachable, {});
932	try cg.body.emit(gpa, .OpFunctionEnd, {});
933	// Append the actual code into the functions section.
934	try cg.sections.functions.append(gpa, cg.prologue);
935	try cg.sections.functions.append(gpa, cg.body);
936
937	// Temporarily generate a test kernel declaration if this is a test function.
938	if (is_test) {
939	try cg.generateTestEntryPoint(nav.fqn.toSlice(ip), spv_decl_index, func_result_id);
940	}
941
942	try cg.debugName(func_result_id, nav.fqn.toSlice(ip));
943	},
944	.global => {
945	const key = ip.indexToKey(val.toIntern()).@"extern";
946
947	const storage_class = cg.storageClass(nav.resolved.?.@"addrspace");
948	assert(storage_class != .generic); // These should be instance globals
949
950	const as = nav.resolved.?.@"addrspace";
951	const ty_id = try cg.pointeeType(as, ty, true);
952	const ptr_ty_id = try cg.ptrType(ty_id, storage_class);
953
954	try cg.sections.globals.emit(gpa, .OpVariable, .{
955	.id_result_type = ptr_ty_id,
956	.id_result = result_id,
957	.storage_class = storage_class,
958	});
959
960	switch (target.os.tag) {
961	.vulkan, .opengl => {
962	switch (storage_class) {
963	.uniform,
964	.push_constant,
965	.storage_buffer,
966	.physical_storage_buffer,
967	=> {
968	if (ty.hasRuntimeBits(zcu)) {
969	if (!ty.isSpirvRuntimeArray(zcu)) {
970	try cg.decorate(
971	ptr_ty_id,
972	.{ .array_stride = .{ .array_stride = @intCast(ty.abiSize(zcu)) } },
973	);
974	}
975	if (!cg.needsLayout(as, ty)) try cg.decorateLayout(ty, ty_id);
976	}
977	if (key.is_const and storage_class == .storage_buffer) {
978	try cg.decorate(result_id, .non_writable);
979	}
980	},
981	else => {},
982	}
983
984	if (key.decoration) |decoration| switch (decoration) {
985	.location => |location| {
986	if (storage_class != .output and storage_class != .input and storage_class != .uniform_constant) {
987	return cg.fail("storage class must be one of (output, input, uniform_constant) but is {s}", .{@tagName(storage_class)});
988	}
989	try cg.decorate(result_id, .{
990	.location = .{ .location = location },
991	});
992	},
993	.flat => |location| {
994	try cg.decorate(result_id, .{ .location = .{ .location = location } });
995	try cg.decorate(result_id, .flat);
996	},
997	.descriptor => |descriptor| {
998	if (storage_class != .storage_buffer and storage_class != .uniform and storage_class != .uniform_constant) {
999	return cg.fail("storage class must be one of (storage_buffer, uniform, uniform_constant) but is {s}", .{@tagName(storage_class)});
1000	}
1001	try cg.decorate(result_id, .{
1002	.binding = .{ .binding_point = descriptor.binding },
1003	});
1004
1005	try cg.decorate(result_id, .{
1006	.descriptor_set = .{ .descriptor_set = descriptor.set },
1007	});
1008	},
1009	};
1010	},
1011	else => {},
1012	}
1013
1014	if (std.meta.stringToEnum(spec.BuiltIn, nav.fqn.toSlice(ip))) |built_in| {
1015	try cg.decorate(result_id, .{ .built_in = .{ .built_in = built_in } });
1016	}
1017
1018	try cg.debugName(result_id, nav.fqn.toSlice(ip));
1019	},
1020	.invocation_global => {
1021	// `@extern()` produces an invocation_global whose value is a
1022	// comptime-known pointer to an underlying extern symbol's Nav.
1023	// The pointer is inlined at use sites so we don't need a Function-scope wrapper here.
1024	if (ip.indexToKey(val.toIntern()) == .ptr) alias: {
1025	const ptr_key = ip.indexToKey(val.toIntern()).ptr;
1026	if (ptr_key.base_addr != .nav or ptr_key.byte_offset != 0) break :alias;
1027	const underlying_nav = ip.getNav(ptr_key.base_addr.nav);
1028	if (!underlying_nav.resolved.?.is_extern_decl) break :alias;
1029	cg.declPtr(spv_decl_index).end_dep = cg.decl_deps.items.len;
1030	return;
1031	}
1032
1033	const ty_id = try cg.resolveType(ty, .indirect);
1034	const ptr_ty_id = try cg.ptrType(ty_id, .function);
1035
1036	// TODO: Combine with resolveAnonDecl?
1037	const void_ty_id = try cg.resolveType(.void, .direct);
1038	const initializer_proto_ty_id = try cg.functionType(void_ty_id, &.{});
1039
1040	const initializer_id = cg.allocId();
1041	try cg.prologue.emit(gpa, .OpFunction, .{
1042	.id_result_type = try cg.resolveType(.void, .direct),
1043	.id_result = initializer_id,
1044	.function_control = .{},
1045	.function_type = initializer_proto_ty_id,
1046	});
1047
1048	const root_block_id = cg.allocId();
1049	try cg.prologue.emit(gpa, .OpLabel, .{
1050	.id_result = root_block_id,
1051	});
1052	cg.block_label = root_block_id;
1053
1054	const val_id = try cg.constant(ty, val, .indirect);
1055	try cg.body.emit(gpa, .OpStore, .{
1056	.pointer = result_id,
1057	.object = val_id,
1058	});
1059
1060	try cg.body.emit(gpa, .OpReturn, {});
1061	try cg.body.emit(gpa, .OpFunctionEnd, {});
1062	try cg.sections.functions.append(gpa, cg.prologue);
1063	try cg.sections.functions.append(gpa, cg.body);
1064
1065	try cg.debugNameFmt(initializer_id, "initializer of {f}", .{nav.fqn.fmt(ip)});
1066	try cg.debugName(result_id, nav.fqn.toSlice(ip));
1067
1068	try cg.sections.globals.emit(gpa, .OpExtInst, .{
1069	.id_result_type = ptr_ty_id,
1070	.id_result = result_id,
1071	.set = try cg.importInstructionSet(.zig),
1072	.instruction = .{ .inst = @backingInt(spec.Zig.InvocationGlobal) },
1073	.id_ref_4 = &.{initializer_id},
1074	});
1075	},
1076	}
1077
1078	cg.declPtr(spv_decl_index).end_dep = cg.decl_deps.items.len;
1079	}
1080
1081	fn decorateLayout(cg: *CodeGen, ty: Type, ty_id: spec.Id) Error!void {
1082	const zcu = cg.zcu;
1083	const ip = &zcu.intern_pool;
1084	switch (ty.zigTypeTag(zcu)) {
1085	.array => {
1086	const elem_ty = ty.childType(zcu);
1087	if (!elem_ty.hasRuntimeBits(zcu)) return;
1088	try cg.decorate(ty_id, .{
1089	.array_stride = .{ .array_stride = @intCast(elem_ty.abiSize(zcu)) },
1090	});
1091	try cg.decorateLayout(elem_ty, try cg.resolveType(elem_ty, .indirect));
1092	},
1093	.vector => {
1094	const elem_ty = ty.childType(zcu);
1095	try cg.decorateLayout(elem_ty, try cg.resolveType(elem_ty, .indirect));
1096	if (cg.isSpvVector(ty)) return;
1097	try cg.decorate(ty_id, .{
1098	.array_stride = .{ .array_stride = @intCast(elem_ty.abiSize(zcu)) },
1099	});
1100	},
1101	.@"struct" => switch (ip.indexToKey(ty.toIntern())) {
1102	.struct_type => {
1103	const struct_type = ip.loadStructType(ty.toIntern());
1104	if (struct_type.layout == .@"packed") return;
1105	var it = struct_type.iterateRuntimeOrder(ip);
1106	var member: u32 = 0;
1107	while (it.next()) |field_index| {
1108	const field_ty: Type = .fromInterned(struct_type.field_types.get(ip)[field_index]);
1109	if (!field_ty.hasRuntimeBits(zcu)) continue;
1110	const offset: u32 = @intCast(ty.structFieldOffset(field_index, zcu));
1111	try cg.decorateMember(ty_id, member, .{ .offset = .{ .byte_offset = offset } });
1112	try cg.decorateLayout(field_ty, try cg.resolveType(field_ty, .indirect));
1113	member += 1;
1114	}
1115	},
1116	.tuple_type => |tuple| {
1117	for (tuple.types.get(ip), tuple.values.get(ip)) |field_ty, field_val| {
1118	if (field_val != .none) continue;
1119	const ft: Type = .fromInterned(field_ty);
1120	if (ft.hasRuntimeBits(zcu)) try cg.decorateLayout(ft, try cg.resolveType(ft, .indirect));
1121	}
1122	},
1123	else => {},
1124	},
1125	.@"union" => {
1126	const union_obj = zcu.typeToUnion(ty).?;
1127	if (union_obj.layout == .@"packed") return;
1128	const layout = cg.unionLayout(ty);
1129	if (layout.tag_size != 0) {
1130	const tag_ty: Type = .fromInterned(union_obj.enum_tag_type);
1131	try cg.decorateLayout(tag_ty, try cg.resolveType(tag_ty, .indirect));
1132	}
1133	if (layout.has_payload) {
1134	try cg.decorateLayout(layout.payload_ty, try cg.resolveType(layout.payload_ty, .indirect));
1135	}
1136	const u8_id = try cg.resolveType(.u8, .direct);
1137	if (layout.payload_padding_size != 0) {
1138	const len_id = try cg.constInt(.u32, layout.payload_padding_size);
1139	const arr_id = try cg.arrayType(len_id, u8_id);
1140	try cg.decorate(arr_id, .{ .array_stride = .{ .array_stride = 1 } });
1141	}
1142	if (layout.padding_size != 0) {
1143	const len_id = try cg.constInt(.u32, layout.padding_size);
1144	const arr_id = try cg.arrayType(len_id, u8_id);
1145	try cg.decorate(arr_id, .{ .array_stride = .{ .array_stride = 1 } });
1146	}
1147	},
1148	.optional => {
1149	const payload_ty = ty.optionalChild(zcu);
1150	if (payload_ty.hasRuntimeBits(zcu)) try cg.decorateLayout(payload_ty, try cg.resolveType(payload_ty, .indirect));
1151	},
1152	.error_union => {
1153	const payload_ty = ty.errorUnionPayload(zcu);
1154	if (payload_ty.hasRuntimeBits(zcu)) try cg.decorateLayout(payload_ty, try cg.resolveType(payload_ty, .indirect));
1155	},
1156	else => {},
1157	}
1158	}
1159
1160	pub fn fail(cg: *CodeGen, comptime format: []const u8, args: anytype) Error {
1161	@branchHint(.cold);
1162	return cg.zcu.codegenFail(cg.owner_nav, format, args);
1163	}
1164
1165	pub fn todo(cg: *CodeGen, comptime format: []const u8, args: anytype) Error {
1166	return cg.fail("TODO (SPIR-V): " ++ format, args);
1167	}
1168
1169	/// This imports the "default" extended instruction set for the target
1170	/// For OpenCL, OpenCL.std.100. For Vulkan and OpenGL, GLSL.std.450.
1171	fn importExtendedSet(cg: *CodeGen) !Id {
1172	const target = cg.zcu.getTarget();
1173	return switch (target.os.tag) {
1174	.opencl, .amdhsa => try cg.importInstructionSet(.@"OpenCL.std"),
1175	.vulkan, .opengl => try cg.importInstructionSet(.@"GLSL.std.450"),
1176	else => unreachable,
1177	};
1178	}
1179
1180	/// Fetch the result-id for a previously generated instruction or constant.
1181	fn resolve(cg: *CodeGen, inst: Air.Inst.Ref) !Id {
1182	const zcu = cg.zcu;
1183	const ip = &zcu.intern_pool;
1184	if (inst.toInterned()) |val_ip_index| {
1185	const ty = cg.typeOf(inst);
1186	if (ty.zigTypeTag(zcu) == .@"fn") {
1187	const val_key = zcu.intern_pool.indexToKey(val_ip_index);
1188	const fn_nav = switch (val_key) {
1189	.@"extern" => |@"extern"| @"extern".owner_nav,
1190	.func => |func| func.owner_nav,
1191	else => unreachable,
1192	};
1193	const spv_decl_index = try cg.resolveNav(ip, fn_nav);
1194	try cg.decl_deps.append(cg.gpa, spv_decl_index);
1195	const decl = cg.declPtr(spv_decl_index);
1196	if (val_key == .@"extern") {
1197	const nav = ip.getNav(fn_nav);
1198	const nav_ty: Type = .fromInterned(nav.resolved.?.type);
1199	try emitExternFnStub(cg, nav, decl, nav_ty);
1200	}
1201	return decl.result_id;
1202	}
1203
1204	return try cg.constant(ty, .fromInterned(val_ip_index), .direct);
1205	}
1206	const index = inst.toIndex().?;
1207	return cg.inst_results.get(index).?; // Assertion means instruction does not dominate usage.
1208	}
1209
1210	fn resolveUav(cg: *CodeGen, val: InternPool.Index) !Id {
1211	const gpa = cg.gpa;
1212
1213	// TODO: This cannot be a function at this point, but it should probably be handled anyway.
1214
1215	const zcu = cg.zcu;
1216	const ty: Type = .fromInterned(zcu.intern_pool.typeOf(val));
1217	const ty_id = try cg.resolveType(ty, .indirect);
1218
1219	const spv_decl_index = blk: {
1220	const entry = try cg.uav_link.getOrPut(gpa, .{ val, .function });
1221	if (entry.found_existing) {
1222	try cg.addFunctionDep(entry.value_ptr.*, .function);
1223	return cg.declPtr(entry.value_ptr.*).result_id;
1224	}
1225
1226	const spv_decl_index = try cg.allocDecl(.invocation_global);
1227	try cg.addFunctionDep(spv_decl_index, .function);
1228	entry.value_ptr.* = spv_decl_index;
1229	break :blk spv_decl_index;
1230	};
1231
1232	// TODO: At some point we will be able to generate this all constant here, but then all of
1233	// constant() will need to be implemented such that it doesn't generate any at-runtime code.
1234	// NOTE: Because this is a global, we really only want to initialize it once. Therefore the
1235	// constant lowering of this value will need to be deferred to an initializer similar to
1236	// other globals.
1237
1238	const result_id = cg.declPtr(spv_decl_index).result_id;
1239
1240	{
1241	// Save the current state so that we can temporarily generate into a different function.
1242	// TODO: This should probably be made a little more robust.
1243	const func_prologue = cg.prologue;
1244	const func_body = cg.body;
1245	const block_label = cg.block_label;
1246	defer {
1247	cg.prologue = func_prologue;
1248	cg.body = func_body;
1249	cg.block_label = block_label;
1250	}
1251
1252	cg.prologue = .{};
1253	cg.body = .{};
1254	defer {
1255	cg.prologue.deinit(gpa);
1256	cg.body.deinit(gpa);
1257	}
1258
1259	const void_ty_id = try cg.resolveType(.void, .direct);
1260	const initializer_proto_ty_id = try cg.functionType(void_ty_id, &.{});
1261
1262	const initializer_id = cg.allocId();
1263	try cg.prologue.emit(gpa, .OpFunction, .{
1264	.id_result_type = try cg.resolveType(.void, .direct),
1265	.id_result = initializer_id,
1266	.function_control = .{},
1267	.function_type = initializer_proto_ty_id,
1268	});
1269	const root_block_id = cg.allocId();
1270	try cg.prologue.emit(gpa, .OpLabel, .{
1271	.id_result = root_block_id,
1272	});
1273	cg.block_label = root_block_id;
1274
1275	const val_id = try cg.constant(ty, .fromInterned(val), .indirect);
1276	try cg.body.emit(gpa, .OpStore, .{
1277	.pointer = result_id,
1278	.object = val_id,
1279	});
1280
1281	try cg.body.emit(gpa, .OpReturn, {});
1282	try cg.body.emit(gpa, .OpFunctionEnd, {});
1283
1284	try cg.sections.functions.append(gpa, cg.prologue);
1285	try cg.sections.functions.append(gpa, cg.body);
1286
1287	try cg.debugNameFmt(initializer_id, "initializer of __anon_{d}", .{@backingInt(val)});
1288
1289	const fn_decl_ptr_ty_id = try cg.ptrType(ty_id, .function);
1290	try cg.sections.globals.emit(gpa, .OpExtInst, .{
1291	.id_result_type = fn_decl_ptr_ty_id,
1292	.id_result = result_id,
1293	.set = try cg.importInstructionSet(.zig),
1294	.instruction = .{ .inst = @backingInt(spec.Zig.InvocationGlobal) },
1295	.id_ref_4 = &.{initializer_id},
1296	});
1297	}
1298
1299	return result_id;
1300	}
1301
1302	fn resolvePtr(cg: *CodeGen, ref: Air.Inst.Ref) !Ptr {
1303	const id = try cg.resolve(ref);
1304	if (cg.tracked_allocas.getPtr(id)) |slot| return .{ .tracked = .{ .id = id, .slot = slot } };
1305	return .{ .id = id };
1306	}
1307
1308	fn addFunctionDep(cg: *CodeGen, decl_index: Decl.Index, storage_class: StorageClass) !void {
1309	const gpa = cg.gpa;
1310	const target = cg.zcu.getTarget();
1311	if (target.cpu.has(.spirv, .v1_4)) {
1312	try cg.decl_deps.append(gpa, decl_index);
1313	} else {
1314	// Before version 1.4, the interface’s storage classes are limited to the Input and Output
1315	if (storage_class == .input or storage_class == .output) {
1316	try cg.decl_deps.append(gpa, decl_index);
1317	}
1318	}
1319	}
1320
1321	/// Start a new SPIR-V block, Emits the label of the new block, and stores which
1322	/// block we are currently generating.
1323	/// Note that there is no such thing as nested blocks like in ZIR or AIR, so we don't need to
1324	/// keep track of the previous block.
1325	fn beginSpvBlock(cg: *CodeGen, label: Id) !void {
1326	try cg.body.emit(cg.gpa, .OpLabel, .{ .id_result = label });
1327	cg.block_label = label;
1328	cg.block_terminated = false;
1329	}
1330
1331	const ArithmeticTypeInfo = struct {
1332	const Class = enum {
1333	bool,
1334	/// A regular, **native**, integer.
1335	/// This is only returned when the backend supports this int as a native type (when
1336	/// the relevant capability is enabled).
1337	integer,
1338	/// A regular float. These are all required to be natively supported. Floating points
1339	/// for which the relevant capability is not enabled are not emulated.
1340	float,
1341	/// An integer of a 'strange' size (which' bit size is not the same as its backing
1342	/// type. **Note**: this may **also** include power-of-2 integers for which the
1343	/// relevant capability is not enabled), but still within the limits of the largest
1344	/// natively supported integer type.
1345	strange_integer,
1346	/// An integer with more bits than the largest natively supported integer type.
1347	composite_integer,
1348	};
1349
1350	/// A classification of the inner type.
1351	/// These scenarios will all have to be handled slightly different.
1352	class: Class,
1353	/// The number of bits in the inner type.
1354	/// This is the actual number of bits of the type, not the size of the backing integer.
1355	bits: u16,
1356	/// The number of bits required to store the type.
1357	/// For `integer` and `float`, this is equal to `bits`.
1358	/// For `strange_integer` and `bool` this is the size of the backing integer.
1359	/// For `composite_integer` this is the elements count.
1360	backing_bits: u16,
1361	/// Null if this type is a scalar, or the length of the vector otherwise.
1362	vector_len: ?u32,
1363	/// Whether the inner type is signed. Only relevant for integers.
1364	signedness: std.lang.Signedness,
1365	};
1366
1367	fn arithmeticTypeInfo(cg: *CodeGen, ty: Type) ArithmeticTypeInfo {
1368	const zcu = cg.zcu;
1369	const target = cg.zcu.getTarget();
1370	var scalar_ty = ty.scalarType(zcu);
1371	if (scalar_ty.zigTypeTag(zcu) == .@"enum") {
1372	scalar_ty = scalar_ty.backingIntType(zcu);
1373	}
1374	const vector_len = if (ty.isVector(zcu)) ty.vectorLen(zcu) else null;
1375	return switch (scalar_ty.zigTypeTag(zcu)) {
1376	.bool => .{
1377	.bits = 1, // Doesn't matter for this class.
1378	.backing_bits = cg.backingIntBits(1).@"0",
1379	.vector_len = vector_len,
1380	.signedness = .unsigned, // Technically, but doesn't matter for this class.
1381	.class = .bool,
1382	},
1383	.float => .{
1384	.bits = scalar_ty.floatBits(target),
1385	.backing_bits = scalar_ty.floatBits(target), // TODO: F80?
1386	.vector_len = vector_len,
1387	.signedness = .signed, // Technically, but doesn't matter for this class.
1388	.class = .float,
1389	},
1390	.int => blk: {
1391	const int_info = scalar_ty.intInfo(zcu);
1392	// TODO: Maybe it's useful to also return this value.
1393	const backing_bits, const big_int = cg.backingIntBits(int_info.bits);
1394	break :blk .{
1395	.bits = int_info.bits,
1396	.backing_bits = backing_bits,
1397	.vector_len = vector_len,
1398	.signedness = int_info.signedness,
1399	.class = class: {
1400	if (big_int) break :class .composite_integer;
1401	break :class if (backing_bits == int_info.bits) .integer else .strange_integer;
1402	},
1403	};
1404	},
1405	.@"enum" => unreachable,
1406	.vector => unreachable,
1407	else => unreachable, // Unhandled arithmetic type
1408	};
1409	}
1410
1411	/// Checks whether the type can be directly translated to SPIR-V vectors
1412	fn isSpvVector(cg: *CodeGen, ty: Type) bool {
1413	const zcu = cg.zcu;
1414	const target = cg.zcu.getTarget();
1415	if (ty.zigTypeTag(zcu) != .vector) return false;
1416
1417	// TODO: This check must be expanded for types that can be represented
1418	// as integers (enums / packed structs?) and types that are represented
1419	// by multiple SPIR-V values.
1420	const scalar_ty = ty.scalarType(zcu);
1421	switch (scalar_ty.zigTypeTag(zcu)) {
1422	.bool,
1423	.int,
1424	.float,
1425	=> {},
1426	else => return false,
1427	}
1428
1429	const elem_ty = ty.childType(zcu);
1430	const len = ty.vectorLen(zcu);
1431
1432	if (elem_ty.isNumeric(zcu) or elem_ty.toIntern() == .bool_type) {
1433	if (len > 1 and len <= 4) return true;
1434	if (target.cpu.has(.spirv, .vector16)) return (len == 8 or len == 16);
1435	}
1436
1437	return false;
1438	}
1439
1440	/// Emits a bool constant in a particular representation.
1441	fn constBool(cg: *CodeGen, value: bool, repr: Repr) !Id {
1442	switch (repr) {
1443	.indirect => return cg.constInt(.u1, @intFromBool(value)),
1444	.direct => {
1445	const result_ty_id = try cg.boolType();
1446	const result_id = cg.allocId();
1447	switch (value) {
1448	inline else => |value_ct| try cg.sections.globals.emit(
1449	cg.gpa,
1450	if (value_ct) .OpConstantTrue else .OpConstantFalse,
1451	.{ .id_result_type = result_ty_id, .id_result = result_id },
1452	),
1453	}
1454	return result_id;
1455	},
1456	}
1457	}
1458
1459	/// Emits an integer constant.
1460	/// This function, unlike cg.constInt, takes care to bitcast
1461	/// the value to an unsigned int first for Kernels.
1462	fn constInt(cg: *CodeGen, ty: Type, value: anytype) !Id {
1463	const gpa = cg.gpa;
1464	const zcu = cg.zcu;
1465	const target = cg.zcu.getTarget();
1466	const scalar_ty = ty.scalarType(zcu);
1467	const int_info = scalar_ty.intInfo(zcu);
1468	// Use backing bits so that negatives are sign extended
1469	const backing_bits, const big_int = cg.backingIntBits(int_info.bits);
1470	assert(backing_bits != 0); // u0 is comptime
1471
1472	const result_ty_id = try cg.resolveType(scalar_ty, .indirect);
1473	const signedness: Signedness = switch (@typeInfo(@TypeOf(value))) {
1474	.int => |int| int.signedness,
1475	.comptime_int => if (value < 0) .signed else .unsigned,
1476	else => unreachable,
1477	};
1478	if (@TypeOf(value) != comptime_int and @sizeOf(@TypeOf(value)) >= 4 and big_int) {
1479	const value64: u64 = switch (signedness) {
1480	.signed => @bitCast(@as(i64, @intCast(value))),
1481	.unsigned => @as(u64, @intCast(value)),
1482	};
1483	const n_limbs = backing_bits / cg.bigIntBits();
1484	const fill: u32 = if (signedness == .signed and value < 0) 0xFFFFFFFF else 0;
1485	const scratch_top = cg.id_scratch.items.len;
1486	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
1487	const constituents = try cg.id_scratch.addManyAsSlice(gpa, n_limbs);
1488	for (constituents, 0..) |*c, i| {
1489	c.* = try cg.constInt(
1490	.u32,
1491	if (i < 2) @as(u32, @truncate(value64 >> @intCast(i * 32))) else fill,
1492	);
1493	}
1494	return cg.constructComposite(result_ty_id, constituents);
1495	}
1496
1497	const final_value: spec.LiteralContextDependentNumber = switch (target.os.tag) {
1498	.opencl, .amdhsa => blk: {
1499	const value64: u64 = switch (signedness) {
1500	.signed => @bitCast(@as(i64, @intCast(value))),
1501	.unsigned => @as(u64, @intCast(value)),
1502	};
1503
1504	// Manually truncate the value to the right amount of bits.
1505	const truncated_value = if (backing_bits == 64)
1506	value64
1507	else
1508	value64 & (@as(u64, 1) << @intCast(backing_bits)) - 1;
1509
1510	break :blk switch (backing_bits) {
1511	1...32 => .{ .uint32 = @truncate(truncated_value) },
1512	33...64 => .{ .uint64 = truncated_value },
1513	else => unreachable,
1514	};
1515	},
1516	else => switch (backing_bits) {
1517	1...32 => if (signedness == .signed) .{ .int32 = @intCast(value) } else .{ .uint32 = @intCast(value) },
1518	33...64 => if (signedness == .signed) .{ .int64 = value } else .{ .uint64 = value },
1519	else => unreachable,
1520	},
1521	};
1522
1523	const result_id = cg.allocId();
1524	try cg.sections.globals.emit(cg.gpa, .OpConstant, .{
1525	.id_result_type = result_ty_id,
1526	.id_result = result_id,
1527	.value = final_value,
1528	});
1529
1530	if (!ty.isVector(zcu)) return result_id;
1531	return cg.constructCompositeSplat(ty, result_id);
1532	}
1533
1534	/// Construct a composite value from its constituents.
1535	/// In logical addressing mode (Vulkan/OpenGL), OpCompositeConstruct cannot accept
1536	/// pointer operands, so for struct types we use alloc, store for each field and load instead.
1537	pub fn constructComposite(cg: *CodeGen, result_ty_id: Id, constituents: []const Id) !Id {
1538	const gpa = cg.gpa;
1539
1540	const maybe_fields: ?[]const Id = for (cg.struct_types.keys(), cg.struct_types.values()) |key, val| {
1541	if (val == result_ty_id) break key.fields;
1542	} else null;
1543	if (maybe_fields) |fields| {
1544	assert(fields.len == constituents.len);
1545	const u32_ty_id = try cg.intType(.unsigned, 32);
1546	const var_id = try cg.alloc(result_ty_id, null);
1547	for (fields, constituents, 0..) |field_ty_id, constituent, i| {
1548	const field_ptr_ty_id = try cg.ptrType(field_ty_id, .function);
1549	const index_id = cg.allocId();
1550	try cg.sections.globals.emit(gpa, .OpConstant, .{
1551	.id_result_type = u32_ty_id,
1552	.id_result = index_id,
1553	.value = .{ .uint32 = @intCast(i) },
1554	});
1555	const field_ptr = try cg.accessChainId(field_ptr_ty_id, var_id, &.{index_id});
1556	try cg.body.emit(gpa, .OpStore, .{
1557	.pointer = field_ptr,
1558	.object = constituent,
1559	});
1560	}
1561	const result_id = cg.allocId();
1562	try cg.body.emit(gpa, .OpLoad, .{
1563	.id_result_type = result_ty_id,
1564	.id_result = result_id,
1565	.pointer = var_id,
1566	});
1567	return result_id;
1568	}
1569
1570	const result_id = cg.allocId();
1571	try cg.body.emit(gpa, .OpCompositeConstruct, .{
1572	.id_result_type = result_ty_id,
1573	.id_result = result_id,
1574	.constituents = constituents,
1575	});
1576	return result_id;
1577	}
1578
1579	/// Construct a composite at runtime with all lanes set to the same value.
1580	/// ty must be an aggregate type.
1581	fn constructCompositeSplat(cg: *CodeGen, ty: Type, constituent: Id) !Id {
1582	const gpa = cg.gpa;
1583	const zcu = cg.zcu;
1584	const n: usize = @intCast(ty.arrayLen(zcu));
1585
1586	const scratch_top = cg.id_scratch.items.len;
1587	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
1588
1589	const constituents = try cg.id_scratch.addManyAsSlice(gpa, n);
1590	@memset(constituents, constituent);
1591
1592	const result_ty_id = try cg.resolveType(ty, .direct);
1593	return cg.constructComposite(result_ty_id, constituents);
1594	}
1595
1596	/// This function generates a load for a constant in direct (ie, non-memory) representation.
1597	/// When the constant is simple, it can be generated directly using OpConstant instructions.
1598	/// When the constant is more complicated however, it needs to be constructed using multiple values. This
1599	/// is done by emitting a sequence of instructions that initialize the value.
1600	//
1601	/// This function should only be called during function code generation.
1602	fn constant(cg: *CodeGen, ty: Type, val: Value, repr: Repr) Error!Id {
1603	const gpa = cg.gpa;
1604
1605	const pt = cg.pt;
1606	const zcu = cg.zcu;
1607	const target = cg.zcu.getTarget();
1608	const result_ty_id = try cg.resolveType(ty, repr);
1609	const ip = &zcu.intern_pool;
1610
1611	log.debug("lowering constant: ty = {f}, val = {f}, key = {s}", .{ ty.fmt(pt), val.fmtValue(pt), @tagName(ip.indexToKey(val.toIntern())) });
1612	if (val.isUndef(zcu)) {
1613	return cg.constUndef(result_ty_id);
1614	}
1615
1616	const cacheable_id = cache: {
1617	switch (ip.indexToKey(val.toIntern())) {
1618	.int_type,
1619	.ptr_type,
1620	.array_type,
1621	.vector_type,
1622	.opt_type,
1623	.anyframe_type,
1624	.error_union_type,
1625	.simple_type,
1626	.struct_type,
1627	.tuple_type,
1628	.union_type,
1629	.opaque_type,
1630	.spirv_type,
1631	.enum_type,
1632	.func_type,
1633	.error_set_type,
1634	.inferred_error_set_type,
1635	=> unreachable, // types, not values
1636
1637	.undef => unreachable, // handled above
1638
1639	.@"extern",
1640	.func,
1641	.enum_literal,
1642	=> unreachable, // non-runtime values
1643
1644	.simple_value => |simple_value| switch (simple_value) {
1645	.void,
1646	.null,
1647	.@"unreachable",
1648	=> unreachable, // non-runtime values
1649
1650	.false, .true => break :cache try cg.constBool(val.toBool(), repr),
1651	},
1652	.int => {
1653	const int_info = ty.intInfo(zcu);
1654	const backing_bits, const is_big_int = cg.backingIntBits(int_info.bits);
1655	if (is_big_int) {
1656	const limb_bits = cg.bigIntBits();
1657	const n_limbs = backing_bits / limb_bits;
1658	const big_result_ty_id = try cg.resolveType(ty, .indirect);
1659	var bigint_space: Value.BigIntSpace = undefined;
1660	const bigint = val.toBigInt(&bigint_space, zcu);
1661	const limb_bytes = try gpa.alloc(u8, backing_bits / 8);
1662	defer gpa.free(limb_bytes);
1663	bigint.writeTwosComplement(limb_bytes, .little);
1664	const scratch_top = cg.id_scratch.items.len;
1665	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
1666	const constituents = try cg.id_scratch.addManyAsSlice(gpa, n_limbs);
1667	switch (limb_bits) {
1668	32 => {
1669	const limbs_u32: []u32 = @ptrCast(@alignCast(limb_bytes));
1670	for (constituents, limbs_u32) |*c, v| {
1671	const host_v = if (builtin.cpu.arch.endian() == .big) @byteSwap(v) else v;
1672	c.* = try cg.constInt(.u32, host_v);
1673	}
1674	},
1675	64 => {
1676	const limbs_u64: []u64 = @ptrCast(@alignCast(limb_bytes));
1677	for (constituents, limbs_u64) |*c, v| {
1678	const host_v = if (builtin.cpu.arch.endian() == .big) @byteSwap(v) else v;
1679	c.* = try cg.constInt(.u64, host_v);
1680	}
1681	},
1682	else => unreachable,
1683	}
1684	break :cache try cg.constructComposite(big_result_ty_id, constituents);
1685	}
1686	if (ty.isSignedInt(zcu)) {
1687	break :cache try cg.constInt(ty, val.toSignedInt(zcu));
1688	} else {
1689	break :cache try cg.constInt(ty, val.toUnsignedInt(zcu));
1690	}
1691	},
1692	.float => {
1693	const lit: spec.LiteralContextDependentNumber = switch (ty.floatBits(target)) {
1694	16 => .{ .uint32 = @as(u16, @bitCast(val.toFloat(f16, zcu))) },
1695	32 => .{ .float32 = val.toFloat(f32, zcu) },
1696	64 => .{ .float64 = val.toFloat(f64, zcu) },
1697	80, 128 => unreachable, // TODO
1698	else => unreachable,
1699	};
1700	const lit_id = cg.allocId();
1701	try cg.sections.globals.emit(gpa, .OpConstant, .{
1702	.id_result_type = result_ty_id,
1703	.id_result = lit_id,
1704	.value = lit,
1705	});
1706	break :cache lit_id;
1707	},
1708	.err => |err| {
1709	const value = try pt.getErrorValue(err.name);
1710	break :cache try cg.constInt(ty, value);
1711	},
1712	.error_union => |error_union| {
1713	// TODO: Error unions may be constructed with constant instructions if the payload type
1714	// allows it. For now, just generate it here regardless.
1715	const err_ty = ty.errorUnionSet(zcu);
1716	const payload_ty = ty.errorUnionPayload(zcu);
1717	const err_val_id = switch (error_union.val) {
1718	.err_name => |err_name| try cg.constInt(
1719	err_ty,
1720	try pt.getErrorValue(err_name),
1721	),
1722	.payload => try cg.constInt(err_ty, 0),
1723	};
1724	const eu_layout = cg.errorUnionLayout(payload_ty);
1725	if (!eu_layout.payload_has_bits) {
1726	// We use the error type directly as the type.
1727	break :cache err_val_id;
1728	}
1729
1730	const payload_val_id = switch (error_union.val) {
1731	.err_name => try cg.constant(payload_ty, .undef, .indirect),
1732	.payload => |p| try cg.constant(payload_ty, .fromInterned(p), .indirect),
1733	};
1734
1735	var constituents: [2]Id = undefined;
1736	var types: [2]Type = undefined;
1737	if (eu_layout.error_first) {
1738	constituents[0] = err_val_id;
1739	constituents[1] = payload_val_id;
1740	types = .{ err_ty, payload_ty };
1741	} else {
1742	constituents[0] = payload_val_id;
1743	constituents[1] = err_val_id;
1744	types = .{ payload_ty, err_ty };
1745	}
1746
1747	const comp_ty_id = try cg.resolveType(ty, .direct);
1748	return try cg.constructComposite(comp_ty_id, &constituents);
1749	},
1750	.enum_tag => {
1751	const int_val = val.backingInt(zcu);
1752	const int_ty = ty.backingIntType(zcu);
1753	break :cache try cg.constant(int_ty, int_val, repr);
1754	},
1755	.ptr => return cg.constantPtr(val),
1756	.slice => |slice| {
1757	const ptr_id = try cg.constantPtr(.fromInterned(slice.ptr));
1758	const len_id = try cg.constant(.usize, .fromInterned(slice.len), .indirect);
1759	const comp_ty_id = try cg.resolveType(ty, .direct);
1760	return try cg.constructComposite(comp_ty_id, &.{ ptr_id, len_id });
1761	},
1762	.opt => {
1763	const payload_ty = ty.optionalChild(zcu);
1764	const maybe_payload_val = val.optionalValue(zcu);
1765
1766	if (!payload_ty.hasRuntimeBits(zcu)) {
1767	break :cache try cg.constBool(maybe_payload_val != null, .indirect);
1768	} else if (ty.optionalReprIsPayload(zcu)) {
1769	// Optional representation is a nullable pointer or slice.
1770	if (maybe_payload_val) |payload_val| {
1771	return try cg.constant(payload_ty, payload_val, .indirect);
1772	} else {
1773	break :cache try cg.constNull(result_ty_id);
1774	}
1775	}
1776
1777	// Optional representation is a structure.
1778	// { Payload, Bool }
1779
1780	const has_pl_id = try cg.constBool(maybe_payload_val != null, .indirect);
1781	const payload_id = if (maybe_payload_val) |payload_val|
1782	try cg.constant(payload_ty, payload_val, .indirect)
1783	else
1784	try cg.constUndef(try cg.resolveType(payload_ty, .indirect));
1785
1786	const comp_ty_id = try cg.resolveType(ty, .direct);
1787	return try cg.constructComposite(comp_ty_id, &.{ payload_id, has_pl_id });
1788	},
1789	.aggregate => |aggregate| switch (ip.indexToKey(ty.ip_index)) {
1790	inline .array_type, .vector_type => |array_type, tag| {
1791	const elem_ty: Type = .fromInterned(array_type.child);
1792
1793	const scratch_top = cg.id_scratch.items.len;
1794	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
1795	const constituents = try cg.id_scratch.addManyAsSlice(gpa, @intCast(ty.arrayLenIncludingSentinel(zcu)));
1796
1797	const child_repr: Repr = switch (tag) {
1798	.array_type => .indirect,
1799	.vector_type => .direct,
1800	else => unreachable,
1801	};
1802
1803	switch (aggregate.storage) {
1804	.bytes => |bytes| {
1805	// TODO: This is really space inefficient, perhaps there is a better
1806	// way to do it?
1807	for (constituents, bytes.toSlice(constituents.len, ip)) |*constituent, byte| {
1808	constituent.* = try cg.constInt(elem_ty, byte);
1809	}
1810	},
1811	.elems => |elems| {
1812	for (constituents, elems) |*constituent, elem| {
1813	constituent.* = try cg.constant(elem_ty, .fromInterned(elem), child_repr);
1814	}
1815	},
1816	.repeated_elem => |elem| {
1817	@memset(constituents, try cg.constant(elem_ty, .fromInterned(elem), child_repr));
1818	},
1819	}
1820
1821	const comp_ty_id = try cg.resolveType(ty, .direct);
1822	return cg.constructComposite(comp_ty_id, constituents);
1823	},
1824	.struct_type => {
1825	const struct_type = zcu.typeToStruct(ty).?;
1826	assert(struct_type.layout != .@"packed"); // packed structs use `bitpack`
1827
1828	var types: std.ArrayList(Type) = .empty;
1829	defer types.deinit(gpa);
1830
1831	var constituents: std.ArrayList(Id) = .empty;
1832	defer constituents.deinit(gpa);
1833
1834	var it = struct_type.iterateRuntimeOrder(ip);
1835	while (it.next()) |field_index| {
1836	const field_ty: Type = .fromInterned(struct_type.field_types.get(ip)[field_index]);
1837	if (!field_ty.hasRuntimeBits(zcu)) {
1838	// This is a zero-bit field - we only needed it for the alignment.
1839	continue;
1840	}
1841
1842	// TODO: Padding?
1843	const field_val = try val.fieldValue(pt, field_index);
1844	const field_id = try cg.constant(field_ty, field_val, .indirect);
1845
1846	try types.append(gpa, field_ty);
1847	try constituents.append(gpa, field_id);
1848	}
1849
1850	const comp_ty_id = try cg.resolveType(ty, .direct);
1851	return try cg.constructComposite(comp_ty_id, constituents.items);
1852	},
1853	.tuple_type => |tuple| {
1854	var constituents: std.ArrayList(Id) = .empty;
1855	defer constituents.deinit(gpa);
1856
1857	for (tuple.types.get(ip), tuple.values.get(ip), 0..) |field_ty, field_val, i| {
1858	if (field_val != .none) continue;
1859	const ft: Type = .fromInterned(field_ty);
1860	if (!ft.hasRuntimeBits(zcu)) continue;
1861
1862	const fv = try val.fieldValue(pt, i);
1863	const field_id = try cg.constant(ft, fv, .indirect);
1864	try constituents.append(gpa, field_id);
1865	}
1866
1867	const comp_ty_id = try cg.resolveType(ty, .direct);
1868	return try cg.constructComposite(comp_ty_id, constituents.items);
1869	},
1870	else => unreachable,
1871	},
1872	.un => |un| {
1873	assert(ty.containerLayout(zcu) != .@"packed"); // packed unions use `bitpack`
1874	if (un.tag == .none) {
1875	@panic("TODO");
1876	}
1877	const active_field = ty.unionTagFieldIndex(.fromInterned(un.tag), zcu).?;
1878	const union_obj = zcu.typeToUnion(ty).?;
1879	const field_ty: Type = .fromInterned(union_obj.field_types.get(ip)[active_field]);
1880	const payload = if (field_ty.hasRuntimeBits(zcu))
1881	try cg.constant(field_ty, .fromInterned(un.val), .direct)
1882	else
1883	null;
1884	return try cg.unionInit(ty, active_field, payload);
1885	},
1886	.bitpack => |bitpack| {
1887	const int_val: Value = .fromInterned(bitpack.backing_int_val);
1888	break :cache try cg.constant(int_val.typeOf(zcu), int_val, repr);
1889	},
1890
1891	.memoized_call => unreachable,
1892	}
1893	};
1894	return cacheable_id;
1895	}
1896
1897	fn constantPtr(cg: *CodeGen, ptr_val: Value) !Id {
1898	const pt = cg.pt;
1899	const zcu = cg.zcu;
1900	const gpa = cg.gpa;
1901
1902	if (ptr_val.isUndef(zcu)) {
1903	const result_ty = ptr_val.typeOf(zcu);
1904	const result_ty_id = try cg.resolveType(result_ty, .direct);
1905	return cg.constUndef(result_ty_id);
1906	}
1907
1908	var arena = std.heap.ArenaAllocator.init(gpa);
1909	defer arena.deinit();
1910
1911	const derivation = try ptr_val.pointerDerivation(arena.allocator(), pt, null);
1912	return cg.derivePtr(derivation);
1913	}
1914
1915	fn derivePtr(cg: *CodeGen, derivation: Value.PointerDeriveStep) !Id {
1916	const gpa = cg.gpa;
1917	const pt = cg.pt;
1918	const zcu = cg.zcu;
1919	const target = zcu.getTarget();
1920	switch (derivation) {
1921	.comptime_alloc_ptr, .comptime_field_ptr => unreachable,
1922	.int => |int| {
1923	if (target.os.tag != .opencl) {
1924	if (int.ptr_ty.ptrAddressSpace(zcu) != .physical_storage_buffer) {
1925	return cg.fail(
1926	"cannot cast integer to pointer with address space '{s}'",
1927	.{@tagName(int.ptr_ty.ptrAddressSpace(zcu))},
1928	);
1929	}
1930	}
1931	const result_ty_id = try cg.resolveType(int.ptr_ty, .direct);
1932	// TODO: This can probably be an OpSpecConstantOp Bitcast, but
1933	// that is not implemented by Mesa yet. Therefore, just generate it
1934	// as a runtime operation.
1935	const result_ptr_id = cg.allocId();
1936	const value_id = try cg.constInt(.usize, int.addr);
1937	try cg.body.emit(gpa, .OpConvertUToPtr, .{
1938	.id_result_type = result_ty_id,
1939	.id_result = result_ptr_id,
1940	.integer_value = value_id,
1941	});
1942	return result_ptr_id;
1943	},
1944	.nav_ptr => |nav_index| {
1945	const ip = &zcu.intern_pool;
1946	const result_ptr_ty = try pt.navPtrType(nav_index);
1947	const ty_id = try cg.resolveType(result_ptr_ty, .direct);
1948	const nav = ip.getNav(nav_index);
1949	const nav_ty: Type = .fromInterned(nav.resolved.?.type);
1950
1951	switch (nav.resolved.?.value) {
1952	.none => {},
1953	else => |value| switch (ip.indexToKey(value)) {
1954	// TODO: Properly lower function pointers; for now substitute undef.
1955	.func => return try cg.constUndef(ty_id),
1956	.@"extern" => if (ip.isFunctionType(nav_ty.toIntern())) {
1957	const spv_decl_index = try cg.resolveNav(ip, nav_index);
1958	const decl = cg.declPtr(spv_decl_index);
1959	try emitExternFnStub(cg, nav, decl, nav_ty);
1960	return decl.result_id;
1961	},
1962	else => {},
1963	},
1964	}
1965
1966	if (!nav_ty.hasRuntimeBits(zcu) and nav_ty.zigTypeTag(zcu) != .spirv) {
1967	return cg.constUndef(ty_id);
1968	}
1969
1970	const spv_decl_index = try cg.resolveNav(ip, nav_index);
1971	const spv_decl = cg.declPtr(spv_decl_index);
1972	assert(spv_decl.kind != .func);
1973	const storage_class = cg.storageClass(nav.resolved.?.@"addrspace");
1974	try cg.addFunctionDep(spv_decl_index, storage_class);
1975
1976	const nav_ty_id = try cg.resolveType(nav_ty, .indirect);
1977	const decl_ptr_ty_id = try cg.ptrType(nav_ty_id, storage_class);
1978	if (cg.needsLayout(nav.resolved.?.@"addrspace", nav_ty)) {
1979	try cg.block_var_ids.put(gpa, spv_decl.result_id, {});
1980	}
1981	if (decl_ptr_ty_id == ty_id) return spv_decl.result_id;
1982	switch (target.os.tag) {
1983	.vulkan, .opengl => return spv_decl.result_id,
1984	else => {},
1985	}
1986	const casted_ptr_id = cg.allocId();
1987	try cg.body.emit(gpa, .OpBitcast, .{
1988	.id_result_type = ty_id,
1989	.id_result = casted_ptr_id,
1990	.operand = spv_decl.result_id,
1991	});
1992	return casted_ptr_id;
1993	},
1994	.uav_ptr => |uav| {
1995	const ip = &zcu.intern_pool;
1996	const result_ptr_ty: Type = .fromInterned(uav.orig_ty);
1997	const ty_id = try cg.resolveType(result_ptr_ty, .direct);
1998	const uav_ty: Type = .fromInterned(ip.typeOf(uav.val));
1999
2000	switch (ip.indexToKey(uav.val)) {
2001	.func => unreachable, // TODO
2002	.@"extern" => assert(!ip.isFunctionType(uav_ty.toIntern())),
2003	else => {},
2004	}
2005
2006	if (!uav_ty.hasRuntimeBits(zcu) and uav_ty.zigTypeTag(zcu) != .spirv) {
2007	return cg.constUndef(ty_id);
2008	}
2009
2010	// Uav refs are always generic.
2011	assert(result_ptr_ty.ptrAddressSpace(zcu) == .generic);
2012	const uav_ty_id = try cg.resolveType(uav_ty, .indirect);
2013	const decl_ptr_ty_id = try cg.ptrType(uav_ty_id, .function);
2014	const ptr_id = try cg.resolveUav(uav.val);
2015
2016	if (decl_ptr_ty_id == ty_id) return ptr_id;
2017	switch (target.os.tag) {
2018	.vulkan, .opengl => return ptr_id,
2019	else => {},
2020	}
2021	const casted_ptr_id = cg.allocId();
2022	try cg.body.emit(gpa, .OpBitcast, .{
2023	.id_result_type = ty_id,
2024	.id_result = casted_ptr_id,
2025	.operand = ptr_id,
2026	});
2027	return casted_ptr_id;
2028	},
2029	.eu_payload_ptr => @panic("TODO"),
2030	.opt_payload_ptr => @panic("TODO"),
2031	.field_ptr => |field| {
2032	const parent_ptr_id = try cg.derivePtr(field.parent.*);
2033	const parent_ptr_ty = try field.parent.ptrType(pt);
2034	return cg.structFieldPtr(field.result_ptr_ty, parent_ptr_ty, parent_ptr_id, field.field_idx);
2035	},
2036	.elem_ptr => |elem| {
2037	const parent_ptr_id = try cg.derivePtr(elem.parent.*);
2038	const parent_ptr_ty = try elem.parent.ptrType(pt);
2039	const index_id = try cg.constInt(.usize, elem.elem_idx);
2040	return cg.ptrElemPtr(parent_ptr_ty, parent_ptr_id, index_id);
2041	},
2042	.offset_and_cast => |oac| {
2043	const parent_ptr_id = try cg.derivePtr(oac.parent.*);
2044	const parent_ptr_ty = try oac.parent.ptrType(pt);
2045
2046	if (oac.new_ptr_ty.ptrInfo(zcu).flags.vector_index != .none) {
2047	return parent_ptr_id;
2048	}
2049
2050	if (oac.byte_offset == 0) {
2051	var depth: u32 = 0;
2052	var cur = parent_ptr_ty.childType(zcu);
2053	const dst_child = oac.new_ptr_ty.childType(zcu);
2054	while (cur.toIntern() != dst_child.toIntern()) {
2055	switch (cur.zigTypeTag(zcu)) {
2056	.array => {
2057	if (dst_child.zigTypeTag(zcu) == .array and
2058	dst_child.childType(zcu).toIntern() == cur.childType(zcu).toIntern() and
2059	dst_child.arrayLenIncludingSentinel(zcu) <= cur.arrayLenIncludingSentinel(zcu))
2060	{
2061	cur = dst_child;
2062	break;
2063	}
2064	cur = cur.childType(zcu);
2065	depth += 1;
2066	},
2067	.@"struct" => {
2068	if (cur.structFieldCount(zcu) == 0) break;
2069	if (cur.structFieldOffset(0, zcu) != 0) break;
2070	cur = cur.fieldType(0, zcu);
2071	depth += 1;
2072	},
2073	else => break,
2074	}
2075	}
2076	if (cur.toIntern() == dst_child.toIntern()) {
2077	if (depth != 0) {
2078	const as = oac.new_ptr_ty.ptrAddressSpace(zcu);
2079	const child_ty_id = try cg.pointeeType(as, dst_child, false);
2080	const result_ty_id = try cg.ptrType(child_ty_id, cg.storageClass(as));
2081	const scratch_top = cg.id_scratch.items.len;
2082	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
2083	const zero = try cg.constInt(.u32, 0);
2084	const ids = try cg.id_scratch.addManyAsSlice(gpa, depth);
2085	@memset(ids, zero);
2086	return cg.accessChainId(result_ty_id, parent_ptr_id, ids);
2087	} else {
2088	return parent_ptr_id;
2089	}
2090	}
2091	const result_ty_id = try cg.resolveType(oac.new_ptr_ty, .direct);
2092	if (target.os.tag == .opencl) {
2093	const result_ptr_id = cg.allocId();
2094	try cg.body.emit(gpa, .OpBitcast, .{
2095	.id_result_type = result_ty_id,
2096	.id_result = result_ptr_id,
2097	.operand = parent_ptr_id,
2098	});
2099	return result_ptr_id;
2100	}
2101	}
2102
2103	return cg.fail("cannot cast pointer '{f}' to '{f}'", .{
2104	parent_ptr_ty.fmt(pt),
2105	oac.new_ptr_ty.fmt(pt),
2106	});
2107	},
2108	}
2109	}
2110
2111	/// Emit a stub OpFunction/OpFunctionEnd + Import linkage decoration for an
2112	/// extern function so the module is structurally valid. The stub will be
2113	/// replaced by the real definition at link time.
2114	fn emitExternFnStub(cg: *CodeGen, nav: InternPool.Nav, decl: *Decl, fn_ty: Type) !void {
2115	if (decl.has_extern_stub) return;
2116	decl.has_extern_stub = true;
2117
2118	const gpa = cg.gpa;
2119	const zcu = cg.zcu;
2120	const ip = &zcu.intern_pool;
2121	const fn_info = zcu.typeToFunc(fn_ty).?;
2122	const return_ty_id = try cg.resolveFnReturnType(.fromInterned(fn_info.return_type));
2123	const prototype_ty_id = try cg.resolveType(fn_ty, .direct);
2124
2125	var stub: Section = .{};
2126	defer stub.deinit(gpa);
2127	try stub.emit(gpa, .OpFunction, .{
2128	.id_result_type = return_ty_id,
2129	.id_result = decl.result_id,
2130	.function_type = prototype_ty_id,
2131	.function_control = .{},
2132	});
2133	for (fn_info.param_types.get(ip)) |param_ty_index| {
2134	const param_ty: Type = .fromInterned(param_ty_index);
2135	if (!param_ty.hasRuntimeBits(zcu)) continue;
2136	const param_type_id = try cg.resolveType(param_ty, .direct);
2137	try stub.emit(gpa, .OpFunctionParameter, .{
2138	.id_result_type = param_type_id,
2139	.id_result = cg.allocId(),
2140	});
2141	}
2142	try stub.emit(gpa, .OpFunctionEnd, {});
2143	try cg.sections.functions.append(gpa, stub);
2144
2145	const extern_name = nav.getExtern(ip).?.name.toSlice(ip);
2146	try cg.sections.annotations.emit(gpa, .OpDecorate, .{
2147	.target = decl.result_id,
2148	.decoration = .{ .linkage_attributes = .{
2149	.name = extern_name,
2150	.linkage_type = .import,
2151	} },
2152	});
2153	try cg.debugName(decl.result_id, extern_name);
2154	}
2155
2156	fn resolveTypeName(cg: *CodeGen, ty: Type) ![]const u8 {
2157	const gpa = cg.gpa;
2158	var aw: std.Io.Writer.Allocating = .init(gpa);
2159	defer aw.deinit();
2160	ty.print(&aw.writer, cg.pt, null) catch |err| switch (err) {
2161	error.WriteFailed => return error.OutOfMemory,
2162	};
2163	return try aw.toOwnedSlice();
2164	}
2165
2166	/// Generate a union type. Union types are always generated with the
2167	/// most aligned field active. If the tag alignment is greater
2168	/// than that of the payload, a regular union (non-packed, with both tag and
2169	/// payload), will be generated as follows:
2170	/// struct {
2171	/// tag: TagType,
2172	/// payload: MostAlignedFieldType,
2173	/// payload_padding: [payload_size - @sizeOf(MostAlignedFieldType)]u8,
2174	/// padding: [padding_size]u8,
2175	/// }
2176	/// If the payload alignment is greater than that of the tag:
2177	/// struct {
2178	/// payload: MostAlignedFieldType,
2179	/// payload_padding: [payload_size - @sizeOf(MostAlignedFieldType)]u8,
2180	/// tag: TagType,
2181	/// padding: [padding_size]u8,
2182	/// }
2183	/// If any of the fields' size is 0, it will be omitted.
2184	fn resolveFnReturnType(cg: *CodeGen, ret_ty: Type) !Id {
2185	const zcu = cg.zcu;
2186	if (!ret_ty.hasRuntimeBits(zcu)) {
2187	// If the return type is an error set or an error union, then we make this
2188	// anyerror return type instead, so that it can be coerced into a function
2189	// pointer type which has anyerror as the return type.
2190	if (ret_ty.isError(zcu)) {
2191	return cg.resolveType(.anyerror, .direct);
2192	} else {
2193	return cg.resolveType(.void, .direct);
2194	}
2195	}
2196
2197	return try cg.resolveType(ret_ty, .direct);
2198	}
2199
2200	fn resolveType(cg: *CodeGen, ty: Type, repr: Repr) Error!Id {
2201	const gpa = cg.gpa;
2202	const pt = cg.pt;
2203	const zcu = cg.zcu;
2204	const ip = &zcu.intern_pool;
2205	const target = cg.zcu.getTarget();
2206
2207	log.debug("resolveType: ty = {f}", .{ty.fmt(pt)});
2208
2209	switch (ty.zigTypeTag(zcu)) {
2210	.noreturn => {
2211	assert(repr == .direct);
2212	return try cg.voidType();
2213	},
2214	.void => switch (repr) {
2215	.direct => return try cg.voidType(),
2216	.indirect => {
2217	if (target.os.tag != .opencl) return cg.fail("cannot generate opaque type", .{});
2218	return try cg.opaqueType("void");
2219	},
2220	},
2221	.bool => switch (repr) {
2222	.direct => return try cg.boolType(),
2223	.indirect => return try cg.resolveType(.u1, .indirect),
2224	},
2225	.int => {
2226	if (ty.toIntern() == .u0_type) {
2227	assert(repr == .indirect);
2228	if (target.os.tag != .opencl) return cg.fail("cannot generate opaque type", .{});
2229	return try cg.opaqueType("u0");
2230	}
2231	const int_info = ty.intInfo(zcu);
2232	return try cg.intType(int_info.signedness, int_info.bits);
2233	},
2234	.@"enum" => return try cg.resolveType(ty.backingIntType(zcu), repr),
2235	.float => {
2236	const bits = ty.floatBits(target);
2237	const supported = switch (bits) {
2238	16 => target.cpu.has(.spirv, .float16),
2239	32 => true,
2240	64 => target.cpu.has(.spirv, .float64),
2241	else => false,
2242	};
2243	if (!supported) return cg.fail(
2244	"'{f}' is not supported on the current SPIR-V feature set",
2245	.{ty.fmt(cg.pt)},
2246	);
2247	return try cg.floatType(bits);
2248	},
2249	.array => {
2250	const elem_ty = ty.childType(zcu);
2251	const elem_ty_id = try cg.resolveType(elem_ty, .indirect);
2252	const total_len = std.math.cast(u32, ty.arrayLenIncludingSentinel(zcu)) orelse {
2253	return cg.fail("array type of {} elements is too large", .{ty.arrayLenIncludingSentinel(zcu)});
2254	};
2255
2256	if (!elem_ty.hasRuntimeBits(zcu)) {
2257	assert(repr == .indirect);
2258	if (target.os.tag != .opencl) return cg.fail("cannot generate opaque type", .{});
2259	return try cg.opaqueType("zero-sized-array");
2260	} else if (total_len == 0) {
2261	// The size of the array would be 0, but that is not allowed in SPIR-V.
2262	// This path can be reached for example when there is a slicing of a pointer
2263	// that produces a zero-length array. In all cases where this type can be generated,
2264	// this should be an indirect path.
2265	assert(repr == .indirect);
2266	// In this case, we have an array of a non-zero sized type. In this case,
2267	// generate an array of 1 element instead, so that ptr_elem_ptr instructions
2268	// can be lowered to ptrAccessChain instead of manually performing the math.
2269	const len_id = try cg.constInt(.u32, 1);
2270	return try cg.arrayType(len_id, elem_ty_id);
2271	} else {
2272	const total_len_id = try cg.constInt(.u32, total_len);
2273	return try cg.arrayType(total_len_id, elem_ty_id);
2274	}
2275	},
2276	.vector => {
2277	const elem_ty = ty.childType(zcu);
2278	const elem_ty_id = try cg.resolveType(elem_ty, repr);
2279	const len = ty.vectorLen(zcu);
2280	if (cg.isSpvVector(ty)) return try cg.vectorType(len, elem_ty_id);
2281	const len_id = try cg.constInt(.u32, len);
2282	return try cg.arrayType(len_id, elem_ty_id);
2283	},
2284	.@"fn" => switch (repr) {
2285	.direct => {
2286	const fn_info = zcu.typeToFunc(ty).?;
2287
2288	assert(!fn_info.is_var_args);
2289	switch (fn_info.cc) {
2290	.auto,
2291	.spirv_kernel,
2292	.spirv_fragment,
2293	.spirv_vertex,
2294	.spirv_device,
2295	.spirv_task,
2296	.spirv_mesh,
2297	=> {},
2298	else => unreachable,
2299	}
2300
2301	const return_ty_id = try cg.resolveFnReturnType(.fromInterned(fn_info.return_type));
2302
2303	const scratch_top = cg.id_scratch.items.len;
2304	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
2305	const param_ty_ids = try cg.id_scratch.addManyAsSlice(gpa, fn_info.param_types.len);
2306
2307	var param_index: usize = 0;
2308	for (fn_info.param_types.get(ip)) |param_ty_index| {
2309	const param_ty: Type = .fromInterned(param_ty_index);
2310	if (!param_ty.hasRuntimeBits(zcu)) continue;
2311
2312	param_ty_ids[param_index] = try cg.resolveType(param_ty, .direct);
2313	param_index += 1;
2314	}
2315
2316	return try cg.functionType(return_ty_id, param_ty_ids[0..param_index]);
2317	},
2318	.indirect => {
2319	// TODO: Represent function pointers properly.
2320	// For now, just use an usize type.
2321	return try cg.resolveType(.usize, .indirect);
2322	},
2323	},
2324	.pointer => {
2325	const ptr_info = ty.ptrInfo(zcu);
2326
2327	const child_ty: Type = switch (ptr_info.packed_offset.host_size) {
2328	0 => .fromInterned(ptr_info.child),
2329	else => switch (ptr_info.flags.vector_index) {
2330	// Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate these usages of `pt`.
2331	.none => try pt.intType(.unsigned, ptr_info.packed_offset.host_size * 8),
2332	else => try pt.vectorType(.{
2333	.child = ptr_info.child,
2334	.len = ptr_info.packed_offset.host_size,
2335	}),
2336	},
2337	};
2338	const child_ty_id = try cg.pointeeType(ptr_info.flags.address_space, child_ty, false);
2339	const storage_class = cg.storageClass(ptr_info.flags.address_space);
2340	const ptr_ty_id = try cg.ptrType(child_ty_id, storage_class);
2341
2342	if (ptr_info.flags.size != .slice) {
2343	return ptr_ty_id;
2344	}
2345
2346	const size_ty_id = try cg.resolveType(.usize, .direct);
2347	return try cg.structType(
2348	&.{ ptr_ty_id, size_ty_id },
2349	&.{ "ptr", "len" },
2350	.none,
2351	);
2352	},
2353	.@"struct" => {
2354	const struct_type = switch (ip.indexToKey(ty.toIntern())) {
2355	.tuple_type => |tuple| {
2356	const scratch_top = cg.id_scratch.items.len;
2357	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
2358	const member_types = try cg.id_scratch.addManyAsSlice(gpa, tuple.values.len);
2359
2360	var member_index: usize = 0;
2361	for (tuple.types.get(ip), tuple.values.get(ip)) |field_ty, field_val| {
2362	if (field_val != .none or !Type.fromInterned(field_ty).hasRuntimeBits(zcu)) continue;
2363
2364	member_types[member_index] = try cg.resolveType(.fromInterned(field_ty), .indirect);
2365	member_index += 1;
2366	}
2367
2368	const result_id = try cg.structType(
2369	member_types[0..member_index],
2370	null,
2371	.none,
2372	);
2373	const type_name = try cg.resolveTypeName(ty);
2374	defer gpa.free(type_name);
2375	try cg.debugName(result_id, type_name);
2376	return result_id;
2377	},
2378	.struct_type => ip.loadStructType(ty.toIntern()),
2379	else => unreachable,
2380	};
2381
2382	if (struct_type.layout == .@"packed") {
2383	return try cg.resolveType(.fromInterned(struct_type.packed_backing_int_type), .direct);
2384	}
2385
2386	var member_types: std.ArrayList(Id) = .empty;
2387	defer member_types.deinit(gpa);
2388
2389	var member_names: std.ArrayList([]const u8) = .empty;
2390	defer member_names.deinit(gpa);
2391
2392	var it = struct_type.iterateRuntimeOrder(ip);
2393	while (it.next()) |field_index| {
2394	const field_ty: Type = .fromInterned(struct_type.field_types.get(ip)[field_index]);
2395	if (!field_ty.hasRuntimeBits(zcu)) continue;
2396
2397	const field_name = struct_type.field_names.get(ip)[field_index];
2398	try member_types.append(gpa, try cg.resolveType(field_ty, .indirect));
2399	try member_names.append(gpa, field_name.toSlice(ip));
2400	}
2401
2402	const result_id = try cg.structType(
2403	member_types.items,
2404	member_names.items,
2405	ty.toIntern(),
2406	);
2407
2408	const type_name = try cg.resolveTypeName(ty);
2409	defer gpa.free(type_name);
2410	try cg.debugName(result_id, type_name);
2411
2412	return result_id;
2413	},
2414	.optional => {
2415	const payload_ty = ty.optionalChild(zcu);
2416	if (!payload_ty.hasRuntimeBits(zcu)) {
2417	// Just use a bool.
2418	// Note: Always generate the bool with indirect format, to save on some sanity
2419	// Perform the conversion to a direct bool when the field is extracted.
2420	return try cg.resolveType(.bool, .indirect);
2421	}
2422
2423	const payload_ty_id = try cg.resolveType(payload_ty, .indirect);
2424	if (ty.optionalReprIsPayload(zcu)) {
2425	// Optional is actually a pointer or a slice.
2426	return payload_ty_id;
2427	}
2428
2429	const bool_ty_id = try cg.resolveType(.bool, .indirect);
2430
2431	return try cg.structType(
2432	&.{ payload_ty_id, bool_ty_id },
2433	&.{ "payload", "valid" },
2434	.none,
2435	);
2436	},
2437	.@"union" => {
2438	const union_obj = zcu.typeToUnion(ty).?;
2439	if (union_obj.layout == .@"packed") {
2440	return try cg.intType(.unsigned, @intCast(ty.bitSize(zcu)));
2441	}
2442	const layout = cg.unionLayout(ty);
2443	if (!layout.has_payload) {
2444	return try cg.resolveType(.fromInterned(union_obj.enum_tag_type), .indirect);
2445	}
2446	var member_types: [4]Id = undefined;
2447	var member_names: [4][]const u8 = undefined;
2448	const u8_ty_id = try cg.resolveType(.u8, .direct);
2449	if (layout.tag_size != 0) {
2450	member_types[layout.tag_index] = try cg.resolveType(.fromInterned(union_obj.enum_tag_type), .indirect);
2451	member_names[layout.tag_index] = "(tag)";
2452	}
2453	if (layout.payload_size != 0) {
2454	member_types[layout.payload_index] = try cg.resolveType(layout.payload_ty, .indirect);
2455	member_names[layout.payload_index] = "(payload)";
2456	}
2457	if (layout.payload_padding_size != 0) {
2458	const len_id = try cg.constInt(.u32, layout.payload_padding_size);
2459	member_types[layout.payload_padding_index] = try cg.arrayType(len_id, u8_ty_id);
2460	member_names[layout.payload_padding_index] = "(payload padding)";
2461	}
2462	if (layout.padding_size != 0) {
2463	const len_id = try cg.constInt(.u32, layout.padding_size);
2464	member_types[layout.padding_index] = try cg.arrayType(len_id, u8_ty_id);
2465	member_names[layout.padding_index] = "(padding)";
2466	}
2467	const result_id = try cg.structType(
2468	member_types[0..layout.total_fields],
2469	member_names[0..layout.total_fields],
2470	.none,
2471	);
2472	const type_name = try cg.resolveTypeName(ty);
2473	defer gpa.free(type_name);
2474	try cg.debugName(result_id, type_name);
2475	return result_id;
2476	},
2477	.error_set => {
2478	const err_int_ty = try pt.errorIntType();
2479	return try cg.resolveType(err_int_ty, repr);
2480	},
2481	.error_union => {
2482	const payload_ty = ty.errorUnionPayload(zcu);
2483	const err_ty = ty.errorUnionSet(zcu);
2484	const error_ty_id = try cg.resolveType(err_ty, .indirect);
2485
2486	const eu_layout = cg.errorUnionLayout(payload_ty);
2487	if (!eu_layout.payload_has_bits) {
2488	return error_ty_id;
2489	}
2490
2491	const payload_ty_id = try cg.resolveType(payload_ty, .indirect);
2492
2493	var member_types: [2]Id = undefined;
2494	var member_names: [2][]const u8 = undefined;
2495	if (eu_layout.error_first) {
2496	// Put the error first
2497	member_types = .{ error_ty_id, payload_ty_id };
2498	member_names = .{ "error", "payload" };
2499	// TODO: ABI padding?
2500	} else {
2501	// Put the payload first.
2502	member_types = .{ payload_ty_id, error_ty_id };
2503	member_names = .{ "payload", "error" };
2504	// TODO: ABI padding?
2505	}
2506
2507	return try cg.structType(&member_types, &member_names, .none);
2508	},
2509	.@"opaque" => {
2510	if (target.os.tag != .opencl) return cg.fail("cannot generate opaque type", .{});
2511	const type_name = try cg.resolveTypeName(ty);
2512	defer gpa.free(type_name);
2513	return try cg.opaqueType(type_name);
2514	},
2515	.spirv => {
2516	const spirv_type = ip.loadSpirvType(ty.toIntern());
2517	const result_id = cg.allocId();
2518	switch (spirv_type.flags.tag) {
2519	.sampler => try cg.sections.globals.emit(gpa, .OpTypeSampler, .{ .id_result = result_id }),
2520	.image => {
2521	const sampled_type_id = try cg.resolveType(.fromInterned(spirv_type.ty), .direct);
2522	try cg.sections.globals.emit(gpa, .OpTypeImage, .{
2523	.id_result = result_id,
2524	.sampled_type = sampled_type_id,
2525	.dim = switch (spirv_type.flags.dim) {
2526	.@"1d" => .@"1d",
2527	.@"2d" => .@"2d",
2528	.@"3d" => .@"3d",
2529	.cube => .cube,
2530	},
2531	.depth = switch (spirv_type.flags.depth) {
2532	.not_depth => 0,
2533	.depth => 1,
2534	.unknown => 2,
2535	},
2536	.arrayed = @intFromBool(spirv_type.flags.is_arrayed),
2537	.ms = @intFromBool(spirv_type.flags.is_multisampled),
2538	.sampled = switch (spirv_type.flags.usage) {
2539	.unknown => 0,
2540	.sampled => 1,
2541	.storage => 2,
2542	},
2543	.image_format = switch (spirv_type.flags.format) {
2544	.unknown => .unknown,
2545	.rgba32f => .rgba32f,
2546	.rgba32i => .rgba32i,
2547	.rgba32u => .rgba32ui,
2548	.rgba16f => .rgba16f,
2549	.rgba16i => .rgba16i,
2550	.rgba16u => .rgba16ui,
2551	.rgba8unorm => .rgba8,
2552	.rgba8snorm => .rgba8snorm,
2553	.rgba8i => .rgba8i,
2554	.rgba8u => .rgba8ui,
2555	.r32f => .r32f,
2556	.r32i => .r32i,
2557	.r32u => .r32ui,
2558	},
2559	.access_qualifier = switch (spirv_type.flags.access) {
2560	.unknown => null,
2561	.read_only => .read_only,
2562	.write_only => .write_only,
2563	.read_write => .read_write,
2564	},
2565	});
2566	},
2567	.sampled_image => {
2568	const image_ty_id = try cg.resolveType(.fromInterned(spirv_type.ty), .indirect);
2569	try cg.sections.globals.emit(gpa, .OpTypeSampledImage, .{
2570	.id_result = result_id,
2571	.image_type = image_ty_id,
2572	});
2573	},
2574	.runtime_array => {
2575	const elem_ty: Type = .fromInterned(spirv_type.ty);
2576	const elem_ty_id = try cg.resolveType(elem_ty, .indirect);
2577	try cg.sections.globals.emit(gpa, .OpTypeRuntimeArray, .{
2578	.id_result = result_id,
2579	.element_type = elem_ty_id,
2580	});
2581	if (elem_ty.hasRuntimeBits(zcu)) {
2582	try cg.decorate(result_id, .{ .array_stride = .{
2583	.array_stride = @intCast(elem_ty.abiSize(zcu)),
2584	} });
2585	}
2586	},
2587	}
2588	return result_id;
2589	},
2590
2591	.null,
2592	.undefined,
2593	.enum_literal,
2594	.comptime_float,
2595	.comptime_int,
2596	.type,
2597	=> unreachable, // Must be comptime.
2598
2599	.frame, .@"anyframe" => unreachable, // TODO
2600	}
2601	}
2602
2603	const ErrorUnionLayout = struct {
2604	payload_has_bits: bool,
2605	error_first: bool,
2606
2607	fn errorFieldIndex(cg: @This()) u32 {
2608	assert(cg.payload_has_bits);
2609	return if (cg.error_first) 0 else 1;
2610	}
2611
2612	fn payloadFieldIndex(cg: @This()) u32 {
2613	assert(cg.payload_has_bits);
2614	return if (cg.error_first) 1 else 0;
2615	}
2616	};
2617
2618	fn errorUnionLayout(cg: *CodeGen, payload_ty: Type) ErrorUnionLayout {
2619	const zcu = cg.zcu;
2620
2621	const error_align = Type.abiAlignment(.anyerror, zcu);
2622	const payload_align = payload_ty.abiAlignment(zcu);
2623
2624	const error_first = error_align.compare(.gt, payload_align);
2625	return .{
2626	.payload_has_bits = payload_ty.hasRuntimeBits(zcu),
2627	.error_first = error_first,
2628	};
2629	}
2630
2631	const UnionLayout = struct {
2632	/// If false, this union is represented
2633	/// by only an integer of the tag type.
2634	has_payload: bool,
2635	tag_size: u32,
2636	tag_index: u32,
2637	/// Note: This is the size of the payload type itcg, NOT the size of the ENTIRE payload.
2638	/// Use `has_payload` instead!!
2639	payload_ty: Type,
2640	payload_size: u32,
2641	payload_index: u32,
2642	payload_padding_size: u32,
2643	payload_padding_index: u32,
2644	padding_size: u32,
2645	padding_index: u32,
2646	total_fields: u32,
2647	};
2648
2649	fn unionLayout(cg: *CodeGen, ty: Type) UnionLayout {
2650	const zcu = cg.zcu;
2651	const ip = &zcu.intern_pool;
2652	const layout = ty.unionGetLayout(zcu);
2653	const union_obj = zcu.typeToUnion(ty).?;
2654
2655	var union_layout: UnionLayout = .{
2656	.has_payload = layout.payload_size != 0,
2657	.tag_size = @intCast(layout.tag_size),
2658	.tag_index = undefined,
2659	.payload_ty = undefined,
2660	.payload_size = undefined,
2661	.payload_index = undefined,
2662	.payload_padding_size = undefined,
2663	.payload_padding_index = undefined,
2664	.padding_size = @intCast(layout.padding),
2665	.padding_index = undefined,
2666	.total_fields = undefined,
2667	};
2668
2669	if (union_layout.has_payload) {
2670	const most_aligned_field = layout.most_aligned_field;
2671	const most_aligned_field_ty: Type = .fromInterned(union_obj.field_types.get(ip)[most_aligned_field]);
2672	union_layout.payload_ty = most_aligned_field_ty;
2673	union_layout.payload_size = @intCast(most_aligned_field_ty.abiSize(zcu));
2674	} else {
2675	union_layout.payload_size = 0;
2676	}
2677
2678	union_layout.payload_padding_size = @intCast(layout.payload_size - union_layout.payload_size);
2679
2680	const tag_first = layout.tag_align.compare(.gte, layout.payload_align);
2681	var field_index: u32 = 0;
2682
2683	if (union_layout.tag_size != 0 and tag_first) {
2684	union_layout.tag_index = field_index;
2685	field_index += 1;
2686	}
2687
2688	if (union_layout.payload_size != 0) {
2689	union_layout.payload_index = field_index;
2690	field_index += 1;
2691	}
2692
2693	if (union_layout.payload_padding_size != 0) {
2694	union_layout.payload_padding_index = field_index;
2695	field_index += 1;
2696	}
2697
2698	if (union_layout.tag_size != 0 and !tag_first) {
2699	union_layout.tag_index = field_index;
2700	field_index += 1;
2701	}
2702
2703	if (union_layout.padding_size != 0) {
2704	union_layout.padding_index = field_index;
2705	field_index += 1;
2706	}
2707
2708	union_layout.total_fields = field_index;
2709
2710	return union_layout;
2711	}
2712
2713	/// This structure represents a "temporary" value: Something we are currently
2714	/// operating on. It typically lives no longer than the function that
2715	/// implements a particular AIR operation. These are used to easier
2716	/// implement vectorizable operations (see Vectorization and the build*
2717	/// functions), and typically are only used for vectors of primitive types.
2718	const Temporary = struct {
2719	/// The type of the temporary. This is here mainly
2720	/// for easier bookkeeping. Because we will never really
2721	/// store Temporaries, they only cause extra stack space,
2722	/// therefore no real storage is wasted.
2723	ty: Type,
2724	/// The value that this temporary holds. This is not necessarily
2725	/// a value that is actually usable, or a single value: It is virtual
2726	/// until materialize() is called, at which point is turned into
2727	/// the usual SPIR-V representation of `cg.ty`.
2728	value: Temporary.Value,
2729
2730	const Value = union(enum) {
2731	singleton: Id,
2732	exploded_vector: IdRange,
2733	};
2734
2735	fn init(ty: Type, singleton: Id) Temporary {
2736	return .{ .ty = ty, .value = .{ .singleton = singleton } };
2737	}
2738
2739	fn materialize(temp: Temporary, cg: *CodeGen) !Id {
2740	const gpa = cg.gpa;
2741	const zcu = cg.zcu;
2742	switch (temp.value) {
2743	.singleton => |id| return id,
2744	.exploded_vector => |range| {
2745	assert(temp.ty.isVector(zcu));
2746	assert(temp.ty.vectorLen(zcu) == range.len);
2747
2748	const scratch_top = cg.id_scratch.items.len;
2749	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
2750	const constituents = try cg.id_scratch.addManyAsSlice(gpa, range.len);
2751	for (constituents, 0..range.len) |*id, i| {
2752	id.* = range.at(i);
2753	}
2754
2755	const result_ty_id = try cg.resolveType(temp.ty, .direct);
2756	return cg.constructComposite(result_ty_id, constituents);
2757	},
2758	}
2759	}
2760
2761	fn vectorization(temp: Temporary, cg: *CodeGen) Vectorization {
2762	return .fromType(temp.ty, cg);
2763	}
2764
2765	fn pun(temp: Temporary, new_ty: Type) Temporary {
2766	return .{
2767	.ty = new_ty,
2768	.value = temp.value,
2769	};
2770	}
2771
2772	/// 'Explode' a temporary into separate elements. This turns a vector
2773	/// into a bag of elements.
2774	fn explode(temp: Temporary, cg: *CodeGen) !IdRange {
2775	const zcu = cg.zcu;
2776
2777	// If the value is a scalar, then this is a no-op.
2778	if (!temp.ty.isVector(zcu)) {
2779	return switch (temp.value) {
2780	.singleton => |id| .{ .base = @backingInt(id), .len = 1 },
2781	.exploded_vector => |range| range,
2782	};
2783	}
2784
2785	const ty_id = try cg.resolveType(temp.ty.scalarType(zcu), .direct);
2786	const n = temp.ty.vectorLen(zcu);
2787	const results = cg.allocIds(n);
2788
2789	const id = switch (temp.value) {
2790	.singleton => |id| id,
2791	.exploded_vector => |range| return range,
2792	};
2793
2794	for (0..n) |i| {
2795	const indexes = [_]u32{@intCast(i)};
2796	try cg.body.emit(cg.gpa, .OpCompositeExtract, .{
2797	.id_result_type = ty_id,
2798	.id_result = results.at(i),
2799	.composite = id,
2800	.indexes = &indexes,
2801	});
2802	}
2803
2804	return results;
2805	}
2806	};
2807
2808	/// composite integers are represented as [N]u32 arrays
2809	const CompositeInt = struct {
2810	cg: *CodeGen,
2811	limbs: []Id,
2812	n_limbs: u16,
2813	info: ArithmeticTypeInfo,
2814
2815	fn init(cg: *CodeGen, composite_id: Id, info: ArithmeticTypeInfo) !CompositeInt {
2816	const n_limbs: u16 = info.backing_bits / cg.bigIntBits();
2817	const gpa = cg.gpa;
2818	if (cg.composite_limbs.get(composite_id)) |cached| {
2819	assert(cached.len == n_limbs);
2820	const limbs = try cg.id_scratch.addManyAsSlice(gpa, n_limbs);
2821	@memcpy(limbs, cached);
2822	return .{ .cg = cg, .limbs = limbs, .n_limbs = n_limbs, .info = info };
2823	}
2824	const limb_ty_id = try cg.limbTypeId();
2825	const limbs = try cg.id_scratch.addManyAsSlice(gpa, n_limbs);
2826	for (limbs, 0..) |*limb, i| {
2827	const result_id = cg.allocId();
2828	try cg.body.emit(gpa, .OpCompositeExtract, .{
2829	.id_result_type = limb_ty_id,
2830	.id_result = result_id,
2831	.composite = composite_id,
2832	.indexes = &.{@as(u32, @intCast(i))},
2833	});
2834	limb.* = result_id;
2835	}
2836	const cached = try cg.arena.dupe(Id, limbs);
2837	try cg.composite_limbs.put(gpa, composite_id, cached);
2838	return .{ .cg = cg, .limbs = limbs, .n_limbs = n_limbs, .info = info };
2839	}
2840
2841	fn fromLimbs(cg: *CodeGen, limbs: []Id, info: ArithmeticTypeInfo) CompositeInt {
2842	return .{
2843	.cg = cg,
2844	.limbs = limbs,
2845	.n_limbs = @intCast(limbs.len),
2846	.info = info,
2847	};
2848	}
2849
2850	fn zero(cg: *CodeGen, info: ArithmeticTypeInfo) !CompositeInt {
2851	const n_limbs: u16 = info.backing_bits / cg.bigIntBits();
2852	const limbs = try cg.id_scratch.addManyAsSlice(cg.gpa, n_limbs);
2853	const zero_id = try cg.constInt(cg.limbType(), @as(u64, 0));
2854	for (limbs) |*limb| limb.* = zero_id;
2855	return .{ .cg = cg, .limbs = limbs, .n_limbs = n_limbs, .info = info };
2856	}
2857
2858	fn materialize(ci: CompositeInt, ty: Type) !Id {
2859	const result_ty_id = try ci.cg.resolveType(ty, .indirect);
2860	return ci.cg.constructComposite(result_ty_id, ci.limbs);
2861	}
2862
2863	fn limbBinOp(ci: CompositeInt, opcode: Opcode, lhs: Id, rhs: Id) !Id {
2864	const cg = ci.cg;
2865	const gpa = cg.gpa;
2866	const limb_ty_id = try cg.limbTypeId();
2867	const result_id = cg.allocId();
2868	try cg.body.emitRaw(gpa, opcode, 4);
2869	cg.body.writeOperand(Id, limb_ty_id);
2870	cg.body.writeOperand(Id, result_id);
2871	cg.body.writeOperand(Id, lhs);
2872	cg.body.writeOperand(Id, rhs);
2873	return result_id;
2874	}
2875
2876	fn limbUnOp(ci: CompositeInt, opcode: Opcode, operand: Id) !Id {
2877	const cg = ci.cg;
2878	const gpa = cg.gpa;
2879	const limb_ty_id = try cg.limbTypeId();
2880	const result_id = cg.allocId();
2881	try cg.body.emitRaw(gpa, opcode, 3);
2882	cg.body.writeOperand(Id, limb_ty_id);
2883	cg.body.writeOperand(Id, result_id);
2884	cg.body.writeOperand(Id, operand);
2885	return result_id;
2886	}
2887
2888	fn bitwiseOp(ci: CompositeInt, other: CompositeInt, opcode: Opcode) !CompositeInt {
2889	const cg = ci.cg;
2890	const gpa = cg.gpa;
2891	const result_limbs = try cg.id_scratch.addManyAsSlice(gpa, ci.n_limbs);
2892	for (result_limbs, 0..) |*r, i| {
2893	r.* = try ci.limbBinOp(opcode, ci.limbs[i], other.limbs[i]);
2894	}
2895	return .fromLimbs(cg, result_limbs, ci.info);
2896	}
2897
2898	fn bitwiseNot(ci: CompositeInt) !CompositeInt {
2899	const cg = ci.cg;
2900	const gpa = cg.gpa;
2901	const result_limbs = try cg.id_scratch.addManyAsSlice(gpa, ci.n_limbs);
2902	for (result_limbs, 0..) |*r, i| {
2903	r.* = try ci.limbUnOp(.OpNot, ci.limbs[i]);
2904	}
2905	return .fromLimbs(cg, result_limbs, ci.info);
2906	}
2907
2908	fn cmp(ci: CompositeInt, other: CompositeInt, op: std.math.CompareOperator) !Id {
2909	const cg = ci.cg;
2910	const gpa = cg.gpa;
2911	const bool_ty_id = try cg.resolveType(.bool, .direct);
2912
2913	switch (op) {
2914	.eq, .neq => {
2915	var result = blk: {
2916	const r = cg.allocId();
2917	try cg.body.emit(gpa, .OpIEqual, .{
2918	.id_result_type = bool_ty_id,
2919	.id_result = r,
2920	.operand_1 = ci.limbs[0],
2921	.operand_2 = other.limbs[0],
2922	});
2923	break :blk r;
2924	};
2925	for (1..ci.n_limbs) |i| {
2926	const limb_eq = cg.allocId();
2927	try cg.body.emit(gpa, .OpIEqual, .{
2928	.id_result_type = bool_ty_id,
2929	.id_result = limb_eq,
2930	.operand_1 = ci.limbs[i],
2931	.operand_2 = other.limbs[i],
2932	});
2933	const combined = cg.allocId();
2934	try cg.body.emit(gpa, .OpLogicalAnd, .{
2935	.id_result_type = bool_ty_id,
2936	.id_result = combined,
2937	.operand_1 = result,
2938	.operand_2 = limb_eq,
2939	});
2940	result = combined;
2941	}
2942	if (op == .neq) {
2943	const negated = cg.allocId();
2944	try cg.body.emit(gpa, .OpLogicalNot, .{
2945	.id_result_type = bool_ty_id,
2946	.id_result = negated,
2947	.operand = result,
2948	});
2949	result = negated;
2950	}
2951	return result;
2952	},
2953	.lt, .lte, .gt, .gte => {
2954	const is_lt = (op == .lt or op == .lte);
2955	const is_strict = (op == .lt or op == .gt);
2956	var result = try cg.constBool(!is_strict, .direct);
2957
2958	for (0..ci.n_limbs) |i| {
2959	const l = ci.limbs[i];
2960	const r = other.limbs[i];
2961	const limb_ne = cg.allocId();
2962	try cg.body.emit(gpa, .OpINotEqual, .{
2963	.id_result_type = bool_ty_id,
2964	.id_result = limb_ne,
2965	.operand_1 = l,
2966	.operand_2 = r,
2967	});
2968
2969	const is_top = (i == ci.n_limbs - 1);
2970	const use_signed = is_top and ci.info.signedness == .signed;
2971	var cmp_l = l;
2972	var cmp_r = r;
2973	if (use_signed) {
2974	const signed_limb_ty: Type = if (cg.bigIntBits() == 64) .i64 else .i32;
2975	const signed_limb_ty_id = try cg.resolveType(signed_limb_ty, .direct);
2976	const sl = cg.allocId();
2977	try cg.body.emit(gpa, .OpBitcast, .{
2978	.id_result_type = signed_limb_ty_id,
2979	.id_result = sl,
2980	.operand = l,
2981	});
2982	const sr = cg.allocId();
2983	try cg.body.emit(gpa, .OpBitcast, .{
2984	.id_result_type = signed_limb_ty_id,
2985	.id_result = sr,
2986	.operand = r,
2987	});
2988	cmp_l = sl;
2989	cmp_r = sr;
2990	}
2991
2992	const cmp_opcode: Opcode = if (is_lt)
2993	(if (use_signed) .OpSLessThan else .OpULessThan)
2994	else
2995	(if (use_signed) .OpSGreaterThan else .OpUGreaterThan);
2996
2997	const limb_cmp = cg.allocId();
2998	try cg.body.emitRaw(gpa, cmp_opcode, 4);
2999	cg.body.writeOperand(Id, bool_ty_id);
3000	cg.body.writeOperand(Id, limb_cmp);
3001	cg.body.writeOperand(Id, cmp_l);
3002	cg.body.writeOperand(Id, cmp_r);
3003
3004	const selected = cg.allocId();
3005	try cg.body.emit(gpa, .OpSelect, .{
3006	.id_result_type = bool_ty_id,
3007	.id_result = selected,
3008	.condition = limb_ne,
3009	.object_1 = limb_cmp,
3010	.object_2 = result,
3011	});
3012	result = selected;
3013	}
3014	return result;
3015	},
3016	}
3017	}
3018
3019	fn addSub(ci: CompositeInt, other: CompositeInt, comptime is_add: bool) !CompositeInt {
3020	const cg = ci.cg;
3021	const gpa = cg.gpa;
3022	const pt = cg.pt;
3023	const zcu = cg.zcu;
3024	const ip = &zcu.intern_pool;
3025	const comp = zcu.comp;
3026	const io = comp.io;
3027
3028	const limb_bits = cg.bigIntBits();
3029	const limb_zig = try pt.intType(.unsigned, limb_bits);
3030	const limb_ty_id = try cg.limbTypeId();
3031	const carry_struct_ty: Type = .fromInterned(try ip.getTupleType(gpa, io, pt.tid, .{
3032	.types = &.{ limb_zig.toIntern(), limb_zig.toIntern() },
3033	.values = &.{ .none, .none },
3034	}));
3035	const carry_struct_ty_id = try cg.resolveType(carry_struct_ty, .direct);
3036
3037	const result_limbs = try cg.id_scratch.addManyAsSlice(gpa, ci.n_limbs);
3038	var carry_id = try cg.constInt(cg.limbType(), @as(u64, 0));
3039
3040	const opcode: Opcode = if (is_add) .OpIAddCarry else .OpISubBorrow;
3041
3042	for (0..ci.n_limbs) |i| {
3043	const op1 = cg.allocId();
3044	try cg.body.emitRaw(gpa, opcode, 4);
3045	cg.body.writeOperand(Id, carry_struct_ty_id);
3046	cg.body.writeOperand(Id, op1);
3047	cg.body.writeOperand(Id, ci.limbs[i]);
3048	cg.body.writeOperand(Id, other.limbs[i]);
3049
3050	const sum1 = cg.allocId();
3051	try cg.body.emit(gpa, .OpCompositeExtract, .{
3052	.id_result_type = limb_ty_id,
3053	.id_result = sum1,
3054	.composite = op1,
3055	.indexes = &.{0},
3056	});
3057	const carry1 = cg.allocId();
3058	try cg.body.emit(gpa, .OpCompositeExtract, .{
3059	.id_result_type = limb_ty_id,
3060	.id_result = carry1,
3061	.composite = op1,
3062	.indexes = &.{1},
3063	});
3064
3065	const op2 = cg.allocId();
3066	try cg.body.emitRaw(gpa, opcode, 4);
3067	cg.body.writeOperand(Id, carry_struct_ty_id);
3068	cg.body.writeOperand(Id, op2);
3069	cg.body.writeOperand(Id, sum1);
3070	cg.body.writeOperand(Id, carry_id);
3071
3072	result_limbs[i] = cg.allocId();
3073	try cg.body.emit(gpa, .OpCompositeExtract, .{
3074	.id_result_type = limb_ty_id,
3075	.id_result = result_limbs[i],
3076	.composite = op2,
3077	.indexes = &.{0},
3078	});
3079	const carry2 = cg.allocId();
3080	try cg.body.emit(gpa, .OpCompositeExtract, .{
3081	.id_result_type = limb_ty_id,
3082	.id_result = carry2,
3083	.composite = op2,
3084	.indexes = &.{1},
3085	});
3086
3087	carry_id = try ci.limbBinOp(.OpBitwiseOr, carry1, carry2);
3088	}
3089
3090	return .fromLimbs(cg, result_limbs, ci.info);
3091	}
3092
3093	fn shl(ci: CompositeInt, shift_amt_id: Id) !CompositeInt {
3094	const cg = ci.cg;
3095	const gpa = cg.gpa;
3096	const limb_bits = cg.bigIntBits();
3097	const limb_ty = cg.limbType();
3098	const limb_ty_id = try cg.limbTypeId();
3099	const bool_ty_id = try cg.resolveType(.bool, .direct);
3100	const zero_id = try cg.constInt(limb_ty, @as(u64, 0));
3101	const log2_bits_id = try cg.constInt(limb_ty, @as(u64, std.math.log2_int(u16, limb_bits)));
3102	const bits_minus_1_id = try cg.constInt(limb_ty, @as(u64, limb_bits - 1));
3103	const bits_id = try cg.constInt(limb_ty, @as(u64, limb_bits));
3104
3105	const whole = try ci.limbBinOp(.OpShiftRightLogical, shift_amt_id, log2_bits_id);
3106	const frac = try ci.limbBinOp(.OpBitwiseAnd, shift_amt_id, bits_minus_1_id);
3107	const comp_frac = try ci.limbBinOp(.OpISub, bits_id, frac);
3108	const frac_is_zero = blk: {
3109	const r = cg.allocId();
3110	try cg.body.emit(gpa, .OpIEqual, .{
3111	.id_result_type = bool_ty_id,
3112	.id_result = r,
3113	.operand_1 = frac,
3114	.operand_2 = zero_id,
3115	});
3116	break :blk r;
3117	};
3118
3119	const result_limbs = try cg.id_scratch.addManyAsSlice(gpa, ci.n_limbs);
3120
3121	for (0..ci.n_limbs) |i| {
3122	const i_id = try cg.constInt(limb_ty, @as(u64, @intCast(i)));
3123	var main_val = zero_id;
3124	var carry_val = zero_id;
3125
3126	for (0..ci.n_limbs) |j| {
3127	const j_id = try cg.constInt(limb_ty, @as(u64, @intCast(j)));
3128	const j_plus_whole = try ci.limbBinOp(.OpIAdd, j_id, whole);
3129
3130	const is_main = blk: {
3131	const r = cg.allocId();
3132	try cg.body.emit(gpa, .OpIEqual, .{
3133	.id_result_type = bool_ty_id,
3134	.id_result = r,
3135	.operand_1 = j_plus_whole,
3136	.operand_2 = i_id,
3137	});
3138	break :blk r;
3139	};
3140	const shifted = try ci.limbBinOp(.OpShiftLeftLogical, ci.limbs[j], frac);
3141	main_val = blk: {
3142	const r = cg.allocId();
3143	try cg.body.emit(gpa, .OpSelect, .{
3144	.id_result_type = limb_ty_id,
3145	.id_result = r,
3146	.condition = is_main,
3147	.object_1 = shifted,
3148	.object_2 = main_val,
3149	});
3150	break :blk r;
3151	};
3152
3153	const one_id = try cg.constInt(limb_ty, @as(u64, 1));
3154	const j_plus_whole_plus_1 = try ci.limbBinOp(.OpIAdd, j_plus_whole, one_id);
3155	const is_carry = blk: {
3156	const r = cg.allocId();
3157	try cg.body.emit(gpa, .OpIEqual, .{
3158	.id_result_type = bool_ty_id,
3159	.id_result = r,
3160	.operand_1 = j_plus_whole_plus_1,
3161	.operand_2 = i_id,
3162	});
3163	break :blk r;
3164	};
3165	const carry_shifted = try ci.limbBinOp(.OpShiftRightLogical, ci.limbs[j], comp_frac);
3166	const guarded_carry = blk: {
3167	const r = cg.allocId();
3168	try cg.body.emit(gpa, .OpSelect, .{
3169	.id_result_type = limb_ty_id,
3170	.id_result = r,
3171	.condition = frac_is_zero,
3172	.object_1 = zero_id,
3173	.object_2 = carry_shifted,
3174	});
3175	break :blk r;
3176	};
3177	carry_val = blk: {
3178	const r = cg.allocId();
3179	try cg.body.emit(gpa, .OpSelect, .{
3180	.id_result_type = limb_ty_id,
3181	.id_result = r,
3182	.condition = is_carry,
3183	.object_1 = guarded_carry,
3184	.object_2 = carry_val,
3185	});
3186	break :blk r;
3187	};
3188	}
3189
3190	result_limbs[i] = try ci.limbBinOp(.OpBitwiseOr, main_val, carry_val);
3191	}
3192
3193	return .fromLimbs(cg, result_limbs, ci.info);
3194	}
3195
3196	fn shr(ci: CompositeInt, shift_amt_id: Id, comptime is_arithmetic: bool) !CompositeInt {
3197	const cg = ci.cg;
3198	const gpa = cg.gpa;
3199	const limb_bits = cg.bigIntBits();
3200	const limb_ty = cg.limbType();
3201	const limb_ty_id = try cg.limbTypeId();
3202	const bool_ty_id = try cg.resolveType(.bool, .direct);
3203	const zero_id = try cg.constInt(limb_ty, @as(u64, 0));
3204	const log2_bits_id = try cg.constInt(limb_ty, @as(u64, std.math.log2_int(u16, limb_bits)));
3205	const bits_minus_1_id = try cg.constInt(limb_ty, @as(u64, limb_bits - 1));
3206	const bits_id = try cg.constInt(limb_ty, @as(u64, limb_bits));
3207
3208	const whole = try ci.limbBinOp(.OpShiftRightLogical, shift_amt_id, log2_bits_id);
3209	const frac = try ci.limbBinOp(.OpBitwiseAnd, shift_amt_id, bits_minus_1_id);
3210	const comp_frac = try ci.limbBinOp(.OpISub, bits_id, frac);
3211	const frac_is_zero = blk: {
3212	const r = cg.allocId();
3213	try cg.body.emit(gpa, .OpIEqual, .{
3214	.id_result_type = bool_ty_id,
3215	.id_result = r,
3216	.operand_1 = frac,
3217	.operand_2 = zero_id,
3218	});
3219	break :blk r;
3220	};
3221
3222	const fill_id = if (is_arithmetic) blk: {
3223	const signed_limb_ty: Type = if (limb_bits == 64) .i64 else .i32;
3224	const signed_limb_ty_id = try cg.resolveType(signed_limb_ty, .direct);
3225	const msb_signed = cg.allocId();
3226	try cg.body.emit(gpa, .OpBitcast, .{
3227	.id_result_type = signed_limb_ty_id,
3228	.id_result = msb_signed,
3229	.operand = ci.limbs[ci.n_limbs - 1],
3230	});
3231	const shift_amt = try cg.constInt(signed_limb_ty, @as(u64, limb_bits - 1));
3232	const sign_ext = cg.allocId();
3233	try cg.body.emit(gpa, .OpShiftRightArithmetic, .{
3234	.id_result_type = signed_limb_ty_id,
3235	.id_result = sign_ext,
3236	.base = msb_signed,
3237	.shift = shift_amt,
3238	});
3239	const back = cg.allocId();
3240	try cg.body.emit(gpa, .OpBitcast, .{
3241	.id_result_type = limb_ty_id,
3242	.id_result = back,
3243	.operand = sign_ext,
3244	});
3245	break :blk back;
3246	} else zero_id;
3247
3248	const result_limbs = try cg.id_scratch.addManyAsSlice(gpa, ci.n_limbs);
3249
3250	const arith_carry_init = if (is_arithmetic) blk: {
3251	const shifted_fill = try ci.limbBinOp(.OpShiftLeftLogical, fill_id, comp_frac);
3252	const guarded = cg.allocId();
3253	try cg.body.emit(gpa, .OpSelect, .{
3254	.id_result_type = limb_ty_id,
3255	.id_result = guarded,
3256	.condition = frac_is_zero,
3257	.object_1 = zero_id,
3258	.object_2 = shifted_fill,
3259	});
3260	break :blk guarded;
3261	} else zero_id;
3262
3263	for (0..ci.n_limbs) |i| {
3264	const i_id = try cg.constInt(limb_ty, @as(u64, @intCast(i)));
3265	var main_val = fill_id;
3266	var carry_val = arith_carry_init;
3267
3268	for (0..ci.n_limbs) |j| {
3269	const j_id = try cg.constInt(limb_ty, @as(u64, @intCast(j)));
3270	const i_plus_whole = try ci.limbBinOp(.OpIAdd, i_id, whole);
3271	const is_main = blk: {
3272	const r = cg.allocId();
3273	try cg.body.emit(gpa, .OpIEqual, .{
3274	.id_result_type = bool_ty_id,
3275	.id_result = r,
3276	.operand_1 = j_id,
3277	.operand_2 = i_plus_whole,
3278	});
3279	break :blk r;
3280	};
3281	const shifted = try ci.limbBinOp(.OpShiftRightLogical, ci.limbs[j], frac);
3282	main_val = blk: {
3283	const r = cg.allocId();
3284	try cg.body.emit(gpa, .OpSelect, .{
3285	.id_result_type = limb_ty_id,
3286	.id_result = r,
3287	.condition = is_main,
3288	.object_1 = shifted,
3289	.object_2 = main_val,
3290	});
3291	break :blk r;
3292	};
3293
3294	const one_id = try cg.constInt(limb_ty, @as(u64, 1));
3295	const i_plus_whole_plus_1 = try ci.limbBinOp(.OpIAdd, i_plus_whole, one_id);
3296	const is_carry = blk: {
3297	const r = cg.allocId();
3298	try cg.body.emit(gpa, .OpIEqual, .{
3299	.id_result_type = bool_ty_id,
3300	.id_result = r,
3301	.operand_1 = j_id,
3302	.operand_2 = i_plus_whole_plus_1,
3303	});
3304	break :blk r;
3305	};
3306	const carry_shifted = try ci.limbBinOp(.OpShiftLeftLogical, ci.limbs[j], comp_frac);
3307	const guarded_carry = blk: {
3308	const r = cg.allocId();
3309	try cg.body.emit(gpa, .OpSelect, .{
3310	.id_result_type = limb_ty_id,
3311	.id_result = r,
3312	.condition = frac_is_zero,
3313	.object_1 = zero_id,
3314	.object_2 = carry_shifted,
3315	});
3316	break :blk r;
3317	};
3318	carry_val = blk: {
3319	const r = cg.allocId();
3320	try cg.body.emit(gpa, .OpSelect, .{
3321	.id_result_type = limb_ty_id,
3322	.id_result = r,
3323	.condition = is_carry,
3324	.object_1 = guarded_carry,
3325	.object_2 = carry_val,
3326	});
3327	break :blk r;
3328	};
3329	}
3330
3331	result_limbs[i] = try ci.limbBinOp(.OpBitwiseOr, main_val, carry_val);
3332	}
3333
3334	return .fromLimbs(cg, result_limbs, ci.info);
3335	}
3336
3337	fn mul(ci: CompositeInt, other: CompositeInt, comptime wide: bool) ![]Id {
3338	const cg = ci.cg;
3339	const gpa = cg.gpa;
3340	const pt = cg.pt;
3341	const zcu = cg.zcu;
3342	const ip = &zcu.intern_pool;
3343	const comp = zcu.comp;
3344	const io = comp.io;
3345	const target = zcu.getTarget();
3346
3347	const n: usize = ci.n_limbs;
3348	const total: usize = if (wide) 2 * n else n;
3349	const limb_bits = cg.bigIntBits();
3350	const limb_zig = try pt.intType(.unsigned, limb_bits);
3351	const limb_ty_id = try cg.limbTypeId();
3352
3353	const pair_struct_ty: Type = .fromInterned(try ip.getTupleType(gpa, io, pt.tid, .{
3354	.types = &.{ limb_zig.toIntern(), limb_zig.toIntern() },
3355	.values = &.{ .none, .none },
3356	}));
3357	const pair_struct_ty_id = try cg.resolveType(pair_struct_ty, .direct);
3358
3359	const result_limbs = try cg.id_scratch.addManyAsSlice(gpa, total);
3360	const zero_id = try cg.constInt(cg.limbType(), @as(u64, 0));
3361	for (result_limbs) |*r| r.* = zero_id;
3362
3363	for (0..n) |i| {
3364	var carry_id = zero_id;
3365	for (0..n) |j| {
3366	const k = i + j;
3367	if (k >= total) break;
3368
3369	var lo: Id = undefined;
3370	var hi: Id = undefined;
3371	switch (target.os.tag) {
3372	.opencl => {
3373	lo = cg.allocId();
3374	try cg.body.emit(gpa, .OpIMul, .{
3375	.id_result_type = limb_ty_id,
3376	.id_result = lo,
3377	.operand_1 = ci.limbs[i],
3378	.operand_2 = other.limbs[j],
3379	});
3380
3381	const set = try cg.importExtendedSet();
3382	hi = cg.allocId();
3383	try cg.body.emit(gpa, .OpExtInst, .{
3384	.id_result_type = limb_ty_id,
3385	.id_result = hi,
3386	.set = set,
3387	.instruction = .{ .inst = @backingInt(spec.OpenClOpcode.u_mul_hi) },
3388	.id_ref_4 = &.{ ci.limbs[i], other.limbs[j] },
3389	});
3390	},
3391	else => {
3392	const mul_result = cg.allocId();
3393	try cg.body.emit(gpa, .OpUMulExtended, .{
3394	.id_result_type = pair_struct_ty_id,
3395	.id_result = mul_result,
3396	.operand_1 = ci.limbs[i],
3397	.operand_2 = other.limbs[j],
3398	});
3399
3400	lo = cg.allocId();
3401	try cg.body.emit(gpa, .OpCompositeExtract, .{
3402	.id_result_type = limb_ty_id,
3403	.id_result = lo,
3404	.composite = mul_result,
3405	.indexes = &.{0},
3406	});
3407	hi = cg.allocId();
3408	try cg.body.emit(gpa, .OpCompositeExtract, .{
3409	.id_result_type = limb_ty_id,
3410	.id_result = hi,
3411	.composite = mul_result,
3412	.indexes = &.{1},
3413	});
3414	},
3415	}
3416
3417	const add1 = cg.allocId();
3418	try cg.body.emit(gpa, .OpIAddCarry, .{
3419	.id_result_type = pair_struct_ty_id,
3420	.id_result = add1,
3421	.operand_1 = result_limbs[k],
3422	.operand_2 = lo,
3423	});
3424
3425	const sum1 = cg.allocId();
3426	try cg.body.emit(gpa, .OpCompositeExtract, .{
3427	.id_result_type = limb_ty_id,
3428	.id_result = sum1,
3429	.composite = add1,
3430	.indexes = &.{0},
3431	});
3432	const c1 = cg.allocId();
3433	try cg.body.emit(gpa, .OpCompositeExtract, .{
3434	.id_result_type = limb_ty_id,
3435	.id_result = c1,
3436	.composite = add1,
3437	.indexes = &.{1},
3438	});
3439
3440	const add2 = cg.allocId();
3441	try cg.body.emit(gpa, .OpIAddCarry, .{
3442	.id_result_type = pair_struct_ty_id,
3443	.id_result = add2,
3444	.operand_1 = sum1,
3445	.operand_2 = carry_id,
3446	});
3447
3448	result_limbs[k] = cg.allocId();
3449	try cg.body.emit(gpa, .OpCompositeExtract, .{
3450	.id_result_type = limb_ty_id,
3451	.id_result = result_limbs[k],
3452	.composite = add2,
3453	.indexes = &.{0},
3454	});
3455	const c2 = cg.allocId();
3456	try cg.body.emit(gpa, .OpCompositeExtract, .{
3457	.id_result_type = limb_ty_id,
3458	.id_result = c2,
3459	.composite = add2,
3460	.indexes = &.{1},
3461	});
3462
3463	const hi_plus_c1 = try ci.limbBinOp(.OpIAdd, hi, c1);
3464	carry_id = try ci.limbBinOp(.OpIAdd, hi_plus_c1, c2);
3465	}
3466	if (wide and i + n < 2 * n) {
3467	result_limbs[i + n] = try ci.limbBinOp(.OpIAdd, result_limbs[i + n], carry_id);
3468	}
3469	}
3470
3471	return result_limbs;
3472	}
3473
3474	fn normalize(ci: CompositeInt) !CompositeInt {
3475	if (ci.info.bits == ci.info.backing_bits) return ci;
3476	const cg = ci.cg;
3477	const gpa = cg.gpa;
3478	const limb_bits = cg.bigIntBits();
3479	const top_bits: u16 = ci.info.bits % limb_bits;
3480	assert(top_bits != 0);
3481
3482	const result_limbs = try cg.id_scratch.addManyAsSlice(gpa, ci.n_limbs);
3483	for (0..ci.n_limbs - 1) |i| {
3484	result_limbs[i] = ci.limbs[i];
3485	}
3486
3487	const top_limb = ci.limbs[ci.n_limbs - 1];
3488	const limb_ty = cg.limbType();
3489	const limb_signed_ty: Type = if (limb_bits == 64) .i64 else .i32;
3490	switch (ci.info.signedness) {
3491	.unsigned => {
3492	const mask_val: u64 = (@as(u64, 1) << @as(u6, @intCast(top_bits))) - 1;
3493	const mask_id = try cg.constInt(limb_ty, mask_val);
3494	result_limbs[ci.n_limbs - 1] = try ci.limbBinOp(.OpBitwiseAnd, top_limb, mask_id);
3495	},
3496	.signed => {
3497	const limb_ty_id = try cg.limbTypeId();
3498	const signed_ty_id = try cg.resolveType(limb_signed_ty, .direct);
3499	const shift_amt: u32 = @intCast(limb_bits - top_bits);
3500	const shift_id = try cg.constInt(limb_ty, shift_amt);
3501
3502	const as_signed = cg.allocId();
3503	try cg.body.emit(gpa, .OpBitcast, .{
3504	.id_result_type = signed_ty_id,
3505	.id_result = as_signed,
3506	.operand = top_limb,
3507	});
3508	const shifted_left = cg.allocId();
3509	try cg.body.emit(gpa, .OpShiftLeftLogical, .{
3510	.id_result_type = signed_ty_id,
3511	.id_result = shifted_left,
3512	.base = as_signed,
3513	.shift = shift_id,
3514	});
3515	const shifted_right = cg.allocId();
3516	try cg.body.emit(gpa, .OpShiftRightArithmetic, .{
3517	.id_result_type = signed_ty_id,
3518	.id_result = shifted_right,
3519	.base = shifted_left,
3520	.shift = shift_id,
3521	});
3522	const back = cg.allocId();
3523	try cg.body.emit(gpa, .OpBitcast, .{
3524	.id_result_type = limb_ty_id,
3525	.id_result = back,
3526	.operand = shifted_right,
3527	});
3528	result_limbs[ci.n_limbs - 1] = back;
3529	},
3530	}
3531
3532	return .fromLimbs(cg, result_limbs, ci.info);
3533	}
3534	};
3535
3536	/// Initialize a `Temporary` from an AIR value.
3537	fn temporary(cg: *CodeGen, inst: Air.Inst.Ref) !Temporary {
3538	return .{
3539	.ty = cg.typeOf(inst),
3540	.value = .{ .singleton = try cg.resolve(inst) },
3541	};
3542	}
3543
3544	/// This union describes how a particular operation should be vectorized.
3545	/// That depends on the operation and number of components of the inputs.
3546	const Vectorization = union(enum) {
3547	/// This is an operation between scalars.
3548	scalar,
3549	/// This operation is unrolled into separate operations.
3550	/// Inputs may still be SPIR-V vectors, for example,
3551	/// when the operation can't be vectorized in SPIR-V.
3552	/// Value is number of components.
3553	unrolled: u32,
3554
3555	/// Derive a vectorization from a particular type
3556	fn fromType(ty: Type, cg: *CodeGen) Vectorization {
3557	const zcu = cg.zcu;
3558	if (!ty.isVector(zcu)) return .scalar;
3559	return .{ .unrolled = ty.vectorLen(zcu) };
3560	}
3561
3562	/// Given two vectorization methods, compute a "unification": a fallback
3563	/// that works for both, according to the following rules:
3564	/// - Scalars may broadcast
3565	/// - SPIR-V vectorized operations will unroll
3566	/// - Prefer scalar > unrolled
3567	fn unify(a: Vectorization, b: Vectorization) Vectorization {
3568	if (a == .scalar and b == .scalar) return .scalar;
3569	if (a == .unrolled or b == .unrolled) {
3570	if (a == .unrolled and b == .unrolled) assert(a.components() == b.components());
3571	if (a == .unrolled) return .{ .unrolled = a.components() };
3572	return .{ .unrolled = b.components() };
3573	}
3574	unreachable;
3575	}
3576
3577	/// Query the number of components that inputs of this operation have.
3578	/// Note: for broadcasting scalars, this returns the number of elements
3579	/// that the broadcasted vector would have.
3580	fn components(vec: Vectorization) u32 {
3581	return switch (vec) {
3582	.scalar => 1,
3583	.unrolled => |n| n,
3584	};
3585	}
3586
3587	/// Turns `ty` into the result-type of the entire operation.
3588	/// `ty` may be a scalar or vector, it doesn't matter.
3589	fn resultType(vec: Vectorization, cg: *CodeGen, ty: Type) !Type {
3590	const pt = cg.pt;
3591	const zcu = cg.zcu;
3592	const scalar_ty = ty.scalarType(zcu);
3593	return switch (vec) {
3594	.scalar => scalar_ty,
3595	.unrolled => |n| try pt.vectorType(.{ .len = n, .child = scalar_ty.toIntern() }),
3596	};
3597	}
3598
3599	/// Before a temporary can be used, some setup may need to be one. This function implements
3600	/// this setup, and returns a new type that holds the relevant information on how to access
3601	/// elements of the input.
3602	fn prepare(vec: Vectorization, cg: *CodeGen, tmp: Temporary) !PreparedOperand {
3603	const zcu = cg.zcu;
3604	const is_vector = tmp.ty.isVector(zcu);
3605	const value: PreparedOperand.Value = switch (tmp.value) {
3606	.singleton => |id| switch (vec) {
3607	.scalar => blk: {
3608	assert(!is_vector);
3609	break :blk .{ .scalar = id };
3610	},
3611	.unrolled => blk: {
3612	if (is_vector) break :blk .{ .vector_exploded = try tmp.explode(cg) };
3613	break :blk .{ .scalar_broadcast = id };
3614	},
3615	},
3616	.exploded_vector => |range| switch (vec) {
3617	.scalar => unreachable,
3618	.unrolled => |n| blk: {
3619	assert(range.len == n);
3620	break :blk .{ .vector_exploded = range };
3621	},
3622	},
3623	};
3624
3625	return .{
3626	.ty = tmp.ty,
3627	.value = value,
3628	};
3629	}
3630
3631	/// Finalize the results of an operation back into a temporary. `results` is
3632	/// a list of result-ids of the operation.
3633	fn finalize(vec: Vectorization, ty: Type, results: IdRange) Temporary {
3634	assert(vec.components() == results.len);
3635	return .{
3636	.ty = ty,
3637	.value = switch (vec) {
3638	.scalar => .{ .singleton = results.at(0) },
3639	.unrolled => .{ .exploded_vector = results },
3640	},
3641	};
3642	}
3643
3644	/// This struct represents an operand that has gone through some setup, and is
3645	/// ready to be used as part of an operation.
3646	const PreparedOperand = struct {
3647	ty: Type,
3648	value: PreparedOperand.Value,
3649
3650	/// The types of value that a prepared operand can hold internally. Depends
3651	/// on the operation and input value.
3652	const Value = union(enum) {
3653	/// A single scalar value that is used by a scalar operation.
3654	scalar: Id,
3655	/// A single scalar that is broadcasted in an unrolled operation.
3656	scalar_broadcast: Id,
3657	/// A vector represented by a consecutive list of IDs that is used in an unrolled operation.
3658	vector_exploded: IdRange,
3659	};
3660
3661	/// Query the value at a particular index of the operation. Note that
3662	/// the index is *not* the component/lane, but the index of the *operation*.
3663	fn at(op: PreparedOperand, i: usize) Id {
3664	switch (op.value) {
3665	.scalar => |id| {
3666	assert(i == 0);
3667	return id;
3668	},
3669	.scalar_broadcast => |id| return id,
3670	.vector_exploded => |range| return range.at(i),
3671	}
3672	}
3673	};
3674	};
3675
3676	/// A utility function to compute the vectorization style of
3677	/// a list of values. These values may be any of the following:
3678	/// - A `Vectorization` instance
3679	/// - A Type, in which case the vectorization is computed via `Vectorization.fromType`.
3680	/// - A Temporary, in which case the vectorization is computed via `Temporary.vectorization`.
3681	fn vectorization(cg: *CodeGen, args: anytype) Vectorization {
3682	var v: Vectorization = undefined;
3683	assert(args.len >= 1);
3684	inline for (args, 0..) |arg, i| {
3685	const iv: Vectorization = switch (@TypeOf(arg)) {
3686	Vectorization => arg,
3687	Type => Vectorization.fromType(arg, cg),
3688	Temporary => arg.vectorization(cg),
3689	else => @compileError("invalid type"),
3690	};
3691	if (i == 0) {
3692	v = iv;
3693	} else {
3694	v = v.unify(iv);
3695	}
3696	}
3697	return v;
3698	}
3699
3700	/// This function builds an OpSConvert of OpUConvert depending on the
3701	/// signedness of the types.
3702	fn buildConvert(cg: *CodeGen, dst_ty: Type, src: Temporary) !Temporary {
3703	const zcu = cg.zcu;
3704
3705	const v = cg.vectorization(.{ dst_ty, src });
3706	const result_ty = try v.resultType(cg, dst_ty);
3707
3708	const dst_scalar = dst_ty.scalarType(zcu);
3709	const src_scalar = src.ty.scalarType(zcu);
3710	if (dst_scalar.toIntern() == src_scalar.toIntern()) {
3711	return src.pun(result_ty);
3712	}
3713	if (dst_scalar.isInt(zcu) and src_scalar.isInt(zcu)) {
3714	const dst_info = dst_scalar.intInfo(zcu);
3715	const src_info = src_scalar.intInfo(zcu);
3716	if (cg.backingIntBits(dst_info.bits).@"0" == cg.backingIntBits(src_info.bits).@"0" and
3717	dst_info.signedness == src_info.signedness)
3718	{
3719	return src.pun(result_ty);
3720	}
3721	}
3722
3723	const ops = v.components();
3724	const results = cg.allocIds(ops);
3725
3726	const op_result_ty = dst_ty.scalarType(zcu);
3727	const op_result_ty_id = try cg.resolveType(op_result_ty, .direct);
3728
3729	const opcode: Opcode = blk: {
3730	if (dst_ty.scalarType(zcu).isAnyFloat()) break :blk .OpFConvert;
3731	if (dst_ty.scalarType(zcu).isSignedInt(zcu)) break :blk .OpSConvert;
3732	break :blk .OpUConvert;
3733	};
3734
3735	const op_src = try v.prepare(cg, src);
3736
3737	for (0..ops) |i| {
3738	try cg.body.emitRaw(cg.gpa, opcode, 3);
3739	cg.body.writeOperand(Id, op_result_ty_id);
3740	cg.body.writeOperand(Id, results.at(i));
3741	cg.body.writeOperand(Id, op_src.at(i));
3742	}
3743
3744	return v.finalize(result_ty, results);
3745	}
3746
3747	fn buildSelect(cg: *CodeGen, condition: Temporary, lhs: Temporary, rhs: Temporary) !Temporary {
3748	const zcu = cg.zcu;
3749
3750	const v = cg.vectorization(.{ condition, lhs, rhs });
3751	const ops = v.components();
3752	const results = cg.allocIds(ops);
3753
3754	const op_result_ty = lhs.ty.scalarType(zcu);
3755	const op_result_ty_id = try cg.resolveType(op_result_ty, .direct);
3756	const result_ty = try v.resultType(cg, lhs.ty);
3757
3758	assert(condition.ty.scalarType(zcu).zigTypeTag(zcu) == .bool);
3759
3760	const cond = try v.prepare(cg, condition);
3761	const object_1 = try v.prepare(cg, lhs);
3762	const object_2 = try v.prepare(cg, rhs);
3763
3764	for (0..ops) |i| {
3765	try cg.body.emit(cg.gpa, .OpSelect, .{
3766	.id_result_type = op_result_ty_id,
3767	.id_result = results.at(i),
3768	.condition = cond.at(i),
3769	.object_1 = object_1.at(i),
3770	.object_2 = object_2.at(i),
3771	});
3772	}
3773
3774	return v.finalize(result_ty, results);
3775	}
3776
3777	fn buildCmp(cg: *CodeGen, opcode: Opcode, lhs: Temporary, rhs: Temporary) !Temporary {
3778	const v = cg.vectorization(.{ lhs, rhs });
3779	const ops = v.components();
3780	const results = cg.allocIds(ops);
3781
3782	const op_result_ty: Type = .bool;
3783	const op_result_ty_id = try cg.resolveType(op_result_ty, .direct);
3784	const result_ty = try v.resultType(cg, Type.bool);
3785
3786	const op_lhs = try v.prepare(cg, lhs);
3787	const op_rhs = try v.prepare(cg, rhs);
3788
3789	for (0..ops) |i| {
3790	try cg.body.emitRaw(cg.gpa, opcode, 4);
3791	cg.body.writeOperand(Id, op_result_ty_id);
3792	cg.body.writeOperand(Id, results.at(i));
3793	cg.body.writeOperand(Id, op_lhs.at(i));
3794	cg.body.writeOperand(Id, op_rhs.at(i));
3795	}
3796
3797	return v.finalize(result_ty, results);
3798	}
3799
3800	const UnaryOp = enum {
3801	l_not,
3802	bit_not,
3803	i_neg,
3804	f_neg,
3805	i_abs,
3806	f_abs,
3807	clz,
3808	ctz,
3809	floor,
3810	ceil,
3811	trunc,
3812	round,
3813	sqrt,
3814	sin,
3815	cos,
3816	tan,
3817	exp,
3818	exp2,
3819	log,
3820	log2,
3821	log10,
3822
3823	pub fn extInstOpcode(op: UnaryOp, target: *const std.Target) ?u32 {
3824	return switch (target.os.tag) {
3825	.opencl => @backingInt(@as(spec.OpenClOpcode, switch (op) {
3826	.i_abs => .s_abs,
3827	.f_abs => .fabs,
3828	.clz => .clz,
3829	.ctz => .ctz,
3830	.floor => .floor,
3831	.ceil => .ceil,
3832	.trunc => .trunc,
3833	.round => .round,
3834	.sqrt => .sqrt,
3835	.sin => .sin,
3836	.cos => .cos,
3837	.tan => .tan,
3838	.exp => .exp,
3839	.exp2 => .exp2,
3840	.log => .log,
3841	.log2 => .log2,
3842	.log10 => .log10,
3843	else => return null,
3844	})),
3845	// Note: We'll need to check these for floating point accuracy
3846	// Vulkan does not put tight requirements on these, for correction
3847	// we might want to emulate them at some point.
3848	.vulkan, .opengl => @backingInt(@as(spec.GlslOpcode, switch (op) {
3849	.i_abs => .SAbs,
3850	.f_abs => .FAbs,
3851	.floor => .Floor,
3852	.ceil => .Ceil,
3853	.trunc => .Trunc,
3854	.round => .Round,
3855	.sin => .Sin,
3856	.cos => .Cos,
3857	.tan => .Tan,
3858	.sqrt => .Sqrt,
3859	.exp => .Exp,
3860	.exp2 => .Exp2,
3861	.log => .Log,
3862	.log2 => .Log2,
3863	else => return null,
3864	})),
3865	else => unreachable,
3866	};
3867	}
3868	};
3869
3870	fn buildUnary(cg: *CodeGen, op: UnaryOp, operand: Temporary) !Temporary {
3871	const zcu = cg.zcu;
3872	const target = cg.zcu.getTarget();
3873	const v = cg.vectorization(.{operand});
3874	const ops = v.components();
3875	const results = cg.allocIds(ops);
3876	const op_result_ty = operand.ty.scalarType(zcu);
3877	const op_result_ty_id = try cg.resolveType(op_result_ty, .direct);
3878	const result_ty = try v.resultType(cg, operand.ty);
3879	const op_operand = try v.prepare(cg, operand);
3880
3881	if (op.extInstOpcode(target)) |opcode| {
3882	const set = try cg.importExtendedSet();
3883	for (0..ops) |i| {
3884	try cg.body.emit(cg.gpa, .OpExtInst, .{
3885	.id_result_type = op_result_ty_id,
3886	.id_result = results.at(i),
3887	.set = set,
3888	.instruction = .{ .inst = opcode },
3889	.id_ref_4 = &.{op_operand.at(i)},
3890	});
3891	}
3892	} else {
3893	const opcode: Opcode = switch (op) {
3894	.l_not => .OpLogicalNot,
3895	.bit_not => .OpNot,
3896	.i_neg => .OpSNegate,
3897	.f_neg => .OpFNegate,
3898	else => return cg.todo(
3899	"implement unary operation '{s}' for {s} os",
3900	.{ @tagName(op), @tagName(target.os.tag) },
3901	),
3902	};
3903	for (0..ops) |i| {
3904	try cg.body.emitRaw(cg.gpa, opcode, 3);
3905	cg.body.writeOperand(Id, op_result_ty_id);
3906	cg.body.writeOperand(Id, results.at(i));
3907	cg.body.writeOperand(Id, op_operand.at(i));
3908	}
3909	}
3910
3911	return v.finalize(result_ty, results);
3912	}
3913
3914	fn buildBinary(cg: *CodeGen, opcode: Opcode, lhs: Temporary, rhs: Temporary) !Temporary {
3915	const zcu = cg.zcu;
3916
3917	const v = cg.vectorization(.{ lhs, rhs });
3918	const ops = v.components();
3919	const results = cg.allocIds(ops);
3920
3921	const op_result_ty = lhs.ty.scalarType(zcu);
3922	const op_result_ty_id = try cg.resolveType(op_result_ty, .direct);
3923	const result_ty = try v.resultType(cg, lhs.ty);
3924
3925	const op_lhs = try v.prepare(cg, lhs);
3926	const op_rhs = try v.prepare(cg, rhs);
3927
3928	for (0..ops) |i| {
3929	try cg.body.emitRaw(cg.gpa, opcode, 4);
3930	cg.body.writeOperand(Id, op_result_ty_id);
3931	cg.body.writeOperand(Id, results.at(i));
3932	cg.body.writeOperand(Id, op_lhs.at(i));
3933	cg.body.writeOperand(Id, op_rhs.at(i));
3934	}
3935
3936	return v.finalize(result_ty, results);
3937	}
3938
3939	/// This function builds an extended multiplication, either OpSMulExtended or OpUMulExtended on Vulkan,
3940	/// or OpIMul and s_mul_hi or u_mul_hi on OpenCL.
3941	fn buildWideMul(
3942	cg: *CodeGen,
3943	signedness: std.lang.Signedness,
3944	lhs: Temporary,
3945	rhs: Temporary,
3946	) !struct { Temporary, Temporary } {
3947	const pt = cg.pt;
3948	const zcu = cg.zcu;
3949	const comp = zcu.comp;
3950	const gpa = comp.gpa;
3951	const io = comp.io;
3952	const target = cg.zcu.getTarget();
3953	const ip = &zcu.intern_pool;
3954
3955	const v = lhs.vectorization(cg).unify(rhs.vectorization(cg));
3956	const ops = v.components();
3957
3958	const arith_op_ty = lhs.ty.scalarType(zcu);
3959	const arith_op_ty_id = try cg.resolveType(arith_op_ty, .direct);
3960
3961	const lhs_op = try v.prepare(cg, lhs);
3962	const rhs_op = try v.prepare(cg, rhs);
3963
3964	const value_results = cg.allocIds(ops);
3965	const overflow_results = cg.allocIds(ops);
3966
3967	switch (target.os.tag) {
3968	.opencl => {
3969	// Currently, SPIRV-LLVM-Translator based backends cannot deal with OpSMulExtended and
3970	// OpUMulExtended. For these we will use the OpenCL s_mul_hi to compute the high-order bits
3971	// instead.
3972	const set = try cg.importExtendedSet();
3973	const overflow_inst: spec.OpenClOpcode = switch (signedness) {
3974	.signed => .s_mul_hi,
3975	.unsigned => .u_mul_hi,
3976	};
3977
3978	for (0..ops) |i| {
3979	try cg.body.emit(gpa, .OpIMul, .{
3980	.id_result_type = arith_op_ty_id,
3981	.id_result = value_results.at(i),
3982	.operand_1 = lhs_op.at(i),
3983	.operand_2 = rhs_op.at(i),
3984	});
3985
3986	try cg.body.emit(gpa, .OpExtInst, .{
3987	.id_result_type = arith_op_ty_id,
3988	.id_result = overflow_results.at(i),
3989	.set = set,
3990	.instruction = .{ .inst = @backingInt(overflow_inst) },
3991	.id_ref_4 = &.{ lhs_op.at(i), rhs_op.at(i) },
3992	});
3993	}
3994	},
3995	.vulkan, .opengl => {
3996	// Operations return a struct{T, T}
3997	// where T is maybe vectorized.
3998	const op_result_ty: Type = .fromInterned(try ip.getTupleType(gpa, io, pt.tid, .{
3999	.types = &.{ arith_op_ty.toIntern(), arith_op_ty.toIntern() },
4000	.values = &.{ .none, .none },
4001	}));
4002	const op_result_ty_id = try cg.resolveType(op_result_ty, .direct);
4003
4004	const opcode: Opcode = switch (signedness) {
4005	.signed => .OpSMulExtended,
4006	.unsigned => .OpUMulExtended,
4007	};
4008
4009	for (0..ops) |i| {
4010	const op_result = cg.allocId();
4011
4012	try cg.body.emitRaw(gpa, opcode, 4);
4013	cg.body.writeOperand(Id, op_result_ty_id);
4014	cg.body.writeOperand(Id, op_result);
4015	cg.body.writeOperand(Id, lhs_op.at(i));
4016	cg.body.writeOperand(Id, rhs_op.at(i));
4017
4018	// The above operation returns a struct. We might want to expand
4019	// Temporary to deal with the fact that these are structs eventually,
4020	// but for now, take the struct apart and return two separate vectors.
4021
4022	try cg.body.emit(gpa, .OpCompositeExtract, .{
4023	.id_result_type = arith_op_ty_id,
4024	.id_result = value_results.at(i),
4025	.composite = op_result,
4026	.indexes = &.{0},
4027	});
4028
4029	try cg.body.emit(gpa, .OpCompositeExtract, .{
4030	.id_result_type = arith_op_ty_id,
4031	.id_result = overflow_results.at(i),
4032	.composite = op_result,
4033	.indexes = &.{1},
4034	});
4035	}
4036	},
4037	else => unreachable,
4038	}
4039
4040	const result_ty = try v.resultType(cg, lhs.ty);
4041	return .{
4042	v.finalize(result_ty, value_results),
4043	v.finalize(result_ty, overflow_results),
4044	};
4045	}
4046
4047	/// The SPIR-V backend is not yet advanced enough to support the std testing infrastructure.
4048	/// In order to be able to run tests, we "temporarily" lower test kernels into separate entry-
4049	/// points. The test executor will then be able to invoke these to run the tests.
4050	/// Note that tests are lowered according to std.lang.TestFn, which is `fn () anyerror!void`.
4051	/// (anyerror!void has the same layout as anyerror).
4052	/// Each test declaration generates a function like.
4053	/// %anyerror = OpTypeInt 0 16
4054	/// %p_invocation_globals_struct_ty = ...
4055	/// %p_anyerror = OpTypePointer CrossWorkgroup %anyerror
4056	/// %K = OpTypeFunction %void %p_invocation_globals_struct_ty %p_anyerror
4057	///
4058	/// %test = OpFunction %void %K
4059	/// %p_invocation_globals = OpFunctionParameter p_invocation_globals_struct_ty
4060	/// %p_err = OpFunctionParameter %p_anyerror
4061	/// %lbl = OpLabel
4062	/// %result = OpFunctionCall %anyerror %func %p_invocation_globals
4063	/// OpStore %p_err %result
4064	/// OpFunctionEnd
4065	/// TODO is to also write out the error as a function call parameter, and to somehow fetch
4066	/// the name of an error in the text executor.
4067	fn generateTestEntryPoint(
4068	cg: *CodeGen,
4069	name: []const u8,
4070	spv_decl_index: Decl.Index,
4071	test_id: Id,
4072	) !void {
4073	const gpa = cg.gpa;
4074	const zcu = cg.zcu;
4075	const target = cg.zcu.getTarget();
4076
4077	const anyerror_ty_id = try cg.resolveType(.anyerror, .direct);
4078	const ptr_anyerror_ty = try cg.pt.ptrType(.{
4079	.child = .anyerror_type,
4080	.flags = .{ .address_space = .global },
4081	});
4082	const ptr_anyerror_ty_id = try cg.resolveType(ptr_anyerror_ty, .direct);
4083
4084	const kernel_id = cg.declPtr(spv_decl_index).result_id;
4085
4086	const section = &cg.sections.functions;
4087
4088	const p_error_id = cg.allocId();
4089	switch (target.os.tag) {
4090	.opencl, .amdhsa => {
4091	const void_ty_id = try cg.resolveType(.void, .direct);
4092	const kernel_proto_ty_id = try cg.functionType(void_ty_id, &.{ptr_anyerror_ty_id});
4093
4094	try section.emit(gpa, .OpFunction, .{
4095	.id_result_type = try cg.resolveType(.void, .direct),
4096	.id_result = kernel_id,
4097	.function_control = .{},
4098	.function_type = kernel_proto_ty_id,
4099	});
4100
4101	try section.emit(gpa, .OpFunctionParameter, .{
4102	.id_result_type = ptr_anyerror_ty_id,
4103	.id_result = p_error_id,
4104	});
4105
4106	try section.emit(gpa, .OpLabel, .{
4107	.id_result = cg.allocId(),
4108	});
4109	},
4110	.vulkan, .opengl => {
4111	if (cg.error_buffer == null) {
4112	const spv_err_decl_index = try cg.allocDecl(.global);
4113	const err_buf_result_id = cg.declPtr(spv_err_decl_index).result_id;
4114
4115	const buffer_struct_ty_id = cg.allocId();
4116	try cg.sections.globals.emit(gpa, .OpTypeStruct, .{
4117	.id_result = buffer_struct_ty_id,
4118	.id_ref = &.{anyerror_ty_id},
4119	});
4120	try cg.memberDebugName(buffer_struct_ty_id, 0, "error_out");
4121	try cg.decorate(buffer_struct_ty_id, .block);
4122	try cg.decorateMember(buffer_struct_ty_id, 0, .{ .offset = .{ .byte_offset = 0 } });
4123
4124	const ptr_buffer_struct_ty_id = cg.allocId();
4125	try cg.sections.globals.emit(gpa, .OpTypePointer, .{
4126	.id_result = ptr_buffer_struct_ty_id,
4127	.storage_class = cg.storageClass(.global),
4128	.type = buffer_struct_ty_id,
4129	});
4130
4131	try cg.sections.globals.emit(gpa, .OpVariable, .{
4132	.id_result_type = ptr_buffer_struct_ty_id,
4133	.id_result = err_buf_result_id,
4134	.storage_class = cg.storageClass(.global),
4135	});
4136	try cg.decorate(err_buf_result_id, .{ .descriptor_set = .{ .descriptor_set = 0 } });
4137	try cg.decorate(err_buf_result_id, .{ .binding = .{ .binding_point = 0 } });
4138
4139	cg.error_buffer = spv_err_decl_index;
4140	}
4141
4142	const void_ty_id = try cg.resolveType(.void, .direct);
4143	const kernel_proto_ty_id = try cg.functionType(void_ty_id, &.{});
4144	try section.emit(gpa, .OpFunction, .{
4145	.id_result_type = try cg.resolveType(.void, .direct),
4146	.id_result = kernel_id,
4147	.function_control = .{},
4148	.function_type = kernel_proto_ty_id,
4149	});
4150	try section.emit(gpa, .OpLabel, .{
4151	.id_result = cg.allocId(),
4152	});
4153
4154	const spv_err_decl_index = cg.error_buffer.?;
4155	const buffer_id = cg.declPtr(spv_err_decl_index).result_id;
4156	try cg.decl_deps.append(gpa, spv_err_decl_index);
4157
4158	const zero_id = try cg.constInt(.u32, 0);
4159	try section.emit(gpa, .OpInBoundsAccessChain, .{
4160	.id_result_type = ptr_anyerror_ty_id,
4161	.id_result = p_error_id,
4162	.base = buffer_id,
4163	.indexes = &.{zero_id},
4164	});
4165	},
4166	else => unreachable,
4167	}
4168
4169	const error_id = cg.allocId();
4170	try section.emit(gpa, .OpFunctionCall, .{
4171	.id_result_type = anyerror_ty_id,
4172	.id_result = error_id,
4173	.function = test_id,
4174	});
4175	// Note: Convert to direct not required.
4176	try section.emit(gpa, .OpStore, .{
4177	.pointer = p_error_id,
4178	.object = error_id,
4179	.memory_access = .{
4180	.aligned = .{ .literal_integer = @intCast(Type.abiAlignment(.anyerror, zcu).toByteUnits().?) },
4181	},
4182	});
4183	try section.emit(gpa, .OpReturn, {});
4184	try section.emit(gpa, .OpFunctionEnd, {});
4185
4186	// Just generate a quick other name because the intel runtime crashes when the entry-
4187	// point name is the same as a different OpName.
4188	const test_name = try std.fmt.allocPrint(cg.arena, "test {s}", .{name});
4189
4190	const ep_gop = try cg.entry_points.getOrPut(cg.gpa, cg.declPtr(spv_decl_index).result_id);
4191	ep_gop.value_ptr.* = .{
4192	.decl_index = spv_decl_index,
4193	.name = test_name,
4194	.cc = .{ .spirv_kernel = .{ .x = 1, .y = 1, .z = 1 } },
4195	};
4196	}
4197
4198	fn intFromBool(cg: *CodeGen, value: Temporary, result_ty: Type) !Temporary {
4199	const zero_id = try cg.constInt(result_ty, 0);
4200	const one_id = try cg.constInt(result_ty, 1);
4201
4202	return try cg.buildSelect(
4203	value,
4204	Temporary.init(result_ty, one_id),
4205	Temporary.init(result_ty, zero_id),
4206	);
4207	}
4208
4209	/// Convert representation from indirect (in memory) to direct (in 'register')
4210	/// This converts the argument type from resolveType(ty, .indirect) to resolveType(ty, .direct).
4211	fn convertToDirect(cg: *CodeGen, ty: Type, operand_id: Id) !Id {
4212	const pt = cg.pt;
4213	const zcu = cg.zcu;
4214	switch (ty.scalarType(zcu).zigTypeTag(zcu)) {
4215	.bool => {
4216	const false_id = try cg.constBool(false, .indirect);
4217	const operand_ty = blk: {
4218	if (!ty.isVector(zcu)) break :blk Type.u1;
4219	break :blk try pt.vectorType(.{
4220	.len = ty.vectorLen(zcu),
4221	.child = .u1_type,
4222	});
4223	};
4224
4225	const result = try cg.buildCmp(
4226	.OpINotEqual,
4227	Temporary.init(operand_ty, operand_id),
4228	Temporary.init(.u1, false_id),
4229	);
4230	return try result.materialize(cg);
4231	},
4232	else => return operand_id,
4233	}
4234	}
4235
4236	/// Convert representation from direct (in 'register) to direct (in memory)
4237	/// This converts the argument type from resolveType(ty, .direct) to resolveType(ty, .indirect).
4238	fn convertToIndirect(cg: *CodeGen, ty: Type, operand_id: Id) !Id {
4239	const zcu = cg.zcu;
4240	switch (ty.scalarType(zcu).zigTypeTag(zcu)) {
4241	.bool => {
4242	const result = try cg.intFromBool(.init(ty, operand_id), .u1);
4243	return try result.materialize(cg);
4244	},
4245	else => return operand_id,
4246	}
4247	}
4248
4249	fn extractField(cg: *CodeGen, result_ty: Type, object: Id, field: u32) !Id {
4250	const result_ty_id = try cg.resolveType(result_ty, .indirect);
4251	const result_id = cg.allocId();
4252	const indexes = [_]u32{field};
4253	try cg.body.emit(cg.gpa, .OpCompositeExtract, .{
4254	.id_result_type = result_ty_id,
4255	.id_result = result_id,
4256	.composite = object,
4257	.indexes = &indexes,
4258	});
4259	// Convert bools; direct structs have their field types as indirect values.
4260	return try cg.convertToDirect(result_ty, result_id);
4261	}
4262
4263	fn extractVectorComponent(cg: *CodeGen, result_ty: Type, vector_id: Id, field: u32) !Id {
4264	const result_ty_id = try cg.resolveType(result_ty, .direct);
4265	const result_id = cg.allocId();
4266	const indexes = [_]u32{field};
4267	try cg.body.emit(cg.gpa, .OpCompositeExtract, .{
4268	.id_result_type = result_ty_id,
4269	.id_result = result_id,
4270	.composite = vector_id,
4271	.indexes = &indexes,
4272	});
4273	// Vector components are already stored in direct representation.
4274	return result_id;
4275	}
4276
4277	const MemoryOptions = struct {
4278	is_volatile: bool = false,
4279	ptr_address_space: std.lang.AddressSpace = .generic,
4280	};
4281
4282	/// Returns true if a pointee at address space must use the
4283	/// layout-decorated variant rather than the bare type.
4284	fn needsLayout(cg: *CodeGen, as: std.lang.AddressSpace, pointee_ty: Type) bool {
4285	const target = cg.zcu.getTarget();
4286	if (target.os.tag != .vulkan and target.os.tag != .opengl) return false;
4287	return switch (as) {
4288	.uniform,
4289	.push_constant,
4290	.storage_buffer,
4291	.physical_storage_buffer,
4292	=> switch (pointee_ty.zigTypeTag(cg.zcu)) {
4293	.@"struct", .@"union", .array => true,
4294	.spirv => pointee_ty.isSpirvRuntimeArray(cg.zcu),
4295	else => false,
4296	},
4297	else => false,
4298	};
4299	}
4300
4301	fn pointeeType(cg: *CodeGen, as: std.lang.AddressSpace, ty: Type, is_block_root: bool) !Id {
4302	return if (cg.needsLayout(as, ty))
4303	cg.layoutType(ty, is_block_root)
4304	else
4305	cg.resolveType(ty, .indirect);
4306	}
4307
4308	fn convertLayout(cg: *CodeGen, dst_ty_id: Id, src_id: Id, src_ty_id: Id) !Id {
4309	if (dst_ty_id == src_ty_id) return src_id;
4310	const id = cg.allocId();
4311	try cg.body.emit(cg.gpa, .OpCopyLogical, .{
4312	.id_result_type = dst_ty_id,
4313	.id_result = id,
4314	.operand = src_id,
4315	});
4316	return id;
4317	}
4318
4319	fn load(cg: *CodeGen, value_ty: Type, ptr_id: Id, options: MemoryOptions) !Id {
4320	const zcu = cg.zcu;
4321	const alignment: u32 = @intCast(value_ty.abiAlignment(zcu).toByteUnits().?);
4322	const bare_ty_id = try cg.resolveType(value_ty, .indirect);
4323	const load_ty_id = if (cg.needsLayout(options.ptr_address_space, value_ty))
4324	try cg.layoutType(value_ty, cg.block_var_ids.contains(ptr_id))
4325	else
4326	bare_ty_id;
4327	const loaded_id = cg.allocId();
4328	try cg.body.emit(cg.gpa, .OpLoad, .{
4329	.id_result_type = load_ty_id,
4330	.id_result = loaded_id,
4331	.pointer = ptr_id,
4332	.memory_access = .{
4333	.@"volatile" = options.is_volatile,
4334	.aligned = .{ .literal_integer = alignment },
4335	},
4336	});
4337	const result_id = try cg.convertLayout(bare_ty_id, loaded_id, load_ty_id);
4338	return try cg.convertToDirect(value_ty, result_id);
4339	}
4340
4341	fn store(cg: *CodeGen, value_ty: Type, ptr_id: Id, value_id: Id, options: MemoryOptions) !void {
4342	const zcu = cg.zcu;
4343	const alignment: u32 = @intCast(value_ty.abiAlignment(zcu).toByteUnits().?);
4344	const bare_value_id = try cg.convertToIndirect(value_ty, value_id);
4345	const bare_ty_id = try cg.resolveType(value_ty, .indirect);
4346	const store_ty_id = if (cg.needsLayout(options.ptr_address_space, value_ty))
4347	try cg.layoutType(value_ty, cg.block_var_ids.contains(ptr_id))
4348	else
4349	bare_ty_id;
4350	const object_id = try cg.convertLayout(store_ty_id, bare_value_id, bare_ty_id);
4351	try cg.body.emit(cg.gpa, .OpStore, .{
4352	.pointer = ptr_id,
4353	.object = object_id,
4354	.memory_access = .{
4355	.@"volatile" = options.is_volatile,
4356	.aligned = .{ .literal_integer = alignment },
4357	},
4358	});
4359	}
4360
4361	fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) !void {
4362	for (body) |inst| {
4363	try cg.genInst(inst);
4364	}
4365	}
4366
4367	fn genInst(cg: *CodeGen, inst: Air.Inst.Index) Error!void {
4368	const gpa = cg.gpa;
4369	const zcu = cg.zcu;
4370	const ip = &zcu.intern_pool;
4371	if (cg.liveness.isUnused(inst) and !cg.air.mustLower(inst, ip))
4372	return;
4373
4374	const air_tags = cg.air.instructions.items(.tag);
4375	const maybe_result_id: ?Id = switch (air_tags[@backingInt(inst)]) {
4376	// zig fmt: off
4377	.add, .add_wrap, .add_optimized => try cg.airArithOp(inst, .OpFAdd, .OpIAdd, .OpIAdd),
4378	.sub, .sub_wrap, .sub_optimized => try cg.airArithOp(inst, .OpFSub, .OpISub, .OpISub),
4379	.mul, .mul_wrap, .mul_optimized => try cg.airArithOp(inst, .OpFMul, .OpIMul, .OpIMul),
4380
4381	.sqrt => try cg.airUnOpSimple(inst, .sqrt),
4382	.sin => try cg.airUnOpSimple(inst, .sin),
4383	.cos => try cg.airUnOpSimple(inst, .cos),
4384	.tan => try cg.airUnOpSimple(inst, .tan),
4385	.exp => try cg.airUnOpSimple(inst, .exp),
4386	.exp2 => try cg.airUnOpSimple(inst, .exp2),
4387	.log => try cg.airUnOpSimple(inst, .log),
4388	.log2 => try cg.airUnOpSimple(inst, .log2),
4389	.log10 => try cg.airUnOpSimple(inst, .log10),
4390	.abs => try cg.airAbs(inst),
4391	.floor => try cg.airUnOpSimple(inst, .floor),
4392	.ceil => try cg.airUnOpSimple(inst, .ceil),
4393	.round => try cg.airUnOpSimple(inst, .round),
4394	.trunc_float => try cg.airUnOpSimple(inst, .trunc),
4395	.neg, .neg_optimized => try cg.airUnOpSimple(inst, .f_neg),
4396
4397	.div_float, .div_float_optimized => try cg.airArithOp(inst, .OpFDiv, .OpSDiv, .OpUDiv),
4398	.div_floor, .div_floor_optimized => try cg.airDivFloor(inst),
4399	.div_trunc, .div_trunc_optimized => try cg.airDivTrunc(inst),
4400
4401	.rem, .rem_optimized => try cg.airArithOp(inst, .OpFRem, .OpSRem, .OpUMod),
4402	.mod, .mod_optimized => try cg.airArithOp(inst, .OpFMod, .OpSMod, .OpUMod),
4403
4404	.add_with_overflow => try cg.airAddSubOverflow(inst, .OpIAdd, .OpULessThan, .OpSLessThan),
4405	.sub_with_overflow => try cg.airAddSubOverflow(inst, .OpISub, .OpUGreaterThan, .OpSGreaterThan),
4406	.mul_with_overflow => try cg.airMulOverflow(inst),
4407	.shl_with_overflow => try cg.airShlOverflow(inst),
4408
4409	.mul_add => try cg.airMulAdd(inst),
4410
4411	.ctz => try cg.airClzCtz(inst, .ctz),
4412	.clz => try cg.airClzCtz(inst, .clz),
4413
4414	.select => try cg.airSelect(inst),
4415
4416	.splat => try cg.airSplat(inst),
4417	.reduce, .reduce_optimized => try cg.airReduce(inst),
4418	.shuffle_one => try cg.airShuffleOne(inst),
4419	.shuffle_two => try cg.airShuffleTwo(inst),
4420
4421	.ptr_add => try cg.airPtrAdd(inst),
4422	.ptr_sub => try cg.airPtrSub(inst),
4423
4424	.bit_and => try cg.airBitwiseOp(inst, .bit_and),
4425	.bit_or => try cg.airBitwiseOp(inst, .bit_or),
4426	.xor => try cg.airBitwiseOp(inst, .xor),
4427
4428	.shl, .shl_exact => try cg.airShift(inst, .OpShiftLeftLogical, .OpShiftLeftLogical),
4429	.shr, .shr_exact => try cg.airShift(inst, .OpShiftRightLogical, .OpShiftRightArithmetic),
4430
4431	.min => try cg.airMinMax(inst, .min),
4432	.max => try cg.airMinMax(inst, .max),
4433
4434	.bit_cast => try cg.airBitCast(inst),
4435	.ptr_cast => try cg.airBitCast(inst),
4436	.ptr_from_int => try cg.airBitCast(inst),
4437	.int_from_ptr => try cg.airBitCast(inst),
4438	.error_cast => try cg.airBitCast(inst),
4439	.error_from_int => try cg.airBitCast(inst),
4440	.int_from_error => try cg.airBitCast(inst),
4441	.union_from_enum => try cg.airBitCast(inst),
4442	.int_cast, .trunc => try cg.airIntCast(inst),
4443	.float_from_int => try cg.airFloatFromInt(inst),
4444	.int_from_float => try cg.airIntFromFloat(inst),
4445	.fpext, .fptrunc => try cg.airFloatCast(inst),
4446	.not => try cg.airNot(inst),
4447
4448	.array_to_slice => try cg.airArrayToSlice(inst),
4449	.array_to_vector => unreachable, // legalize .expand_array_to_vector
4450	.slice => try cg.airSlice(inst),
4451	.aggregate_init => try cg.airAggregateInit(inst),
4452	.memcpy => return cg.airMemcpy(inst),
4453	.memmove => return cg.airMemmove(inst),
4454
4455	.slice_ptr => try cg.airSliceField(inst, 0),
4456	.slice_len => try cg.airSliceField(inst, 1),
4457	.ptr_slice_ptr_ptr => try cg.airStructFieldPtrIndex(inst, 0),
4458	.ptr_slice_len_ptr => try cg.airStructFieldPtrIndex(inst, 1),
4459	.spirv_runtime_array_len => try cg.airSpirvRuntimeArrayLen(inst),
4460	.slice_elem_ptr => try cg.airSliceElemPtr(inst),
4461	.slice_elem_val => try cg.airSliceElemVal(inst),
4462	.ptr_elem_ptr => try cg.airPtrElemPtr(inst),
4463	.ptr_elem_val => try cg.airPtrElemVal(inst),
4464	.array_elem_val => try cg.airArrayElemVal(inst),
4465
4466	.set_union_tag => return cg.airSetUnionTag(inst),
4467	.get_union_tag => try cg.airGetUnionTag(inst),
4468	.union_init => try cg.airUnionInit(inst),
4469
4470	.agg_field_val => try cg.airAggFieldVal(inst),
4471	.field_parent_ptr => try cg.airFieldParentPtr(inst),
4472
4473	.struct_field_ptr => try cg.airStructFieldPtr(inst),
4474
4475	.struct_field_ptr_index_0 => try cg.airStructFieldPtrIndex(inst, 0),
4476	.struct_field_ptr_index_1 => try cg.airStructFieldPtrIndex(inst, 1),
4477	.struct_field_ptr_index_2 => try cg.airStructFieldPtrIndex(inst, 2),
4478	.struct_field_ptr_index_3 => try cg.airStructFieldPtrIndex(inst, 3),
4479
4480	.cmp_eq => try cg.airCmp(inst, .eq),
4481	.cmp_neq => try cg.airCmp(inst, .neq),
4482	.cmp_gt => try cg.airCmp(inst, .gt),
4483	.cmp_gte => try cg.airCmp(inst, .gte),
4484	.cmp_lt => try cg.airCmp(inst, .lt),
4485	.cmp_lte => try cg.airCmp(inst, .lte),
4486	.cmp_vector => try cg.airVectorCmp(inst),
4487
4488	.arg => cg.airArg(),
4489	.alloc => try cg.airAlloc(inst),
4490	// TODO: We probably need to have a special implementation of this for the C abi.
4491	.ret_ptr => try cg.airAlloc(inst),
4492	.block => try cg.airBlock(inst),
4493
4494	.load => try cg.airLoad(inst),
4495	.store, .store_safe => return cg.airStore(inst),
4496
4497	.br => return cg.airBr(inst),
4498	// For now just ignore this instruction. This effectively falls back on the old implementation,
4499	// this doesn't change anything for us.
4500	.repeat => return,
4501	.breakpoint => return,
4502	.cond_br => return cg.airCondBr(inst),
4503	.loop => return cg.airLoop(inst),
4504	.ret => return cg.airRet(inst),
4505	.ret_safe => return cg.airRet(inst), // TODO
4506	.ret_load => return cg.airRetLoad(inst),
4507	.@"try" => try cg.airTry(inst),
4508	.switch_br => return cg.airSwitchBr(inst),
4509	.loop_switch_br => return cg.airLoopSwitchBr(inst),
4510	.switch_dispatch => return cg.airSwitchDispatch(inst),
4511	.unreach, .trap => return cg.airUnreach(),
4512
4513	.dbg_empty_stmt => return,
4514	.dbg_stmt => return cg.airDbgStmt(inst),
4515	.dbg_inline_block => try cg.airDbgInlineBlock(inst),
4516	.dbg_var_ptr, .dbg_var_val, .dbg_arg_inline => return cg.airDbgVar(inst),
4517
4518	.unwrap_errunion_err => try cg.airErrUnionErr(inst),
4519	.unwrap_errunion_payload => try cg.airErrUnionPayload(inst),
4520	.wrap_errunion_err => try cg.airWrapErrUnionErr(inst),
4521	.wrap_errunion_payload => try cg.airWrapErrUnionPayload(inst),
4522
4523	.is_null => try cg.airIsNull(inst, false, .is_null),
4524	.is_non_null => try cg.airIsNull(inst, false, .is_non_null),
4525	.is_null_ptr => try cg.airIsNull(inst, true, .is_null),
4526	.is_non_null_ptr => try cg.airIsNull(inst, true, .is_non_null),
4527	.is_err => try cg.airIsErr(inst, .is_err),
4528	.is_non_err => try cg.airIsErr(inst, .is_non_err),
4529
4530	.optional_payload => try cg.airUnwrapOptional(inst),
4531	.optional_payload_ptr => try cg.airUnwrapOptionalPtr(inst),
4532	.optional_payload_ptr_set => try cg.airSetOptionalPtr(inst),
4533	.wrap_optional => try cg.airWrapOptional(inst),
4534
4535	.assembly => try cg.airAssembly(inst),
4536
4537	.call => try cg.airCall(inst, .auto),
4538	.call_always_tail => try cg.airCall(inst, .always_tail),
4539	.call_never_tail => try cg.airCall(inst, .never_tail),
4540	.call_never_inline => try cg.airCall(inst, .never_inline),
4541
4542	.work_item_id => try cg.airWorkItemId(inst),
4543	.work_group_size => try cg.airWorkGroupSize(inst),
4544	.work_group_id => try cg.airWorkGroupId(inst),
4545
4546	// zig fmt: on
4547
4548	else => |tag| return cg.todo("implement AIR tag {s}", .{@tagName(tag)}),
4549	};
4550
4551	const result_id = maybe_result_id orelse return;
4552	try cg.inst_results.putNoClobber(gpa, inst, result_id);
4553	}
4554
4555	fn airBinOpSimple(cg: *CodeGen, inst: Air.Inst.Index, op: Opcode) !?Id {
4556	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
4557	const lhs = try cg.temporary(bin_op.lhs);
4558	const rhs = try cg.temporary(bin_op.rhs);
4559
4560	const result = try cg.buildBinary(op, lhs, rhs);
4561	return try result.materialize(cg);
4562	}
4563
4564	const BitwiseOp = enum { bit_and, bit_or, xor };
4565
4566	fn airBitwiseOp(cg: *CodeGen, inst: Air.Inst.Index, op: BitwiseOp) !?Id {
4567	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
4568	const lhs = try cg.temporary(bin_op.lhs);
4569	const rhs = try cg.temporary(bin_op.rhs);
4570	const info = cg.arithmeticTypeInfo(lhs.ty);
4571
4572	// SPIR-V requires logical opcodes for booleans, bitwise opcodes for integers.
4573	const opcode: Opcode = switch (info.class) {
4574	.bool => switch (op) {
4575	.bit_and => .OpLogicalAnd,
4576	.bit_or => .OpLogicalOr,
4577	.xor => .OpLogicalNotEqual,
4578	},
4579	.integer, .strange_integer => switch (op) {
4580	.bit_and => .OpBitwiseAnd,
4581	.bit_or => .OpBitwiseOr,
4582	.xor => .OpBitwiseXor,
4583	},
4584	.float => unreachable,
4585	.composite_integer => {
4586	const spv_opcode: Opcode = switch (op) {
4587	.bit_and => .OpBitwiseAnd,
4588	.bit_or => .OpBitwiseOr,
4589	.xor => .OpBitwiseXor,
4590	};
4591	const lhs_id = try lhs.materialize(cg);
4592	const rhs_id = try rhs.materialize(cg);
4593	const scratch_top = cg.id_scratch.items.len;
4594	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
4595	const ci_lhs = try CompositeInt.init(cg, lhs_id, info);
4596	const ci_rhs = try CompositeInt.init(cg, rhs_id, info);
4597	const ci_result = try ci_lhs.bitwiseOp(ci_rhs, spv_opcode);
4598	return try ci_result.materialize(lhs.ty);
4599	},
4600	};
4601
4602	const result = try cg.buildBinary(opcode, lhs, rhs);
4603	return try result.materialize(cg);
4604	}
4605
4606	fn airShift(cg: *CodeGen, inst: Air.Inst.Index, unsigned: Opcode, signed: Opcode) !?Id {
4607	const zcu = cg.zcu;
4608	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
4609
4610	const base = try cg.temporary(bin_op.lhs);
4611	const shift = try cg.temporary(bin_op.rhs);
4612
4613	const result_ty = cg.typeOfIndex(inst);
4614
4615	const info = cg.arithmeticTypeInfo(result_ty);
4616	switch (info.class) {
4617	.composite_integer => {
4618	const shift_info = cg.arithmeticTypeInfo(shift.ty);
4619	const limb_ty = cg.limbType();
4620	const shift_amt_id = switch (shift_info.class) {
4621	.composite_integer => blk: {
4622	const shift_id = try shift.materialize(cg);
4623	const limb_ty_id = try cg.limbTypeId();
4624	const result_id = cg.allocId();
4625	try cg.body.emit(cg.gpa, .OpCompositeExtract, .{
4626	.id_result_type = limb_ty_id,
4627	.id_result = result_id,
4628	.composite = shift_id,
4629	.indexes = &.{@as(u32, 0)},
4630	});
4631	break :blk result_id;
4632	},
4633	else => blk: {
4634	const converted = try cg.buildConvert(limb_ty, shift);
4635	break :blk try converted.materialize(cg);
4636	},
4637	};
4638	const base_id = try base.materialize(cg);
4639	const scratch_top = cg.id_scratch.items.len;
4640	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
4641	const ci = try CompositeInt.init(cg, base_id, info);
4642	const ci_result = if (unsigned == .OpShiftLeftLogical)
4643	try ci.shl(shift_amt_id)
4644	else switch (info.signedness) {
4645	.unsigned => try ci.shr(shift_amt_id, false),
4646	.signed => try ci.shr(shift_amt_id, true),
4647	};
4648	const normalized = try ci_result.normalize();
4649	return try normalized.materialize(result_ty);
4650	},
4651	.integer, .strange_integer => {},
4652	.float, .bool => unreachable,
4653	}
4654
4655	// Sometimes Zig doesn't make both of the arguments the same types here. SPIR-V expects that,
4656	// so just manually upcast it if required.
4657
4658	// Note: The sign may differ here between the shift and the base type, in case
4659	// of an arithmetic right shift. SPIR-V still expects the same type,
4660	// so in that case we have to cast convert to signed.
4661	const casted_shift = try cg.buildConvert(base.ty.scalarType(zcu), shift);
4662
4663	const shifted = switch (info.signedness) {
4664	.unsigned => try cg.buildBinary(unsigned, base, casted_shift),
4665	.signed => try cg.buildBinary(signed, base, casted_shift),
4666	};
4667
4668	const result = try cg.normalize(shifted, info);
4669	return try result.materialize(cg);
4670	}
4671
4672	const MinMax = enum {
4673	min,
4674	max,
4675
4676	pub fn extInstOpcode(
4677	op: MinMax,
4678	target: *const std.Target,
4679	info: ArithmeticTypeInfo,
4680	) u32 {
4681	return switch (target.os.tag) {
4682	.opencl => @backingInt(@as(spec.OpenClOpcode, switch (info.class) {
4683	.float => switch (op) {
4684	.min => .fmin,
4685	.max => .fmax,
4686	},
4687	.integer, .strange_integer, .composite_integer => switch (info.signedness) {
4688	.signed => switch (op) {
4689	.min => .s_min,
4690	.max => .s_max,
4691	},
4692	.unsigned => switch (op) {
4693	.min => .u_min,
4694	.max => .u_max,
4695	},
4696	},
4697	.bool => unreachable,
4698	})),
4699	.vulkan, .opengl => @backingInt(@as(spec.GlslOpcode, switch (info.class) {
4700	.float => switch (op) {
4701	.min => .FMin,
4702	.max => .FMax,
4703	},
4704	.integer, .strange_integer, .composite_integer => switch (info.signedness) {
4705	.signed => switch (op) {
4706	.min => .SMin,
4707	.max => .SMax,
4708	},
4709	.unsigned => switch (op) {
4710	.min => .UMin,
4711	.max => .UMax,
4712	},
4713	},
4714	.bool => unreachable,
4715	})),
4716	else => unreachable,
4717	};
4718	}
4719	};
4720
4721	fn airMinMax(cg: *CodeGen, inst: Air.Inst.Index, op: MinMax) !?Id {
4722	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
4723
4724	const lhs = try cg.temporary(bin_op.lhs);
4725	const rhs = try cg.temporary(bin_op.rhs);
4726
4727	const result = try cg.minMax(lhs, rhs, op);
4728	return try result.materialize(cg);
4729	}
4730
4731	fn minMax(cg: *CodeGen, lhs: Temporary, rhs: Temporary, op: MinMax) !Temporary {
4732	const zcu = cg.zcu;
4733	const target = zcu.getTarget();
4734	const info = cg.arithmeticTypeInfo(lhs.ty);
4735
4736	const v = cg.vectorization(.{ lhs, rhs });
4737	const ops = v.components();
4738	const results = cg.allocIds(ops);
4739
4740	const op_result_ty = lhs.ty.scalarType(zcu);
4741	const op_result_ty_id = try cg.resolveType(op_result_ty, .direct);
4742	const result_ty = try v.resultType(cg, lhs.ty);
4743
4744	const op_lhs = try v.prepare(cg, lhs);
4745	const op_rhs = try v.prepare(cg, rhs);
4746
4747	const set = try cg.importExtendedSet();
4748	const opcode = op.extInstOpcode(target, info);
4749	for (0..ops) |i| {
4750	try cg.body.emit(cg.gpa, .OpExtInst, .{
4751	.id_result_type = op_result_ty_id,
4752	.id_result = results.at(i),
4753	.set = set,
4754	.instruction = .{ .inst = opcode },
4755	.id_ref_4 = &.{ op_lhs.at(i), op_rhs.at(i) },
4756	});
4757	}
4758
4759	return v.finalize(result_ty, results);
4760	}
4761
4762	/// This function normalizes values to a canonical representation
4763	/// after some arithmetic operation. This mostly consists of wrapping
4764	/// behavior for strange integers:
4765	/// - Unsigned integers are bitwise masked with a mask that only passes
4766	/// the valid bits through.
4767	/// - Signed integers are also sign extended if they are negative.
4768	/// All other values are returned unmodified (this makes strange integer
4769	/// wrapping easier to use in generic operations).
4770	fn normalize(cg: *CodeGen, value: Temporary, info: ArithmeticTypeInfo) !Temporary {
4771	const zcu = cg.zcu;
4772	const ty = value.ty;
4773	switch (info.class) {
4774	.integer, .bool, .float => return value,
4775	.composite_integer => {
4776	if (info.bits == info.backing_bits) return value;
4777	const val_id = try value.materialize(cg);
4778	const scratch_top = cg.id_scratch.items.len;
4779	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
4780	const ci = try CompositeInt.init(cg, val_id, info);
4781	const normalized = try ci.normalize();
4782	return .init(ty, try normalized.materialize(ty));
4783	},
4784	.strange_integer => switch (info.signedness) {
4785	.unsigned => {
4786	const mask_value = @as(u64, std.math.maxInt(u64)) >> @as(u6, @intCast(64 - info.bits));
4787	const mask_id = try cg.constInt(ty.scalarType(zcu), mask_value);
4788	return try cg.buildBinary(.OpBitwiseAnd, value, Temporary.init(ty.scalarType(zcu), mask_id));
4789	},
4790	.signed => {
4791	// Shift left and right so that we can copy the sight bit that way.
4792	const shift_amt_id = try cg.constInt(ty.scalarType(zcu), info.backing_bits - info.bits);
4793	const shift_amt: Temporary = .init(ty.scalarType(zcu), shift_amt_id);
4794	const left = try cg.buildBinary(.OpShiftLeftLogical, value, shift_amt);
4795	return try cg.buildBinary(.OpShiftRightArithmetic, left, shift_amt);
4796	},
4797	},
4798	}
4799	}
4800
4801	fn airDivFloor(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
4802	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
4803
4804	const lhs = try cg.temporary(bin_op.lhs);
4805	const rhs = try cg.temporary(bin_op.rhs);
4806
4807	const info = cg.arithmeticTypeInfo(lhs.ty);
4808	switch (info.class) {
4809	.composite_integer => return cg.todo("div_floor for composite integers", .{}),
4810	.integer, .strange_integer => {
4811	switch (info.signedness) {
4812	.unsigned => {
4813	const result = try cg.buildBinary(.OpUDiv, lhs, rhs);
4814	return try result.materialize(cg);
4815	},
4816	.signed => {},
4817	}
4818
4819	// For signed integers:
4820	// (a / b) - (a % b != 0 && a < 0 != b < 0);
4821	// There shouldn't be any overflow issues.
4822
4823	const div = try cg.buildBinary(.OpSDiv, lhs, rhs);
4824	const rem = try cg.buildBinary(.OpSRem, lhs, rhs);
4825	const zero: Temporary = .init(lhs.ty, try cg.constInt(lhs.ty, 0));
4826	const rem_non_zero = try cg.buildCmp(.OpINotEqual, rem, zero);
4827	const lhs_rhs_xor = try cg.buildBinary(.OpBitwiseXor, lhs, rhs);
4828	const signs_differ = try cg.buildCmp(.OpSLessThan, lhs_rhs_xor, zero);
4829	const adjust = try cg.buildBinary(.OpLogicalAnd, rem_non_zero, signs_differ);
4830	const result = try cg.buildBinary(.OpISub, div, try cg.intFromBool(adjust, div.ty));
4831	return try result.materialize(cg);
4832	},
4833	.float => {
4834	const div = try cg.buildBinary(.OpFDiv, lhs, rhs);
4835	const result = try cg.buildUnary(.floor, div);
4836	return try result.materialize(cg);
4837	},
4838	.bool => unreachable,
4839	}
4840	}
4841
4842	fn airDivTrunc(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
4843	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
4844	const lhs = try cg.temporary(bin_op.lhs);
4845	const rhs = try cg.temporary(bin_op.rhs);
4846	const info = cg.arithmeticTypeInfo(lhs.ty);
4847	switch (info.class) {
4848	.composite_integer => return cg.todo("div_trunc for composite integers", .{}),
4849	.integer, .strange_integer => switch (info.signedness) {
4850	.unsigned => {
4851	const result = try cg.buildBinary(.OpUDiv, lhs, rhs);
4852	return try result.materialize(cg);
4853	},
4854	.signed => {
4855	const result = try cg.buildBinary(.OpSDiv, lhs, rhs);
4856	return try result.materialize(cg);
4857	},
4858	},
4859	.float => {
4860	const div = try cg.buildBinary(.OpFDiv, lhs, rhs);
4861	const result = try cg.buildUnary(.trunc, div);
4862	return try result.materialize(cg);
4863	},
4864	.bool => unreachable,
4865	}
4866	}
4867
4868	fn airUnOpSimple(cg: *CodeGen, inst: Air.Inst.Index, op: UnaryOp) !?Id {
4869	const un_op = cg.air.instructions.items(.data)[@backingInt(inst)].un_op;
4870	const operand = try cg.temporary(un_op);
4871	const result = try cg.buildUnary(op, operand);
4872	return try result.materialize(cg);
4873	}
4874
4875	fn airArithOp(
4876	cg: *CodeGen,
4877	inst: Air.Inst.Index,
4878	comptime fop: Opcode,
4879	comptime sop: Opcode,
4880	comptime uop: Opcode,
4881	) !?Id {
4882	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
4883	const lhs = try cg.temporary(bin_op.lhs);
4884	const rhs = try cg.temporary(bin_op.rhs);
4885	const info = cg.arithmeticTypeInfo(lhs.ty);
4886	const result = switch (info.class) {
4887	.composite_integer => res: {
4888	const lhs_id = try lhs.materialize(cg);
4889	const rhs_id = try rhs.materialize(cg);
4890	const scratch_top = cg.id_scratch.items.len;
4891	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
4892	const ci_lhs = try CompositeInt.init(cg, lhs_id, info);
4893	const ci_rhs = try CompositeInt.init(cg, rhs_id, info);
4894	const ci_result = switch (uop) {
4895	.OpIAdd => try ci_lhs.addSub(ci_rhs, true),
4896	.OpISub => try ci_lhs.addSub(ci_rhs, false),
4897	.OpIMul => CompositeInt.fromLimbs(cg, try ci_lhs.mul(ci_rhs, false), info),
4898	else => return cg.todo("arith op for composite integers", .{}),
4899	};
4900	const normalized = try ci_result.normalize();
4901	break :res Temporary.init(lhs.ty, try normalized.materialize(lhs.ty));
4902	},
4903	.integer, .strange_integer => res: {
4904	const raw = switch (info.signedness) {
4905	.signed => try cg.buildBinary(sop, lhs, rhs),
4906	.unsigned => try cg.buildBinary(uop, lhs, rhs),
4907	};
4908	break :res try cg.normalize(raw, info);
4909	},
4910	.float => try cg.buildBinary(fop, lhs, rhs),
4911	.bool => unreachable,
4912	};
4913	return try result.materialize(cg);
4914	}
4915
4916	fn airAbs(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
4917	const zcu = cg.zcu;
4918	const target = zcu.getTarget();
4919	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
4920	const value = try cg.temporary(ty_op.operand);
4921	// Note: operand_ty may be signed, while ty is always unsigned.
4922	const result_ty = cg.typeOfIndex(inst);
4923	const operand_info = cg.arithmeticTypeInfo(value.ty);
4924	const result: Temporary = switch (operand_info.class) {
4925	.float => try cg.buildUnary(.f_abs, value),
4926	.integer, .strange_integer => abs: {
4927	var abs_value = try cg.buildUnary(.i_abs, value);
4928	switch (target.os.tag) {
4929	.vulkan, .opengl => {
4930	if (value.ty.intInfo(zcu).signedness == .signed) {
4931	const abs_id = try abs_value.materialize(cg);
4932	const dst_ty_id = try cg.resolveType(result_ty, .direct);
4933	const cast_id = cg.allocId();
4934	try cg.body.emit(cg.gpa, .OpBitcast, .{
4935	.id_result_type = dst_ty_id,
4936	.id_result = cast_id,
4937	.operand = abs_id,
4938	});
4939	abs_value = .init(result_ty, cast_id);
4940	}
4941	},
4942	else => {},
4943	}
4944	break :abs try cg.normalize(abs_value, cg.arithmeticTypeInfo(result_ty));
4945	},
4946	.composite_integer => abs: {
4947	const val_id = try value.materialize(cg);
4948	const scratch_top = cg.id_scratch.items.len;
4949	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
4950	const ci = try CompositeInt.init(cg, val_id, operand_info);
4951	const ci_z = try CompositeInt.zero(cg, operand_info);
4952	const is_neg = try ci.cmp(ci_z, .lt);
4953	const ci_neg = try ci_z.addSub(ci, false);
4954	const result_info = cg.arithmeticTypeInfo(result_ty);
4955	const limb_ty_id = try cg.limbTypeId();
4956	const result_limbs = try cg.id_scratch.addManyAsSlice(cg.gpa, ci.n_limbs);
4957	for (0..ci.n_limbs) |i| {
4958	result_limbs[i] = cg.allocId();
4959	try cg.body.emit(cg.gpa, .OpSelect, .{
4960	.id_result_type = limb_ty_id,
4961	.id_result = result_limbs[i],
4962	.condition = is_neg,
4963	.object_1 = ci_neg.limbs[i],
4964	.object_2 = ci.limbs[i],
4965	});
4966	}
4967	const ci_result = CompositeInt.fromLimbs(cg, result_limbs, result_info);
4968	const normalized = try ci_result.normalize();
4969	break :abs .init(result_ty, try normalized.materialize(result_ty));
4970	},
4971	.bool => unreachable,
4972	};
4973	return try result.materialize(cg);
4974	}
4975
4976	fn airAddSubOverflow(
4977	cg: *CodeGen,
4978	inst: Air.Inst.Index,
4979	comptime add: Opcode,
4980	u_opcode: Opcode,
4981	s_opcode: Opcode,
4982	) !?Id {
4983	// Note: OpIAddCarry and OpISubBorrow are not really useful here: For unsigned numbers,
4984	// there is in both cases only one extra operation required. For signed operations,
4985	// the overflow bit is set then going from 0x80.. to 0x00.., but this doesn't actually
4986	// normally set a carry bit. So the SPIR-V overflow operations are not particularly
4987	// useful here.
4988
4989	_ = s_opcode;
4990
4991	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
4992	const extra = cg.air.extraData(Air.Bin, ty_pl.payload).data;
4993	const lhs = try cg.temporary(extra.lhs);
4994	const rhs = try cg.temporary(extra.rhs);
4995	const result_ty = cg.typeOfIndex(inst);
4996
4997	const info = cg.arithmeticTypeInfo(lhs.ty);
4998	switch (info.class) {
4999	.composite_integer => {
5000	const lhs_id = try lhs.materialize(cg);
5001	const rhs_id = try rhs.materialize(cg);
5002	const scratch_top = cg.id_scratch.items.len;
5003	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
5004	const ci_lhs = try CompositeInt.init(cg, lhs_id, info);
5005	const ci_rhs = try CompositeInt.init(cg, rhs_id, info);
5006	const ci_sum = if (add == .OpIAdd) try ci_lhs.addSub(ci_rhs, true) else try ci_lhs.addSub(ci_rhs, false);
5007	const ci_result = try ci_sum.normalize();
5008	const result_val_id = try ci_result.materialize(lhs.ty);
5009
5010	const ov_bool = switch (info.signedness) {
5011	.unsigned => blk: {
5012	const ci_res2 = try CompositeInt.init(cg, result_val_id, info);
5013	const ci_lhs2 = try CompositeInt.init(cg, lhs_id, info);
5014	break :blk if (add == .OpIAdd)
5015	try ci_res2.cmp(ci_lhs2, .lt)
5016	else
5017	try ci_res2.cmp(ci_lhs2, .gt);
5018	},
5019	.signed => blk: {
5020	const ci_res2 = try CompositeInt.init(cg, result_val_id, info);
5021	const ci_lhs2 = try CompositeInt.init(cg, lhs_id, info);
5022	const ci_rhs2 = try CompositeInt.init(cg, rhs_id, info);
5023	const ci_z = try CompositeInt.zero(cg, info);
5024	const lhs_neg = try ci_lhs2.cmp(ci_z, .lt);
5025	const rhs_neg = try ci_rhs2.cmp(ci_z, .lt);
5026	const res_neg = try ci_res2.cmp(ci_z, .lt);
5027
5028	const bool_ty_id = try cg.resolveType(.bool, .direct);
5029	const signs_match = cg.allocId();
5030	try cg.body.emit(cg.gpa, .OpLogicalEqual, .{
5031	.id_result_type = bool_ty_id,
5032	.id_result = signs_match,
5033	.operand_1 = lhs_neg,
5034	.operand_2 = rhs_neg,
5035	});
5036	const res_sign_diff = cg.allocId();
5037	try cg.body.emit(cg.gpa, .OpLogicalNotEqual, .{
5038	.id_result_type = bool_ty_id,
5039	.id_result = res_sign_diff,
5040	.operand_1 = lhs_neg,
5041	.operand_2 = res_neg,
5042	});
5043	const ov_cond = if (add == .OpIAdd) signs_match else blk2: {
5044	const not_match = cg.allocId();
5045	try cg.body.emit(cg.gpa, .OpLogicalNot, .{
5046	.id_result_type = bool_ty_id,
5047	.id_result = not_match,
5048	.operand = signs_match,
5049	});
5050	break :blk2 not_match;
5051	};
5052	const ov_result = cg.allocId();
5053	try cg.body.emit(cg.gpa, .OpLogicalAnd, .{
5054	.id_result_type = bool_ty_id,
5055	.id_result = ov_result,
5056	.operand_1 = ov_cond,
5057	.operand_2 = res_sign_diff,
5058	});
5059	break :blk ov_result;
5060	},
5061	};
5062	const ov = try cg.intFromBool(.init(.bool, ov_bool), .u1);
5063	const result_ty_id = try cg.resolveType(result_ty, .direct);
5064	return try cg.constructComposite(result_ty_id, &.{ result_val_id, try ov.materialize(cg) });
5065	},
5066	.strange_integer, .integer => {},
5067	.float, .bool => unreachable,
5068	}
5069
5070	const sum = try cg.buildBinary(add, lhs, rhs);
5071	const result = try cg.normalize(sum, info);
5072	const overflowed = switch (info.signedness) {
5073	// Overflow happened if the result is smaller than either of the operands. It doesn't matter which.
5074	// For subtraction the conditions need to be swapped.
5075	.unsigned => try cg.buildCmp(u_opcode, result, lhs),
5076	// For signed operations, we check the signs of the operands and the result.
5077	.signed => blk: {
5078	// Signed overflow detection using the sign bits of the operands and the result.
5079	// For addition (a + b), overflow occurs if the operands have the same sign
5080	// and the result's sign is different from the operands' sign.
5081	// (sign(a) == sign(b)) && (sign(a) != sign(result))
5082	// For subtraction (a - b), overflow occurs if the operands have different signs
5083	// and the result's sign is different from the minuend's (a's) sign.
5084	// (sign(a) != sign(b)) && (sign(a) != sign(result))
5085	const zero: Temporary = .init(rhs.ty, try cg.constInt(rhs.ty, 0));
5086	const lhs_is_neg = try cg.buildCmp(.OpSLessThan, lhs, zero);
5087	const rhs_is_neg = try cg.buildCmp(.OpSLessThan, rhs, zero);
5088	const result_is_neg = try cg.buildCmp(.OpSLessThan, result, zero);
5089	const signs_match = try cg.buildCmp(.OpLogicalEqual, lhs_is_neg, rhs_is_neg);
5090	const result_sign_differs = try cg.buildCmp(.OpLogicalNotEqual, lhs_is_neg, result_is_neg);
5091	const overflow_condition = switch (add) {
5092	.OpIAdd => signs_match,
5093	.OpISub => try cg.buildUnary(.l_not, signs_match),
5094	else => unreachable,
5095	};
5096	break :blk try cg.buildCmp(.OpLogicalAnd, overflow_condition, result_sign_differs);
5097	},
5098	};
5099
5100	const ov = try cg.intFromBool(overflowed, .u1);
5101	const result_ty_id = try cg.resolveType(result_ty, .direct);
5102	return try cg.constructComposite(result_ty_id, &.{ try result.materialize(cg), try ov.materialize(cg) });
5103	}
5104
5105	fn airMulOverflow(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
5106	const pt = cg.pt;
5107	const gpa = cg.gpa;
5108	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
5109	const extra = cg.air.extraData(Air.Bin, ty_pl.payload).data;
5110	const lhs = try cg.temporary(extra.lhs);
5111	const rhs = try cg.temporary(extra.rhs);
5112	const result_ty = cg.typeOfIndex(inst);
5113
5114	const info = cg.arithmeticTypeInfo(lhs.ty);
5115	switch (info.class) {
5116	.composite_integer => {
5117	const lhs_id = try lhs.materialize(cg);
5118	const rhs_id = try rhs.materialize(cg);
5119	const scratch_top = cg.id_scratch.items.len;
5120	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
5121	const ci_lhs = try CompositeInt.init(cg, lhs_id, info);
5122	const ci_rhs = try CompositeInt.init(cg, rhs_id, info);
5123
5124	const low_limbs = try ci_lhs.mul(ci_rhs, false);
5125	const ci_result = try CompositeInt.fromLimbs(cg, low_limbs, info).normalize();
5126	const result_val_id = try ci_result.materialize(lhs.ty);
5127
5128	const ci_lhs2 = try CompositeInt.init(cg, lhs_id, info);
5129	const ci_rhs2 = try CompositeInt.init(cg, rhs_id, info);
5130	const wide_limbs = try ci_lhs2.mul(ci_rhs2, true);
5131	const high_limbs = wide_limbs[ci_lhs2.n_limbs..];
5132
5133	const bool_ty_id = try cg.resolveType(.bool, .direct);
5134	const limb_ty_id = try cg.limbTypeId();
5135	const limb_ty = cg.limbType();
5136	const n: usize = info.backing_bits / cg.bigIntBits();
5137
5138	const ov_bool = switch (info.signedness) {
5139	.unsigned => blk: {
5140	const zero_id = try cg.constInt(limb_ty, @as(u64, 0));
5141	var any_nonzero = cg.allocId();
5142	try cg.body.emit(gpa, .OpINotEqual, .{
5143	.id_result_type = bool_ty_id,
5144	.id_result = any_nonzero,
5145	.operand_1 = high_limbs[0],
5146	.operand_2 = zero_id,
5147	});
5148
5149	for (1..n) |i| {
5150	const limb_nz = cg.allocId();
5151	try cg.body.emit(gpa, .OpINotEqual, .{
5152	.id_result_type = bool_ty_id,
5153	.id_result = limb_nz,
5154	.operand_1 = high_limbs[i],
5155	.operand_2 = zero_id,
5156	});
5157
5158	const combined = cg.allocId();
5159	try cg.body.emit(gpa, .OpLogicalOr, .{
5160	.id_result_type = bool_ty_id,
5161	.id_result = combined,
5162	.operand_1 = any_nonzero,
5163	.operand_2 = limb_nz,
5164	});
5165	any_nonzero = combined;
5166	}
5167
5168	break :blk any_nonzero;
5169	},
5170	.signed => blk: {
5171	const ci_res = try CompositeInt.init(cg, result_val_id, info);
5172	const top_limb = ci_res.limbs[n - 1];
5173	const signed_limb_ty: Type = if (cg.bigIntBits() == 64) .i64 else .i32;
5174	const signed_limb_ty_id = try cg.resolveType(signed_limb_ty, .direct);
5175
5176	const top_bits: u16 = if (info.bits % cg.bigIntBits() == 0)
5177	cg.bigIntBits()
5178	else
5179	info.bits % cg.bigIntBits();
5180
5181	const shift_amt: u64 = top_bits - 1;
5182	const shift_id = try cg.constInt(limb_ty, shift_amt);
5183
5184	const as_signed = cg.allocId();
5185	try cg.body.emit(gpa, .OpBitcast, .{
5186	.id_result_type = signed_limb_ty_id,
5187	.id_result = as_signed,
5188	.operand = top_limb,
5189	});
5190	const sign_ext = cg.allocId();
5191	try cg.body.emit(gpa, .OpShiftRightArithmetic, .{
5192	.id_result_type = signed_limb_ty_id,
5193	.id_result = sign_ext,
5194	.base = as_signed,
5195	.shift = shift_id,
5196	});
5197	const expected = cg.allocId();
5198	try cg.body.emit(gpa, .OpBitcast, .{
5199	.id_result_type = limb_ty_id,
5200	.id_result = expected,
5201	.operand = sign_ext,
5202	});
5203
5204	var any_mismatch = cg.allocId();
5205	try cg.body.emit(gpa, .OpINotEqual, .{
5206	.id_result_type = bool_ty_id,
5207	.id_result = any_mismatch,
5208	.operand_1 = high_limbs[0],
5209	.operand_2 = expected,
5210	});
5211
5212	for (1..n) |i| {
5213	const limb_ne = cg.allocId();
5214	try cg.body.emit(gpa, .OpINotEqual, .{
5215	.id_result_type = bool_ty_id,
5216	.id_result = limb_ne,
5217	.operand_1 = high_limbs[i],
5218	.operand_2 = expected,
5219	});
5220
5221	const combined = cg.allocId();
5222	try cg.body.emit(gpa, .OpLogicalOr, .{
5223	.id_result_type = bool_ty_id,
5224	.id_result = combined,
5225	.operand_1 = any_mismatch,
5226	.operand_2 = limb_ne,
5227	});
5228	any_mismatch = combined;
5229	}
5230
5231	if (info.bits != info.backing_bits) {
5232	const top_bits_s: u16 = info.bits % cg.bigIntBits();
5233	const s_shift_id = try cg.constInt(limb_ty, @as(u64, top_bits_s - 1));
5234
5235	const top_as_signed = cg.allocId();
5236	try cg.body.emit(gpa, .OpBitcast, .{
5237	.id_result_type = signed_limb_ty_id,
5238	.id_result = top_as_signed,
5239	.operand = top_limb,
5240	});
5241	const top_sign_ext = cg.allocId();
5242	try cg.body.emit(gpa, .OpShiftRightArithmetic, .{
5243	.id_result_type = signed_limb_ty_id,
5244	.id_result = top_sign_ext,
5245	.base = top_as_signed,
5246	.shift = s_shift_id,
5247	});
5248	const top_expected = cg.allocId();
5249	try cg.body.emit(gpa, .OpBitcast, .{
5250	.id_result_type = limb_ty_id,
5251	.id_result = top_expected,
5252	.operand = top_sign_ext,
5253	});
5254	const top_mismatch = cg.allocId();
5255	try cg.body.emit(gpa, .OpINotEqual, .{
5256	.id_result_type = bool_ty_id,
5257	.id_result = top_mismatch,
5258	.operand_1 = top_limb,
5259	.operand_2 = top_expected,
5260	});
5261
5262	const combined = cg.allocId();
5263	try cg.body.emit(gpa, .OpLogicalOr, .{
5264	.id_result_type = bool_ty_id,
5265	.id_result = combined,
5266	.operand_1 = any_mismatch,
5267	.operand_2 = top_mismatch,
5268	});
5269	any_mismatch = combined;
5270	}
5271
5272	break :blk any_mismatch;
5273	},
5274	};
5275
5276	const ov = try cg.intFromBool(.init(.bool, ov_bool), .u1);
5277	const result_ty_id = try cg.resolveType(result_ty, .direct);
5278	return try cg.constructComposite(result_ty_id, &.{ result_val_id, try ov.materialize(cg) });
5279	},
5280	.strange_integer, .integer => {},
5281	.float, .bool => unreachable,
5282	}
5283
5284	// There are 3 cases which we have to deal with:
5285	// - If info.bits < 32 / 2, we will upcast to 32 and check the higher bits
5286	// - If info.bits > 32 / 2, we have to use extended multiplication
5287	// - Additionally, if info.bits != 32, we'll have to check the high bits
5288	// of the result too.
5289
5290	const target = cg.zcu.getTarget();
5291	const largest_int_bits: u16 = if (hasInt64(target)) 64 else 32;
5292	// If non-null, the number of bits that the multiplication should be performed in. If
5293	// null, we have to use wide multiplication.
5294	const maybe_op_ty_bits: ?u16 = switch (info.bits) {
5295	0 => unreachable,
5296	1...16 => 32,
5297	17...32 => if (largest_int_bits > 32) 64 else null, // Upcast if we can.
5298	33...64 => null, // Always use wide multiplication.
5299	else => unreachable,
5300	};
5301
5302	const result, const overflowed = switch (info.signedness) {
5303	.unsigned => blk: {
5304	if (maybe_op_ty_bits) |op_ty_bits| {
5305	const op_ty = try pt.intType(.unsigned, op_ty_bits);
5306	const casted_lhs = try cg.buildConvert(op_ty, lhs);
5307	const casted_rhs = try cg.buildConvert(op_ty, rhs);
5308	const full_result = try cg.buildBinary(.OpIMul, casted_lhs, casted_rhs);
5309	const low_bits = try cg.buildConvert(lhs.ty, full_result);
5310	const result = try cg.normalize(low_bits, info);
5311	// Shift the result bits away to get the overflow bits.
5312	const shift: Temporary = .init(full_result.ty, try cg.constInt(full_result.ty, info.bits));
5313	const overflow = try cg.buildBinary(.OpShiftRightLogical, full_result, shift);
5314	// Directly check if its zero in the op_ty without converting first.
5315	const zero: Temporary = .init(full_result.ty, try cg.constInt(full_result.ty, 0));
5316	const overflowed = try cg.buildCmp(.OpINotEqual, zero, overflow);
5317	break :blk .{ result, overflowed };
5318	}
5319
5320	const low_bits, const high_bits = try cg.buildWideMul(.unsigned, lhs, rhs);
5321
5322	// Truncate the result, if required.
5323	const result = try cg.normalize(low_bits, info);
5324
5325	// Overflow happened if the high-bits of the result are non-zero OR if the
5326	// high bits of the low word of the result (those outside the range of the
5327	// int) are nonzero.
5328	const zero: Temporary = .init(lhs.ty, try cg.constInt(lhs.ty, 0));
5329	const high_overflowed = try cg.buildCmp(.OpINotEqual, zero, high_bits);
5330
5331	// If no overflow bits in low_bits, no extra work needs to be done.
5332	if (info.backing_bits == info.bits) break :blk .{ result, high_overflowed };
5333
5334	// Shift the result bits away to get the overflow bits.
5335	const shift: Temporary = .init(lhs.ty, try cg.constInt(lhs.ty, info.bits));
5336	const low_overflow = try cg.buildBinary(.OpShiftRightLogical, low_bits, shift);
5337	const low_overflowed = try cg.buildCmp(.OpINotEqual, zero, low_overflow);
5338
5339	const overflowed = try cg.buildCmp(.OpLogicalOr, low_overflowed, high_overflowed);
5340
5341	break :blk .{ result, overflowed };
5342	},
5343	.signed => blk: {
5344	// - lhs >= 0, rhxs >= 0: expect positive; overflow should be 0
5345	// - lhs == 0 : expect positive; overflow should be 0
5346	// - rhs == 0: expect positive; overflow should be 0
5347	// - lhs > 0, rhs < 0: expect negative; overflow should be -1
5348	// - lhs < 0, rhs > 0: expect negative; overflow should be -1
5349	// - lhs <= 0, rhs <= 0: expect positive; overflow should be 0
5350	// ------
5351	// overflow should be -1 when
5352	// (lhs > 0 && rhs < 0) || (lhs < 0 && rhs > 0)
5353
5354	const zero: Temporary = .init(lhs.ty, try cg.constInt(lhs.ty, 0));
5355	const lhs_negative = try cg.buildCmp(.OpSLessThan, lhs, zero);
5356	const rhs_negative = try cg.buildCmp(.OpSLessThan, rhs, zero);
5357	const lhs_positive = try cg.buildCmp(.OpSGreaterThan, lhs, zero);
5358	const rhs_positive = try cg.buildCmp(.OpSGreaterThan, rhs, zero);
5359
5360	// Set to `true` if we expect -1.
5361	const expected_overflow_bit = try cg.buildBinary(
5362	.OpLogicalOr,
5363	try cg.buildCmp(.OpLogicalAnd, lhs_positive, rhs_negative),
5364	try cg.buildCmp(.OpLogicalAnd, lhs_negative, rhs_positive),
5365	);
5366
5367	if (maybe_op_ty_bits) |op_ty_bits| {
5368	const op_ty = try pt.intType(.signed, op_ty_bits);
5369	// Assume normalized; sign bit is set. We want a sign extend.
5370	const casted_lhs = try cg.buildConvert(op_ty, lhs);
5371	const casted_rhs = try cg.buildConvert(op_ty, rhs);
5372
5373	const full_result = try cg.buildBinary(.OpIMul, casted_lhs, casted_rhs);
5374
5375	// Truncate to the result type.
5376	const low_bits = try cg.buildConvert(lhs.ty, full_result);
5377	const result = try cg.normalize(low_bits, info);
5378
5379	// Now, we need to check the overflow bits AND the sign
5380	// bit for the expected overflow bits.
5381	// To do that, shift out everything bit the sign bit and
5382	// then check what remains.
5383	const shift: Temporary = .init(full_result.ty, try cg.constInt(full_result.ty, info.bits - 1));
5384	// Use SRA so that any sign bits are duplicated. Now we can just check if ALL bits are set
5385	// for negative cases.
5386	const overflow = try cg.buildBinary(.OpShiftRightArithmetic, full_result, shift);
5387
5388	const long_all_set: Temporary = .init(full_result.ty, try cg.constInt(full_result.ty, -1));
5389	const long_zero: Temporary = .init(full_result.ty, try cg.constInt(full_result.ty, 0));
5390	const mask = try cg.buildSelect(expected_overflow_bit, long_all_set, long_zero);
5391
5392	const overflowed = try cg.buildCmp(.OpINotEqual, mask, overflow);
5393
5394	break :blk .{ result, overflowed };
5395	}
5396
5397	const low_bits, const high_bits = try cg.buildWideMul(.signed, lhs, rhs);
5398
5399	// Truncate result if required.
5400	const result = try cg.normalize(low_bits, info);
5401
5402	const all_set: Temporary = .init(lhs.ty, try cg.constInt(lhs.ty, -1));
5403	const mask = try cg.buildSelect(expected_overflow_bit, all_set, zero);
5404
5405	// Like with unsigned, overflow happened if high_bits are not the ones we expect,
5406	// and we also need to check some ones from the low bits.
5407
5408	const high_overflowed = try cg.buildCmp(.OpINotEqual, mask, high_bits);
5409
5410	// If no overflow bits in low_bits, no extra work needs to be done.
5411	// Careful, we still have to check the sign bit, so this branch
5412	// only goes for i33 and such.
5413	if (info.backing_bits == info.bits + 1) break :blk .{ result, high_overflowed };
5414
5415	// Shift the result bits away to get the overflow bits.
5416	const shift: Temporary = .init(lhs.ty, try cg.constInt(lhs.ty, info.bits - 1));
5417	// Use SRA so that any sign bits are duplicated. Now we can just check if ALL bits are set
5418	// for negative cases.
5419	const low_overflow = try cg.buildBinary(.OpShiftRightArithmetic, low_bits, shift);
5420	const low_overflowed = try cg.buildCmp(.OpINotEqual, mask, low_overflow);
5421
5422	const overflowed = try cg.buildCmp(.OpLogicalOr, low_overflowed, high_overflowed);
5423
5424	break :blk .{ result, overflowed };
5425	},
5426	};
5427
5428	const ov = try cg.intFromBool(overflowed, .u1);
5429
5430	const result_ty_id = try cg.resolveType(result_ty, .direct);
5431	return try cg.constructComposite(result_ty_id, &.{ try result.materialize(cg), try ov.materialize(cg) });
5432	}
5433
5434	fn airShlOverflow(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
5435	const zcu = cg.zcu;
5436
5437	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
5438	const extra = cg.air.extraData(Air.Bin, ty_pl.payload).data;
5439
5440	const base = try cg.temporary(extra.lhs);
5441	const shift = try cg.temporary(extra.rhs);
5442
5443	const result_ty = cg.typeOfIndex(inst);
5444
5445	const info = cg.arithmeticTypeInfo(base.ty);
5446	switch (info.class) {
5447	.composite_integer => return cg.todo("shl-with-overflow for composite integers", .{}),
5448	.integer, .strange_integer => {},
5449	.float, .bool => unreachable,
5450	}
5451
5452	// Sometimes Zig doesn't make both of the arguments the same types here. SPIR-V expects that,
5453	// so just manually upcast it if required.
5454	const casted_shift = try cg.buildConvert(base.ty.scalarType(zcu), shift);
5455
5456	const left = try cg.buildBinary(.OpShiftLeftLogical, base, casted_shift);
5457	const result = try cg.normalize(left, info);
5458
5459	const right = switch (info.signedness) {
5460	.unsigned => try cg.buildBinary(.OpShiftRightLogical, result, casted_shift),
5461	.signed => try cg.buildBinary(.OpShiftRightArithmetic, result, casted_shift),
5462	};
5463
5464	const overflowed = try cg.buildCmp(.OpINotEqual, base, right);
5465	const ov = try cg.intFromBool(overflowed, .u1);
5466
5467	const result_ty_id = try cg.resolveType(result_ty, .direct);
5468	return try cg.constructComposite(result_ty_id, &.{ try result.materialize(cg), try ov.materialize(cg) });
5469	}
5470
5471	fn airMulAdd(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
5472	const pl_op = cg.air.instructions.items(.data)[@backingInt(inst)].pl_op;
5473	const extra = cg.air.extraData(Air.Bin, pl_op.payload).data;
5474
5475	const a = try cg.temporary(extra.lhs);
5476	const b = try cg.temporary(extra.rhs);
5477	const c = try cg.temporary(pl_op.operand);
5478
5479	const result_ty = cg.typeOfIndex(inst);
5480	const info = cg.arithmeticTypeInfo(result_ty);
5481	assert(info.class == .float); // .mul_add is only emitted for floats
5482
5483	const zcu = cg.zcu;
5484	const target = zcu.getTarget();
5485
5486	const v = cg.vectorization(.{ a, b, c });
5487	const ops = v.components();
5488	const results = cg.allocIds(ops);
5489
5490	const op_result_ty = a.ty.scalarType(zcu);
5491	const op_result_ty_id = try cg.resolveType(op_result_ty, .direct);
5492	const result_temp_ty = try v.resultType(cg, a.ty);
5493
5494	const op_a = try v.prepare(cg, a);
5495	const op_b = try v.prepare(cg, b);
5496	const op_c = try v.prepare(cg, c);
5497
5498	const set = try cg.importExtendedSet();
5499	const opcode: u32 = switch (target.os.tag) {
5500	.opencl => @backingInt(spec.OpenClOpcode.fma),
5501	// NOTE: Vulkan's FMA does not meet Zig's nor OpenCL's precision guarantees and needs
5502	// to be emulated.
5503	.vulkan, .opengl => @backingInt(spec.GlslOpcode.Fma),
5504	else => unreachable,
5505	};
5506
5507	for (0..ops) |i| {
5508	try cg.body.emit(cg.gpa, .OpExtInst, .{
5509	.id_result_type = op_result_ty_id,
5510	.id_result = results.at(i),
5511	.set = set,
5512	.instruction = .{ .inst = opcode },
5513	.id_ref_4 = &.{ op_a.at(i), op_b.at(i), op_c.at(i) },
5514	});
5515	}
5516
5517	const result = v.finalize(result_temp_ty, results);
5518	return try result.materialize(cg);
5519	}
5520
5521	fn airClzCtz(cg: *CodeGen, inst: Air.Inst.Index, op: UnaryOp) !?Id {
5522	if (cg.liveness.isUnused(inst)) return null;
5523
5524	const zcu = cg.zcu;
5525	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
5526	const operand = try cg.temporary(ty_op.operand);
5527
5528	const scalar_result_ty = cg.typeOfIndex(inst).scalarType(zcu);
5529
5530	const info = cg.arithmeticTypeInfo(operand.ty);
5531	switch (info.class) {
5532	.composite_integer => return cg.todo("@clz/@ctz for composite integers", .{}),
5533	.integer, .strange_integer => {},
5534	.float, .bool => unreachable,
5535	}
5536
5537	const count = try cg.buildUnary(op, operand);
5538
5539	// Result of OpenCL ctz/clz returns operand.ty, and we want result_ty.
5540	// result_ty is always large enough to hold the result, so we might have to down
5541	// cast it.
5542	const result = try cg.buildConvert(scalar_result_ty, count);
5543	return try result.materialize(cg);
5544	}
5545
5546	fn airSelect(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
5547	const pl_op = cg.air.instructions.items(.data)[@backingInt(inst)].pl_op;
5548	const extra = cg.air.extraData(Air.Bin, pl_op.payload).data;
5549	const pred = try cg.temporary(pl_op.operand);
5550	const a = try cg.temporary(extra.lhs);
5551	const b = try cg.temporary(extra.rhs);
5552
5553	const result = try cg.buildSelect(pred, a, b);
5554	return try result.materialize(cg);
5555	}
5556
5557	fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
5558	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
5559
5560	const operand_id = try cg.resolve(ty_op.operand);
5561	const result_ty = cg.typeOfIndex(inst);
5562
5563	return try cg.constructCompositeSplat(result_ty, operand_id);
5564	}
5565
5566	fn airReduce(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
5567	const zcu = cg.zcu;
5568	const reduce = cg.air.instructions.items(.data)[@backingInt(inst)].reduce;
5569	const operand = try cg.resolve(reduce.operand);
5570	const operand_ty = cg.typeOf(reduce.operand);
5571	const scalar_ty = operand_ty.scalarType(zcu);
5572	const info = cg.arithmeticTypeInfo(operand_ty);
5573	const len = operand_ty.vectorLen(zcu);
5574	const first = try cg.extractVectorComponent(scalar_ty, operand, 0);
5575
5576	switch (reduce.operation) {
5577	.Min, .Max => |op| {
5578	var result: Temporary = .init(scalar_ty, first);
5579	const cmp_op: MinMax = switch (op) {
5580	.Max => .max,
5581	.Min => .min,
5582	else => unreachable,
5583	};
5584	for (1..len) |i| {
5585	const lhs = result;
5586	const rhs_id = try cg.extractVectorComponent(scalar_ty, operand, @intCast(i));
5587	const rhs: Temporary = .init(scalar_ty, rhs_id);
5588
5589	result = try cg.minMax(lhs, rhs, cmp_op);
5590	}
5591
5592	return try result.materialize(cg);
5593	},
5594	else => {},
5595	}
5596
5597	const opcode: Opcode = switch (info.class) {
5598	.bool => switch (reduce.operation) {
5599	.And => .OpLogicalAnd,
5600	.Or => .OpLogicalOr,
5601	.Xor => .OpLogicalNotEqual,
5602	else => unreachable,
5603	},
5604	.strange_integer, .integer => switch (reduce.operation) {
5605	.And => .OpBitwiseAnd,
5606	.Or => .OpBitwiseOr,
5607	.Xor => .OpBitwiseXor,
5608	.Add => .OpIAdd,
5609	.Mul => .OpIMul,
5610	else => unreachable,
5611	},
5612	.float => switch (reduce.operation) {
5613	.Add => .OpFAdd,
5614	.Mul => .OpFMul,
5615	else => unreachable,
5616	},
5617	.composite_integer => return cg.todo("@reduce for composite integers", .{}),
5618	};
5619
5620	const needs_normalize = info.class == .strange_integer and
5621	(reduce.operation == .Add or reduce.operation == .Mul);
5622
5623	var result: Temporary = .init(scalar_ty, first);
5624	for (1..len) |i| {
5625	const rhs_id = try cg.extractVectorComponent(scalar_ty, operand, @intCast(i));
5626	const rhs: Temporary = .init(scalar_ty, rhs_id);
5627	const stepped = try cg.buildBinary(opcode, result, rhs);
5628	result = if (needs_normalize) try cg.normalize(stepped, info) else stepped;
5629	}
5630
5631	return try result.materialize(cg);
5632	}
5633
5634	fn airShuffleOne(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
5635	const zcu = cg.zcu;
5636	const gpa = zcu.gpa;
5637
5638	const unwrapped = cg.air.unwrapShuffleOne(zcu, inst);
5639	const mask = unwrapped.mask;
5640	const result_ty = unwrapped.result_ty;
5641	const elem_ty = result_ty.childType(zcu);
5642	const operand = try cg.resolve(unwrapped.operand);
5643
5644	const scratch_top = cg.id_scratch.items.len;
5645	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
5646	const constituents = try cg.id_scratch.addManyAsSlice(gpa, mask.len);
5647
5648	for (constituents, mask) |*id, mask_elem| {
5649	id.* = switch (mask_elem.unwrap()) {
5650	.elem => |idx| try cg.extractVectorComponent(elem_ty, operand, idx),
5651	.value => |val| try cg.constant(elem_ty, .fromInterned(val), .direct),
5652	};
5653	}
5654
5655	const result_ty_id = try cg.resolveType(result_ty, .direct);
5656	return try cg.constructComposite(result_ty_id, constituents);
5657	}
5658
5659	fn airShuffleTwo(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
5660	const zcu = cg.zcu;
5661	const gpa = zcu.gpa;
5662
5663	const unwrapped = cg.air.unwrapShuffleTwo(zcu, inst);
5664	const mask = unwrapped.mask;
5665	const result_ty = unwrapped.result_ty;
5666	const elem_ty = result_ty.childType(zcu);
5667	const elem_ty_id = try cg.resolveType(elem_ty, .direct);
5668	const operand_a = try cg.resolve(unwrapped.operand_a);
5669	const operand_b = try cg.resolve(unwrapped.operand_b);
5670
5671	const scratch_top = cg.id_scratch.items.len;
5672	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
5673	const constituents = try cg.id_scratch.addManyAsSlice(gpa, mask.len);
5674
5675	for (constituents, mask) |*id, mask_elem| {
5676	id.* = switch (mask_elem.unwrap()) {
5677	.a_elem => |idx| try cg.extractVectorComponent(elem_ty, operand_a, idx),
5678	.b_elem => |idx| try cg.extractVectorComponent(elem_ty, operand_b, idx),
5679	.undef => try cg.constUndef(elem_ty_id),
5680	};
5681	}
5682
5683	const result_ty_id = try cg.resolveType(result_ty, .direct);
5684	return try cg.constructComposite(result_ty_id, constituents);
5685	}
5686
5687	fn accessChainId(
5688	cg: *CodeGen,
5689	result_ty_id: Id,
5690	base: Id,
5691	indices: []const Id,
5692	) !Id {
5693	const result_id = cg.allocId();
5694	try cg.body.emit(cg.gpa, .OpInBoundsAccessChain, .{
5695	.id_result_type = result_ty_id,
5696	.id_result = result_id,
5697	.base = base,
5698	.indexes = indices,
5699	});
5700	return result_id;
5701	}
5702
5703	/// AccessChain is essentially PtrAccessChain with 0 as initial argument. The effective
5704	/// difference lies in whether the resulting type of the first dereference will be the
5705	/// same as that of the base pointer, or that of a dereferenced base pointer. AccessChain
5706	/// is the latter and PtrAccessChain is the former.
5707	fn accessChain(
5708	cg: *CodeGen,
5709	result_ty_id: Id,
5710	base: Id,
5711	indices: []const u32,
5712	) !Id {
5713	const gpa = cg.gpa;
5714	const scratch_top = cg.id_scratch.items.len;
5715	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
5716	const ids = try cg.id_scratch.addManyAsSlice(gpa, indices.len);
5717	for (indices, ids) |index, *id| {
5718	id.* = try cg.constInt(.u32, index);
5719	}
5720	return try cg.accessChainId(result_ty_id, base, ids);
5721	}
5722
5723	fn ptrAccessChain(
5724	cg: *CodeGen,
5725	result_ty_id: Id,
5726	base: Id,
5727	element: Id,
5728	indices: []const u32,
5729	) !Id {
5730	const gpa = cg.gpa;
5731	const target = cg.zcu.getTarget();
5732
5733	const scratch_top = cg.id_scratch.items.len;
5734	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
5735	const ids = try cg.id_scratch.addManyAsSlice(gpa, indices.len);
5736	for (indices, ids) |index, *id| {
5737	id.* = try cg.constInt(.u32, index);
5738	}
5739
5740	const result_id = cg.allocId();
5741	switch (target.os.tag) {
5742	.opencl, .amdhsa => {
5743	try cg.body.emit(gpa, .OpInBoundsPtrAccessChain, .{
5744	.id_result_type = result_ty_id,
5745	.id_result = result_id,
5746	.base = base,
5747	.element = element,
5748	.indexes = ids,
5749	});
5750	},
5751	.vulkan, .opengl => {
5752	assert(target.cpu.has(.spirv, .variable_pointers) or
5753	target.cpu.has(.spirv, .variable_pointers_storage_buffer));
5754	try cg.body.emit(gpa, .OpPtrAccessChain, .{
5755	.id_result_type = result_ty_id,
5756	.id_result = result_id,
5757	.base = base,
5758	.element = element,
5759	.indexes = ids,
5760	});
5761	},
5762	else => unreachable,
5763	}
5764	return result_id;
5765	}
5766
5767	fn ptrAdd(cg: *CodeGen, result_ty: Type, ptr_ty: Type, ptr_id: Id, offset_id: Id) !Id {
5768	const zcu = cg.zcu;
5769	const as = result_ty.ptrAddressSpace(zcu);
5770	const child_ty_id = try cg.pointeeType(as, result_ty.childType(zcu), false);
5771	const result_ty_id = try cg.ptrType(child_ty_id, cg.storageClass(as));
5772	return switch (ptr_ty.ptrSize(zcu)) {
5773	.one => cg.accessChainId(result_ty_id, ptr_id, &.{offset_id}),
5774	.c, .many => cg.ptrAccessChain(result_ty_id, ptr_id, offset_id, &.{}),
5775	.slice => blk: {
5776	// TODO: This is probably incorrect. A slice should be returned here, though this is what llvm does.
5777	const slice_ptr_id = try cg.extractField(result_ty, ptr_id, 0);
5778	break :blk cg.ptrAccessChain(result_ty_id, slice_ptr_id, offset_id, &.{});
5779	},
5780	};
5781	}
5782
5783	fn airPtrAdd(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
5784	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
5785	const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
5786	const ptr_id = try cg.resolve(bin_op.lhs);
5787	const offset_id = try cg.resolve(bin_op.rhs);
5788	const ptr_ty = cg.typeOf(bin_op.lhs);
5789	const result_ty = cg.typeOfIndex(inst);
5790
5791	return try cg.ptrAdd(result_ty, ptr_ty, ptr_id, offset_id);
5792	}
5793
5794	fn airPtrSub(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
5795	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
5796	const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
5797	const ptr_id = try cg.resolve(bin_op.lhs);
5798	const ptr_ty = cg.typeOf(bin_op.lhs);
5799	const offset_id = try cg.resolve(bin_op.rhs);
5800	const offset_ty = cg.typeOf(bin_op.rhs);
5801	const offset_ty_id = try cg.resolveType(offset_ty, .direct);
5802	const result_ty = cg.typeOfIndex(inst);
5803
5804	const negative_offset_id = cg.allocId();
5805	try cg.body.emit(cg.gpa, .OpSNegate, .{
5806	.id_result_type = offset_ty_id,
5807	.id_result = negative_offset_id,
5808	.operand = offset_id,
5809	});
5810	return try cg.ptrAdd(result_ty, ptr_ty, ptr_id, negative_offset_id);
5811	}
5812
5813	fn cmp(
5814	cg: *CodeGen,
5815	op: std.math.CompareOperator,
5816	lhs: Temporary,
5817	rhs: Temporary,
5818	) !Temporary {
5819	const gpa = cg.gpa;
5820	const pt = cg.pt;
5821	const zcu = cg.zcu;
5822	const scalar_ty = lhs.ty.scalarType(zcu);
5823	const is_vector = lhs.ty.isVector(zcu);
5824
5825	switch (scalar_ty.zigTypeTag(zcu)) {
5826	.int, .bool, .float => {},
5827	.@"enum" => {
5828	assert(!is_vector);
5829	const ty = lhs.ty.backingIntType(zcu);
5830	return try cg.cmp(op, lhs.pun(ty), rhs.pun(ty));
5831	},
5832	.@"struct" => {
5833	const struct_ty = zcu.typeToPackedStruct(scalar_ty).?;
5834	const ty: Type = .fromInterned(struct_ty.packed_backing_int_type);
5835	return try cg.cmp(op, lhs.pun(ty), rhs.pun(ty));
5836	},
5837	.error_set => {
5838	assert(!is_vector);
5839	const err_int_ty = try pt.errorIntType();
5840	return try cg.cmp(op, lhs.pun(err_int_ty), rhs.pun(err_int_ty));
5841	},
5842	.pointer => {
5843	assert(!is_vector);
5844	// Note that while SPIR-V offers OpPtrEqual and OpPtrNotEqual, they are
5845	// currently not implemented in the SPIR-V LLVM translator. Thus, we emit these using
5846	// OpConvertPtrToU...
5847
5848	const usize_ty_id = try cg.resolveType(.usize, .direct);
5849
5850	const lhs_int_id = cg.allocId();
5851	try cg.body.emit(gpa, .OpConvertPtrToU, .{
5852	.id_result_type = usize_ty_id,
5853	.id_result = lhs_int_id,
5854	.pointer = try lhs.materialize(cg),
5855	});
5856
5857	const rhs_int_id = cg.allocId();
5858	try cg.body.emit(gpa, .OpConvertPtrToU, .{
5859	.id_result_type = usize_ty_id,
5860	.id_result = rhs_int_id,
5861	.pointer = try rhs.materialize(cg),
5862	});
5863
5864	const lhs_int: Temporary = .init(.usize, lhs_int_id);
5865	const rhs_int: Temporary = .init(.usize, rhs_int_id);
5866	return try cg.cmp(op, lhs_int, rhs_int);
5867	},
5868	.optional => {
5869	assert(!is_vector);
5870
5871	const ty = lhs.ty;
5872
5873	const payload_ty = ty.optionalChild(zcu);
5874	if (ty.optionalReprIsPayload(zcu)) {
5875	assert(payload_ty.hasRuntimeBits(zcu));
5876	assert(!payload_ty.isSlice(zcu));
5877
5878	return try cg.cmp(op, lhs.pun(payload_ty), rhs.pun(payload_ty));
5879	}
5880
5881	const lhs_id = try lhs.materialize(cg);
5882	const rhs_id = try rhs.materialize(cg);
5883
5884	const lhs_valid_id = if (payload_ty.hasRuntimeBits(zcu))
5885	try cg.extractField(.bool, lhs_id, 1)
5886	else
5887	try cg.convertToDirect(.bool, lhs_id);
5888
5889	const rhs_valid_id = if (payload_ty.hasRuntimeBits(zcu))
5890	try cg.extractField(.bool, rhs_id, 1)
5891	else
5892	try cg.convertToDirect(.bool, rhs_id);
5893
5894	const lhs_valid: Temporary = .init(.bool, lhs_valid_id);
5895	const rhs_valid: Temporary = .init(.bool, rhs_valid_id);
5896
5897	if (!payload_ty.hasRuntimeBits(zcu)) {
5898	return try cg.cmp(op, lhs_valid, rhs_valid);
5899	}
5900
5901	// a = lhs_valid
5902	// b = rhs_valid
5903	// c = lhs_pl == rhs_pl
5904	//
5905	// For op == .eq we have:
5906	// a == b && a -> c
5907	// = a == b && (!a || c)
5908	//
5909	// For op == .neq we have
5910	// a == b && a -> c
5911	// = !(a == b && a -> c)
5912	// = a != b || !(a -> c
5913	// = a != b || !(!a || c)
5914	// = a != b || a && !c
5915
5916	const lhs_pl_id = try cg.extractField(payload_ty, lhs_id, 0);
5917	const rhs_pl_id = try cg.extractField(payload_ty, rhs_id, 0);
5918
5919	const lhs_pl: Temporary = .init(payload_ty, lhs_pl_id);
5920	const rhs_pl: Temporary = .init(payload_ty, rhs_pl_id);
5921
5922	return switch (op) {
5923	.eq => try cg.buildBinary(
5924	.OpLogicalAnd,
5925	try cg.cmp(.eq, lhs_valid, rhs_valid),
5926	try cg.buildBinary(
5927	.OpLogicalOr,
5928	try cg.buildUnary(.l_not, lhs_valid),
5929	try cg.cmp(.eq, lhs_pl, rhs_pl),
5930	),
5931	),
5932	.neq => try cg.buildBinary(
5933	.OpLogicalOr,
5934	try cg.cmp(.neq, lhs_valid, rhs_valid),
5935	try cg.buildBinary(
5936	.OpLogicalAnd,
5937	lhs_valid,
5938	try cg.cmp(.neq, lhs_pl, rhs_pl),
5939	),
5940	),
5941	else => unreachable,
5942	};
5943	},
5944	else => |ty| return cg.todo("implement cmp operation for '{s}' type", .{@tagName(ty)}),
5945	}
5946
5947	const info = cg.arithmeticTypeInfo(scalar_ty);
5948	const pred: Opcode = switch (info.class) {
5949	.composite_integer => {
5950	const lhs_id = try lhs.materialize(cg);
5951	const rhs_id = try rhs.materialize(cg);
5952	const scratch_top = cg.id_scratch.items.len;
5953	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
5954	const ci_lhs = try CompositeInt.init(cg, lhs_id, info);
5955	const ci_rhs = try CompositeInt.init(cg, rhs_id, info);
5956	const result_id = try ci_lhs.cmp(ci_rhs, op);
5957	return .init(.bool, result_id);
5958	},
5959	.float => switch (op) {
5960	.eq => .OpFOrdEqual,
5961	.neq => .OpFUnordNotEqual,
5962	.lt => .OpFOrdLessThan,
5963	.lte => .OpFOrdLessThanEqual,
5964	.gt => .OpFOrdGreaterThan,
5965	.gte => .OpFOrdGreaterThanEqual,
5966	},
5967	.bool => switch (op) {
5968	.eq => .OpLogicalEqual,
5969	.neq => .OpLogicalNotEqual,
5970	else => unreachable,
5971	},
5972	.integer, .strange_integer => switch (info.signedness) {
5973	.signed => switch (op) {
5974	.eq => .OpIEqual,
5975	.neq => .OpINotEqual,
5976	.lt => .OpSLessThan,
5977	.lte => .OpSLessThanEqual,
5978	.gt => .OpSGreaterThan,
5979	.gte => .OpSGreaterThanEqual,
5980	},
5981	.unsigned => switch (op) {
5982	.eq => .OpIEqual,
5983	.neq => .OpINotEqual,
5984	.lt => .OpULessThan,
5985	.lte => .OpULessThanEqual,
5986	.gt => .OpUGreaterThan,
5987	.gte => .OpUGreaterThanEqual,
5988	},
5989	},
5990	};
5991
5992	return try cg.buildCmp(pred, lhs, rhs);
5993	}
5994
5995	fn airCmp(
5996	cg: *CodeGen,
5997	inst: Air.Inst.Index,
5998	comptime op: std.math.CompareOperator,
5999	) !?Id {
6000	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
6001	const lhs = try cg.temporary(bin_op.lhs);
6002	const rhs = try cg.temporary(bin_op.rhs);
6003
6004	const result = try cg.cmp(op, lhs, rhs);
6005	return try result.materialize(cg);
6006	}
6007
6008	fn airVectorCmp(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6009	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6010	const vec_cmp = cg.air.extraData(Air.VectorCmp, ty_pl.payload).data;
6011	const lhs = try cg.temporary(vec_cmp.lhs);
6012	const rhs = try cg.temporary(vec_cmp.rhs);
6013	const op = vec_cmp.compareOperator();
6014
6015	const result = try cg.cmp(op, lhs, rhs);
6016	return try result.materialize(cg);
6017	}
6018
6019	/// Bitcast one type to another. Note: both types, input, output are expected in **direct** representation.
6020	fn bitCast(
6021	cg: *CodeGen,
6022	dst_ty: Type,
6023	src_ty: Type,
6024	src_id: Id,
6025	) !Id {
6026	const gpa = cg.gpa;
6027	const zcu = cg.zcu;
6028	const target = zcu.getTarget();
6029
6030	if (src_ty.toIntern() == dst_ty.toIntern()) return src_id;
6031	if (src_ty.isPtrAtRuntime(zcu) and dst_ty.isPtrAtRuntime(zcu)) switch (target.os.tag) {
6032	.vulkan, .opengl => if (src_ty.ptrAddressSpace(zcu) != .physical_storage_buffer) {
6033	const src_child = src_ty.childType(zcu);
6034	const dst_child = dst_ty.childType(zcu);
6035	if (!dst_child.hasRuntimeBits(zcu)) return src_id;
6036	if (src_child.toIntern() == dst_child.toIntern()) return src_id;
6037	if (src_ty.ptrInfo(zcu).packed_offset.host_size != 0 or
6038	dst_ty.ptrInfo(zcu).packed_offset.host_size != 0) return src_id;
6039
6040	var indices: std.ArrayList(u32) = .empty;
6041	defer indices.deinit(gpa);
6042	var cur = src_child;
6043	while (cur.toIntern() != dst_child.toIntern()) : (try indices.append(gpa, 0)) {
6044	cur = switch (cur.zigTypeTag(zcu)) {
6045	.array, .vector => cur.childType(zcu),
6046	.@"struct" => field: {
6047	for (0..cur.structFieldCount(zcu)) |i| {
6048	const field_ty = cur.fieldType(i, zcu);
6049	if (field_ty.hasRuntimeBits(zcu) and cur.structFieldOffset(i, zcu) == 0) break :field field_ty;
6050	}
6051	unreachable;
6052	},
6053	else => unreachable,
6054	};
6055	}
6056	const dst_ty_id = try cg.resolveType(dst_ty, .direct);
6057	return try cg.accessChain(dst_ty_id, src_id, indices.items);
6058	},
6059	else => {},
6060	};
6061
6062	const dst_ty_id = try cg.resolveType(dst_ty, .direct);
6063	const result_id = blk: {
6064	// Big-int ↔ big-int bitcast: the indirect representation is an array,
6065	// which OpBitcast cannot operate on. The arrays are bitwise identical
6066	// apart from the top limb's padding; the normalize pass below fixes
6067	// the padding.
6068	if (src_ty.isInt(zcu) and dst_ty.isInt(zcu)) {
6069	const src_info = src_ty.intInfo(zcu);
6070	const dst_info = dst_ty.intInfo(zcu);
6071	const src_backing, const src_big = cg.backingIntBits(src_info.bits);
6072	const dst_backing, const dst_big = cg.backingIntBits(dst_info.bits);
6073	if (src_backing == dst_backing and src_big and dst_big) break :blk src_id;
6074	}
6075
6076	// TODO: Some more cases are missing here
6077	// See fn bitCast in llvm.zig
6078
6079	if (src_ty.zigTypeTag(zcu) == .int and dst_ty.isPtrAtRuntime(zcu)) {
6080	if (target.os.tag != .opencl) {
6081	if (dst_ty.ptrAddressSpace(zcu) != .physical_storage_buffer) {
6082	return cg.fail(
6083	"cannot cast integer to pointer with address space '{s}'",
6084	.{@tagName(dst_ty.ptrAddressSpace(zcu))},
6085	);
6086	}
6087	}
6088
6089	const result_id = cg.allocId();
6090	try cg.body.emit(gpa, .OpConvertUToPtr, .{
6091	.id_result_type = dst_ty_id,
6092	.id_result = result_id,
6093	.integer_value = src_id,
6094	});
6095	break :blk result_id;
6096	}
6097
6098	// We can only use OpBitcast for specific conversions: between numerical types, and
6099	// between pointers. If the resolved spir-v types fall into this category then emit OpBitcast,
6100	// otherwise use a temporary and perform a pointer cast.
6101	const can_bitcast = (src_ty.isNumeric(zcu) and dst_ty.isNumeric(zcu)) or (src_ty.isPtrAtRuntime(zcu) and dst_ty.isPtrAtRuntime(zcu));
6102	if (can_bitcast) {
6103	const result_id = cg.allocId();
6104	try cg.body.emit(gpa, .OpBitcast, .{
6105	.id_result_type = dst_ty_id,
6106	.id_result = result_id,
6107	.operand = src_id,
6108	});
6109
6110	break :blk result_id;
6111	}
6112
6113	switch (target.os.tag) {
6114	.vulkan, .opengl => {
6115	// Logical addressing forbids OpBitcast on pointers. Allocate
6116	// the temp with dst_ty so the load reads through a slot of the right type.
6117	const dst_ty_indirect_id = try cg.resolveType(dst_ty, .indirect);
6118	const tmp_id = try cg.alloc(dst_ty_indirect_id, null);
6119	try cg.store(dst_ty, tmp_id, src_id, .{});
6120	break :blk try cg.load(dst_ty, tmp_id, .{});
6121	},
6122	else => {},
6123	}
6124
6125	const dst_ptr_ty_id = try cg.ptrType(dst_ty_id, .function);
6126
6127	const src_ty_indirect_id = try cg.resolveType(src_ty, .indirect);
6128	const tmp_id = try cg.alloc(src_ty_indirect_id, null);
6129	try cg.store(src_ty, tmp_id, src_id, .{});
6130	const casted_ptr_id = cg.allocId();
6131	try cg.body.emit(gpa, .OpBitcast, .{
6132	.id_result_type = dst_ptr_ty_id,
6133	.id_result = casted_ptr_id,
6134	.operand = tmp_id,
6135	});
6136	break :blk try cg.load(dst_ty, casted_ptr_id, .{});
6137	};
6138
6139	// Because strange integers use sign-extended representation, we may need to normalize
6140	// the result here.
6141	// TODO: This detail could cause stuff like @as(*const i1, @ptrCast(&@as(u1, 1))) to break
6142	// should we change the representation of strange integers?
6143	if (dst_ty.zigTypeTag(zcu) == .int) {
6144	const info = cg.arithmeticTypeInfo(dst_ty);
6145	const result = try cg.normalize(Temporary.init(dst_ty, result_id), info);
6146	return try result.materialize(cg);
6147	}
6148
6149	return result_id;
6150	}
6151
6152	fn airBitCast(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6153	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6154	const operand_ty = cg.typeOf(ty_op.operand);
6155	const result_ty = cg.typeOfIndex(inst);
6156	if (operand_ty.toIntern() == .bool_type) {
6157	const operand = try cg.temporary(ty_op.operand);
6158	const result = try cg.intFromBool(operand, .u1);
6159	return try result.materialize(cg);
6160	}
6161	if (operand_ty.zigTypeTag(cg.zcu) == .pointer) {
6162	switch (try cg.resolvePtr(ty_op.operand)) {
6163	.tracked => |t| return t.id, // TODO
6164	.id => |operand_id| return try cg.bitCast(result_ty, operand_ty, operand_id),
6165	}
6166	}
6167	const operand_id = try cg.resolve(ty_op.operand);
6168	return try cg.bitCast(result_ty, operand_ty, operand_id);
6169	}
6170
6171	fn airIntCast(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6172	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6173	const src = try cg.temporary(ty_op.operand);
6174	const dst_ty = cg.typeOfIndex(inst);
6175
6176	const src_info = cg.arithmeticTypeInfo(src.ty);
6177	const dst_info = cg.arithmeticTypeInfo(dst_ty);
6178
6179	const src_composite = src_info.class == .composite_integer;
6180	const dst_composite = dst_info.class == .composite_integer;
6181
6182	if (src_composite or dst_composite) {
6183	const gpa = cg.gpa;
6184	const scratch_top = cg.id_scratch.items.len;
6185	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
6186
6187	if (src_composite and dst_composite) {
6188	const src_id = try src.materialize(cg);
6189	const limb_bits = cg.bigIntBits();
6190	const limb_ty = cg.limbType();
6191	const limb_ty_id = try cg.limbTypeId();
6192	const src_n: u16 = src_info.backing_bits / limb_bits;
6193	const dst_n: u16 = dst_info.backing_bits / limb_bits;
6194	const result_limbs = try cg.id_scratch.addManyAsSlice(gpa, dst_n);
6195	const min_n = @min(src_n, dst_n);
6196	for (0..min_n) |i| {
6197	result_limbs[i] = cg.allocId();
6198	try cg.body.emit(gpa, .OpCompositeExtract, .{
6199	.id_result_type = limb_ty_id,
6200	.id_result = result_limbs[i],
6201	.composite = src_id,
6202	.indexes = &.{@as(u32, @intCast(i))},
6203	});
6204	}
6205	if (dst_n > src_n) {
6206	const fill = if (src_info.signedness == .signed) blk: {
6207	const signed_limb_ty: Type = if (limb_bits == 64) .i64 else .i32;
6208	const signed_limb_ty_id = try cg.resolveType(signed_limb_ty, .direct);
6209	const msb = result_limbs[src_n - 1];
6210	const msb_signed = cg.allocId();
6211	try cg.body.emit(gpa, .OpBitcast, .{
6212	.id_result_type = signed_limb_ty_id,
6213	.id_result = msb_signed,
6214	.operand = msb,
6215	});
6216	const shift_amt = try cg.constInt(signed_limb_ty, @as(u64, limb_bits - 1));
6217	const sign_ext = cg.allocId();
6218	try cg.body.emit(gpa, .OpShiftRightArithmetic, .{
6219	.id_result_type = signed_limb_ty_id,
6220	.id_result = sign_ext,
6221	.base = msb_signed,
6222	.shift = shift_amt,
6223	});
6224	const back = cg.allocId();
6225	try cg.body.emit(gpa, .OpBitcast, .{
6226	.id_result_type = limb_ty_id,
6227	.id_result = back,
6228	.operand = sign_ext,
6229	});
6230	break :blk back;
6231	} else try cg.constInt(limb_ty, @as(u64, 0));
6232	for (min_n..dst_n) |i| {
6233	result_limbs[i] = fill;
6234	}
6235	}
6236	const ci = CompositeInt.fromLimbs(cg, result_limbs, dst_info);
6237	const normalized = try ci.normalize();
6238	return try normalized.materialize(dst_ty);
6239	} else if (src_composite and !dst_composite) {
6240	const src_id = try src.materialize(cg);
6241	const limb_bits = cg.bigIntBits();
6242	const limb_ty = cg.limbType();
6243	const limb_ty_id = try cg.limbTypeId();
6244	if (dst_info.backing_bits <= limb_bits) {
6245	const limb0 = cg.allocId();
6246	try cg.body.emit(gpa, .OpCompositeExtract, .{
6247	.id_result_type = limb_ty_id,
6248	.id_result = limb0,
6249	.composite = src_id,
6250	.indexes = &.{@as(u32, 0)},
6251	});
6252	const tmp: Temporary = .init(limb_ty, limb0);
6253	const converted = try cg.buildConvert(dst_ty, tmp);
6254	const result = if (dst_info.bits < src_info.bits)
6255	try cg.normalize(converted, dst_info)
6256	else
6257	converted;
6258	return try result.materialize(cg);
6259	} else {
6260	assert(limb_bits == 32); // dst > 64 while limbs are 64 shouldn't happen — dst fits in one 64-bit limb.
6261	const limb0 = cg.allocId();
6262	try cg.body.emit(gpa, .OpCompositeExtract, .{
6263	.id_result_type = limb_ty_id,
6264	.id_result = limb0,
6265	.composite = src_id,
6266	.indexes = &.{@as(u32, 0)},
6267	});
6268	const limb1 = cg.allocId();
6269	try cg.body.emit(gpa, .OpCompositeExtract, .{
6270	.id_result_type = limb_ty_id,
6271	.id_result = limb1,
6272	.composite = src_id,
6273	.indexes = &.{@as(u32, 1)},
6274	});
6275	const u64_ty_id = try cg.resolveType(.u64, .direct);
6276	const lo = cg.allocId();
6277	try cg.body.emit(gpa, .OpUConvert, .{
6278	.id_result_type = u64_ty_id,
6279	.id_result = lo,
6280	.unsigned_value = limb0,
6281	});
6282	const hi = cg.allocId();
6283	try cg.body.emit(gpa, .OpUConvert, .{
6284	.id_result_type = u64_ty_id,
6285	.id_result = hi,
6286	.unsigned_value = limb1,
6287	});
6288	const shift32 = try cg.constInt(.u64, @as(u64, 32));
6289	const hi_shifted = cg.allocId();
6290	try cg.body.emit(gpa, .OpShiftLeftLogical, .{
6291	.id_result_type = u64_ty_id,
6292	.id_result = hi_shifted,
6293	.base = hi,
6294	.shift = shift32,
6295	});
6296	const combined = cg.allocId();
6297	try cg.body.emit(gpa, .OpBitwiseOr, .{
6298	.id_result_type = u64_ty_id,
6299	.id_result = combined,
6300	.operand_1 = lo,
6301	.operand_2 = hi_shifted,
6302	});
6303	const tmp: Temporary = .init(.u64, combined);
6304	const converted = try cg.buildConvert(dst_ty, tmp);
6305	const result = if (dst_info.bits < src_info.bits)
6306	try cg.normalize(converted, dst_info)
6307	else
6308	converted;
6309	return try result.materialize(cg);
6310	}
6311	} else {
6312	const limb_bits = cg.bigIntBits();
6313	const limb_ty = cg.limbType();
6314	const limb_ty_id = try cg.limbTypeId();
6315	const dst_n: u16 = dst_info.backing_bits / limb_bits;
6316	const result_limbs = try cg.id_scratch.addManyAsSlice(gpa, dst_n);
6317
6318	if (src_info.backing_bits <= limb_bits) {
6319	const converted = try cg.buildConvert(limb_ty, src);
6320	result_limbs[0] = try converted.materialize(cg);
6321	} else {
6322	const src_as_u64 = try cg.buildConvert(.u64, src);
6323	const src_id = try src_as_u64.materialize(cg);
6324	result_limbs[0] = cg.allocId();
6325	try cg.body.emit(gpa, .OpUConvert, .{
6326	.id_result_type = limb_ty_id,
6327	.id_result = result_limbs[0],
6328	.unsigned_value = src_id,
6329	});
6330	const u64_ty_id = try cg.resolveType(.u64, .direct);
6331	const shift32 = try cg.constInt(.u64, @as(u64, 32));
6332	const hi = cg.allocId();
6333	try cg.body.emit(gpa, .OpShiftRightLogical, .{
6334	.id_result_type = u64_ty_id,
6335	.id_result = hi,
6336	.base = src_id,
6337	.shift = shift32,
6338	});
6339	result_limbs[1] = cg.allocId();
6340	try cg.body.emit(gpa, .OpUConvert, .{
6341	.id_result_type = limb_ty_id,
6342	.id_result = result_limbs[1],
6343	.unsigned_value = hi,
6344	});
6345	}
6346	// Sign/zero-extend remaining limbs.
6347	const fill_start: u16 = if (src_info.backing_bits <= limb_bits) 1 else 2;
6348	const fill = if (src_info.signedness == .signed) blk: {
6349	const signed_limb_ty: Type = if (limb_bits == 64) .i64 else .i32;
6350	const signed_limb_ty_id = try cg.resolveType(signed_limb_ty, .direct);
6351	const msb = result_limbs[fill_start - 1];
6352	const msb_signed = cg.allocId();
6353	try cg.body.emit(gpa, .OpBitcast, .{
6354	.id_result_type = signed_limb_ty_id,
6355	.id_result = msb_signed,
6356	.operand = msb,
6357	});
6358	const shift_amt = try cg.constInt(signed_limb_ty, @as(u64, limb_bits - 1));
6359	const sign_ext = cg.allocId();
6360	try cg.body.emit(gpa, .OpShiftRightArithmetic, .{
6361	.id_result_type = signed_limb_ty_id,
6362	.id_result = sign_ext,
6363	.base = msb_signed,
6364	.shift = shift_amt,
6365	});
6366	const back = cg.allocId();
6367	try cg.body.emit(gpa, .OpBitcast, .{
6368	.id_result_type = limb_ty_id,
6369	.id_result = back,
6370	.operand = sign_ext,
6371	});
6372	break :blk back;
6373	} else try cg.constInt(limb_ty, @as(u64, 0));
6374	for (fill_start..dst_n) |i| {
6375	result_limbs[i] = fill;
6376	}
6377	const ci = CompositeInt.fromLimbs(cg, result_limbs, dst_info);
6378	const normalized = try ci.normalize();
6379	return try normalized.materialize(dst_ty);
6380	}
6381	}
6382
6383	if (src_info.backing_bits == dst_info.backing_bits) {
6384	const result = if (dst_info.bits < src_info.bits)
6385	try cg.normalize(src.pun(dst_ty), dst_info)
6386	else
6387	src.pun(dst_ty);
6388	return try result.materialize(cg);
6389	}
6390
6391	const converted = try cg.buildConvert(dst_ty, src);
6392
6393	// Make sure to normalize the result if shrinking.
6394	// Because strange ints are sign extended in their backing
6395	// type, we don't need to normalize when growing the type. The
6396	// representation is already the same.
6397	const result = if (dst_info.bits < src_info.bits)
6398	try cg.normalize(converted, dst_info)
6399	else
6400	converted;
6401
6402	return try result.materialize(cg);
6403	}
6404
6405	fn intFromPtr(cg: *CodeGen, operand_id: Id) !Id {
6406	const result_type_id = try cg.resolveType(.usize, .direct);
6407	const result_id = cg.allocId();
6408	try cg.body.emit(cg.gpa, .OpConvertPtrToU, .{
6409	.id_result_type = result_type_id,
6410	.id_result = result_id,
6411	.pointer = operand_id,
6412	});
6413	return result_id;
6414	}
6415
6416	fn airFloatFromInt(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6417	const gpa = cg.gpa;
6418	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6419	const operand_ty = cg.typeOf(ty_op.operand);
6420	const operand_id = try cg.resolve(ty_op.operand);
6421	const result_ty = cg.typeOfIndex(inst);
6422	const operand_info = cg.arithmeticTypeInfo(operand_ty);
6423	const result_id = cg.allocId();
6424	const result_ty_id = try cg.resolveType(result_ty, .direct);
6425	switch (operand_info.signedness) {
6426	.signed => try cg.body.emit(gpa, .OpConvertSToF, .{
6427	.id_result_type = result_ty_id,
6428	.id_result = result_id,
6429	.signed_value = operand_id,
6430	}),
6431	.unsigned => try cg.body.emit(gpa, .OpConvertUToF, .{
6432	.id_result_type = result_ty_id,
6433	.id_result = result_id,
6434	.unsigned_value = operand_id,
6435	}),
6436	}
6437	return result_id;
6438	}
6439
6440	fn airIntFromFloat(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6441	const gpa = cg.gpa;
6442	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6443	const operand_id = try cg.resolve(ty_op.operand);
6444	const result_ty = cg.typeOfIndex(inst);
6445	const result_info = cg.arithmeticTypeInfo(result_ty);
6446	const result_ty_id = try cg.resolveType(result_ty, .direct);
6447	const result_id = cg.allocId();
6448	switch (result_info.signedness) {
6449	.signed => try cg.body.emit(gpa, .OpConvertFToS, .{
6450	.id_result_type = result_ty_id,
6451	.id_result = result_id,
6452	.float_value = operand_id,
6453	}),
6454	.unsigned => try cg.body.emit(gpa, .OpConvertFToU, .{
6455	.id_result_type = result_ty_id,
6456	.id_result = result_id,
6457	.float_value = operand_id,
6458	}),
6459	}
6460	return result_id;
6461	}
6462
6463	fn airFloatCast(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6464	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6465	const operand = try cg.temporary(ty_op.operand);
6466	const dest_ty = cg.typeOfIndex(inst);
6467	const result = try cg.buildConvert(dest_ty, operand);
6468	return try result.materialize(cg);
6469	}
6470
6471	fn airNot(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6472	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6473	const operand = try cg.temporary(ty_op.operand);
6474	const result_ty = cg.typeOfIndex(inst);
6475	const info = cg.arithmeticTypeInfo(result_ty);
6476
6477	const result = switch (info.class) {
6478	.bool => try cg.buildUnary(.l_not, operand),
6479	.float => unreachable,
6480	.composite_integer => blk: {
6481	const op_id = try operand.materialize(cg);
6482	const scratch_top = cg.id_scratch.items.len;
6483	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
6484	const ci = try CompositeInt.init(cg, op_id, info);
6485	const notted = try ci.bitwiseNot();
6486	const normalized = try notted.normalize();
6487	break :blk Temporary.init(result_ty, try normalized.materialize(result_ty));
6488	},
6489	.strange_integer, .integer => blk: {
6490	const complement = try cg.buildUnary(.bit_not, operand);
6491	break :blk try cg.normalize(complement, info);
6492	},
6493	};
6494
6495	return try result.materialize(cg);
6496	}
6497
6498	fn airArrayToSlice(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6499	const zcu = cg.zcu;
6500	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6501	const array_ptr_ty = cg.typeOf(ty_op.operand);
6502	const array_ty = array_ptr_ty.childType(zcu);
6503	const slice_ty = cg.typeOfIndex(inst);
6504	const elem_ptr_ty = slice_ty.slicePtrFieldType(zcu);
6505
6506	const elem_ptr_ty_id = try cg.resolveType(elem_ptr_ty, .direct);
6507
6508	const array_ptr_id = try cg.resolve(ty_op.operand);
6509	const len_id = try cg.constInt(.usize, array_ty.arrayLen(zcu));
6510
6511	const elem_ptr_id = if (!array_ty.hasRuntimeBits(zcu))
6512	// Note: The pointer is something like *opaque{}, so we need to bitcast it to the element type.
6513	try cg.bitCast(elem_ptr_ty, array_ptr_ty, array_ptr_id)
6514	else
6515	// Convert the pointer-to-array to a pointer to the first element.
6516	try cg.accessChain(elem_ptr_ty_id, array_ptr_id, &.{0});
6517
6518	const slice_ty_id = try cg.resolveType(slice_ty, .direct);
6519	return try cg.constructComposite(slice_ty_id, &.{ elem_ptr_id, len_id });
6520	}
6521
6522	fn airSlice(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6523	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6524	const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
6525	const ptr_id = try cg.resolve(bin_op.lhs);
6526	const len_id = try cg.resolve(bin_op.rhs);
6527	const slice_ty = cg.typeOfIndex(inst);
6528	const slice_ty_id = try cg.resolveType(slice_ty, .direct);
6529	return try cg.constructComposite(slice_ty_id, &.{ ptr_id, len_id });
6530	}
6531
6532	fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6533	const gpa = cg.gpa;
6534	const pt = cg.pt;
6535	const zcu = cg.zcu;
6536	const ip = &zcu.intern_pool;
6537	const target = cg.zcu.getTarget();
6538	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6539	const result_ty = cg.typeOfIndex(inst);
6540	const len: usize = @intCast(result_ty.arrayLen(zcu));
6541	const elements: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[ty_pl.payload..][0..len]);
6542
6543	switch (result_ty.zigTypeTag(zcu)) {
6544	.@"struct" => {
6545	if (zcu.typeToPackedStruct(result_ty)) |struct_type| {
6546	comptime assert(Type.packed_struct_layout_version == 2);
6547	const backing_int_ty: Type = .fromInterned(struct_type.packed_backing_int_type);
6548	var running_int_id = try cg.constInt(backing_int_ty, 0);
6549	var running_bits: u16 = 0;
6550	for (struct_type.field_types.get(ip), elements) |field_ty_ip, element| {
6551	const field_ty: Type = .fromInterned(field_ty_ip);
6552	if (!field_ty.hasRuntimeBits(zcu)) continue;
6553	const field_id = try cg.resolve(element);
6554	const ty_bit_size: u16 = @intCast(field_ty.bitSize(zcu));
6555	const field_int_ty = try cg.pt.intType(.unsigned, ty_bit_size);
6556	const field_int_id = blk: {
6557	if (field_ty.isPtrAtRuntime(zcu)) {
6558	assert(target.cpu.arch == .spirv64 and
6559	field_ty.ptrAddressSpace(zcu) == .storage_buffer);
6560	break :blk try cg.intFromPtr(field_id);
6561	}
6562	break :blk try cg.bitCast(field_int_ty, field_ty, field_id);
6563	};
6564	const shift_rhs = try cg.constInt(backing_int_ty, running_bits);
6565	const extended_int_conv = try cg.buildConvert(backing_int_ty, .{
6566	.ty = field_int_ty,
6567	.value = .{ .singleton = field_int_id },
6568	});
6569	const shifted = try cg.buildBinary(.OpShiftLeftLogical, extended_int_conv, .{
6570	.ty = backing_int_ty,
6571	.value = .{ .singleton = shift_rhs },
6572	});
6573	const running_int_tmp = try cg.buildBinary(
6574	.OpBitwiseOr,
6575	.{ .ty = backing_int_ty, .value = .{ .singleton = running_int_id } },
6576	shifted,
6577	);
6578	running_int_id = try running_int_tmp.materialize(cg);
6579	running_bits += ty_bit_size;
6580	}
6581	return running_int_id;
6582	}
6583
6584	const scratch_top = cg.id_scratch.items.len;
6585	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
6586	const constituents = try cg.id_scratch.addManyAsSlice(gpa, elements.len);
6587
6588	const types = try gpa.alloc(Type, elements.len);
6589	defer gpa.free(types);
6590
6591	var index: usize = 0;
6592
6593	switch (ip.indexToKey(result_ty.toIntern())) {
6594	.tuple_type => |tuple| {
6595	for (tuple.types.get(ip), elements, 0..) |field_ty, element, i| {
6596	if ((try result_ty.structFieldValueComptime(pt, i)) != null) continue;
6597	assert(Type.fromInterned(field_ty).hasRuntimeBits(zcu));
6598
6599	const id = try cg.resolve(element);
6600	types[index] = .fromInterned(field_ty);
6601	constituents[index] = try cg.convertToIndirect(.fromInterned(field_ty), id);
6602	index += 1;
6603	}
6604	},
6605	.struct_type => {
6606	const struct_type = ip.loadStructType(result_ty.toIntern());
6607	var it = struct_type.iterateRuntimeOrder(ip);
6608	for (elements, 0..) |element, i| {
6609	const field_index = it.next().?;
6610	if ((try result_ty.structFieldValueComptime(pt, i)) != null) continue;
6611	const field_ty: Type = .fromInterned(struct_type.field_types.get(ip)[field_index]);
6612	assert(field_ty.hasRuntimeBits(zcu));
6613
6614	const id = try cg.resolve(element);
6615	types[index] = field_ty;
6616	constituents[index] = try cg.convertToIndirect(field_ty, id);
6617	index += 1;
6618	}
6619	},
6620	else => unreachable,
6621	}
6622
6623	const result_ty_id = try cg.resolveType(result_ty, .direct);
6624	return try cg.constructComposite(result_ty_id, constituents[0..index]);
6625	},
6626	.vector => {
6627	const n_elems = result_ty.vectorLen(zcu);
6628	const scratch_top = cg.id_scratch.items.len;
6629	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
6630	const elem_ids = try cg.id_scratch.addManyAsSlice(gpa, n_elems);
6631
6632	for (elements, 0..) |element, i| {
6633	elem_ids[i] = try cg.resolve(element);
6634	}
6635
6636	const result_ty_id = try cg.resolveType(result_ty, .direct);
6637	return try cg.constructComposite(result_ty_id, elem_ids);
6638	},
6639	.array => {
6640	const array_info = result_ty.arrayInfo(zcu);
6641	const n_elems: usize = @intCast(result_ty.arrayLenIncludingSentinel(zcu));
6642	const scratch_top = cg.id_scratch.items.len;
6643	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
6644	const elem_ids = try cg.id_scratch.addManyAsSlice(gpa, n_elems);
6645
6646	for (elements, 0..) |element, i| {
6647	const id = try cg.resolve(element);
6648	elem_ids[i] = try cg.convertToIndirect(array_info.elem_type, id);
6649	}
6650
6651	if (array_info.sentinel) |sentinel_val| {
6652	elem_ids[n_elems - 1] = try cg.constant(array_info.elem_type, sentinel_val, .indirect);
6653	}
6654
6655	const result_ty_id = try cg.resolveType(result_ty, .direct);
6656	return try cg.constructComposite(result_ty_id, elem_ids);
6657	},
6658	else => unreachable,
6659	}
6660	}
6661
6662	fn sliceOrArrayPtr(cg: *CodeGen, operand_id: Id, ty: Type) !Id {
6663	const zcu = cg.zcu;
6664	if (ty.isSlice(zcu)) {
6665	const ptr_ty = ty.slicePtrFieldType(zcu);
6666	return cg.extractField(ptr_ty, operand_id, 0);
6667	}
6668	return operand_id;
6669	}
6670
6671	fn airMemcpy(cg: *CodeGen, inst: Air.Inst.Index) !void {
6672	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
6673	const dest_slice = try cg.resolve(bin_op.lhs);
6674	const src_slice = try cg.resolve(bin_op.rhs);
6675	const dest_ty = cg.typeOf(bin_op.lhs);
6676	const src_ty = cg.typeOf(bin_op.rhs);
6677	const dest_ptr = try cg.sliceOrArrayPtr(dest_slice, dest_ty);
6678	const src_ptr = try cg.sliceOrArrayPtr(src_slice, src_ty);
6679	const len = switch (dest_ty.ptrSize(cg.zcu)) {
6680	.slice => try cg.extractField(.usize, dest_slice, 1),
6681	.one => len: {
6682	const array_ty = dest_ty.childType(cg.zcu);
6683	const elem_ty = array_ty.childType(cg.zcu);
6684	const size = array_ty.arrayLenIncludingSentinel(cg.zcu) * elem_ty.abiSize(cg.zcu);
6685	break :len try cg.constInt(.usize, size);
6686	},
6687	.many, .c => unreachable,
6688	};
6689	try cg.body.emit(cg.gpa, .OpCopyMemorySized, .{
6690	.target = dest_ptr,
6691	.source = src_ptr,
6692	.size = len,
6693	});
6694	}
6695
6696	fn airMemmove(cg: *CodeGen, inst: Air.Inst.Index) !void {
6697	_ = inst;
6698	return cg.fail("TODO implement airMemcpy for spirv", .{});
6699	}
6700
6701	fn airSliceField(cg: *CodeGen, inst: Air.Inst.Index, field: u32) !?Id {
6702	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6703	const field_ty = cg.typeOfIndex(inst);
6704	const operand_id = try cg.resolve(ty_op.operand);
6705	return try cg.extractField(field_ty, operand_id, field);
6706	}
6707
6708	fn airSpirvRuntimeArrayLen(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6709	const gpa = cg.gpa;
6710	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6711	const extra = cg.air.extraData(Air.StructField, ty_pl.payload).data;
6712	const struct_ptr_id = try cg.resolve(extra.struct_operand);
6713	const u32_ty_id = try cg.intType(.unsigned, 32);
6714	const result_id = cg.allocId();
6715	try cg.body.emit(gpa, .OpArrayLength, .{
6716	.id_result_type = u32_ty_id,
6717	.id_result = result_id,
6718	.structure = struct_ptr_id,
6719	.array_member = extra.field_index,
6720	});
6721	return result_id;
6722	}
6723
6724	fn airSliceElemPtr(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6725	const zcu = cg.zcu;
6726	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6727	const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
6728	const slice_ty = cg.typeOf(bin_op.lhs);
6729	if (!slice_ty.isVolatilePtr(zcu) and cg.liveness.isUnused(inst)) return null;
6730
6731	const slice_id = try cg.resolve(bin_op.lhs);
6732	const index_id = try cg.resolve(bin_op.rhs);
6733
6734	const ptr_ty = cg.typeOfIndex(inst);
6735	const ptr_ty_id = try cg.resolveType(ptr_ty, .direct);
6736
6737	const slice_ptr = try cg.extractField(ptr_ty, slice_id, 0);
6738	return try cg.ptrAccessChain(ptr_ty_id, slice_ptr, index_id, &.{});
6739	}
6740
6741	fn airSliceElemVal(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6742	const zcu = cg.zcu;
6743	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
6744	const slice_ty = cg.typeOf(bin_op.lhs);
6745	if (!slice_ty.isVolatilePtr(zcu) and cg.liveness.isUnused(inst)) return null;
6746
6747	const slice_id = try cg.resolve(bin_op.lhs);
6748	const index_id = try cg.resolve(bin_op.rhs);
6749
6750	const ptr_ty = slice_ty.slicePtrFieldType(zcu);
6751	const ptr_ty_id = try cg.resolveType(ptr_ty, .direct);
6752
6753	const slice_ptr = try cg.extractField(ptr_ty, slice_id, 0);
6754	const elem_ptr = try cg.ptrAccessChain(ptr_ty_id, slice_ptr, index_id, &.{});
6755	return try cg.load(slice_ty.childType(zcu), elem_ptr, .{ .is_volatile = slice_ty.isVolatilePtr(zcu) });
6756	}
6757
6758	fn ptrElemPtr(cg: *CodeGen, ptr_ty: Type, ptr_id: Id, index_id: Id) !Id {
6759	const zcu = cg.zcu;
6760	// Construct new pointer type for the resulting pointer
6761	const as = ptr_ty.ptrAddressSpace(zcu);
6762	const is_single_ptr = ptr_ty.isSinglePointer(zcu);
6763	const elem_is_block = cg.block_var_ids.contains(ptr_id);
6764	const elem_ty_id = try cg.pointeeType(as, ptr_ty.indexableElem(zcu), elem_is_block);
6765	const elem_ptr_ty_id = try cg.ptrType(elem_ty_id, cg.storageClass(as));
6766	if (is_single_ptr) {
6767	// Pointer-to-array. In this case, the resulting pointer is not of the same type
6768	// as the ptr_ty (we want a *T, not a *[N]T), and hence we need to use accessChain.
6769	return cg.accessChainId(elem_ptr_ty_id, ptr_id, &.{index_id});
6770	} else {
6771	// Resulting pointer type is the same as the ptr_ty, so use ptrAccessChain
6772	return cg.ptrAccessChain(elem_ptr_ty_id, ptr_id, index_id, &.{});
6773	}
6774	}
6775
6776	fn airPtrElemPtr(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6777	const zcu = cg.zcu;
6778	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6779	const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
6780	const src_ptr_ty = cg.typeOf(bin_op.lhs);
6781	const elem_ty = src_ptr_ty.childType(zcu);
6782	const ptr_id = try cg.resolve(bin_op.lhs);
6783
6784	assert(elem_ty.hasRuntimeBits(zcu));
6785
6786	const index_id = try cg.resolve(bin_op.rhs);
6787	return try cg.ptrElemPtr(src_ptr_ty, ptr_id, index_id);
6788	}
6789
6790	fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6791	const gpa = cg.gpa;
6792	const zcu = cg.zcu;
6793	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
6794	const array_ty = cg.typeOf(bin_op.lhs);
6795	const elem_ty = array_ty.childType(zcu);
6796	const array_id = try cg.resolve(bin_op.lhs);
6797	const index_id = try cg.resolve(bin_op.rhs);
6798
6799	// SPIR-V doesn't have an array indexing function for some damn reason.
6800	// For now, just generate a temporary and use that.
6801	// TODO: This backend probably also should use isByRef from llvm...
6802
6803	const is_vector = array_ty.isVector(zcu);
6804	const elem_repr: Repr = if (is_vector) .direct else .indirect;
6805	const array_ty_id = try cg.resolveType(array_ty, .direct);
6806	const elem_ty_id = try cg.resolveType(elem_ty, elem_repr);
6807	const ptr_array_ty_id = try cg.ptrType(array_ty_id, .function);
6808	const ptr_elem_ty_id = try cg.ptrType(elem_ty_id, .function);
6809
6810	const tmp_id = cg.allocId();
6811	try cg.prologue.emit(gpa, .OpVariable, .{
6812	.id_result_type = ptr_array_ty_id,
6813	.id_result = tmp_id,
6814	.storage_class = .function,
6815	});
6816
6817	try cg.body.emit(gpa, .OpStore, .{
6818	.pointer = tmp_id,
6819	.object = array_id,
6820	});
6821
6822	const elem_ptr_id = try cg.accessChainId(ptr_elem_ty_id, tmp_id, &.{index_id});
6823
6824	const result_id = cg.allocId();
6825	try cg.body.emit(gpa, .OpLoad, .{
6826	.id_result_type = try cg.resolveType(elem_ty, elem_repr),
6827	.id_result = result_id,
6828	.pointer = elem_ptr_id,
6829	});
6830
6831	if (is_vector) {
6832	// Result is already in direct representation
6833	return result_id;
6834	}
6835
6836	// This is an array type; the elements are stored in indirect representation.
6837	// We have to convert the type to direct.
6838
6839	return try cg.convertToDirect(elem_ty, result_id);
6840	}
6841
6842	fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6843	const zcu = cg.zcu;
6844	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
6845	const ptr_ty = cg.typeOf(bin_op.lhs);
6846	const elem_ty = cg.typeOfIndex(inst);
6847	const ptr_id = try cg.resolve(bin_op.lhs);
6848	const index_id = try cg.resolve(bin_op.rhs);
6849	const elem_ptr_id = try cg.ptrElemPtr(ptr_ty, ptr_id, index_id);
6850	return try cg.load(elem_ty, elem_ptr_id, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) });
6851	}
6852
6853	fn airSetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) !void {
6854	const zcu = cg.zcu;
6855	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
6856	const un_ptr_ty = cg.typeOf(bin_op.lhs);
6857	const un_ty = un_ptr_ty.childType(zcu);
6858	const layout = cg.unionLayout(un_ty);
6859
6860	if (layout.tag_size == 0) return;
6861
6862	const tag_ty = un_ty.unionTagTypeRuntime(zcu).?;
6863	const tag_ty_id = try cg.resolveType(tag_ty, .indirect);
6864	const tag_ptr_ty_id = try cg.ptrType(tag_ty_id, cg.storageClass(un_ptr_ty.ptrAddressSpace(zcu)));
6865
6866	const union_ptr_id = try cg.resolve(bin_op.lhs);
6867	const new_tag_id = try cg.resolve(bin_op.rhs);
6868
6869	if (!layout.has_payload) {
6870	try cg.store(tag_ty, union_ptr_id, new_tag_id, .{ .is_volatile = un_ptr_ty.isVolatilePtr(zcu) });
6871	} else {
6872	const ptr_id = try cg.accessChain(tag_ptr_ty_id, union_ptr_id, &.{layout.tag_index});
6873	try cg.store(tag_ty, ptr_id, new_tag_id, .{ .is_volatile = un_ptr_ty.isVolatilePtr(zcu) });
6874	}
6875	}
6876
6877	fn airGetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6878	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6879	const un_ty = cg.typeOf(ty_op.operand);
6880
6881	const zcu = cg.zcu;
6882	const layout = cg.unionLayout(un_ty);
6883	if (layout.tag_size == 0) return null;
6884
6885	const union_handle = try cg.resolve(ty_op.operand);
6886	if (!layout.has_payload) return union_handle;
6887
6888	const tag_ty = un_ty.unionTagTypeRuntime(zcu).?;
6889	return try cg.extractField(tag_ty, union_handle, layout.tag_index);
6890	}
6891
6892	fn unionInit(
6893	cg: *CodeGen,
6894	ty: Type,
6895	active_field: u32,
6896	payload: ?Id,
6897	) !Id {
6898	// To initialize a union, generate a temporary variable with the
6899	// union type, then get the field pointer and pointer-cast it to the
6900	// right type to store it. Finally load the entire union.
6901
6902	// Note: The result here is not cached, because it generates runtime code.
6903
6904	const pt = cg.pt;
6905	const zcu = cg.zcu;
6906	const ip = &zcu.intern_pool;
6907	const union_ty = zcu.typeToUnion(ty).?;
6908	const tag_ty: Type = .fromInterned(union_ty.enum_tag_type);
6909
6910	const layout = cg.unionLayout(ty);
6911	const payload_ty: Type = .fromInterned(union_ty.field_types.get(ip)[active_field]);
6912
6913	assert(union_ty.layout != .@"packed");
6914
6915	const tag_int = if (layout.tag_size != 0) blk: {
6916	const tag_val = try pt.enumValueFieldIndex(tag_ty, active_field);
6917	const tag_int_val = tag_val.backingInt(zcu);
6918	break :blk tag_int_val.toUnsignedInt(zcu);
6919	} else 0;
6920
6921	if (!layout.has_payload) {
6922	return try cg.constInt(tag_ty, tag_int);
6923	}
6924
6925	const ty_id = try cg.resolveType(ty, .indirect);
6926	const tmp_id = try cg.alloc(ty_id, null);
6927
6928	if (layout.tag_size != 0) {
6929	const tag_ty_id = try cg.resolveType(tag_ty, .indirect);
6930	const tag_ptr_ty_id = try cg.ptrType(tag_ty_id, .function);
6931	const ptr_id = try cg.accessChain(tag_ptr_ty_id, tmp_id, &.{@as(u32, @intCast(layout.tag_index))});
6932	const tag_id = try cg.constInt(tag_ty, tag_int);
6933	try cg.store(tag_ty, ptr_id, tag_id, .{});
6934	}
6935
6936	if (payload_ty.hasRuntimeBits(zcu)) {
6937	const layout_payload_ty_id = try cg.resolveType(layout.payload_ty, .indirect);
6938	const pl_ptr_ty_id = try cg.ptrType(layout_payload_ty_id, .function);
6939	const pl_ptr_id = try cg.accessChain(pl_ptr_ty_id, tmp_id, &.{layout.payload_index});
6940	const active_pl_ptr_id = if (!layout.payload_ty.eql(payload_ty)) blk: {
6941	const payload_ty_id = try cg.resolveType(payload_ty, .indirect);
6942	const active_pl_ptr_ty_id = try cg.ptrType(payload_ty_id, .function);
6943	const active_pl_ptr_id = cg.allocId();
6944	try cg.body.emit(cg.gpa, .OpBitcast, .{
6945	.id_result_type = active_pl_ptr_ty_id,
6946	.id_result = active_pl_ptr_id,
6947	.operand = pl_ptr_id,
6948	});
6949	break :blk active_pl_ptr_id;
6950	} else pl_ptr_id;
6951
6952	try cg.store(payload_ty, active_pl_ptr_id, payload.?, .{});
6953	} else {
6954	assert(payload == null);
6955	}
6956
6957	// Just leave the padding fields uninitialized...
6958	// TODO: Or should we initialize them with undef explicitly?
6959
6960	return try cg.load(ty, tmp_id, .{});
6961	}
6962
6963	fn airUnionInit(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6964	const zcu = cg.zcu;
6965	const ip = &zcu.intern_pool;
6966	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6967	const extra = cg.air.extraData(Air.UnionInit, ty_pl.payload).data;
6968	const ty = cg.typeOfIndex(inst);
6969
6970	const union_obj = zcu.typeToUnion(ty).?;
6971	const field_ty: Type = .fromInterned(union_obj.field_types.get(ip)[extra.field_index]);
6972	const payload = if (field_ty.hasRuntimeBits(zcu))
6973	try cg.resolve(extra.init)
6974	else
6975	null;
6976	return try cg.unionInit(ty, extra.field_index, payload);
6977	}
6978
6979	fn airAggFieldVal(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6980	const pt = cg.pt;
6981	const zcu = cg.zcu;
6982	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6983	const struct_field = cg.air.extraData(Air.StructField, ty_pl.payload).data;
6984
6985	const object_ty = cg.typeOf(struct_field.struct_operand);
6986	const object_id = try cg.resolve(struct_field.struct_operand);
6987	const field_index = struct_field.field_index;
6988	const field_ty = object_ty.fieldType(field_index, zcu);
6989
6990	assert(field_ty.hasRuntimeBits(zcu));
6991
6992	switch (object_ty.zigTypeTag(zcu)) {
6993	.@"struct" => switch (object_ty.containerLayout(zcu)) {
6994	.@"packed" => {
6995	const struct_ty = zcu.typeToPackedStruct(object_ty).?;
6996	const struct_backing_int_bits = cg.backingIntBits(@intCast(object_ty.bitSize(zcu))).@"0";
6997	const bit_offset = zcu.structPackedFieldBitOffset(struct_ty, field_index);
6998	// We use the same int type the packed struct is backed by, because even though it would
6999	// be valid SPIR-V to use an smaller type like u16, some implementations like PoCL will complain.
7000	const bit_offset_id = try cg.constInt(object_ty, bit_offset);
7001	const signedness = if (field_ty.isInt(zcu)) field_ty.intInfo(zcu).signedness else .unsigned;
7002	const field_bit_size: u16 = @intCast(field_ty.bitSize(zcu));
7003	const field_int_ty = try pt.intType(signedness, field_bit_size);
7004	const shift_lhs: Temporary = .{ .ty = object_ty, .value = .{ .singleton = object_id } };
7005	const shift = try cg.buildBinary(.OpShiftRightLogical, shift_lhs, .{ .ty = object_ty, .value = .{ .singleton = bit_offset_id } });
7006	const mask_id = try cg.constInt(object_ty, (@as(u64, 1) << @as(u6, @intCast(field_bit_size))) - 1);
7007	const masked = try cg.buildBinary(.OpBitwiseAnd, shift, .{ .ty = object_ty, .value = .{ .singleton = mask_id } });
7008	const result_id = blk: {
7009	if (cg.backingIntBits(field_bit_size).@"0" == struct_backing_int_bits)
7010	break :blk try cg.bitCast(field_int_ty, object_ty, try masked.materialize(cg));
7011	const trunc = try cg.buildConvert(field_int_ty, masked);
7012	break :blk try trunc.materialize(cg);
7013	};
7014	if (field_ty.ip_index == .bool_type) return try cg.convertToDirect(.bool, result_id);
7015	if (field_ty.isInt(zcu)) return result_id;
7016	return try cg.bitCast(field_ty, field_int_ty, result_id);
7017	},
7018	else => return try cg.extractField(field_ty, object_id, field_index),
7019	},
7020	.@"union" => switch (object_ty.containerLayout(zcu)) {
7021	.@"packed" => {
7022	const backing_int_ty = try pt.intType(.unsigned, @intCast(object_ty.bitSize(zcu)));
7023	const signedness = if (field_ty.isInt(zcu)) field_ty.intInfo(zcu).signedness else .unsigned;
7024	const field_bit_size: u16 = @intCast(field_ty.bitSize(zcu));
7025	const int_ty = try pt.intType(signedness, field_bit_size);
7026	const mask_id = try cg.constInt(backing_int_ty, (@as(u64, 1) << @as(u6, @intCast(field_bit_size))) - 1);
7027	const masked = try cg.buildBinary(
7028	.OpBitwiseAnd,
7029	.{ .ty = backing_int_ty, .value = .{ .singleton = object_id } },
7030	.{ .ty = backing_int_ty, .value = .{ .singleton = mask_id } },
7031	);
7032	const result_id = blk: {
7033	if (cg.backingIntBits(field_bit_size).@"0" == cg.backingIntBits(@intCast(backing_int_ty.bitSize(zcu))).@"0")
7034	break :blk try cg.bitCast(int_ty, backing_int_ty, try masked.materialize(cg));
7035	const trunc = try cg.buildConvert(int_ty, masked);
7036	break :blk try trunc.materialize(cg);
7037	};
7038	if (field_ty.ip_index == .bool_type) return try cg.convertToDirect(.bool, result_id);
7039	if (field_ty.isInt(zcu)) return result_id;
7040	return try cg.bitCast(field_ty, int_ty, result_id);
7041	},
7042	else => {
7043	// Store, ptr-elem-ptr, pointer-cast, load
7044	const layout = cg.unionLayout(object_ty);
7045	assert(layout.has_payload);
7046
7047	const object_ty_id = try cg.resolveType(object_ty, .indirect);
7048	const tmp_id = try cg.alloc(object_ty_id, null);
7049	try cg.store(object_ty, tmp_id, object_id, .{});
7050
7051	const layout_payload_ty_id = try cg.resolveType(layout.payload_ty, .indirect);
7052	const pl_ptr_ty_id = try cg.ptrType(layout_payload_ty_id, .function);
7053	const pl_ptr_id = try cg.accessChain(pl_ptr_ty_id, tmp_id, &.{layout.payload_index});
7054
7055	if (field_ty.toIntern() == layout.payload_ty.toIntern()) {
7056	return try cg.load(field_ty, pl_ptr_id, .{});
7057	}
7058
7059	switch (zcu.getTarget().os.tag) {
7060	.vulkan, .opengl => {
7061	// Logical addressing forbids OpBitcast on pointers. Load the
7062	// payload as its type and bitcast the value instead.
7063	const payload_id = try cg.load(layout.payload_ty, pl_ptr_id, .{});
7064	return try cg.bitCast(field_ty, layout.payload_ty, payload_id);
7065	},
7066	else => {},
7067	}
7068
7069	const field_ty_id = try cg.resolveType(field_ty, .indirect);
7070	const active_pl_ptr_ty_id = try cg.ptrType(field_ty_id, .function);
7071	const active_pl_ptr_id = cg.allocId();
7072	try cg.body.emit(cg.gpa, .OpBitcast, .{
7073	.id_result_type = active_pl_ptr_ty_id,
7074	.id_result = active_pl_ptr_id,
7075	.operand = pl_ptr_id,
7076	});
7077	return try cg.load(field_ty, active_pl_ptr_id, .{});
7078	},
7079	},
7080	else => unreachable,
7081	}
7082	}
7083
7084	fn airFieldParentPtr(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
7085	const zcu = cg.zcu;
7086	const target = zcu.getTarget();
7087	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
7088	const extra = cg.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
7089
7090	const parent_ptr_ty = ty_pl.ty;
7091	const parent_ty = parent_ptr_ty.childType(zcu);
7092	const result_ty_id = try cg.resolveType(parent_ptr_ty, .indirect);
7093
7094	const field_ptr = try cg.resolve(extra.field_ptr);
7095	const field_ptr_ty = cg.typeOf(extra.field_ptr);
7096	const field_ptr_int = try cg.intFromPtr(field_ptr);
7097	const field_offset = parent_ty.structFieldOffset(extra.field_index, zcu);
7098
7099	const base_ptr_int = base_ptr_int: {
7100	if (field_offset == 0) break :base_ptr_int field_ptr_int;
7101
7102	const field_offset_id = try cg.constInt(.usize, field_offset);
7103	const field_ptr_tmp: Temporary = .init(.usize, field_ptr_int);
7104	const field_offset_tmp: Temporary = .init(.usize, field_offset_id);
7105	const result = try cg.buildBinary(.OpISub, field_ptr_tmp, field_offset_tmp);
7106	break :base_ptr_int try result.materialize(cg);
7107	};
7108
7109	if (target.os.tag != .opencl) {
7110	if (field_ptr_ty.ptrAddressSpace(zcu) != .physical_storage_buffer) {
7111	return cg.fail(
7112	"cannot cast integer to pointer with address space '{s}'",
7113	.{@tagName(field_ptr_ty.ptrAddressSpace(zcu))},
7114	);
7115	}
7116	}
7117
7118	const base_ptr = cg.allocId();
7119	try cg.body.emit(cg.gpa, .OpConvertUToPtr, .{
7120	.id_result_type = result_ty_id,
7121	.id_result = base_ptr,
7122	.integer_value = base_ptr_int,
7123	});
7124
7125	return base_ptr;
7126	}
7127
7128	fn structFieldPtr(
7129	cg: *CodeGen,
7130	result_ptr_ty: Type,
7131	object_ptr_ty: Type,
7132	object_ptr: Id,
7133	field_index: u32,
7134	) !Id {
7135	const result_ty_id = try cg.resolveType(result_ptr_ty, .direct);
7136
7137	const zcu = cg.zcu;
7138	const object_ty = object_ptr_ty.childType(zcu);
7139	switch (object_ty.zigTypeTag(zcu)) {
7140	.pointer => {
7141	assert(object_ty.isSlice(zcu));
7142	return cg.accessChain(result_ty_id, object_ptr, &.{field_index});
7143	},
7144	.@"struct" => switch (object_ty.containerLayout(zcu)) {
7145	.@"packed" => {
7146	const byte_offset = codegen.fieldOffset(object_ptr_ty, result_ptr_ty, field_index, zcu);
7147	if (byte_offset == 0) return object_ptr;
7148	const usize_ty_id = try cg.resolveType(.usize, .direct);
7149	const base_int = cg.allocId();
7150	try cg.body.emit(cg.gpa, .OpConvertPtrToU, .{
7151	.id_result_type = usize_ty_id,
7152	.id_result = base_int,
7153	.pointer = object_ptr,
7154	});
7155	const offset_id = try cg.constInt(.usize, byte_offset);
7156	const adjusted = try cg.buildBinary(.OpIAdd, .{ .ty = .usize, .value = .{ .singleton = base_int } }, .{ .ty = .usize, .value = .{ .singleton = offset_id } });
7157	const adjusted_id = try adjusted.materialize(cg);
7158	const result_id = cg.allocId();
7159	try cg.body.emit(cg.gpa, .OpConvertUToPtr, .{
7160	.id_result_type = result_ty_id,
7161	.id_result = result_id,
7162	.integer_value = adjusted_id,
7163	});
7164	return result_id;
7165	},
7166	.auto, .@"extern" => {
7167	return try cg.accessChain(result_ty_id, object_ptr, &.{field_index});
7168	},
7169	},
7170	.@"union" => switch (object_ty.containerLayout(zcu)) {
7171	.@"packed" => return cg.todo("implement field access for packed unions", .{}),
7172	.auto, .@"extern" => {
7173	const layout = cg.unionLayout(object_ty);
7174	if (!layout.has_payload) {
7175	// Asked to get a pointer to a zero-sized field. Just lower this
7176	// to undefined, there is no reason to make it be a valid pointer.
7177	return try cg.constUndef(result_ty_id);
7178	}
7179
7180	const storage_class = cg.storageClass(object_ptr_ty.ptrAddressSpace(zcu));
7181	const field_ty = result_ptr_ty.childType(zcu);
7182	if (field_ty.toIntern() == layout.payload_ty.toIntern()) {
7183	if (object_ty.containerLayout(zcu) == .@"packed") return object_ptr;
7184	return try cg.accessChain(result_ty_id, object_ptr, &.{layout.payload_index});
7185	}
7186
7187	switch (zcu.getTarget().os.tag) {
7188	.vulkan, .opengl => {
7189	// Logical addressing forbids OpBitcast on pointers. If the field
7190	// type is structurally identical to the payload type (dedup will
7191	// unify them) the access chain typed as the field type is valid.
7192	if (object_ty.containerLayout(zcu) == .@"packed") return object_ptr;
7193	return try cg.accessChain(result_ty_id, object_ptr, &.{layout.payload_index});
7194	},
7195	else => {},
7196	}
7197
7198	const layout_payload_ty_id = try cg.resolveType(layout.payload_ty, .indirect);
7199	const pl_ptr_ty_id = try cg.ptrType(layout_payload_ty_id, storage_class);
7200	const pl_ptr_id = blk: {
7201	if (object_ty.containerLayout(zcu) == .@"packed") break :blk object_ptr;
7202	break :blk try cg.accessChain(pl_ptr_ty_id, object_ptr, &.{layout.payload_index});
7203	};
7204
7205	const active_pl_ptr_id = cg.allocId();
7206	try cg.body.emit(cg.gpa, .OpBitcast, .{
7207	.id_result_type = result_ty_id,
7208	.id_result = active_pl_ptr_id,
7209	.operand = pl_ptr_id,
7210	});
7211	return active_pl_ptr_id;
7212	},
7213	},
7214	else => unreachable,
7215	}
7216	}
7217
7218	fn airStructFieldPtr(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
7219	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
7220	const struct_field = cg.air.extraData(Air.StructField, ty_pl.payload).data;
7221	const struct_ptr = try cg.resolve(struct_field.struct_operand);
7222	const struct_ptr_ty = cg.typeOf(struct_field.struct_operand);
7223	const result_ptr_ty = cg.typeOfIndex(inst);
7224	return try cg.structFieldPtr(result_ptr_ty, struct_ptr_ty, struct_ptr, struct_field.field_index);
7225	}
7226
7227	fn airStructFieldPtrIndex(cg: *CodeGen, inst: Air.Inst.Index, field_index: u32) !?Id {
7228	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
7229	const struct_ptr = try cg.resolve(ty_op.operand);
7230	const struct_ptr_ty = cg.typeOf(ty_op.operand);
7231	const result_ptr_ty = cg.typeOfIndex(inst);
7232	return try cg.structFieldPtr(result_ptr_ty, struct_ptr_ty, struct_ptr, field_index);
7233	}
7234
7235	fn alloc(cg: *CodeGen, ty_id: Id, initializer: ?Id) !Id {
7236	const ptr_ty_id = try cg.ptrType(ty_id, .function);
7237	const result_id = cg.allocId();
7238	try cg.prologue.emit(cg.gpa, .OpVariable, .{
7239	.id_result_type = ptr_ty_id,
7240	.id_result = result_id,
7241	.storage_class = .function,
7242	.initializer = initializer,
7243	});
7244	return result_id;
7245	}
7246
7247	fn airAlloc(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
7248	const zcu = cg.zcu;
7249	const target = zcu.getTarget();
7250	const ptr_ty = cg.typeOfIndex(inst);
7251	const child_ty = ptr_ty.childType(zcu);
7252
7253	switch (target.os.tag) {
7254	.vulkan, .opengl => {
7255	if (child_ty.zigTypeTag(zcu) == .pointer and !child_ty.isSlice(zcu)) {
7256	const as = child_ty.ptrAddressSpace(zcu);
7257	if (cg.storageClass(as) == .function) {
7258	const result_id = cg.allocId();
7259	try cg.tracked_allocas.put(cg.gpa, result_id, null);
7260	return result_id;
7261	}
7262	}
7263	},
7264	else => {},
7265	}
7266
7267	const child_ty_id = try cg.resolveType(child_ty, .indirect);
7268	const ptr_align = ptr_ty.ptrAlignment(zcu);
7269	const result_id = try cg.alloc(child_ty_id, null);
7270	if (ptr_align != child_ty.abiAlignment(zcu)) {
7271	if (target.os.tag != .opencl) return cg.fail("cannot apply alignment to variables", .{});
7272	try cg.decorate(result_id, .{
7273	.alignment = .{ .alignment = @intCast(ptr_align.toByteUnits().?) },
7274	});
7275	}
7276	return result_id;
7277	}
7278
7279	fn airArg(cg: *CodeGen) Id {
7280	defer cg.next_arg_index += 1;
7281	return cg.args.items[cg.next_arg_index];
7282	}
7283
7284	/// Given a slice of incoming block connections, returns the block-id of the next
7285	/// block to jump to. This function emits instructions, so it should be emitted
7286	/// inside the merge block of the block.
7287	/// This function should only be called with structured control flow generation.
7288	fn structuredNextBlock(cg: *CodeGen, incoming: []const Block.Incoming) !Id {
7289	const result_id = cg.allocId();
7290	const block_id_ty_id = try cg.resolveType(.u32, .direct);
7291	try cg.body.emitRaw(cg.gpa, .OpPhi, @intCast(2 + incoming.len * 2)); // result type + result + variable/parent...
7292	cg.body.writeOperand(Id, block_id_ty_id);
7293	cg.body.writeOperand(Id, result_id);
7294
7295	for (incoming) |incoming_block| {
7296	cg.body.writeOperand(spec.PairIdRefIdRef, .{ incoming_block.next_block, incoming_block.src_label });
7297	}
7298
7299	return result_id;
7300	}
7301
7302	/// Jumps to the block with the target block-id. This function must only be called when
7303	/// terminating a body, there should be no instructions after it.
7304	/// This function should only be called with structured control flow generation.
7305	fn structuredBreak(cg: *CodeGen, target_block: Id) !void {
7306	if (cg.block_terminated) return;
7307
7308	const gpa = cg.gpa;
7309	const sblock = cg.block_stack.last().?;
7310	const merge_block = switch (sblock.*) {
7311	.selection => |*merge| blk: {
7312	const merge_label = cg.allocId();
7313	try merge.merge_stack.append(gpa, .{
7314	.incoming = .{
7315	.src_label = cg.block_label,
7316	.next_block = target_block,
7317	},
7318	.merge_block = merge_label,
7319	});
7320	break :blk merge_label;
7321	},
7322	// Loop blocks do not end in a break. Not through a direct break,
7323	// and also not through another instruction like cond_br or unreachable (these
7324	// situations are replaced by `cond_br` in sema, or there is a `block` instruction
7325	// placed around them).
7326	.loop => unreachable,
7327	};
7328
7329	try cg.body.emit(gpa, .OpBranch, .{ .target_label = merge_block });
7330	}
7331
7332	/// Generate a body in a way that exits the body using only structured constructs.
7333	/// Returns the block-id of the next block to jump to. After this function, a jump
7334	/// should still be emitted to the block that should follow this structured body.
7335	/// This function should only be called with structured control flow generation.
7336	fn genStructuredBody(
7337	cg: *CodeGen,
7338	/// This parameter defines the method that this structured body is exited with.
7339	block_merge_type: union(enum) {
7340	/// Using selection; early exits from this body are surrounded with
7341	/// if() statements.
7342	selection,
7343	/// Using loops; loops can be early exited by jumping to the merge block at
7344	/// any time.
7345	loop: struct {
7346	merge_label: Id,
7347	continue_label: Id,
7348	},
7349	},
7350	body: []const Air.Inst.Index,
7351	) !Id {
7352	const gpa = cg.gpa;
7353
7354	var sblock: Block = switch (block_merge_type) {
7355	.loop => |merge| .{ .loop = .{
7356	.merge_block = merge.merge_label,
7357	} },
7358	.selection => .{ .selection = .{} },
7359	};
7360	defer sblock.deinit(gpa);
7361
7362	{
7363	try cg.block_stack.append(gpa, &sblock);
7364	defer _ = cg.block_stack.pop();
7365
7366	try cg.genBody(body);
7367	}
7368
7369	switch (sblock) {
7370	.selection => |merge| {
7371	// Now generate the merge block for all merges that
7372	// still need to be performed.
7373	const merge_stack = merge.merge_stack.items;
7374
7375	// If no merges on the stack, this block didn't generate any jumps (all paths
7376	// ended with a return or an unreachable). In that case, we don't need to do
7377	// any merging.
7378	if (merge_stack.len == 0) {
7379	// We still need to return a value of a next block to jump to.
7380	// For example, if we have code like
7381	// if (x) {
7382	// if (y) return else return;
7383	// } else {}
7384	// then we still need the outer to have an OpSelectionMerge and consequently
7385	// a phi node. In that case we can just return bogus, since we know that its
7386	// path will never be taken.
7387
7388	// Make sure that we are still in a block when exiting the function.
7389	// TODO: Can we get rid of that?
7390	try cg.beginSpvBlock(cg.allocId());
7391	const block_id_ty_id = try cg.resolveType(.u32, .direct);
7392	return try cg.constUndef(block_id_ty_id);
7393	}
7394
7395	// The top-most merge actually only has a single source, the
7396	// final jump of the block, or the merge block of a sub-block, cond_br,
7397	// or loop. Therefore we just need to generate a block with a jump to the
7398	// next merge block.
7399	try cg.beginSpvBlock(merge_stack[merge_stack.len - 1].merge_block);
7400
7401	// Now generate a merge ladder for the remaining merges in the stack.
7402	var incoming: Block.Incoming = .{
7403	.src_label = cg.block_label,
7404	.next_block = merge_stack[merge_stack.len - 1].incoming.next_block,
7405	};
7406	var i = merge_stack.len - 1;
7407	while (i > 0) {
7408	i -= 1;
7409	const step = merge_stack[i];
7410
7411	try cg.body.emit(gpa, .OpBranch, .{ .target_label = step.merge_block });
7412	try cg.beginSpvBlock(step.merge_block);
7413	const next_block = try cg.structuredNextBlock(&.{ incoming, step.incoming });
7414	incoming = .{
7415	.src_label = step.merge_block,
7416	.next_block = next_block,
7417	};
7418	}
7419
7420	return incoming.next_block;
7421	},
7422	.loop => |merge| {
7423	// Close the loop by jumping to the continue label
7424
7425	try cg.body.emit(gpa, .OpBranch, .{ .target_label = block_merge_type.loop.continue_label });
7426	// For blocks we must simple merge all the incoming blocks to get the next block.
7427	try cg.beginSpvBlock(merge.merge_block);
7428	return try cg.structuredNextBlock(merge.merges.items);
7429	},
7430	}
7431	}
7432
7433	fn airBlock(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
7434	const block = cg.air.unwrapBlock(inst);
7435	return cg.lowerBlock(inst, block.body);
7436	}
7437
7438	fn lowerBlock(cg: *CodeGen, inst: Air.Inst.Index, body: []const Air.Inst.Index) !?Id {
7439	// In AIR, a block doesn't really define an entry point like a block, but
7440	// more like a scope that breaks can jump out of and "return" a value from.
7441	// This cannot be directly modelled in SPIR-V, so in a block instruction,
7442	// we're going to split up the current block by first generating the code
7443	// of the block, then a label, and then generate the rest of the current
7444	// ir.Block in a different SPIR-V block.
7445
7446	const gpa = cg.gpa;
7447	const zcu = cg.zcu;
7448	const ty = cg.typeOfIndex(inst);
7449	const have_block_result = ty.hasRuntimeBits(zcu);
7450
7451	const maybe_block_result_var_id = if (have_block_result) blk: {
7452	const ty_id = try cg.resolveType(ty, .indirect);
7453	const block_result_var_id = try cg.alloc(ty_id, null);
7454	try cg.block_results.putNoClobber(gpa, inst, block_result_var_id);
7455	break :blk block_result_var_id;
7456	} else null;
7457	defer if (have_block_result) assert(cg.block_results.remove(inst));
7458
7459	const next_block = try cg.genStructuredBody(.selection, body);
7460
7461	// When encountering a block instruction, we are always at least in the function's scope,
7462	// so there always has to be another entry.
7463	assert(cg.block_stack.items.len > 0);
7464
7465	// Check if the target of the branch was this current block.
7466	const this_block = try cg.constInt(.u32, @backingInt(inst));
7467	const jump_to_this_block_id = cg.allocId();
7468	const bool_ty_id = try cg.resolveType(.bool, .direct);
7469	try cg.body.emit(gpa, .OpIEqual, .{
7470	.id_result_type = bool_ty_id,
7471	.id_result = jump_to_this_block_id,
7472	.operand_1 = next_block,
7473	.operand_2 = this_block,
7474	});
7475
7476	const sblock = cg.block_stack.last().?;
7477
7478	if (ty.isNoReturn(zcu)) {
7479	// If this block is noreturn, this instruction is the last of a block,
7480	// and we must simply jump to the block's merge unconditionally.
7481	try cg.structuredBreak(next_block);
7482	} else {
7483	switch (sblock.*) {
7484	.selection => |*merge| {
7485	// To jump out of a selection block, push a new entry onto its merge stack and
7486	// generate a conditional branch to there and to the instructions following this block.
7487	const merge_label = cg.allocId();
7488	const then_label = cg.allocId();
7489	try cg.body.emit(gpa, .OpSelectionMerge, .{
7490	.merge_block = merge_label,
7491	.selection_control = .{},
7492	});
7493	try cg.body.emit(gpa, .OpBranchConditional, .{
7494	.condition = jump_to_this_block_id,
7495	.true_label = then_label,
7496	.false_label = merge_label,
7497	});
7498	try merge.merge_stack.append(gpa, .{
7499	.incoming = .{
7500	.src_label = cg.block_label,
7501	.next_block = next_block,
7502	},
7503	.merge_block = merge_label,
7504	});
7505
7506	try cg.beginSpvBlock(then_label);
7507	},
7508	.loop => |*merge| {
7509	// To jump out of a loop block, generate a conditional that exits the block
7510	// to the loop merge if the target ID is not the one of this block.
7511	const continue_label = cg.allocId();
7512	try cg.body.emit(gpa, .OpBranchConditional, .{
7513	.condition = jump_to_this_block_id,
7514	.true_label = continue_label,
7515	.false_label = merge.merge_block,
7516	});
7517	try merge.merges.append(gpa, .{
7518	.src_label = cg.block_label,
7519	.next_block = next_block,
7520	});
7521	try cg.beginSpvBlock(continue_label);
7522	},
7523	}
7524	}
7525
7526	if (maybe_block_result_var_id) |block_result_var_id| {
7527	return try cg.load(ty, block_result_var_id, .{});
7528	}
7529
7530	return null;
7531	}
7532
7533	fn airBr(cg: *CodeGen, inst: Air.Inst.Index) !void {
7534	const zcu = cg.zcu;
7535	const br = cg.air.instructions.items(.data)[@backingInt(inst)].br;
7536	const operand_ty = cg.typeOf(br.operand);
7537
7538	if (operand_ty.hasRuntimeBits(zcu)) {
7539	const operand_id = try cg.resolve(br.operand);
7540	const block_result_var_id = cg.block_results.get(br.block_inst).?;
7541	try cg.store(operand_ty, block_result_var_id, operand_id, .{});
7542	}
7543
7544	const next_block = try cg.constInt(.u32, @backingInt(br.block_inst));
7545	try cg.structuredBreak(next_block);
7546	}
7547
7548	fn airCondBr(cg: *CodeGen, inst: Air.Inst.Index) !void {
7549	const gpa = cg.gpa;
7550	const cond_br = cg.air.unwrapCondBr(inst);
7551	const then_body = cond_br.then_body;
7552	const else_body = cond_br.else_body;
7553	const condition_id = try cg.resolve(cond_br.condition);
7554
7555	const then_label = cg.allocId();
7556	const else_label = cg.allocId();
7557
7558	const merge_label = cg.allocId();
7559
7560	try cg.body.emit(gpa, .OpSelectionMerge, .{
7561	.merge_block = merge_label,
7562	.selection_control = .{},
7563	});
7564	try cg.body.emit(gpa, .OpBranchConditional, .{
7565	.condition = condition_id,
7566	.true_label = then_label,
7567	.false_label = else_label,
7568	});
7569
7570	try cg.beginSpvBlock(then_label);
7571	const then_next = try cg.genStructuredBody(.selection, then_body);
7572	const then_incoming: Block.Incoming = .{
7573	.src_label = cg.block_label,
7574	.next_block = then_next,
7575	};
7576
7577	if (!cg.block_terminated) {
7578	try cg.body.emit(gpa, .OpBranch, .{ .target_label = merge_label });
7579	}
7580
7581	try cg.beginSpvBlock(else_label);
7582	const else_next = try cg.genStructuredBody(.selection, else_body);
7583	const else_incoming: Block.Incoming = .{
7584	.src_label = cg.block_label,
7585	.next_block = else_next,
7586	};
7587
7588	if (!cg.block_terminated) {
7589	try cg.body.emit(gpa, .OpBranch, .{ .target_label = merge_label });
7590	}
7591
7592	try cg.beginSpvBlock(merge_label);
7593	const next_block = try cg.structuredNextBlock(&.{ then_incoming, else_incoming });
7594
7595	try cg.structuredBreak(next_block);
7596	}
7597
7598	fn airLoop(cg: *CodeGen, inst: Air.Inst.Index) !void {
7599	const gpa = cg.gpa;
7600	const block = cg.air.unwrapBlock(inst);
7601
7602	const body_label = cg.allocId();
7603
7604	const header_label = cg.allocId();
7605	const merge_label = cg.allocId();
7606	const continue_label = cg.allocId();
7607
7608	// The back-edge must point to the loop header, so generate a separate block for the
7609	// loop header so that we don't accidentally include some instructions from there
7610	// in the loop.
7611
7612	try cg.body.emit(gpa, .OpBranch, .{ .target_label = header_label });
7613	try cg.beginSpvBlock(header_label);
7614
7615	// Emit loop header and jump to loop body
7616	try cg.body.emit(gpa, .OpLoopMerge, .{
7617	.merge_block = merge_label,
7618	.continue_target = continue_label,
7619	.loop_control = .{},
7620	});
7621
7622	try cg.body.emit(gpa, .OpBranch, .{ .target_label = body_label });
7623
7624	try cg.beginSpvBlock(body_label);
7625
7626	const next_block = try cg.genStructuredBody(.{ .loop = .{
7627	.merge_label = merge_label,
7628	.continue_label = continue_label,
7629	} }, block.body);
7630	try cg.structuredBreak(next_block);
7631
7632	try cg.beginSpvBlock(continue_label);
7633
7634	try cg.body.emit(gpa, .OpBranch, .{ .target_label = header_label });
7635	}
7636
7637	fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
7638	const zcu = cg.zcu;
7639	const pt = cg.pt;
7640	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
7641	const ptr_ty = cg.typeOf(ty_op.operand);
7642	const ptr_info = ptr_ty.ptrInfo(zcu);
7643	const elem_ty = cg.typeOfIndex(inst);
7644	const ptr = try cg.resolvePtr(ty_op.operand);
7645	assert(ptr_info.child == elem_ty.toIntern());
7646
7647	const operand_ptr_id = switch (ptr) {
7648	.tracked => |t| return t.slot.*.?,
7649	.id => |id| id,
7650	};
7651
7652	if (ptr_info.packed_offset.host_size != 0 and
7653	ptr_info.flags.vector_index == .none)
7654	{
7655	const host_bits: u16 = ptr_info.packed_offset.host_size * 8;
7656	const elem_bit_size: u16 = @intCast(elem_ty.bitSize(zcu));
7657	const host_int_ty = try pt.intType(.unsigned, host_bits);
7658	const host_val = try cg.load(host_int_ty, operand_ptr_id, .{ .is_volatile = ptr_info.flags.is_volatile });
7659	const signedness: Signedness = if (elem_ty.isInt(zcu)) elem_ty.intInfo(zcu).signedness else .unsigned;
7660	const field_int_ty = try pt.intType(signedness, elem_bit_size);
7661	const narrowed = if (ptr_info.packed_offset.bit_offset > 0) blk: {
7662	const bit_offset_id = try cg.constInt(host_int_ty, ptr_info.packed_offset.bit_offset);
7663	const shifted = try cg.buildBinary(.OpShiftRightLogical, .{ .ty = host_int_ty, .value = .{ .singleton = host_val } }, .{ .ty = host_int_ty, .value = .{ .singleton = bit_offset_id } });
7664	break :blk try shifted.materialize(cg);
7665	} else host_val;
7666	const result_id = blk: {
7667	if (cg.backingIntBits(elem_bit_size).@"0" == cg.backingIntBits(host_bits).@"0")
7668	break :blk try cg.bitCast(field_int_ty, host_int_ty, narrowed);
7669	const trunc = try cg.buildConvert(field_int_ty, .{ .ty = host_int_ty, .value = .{ .singleton = narrowed } });
7670	break :blk try trunc.materialize(cg);
7671	};
7672	if (elem_ty.ip_index == .bool_type) return try cg.convertToDirect(.bool, result_id);
7673	if (elem_ty.isInt(zcu)) return result_id;
7674	return try cg.bitCast(elem_ty, field_int_ty, result_id);
7675	}
7676
7677	const ptr_id = switch (ptr_info.flags.vector_index) {
7678	.none => operand_ptr_id,
7679	else => |index| ptr_id: {
7680	const elem_ptr_ty_id = try cg.ptrType(
7681	try cg.resolveType(elem_ty, .indirect),
7682	cg.storageClass(ptr_info.flags.address_space),
7683	);
7684	break :ptr_id try cg.accessChain(elem_ptr_ty_id, operand_ptr_id, &.{@backingInt(index)});
7685	},
7686	};
7687	return try cg.load(elem_ty, ptr_id, .{
7688	.is_volatile = ptr_info.flags.is_volatile,
7689	.ptr_address_space = ptr_info.flags.address_space,
7690	});
7691	}
7692
7693	fn airStore(cg: *CodeGen, inst: Air.Inst.Index) !void {
7694	const zcu = cg.zcu;
7695	const pt = cg.pt;
7696	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
7697	const ptr_ty = cg.typeOf(bin_op.lhs);
7698	const ptr_info = ptr_ty.ptrInfo(zcu);
7699	const elem_ty: Type = .fromInterned(ptr_info.child);
7700	const value_id = try cg.resolve(bin_op.rhs);
7701	const operand_ptr_id = switch (try cg.resolvePtr(bin_op.lhs)) {
7702	.tracked => |t| {
7703	t.slot.* = value_id;
7704	return;
7705	},
7706	.id => |id| id,
7707	};
7708
7709	if (ptr_info.packed_offset.host_size != 0 and
7710	ptr_info.flags.vector_index == .none)
7711	{
7712	const host_bits: u16 = ptr_info.packed_offset.host_size * 8;
7713	const host_int_ty = try pt.intType(.unsigned, host_bits);
7714	const host_val = try cg.load(host_int_ty, operand_ptr_id, .{ .is_volatile = ptr_info.flags.is_volatile });
7715	const elem_bit_size: u16 = @intCast(elem_ty.bitSize(zcu));
7716	const signedness: Signedness = if (elem_ty.isInt(zcu)) elem_ty.intInfo(zcu).signedness else .unsigned;
7717	const field_int_ty = try pt.intType(signedness, elem_bit_size);
7718
7719	var value_as_int: Id = undefined;
7720	if (elem_ty.ip_index == .bool_type) {
7721	value_as_int = try cg.convertToIndirect(.bool, value_id);
7722	value_as_int = try cg.bitCast(field_int_ty, .u1, value_as_int);
7723	} else if (elem_ty.isInt(zcu)) {
7724	value_as_int = value_id;
7725	} else {
7726	value_as_int = try cg.bitCast(field_int_ty, elem_ty, value_id);
7727	}
7728
7729	const extended = blk: {
7730	if (cg.backingIntBits(elem_bit_size).@"0" == cg.backingIntBits(host_bits).@"0")
7731	break :blk try cg.bitCast(host_int_ty, field_int_ty, value_as_int);
7732	const conv = try cg.buildConvert(host_int_ty, .{ .ty = field_int_ty, .value = .{ .singleton = value_as_int } });
7733	break :blk try conv.materialize(cg);
7734	};
7735
7736	const bit_offset = ptr_info.packed_offset.bit_offset;
7737	const field_mask = (@as(u64, 1) << @as(u6, @intCast(elem_bit_size))) - 1;
7738	const host_mask = if (host_bits == 64) @as(u64, std.math.maxInt(u64)) else (@as(u64, 1) << @as(u6, @intCast(host_bits))) - 1;
7739	const clear_mask = ~(field_mask << @as(u6, @intCast(bit_offset))) & host_mask;
7740	const clear_mask_id = try cg.constInt(host_int_ty, clear_mask);
7741	const cleared = try cg.buildBinary(.OpBitwiseAnd, .{ .ty = host_int_ty, .value = .{ .singleton = host_val } }, .{ .ty = host_int_ty, .value = .{ .singleton = clear_mask_id } });
7742	const bit_offset_id = try cg.constInt(host_int_ty, bit_offset);
7743	const shifted_val = try cg.buildBinary(.OpShiftLeftLogical, .{ .ty = host_int_ty, .value = .{ .singleton = extended } }, .{ .ty = host_int_ty, .value = .{ .singleton = bit_offset_id } });
7744	const combined = try cg.buildBinary(.OpBitwiseOr, cleared, shifted_val);
7745	const combined_id = try combined.materialize(cg);
7746
7747	try cg.store(host_int_ty, operand_ptr_id, combined_id, .{ .is_volatile = ptr_info.flags.is_volatile });
7748	return;
7749	}
7750
7751	const ptr_id = switch (ptr_info.flags.vector_index) {
7752	.none => operand_ptr_id,
7753	else => |index| ptr_id: {
7754	const elem_ptr_ty_id = try cg.ptrType(
7755	try cg.resolveType(elem_ty, .indirect),
7756	cg.storageClass(ptr_info.flags.address_space),
7757	);
7758	break :ptr_id try cg.accessChain(elem_ptr_ty_id, operand_ptr_id, &.{@backingInt(index)});
7759	},
7760	};
7761
7762	try cg.store(elem_ty, ptr_id, value_id, .{
7763	.is_volatile = ptr_info.flags.is_volatile,
7764	.ptr_address_space = ptr_info.flags.address_space,
7765	});
7766	}
7767
7768	fn airRet(cg: *CodeGen, inst: Air.Inst.Index) !void {
7769	const gpa = cg.gpa;
7770	const zcu = cg.zcu;
7771	const operand = cg.air.instructions.items(.data)[@backingInt(inst)].un_op;
7772	const ret_ty = cg.typeOf(operand);
7773	if (!ret_ty.hasRuntimeBits(zcu)) {
7774	const fn_info = zcu.typeToFunc(zcu.navValue(cg.owner_nav).typeOf(zcu)).?;
7775	if (Type.fromInterned(fn_info.return_type).isError(zcu)) {
7776	// Functions with an empty error set are emitted with an error code
7777	// return type and return zero so they can be function pointers coerced
7778	// to functions that return anyerror.
7779	const no_err_id = try cg.constInt(.anyerror, 0);
7780	return try cg.body.emit(gpa, .OpReturnValue, .{ .value = no_err_id });
7781	} else {
7782	return try cg.body.emit(gpa, .OpReturn, {});
7783	}
7784	}
7785
7786	const operand_id = try cg.resolve(operand);
7787	try cg.body.emit(gpa, .OpReturnValue, .{ .value = operand_id });
7788	}
7789
7790	fn airRetLoad(cg: *CodeGen, inst: Air.Inst.Index) !void {
7791	const gpa = cg.gpa;
7792	const zcu = cg.zcu;
7793	const un_op = cg.air.instructions.items(.data)[@backingInt(inst)].un_op;
7794	const ptr_ty = cg.typeOf(un_op);
7795	const ret_ty = ptr_ty.childType(zcu);
7796
7797	if (!ret_ty.hasRuntimeBits(zcu)) {
7798	const fn_info = zcu.typeToFunc(zcu.navValue(cg.owner_nav).typeOf(zcu)).?;
7799	if (Type.fromInterned(fn_info.return_type).isError(zcu)) {
7800	// Functions with an empty error set are emitted with an error code
7801	// return type and return zero so they can be function pointers coerced
7802	// to functions that return anyerror.
7803	const no_err_id = try cg.constInt(.anyerror, 0);
7804	return try cg.body.emit(gpa, .OpReturnValue, .{ .value = no_err_id });
7805	} else {
7806	return try cg.body.emit(gpa, .OpReturn, {});
7807	}
7808	}
7809
7810	const value = switch (try cg.resolvePtr(un_op)) {
7811	.tracked => |t| t.slot.*.?,
7812	.id => |ptr| try cg.load(ret_ty, ptr, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) }),
7813	};
7814	try cg.body.emit(gpa, .OpReturnValue, .{
7815	.value = value,
7816	});
7817	}
7818
7819	fn airTry(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
7820	const gpa = cg.gpa;
7821	const zcu = cg.zcu;
7822	const unwrapped_try = cg.air.unwrapTry(inst);
7823	const body = unwrapped_try.else_body;
7824
7825	const err_union_id = try cg.resolve(unwrapped_try.error_union);
7826	const err_union_ty = cg.air.typeOf(unwrapped_try.error_union, &zcu.intern_pool);
7827	const payload_ty = cg.typeOfIndex(inst);
7828
7829	const bool_ty_id = try cg.resolveType(.bool, .direct);
7830
7831	const eu_layout = cg.errorUnionLayout(payload_ty);
7832
7833	if (!err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
7834	const err_id = if (eu_layout.payload_has_bits)
7835	try cg.extractField(.anyerror, err_union_id, eu_layout.errorFieldIndex())
7836	else
7837	err_union_id;
7838
7839	const zero_id = try cg.constInt(.anyerror, 0);
7840	const is_err_id = cg.allocId();
7841	try cg.body.emit(gpa, .OpINotEqual, .{
7842	.id_result_type = bool_ty_id,
7843	.id_result = is_err_id,
7844	.operand_1 = err_id,
7845	.operand_2 = zero_id,
7846	});
7847
7848	// When there is an error, we must evaluate `body`. Otherwise we must continue
7849	// with the current body.
7850	// Just generate a new block here, then generate a new block inline for the remainder of the body.
7851
7852	const err_block = cg.allocId();
7853	const ok_block = cg.allocId();
7854
7855	// According to AIR documentation, this block is guaranteed
7856	// to not break and end in a return instruction. Thus,
7857	// we can just naively use the ok block as the merge block here.
7858	try cg.body.emit(gpa, .OpSelectionMerge, .{
7859	.merge_block = ok_block,
7860	.selection_control = .{},
7861	});
7862
7863	try cg.body.emit(gpa, .OpBranchConditional, .{
7864	.condition = is_err_id,
7865	.true_label = err_block,
7866	.false_label = ok_block,
7867	});
7868
7869	try cg.beginSpvBlock(err_block);
7870	try cg.genBody(body);
7871
7872	try cg.beginSpvBlock(ok_block);
7873	}
7874
7875	if (!eu_layout.payload_has_bits) {
7876	return null;
7877	}
7878
7879	// Now just extract the payload, if required.
7880	return try cg.extractField(payload_ty, err_union_id, eu_layout.payloadFieldIndex());
7881	}
7882
7883	fn airErrUnionErr(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
7884	const zcu = cg.zcu;
7885	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
7886	const operand_id = try cg.resolve(ty_op.operand);
7887	const err_union_ty = cg.typeOf(ty_op.operand);
7888	const err_ty_id = try cg.resolveType(.anyerror, .direct);
7889
7890	if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
7891	// No error possible, so just return undefined.
7892	return try cg.constUndef(err_ty_id);
7893	}
7894
7895	const payload_ty = err_union_ty.errorUnionPayload(zcu);
7896	const eu_layout = cg.errorUnionLayout(payload_ty);
7897
7898	if (!eu_layout.payload_has_bits) {
7899	// If no payload, error union is represented by error set.
7900	return operand_id;
7901	}
7902
7903	return try cg.extractField(.anyerror, operand_id, eu_layout.errorFieldIndex());
7904	}
7905
7906	fn airErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
7907	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
7908	const operand_id = try cg.resolve(ty_op.operand);
7909	const payload_ty = cg.typeOfIndex(inst);
7910	const eu_layout = cg.errorUnionLayout(payload_ty);
7911
7912	if (!eu_layout.payload_has_bits) {
7913	return null; // No error possible.
7914	}
7915
7916	return try cg.extractField(payload_ty, operand_id, eu_layout.payloadFieldIndex());
7917	}
7918
7919	fn airWrapErrUnionErr(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
7920	const zcu = cg.zcu;
7921	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
7922	const err_union_ty = cg.typeOfIndex(inst);
7923	const payload_ty = err_union_ty.errorUnionPayload(zcu);
7924	const operand_id = try cg.resolve(ty_op.operand);
7925	const eu_layout = cg.errorUnionLayout(payload_ty);
7926
7927	if (!eu_layout.payload_has_bits) {
7928	return operand_id;
7929	}
7930
7931	const payload_ty_id = try cg.resolveType(payload_ty, .indirect);
7932
7933	var members: [2]Id = undefined;
7934	members[eu_layout.errorFieldIndex()] = operand_id;
7935	members[eu_layout.payloadFieldIndex()] = try cg.constUndef(payload_ty_id);
7936
7937	var types: [2]Type = undefined;
7938	types[eu_layout.errorFieldIndex()] = .anyerror;
7939	types[eu_layout.payloadFieldIndex()] = payload_ty;
7940
7941	const err_union_ty_id = try cg.resolveType(err_union_ty, .direct);
7942	return try cg.constructComposite(err_union_ty_id, &members);
7943	}
7944
7945	fn airWrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
7946	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
7947	const err_union_ty = cg.typeOfIndex(inst);
7948	const operand_id = try cg.resolve(ty_op.operand);
7949	const payload_ty = cg.typeOf(ty_op.operand);
7950	const eu_layout = cg.errorUnionLayout(payload_ty);
7951
7952	if (!eu_layout.payload_has_bits) {
7953	return try cg.constInt(.anyerror, 0);
7954	}
7955
7956	var members: [2]Id = undefined;
7957	members[eu_layout.errorFieldIndex()] = try cg.constInt(.anyerror, 0);
7958	members[eu_layout.payloadFieldIndex()] = try cg.convertToIndirect(payload_ty, operand_id);
7959
7960	var types: [2]Type = undefined;
7961	types[eu_layout.errorFieldIndex()] = .anyerror;
7962	types[eu_layout.payloadFieldIndex()] = payload_ty;
7963
7964	const err_union_ty_id = try cg.resolveType(err_union_ty, .direct);
7965	return try cg.constructComposite(err_union_ty_id, &members);
7966	}
7967
7968	fn airIsNull(cg: *CodeGen, inst: Air.Inst.Index, is_pointer: bool, pred: enum { is_null, is_non_null }) !?Id {
7969	const zcu = cg.zcu;
7970	const un_op = cg.air.instructions.items(.data)[@backingInt(inst)].un_op;
7971	const operand_id = try cg.resolve(un_op);
7972	const operand_ty = cg.typeOf(un_op);
7973	const optional_ty = if (is_pointer) operand_ty.childType(zcu) else operand_ty;
7974	const payload_ty = optional_ty.optionalChild(zcu);
7975
7976	const bool_ty_id = try cg.resolveType(.bool, .direct);
7977
7978	if (optional_ty.optionalReprIsPayload(zcu)) {
7979	// Pointer payload represents nullability: pointer or slice.
7980	const loaded_id = if (is_pointer)
7981	try cg.load(optional_ty, operand_id, .{})
7982	else
7983	operand_id;
7984
7985	const ptr_ty = if (payload_ty.isSlice(zcu))
7986	payload_ty.slicePtrFieldType(zcu)
7987	else
7988	payload_ty;
7989
7990	const ptr_id = if (payload_ty.isSlice(zcu))
7991	try cg.extractField(ptr_ty, loaded_id, 0)
7992	else
7993	loaded_id;
7994
7995	const ptr_ty_id = try cg.resolveType(ptr_ty, .direct);
7996	const null_id = try cg.constNull(ptr_ty_id);
7997	const null_tmp: Temporary = .init(ptr_ty, null_id);
7998	const ptr: Temporary = .init(ptr_ty, ptr_id);
7999
8000	const op: std.math.CompareOperator = switch (pred) {
8001	.is_null => .eq,
8002	.is_non_null => .neq,
8003	};
8004	const result = try cg.cmp(op, ptr, null_tmp);
8005	return try result.materialize(cg);
8006	}
8007
8008	const is_non_null_id = blk: {
8009	if (is_pointer) {
8010	if (payload_ty.hasRuntimeBits(zcu)) {
8011	const storage_class = cg.storageClass(operand_ty.ptrAddressSpace(zcu));
8012	const bool_indirect_ty_id = try cg.resolveType(.bool, .indirect);
8013	const bool_ptr_ty_id = try cg.ptrType(bool_indirect_ty_id, storage_class);
8014	const tag_ptr_id = try cg.accessChain(bool_ptr_ty_id, operand_id, &.{1});
8015	break :blk try cg.load(.bool, tag_ptr_id, .{});
8016	}
8017
8018	break :blk try cg.load(.bool, operand_id, .{});
8019	}
8020
8021	break :blk if (payload_ty.hasRuntimeBits(zcu))
8022	try cg.extractField(.bool, operand_id, 1)
8023	else
8024	// Optional representation is bool indicating whether the optional is set
8025	// Optionals with no payload are represented as an (indirect) bool, so convert
8026	// it back to the direct bool here.
8027	try cg.convertToDirect(.bool, operand_id);
8028	};
8029
8030	return switch (pred) {
8031	.is_null => blk: {
8032	// Invert condition
8033	const result_id = cg.allocId();
8034	try cg.body.emit(cg.gpa, .OpLogicalNot, .{
8035	.id_result_type = bool_ty_id,
8036	.id_result = result_id,
8037	.operand = is_non_null_id,
8038	});
8039	break :blk result_id;
8040	},
8041	.is_non_null => is_non_null_id,
8042	};
8043	}
8044
8045	fn airIsErr(cg: *CodeGen, inst: Air.Inst.Index, pred: enum { is_err, is_non_err }) !?Id {
8046	const zcu = cg.zcu;
8047	const un_op = cg.air.instructions.items(.data)[@backingInt(inst)].un_op;
8048	const operand_id = try cg.resolve(un_op);
8049	const err_union_ty = cg.typeOf(un_op);
8050
8051	if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
8052	return try cg.constBool(pred == .is_non_err, .direct);
8053	}
8054
8055	const payload_ty = err_union_ty.errorUnionPayload(zcu);
8056	const eu_layout = cg.errorUnionLayout(payload_ty);
8057	const bool_ty_id = try cg.resolveType(.bool, .direct);
8058
8059	const error_id = if (!eu_layout.payload_has_bits)
8060	operand_id
8061	else
8062	try cg.extractField(.anyerror, operand_id, eu_layout.errorFieldIndex());
8063
8064	const result_id = cg.allocId();
8065	switch (pred) {
8066	inline else => |pred_ct| try cg.body.emit(
8067	cg.gpa,
8068	switch (pred_ct) {
8069	.is_err => .OpINotEqual,
8070	.is_non_err => .OpIEqual,
8071	},
8072	.{
8073	.id_result_type = bool_ty_id,
8074	.id_result = result_id,
8075	.operand_1 = error_id,
8076	.operand_2 = try cg.constInt(.anyerror, 0),
8077	},
8078	),
8079	}
8080	return result_id;
8081	}
8082
8083	fn airUnwrapOptional(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
8084	const zcu = cg.zcu;
8085	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
8086	const operand_id = try cg.resolve(ty_op.operand);
8087	const optional_ty = cg.typeOf(ty_op.operand);
8088	const payload_ty = cg.typeOfIndex(inst);
8089
8090	if (!payload_ty.hasRuntimeBits(zcu)) return null;
8091
8092	if (optional_ty.optionalReprIsPayload(zcu)) {
8093	return operand_id;
8094	}
8095
8096	return try cg.extractField(payload_ty, operand_id, 0);
8097	}
8098
8099	fn airUnwrapOptionalPtr(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
8100	const zcu = cg.zcu;
8101	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
8102	const operand_id = try cg.resolve(ty_op.operand);
8103	const operand_ty = cg.typeOf(ty_op.operand);
8104	const optional_ty = operand_ty.childType(zcu);
8105	const payload_ty = optional_ty.optionalChild(zcu);
8106	const result_ty = cg.typeOfIndex(inst);
8107	const result_ty_id = try cg.resolveType(result_ty, .direct);
8108
8109	if (!payload_ty.hasRuntimeBits(zcu)) {
8110	// There is no payload, but we still need to return a valid pointer.
8111	// We can just return anything here, so just return a pointer to the operand.
8112	return try cg.bitCast(result_ty, operand_ty, operand_id);
8113	}
8114
8115	if (optional_ty.optionalReprIsPayload(zcu)) {
8116	// They are the same value.
8117	return try cg.bitCast(result_ty, operand_ty, operand_id);
8118	}
8119
8120	return try cg.accessChain(result_ty_id, operand_id, &.{0});
8121	}
8122
8123	fn airSetOptionalPtr(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
8124	const zcu = cg.zcu;
8125	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
8126
8127	const ptr_ty = cg.typeOf(ty_op.operand);
8128	const ptr_id = try cg.resolve(ty_op.operand);
8129
8130	const optional_ty = ptr_ty.childType(zcu);
8131	const payload_ty = optional_ty.optionalChild(zcu);
8132	const result_ty = cg.typeOfIndex(inst);
8133
8134	if (optional_ty.optionalReprIsPayload(zcu)) {
8135	return try cg.bitCast(result_ty, ptr_ty, ptr_id);
8136	}
8137
8138	const storage_class = cg.storageClass(ptr_ty.ptrAddressSpace(zcu));
8139	const bool_indirect_ty_id = try cg.resolveType(.bool, .indirect);
8140	const bool_ptr_ty_id = try cg.ptrType(bool_indirect_ty_id, storage_class);
8141	const result_ty_id = try cg.resolveType(result_ty, .direct);
8142
8143	const bool_ptr_id, const ret = switch (payload_ty.hasRuntimeBits(zcu)) {
8144	true => .{
8145	try cg.accessChain(bool_ptr_ty_id, ptr_id, &.{1}),
8146	try cg.accessChain(result_ty_id, ptr_id, &.{0}),
8147	},
8148	false => .{ ptr_id, try cg.bitCast(result_ty, ptr_ty, ptr_id) },
8149	};
8150
8151	try cg.store(.bool, bool_ptr_id, try cg.constBool(true, .direct), .{});
8152
8153	return ret;
8154	}
8155
8156	fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
8157	const zcu = cg.zcu;
8158	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
8159	const payload_ty = cg.typeOf(ty_op.operand);
8160
8161	assert(payload_ty.hasRuntimeBits(zcu));
8162
8163	const operand_id = try cg.resolve(ty_op.operand);
8164
8165	const optional_ty = cg.typeOfIndex(inst);
8166	if (optional_ty.optionalReprIsPayload(zcu)) {
8167	return operand_id;
8168	}
8169
8170	const payload_id = try cg.convertToIndirect(payload_ty, operand_id);
8171	const members = [_]Id{ payload_id, try cg.constBool(true, .indirect) };
8172	const optional_ty_id = try cg.resolveType(optional_ty, .direct);
8173	return try cg.constructComposite(optional_ty_id, &members);
8174	}
8175
8176	fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index) !void {
8177	const gpa = cg.gpa;
8178	const zcu = cg.zcu;
8179	const target = cg.zcu.getTarget();
8180	const switch_br = cg.air.unwrapSwitch(inst);
8181	const cond_ty = cg.typeOf(switch_br.operand);
8182	const cond = try cg.resolve(switch_br.operand);
8183	var cond_indirect = try cg.convertToIndirect(cond_ty, cond);
8184
8185	const cond_words: u32 = switch (cond_ty.zigTypeTag(zcu)) {
8186	.bool, .error_set => 1,
8187	.int => blk: {
8188	const bits = cond_ty.intInfo(zcu).bits;
8189	const backing_bits, const big_int = cg.backingIntBits(bits);
8190	if (big_int) return cg.todo("implement composite int switch", .{});
8191	break :blk if (backing_bits <= 32) 1 else 2;
8192	},
8193	.@"enum" => blk: {
8194	const int_ty = cond_ty.backingIntType(zcu);
8195	const int_info = int_ty.intInfo(zcu);
8196	const backing_bits, const big_int = cg.backingIntBits(int_info.bits);
8197	if (big_int) return cg.todo("implement composite int switch", .{});
8198	break :blk if (backing_bits <= 32) 1 else 2;
8199	},
8200	.pointer => blk: {
8201	cond_indirect = try cg.intFromPtr(cond_indirect);
8202	break :blk target.ptrBitWidth() / 32;
8203	},
8204	// TODO: Figure out which types apply here, and work around them as we can only do integers.
8205	else => return cg.todo("implement switch for type {s}", .{@tagName(cond_ty.zigTypeTag(zcu))}),
8206	};
8207
8208	const num_cases = switch_br.cases_len;
8209
8210	// compute the total number of scalar arms and find the last range case
8211	var num_conditions: u32 = 0;
8212	var last_range_case: ?u32 = null;
8213	{
8214	var it = switch_br.iterateCases();
8215	while (it.next()) |case| {
8216	if (case.ranges.len > 0) {
8217	last_range_case = case.idx;
8218	} else {
8219	num_conditions += @intCast(case.items.len);
8220	}
8221	}
8222	}
8223
8224	// First, pre-allocate the labels for the cases.
8225	const case_labels = cg.allocIds(num_cases);
8226	// We always need the default case - if zig has none, we will generate unreachable there.
8227	const default_label = cg.allocId();
8228	const switch_default = if (last_range_case != null) cg.allocId() else default_label;
8229
8230	const merge_label = cg.allocId();
8231
8232	try cg.body.emit(gpa, .OpSelectionMerge, .{
8233	.merge_block = merge_label,
8234	.selection_control = .{},
8235	});
8236
8237	// Emit the instruction before generating the blocks.
8238	try cg.body.emitRaw(gpa, .OpSwitch, 2 + (cond_words + 1) * num_conditions);
8239	cg.body.writeOperand(Id, cond_indirect);
8240	cg.body.writeOperand(Id, switch_default);
8241
8242	// Emit the non-range cases into the OpSwitch.
8243	// Cases with ranges are handled by the conditional chain below.
8244	{
8245	var it = switch_br.iterateCases();
8246	while (it.next()) |case| {
8247	if (case.ranges.len > 0) continue;
8248	const label = case_labels.at(case.idx);
8249
8250	for (case.items) |item| {
8251	const value: Value = .fromInterned(item.toInterned().?);
8252	const int_val: u64 = switch (cond_ty.zigTypeTag(zcu)) {
8253	.bool, .int => if (cond_ty.isSignedInt(zcu)) @bitCast(value.toSignedInt(zcu)) else value.toUnsignedInt(zcu),
8254	.@"enum" => value.backingInt(zcu).toUnsignedInt(zcu),
8255	.error_set => value.getErrorInt(zcu),
8256	.pointer => value.toUnsignedInt(zcu),
8257	else => unreachable,
8258	};
8259	const int_lit: spec.LiteralContextDependentNumber = switch (cond_words) {
8260	1 => .{ .uint32 = @intCast(int_val) },
8261	2 => .{ .uint64 = int_val },
8262	else => unreachable,
8263	};
8264	cg.body.writeOperand(spec.LiteralContextDependentNumber, int_lit);
8265	cg.body.writeOperand(Id, label);
8266	}
8267	}
8268	}
8269
8270	var incoming_structured_blocks: std.ArrayList(Block.Incoming) = .empty;
8271	defer incoming_structured_blocks.deinit(gpa);
8272	try incoming_structured_blocks.ensureUnusedCapacity(gpa, num_cases + 1);
8273
8274	// emit the range-checking chain as nested if-else inside the switch's default branch.
8275	// each range case becomes:
8276	// - check condition,
8277	// - if true emit case body and branch to merge,
8278	// - else continue to next check or default
8279	if (last_range_case != null) {
8280	const cond_tmp: Temporary = .init(cond_ty, cond);
8281	const bool_ty_id = try cg.resolveType(.bool, .direct);
8282
8283	try cg.beginSpvBlock(switch_default);
8284
8285	var it_range = switch_br.iterateCases();
8286	while (it_range.next()) |case| {
8287	if (case.ranges.len == 0) continue;
8288
8289	var case_cond: ?Id = null;
8290
8291	for (case.items) |item| {
8292	const item_tmp: Temporary = try cg.temporary(item);
8293	const eq = try (try cg.cmp(.eq, cond_tmp, item_tmp)).materialize(cg);
8294	case_cond = if (case_cond) |prev| blk: {
8295	const combined = cg.allocId();
8296	try cg.body.emit(gpa, .OpLogicalOr, .{
8297	.id_result_type = bool_ty_id,
8298	.id_result = combined,
8299	.operand_1 = prev,
8300	.operand_2 = eq,
8301	});
8302	break :blk combined;
8303	} else eq;
8304	}
8305
8306	for (case.ranges) |range| {
8307	const lo_tmp: Temporary = try cg.temporary(range[0]);
8308	const hi_tmp: Temporary = try cg.temporary(range[1]);
8309	const ge = try (try cg.cmp(.gte, cond_tmp, lo_tmp)).materialize(cg);
8310	const le = try (try cg.cmp(.lte, cond_tmp, hi_tmp)).materialize(cg);
8311	const in_range = cg.allocId();
8312	try cg.body.emit(gpa, .OpLogicalAnd, .{
8313	.id_result_type = bool_ty_id,
8314	.id_result = in_range,
8315	.operand_1 = ge,
8316	.operand_2 = le,
8317	});
8318	case_cond = if (case_cond) |prev| blk: {
8319	const combined = cg.allocId();
8320	try cg.body.emit(gpa, .OpLogicalOr, .{
8321	.id_result_type = bool_ty_id,
8322	.id_result = combined,
8323	.operand_1 = prev,
8324	.operand_2 = in_range,
8325	});
8326	break :blk combined;
8327	} else in_range;
8328	}
8329
8330	const case_label = case_labels.at(case.idx);
8331	const is_last = case.idx == last_range_case.?;
8332	const next_check = if (is_last) default_label else cg.allocId();
8333
8334	try cg.body.emit(gpa, .OpSelectionMerge, .{
8335	.merge_block = next_check,
8336	.selection_control = .{},
8337	});
8338
8339	try cg.body.emit(gpa, .OpBranchConditional, .{
8340	.condition = case_cond.?,
8341	.true_label = case_label,
8342	.false_label = next_check,
8343	});
8344
8345	if (!is_last) {
8346	try cg.beginSpvBlock(next_check);
8347	}
8348	}
8349	}
8350
8351	// emit bodies
8352	var it = switch_br.iterateCases();
8353	while (it.next()) |case| {
8354	const label = case_labels.at(case.idx);
8355
8356	try cg.beginSpvBlock(label);
8357
8358	const next_block = try cg.genStructuredBody(.selection, case.body);
8359	incoming_structured_blocks.appendAssumeCapacity(.{
8360	.src_label = cg.block_label,
8361	.next_block = next_block,
8362	});
8363
8364	try cg.body.emit(gpa, .OpBranch, .{ .target_label = merge_label });
8365	}
8366
8367	const else_body = blk: {
8368	var it_else = switch_br.iterateCases();
8369	while (it_else.next()) |_| {}
8370	break :blk it_else.elseBody();
8371	};
8372	try cg.beginSpvBlock(default_label);
8373	if (else_body.len != 0) {
8374	const next_block = try cg.genStructuredBody(.selection, else_body);
8375	incoming_structured_blocks.appendAssumeCapacity(.{
8376	.src_label = cg.block_label,
8377	.next_block = next_block,
8378	});
8379
8380	try cg.body.emit(gpa, .OpBranch, .{ .target_label = merge_label });
8381	} else {
8382	try cg.body.emit(gpa, .OpUnreachable, {});
8383	}
8384
8385	try cg.beginSpvBlock(merge_label);
8386	const next_block = try cg.structuredNextBlock(incoming_structured_blocks.items);
8387	try cg.structuredBreak(next_block);
8388	}
8389
8390	fn airLoopSwitchBr(cg: *CodeGen, inst: Air.Inst.Index) !void {
8391	const gpa = cg.gpa;
8392	const zcu = cg.zcu;
8393	const target = cg.zcu.getTarget();
8394	const switch_br = cg.air.unwrapSwitch(inst);
8395	const cond_ty = cg.typeOf(switch_br.operand);
8396	const initial_cond = try cg.resolve(switch_br.operand);
8397	var initial_cond_indirect = try cg.convertToIndirect(cond_ty, initial_cond);
8398
8399	const cond_words: u32 = switch (cond_ty.zigTypeTag(zcu)) {
8400	.bool, .error_set => 1,
8401	.int => blk: {
8402	const bits = cond_ty.intInfo(zcu).bits;
8403	const backing_bits, const big_int = cg.backingIntBits(bits);
8404	if (big_int) return cg.todo("implement composite int loop switch", .{});
8405	break :blk if (backing_bits <= 32) 1 else 2;
8406	},
8407	.@"enum" => blk: {
8408	const int_ty = cond_ty.backingIntType(zcu);
8409	const int_info = int_ty.intInfo(zcu);
8410	const backing_bits, const big_int = cg.backingIntBits(int_info.bits);
8411	if (big_int) return cg.todo("implement composite int loop switch", .{});
8412	break :blk if (backing_bits <= 32) 1 else 2;
8413	},
8414	.pointer => blk: {
8415	initial_cond_indirect = try cg.intFromPtr(initial_cond_indirect);
8416	break :blk target.ptrBitWidth() / 32;
8417	},
8418	else => return cg.todo("implement loop switch for type {s}", .{@tagName(cond_ty.zigTypeTag(zcu))}),
8419	};
8420
8421	const cond_ty_id = try cg.resolveType(cond_ty, .indirect);
8422	const cond_var = try cg.alloc(cond_ty_id, null);
8423	try cg.store(cond_ty, cond_var, initial_cond_indirect, .{});
8424
8425	const num_cases = switch_br.cases_len;
8426
8427	var num_conditions: u32 = 0;
8428	var last_range_case: ?u32 = null;
8429	{
8430	var it = switch_br.iterateCases();
8431	while (it.next()) |case| {
8432	if (case.ranges.len > 0) {
8433	last_range_case = case.idx;
8434	} else {
8435	num_conditions += @intCast(case.items.len);
8436	}
8437	}
8438	}
8439
8440	const case_labels = cg.allocIds(num_cases);
8441	const default_label = cg.allocId();
8442	const switch_default = if (last_range_case != null) cg.allocId() else default_label;
8443
8444	const header_label = cg.allocId();
8445	const loop_merge = cg.allocId();
8446	const continue_label = cg.allocId();
8447	const switch_merge = cg.allocId();
8448	const body_label = cg.allocId();
8449
8450	// switch_dispatch signals "continue the loop" by using this sentinel as the
8451	// next_block in structuredBreak. at switch_merge, a phi + comparison distinguishes
8452	// dispatch (continue) from break (exit)
8453	const dispatch_sentinel = try cg.constInt(.u32, @backingInt(inst));
8454
8455	try cg.loop_switches.putNoClobber(gpa, inst, .{
8456	.cond_var = cond_var,
8457	.continue_label = dispatch_sentinel,
8458	});
8459	defer assert(cg.loop_switches.remove(inst));
8460
8461	try cg.body.emit(gpa, .OpBranch, .{ .target_label = header_label });
8462	try cg.beginSpvBlock(header_label);
8463
8464	try cg.body.emit(gpa, .OpLoopMerge, .{
8465	.merge_block = loop_merge,
8466	.continue_target = continue_label,
8467	.loop_control = .{},
8468	});
8469
8470	try cg.body.emit(gpa, .OpBranch, .{ .target_label = body_label });
8471	try cg.beginSpvBlock(body_label);
8472
8473	const cond = try cg.load(cond_ty, cond_var, .{});
8474	const cond_indirect = try cg.convertToIndirect(cond_ty, cond);
8475
8476	try cg.body.emit(gpa, .OpSelectionMerge, .{
8477	.merge_block = switch_merge,
8478	.selection_control = .{},
8479	});
8480
8481	try cg.body.emitRaw(gpa, .OpSwitch, 2 + (cond_words + 1) * num_conditions);
8482	cg.body.writeOperand(Id, cond_indirect);
8483	cg.body.writeOperand(Id, switch_default);
8484
8485	{
8486	var it = switch_br.iterateCases();
8487	while (it.next()) |case| {
8488	if (case.ranges.len > 0) continue;
8489	const label = case_labels.at(case.idx);
8490	for (case.items) |item| {
8491	const value: Value = .fromInterned(item.toInterned().?);
8492	const int_val: u64 = switch (cond_ty.zigTypeTag(zcu)) {
8493	.bool, .int => if (cond_ty.isSignedInt(zcu)) @bitCast(value.toSignedInt(zcu)) else value.toUnsignedInt(zcu),
8494	.@"enum" => value.backingInt(zcu).toUnsignedInt(zcu),
8495	.error_set => value.getErrorInt(zcu),
8496	.pointer => value.toUnsignedInt(zcu),
8497	else => unreachable,
8498	};
8499	const int_lit: spec.LiteralContextDependentNumber = switch (cond_words) {
8500	1 => .{ .uint32 = @intCast(int_val) },
8501	2 => .{ .uint64 = int_val },
8502	else => unreachable,
8503	};
8504	cg.body.writeOperand(spec.LiteralContextDependentNumber, int_lit);
8505	cg.body.writeOperand(Id, label);
8506	}
8507	}
8508	}
8509
8510	var incoming_structured_blocks: std.ArrayList(Block.Incoming) = .empty;
8511	defer incoming_structured_blocks.deinit(gpa);
8512	try incoming_structured_blocks.ensureUnusedCapacity(gpa, num_cases + 1);
8513
8514	if (last_range_case != null) {
8515	const cond_tmp: Temporary = .init(cond_ty, cond);
8516	const bool_ty_id = try cg.resolveType(.bool, .direct);
8517
8518	try cg.beginSpvBlock(switch_default);
8519
8520	var it_range = switch_br.iterateCases();
8521	while (it_range.next()) |case| {
8522	if (case.ranges.len == 0) continue;
8523
8524	var case_cond: ?Id = null;
8525
8526	for (case.items) |item| {
8527	const item_tmp: Temporary = try cg.temporary(item);
8528	const eq = try (try cg.cmp(.eq, cond_tmp, item_tmp)).materialize(cg);
8529	case_cond = if (case_cond) |prev| blk: {
8530	const combined = cg.allocId();
8531	try cg.body.emit(gpa, .OpLogicalOr, .{
8532	.id_result_type = bool_ty_id,
8533	.id_result = combined,
8534	.operand_1 = prev,
8535	.operand_2 = eq,
8536	});
8537	break :blk combined;
8538	} else eq;
8539	}
8540
8541	for (case.ranges) |range| {
8542	const lo_tmp: Temporary = try cg.temporary(range[0]);
8543	const hi_tmp: Temporary = try cg.temporary(range[1]);
8544	const ge = try (try cg.cmp(.gte, cond_tmp, lo_tmp)).materialize(cg);
8545	const le = try (try cg.cmp(.lte, cond_tmp, hi_tmp)).materialize(cg);
8546	const in_range = cg.allocId();
8547	try cg.body.emit(gpa, .OpLogicalAnd, .{
8548	.id_result_type = bool_ty_id,
8549	.id_result = in_range,
8550	.operand_1 = ge,
8551	.operand_2 = le,
8552	});
8553	case_cond = if (case_cond) |prev| blk: {
8554	const combined = cg.allocId();
8555	try cg.body.emit(gpa, .OpLogicalOr, .{
8556	.id_result_type = bool_ty_id,
8557	.id_result = combined,
8558	.operand_1 = prev,
8559	.operand_2 = in_range,
8560	});
8561	break :blk combined;
8562	} else in_range;
8563	}
8564
8565	const case_label = case_labels.at(case.idx);
8566	const is_last = case.idx == last_range_case.?;
8567	const next_check = if (is_last) default_label else cg.allocId();
8568
8569	try cg.body.emit(gpa, .OpSelectionMerge, .{
8570	.merge_block = next_check,
8571	.selection_control = .{},
8572	});
8573
8574	try cg.body.emit(gpa, .OpBranchConditional, .{
8575	.condition = case_cond.?,
8576	.true_label = case_label,
8577	.false_label = next_check,
8578	});
8579
8580	if (!is_last) {
8581	try cg.beginSpvBlock(next_check);
8582	}
8583	}
8584	}
8585
8586	{
8587	var it = switch_br.iterateCases();
8588	while (it.next()) |case| {
8589	const label = case_labels.at(case.idx);
8590	try cg.beginSpvBlock(label);
8591
8592	const next_block = try cg.genStructuredBody(.selection, case.body);
8593	incoming_structured_blocks.appendAssumeCapacity(.{
8594	.src_label = cg.block_label,
8595	.next_block = next_block,
8596	});
8597	try cg.body.emit(gpa, .OpBranch, .{ .target_label = switch_merge });
8598	}
8599	}
8600
8601	const else_body = blk: {
8602	var it_else = switch_br.iterateCases();
8603	while (it_else.next()) |_| {}
8604	break :blk it_else.elseBody();
8605	};
8606	try cg.beginSpvBlock(default_label);
8607	if (else_body.len != 0) {
8608	const next_block = try cg.genStructuredBody(.selection, else_body);
8609	incoming_structured_blocks.appendAssumeCapacity(.{
8610	.src_label = cg.block_label,
8611	.next_block = next_block,
8612	});
8613	try cg.body.emit(gpa, .OpBranch, .{ .target_label = switch_merge });
8614	} else {
8615	try cg.body.emit(gpa, .OpUnreachable, {});
8616	}
8617
8618	try cg.beginSpvBlock(switch_merge);
8619	const next_block = try cg.structuredNextBlock(incoming_structured_blocks.items);
8620
8621	const is_dispatch = cg.allocId();
8622	const bool_ty_id = try cg.resolveType(.bool, .direct);
8623	try cg.body.emit(gpa, .OpIEqual, .{
8624	.id_result_type = bool_ty_id,
8625	.id_result = is_dispatch,
8626	.operand_1 = next_block,
8627	.operand_2 = dispatch_sentinel,
8628	});
8629
8630	const dispatch_check_merge = cg.allocId();
8631	try cg.body.emit(gpa, .OpSelectionMerge, .{
8632	.merge_block = dispatch_check_merge,
8633	.selection_control = .{},
8634	});
8635	const exit_block = cg.allocId();
8636	try cg.body.emit(gpa, .OpBranchConditional, .{
8637	.condition = is_dispatch,
8638	.true_label = dispatch_check_merge,
8639	.false_label = exit_block,
8640	});
8641
8642	try cg.beginSpvBlock(exit_block);
8643	try cg.body.emit(gpa, .OpBranch, .{ .target_label = loop_merge });
8644
8645	try cg.beginSpvBlock(dispatch_check_merge);
8646	try cg.body.emit(gpa, .OpBranch, .{ .target_label = continue_label });
8647
8648	try cg.beginSpvBlock(continue_label);
8649	try cg.body.emit(gpa, .OpBranch, .{ .target_label = header_label });
8650
8651	try cg.beginSpvBlock(loop_merge);
8652	try cg.structuredBreak(next_block);
8653	}
8654
8655	fn airSwitchDispatch(cg: *CodeGen, inst: Air.Inst.Index) !void {
8656	const br = cg.air.instructions.items(.data)[@backingInt(inst)].br;
8657	const loop_switch = cg.loop_switches.get(br.block_inst).?;
8658	const cond_ty = cg.typeOf(br.operand);
8659	const operand = try cg.resolve(br.operand);
8660	const operand_indirect = try cg.convertToIndirect(cond_ty, operand);
8661
8662	try cg.store(cond_ty, loop_switch.cond_var, operand_indirect, .{});
8663	try cg.structuredBreak(loop_switch.continue_label);
8664	}
8665
8666	fn airUnreach(cg: *CodeGen) !void {
8667	try cg.body.emit(cg.gpa, .OpUnreachable, {});
8668	}
8669
8670	fn airDbgStmt(cg: *CodeGen, inst: Air.Inst.Index) !void {
8671	const zcu = cg.zcu;
8672	const dbg_stmt = cg.air.instructions.items(.data)[@backingInt(inst)].dbg_stmt;
8673	const path = zcu.navFileScope(cg.owner_nav).sub_file_path;
8674
8675	if (zcu.comp.config.root_strip) return;
8676
8677	const path_id = cg.allocId();
8678	try cg.sections.debug_strings.emit(cg.gpa, .OpString, .{
8679	.id_result = path_id,
8680	.string = path,
8681	});
8682	try cg.body.emit(cg.gpa, .OpLine, .{
8683	.file = path_id,
8684	.line = cg.base_line + dbg_stmt.line + 1,
8685	.column = dbg_stmt.column + 1,
8686	});
8687	}
8688
8689	fn airDbgInlineBlock(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
8690	const zcu = cg.zcu;
8691	const block = cg.air.unwrapDbgBlock(inst);
8692	const old_base_line = cg.base_line;
8693	defer cg.base_line = old_base_line;
8694	cg.base_line = zcu.navSrcLine(zcu.funcInfo(block.func).owner_nav);
8695	return cg.lowerBlock(inst, block.body);
8696	}
8697
8698	fn airDbgVar(cg: *CodeGen, inst: Air.Inst.Index) !void {
8699	const pl_op = cg.air.instructions.items(.data)[@backingInt(inst)].pl_op;
8700	const target_id = switch (try cg.resolvePtr(pl_op.operand)) {
8701	.tracked => return,
8702	.id => |id| id,
8703	};
8704	const name: Air.NullTerminatedString = @fromBackingInt(@intCast(pl_op.payload));
8705	try cg.debugName(target_id, name.toSlice(cg.air));
8706	}
8707
8708	fn airAssembly(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
8709	const gpa = cg.gpa;
8710	const zcu = cg.zcu;
8711	const unwrapped_asm = cg.air.unwrapAsm(inst);
8712
8713	const is_volatile = unwrapped_asm.is_volatile;
8714	const outputs_len = unwrapped_asm.outputs.len;
8715
8716	if (!is_volatile and cg.liveness.isUnused(inst)) return null;
8717
8718	if (outputs_len > 1) {
8719	return cg.todo("implement inline asm with more than 1 output", .{});
8720	}
8721
8722	var ass: Assembler = .{ .cg = cg };
8723	defer ass.deinit();
8724
8725	var it = unwrapped_asm.iterateOutputs();
8726	while (it.next()) |out| {
8727	if (out.operand != .none) {
8728	return cg.todo("implement inline asm with non-returned output", .{});
8729	}
8730	}
8731
8732	it = unwrapped_asm.iterateInputs();
8733	while (it.next()) |in| {
8734	const input_ty = cg.typeOf(in.operand);
8735
8736	if (std.mem.eql(u8, in.constraint, "c")) {
8737	const val: Value = .fromInterned(in.operand.toInterned().?);
8738	const ip = &zcu.intern_pool;
8739	const target = cg.pt.zcu.getTarget();
8740	switch (input_ty.zigTypeTag(zcu)) {
8741	.int => {
8742	const bits: u64 = switch (input_ty.intInfo(zcu).signedness) {
8743	.unsigned => val.toUnsignedInt(zcu),
8744	.signed => @bitCast(val.toSignedInt(zcu)),
8745	};
8746	try ass.value_map.put(gpa, in.name, .{ .constant = bits });
8747	},
8748	.float => {
8749	const bits: u64 = switch (input_ty.floatBits(target)) {
8750	16 => @as(u16, @bitCast(val.toFloat(f16, zcu))),
8751	32 => @as(u32, @bitCast(val.toFloat(f32, zcu))),
8752	64 => @bitCast(val.toFloat(f64, zcu)),
8753	else => unreachable, // Sema rejects unsupported float widths.
8754	};
8755	try ass.value_map.put(gpa, in.name, .{ .constant = bits });
8756	},
8757	.vector => {
8758	const child_ty = input_ty.childType(zcu);
8759	const child_kind = child_ty.zigTypeTag(zcu);
8760	const child_bit_width: u16 = switch (child_kind) {
8761	.bool => 0,
8762	.int => @intCast(child_ty.intInfo(zcu).bits),
8763	.float => child_ty.floatBits(target),
8764	else => unreachable, // Sema rejects unsupported vector element types.
8765	};
8766	const vec_len: usize = @intCast(input_ty.vectorLen(zcu));
8767	const values = try gpa.alloc(u64, vec_len);
8768	errdefer gpa.free(values);
8769	for (values, 0..) |*out, i| {
8770	const elem: Value = try val.elemValue(cg.pt, i);
8771	out.* = switch (child_kind) {
8772	.bool => @intFromBool(elem.toBool()),
8773	.int => switch (child_ty.intInfo(zcu).signedness) {
8774	.unsigned => elem.toUnsignedInt(zcu),
8775	.signed => @bitCast(elem.toSignedInt(zcu)),
8776	},
8777	.float => switch (child_bit_width) {
8778	16 => @as(u16, @bitCast(elem.toFloat(f16, zcu))),
8779	32 => @as(u32, @bitCast(elem.toFloat(f32, zcu))),
8780	64 => @bitCast(elem.toFloat(f64, zcu)),
8781	else => unreachable,
8782	},
8783	else => unreachable,
8784	};
8785	}
8786	const child_ty_id = try cg.resolveType(child_ty, .direct);
8787	try ass.value_map.put(gpa, in.name, .{ .constant_composite = .{
8788	.child = child_ty_id,
8789	.child_kind = child_kind,
8790	.child_bit_width = child_bit_width,
8791	.values = values,
8792	} });
8793	},
8794	.@"enum" => switch (ip.indexToKey(val.toIntern())) {
8795	.enum_literal => |str| try ass.value_map.put(gpa, in.name, .{ .string = str.toSlice(ip) }),
8796	else => unreachable,
8797	},
8798	else => unreachable, // Sema rejects unsupported types.
8799	}
8800	} else if (std.mem.eql(u8, in.constraint, "t")) {
8801	// type
8802	if (input_ty.zigTypeTag(zcu) == .type) {
8803	// This assembly input is a type instead of a value.
8804	// That's fine for now, just make sure to resolve it as such.
8805	const ty_id = try cg.resolveType(in.operand.toType(), .direct);
8806	try ass.value_map.put(gpa, in.name, .{ .ty = ty_id });
8807	} else {
8808	const ty_id = try cg.resolveType(input_ty, .direct);
8809	try ass.value_map.put(gpa, in.name, .{ .ty = ty_id });
8810	}
8811	} else {
8812	if (input_ty.zigTypeTag(zcu) == .type) {
8813	return cg.fail("use the 't' constraint to supply types to SPIR-V inline assembly", .{});
8814	}
8815
8816	const val_id = try cg.resolve(in.operand);
8817	try ass.value_map.put(gpa, in.name, .{ .value = val_id });
8818	}
8819	}
8820	// TODO: do something with clobbers
8821	_ = unwrapped_asm.clobbers;
8822
8823	const asm_source = unwrapped_asm.source;
8824
8825	ass.assemble(asm_source) catch |err| switch (err) {
8826	error.AssembleFail => {
8827	// TODO: For now the compiler only supports a single error message per decl,
8828	// so to translate the possible multiple errors from the assembler, emit
8829	// them as notes here.
8830	// TODO: Translate proper error locations.
8831	assert(ass.errors.items.len != 0);
8832	const msg: *Zcu.ErrorMsg = msg: {
8833	const src_loc = zcu.navSrcLoc(cg.owner_nav);
8834	var msg: *Zcu.ErrorMsg = try .create(zcu.gpa, src_loc, "failed to assemble SPIR-V inline assembly", .{});
8835	errdefer msg.destroy(zcu.gpa);
8836
8837	const notes = try zcu.gpa.alloc(Zcu.ErrorMsg, ass.errors.items.len);
8838	errdefer zcu.gpa.free(notes);
8839
8840	var i: usize = 0;
8841	errdefer for (notes[0..i]) |*note| {
8842	note.deinit(zcu.gpa);
8843	};
8844
8845	while (i < ass.errors.items.len) : (i += 1) {
8846	notes[i] = try Zcu.ErrorMsg.init(zcu.gpa, src_loc, "{s}", .{ass.errors.items[i].msg});
8847	}
8848
8849	msg.notes = notes;
8850	break :msg msg;
8851	};
8852	return zcu.codegenFailMsg(cg.owner_nav, msg);
8853	},
8854	else => |others| return others,
8855	};
8856
8857	it = unwrapped_asm.iterateOutputs();
8858	while (it.next()) |out| {
8859	const result = ass.value_map.get(out.name) orelse return {
8860	return cg.fail("invalid asm output '{s}'", .{out.name});
8861	};
8862	switch (result) {
8863	.just_declared, .unresolved_forward_reference => unreachable,
8864	.ty => return cg.fail("cannot return spir-v type as value from assembly", .{}),
8865	.value => |ref| return ref,
8866	.constant, .constant_composite, .string => return cg.fail("cannot return constant from assembly", .{}),
8867	}
8868	// TODO: Multiple results
8869	// TODO: Check that the output type from assembly is the same as the type actually expected by Zig.
8870
8871	}
8872
8873	return null;
8874	}
8875
8876	fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier) !?Id {
8877	_ = modifier;
8878
8879	const gpa = cg.gpa;
8880	const zcu = cg.zcu;
8881	const air_call = cg.air.unwrapCall(inst);
8882	const args = air_call.args;
8883	const callee_ty = cg.typeOf(air_call.callee);
8884	const zig_fn_ty = switch (callee_ty.zigTypeTag(zcu)) {
8885	.@"fn" => callee_ty,
8886	else => unreachable, // rejected by Sema for SPIR-V
8887	};
8888	const fn_info = zcu.typeToFunc(zig_fn_ty).?;
8889	const return_type = fn_info.return_type;
8890
8891	const result_type_id = try cg.resolveFnReturnType(.fromInterned(return_type));
8892	const result_id = cg.allocId();
8893	const callee_id = try cg.resolve(air_call.callee);
8894
8895	const scratch_top = cg.id_scratch.items.len;
8896	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
8897	const params = try cg.id_scratch.addManyAsSlice(gpa, args.len);
8898
8899	var n_params: usize = 0;
8900	for (args) |arg| {
8901	// Note: resolve() might emit instructions, so we need to call it
8902	// before starting to emit OpFunctionCall instructions. Hence the
8903	// temporary params buffer.
8904	const arg_ty = cg.typeOf(arg);
8905	if (!arg_ty.hasRuntimeBits(zcu)) continue;
8906
8907	if (arg_ty.zigTypeTag(zcu) == .pointer and !arg_ty.isSlice(zcu) and
8908	!arg_ty.childType(zcu).hasRuntimeBits(zcu) and
8909	cg.storageClass(arg_ty.ptrAddressSpace(zcu)) == .function)
8910	{
8911	// in logical addressing, pointer arguments to function calls
8912	// must be memory object declarations (OpVariable). for pointers to
8913	// zero-sized types, the source value may not be a variable, so just
8914	// allocate a dummy one.
8915	const child_ty_id = try cg.resolveType(arg_ty.childType(zcu), .indirect);
8916	params[n_params] = try cg.alloc(child_ty_id, null);
8917	} else {
8918	params[n_params] = try cg.resolve(arg);
8919	}
8920	n_params += 1;
8921	}
8922
8923	try cg.body.emit(gpa, .OpFunctionCall, .{
8924	.id_result_type = result_type_id,
8925	.id_result = result_id,
8926	.function = callee_id,
8927	.id_ref_3 = params[0..n_params],
8928	});
8929
8930	if (cg.liveness.isUnused(inst) or !Type.fromInterned(return_type).hasRuntimeBits(zcu)) {
8931	return null;
8932	}
8933
8934	return result_id;
8935	}
8936
8937	fn builtin3D(
8938	cg: *CodeGen,
8939	result_ty: Type,
8940	built_in: spec.BuiltIn,
8941	dimension: u32,
8942	out_of_range_value: anytype,
8943	) !Id {
8944	const gpa = cg.gpa;
8945	if (dimension >= 3) return try cg.constInt(result_ty, out_of_range_value);
8946	const u32_ty_id = try cg.intType(.unsigned, 32);
8947	const vec_ty_id = try cg.vectorType(3, u32_ty_id);
8948	const ptr_ty_id = try cg.ptrType(vec_ty_id, .input);
8949	const builtins_gop = try cg.builtins.getOrPut(gpa, .{ built_in, .input });
8950	if (!builtins_gop.found_existing) {
8951	builtins_gop.value_ptr.* = try cg.allocDecl(.global);
8952	const decl = cg.declPtr(builtins_gop.value_ptr.*);
8953	try cg.sections.globals.emit(gpa, .OpVariable, .{
8954	.id_result_type = ptr_ty_id,
8955	.id_result = decl.result_id,
8956	.storage_class = .input,
8957	});
8958	try cg.decorate(decl.result_id, .{ .built_in = .{ .built_in = built_in } });
8959	}
8960	const spv_decl_index = builtins_gop.value_ptr.*;
8961	try cg.decl_deps.append(gpa, spv_decl_index);
8962	const ptr_id = cg.declPtr(spv_decl_index).result_id;
8963	const vec_id = cg.allocId();
8964	try cg.body.emit(gpa, .OpLoad, .{
8965	.id_result_type = vec_ty_id,
8966	.id_result = vec_id,
8967	.pointer = ptr_id,
8968	});
8969	return try cg.extractVectorComponent(result_ty, vec_id, dimension);
8970	}
8971
8972	fn airWorkItemId(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
8973	if (cg.liveness.isUnused(inst)) return null;
8974	const pl_op = cg.air.instructions.items(.data)[@backingInt(inst)].pl_op;
8975	const dimension = pl_op.payload;
8976	return try cg.builtin3D(.u32, .local_invocation_id, dimension, 0);
8977	}
8978
8979	// TODO: this must be an OpConstant/OpSpec but even then the driver crashes.
8980	fn airWorkGroupSize(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
8981	if (cg.liveness.isUnused(inst)) return null;
8982	const pl_op = cg.air.instructions.items(.data)[@backingInt(inst)].pl_op;
8983	const dimension = pl_op.payload;
8984	return try cg.builtin3D(.u32, .workgroup_size, dimension, 0);
8985	}
8986
8987	fn airWorkGroupId(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
8988	if (cg.liveness.isUnused(inst)) return null;
8989	const pl_op = cg.air.instructions.items(.data)[@backingInt(inst)].pl_op;
8990	const dimension = pl_op.payload;
8991	return try cg.builtin3D(.u32, .workgroup_id, dimension, 0);
8992	}
8993
8994	const std = @import("std");
8995	const Allocator = std.mem.Allocator;
8996	const Target = std.Target;
8997	const Signedness = std.lang.Signedness;
8998	const assert = std.debug.assert;
8999	const log = std.log.scoped(.codegen);
9000
9001	const builtin = @import("builtin");
9002	const link = @import("../../link.zig");
9003	const codegen = @import("../../codegen.zig");
9004	const Zcu = @import("../../Zcu.zig");
9005	const Type = @import("../../Type.zig");
9006	const Value = @import("../../Value.zig");
9007	const Air = @import("../../Air.zig");
9008	const InternPool = @import("../../InternPool.zig");
9009	const Section = @import("Section.zig");
9010	const Assembler = @import("Assembler.zig");
9011	const Mir = @import("Mir.zig");
9012
9013	const spec = @import("spec.zig");
9014	const Opcode = spec.Opcode;
9015	const Word = spec.Word;
9016	const Id = spec.Id;
9017	const IdRange = spec.IdRange;
9018	const StorageClass = spec.StorageClass;
9019
9020	const CodeGen = @This();