Meg's Mirrors

zig

1	const std = @import("std");
2	const builtin = @import("builtin");
3	const build_options = @import("build_options");
4
5	const mem = std.mem;
6	const math = std.math;
7	const assert = std.debug.assert;
8	const Allocator = mem.Allocator;
9
10	const Air = @import("../../Air.zig");
11	const Mir = @import("Mir.zig");
12	const Emit = @import("Emit.zig");
13	const Type = @import("../../Type.zig");
14	const Value = @import("../../Value.zig");
15	const link = @import("../../link.zig");
16	const Zcu = @import("../../Zcu.zig");
17	const Module = @import("../../Module.zig");
18	const InternPool = @import("../../InternPool.zig");
19	const Compilation = @import("../../Compilation.zig");
20	const target_util = @import("../../target.zig");
21	const trace = @import("../../tracy.zig").trace;
22	const codegen = @import("../../codegen.zig");
23
24	const ErrorMsg = Zcu.ErrorMsg;
25	const Target = std.Target;
26
27	const log = std.log.scoped(.riscv_codegen);
28	const tracking_log = std.log.scoped(.tracking);
29	const verbose_tracking_log = std.log.scoped(.verbose_tracking);
30	const wip_mir_log = std.log.scoped(.wip_mir);
31	const Alignment = InternPool.Alignment;
32
33	const bits = @import("bits.zig");
34	const abi = @import("abi.zig");
35	const Lower = @import("Lower.zig");
36	const mnem_import = @import("mnem.zig");
37	const Mnemonic = mnem_import.Mnemonic;
38	const Pseudo = mnem_import.Pseudo;
39	const encoding = @import("encoding.zig");
40
41	const Register = bits.Register;
42	const CSR = bits.CSR;
43	const Immediate = bits.Immediate;
44	const Memory = bits.Memory;
45	const FrameIndex = bits.FrameIndex;
46	const RegisterManager = abi.RegisterManager;
47	const RegisterLock = RegisterManager.RegisterLock;
48	const Instruction = encoding.Instruction;
49
50	const InnerError = codegen.Error || error{OutOfRegisters};
51
52	pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
53	return comptime &.initMany(&.{
54	.expand_bit_cast_safe,
55	.expand_int_cast_safe,
56	.expand_int_from_float_safe,
57	.expand_int_from_float_optimized_safe,
58	.expand_add_safe,
59	.expand_sub_safe,
60	.expand_mul_safe,
61
62	.expand_array_splat,
63	.expand_array_to_vector,
64	});
65	}
66
67	pt: Zcu.PerThread,
68	air: Air,
69	liveness: Air.Liveness,
70	bin_file: *link.File,
71	gpa: Allocator,
72
73	mod: *Module,
74	target: *const std.Target,
75	args: []MCValue,
76	ret_mcv: InstTracking,
77	func_index: InternPool.Index,
78	fn_type: Type,
79	arg_index: usize,
80
81	mir_instructions: std.MultiArrayList(Mir.Inst) = .{},
82
83	owner: Owner,
84
85	/// Byte offset within the source file of the ending curly.
86	end_di_line: u32,
87	end_di_column: u32,
88
89	scope_generation: u32,
90
91	/// The value is an offset into the `Function` `code` from the beginning.
92	/// To perform the reloc, write 32-bit signed little-endian integer
93	/// which is a relative jump, based on the address following the reloc.
94	exitlude_jump_relocs: std.ArrayList(usize) = .empty,
95
96	reused_operands: std.bit_set.Static(Air.Liveness.bpi - 1) = undefined,
97
98	/// Whenever there is a runtime branch, we push a Branch onto this stack,
99	/// and pop it off when the runtime branch joins. This provides an "overlay"
100	/// of the table of mappings from instructions to `MCValue` from within the branch.
101	/// This way we can modify the `MCValue` for an instruction in different ways
102	/// within different branches. Special consideration is needed when a branch
103	/// joins with its parent, to make sure all instructions have the same MCValue
104	/// across each runtime branch upon joining.
105	branch_stack: *std.array_list.Managed(Branch),
106
107	// Currently set vector properties, null means they haven't been set yet in the function.
108	avl: ?u64,
109	vtype: ?bits.VType,
110
111	// Key is the block instruction
112	blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .empty,
113	register_manager: RegisterManager = .{},
114
115	const_tracking: ConstTrackingMap = .{},
116	inst_tracking: InstTrackingMap = .{},
117
118	frame_allocs: std.MultiArrayList(FrameAlloc) = .{},
119	free_frame_indices: std.array_hash_map.Auto(FrameIndex, void) = .empty,
120	frame_locs: std.MultiArrayList(Mir.FrameLoc) = .{},
121
122	loops: std.AutoHashMapUnmanaged(Air.Inst.Index, struct {
123	/// The state to restore before branching.
124	state: State,
125	/// The branch target.
126	jmp_target: Mir.Inst.Index,
127	}) = .{},
128
129	/// Debug field, used to find bugs in the compiler.
130	air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init,
131
132	const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {};
133
134	const SymbolOffset = struct { sym: link.File.SymbolId, off: i32 = 0 };
135	const RegisterOffset = struct { reg: Register, off: i32 = 0 };
136	pub const FrameAddr = struct { index: FrameIndex, off: i32 = 0 };
137
138	const Owner = union(enum) {
139	nav_index: InternPool.Nav.Index,
140	lazy_sym: link.File.LazySymbol,
141
142	fn getSymbolIndex(owner: Owner, func: *Func) !u32 {
143	const pt = func.pt;
144	switch (owner) {
145	.nav_index => |nav_index| {
146	const elf_file = func.bin_file.cast(.elf).?;
147	return elf_file.zigObjectPtr().?.getOrCreateMetadataForNav(pt.zcu, nav_index);
148	},
149	.lazy_sym => |lazy_sym| {
150	const elf_file = func.bin_file.cast(.elf).?;
151	return elf_file.zigObjectPtr().?.getOrCreateMetadataForLazySymbol(elf_file, pt, lazy_sym) catch |err|
152	func.fail("{s} creating lazy symbol", .{@errorName(err)});
153	},
154	}
155	}
156	};
157
158	const MCValue = union(enum) {
159	/// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc.
160	/// TODO Look into deleting this tag and using `dead` instead, since every use
161	/// of MCValue.none should be instead looking at the type and noticing it is 0 bits.
162	none,
163	/// Control flow will not allow this value to be observed.
164	unreach,
165	/// No more references to this value remain.
166	/// The payload is the value of scope_generation at the point where the death occurred
167	dead: u32,
168	/// The value is undefined. Contains a symbol index to an undefined constant. Null means
169	/// set the undefined value via immediate instead of a load.
170	undef: ?link.File.SymbolId,
171	/// A pointer-sized integer that fits in a register.
172	/// If the type is a pointer, this is the pointer address in virtual address space.
173	immediate: u64,
174	/// The value doesn't exist in memory yet.
175	load_symbol: SymbolOffset,
176	/// The address of the memory location not-yet-allocated by the linker.
177	lea_symbol: SymbolOffset,
178	/// The value is in a target-specific register.
179	register: Register,
180	/// The value is split across two registers
181	register_pair: [2]Register,
182	/// The value is in memory at a hard-coded address.
183	/// If the type is a pointer, it means the pointer address is at this memory location.
184	memory: u64,
185	/// The value stored at an offset from a frame index
186	/// Payload is a frame address.
187	load_frame: FrameAddr,
188	/// The address of an offset from a frame index
189	/// Payload is a frame address.
190	lea_frame: FrameAddr,
191	air_ref: Air.Inst.Ref,
192	/// The value is in memory at a constant offset from the address in a register.
193	indirect: RegisterOffset,
194	/// The value is a constant offset from the value in a register.
195	register_offset: RegisterOffset,
196	/// This indicates that we have already allocated a frame index for this instruction,
197	/// but it has not been spilled there yet in the current control flow.
198	/// Payload is a frame index.
199	reserved_frame: FrameIndex,
200
201	fn isMemory(mcv: MCValue) bool {
202	return switch (mcv) {
203	.memory, .indirect, .load_frame => true,
204	else => false,
205	};
206	}
207
208	fn isImmediate(mcv: MCValue) bool {
209	return switch (mcv) {
210	.immediate => true,
211	else => false,
212	};
213	}
214
215	fn isRegister(mcv: MCValue) bool {
216	return switch (mcv) {
217	.register => true,
218	.register_offset => |reg_off| return reg_off.off == 0,
219	else => false,
220	};
221	}
222
223	fn isMutable(mcv: MCValue) bool {
224	return switch (mcv) {
225	.none => unreachable,
226	.unreach => unreachable,
227	.dead => unreachable,
228
229	.immediate,
230	.memory,
231	.lea_frame,
232	.undef,
233	.lea_symbol,
234	.air_ref,
235	.reserved_frame,
236	=> false,
237
238	.register,
239	.register_pair,
240	.register_offset,
241	.load_symbol,
242	.indirect,
243	=> true,
244
245	.load_frame => |frame_addr| !frame_addr.index.isNamed(),
246	};
247	}
248
249	fn address(mcv: MCValue) MCValue {
250	return switch (mcv) {
251	.none,
252	.unreach,
253	.dead,
254	.immediate,
255	.lea_frame,
256	.register_offset,
257	.register_pair,
258	.register,
259	.undef,
260	.air_ref,
261	.lea_symbol,
262	.reserved_frame,
263	=> unreachable, // not in memory
264
265	.load_symbol => |sym_off| .{ .lea_symbol = sym_off },
266	.memory => |addr| .{ .immediate = addr },
267	.load_frame => |off| .{ .lea_frame = off },
268	.indirect => |reg_off| switch (reg_off.off) {
269	0 => .{ .register = reg_off.reg },
270	else => .{ .register_offset = reg_off },
271	},
272	};
273	}
274
275	fn deref(mcv: MCValue) MCValue {
276	return switch (mcv) {
277	.none,
278	.unreach,
279	.dead,
280	.memory,
281	.indirect,
282	.undef,
283	.air_ref,
284	.register_pair,
285	.load_frame,
286	.load_symbol,
287	.reserved_frame,
288	=> unreachable, // not a pointer
289
290	.immediate => |addr| .{ .memory = addr },
291	.register => |reg| .{ .indirect = .{ .reg = reg } },
292	.register_offset => |reg_off| .{ .indirect = reg_off },
293	.lea_frame => |off| .{ .load_frame = off },
294	.lea_symbol => |sym_off| .{ .load_symbol = sym_off },
295	};
296	}
297
298	fn offset(mcv: MCValue, off: i32) MCValue {
299	return switch (mcv) {
300	.none,
301	.unreach,
302	.dead,
303	.undef,
304	.air_ref,
305	.reserved_frame,
306	=> unreachable, // not valid
307	.register_pair,
308	.memory,
309	.indirect,
310	.load_symbol,
311	.lea_symbol,
312	=> switch (off) {
313	0 => mcv,
314	else => unreachable,
315	},
316	.load_frame => |frame| .{ .load_frame = .{ .index = frame.index, .off = frame.off + off } },
317	.immediate => |imm| .{ .immediate = @bitCast(@as(i64, @bitCast(imm)) +% off) },
318	.register => |reg| .{ .register_offset = .{ .reg = reg, .off = off } },
319	.register_offset => |reg_off| .{ .register_offset = .{ .reg = reg_off.reg, .off = reg_off.off + off } },
320	.lea_frame => |frame_addr| .{
321	.lea_frame = .{ .index = frame_addr.index, .off = frame_addr.off + off },
322	},
323	};
324	}
325
326	fn getReg(mcv: MCValue) ?Register {
327	return switch (mcv) {
328	.register => |reg| reg,
329	.register_offset, .indirect => |ro| ro.reg,
330	else => null,
331	};
332	}
333
334	fn getRegs(mcv: *const MCValue) []const Register {
335	return switch (mcv.*) {
336	.register => |*reg| @as(*const [1]Register, reg),
337	.register_pair => |*regs| regs,
338	.register_offset, .indirect => |*ro| @as(*const [1]Register, &ro.reg),
339	else => &.{},
340	};
341	}
342	};
343
344	const Branch = struct {
345	inst_table: std.array_hash_map.Auto(Air.Inst.Index, MCValue) = .empty,
346
347	fn deinit(func: *Branch, gpa: Allocator) void {
348	func.inst_table.deinit(gpa);
349	func.* = undefined;
350	}
351	};
352
353	const InstTrackingMap = std.array_hash_map.Auto(Air.Inst.Index, InstTracking);
354	const ConstTrackingMap = std.array_hash_map.Auto(InternPool.Index, InstTracking);
355
356	const InstTracking = struct {
357	long: MCValue,
358	short: MCValue,
359
360	fn init(result: MCValue) InstTracking {
361	return .{ .long = switch (result) {
362	.none,
363	.unreach,
364	.undef,
365	.immediate,
366	.memory,
367	.load_frame,
368	.lea_frame,
369	.load_symbol,
370	.lea_symbol,
371	=> result,
372	.dead,
373	.reserved_frame,
374	.air_ref,
375	=> unreachable,
376	.register,
377	.register_pair,
378	.register_offset,
379	.indirect,
380	=> .none,
381	}, .short = result };
382	}
383
384	fn getReg(inst_tracking: InstTracking) ?Register {
385	return inst_tracking.short.getReg();
386	}
387
388	fn getRegs(inst_tracking: *const InstTracking) []const Register {
389	return inst_tracking.short.getRegs();
390	}
391
392	fn spill(inst_tracking: *InstTracking, function: *Func, inst: Air.Inst.Index) !void {
393	if (std.meta.eql(inst_tracking.long, inst_tracking.short)) return; // Already spilled
394	// Allocate or reuse frame index
395	switch (inst_tracking.long) {
396	.none => inst_tracking.long = try function.allocRegOrMem(
397	function.typeOfIndex(inst),
398	inst,
399	false,
400	),
401	.load_frame => {},
402	.reserved_frame => |index| inst_tracking.long = .{ .load_frame = .{ .index = index } },
403	else => unreachable,
404	}
405	tracking_log.debug("spill %{d} from {} to {}", .{ inst, inst_tracking.short, inst_tracking.long });
406	try function.genCopy(function.typeOfIndex(inst), inst_tracking.long, inst_tracking.short);
407	}
408
409	fn reuseFrame(inst_tracking: *InstTracking) void {
410	switch (inst_tracking.long) {
411	.reserved_frame => |index| inst_tracking.long = .{ .load_frame = .{ .index = index } },
412	else => {},
413	}
414	inst_tracking.short = switch (inst_tracking.long) {
415	.none,
416	.unreach,
417	.undef,
418	.immediate,
419	.memory,
420	.load_frame,
421	.lea_frame,
422	.load_symbol,
423	.lea_symbol,
424	=> inst_tracking.long,
425	.dead,
426	.register,
427	.register_pair,
428	.register_offset,
429	.indirect,
430	.reserved_frame,
431	.air_ref,
432	=> unreachable,
433	};
434	}
435
436	fn trackSpill(inst_tracking: *InstTracking, function: *Func, inst: Air.Inst.Index) !void {
437	try function.freeValue(inst_tracking.short);
438	inst_tracking.reuseFrame();
439	tracking_log.debug("%{d} => {f} (spilled)", .{ inst, inst_tracking.* });
440	}
441
442	fn verifyMaterialize(inst_tracking: InstTracking, target: InstTracking) void {
443	switch (inst_tracking.long) {
444	.none,
445	.unreach,
446	.undef,
447	.immediate,
448	.memory,
449	.lea_frame,
450	.load_symbol,
451	.lea_symbol,
452	=> assert(std.meta.eql(inst_tracking.long, target.long)),
453	.load_frame,
454	.reserved_frame,
455	=> switch (target.long) {
456	.none,
457	.load_frame,
458	.reserved_frame,
459	=> {},
460	else => unreachable,
461	},
462	.dead,
463	.register,
464	.register_pair,
465	.register_offset,
466	.indirect,
467	.air_ref,
468	=> unreachable,
469	}
470	}
471
472	fn materialize(
473	inst_tracking: *InstTracking,
474	function: *Func,
475	inst: Air.Inst.Index,
476	target: InstTracking,
477	) !void {
478	inst_tracking.verifyMaterialize(target);
479	try inst_tracking.materializeUnsafe(function, inst, target);
480	}
481
482	fn materializeUnsafe(
483	inst_tracking: InstTracking,
484	function: *Func,
485	inst: Air.Inst.Index,
486	target: InstTracking,
487	) !void {
488	const ty = function.typeOfIndex(inst);
489	if ((inst_tracking.long == .none or inst_tracking.long == .reserved_frame) and target.long == .load_frame)
490	try function.genCopy(ty, target.long, inst_tracking.short);
491	try function.genCopy(ty, target.short, inst_tracking.short);
492	}
493
494	fn trackMaterialize(inst_tracking: *InstTracking, inst: Air.Inst.Index, target: InstTracking) void {
495	inst_tracking.verifyMaterialize(target);
496	// Don't clobber reserved frame indices
497	inst_tracking.long = if (target.long == .none) switch (inst_tracking.long) {
498	.load_frame => |addr| .{ .reserved_frame = addr.index },
499	.reserved_frame => inst_tracking.long,
500	else => target.long,
501	} else target.long;
502	inst_tracking.short = target.short;
503	tracking_log.debug("%{d} => {f} (materialize)", .{ inst, inst_tracking.* });
504	}
505
506	fn resurrect(inst_tracking: *InstTracking, inst: Air.Inst.Index, scope_generation: u32) void {
507	switch (inst_tracking.short) {
508	.dead => |die_generation| if (die_generation >= scope_generation) {
509	inst_tracking.reuseFrame();
510	tracking_log.debug("%{d} => {f} (resurrect)", .{ inst, inst_tracking.* });
511	},
512	else => {},
513	}
514	}
515
516	fn die(inst_tracking: *InstTracking, function: *Func, inst: Air.Inst.Index) !void {
517	if (inst_tracking.short == .dead) return;
518	try function.freeValue(inst_tracking.short);
519	inst_tracking.short = .{ .dead = function.scope_generation };
520	tracking_log.debug("%{d} => {f} (death)", .{ inst, inst_tracking.* });
521	}
522
523	fn reuse(
524	inst_tracking: *InstTracking,
525	function: *Func,
526	new_inst: ?Air.Inst.Index,
527	old_inst: Air.Inst.Index,
528	) void {
529	inst_tracking.short = .{ .dead = function.scope_generation };
530	if (new_inst) |inst|
531	tracking_log.debug("%{d} => {f} (reuse %{d})", .{ inst, inst_tracking.*, old_inst })
532	else
533	tracking_log.debug("tmp => {f} (reuse %{d})", .{ inst_tracking.*, old_inst });
534	}
535
536	fn liveOut(inst_tracking: *InstTracking, function: *Func, inst: Air.Inst.Index) void {
537	for (inst_tracking.getRegs()) |reg| {
538	if (function.register_manager.isRegFree(reg)) {
539	tracking_log.debug("%{d} => {f} (live-out)", .{ inst, inst_tracking.* });
540	continue;
541	}
542
543	const index = RegisterManager.indexOfRegIntoTracked(reg).?;
544	const tracked_inst = function.register_manager.registers[index];
545	const tracking = function.getResolvedInstValue(tracked_inst);
546
547	// Disable death.
548	var found_reg = false;
549	var remaining_reg: Register = .zero;
550	for (tracking.getRegs()) |tracked_reg| if (tracked_reg.id() == reg.id()) {
551	assert(!found_reg);
552	found_reg = true;
553	} else {
554	assert(remaining_reg == .zero);
555	remaining_reg = tracked_reg;
556	};
557	assert(found_reg);
558	tracking.short = switch (remaining_reg) {
559	.zero => .{ .dead = function.scope_generation },
560	else => .{ .register = remaining_reg },
561	};
562
563	// Perform side-effects of freeValue manually.
564	function.register_manager.freeReg(reg);
565
566	tracking_log.debug("%{d} => {f} (live-out %{d})", .{ inst, inst_tracking.*, tracked_inst });
567	}
568	}
569
570	pub fn format(inst_tracking: InstTracking, writer: *std.Io.Writer) std.Io.Writer.Error!void {
571	if (!std.meta.eql(inst_tracking.long, inst_tracking.short)) try writer.print("|{}| ", .{inst_tracking.long});
572	try writer.print("{}", .{inst_tracking.short});
573	}
574	};
575
576	const FrameAlloc = struct {
577	abi_size: u31,
578	spill_pad: u3,
579	abi_align: Alignment,
580	ref_count: u16,
581
582	fn init(alloc_abi: struct { size: u64, pad: u3 = 0, alignment: Alignment }) FrameAlloc {
583	return .{
584	.abi_size = @intCast(alloc_abi.size),
585	.spill_pad = alloc_abi.pad,
586	.abi_align = alloc_abi.alignment,
587	.ref_count = 0,
588	};
589	}
590	fn initType(ty: Type, zcu: *Zcu) FrameAlloc {
591	return init(.{
592	.size = ty.abiSize(zcu),
593	.alignment = ty.abiAlignment(zcu),
594	});
595	}
596	fn initSpill(ty: Type, zcu: *Zcu) FrameAlloc {
597	const abi_size = ty.abiSize(zcu);
598	const spill_size = if (abi_size < 8)
599	math.ceilPowerOfTwoAssert(u64, abi_size)
600	else
601	std.mem.alignForward(u64, abi_size, 8);
602	return init(.{
603	.size = spill_size,
604	.pad = @intCast(spill_size - abi_size),
605	.alignment = ty.abiAlignment(zcu).maxStrict(
606	Alignment.fromNonzeroByteUnits(@min(spill_size, 8)),
607	),
608	});
609	}
610	};
611
612	const BlockData = struct {
613	relocs: std.ArrayList(Mir.Inst.Index) = .empty,
614	state: State,
615
616	fn deinit(bd: *BlockData, gpa: Allocator) void {
617	bd.relocs.deinit(gpa);
618	bd.* = undefined;
619	}
620	};
621
622	const State = struct {
623	registers: RegisterManager.TrackedRegisters,
624	reg_tracking: [RegisterManager.RegisterBitSet.bit_length]InstTracking,
625	free_registers: RegisterManager.RegisterBitSet,
626	inst_tracking_len: u32,
627	scope_generation: u32,
628	};
629
630	fn initRetroactiveState(func: *Func) State {
631	var state: State = undefined;
632	state.inst_tracking_len = @intCast(func.inst_tracking.count());
633	state.scope_generation = func.scope_generation;
634	return state;
635	}
636
637	fn saveRetroactiveState(func: *Func, state: *State) !void {
638	const free_registers = func.register_manager.free_registers;
639	var it = free_registers.iterator(.{ .kind = .unset });
640	while (it.next()) |index| {
641	const tracked_inst = func.register_manager.registers[index];
642	state.registers[index] = tracked_inst;
643	state.reg_tracking[index] = func.inst_tracking.get(tracked_inst).?;
644	}
645	state.free_registers = free_registers;
646	}
647
648	fn saveState(func: *Func) !State {
649	var state = func.initRetroactiveState();
650	try func.saveRetroactiveState(&state);
651	return state;
652	}
653
654	fn restoreState(func: *Func, state: State, deaths: []const Air.Inst.Index, comptime opts: struct {
655	emit_instructions: bool,
656	update_tracking: bool,
657	resurrect: bool,
658	close_scope: bool,
659	}) !void {
660	if (opts.close_scope) {
661	for (
662	func.inst_tracking.keys()[state.inst_tracking_len..],
663	func.inst_tracking.values()[state.inst_tracking_len..],
664	) |inst, *tracking| try tracking.die(func, inst);
665	func.inst_tracking.shrinkRetainingCapacity(state.inst_tracking_len);
666	}
667
668	if (opts.resurrect) for (
669	func.inst_tracking.keys()[0..state.inst_tracking_len],
670	func.inst_tracking.values()[0..state.inst_tracking_len],
671	) |inst, *tracking| tracking.resurrect(inst, state.scope_generation);
672	for (deaths) |death| try func.processDeath(death);
673
674	const ExpectedContents = [@typeInfo(RegisterManager.TrackedRegisters).array.len]RegisterLock;
675	const stack_buf_len = if (opts.update_tracking) 0 else 1;
676	var bfa_buf: [stack_buf_len]ExpectedContents = undefined;
677	var bfa = if (opts.update_tracking) {} else std.heap.BufferFirstAllocator.init(@ptrCast(&bfa_buf), func.gpa);
678
679	var reg_locks = if (opts.update_tracking) {} else try std.array_list.Managed(RegisterLock).initCapacity(
680	bfa.allocator(),
681	@typeInfo(ExpectedContents).array.len,
682	);
683	defer if (!opts.update_tracking) {
684	for (reg_locks.items) |lock| func.register_manager.unlockReg(lock);
685	reg_locks.deinit();
686	};
687
688	for (0..state.registers.len) |index| {
689	const current_maybe_inst = if (func.register_manager.free_registers.isSet(index))
690	null
691	else
692	func.register_manager.registers[index];
693	const target_maybe_inst = if (state.free_registers.isSet(index))
694	null
695	else
696	state.registers[index];
697	if (std.debug.runtime_safety) if (target_maybe_inst) |target_inst|
698	assert(func.inst_tracking.getIndex(target_inst).? < state.inst_tracking_len);
699	if (opts.emit_instructions) {
700	if (current_maybe_inst) |current_inst| {
701	try func.inst_tracking.getPtr(current_inst).?.spill(func, current_inst);
702	}
703	if (target_maybe_inst) |target_inst| {
704	const target_tracking = func.inst_tracking.getPtr(target_inst).?;
705	try target_tracking.materialize(func, target_inst, state.reg_tracking[index]);
706	}
707	}
708	if (opts.update_tracking) {
709	if (current_maybe_inst) |current_inst| {
710	try func.inst_tracking.getPtr(current_inst).?.trackSpill(func, current_inst);
711	}
712	blk: {
713	const inst = target_maybe_inst orelse break :blk;
714	const reg = RegisterManager.regAtTrackedIndex(@intCast(index));
715	func.register_manager.freeReg(reg);
716	func.register_manager.getRegAssumeFree(reg, inst);
717	}
718	if (target_maybe_inst) |target_inst| {
719	func.inst_tracking.getPtr(target_inst).?.trackMaterialize(
720	target_inst,
721	state.reg_tracking[index],
722	);
723	}
724	} else if (target_maybe_inst) |_|
725	try reg_locks.append(func.register_manager.lockRegIndexAssumeUnused(@intCast(index)));
726	}
727
728	if (opts.update_tracking and std.debug.runtime_safety) {
729	assert(func.register_manager.free_registers.eql(state.free_registers));
730	var used_reg_it = state.free_registers.iterator(.{ .kind = .unset });
731	while (used_reg_it.next()) |index|
732	assert(func.register_manager.registers[index] == state.registers[index]);
733	}
734	}
735
736	const Func = @This();
737
738	const CallView = enum(u1) {
739	callee,
740	caller,
741	};
742
743	pub fn generate(
744	bin_file: *link.File,
745	pt: Zcu.PerThread,
746	func_index: InternPool.Index,
747	air: *const Air,
748	liveness: *const ?Air.Liveness,
749	) codegen.Error!Mir {
750	const zcu = pt.zcu;
751	const gpa = zcu.gpa;
752	const ip = &zcu.intern_pool;
753	const func = zcu.funcInfo(func_index);
754	const fn_type = Type.fromInterned(func.ty);
755	const mod = zcu.navFileScope(func.owner_nav).mod.?;
756
757	var branch_stack = std.array_list.Managed(Branch).init(gpa);
758	defer {
759	assert(branch_stack.items.len == 1);
760	branch_stack.items[0].deinit(gpa);
761	branch_stack.deinit();
762	}
763	try branch_stack.append(.{});
764
765	var function: Func = .{
766	.gpa = gpa,
767	.air = air.*,
768	.pt = pt,
769	.mod = mod,
770	.bin_file = bin_file,
771	.liveness = liveness.*.?,
772	.target = &mod.resolved_target.result,
773	.owner = .{ .nav_index = func.owner_nav },
774	.args = undefined, // populated after `resolveCallingConventionValues`
775	.ret_mcv = undefined, // populated after `resolveCallingConventionValues`
776	.func_index = func_index,
777	.fn_type = fn_type,
778	.arg_index = 0,
779	.branch_stack = &branch_stack,
780	.end_di_line = func.rbrace_line,
781	.end_di_column = func.rbrace_column,
782	.scope_generation = 0,
783	.avl = null,
784	.vtype = null,
785	};
786	defer {
787	function.frame_allocs.deinit(gpa);
788	function.free_frame_indices.deinit(gpa);
789	function.frame_locs.deinit(gpa);
790	function.loops.deinit(gpa);
791	var block_it = function.blocks.valueIterator();
792	while (block_it.next()) |block| block.deinit(gpa);
793	function.blocks.deinit(gpa);
794	function.inst_tracking.deinit(gpa);
795	function.const_tracking.deinit(gpa);
796	function.exitlude_jump_relocs.deinit(gpa);
797	function.mir_instructions.deinit(gpa);
798	}
799
800	wip_mir_log.debug("{f}:", .{fmtNav(func.owner_nav, ip)});
801
802	try function.frame_allocs.resize(gpa, FrameIndex.named_count);
803	function.frame_allocs.set(
804	@backingInt(FrameIndex.stack_frame),
805	FrameAlloc.init(.{ .size = 0, .alignment = .@"1" }),
806	);
807	function.frame_allocs.set(
808	@backingInt(FrameIndex.call_frame),
809	FrameAlloc.init(.{ .size = 0, .alignment = .@"1" }),
810	);
811
812	const fn_info = zcu.typeToFunc(fn_type).?;
813	var call_info = try function.resolveCallingConventionValues(fn_info, &.{});
814
815	defer call_info.deinit(&function);
816
817	function.args = call_info.args;
818	function.ret_mcv = call_info.return_value;
819	function.frame_allocs.set(@backingInt(FrameIndex.ret_addr), FrameAlloc.init(.{
820	.size = Type.u64.abiSize(zcu),
821	.alignment = Type.u64.abiAlignment(zcu).min(call_info.stack_align),
822	}));
823	function.frame_allocs.set(@backingInt(FrameIndex.base_ptr), FrameAlloc.init(.{
824	.size = Type.u64.abiSize(zcu),
825	.alignment = Alignment.min(
826	call_info.stack_align,
827	Alignment.fromNonzeroByteUnits(function.target.stackAlignment()),
828	),
829	}));
830	function.frame_allocs.set(@backingInt(FrameIndex.args_frame), FrameAlloc.init(.{
831	.size = call_info.stack_byte_count,
832	.alignment = call_info.stack_align,
833	}));
834	function.frame_allocs.set(@backingInt(FrameIndex.spill_frame), FrameAlloc.init(.{
835	.size = 0,
836	.alignment = Type.u64.abiAlignment(zcu),
837	}));
838
839	function.gen() catch |err| switch (err) {
840	error.OutOfRegisters => return function.fail("ran out of registers (Zig compiler bug)", .{}),
841	else => |e| return e,
842	};
843
844	var mir: Mir = .{
845	.instructions = function.mir_instructions.toOwnedSlice(),
846	.frame_locs = function.frame_locs.toOwnedSlice(),
847	};
848	errdefer mir.deinit(gpa);
849	return mir;
850	}
851
852	pub fn generateLazy(
853	bin_file: *link.File,
854	pt: Zcu.PerThread,
855	lazy_sym: link.File.LazySymbol,
856	atom_index: link.File.AtomId,
857	w: *std.Io.Writer,
858	debug_output: link.File.DebugInfoOutput,
859	) (codegen.Error || std.Io.Writer.Error)!void {
860	_ = atom_index;
861	const comp = bin_file.comp;
862	const gpa = comp.gpa;
863	const mod = comp.root_mod;
864
865	var function: Func = .{
866	.gpa = gpa,
867	.air = undefined,
868	.pt = pt,
869	.mod = mod,
870	.bin_file = bin_file,
871	.liveness = undefined,
872	.target = &mod.resolved_target.result,
873	.owner = .{ .lazy_sym = lazy_sym },
874	.args = undefined, // populated after `resolveCallingConventionValues`
875	.ret_mcv = undefined, // populated after `resolveCallingConventionValues`
876	.func_index = undefined,
877	.fn_type = undefined,
878	.arg_index = 0,
879	.branch_stack = undefined,
880	.end_di_line = undefined,
881	.end_di_column = undefined,
882	.scope_generation = 0,
883	.avl = null,
884	.vtype = null,
885	};
886	defer function.mir_instructions.deinit(gpa);
887
888	function.genLazy(lazy_sym) catch |err| switch (err) {
889	error.OutOfRegisters => return function.fail("ran out of registers (Zig compiler bug)", .{}),
890	else => |e| return e,
891	};
892
893	var mir: Mir = .{
894	.instructions = function.mir_instructions.toOwnedSlice(),
895	.frame_locs = function.frame_locs.toOwnedSlice(),
896	};
897	defer mir.deinit(gpa);
898
899	var emit: Emit = .{
900	.lower = .{
901	.pt = pt,
902	.allocator = gpa,
903	.mir = mir,
904	.cc = .auto,
905	.src_loc = Type.fromInterned(lazy_sym.ty).srcLocOrNull(pt.zcu) orelse .unneeded,
906	.output_mode = comp.config.output_mode,
907	.link_mode = comp.config.link_mode,
908	.pic = mod.pic,
909	},
910	.bin_file = bin_file,
911	.debug_output = debug_output,
912	.w = w,
913	.prev_di_pc = undefined, // no debug info yet
914	.prev_di_line = undefined, // no debug info yet
915	.prev_di_column = undefined, // no debug info yet
916	};
917	defer emit.deinit();
918
919	emit.emitMir() catch |err| switch (err) {
920	error.LowerFail, error.EmitFail => return function.failMsg(emit.lower.err_msg.?),
921	error.InvalidInstruction => |e| return function.fail("emit MIR failed: {s} (Zig compiler bug)", .{@errorName(e)}),
922	else => |e| return e,
923	};
924	}
925
926	const FormatWipMirData = struct {
927	func: *Func,
928	inst: Mir.Inst.Index,
929	};
930	fn formatWipMir(data: FormatWipMirData, writer: *std.Io.Writer) std.Io.Writer.Error!void {
931	const pt = data.func.pt;
932	const comp = pt.zcu.comp;
933	var lower: Lower = .{
934	.pt = pt,
935	.allocator = data.func.gpa,
936	.mir = .{
937	.instructions = data.func.mir_instructions.slice(),
938	.frame_locs = data.func.frame_locs.slice(),
939	},
940	.cc = .auto,
941	.src_loc = switch (data.func.owner) {
942	.nav_index => |nav| pt.zcu.navSrcLoc(nav),
943	.lazy_sym => |lazy_sym| Type.fromInterned(lazy_sym.ty).srcLocOrNull(pt.zcu) orelse .unneeded,
944	},
945	.output_mode = comp.config.output_mode,
946	.link_mode = comp.config.link_mode,
947	.pic = comp.root_mod.pic,
948	};
949	var first = true;
950	for ((lower.lowerMir(data.inst, .{ .allow_frame_locs = false }) catch |err| switch (err) {
951	error.LowerFail => {
952	defer {
953	lower.err_msg.?.deinit(data.func.gpa);
954	lower.err_msg = null;
955	}
956	try writer.writeAll(lower.err_msg.?.msg);
957	return;
958	},
959	error.OutOfMemory, error.InvalidInstruction => |e| {
960	try writer.writeAll(switch (e) {
961	error.OutOfMemory => "Out of memory",
962	error.InvalidInstruction => "CodeGen failed to find a viable instruction.",
963	});
964	return;
965	},
966	else => |e| return e,
967	}).insts) |lowered_inst| {
968	if (!first) try writer.writeAll("\ndebug(wip_mir): ");
969	try writer.print(" | {}", .{lowered_inst});
970	first = false;
971	}
972	}
973	fn fmtWipMir(func: *Func, inst: Mir.Inst.Index) std.fmt.Alt(FormatWipMirData, formatWipMir) {
974	return .{ .data = .{ .func = func, .inst = inst } };
975	}
976
977	const FormatNavData = struct {
978	ip: *const InternPool,
979	nav_index: InternPool.Nav.Index,
980	};
981	fn formatNav(data: FormatNavData, writer: *std.Io.Writer) std.Io.Writer.Error!void {
982	try writer.print("{f}", .{data.ip.getNav(data.nav_index).fqn.fmt(data.ip)});
983	}
984	fn fmtNav(nav_index: InternPool.Nav.Index, ip: *const InternPool) std.fmt.Alt(FormatNavData, formatNav) {
985	return .{ .data = .{
986	.ip = ip,
987	.nav_index = nav_index,
988	} };
989	}
990
991	const FormatAirData = struct {
992	func: *Func,
993	inst: Air.Inst.Index,
994	};
995	fn formatAir(data: FormatAirData, writer: *std.Io.Writer) std.Io.Writer.Error!void {
996	// Not acceptable implementation because it ignores `writer`:
997	//data.func.air.dumpInst(data.inst, data.func.pt, data.func.liveness);
998	_ = data;
999	_ = writer;
1000	@panic("unimplemented");
1001	}
1002	fn fmtAir(func: *Func, inst: Air.Inst.Index) std.fmt.Alt(FormatAirData, formatAir) {
1003	return .{ .data = .{ .func = func, .inst = inst } };
1004	}
1005
1006	const FormatTrackingData = struct {
1007	func: *Func,
1008	};
1009	fn formatTracking(data: FormatTrackingData, writer: *std.Io.Writer) std.Io.Writer.Error!void {
1010	var it = data.func.inst_tracking.iterator();
1011	while (it.next()) |entry| try writer.print("\n%{d} = {f}", .{ entry.key_ptr.*, entry.value_ptr.* });
1012	}
1013	fn fmtTracking(func: *Func) std.fmt.Alt(FormatTrackingData, formatTracking) {
1014	return .{ .data = .{ .func = func } };
1015	}
1016
1017	fn addInst(func: *Func, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {
1018	const gpa = func.gpa;
1019	try func.mir_instructions.ensureUnusedCapacity(gpa, 1);
1020	const result_index: Mir.Inst.Index = @intCast(func.mir_instructions.len);
1021	func.mir_instructions.appendAssumeCapacity(inst);
1022	if (switch (inst.tag) {
1023	else => true,
1024	.pseudo_dbg_prologue_end,
1025	.pseudo_dbg_line_column,
1026	.pseudo_dbg_epilogue_begin,
1027	.pseudo_dead,
1028	=> false,
1029	}) wip_mir_log.debug("{f}", .{func.fmtWipMir(result_index)});
1030	return result_index;
1031	}
1032
1033	fn addPseudo(func: *Func, mnem: Mnemonic) error{OutOfMemory}!Mir.Inst.Index {
1034	return func.addInst(.{
1035	.tag = mnem,
1036	.data = .none,
1037	});
1038	}
1039
1040	/// Returns a temporary register that contains the value of the `reg` csr.
1041	///
1042	/// Caller's duty to lock the return register is needed.
1043	fn getCsr(func: *Func, csr: CSR) !Register {
1044	assert(func.hasFeature(.zicsr));
1045	const dst_reg = try func.register_manager.allocReg(null, func.regTempClassForType(Type.u64));
1046	_ = try func.addInst(.{
1047	.tag = .csrrs,
1048	.data = .{ .csr = .{
1049	.csr = csr,
1050	.rd = dst_reg,
1051	.rs1 = .x0,
1052	} },
1053	});
1054	return dst_reg;
1055	}
1056
1057	fn setVl(func: *Func, dst_reg: Register, avl: u64, options: bits.VType) !void {
1058	if (func.avl == avl) if (func.vtype) |vtype| {
1059	// it's already set, we don't need to do anything
1060	if (@as(u8, @bitCast(vtype)) == @as(u8, @bitCast(options))) return;
1061	};
1062
1063	func.avl = avl;
1064	func.vtype = options;
1065
1066	if (avl == 0) {
1067	// the caller means to do "vsetvli zero, zero ..." which keeps the avl to whatever it was before
1068	const options_int: u12 = @as(u12, 0) | @as(u8, @bitCast(options));
1069	_ = try func.addInst(.{
1070	.tag = .vsetvli,
1071	.data = .{ .i_type = .{
1072	.rd = dst_reg,
1073	.rs1 = .zero,
1074	.imm12 = Immediate.u(options_int),
1075	} },
1076	});
1077	} else {
1078	// if the avl can fit into u5 we can use vsetivli otherwise use vsetvli
1079	if (avl <= std.math.maxInt(u5)) {
1080	const options_int: u12 = (~@as(u12, 0) << 10) | @as(u8, @bitCast(options));
1081	_ = try func.addInst(.{
1082	.tag = .vsetivli,
1083	.data = .{
1084	.i_type = .{
1085	.rd = dst_reg,
1086	.rs1 = @fromBackingInt(@intCast(avl)),
1087	.imm12 = Immediate.u(options_int),
1088	},
1089	},
1090	});
1091	} else {
1092	const options_int: u12 = @as(u12, 0) | @as(u8, @bitCast(options));
1093	const temp_reg = try func.copyToTmpRegister(Type.u64, .{ .immediate = avl });
1094	_ = try func.addInst(.{
1095	.tag = .vsetvli,
1096	.data = .{ .i_type = .{
1097	.rd = dst_reg,
1098	.rs1 = temp_reg,
1099	.imm12 = Immediate.u(options_int),
1100	} },
1101	});
1102	}
1103	}
1104	}
1105
1106	const required_features = [_]Target.riscv.Feature{
1107	.d,
1108	.m,
1109	.a,
1110	.zicsr,
1111	.v,
1112	.zbb,
1113	};
1114
1115	fn gen(func: *Func) !void {
1116	const pt = func.pt;
1117	const zcu = pt.zcu;
1118	const fn_info = zcu.typeToFunc(func.fn_type).?;
1119
1120	inline for (required_features) |feature| {
1121	if (!func.hasFeature(feature)) {
1122	return func.fail(
1123	"target missing required feature {s}",
1124	.{@tagName(feature)},
1125	);
1126	}
1127	}
1128
1129	if (fn_info.cc != .naked) {
1130	_ = try func.addPseudo(.pseudo_dbg_prologue_end);
1131
1132	const backpatch_stack_alloc = try func.addPseudo(.pseudo_dead);
1133	const backpatch_ra_spill = try func.addPseudo(.pseudo_dead);
1134	const backpatch_fp_spill = try func.addPseudo(.pseudo_dead);
1135	const backpatch_fp_add = try func.addPseudo(.pseudo_dead);
1136	const backpatch_spill_callee_preserved_regs = try func.addPseudo(.pseudo_dead);
1137
1138	switch (func.ret_mcv.long) {
1139	.none, .unreach => {},
1140	.indirect => {
1141	// The address where to store the return value for the caller is in a
1142	// register which the callee is free to clobber. Therefore, we purposely
1143	// spill it to stack immediately.
1144	const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(Type.u64, zcu));
1145	try func.genSetMem(
1146	.{ .frame = frame_index },
1147	0,
1148	Type.u64,
1149	func.ret_mcv.long.address().offset(-func.ret_mcv.short.indirect.off),
1150	);
1151	func.ret_mcv.long = .{ .load_frame = .{ .index = frame_index } };
1152	tracking_log.debug("spill {} to {}", .{ func.ret_mcv.long, frame_index });
1153	},
1154	else => unreachable,
1155	}
1156
1157	try func.genBody(func.air.getMainBody());
1158
1159	for (func.exitlude_jump_relocs.items) |jmp_reloc| {
1160	func.mir_instructions.items(.data)[jmp_reloc].j_type.inst =
1161	@intCast(func.mir_instructions.len);
1162	}
1163
1164	_ = try func.addPseudo(.pseudo_dbg_epilogue_begin);
1165
1166	const backpatch_restore_callee_preserved_regs = try func.addPseudo(.pseudo_dead);
1167	const backpatch_ra_restore = try func.addPseudo(.pseudo_dead);
1168	const backpatch_fp_restore = try func.addPseudo(.pseudo_dead);
1169	const backpatch_stack_alloc_restore = try func.addPseudo(.pseudo_dead);
1170
1171	// ret
1172	_ = try func.addInst(.{
1173	.tag = .jalr,
1174	.data = .{ .i_type = .{
1175	.rd = .zero,
1176	.rs1 = .ra,
1177	.imm12 = Immediate.s(0),
1178	} },
1179	});
1180
1181	const frame_layout = try func.computeFrameLayout();
1182	const need_save_reg = frame_layout.save_reg_list.count() > 0;
1183
1184	func.mir_instructions.set(backpatch_stack_alloc, .{
1185	.tag = .addi,
1186	.data = .{ .i_type = .{
1187	.rd = .sp,
1188	.rs1 = .sp,
1189	.imm12 = Immediate.s(-@as(i32, @intCast(frame_layout.stack_adjust))),
1190	} },
1191	});
1192	func.mir_instructions.set(backpatch_ra_spill, .{
1193	.tag = .pseudo_store_rm,
1194	.data = .{ .rm = .{
1195	.r = .ra,
1196	.m = .{
1197	.base = .{ .frame = .ret_addr },
1198	.mod = .{ .size = .dword, .unsigned = false },
1199	},
1200	} },
1201	});
1202	func.mir_instructions.set(backpatch_ra_restore, .{
1203	.tag = .pseudo_load_rm,
1204	.data = .{ .rm = .{
1205	.r = .ra,
1206	.m = .{
1207	.base = .{ .frame = .ret_addr },
1208	.mod = .{ .size = .dword, .unsigned = false },
1209	},
1210	} },
1211	});
1212	func.mir_instructions.set(backpatch_fp_spill, .{
1213	.tag = .pseudo_store_rm,
1214	.data = .{ .rm = .{
1215	.r = .s0,
1216	.m = .{
1217	.base = .{ .frame = .base_ptr },
1218	.mod = .{ .size = .dword, .unsigned = false },
1219	},
1220	} },
1221	});
1222	func.mir_instructions.set(backpatch_fp_restore, .{
1223	.tag = .pseudo_load_rm,
1224	.data = .{ .rm = .{
1225	.r = .s0,
1226	.m = .{
1227	.base = .{ .frame = .base_ptr },
1228	.mod = .{ .size = .dword, .unsigned = false },
1229	},
1230	} },
1231	});
1232	func.mir_instructions.set(backpatch_fp_add, .{
1233	.tag = .addi,
1234	.data = .{ .i_type = .{
1235	.rd = .s0,
1236	.rs1 = .sp,
1237	.imm12 = Immediate.s(@intCast(frame_layout.stack_adjust)),
1238	} },
1239	});
1240	func.mir_instructions.set(backpatch_stack_alloc_restore, .{
1241	.tag = .addi,
1242	.data = .{ .i_type = .{
1243	.rd = .sp,
1244	.rs1 = .sp,
1245	.imm12 = Immediate.s(@intCast(frame_layout.stack_adjust)),
1246	} },
1247	});
1248
1249	if (need_save_reg) {
1250	func.mir_instructions.set(backpatch_spill_callee_preserved_regs, .{
1251	.tag = .pseudo_spill_regs,
1252	.data = .{ .reg_list = frame_layout.save_reg_list },
1253	});
1254
1255	func.mir_instructions.set(backpatch_restore_callee_preserved_regs, .{
1256	.tag = .pseudo_restore_regs,
1257	.data = .{ .reg_list = frame_layout.save_reg_list },
1258	});
1259	}
1260	} else {
1261	_ = try func.addPseudo(.pseudo_dbg_prologue_end);
1262	try func.genBody(func.air.getMainBody());
1263	_ = try func.addPseudo(.pseudo_dbg_epilogue_begin);
1264	}
1265
1266	// Drop them off at the rbrace.
1267	_ = try func.addInst(.{
1268	.tag = .pseudo_dbg_line_column,
1269	.data = .{ .pseudo_dbg_line_column = .{
1270	.line = func.end_di_line,
1271	.column = func.end_di_column,
1272	} },
1273	});
1274	}
1275
1276	fn genLazy(func: *Func, lazy_sym: link.File.LazySymbol) InnerError!void {
1277	const pt = func.pt;
1278	const zcu = pt.zcu;
1279	const ip = &zcu.intern_pool;
1280	switch (Type.fromInterned(lazy_sym.ty).zigTypeTag(zcu)) {
1281	.@"enum" => {
1282	const enum_ty = Type.fromInterned(lazy_sym.ty);
1283	wip_mir_log.debug("{f}.@tagName:", .{enum_ty.fmt(pt)});
1284
1285	const param_regs = abi.Registers.Integer.function_arg_regs;
1286	const ret_reg = param_regs[0];
1287	const enum_mcv: MCValue = .{ .register = param_regs[1] };
1288
1289	const exitlude_jump_relocs = try func.gpa.alloc(Mir.Inst.Index, enum_ty.enumFieldCount(zcu));
1290	defer func.gpa.free(exitlude_jump_relocs);
1291
1292	const data_reg, const data_lock = try func.allocReg(.int);
1293	defer func.register_manager.unlockReg(data_lock);
1294
1295	const elf_file = func.bin_file.cast(.elf).?;
1296	const zo = elf_file.zigObjectPtr().?;
1297	const sym_index = zo.getOrCreateMetadataForLazySymbol(elf_file, pt, .{
1298	.kind = .const_data,
1299	.ty = enum_ty.toIntern(),
1300	}) catch |err|
1301	return func.fail("{s} creating lazy symbol", .{@errorName(err)});
1302
1303	try func.genSetReg(Type.u64, data_reg, .{ .lea_symbol = .{ .sym = @fromBackingInt(@intCast(sym_index)) } });
1304
1305	const cmp_reg, const cmp_lock = try func.allocReg(.int);
1306	defer func.register_manager.unlockReg(cmp_lock);
1307
1308	var data_off: i32 = 0;
1309	const tag_names = enum_ty.enumFields(zcu);
1310	for (exitlude_jump_relocs, 0..) |*exitlude_jump_reloc, tag_index| {
1311	const tag_name_len = tag_names.get(ip)[tag_index].length(ip);
1312	const tag_val = try pt.enumValueFieldIndex(enum_ty, @intCast(tag_index));
1313	const tag_mcv = try func.genTypedValue(tag_val);
1314
1315	_ = try func.genBinOp(
1316	.cmp_neq,
1317	enum_mcv,
1318	enum_ty,
1319	tag_mcv,
1320	enum_ty,
1321	cmp_reg,
1322	);
1323	const skip_reloc = try func.condBr(Type.bool, .{ .register = cmp_reg });
1324
1325	try func.genSetMem(
1326	.{ .reg = ret_reg },
1327	0,
1328	Type.u64,
1329	.{ .register_offset = .{ .reg = data_reg, .off = data_off } },
1330	);
1331
1332	try func.genSetMem(
1333	.{ .reg = ret_reg },
1334	8,
1335	Type.u64,
1336	.{ .immediate = tag_name_len },
1337	);
1338
1339	exitlude_jump_reloc.* = try func.addInst(.{
1340	.tag = .pseudo_j,
1341	.data = .{ .j_type = .{
1342	.rd = .zero,
1343	.inst = undefined,
1344	} },
1345	});
1346	func.performReloc(skip_reloc);
1347
1348	data_off += @intCast(tag_name_len + 1);
1349	}
1350
1351	try func.airTrap();
1352
1353	for (exitlude_jump_relocs) |reloc| func.performReloc(reloc);
1354
1355	_ = try func.addInst(.{
1356	.tag = .jalr,
1357	.data = .{ .i_type = .{
1358	.rd = .zero,
1359	.rs1 = .ra,
1360	.imm12 = Immediate.s(0),
1361	} },
1362	});
1363	},
1364	else => return func.fail(
1365	"TODO implement {s} for {f}",
1366	.{ @tagName(lazy_sym.kind), Type.fromInterned(lazy_sym.ty).fmt(pt) },
1367	),
1368	}
1369	}
1370
1371	fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
1372	const pt = func.pt;
1373	const zcu = pt.zcu;
1374	const ip = &zcu.intern_pool;
1375	const air_tags = func.air.instructions.items(.tag);
1376
1377	for (body) |inst| {
1378	if (func.liveness.isUnused(inst) and !func.air.mustLower(inst, ip)) continue;
1379	wip_mir_log.debug("{f}", .{func.fmtAir(inst)});
1380	verbose_tracking_log.debug("{f}", .{func.fmtTracking()});
1381
1382	const old_air_bookkeeping = func.air_bookkeeping;
1383	try func.ensureProcessDeathCapacity(Air.Liveness.bpi);
1384
1385	func.reused_operands = @TypeOf(func.reused_operands).empty;
1386	try func.inst_tracking.ensureUnusedCapacity(func.gpa, 1);
1387	const tag = air_tags[@backingInt(inst)];
1388	switch (tag) {
1389	// zig fmt: off
1390
1391	// No "scalarize" legalizations are enabled, so these instructions never appear.
1392	.legalize_vec_elem_val => unreachable,
1393	.legalize_vec_store_elem => unreachable,
1394	// No soft float legalizations are enabled.
1395	.legalize_compiler_rt_call => unreachable,
1396
1397	.add,
1398	.add_wrap,
1399	.sub,
1400	.sub_wrap,
1401
1402	.add_sat,
1403
1404	.mul,
1405	.mul_wrap,
1406	.div_trunc,
1407	.div_exact,
1408	.rem,
1409
1410	.shl, .shl_exact,
1411	.shr, .shr_exact,
1412
1413	.bit_and,
1414	.bit_or,
1415
1416	.xor,
1417
1418	.min,
1419	.max,
1420	=> try func.airBinOp(inst, tag),
1421
1422
1423	.ptr_add,
1424	.ptr_sub => try func.airPtrArithmetic(inst, tag),
1425
1426	.mod,
1427	.div_float,
1428	.div_floor,
1429	.div_ceil,
1430	=> return func.fail("TODO: {s}", .{@tagName(tag)}),
1431
1432	.sqrt,
1433	.sin,
1434	.cos,
1435	.tan,
1436	.exp,
1437	.exp2,
1438	.log,
1439	.log2,
1440	.log10,
1441	.floor,
1442	.ceil,
1443	.round,
1444	.trunc_float,
1445	.neg,
1446	=> try func.airUnaryMath(inst, tag),
1447
1448	.add_with_overflow => try func.airAddWithOverflow(inst),
1449	.sub_with_overflow => try func.airSubWithOverflow(inst),
1450	.mul_with_overflow => try func.airMulWithOverflow(inst),
1451	.shl_with_overflow => try func.airShlWithOverflow(inst),
1452
1453
1454	.sub_sat => try func.airSubSat(inst),
1455	.mul_sat => try func.airMulSat(inst),
1456	.shl_sat => try func.airShlSat(inst),
1457
1458	.add_safe,
1459	.sub_safe,
1460	.mul_safe,
1461	.bit_cast_safe,
1462	.int_cast_safe,
1463	.int_from_float_safe,
1464	.int_from_float_optimized_safe,
1465	=> return func.fail("TODO implement safety_checked_instructions", .{}),
1466
1467	.cmp_lt,
1468	.cmp_lte,
1469	.cmp_eq,
1470	.cmp_gte,
1471	.cmp_gt,
1472	.cmp_neq,
1473	=> try func.airCmp(inst, tag),
1474
1475	.cmp_vector => try func.airCmpVector(inst),
1476	.cmp_lte_errors_len => try func.airCmpLteErrorsLen(inst),
1477
1478	.slice => try func.airSlice(inst),
1479	.array_to_slice => try func.airArrayToSlice(inst),
1480	.array_to_vector => unreachable, // legalize .expand_array_to_vector
1481
1482	.slice_ptr => try func.airSlicePtr(inst),
1483	.slice_len => try func.airSliceLen(inst),
1484
1485	.alloc => try func.airAlloc(inst),
1486	.ret_ptr => try func.airRetPtr(inst),
1487	.arg => try func.airArg(inst),
1488	.assembly => try func.airAsm(inst),
1489	.bit_cast => try func.airBitCast(inst),
1490	.ptr_cast => try func.airBitCast(inst),
1491	.ptr_from_int => try func.airBitCast(inst),
1492	.int_from_ptr => try func.airBitCast(inst),
1493	.error_cast => try func.airBitCast(inst),
1494	.error_from_int => try func.airBitCast(inst),
1495	.int_from_error => try func.airBitCast(inst),
1496	.union_from_enum => try func.airBitCast(inst),
1497	.block => try func.airBlock(inst),
1498	.br => try func.airBr(inst),
1499	.repeat => try func.airRepeat(inst),
1500	.switch_dispatch => try func.airSwitchDispatch(inst),
1501	.trap => try func.airTrap(),
1502	.breakpoint => try func.airBreakpoint(),
1503	.ret_addr => try func.airRetAddr(inst),
1504	.frame_addr => try func.airFrameAddress(inst),
1505	.cond_br => try func.airCondBr(inst),
1506	.dbg_stmt => try func.airDbgStmt(inst),
1507	.dbg_empty_stmt => func.finishAirBookkeeping(),
1508	.fptrunc => try func.airFptrunc(inst),
1509	.fpext => try func.airFpext(inst),
1510	.int_cast => try func.airIntCast(inst),
1511	.trunc => try func.airTrunc(inst),
1512	.is_non_null => try func.airIsNonNull(inst),
1513	.is_non_null_ptr => try func.airIsNonNullPtr(inst),
1514	.is_null => try func.airIsNull(inst),
1515	.is_null_ptr => try func.airIsNullPtr(inst),
1516	.is_non_err => try func.airIsNonErr(inst),
1517	.is_non_err_ptr => try func.airIsNonErrPtr(inst),
1518	.is_err => try func.airIsErr(inst),
1519	.is_err_ptr => try func.airIsErrPtr(inst),
1520	.load => try func.airLoad(inst),
1521	.loop => try func.airLoop(inst),
1522	.not => try func.airNot(inst),
1523	.ret => try func.airRet(inst, false),
1524	.ret_safe => try func.airRet(inst, true),
1525	.ret_load => try func.airRetLoad(inst),
1526	.store => try func.airStore(inst, false),
1527	.store_safe => try func.airStore(inst, true),
1528	.struct_field_ptr=> try func.airStructFieldPtr(inst),
1529	.agg_field_val => try func.airAggFieldVal(inst),
1530	.float_from_int => try func.airFloatFromInt(inst),
1531	.int_from_float => try func.airIntFromFloat(inst),
1532	.cmpxchg_strong => try func.airCmpxchg(inst, .strong),
1533	.cmpxchg_weak => try func.airCmpxchg(inst, .weak),
1534	.atomic_rmw => try func.airAtomicRmw(inst),
1535	.atomic_load => try func.airAtomicLoad(inst),
1536	.memcpy => try func.airMemcpy(inst),
1537	.memmove => try func.airMemmove(inst),
1538	.memset => try func.airMemset(inst, false),
1539	.memset_safe => try func.airMemset(inst, true),
1540	.set_union_tag => try func.airSetUnionTag(inst),
1541	.get_union_tag => try func.airGetUnionTag(inst),
1542	.clz => try func.airClz(inst),
1543	.ctz => try func.airCtz(inst),
1544	.popcount => try func.airPopcount(inst),
1545	.abs => try func.airAbs(inst),
1546	.byte_swap => try func.airByteSwap(inst),
1547	.bit_reverse => try func.airBitReverse(inst),
1548	.tag_name => try func.airTagName(inst),
1549	.error_name => try func.airErrorName(inst),
1550	.splat => try func.airSplat(inst),
1551	.select => try func.airSelect(inst),
1552	.shuffle_one => try func.airShuffleOne(inst),
1553	.shuffle_two => try func.airShuffleTwo(inst),
1554	.reduce => try func.airReduce(inst),
1555	.aggregate_init => try func.airAggregateInit(inst),
1556	.union_init => try func.airUnionInit(inst),
1557	.prefetch => try func.airPrefetch(inst),
1558	.mul_add => try func.airMulAdd(inst),
1559	.addrspace_cast => return func.fail("TODO: addrspace_cast", .{}),
1560
1561	.@"try" => try func.airTry(inst),
1562	.try_cold => try func.airTry(inst),
1563	.try_ptr => return func.fail("TODO: try_ptr", .{}),
1564	.try_ptr_cold => return func.fail("TODO: try_ptr_cold", .{}),
1565
1566	.dbg_var_ptr,
1567	.dbg_var_val,
1568	.dbg_arg_inline,
1569	=> try func.airDbgVar(inst),
1570
1571	.dbg_inline_block => try func.airDbgInlineBlock(inst),
1572
1573	.call => try func.airCall(inst, .auto),
1574	.call_always_tail => try func.airCall(inst, .always_tail),
1575	.call_never_tail => try func.airCall(inst, .never_tail),
1576	.call_never_inline => try func.airCall(inst, .never_inline),
1577
1578	.atomic_store_unordered => try func.airAtomicStore(inst, .unordered),
1579	.atomic_store_monotonic => try func.airAtomicStore(inst, .monotonic),
1580	.atomic_store_release => try func.airAtomicStore(inst, .release),
1581	.atomic_store_seq_cst => try func.airAtomicStore(inst, .seq_cst),
1582	.struct_field_ptr_index_0 => try func.airStructFieldPtrIndex(inst, 0),
1583	.struct_field_ptr_index_1 => try func.airStructFieldPtrIndex(inst, 1),
1584	.struct_field_ptr_index_2 => try func.airStructFieldPtrIndex(inst, 2),
1585	.struct_field_ptr_index_3 => try func.airStructFieldPtrIndex(inst, 3),
1586
1587	.field_parent_ptr => try func.airFieldParentPtr(inst),
1588
1589	.switch_br => try func.airSwitchBr(inst),
1590	.loop_switch_br => try func.airLoopSwitchBr(inst),
1591
1592	.ptr_slice_len_ptr => try func.airPtrSliceLenPtr(inst),
1593	.ptr_slice_ptr_ptr => try func.airPtrSlicePtrPtr(inst),
1594
1595	.array_elem_val => try func.airArrayElemVal(inst),
1596
1597	.slice_elem_val => try func.airSliceElemVal(inst),
1598	.slice_elem_ptr => try func.airSliceElemPtr(inst),
1599
1600	.ptr_elem_val => try func.airPtrElemVal(inst),
1601	.ptr_elem_ptr => try func.airPtrElemPtr(inst),
1602
1603	.inferred_alloc, .inferred_alloc_comptime => unreachable,
1604	.unreach => func.finishAirBookkeeping(),
1605
1606	.optional_payload => try func.airOptionalPayload(inst),
1607	.optional_payload_ptr => try func.airOptionalPayloadPtr(inst),
1608	.optional_payload_ptr_set => try func.airOptionalPayloadPtrSet(inst),
1609	.unwrap_errunion_err => try func.airUnwrapErrErr(inst),
1610	.unwrap_errunion_payload => try func.airUnwrapErrPayload(inst),
1611	.unwrap_errunion_err_ptr => try func.airUnwrapErrErrPtr(inst),
1612	.unwrap_errunion_payload_ptr=> try func.airUnwrapErrPayloadPtr(inst),
1613	.errunion_payload_ptr_set => try func.airErrUnionPayloadPtrSet(inst),
1614	.err_return_trace => try func.airErrReturnTrace(inst),
1615	.set_err_return_trace => try func.airSetErrReturnTrace(inst),
1616	.save_err_return_trace_index=> try func.airSaveErrReturnTraceIndex(inst),
1617
1618	.wrap_optional => try func.airWrapOptional(inst),
1619	.wrap_errunion_payload => try func.airWrapErrUnionPayload(inst),
1620	.wrap_errunion_err => try func.airWrapErrUnionErr(inst),
1621
1622	.runtime_nav_ptr => try func.airRuntimeNavPtr(inst),
1623
1624	.add_optimized,
1625	.sub_optimized,
1626	.mul_optimized,
1627	.div_float_optimized,
1628	.div_trunc_optimized,
1629	.div_floor_optimized,
1630	.div_exact_optimized,
1631	.div_ceil_optimized,
1632	.rem_optimized,
1633	.mod_optimized,
1634	.neg_optimized,
1635	.cmp_lt_optimized,
1636	.cmp_lte_optimized,
1637	.cmp_eq_optimized,
1638	.cmp_gte_optimized,
1639	.cmp_gt_optimized,
1640	.cmp_neq_optimized,
1641	.cmp_vector_optimized,
1642	.reduce_optimized,
1643	.int_from_float_optimized,
1644	=> return func.fail("TODO implement optimized float mode", .{}),
1645
1646	.is_named_enum_value => return func.fail("TODO implement is_named_enum_value", .{}),
1647	.error_set_has_value => return func.fail("TODO implement error_set_has_value", .{}),
1648
1649	.c_va_arg => return func.fail("TODO implement c_va_arg", .{}),
1650	.c_va_copy => return func.fail("TODO implement c_va_copy", .{}),
1651	.c_va_end => return func.fail("TODO implement c_va_end", .{}),
1652	.c_va_start => return func.fail("TODO implement c_va_start", .{}),
1653
1654	.wasm_memory_size => unreachable,
1655	.wasm_memory_grow => unreachable,
1656
1657	.work_item_id => unreachable,
1658	.work_group_size => unreachable,
1659	.work_group_id => unreachable,
1660	.spirv_runtime_array_len => unreachable,
1661	// zig fmt: on
1662	}
1663
1664	assert(!func.register_manager.lockedRegsExist());
1665
1666	if (std.debug.runtime_safety) {
1667	if (func.air_bookkeeping < old_air_bookkeeping + 1) {
1668	std.debug.panic("in codegen.zig, handling of AIR instruction %{d} ('{}') did not do proper bookkeeping. Look for a missing call to finishAir.", .{ inst, air_tags[@backingInt(inst)] });
1669	}
1670
1671	{ // check consistency of tracked registers
1672	var it = func.register_manager.free_registers.iterator(.{ .kind = .unset });
1673	while (it.next()) |index| {
1674	const tracked_inst = func.register_manager.registers[index];
1675	tracking_log.debug("tracked inst: {f}", .{tracked_inst});
1676	const tracking = func.getResolvedInstValue(tracked_inst);
1677	for (tracking.getRegs()) |reg| {
1678	if (RegisterManager.indexOfRegIntoTracked(reg).? == index) break;
1679	} else return std.debug.panic(
1680	\\%{f} takes up these regs: {any}, however this regs {any}, don't use it
1681	, .{ tracked_inst, tracking.getRegs(), RegisterManager.regAtTrackedIndex(@intCast(index)) });
1682	}
1683	}
1684	}
1685	}
1686	verbose_tracking_log.debug("{f}", .{func.fmtTracking()});
1687	}
1688
1689	fn getValue(func: *Func, value: MCValue, inst: ?Air.Inst.Index) !void {
1690	for (value.getRegs()) |reg| try func.register_manager.getReg(reg, inst);
1691	}
1692
1693	fn getValueIfFree(func: *Func, value: MCValue, inst: ?Air.Inst.Index) void {
1694	for (value.getRegs()) |reg| if (func.register_manager.isRegFree(reg))
1695	func.register_manager.getRegAssumeFree(reg, inst);
1696	}
1697
1698	fn freeValue(func: *Func, value: MCValue) !void {
1699	switch (value) {
1700	.register => |reg| func.register_manager.freeReg(reg),
1701	.register_pair => |regs| for (regs) |reg| func.register_manager.freeReg(reg),
1702	.register_offset => |reg_off| func.register_manager.freeReg(reg_off.reg),
1703	else => {}, // TODO process stack allocation death
1704	}
1705	}
1706
1707	fn feed(func: *Func, bt: *Air.Liveness.BigTomb, operand: Air.Inst.Ref) !void {
1708	if (bt.feed()) if (operand.toIndex()) |inst| {
1709	log.debug("feed inst: %{f}", .{inst});
1710	try func.processDeath(inst);
1711	};
1712	}
1713
1714	/// Asserts there is already capacity to insert into top branch inst_table.
1715	fn processDeath(func: *Func, inst: Air.Inst.Index) !void {
1716	try func.inst_tracking.getPtr(inst).?.die(func, inst);
1717	}
1718
1719	/// Called when there are no operands, and the instruction is always unreferenced.
1720	fn finishAirBookkeeping(func: *Func) void {
1721	if (std.debug.runtime_safety) {
1722	func.air_bookkeeping += 1;
1723	}
1724	}
1725
1726	fn finishAirResult(func: *Func, inst: Air.Inst.Index, result: MCValue) void {
1727	if (func.liveness.isUnused(inst)) switch (result) {
1728	.none, .dead, .unreach => {},
1729	// Why didn't the result die?
1730	.register => |r| if (r != .zero) unreachable,
1731	else => unreachable,
1732	} else {
1733	switch (result) {
1734	.register => |r| if (r == .zero) unreachable, // Why did we discard a used result?
1735	else => {},
1736	}
1737
1738	tracking_log.debug("%{d} => {} (birth)", .{ inst, result });
1739	func.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(result));
1740	// In some cases, an operand may be reused as the result.
1741	// If that operand died and was a register, it was freed by
1742	// processDeath, so we have to "re-allocate" the register.
1743	func.getValueIfFree(result, inst);
1744	}
1745	func.finishAirBookkeeping();
1746	}
1747
1748	fn finishAir(
1749	func: *Func,
1750	inst: Air.Inst.Index,
1751	result: MCValue,
1752	operands: [Air.Liveness.bpi - 1]Air.Inst.Ref,
1753	) !void {
1754	const tomb_bits = func.liveness.getTombBits(inst);
1755	for (0.., operands) |op_index, op| {
1756	if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
1757	if (func.reused_operands.isSet(op_index)) continue;
1758	try func.processDeath(op.toIndexAllowNone() orelse continue);
1759	}
1760	func.finishAirResult(inst, result);
1761	}
1762
1763	const FrameLayout = struct {
1764	stack_adjust: i12,
1765	save_reg_list: Mir.RegisterList,
1766	};
1767
1768	fn setFrameLoc(
1769	func: *Func,
1770	frame_index: FrameIndex,
1771	base: Register,
1772	offset: *i32,
1773	comptime aligned: bool,
1774	) void {
1775	const frame_i = @backingInt(frame_index);
1776	if (aligned) {
1777	const alignment: InternPool.Alignment = func.frame_allocs.items(.abi_align)[frame_i];
1778	offset.* = math.sign(offset.*) * @as(i32, @intCast(alignment.backward(@intCast(@abs(offset.*)))));
1779	}
1780	func.frame_locs.set(frame_i, .{ .base = base, .disp = offset.* });
1781	offset.* += func.frame_allocs.items(.abi_size)[frame_i];
1782	}
1783
1784	fn computeFrameLayout(func: *Func) !FrameLayout {
1785	const frame_allocs_len = func.frame_allocs.len;
1786	try func.frame_locs.resize(func.gpa, frame_allocs_len);
1787	const stack_frame_order = try func.gpa.alloc(FrameIndex, frame_allocs_len - FrameIndex.named_count);
1788	defer func.gpa.free(stack_frame_order);
1789
1790	const frame_size = func.frame_allocs.items(.abi_size);
1791	const frame_align = func.frame_allocs.items(.abi_align);
1792
1793	for (stack_frame_order, FrameIndex.named_count..) |*frame_order, frame_index|
1794	frame_order.* = @fromBackingInt(@intCast(frame_index));
1795
1796	{
1797	const SortContext = struct {
1798	frame_align: @TypeOf(frame_align),
1799	pub fn lessThan(context: @This(), lhs: FrameIndex, rhs: FrameIndex) bool {
1800	return context.frame_align[@backingInt(lhs)].compare(.gt, context.frame_align[@backingInt(rhs)]);
1801	}
1802	};
1803	const sort_context = SortContext{ .frame_align = frame_align };
1804	mem.sort(FrameIndex, stack_frame_order, sort_context, SortContext.lessThan);
1805	}
1806
1807	var save_reg_list = Mir.RegisterList{};
1808	for (abi.Registers.all_preserved) |reg| {
1809	if (func.register_manager.isRegAllocated(reg)) {
1810	save_reg_list.push(&abi.Registers.all_preserved, reg);
1811	}
1812	}
1813
1814	const total_alloc_size: i32 = blk: {
1815	var i: i32 = 0;
1816	for (stack_frame_order) |frame_index| {
1817	i += frame_size[@backingInt(frame_index)];
1818	}
1819	break :blk i;
1820	};
1821
1822	const saved_reg_size = save_reg_list.size();
1823	frame_size[@backingInt(FrameIndex.spill_frame)] = @intCast(saved_reg_size);
1824
1825	// The total frame size is calculated by the amount of s registers you need to save * 8, as each
1826	// register is 8 bytes, the total allocation sizes, and 16 more register for the spilled ra and s0
1827	// register. Finally we align the frame size to the alignment of the base pointer.
1828	const args_frame_size = frame_size[@backingInt(FrameIndex.args_frame)];
1829	const spill_frame_size = frame_size[@backingInt(FrameIndex.spill_frame)];
1830	const call_frame_size = frame_size[@backingInt(FrameIndex.call_frame)];
1831
1832	// TODO: this 64 should be a 16, but we were clobbering the top and bottom of the frame.
1833	// maybe everything can go from the bottom?
1834	const acc_frame_size: i32 = std.mem.alignForward(
1835	i32,
1836	total_alloc_size + 64 + args_frame_size + spill_frame_size + call_frame_size,
1837	@intCast(frame_align[@backingInt(FrameIndex.base_ptr)].toByteUnits().?),
1838	);
1839	log.debug("frame size: {d}", .{acc_frame_size});
1840
1841	// store the ra at total_size - 8, so it's the very first thing in the stack
1842	// relative to the fp
1843	func.frame_locs.set(
1844	@backingInt(FrameIndex.ret_addr),
1845	.{ .base = .sp, .disp = acc_frame_size - 8 },
1846	);
1847	func.frame_locs.set(
1848	@backingInt(FrameIndex.base_ptr),
1849	.{ .base = .sp, .disp = acc_frame_size - 16 },
1850	);
1851
1852	// now we grow the stack frame from the bottom of total frame in order to
1853	// not need to know the size of the first allocation. Stack offsets point at the "bottom"
1854	// of variables.
1855	var s0_offset: i32 = -acc_frame_size;
1856	func.setFrameLoc(.stack_frame, .s0, &s0_offset, true);
1857	for (stack_frame_order) |frame_index| func.setFrameLoc(frame_index, .s0, &s0_offset, true);
1858	func.setFrameLoc(.args_frame, .s0, &s0_offset, true);
1859	func.setFrameLoc(.call_frame, .s0, &s0_offset, true);
1860	func.setFrameLoc(.spill_frame, .s0, &s0_offset, true);
1861
1862	return .{
1863	.stack_adjust = @intCast(acc_frame_size),
1864	.save_reg_list = save_reg_list,
1865	};
1866	}
1867
1868	fn ensureProcessDeathCapacity(func: *Func, additional_count: usize) !void {
1869	const table = &func.branch_stack.items[func.branch_stack.items.len - 1].inst_table;
1870	try table.ensureUnusedCapacity(func.gpa, additional_count);
1871	}
1872
1873	fn memSize(func: *Func, ty: Type) Memory.Size {
1874	const pt = func.pt;
1875	const zcu = pt.zcu;
1876	return switch (ty.zigTypeTag(zcu)) {
1877	.float => Memory.Size.fromBitSize(ty.floatBits(func.target)),
1878	else => Memory.Size.fromByteSize(ty.abiSize(zcu)),
1879	};
1880	}
1881
1882	fn splitType(func: *Func, ty: Type) ![2]Type {
1883	const zcu = func.pt.zcu;
1884	const classes = mem.sliceTo(&abi.classifySystem(ty, zcu), .none);
1885	var parts: [2]Type = undefined;
1886	if (classes.len == 2) for (&parts, classes, 0..) |*part, class, part_i| {
1887	part.* = switch (class) {
1888	.integer => switch (part_i) {
1889	0 => Type.u64,
1890	1 => part: {
1891	const elem_size = ty.abiAlignment(zcu).minStrict(.@"8").toByteUnits().?;
1892	const elem_ty = try func.pt.intType(.unsigned, @intCast(elem_size * 8));
1893	break :part switch (@divExact(ty.abiSize(zcu) - 8, elem_size)) {
1894	1 => elem_ty,
1895	else => |len| try func.pt.arrayType(.{ .len = len, .child = elem_ty.toIntern() }),
1896	};
1897	},
1898	else => unreachable,
1899	},
1900	else => return func.fail("TODO: splitType class {}", .{class}),
1901	};
1902	} else if (parts[0].abiSize(zcu) + parts[1].abiSize(zcu) == ty.abiSize(zcu)) return parts;
1903	return func.fail("TODO implement splitType for {f}", .{ty.fmt(func.pt)});
1904	}
1905
1906	/// Truncates the value in the register in place.
1907	/// Clobbers any remaining bits.
1908	fn truncateRegister(func: *Func, ty: Type, reg: Register) !void {
1909	const pt = func.pt;
1910	const zcu = pt.zcu;
1911	const int_info = if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else std.lang.Type.Int{
1912	.signedness = .unsigned,
1913	.bits = @intCast(ty.bitSize(zcu)),
1914	};
1915	assert(reg.class() == .int);
1916
1917	const shift = math.cast(u6, 64 - int_info.bits % 64) orelse return;
1918	switch (int_info.signedness) {
1919	.signed => {
1920	_ = try func.addInst(.{
1921	.tag = .slli,
1922
1923	.data = .{
1924	.i_type = .{
1925	.rd = reg,
1926	.rs1 = reg,
1927	.imm12 = Immediate.u(shift),
1928	},
1929	},
1930	});
1931	_ = try func.addInst(.{
1932	.tag = .srai,
1933
1934	.data = .{
1935	.i_type = .{
1936	.rd = reg,
1937	.rs1 = reg,
1938	.imm12 = Immediate.u(shift),
1939	},
1940	},
1941	});
1942	},
1943	.unsigned => {
1944	const mask = ~@as(u64, 0) >> shift;
1945	if (mask < 256) {
1946	_ = try func.addInst(.{
1947	.tag = .andi,
1948
1949	.data = .{
1950	.i_type = .{
1951	.rd = reg,
1952	.rs1 = reg,
1953	.imm12 = Immediate.u(@intCast(mask)),
1954	},
1955	},
1956	});
1957	} else {
1958	_ = try func.addInst(.{
1959	.tag = .slli,
1960
1961	.data = .{
1962	.i_type = .{
1963	.rd = reg,
1964	.rs1 = reg,
1965	.imm12 = Immediate.u(shift),
1966	},
1967	},
1968	});
1969	_ = try func.addInst(.{
1970	.tag = .srli,
1971
1972	.data = .{
1973	.i_type = .{
1974	.rd = reg,
1975	.rs1 = reg,
1976	.imm12 = Immediate.u(shift),
1977	},
1978	},
1979	});
1980	}
1981	},
1982	}
1983	}
1984
1985	fn allocFrameIndex(func: *Func, alloc: FrameAlloc) !FrameIndex {
1986	const frame_allocs_slice = func.frame_allocs.slice();
1987	const frame_size = frame_allocs_slice.items(.abi_size);
1988	const frame_align = frame_allocs_slice.items(.abi_align);
1989
1990	const stack_frame_align = &frame_align[@backingInt(FrameIndex.stack_frame)];
1991	stack_frame_align.* = stack_frame_align.max(alloc.abi_align);
1992
1993	for (func.free_frame_indices.keys(), 0..) |frame_index, free_i| {
1994	const abi_size = frame_size[@backingInt(frame_index)];
1995	if (abi_size != alloc.abi_size) continue;
1996	const abi_align = &frame_align[@backingInt(frame_index)];
1997	abi_align.* = abi_align.max(alloc.abi_align);
1998
1999	_ = func.free_frame_indices.swapRemoveAt(free_i);
2000	return frame_index;
2001	}
2002	const frame_index: FrameIndex = @fromBackingInt(@intCast(func.frame_allocs.len));
2003	try func.frame_allocs.append(func.gpa, alloc);
2004	log.debug("allocated frame {}", .{frame_index});
2005	return frame_index;
2006	}
2007
2008	/// Use a pointer instruction as the basis for allocating stack memory.
2009	fn allocMemPtr(func: *Func, inst: Air.Inst.Index) !FrameIndex {
2010	const pt = func.pt;
2011	const zcu = pt.zcu;
2012	const ptr_ty = func.typeOfIndex(inst);
2013	const val_ty = ptr_ty.childType(zcu);
2014	return func.allocFrameIndex(FrameAlloc.init(.{
2015	.size = math.cast(u32, val_ty.abiSize(zcu)) orelse {
2016	return func.fail("type '{f}' too big to fit into stack frame", .{val_ty.fmt(pt)});
2017	},
2018	.alignment = ptr_ty.ptrAlignment(zcu).max(.@"1"),
2019	}));
2020	}
2021
2022	fn typeRegClass(func: *Func, ty: Type) abi.RegisterClass {
2023	const pt = func.pt;
2024	const zcu = pt.zcu;
2025	return switch (ty.zigTypeTag(zcu)) {
2026	.float => .float,
2027	.vector => .vector,
2028	else => .int,
2029	};
2030	}
2031
2032	fn regGeneralClassForType(func: *Func, ty: Type) RegisterManager.RegisterBitSet {
2033	return switch (ty.zigTypeTag(func.pt.zcu)) {
2034	.float => abi.Registers.Float.general_purpose,
2035	.vector => abi.Registers.Vector.general_purpose,
2036	else => abi.Registers.Integer.general_purpose,
2037	};
2038	}
2039
2040	fn regTempClassForType(func: *Func, ty: Type) RegisterManager.RegisterBitSet {
2041	return switch (ty.zigTypeTag(func.pt.zcu)) {
2042	.float => abi.Registers.Float.temporary,
2043	.vector => abi.Registers.Vector.general_purpose, // there are no temporary vector registers
2044	else => abi.Registers.Integer.temporary,
2045	};
2046	}
2047
2048	fn allocRegOrMem(func: *Func, elem_ty: Type, inst: ?Air.Inst.Index, reg_ok: bool) !MCValue {
2049	const pt = func.pt;
2050	const zcu = pt.zcu;
2051
2052	const bit_size = elem_ty.bitSize(zcu);
2053	const min_size: u64 = switch (elem_ty.zigTypeTag(zcu)) {
2054	.float => if (func.hasFeature(.d)) 64 else 32,
2055	.vector => 256, // TODO: calculate it from avl * vsew
2056	else => 64,
2057	};
2058
2059	if (reg_ok and bit_size <= min_size) {
2060	if (func.register_manager.tryAllocReg(inst, func.regGeneralClassForType(elem_ty))) |reg| {
2061	return .{ .register = reg };
2062	}
2063	} else if (reg_ok and elem_ty.zigTypeTag(zcu) == .vector) {
2064	return func.fail("did you forget to extend vector registers before allocating", .{});
2065	}
2066
2067	const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(elem_ty, zcu));
2068	return .{ .load_frame = .{ .index = frame_index } };
2069	}
2070
2071	/// Allocates a register from the general purpose set and returns the Register and the Lock.
2072	///
2073	/// Up to the caller to unlock the register later.
2074	fn allocReg(func: *Func, reg_class: abi.RegisterClass) !struct { Register, RegisterLock } {
2075	if (reg_class == .float and !func.hasFeature(.f))
2076	std.debug.panic("allocReg class == float where F isn't enabled", .{});
2077	if (reg_class == .vector and !func.hasFeature(.v))
2078	std.debug.panic("allocReg class == vector where V isn't enabled", .{});
2079
2080	const class = switch (reg_class) {
2081	.int => abi.Registers.Integer.general_purpose,
2082	.float => abi.Registers.Float.general_purpose,
2083	.vector => abi.Registers.Vector.general_purpose,
2084	};
2085
2086	const reg = try func.register_manager.allocReg(null, class);
2087	const lock = func.register_manager.lockRegAssumeUnused(reg);
2088	return .{ reg, lock };
2089	}
2090
2091	/// Similar to `allocReg` but will copy the MCValue into the Register unless `operand` is already
2092	/// a register, in which case it will return a possible lock to that register.
2093	fn promoteReg(func: *Func, ty: Type, operand: MCValue) !struct { Register, ?RegisterLock } {
2094	if (operand == .register) {
2095	const op_reg = operand.register;
2096	return .{ op_reg, func.register_manager.lockReg(operand.register) };
2097	}
2098
2099	const class = func.typeRegClass(ty);
2100	const reg, const lock = try func.allocReg(class);
2101	try func.genSetReg(ty, reg, operand);
2102	return .{ reg, lock };
2103	}
2104
2105	fn elemOffset(func: *Func, index_ty: Type, index: MCValue, elem_size: u64) !Register {
2106	const reg: Register = blk: {
2107	switch (index) {
2108	.immediate => |imm| {
2109	// Optimisation: if index MCValue is an immediate, we can multiply in `comptime`
2110	// and set the register directly to the scaled offset as an immediate.
2111	const reg = try func.register_manager.allocReg(null, func.regGeneralClassForType(index_ty));
2112	try func.genSetReg(index_ty, reg, .{ .immediate = imm * elem_size });
2113	break :blk reg;
2114	},
2115	else => {
2116	const reg = try func.copyToTmpRegister(index_ty, index);
2117	const lock = func.register_manager.lockRegAssumeUnused(reg);
2118	defer func.register_manager.unlockReg(lock);
2119
2120	const result_reg, const result_lock = try func.allocReg(.int);
2121	defer func.register_manager.unlockReg(result_lock);
2122
2123	try func.genBinOp(
2124	.mul,
2125	.{ .register = reg },
2126	index_ty,
2127	.{ .immediate = elem_size },
2128	index_ty,
2129	result_reg,
2130	);
2131
2132	break :blk result_reg;
2133	},
2134	}
2135	};
2136	return reg;
2137	}
2138
2139	pub fn spillInstruction(func: *Func, reg: Register, inst: Air.Inst.Index) !void {
2140	const tracking = func.inst_tracking.getPtr(inst) orelse return;
2141	for (tracking.getRegs()) |tracked_reg| {
2142	if (tracked_reg.id() == reg.id()) break;
2143	} else unreachable; // spilled reg not tracked with spilled instruciton
2144	try tracking.spill(func, inst);
2145	try tracking.trackSpill(func, inst);
2146	}
2147
2148	pub fn spillRegisters(func: *Func, comptime registers: []const Register) !void {
2149	inline for (registers) |reg| try func.register_manager.getKnownReg(reg, null);
2150	}
2151
2152	/// Copies a value to a register without tracking the register. The register is not considered
2153	/// allocated. A second call to `copyToTmpRegister` may return the same register.
2154	/// This can have a side effect of spilling instructions to the stack to free up a register.
2155	fn copyToTmpRegister(func: *Func, ty: Type, mcv: MCValue) !Register {
2156	log.debug("copyToTmpRegister ty: {f}", .{ty.fmt(func.pt)});
2157	const reg = try func.register_manager.allocReg(null, func.regTempClassForType(ty));
2158	try func.genSetReg(ty, reg, mcv);
2159	return reg;
2160	}
2161
2162	/// Allocates a new register and copies `mcv` into it.
2163	/// `reg_owner` is the instruction that gets associated with the register in the register table.
2164	/// This can have a side effect of spilling instructions to the stack to free up a register.
2165	fn copyToNewRegister(func: *Func, reg_owner: Air.Inst.Index, mcv: MCValue) !MCValue {
2166	const ty = func.typeOfIndex(reg_owner);
2167	const reg = try func.register_manager.allocReg(reg_owner, func.regGeneralClassForType(ty));
2168	try func.genSetReg(func.typeOfIndex(reg_owner), reg, mcv);
2169	return MCValue{ .register = reg };
2170	}
2171
2172	fn airAlloc(func: *Func, inst: Air.Inst.Index) !void {
2173	const result = MCValue{ .lea_frame = .{ .index = try func.allocMemPtr(inst) } };
2174	return func.finishAir(inst, result, .{ .none, .none, .none });
2175	}
2176
2177	fn airRetPtr(func: *Func, inst: Air.Inst.Index) !void {
2178	const result: MCValue = switch (func.ret_mcv.long) {
2179	.none => .{ .lea_frame = .{ .index = try func.allocMemPtr(inst) } },
2180	.load_frame => .{ .register_offset = .{
2181	.reg = (try func.copyToNewRegister(
2182	inst,
2183	func.ret_mcv.long,
2184	)).register,
2185	.off = func.ret_mcv.short.indirect.off,
2186	} },
2187	else => |t| return func.fail("TODO: airRetPtr {s}", .{@tagName(t)}),
2188	};
2189	return func.finishAir(inst, result, .{ .none, .none, .none });
2190	}
2191
2192	fn airFptrunc(func: *Func, inst: Air.Inst.Index) !void {
2193	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2194	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airFptrunc for {}", .{func.target.cpu.arch});
2195	return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2196	}
2197
2198	fn airFpext(func: *Func, inst: Air.Inst.Index) !void {
2199	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2200	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airFpext for {}", .{func.target.cpu.arch});
2201	return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2202	}
2203
2204	fn airIntCast(func: *Func, inst: Air.Inst.Index) !void {
2205	const pt = func.pt;
2206	const zcu = pt.zcu;
2207	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2208	const src_ty = func.typeOf(ty_op.operand);
2209	const dst_ty = func.typeOfIndex(inst);
2210
2211	const result: MCValue = result: {
2212	const src_int_info = src_ty.intInfo(zcu);
2213	const dst_int_info = dst_ty.intInfo(zcu);
2214
2215	const min_ty = if (dst_int_info.bits < src_int_info.bits) dst_ty else src_ty;
2216
2217	const src_mcv = try func.resolveInst(ty_op.operand);
2218
2219	const src_storage_bits: u16 = switch (src_mcv) {
2220	.register => 64,
2221	.load_frame => src_int_info.bits,
2222	else => return func.fail("airIntCast from {s}", .{@tagName(src_mcv)}),
2223	};
2224
2225	const dst_mcv = if (dst_int_info.bits <= src_storage_bits and
2226	@divCeil(dst_int_info.bits, 64) == @divCeil(src_storage_bits, 64) and
2227	func.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {
2228	const dst_mcv = try func.allocRegOrMem(dst_ty, inst, true);
2229	try func.genCopy(min_ty, dst_mcv, src_mcv);
2230	break :dst dst_mcv;
2231	};
2232
2233	if (dst_int_info.bits <= src_int_info.bits)
2234	break :result dst_mcv;
2235
2236	if (dst_int_info.bits > 64 or src_int_info.bits > 64)
2237	break :result null; // TODO
2238
2239	break :result dst_mcv;
2240	} orelse return func.fail("TODO: implement airIntCast from {f} to {f}", .{
2241	src_ty.fmt(pt), dst_ty.fmt(pt),
2242	});
2243
2244	return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2245	}
2246
2247	fn airTrunc(func: *Func, inst: Air.Inst.Index) !void {
2248	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2249	if (func.liveness.isUnused(inst))
2250	return func.finishAir(inst, .unreach, .{ ty_op.operand, .none, .none });
2251	// we assume no zeroext in the "Zig ABI", so it's fine to just not truncate it.
2252	const operand = try func.resolveInst(ty_op.operand);
2253
2254	// we can do it just to be safe, but this shouldn't be needed for no-runtime safety modes
2255	switch (operand) {
2256	.register => |reg| try func.truncateRegister(func.typeOf(ty_op.operand), reg),
2257	else => {},
2258	}
2259
2260	return func.finishAir(inst, operand, .{ ty_op.operand, .none, .none });
2261	}
2262
2263	fn airNot(func: *Func, inst: Air.Inst.Index) !void {
2264	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2265	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
2266	const pt = func.pt;
2267	const zcu = pt.zcu;
2268
2269	const operand = try func.resolveInst(ty_op.operand);
2270	const ty = func.typeOf(ty_op.operand);
2271
2272	const operand_reg, const operand_lock = try func.promoteReg(ty, operand);
2273	defer if (operand_lock) |lock| func.register_manager.unlockReg(lock);
2274
2275	const dst_reg: Register =
2276	if (func.reuseOperand(inst, ty_op.operand, 0, operand) and operand == .register)
2277	operand.register
2278	else
2279	(try func.allocRegOrMem(func.typeOfIndex(inst), inst, true)).register;
2280
2281	switch (ty.zigTypeTag(zcu)) {
2282	.bool => {
2283	_ = try func.addInst(.{
2284	.tag = .pseudo_not,
2285	.data = .{
2286	.rr = .{
2287	.rs = operand_reg,
2288	.rd = dst_reg,
2289	},
2290	},
2291	});
2292	},
2293	.int => {
2294	const size = ty.bitSize(zcu);
2295	if (!math.isPowerOfTwo(size))
2296	return func.fail("TODO: airNot non-pow 2 int size", .{});
2297
2298	switch (size) {
2299	32, 64 => {
2300	_ = try func.addInst(.{
2301	.tag = .xori,
2302	.data = .{
2303	.i_type = .{
2304	.rd = dst_reg,
2305	.rs1 = operand_reg,
2306	.imm12 = Immediate.s(-1),
2307	},
2308	},
2309	});
2310	},
2311	8, 16 => return func.fail("TODO: airNot 8 or 16, {}", .{size}),
2312	else => unreachable,
2313	}
2314	},
2315	else => unreachable,
2316	}
2317
2318	break :result .{ .register = dst_reg };
2319	};
2320	return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2321	}
2322
2323	fn airSlice(func: *Func, inst: Air.Inst.Index) !void {
2324	const pt = func.pt;
2325	const zcu = pt.zcu;
2326	const ty_pl = func.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
2327	const bin_op = func.air.extraData(Air.Bin, ty_pl.payload).data;
2328
2329	const slice_ty = func.typeOfIndex(inst);
2330	const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(slice_ty, zcu));
2331
2332	const ptr_ty = func.typeOf(bin_op.lhs);
2333	try func.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, .{ .air_ref = bin_op.lhs });
2334
2335	const len_ty = func.typeOf(bin_op.rhs);
2336	try func.genSetMem(
2337	.{ .frame = frame_index },
2338	@intCast(ptr_ty.abiSize(zcu)),
2339	len_ty,
2340	.{ .air_ref = bin_op.rhs },
2341	);
2342
2343	const result = MCValue{ .load_frame = .{ .index = frame_index } };
2344	return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2345	}
2346
2347	fn airBinOp(func: *Func, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
2348	const pt = func.pt;
2349	const zcu = pt.zcu;
2350	const bin_op = func.air.instructions.items(.data)[@backingInt(inst)].bin_op;
2351	const dst_mcv = try func.binOp(inst, tag, bin_op.lhs, bin_op.rhs);
2352
2353	const dst_ty = func.typeOfIndex(inst);
2354	if (dst_ty.isAbiInt(zcu)) {
2355	const abi_size: u32 = @intCast(dst_ty.abiSize(zcu));
2356	const bit_size: u32 = @intCast(dst_ty.bitSize(zcu));
2357	if (abi_size * 8 > bit_size) {
2358	const dst_lock = switch (dst_mcv) {
2359	.register => |dst_reg| func.register_manager.lockRegAssumeUnused(dst_reg),
2360	else => null,
2361	};
2362	defer if (dst_lock) |lock| func.register_manager.unlockReg(lock);
2363
2364	if (dst_mcv.isRegister()) {
2365	try func.truncateRegister(dst_ty, dst_mcv.getReg().?);
2366	} else {
2367	const tmp_reg, const tmp_lock = try func.allocReg(.int);
2368	defer func.register_manager.unlockReg(tmp_lock);
2369
2370	const hi_ty = try pt.intType(.unsigned, @intCast((dst_ty.bitSize(zcu) - 1) % 64 + 1));
2371	const hi_mcv = dst_mcv.address().offset(@intCast(bit_size / 64 * 8)).deref();
2372	try func.genSetReg(hi_ty, tmp_reg, hi_mcv);
2373	try func.truncateRegister(dst_ty, tmp_reg);
2374	try func.genCopy(hi_ty, hi_mcv, .{ .register = tmp_reg });
2375	}
2376	}
2377	}
2378
2379	return func.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
2380	}
2381
2382	fn binOp(
2383	func: *Func,
2384	maybe_inst: ?Air.Inst.Index,
2385	air_tag: Air.Inst.Tag,
2386	lhs_air: Air.Inst.Ref,
2387	rhs_air: Air.Inst.Ref,
2388	) !MCValue {
2389	_ = maybe_inst;
2390	const pt = func.pt;
2391	const zcu = pt.zcu;
2392	const lhs_ty = func.typeOf(lhs_air);
2393	const rhs_ty = func.typeOf(rhs_air);
2394
2395	if (lhs_ty.isRuntimeFloat()) libcall: {
2396	const float_bits = lhs_ty.floatBits(func.target);
2397	const type_needs_libcall = switch (float_bits) {
2398	16 => true,
2399	32, 64 => false,
2400	80, 128 => true,
2401	else => unreachable,
2402	};
2403	if (!type_needs_libcall) break :libcall;
2404	return func.fail("binOp libcall runtime-float ops", .{});
2405	}
2406
2407	// don't have support for certain sizes of addition
2408	switch (lhs_ty.zigTypeTag(zcu)) {
2409	.vector => {}, // works differently and fails in a different place
2410	else => if (lhs_ty.bitSize(zcu) > 64) return func.fail("TODO: binOp >= 64 bits", .{}),
2411	}
2412
2413	const lhs_mcv = try func.resolveInst(lhs_air);
2414	const rhs_mcv = try func.resolveInst(rhs_air);
2415
2416	const class_for_dst_ty: abi.RegisterClass = switch (air_tag) {
2417	// will always return int register no matter the input
2418	.cmp_eq,
2419	.cmp_neq,
2420	.cmp_lt,
2421	.cmp_lte,
2422	.cmp_gt,
2423	.cmp_gte,
2424	=> .int,
2425
2426	else => func.typeRegClass(lhs_ty),
2427	};
2428
2429	const dst_reg, const dst_lock = try func.allocReg(class_for_dst_ty);
2430	defer func.register_manager.unlockReg(dst_lock);
2431
2432	try func.genBinOp(
2433	air_tag,
2434	lhs_mcv,
2435	lhs_ty,
2436	rhs_mcv,
2437	rhs_ty,
2438	dst_reg,
2439	);
2440
2441	return .{ .register = dst_reg };
2442	}
2443
2444	/// Does the same thing as binOp however is meant to be used internally to the backend.
2445	///
2446	/// The `dst_reg` argument is meant to be caller-locked. Asserts that the binOp result can be
2447	/// fit into the register.
2448	///
2449	/// Assumes that the `dst_reg` class is correct.
2450	fn genBinOp(
2451	func: *Func,
2452	tag: Air.Inst.Tag,
2453	lhs_mcv: MCValue,
2454	lhs_ty: Type,
2455	rhs_mcv: MCValue,
2456	rhs_ty: Type,
2457	dst_reg: Register,
2458	) !void {
2459	const pt = func.pt;
2460	const zcu = pt.zcu;
2461	const bit_size = lhs_ty.bitSize(zcu);
2462
2463	const is_unsigned = lhs_ty.isUnsignedInt(zcu);
2464
2465	const lhs_reg, const maybe_lhs_lock = try func.promoteReg(lhs_ty, lhs_mcv);
2466	const rhs_reg, const maybe_rhs_lock = try func.promoteReg(rhs_ty, rhs_mcv);
2467
2468	defer if (maybe_lhs_lock) |lock| func.register_manager.unlockReg(lock);
2469	defer if (maybe_rhs_lock) |lock| func.register_manager.unlockReg(lock);
2470
2471	switch (tag) {
2472	.add,
2473	.add_wrap,
2474	.sub,
2475	.sub_wrap,
2476	.mul,
2477	.mul_wrap,
2478	.rem,
2479	.div_trunc,
2480	.div_exact,
2481	=> {
2482	switch (tag) {
2483	.rem,
2484	.div_trunc,
2485	.div_exact,
2486	=> {
2487	if (!math.isPowerOfTwo(bit_size)) {
2488	try func.truncateRegister(lhs_ty, lhs_reg);
2489	try func.truncateRegister(rhs_ty, rhs_reg);
2490	}
2491	},
2492	else => {
2493	if (!math.isPowerOfTwo(bit_size))
2494	return func.fail(
2495	"TODO: genBinOp verify if needs to truncate {s} non-pow 2, found {}",
2496	.{ @tagName(tag), bit_size },
2497	);
2498	},
2499	}
2500
2501	switch (lhs_ty.zigTypeTag(zcu)) {
2502	.int => {
2503	const mnem: Mnemonic = switch (tag) {
2504	.add, .add_wrap => switch (bit_size) {
2505	8, 16, 64 => .add,
2506	32 => .addw,
2507	else => unreachable,
2508	},
2509	.sub, .sub_wrap => switch (bit_size) {
2510	8, 16, 32 => .subw,
2511	64 => .sub,
2512	else => unreachable,
2513	},
2514	.mul, .mul_wrap => switch (bit_size) {
2515	8, 16, 64 => .mul,
2516	32 => .mulw,
2517	else => unreachable,
2518	},
2519	.rem => switch (bit_size) {
2520	8, 16, 32 => if (is_unsigned) .remuw else .remw,
2521	else => if (is_unsigned) .remu else .rem,
2522	},
2523	.div_trunc, .div_exact => switch (bit_size) {
2524	8, 16, 32 => if (is_unsigned) .divuw else .divw,
2525	else => if (is_unsigned) .divu else .div,
2526	},
2527	else => unreachable,
2528	};
2529
2530	_ = try func.addInst(.{
2531	.tag = mnem,
2532	.data = .{
2533	.r_type = .{
2534	.rd = dst_reg,
2535	.rs1 = lhs_reg,
2536	.rs2 = rhs_reg,
2537	},
2538	},
2539	});
2540	},
2541	.float => {
2542	const mir_tag: Mnemonic = switch (tag) {
2543	.add => switch (bit_size) {
2544	32 => .fadds,
2545	64 => .faddd,
2546	else => unreachable,
2547	},
2548	.sub => switch (bit_size) {
2549	32 => .fsubs,
2550	64 => .fsubd,
2551	else => unreachable,
2552	},
2553	.mul => switch (bit_size) {
2554	32 => .fmuls,
2555	64 => .fmuld,
2556	else => unreachable,
2557	},
2558	else => return func.fail("TODO: genBinOp {s} Float", .{@tagName(tag)}),
2559	};
2560
2561	_ = try func.addInst(.{
2562	.tag = mir_tag,
2563	.data = .{
2564	.r_type = .{
2565	.rd = dst_reg,
2566	.rs1 = lhs_reg,
2567	.rs2 = rhs_reg,
2568	},
2569	},
2570	});
2571	},
2572	.vector => {
2573	const num_elem = lhs_ty.vectorLen(zcu);
2574	const elem_size = lhs_ty.childType(zcu).bitSize(zcu);
2575
2576	const child_ty = lhs_ty.childType(zcu);
2577
2578	const mir_tag: Mnemonic = switch (tag) {
2579	.add => switch (child_ty.zigTypeTag(zcu)) {
2580	.int => .vaddvv,
2581	.float => .vfaddvv,
2582	else => unreachable,
2583	},
2584	.sub => switch (child_ty.zigTypeTag(zcu)) {
2585	.int => .vsubvv,
2586	.float => .vfsubvv,
2587	else => unreachable,
2588	},
2589	.mul => switch (child_ty.zigTypeTag(zcu)) {
2590	.int => .vmulvv,
2591	.float => .vfmulvv,
2592	else => unreachable,
2593	},
2594	else => return func.fail("TODO: genBinOp {s} Vector", .{@tagName(tag)}),
2595	};
2596
2597	try func.setVl(.zero, num_elem, .{
2598	.vsew = switch (elem_size) {
2599	8 => .@"8",
2600	16 => .@"16",
2601	32 => .@"32",
2602	64 => .@"64",
2603	else => return func.fail("TODO: genBinOp > 64 bit elements, found {d}", .{elem_size}),
2604	},
2605	.vlmul = .m1,
2606	.vma = true,
2607	.vta = true,
2608	});
2609
2610	_ = try func.addInst(.{
2611	.tag = mir_tag,
2612	.data = .{
2613	.r_type = .{
2614	.rd = dst_reg,
2615	.rs1 = rhs_reg,
2616	.rs2 = lhs_reg,
2617	},
2618	},
2619	});
2620	},
2621	else => unreachable,
2622	}
2623	},
2624
2625	.add_sat,
2626	=> {
2627	if (bit_size != 64 or !is_unsigned)
2628	return func.fail("TODO: genBinOp ty: {f}", .{lhs_ty.fmt(pt)});
2629
2630	const tmp_reg = try func.copyToTmpRegister(rhs_ty, .{ .register = rhs_reg });
2631	const tmp_lock = func.register_manager.lockRegAssumeUnused(tmp_reg);
2632	defer func.register_manager.unlockReg(tmp_lock);
2633
2634	_ = try func.addInst(.{
2635	.tag = .add,
2636	.data = .{ .r_type = .{
2637	.rd = tmp_reg,
2638	.rs1 = rhs_reg,
2639	.rs2 = lhs_reg,
2640	} },
2641	});
2642
2643	_ = try func.addInst(.{
2644	.tag = .sltu,
2645	.data = .{ .r_type = .{
2646	.rd = dst_reg,
2647	.rs1 = tmp_reg,
2648	.rs2 = lhs_reg,
2649	} },
2650	});
2651
2652	// neg dst_reg, dst_reg
2653	_ = try func.addInst(.{
2654	.tag = .sub,
2655	.data = .{ .r_type = .{
2656	.rd = dst_reg,
2657	.rs1 = .zero,
2658	.rs2 = dst_reg,
2659	} },
2660	});
2661
2662	_ = try func.addInst(.{
2663	.tag = .@"or",
2664	.data = .{ .r_type = .{
2665	.rd = dst_reg,
2666	.rs1 = dst_reg,
2667	.rs2 = tmp_reg,
2668	} },
2669	});
2670	},
2671
2672	.ptr_add,
2673	.ptr_sub,
2674	=> {
2675	const tmp_reg = try func.copyToTmpRegister(rhs_ty, .{ .register = rhs_reg });
2676	const tmp_mcv = MCValue{ .register = tmp_reg };
2677	const tmp_lock = func.register_manager.lockRegAssumeUnused(tmp_reg);
2678	defer func.register_manager.unlockReg(tmp_lock);
2679
2680	// RISC-V has no immediate mul, so we copy the size to a temporary register
2681	const elem_size = lhs_ty.indexableElem(zcu).abiSize(zcu);
2682	const elem_size_reg = try func.copyToTmpRegister(Type.u64, .{ .immediate = elem_size });
2683
2684	try func.genBinOp(
2685	.mul,
2686	tmp_mcv,
2687	rhs_ty,
2688	.{ .register = elem_size_reg },
2689	Type.u64,
2690	tmp_reg,
2691	);
2692
2693	try func.genBinOp(
2694	switch (tag) {
2695	.ptr_add => .add,
2696	.ptr_sub => .sub,
2697	else => unreachable,
2698	},
2699	lhs_mcv,
2700	Type.u64, // we know it's a pointer, so it'll be usize.
2701	tmp_mcv,
2702	Type.u64,
2703	dst_reg,
2704	);
2705	},
2706
2707	.bit_and,
2708	.bit_or,
2709	=> {
2710	_ = try func.addInst(.{
2711	.tag = switch (tag) {
2712	.bit_and => .@"and",
2713	.bit_or => .@"or",
2714	else => unreachable,
2715	},
2716	.data = .{
2717	.r_type = .{
2718	.rd = dst_reg,
2719	.rs1 = lhs_reg,
2720	.rs2 = rhs_reg,
2721	},
2722	},
2723	});
2724	},
2725
2726	.shr,
2727	.shr_exact,
2728	.shl,
2729	.shl_exact,
2730	=> {
2731	if (lhs_ty.isVector(zcu) and !rhs_ty.isVector(zcu)) return func.fail("TODO: vector shift with scalar rhs", .{});
2732	if (bit_size > 64) return func.fail("TODO: genBinOp shift > 64 bits, {}", .{bit_size});
2733	try func.truncateRegister(rhs_ty, rhs_reg);
2734
2735	const mir_tag: Mnemonic = switch (tag) {
2736	.shl, .shl_exact => switch (bit_size) {
2737	1...31, 33...64 => .sll,
2738	32 => .sllw,
2739	else => unreachable,
2740	},
2741	.shr, .shr_exact => switch (bit_size) {
2742	1...31, 33...64 => .srl,
2743	32 => .srlw,
2744	else => unreachable,
2745	},
2746	else => unreachable,
2747	};
2748
2749	_ = try func.addInst(.{
2750	.tag = mir_tag,
2751	.data = .{ .r_type = .{
2752	.rd = dst_reg,
2753	.rs1 = lhs_reg,
2754	.rs2 = rhs_reg,
2755	} },
2756	});
2757	},
2758
2759	// TODO: move the isel logic out of lower and into here.
2760	.cmp_eq,
2761	.cmp_neq,
2762	.cmp_lt,
2763	.cmp_lte,
2764	.cmp_gt,
2765	.cmp_gte,
2766	=> {
2767	assert(lhs_reg.class() == rhs_reg.class());
2768	if (lhs_reg.class() == .int) {
2769	try func.truncateRegister(lhs_ty, lhs_reg);
2770	try func.truncateRegister(rhs_ty, rhs_reg);
2771	}
2772
2773	_ = try func.addInst(.{
2774	.tag = .pseudo_compare,
2775	.data = .{
2776	.compare = .{
2777	.op = switch (tag) {
2778	.cmp_eq => .eq,
2779	.cmp_neq => .neq,
2780	.cmp_lt => .lt,
2781	.cmp_lte => .lte,
2782	.cmp_gt => .gt,
2783	.cmp_gte => .gte,
2784	else => unreachable,
2785	},
2786	.rd = dst_reg,
2787	.rs1 = lhs_reg,
2788	.rs2 = rhs_reg,
2789	.ty = lhs_ty,
2790	},
2791	},
2792	});
2793	},
2794
2795	// A branchless @min/@max sequence.
2796	//
2797	// Assume that a0 and a1 are the lhs and rhs respectively.
2798	// Also assume that a2 is the destination register.
2799	//
2800	// Algorithm:
2801	// slt s0, a0, a1
2802	// sub s0, zero, s0
2803	// xor a2, a0, a1
2804	// and s0, a2, s0
2805	// xor a2, a0, s0 # a0 is @min, a1 is @max
2806	//
2807	// "slt s0, a0, a1" will set s0 to 1 if a0 is less than a1, and 1 otherwise.
2808	//
2809	// "sub s0, zero, s0" will set all the bits of s0 to 1 if it was 1, otherwise it'll remain at 0.
2810	//
2811	// "xor a2, a0, a1" stores the bitwise XOR of a0 and a1 in a2. Effectively getting the difference between them.
2812	//
2813	// "and a0, a2, s0" here we mask the result of the XOR with the negated s0. If a0 < a1, s0 is -1, which
2814	// doesn't change the bits of a2. If a0 >= a1, s0 is 0, nullifying a2.
2815	//
2816	// "xor a2, a0, s0" the final XOR operation adjusts a2 to be the minimum value of a0 and a1. If a0 was less than
2817	// a1, s0 was -1, flipping all the bits in a2 and effectively restoring a0. If a0 was greater than or equal to a1,
2818	// s0 was 0, leaving a2 unchanged as a0.
2819	.min, .max => {
2820	switch (lhs_ty.zigTypeTag(zcu)) {
2821	.int => {
2822	const int_info = lhs_ty.intInfo(zcu);
2823
2824	const mask_reg, const mask_lock = try func.allocReg(.int);
2825	defer func.register_manager.unlockReg(mask_lock);
2826
2827	_ = try func.addInst(.{
2828	.tag = if (int_info.signedness == .unsigned) .sltu else .slt,
2829	.data = .{ .r_type = .{
2830	.rd = mask_reg,
2831	.rs1 = lhs_reg,
2832	.rs2 = rhs_reg,
2833	} },
2834	});
2835
2836	_ = try func.addInst(.{
2837	.tag = .sub,
2838	.data = .{ .r_type = .{
2839	.rd = mask_reg,
2840	.rs1 = .zero,
2841	.rs2 = mask_reg,
2842	} },
2843	});
2844
2845	_ = try func.addInst(.{
2846	.tag = .xor,
2847	.data = .{ .r_type = .{
2848	.rd = dst_reg,
2849	.rs1 = lhs_reg,
2850	.rs2 = rhs_reg,
2851	} },
2852	});
2853
2854	_ = try func.addInst(.{
2855	.tag = .@"and",
2856	.data = .{ .r_type = .{
2857	.rd = mask_reg,
2858	.rs1 = dst_reg,
2859	.rs2 = mask_reg,
2860	} },
2861	});
2862
2863	_ = try func.addInst(.{
2864	.tag = .xor,
2865	.data = .{ .r_type = .{
2866	.rd = dst_reg,
2867	.rs1 = if (tag == .min) rhs_reg else lhs_reg,
2868	.rs2 = mask_reg,
2869	} },
2870	});
2871	},
2872	else => |t| return func.fail("TODO: genBinOp min/max for {s}", .{@tagName(t)}),
2873	}
2874	},
2875	else => return func.fail("TODO: genBinOp {}", .{tag}),
2876	}
2877	}
2878
2879	fn airPtrArithmetic(func: *Func, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
2880	const ty_pl = func.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
2881	const bin_op = func.air.extraData(Air.Bin, ty_pl.payload).data;
2882	const dst_mcv = try func.binOp(inst, tag, bin_op.lhs, bin_op.rhs);
2883	return func.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
2884	}
2885
2886	fn airAddWithOverflow(func: *Func, inst: Air.Inst.Index) !void {
2887	const pt = func.pt;
2888	const zcu = pt.zcu;
2889	const ty_pl = func.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
2890	const extra = func.air.extraData(Air.Bin, ty_pl.payload).data;
2891
2892	const rhs_ty = func.typeOf(extra.rhs);
2893	const lhs_ty = func.typeOf(extra.lhs);
2894
2895	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
2896	switch (lhs_ty.zigTypeTag(zcu)) {
2897	.vector => return func.fail("TODO implement add with overflow for Vector type", .{}),
2898	.int => {
2899	const int_info = lhs_ty.intInfo(zcu);
2900
2901	const tuple_ty = func.typeOfIndex(inst);
2902	const result_mcv = try func.allocRegOrMem(tuple_ty, inst, false);
2903	const offset = result_mcv.load_frame;
2904
2905	if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) {
2906	const add_result = try func.binOp(null, .add, extra.lhs, extra.rhs);
2907
2908	try func.genSetMem(
2909	.{ .frame = offset.index },
2910	offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(0, zcu))),
2911	lhs_ty,
2912	add_result,
2913	);
2914
2915	const trunc_reg = try func.copyToTmpRegister(lhs_ty, add_result);
2916	const trunc_reg_lock = func.register_manager.lockRegAssumeUnused(trunc_reg);
2917	defer func.register_manager.unlockReg(trunc_reg_lock);
2918
2919	const overflow_reg, const overflow_lock = try func.allocReg(.int);
2920	defer func.register_manager.unlockReg(overflow_lock);
2921
2922	// if the result isn't equal after truncating it to the given type,
2923	// an overflow must have happened.
2924	try func.truncateRegister(lhs_ty, trunc_reg);
2925	try func.genBinOp(
2926	.cmp_neq,
2927	add_result,
2928	lhs_ty,
2929	.{ .register = trunc_reg },
2930	rhs_ty,
2931	overflow_reg,
2932	);
2933
2934	try func.genSetMem(
2935	.{ .frame = offset.index },
2936	offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(1, zcu))),
2937	Type.u1,
2938	.{ .register = overflow_reg },
2939	);
2940
2941	break :result result_mcv;
2942	} else {
2943	const rhs_mcv = try func.resolveInst(extra.rhs);
2944	const lhs_mcv = try func.resolveInst(extra.lhs);
2945
2946	const rhs_reg, const rhs_lock = try func.promoteReg(rhs_ty, rhs_mcv);
2947	const lhs_reg, const lhs_lock = try func.promoteReg(lhs_ty, lhs_mcv);
2948	defer {
2949	if (rhs_lock) |lock| func.register_manager.unlockReg(lock);
2950	if (lhs_lock) |lock| func.register_manager.unlockReg(lock);
2951	}
2952
2953	try func.truncateRegister(rhs_ty, rhs_reg);
2954	try func.truncateRegister(lhs_ty, lhs_reg);
2955
2956	const dest_reg, const dest_lock = try func.allocReg(.int);
2957	defer func.register_manager.unlockReg(dest_lock);
2958
2959	_ = try func.addInst(.{
2960	.tag = .add,
2961	.data = .{ .r_type = .{
2962	.rs1 = rhs_reg,
2963	.rs2 = lhs_reg,
2964	.rd = dest_reg,
2965	} },
2966	});
2967
2968	try func.truncateRegister(func.typeOfIndex(inst), dest_reg);
2969	const add_result: MCValue = .{ .register = dest_reg };
2970
2971	try func.genSetMem(
2972	.{ .frame = offset.index },
2973	offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(0, zcu))),
2974	lhs_ty,
2975	add_result,
2976	);
2977
2978	const trunc_reg = try func.copyToTmpRegister(lhs_ty, add_result);
2979	const trunc_reg_lock = func.register_manager.lockRegAssumeUnused(trunc_reg);
2980	defer func.register_manager.unlockReg(trunc_reg_lock);
2981
2982	const overflow_reg, const overflow_lock = try func.allocReg(.int);
2983	defer func.register_manager.unlockReg(overflow_lock);
2984
2985	// if the result isn't equal after truncating it to the given type,
2986	// an overflow must have happened.
2987	try func.truncateRegister(lhs_ty, trunc_reg);
2988	try func.genBinOp(
2989	.cmp_neq,
2990	add_result,
2991	lhs_ty,
2992	.{ .register = trunc_reg },
2993	rhs_ty,
2994	overflow_reg,
2995	);
2996
2997	try func.genSetMem(
2998	.{ .frame = offset.index },
2999	offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(1, zcu))),
3000	Type.u1,
3001	.{ .register = overflow_reg },
3002	);
3003
3004	break :result result_mcv;
3005	}
3006	},
3007	else => unreachable,
3008	}
3009	};
3010
3011	return func.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
3012	}
3013
3014	fn airSubWithOverflow(func: *Func, inst: Air.Inst.Index) !void {
3015	const pt = func.pt;
3016	const zcu = pt.zcu;
3017	const ty_pl = func.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
3018	const extra = func.air.extraData(Air.Bin, ty_pl.payload).data;
3019
3020	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
3021	const lhs = try func.resolveInst(extra.lhs);
3022	const rhs = try func.resolveInst(extra.rhs);
3023	const lhs_ty = func.typeOf(extra.lhs);
3024	const rhs_ty = func.typeOf(extra.rhs);
3025
3026	const int_info = lhs_ty.intInfo(zcu);
3027
3028	if (!math.isPowerOfTwo(int_info.bits) or int_info.bits < 8) {
3029	return func.fail("TODO: airSubWithOverflow non-power of 2 and less than 8 bits", .{});
3030	}
3031
3032	if (int_info.bits > 64) {
3033	return func.fail("TODO: airSubWithOverflow > 64 bits", .{});
3034	}
3035
3036	const tuple_ty = func.typeOfIndex(inst);
3037	const result_mcv = try func.allocRegOrMem(tuple_ty, inst, false);
3038	const offset = result_mcv.load_frame;
3039
3040	const dest_mcv = try func.binOp(null, .sub, extra.lhs, extra.rhs);
3041	assert(dest_mcv == .register);
3042	const dest_reg = dest_mcv.register;
3043
3044	try func.genSetMem(
3045	.{ .frame = offset.index },
3046	offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(0, zcu))),
3047	lhs_ty,
3048	.{ .register = dest_reg },
3049	);
3050
3051	const lhs_reg, const lhs_lock = try func.promoteReg(lhs_ty, lhs);
3052	defer if (lhs_lock) |lock| func.register_manager.unlockReg(lock);
3053
3054	const rhs_reg, const rhs_lock = try func.promoteReg(rhs_ty, rhs);
3055	defer if (rhs_lock) |lock| func.register_manager.unlockReg(lock);
3056
3057	const overflow_reg = try func.copyToTmpRegister(Type.u64, .{ .immediate = 0 });
3058
3059	const overflow_lock = func.register_manager.lockRegAssumeUnused(overflow_reg);
3060	defer func.register_manager.unlockReg(overflow_lock);
3061
3062	switch (int_info.signedness) {
3063	.unsigned => {
3064	_ = try func.addInst(.{
3065	.tag = .sltu,
3066	.data = .{ .r_type = .{
3067	.rd = overflow_reg,
3068	.rs1 = lhs_reg,
3069	.rs2 = rhs_reg,
3070	} },
3071	});
3072
3073	try func.genSetMem(
3074	.{ .frame = offset.index },
3075	offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(1, zcu))),
3076	Type.u1,
3077	.{ .register = overflow_reg },
3078	);
3079
3080	break :result result_mcv;
3081	},
3082	.signed => {
3083	switch (int_info.bits) {
3084	64 => {
3085	_ = try func.addInst(.{
3086	.tag = .slt,
3087	.data = .{ .r_type = .{
3088	.rd = overflow_reg,
3089	.rs1 = overflow_reg,
3090	.rs2 = rhs_reg,
3091	} },
3092	});
3093
3094	_ = try func.addInst(.{
3095	.tag = .slt,
3096	.data = .{ .r_type = .{
3097	.rd = rhs_reg,
3098	.rs1 = rhs_reg,
3099	.rs2 = lhs_reg,
3100	} },
3101	});
3102
3103	_ = try func.addInst(.{
3104	.tag = .xor,
3105	.data = .{ .r_type = .{
3106	.rd = lhs_reg,
3107	.rs1 = overflow_reg,
3108	.rs2 = rhs_reg,
3109	} },
3110	});
3111
3112	try func.genBinOp(
3113	.cmp_neq,
3114	.{ .register = overflow_reg },
3115	Type.u64,
3116	.{ .register = rhs_reg },
3117	Type.u64,
3118	overflow_reg,
3119	);
3120
3121	try func.genSetMem(
3122	.{ .frame = offset.index },
3123	offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(1, zcu))),
3124	Type.u1,
3125	.{ .register = overflow_reg },
3126	);
3127
3128	break :result result_mcv;
3129	},
3130	else => |int_bits| return func.fail("TODO: airSubWithOverflow signed {}", .{int_bits}),
3131	}
3132	},
3133	}
3134	};
3135
3136	return func.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
3137	}
3138
3139	fn airMulWithOverflow(func: *Func, inst: Air.Inst.Index) !void {
3140	const pt = func.pt;
3141	const zcu = pt.zcu;
3142	const ty_pl = func.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
3143	const extra = func.air.extraData(Air.Bin, ty_pl.payload).data;
3144
3145	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
3146	const lhs = try func.resolveInst(extra.lhs);
3147	const rhs = try func.resolveInst(extra.rhs);
3148	const lhs_ty = func.typeOf(extra.lhs);
3149	const rhs_ty = func.typeOf(extra.rhs);
3150
3151	const tuple_ty = func.typeOfIndex(inst);
3152
3153	// genSetReg needs to support register_offset src_mcv for this to be true.
3154	const result_mcv = try func.allocRegOrMem(tuple_ty, inst, false);
3155
3156	const result_off: i32 = @intCast(tuple_ty.structFieldOffset(0, zcu));
3157	const overflow_off: i32 = @intCast(tuple_ty.structFieldOffset(1, zcu));
3158
3159	const dest_reg, const dest_lock = try func.allocReg(.int);
3160	defer func.register_manager.unlockReg(dest_lock);
3161
3162	try func.genBinOp(
3163	.mul,
3164	lhs,
3165	lhs_ty,
3166	rhs,
3167	rhs_ty,
3168	dest_reg,
3169	);
3170
3171	try func.genCopy(
3172	lhs_ty,
3173	result_mcv.offset(result_off),
3174	.{ .register = dest_reg },
3175	);
3176
3177	switch (lhs_ty.zigTypeTag(zcu)) {
3178	else => |x| return func.fail("TODO: airMulWithOverflow {s}", .{@tagName(x)}),
3179	.int => {
3180	if (std.debug.runtime_safety) assert(lhs_ty.eql(rhs_ty));
3181
3182	const trunc_reg = try func.copyToTmpRegister(lhs_ty, .{ .register = dest_reg });
3183	const trunc_reg_lock = func.register_manager.lockRegAssumeUnused(trunc_reg);
3184	defer func.register_manager.unlockReg(trunc_reg_lock);
3185
3186	const overflow_reg, const overflow_lock = try func.allocReg(.int);
3187	defer func.register_manager.unlockReg(overflow_lock);
3188
3189	// if the result isn't equal after truncating it to the given type,
3190	// an overflow must have happened.
3191	try func.truncateRegister(func.typeOf(extra.lhs), trunc_reg);
3192	try func.genBinOp(
3193	.cmp_neq,
3194	.{ .register = dest_reg },
3195	lhs_ty,
3196	.{ .register = trunc_reg },
3197	rhs_ty,
3198	overflow_reg,
3199	);
3200
3201	try func.genCopy(
3202	lhs_ty,
3203	result_mcv.offset(overflow_off),
3204	.{ .register = overflow_reg },
3205	);
3206
3207	break :result result_mcv;
3208	},
3209	}
3210	};
3211
3212	return func.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
3213	}
3214
3215	fn airShlWithOverflow(func: *Func, inst: Air.Inst.Index) !void {
3216	const zcu = func.pt.zcu;
3217	const bin_op = func.air.instructions.items(.data)[@backingInt(inst)].bin_op;
3218	const result: MCValue = if (func.liveness.isUnused(inst))
3219	.unreach
3220	else if (func.typeOf(bin_op.lhs).isVector(zcu) and !func.typeOf(bin_op.rhs).isVector(zcu))
3221	return func.fail("TODO implement vector airShlWithOverflow with scalar rhs", .{})
3222	else
3223	return func.fail("TODO implement airShlWithOverflow", .{});
3224	return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
3225	}
3226
3227	fn airSubSat(func: *Func, inst: Air.Inst.Index) !void {
3228	const bin_op = func.air.instructions.items(.data)[@backingInt(inst)].bin_op;
3229	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airSubSat", .{});
3230	return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
3231	}
3232
3233	fn airMulSat(func: *Func, inst: Air.Inst.Index) !void {
3234	const bin_op = func.air.instructions.items(.data)[@backingInt(inst)].bin_op;
3235	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airMulSat", .{});
3236	return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
3237	}
3238
3239	fn airShlSat(func: *Func, inst: Air.Inst.Index) !void {
3240	const zcu = func.pt.zcu;
3241	const bin_op = func.air.instructions.items(.data)[@backingInt(inst)].bin_op;
3242	const result: MCValue = if (func.liveness.isUnused(inst))
3243	.unreach
3244	else if (func.typeOf(bin_op.lhs).isVector(zcu) and !func.typeOf(bin_op.rhs).isVector(zcu))
3245	return func.fail("TODO implement vector airShlSat with scalar rhs", .{})
3246	else
3247	return func.fail("TODO implement airShlSat", .{});
3248	return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
3249	}
3250
3251	fn airOptionalPayload(func: *Func, inst: Air.Inst.Index) !void {
3252	const zcu = func.pt.zcu;
3253	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
3254	const result: MCValue = result: {
3255	const pl_ty = func.typeOfIndex(inst);
3256	if (!pl_ty.hasRuntimeBits(zcu)) break :result .none;
3257
3258	const opt_mcv = try func.resolveInst(ty_op.operand);
3259	if (func.reuseOperand(inst, ty_op.operand, 0, opt_mcv)) {
3260	switch (opt_mcv) {
3261	.register => |pl_reg| try func.truncateRegister(pl_ty, pl_reg),
3262	else => {},
3263	}
3264	break :result opt_mcv;
3265	}
3266
3267	const pl_mcv = try func.allocRegOrMem(pl_ty, inst, true);
3268	try func.genCopy(pl_ty, pl_mcv, opt_mcv);
3269	break :result pl_mcv;
3270	};
3271	return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3272	}
3273
3274	fn airOptionalPayloadPtr(func: *Func, inst: Air.Inst.Index) !void {
3275	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
3276	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement .optional_payload_ptr for {}", .{func.target.cpu.arch});
3277	return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3278	}
3279
3280	fn airOptionalPayloadPtrSet(func: *Func, inst: Air.Inst.Index) !void {
3281	const zcu = func.pt.zcu;
3282
3283	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
3284	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
3285	const dst_ty = func.typeOfIndex(inst);
3286	const src_ty = func.typeOf(ty_op.operand);
3287	const opt_ty = src_ty.childType(zcu);
3288	const src_mcv = try func.resolveInst(ty_op.operand);
3289
3290	if (opt_ty.optionalReprIsPayload(zcu)) {
3291	break :result if (func.reuseOperand(inst, ty_op.operand, 0, src_mcv))
3292	src_mcv
3293	else
3294	try func.copyToNewRegister(inst, src_mcv);
3295	}
3296
3297	const dst_mcv: MCValue = if (src_mcv.isRegister() and
3298	func.reuseOperand(inst, ty_op.operand, 0, src_mcv))
3299	src_mcv
3300	else
3301	try func.copyToNewRegister(inst, src_mcv);
3302
3303	const pl_ty = dst_ty.childType(zcu);
3304	const pl_abi_size: i32 = @intCast(pl_ty.abiSize(zcu));
3305	try func.genSetMem(
3306	.{ .reg = dst_mcv.getReg().? },
3307	pl_abi_size,
3308	Type.bool,
3309	.{ .immediate = 1 },
3310	);
3311	break :result dst_mcv;
3312	};
3313	return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3314	}
3315
3316	fn airUnwrapErrErr(func: *Func, inst: Air.Inst.Index) !void {
3317	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
3318	const pt = func.pt;
3319	const zcu = pt.zcu;
3320	const err_union_ty = func.typeOf(ty_op.operand);
3321	const err_ty = err_union_ty.errorUnionSet(zcu);
3322	const payload_ty = err_union_ty.errorUnionPayload(zcu);
3323	const operand = try func.resolveInst(ty_op.operand);
3324
3325	const result: MCValue = result: {
3326	if (err_ty.errorSetIsEmpty(zcu)) {
3327	break :result .{ .immediate = 0 };
3328	}
3329
3330	if (!payload_ty.hasRuntimeBits(zcu)) {
3331	break :result operand;
3332	}
3333
3334	const err_off: u32 = @intCast(errUnionErrorOffset(payload_ty, zcu));
3335
3336	switch (operand) {
3337	.register => |reg| {
3338	const eu_lock = func.register_manager.lockReg(reg);
3339	defer if (eu_lock) |lock| func.register_manager.unlockReg(lock);
3340
3341	const result = try func.copyToNewRegister(inst, operand);
3342	if (err_off > 0) {
3343	try func.genBinOp(
3344	.shr,
3345	result,
3346	err_union_ty,
3347	.{ .immediate = @as(u6, @intCast(err_off * 8)) },
3348	Type.u8,
3349	result.register,
3350	);
3351	}
3352	break :result result;
3353	},
3354	.load_frame => |frame_addr| break :result .{ .load_frame = .{
3355	.index = frame_addr.index,
3356	.off = frame_addr.off + @as(i32, @intCast(err_off)),
3357	} },
3358	else => return func.fail("TODO implement unwrap_err_err for {}", .{operand}),
3359	}
3360	};
3361
3362	return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3363	}
3364
3365	fn airUnwrapErrPayload(func: *Func, inst: Air.Inst.Index) !void {
3366	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
3367	const operand_ty = func.typeOf(ty_op.operand);
3368	const operand = try func.resolveInst(ty_op.operand);
3369	const result = try func.genUnwrapErrUnionPayloadMir(operand_ty, operand);
3370	return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3371	}
3372
3373	fn genUnwrapErrUnionPayloadMir(
3374	func: *Func,
3375	err_union_ty: Type,
3376	err_union: MCValue,
3377	) !MCValue {
3378	const pt = func.pt;
3379	const zcu = pt.zcu;
3380	const payload_ty = err_union_ty.errorUnionPayload(zcu);
3381
3382	const result: MCValue = result: {
3383	if (!payload_ty.hasRuntimeBits(zcu)) break :result .none;
3384
3385	const payload_off: u31 = @intCast(errUnionPayloadOffset(payload_ty, zcu));
3386	switch (err_union) {
3387	.load_frame => |frame_addr| break :result .{ .load_frame = .{
3388	.index = frame_addr.index,
3389	.off = frame_addr.off + payload_off,
3390	} },
3391	.register => |reg| {
3392	const eu_lock = func.register_manager.lockReg(reg);
3393	defer if (eu_lock) |lock| func.register_manager.unlockReg(lock);
3394
3395	const result_reg = try func.copyToTmpRegister(err_union_ty, err_union);
3396	if (payload_off > 0) {
3397	try func.genBinOp(
3398	.shr,
3399	.{ .register = result_reg },
3400	err_union_ty,
3401	.{ .immediate = @as(u6, @intCast(payload_off * 8)) },
3402	Type.u8,
3403	result_reg,
3404	);
3405	}
3406	break :result .{ .register = result_reg };
3407	},
3408	else => return func.fail("TODO implement genUnwrapErrUnionPayloadMir for {}", .{err_union}),
3409	}
3410	};
3411
3412	return result;
3413	}
3414
3415	// *(E!T) -> E
3416	fn airUnwrapErrErrPtr(func: *Func, inst: Air.Inst.Index) !void {
3417	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
3418	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement unwrap error union error ptr for {}", .{func.target.cpu.arch});
3419	return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3420	}
3421
3422	// *(E!T) -> *T
3423	fn airUnwrapErrPayloadPtr(func: *Func, inst: Air.Inst.Index) !void {
3424	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
3425	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement unwrap error union payload ptr for {}", .{func.target.cpu.arch});
3426	return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3427	}
3428
3429	// *(E!T) => *T
3430	fn airErrUnionPayloadPtrSet(func: *Func, inst: Air.Inst.Index) !void {
3431	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
3432	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
3433	const zcu = func.pt.zcu;
3434	const src_ty = func.typeOf(ty_op.operand);
3435	const src_mcv = try func.resolveInst(ty_op.operand);
3436
3437	// `src_reg` contains the pointer to the error union
3438	const src_reg = switch (src_mcv) {
3439	.register => |reg| reg,
3440	else => try func.copyToTmpRegister(src_ty, src_mcv),
3441	};
3442	const src_lock = func.register_manager.lockRegAssumeUnused(src_reg);
3443	defer func.register_manager.unlockReg(src_lock);
3444
3445	// we set the place of where the error would have been to 0
3446	const eu_ty = src_ty.childType(zcu);
3447	const pl_ty = eu_ty.errorUnionPayload(zcu);
3448	const err_ty = eu_ty.errorUnionSet(zcu);
3449	const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
3450	try func.genSetMem(.{ .reg = src_reg }, err_off, err_ty, .{ .immediate = 0 });
3451
3452	const dst_reg, const dst_lock = if (func.reuseOperand(inst, ty_op.operand, 0, src_mcv))
3453	.{ src_reg, null }
3454	else
3455	try func.allocReg(.int);
3456	defer if (dst_lock) |lock| func.register_manager.unlockReg(lock);
3457
3458	// move the pointer to be at the payload
3459	const pl_off = errUnionPayloadOffset(pl_ty, zcu);
3460	try func.genBinOp(
3461	.add,
3462	.{ .register = src_reg },
3463	Type.u64,
3464	.{ .immediate = pl_off },
3465	Type.u64,
3466	dst_reg,
3467	);
3468
3469	break :result .{ .register = dst_reg };
3470	};
3471	return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3472	}
3473
3474	fn airErrReturnTrace(func: *Func, inst: Air.Inst.Index) !void {
3475	const result: MCValue = if (func.liveness.isUnused(inst))
3476	.unreach
3477	else
3478	return func.fail("TODO implement airErrReturnTrace for {}", .{func.target.cpu.arch});
3479	return func.finishAir(inst, result, .{ .none, .none, .none });
3480	}
3481
3482	fn airSetErrReturnTrace(func: *Func, inst: Air.Inst.Index) !void {
3483	_ = inst;
3484	return func.fail("TODO implement airSetErrReturnTrace for {}", .{func.target.cpu.arch});
3485	}
3486
3487	fn airSaveErrReturnTraceIndex(func: *Func, inst: Air.Inst.Index) !void {
3488	_ = inst;
3489	return func.fail("TODO implement airSaveErrReturnTraceIndex for {}", .{func.target.cpu.arch});
3490	}
3491
3492	fn airWrapOptional(func: *Func, inst: Air.Inst.Index) !void {
3493	const pt = func.pt;
3494	const zcu = pt.zcu;
3495	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
3496	const result: MCValue = result: {
3497	const pl_ty = func.typeOf(ty_op.operand);
3498	if (!pl_ty.hasRuntimeBits(zcu)) break :result .{ .immediate = 1 };
3499
3500	const opt_ty = func.typeOfIndex(inst);
3501	const pl_mcv = try func.resolveInst(ty_op.operand);
3502	const same_repr = opt_ty.optionalReprIsPayload(zcu);
3503	if (same_repr and func.reuseOperand(inst, ty_op.operand, 0, pl_mcv)) break :result pl_mcv;
3504
3505	const pl_lock: ?RegisterLock = switch (pl_mcv) {
3506	.register => |reg| func.register_manager.lockRegAssumeUnused(reg),
3507	else => null,
3508	};
3509	defer if (pl_lock) |lock| func.register_manager.unlockReg(lock);
3510
3511	const opt_mcv = try func.allocRegOrMem(opt_ty, inst, false);
3512	try func.genCopy(pl_ty, opt_mcv, pl_mcv);
3513
3514	if (!same_repr) {
3515	const pl_abi_size: i32 = @intCast(pl_ty.abiSize(zcu));
3516	switch (opt_mcv) {
3517	.load_frame => |frame_addr| {
3518	try func.genCopy(pl_ty, opt_mcv, pl_mcv);
3519	try func.genSetMem(
3520	.{ .frame = frame_addr.index },
3521	frame_addr.off + pl_abi_size,
3522	Type.u8,
3523	.{ .immediate = 1 },
3524	);
3525	},
3526	else => unreachable,
3527	}
3528	}
3529	break :result opt_mcv;
3530	};
3531	return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3532	}
3533
3534	/// T to E!T
3535	fn airWrapErrUnionPayload(func: *Func, inst: Air.Inst.Index) !void {
3536	const pt = func.pt;
3537	const zcu = pt.zcu;
3538	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
3539
3540	const eu_ty = ty_op.ty;
3541	const pl_ty = eu_ty.errorUnionPayload(zcu);
3542	const err_ty = eu_ty.errorUnionSet(zcu);
3543	const operand = try func.resolveInst(ty_op.operand);
3544
3545	const result: MCValue = result: {
3546	if (!pl_ty.hasRuntimeBits(zcu)) break :result .{ .immediate = 0 };
3547
3548	const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(eu_ty, zcu));
3549	const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, zcu));
3550	const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
3551	try func.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, operand);
3552	try func.genSetMem(.{ .frame = frame_index }, err_off, err_ty, .{ .immediate = 0 });
3553	break :result .{ .load_frame = .{ .index = frame_index } };
3554	};
3555
3556	return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3557	}
3558
3559	/// E to E!T
3560	fn airWrapErrUnionErr(func: *Func, inst: Air.Inst.Index) !void {
3561	const pt = func.pt;
3562	const zcu = pt.zcu;
3563	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
3564
3565	const eu_ty = ty_op.ty;
3566	const pl_ty = eu_ty.errorUnionPayload(zcu);
3567	const err_ty = eu_ty.errorUnionSet(zcu);
3568
3569	const result: MCValue = result: {
3570	if (!pl_ty.hasRuntimeBits(zcu)) break :result try func.resolveInst(ty_op.operand);
3571
3572	const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(eu_ty, zcu));
3573	const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, zcu));
3574	const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
3575	try func.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .{ .undef = null });
3576	const operand = try func.resolveInst(ty_op.operand);
3577	try func.genSetMem(.{ .frame = frame_index }, err_off, err_ty, operand);
3578	break :result .{ .load_frame = .{ .index = frame_index } };
3579	};
3580	return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3581	}
3582
3583	fn airRuntimeNavPtr(func: *Func, inst: Air.Inst.Index) !void {
3584	const zcu = func.pt.zcu;
3585	const ip = &zcu.intern_pool;
3586	const ty_nav = func.air.instructions.items(.data)[@backingInt(inst)].ty_nav;
3587	const ptr_ty: Type = ty_nav.ty;
3588
3589	const nav = ip.getNav(ty_nav.nav);
3590	const tlv_sym_index = if (func.bin_file.cast(.elf)) |elf_file| sym: {
3591	const zo = elf_file.zigObjectPtr().?;
3592	if (nav.getExtern(ip)) |e| {
3593	break :sym try elf_file.getGlobalSymbol(nav.name.toSlice(ip), e.lib_name.toSlice(ip));
3594	}
3595	break :sym try zo.getOrCreateMetadataForNav(zcu, ty_nav.nav);
3596	} else return func.fail("TODO runtime_nav_ptr on {}", .{func.bin_file.tag});
3597
3598	const dest_mcv = try func.allocRegOrMem(ptr_ty, inst, true);
3599	if (dest_mcv.isRegister()) {
3600	_ = try func.addInst(.{
3601	.tag = .pseudo_load_tlv,
3602	.data = .{ .reloc = .{
3603	.register = dest_mcv.getReg().?,
3604	.atom_index = @fromBackingInt(@intCast(try func.owner.getSymbolIndex(func))),
3605	.sym_index = @fromBackingInt(@intCast(tlv_sym_index)),
3606	} },
3607	});
3608	} else {
3609	const tmp_reg, const tmp_lock = try func.allocReg(.int);
3610	defer func.register_manager.unlockReg(tmp_lock);
3611	_ = try func.addInst(.{
3612	.tag = .pseudo_load_tlv,
3613	.data = .{ .reloc = .{
3614	.register = tmp_reg,
3615	.atom_index = @fromBackingInt(@intCast(try func.owner.getSymbolIndex(func))),
3616	.sym_index = @fromBackingInt(@intCast(tlv_sym_index)),
3617	} },
3618	});
3619	try func.genCopy(ptr_ty, dest_mcv, .{ .register = tmp_reg });
3620	}
3621
3622	return func.finishAir(inst, dest_mcv, .{ .none, .none, .none });
3623	}
3624
3625	fn airTry(func: *Func, inst: Air.Inst.Index) !void {
3626	const zcu = func.pt.zcu;
3627	const unwrapped_try = func.air.unwrapTry(inst);
3628	const body = unwrapped_try.else_body;
3629	const operand_ty = func.air.typeOf(unwrapped_try.error_union, &zcu.intern_pool);
3630	const result = try func.genTry(inst, unwrapped_try.error_union, body, operand_ty, false);
3631	return func.finishAir(inst, result, .{ .none, .none, .none });
3632	}
3633
3634	fn genTry(
3635	func: *Func,
3636	inst: Air.Inst.Index,
3637	operand: Air.Inst.Ref,
3638	body: []const Air.Inst.Index,
3639	operand_ty: Type,
3640	operand_is_ptr: bool,
3641	) !MCValue {
3642	_ = operand_is_ptr;
3643
3644	const liveness_cond_br = func.liveness.getCondBr(inst);
3645
3646	const operand_mcv = try func.resolveInst(operand);
3647	const is_err_mcv = try func.isErr(null, operand_ty, operand_mcv);
3648
3649	// A branch to the false section. Uses beq. 1 is the default "true" state.
3650	const reloc = try func.condBr(Type.anyerror, is_err_mcv);
3651
3652	if (func.liveness.operandDies(inst, 0)) {
3653	if (operand.toIndex()) |operand_inst| try func.processDeath(operand_inst);
3654	}
3655
3656	func.scope_generation += 1;
3657	const state = try func.saveState();
3658
3659	for (liveness_cond_br.else_deaths) |death| try func.processDeath(death);
3660	try func.genBody(body);
3661	try func.restoreState(state, &.{}, .{
3662	.emit_instructions = false,
3663	.update_tracking = true,
3664	.resurrect = true,
3665	.close_scope = true,
3666	});
3667
3668	func.performReloc(reloc);
3669
3670	for (liveness_cond_br.then_deaths) |death| try func.processDeath(death);
3671
3672	const result = if (func.liveness.isUnused(inst))
3673	.unreach
3674	else
3675	try func.genUnwrapErrUnionPayloadMir(operand_ty, operand_mcv);
3676
3677	return result;
3678	}
3679
3680	fn airSlicePtr(func: *Func, inst: Air.Inst.Index) !void {
3681	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
3682	const result = result: {
3683	const src_mcv = try func.resolveInst(ty_op.operand);
3684	if (func.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;
3685
3686	const dst_mcv = try func.allocRegOrMem(func.typeOfIndex(inst), inst, true);
3687	const dst_ty = func.typeOfIndex(inst);
3688	try func.genCopy(dst_ty, dst_mcv, src_mcv);
3689	break :result dst_mcv;
3690	};
3691	return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3692	}
3693
3694	fn airSliceLen(func: *Func, inst: Air.Inst.Index) !void {
3695	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
3696	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
3697	const src_mcv = try func.resolveInst(ty_op.operand);
3698	const ty = func.typeOfIndex(inst);
3699
3700	switch (src_mcv) {
3701	.load_frame => |frame_addr| {
3702	const len_mcv: MCValue = .{ .load_frame = .{
3703	.index = frame_addr.index,
3704	.off = frame_addr.off + 8,
3705	} };
3706	if (func.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result len_mcv;
3707
3708	const dst_mcv = try func.allocRegOrMem(ty, inst, true);
3709	try func.genCopy(Type.u64, dst_mcv, len_mcv);
3710	break :result dst_mcv;
3711	},
3712	.register_pair => |pair| {
3713	const len_mcv: MCValue = .{ .register = pair[1] };
3714
3715	if (func.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result len_mcv;
3716
3717	const dst_mcv = try func.allocRegOrMem(ty, inst, true);
3718	try func.genCopy(Type.u64, dst_mcv, len_mcv);
3719	break :result dst_mcv;
3720	},
3721	else => return func.fail("TODO airSliceLen for {}", .{src_mcv}),
3722	}
3723	};
3724	return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3725	}
3726
3727	fn airPtrSliceLenPtr(func: *Func, inst: Air.Inst.Index) !void {
3728	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
3729	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
3730	const src_mcv = try func.resolveInst(ty_op.operand);
3731
3732	const dst_reg, const dst_lock = try func.allocReg(.int);
3733	defer func.register_manager.unlockReg(dst_lock);
3734	const dst_mcv: MCValue = .{ .register = dst_reg };
3735
3736	try func.genCopy(Type.u64, dst_mcv, src_mcv.offset(8));
3737	break :result dst_mcv;
3738	};
3739	return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3740	}
3741
3742	fn airPtrSlicePtrPtr(func: *Func, inst: Air.Inst.Index) !void {
3743	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
3744
3745	const opt_mcv = try func.resolveInst(ty_op.operand);
3746	const dst_mcv = if (func.reuseOperand(inst, ty_op.operand, 0, opt_mcv))
3747	opt_mcv
3748	else
3749	try func.copyToNewRegister(inst, opt_mcv);
3750	return func.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
3751	}
3752
3753	fn airSliceElemVal(func: *Func, inst: Air.Inst.Index) !void {
3754	const pt = func.pt;
3755	const zcu = pt.zcu;
3756	const bin_op = func.air.instructions.items(.data)[@backingInt(inst)].bin_op;
3757
3758	const result: MCValue = result: {
3759	const elem_ty = func.typeOfIndex(inst);
3760	assert(elem_ty.hasRuntimeBits(zcu));
3761
3762	const slice_ty = func.typeOf(bin_op.lhs);
3763	const slice_ptr_field_type = slice_ty.slicePtrFieldType(zcu);
3764	const elem_ptr = try func.genSliceElemPtr(bin_op.lhs, bin_op.rhs);
3765	const dst_mcv = try func.allocRegOrMem(elem_ty, inst, false);
3766	try func.load(dst_mcv, elem_ptr, slice_ptr_field_type);
3767	break :result dst_mcv;
3768	};
3769	return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
3770	}
3771
3772	fn airSliceElemPtr(func: *Func, inst: Air.Inst.Index) !void {
3773	const ty_pl = func.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
3774	const extra = func.air.extraData(Air.Bin, ty_pl.payload).data;
3775	const dst_mcv = try func.genSliceElemPtr(extra.lhs, extra.rhs);
3776	return func.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none });
3777	}
3778
3779	fn genSliceElemPtr(func: *Func, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
3780	const pt = func.pt;
3781	const zcu = pt.zcu;
3782	const slice_ty = func.typeOf(lhs);
3783	const slice_mcv = try func.resolveInst(lhs);
3784	const slice_mcv_lock: ?RegisterLock = switch (slice_mcv) {
3785	.register => |reg| func.register_manager.lockRegAssumeUnused(reg),
3786	else => null,
3787	};
3788	defer if (slice_mcv_lock) |lock| func.register_manager.unlockReg(lock);
3789
3790	const elem_ty = slice_ty.childType(zcu);
3791	const elem_size = elem_ty.abiSize(zcu);
3792
3793	const index_ty = func.typeOf(rhs);
3794	const index_mcv = try func.resolveInst(rhs);
3795	const index_mcv_lock: ?RegisterLock = switch (index_mcv) {
3796	.register => |reg| func.register_manager.lockRegAssumeUnused(reg),
3797	else => null,
3798	};
3799	defer if (index_mcv_lock) |lock| func.register_manager.unlockReg(lock);
3800
3801	const offset_reg = try func.elemOffset(index_ty, index_mcv, elem_size);
3802	const offset_reg_lock = func.register_manager.lockRegAssumeUnused(offset_reg);
3803	defer func.register_manager.unlockReg(offset_reg_lock);
3804
3805	const addr_reg, const addr_lock = try func.allocReg(.int);
3806	defer func.register_manager.unlockReg(addr_lock);
3807	try func.genSetReg(Type.u64, addr_reg, slice_mcv);
3808
3809	_ = try func.addInst(.{
3810	.tag = .add,
3811	.data = .{ .r_type = .{
3812	.rd = addr_reg,
3813	.rs1 = addr_reg,
3814	.rs2 = offset_reg,
3815	} },
3816	});
3817
3818	return .{ .register = addr_reg };
3819	}
3820
3821	fn airArrayElemVal(func: *Func, inst: Air.Inst.Index) !void {
3822	const pt = func.pt;
3823	const zcu = pt.zcu;
3824	const bin_op = func.air.instructions.items(.data)[@backingInt(inst)].bin_op;
3825	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
3826	const result_ty = func.typeOfIndex(inst);
3827
3828	const array_ty = func.typeOf(bin_op.lhs);
3829	const array_mcv = try func.resolveInst(bin_op.lhs);
3830
3831	const index_mcv = try func.resolveInst(bin_op.rhs);
3832	const index_ty = func.typeOf(bin_op.rhs);
3833
3834	const elem_ty = array_ty.childType(zcu);
3835	const elem_abi_size = elem_ty.abiSize(zcu);
3836
3837	const addr_reg, const addr_reg_lock = try func.allocReg(.int);
3838	defer func.register_manager.unlockReg(addr_reg_lock);
3839
3840	switch (array_mcv) {
3841	.register => {
3842	const frame_index = try func.allocFrameIndex(FrameAlloc.initType(array_ty, zcu));
3843	try func.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mcv);
3844	try func.genSetReg(Type.u64, addr_reg, .{ .lea_frame = .{ .index = frame_index } });
3845	},
3846	.load_frame => |frame_addr| {
3847	try func.genSetReg(Type.u64, addr_reg, .{ .lea_frame = frame_addr });
3848	},
3849	else => try func.genSetReg(Type.u64, addr_reg, array_mcv.address()),
3850	}
3851
3852	const dst_mcv = try func.allocRegOrMem(result_ty, inst, false);
3853
3854	if (array_ty.isVector(zcu)) {
3855	// we need to load the vector, vslidedown to get the element we want
3856	// and store that element in a load frame.
3857
3858	const src_reg, const src_lock = try func.allocReg(.vector);
3859	defer func.register_manager.unlockReg(src_lock);
3860
3861	// load the vector into a temporary register
3862	try func.genCopy(array_ty, .{ .register = src_reg }, .{ .indirect = .{ .reg = addr_reg } });
3863
3864	// we need to construct a 1xbitSize vector because of how lane splitting works in RISC-V
3865	const single_ty = try pt.vectorType(.{ .child = elem_ty.toIntern(), .len = 1 });
3866
3867	// we can do a shortcut here where we don't need a vslicedown
3868	// and can just copy to the frame index.
3869	if (!(index_mcv == .immediate and index_mcv.immediate == 0)) {
3870	const index_reg = try func.copyToTmpRegister(Type.u64, index_mcv);
3871
3872	_ = try func.addInst(.{
3873	.tag = .vslidedownvx,
3874	.data = .{ .r_type = .{
3875	.rd = src_reg,
3876	.rs1 = index_reg,
3877	.rs2 = src_reg,
3878	} },
3879	});
3880	}
3881
3882	try func.genCopy(single_ty, dst_mcv, .{ .register = src_reg });
3883	break :result dst_mcv;
3884	}
3885
3886	const offset_reg = try func.elemOffset(index_ty, index_mcv, elem_abi_size);
3887	const offset_lock = func.register_manager.lockRegAssumeUnused(offset_reg);
3888	defer func.register_manager.unlockReg(offset_lock);
3889	_ = try func.addInst(.{
3890	.tag = .add,
3891	.data = .{ .r_type = .{
3892	.rd = addr_reg,
3893	.rs1 = addr_reg,
3894	.rs2 = offset_reg,
3895	} },
3896	});
3897
3898	try func.genCopy(elem_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } });
3899	break :result dst_mcv;
3900	};
3901	return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
3902	}
3903
3904	fn airPtrElemVal(func: *Func, inst: Air.Inst.Index) !void {
3905	const is_volatile = false; // TODO
3906	const pt = func.pt;
3907	const zcu = pt.zcu;
3908	const bin_op = func.air.instructions.items(.data)[@backingInt(inst)].bin_op;
3909	const base_ptr_ty = func.typeOf(bin_op.lhs);
3910
3911	const result: MCValue = if (!is_volatile and func.liveness.isUnused(inst)) .unreach else result: {
3912	const elem_ty = base_ptr_ty.indexableElem(zcu);
3913	assert(elem_ty.hasRuntimeBits(zcu));
3914	const base_ptr_mcv = try func.resolveInst(bin_op.lhs);
3915	const base_ptr_lock: ?RegisterLock = switch (base_ptr_mcv) {
3916	.register => |reg| func.register_manager.lockRegAssumeUnused(reg),
3917	else => null,
3918	};
3919	defer if (base_ptr_lock) |lock| func.register_manager.unlockReg(lock);
3920
3921	const index_mcv = try func.resolveInst(bin_op.rhs);
3922	const index_lock: ?RegisterLock = switch (index_mcv) {
3923	.register => |reg| func.register_manager.lockRegAssumeUnused(reg),
3924	else => null,
3925	};
3926	defer if (index_lock) |lock| func.register_manager.unlockReg(lock);
3927
3928	const elem_ptr_reg, const elem_ptr_lock = if (base_ptr_mcv.isRegister() and
3929	func.liveness.operandDies(inst, 0))
3930	.{ base_ptr_mcv.register, null }
3931	else blk: {
3932	const reg, const lock = try func.allocReg(.int);
3933	try func.genSetReg(base_ptr_ty, reg, base_ptr_mcv);
3934	break :blk .{ reg, lock };
3935	};
3936	defer if (elem_ptr_lock) |lock| func.register_manager.unlockReg(lock);
3937
3938	try func.genBinOp(
3939	.ptr_add,
3940	base_ptr_mcv,
3941	base_ptr_ty,
3942	index_mcv,
3943	Type.u64,
3944	elem_ptr_reg,
3945	);
3946
3947	const dst_mcv = try func.allocRegOrMem(func.typeOfIndex(inst), inst, true);
3948	const dst_lock = switch (dst_mcv) {
3949	.register => |reg| func.register_manager.lockRegAssumeUnused(reg),
3950	else => null,
3951	};
3952	defer if (dst_lock) |lock| func.register_manager.unlockReg(lock);
3953
3954	try func.load(dst_mcv, .{ .register = elem_ptr_reg }, base_ptr_ty);
3955	break :result dst_mcv;
3956	};
3957	return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
3958	}
3959
3960	fn airPtrElemPtr(func: *Func, inst: Air.Inst.Index) !void {
3961	const pt = func.pt;
3962	const zcu = pt.zcu;
3963	const ty_pl = func.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
3964	const extra = func.air.extraData(Air.Bin, ty_pl.payload).data;
3965
3966	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
3967	const elem_ptr_ty = func.typeOfIndex(inst);
3968	const base_ptr_ty = func.typeOf(extra.lhs);
3969
3970	assert(elem_ptr_ty.ptrInfo(zcu).flags.vector_index == .none);
3971
3972	const base_ptr_mcv = try func.resolveInst(extra.lhs);
3973	const base_ptr_lock: ?RegisterLock = switch (base_ptr_mcv) {
3974	.register => |reg| func.register_manager.lockRegAssumeUnused(reg),
3975	else => null,
3976	};
3977	defer if (base_ptr_lock) |lock| func.register_manager.unlockReg(lock);
3978
3979	const index_mcv = try func.resolveInst(extra.rhs);
3980	const index_lock: ?RegisterLock = switch (index_mcv) {
3981	.register => |reg| func.register_manager.lockRegAssumeUnused(reg),
3982	else => null,
3983	};
3984	defer if (index_lock) |lock| func.register_manager.unlockReg(lock);
3985
3986	const result_reg, const result_lock = try func.allocReg(.int);
3987	defer func.register_manager.unlockReg(result_lock);
3988
3989	try func.genBinOp(
3990	.ptr_add,
3991	base_ptr_mcv,
3992	base_ptr_ty,
3993	index_mcv,
3994	Type.u64,
3995	result_reg,
3996	);
3997
3998	break :result MCValue{ .register = result_reg };
3999	};
4000
4001	return func.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
4002	}
4003
4004	fn airSetUnionTag(func: *Func, inst: Air.Inst.Index) !void {
4005	const bin_op = func.air.instructions.items(.data)[@backingInt(inst)].bin_op;
4006	_ = bin_op;
4007	return func.fail("TODO implement airSetUnionTag for {}", .{func.target.cpu.arch});
4008	// return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
4009	}
4010
4011	fn airGetUnionTag(func: *Func, inst: Air.Inst.Index) !void {
4012	const pt = func.pt;
4013	const zcu = pt.zcu;
4014	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
4015
4016	const tag_ty = func.typeOfIndex(inst);
4017	const union_ty = func.typeOf(ty_op.operand);
4018	const layout = union_ty.unionGetLayout(zcu);
4019
4020	if (layout.tag_size == 0) {
4021	return func.finishAir(inst, .none, .{ ty_op.operand, .none, .none });
4022	}
4023
4024	const operand = try func.resolveInst(ty_op.operand);
4025
4026	const frame_mcv = try func.allocRegOrMem(union_ty, null, false);
4027	try func.genCopy(union_ty, frame_mcv, operand);
4028
4029	const tag_abi_size = tag_ty.abiSize(zcu);
4030	const result_reg, const result_lock = try func.allocReg(.int);
4031	defer func.register_manager.unlockReg(result_lock);
4032
4033	switch (frame_mcv) {
4034	.load_frame => {
4035	if (tag_abi_size <= 8) {
4036	const off: i32 = if (layout.tag_align.compare(.lt, layout.payload_align))
4037	@intCast(layout.payload_size)
4038	else
4039	0;
4040
4041	try func.genCopy(
4042	tag_ty,
4043	.{ .register = result_reg },
4044	frame_mcv.offset(off),
4045	);
4046	} else {
4047	return func.fail(
4048	"TODO implement get_union_tag for ABI larger than 8 bytes and operand {}, tag {f}",
4049	.{ frame_mcv, tag_ty.fmt(pt) },
4050	);
4051	}
4052	},
4053	else => return func.fail("TODO: airGetUnionTag {s}", .{@tagName(operand)}),
4054	}
4055
4056	return func.finishAir(inst, .{ .register = result_reg }, .{ ty_op.operand, .none, .none });
4057	}
4058
4059	fn airClz(func: *Func, inst: Air.Inst.Index) !void {
4060	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
4061	const operand = try func.resolveInst(ty_op.operand);
4062	const ty = func.typeOf(ty_op.operand);
4063
4064	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
4065	const src_reg, const src_lock = try func.promoteReg(ty, operand);
4066	defer if (src_lock) |lock| func.register_manager.unlockReg(lock);
4067
4068	const dst_reg: Register = if (func.reuseOperand(
4069	inst,
4070	ty_op.operand,
4071	0,
4072	operand,
4073	) and operand == .register)
4074	operand.register
4075	else
4076	(try func.allocRegOrMem(func.typeOfIndex(inst), inst, true)).register;
4077
4078	const bit_size = ty.bitSize(func.pt.zcu);
4079	if (!math.isPowerOfTwo(bit_size)) try func.truncateRegister(ty, src_reg);
4080
4081	if (bit_size > 64) {
4082	return func.fail("TODO: airClz > 64 bits, found {d}", .{bit_size});
4083	}
4084
4085	_ = try func.addInst(.{
4086	.tag = switch (bit_size) {
4087	32 => .clzw,
4088	else => .clz,
4089	},
4090	.data = .{
4091	.r_type = .{
4092	.rs2 = .zero, // rs2 is 0 filled in the spec
4093	.rs1 = src_reg,
4094	.rd = dst_reg,
4095	},
4096	},
4097	});
4098
4099	if (!(bit_size == 32 or bit_size == 64)) {
4100	_ = try func.addInst(.{
4101	.tag = .addi,
4102	.data = .{ .i_type = .{
4103	.rd = dst_reg,
4104	.rs1 = dst_reg,
4105	.imm12 = Immediate.s(-@as(i12, @intCast(64 - bit_size % 64))),
4106	} },
4107	});
4108	}
4109
4110	break :result .{ .register = dst_reg };
4111	};
4112	return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
4113	}
4114
4115	fn airCtz(func: *Func, inst: Air.Inst.Index) !void {
4116	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
4117	_ = ty_op;
4118	return func.fail("TODO: finish ctz", .{});
4119	}
4120
4121	fn airPopcount(func: *Func, inst: Air.Inst.Index) !void {
4122	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
4123	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
4124	const pt = func.pt;
4125	const zcu = pt.zcu;
4126
4127	const operand = try func.resolveInst(ty_op.operand);
4128	const src_ty = func.typeOf(ty_op.operand);
4129	const operand_reg, const operand_lock = try func.promoteReg(src_ty, operand);
4130	defer if (operand_lock) |lock| func.register_manager.unlockReg(lock);
4131
4132	const dst_reg, const dst_lock = try func.allocReg(.int);
4133	defer func.register_manager.unlockReg(dst_lock);
4134
4135	const bit_size = src_ty.bitSize(zcu);
4136	switch (bit_size) {
4137	32, 64 => {},
4138	1...31, 33...63 => try func.truncateRegister(src_ty, operand_reg),
4139	else => return func.fail("TODO: airPopcount > 64 bits", .{}),
4140	}
4141
4142	_ = try func.addInst(.{
4143	.tag = if (bit_size <= 32) .cpopw else .cpop,
4144	.data = .{
4145	.r_type = .{
4146	.rd = dst_reg,
4147	.rs1 = operand_reg,
4148	.rs2 = @fromBackingInt(@intCast(0b00010)), // this is the cpop funct5
4149	},
4150	},
4151	});
4152
4153	break :result .{ .register = dst_reg };
4154	};
4155	return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
4156	}
4157
4158	fn airAbs(func: *Func, inst: Air.Inst.Index) !void {
4159	const pt = func.pt;
4160	const zcu = pt.zcu;
4161	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
4162	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
4163	const ty = func.typeOf(ty_op.operand);
4164	const scalar_ty = ty.scalarType(zcu);
4165	const operand = try func.resolveInst(ty_op.operand);
4166
4167	switch (scalar_ty.zigTypeTag(zcu)) {
4168	.int => if (ty.zigTypeTag(zcu) == .vector) {
4169	return func.fail("TODO implement airAbs for {f}", .{ty.fmt(pt)});
4170	} else {
4171	const int_info = scalar_ty.intInfo(zcu);
4172	const int_bits = int_info.bits;
4173	switch (int_bits) {
4174	32, 64 => {},
4175	else => return func.fail("TODO: airAbs Int size {d}", .{int_bits}),
4176	}
4177
4178	const return_mcv = try func.copyToNewRegister(inst, operand);
4179	const operand_reg = return_mcv.register;
4180
4181	const temp_reg, const temp_lock = try func.allocReg(.int);
4182	defer func.register_manager.unlockReg(temp_lock);
4183
4184	_ = try func.addInst(.{
4185	.tag = switch (int_bits) {
4186	32 => .sraiw,
4187	64 => .srai,
4188	else => unreachable,
4189	},
4190	.data = .{ .i_type = .{
4191	.rd = temp_reg,
4192	.rs1 = operand_reg,
4193	.imm12 = Immediate.u(int_bits - 1),
4194	} },
4195	});
4196
4197	_ = try func.addInst(.{
4198	.tag = .xor,
4199	.data = .{ .r_type = .{
4200	.rd = operand_reg,
4201	.rs1 = operand_reg,
4202	.rs2 = temp_reg,
4203	} },
4204	});
4205
4206	_ = try func.addInst(.{
4207	.tag = switch (int_bits) {
4208	32 => .subw,
4209	64 => .sub,
4210	else => unreachable,
4211	},
4212	.data = .{ .r_type = .{
4213	.rd = operand_reg,
4214	.rs1 = operand_reg,
4215	.rs2 = temp_reg,
4216	} },
4217	});
4218
4219	break :result return_mcv;
4220	},
4221	.float => {
4222	const float_bits = scalar_ty.floatBits(zcu.getTarget());
4223	const mnem: Mnemonic = switch (float_bits) {
4224	16 => return func.fail("TODO: airAbs 16-bit float", .{}),
4225	32 => .fsgnjxs,
4226	64 => .fsgnjxd,
4227	80 => return func.fail("TODO: airAbs 80-bit float", .{}),
4228	128 => return func.fail("TODO: airAbs 128-bit float", .{}),
4229	else => unreachable,
4230	};
4231
4232	const return_mcv = try func.copyToNewRegister(inst, operand);
4233	const operand_reg = return_mcv.register;
4234
4235	assert(operand_reg.class() == .float);
4236
4237	_ = try func.addInst(.{
4238	.tag = mnem,
4239	.data = .{
4240	.r_type = .{
4241	.rd = operand_reg,
4242	.rs1 = operand_reg,
4243	.rs2 = operand_reg,
4244	},
4245	},
4246	});
4247
4248	break :result return_mcv;
4249	},
4250	else => return func.fail("TODO: implement airAbs {f}", .{scalar_ty.fmt(pt)}),
4251	}
4252
4253	break :result .unreach;
4254	};
4255	return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
4256	}
4257
4258	fn airByteSwap(func: *Func, inst: Air.Inst.Index) !void {
4259	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
4260	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
4261	const pt = func.pt;
4262	const zcu = pt.zcu;
4263	const ty = func.typeOf(ty_op.operand);
4264	const operand = try func.resolveInst(ty_op.operand);
4265
4266	switch (ty.zigTypeTag(zcu)) {
4267	.int => {
4268	const int_bits = ty.intInfo(zcu).bits;
4269
4270	// bytes are no-op
4271	if (int_bits == 8 and func.reuseOperand(inst, ty_op.operand, 0, operand)) {
4272	return func.finishAir(inst, operand, .{ ty_op.operand, .none, .none });
4273	}
4274
4275	const dest_mcv = try func.copyToNewRegister(inst, operand);
4276	const dest_reg = dest_mcv.register;
4277
4278	switch (int_bits) {
4279	16 => {
4280	const temp_reg, const temp_lock = try func.allocReg(.int);
4281	defer func.register_manager.unlockReg(temp_lock);
4282
4283	_ = try func.addInst(.{
4284	.tag = .srli,
4285	.data = .{ .i_type = .{
4286	.imm12 = Immediate.s(8),
4287	.rd = temp_reg,
4288	.rs1 = dest_reg,
4289	} },
4290	});
4291
4292	_ = try func.addInst(.{
4293	.tag = .slli,
4294	.data = .{ .i_type = .{
4295	.imm12 = Immediate.s(8),
4296	.rd = dest_reg,
4297	.rs1 = dest_reg,
4298	} },
4299	});
4300	_ = try func.addInst(.{
4301	.tag = .@"or",
4302	.data = .{ .r_type = .{
4303	.rd = dest_reg,
4304	.rs1 = dest_reg,
4305	.rs2 = temp_reg,
4306	} },
4307	});
4308	},
4309	else => return func.fail("TODO: {d} bits for airByteSwap", .{int_bits}),
4310	}
4311
4312	break :result dest_mcv;
4313	},
4314	else => return func.fail("TODO: airByteSwap {f}", .{ty.fmt(pt)}),
4315	}
4316	};
4317	return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
4318	}
4319
4320	fn airBitReverse(func: *Func, inst: Air.Inst.Index) !void {
4321	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
4322	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airBitReverse for {}", .{func.target.cpu.arch});
4323	return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
4324	}
4325
4326	fn airUnaryMath(func: *Func, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
4327	const pt = func.pt;
4328	const zcu = pt.zcu;
4329	const un_op = func.air.instructions.items(.data)[@backingInt(inst)].un_op;
4330	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
4331	const ty = func.typeOf(un_op);
4332
4333	const operand = try func.resolveInst(un_op);
4334	const operand_bit_size = ty.bitSize(zcu);
4335
4336	if (!math.isPowerOfTwo(operand_bit_size))
4337	return func.fail("TODO: airUnaryMath non-pow 2", .{});
4338
4339	const operand_reg, const operand_lock = try func.promoteReg(ty, operand);
4340	defer if (operand_lock) |lock| func.register_manager.unlockReg(lock);
4341
4342	const dst_class = func.typeRegClass(ty);
4343	const dst_reg, const dst_lock = try func.allocReg(dst_class);
4344	defer func.register_manager.unlockReg(dst_lock);
4345
4346	switch (ty.zigTypeTag(zcu)) {
4347	.float => {
4348	assert(dst_class == .float);
4349
4350	switch (operand_bit_size) {
4351	16, 80, 128 => return func.fail("TODO: airUnaryMath Float bit-size {}", .{operand_bit_size}),
4352	32, 64 => {},
4353	else => unreachable,
4354	}
4355
4356	switch (tag) {
4357	.sqrt => {
4358	_ = try func.addInst(.{
4359	.tag = if (operand_bit_size == 64) .fsqrtd else .fsqrts,
4360	.data = .{
4361	.r_type = .{
4362	.rd = dst_reg,
4363	.rs1 = operand_reg,
4364	.rs2 = .f0, // unused, spec says it's 0
4365	},
4366	},
4367	});
4368	},
4369
4370	else => return func.fail("TODO: airUnaryMath Float {s}", .{@tagName(tag)}),
4371	}
4372	},
4373	.int => {
4374	assert(dst_class == .int);
4375
4376	switch (tag) {
4377	else => return func.fail("TODO: airUnaryMath Float {s}", .{@tagName(tag)}),
4378	}
4379	},
4380	else => return func.fail("TODO: airUnaryMath ty: {f}", .{ty.fmt(pt)}),
4381	}
4382
4383	break :result MCValue{ .register = dst_reg };
4384	};
4385
4386	return func.finishAir(inst, result, .{ un_op, .none, .none });
4387	}
4388
4389	fn reuseOperand(
4390	func: *Func,
4391	inst: Air.Inst.Index,
4392	operand: Air.Inst.Ref,
4393	op_index: Air.Liveness.OperandInt,
4394	mcv: MCValue,
4395	) bool {
4396	return func.reuseOperandAdvanced(inst, operand, op_index, mcv, inst);
4397	}
4398
4399	fn reuseOperandAdvanced(
4400	func: *Func,
4401	inst: Air.Inst.Index,
4402	operand: Air.Inst.Ref,
4403	op_index: Air.Liveness.OperandInt,
4404	mcv: MCValue,
4405	maybe_tracked_inst: ?Air.Inst.Index,
4406	) bool {
4407	if (!func.liveness.operandDies(inst, op_index))
4408	return false;
4409
4410	switch (mcv) {
4411	.register,
4412	.register_pair,
4413	=> for (mcv.getRegs()) |reg| {
4414	// If it's in the registers table, need to associate the register(s) with the
4415	// new instruction.
4416	if (maybe_tracked_inst) |tracked_inst| {
4417	if (!func.register_manager.isRegFree(reg)) {
4418	if (RegisterManager.indexOfRegIntoTracked(reg)) |index| {
4419	func.register_manager.registers[index] = tracked_inst;
4420	}
4421	}
4422	} else func.register_manager.freeReg(reg);
4423	},
4424	.load_frame => |frame_addr| if (frame_addr.index.isNamed()) return false,
4425	else => return false,
4426	}
4427
4428	// Prevent the operand deaths processing code from deallocating it.
4429	func.reused_operands.set(op_index);
4430	const op_inst = operand.toIndex().?;
4431	func.getResolvedInstValue(op_inst).reuse(func, maybe_tracked_inst, op_inst);
4432
4433	return true;
4434	}
4435
4436	fn airLoad(func: *Func, inst: Air.Inst.Index) !void {
4437	const pt = func.pt;
4438	const zcu = pt.zcu;
4439	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
4440	const elem_ty = func.typeOfIndex(inst);
4441
4442	const result: MCValue = result: {
4443	if (!elem_ty.hasRuntimeBits(zcu))
4444	break :result .none;
4445
4446	const ptr = try func.resolveInst(ty_op.operand);
4447	const is_volatile = func.typeOf(ty_op.operand).isVolatilePtr(zcu);
4448	if (func.liveness.isUnused(inst) and !is_volatile)
4449	break :result .unreach;
4450
4451	const elem_size = elem_ty.abiSize(zcu);
4452
4453	const dst_mcv: MCValue = blk: {
4454	// The MCValue that holds the pointer can be re-used as the value.
4455	// - "ptr" is 8 bytes, and if the element is more than that, we cannot reuse it.
4456	//
4457	// - "ptr" will be stored in an integer register, so the type that we're gonna
4458	// load into it must also be a type that can be inside of an integer register
4459	if (elem_size <= 8 and
4460	(if (ptr == .register) func.typeRegClass(elem_ty) == ptr.register.class() else true) and
4461	func.reuseOperand(inst, ty_op.operand, 0, ptr))
4462	{
4463	break :blk ptr;
4464	} else {
4465	break :blk try func.allocRegOrMem(elem_ty, inst, true);
4466	}
4467	};
4468
4469	try func.load(dst_mcv, ptr, func.typeOf(ty_op.operand));
4470	break :result dst_mcv;
4471	};
4472	return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
4473	}
4474
4475	fn load(func: *Func, dst_mcv: MCValue, ptr_mcv: MCValue, ptr_ty: Type) InnerError!void {
4476	const pt = func.pt;
4477	const zcu = pt.zcu;
4478	const dst_ty = ptr_ty.childType(zcu);
4479
4480	log.debug("loading {}:{f} into {}", .{ ptr_mcv, ptr_ty.fmt(pt), dst_mcv });
4481
4482	switch (ptr_mcv) {
4483	.none,
4484	.undef,
4485	.unreach,
4486	.dead,
4487	.register_pair,
4488	.reserved_frame,
4489	=> unreachable, // not a valid pointer
4490
4491	.immediate,
4492	.register,
4493	.register_offset,
4494	.lea_frame,
4495	.lea_symbol,
4496	=> try func.genCopy(dst_ty, dst_mcv, ptr_mcv.deref()),
4497
4498	.memory,
4499	.indirect,
4500	.load_symbol,
4501	.load_frame,
4502	=> {
4503	const addr_reg = try func.copyToTmpRegister(ptr_ty, ptr_mcv);
4504	const addr_lock = func.register_manager.lockRegAssumeUnused(addr_reg);
4505	defer func.register_manager.unlockReg(addr_lock);
4506
4507	try func.genCopy(dst_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } });
4508	},
4509	.air_ref => |ptr_ref| try func.load(dst_mcv, try func.resolveInst(ptr_ref), ptr_ty),
4510	}
4511	}
4512
4513	fn airStore(func: *Func, inst: Air.Inst.Index, safety: bool) !void {
4514	if (safety) {
4515	// TODO if the value is undef, write 0xaa bytes to dest
4516	} else {
4517	// TODO if the value is undef, don't lower this instruction
4518	}
4519	const bin_op = func.air.instructions.items(.data)[@backingInt(inst)].bin_op;
4520	const ptr = try func.resolveInst(bin_op.lhs);
4521	const value = try func.resolveInst(bin_op.rhs);
4522	const ptr_ty = func.typeOf(bin_op.lhs);
4523
4524	try func.store(ptr, value, ptr_ty);
4525
4526	return func.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });
4527	}
4528
4529	/// Loads `value` into the "payload" of `pointer`.
4530	fn store(func: *Func, ptr_mcv: MCValue, src_mcv: MCValue, ptr_ty: Type) !void {
4531	const zcu = func.pt.zcu;
4532	const src_ty = ptr_ty.childType(zcu);
4533	log.debug("storing {}:{f} in {}:{f}", .{ src_mcv, src_ty.fmt(func.pt), ptr_mcv, ptr_ty.fmt(func.pt) });
4534
4535	switch (ptr_mcv) {
4536	.none => unreachable,
4537	.undef => unreachable,
4538	.unreach => unreachable,
4539	.dead => unreachable,
4540	.register_pair => unreachable,
4541	.reserved_frame => unreachable,
4542
4543	.immediate,
4544	.register,
4545	.register_offset,
4546	.lea_symbol,
4547	.lea_frame,
4548	=> try func.genCopy(src_ty, ptr_mcv.deref(), src_mcv),
4549
4550	.memory,
4551	.indirect,
4552	.load_symbol,
4553	.load_frame,
4554	=> {
4555	const addr_reg = try func.copyToTmpRegister(ptr_ty, ptr_mcv);
4556	const addr_lock = func.register_manager.lockRegAssumeUnused(addr_reg);
4557	defer func.register_manager.unlockReg(addr_lock);
4558
4559	try func.genCopy(src_ty, .{ .indirect = .{ .reg = addr_reg } }, src_mcv);
4560	},
4561	.air_ref => |ptr_ref| try func.store(try func.resolveInst(ptr_ref), src_mcv, ptr_ty),
4562	}
4563	}
4564
4565	fn airStructFieldPtr(func: *Func, inst: Air.Inst.Index) !void {
4566	const ty_pl = func.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
4567	const extra = func.air.extraData(Air.StructField, ty_pl.payload).data;
4568	const result = try func.structFieldPtr(inst, extra.struct_operand, extra.field_index);
4569	return func.finishAir(inst, result, .{ extra.struct_operand, .none, .none });
4570	}
4571
4572	fn airStructFieldPtrIndex(func: *Func, inst: Air.Inst.Index, index: u8) !void {
4573	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
4574	const result = try func.structFieldPtr(inst, ty_op.operand, index);
4575	return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
4576	}
4577
4578	fn structFieldPtr(func: *Func, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32) !MCValue {
4579	const pt = func.pt;
4580	const zcu = pt.zcu;
4581	const ptr_field_ty = func.typeOfIndex(inst);
4582	const ptr_container_ty = func.typeOf(operand);
4583	const container_ty = ptr_container_ty.childType(zcu);
4584
4585	const field_offset: i32 = switch (container_ty.containerLayout(zcu)) {
4586	.auto, .@"extern" => @intCast(container_ty.structFieldOffset(index, zcu)),
4587	.@"packed" => @divExact(@as(i32, ptr_container_ty.ptrInfo(zcu).packed_offset.bit_offset) +
4588	(if (zcu.typeToStruct(container_ty)) |struct_obj| zcu.structPackedFieldBitOffset(struct_obj, index) else 0) -
4589	ptr_field_ty.ptrInfo(zcu).packed_offset.bit_offset, 8),
4590	};
4591
4592	const src_mcv = try func.resolveInst(operand);
4593	const dst_mcv = if (switch (src_mcv) {
4594	.immediate, .lea_frame => true,
4595	.register, .register_offset => func.reuseOperand(inst, operand, 0, src_mcv),
4596	else => false,
4597	}) src_mcv else try func.copyToNewRegister(inst, src_mcv);
4598	return dst_mcv.offset(field_offset);
4599	}
4600
4601	fn airAggFieldVal(func: *Func, inst: Air.Inst.Index) !void {
4602	const pt = func.pt;
4603	const zcu = pt.zcu;
4604
4605	const ty_pl = func.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
4606	const extra = func.air.extraData(Air.StructField, ty_pl.payload).data;
4607	const operand = extra.struct_operand;
4608	const index = extra.field_index;
4609
4610	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
4611	const src_mcv = try func.resolveInst(operand);
4612	const struct_ty = func.typeOf(operand);
4613	const field_ty = struct_ty.fieldType(index, zcu);
4614	assert(field_ty.hasRuntimeBits(zcu));
4615
4616	const field_off: u32 = switch (struct_ty.containerLayout(zcu)) {
4617	.auto, .@"extern" => @intCast(struct_ty.structFieldOffset(index, zcu) * 8),
4618	.@"packed" => if (zcu.typeToStruct(struct_ty)) |struct_type|
4619	zcu.structPackedFieldBitOffset(struct_type, index)
4620	else
4621	0,
4622	};
4623
4624	switch (src_mcv) {
4625	.dead, .unreach => unreachable,
4626	.register => |src_reg| {
4627	const src_reg_lock = func.register_manager.lockRegAssumeUnused(src_reg);
4628	defer func.register_manager.unlockReg(src_reg_lock);
4629
4630	const dst_reg = if (field_off == 0)
4631	(try func.copyToNewRegister(inst, src_mcv)).register
4632	else
4633	try func.copyToTmpRegister(Type.u64, .{ .register = src_reg });
4634
4635	const dst_mcv: MCValue = .{ .register = dst_reg };
4636	const dst_lock = func.register_manager.lockReg(dst_reg);
4637	defer if (dst_lock) |lock| func.register_manager.unlockReg(lock);
4638
4639	if (field_off > 0) {
4640	_ = try func.addInst(.{
4641	.tag = .srli,
4642	.data = .{ .i_type = .{
4643	.imm12 = Immediate.u(@intCast(field_off)),
4644	.rd = dst_reg,
4645	.rs1 = dst_reg,
4646	} },
4647	});
4648	}
4649
4650	if (field_off == 0) {
4651	try func.truncateRegister(field_ty, dst_reg);
4652	}
4653
4654	break :result if (field_off == 0) dst_mcv else try func.copyToNewRegister(inst, dst_mcv);
4655	},
4656	.load_frame => {
4657	const field_abi_size: u32 = @intCast(field_ty.abiSize(zcu));
4658	if (field_off % 8 == 0) {
4659	const field_byte_off = @divExact(field_off, 8);
4660	const off_mcv = src_mcv.address().offset(@intCast(field_byte_off)).deref();
4661	const field_bit_size = field_ty.bitSize(zcu);
4662
4663	if (field_abi_size <= 8) {
4664	const int_ty = try pt.intType(
4665	if (field_ty.isAbiInt(zcu)) field_ty.intInfo(zcu).signedness else .unsigned,
4666	@intCast(field_bit_size),
4667	);
4668
4669	const dst_reg, const dst_lock = try func.allocReg(.int);
4670	const dst_mcv = MCValue{ .register = dst_reg };
4671	defer func.register_manager.unlockReg(dst_lock);
4672
4673	try func.genCopy(int_ty, dst_mcv, off_mcv);
4674	break :result try func.copyToNewRegister(inst, dst_mcv);
4675	}
4676
4677	const container_abi_size: u32 = @intCast(struct_ty.abiSize(zcu));
4678	const dst_mcv = if (field_byte_off + field_abi_size <= container_abi_size and
4679	func.reuseOperand(inst, operand, 0, src_mcv))
4680	off_mcv
4681	else dst: {
4682	const dst_mcv = try func.allocRegOrMem(func.typeOfIndex(inst), inst, true);
4683	try func.genCopy(field_ty, dst_mcv, off_mcv);
4684	break :dst dst_mcv;
4685	};
4686	if (field_abi_size * 8 > field_bit_size and dst_mcv.isMemory()) {
4687	const tmp_reg, const tmp_lock = try func.allocReg(.int);
4688	defer func.register_manager.unlockReg(tmp_lock);
4689
4690	const hi_mcv =
4691	dst_mcv.address().offset(@intCast(field_bit_size / 64 * 8)).deref();
4692	try func.genSetReg(Type.u64, tmp_reg, hi_mcv);
4693	try func.genCopy(Type.u64, hi_mcv, .{ .register = tmp_reg });
4694	}
4695	break :result dst_mcv;
4696	}
4697
4698	return func.fail("TODO: airAggFieldVal load_frame field_off non multiple of 8", .{});
4699	},
4700	else => return func.fail("TODO: airStructField {s}", .{@tagName(src_mcv)}),
4701	}
4702	};
4703
4704	return func.finishAir(inst, result, .{ extra.struct_operand, .none, .none });
4705	}
4706
4707	fn airFieldParentPtr(func: *Func, inst: Air.Inst.Index) !void {
4708	_ = inst;
4709	return func.fail("TODO implement codegen airFieldParentPtr", .{});
4710	}
4711
4712	fn genArgDbgInfo(func: *const Func, name: []const u8, ty: Type, mcv: MCValue) InnerError!void {
4713	assert(!func.mod.strip);
4714
4715	// TODO: Add a pseudo-instruction or something to defer this work until Emit.
4716	// We aren't allowed to interact with linker state here.
4717	if (true) return;
4718	switch (func.debug_output) {
4719	.dwarf => |dw| switch (mcv) {
4720	.register => |reg| dw.genLocalDebugInfo(
4721	.local_arg,
4722	name,
4723	ty,
4724	.{ .reg = reg.dwarfNum() },
4725	) catch |err| return func.fail("failed to generate debug info: {s}", .{@errorName(err)}),
4726	.load_frame => {},
4727	else => {},
4728	},
4729	.plan9 => {},
4730	.none => {},
4731	}
4732	}
4733
4734	fn airArg(func: *Func, inst: Air.Inst.Index) InnerError!void {
4735	const zcu = func.pt.zcu;
4736
4737	var arg_index = func.arg_index;
4738
4739	// we skip over args that have no bits
4740	while (func.args[arg_index] == .none) arg_index += 1;
4741	func.arg_index = arg_index + 1;
4742
4743	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
4744	const src_mcv = func.args[arg_index];
4745	const arg_ty = func.typeOfIndex(inst);
4746
4747	const dst_mcv = try func.allocRegOrMem(arg_ty, inst, false);
4748
4749	log.debug("airArg {} -> {}", .{ src_mcv, dst_mcv });
4750
4751	try func.genCopy(arg_ty, dst_mcv, src_mcv);
4752
4753	const arg = func.air.instructions.items(.data)[@backingInt(inst)].arg;
4754	// can delete `func.func_index` if this logic is moved to emit
4755	const func_zir = zcu.funcInfo(func.func_index).zir_body_inst.resolveFull(&zcu.intern_pool).?;
4756	const file = zcu.fileByIndex(func_zir.file);
4757	const zir = &file.zir.?;
4758	const name = zir.nullTerminatedString(zir.getParamName(zir.getParamBody(func_zir.inst)[arg.zir_param_index]).?);
4759
4760	try func.genArgDbgInfo(name, arg_ty, src_mcv);
4761	break :result dst_mcv;
4762	};
4763
4764	return func.finishAir(inst, result, .{ .none, .none, .none });
4765	}
4766
4767	fn airTrap(func: *Func) !void {
4768	_ = try func.addInst(.{
4769	.tag = .unimp,
4770	.data = .none,
4771	});
4772	return func.finishAirBookkeeping();
4773	}
4774
4775	fn airBreakpoint(func: *Func) !void {
4776	_ = try func.addInst(.{
4777	.tag = .ebreak,
4778	.data = .none,
4779	});
4780	return func.finishAirBookkeeping();
4781	}
4782
4783	fn airRetAddr(func: *Func, inst: Air.Inst.Index) !void {
4784	const dst_mcv = try func.allocRegOrMem(func.typeOfIndex(inst), inst, true);
4785	try func.genCopy(Type.u64, dst_mcv, .{ .load_frame = .{ .index = .ret_addr } });
4786	return func.finishAir(inst, dst_mcv, .{ .none, .none, .none });
4787	}
4788
4789	fn airFrameAddress(func: *Func, inst: Air.Inst.Index) !void {
4790	const dst_mcv = try func.allocRegOrMem(func.typeOfIndex(inst), inst, true);
4791	try func.genCopy(Type.u64, dst_mcv, .{ .lea_frame = .{ .index = .base_ptr } });
4792	return func.finishAir(inst, dst_mcv, .{ .none, .none, .none });
4793	}
4794
4795	fn airCall(func: *Func, inst: Air.Inst.Index, modifier: std.lang.CallModifier) !void {
4796	if (modifier == .always_tail) return func.fail("TODO implement tail calls for riscv64", .{});
4797	const call = func.air.unwrapCall(inst);
4798	const arg_refs = call.args;
4799
4800	const expected_num_args = 8;
4801	const ExpectedContents = extern struct {
4802	vals: [expected_num_args][@sizeOf(MCValue)]u8 align(@alignOf(MCValue)),
4803	};
4804	var bfa_buf: ExpectedContents = undefined;
4805	var bfa: std.heap.BufferFirstAllocator = .init(@ptrCast(&bfa_buf), func.gpa);
4806	const allocator = bfa.allocator();
4807
4808	const arg_tys = try allocator.alloc(Type, arg_refs.len);
4809	defer allocator.free(arg_tys);
4810	for (arg_tys, arg_refs) |*arg_ty, arg_ref| arg_ty.* = func.typeOf(arg_ref);
4811
4812	const arg_vals = try allocator.alloc(MCValue, arg_refs.len);
4813	defer allocator.free(arg_vals);
4814	for (arg_vals, arg_refs) |*arg_val, arg_ref| arg_val.* = .{ .air_ref = arg_ref };
4815
4816	const call_ret = try func.genCall(.{ .air = call.callee }, arg_tys, arg_vals);
4817
4818	var bt = func.liveness.iterateBigTomb(inst);
4819	try func.feed(&bt, call.callee);
4820	for (arg_refs) |arg_ref| try func.feed(&bt, arg_ref);
4821
4822	const result = if (func.liveness.isUnused(inst)) .unreach else call_ret;
4823	return func.finishAirResult(inst, result);
4824	}
4825
4826	fn genCall(
4827	func: *Func,
4828	info: union(enum) {
4829	air: Air.Inst.Ref,
4830	lib: struct {
4831	return_type: InternPool.Index,
4832	param_types: []const InternPool.Index,
4833	lib: ?[]const u8 = null,
4834	callee: []const u8,
4835	},
4836	},
4837	arg_tys: []const Type,
4838	args: []const MCValue,
4839	) !MCValue {
4840	const pt = func.pt;
4841	const zcu = pt.zcu;
4842
4843	const fn_ty = switch (info) {
4844	.air => |callee| fn_info: {
4845	const callee_ty = func.typeOf(callee);
4846	break :fn_info switch (callee_ty.zigTypeTag(zcu)) {
4847	.@"fn" => callee_ty,
4848	.pointer => callee_ty.childType(zcu),
4849	else => unreachable,
4850	};
4851	},
4852	.lib => |lib| try pt.funcType(.{
4853	.param_types = lib.param_types,
4854	.return_type = lib.return_type,
4855	.cc = func.target.cCallingConvention().?,
4856	}),
4857	};
4858
4859	const fn_info = zcu.typeToFunc(fn_ty).?;
4860
4861	const allocator = func.gpa;
4862
4863	const var_args = try allocator.alloc(Type, args.len - fn_info.param_types.len);
4864	defer allocator.free(var_args);
4865	for (var_args, arg_tys[fn_info.param_types.len..]) |*var_arg, arg_ty| var_arg.* = arg_ty;
4866
4867	var call_info = try func.resolveCallingConventionValues(fn_info, var_args);
4868	defer call_info.deinit(func);
4869
4870	// We need a properly aligned and sized call frame to be able to call this function.
4871	{
4872	const needed_call_frame = FrameAlloc.init(.{
4873	.size = call_info.stack_byte_count,
4874	.alignment = call_info.stack_align,
4875	});
4876	const frame_allocs_slice = func.frame_allocs.slice();
4877	const stack_frame_size =
4878	&frame_allocs_slice.items(.abi_size)[@backingInt(FrameIndex.call_frame)];
4879	stack_frame_size.* = @max(stack_frame_size.*, needed_call_frame.abi_size);
4880	const stack_frame_align =
4881	&frame_allocs_slice.items(.abi_align)[@backingInt(FrameIndex.call_frame)];
4882	stack_frame_align.* = stack_frame_align.max(needed_call_frame.abi_align);
4883	}
4884
4885	var reg_locks = std.array_list.Managed(?RegisterLock).init(allocator);
4886	defer reg_locks.deinit();
4887	try reg_locks.ensureTotalCapacity(8);
4888	defer for (reg_locks.items) |reg_lock| if (reg_lock) |lock| func.register_manager.unlockReg(lock);
4889
4890	const frame_indices = try allocator.alloc(FrameIndex, args.len);
4891	defer allocator.free(frame_indices);
4892
4893	switch (call_info.return_value.long) {
4894	.none, .unreach => {},
4895	.indirect => |reg_off| try func.register_manager.getReg(reg_off.reg, null),
4896	else => unreachable,
4897	}
4898	for (call_info.args, args, arg_tys, frame_indices) |dst_arg, src_arg, arg_ty, *frame_index| {
4899	switch (dst_arg) {
4900	.none => {},
4901	.register => |reg| {
4902	try func.register_manager.getReg(reg, null);
4903	try reg_locks.append(func.register_manager.lockReg(reg));
4904	},
4905	.register_pair => |regs| {
4906	for (regs) |reg| try func.register_manager.getReg(reg, null);
4907	try reg_locks.appendSlice(&func.register_manager.lockRegs(2, regs));
4908	},
4909	.indirect => |reg_off| {
4910	frame_index.* = try func.allocFrameIndex(FrameAlloc.initType(arg_ty, zcu));
4911	try func.genSetMem(.{ .frame = frame_index.* }, 0, arg_ty, src_arg);
4912	try func.register_manager.getReg(reg_off.reg, null);
4913	try reg_locks.append(func.register_manager.lockReg(reg_off.reg));
4914	},
4915	else => return func.fail("TODO: genCall set arg {s}", .{@tagName(dst_arg)}),
4916	}
4917	}
4918
4919	switch (call_info.return_value.long) {
4920	.none, .unreach => {},
4921	.indirect => |reg_off| {
4922	const ret_ty = Type.fromInterned(fn_info.return_type);
4923	const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(ret_ty, zcu));
4924	try func.genSetReg(Type.u64, reg_off.reg, .{
4925	.lea_frame = .{ .index = frame_index, .off = -reg_off.off },
4926	});
4927	call_info.return_value.short = .{ .load_frame = .{ .index = frame_index } };
4928	try reg_locks.append(func.register_manager.lockReg(reg_off.reg));
4929	},
4930	else => unreachable,
4931	}
4932
4933	for (call_info.args, arg_tys, args, frame_indices) |dst_arg, arg_ty, src_arg, frame_index| {
4934	switch (dst_arg) {
4935	.none, .load_frame => {},
4936	.register_pair => try func.genCopy(arg_ty, dst_arg, src_arg),
4937	.register => |dst_reg| try func.genSetReg(
4938	arg_ty,
4939	dst_reg,
4940	src_arg,
4941	),
4942	.indirect => |reg_off| try func.genSetReg(Type.u64, reg_off.reg, .{
4943	.lea_frame = .{ .index = frame_index, .off = -reg_off.off },
4944	}),
4945	else => return func.fail("TODO: genCall actual set {s}", .{@tagName(dst_arg)}),
4946	}
4947	}
4948
4949	// Due to incremental compilation, how function calls are generated depends
4950	// on linking.
4951	switch (info) {
4952	.air => |callee| {
4953	if (callee.toInterned()) |func_ip_index| {
4954	const func_key = zcu.intern_pool.indexToKey(func_ip_index);
4955	switch (switch (func_key) {
4956	else => func_key,
4957	.ptr => |ptr| if (ptr.byte_offset == 0) switch (ptr.base_addr) {
4958	.nav => |nav| zcu.intern_pool.indexToKey(zcu.navValue(nav).toIntern()),
4959	else => func_key,
4960	} else func_key,
4961	}) {
4962	.func => |func_val| {
4963	if (func.bin_file.cast(.elf)) |elf_file| {
4964	const zo = elf_file.zigObjectPtr().?;
4965	const sym_index: link.File.SymbolId = @fromBackingInt(@intCast(try zo.getOrCreateMetadataForNav(zcu, func_val.owner_nav)));
4966
4967	if (func.mod.pic) {
4968	return func.fail("TODO: genCall pic", .{});
4969	} else {
4970	try func.genSetReg(Type.u64, .ra, .{ .lea_symbol = .{ .sym = sym_index } });
4971	_ = try func.addInst(.{
4972	.tag = .jalr,
4973	.data = .{ .i_type = .{
4974	.rd = .ra,
4975	.rs1 = .ra,
4976	.imm12 = Immediate.s(0),
4977	} },
4978	});
4979	}
4980	} else unreachable; // not a valid riscv64 format
4981	},
4982	.@"extern" => |@"extern"| {
4983	const lib_name = @"extern".lib_name.toSlice(&zcu.intern_pool);
4984	const name = @"extern".name.toSlice(&zcu.intern_pool);
4985	const atom_index: link.File.AtomId = @fromBackingInt(@intCast(try func.owner.getSymbolIndex(func)));
4986
4987	const elf_file = func.bin_file.cast(.elf).?;
4988	_ = try func.addInst(.{
4989	.tag = .pseudo_extern_fn_reloc,
4990	.data = .{ .reloc = .{
4991	.register = .ra,
4992	.atom_index = atom_index,
4993	.sym_index = @fromBackingInt(@intCast(try elf_file.getGlobalSymbol(name, lib_name))),
4994	} },
4995	});
4996	},
4997	else => return func.fail("TODO implement calling bitcasted functions", .{}),
4998	}
4999	} else {
5000	assert(func.typeOf(callee).zigTypeTag(zcu) == .pointer);
5001	const addr_reg, const addr_lock = try func.allocReg(.int);
5002	defer func.register_manager.unlockReg(addr_lock);
5003	try func.genSetReg(Type.u64, addr_reg, .{ .air_ref = callee });
5004
5005	_ = try func.addInst(.{
5006	.tag = .jalr,
5007	.data = .{ .i_type = .{
5008	.rd = .ra,
5009	.rs1 = addr_reg,
5010	.imm12 = Immediate.s(0),
5011	} },
5012	});
5013	}
5014	},
5015	.lib => return func.fail("TODO: lib func calls", .{}),
5016	}
5017
5018	// reset the vector settings as they might have changed in the function
5019	func.avl = null;
5020	func.vtype = null;
5021
5022	return call_info.return_value.short;
5023	}
5024
5025	fn airRet(func: *Func, inst: Air.Inst.Index, safety: bool) !void {
5026	const pt = func.pt;
5027	const zcu = pt.zcu;
5028	const un_op = func.air.instructions.items(.data)[@backingInt(inst)].un_op;
5029
5030	if (safety) {
5031	// safe
5032	} else {
5033	// not safe
5034	}
5035
5036	const ret_ty = func.fn_type.fnReturnType(zcu);
5037	switch (func.ret_mcv.short) {
5038	.none => {},
5039	.register,
5040	.register_pair,
5041	=> {
5042	if (ret_ty.isVector(zcu)) {
5043	const bit_size = ret_ty.bitSize(zcu);
5044
5045	// set the vtype to hold the entire vector's contents in a single element
5046	try func.setVl(.zero, 0, .{
5047	.vsew = switch (bit_size) {
5048	8 => .@"8",
5049	16 => .@"16",
5050	32 => .@"32",
5051	64 => .@"64",
5052	else => unreachable,
5053	},
5054	.vlmul = .m1,
5055	.vma = true,
5056	.vta = true,
5057	});
5058	}
5059
5060	try func.genCopy(ret_ty, func.ret_mcv.short, .{ .air_ref = un_op });
5061	},
5062	.indirect => |reg_off| {
5063	try func.register_manager.getReg(reg_off.reg, null);
5064	const lock = func.register_manager.lockRegAssumeUnused(reg_off.reg);
5065	defer func.register_manager.unlockReg(lock);
5066
5067	try func.genSetReg(Type.u64, reg_off.reg, func.ret_mcv.long);
5068	try func.genSetMem(
5069	.{ .reg = reg_off.reg },
5070	reg_off.off,
5071	ret_ty,
5072	.{ .air_ref = un_op },
5073	);
5074	},
5075	else => unreachable,
5076	}
5077
5078	func.ret_mcv.liveOut(func, inst);
5079	try func.finishAir(inst, .unreach, .{ un_op, .none, .none });
5080
5081	// Just add space for an instruction, reloced this later
5082	const index = try func.addInst(.{
5083	.tag = .pseudo_j,
5084	.data = .{ .j_type = .{
5085	.rd = .zero,
5086	.inst = undefined,
5087	} },
5088	});
5089
5090	try func.exitlude_jump_relocs.append(func.gpa, index);
5091	}
5092
5093	fn airRetLoad(func: *Func, inst: Air.Inst.Index) !void {
5094	const un_op = func.air.instructions.items(.data)[@backingInt(inst)].un_op;
5095	const ptr = try func.resolveInst(un_op);
5096
5097	const ptr_ty = func.typeOf(un_op);
5098	switch (func.ret_mcv.short) {
5099	.none => {},
5100	.register, .register_pair => try func.load(func.ret_mcv.short, ptr, ptr_ty),
5101	.indirect => |reg_off| try func.genSetReg(ptr_ty, reg_off.reg, ptr),
5102	else => unreachable,
5103	}
5104	func.ret_mcv.liveOut(func, inst);
5105	try func.finishAir(inst, .unreach, .{ un_op, .none, .none });
5106
5107	// Just add space for an instruction, reloced this later
5108	const index = try func.addInst(.{
5109	.tag = .pseudo_j,
5110	.data = .{ .j_type = .{
5111	.rd = .zero,
5112	.inst = undefined,
5113	} },
5114	});
5115
5116	try func.exitlude_jump_relocs.append(func.gpa, index);
5117	}
5118
5119	fn airCmp(func: *Func, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
5120	const bin_op = func.air.instructions.items(.data)[@backingInt(inst)].bin_op;
5121	const pt = func.pt;
5122	const zcu = pt.zcu;
5123
5124	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
5125	const lhs_ty = func.typeOf(bin_op.lhs);
5126
5127	switch (lhs_ty.zigTypeTag(zcu)) {
5128	.int,
5129	.@"enum",
5130	.bool,
5131	.pointer,
5132	.error_set,
5133	.optional,
5134	.@"struct",
5135	=> {
5136	const int_ty: Type = switch (lhs_ty.zigTypeTag(zcu)) {
5137	.int => lhs_ty,
5138	.bool => .u1,
5139	.pointer => .u64,
5140	.error_set => .anyerror,
5141	.optional => blk: {
5142	const payload_ty = lhs_ty.optionalChild(zcu);
5143	if (!payload_ty.hasRuntimeBits(zcu)) {
5144	break :blk .u1;
5145	} else if (lhs_ty.isPtrLikeOptional(zcu)) {
5146	break :blk .u64;
5147	} else {
5148	return func.fail("TODO riscv cmp non-pointer optionals", .{});
5149	}
5150	},
5151	.@"enum", .@"struct", .@"union" => lhs_ty.backingIntType(zcu),
5152	else => unreachable,
5153	};
5154
5155	const int_info = int_ty.intInfo(zcu);
5156	if (int_info.bits <= 64) {
5157	break :result try func.binOp(inst, tag, bin_op.lhs, bin_op.rhs);
5158	} else {
5159	return func.fail("TODO riscv cmp for ints > 64 bits", .{});
5160	}
5161	},
5162	.float => {
5163	const float_bits = lhs_ty.floatBits(func.target);
5164	const float_reg_size: u32 = if (func.hasFeature(.d)) 64 else 32;
5165	if (float_bits > float_reg_size) {
5166	return func.fail("TODO: airCmp float > 64/32 bits", .{});
5167	}
5168	break :result try func.binOp(inst, tag, bin_op.lhs, bin_op.rhs);
5169	},
5170	else => unreachable,
5171	}
5172	};
5173
5174	return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
5175	}
5176
5177	fn airCmpVector(func: *Func, inst: Air.Inst.Index) !void {
5178	_ = inst;
5179	return func.fail("TODO implement airCmpVector for {}", .{func.target.cpu.arch});
5180	}
5181
5182	fn airCmpLteErrorsLen(func: *Func, inst: Air.Inst.Index) !void {
5183	const un_op = func.air.instructions.items(.data)[@backingInt(inst)].un_op;
5184	const operand = try func.resolveInst(un_op);
5185	_ = operand;
5186	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airCmpLteErrorsLen for {}", .{func.target.cpu.arch});
5187	return func.finishAir(inst, result, .{ un_op, .none, .none });
5188	}
5189
5190	fn airDbgStmt(func: *Func, inst: Air.Inst.Index) !void {
5191	const dbg_stmt = func.air.instructions.items(.data)[@backingInt(inst)].dbg_stmt;
5192
5193	_ = try func.addInst(.{
5194	.tag = .pseudo_dbg_line_column,
5195	.data = .{ .pseudo_dbg_line_column = .{
5196	.line = dbg_stmt.line,
5197	.column = dbg_stmt.column,
5198	} },
5199	});
5200
5201	return func.finishAirBookkeeping();
5202	}
5203
5204	fn airDbgInlineBlock(func: *Func, inst: Air.Inst.Index) !void {
5205	const block = func.air.unwrapDbgBlock(inst);
5206	try func.lowerBlock(inst, block.body);
5207	}
5208
5209	fn airDbgVar(func: *Func, inst: Air.Inst.Index) InnerError!void {
5210	const pl_op = func.air.instructions.items(.data)[@backingInt(inst)].pl_op;
5211	const operand = pl_op.operand;
5212	const ty = func.typeOf(operand);
5213	const mcv = try func.resolveInst(operand);
5214	const name: Air.NullTerminatedString = @fromBackingInt(@intCast(pl_op.payload));
5215
5216	const tag = func.air.instructions.items(.tag)[@backingInt(inst)];
5217	func.genVarDbgInfo(tag, ty, mcv, name.toSlice(func.air)) catch |err|
5218	return func.fail("failed to generate variable debug info: {s}", .{@errorName(err)});
5219
5220	return func.finishAir(inst, .unreach, .{ operand, .none, .none });
5221	}
5222
5223	fn genVarDbgInfo(
5224	func: Func,
5225	tag: Air.Inst.Tag,
5226	ty: Type,
5227	mcv: MCValue,
5228	name: []const u8,
5229	) !void {
5230	// TODO: Add a pseudo-instruction or something to defer this work until Emit.
5231	// We aren't allowed to interact with linker state here.
5232	if (true) return;
5233	switch (func.debug_output) {
5234	.dwarf => |dwarf| {
5235	const loc: link.File.Dwarf.Loc = switch (mcv) {
5236	.register => |reg| .{ .reg = reg.dwarfNum() },
5237	.memory => |address| .{ .constu = address },
5238	.immediate => |x| .{ .constu = x },
5239	.none => .empty,
5240	else => blk: {
5241	// log.warn("TODO generate debug info for {}", .{mcv});
5242	break :blk .empty;
5243	},
5244	};
5245	try dwarf.genLocalDebugInfo(switch (tag) {
5246	else => unreachable,
5247	.dbg_var_ptr, .dbg_var_val => .local_var,
5248	.dbg_arg_inline => .local_arg,
5249	}, name, ty, loc);
5250	},
5251	.plan9 => {},
5252	.none => {},
5253	}
5254	}
5255
5256	fn airCondBr(func: *Func, inst: Air.Inst.Index) !void {
5257	const cond_br = func.air.unwrapCondBr(inst);
5258	const cond = try func.resolveInst(cond_br.condition);
5259	const cond_ty = func.typeOf(cond_br.condition);
5260	const then_body = cond_br.then_body;
5261	const else_body = cond_br.else_body;
5262	const liveness_cond_br = func.liveness.getCondBr(inst);
5263
5264	// If the condition dies here in this condbr instruction, process
5265	// that death now instead of later as this has an effect on
5266	// whether it needs to be spilled in the branches
5267	if (func.liveness.operandDies(inst, 0)) {
5268	if (cond_br.condition.toIndex()) |op_inst| try func.processDeath(op_inst);
5269	}
5270
5271	func.scope_generation += 1;
5272	const state = try func.saveState();
5273	const reloc = try func.condBr(cond_ty, cond);
5274
5275	for (liveness_cond_br.then_deaths) |death| try func.processDeath(death);
5276	try func.genBody(then_body);
5277	try func.restoreState(state, &.{}, .{
5278	.emit_instructions = false,
5279	.update_tracking = true,
5280	.resurrect = true,
5281	.close_scope = true,
5282	});
5283
5284	func.performReloc(reloc);
5285
5286	for (liveness_cond_br.else_deaths) |death| try func.processDeath(death);
5287	try func.genBody(else_body);
5288	try func.restoreState(state, &.{}, .{
5289	.emit_instructions = false,
5290	.update_tracking = true,
5291	.resurrect = true,
5292	.close_scope = true,
5293	});
5294
5295	// We already took care of pl_op.operand earlier, so there's nothing left to do.
5296	func.finishAirBookkeeping();
5297	}
5298
5299	fn condBr(func: *Func, cond_ty: Type, condition: MCValue) !Mir.Inst.Index {
5300	const cond_reg = try func.copyToTmpRegister(cond_ty, condition);
5301
5302	return try func.addInst(.{
5303	.tag = .beq,
5304	.data = .{
5305	.b_type = .{
5306	.rs1 = cond_reg,
5307	.rs2 = .zero,
5308	.inst = undefined,
5309	},
5310	},
5311	});
5312	}
5313
5314	fn isNull(func: *Func, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MCValue {
5315	const pt = func.pt;
5316	const zcu = pt.zcu;
5317	const pl_ty = opt_ty.optionalChild(zcu);
5318
5319	const some_info: struct { off: i32, ty: Type } = if (opt_ty.optionalReprIsPayload(zcu))
5320	.{ .off = 0, .ty = if (pl_ty.isSlice(zcu)) pl_ty.slicePtrFieldType(zcu) else pl_ty }
5321	else
5322	.{ .off = @intCast(pl_ty.abiSize(zcu)), .ty = Type.bool };
5323
5324	const return_mcv = try func.allocRegOrMem(func.typeOfIndex(inst), inst, true);
5325	assert(return_mcv == .register); // should not be larger 8 bytes
5326	const return_reg = return_mcv.register;
5327
5328	switch (opt_mcv) {
5329	.none,
5330	.unreach,
5331	.dead,
5332	.undef,
5333	.immediate,
5334	.register_offset,
5335	.lea_frame,
5336	.lea_symbol,
5337	.reserved_frame,
5338	.air_ref,
5339	.register_pair,
5340	=> unreachable,
5341
5342	.register => |opt_reg| {
5343	if (some_info.off == 0) {
5344	_ = try func.addInst(.{
5345	.tag = .pseudo_compare,
5346	.data = .{
5347	.compare = .{
5348	.op = .eq,
5349	.rd = return_reg,
5350	.rs1 = opt_reg,
5351	.rs2 = try func.copyToTmpRegister(
5352	some_info.ty,
5353	.{ .immediate = 0 },
5354	),
5355	.ty = Type.bool,
5356	},
5357	},
5358	});
5359	return return_mcv;
5360	}
5361	assert(some_info.ty.ip_index == .bool_type);
5362	const bit_offset: u7 = @intCast(some_info.off * 8);
5363
5364	try func.genBinOp(
5365	.shr,
5366	.{ .register = opt_reg },
5367	Type.u64,
5368	.{ .immediate = bit_offset },
5369	Type.u8,
5370	return_reg,
5371	);
5372	try func.truncateRegister(Type.u8, return_reg);
5373	try func.genBinOp(
5374	.cmp_eq,
5375	.{ .register = return_reg },
5376	Type.u64,
5377	.{ .immediate = 0 },
5378	Type.u8,
5379	return_reg,
5380	);
5381
5382	return return_mcv;
5383	},
5384
5385	.load_frame => {
5386	const opt_reg = try func.copyToTmpRegister(
5387	some_info.ty,
5388	opt_mcv.address().offset(some_info.off).deref(),
5389	);
5390	const opt_reg_lock = func.register_manager.lockRegAssumeUnused(opt_reg);
5391	defer func.register_manager.unlockReg(opt_reg_lock);
5392
5393	_ = try func.addInst(.{
5394	.tag = .pseudo_compare,
5395	.data = .{
5396	.compare = .{
5397	.op = .eq,
5398	.rd = return_reg,
5399	.rs1 = opt_reg,
5400	.rs2 = try func.copyToTmpRegister(
5401	some_info.ty,
5402	.{ .immediate = 0 },
5403	),
5404	.ty = Type.bool,
5405	},
5406	},
5407	});
5408	return return_mcv;
5409	},
5410
5411	else => return func.fail("TODO: isNull {}", .{opt_mcv}),
5412	}
5413	}
5414
5415	fn airIsNull(func: *Func, inst: Air.Inst.Index) !void {
5416	const un_op = func.air.instructions.items(.data)[@backingInt(inst)].un_op;
5417	const operand = try func.resolveInst(un_op);
5418	const ty = func.typeOf(un_op);
5419	const result = try func.isNull(inst, ty, operand);
5420	return func.finishAir(inst, result, .{ un_op, .none, .none });
5421	}
5422
5423	fn airIsNullPtr(func: *Func, inst: Air.Inst.Index) !void {
5424	const un_op = func.air.instructions.items(.data)[@backingInt(inst)].un_op;
5425	const operand = try func.resolveInst(un_op);
5426	_ = operand;
5427	const ty = func.typeOf(un_op);
5428	_ = ty;
5429
5430	if (true) return func.fail("TODO: airIsNullPtr", .{});
5431
5432	return func.finishAir(inst, .unreach, .{ un_op, .none, .none });
5433	}
5434
5435	fn airIsNonNull(func: *Func, inst: Air.Inst.Index) !void {
5436	const un_op = func.air.instructions.items(.data)[@backingInt(inst)].un_op;
5437	const operand = try func.resolveInst(un_op);
5438	const ty = func.typeOf(un_op);
5439	const result = try func.isNull(inst, ty, operand);
5440	assert(result == .register);
5441
5442	_ = try func.addInst(.{
5443	.tag = .pseudo_not,
5444	.data = .{
5445	.rr = .{
5446	.rd = result.register,
5447	.rs = result.register,
5448	},
5449	},
5450	});
5451
5452	return func.finishAir(inst, result, .{ un_op, .none, .none });
5453	}
5454
5455	fn airIsNonNullPtr(func: *Func, inst: Air.Inst.Index) !void {
5456	const un_op = func.air.instructions.items(.data)[@backingInt(inst)].un_op;
5457	const operand = try func.resolveInst(un_op);
5458	_ = operand;
5459	const ty = func.typeOf(un_op);
5460	_ = ty;
5461
5462	if (true) return func.fail("TODO: airIsNonNullPtr", .{});
5463
5464	return func.finishAir(inst, .unreach, .{ un_op, .none, .none });
5465	}
5466
5467	fn airIsErr(func: *Func, inst: Air.Inst.Index) !void {
5468	const un_op = func.air.instructions.items(.data)[@backingInt(inst)].un_op;
5469	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
5470	const operand = try func.resolveInst(un_op);
5471	const operand_ty = func.typeOf(un_op);
5472	break :result try func.isErr(inst, operand_ty, operand);
5473	};
5474	return func.finishAir(inst, result, .{ un_op, .none, .none });
5475	}
5476
5477	fn airIsErrPtr(func: *Func, inst: Air.Inst.Index) !void {
5478	const pt = func.pt;
5479	const zcu = pt.zcu;
5480	const un_op = func.air.instructions.items(.data)[@backingInt(inst)].un_op;
5481	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
5482	const operand_ptr = try func.resolveInst(un_op);
5483	const operand: MCValue = blk: {
5484	if (func.reuseOperand(inst, un_op, 0, operand_ptr)) {
5485	// The MCValue that holds the pointer can be re-used as the value.
5486	break :blk operand_ptr;
5487	} else {
5488	break :blk try func.allocRegOrMem(func.typeOfIndex(inst), inst, true);
5489	}
5490	};
5491	try func.load(operand, operand_ptr, func.typeOf(un_op));
5492	const operand_ptr_ty = func.typeOf(un_op);
5493	const operand_ty = operand_ptr_ty.childType(zcu);
5494
5495	break :result try func.isErr(inst, operand_ty, operand);
5496	};
5497	return func.finishAir(inst, result, .{ un_op, .none, .none });
5498	}
5499
5500	/// Generates a compare instruction which will indicate if `eu_mcv` is an error.
5501	///
5502	/// Result is in the return register.
5503	fn isErr(func: *Func, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MCValue {
5504	_ = maybe_inst;
5505	const zcu = func.pt.zcu;
5506	const err_ty = eu_ty.errorUnionSet(zcu);
5507	if (err_ty.errorSetIsEmpty(zcu)) return MCValue{ .immediate = 0 }; // always false
5508	const err_off: u31 = @intCast(errUnionErrorOffset(eu_ty.errorUnionPayload(zcu), zcu));
5509
5510	const return_reg, const return_lock = try func.allocReg(.int);
5511	defer func.register_manager.unlockReg(return_lock);
5512
5513	switch (eu_mcv) {
5514	.register => |reg| {
5515	const eu_lock = func.register_manager.lockReg(reg);
5516	defer if (eu_lock) |lock| func.register_manager.unlockReg(lock);
5517
5518	try func.genCopy(eu_ty, .{ .register = return_reg }, eu_mcv);
5519
5520	if (err_off > 0) {
5521	try func.genBinOp(
5522	.shr,
5523	.{ .register = return_reg },
5524	eu_ty,
5525	.{ .immediate = @as(u6, @intCast(err_off * 8)) },
5526	Type.u8,
5527	return_reg,
5528	);
5529	}
5530
5531	try func.genBinOp(
5532	.cmp_neq,
5533	.{ .register = return_reg },
5534	Type.anyerror,
5535	.{ .immediate = 0 },
5536	Type.u8,
5537	return_reg,
5538	);
5539	},
5540	.load_frame => |frame_addr| {
5541	try func.genBinOp(
5542	.cmp_neq,
5543	.{ .load_frame = .{
5544	.index = frame_addr.index,
5545	.off = frame_addr.off + err_off,
5546	} },
5547	Type.anyerror,
5548	.{ .immediate = 0 },
5549	Type.anyerror,
5550	return_reg,
5551	);
5552	},
5553	else => return func.fail("TODO implement isErr for {}", .{eu_mcv}),
5554	}
5555
5556	return .{ .register = return_reg };
5557	}
5558
5559	fn airIsNonErr(func: *Func, inst: Air.Inst.Index) !void {
5560	const un_op = func.air.instructions.items(.data)[@backingInt(inst)].un_op;
5561	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
5562	const operand = try func.resolveInst(un_op);
5563	const ty = func.typeOf(un_op);
5564	break :result try func.isNonErr(inst, ty, operand);
5565	};
5566	return func.finishAir(inst, result, .{ un_op, .none, .none });
5567	}
5568
5569	fn isNonErr(func: *Func, inst: Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MCValue {
5570	const is_err_res = try func.isErr(inst, eu_ty, eu_mcv);
5571	switch (is_err_res) {
5572	.register => |reg| {
5573	_ = try func.addInst(.{
5574	.tag = .pseudo_not,
5575	.data = .{
5576	.rr = .{
5577	.rd = reg,
5578	.rs = reg,
5579	},
5580	},
5581	});
5582	return is_err_res;
5583	},
5584	// always false case
5585	.immediate => |imm| {
5586	assert(imm == 0);
5587	return MCValue{ .immediate = @intFromBool(imm == 0) };
5588	},
5589	else => unreachable,
5590	}
5591	}
5592
5593	fn airIsNonErrPtr(func: *Func, inst: Air.Inst.Index) !void {
5594	const pt = func.pt;
5595	const zcu = pt.zcu;
5596	const un_op = func.air.instructions.items(.data)[@backingInt(inst)].un_op;
5597	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
5598	const operand_ptr = try func.resolveInst(un_op);
5599	const operand: MCValue = blk: {
5600	if (func.reuseOperand(inst, un_op, 0, operand_ptr)) {
5601	// The MCValue that holds the pointer can be re-used as the value.
5602	break :blk operand_ptr;
5603	} else {
5604	break :blk try func.allocRegOrMem(func.typeOfIndex(inst), inst, true);
5605	}
5606	};
5607	const operand_ptr_ty = func.typeOf(un_op);
5608	const operand_ty = operand_ptr_ty.childType(zcu);
5609
5610	try func.load(operand, operand_ptr, func.typeOf(un_op));
5611	break :result try func.isNonErr(inst, operand_ty, operand);
5612	};
5613	return func.finishAir(inst, result, .{ un_op, .none, .none });
5614	}
5615
5616	fn airLoop(func: *Func, inst: Air.Inst.Index) !void {
5617	// A loop is a setup to be able to jump back to the beginning.
5618	const body = func.air.unwrapBlock(inst);
5619	func.scope_generation += 1;
5620	const state = try func.saveState();
5621
5622	try func.loops.putNoClobber(func.gpa, inst, .{
5623	.state = state,
5624	.jmp_target = @intCast(func.mir_instructions.len),
5625	});
5626	defer assert(func.loops.remove(inst));
5627
5628	try func.genBody(body.body);
5629
5630	func.finishAirBookkeeping();
5631	}
5632
5633	/// Send control flow to the `index` of `func.code`.
5634	fn jump(func: *Func, index: Mir.Inst.Index) !Mir.Inst.Index {
5635	return func.addInst(.{
5636	.tag = .pseudo_j,
5637	.data = .{ .j_type = .{
5638	.rd = .zero,
5639	.inst = index,
5640	} },
5641	});
5642	}
5643
5644	fn airBlock(func: *Func, inst: Air.Inst.Index) !void {
5645	const block = func.air.unwrapBlock(inst);
5646	try func.lowerBlock(inst, block.body);
5647	}
5648
5649	fn lowerBlock(func: *Func, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void {
5650	// A block is a setup to be able to jump to the end.
5651	const inst_tracking_i = func.inst_tracking.count();
5652	func.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(.unreach));
5653
5654	func.scope_generation += 1;
5655	try func.blocks.putNoClobber(func.gpa, inst, .{ .state = func.initRetroactiveState() });
5656	const liveness = func.liveness.getBlock(inst);
5657
5658	// TODO emit debug info lexical block
5659	try func.genBody(body);
5660
5661	var block_data = func.blocks.fetchRemove(inst).?;
5662	defer block_data.value.deinit(func.gpa);
5663	if (block_data.value.relocs.items.len > 0) {
5664	try func.restoreState(block_data.value.state, liveness.deaths, .{
5665	.emit_instructions = false,
5666	.update_tracking = true,
5667	.resurrect = true,
5668	.close_scope = true,
5669	});
5670	for (block_data.value.relocs.items) |reloc| func.performReloc(reloc);
5671	}
5672
5673	if (std.debug.runtime_safety) assert(func.inst_tracking.getIndex(inst).? == inst_tracking_i);
5674	const tracking = &func.inst_tracking.values()[inst_tracking_i];
5675	if (func.liveness.isUnused(inst)) try tracking.die(func, inst);
5676	func.getValueIfFree(tracking.short, inst);
5677	func.finishAirBookkeeping();
5678	}
5679
5680	fn airSwitchBr(func: *Func, inst: Air.Inst.Index) !void {
5681	const switch_br = func.air.unwrapSwitch(inst);
5682	const condition = try func.resolveInst(switch_br.operand);
5683
5684	// If the condition dies here in this switch instruction, process
5685	// that death now instead of later as this has an effect on
5686	// whether it needs to be spilled in the branches
5687	if (func.liveness.operandDies(inst, 0)) {
5688	if (switch_br.operand.toIndex()) |op_inst| try func.processDeath(op_inst);
5689	}
5690
5691	try func.lowerSwitchBr(inst, switch_br, condition);
5692
5693	// We already took care of pl_op.operand earlier, so there's nothing left to do
5694	func.finishAirBookkeeping();
5695	}
5696
5697	fn lowerSwitchBr(
5698	func: *Func,
5699	inst: Air.Inst.Index,
5700	switch_br: Air.UnwrappedSwitch,
5701	condition: MCValue,
5702	) !void {
5703	const condition_ty = func.typeOf(switch_br.operand);
5704	const liveness = try func.liveness.getSwitchBr(func.gpa, inst, switch_br.cases_len + 1);
5705	defer func.gpa.free(liveness.deaths);
5706
5707	func.scope_generation += 1;
5708	const state = try func.saveState();
5709
5710	var it = switch_br.iterateCases();
5711	while (it.next()) |case| {
5712	var relocs = try func.gpa.alloc(Mir.Inst.Index, case.items.len + case.ranges.len);
5713	defer func.gpa.free(relocs);
5714
5715	for (case.items, relocs[0..case.items.len]) |item, *reloc| {
5716	const item_mcv = try func.resolveInst(item);
5717
5718	const cond_lock = switch (condition) {
5719	.register => func.register_manager.lockRegAssumeUnused(condition.register),
5720	else => null,
5721	};
5722	defer if (cond_lock) |lock| func.register_manager.unlockReg(lock);
5723
5724	const cmp_reg, const cmp_lock = try func.allocReg(.int);
5725	defer func.register_manager.unlockReg(cmp_lock);
5726
5727	try func.genBinOp(
5728	.cmp_neq,
5729	condition,
5730	condition_ty,
5731	item_mcv,
5732	condition_ty,
5733	cmp_reg,
5734	);
5735
5736	reloc.* = try func.condBr(condition_ty, .{ .register = cmp_reg });
5737	}
5738
5739	for (case.ranges, relocs[case.items.len..]) |range, *reloc| {
5740	const min_mcv = try func.resolveInst(range[0]);
5741	const max_mcv = try func.resolveInst(range[1]);
5742	const cond_lock = switch (condition) {
5743	.register => func.register_manager.lockRegAssumeUnused(condition.register),
5744	else => null,
5745	};
5746	defer if (cond_lock) |lock| func.register_manager.unlockReg(lock);
5747
5748	const temp_cmp_reg, const temp_cmp_lock = try func.allocReg(.int);
5749	defer func.register_manager.unlockReg(temp_cmp_lock);
5750
5751	// is `condition` less than `min`? is "true", we've failed
5752	try func.genBinOp(
5753	.cmp_gte,
5754	condition,
5755	condition_ty,
5756	min_mcv,
5757	condition_ty,
5758	temp_cmp_reg,
5759	);
5760
5761	// if the compare was true, we will jump to the fail case and fall through
5762	// to the next checks
5763	const lt_fail_reloc = try func.condBr(condition_ty, .{ .register = temp_cmp_reg });
5764	try func.genBinOp(
5765	.cmp_gt,
5766	condition,
5767	condition_ty,
5768	max_mcv,
5769	condition_ty,
5770	temp_cmp_reg,
5771	);
5772
5773	reloc.* = try func.condBr(condition_ty, .{ .register = temp_cmp_reg });
5774	func.performReloc(lt_fail_reloc);
5775	}
5776
5777	const skip_case_reloc = try func.jump(undefined);
5778
5779	for (liveness.deaths[case.idx]) |operand| try func.processDeath(operand);
5780
5781	for (relocs) |reloc| func.performReloc(reloc);
5782	try func.genBody(case.body);
5783	try func.restoreState(state, &.{}, .{
5784	.emit_instructions = false,
5785	.update_tracking = true,
5786	.resurrect = true,
5787	.close_scope = true,
5788	});
5789
5790	func.performReloc(skip_case_reloc);
5791	}
5792
5793	if (switch_br.else_body_len > 0) {
5794	const else_body = it.elseBody();
5795
5796	const else_deaths = liveness.deaths.len - 1;
5797	for (liveness.deaths[else_deaths]) |operand| try func.processDeath(operand);
5798
5799	try func.genBody(else_body);
5800	try func.restoreState(state, &.{}, .{
5801	.emit_instructions = false,
5802	.update_tracking = true,
5803	.resurrect = true,
5804	.close_scope = true,
5805	});
5806	}
5807	}
5808
5809	fn airLoopSwitchBr(func: *Func, inst: Air.Inst.Index) !void {
5810	const switch_br = func.air.unwrapSwitch(inst);
5811	const condition = try func.resolveInst(switch_br.operand);
5812
5813	const mat_cond = if (condition.isMutable() and
5814	func.reuseOperand(inst, switch_br.operand, 0, condition))
5815	condition
5816	else mat_cond: {
5817	const ty = func.typeOf(switch_br.operand);
5818	const mat_cond = try func.allocRegOrMem(ty, inst, true);
5819	try func.genCopy(ty, mat_cond, condition);
5820	break :mat_cond mat_cond;
5821	};
5822	func.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(mat_cond));
5823
5824	// If the condition dies here in this switch instruction, process
5825	// that death now instead of later as this has an effect on
5826	// whether it needs to be spilled in the branches
5827	if (func.liveness.operandDies(inst, 0)) {
5828	if (switch_br.operand.toIndex()) |op_inst| try func.processDeath(op_inst);
5829	}
5830
5831	func.scope_generation += 1;
5832	const state = try func.saveState();
5833
5834	try func.loops.putNoClobber(func.gpa, inst, .{
5835	.state = state,
5836	.jmp_target = @intCast(func.mir_instructions.len),
5837	});
5838	defer assert(func.loops.remove(inst));
5839
5840	// Stop tracking block result without forgetting tracking info
5841	try func.freeValue(mat_cond);
5842
5843	try func.lowerSwitchBr(inst, switch_br, mat_cond);
5844
5845	try func.processDeath(inst);
5846	func.finishAirBookkeeping();
5847	}
5848
5849	fn airSwitchDispatch(func: *Func, inst: Air.Inst.Index) !void {
5850	const br = func.air.instructions.items(.data)[@backingInt(inst)].br;
5851
5852	const block_ty = func.typeOfIndex(br.block_inst);
5853	const block_tracking = func.inst_tracking.getPtr(br.block_inst).?;
5854	const loop_data = func.loops.getPtr(br.block_inst).?;
5855	done: {
5856	try func.getValue(block_tracking.short, null);
5857	const src_mcv = try func.resolveInst(br.operand);
5858
5859	if (func.reuseOperandAdvanced(inst, br.operand, 0, src_mcv, br.block_inst)) {
5860	try func.getValue(block_tracking.short, br.block_inst);
5861	// .long = .none to avoid merging operand and block result stack frames.
5862	const current_tracking: InstTracking = .{ .long = .none, .short = src_mcv };
5863	try current_tracking.materializeUnsafe(func, br.block_inst, block_tracking.*);
5864	for (current_tracking.getRegs()) |src_reg| func.register_manager.freeReg(src_reg);
5865	break :done;
5866	}
5867
5868	try func.getValue(block_tracking.short, br.block_inst);
5869	const dst_mcv = block_tracking.short;
5870	try func.genCopy(block_ty, dst_mcv, try func.resolveInst(br.operand));
5871	break :done;
5872	}
5873
5874	// Process operand death so that it is properly accounted for in the State below.
5875	if (func.liveness.operandDies(inst, 0)) {
5876	if (br.operand.toIndex()) |op_inst| try func.processDeath(op_inst);
5877	}
5878
5879	try func.restoreState(loop_data.state, &.{}, .{
5880	.emit_instructions = true,
5881	.update_tracking = false,
5882	.resurrect = false,
5883	.close_scope = false,
5884	});
5885
5886	// Emit a jump with a relocation. It will be patched up after the block ends.
5887	// Leave the jump offset undefined
5888	_ = try func.jump(loop_data.jmp_target);
5889
5890	// Stop tracking block result without forgetting tracking info
5891	try func.freeValue(block_tracking.short);
5892
5893	func.finishAirBookkeeping();
5894	}
5895
5896	fn performReloc(func: *Func, inst: Mir.Inst.Index) void {
5897	const tag = func.mir_instructions.items(.tag)[inst];
5898	const target: Mir.Inst.Index = @intCast(func.mir_instructions.len);
5899
5900	switch (tag) {
5901	.beq,
5902	.bne,
5903	=> func.mir_instructions.items(.data)[inst].b_type.inst = target,
5904	.jal => func.mir_instructions.items(.data)[inst].j_type.inst = target,
5905	.pseudo_j => func.mir_instructions.items(.data)[inst].j_type.inst = target,
5906	else => std.debug.panic("TODO: performReloc {s}", .{@tagName(tag)}),
5907	}
5908	}
5909
5910	fn airBr(func: *Func, inst: Air.Inst.Index) !void {
5911	const zcu = func.pt.zcu;
5912	const br = func.air.instructions.items(.data)[@backingInt(inst)].br;
5913
5914	const block_ty = func.typeOfIndex(br.block_inst);
5915	const block_unused = !block_ty.hasRuntimeBits(zcu) or func.liveness.isUnused(br.block_inst);
5916	const block_tracking = func.inst_tracking.getPtr(br.block_inst).?;
5917	const block_data = func.blocks.getPtr(br.block_inst).?;
5918	const first_br = block_data.relocs.items.len == 0;
5919	const block_result = result: {
5920	if (block_unused) break :result .none;
5921
5922	if (!first_br) try func.getValue(block_tracking.short, null);
5923	const src_mcv = try func.resolveInst(br.operand);
5924
5925	if (func.reuseOperandAdvanced(inst, br.operand, 0, src_mcv, br.block_inst)) {
5926	if (first_br) break :result src_mcv;
5927
5928	try func.getValue(block_tracking.short, br.block_inst);
5929	try InstTracking.materializeUnsafe(
5930	// .long = .none to avoid merging operand and block result stack frames.
5931	.{ .long = .none, .short = src_mcv },
5932	func,
5933	br.block_inst,
5934	block_tracking.*,
5935	);
5936	try func.freeValue(src_mcv);
5937	break :result block_tracking.short;
5938	}
5939
5940	const dst_mcv = if (first_br) try func.allocRegOrMem(block_ty, br.block_inst, true) else dst: {
5941	try func.getValue(block_tracking.short, br.block_inst);
5942	break :dst block_tracking.short;
5943	};
5944	try func.genCopy(block_ty, dst_mcv, try func.resolveInst(br.operand));
5945	break :result dst_mcv;
5946	};
5947
5948	// Process operand death so that it is properly accounted for in the State below.
5949	if (func.liveness.operandDies(inst, 0)) {
5950	if (br.operand.toIndex()) |op_inst| try func.processDeath(op_inst);
5951	}
5952
5953	if (first_br) {
5954	block_tracking.* = InstTracking.init(block_result);
5955	try func.saveRetroactiveState(&block_data.state);
5956	} else try func.restoreState(block_data.state, &.{}, .{
5957	.emit_instructions = true,
5958	.update_tracking = false,
5959	.resurrect = false,
5960	.close_scope = false,
5961	});
5962
5963	// Emit a jump with a relocation. It will be patched up after the block ends.
5964	// Leave the jump offset undefined
5965	const jmp_reloc = try func.jump(undefined);
5966	try block_data.relocs.append(func.gpa, jmp_reloc);
5967
5968	// Stop tracking block result without forgetting tracking info
5969	try func.freeValue(block_tracking.short);
5970
5971	func.finishAirBookkeeping();
5972	}
5973
5974	fn airRepeat(func: *Func, inst: Air.Inst.Index) !void {
5975	const loop_inst = func.air.instructions.items(.data)[@backingInt(inst)].repeat.loop_inst;
5976	const repeat_info = func.loops.get(loop_inst).?;
5977	try func.restoreState(repeat_info.state, &.{}, .{
5978	.emit_instructions = true,
5979	.update_tracking = false,
5980	.resurrect = false,
5981	.close_scope = true,
5982	});
5983	_ = try func.jump(repeat_info.jmp_target);
5984	func.finishAirBookkeeping();
5985	}
5986
5987	fn airBoolOp(func: *Func, inst: Air.Inst.Index) !void {
5988	const bin_op = func.air.instructions.items(.data)[@backingInt(inst)].bin_op;
5989	const tag: Air.Inst.Tag = func.air.instructions.items(.tag)[@backingInt(inst)];
5990
5991	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
5992	const lhs = try func.resolveInst(bin_op.lhs);
5993	const rhs = try func.resolveInst(bin_op.rhs);
5994	const lhs_ty = Type.bool;
5995	const rhs_ty = Type.bool;
5996
5997	const lhs_reg, const lhs_lock = try func.promoteReg(lhs_ty, lhs);
5998	defer if (lhs_lock) |lock| func.register_manager.unlockReg(lock);
5999
6000	const rhs_reg, const rhs_lock = try func.promoteReg(rhs_ty, rhs);
6001	defer if (rhs_lock) |lock| func.register_manager.unlockReg(lock);
6002
6003	const result_reg, const result_lock = try func.allocReg(.int);
6004	defer func.register_manager.unlockReg(result_lock);
6005
6006	_ = try func.addInst(.{
6007	.tag = if (tag == .bool_or) .@"or" else .@"and",
6008	.data = .{ .r_type = .{
6009	.rd = result_reg,
6010	.rs1 = lhs_reg,
6011	.rs2 = rhs_reg,
6012	} },
6013	});
6014
6015	// safety truncate
6016	if (func.wantSafety()) {
6017	_ = try func.addInst(.{
6018	.tag = .andi,
6019	.data = .{ .i_type = .{
6020	.rd = result_reg,
6021	.rs1 = result_reg,
6022	.imm12 = Immediate.s(1),
6023	} },
6024	});
6025	}
6026
6027	break :result .{ .register = result_reg };
6028	};
6029	return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
6030	}
6031
6032	fn airAsm(func: *Func, inst: Air.Inst.Index) !void {
6033	const unwrapped_asm = func.air.unwrapAsm(inst);
6034	const outputs = unwrapped_asm.outputs;
6035	const inputs = unwrapped_asm.inputs;
6036
6037	var result: MCValue = .none;
6038	var args = std.array_list.Managed(MCValue).init(func.gpa);
6039	try args.ensureTotalCapacity(outputs.len + inputs.len);
6040	defer {
6041	for (args.items) |arg| if (arg.getReg()) |reg| func.register_manager.unlockReg(.{
6042	.tracked_index = RegisterManager.indexOfRegIntoTracked(reg) orelse continue,
6043	});
6044	args.deinit();
6045	}
6046	var arg_map = std.StringHashMap(u8).init(func.gpa);
6047	try arg_map.ensureTotalCapacity(@intCast(outputs.len + inputs.len));
6048	defer arg_map.deinit();
6049
6050	var it = unwrapped_asm.iterateOutputs();
6051	while (it.next()) |output| {
6052	const constraint = output.constraint;
6053	const name = output.name;
6054
6055	const is_read = switch (constraint[0]) {
6056	'=' => false,
6057	'+' => read: {
6058	if (output.operand == .none) return func.fail(
6059	"read-write constraint unsupported for asm result: '{s}'",
6060	.{constraint},
6061	);
6062	break :read true;
6063	},
6064	else => return func.fail("invalid constraint: '{s}'", .{constraint}),
6065	};
6066	const is_early_clobber = constraint[1] == '&';
6067	const rest = constraint[@as(usize, 1) + @intFromBool(is_early_clobber) ..];
6068	const arg_mcv: MCValue = arg_mcv: {
6069	const arg_maybe_reg: ?Register = if (mem.eql(u8, rest, "m"))
6070	if (output.operand != .none) null else return func.fail(
6071	"memory constraint unsupported for asm result: '{s}'",
6072	.{constraint},
6073	)
6074	else if (mem.startsWith(u8, rest, "{") and mem.endsWith(u8, rest, "}"))
6075	parseRegName(rest["{".len .. rest.len - "}".len]) orelse
6076	return func.fail("invalid register constraint: '{s}'", .{constraint})
6077	else if (rest.len == 1 and std.ascii.isDigit(rest[0])) {
6078	const index = std.fmt.charToDigit(rest[0], 10) catch unreachable;
6079	if (index >= args.items.len) return func.fail("constraint out of bounds: '{s}'", .{
6080	constraint,
6081	});
6082	break :arg_mcv args.items[index];
6083	} else return func.fail("invalid constraint: '{s}'", .{constraint});
6084	break :arg_mcv if (arg_maybe_reg) |reg| .{ .register = reg } else arg: {
6085	const ptr_mcv = try func.resolveInst(output.operand);
6086	switch (ptr_mcv) {
6087	.immediate => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |_|
6088	break :arg ptr_mcv.deref(),
6089	.register, .register_offset, .lea_frame => break :arg ptr_mcv.deref(),
6090	else => {},
6091	}
6092	break :arg .{ .indirect = .{ .reg = try func.copyToTmpRegister(Type.usize, ptr_mcv) } };
6093	};
6094	};
6095	if (arg_mcv.getReg()) |reg| if (RegisterManager.indexOfRegIntoTracked(reg)) |_| {
6096	_ = func.register_manager.lockReg(reg);
6097	};
6098	if (!mem.eql(u8, name, "_"))
6099	arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len));
6100	args.appendAssumeCapacity(arg_mcv);
6101	if (output.operand == .none) result = arg_mcv;
6102	if (is_read) try func.load(arg_mcv, .{ .air_ref = output.operand }, func.typeOf(output.operand));
6103	}
6104
6105	it = unwrapped_asm.iterateInputs();
6106	while (it.next()) |input| {
6107	const constraint = input.constraint;
6108	const name = input.name;
6109
6110	const ty = func.typeOf(input.operand);
6111	const input_mcv = try func.resolveInst(input.operand);
6112	const arg_mcv: MCValue = if (mem.eql(u8, constraint, "X"))
6113	input_mcv
6114	else if (mem.startsWith(u8, constraint, "{") and mem.endsWith(u8, constraint, "}")) arg: {
6115	const reg = parseRegName(constraint["{".len .. constraint.len - "}".len]) orelse
6116	return func.fail("invalid register constraint: '{s}'", .{constraint});
6117	try func.register_manager.getReg(reg, null);
6118	try func.genSetReg(ty, reg, input_mcv);
6119	break :arg .{ .register = reg };
6120	} else if (mem.eql(u8, constraint, "r")) arg: {
6121	switch (input_mcv) {
6122	.register => break :arg input_mcv,
6123	else => {},
6124	}
6125	const temp_reg = try func.copyToTmpRegister(ty, input_mcv);
6126	break :arg .{ .register = temp_reg };
6127	} else return func.fail("invalid input constraint: '{s}'", .{constraint});
6128	if (arg_mcv.getReg()) |reg| if (RegisterManager.indexOfRegIntoTracked(reg)) |_| {
6129	_ = func.register_manager.lockReg(reg);
6130	};
6131	if (!mem.eql(u8, name, "_"))
6132	arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len));
6133	args.appendAssumeCapacity(arg_mcv);
6134	}
6135
6136	const zcu = func.pt.zcu;
6137	const ip = &zcu.intern_pool;
6138	const clobbers_val: Value = .fromInterned(unwrapped_asm.clobbers);
6139	const clobbers_ty = clobbers_val.typeOf(zcu);
6140	var clobbers_bigint_buf: Value.BigIntSpace = undefined;
6141	const clobbers_bigint = clobbers_val.toBigInt(&clobbers_bigint_buf, zcu);
6142	for (0..clobbers_ty.structFieldCount(zcu)) |field_index| {
6143	assert(clobbers_ty.fieldType(field_index, zcu).toIntern() == .bool_type);
6144	const limb_bits = @bitSizeOf(std.math.big.Limb);
6145	if (field_index / limb_bits >= clobbers_bigint.limbs.len) continue; // field is false
6146	switch (@as(u1, @truncate(clobbers_bigint.limbs[field_index / limb_bits] >> @intCast(field_index % limb_bits)))) {
6147	0 => continue, // field is false
6148	1 => {}, // field is true
6149	}
6150	const clobber = clobbers_ty.structFieldName(field_index, zcu).toSlice(ip).?;
6151	assert(clobber.len != 0);
6152	if (std.mem.eql(u8, clobber, "memory")) {
6153	// nothing really to do
6154	} else {
6155	try func.register_manager.getReg(parseRegName(clobber) orelse
6156	return func.fail("invalid clobber: '{s}'", .{clobber}), null);
6157	}
6158	}
6159
6160	const Label = struct {
6161	target: Mir.Inst.Index = undefined,
6162	pending_relocs: std.ArrayList(Mir.Inst.Index) = .empty,
6163
6164	const Kind = enum { definition, reference };
6165
6166	fn isValid(kind: Kind, name: []const u8) bool {
6167	for (name, 0..) |c, i| switch (c) {
6168	else => return false,
6169	'$' => if (i == 0) return false,
6170	'.' => {},
6171	'0'...'9' => if (i == 0) switch (kind) {
6172	.definition => if (name.len != 1) return false,
6173	.reference => {
6174	if (name.len != 2) return false;
6175	switch (name[1]) {
6176	else => return false,
6177	'B', 'F', 'b', 'f' => {},
6178	}
6179	},
6180	},
6181	'@', 'A'...'Z', '_', 'a'...'z' => {},
6182	};
6183	return name.len > 0;
6184	}
6185	};
6186	var labels: std.StringHashMapUnmanaged(Label) = .empty;
6187	defer {
6188	var label_it = labels.valueIterator();
6189	while (label_it.next()) |label| label.pending_relocs.deinit(func.gpa);
6190	labels.deinit(func.gpa);
6191	}
6192
6193	const asm_source = unwrapped_asm.source;
6194	var line_it = mem.tokenizeAny(u8, asm_source, "\n\r;");
6195	next_line: while (line_it.next()) |line| {
6196	var mnem_it = mem.tokenizeAny(u8, line, " \t");
6197	const mnem_str = while (mnem_it.next()) |mnem_str| {
6198	if (mem.startsWith(u8, mnem_str, "#")) continue :next_line;
6199	if (mem.startsWith(u8, mnem_str, "//")) continue :next_line;
6200	if (!mem.endsWith(u8, mnem_str, ":")) break mnem_str;
6201	const label_name = mnem_str[0 .. mnem_str.len - ":".len];
6202	if (!Label.isValid(.definition, label_name))
6203	return func.fail("invalid label: '{s}'", .{label_name});
6204
6205	const label_gop = try labels.getOrPut(func.gpa, label_name);
6206	if (!label_gop.found_existing) label_gop.value_ptr.* = .{} else {
6207	const anon = std.ascii.isDigit(label_name[0]);
6208	if (!anon and label_gop.value_ptr.pending_relocs.items.len == 0)
6209	return func.fail("redefined label: '{s}'", .{label_name});
6210	for (label_gop.value_ptr.pending_relocs.items) |pending_reloc|
6211	func.performReloc(pending_reloc);
6212	if (anon)
6213	label_gop.value_ptr.pending_relocs.clearRetainingCapacity()
6214	else
6215	label_gop.value_ptr.pending_relocs.clearAndFree(func.gpa);
6216	}
6217	label_gop.value_ptr.target = @intCast(func.mir_instructions.len);
6218	} else continue;
6219
6220	const instruction: union(enum) { mnem: Mnemonic, pseudo: Pseudo } =
6221	if (std.meta.stringToEnum(Mnemonic, mnem_str)) |mnem|
6222	.{ .mnem = mnem }
6223	else if (std.meta.stringToEnum(Pseudo, mnem_str)) |pseudo|
6224	.{ .pseudo = pseudo }
6225	else
6226	return func.fail("invalid mnem str '{s}'", .{mnem_str});
6227
6228	const Operand = union(enum) {
6229	none,
6230	reg: Register,
6231	imm: Immediate,
6232	inst: Mir.Inst.Index,
6233	sym: SymbolOffset,
6234	};
6235
6236	var ops: [4]Operand = @splat(.none);
6237	var last_op = false;
6238	var op_it = mem.splitAny(u8, mnem_it.rest(), ",(");
6239	next_op: for (&ops) |*op| {
6240	const op_str = while (!last_op) {
6241	const full_str = op_it.next() orelse break :next_op;
6242	const code_str = if (mem.findScalar(u8, full_str, '#') orelse
6243	mem.find(u8, full_str, "//")) |comment|
6244	code: {
6245	last_op = true;
6246	break :code full_str[0..comment];
6247	} else full_str;
6248	const trim_str = mem.trim(u8, code_str, " \t*");
6249	if (trim_str.len > 0) break trim_str;
6250	} else break;
6251
6252	if (parseRegName(op_str)) |reg| {
6253	op.* = .{ .reg = reg };
6254	} else if (std.fmt.parseInt(i12, op_str, 10)) |int| {
6255	op.* = .{ .imm = Immediate.s(int) };
6256	} else |_| if (mem.startsWith(u8, op_str, "%[")) {
6257	const mod_index = mem.find(u8, op_str, "]@");
6258	const modifier = if (mod_index) |index|
6259	op_str[index + "]@".len ..]
6260	else
6261	"";
6262
6263	op.* = switch (args.items[
6264	arg_map.get(op_str["%[".len .. mod_index orelse op_str.len - "]".len]) orelse
6265	return func.fail("no matching constraint: '{s}'", .{op_str})
6266	]) {
6267	.lea_symbol => |sym_off| if (mem.eql(u8, modifier, "plt")) blk: {
6268	assert(sym_off.off == 0);
6269	break :blk .{ .sym = sym_off };
6270	} else return func.fail("invalid modifier: '{s}'", .{modifier}),
6271	.register => |reg| if (modifier.len == 0)
6272	.{ .reg = reg }
6273	else
6274	return func.fail("invalid modified '{s}'", .{modifier}),
6275	else => return func.fail("invalid constraint: '{s}'", .{op_str}),
6276	};
6277	} else if (mem.endsWith(u8, op_str, ")")) {
6278	const reg = op_str[0 .. op_str.len - ")".len];
6279	const addr_reg = parseRegName(reg) orelse
6280	return func.fail("expected valid register, found '{s}'", .{reg});
6281
6282	op.* = .{ .reg = addr_reg };
6283	} else if (Label.isValid(.reference, op_str)) {
6284	const anon = std.ascii.isDigit(op_str[0]);
6285	const label_gop = try labels.getOrPut(func.gpa, op_str[0..if (anon) 1 else op_str.len]);
6286	if (!label_gop.found_existing) label_gop.value_ptr.* = .{};
6287	if (anon and (op_str[1] == 'b' or op_str[1] == 'B') and !label_gop.found_existing)
6288	return func.fail("undefined label: '{s}'", .{op_str});
6289	const pending_relocs = &label_gop.value_ptr.pending_relocs;
6290	if (if (anon)
6291	op_str[1] == 'f' or op_str[1] == 'F'
6292	else
6293	!label_gop.found_existing or pending_relocs.items.len > 0)
6294	try pending_relocs.append(func.gpa, @intCast(func.mir_instructions.len));
6295	op.* = .{ .inst = label_gop.value_ptr.target };
6296	} else return func.fail("invalid operand: '{s}'", .{op_str});
6297	} else if (op_it.next()) |op_str| return func.fail("extra operand: '{s}'", .{op_str});
6298
6299	switch (instruction) {
6300	.mnem => |mnem| {
6301	_ = (switch (ops[0]) {
6302	.none => try func.addInst(.{
6303	.tag = mnem,
6304	.data = .none,
6305	}),
6306	.reg => |reg1| switch (ops[1]) {
6307	.reg => |reg2| switch (ops[2]) {
6308	.imm => |imm1| try func.addInst(.{
6309	.tag = mnem,
6310	.data = .{ .i_type = .{
6311	.rd = reg1,
6312	.rs1 = reg2,
6313	.imm12 = imm1,
6314	} },
6315	}),
6316	else => error.InvalidInstruction,
6317	},
6318	.imm => |imm1| switch (ops[2]) {
6319	.reg => |reg2| switch (mnem) {
6320	.sd => try func.addInst(.{
6321	.tag = mnem,
6322	.data = .{ .i_type = .{
6323	.rd = reg2,
6324	.rs1 = reg1,
6325	.imm12 = imm1,
6326	} },
6327	}),
6328	.ld => try func.addInst(.{
6329	.tag = mnem,
6330	.data = .{ .i_type = .{
6331	.rd = reg1,
6332	.rs1 = reg2,
6333	.imm12 = imm1,
6334	} },
6335	}),
6336	else => error.InvalidInstruction,
6337	},
6338	else => error.InvalidInstruction,
6339	},
6340	.none => switch (mnem) {
6341	.jalr => try func.addInst(.{
6342	.tag = mnem,
6343	.data = .{ .i_type = .{
6344	.rd = .ra,
6345	.rs1 = reg1,
6346	.imm12 = Immediate.s(0),
6347	} },
6348	}),
6349	else => error.InvalidInstruction,
6350	},
6351	else => error.InvalidInstruction,
6352	},
6353	else => error.InvalidInstruction,
6354	}) catch |err| {
6355	switch (err) {
6356	error.InvalidInstruction => return func.fail(
6357	"invalid instruction: {s} {s} {s} {s} {s}",
6358	.{
6359	@tagName(mnem),
6360	@tagName(ops[0]),
6361	@tagName(ops[1]),
6362	@tagName(ops[2]),
6363	@tagName(ops[3]),
6364	},
6365	),
6366	else => |e| return e,
6367	}
6368	};
6369	},
6370	.pseudo => |pseudo| {
6371	(@as(error{InvalidInstruction}!void, switch (pseudo) {
6372	.li => blk: {
6373	if (ops[0] != .reg or ops[1] != .imm) {
6374	break :blk error.InvalidInstruction;
6375	}
6376
6377	const reg = ops[0].reg;
6378	const imm = ops[1].imm;
6379
6380	try func.genSetReg(Type.usize, reg, .{ .immediate = imm.asBits(u64) });
6381	},
6382	.mv => blk: {
6383	if (ops[0] != .reg or ops[1] != .reg) {
6384	break :blk error.InvalidInstruction;
6385	}
6386
6387	const dst = ops[0].reg;
6388	const src = ops[1].reg;
6389
6390	if (dst.class() != .int or src.class() != .int) {
6391	return func.fail("pseudo instruction 'mv' only works on integer registers", .{});
6392	}
6393
6394	try func.genSetReg(Type.usize, dst, .{ .register = src });
6395	},
6396	.tail => blk: {
6397	if (ops[0] != .sym) {
6398	break :blk error.InvalidInstruction;
6399	}
6400
6401	const sym_offset = ops[0].sym;
6402	assert(sym_offset.off == 0);
6403
6404	const random_link_reg, const lock = try func.allocReg(.int);
6405	defer func.register_manager.unlockReg(lock);
6406
6407	_ = try func.addInst(.{
6408	.tag = .pseudo_extern_fn_reloc,
6409	.data = .{ .reloc = .{
6410	.register = random_link_reg,
6411	.atom_index = @fromBackingInt(@intCast(try func.owner.getSymbolIndex(func))),
6412	.sym_index = sym_offset.sym,
6413	} },
6414	});
6415	},
6416	.ret => _ = try func.addInst(.{
6417	.tag = .jalr,
6418	.data = .{ .i_type = .{
6419	.rd = .zero,
6420	.rs1 = .ra,
6421	.imm12 = Immediate.s(0),
6422	} },
6423	}),
6424	.beqz => blk: {
6425	if (ops[0] != .reg or ops[1] != .inst) {
6426	break :blk error.InvalidInstruction;
6427	}
6428
6429	_ = try func.addInst(.{
6430	.tag = .beq,
6431	.data = .{ .b_type = .{
6432	.rs1 = ops[0].reg,
6433	.rs2 = .zero,
6434	.inst = ops[1].inst,
6435	} },
6436	});
6437	},
6438	})) catch |err| {
6439	switch (err) {
6440	error.InvalidInstruction => return func.fail(
6441	"invalid instruction: {s} {s} {s} {s} {s}",
6442	.{
6443	@tagName(pseudo),
6444	@tagName(ops[0]),
6445	@tagName(ops[1]),
6446	@tagName(ops[2]),
6447	@tagName(ops[3]),
6448	},
6449	),
6450	else => |e| return e,
6451	}
6452	};
6453	},
6454	}
6455	}
6456
6457	var label_it = labels.iterator();
6458	while (label_it.next()) |label| if (label.value_ptr.pending_relocs.items.len > 0)
6459	return func.fail("undefined label: '{s}'", .{label.key_ptr.*});
6460
6461	it = unwrapped_asm.iterateOutputs();
6462	while (it.next()) |output| {
6463	const constraint = output.constraint;
6464
6465	if (output.operand == .none) continue;
6466	if (args.items[output.index] != .register) continue;
6467	if (constraint.len == 2 and std.ascii.isDigit(constraint[1])) continue;
6468	try func.store(.{ .air_ref = output.operand }, args.items[output.index], func.typeOf(output.operand));
6469	}
6470
6471	simple: {
6472	var buf: [Air.Liveness.bpi - 1]Air.Inst.Ref = @splat(.none);
6473	var buf_index: usize = 0;
6474	for (outputs) |output| {
6475	if (output == .none) continue;
6476
6477	if (buf_index >= buf.len) break :simple;
6478	buf[buf_index] = output;
6479	buf_index += 1;
6480	}
6481	if (buf_index + inputs.len > buf.len) break :simple;
6482	@memcpy(buf[buf_index..][0..inputs.len], inputs);
6483	return func.finishAir(inst, result, buf);
6484	}
6485	var bt = func.liveness.iterateBigTomb(inst);
6486	for (outputs) |output| if (output != .none) try func.feed(&bt, output);
6487	for (inputs) |input| try func.feed(&bt, input);
6488	return func.finishAirResult(inst, result);
6489	}
6490
6491	/// Sets the value of `dst_mcv` to the value of `src_mcv`.
6492	fn genCopy(func: *Func, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void {
6493	// There isn't anything to store
6494	if (dst_mcv == .none) return;
6495
6496	if (!dst_mcv.isMutable()) {
6497	// panic so we can see the trace
6498	return std.debug.panic("tried to genCopy immutable: {s}", .{@tagName(dst_mcv)});
6499	}
6500
6501	const zcu = func.pt.zcu;
6502
6503	switch (dst_mcv) {
6504	.register => |reg| return func.genSetReg(ty, reg, src_mcv),
6505	.register_offset => |dst_reg_off| try func.genSetReg(ty, dst_reg_off.reg, switch (src_mcv) {
6506	.none,
6507	.unreach,
6508	.dead,
6509	.undef,
6510	=> unreachable,
6511	.immediate,
6512	.register,
6513	.register_offset,
6514	=> src_mcv.offset(-dst_reg_off.off),
6515	else => .{ .register_offset = .{
6516	.reg = try func.copyToTmpRegister(ty, src_mcv),
6517	.off = -dst_reg_off.off,
6518	} },
6519	}),
6520	.indirect => |reg_off| try func.genSetMem(
6521	.{ .reg = reg_off.reg },
6522	reg_off.off,
6523	ty,
6524	src_mcv,
6525	),
6526	.load_frame => |frame_addr| try func.genSetMem(
6527	.{ .frame = frame_addr.index },
6528	frame_addr.off,
6529	ty,
6530	src_mcv,
6531	),
6532	.load_symbol => {
6533	const addr_reg, const addr_lock = try func.allocReg(.int);
6534	defer func.register_manager.unlockReg(addr_lock);
6535
6536	try func.genSetReg(ty, addr_reg, dst_mcv.address());
6537	try func.genCopy(ty, .{ .indirect = .{ .reg = addr_reg } }, src_mcv);
6538	},
6539	.memory => return func.fail("TODO: genCopy memory", .{}),
6540	.register_pair => |dst_regs| {
6541	const src_info: ?struct { addr_reg: Register, addr_lock: ?RegisterLock } = switch (src_mcv) {
6542	.register_pair, .memory, .indirect, .load_frame => null,
6543	.load_symbol => src: {
6544	const src_addr_reg, const src_addr_lock = try func.promoteReg(Type.u64, src_mcv.address());
6545	errdefer func.register_manager.unlockReg(src_addr_lock);
6546
6547	break :src .{ .addr_reg = src_addr_reg, .addr_lock = src_addr_lock };
6548	},
6549	.air_ref => |src_ref| return func.genCopy(
6550	ty,
6551	dst_mcv,
6552	try func.resolveInst(src_ref),
6553	),
6554	else => return func.fail("genCopy register_pair src: {}", .{src_mcv}),
6555	};
6556
6557	defer if (src_info) |info| {
6558	if (info.addr_lock) |lock| {
6559	func.register_manager.unlockReg(lock);
6560	}
6561	};
6562
6563	var part_disp: i32 = 0;
6564	for (dst_regs, try func.splitType(ty), 0..) |dst_reg, dst_ty, part_i| {
6565	try func.genSetReg(dst_ty, dst_reg, switch (src_mcv) {
6566	.register_pair => |src_regs| .{ .register = src_regs[part_i] },
6567	.memory, .indirect, .load_frame => src_mcv.address().offset(part_disp).deref(),
6568	.load_symbol => .{ .indirect = .{
6569	.reg = src_info.?.addr_reg,
6570	.off = part_disp,
6571	} },
6572	else => unreachable,
6573	});
6574	part_disp += @intCast(dst_ty.abiSize(zcu));
6575	}
6576	},
6577	else => return std.debug.panic("TODO: genCopy to {s} from {s}", .{ @tagName(dst_mcv), @tagName(src_mcv) }),
6578	}
6579	}
6580
6581	fn genInlineMemcpy(
6582	func: *Func,
6583	dst_ptr: MCValue,
6584	src_ptr: MCValue,
6585	len: MCValue,
6586	) !void {
6587	const regs = try func.register_manager.allocRegs(4, @splat(null), abi.Registers.Integer.temporary);
6588	const locks = func.register_manager.lockRegsAssumeUnused(4, regs);
6589	defer for (locks) |lock| func.register_manager.unlockReg(lock);
6590
6591	const count = regs[0];
6592	const tmp = regs[1];
6593	const src = regs[2];
6594	const dst = regs[3];
6595
6596	try func.genSetReg(Type.u64, count, len);
6597	try func.genSetReg(Type.u64, src, src_ptr);
6598	try func.genSetReg(Type.u64, dst, dst_ptr);
6599
6600	// if count is 0, there's nothing to copy
6601	_ = try func.addInst(.{
6602	.tag = .beq,
6603	.data = .{ .b_type = .{
6604	.rs1 = count,
6605	.rs2 = .zero,
6606	.inst = @intCast(func.mir_instructions.len + 9),
6607	} },
6608	});
6609
6610	// lb tmp, 0(src)
6611	const first_inst = try func.addInst(.{
6612	.tag = .lb,
6613	.data = .{
6614	.i_type = .{
6615	.rd = tmp,
6616	.rs1 = src,
6617	.imm12 = Immediate.s(0),
6618	},
6619	},
6620	});
6621
6622	// sb tmp, 0(dst)
6623	_ = try func.addInst(.{
6624	.tag = .sb,
6625	.data = .{
6626	.i_type = .{
6627	.rd = dst,
6628	.rs1 = tmp,
6629	.imm12 = Immediate.s(0),
6630	},
6631	},
6632	});
6633
6634	// dec count by 1
6635	_ = try func.addInst(.{
6636	.tag = .addi,
6637	.data = .{
6638	.i_type = .{
6639	.rd = count,
6640	.rs1 = count,
6641	.imm12 = Immediate.s(-1),
6642	},
6643	},
6644	});
6645
6646	// branch if count is 0
6647	_ = try func.addInst(.{
6648	.tag = .beq,
6649	.data = .{
6650	.b_type = .{
6651	.inst = @intCast(func.mir_instructions.len + 4), // points after the last inst
6652	.rs1 = count,
6653	.rs2 = .zero,
6654	},
6655	},
6656	});
6657
6658	// increment the pointers
6659	_ = try func.addInst(.{
6660	.tag = .addi,
6661	.data = .{
6662	.i_type = .{
6663	.rd = src,
6664	.rs1 = src,
6665	.imm12 = Immediate.s(1),
6666	},
6667	},
6668	});
6669
6670	_ = try func.addInst(.{
6671	.tag = .addi,
6672	.data = .{
6673	.i_type = .{
6674	.rd = dst,
6675	.rs1 = dst,
6676	.imm12 = Immediate.s(1),
6677	},
6678	},
6679	});
6680
6681	// jump back to start of loop
6682	_ = try func.addInst(.{
6683	.tag = .pseudo_j,
6684	.data = .{ .j_type = .{
6685	.rd = .zero,
6686	.inst = first_inst,
6687	} },
6688	});
6689	}
6690
6691	fn genInlineMemset(
6692	func: *Func,
6693	dst_ptr: MCValue,
6694	src_value: MCValue,
6695	len: MCValue,
6696	) !void {
6697	const regs = try func.register_manager.allocRegs(3, @splat(null), abi.Registers.Integer.temporary);
6698	const locks = func.register_manager.lockRegsAssumeUnused(3, regs);
6699	defer for (locks) |lock| func.register_manager.unlockReg(lock);
6700
6701	const count = regs[0];
6702	const src = regs[1];
6703	const dst = regs[2];
6704
6705	try func.genSetReg(Type.u64, count, len);
6706	try func.genSetReg(Type.u64, src, src_value);
6707	try func.genSetReg(Type.u64, dst, dst_ptr);
6708
6709	// sb src, 0(dst)
6710	const first_inst = try func.addInst(.{
6711	.tag = .sb,
6712	.data = .{
6713	.i_type = .{
6714	.rd = dst,
6715	.rs1 = src,
6716	.imm12 = Immediate.s(0),
6717	},
6718	},
6719	});
6720
6721	// dec count by 1
6722	_ = try func.addInst(.{
6723	.tag = .addi,
6724	.data = .{
6725	.i_type = .{
6726	.rd = count,
6727	.rs1 = count,
6728	.imm12 = Immediate.s(-1),
6729	},
6730	},
6731	});
6732
6733	// branch if count is 0
6734	_ = try func.addInst(.{
6735	.tag = .beq,
6736	.data = .{
6737	.b_type = .{
6738	.inst = @intCast(func.mir_instructions.len + 3), // points after the last inst
6739	.rs1 = count,
6740	.rs2 = .zero,
6741	},
6742	},
6743	});
6744
6745	// increment the pointers
6746	_ = try func.addInst(.{
6747	.tag = .addi,
6748	.data = .{
6749	.i_type = .{
6750	.rd = dst,
6751	.rs1 = dst,
6752	.imm12 = Immediate.s(1),
6753	},
6754	},
6755	});
6756
6757	// jump back to start of loop
6758	_ = try func.addInst(.{
6759	.tag = .pseudo_j,
6760	.data = .{ .j_type = .{
6761	.rd = .zero,
6762	.inst = first_inst,
6763	} },
6764	});
6765	}
6766
6767	/// Sets the value of `src_mcv` into `reg`. Assumes you have a lock on it.
6768	fn genSetReg(func: *Func, ty: Type, reg: Register, src_mcv: MCValue) InnerError!void {
6769	const pt = func.pt;
6770	const zcu = pt.zcu;
6771	const abi_size: u32 = @intCast(ty.abiSize(zcu));
6772
6773	const max_size: u32 = switch (reg.class()) {
6774	.int => 64,
6775	.float => if (func.hasFeature(.d)) 64 else 32,
6776	.vector => 64, // TODO: calculate it from avl * vsew
6777	};
6778	if (abi_size > max_size) return std.debug.panic("tried to set reg with size {}", .{abi_size});
6779	const dst_reg_class = reg.class();
6780
6781	switch (src_mcv) {
6782	.unreach,
6783	.none,
6784	.dead,
6785	=> unreachable,
6786	.undef => |sym_index| {
6787	if (!func.wantSafety())
6788	return;
6789
6790	if (sym_index) |index| {
6791	return func.genSetReg(ty, reg, .{ .load_symbol = .{ .sym = index } });
6792	}
6793
6794	switch (abi_size) {
6795	1 => return func.genSetReg(ty, reg, .{ .immediate = 0xAA }),
6796	2 => return func.genSetReg(ty, reg, .{ .immediate = 0xAAAA }),
6797	3...4 => return func.genSetReg(ty, reg, .{ .immediate = 0xAAAAAAAA }),
6798	5...8 => return func.genSetReg(ty, reg, .{ .immediate = 0xAAAAAAAAAAAAAAAA }),
6799	else => unreachable,
6800	}
6801	},
6802	.immediate => |unsigned_x| {
6803	assert(dst_reg_class == .int);
6804
6805	const x: i64 = @bitCast(unsigned_x);
6806	if (math.minInt(i12) <= x and x <= math.maxInt(i12)) {
6807	_ = try func.addInst(.{
6808	.tag = .addi,
6809	.data = .{ .i_type = .{
6810	.rd = reg,
6811	.rs1 = .zero,
6812	.imm12 = Immediate.s(@intCast(x)),
6813	} },
6814	});
6815	} else if (math.minInt(i32) <= x and x <= math.maxInt(i32)) {
6816	const lo12: i12 = @truncate(x);
6817	const carry: i32 = if (lo12 < 0) 1 else 0;
6818	const hi20: i20 = @truncate((x >> 12) +% carry);
6819
6820	_ = try func.addInst(.{
6821	.tag = .lui,
6822	.data = .{ .u_type = .{
6823	.rd = reg,
6824	.imm20 = Immediate.s(hi20),
6825	} },
6826	});
6827	_ = try func.addInst(.{
6828	.tag = .addi,
6829	.data = .{ .i_type = .{
6830	.rd = reg,
6831	.rs1 = reg,
6832	.imm12 = Immediate.s(lo12),
6833	} },
6834	});
6835	} else {
6836	// TODO: use a more advanced myriad seq to do this without a reg.
6837	// see: https://github.com/llvm/llvm-project/blob/081a66ffacfe85a37ff775addafcf3371e967328/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp#L224
6838
6839	const temp, const temp_lock = try func.allocReg(.int);
6840	defer func.register_manager.unlockReg(temp_lock);
6841
6842	const lo32: i32 = @truncate(x);
6843	const carry: i32 = if (lo32 < 0) 1 else 0;
6844	const hi32: i32 = @truncate((x >> 32) +% carry);
6845
6846	try func.genSetReg(Type.i32, temp, .{ .immediate = @bitCast(@as(i64, lo32)) });
6847	try func.genSetReg(Type.i32, reg, .{ .immediate = @bitCast(@as(i64, hi32)) });
6848
6849	_ = try func.addInst(.{
6850	.tag = .slli,
6851	.data = .{ .i_type = .{
6852	.rd = reg,
6853	.rs1 = reg,
6854	.imm12 = Immediate.u(32),
6855	} },
6856	});
6857
6858	_ = try func.addInst(.{
6859	.tag = .add,
6860	.data = .{ .r_type = .{
6861	.rd = reg,
6862	.rs1 = reg,
6863	.rs2 = temp,
6864	} },
6865	});
6866	}
6867	},
6868	.register => |src_reg| {
6869	// If the registers are the same, nothing to do.
6870	if (src_reg.id() == reg.id())
6871	return;
6872
6873	// there is no instruction for loading the contents of a vector register
6874	// into an integer register, however we can cheat a bit by setting the element
6875	// size to the total size of the vector, and vmv.x.s will work then
6876	if (src_reg.class() == .vector) {
6877	try func.setVl(.zero, 0, .{
6878	.vsew = switch (ty.bitSize(zcu)) {
6879	8 => .@"8",
6880	16 => .@"16",
6881	32 => .@"32",
6882	64 => .@"64",
6883	else => |vec_bits| return func.fail("TODO: genSetReg vec -> {s} bits {d}", .{
6884	@tagName(reg.class()),
6885	vec_bits,
6886	}),
6887	},
6888	.vlmul = .m1,
6889	.vta = true,
6890	.vma = true,
6891	});
6892	}
6893
6894	// mv reg, src_reg
6895	_ = try func.addInst(.{
6896	.tag = .pseudo_mv,
6897	.data = .{ .rr = .{
6898	.rd = reg,
6899	.rs = src_reg,
6900	} },
6901	});
6902	},
6903	// useful in cases like slice_ptr, which can easily reuse the operand
6904	// but we need to get only the pointer out.
6905	.register_pair => |pair| try func.genSetReg(ty, reg, .{ .register = pair[0] }),
6906	.load_frame => |frame| {
6907	if (reg.class() == .vector) {
6908	// vectors don't support an offset memory load so we need to put the true
6909	// address into a register before loading from it.
6910	const addr_reg, const addr_lock = try func.allocReg(.int);
6911	defer func.register_manager.unlockReg(addr_lock);
6912
6913	try func.genCopy(ty, .{ .register = addr_reg }, src_mcv.address());
6914	try func.genCopy(ty, .{ .register = reg }, .{ .indirect = .{ .reg = addr_reg } });
6915	} else {
6916	_ = try func.addInst(.{
6917	.tag = .pseudo_load_rm,
6918	.data = .{ .rm = .{
6919	.r = reg,
6920	.m = .{
6921	.base = .{ .frame = frame.index },
6922	.mod = .{
6923	.size = func.memSize(ty),
6924	.unsigned = ty.isUnsignedInt(zcu),
6925	.disp = frame.off,
6926	},
6927	},
6928	} },
6929	});
6930	}
6931	},
6932	.memory => |addr| {
6933	try func.genSetReg(ty, reg, .{ .immediate = addr });
6934
6935	_ = try func.addInst(.{
6936	.tag = .ld,
6937	.data = .{ .i_type = .{
6938	.rd = reg,
6939	.rs1 = reg,
6940	.imm12 = Immediate.u(0),
6941	} },
6942	});
6943	},
6944	.lea_frame, .register_offset => {
6945	_ = try func.addInst(.{
6946	.tag = .pseudo_lea_rm,
6947	.data = .{
6948	.rm = .{
6949	.r = reg,
6950	.m = switch (src_mcv) {
6951	.register_offset => |reg_off| .{
6952	.base = .{ .reg = reg_off.reg },
6953	.mod = .{
6954	.size = .byte, // the size doesn't matter
6955	.disp = reg_off.off,
6956	.unsigned = false,
6957	},
6958	},
6959	.lea_frame => |frame| .{
6960	.base = .{ .frame = frame.index },
6961	.mod = .{
6962	.size = .byte, // the size doesn't matter
6963	.disp = frame.off,
6964	.unsigned = false,
6965	},
6966	},
6967	else => unreachable,
6968	},
6969	},
6970	},
6971	});
6972	},
6973	.indirect => |reg_off| {
6974	const load_tag: Mnemonic = switch (reg.class()) {
6975	.float => switch (abi_size) {
6976	1 => unreachable, // Zig does not support 8-bit floats
6977	2 => return func.fail("TODO: genSetReg indirect 16-bit float", .{}),
6978	4 => .flw,
6979	8 => .fld,
6980	else => return std.debug.panic("TODO: genSetReg for float size {d}", .{abi_size}),
6981	},
6982	.int => switch (abi_size) {
6983	1...1 => .lb,
6984	2...2 => .lh,
6985	3...4 => .lw,
6986	5...8 => .ld,
6987	else => return std.debug.panic("TODO: genSetReg for int size {d}", .{abi_size}),
6988	},
6989	.vector => {
6990	assert(reg_off.off == 0);
6991
6992	// There is no vector instruction for loading with an offset to a base register,
6993	// so we need to get an offset register containing the address of the vector first
6994	// and load from it.
6995	const len = ty.vectorLen(zcu);
6996	const elem_ty = ty.childType(zcu);
6997	const elem_size = elem_ty.abiSize(zcu);
6998
6999	try func.setVl(.zero, len, .{
7000	.vsew = switch (elem_size) {
7001	1 => .@"8",
7002	2 => .@"16",
7003	4 => .@"32",
7004	8 => .@"64",
7005	else => unreachable,
7006	},
7007	.vlmul = .m1,
7008	.vma = true,
7009	.vta = true,
7010	});
7011
7012	_ = try func.addInst(.{
7013	.tag = .pseudo_load_rm,
7014	.data = .{ .rm = .{
7015	.r = reg,
7016	.m = .{
7017	.base = .{ .reg = reg_off.reg },
7018	.mod = .{
7019	.size = func.memSize(elem_ty),
7020	.unsigned = false,
7021	.disp = 0,
7022	},
7023	},
7024	} },
7025	});
7026
7027	return;
7028	},
7029	};
7030
7031	_ = try func.addInst(.{
7032	.tag = load_tag,
7033	.data = .{ .i_type = .{
7034	.rd = reg,
7035	.rs1 = reg_off.reg,
7036	.imm12 = Immediate.s(reg_off.off),
7037	} },
7038	});
7039	},
7040	.lea_symbol => |sym_off| {
7041	assert(sym_off.off == 0);
7042	const atom_index: link.File.AtomId = @fromBackingInt(@intCast(try func.owner.getSymbolIndex(func)));
7043
7044	_ = try func.addInst(.{
7045	.tag = .pseudo_load_symbol,
7046	.data = .{ .reloc = .{
7047	.register = reg,
7048	.atom_index = atom_index,
7049	.sym_index = sym_off.sym,
7050	} },
7051	});
7052	},
7053	.load_symbol => {
7054	const addr_reg, const addr_lock = try func.allocReg(.int);
7055	defer func.register_manager.unlockReg(addr_lock);
7056
7057	try func.genSetReg(ty, addr_reg, src_mcv.address());
7058	try func.genSetReg(ty, reg, .{ .indirect = .{ .reg = addr_reg } });
7059	},
7060	.air_ref => |ref| try func.genSetReg(ty, reg, try func.resolveInst(ref)),
7061	else => return func.fail("TODO: genSetReg {s}", .{@tagName(src_mcv)}),
7062	}
7063	}
7064
7065	fn genSetMem(
7066	func: *Func,
7067	base: Memory.Base,
7068	disp: i32,
7069	ty: Type,
7070	src_mcv: MCValue,
7071	) InnerError!void {
7072	const pt = func.pt;
7073	const zcu = pt.zcu;
7074
7075	const abi_size: u32 = @intCast(ty.abiSize(zcu));
7076	const dst_ptr_mcv: MCValue = switch (base) {
7077	.reg => |base_reg| .{ .register_offset = .{ .reg = base_reg, .off = disp } },
7078	.frame => |base_frame_index| .{ .lea_frame = .{ .index = base_frame_index, .off = disp } },
7079	};
7080	switch (src_mcv) {
7081	.none,
7082	.unreach,
7083	.dead,
7084	.reserved_frame,
7085	=> unreachable,
7086	.undef => |sym_index| {
7087	if (sym_index) |index| {
7088	return func.genSetMem(base, disp, ty, .{ .load_symbol = .{ .sym = index } });
7089	}
7090
7091	try func.genInlineMemset(
7092	dst_ptr_mcv,
7093	src_mcv,
7094	.{ .immediate = abi_size },
7095	);
7096	},
7097	.register_offset,
7098	.memory,
7099	.indirect,
7100	.load_frame,
7101	.lea_frame,
7102	.load_symbol,
7103	.lea_symbol,
7104	=> switch (abi_size) {
7105	0 => {},
7106	1, 2, 4, 8 => {
7107	const reg = try func.register_manager.allocReg(null, abi.Registers.Integer.temporary);
7108	const src_lock = func.register_manager.lockRegAssumeUnused(reg);
7109	defer func.register_manager.unlockReg(src_lock);
7110
7111	try func.genSetReg(ty, reg, src_mcv);
7112	try func.genSetMem(base, disp, ty, .{ .register = reg });
7113	},
7114	else => try func.genInlineMemcpy(
7115	dst_ptr_mcv,
7116	src_mcv.address(),
7117	.{ .immediate = abi_size },
7118	),
7119	},
7120	.register => |reg| {
7121	if (reg.class() == .vector) {
7122	const addr_reg = try func.copyToTmpRegister(Type.u64, dst_ptr_mcv);
7123
7124	const num_elem = ty.vectorLen(zcu);
7125	const elem_size = ty.childType(zcu).bitSize(zcu);
7126
7127	try func.setVl(.zero, num_elem, .{
7128	.vsew = switch (elem_size) {
7129	8 => .@"8",
7130	16 => .@"16",
7131	32 => .@"32",
7132	64 => .@"64",
7133	else => unreachable,
7134	},
7135	.vlmul = .m1,
7136	.vma = true,
7137	.vta = true,
7138	});
7139
7140	_ = try func.addInst(.{
7141	.tag = .pseudo_store_rm,
7142	.data = .{ .rm = .{
7143	.r = reg,
7144	.m = .{
7145	.base = .{ .reg = addr_reg },
7146	.mod = .{
7147	.disp = 0,
7148	.size = func.memSize(ty.childType(zcu)),
7149	.unsigned = false,
7150	},
7151	},
7152	} },
7153	});
7154
7155	return;
7156	}
7157
7158	const mem_size = switch (base) {
7159	.frame => |base_fi| mem_size: {
7160	assert(disp >= 0);
7161	const frame_abi_size = func.frame_allocs.items(.abi_size)[@backingInt(base_fi)];
7162	const frame_spill_pad = func.frame_allocs.items(.spill_pad)[@backingInt(base_fi)];
7163	assert(frame_abi_size - frame_spill_pad - disp >= abi_size);
7164	break :mem_size if (frame_abi_size - frame_spill_pad - disp == abi_size)
7165	frame_abi_size
7166	else
7167	abi_size;
7168	},
7169	else => abi_size,
7170	};
7171	const src_size = math.ceilPowerOfTwoAssert(u32, abi_size);
7172	const src_align = Alignment.fromNonzeroByteUnits(math.ceilPowerOfTwoAssert(u32, src_size));
7173	if (src_size > mem_size) {
7174	const frame_index = try func.allocFrameIndex(FrameAlloc.init(.{
7175	.size = src_size,
7176	.alignment = src_align,
7177	}));
7178	const frame_mcv: MCValue = .{ .load_frame = .{ .index = frame_index } };
7179	_ = try func.addInst(.{
7180	.tag = .pseudo_store_rm,
7181	.data = .{ .rm = .{
7182	.r = reg,
7183	.m = .{
7184	.base = .{ .frame = frame_index },
7185	.mod = .{
7186	.size = Memory.Size.fromByteSize(src_size),
7187	.unsigned = false,
7188	},
7189	},
7190	} },
7191	});
7192	try func.genSetMem(base, disp, ty, frame_mcv);
7193	try func.freeValue(frame_mcv);
7194	} else _ = try func.addInst(.{
7195	.tag = .pseudo_store_rm,
7196	.data = .{ .rm = .{
7197	.r = reg,
7198	.m = .{
7199	.base = base,
7200	.mod = .{
7201	.size = func.memSize(ty),
7202	.disp = disp,
7203	.unsigned = false,
7204	},
7205	},
7206	} },
7207	});
7208	},
7209	.register_pair => |src_regs| {
7210	var part_disp: i32 = disp;
7211	for (try func.splitType(ty), src_regs) |src_ty, src_reg| {
7212	try func.genSetMem(base, part_disp, src_ty, .{ .register = src_reg });
7213	part_disp += @intCast(src_ty.abiSize(zcu));
7214	}
7215	},
7216	.immediate => {
7217	// TODO: remove this lock in favor of a copyToTmpRegister when we load 64 bit immediates with
7218	// a register allocation.
7219	const reg, const reg_lock = try func.promoteReg(ty, src_mcv);
7220	defer if (reg_lock) |lock| func.register_manager.unlockReg(lock);
7221
7222	return func.genSetMem(base, disp, ty, .{ .register = reg });
7223	},
7224	.air_ref => |src_ref| try func.genSetMem(base, disp, ty, try func.resolveInst(src_ref)),
7225	}
7226	}
7227
7228	fn airBitCast(func: *Func, inst: Air.Inst.Index) !void {
7229	const pt = func.pt;
7230	const zcu = pt.zcu;
7231
7232	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
7233	const result = if (func.liveness.isUnused(inst)) .unreach else result: {
7234	const src_mcv = try func.resolveInst(ty_op.operand);
7235
7236	const src_ty = func.typeOf(ty_op.operand);
7237	if (src_ty.toIntern() == .bool_type) break :result src_mcv;
7238	const dst_ty = func.typeOfIndex(inst);
7239
7240	const src_lock = if (src_mcv.getReg()) |reg| func.register_manager.lockReg(reg) else null;
7241	defer if (src_lock) |lock| func.register_manager.unlockReg(lock);
7242
7243	const dst_mcv = if (dst_ty.abiSize(zcu) <= src_ty.abiSize(zcu) and src_mcv != .register_pair and
7244	func.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {
7245	const dst_mcv = try func.allocRegOrMem(dst_ty, inst, true);
7246	try func.genCopy(switch (math.order(dst_ty.abiSize(zcu), src_ty.abiSize(zcu))) {
7247	.lt => dst_ty,
7248	.eq => if (!dst_mcv.isMemory() or src_mcv.isMemory()) dst_ty else src_ty,
7249	.gt => src_ty,
7250	}, dst_mcv, src_mcv);
7251	break :dst dst_mcv;
7252	};
7253
7254	if (dst_ty.isAbiInt(zcu) and src_ty.isAbiInt(zcu) and
7255	dst_ty.intInfo(zcu).signedness == src_ty.intInfo(zcu).signedness) break :result dst_mcv;
7256
7257	const abi_size = dst_ty.abiSize(zcu);
7258	const bit_size = dst_ty.bitSize(zcu);
7259	if (abi_size * 8 <= bit_size) break :result dst_mcv;
7260
7261	return func.fail("TODO: airBitCast {f} to {f}", .{ src_ty.fmt(pt), dst_ty.fmt(pt) });
7262	};
7263	return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
7264	}
7265
7266	fn airArrayToSlice(func: *Func, inst: Air.Inst.Index) !void {
7267	const pt = func.pt;
7268	const zcu = pt.zcu;
7269	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
7270
7271	const slice_ty = func.typeOfIndex(inst);
7272	const ptr_ty = func.typeOf(ty_op.operand);
7273	const ptr = try func.resolveInst(ty_op.operand);
7274	const array_ty = ptr_ty.childType(zcu);
7275	const array_len = array_ty.arrayLen(zcu);
7276
7277	const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(slice_ty, zcu));
7278	try func.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr);
7279	try func.genSetMem(
7280	.{ .frame = frame_index },
7281	@intCast(ptr_ty.abiSize(zcu)),
7282	Type.u64,
7283	.{ .immediate = array_len },
7284	);
7285
7286	const result = MCValue{ .load_frame = .{ .index = frame_index } };
7287	return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
7288	}
7289
7290	fn airFloatFromInt(func: *Func, inst: Air.Inst.Index) !void {
7291	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
7292	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
7293	const pt = func.pt;
7294	const zcu = pt.zcu;
7295
7296	const operand = try func.resolveInst(ty_op.operand);
7297
7298	const src_ty = func.typeOf(ty_op.operand);
7299	const dst_ty = ty_op.ty;
7300
7301	const src_reg, const src_lock = try func.promoteReg(src_ty, operand);
7302	defer if (src_lock) |lock| func.register_manager.unlockReg(lock);
7303
7304	const is_unsigned = dst_ty.isUnsignedInt(zcu);
7305	const src_bits = src_ty.bitSize(zcu);
7306	const dst_bits = dst_ty.bitSize(zcu);
7307
7308	switch (src_bits) {
7309	32, 64 => {},
7310	else => try func.truncateRegister(src_ty, src_reg),
7311	}
7312
7313	const int_zcu: Mir.FcvtOp = switch (src_bits) {
7314	8, 16, 32 => if (is_unsigned) .wu else .w,
7315	64 => if (is_unsigned) .lu else .l,
7316	else => return func.fail("TODO: airFloatFromInt src size: {d}", .{src_bits}),
7317	};
7318
7319	const float_zcu: enum { s, d } = switch (dst_bits) {
7320	32 => .s,
7321	64 => .d,
7322	else => return func.fail("TODO: airFloatFromInt dst size {d}", .{dst_bits}),
7323	};
7324
7325	const dst_reg, const dst_lock = try func.allocReg(.int);
7326	defer func.register_manager.unlockReg(dst_lock);
7327
7328	_ = try func.addInst(.{
7329	.tag = switch (float_zcu) {
7330	.s => switch (int_zcu) {
7331	.l => .fcvtsl,
7332	.lu => .fcvtslu,
7333	.w => .fcvtsw,
7334	.wu => .fcvtswu,
7335	},
7336	.d => switch (int_zcu) {
7337	.l => .fcvtdl,
7338	.lu => .fcvtdlu,
7339	.w => .fcvtdw,
7340	.wu => .fcvtdwu,
7341	},
7342	},
7343	.data = .{ .rr = .{
7344	.rd = dst_reg,
7345	.rs = src_reg,
7346	} },
7347	});
7348
7349	break :result .{ .register = dst_reg };
7350	};
7351	return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
7352	}
7353
7354	fn airIntFromFloat(func: *Func, inst: Air.Inst.Index) !void {
7355	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
7356	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
7357	const pt = func.pt;
7358	const zcu = pt.zcu;
7359
7360	const operand = try func.resolveInst(ty_op.operand);
7361	const src_ty = func.typeOf(ty_op.operand);
7362	const dst_ty = ty_op.ty;
7363
7364	const is_unsigned = dst_ty.isUnsignedInt(zcu);
7365	const src_bits = src_ty.bitSize(zcu);
7366	const dst_bits = dst_ty.bitSize(zcu);
7367
7368	const float_zcu: enum { s, d } = switch (src_bits) {
7369	32 => .s,
7370	64 => .d,
7371	else => return func.fail("TODO: airIntFromFloat src size {d}", .{src_bits}),
7372	};
7373
7374	const int_zcu: Mir.FcvtOp = switch (dst_bits) {
7375	32 => if (is_unsigned) .wu else .w,
7376	8, 16, 64 => if (is_unsigned) .lu else .l,
7377	else => return func.fail("TODO: airIntFromFloat dst size: {d}", .{dst_bits}),
7378	};
7379
7380	const src_reg, const src_lock = try func.promoteReg(src_ty, operand);
7381	defer if (src_lock) |lock| func.register_manager.unlockReg(lock);
7382
7383	const dst_reg, const dst_lock = try func.allocReg(.int);
7384	defer func.register_manager.unlockReg(dst_lock);
7385
7386	_ = try func.addInst(.{
7387	.tag = switch (float_zcu) {
7388	.s => switch (int_zcu) {
7389	.l => .fcvtls,
7390	.lu => .fcvtlus,
7391	.w => .fcvtws,
7392	.wu => .fcvtwus,
7393	},
7394	.d => switch (int_zcu) {
7395	.l => .fcvtld,
7396	.lu => .fcvtlud,
7397	.w => .fcvtwd,
7398	.wu => .fcvtwud,
7399	},
7400	},
7401	.data = .{ .rr = .{
7402	.rd = dst_reg,
7403	.rs = src_reg,
7404	} },
7405	});
7406
7407	break :result .{ .register = dst_reg };
7408	};
7409	return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
7410	}
7411
7412	fn airCmpxchg(func: *Func, inst: Air.Inst.Index, strength: enum { weak, strong }) !void {
7413	_ = strength; // TODO: do something with this
7414
7415	const pt = func.pt;
7416	const zcu = pt.zcu;
7417	const ty_pl = func.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
7418	const extra = func.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
7419
7420	const ptr_ty = func.typeOf(extra.ptr);
7421	const val_ty = func.typeOf(extra.expected_value);
7422	const val_abi_size: u32 = @intCast(val_ty.abiSize(pt.zcu));
7423
7424	switch (val_abi_size) {
7425	1, 2, 4, 8 => {},
7426	else => return func.fail("TODO: airCmpxchg Int size {}", .{val_abi_size}),
7427	}
7428
7429	const lr_order: struct { aq: Mir.Barrier, rl: Mir.Barrier } = switch (extra.successOrder()) {
7430	.unordered,
7431	=> unreachable,
7432
7433	.monotonic,
7434	.release,
7435	=> .{ .aq = .none, .rl = .none },
7436	.acquire,
7437	.acq_rel,
7438	=> .{ .aq = .aq, .rl = .none },
7439	.seq_cst => .{ .aq = .aq, .rl = .rl },
7440	};
7441
7442	const sc_order: struct { aq: Mir.Barrier, rl: Mir.Barrier } = switch (extra.failureOrder()) {
7443	.unordered,
7444	.release,
7445	.acq_rel,
7446	=> unreachable,
7447
7448	.monotonic,
7449	.acquire,
7450	.seq_cst,
7451	=> switch (extra.successOrder()) {
7452	.release,
7453	.seq_cst,
7454	=> .{ .aq = .none, .rl = .rl },
7455	else => .{ .aq = .none, .rl = .none },
7456	},
7457	};
7458
7459	const ptr_mcv = try func.resolveInst(extra.ptr);
7460	const ptr_reg, const ptr_lock = try func.promoteReg(ptr_ty, ptr_mcv);
7461	defer if (ptr_lock) |lock| func.register_manager.unlockReg(lock);
7462
7463	const exp_mcv = try func.resolveInst(extra.expected_value);
7464	const exp_reg, const exp_lock = try func.promoteReg(val_ty, exp_mcv);
7465	defer if (exp_lock) |lock| func.register_manager.unlockReg(lock);
7466	try func.truncateRegister(val_ty, exp_reg);
7467
7468	const new_mcv = try func.resolveInst(extra.new_value);
7469	const new_reg, const new_lock = try func.promoteReg(val_ty, new_mcv);
7470	defer if (new_lock) |lock| func.register_manager.unlockReg(lock);
7471	try func.truncateRegister(val_ty, new_reg);
7472
7473	const branch_reg, const branch_lock = try func.allocReg(.int);
7474	defer func.register_manager.unlockReg(branch_lock);
7475
7476	const fallthrough_reg, const fallthrough_lock = try func.allocReg(.int);
7477	defer func.register_manager.unlockReg(fallthrough_lock);
7478
7479	const jump_back = try func.addInst(.{
7480	.tag = if (val_ty.bitSize(zcu) <= 32) .lrw else .lrd,
7481	.data = .{ .amo = .{
7482	.aq = lr_order.aq,
7483	.rl = lr_order.rl,
7484	.rd = branch_reg,
7485	.rs1 = ptr_reg,
7486	.rs2 = .zero,
7487	} },
7488	});
7489	try func.truncateRegister(val_ty, branch_reg);
7490
7491	const jump_forward = try func.addInst(.{
7492	.tag = .bne,
7493	.data = .{ .b_type = .{
7494	.rs1 = branch_reg,
7495	.rs2 = exp_reg,
7496	.inst = undefined,
7497	} },
7498	});
7499
7500	_ = try func.addInst(.{
7501	.tag = if (val_ty.bitSize(zcu) <= 32) .scw else .scd,
7502	.data = .{ .amo = .{
7503	.aq = sc_order.aq,
7504	.rl = sc_order.rl,
7505	.rd = fallthrough_reg,
7506	.rs1 = ptr_reg,
7507	.rs2 = new_reg,
7508	} },
7509	});
7510	try func.truncateRegister(Type.bool, fallthrough_reg);
7511
7512	_ = try func.addInst(.{
7513	.tag = .bne,
7514	.data = .{ .b_type = .{
7515	.rs1 = fallthrough_reg,
7516	.rs2 = .zero,
7517	.inst = jump_back,
7518	} },
7519	});
7520
7521	func.performReloc(jump_forward);
7522
7523	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
7524	const dst_mcv = try func.allocRegOrMem(func.typeOfIndex(inst), inst, false);
7525
7526	const tmp_reg, const tmp_lock = try func.allocReg(.int);
7527	defer func.register_manager.unlockReg(tmp_lock);
7528
7529	try func.genBinOp(
7530	.cmp_neq,
7531	.{ .register = branch_reg },
7532	val_ty,
7533	.{ .register = exp_reg },
7534	val_ty,
7535	tmp_reg,
7536	);
7537
7538	try func.genCopy(val_ty, dst_mcv, .{ .register = branch_reg });
7539	try func.genCopy(
7540	Type.bool,
7541	dst_mcv.address().offset(@intCast(val_abi_size)).deref(),
7542	.{ .register = tmp_reg },
7543	);
7544
7545	break :result dst_mcv;
7546	};
7547
7548	return func.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });
7549	}
7550
7551	fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void {
7552	const pt = func.pt;
7553	const zcu = pt.zcu;
7554	const pl_op = func.air.instructions.items(.data)[@backingInt(inst)].pl_op;
7555	const extra = func.air.extraData(Air.AtomicRmw, pl_op.payload).data;
7556
7557	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
7558	const op = extra.op();
7559	const order = extra.ordering();
7560
7561	const ptr_ty = func.typeOf(pl_op.operand);
7562	const ptr_mcv = try func.resolveInst(pl_op.operand);
7563
7564	const val_ty = func.typeOf(extra.operand);
7565	const val_size = val_ty.abiSize(zcu);
7566	const val_mcv = try func.resolveInst(extra.operand);
7567
7568	if (!math.isPowerOfTwo(val_size))
7569	return func.fail("TODO: airAtomicRmw non-pow 2", .{});
7570
7571	switch (val_ty.zigTypeTag(pt.zcu)) {
7572	.@"enum", .int => {},
7573	inline .bool, .float, .pointer => |ty| return func.fail("TODO: airAtomicRmw {s}", .{@tagName(ty)}),
7574	else => unreachable,
7575	}
7576
7577	const method: enum { amo, loop } = switch (val_size) {
7578	1, 2 => .loop,
7579	4, 8 => .amo,
7580	else => unreachable,
7581	};
7582
7583	const ptr_register, const ptr_lock = try func.promoteReg(ptr_ty, ptr_mcv);
7584	defer if (ptr_lock) |lock| func.register_manager.unlockReg(lock);
7585
7586	const val_register, const val_lock = try func.promoteReg(val_ty, val_mcv);
7587	defer if (val_lock) |lock| func.register_manager.unlockReg(lock);
7588
7589	const result_mcv = try func.allocRegOrMem(val_ty, inst, true);
7590	assert(result_mcv == .register); // should fit into 8 bytes
7591	const result_reg = result_mcv.register;
7592
7593	const aq, const rl = switch (order) {
7594	.unordered => unreachable,
7595	.monotonic => .{ false, false },
7596	.acquire => .{ true, false },
7597	.release => .{ false, true },
7598	.acq_rel => .{ true, true },
7599	.seq_cst => .{ true, true },
7600	};
7601
7602	switch (method) {
7603	.amo => {
7604	const is_d = val_ty.abiSize(zcu) == 8;
7605	const is_un = val_ty.isUnsignedInt(zcu);
7606
7607	const mnem: Mnemonic = switch (op) {
7608	// zig fmt: off
7609	.Xchg => if (is_d) .amoswapd else .amoswapw,
7610	.Add => if (is_d) .amoaddd else .amoaddw,
7611	.And => if (is_d) .amoandd else .amoandw,
7612	.Or => if (is_d) .amoord else .amoorw,
7613	.Xor => if (is_d) .amoxord else .amoxorw,
7614	.Max => if (is_d) if (is_un) .amomaxud else .amomaxd else if (is_un) .amomaxuw else .amomaxw,
7615	.Min => if (is_d) if (is_un) .amominud else .amomind else if (is_un) .amominuw else .amominw,
7616	else => return func.fail("TODO: airAtomicRmw amo {s}", .{@tagName(op)}),
7617	// zig fmt: on
7618	};
7619
7620	_ = try func.addInst(.{
7621	.tag = mnem,
7622	.data = .{ .amo = .{
7623	.rd = result_reg,
7624	.rs1 = ptr_register,
7625	.rs2 = val_register,
7626	.aq = if (aq) .aq else .none,
7627	.rl = if (rl) .rl else .none,
7628	} },
7629	});
7630	},
7631	.loop => {
7632	// where we'll jump back when the sc fails
7633	const jump_back = try func.addInst(.{
7634	.tag = .lrw,
7635	.data = .{ .amo = .{
7636	.rd = result_reg,
7637	.rs1 = ptr_register,
7638	.rs2 = .zero,
7639	.aq = if (aq) .aq else .none,
7640	.rl = if (rl) .rl else .none,
7641	} },
7642	});
7643
7644	const after_reg, const after_lock = try func.allocReg(.int);
7645	defer func.register_manager.unlockReg(after_lock);
7646
7647	switch (op) {
7648	.Add, .Sub => |tag| {
7649	_ = try func.genBinOp(
7650	switch (tag) {
7651	.Add => .add,
7652	.Sub => .sub,
7653	else => unreachable,
7654	},
7655	.{ .register = result_reg },
7656	val_ty,
7657	.{ .register = val_register },
7658	val_ty,
7659	after_reg,
7660	);
7661	},
7662
7663	else => return func.fail("TODO: airAtomicRmw loop {s}", .{@tagName(op)}),
7664	}
7665
7666	_ = try func.addInst(.{
7667	.tag = .scw,
7668	.data = .{ .amo = .{
7669	.rd = after_reg,
7670	.rs1 = ptr_register,
7671	.rs2 = after_reg,
7672	.aq = if (aq) .aq else .none,
7673	.rl = if (rl) .rl else .none,
7674	} },
7675	});
7676
7677	_ = try func.addInst(.{
7678	.tag = .bne,
7679	.data = .{ .b_type = .{
7680	.inst = jump_back,
7681	.rs1 = after_reg,
7682	.rs2 = .zero,
7683	} },
7684	});
7685	},
7686	}
7687	break :result result_mcv;
7688	};
7689
7690	return func.finishAir(inst, result, .{ pl_op.operand, extra.operand, .none });
7691	}
7692
7693	fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void {
7694	const pt = func.pt;
7695	const zcu = pt.zcu;
7696	const atomic_load = func.air.instructions.items(.data)[@backingInt(inst)].atomic_load;
7697	const order: std.lang.AtomicOrder = atomic_load.order;
7698
7699	const ptr_ty = func.typeOf(atomic_load.ptr);
7700	const elem_ty = ptr_ty.childType(zcu);
7701	const ptr_mcv = try func.resolveInst(atomic_load.ptr);
7702
7703	const bit_size = elem_ty.bitSize(zcu);
7704	if (bit_size > 64) return func.fail("TODO: airAtomicLoad > 64 bits", .{});
7705
7706	const result_mcv: MCValue = if (func.liveness.isUnused(inst))
7707	.{ .register = .zero }
7708	else
7709	try func.allocRegOrMem(elem_ty, inst, true);
7710	assert(result_mcv == .register); // should be less than 8 bytes
7711
7712	if (order == .seq_cst) {
7713	_ = try func.addInst(.{
7714	.tag = .fence,
7715	.data = .{ .fence = .{
7716	.pred = .rw,
7717	.succ = .rw,
7718	} },
7719	});
7720	}
7721
7722	try func.load(result_mcv, ptr_mcv, ptr_ty);
7723
7724	switch (order) {
7725	// Don't guarantee other memory operations to be ordered after the load.
7726	.unordered, .monotonic => {},
7727	// Make sure all previous reads happen before any reading or writing occurs.
7728	.acquire, .seq_cst => {
7729	_ = try func.addInst(.{
7730	.tag = .fence,
7731	.data = .{ .fence = .{
7732	.pred = .r,
7733	.succ = .rw,
7734	} },
7735	});
7736	},
7737	else => unreachable,
7738	}
7739
7740	return func.finishAir(inst, result_mcv, .{ atomic_load.ptr, .none, .none });
7741	}
7742
7743	fn airAtomicStore(func: *Func, inst: Air.Inst.Index, order: std.lang.AtomicOrder) !void {
7744	const bin_op = func.air.instructions.items(.data)[@backingInt(inst)].bin_op;
7745
7746	const ptr_ty = func.typeOf(bin_op.lhs);
7747	const ptr_mcv = try func.resolveInst(bin_op.lhs);
7748
7749	const val_ty = func.typeOf(bin_op.rhs);
7750	const val_mcv = try func.resolveInst(bin_op.rhs);
7751
7752	const bit_size = val_ty.bitSize(func.pt.zcu);
7753	if (bit_size > 64) return func.fail("TODO: airAtomicStore > 64 bits", .{});
7754
7755	switch (order) {
7756	.unordered, .monotonic => {},
7757	.release, .seq_cst => {
7758	_ = try func.addInst(.{
7759	.tag = .fence,
7760	.data = .{ .fence = .{
7761	.pred = .rw,
7762	.succ = .w,
7763	} },
7764	});
7765	},
7766	else => unreachable,
7767	}
7768
7769	try func.store(ptr_mcv, val_mcv, ptr_ty);
7770
7771	if (order == .seq_cst) {
7772	_ = try func.addInst(.{
7773	.tag = .fence,
7774	.data = .{ .fence = .{
7775	.pred = .rw,
7776	.succ = .rw,
7777	} },
7778	});
7779	}
7780
7781	return func.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
7782	}
7783
7784	fn airMemset(func: *Func, inst: Air.Inst.Index, safety: bool) !void {
7785	const pt = func.pt;
7786	const zcu = pt.zcu;
7787	const bin_op = func.air.instructions.items(.data)[@backingInt(inst)].bin_op;
7788
7789	result: {
7790	if (!safety and (try func.resolveInst(bin_op.rhs)) == .undef) break :result;
7791
7792	const dst_ptr = try func.resolveInst(bin_op.lhs);
7793	const dst_ptr_ty = func.typeOf(bin_op.lhs);
7794	const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) {
7795	.register => |reg| func.register_manager.lockRegAssumeUnused(reg),
7796	else => null,
7797	};
7798	defer if (dst_ptr_lock) |lock| func.register_manager.unlockReg(lock);
7799
7800	const src_val = try func.resolveInst(bin_op.rhs);
7801	const elem_ty = func.typeOf(bin_op.rhs);
7802	const src_val_lock: ?RegisterLock = switch (src_val) {
7803	.register => |reg| func.register_manager.lockRegAssumeUnused(reg),
7804	else => null,
7805	};
7806	defer if (src_val_lock) |lock| func.register_manager.unlockReg(lock);
7807
7808	const elem_abi_size: u31 = @intCast(elem_ty.abiSize(zcu));
7809
7810	if (elem_abi_size == 1) {
7811	const ptr: MCValue = switch (dst_ptr_ty.ptrSize(zcu)) {
7812	// TODO: this only handles slices stored in the stack
7813	.slice => dst_ptr,
7814	.one => dst_ptr,
7815	.c, .many => unreachable,
7816	};
7817	const len: MCValue = switch (dst_ptr_ty.ptrSize(zcu)) {
7818	// TODO: this only handles slices stored in the stack
7819	.slice => dst_ptr.address().offset(8).deref(),
7820	.one => .{ .immediate = dst_ptr_ty.childType(zcu).arrayLen(zcu) },
7821	.c, .many => unreachable,
7822	};
7823	const len_lock: ?RegisterLock = switch (len) {
7824	.register => |reg| func.register_manager.lockRegAssumeUnused(reg),
7825	else => null,
7826	};
7827	defer if (len_lock) |lock| func.register_manager.unlockReg(lock);
7828
7829	try func.genInlineMemset(ptr, src_val, len);
7830	break :result;
7831	}
7832
7833	// Store the first element, and then rely on memcpy copying forwards.
7834	// Length zero requires a runtime check - so we handle arrays specially
7835	// here to elide it.
7836	switch (dst_ptr_ty.ptrSize(zcu)) {
7837	.slice => return func.fail("TODO: airMemset Slices", .{}),
7838	.one => {
7839	const elem_ptr_ty = try pt.singleMutPtrType(elem_ty);
7840
7841	const len = dst_ptr_ty.childType(zcu).arrayLen(zcu);
7842
7843	assert(len != 0); // prevented by Sema
7844	try func.store(dst_ptr, src_val, elem_ptr_ty);
7845
7846	const second_elem_ptr_reg, const second_elem_ptr_lock = try func.allocReg(.int);
7847	defer func.register_manager.unlockReg(second_elem_ptr_lock);
7848
7849	const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg };
7850
7851	try func.genSetReg(Type.u64, second_elem_ptr_reg, .{ .register_offset = .{
7852	.reg = try func.copyToTmpRegister(Type.u64, dst_ptr),
7853	.off = elem_abi_size,
7854	} });
7855
7856	const bytes_to_copy: MCValue = .{ .immediate = elem_abi_size * (len - 1) };
7857	try func.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, bytes_to_copy);
7858	},
7859	.c, .many => unreachable,
7860	}
7861	}
7862	return func.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
7863	}
7864
7865	fn airMemcpy(func: *Func, inst: Air.Inst.Index) !void {
7866	const pt = func.pt;
7867	const zcu = pt.zcu;
7868	const bin_op = func.air.instructions.items(.data)[@backingInt(inst)].bin_op;
7869
7870	const dst_ptr = try func.resolveInst(bin_op.lhs);
7871	const src_ptr = try func.resolveInst(bin_op.rhs);
7872
7873	const dst_ty = func.typeOf(bin_op.lhs);
7874
7875	const len_mcv: MCValue = switch (dst_ty.ptrSize(zcu)) {
7876	.slice => len: {
7877	const len_reg, const len_lock = try func.allocReg(.int);
7878	defer func.register_manager.unlockReg(len_lock);
7879
7880	const elem_size = dst_ty.childType(zcu).abiSize(zcu);
7881	try func.genBinOp(
7882	.mul,
7883	.{ .immediate = elem_size },
7884	Type.u64,
7885	dst_ptr.address().offset(8).deref(),
7886	Type.u64,
7887	len_reg,
7888	);
7889	break :len .{ .register = len_reg };
7890	},
7891	.one => len: {
7892	const array_ty = dst_ty.childType(zcu);
7893	break :len .{ .immediate = array_ty.arrayLen(zcu) * array_ty.childType(zcu).abiSize(zcu) };
7894	},
7895	else => |size| return func.fail("TODO: airMemcpy size {s}", .{@tagName(size)}),
7896	};
7897	const len_lock: ?RegisterLock = switch (len_mcv) {
7898	.register => |reg| func.register_manager.lockRegAssumeUnused(reg),
7899	else => null,
7900	};
7901	defer if (len_lock) |lock| func.register_manager.unlockReg(lock);
7902
7903	try func.genInlineMemcpy(dst_ptr, src_ptr, len_mcv);
7904
7905	return func.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
7906	}
7907
7908	fn airMemmove(func: *Func, inst: Air.Inst.Index) !void {
7909	_ = inst;
7910	return func.fail("TODO implement airMemmove for riscv64", .{});
7911	}
7912
7913	fn airTagName(func: *Func, inst: Air.Inst.Index) !void {
7914	const pt = func.pt;
7915
7916	const un_op = func.air.instructions.items(.data)[@backingInt(inst)].un_op;
7917	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
7918	const enum_ty = func.typeOf(un_op);
7919
7920	// TODO: work out the bugs
7921	if (true) return func.fail("TODO: airTagName", .{});
7922
7923	const param_regs = abi.Registers.Integer.function_arg_regs;
7924	const dst_mcv = try func.allocRegOrMem(Type.u64, inst, false);
7925	try func.genSetReg(Type.u64, param_regs[0], dst_mcv.address());
7926
7927	const operand = try func.resolveInst(un_op);
7928	try func.genSetReg(enum_ty, param_regs[1], operand);
7929
7930	const lazy_sym: link.File.LazySymbol = .{ .kind = .code, .ty = enum_ty.toIntern() };
7931	const elf_file = func.bin_file.cast(link.File.Elf).?;
7932	const zo = elf_file.zigObjectPtr().?;
7933	const sym_index = zo.getOrCreateMetadataForLazySymbol(elf_file, pt, lazy_sym) catch |err|
7934	return func.fail("{s} creating lazy symbol", .{@errorName(err)});
7935
7936	if (func.mod.pic) {
7937	return func.fail("TODO: airTagName pic", .{});
7938	} else {
7939	try func.genSetReg(Type.u64, .ra, .{ .load_symbol = .{ .sym = sym_index } });
7940	_ = try func.addInst(.{
7941	.tag = .jalr,
7942	.data = .{ .i_type = .{
7943	.rd = .ra,
7944	.rs1 = .ra,
7945	.imm12 = Immediate.s(0),
7946	} },
7947	});
7948	}
7949
7950	break :result dst_mcv;
7951	};
7952	return func.finishAir(inst, result, .{ un_op, .none, .none });
7953	}
7954
7955	fn airErrorName(func: *Func, inst: Air.Inst.Index) !void {
7956	_ = inst;
7957	return func.fail("TODO: airErrorName", .{});
7958	}
7959
7960	fn airSplat(func: *Func, inst: Air.Inst.Index) !void {
7961	const ty_op = func.air.instructions.items(.data)[@backingInt(inst)].ty_op;
7962	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airSplat for riscv64", .{});
7963	return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
7964	}
7965
7966	fn airSelect(func: *Func, inst: Air.Inst.Index) !void {
7967	const pl_op = func.air.instructions.items(.data)[@backingInt(inst)].pl_op;
7968	const extra = func.air.extraData(Air.Bin, pl_op.payload).data;
7969	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airSelect for riscv64", .{});
7970	return func.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs });
7971	}
7972
7973	fn airShuffleOne(func: *Func, inst: Air.Inst.Index) !void {
7974	_ = inst;
7975	return func.fail("TODO implement airShuffleOne for riscv64", .{});
7976	}
7977
7978	fn airShuffleTwo(func: *Func, inst: Air.Inst.Index) !void {
7979	_ = inst;
7980	return func.fail("TODO implement airShuffleTwo for riscv64", .{});
7981	}
7982
7983	fn airReduce(func: *Func, inst: Air.Inst.Index) !void {
7984	const reduce = func.air.instructions.items(.data)[@backingInt(inst)].reduce;
7985	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airReduce for riscv64", .{});
7986	return func.finishAir(inst, result, .{ reduce.operand, .none, .none });
7987	}
7988
7989	fn airAggregateInit(func: *Func, inst: Air.Inst.Index) !void {
7990	const pt = func.pt;
7991	const zcu = pt.zcu;
7992	const result_ty = func.typeOfIndex(inst);
7993	const len: usize = @intCast(result_ty.arrayLen(zcu));
7994	const ty_pl = func.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
7995	const elements: []const Air.Inst.Ref = @ptrCast(func.air.extra.items[ty_pl.payload..][0..len]);
7996
7997	const result: MCValue = result: {
7998	switch (result_ty.zigTypeTag(zcu)) {
7999	.@"struct" => {
8000	const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(result_ty, zcu));
8001	if (result_ty.containerLayout(zcu) == .@"packed") {
8002	const struct_obj = zcu.typeToStruct(result_ty).?;
8003	try func.genInlineMemset(
8004	.{ .lea_frame = .{ .index = frame_index } },
8005	.{ .immediate = 0 },
8006	.{ .immediate = result_ty.abiSize(zcu) },
8007	);
8008
8009	for (elements, 0..) |elem, elem_i_usize| {
8010	const elem_i: u32 = @intCast(elem_i_usize);
8011	if ((try result_ty.structFieldValueComptime(pt, elem_i)) != null) continue;
8012
8013	const elem_ty = result_ty.fieldType(elem_i, zcu);
8014	const elem_bit_size: u32 = @intCast(elem_ty.bitSize(zcu));
8015	if (elem_bit_size > 64) {
8016	return func.fail(
8017	"TODO airAggregateInit implement packed structs with large fields",
8018	.{},
8019	);
8020	}
8021
8022	const elem_abi_size: u32 = @intCast(elem_ty.abiSize(zcu));
8023	const elem_abi_bits = elem_abi_size * 8;
8024	const elem_off = zcu.structPackedFieldBitOffset(struct_obj, elem_i);
8025	const elem_byte_off: i32 = @intCast(elem_off / elem_abi_bits * elem_abi_size);
8026	const elem_bit_off = elem_off % elem_abi_bits;
8027	const elem_mcv = try func.resolveInst(elem);
8028
8029	_ = elem_byte_off;
8030	_ = elem_bit_off;
8031
8032	const elem_lock = switch (elem_mcv) {
8033	.register => |reg| func.register_manager.lockReg(reg),
8034	.immediate => |imm| lock: {
8035	if (imm == 0) continue;
8036	break :lock null;
8037	},
8038	else => null,
8039	};
8040	defer if (elem_lock) |lock| func.register_manager.unlockReg(lock);
8041
8042	return func.fail("TODO: airAggregateInit packed structs", .{});
8043	}
8044	} else for (elements, 0..) |elem, elem_i| {
8045	if ((try result_ty.structFieldValueComptime(pt, elem_i)) != null) continue;
8046
8047	const elem_ty = result_ty.fieldType(elem_i, zcu);
8048	const elem_off: i32 = @intCast(result_ty.structFieldOffset(elem_i, zcu));
8049	const elem_mcv = try func.resolveInst(elem);
8050	try func.genSetMem(.{ .frame = frame_index }, elem_off, elem_ty, elem_mcv);
8051	}
8052	break :result .{ .load_frame = .{ .index = frame_index } };
8053	},
8054	.array => {
8055	const elem_ty = result_ty.childType(zcu);
8056	const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(result_ty, zcu));
8057	const elem_size: u32 = @intCast(elem_ty.abiSize(zcu));
8058
8059	for (elements, 0..) |elem, elem_i| {
8060	const elem_mcv = try func.resolveInst(elem);
8061	const elem_off: i32 = @intCast(elem_size * elem_i);
8062	try func.genSetMem(
8063	.{ .frame = frame_index },
8064	elem_off,
8065	elem_ty,
8066	elem_mcv,
8067	);
8068	}
8069	if (result_ty.sentinel(zcu)) |sentinel| try func.genSetMem(
8070	.{ .frame = frame_index },
8071	@intCast(elem_size * elements.len),
8072	elem_ty,
8073	try func.genTypedValue(sentinel),
8074	);
8075	break :result .{ .load_frame = .{ .index = frame_index } };
8076	},
8077	else => return func.fail("TODO: airAggregate {f}", .{result_ty.fmt(pt)}),
8078	}
8079	};
8080
8081	if (elements.len <= Air.Liveness.bpi - 1) {
8082	var buf: [Air.Liveness.bpi - 1]Air.Inst.Ref = @splat(.none);
8083	@memcpy(buf[0..elements.len], elements);
8084	return func.finishAir(inst, result, buf);
8085	}
8086	var bt = func.liveness.iterateBigTomb(inst);
8087	for (elements) |elem| try func.feed(&bt, elem);
8088	return func.finishAirResult(inst, result);
8089	}
8090
8091	fn airUnionInit(func: *Func, inst: Air.Inst.Index) !void {
8092	const ty_pl = func.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
8093	const extra = func.air.extraData(Air.UnionInit, ty_pl.payload).data;
8094	_ = extra;
8095	return func.fail("TODO implement airUnionInit for riscv64", .{});
8096	// return func.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });
8097	}
8098
8099	fn airPrefetch(func: *Func, inst: Air.Inst.Index) !void {
8100	const prefetch = func.air.instructions.items(.data)[@backingInt(inst)].prefetch;
8101	// TODO: RISC-V does have prefetch instruction variants.
8102	// see here: https://raw.githubusercontent.com/riscv/riscv-CMOs/master/specifications/cmobase-v1.0.1.pdf
8103	return func.finishAir(inst, .unreach, .{ prefetch.ptr, .none, .none });
8104	}
8105
8106	fn airMulAdd(func: *Func, inst: Air.Inst.Index) !void {
8107	const pl_op = func.air.instructions.items(.data)[@backingInt(inst)].pl_op;
8108	const extra = func.air.extraData(Air.Bin, pl_op.payload).data;
8109	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else {
8110	return func.fail("TODO implement airMulAdd for riscv64", .{});
8111	};
8112	return func.finishAir(inst, result, .{ extra.lhs, extra.rhs, pl_op.operand });
8113	}
8114
8115	fn resolveInst(func: *Func, ref: Air.Inst.Ref) InnerError!MCValue {
8116	const pt = func.pt;
8117	const zcu = pt.zcu;
8118
8119	// If the type has no codegen bits, no need to store it.
8120	const inst_ty = func.typeOf(ref);
8121	if (!inst_ty.hasRuntimeBits(zcu))
8122	return .none;
8123
8124	const mcv = if (ref.toIndex()) |inst| mcv: {
8125	break :mcv func.inst_tracking.getPtr(inst).?.short;
8126	} else mcv: {
8127	const ip_index = ref.toInterned().?;
8128	const gop = try func.const_tracking.getOrPut(func.gpa, ip_index);
8129	if (!gop.found_existing) gop.value_ptr.* = InstTracking.init(
8130	try func.genTypedValue(Value.fromInterned(ip_index)),
8131	);
8132	break :mcv gop.value_ptr.short;
8133	};
8134
8135	return mcv;
8136	}
8137
8138	fn getResolvedInstValue(func: *Func, inst: Air.Inst.Index) *InstTracking {
8139	const tracking = func.inst_tracking.getPtr(inst).?;
8140	return switch (tracking.short) {
8141	.none, .unreach, .dead => unreachable,
8142	else => tracking,
8143	};
8144	}
8145
8146	fn genTypedValue(func: *Func, val: Value) InnerError!MCValue {
8147	const pt = func.pt;
8148
8149	const lf = func.bin_file;
8150
8151	const result: codegen.MCValue = if (val.isUndef(pt.zcu))
8152	.{ .load_symbol = try lf.lowerUav(pt, val.toIntern(), .none) }
8153	else
8154	try codegen.genTypedValue(lf, pt, val, func.target);
8155	const mcv: MCValue = switch (result) {
8156	.none => .none,
8157	.undef => unreachable,
8158	.lea_symbol => |sym_index| .{ .lea_symbol = .{ .sym = sym_index } },
8159	.load_symbol => |sym_index| .{ .load_symbol = .{ .sym = sym_index } },
8160	.immediate => |imm| .{ .immediate = imm },
8161	.memory => |addr| .{ .memory = addr },
8162	.load_got, .load_direct, .lea_direct => {
8163	return func.fail("TODO: genTypedValue {s}", .{@tagName(result)});
8164	},
8165	};
8166	return mcv;
8167	}
8168
8169	const CallMCValues = struct {
8170	args: []MCValue,
8171	return_value: InstTracking,
8172	stack_byte_count: u31,
8173	stack_align: Alignment,
8174
8175	fn deinit(call: *CallMCValues, func: *Func) void {
8176	func.gpa.free(call.args);
8177	call.* = undefined;
8178	}
8179	};
8180
8181	/// Caller must call `CallMCValues.deinit`.
8182	fn resolveCallingConventionValues(
8183	func: *Func,
8184	fn_info: InternPool.Key.FuncType,
8185	var_args: []const Type,
8186	) !CallMCValues {
8187	const pt = func.pt;
8188	const zcu = pt.zcu;
8189	const ip = &zcu.intern_pool;
8190
8191	const param_types = try func.gpa.alloc(Type, fn_info.param_types.len + var_args.len);
8192	defer func.gpa.free(param_types);
8193
8194	for (param_types[0..fn_info.param_types.len], fn_info.param_types.get(ip)) |*dest, src| {
8195	dest.* = Type.fromInterned(src);
8196	}
8197	for (param_types[fn_info.param_types.len..], var_args) |*param_ty, arg_ty|
8198	param_ty.* = func.promoteVarArg(arg_ty);
8199
8200	const cc = fn_info.cc;
8201	var result: CallMCValues = .{
8202	.args = try func.gpa.alloc(MCValue, param_types.len),
8203	// These undefined values must be populated before returning from this function.
8204	.return_value = undefined,
8205	.stack_byte_count = 0,
8206	.stack_align = undefined,
8207	};
8208	errdefer func.gpa.free(result.args);
8209
8210	const ret_ty = Type.fromInterned(fn_info.return_type);
8211
8212	switch (cc) {
8213	.naked => {
8214	assert(result.args.len == 0);
8215	result.return_value = InstTracking.init(.unreach);
8216	result.stack_align = .@"8";
8217	},
8218	.riscv64_lp64, .auto => {
8219	if (result.args.len > 8) {
8220	return func.fail("RISC-V calling convention does not support more than 8 arguments", .{});
8221	}
8222
8223	var ret_int_reg_i: u32 = 0;
8224	var param_int_reg_i: u32 = 0;
8225
8226	result.stack_align = .@"16";
8227
8228	// Return values
8229	if (ret_ty.zigTypeTag(zcu) == .noreturn) {
8230	result.return_value = InstTracking.init(.unreach);
8231	} else if (!ret_ty.hasRuntimeBits(zcu)) {
8232	result.return_value = InstTracking.init(.none);
8233	} else {
8234	var ret_tracking: [2]InstTracking = undefined;
8235	var ret_tracking_i: usize = 0;
8236	var ret_float_reg_i: usize = 0;
8237
8238	const classes = mem.sliceTo(&abi.classifySystem(ret_ty, zcu), .none);
8239
8240	for (classes) |class| switch (class) {
8241	.integer => {
8242	const ret_int_reg = abi.Registers.Integer.function_ret_regs[ret_int_reg_i];
8243	ret_int_reg_i += 1;
8244
8245	ret_tracking[ret_tracking_i] = InstTracking.init(.{ .register = ret_int_reg });
8246	ret_tracking_i += 1;
8247	},
8248	.float => {
8249	const ret_float_reg = abi.Registers.Float.function_ret_regs[ret_float_reg_i];
8250	ret_float_reg_i += 1;
8251
8252	ret_tracking[ret_tracking_i] = InstTracking.init(.{ .register = ret_float_reg });
8253	ret_tracking_i += 1;
8254	},
8255	.memory => {
8256	const ret_int_reg = abi.Registers.Integer.function_ret_regs[ret_int_reg_i];
8257	ret_int_reg_i += 1;
8258	const ret_indirect_reg = abi.Registers.Integer.function_arg_regs[param_int_reg_i];
8259	param_int_reg_i += 1;
8260
8261	ret_tracking[ret_tracking_i] = .{
8262	.short = .{ .indirect = .{ .reg = ret_int_reg } },
8263	.long = .{ .indirect = .{ .reg = ret_indirect_reg } },
8264	};
8265	ret_tracking_i += 1;
8266	},
8267	else => return func.fail("TODO: C calling convention return class {}", .{class}),
8268	};
8269
8270	result.return_value = switch (ret_tracking_i) {
8271	else => return func.fail("ty {f} took {} tracking return indices", .{ ret_ty.fmt(pt), ret_tracking_i }),
8272	1 => ret_tracking[0],
8273	2 => InstTracking.init(.{ .register_pair = .{
8274	ret_tracking[0].short.register, ret_tracking[1].short.register,
8275	} }),
8276	};
8277	}
8278
8279	var param_float_reg_i: usize = 0;
8280
8281	for (param_types, result.args) |ty, *arg| {
8282	if (!ty.hasRuntimeBits(zcu)) {
8283	assert(cc == .auto);
8284	arg.* = .none;
8285	continue;
8286	}
8287
8288	var arg_mcv: [2]MCValue = undefined;
8289	var arg_mcv_i: usize = 0;
8290
8291	const classes = mem.sliceTo(&abi.classifySystem(ty, zcu), .none);
8292
8293	for (classes) |class| switch (class) {
8294	.integer => {
8295	const param_int_regs = abi.Registers.Integer.function_arg_regs;
8296	if (param_int_reg_i >= param_int_regs.len) break;
8297
8298	const param_int_reg = param_int_regs[param_int_reg_i];
8299	param_int_reg_i += 1;
8300
8301	arg_mcv[arg_mcv_i] = .{ .register = param_int_reg };
8302	arg_mcv_i += 1;
8303	},
8304	.float => {
8305	const param_float_regs = abi.Registers.Float.function_arg_regs;
8306	if (param_float_reg_i >= param_float_regs.len) break;
8307
8308	const param_float_reg = param_float_regs[param_float_reg_i];
8309	param_float_reg_i += 1;
8310
8311	arg_mcv[arg_mcv_i] = .{ .register = param_float_reg };
8312	arg_mcv_i += 1;
8313	},
8314	.memory => {
8315	const param_int_regs = abi.Registers.Integer.function_arg_regs;
8316
8317	const param_int_reg = param_int_regs[param_int_reg_i];
8318	param_int_reg_i += 1;
8319
8320	arg_mcv[arg_mcv_i] = .{ .indirect = .{ .reg = param_int_reg } };
8321	arg_mcv_i += 1;
8322	},
8323	else => return func.fail("TODO: C calling convention arg class {}", .{class}),
8324	} else {
8325	arg.* = switch (arg_mcv_i) {
8326	else => return func.fail("ty {f} took {} tracking arg indices", .{ ty.fmt(pt), arg_mcv_i }),
8327	1 => arg_mcv[0],
8328	2 => .{ .register_pair = .{ arg_mcv[0].register, arg_mcv[1].register } },
8329	};
8330	continue;
8331	}
8332
8333	return func.fail("TODO: pass args by stack", .{});
8334	}
8335	},
8336	else => return func.fail("TODO implement function parameters for {} on riscv64", .{cc}),
8337	}
8338
8339	result.stack_byte_count = @intCast(result.stack_align.forward(result.stack_byte_count));
8340	return result;
8341	}
8342
8343	fn wantSafety(func: *Func) bool {
8344	return switch (func.mod.optimize_mode) {
8345	.debug => true,
8346	.safe => true,
8347	.fast => false,
8348	.small => false,
8349	};
8350	}
8351
8352	fn fail(func: *const Func, comptime format: []const u8, args: anytype) error{ OutOfMemory, AlreadyReported } {
8353	@branchHint(.cold);
8354	const zcu = func.pt.zcu;
8355	switch (func.owner) {
8356	.nav_index => |i| return zcu.codegenFail(i, format, args),
8357	.lazy_sym => |s| return zcu.codegenFailType(s.ty, format, args),
8358	}
8359	return error.AlreadyReported;
8360	}
8361
8362	fn failMsg(func: *const Func, msg: *ErrorMsg) error{ OutOfMemory, AlreadyReported } {
8363	@branchHint(.cold);
8364	const zcu = func.pt.zcu;
8365	switch (func.owner) {
8366	.nav_index => |i| return zcu.codegenFailMsg(i, msg),
8367	.lazy_sym => |s| return zcu.codegenFailTypeMsg(s.ty, msg),
8368	}
8369	return error.AlreadyReported;
8370	}
8371
8372	fn parseRegName(name: []const u8) ?Register {
8373	// The `fp` alias for `s0` is awkward to fit into the current `Register` scheme, so for now we
8374	// special-case it here.
8375	if (std.mem.eql(u8, name, "fp")) return .s0;
8376
8377	return std.meta.stringToEnum(Register, name);
8378	}
8379
8380	fn typeOf(func: *Func, inst: Air.Inst.Ref) Type {
8381	return func.air.typeOf(inst, &func.pt.zcu.intern_pool);
8382	}
8383
8384	fn typeOfIndex(func: *Func, inst: Air.Inst.Index) Type {
8385	const zcu = func.pt.zcu;
8386	return switch (func.air.instructions.items(.tag)[@backingInt(inst)]) {
8387	.loop_switch_br => func.typeOf(func.air.unwrapSwitch(inst).operand),
8388	else => func.air.typeOfIndex(inst, &zcu.intern_pool),
8389	};
8390	}
8391
8392	fn hasFeature(func: *Func, feature: Target.riscv.Feature) bool {
8393	return func.target.cpu.has(.riscv, feature);
8394	}
8395
8396	pub fn errUnionPayloadOffset(payload_ty: Type, zcu: *Zcu) u64 {
8397	if (!payload_ty.hasRuntimeBits(zcu)) return 0;
8398	const payload_align = payload_ty.abiAlignment(zcu);
8399	const error_align = Type.anyerror.abiAlignment(zcu);
8400	if (payload_align.compare(.gte, error_align) or !payload_ty.hasRuntimeBits(zcu)) {
8401	return 0;
8402	} else {
8403	return payload_align.forward(Type.anyerror.abiSize(zcu));
8404	}
8405	}
8406
8407	pub fn errUnionErrorOffset(payload_ty: Type, zcu: *Zcu) u64 {
8408	if (!payload_ty.hasRuntimeBits(zcu)) return 0;
8409	const payload_align = payload_ty.abiAlignment(zcu);
8410	const error_align = Type.anyerror.abiAlignment(zcu);
8411	if (payload_align.compare(.gte, error_align) and payload_ty.hasRuntimeBits(zcu)) {
8412	return error_align.forward(payload_ty.abiSize(zcu));
8413	} else {
8414	return 0;
8415	}
8416	}
8417
8418	fn promoteInt(func: *Func, ty: Type) Type {
8419	const pt = func.pt;
8420	const zcu = pt.zcu;
8421	const int_info: InternPool.Key.IntType = switch (ty.toIntern()) {
8422	.bool_type => .{ .signedness = .unsigned, .bits = 1 },
8423	else => if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else return ty,
8424	};
8425	for ([_]Type{
8426	Type.c_int, Type.c_uint,
8427	Type.c_long, Type.c_ulong,
8428	Type.c_longlong, Type.c_ulonglong,
8429	}) |promote_ty| {
8430	const promote_info = promote_ty.intInfo(zcu);
8431	if (int_info.signedness == .signed and promote_info.signedness == .unsigned) continue;
8432	if (int_info.bits + @intFromBool(int_info.signedness == .unsigned and
8433	promote_info.signedness == .signed) <= promote_info.bits) return promote_ty;
8434	}
8435	return ty;
8436	}
8437
8438	fn promoteVarArg(func: *Func, ty: Type) Type {
8439	if (!ty.isRuntimeFloat()) return func.promoteInt(ty);
8440	switch (ty.floatBits(func.target)) {
8441	32, 64 => return Type.f64,
8442	else => |float_bits| {
8443	assert(float_bits == func.target.cTypeBitSize(.longdouble));
8444	return Type.c_longdouble;
8445	},
8446	}
8447	}