Meg's Mirrors

zig

1	//! SPARCv9 codegen.
2	//! This lowers AIR into MIR.
3	//! For now this only implements medium/low code model with absolute addressing.
4	//! TODO add support for other code models.
5	const std = @import("std");
6	const assert = std.debug.assert;
7	const log = std.log.scoped(.codegen);
8	const math = std.math;
9	const mem = std.mem;
10	const Allocator = mem.Allocator;
11	const builtin = @import("builtin");
12	const link = @import("../../link.zig");
13	const Zcu = @import("../../Zcu.zig");
14	const InternPool = @import("../../InternPool.zig");
15	const Value = @import("../../Value.zig");
16	const ErrorMsg = Zcu.ErrorMsg;
17	const codegen = @import("../../codegen.zig");
18	const Air = @import("../../Air.zig");
19	const Mir = @import("Mir.zig");
20	const Emit = @import("Emit.zig");
21	const Type = @import("../../Type.zig");
22	const Endian = std.lang.Endian;
23	const Alignment = InternPool.Alignment;
24
25	const build_options = @import("build_options");
26
27	const bits = @import("bits.zig");
28	const abi = @import("abi.zig");
29	const errUnionPayloadOffset = codegen.errUnionPayloadOffset;
30	const errUnionErrorOffset = codegen.errUnionErrorOffset;
31	const Instruction = bits.Instruction;
32	const ASI = Instruction.ASI;
33	const ShiftWidth = Instruction.ShiftWidth;
34	const RegisterManager = abi.RegisterManager;
35	const RegisterLock = RegisterManager.RegisterLock;
36	const Register = bits.Register;
37	const gp = abi.RegisterClass.gp;
38
39	const Self = @This();
40
41	const InnerError = codegen.Error || error{OutOfRegisters};
42
43	pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features {
44	return comptime &.initMany(&.{
45	.expand_array_splat,
46	.expand_array_to_vector,
47	});
48	}
49
50	const RegisterView = enum(u1) {
51	caller,
52	callee,
53	};
54
55	gpa: Allocator,
56	pt: Zcu.PerThread,
57	air: Air,
58	liveness: Air.Liveness,
59	bin_file: *link.File,
60	target: *const std.Target,
61	func_index: InternPool.Index,
62	args: []MCValue,
63	ret_mcv: MCValue,
64	fn_type: Type,
65	arg_index: usize,
66	stack_align: Alignment,
67
68	/// MIR Instructions
69	mir_instructions: std.MultiArrayList(Mir.Inst) = .{},
70	/// MIR extra data
71	mir_extra: std.ArrayList(u32) = .empty,
72
73	/// Byte offset within the source file of the ending curly.
74	end_di_line: u32,
75	end_di_column: u32,
76
77	/// The value is an offset into the `Function` `code` from the beginning.
78	/// To perform the reloc, write 32-bit signed little-endian integer
79	/// which is a relative jump, based on the address following the reloc.
80	exitlude_jump_relocs: std.ArrayList(usize) = .empty,
81
82	reused_operands: std.bit_set.Static(Air.Liveness.bpi - 1) = undefined,
83
84	/// Whenever there is a runtime branch, we push a Branch onto this stack,
85	/// and pop it off when the runtime branch joins. This provides an "overlay"
86	/// of the table of mappings from instructions to `MCValue` from within the branch.
87	/// This way we can modify the `MCValue` for an instruction in different ways
88	/// within different branches. Special consideration is needed when a branch
89	/// joins with its parent, to make sure all instructions have the same MCValue
90	/// across each runtime branch upon joining.
91	branch_stack: *std.array_list.Managed(Branch),
92
93	// Key is the block instruction
94	blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .empty,
95
96	register_manager: RegisterManager = .{},
97
98	/// Maps offset to what is stored there.
99	stack: std.AutoHashMapUnmanaged(u32, StackAllocation) = .empty,
100
101	/// Tracks the current instruction allocated to the condition flags
102	condition_flags_inst: ?Air.Inst.Index = null,
103
104	/// Tracks the current instruction allocated to the condition register
105	condition_register_inst: ?Air.Inst.Index = null,
106
107	/// Offset from the stack base, representing the end of the stack frame.
108	max_end_stack: u32 = 0,
109	/// Represents the current end stack offset. If there is no existing slot
110	/// to place a new stack allocation, it goes here, and then bumps `max_end_stack`.
111	next_stack_offset: u32 = 0,
112
113	/// Debug field, used to find bugs in the compiler.
114	air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init,
115
116	const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {};
117
118	const MCValue = union(enum) {
119	/// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc.
120	/// TODO Look into deleting this tag and using `dead` instead, since every use
121	/// of MCValue.none should be instead looking at the type and noticing it is 0 bits.
122	none,
123	/// Control flow will not allow this value to be observed.
124	unreach,
125	/// No more references to this value remain.
126	dead,
127	/// The value is undefined.
128	undef,
129	/// A pointer-sized integer that fits in a register.
130	/// If the type is a pointer, this is the pointer address in virtual address space.
131	immediate: u64,
132	/// The value is in a target-specific register.
133	register: Register,
134	/// The value is a tuple { wrapped, overflow } where
135	/// wrapped is stored in the register and the overflow bit is
136	/// stored in the C (signed) or V (unsigned) flag of the CCR.
137	///
138	/// This MCValue is only generated by a add_with_overflow or
139	/// sub_with_overflow instruction operating on 32- or 64-bit values.
140	register_with_overflow: struct {
141	reg: Register,
142	flag: struct { cond: Instruction.ICondition, ccr: Instruction.CCR },
143	},
144	/// The value is in memory at a hard-coded address.
145	/// If the type is a pointer, it means the pointer address is at this memory location.
146	memory: u64,
147	/// The value is one of the stack variables.
148	/// If the type is a pointer, it means the pointer address is in the stack at this offset.
149	/// Note that this stores the plain value (i.e without the effects of the stack bias).
150	/// Always convert this value into machine offsets with realStackOffset() before
151	/// lowering into asm!
152	stack_offset: u32,
153	/// The value is a pointer to one of the stack variables (payload is stack offset).
154	ptr_stack_offset: u32,
155	/// The value is in the specified CCR. The value is 1 (if
156	/// the type is u1) or true (if the type in bool) iff the
157	/// specified condition is true.
158	condition_flags: struct {
159	cond: Instruction.Condition,
160	ccr: Instruction.CCR,
161	},
162	/// The value is in the specified Register. The value is 1 (if
163	/// the type is u1) or true (if the type in bool) iff the
164	/// specified condition is true.
165	condition_register: struct {
166	cond: Instruction.RCondition,
167	reg: Register,
168	},
169
170	fn isMemory(mcv: MCValue) bool {
171	return switch (mcv) {
172	.memory, .stack_offset => true,
173	else => false,
174	};
175	}
176
177	fn isImmediate(mcv: MCValue) bool {
178	return switch (mcv) {
179	.immediate => true,
180	else => false,
181	};
182	}
183
184	fn isMutable(mcv: MCValue) bool {
185	return switch (mcv) {
186	.none => unreachable,
187	.unreach => unreachable,
188	.dead => unreachable,
189
190	.immediate,
191	.memory,
192	.condition_flags,
193	.condition_register,
194	.ptr_stack_offset,
195	.undef,
196	=> false,
197
198	.register,
199	.stack_offset,
200	=> true,
201	};
202	}
203	};
204
205	const Branch = struct {
206	inst_table: std.array_hash_map.Auto(Air.Inst.Index, MCValue) = .empty,
207
208	fn deinit(self: *Branch, gpa: Allocator) void {
209	self.inst_table.deinit(gpa);
210	self.* = undefined;
211	}
212	};
213
214	const StackAllocation = struct {
215	inst: Air.Inst.Index,
216	/// TODO do we need size? should be determined by inst.ty.abiSize()
217	size: u32,
218	};
219
220	const BlockData = struct {
221	relocs: std.ArrayList(Mir.Inst.Index),
222	/// The first break instruction encounters `null` here and chooses a
223	/// machine code value for the block result, populating this field.
224	/// Following break instructions encounter that value and use it for
225	/// the location to store their block results.
226	mcv: MCValue,
227	};
228
229	const CallMCValues = struct {
230	args: []MCValue,
231	return_value: MCValue,
232	stack_byte_count: u32,
233	stack_align: Alignment,
234
235	fn deinit(self: *CallMCValues, func: *Self) void {
236	func.gpa.free(self.args);
237	self.* = undefined;
238	}
239	};
240
241	const BigTomb = struct {
242	function: *Self,
243	inst: Air.Inst.Index,
244	lbt: Air.Liveness.BigTomb,
245
246	fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void {
247	const dies = bt.lbt.feed();
248	const op_index = op_ref.toIndex() orelse return;
249	if (!dies) return;
250	bt.function.processDeath(op_index);
251	}
252
253	fn finishAir(bt: *BigTomb, result: MCValue) void {
254	const is_used = !bt.function.liveness.isUnused(bt.inst);
255	if (is_used) {
256	log.debug("%{d} => {}", .{ bt.inst, result });
257	const branch = &bt.function.branch_stack.items[bt.function.branch_stack.items.len - 1];
258	branch.inst_table.putAssumeCapacityNoClobber(bt.inst, result);
259	}
260	bt.function.finishAirBookkeeping();
261	}
262	};
263
264	pub fn generate(
265	lf: *link.File,
266	pt: Zcu.PerThread,
267	func_index: InternPool.Index,
268	air: *const Air,
269	liveness: *const ?Air.Liveness,
270	) codegen.Error!Mir {
271	const zcu = pt.zcu;
272	const gpa = zcu.gpa;
273	const func = zcu.funcInfo(func_index);
274	const func_ty = Type.fromInterned(func.ty);
275	const file_scope = zcu.navFileScope(func.owner_nav);
276	const target = &file_scope.mod.?.resolved_target.result;
277
278	var branch_stack = std.array_list.Managed(Branch).init(gpa);
279	defer {
280	assert(branch_stack.items.len == 1);
281	branch_stack.items[0].deinit(gpa);
282	branch_stack.deinit();
283	}
284	try branch_stack.append(.{});
285
286	var function: Self = .{
287	.gpa = gpa,
288	.pt = pt,
289	.air = air.*,
290	.liveness = liveness.*.?,
291	.target = target,
292	.bin_file = lf,
293	.func_index = func_index,
294	.args = undefined, // populated after `resolveCallingConventionValues`
295	.ret_mcv = undefined, // populated after `resolveCallingConventionValues`
296	.fn_type = func_ty,
297	.arg_index = 0,
298	.branch_stack = &branch_stack,
299	.stack_align = undefined,
300	.end_di_line = func.rbrace_line,
301	.end_di_column = func.rbrace_column,
302	};
303	defer function.stack.deinit(gpa);
304	defer function.blocks.deinit(gpa);
305	defer function.exitlude_jump_relocs.deinit(gpa);
306
307	var call_info = try function.resolveCallingConventionValues(func_ty, .callee);
308	defer call_info.deinit(&function);
309
310	function.args = call_info.args;
311	function.ret_mcv = call_info.return_value;
312	function.stack_align = call_info.stack_align;
313	function.max_end_stack = call_info.stack_byte_count;
314
315	function.gen() catch |err| switch (err) {
316	error.OutOfRegisters => return function.fail("ran out of registers (Zig compiler bug)", .{}),
317	else => |e| return e,
318	};
319
320	try function.mir_extra.shrinkToLen(gpa);
321
322	return .{
323	.instructions = function.mir_instructions.toOwnedSlice(),
324	.extra = function.mir_extra.toOwnedSliceAssert(),
325	};
326	}
327
328	fn gen(self: *Self) !void {
329	const pt = self.pt;
330	const zcu = pt.zcu;
331	const cc = self.fn_type.fnCallingConvention(zcu);
332	if (cc != .naked) {
333	// TODO Finish function prologue and epilogue for sparc64.
334
335	// save %sp, stack_reserved_area, %sp
336	const save_inst = try self.addInst(.{
337	.tag = .save,
338	.data = .{
339	.arithmetic_3op = .{
340	.is_imm = true,
341	.rd = .sp,
342	.rs1 = .sp,
343	.rs2_or_imm = .{ .imm = -abi.stack_reserved_area },
344	},
345	},
346	});
347
348	_ = try self.addInst(.{
349	.tag = .dbg_prologue_end,
350	.data = .{ .nop = {} },
351	});
352
353	try self.genBody(self.air.getMainBody());
354
355	_ = try self.addInst(.{
356	.tag = .dbg_epilogue_begin,
357	.data = .{ .nop = {} },
358	});
359
360	// exitlude jumps
361	if (self.exitlude_jump_relocs.items.len > 0 and
362	self.exitlude_jump_relocs.items[self.exitlude_jump_relocs.items.len - 1] == self.mir_instructions.len - 3)
363	{
364	// If the last Mir instruction (apart from the
365	// dbg_epilogue_begin) is the last exitlude jump
366	// relocation (which would just jump two instructions
367	// further), it can be safely removed
368	const index = self.exitlude_jump_relocs.pop().?;
369
370	// First, remove the delay slot, then remove
371	// the branch instruction itself.
372	self.mir_instructions.orderedRemove(index + 1);
373	self.mir_instructions.orderedRemove(index);
374	}
375
376	for (self.exitlude_jump_relocs.items) |jmp_reloc| {
377	self.mir_instructions.set(jmp_reloc, .{
378	.tag = .bpcc,
379	.data = .{
380	.branch_predict_int = .{
381	.ccr = .xcc,
382	.cond = .al,
383	.inst = @as(u32, @intCast(self.mir_instructions.len)),
384	},
385	},
386	});
387	}
388
389	// Backpatch stack offset
390	const total_stack_size = self.max_end_stack + abi.stack_reserved_area;
391	const stack_size = self.stack_align.forward(total_stack_size);
392	if (math.cast(i13, stack_size)) |size| {
393	self.mir_instructions.set(save_inst, .{
394	.tag = .save,
395	.data = .{
396	.arithmetic_3op = .{
397	.is_imm = true,
398	.rd = .sp,
399	.rs1 = .sp,
400	.rs2_or_imm = .{ .imm = -size },
401	},
402	},
403	});
404	} else {
405	// TODO for large stacks, replace the prologue with:
406	// setx stack_size, %g1
407	// save %sp, %g1, %sp
408	return self.fail("TODO SPARCv9: allow larger stacks", .{});
409	}
410
411	// return %i7 + 8
412	_ = try self.addInst(.{
413	.tag = .@"return",
414	.data = .{
415	.arithmetic_2op = .{
416	.is_imm = true,
417	.rs1 = .i7,
418	.rs2_or_imm = .{ .imm = 8 },
419	},
420	},
421	});
422
423	// Branches in SPARC have a delay slot, that is, the instruction
424	// following it will unconditionally be executed.
425	// See: Section 3.2.3 Control Transfer in SPARCv9 manual.
426	// See also: https://arcb.csc.ncsu.edu/~mueller/codeopt/codeopt00/notes/delaybra.html
427	// TODO Find a way to fill this delay slot
428	// nop
429	_ = try self.addInst(.{
430	.tag = .nop,
431	.data = .{ .nop = {} },
432	});
433	} else {
434	_ = try self.addInst(.{
435	.tag = .dbg_prologue_end,
436	.data = .{ .nop = {} },
437	});
438
439	try self.genBody(self.air.getMainBody());
440
441	_ = try self.addInst(.{
442	.tag = .dbg_epilogue_begin,
443	.data = .{ .nop = {} },
444	});
445	}
446
447	// Drop them off at the rbrace.
448	_ = try self.addInst(.{
449	.tag = .dbg_line,
450	.data = .{ .dbg_line_column = .{
451	.line = self.end_di_line,
452	.column = self.end_di_column,
453	} },
454	});
455	}
456
457	fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
458	const pt = self.pt;
459	const zcu = pt.zcu;
460	const ip = &zcu.intern_pool;
461	const air_tags = self.air.instructions.items(.tag);
462
463	for (body) |inst| {
464	// TODO: remove now-redundant isUnused calls from AIR handler functions
465	if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip))
466	continue;
467
468	const old_air_bookkeeping = self.air_bookkeeping;
469	try self.ensureProcessDeathCapacity(Air.Liveness.bpi);
470
471	self.reused_operands = @TypeOf(self.reused_operands).empty;
472	switch (air_tags[@backingInt(inst)]) {
473	// zig fmt: off
474
475	// No "scalarize" legalizations are enabled, so these instructions never appear.
476	.legalize_vec_elem_val => unreachable,
477	.legalize_vec_store_elem => unreachable,
478	// No soft float legalizations are enabled.
479	.legalize_compiler_rt_call => unreachable,
480
481	.ptr_add => try self.airPtrArithmetic(inst, .ptr_add),
482	.ptr_sub => try self.airPtrArithmetic(inst, .ptr_sub),
483
484	.add => try self.airBinOp(inst, .add),
485	.add_wrap => try self.airBinOp(inst, .add_wrap),
486	.sub => try self.airBinOp(inst, .sub),
487	.sub_wrap => try self.airBinOp(inst, .sub_wrap),
488	.mul => try self.airBinOp(inst, .mul),
489	.mul_wrap => try self.airBinOp(inst, .mul_wrap),
490	.shl => try self.airBinOp(inst, .shl),
491	.shl_exact => try self.airBinOp(inst, .shl_exact),
492	.shr => try self.airBinOp(inst, .shr),
493	.shr_exact => try self.airBinOp(inst, .shr_exact),
494	.bit_and => try self.airBinOp(inst, .bit_and),
495	.bit_or => try self.airBinOp(inst, .bit_or),
496	.xor => try self.airBinOp(inst, .xor),
497
498	.add_sat => try self.airAddSat(inst),
499	.sub_sat => try self.airSubSat(inst),
500	.mul_sat => try self.airMulSat(inst),
501	.shl_sat => try self.airShlSat(inst),
502	.min, .max => try self.airMinMax(inst),
503	.rem => try self.airRem(inst),
504	.mod => try self.airMod(inst),
505	.slice => try self.airSlice(inst),
506
507	.sqrt,
508	.sin,
509	.cos,
510	.tan,
511	.exp,
512	.exp2,
513	.log,
514	.log2,
515	.log10,
516	.abs,
517	.floor,
518	.ceil,
519	.round,
520	.trunc_float,
521	.neg,
522	=> try self.airUnaryMath(inst),
523
524	.add_with_overflow => try self.airAddSubWithOverflow(inst),
525	.sub_with_overflow => try self.airAddSubWithOverflow(inst),
526	.mul_with_overflow => try self.airMulWithOverflow(inst),
527	.shl_with_overflow => try self.airShlWithOverflow(inst),
528
529	.div_float, .div_trunc, .div_floor, .div_ceil, .div_exact => try self.airDiv(inst),
530
531	.cmp_lt => try self.airCmp(inst, .lt),
532	.cmp_lte => try self.airCmp(inst, .lte),
533	.cmp_eq => try self.airCmp(inst, .eq),
534	.cmp_gte => try self.airCmp(inst, .gte),
535	.cmp_gt => try self.airCmp(inst, .gt),
536	.cmp_neq => try self.airCmp(inst, .neq),
537	.cmp_vector => @panic("TODO try self.airCmpVector(inst)"),
538	.cmp_lte_errors_len => try self.airCmpLteErrorsLen(inst),
539
540	.alloc => try self.airAlloc(inst),
541	.ret_ptr => try self.airRetPtr(inst),
542	.arg => try self.airArg(inst),
543	.assembly => try self.airAsm(inst),
544	.bit_cast => try self.airBitCast(inst),
545	.ptr_cast => try self.airBitCast(inst),
546	.ptr_from_int => try self.airBitCast(inst),
547	.int_from_ptr => try self.airBitCast(inst),
548	.error_cast => try self.airBitCast(inst),
549	.error_from_int => try self.airBitCast(inst),
550	.int_from_error => try self.airBitCast(inst),
551	.union_from_enum => try self.airBitCast(inst),
552	.block => try self.airBlock(inst),
553	.br => try self.airBr(inst),
554	.repeat => return self.fail("TODO implement `repeat`", .{}),
555	.switch_dispatch => return self.fail("TODO implement `switch_dispatch`", .{}),
556	.trap => try self.airTrap(),
557	.breakpoint => try self.airBreakpoint(),
558	.ret_addr => @panic("TODO try self.airRetAddr(inst)"),
559	.frame_addr => @panic("TODO try self.airFrameAddress(inst)"),
560	.cond_br => try self.airCondBr(inst),
561	.fptrunc => @panic("TODO try self.airFptrunc(inst)"),
562	.fpext => @panic("TODO try self.airFpext(inst)"),
563	.int_cast => try self.airIntCast(inst),
564	.trunc => try self.airTrunc(inst),
565	.is_non_null => try self.airIsNonNull(inst),
566	.is_non_null_ptr => @panic("TODO try self.airIsNonNullPtr(inst)"),
567	.is_null => try self.airIsNull(inst),
568	.is_null_ptr => @panic("TODO try self.airIsNullPtr(inst)"),
569	.is_non_err => try self.airIsNonErr(inst),
570	.is_non_err_ptr => @panic("TODO try self.airIsNonErrPtr(inst)"),
571	.is_err => try self.airIsErr(inst),
572	.is_err_ptr => @panic("TODO try self.airIsErrPtr(inst)"),
573	.load => try self.airLoad(inst),
574	.loop => try self.airLoop(inst),
575	.not => try self.airNot(inst),
576	.ret => try self.airRet(inst),
577	.ret_safe => try self.airRet(inst), // TODO
578	.ret_load => try self.airRetLoad(inst),
579	.store => try self.airStore(inst, false),
580	.store_safe => try self.airStore(inst, true),
581	.struct_field_ptr=> try self.airStructFieldPtr(inst),
582	.agg_field_val => try self.airAggFieldVal(inst),
583	.array_to_slice => try self.airArrayToSlice(inst),
584	.array_to_vector => unreachable, // legalize .expand_array_to_vector
585	.float_from_int => try self.airFloatFromInt(inst),
586	.int_from_float => try self.airIntFromFloat(inst),
587	.cmpxchg_strong,
588	.cmpxchg_weak,
589	=> try self.airCmpxchg(inst),
590	.atomic_rmw => try self.airAtomicRmw(inst),
591	.atomic_load => try self.airAtomicLoad(inst),
592	.memcpy => @panic("TODO try self.airMemcpy(inst)"),
593	.memmove => @panic("TODO try self.airMemmove(inst)"),
594	.memset => try self.airMemset(inst, false),
595	.memset_safe => try self.airMemset(inst, true),
596	.set_union_tag => try self.airSetUnionTag(inst),
597	.get_union_tag => try self.airGetUnionTag(inst),
598	.clz => try self.airClz(inst),
599	.ctz => try self.airCtz(inst),
600	.popcount => try self.airPopcount(inst),
601	.byte_swap => try self.airByteSwap(inst),
602	.bit_reverse => try self.airBitReverse(inst),
603	.tag_name => try self.airTagName(inst),
604	.error_name => try self.airErrorName(inst),
605	.splat => try self.airSplat(inst),
606	.select => @panic("TODO try self.airSelect(inst)"),
607	.shuffle_one => @panic("TODO try self.airShuffleOne(inst)"),
608	.shuffle_two => @panic("TODO try self.airShuffleTwo(inst)"),
609	.reduce => @panic("TODO try self.airReduce(inst)"),
610	.aggregate_init => try self.airAggregateInit(inst),
611	.union_init => try self.airUnionInit(inst),
612	.prefetch => try self.airPrefetch(inst),
613	.mul_add => @panic("TODO try self.airMulAdd(inst)"),
614	.addrspace_cast => @panic("TODO try self.airAddrSpaceCast(int)"),
615
616	.@"try" => try self.airTry(inst),
617	.try_cold => try self.airTry(inst),
618	.try_ptr => @panic("TODO try self.airTryPtr(inst)"),
619	.try_ptr_cold => @panic("TODO try self.airTryPtrCold(inst)"),
620
621	.dbg_stmt => try self.airDbgStmt(inst),
622	.dbg_empty_stmt => self.finishAirBookkeeping(),
623	.dbg_inline_block => try self.airDbgInlineBlock(inst),
624	.dbg_var_ptr,
625	.dbg_var_val,
626	.dbg_arg_inline,
627	=> try self.airDbgVar(inst),
628
629	.call => try self.airCall(inst, .auto),
630	.call_always_tail => try self.airCall(inst, .always_tail),
631	.call_never_tail => try self.airCall(inst, .never_tail),
632	.call_never_inline => try self.airCall(inst, .never_inline),
633
634	.atomic_store_unordered => @panic("TODO try self.airAtomicStore(inst, .unordered)"),
635	.atomic_store_monotonic => @panic("TODO try self.airAtomicStore(inst, .monotonic)"),
636	.atomic_store_release => @panic("TODO try self.airAtomicStore(inst, .release)"),
637	.atomic_store_seq_cst => @panic("TODO try self.airAtomicStore(inst, .seq_cst)"),
638
639	.struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0),
640	.struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1),
641	.struct_field_ptr_index_2 => try self.airStructFieldPtrIndex(inst, 2),
642	.struct_field_ptr_index_3 => try self.airStructFieldPtrIndex(inst, 3),
643
644	.field_parent_ptr => @panic("TODO try self.airFieldParentPtr(inst)"),
645
646	.switch_br => try self.airSwitch(inst),
647	.loop_switch_br => return self.fail("TODO implement `loop_switch_br`", .{}),
648	.slice_ptr => try self.airSlicePtr(inst),
649	.slice_len => try self.airSliceLen(inst),
650
651	.ptr_slice_len_ptr => try self.airPtrSliceLenPtr(inst),
652	.ptr_slice_ptr_ptr => try self.airPtrSlicePtrPtr(inst),
653
654	.array_elem_val => try self.airArrayElemVal(inst),
655	.slice_elem_val => try self.airSliceElemVal(inst),
656	.slice_elem_ptr => @panic("TODO try self.airSliceElemPtr(inst)"),
657	.ptr_elem_val => try self.airPtrElemVal(inst),
658	.ptr_elem_ptr => try self.airPtrElemPtr(inst),
659
660	.inferred_alloc, .inferred_alloc_comptime => unreachable,
661	.unreach => self.finishAirBookkeeping(),
662
663	.optional_payload => try self.airOptionalPayload(inst),
664	.optional_payload_ptr => try self.airOptionalPayloadPtr(inst),
665	.optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst),
666	.unwrap_errunion_err => try self.airUnwrapErrErr(inst),
667	.unwrap_errunion_payload => try self.airUnwrapErrPayload(inst),
668	.unwrap_errunion_err_ptr => @panic("TODO try self.airUnwrapErrErrPtr(inst)"),
669	.unwrap_errunion_payload_ptr=> @panic("TODO try self.airUnwrapErrPayloadPtr(inst)"),
670	.errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst),
671	.err_return_trace => @panic("TODO try self.airErrReturnTrace(inst)"),
672	.set_err_return_trace => @panic("TODO try self.airSetErrReturnTrace(inst)"),
673	.save_err_return_trace_index=> @panic("TODO try self.airSaveErrReturnTraceIndex(inst)"),
674
675	.wrap_optional => try self.airWrapOptional(inst),
676	.wrap_errunion_payload => try self.airWrapErrUnionPayload(inst),
677	.wrap_errunion_err => try self.airWrapErrUnionErr(inst),
678
679	.add_optimized,
680	.sub_optimized,
681	.mul_optimized,
682	.div_float_optimized,
683	.div_trunc_optimized,
684	.div_floor_optimized,
685	.div_ceil_optimized,
686	.div_exact_optimized,
687	.rem_optimized,
688	.mod_optimized,
689	.neg_optimized,
690	.cmp_lt_optimized,
691	.cmp_lte_optimized,
692	.cmp_eq_optimized,
693	.cmp_gte_optimized,
694	.cmp_gt_optimized,
695	.cmp_neq_optimized,
696	.cmp_vector_optimized,
697	.reduce_optimized,
698	.int_from_float_optimized,
699	=> @panic("TODO implement optimized float mode"),
700
701	.add_safe,
702	.sub_safe,
703	.mul_safe,
704	.bit_cast_safe,
705	.int_cast_safe,
706	.int_from_float_safe,
707	.int_from_float_optimized_safe,
708	=> @panic("TODO implement safety_checked_instructions"),
709
710	.is_named_enum_value => @panic("TODO implement is_named_enum_value"),
711	.error_set_has_value => @panic("TODO implement error_set_has_value"),
712	.runtime_nav_ptr => @panic("TODO implement runtime_nav_ptr"),
713
714	.c_va_arg => return self.fail("TODO implement c_va_arg", .{}),
715	.c_va_copy => return self.fail("TODO implement c_va_copy", .{}),
716	.c_va_end => return self.fail("TODO implement c_va_end", .{}),
717	.c_va_start => return self.fail("TODO implement c_va_start", .{}),
718
719	.wasm_memory_size => unreachable,
720	.wasm_memory_grow => unreachable,
721
722	.work_item_id => unreachable,
723	.work_group_size => unreachable,
724	.work_group_id => unreachable,
725	.spirv_runtime_array_len => unreachable,
726	// zig fmt: on
727	}
728
729	assert(!self.register_manager.lockedRegsExist());
730
731	if (std.debug.runtime_safety) {
732	if (self.air_bookkeeping < old_air_bookkeeping + 1) {
733	std.debug.panic("in codegen.zig, handling of AIR instruction %{d} ('{t}') did not do proper bookkeeping. Look for a missing call to finishAir.", .{ inst, air_tags[@backingInt(inst)] });
734	}
735	}
736	}
737	}
738
739	fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
740	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
741	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement add_sat for {}", .{self.target.cpu.arch});
742	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
743	}
744
745	fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
746	const tag = self.air.instructions.items(.tag)[@backingInt(inst)];
747	const ty_pl = self.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
748	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
749	const pt = self.pt;
750	const zcu = pt.zcu;
751	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
752	const lhs = try self.resolveInst(extra.lhs);
753	const rhs = try self.resolveInst(extra.rhs);
754	const lhs_ty = self.typeOf(extra.lhs);
755	const rhs_ty = self.typeOf(extra.rhs);
756
757	switch (lhs_ty.zigTypeTag(zcu)) {
758	.vector => return self.fail("TODO implement add_with_overflow/sub_with_overflow for vectors", .{}),
759	.int => {
760	assert(lhs_ty.eql(rhs_ty));
761	const int_info = lhs_ty.intInfo(zcu);
762	switch (int_info.bits) {
763	32, 64 => {
764	// Only say yes if the operation is
765	// commutative, i.e. we can swap both of the
766	// operands
767	const lhs_immediate_ok = switch (tag) {
768	.add_with_overflow => lhs == .immediate and lhs.immediate <= std.math.maxInt(u12),
769	.sub_with_overflow => false,
770	else => unreachable,
771	};
772	const rhs_immediate_ok = switch (tag) {
773	.add_with_overflow,
774	.sub_with_overflow,
775	=> rhs == .immediate and rhs.immediate <= std.math.maxInt(u12),
776	else => unreachable,
777	};
778
779	const mir_tag: Mir.Inst.Tag = switch (tag) {
780	.add_with_overflow => .addcc,
781	.sub_with_overflow => .subcc,
782	else => unreachable,
783	};
784
785	try self.spillConditionFlagsIfOccupied();
786
787	const dest = blk: {
788	if (rhs_immediate_ok) {
789	break :blk try self.binOpImmediate(mir_tag, lhs, rhs, lhs_ty, false, null);
790	} else if (lhs_immediate_ok) {
791	// swap lhs and rhs
792	break :blk try self.binOpImmediate(mir_tag, rhs, lhs, rhs_ty, true, null);
793	} else {
794	break :blk try self.binOpRegister(mir_tag, lhs, rhs, lhs_ty, rhs_ty, null);
795	}
796	};
797
798	const cond = switch (int_info.signedness) {
799	.unsigned => switch (tag) {
800	.add_with_overflow => Instruction.ICondition.cs,
801	.sub_with_overflow => Instruction.ICondition.cc,
802	else => unreachable,
803	},
804	.signed => Instruction.ICondition.vs,
805	};
806
807	const ccr = switch (int_info.bits) {
808	32 => Instruction.CCR.icc,
809	64 => Instruction.CCR.xcc,
810	else => unreachable,
811	};
812
813	break :result MCValue{ .register_with_overflow = .{
814	.reg = dest.register,
815	.flag = .{ .cond = cond, .ccr = ccr },
816	} };
817	},
818	else => return self.fail("TODO overflow operations on other integer sizes", .{}),
819	}
820	},
821	else => unreachable,
822	}
823	};
824	return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
825	}
826
827	fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
828	const pt = self.pt;
829	const zcu = pt.zcu;
830	const vector_ty = self.typeOfIndex(inst);
831	const len = vector_ty.vectorLen(zcu);
832	const ty_pl = self.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
833	const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]);
834	const result: MCValue = res: {
835	if (self.liveness.isUnused(inst)) break :res MCValue.dead;
836	return self.fail("TODO implement airAggregateInit for {}", .{self.target.cpu.arch});
837	};
838
839	if (elements.len <= Air.Liveness.bpi - 1) {
840	var buf: [Air.Liveness.bpi - 1]Air.Inst.Ref = @splat(.none);
841	@memcpy(buf[0..elements.len], elements);
842	return self.finishAir(inst, result, buf);
843	}
844	var bt = try self.iterateBigTomb(inst, elements.len);
845	for (elements) |elem| {
846	bt.feed(elem);
847	}
848	return bt.finishAir(result);
849	}
850
851	fn airAlloc(self: *Self, inst: Air.Inst.Index) !void {
852	const stack_offset = try self.allocMemPtr(inst);
853	return self.finishAir(inst, .{ .ptr_stack_offset = stack_offset }, .{ .none, .none, .none });
854	}
855
856	fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
857	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
858	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement array_elem_val for {}", .{self.target.cpu.arch});
859	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
860	}
861
862	fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
863	const pt = self.pt;
864	const zcu = pt.zcu;
865	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
866	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
867	const ptr_ty = self.typeOf(ty_op.operand);
868	const ptr = try self.resolveInst(ty_op.operand);
869	const array_ty = ptr_ty.childType(zcu);
870	const array_len: u32 = @intCast(array_ty.arrayLen(zcu));
871	const ptr_bytes = 8;
872	const stack_offset = try self.allocMem(inst, ptr_bytes * 2, .@"8");
873	try self.genSetStack(ptr_ty, stack_offset, ptr);
874	try self.genSetStack(Type.usize, stack_offset - ptr_bytes, .{ .immediate = array_len });
875	break :result MCValue{ .stack_offset = stack_offset };
876	};
877	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
878	}
879
880	fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
881	const unwrapped_asm = self.air.unwrapAsm(inst);
882	const is_volatile = unwrapped_asm.is_volatile;
883	const outputs = unwrapped_asm.outputs;
884	const inputs = unwrapped_asm.inputs;
885
886	const dead = !is_volatile and self.liveness.isUnused(inst);
887	const result: MCValue = if (dead) .dead else result: {
888	if (outputs.len > 1) {
889	return self.fail("TODO implement codegen for asm with more than 1 output", .{});
890	}
891
892	var it = unwrapped_asm.iterateOutputs();
893	const output_constraint: ?[]const u8 = while (it.next()) |output| {
894	if (output.operand != .none) {
895	return self.fail("TODO implement codegen for non-expr asm", .{});
896	}
897
898	break output.constraint;
899	} else null;
900
901	it = unwrapped_asm.iterateInputs();
902	while (it.next()) |input| {
903	const constraint = input.constraint;
904
905	if (constraint.len < 3 or constraint[0] != '{' or constraint[constraint.len - 1] != '}') {
906	return self.fail("unrecognized asm input constraint: '{s}'", .{constraint});
907	}
908	const reg_name = constraint[1 .. constraint.len - 1];
909	const reg = parseRegName(reg_name) orelse
910	return self.fail("unrecognized register: '{s}'", .{reg_name});
911
912	const arg_mcv = try self.resolveInst(input.operand);
913	try self.register_manager.getReg(reg, null);
914	try self.genSetReg(self.typeOf(input.operand), reg, arg_mcv);
915	}
916
917	// TODO honor the clobbers
918	_ = unwrapped_asm.clobbers;
919
920	const asm_source = unwrapped_asm.source;
921
922	if (mem.eql(u8, asm_source, "ta 0x6d")) {
923	_ = try self.addInst(.{
924	.tag = .tcc,
925	.data = .{
926	.trap = .{
927	.is_imm = true,
928	.cond = .al,
929	.rs2_or_imm = .{ .imm = 0x6d },
930	},
931	},
932	});
933	} else {
934	return self.fail("TODO implement a full SPARCv9 assembly parsing", .{});
935	}
936
937	if (output_constraint) |output| {
938	if (output.len < 4 or output[0] != '=' or output[1] != '{' or output[output.len - 1] != '}') {
939	return self.fail("unrecognized asm output constraint: '{s}'", .{output});
940	}
941	const reg_name = output[2 .. output.len - 1];
942	const reg = parseRegName(reg_name) orelse
943	return self.fail("unrecognized register: '{s}'", .{reg_name});
944	break :result MCValue{ .register = reg };
945	} else {
946	break :result MCValue{ .none = {} };
947	}
948	};
949
950	simple: {
951	var buf: [Air.Liveness.bpi - 1]Air.Inst.Ref = @splat(.none);
952	var buf_index: usize = 0;
953	for (outputs) |output| {
954	if (output == .none) continue;
955
956	if (buf_index >= buf.len) break :simple;
957	buf[buf_index] = output;
958	buf_index += 1;
959	}
960	if (buf_index + inputs.len > buf.len) break :simple;
961	@memcpy(buf[buf_index..][0..inputs.len], inputs);
962	return self.finishAir(inst, result, buf);
963	}
964
965	var bt = try self.iterateBigTomb(inst, outputs.len + inputs.len);
966	for (outputs) |output| {
967	if (output == .none) continue;
968
969	bt.feed(output);
970	}
971	for (inputs) |input| {
972	bt.feed(input);
973	}
974	return bt.finishAir(result);
975	}
976
977	fn airArg(self: *Self, inst: Air.Inst.Index) InnerError!void {
978	const pt = self.pt;
979	const zcu = pt.zcu;
980	const arg_index = self.arg_index;
981	self.arg_index += 1;
982
983	const ty = self.typeOfIndex(inst);
984	const mcv: MCValue = blk: {
985	switch (self.args[arg_index]) {
986	.stack_offset => |off| {
987	const abi_size = math.cast(u32, ty.abiSize(zcu)) orelse {
988	return self.fail("type '{f}' too big to fit into stack frame", .{ty.fmt(pt)});
989	};
990	const offset = off + abi_size;
991	break :blk .{ .stack_offset = offset };
992	},
993	else => |mcv| break :blk mcv,
994	}
995	};
996
997	const func_zir = zcu.funcInfo(self.func_index).zir_body_inst.resolveFull(&zcu.intern_pool).?;
998	const file = zcu.fileByIndex(func_zir.file);
999	if (!file.mod.?.strip) {
1000	const arg = self.air.instructions.items(.data)[@backingInt(inst)].arg;
1001	const zir = &file.zir.?;
1002	const name = zir.nullTerminatedString(zir.getParamName(zir.getParamBody(func_zir.inst)[arg.zir_param_index]).?);
1003
1004	self.genArgDbgInfo(name, ty, mcv) catch |err|
1005	return self.fail("failed to generate debug info for parameter: {s}", .{@errorName(err)});
1006	}
1007
1008	if (self.liveness.isUnused(inst))
1009	return self.finishAirBookkeeping();
1010
1011	switch (mcv) {
1012	.register => |reg| {
1013	self.register_manager.getRegAssumeFree(reg, inst);
1014	},
1015	else => {},
1016	}
1017
1018	return self.finishAir(inst, mcv, .{ .none, .none, .none });
1019	}
1020
1021	fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {
1022	_ = self.air.instructions.items(.data)[@backingInt(inst)].atomic_load;
1023
1024	return self.fail("TODO implement airAtomicLoad for {}", .{
1025	self.target.cpu.arch,
1026	});
1027	}
1028
1029	fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void {
1030	_ = self.air.instructions.items(.data)[@backingInt(inst)].pl_op;
1031
1032	return self.fail("TODO implement airAtomicRmw for {}", .{
1033	self.target.cpu.arch,
1034	});
1035	}
1036
1037	fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
1038	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
1039	const lhs = try self.resolveInst(bin_op.lhs);
1040	const rhs = try self.resolveInst(bin_op.rhs);
1041	const lhs_ty = self.typeOf(bin_op.lhs);
1042	const rhs_ty = self.typeOf(bin_op.rhs);
1043	const result: MCValue = if (self.liveness.isUnused(inst))
1044	.dead
1045	else
1046	try self.binOp(tag, lhs, rhs, lhs_ty, rhs_ty, BinOpMetadata{
1047	.lhs = bin_op.lhs,
1048	.rhs = bin_op.rhs,
1049	.inst = inst,
1050	});
1051	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1052	}
1053
1054	fn airPtrArithmetic(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
1055	const ty_pl = self.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
1056	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
1057	const lhs = try self.resolveInst(bin_op.lhs);
1058	const rhs = try self.resolveInst(bin_op.rhs);
1059	const lhs_ty = self.typeOf(bin_op.lhs);
1060	const rhs_ty = self.typeOf(bin_op.rhs);
1061	const result: MCValue = if (self.liveness.isUnused(inst))
1062	.dead
1063	else
1064	try self.binOp(tag, lhs, rhs, lhs_ty, rhs_ty, BinOpMetadata{
1065	.lhs = bin_op.lhs,
1066	.rhs = bin_op.rhs,
1067	.inst = inst,
1068	});
1069	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1070	}
1071
1072	fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
1073	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
1074	const result = if (self.liveness.isUnused(inst)) .dead else result: {
1075	const operand = try self.resolveInst(ty_op.operand);
1076	if (self.reuseOperand(inst, ty_op.operand, 0, operand)) break :result operand;
1077
1078	const operand_lock = switch (operand) {
1079	.register => |reg| self.register_manager.lockReg(reg),
1080	.register_with_overflow => |rwo| self.register_manager.lockReg(rwo.reg),
1081	else => null,
1082	};
1083	defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
1084
1085	const dest = try self.allocRegOrMem(inst, true);
1086	try self.setRegOrMem(self.typeOfIndex(inst), dest, operand);
1087	break :result dest;
1088	};
1089	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1090	}
1091
1092	fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
1093	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
1094	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airBitReverse for {}", .{self.target.cpu.arch});
1095	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1096	}
1097
1098	fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
1099	const block = self.air.unwrapBlock(inst);
1100	try self.lowerBlock(inst, block.body);
1101	}
1102
1103	fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void {
1104	try self.blocks.putNoClobber(self.gpa, inst, .{
1105	// A block is a setup to be able to jump to the end.
1106	.relocs = .empty,
1107	// It also acts as a receptacle for break operands.
1108	// Here we use `MCValue.none` to represent a null value so that the first
1109	// break instruction will choose a MCValue for the block result and overwrite
1110	// this field. Following break instructions will use that MCValue to put their
1111	// block results.
1112	.mcv = MCValue{ .none = {} },
1113	});
1114	defer self.blocks.getPtr(inst).?.relocs.deinit(self.gpa);
1115
1116	// TODO emit debug info lexical block
1117	try self.genBody(body);
1118
1119	// relocations for `bpcc` instructions
1120	const relocs = &self.blocks.getPtr(inst).?.relocs;
1121	if (relocs.items.len > 0 and relocs.items[relocs.items.len - 1] == self.mir_instructions.len - 1) {
1122	// If the last Mir instruction is the last relocation (which
1123	// would just jump two instruction further), it can be safely
1124	// removed
1125	const index = relocs.pop().?;
1126
1127	// First, remove the delay slot, then remove
1128	// the branch instruction itself.
1129	self.mir_instructions.orderedRemove(index + 1);
1130	self.mir_instructions.orderedRemove(index);
1131	}
1132	for (relocs.items) |reloc| {
1133	try self.performReloc(reloc);
1134	}
1135
1136	const result = self.blocks.getPtr(inst).?.mcv;
1137	return self.finishAir(inst, result, .{ .none, .none, .none });
1138	}
1139
1140	fn airBr(self: *Self, inst: Air.Inst.Index) !void {
1141	const branch = self.air.instructions.items(.data)[@backingInt(inst)].br;
1142	try self.br(branch.block_inst, branch.operand);
1143	return self.finishAir(inst, .dead, .{ branch.operand, .none, .none });
1144	}
1145
1146	fn airTrap(self: *Self) !void {
1147	// ta 0x05
1148	_ = try self.addInst(.{
1149	.tag = .tcc,
1150	.data = .{
1151	.trap = .{
1152	.is_imm = true,
1153	.cond = .al,
1154	.rs2_or_imm = .{ .imm = 0x05 },
1155	},
1156	},
1157	});
1158	return self.finishAirBookkeeping();
1159	}
1160
1161	fn airBreakpoint(self: *Self) !void {
1162	// ta 0x01
1163	_ = try self.addInst(.{
1164	.tag = .tcc,
1165	.data = .{
1166	.trap = .{
1167	.is_imm = true,
1168	.cond = .al,
1169	.rs2_or_imm = .{ .imm = 0x01 },
1170	},
1171	},
1172	});
1173	return self.finishAirBookkeeping();
1174	}
1175
1176	fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {
1177	const pt = self.pt;
1178	const zcu = pt.zcu;
1179	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
1180
1181	// We have hardware byteswapper in SPARCv9, don't let mainstream compilers mislead you.
1182	// That being said, the strategy to lower this is:
1183	// - If src is an immediate, comptime-swap it.
1184	// - If src is in memory then issue an LD*A with #ASI_P_[oppposite-endian]
1185	// - If src is a register then issue an ST*A with #ASI_P_[oppposite-endian]
1186	// to a stack slot, then follow with a normal load from said stack slot.
1187	// This is because on some implementations, ASI-tagged memory operations are non-piplelinable
1188	// and loads tend to have longer latency than stores, so the sequence will minimize stall.
1189	// The result will always be either another immediate or stored in a register.
1190	// TODO: Fold byteswap+store into a single ST*A and load+byteswap into a single LD*A.
1191	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1192	const operand = try self.resolveInst(ty_op.operand);
1193	const operand_ty = self.typeOf(ty_op.operand);
1194	switch (operand_ty.zigTypeTag(zcu)) {
1195	.vector => return self.fail("TODO byteswap for vectors", .{}),
1196	.int => {
1197	const int_info = operand_ty.intInfo(zcu);
1198	if (int_info.bits == 8) break :result operand;
1199
1200	const abi_size = int_info.bits >> 3;
1201	const abi_align = operand_ty.abiAlignment(zcu);
1202	const opposite_endian_asi = switch (self.target.cpu.arch.endian()) {
1203	Endian.big => ASI.asi_primary_little,
1204	Endian.little => ASI.asi_primary,
1205	};
1206
1207	switch (operand) {
1208	.immediate => |imm| {
1209	const swapped = switch (int_info.bits) {
1210	16 => @byteSwap(@as(u16, @intCast(imm))),
1211	24 => @byteSwap(@as(u24, @intCast(imm))),
1212	32 => @byteSwap(@as(u32, @intCast(imm))),
1213	40 => @byteSwap(@as(u40, @intCast(imm))),
1214	48 => @byteSwap(@as(u48, @intCast(imm))),
1215	56 => @byteSwap(@as(u56, @intCast(imm))),
1216	64 => @byteSwap(@as(u64, @intCast(imm))),
1217	else => return self.fail("TODO synthesize SPARCv9 byteswap for other integer sizes", .{}),
1218	};
1219	break :result .{ .immediate = swapped };
1220	},
1221	.register => |reg| {
1222	if (int_info.bits > 64 or @popCount(int_info.bits) != 1)
1223	return self.fail("TODO synthesize SPARCv9 byteswap for other integer sizes", .{});
1224
1225	const off = try self.allocMem(inst, abi_size, abi_align);
1226	const off_reg = try self.copyToTmpRegister(operand_ty, .{ .immediate = realStackOffset(off) });
1227
1228	try self.genStoreASI(reg, .sp, off_reg, abi_size, opposite_endian_asi);
1229	try self.genLoad(reg, .sp, Register, off_reg, abi_size);
1230	break :result .{ .register = reg };
1231	},
1232	.memory => {
1233	if (int_info.bits > 64 or @popCount(int_info.bits) != 1)
1234	return self.fail("TODO synthesize SPARCv9 byteswap for other integer sizes", .{});
1235
1236	const addr_reg = try self.copyToTmpRegister(operand_ty, operand);
1237	const dst_reg = try self.register_manager.allocReg(null, gp);
1238
1239	try self.genLoadASI(dst_reg, addr_reg, .g0, abi_size, opposite_endian_asi);
1240	break :result .{ .register = dst_reg };
1241	},
1242	.stack_offset => |off| {
1243	if (int_info.bits > 64 or @popCount(int_info.bits) != 1)
1244	return self.fail("TODO synthesize SPARCv9 byteswap for other integer sizes", .{});
1245
1246	const off_reg = try self.copyToTmpRegister(operand_ty, .{ .immediate = realStackOffset(off) });
1247	const dst_reg = try self.register_manager.allocReg(null, gp);
1248
1249	try self.genLoadASI(dst_reg, .sp, off_reg, abi_size, opposite_endian_asi);
1250	break :result .{ .register = dst_reg };
1251	},
1252	else => unreachable,
1253	}
1254	},
1255	else => unreachable,
1256	}
1257	};
1258
1259	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1260	}
1261
1262	fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.lang.CallModifier) !void {
1263	if (modifier == .always_tail) return self.fail("TODO implement tail calls for {}", .{self.target.cpu.arch});
1264
1265	const call = self.air.unwrapCall(inst);
1266	const args = call.args;
1267	const ty = self.typeOf(call.callee);
1268	const pt = self.pt;
1269	const zcu = pt.zcu;
1270	const ip = &zcu.intern_pool;
1271	const fn_ty = switch (ty.zigTypeTag(zcu)) {
1272	.@"fn" => ty,
1273	.pointer => ty.childType(zcu),
1274	else => unreachable,
1275	};
1276
1277	var info = try self.resolveCallingConventionValues(fn_ty, .caller);
1278	defer info.deinit(self);
1279
1280	// CCR is volatile across function calls
1281	// (SCD 2.4.1, page 3P-10)
1282	try self.spillConditionFlagsIfOccupied();
1283
1284	// Save caller-saved registers, but crucially *after* we save the
1285	// compare flags as saving compare flags may require a new
1286	// caller-saved register
1287	for (abi.caller_preserved_regs) |reg| {
1288	try self.register_manager.getReg(reg, null);
1289	}
1290
1291	for (info.args, 0..) |mc_arg, arg_i| {
1292	const arg = args[arg_i];
1293	const arg_ty = self.typeOf(arg);
1294	const arg_mcv = try self.resolveInst(arg);
1295
1296	switch (mc_arg) {
1297	.none => continue,
1298	.register => |reg| {
1299	try self.register_manager.getReg(reg, null);
1300	try self.genSetReg(arg_ty, reg, arg_mcv);
1301	},
1302	.stack_offset => {
1303	return self.fail("TODO implement calling with parameters in memory", .{});
1304	},
1305	.ptr_stack_offset => {
1306	return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{});
1307	},
1308	else => unreachable,
1309	}
1310	}
1311
1312	// Due to incremental compilation, how function calls are generated depends
1313	// on linking.
1314	if (call.callee.toInterned()) |func_ip_index| switch (ip.indexToKey(func_ip_index)) {
1315	.func => {
1316	return self.fail("TODO implement calling functions", .{});
1317	},
1318	.@"extern" => {
1319	return self.fail("TODO implement calling extern functions", .{});
1320	},
1321	else => {
1322	return self.fail("TODO implement calling bitcasted functions", .{});
1323	},
1324	} else {
1325	assert(ty.zigTypeTag(zcu) == .pointer);
1326	const mcv = try self.resolveInst(call.callee);
1327	try self.genSetReg(ty, .o7, mcv);
1328
1329	_ = try self.addInst(.{
1330	.tag = .jmpl,
1331	.data = .{
1332	.arithmetic_3op = .{
1333	.is_imm = false,
1334	.rd = .o7,
1335	.rs1 = .o7,
1336	.rs2_or_imm = .{ .rs2 = .g0 },
1337	},
1338	},
1339	});
1340
1341	// TODO Find a way to fill this delay slot
1342	_ = try self.addInst(.{
1343	.tag = .nop,
1344	.data = .{ .nop = {} },
1345	});
1346	}
1347
1348	const result = info.return_value;
1349
1350	if (args.len + 1 <= Air.Liveness.bpi - 1) {
1351	var buf: [Air.Liveness.bpi - 1]Air.Inst.Ref = @splat(.none);
1352	buf[0] = call.callee;
1353	@memcpy(buf[1..][0..args.len], args);
1354	return self.finishAir(inst, result, buf);
1355	}
1356
1357	var bt = try self.iterateBigTomb(inst, 1 + args.len);
1358	bt.feed(call.callee);
1359	for (args) |arg| {
1360	bt.feed(arg);
1361	}
1362	return bt.finishAir(result);
1363	}
1364
1365	fn airClz(self: *Self, inst: Air.Inst.Index) !void {
1366	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
1367	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airClz for {}", .{self.target.cpu.arch});
1368	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1369	}
1370
1371	fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
1372	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
1373	const pt = self.pt;
1374	const zcu = pt.zcu;
1375	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1376	const lhs = try self.resolveInst(bin_op.lhs);
1377	const rhs = try self.resolveInst(bin_op.rhs);
1378	const lhs_ty = self.typeOf(bin_op.lhs);
1379
1380	const int_ty: Type = switch (lhs_ty.zigTypeTag(zcu)) {
1381	.vector => unreachable, // Handled by cmp_vector.
1382	.@"enum" => lhs_ty.backingIntType(zcu),
1383	.int => lhs_ty,
1384	.bool => .u1,
1385	.pointer => .usize,
1386	.error_set => .u16,
1387	.optional => blk: {
1388	const payload_ty = lhs_ty.optionalChild(zcu);
1389	if (!payload_ty.hasRuntimeBits(zcu)) {
1390	break :blk .u1;
1391	} else if (lhs_ty.isPtrLikeOptional(zcu)) {
1392	break :blk .usize;
1393	} else {
1394	return self.fail("TODO SPARCv9 cmp non-pointer optionals", .{});
1395	}
1396	},
1397	.float => return self.fail("TODO SPARCv9 cmp floats", .{}),
1398	else => unreachable,
1399	};
1400
1401	const int_info = int_ty.intInfo(zcu);
1402	if (int_info.bits <= 64) {
1403	_ = try self.binOp(.cmp_eq, lhs, rhs, int_ty, int_ty, BinOpMetadata{
1404	.lhs = bin_op.lhs,
1405	.rhs = bin_op.rhs,
1406	.inst = inst,
1407	});
1408
1409	try self.spillConditionFlagsIfOccupied();
1410	self.condition_flags_inst = inst;
1411
1412	break :result switch (int_info.signedness) {
1413	.signed => MCValue{ .condition_flags = .{
1414	.cond = .{ .icond = Instruction.ICondition.fromCompareOperatorSigned(op) },
1415	.ccr = .xcc,
1416	} },
1417	.unsigned => MCValue{ .condition_flags = .{
1418	.cond = .{ .icond = Instruction.ICondition.fromCompareOperatorUnsigned(op) },
1419	.ccr = .xcc,
1420	} },
1421	};
1422	} else {
1423	return self.fail("TODO SPARCv9 cmp for ints > 64 bits", .{});
1424	}
1425	};
1426	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1427	}
1428
1429	fn airCmpLteErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
1430	const un_op = self.air.instructions.items(.data)[@backingInt(inst)].un_op;
1431	const operand = try self.resolveInst(un_op);
1432	_ = operand;
1433	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airCmpLteErrorsLen for {}", .{self.target.cpu.arch});
1434	return self.finishAir(inst, result, .{ un_op, .none, .none });
1435	}
1436
1437	fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
1438	_ = inst;
1439
1440	return self.fail("TODO implement airCmpxchg for {}", .{
1441	self.target.cpu.arch,
1442	});
1443	}
1444
1445	fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
1446	const cond_br = self.air.unwrapCondBr(inst);
1447	const condition = try self.resolveInst(cond_br.condition);
1448	const then_body = cond_br.then_body;
1449	const else_body = cond_br.else_body;
1450	const liveness_condbr = self.liveness.getCondBr(inst);
1451
1452	// Here we emit a branch to the false section.
1453	const reloc: Mir.Inst.Index = try self.condBr(condition);
1454
1455	// If the condition dies here in this condbr instruction, process
1456	// that death now instead of later as this has an effect on
1457	// whether it needs to be spilled in the branches
1458	if (self.liveness.operandDies(inst, 0)) {
1459	if (cond_br.condition.toIndex()) |op_index| {
1460	self.processDeath(op_index);
1461	}
1462	}
1463
1464	// Capture the state of register and stack allocation state so that we can revert to it.
1465	const parent_next_stack_offset = self.next_stack_offset;
1466	const parent_free_registers = self.register_manager.free_registers;
1467	var parent_stack = try self.stack.clone(self.gpa);
1468	defer parent_stack.deinit(self.gpa);
1469	const parent_registers = self.register_manager.registers;
1470	const parent_condition_flags_inst = self.condition_flags_inst;
1471
1472	try self.branch_stack.append(.{});
1473	errdefer {
1474	_ = self.branch_stack.pop().?;
1475	}
1476
1477	try self.ensureProcessDeathCapacity(liveness_condbr.then_deaths.len);
1478	for (liveness_condbr.then_deaths) |operand| {
1479	self.processDeath(operand);
1480	}
1481	try self.genBody(then_body);
1482
1483	// Revert to the previous register and stack allocation state.
1484
1485	var saved_then_branch = self.branch_stack.pop().?;
1486	defer saved_then_branch.deinit(self.gpa);
1487
1488	self.register_manager.registers = parent_registers;
1489	self.condition_flags_inst = parent_condition_flags_inst;
1490
1491	self.stack.deinit(self.gpa);
1492	self.stack = parent_stack;
1493	parent_stack = .{};
1494
1495	self.next_stack_offset = parent_next_stack_offset;
1496	self.register_manager.free_registers = parent_free_registers;
1497
1498	try self.performReloc(reloc);
1499	const else_branch = self.branch_stack.addOneAssumeCapacity();
1500	else_branch.* = .{};
1501
1502	try self.ensureProcessDeathCapacity(liveness_condbr.else_deaths.len);
1503	for (liveness_condbr.else_deaths) |operand| {
1504	self.processDeath(operand);
1505	}
1506	try self.genBody(else_body);
1507
1508	// At this point, each branch will possibly have conflicting values for where
1509	// each instruction is stored. They agree, however, on which instructions are alive/dead.
1510	// We use the first ("then") branch as canonical, and here emit
1511	// instructions into the second ("else") branch to make it conform.
1512	// We continue respect the data structure semantic guarantees of the else_branch so
1513	// that we can use all the code emitting abstractions. This is why at the bottom we
1514	// assert that parent_branch.free_registers equals the saved_then_branch.free_registers
1515	// rather than assigning it.
1516	const parent_branch = &self.branch_stack.items[self.branch_stack.items.len - 2];
1517	try parent_branch.inst_table.ensureUnusedCapacity(self.gpa, else_branch.inst_table.count());
1518
1519	const else_slice = else_branch.inst_table.entries.slice();
1520	const else_keys = else_slice.items(.key);
1521	const else_values = else_slice.items(.value);
1522	for (else_keys, 0..) |else_key, else_idx| {
1523	const else_value = else_values[else_idx];
1524	const canon_mcv = if (saved_then_branch.inst_table.fetchSwapRemove(else_key)) |then_entry| blk: {
1525	// The instruction's MCValue is overridden in both branches.
1526	log.debug("condBr put branch table (key = %{d}, value = {})", .{ else_key, then_entry.value });
1527	parent_branch.inst_table.putAssumeCapacity(else_key, then_entry.value);
1528	if (else_value == .dead) {
1529	assert(then_entry.value == .dead);
1530	continue;
1531	}
1532	break :blk then_entry.value;
1533	} else blk: {
1534	if (else_value == .dead)
1535	continue;
1536	// The instruction is only overridden in the else branch.
1537	var i: usize = self.branch_stack.items.len - 2;
1538	while (true) {
1539	i -= 1; // If this overflows, the question is: why wasn't the instruction marked dead?
1540	if (self.branch_stack.items[i].inst_table.get(else_key)) |mcv| {
1541	assert(mcv != .dead);
1542	break :blk mcv;
1543	}
1544	}
1545	};
1546	log.debug("consolidating else_entry {d} {}=>{}", .{ else_key, else_value, canon_mcv });
1547	// TODO make sure the destination stack offset / register does not already have something
1548	// going on there.
1549	try self.setRegOrMem(self.typeOfIndex(else_key), canon_mcv, else_value);
1550	// TODO track the new register / stack allocation
1551	}
1552	try parent_branch.inst_table.ensureUnusedCapacity(self.gpa, saved_then_branch.inst_table.count());
1553	const then_slice = saved_then_branch.inst_table.entries.slice();
1554	const then_keys = then_slice.items(.key);
1555	const then_values = then_slice.items(.value);
1556	for (then_keys, 0..) |then_key, then_idx| {
1557	const then_value = then_values[then_idx];
1558	// We already deleted the items from this table that matched the else_branch.
1559	// So these are all instructions that are only overridden in the then branch.
1560	parent_branch.inst_table.putAssumeCapacity(then_key, then_value);
1561	if (then_value == .dead)
1562	continue;
1563	const parent_mcv = blk: {
1564	var i: usize = self.branch_stack.items.len - 2;
1565	while (true) {
1566	i -= 1;
1567	if (self.branch_stack.items[i].inst_table.get(then_key)) |mcv| {
1568	assert(mcv != .dead);
1569	break :blk mcv;
1570	}
1571	}
1572	};
1573	log.debug("consolidating then_entry {d} {}=>{}", .{ then_key, parent_mcv, then_value });
1574	// TODO make sure the destination stack offset / register does not already have something
1575	// going on there.
1576	try self.setRegOrMem(self.typeOfIndex(then_key), parent_mcv, then_value);
1577	// TODO track the new register / stack allocation
1578	}
1579
1580	{
1581	var item = self.branch_stack.pop().?;
1582	item.deinit(self.gpa);
1583	}
1584
1585	// We already took care of pl_op.operand earlier, so we're going
1586	// to pass .none here
1587	return self.finishAir(inst, .unreach, .{ .none, .none, .none });
1588	}
1589
1590	fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
1591	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
1592	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airCtz for {}", .{self.target.cpu.arch});
1593	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1594	}
1595
1596	fn airDbgInlineBlock(self: *Self, inst: Air.Inst.Index) !void {
1597	const block = self.air.unwrapDbgBlock(inst);
1598	// TODO emit debug info for function change
1599	try self.lowerBlock(inst, block.body);
1600	}
1601
1602	fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
1603	const dbg_stmt = self.air.instructions.items(.data)[@backingInt(inst)].dbg_stmt;
1604
1605	_ = try self.addInst(.{
1606	.tag = .dbg_line,
1607	.data = .{
1608	.dbg_line_column = .{
1609	.line = dbg_stmt.line,
1610	.column = dbg_stmt.column,
1611	},
1612	},
1613	});
1614
1615	return self.finishAirBookkeeping();
1616	}
1617
1618	fn airDbgVar(self: *Self, inst: Air.Inst.Index) !void {
1619	const pl_op = self.air.instructions.items(.data)[@backingInt(inst)].pl_op;
1620	const name: Air.NullTerminatedString = @fromBackingInt(@intCast(pl_op.payload));
1621	const operand = pl_op.operand;
1622	// TODO emit debug info for this variable
1623	_ = name;
1624	return self.finishAir(inst, .dead, .{ operand, .none, .none });
1625	}
1626
1627	fn airDiv(self: *Self, inst: Air.Inst.Index) !void {
1628	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
1629	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement div for {}", .{self.target.cpu.arch});
1630	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1631	}
1632
1633	fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
1634	const un_op = self.air.instructions.items(.data)[@backingInt(inst)].un_op;
1635	const operand = try self.resolveInst(un_op);
1636	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else {
1637	_ = operand;
1638	return self.fail("TODO implement airErrorName for {}", .{self.target.cpu.arch});
1639	};
1640	return self.finishAir(inst, result, .{ un_op, .none, .none });
1641	}
1642
1643	fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
1644	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
1645	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement .errunion_payload_ptr_set for {}", .{self.target.cpu.arch});
1646	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1647	}
1648
1649	fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {
1650	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
1651	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airIntFromFloat for {}", .{
1652	self.target.cpu.arch,
1653	});
1654	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1655	}
1656
1657	fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
1658	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
1659	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airGetUnionTag for {}", .{self.target.cpu.arch});
1660	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1661	}
1662
1663	fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
1664	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
1665	if (self.liveness.isUnused(inst))
1666	return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
1667
1668	const pt = self.pt;
1669	const zcu = pt.zcu;
1670	const operand_ty = self.typeOf(ty_op.operand);
1671	const operand = try self.resolveInst(ty_op.operand);
1672	const info_a = operand_ty.intInfo(zcu);
1673	const info_b = self.typeOfIndex(inst).intInfo(zcu);
1674	if (info_a.signedness != info_b.signedness)
1675	return self.fail("TODO gen int_cast sign safety in semantic analysis", .{});
1676
1677	if (info_a.bits == info_b.bits)
1678	return self.finishAir(inst, operand, .{ ty_op.operand, .none, .none });
1679
1680	return self.fail("TODO implement intCast for {}", .{self.target.cpu.arch});
1681	}
1682
1683	fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {
1684	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
1685	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airFloatFromInt for {}", .{
1686	self.target.cpu.arch,
1687	});
1688	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1689	}
1690
1691	fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {
1692	const un_op = self.air.instructions.items(.data)[@backingInt(inst)].un_op;
1693	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1694	const operand = try self.resolveInst(un_op);
1695	const ty = self.typeOf(un_op);
1696	break :result try self.isErr(ty, operand);
1697	};
1698	return self.finishAir(inst, result, .{ un_op, .none, .none });
1699	}
1700
1701	fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void {
1702	const un_op = self.air.instructions.items(.data)[@backingInt(inst)].un_op;
1703	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1704	const operand = try self.resolveInst(un_op);
1705	const ty = self.typeOf(un_op);
1706	break :result try self.isNonErr(ty, operand);
1707	};
1708	return self.finishAir(inst, result, .{ un_op, .none, .none });
1709	}
1710
1711	fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {
1712	const un_op = self.air.instructions.items(.data)[@backingInt(inst)].un_op;
1713	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1714	const operand = try self.resolveInst(un_op);
1715	break :result try self.isNull(operand);
1716	};
1717	return self.finishAir(inst, result, .{ un_op, .none, .none });
1718	}
1719
1720	fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {
1721	const un_op = self.air.instructions.items(.data)[@backingInt(inst)].un_op;
1722	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1723	const operand = try self.resolveInst(un_op);
1724	break :result try self.isNonNull(operand);
1725	};
1726	return self.finishAir(inst, result, .{ un_op, .none, .none });
1727	}
1728
1729	fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
1730	const pt = self.pt;
1731	const zcu = pt.zcu;
1732	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
1733	const elem_ty = self.typeOfIndex(inst);
1734	const elem_size = elem_ty.abiSize(zcu);
1735	const result: MCValue = result: {
1736	if (!elem_ty.hasRuntimeBits(zcu))
1737	break :result MCValue.none;
1738
1739	const ptr = try self.resolveInst(ty_op.operand);
1740	const is_volatile = self.typeOf(ty_op.operand).isVolatilePtr(zcu);
1741	if (self.liveness.isUnused(inst) and !is_volatile)
1742	break :result MCValue.dead;
1743
1744	const dst_mcv: MCValue = blk: {
1745	if (elem_size <= 8 and self.reuseOperand(inst, ty_op.operand, 0, ptr)) {
1746	// The MCValue that holds the pointer can be re-used as the value.
1747	break :blk switch (ptr) {
1748	.register => |r| MCValue{ .register = r },
1749	else => ptr,
1750	};
1751	} else {
1752	break :blk try self.allocRegOrMem(inst, true);
1753	}
1754	};
1755	try self.load(dst_mcv, ptr, self.typeOf(ty_op.operand));
1756	break :result dst_mcv;
1757	};
1758	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1759	}
1760
1761	fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
1762	// A loop is a setup to be able to jump back to the beginning.
1763	const block = self.air.unwrapBlock(inst);
1764	const start: u32 = @intCast(self.mir_instructions.len);
1765
1766	try self.genBody(block.body);
1767	try self.jump(start);
1768
1769	return self.finishAirBookkeeping();
1770	}
1771
1772	fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
1773	if (safety) {
1774	// TODO if the value is undef, write 0xaa bytes to dest
1775	} else {
1776	// TODO if the value is undef, don't lower this instruction
1777	}
1778	const pl_op = self.air.instructions.items(.data)[@backingInt(inst)].pl_op;
1779	const extra = self.air.extraData(Air.Bin, pl_op.payload);
1780
1781	const operand = pl_op.operand;
1782	const value = extra.data.lhs;
1783	const length = extra.data.rhs;
1784	_ = operand;
1785	_ = value;
1786	_ = length;
1787
1788	return self.fail("TODO implement airMemset for {}", .{self.target.cpu.arch});
1789	}
1790
1791	fn airMinMax(self: *Self, inst: Air.Inst.Index) !void {
1792	const tag = self.air.instructions.items(.tag)[@backingInt(inst)];
1793	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
1794	const lhs = try self.resolveInst(bin_op.lhs);
1795	const rhs = try self.resolveInst(bin_op.rhs);
1796	const lhs_ty = self.typeOf(bin_op.lhs);
1797	const rhs_ty = self.typeOf(bin_op.rhs);
1798
1799	const result: MCValue = if (self.liveness.isUnused(inst))
1800	.dead
1801	else
1802	try self.minMax(tag, lhs, rhs, lhs_ty, rhs_ty);
1803
1804	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1805	}
1806
1807	fn airMod(self: *Self, inst: Air.Inst.Index) !void {
1808	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
1809	const lhs = try self.resolveInst(bin_op.lhs);
1810	const rhs = try self.resolveInst(bin_op.rhs);
1811	const lhs_ty = self.typeOf(bin_op.lhs);
1812	const rhs_ty = self.typeOf(bin_op.rhs);
1813	assert(lhs_ty.eql(rhs_ty));
1814
1815	if (self.liveness.isUnused(inst))
1816	return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
1817
1818	// TODO add safety check
1819
1820	// We use manual assembly emission to generate faster code
1821	// First, ensure lhs, rhs, rem, and added are in registers
1822
1823	const lhs_is_register = lhs == .register;
1824	const rhs_is_register = rhs == .register;
1825
1826	const lhs_reg = if (lhs_is_register)
1827	lhs.register
1828	else
1829	try self.register_manager.allocReg(null, gp);
1830
1831	const lhs_lock = self.register_manager.lockReg(lhs_reg);
1832	defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg);
1833
1834	const rhs_reg = if (rhs_is_register)
1835	rhs.register
1836	else
1837	try self.register_manager.allocReg(null, gp);
1838	const rhs_lock = self.register_manager.lockReg(rhs_reg);
1839	defer if (rhs_lock) |reg| self.register_manager.unlockReg(reg);
1840
1841	if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs);
1842	if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs);
1843
1844	const regs = try self.register_manager.allocRegs(2, .{ null, null }, gp);
1845	const regs_locks = self.register_manager.lockRegsAssumeUnused(2, regs);
1846	defer for (regs_locks) |reg| {
1847	self.register_manager.unlockReg(reg);
1848	};
1849
1850	const add_reg = regs[0];
1851	const mod_reg = regs[1];
1852
1853	// mod_reg = @rem(lhs_reg, rhs_reg)
1854	_ = try self.addInst(.{
1855	.tag = .sdivx,
1856	.data = .{
1857	.arithmetic_3op = .{
1858	.is_imm = false,
1859	.rd = mod_reg,
1860	.rs1 = lhs_reg,
1861	.rs2_or_imm = .{ .rs2 = rhs_reg },
1862	},
1863	},
1864	});
1865
1866	_ = try self.addInst(.{
1867	.tag = .mulx,
1868	.data = .{
1869	.arithmetic_3op = .{
1870	.is_imm = false,
1871	.rd = mod_reg,
1872	.rs1 = mod_reg,
1873	.rs2_or_imm = .{ .rs2 = rhs_reg },
1874	},
1875	},
1876	});
1877
1878	_ = try self.addInst(.{
1879	.tag = .sub,
1880	.data = .{
1881	.arithmetic_3op = .{
1882	.is_imm = false,
1883	.rd = mod_reg,
1884	.rs1 = lhs_reg,
1885	.rs2_or_imm = .{ .rs2 = mod_reg },
1886	},
1887	},
1888	});
1889
1890	// add_reg = mod_reg + rhs_reg
1891	_ = try self.addInst(.{
1892	.tag = .add,
1893	.data = .{
1894	.arithmetic_3op = .{
1895	.is_imm = false,
1896	.rd = add_reg,
1897	.rs1 = mod_reg,
1898	.rs2_or_imm = .{ .rs2 = rhs_reg },
1899	},
1900	},
1901	});
1902
1903	// if (add_reg == rhs_reg) add_reg = 0
1904	_ = try self.addInst(.{
1905	.tag = .cmp,
1906	.data = .{
1907	.arithmetic_2op = .{
1908	.is_imm = false,
1909	.rs1 = add_reg,
1910	.rs2_or_imm = .{ .rs2 = rhs_reg },
1911	},
1912	},
1913	});
1914
1915	_ = try self.addInst(.{
1916	.tag = .movcc,
1917	.data = .{
1918	.conditional_move_int = .{
1919	.is_imm = true,
1920	.ccr = .xcc,
1921	.cond = .{ .icond = .eq },
1922	.rd = add_reg,
1923	.rs2_or_imm = .{ .imm = 0 },
1924	},
1925	},
1926	});
1927
1928	// if (lhs_reg < 0) mod_reg = add_reg
1929	_ = try self.addInst(.{
1930	.tag = .movr,
1931	.data = .{
1932	.conditional_move_reg = .{
1933	.is_imm = false,
1934	.cond = .lt_zero,
1935	.rd = mod_reg,
1936	.rs1 = lhs_reg,
1937	.rs2_or_imm = .{ .rs2 = add_reg },
1938	},
1939	},
1940	});
1941
1942	return self.finishAir(inst, .{ .register = mod_reg }, .{ bin_op.lhs, bin_op.rhs, .none });
1943	}
1944
1945	fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
1946	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
1947	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement mul_sat for {}", .{self.target.cpu.arch});
1948	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1949	}
1950
1951	fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1952	//const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)];
1953	const ty_pl = self.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
1954	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
1955	const pt = self.pt;
1956	const zcu = pt.zcu;
1957	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1958	const lhs = try self.resolveInst(extra.lhs);
1959	const rhs = try self.resolveInst(extra.rhs);
1960	const lhs_ty = self.typeOf(extra.lhs);
1961	const rhs_ty = self.typeOf(extra.rhs);
1962
1963	switch (lhs_ty.zigTypeTag(zcu)) {
1964	.vector => return self.fail("TODO implement mul_with_overflow for vectors", .{}),
1965	.int => {
1966	assert(lhs_ty.eql(rhs_ty));
1967	const int_info = lhs_ty.intInfo(zcu);
1968	switch (int_info.bits) {
1969	1...32 => {
1970	try self.spillConditionFlagsIfOccupied();
1971
1972	const dest = try self.binOp(.mul, lhs, rhs, lhs_ty, rhs_ty, null);
1973
1974	const dest_reg = dest.register;
1975	const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg);
1976	defer self.register_manager.unlockReg(dest_reg_lock);
1977
1978	const truncated_reg = try self.register_manager.allocReg(null, gp);
1979	const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg);
1980	defer self.register_manager.unlockReg(truncated_reg_lock);
1981
1982	try self.truncRegister(
1983	dest_reg,
1984	truncated_reg,
1985	int_info.signedness,
1986	int_info.bits,
1987	);
1988
1989	_ = try self.addInst(.{
1990	.tag = .cmp,
1991	.data = .{ .arithmetic_2op = .{
1992	.is_imm = false,
1993	.rs1 = dest_reg,
1994	.rs2_or_imm = .{ .rs2 = truncated_reg },
1995	} },
1996	});
1997
1998	const cond = Instruction.ICondition.ne;
1999	const ccr = Instruction.CCR.xcc;
2000
2001	break :result MCValue{ .register_with_overflow = .{
2002	.reg = truncated_reg,
2003	.flag = .{ .cond = cond, .ccr = ccr },
2004	} };
2005	},
2006	// XXX DO NOT call __multi3 directly as it'll result in us doing six multiplications,
2007	// which is far more than strictly necessary
2008	33...64 => return self.fail("TODO copy compiler-rt's mulddi3 for a 64x64->128 multiply", .{}),
2009	else => return self.fail("TODO overflow operations on other integer sizes", .{}),
2010	}
2011	},
2012	else => unreachable,
2013	}
2014	};
2015	return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
2016	}
2017
2018	fn airNot(self: *Self, inst: Air.Inst.Index) !void {
2019	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2020	const pt = self.pt;
2021	const zcu = pt.zcu;
2022	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2023	const operand = try self.resolveInst(ty_op.operand);
2024	const operand_ty = self.typeOf(ty_op.operand);
2025	switch (operand) {
2026	.dead => unreachable,
2027	.unreach => unreachable,
2028	.condition_flags => |op| {
2029	break :result MCValue{
2030	.condition_flags = .{
2031	.cond = op.cond.negate(),
2032	.ccr = op.ccr,
2033	},
2034	};
2035	},
2036	else => {
2037	switch (operand_ty.zigTypeTag(zcu)) {
2038	.bool => {
2039	const op_reg = switch (operand) {
2040	.register => |r| r,
2041	else => try self.copyToTmpRegister(operand_ty, operand),
2042	};
2043	const reg_lock = self.register_manager.lockRegAssumeUnused(op_reg);
2044	defer self.register_manager.unlockReg(reg_lock);
2045
2046	const dest_reg = blk: {
2047	if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) {
2048	break :blk op_reg;
2049	}
2050
2051	const reg = try self.register_manager.allocReg(null, gp);
2052	break :blk reg;
2053	};
2054
2055	_ = try self.addInst(.{
2056	.tag = .xor,
2057	.data = .{
2058	.arithmetic_3op = .{
2059	.is_imm = true,
2060	.rd = dest_reg,
2061	.rs1 = op_reg,
2062	.rs2_or_imm = .{ .imm = 1 },
2063	},
2064	},
2065	});
2066
2067	break :result MCValue{ .register = dest_reg };
2068	},
2069	.vector => return self.fail("TODO bitwise not for vectors", .{}),
2070	.int => {
2071	const int_info = operand_ty.intInfo(zcu);
2072	if (int_info.bits <= 64) {
2073	const op_reg = switch (operand) {
2074	.register => |r| r,
2075	else => try self.copyToTmpRegister(operand_ty, operand),
2076	};
2077	const reg_lock = self.register_manager.lockRegAssumeUnused(op_reg);
2078	defer self.register_manager.unlockReg(reg_lock);
2079
2080	const dest_reg = blk: {
2081	if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) {
2082	break :blk op_reg;
2083	}
2084
2085	const reg = try self.register_manager.allocReg(null, gp);
2086	break :blk reg;
2087	};
2088
2089	_ = try self.addInst(.{
2090	.tag = .not,
2091	.data = .{
2092	.arithmetic_2op = .{
2093	.is_imm = false,
2094	.rs1 = dest_reg,
2095	.rs2_or_imm = .{ .rs2 = op_reg },
2096	},
2097	},
2098	});
2099
2100	try self.truncRegister(dest_reg, dest_reg, int_info.signedness, int_info.bits);
2101
2102	break :result MCValue{ .register = dest_reg };
2103	} else {
2104	return self.fail("TODO sparc64 not on integers > u64/i64", .{});
2105	}
2106	},
2107	else => unreachable,
2108	}
2109	},
2110	}
2111	};
2112	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2113	}
2114
2115	fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {
2116	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2117	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement .optional_payload for {}", .{self.target.cpu.arch});
2118	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2119	}
2120
2121	fn airOptionalPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
2122	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2123	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement .optional_payload_ptr for {}", .{self.target.cpu.arch});
2124	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2125	}
2126
2127	fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
2128	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2129	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement .optional_payload_ptr_set for {}", .{self.target.cpu.arch});
2130	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2131	}
2132
2133	fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
2134	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2135	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airPopcount for {}", .{self.target.cpu.arch});
2136	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2137	}
2138
2139	fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void {
2140	const prefetch = self.air.instructions.items(.data)[@backingInt(inst)].prefetch;
2141	// TODO Emit a PREFETCH/IPREFETCH as necessary, see A.7 and A.42
2142	return self.finishAir(inst, MCValue.dead, .{ prefetch.ptr, .none, .none });
2143	}
2144
2145	fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
2146	const is_volatile = false; // TODO
2147	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
2148	const result: MCValue = if (!is_volatile and self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement ptr_elem_val for {}", .{self.target.cpu.arch});
2149	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2150	}
2151
2152	fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
2153	const ty_pl = self.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
2154	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
2155	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement ptr_elem_ptr for {}", .{self.target.cpu.arch});
2156	return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
2157	}
2158
2159	fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
2160	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2161	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2162	const ptr_bits = self.target.ptrBitWidth();
2163	const ptr_bytes = @divExact(ptr_bits, 8);
2164	const mcv = try self.resolveInst(ty_op.operand);
2165	switch (mcv) {
2166	.dead, .unreach, .none => unreachable,
2167	.ptr_stack_offset => |off| {
2168	break :result MCValue{ .ptr_stack_offset = off - ptr_bytes };
2169	},
2170	else => return self.fail("TODO implement ptr_slice_len_ptr for {}", .{mcv}),
2171	}
2172	};
2173	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2174	}
2175
2176	fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void {
2177	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2178	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2179	const mcv = try self.resolveInst(ty_op.operand);
2180	switch (mcv) {
2181	.dead, .unreach, .none => unreachable,
2182	.ptr_stack_offset => |off| {
2183	break :result MCValue{ .ptr_stack_offset = off };
2184	},
2185	else => return self.fail("TODO implement ptr_slice_len_ptr for {}", .{mcv}),
2186	}
2187	};
2188	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2189	}
2190
2191	fn airRem(self: *Self, inst: Air.Inst.Index) !void {
2192	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
2193	const lhs = try self.resolveInst(bin_op.lhs);
2194	const rhs = try self.resolveInst(bin_op.rhs);
2195	const lhs_ty = self.typeOf(bin_op.lhs);
2196	const rhs_ty = self.typeOf(bin_op.rhs);
2197
2198	// TODO add safety check
2199
2200	// result = lhs - @divTrunc(lhs, rhs) * rhs
2201	const result: MCValue = if (self.liveness.isUnused(inst)) blk: {
2202	break :blk .dead;
2203	} else blk: {
2204	const tmp0 = try self.binOp(.div_trunc, lhs, rhs, lhs_ty, rhs_ty, null);
2205	const tmp1 = try self.binOp(.mul, tmp0, rhs, lhs_ty, rhs_ty, null);
2206	break :blk try self.binOp(.sub, lhs, tmp1, lhs_ty, rhs_ty, null);
2207	};
2208
2209	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2210	}
2211
2212	fn airRet(self: *Self, inst: Air.Inst.Index) !void {
2213	const un_op = self.air.instructions.items(.data)[@backingInt(inst)].un_op;
2214	const operand = try self.resolveInst(un_op);
2215	try self.ret(operand);
2216	return self.finishAir(inst, .dead, .{ un_op, .none, .none });
2217	}
2218
2219	fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
2220	const un_op = self.air.instructions.items(.data)[@backingInt(inst)].un_op;
2221	const ptr = try self.resolveInst(un_op);
2222	_ = ptr;
2223	return self.fail("TODO implement airRetLoad for {}", .{self.target.cpu.arch});
2224	//return self.finishAir(inst, .dead, .{ un_op, .none, .none });
2225	}
2226
2227	fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {
2228	const stack_offset = try self.allocMemPtr(inst);
2229	return self.finishAir(inst, .{ .ptr_stack_offset = stack_offset }, .{ .none, .none, .none });
2230	}
2231
2232	fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
2233	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
2234	_ = bin_op;
2235	return self.fail("TODO implement airSetUnionTag for {}", .{self.target.cpu.arch});
2236	}
2237
2238	fn airShlSat(self: *Self, inst: Air.Inst.Index) !void {
2239	const zcu = self.pt.zcu;
2240	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
2241	const result: MCValue = if (self.liveness.isUnused(inst))
2242	.dead
2243	else if (self.typeOf(bin_op.lhs).isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu))
2244	return self.fail("TODO implement vector shl_sat with scalar rhs for {}", .{self.target.cpu.arch})
2245	else
2246	return self.fail("TODO implement shl_sat for {}", .{self.target.cpu.arch});
2247	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2248	}
2249
2250	fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2251	const ty_pl = self.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
2252	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
2253	const pt = self.pt;
2254	const zcu = pt.zcu;
2255	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2256	const lhs = try self.resolveInst(extra.lhs);
2257	const rhs = try self.resolveInst(extra.rhs);
2258	const lhs_ty = self.typeOf(extra.lhs);
2259	const rhs_ty = self.typeOf(extra.rhs);
2260
2261	switch (lhs_ty.zigTypeTag(zcu)) {
2262	.vector => if (!rhs_ty.isVector(zcu))
2263	return self.fail("TODO implement vector shl_with_overflow with scalar rhs", .{})
2264	else
2265	return self.fail("TODO implement mul_with_overflow for vectors", .{}),
2266	.int => {
2267	const int_info = lhs_ty.intInfo(zcu);
2268	if (int_info.bits <= 64) {
2269	try self.spillConditionFlagsIfOccupied();
2270
2271	const lhs_lock: ?RegisterLock = if (lhs == .register)
2272	self.register_manager.lockRegAssumeUnused(lhs.register)
2273	else
2274	null;
2275	// TODO this currently crashes stage1
2276	// defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg);
2277
2278	// Increase shift amount (i.e, rhs) by shamt_bits - int_info.bits
2279	// e.g if shifting a i48 then use sr*x (shamt_bits == 64) but increase rhs by 16
2280	// and if shifting a i24 then use sr* (shamt_bits == 32) but increase rhs by 8
2281	const new_rhs = switch (int_info.bits) {
2282	1...31 => if (rhs == .immediate) MCValue{
2283	.immediate = rhs.immediate + 32 - int_info.bits,
2284	} else try self.binOp(.add, rhs, .{ .immediate = 32 - int_info.bits }, rhs_ty, rhs_ty, null),
2285	33...63 => if (rhs == .immediate) MCValue{
2286	.immediate = rhs.immediate + 64 - int_info.bits,
2287	} else try self.binOp(.add, rhs, .{ .immediate = 64 - int_info.bits }, rhs_ty, rhs_ty, null),
2288	32, 64 => rhs,
2289	else => unreachable,
2290	};
2291
2292	const new_rhs_lock: ?RegisterLock = if (new_rhs == .register)
2293	self.register_manager.lockRegAssumeUnused(new_rhs.register)
2294	else
2295	null;
2296	// TODO this currently crashes stage1
2297	// defer if (new_rhs_lock) |reg| self.register_manager.unlockReg(reg);
2298
2299	const dest = try self.binOp(.shl, lhs, new_rhs, lhs_ty, rhs_ty, null);
2300	const dest_reg = dest.register;
2301	const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg);
2302	defer self.register_manager.unlockReg(dest_reg_lock);
2303
2304	const shr = try self.binOp(.shr, dest, new_rhs, lhs_ty, rhs_ty, null);
2305
2306	_ = try self.addInst(.{
2307	.tag = .cmp,
2308	.data = .{ .arithmetic_2op = .{
2309	.is_imm = false,
2310	.rs1 = dest_reg,
2311	.rs2_or_imm = .{ .rs2 = shr.register },
2312	} },
2313	});
2314
2315	const cond = Instruction.ICondition.ne;
2316	const ccr = switch (int_info.bits) {
2317	1...32 => Instruction.CCR.icc,
2318	33...64 => Instruction.CCR.xcc,
2319	else => unreachable,
2320	};
2321
2322	// TODO Those should really be written as defers, however stage1 currently
2323	// panics when those are turned into defer statements so those are
2324	// written here at the end as ordinary statements.
2325	// Because of that, on failure, the lock on those registers wouldn't be
2326	// released.
2327	if (lhs_lock) |reg| self.register_manager.unlockReg(reg);
2328	if (new_rhs_lock) |reg| self.register_manager.unlockReg(reg);
2329
2330	break :result MCValue{ .register_with_overflow = .{
2331	.reg = dest_reg,
2332	.flag = .{ .cond = cond, .ccr = ccr },
2333	} };
2334	} else {
2335	return self.fail("TODO overflow operations on other integer sizes", .{});
2336	}
2337	},
2338	else => unreachable,
2339	}
2340	};
2341	return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
2342	}
2343
2344	fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
2345	const ty_pl = self.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
2346	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
2347	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2348	const ptr = try self.resolveInst(bin_op.lhs);
2349	const ptr_ty = self.typeOf(bin_op.lhs);
2350	const len = try self.resolveInst(bin_op.rhs);
2351	const len_ty = self.typeOf(bin_op.rhs);
2352	const ptr_bytes = 8;
2353	const stack_offset = try self.allocMem(inst, ptr_bytes * 2, .@"8");
2354	try self.genSetStack(ptr_ty, stack_offset, ptr);
2355	try self.genSetStack(len_ty, stack_offset - ptr_bytes, len);
2356	break :result MCValue{ .stack_offset = stack_offset };
2357	};
2358	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2359	}
2360
2361	fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
2362	const pt = self.pt;
2363	const zcu = pt.zcu;
2364	const is_volatile = false; // TODO
2365	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
2366
2367	if (!is_volatile and self.liveness.isUnused(inst)) return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
2368	const result: MCValue = result: {
2369	const slice_mcv = try self.resolveInst(bin_op.lhs);
2370	const index_mcv = try self.resolveInst(bin_op.rhs);
2371
2372	const slice_ty = self.typeOf(bin_op.lhs);
2373	const elem_ty = slice_ty.childType(zcu);
2374	const elem_size = elem_ty.abiSize(zcu);
2375
2376	const slice_ptr_field_type = slice_ty.slicePtrFieldType(zcu);
2377
2378	const index_lock: ?RegisterLock = if (index_mcv == .register)
2379	self.register_manager.lockRegAssumeUnused(index_mcv.register)
2380	else
2381	null;
2382	defer if (index_lock) |reg| self.register_manager.unlockReg(reg);
2383
2384	const base_mcv: MCValue = switch (slice_mcv) {
2385	.stack_offset => |off| .{ .register = try self.copyToTmpRegister(slice_ptr_field_type, .{ .stack_offset = off }) },
2386	else => return self.fail("TODO slice_elem_val when slice is {}", .{slice_mcv}),
2387	};
2388	const base_lock = self.register_manager.lockRegAssumeUnused(base_mcv.register);
2389	defer self.register_manager.unlockReg(base_lock);
2390
2391	switch (elem_size) {
2392	else => {
2393	// TODO skip the ptr_add emission entirely and use native addressing modes
2394	// i.e sllx/mulx then R+R or scale immediate then R+I
2395	const dest = try self.allocRegOrMem(inst, true);
2396	const addr = try self.binOp(.ptr_add, base_mcv, index_mcv, slice_ptr_field_type, Type.usize, null);
2397	try self.load(dest, addr, slice_ptr_field_type);
2398
2399	break :result dest;
2400	},
2401	}
2402	};
2403	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2404	}
2405
2406	fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {
2407	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2408	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2409	const ptr_bits = self.target.ptrBitWidth();
2410	const ptr_bytes = @divExact(ptr_bits, 8);
2411	const mcv = try self.resolveInst(ty_op.operand);
2412	switch (mcv) {
2413	.dead, .unreach, .none => unreachable,
2414	.register => unreachable, // a slice doesn't fit in one register
2415	.stack_offset => |off| {
2416	break :result MCValue{ .stack_offset = off - ptr_bytes };
2417	},
2418	.memory => |addr| {
2419	break :result MCValue{ .memory = addr + ptr_bytes };
2420	},
2421	else => return self.fail("TODO implement slice_len for {}", .{mcv}),
2422	}
2423	};
2424	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2425	}
2426
2427	fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {
2428	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2429	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2430	const mcv = try self.resolveInst(ty_op.operand);
2431	switch (mcv) {
2432	.dead, .unreach, .none => unreachable,
2433	.register => unreachable, // a slice doesn't fit in one register
2434	.stack_offset => |off| {
2435	break :result MCValue{ .stack_offset = off };
2436	},
2437	.memory => |addr| {
2438	break :result MCValue{ .memory = addr };
2439	},
2440	else => return self.fail("TODO implement slice_len for {}", .{mcv}),
2441	}
2442	};
2443	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2444	}
2445
2446	fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
2447	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2448	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airSplat for {}", .{self.target.cpu.arch});
2449	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2450	}
2451
2452	fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
2453	if (safety) {
2454	// TODO if the value is undef, write 0xaa bytes to dest
2455	} else {
2456	// TODO if the value is undef, don't lower this instruction
2457	}
2458	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
2459	const ptr = try self.resolveInst(bin_op.lhs);
2460	const value = try self.resolveInst(bin_op.rhs);
2461	const ptr_ty = self.typeOf(bin_op.lhs);
2462	const value_ty = self.typeOf(bin_op.rhs);
2463
2464	try self.store(ptr, value, ptr_ty, value_ty);
2465
2466	return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
2467	}
2468
2469	fn airStructFieldPtr(self: *Self, inst: Air.Inst.Index) !void {
2470	const ty_pl = self.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
2471	const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
2472	const result = try self.structFieldPtr(inst, extra.struct_operand, extra.field_index);
2473	return self.finishAir(inst, result, .{ extra.struct_operand, .none, .none });
2474	}
2475
2476	fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, index: u8) !void {
2477	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2478	const result = try self.structFieldPtr(inst, ty_op.operand, index);
2479	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2480	}
2481
2482	fn airAggFieldVal(self: *Self, inst: Air.Inst.Index) !void {
2483	const ty_pl = self.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
2484	const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
2485	const operand = extra.struct_operand;
2486	const index = extra.field_index;
2487	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2488	const zcu = self.pt.zcu;
2489	const mcv = try self.resolveInst(operand);
2490	const struct_ty = self.typeOf(operand);
2491	const struct_field_offset: u32 = @intCast(struct_ty.structFieldOffset(index, zcu));
2492
2493	switch (mcv) {
2494	.dead, .unreach => unreachable,
2495	.stack_offset => |off| {
2496	break :result MCValue{ .stack_offset = off - struct_field_offset };
2497	},
2498	.memory => |addr| {
2499	break :result MCValue{ .memory = addr + struct_field_offset };
2500	},
2501	.register_with_overflow => |rwo| {
2502	switch (index) {
2503	0 => {
2504	// get wrapped value: return register
2505	break :result MCValue{ .register = rwo.reg };
2506	},
2507	1 => {
2508	// TODO return special MCValue condition flags
2509	// get overflow bit: set register to C flag
2510	// resp. V flag
2511	const dest_reg = try self.register_manager.allocReg(null, gp);
2512
2513	// TODO handle floating point CCRs
2514	assert(rwo.flag.ccr == .xcc or rwo.flag.ccr == .icc);
2515
2516	_ = try self.addInst(.{
2517	.tag = .mov,
2518	.data = .{
2519	.arithmetic_2op = .{
2520	.is_imm = false,
2521	.rs1 = dest_reg,
2522	.rs2_or_imm = .{ .rs2 = .g0 },
2523	},
2524	},
2525	});
2526
2527	_ = try self.addInst(.{
2528	.tag = .movcc,
2529	.data = .{
2530	.conditional_move_int = .{
2531	.ccr = rwo.flag.ccr,
2532	.cond = .{ .icond = rwo.flag.cond },
2533	.is_imm = true,
2534	.rd = dest_reg,
2535	.rs2_or_imm = .{ .imm = 1 },
2536	},
2537	},
2538	});
2539
2540	break :result MCValue{ .register = dest_reg };
2541	},
2542	else => unreachable,
2543	}
2544	},
2545	else => return self.fail("TODO implement codegen agg_field_val for {}", .{mcv}),
2546	}
2547	};
2548
2549	return self.finishAir(inst, result, .{ extra.struct_operand, .none, .none });
2550	}
2551
2552	fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
2553	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
2554	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement sub_sat for {}", .{self.target.cpu.arch});
2555	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2556	}
2557
2558	fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
2559	_ = inst;
2560	return self.fail("TODO implement switch for {}", .{self.target.cpu.arch});
2561	}
2562
2563	fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
2564	const un_op = self.air.instructions.items(.data)[@backingInt(inst)].un_op;
2565	const operand = try self.resolveInst(un_op);
2566	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else {
2567	_ = operand;
2568	return self.fail("TODO implement airTagName for {}", .{self.target.cpu.arch});
2569	};
2570	return self.finishAir(inst, result, .{ un_op, .none, .none });
2571	}
2572
2573	fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
2574	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2575	const operand = try self.resolveInst(ty_op.operand);
2576	const operand_ty = self.typeOf(ty_op.operand);
2577	const dest_ty = self.typeOfIndex(inst);
2578
2579	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else blk: {
2580	break :blk try self.trunc(inst, operand, operand_ty, dest_ty);
2581	};
2582
2583	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2584	}
2585
2586	fn airTry(self: *Self, inst: Air.Inst.Index) !void {
2587	const unwrapped_try = self.air.unwrapTry(inst);
2588	const body = unwrapped_try.else_body;
2589	const result: MCValue = result: {
2590	const error_union_ty = self.air.typeOf(unwrapped_try.error_union, &self.pt.zcu.intern_pool);
2591	const error_union = try self.resolveInst(unwrapped_try.error_union);
2592	const is_err_result = try self.isErr(error_union_ty, error_union);
2593	const reloc = try self.condBr(is_err_result);
2594
2595	try self.genBody(body);
2596
2597	try self.performReloc(reloc);
2598	break :result try self.errUnionPayload(error_union, error_union_ty);
2599	};
2600	return self.finishAir(inst, result, .{ unwrapped_try.error_union, .none, .none });
2601	}
2602
2603	fn airUnaryMath(self: *Self, inst: Air.Inst.Index) !void {
2604	const un_op = self.air.instructions.items(.data)[@backingInt(inst)].un_op;
2605	const result: MCValue = if (self.liveness.isUnused(inst))
2606	.dead
2607	else
2608	return self.fail("TODO implement airUnaryMath for {}", .{self.target.cpu.arch});
2609	return self.finishAir(inst, result, .{ un_op, .none, .none });
2610	}
2611
2612	fn airUnionInit(self: *Self, inst: Air.Inst.Index) !void {
2613	const ty_pl = self.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
2614	const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data;
2615	_ = extra;
2616	return self.fail("TODO implement airUnionInit for {}", .{self.target.cpu.arch});
2617	}
2618
2619	fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) !void {
2620	const pt = self.pt;
2621	const zcu = pt.zcu;
2622	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2623	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2624	const error_union_ty = self.typeOf(ty_op.operand);
2625	const payload_ty = error_union_ty.errorUnionPayload(zcu);
2626	const mcv = try self.resolveInst(ty_op.operand);
2627	if (!payload_ty.hasRuntimeBits(zcu)) break :result mcv;
2628
2629	return self.fail("TODO implement unwrap error union error for non-empty payloads", .{});
2630	};
2631	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2632	}
2633
2634	fn airUnwrapErrPayload(self: *Self, inst: Air.Inst.Index) !void {
2635	const pt = self.pt;
2636	const zcu = pt.zcu;
2637	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2638	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2639	const error_union_ty = self.typeOf(ty_op.operand);
2640	const payload_ty = error_union_ty.errorUnionPayload(zcu);
2641	if (!payload_ty.hasRuntimeBits(zcu)) break :result MCValue.none;
2642
2643	return self.fail("TODO implement unwrap error union payload for non-empty payloads", .{});
2644	};
2645	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2646	}
2647
2648	/// E to E!T
2649	fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
2650	const pt = self.pt;
2651	const zcu = pt.zcu;
2652	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2653	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2654	const error_union_ty = ty_op.ty;
2655	const payload_ty = error_union_ty.errorUnionPayload(zcu);
2656	const mcv = try self.resolveInst(ty_op.operand);
2657	if (!payload_ty.hasRuntimeBits(zcu)) break :result mcv;
2658
2659	return self.fail("TODO implement wrap errunion error for non-empty payloads", .{});
2660	};
2661	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2662	}
2663
2664	/// T to E!T
2665	fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
2666	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2667	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement wrap errunion payload for {}", .{self.target.cpu.arch});
2668	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2669	}
2670
2671	fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
2672	const pt = self.pt;
2673	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2674	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2675	const optional_ty = self.typeOfIndex(inst);
2676
2677	// Optional with a zero-bit payload type is just a boolean true
2678	if (optional_ty.abiSize(pt.zcu) == 1)
2679	break :result MCValue{ .immediate = 1 };
2680
2681	return self.fail("TODO implement wrap optional for {}", .{self.target.cpu.arch});
2682	};
2683	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2684	}
2685
2686	// Common helper functions
2687
2688	fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {
2689	const gpa = self.gpa;
2690	try self.mir_instructions.ensureUnusedCapacity(gpa, 1);
2691	const result_index: Mir.Inst.Index = @intCast(self.mir_instructions.len);
2692	self.mir_instructions.appendAssumeCapacity(inst);
2693	return result_index;
2694	}
2695
2696	fn allocMem(self: *Self, inst: Air.Inst.Index, abi_size: u32, abi_align: Alignment) !u32 {
2697	self.stack_align = self.stack_align.max(abi_align);
2698	// TODO find a free slot instead of always appending
2699	const offset: u32 = @intCast(abi_align.forward(self.next_stack_offset) + abi_size);
2700	self.next_stack_offset = offset;
2701	if (self.next_stack_offset > self.max_end_stack)
2702	self.max_end_stack = self.next_stack_offset;
2703	try self.stack.putNoClobber(self.gpa, offset, .{
2704	.inst = inst,
2705	.size = abi_size,
2706	});
2707	return offset;
2708	}
2709
2710	/// Use a pointer instruction as the basis for allocating stack memory.
2711	fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !u32 {
2712	const pt = self.pt;
2713	const zcu = pt.zcu;
2714	const elem_ty = self.typeOfIndex(inst).childType(zcu);
2715
2716	if (!elem_ty.hasRuntimeBits(zcu)) {
2717	// As this stack item will never be dereferenced at runtime,
2718	// return the stack offset 0. Stack offset 0 will be where all
2719	// zero-sized stack allocations live as non-zero-sized
2720	// allocations will always have an offset > 0.
2721	return @as(u32, 0);
2722	}
2723
2724	const abi_size = math.cast(u32, elem_ty.abiSize(zcu)) orelse {
2725	return self.fail("type '{f}' too big to fit into stack frame", .{elem_ty.fmt(pt)});
2726	};
2727	// TODO swap this for inst.ty.ptrAlign
2728	const abi_align = elem_ty.abiAlignment(zcu);
2729	return self.allocMem(inst, abi_size, abi_align);
2730	}
2731
2732	fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
2733	const pt = self.pt;
2734	const zcu = pt.zcu;
2735	const elem_ty = self.typeOfIndex(inst);
2736	const abi_size = math.cast(u32, elem_ty.abiSize(zcu)) orelse {
2737	return self.fail("type '{f}' too big to fit into stack frame", .{elem_ty.fmt(pt)});
2738	};
2739	const abi_align = elem_ty.abiAlignment(zcu);
2740	self.stack_align = self.stack_align.max(abi_align);
2741
2742	if (reg_ok) {
2743	// Make sure the type can fit in a register before we try to allocate one.
2744	if (abi_size <= 8) {
2745	if (self.register_manager.tryAllocReg(inst, gp)) |reg| {
2746	return MCValue{ .register = reg };
2747	}
2748	}
2749	}
2750	const stack_offset = try self.allocMem(inst, abi_size, abi_align);
2751	return MCValue{ .stack_offset = stack_offset };
2752	}
2753
2754	const BinOpMetadata = struct {
2755	inst: Air.Inst.Index,
2756	lhs: Air.Inst.Ref,
2757	rhs: Air.Inst.Ref,
2758	};
2759
2760	/// For all your binary operation needs, this function will generate
2761	/// the corresponding Mir instruction(s). Returns the location of the
2762	/// result.
2763	///
2764	/// If the binary operation itself happens to be an Air instruction,
2765	/// pass the corresponding index in the inst parameter. That helps
2766	/// this function do stuff like reusing operands.
2767	///
2768	/// This function does not do any lowering to Mir itself, but instead
2769	/// looks at the lhs and rhs and determines which kind of lowering
2770	/// would be best suitable and then delegates the lowering to other
2771	/// functions.
2772	fn binOp(
2773	self: *Self,
2774	tag: Air.Inst.Tag,
2775	lhs: MCValue,
2776	rhs: MCValue,
2777	lhs_ty: Type,
2778	rhs_ty: Type,
2779	metadata: ?BinOpMetadata,
2780	) InnerError!MCValue {
2781	const pt = self.pt;
2782	const zcu = pt.zcu;
2783	switch (tag) {
2784	.add,
2785	.sub,
2786	.mul,
2787	.bit_and,
2788	.bit_or,
2789	.xor,
2790	.cmp_eq,
2791	=> {
2792	switch (lhs_ty.zigTypeTag(zcu)) {
2793	.float => return self.fail("TODO binary operations on floats", .{}),
2794	.vector => return self.fail("TODO binary operations on vectors", .{}),
2795	.int => {
2796	assert(lhs_ty.eql(rhs_ty));
2797	const int_info = lhs_ty.intInfo(zcu);
2798	if (int_info.bits <= 64) {
2799	// Only say yes if the operation is
2800	// commutative, i.e. we can swap both of the
2801	// operands
2802	const lhs_immediate_ok = switch (tag) {
2803	.add => lhs == .immediate and lhs.immediate <= std.math.maxInt(u12),
2804	.mul => lhs == .immediate and lhs.immediate <= std.math.maxInt(u12),
2805	.bit_and => lhs == .immediate and lhs.immediate <= std.math.maxInt(u12),
2806	.bit_or => lhs == .immediate and lhs.immediate <= std.math.maxInt(u12),
2807	.xor => lhs == .immediate and lhs.immediate <= std.math.maxInt(u12),
2808	.sub, .cmp_eq => false,
2809	else => unreachable,
2810	};
2811	const rhs_immediate_ok = switch (tag) {
2812	.add,
2813	.sub,
2814	.mul,
2815	.bit_and,
2816	.bit_or,
2817	.xor,
2818	.cmp_eq,
2819	=> rhs == .immediate and rhs.immediate <= std.math.maxInt(u12),
2820	else => unreachable,
2821	};
2822
2823	const mir_tag: Mir.Inst.Tag = switch (tag) {
2824	.add => .add,
2825	.sub => .sub,
2826	.mul => .mulx,
2827	.bit_and => .@"and",
2828	.bit_or => .@"or",
2829	.xor => .xor,
2830	.cmp_eq => .cmp,
2831	else => unreachable,
2832	};
2833
2834	if (rhs_immediate_ok) {
2835	return try self.binOpImmediate(mir_tag, lhs, rhs, lhs_ty, false, metadata);
2836	} else if (lhs_immediate_ok) {
2837	// swap lhs and rhs
2838	return try self.binOpImmediate(mir_tag, rhs, lhs, rhs_ty, true, metadata);
2839	} else {
2840	// TODO convert large immediates to register before adding
2841	return try self.binOpRegister(mir_tag, lhs, rhs, lhs_ty, rhs_ty, metadata);
2842	}
2843	} else {
2844	return self.fail("TODO binary operations on int with bits > 64", .{});
2845	}
2846	},
2847	else => unreachable,
2848	}
2849	},
2850
2851	.add_wrap,
2852	.sub_wrap,
2853	.mul_wrap,
2854	=> {
2855	const base_tag: Air.Inst.Tag = switch (tag) {
2856	.add_wrap => .add,
2857	.sub_wrap => .sub,
2858	.mul_wrap => .mul,
2859	else => unreachable,
2860	};
2861
2862	// Generate the base operation
2863	const result = try self.binOp(base_tag, lhs, rhs, lhs_ty, rhs_ty, metadata);
2864
2865	// Truncate if necessary
2866	switch (lhs_ty.zigTypeTag(zcu)) {
2867	.vector => return self.fail("TODO binary operations on vectors", .{}),
2868	.int => {
2869	const int_info = lhs_ty.intInfo(zcu);
2870	if (int_info.bits <= 64) {
2871	const result_reg = result.register;
2872	try self.truncRegister(result_reg, result_reg, int_info.signedness, int_info.bits);
2873	return result;
2874	} else {
2875	return self.fail("TODO binary operations on integers > u64/i64", .{});
2876	}
2877	},
2878	else => unreachable,
2879	}
2880	},
2881
2882	.div_trunc => {
2883	switch (lhs_ty.zigTypeTag(zcu)) {
2884	.vector => return self.fail("TODO binary operations on vectors", .{}),
2885	.int => {
2886	assert(lhs_ty.eql(rhs_ty));
2887	const int_info = lhs_ty.intInfo(zcu);
2888	if (int_info.bits <= 64) {
2889	const rhs_immediate_ok = switch (tag) {
2890	.div_trunc => rhs == .immediate and rhs.immediate <= std.math.maxInt(u12),
2891	else => unreachable,
2892	};
2893
2894	const mir_tag: Mir.Inst.Tag = switch (tag) {
2895	.div_trunc => switch (int_info.signedness) {
2896	.signed => Mir.Inst.Tag.sdivx,
2897	.unsigned => Mir.Inst.Tag.udivx,
2898	},
2899	else => unreachable,
2900	};
2901
2902	if (rhs_immediate_ok) {
2903	return try self.binOpImmediate(mir_tag, lhs, rhs, lhs_ty, true, metadata);
2904	} else {
2905	return try self.binOpRegister(mir_tag, lhs, rhs, lhs_ty, rhs_ty, metadata);
2906	}
2907	} else {
2908	return self.fail("TODO binary operations on int with bits > 64", .{});
2909	}
2910	},
2911	else => unreachable,
2912	}
2913	},
2914
2915	.ptr_add => {
2916	switch (lhs_ty.zigTypeTag(zcu)) {
2917	.pointer => {
2918	const ptr_ty = lhs_ty;
2919	const elem_ty = switch (ptr_ty.ptrSize(zcu)) {
2920	.one => ptr_ty.childType(zcu).childType(zcu), // ptr to array, so get array element type
2921	else => ptr_ty.childType(zcu),
2922	};
2923	const elem_size = elem_ty.abiSize(zcu);
2924
2925	if (elem_size == 1) {
2926	const base_tag: Mir.Inst.Tag = switch (tag) {
2927	.ptr_add => .add,
2928	else => unreachable,
2929	};
2930
2931	return try self.binOpRegister(base_tag, lhs, rhs, lhs_ty, rhs_ty, metadata);
2932	} else {
2933	// convert the offset into a byte offset by
2934	// multiplying it with elem_size
2935
2936	const offset = try self.binOp(.mul, rhs, .{ .immediate = elem_size }, Type.usize, Type.usize, null);
2937	const addr = try self.binOp(tag, lhs, offset, Type.manyptr_u8, Type.usize, null);
2938	return addr;
2939	}
2940	},
2941	else => unreachable,
2942	}
2943	},
2944
2945	.shl,
2946	.shr,
2947	=> {
2948	const base_tag: Air.Inst.Tag = switch (tag) {
2949	.shl => .shl_exact,
2950	.shr => .shr_exact,
2951	else => unreachable,
2952	};
2953
2954	// Generate the base operation
2955	const result = try self.binOp(base_tag, lhs, rhs, lhs_ty, rhs_ty, metadata);
2956
2957	// Truncate if necessary
2958	switch (lhs_ty.zigTypeTag(zcu)) {
2959	.vector => if (rhs_ty.isVector(zcu))
2960	return self.fail("TODO vector shift with scalar rhs", .{})
2961	else
2962	return self.fail("TODO binary operations on vectors", .{}),
2963	.int => {
2964	const int_info = lhs_ty.intInfo(zcu);
2965	if (int_info.bits <= 64) {
2966	// 32 and 64 bit operands doesn't need truncating
2967	if (int_info.bits == 32 or int_info.bits == 64) return result;
2968
2969	const result_reg = result.register;
2970	try self.truncRegister(result_reg, result_reg, int_info.signedness, int_info.bits);
2971	return result;
2972	} else {
2973	return self.fail("TODO binary operations on integers > u64/i64", .{});
2974	}
2975	},
2976	else => unreachable,
2977	}
2978	},
2979
2980	.shl_exact,
2981	.shr_exact,
2982	=> {
2983	switch (lhs_ty.zigTypeTag(zcu)) {
2984	.vector => if (rhs_ty.isVector(zcu))
2985	return self.fail("TODO vector shift with scalar rhs", .{})
2986	else
2987	return self.fail("TODO binary operations on vectors", .{}),
2988	.int => {
2989	const int_info = lhs_ty.intInfo(zcu);
2990	if (int_info.bits <= 64) {
2991	const rhs_immediate_ok = rhs == .immediate;
2992
2993	const mir_tag: Mir.Inst.Tag = switch (tag) {
2994	.shl_exact => if (int_info.bits <= 32) Mir.Inst.Tag.sll else Mir.Inst.Tag.sllx,
2995	.shr_exact => switch (int_info.signedness) {
2996	.signed => if (int_info.bits <= 32) Mir.Inst.Tag.sra else Mir.Inst.Tag.srax,
2997	.unsigned => if (int_info.bits <= 32) Mir.Inst.Tag.srl else Mir.Inst.Tag.srlx,
2998	},
2999	else => unreachable,
3000	};
3001
3002	if (rhs_immediate_ok) {
3003	return try self.binOpImmediate(mir_tag, lhs, rhs, lhs_ty, false, metadata);
3004	} else {
3005	return try self.binOpRegister(mir_tag, lhs, rhs, lhs_ty, rhs_ty, metadata);
3006	}
3007	} else {
3008	return self.fail("TODO binary operations on int with bits > 64", .{});
3009	}
3010	},
3011	else => unreachable,
3012	}
3013	},
3014
3015	else => return self.fail("TODO implement {} binOp for SPARCv9", .{tag}),
3016	}
3017	}
3018
3019	/// Don't call this function directly. Use binOp instead.
3020	///
3021	/// Calling this function signals an intention to generate a Mir
3022	/// instruction of the form
3023	///
3024	/// op dest, lhs, #rhs_imm
3025	///
3026	/// Set lhs_and_rhs_swapped to true iff inst.bin_op.lhs corresponds to
3027	/// rhs and vice versa. This parameter is only used when metadata != null.
3028	///
3029	/// Asserts that generating an instruction of that form is possible.
3030	fn binOpImmediate(
3031	self: *Self,
3032	mir_tag: Mir.Inst.Tag,
3033	lhs: MCValue,
3034	rhs: MCValue,
3035	lhs_ty: Type,
3036	lhs_and_rhs_swapped: bool,
3037	metadata: ?BinOpMetadata,
3038	) !MCValue {
3039	const lhs_is_register = lhs == .register;
3040
3041	const lhs_lock: ?RegisterLock = if (lhs_is_register)
3042	self.register_manager.lockReg(lhs.register)
3043	else
3044	null;
3045	defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg);
3046
3047	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
3048
3049	const lhs_reg = if (lhs_is_register) lhs.register else blk: {
3050	const track_inst: ?Air.Inst.Index = if (metadata) |md| inst: {
3051	break :inst (if (lhs_and_rhs_swapped) md.rhs else md.lhs).toIndex().?;
3052	} else null;
3053
3054	const reg = try self.register_manager.allocReg(track_inst, gp);
3055
3056	if (track_inst) |inst| {
3057	const mcv: MCValue = .{ .register = reg };
3058	log.debug("binOpRegister move lhs %{d} to register: {} -> {}", .{ inst, lhs, mcv });
3059	branch.inst_table.putAssumeCapacity(inst, mcv);
3060
3061	// If we're moving a condition flag MCV to register,
3062	// mark it as free.
3063	if (lhs == .condition_flags) {
3064	assert(self.condition_flags_inst.? == inst);
3065	self.condition_flags_inst = null;
3066	}
3067	}
3068
3069	break :blk reg;
3070	};
3071	const new_lhs_lock = self.register_manager.lockReg(lhs_reg);
3072	defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg);
3073
3074	const dest_reg = switch (mir_tag) {
3075	.cmp => undefined, // cmp has no destination register
3076	else => if (metadata) |md| blk: {
3077	if (lhs_is_register and self.reuseOperand(
3078	md.inst,
3079	if (lhs_and_rhs_swapped) md.rhs else md.lhs,
3080	if (lhs_and_rhs_swapped) 1 else 0,
3081	lhs,
3082	)) {
3083	break :blk lhs_reg;
3084	} else {
3085	break :blk try self.register_manager.allocReg(md.inst, gp);
3086	}
3087	} else blk: {
3088	break :blk try self.register_manager.allocReg(null, gp);
3089	},
3090	};
3091
3092	if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs);
3093
3094	const mir_data: Mir.Inst.Data = switch (mir_tag) {
3095	.add,
3096	.addcc,
3097	.@"and",
3098	.@"or",
3099	.xor,
3100	.xnor,
3101	.mulx,
3102	.sdivx,
3103	.udivx,
3104	.sub,
3105	.subcc,
3106	=> .{
3107	.arithmetic_3op = .{
3108	.is_imm = true,
3109	.rd = dest_reg,
3110	.rs1 = lhs_reg,
3111	.rs2_or_imm = .{ .imm = @as(u12, @intCast(rhs.immediate)) },
3112	},
3113	},
3114	.sll,
3115	.srl,
3116	.sra,
3117	=> .{
3118	.shift = .{
3119	.is_imm = true,
3120	.rd = dest_reg,
3121	.rs1 = lhs_reg,
3122	.rs2_or_imm = .{ .imm = @as(u5, @intCast(rhs.immediate)) },
3123	},
3124	},
3125	.sllx,
3126	.srlx,
3127	.srax,
3128	=> .{
3129	.shift = .{
3130	.is_imm = true,
3131	.rd = dest_reg,
3132	.rs1 = lhs_reg,
3133	.rs2_or_imm = .{ .imm = @as(u6, @intCast(rhs.immediate)) },
3134	},
3135	},
3136	.cmp => .{
3137	.arithmetic_2op = .{
3138	.is_imm = true,
3139	.rs1 = lhs_reg,
3140	.rs2_or_imm = .{ .imm = @as(u12, @intCast(rhs.immediate)) },
3141	},
3142	},
3143	else => unreachable,
3144	};
3145
3146	_ = try self.addInst(.{
3147	.tag = mir_tag,
3148	.data = mir_data,
3149	});
3150
3151	return MCValue{ .register = dest_reg };
3152	}
3153
3154	/// Don't call this function directly. Use binOp instead.
3155	///
3156	/// Calling this function signals an intention to generate a Mir
3157	/// instruction of the form
3158	///
3159	/// op dest, lhs, rhs
3160	///
3161	/// Asserts that generating an instruction of that form is possible.
3162	fn binOpRegister(
3163	self: *Self,
3164	mir_tag: Mir.Inst.Tag,
3165	lhs: MCValue,
3166	rhs: MCValue,
3167	lhs_ty: Type,
3168	rhs_ty: Type,
3169	metadata: ?BinOpMetadata,
3170	) !MCValue {
3171	const lhs_is_register = lhs == .register;
3172	const rhs_is_register = rhs == .register;
3173
3174	const lhs_lock: ?RegisterLock = if (lhs_is_register)
3175	self.register_manager.lockReg(lhs.register)
3176	else
3177	null;
3178	defer if (lhs_lock) |reg| self.register_manager.unlockReg(reg);
3179
3180	const rhs_lock: ?RegisterLock = if (rhs_is_register)
3181	self.register_manager.lockReg(rhs.register)
3182	else
3183	null;
3184	defer if (rhs_lock) |reg| self.register_manager.unlockReg(reg);
3185
3186	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
3187
3188	const lhs_reg = if (lhs_is_register) lhs.register else blk: {
3189	const track_inst: ?Air.Inst.Index = if (metadata) |md| inst: {
3190	break :inst md.lhs.toIndex().?;
3191	} else null;
3192
3193	const reg = try self.register_manager.allocReg(track_inst, gp);
3194	if (track_inst) |inst| {
3195	const mcv: MCValue = .{ .register = reg };
3196	log.debug("binOpRegister move lhs %{d} to register: {} -> {}", .{ inst, lhs, mcv });
3197	branch.inst_table.putAssumeCapacity(inst, mcv);
3198
3199	// If we're moving a condition flag MCV to register,
3200	// mark it as free.
3201	if (lhs == .condition_flags) {
3202	assert(self.condition_flags_inst.? == inst);
3203	self.condition_flags_inst = null;
3204	}
3205	}
3206
3207	break :blk reg;
3208	};
3209	const new_lhs_lock = self.register_manager.lockReg(lhs_reg);
3210	defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg);
3211
3212	const rhs_reg = if (rhs_is_register) rhs.register else blk: {
3213	const track_inst: ?Air.Inst.Index = if (metadata) |md| inst: {
3214	break :inst md.rhs.toIndex().?;
3215	} else null;
3216
3217	const reg = try self.register_manager.allocReg(track_inst, gp);
3218	if (track_inst) |inst| {
3219	const mcv: MCValue = .{ .register = reg };
3220	log.debug("binOpRegister move rhs %{d} to register: {} -> {}", .{ inst, rhs, mcv });
3221	branch.inst_table.putAssumeCapacity(inst, mcv);
3222
3223	// If we're moving a condition flag MCV to register,
3224	// mark it as free.
3225	if (rhs == .condition_flags) {
3226	assert(self.condition_flags_inst.? == inst);
3227	self.condition_flags_inst = null;
3228	}
3229	}
3230
3231	break :blk reg;
3232	};
3233	const new_rhs_lock = self.register_manager.lockReg(rhs_reg);
3234	defer if (new_rhs_lock) |reg| self.register_manager.unlockReg(reg);
3235
3236	const dest_reg = switch (mir_tag) {
3237	.cmp => undefined, // cmp has no destination register
3238	else => if (metadata) |md| blk: {
3239	if (lhs_is_register and self.reuseOperand(md.inst, md.lhs, 0, lhs)) {
3240	break :blk lhs_reg;
3241	} else if (rhs_is_register and self.reuseOperand(md.inst, md.rhs, 1, rhs)) {
3242	break :blk rhs_reg;
3243	} else {
3244	break :blk try self.register_manager.allocReg(md.inst, gp);
3245	}
3246	} else blk: {
3247	break :blk try self.register_manager.allocReg(null, gp);
3248	},
3249	};
3250
3251	if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs);
3252	if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs);
3253
3254	const mir_data: Mir.Inst.Data = switch (mir_tag) {
3255	.add,
3256	.addcc,
3257	.@"and",
3258	.@"or",
3259	.xor,
3260	.xnor,
3261	.mulx,
3262	.sdivx,
3263	.udivx,
3264	.sub,
3265	.subcc,
3266	=> .{
3267	.arithmetic_3op = .{
3268	.is_imm = false,
3269	.rd = dest_reg,
3270	.rs1 = lhs_reg,
3271	.rs2_or_imm = .{ .rs2 = rhs_reg },
3272	},
3273	},
3274	.sll,
3275	.srl,
3276	.sra,
3277	.sllx,
3278	.srlx,
3279	.srax,
3280	=> .{
3281	.shift = .{
3282	.is_imm = false,
3283	.rd = dest_reg,
3284	.rs1 = lhs_reg,
3285	.rs2_or_imm = .{ .rs2 = rhs_reg },
3286	},
3287	},
3288	.cmp => .{
3289	.arithmetic_2op = .{
3290	.is_imm = false,
3291	.rs1 = lhs_reg,
3292	.rs2_or_imm = .{ .rs2 = rhs_reg },
3293	},
3294	},
3295	else => unreachable,
3296	};
3297
3298	_ = try self.addInst(.{
3299	.tag = mir_tag,
3300	.data = mir_data,
3301	});
3302
3303	return MCValue{ .register = dest_reg };
3304	}
3305
3306	fn br(self: *Self, block: Air.Inst.Index, operand: Air.Inst.Ref) !void {
3307	const block_data = self.blocks.getPtr(block).?;
3308
3309	const zcu = self.pt.zcu;
3310	if (self.typeOf(operand).hasRuntimeBits(zcu)) {
3311	const operand_mcv = try self.resolveInst(operand);
3312	const block_mcv = block_data.mcv;
3313	if (block_mcv == .none) {
3314	block_data.mcv = switch (operand_mcv) {
3315	.none, .dead, .unreach => unreachable,
3316	.register, .stack_offset, .memory => operand_mcv,
3317	.immediate => blk: {
3318	const new_mcv = try self.allocRegOrMem(block, true);
3319	try self.setRegOrMem(self.typeOfIndex(block), new_mcv, operand_mcv);
3320	break :blk new_mcv;
3321	},
3322	else => return self.fail("TODO implement block_data.mcv = operand_mcv for {}", .{operand_mcv}),
3323	};
3324	} else {
3325	try self.setRegOrMem(self.typeOfIndex(block), block_mcv, operand_mcv);
3326	}
3327	}
3328	return self.brVoid(block);
3329	}
3330
3331	fn brVoid(self: *Self, block: Air.Inst.Index) !void {
3332	const block_data = self.blocks.getPtr(block).?;
3333
3334	// Emit a jump with a relocation. It will be patched up after the block ends.
3335	try block_data.relocs.ensureUnusedCapacity(self.gpa, 1);
3336
3337	const br_index = try self.addInst(.{
3338	.tag = .bpcc,
3339	.data = .{
3340	.branch_predict_int = .{
3341	.ccr = .xcc,
3342	.cond = .al,
3343	.inst = undefined, // Will be filled by performReloc
3344	},
3345	},
3346	});
3347
3348	// TODO Find a way to fill this delay slot
3349	_ = try self.addInst(.{
3350	.tag = .nop,
3351	.data = .{ .nop = {} },
3352	});
3353
3354	block_data.relocs.appendAssumeCapacity(br_index);
3355	}
3356
3357	fn condBr(self: *Self, condition: MCValue) !Mir.Inst.Index {
3358	// Here we either emit a BPcc for branching on CCR content,
3359	// or emit a BPr to branch on register content.
3360	const reloc: Mir.Inst.Index = switch (condition) {
3361	.condition_flags => |flags| try self.addInst(.{
3362	.tag = .bpcc,
3363	.data = .{
3364	.branch_predict_int = .{
3365	.ccr = flags.ccr,
3366	// Here we map to the opposite condition because the jump is to the false branch.
3367	.cond = flags.cond.icond.negate(),
3368	.inst = undefined, // Will be filled by performReloc
3369	},
3370	},
3371	}),
3372	.condition_register => |reg| try self.addInst(.{
3373	.tag = .bpr,
3374	.data = .{
3375	.branch_predict_reg = .{
3376	.rs1 = reg.reg,
3377	// Here we map to the opposite condition because the jump is to the false branch.
3378	.cond = reg.cond.negate(),
3379	.inst = undefined, // Will be filled by performReloc
3380	},
3381	},
3382	}),
3383	else => blk: {
3384	const reg = switch (condition) {
3385	.register => |r| r,
3386	else => try self.copyToTmpRegister(Type.bool, condition),
3387	};
3388
3389	break :blk try self.addInst(.{
3390	.tag = .bpr,
3391	.data = .{
3392	.branch_predict_reg = .{
3393	.cond = .eq_zero,
3394	.rs1 = reg,
3395	.inst = undefined, // populated later through performReloc
3396	},
3397	},
3398	});
3399	},
3400	};
3401
3402	// Regardless of the branch type that's emitted, we need to reserve
3403	// a space for the delay slot.
3404	// TODO Find a way to fill this delay slot
3405	_ = try self.addInst(.{
3406	.tag = .nop,
3407	.data = .{ .nop = {} },
3408	});
3409
3410	return reloc;
3411	}
3412
3413	/// Copies a value to a register without tracking the register. The register is not considered
3414	/// allocated. A second call to `copyToTmpRegister` may return the same register.
3415	/// This can have a side effect of spilling instructions to the stack to free up a register.
3416	fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
3417	const reg = try self.register_manager.allocReg(null, gp);
3418	try self.genSetReg(ty, reg, mcv);
3419	return reg;
3420	}
3421
3422	fn ensureProcessDeathCapacity(self: *Self, additional_count: usize) !void {
3423	const table = &self.branch_stack.items[self.branch_stack.items.len - 1].inst_table;
3424	try table.ensureUnusedCapacity(self.gpa, additional_count);
3425	}
3426
3427	/// Given an error union, returns the payload
3428	fn errUnionPayload(self: *Self, error_union_mcv: MCValue, error_union_ty: Type) !MCValue {
3429	const pt = self.pt;
3430	const zcu = pt.zcu;
3431	const err_ty = error_union_ty.errorUnionSet(zcu);
3432	const payload_ty = error_union_ty.errorUnionPayload(zcu);
3433	if (err_ty.errorSetIsEmpty(zcu)) {
3434	return error_union_mcv;
3435	}
3436	if (!payload_ty.hasRuntimeBits(zcu)) {
3437	return .none;
3438	}
3439
3440	const payload_offset: u32 = @intCast(errUnionPayloadOffset(payload_ty, zcu));
3441	switch (error_union_mcv) {
3442	.register => return self.fail("TODO errUnionPayload for registers", .{}),
3443	.stack_offset => |off| {
3444	return MCValue{ .stack_offset = off - payload_offset };
3445	},
3446	.memory => |addr| {
3447	return MCValue{ .memory = addr + payload_offset };
3448	},
3449	else => unreachable, // invalid MCValue for an error union
3450	}
3451	}
3452
3453	fn fail(self: *Self, comptime format: []const u8, args: anytype) error{ OutOfMemory, AlreadyReported } {
3454	@branchHint(.cold);
3455	const zcu = self.pt.zcu;
3456	const func = zcu.funcInfo(self.func_index);
3457	const msg = try ErrorMsg.create(zcu.gpa, zcu.navSrcLoc(func.owner_nav), format, args);
3458	return zcu.codegenFailMsg(func.owner_nav, msg);
3459	}
3460
3461	fn failMsg(self: *Self, msg: *ErrorMsg) error{ OutOfMemory, AlreadyReported } {
3462	@branchHint(.cold);
3463	const zcu = self.pt.zcu;
3464	const func = zcu.funcInfo(self.func_index);
3465	return zcu.codegenFailMsg(func.owner_nav, msg);
3466	}
3467
3468	/// Called when there are no operands, and the instruction is always unreferenced.
3469	fn finishAirBookkeeping(self: *Self) void {
3470	if (std.debug.runtime_safety) {
3471	self.air_bookkeeping += 1;
3472	}
3473	}
3474
3475	fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Air.Liveness.bpi - 1]Air.Inst.Ref) void {
3476	const tomb_bits = self.liveness.getTombBits(inst);
3477	for (0.., operands) |op_index, op| {
3478	if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
3479	if (self.reused_operands.isSet(op_index)) continue;
3480	self.processDeath(op.toIndexAllowNone() orelse continue);
3481	}
3482	if (tomb_bits & 1 << (Air.Liveness.bpi - 1) == 0) {
3483	log.debug("%{d} => {}", .{ inst, result });
3484	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
3485	branch.inst_table.putAssumeCapacityNoClobber(inst, result);
3486
3487	switch (result) {
3488	.register => |reg| {
3489	// In some cases (such as bitcast), an operand
3490	// may be the same MCValue as the result. If
3491	// that operand died and was a register, it
3492	// was freed by processDeath. We have to
3493	// "re-allocate" the register.
3494	if (self.register_manager.isRegFree(reg)) {
3495	self.register_manager.getRegAssumeFree(reg, inst);
3496	}
3497	},
3498	else => {},
3499	}
3500	}
3501	self.finishAirBookkeeping();
3502	}
3503
3504	fn genArgDbgInfo(self: Self, name: []const u8, ty: Type, mcv: MCValue) !void {
3505	// TODO: Add a pseudo-instruction or something to defer this work until Emit.
3506	// We aren't allowed to interact with linker state here.
3507	if (true) return;
3508	switch (self.debug_output) {
3509	.dwarf => |dw| switch (mcv) {
3510	.register => |reg| try dw.genLocalDebugInfo(
3511	.local_arg,
3512	name,
3513	ty,
3514	.{ .reg = reg.dwarfNum() },
3515	),
3516	else => {},
3517	},
3518	else => {},
3519	}
3520	}
3521
3522	// TODO replace this to call to extern memcpy
3523	fn genInlineMemcpy(
3524	self: *Self,
3525	src: Register,
3526	dst: Register,
3527	len: Register,
3528	tmp: Register,
3529	) !void {
3530	// Here we assume that len > 0.
3531	// Also we do the copy from end -> start address to save a register.
3532
3533	// sub len, 1, len
3534	_ = try self.addInst(.{
3535	.tag = .sub,
3536	.data = .{ .arithmetic_3op = .{
3537	.is_imm = true,
3538	.rs1 = len,
3539	.rs2_or_imm = .{ .imm = 1 },
3540	.rd = len,
3541	} },
3542	});
3543
3544	// loop:
3545	// ldub [src + len], tmp
3546	_ = try self.addInst(.{
3547	.tag = .ldub,
3548	.data = .{ .arithmetic_3op = .{
3549	.is_imm = false,
3550	.rs1 = src,
3551	.rs2_or_imm = .{ .rs2 = len },
3552	.rd = tmp,
3553	} },
3554	});
3555
3556	// stb tmp, [dst + len]
3557	_ = try self.addInst(.{
3558	.tag = .stb,
3559	.data = .{ .arithmetic_3op = .{
3560	.is_imm = false,
3561	.rs1 = dst,
3562	.rs2_or_imm = .{ .rs2 = len },
3563	.rd = tmp,
3564	} },
3565	});
3566
3567	// brnz len, loop
3568	_ = try self.addInst(.{
3569	.tag = .bpr,
3570	.data = .{ .branch_predict_reg = .{
3571	.cond = .ne_zero,
3572	.rs1 = len,
3573	.inst = @as(u32, @intCast(self.mir_instructions.len - 2)),
3574	} },
3575	});
3576
3577	// Delay slot:
3578	// sub len, 1, len
3579	_ = try self.addInst(.{
3580	.tag = .sub,
3581	.data = .{ .arithmetic_3op = .{
3582	.is_imm = true,
3583	.rs1 = len,
3584	.rs2_or_imm = .{ .imm = 1 },
3585	.rd = len,
3586	} },
3587	});
3588
3589	// end:
3590	}
3591
3592	fn genLoad(self: *Self, value_reg: Register, addr_reg: Register, comptime off_type: type, off: off_type, abi_size: u64) !void {
3593	assert(off_type == Register or off_type == i13);
3594
3595	const is_imm = (off_type == i13);
3596
3597	switch (abi_size) {
3598	1, 2, 4, 8 => {
3599	const tag: Mir.Inst.Tag = switch (abi_size) {
3600	1 => .ldub,
3601	2 => .lduh,
3602	4 => .lduw,
3603	8 => .ldx,
3604	else => unreachable, // unexpected abi size
3605	};
3606
3607	_ = try self.addInst(.{
3608	.tag = tag,
3609	.data = .{
3610	.arithmetic_3op = .{
3611	.is_imm = is_imm,
3612	.rd = value_reg,
3613	.rs1 = addr_reg,
3614	.rs2_or_imm = if (is_imm) .{ .imm = off } else .{ .rs2 = off },
3615	},
3616	},
3617	});
3618	},
3619	3, 5, 6, 7 => return self.fail("TODO: genLoad for more abi_sizes", .{}),
3620	else => unreachable,
3621	}
3622	}
3623
3624	fn genLoadASI(self: *Self, value_reg: Register, addr_reg: Register, off_reg: Register, abi_size: u64, asi: ASI) !void {
3625	switch (abi_size) {
3626	1, 2, 4, 8 => {
3627	const tag: Mir.Inst.Tag = switch (abi_size) {
3628	1 => .lduba,
3629	2 => .lduha,
3630	4 => .lduwa,
3631	8 => .ldxa,
3632	else => unreachable, // unexpected abi size
3633	};
3634
3635	_ = try self.addInst(.{
3636	.tag = tag,
3637	.data = .{
3638	.mem_asi = .{
3639	.rd = value_reg,
3640	.rs1 = addr_reg,
3641	.rs2 = off_reg,
3642	.asi = asi,
3643	},
3644	},
3645	});
3646	},
3647	3, 5, 6, 7 => return self.fail("TODO: genLoad for more abi_sizes", .{}),
3648	else => unreachable,
3649	}
3650	}
3651
3652	fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void {
3653	const pt = self.pt;
3654	const zcu = pt.zcu;
3655	switch (mcv) {
3656	.dead => unreachable,
3657	.unreach, .none => return, // Nothing to do.
3658	.condition_flags => |op| {
3659	const condition = op.cond;
3660	const ccr = op.ccr;
3661
3662	// TODO handle floating point CCRs
3663	assert(ccr == .xcc or ccr == .icc);
3664
3665	_ = try self.addInst(.{
3666	.tag = .mov,
3667	.data = .{
3668	.arithmetic_2op = .{
3669	.is_imm = false,
3670	.rs1 = reg,
3671	.rs2_or_imm = .{ .rs2 = .g0 },
3672	},
3673	},
3674	});
3675
3676	_ = try self.addInst(.{
3677	.tag = .movcc,
3678	.data = .{
3679	.conditional_move_int = .{
3680	.ccr = ccr,
3681	.cond = condition,
3682	.is_imm = true,
3683	.rd = reg,
3684	.rs2_or_imm = .{ .imm = 1 },
3685	},
3686	},
3687	});
3688	},
3689	.condition_register => |op| {
3690	const condition = op.cond;
3691	const register = op.reg;
3692
3693	_ = try self.addInst(.{
3694	.tag = .mov,
3695	.data = .{
3696	.arithmetic_2op = .{
3697	.is_imm = false,
3698	.rs1 = reg,
3699	.rs2_or_imm = .{ .rs2 = .g0 },
3700	},
3701	},
3702	});
3703
3704	_ = try self.addInst(.{
3705	.tag = .movr,
3706	.data = .{
3707	.conditional_move_reg = .{
3708	.cond = condition,
3709	.is_imm = true,
3710	.rd = reg,
3711	.rs1 = register,
3712	.rs2_or_imm = .{ .imm = 1 },
3713	},
3714	},
3715	});
3716	},
3717	.undef => {
3718	if (!self.wantSafety())
3719	return; // The already existing value will do just fine.
3720	// Write the debug undefined value.
3721	return self.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaaaaaaaaaa });
3722	},
3723	.ptr_stack_offset => |off| {
3724	const real_offset = realStackOffset(off);
3725	const simm13 = math.cast(i13, real_offset) orelse
3726	return self.fail("TODO larger stack offsets: {}", .{real_offset});
3727
3728	_ = try self.addInst(.{
3729	.tag = .add,
3730	.data = .{
3731	.arithmetic_3op = .{
3732	.is_imm = true,
3733	.rd = reg,
3734	.rs1 = .sp,
3735	.rs2_or_imm = .{ .imm = simm13 },
3736	},
3737	},
3738	});
3739	},
3740	.immediate => |x| {
3741	if (x <= math.maxInt(u12)) {
3742	_ = try self.addInst(.{
3743	.tag = .mov,
3744	.data = .{
3745	.arithmetic_2op = .{
3746	.is_imm = true,
3747	.rs1 = reg,
3748	.rs2_or_imm = .{ .imm = @as(u12, @truncate(x)) },
3749	},
3750	},
3751	});
3752	} else if (x <= math.maxInt(u32)) {
3753	_ = try self.addInst(.{
3754	.tag = .sethi,
3755	.data = .{
3756	.sethi = .{
3757	.rd = reg,
3758	.imm = @as(u22, @truncate(x >> 10)),
3759	},
3760	},
3761	});
3762
3763	_ = try self.addInst(.{
3764	.tag = .@"or",
3765	.data = .{
3766	.arithmetic_3op = .{
3767	.is_imm = true,
3768	.rd = reg,
3769	.rs1 = reg,
3770	.rs2_or_imm = .{ .imm = @as(u10, @truncate(x)) },
3771	},
3772	},
3773	});
3774	} else if (x <= math.maxInt(u44)) {
3775	try self.genSetReg(ty, reg, .{ .immediate = @as(u32, @truncate(x >> 12)) });
3776
3777	_ = try self.addInst(.{
3778	.tag = .sllx,
3779	.data = .{
3780	.shift = .{
3781	.is_imm = true,
3782	.rd = reg,
3783	.rs1 = reg,
3784	.rs2_or_imm = .{ .imm = 12 },
3785	},
3786	},
3787	});
3788
3789	_ = try self.addInst(.{
3790	.tag = .@"or",
3791	.data = .{
3792	.arithmetic_3op = .{
3793	.is_imm = true,
3794	.rd = reg,
3795	.rs1 = reg,
3796	.rs2_or_imm = .{ .imm = @as(u12, @truncate(x)) },
3797	},
3798	},
3799	});
3800	} else {
3801	// Need to allocate a temporary register to load 64-bit immediates.
3802	const tmp_reg = try self.register_manager.allocReg(null, gp);
3803
3804	try self.genSetReg(ty, tmp_reg, .{ .immediate = @as(u32, @truncate(x)) });
3805	try self.genSetReg(ty, reg, .{ .immediate = @as(u32, @truncate(x >> 32)) });
3806
3807	_ = try self.addInst(.{
3808	.tag = .sllx,
3809	.data = .{
3810	.shift = .{
3811	.is_imm = true,
3812	.rd = reg,
3813	.rs1 = reg,
3814	.rs2_or_imm = .{ .imm = 32 },
3815	},
3816	},
3817	});
3818
3819	_ = try self.addInst(.{
3820	.tag = .@"or",
3821	.data = .{
3822	.arithmetic_3op = .{
3823	.is_imm = false,
3824	.rd = reg,
3825	.rs1 = reg,
3826	.rs2_or_imm = .{ .rs2 = tmp_reg },
3827	},
3828	},
3829	});
3830	}
3831	},
3832	.register => |src_reg| {
3833	// If the registers are the same, nothing to do.
3834	if (src_reg.id() == reg.id())
3835	return;
3836
3837	_ = try self.addInst(.{
3838	.tag = .mov,
3839	.data = .{
3840	.arithmetic_2op = .{
3841	.is_imm = false,
3842	.rs1 = reg,
3843	.rs2_or_imm = .{ .rs2 = src_reg },
3844	},
3845	},
3846	});
3847	},
3848	.register_with_overflow => unreachable,
3849	.memory => |addr| {
3850	// The value is in memory at a hard-coded address.
3851	// If the type is a pointer, it means the pointer address is at this memory location.
3852	try self.genSetReg(ty, reg, .{ .immediate = addr });
3853	try self.genLoad(reg, reg, i13, 0, ty.abiSize(zcu));
3854	},
3855	.stack_offset => |off| {
3856	const real_offset = realStackOffset(off);
3857	const simm13 = math.cast(i13, real_offset) orelse
3858	return self.fail("TODO larger stack offsets: {}", .{real_offset});
3859	try self.genLoad(reg, .sp, i13, simm13, ty.abiSize(zcu));
3860	},
3861	}
3862	}
3863
3864	fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerError!void {
3865	const pt = self.pt;
3866	const zcu = pt.zcu;
3867	const abi_size = ty.abiSize(zcu);
3868	switch (mcv) {
3869	.dead => unreachable,
3870	.unreach, .none => return, // Nothing to do.
3871	.undef => {
3872	if (!self.wantSafety())
3873	return; // The already existing value will do just fine.
3874	// TODO Upgrade this to a memset call when we have that available.
3875	switch (ty.abiSize(zcu)) {
3876	1 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaa }),
3877	2 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaa }),
3878	4 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaa }),
3879	8 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaaaaaaaaaa }),
3880	else => return self.fail("TODO implement memset", .{}),
3881	}
3882	},
3883	.condition_flags,
3884	.condition_register,
3885	.immediate,
3886	.ptr_stack_offset,
3887	=> {
3888	const reg = try self.copyToTmpRegister(ty, mcv);
3889	return self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
3890	},
3891	.register => |reg| {
3892	const real_offset = realStackOffset(stack_offset);
3893	const simm13 = math.cast(i13, real_offset) orelse
3894	return self.fail("TODO larger stack offsets: {}", .{real_offset});
3895	return self.genStore(reg, .sp, i13, simm13, abi_size);
3896	},
3897	.register_with_overflow => |rwo| {
3898	const reg_lock = self.register_manager.lockReg(rwo.reg);
3899	defer if (reg_lock) |locked_reg| self.register_manager.unlockReg(locked_reg);
3900
3901	const wrapped_ty = ty.fieldType(0, zcu);
3902	try self.genSetStack(wrapped_ty, stack_offset, .{ .register = rwo.reg });
3903
3904	const overflow_bit_ty = ty.fieldType(1, zcu);
3905	const overflow_bit_offset: u32 = @intCast(ty.structFieldOffset(1, zcu));
3906	const cond_reg = try self.register_manager.allocReg(null, gp);
3907
3908	// TODO handle floating point CCRs
3909	assert(rwo.flag.ccr == .xcc or rwo.flag.ccr == .icc);
3910
3911	_ = try self.addInst(.{
3912	.tag = .mov,
3913	.data = .{
3914	.arithmetic_2op = .{
3915	.is_imm = false,
3916	.rs1 = cond_reg,
3917	.rs2_or_imm = .{ .rs2 = .g0 },
3918	},
3919	},
3920	});
3921
3922	_ = try self.addInst(.{
3923	.tag = .movcc,
3924	.data = .{
3925	.conditional_move_int = .{
3926	.ccr = rwo.flag.ccr,
3927	.cond = .{ .icond = rwo.flag.cond },
3928	.is_imm = true,
3929	.rd = cond_reg,
3930	.rs2_or_imm = .{ .imm = 1 },
3931	},
3932	},
3933	});
3934	try self.genSetStack(overflow_bit_ty, stack_offset - overflow_bit_offset, .{
3935	.register = cond_reg,
3936	});
3937	},
3938	.memory, .stack_offset => {
3939	switch (mcv) {
3940	.stack_offset => |off| {
3941	if (stack_offset == off)
3942	return; // Copy stack variable to itself; nothing to do.
3943	},
3944	else => {},
3945	}
3946
3947	if (abi_size <= 8) {
3948	const reg = try self.copyToTmpRegister(ty, mcv);
3949	return self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
3950	} else {
3951	const ptr_ty = try pt.singleMutPtrType(ty);
3952
3953	const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null }, gp);
3954	const regs_locks = self.register_manager.lockRegsAssumeUnused(4, regs);
3955	defer for (regs_locks) |reg| {
3956	self.register_manager.unlockReg(reg);
3957	};
3958
3959	const src_reg = regs[0];
3960	const dst_reg = regs[1];
3961	const len_reg = regs[2];
3962	const tmp_reg = regs[3];
3963
3964	switch (mcv) {
3965	.stack_offset => |off| try self.genSetReg(ptr_ty, src_reg, .{ .ptr_stack_offset = off }),
3966	.memory => |addr| try self.genSetReg(Type.usize, src_reg, .{ .immediate = addr }),
3967	else => unreachable,
3968	}
3969
3970	try self.genSetReg(ptr_ty, dst_reg, .{ .ptr_stack_offset = stack_offset });
3971	try self.genSetReg(Type.usize, len_reg, .{ .immediate = abi_size });
3972	try self.genInlineMemcpy(src_reg, dst_reg, len_reg, tmp_reg);
3973	}
3974	},
3975	}
3976	}
3977
3978	fn genStore(self: *Self, value_reg: Register, addr_reg: Register, comptime off_type: type, off: off_type, abi_size: u64) !void {
3979	assert(off_type == Register or off_type == i13);
3980
3981	const is_imm = (off_type == i13);
3982
3983	switch (abi_size) {
3984	1, 2, 4, 8 => {
3985	const tag: Mir.Inst.Tag = switch (abi_size) {
3986	1 => .stb,
3987	2 => .sth,
3988	4 => .stw,
3989	8 => .stx,
3990	else => unreachable, // unexpected abi size
3991	};
3992
3993	_ = try self.addInst(.{
3994	.tag = tag,
3995	.data = .{
3996	.arithmetic_3op = .{
3997	.is_imm = is_imm,
3998	.rd = value_reg,
3999	.rs1 = addr_reg,
4000	.rs2_or_imm = if (is_imm) .{ .imm = off } else .{ .rs2 = off },
4001	},
4002	},
4003	});
4004	},
4005	3, 5, 6, 7 => return self.fail("TODO: genLoad for more abi_sizes", .{}),
4006	else => unreachable,
4007	}
4008	}
4009
4010	fn genStoreASI(self: *Self, value_reg: Register, addr_reg: Register, off_reg: Register, abi_size: u64, asi: ASI) !void {
4011	switch (abi_size) {
4012	1, 2, 4, 8 => {
4013	const tag: Mir.Inst.Tag = switch (abi_size) {
4014	1 => .stba,
4015	2 => .stha,
4016	4 => .stwa,
4017	8 => .stxa,
4018	else => unreachable, // unexpected abi size
4019	};
4020
4021	_ = try self.addInst(.{
4022	.tag = tag,
4023	.data = .{
4024	.mem_asi = .{
4025	.rd = value_reg,
4026	.rs1 = addr_reg,
4027	.rs2 = off_reg,
4028	.asi = asi,
4029	},
4030	},
4031	});
4032	},
4033	3, 5, 6, 7 => return self.fail("TODO: genLoad for more abi_sizes", .{}),
4034	else => unreachable,
4035	}
4036	}
4037
4038	fn genTypedValue(self: *Self, val: Value) InnerError!MCValue {
4039	const pt = self.pt;
4040	const mcv: MCValue = switch (try codegen.genTypedValue(
4041	self.bin_file,
4042	pt,
4043	val,
4044	self.target,
4045	)) {
4046	.none => .none,
4047	.undef => .undef,
4048	.load_got, .load_symbol, .load_direct, .lea_symbol, .lea_direct => unreachable, // TODO
4049	.immediate => |imm| .{ .immediate = imm },
4050	.memory => |addr| .{ .memory = addr },
4051	};
4052	return mcv;
4053	}
4054
4055	fn getResolvedInstValue(self: *Self, inst: Air.Inst.Index) MCValue {
4056	// Treat each stack item as a "layer" on top of the previous one.
4057	var i: usize = self.branch_stack.items.len;
4058	while (true) {
4059	i -= 1;
4060	if (self.branch_stack.items[i].inst_table.get(inst)) |mcv| {
4061	log.debug("getResolvedInstValue %{f} => {}", .{ inst, mcv });
4062	assert(mcv != .dead);
4063	return mcv;
4064	}
4065	}
4066	}
4067
4068	fn isErr(self: *Self, ty: Type, operand: MCValue) !MCValue {
4069	const pt = self.pt;
4070	const zcu = pt.zcu;
4071	const error_type = ty.errorUnionSet(zcu);
4072	const payload_type = ty.errorUnionPayload(zcu);
4073
4074	if (!error_type.hasRuntimeBits(zcu)) {
4075	return MCValue{ .immediate = 0 }; // always false
4076	} else if (!payload_type.hasRuntimeBits(zcu)) {
4077	if (error_type.abiSize(zcu) <= 8) {
4078	const reg_mcv: MCValue = switch (operand) {
4079	.register => operand,
4080	else => .{ .register = try self.copyToTmpRegister(error_type, operand) },
4081	};
4082
4083	_ = try self.addInst(.{
4084	.tag = .cmp,
4085	.data = .{ .arithmetic_2op = .{
4086	.is_imm = true,
4087	.rs1 = reg_mcv.register,
4088	.rs2_or_imm = .{ .imm = 0 },
4089	} },
4090	});
4091
4092	return MCValue{ .condition_flags = .{ .cond = .{ .icond = .gu }, .ccr = .xcc } };
4093	} else {
4094	return self.fail("TODO isErr for errors with size > 8", .{});
4095	}
4096	} else {
4097	return self.fail("TODO isErr for non-empty payloads", .{});
4098	}
4099	}
4100
4101	fn isNonErr(self: *Self, ty: Type, operand: MCValue) !MCValue {
4102	// Call isErr, then negate the result.
4103	const is_err_result = try self.isErr(ty, operand);
4104	switch (is_err_result) {
4105	.condition_flags => |op| {
4106	return MCValue{ .condition_flags = .{ .cond = op.cond.negate(), .ccr = op.ccr } };
4107	},
4108	.immediate => |imm| {
4109	assert(imm == 0);
4110	return MCValue{ .immediate = 1 };
4111	},
4112	else => unreachable,
4113	}
4114	}
4115
4116	fn isNull(self: *Self, operand: MCValue) !MCValue {
4117	_ = operand;
4118	// Here you can specialize this instruction if it makes sense to, otherwise the default
4119	// will call isNonNull and invert the result.
4120	return self.fail("TODO call isNonNull and invert the result", .{});
4121	}
4122
4123	fn isNonNull(self: *Self, operand: MCValue) !MCValue {
4124	// Call isNull, then negate the result.
4125	const is_null_result = try self.isNull(operand);
4126	switch (is_null_result) {
4127	.condition_flags => |op| {
4128	return MCValue{ .condition_flags = .{ .cond = op.cond.negate(), .ccr = op.ccr } };
4129	},
4130	.immediate => |imm| {
4131	assert(imm == 0);
4132	return MCValue{ .immediate = 1 };
4133	},
4134	else => unreachable,
4135	}
4136	}
4137
4138	fn iterateBigTomb(self: *Self, inst: Air.Inst.Index, operand_count: usize) !BigTomb {
4139	try self.ensureProcessDeathCapacity(operand_count + 1);
4140	return BigTomb{
4141	.function = self,
4142	.inst = inst,
4143	.lbt = self.liveness.iterateBigTomb(inst),
4144	};
4145	}
4146
4147	/// Send control flow to `inst`.
4148	fn jump(self: *Self, inst: Mir.Inst.Index) !void {
4149	_ = try self.addInst(.{
4150	.tag = .bpcc,
4151	.data = .{
4152	.branch_predict_int = .{
4153	.cond = .al,
4154	.ccr = .xcc,
4155	.inst = inst,
4156	},
4157	},
4158	});
4159
4160	// TODO find out a way to fill this delay slot
4161	_ = try self.addInst(.{
4162	.tag = .nop,
4163	.data = .{ .nop = {} },
4164	});
4165	}
4166
4167	fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!void {
4168	const pt = self.pt;
4169	const zcu = pt.zcu;
4170	const elem_ty = ptr_ty.childType(zcu);
4171	const elem_size = elem_ty.abiSize(zcu);
4172
4173	switch (ptr) {
4174	.none => unreachable,
4175	.undef => unreachable,
4176	.unreach => unreachable,
4177	.dead => unreachable,
4178	.condition_flags,
4179	.condition_register,
4180	.register_with_overflow,
4181	=> unreachable, // cannot hold an address
4182	.immediate => |imm| try self.setRegOrMem(elem_ty, dst_mcv, .{ .memory = imm }),
4183	.ptr_stack_offset => |off| try self.setRegOrMem(elem_ty, dst_mcv, .{ .stack_offset = off }),
4184	.register => |addr_reg| {
4185	const addr_reg_lock = self.register_manager.lockReg(addr_reg);
4186	defer if (addr_reg_lock) |reg| self.register_manager.unlockReg(reg);
4187
4188	switch (dst_mcv) {
4189	.dead => unreachable,
4190	.undef => unreachable,
4191	.condition_flags => unreachable,
4192	.register => |dst_reg| {
4193	try self.genLoad(dst_reg, addr_reg, i13, 0, elem_size);
4194	},
4195	.stack_offset => |off| {
4196	if (elem_size <= 8) {
4197	const tmp_reg = try self.register_manager.allocReg(null, gp);
4198	const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
4199	defer self.register_manager.unlockReg(tmp_reg_lock);
4200
4201	try self.load(.{ .register = tmp_reg }, ptr, ptr_ty);
4202	try self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg });
4203	} else {
4204	const regs = try self.register_manager.allocRegs(3, .{ null, null, null }, gp);
4205	const regs_locks = self.register_manager.lockRegsAssumeUnused(3, regs);
4206	defer for (regs_locks) |reg| {
4207	self.register_manager.unlockReg(reg);
4208	};
4209
4210	const src_reg = addr_reg;
4211	const dst_reg = regs[0];
4212	const len_reg = regs[1];
4213	const tmp_reg = regs[2];
4214
4215	try self.genSetReg(ptr_ty, dst_reg, .{ .ptr_stack_offset = off });
4216	try self.genSetReg(Type.usize, len_reg, .{ .immediate = elem_size });
4217	try self.genInlineMemcpy(src_reg, dst_reg, len_reg, tmp_reg);
4218	}
4219	},
4220	else => return self.fail("TODO load from register into {}", .{dst_mcv}),
4221	}
4222	},
4223	.memory,
4224	.stack_offset,
4225	=> {
4226	const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr);
4227	try self.load(dst_mcv, .{ .register = addr_reg }, ptr_ty);
4228	},
4229	}
4230	}
4231
4232	fn minMax(
4233	self: *Self,
4234	tag: Air.Inst.Tag,
4235	lhs: MCValue,
4236	rhs: MCValue,
4237	lhs_ty: Type,
4238	rhs_ty: Type,
4239	) InnerError!MCValue {
4240	const pt = self.pt;
4241	const zcu = pt.zcu;
4242	assert(lhs_ty.eql(rhs_ty));
4243	switch (lhs_ty.zigTypeTag(zcu)) {
4244	.float => return self.fail("TODO min/max on floats", .{}),
4245	.vector => return self.fail("TODO min/max on vectors", .{}),
4246	.int => {
4247	const int_info = lhs_ty.intInfo(zcu);
4248	if (int_info.bits <= 64) {
4249	// TODO skip register setting when one of the operands
4250	// is a small (fits in i13) immediate.
4251	const rhs_is_register = rhs == .register;
4252	const rhs_reg = if (rhs_is_register)
4253	rhs.register
4254	else
4255	try self.register_manager.allocReg(null, gp);
4256	const rhs_lock = self.register_manager.lockReg(rhs_reg);
4257	defer if (rhs_lock) |reg| self.register_manager.unlockReg(reg);
4258	if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs);
4259
4260	const result_reg = try self.register_manager.allocReg(null, gp);
4261	const result_lock = self.register_manager.lockReg(result_reg);
4262	defer if (result_lock) |reg| self.register_manager.unlockReg(reg);
4263	try self.genSetReg(lhs_ty, result_reg, lhs);
4264
4265	const cond_choose_rhs: Instruction.ICondition = switch (tag) {
4266	.max => switch (int_info.signedness) {
4267	.signed => Instruction.ICondition.gt,
4268	.unsigned => Instruction.ICondition.gu,
4269	},
4270	.min => switch (int_info.signedness) {
4271	.signed => Instruction.ICondition.lt,
4272	.unsigned => Instruction.ICondition.cs,
4273	},
4274	else => unreachable,
4275	};
4276
4277	_ = try self.addInst(.{
4278	.tag = .cmp,
4279	.data = .{
4280	.arithmetic_2op = .{
4281	.is_imm = false,
4282	.rs1 = result_reg,
4283	.rs2_or_imm = .{ .rs2 = rhs_reg },
4284	},
4285	},
4286	});
4287
4288	_ = try self.addInst(.{
4289	.tag = .movcc,
4290	.data = .{
4291	.conditional_move_int = .{
4292	.is_imm = false,
4293	.ccr = .xcc,
4294	.cond = .{ .icond = cond_choose_rhs },
4295	.rd = result_reg,
4296	.rs2_or_imm = .{ .rs2 = rhs_reg },
4297	},
4298	},
4299	});
4300
4301	return MCValue{ .register = result_reg };
4302	} else {
4303	return self.fail("TODO min/max on integers > u64/i64", .{});
4304	}
4305	},
4306	else => unreachable,
4307	}
4308	}
4309
4310	fn parseRegName(name: []const u8) ?Register {
4311	if (@hasDecl(Register, "parseRegName")) {
4312	return Register.parseRegName(name);
4313	}
4314	return std.meta.stringToEnum(Register, name);
4315	}
4316
4317	fn performReloc(self: *Self, inst: Mir.Inst.Index) !void {
4318	const tag = self.mir_instructions.items(.tag)[inst];
4319	switch (tag) {
4320	.bpcc => self.mir_instructions.items(.data)[inst].branch_predict_int.inst = @intCast(self.mir_instructions.len),
4321	.bpr => self.mir_instructions.items(.data)[inst].branch_predict_reg.inst = @intCast(self.mir_instructions.len),
4322	else => unreachable,
4323	}
4324	}
4325
4326	/// Asserts there is already capacity to insert into top branch inst_table.
4327	fn processDeath(self: *Self, inst: Air.Inst.Index) void {
4328	// When editing this function, note that the logic must synchronize with `reuseOperand`.
4329	const prev_value = self.getResolvedInstValue(inst);
4330	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
4331	branch.inst_table.putAssumeCapacity(inst, .dead);
4332	log.debug("%{f} death: {} -> .dead", .{ inst, prev_value });
4333	switch (prev_value) {
4334	.register => |reg| {
4335	self.register_manager.freeReg(reg);
4336	},
4337	.register_with_overflow => |rwo| {
4338	self.register_manager.freeReg(rwo.reg);
4339	self.condition_flags_inst = null;
4340	},
4341	.condition_flags => {
4342	self.condition_flags_inst = null;
4343	},
4344	else => {}, // TODO process stack allocation death
4345	}
4346	}
4347
4348	/// Turns stack_offset MCV into a real SPARCv9 stack offset usable for asm.
4349	fn realStackOffset(off: u32) u32 {
4350	return off +
4351	// SPARCv9 %sp points away from the stack by some amount.
4352	abi.stack_bias +
4353	// The first couple bytes of each stack frame is reserved
4354	// for ABI and hardware purposes.
4355	abi.stack_reserved_area;
4356	// Only after that we have the usable stack frame portion.
4357	}
4358
4359	/// Caller must call `CallMCValues.deinit`.
4360	fn resolveCallingConventionValues(self: *Self, fn_ty: Type, role: RegisterView) !CallMCValues {
4361	const pt = self.pt;
4362	const zcu = pt.zcu;
4363	const ip = &zcu.intern_pool;
4364	const fn_info = zcu.typeToFunc(fn_ty).?;
4365	const cc = fn_info.cc;
4366	var result: CallMCValues = .{
4367	.args = try self.gpa.alloc(MCValue, fn_info.param_types.len),
4368	// These undefined values must be populated before returning from this function.
4369	.return_value = undefined,
4370	.stack_byte_count = undefined,
4371	.stack_align = undefined,
4372	};
4373	errdefer self.gpa.free(result.args);
4374
4375	const ret_ty = fn_ty.fnReturnType(zcu);
4376
4377	switch (cc) {
4378	.naked => {
4379	assert(result.args.len == 0);
4380	result.return_value = .{ .unreach = {} };
4381	result.stack_byte_count = 0;
4382	result.stack_align = .@"1";
4383	return result;
4384	},
4385	.auto, .sparc64_sysv => {
4386	// SPARC Compliance Definition 2.4.1, Chapter 3
4387	// Low-Level System Information (64-bit psABI) - Function Calling Sequence
4388
4389	var next_register: usize = 0;
4390	var next_stack_offset: u32 = 0;
4391	// TODO: this is never assigned, which is a bug, but I don't know how this code works
4392	// well enough to try and fix it. I *think* `next_register += next_stack_offset` is
4393	// supposed to be `next_stack_offset += param_size` in every case where it appears.
4394	_ = &next_stack_offset;
4395
4396	// The caller puts the argument in %o0-%o5, which becomes %i0-%i5 inside the callee.
4397	const argument_registers = switch (role) {
4398	.caller => abi.c_abi_int_param_regs_caller_view,
4399	.callee => abi.c_abi_int_param_regs_callee_view,
4400	};
4401
4402	for (fn_info.param_types.get(ip), result.args) |ty, *result_arg| {
4403	const param_size: u32 = @intCast(Type.fromInterned(ty).abiSize(zcu));
4404	if (param_size <= 8) {
4405	if (next_register < argument_registers.len) {
4406	result_arg.* = .{ .register = argument_registers[next_register] };
4407	next_register += 1;
4408	} else {
4409	result_arg.* = .{ .stack_offset = next_stack_offset };
4410	next_register += next_stack_offset;
4411	}
4412	} else if (param_size <= 16) {
4413	if (next_register < argument_registers.len - 1) {
4414	return self.fail("TODO MCValues with 2 registers", .{});
4415	} else if (next_register < argument_registers.len) {
4416	return self.fail("TODO MCValues split register + stack", .{});
4417	} else {
4418	result_arg.* = .{ .stack_offset = next_stack_offset };
4419	next_register += next_stack_offset;
4420	}
4421	} else {
4422	result_arg.* = .{ .stack_offset = next_stack_offset };
4423	next_register += next_stack_offset;
4424	}
4425	}
4426
4427	result.stack_byte_count = next_stack_offset;
4428	result.stack_align = .@"16";
4429
4430	if (ret_ty.zigTypeTag(zcu) == .noreturn) {
4431	result.return_value = .{ .unreach = {} };
4432	} else if (!ret_ty.hasRuntimeBits(zcu)) {
4433	result.return_value = .{ .none = {} };
4434	} else {
4435	const ret_ty_size: u32 = @intCast(ret_ty.abiSize(zcu));
4436	// The callee puts the return values in %i0-%i3, which becomes %o0-%o3 inside the caller.
4437	if (ret_ty_size <= 8) {
4438	result.return_value = switch (role) {
4439	.caller => .{ .register = abi.c_abi_int_return_regs_caller_view[0] },
4440	.callee => .{ .register = abi.c_abi_int_return_regs_callee_view[0] },
4441	};
4442	} else {
4443	return self.fail("TODO support more return values for sparc64", .{});
4444	}
4445	}
4446	},
4447	else => return self.fail("TODO implement function parameters for {} on sparc64", .{cc}),
4448	}
4449
4450	return result;
4451	}
4452
4453	fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {
4454	const pt = self.pt;
4455	const ty = self.typeOf(ref);
4456
4457	// If the type has no codegen bits, no need to store it.
4458	if (!ty.hasRuntimeBits(pt.zcu)) return .none;
4459
4460	if (ref.toIndex()) |inst| {
4461	return self.getResolvedInstValue(inst);
4462	}
4463
4464	return self.genTypedValue(.fromInterned(ref.toInterned().?));
4465	}
4466
4467	fn ret(self: *Self, mcv: MCValue) !void {
4468	const pt = self.pt;
4469	const zcu = pt.zcu;
4470	const ret_ty = self.fn_type.fnReturnType(zcu);
4471	try self.setRegOrMem(ret_ty, self.ret_mcv, mcv);
4472
4473	// Just add space for a branch instruction, patch this later
4474	const index = try self.addInst(.{
4475	.tag = .nop,
4476	.data = .{ .nop = {} },
4477	});
4478
4479	// Reserve space for the delay slot too
4480	// TODO find out a way to fill this
4481	_ = try self.addInst(.{
4482	.tag = .nop,
4483	.data = .{ .nop = {} },
4484	});
4485	try self.exitlude_jump_relocs.append(self.gpa, index);
4486	}
4487
4488	fn reuseOperand(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, op_index: Air.Liveness.OperandInt, mcv: MCValue) bool {
4489	if (!self.liveness.operandDies(inst, op_index))
4490	return false;
4491
4492	switch (mcv) {
4493	.register => |reg| {
4494	// If it's in the registers table, need to associate the register with the
4495	// new instruction.
4496	if (RegisterManager.indexOfRegIntoTracked(reg)) |index| {
4497	if (!self.register_manager.isRegFree(reg)) {
4498	self.register_manager.registers[index] = inst;
4499	}
4500	}
4501	log.debug("%{d} => {} (reused)", .{ inst, reg });
4502	},
4503	.stack_offset => |off| {
4504	log.debug("%{d} => stack offset {d} (reused)", .{ inst, off });
4505	},
4506	else => return false,
4507	}
4508
4509	// Prevent the operand deaths processing code from deallocating it.
4510	self.reused_operands.set(op_index);
4511
4512	// That makes us responsible for doing the rest of the stuff that processDeath would have done.
4513	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
4514	branch.inst_table.putAssumeCapacity(operand.toIndex().?, .dead);
4515
4516	return true;
4517	}
4518
4519	/// Sets the value without any modifications to register allocation metadata or stack allocation metadata.
4520	fn setRegOrMem(self: *Self, ty: Type, loc: MCValue, val: MCValue) !void {
4521	switch (loc) {
4522	.none => return,
4523	.register => |reg| return self.genSetReg(ty, reg, val),
4524	.stack_offset => |off| return self.genSetStack(ty, off, val),
4525	.memory => {
4526	return self.fail("TODO implement setRegOrMem for memory", .{});
4527	},
4528	else => unreachable,
4529	}
4530	}
4531
4532	/// Save the current instruction stored in the condition flags if
4533	/// occupied
4534	fn spillConditionFlagsIfOccupied(self: *Self) !void {
4535	if (self.condition_flags_inst) |inst_to_save| {
4536	const mcv = self.getResolvedInstValue(inst_to_save);
4537	const new_mcv = switch (mcv) {
4538	.condition_flags => try self.allocRegOrMem(inst_to_save, true),
4539	.register_with_overflow => try self.allocRegOrMem(inst_to_save, false),
4540	else => unreachable, // mcv doesn't occupy the compare flags
4541	};
4542
4543	try self.setRegOrMem(self.typeOfIndex(inst_to_save), new_mcv, mcv);
4544	log.debug("spilling {d} to mcv {any}", .{ inst_to_save, new_mcv });
4545
4546	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
4547	try branch.inst_table.put(self.gpa, inst_to_save, new_mcv);
4548
4549	self.condition_flags_inst = null;
4550
4551	// TODO consolidate with register manager and spillInstruction
4552	// this call should really belong in the register manager!
4553	switch (mcv) {
4554	.register_with_overflow => |rwo| self.register_manager.freeReg(rwo.reg),
4555	else => {},
4556	}
4557	}
4558	}
4559
4560	pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {
4561	const stack_mcv = try self.allocRegOrMem(inst, false);
4562	log.debug("spilling {d} to stack mcv {any}", .{ inst, stack_mcv });
4563	const reg_mcv = self.getResolvedInstValue(inst);
4564	assert(reg == reg_mcv.register);
4565	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
4566	try branch.inst_table.put(self.gpa, inst, stack_mcv);
4567	try self.genSetStack(self.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv);
4568	}
4569
4570	fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type) InnerError!void {
4571	const pt = self.pt;
4572	const abi_size = value_ty.abiSize(pt.zcu);
4573
4574	switch (ptr) {
4575	.none => unreachable,
4576	.undef => unreachable,
4577	.unreach => unreachable,
4578	.dead => unreachable,
4579	.condition_flags,
4580	.condition_register,
4581	.register_with_overflow,
4582	=> unreachable, // cannot hold an address
4583	.immediate => |imm| {
4584	try self.setRegOrMem(value_ty, .{ .memory = imm }, value);
4585	},
4586	.ptr_stack_offset => |off| {
4587	try self.genSetStack(value_ty, off, value);
4588	},
4589	.register => |addr_reg| {
4590	const addr_reg_lock = self.register_manager.lockReg(addr_reg);
4591	defer if (addr_reg_lock) |reg| self.register_manager.unlockReg(reg);
4592
4593	switch (value) {
4594	.register => |value_reg| {
4595	try self.genStore(value_reg, addr_reg, i13, 0, abi_size);
4596	},
4597	else => {
4598	return self.fail("TODO implement copying of memory", .{});
4599	},
4600	}
4601	},
4602	.memory,
4603	.stack_offset,
4604	=> {
4605	const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr);
4606	try self.store(.{ .register = addr_reg }, value, ptr_ty, value_ty);
4607	},
4608	}
4609	}
4610
4611	fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32) !MCValue {
4612	return if (self.liveness.isUnused(inst)) .dead else result: {
4613	const pt = self.pt;
4614	const zcu = pt.zcu;
4615	const mcv = try self.resolveInst(operand);
4616	const ptr_ty = self.typeOf(operand);
4617	const struct_ty = ptr_ty.childType(zcu);
4618	const struct_field_offset: u32 = @intCast(struct_ty.structFieldOffset(index, zcu));
4619	switch (mcv) {
4620	.ptr_stack_offset => |off| {
4621	break :result MCValue{ .ptr_stack_offset = off - struct_field_offset };
4622	},
4623	else => {
4624	const offset_reg = try self.copyToTmpRegister(ptr_ty, .{
4625	.immediate = struct_field_offset,
4626	});
4627	const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
4628	defer self.register_manager.unlockReg(offset_reg_lock);
4629
4630	const addr_reg = try self.copyToTmpRegister(ptr_ty, mcv);
4631	const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
4632	defer self.register_manager.unlockReg(addr_reg_lock);
4633
4634	const dest = try self.binOp(
4635	.add,
4636	.{ .register = addr_reg },
4637	.{ .register = offset_reg },
4638	Type.usize,
4639	Type.usize,
4640	null,
4641	);
4642
4643	break :result dest;
4644	},
4645	}
4646	};
4647	}
4648
4649	fn trunc(
4650	self: *Self,
4651	maybe_inst: ?Air.Inst.Index,
4652	operand: MCValue,
4653	operand_ty: Type,
4654	dest_ty: Type,
4655	) !MCValue {
4656	const pt = self.pt;
4657	const zcu = pt.zcu;
4658	const info_a = operand_ty.intInfo(zcu);
4659	const info_b = dest_ty.intInfo(zcu);
4660
4661	if (info_b.bits <= 64) {
4662	const operand_reg = switch (operand) {
4663	.register => |r| r,
4664	else => operand_reg: {
4665	if (info_a.bits <= 64) {
4666	const reg = try self.copyToTmpRegister(operand_ty, operand);
4667	break :operand_reg reg;
4668	} else {
4669	return self.fail("TODO load least significant word into register", .{});
4670	}
4671	},
4672	};
4673	const lock = self.register_manager.lockReg(operand_reg);
4674	defer if (lock) |reg| self.register_manager.unlockReg(reg);
4675
4676	const dest_reg = if (maybe_inst) |inst| blk: {
4677	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
4678
4679	if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) {
4680	break :blk operand_reg;
4681	} else {
4682	const reg = try self.register_manager.allocReg(inst, gp);
4683	break :blk reg;
4684	}
4685	} else blk: {
4686	const reg = try self.register_manager.allocReg(null, gp);
4687	break :blk reg;
4688	};
4689
4690	try self.truncRegister(operand_reg, dest_reg, info_b.signedness, info_b.bits);
4691
4692	return MCValue{ .register = dest_reg };
4693	} else {
4694	return self.fail("TODO: truncate to ints > 64 bits", .{});
4695	}
4696	}
4697
4698	fn truncRegister(
4699	self: *Self,
4700	operand_reg: Register,
4701	dest_reg: Register,
4702	int_signedness: std.lang.Signedness,
4703	int_bits: u16,
4704	) !void {
4705	switch (int_bits) {
4706	1...31, 33...63 => {
4707	_ = try self.addInst(.{
4708	.tag = .sllx,
4709	.data = .{
4710	.shift = .{
4711	.is_imm = true,
4712	.rd = dest_reg,
4713	.rs1 = operand_reg,
4714	.rs2_or_imm = .{ .imm = @as(u6, @intCast(64 - int_bits)) },
4715	},
4716	},
4717	});
4718	_ = try self.addInst(.{
4719	.tag = switch (int_signedness) {
4720	.signed => .srax,
4721	.unsigned => .srlx,
4722	},
4723	.data = .{
4724	.shift = .{
4725	.is_imm = true,
4726	.rd = dest_reg,
4727	.rs1 = dest_reg,
4728	.rs2_or_imm = .{ .imm = @as(u6, @intCast(int_bits)) },
4729	},
4730	},
4731	});
4732	},
4733	32 => {
4734	_ = try self.addInst(.{
4735	.tag = switch (int_signedness) {
4736	.signed => .sra,
4737	.unsigned => .srl,
4738	},
4739	.data = .{
4740	.shift = .{
4741	.is_imm = true,
4742	.rd = dest_reg,
4743	.rs1 = operand_reg,
4744	.rs2_or_imm = .{ .imm = 0 },
4745	},
4746	},
4747	});
4748	},
4749	64 => {
4750	if (dest_reg == operand_reg)
4751	return; // Copy register to itself; nothing to do.
4752	_ = try self.addInst(.{
4753	.tag = .mov,
4754	.data = .{
4755	.arithmetic_2op = .{
4756	.is_imm = false,
4757	.rs1 = dest_reg,
4758	.rs2_or_imm = .{ .rs2 = operand_reg },
4759	},
4760	},
4761	});
4762	},
4763	else => unreachable,
4764	}
4765	}
4766
4767	/// TODO support scope overrides. Also note this logic is duplicated with `Zcu.wantSafety`.
4768	fn wantSafety(self: *Self) bool {
4769	return switch (self.bin_file.comp.root_mod.optimize_mode) {
4770	.debug => true,
4771	.safe => true,
4772	.fast => false,
4773	.small => false,
4774	};
4775	}
4776
4777	fn typeOf(self: *Self, inst: Air.Inst.Ref) Type {
4778	return self.air.typeOf(inst, &self.pt.zcu.intern_pool);
4779	}
4780
4781	fn typeOfIndex(self: *Self, inst: Air.Inst.Index) Type {
4782	return self.air.typeOfIndex(inst, &self.pt.zcu.intern_pool);
4783	}