Commit 91d93b6395bf4a5718cffe18d4a9351b2ff06492 | zig

author	Jakub Konka <kubkon@jakubkonka.com> 2021-10-31 13:01:00+01:00
committer	Andrew Kelley <andrew@ziglang.org> 2021-10-31 17:32:39-04:00
log	91d93b6395bf4a5718cffe18d4a9351b2ff06492
tree	9166a411461ab1df66e00244049008ff42a51274
parent	c452d10953366bae4f2c6d2bf689eaaf51da739a

stage2: move x86_64 codegen to arch/x86_64/CodeGen.zig

This mimics steps taken for aarch64 and preps stage2 x86_64
for a rewrite introducing MIR for this arch.

2 files changed, 3663 insertions(+), 1281 deletions(-)

src
- arch
  - x86_64
    CodeGen.zig
- codegen.zig

src/arch/x86_64/CodeGen.zig created+3647

...	@@ -0,0 +1,3647 @@
1	const std = @import("std");
2	const build_options = @import("build_options");
3	const builtin = @import("builtin");
4	const assert = std.debug.assert;
5	const leb128 = std.leb;
6	const link = @import("../../link.zig");
7	const log = std.log.scoped(.codegen);
8	const math = std.math;
9	const mem = std.mem;
10	const trace = @import("../../tracy.zig").trace;
11
12	const Air = @import("../../Air.zig");
13	const Allocator = mem.Allocator;
14	const Compilation = @import("../../Compilation.zig");
15	const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;
16	const DW = std.dwarf;
17	const Encoder = @import("bits.zig").Encoder;
18	const ErrorMsg = Module.ErrorMsg;
19	const FnResult = @import("../../codegen.zig").FnResult;
20	const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError;
21	const Liveness = @import("../../Liveness.zig");
22	const Module = @import("../../Module.zig");
23	const RegisterManager = @import("../../register_manager.zig").RegisterManager;
24	const Target = std.Target;
25	const Type = @import("../../type.zig").Type;
26	const TypedValue = @import("../../TypedValue.zig");
27	const Value = @import("../../value.zig").Value;
28	const Zir = @import("../../Zir.zig");
29
30	const InnerError = error{
31	OutOfMemory,
32	CodegenFail,
33	};
34
35	arch: std.Target.Cpu.Arch,
36	gpa: *Allocator,
37	air: Air,
38	liveness: Liveness,
39	bin_file: *link.File,
40	target: *const std.Target,
41	mod_fn: *const Module.Fn,
42	code: *std.ArrayList(u8),
43	debug_output: DebugInfoOutput,
44	err_msg: ?*ErrorMsg,
45	args: []MCValue,
46	ret_mcv: MCValue,
47	fn_type: Type,
48	arg_index: usize,
49	src_loc: Module.SrcLoc,
50	stack_align: u32,
51
52	prev_di_line: u32,
53	prev_di_column: u32,
54	/// Byte offset within the source file of the ending curly.
55	end_di_line: u32,
56	end_di_column: u32,
57	/// Relative to the beginning of `code`.
58	prev_di_pc: usize,
59
60	/// The value is an offset into the `Function` `code` from the beginning.
61	/// To perform the reloc, write 32-bit signed little-endian integer
62	/// which is a relative jump, based on the address following the reloc.
63	exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .{},
64
65	/// Whenever there is a runtime branch, we push a Branch onto this stack,
66	/// and pop it off when the runtime branch joins. This provides an "overlay"
67	/// of the table of mappings from instructions to `MCValue` from within the branch.
68	/// This way we can modify the `MCValue` for an instruction in different ways
69	/// within different branches. Special consideration is needed when a branch
70	/// joins with its parent, to make sure all instructions have the same MCValue
71	/// across each runtime branch upon joining.
72	branch_stack: *std.ArrayList(Branch),
73
74	// Key is the block instruction
75	blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .{},
76
77	register_manager: RegisterManager(Self, Register, &callee_preserved_regs) = .{},
78	/// Maps offset to what is stored there.
79	stack: std.AutoHashMapUnmanaged(u32, StackAllocation) = .{},
80
81	/// Offset from the stack base, representing the end of the stack frame.
82	max_end_stack: u32 = 0,
83	/// Represents the current end stack offset. If there is no existing slot
84	/// to place a new stack allocation, it goes here, and then bumps `max_end_stack`.
85	next_stack_offset: u32 = 0,
86
87	/// Debug field, used to find bugs in the compiler.
88	air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init,
89
90	const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {};
91
92	const MCValue = union(enum) {
93	/// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc.
94	/// TODO Look into deleting this tag and using `dead` instead, since every use
95	/// of MCValue.none should be instead looking at the type and noticing it is 0 bits.
96	none,
97	/// Control flow will not allow this value to be observed.
98	unreach,
99	/// No more references to this value remain.
100	dead,
101	/// The value is undefined.
102	undef,
103	/// A pointer-sized integer that fits in a register.
104	/// If the type is a pointer, this is the pointer address in virtual address space.
105	immediate: u64,
106	/// The constant was emitted into the code, at this offset.
107	/// If the type is a pointer, it means the pointer address is embedded in the code.
108	embedded_in_code: usize,
109	/// The value is a pointer to a constant which was emitted into the code, at this offset.
110	ptr_embedded_in_code: usize,
111	/// The value is in a target-specific register.
112	register: Register,
113	/// The value is in memory at a hard-coded address.
114	/// If the type is a pointer, it means the pointer address is at this memory location.
115	memory: u64,
116	/// The value is one of the stack variables.
117	/// If the type is a pointer, it means the pointer address is in the stack at this offset.
118	stack_offset: u32,
119	/// The value is a pointer to one of the stack variables (payload is stack offset).
120	ptr_stack_offset: u32,
121	/// The value is in the compare flags assuming an unsigned operation,
122	/// with this operator applied on top of it.
123	compare_flags_unsigned: math.CompareOperator,
124	/// The value is in the compare flags assuming a signed operation,
125	/// with this operator applied on top of it.
126	compare_flags_signed: math.CompareOperator,
127
128	fn isMemory(mcv: MCValue) bool {
129	return switch (mcv) {
130	.embedded_in_code, .memory, .stack_offset => true,
131	else => false,
132	};
133	}
134
135	fn isImmediate(mcv: MCValue) bool {
136	return switch (mcv) {
137	.immediate => true,
138	else => false,
139	};
140	}
141
142	fn isMutable(mcv: MCValue) bool {
143	return switch (mcv) {
144	.none => unreachable,
145	.unreach => unreachable,
146	.dead => unreachable,
147
148	.immediate,
149	.embedded_in_code,
150	.memory,
151	.compare_flags_unsigned,
152	.compare_flags_signed,
153	.ptr_stack_offset,
154	.ptr_embedded_in_code,
155	.undef,
156	=> false,
157
158	.register,
159	.stack_offset,
160	=> true,
161	};
162	}
163	};
164
165	const Branch = struct {
166	inst_table: std.AutoArrayHashMapUnmanaged(Air.Inst.Index, MCValue) = .{},
167
168	fn deinit(self: Branch, gpa: Allocator) void {
169	self.inst_table.deinit(gpa);
170	self.* = undefined;
171	}
172	};
173
174	const StackAllocation = struct {
175	inst: Air.Inst.Index,
176	/// TODO do we need size? should be determined by inst.ty.abiSize()
177	size: u32,
178	};
179
180	const BlockData = struct {
181	relocs: std.ArrayListUnmanaged(Reloc),
182	/// The first break instruction encounters `null` here and chooses a
183	/// machine code value for the block result, populating this field.
184	/// Following break instructions encounter that value and use it for
185	/// the location to store their block results.
186	mcv: MCValue,
187	};
188
189	const Reloc = union(enum) {
190	/// The value is an offset into the `Function` `code` from the beginning.
191	/// To perform the reloc, write 32-bit signed little-endian integer
192	/// which is a relative jump, based on the address following the reloc.
193	rel32: usize,
194	/// A branch in the ARM instruction set
195	arm_branch: struct {
196	pos: usize,
197	cond: @import("../../arch/arm/bits.zig").Condition,
198	},
199	};
200
201	const BigTomb = struct {
202	function: *Self,
203	inst: Air.Inst.Index,
204	tomb_bits: Liveness.Bpi,
205	big_tomb_bits: u32,
206	bit_index: usize,
207
208	fn feed(bt: *BigTomb, op_ref: Air.Inst.Ref) void {
209	const this_bit_index = bt.bit_index;
210	bt.bit_index += 1;
211
212	const op_int = @enumToInt(op_ref);
213	if (op_int < Air.Inst.Ref.typed_value_map.len) return;
214	const op_index = @intCast(Air.Inst.Index, op_int - Air.Inst.Ref.typed_value_map.len);
215
216	if (this_bit_index < Liveness.bpi - 1) {
217	const dies = @truncate(u1, bt.tomb_bits >> @intCast(Liveness.OperandInt, this_bit_index)) != 0;
218	if (!dies) return;
219	} else {
220	const big_bit_index = @intCast(u5, this_bit_index - (Liveness.bpi - 1));
221	const dies = @truncate(u1, bt.big_tomb_bits >> big_bit_index) != 0;
222	if (!dies) return;
223	}
224	bt.function.processDeath(op_index);
225	}
226
227	fn finishAir(bt: *BigTomb, result: MCValue) void {
228	const is_used = !bt.function.liveness.isUnused(bt.inst);
229	if (is_used) {
230	log.debug("%{d} => {}", .{ bt.inst, result });
231	const branch = &bt.function.branch_stack.items[bt.function.branch_stack.items.len - 1];
232	branch.inst_table.putAssumeCapacityNoClobber(bt.inst, result);
233	}
234	bt.function.finishAirBookkeeping();
235	}
236	};
237
238	const Self = @This();
239
240	pub fn generate(
241	arch: std.Target.Cpu.Arch,
242	bin_file: *link.File,
243	src_loc: Module.SrcLoc,
244	module_fn: *Module.Fn,
245	air: Air,
246	liveness: Liveness,
247	code: *std.ArrayList(u8),
248	debug_output: DebugInfoOutput,
249	) GenerateSymbolError!FnResult {
250	if (build_options.skip_non_native and builtin.cpu.arch != arch) {
251	@panic("Attempted to compile for architecture that was disabled by build configuration");
252	}
253
254	assert(module_fn.owner_decl.has_tv);
255	const fn_type = module_fn.owner_decl.ty;
256
257	var branch_stack = std.ArrayList(Branch).init(bin_file.allocator);
258	defer {
259	assert(branch_stack.items.len == 1);
260	branch_stack.items[0].deinit(bin_file.allocator);
261	branch_stack.deinit();
262	}
263	try branch_stack.append(.{});
264
265	var function = Self{
266	.arch = arch,
267	.gpa = bin_file.allocator,
268	.air = air,
269	.liveness = liveness,
270	.target = &bin_file.options.target,
271	.bin_file = bin_file,
272	.mod_fn = module_fn,
273	.code = code,
274	.debug_output = debug_output,
275	.err_msg = null,
276	.args = undefined, // populated after `resolveCallingConventionValues`
277	.ret_mcv = undefined, // populated after `resolveCallingConventionValues`
278	.fn_type = fn_type,
279	.arg_index = 0,
280	.branch_stack = &branch_stack,
281	.src_loc = src_loc,
282	.stack_align = undefined,
283	.prev_di_pc = 0,
284	.prev_di_line = module_fn.lbrace_line,
285	.prev_di_column = module_fn.lbrace_column,
286	.end_di_line = module_fn.rbrace_line,
287	.end_di_column = module_fn.rbrace_column,
288	};
289	defer function.stack.deinit(bin_file.allocator);
290	defer function.blocks.deinit(bin_file.allocator);
291	defer function.exitlude_jump_relocs.deinit(bin_file.allocator);
292
293	var call_info = function.resolveCallingConventionValues(fn_type) catch \|err\| switch (err) {
294	error.CodegenFail => return FnResult{ .fail = function.err_msg.? },
295	else => \|e\| return e,
296	};
297	defer call_info.deinit(&function);
298
299	function.args = call_info.args;
300	function.ret_mcv = call_info.return_value;
301	function.stack_align = call_info.stack_align;
302	function.max_end_stack = call_info.stack_byte_count;
303
304	function.gen() catch \|err\| switch (err) {
305	error.CodegenFail => return FnResult{ .fail = function.err_msg.? },
306	else => \|e\| return e,
307	};
308
309	if (function.err_msg) \|em\| {
310	return FnResult{ .fail = em };
311	} else {
312	return FnResult{ .appended = {} };
313	}
314	}
315
316	fn gen(self: *Self) !void {
317	try self.code.ensureUnusedCapacity(11);
318
319	const cc = self.fn_type.fnCallingConvention();
320	if (cc != .Naked) {
321	// We want to subtract the aligned stack frame size from rsp here, but we don't
322	// yet know how big it will be, so we leave room for a 4-byte stack size.
323	// TODO During semantic analysis, check if there are no function calls. If there
324	// are none, here we can omit the part where we subtract and then add rsp.
325	self.code.appendSliceAssumeCapacity(&[_]u8{
326	0x55, // push rbp
327	0x48, 0x89, 0xe5, // mov rbp, rsp
328	0x48, 0x81, 0xec, // sub rsp, imm32 (with reloc)
329	});
330	const reloc_index = self.code.items.len;
331	self.code.items.len += 4;
332
333	try self.dbgSetPrologueEnd();
334	try self.genBody(self.air.getMainBody());
335
336	const stack_end = self.max_end_stack;
337	if (stack_end > math.maxInt(i32))
338	return self.failSymbol("too much stack used in call parameters", .{});
339	const aligned_stack_end = mem.alignForward(stack_end, self.stack_align);
340	mem.writeIntLittle(u32, self.code.items[reloc_index..][0..4], @intCast(u32, aligned_stack_end));
341
342	if (self.code.items.len >= math.maxInt(i32)) {
343	return self.failSymbol("unable to perform relocation: jump too far", .{});
344	}
345	if (self.exitlude_jump_relocs.items.len == 1) {
346	self.code.items.len -= 5;
347	} else for (self.exitlude_jump_relocs.items) \|jmp_reloc\| {
348	const amt = self.code.items.len - (jmp_reloc + 4);
349	const s32_amt = @intCast(i32, amt);
350	mem.writeIntLittle(i32, self.code.items[jmp_reloc..][0..4], s32_amt);
351	}
352
353	// Important to be after the possible self.code.items.len -= 5 above.
354	try self.dbgSetEpilogueBegin();
355
356	try self.code.ensureUnusedCapacity(9);
357	// add rsp, x
358	if (aligned_stack_end > math.maxInt(i8)) {
359	// example: 48 81 c4 ff ff ff 7f add rsp,0x7fffffff
360	self.code.appendSliceAssumeCapacity(&[_]u8{ 0x48, 0x81, 0xc4 });
361	const x = @intCast(u32, aligned_stack_end);
362	mem.writeIntLittle(u32, self.code.addManyAsArrayAssumeCapacity(4), x);
363	} else if (aligned_stack_end != 0) {
364	// example: 48 83 c4 7f add rsp,0x7f
365	const x = @intCast(u8, aligned_stack_end);
366	self.code.appendSliceAssumeCapacity(&[_]u8{ 0x48, 0x83, 0xc4, x });
367	}
368
369	self.code.appendSliceAssumeCapacity(&[_]u8{
370	0x5d, // pop rbp
371	0xc3, // ret
372	});
373	} else {
374	try self.dbgSetPrologueEnd();
375	try self.genBody(self.air.getMainBody());
376	try self.dbgSetEpilogueBegin();
377	}
378
379	// Drop them off at the rbrace.
380	try self.dbgAdvancePCAndLine(self.end_di_line, self.end_di_column);
381	}
382
383	fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
384	const air_tags = self.air.instructions.items(.tag);
385
386	for (body) \|inst\| {
387	const old_air_bookkeeping = self.air_bookkeeping;
388	try self.ensureProcessDeathCapacity(Liveness.bpi);
389
390	switch (air_tags[inst]) {
391	// zig fmt: off
392	.add, .ptr_add => try self.airAdd(inst),
393	.addwrap => try self.airAddWrap(inst),
394	.add_sat => try self.airAddSat(inst),
395	.sub, .ptr_sub => try self.airSub(inst),
396	.subwrap => try self.airSubWrap(inst),
397	.sub_sat => try self.airSubSat(inst),
398	.mul => try self.airMul(inst),
399	.mulwrap => try self.airMulWrap(inst),
400	.mul_sat => try self.airMulSat(inst),
401	.rem => try self.airRem(inst),
402	.mod => try self.airMod(inst),
403	.shl, .shl_exact => try self.airShl(inst),
404	.shl_sat => try self.airShlSat(inst),
405	.min => try self.airMin(inst),
406	.max => try self.airMax(inst),
407	.slice => try self.airSlice(inst),
408
409	.div_float, .div_trunc, .div_floor, .div_exact => try self.airDiv(inst),
410
411	.cmp_lt => try self.airCmp(inst, .lt),
412	.cmp_lte => try self.airCmp(inst, .lte),
413	.cmp_eq => try self.airCmp(inst, .eq),
414	.cmp_gte => try self.airCmp(inst, .gte),
415	.cmp_gt => try self.airCmp(inst, .gt),
416	.cmp_neq => try self.airCmp(inst, .neq),
417
418	.bool_and => try self.airBoolOp(inst),
419	.bool_or => try self.airBoolOp(inst),
420	.bit_and => try self.airBitAnd(inst),
421	.bit_or => try self.airBitOr(inst),
422	.xor => try self.airXor(inst),
423	.shr => try self.airShr(inst),
424
425	.alloc => try self.airAlloc(inst),
426	.ret_ptr => try self.airRetPtr(inst),
427	.arg => try self.airArg(inst),
428	.assembly => try self.airAsm(inst),
429	.bitcast => try self.airBitCast(inst),
430	.block => try self.airBlock(inst),
431	.br => try self.airBr(inst),
432	.breakpoint => try self.airBreakpoint(),
433	.fence => try self.airFence(),
434	.call => try self.airCall(inst),
435	.cond_br => try self.airCondBr(inst),
436	.dbg_stmt => try self.airDbgStmt(inst),
437	.fptrunc => try self.airFptrunc(inst),
438	.fpext => try self.airFpext(inst),
439	.intcast => try self.airIntCast(inst),
440	.trunc => try self.airTrunc(inst),
441	.bool_to_int => try self.airBoolToInt(inst),
442	.is_non_null => try self.airIsNonNull(inst),
443	.is_non_null_ptr => try self.airIsNonNullPtr(inst),
444	.is_null => try self.airIsNull(inst),
445	.is_null_ptr => try self.airIsNullPtr(inst),
446	.is_non_err => try self.airIsNonErr(inst),
447	.is_non_err_ptr => try self.airIsNonErrPtr(inst),
448	.is_err => try self.airIsErr(inst),
449	.is_err_ptr => try self.airIsErrPtr(inst),
450	.load => try self.airLoad(inst),
451	.loop => try self.airLoop(inst),
452	.not => try self.airNot(inst),
453	.ptrtoint => try self.airPtrToInt(inst),
454	.ret => try self.airRet(inst),
455	.ret_load => try self.airRetLoad(inst),
456	.store => try self.airStore(inst),
457	.struct_field_ptr=> try self.airStructFieldPtr(inst),
458	.struct_field_val=> try self.airStructFieldVal(inst),
459	.array_to_slice => try self.airArrayToSlice(inst),
460	.int_to_float => try self.airIntToFloat(inst),
461	.float_to_int => try self.airFloatToInt(inst),
462	.cmpxchg_strong => try self.airCmpxchg(inst),
463	.cmpxchg_weak => try self.airCmpxchg(inst),
464	.atomic_rmw => try self.airAtomicRmw(inst),
465	.atomic_load => try self.airAtomicLoad(inst),
466	.memcpy => try self.airMemcpy(inst),
467	.memset => try self.airMemset(inst),
468	.set_union_tag => try self.airSetUnionTag(inst),
469	.get_union_tag => try self.airGetUnionTag(inst),
470	.clz => try self.airClz(inst),
471	.ctz => try self.airCtz(inst),
472	.popcount => try self.airPopcount(inst),
473
474	.atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),
475	.atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),
476	.atomic_store_release => try self.airAtomicStore(inst, .Release),
477	.atomic_store_seq_cst => try self.airAtomicStore(inst, .SeqCst),
478
479	.struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0),
480	.struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1),
481	.struct_field_ptr_index_2 => try self.airStructFieldPtrIndex(inst, 2),
482	.struct_field_ptr_index_3 => try self.airStructFieldPtrIndex(inst, 3),
483
484	.switch_br => try self.airSwitch(inst),
485	.slice_ptr => try self.airSlicePtr(inst),
486	.slice_len => try self.airSliceLen(inst),
487
488	.ptr_slice_len_ptr => try self.airPtrSliceLenPtr(inst),
489	.ptr_slice_ptr_ptr => try self.airPtrSlicePtrPtr(inst),
490
491	.array_elem_val => try self.airArrayElemVal(inst),
492	.slice_elem_val => try self.airSliceElemVal(inst),
493	.slice_elem_ptr => try self.airSliceElemPtr(inst),
494	.ptr_elem_val => try self.airPtrElemVal(inst),
495	.ptr_elem_ptr => try self.airPtrElemPtr(inst),
496
497	.constant => unreachable, // excluded from function bodies
498	.const_ty => unreachable, // excluded from function bodies
499	.unreach => self.finishAirBookkeeping(),
500
501	.optional_payload => try self.airOptionalPayload(inst),
502	.optional_payload_ptr => try self.airOptionalPayloadPtr(inst),
503	.unwrap_errunion_err => try self.airUnwrapErrErr(inst),
504	.unwrap_errunion_payload => try self.airUnwrapErrPayload(inst),
505	.unwrap_errunion_err_ptr => try self.airUnwrapErrErrPtr(inst),
506	.unwrap_errunion_payload_ptr=> try self.airUnwrapErrPayloadPtr(inst),
507
508	.wrap_optional => try self.airWrapOptional(inst),
509	.wrap_errunion_payload => try self.airWrapErrUnionPayload(inst),
510	.wrap_errunion_err => try self.airWrapErrUnionErr(inst),
511	// zig fmt: on
512	}
513	if (std.debug.runtime_safety) {
514	if (self.air_bookkeeping < old_air_bookkeeping + 1) {
515	std.debug.panic("in codegen.zig, handling of AIR instruction %{d} ('{}') did not do proper bookkeeping. Look for a missing call to finishAir.", .{ inst, air_tags[inst] });
516	}
517	}
518	}
519	}
520
521	fn dbgSetPrologueEnd(self: *Self) InnerError!void {
522	switch (self.debug_output) {
523	.dwarf => \|dbg_out\| {
524	try dbg_out.dbg_line.append(DW.LNS.set_prologue_end);
525	try self.dbgAdvancePCAndLine(self.prev_di_line, self.prev_di_column);
526	},
527	.plan9 => {},
528	.none => {},
529	}
530	}
531
532	fn dbgSetEpilogueBegin(self: *Self) InnerError!void {
533	switch (self.debug_output) {
534	.dwarf => \|dbg_out\| {
535	try dbg_out.dbg_line.append(DW.LNS.set_epilogue_begin);
536	try self.dbgAdvancePCAndLine(self.prev_di_line, self.prev_di_column);
537	},
538	.plan9 => {},
539	.none => {},
540	}
541	}
542
543	fn dbgAdvancePCAndLine(self: *Self, line: u32, column: u32) InnerError!void {
544	const delta_line = @intCast(i32, line) - @intCast(i32, self.prev_di_line);
545	const delta_pc: usize = self.code.items.len - self.prev_di_pc;
546	switch (self.debug_output) {
547	.dwarf => \|dbg_out\| {
548	// TODO Look into using the DWARF special opcodes to compress this data.
549	// It lets you emit single-byte opcodes that add different numbers to
550	// both the PC and the line number at the same time.
551	try dbg_out.dbg_line.ensureUnusedCapacity(11);
552	dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_pc);
553	leb128.writeULEB128(dbg_out.dbg_line.writer(), delta_pc) catch unreachable;
554	if (delta_line != 0) {
555	dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_line);
556	leb128.writeILEB128(dbg_out.dbg_line.writer(), delta_line) catch unreachable;
557	}
558	dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.copy);
559	self.prev_di_pc = self.code.items.len;
560	self.prev_di_line = line;
561	self.prev_di_column = column;
562	self.prev_di_pc = self.code.items.len;
563	},
564	.plan9 => \|dbg_out\| {
565	if (delta_pc <= 0) return; // only do this when the pc changes
566	// we have already checked the target in the linker to make sure it is compatable
567	const quant = @import("../../link/Plan9/aout.zig").getPCQuant(self.target.cpu.arch) catch unreachable;
568
569	// increasing the line number
570	try @import("../../link/Plan9.zig").changeLine(dbg_out.dbg_line, delta_line);
571	// increasing the pc
572	const d_pc_p9 = @intCast(i64, delta_pc) - quant;
573	if (d_pc_p9 > 0) {
574	// minus one because if its the last one, we want to leave space to change the line which is one quanta
575	try dbg_out.dbg_line.append(@intCast(u8, @divExact(d_pc_p9, quant) + 128) - quant);
576	if (dbg_out.pcop_change_index.*) \|pci\|
577	dbg_out.dbg_line.items[pci] += 1;
578	dbg_out.pcop_change_index.* = @intCast(u32, dbg_out.dbg_line.items.len - 1);
579	} else if (d_pc_p9 == 0) {
580	// we don't need to do anything, because adding the quant does it for us
581	} else unreachable;
582	if (dbg_out.start_line.* == null)
583	dbg_out.start_line.* = self.prev_di_line;
584	dbg_out.end_line.* = line;
585	// only do this if the pc changed
586	self.prev_di_line = line;
587	self.prev_di_column = column;
588	self.prev_di_pc = self.code.items.len;
589	},
590	.none => {},
591	}
592	}
593
594	/// Asserts there is already capacity to insert into top branch inst_table.
595	fn processDeath(self: *Self, inst: Air.Inst.Index) void {
596	const air_tags = self.air.instructions.items(.tag);
597	if (air_tags[inst] == .constant) return; // Constants are immortal.
598	// When editing this function, note that the logic must synchronize with `reuseOperand`.
599	const prev_value = self.getResolvedInstValue(inst);
600	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
601	branch.inst_table.putAssumeCapacity(inst, .dead);
602	switch (prev_value) {
603	.register => \|reg\| {
604	const canon_reg = reg.to64();
605	self.register_manager.freeReg(canon_reg);
606	},
607	else => {}, // TODO process stack allocation death
608	}
609	}
610
611	/// Called when there are no operands, and the instruction is always unreferenced.
612	fn finishAirBookkeeping(self: *Self) void {
613	if (std.debug.runtime_safety) {
614	self.air_bookkeeping += 1;
615	}
616	}
617
618	fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref) void {
619	var tomb_bits = self.liveness.getTombBits(inst);
620	for (operands) \|op\| {
621	const dies = @truncate(u1, tomb_bits) != 0;
622	tomb_bits >>= 1;
623	if (!dies) continue;
624	const op_int = @enumToInt(op);
625	if (op_int < Air.Inst.Ref.typed_value_map.len) continue;
626	const op_index = @intCast(Air.Inst.Index, op_int - Air.Inst.Ref.typed_value_map.len);
627	self.processDeath(op_index);
628	}
629	const is_used = @truncate(u1, tomb_bits) == 0;
630	if (is_used) {
631	log.debug("%{d} => {}", .{ inst, result });
632	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
633	branch.inst_table.putAssumeCapacityNoClobber(inst, result);
634
635	switch (result) {
636	.register => \|reg\| {
637	// In some cases (such as bitcast), an operand
638	// may be the same MCValue as the result. If
639	// that operand died and was a register, it
640	// was freed by processDeath. We have to
641	// "re-allocate" the register.
642	if (self.register_manager.isRegFree(reg)) {
643	self.register_manager.getRegAssumeFree(reg, inst);
644	}
645	},
646	else => {},
647	}
648	}
649	self.finishAirBookkeeping();
650	}
651
652	fn ensureProcessDeathCapacity(self: *Self, additional_count: usize) !void {
653	const table = &self.branch_stack.items[self.branch_stack.items.len - 1].inst_table;
654	try table.ensureUnusedCapacity(self.gpa, additional_count);
655	}
656
657	/// Adds a Type to the .debug_info at the current position. The bytes will be populated later,
658	/// after codegen for this symbol is done.
659	fn addDbgInfoTypeReloc(self: *Self, ty: Type) !void {
660	switch (self.debug_output) {
661	.dwarf => \|dbg_out\| {
662	assert(ty.hasCodeGenBits());
663	const index = dbg_out.dbg_info.items.len;
664	try dbg_out.dbg_info.resize(index + 4); // DW.AT.type, DW.FORM.ref4
665
666	const gop = try dbg_out.dbg_info_type_relocs.getOrPut(self.gpa, ty);
667	if (!gop.found_existing) {
668	gop.value_ptr.* = .{
669	.off = undefined,
670	.relocs = .{},
671	};
672	}
673	try gop.value_ptr.relocs.append(self.gpa, @intCast(u32, index));
674	},
675	.plan9 => {},
676	.none => {},
677	}
678	}
679
680	fn allocMem(self: *Self, inst: Air.Inst.Index, abi_size: u32, abi_align: u32) !u32 {
681	if (abi_align > self.stack_align)
682	self.stack_align = abi_align;
683	// TODO find a free slot instead of always appending
684	const offset = mem.alignForwardGeneric(u32, self.next_stack_offset, abi_align);
685	self.next_stack_offset = offset + abi_size;
686	if (self.next_stack_offset > self.max_end_stack)
687	self.max_end_stack = self.next_stack_offset;
688	try self.stack.putNoClobber(self.gpa, offset, .{
689	.inst = inst,
690	.size = abi_size,
691	});
692	return offset;
693	}
694
695	/// Use a pointer instruction as the basis for allocating stack memory.
696	fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !u32 {
697	const elem_ty = self.air.typeOfIndex(inst).elemType();
698	const abi_size = math.cast(u32, elem_ty.abiSize(self.target.*)) catch {
699	return self.fail("type '{}' too big to fit into stack frame", .{elem_ty});
700	};
701	// TODO swap this for inst.ty.ptrAlign
702	const abi_align = elem_ty.abiAlignment(self.target.*);
703	return self.allocMem(inst, abi_size, abi_align);
704	}
705
706	fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
707	const elem_ty = self.air.typeOfIndex(inst);
708	const abi_size = math.cast(u32, elem_ty.abiSize(self.target.*)) catch {
709	return self.fail("type '{}' too big to fit into stack frame", .{elem_ty});
710	};
711	const abi_align = elem_ty.abiAlignment(self.target.*);
712	if (abi_align > self.stack_align)
713	self.stack_align = abi_align;
714
715	if (reg_ok) {
716	// Make sure the type can fit in a register before we try to allocate one.
717	const ptr_bits = self.target.cpu.arch.ptrBitWidth();
718	const ptr_bytes: u64 = @divExact(ptr_bits, 8);
719	if (abi_size <= ptr_bytes) {
720	if (self.register_manager.tryAllocReg(inst, &.{})) \|reg\| {
721	return MCValue{ .register = registerAlias(reg, abi_size) };
722	}
723	}
724	}
725	const stack_offset = try self.allocMem(inst, abi_size, abi_align);
726	return MCValue{ .stack_offset = stack_offset };
727	}
728
729	pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {
730	const stack_mcv = try self.allocRegOrMem(inst, false);
731	log.debug("spilling {d} to stack mcv {any}", .{ inst, stack_mcv });
732	const reg_mcv = self.getResolvedInstValue(inst);
733	assert(reg == reg_mcv.register.to64());
734	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
735	try branch.inst_table.put(self.gpa, inst, stack_mcv);
736	try self.genSetStack(self.air.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv);
737	}
738
739	/// Copies a value to a register without tracking the register. The register is not considered
740	/// allocated. A second call to `copyToTmpRegister` may return the same register.
741	/// This can have a side effect of spilling instructions to the stack to free up a register.
742	fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
743	const reg = try self.register_manager.allocReg(null, &.{});
744	try self.genSetReg(ty, reg, mcv);
745	return reg;
746	}
747
748	/// Allocates a new register and copies `mcv` into it.
749	/// `reg_owner` is the instruction that gets associated with the register in the register table.
750	/// This can have a side effect of spilling instructions to the stack to free up a register.
751	fn copyToNewRegister(self: *Self, reg_owner: Air.Inst.Index, mcv: MCValue) !MCValue {
752	const reg = try self.register_manager.allocReg(reg_owner, &.{});
753	try self.genSetReg(self.air.typeOfIndex(reg_owner), reg, mcv);
754	return MCValue{ .register = reg };
755	}
756
757	fn airAlloc(self: *Self, inst: Air.Inst.Index) !void {
758	const stack_offset = try self.allocMemPtr(inst);
759	return self.finishAir(inst, .{ .ptr_stack_offset = stack_offset }, .{ .none, .none, .none });
760	}
761
762	fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {
763	const stack_offset = try self.allocMemPtr(inst);
764	return self.finishAir(inst, .{ .ptr_stack_offset = stack_offset }, .{ .none, .none, .none });
765	}
766
767	fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
768	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
769	_ = ty_op;
770	return self.fail("TODO implement airFptrunc for {}", .{self.target.cpu.arch});
771	// return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
772	}
773
774	fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
775	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
776	_ = ty_op;
777	return self.fail("TODO implement airFpext for {}", .{self.target.cpu.arch});
778	// return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
779	}
780
781	fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
782	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
783	if (self.liveness.isUnused(inst))
784	return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
785
786	const operand_ty = self.air.typeOf(ty_op.operand);
787	const operand = try self.resolveInst(ty_op.operand);
788	const info_a = operand_ty.intInfo(self.target.*);
789	const info_b = self.air.typeOfIndex(inst).intInfo(self.target.*);
790	if (info_a.signedness != info_b.signedness)
791	return self.fail("TODO gen intcast sign safety in semantic analysis", .{});
792
793	if (info_a.bits == info_b.bits)
794	return self.finishAir(inst, operand, .{ ty_op.operand, .none, .none });
795
796	return self.fail("TODO implement intCast for {}", .{self.target.cpu.arch});
797	}
798
799	fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
800	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
801	if (self.liveness.isUnused(inst))
802	return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
803
804	const operand = try self.resolveInst(ty_op.operand);
805	_ = operand;
806	return self.fail("TODO implement trunc for {}", .{self.target.cpu.arch});
807	}
808
809	fn airBoolToInt(self: *Self, inst: Air.Inst.Index) !void {
810	const un_op = self.air.instructions.items(.data)[inst].un_op;
811	const operand = try self.resolveInst(un_op);
812	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else operand;
813	return self.finishAir(inst, result, .{ un_op, .none, .none });
814	}
815
816	fn airNot(self: *Self, inst: Air.Inst.Index) !void {
817	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
818	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
819	const operand = try self.resolveInst(ty_op.operand);
820	switch (operand) {
821	.dead => unreachable,
822	.unreach => unreachable,
823	.compare_flags_unsigned => \|op\| {
824	const r = MCValue{
825	.compare_flags_unsigned = switch (op) {
826	.gte => .lt,
827	.gt => .lte,
828	.neq => .eq,
829	.lt => .gte,
830	.lte => .gt,
831	.eq => .neq,
832	},
833	};
834	break :result r;
835	},
836	.compare_flags_signed => \|op\| {
837	const r = MCValue{
838	.compare_flags_signed = switch (op) {
839	.gte => .lt,
840	.gt => .lte,
841	.neq => .eq,
842	.lt => .gte,
843	.lte => .gt,
844	.eq => .neq,
845	},
846	};
847	break :result r;
848	},
849	else => {},
850	}
851	break :result try self.genX8664BinMath(inst, ty_op.operand, .bool_true);
852	};
853	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
854	}
855
856	fn airMin(self: *Self, inst: Air.Inst.Index) !void {
857	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
858	const result: MCValue = if (self.liveness.isUnused(inst))
859	.dead
860	else
861	return self.fail("TODO implement min for {}", .{self.target.cpu.arch});
862	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
863	}
864
865	fn airMax(self: *Self, inst: Air.Inst.Index) !void {
866	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
867	const result: MCValue = if (self.liveness.isUnused(inst))
868	.dead
869	else
870	return self.fail("TODO implement max for {}", .{self.target.cpu.arch});
871	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
872	}
873
874	fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
875	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
876	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
877	const result: MCValue = if (self.liveness.isUnused(inst))
878	.dead
879	else
880	return self.fail("TODO implement slice for {}", .{self.target.cpu.arch});
881	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
882	}
883
884	fn airAdd(self: *Self, inst: Air.Inst.Index) !void {
885	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
886	const result: MCValue = if (self.liveness.isUnused(inst))
887	.dead
888	else
889	try self.genX8664BinMath(inst, bin_op.lhs, bin_op.rhs);
890	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
891	}
892
893	fn airAddWrap(self: *Self, inst: Air.Inst.Index) !void {
894	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
895	const result: MCValue = if (self.liveness.isUnused(inst))
896	.dead
897	else
898	return self.fail("TODO implement addwrap for {}", .{self.target.cpu.arch});
899	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
900	}
901
902	fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
903	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
904	const result: MCValue = if (self.liveness.isUnused(inst))
905	.dead
906	else
907	return self.fail("TODO implement add_sat for {}", .{self.target.cpu.arch});
908	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
909	}
910
911	fn airSub(self: *Self, inst: Air.Inst.Index) !void {
912	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
913	const result: MCValue = if (self.liveness.isUnused(inst))
914	.dead
915	else
916	try self.genX8664BinMath(inst, bin_op.lhs, bin_op.rhs);
917	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
918	}
919
920	fn airSubWrap(self: *Self, inst: Air.Inst.Index) !void {
921	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
922	const result: MCValue = if (self.liveness.isUnused(inst))
923	.dead
924	else
925	return self.fail("TODO implement subwrap for {}", .{self.target.cpu.arch});
926	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
927	}
928
929	fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
930	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
931	const result: MCValue = if (self.liveness.isUnused(inst))
932	.dead
933	else
934	return self.fail("TODO implement sub_sat for {}", .{self.target.cpu.arch});
935	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
936	}
937
938	fn airMul(self: *Self, inst: Air.Inst.Index) !void {
939	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
940	const result: MCValue = if (self.liveness.isUnused(inst))
941	.dead
942	else
943	try self.genX8664BinMath(inst, bin_op.lhs, bin_op.rhs);
944	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
945	}
946
947	fn airMulWrap(self: *Self, inst: Air.Inst.Index) !void {
948	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
949	const result: MCValue = if (self.liveness.isUnused(inst))
950	.dead
951	else
952	return self.fail("TODO implement mulwrap for {}", .{self.target.cpu.arch});
953	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
954	}
955
956	fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
957	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
958	const result: MCValue = if (self.liveness.isUnused(inst))
959	.dead
960	else
961	return self.fail("TODO implement mul_sat for {}", .{self.target.cpu.arch});
962	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
963	}
964
965	fn airDiv(self: *Self, inst: Air.Inst.Index) !void {
966	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
967	const result: MCValue = if (self.liveness.isUnused(inst))
968	.dead
969	else
970	return self.fail("TODO implement div for {}", .{self.target.cpu.arch});
971	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
972	}
973
974	fn airRem(self: *Self, inst: Air.Inst.Index) !void {
975	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
976	const result: MCValue = if (self.liveness.isUnused(inst))
977	.dead
978	else
979	return self.fail("TODO implement rem for {}", .{self.target.cpu.arch});
980	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
981	}
982
983	fn airMod(self: *Self, inst: Air.Inst.Index) !void {
984	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
985	const result: MCValue = if (self.liveness.isUnused(inst))
986	.dead
987	else
988	return self.fail("TODO implement mod for {}", .{self.target.cpu.arch});
989	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
990	}
991
992	fn airBitAnd(self: *Self, inst: Air.Inst.Index) !void {
993	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
994	const result: MCValue = if (self.liveness.isUnused(inst))
995	.dead
996	else
997	try self.genX8664BinMath(inst, bin_op.lhs, bin_op.rhs);
998	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
999	}
1000
1001	fn airBitOr(self: *Self, inst: Air.Inst.Index) !void {
1002	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1003	const result: MCValue = if (self.liveness.isUnused(inst))
1004	.dead
1005	else
1006	try self.genX8664BinMath(inst, bin_op.lhs, bin_op.rhs);
1007	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1008	}
1009
1010	fn airXor(self: *Self, inst: Air.Inst.Index) !void {
1011	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1012	const result: MCValue = if (self.liveness.isUnused(inst))
1013	.dead
1014	else
1015	return self.fail("TODO implement xor for {}", .{self.target.cpu.arch});
1016	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1017	}
1018
1019	fn airShl(self: *Self, inst: Air.Inst.Index) !void {
1020	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1021	const result: MCValue = if (self.liveness.isUnused(inst))
1022	.dead
1023	else
1024	return self.fail("TODO implement shl for {}", .{self.target.cpu.arch});
1025	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1026	}
1027
1028	fn airShlSat(self: *Self, inst: Air.Inst.Index) !void {
1029	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1030	const result: MCValue = if (self.liveness.isUnused(inst))
1031	.dead
1032	else
1033	return self.fail("TODO implement shl_sat for {}", .{self.target.cpu.arch});
1034	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1035	}
1036
1037	fn airShr(self: *Self, inst: Air.Inst.Index) !void {
1038	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1039	const result: MCValue = if (self.liveness.isUnused(inst))
1040	.dead
1041	else
1042	return self.fail("TODO implement shr for {}", .{self.target.cpu.arch});
1043	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1044	}
1045
1046	fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {
1047	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1048	const result: MCValue = if (self.liveness.isUnused(inst))
1049	.dead
1050	else
1051	return self.fail("TODO implement .optional_payload for {}", .{self.target.cpu.arch});
1052	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1053	}
1054
1055	fn airOptionalPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
1056	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1057	const result: MCValue = if (self.liveness.isUnused(inst))
1058	.dead
1059	else
1060	return self.fail("TODO implement .optional_payload_ptr for {}", .{self.target.cpu.arch});
1061	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1062	}
1063
1064	fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) !void {
1065	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1066	const result: MCValue = if (self.liveness.isUnused(inst))
1067	.dead
1068	else
1069	return self.fail("TODO implement unwrap error union error for {}", .{self.target.cpu.arch});
1070	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1071	}
1072
1073	fn airUnwrapErrPayload(self: *Self, inst: Air.Inst.Index) !void {
1074	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1075	const result: MCValue = if (self.liveness.isUnused(inst))
1076	.dead
1077	else
1078	return self.fail("TODO implement unwrap error union payload for {}", .{self.target.cpu.arch});
1079	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1080	}
1081
1082	// *(E!T) -> E
1083	fn airUnwrapErrErrPtr(self: *Self, inst: Air.Inst.Index) !void {
1084	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1085	const result: MCValue = if (self.liveness.isUnused(inst))
1086	.dead
1087	else
1088	return self.fail("TODO implement unwrap error union error ptr for {}", .{self.target.cpu.arch});
1089	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1090	}
1091
1092	// (E!T) -> T
1093	fn airUnwrapErrPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
1094	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1095	const result: MCValue = if (self.liveness.isUnused(inst))
1096	.dead
1097	else
1098	return self.fail("TODO implement unwrap error union payload ptr for {}", .{self.target.cpu.arch});
1099	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1100	}
1101
1102	fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
1103	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1104	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1105	const optional_ty = self.air.typeOfIndex(inst);
1106
1107	// Optional with a zero-bit payload type is just a boolean true
1108	if (optional_ty.abiSize(self.target.*) == 1)
1109	break :result MCValue{ .immediate = 1 };
1110
1111	return self.fail("TODO implement wrap optional for {}", .{self.target.cpu.arch});
1112	};
1113	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1114	}
1115
1116	/// T to E!T
1117	fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
1118	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1119	const result: MCValue = if (self.liveness.isUnused(inst))
1120	.dead
1121	else
1122	return self.fail("TODO implement wrap errunion payload for {}", .{self.target.cpu.arch});
1123	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1124	}
1125
1126	/// E to E!T
1127	fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
1128	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1129	const result: MCValue = if (self.liveness.isUnused(inst))
1130	.dead
1131	else
1132	return self.fail("TODO implement wrap errunion error for {}", .{self.target.cpu.arch});
1133	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1134	}
1135
1136	fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {
1137	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1138	const result: MCValue = if (self.liveness.isUnused(inst))
1139	.dead
1140	else
1141	return self.fail("TODO implement slice_ptr for {}", .{self.target.cpu.arch});
1142	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1143	}
1144
1145	fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {
1146	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1147	const result: MCValue = if (self.liveness.isUnused(inst))
1148	.dead
1149	else
1150	return self.fail("TODO implement slice_len for {}", .{self.target.cpu.arch});
1151	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1152	}
1153
1154	fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
1155	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1156	const result: MCValue = if (self.liveness.isUnused(inst))
1157	.dead
1158	else
1159	return self.fail("TODO implement ptr_slice_len_ptr for {}", .{self.target.cpu.arch});
1160	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1161	}
1162
1163	fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void {
1164	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1165	const result: MCValue = if (self.liveness.isUnused(inst))
1166	.dead
1167	else
1168	return self.fail("TODO implement ptr_slice_ptr_ptr for {}", .{self.target.cpu.arch});
1169	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1170	}
1171
1172	fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
1173	const is_volatile = false; // TODO
1174	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1175	const result: MCValue = if (!is_volatile and self.liveness.isUnused(inst))
1176	.dead
1177	else
1178	return self.fail("TODO implement slice_elem_val for {}", .{self.target.cpu.arch});
1179	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1180	}
1181
1182	fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) !void {
1183	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1184	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
1185	const result: MCValue = if (self.liveness.isUnused(inst))
1186	.dead
1187	else
1188	return self.fail("TODO implement slice_elem_ptr for {}", .{self.target.cpu.arch});
1189	return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
1190	}
1191
1192	fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
1193	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1194	const result: MCValue = if (self.liveness.isUnused(inst))
1195	.dead
1196	else
1197	return self.fail("TODO implement array_elem_val for {}", .{self.target.cpu.arch});
1198	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1199	}
1200
1201	fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
1202	const is_volatile = false; // TODO
1203	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1204	const result: MCValue = if (!is_volatile and self.liveness.isUnused(inst))
1205	.dead
1206	else
1207	return self.fail("TODO implement ptr_elem_val for {}", .{self.target.cpu.arch});
1208	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1209	}
1210
1211	fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
1212	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1213	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
1214	const result: MCValue = if (self.liveness.isUnused(inst))
1215	.dead
1216	else
1217	return self.fail("TODO implement ptr_elem_ptr for {}", .{self.target.cpu.arch});
1218	return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
1219	}
1220
1221	fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
1222	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1223	_ = bin_op;
1224	return self.fail("TODO implement airSetUnionTag for {}", .{self.target.cpu.arch});
1225	}
1226
1227	fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
1228	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1229	const result: MCValue = if (self.liveness.isUnused(inst))
1230	.dead
1231	else
1232	return self.fail("TODO implement airGetUnionTag for {}", .{self.target.cpu.arch});
1233	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1234	}
1235
1236	fn airClz(self: *Self, inst: Air.Inst.Index) !void {
1237	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1238	const result: MCValue = if (self.liveness.isUnused(inst))
1239	.dead
1240	else
1241	return self.fail("TODO implement airClz for {}", .{self.target.cpu.arch});
1242	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1243	}
1244
1245	fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
1246	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1247	const result: MCValue = if (self.liveness.isUnused(inst))
1248	.dead
1249	else
1250	return self.fail("TODO implement airCtz for {}", .{self.target.cpu.arch});
1251	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1252	}
1253
1254	fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
1255	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1256	const result: MCValue = if (self.liveness.isUnused(inst))
1257	.dead
1258	else
1259	return self.fail("TODO implement airPopcount for {}", .{self.target.cpu.arch});
1260	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1261	}
1262
1263	fn reuseOperand(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, op_index: Liveness.OperandInt, mcv: MCValue) bool {
1264	if (!self.liveness.operandDies(inst, op_index))
1265	return false;
1266
1267	switch (mcv) {
1268	.register => \|reg\| {
1269	// If it's in the registers table, need to associate the register with the
1270	// new instruction.
1271	if (reg.allocIndex()) \|index\| {
1272	if (!self.register_manager.isRegFree(reg)) {
1273	self.register_manager.registers[index] = inst;
1274	}
1275	}
1276	log.debug("%{d} => {} (reused)", .{ inst, reg });
1277	},
1278	.stack_offset => \|off\| {
1279	log.debug("%{d} => stack offset {d} (reused)", .{ inst, off });
1280	},
1281	else => return false,
1282	}
1283
1284	// Prevent the operand deaths processing code from deallocating it.
1285	self.liveness.clearOperandDeath(inst, op_index);
1286
1287	// That makes us responsible for doing the rest of the stuff that processDeath would have done.
1288	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
1289	branch.inst_table.putAssumeCapacity(Air.refToIndex(operand).?, .dead);
1290
1291	return true;
1292	}
1293
1294	fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!void {
1295	const elem_ty = ptr_ty.elemType();
1296	switch (ptr) {
1297	.none => unreachable,
1298	.undef => unreachable,
1299	.unreach => unreachable,
1300	.dead => unreachable,
1301	.compare_flags_unsigned => unreachable,
1302	.compare_flags_signed => unreachable,
1303	.immediate => \|imm\| try self.setRegOrMem(elem_ty, dst_mcv, .{ .memory = imm }),
1304	.ptr_stack_offset => \|off\| try self.setRegOrMem(elem_ty, dst_mcv, .{ .stack_offset = off }),
1305	.ptr_embedded_in_code => \|off\| {
1306	try self.setRegOrMem(elem_ty, dst_mcv, .{ .embedded_in_code = off });
1307	},
1308	.embedded_in_code => {
1309	return self.fail("TODO implement loading from MCValue.embedded_in_code", .{});
1310	},
1311	.register => {
1312	return self.fail("TODO implement loading from MCValue.register for {}", .{self.target.cpu.arch});
1313	},
1314	.memory => \|addr\| {
1315	const reg = try self.register_manager.allocReg(null, &.{});
1316	try self.genSetReg(ptr_ty, reg, .{ .memory = addr });
1317	try self.load(dst_mcv, .{ .register = reg }, ptr_ty);
1318	},
1319	.stack_offset => {
1320	return self.fail("TODO implement loading from MCValue.stack_offset", .{});
1321	},
1322	}
1323	}
1324
1325	fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
1326	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1327	const elem_ty = self.air.typeOfIndex(inst);
1328	const result: MCValue = result: {
1329	if (!elem_ty.hasCodeGenBits())
1330	break :result MCValue.none;
1331
1332	const ptr = try self.resolveInst(ty_op.operand);
1333	const is_volatile = self.air.typeOf(ty_op.operand).isVolatilePtr();
1334	if (self.liveness.isUnused(inst) and !is_volatile)
1335	break :result MCValue.dead;
1336
1337	const dst_mcv: MCValue = blk: {
1338	if (self.reuseOperand(inst, ty_op.operand, 0, ptr)) {
1339	// The MCValue that holds the pointer can be re-used as the value.
1340	break :blk ptr;
1341	} else {
1342	break :blk try self.allocRegOrMem(inst, true);
1343	}
1344	};
1345	try self.load(dst_mcv, ptr, self.air.typeOf(ty_op.operand));
1346	break :result dst_mcv;
1347	};
1348	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1349	}
1350
1351	fn airStore(self: *Self, inst: Air.Inst.Index) !void {
1352	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1353	const ptr = try self.resolveInst(bin_op.lhs);
1354	const value = try self.resolveInst(bin_op.rhs);
1355	const elem_ty = self.air.typeOf(bin_op.rhs);
1356	switch (ptr) {
1357	.none => unreachable,
1358	.undef => unreachable,
1359	.unreach => unreachable,
1360	.dead => unreachable,
1361	.compare_flags_unsigned => unreachable,
1362	.compare_flags_signed => unreachable,
1363	.immediate => \|imm\| {
1364	try self.setRegOrMem(elem_ty, .{ .memory = imm }, value);
1365	},
1366	.ptr_stack_offset => \|off\| {
1367	try self.genSetStack(elem_ty, off, value);
1368	},
1369	.ptr_embedded_in_code => \|off\| {
1370	try self.setRegOrMem(elem_ty, .{ .embedded_in_code = off }, value);
1371	},
1372	.embedded_in_code => {
1373	return self.fail("TODO implement storing to MCValue.embedded_in_code", .{});
1374	},
1375	.register => {
1376	return self.fail("TODO implement storing to MCValue.register", .{});
1377	},
1378	.memory => {
1379	return self.fail("TODO implement storing to MCValue.memory", .{});
1380	},
1381	.stack_offset => {
1382	return self.fail("TODO implement storing to MCValue.stack_offset", .{});
1383	},
1384	}
1385	return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
1386	}
1387
1388	fn airStructFieldPtr(self: *Self, inst: Air.Inst.Index) !void {
1389	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1390	const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
1391	return self.structFieldPtr(extra.struct_operand, ty_pl.ty, extra.field_index);
1392	}
1393
1394	fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, index: u8) !void {
1395	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1396	return self.structFieldPtr(ty_op.operand, ty_op.ty, index);
1397	}
1398	fn structFieldPtr(self: *Self, operand: Air.Inst.Ref, ty: Air.Inst.Ref, index: u32) !void {
1399	_ = self;
1400	_ = operand;
1401	_ = ty;
1402	_ = index;
1403	return self.fail("TODO implement codegen struct_field_ptr", .{});
1404	//return self.finishAir(inst, result, .{ extra.struct_ptr, .none, .none });
1405	}
1406
1407	fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
1408	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1409	const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
1410	_ = extra;
1411	return self.fail("TODO implement codegen struct_field_val", .{});
1412	//return self.finishAir(inst, result, .{ extra.struct_ptr, .none, .none });
1413	}
1414
1415	/// Perform "binary" operators, excluding comparisons.
1416	/// Currently, the following ops are supported:
1417	/// ADD, SUB, XOR, OR, AND
1418	fn genX8664BinMath(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air.Inst.Ref) !MCValue {
1419	// We'll handle these ops in two steps.
1420	// 1) Prepare an output location (register or memory)
1421	// This location will be the location of the operand that dies (if one exists)
1422	// or just a temporary register (if one doesn't exist)
1423	// 2) Perform the op with the other argument
1424	// 3) Sometimes, the output location is memory but the op doesn't support it.
1425	// In this case, copy that location to a register, then perform the op to that register instead.
1426	//
1427	// TODO: make this algorithm less bad
1428
1429	try self.code.ensureUnusedCapacity(8);
1430
1431	const lhs = try self.resolveInst(op_lhs);
1432	const rhs = try self.resolveInst(op_rhs);
1433
1434	// There are 2 operands, destination and source.
1435	// Either one, but not both, can be a memory operand.
1436	// Source operand can be an immediate, 8 bits or 32 bits.
1437	// So, if either one of the operands dies with this instruction, we can use it
1438	// as the result MCValue.
1439	var dst_mcv: MCValue = undefined;
1440	var src_mcv: MCValue = undefined;
1441	var src_inst: Air.Inst.Ref = undefined;
1442	if (self.reuseOperand(inst, op_lhs, 0, lhs)) {
1443	// LHS dies; use it as the destination.
1444	// Both operands cannot be memory.
1445	src_inst = op_rhs;
1446	if (lhs.isMemory() and rhs.isMemory()) {
1447	dst_mcv = try self.copyToNewRegister(inst, lhs);
1448	src_mcv = rhs;
1449	} else {
1450	dst_mcv = lhs;
1451	src_mcv = rhs;
1452	}
1453	} else if (self.reuseOperand(inst, op_rhs, 1, rhs)) {
1454	// RHS dies; use it as the destination.
1455	// Both operands cannot be memory.
1456	src_inst = op_lhs;
1457	if (lhs.isMemory() and rhs.isMemory()) {
1458	dst_mcv = try self.copyToNewRegister(inst, rhs);
1459	src_mcv = lhs;
1460	} else {
1461	dst_mcv = rhs;
1462	src_mcv = lhs;
1463	}
1464	} else {
1465	if (lhs.isMemory()) {
1466	dst_mcv = try self.copyToNewRegister(inst, lhs);
1467	src_mcv = rhs;
1468	src_inst = op_rhs;
1469	} else {
1470	dst_mcv = try self.copyToNewRegister(inst, rhs);
1471	src_mcv = lhs;
1472	src_inst = op_lhs;
1473	}
1474	}
1475	// This instruction supports only signed 32-bit immediates at most. If the immediate
1476	// value is larger than this, we put it in a register.
1477	// A potential opportunity for future optimization here would be keeping track
1478	// of the fact that the instruction is available both as an immediate
1479	// and as a register.
1480	switch (src_mcv) {
1481	.immediate => \|imm\| {
1482	if (imm > math.maxInt(u31)) {
1483	src_mcv = MCValue{ .register = try self.copyToTmpRegister(Type.initTag(.u64), src_mcv) };
1484	}
1485	},
1486	else => {},
1487	}
1488
1489	// Now for step 2, we perform the actual op
1490	const inst_ty = self.air.typeOfIndex(inst);
1491	const air_tags = self.air.instructions.items(.tag);
1492	switch (air_tags[inst]) {
1493	// TODO: Generate wrapping and non-wrapping versions separately
1494	.add, .addwrap => try self.genX8664BinMathCode(inst_ty, dst_mcv, src_mcv, 0, 0x00),
1495	.bool_or, .bit_or => try self.genX8664BinMathCode(inst_ty, dst_mcv, src_mcv, 1, 0x08),
1496	.bool_and, .bit_and => try self.genX8664BinMathCode(inst_ty, dst_mcv, src_mcv, 4, 0x20),
1497	.sub, .subwrap => try self.genX8664BinMathCode(inst_ty, dst_mcv, src_mcv, 5, 0x28),
1498	.xor, .not => try self.genX8664BinMathCode(inst_ty, dst_mcv, src_mcv, 6, 0x30),
1499
1500	.mul, .mulwrap => try self.genX8664Imul(inst_ty, dst_mcv, src_mcv),
1501	else => unreachable,
1502	}
1503
1504	return dst_mcv;
1505	}
1506
1507	/// Wrap over Instruction.encodeInto to translate errors
1508	fn encodeX8664Instruction(self: *Self, inst: Instruction) !void {
1509	inst.encodeInto(self.code) catch \|err\| {
1510	if (err == error.OutOfMemory)
1511	return error.OutOfMemory
1512	else
1513	return self.fail("Instruction.encodeInto failed because {s}", .{@errorName(err)});
1514	};
1515	}
1516
1517	/// This function encodes a binary operation for x86_64
1518	/// intended for use with the following opcode ranges
1519	/// because they share the same structure.
1520	///
1521	/// Thus not all binary operations can be used here
1522	/// -- multiplication needs to be done with imul,
1523	/// which doesn't have as convenient an interface.
1524	///
1525	/// "opx"-style instructions use the opcode extension field to indicate which instruction to execute:
1526	///
1527	/// opx = /0: add
1528	/// opx = /1: or
1529	/// opx = /2: adc
1530	/// opx = /3: sbb
1531	/// opx = /4: and
1532	/// opx = /5: sub
1533	/// opx = /6: xor
1534	/// opx = /7: cmp
1535	///
1536	/// opcode \| operand shape
1537	/// --------+----------------------
1538	/// 80 /opx \| r/m8, imm8
1539	/// 81 /opx \| r/m16/32/64, imm16/32
1540	/// 83 /opx \| r/m16/32/64, imm8
1541	///
1542	/// "mr"-style instructions use the low bits of opcode to indicate shape of instruction:
1543	///
1544	/// mr = 00: add
1545	/// mr = 08: or
1546	/// mr = 10: adc
1547	/// mr = 18: sbb
1548	/// mr = 20: and
1549	/// mr = 28: sub
1550	/// mr = 30: xor
1551	/// mr = 38: cmp
1552	///
1553	/// opcode \| operand shape
1554	/// -------+-------------------------
1555	/// mr + 0 \| r/m8, r8
1556	/// mr + 1 \| r/m16/32/64, r16/32/64
1557	/// mr + 2 \| r8, r/m8
1558	/// mr + 3 \| r16/32/64, r/m16/32/64
1559	/// mr + 4 \| AL, imm8
1560	/// mr + 5 \| rAX, imm16/32
1561	///
1562	/// TODO: rotates and shifts share the same structure, so we can potentially implement them
1563	/// at a later date with very similar code.
1564	/// They have "opx"-style instructions, but no "mr"-style instructions.
1565	///
1566	/// opx = /0: rol,
1567	/// opx = /1: ror,
1568	/// opx = /2: rcl,
1569	/// opx = /3: rcr,
1570	/// opx = /4: shl sal,
1571	/// opx = /5: shr,
1572	/// opx = /6: sal shl,
1573	/// opx = /7: sar,
1574	///
1575	/// opcode \| operand shape
1576	/// --------+------------------
1577	/// c0 /opx \| r/m8, imm8
1578	/// c1 /opx \| r/m16/32/64, imm8
1579	/// d0 /opx \| r/m8, 1
1580	/// d1 /opx \| r/m16/32/64, 1
1581	/// d2 /opx \| r/m8, CL (for context, CL is register 1)
1582	/// d3 /opx \| r/m16/32/64, CL (for context, CL is register 1)
1583	fn genX8664BinMathCode(
1584	self: *Self,
1585	dst_ty: Type,
1586	dst_mcv: MCValue,
1587	src_mcv: MCValue,
1588	opx: u3,
1589	mr: u8,
1590	) !void {
1591	switch (dst_mcv) {
1592	.none => unreachable,
1593	.undef => unreachable,
1594	.dead, .unreach, .immediate => unreachable,
1595	.compare_flags_unsigned => unreachable,
1596	.compare_flags_signed => unreachable,
1597	.ptr_stack_offset => unreachable,
1598	.ptr_embedded_in_code => unreachable,
1599	.register => \|dst_reg\| {
1600	switch (src_mcv) {
1601	.none => unreachable,
1602	.undef => try self.genSetReg(dst_ty, dst_reg, .undef),
1603	.dead, .unreach => unreachable,
1604	.ptr_stack_offset => unreachable,
1605	.ptr_embedded_in_code => unreachable,
1606	.register => \|src_reg\| {
1607	// for register, register use mr + 1
1608	// addressing mode: r/m16/32/64, r16/32/64
1609	const abi_size = dst_ty.abiSize(self.target.*);
1610	const encoder = try Encoder.init(self.code, 3);
1611	encoder.rex(.{
1612	.w = abi_size == 8,
1613	.r = src_reg.isExtended(),
1614	.b = dst_reg.isExtended(),
1615	});
1616	encoder.opcode_1byte(mr + 1);
1617	encoder.modRm_direct(
1618	src_reg.low_id(),
1619	dst_reg.low_id(),
1620	);
1621	},
1622	.immediate => \|imm\| {
1623	// register, immediate use opx = 81 or 83 addressing modes:
1624	// opx = 81: r/m16/32/64, imm16/32
1625	// opx = 83: r/m16/32/64, imm8
1626	const imm32 = @intCast(i32, imm); // This case must be handled before calling genX8664BinMathCode.
1627	if (imm32 <= math.maxInt(i8)) {
1628	const abi_size = dst_ty.abiSize(self.target.*);
1629	const encoder = try Encoder.init(self.code, 4);
1630	encoder.rex(.{
1631	.w = abi_size == 8,
1632	.b = dst_reg.isExtended(),
1633	});
1634	encoder.opcode_1byte(0x83);
1635	encoder.modRm_direct(
1636	opx,
1637	dst_reg.low_id(),
1638	);
1639	encoder.imm8(@intCast(i8, imm32));
1640	} else {
1641	const abi_size = dst_ty.abiSize(self.target.*);
1642	const encoder = try Encoder.init(self.code, 7);
1643	encoder.rex(.{
1644	.w = abi_size == 8,
1645	.b = dst_reg.isExtended(),
1646	});
1647	encoder.opcode_1byte(0x81);
1648	encoder.modRm_direct(
1649	opx,
1650	dst_reg.low_id(),
1651	);
1652	encoder.imm32(@intCast(i32, imm32));
1653	}
1654	},
1655	.embedded_in_code, .memory => {
1656	return self.fail("TODO implement x86 ADD/SUB/CMP source memory", .{});
1657	},
1658	.stack_offset => \|off\| {
1659	// register, indirect use mr + 3
1660	// addressing mode: r16/32/64, r/m16/32/64
1661	const abi_size = dst_ty.abiSize(self.target.*);
1662	const adj_off = off + abi_size;
1663	if (off > math.maxInt(i32)) {
1664	return self.fail("stack offset too large", .{});
1665	}
1666	const encoder = try Encoder.init(self.code, 7);
1667	encoder.rex(.{
1668	.w = abi_size == 8,
1669	.r = dst_reg.isExtended(),
1670	});
1671	encoder.opcode_1byte(mr + 3);
1672	if (adj_off <= std.math.maxInt(i8)) {
1673	encoder.modRm_indirectDisp8(
1674	dst_reg.low_id(),
1675	Register.ebp.low_id(),
1676	);
1677	encoder.disp8(-@intCast(i8, adj_off));
1678	} else {
1679	encoder.modRm_indirectDisp32(
1680	dst_reg.low_id(),
1681	Register.ebp.low_id(),
1682	);
1683	encoder.disp32(-@intCast(i32, adj_off));
1684	}
1685	},
1686	.compare_flags_unsigned => {
1687	return self.fail("TODO implement x86 ADD/SUB/CMP source compare flag (unsigned)", .{});
1688	},
1689	.compare_flags_signed => {
1690	return self.fail("TODO implement x86 ADD/SUB/CMP source compare flag (signed)", .{});
1691	},
1692	}
1693	},
1694	.stack_offset => \|off\| {
1695	switch (src_mcv) {
1696	.none => unreachable,
1697	.undef => return self.genSetStack(dst_ty, off, .undef),
1698	.dead, .unreach => unreachable,
1699	.ptr_stack_offset => unreachable,
1700	.ptr_embedded_in_code => unreachable,
1701	.register => \|src_reg\| {
1702	try self.genX8664ModRMRegToStack(dst_ty, off, src_reg, mr + 0x1);
1703	},
1704	.immediate => \|imm\| {
1705	_ = imm;
1706	return self.fail("TODO implement x86 ADD/SUB/CMP source immediate", .{});
1707	},
1708	.embedded_in_code, .memory, .stack_offset => {
1709	return self.fail("TODO implement x86 ADD/SUB/CMP source memory", .{});
1710	},
1711	.compare_flags_unsigned => {
1712	return self.fail("TODO implement x86 ADD/SUB/CMP source compare flag (unsigned)", .{});
1713	},
1714	.compare_flags_signed => {
1715	return self.fail("TODO implement x86 ADD/SUB/CMP source compare flag (signed)", .{});
1716	},
1717	}
1718	},
1719	.embedded_in_code, .memory => {
1720	return self.fail("TODO implement x86 ADD/SUB/CMP destination memory", .{});
1721	},
1722	}
1723	}
1724
1725	/// Performs integer multiplication between dst_mcv and src_mcv, storing the result in dst_mcv.
1726	fn genX8664Imul(
1727	self: *Self,
1728	dst_ty: Type,
1729	dst_mcv: MCValue,
1730	src_mcv: MCValue,
1731	) !void {
1732	switch (dst_mcv) {
1733	.none => unreachable,
1734	.undef => unreachable,
1735	.dead, .unreach, .immediate => unreachable,
1736	.compare_flags_unsigned => unreachable,
1737	.compare_flags_signed => unreachable,
1738	.ptr_stack_offset => unreachable,
1739	.ptr_embedded_in_code => unreachable,
1740	.register => \|dst_reg\| {
1741	switch (src_mcv) {
1742	.none => unreachable,
1743	.undef => try self.genSetReg(dst_ty, dst_reg, .undef),
1744	.dead, .unreach => unreachable,
1745	.ptr_stack_offset => unreachable,
1746	.ptr_embedded_in_code => unreachable,
1747	.register => \|src_reg\| {
1748	// register, register
1749	//
1750	// Use the following imul opcode
1751	// 0F AF /r: IMUL r32/64, r/m32/64
1752	const abi_size = dst_ty.abiSize(self.target.*);
1753	const encoder = try Encoder.init(self.code, 4);
1754	encoder.rex(.{
1755	.w = abi_size == 8,
1756	.r = dst_reg.isExtended(),
1757	.b = src_reg.isExtended(),
1758	});
1759	encoder.opcode_2byte(0x0f, 0xaf);
1760	encoder.modRm_direct(
1761	dst_reg.low_id(),
1762	src_reg.low_id(),
1763	);
1764	},
1765	.immediate => \|imm\| {
1766	// register, immediate:
1767	// depends on size of immediate.
1768	//
1769	// immediate fits in i8:
1770	// 6B /r ib: IMUL r32/64, r/m32/64, imm8
1771	//
1772	// immediate fits in i32:
1773	// 69 /r id: IMUL r32/64, r/m32/64, imm32
1774	//
1775	// immediate is huge:
1776	// split into 2 instructions
1777	// 1) copy the 64 bit immediate into a tmp register
1778	// 2) perform register,register mul
1779	// 0F AF /r: IMUL r32/64, r/m32/64
1780	if (math.minInt(i8) <= imm and imm <= math.maxInt(i8)) {
1781	const abi_size = dst_ty.abiSize(self.target.*);
1782	const encoder = try Encoder.init(self.code, 4);
1783	encoder.rex(.{
1784	.w = abi_size == 8,
1785	.r = dst_reg.isExtended(),
1786	.b = dst_reg.isExtended(),
1787	});
1788	encoder.opcode_1byte(0x6B);
1789	encoder.modRm_direct(
1790	dst_reg.low_id(),
1791	dst_reg.low_id(),
1792	);
1793	encoder.imm8(@intCast(i8, imm));
1794	} else if (math.minInt(i32) <= imm and imm <= math.maxInt(i32)) {
1795	const abi_size = dst_ty.abiSize(self.target.*);
1796	const encoder = try Encoder.init(self.code, 7);
1797	encoder.rex(.{
1798	.w = abi_size == 8,
1799	.r = dst_reg.isExtended(),
1800	.b = dst_reg.isExtended(),
1801	});
1802	encoder.opcode_1byte(0x69);
1803	encoder.modRm_direct(
1804	dst_reg.low_id(),
1805	dst_reg.low_id(),
1806	);
1807	encoder.imm32(@intCast(i32, imm));
1808	} else {
1809	const src_reg = try self.copyToTmpRegister(dst_ty, src_mcv);
1810	return self.genX8664Imul(dst_ty, dst_mcv, MCValue{ .register = src_reg });
1811	}
1812	},
1813	.embedded_in_code, .memory, .stack_offset => {
1814	return self.fail("TODO implement x86 multiply source memory", .{});
1815	},
1816	.compare_flags_unsigned => {
1817	return self.fail("TODO implement x86 multiply source compare flag (unsigned)", .{});
1818	},
1819	.compare_flags_signed => {
1820	return self.fail("TODO implement x86 multiply source compare flag (signed)", .{});
1821	},
1822	}
1823	},
1824	.stack_offset => \|off\| {
1825	switch (src_mcv) {
1826	.none => unreachable,
1827	.undef => return self.genSetStack(dst_ty, off, .undef),
1828	.dead, .unreach => unreachable,
1829	.ptr_stack_offset => unreachable,
1830	.ptr_embedded_in_code => unreachable,
1831	.register => \|src_reg\| {
1832	// copy dst to a register
1833	const dst_reg = try self.copyToTmpRegister(dst_ty, dst_mcv);
1834	// multiply into dst_reg
1835	// register, register
1836	// Use the following imul opcode
1837	// 0F AF /r: IMUL r32/64, r/m32/64
1838	const abi_size = dst_ty.abiSize(self.target.*);
1839	const encoder = try Encoder.init(self.code, 4);
1840	encoder.rex(.{
1841	.w = abi_size == 8,
1842	.r = dst_reg.isExtended(),
1843	.b = src_reg.isExtended(),
1844	});
1845	encoder.opcode_2byte(0x0f, 0xaf);
1846	encoder.modRm_direct(
1847	dst_reg.low_id(),
1848	src_reg.low_id(),
1849	);
1850	// copy dst_reg back out
1851	return self.genSetStack(dst_ty, off, MCValue{ .register = dst_reg });
1852	},
1853	.immediate => \|imm\| {
1854	_ = imm;
1855	return self.fail("TODO implement x86 multiply source immediate", .{});
1856	},
1857	.embedded_in_code, .memory, .stack_offset => {
1858	return self.fail("TODO implement x86 multiply source memory", .{});
1859	},
1860	.compare_flags_unsigned => {
1861	return self.fail("TODO implement x86 multiply source compare flag (unsigned)", .{});
1862	},
1863	.compare_flags_signed => {
1864	return self.fail("TODO implement x86 multiply source compare flag (signed)", .{});
1865	},
1866	}
1867	},
1868	.embedded_in_code, .memory => {
1869	return self.fail("TODO implement x86 multiply destination memory", .{});
1870	},
1871	}
1872	}
1873
1874	fn genX8664ModRMRegToStack(self: *Self, ty: Type, off: u32, reg: Register, opcode: u8) !void {
1875	const abi_size = ty.abiSize(self.target.*);
1876	const adj_off = off + abi_size;
1877	if (off > math.maxInt(i32)) {
1878	return self.fail("stack offset too large", .{});
1879	}
1880
1881	const i_adj_off = -@intCast(i32, adj_off);
1882	const encoder = try Encoder.init(self.code, 7);
1883	encoder.rex(.{
1884	.w = abi_size == 8,
1885	.r = reg.isExtended(),
1886	});
1887	encoder.opcode_1byte(opcode);
1888	if (i_adj_off < std.math.maxInt(i8)) {
1889	// example: 48 89 55 7f mov QWORD PTR [rbp+0x7f],rdx
1890	encoder.modRm_indirectDisp8(
1891	reg.low_id(),
1892	Register.ebp.low_id(),
1893	);
1894	encoder.disp8(@intCast(i8, i_adj_off));
1895	} else {
1896	// example: 48 89 95 80 00 00 00 mov QWORD PTR [rbp+0x80],rdx
1897	encoder.modRm_indirectDisp32(
1898	reg.low_id(),
1899	Register.ebp.low_id(),
1900	);
1901	encoder.disp32(i_adj_off);
1902	}
1903	}
1904
1905	fn genArgDbgInfo(self: *Self, inst: Air.Inst.Index, mcv: MCValue) !void {
1906	const ty_str = self.air.instructions.items(.data)[inst].ty_str;
1907	const zir = &self.mod_fn.owner_decl.getFileScope().zir;
1908	const name = zir.nullTerminatedString(ty_str.str);
1909	const name_with_null = name.ptr[0 .. name.len + 1];
1910	const ty = self.air.getRefType(ty_str.ty);
1911
1912	switch (mcv) {
1913	.register => \|reg\| {
1914	switch (self.debug_output) {
1915	.dwarf => \|dbg_out\| {
1916	try dbg_out.dbg_info.ensureUnusedCapacity(3);
1917	dbg_out.dbg_info.appendAssumeCapacity(link.File.Elf.abbrev_parameter);
1918	dbg_out.dbg_info.appendSliceAssumeCapacity(&[2]u8{ // DW.AT.location, DW.FORM.exprloc
1919	1, // ULEB128 dwarf expression length
1920	reg.dwarfLocOp(),
1921	});
1922	try dbg_out.dbg_info.ensureUnusedCapacity(5 + name_with_null.len);
1923	try self.addDbgInfoTypeReloc(ty); // DW.AT.type, DW.FORM.ref4
1924	dbg_out.dbg_info.appendSliceAssumeCapacity(name_with_null); // DW.AT.name, DW.FORM.string
1925	},
1926	.plan9 => {},
1927	.none => {},
1928	}
1929	},
1930	.stack_offset => {
1931	switch (self.debug_output) {
1932	.dwarf => {},
1933	.plan9 => {},
1934	.none => {},
1935	}
1936	},
1937	else => {},
1938	}
1939	}
1940
1941	fn airArg(self: *Self, inst: Air.Inst.Index) !void {
1942	const arg_index = self.arg_index;
1943	self.arg_index += 1;
1944
1945	const ty = self.air.typeOfIndex(inst);
1946	_ = ty;
1947
1948	const mcv = self.args[arg_index];
1949	try self.genArgDbgInfo(inst, mcv);
1950
1951	if (self.liveness.isUnused(inst))
1952	return self.finishAirBookkeeping();
1953
1954	switch (mcv) {
1955	.register => \|reg\| {
1956	self.register_manager.getRegAssumeFree(reg.to64(), inst);
1957	},
1958	else => {},
1959	}
1960
1961	return self.finishAir(inst, mcv, .{ .none, .none, .none });
1962	}
1963
1964	fn airBreakpoint(self: *Self) !void {
1965	try self.code.append(0xcc); // int3
1966	return self.finishAirBookkeeping();
1967	}
1968
1969	fn airFence(self: *Self) !void {
1970	return self.fail("TODO implement fence() for {}", .{self.target.cpu.arch});
1971	//return self.finishAirBookkeeping();
1972	}
1973
1974	fn airCall(self: *Self, inst: Air.Inst.Index) !void {
1975	const pl_op = self.air.instructions.items(.data)[inst].pl_op;
1976	const fn_ty = self.air.typeOf(pl_op.operand);
1977	const callee = pl_op.operand;
1978	const extra = self.air.extraData(Air.Call, pl_op.payload);
1979	const args = @bitCast([]const Air.Inst.Ref, self.air.extra[extra.end..][0..extra.data.args_len]);
1980
1981	var info = try self.resolveCallingConventionValues(fn_ty);
1982	defer info.deinit(self);
1983
1984	// Due to incremental compilation, how function calls are generated depends
1985	// on linking.
1986	if (self.bin_file.tag == link.File.Elf.base_tag or self.bin_file.tag == link.File.Coff.base_tag) {
1987	for (info.args) \|mc_arg, arg_i\| {
1988	const arg = args[arg_i];
1989	const arg_ty = self.air.typeOf(arg);
1990	const arg_mcv = try self.resolveInst(args[arg_i]);
1991	// Here we do not use setRegOrMem even though the logic is similar, because
1992	// the function call will move the stack pointer, so the offsets are different.
1993	switch (mc_arg) {
1994	.none => continue,
1995	.register => \|reg\| {
1996	try self.register_manager.getReg(reg, null);
1997	try self.genSetReg(arg_ty, reg, arg_mcv);
1998	},
1999	.stack_offset => \|off\| {
2000	// Here we need to emit instructions like this:
2001	// mov qword ptr [rsp + stack_offset], x
2002	try self.genSetStack(arg_ty, off, arg_mcv);
2003	},
2004	.ptr_stack_offset => {
2005	return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{});
2006	},
2007	.ptr_embedded_in_code => {
2008	return self.fail("TODO implement calling with MCValue.ptr_embedded_in_code arg", .{});
2009	},
2010	.undef => unreachable,
2011	.immediate => unreachable,
2012	.unreach => unreachable,
2013	.dead => unreachable,
2014	.embedded_in_code => unreachable,
2015	.memory => unreachable,
2016	.compare_flags_signed => unreachable,
2017	.compare_flags_unsigned => unreachable,
2018	}
2019	}
2020
2021	if (self.air.value(callee)) \|func_value\| {
2022	if (func_value.castTag(.function)) \|func_payload\| {
2023	const func = func_payload.data;
2024
2025	const ptr_bits = self.target.cpu.arch.ptrBitWidth();
2026	const ptr_bytes: u64 = @divExact(ptr_bits, 8);
2027	const got_addr = if (self.bin_file.cast(link.File.Elf)) \|elf_file\| blk: {
2028	const got = &elf_file.program_headers.items[elf_file.phdr_got_index.?];
2029	break :blk @intCast(u32, got.p_vaddr + func.owner_decl.link.elf.offset_table_index * ptr_bytes);
2030	} else if (self.bin_file.cast(link.File.Coff)) \|coff_file\|
2031	@intCast(u32, coff_file.offset_table_virtual_address + func.owner_decl.link.coff.offset_table_index * ptr_bytes)
2032	else
2033	unreachable;
2034
2035	// ff 14 25 xx xx xx xx call [addr]
2036	try self.code.ensureUnusedCapacity(7);
2037	self.code.appendSliceAssumeCapacity(&[3]u8{ 0xff, 0x14, 0x25 });
2038	mem.writeIntLittle(u32, self.code.addManyAsArrayAssumeCapacity(4), got_addr);
2039	} else if (func_value.castTag(.extern_fn)) \|_\| {
2040	return self.fail("TODO implement calling extern functions", .{});
2041	} else {
2042	return self.fail("TODO implement calling bitcasted functions", .{});
2043	}
2044	} else {
2045	return self.fail("TODO implement calling runtime known function pointer", .{});
2046	}
2047	} else if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {
2048	for (info.args) \|mc_arg, arg_i\| {
2049	const arg = args[arg_i];
2050	const arg_ty = self.air.typeOf(arg);
2051	const arg_mcv = try self.resolveInst(args[arg_i]);
2052	// Here we do not use setRegOrMem even though the logic is similar, because
2053	// the function call will move the stack pointer, so the offsets are different.
2054	switch (mc_arg) {
2055	.none => continue,
2056	.register => \|reg\| {
2057	// TODO prevent this macho if block to be generated for all archs
2058	try self.register_manager.getReg(reg, null);
2059	try self.genSetReg(arg_ty, reg, arg_mcv);
2060	},
2061	.stack_offset => {
2062	// Here we need to emit instructions like this:
2063	// mov qword ptr [rsp + stack_offset], x
2064	return self.fail("TODO implement calling with parameters in memory", .{});
2065	},
2066	.ptr_stack_offset => {
2067	return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{});
2068	},
2069	.ptr_embedded_in_code => {
2070	return self.fail("TODO implement calling with MCValue.ptr_embedded_in_code arg", .{});
2071	},
2072	.undef => unreachable,
2073	.immediate => unreachable,
2074	.unreach => unreachable,
2075	.dead => unreachable,
2076	.embedded_in_code => unreachable,
2077	.memory => unreachable,
2078	.compare_flags_signed => unreachable,
2079	.compare_flags_unsigned => unreachable,
2080	}
2081	}
2082
2083	if (self.air.value(callee)) \|func_value\| {
2084	if (func_value.castTag(.function)) \|func_payload\| {
2085	const func = func_payload.data;
2086	// TODO I'm hacking my way through here by repurposing .memory for storing
2087	// index to the GOT target symbol index.
2088	try self.genSetReg(Type.initTag(.u64), .rax, .{
2089	.memory = func.owner_decl.link.macho.local_sym_index,
2090	});
2091	// callq *%rax
2092	try self.code.ensureUnusedCapacity(2);
2093	self.code.appendSliceAssumeCapacity(&[2]u8{ 0xff, 0xd0 });
2094	} else if (func_value.castTag(.extern_fn)) \|func_payload\| {
2095	const decl = func_payload.data;
2096	const n_strx = try macho_file.addExternFn(mem.spanZ(decl.name));
2097	const offset = blk: {
2098	// callq
2099	try self.code.ensureUnusedCapacity(5);
2100	self.code.appendSliceAssumeCapacity(&[5]u8{ 0xe8, 0x0, 0x0, 0x0, 0x0 });
2101	break :blk @intCast(u32, self.code.items.len) - 4;
2102	};
2103	// Add relocation to the decl.
2104	try macho_file.active_decl.?.link.macho.relocs.append(self.bin_file.allocator, .{
2105	.offset = offset,
2106	.target = .{ .global = n_strx },
2107	.addend = 0,
2108	.subtractor = null,
2109	.pcrel = true,
2110	.length = 2,
2111	.@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_BRANCH),
2112	});
2113	} else {
2114	return self.fail("TODO implement calling bitcasted functions", .{});
2115	}
2116	} else {
2117	return self.fail("TODO implement calling runtime known function pointer", .{});
2118	}
2119	} else if (self.bin_file.cast(link.File.Plan9)) \|p9\| {
2120	for (info.args) \|mc_arg, arg_i\| {
2121	const arg = args[arg_i];
2122	const arg_ty = self.air.typeOf(arg);
2123	const arg_mcv = try self.resolveInst(args[arg_i]);
2124	// Here we do not use setRegOrMem even though the logic is similar, because
2125	// the function call will move the stack pointer, so the offsets are different.
2126	switch (mc_arg) {
2127	.none => continue,
2128	.register => \|reg\| {
2129	try self.register_manager.getReg(reg, null);
2130	try self.genSetReg(arg_ty, reg, arg_mcv);
2131	},
2132	.stack_offset => {
2133	// Here we need to emit instructions like this:
2134	// mov qword ptr [rsp + stack_offset], x
2135	return self.fail("TODO implement calling with parameters in memory", .{});
2136	},
2137	.ptr_stack_offset => {
2138	return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{});
2139	},
2140	.ptr_embedded_in_code => {
2141	return self.fail("TODO implement calling with MCValue.ptr_embedded_in_code arg", .{});
2142	},
2143	.undef => unreachable,
2144	.immediate => unreachable,
2145	.unreach => unreachable,
2146	.dead => unreachable,
2147	.embedded_in_code => unreachable,
2148	.memory => unreachable,
2149	.compare_flags_signed => unreachable,
2150	.compare_flags_unsigned => unreachable,
2151	}
2152	}
2153	if (self.air.value(callee)) \|func_value\| {
2154	if (func_value.castTag(.function)) \|func_payload\| {
2155	try p9.seeDecl(func_payload.data.owner_decl);
2156	const ptr_bits = self.target.cpu.arch.ptrBitWidth();
2157	const ptr_bytes: u64 = @divExact(ptr_bits, 8);
2158	const got_addr = p9.bases.data;
2159	const got_index = func_payload.data.owner_decl.link.plan9.got_index.?;
2160	// ff 14 25 xx xx xx xx call [addr]
2161	try self.code.ensureUnusedCapacity(7);
2162	self.code.appendSliceAssumeCapacity(&[3]u8{ 0xff, 0x14, 0x25 });
2163	const fn_got_addr = got_addr + got_index * ptr_bytes;
2164	mem.writeIntLittle(u32, self.code.addManyAsArrayAssumeCapacity(4), @intCast(u32, fn_got_addr));
2165	} else return self.fail("TODO implement calling extern fn on plan9", .{});
2166	} else {
2167	return self.fail("TODO implement calling runtime known function pointer", .{});
2168	}
2169	} else unreachable;
2170
2171	const result: MCValue = result: {
2172	switch (info.return_value) {
2173	.register => \|reg\| {
2174	if (Register.allocIndex(reg) == null) {
2175	// Save function return value in a callee saved register
2176	break :result try self.copyToNewRegister(inst, info.return_value);
2177	}
2178	},
2179	else => {},
2180	}
2181	break :result info.return_value;
2182	};
2183
2184	if (args.len <= Liveness.bpi - 2) {
2185	var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
2186	buf[0] = callee;
2187	std.mem.copy(Air.Inst.Ref, buf[1..], args);
2188	return self.finishAir(inst, result, buf);
2189	}
2190	var bt = try self.iterateBigTomb(inst, 1 + args.len);
2191	bt.feed(callee);
2192	for (args) \|arg\| {
2193	bt.feed(arg);
2194	}
2195	return bt.finishAir(result);
2196	}
2197
2198	fn ret(self: *Self, mcv: MCValue) !void {
2199	const ret_ty = self.fn_type.fnReturnType();
2200	try self.setRegOrMem(ret_ty, self.ret_mcv, mcv);
2201	// TODO when implementing defer, this will need to jump to the appropriate defer expression.
2202	// TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
2203	// which is available if the jump is 127 bytes or less forward.
2204	try self.code.resize(self.code.items.len + 5);
2205	self.code.items[self.code.items.len - 5] = 0xe9; // jmp rel32
2206	try self.exitlude_jump_relocs.append(self.gpa, self.code.items.len - 4);
2207	}
2208
2209	fn airRet(self: *Self, inst: Air.Inst.Index) !void {
2210	const un_op = self.air.instructions.items(.data)[inst].un_op;
2211	const operand = try self.resolveInst(un_op);
2212	try self.ret(operand);
2213	return self.finishAir(inst, .dead, .{ un_op, .none, .none });
2214	}
2215
2216	fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
2217	const un_op = self.air.instructions.items(.data)[inst].un_op;
2218	const ptr = try self.resolveInst(un_op);
2219	_ = ptr;
2220	return self.fail("TODO implement airRetLoad for {}", .{self.target.cpu.arch});
2221	//return self.finishAir(inst, .dead, .{ un_op, .none, .none });
2222	}
2223
2224	fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
2225	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
2226	if (self.liveness.isUnused(inst))
2227	return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
2228	const ty = self.air.typeOf(bin_op.lhs);
2229	assert(ty.eql(self.air.typeOf(bin_op.rhs)));
2230	if (ty.zigTypeTag() == .ErrorSet)
2231	return self.fail("TODO implement cmp for errors", .{});
2232
2233	const lhs = try self.resolveInst(bin_op.lhs);
2234	const rhs = try self.resolveInst(bin_op.rhs);
2235	const result: MCValue = result: {
2236	try self.code.ensureUnusedCapacity(8);
2237
2238	// There are 2 operands, destination and source.
2239	// Either one, but not both, can be a memory operand.
2240	// Source operand can be an immediate, 8 bits or 32 bits.
2241	const dst_mcv = if (lhs.isImmediate() or (lhs.isMemory() and rhs.isMemory()))
2242	try self.copyToNewRegister(inst, lhs)
2243	else
2244	lhs;
2245	// This instruction supports only signed 32-bit immediates at most.
2246	const src_mcv = try self.limitImmediateType(bin_op.rhs, i32);
2247
2248	try self.genX8664BinMathCode(Type.initTag(.bool), dst_mcv, src_mcv, 7, 0x38);
2249	break :result switch (ty.isSignedInt()) {
2250	true => MCValue{ .compare_flags_signed = op },
2251	false => MCValue{ .compare_flags_unsigned = op },
2252	};
2253	};
2254	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2255	}
2256
2257	fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
2258	const dbg_stmt = self.air.instructions.items(.data)[inst].dbg_stmt;
2259	try self.dbgAdvancePCAndLine(dbg_stmt.line, dbg_stmt.column);
2260	return self.finishAirBookkeeping();
2261	}
2262
2263	fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
2264	const pl_op = self.air.instructions.items(.data)[inst].pl_op;
2265	const cond = try self.resolveInst(pl_op.operand);
2266	const extra = self.air.extraData(Air.CondBr, pl_op.payload);
2267	const then_body = self.air.extra[extra.end..][0..extra.data.then_body_len];
2268	const else_body = self.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len];
2269	const liveness_condbr = self.liveness.getCondBr(inst);
2270
2271	const reloc: Reloc = reloc: {
2272	try self.code.ensureUnusedCapacity(6);
2273
2274	const opcode: u8 = switch (cond) {
2275	.compare_flags_signed => \|cmp_op\| blk: {
2276	// Here we map to the opposite opcode because the jump is to the false branch.
2277	const opcode: u8 = switch (cmp_op) {
2278	.gte => 0x8c,
2279	.gt => 0x8e,
2280	.neq => 0x84,
2281	.lt => 0x8d,
2282	.lte => 0x8f,
2283	.eq => 0x85,
2284	};
2285	break :blk opcode;
2286	},
2287	.compare_flags_unsigned => \|cmp_op\| blk: {
2288	// Here we map to the opposite opcode because the jump is to the false branch.
2289	const opcode: u8 = switch (cmp_op) {
2290	.gte => 0x82,
2291	.gt => 0x86,
2292	.neq => 0x84,
2293	.lt => 0x83,
2294	.lte => 0x87,
2295	.eq => 0x85,
2296	};
2297	break :blk opcode;
2298	},
2299	.register => \|reg\| blk: {
2300	// test reg, 1
2301	// TODO detect al, ax, eax
2302	const encoder = try Encoder.init(self.code, 4);
2303	encoder.rex(.{
2304	// TODO audit this codegen: we force w = true here to make
2305	// the value affect the big register
2306	.w = true,
2307	.b = reg.isExtended(),
2308	});
2309	encoder.opcode_1byte(0xf6);
2310	encoder.modRm_direct(
2311	0,
2312	reg.low_id(),
2313	);
2314	encoder.disp8(1);
2315	break :blk 0x84;
2316	},
2317	else => return self.fail("TODO implement condbr {s} when condition is {s}", .{ self.target.cpu.arch, @tagName(cond) }),
2318	};
2319	self.code.appendSliceAssumeCapacity(&[_]u8{ 0x0f, opcode });
2320	const reloc = Reloc{ .rel32 = self.code.items.len };
2321	self.code.items.len += 4;
2322	break :reloc reloc;
2323	};
2324
2325	// Capture the state of register and stack allocation state so that we can revert to it.
2326	const parent_next_stack_offset = self.next_stack_offset;
2327	const parent_free_registers = self.register_manager.free_registers;
2328	var parent_stack = try self.stack.clone(self.gpa);
2329	defer parent_stack.deinit(self.gpa);
2330	const parent_registers = self.register_manager.registers;
2331
2332	try self.branch_stack.append(.{});
2333
2334	try self.ensureProcessDeathCapacity(liveness_condbr.then_deaths.len);
2335	for (liveness_condbr.then_deaths) \|operand\| {
2336	self.processDeath(operand);
2337	}
2338	try self.genBody(then_body);
2339
2340	// Revert to the previous register and stack allocation state.
2341
2342	var saved_then_branch = self.branch_stack.pop();
2343	defer saved_then_branch.deinit(self.gpa);
2344
2345	self.register_manager.registers = parent_registers;
2346
2347	self.stack.deinit(self.gpa);
2348	self.stack = parent_stack;
2349	parent_stack = .{};
2350
2351	self.next_stack_offset = parent_next_stack_offset;
2352	self.register_manager.free_registers = parent_free_registers;
2353
2354	try self.performReloc(reloc);
2355	const else_branch = self.branch_stack.addOneAssumeCapacity();
2356	else_branch.* = .{};
2357
2358	try self.ensureProcessDeathCapacity(liveness_condbr.else_deaths.len);
2359	for (liveness_condbr.else_deaths) \|operand\| {
2360	self.processDeath(operand);
2361	}
2362	try self.genBody(else_body);
2363
2364	// At this point, each branch will possibly have conflicting values for where
2365	// each instruction is stored. They agree, however, on which instructions are alive/dead.
2366	// We use the first ("then") branch as canonical, and here emit
2367	// instructions into the second ("else") branch to make it conform.
2368	// We continue respect the data structure semantic guarantees of the else_branch so
2369	// that we can use all the code emitting abstractions. This is why at the bottom we
2370	// assert that parent_branch.free_registers equals the saved_then_branch.free_registers
2371	// rather than assigning it.
2372	const parent_branch = &self.branch_stack.items[self.branch_stack.items.len - 2];
2373	try parent_branch.inst_table.ensureUnusedCapacity(self.gpa, else_branch.inst_table.count());
2374
2375	const else_slice = else_branch.inst_table.entries.slice();
2376	const else_keys = else_slice.items(.key);
2377	const else_values = else_slice.items(.value);
2378	for (else_keys) \|else_key, else_idx\| {
2379	const else_value = else_values[else_idx];
2380	const canon_mcv = if (saved_then_branch.inst_table.fetchSwapRemove(else_key)) \|then_entry\| blk: {
2381	// The instruction's MCValue is overridden in both branches.
2382	parent_branch.inst_table.putAssumeCapacity(else_key, then_entry.value);
2383	if (else_value == .dead) {
2384	assert(then_entry.value == .dead);
2385	continue;
2386	}
2387	break :blk then_entry.value;
2388	} else blk: {
2389	if (else_value == .dead)
2390	continue;
2391	// The instruction is only overridden in the else branch.
2392	var i: usize = self.branch_stack.items.len - 2;
2393	while (true) {
2394	i -= 1; // If this overflows, the question is: why wasn't the instruction marked dead?
2395	if (self.branch_stack.items[i].inst_table.get(else_key)) \|mcv\| {
2396	assert(mcv != .dead);
2397	break :blk mcv;
2398	}
2399	}
2400	};
2401	log.debug("consolidating else_entry {d} {}=>{}", .{ else_key, else_value, canon_mcv });
2402	// TODO make sure the destination stack offset / register does not already have something
2403	// going on there.
2404	try self.setRegOrMem(self.air.typeOfIndex(else_key), canon_mcv, else_value);
2405	// TODO track the new register / stack allocation
2406	}
2407	try parent_branch.inst_table.ensureUnusedCapacity(self.gpa, saved_then_branch.inst_table.count());
2408	const then_slice = saved_then_branch.inst_table.entries.slice();
2409	const then_keys = then_slice.items(.key);
2410	const then_values = then_slice.items(.value);
2411	for (then_keys) \|then_key, then_idx\| {
2412	const then_value = then_values[then_idx];
2413	// We already deleted the items from this table that matched the else_branch.
2414	// So these are all instructions that are only overridden in the then branch.
2415	parent_branch.inst_table.putAssumeCapacity(then_key, then_value);
2416	if (then_value == .dead)
2417	continue;
2418	const parent_mcv = blk: {
2419	var i: usize = self.branch_stack.items.len - 2;
2420	while (true) {
2421	i -= 1;
2422	if (self.branch_stack.items[i].inst_table.get(then_key)) \|mcv\| {
2423	assert(mcv != .dead);
2424	break :blk mcv;
2425	}
2426	}
2427	};
2428	log.debug("consolidating then_entry {d} {}=>{}", .{ then_key, parent_mcv, then_value });
2429	// TODO make sure the destination stack offset / register does not already have something
2430	// going on there.
2431	try self.setRegOrMem(self.air.typeOfIndex(then_key), parent_mcv, then_value);
2432	// TODO track the new register / stack allocation
2433	}
2434
2435	self.branch_stack.pop().deinit(self.gpa);
2436
2437	return self.finishAir(inst, .unreach, .{ pl_op.operand, .none, .none });
2438	}
2439
2440	fn isNull(self: *Self, operand: MCValue) !MCValue {
2441	_ = operand;
2442	// Here you can specialize this instruction if it makes sense to, otherwise the default
2443	// will call isNonNull and invert the result.
2444	return self.fail("TODO call isNonNull and invert the result", .{});
2445	}
2446
2447	fn isNonNull(self: *Self, operand: MCValue) !MCValue {
2448	_ = operand;
2449	// Here you can specialize this instruction if it makes sense to, otherwise the default
2450	// will call isNull and invert the result.
2451	return self.fail("TODO call isNull and invert the result", .{});
2452	}
2453
2454	fn isErr(self: *Self, operand: MCValue) !MCValue {
2455	_ = operand;
2456	// Here you can specialize this instruction if it makes sense to, otherwise the default
2457	// will call isNonNull and invert the result.
2458	return self.fail("TODO call isNonErr and invert the result", .{});
2459	}
2460
2461	fn isNonErr(self: *Self, operand: MCValue) !MCValue {
2462	_ = operand;
2463	// Here you can specialize this instruction if it makes sense to, otherwise the default
2464	// will call isNull and invert the result.
2465	return self.fail("TODO call isErr and invert the result", .{});
2466	}
2467
2468	fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {
2469	const un_op = self.air.instructions.items(.data)[inst].un_op;
2470	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2471	const operand = try self.resolveInst(un_op);
2472	break :result try self.isNull(operand);
2473	};
2474	return self.finishAir(inst, result, .{ un_op, .none, .none });
2475	}
2476
2477	fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {
2478	const un_op = self.air.instructions.items(.data)[inst].un_op;
2479	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2480	const operand_ptr = try self.resolveInst(un_op);
2481	const operand: MCValue = blk: {
2482	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
2483	// The MCValue that holds the pointer can be re-used as the value.
2484	break :blk operand_ptr;
2485	} else {
2486	break :blk try self.allocRegOrMem(inst, true);
2487	}
2488	};
2489	try self.load(operand, operand_ptr, self.air.typeOf(un_op));
2490	break :result try self.isNull(operand);
2491	};
2492	return self.finishAir(inst, result, .{ un_op, .none, .none });
2493	}
2494
2495	fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {
2496	const un_op = self.air.instructions.items(.data)[inst].un_op;
2497	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2498	const operand = try self.resolveInst(un_op);
2499	break :result try self.isNonNull(operand);
2500	};
2501	return self.finishAir(inst, result, .{ un_op, .none, .none });
2502	}
2503
2504	fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {
2505	const un_op = self.air.instructions.items(.data)[inst].un_op;
2506	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2507	const operand_ptr = try self.resolveInst(un_op);
2508	const operand: MCValue = blk: {
2509	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
2510	// The MCValue that holds the pointer can be re-used as the value.
2511	break :blk operand_ptr;
2512	} else {
2513	break :blk try self.allocRegOrMem(inst, true);
2514	}
2515	};
2516	try self.load(operand, operand_ptr, self.air.typeOf(un_op));
2517	break :result try self.isNonNull(operand);
2518	};
2519	return self.finishAir(inst, result, .{ un_op, .none, .none });
2520	}
2521
2522	fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {
2523	const un_op = self.air.instructions.items(.data)[inst].un_op;
2524	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2525	const operand = try self.resolveInst(un_op);
2526	break :result try self.isErr(operand);
2527	};
2528	return self.finishAir(inst, result, .{ un_op, .none, .none });
2529	}
2530
2531	fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {
2532	const un_op = self.air.instructions.items(.data)[inst].un_op;
2533	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2534	const operand_ptr = try self.resolveInst(un_op);
2535	const operand: MCValue = blk: {
2536	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
2537	// The MCValue that holds the pointer can be re-used as the value.
2538	break :blk operand_ptr;
2539	} else {
2540	break :blk try self.allocRegOrMem(inst, true);
2541	}
2542	};
2543	try self.load(operand, operand_ptr, self.air.typeOf(un_op));
2544	break :result try self.isErr(operand);
2545	};
2546	return self.finishAir(inst, result, .{ un_op, .none, .none });
2547	}
2548
2549	fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void {
2550	const un_op = self.air.instructions.items(.data)[inst].un_op;
2551	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2552	const operand = try self.resolveInst(un_op);
2553	break :result try self.isNonErr(operand);
2554	};
2555	return self.finishAir(inst, result, .{ un_op, .none, .none });
2556	}
2557
2558	fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {
2559	const un_op = self.air.instructions.items(.data)[inst].un_op;
2560	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2561	const operand_ptr = try self.resolveInst(un_op);
2562	const operand: MCValue = blk: {
2563	if (self.reuseOperand(inst, un_op, 0, operand_ptr)) {
2564	// The MCValue that holds the pointer can be re-used as the value.
2565	break :blk operand_ptr;
2566	} else {
2567	break :blk try self.allocRegOrMem(inst, true);
2568	}
2569	};
2570	try self.load(operand, operand_ptr, self.air.typeOf(un_op));
2571	break :result try self.isNonErr(operand);
2572	};
2573	return self.finishAir(inst, result, .{ un_op, .none, .none });
2574	}
2575
2576	fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
2577	// A loop is a setup to be able to jump back to the beginning.
2578	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
2579	const loop = self.air.extraData(Air.Block, ty_pl.payload);
2580	const body = self.air.extra[loop.end..][0..loop.data.body_len];
2581	const start_index = self.code.items.len;
2582	try self.genBody(body);
2583	try self.jump(start_index);
2584	return self.finishAirBookkeeping();
2585	}
2586
2587	/// Send control flow to the `index` of `self.code`.
2588	fn jump(self: *Self, index: usize) !void {
2589	try self.code.ensureUnusedCapacity(5);
2590	if (math.cast(i8, @intCast(i32, index) - (@intCast(i32, self.code.items.len + 2)))) \|delta\| {
2591	self.code.appendAssumeCapacity(0xeb); // jmp rel8
2592	self.code.appendAssumeCapacity(@bitCast(u8, delta));
2593	} else \|_\| {
2594	const delta = @intCast(i32, index) - (@intCast(i32, self.code.items.len + 5));
2595	self.code.appendAssumeCapacity(0xe9); // jmp rel32
2596	mem.writeIntLittle(i32, self.code.addManyAsArrayAssumeCapacity(4), delta);
2597	}
2598	}
2599
2600	fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
2601	try self.blocks.putNoClobber(self.gpa, inst, .{
2602	// A block is a setup to be able to jump to the end.
2603	.relocs = .{},
2604	// It also acts as a receptacle for break operands.
2605	// Here we use `MCValue.none` to represent a null value so that the first
2606	// break instruction will choose a MCValue for the block result and overwrite
2607	// this field. Following break instructions will use that MCValue to put their
2608	// block results.
2609	.mcv = MCValue{ .none = {} },
2610	});
2611	const block_data = self.blocks.getPtr(inst).?;
2612	defer block_data.relocs.deinit(self.gpa);
2613
2614	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
2615	const extra = self.air.extraData(Air.Block, ty_pl.payload);
2616	const body = self.air.extra[extra.end..][0..extra.data.body_len];
2617	try self.genBody(body);
2618
2619	for (block_data.relocs.items) \|reloc\| try self.performReloc(reloc);
2620
2621	const result = @bitCast(MCValue, block_data.mcv);
2622	return self.finishAir(inst, result, .{ .none, .none, .none });
2623	}
2624
2625	fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
2626	const pl_op = self.air.instructions.items(.data)[inst].pl_op;
2627	const condition = pl_op.operand;
2628	_ = condition;
2629	return self.fail("TODO airSwitch for {}", .{self.target.cpu.arch});
2630	// return self.finishAir(inst, .dead, .{ condition, .none, .none });
2631	}
2632
2633	fn performReloc(self: *Self, reloc: Reloc) !void {
2634	switch (reloc) {
2635	.rel32 => \|pos\| {
2636	const amt = self.code.items.len - (pos + 4);
2637	// Here it would be tempting to implement testing for amt == 0 and then elide the
2638	// jump. However, that will cause a problem because other jumps may assume that they
2639	// can jump to this code. Or maybe I didn't understand something when I was debugging.
2640	// It could be worth another look. Anyway, that's why that isn't done here. Probably the
2641	// best place to elide jumps will be in semantic analysis, by inlining blocks that only
2642	// only have 1 break instruction.
2643	const s32_amt = math.cast(i32, amt) catch
2644	return self.fail("unable to perform relocation: jump too far", .{});
2645	mem.writeIntLittle(i32, self.code.items[pos..][0..4], s32_amt);
2646	},
2647	.arm_branch => unreachable,
2648	}
2649	}
2650
2651	fn airBr(self: *Self, inst: Air.Inst.Index) !void {
2652	const branch = self.air.instructions.items(.data)[inst].br;
2653	try self.br(branch.block_inst, branch.operand);
2654	return self.finishAir(inst, .dead, .{ branch.operand, .none, .none });
2655	}
2656
2657	fn airBoolOp(self: *Self, inst: Air.Inst.Index) !void {
2658	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
2659	const air_tags = self.air.instructions.items(.tag);
2660	const result: MCValue = if (self.liveness.isUnused(inst))
2661	.dead
2662	else switch (air_tags[inst]) {
2663	// lhs AND rhs
2664	.bool_and => try self.genX8664BinMath(inst, bin_op.lhs, bin_op.rhs),
2665	// lhs OR rhs
2666	.bool_or => try self.genX8664BinMath(inst, bin_op.lhs, bin_op.rhs),
2667	else => unreachable, // Not a boolean operation
2668	};
2669	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2670	}
2671
2672	fn br(self: *Self, block: Air.Inst.Index, operand: Air.Inst.Ref) !void {
2673	const block_data = self.blocks.getPtr(block).?;
2674
2675	if (self.air.typeOf(operand).hasCodeGenBits()) {
2676	const operand_mcv = try self.resolveInst(operand);
2677	const block_mcv = block_data.mcv;
2678	if (block_mcv == .none) {
2679	block_data.mcv = operand_mcv;
2680	} else {
2681	try self.setRegOrMem(self.air.typeOfIndex(block), block_mcv, operand_mcv);
2682	}
2683	}
2684	return self.brVoid(block);
2685	}
2686
2687	fn brVoid(self: *Self, block: Air.Inst.Index) !void {
2688	const block_data = self.blocks.getPtr(block).?;
2689	// Emit a jump with a relocation. It will be patched up after the block ends.
2690	try block_data.relocs.ensureUnusedCapacity(self.gpa, 1);
2691	// TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
2692	// which is available if the jump is 127 bytes or less forward.
2693	try self.code.resize(self.code.items.len + 5);
2694	self.code.items[self.code.items.len - 5] = 0xe9; // jmp rel32
2695	// Leave the jump offset undefined
2696	block_data.relocs.appendAssumeCapacity(.{ .rel32 = self.code.items.len - 4 });
2697	}
2698
2699	fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
2700	const air_datas = self.air.instructions.items(.data);
2701	const air_extra = self.air.extraData(Air.Asm, air_datas[inst].ty_pl.payload);
2702	const zir = self.mod_fn.owner_decl.getFileScope().zir;
2703	const extended = zir.instructions.items(.data)[air_extra.data.zir_index].extended;
2704	const zir_extra = zir.extraData(Zir.Inst.Asm, extended.operand);
2705	const asm_source = zir.nullTerminatedString(zir_extra.data.asm_source);
2706	const outputs_len = @truncate(u5, extended.small);
2707	const args_len = @truncate(u5, extended.small >> 5);
2708	const clobbers_len = @truncate(u5, extended.small >> 10);
2709	_ = clobbers_len; // TODO honor these
2710	const is_volatile = @truncate(u1, extended.small >> 15) != 0;
2711	const outputs = @bitCast([]const Air.Inst.Ref, self.air.extra[air_extra.end..][0..outputs_len]);
2712	const args = @bitCast([]const Air.Inst.Ref, self.air.extra[air_extra.end + outputs.len ..][0..args_len]);
2713
2714	if (outputs_len > 1) {
2715	return self.fail("TODO implement codegen for asm with more than 1 output", .{});
2716	}
2717	var extra_i: usize = zir_extra.end;
2718	const output_constraint: ?[]const u8 = out: {
2719	var i: usize = 0;
2720	while (i < outputs_len) : (i += 1) {
2721	const output = zir.extraData(Zir.Inst.Asm.Output, extra_i);
2722	extra_i = output.end;
2723	break :out zir.nullTerminatedString(output.data.constraint);
2724	}
2725	break :out null;
2726	};
2727
2728	const dead = !is_volatile and self.liveness.isUnused(inst);
2729	const result: MCValue = if (dead)
2730	.dead
2731	else result: {
2732	for (args) \|arg\| {
2733	const input = zir.extraData(Zir.Inst.Asm.Input, extra_i);
2734	extra_i = input.end;
2735	const constraint = zir.nullTerminatedString(input.data.constraint);
2736
2737	if (constraint.len < 3 or constraint[0] != '{' or constraint[constraint.len - 1] != '}') {
2738	return self.fail("unrecognized asm input constraint: '{s}'", .{constraint});
2739	}
2740	const reg_name = constraint[1 .. constraint.len - 1];
2741	const reg = parseRegName(reg_name) orelse
2742	return self.fail("unrecognized register: '{s}'", .{reg_name});
2743
2744	const arg_mcv = try self.resolveInst(arg);
2745	try self.register_manager.getReg(reg, null);
2746	try self.genSetReg(self.air.typeOf(arg), reg, arg_mcv);
2747	}
2748
2749	{
2750	var iter = std.mem.tokenize(u8, asm_source, "\n\r");
2751	while (iter.next()) \|ins\| {
2752	if (mem.eql(u8, ins, "syscall")) {
2753	try self.code.appendSlice(&[_]u8{ 0x0f, 0x05 });
2754	} else if (mem.indexOf(u8, ins, "push")) \|_\| {
2755	const arg = ins[4..];
2756	if (mem.indexOf(u8, arg, "$")) \|l\| {
2757	const n = std.fmt.parseInt(u8, ins[4 + l + 1 ..], 10) catch return self.fail("TODO implement more inline asm int parsing", .{});
2758	try self.code.appendSlice(&.{ 0x6a, n });
2759	} else if (mem.indexOf(u8, arg, "%%")) \|l\| {
2760	const reg_name = ins[4 + l + 2 ..];
2761	const reg = parseRegName(reg_name) orelse
2762	return self.fail("unrecognized register: '{s}'", .{reg_name});
2763	const low_id: u8 = reg.low_id();
2764	if (reg.isExtended()) {
2765	try self.code.appendSlice(&.{ 0x41, 0b1010000 \| low_id });
2766	} else {
2767	try self.code.append(0b1010000 \| low_id);
2768	}
2769	} else return self.fail("TODO more push operands", .{});
2770	} else if (mem.indexOf(u8, ins, "pop")) \|_\| {
2771	const arg = ins[3..];
2772	if (mem.indexOf(u8, arg, "%%")) \|l\| {
2773	const reg_name = ins[3 + l + 2 ..];
2774	const reg = parseRegName(reg_name) orelse
2775	return self.fail("unrecognized register: '{s}'", .{reg_name});
2776	const low_id: u8 = reg.low_id();
2777	if (reg.isExtended()) {
2778	try self.code.appendSlice(&.{ 0x41, 0b1011000 \| low_id });
2779	} else {
2780	try self.code.append(0b1011000 \| low_id);
2781	}
2782	} else return self.fail("TODO more pop operands", .{});
2783	} else {
2784	return self.fail("TODO implement support for more x86 assembly instructions", .{});
2785	}
2786	}
2787	}
2788
2789	if (output_constraint) \|output\| {
2790	if (output.len < 4 or output[0] != '=' or output[1] != '{' or output[output.len - 1] != '}') {
2791	return self.fail("unrecognized asm output constraint: '{s}'", .{output});
2792	}
2793	const reg_name = output[2 .. output.len - 1];
2794	const reg = parseRegName(reg_name) orelse
2795	return self.fail("unrecognized register: '{s}'", .{reg_name});
2796	break :result MCValue{ .register = reg };
2797	} else {
2798	break :result MCValue{ .none = {} };
2799	}
2800	};
2801	if (outputs.len + args.len <= Liveness.bpi - 1) {
2802	var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
2803	std.mem.copy(Air.Inst.Ref, &buf, outputs);
2804	std.mem.copy(Air.Inst.Ref, buf[outputs.len..], args);
2805	return self.finishAir(inst, result, buf);
2806	}
2807	var bt = try self.iterateBigTomb(inst, outputs.len + args.len);
2808	for (outputs) \|output\| {
2809	bt.feed(output);
2810	}
2811	for (args) \|arg\| {
2812	bt.feed(arg);
2813	}
2814	return bt.finishAir(result);
2815	}
2816
2817	fn iterateBigTomb(self: *Self, inst: Air.Inst.Index, operand_count: usize) !BigTomb {
2818	try self.ensureProcessDeathCapacity(operand_count + 1);
2819	return BigTomb{
2820	.function = self,
2821	.inst = inst,
2822	.tomb_bits = self.liveness.getTombBits(inst),
2823	.big_tomb_bits = self.liveness.special.get(inst) orelse 0,
2824	.bit_index = 0,
2825	};
2826	}
2827
2828	/// Sets the value without any modifications to register allocation metadata or stack allocation metadata.
2829	fn setRegOrMem(self: *Self, ty: Type, loc: MCValue, val: MCValue) !void {
2830	switch (loc) {
2831	.none => return,
2832	.register => \|reg\| return self.genSetReg(ty, reg, val),
2833	.stack_offset => \|off\| return self.genSetStack(ty, off, val),
2834	.memory => {
2835	return self.fail("TODO implement setRegOrMem for memory", .{});
2836	},
2837	else => unreachable,
2838	}
2839	}
2840
2841	fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerError!void {
2842	switch (mcv) {
2843	.dead => unreachable,
2844	.ptr_stack_offset => unreachable,
2845	.ptr_embedded_in_code => unreachable,
2846	.unreach, .none => return, // Nothing to do.
2847	.undef => {
2848	if (!self.wantSafety())
2849	return; // The already existing value will do just fine.
2850	// TODO Upgrade this to a memset call when we have that available.
2851	switch (ty.abiSize(self.target.*)) {
2852	1 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaa }),
2853	2 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaa }),
2854	4 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaa }),
2855	8 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaaaaaaaaaa }),
2856	else => return self.fail("TODO implement memset", .{}),
2857	}
2858	},
2859	.compare_flags_unsigned => \|op\| {
2860	_ = op;
2861	return self.fail("TODO implement set stack variable with compare flags value (unsigned)", .{});
2862	},
2863	.compare_flags_signed => \|op\| {
2864	_ = op;
2865	return self.fail("TODO implement set stack variable with compare flags value (signed)", .{});
2866	},
2867	.immediate => \|x_big\| {
2868	const abi_size = ty.abiSize(self.target.*);
2869	const adj_off = stack_offset + abi_size;
2870	if (adj_off > 128) {
2871	return self.fail("TODO implement set stack variable with large stack offset", .{});
2872	}
2873	try self.code.ensureUnusedCapacity(8);
2874	switch (abi_size) {
2875	1 => {
2876	return self.fail("TODO implement set abi_size=1 stack variable with immediate", .{});
2877	},
2878	2 => {
2879	return self.fail("TODO implement set abi_size=2 stack variable with immediate", .{});
2880	},
2881	4 => {
2882	const x = @intCast(u32, x_big);
2883	// We have a positive stack offset value but we want a twos complement negative
2884	// offset from rbp, which is at the top of the stack frame.
2885	const negative_offset = @intCast(i8, -@intCast(i32, adj_off));
2886	const twos_comp = @bitCast(u8, negative_offset);
2887	// mov DWORD PTR [rbp+offset], immediate
2888	self.code.appendSliceAssumeCapacity(&[_]u8{ 0xc7, 0x45, twos_comp });
2889	mem.writeIntLittle(u32, self.code.addManyAsArrayAssumeCapacity(4), x);
2890	},
2891	8 => {
2892	// We have a positive stack offset value but we want a twos complement negative
2893	// offset from rbp, which is at the top of the stack frame.
2894	const negative_offset = @intCast(i8, -@intCast(i32, adj_off));
2895	const twos_comp = @bitCast(u8, negative_offset);
2896
2897	// 64 bit write to memory would take two mov's anyways so we
2898	// insted just use two 32 bit writes to avoid register allocation
2899	try self.code.ensureUnusedCapacity(14);
2900	var buf: [8]u8 = undefined;
2901	mem.writeIntLittle(u64, &buf, x_big);
2902
2903	// mov DWORD PTR [rbp+offset+4], immediate
2904	self.code.appendSliceAssumeCapacity(&[_]u8{ 0xc7, 0x45, twos_comp + 4 });
2905	self.code.appendSliceAssumeCapacity(buf[4..8]);
2906
2907	// mov DWORD PTR [rbp+offset], immediate
2908	self.code.appendSliceAssumeCapacity(&[_]u8{ 0xc7, 0x45, twos_comp });
2909	self.code.appendSliceAssumeCapacity(buf[0..4]);
2910	},
2911	else => {
2912	return self.fail("TODO implement set abi_size=large stack variable with immediate", .{});
2913	},
2914	}
2915	},
2916	.embedded_in_code => {
2917	// TODO this and `.stack_offset` below need to get improved to support types greater than
2918	// register size, and do general memcpy
2919	const reg = try self.copyToTmpRegister(ty, mcv);
2920	return self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
2921	},
2922	.register => \|reg\| {
2923	try self.genX8664ModRMRegToStack(ty, stack_offset, reg, 0x89);
2924	},
2925	.memory => \|vaddr\| {
2926	_ = vaddr;
2927	return self.fail("TODO implement set stack variable from memory vaddr", .{});
2928	},
2929	.stack_offset => \|off\| {
2930	// TODO this and `.embedded_in_code` above need to get improved to support types greater than
2931	// register size, and do general memcpy
2932
2933	if (stack_offset == off)
2934	return; // Copy stack variable to itself; nothing to do.
2935
2936	const reg = try self.copyToTmpRegister(ty, mcv);
2937	return self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
2938	},
2939	}
2940	}
2941
2942	fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void {
2943	switch (mcv) {
2944	.dead => unreachable,
2945	.ptr_stack_offset => unreachable,
2946	.ptr_embedded_in_code => unreachable,
2947	.unreach, .none => return, // Nothing to do.
2948	.undef => {
2949	if (!self.wantSafety())
2950	return; // The already existing value will do just fine.
2951	// Write the debug undefined value.
2952	switch (reg.size()) {
2953	8 => return self.genSetReg(ty, reg, .{ .immediate = 0xaa }),
2954	16 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaa }),
2955	32 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaa }),
2956	64 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaaaaaaaaaa }),
2957	else => unreachable,
2958	}
2959	},
2960	.compare_flags_unsigned => \|op\| {
2961	const encoder = try Encoder.init(self.code, 7);
2962	// TODO audit this codegen: we force w = true here to make
2963	// the value affect the big register
2964	encoder.rex(.{
2965	.w = true,
2966	.b = reg.isExtended(),
2967	});
2968	encoder.opcode_2byte(0x0f, switch (op) {
2969	.gte => 0x93,
2970	.gt => 0x97,
2971	.neq => 0x95,
2972	.lt => 0x92,
2973	.lte => 0x96,
2974	.eq => 0x94,
2975	});
2976	encoder.modRm_direct(
2977	0,
2978	reg.low_id(),
2979	);
2980	},
2981	.compare_flags_signed => \|op\| {
2982	_ = op;
2983	return self.fail("TODO set register with compare flags value (signed)", .{});
2984	},
2985	.immediate => \|x\| {
2986	// 32-bit moves zero-extend to 64-bit, so xoring the 32-bit
2987	// register is the fastest way to zero a register.
2988	if (x == 0) {
2989	// The encoding for `xor r32, r32` is `0x31 /r`.
2990	const encoder = try Encoder.init(self.code, 3);
2991
2992	// If we're accessing e.g. r8d, we need to use a REX prefix before the actual operation. Since
2993	// this is a 32-bit operation, the W flag is set to zero. X is also zero, as we're not using a SIB.
2994	// Both R and B are set, as we're extending, in effect, the register bits and the operand.
2995	encoder.rex(.{
2996	.r = reg.isExtended(),
2997	.b = reg.isExtended(),
2998	});
2999	encoder.opcode_1byte(0x31);
3000	// Section 3.1.1.1 of the Intel x64 Manual states that "/r indicates that the
3001	// ModR/M byte of the instruction contains a register operand and an r/m operand."
3002	encoder.modRm_direct(
3003	reg.low_id(),
3004	reg.low_id(),
3005	);
3006
3007	return;
3008	}
3009	if (x <= math.maxInt(i32)) {
3010	// Next best case: if we set the lower four bytes, the upper four will be zeroed.
3011	//
3012	// The encoding for `mov IMM32 -> REG` is (0xB8 + R) IMM.
3013
3014	const encoder = try Encoder.init(self.code, 6);
3015	// Just as with XORing, we need a REX prefix. This time though, we only
3016	// need the B bit set, as we're extending the opcode's register field,
3017	// and there is no Mod R/M byte.
3018	encoder.rex(.{
3019	.b = reg.isExtended(),
3020	});
3021	encoder.opcode_withReg(0xB8, reg.low_id());
3022
3023	// no ModR/M byte
3024
3025	// IMM
3026	encoder.imm32(@intCast(i32, x));
3027	return;
3028	}
3029	// Worst case: we need to load the 64-bit register with the IMM. GNU's assemblers calls
3030	// this `movabs`, though this is officially just a different variant of the plain `mov`
3031	// instruction.
3032	//
3033	// This encoding is, in fact, the same as the one used for 32-bit loads. The only
3034	// difference is that we set REX.W before the instruction, which extends the load to
3035	// 64-bit and uses the full bit-width of the register.
3036	{
3037	const encoder = try Encoder.init(self.code, 10);
3038	encoder.rex(.{
3039	.w = true,
3040	.b = reg.isExtended(),
3041	});
3042	encoder.opcode_withReg(0xB8, reg.low_id());
3043	encoder.imm64(x);
3044	}
3045	},
3046	.embedded_in_code => \|code_offset\| {
3047	// We need the offset from RIP in a signed i32 twos complement.
3048	// The instruction is 7 bytes long and RIP points to the next instruction.
3049
3050	// 64-bit LEA is encoded as REX.W 8D /r.
3051	const rip = self.code.items.len + 7;
3052	const big_offset = @intCast(i64, code_offset) - @intCast(i64, rip);
3053	const offset = @intCast(i32, big_offset);
3054	const encoder = try Encoder.init(self.code, 7);
3055
3056	// byte 1, always exists because w = true
3057	encoder.rex(.{
3058	.w = true,
3059	.r = reg.isExtended(),
3060	});
3061	// byte 2
3062	encoder.opcode_1byte(0x8D);
3063	// byte 3
3064	encoder.modRm_RIPDisp32(reg.low_id());
3065	// byte 4-7
3066	encoder.disp32(offset);
3067
3068	// Double check that we haven't done any math errors
3069	assert(rip == self.code.items.len);
3070	},
3071	.register => \|src_reg\| {
3072	// If the registers are the same, nothing to do.
3073	if (src_reg.id() == reg.id())
3074	return;
3075
3076	// This is a variant of 8B /r.
3077	const abi_size = ty.abiSize(self.target.*);
3078	const encoder = try Encoder.init(self.code, 3);
3079	encoder.rex(.{
3080	.w = abi_size == 8,
3081	.r = reg.isExtended(),
3082	.b = src_reg.isExtended(),
3083	});
3084	encoder.opcode_1byte(0x8B);
3085	encoder.modRm_direct(reg.low_id(), src_reg.low_id());
3086	},
3087	.memory => \|x\| {
3088	if (self.bin_file.options.pie) {
3089	// RIP-relative displacement to the entry in the GOT table.
3090	const abi_size = ty.abiSize(self.target.*);
3091	const encoder = try Encoder.init(self.code, 10);
3092
3093	// LEA reg, [<offset>]
3094
3095	// We encode the instruction FIRST because prefixes may or may not appear.
3096	// After we encode the instruction, we will know that the displacement bytes
3097	// for [<offset>] will be at self.code.items.len - 4.
3098	encoder.rex(.{
3099	.w = true, // force 64 bit because loading an address (to the GOT)
3100	.r = reg.isExtended(),
3101	});
3102	encoder.opcode_1byte(0x8D);
3103	encoder.modRm_RIPDisp32(reg.low_id());
3104	encoder.disp32(0);
3105
3106	const offset = @intCast(u32, self.code.items.len);
3107
3108	if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {
3109	// TODO I think the reloc might be in the wrong place.
3110	const decl = macho_file.active_decl.?;
3111	// Load reloc for LEA instruction.
3112	try decl.link.macho.relocs.append(self.bin_file.allocator, .{
3113	.offset = offset - 4,
3114	.target = .{ .local = @intCast(u32, x) },
3115	.addend = 0,
3116	.subtractor = null,
3117	.pcrel = true,
3118	.length = 2,
3119	.@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_GOT),
3120	});
3121	} else {
3122	return self.fail("TODO implement genSetReg for PIE GOT indirection on this platform", .{});
3123	}
3124
3125	// MOV reg, [reg]
3126	encoder.rex(.{
3127	.w = abi_size == 8,
3128	.r = reg.isExtended(),
3129	.b = reg.isExtended(),
3130	});
3131	encoder.opcode_1byte(0x8B);
3132	encoder.modRm_indirectDisp0(reg.low_id(), reg.low_id());
3133	} else if (x <= math.maxInt(i32)) {
3134	// Moving from memory to a register is a variant of `8B /r`.
3135	// Since we're using 64-bit moves, we require a REX.
3136	// This variant also requires a SIB, as it would otherwise be RIP-relative.
3137	// We want mode zero with the lower three bits set to four to indicate an SIB with no other displacement.
3138	// The SIB must be 0x25, to indicate a disp32 with no scaled index.
3139	// 0b00RRR100, where RRR is the lower three bits of the register ID.
3140	// The instruction is thus eight bytes; REX 0x8B 0b00RRR100 0x25 followed by a four-byte disp32.
3141	const abi_size = ty.abiSize(self.target.*);
3142	const encoder = try Encoder.init(self.code, 8);
3143	encoder.rex(.{
3144	.w = abi_size == 8,
3145	.r = reg.isExtended(),
3146	});
3147	encoder.opcode_1byte(0x8B);
3148	// effective address = [SIB]
3149	encoder.modRm_SIBDisp0(reg.low_id());
3150	// SIB = disp32
3151	encoder.sib_disp32();
3152	encoder.disp32(@intCast(i32, x));
3153	} else {
3154	// If this is RAX, we can use a direct load; otherwise, we need to load the address, then indirectly load
3155	// the value.
3156	if (reg.id() == 0) {
3157	// REX.W 0xA1 moffs64*
3158	// moffs64* is a 64-bit offset "relative to segment base", which really just means the
3159	// absolute address for all practical purposes.
3160
3161	const encoder = try Encoder.init(self.code, 10);
3162	encoder.rex(.{
3163	.w = true,
3164	});
3165	encoder.opcode_1byte(0xA1);
3166	encoder.writeIntLittle(u64, x);
3167	} else {
3168	// This requires two instructions; a move imm as used above, followed by an indirect load using the register
3169	// as the address and the register as the destination.
3170	//
3171	// This cannot be used if the lower three bits of the id are equal to four or five, as there
3172	// is no way to possibly encode it. This means that RSP, RBP, R12, and R13 cannot be used with
3173	// this instruction.
3174	const id3 = @truncate(u3, reg.id());
3175	assert(id3 != 4 and id3 != 5);
3176
3177	// Rather than duplicate the logic used for the move, we just use a self-call with a new MCValue.
3178	try self.genSetReg(ty, reg, MCValue{ .immediate = x });
3179
3180	// Now, the register contains the address of the value to load into it
3181	// Currently, we're only allowing 64-bit registers, so we need the `REX.W 8B /r` variant.
3182	// TODO: determine whether to allow other sized registers, and if so, handle them properly.
3183
3184	// mov reg, [reg]
3185	const abi_size = ty.abiSize(self.target.*);
3186	const encoder = try Encoder.init(self.code, 3);
3187	encoder.rex(.{
3188	.w = abi_size == 8,
3189	.r = reg.isExtended(),
3190	.b = reg.isExtended(),
3191	});
3192	encoder.opcode_1byte(0x8B);
3193	encoder.modRm_indirectDisp0(reg.low_id(), reg.low_id());
3194	}
3195	}
3196	},
3197	.stack_offset => \|unadjusted_off\| {
3198	const abi_size = ty.abiSize(self.target.*);
3199	const off = unadjusted_off + abi_size;
3200	if (off < std.math.minInt(i32) or off > std.math.maxInt(i32)) {
3201	return self.fail("stack offset too large", .{});
3202	}
3203	const ioff = -@intCast(i32, off);
3204	const encoder = try Encoder.init(self.code, 3);
3205	encoder.rex(.{
3206	.w = abi_size == 8,
3207	.r = reg.isExtended(),
3208	});
3209	encoder.opcode_1byte(0x8B);
3210	if (std.math.minInt(i8) <= ioff and ioff <= std.math.maxInt(i8)) {
3211	// Example: 48 8b 4d 7f mov rcx,QWORD PTR [rbp+0x7f]
3212	encoder.modRm_indirectDisp8(reg.low_id(), Register.ebp.low_id());
3213	encoder.disp8(@intCast(i8, ioff));
3214	} else {
3215	// Example: 48 8b 8d 80 00 00 00 mov rcx,QWORD PTR [rbp+0x80]
3216	encoder.modRm_indirectDisp32(reg.low_id(), Register.ebp.low_id());
3217	encoder.disp32(ioff);
3218	}
3219	},
3220	}
3221	}
3222
3223	fn airPtrToInt(self: *Self, inst: Air.Inst.Index) !void {
3224	const un_op = self.air.instructions.items(.data)[inst].un_op;
3225	const result = try self.resolveInst(un_op);
3226	return self.finishAir(inst, result, .{ un_op, .none, .none });
3227	}
3228
3229	fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
3230	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
3231	const result = try self.resolveInst(ty_op.operand);
3232	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3233	}
3234
3235	fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
3236	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
3237	const result: MCValue = if (self.liveness.isUnused(inst))
3238	.dead
3239	else
3240	return self.fail("TODO implement airArrayToSlice for {}", .{self.target.cpu.arch});
3241	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3242	}
3243
3244	fn airIntToFloat(self: *Self, inst: Air.Inst.Index) !void {
3245	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
3246	const result: MCValue = if (self.liveness.isUnused(inst))
3247	.dead
3248	else
3249	return self.fail("TODO implement airIntToFloat for {}", .{self.target.cpu.arch});
3250	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3251	}
3252
3253	fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
3254	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
3255	const result: MCValue = if (self.liveness.isUnused(inst))
3256	.dead
3257	else
3258	return self.fail("TODO implement airFloatToInt for {}", .{self.target.cpu.arch});
3259	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3260	}
3261
3262	fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
3263	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
3264	const extra = self.air.extraData(Air.Block, ty_pl.payload);
3265	_ = ty_pl;
3266	_ = extra;
3267	return self.fail("TODO implement airCmpxchg for {}", .{self.target.cpu.arch});
3268	// return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });
3269	}
3270
3271	fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void {
3272	_ = inst;
3273	return self.fail("TODO implement airCmpxchg for {}", .{self.target.cpu.arch});
3274	}
3275
3276	fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {
3277	_ = inst;
3278	return self.fail("TODO implement airAtomicLoad for {}", .{self.target.cpu.arch});
3279	}
3280
3281	fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void {
3282	_ = inst;
3283	_ = order;
3284	return self.fail("TODO implement airAtomicStore for {}", .{self.target.cpu.arch});
3285	}
3286
3287	fn airMemset(self: *Self, inst: Air.Inst.Index) !void {
3288	_ = inst;
3289	return self.fail("TODO implement airMemset for {}", .{self.target.cpu.arch});
3290	}
3291
3292	fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
3293	_ = inst;
3294	return self.fail("TODO implement airMemcpy for {}", .{self.target.cpu.arch});
3295	}
3296
3297	fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {
3298	// First section of indexes correspond to a set number of constant values.
3299	const ref_int = @enumToInt(inst);
3300	if (ref_int < Air.Inst.Ref.typed_value_map.len) {
3301	const tv = Air.Inst.Ref.typed_value_map[ref_int];
3302	if (!tv.ty.hasCodeGenBits()) {
3303	return MCValue{ .none = {} };
3304	}
3305	return self.genTypedValue(tv);
3306	}
3307
3308	// If the type has no codegen bits, no need to store it.
3309	const inst_ty = self.air.typeOf(inst);
3310	if (!inst_ty.hasCodeGenBits())
3311	return MCValue{ .none = {} };
3312
3313	const inst_index = @intCast(Air.Inst.Index, ref_int - Air.Inst.Ref.typed_value_map.len);
3314	switch (self.air.instructions.items(.tag)[inst_index]) {
3315	.constant => {
3316	// Constants have static lifetimes, so they are always memoized in the outer most table.
3317	const branch = &self.branch_stack.items[0];
3318	const gop = try branch.inst_table.getOrPut(self.gpa, inst_index);
3319	if (!gop.found_existing) {
3320	const ty_pl = self.air.instructions.items(.data)[inst_index].ty_pl;
3321	gop.value_ptr.* = try self.genTypedValue(.{
3322	.ty = inst_ty,
3323	.val = self.air.values[ty_pl.payload],
3324	});
3325	}
3326	return gop.value_ptr.*;
3327	},
3328	.const_ty => unreachable,
3329	else => return self.getResolvedInstValue(inst_index),
3330	}
3331	}
3332
3333	fn getResolvedInstValue(self: *Self, inst: Air.Inst.Index) MCValue {
3334	// Treat each stack item as a "layer" on top of the previous one.
3335	var i: usize = self.branch_stack.items.len;
3336	while (true) {
3337	i -= 1;
3338	if (self.branch_stack.items[i].inst_table.get(inst)) \|mcv\| {
3339	assert(mcv != .dead);
3340	return mcv;
3341	}
3342	}
3343	}
3344
3345	/// If the MCValue is an immediate, and it does not fit within this type,
3346	/// we put it in a register.
3347	/// A potential opportunity for future optimization here would be keeping track
3348	/// of the fact that the instruction is available both as an immediate
3349	/// and as a register.
3350	fn limitImmediateType(self: *Self, operand: Air.Inst.Ref, comptime T: type) !MCValue {
3351	const mcv = try self.resolveInst(operand);
3352	const ti = @typeInfo(T).Int;
3353	switch (mcv) {
3354	.immediate => \|imm\| {
3355	// This immediate is unsigned.
3356	const U = std.meta.Int(.unsigned, ti.bits - @boolToInt(ti.signedness == .signed));
3357	if (imm >= math.maxInt(U)) {
3358	return MCValue{ .register = try self.copyToTmpRegister(Type.initTag(.usize), mcv) };
3359	}
3360	},
3361	else => {},
3362	}
3363	return mcv;
3364	}
3365
3366	fn genTypedValue(self: *Self, typed_value: TypedValue) InnerError!MCValue {
3367	if (typed_value.val.isUndef())
3368	return MCValue{ .undef = {} };
3369	const ptr_bits = self.target.cpu.arch.ptrBitWidth();
3370	const ptr_bytes: u64 = @divExact(ptr_bits, 8);
3371	switch (typed_value.ty.zigTypeTag()) {
3372	.Pointer => switch (typed_value.ty.ptrSize()) {
3373	.Slice => {
3374	var buf: Type.SlicePtrFieldTypeBuffer = undefined;
3375	const ptr_type = typed_value.ty.slicePtrFieldType(&buf);
3376	const ptr_mcv = try self.genTypedValue(.{ .ty = ptr_type, .val = typed_value.val });
3377	const slice_len = typed_value.val.sliceLen();
3378	// Codegen can't handle some kinds of indirection. If the wrong union field is accessed here it may mean
3379	// the Sema code needs to use anonymous Decls or alloca instructions to store data.
3380	const ptr_imm = ptr_mcv.memory;
3381	_ = slice_len;
3382	_ = ptr_imm;
3383	// We need more general support for const data being stored in memory to make this work.
3384	return self.fail("TODO codegen for const slices", .{});
3385	},
3386	else => {
3387	if (typed_value.val.castTag(.decl_ref)) \|payload\| {
3388	const decl = payload.data;
3389	decl.alive = true;
3390	if (self.bin_file.cast(link.File.Elf)) \|elf_file\| {
3391	const got = &elf_file.program_headers.items[elf_file.phdr_got_index.?];
3392	const got_addr = got.p_vaddr + decl.link.elf.offset_table_index * ptr_bytes;
3393	return MCValue{ .memory = got_addr };
3394	} else if (self.bin_file.cast(link.File.MachO)) \|_\| {
3395	// TODO I'm hacking my way through here by repurposing .memory for storing
3396	// index to the GOT target symbol index.
3397	return MCValue{ .memory = decl.link.macho.local_sym_index };
3398	} else if (self.bin_file.cast(link.File.Coff)) \|coff_file\| {
3399	const got_addr = coff_file.offset_table_virtual_address + decl.link.coff.offset_table_index * ptr_bytes;
3400	return MCValue{ .memory = got_addr };
3401	} else if (self.bin_file.cast(link.File.Plan9)) \|p9\| {
3402	try p9.seeDecl(decl);
3403	const got_addr = p9.bases.data + decl.link.plan9.got_index.? * ptr_bytes;
3404	return MCValue{ .memory = got_addr };
3405	} else {
3406	return self.fail("TODO codegen non-ELF const Decl pointer", .{});
3407	}
3408	}
3409	if (typed_value.val.tag() == .int_u64) {
3410	return MCValue{ .immediate = typed_value.val.toUnsignedInt() };
3411	}
3412	return self.fail("TODO codegen more kinds of const pointers", .{});
3413	},
3414	},
3415	.Int => {
3416	const info = typed_value.ty.intInfo(self.target.*);
3417	if (info.bits > ptr_bits or info.signedness == .signed) {
3418	return self.fail("TODO const int bigger than ptr and signed int", .{});
3419	}
3420	return MCValue{ .immediate = typed_value.val.toUnsignedInt() };
3421	},
3422	.Bool => {
3423	return MCValue{ .immediate = @boolToInt(typed_value.val.toBool()) };
3424	},
3425	.ComptimeInt => unreachable, // semantic analysis prevents this
3426	.ComptimeFloat => unreachable, // semantic analysis prevents this
3427	.Optional => {
3428	if (typed_value.ty.isPtrLikeOptional()) {
3429	if (typed_value.val.isNull())
3430	return MCValue{ .immediate = 0 };
3431
3432	var buf: Type.Payload.ElemType = undefined;
3433	return self.genTypedValue(.{
3434	.ty = typed_value.ty.optionalChild(&buf),
3435	.val = typed_value.val,
3436	});
3437	} else if (typed_value.ty.abiSize(self.target.*) == 1) {
3438	return MCValue{ .immediate = @boolToInt(typed_value.val.isNull()) };
3439	}
3440	return self.fail("TODO non pointer optionals", .{});
3441	},
3442	.Enum => {
3443	if (typed_value.val.castTag(.enum_field_index)) \|field_index\| {
3444	switch (typed_value.ty.tag()) {
3445	.enum_simple => {
3446	return MCValue{ .immediate = field_index.data };
3447	},
3448	.enum_full, .enum_nonexhaustive => {
3449	const enum_full = typed_value.ty.cast(Type.Payload.EnumFull).?.data;
3450	if (enum_full.values.count() != 0) {
3451	const tag_val = enum_full.values.keys()[field_index.data];
3452	return self.genTypedValue(.{ .ty = enum_full.tag_ty, .val = tag_val });
3453	} else {
3454	return MCValue{ .immediate = field_index.data };
3455	}
3456	},
3457	else => unreachable,
3458	}
3459	} else {
3460	var int_tag_buffer: Type.Payload.Bits = undefined;
3461	const int_tag_ty = typed_value.ty.intTagType(&int_tag_buffer);
3462	return self.genTypedValue(.{ .ty = int_tag_ty, .val = typed_value.val });
3463	}
3464	},
3465	.ErrorSet => {
3466	switch (typed_value.val.tag()) {
3467	.@"error" => {
3468	const err_name = typed_value.val.castTag(.@"error").?.data.name;
3469	const module = self.bin_file.options.module.?;
3470	const global_error_set = module.global_error_set;
3471	const error_index = global_error_set.get(err_name).?;
3472	return MCValue{ .immediate = error_index };
3473	},
3474	else => {
3475	// In this case we are rendering an error union which has a 0 bits payload.
3476	return MCValue{ .immediate = 0 };
3477	},
3478	}
3479	},
3480	.ErrorUnion => {
3481	const error_type = typed_value.ty.errorUnionSet();
3482	const payload_type = typed_value.ty.errorUnionPayload();
3483	const sub_val = typed_value.val.castTag(.eu_payload).?.data;
3484
3485	if (!payload_type.hasCodeGenBits()) {
3486	// We use the error type directly as the type.
3487	return self.genTypedValue(.{ .ty = error_type, .val = sub_val });
3488	}
3489
3490	return self.fail("TODO implement error union const of type '{}'", .{typed_value.ty});
3491	},
3492	else => return self.fail("TODO implement const of type '{}'", .{typed_value.ty}),
3493	}
3494	}
3495
3496	const CallMCValues = struct {
3497	args: []MCValue,
3498	return_value: MCValue,
3499	stack_byte_count: u32,
3500	stack_align: u32,
3501
3502	fn deinit(self: CallMCValues, func: Self) void {
3503	func.gpa.free(self.args);
3504	self.* = undefined;
3505	}
3506	};
3507
3508	/// Caller must call `CallMCValues.deinit`.
3509	fn resolveCallingConventionValues(self: *Self, fn_ty: Type) !CallMCValues {
3510	const cc = fn_ty.fnCallingConvention();
3511	const param_types = try self.gpa.alloc(Type, fn_ty.fnParamLen());
3512	defer self.gpa.free(param_types);
3513	fn_ty.fnParamTypes(param_types);
3514	var result: CallMCValues = .{
3515	.args = try self.gpa.alloc(MCValue, param_types.len),
3516	// These undefined values must be populated before returning from this function.
3517	.return_value = undefined,
3518	.stack_byte_count = undefined,
3519	.stack_align = undefined,
3520	};
3521	errdefer self.gpa.free(result.args);
3522
3523	const ret_ty = fn_ty.fnReturnType();
3524
3525	switch (cc) {
3526	.Naked => {
3527	assert(result.args.len == 0);
3528	result.return_value = .{ .unreach = {} };
3529	result.stack_byte_count = 0;
3530	result.stack_align = 1;
3531	return result;
3532	},
3533	.Unspecified, .C => {
3534	var next_int_reg: usize = 0;
3535	var next_stack_offset: u32 = 0;
3536
3537	for (param_types) \|ty, i\| {
3538	if (!ty.hasCodeGenBits()) {
3539	assert(cc != .C);
3540	result.args[i] = .{ .none = {} };
3541	continue;
3542	}
3543	const param_size = @intCast(u32, ty.abiSize(self.target.*));
3544	const pass_in_reg = switch (ty.zigTypeTag()) {
3545	.Bool => true,
3546	.Int => param_size <= 8,
3547	.Pointer => ty.ptrSize() != .Slice,
3548	.Optional => ty.isPtrLikeOptional(),
3549	else => false,
3550	};
3551	if (pass_in_reg) {
3552	if (next_int_reg >= c_abi_int_param_regs.len) {
3553	result.args[i] = .{ .stack_offset = next_stack_offset };
3554	next_stack_offset += param_size;
3555	} else {
3556	const aliased_reg = registerAlias(
3557	c_abi_int_param_regs[next_int_reg],
3558	param_size,
3559	);
3560	result.args[i] = .{ .register = aliased_reg };
3561	next_int_reg += 1;
3562	}
3563	} else {
3564	// For simplicity of codegen, slices and other types are always pushed onto the stack.
3565	// TODO: look into optimizing this by passing things as registers sometimes,
3566	// such as ptr and len of slices as separate registers.
3567	// TODO: also we need to honor the C ABI for relevant types rather than passing on
3568	// the stack here.
3569	result.args[i] = .{ .stack_offset = next_stack_offset };
3570	next_stack_offset += param_size;
3571	}
3572	}
3573	result.stack_byte_count = next_stack_offset;
3574	result.stack_align = 16;
3575	},
3576	else => return self.fail("TODO implement function parameters for {} on x86_64", .{cc}),
3577	}
3578
3579	if (ret_ty.zigTypeTag() == .NoReturn) {
3580	result.return_value = .{ .unreach = {} };
3581	} else if (!ret_ty.hasCodeGenBits()) {
3582	result.return_value = .{ .none = {} };
3583	} else switch (cc) {
3584	.Naked => unreachable,
3585	.Unspecified, .C => {
3586	const ret_ty_size = @intCast(u32, ret_ty.abiSize(self.target.*));
3587	const aliased_reg = registerAlias(c_abi_int_return_regs[0], ret_ty_size);
3588	result.return_value = .{ .register = aliased_reg };
3589	},
3590	else => return self.fail("TODO implement function return values for {}", .{cc}),
3591	}
3592	return result;
3593	}
3594
3595	/// TODO support scope overrides. Also note this logic is duplicated with `Module.wantSafety`.
3596	fn wantSafety(self: *Self) bool {
3597	return switch (self.bin_file.options.optimize_mode) {
3598	.Debug => true,
3599	.ReleaseSafe => true,
3600	.ReleaseFast => false,
3601	.ReleaseSmall => false,
3602	};
3603	}
3604
3605	fn fail(self: *Self, comptime format: []const u8, args: anytype) InnerError {
3606	@setCold(true);
3607	assert(self.err_msg == null);
3608	self.err_msg = try ErrorMsg.create(self.bin_file.allocator, self.src_loc, format, args);
3609	return error.CodegenFail;
3610	}
3611
3612	fn failSymbol(self: *Self, comptime format: []const u8, args: anytype) InnerError {
3613	@setCold(true);
3614	assert(self.err_msg == null);
3615	self.err_msg = try ErrorMsg.create(self.bin_file.allocator, self.src_loc, format, args);
3616	return error.CodegenFail;
3617	}
3618
3619	const Register = @import("bits.zig").Register;
3620
3621	const Instruction = void;
3622
3623	const Condition = void;
3624
3625	const callee_preserved_regs = @import("bits.zig").callee_preserved_regs;
3626
3627	const c_abi_int_param_regs = @import("bits.zig").c_abi_int_param_regs;
3628
3629	const c_abi_int_return_regs = @import("bits.zig").c_abi_int_return_regs;
3630
3631	fn parseRegName(name: []const u8) ?Register {
3632	if (@hasDecl(Register, "parseRegName")) {
3633	return Register.parseRegName(name);
3634	}
3635	return std.meta.stringToEnum(Register, name);
3636	}
3637
3638	fn registerAlias(reg: Register, size_bytes: u32) Register {
3639	// For x86_64 we have to pick a smaller register alias depending on abi size.
3640	switch (size_bytes) {
3641	1 => return reg.to8(),
3642	2 => return reg.to16(),
3643	4 => return reg.to32(),
3644	8 => return reg.to64(),
3645	else => unreachable,
3646	}
3647	}

src/codegen.zig+16-1281

...	@@ -22,8 +22,6 @@ const log = std.log.scoped(.codegen);	...	@@ -22,8 +22,6 @@ const log = std.log.scoped(.codegen);
22	const build_options = @import("build_options");	22	const build_options = @import("build_options");
23	const RegisterManager = @import("register_manager.zig").RegisterManager;	23	const RegisterManager = @import("register_manager.zig").RegisterManager;
24		24
25	const X8664Encoder = @import("arch/x86_64/bits.zig").Encoder;
26
27	pub const FnResult = union(enum) {	25	pub const FnResult = union(enum) {
28	/// The `code` parameter passed to `generateSymbol` has the value appended.	26	/// The `code` parameter passed to `generateSymbol` has the value appended.
29	appended: void,	27	appended: void,
...	@@ -118,7 +116,7 @@ pub fn generateFunction(	...	@@ -118,7 +116,7 @@ pub fn generateFunction(
118	//.thumb => return Function(.thumb).generate(bin_file, src_loc, func, air, liveness, code, debug_output),	116	//.thumb => return Function(.thumb).generate(bin_file, src_loc, func, air, liveness, code, debug_output),
119	//.thumbeb => return Function(.thumbeb).generate(bin_file, src_loc, func, air, liveness, code, debug_output),	117	//.thumbeb => return Function(.thumbeb).generate(bin_file, src_loc, func, air, liveness, code, debug_output),
120	//.i386 => return Function(.i386).generate(bin_file, src_loc, func, air, liveness, code, debug_output),	118	//.i386 => return Function(.i386).generate(bin_file, src_loc, func, air, liveness, code, debug_output),
121	.x86_64 => return Function(.x86_64).generate(bin_file, src_loc, func, air, liveness, code, debug_output),	119	.x86_64 => return @import("arch/x86_64/CodeGen.zig").generate(.x86_64, bin_file, src_loc, func, air, liveness, code, debug_output),
122	//.xcore => return Function(.xcore).generate(bin_file, src_loc, func, air, liveness, code, debug_output),	120	//.xcore => return Function(.xcore).generate(bin_file, src_loc, func, air, liveness, code, debug_output),
123	//.nvptx => return Function(.nvptx).generate(bin_file, src_loc, func, air, liveness, code, debug_output),	121	//.nvptx => return Function(.nvptx).generate(bin_file, src_loc, func, air, liveness, code, debug_output),
124	//.nvptx64 => return Function(.nvptx64).generate(bin_file, src_loc, func, air, liveness, code, debug_output),	122	//.nvptx64 => return Function(.nvptx64).generate(bin_file, src_loc, func, air, liveness, code, debug_output),
...	@@ -598,69 +596,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -598,69 +596,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
598		596
599	fn gen(self: *Self) !void {	597	fn gen(self: *Self) !void {
600	switch (arch) {	598	switch (arch) {
601	.x86_64 => {
602	try self.code.ensureUnusedCapacity(11);
603
604	const cc = self.fn_type.fnCallingConvention();
605	if (cc != .Naked) {
606	// We want to subtract the aligned stack frame size from rsp here, but we don't
607	// yet know how big it will be, so we leave room for a 4-byte stack size.
608	// TODO During semantic analysis, check if there are no function calls. If there
609	// are none, here we can omit the part where we subtract and then add rsp.
610	self.code.appendSliceAssumeCapacity(&[_]u8{
611	0x55, // push rbp
612	0x48, 0x89, 0xe5, // mov rbp, rsp
613	0x48, 0x81, 0xec, // sub rsp, imm32 (with reloc)
614	});
615	const reloc_index = self.code.items.len;
616	self.code.items.len += 4;
617
618	try self.dbgSetPrologueEnd();
619	try self.genBody(self.air.getMainBody());
620
621	const stack_end = self.max_end_stack;
622	if (stack_end > math.maxInt(i32))
623	return self.failSymbol("too much stack used in call parameters", .{});
624	const aligned_stack_end = mem.alignForward(stack_end, self.stack_align);
625	mem.writeIntLittle(u32, self.code.items[reloc_index..][0..4], @intCast(u32, aligned_stack_end));
626
627	if (self.code.items.len >= math.maxInt(i32)) {
628	return self.failSymbol("unable to perform relocation: jump too far", .{});
629	}
630	if (self.exitlude_jump_relocs.items.len == 1) {
631	self.code.items.len -= 5;
632	} else for (self.exitlude_jump_relocs.items) \|jmp_reloc\| {
633	const amt = self.code.items.len - (jmp_reloc + 4);
634	const s32_amt = @intCast(i32, amt);
635	mem.writeIntLittle(i32, self.code.items[jmp_reloc..][0..4], s32_amt);
636	}
637
638	// Important to be after the possible self.code.items.len -= 5 above.
639	try self.dbgSetEpilogueBegin();
640
641	try self.code.ensureUnusedCapacity(9);
642	// add rsp, x
643	if (aligned_stack_end > math.maxInt(i8)) {
644	// example: 48 81 c4 ff ff ff 7f add rsp,0x7fffffff
645	self.code.appendSliceAssumeCapacity(&[_]u8{ 0x48, 0x81, 0xc4 });
646	const x = @intCast(u32, aligned_stack_end);
647	mem.writeIntLittle(u32, self.code.addManyAsArrayAssumeCapacity(4), x);
648	} else if (aligned_stack_end != 0) {
649	// example: 48 83 c4 7f add rsp,0x7f
650	const x = @intCast(u8, aligned_stack_end);
651	self.code.appendSliceAssumeCapacity(&[_]u8{ 0x48, 0x83, 0xc4, x });
652	}
653
654	self.code.appendSliceAssumeCapacity(&[_]u8{
655	0x5d, // pop rbp
656	0xc3, // ret
657	});
658	} else {
659	try self.dbgSetPrologueEnd();
660	try self.genBody(self.air.getMainBody());
661	try self.dbgSetEpilogueBegin();
662	}
663	},
664	.arm, .armeb => {	599	.arm, .armeb => {
665	const cc = self.fn_type.fnCallingConvention();	600	const cc = self.fn_type.fnCallingConvention();
666	if (cc != .Naked) {	601	if (cc != .Naked) {
...	@@ -969,8 +904,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -969,8 +904,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
969	branch.inst_table.putAssumeCapacity(inst, .dead);	904	branch.inst_table.putAssumeCapacity(inst, .dead);
970	switch (prev_value) {	905	switch (prev_value) {
971	.register => \|reg\| {	906	.register => \|reg\| {
972	const canon_reg = toCanonicalReg(reg);	907	self.register_manager.freeReg(reg);
973	self.register_manager.freeReg(canon_reg);
974	},	908	},
975	else => {}, // TODO process stack allocation death	909	else => {}, // TODO process stack allocation death
976	}	910	}
...	@@ -1086,7 +1020,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -1086,7 +1020,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
1086	const ptr_bytes: u64 = @divExact(ptr_bits, 8);	1020	const ptr_bytes: u64 = @divExact(ptr_bits, 8);
1087	if (abi_size <= ptr_bytes) {	1021	if (abi_size <= ptr_bytes) {
1088	if (self.register_manager.tryAllocReg(inst, &.{})) \|reg\| {	1022	if (self.register_manager.tryAllocReg(inst, &.{})) \|reg\| {
1089	return MCValue{ .register = registerAlias(reg, abi_size) };	1023	return MCValue{ .register = reg };
1090	}	1024	}
1091	}	1025	}
1092	}	1026	}
...	@@ -1098,7 +1032,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -1098,7 +1032,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
1098	const stack_mcv = try self.allocRegOrMem(inst, false);	1032	const stack_mcv = try self.allocRegOrMem(inst, false);
1099	log.debug("spilling {d} to stack mcv {any}", .{ inst, stack_mcv });	1033	log.debug("spilling {d} to stack mcv {any}", .{ inst, stack_mcv });
1100	const reg_mcv = self.getResolvedInstValue(inst);	1034	const reg_mcv = self.getResolvedInstValue(inst);
1101	assert(reg == toCanonicalReg(reg_mcv.register));	1035	assert(reg == reg_mcv.register);
1102	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];	1036	const branch = &self.branch_stack.items[self.branch_stack.items.len - 1];
1103	try branch.inst_table.put(self.gpa, inst, stack_mcv);	1037	try branch.inst_table.put(self.gpa, inst, stack_mcv);
1104	try self.genSetStack(self.air.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv);	1038	try self.genSetStack(self.air.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv);
...	@@ -1226,9 +1160,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -1226,9 +1160,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
1226	}	1160	}
1227		1161
1228	switch (arch) {	1162	switch (arch) {
1229	.x86_64 => {
1230	break :result try self.genX8664BinMath(inst, ty_op.operand, .bool_true);
1231	},
1232	.arm, .armeb => {	1163	.arm, .armeb => {
1233	break :result try self.genArmBinOp(inst, ty_op.operand, .bool_true, .not);	1164	break :result try self.genArmBinOp(inst, ty_op.operand, .bool_true, .not);
1234	},	1165	},
...	@@ -1266,7 +1197,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -1266,7 +1197,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
1266	fn airAdd(self: *Self, inst: Air.Inst.Index) !void {	1197	fn airAdd(self: *Self, inst: Air.Inst.Index) !void {
1267	const bin_op = self.air.instructions.items(.data)[inst].bin_op;	1198	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1268	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {	1199	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1269	.x86_64 => try self.genX8664BinMath(inst, bin_op.lhs, bin_op.rhs),
1270	.arm, .armeb => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .add),	1200	.arm, .armeb => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .add),
1271	else => return self.fail("TODO implement add for {}", .{self.target.cpu.arch}),	1201	else => return self.fail("TODO implement add for {}", .{self.target.cpu.arch}),
1272	};	1202	};
...	@@ -1292,7 +1222,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -1292,7 +1222,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
1292	fn airSub(self: *Self, inst: Air.Inst.Index) !void {	1222	fn airSub(self: *Self, inst: Air.Inst.Index) !void {
1293	const bin_op = self.air.instructions.items(.data)[inst].bin_op;	1223	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1294	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {	1224	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1295	.x86_64 => try self.genX8664BinMath(inst, bin_op.lhs, bin_op.rhs),
1296	.arm, .armeb => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .sub),	1225	.arm, .armeb => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .sub),
1297	else => return self.fail("TODO implement sub for {}", .{self.target.cpu.arch}),	1226	else => return self.fail("TODO implement sub for {}", .{self.target.cpu.arch}),
1298	};	1227	};
...	@@ -1318,7 +1247,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -1318,7 +1247,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
1318	fn airMul(self: *Self, inst: Air.Inst.Index) !void {	1247	fn airMul(self: *Self, inst: Air.Inst.Index) !void {
1319	const bin_op = self.air.instructions.items(.data)[inst].bin_op;	1248	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1320	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {	1249	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1321	.x86_64 => try self.genX8664BinMath(inst, bin_op.lhs, bin_op.rhs),
1322	.arm, .armeb => try self.genArmMul(inst, bin_op.lhs, bin_op.rhs),	1250	.arm, .armeb => try self.genArmMul(inst, bin_op.lhs, bin_op.rhs),
1323	else => return self.fail("TODO implement mul for {}", .{self.target.cpu.arch}),	1251	else => return self.fail("TODO implement mul for {}", .{self.target.cpu.arch}),
1324	};	1252	};
...	@@ -1369,7 +1297,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -1369,7 +1297,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
1369	const bin_op = self.air.instructions.items(.data)[inst].bin_op;	1297	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1370	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {	1298	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1371	.arm, .armeb => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .bit_and),	1299	.arm, .armeb => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .bit_and),
1372	.x86_64 => try self.genX8664BinMath(inst, bin_op.lhs, bin_op.rhs),
1373	else => return self.fail("TODO implement bitwise and for {}", .{self.target.cpu.arch}),	1300	else => return self.fail("TODO implement bitwise and for {}", .{self.target.cpu.arch}),
1374	};	1301	};
1375	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	1302	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
...	@@ -1379,7 +1306,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -1379,7 +1306,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
1379	const bin_op = self.air.instructions.items(.data)[inst].bin_op;	1306	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1380	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {	1307	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
1381	.arm, .armeb => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .bit_or),	1308	.arm, .armeb => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .bit_or),
1382	.x86_64 => try self.genX8664BinMath(inst, bin_op.lhs, bin_op.rhs),
1383	else => return self.fail("TODO implement bitwise or for {}", .{self.target.cpu.arch}),	1309	else => return self.fail("TODO implement bitwise or for {}", .{self.target.cpu.arch}),
1384	};	1310	};
1385	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	1311	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
...	@@ -2088,496 +2014,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -2088,496 +2014,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
2088	return dst_mcv;	2014	return dst_mcv;
2089	}	2015	}
2090		2016
2091	/// Perform "binary" operators, excluding comparisons.
2092	/// Currently, the following ops are supported:
2093	/// ADD, SUB, XOR, OR, AND
2094	fn genX8664BinMath(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air.Inst.Ref) !MCValue {
2095	// We'll handle these ops in two steps.
2096	// 1) Prepare an output location (register or memory)
2097	// This location will be the location of the operand that dies (if one exists)
2098	// or just a temporary register (if one doesn't exist)
2099	// 2) Perform the op with the other argument
2100	// 3) Sometimes, the output location is memory but the op doesn't support it.
2101	// In this case, copy that location to a register, then perform the op to that register instead.
2102	//
2103	// TODO: make this algorithm less bad
2104
2105	try self.code.ensureUnusedCapacity(8);
2106
2107	const lhs = try self.resolveInst(op_lhs);
2108	const rhs = try self.resolveInst(op_rhs);
2109
2110	// There are 2 operands, destination and source.
2111	// Either one, but not both, can be a memory operand.
2112	// Source operand can be an immediate, 8 bits or 32 bits.
2113	// So, if either one of the operands dies with this instruction, we can use it
2114	// as the result MCValue.
2115	var dst_mcv: MCValue = undefined;
2116	var src_mcv: MCValue = undefined;
2117	var src_inst: Air.Inst.Ref = undefined;
2118	if (self.reuseOperand(inst, op_lhs, 0, lhs)) {
2119	// LHS dies; use it as the destination.
2120	// Both operands cannot be memory.
2121	src_inst = op_rhs;
2122	if (lhs.isMemory() and rhs.isMemory()) {
2123	dst_mcv = try self.copyToNewRegister(inst, lhs);
2124	src_mcv = rhs;
2125	} else {
2126	dst_mcv = lhs;
2127	src_mcv = rhs;
2128	}
2129	} else if (self.reuseOperand(inst, op_rhs, 1, rhs)) {
2130	// RHS dies; use it as the destination.
2131	// Both operands cannot be memory.
2132	src_inst = op_lhs;
2133	if (lhs.isMemory() and rhs.isMemory()) {
2134	dst_mcv = try self.copyToNewRegister(inst, rhs);
2135	src_mcv = lhs;
2136	} else {
2137	dst_mcv = rhs;
2138	src_mcv = lhs;
2139	}
2140	} else {
2141	if (lhs.isMemory()) {
2142	dst_mcv = try self.copyToNewRegister(inst, lhs);
2143	src_mcv = rhs;
2144	src_inst = op_rhs;
2145	} else {
2146	dst_mcv = try self.copyToNewRegister(inst, rhs);
2147	src_mcv = lhs;
2148	src_inst = op_lhs;
2149	}
2150	}
2151	// This instruction supports only signed 32-bit immediates at most. If the immediate
2152	// value is larger than this, we put it in a register.
2153	// A potential opportunity for future optimization here would be keeping track
2154	// of the fact that the instruction is available both as an immediate
2155	// and as a register.
2156	switch (src_mcv) {
2157	.immediate => \|imm\| {
2158	if (imm > math.maxInt(u31)) {
2159	src_mcv = MCValue{ .register = try self.copyToTmpRegister(Type.initTag(.u64), src_mcv) };
2160	}
2161	},
2162	else => {},
2163	}
2164
2165	// Now for step 2, we perform the actual op
2166	const inst_ty = self.air.typeOfIndex(inst);
2167	const air_tags = self.air.instructions.items(.tag);
2168	switch (air_tags[inst]) {
2169	// TODO: Generate wrapping and non-wrapping versions separately
2170	.add, .addwrap => try self.genX8664BinMathCode(inst_ty, dst_mcv, src_mcv, 0, 0x00),
2171	.bool_or, .bit_or => try self.genX8664BinMathCode(inst_ty, dst_mcv, src_mcv, 1, 0x08),
2172	.bool_and, .bit_and => try self.genX8664BinMathCode(inst_ty, dst_mcv, src_mcv, 4, 0x20),
2173	.sub, .subwrap => try self.genX8664BinMathCode(inst_ty, dst_mcv, src_mcv, 5, 0x28),
2174	.xor, .not => try self.genX8664BinMathCode(inst_ty, dst_mcv, src_mcv, 6, 0x30),
2175
2176	.mul, .mulwrap => try self.genX8664Imul(inst_ty, dst_mcv, src_mcv),
2177	else => unreachable,
2178	}
2179
2180	return dst_mcv;
2181	}
2182
2183	/// Wrap over Instruction.encodeInto to translate errors
2184	fn encodeX8664Instruction(self: *Self, inst: Instruction) !void {
2185	inst.encodeInto(self.code) catch \|err\| {
2186	if (err == error.OutOfMemory)
2187	return error.OutOfMemory
2188	else
2189	return self.fail("Instruction.encodeInto failed because {s}", .{@errorName(err)});
2190	};
2191	}
2192
2193	/// This function encodes a binary operation for x86_64
2194	/// intended for use with the following opcode ranges
2195	/// because they share the same structure.
2196	///
2197	/// Thus not all binary operations can be used here
2198	/// -- multiplication needs to be done with imul,
2199	/// which doesn't have as convenient an interface.
2200	///
2201	/// "opx"-style instructions use the opcode extension field to indicate which instruction to execute:
2202	///
2203	/// opx = /0: add
2204	/// opx = /1: or
2205	/// opx = /2: adc
2206	/// opx = /3: sbb
2207	/// opx = /4: and
2208	/// opx = /5: sub
2209	/// opx = /6: xor
2210	/// opx = /7: cmp
2211	///
2212	/// opcode \| operand shape
2213	/// --------+----------------------
2214	/// 80 /opx \| r/m8, imm8
2215	/// 81 /opx \| r/m16/32/64, imm16/32
2216	/// 83 /opx \| r/m16/32/64, imm8
2217	///
2218	/// "mr"-style instructions use the low bits of opcode to indicate shape of instruction:
2219	///
2220	/// mr = 00: add
2221	/// mr = 08: or
2222	/// mr = 10: adc
2223	/// mr = 18: sbb
2224	/// mr = 20: and
2225	/// mr = 28: sub
2226	/// mr = 30: xor
2227	/// mr = 38: cmp
2228	///
2229	/// opcode \| operand shape
2230	/// -------+-------------------------
2231	/// mr + 0 \| r/m8, r8
2232	/// mr + 1 \| r/m16/32/64, r16/32/64
2233	/// mr + 2 \| r8, r/m8
2234	/// mr + 3 \| r16/32/64, r/m16/32/64
2235	/// mr + 4 \| AL, imm8
2236	/// mr + 5 \| rAX, imm16/32
2237	///
2238	/// TODO: rotates and shifts share the same structure, so we can potentially implement them
2239	/// at a later date with very similar code.
2240	/// They have "opx"-style instructions, but no "mr"-style instructions.
2241	///
2242	/// opx = /0: rol,
2243	/// opx = /1: ror,
2244	/// opx = /2: rcl,
2245	/// opx = /3: rcr,
2246	/// opx = /4: shl sal,
2247	/// opx = /5: shr,
2248	/// opx = /6: sal shl,
2249	/// opx = /7: sar,
2250	///
2251	/// opcode \| operand shape
2252	/// --------+------------------
2253	/// c0 /opx \| r/m8, imm8
2254	/// c1 /opx \| r/m16/32/64, imm8
2255	/// d0 /opx \| r/m8, 1
2256	/// d1 /opx \| r/m16/32/64, 1
2257	/// d2 /opx \| r/m8, CL (for context, CL is register 1)
2258	/// d3 /opx \| r/m16/32/64, CL (for context, CL is register 1)
2259	fn genX8664BinMathCode(
2260	self: *Self,
2261	dst_ty: Type,
2262	dst_mcv: MCValue,
2263	src_mcv: MCValue,
2264	opx: u3,
2265	mr: u8,
2266	) !void {
2267	switch (dst_mcv) {
2268	.none => unreachable,
2269	.undef => unreachable,
2270	.dead, .unreach, .immediate => unreachable,
2271	.compare_flags_unsigned => unreachable,
2272	.compare_flags_signed => unreachable,
2273	.ptr_stack_offset => unreachable,
2274	.ptr_embedded_in_code => unreachable,
2275	.register => \|dst_reg\| {
2276	switch (src_mcv) {
2277	.none => unreachable,
2278	.undef => try self.genSetReg(dst_ty, dst_reg, .undef),
2279	.dead, .unreach => unreachable,
2280	.ptr_stack_offset => unreachable,
2281	.ptr_embedded_in_code => unreachable,
2282	.register => \|src_reg\| {
2283	// for register, register use mr + 1
2284	// addressing mode: r/m16/32/64, r16/32/64
2285	const abi_size = dst_ty.abiSize(self.target.*);
2286	const encoder = try X8664Encoder.init(self.code, 3);
2287	encoder.rex(.{
2288	.w = abi_size == 8,
2289	.r = src_reg.isExtended(),
2290	.b = dst_reg.isExtended(),
2291	});
2292	encoder.opcode_1byte(mr + 1);
2293	encoder.modRm_direct(
2294	src_reg.low_id(),
2295	dst_reg.low_id(),
2296	);
2297	},
2298	.immediate => \|imm\| {
2299	// register, immediate use opx = 81 or 83 addressing modes:
2300	// opx = 81: r/m16/32/64, imm16/32
2301	// opx = 83: r/m16/32/64, imm8
2302	const imm32 = @intCast(i32, imm); // This case must be handled before calling genX8664BinMathCode.
2303	if (imm32 <= math.maxInt(i8)) {
2304	const abi_size = dst_ty.abiSize(self.target.*);
2305	const encoder = try X8664Encoder.init(self.code, 4);
2306	encoder.rex(.{
2307	.w = abi_size == 8,
2308	.b = dst_reg.isExtended(),
2309	});
2310	encoder.opcode_1byte(0x83);
2311	encoder.modRm_direct(
2312	opx,
2313	dst_reg.low_id(),
2314	);
2315	encoder.imm8(@intCast(i8, imm32));
2316	} else {
2317	const abi_size = dst_ty.abiSize(self.target.*);
2318	const encoder = try X8664Encoder.init(self.code, 7);
2319	encoder.rex(.{
2320	.w = abi_size == 8,
2321	.b = dst_reg.isExtended(),
2322	});
2323	encoder.opcode_1byte(0x81);
2324	encoder.modRm_direct(
2325	opx,
2326	dst_reg.low_id(),
2327	);
2328	encoder.imm32(@intCast(i32, imm32));
2329	}
2330	},
2331	.embedded_in_code, .memory => {
2332	return self.fail("TODO implement x86 ADD/SUB/CMP source memory", .{});
2333	},
2334	.stack_offset => \|off\| {
2335	// register, indirect use mr + 3
2336	// addressing mode: r16/32/64, r/m16/32/64
2337	const abi_size = dst_ty.abiSize(self.target.*);
2338	const adj_off = off + abi_size;
2339	if (off > math.maxInt(i32)) {
2340	return self.fail("stack offset too large", .{});
2341	}
2342	const encoder = try X8664Encoder.init(self.code, 7);
2343	encoder.rex(.{
2344	.w = abi_size == 8,
2345	.r = dst_reg.isExtended(),
2346	});
2347	encoder.opcode_1byte(mr + 3);
2348	if (adj_off <= std.math.maxInt(i8)) {
2349	encoder.modRm_indirectDisp8(
2350	dst_reg.low_id(),
2351	Register.ebp.low_id(),
2352	);
2353	encoder.disp8(-@intCast(i8, adj_off));
2354	} else {
2355	encoder.modRm_indirectDisp32(
2356	dst_reg.low_id(),
2357	Register.ebp.low_id(),
2358	);
2359	encoder.disp32(-@intCast(i32, adj_off));
2360	}
2361	},
2362	.compare_flags_unsigned => {
2363	return self.fail("TODO implement x86 ADD/SUB/CMP source compare flag (unsigned)", .{});
2364	},
2365	.compare_flags_signed => {
2366	return self.fail("TODO implement x86 ADD/SUB/CMP source compare flag (signed)", .{});
2367	},
2368	}
2369	},
2370	.stack_offset => \|off\| {
2371	switch (src_mcv) {
2372	.none => unreachable,
2373	.undef => return self.genSetStack(dst_ty, off, .undef),
2374	.dead, .unreach => unreachable,
2375	.ptr_stack_offset => unreachable,
2376	.ptr_embedded_in_code => unreachable,
2377	.register => \|src_reg\| {
2378	try self.genX8664ModRMRegToStack(dst_ty, off, src_reg, mr + 0x1);
2379	},
2380	.immediate => \|imm\| {
2381	_ = imm;
2382	return self.fail("TODO implement x86 ADD/SUB/CMP source immediate", .{});
2383	},
2384	.embedded_in_code, .memory, .stack_offset => {
2385	return self.fail("TODO implement x86 ADD/SUB/CMP source memory", .{});
2386	},
2387	.compare_flags_unsigned => {
2388	return self.fail("TODO implement x86 ADD/SUB/CMP source compare flag (unsigned)", .{});
2389	},
2390	.compare_flags_signed => {
2391	return self.fail("TODO implement x86 ADD/SUB/CMP source compare flag (signed)", .{});
2392	},
2393	}
2394	},
2395	.embedded_in_code, .memory => {
2396	return self.fail("TODO implement x86 ADD/SUB/CMP destination memory", .{});
2397	},
2398	}
2399	}
2400
2401	/// Performs integer multiplication between dst_mcv and src_mcv, storing the result in dst_mcv.
2402	fn genX8664Imul(
2403	self: *Self,
2404	dst_ty: Type,
2405	dst_mcv: MCValue,
2406	src_mcv: MCValue,
2407	) !void {
2408	switch (dst_mcv) {
2409	.none => unreachable,
2410	.undef => unreachable,
2411	.dead, .unreach, .immediate => unreachable,
2412	.compare_flags_unsigned => unreachable,
2413	.compare_flags_signed => unreachable,
2414	.ptr_stack_offset => unreachable,
2415	.ptr_embedded_in_code => unreachable,
2416	.register => \|dst_reg\| {
2417	switch (src_mcv) {
2418	.none => unreachable,
2419	.undef => try self.genSetReg(dst_ty, dst_reg, .undef),
2420	.dead, .unreach => unreachable,
2421	.ptr_stack_offset => unreachable,
2422	.ptr_embedded_in_code => unreachable,
2423	.register => \|src_reg\| {
2424	// register, register
2425	//
2426	// Use the following imul opcode
2427	// 0F AF /r: IMUL r32/64, r/m32/64
2428	const abi_size = dst_ty.abiSize(self.target.*);
2429	const encoder = try X8664Encoder.init(self.code, 4);
2430	encoder.rex(.{
2431	.w = abi_size == 8,
2432	.r = dst_reg.isExtended(),
2433	.b = src_reg.isExtended(),
2434	});
2435	encoder.opcode_2byte(0x0f, 0xaf);
2436	encoder.modRm_direct(
2437	dst_reg.low_id(),
2438	src_reg.low_id(),
2439	);
2440	},
2441	.immediate => \|imm\| {
2442	// register, immediate:
2443	// depends on size of immediate.
2444	//
2445	// immediate fits in i8:
2446	// 6B /r ib: IMUL r32/64, r/m32/64, imm8
2447	//
2448	// immediate fits in i32:
2449	// 69 /r id: IMUL r32/64, r/m32/64, imm32
2450	//
2451	// immediate is huge:
2452	// split into 2 instructions
2453	// 1) copy the 64 bit immediate into a tmp register
2454	// 2) perform register,register mul
2455	// 0F AF /r: IMUL r32/64, r/m32/64
2456	if (math.minInt(i8) <= imm and imm <= math.maxInt(i8)) {
2457	const abi_size = dst_ty.abiSize(self.target.*);
2458	const encoder = try X8664Encoder.init(self.code, 4);
2459	encoder.rex(.{
2460	.w = abi_size == 8,
2461	.r = dst_reg.isExtended(),
2462	.b = dst_reg.isExtended(),
2463	});
2464	encoder.opcode_1byte(0x6B);
2465	encoder.modRm_direct(
2466	dst_reg.low_id(),
2467	dst_reg.low_id(),
2468	);
2469	encoder.imm8(@intCast(i8, imm));
2470	} else if (math.minInt(i32) <= imm and imm <= math.maxInt(i32)) {
2471	const abi_size = dst_ty.abiSize(self.target.*);
2472	const encoder = try X8664Encoder.init(self.code, 7);
2473	encoder.rex(.{
2474	.w = abi_size == 8,
2475	.r = dst_reg.isExtended(),
2476	.b = dst_reg.isExtended(),
2477	});
2478	encoder.opcode_1byte(0x69);
2479	encoder.modRm_direct(
2480	dst_reg.low_id(),
2481	dst_reg.low_id(),
2482	);
2483	encoder.imm32(@intCast(i32, imm));
2484	} else {
2485	const src_reg = try self.copyToTmpRegister(dst_ty, src_mcv);
2486	return self.genX8664Imul(dst_ty, dst_mcv, MCValue{ .register = src_reg });
2487	}
2488	},
2489	.embedded_in_code, .memory, .stack_offset => {
2490	return self.fail("TODO implement x86 multiply source memory", .{});
2491	},
2492	.compare_flags_unsigned => {
2493	return self.fail("TODO implement x86 multiply source compare flag (unsigned)", .{});
2494	},
2495	.compare_flags_signed => {
2496	return self.fail("TODO implement x86 multiply source compare flag (signed)", .{});
2497	},
2498	}
2499	},
2500	.stack_offset => \|off\| {
2501	switch (src_mcv) {
2502	.none => unreachable,
2503	.undef => return self.genSetStack(dst_ty, off, .undef),
2504	.dead, .unreach => unreachable,
2505	.ptr_stack_offset => unreachable,
2506	.ptr_embedded_in_code => unreachable,
2507	.register => \|src_reg\| {
2508	// copy dst to a register
2509	const dst_reg = try self.copyToTmpRegister(dst_ty, dst_mcv);
2510	// multiply into dst_reg
2511	// register, register
2512	// Use the following imul opcode
2513	// 0F AF /r: IMUL r32/64, r/m32/64
2514	const abi_size = dst_ty.abiSize(self.target.*);
2515	const encoder = try X8664Encoder.init(self.code, 4);
2516	encoder.rex(.{
2517	.w = abi_size == 8,
2518	.r = dst_reg.isExtended(),
2519	.b = src_reg.isExtended(),
2520	});
2521	encoder.opcode_2byte(0x0f, 0xaf);
2522	encoder.modRm_direct(
2523	dst_reg.low_id(),
2524	src_reg.low_id(),
2525	);
2526	// copy dst_reg back out
2527	return self.genSetStack(dst_ty, off, MCValue{ .register = dst_reg });
2528	},
2529	.immediate => \|imm\| {
2530	_ = imm;
2531	return self.fail("TODO implement x86 multiply source immediate", .{});
2532	},
2533	.embedded_in_code, .memory, .stack_offset => {
2534	return self.fail("TODO implement x86 multiply source memory", .{});
2535	},
2536	.compare_flags_unsigned => {
2537	return self.fail("TODO implement x86 multiply source compare flag (unsigned)", .{});
2538	},
2539	.compare_flags_signed => {
2540	return self.fail("TODO implement x86 multiply source compare flag (signed)", .{});
2541	},
2542	}
2543	},
2544	.embedded_in_code, .memory => {
2545	return self.fail("TODO implement x86 multiply destination memory", .{});
2546	},
2547	}
2548	}
2549
2550	fn genX8664ModRMRegToStack(self: *Self, ty: Type, off: u32, reg: Register, opcode: u8) !void {
2551	const abi_size = ty.abiSize(self.target.*);
2552	const adj_off = off + abi_size;
2553	if (off > math.maxInt(i32)) {
2554	return self.fail("stack offset too large", .{});
2555	}
2556
2557	const i_adj_off = -@intCast(i32, adj_off);
2558	const encoder = try X8664Encoder.init(self.code, 7);
2559	encoder.rex(.{
2560	.w = abi_size == 8,
2561	.r = reg.isExtended(),
2562	});
2563	encoder.opcode_1byte(opcode);
2564	if (i_adj_off < std.math.maxInt(i8)) {
2565	// example: 48 89 55 7f mov QWORD PTR [rbp+0x7f],rdx
2566	encoder.modRm_indirectDisp8(
2567	reg.low_id(),
2568	Register.ebp.low_id(),
2569	);
2570	encoder.disp8(@intCast(i8, i_adj_off));
2571	} else {
2572	// example: 48 89 95 80 00 00 00 mov QWORD PTR [rbp+0x80],rdx
2573	encoder.modRm_indirectDisp32(
2574	reg.low_id(),
2575	Register.ebp.low_id(),
2576	);
2577	encoder.disp32(i_adj_off);
2578	}
2579	}
2580
2581	fn genArgDbgInfo(self: *Self, inst: Air.Inst.Index, mcv: MCValue) !void {	2017	fn genArgDbgInfo(self: *Self, inst: Air.Inst.Index, mcv: MCValue) !void {
2582	const ty_str = self.air.instructions.items(.data)[inst].ty_str;	2018	const ty_str = self.air.instructions.items(.data)[inst].ty_str;
2583	const zir = &self.mod_fn.owner_decl.getFileScope().zir;	2019	const zir = &self.mod_fn.owner_decl.getFileScope().zir;
...	@@ -2674,7 +2110,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -2674,7 +2110,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
2674		2110
2675	switch (mcv) {	2111	switch (mcv) {
2676	.register => \|reg\| {	2112	.register => \|reg\| {
2677	self.register_manager.getRegAssumeFree(toCanonicalReg(reg), inst);	2113	self.register_manager.getRegAssumeFree(reg, inst);
2678	},	2114	},
2679	else => {},	2115	else => {},
2680	}	2116	}
...	@@ -2684,7 +2120,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -2684,7 +2120,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
2684		2120
2685	fn airBreakpoint(self: *Self) !void {	2121	fn airBreakpoint(self: *Self) !void {
2686	switch (arch) {	2122	switch (arch) {
2687	.i386, .x86_64 => {	2123	.i386 => {
2688	try self.code.append(0xcc); // int3	2124	try self.code.append(0xcc); // int3
2689	},	2125	},
2690	.riscv64 => {	2126	.riscv64 => {
...	@@ -2717,68 +2153,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -2717,68 +2153,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
2717	// on linking.	2153	// on linking.
2718	if (self.bin_file.tag == link.File.Elf.base_tag or self.bin_file.tag == link.File.Coff.base_tag) {	2154	if (self.bin_file.tag == link.File.Elf.base_tag or self.bin_file.tag == link.File.Coff.base_tag) {
2719	switch (arch) {	2155	switch (arch) {
2720	.x86_64 => {
2721	for (info.args) \|mc_arg, arg_i\| {
2722	const arg = args[arg_i];
2723	const arg_ty = self.air.typeOf(arg);
2724	const arg_mcv = try self.resolveInst(args[arg_i]);
2725	// Here we do not use setRegOrMem even though the logic is similar, because
2726	// the function call will move the stack pointer, so the offsets are different.
2727	switch (mc_arg) {
2728	.none => continue,
2729	.register => \|reg\| {
2730	try self.register_manager.getReg(reg, null);
2731	try self.genSetReg(arg_ty, reg, arg_mcv);
2732	},
2733	.stack_offset => \|off\| {
2734	// Here we need to emit instructions like this:
2735	// mov qword ptr [rsp + stack_offset], x
2736	try self.genSetStack(arg_ty, off, arg_mcv);
2737	},
2738	.ptr_stack_offset => {
2739	return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{});
2740	},
2741	.ptr_embedded_in_code => {
2742	return self.fail("TODO implement calling with MCValue.ptr_embedded_in_code arg", .{});
2743	},
2744	.undef => unreachable,
2745	.immediate => unreachable,
2746	.unreach => unreachable,
2747	.dead => unreachable,
2748	.embedded_in_code => unreachable,
2749	.memory => unreachable,
2750	.compare_flags_signed => unreachable,
2751	.compare_flags_unsigned => unreachable,
2752	}
2753	}
2754
2755	if (self.air.value(callee)) \|func_value\| {
2756	if (func_value.castTag(.function)) \|func_payload\| {
2757	const func = func_payload.data;
2758
2759	const ptr_bits = self.target.cpu.arch.ptrBitWidth();
2760	const ptr_bytes: u64 = @divExact(ptr_bits, 8);
2761	const got_addr = if (self.bin_file.cast(link.File.Elf)) \|elf_file\| blk: {
2762	const got = &elf_file.program_headers.items[elf_file.phdr_got_index.?];
2763	break :blk @intCast(u32, got.p_vaddr + func.owner_decl.link.elf.offset_table_index * ptr_bytes);
2764	} else if (self.bin_file.cast(link.File.Coff)) \|coff_file\|
2765	@intCast(u32, coff_file.offset_table_virtual_address + func.owner_decl.link.coff.offset_table_index * ptr_bytes)
2766	else
2767	unreachable;
2768
2769	// ff 14 25 xx xx xx xx call [addr]
2770	try self.code.ensureUnusedCapacity(7);
2771	self.code.appendSliceAssumeCapacity(&[3]u8{ 0xff, 0x14, 0x25 });
2772	mem.writeIntLittle(u32, self.code.addManyAsArrayAssumeCapacity(4), got_addr);
2773	} else if (func_value.castTag(.extern_fn)) \|_\| {
2774	return self.fail("TODO implement calling extern functions", .{});
2775	} else {
2776	return self.fail("TODO implement calling bitcasted functions", .{});
2777	}
2778	} else {
2779	return self.fail("TODO implement calling runtime known function pointer", .{});
2780	}
2781	},
2782	.riscv64 => {	2156	.riscv64 => {
2783	if (info.args.len > 0) return self.fail("TODO implement fn args for {}", .{self.target.cpu.arch});	2157	if (info.args.len > 0) return self.fail("TODO implement fn args for {}", .{self.target.cpu.arch});
2784		2158
...	@@ -2873,149 +2247,10 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -2873,149 +2247,10 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
2873	},	2247	},
2874	else => return self.fail("TODO implement call for {}", .{self.target.cpu.arch}),	2248	else => return self.fail("TODO implement call for {}", .{self.target.cpu.arch}),
2875	}	2249	}
2876	} else if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {	2250	} else if (self.bin_file.cast(link.File.MachO)) \|_\| {
2877	for (info.args) \|mc_arg, arg_i\| {	2251	unreachable; // unsupported architecture for MachO
2878	const arg = args[arg_i];	2252	} else if (self.bin_file.cast(link.File.Plan9)) \|_\| {
2879	const arg_ty = self.air.typeOf(arg);	2253	return self.fail("TODO implement call on plan9 for {}", .{self.target.cpu.arch});
2880	const arg_mcv = try self.resolveInst(args[arg_i]);
2881	// Here we do not use setRegOrMem even though the logic is similar, because
2882	// the function call will move the stack pointer, so the offsets are different.
2883	switch (mc_arg) {
2884	.none => continue,
2885	.register => \|reg\| {
2886	// TODO prevent this macho if block to be generated for all archs
2887	switch (arch) {
2888	.x86_64 => try self.register_manager.getReg(reg, null),
2889	else => unreachable,
2890	}
2891	try self.genSetReg(arg_ty, reg, arg_mcv);
2892	},
2893	.stack_offset => {
2894	// Here we need to emit instructions like this:
2895	// mov qword ptr [rsp + stack_offset], x
2896	return self.fail("TODO implement calling with parameters in memory", .{});
2897	},
2898	.ptr_stack_offset => {
2899	return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{});
2900	},
2901	.ptr_embedded_in_code => {
2902	return self.fail("TODO implement calling with MCValue.ptr_embedded_in_code arg", .{});
2903	},
2904	.undef => unreachable,
2905	.immediate => unreachable,
2906	.unreach => unreachable,
2907	.dead => unreachable,
2908	.embedded_in_code => unreachable,
2909	.memory => unreachable,
2910	.compare_flags_signed => unreachable,
2911	.compare_flags_unsigned => unreachable,
2912	}
2913	}
2914
2915	if (self.air.value(callee)) \|func_value\| {
2916	if (func_value.castTag(.function)) \|func_payload\| {
2917	const func = func_payload.data;
2918	// TODO I'm hacking my way through here by repurposing .memory for storing
2919	// index to the GOT target symbol index.
2920	switch (arch) {
2921	.x86_64 => {
2922	try self.genSetReg(Type.initTag(.u64), .rax, .{
2923	.memory = func.owner_decl.link.macho.local_sym_index,
2924	});
2925	// callq *%rax
2926	try self.code.ensureUnusedCapacity(2);
2927	self.code.appendSliceAssumeCapacity(&[2]u8{ 0xff, 0xd0 });
2928	},
2929	else => unreachable, // unsupported architecture on MachO
2930	}
2931	} else if (func_value.castTag(.extern_fn)) \|func_payload\| {
2932	const decl = func_payload.data;
2933	const n_strx = try macho_file.addExternFn(mem.spanZ(decl.name));
2934	const offset = blk: {
2935	switch (arch) {
2936	.x86_64 => {
2937	// callq
2938	try self.code.ensureUnusedCapacity(5);
2939	self.code.appendSliceAssumeCapacity(&[5]u8{ 0xe8, 0x0, 0x0, 0x0, 0x0 });
2940	break :blk @intCast(u32, self.code.items.len) - 4;
2941	},
2942	else => unreachable, // unsupported architecture on MachO
2943	}
2944	};
2945	// Add relocation to the decl.
2946	try macho_file.active_decl.?.link.macho.relocs.append(self.bin_file.allocator, .{
2947	.offset = offset,
2948	.target = .{ .global = n_strx },
2949	.addend = 0,
2950	.subtractor = null,
2951	.pcrel = true,
2952	.length = 2,
2953	.@"type" = switch (arch) {
2954	.x86_64 => @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_BRANCH),
2955	else => unreachable,
2956	},
2957	});
2958	} else {
2959	return self.fail("TODO implement calling bitcasted functions", .{});
2960	}
2961	} else {
2962	return self.fail("TODO implement calling runtime known function pointer", .{});
2963	}
2964	} else if (self.bin_file.cast(link.File.Plan9)) \|p9\| {
2965	switch (arch) {
2966	.x86_64 => {
2967	for (info.args) \|mc_arg, arg_i\| {
2968	const arg = args[arg_i];
2969	const arg_ty = self.air.typeOf(arg);
2970	const arg_mcv = try self.resolveInst(args[arg_i]);
2971	// Here we do not use setRegOrMem even though the logic is similar, because
2972	// the function call will move the stack pointer, so the offsets are different.
2973	switch (mc_arg) {
2974	.none => continue,
2975	.register => \|reg\| {
2976	try self.register_manager.getReg(reg, null);
2977	try self.genSetReg(arg_ty, reg, arg_mcv);
2978	},
2979	.stack_offset => {
2980	// Here we need to emit instructions like this:
2981	// mov qword ptr [rsp + stack_offset], x
2982	return self.fail("TODO implement calling with parameters in memory", .{});
2983	},
2984	.ptr_stack_offset => {
2985	return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{});
2986	},
2987	.ptr_embedded_in_code => {
2988	return self.fail("TODO implement calling with MCValue.ptr_embedded_in_code arg", .{});
2989	},
2990	.undef => unreachable,
2991	.immediate => unreachable,
2992	.unreach => unreachable,
2993	.dead => unreachable,
2994	.embedded_in_code => unreachable,
2995	.memory => unreachable,
2996	.compare_flags_signed => unreachable,
2997	.compare_flags_unsigned => unreachable,
2998	}
2999	}
3000	if (self.air.value(callee)) \|func_value\| {
3001	if (func_value.castTag(.function)) \|func_payload\| {
3002	try p9.seeDecl(func_payload.data.owner_decl);
3003	const ptr_bits = self.target.cpu.arch.ptrBitWidth();
3004	const ptr_bytes: u64 = @divExact(ptr_bits, 8);
3005	const got_addr = p9.bases.data;
3006	const got_index = func_payload.data.owner_decl.link.plan9.got_index.?;
3007	// ff 14 25 xx xx xx xx call [addr]
3008	try self.code.ensureUnusedCapacity(7);
3009	self.code.appendSliceAssumeCapacity(&[3]u8{ 0xff, 0x14, 0x25 });
3010	const fn_got_addr = got_addr + got_index * ptr_bytes;
3011	mem.writeIntLittle(u32, self.code.addManyAsArrayAssumeCapacity(4), @intCast(u32, fn_got_addr));
3012	} else return self.fail("TODO implement calling extern fn on plan9", .{});
3013	} else {
3014	return self.fail("TODO implement calling runtime known function pointer", .{});
3015	}
3016	},
3017	else => return self.fail("TODO implement call on plan9 for {}", .{self.target.cpu.arch}),
3018	}
3019	} else unreachable;	2254	} else unreachable;
3020		2255
3021	const result: MCValue = result: {	2256	const result: MCValue = result: {
...	@@ -3052,14 +2287,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -3052,14 +2287,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
3052	.i386 => {	2287	.i386 => {
3053	try self.code.append(0xc3); // ret	2288	try self.code.append(0xc3); // ret
3054	},	2289	},
3055	.x86_64 => {
3056	// TODO when implementing defer, this will need to jump to the appropriate defer expression.
3057	// TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
3058	// which is available if the jump is 127 bytes or less forward.
3059	try self.code.resize(self.code.items.len + 5);
3060	self.code.items[self.code.items.len - 5] = 0xe9; // jmp rel32
3061	try self.exitlude_jump_relocs.append(self.gpa, self.code.items.len - 4);
3062	},
3063	.riscv64 => {	2290	.riscv64 => {
3064	mem.writeIntLittle(u32, try self.code.addManyAsArray(4), Instruction.jalr(.zero, 0, .ra).toU32());	2291	mem.writeIntLittle(u32, try self.code.addManyAsArray(4), Instruction.jalr(.zero, 0, .ra).toU32());
3065	},	2292	},
...	@@ -3099,25 +2326,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -3099,25 +2326,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
3099	const lhs = try self.resolveInst(bin_op.lhs);	2326	const lhs = try self.resolveInst(bin_op.lhs);
3100	const rhs = try self.resolveInst(bin_op.rhs);	2327	const rhs = try self.resolveInst(bin_op.rhs);
3101	const result: MCValue = switch (arch) {	2328	const result: MCValue = switch (arch) {
3102	.x86_64 => result: {
3103	try self.code.ensureUnusedCapacity(8);
3104
3105	// There are 2 operands, destination and source.
3106	// Either one, but not both, can be a memory operand.
3107	// Source operand can be an immediate, 8 bits or 32 bits.
3108	const dst_mcv = if (lhs.isImmediate() or (lhs.isMemory() and rhs.isMemory()))
3109	try self.copyToNewRegister(inst, lhs)
3110	else
3111	lhs;
3112	// This instruction supports only signed 32-bit immediates at most.
3113	const src_mcv = try self.limitImmediateType(bin_op.rhs, i32);
3114
3115	try self.genX8664BinMathCode(Type.initTag(.bool), dst_mcv, src_mcv, 7, 0x38);
3116	break :result switch (ty.isSignedInt()) {
3117	true => MCValue{ .compare_flags_signed = op },
3118	false => MCValue{ .compare_flags_unsigned = op },
3119	};
3120	},
3121	.arm, .armeb => result: {	2329	.arm, .armeb => result: {
3122	const lhs_is_register = lhs == .register;	2330	const lhs_is_register = lhs == .register;
3123	const rhs_is_register = rhs == .register;	2331	const rhs_is_register = rhs == .register;
...	@@ -3183,7 +2391,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -3183,7 +2391,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
3183	const liveness_condbr = self.liveness.getCondBr(inst);	2391	const liveness_condbr = self.liveness.getCondBr(inst);
3184		2392
3185	const reloc: Reloc = switch (arch) {	2393	const reloc: Reloc = switch (arch) {
3186	.i386, .x86_64 => reloc: {	2394	.i386 => reloc: {
3187	try self.code.ensureUnusedCapacity(6);	2395	try self.code.ensureUnusedCapacity(6);
3188		2396
3189	const opcode: u8 = switch (cond) {	2397	const opcode: u8 = switch (cond) {
...	@@ -3214,7 +2422,8 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -3214,7 +2422,8 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
3214	.register => \|reg\| blk: {	2422	.register => \|reg\| blk: {
3215	// test reg, 1	2423	// test reg, 1
3216	// TODO detect al, ax, eax	2424	// TODO detect al, ax, eax
3217	const encoder = try X8664Encoder.init(self.code, 4);	2425	const Encoder = @import("arch/x86_64/bits.zig").Encoder;
		2426	const encoder = try Encoder.init(self.code, 4);
3218	encoder.rex(.{	2427	encoder.rex(.{
3219	// TODO audit this codegen: we force w = true here to make	2428	// TODO audit this codegen: we force w = true here to make
3220	// the value affect the big register	2429	// the value affect the big register
...	@@ -3543,7 +2752,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -3543,7 +2752,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
3543	/// Send control flow to the `index` of `self.code`.	2752	/// Send control flow to the `index` of `self.code`.
3544	fn jump(self: *Self, index: usize) !void {	2753	fn jump(self: *Self, index: usize) !void {
3545	switch (arch) {	2754	switch (arch) {
3546	.i386, .x86_64 => {	2755	.i386 => {
3547	try self.code.ensureUnusedCapacity(5);	2756	try self.code.ensureUnusedCapacity(5);
3548	if (math.cast(i8, @intCast(i32, index) - (@intCast(i32, self.code.items.len + 2)))) \|delta\| {	2757	if (math.cast(i8, @intCast(i32, index) - (@intCast(i32, self.code.items.len + 2)))) \|delta\| {
3549	self.code.appendAssumeCapacity(0xeb); // jmp rel8	2758	self.code.appendAssumeCapacity(0xeb); // jmp rel8
...	@@ -3639,13 +2848,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -3639,13 +2848,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
3639	const bin_op = self.air.instructions.items(.data)[inst].bin_op;	2848	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
3640	const air_tags = self.air.instructions.items(.tag);	2849	const air_tags = self.air.instructions.items(.tag);
3641	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {	2850	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else switch (arch) {
3642	.x86_64 => switch (air_tags[inst]) {
3643	// lhs AND rhs
3644	.bool_and => try self.genX8664BinMath(inst, bin_op.lhs, bin_op.rhs),
3645	// lhs OR rhs
3646	.bool_or => try self.genX8664BinMath(inst, bin_op.lhs, bin_op.rhs),
3647	else => unreachable, // Not a boolean operation
3648	},
3649	.arm, .armeb => switch (air_tags[inst]) {	2851	.arm, .armeb => switch (air_tags[inst]) {
3650	.bool_and => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .bool_and),	2852	.bool_and => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .bool_and),
3651	.bool_or => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .bool_or),	2853	.bool_or => try self.genArmBinOp(inst, bin_op.lhs, bin_op.rhs, .bool_or),
...	@@ -3678,7 +2880,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -3678,7 +2880,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
3678	try block_data.relocs.ensureUnusedCapacity(self.gpa, 1);	2880	try block_data.relocs.ensureUnusedCapacity(self.gpa, 1);
3679		2881
3680	switch (arch) {	2882	switch (arch) {
3681	.i386, .x86_64 => {	2883	.i386 => {
3682	// TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction	2884	// TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
3683	// which is available if the jump is 127 bytes or less forward.	2885	// which is available if the jump is 127 bytes or less forward.
3684	try self.code.resize(self.code.items.len + 5);	2886	try self.code.resize(self.code.items.len + 5);
...	@@ -3803,7 +3005,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -3803,7 +3005,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
3803	break :result MCValue{ .none = {} };	3005	break :result MCValue{ .none = {} };
3804	}	3006	}
3805	},	3007	},
3806	.x86_64, .i386 => result: {	3008	.i386 => result: {
3807	for (args) \|arg\| {	3009	for (args) \|arg\| {
3808	const input = zir.extraData(Zir.Inst.Asm.Input, extra_i);	3010	const input = zir.extraData(Zir.Inst.Asm.Input, extra_i);
3809	extra_i = input.end;	3011	extra_i = input.end;
...	@@ -3990,104 +3192,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -3990,104 +3192,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
3990	return self.genSetStack(ty, stack_offset, MCValue{ .register = reg });	3192	return self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
3991	},	3193	},
3992	},	3194	},
3993	.x86_64 => switch (mcv) {
3994	.dead => unreachable,
3995	.ptr_stack_offset => unreachable,
3996	.ptr_embedded_in_code => unreachable,
3997	.unreach, .none => return, // Nothing to do.
3998	.undef => {
3999	if (!self.wantSafety())
4000	return; // The already existing value will do just fine.
4001	// TODO Upgrade this to a memset call when we have that available.
4002	switch (ty.abiSize(self.target.*)) {
4003	1 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaa }),
4004	2 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaa }),
4005	4 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaa }),
4006	8 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaaaaaaaaaa }),
4007	else => return self.fail("TODO implement memset", .{}),
4008	}
4009	},
4010	.compare_flags_unsigned => \|op\| {
4011	_ = op;
4012	return self.fail("TODO implement set stack variable with compare flags value (unsigned)", .{});
4013	},
4014	.compare_flags_signed => \|op\| {
4015	_ = op;
4016	return self.fail("TODO implement set stack variable with compare flags value (signed)", .{});
4017	},
4018	.immediate => \|x_big\| {
4019	const abi_size = ty.abiSize(self.target.*);
4020	const adj_off = stack_offset + abi_size;
4021	if (adj_off > 128) {
4022	return self.fail("TODO implement set stack variable with large stack offset", .{});
4023	}
4024	try self.code.ensureUnusedCapacity(8);
4025	switch (abi_size) {
4026	1 => {
4027	return self.fail("TODO implement set abi_size=1 stack variable with immediate", .{});
4028	},
4029	2 => {
4030	return self.fail("TODO implement set abi_size=2 stack variable with immediate", .{});
4031	},
4032	4 => {
4033	const x = @intCast(u32, x_big);
4034	// We have a positive stack offset value but we want a twos complement negative
4035	// offset from rbp, which is at the top of the stack frame.
4036	const negative_offset = @intCast(i8, -@intCast(i32, adj_off));
4037	const twos_comp = @bitCast(u8, negative_offset);
4038	// mov DWORD PTR [rbp+offset], immediate
4039	self.code.appendSliceAssumeCapacity(&[_]u8{ 0xc7, 0x45, twos_comp });
4040	mem.writeIntLittle(u32, self.code.addManyAsArrayAssumeCapacity(4), x);
4041	},
4042	8 => {
4043	// We have a positive stack offset value but we want a twos complement negative
4044	// offset from rbp, which is at the top of the stack frame.
4045	const negative_offset = @intCast(i8, -@intCast(i32, adj_off));
4046	const twos_comp = @bitCast(u8, negative_offset);
4047
4048	// 64 bit write to memory would take two mov's anyways so we
4049	// insted just use two 32 bit writes to avoid register allocation
4050	try self.code.ensureUnusedCapacity(14);
4051	var buf: [8]u8 = undefined;
4052	mem.writeIntLittle(u64, &buf, x_big);
4053
4054	// mov DWORD PTR [rbp+offset+4], immediate
4055	self.code.appendSliceAssumeCapacity(&[_]u8{ 0xc7, 0x45, twos_comp + 4 });
4056	self.code.appendSliceAssumeCapacity(buf[4..8]);
4057
4058	// mov DWORD PTR [rbp+offset], immediate
4059	self.code.appendSliceAssumeCapacity(&[_]u8{ 0xc7, 0x45, twos_comp });
4060	self.code.appendSliceAssumeCapacity(buf[0..4]);
4061	},
4062	else => {
4063	return self.fail("TODO implement set abi_size=large stack variable with immediate", .{});
4064	},
4065	}
4066	},
4067	.embedded_in_code => {
4068	// TODO this and `.stack_offset` below need to get improved to support types greater than
4069	// register size, and do general memcpy
4070	const reg = try self.copyToTmpRegister(ty, mcv);
4071	return self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
4072	},
4073	.register => \|reg\| {
4074	try self.genX8664ModRMRegToStack(ty, stack_offset, reg, 0x89);
4075	},
4076	.memory => \|vaddr\| {
4077	_ = vaddr;
4078	return self.fail("TODO implement set stack variable from memory vaddr", .{});
4079	},
4080	.stack_offset => \|off\| {
4081	// TODO this and `.embedded_in_code` above need to get improved to support types greater than
4082	// register size, and do general memcpy
4083
4084	if (stack_offset == off)
4085	return; // Copy stack variable to itself; nothing to do.
4086
4087	const reg = try self.copyToTmpRegister(ty, mcv);
4088	return self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
4089	},
4090	},
4091	else => return self.fail("TODO implement getSetStack for {}", .{self.target.cpu.arch}),	3195	else => return self.fail("TODO implement getSetStack for {}", .{self.target.cpu.arch}),
4092	}	3196	}
4093	}	3197	}
...	@@ -4250,284 +3354,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -4250,284 +3354,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
4250	},	3354	},
4251	else => return self.fail("TODO implement getSetReg for riscv64 {}", .{mcv}),	3355	else => return self.fail("TODO implement getSetReg for riscv64 {}", .{mcv}),
4252	},	3356	},
4253	.x86_64 => switch (mcv) {
4254	.dead => unreachable,
4255	.ptr_stack_offset => unreachable,
4256	.ptr_embedded_in_code => unreachable,
4257	.unreach, .none => return, // Nothing to do.
4258	.undef => {
4259	if (!self.wantSafety())
4260	return; // The already existing value will do just fine.
4261	// Write the debug undefined value.
4262	switch (reg.size()) {
4263	8 => return self.genSetReg(ty, reg, .{ .immediate = 0xaa }),
4264	16 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaa }),
4265	32 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaa }),
4266	64 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaaaaaaaaaa }),
4267	else => unreachable,
4268	}
4269	},
4270	.compare_flags_unsigned => \|op\| {
4271	const encoder = try X8664Encoder.init(self.code, 7);
4272	// TODO audit this codegen: we force w = true here to make
4273	// the value affect the big register
4274	encoder.rex(.{
4275	.w = true,
4276	.b = reg.isExtended(),
4277	});
4278	encoder.opcode_2byte(0x0f, switch (op) {
4279	.gte => 0x93,
4280	.gt => 0x97,
4281	.neq => 0x95,
4282	.lt => 0x92,
4283	.lte => 0x96,
4284	.eq => 0x94,
4285	});
4286	encoder.modRm_direct(
4287	0,
4288	reg.low_id(),
4289	);
4290	},
4291	.compare_flags_signed => \|op\| {
4292	_ = op;
4293	return self.fail("TODO set register with compare flags value (signed)", .{});
4294	},
4295	.immediate => \|x\| {
4296	// 32-bit moves zero-extend to 64-bit, so xoring the 32-bit
4297	// register is the fastest way to zero a register.
4298	if (x == 0) {
4299	// The encoding for `xor r32, r32` is `0x31 /r`.
4300	const encoder = try X8664Encoder.init(self.code, 3);
4301
4302	// If we're accessing e.g. r8d, we need to use a REX prefix before the actual operation. Since
4303	// this is a 32-bit operation, the W flag is set to zero. X is also zero, as we're not using a SIB.
4304	// Both R and B are set, as we're extending, in effect, the register bits and the operand.
4305	encoder.rex(.{
4306	.r = reg.isExtended(),
4307	.b = reg.isExtended(),
4308	});
4309	encoder.opcode_1byte(0x31);
4310	// Section 3.1.1.1 of the Intel x64 Manual states that "/r indicates that the
4311	// ModR/M byte of the instruction contains a register operand and an r/m operand."
4312	encoder.modRm_direct(
4313	reg.low_id(),
4314	reg.low_id(),
4315	);
4316
4317	return;
4318	}
4319	if (x <= math.maxInt(i32)) {
4320	// Next best case: if we set the lower four bytes, the upper four will be zeroed.
4321	//
4322	// The encoding for `mov IMM32 -> REG` is (0xB8 + R) IMM.
4323
4324	const encoder = try X8664Encoder.init(self.code, 6);
4325	// Just as with XORing, we need a REX prefix. This time though, we only
4326	// need the B bit set, as we're extending the opcode's register field,
4327	// and there is no Mod R/M byte.
4328	encoder.rex(.{
4329	.b = reg.isExtended(),
4330	});
4331	encoder.opcode_withReg(0xB8, reg.low_id());
4332
4333	// no ModR/M byte
4334
4335	// IMM
4336	encoder.imm32(@intCast(i32, x));
4337	return;
4338	}
4339	// Worst case: we need to load the 64-bit register with the IMM. GNU's assemblers calls
4340	// this `movabs`, though this is officially just a different variant of the plain `mov`
4341	// instruction.
4342	//
4343	// This encoding is, in fact, the same as the one used for 32-bit loads. The only
4344	// difference is that we set REX.W before the instruction, which extends the load to
4345	// 64-bit and uses the full bit-width of the register.
4346	{
4347	const encoder = try X8664Encoder.init(self.code, 10);
4348	encoder.rex(.{
4349	.w = true,
4350	.b = reg.isExtended(),
4351	});
4352	encoder.opcode_withReg(0xB8, reg.low_id());
4353	encoder.imm64(x);
4354	}
4355	},
4356	.embedded_in_code => \|code_offset\| {
4357	// We need the offset from RIP in a signed i32 twos complement.
4358	// The instruction is 7 bytes long and RIP points to the next instruction.
4359
4360	// 64-bit LEA is encoded as REX.W 8D /r.
4361	const rip = self.code.items.len + 7;
4362	const big_offset = @intCast(i64, code_offset) - @intCast(i64, rip);
4363	const offset = @intCast(i32, big_offset);
4364	const encoder = try X8664Encoder.init(self.code, 7);
4365
4366	// byte 1, always exists because w = true
4367	encoder.rex(.{
4368	.w = true,
4369	.r = reg.isExtended(),
4370	});
4371	// byte 2
4372	encoder.opcode_1byte(0x8D);
4373	// byte 3
4374	encoder.modRm_RIPDisp32(reg.low_id());
4375	// byte 4-7
4376	encoder.disp32(offset);
4377
4378	// Double check that we haven't done any math errors
4379	assert(rip == self.code.items.len);
4380	},
4381	.register => \|src_reg\| {
4382	// If the registers are the same, nothing to do.
4383	if (src_reg.id() == reg.id())
4384	return;
4385
4386	// This is a variant of 8B /r.
4387	const abi_size = ty.abiSize(self.target.*);
4388	const encoder = try X8664Encoder.init(self.code, 3);
4389	encoder.rex(.{
4390	.w = abi_size == 8,
4391	.r = reg.isExtended(),
4392	.b = src_reg.isExtended(),
4393	});
4394	encoder.opcode_1byte(0x8B);
4395	encoder.modRm_direct(reg.low_id(), src_reg.low_id());
4396	},
4397	.memory => \|x\| {
4398	if (self.bin_file.options.pie) {
4399	// RIP-relative displacement to the entry in the GOT table.
4400	const abi_size = ty.abiSize(self.target.*);
4401	const encoder = try X8664Encoder.init(self.code, 10);
4402
4403	// LEA reg, [<offset>]
4404
4405	// We encode the instruction FIRST because prefixes may or may not appear.
4406	// After we encode the instruction, we will know that the displacement bytes
4407	// for [<offset>] will be at self.code.items.len - 4.
4408	encoder.rex(.{
4409	.w = true, // force 64 bit because loading an address (to the GOT)
4410	.r = reg.isExtended(),
4411	});
4412	encoder.opcode_1byte(0x8D);
4413	encoder.modRm_RIPDisp32(reg.low_id());
4414	encoder.disp32(0);
4415
4416	const offset = @intCast(u32, self.code.items.len);
4417
4418	if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {
4419	// TODO I think the reloc might be in the wrong place.
4420	const decl = macho_file.active_decl.?;
4421	// Load reloc for LEA instruction.
4422	try decl.link.macho.relocs.append(self.bin_file.allocator, .{
4423	.offset = offset - 4,
4424	.target = .{ .local = @intCast(u32, x) },
4425	.addend = 0,
4426	.subtractor = null,
4427	.pcrel = true,
4428	.length = 2,
4429	.@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_GOT),
4430	});
4431	} else {
4432	return self.fail("TODO implement genSetReg for PIE GOT indirection on this platform", .{});
4433	}
4434
4435	// MOV reg, [reg]
4436	encoder.rex(.{
4437	.w = abi_size == 8,
4438	.r = reg.isExtended(),
4439	.b = reg.isExtended(),
4440	});
4441	encoder.opcode_1byte(0x8B);
4442	encoder.modRm_indirectDisp0(reg.low_id(), reg.low_id());
4443	} else if (x <= math.maxInt(i32)) {
4444	// Moving from memory to a register is a variant of `8B /r`.
4445	// Since we're using 64-bit moves, we require a REX.
4446	// This variant also requires a SIB, as it would otherwise be RIP-relative.
4447	// We want mode zero with the lower three bits set to four to indicate an SIB with no other displacement.
4448	// The SIB must be 0x25, to indicate a disp32 with no scaled index.
4449	// 0b00RRR100, where RRR is the lower three bits of the register ID.
4450	// The instruction is thus eight bytes; REX 0x8B 0b00RRR100 0x25 followed by a four-byte disp32.
4451	const abi_size = ty.abiSize(self.target.*);
4452	const encoder = try X8664Encoder.init(self.code, 8);
4453	encoder.rex(.{
4454	.w = abi_size == 8,
4455	.r = reg.isExtended(),
4456	});
4457	encoder.opcode_1byte(0x8B);
4458	// effective address = [SIB]
4459	encoder.modRm_SIBDisp0(reg.low_id());
4460	// SIB = disp32
4461	encoder.sib_disp32();
4462	encoder.disp32(@intCast(i32, x));
4463	} else {
4464	// If this is RAX, we can use a direct load; otherwise, we need to load the address, then indirectly load
4465	// the value.
4466	if (reg.id() == 0) {
4467	// REX.W 0xA1 moffs64*
4468	// moffs64* is a 64-bit offset "relative to segment base", which really just means the
4469	// absolute address for all practical purposes.
4470
4471	const encoder = try X8664Encoder.init(self.code, 10);
4472	encoder.rex(.{
4473	.w = true,
4474	});
4475	encoder.opcode_1byte(0xA1);
4476	encoder.writeIntLittle(u64, x);
4477	} else {
4478	// This requires two instructions; a move imm as used above, followed by an indirect load using the register
4479	// as the address and the register as the destination.
4480	//
4481	// This cannot be used if the lower three bits of the id are equal to four or five, as there
4482	// is no way to possibly encode it. This means that RSP, RBP, R12, and R13 cannot be used with
4483	// this instruction.
4484	const id3 = @truncate(u3, reg.id());
4485	assert(id3 != 4 and id3 != 5);
4486
4487	// Rather than duplicate the logic used for the move, we just use a self-call with a new MCValue.
4488	try self.genSetReg(ty, reg, MCValue{ .immediate = x });
4489
4490	// Now, the register contains the address of the value to load into it
4491	// Currently, we're only allowing 64-bit registers, so we need the `REX.W 8B /r` variant.
4492	// TODO: determine whether to allow other sized registers, and if so, handle them properly.
4493
4494	// mov reg, [reg]
4495	const abi_size = ty.abiSize(self.target.*);
4496	const encoder = try X8664Encoder.init(self.code, 3);
4497	encoder.rex(.{
4498	.w = abi_size == 8,
4499	.r = reg.isExtended(),
4500	.b = reg.isExtended(),
4501	});
4502	encoder.opcode_1byte(0x8B);
4503	encoder.modRm_indirectDisp0(reg.low_id(), reg.low_id());
4504	}
4505	}
4506	},
4507	.stack_offset => \|unadjusted_off\| {
4508	const abi_size = ty.abiSize(self.target.*);
4509	const off = unadjusted_off + abi_size;
4510	if (off < std.math.minInt(i32) or off > std.math.maxInt(i32)) {
4511	return self.fail("stack offset too large", .{});
4512	}
4513	const ioff = -@intCast(i32, off);
4514	const encoder = try X8664Encoder.init(self.code, 3);
4515	encoder.rex(.{
4516	.w = abi_size == 8,
4517	.r = reg.isExtended(),
4518	});
4519	encoder.opcode_1byte(0x8B);
4520	if (std.math.minInt(i8) <= ioff and ioff <= std.math.maxInt(i8)) {
4521	// Example: 48 8b 4d 7f mov rcx,QWORD PTR [rbp+0x7f]
4522	encoder.modRm_indirectDisp8(reg.low_id(), Register.ebp.low_id());
4523	encoder.disp8(@intCast(i8, ioff));
4524	} else {
4525	// Example: 48 8b 8d 80 00 00 00 mov rcx,QWORD PTR [rbp+0x80]
4526	encoder.modRm_indirectDisp32(reg.low_id(), Register.ebp.low_id());
4527	encoder.disp32(ioff);
4528	}
4529	},
4530	},
4531	else => return self.fail("TODO implement getSetReg for {}", .{self.target.cpu.arch}),	3357	else => return self.fail("TODO implement getSetReg for {}", .{self.target.cpu.arch}),
4532	}	3358	}
4533	}	3359	}
...	@@ -4840,61 +3666,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -4840,61 +3666,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
4840	const ret_ty = fn_ty.fnReturnType();	3666	const ret_ty = fn_ty.fnReturnType();
4841		3667
4842	switch (arch) {	3668	switch (arch) {
4843	.x86_64 => {
4844	switch (cc) {
4845	.Naked => {
4846	assert(result.args.len == 0);
4847	result.return_value = .{ .unreach = {} };
4848	result.stack_byte_count = 0;
4849	result.stack_align = 1;
4850	return result;
4851	},
4852	.Unspecified, .C => {
4853	var next_int_reg: usize = 0;
4854	var next_stack_offset: u32 = 0;
4855
4856	for (param_types) \|ty, i\| {
4857	if (!ty.hasCodeGenBits()) {
4858	assert(cc != .C);
4859	result.args[i] = .{ .none = {} };
4860	continue;
4861	}
4862	const param_size = @intCast(u32, ty.abiSize(self.target.*));
4863	const pass_in_reg = switch (ty.zigTypeTag()) {
4864	.Bool => true,
4865	.Int => param_size <= 8,
4866	.Pointer => ty.ptrSize() != .Slice,
4867	.Optional => ty.isPtrLikeOptional(),
4868	else => false,
4869	};
4870	if (pass_in_reg) {
4871	if (next_int_reg >= c_abi_int_param_regs.len) {
4872	result.args[i] = .{ .stack_offset = next_stack_offset };
4873	next_stack_offset += param_size;
4874	} else {
4875	const aliased_reg = registerAlias(
4876	c_abi_int_param_regs[next_int_reg],
4877	param_size,
4878	);
4879	result.args[i] = .{ .register = aliased_reg };
4880	next_int_reg += 1;
4881	}
4882	} else {
4883	// For simplicity of codegen, slices and other types are always pushed onto the stack.
4884	// TODO: look into optimizing this by passing things as registers sometimes,
4885	// such as ptr and len of slices as separate registers.
4886	// TODO: also we need to honor the C ABI for relevant types rather than passing on
4887	// the stack here.
4888	result.args[i] = .{ .stack_offset = next_stack_offset };
4889	next_stack_offset += param_size;
4890	}
4891	}
4892	result.stack_byte_count = next_stack_offset;
4893	result.stack_align = 16;
4894	},
4895	else => return self.fail("TODO implement function parameters for {} on x86_64", .{cc}),
4896	}
4897	},
4898	.arm, .armeb => {	3669	.arm, .armeb => {
4899	switch (cc) {	3670	switch (cc) {
4900	.Naked => {	3671	.Naked => {
...	@@ -4948,15 +3719,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -4948,15 +3719,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
4948	} else if (!ret_ty.hasCodeGenBits()) {	3719	} else if (!ret_ty.hasCodeGenBits()) {
4949	result.return_value = .{ .none = {} };	3720	result.return_value = .{ .none = {} };
4950	} else switch (arch) {	3721	} else switch (arch) {
4951	.x86_64 => switch (cc) {
4952	.Naked => unreachable,
4953	.Unspecified, .C => {
4954	const ret_ty_size = @intCast(u32, ret_ty.abiSize(self.target.*));
4955	const aliased_reg = registerAlias(c_abi_int_return_regs[0], ret_ty_size);
4956	result.return_value = .{ .register = aliased_reg };
4957	},
4958	else => return self.fail("TODO implement function return values for {}", .{cc}),
4959	},
4960	.arm, .armeb => switch (cc) {	3722	.arm, .armeb => switch (cc) {
4961	.Naked => unreachable,	3723	.Naked => unreachable,
4962	.Unspecified, .C => {	3724	.Unspecified, .C => {
...	@@ -5000,7 +3762,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -5000,7 +3762,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
5000		3762
5001	const Register = switch (arch) {	3763	const Register = switch (arch) {
5002	.i386 => @import("arch/x86/bits.zig").Register,	3764	.i386 => @import("arch/x86/bits.zig").Register,
5003	.x86_64 => @import("arch/x86_64/bits.zig").Register,
5004	.riscv64 => @import("arch/riscv64/bits.zig").Register,	3765	.riscv64 => @import("arch/riscv64/bits.zig").Register,
5005	.arm, .armeb => @import("arch/arm/bits.zig").Register,	3766	.arm, .armeb => @import("arch/arm/bits.zig").Register,
5006	else => enum {	3767	else => enum {
...	@@ -5026,7 +3787,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -5026,7 +3787,6 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
5026		3787
5027	const callee_preserved_regs = switch (arch) {	3788	const callee_preserved_regs = switch (arch) {
5028	.i386 => @import("arch/x86/bits.zig").callee_preserved_regs,	3789	.i386 => @import("arch/x86/bits.zig").callee_preserved_regs,
5029	.x86_64 => @import("arch/x86_64/bits.zig").callee_preserved_regs,
5030	.riscv64 => @import("arch/riscv64/bits.zig").callee_preserved_regs,	3790	.riscv64 => @import("arch/riscv64/bits.zig").callee_preserved_regs,
5031	.arm, .armeb => @import("arch/arm/bits.zig").callee_preserved_regs,	3791	.arm, .armeb => @import("arch/arm/bits.zig").callee_preserved_regs,
5032	else => [_]Register{},	3792	else => [_]Register{},
...	@@ -5034,14 +3794,12 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -5034,14 +3794,12 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
5034		3794
5035	const c_abi_int_param_regs = switch (arch) {	3795	const c_abi_int_param_regs = switch (arch) {
5036	.i386 => @import("arch/x86/bits.zig").c_abi_int_param_regs,	3796	.i386 => @import("arch/x86/bits.zig").c_abi_int_param_regs,
5037	.x86_64 => @import("arch/x86_64/bits.zig").c_abi_int_param_regs,
5038	.arm, .armeb => @import("arch/arm/bits.zig").c_abi_int_param_regs,	3797	.arm, .armeb => @import("arch/arm/bits.zig").c_abi_int_param_regs,
5039	else => [_]Register{},	3798	else => [_]Register{},
5040	};	3799	};
5041		3800
5042	const c_abi_int_return_regs = switch (arch) {	3801	const c_abi_int_return_regs = switch (arch) {
5043	.i386 => @import("arch/x86/bits.zig").c_abi_int_return_regs,	3802	.i386 => @import("arch/x86/bits.zig").c_abi_int_return_regs,
5044	.x86_64 => @import("arch/x86_64/bits.zig").c_abi_int_return_regs,
5045	.arm, .armeb => @import("arch/arm/bits.zig").c_abi_int_return_regs,	3803	.arm, .armeb => @import("arch/arm/bits.zig").c_abi_int_return_regs,
5046	else => [_]Register{},	3804	else => [_]Register{},
5047	};	3805	};
...	@@ -5052,28 +3810,5 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {	...	@@ -5052,28 +3810,5 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
5052	}	3810	}
5053	return std.meta.stringToEnum(Register, name);	3811	return std.meta.stringToEnum(Register, name);
5054	}	3812	}
5055
5056	fn registerAlias(reg: Register, size_bytes: u32) Register {
5057	switch (arch) {
5058	// For x86_64 we have to pick a smaller register alias depending on abi size.
5059	.x86_64 => switch (size_bytes) {
5060	1 => return reg.to8(),
5061	2 => return reg.to16(),
5062	4 => return reg.to32(),
5063	8 => return reg.to64(),
5064	else => unreachable,
5065	},
5066	else => return reg,
5067	}
5068	}
5069
5070	/// For most architectures this does nothing. For x86_64 it resolves any aliased registers
5071	/// to the 64-bit wide ones.
5072	fn toCanonicalReg(reg: Register) Register {
5073	return switch (arch) {
5074	.x86_64 => reg.to64(),
5075	else => reg,
5076	};
5077	}
5078	};	3813	};
5079	}	3814	}