Commit 06286b05a41c6a6dcc28b62edfc41063e61e1588 | zig

author	Andrew Kelley <andrew@ziglang.org> 2021-12-29 22:22:26-05:00
committer	GitHub <noreply@github.com> 2021-12-29 22:22:26-05:00
log	06286b05a41c6a6dcc28b62edfc41063e61e1588
tree	908340c747ac198e9c9530fb66774b5dd8d77f38
parent	be5130ec535456559497ac241ac9fa76c4bbb8ca
parent	4ecc5956f62166708f3f47522e326452b0166dda
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #10446 from ziglang/stage2-x86_64-mov-mem-imm

stage2: more fixes to x86_64 codegen, mir and isel

5 files changed, 1291 insertions(+), 1173 deletions(-)

src
- arch
  - x86_64
    CodeGen.zig
    Emit.zig
    Mir.zig
    PrintMir.zig
test
- stage2
  - x86_64.zig

src/arch/x86_64/CodeGen.zig+30-25

...	...	@@ -329,7 +329,11 @@ pub fn generate(
329	329	if (builtin.mode == .Debug and bin_file.options.module.?.comp.verbose_mir) {
330	330	const w = std.io.getStdErr().writer();
331	331	w.print("# Begin Function MIR: {s}:\n", .{module_fn.owner_decl.name}) catch {};
332		const print = @import("./PrintMir.zig"){ .mir = mir };
	332	const PrintMir = @import("PrintMir.zig");
	333	const print = PrintMir{
	334	.mir = mir,
	335	.bin_file = bin_file,
	336	};
333	337	print.printMir(w, function.mir_to_air_map, air) catch {}; // we don't care if the debug printing fails
334	338	w.print("# End Function MIR: {s}\n\n", .{module_fn.owner_decl.name}) catch {};
335	339	}
...	...	@@ -1642,11 +1646,11 @@ fn genBinMathOpMir(
1642	1646	});
1643	1647	},
1644	1648	.immediate => \|imm\| {
1645		// TODO I am not quite sure why we need to set the size of the register here...
	1649	const abi_size = dst_ty.abiSize(self.target.*);
1646	1650	_ = try self.addInst(.{
1647	1651	.tag = mir_tag,
1648	1652	.ops = (Mir.Ops{
1649		.reg1 = registerAlias(dst_reg, 4),
	1653	.reg1 = registerAlias(dst_reg, @intCast(u32, abi_size)),
1650	1654	}).encode(),
1651	1655	.data = .{ .imm = @intCast(i32, imm) },
1652	1656	});
...	...	@@ -1751,13 +1755,14 @@ fn genIMulOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !
1751	1755	});
1752	1756	},
1753	1757	.immediate => \|imm\| {
	1758	// TODO take into account the type's ABI size when selecting the register alias
1754	1759	// register, immediate
1755	1760	if (imm <= math.maxInt(i32)) {
1756	1761	_ = try self.addInst(.{
1757	1762	.tag = .imul_complex,
1758	1763	.ops = (Mir.Ops{
1759		.reg1 = dst_reg,
1760		.reg2 = dst_reg,
	1764	.reg1 = dst_reg.to32(),
	1765	.reg2 = dst_reg.to32(),
1761	1766	.flags = 0b10,
1762	1767	}).encode(),
1763	1768	.data = .{ .imm = @intCast(i32, imm) },
...	...	@@ -2147,7 +2152,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
2147	2152	// This instruction supports only signed 32-bit immediates at most.
2148	2153	const src_mcv = try self.limitImmediateType(bin_op.rhs, i32);
2149	2154
2150		try self.genBinMathOpMir(.cmp, Type.initTag(.bool), dst_mcv, src_mcv);
	2155	try self.genBinMathOpMir(.cmp, ty, dst_mcv, src_mcv);
2151	2156	break :result switch (ty.isSignedInt()) {
2152	2157	true => MCValue{ .compare_flags_signed = op },
2153	2158	false => MCValue{ .compare_flags_unsigned = op },
...	...	@@ -2792,25 +2797,24 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
2792	2797	return self.fail("TODO implement set stack variable with large stack offset", .{});
2793	2798	}
2794	2799	switch (abi_size) {
2795		1 => {
2796		return self.fail("TODO implement set abi_size=1 stack variable with immediate", .{});
2797		},
2798		2 => {
2799		return self.fail("TODO implement set abi_size=2 stack variable with immediate", .{});
2800		},
2801		4 => {
	2800	1, 2, 4 => {
2802	2801	// We have a positive stack offset value but we want a twos complement negative
2803	2802	// offset from rbp, which is at the top of the stack frame.
2804		// mov DWORD PTR [rbp+offset], immediate
	2803	// mov [rbp+offset], immediate
2805	2804	const payload = try self.addExtra(Mir.ImmPair{
2806	2805	.dest_off = -@intCast(i32, adj_off),
2807	2806	.operand = @bitCast(i32, @intCast(u32, x_big)),
2808	2807	});
2809	2808	_ = try self.addInst(.{
2810		.tag = .mov,
	2809	.tag = .mov_mem_imm,
2811	2810	.ops = (Mir.Ops{
2812	2811	.reg1 = .rbp,
2813		.flags = 0b11,
	2812	.flags = switch (abi_size) {
	2813	1 => 0b00,
	2814	2 => 0b01,
	2815	4 => 0b10,
	2816	else => unreachable,
	2817	},
2814	2818	}).encode(),
2815	2819	.data = .{ .payload = payload },
2816	2820	});
...	...	@@ -2828,10 +2832,10 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
2828	2832	.operand = @bitCast(i32, @truncate(u32, x_big >> 32)),
2829	2833	});
2830	2834	_ = try self.addInst(.{
2831		.tag = .mov,
	2835	.tag = .mov_mem_imm,
2832	2836	.ops = (Mir.Ops{
2833	2837	.reg1 = .rbp,
2834		.flags = 0b11,
	2838	.flags = 0b10,
2835	2839	}).encode(),
2836	2840	.data = .{ .payload = payload },
2837	2841	});
...	...	@@ -2842,10 +2846,10 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
2842	2846	.operand = @bitCast(i32, @truncate(u32, x_big)),
2843	2847	});
2844	2848	_ = try self.addInst(.{
2845		.tag = .mov,
	2849	.tag = .mov_mem_imm,
2846	2850	.ops = (Mir.Ops{
2847	2851	.reg1 = .rbp,
2848		.flags = 0b11,
	2852	.flags = 0b10,
2849	2853	}).encode(),
2850	2854	.data = .{ .payload = payload },
2851	2855	});
...	...	@@ -2954,12 +2958,12 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
2954	2958	return;
2955	2959	}
2956	2960	if (x <= math.maxInt(i32)) {
	2961	const abi_size = ty.abiSize(self.target.*);
2957	2962	// Next best case: if we set the lower four bytes, the upper four will be zeroed.
2958		// TODO I am not quite sure why we need to set the size of the register here...
2959	2963	_ = try self.addInst(.{
2960	2964	.tag = .mov,
2961	2965	.ops = (Mir.Ops{
2962		.reg1 = registerAlias(reg, 4),
	2966	.reg1 = registerAlias(reg, @intCast(u32, abi_size)),
2963	2967	}).encode(),
2964	2968	.data = .{ .imm = @intCast(i32, x) },
2965	2969	});
...	...	@@ -2985,9 +2989,10 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
2985	2989	// We need the offset from RIP in a signed i32 twos complement.
2986	2990	const payload = try self.addExtra(Mir.Imm64.encode(code_offset));
2987	2991	_ = try self.addInst(.{
2988		.tag = .lea_rip,
	2992	.tag = .lea,
2989	2993	.ops = (Mir.Ops{
2990	2994	.reg1 = reg,
	2995	.flags = 0b01,
2991	2996	}).encode(),
2992	2997	.data = .{ .payload = payload },
2993	2998	});
...	...	@@ -3011,10 +3016,10 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
3011	3016	if (self.bin_file.options.pie) {
3012	3017	// TODO we should flag up `x` as GOT symbol entry explicitly rather than as a hack.
3013	3018	_ = try self.addInst(.{
3014		.tag = .lea_rip,
	3019	.tag = .lea,
3015	3020	.ops = (Mir.Ops{
3016	3021	.reg1 = reg,
3017		.flags = 0b01,
	3022	.flags = 0b10,
3018	3023	}).encode(),
3019	3024	.data = .{ .got_entry = @intCast(u32, x) },
3020	3025	});

src/arch/x86_64/Emit.zig+1052-855

...	...	@@ -76,6 +76,16 @@ pub fn emitMir(emit: *Emit) InnerError!void {
76	76	.cmp => try emit.mirArith(.cmp, inst),
77	77	.mov => try emit.mirArith(.mov, inst),
78	78
	79	.adc_mem_imm => try emit.mirArithMemImm(.adc, inst),
	80	.add_mem_imm => try emit.mirArithMemImm(.add, inst),
	81	.sub_mem_imm => try emit.mirArithMemImm(.sub, inst),
	82	.xor_mem_imm => try emit.mirArithMemImm(.xor, inst),
	83	.and_mem_imm => try emit.mirArithMemImm(.@"and", inst),
	84	.or_mem_imm => try emit.mirArithMemImm(.@"or", inst),
	85	.sbb_mem_imm => try emit.mirArithMemImm(.sbb, inst),
	86	.cmp_mem_imm => try emit.mirArithMemImm(.cmp, inst),
	87	.mov_mem_imm => try emit.mirArithMemImm(.mov, inst),
	88
79	89	.adc_scale_src => try emit.mirArithScaleSrc(.adc, inst),
80	90	.add_scale_src => try emit.mirArithScaleSrc(.add, inst),
81	91	.sub_scale_src => try emit.mirArithScaleSrc(.sub, inst),
...	...	@@ -109,7 +119,6 @@ pub fn emitMir(emit: *Emit) InnerError!void {
109	119	.movabs => try emit.mirMovabs(inst),
110	120
111	121	.lea => try emit.mirLea(inst),
112		.lea_rip => try emit.mirLeaRip(inst),
113	122
114	123	.imul_complex => try emit.mirIMulComplex(inst),
115	124
...	...	@@ -170,6 +179,14 @@ fn fail(emit: *Emit, comptime format: []const u8, args: anytype) InnerError {
170	179	return error.EmitFail;
171	180	}
172	181
	182	fn failWithLoweringError(emit: *Emit, err: LoweringError) InnerError {
	183	return switch (err) {
	184	error.RaxOperandExpected => emit.fail("Register.rax expected as destination operand", .{}),
	185	error.OperandSizeMismatch => emit.fail("operand size mismatch", .{}),
	186	else => \|e\| e,
	187	};
	188	}
	189
173	190	fn fixupRelocs(emit: *Emit) InnerError!void {
174	191	// TODO this function currently assumes all relocs via JMP/CALL instructions are 32bit in size.
175	192	// This should be reversed like it is done in aarch64 MIR emit code: start with the smallest
...	...	@@ -185,15 +202,15 @@ fn fixupRelocs(emit: *Emit) InnerError!void {
185	202	}
186	203
187	204	fn mirBrk(emit: *Emit) InnerError!void {
188		return lowerToZoEnc(.brk, emit.code);
	205	return lowerToZoEnc(.brk, emit.code) catch \|err\| emit.failWithLoweringError(err);
189	206	}
190	207
191	208	fn mirNop(emit: *Emit) InnerError!void {
192		return lowerToZoEnc(.nop, emit.code);
	209	return lowerToZoEnc(.nop, emit.code) catch \|err\| emit.failWithLoweringError(err);
193	210	}
194	211
195	212	fn mirSyscall(emit: *Emit) InnerError!void {
196		return lowerToZoEnc(.syscall, emit.code);
	213	return lowerToZoEnc(.syscall, emit.code) catch \|err\| emit.failWithLoweringError(err);
197	214	}
198	215
199	216	fn mirPushPop(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
...	...	@@ -201,18 +218,24 @@ fn mirPushPop(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
201	218	switch (ops.flags) {
202	219	0b00 => {
203	220	// PUSH/POP reg
204		return lowerToOEnc(tag, ops.reg1, emit.code);
	221	return lowerToOEnc(tag, ops.reg1, emit.code) catch \|err\| emit.failWithLoweringError(err);
205	222	},
206	223	0b01 => {
207	224	// PUSH/POP r/m64
208	225	const imm = emit.mir.instructions.items(.data)[inst].imm;
209		return lowerToMEnc(tag, RegisterOrMemory.mem(ops.reg1, imm), emit.code);
	226	const ptr_size: Memory.PtrSize = switch (immOpSize(imm)) {
	227	16 => .word_ptr,
	228	else => .qword_ptr,
	229	};
	230	return lowerToMEnc(tag, RegisterOrMemory.mem(ops.reg1, imm, ptr_size), emit.code) catch \|err\|
	231	emit.failWithLoweringError(err);
210	232	},
211	233	0b10 => {
212	234	// PUSH imm32
213	235	assert(tag == .push);
214	236	const imm = emit.mir.instructions.items(.data)[inst].imm;
215		return lowerToIEnc(.push, imm, emit.code);
	237	return lowerToIEnc(.push, imm, emit.code) catch \|err\|
	238	emit.failWithLoweringError(err);
216	239	},
217	240	0b11 => unreachable,
218	241	}
...	...	@@ -223,7 +246,8 @@ fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Tag, inst: Mir.Inst.I
223	246	if (tag == .push) {
224	247	for (callee_preserved_regs) \|reg, i\| {
225	248	if ((regs >> @intCast(u5, i)) & 1 == 0) continue;
226		try lowerToOEnc(.push, reg, emit.code);
	249	lowerToOEnc(.push, reg, emit.code) catch \|err\|
	250	return emit.failWithLoweringError(err);
227	251	}
228	252	} else {
229	253	// pop in the reverse direction
...	...	@@ -231,7 +255,8 @@ fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Tag, inst: Mir.Inst.I
231	255	while (i > 0) : (i -= 1) {
232	256	const reg = callee_preserved_regs[i - 1];
233	257	if ((regs >> @intCast(u5, i - 1)) & 1 == 0) continue;
234		try lowerToOEnc(.pop, reg, emit.code);
	258	lowerToOEnc(.pop, reg, emit.code) catch \|err\|
	259	return emit.failWithLoweringError(err);
235	260	}
236	261	}
237	262	}
...	...	@@ -242,7 +267,8 @@ fn mirJmpCall(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
242	267	if (flag == 0) {
243	268	const target = emit.mir.instructions.items(.data)[inst].inst;
244	269	const source = emit.code.items.len;
245		try lowerToDEnc(tag, 0, emit.code);
	270	lowerToDEnc(tag, 0, emit.code) catch \|err\|
	271	return emit.failWithLoweringError(err);
246	272	try emit.relocs.append(emit.bin_file.allocator, .{
247	273	.source = source,
248	274	.target = target,
...	...	@@ -254,10 +280,15 @@ fn mirJmpCall(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
254	280	if (ops.reg1 == .none) {
255	281	// JMP/CALL [imm]
256	282	const imm = emit.mir.instructions.items(.data)[inst].imm;
257		return lowerToMEnc(tag, RegisterOrMemory.mem(null, imm), emit.code);
	283	const ptr_size: Memory.PtrSize = switch (immOpSize(imm)) {
	284	16 => .word_ptr,
	285	else => .qword_ptr,
	286	};
	287	return lowerToMEnc(tag, RegisterOrMemory.mem(null, imm, ptr_size), emit.code) catch \|err\|
	288	emit.failWithLoweringError(err);
258	289	}
259	290	// JMP/CALL reg
260		return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code);
	291	return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code) catch \|err\| emit.failWithLoweringError(err);
261	292	}
262	293
263	294	fn mirCondJmp(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
...	...	@@ -283,7 +314,8 @@ fn mirCondJmp(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerErr
283	314	else => unreachable,
284	315	};
285	316	const source = emit.code.items.len;
286		try lowerToDEnc(tag, 0, emit.code);
	317	lowerToDEnc(tag, 0, emit.code) catch \|err\|
	318	return emit.failWithLoweringError(err);
287	319	try emit.relocs.append(emit.bin_file.allocator, .{
288	320	.source = source,
289	321	.target = target,
...	...	@@ -313,7 +345,8 @@ fn mirCondSetByte(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) Inne
313	345	},
314	346	else => unreachable,
315	347	};
316		return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code);
	348	return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1.to8()), emit.code) catch \|err\|
	349	emit.failWithLoweringError(err);
317	350	}
318	351
319	352	fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
...	...	@@ -329,9 +362,11 @@ fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
329	362	if (ops.reg1.to64() == .rax) {
330	363	// TEST rax, imm32
331	364	// I
332		return lowerToIEnc(.@"test", imm, emit.code);
	365	return lowerToIEnc(.@"test", imm, emit.code) catch \|err\|
	366	emit.failWithLoweringError(err);
333	367	}
334		return lowerToMiEnc(.@"test", RegisterOrMemory.reg(ops.reg1), imm, emit.code);
	368	return lowerToMiEnc(.@"test", RegisterOrMemory.reg(ops.reg1), imm, emit.code) catch \|err\|
	369	emit.failWithLoweringError(err);
335	370	}
336	371	// TEST r/m64, r64
337	372	return emit.fail("TODO TEST r/m64, r64", .{});
...	...	@@ -349,207 +384,690 @@ fn mirRet(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
349	384	// RETF imm16
350	385	// I
351	386	const imm = emit.mir.instructions.items(.data)[inst].imm;
352		return lowerToIEnc(.ret_far, imm, emit.code);
	387	return lowerToIEnc(.ret_far, imm, emit.code) catch \|err\| emit.failWithLoweringError(err);
	388	},
	389	0b01 => {
	390	return lowerToZoEnc(.ret_far, emit.code) catch \|err\| emit.failWithLoweringError(err);
353	391	},
354		0b01 => return lowerToZoEnc(.ret_far, emit.code),
355	392	0b10 => {
356	393	// RET imm16
357	394	// I
358	395	const imm = emit.mir.instructions.items(.data)[inst].imm;
359		return lowerToIEnc(.ret_near, imm, emit.code);
	396	return lowerToIEnc(.ret_near, imm, emit.code) catch \|err\| emit.failWithLoweringError(err);
	397	},
	398	0b11 => {
	399	return lowerToZoEnc(.ret_near, emit.code) catch \|err\| emit.failWithLoweringError(err);
360	400	},
361		0b11 => return lowerToZoEnc(.ret_near, emit.code),
362	401	}
363	402	}
364	403
365		const Tag = enum {
366		adc,
367		add,
368		sub,
369		xor,
370		@"and",
371		@"or",
372		sbb,
373		cmp,
374		mov,
375		lea,
376		jmp_near,
377		call_near,
378		push,
379		pop,
380		@"test",
381		brk,
382		nop,
383		imul,
384		syscall,
385		ret_near,
386		ret_far,
387		jo,
388		jno,
389		jb,
390		jbe,
391		jc,
392		jnae,
393		jnc,
394		jae,
395		je,
396		jz,
397		jne,
398		jnz,
399		jna,
400		jnb,
401		jnbe,
402		ja,
403		js,
404		jns,
405		jpe,
406		jp,
407		jpo,
408		jnp,
409		jnge,
410		jl,
411		jge,
412		jnl,
413		jle,
414		jng,
415		jg,
416		jnle,
417		seto,
418		setno,
419		setb,
420		setc,
421		setnae,
422		setnb,
423		setnc,
424		setae,
425		sete,
426		setz,
427		setne,
428		setnz,
429		setbe,
430		setna,
431		seta,
432		setnbe,
433		sets,
434		setns,
435		setp,
436		setpe,
437		setnp,
438		setop,
439		setl,
440		setnge,
441		setnl,
442		setge,
443		setle,
444		setng,
445		setnle,
446		setg,
447
448		fn isSetCC(tag: Tag) bool {
449		return switch (tag) {
450		.seto,
451		.setno,
452		.setb,
453		.setc,
454		.setnae,
455		.setnb,
456		.setnc,
457		.setae,
458		.sete,
459		.setz,
460		.setne,
461		.setnz,
462		.setbe,
463		.setna,
464		.seta,
465		.setnbe,
466		.sets,
467		.setns,
468		.setp,
469		.setpe,
470		.setnp,
471		.setop,
472		.setl,
473		.setnge,
474		.setnl,
475		.setge,
476		.setle,
477		.setng,
478		.setnle,
479		.setg,
480		=> true,
481		else => false,
482		};
	404	fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	405	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	406	switch (ops.flags) {
	407	0b00 => {
	408	if (ops.reg2 == .none) {
	409	// mov reg1, imm32
	410	// MI
	411	const imm = emit.mir.instructions.items(.data)[inst].imm;
	412	return lowerToMiEnc(tag, RegisterOrMemory.reg(ops.reg1), imm, emit.code) catch \|err\|
	413	emit.failWithLoweringError(err);
	414	}
	415	// mov reg1, reg2
	416	// RM
	417	return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code) catch \|err\|
	418	emit.failWithLoweringError(err);
	419	},
	420	0b01 => {
	421	// mov reg1, [reg2 + imm32]
	422	// RM
	423	const imm = emit.mir.instructions.items(.data)[inst].imm;
	424	const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
	425	return lowerToRmEnc(
	426	tag,
	427	ops.reg1,
	428	RegisterOrMemory.mem(src_reg, imm, Memory.PtrSize.fromBits(ops.reg1.size())),
	429	emit.code,
	430	) catch \|err\| emit.failWithLoweringError(err);
	431	},
	432	0b10 => {
	433	if (ops.reg2 == .none) {
	434	return emit.fail("TODO unused variant: mov reg1, none, 0b10", .{});
	435	}
	436	// mov [reg1 + imm32], reg2
	437	// MR
	438	const imm = emit.mir.instructions.items(.data)[inst].imm;
	439	return lowerToMrEnc(
	440	tag,
	441	RegisterOrMemory.mem(ops.reg1, imm, Memory.PtrSize.fromBits(ops.reg2.size())),
	442	ops.reg2,
	443	emit.code,
	444	) catch \|err\| emit.failWithLoweringError(err);
	445	},
	446	0b11 => {
	447	return emit.fail("TODO unused variant: mov reg1, reg2, 0b11", .{});
	448	},
483	449	}
484		};
485
486		const Encoding = enum {
487		/// OP
488		zo,
489
490		/// OP rel32
491		d,
492
493		/// OP r/m64
494		m,
495
496		/// OP r64
497		o,
498
499		/// OP imm32
500		i,
501
502		/// OP r/m64, imm32
503		mi,
504
505		/// OP r/m64, r64
506		mr,
507
508		/// OP r64, r/m64
509		rm,
510
511		/// OP r64, imm64
512		oi,
513
514		/// OP al/ax/eax/rax, moffs
515		fd,
516
517		/// OP moffs, al/ax/eax/rax
518		td,
	450	}
519	451
520		/// OP r64, r/m64, imm32
521		rmi,
522		};
	452	fn mirArithMemImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	453	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	454	assert(ops.reg2 == .none);
	455	const payload = emit.mir.instructions.items(.data)[inst].payload;
	456	const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
	457	const ptr_size: Memory.PtrSize = switch (ops.flags) {
	458	0b00 => .byte_ptr,
	459	0b01 => .word_ptr,
	460	0b10 => .dword_ptr,
	461	0b11 => .qword_ptr,
	462	};
	463	return lowerToMiEnc(
	464	tag,
	465	RegisterOrMemory.mem(ops.reg1, imm_pair.dest_off, ptr_size),
	466	imm_pair.operand,
	467	emit.code,
	468	) catch \|err\| emit.failWithLoweringError(err);
	469	}
523	470
524		const OpCode = union(enum) {
525		one_byte: u8,
526		two_byte: struct { _1: u8, _2: u8 },
	471	inline fn setRexWRegister(reg: Register) bool {
	472	if (reg.size() == 64) return true;
	473	return switch (reg) {
	474	.ah, .bh, .ch, .dh => true,
	475	else => false,
	476	};
	477	}
527	478
528		fn oneByte(opc: u8) OpCode {
529		return .{ .one_byte = opc };
	479	inline fn immOpSize(imm: i64) u8 {
	480	blk: {
	481	_ = math.cast(i8, imm) catch break :blk;
	482	return 8;
530	483	}
531
532		fn twoByte(opc1: u8, opc2: u8) OpCode {
533		return .{ .two_byte = .{ ._1 = opc1, ._2 = opc2 } };
	484	blk: {
	485	_ = math.cast(i16, imm) catch break :blk;
	486	return 16;
534	487	}
535
536		fn encode(opc: OpCode, encoder: Encoder) void {
537		switch (opc) {
538		.one_byte => \|v\| encoder.opcode_1byte(v),
539		.two_byte => \|v\| encoder.opcode_2byte(v._1, v._2),
540		}
	488	blk: {
	489	_ = math.cast(i32, imm) catch break :blk;
	490	return 32;
541	491	}
	492	return 64;
	493	}
542	494
543		fn encodeWithReg(opc: OpCode, encoder: Encoder, reg: Register) void {
544		assert(opc == .one_byte);
545		encoder.opcode_withReg(opc.one_byte, reg.lowId());
	495	// TODO
	496	fn mirArithScaleSrc(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	497	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	498	const scale = ops.flags;
	499	// OP reg1, [reg2 + scale*rcx + imm32]
	500	const opc = getOpCode(tag, .rm, ops.reg1.size() == 8).?;
	501	const imm = emit.mir.instructions.items(.data)[inst].imm;
	502	const encoder = try Encoder.init(emit.code, 8);
	503	encoder.rex(.{
	504	.w = ops.reg1.size() == 64,
	505	.r = ops.reg1.isExtended(),
	506	.b = ops.reg2.isExtended(),
	507	});
	508	opc.encode(encoder);
	509	if (imm <= math.maxInt(i8)) {
	510	encoder.modRm_SIBDisp8(ops.reg1.lowId());
	511	encoder.sib_scaleIndexBaseDisp8(scale, Register.rcx.lowId(), ops.reg2.lowId());
	512	encoder.disp8(@intCast(i8, imm));
	513	} else {
	514	encoder.modRm_SIBDisp32(ops.reg1.lowId());
	515	encoder.sib_scaleIndexBaseDisp32(scale, Register.rcx.lowId(), ops.reg2.lowId());
	516	encoder.disp32(imm);
546	517	}
547		};
	518	}
548	519
549		inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode {
550		switch (enc) {
551		.zo => return switch (tag) {
552		.ret_near => OpCode.oneByte(0xc3),
	520	// TODO
	521	fn mirArithScaleDst(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	522	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	523	const scale = ops.flags;
	524	const imm = emit.mir.instructions.items(.data)[inst].imm;
	525
	526	if (ops.reg2 == .none) {
	527	// OP [reg1 + scale*rax + 0], imm32
	528	const opc = getOpCode(tag, .mi, ops.reg1.size() == 8).?;
	529	const modrm_ext = getModRmExt(tag).?;
	530	const encoder = try Encoder.init(emit.code, 8);
	531	encoder.rex(.{
	532	.w = ops.reg1.size() == 64,
	533	.b = ops.reg1.isExtended(),
	534	});
	535	opc.encode(encoder);
	536	encoder.modRm_SIBDisp0(modrm_ext);
	537	encoder.sib_scaleIndexBase(scale, Register.rax.lowId(), ops.reg1.lowId());
	538	if (imm <= math.maxInt(i8)) {
	539	encoder.imm8(@intCast(i8, imm));
	540	} else if (imm <= math.maxInt(i16)) {
	541	encoder.imm16(@intCast(i16, imm));
	542	} else {
	543	encoder.imm32(imm);
	544	}
	545	return;
	546	}
	547
	548	// OP [reg1 + scale*rax + imm32], reg2
	549	const opc = getOpCode(tag, .mr, ops.reg1.size() == 8).?;
	550	const encoder = try Encoder.init(emit.code, 8);
	551	encoder.rex(.{
	552	.w = ops.reg1.size() == 64,
	553	.r = ops.reg2.isExtended(),
	554	.b = ops.reg1.isExtended(),
	555	});
	556	opc.encode(encoder);
	557	if (imm <= math.maxInt(i8)) {
	558	encoder.modRm_SIBDisp8(ops.reg2.lowId());
	559	encoder.sib_scaleIndexBaseDisp8(scale, Register.rax.lowId(), ops.reg1.lowId());
	560	encoder.disp8(@intCast(i8, imm));
	561	} else {
	562	encoder.modRm_SIBDisp32(ops.reg2.lowId());
	563	encoder.sib_scaleIndexBaseDisp32(scale, Register.rax.lowId(), ops.reg1.lowId());
	564	encoder.disp32(imm);
	565	}
	566	}
	567
	568	// TODO
	569	fn mirArithScaleImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	570	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	571	const scale = ops.flags;
	572	const payload = emit.mir.instructions.items(.data)[inst].payload;
	573	const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
	574	const opc = getOpCode(tag, .mi, ops.reg1.size() == 8).?;
	575	const modrm_ext = getModRmExt(tag).?;
	576	const encoder = try Encoder.init(emit.code, 2);
	577	encoder.rex(.{
	578	.w = ops.reg1.size() == 64,
	579	.b = ops.reg1.isExtended(),
	580	});
	581	opc.encode(encoder);
	582	if (imm_pair.dest_off <= math.maxInt(i8)) {
	583	encoder.modRm_SIBDisp8(modrm_ext);
	584	encoder.sib_scaleIndexBaseDisp8(scale, Register.rax.lowId(), ops.reg1.lowId());
	585	encoder.disp8(@intCast(i8, imm_pair.dest_off));
	586	} else {
	587	encoder.modRm_SIBDisp32(modrm_ext);
	588	encoder.sib_scaleIndexBaseDisp32(scale, Register.rax.lowId(), ops.reg1.lowId());
	589	encoder.disp32(imm_pair.dest_off);
	590	}
	591	encoder.imm32(imm_pair.operand);
	592	}
	593
	594	fn mirMovabs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	595	const tag = emit.mir.instructions.items(.tag)[inst];
	596	assert(tag == .movabs);
	597	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	598	const imm: i64 = if (ops.reg1.size() == 64) blk: {
	599	const payload = emit.mir.instructions.items(.data)[inst].payload;
	600	const imm = emit.mir.extraData(Mir.Imm64, payload).data;
	601	break :blk @bitCast(i64, imm.decode());
	602	} else emit.mir.instructions.items(.data)[inst].imm;
	603	if (ops.flags == 0b00) {
	604	// movabs reg, imm64
	605	// OI
	606	return lowerToOiEnc(.mov, ops.reg1, imm, emit.code) catch \|err\| emit.failWithLoweringError(err);
	607	}
	608	if (ops.reg1 == .none) {
	609	// movabs moffs64, rax
	610	// TD
	611	return lowerToTdEnc(.mov, imm, ops.reg2, emit.code) catch \|err\| emit.failWithLoweringError(err);
	612	}
	613	// movabs rax, moffs64
	614	// FD
	615	return lowerToFdEnc(.mov, ops.reg1, imm, emit.code) catch \|err\| emit.failWithLoweringError(err);
	616	}
	617
	618	fn mirIMulComplex(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	619	const tag = emit.mir.instructions.items(.tag)[inst];
	620	assert(tag == .imul_complex);
	621	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	622	switch (ops.flags) {
	623	0b00 => {
	624	return lowerToRmEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code) catch \|err\|
	625	emit.failWithLoweringError(err);
	626	},
	627	0b10 => {
	628	const imm = emit.mir.instructions.items(.data)[inst].imm;
	629	return lowerToRmiEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), imm, emit.code) catch \|err\|
	630	emit.failWithLoweringError(err);
	631	},
	632	else => return emit.fail("TODO implement imul", .{}),
	633	}
	634	}
	635
	636	fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	637	const tag = emit.mir.instructions.items(.tag)[inst];
	638	assert(tag == .lea);
	639	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
	640	switch (ops.flags) {
	641	0b00 => {
	642	// lea reg1, [reg2 + imm32]
	643	// RM
	644	const imm = emit.mir.instructions.items(.data)[inst].imm;
	645	const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
	646	return lowerToRmEnc(
	647	.lea,
	648	ops.reg1,
	649	RegisterOrMemory.mem(src_reg, imm, Memory.PtrSize.fromBits(ops.reg1.size())),
	650	emit.code,
	651	) catch \|err\| emit.failWithLoweringError(err);
	652	},
	653	0b01 => {
	654	// lea reg1, [rip + imm32]
	655	// RM
	656	const start_offset = emit.code.items.len;
	657	lowerToRmEnc(
	658	.lea,
	659	ops.reg1,
	660	RegisterOrMemory.rip(0, Memory.PtrSize.fromBits(ops.reg1.size())),
	661	emit.code,
	662	) catch \|err\| return emit.failWithLoweringError(err);
	663	const end_offset = emit.code.items.len;
	664	// Backpatch the displacement
	665	const payload = emit.mir.instructions.items(.data)[inst].payload;
	666	const imm = emit.mir.extraData(Mir.Imm64, payload).data.decode();
	667	const disp = @intCast(i32, @intCast(i64, imm) - @intCast(i64, end_offset - start_offset));
	668	mem.writeIntLittle(i32, emit.code.items[end_offset - 4 ..][0..4], disp);
	669	},
	670	0b10 => {
	671	// lea reg1, [rip + reloc]
	672	// RM
	673	lowerToRmEnc(
	674	.lea,
	675	ops.reg1,
	676	RegisterOrMemory.rip(0, Memory.PtrSize.fromBits(ops.reg1.size())),
	677	emit.code,
	678	) catch \|err\| return emit.failWithLoweringError(err);
	679	const end_offset = emit.code.items.len;
	680	const got_entry = emit.mir.instructions.items(.data)[inst].got_entry;
	681	if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
	682	// TODO I think the reloc might be in the wrong place.
	683	const decl = macho_file.active_decl.?;
	684	try decl.link.macho.relocs.append(emit.bin_file.allocator, .{
	685	.offset = @intCast(u32, end_offset - 4),
	686	.target = .{ .local = got_entry },
	687	.addend = 0,
	688	.subtractor = null,
	689	.pcrel = true,
	690	.length = 2,
	691	.@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_GOT),
	692	});
	693	} else {
	694	return emit.fail(
	695	"TODO implement lea reg, [rip + reloc] for linking backends different than MachO",
	696	.{},
	697	);
	698	}
	699	},
	700	0b11 => return emit.fail("TODO unused variant lea reg1, reg2, 0b11", .{}),
	701	}
	702	}
	703
	704	fn mirCallExtern(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	705	const tag = emit.mir.instructions.items(.tag)[inst];
	706	assert(tag == .call_extern);
	707	const n_strx = emit.mir.instructions.items(.data)[inst].extern_fn;
	708	const offset = blk: {
	709	// callq
	710	lowerToDEnc(.call_near, 0, emit.code) catch \|err\|
	711	return emit.failWithLoweringError(err);
	712	break :blk @intCast(u32, emit.code.items.len) - 4;
	713	};
	714	if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
	715	// Add relocation to the decl.
	716	try macho_file.active_decl.?.link.macho.relocs.append(emit.bin_file.allocator, .{
	717	.offset = offset,
	718	.target = .{ .global = n_strx },
	719	.addend = 0,
	720	.subtractor = null,
	721	.pcrel = true,
	722	.length = 2,
	723	.@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_BRANCH),
	724	});
	725	} else {
	726	return emit.fail("TODO implement call_extern for linking backends different than MachO", .{});
	727	}
	728	}
	729
	730	fn mirDbgLine(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	731	const tag = emit.mir.instructions.items(.tag)[inst];
	732	assert(tag == .dbg_line);
	733	const payload = emit.mir.instructions.items(.data)[inst].payload;
	734	const dbg_line_column = emit.mir.extraData(Mir.DbgLineColumn, payload).data;
	735	try emit.dbgAdvancePCAndLine(dbg_line_column.line, dbg_line_column.column);
	736	}
	737
	738	fn dbgAdvancePCAndLine(emit: *Emit, line: u32, column: u32) InnerError!void {
	739	const delta_line = @intCast(i32, line) - @intCast(i32, emit.prev_di_line);
	740	const delta_pc: usize = emit.code.items.len - emit.prev_di_pc;
	741	switch (emit.debug_output) {
	742	.dwarf => \|dbg_out\| {
	743	// TODO Look into using the DWARF special opcodes to compress this data.
	744	// It lets you emit single-byte opcodes that add different numbers to
	745	// both the PC and the line number at the same time.
	746	try dbg_out.dbg_line.ensureUnusedCapacity(11);
	747	dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_pc);
	748	leb128.writeULEB128(dbg_out.dbg_line.writer(), delta_pc) catch unreachable;
	749	if (delta_line != 0) {
	750	dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_line);
	751	leb128.writeILEB128(dbg_out.dbg_line.writer(), delta_line) catch unreachable;
	752	}
	753	dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.copy);
	754	emit.prev_di_pc = emit.code.items.len;
	755	emit.prev_di_line = line;
	756	emit.prev_di_column = column;
	757	emit.prev_di_pc = emit.code.items.len;
	758	},
	759	.plan9 => \|dbg_out\| {
	760	if (delta_pc <= 0) return; // only do this when the pc changes
	761	// we have already checked the target in the linker to make sure it is compatable
	762	const quant = @import("../../link/Plan9/aout.zig").getPCQuant(emit.target.cpu.arch) catch unreachable;
	763
	764	// increasing the line number
	765	try @import("../../link/Plan9.zig").changeLine(dbg_out.dbg_line, delta_line);
	766	// increasing the pc
	767	const d_pc_p9 = @intCast(i64, delta_pc) - quant;
	768	if (d_pc_p9 > 0) {
	769	// minus one because if its the last one, we want to leave space to change the line which is one quanta
	770	try dbg_out.dbg_line.append(@intCast(u8, @divExact(d_pc_p9, quant) + 128) - quant);
	771	if (dbg_out.pcop_change_index.*) \|pci\|
	772	dbg_out.dbg_line.items[pci] += 1;
	773	dbg_out.pcop_change_index.* = @intCast(u32, dbg_out.dbg_line.items.len - 1);
	774	} else if (d_pc_p9 == 0) {
	775	// we don't need to do anything, because adding the quant does it for us
	776	} else unreachable;
	777	if (dbg_out.start_line.* == null)
	778	dbg_out.start_line.* = emit.prev_di_line;
	779	dbg_out.end_line.* = line;
	780	// only do this if the pc changed
	781	emit.prev_di_line = line;
	782	emit.prev_di_column = column;
	783	emit.prev_di_pc = emit.code.items.len;
	784	},
	785	.none => {},
	786	}
	787	}
	788
	789	fn mirDbgPrologueEnd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	790	const tag = emit.mir.instructions.items(.tag)[inst];
	791	assert(tag == .dbg_prologue_end);
	792	switch (emit.debug_output) {
	793	.dwarf => \|dbg_out\| {
	794	try dbg_out.dbg_line.append(DW.LNS.set_prologue_end);
	795	try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
	796	},
	797	.plan9 => {},
	798	.none => {},
	799	}
	800	}
	801
	802	fn mirDbgEpilogueBegin(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	803	const tag = emit.mir.instructions.items(.tag)[inst];
	804	assert(tag == .dbg_epilogue_begin);
	805	switch (emit.debug_output) {
	806	.dwarf => \|dbg_out\| {
	807	try dbg_out.dbg_line.append(DW.LNS.set_epilogue_begin);
	808	try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
	809	},
	810	.plan9 => {},
	811	.none => {},
	812	}
	813	}
	814
	815	fn mirArgDbgInfo(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	816	const tag = emit.mir.instructions.items(.tag)[inst];
	817	assert(tag == .arg_dbg_info);
	818	const payload = emit.mir.instructions.items(.data)[inst].payload;
	819	const arg_dbg_info = emit.mir.extraData(Mir.ArgDbgInfo, payload).data;
	820	const mcv = emit.mir.function.args[arg_dbg_info.arg_index];
	821	try emit.genArgDbgInfo(arg_dbg_info.air_inst, mcv);
	822	}
	823
	824	fn genArgDbgInfo(emit: *Emit, inst: Air.Inst.Index, mcv: MCValue) !void {
	825	const ty_str = emit.mir.function.air.instructions.items(.data)[inst].ty_str;
	826	const zir = &emit.mir.function.mod_fn.owner_decl.getFileScope().zir;
	827	const name = zir.nullTerminatedString(ty_str.str);
	828	const name_with_null = name.ptr[0 .. name.len + 1];
	829	const ty = emit.mir.function.air.getRefType(ty_str.ty);
	830
	831	switch (mcv) {
	832	.register => \|reg\| {
	833	switch (emit.debug_output) {
	834	.dwarf => \|dbg_out\| {
	835	try dbg_out.dbg_info.ensureUnusedCapacity(3);
	836	dbg_out.dbg_info.appendAssumeCapacity(link.File.Elf.abbrev_parameter);
	837	dbg_out.dbg_info.appendSliceAssumeCapacity(&[2]u8{ // DW.AT.location, DW.FORM.exprloc
	838	1, // ULEB128 dwarf expression length
	839	reg.dwarfLocOp(),
	840	});
	841	try dbg_out.dbg_info.ensureUnusedCapacity(5 + name_with_null.len);
	842	try emit.addDbgInfoTypeReloc(ty); // DW.AT.type, DW.FORM.ref4
	843	dbg_out.dbg_info.appendSliceAssumeCapacity(name_with_null); // DW.AT.name, DW.FORM.string
	844	},
	845	.plan9 => {},
	846	.none => {},
	847	}
	848	},
	849	.stack_offset => {
	850	switch (emit.debug_output) {
	851	.dwarf => {},
	852	.plan9 => {},
	853	.none => {},
	854	}
	855	},
	856	else => {},
	857	}
	858	}
	859
	860	/// Adds a Type to the .debug_info at the current position. The bytes will be populated later,
	861	/// after codegen for this symbol is done.
	862	fn addDbgInfoTypeReloc(emit: *Emit, ty: Type) !void {
	863	switch (emit.debug_output) {
	864	.dwarf => \|dbg_out\| {
	865	assert(ty.hasCodeGenBits());
	866	const index = dbg_out.dbg_info.items.len;
	867	try dbg_out.dbg_info.resize(index + 4); // DW.AT.type, DW.FORM.ref4
	868
	869	const gop = try dbg_out.dbg_info_type_relocs.getOrPut(emit.bin_file.allocator, ty);
	870	if (!gop.found_existing) {
	871	gop.value_ptr.* = .{
	872	.off = undefined,
	873	.relocs = .{},
	874	};
	875	}
	876	try gop.value_ptr.relocs.append(emit.bin_file.allocator, @intCast(u32, index));
	877	},
	878	.plan9 => {},
	879	.none => {},
	880	}
	881	}
	882
	883	const Tag = enum {
	884	adc,
	885	add,
	886	sub,
	887	xor,
	888	@"and",
	889	@"or",
	890	sbb,
	891	cmp,
	892	mov,
	893	lea,
	894	jmp_near,
	895	call_near,
	896	push,
	897	pop,
	898	@"test",
	899	brk,
	900	nop,
	901	imul,
	902	syscall,
	903	ret_near,
	904	ret_far,
	905	jo,
	906	jno,
	907	jb,
	908	jbe,
	909	jc,
	910	jnae,
	911	jnc,
	912	jae,
	913	je,
	914	jz,
	915	jne,
	916	jnz,
	917	jna,
	918	jnb,
	919	jnbe,
	920	ja,
	921	js,
	922	jns,
	923	jpe,
	924	jp,
	925	jpo,
	926	jnp,
	927	jnge,
	928	jl,
	929	jge,
	930	jnl,
	931	jle,
	932	jng,
	933	jg,
	934	jnle,
	935	seto,
	936	setno,
	937	setb,
	938	setc,
	939	setnae,
	940	setnb,
	941	setnc,
	942	setae,
	943	sete,
	944	setz,
	945	setne,
	946	setnz,
	947	setbe,
	948	setna,
	949	seta,
	950	setnbe,
	951	sets,
	952	setns,
	953	setp,
	954	setpe,
	955	setnp,
	956	setop,
	957	setl,
	958	setnge,
	959	setnl,
	960	setge,
	961	setle,
	962	setng,
	963	setnle,
	964	setg,
	965
	966	fn isSetCC(tag: Tag) bool {
	967	return switch (tag) {
	968	.seto,
	969	.setno,
	970	.setb,
	971	.setc,
	972	.setnae,
	973	.setnb,
	974	.setnc,
	975	.setae,
	976	.sete,
	977	.setz,
	978	.setne,
	979	.setnz,
	980	.setbe,
	981	.setna,
	982	.seta,
	983	.setnbe,
	984	.sets,
	985	.setns,
	986	.setp,
	987	.setpe,
	988	.setnp,
	989	.setop,
	990	.setl,
	991	.setnge,
	992	.setnl,
	993	.setge,
	994	.setle,
	995	.setng,
	996	.setnle,
	997	.setg,
	998	=> true,
	999	else => false,
	1000	};
	1001	}
	1002	};
	1003
	1004	const Encoding = enum {
	1005	/// OP
	1006	zo,
	1007
	1008	/// OP rel32
	1009	d,
	1010
	1011	/// OP r/m64
	1012	m,
	1013
	1014	/// OP r64
	1015	o,
	1016
	1017	/// OP imm32
	1018	i,
	1019
	1020	/// OP r/m64, imm32
	1021	mi,
	1022
	1023	/// OP r/m64, r64
	1024	mr,
	1025
	1026	/// OP r64, r/m64
	1027	rm,
	1028
	1029	/// OP r64, imm64
	1030	oi,
	1031
	1032	/// OP al/ax/eax/rax, moffs
	1033	fd,
	1034
	1035	/// OP moffs, al/ax/eax/rax
	1036	td,
	1037
	1038	/// OP r64, r/m64, imm32
	1039	rmi,
	1040	};
	1041
	1042	const OpCode = union(enum) {
	1043	one_byte: u8,
	1044	two_byte: struct { _1: u8, _2: u8 },
	1045
	1046	fn oneByte(opc: u8) OpCode {
	1047	return .{ .one_byte = opc };
	1048	}
	1049
	1050	fn twoByte(opc1: u8, opc2: u8) OpCode {
	1051	return .{ .two_byte = .{ ._1 = opc1, ._2 = opc2 } };
	1052	}
	1053
	1054	fn encode(opc: OpCode, encoder: Encoder) void {
	1055	switch (opc) {
	1056	.one_byte => \|v\| encoder.opcode_1byte(v),
	1057	.two_byte => \|v\| encoder.opcode_2byte(v._1, v._2),
	1058	}
	1059	}
	1060
	1061	fn encodeWithReg(opc: OpCode, encoder: Encoder, reg: Register) void {
	1062	assert(opc == .one_byte);
	1063	encoder.opcode_withReg(opc.one_byte, reg.lowId());
	1064	}
	1065	};
	1066
	1067	inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode {
	1068	switch (enc) {
	1069	.zo => return switch (tag) {
	1070	.ret_near => OpCode.oneByte(0xc3),
553	1071	.ret_far => OpCode.oneByte(0xcb),
554	1072	.brk => OpCode.oneByte(0xcc),
555	1073	.nop => OpCode.oneByte(0x90),
...	...	@@ -722,7 +1240,35 @@ const Memory = struct {
722	1240	reg: ?Register,
723	1241	rip: bool = false,
724	1242	disp: i32,
	1243	ptr_size: PtrSize,
725	1244	sib: ?ScaleIndexBase = null,
	1245
	1246	const PtrSize = enum {
	1247	byte_ptr,
	1248	word_ptr,
	1249	dword_ptr,
	1250	qword_ptr,
	1251
	1252	fn fromBits(in_bits: u64) PtrSize {
	1253	return switch (in_bits) {
	1254	8 => .byte_ptr,
	1255	16 => .word_ptr,
	1256	32 => .dword_ptr,
	1257	64 => .qword_ptr,
	1258	else => unreachable,
	1259	};
	1260	}
	1261
	1262	/// Returns size in bits.
	1263	fn size(ptr_size: PtrSize) u64 {
	1264	return switch (ptr_size) {
	1265	.byte_ptr => 8,
	1266	.word_ptr => 16,
	1267	.dword_ptr => 32,
	1268	.qword_ptr => 64,
	1269	};
	1270	}
	1271	};
726	1272	};
727	1273
728	1274	const RegisterOrMemory = union(enum) {
...	...	@@ -733,33 +1279,42 @@ const RegisterOrMemory = union(enum) {
733	1279	return .{ .register = register };
734	1280	}
735	1281
736		fn mem(register: ?Register, disp: i32) RegisterOrMemory {
	1282	fn mem(register: ?Register, disp: i32, ptr_size: Memory.PtrSize) RegisterOrMemory {
737	1283	return .{
738	1284	.memory = .{
739	1285	.reg = register,
740	1286	.disp = disp,
	1287	.ptr_size = ptr_size,
741	1288	},
742	1289	};
743	1290	}
744	1291
745		fn rip(disp: i32) RegisterOrMemory {
	1292	fn rip(disp: i32, ptr_size: Memory.PtrSize) RegisterOrMemory {
746	1293	return .{
747	1294	.memory = .{
748	1295	.reg = null,
749	1296	.rip = true,
750	1297	.disp = disp,
	1298	.ptr_size = ptr_size,
751	1299	},
752	1300	};
753	1301	}
754	1302	};
755	1303
756		fn lowerToZoEnc(tag: Tag, code: *std.ArrayList(u8)) InnerError!void {
	1304	const LoweringError = error{
	1305	OutOfMemory,
	1306	Overflow,
	1307	OperandSizeMismatch,
	1308	RaxOperandExpected,
	1309	};
	1310
	1311	fn lowerToZoEnc(tag: Tag, code: *std.ArrayList(u8)) LoweringError!void {
757	1312	const opc = getOpCode(tag, .zo, false).?;
758	1313	const encoder = try Encoder.init(code, 1);
759	1314	opc.encode(encoder);
760	1315	}
761	1316
762		fn lowerToIEnc(tag: Tag, imm: i32, code: *std.ArrayList(u8)) InnerError!void {
	1317	fn lowerToIEnc(tag: Tag, imm: i32, code: *std.ArrayList(u8)) LoweringError!void {
763	1318	if (tag == .ret_far or tag == .ret_near) {
764	1319	const encoder = try Encoder.init(code, 3);
765	1320	const opc = getOpCode(tag, .i, false).?;
...	...	@@ -782,8 +1337,10 @@ fn lowerToIEnc(tag: Tag, imm: i32, code: *std.ArrayList(u8)) InnerError!void {
782	1337	}
783	1338	}
784	1339
785		fn lowerToOEnc(tag: Tag, reg: Register, code: *std.ArrayList(u8)) InnerError!void {
786		if (reg.size() != 16 and reg.size() != 64) return error.EmitFail; // TODO correct for push/pop, but is it universal?
	1340	fn lowerToOEnc(tag: Tag, reg: Register, code: *std.ArrayList(u8)) LoweringError!void {
	1341	if (reg.size() != 16 and reg.size() != 64) {
	1342	return error.OperandSizeMismatch; // TODO correct for push/pop, but is it universal?
	1343	}
787	1344	const opc = getOpCode(tag, .o, false).?;
788	1345	const encoder = try Encoder.init(code, 3);
789	1346	if (reg.size() == 16) {
...	...	@@ -796,39 +1353,54 @@ fn lowerToOEnc(tag: Tag, reg: Register, code: *std.ArrayList(u8)) InnerError!voi
796	1353	opc.encodeWithReg(encoder, reg);
797	1354	}
798	1355
799		fn lowerToDEnc(tag: Tag, imm: i32, code: *std.ArrayList(u8)) InnerError!void {
	1356	fn lowerToDEnc(tag: Tag, imm: i32, code: *std.ArrayList(u8)) LoweringError!void {
800	1357	const opc = getOpCode(tag, .d, false).?;
801	1358	const encoder = try Encoder.init(code, 6);
802	1359	opc.encode(encoder);
803	1360	encoder.imm32(imm);
804	1361	}
805	1362
806		fn lowerToMEnc(tag: Tag, reg_or_mem: RegisterOrMemory, code: *std.ArrayList(u8)) InnerError!void {
	1363	fn lowerToMEnc(tag: Tag, reg_or_mem: RegisterOrMemory, code: *std.ArrayList(u8)) LoweringError!void {
807	1364	const opc = getOpCode(tag, .m, false).?;
808	1365	const modrm_ext = getModRmExt(tag).?;
809	1366	switch (reg_or_mem) {
810	1367	.register => \|reg\| {
811		// TODO clean this up!
812		if (reg.size() != 64) {
813		if (reg.size() != 8 and !tag.isSetCC()) return error.EmitFail;
	1368	const op_size_mismatch = blk: {
	1369	if (tag.isSetCC() and reg.size() == 8)
	1370	break :blk false;
	1371	break :blk reg.size() != 64 and reg.size() != 16;
	1372	};
	1373	if (op_size_mismatch) {
	1374	return error.OperandSizeMismatch;
	1375	}
	1376	const encoder = try Encoder.init(code, 4);
	1377	if (reg.size() == 16) {
	1378	encoder.opcode_1byte(0x66);
814	1379	}
815		const encoder = try Encoder.init(code, 3);
816	1380	encoder.rex(.{
817		.w = tag.isSetCC(),
	1381	.w = switch (reg) {
	1382	.ah, .bh, .ch, .dh => true,
	1383	else => false,
	1384	},
818	1385	.b = reg.isExtended(),
819	1386	});
820	1387	opc.encode(encoder);
821	1388	encoder.modRm_direct(modrm_ext, reg.lowId());
822	1389	},
823	1390	.memory => \|mem_op\| {
	1391	if (mem_op.ptr_size != .qword_ptr and mem_op.ptr_size != .word_ptr) {
	1392	return error.OperandSizeMismatch;
	1393	}
824	1394	const encoder = try Encoder.init(code, 8);
	1395	if (mem_op.ptr_size == .word_ptr) {
	1396	encoder.opcode_1byte(0x66);
	1397	}
825	1398	if (mem_op.reg) \|reg\| {
826		// TODO clean this up!
827	1399	if (reg.size() != 64) {
828		if (reg.size() != 8 and !tag.isSetCC()) return error.EmitFail;
	1400	return error.OperandSizeMismatch;
829	1401	}
830	1402	encoder.rex(.{
831		.w = tag.isSetCC(),
	1403	.w = false,
832	1404	.b = reg.isExtended(),
833	1405	});
834	1406	opc.encode(encoder);
...	...	@@ -870,17 +1442,21 @@ fn lowerToMEnc(tag: Tag, reg_or_mem: RegisterOrMemory, code: *std.ArrayList(u8))
870	1442	}
871	1443	}
872	1444
873		fn lowerToTdEnc(tag: Tag, moffs: i64, reg: Register, code: *std.ArrayList(u8)) InnerError!void {
	1445	fn lowerToTdEnc(tag: Tag, moffs: i64, reg: Register, code: *std.ArrayList(u8)) LoweringError!void {
874	1446	return lowerToTdFdEnc(tag, reg, moffs, code, true);
875	1447	}
876	1448
877		fn lowerToFdEnc(tag: Tag, reg: Register, moffs: i64, code: *std.ArrayList(u8)) InnerError!void {
	1449	fn lowerToFdEnc(tag: Tag, reg: Register, moffs: i64, code: *std.ArrayList(u8)) LoweringError!void {
878	1450	return lowerToTdFdEnc(tag, reg, moffs, code, false);
879	1451	}
880	1452
881		fn lowerToTdFdEnc(tag: Tag, reg: Register, moffs: i64, code: *std.ArrayList(u8), td: bool) InnerError!void {
882		if (reg.lowId() != Register.rax.lowId()) return error.EmitFail;
883		if (reg.size() != immOpSize(moffs)) return error.EmitFail;
	1453	fn lowerToTdFdEnc(tag: Tag, reg: Register, moffs: i64, code: *std.ArrayList(u8), td: bool) LoweringError!void {
	1454	if (reg.lowId() != Register.rax.lowId()) {
	1455	return error.RaxOperandExpected;
	1456	}
	1457	if (reg.size() != immOpSize(moffs)) {
	1458	return error.OperandSizeMismatch;
	1459	}
884	1460	const opc = if (td)
885	1461	getOpCode(tag, .td, reg.size() == 8).?
886	1462	else
...	...	@@ -890,7 +1466,7 @@ fn lowerToTdFdEnc(tag: Tag, reg: Register, moffs: i64, code: *std.ArrayList(u8),
890	1466	encoder.opcode_1byte(0x66);
891	1467	}
892	1468	encoder.rex(.{
893		.w = reg.size() == 64,
	1469	.w = setRexWRegister(reg),
894	1470	});
895	1471	opc.encode(encoder);
896	1472	switch (reg.size()) {
...	...	@@ -913,15 +1489,17 @@ fn lowerToTdFdEnc(tag: Tag, reg: Register, moffs: i64, code: *std.ArrayList(u8),
913	1489	}
914	1490	}
915	1491
916		fn lowerToOiEnc(tag: Tag, reg: Register, imm: i64, code: *std.ArrayList(u8)) InnerError!void {
917		if (reg.size() != immOpSize(imm)) return error.EmitFail;
	1492	fn lowerToOiEnc(tag: Tag, reg: Register, imm: i64, code: *std.ArrayList(u8)) LoweringError!void {
	1493	if (reg.size() != immOpSize(imm)) {
	1494	return error.OperandSizeMismatch;
	1495	}
918	1496	const opc = getOpCode(tag, .oi, reg.size() == 8).?;
919	1497	const encoder = try Encoder.init(code, 10);
920	1498	if (reg.size() == 16) {
921	1499	encoder.opcode_1byte(0x66);
922	1500	}
923	1501	encoder.rex(.{
924		.w = reg.size() == 64,
	1502	.w = setRexWRegister(reg),
925	1503	.b = reg.isExtended(),
926	1504	});
927	1505	opc.encodeWithReg(encoder, reg);
...	...	@@ -945,7 +1523,7 @@ fn lowerToOiEnc(tag: Tag, reg: Register, imm: i64, code: *std.ArrayList(u8)) Inn
945	1523	}
946	1524	}
947	1525
948		fn lowerToMiEnc(tag: Tag, reg_or_mem: RegisterOrMemory, imm: i32, code: *std.ArrayList(u8)) InnerError!void {
	1526	fn lowerToMiEnc(tag: Tag, reg_or_mem: RegisterOrMemory, imm: i32, code: *std.ArrayList(u8)) LoweringError!void {
949	1527	const modrm_ext = getModRmExt(tag).?;
950	1528	switch (reg_or_mem) {
951	1529	.register => \|dst_reg\| {
...	...	@@ -958,7 +1536,7 @@ fn lowerToMiEnc(tag: Tag, reg_or_mem: RegisterOrMemory, imm: i32, code: *std.Arr
958	1536	encoder.opcode_1byte(0x66);
959	1537	}
960	1538	encoder.rex(.{
961		.w = dst_reg.size() == 64,
	1539	.w = setRexWRegister(dst_reg),
962	1540	.b = dst_reg.isExtended(),
963	1541	});
964	1542	opc.encode(encoder);
...	...	@@ -977,18 +1555,17 @@ fn lowerToMiEnc(tag: Tag, reg_or_mem: RegisterOrMemory, imm: i32, code: *std.Arr
977	1555	}
978	1556	},
979	1557	.memory => \|dst_mem\| {
980		const opc = getOpCode(tag, .mi, false).?;
	1558	const opc = getOpCode(tag, .mi, dst_mem.ptr_size == .byte_ptr).?;
981	1559	const encoder = try Encoder.init(code, 12);
	1560	if (dst_mem.ptr_size == .word_ptr) {
	1561	encoder.opcode_1byte(0x66);
	1562	}
982	1563	if (dst_mem.reg) \|dst_reg\| {
983		// Register dst_reg can either be 64bit or 32bit in size.
984		// TODO for memory operand, immediate operand pair, we currently
985		// have no way of flagging whether the immediate can be 8-, 16- or
986		// 32-bit and whether the corresponding memory operand is respectively
987		// a byte, word or dword ptr.
988		// TODO we currently don't have a way to flag imm32 64bit sign extended
989		if (dst_reg.size() != 64) return error.EmitFail;
	1564	if (dst_reg.size() != 64) {
	1565	return error.OperandSizeMismatch;
	1566	}
990	1567	encoder.rex(.{
991		.w = false,
	1568	.w = dst_mem.ptr_size == .qword_ptr,
992	1569	.b = dst_reg.isExtended(),
993	1570	});
994	1571	opc.encode(encoder);
...	...	@@ -1026,7 +1603,19 @@ fn lowerToMiEnc(tag: Tag, reg_or_mem: RegisterOrMemory, imm: i32, code: *std.Arr
1026	1603	}
1027	1604	encoder.disp32(dst_mem.disp);
1028	1605	}
1029		encoder.imm32(imm);
	1606	switch (dst_mem.ptr_size) {
	1607	.byte_ptr => {
	1608	const imm8 = try math.cast(i8, imm);
	1609	encoder.imm8(imm8);
	1610	},
	1611	.word_ptr => {
	1612	const imm16 = try math.cast(i16, imm);
	1613	encoder.imm16(imm16);
	1614	},
	1615	.dword_ptr, .qword_ptr => {
	1616	encoder.imm32(imm);
	1617	},
	1618	}
1030	1619	},
1031	1620	}
1032	1621	}
...	...	@@ -1036,14 +1625,16 @@ fn lowerToRmEnc(
1036	1625	reg: Register,
1037	1626	reg_or_mem: RegisterOrMemory,
1038	1627	code: *std.ArrayList(u8),
1039		) InnerError!void {
	1628	) LoweringError!void {
1040	1629	const opc = getOpCode(tag, .rm, reg.size() == 8).?;
1041	1630	switch (reg_or_mem) {
1042	1631	.register => \|src_reg\| {
1043		if (reg.size() != src_reg.size()) return error.EmitFail;
	1632	if (reg.size() != src_reg.size()) {
	1633	return error.OperandSizeMismatch;
	1634	}
1044	1635	const encoder = try Encoder.init(code, 3);
1045	1636	encoder.rex(.{
1046		.w = reg.size() == 64,
	1637	.w = setRexWRegister(reg) or setRexWRegister(src_reg),
1047	1638	.r = reg.isExtended(),
1048	1639	.b = src_reg.isExtended(),
1049	1640	});
...	...	@@ -1051,185 +1642,21 @@ fn lowerToRmEnc(
1051	1642	encoder.modRm_direct(reg.lowId(), src_reg.lowId());
1052	1643	},
1053	1644	.memory => \|src_mem\| {
1054		const encoder = try Encoder.init(code, 9);
1055		if (reg.size() == 16) {
1056		encoder.opcode_1byte(0x66);
1057		}
1058		if (src_mem.reg) \|src_reg\| {
1059		// TODO handle 32-bit base register - requires prefix 0x67
1060		// Intel Manual, Vol 1, chapter 3.6 and 3.6.1
1061		if (src_reg.size() != 64) return error.EmitFail;
1062		encoder.rex(.{
1063		.w = reg.size() == 64,
1064		.r = reg.isExtended(),
1065		.b = src_reg.isExtended(),
1066		});
1067		opc.encode(encoder);
1068		if (src_reg.lowId() == 4) {
1069		if (src_mem.disp == 0) {
1070		encoder.modRm_SIBDisp0(reg.lowId());
1071		encoder.sib_base(src_reg.lowId());
1072		} else if (immOpSize(src_mem.disp) == 8) {
1073		encoder.modRm_SIBDisp8(reg.lowId());
1074		encoder.sib_baseDisp8(src_reg.lowId());
1075		encoder.disp8(@intCast(i8, src_mem.disp));
1076		} else {
1077		encoder.modRm_SIBDisp32(reg.lowId());
1078		encoder.sib_baseDisp32(src_reg.lowId());
1079		encoder.disp32(src_mem.disp);
1080		}
1081		} else {
1082		if (src_mem.disp == 0) {
1083		encoder.modRm_indirectDisp0(reg.lowId(), src_reg.lowId());
1084		} else if (immOpSize(src_mem.disp) == 8) {
1085		encoder.modRm_indirectDisp8(reg.lowId(), src_reg.lowId());
1086		encoder.disp8(@intCast(i8, src_mem.disp));
1087		} else {
1088		encoder.modRm_indirectDisp32(reg.lowId(), src_reg.lowId());
1089		encoder.disp32(src_mem.disp);
1090		}
1091		}
1092		} else {
1093		encoder.rex(.{
1094		.w = reg.size() == 64,
1095		.r = reg.isExtended(),
1096		});
1097		opc.encode(encoder);
1098		if (src_mem.rip) {
1099		encoder.modRm_RIPDisp32(reg.lowId());
1100		} else {
1101		encoder.modRm_SIBDisp0(reg.lowId());
1102		encoder.sib_disp32();
1103		}
1104		encoder.disp32(src_mem.disp);
	1645	if (reg.size() != src_mem.ptr_size.size()) {
	1646	return error.OperandSizeMismatch;
1105	1647	}
1106		},
1107		}
1108		}
1109
1110		fn lowerToMrEnc(
1111		tag: Tag,
1112		reg_or_mem: RegisterOrMemory,
1113		reg: Register,
1114		code: *std.ArrayList(u8),
1115		) InnerError!void {
1116		// We use size of source register reg to work out which
1117		// variant of memory ptr to pick:
1118		// * reg is 64bit - qword ptr
1119		// * reg is 32bit - dword ptr
1120		// * reg is 16bit - word ptr
1121		// * reg is 8bit - byte ptr
1122		const opc = getOpCode(tag, .mr, reg.size() == 8).?;
1123		switch (reg_or_mem) {
1124		.register => \|dst_reg\| {
1125		if (dst_reg.size() != reg.size()) return error.EmitFail;
1126		const encoder = try Encoder.init(code, 3);
1127		encoder.rex(.{
1128		.w = dst_reg.size() == 64,
1129		.r = reg.isExtended(),
1130		.b = dst_reg.isExtended(),
1131		});
1132		opc.encode(encoder);
1133		encoder.modRm_direct(reg.lowId(), dst_reg.lowId());
1134		},
1135		.memory => \|dst_mem\| {
1136	1648	const encoder = try Encoder.init(code, 9);
1137	1649	if (reg.size() == 16) {
1138	1650	encoder.opcode_1byte(0x66);
1139	1651	}
1140		if (dst_mem.reg) \|dst_reg\| {
1141		if (dst_reg.size() != 64) return error.EmitFail;
1142		encoder.rex(.{
1143		.w = reg.size() == 64,
1144		.r = reg.isExtended(),
1145		.b = dst_reg.isExtended(),
1146		});
1147		opc.encode(encoder);
1148		if (dst_reg.lowId() == 4) {
1149		if (dst_mem.disp == 0) {
1150		encoder.modRm_SIBDisp0(reg.lowId());
1151		encoder.sib_base(dst_reg.lowId());
1152		} else if (immOpSize(dst_mem.disp) == 8) {
1153		encoder.modRm_SIBDisp8(reg.lowId());
1154		encoder.sib_baseDisp8(dst_reg.lowId());
1155		encoder.disp8(@intCast(i8, dst_mem.disp));
1156		} else {
1157		encoder.modRm_SIBDisp32(reg.lowId());
1158		encoder.sib_baseDisp32(dst_reg.lowId());
1159		encoder.disp32(dst_mem.disp);
1160		}
1161		} else {
1162		if (dst_mem.disp == 0) {
1163		encoder.modRm_indirectDisp0(reg.lowId(), dst_reg.lowId());
1164		} else if (immOpSize(dst_mem.disp) == 8) {
1165		encoder.modRm_indirectDisp8(reg.lowId(), dst_reg.lowId());
1166		encoder.disp8(@intCast(i8, dst_mem.disp));
1167		} else {
1168		encoder.modRm_indirectDisp32(reg.lowId(), dst_reg.lowId());
1169		encoder.disp32(dst_mem.disp);
1170		}
1171		}
1172		} else {
1173		encoder.rex(.{
1174		.w = reg.size() == 64,
1175		.r = reg.isExtended(),
1176		});
1177		opc.encode(encoder);
1178		if (dst_mem.rip) {
1179		encoder.modRm_RIPDisp32(reg.lowId());
1180		} else {
1181		encoder.modRm_SIBDisp0(reg.lowId());
1182		encoder.sib_disp32();
1183		}
1184		encoder.disp32(dst_mem.disp);
1185		}
1186		},
1187		}
1188		}
1189
1190		fn lowerToRmiEnc(
1191		tag: Tag,
1192		reg: Register,
1193		reg_or_mem: RegisterOrMemory,
1194		imm: i32,
1195		code: *std.ArrayList(u8),
1196		) InnerError!void {
1197		const opc = getOpCode(tag, .rmi, reg.size() == 8).?;
1198		switch (reg_or_mem) {
1199		.register => \|src_reg\| {
1200		if (reg.size() != src_reg.size()) return error.EmitFail;
1201		const encoder = try Encoder.init(code, 7);
1202		encoder.rex(.{
1203		.w = reg.size() == 64,
1204		.r = reg.isExtended(),
1205		.b = src_reg.isExtended(),
1206		});
1207		opc.encode(encoder);
1208		encoder.modRm_direct(reg.lowId(), src_reg.lowId());
1209		switch (reg.size()) {
1210		8 => {
1211		const imm8 = try math.cast(i8, imm);
1212		encoder.imm8(imm8);
1213		},
1214		16 => {
1215		const imm16 = try math.cast(i16, imm);
1216		encoder.imm16(imm16);
1217		},
1218		32, 64 => encoder.imm32(imm),
1219		else => unreachable,
1220		}
1221		},
1222		.memory => \|src_mem\| {
1223		const encoder = try Encoder.init(code, 13);
1224		if (reg.size() == 16) {
1225		encoder.opcode_1byte(0x66);
1226		}
1227	1652	if (src_mem.reg) \|src_reg\| {
1228	1653	// TODO handle 32-bit base register - requires prefix 0x67
1229	1654	// Intel Manual, Vol 1, chapter 3.6 and 3.6.1
1230		if (src_reg.size() != 64) return error.EmitFail;
	1655	if (src_reg.size() != 64) {
	1656	return error.OperandSizeMismatch;
	1657	}
1231	1658	encoder.rex(.{
1232		.w = reg.size() == 64,
	1659	.w = setRexWRegister(reg),
1233	1660	.r = reg.isExtended(),
1234	1661	.b = src_reg.isExtended(),
1235	1662	});
...	...	@@ -1260,7 +1687,7 @@ fn lowerToRmiEnc(
1260	1687	}
1261	1688	} else {
1262	1689	encoder.rex(.{
1263		.w = reg.size() == 64,
	1690	.w = setRexWRegister(reg),
1264	1691	.r = reg.isExtended(),
1265	1692	});
1266	1693	opc.encode(encoder);
...	...	@@ -1269,441 +1696,187 @@ fn lowerToRmiEnc(
1269	1696	} else {
1270	1697	encoder.modRm_SIBDisp0(reg.lowId());
1271	1698	encoder.sib_disp32();
1272		}
1273		encoder.disp32(src_mem.disp);
1274		}
1275		encoder.imm32(imm);
1276		},
1277		}
1278		}
1279
1280		fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
1281		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
1282		switch (ops.flags) {
1283		0b00 => {
1284		if (ops.reg2 == .none) {
1285		// mov reg1, imm32
1286		// MI
1287		const imm = emit.mir.instructions.items(.data)[inst].imm;
1288		return lowerToMiEnc(tag, RegisterOrMemory.reg(ops.reg1), imm, emit.code);
1289		}
1290		// mov reg1, reg2
1291		// RM
1292		return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
1293		},
1294		0b01 => {
1295		const imm = emit.mir.instructions.items(.data)[inst].imm;
1296		if (ops.reg2 == .none) {
1297		// mov reg1, [imm32]
1298		// RM
1299		return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(null, imm), emit.code);
1300		}
1301		// mov reg1, [reg2 + imm32]
1302		// RM
1303		return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(ops.reg2, imm), emit.code);
1304		},
1305		0b10 => {
1306		if (ops.reg2 == .none) {
1307		// mov dword ptr [reg1 + 0], imm32
1308		// MI
1309		const imm = emit.mir.instructions.items(.data)[inst].imm;
1310		return lowerToMiEnc(tag, RegisterOrMemory.mem(ops.reg1, 0), imm, emit.code);
1311		}
1312		// mov [reg1 + imm32], reg2
1313		// MR
1314		const imm = emit.mir.instructions.items(.data)[inst].imm;
1315		return lowerToMrEnc(tag, RegisterOrMemory.mem(ops.reg1, imm), ops.reg2, emit.code);
1316		},
1317		0b11 => {
1318		if (ops.reg2 == .none) {
1319		// mov dword ptr [reg1 + imm32], imm32
1320		// MI
1321		const payload = emit.mir.instructions.items(.data)[inst].payload;
1322		const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
1323		return lowerToMiEnc(
1324		tag,
1325		RegisterOrMemory.mem(ops.reg1, imm_pair.dest_off),
1326		imm_pair.operand,
1327		emit.code,
1328		);
1329		}
1330		return emit.fail("TODO unused variant: mov reg1, reg2, 0b11", .{});
1331		},
1332		}
1333		}
1334
1335		fn immOpSize(imm: i64) u8 {
1336		blk: {
1337		_ = math.cast(i8, imm) catch break :blk;
1338		return 8;
1339		}
1340		blk: {
1341		_ = math.cast(i16, imm) catch break :blk;
1342		return 16;
1343		}
1344		blk: {
1345		_ = math.cast(i32, imm) catch break :blk;
1346		return 32;
1347		}
1348		return 64;
1349		}
1350
1351		// TODO
1352		fn mirArithScaleSrc(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
1353		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
1354		const scale = ops.flags;
1355		// OP reg1, [reg2 + scale*rcx + imm32]
1356		const opc = getOpCode(tag, .rm, ops.reg1.size() == 8).?;
1357		const imm = emit.mir.instructions.items(.data)[inst].imm;
1358		const encoder = try Encoder.init(emit.code, 8);
1359		encoder.rex(.{
1360		.w = ops.reg1.size() == 64,
1361		.r = ops.reg1.isExtended(),
1362		.b = ops.reg2.isExtended(),
1363		});
1364		opc.encode(encoder);
1365		if (imm <= math.maxInt(i8)) {
1366		encoder.modRm_SIBDisp8(ops.reg1.lowId());
1367		encoder.sib_scaleIndexBaseDisp8(scale, Register.rcx.lowId(), ops.reg2.lowId());
1368		encoder.disp8(@intCast(i8, imm));
1369		} else {
1370		encoder.modRm_SIBDisp32(ops.reg1.lowId());
1371		encoder.sib_scaleIndexBaseDisp32(scale, Register.rcx.lowId(), ops.reg2.lowId());
1372		encoder.disp32(imm);
1373		}
1374		}
1375
1376		// TODO
1377		fn mirArithScaleDst(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
1378		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
1379		const scale = ops.flags;
1380		const imm = emit.mir.instructions.items(.data)[inst].imm;
1381
1382		if (ops.reg2 == .none) {
1383		// OP [reg1 + scale*rax + 0], imm32
1384		const opc = getOpCode(tag, .mi, ops.reg1.size() == 8).?;
1385		const modrm_ext = getModRmExt(tag).?;
1386		const encoder = try Encoder.init(emit.code, 8);
1387		encoder.rex(.{
1388		.w = ops.reg1.size() == 64,
1389		.b = ops.reg1.isExtended(),
1390		});
1391		opc.encode(encoder);
1392		encoder.modRm_SIBDisp0(modrm_ext);
1393		encoder.sib_scaleIndexBase(scale, Register.rax.lowId(), ops.reg1.lowId());
1394		if (imm <= math.maxInt(i8)) {
1395		encoder.imm8(@intCast(i8, imm));
1396		} else if (imm <= math.maxInt(i16)) {
1397		encoder.imm16(@intCast(i16, imm));
1398		} else {
1399		encoder.imm32(imm);
1400		}
1401		return;
1402		}
1403
1404		// OP [reg1 + scale*rax + imm32], reg2
1405		const opc = getOpCode(tag, .mr, ops.reg1.size() == 8).?;
1406		const encoder = try Encoder.init(emit.code, 8);
1407		encoder.rex(.{
1408		.w = ops.reg1.size() == 64,
1409		.r = ops.reg2.isExtended(),
1410		.b = ops.reg1.isExtended(),
1411		});
1412		opc.encode(encoder);
1413		if (imm <= math.maxInt(i8)) {
1414		encoder.modRm_SIBDisp8(ops.reg2.lowId());
1415		encoder.sib_scaleIndexBaseDisp8(scale, Register.rax.lowId(), ops.reg1.lowId());
1416		encoder.disp8(@intCast(i8, imm));
1417		} else {
1418		encoder.modRm_SIBDisp32(ops.reg2.lowId());
1419		encoder.sib_scaleIndexBaseDisp32(scale, Register.rax.lowId(), ops.reg1.lowId());
1420		encoder.disp32(imm);
1421		}
1422		}
1423
1424		// TODO
1425		fn mirArithScaleImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
1426		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
1427		const scale = ops.flags;
1428		const payload = emit.mir.instructions.items(.data)[inst].payload;
1429		const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
1430		const opc = getOpCode(tag, .mi, ops.reg1.size() == 8).?;
1431		const modrm_ext = getModRmExt(tag).?;
1432		const encoder = try Encoder.init(emit.code, 2);
1433		encoder.rex(.{
1434		.w = ops.reg1.size() == 64,
1435		.b = ops.reg1.isExtended(),
1436		});
1437		opc.encode(encoder);
1438		if (imm_pair.dest_off <= math.maxInt(i8)) {
1439		encoder.modRm_SIBDisp8(modrm_ext);
1440		encoder.sib_scaleIndexBaseDisp8(scale, Register.rax.lowId(), ops.reg1.lowId());
1441		encoder.disp8(@intCast(i8, imm_pair.dest_off));
1442		} else {
1443		encoder.modRm_SIBDisp32(modrm_ext);
1444		encoder.sib_scaleIndexBaseDisp32(scale, Register.rax.lowId(), ops.reg1.lowId());
1445		encoder.disp32(imm_pair.dest_off);
1446		}
1447		encoder.imm32(imm_pair.operand);
1448		}
1449
1450		fn mirMovabs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1451		const tag = emit.mir.instructions.items(.tag)[inst];
1452		assert(tag == .movabs);
1453		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
1454		const imm: i64 = if (ops.reg1.size() == 64) blk: {
1455		const payload = emit.mir.instructions.items(.data)[inst].payload;
1456		const imm = emit.mir.extraData(Mir.Imm64, payload).data;
1457		break :blk @bitCast(i64, imm.decode());
1458		} else emit.mir.instructions.items(.data)[inst].imm;
1459		if (ops.flags == 0b00) {
1460		// movabs reg, imm64
1461		// OI
1462		return lowerToOiEnc(.mov, ops.reg1, imm, emit.code);
1463		}
1464		if (ops.reg1 == .none) {
1465		// movabs moffs64, rax
1466		// TD
1467		return lowerToTdEnc(.mov, imm, ops.reg2, emit.code);
1468		} else {
1469		// movabs rax, moffs64
1470		// FD
1471		return lowerToFdEnc(.mov, ops.reg1, imm, emit.code);
1472		}
1473		}
1474
1475		fn mirIMulComplex(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1476		const tag = emit.mir.instructions.items(.tag)[inst];
1477		assert(tag == .imul_complex);
1478		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
1479		switch (ops.flags) {
1480		0b00 => return lowerToRmEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code),
1481		0b10 => {
1482		const imm = emit.mir.instructions.items(.data)[inst].imm;
1483		return lowerToRmiEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), imm, emit.code);
1484		},
1485		else => return emit.fail("TODO implement imul", .{}),
1486		}
1487		}
1488
1489		fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1490		const tag = emit.mir.instructions.items(.tag)[inst];
1491		assert(tag == .lea);
1492		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
1493		assert(ops.flags == 0b01);
1494		const imm = emit.mir.instructions.items(.data)[inst].imm;
1495		return lowerToRmEnc(.lea, ops.reg1, RegisterOrMemory.mem(ops.reg2, imm), emit.code);
1496		}
1497
1498		fn mirLeaRip(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1499		const tag = emit.mir.instructions.items(.tag)[inst];
1500		assert(tag == .lea_rip);
1501		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
1502		const start_offset = emit.code.items.len;
1503		try lowerToRmEnc(.lea, ops.reg1, RegisterOrMemory.rip(0), emit.code);
1504		const end_offset = emit.code.items.len;
1505		if (@truncate(u1, ops.flags) == 0b0) {
1506		// Backpatch the displacement
1507		// TODO figure out if this can be simplified
1508		const payload = emit.mir.instructions.items(.data)[inst].payload;
1509		const imm = emit.mir.extraData(Mir.Imm64, payload).data.decode();
1510		const disp = @intCast(i32, @intCast(i64, imm) - @intCast(i64, end_offset - start_offset));
1511		mem.writeIntLittle(i32, emit.code.items[end_offset - 4 ..][0..4], disp);
1512		} else {
1513		const got_entry = emit.mir.instructions.items(.data)[inst].got_entry;
1514		if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
1515		// TODO I think the reloc might be in the wrong place.
1516		const decl = macho_file.active_decl.?;
1517		try decl.link.macho.relocs.append(emit.bin_file.allocator, .{
1518		.offset = @intCast(u32, end_offset - 4),
1519		.target = .{ .local = got_entry },
1520		.addend = 0,
1521		.subtractor = null,
1522		.pcrel = true,
1523		.length = 2,
1524		.@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_GOT),
1525		});
1526		} else {
1527		return emit.fail("TODO implement lea_rip for linking backends different than MachO", .{});
1528		}
1529		}
1530		}
1531
1532		fn mirCallExtern(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1533		const tag = emit.mir.instructions.items(.tag)[inst];
1534		assert(tag == .call_extern);
1535		const n_strx = emit.mir.instructions.items(.data)[inst].extern_fn;
1536		const offset = blk: {
1537		// callq
1538		try lowerToDEnc(.call_near, 0, emit.code);
1539		break :blk @intCast(u32, emit.code.items.len) - 4;
1540		};
1541		if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
1542		// Add relocation to the decl.
1543		try macho_file.active_decl.?.link.macho.relocs.append(emit.bin_file.allocator, .{
1544		.offset = offset,
1545		.target = .{ .global = n_strx },
1546		.addend = 0,
1547		.subtractor = null,
1548		.pcrel = true,
1549		.length = 2,
1550		.@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_BRANCH),
1551		});
1552		} else {
1553		return emit.fail("TODO implement call_extern for linking backends different than MachO", .{});
1554		}
1555		}
1556
1557		fn mirDbgLine(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1558		const tag = emit.mir.instructions.items(.tag)[inst];
1559		assert(tag == .dbg_line);
1560		const payload = emit.mir.instructions.items(.data)[inst].payload;
1561		const dbg_line_column = emit.mir.extraData(Mir.DbgLineColumn, payload).data;
1562		try emit.dbgAdvancePCAndLine(dbg_line_column.line, dbg_line_column.column);
1563		}
1564
1565		fn dbgAdvancePCAndLine(emit: *Emit, line: u32, column: u32) InnerError!void {
1566		const delta_line = @intCast(i32, line) - @intCast(i32, emit.prev_di_line);
1567		const delta_pc: usize = emit.code.items.len - emit.prev_di_pc;
1568		switch (emit.debug_output) {
1569		.dwarf => \|dbg_out\| {
1570		// TODO Look into using the DWARF special opcodes to compress this data.
1571		// It lets you emit single-byte opcodes that add different numbers to
1572		// both the PC and the line number at the same time.
1573		try dbg_out.dbg_line.ensureUnusedCapacity(11);
1574		dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_pc);
1575		leb128.writeULEB128(dbg_out.dbg_line.writer(), delta_pc) catch unreachable;
1576		if (delta_line != 0) {
1577		dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_line);
1578		leb128.writeILEB128(dbg_out.dbg_line.writer(), delta_line) catch unreachable;
	1699	}
	1700	encoder.disp32(src_mem.disp);
1579	1701	}
1580		dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.copy);
1581		emit.prev_di_pc = emit.code.items.len;
1582		emit.prev_di_line = line;
1583		emit.prev_di_column = column;
1584		emit.prev_di_pc = emit.code.items.len;
1585		},
1586		.plan9 => \|dbg_out\| {
1587		if (delta_pc <= 0) return; // only do this when the pc changes
1588		// we have already checked the target in the linker to make sure it is compatable
1589		const quant = @import("../../link/Plan9/aout.zig").getPCQuant(emit.target.cpu.arch) catch unreachable;
1590
1591		// increasing the line number
1592		try @import("../../link/Plan9.zig").changeLine(dbg_out.dbg_line, delta_line);
1593		// increasing the pc
1594		const d_pc_p9 = @intCast(i64, delta_pc) - quant;
1595		if (d_pc_p9 > 0) {
1596		// minus one because if its the last one, we want to leave space to change the line which is one quanta
1597		try dbg_out.dbg_line.append(@intCast(u8, @divExact(d_pc_p9, quant) + 128) - quant);
1598		if (dbg_out.pcop_change_index.*) \|pci\|
1599		dbg_out.dbg_line.items[pci] += 1;
1600		dbg_out.pcop_change_index.* = @intCast(u32, dbg_out.dbg_line.items.len - 1);
1601		} else if (d_pc_p9 == 0) {
1602		// we don't need to do anything, because adding the quant does it for us
1603		} else unreachable;
1604		if (dbg_out.start_line.* == null)
1605		dbg_out.start_line.* = emit.prev_di_line;
1606		dbg_out.end_line.* = line;
1607		// only do this if the pc changed
1608		emit.prev_di_line = line;
1609		emit.prev_di_column = column;
1610		emit.prev_di_pc = emit.code.items.len;
1611	1702	},
1612		.none => {},
1613	1703	}
1614	1704	}
1615	1705
1616		fn mirDbgPrologueEnd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1617		const tag = emit.mir.instructions.items(.tag)[inst];
1618		assert(tag == .dbg_prologue_end);
1619		switch (emit.debug_output) {
1620		.dwarf => \|dbg_out\| {
1621		try dbg_out.dbg_line.append(DW.LNS.set_prologue_end);
1622		try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
	1706	fn lowerToMrEnc(
	1707	tag: Tag,
	1708	reg_or_mem: RegisterOrMemory,
	1709	reg: Register,
	1710	code: *std.ArrayList(u8),
	1711	) LoweringError!void {
	1712	const opc = getOpCode(tag, .mr, reg.size() == 8).?;
	1713	switch (reg_or_mem) {
	1714	.register => \|dst_reg\| {
	1715	if (dst_reg.size() != reg.size()) {
	1716	return error.OperandSizeMismatch;
	1717	}
	1718	const encoder = try Encoder.init(code, 3);
	1719	encoder.rex(.{
	1720	.w = setRexWRegister(dst_reg) or setRexWRegister(reg),
	1721	.r = reg.isExtended(),
	1722	.b = dst_reg.isExtended(),
	1723	});
	1724	opc.encode(encoder);
	1725	encoder.modRm_direct(reg.lowId(), dst_reg.lowId());
1623	1726	},
1624		.plan9 => {},
1625		.none => {},
1626		}
1627		}
1628
1629		fn mirDbgEpilogueBegin(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1630		const tag = emit.mir.instructions.items(.tag)[inst];
1631		assert(tag == .dbg_epilogue_begin);
1632		switch (emit.debug_output) {
1633		.dwarf => \|dbg_out\| {
1634		try dbg_out.dbg_line.append(DW.LNS.set_epilogue_begin);
1635		try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
	1727	.memory => \|dst_mem\| {
	1728	if (dst_mem.ptr_size.size() != reg.size()) {
	1729	return error.OperandSizeMismatch;
	1730	}
	1731	const encoder = try Encoder.init(code, 9);
	1732	if (reg.size() == 16) {
	1733	encoder.opcode_1byte(0x66);
	1734	}
	1735	if (dst_mem.reg) \|dst_reg\| {
	1736	if (dst_reg.size() != 64) {
	1737	return error.OperandSizeMismatch;
	1738	}
	1739	encoder.rex(.{
	1740	.w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg),
	1741	.r = reg.isExtended(),
	1742	.b = dst_reg.isExtended(),
	1743	});
	1744	opc.encode(encoder);
	1745	if (dst_reg.lowId() == 4) {
	1746	if (dst_mem.disp == 0) {
	1747	encoder.modRm_SIBDisp0(reg.lowId());
	1748	encoder.sib_base(dst_reg.lowId());
	1749	} else if (immOpSize(dst_mem.disp) == 8) {
	1750	encoder.modRm_SIBDisp8(reg.lowId());
	1751	encoder.sib_baseDisp8(dst_reg.lowId());
	1752	encoder.disp8(@intCast(i8, dst_mem.disp));
	1753	} else {
	1754	encoder.modRm_SIBDisp32(reg.lowId());
	1755	encoder.sib_baseDisp32(dst_reg.lowId());
	1756	encoder.disp32(dst_mem.disp);
	1757	}
	1758	} else {
	1759	if (dst_mem.disp == 0) {
	1760	encoder.modRm_indirectDisp0(reg.lowId(), dst_reg.lowId());
	1761	} else if (immOpSize(dst_mem.disp) == 8) {
	1762	encoder.modRm_indirectDisp8(reg.lowId(), dst_reg.lowId());
	1763	encoder.disp8(@intCast(i8, dst_mem.disp));
	1764	} else {
	1765	encoder.modRm_indirectDisp32(reg.lowId(), dst_reg.lowId());
	1766	encoder.disp32(dst_mem.disp);
	1767	}
	1768	}
	1769	} else {
	1770	encoder.rex(.{
	1771	.w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg),
	1772	.r = reg.isExtended(),
	1773	});
	1774	opc.encode(encoder);
	1775	if (dst_mem.rip) {
	1776	encoder.modRm_RIPDisp32(reg.lowId());
	1777	} else {
	1778	encoder.modRm_SIBDisp0(reg.lowId());
	1779	encoder.sib_disp32();
	1780	}
	1781	encoder.disp32(dst_mem.disp);
	1782	}
1636	1783	},
1637		.plan9 => {},
1638		.none => {},
1639	1784	}
1640	1785	}
1641	1786
1642		fn mirArgDbgInfo(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1643		const tag = emit.mir.instructions.items(.tag)[inst];
1644		assert(tag == .arg_dbg_info);
1645		const payload = emit.mir.instructions.items(.data)[inst].payload;
1646		const arg_dbg_info = emit.mir.extraData(Mir.ArgDbgInfo, payload).data;
1647		const mcv = emit.mir.function.args[arg_dbg_info.arg_index];
1648		try emit.genArgDbgInfo(arg_dbg_info.air_inst, mcv);
1649		}
1650
1651		fn genArgDbgInfo(emit: *Emit, inst: Air.Inst.Index, mcv: MCValue) !void {
1652		const ty_str = emit.mir.function.air.instructions.items(.data)[inst].ty_str;
1653		const zir = &emit.mir.function.mod_fn.owner_decl.getFileScope().zir;
1654		const name = zir.nullTerminatedString(ty_str.str);
1655		const name_with_null = name.ptr[0 .. name.len + 1];
1656		const ty = emit.mir.function.air.getRefType(ty_str.ty);
1657
1658		switch (mcv) {
1659		.register => \|reg\| {
1660		switch (emit.debug_output) {
1661		.dwarf => \|dbg_out\| {
1662		try dbg_out.dbg_info.ensureUnusedCapacity(3);
1663		dbg_out.dbg_info.appendAssumeCapacity(link.File.Elf.abbrev_parameter);
1664		dbg_out.dbg_info.appendSliceAssumeCapacity(&[2]u8{ // DW.AT.location, DW.FORM.exprloc
1665		1, // ULEB128 dwarf expression length
1666		reg.dwarfLocOp(),
1667		});
1668		try dbg_out.dbg_info.ensureUnusedCapacity(5 + name_with_null.len);
1669		try emit.addDbgInfoTypeReloc(ty); // DW.AT.type, DW.FORM.ref4
1670		dbg_out.dbg_info.appendSliceAssumeCapacity(name_with_null); // DW.AT.name, DW.FORM.string
1671		},
1672		.plan9 => {},
1673		.none => {},
	1787	fn lowerToRmiEnc(
	1788	tag: Tag,
	1789	reg: Register,
	1790	reg_or_mem: RegisterOrMemory,
	1791	imm: i32,
	1792	code: *std.ArrayList(u8),
	1793	) LoweringError!void {
	1794	if (reg.size() == 8) {
	1795	return error.OperandSizeMismatch;
	1796	}
	1797	const opc = getOpCode(tag, .rmi, false).?;
	1798	const encoder = try Encoder.init(code, 13);
	1799	if (reg.size() == 16) {
	1800	encoder.opcode_1byte(0x66);
	1801	}
	1802	switch (reg_or_mem) {
	1803	.register => \|src_reg\| {
	1804	if (reg.size() != src_reg.size()) {
	1805	return error.OperandSizeMismatch;
1674	1806	}
	1807	encoder.rex(.{
	1808	.w = setRexWRegister(reg) or setRexWRegister(src_reg),
	1809	.r = reg.isExtended(),
	1810	.b = src_reg.isExtended(),
	1811	});
	1812	opc.encode(encoder);
	1813	encoder.modRm_direct(reg.lowId(), src_reg.lowId());
1675	1814	},
1676		.stack_offset => {
1677		switch (emit.debug_output) {
1678		.dwarf => {},
1679		.plan9 => {},
1680		.none => {},
	1815	.memory => \|src_mem\| {
	1816	if (src_mem.reg) \|src_reg\| {
	1817	// TODO handle 32-bit base register - requires prefix 0x67
	1818	// Intel Manual, Vol 1, chapter 3.6 and 3.6.1
	1819	if (src_reg.size() != 64) {
	1820	return error.OperandSizeMismatch;
	1821	}
	1822	if (src_mem.ptr_size == .byte_ptr) {
	1823	return error.OperandSizeMismatch;
	1824	}
	1825	encoder.rex(.{
	1826	.w = setRexWRegister(reg),
	1827	.r = reg.isExtended(),
	1828	.b = src_reg.isExtended(),
	1829	});
	1830	opc.encode(encoder);
	1831	if (src_reg.lowId() == 4) {
	1832	if (src_mem.disp == 0) {
	1833	encoder.modRm_SIBDisp0(reg.lowId());
	1834	encoder.sib_base(src_reg.lowId());
	1835	} else if (immOpSize(src_mem.disp) == 8) {
	1836	encoder.modRm_SIBDisp8(reg.lowId());
	1837	encoder.sib_baseDisp8(src_reg.lowId());
	1838	encoder.disp8(@intCast(i8, src_mem.disp));
	1839	} else {
	1840	encoder.modRm_SIBDisp32(reg.lowId());
	1841	encoder.sib_baseDisp32(src_reg.lowId());
	1842	encoder.disp32(src_mem.disp);
	1843	}
	1844	} else {
	1845	if (src_mem.disp == 0) {
	1846	encoder.modRm_indirectDisp0(reg.lowId(), src_reg.lowId());
	1847	} else if (immOpSize(src_mem.disp) == 8) {
	1848	encoder.modRm_indirectDisp8(reg.lowId(), src_reg.lowId());
	1849	encoder.disp8(@intCast(i8, src_mem.disp));
	1850	} else {
	1851	encoder.modRm_indirectDisp32(reg.lowId(), src_reg.lowId());
	1852	encoder.disp32(src_mem.disp);
	1853	}
	1854	}
	1855	} else {
	1856	encoder.rex(.{
	1857	.w = setRexWRegister(reg),
	1858	.r = reg.isExtended(),
	1859	});
	1860	opc.encode(encoder);
	1861	if (src_mem.rip) {
	1862	encoder.modRm_RIPDisp32(reg.lowId());
	1863	} else {
	1864	encoder.modRm_SIBDisp0(reg.lowId());
	1865	encoder.sib_disp32();
	1866	}
	1867	encoder.disp32(src_mem.disp);
1681	1868	}
1682	1869	},
1683		else => {},
1684	1870	}
1685		}
1686
1687		/// Adds a Type to the .debug_info at the current position. The bytes will be populated later,
1688		/// after codegen for this symbol is done.
1689		fn addDbgInfoTypeReloc(emit: *Emit, ty: Type) !void {
1690		switch (emit.debug_output) {
1691		.dwarf => \|dbg_out\| {
1692		assert(ty.hasCodeGenBits());
1693		const index = dbg_out.dbg_info.items.len;
1694		try dbg_out.dbg_info.resize(index + 4); // DW.AT.type, DW.FORM.ref4
1695
1696		const gop = try dbg_out.dbg_info_type_relocs.getOrPut(emit.bin_file.allocator, ty);
1697		if (!gop.found_existing) {
1698		gop.value_ptr.* = .{
1699		.off = undefined,
1700		.relocs = .{},
1701		};
1702		}
1703		try gop.value_ptr.relocs.append(emit.bin_file.allocator, @intCast(u32, index));
	1871	switch (reg.size()) {
	1872	// TODO 8bit immediate
	1873	8 => unreachable,
	1874	16 => {
	1875	const imm16 = try math.cast(i16, imm);
	1876	encoder.imm16(imm16);
1704	1877	},
1705		.plan9 => {},
1706		.none => {},
	1878	32, 64 => encoder.imm32(imm),
	1879	else => unreachable,
1707	1880	}
1708	1881	}
1709	1882
...	...	@@ -1757,34 +1930,44 @@ test "lower MI encoding" {
1757	1930	defer code.deinit();
1758	1931	try lowerToMiEnc(.mov, RegisterOrMemory.reg(.rax), 0x10, code.buffer());
1759	1932	try expectEqualHexStrings("\x48\xc7\xc0\x10\x00\x00\x00", code.emitted(), "mov rax, 0x10");
1760		try lowerToMiEnc(.mov, RegisterOrMemory.mem(.r11, 0), 0x10, code.buffer());
	1933	try lowerToMiEnc(.mov, RegisterOrMemory.mem(.r11, 0, .dword_ptr), 0x10, code.buffer());
1761	1934	try expectEqualHexStrings("\x41\xc7\x03\x10\x00\x00\x00", code.emitted(), "mov dword ptr [r11 + 0], 0x10");
1762		try lowerToMiEnc(.add, RegisterOrMemory.mem(.rdx, -8), 0x10, code.buffer());
	1935	try lowerToMiEnc(.add, RegisterOrMemory.mem(.rdx, -8, .dword_ptr), 0x10, code.buffer());
1763	1936	try expectEqualHexStrings("\x81\x42\xF8\x10\x00\x00\x00", code.emitted(), "add dword ptr [rdx - 8], 0x10");
1764		try lowerToMiEnc(.sub, RegisterOrMemory.mem(.r11, 0x10000000), 0x10, code.buffer());
	1937	try lowerToMiEnc(.sub, RegisterOrMemory.mem(.r11, 0x10000000, .dword_ptr), 0x10, code.buffer());
1765	1938	try expectEqualHexStrings(
1766	1939	"\x41\x81\xab\x00\x00\x00\x10\x10\x00\x00\x00",
1767	1940	code.emitted(),
1768	1941	"sub dword ptr [r11 + 0x10000000], 0x10",
1769	1942	);
1770		try lowerToMiEnc(.@"and", RegisterOrMemory.mem(null, 0x10000000), 0x10, code.buffer());
	1943	try lowerToMiEnc(.@"and", RegisterOrMemory.mem(null, 0x10000000, .dword_ptr), 0x10, code.buffer());
1771	1944	try expectEqualHexStrings(
1772	1945	"\x81\x24\x25\x00\x00\x00\x10\x10\x00\x00\x00",
1773	1946	code.emitted(),
1774	1947	"and dword ptr [ds:0x10000000], 0x10",
1775	1948	);
1776		try lowerToMiEnc(.@"and", RegisterOrMemory.mem(.r12, 0x10000000), 0x10, code.buffer());
	1949	try lowerToMiEnc(.@"and", RegisterOrMemory.mem(.r12, 0x10000000, .dword_ptr), 0x10, code.buffer());
1777	1950	try expectEqualHexStrings(
1778	1951	"\x41\x81\xA4\x24\x00\x00\x00\x10\x10\x00\x00\x00",
1779	1952	code.emitted(),
1780	1953	"and dword ptr [r12 + 0x10000000], 0x10",
1781	1954	);
1782		try lowerToMiEnc(.mov, RegisterOrMemory.rip(0x10), 0x10, code.buffer());
	1955	try lowerToMiEnc(.mov, RegisterOrMemory.rip(0x10, .qword_ptr), 0x10, code.buffer());
1783	1956	try expectEqualHexStrings(
1784	1957	"\xC7\x05\x10\x00\x00\x00\x10\x00\x00\x00",
1785	1958	code.emitted(),
1786		"mov [rip + 0x10], 0x10",
	1959	"mov qword ptr [rip + 0x10], 0x10",
	1960	);
	1961	try lowerToMiEnc(.mov, RegisterOrMemory.mem(.rbp, -8, .qword_ptr), 0x10, code.buffer());
	1962	try expectEqualHexStrings(
	1963	"\x48\xc7\x45\xf8\x10\x00\x00\x00",
	1964	code.emitted(),
	1965	"mov qword ptr [rbp - 8], 0x10",
1787	1966	);
	1967	try lowerToMiEnc(.mov, RegisterOrMemory.mem(.rbp, -2, .word_ptr), 0x10, code.buffer());
	1968	try expectEqualHexStrings("\x66\xC7\x45\xFE\x10\x00", code.emitted(), "mov word ptr [rbp - 2], 0x10");
	1969	try lowerToMiEnc(.mov, RegisterOrMemory.mem(.rbp, -1, .byte_ptr), 0x10, code.buffer());
	1970	try expectEqualHexStrings("\xC6\x45\xFF\x10", code.emitted(), "mov byte ptr [rbp - 1], 0x10");
1788	1971	}
1789	1972
1790	1973	test "lower RM encoding" {
...	...	@@ -1792,35 +1975,35 @@ test "lower RM encoding" {
1792	1975	defer code.deinit();
1793	1976	try lowerToRmEnc(.mov, .rax, RegisterOrMemory.reg(.rbx), code.buffer());
1794	1977	try expectEqualHexStrings("\x48\x8b\xc3", code.emitted(), "mov rax, rbx");
1795		try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.r11, 0), code.buffer());
	1978	try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.r11, 0, .qword_ptr), code.buffer());
1796	1979	try expectEqualHexStrings("\x49\x8b\x03", code.emitted(), "mov rax, qword ptr [r11 + 0]");
1797		try lowerToRmEnc(.add, .r11, RegisterOrMemory.mem(null, 0x10000000), code.buffer());
	1980	try lowerToRmEnc(.add, .r11, RegisterOrMemory.mem(null, 0x10000000, .qword_ptr), code.buffer());
1798	1981	try expectEqualHexStrings(
1799	1982	"\x4C\x03\x1C\x25\x00\x00\x00\x10",
1800	1983	code.emitted(),
1801	1984	"add r11, qword ptr [ds:0x10000000]",
1802	1985	);
1803		try lowerToRmEnc(.add, .r12b, RegisterOrMemory.mem(null, 0x10000000), code.buffer());
	1986	try lowerToRmEnc(.add, .r12b, RegisterOrMemory.mem(null, 0x10000000, .byte_ptr), code.buffer());
1804	1987	try expectEqualHexStrings(
1805	1988	"\x44\x02\x24\x25\x00\x00\x00\x10",
1806	1989	code.emitted(),
1807	1990	"add r11b, byte ptr [ds:0x10000000]",
1808	1991	);
1809		try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.r13, 0x10000000), code.buffer());
	1992	try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.r13, 0x10000000, .qword_ptr), code.buffer());
1810	1993	try expectEqualHexStrings(
1811	1994	"\x4D\x2B\x9D\x00\x00\x00\x10",
1812	1995	code.emitted(),
1813	1996	"sub r11, qword ptr [r13 + 0x10000000]",
1814	1997	);
1815		try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.r12, 0x10000000), code.buffer());
	1998	try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.r12, 0x10000000, .qword_ptr), code.buffer());
1816	1999	try expectEqualHexStrings(
1817	2000	"\x4D\x2B\x9C\x24\x00\x00\x00\x10",
1818	2001	code.emitted(),
1819	2002	"sub r11, qword ptr [r12 + 0x10000000]",
1820	2003	);
1821		try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.rbp, -4), code.buffer());
	2004	try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.rbp, -4, .qword_ptr), code.buffer());
1822	2005	try expectEqualHexStrings("\x48\x8B\x45\xFC", code.emitted(), "mov rax, qword ptr [rbp - 4]");
1823		try lowerToRmEnc(.lea, .rax, RegisterOrMemory.rip(0x10), code.buffer());
	2006	try lowerToRmEnc(.lea, .rax, RegisterOrMemory.rip(0x10, .qword_ptr), code.buffer());
1824	2007	try expectEqualHexStrings("\x48\x8D\x05\x10\x00\x00\x00", code.emitted(), "lea rax, [rip + 0x10]");
1825	2008	}
1826	2009
...	...	@@ -1829,27 +2012,27 @@ test "lower MR encoding" {
1829	2012	defer code.deinit();
1830	2013	try lowerToMrEnc(.mov, RegisterOrMemory.reg(.rax), .rbx, code.buffer());
1831	2014	try expectEqualHexStrings("\x48\x89\xd8", code.emitted(), "mov rax, rbx");
1832		try lowerToMrEnc(.mov, RegisterOrMemory.mem(.rbp, -4), .r11, code.buffer());
	2015	try lowerToMrEnc(.mov, RegisterOrMemory.mem(.rbp, -4, .qword_ptr), .r11, code.buffer());
1833	2016	try expectEqualHexStrings("\x4c\x89\x5d\xfc", code.emitted(), "mov qword ptr [rbp - 4], r11");
1834		try lowerToMrEnc(.add, RegisterOrMemory.mem(null, 0x10000000), .r12b, code.buffer());
	2017	try lowerToMrEnc(.add, RegisterOrMemory.mem(null, 0x10000000, .byte_ptr), .r12b, code.buffer());
1835	2018	try expectEqualHexStrings(
1836	2019	"\x44\x00\x24\x25\x00\x00\x00\x10",
1837	2020	code.emitted(),
1838	2021	"add byte ptr [ds:0x10000000], r12b",
1839	2022	);
1840		try lowerToMrEnc(.add, RegisterOrMemory.mem(null, 0x10000000), .r12d, code.buffer());
	2023	try lowerToMrEnc(.add, RegisterOrMemory.mem(null, 0x10000000, .dword_ptr), .r12d, code.buffer());
1841	2024	try expectEqualHexStrings(
1842	2025	"\x44\x01\x24\x25\x00\x00\x00\x10",
1843	2026	code.emitted(),
1844	2027	"add dword ptr [ds:0x10000000], r12d",
1845	2028	);
1846		try lowerToMrEnc(.sub, RegisterOrMemory.mem(.r11, 0x10000000), .r12, code.buffer());
	2029	try lowerToMrEnc(.sub, RegisterOrMemory.mem(.r11, 0x10000000, .qword_ptr), .r12, code.buffer());
1847	2030	try expectEqualHexStrings(
1848	2031	"\x4D\x29\xA3\x00\x00\x00\x10",
1849	2032	code.emitted(),
1850	2033	"sub qword ptr [r11 + 0x10000000], r12",
1851	2034	);
1852		try lowerToMrEnc(.mov, RegisterOrMemory.rip(0x10), .r12, code.buffer());
	2035	try lowerToMrEnc(.mov, RegisterOrMemory.rip(0x10, .qword_ptr), .r12, code.buffer());
1853	2036	try expectEqualHexStrings("\x4C\x89\x25\x10\x00\x00\x00", code.emitted(), "mov qword ptr [rip + 0x10], r12");
1854	2037	}
1855	2038
...	...	@@ -1898,22 +2081,26 @@ test "lower M encoding" {
1898	2081	defer code.deinit();
1899	2082	try lowerToMEnc(.jmp_near, RegisterOrMemory.reg(.r12), code.buffer());
1900	2083	try expectEqualHexStrings("\x41\xFF\xE4", code.emitted(), "jmp r12");
1901		try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.r12, 0), code.buffer());
	2084	try lowerToMEnc(.jmp_near, RegisterOrMemory.reg(.r12w), code.buffer());
	2085	try expectEqualHexStrings("\x66\x41\xFF\xE4", code.emitted(), "jmp r12w");
	2086	try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.r12, 0, .qword_ptr), code.buffer());
1902	2087	try expectEqualHexStrings("\x41\xFF\x24\x24", code.emitted(), "jmp qword ptr [r12]");
1903		try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.r12, 0x10), code.buffer());
	2088	try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.r12, 0, .word_ptr), code.buffer());
	2089	try expectEqualHexStrings("\x66\x41\xFF\x24\x24", code.emitted(), "jmp word ptr [r12]");
	2090	try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.r12, 0x10, .qword_ptr), code.buffer());
1904	2091	try expectEqualHexStrings("\x41\xFF\x64\x24\x10", code.emitted(), "jmp qword ptr [r12 + 0x10]");
1905		try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.r12, 0x1000), code.buffer());
	2092	try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.r12, 0x1000, .qword_ptr), code.buffer());
1906	2093	try expectEqualHexStrings(
1907	2094	"\x41\xFF\xA4\x24\x00\x10\x00\x00",
1908	2095	code.emitted(),
1909	2096	"jmp qword ptr [r12 + 0x1000]",
1910	2097	);
1911		try lowerToMEnc(.jmp_near, RegisterOrMemory.rip(0x10), code.buffer());
	2098	try lowerToMEnc(.jmp_near, RegisterOrMemory.rip(0x10, .qword_ptr), code.buffer());
1912	2099	try expectEqualHexStrings("\xFF\x25\x10\x00\x00\x00", code.emitted(), "jmp qword ptr [rip + 0x10]");
1913		try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(null, 0x10), code.buffer());
	2100	try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(null, 0x10, .qword_ptr), code.buffer());
1914	2101	try expectEqualHexStrings("\xFF\x24\x25\x10\x00\x00\x00", code.emitted(), "jmp qword ptr [ds:0x10]");
1915	2102	try lowerToMEnc(.seta, RegisterOrMemory.reg(.r11b), code.buffer());
1916		try expectEqualHexStrings("\x49\x0F\x97\xC3", code.emitted(), "seta r11b");
	2103	try expectEqualHexStrings("\x41\x0F\x97\xC3", code.emitted(), "seta r11b");
1917	2104	}
1918	2105
1919	2106	test "lower O encoding" {
...	...	@@ -1928,8 +2115,18 @@ test "lower O encoding" {
1928	2115	test "lower RMI encoding" {
1929	2116	var code = TestEmitCode.init();
1930	2117	defer code.deinit();
1931		try lowerToRmiEnc(.imul, .rax, RegisterOrMemory.mem(.rbp, -8), 0x10, code.buffer());
1932		try expectEqualHexStrings("\x48\x69\x45\xF8\x10\x00\x00\x00", code.emitted(), "imul rax, [rbp - 8], 0x10");
	2118	try lowerToRmiEnc(.imul, .rax, RegisterOrMemory.mem(.rbp, -8, .qword_ptr), 0x10, code.buffer());
	2119	try expectEqualHexStrings(
	2120	"\x48\x69\x45\xF8\x10\x00\x00\x00",
	2121	code.emitted(),
	2122	"imul rax, qword ptr [rbp - 8], 0x10",
	2123	);
	2124	try lowerToRmiEnc(.imul, .eax, RegisterOrMemory.mem(.rbp, -4, .dword_ptr), 0x10, code.buffer());
	2125	try expectEqualHexStrings("\x69\x45\xFC\x10\x00\x00\x00", code.emitted(), "imul eax, dword ptr [rbp - 4], 0x10");
	2126	try lowerToRmiEnc(.imul, .ax, RegisterOrMemory.mem(.rbp, -2, .word_ptr), 0x10, code.buffer());
	2127	try expectEqualHexStrings("\x66\x69\x45\xFE\x10\x00", code.emitted(), "imul ax, word ptr [rbp - 2], 0x10");
1933	2128	try lowerToRmiEnc(.imul, .r12, RegisterOrMemory.reg(.r12), 0x10, code.buffer());
1934	2129	try expectEqualHexStrings("\x4D\x69\xE4\x10\x00\x00\x00", code.emitted(), "imul r12, r12, 0x10");
	2130	try lowerToRmiEnc(.imul, .r12w, RegisterOrMemory.reg(.r12w), 0x10, code.buffer());
	2131	try expectEqualHexStrings("\x66\x45\x69\xE4\x10\x00", code.emitted(), "imul r12w, r12w, 0x10");
1935	2132	}

src/arch/x86_64/Mir.zig+32-23

...	...	@@ -38,13 +38,18 @@ pub const Inst = struct {
38	38	/// 0b01 reg1, [reg2 + imm32]
39	39	/// 0b01 reg1, [ds:imm32]
40	40	/// 0b10 [reg1 + imm32], reg2
41		/// 0b10 [reg1 + 0], imm32
42		/// 0b11 [reg1 + imm32], imm32
43	41	/// Notes:
44	42	/// * If reg2 is `none` then it means Data field `imm` is used as the immediate.
45	43	/// * When two imm32 values are required, Data field `payload` points at `ImmPair`.
46	44	adc,
47	45
	46	/// ops flags: form:
	47	/// 0b00 byte ptr [reg1 + imm32], imm8
	48	/// 0b01 word ptr [reg1 + imm32], imm16
	49	/// 0b10 dword ptr [reg1 + imm32], imm32
	50	/// 0b11 qword ptr [reg1 + imm32], imm32 (sign-extended to imm64)
	51	adc_mem_imm,
	52
48	53	/// form: reg1, [reg2 + scale*rcx + imm32]
49	54	/// ops flags scale
50	55	/// 0b00 1
...	...	@@ -77,74 +82,95 @@ pub const Inst = struct {
77	82	// The following instructions all have the same encoding as `adc`.
78	83
79	84	add,
	85	add_mem_imm,
80	86	add_scale_src,
81	87	add_scale_dst,
82	88	add_scale_imm,
83	89	sub,
	90	sub_mem_imm,
84	91	sub_scale_src,
85	92	sub_scale_dst,
86	93	sub_scale_imm,
87	94	xor,
	95	xor_mem_imm,
88	96	xor_scale_src,
89	97	xor_scale_dst,
90	98	xor_scale_imm,
91	99	@"and",
	100	and_mem_imm,
92	101	and_scale_src,
93	102	and_scale_dst,
94	103	and_scale_imm,
95	104	@"or",
	105	or_mem_imm,
96	106	or_scale_src,
97	107	or_scale_dst,
98	108	or_scale_imm,
99	109	rol,
	110	rol_mem_imm,
100	111	rol_scale_src,
101	112	rol_scale_dst,
102	113	rol_scale_imm,
103	114	ror,
	115	ror_mem_imm,
104	116	ror_scale_src,
105	117	ror_scale_dst,
106	118	ror_scale_imm,
107	119	rcl,
	120	rcl_mem_imm,
108	121	rcl_scale_src,
109	122	rcl_scale_dst,
110	123	rcl_scale_imm,
111	124	rcr,
	125	rcr_mem_imm,
112	126	rcr_scale_src,
113	127	rcr_scale_dst,
114	128	rcr_scale_imm,
115	129	shl,
	130	shl_mem_imm,
116	131	shl_scale_src,
117	132	shl_scale_dst,
118	133	shl_scale_imm,
119	134	sal,
	135	sal_mem_imm,
120	136	sal_scale_src,
121	137	sal_scale_dst,
122	138	sal_scale_imm,
123	139	shr,
	140	shr_mem_imm,
124	141	shr_scale_src,
125	142	shr_scale_dst,
126	143	shr_scale_imm,
127	144	sar,
	145	sar_mem_imm,
128	146	sar_scale_src,
129	147	sar_scale_dst,
130	148	sar_scale_imm,
131	149	sbb,
	150	sbb_mem_imm,
132	151	sbb_scale_src,
133	152	sbb_scale_dst,
134	153	sbb_scale_imm,
135	154	cmp,
	155	cmp_mem_imm,
136	156	cmp_scale_src,
137	157	cmp_scale_dst,
138	158	cmp_scale_imm,
139	159	mov,
	160	mov_mem_imm,
140	161	mov_scale_src,
141	162	mov_scale_dst,
142	163	mov_scale_imm,
143	164
	165	/// ops flags: form:
	166	/// 0b00 reg1, [reg2 + imm32]
	167	/// 0b00 reg1, [ds:imm32]
	168	/// 0b01 reg1, [rip + imm32]
	169	/// 0b10 reg1, [rip + reloc]
	170	/// Notes:
	171	/// * if flags are 0b10, `Data` contains `got_entry` for the linker to generate
	172	/// a valid relocation for.
144	173	lea,
145		lea_scale_src,
146		lea_scale_dst,
147		lea_scale_imm,
148	174
149	175	/// ops flags: form:
150	176	/// 0bX0 reg1
...	...	@@ -160,15 +186,6 @@ pub const Inst = struct {
160	186	/// 0b11 reg1, [reg2 + imm32], imm32
161	187	imul_complex,
162	188
163		/// ops flags: form:
164		/// 0bX0 reg1, [rip + imm32]
165		/// 0bX1 reg1, [rip + reloc]
166		/// Notes:
167		/// * if flags are 0bX1, `Data` contains `got_entry` for linker to generate
168		/// valid relocation for.
169		/// TODO handle more cases
170		lea_rip,
171
172	189	/// ops flags: form:
173	190	/// 0bX0 reg1, imm64
174	191	/// 0bX1 rax, moffs64
...	...	@@ -233,16 +250,8 @@ pub const Inst = struct {
233	250	syscall,
234	251
235	252	/// ops flags: form:
236		/// 0b00 reg1, reg2
237	253	/// 0b00 reg1, imm32
238		/// 0b01 reg1, [reg2 + imm32]
239		/// 0b01 reg1, [ds:imm32]
240		/// 0b10 [reg1 + imm32], reg2
241		/// 0b10 [reg1 + 0], imm32
242		/// 0b11 [reg1 + imm32], imm32
243		/// Notes:
244		/// * If reg2 is `none` then it means Data field `imm` is used as the immediate.
245		/// * When two imm32 values are required, Data field `payload` points at `ImmPair`.
	254	/// TODO handle more cases
246	255	@"test",
247	256
248	257	/// Breakpoint

src/arch/x86_64/PrintMir.zig+119-270

...	...	@@ -25,6 +25,7 @@ const Type = @import("../../type.zig").Type;
25	25	const fmtIntSizeBin = std.fmt.fmtIntSizeBin;
26	26
27	27	mir: Mir,
	28	bin_file: *link.File,
28	29
29	30	pub fn printMir(print: *const Print, w: anytype, mir_to_air_map: std.AutoHashMap(Mir.Inst.Index, Air.Inst.Index), air: Air) !void {
30	31	const instruction_bytes = print.mir.instructions.len *
...	...	@@ -64,6 +65,15 @@ pub fn printMir(print: *const Print, w: anytype, mir_to_air_map: std.AutoHashMap
64	65	.sbb => try print.mirArith(.sbb, inst, w),
65	66	.cmp => try print.mirArith(.cmp, inst, w),
66	67
	68	.adc_mem_imm => try print.mirArithMemImm(.adc, inst, w),
	69	.add_mem_imm => try print.mirArithMemImm(.add, inst, w),
	70	.sub_mem_imm => try print.mirArithMemImm(.sub, inst, w),
	71	.xor_mem_imm => try print.mirArithMemImm(.xor, inst, w),
	72	.and_mem_imm => try print.mirArithMemImm(.@"and", inst, w),
	73	.or_mem_imm => try print.mirArithMemImm(.@"or", inst, w),
	74	.sbb_mem_imm => try print.mirArithMemImm(.sbb, inst, w),
	75	.cmp_mem_imm => try print.mirArithMemImm(.cmp, inst, w),
	76
67	77	.adc_scale_src => try print.mirArithScaleSrc(.adc, inst, w),
68	78	.add_scale_src => try print.mirArithScaleSrc(.add, inst, w),
69	79	.sub_scale_src => try print.mirArithScaleSrc(.sub, inst, w),
...	...	@@ -98,7 +108,6 @@ pub fn printMir(print: *const Print, w: anytype, mir_to_air_map: std.AutoHashMap
98	108	.movabs => try print.mirMovabs(inst, w),
99	109
100	110	.lea => try print.mirLea(inst, w),
101		.lea_rip => try print.mirLeaRip(inst, w),
102	111
103	112	.imul_complex => try print.mirIMulComplex(inst, w),
104	113
...	...	@@ -108,15 +117,15 @@ pub fn printMir(print: *const Print, w: anytype, mir_to_air_map: std.AutoHashMap
108	117	.jmp => try print.mirJmpCall(.jmp, inst, w),
109	118	.call => try print.mirJmpCall(.call, inst, w),
110	119
111		// .cond_jmp_greater_less => try print.mirCondJmp(.cond_jmp_greater_less, inst, w),
112		// .cond_jmp_above_below => try print.mirCondJmp(.cond_jmp_above_below, inst, w),
113		// .cond_jmp_eq_ne => try print.mirCondJmp(.cond_jmp_eq_ne, inst, w),
	120	.cond_jmp_greater_less => try print.mirCondJmp(.cond_jmp_greater_less, inst, w),
	121	.cond_jmp_above_below => try print.mirCondJmp(.cond_jmp_above_below, inst, w),
	122	.cond_jmp_eq_ne => try print.mirCondJmp(.cond_jmp_eq_ne, inst, w),
114	123
115		// .cond_set_byte_greater_less => try print.mirCondSetByte(.cond_set_byte_greater_less, inst, w),
116		// .cond_set_byte_above_below => try print.mirCondSetByte(.cond_set_byte_above_below, inst, w),
117		// .cond_set_byte_eq_ne => try print.mirCondSetByte(.cond_set_byte_eq_ne, inst, w),
	124	.cond_set_byte_greater_less => try print.mirCondSetByte(.cond_set_byte_greater_less, inst, w),
	125	.cond_set_byte_above_below => try print.mirCondSetByte(.cond_set_byte_above_below, inst, w),
	126	.cond_set_byte_eq_ne => try print.mirCondSetByte(.cond_set_byte_eq_ne, inst, w),
118	127
119		// .@"test" => try print.mirTest(inst, w),
	128	.@"test" => try print.mirTest(inst, w),
120	129
121	130	.brk => try w.writeAll("brk\n"),
122	131	.ret => try w.writeAll("ret\n"),
...	...	@@ -201,254 +210,24 @@ fn mirJmpCall(print: *const Print, tag: Mir.Inst.Tag, inst: Mir.Inst.Index, w: a
201	210	try w.print("{s}\n", .{@tagName(ops.reg1)});
202	211	}
203	212
204		const CondType = enum {
205		/// greater than or equal
206		gte,
207
208		/// greater than
209		gt,
210
211		/// less than
212		lt,
213
214		/// less than or equal
215		lte,
216
217		/// above or equal
218		ae,
219
220		/// above
221		a,
222
223		/// below
224		b,
225
226		/// below or equal
227		be,
228
229		/// not equal
230		ne,
231
232		/// equal
233		eq,
234
235		fn fromTagAndFlags(tag: Mir.Inst.Tag, flags: u2) CondType {
236		return switch (tag) {
237		.cond_jmp_greater_less,
238		.cond_set_byte_greater_less,
239		=> switch (flags) {
240		0b00 => CondType.gte,
241		0b01 => CondType.gt,
242		0b10 => CondType.lt,
243		0b11 => CondType.lte,
244		},
245		.cond_jmp_above_below,
246		.cond_set_byte_above_below,
247		=> switch (flags) {
248		0b00 => CondType.ae,
249		0b01 => CondType.a,
250		0b10 => CondType.b,
251		0b11 => CondType.be,
252		},
253		.cond_jmp_eq_ne,
254		.cond_set_byte_eq_ne,
255		=> switch (@truncate(u1, flags)) {
256		0b0 => CondType.ne,
257		0b1 => CondType.eq,
258		},
259		else => unreachable,
260		};
261		}
262		};
263
264		inline fn getCondOpCode(tag: Mir.Inst.Tag, cond: CondType) u8 {
265		switch (cond) {
266		.gte => return switch (tag) {
267		.cond_jmp_greater_less => 0x8d,
268		.cond_set_byte_greater_less => 0x9d,
269		else => unreachable,
270		},
271		.gt => return switch (tag) {
272		.cond_jmp_greater_less => 0x8f,
273		.cond_set_byte_greater_less => 0x9f,
274		else => unreachable,
275		},
276		.lt => return switch (tag) {
277		.cond_jmp_greater_less => 0x8c,
278		.cond_set_byte_greater_less => 0x9c,
279		else => unreachable,
280		},
281		.lte => return switch (tag) {
282		.cond_jmp_greater_less => 0x8e,
283		.cond_set_byte_greater_less => 0x9e,
284		else => unreachable,
285		},
286		.ae => return switch (tag) {
287		.cond_jmp_above_below => 0x83,
288		.cond_set_byte_above_below => 0x93,
289		else => unreachable,
290		},
291		.a => return switch (tag) {
292		.cond_jmp_above_below => 0x87,
293		.cond_set_byte_greater_less => 0x97,
294		else => unreachable,
295		},
296		.b => return switch (tag) {
297		.cond_jmp_above_below => 0x82,
298		.cond_set_byte_greater_less => 0x92,
299		else => unreachable,
300		},
301		.be => return switch (tag) {
302		.cond_jmp_above_below => 0x86,
303		.cond_set_byte_greater_less => 0x96,
304		else => unreachable,
305		},
306		.eq => return switch (tag) {
307		.cond_jmp_eq_ne => 0x84,
308		.cond_set_byte_eq_ne => 0x94,
309		else => unreachable,
310		},
311		.ne => return switch (tag) {
312		.cond_jmp_eq_ne => 0x85,
313		.cond_set_byte_eq_ne => 0x95,
314		else => unreachable,
315		},
316		}
317		}
318
319	213	fn mirCondJmp(print: *const Print, tag: Mir.Inst.Tag, inst: Mir.Inst.Index, w: anytype) !void {
320		_ = w; // TODO
321		const ops = Mir.Ops.decode(print.mir.instructions.items(.ops)[inst]);
322		const target = print.mir.instructions.items(.data)[inst].inst;
323		const cond = CondType.fromTagAndFlags(tag, ops.flags);
324		const opc = getCondOpCode(tag, cond);
325		const source = print.code.items.len;
326		const encoder = try Encoder.init(print.code, 6);
327		encoder.opcode_2byte(0x0f, opc);
328		try print.relocs.append(print.bin_file.allocator, .{
329		.source = source,
330		.target = target,
331		.offset = print.code.items.len,
332		.length = 6,
333		});
334		encoder.imm32(0);
	214	_ = print;
	215	_ = tag;
	216	_ = inst;
	217	try w.writeAll("TODO print mirCondJmp\n");
335	218	}
336	219
337	220	fn mirCondSetByte(print: *const Print, tag: Mir.Inst.Tag, inst: Mir.Inst.Index, w: anytype) !void {
338		_ = w; // TODO
339		const ops = Mir.Ops.decode(print.mir.instructions.items(.ops)[inst]);
340		const cond = CondType.fromTagAndFlags(tag, ops.flags);
341		const opc = getCondOpCode(tag, cond);
342		const encoder = try Encoder.init(print.code, 4);
343		encoder.rex(.{
344		.w = true,
345		.b = ops.reg1.isExtended(),
346		});
347		encoder.opcode_2byte(0x0f, opc);
348		encoder.modRm_direct(0x0, ops.reg1.lowId());
	221	_ = tag;
	222	_ = inst;
	223	_ = print;
	224	try w.writeAll("TODO print mirCondSetByte\n");
349	225	}
350	226
351	227	fn mirTest(print: *const Print, inst: Mir.Inst.Index, w: anytype) !void {
352		_ = w; // TODO
353		const tag = print.mir.instructions.items(.tag)[inst];
354		assert(tag == .@"test");
355		const ops = Mir.Ops.decode(print.mir.instructions.items(.ops)[inst]);
356		switch (ops.flags) {
357		0b00 => blk: {
358		if (ops.reg2 == .none) {
359		// TEST r/m64, imm32
360		const imm = print.mir.instructions.items(.data)[inst].imm;
361		if (ops.reg1.to64() == .rax) {
362		// TODO reduce the size of the instruction if the immediate
363		// is smaller than 32 bits
364		const encoder = try Encoder.init(print.code, 6);
365		encoder.rex(.{
366		.w = true,
367		});
368		encoder.opcode_1byte(0xa9);
369		encoder.imm32(imm);
370		break :blk;
371		}
372		const opc: u8 = if (ops.reg1.size() == 8) 0xf6 else 0xf7;
373		const encoder = try Encoder.init(print.code, 7);
374		encoder.rex(.{
375		.w = true,
376		.b = ops.reg1.isExtended(),
377		});
378		encoder.opcode_1byte(opc);
379		encoder.modRm_direct(0, ops.reg1.lowId());
380		encoder.imm8(@intCast(i8, imm));
381		break :blk;
382		}
383		// TEST r/m64, r64
384		return print.fail("TODO TEST r/m64, r64", .{});
385		},
386		else => return print.fail("TODO more TEST alternatives", .{}),
387		}
388		}
389
390		const EncType = enum {
391		/// OP r/m64, imm32
392		mi,
393
394		/// OP r/m64, r64
395		mr,
396
397		/// OP r64, r/m64
398		rm,
399		};
400
401		const OpCode = struct {
402		opc: u8,
403		/// Only used if `EncType == .mi`.
404		modrm_ext: u3,
405		};
406
407		inline fn getArithOpCode(tag: Mir.Inst.Tag, enc: EncType) OpCode {
408		switch (enc) {
409		.mi => return switch (tag) {
410		.adc => .{ .opc = 0x81, .modrm_ext = 0x2 },
411		.add => .{ .opc = 0x81, .modrm_ext = 0x0 },
412		.sub => .{ .opc = 0x81, .modrm_ext = 0x5 },
413		.xor => .{ .opc = 0x81, .modrm_ext = 0x6 },
414		.@"and" => .{ .opc = 0x81, .modrm_ext = 0x4 },
415		.@"or" => .{ .opc = 0x81, .modrm_ext = 0x1 },
416		.sbb => .{ .opc = 0x81, .modrm_ext = 0x3 },
417		.cmp => .{ .opc = 0x81, .modrm_ext = 0x7 },
418		.mov => .{ .opc = 0xc7, .modrm_ext = 0x0 },
419		else => unreachable,
420		},
421		.mr => {
422		const opc: u8 = switch (tag) {
423		.adc => 0x11,
424		.add => 0x01,
425		.sub => 0x29,
426		.xor => 0x31,
427		.@"and" => 0x21,
428		.@"or" => 0x09,
429		.sbb => 0x19,
430		.cmp => 0x39,
431		.mov => 0x89,
432		else => unreachable,
433		};
434		return .{ .opc = opc, .modrm_ext = undefined };
435		},
436		.rm => {
437		const opc: u8 = switch (tag) {
438		.adc => 0x13,
439		.add => 0x03,
440		.sub => 0x2b,
441		.xor => 0x33,
442		.@"and" => 0x23,
443		.@"or" => 0x0b,
444		.sbb => 0x1b,
445		.cmp => 0x3b,
446		.mov => 0x8b,
447		else => unreachable,
448		};
449		return .{ .opc = opc, .modrm_ext = undefined };
450		},
451		}
	228	_ = print;
	229	_ = inst;
	230	try w.writeAll("TODO print mirTest\n");
452	231	}
453	232
454	233	fn mirArith(print: *const Print, tag: Mir.Inst.Tag, inst: Mir.Inst.Index, w: anytype) !void {
...	...	@@ -465,31 +244,63 @@ fn mirArith(print: *const Print, tag: Mir.Inst.Tag, inst: Mir.Inst.Index, w: any
465	244	0b01 => {
466	245	const imm = print.mir.instructions.items(.data)[inst].imm;
467	246	if (ops.reg2 == .none) {
468		try w.print("{s}, [ds:{d}]", .{ @tagName(ops.reg1), imm });
	247	try w.print("{s}, ", .{@tagName(ops.reg1)});
	248	switch (ops.reg1.size()) {
	249	8 => try w.print("byte ptr ", .{}),
	250	16 => try w.print("word ptr ", .{}),
	251	32 => try w.print("dword ptr ", .{}),
	252	64 => try w.print("qword ptr ", .{}),
	253	else => unreachable,
	254	}
	255	try w.print("[ds:{d}]", .{imm});
469	256	} else {
470		try w.print("{s}, [{s} + {d}]", .{ @tagName(ops.reg1), @tagName(ops.reg2), imm });
	257	try w.print("{s}, ", .{@tagName(ops.reg1)});
	258	switch (ops.reg1.size()) {
	259	8 => try w.print("byte ptr ", .{}),
	260	16 => try w.print("word ptr ", .{}),
	261	32 => try w.print("dword ptr ", .{}),
	262	64 => try w.print("qword ptr ", .{}),
	263	else => unreachable,
	264	}
	265	try w.print("[{s} + {d}]", .{ @tagName(ops.reg2), imm });
471	266	}
472	267	},
473	268	0b10 => {
474	269	const imm = print.mir.instructions.items(.data)[inst].imm;
475	270	if (ops.reg2 == .none) {
476		try w.print("[{s} + 0], {d}", .{ @tagName(ops.reg1), imm });
	271	try w.writeAll("unused variant");
477	272	} else {
	273	switch (ops.reg2.size()) {
	274	8 => try w.print("byte ptr ", .{}),
	275	16 => try w.print("word ptr ", .{}),
	276	32 => try w.print("dword ptr ", .{}),
	277	64 => try w.print("qword ptr ", .{}),
	278	else => unreachable,
	279	}
478	280	try w.print("[{s} + {d}], {s}", .{ @tagName(ops.reg1), imm, @tagName(ops.reg2) });
479	281	}
480	282	},
481	283	0b11 => {
482		if (ops.reg2 == .none) {
483		const payload = print.mir.instructions.items(.data)[inst].payload;
484		const imm_pair = print.mir.extraData(Mir.ImmPair, payload).data;
485		try w.print("[{s} + {d}], {d}", .{ @tagName(ops.reg1), imm_pair.dest_off, imm_pair.operand });
486		}
487		try w.writeAll("TODO");
	284	try w.writeAll("unused variant");
488	285	},
489	286	}
490	287	try w.writeByte('\n');
491	288	}
492	289
	290	fn mirArithMemImm(print: *const Print, tag: Mir.Inst.Tag, inst: Mir.Inst.Index, w: anytype) !void {
	291	const ops = Mir.Ops.decode(print.mir.instructions.items(.ops)[inst]);
	292	const payload = print.mir.instructions.items(.data)[inst].payload;
	293	const imm_pair = print.mir.extraData(Mir.ImmPair, payload).data;
	294	try w.print("{s} ", .{@tagName(tag)});
	295	switch (ops.flags) {
	296	0b00 => try w.print("byte ptr ", .{}),
	297	0b01 => try w.print("word ptr ", .{}),
	298	0b10 => try w.print("dword ptr ", .{}),
	299	0b11 => try w.print("qword ptr ", .{}),
	300	}
	301	try w.print("[{s} + {d}], {d}\n", .{ @tagName(ops.reg1), imm_pair.dest_off, imm_pair.operand });
	302	}
	303
493	304	fn mirArithScaleSrc(print: *const Print, tag: Mir.Inst.Tag, inst: Mir.Inst.Index, w: anytype) !void {
494	305	const ops = Mir.Ops.decode(print.mir.instructions.items(.ops)[inst]);
495	306	const scale = ops.flags;
...	...	@@ -560,19 +371,57 @@ fn mirIMulComplex(print: *const Print, inst: Mir.Inst.Index, w: anytype) !void {
560	371	}
561	372
562	373	fn mirLea(print: *const Print, inst: Mir.Inst.Index, w: anytype) !void {
563		const tag = print.mir.instructions.items(.tag)[inst];
564		assert(tag == .lea);
565	374	const ops = Mir.Ops.decode(print.mir.instructions.items(.ops)[inst]);
566		assert(ops.flags == 0b01);
567		const imm = print.mir.instructions.items(.data)[inst].imm;
568
569		try w.print("lea {s} [{s} + {d}]\n", .{ @tagName(ops.reg1), @tagName(ops.reg2), imm });
570		}
571
572		fn mirLeaRip(print: *const Print, inst: Mir.Inst.Index, w: anytype) !void {
573		_ = print;
574		_ = inst;
575		return w.writeAll("TODO lea_rip\n");
	375	try w.writeAll("lea ");
	376	switch (ops.flags) {
	377	0b00 => {
	378	const imm = print.mir.instructions.items(.data)[inst].imm;
	379	try w.print("{s} [", .{@tagName(ops.reg1)});
	380	if (ops.reg2 != .none) {
	381	try w.print("{s} + ", .{@tagName(ops.reg2)});
	382	} else {
	383	try w.print("ds:", .{});
	384	}
	385	try w.print("{d}]\n", .{imm});
	386	},
	387	0b01 => {
	388	try w.print("{s}, ", .{@tagName(ops.reg1)});
	389	switch (ops.reg1.size()) {
	390	8 => try w.print("byte ptr ", .{}),
	391	16 => try w.print("word ptr ", .{}),
	392	32 => try w.print("dword ptr ", .{}),
	393	64 => try w.print("qword ptr ", .{}),
	394	else => unreachable,
	395	}
	396	try w.print("[rip + 0x0] ", .{});
	397	const payload = print.mir.instructions.items(.data)[inst].payload;
	398	const imm = print.mir.extraData(Mir.Imm64, payload).data.decode();
	399	try w.print("target@{x}", .{imm});
	400	},
	401	0b10 => {
	402	try w.print("{s}, ", .{@tagName(ops.reg1)});
	403	switch (ops.reg1.size()) {
	404	8 => try w.print("byte ptr ", .{}),
	405	16 => try w.print("word ptr ", .{}),
	406	32 => try w.print("dword ptr ", .{}),
	407	64 => try w.print("qword ptr ", .{}),
	408	else => unreachable,
	409	}
	410	try w.print("[rip + 0x0] ", .{});
	411	const got_entry = print.mir.instructions.items(.data)[inst].got_entry;
	412	if (print.bin_file.cast(link.File.MachO)) \|macho_file\| {
	413	const target = macho_file.locals.items[got_entry];
	414	const target_name = macho_file.getString(target.n_strx);
	415	try w.print("target@{s}", .{target_name});
	416	} else {
	417	try w.writeAll("TODO lea reg, [rip + reloc] for linking backends different than MachO");
	418	}
	419	},
	420	0b11 => {
	421	try w.writeAll("unused variant\n");
	422	},
	423	}
	424	try w.writeAll("\n");
576	425	}
577	426
578	427	fn mirCallExtern(print: *const Print, inst: Mir.Inst.Index, w: anytype) !void {

test/stage2/x86_64.zig+58

...	...	@@ -1604,6 +1604,64 @@ pub fn addCases(ctx: *TestContext) !void {
1604	1604	":2:28: error: cannot set address space of local variable 'foo'",
1605	1605	});
1606	1606	}
	1607
	1608	{
	1609	var case = ctx.exe("saving vars of different ABI size to stack", target);
	1610
	1611	case.addCompareOutput(
	1612	\\pub fn main() void {
	1613	\\ assert(callMe(2) == 24);
	1614	\\}
	1615	\\
	1616	\\fn callMe(a: u8) u8 {
	1617	\\ var b: u8 = a + 10;
	1618	\\ const c = 2 * b;
	1619	\\ return c;
	1620	\\}
	1621	\\
	1622	\\pub fn assert(ok: bool) void {
	1623	\\ if (!ok) unreachable; // assertion failure
	1624	\\}
	1625	,
	1626	"",
	1627	);
	1628
	1629	case.addCompareOutput(
	1630	\\pub fn main() void {
	1631	\\ assert(callMe(2) == 24);
	1632	\\}
	1633	\\
	1634	\\fn callMe(a: u16) u16 {
	1635	\\ var b: u16 = a + 10;
	1636	\\ const c = 2 * b;
	1637	\\ return c;
	1638	\\}
	1639	\\
	1640	\\pub fn assert(ok: bool) void {
	1641	\\ if (!ok) unreachable; // assertion failure
	1642	\\}
	1643	,
	1644	"",
	1645	);
	1646
	1647	case.addCompareOutput(
	1648	\\pub fn main() void {
	1649	\\ assert(callMe(2) == 24);
	1650	\\}
	1651	\\
	1652	\\fn callMe(a: u32) u32 {
	1653	\\ var b: u32 = a + 10;
	1654	\\ const c = 2 * b;
	1655	\\ return c;
	1656	\\}
	1657	\\
	1658	\\pub fn assert(ok: bool) void {
	1659	\\ if (!ok) unreachable; // assertion failure
	1660	\\}
	1661	,
	1662	"",
	1663	);
	1664	}
1607	1665	}
1608	1666	}
1609	1667