Commit e7ac05e882fa4290af4a41e9cae63105bcacb283 | zig

author	Jakub Konka <kubkon@jakubkonka.com> 2021-12-31 11:18:23+01:00
committer	Jakub Konka <kubkon@jakubkonka.com> 2021-12-31 11:18:23+01:00
log	e7ac05e882fa4290af4a41e9cae63105bcacb283
tree	b244a82307a3abccec98a777af0d9d13362e1787
parent	2d9508780a3578ecddb4948eb459768ad5bf8720

stage2: rename Emit to Isel for x86_64

4 files changed, 2138 insertions(+), 2138 deletions(-)

CMakeLists.txt
src
- arch
  - x86_64
    CodeGen.zig
    Emit.zig
    Isel.zig

CMakeLists.txt+1-1

...	...	@@ -578,7 +578,7 @@ set(ZIG_STAGE2_SOURCES
578	578	"${CMAKE_SOURCE_DIR}/src/arch/wasm/Emit.zig"
579	579	"${CMAKE_SOURCE_DIR}/src/arch/wasm/Mir.zig"
580	580	"${CMAKE_SOURCE_DIR}/src/arch/x86_64/CodeGen.zig"
581		"${CMAKE_SOURCE_DIR}/src/arch/x86_64/Emit.zig"
	581	"${CMAKE_SOURCE_DIR}/src/arch/x86_64/Isel.zig"
582	582	"${CMAKE_SOURCE_DIR}/src/arch/x86_64/Mir.zig"
583	583	"${CMAKE_SOURCE_DIR}/src/arch/x86_64/bits.zig"
584	584	"${CMAKE_SOURCE_DIR}/src/clang.zig"

src/arch/x86_64/CodeGen.zig+5-5

...	...	@@ -14,10 +14,10 @@ const Allocator = mem.Allocator;
14	14	const Compilation = @import("../../Compilation.zig");
15	15	const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;
16	16	const DW = std.dwarf;
17		const Emit = @import("Emit.zig");
18	17	const ErrorMsg = Module.ErrorMsg;
19	18	const FnResult = @import("../../codegen.zig").FnResult;
20	19	const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError;
	20	const Isel = @import("Isel.zig");
21	21	const Liveness = @import("../../Liveness.zig");
22	22	const Mir = @import("Mir.zig");
23	23	const Module = @import("../../Module.zig");
...	...	@@ -309,7 +309,7 @@ pub fn generate(
309	309	};
310	310	defer mir.deinit(bin_file.allocator);
311	311
312		var emit = Emit{
	312	var isel = Isel{
313	313	.mir = mir,
314	314	.bin_file = bin_file,
315	315	.debug_output = debug_output,
...	...	@@ -320,9 +320,9 @@ pub fn generate(
320	320	.prev_di_line = module_fn.lbrace_line,
321	321	.prev_di_column = module_fn.lbrace_column,
322	322	};
323		defer emit.deinit();
324		emit.emitMir() catch \|err\| switch (err) {
325		error.EmitFail => return FnResult{ .fail = emit.err_msg.? },
	323	defer isel.deinit();
	324	isel.lowerMir() catch \|err\| switch (err) {
	325	error.IselFail => return FnResult{ .fail = isel.err_msg.? },
326	326	else => \|e\| return e,
327	327	};
328	328

src/arch/x86_64/Emit.zig deleted-2132

...	...	@@ -1,2132 +0,0 @@
1		//! This file contains the functionality for lowering x86_64 MIR into
2		//! machine code
3
4		const Emit = @This();
5
6		const std = @import("std");
7		const assert = std.debug.assert;
8		const bits = @import("bits.zig");
9		const leb128 = std.leb;
10		const link = @import("../../link.zig");
11		const log = std.log.scoped(.codegen);
12		const math = std.math;
13		const mem = std.mem;
14		const testing = std.testing;
15
16		const Air = @import("../../Air.zig");
17		const Allocator = mem.Allocator;
18		const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;
19		const DW = std.dwarf;
20		const Encoder = bits.Encoder;
21		const ErrorMsg = Module.ErrorMsg;
22		const MCValue = @import("CodeGen.zig").MCValue;
23		const Mir = @import("Mir.zig");
24		const Module = @import("../../Module.zig");
25		const Instruction = bits.Instruction;
26		const Register = bits.Register;
27		const Type = @import("../../type.zig").Type;
28
29		mir: Mir,
30		bin_file: *link.File,
31		debug_output: DebugInfoOutput,
32		target: *const std.Target,
33		err_msg: ?*ErrorMsg = null,
34		src_loc: Module.SrcLoc,
35		code: *std.ArrayList(u8),
36
37		prev_di_line: u32,
38		prev_di_column: u32,
39		/// Relative to the beginning of `code`.
40		prev_di_pc: usize,
41
42		code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .{},
43		relocs: std.ArrayListUnmanaged(Reloc) = .{},
44
45		const InnerError = error{
46		OutOfMemory,
47		Overflow,
48		EmitFail,
49		};
50
51		const Reloc = struct {
52		/// Offset of the instruction.
53		source: u64,
54		/// Target of the relocation.
55		target: Mir.Inst.Index,
56		/// Offset of the relocation within the instruction.
57		offset: u64,
58		/// Length of the instruction.
59		length: u5,
60		};
61
62		pub fn emitMir(emit: *Emit) InnerError!void {
63		const mir_tags = emit.mir.instructions.items(.tag);
64
65		for (mir_tags) \|tag, index\| {
66		const inst = @intCast(u32, index);
67		try emit.code_offset_mapping.putNoClobber(emit.bin_file.allocator, inst, emit.code.items.len);
68		switch (tag) {
69		.adc => try emit.mirArith(.adc, inst),
70		.add => try emit.mirArith(.add, inst),
71		.sub => try emit.mirArith(.sub, inst),
72		.xor => try emit.mirArith(.xor, inst),
73		.@"and" => try emit.mirArith(.@"and", inst),
74		.@"or" => try emit.mirArith(.@"or", inst),
75		.sbb => try emit.mirArith(.sbb, inst),
76		.cmp => try emit.mirArith(.cmp, inst),
77		.mov => try emit.mirArith(.mov, inst),
78
79		.adc_mem_imm => try emit.mirArithMemImm(.adc, inst),
80		.add_mem_imm => try emit.mirArithMemImm(.add, inst),
81		.sub_mem_imm => try emit.mirArithMemImm(.sub, inst),
82		.xor_mem_imm => try emit.mirArithMemImm(.xor, inst),
83		.and_mem_imm => try emit.mirArithMemImm(.@"and", inst),
84		.or_mem_imm => try emit.mirArithMemImm(.@"or", inst),
85		.sbb_mem_imm => try emit.mirArithMemImm(.sbb, inst),
86		.cmp_mem_imm => try emit.mirArithMemImm(.cmp, inst),
87		.mov_mem_imm => try emit.mirArithMemImm(.mov, inst),
88
89		.adc_scale_src => try emit.mirArithScaleSrc(.adc, inst),
90		.add_scale_src => try emit.mirArithScaleSrc(.add, inst),
91		.sub_scale_src => try emit.mirArithScaleSrc(.sub, inst),
92		.xor_scale_src => try emit.mirArithScaleSrc(.xor, inst),
93		.and_scale_src => try emit.mirArithScaleSrc(.@"and", inst),
94		.or_scale_src => try emit.mirArithScaleSrc(.@"or", inst),
95		.sbb_scale_src => try emit.mirArithScaleSrc(.sbb, inst),
96		.cmp_scale_src => try emit.mirArithScaleSrc(.cmp, inst),
97		.mov_scale_src => try emit.mirArithScaleSrc(.mov, inst),
98
99		.adc_scale_dst => try emit.mirArithScaleDst(.adc, inst),
100		.add_scale_dst => try emit.mirArithScaleDst(.add, inst),
101		.sub_scale_dst => try emit.mirArithScaleDst(.sub, inst),
102		.xor_scale_dst => try emit.mirArithScaleDst(.xor, inst),
103		.and_scale_dst => try emit.mirArithScaleDst(.@"and", inst),
104		.or_scale_dst => try emit.mirArithScaleDst(.@"or", inst),
105		.sbb_scale_dst => try emit.mirArithScaleDst(.sbb, inst),
106		.cmp_scale_dst => try emit.mirArithScaleDst(.cmp, inst),
107		.mov_scale_dst => try emit.mirArithScaleDst(.mov, inst),
108
109		.adc_scale_imm => try emit.mirArithScaleImm(.adc, inst),
110		.add_scale_imm => try emit.mirArithScaleImm(.add, inst),
111		.sub_scale_imm => try emit.mirArithScaleImm(.sub, inst),
112		.xor_scale_imm => try emit.mirArithScaleImm(.xor, inst),
113		.and_scale_imm => try emit.mirArithScaleImm(.@"and", inst),
114		.or_scale_imm => try emit.mirArithScaleImm(.@"or", inst),
115		.sbb_scale_imm => try emit.mirArithScaleImm(.sbb, inst),
116		.cmp_scale_imm => try emit.mirArithScaleImm(.cmp, inst),
117		.mov_scale_imm => try emit.mirArithScaleImm(.mov, inst),
118
119		.movabs => try emit.mirMovabs(inst),
120
121		.lea => try emit.mirLea(inst),
122
123		.imul_complex => try emit.mirIMulComplex(inst),
124
125		.push => try emit.mirPushPop(.push, inst),
126		.pop => try emit.mirPushPop(.pop, inst),
127
128		.jmp => try emit.mirJmpCall(.jmp_near, inst),
129		.call => try emit.mirJmpCall(.call_near, inst),
130
131		.cond_jmp_greater_less,
132		.cond_jmp_above_below,
133		.cond_jmp_eq_ne,
134		=> try emit.mirCondJmp(tag, inst),
135
136		.cond_set_byte_greater_less,
137		.cond_set_byte_above_below,
138		.cond_set_byte_eq_ne,
139		=> try emit.mirCondSetByte(tag, inst),
140
141		.ret => try emit.mirRet(inst),
142
143		.syscall => try emit.mirSyscall(),
144
145		.@"test" => try emit.mirTest(inst),
146
147		.brk => try emit.mirBrk(),
148		.nop => try emit.mirNop(),
149
150		.call_extern => try emit.mirCallExtern(inst),
151
152		.dbg_line => try emit.mirDbgLine(inst),
153		.dbg_prologue_end => try emit.mirDbgPrologueEnd(inst),
154		.dbg_epilogue_begin => try emit.mirDbgEpilogueBegin(inst),
155		.arg_dbg_info => try emit.mirArgDbgInfo(inst),
156
157		.push_regs_from_callee_preserved_regs => try emit.mirPushPopRegsFromCalleePreservedRegs(.push, inst),
158		.pop_regs_from_callee_preserved_regs => try emit.mirPushPopRegsFromCalleePreservedRegs(.pop, inst),
159
160		else => {
161		return emit.fail("Implement MIR->Isel lowering for x86_64 for pseudo-inst: {s}", .{tag});
162		},
163		}
164		}
165
166		try emit.fixupRelocs();
167		}
168
169		pub fn deinit(emit: *Emit) void {
170		emit.relocs.deinit(emit.bin_file.allocator);
171		emit.code_offset_mapping.deinit(emit.bin_file.allocator);
172		emit.* = undefined;
173		}
174
175		fn fail(emit: *Emit, comptime format: []const u8, args: anytype) InnerError {
176		@setCold(true);
177		assert(emit.err_msg == null);
178		emit.err_msg = try ErrorMsg.create(emit.bin_file.allocator, emit.src_loc, format, args);
179		return error.EmitFail;
180		}
181
182		fn failWithLoweringError(emit: *Emit, err: LoweringError) InnerError {
183		return switch (err) {
184		error.RaxOperandExpected => emit.fail("Register.rax expected as destination operand", .{}),
185		error.OperandSizeMismatch => emit.fail("operand size mismatch", .{}),
186		else => \|e\| e,
187		};
188		}
189
190		fn fixupRelocs(emit: *Emit) InnerError!void {
191		// TODO this function currently assumes all relocs via JMP/CALL instructions are 32bit in size.
192		// This should be reversed like it is done in aarch64 MIR emit code: start with the smallest
193		// possible resolution, i.e., 8bit, and iteratively converge on the minimum required resolution
194		// until the entire decl is correctly emitted with all JMP/CALL instructions within range.
195		for (emit.relocs.items) \|reloc\| {
196		const offset = try math.cast(usize, reloc.offset);
197		const target = emit.code_offset_mapping.get(reloc.target) orelse
198		return emit.fail("JMP/CALL relocation target not found!", .{});
199		const disp = @intCast(i32, @intCast(i64, target) - @intCast(i64, reloc.source + reloc.length));
200		mem.writeIntLittle(i32, emit.code.items[offset..][0..4], disp);
201		}
202		}
203
204		fn mirBrk(emit: *Emit) InnerError!void {
205		return lowerToZoEnc(.brk, emit.code) catch \|err\| emit.failWithLoweringError(err);
206		}
207
208		fn mirNop(emit: *Emit) InnerError!void {
209		return lowerToZoEnc(.nop, emit.code) catch \|err\| emit.failWithLoweringError(err);
210		}
211
212		fn mirSyscall(emit: *Emit) InnerError!void {
213		return lowerToZoEnc(.syscall, emit.code) catch \|err\| emit.failWithLoweringError(err);
214		}
215
216		fn mirPushPop(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
217		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
218		switch (ops.flags) {
219		0b00 => {
220		// PUSH/POP reg
221		return lowerToOEnc(tag, ops.reg1, emit.code) catch \|err\| emit.failWithLoweringError(err);
222		},
223		0b01 => {
224		// PUSH/POP r/m64
225		const imm = emit.mir.instructions.items(.data)[inst].imm;
226		const ptr_size: Memory.PtrSize = switch (immOpSize(imm)) {
227		16 => .word_ptr,
228		else => .qword_ptr,
229		};
230		return lowerToMEnc(tag, RegisterOrMemory.mem(ops.reg1, imm, ptr_size), emit.code) catch \|err\|
231		emit.failWithLoweringError(err);
232		},
233		0b10 => {
234		// PUSH imm32
235		assert(tag == .push);
236		const imm = emit.mir.instructions.items(.data)[inst].imm;
237		return lowerToIEnc(.push, imm, emit.code) catch \|err\|
238		emit.failWithLoweringError(err);
239		},
240		0b11 => unreachable,
241		}
242		}
243		fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
244		const callee_preserved_regs = bits.callee_preserved_regs;
245		const regs = emit.mir.instructions.items(.data)[inst].regs_to_push_or_pop;
246		if (tag == .push) {
247		for (callee_preserved_regs) \|reg, i\| {
248		if ((regs >> @intCast(u5, i)) & 1 == 0) continue;
249		lowerToOEnc(.push, reg, emit.code) catch \|err\|
250		return emit.failWithLoweringError(err);
251		}
252		} else {
253		// pop in the reverse direction
254		var i = callee_preserved_regs.len;
255		while (i > 0) : (i -= 1) {
256		const reg = callee_preserved_regs[i - 1];
257		if ((regs >> @intCast(u5, i - 1)) & 1 == 0) continue;
258		lowerToOEnc(.pop, reg, emit.code) catch \|err\|
259		return emit.failWithLoweringError(err);
260		}
261		}
262		}
263
264		fn mirJmpCall(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
265		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
266		const flag = @truncate(u1, ops.flags);
267		if (flag == 0) {
268		const target = emit.mir.instructions.items(.data)[inst].inst;
269		const source = emit.code.items.len;
270		lowerToDEnc(tag, 0, emit.code) catch \|err\|
271		return emit.failWithLoweringError(err);
272		try emit.relocs.append(emit.bin_file.allocator, .{
273		.source = source,
274		.target = target,
275		.offset = emit.code.items.len - 4,
276		.length = 5,
277		});
278		return;
279		}
280		if (ops.reg1 == .none) {
281		// JMP/CALL [imm]
282		const imm = emit.mir.instructions.items(.data)[inst].imm;
283		const ptr_size: Memory.PtrSize = switch (immOpSize(imm)) {
284		16 => .word_ptr,
285		else => .qword_ptr,
286		};
287		return lowerToMEnc(tag, RegisterOrMemory.mem(null, imm, ptr_size), emit.code) catch \|err\|
288		emit.failWithLoweringError(err);
289		}
290		// JMP/CALL reg
291		return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code) catch \|err\| emit.failWithLoweringError(err);
292		}
293
294		fn mirCondJmp(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
295		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
296		const target = emit.mir.instructions.items(.data)[inst].inst;
297		const tag = switch (mir_tag) {
298		.cond_jmp_greater_less => switch (ops.flags) {
299		0b00 => Tag.jge,
300		0b01 => Tag.jg,
301		0b10 => Tag.jl,
302		0b11 => Tag.jle,
303		},
304		.cond_jmp_above_below => switch (ops.flags) {
305		0b00 => Tag.jae,
306		0b01 => Tag.ja,
307		0b10 => Tag.jb,
308		0b11 => Tag.jbe,
309		},
310		.cond_jmp_eq_ne => switch (@truncate(u1, ops.flags)) {
311		0b0 => Tag.jne,
312		0b1 => Tag.je,
313		},
314		else => unreachable,
315		};
316		const source = emit.code.items.len;
317		lowerToDEnc(tag, 0, emit.code) catch \|err\|
318		return emit.failWithLoweringError(err);
319		try emit.relocs.append(emit.bin_file.allocator, .{
320		.source = source,
321		.target = target,
322		.offset = emit.code.items.len - 4,
323		.length = 6,
324		});
325		}
326
327		fn mirCondSetByte(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
328		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
329		const tag = switch (mir_tag) {
330		.cond_set_byte_greater_less => switch (ops.flags) {
331		0b00 => Tag.setge,
332		0b01 => Tag.setg,
333		0b10 => Tag.setl,
334		0b11 => Tag.setle,
335		},
336		.cond_set_byte_above_below => switch (ops.flags) {
337		0b00 => Tag.setae,
338		0b01 => Tag.seta,
339		0b10 => Tag.setb,
340		0b11 => Tag.setbe,
341		},
342		.cond_set_byte_eq_ne => switch (@truncate(u1, ops.flags)) {
343		0b0 => Tag.setne,
344		0b1 => Tag.sete,
345		},
346		else => unreachable,
347		};
348		return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1.to8()), emit.code) catch \|err\|
349		emit.failWithLoweringError(err);
350		}
351
352		fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
353		const tag = emit.mir.instructions.items(.tag)[inst];
354		assert(tag == .@"test");
355		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
356		switch (ops.flags) {
357		0b00 => {
358		if (ops.reg2 == .none) {
359		// TEST r/m64, imm32
360		// MI
361		const imm = emit.mir.instructions.items(.data)[inst].imm;
362		if (ops.reg1.to64() == .rax) {
363		// TEST rax, imm32
364		// I
365		return lowerToIEnc(.@"test", imm, emit.code) catch \|err\|
366		emit.failWithLoweringError(err);
367		}
368		return lowerToMiEnc(.@"test", RegisterOrMemory.reg(ops.reg1), imm, emit.code) catch \|err\|
369		emit.failWithLoweringError(err);
370		}
371		// TEST r/m64, r64
372		return emit.fail("TODO TEST r/m64, r64", .{});
373		},
374		else => return emit.fail("TODO more TEST alternatives", .{}),
375		}
376		}
377
378		fn mirRet(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
379		const tag = emit.mir.instructions.items(.tag)[inst];
380		assert(tag == .ret);
381		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
382		switch (ops.flags) {
383		0b00 => {
384		// RETF imm16
385		// I
386		const imm = emit.mir.instructions.items(.data)[inst].imm;
387		return lowerToIEnc(.ret_far, imm, emit.code) catch \|err\| emit.failWithLoweringError(err);
388		},
389		0b01 => {
390		return lowerToZoEnc(.ret_far, emit.code) catch \|err\| emit.failWithLoweringError(err);
391		},
392		0b10 => {
393		// RET imm16
394		// I
395		const imm = emit.mir.instructions.items(.data)[inst].imm;
396		return lowerToIEnc(.ret_near, imm, emit.code) catch \|err\| emit.failWithLoweringError(err);
397		},
398		0b11 => {
399		return lowerToZoEnc(.ret_near, emit.code) catch \|err\| emit.failWithLoweringError(err);
400		},
401		}
402		}
403
404		fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
405		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
406		switch (ops.flags) {
407		0b00 => {
408		if (ops.reg2 == .none) {
409		// mov reg1, imm32
410		// MI
411		const imm = emit.mir.instructions.items(.data)[inst].imm;
412		return lowerToMiEnc(tag, RegisterOrMemory.reg(ops.reg1), imm, emit.code) catch \|err\|
413		emit.failWithLoweringError(err);
414		}
415		// mov reg1, reg2
416		// RM
417		return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code) catch \|err\|
418		emit.failWithLoweringError(err);
419		},
420		0b01 => {
421		// mov reg1, [reg2 + imm32]
422		// RM
423		const imm = emit.mir.instructions.items(.data)[inst].imm;
424		const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
425		return lowerToRmEnc(
426		tag,
427		ops.reg1,
428		RegisterOrMemory.mem(src_reg, imm, Memory.PtrSize.fromBits(ops.reg1.size())),
429		emit.code,
430		) catch \|err\| emit.failWithLoweringError(err);
431		},
432		0b10 => {
433		if (ops.reg2 == .none) {
434		return emit.fail("TODO unused variant: mov reg1, none, 0b10", .{});
435		}
436		// mov [reg1 + imm32], reg2
437		// MR
438		const imm = emit.mir.instructions.items(.data)[inst].imm;
439		return lowerToMrEnc(
440		tag,
441		RegisterOrMemory.mem(ops.reg1, imm, Memory.PtrSize.fromBits(ops.reg2.size())),
442		ops.reg2,
443		emit.code,
444		) catch \|err\| emit.failWithLoweringError(err);
445		},
446		0b11 => {
447		return emit.fail("TODO unused variant: mov reg1, reg2, 0b11", .{});
448		},
449		}
450		}
451
452		fn mirArithMemImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
453		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
454		assert(ops.reg2 == .none);
455		const payload = emit.mir.instructions.items(.data)[inst].payload;
456		const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
457		const ptr_size: Memory.PtrSize = switch (ops.flags) {
458		0b00 => .byte_ptr,
459		0b01 => .word_ptr,
460		0b10 => .dword_ptr,
461		0b11 => .qword_ptr,
462		};
463		return lowerToMiEnc(
464		tag,
465		RegisterOrMemory.mem(ops.reg1, imm_pair.dest_off, ptr_size),
466		imm_pair.operand,
467		emit.code,
468		) catch \|err\| emit.failWithLoweringError(err);
469		}
470
471		inline fn setRexWRegister(reg: Register) bool {
472		if (reg.size() == 64) return true;
473		return switch (reg) {
474		.ah, .bh, .ch, .dh => true,
475		else => false,
476		};
477		}
478
479		inline fn immOpSize(imm: i64) u8 {
480		blk: {
481		_ = math.cast(i8, imm) catch break :blk;
482		return 8;
483		}
484		blk: {
485		_ = math.cast(i16, imm) catch break :blk;
486		return 16;
487		}
488		blk: {
489		_ = math.cast(i32, imm) catch break :blk;
490		return 32;
491		}
492		return 64;
493		}
494
495		// TODO
496		fn mirArithScaleSrc(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
497		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
498		const scale = ops.flags;
499		// OP reg1, [reg2 + scale*rcx + imm32]
500		const opc = getOpCode(tag, .rm, ops.reg1.size() == 8).?;
501		const imm = emit.mir.instructions.items(.data)[inst].imm;
502		const encoder = try Encoder.init(emit.code, 8);
503		encoder.rex(.{
504		.w = ops.reg1.size() == 64,
505		.r = ops.reg1.isExtended(),
506		.b = ops.reg2.isExtended(),
507		});
508		opc.encode(encoder);
509		if (imm <= math.maxInt(i8)) {
510		encoder.modRm_SIBDisp8(ops.reg1.lowId());
511		encoder.sib_scaleIndexBaseDisp8(scale, Register.rcx.lowId(), ops.reg2.lowId());
512		encoder.disp8(@intCast(i8, imm));
513		} else {
514		encoder.modRm_SIBDisp32(ops.reg1.lowId());
515		encoder.sib_scaleIndexBaseDisp32(scale, Register.rcx.lowId(), ops.reg2.lowId());
516		encoder.disp32(imm);
517		}
518		}
519
520		// TODO
521		fn mirArithScaleDst(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
522		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
523		const scale = ops.flags;
524		const imm = emit.mir.instructions.items(.data)[inst].imm;
525
526		if (ops.reg2 == .none) {
527		// OP [reg1 + scale*rax + 0], imm32
528		const opc = getOpCode(tag, .mi, ops.reg1.size() == 8).?;
529		const modrm_ext = getModRmExt(tag).?;
530		const encoder = try Encoder.init(emit.code, 8);
531		encoder.rex(.{
532		.w = ops.reg1.size() == 64,
533		.b = ops.reg1.isExtended(),
534		});
535		opc.encode(encoder);
536		encoder.modRm_SIBDisp0(modrm_ext);
537		encoder.sib_scaleIndexBase(scale, Register.rax.lowId(), ops.reg1.lowId());
538		if (imm <= math.maxInt(i8)) {
539		encoder.imm8(@intCast(i8, imm));
540		} else if (imm <= math.maxInt(i16)) {
541		encoder.imm16(@intCast(i16, imm));
542		} else {
543		encoder.imm32(imm);
544		}
545		return;
546		}
547
548		// OP [reg1 + scale*rax + imm32], reg2
549		const opc = getOpCode(tag, .mr, ops.reg1.size() == 8).?;
550		const encoder = try Encoder.init(emit.code, 8);
551		encoder.rex(.{
552		.w = ops.reg1.size() == 64,
553		.r = ops.reg2.isExtended(),
554		.b = ops.reg1.isExtended(),
555		});
556		opc.encode(encoder);
557		if (imm <= math.maxInt(i8)) {
558		encoder.modRm_SIBDisp8(ops.reg2.lowId());
559		encoder.sib_scaleIndexBaseDisp8(scale, Register.rax.lowId(), ops.reg1.lowId());
560		encoder.disp8(@intCast(i8, imm));
561		} else {
562		encoder.modRm_SIBDisp32(ops.reg2.lowId());
563		encoder.sib_scaleIndexBaseDisp32(scale, Register.rax.lowId(), ops.reg1.lowId());
564		encoder.disp32(imm);
565		}
566		}
567
568		// TODO
569		fn mirArithScaleImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
570		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
571		const scale = ops.flags;
572		const payload = emit.mir.instructions.items(.data)[inst].payload;
573		const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
574		const opc = getOpCode(tag, .mi, ops.reg1.size() == 8).?;
575		const modrm_ext = getModRmExt(tag).?;
576		const encoder = try Encoder.init(emit.code, 2);
577		encoder.rex(.{
578		.w = ops.reg1.size() == 64,
579		.b = ops.reg1.isExtended(),
580		});
581		opc.encode(encoder);
582		if (imm_pair.dest_off <= math.maxInt(i8)) {
583		encoder.modRm_SIBDisp8(modrm_ext);
584		encoder.sib_scaleIndexBaseDisp8(scale, Register.rax.lowId(), ops.reg1.lowId());
585		encoder.disp8(@intCast(i8, imm_pair.dest_off));
586		} else {
587		encoder.modRm_SIBDisp32(modrm_ext);
588		encoder.sib_scaleIndexBaseDisp32(scale, Register.rax.lowId(), ops.reg1.lowId());
589		encoder.disp32(imm_pair.dest_off);
590		}
591		encoder.imm32(imm_pair.operand);
592		}
593
594		fn mirMovabs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
595		const tag = emit.mir.instructions.items(.tag)[inst];
596		assert(tag == .movabs);
597		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
598		const imm: i64 = if (ops.reg1.size() == 64) blk: {
599		const payload = emit.mir.instructions.items(.data)[inst].payload;
600		const imm = emit.mir.extraData(Mir.Imm64, payload).data;
601		break :blk @bitCast(i64, imm.decode());
602		} else emit.mir.instructions.items(.data)[inst].imm;
603		if (ops.flags == 0b00) {
604		// movabs reg, imm64
605		// OI
606		return lowerToOiEnc(.mov, ops.reg1, imm, emit.code) catch \|err\| emit.failWithLoweringError(err);
607		}
608		if (ops.reg1 == .none) {
609		// movabs moffs64, rax
610		// TD
611		return lowerToTdEnc(.mov, imm, ops.reg2, emit.code) catch \|err\| emit.failWithLoweringError(err);
612		}
613		// movabs rax, moffs64
614		// FD
615		return lowerToFdEnc(.mov, ops.reg1, imm, emit.code) catch \|err\| emit.failWithLoweringError(err);
616		}
617
618		fn mirIMulComplex(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
619		const tag = emit.mir.instructions.items(.tag)[inst];
620		assert(tag == .imul_complex);
621		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
622		switch (ops.flags) {
623		0b00 => {
624		return lowerToRmEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code) catch \|err\|
625		emit.failWithLoweringError(err);
626		},
627		0b10 => {
628		const imm = emit.mir.instructions.items(.data)[inst].imm;
629		return lowerToRmiEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), imm, emit.code) catch \|err\|
630		emit.failWithLoweringError(err);
631		},
632		else => return emit.fail("TODO implement imul", .{}),
633		}
634		}
635
636		fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
637		const tag = emit.mir.instructions.items(.tag)[inst];
638		assert(tag == .lea);
639		const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
640		switch (ops.flags) {
641		0b00 => {
642		// lea reg1, [reg2 + imm32]
643		// RM
644		const imm = emit.mir.instructions.items(.data)[inst].imm;
645		const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
646		return lowerToRmEnc(
647		.lea,
648		ops.reg1,
649		RegisterOrMemory.mem(src_reg, imm, Memory.PtrSize.fromBits(ops.reg1.size())),
650		emit.code,
651		) catch \|err\| emit.failWithLoweringError(err);
652		},
653		0b01 => {
654		// lea reg1, [rip + imm32]
655		// RM
656		const start_offset = emit.code.items.len;
657		lowerToRmEnc(
658		.lea,
659		ops.reg1,
660		RegisterOrMemory.rip(0, Memory.PtrSize.fromBits(ops.reg1.size())),
661		emit.code,
662		) catch \|err\| return emit.failWithLoweringError(err);
663		const end_offset = emit.code.items.len;
664		// Backpatch the displacement
665		const payload = emit.mir.instructions.items(.data)[inst].payload;
666		const imm = emit.mir.extraData(Mir.Imm64, payload).data.decode();
667		const disp = @intCast(i32, @intCast(i64, imm) - @intCast(i64, end_offset - start_offset));
668		mem.writeIntLittle(i32, emit.code.items[end_offset - 4 ..][0..4], disp);
669		},
670		0b10 => {
671		// lea reg1, [rip + reloc]
672		// RM
673		lowerToRmEnc(
674		.lea,
675		ops.reg1,
676		RegisterOrMemory.rip(0, Memory.PtrSize.fromBits(ops.reg1.size())),
677		emit.code,
678		) catch \|err\| return emit.failWithLoweringError(err);
679		const end_offset = emit.code.items.len;
680		const got_entry = emit.mir.instructions.items(.data)[inst].got_entry;
681		if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
682		// TODO I think the reloc might be in the wrong place.
683		const decl = macho_file.active_decl.?;
684		try decl.link.macho.relocs.append(emit.bin_file.allocator, .{
685		.offset = @intCast(u32, end_offset - 4),
686		.target = .{ .local = got_entry },
687		.addend = 0,
688		.subtractor = null,
689		.pcrel = true,
690		.length = 2,
691		.@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_GOT),
692		});
693		} else {
694		return emit.fail(
695		"TODO implement lea reg, [rip + reloc] for linking backends different than MachO",
696		.{},
697		);
698		}
699		},
700		0b11 => return emit.fail("TODO unused variant lea reg1, reg2, 0b11", .{}),
701		}
702		}
703
704		fn mirCallExtern(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
705		const tag = emit.mir.instructions.items(.tag)[inst];
706		assert(tag == .call_extern);
707		const n_strx = emit.mir.instructions.items(.data)[inst].extern_fn;
708		const offset = blk: {
709		// callq
710		lowerToDEnc(.call_near, 0, emit.code) catch \|err\|
711		return emit.failWithLoweringError(err);
712		break :blk @intCast(u32, emit.code.items.len) - 4;
713		};
714		if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
715		// Add relocation to the decl.
716		try macho_file.active_decl.?.link.macho.relocs.append(emit.bin_file.allocator, .{
717		.offset = offset,
718		.target = .{ .global = n_strx },
719		.addend = 0,
720		.subtractor = null,
721		.pcrel = true,
722		.length = 2,
723		.@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_BRANCH),
724		});
725		} else {
726		return emit.fail("TODO implement call_extern for linking backends different than MachO", .{});
727		}
728		}
729
730		fn mirDbgLine(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
731		const tag = emit.mir.instructions.items(.tag)[inst];
732		assert(tag == .dbg_line);
733		const payload = emit.mir.instructions.items(.data)[inst].payload;
734		const dbg_line_column = emit.mir.extraData(Mir.DbgLineColumn, payload).data;
735		try emit.dbgAdvancePCAndLine(dbg_line_column.line, dbg_line_column.column);
736		}
737
738		fn dbgAdvancePCAndLine(emit: *Emit, line: u32, column: u32) InnerError!void {
739		const delta_line = @intCast(i32, line) - @intCast(i32, emit.prev_di_line);
740		const delta_pc: usize = emit.code.items.len - emit.prev_di_pc;
741		switch (emit.debug_output) {
742		.dwarf => \|dbg_out\| {
743		// TODO Look into using the DWARF special opcodes to compress this data.
744		// It lets you emit single-byte opcodes that add different numbers to
745		// both the PC and the line number at the same time.
746		try dbg_out.dbg_line.ensureUnusedCapacity(11);
747		dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_pc);
748		leb128.writeULEB128(dbg_out.dbg_line.writer(), delta_pc) catch unreachable;
749		if (delta_line != 0) {
750		dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_line);
751		leb128.writeILEB128(dbg_out.dbg_line.writer(), delta_line) catch unreachable;
752		}
753		dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.copy);
754		emit.prev_di_pc = emit.code.items.len;
755		emit.prev_di_line = line;
756		emit.prev_di_column = column;
757		emit.prev_di_pc = emit.code.items.len;
758		},
759		.plan9 => \|dbg_out\| {
760		if (delta_pc <= 0) return; // only do this when the pc changes
761		// we have already checked the target in the linker to make sure it is compatable
762		const quant = @import("../../link/Plan9/aout.zig").getPCQuant(emit.target.cpu.arch) catch unreachable;
763
764		// increasing the line number
765		try @import("../../link/Plan9.zig").changeLine(dbg_out.dbg_line, delta_line);
766		// increasing the pc
767		const d_pc_p9 = @intCast(i64, delta_pc) - quant;
768		if (d_pc_p9 > 0) {
769		// minus one because if its the last one, we want to leave space to change the line which is one quanta
770		try dbg_out.dbg_line.append(@intCast(u8, @divExact(d_pc_p9, quant) + 128) - quant);
771		if (dbg_out.pcop_change_index.*) \|pci\|
772		dbg_out.dbg_line.items[pci] += 1;
773		dbg_out.pcop_change_index.* = @intCast(u32, dbg_out.dbg_line.items.len - 1);
774		} else if (d_pc_p9 == 0) {
775		// we don't need to do anything, because adding the quant does it for us
776		} else unreachable;
777		if (dbg_out.start_line.* == null)
778		dbg_out.start_line.* = emit.prev_di_line;
779		dbg_out.end_line.* = line;
780		// only do this if the pc changed
781		emit.prev_di_line = line;
782		emit.prev_di_column = column;
783		emit.prev_di_pc = emit.code.items.len;
784		},
785		.none => {},
786		}
787		}
788
789		fn mirDbgPrologueEnd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
790		const tag = emit.mir.instructions.items(.tag)[inst];
791		assert(tag == .dbg_prologue_end);
792		switch (emit.debug_output) {
793		.dwarf => \|dbg_out\| {
794		try dbg_out.dbg_line.append(DW.LNS.set_prologue_end);
795		try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
796		},
797		.plan9 => {},
798		.none => {},
799		}
800		}
801
802		fn mirDbgEpilogueBegin(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
803		const tag = emit.mir.instructions.items(.tag)[inst];
804		assert(tag == .dbg_epilogue_begin);
805		switch (emit.debug_output) {
806		.dwarf => \|dbg_out\| {
807		try dbg_out.dbg_line.append(DW.LNS.set_epilogue_begin);
808		try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
809		},
810		.plan9 => {},
811		.none => {},
812		}
813		}
814
815		fn mirArgDbgInfo(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
816		const tag = emit.mir.instructions.items(.tag)[inst];
817		assert(tag == .arg_dbg_info);
818		const payload = emit.mir.instructions.items(.data)[inst].payload;
819		const arg_dbg_info = emit.mir.extraData(Mir.ArgDbgInfo, payload).data;
820		const mcv = emit.mir.function.args[arg_dbg_info.arg_index];
821		try emit.genArgDbgInfo(arg_dbg_info.air_inst, mcv);
822		}
823
824		fn genArgDbgInfo(emit: *Emit, inst: Air.Inst.Index, mcv: MCValue) !void {
825		const ty_str = emit.mir.function.air.instructions.items(.data)[inst].ty_str;
826		const zir = &emit.mir.function.mod_fn.owner_decl.getFileScope().zir;
827		const name = zir.nullTerminatedString(ty_str.str);
828		const name_with_null = name.ptr[0 .. name.len + 1];
829		const ty = emit.mir.function.air.getRefType(ty_str.ty);
830
831		switch (mcv) {
832		.register => \|reg\| {
833		switch (emit.debug_output) {
834		.dwarf => \|dbg_out\| {
835		try dbg_out.dbg_info.ensureUnusedCapacity(3);
836		dbg_out.dbg_info.appendAssumeCapacity(link.File.Elf.abbrev_parameter);
837		dbg_out.dbg_info.appendSliceAssumeCapacity(&[2]u8{ // DW.AT.location, DW.FORM.exprloc
838		1, // ULEB128 dwarf expression length
839		reg.dwarfLocOp(),
840		});
841		try dbg_out.dbg_info.ensureUnusedCapacity(5 + name_with_null.len);
842		try emit.addDbgInfoTypeReloc(ty); // DW.AT.type, DW.FORM.ref4
843		dbg_out.dbg_info.appendSliceAssumeCapacity(name_with_null); // DW.AT.name, DW.FORM.string
844		},
845		.plan9 => {},
846		.none => {},
847		}
848		},
849		.stack_offset => {
850		switch (emit.debug_output) {
851		.dwarf => {},
852		.plan9 => {},
853		.none => {},
854		}
855		},
856		else => {},
857		}
858		}
859
860		/// Adds a Type to the .debug_info at the current position. The bytes will be populated later,
861		/// after codegen for this symbol is done.
862		fn addDbgInfoTypeReloc(emit: *Emit, ty: Type) !void {
863		switch (emit.debug_output) {
864		.dwarf => \|dbg_out\| {
865		assert(ty.hasCodeGenBits());
866		const index = dbg_out.dbg_info.items.len;
867		try dbg_out.dbg_info.resize(index + 4); // DW.AT.type, DW.FORM.ref4
868
869		const gop = try dbg_out.dbg_info_type_relocs.getOrPut(emit.bin_file.allocator, ty);
870		if (!gop.found_existing) {
871		gop.value_ptr.* = .{
872		.off = undefined,
873		.relocs = .{},
874		};
875		}
876		try gop.value_ptr.relocs.append(emit.bin_file.allocator, @intCast(u32, index));
877		},
878		.plan9 => {},
879		.none => {},
880		}
881		}
882
883		const Tag = enum {
884		adc,
885		add,
886		sub,
887		xor,
888		@"and",
889		@"or",
890		sbb,
891		cmp,
892		mov,
893		lea,
894		jmp_near,
895		call_near,
896		push,
897		pop,
898		@"test",
899		brk,
900		nop,
901		imul,
902		syscall,
903		ret_near,
904		ret_far,
905		jo,
906		jno,
907		jb,
908		jbe,
909		jc,
910		jnae,
911		jnc,
912		jae,
913		je,
914		jz,
915		jne,
916		jnz,
917		jna,
918		jnb,
919		jnbe,
920		ja,
921		js,
922		jns,
923		jpe,
924		jp,
925		jpo,
926		jnp,
927		jnge,
928		jl,
929		jge,
930		jnl,
931		jle,
932		jng,
933		jg,
934		jnle,
935		seto,
936		setno,
937		setb,
938		setc,
939		setnae,
940		setnb,
941		setnc,
942		setae,
943		sete,
944		setz,
945		setne,
946		setnz,
947		setbe,
948		setna,
949		seta,
950		setnbe,
951		sets,
952		setns,
953		setp,
954		setpe,
955		setnp,
956		setop,
957		setl,
958		setnge,
959		setnl,
960		setge,
961		setle,
962		setng,
963		setnle,
964		setg,
965
966		fn isSetCC(tag: Tag) bool {
967		return switch (tag) {
968		.seto,
969		.setno,
970		.setb,
971		.setc,
972		.setnae,
973		.setnb,
974		.setnc,
975		.setae,
976		.sete,
977		.setz,
978		.setne,
979		.setnz,
980		.setbe,
981		.setna,
982		.seta,
983		.setnbe,
984		.sets,
985		.setns,
986		.setp,
987		.setpe,
988		.setnp,
989		.setop,
990		.setl,
991		.setnge,
992		.setnl,
993		.setge,
994		.setle,
995		.setng,
996		.setnle,
997		.setg,
998		=> true,
999		else => false,
1000		};
1001		}
1002		};
1003
1004		const Encoding = enum {
1005		/// OP
1006		zo,
1007
1008		/// OP rel32
1009		d,
1010
1011		/// OP r/m64
1012		m,
1013
1014		/// OP r64
1015		o,
1016
1017		/// OP imm32
1018		i,
1019
1020		/// OP r/m64, imm32
1021		mi,
1022
1023		/// OP r/m64, r64
1024		mr,
1025
1026		/// OP r64, r/m64
1027		rm,
1028
1029		/// OP r64, imm64
1030		oi,
1031
1032		/// OP al/ax/eax/rax, moffs
1033		fd,
1034
1035		/// OP moffs, al/ax/eax/rax
1036		td,
1037
1038		/// OP r64, r/m64, imm32
1039		rmi,
1040		};
1041
1042		const OpCode = union(enum) {
1043		one_byte: u8,
1044		two_byte: struct { _1: u8, _2: u8 },
1045
1046		fn oneByte(opc: u8) OpCode {
1047		return .{ .one_byte = opc };
1048		}
1049
1050		fn twoByte(opc1: u8, opc2: u8) OpCode {
1051		return .{ .two_byte = .{ ._1 = opc1, ._2 = opc2 } };
1052		}
1053
1054		fn encode(opc: OpCode, encoder: Encoder) void {
1055		switch (opc) {
1056		.one_byte => \|v\| encoder.opcode_1byte(v),
1057		.two_byte => \|v\| encoder.opcode_2byte(v._1, v._2),
1058		}
1059		}
1060
1061		fn encodeWithReg(opc: OpCode, encoder: Encoder, reg: Register) void {
1062		assert(opc == .one_byte);
1063		encoder.opcode_withReg(opc.one_byte, reg.lowId());
1064		}
1065		};
1066
1067		inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode {
1068		switch (enc) {
1069		.zo => return switch (tag) {
1070		.ret_near => OpCode.oneByte(0xc3),
1071		.ret_far => OpCode.oneByte(0xcb),
1072		.brk => OpCode.oneByte(0xcc),
1073		.nop => OpCode.oneByte(0x90),
1074		.syscall => OpCode.twoByte(0x0f, 0x05),
1075		else => null,
1076		},
1077		.d => return switch (tag) {
1078		.jmp_near => OpCode.oneByte(0xe9),
1079		.call_near => OpCode.oneByte(0xe8),
1080		.jo => if (is_one_byte) OpCode.oneByte(0x70) else OpCode.twoByte(0x0f, 0x80),
1081		.jno => if (is_one_byte) OpCode.oneByte(0x71) else OpCode.twoByte(0x0f, 0x81),
1082		.jb, .jc, .jnae => if (is_one_byte) OpCode.oneByte(0x72) else OpCode.twoByte(0x0f, 0x82),
1083		.jnb, .jnc, .jae => if (is_one_byte) OpCode.oneByte(0x73) else OpCode.twoByte(0x0f, 0x83),
1084		.je, .jz => if (is_one_byte) OpCode.oneByte(0x74) else OpCode.twoByte(0x0f, 0x84),
1085		.jne, .jnz => if (is_one_byte) OpCode.oneByte(0x75) else OpCode.twoByte(0x0f, 0x85),
1086		.jna, .jbe => if (is_one_byte) OpCode.oneByte(0x76) else OpCode.twoByte(0x0f, 0x86),
1087		.jnbe, .ja => if (is_one_byte) OpCode.oneByte(0x77) else OpCode.twoByte(0x0f, 0x87),
1088		.js => if (is_one_byte) OpCode.oneByte(0x78) else OpCode.twoByte(0x0f, 0x88),
1089		.jns => if (is_one_byte) OpCode.oneByte(0x79) else OpCode.twoByte(0x0f, 0x89),
1090		.jpe, .jp => if (is_one_byte) OpCode.oneByte(0x7a) else OpCode.twoByte(0x0f, 0x8a),
1091		.jpo, .jnp => if (is_one_byte) OpCode.oneByte(0x7b) else OpCode.twoByte(0x0f, 0x8b),
1092		.jnge, .jl => if (is_one_byte) OpCode.oneByte(0x7c) else OpCode.twoByte(0x0f, 0x8c),
1093		.jge, .jnl => if (is_one_byte) OpCode.oneByte(0x7d) else OpCode.twoByte(0x0f, 0x8d),
1094		.jle, .jng => if (is_one_byte) OpCode.oneByte(0x7e) else OpCode.twoByte(0x0f, 0x8e),
1095		.jg, .jnle => if (is_one_byte) OpCode.oneByte(0x7f) else OpCode.twoByte(0x0f, 0x8f),
1096		else => null,
1097		},
1098		.m => return switch (tag) {
1099		.jmp_near, .call_near, .push => OpCode.oneByte(0xff),
1100		.pop => OpCode.oneByte(0x8f),
1101		.seto => OpCode.twoByte(0x0f, 0x90),
1102		.setno => OpCode.twoByte(0x0f, 0x91),
1103		.setb, .setc, .setnae => OpCode.twoByte(0x0f, 0x92),
1104		.setnb, .setnc, .setae => OpCode.twoByte(0x0f, 0x93),
1105		.sete, .setz => OpCode.twoByte(0x0f, 0x94),
1106		.setne, .setnz => OpCode.twoByte(0x0f, 0x95),
1107		.setbe, .setna => OpCode.twoByte(0x0f, 0x96),
1108		.seta, .setnbe => OpCode.twoByte(0x0f, 0x97),
1109		.sets => OpCode.twoByte(0x0f, 0x98),
1110		.setns => OpCode.twoByte(0x0f, 0x99),
1111		.setp, .setpe => OpCode.twoByte(0x0f, 0x9a),
1112		.setnp, .setop => OpCode.twoByte(0x0f, 0x9b),
1113		.setl, .setnge => OpCode.twoByte(0x0f, 0x9c),
1114		.setnl, .setge => OpCode.twoByte(0x0f, 0x9d),
1115		.setle, .setng => OpCode.twoByte(0x0f, 0x9e),
1116		.setnle, .setg => OpCode.twoByte(0x0f, 0x9f),
1117		else => null,
1118		},
1119		.o => return switch (tag) {
1120		.push => OpCode.oneByte(0x50),
1121		.pop => OpCode.oneByte(0x58),
1122		else => null,
1123		},
1124		.i => return switch (tag) {
1125		.push => OpCode.oneByte(if (is_one_byte) 0x6a else 0x68),
1126		.@"test" => OpCode.oneByte(if (is_one_byte) 0xa8 else 0xa9),
1127		.ret_near => OpCode.oneByte(0xc2),
1128		.ret_far => OpCode.oneByte(0xca),
1129		else => null,
1130		},
1131		.mi => return switch (tag) {
1132		.adc, .add, .sub, .xor, .@"and", .@"or", .sbb, .cmp => OpCode.oneByte(if (is_one_byte) 0x80 else 0x81),
1133		.mov => OpCode.oneByte(if (is_one_byte) 0xc6 else 0xc7),
1134		.@"test" => OpCode.oneByte(if (is_one_byte) 0xf6 else 0xf7),
1135		else => null,
1136		},
1137		.mr => return switch (tag) {
1138		.adc => OpCode.oneByte(if (is_one_byte) 0x10 else 0x11),
1139		.add => OpCode.oneByte(if (is_one_byte) 0x00 else 0x01),
1140		.sub => OpCode.oneByte(if (is_one_byte) 0x28 else 0x29),
1141		.xor => OpCode.oneByte(if (is_one_byte) 0x30 else 0x31),
1142		.@"and" => OpCode.oneByte(if (is_one_byte) 0x20 else 0x21),
1143		.@"or" => OpCode.oneByte(if (is_one_byte) 0x08 else 0x09),
1144		.sbb => OpCode.oneByte(if (is_one_byte) 0x18 else 0x19),
1145		.cmp => OpCode.oneByte(if (is_one_byte) 0x38 else 0x39),
1146		.mov => OpCode.oneByte(if (is_one_byte) 0x88 else 0x89),
1147		else => null,
1148		},
1149		.rm => return switch (tag) {
1150		.adc => OpCode.oneByte(if (is_one_byte) 0x12 else 0x13),
1151		.add => OpCode.oneByte(if (is_one_byte) 0x02 else 0x03),
1152		.sub => OpCode.oneByte(if (is_one_byte) 0x2a else 0x2b),
1153		.xor => OpCode.oneByte(if (is_one_byte) 0x32 else 0x33),
1154		.@"and" => OpCode.oneByte(if (is_one_byte) 0x22 else 0x23),
1155		.@"or" => OpCode.oneByte(if (is_one_byte) 0x0b else 0x0b),
1156		.sbb => OpCode.oneByte(if (is_one_byte) 0x1a else 0x1b),
1157		.cmp => OpCode.oneByte(if (is_one_byte) 0x3a else 0x3b),
1158		.mov => OpCode.oneByte(if (is_one_byte) 0x8a else 0x8b),
1159		.lea => OpCode.oneByte(if (is_one_byte) 0x8c else 0x8d),
1160		.imul => OpCode.twoByte(0x0f, 0xaf),
1161		else => null,
1162		},
1163		.oi => return switch (tag) {
1164		.mov => OpCode.oneByte(if (is_one_byte) 0xb0 else 0xb8),
1165		else => null,
1166		},
1167		.fd => return switch (tag) {
1168		.mov => OpCode.oneByte(if (is_one_byte) 0xa0 else 0xa1),
1169		else => null,
1170		},
1171		.td => return switch (tag) {
1172		.mov => OpCode.oneByte(if (is_one_byte) 0xa2 else 0xa3),
1173		else => null,
1174		},
1175		.rmi => return switch (tag) {
1176		.imul => OpCode.oneByte(if (is_one_byte) 0x6b else 0x69),
1177		else => null,
1178		},
1179		}
1180		}
1181
1182		inline fn getModRmExt(tag: Tag) ?u3 {
1183		return switch (tag) {
1184		.adc => 0x2,
1185		.add => 0x0,
1186		.sub => 0x5,
1187		.xor => 0x6,
1188		.@"and" => 0x4,
1189		.@"or" => 0x1,
1190		.sbb => 0x3,
1191		.cmp => 0x7,
1192		.mov => 0x0,
1193		.jmp_near => 0x4,
1194		.call_near => 0x2,
1195		.push => 0x6,
1196		.pop => 0x0,
1197		.@"test" => 0x0,
1198		.seto,
1199		.setno,
1200		.setb,
1201		.setc,
1202		.setnae,
1203		.setnb,
1204		.setnc,
1205		.setae,
1206		.sete,
1207		.setz,
1208		.setne,
1209		.setnz,
1210		.setbe,
1211		.setna,
1212		.seta,
1213		.setnbe,
1214		.sets,
1215		.setns,
1216		.setp,
1217		.setpe,
1218		.setnp,
1219		.setop,
1220		.setl,
1221		.setnge,
1222		.setnl,
1223		.setge,
1224		.setle,
1225		.setng,
1226		.setnle,
1227		.setg,
1228		=> 0x0,
1229		else => null,
1230		};
1231		}
1232
1233		const ScaleIndexBase = struct {
1234		scale: u2,
1235		index_reg: ?Register,
1236		base_reg: ?Register,
1237		};
1238
1239		const Memory = struct {
1240		reg: ?Register,
1241		rip: bool = false,
1242		disp: i32,
1243		ptr_size: PtrSize,
1244		sib: ?ScaleIndexBase = null,
1245
1246		const PtrSize = enum {
1247		byte_ptr,
1248		word_ptr,
1249		dword_ptr,
1250		qword_ptr,
1251
1252		fn fromBits(in_bits: u64) PtrSize {
1253		return switch (in_bits) {
1254		8 => .byte_ptr,
1255		16 => .word_ptr,
1256		32 => .dword_ptr,
1257		64 => .qword_ptr,
1258		else => unreachable,
1259		};
1260		}
1261
1262		/// Returns size in bits.
1263		fn size(ptr_size: PtrSize) u64 {
1264		return switch (ptr_size) {
1265		.byte_ptr => 8,
1266		.word_ptr => 16,
1267		.dword_ptr => 32,
1268		.qword_ptr => 64,
1269		};
1270		}
1271		};
1272		};
1273
1274		const RegisterOrMemory = union(enum) {
1275		register: Register,
1276		memory: Memory,
1277
1278		fn reg(register: Register) RegisterOrMemory {
1279		return .{ .register = register };
1280		}
1281
1282		fn mem(register: ?Register, disp: i32, ptr_size: Memory.PtrSize) RegisterOrMemory {
1283		return .{
1284		.memory = .{
1285		.reg = register,
1286		.disp = disp,
1287		.ptr_size = ptr_size,
1288		},
1289		};
1290		}
1291
1292		fn rip(disp: i32, ptr_size: Memory.PtrSize) RegisterOrMemory {
1293		return .{
1294		.memory = .{
1295		.reg = null,
1296		.rip = true,
1297		.disp = disp,
1298		.ptr_size = ptr_size,
1299		},
1300		};
1301		}
1302		};
1303
1304		const LoweringError = error{
1305		OutOfMemory,
1306		Overflow,
1307		OperandSizeMismatch,
1308		RaxOperandExpected,
1309		};
1310
1311		fn lowerToZoEnc(tag: Tag, code: *std.ArrayList(u8)) LoweringError!void {
1312		const opc = getOpCode(tag, .zo, false).?;
1313		const encoder = try Encoder.init(code, 1);
1314		opc.encode(encoder);
1315		}
1316
1317		fn lowerToIEnc(tag: Tag, imm: i32, code: *std.ArrayList(u8)) LoweringError!void {
1318		if (tag == .ret_far or tag == .ret_near) {
1319		const encoder = try Encoder.init(code, 3);
1320		const opc = getOpCode(tag, .i, false).?;
1321		opc.encode(encoder);
1322		encoder.imm16(@intCast(i16, imm));
1323		return;
1324		}
1325		const opc = getOpCode(tag, .i, immOpSize(imm) == 8).?;
1326		const encoder = try Encoder.init(code, 5);
1327		if (immOpSize(imm) == 16) {
1328		encoder.opcode_1byte(0x66);
1329		}
1330		opc.encode(encoder);
1331		if (immOpSize(imm) == 8) {
1332		encoder.imm8(@intCast(i8, imm));
1333		} else if (immOpSize(imm) == 16) {
1334		encoder.imm16(@intCast(i16, imm));
1335		} else {
1336		encoder.imm32(imm);
1337		}
1338		}
1339
1340		fn lowerToOEnc(tag: Tag, reg: Register, code: *std.ArrayList(u8)) LoweringError!void {
1341		if (reg.size() != 16 and reg.size() != 64) {
1342		return error.OperandSizeMismatch; // TODO correct for push/pop, but is it universal?
1343		}
1344		const opc = getOpCode(tag, .o, false).?;
1345		const encoder = try Encoder.init(code, 3);
1346		if (reg.size() == 16) {
1347		encoder.opcode_1byte(0x66);
1348		}
1349		encoder.rex(.{
1350		.w = false,
1351		.b = reg.isExtended(),
1352		});
1353		opc.encodeWithReg(encoder, reg);
1354		}
1355
1356		fn lowerToDEnc(tag: Tag, imm: i32, code: *std.ArrayList(u8)) LoweringError!void {
1357		const opc = getOpCode(tag, .d, false).?;
1358		const encoder = try Encoder.init(code, 6);
1359		opc.encode(encoder);
1360		encoder.imm32(imm);
1361		}
1362
1363		fn lowerToMEnc(tag: Tag, reg_or_mem: RegisterOrMemory, code: *std.ArrayList(u8)) LoweringError!void {
1364		const opc = getOpCode(tag, .m, false).?;
1365		const modrm_ext = getModRmExt(tag).?;
1366		switch (reg_or_mem) {
1367		.register => \|reg\| {
1368		const op_size_mismatch = blk: {
1369		if (tag.isSetCC() and reg.size() == 8)
1370		break :blk false;
1371		break :blk reg.size() != 64 and reg.size() != 16;
1372		};
1373		if (op_size_mismatch) {
1374		return error.OperandSizeMismatch;
1375		}
1376		const encoder = try Encoder.init(code, 4);
1377		if (reg.size() == 16) {
1378		encoder.opcode_1byte(0x66);
1379		}
1380		encoder.rex(.{
1381		.w = switch (reg) {
1382		.ah, .bh, .ch, .dh => true,
1383		else => false,
1384		},
1385		.b = reg.isExtended(),
1386		});
1387		opc.encode(encoder);
1388		encoder.modRm_direct(modrm_ext, reg.lowId());
1389		},
1390		.memory => \|mem_op\| {
1391		if (mem_op.ptr_size != .qword_ptr and mem_op.ptr_size != .word_ptr) {
1392		return error.OperandSizeMismatch;
1393		}
1394		const encoder = try Encoder.init(code, 8);
1395		if (mem_op.ptr_size == .word_ptr) {
1396		encoder.opcode_1byte(0x66);
1397		}
1398		if (mem_op.reg) \|reg\| {
1399		if (reg.size() != 64) {
1400		return error.OperandSizeMismatch;
1401		}
1402		encoder.rex(.{
1403		.w = false,
1404		.b = reg.isExtended(),
1405		});
1406		opc.encode(encoder);
1407		if (reg.lowId() == 4) {
1408		if (mem_op.disp == 0) {
1409		encoder.modRm_SIBDisp0(modrm_ext);
1410		encoder.sib_base(reg.lowId());
1411		} else if (immOpSize(mem_op.disp) == 8) {
1412		encoder.modRm_SIBDisp8(modrm_ext);
1413		encoder.sib_baseDisp8(reg.lowId());
1414		encoder.disp8(@intCast(i8, mem_op.disp));
1415		} else {
1416		encoder.modRm_SIBDisp32(modrm_ext);
1417		encoder.sib_baseDisp32(reg.lowId());
1418		encoder.disp32(mem_op.disp);
1419		}
1420		} else {
1421		if (mem_op.disp == 0) {
1422		encoder.modRm_indirectDisp0(modrm_ext, reg.lowId());
1423		} else if (immOpSize(mem_op.disp) == 8) {
1424		encoder.modRm_indirectDisp8(modrm_ext, reg.lowId());
1425		encoder.disp8(@intCast(i8, mem_op.disp));
1426		} else {
1427		encoder.modRm_indirectDisp32(modrm_ext, reg.lowId());
1428		encoder.disp32(mem_op.disp);
1429		}
1430		}
1431		} else {
1432		opc.encode(encoder);
1433		if (mem_op.rip) {
1434		encoder.modRm_RIPDisp32(modrm_ext);
1435		} else {
1436		encoder.modRm_SIBDisp0(modrm_ext);
1437		encoder.sib_disp32();
1438		}
1439		encoder.disp32(mem_op.disp);
1440		}
1441		},
1442		}
1443		}
1444
1445		fn lowerToTdEnc(tag: Tag, moffs: i64, reg: Register, code: *std.ArrayList(u8)) LoweringError!void {
1446		return lowerToTdFdEnc(tag, reg, moffs, code, true);
1447		}
1448
1449		fn lowerToFdEnc(tag: Tag, reg: Register, moffs: i64, code: *std.ArrayList(u8)) LoweringError!void {
1450		return lowerToTdFdEnc(tag, reg, moffs, code, false);
1451		}
1452
1453		fn lowerToTdFdEnc(tag: Tag, reg: Register, moffs: i64, code: *std.ArrayList(u8), td: bool) LoweringError!void {
1454		if (reg.lowId() != Register.rax.lowId()) {
1455		return error.RaxOperandExpected;
1456		}
1457		if (reg.size() != immOpSize(moffs)) {
1458		return error.OperandSizeMismatch;
1459		}
1460		const opc = if (td)
1461		getOpCode(tag, .td, reg.size() == 8).?
1462		else
1463		getOpCode(tag, .fd, reg.size() == 8).?;
1464		const encoder = try Encoder.init(code, 10);
1465		if (reg.size() == 16) {
1466		encoder.opcode_1byte(0x66);
1467		}
1468		encoder.rex(.{
1469		.w = setRexWRegister(reg),
1470		});
1471		opc.encode(encoder);
1472		switch (reg.size()) {
1473		8 => {
1474		const moffs8 = try math.cast(i8, moffs);
1475		encoder.imm8(moffs8);
1476		},
1477		16 => {
1478		const moffs16 = try math.cast(i16, moffs);
1479		encoder.imm16(moffs16);
1480		},
1481		32 => {
1482		const moffs32 = try math.cast(i32, moffs);
1483		encoder.imm32(moffs32);
1484		},
1485		64 => {
1486		encoder.imm64(@bitCast(u64, moffs));
1487		},
1488		else => unreachable,
1489		}
1490		}
1491
1492		fn lowerToOiEnc(tag: Tag, reg: Register, imm: i64, code: *std.ArrayList(u8)) LoweringError!void {
1493		if (reg.size() != immOpSize(imm)) {
1494		return error.OperandSizeMismatch;
1495		}
1496		const opc = getOpCode(tag, .oi, reg.size() == 8).?;
1497		const encoder = try Encoder.init(code, 10);
1498		if (reg.size() == 16) {
1499		encoder.opcode_1byte(0x66);
1500		}
1501		encoder.rex(.{
1502		.w = setRexWRegister(reg),
1503		.b = reg.isExtended(),
1504		});
1505		opc.encodeWithReg(encoder, reg);
1506		switch (reg.size()) {
1507		8 => {
1508		const imm8 = try math.cast(i8, imm);
1509		encoder.imm8(imm8);
1510		},
1511		16 => {
1512		const imm16 = try math.cast(i16, imm);
1513		encoder.imm16(imm16);
1514		},
1515		32 => {
1516		const imm32 = try math.cast(i32, imm);
1517		encoder.imm32(imm32);
1518		},
1519		64 => {
1520		encoder.imm64(@bitCast(u64, imm));
1521		},
1522		else => unreachable,
1523		}
1524		}
1525
1526		fn lowerToMiEnc(tag: Tag, reg_or_mem: RegisterOrMemory, imm: i32, code: *std.ArrayList(u8)) LoweringError!void {
1527		const modrm_ext = getModRmExt(tag).?;
1528		switch (reg_or_mem) {
1529		.register => \|dst_reg\| {
1530		const opc = getOpCode(tag, .mi, dst_reg.size() == 8).?;
1531		const encoder = try Encoder.init(code, 7);
1532		if (dst_reg.size() == 16) {
1533		// 0x66 prefix switches to the non-default size; here we assume a switch from
1534		// the default 32bits to 16bits operand-size.
1535		// More info: https://www.cs.uni-potsdam.de/desn/lehre/ss15/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf#page=32&zoom=auto,-159,773
1536		encoder.opcode_1byte(0x66);
1537		}
1538		encoder.rex(.{
1539		.w = setRexWRegister(dst_reg),
1540		.b = dst_reg.isExtended(),
1541		});
1542		opc.encode(encoder);
1543		encoder.modRm_direct(modrm_ext, dst_reg.lowId());
1544		switch (dst_reg.size()) {
1545		8 => {
1546		const imm8 = try math.cast(i8, imm);
1547		encoder.imm8(imm8);
1548		},
1549		16 => {
1550		const imm16 = try math.cast(i16, imm);
1551		encoder.imm16(imm16);
1552		},
1553		32, 64 => encoder.imm32(imm),
1554		else => unreachable,
1555		}
1556		},
1557		.memory => \|dst_mem\| {
1558		const opc = getOpCode(tag, .mi, dst_mem.ptr_size == .byte_ptr).?;
1559		const encoder = try Encoder.init(code, 12);
1560		if (dst_mem.ptr_size == .word_ptr) {
1561		encoder.opcode_1byte(0x66);
1562		}
1563		if (dst_mem.reg) \|dst_reg\| {
1564		if (dst_reg.size() != 64) {
1565		return error.OperandSizeMismatch;
1566		}
1567		encoder.rex(.{
1568		.w = dst_mem.ptr_size == .qword_ptr,
1569		.b = dst_reg.isExtended(),
1570		});
1571		opc.encode(encoder);
1572		if (dst_reg.lowId() == 4) {
1573		if (dst_mem.disp == 0) {
1574		encoder.modRm_SIBDisp0(modrm_ext);
1575		encoder.sib_base(dst_reg.lowId());
1576		} else if (immOpSize(dst_mem.disp) == 8) {
1577		encoder.modRm_SIBDisp8(modrm_ext);
1578		encoder.sib_baseDisp8(dst_reg.lowId());
1579		encoder.disp8(@intCast(i8, dst_mem.disp));
1580		} else {
1581		encoder.modRm_SIBDisp32(modrm_ext);
1582		encoder.sib_baseDisp32(dst_reg.lowId());
1583		encoder.disp32(dst_mem.disp);
1584		}
1585		} else {
1586		if (dst_mem.disp == 0) {
1587		encoder.modRm_indirectDisp0(modrm_ext, dst_reg.lowId());
1588		} else if (immOpSize(dst_mem.disp) == 8) {
1589		encoder.modRm_indirectDisp8(modrm_ext, dst_reg.lowId());
1590		encoder.disp8(@intCast(i8, dst_mem.disp));
1591		} else {
1592		encoder.modRm_indirectDisp32(modrm_ext, dst_reg.lowId());
1593		encoder.disp32(dst_mem.disp);
1594		}
1595		}
1596		} else {
1597		opc.encode(encoder);
1598		if (dst_mem.rip) {
1599		encoder.modRm_RIPDisp32(modrm_ext);
1600		} else {
1601		encoder.modRm_SIBDisp0(modrm_ext);
1602		encoder.sib_disp32();
1603		}
1604		encoder.disp32(dst_mem.disp);
1605		}
1606		switch (dst_mem.ptr_size) {
1607		.byte_ptr => {
1608		const imm8 = try math.cast(i8, imm);
1609		encoder.imm8(imm8);
1610		},
1611		.word_ptr => {
1612		const imm16 = try math.cast(i16, imm);
1613		encoder.imm16(imm16);
1614		},
1615		.dword_ptr, .qword_ptr => {
1616		encoder.imm32(imm);
1617		},
1618		}
1619		},
1620		}
1621		}
1622
1623		fn lowerToRmEnc(
1624		tag: Tag,
1625		reg: Register,
1626		reg_or_mem: RegisterOrMemory,
1627		code: *std.ArrayList(u8),
1628		) LoweringError!void {
1629		const opc = getOpCode(tag, .rm, reg.size() == 8).?;
1630		switch (reg_or_mem) {
1631		.register => \|src_reg\| {
1632		if (reg.size() != src_reg.size()) {
1633		return error.OperandSizeMismatch;
1634		}
1635		const encoder = try Encoder.init(code, 3);
1636		encoder.rex(.{
1637		.w = setRexWRegister(reg) or setRexWRegister(src_reg),
1638		.r = reg.isExtended(),
1639		.b = src_reg.isExtended(),
1640		});
1641		opc.encode(encoder);
1642		encoder.modRm_direct(reg.lowId(), src_reg.lowId());
1643		},
1644		.memory => \|src_mem\| {
1645		if (reg.size() != src_mem.ptr_size.size()) {
1646		return error.OperandSizeMismatch;
1647		}
1648		const encoder = try Encoder.init(code, 9);
1649		if (reg.size() == 16) {
1650		encoder.opcode_1byte(0x66);
1651		}
1652		if (src_mem.reg) \|src_reg\| {
1653		// TODO handle 32-bit base register - requires prefix 0x67
1654		// Intel Manual, Vol 1, chapter 3.6 and 3.6.1
1655		if (src_reg.size() != 64) {
1656		return error.OperandSizeMismatch;
1657		}
1658		encoder.rex(.{
1659		.w = setRexWRegister(reg),
1660		.r = reg.isExtended(),
1661		.b = src_reg.isExtended(),
1662		});
1663		opc.encode(encoder);
1664		if (src_reg.lowId() == 4) {
1665		if (src_mem.disp == 0) {
1666		encoder.modRm_SIBDisp0(reg.lowId());
1667		encoder.sib_base(src_reg.lowId());
1668		} else if (immOpSize(src_mem.disp) == 8) {
1669		encoder.modRm_SIBDisp8(reg.lowId());
1670		encoder.sib_baseDisp8(src_reg.lowId());
1671		encoder.disp8(@intCast(i8, src_mem.disp));
1672		} else {
1673		encoder.modRm_SIBDisp32(reg.lowId());
1674		encoder.sib_baseDisp32(src_reg.lowId());
1675		encoder.disp32(src_mem.disp);
1676		}
1677		} else {
1678		if (src_mem.disp == 0) {
1679		encoder.modRm_indirectDisp0(reg.lowId(), src_reg.lowId());
1680		} else if (immOpSize(src_mem.disp) == 8) {
1681		encoder.modRm_indirectDisp8(reg.lowId(), src_reg.lowId());
1682		encoder.disp8(@intCast(i8, src_mem.disp));
1683		} else {
1684		encoder.modRm_indirectDisp32(reg.lowId(), src_reg.lowId());
1685		encoder.disp32(src_mem.disp);
1686		}
1687		}
1688		} else {
1689		encoder.rex(.{
1690		.w = setRexWRegister(reg),
1691		.r = reg.isExtended(),
1692		});
1693		opc.encode(encoder);
1694		if (src_mem.rip) {
1695		encoder.modRm_RIPDisp32(reg.lowId());
1696		} else {
1697		encoder.modRm_SIBDisp0(reg.lowId());
1698		encoder.sib_disp32();
1699		}
1700		encoder.disp32(src_mem.disp);
1701		}
1702		},
1703		}
1704		}
1705
1706		fn lowerToMrEnc(
1707		tag: Tag,
1708		reg_or_mem: RegisterOrMemory,
1709		reg: Register,
1710		code: *std.ArrayList(u8),
1711		) LoweringError!void {
1712		const opc = getOpCode(tag, .mr, reg.size() == 8).?;
1713		switch (reg_or_mem) {
1714		.register => \|dst_reg\| {
1715		if (dst_reg.size() != reg.size()) {
1716		return error.OperandSizeMismatch;
1717		}
1718		const encoder = try Encoder.init(code, 3);
1719		encoder.rex(.{
1720		.w = setRexWRegister(dst_reg) or setRexWRegister(reg),
1721		.r = reg.isExtended(),
1722		.b = dst_reg.isExtended(),
1723		});
1724		opc.encode(encoder);
1725		encoder.modRm_direct(reg.lowId(), dst_reg.lowId());
1726		},
1727		.memory => \|dst_mem\| {
1728		if (dst_mem.ptr_size.size() != reg.size()) {
1729		return error.OperandSizeMismatch;
1730		}
1731		const encoder = try Encoder.init(code, 9);
1732		if (reg.size() == 16) {
1733		encoder.opcode_1byte(0x66);
1734		}
1735		if (dst_mem.reg) \|dst_reg\| {
1736		if (dst_reg.size() != 64) {
1737		return error.OperandSizeMismatch;
1738		}
1739		encoder.rex(.{
1740		.w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg),
1741		.r = reg.isExtended(),
1742		.b = dst_reg.isExtended(),
1743		});
1744		opc.encode(encoder);
1745		if (dst_reg.lowId() == 4) {
1746		if (dst_mem.disp == 0) {
1747		encoder.modRm_SIBDisp0(reg.lowId());
1748		encoder.sib_base(dst_reg.lowId());
1749		} else if (immOpSize(dst_mem.disp) == 8) {
1750		encoder.modRm_SIBDisp8(reg.lowId());
1751		encoder.sib_baseDisp8(dst_reg.lowId());
1752		encoder.disp8(@intCast(i8, dst_mem.disp));
1753		} else {
1754		encoder.modRm_SIBDisp32(reg.lowId());
1755		encoder.sib_baseDisp32(dst_reg.lowId());
1756		encoder.disp32(dst_mem.disp);
1757		}
1758		} else {
1759		if (dst_mem.disp == 0) {
1760		encoder.modRm_indirectDisp0(reg.lowId(), dst_reg.lowId());
1761		} else if (immOpSize(dst_mem.disp) == 8) {
1762		encoder.modRm_indirectDisp8(reg.lowId(), dst_reg.lowId());
1763		encoder.disp8(@intCast(i8, dst_mem.disp));
1764		} else {
1765		encoder.modRm_indirectDisp32(reg.lowId(), dst_reg.lowId());
1766		encoder.disp32(dst_mem.disp);
1767		}
1768		}
1769		} else {
1770		encoder.rex(.{
1771		.w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg),
1772		.r = reg.isExtended(),
1773		});
1774		opc.encode(encoder);
1775		if (dst_mem.rip) {
1776		encoder.modRm_RIPDisp32(reg.lowId());
1777		} else {
1778		encoder.modRm_SIBDisp0(reg.lowId());
1779		encoder.sib_disp32();
1780		}
1781		encoder.disp32(dst_mem.disp);
1782		}
1783		},
1784		}
1785		}
1786
1787		fn lowerToRmiEnc(
1788		tag: Tag,
1789		reg: Register,
1790		reg_or_mem: RegisterOrMemory,
1791		imm: i32,
1792		code: *std.ArrayList(u8),
1793		) LoweringError!void {
1794		if (reg.size() == 8) {
1795		return error.OperandSizeMismatch;
1796		}
1797		const opc = getOpCode(tag, .rmi, false).?;
1798		const encoder = try Encoder.init(code, 13);
1799		if (reg.size() == 16) {
1800		encoder.opcode_1byte(0x66);
1801		}
1802		switch (reg_or_mem) {
1803		.register => \|src_reg\| {
1804		if (reg.size() != src_reg.size()) {
1805		return error.OperandSizeMismatch;
1806		}
1807		encoder.rex(.{
1808		.w = setRexWRegister(reg) or setRexWRegister(src_reg),
1809		.r = reg.isExtended(),
1810		.b = src_reg.isExtended(),
1811		});
1812		opc.encode(encoder);
1813		encoder.modRm_direct(reg.lowId(), src_reg.lowId());
1814		},
1815		.memory => \|src_mem\| {
1816		if (src_mem.reg) \|src_reg\| {
1817		// TODO handle 32-bit base register - requires prefix 0x67
1818		// Intel Manual, Vol 1, chapter 3.6 and 3.6.1
1819		if (src_reg.size() != 64) {
1820		return error.OperandSizeMismatch;
1821		}
1822		if (src_mem.ptr_size == .byte_ptr) {
1823		return error.OperandSizeMismatch;
1824		}
1825		encoder.rex(.{
1826		.w = setRexWRegister(reg),
1827		.r = reg.isExtended(),
1828		.b = src_reg.isExtended(),
1829		});
1830		opc.encode(encoder);
1831		if (src_reg.lowId() == 4) {
1832		if (src_mem.disp == 0) {
1833		encoder.modRm_SIBDisp0(reg.lowId());
1834		encoder.sib_base(src_reg.lowId());
1835		} else if (immOpSize(src_mem.disp) == 8) {
1836		encoder.modRm_SIBDisp8(reg.lowId());
1837		encoder.sib_baseDisp8(src_reg.lowId());
1838		encoder.disp8(@intCast(i8, src_mem.disp));
1839		} else {
1840		encoder.modRm_SIBDisp32(reg.lowId());
1841		encoder.sib_baseDisp32(src_reg.lowId());
1842		encoder.disp32(src_mem.disp);
1843		}
1844		} else {
1845		if (src_mem.disp == 0) {
1846		encoder.modRm_indirectDisp0(reg.lowId(), src_reg.lowId());
1847		} else if (immOpSize(src_mem.disp) == 8) {
1848		encoder.modRm_indirectDisp8(reg.lowId(), src_reg.lowId());
1849		encoder.disp8(@intCast(i8, src_mem.disp));
1850		} else {
1851		encoder.modRm_indirectDisp32(reg.lowId(), src_reg.lowId());
1852		encoder.disp32(src_mem.disp);
1853		}
1854		}
1855		} else {
1856		encoder.rex(.{
1857		.w = setRexWRegister(reg),
1858		.r = reg.isExtended(),
1859		});
1860		opc.encode(encoder);
1861		if (src_mem.rip) {
1862		encoder.modRm_RIPDisp32(reg.lowId());
1863		} else {
1864		encoder.modRm_SIBDisp0(reg.lowId());
1865		encoder.sib_disp32();
1866		}
1867		encoder.disp32(src_mem.disp);
1868		}
1869		},
1870		}
1871		switch (reg.size()) {
1872		// TODO 8bit immediate
1873		8 => unreachable,
1874		16 => {
1875		const imm16 = try math.cast(i16, imm);
1876		encoder.imm16(imm16);
1877		},
1878		32, 64 => encoder.imm32(imm),
1879		else => unreachable,
1880		}
1881		}
1882
1883		fn expectEqualHexStrings(expected: []const u8, given: []const u8, assembly: []const u8) !void {
1884		assert(expected.len > 0);
1885		if (mem.eql(u8, expected, given)) return;
1886		const expected_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{std.fmt.fmtSliceHexLower(expected)});
1887		defer testing.allocator.free(expected_fmt);
1888		const given_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{std.fmt.fmtSliceHexLower(given)});
1889		defer testing.allocator.free(given_fmt);
1890		const idx = mem.indexOfDiff(u8, expected_fmt, given_fmt).?;
1891		var padding = try testing.allocator.alloc(u8, idx + 5);
1892		defer testing.allocator.free(padding);
1893		mem.set(u8, padding, ' ');
1894		std.debug.print("\nASM: {s}\nEXP: {s}\nGIV: {s}\n{s}^ -- first differing byte\n", .{
1895		assembly,
1896		expected_fmt,
1897		given_fmt,
1898		padding,
1899		});
1900		return error.TestFailed;
1901		}
1902
1903		const TestEmitCode = struct {
1904		buf: std.ArrayList(u8),
1905		next: usize = 0,
1906
1907		fn init() TestEmitCode {
1908		return .{
1909		.buf = std.ArrayList(u8).init(testing.allocator),
1910		};
1911		}
1912
1913		fn deinit(emit: *TestEmitCode) void {
1914		emit.buf.deinit();
1915		emit.next = undefined;
1916		}
1917
1918		fn buffer(emit: TestEmitCode) std.ArrayList(u8) {
1919		emit.next = emit.buf.items.len;
1920		return &emit.buf;
1921		}
1922
1923		fn emitted(emit: TestEmitCode) []const u8 {
1924		return emit.buf.items[emit.next..];
1925		}
1926		};
1927
1928		test "lower MI encoding" {
1929		var code = TestEmitCode.init();
1930		defer code.deinit();
1931		try lowerToMiEnc(.mov, RegisterOrMemory.reg(.rax), 0x10, code.buffer());
1932		try expectEqualHexStrings("\x48\xc7\xc0\x10\x00\x00\x00", code.emitted(), "mov rax, 0x10");
1933		try lowerToMiEnc(.mov, RegisterOrMemory.mem(.r11, 0, .dword_ptr), 0x10, code.buffer());
1934		try expectEqualHexStrings("\x41\xc7\x03\x10\x00\x00\x00", code.emitted(), "mov dword ptr [r11 + 0], 0x10");
1935		try lowerToMiEnc(.add, RegisterOrMemory.mem(.rdx, -8, .dword_ptr), 0x10, code.buffer());
1936		try expectEqualHexStrings("\x81\x42\xF8\x10\x00\x00\x00", code.emitted(), "add dword ptr [rdx - 8], 0x10");
1937		try lowerToMiEnc(.sub, RegisterOrMemory.mem(.r11, 0x10000000, .dword_ptr), 0x10, code.buffer());
1938		try expectEqualHexStrings(
1939		"\x41\x81\xab\x00\x00\x00\x10\x10\x00\x00\x00",
1940		code.emitted(),
1941		"sub dword ptr [r11 + 0x10000000], 0x10",
1942		);
1943		try lowerToMiEnc(.@"and", RegisterOrMemory.mem(null, 0x10000000, .dword_ptr), 0x10, code.buffer());
1944		try expectEqualHexStrings(
1945		"\x81\x24\x25\x00\x00\x00\x10\x10\x00\x00\x00",
1946		code.emitted(),
1947		"and dword ptr [ds:0x10000000], 0x10",
1948		);
1949		try lowerToMiEnc(.@"and", RegisterOrMemory.mem(.r12, 0x10000000, .dword_ptr), 0x10, code.buffer());
1950		try expectEqualHexStrings(
1951		"\x41\x81\xA4\x24\x00\x00\x00\x10\x10\x00\x00\x00",
1952		code.emitted(),
1953		"and dword ptr [r12 + 0x10000000], 0x10",
1954		);
1955		try lowerToMiEnc(.mov, RegisterOrMemory.rip(0x10, .qword_ptr), 0x10, code.buffer());
1956		try expectEqualHexStrings(
1957		"\xC7\x05\x10\x00\x00\x00\x10\x00\x00\x00",
1958		code.emitted(),
1959		"mov qword ptr [rip + 0x10], 0x10",
1960		);
1961		try lowerToMiEnc(.mov, RegisterOrMemory.mem(.rbp, -8, .qword_ptr), 0x10, code.buffer());
1962		try expectEqualHexStrings(
1963		"\x48\xc7\x45\xf8\x10\x00\x00\x00",
1964		code.emitted(),
1965		"mov qword ptr [rbp - 8], 0x10",
1966		);
1967		try lowerToMiEnc(.mov, RegisterOrMemory.mem(.rbp, -2, .word_ptr), 0x10, code.buffer());
1968		try expectEqualHexStrings("\x66\xC7\x45\xFE\x10\x00", code.emitted(), "mov word ptr [rbp - 2], 0x10");
1969		try lowerToMiEnc(.mov, RegisterOrMemory.mem(.rbp, -1, .byte_ptr), 0x10, code.buffer());
1970		try expectEqualHexStrings("\xC6\x45\xFF\x10", code.emitted(), "mov byte ptr [rbp - 1], 0x10");
1971		}
1972
1973		test "lower RM encoding" {
1974		var code = TestEmitCode.init();
1975		defer code.deinit();
1976		try lowerToRmEnc(.mov, .rax, RegisterOrMemory.reg(.rbx), code.buffer());
1977		try expectEqualHexStrings("\x48\x8b\xc3", code.emitted(), "mov rax, rbx");
1978		try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.r11, 0, .qword_ptr), code.buffer());
1979		try expectEqualHexStrings("\x49\x8b\x03", code.emitted(), "mov rax, qword ptr [r11 + 0]");
1980		try lowerToRmEnc(.add, .r11, RegisterOrMemory.mem(null, 0x10000000, .qword_ptr), code.buffer());
1981		try expectEqualHexStrings(
1982		"\x4C\x03\x1C\x25\x00\x00\x00\x10",
1983		code.emitted(),
1984		"add r11, qword ptr [ds:0x10000000]",
1985		);
1986		try lowerToRmEnc(.add, .r12b, RegisterOrMemory.mem(null, 0x10000000, .byte_ptr), code.buffer());
1987		try expectEqualHexStrings(
1988		"\x44\x02\x24\x25\x00\x00\x00\x10",
1989		code.emitted(),
1990		"add r11b, byte ptr [ds:0x10000000]",
1991		);
1992		try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.r13, 0x10000000, .qword_ptr), code.buffer());
1993		try expectEqualHexStrings(
1994		"\x4D\x2B\x9D\x00\x00\x00\x10",
1995		code.emitted(),
1996		"sub r11, qword ptr [r13 + 0x10000000]",
1997		);
1998		try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.r12, 0x10000000, .qword_ptr), code.buffer());
1999		try expectEqualHexStrings(
2000		"\x4D\x2B\x9C\x24\x00\x00\x00\x10",
2001		code.emitted(),
2002		"sub r11, qword ptr [r12 + 0x10000000]",
2003		);
2004		try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.rbp, -4, .qword_ptr), code.buffer());
2005		try expectEqualHexStrings("\x48\x8B\x45\xFC", code.emitted(), "mov rax, qword ptr [rbp - 4]");
2006		try lowerToRmEnc(.lea, .rax, RegisterOrMemory.rip(0x10, .qword_ptr), code.buffer());
2007		try expectEqualHexStrings("\x48\x8D\x05\x10\x00\x00\x00", code.emitted(), "lea rax, [rip + 0x10]");
2008		}
2009
2010		test "lower MR encoding" {
2011		var code = TestEmitCode.init();
2012		defer code.deinit();
2013		try lowerToMrEnc(.mov, RegisterOrMemory.reg(.rax), .rbx, code.buffer());
2014		try expectEqualHexStrings("\x48\x89\xd8", code.emitted(), "mov rax, rbx");
2015		try lowerToMrEnc(.mov, RegisterOrMemory.mem(.rbp, -4, .qword_ptr), .r11, code.buffer());
2016		try expectEqualHexStrings("\x4c\x89\x5d\xfc", code.emitted(), "mov qword ptr [rbp - 4], r11");
2017		try lowerToMrEnc(.add, RegisterOrMemory.mem(null, 0x10000000, .byte_ptr), .r12b, code.buffer());
2018		try expectEqualHexStrings(
2019		"\x44\x00\x24\x25\x00\x00\x00\x10",
2020		code.emitted(),
2021		"add byte ptr [ds:0x10000000], r12b",
2022		);
2023		try lowerToMrEnc(.add, RegisterOrMemory.mem(null, 0x10000000, .dword_ptr), .r12d, code.buffer());
2024		try expectEqualHexStrings(
2025		"\x44\x01\x24\x25\x00\x00\x00\x10",
2026		code.emitted(),
2027		"add dword ptr [ds:0x10000000], r12d",
2028		);
2029		try lowerToMrEnc(.sub, RegisterOrMemory.mem(.r11, 0x10000000, .qword_ptr), .r12, code.buffer());
2030		try expectEqualHexStrings(
2031		"\x4D\x29\xA3\x00\x00\x00\x10",
2032		code.emitted(),
2033		"sub qword ptr [r11 + 0x10000000], r12",
2034		);
2035		try lowerToMrEnc(.mov, RegisterOrMemory.rip(0x10, .qword_ptr), .r12, code.buffer());
2036		try expectEqualHexStrings("\x4C\x89\x25\x10\x00\x00\x00", code.emitted(), "mov qword ptr [rip + 0x10], r12");
2037		}
2038
2039		test "lower OI encoding" {
2040		var code = TestEmitCode.init();
2041		defer code.deinit();
2042		try lowerToOiEnc(.mov, .rax, 0x1000000000000000, code.buffer());
2043		try expectEqualHexStrings(
2044		"\x48\xB8\x00\x00\x00\x00\x00\x00\x00\x10",
2045		code.emitted(),
2046		"movabs rax, 0x1000000000000000",
2047		);
2048		try lowerToOiEnc(.mov, .r11, 0x1000000000000000, code.buffer());
2049		try expectEqualHexStrings(
2050		"\x49\xBB\x00\x00\x00\x00\x00\x00\x00\x10",
2051		code.emitted(),
2052		"movabs r11, 0x1000000000000000",
2053		);
2054		try lowerToOiEnc(.mov, .r11d, 0x10000000, code.buffer());
2055		try expectEqualHexStrings("\x41\xBB\x00\x00\x00\x10", code.emitted(), "mov r11d, 0x10000000");
2056		try lowerToOiEnc(.mov, .r11w, 0x1000, code.buffer());
2057		try expectEqualHexStrings("\x66\x41\xBB\x00\x10", code.emitted(), "mov r11w, 0x1000");
2058		try lowerToOiEnc(.mov, .r11b, 0x10, code.buffer());
2059		try expectEqualHexStrings("\x41\xB3\x10", code.emitted(), "mov r11b, 0x10");
2060		}
2061
2062		test "lower FD/TD encoding" {
2063		var code = TestEmitCode.init();
2064		defer code.deinit();
2065		try lowerToFdEnc(.mov, .rax, 0x1000000000000000, code.buffer());
2066		try expectEqualHexStrings(
2067		"\x48\xa1\x00\x00\x00\x00\x00\x00\x00\x10",
2068		code.emitted(),
2069		"mov rax, ds:0x1000000000000000",
2070		);
2071		try lowerToFdEnc(.mov, .eax, 0x10000000, code.buffer());
2072		try expectEqualHexStrings("\xa1\x00\x00\x00\x10", code.emitted(), "mov eax, ds:0x10000000");
2073		try lowerToFdEnc(.mov, .ax, 0x1000, code.buffer());
2074		try expectEqualHexStrings("\x66\xa1\x00\x10", code.emitted(), "mov ax, ds:0x1000");
2075		try lowerToFdEnc(.mov, .al, 0x10, code.buffer());
2076		try expectEqualHexStrings("\xa0\x10", code.emitted(), "mov al, ds:0x10");
2077		}
2078
2079		test "lower M encoding" {
2080		var code = TestEmitCode.init();
2081		defer code.deinit();
2082		try lowerToMEnc(.jmp_near, RegisterOrMemory.reg(.r12), code.buffer());
2083		try expectEqualHexStrings("\x41\xFF\xE4", code.emitted(), "jmp r12");
2084		try lowerToMEnc(.jmp_near, RegisterOrMemory.reg(.r12w), code.buffer());
2085		try expectEqualHexStrings("\x66\x41\xFF\xE4", code.emitted(), "jmp r12w");
2086		try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.r12, 0, .qword_ptr), code.buffer());
2087		try expectEqualHexStrings("\x41\xFF\x24\x24", code.emitted(), "jmp qword ptr [r12]");
2088		try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.r12, 0, .word_ptr), code.buffer());
2089		try expectEqualHexStrings("\x66\x41\xFF\x24\x24", code.emitted(), "jmp word ptr [r12]");
2090		try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.r12, 0x10, .qword_ptr), code.buffer());
2091		try expectEqualHexStrings("\x41\xFF\x64\x24\x10", code.emitted(), "jmp qword ptr [r12 + 0x10]");
2092		try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.r12, 0x1000, .qword_ptr), code.buffer());
2093		try expectEqualHexStrings(
2094		"\x41\xFF\xA4\x24\x00\x10\x00\x00",
2095		code.emitted(),
2096		"jmp qword ptr [r12 + 0x1000]",
2097		);
2098		try lowerToMEnc(.jmp_near, RegisterOrMemory.rip(0x10, .qword_ptr), code.buffer());
2099		try expectEqualHexStrings("\xFF\x25\x10\x00\x00\x00", code.emitted(), "jmp qword ptr [rip + 0x10]");
2100		try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(null, 0x10, .qword_ptr), code.buffer());
2101		try expectEqualHexStrings("\xFF\x24\x25\x10\x00\x00\x00", code.emitted(), "jmp qword ptr [ds:0x10]");
2102		try lowerToMEnc(.seta, RegisterOrMemory.reg(.r11b), code.buffer());
2103		try expectEqualHexStrings("\x41\x0F\x97\xC3", code.emitted(), "seta r11b");
2104		}
2105
2106		test "lower O encoding" {
2107		var code = TestEmitCode.init();
2108		defer code.deinit();
2109		try lowerToOEnc(.pop, .r12, code.buffer());
2110		try expectEqualHexStrings("\x41\x5c", code.emitted(), "pop r12");
2111		try lowerToOEnc(.push, .r12w, code.buffer());
2112		try expectEqualHexStrings("\x66\x41\x54", code.emitted(), "push r12w");
2113		}
2114
2115		test "lower RMI encoding" {
2116		var code = TestEmitCode.init();
2117		defer code.deinit();
2118		try lowerToRmiEnc(.imul, .rax, RegisterOrMemory.mem(.rbp, -8, .qword_ptr), 0x10, code.buffer());
2119		try expectEqualHexStrings(
2120		"\x48\x69\x45\xF8\x10\x00\x00\x00",
2121		code.emitted(),
2122		"imul rax, qword ptr [rbp - 8], 0x10",
2123		);
2124		try lowerToRmiEnc(.imul, .eax, RegisterOrMemory.mem(.rbp, -4, .dword_ptr), 0x10, code.buffer());
2125		try expectEqualHexStrings("\x69\x45\xFC\x10\x00\x00\x00", code.emitted(), "imul eax, dword ptr [rbp - 4], 0x10");
2126		try lowerToRmiEnc(.imul, .ax, RegisterOrMemory.mem(.rbp, -2, .word_ptr), 0x10, code.buffer());
2127		try expectEqualHexStrings("\x66\x69\x45\xFE\x10\x00", code.emitted(), "imul ax, word ptr [rbp - 2], 0x10");
2128		try lowerToRmiEnc(.imul, .r12, RegisterOrMemory.reg(.r12), 0x10, code.buffer());
2129		try expectEqualHexStrings("\x4D\x69\xE4\x10\x00\x00\x00", code.emitted(), "imul r12, r12, 0x10");
2130		try lowerToRmiEnc(.imul, .r12w, RegisterOrMemory.reg(.r12w), 0x10, code.buffer());
2131		try expectEqualHexStrings("\x66\x45\x69\xE4\x10\x00", code.emitted(), "imul r12w, r12w, 0x10");
2132		}

src/arch/x86_64/Isel.zig created+2132

...	...	@@ -0,0 +1,2132 @@
	1	//! This file contains the functionality for lowering x86_64 MIR into
	2	//! machine code
	3
	4	const Isel = @This();
	5
	6	const std = @import("std");
	7	const assert = std.debug.assert;
	8	const bits = @import("bits.zig");
	9	const leb128 = std.leb;
	10	const link = @import("../../link.zig");
	11	const log = std.log.scoped(.codegen);
	12	const math = std.math;
	13	const mem = std.mem;
	14	const testing = std.testing;
	15
	16	const Air = @import("../../Air.zig");
	17	const Allocator = mem.Allocator;
	18	const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;
	19	const DW = std.dwarf;
	20	const Encoder = bits.Encoder;
	21	const ErrorMsg = Module.ErrorMsg;
	22	const MCValue = @import("CodeGen.zig").MCValue;
	23	const Mir = @import("Mir.zig");
	24	const Module = @import("../../Module.zig");
	25	const Instruction = bits.Instruction;
	26	const Register = bits.Register;
	27	const Type = @import("../../type.zig").Type;
	28
	29	mir: Mir,
	30	bin_file: *link.File,
	31	debug_output: DebugInfoOutput,
	32	target: *const std.Target,
	33	err_msg: ?*ErrorMsg = null,
	34	src_loc: Module.SrcLoc,
	35	code: *std.ArrayList(u8),
	36
	37	prev_di_line: u32,
	38	prev_di_column: u32,
	39	/// Relative to the beginning of `code`.
	40	prev_di_pc: usize,
	41
	42	code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .{},
	43	relocs: std.ArrayListUnmanaged(Reloc) = .{},
	44
	45	const InnerError = error{
	46	OutOfMemory,
	47	Overflow,
	48	IselFail,
	49	};
	50
	51	const Reloc = struct {
	52	/// Offset of the instruction.
	53	source: u64,
	54	/// Target of the relocation.
	55	target: Mir.Inst.Index,
	56	/// Offset of the relocation within the instruction.
	57	offset: u64,
	58	/// Length of the instruction.
	59	length: u5,
	60	};
	61
	62	pub fn lowerMir(isel: *Isel) InnerError!void {
	63	const mir_tags = isel.mir.instructions.items(.tag);
	64
	65	for (mir_tags) \|tag, index\| {
	66	const inst = @intCast(u32, index);
	67	try isel.code_offset_mapping.putNoClobber(isel.bin_file.allocator, inst, isel.code.items.len);
	68	switch (tag) {
	69	.adc => try isel.mirArith(.adc, inst),
	70	.add => try isel.mirArith(.add, inst),
	71	.sub => try isel.mirArith(.sub, inst),
	72	.xor => try isel.mirArith(.xor, inst),
	73	.@"and" => try isel.mirArith(.@"and", inst),
	74	.@"or" => try isel.mirArith(.@"or", inst),
	75	.sbb => try isel.mirArith(.sbb, inst),
	76	.cmp => try isel.mirArith(.cmp, inst),
	77	.mov => try isel.mirArith(.mov, inst),
	78
	79	.adc_mem_imm => try isel.mirArithMemImm(.adc, inst),
	80	.add_mem_imm => try isel.mirArithMemImm(.add, inst),
	81	.sub_mem_imm => try isel.mirArithMemImm(.sub, inst),
	82	.xor_mem_imm => try isel.mirArithMemImm(.xor, inst),
	83	.and_mem_imm => try isel.mirArithMemImm(.@"and", inst),
	84	.or_mem_imm => try isel.mirArithMemImm(.@"or", inst),
	85	.sbb_mem_imm => try isel.mirArithMemImm(.sbb, inst),
	86	.cmp_mem_imm => try isel.mirArithMemImm(.cmp, inst),
	87	.mov_mem_imm => try isel.mirArithMemImm(.mov, inst),
	88
	89	.adc_scale_src => try isel.mirArithScaleSrc(.adc, inst),
	90	.add_scale_src => try isel.mirArithScaleSrc(.add, inst),
	91	.sub_scale_src => try isel.mirArithScaleSrc(.sub, inst),
	92	.xor_scale_src => try isel.mirArithScaleSrc(.xor, inst),
	93	.and_scale_src => try isel.mirArithScaleSrc(.@"and", inst),
	94	.or_scale_src => try isel.mirArithScaleSrc(.@"or", inst),
	95	.sbb_scale_src => try isel.mirArithScaleSrc(.sbb, inst),
	96	.cmp_scale_src => try isel.mirArithScaleSrc(.cmp, inst),
	97	.mov_scale_src => try isel.mirArithScaleSrc(.mov, inst),
	98
	99	.adc_scale_dst => try isel.mirArithScaleDst(.adc, inst),
	100	.add_scale_dst => try isel.mirArithScaleDst(.add, inst),
	101	.sub_scale_dst => try isel.mirArithScaleDst(.sub, inst),
	102	.xor_scale_dst => try isel.mirArithScaleDst(.xor, inst),
	103	.and_scale_dst => try isel.mirArithScaleDst(.@"and", inst),
	104	.or_scale_dst => try isel.mirArithScaleDst(.@"or", inst),
	105	.sbb_scale_dst => try isel.mirArithScaleDst(.sbb, inst),
	106	.cmp_scale_dst => try isel.mirArithScaleDst(.cmp, inst),
	107	.mov_scale_dst => try isel.mirArithScaleDst(.mov, inst),
	108
	109	.adc_scale_imm => try isel.mirArithScaleImm(.adc, inst),
	110	.add_scale_imm => try isel.mirArithScaleImm(.add, inst),
	111	.sub_scale_imm => try isel.mirArithScaleImm(.sub, inst),
	112	.xor_scale_imm => try isel.mirArithScaleImm(.xor, inst),
	113	.and_scale_imm => try isel.mirArithScaleImm(.@"and", inst),
	114	.or_scale_imm => try isel.mirArithScaleImm(.@"or", inst),
	115	.sbb_scale_imm => try isel.mirArithScaleImm(.sbb, inst),
	116	.cmp_scale_imm => try isel.mirArithScaleImm(.cmp, inst),
	117	.mov_scale_imm => try isel.mirArithScaleImm(.mov, inst),
	118
	119	.movabs => try isel.mirMovabs(inst),
	120
	121	.lea => try isel.mirLea(inst),
	122
	123	.imul_complex => try isel.mirIMulComplex(inst),
	124
	125	.push => try isel.mirPushPop(.push, inst),
	126	.pop => try isel.mirPushPop(.pop, inst),
	127
	128	.jmp => try isel.mirJmpCall(.jmp_near, inst),
	129	.call => try isel.mirJmpCall(.call_near, inst),
	130
	131	.cond_jmp_greater_less,
	132	.cond_jmp_above_below,
	133	.cond_jmp_eq_ne,
	134	=> try isel.mirCondJmp(tag, inst),
	135
	136	.cond_set_byte_greater_less,
	137	.cond_set_byte_above_below,
	138	.cond_set_byte_eq_ne,
	139	=> try isel.mirCondSetByte(tag, inst),
	140
	141	.ret => try isel.mirRet(inst),
	142
	143	.syscall => try isel.mirSyscall(),
	144
	145	.@"test" => try isel.mirTest(inst),
	146
	147	.brk => try isel.mirBrk(),
	148	.nop => try isel.mirNop(),
	149
	150	.call_extern => try isel.mirCallExtern(inst),
	151
	152	.dbg_line => try isel.mirDbgLine(inst),
	153	.dbg_prologue_end => try isel.mirDbgPrologueEnd(inst),
	154	.dbg_epilogue_begin => try isel.mirDbgEpilogueBegin(inst),
	155	.arg_dbg_info => try isel.mirArgDbgInfo(inst),
	156
	157	.push_regs_from_callee_preserved_regs => try isel.mirPushPopRegsFromCalleePreservedRegs(.push, inst),
	158	.pop_regs_from_callee_preserved_regs => try isel.mirPushPopRegsFromCalleePreservedRegs(.pop, inst),
	159
	160	else => {
	161	return isel.fail("Implement MIR->Isel lowering for x86_64 for pseudo-inst: {s}", .{tag});
	162	},
	163	}
	164	}
	165
	166	try isel.fixupRelocs();
	167	}
	168
	169	pub fn deinit(isel: *Isel) void {
	170	isel.relocs.deinit(isel.bin_file.allocator);
	171	isel.code_offset_mapping.deinit(isel.bin_file.allocator);
	172	isel.* = undefined;
	173	}
	174
	175	fn fail(isel: *Isel, comptime format: []const u8, args: anytype) InnerError {
	176	@setCold(true);
	177	assert(isel.err_msg == null);
	178	isel.err_msg = try ErrorMsg.create(isel.bin_file.allocator, isel.src_loc, format, args);
	179	return error.IselFail;
	180	}
	181
	182	fn failWithLoweringError(isel: *Isel, err: LoweringError) InnerError {
	183	return switch (err) {
	184	error.RaxOperandExpected => isel.fail("Register.rax expected as destination operand", .{}),
	185	error.OperandSizeMismatch => isel.fail("operand size mismatch", .{}),
	186	else => \|e\| e,
	187	};
	188	}
	189
	190	fn fixupRelocs(isel: *Isel) InnerError!void {
	191	// TODO this function currently assumes all relocs via JMP/CALL instructions are 32bit in size.
	192	// This should be reversed like it is done in aarch64 MIR emit code: start with the smallest
	193	// possible resolution, i.e., 8bit, and iteratively converge on the minimum required resolution
	194	// until the entire decl is correctly emitted with all JMP/CALL instructions within range.
	195	for (isel.relocs.items) \|reloc\| {
	196	const offset = try math.cast(usize, reloc.offset);
	197	const target = isel.code_offset_mapping.get(reloc.target) orelse
	198	return isel.fail("JMP/CALL relocation target not found!", .{});
	199	const disp = @intCast(i32, @intCast(i64, target) - @intCast(i64, reloc.source + reloc.length));
	200	mem.writeIntLittle(i32, isel.code.items[offset..][0..4], disp);
	201	}
	202	}
	203
	204	fn mirBrk(isel: *Isel) InnerError!void {
	205	return lowerToZoEnc(.brk, isel.code) catch \|err\| isel.failWithLoweringError(err);
	206	}
	207
	208	fn mirNop(isel: *Isel) InnerError!void {
	209	return lowerToZoEnc(.nop, isel.code) catch \|err\| isel.failWithLoweringError(err);
	210	}
	211
	212	fn mirSyscall(isel: *Isel) InnerError!void {
	213	return lowerToZoEnc(.syscall, isel.code) catch \|err\| isel.failWithLoweringError(err);
	214	}
	215
	216	fn mirPushPop(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	217	const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
	218	switch (ops.flags) {
	219	0b00 => {
	220	// PUSH/POP reg
	221	return lowerToOEnc(tag, ops.reg1, isel.code) catch \|err\| isel.failWithLoweringError(err);
	222	},
	223	0b01 => {
	224	// PUSH/POP r/m64
	225	const imm = isel.mir.instructions.items(.data)[inst].imm;
	226	const ptr_size: Memory.PtrSize = switch (immOpSize(imm)) {
	227	16 => .word_ptr,
	228	else => .qword_ptr,
	229	};
	230	return lowerToMEnc(tag, RegisterOrMemory.mem(ops.reg1, imm, ptr_size), isel.code) catch \|err\|
	231	isel.failWithLoweringError(err);
	232	},
	233	0b10 => {
	234	// PUSH imm32
	235	assert(tag == .push);
	236	const imm = isel.mir.instructions.items(.data)[inst].imm;
	237	return lowerToIEnc(.push, imm, isel.code) catch \|err\|
	238	isel.failWithLoweringError(err);
	239	},
	240	0b11 => unreachable,
	241	}
	242	}
	243	fn mirPushPopRegsFromCalleePreservedRegs(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	244	const callee_preserved_regs = bits.callee_preserved_regs;
	245	const regs = isel.mir.instructions.items(.data)[inst].regs_to_push_or_pop;
	246	if (tag == .push) {
	247	for (callee_preserved_regs) \|reg, i\| {
	248	if ((regs >> @intCast(u5, i)) & 1 == 0) continue;
	249	lowerToOEnc(.push, reg, isel.code) catch \|err\|
	250	return isel.failWithLoweringError(err);
	251	}
	252	} else {
	253	// pop in the reverse direction
	254	var i = callee_preserved_regs.len;
	255	while (i > 0) : (i -= 1) {
	256	const reg = callee_preserved_regs[i - 1];
	257	if ((regs >> @intCast(u5, i - 1)) & 1 == 0) continue;
	258	lowerToOEnc(.pop, reg, isel.code) catch \|err\|
	259	return isel.failWithLoweringError(err);
	260	}
	261	}
	262	}
	263
	264	fn mirJmpCall(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	265	const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
	266	const flag = @truncate(u1, ops.flags);
	267	if (flag == 0) {
	268	const target = isel.mir.instructions.items(.data)[inst].inst;
	269	const source = isel.code.items.len;
	270	lowerToDEnc(tag, 0, isel.code) catch \|err\|
	271	return isel.failWithLoweringError(err);
	272	try isel.relocs.append(isel.bin_file.allocator, .{
	273	.source = source,
	274	.target = target,
	275	.offset = isel.code.items.len - 4,
	276	.length = 5,
	277	});
	278	return;
	279	}
	280	if (ops.reg1 == .none) {
	281	// JMP/CALL [imm]
	282	const imm = isel.mir.instructions.items(.data)[inst].imm;
	283	const ptr_size: Memory.PtrSize = switch (immOpSize(imm)) {
	284	16 => .word_ptr,
	285	else => .qword_ptr,
	286	};
	287	return lowerToMEnc(tag, RegisterOrMemory.mem(null, imm, ptr_size), isel.code) catch \|err\|
	288	isel.failWithLoweringError(err);
	289	}
	290	// JMP/CALL reg
	291	return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), isel.code) catch \|err\| isel.failWithLoweringError(err);
	292	}
	293
	294	fn mirCondJmp(isel: *Isel, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
	295	const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
	296	const target = isel.mir.instructions.items(.data)[inst].inst;
	297	const tag = switch (mir_tag) {
	298	.cond_jmp_greater_less => switch (ops.flags) {
	299	0b00 => Tag.jge,
	300	0b01 => Tag.jg,
	301	0b10 => Tag.jl,
	302	0b11 => Tag.jle,
	303	},
	304	.cond_jmp_above_below => switch (ops.flags) {
	305	0b00 => Tag.jae,
	306	0b01 => Tag.ja,
	307	0b10 => Tag.jb,
	308	0b11 => Tag.jbe,
	309	},
	310	.cond_jmp_eq_ne => switch (@truncate(u1, ops.flags)) {
	311	0b0 => Tag.jne,
	312	0b1 => Tag.je,
	313	},
	314	else => unreachable,
	315	};
	316	const source = isel.code.items.len;
	317	lowerToDEnc(tag, 0, isel.code) catch \|err\|
	318	return isel.failWithLoweringError(err);
	319	try isel.relocs.append(isel.bin_file.allocator, .{
	320	.source = source,
	321	.target = target,
	322	.offset = isel.code.items.len - 4,
	323	.length = 6,
	324	});
	325	}
	326
	327	fn mirCondSetByte(isel: *Isel, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
	328	const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
	329	const tag = switch (mir_tag) {
	330	.cond_set_byte_greater_less => switch (ops.flags) {
	331	0b00 => Tag.setge,
	332	0b01 => Tag.setg,
	333	0b10 => Tag.setl,
	334	0b11 => Tag.setle,
	335	},
	336	.cond_set_byte_above_below => switch (ops.flags) {
	337	0b00 => Tag.setae,
	338	0b01 => Tag.seta,
	339	0b10 => Tag.setb,
	340	0b11 => Tag.setbe,
	341	},
	342	.cond_set_byte_eq_ne => switch (@truncate(u1, ops.flags)) {
	343	0b0 => Tag.setne,
	344	0b1 => Tag.sete,
	345	},
	346	else => unreachable,
	347	};
	348	return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1.to8()), isel.code) catch \|err\|
	349	isel.failWithLoweringError(err);
	350	}
	351
	352	fn mirTest(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
	353	const tag = isel.mir.instructions.items(.tag)[inst];
	354	assert(tag == .@"test");
	355	const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
	356	switch (ops.flags) {
	357	0b00 => {
	358	if (ops.reg2 == .none) {
	359	// TEST r/m64, imm32
	360	// MI
	361	const imm = isel.mir.instructions.items(.data)[inst].imm;
	362	if (ops.reg1.to64() == .rax) {
	363	// TEST rax, imm32
	364	// I
	365	return lowerToIEnc(.@"test", imm, isel.code) catch \|err\|
	366	isel.failWithLoweringError(err);
	367	}
	368	return lowerToMiEnc(.@"test", RegisterOrMemory.reg(ops.reg1), imm, isel.code) catch \|err\|
	369	isel.failWithLoweringError(err);
	370	}
	371	// TEST r/m64, r64
	372	return isel.fail("TODO TEST r/m64, r64", .{});
	373	},
	374	else => return isel.fail("TODO more TEST alternatives", .{}),
	375	}
	376	}
	377
	378	fn mirRet(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
	379	const tag = isel.mir.instructions.items(.tag)[inst];
	380	assert(tag == .ret);
	381	const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
	382	switch (ops.flags) {
	383	0b00 => {
	384	// RETF imm16
	385	// I
	386	const imm = isel.mir.instructions.items(.data)[inst].imm;
	387	return lowerToIEnc(.ret_far, imm, isel.code) catch \|err\| isel.failWithLoweringError(err);
	388	},
	389	0b01 => {
	390	return lowerToZoEnc(.ret_far, isel.code) catch \|err\| isel.failWithLoweringError(err);
	391	},
	392	0b10 => {
	393	// RET imm16
	394	// I
	395	const imm = isel.mir.instructions.items(.data)[inst].imm;
	396	return lowerToIEnc(.ret_near, imm, isel.code) catch \|err\| isel.failWithLoweringError(err);
	397	},
	398	0b11 => {
	399	return lowerToZoEnc(.ret_near, isel.code) catch \|err\| isel.failWithLoweringError(err);
	400	},
	401	}
	402	}
	403
	404	fn mirArith(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	405	const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
	406	switch (ops.flags) {
	407	0b00 => {
	408	if (ops.reg2 == .none) {
	409	// mov reg1, imm32
	410	// MI
	411	const imm = isel.mir.instructions.items(.data)[inst].imm;
	412	return lowerToMiEnc(tag, RegisterOrMemory.reg(ops.reg1), imm, isel.code) catch \|err\|
	413	isel.failWithLoweringError(err);
	414	}
	415	// mov reg1, reg2
	416	// RM
	417	return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), isel.code) catch \|err\|
	418	isel.failWithLoweringError(err);
	419	},
	420	0b01 => {
	421	// mov reg1, [reg2 + imm32]
	422	// RM
	423	const imm = isel.mir.instructions.items(.data)[inst].imm;
	424	const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
	425	return lowerToRmEnc(
	426	tag,
	427	ops.reg1,
	428	RegisterOrMemory.mem(src_reg, imm, Memory.PtrSize.fromBits(ops.reg1.size())),
	429	isel.code,
	430	) catch \|err\| isel.failWithLoweringError(err);
	431	},
	432	0b10 => {
	433	if (ops.reg2 == .none) {
	434	return isel.fail("TODO unused variant: mov reg1, none, 0b10", .{});
	435	}
	436	// mov [reg1 + imm32], reg2
	437	// MR
	438	const imm = isel.mir.instructions.items(.data)[inst].imm;
	439	return lowerToMrEnc(
	440	tag,
	441	RegisterOrMemory.mem(ops.reg1, imm, Memory.PtrSize.fromBits(ops.reg2.size())),
	442	ops.reg2,
	443	isel.code,
	444	) catch \|err\| isel.failWithLoweringError(err);
	445	},
	446	0b11 => {
	447	return isel.fail("TODO unused variant: mov reg1, reg2, 0b11", .{});
	448	},
	449	}
	450	}
	451
	452	fn mirArithMemImm(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	453	const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
	454	assert(ops.reg2 == .none);
	455	const payload = isel.mir.instructions.items(.data)[inst].payload;
	456	const imm_pair = isel.mir.extraData(Mir.ImmPair, payload).data;
	457	const ptr_size: Memory.PtrSize = switch (ops.flags) {
	458	0b00 => .byte_ptr,
	459	0b01 => .word_ptr,
	460	0b10 => .dword_ptr,
	461	0b11 => .qword_ptr,
	462	};
	463	return lowerToMiEnc(
	464	tag,
	465	RegisterOrMemory.mem(ops.reg1, imm_pair.dest_off, ptr_size),
	466	imm_pair.operand,
	467	isel.code,
	468	) catch \|err\| isel.failWithLoweringError(err);
	469	}
	470
	471	inline fn setRexWRegister(reg: Register) bool {
	472	if (reg.size() == 64) return true;
	473	return switch (reg) {
	474	.ah, .bh, .ch, .dh => true,
	475	else => false,
	476	};
	477	}
	478
	479	inline fn immOpSize(imm: i64) u8 {
	480	blk: {
	481	_ = math.cast(i8, imm) catch break :blk;
	482	return 8;
	483	}
	484	blk: {
	485	_ = math.cast(i16, imm) catch break :blk;
	486	return 16;
	487	}
	488	blk: {
	489	_ = math.cast(i32, imm) catch break :blk;
	490	return 32;
	491	}
	492	return 64;
	493	}
	494
	495	// TODO
	496	fn mirArithScaleSrc(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	497	const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
	498	const scale = ops.flags;
	499	// OP reg1, [reg2 + scale*rcx + imm32]
	500	const opc = getOpCode(tag, .rm, ops.reg1.size() == 8).?;
	501	const imm = isel.mir.instructions.items(.data)[inst].imm;
	502	const encoder = try Encoder.init(isel.code, 8);
	503	encoder.rex(.{
	504	.w = ops.reg1.size() == 64,
	505	.r = ops.reg1.isExtended(),
	506	.b = ops.reg2.isExtended(),
	507	});
	508	opc.encode(encoder);
	509	if (imm <= math.maxInt(i8)) {
	510	encoder.modRm_SIBDisp8(ops.reg1.lowId());
	511	encoder.sib_scaleIndexBaseDisp8(scale, Register.rcx.lowId(), ops.reg2.lowId());
	512	encoder.disp8(@intCast(i8, imm));
	513	} else {
	514	encoder.modRm_SIBDisp32(ops.reg1.lowId());
	515	encoder.sib_scaleIndexBaseDisp32(scale, Register.rcx.lowId(), ops.reg2.lowId());
	516	encoder.disp32(imm);
	517	}
	518	}
	519
	520	// TODO
	521	fn mirArithScaleDst(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	522	const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
	523	const scale = ops.flags;
	524	const imm = isel.mir.instructions.items(.data)[inst].imm;
	525
	526	if (ops.reg2 == .none) {
	527	// OP [reg1 + scale*rax + 0], imm32
	528	const opc = getOpCode(tag, .mi, ops.reg1.size() == 8).?;
	529	const modrm_ext = getModRmExt(tag).?;
	530	const encoder = try Encoder.init(isel.code, 8);
	531	encoder.rex(.{
	532	.w = ops.reg1.size() == 64,
	533	.b = ops.reg1.isExtended(),
	534	});
	535	opc.encode(encoder);
	536	encoder.modRm_SIBDisp0(modrm_ext);
	537	encoder.sib_scaleIndexBase(scale, Register.rax.lowId(), ops.reg1.lowId());
	538	if (imm <= math.maxInt(i8)) {
	539	encoder.imm8(@intCast(i8, imm));
	540	} else if (imm <= math.maxInt(i16)) {
	541	encoder.imm16(@intCast(i16, imm));
	542	} else {
	543	encoder.imm32(imm);
	544	}
	545	return;
	546	}
	547
	548	// OP [reg1 + scale*rax + imm32], reg2
	549	const opc = getOpCode(tag, .mr, ops.reg1.size() == 8).?;
	550	const encoder = try Encoder.init(isel.code, 8);
	551	encoder.rex(.{
	552	.w = ops.reg1.size() == 64,
	553	.r = ops.reg2.isExtended(),
	554	.b = ops.reg1.isExtended(),
	555	});
	556	opc.encode(encoder);
	557	if (imm <= math.maxInt(i8)) {
	558	encoder.modRm_SIBDisp8(ops.reg2.lowId());
	559	encoder.sib_scaleIndexBaseDisp8(scale, Register.rax.lowId(), ops.reg1.lowId());
	560	encoder.disp8(@intCast(i8, imm));
	561	} else {
	562	encoder.modRm_SIBDisp32(ops.reg2.lowId());
	563	encoder.sib_scaleIndexBaseDisp32(scale, Register.rax.lowId(), ops.reg1.lowId());
	564	encoder.disp32(imm);
	565	}
	566	}
	567
	568	// TODO
	569	fn mirArithScaleImm(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
	570	const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
	571	const scale = ops.flags;
	572	const payload = isel.mir.instructions.items(.data)[inst].payload;
	573	const imm_pair = isel.mir.extraData(Mir.ImmPair, payload).data;
	574	const opc = getOpCode(tag, .mi, ops.reg1.size() == 8).?;
	575	const modrm_ext = getModRmExt(tag).?;
	576	const encoder = try Encoder.init(isel.code, 2);
	577	encoder.rex(.{
	578	.w = ops.reg1.size() == 64,
	579	.b = ops.reg1.isExtended(),
	580	});
	581	opc.encode(encoder);
	582	if (imm_pair.dest_off <= math.maxInt(i8)) {
	583	encoder.modRm_SIBDisp8(modrm_ext);
	584	encoder.sib_scaleIndexBaseDisp8(scale, Register.rax.lowId(), ops.reg1.lowId());
	585	encoder.disp8(@intCast(i8, imm_pair.dest_off));
	586	} else {
	587	encoder.modRm_SIBDisp32(modrm_ext);
	588	encoder.sib_scaleIndexBaseDisp32(scale, Register.rax.lowId(), ops.reg1.lowId());
	589	encoder.disp32(imm_pair.dest_off);
	590	}
	591	encoder.imm32(imm_pair.operand);
	592	}
	593
	594	fn mirMovabs(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
	595	const tag = isel.mir.instructions.items(.tag)[inst];
	596	assert(tag == .movabs);
	597	const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
	598	const imm: i64 = if (ops.reg1.size() == 64) blk: {
	599	const payload = isel.mir.instructions.items(.data)[inst].payload;
	600	const imm = isel.mir.extraData(Mir.Imm64, payload).data;
	601	break :blk @bitCast(i64, imm.decode());
	602	} else isel.mir.instructions.items(.data)[inst].imm;
	603	if (ops.flags == 0b00) {
	604	// movabs reg, imm64
	605	// OI
	606	return lowerToOiEnc(.mov, ops.reg1, imm, isel.code) catch \|err\| isel.failWithLoweringError(err);
	607	}
	608	if (ops.reg1 == .none) {
	609	// movabs moffs64, rax
	610	// TD
	611	return lowerToTdEnc(.mov, imm, ops.reg2, isel.code) catch \|err\| isel.failWithLoweringError(err);
	612	}
	613	// movabs rax, moffs64
	614	// FD
	615	return lowerToFdEnc(.mov, ops.reg1, imm, isel.code) catch \|err\| isel.failWithLoweringError(err);
	616	}
	617
	618	fn mirIMulComplex(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
	619	const tag = isel.mir.instructions.items(.tag)[inst];
	620	assert(tag == .imul_complex);
	621	const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
	622	switch (ops.flags) {
	623	0b00 => {
	624	return lowerToRmEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), isel.code) catch \|err\|
	625	isel.failWithLoweringError(err);
	626	},
	627	0b10 => {
	628	const imm = isel.mir.instructions.items(.data)[inst].imm;
	629	return lowerToRmiEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), imm, isel.code) catch \|err\|
	630	isel.failWithLoweringError(err);
	631	},
	632	else => return isel.fail("TODO implement imul", .{}),
	633	}
	634	}
	635
	636	fn mirLea(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
	637	const tag = isel.mir.instructions.items(.tag)[inst];
	638	assert(tag == .lea);
	639	const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
	640	switch (ops.flags) {
	641	0b00 => {
	642	// lea reg1, [reg2 + imm32]
	643	// RM
	644	const imm = isel.mir.instructions.items(.data)[inst].imm;
	645	const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
	646	return lowerToRmEnc(
	647	.lea,
	648	ops.reg1,
	649	RegisterOrMemory.mem(src_reg, imm, Memory.PtrSize.fromBits(ops.reg1.size())),
	650	isel.code,
	651	) catch \|err\| isel.failWithLoweringError(err);
	652	},
	653	0b01 => {
	654	// lea reg1, [rip + imm32]
	655	// RM
	656	const start_offset = isel.code.items.len;
	657	lowerToRmEnc(
	658	.lea,
	659	ops.reg1,
	660	RegisterOrMemory.rip(0, Memory.PtrSize.fromBits(ops.reg1.size())),
	661	isel.code,
	662	) catch \|err\| return isel.failWithLoweringError(err);
	663	const end_offset = isel.code.items.len;
	664	// Backpatch the displacement
	665	const payload = isel.mir.instructions.items(.data)[inst].payload;
	666	const imm = isel.mir.extraData(Mir.Imm64, payload).data.decode();
	667	const disp = @intCast(i32, @intCast(i64, imm) - @intCast(i64, end_offset - start_offset));
	668	mem.writeIntLittle(i32, isel.code.items[end_offset - 4 ..][0..4], disp);
	669	},
	670	0b10 => {
	671	// lea reg1, [rip + reloc]
	672	// RM
	673	lowerToRmEnc(
	674	.lea,
	675	ops.reg1,
	676	RegisterOrMemory.rip(0, Memory.PtrSize.fromBits(ops.reg1.size())),
	677	isel.code,
	678	) catch \|err\| return isel.failWithLoweringError(err);
	679	const end_offset = isel.code.items.len;
	680	const got_entry = isel.mir.instructions.items(.data)[inst].got_entry;
	681	if (isel.bin_file.cast(link.File.MachO)) \|macho_file\| {
	682	// TODO I think the reloc might be in the wrong place.
	683	const decl = macho_file.active_decl.?;
	684	try decl.link.macho.relocs.append(isel.bin_file.allocator, .{
	685	.offset = @intCast(u32, end_offset - 4),
	686	.target = .{ .local = got_entry },
	687	.addend = 0,
	688	.subtractor = null,
	689	.pcrel = true,
	690	.length = 2,
	691	.@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_GOT),
	692	});
	693	} else {
	694	return isel.fail(
	695	"TODO implement lea reg, [rip + reloc] for linking backends different than MachO",
	696	.{},
	697	);
	698	}
	699	},
	700	0b11 => return isel.fail("TODO unused variant lea reg1, reg2, 0b11", .{}),
	701	}
	702	}
	703
	704	fn mirCallExtern(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
	705	const tag = isel.mir.instructions.items(.tag)[inst];
	706	assert(tag == .call_extern);
	707	const n_strx = isel.mir.instructions.items(.data)[inst].extern_fn;
	708	const offset = blk: {
	709	// callq
	710	lowerToDEnc(.call_near, 0, isel.code) catch \|err\|
	711	return isel.failWithLoweringError(err);
	712	break :blk @intCast(u32, isel.code.items.len) - 4;
	713	};
	714	if (isel.bin_file.cast(link.File.MachO)) \|macho_file\| {
	715	// Add relocation to the decl.
	716	try macho_file.active_decl.?.link.macho.relocs.append(isel.bin_file.allocator, .{
	717	.offset = offset,
	718	.target = .{ .global = n_strx },
	719	.addend = 0,
	720	.subtractor = null,
	721	.pcrel = true,
	722	.length = 2,
	723	.@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_BRANCH),
	724	});
	725	} else {
	726	return isel.fail("TODO implement call_extern for linking backends different than MachO", .{});
	727	}
	728	}
	729
	730	fn mirDbgLine(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
	731	const tag = isel.mir.instructions.items(.tag)[inst];
	732	assert(tag == .dbg_line);
	733	const payload = isel.mir.instructions.items(.data)[inst].payload;
	734	const dbg_line_column = isel.mir.extraData(Mir.DbgLineColumn, payload).data;
	735	try isel.dbgAdvancePCAndLine(dbg_line_column.line, dbg_line_column.column);
	736	}
	737
	738	fn dbgAdvancePCAndLine(isel: *Isel, line: u32, column: u32) InnerError!void {
	739	const delta_line = @intCast(i32, line) - @intCast(i32, isel.prev_di_line);
	740	const delta_pc: usize = isel.code.items.len - isel.prev_di_pc;
	741	switch (isel.debug_output) {
	742	.dwarf => \|dbg_out\| {
	743	// TODO Look into using the DWARF special opcodes to compress this data.
	744	// It lets you emit single-byte opcodes that add different numbers to
	745	// both the PC and the line number at the same time.
	746	try dbg_out.dbg_line.ensureUnusedCapacity(11);
	747	dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_pc);
	748	leb128.writeULEB128(dbg_out.dbg_line.writer(), delta_pc) catch unreachable;
	749	if (delta_line != 0) {
	750	dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_line);
	751	leb128.writeILEB128(dbg_out.dbg_line.writer(), delta_line) catch unreachable;
	752	}
	753	dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.copy);
	754	isel.prev_di_pc = isel.code.items.len;
	755	isel.prev_di_line = line;
	756	isel.prev_di_column = column;
	757	isel.prev_di_pc = isel.code.items.len;
	758	},
	759	.plan9 => \|dbg_out\| {
	760	if (delta_pc <= 0) return; // only do this when the pc changes
	761	// we have already checked the target in the linker to make sure it is compatable
	762	const quant = @import("../../link/Plan9/aout.zig").getPCQuant(isel.target.cpu.arch) catch unreachable;
	763
	764	// increasing the line number
	765	try @import("../../link/Plan9.zig").changeLine(dbg_out.dbg_line, delta_line);
	766	// increasing the pc
	767	const d_pc_p9 = @intCast(i64, delta_pc) - quant;
	768	if (d_pc_p9 > 0) {
	769	// minus one because if its the last one, we want to leave space to change the line which is one quanta
	770	try dbg_out.dbg_line.append(@intCast(u8, @divExact(d_pc_p9, quant) + 128) - quant);
	771	if (dbg_out.pcop_change_index.*) \|pci\|
	772	dbg_out.dbg_line.items[pci] += 1;
	773	dbg_out.pcop_change_index.* = @intCast(u32, dbg_out.dbg_line.items.len - 1);
	774	} else if (d_pc_p9 == 0) {
	775	// we don't need to do anything, because adding the quant does it for us
	776	} else unreachable;
	777	if (dbg_out.start_line.* == null)
	778	dbg_out.start_line.* = isel.prev_di_line;
	779	dbg_out.end_line.* = line;
	780	// only do this if the pc changed
	781	isel.prev_di_line = line;
	782	isel.prev_di_column = column;
	783	isel.prev_di_pc = isel.code.items.len;
	784	},
	785	.none => {},
	786	}
	787	}
	788
	789	fn mirDbgPrologueEnd(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
	790	const tag = isel.mir.instructions.items(.tag)[inst];
	791	assert(tag == .dbg_prologue_end);
	792	switch (isel.debug_output) {
	793	.dwarf => \|dbg_out\| {
	794	try dbg_out.dbg_line.append(DW.LNS.set_prologue_end);
	795	try isel.dbgAdvancePCAndLine(isel.prev_di_line, isel.prev_di_column);
	796	},
	797	.plan9 => {},
	798	.none => {},
	799	}
	800	}
	801
	802	fn mirDbgEpilogueBegin(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
	803	const tag = isel.mir.instructions.items(.tag)[inst];
	804	assert(tag == .dbg_epilogue_begin);
	805	switch (isel.debug_output) {
	806	.dwarf => \|dbg_out\| {
	807	try dbg_out.dbg_line.append(DW.LNS.set_epilogue_begin);
	808	try isel.dbgAdvancePCAndLine(isel.prev_di_line, isel.prev_di_column);
	809	},
	810	.plan9 => {},
	811	.none => {},
	812	}
	813	}
	814
	815	fn mirArgDbgInfo(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
	816	const tag = isel.mir.instructions.items(.tag)[inst];
	817	assert(tag == .arg_dbg_info);
	818	const payload = isel.mir.instructions.items(.data)[inst].payload;
	819	const arg_dbg_info = isel.mir.extraData(Mir.ArgDbgInfo, payload).data;
	820	const mcv = isel.mir.function.args[arg_dbg_info.arg_index];
	821	try isel.genArgDbgInfo(arg_dbg_info.air_inst, mcv);
	822	}
	823
	824	fn genArgDbgInfo(isel: *Isel, inst: Air.Inst.Index, mcv: MCValue) !void {
	825	const ty_str = isel.mir.function.air.instructions.items(.data)[inst].ty_str;
	826	const zir = &isel.mir.function.mod_fn.owner_decl.getFileScope().zir;
	827	const name = zir.nullTerminatedString(ty_str.str);
	828	const name_with_null = name.ptr[0 .. name.len + 1];
	829	const ty = isel.mir.function.air.getRefType(ty_str.ty);
	830
	831	switch (mcv) {
	832	.register => \|reg\| {
	833	switch (isel.debug_output) {
	834	.dwarf => \|dbg_out\| {
	835	try dbg_out.dbg_info.ensureUnusedCapacity(3);
	836	dbg_out.dbg_info.appendAssumeCapacity(link.File.Elf.abbrev_parameter);
	837	dbg_out.dbg_info.appendSliceAssumeCapacity(&[2]u8{ // DW.AT.location, DW.FORM.exprloc
	838	1, // ULEB128 dwarf expression length
	839	reg.dwarfLocOp(),
	840	});
	841	try dbg_out.dbg_info.ensureUnusedCapacity(5 + name_with_null.len);
	842	try isel.addDbgInfoTypeReloc(ty); // DW.AT.type, DW.FORM.ref4
	843	dbg_out.dbg_info.appendSliceAssumeCapacity(name_with_null); // DW.AT.name, DW.FORM.string
	844	},
	845	.plan9 => {},
	846	.none => {},
	847	}
	848	},
	849	.stack_offset => {
	850	switch (isel.debug_output) {
	851	.dwarf => {},
	852	.plan9 => {},
	853	.none => {},
	854	}
	855	},
	856	else => {},
	857	}
	858	}
	859
	860	/// Adds a Type to the .debug_info at the current position. The bytes will be populated later,
	861	/// after codegen for this symbol is done.
	862	fn addDbgInfoTypeReloc(isel: *Isel, ty: Type) !void {
	863	switch (isel.debug_output) {
	864	.dwarf => \|dbg_out\| {
	865	assert(ty.hasCodeGenBits());
	866	const index = dbg_out.dbg_info.items.len;
	867	try dbg_out.dbg_info.resize(index + 4); // DW.AT.type, DW.FORM.ref4
	868
	869	const gop = try dbg_out.dbg_info_type_relocs.getOrPut(isel.bin_file.allocator, ty);
	870	if (!gop.found_existing) {
	871	gop.value_ptr.* = .{
	872	.off = undefined,
	873	.relocs = .{},
	874	};
	875	}
	876	try gop.value_ptr.relocs.append(isel.bin_file.allocator, @intCast(u32, index));
	877	},
	878	.plan9 => {},
	879	.none => {},
	880	}
	881	}
	882
	883	const Tag = enum {
	884	adc,
	885	add,
	886	sub,
	887	xor,
	888	@"and",
	889	@"or",
	890	sbb,
	891	cmp,
	892	mov,
	893	lea,
	894	jmp_near,
	895	call_near,
	896	push,
	897	pop,
	898	@"test",
	899	brk,
	900	nop,
	901	imul,
	902	syscall,
	903	ret_near,
	904	ret_far,
	905	jo,
	906	jno,
	907	jb,
	908	jbe,
	909	jc,
	910	jnae,
	911	jnc,
	912	jae,
	913	je,
	914	jz,
	915	jne,
	916	jnz,
	917	jna,
	918	jnb,
	919	jnbe,
	920	ja,
	921	js,
	922	jns,
	923	jpe,
	924	jp,
	925	jpo,
	926	jnp,
	927	jnge,
	928	jl,
	929	jge,
	930	jnl,
	931	jle,
	932	jng,
	933	jg,
	934	jnle,
	935	seto,
	936	setno,
	937	setb,
	938	setc,
	939	setnae,
	940	setnb,
	941	setnc,
	942	setae,
	943	sete,
	944	setz,
	945	setne,
	946	setnz,
	947	setbe,
	948	setna,
	949	seta,
	950	setnbe,
	951	sets,
	952	setns,
	953	setp,
	954	setpe,
	955	setnp,
	956	setop,
	957	setl,
	958	setnge,
	959	setnl,
	960	setge,
	961	setle,
	962	setng,
	963	setnle,
	964	setg,
	965
	966	fn isSetCC(tag: Tag) bool {
	967	return switch (tag) {
	968	.seto,
	969	.setno,
	970	.setb,
	971	.setc,
	972	.setnae,
	973	.setnb,
	974	.setnc,
	975	.setae,
	976	.sete,
	977	.setz,
	978	.setne,
	979	.setnz,
	980	.setbe,
	981	.setna,
	982	.seta,
	983	.setnbe,
	984	.sets,
	985	.setns,
	986	.setp,
	987	.setpe,
	988	.setnp,
	989	.setop,
	990	.setl,
	991	.setnge,
	992	.setnl,
	993	.setge,
	994	.setle,
	995	.setng,
	996	.setnle,
	997	.setg,
	998	=> true,
	999	else => false,
	1000	};
	1001	}
	1002	};
	1003
	1004	const Encoding = enum {
	1005	/// OP
	1006	zo,
	1007
	1008	/// OP rel32
	1009	d,
	1010
	1011	/// OP r/m64
	1012	m,
	1013
	1014	/// OP r64
	1015	o,
	1016
	1017	/// OP imm32
	1018	i,
	1019
	1020	/// OP r/m64, imm32
	1021	mi,
	1022
	1023	/// OP r/m64, r64
	1024	mr,
	1025
	1026	/// OP r64, r/m64
	1027	rm,
	1028
	1029	/// OP r64, imm64
	1030	oi,
	1031
	1032	/// OP al/ax/eax/rax, moffs
	1033	fd,
	1034
	1035	/// OP moffs, al/ax/eax/rax
	1036	td,
	1037
	1038	/// OP r64, r/m64, imm32
	1039	rmi,
	1040	};
	1041
	1042	const OpCode = union(enum) {
	1043	one_byte: u8,
	1044	two_byte: struct { _1: u8, _2: u8 },
	1045
	1046	fn oneByte(opc: u8) OpCode {
	1047	return .{ .one_byte = opc };
	1048	}
	1049
	1050	fn twoByte(opc1: u8, opc2: u8) OpCode {
	1051	return .{ .two_byte = .{ ._1 = opc1, ._2 = opc2 } };
	1052	}
	1053
	1054	fn encode(opc: OpCode, encoder: Encoder) void {
	1055	switch (opc) {
	1056	.one_byte => \|v\| encoder.opcode_1byte(v),
	1057	.two_byte => \|v\| encoder.opcode_2byte(v._1, v._2),
	1058	}
	1059	}
	1060
	1061	fn encodeWithReg(opc: OpCode, encoder: Encoder, reg: Register) void {
	1062	assert(opc == .one_byte);
	1063	encoder.opcode_withReg(opc.one_byte, reg.lowId());
	1064	}
	1065	};
	1066
	1067	inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode {
	1068	switch (enc) {
	1069	.zo => return switch (tag) {
	1070	.ret_near => OpCode.oneByte(0xc3),
	1071	.ret_far => OpCode.oneByte(0xcb),
	1072	.brk => OpCode.oneByte(0xcc),
	1073	.nop => OpCode.oneByte(0x90),
	1074	.syscall => OpCode.twoByte(0x0f, 0x05),
	1075	else => null,
	1076	},
	1077	.d => return switch (tag) {
	1078	.jmp_near => OpCode.oneByte(0xe9),
	1079	.call_near => OpCode.oneByte(0xe8),
	1080	.jo => if (is_one_byte) OpCode.oneByte(0x70) else OpCode.twoByte(0x0f, 0x80),
	1081	.jno => if (is_one_byte) OpCode.oneByte(0x71) else OpCode.twoByte(0x0f, 0x81),
	1082	.jb, .jc, .jnae => if (is_one_byte) OpCode.oneByte(0x72) else OpCode.twoByte(0x0f, 0x82),
	1083	.jnb, .jnc, .jae => if (is_one_byte) OpCode.oneByte(0x73) else OpCode.twoByte(0x0f, 0x83),
	1084	.je, .jz => if (is_one_byte) OpCode.oneByte(0x74) else OpCode.twoByte(0x0f, 0x84),
	1085	.jne, .jnz => if (is_one_byte) OpCode.oneByte(0x75) else OpCode.twoByte(0x0f, 0x85),
	1086	.jna, .jbe => if (is_one_byte) OpCode.oneByte(0x76) else OpCode.twoByte(0x0f, 0x86),
	1087	.jnbe, .ja => if (is_one_byte) OpCode.oneByte(0x77) else OpCode.twoByte(0x0f, 0x87),
	1088	.js => if (is_one_byte) OpCode.oneByte(0x78) else OpCode.twoByte(0x0f, 0x88),
	1089	.jns => if (is_one_byte) OpCode.oneByte(0x79) else OpCode.twoByte(0x0f, 0x89),
	1090	.jpe, .jp => if (is_one_byte) OpCode.oneByte(0x7a) else OpCode.twoByte(0x0f, 0x8a),
	1091	.jpo, .jnp => if (is_one_byte) OpCode.oneByte(0x7b) else OpCode.twoByte(0x0f, 0x8b),
	1092	.jnge, .jl => if (is_one_byte) OpCode.oneByte(0x7c) else OpCode.twoByte(0x0f, 0x8c),
	1093	.jge, .jnl => if (is_one_byte) OpCode.oneByte(0x7d) else OpCode.twoByte(0x0f, 0x8d),
	1094	.jle, .jng => if (is_one_byte) OpCode.oneByte(0x7e) else OpCode.twoByte(0x0f, 0x8e),
	1095	.jg, .jnle => if (is_one_byte) OpCode.oneByte(0x7f) else OpCode.twoByte(0x0f, 0x8f),
	1096	else => null,
	1097	},
	1098	.m => return switch (tag) {
	1099	.jmp_near, .call_near, .push => OpCode.oneByte(0xff),
	1100	.pop => OpCode.oneByte(0x8f),
	1101	.seto => OpCode.twoByte(0x0f, 0x90),
	1102	.setno => OpCode.twoByte(0x0f, 0x91),
	1103	.setb, .setc, .setnae => OpCode.twoByte(0x0f, 0x92),
	1104	.setnb, .setnc, .setae => OpCode.twoByte(0x0f, 0x93),
	1105	.sete, .setz => OpCode.twoByte(0x0f, 0x94),
	1106	.setne, .setnz => OpCode.twoByte(0x0f, 0x95),
	1107	.setbe, .setna => OpCode.twoByte(0x0f, 0x96),
	1108	.seta, .setnbe => OpCode.twoByte(0x0f, 0x97),
	1109	.sets => OpCode.twoByte(0x0f, 0x98),
	1110	.setns => OpCode.twoByte(0x0f, 0x99),
	1111	.setp, .setpe => OpCode.twoByte(0x0f, 0x9a),
	1112	.setnp, .setop => OpCode.twoByte(0x0f, 0x9b),
	1113	.setl, .setnge => OpCode.twoByte(0x0f, 0x9c),
	1114	.setnl, .setge => OpCode.twoByte(0x0f, 0x9d),
	1115	.setle, .setng => OpCode.twoByte(0x0f, 0x9e),
	1116	.setnle, .setg => OpCode.twoByte(0x0f, 0x9f),
	1117	else => null,
	1118	},
	1119	.o => return switch (tag) {
	1120	.push => OpCode.oneByte(0x50),
	1121	.pop => OpCode.oneByte(0x58),
	1122	else => null,
	1123	},
	1124	.i => return switch (tag) {
	1125	.push => OpCode.oneByte(if (is_one_byte) 0x6a else 0x68),
	1126	.@"test" => OpCode.oneByte(if (is_one_byte) 0xa8 else 0xa9),
	1127	.ret_near => OpCode.oneByte(0xc2),
	1128	.ret_far => OpCode.oneByte(0xca),
	1129	else => null,
	1130	},
	1131	.mi => return switch (tag) {
	1132	.adc, .add, .sub, .xor, .@"and", .@"or", .sbb, .cmp => OpCode.oneByte(if (is_one_byte) 0x80 else 0x81),
	1133	.mov => OpCode.oneByte(if (is_one_byte) 0xc6 else 0xc7),
	1134	.@"test" => OpCode.oneByte(if (is_one_byte) 0xf6 else 0xf7),
	1135	else => null,
	1136	},
	1137	.mr => return switch (tag) {
	1138	.adc => OpCode.oneByte(if (is_one_byte) 0x10 else 0x11),
	1139	.add => OpCode.oneByte(if (is_one_byte) 0x00 else 0x01),
	1140	.sub => OpCode.oneByte(if (is_one_byte) 0x28 else 0x29),
	1141	.xor => OpCode.oneByte(if (is_one_byte) 0x30 else 0x31),
	1142	.@"and" => OpCode.oneByte(if (is_one_byte) 0x20 else 0x21),
	1143	.@"or" => OpCode.oneByte(if (is_one_byte) 0x08 else 0x09),
	1144	.sbb => OpCode.oneByte(if (is_one_byte) 0x18 else 0x19),
	1145	.cmp => OpCode.oneByte(if (is_one_byte) 0x38 else 0x39),
	1146	.mov => OpCode.oneByte(if (is_one_byte) 0x88 else 0x89),
	1147	else => null,
	1148	},
	1149	.rm => return switch (tag) {
	1150	.adc => OpCode.oneByte(if (is_one_byte) 0x12 else 0x13),
	1151	.add => OpCode.oneByte(if (is_one_byte) 0x02 else 0x03),
	1152	.sub => OpCode.oneByte(if (is_one_byte) 0x2a else 0x2b),
	1153	.xor => OpCode.oneByte(if (is_one_byte) 0x32 else 0x33),
	1154	.@"and" => OpCode.oneByte(if (is_one_byte) 0x22 else 0x23),
	1155	.@"or" => OpCode.oneByte(if (is_one_byte) 0x0b else 0x0b),
	1156	.sbb => OpCode.oneByte(if (is_one_byte) 0x1a else 0x1b),
	1157	.cmp => OpCode.oneByte(if (is_one_byte) 0x3a else 0x3b),
	1158	.mov => OpCode.oneByte(if (is_one_byte) 0x8a else 0x8b),
	1159	.lea => OpCode.oneByte(if (is_one_byte) 0x8c else 0x8d),
	1160	.imul => OpCode.twoByte(0x0f, 0xaf),
	1161	else => null,
	1162	},
	1163	.oi => return switch (tag) {
	1164	.mov => OpCode.oneByte(if (is_one_byte) 0xb0 else 0xb8),
	1165	else => null,
	1166	},
	1167	.fd => return switch (tag) {
	1168	.mov => OpCode.oneByte(if (is_one_byte) 0xa0 else 0xa1),
	1169	else => null,
	1170	},
	1171	.td => return switch (tag) {
	1172	.mov => OpCode.oneByte(if (is_one_byte) 0xa2 else 0xa3),
	1173	else => null,
	1174	},
	1175	.rmi => return switch (tag) {
	1176	.imul => OpCode.oneByte(if (is_one_byte) 0x6b else 0x69),
	1177	else => null,
	1178	},
	1179	}
	1180	}
	1181
	1182	inline fn getModRmExt(tag: Tag) ?u3 {
	1183	return switch (tag) {
	1184	.adc => 0x2,
	1185	.add => 0x0,
	1186	.sub => 0x5,
	1187	.xor => 0x6,
	1188	.@"and" => 0x4,
	1189	.@"or" => 0x1,
	1190	.sbb => 0x3,
	1191	.cmp => 0x7,
	1192	.mov => 0x0,
	1193	.jmp_near => 0x4,
	1194	.call_near => 0x2,
	1195	.push => 0x6,
	1196	.pop => 0x0,
	1197	.@"test" => 0x0,
	1198	.seto,
	1199	.setno,
	1200	.setb,
	1201	.setc,
	1202	.setnae,
	1203	.setnb,
	1204	.setnc,
	1205	.setae,
	1206	.sete,
	1207	.setz,
	1208	.setne,
	1209	.setnz,
	1210	.setbe,
	1211	.setna,
	1212	.seta,
	1213	.setnbe,
	1214	.sets,
	1215	.setns,
	1216	.setp,
	1217	.setpe,
	1218	.setnp,
	1219	.setop,
	1220	.setl,
	1221	.setnge,
	1222	.setnl,
	1223	.setge,
	1224	.setle,
	1225	.setng,
	1226	.setnle,
	1227	.setg,
	1228	=> 0x0,
	1229	else => null,
	1230	};
	1231	}
	1232
	1233	const ScaleIndexBase = struct {
	1234	scale: u2,
	1235	index_reg: ?Register,
	1236	base_reg: ?Register,
	1237	};
	1238
	1239	const Memory = struct {
	1240	reg: ?Register,
	1241	rip: bool = false,
	1242	disp: i32,
	1243	ptr_size: PtrSize,
	1244	sib: ?ScaleIndexBase = null,
	1245
	1246	const PtrSize = enum {
	1247	byte_ptr,
	1248	word_ptr,
	1249	dword_ptr,
	1250	qword_ptr,
	1251
	1252	fn fromBits(in_bits: u64) PtrSize {
	1253	return switch (in_bits) {
	1254	8 => .byte_ptr,
	1255	16 => .word_ptr,
	1256	32 => .dword_ptr,
	1257	64 => .qword_ptr,
	1258	else => unreachable,
	1259	};
	1260	}
	1261
	1262	/// Returns size in bits.
	1263	fn size(ptr_size: PtrSize) u64 {
	1264	return switch (ptr_size) {
	1265	.byte_ptr => 8,
	1266	.word_ptr => 16,
	1267	.dword_ptr => 32,
	1268	.qword_ptr => 64,
	1269	};
	1270	}
	1271	};
	1272	};
	1273
	1274	const RegisterOrMemory = union(enum) {
	1275	register: Register,
	1276	memory: Memory,
	1277
	1278	fn reg(register: Register) RegisterOrMemory {
	1279	return .{ .register = register };
	1280	}
	1281
	1282	fn mem(register: ?Register, disp: i32, ptr_size: Memory.PtrSize) RegisterOrMemory {
	1283	return .{
	1284	.memory = .{
	1285	.reg = register,
	1286	.disp = disp,
	1287	.ptr_size = ptr_size,
	1288	},
	1289	};
	1290	}
	1291
	1292	fn rip(disp: i32, ptr_size: Memory.PtrSize) RegisterOrMemory {
	1293	return .{
	1294	.memory = .{
	1295	.reg = null,
	1296	.rip = true,
	1297	.disp = disp,
	1298	.ptr_size = ptr_size,
	1299	},
	1300	};
	1301	}
	1302	};
	1303
	1304	const LoweringError = error{
	1305	OutOfMemory,
	1306	Overflow,
	1307	OperandSizeMismatch,
	1308	RaxOperandExpected,
	1309	};
	1310
	1311	fn lowerToZoEnc(tag: Tag, code: *std.ArrayList(u8)) LoweringError!void {
	1312	const opc = getOpCode(tag, .zo, false).?;
	1313	const encoder = try Encoder.init(code, 1);
	1314	opc.encode(encoder);
	1315	}
	1316
	1317	fn lowerToIEnc(tag: Tag, imm: i32, code: *std.ArrayList(u8)) LoweringError!void {
	1318	if (tag == .ret_far or tag == .ret_near) {
	1319	const encoder = try Encoder.init(code, 3);
	1320	const opc = getOpCode(tag, .i, false).?;
	1321	opc.encode(encoder);
	1322	encoder.imm16(@intCast(i16, imm));
	1323	return;
	1324	}
	1325	const opc = getOpCode(tag, .i, immOpSize(imm) == 8).?;
	1326	const encoder = try Encoder.init(code, 5);
	1327	if (immOpSize(imm) == 16) {
	1328	encoder.opcode_1byte(0x66);
	1329	}
	1330	opc.encode(encoder);
	1331	if (immOpSize(imm) == 8) {
	1332	encoder.imm8(@intCast(i8, imm));
	1333	} else if (immOpSize(imm) == 16) {
	1334	encoder.imm16(@intCast(i16, imm));
	1335	} else {
	1336	encoder.imm32(imm);
	1337	}
	1338	}
	1339
	1340	fn lowerToOEnc(tag: Tag, reg: Register, code: *std.ArrayList(u8)) LoweringError!void {
	1341	if (reg.size() != 16 and reg.size() != 64) {
	1342	return error.OperandSizeMismatch; // TODO correct for push/pop, but is it universal?
	1343	}
	1344	const opc = getOpCode(tag, .o, false).?;
	1345	const encoder = try Encoder.init(code, 3);
	1346	if (reg.size() == 16) {
	1347	encoder.opcode_1byte(0x66);
	1348	}
	1349	encoder.rex(.{
	1350	.w = false,
	1351	.b = reg.isExtended(),
	1352	});
	1353	opc.encodeWithReg(encoder, reg);
	1354	}
	1355
	1356	fn lowerToDEnc(tag: Tag, imm: i32, code: *std.ArrayList(u8)) LoweringError!void {
	1357	const opc = getOpCode(tag, .d, false).?;
	1358	const encoder = try Encoder.init(code, 6);
	1359	opc.encode(encoder);
	1360	encoder.imm32(imm);
	1361	}
	1362
	1363	fn lowerToMEnc(tag: Tag, reg_or_mem: RegisterOrMemory, code: *std.ArrayList(u8)) LoweringError!void {
	1364	const opc = getOpCode(tag, .m, false).?;
	1365	const modrm_ext = getModRmExt(tag).?;
	1366	switch (reg_or_mem) {
	1367	.register => \|reg\| {
	1368	const op_size_mismatch = blk: {
	1369	if (tag.isSetCC() and reg.size() == 8)
	1370	break :blk false;
	1371	break :blk reg.size() != 64 and reg.size() != 16;
	1372	};
	1373	if (op_size_mismatch) {
	1374	return error.OperandSizeMismatch;
	1375	}
	1376	const encoder = try Encoder.init(code, 4);
	1377	if (reg.size() == 16) {
	1378	encoder.opcode_1byte(0x66);
	1379	}
	1380	encoder.rex(.{
	1381	.w = switch (reg) {
	1382	.ah, .bh, .ch, .dh => true,
	1383	else => false,
	1384	},
	1385	.b = reg.isExtended(),
	1386	});
	1387	opc.encode(encoder);
	1388	encoder.modRm_direct(modrm_ext, reg.lowId());
	1389	},
	1390	.memory => \|mem_op\| {
	1391	if (mem_op.ptr_size != .qword_ptr and mem_op.ptr_size != .word_ptr) {
	1392	return error.OperandSizeMismatch;
	1393	}
	1394	const encoder = try Encoder.init(code, 8);
	1395	if (mem_op.ptr_size == .word_ptr) {
	1396	encoder.opcode_1byte(0x66);
	1397	}
	1398	if (mem_op.reg) \|reg\| {
	1399	if (reg.size() != 64) {
	1400	return error.OperandSizeMismatch;
	1401	}
	1402	encoder.rex(.{
	1403	.w = false,
	1404	.b = reg.isExtended(),
	1405	});
	1406	opc.encode(encoder);
	1407	if (reg.lowId() == 4) {
	1408	if (mem_op.disp == 0) {
	1409	encoder.modRm_SIBDisp0(modrm_ext);
	1410	encoder.sib_base(reg.lowId());
	1411	} else if (immOpSize(mem_op.disp) == 8) {
	1412	encoder.modRm_SIBDisp8(modrm_ext);
	1413	encoder.sib_baseDisp8(reg.lowId());
	1414	encoder.disp8(@intCast(i8, mem_op.disp));
	1415	} else {
	1416	encoder.modRm_SIBDisp32(modrm_ext);
	1417	encoder.sib_baseDisp32(reg.lowId());
	1418	encoder.disp32(mem_op.disp);
	1419	}
	1420	} else {
	1421	if (mem_op.disp == 0) {
	1422	encoder.modRm_indirectDisp0(modrm_ext, reg.lowId());
	1423	} else if (immOpSize(mem_op.disp) == 8) {
	1424	encoder.modRm_indirectDisp8(modrm_ext, reg.lowId());
	1425	encoder.disp8(@intCast(i8, mem_op.disp));
	1426	} else {
	1427	encoder.modRm_indirectDisp32(modrm_ext, reg.lowId());
	1428	encoder.disp32(mem_op.disp);
	1429	}
	1430	}
	1431	} else {
	1432	opc.encode(encoder);
	1433	if (mem_op.rip) {
	1434	encoder.modRm_RIPDisp32(modrm_ext);
	1435	} else {
	1436	encoder.modRm_SIBDisp0(modrm_ext);
	1437	encoder.sib_disp32();
	1438	}
	1439	encoder.disp32(mem_op.disp);
	1440	}
	1441	},
	1442	}
	1443	}
	1444
	1445	fn lowerToTdEnc(tag: Tag, moffs: i64, reg: Register, code: *std.ArrayList(u8)) LoweringError!void {
	1446	return lowerToTdFdEnc(tag, reg, moffs, code, true);
	1447	}
	1448
	1449	fn lowerToFdEnc(tag: Tag, reg: Register, moffs: i64, code: *std.ArrayList(u8)) LoweringError!void {
	1450	return lowerToTdFdEnc(tag, reg, moffs, code, false);
	1451	}
	1452
	1453	fn lowerToTdFdEnc(tag: Tag, reg: Register, moffs: i64, code: *std.ArrayList(u8), td: bool) LoweringError!void {
	1454	if (reg.lowId() != Register.rax.lowId()) {
	1455	return error.RaxOperandExpected;
	1456	}
	1457	if (reg.size() != immOpSize(moffs)) {
	1458	return error.OperandSizeMismatch;
	1459	}
	1460	const opc = if (td)
	1461	getOpCode(tag, .td, reg.size() == 8).?
	1462	else
	1463	getOpCode(tag, .fd, reg.size() == 8).?;
	1464	const encoder = try Encoder.init(code, 10);
	1465	if (reg.size() == 16) {
	1466	encoder.opcode_1byte(0x66);
	1467	}
	1468	encoder.rex(.{
	1469	.w = setRexWRegister(reg),
	1470	});
	1471	opc.encode(encoder);
	1472	switch (reg.size()) {
	1473	8 => {
	1474	const moffs8 = try math.cast(i8, moffs);
	1475	encoder.imm8(moffs8);
	1476	},
	1477	16 => {
	1478	const moffs16 = try math.cast(i16, moffs);
	1479	encoder.imm16(moffs16);
	1480	},
	1481	32 => {
	1482	const moffs32 = try math.cast(i32, moffs);
	1483	encoder.imm32(moffs32);
	1484	},
	1485	64 => {
	1486	encoder.imm64(@bitCast(u64, moffs));
	1487	},
	1488	else => unreachable,
	1489	}
	1490	}
	1491
	1492	fn lowerToOiEnc(tag: Tag, reg: Register, imm: i64, code: *std.ArrayList(u8)) LoweringError!void {
	1493	if (reg.size() != immOpSize(imm)) {
	1494	return error.OperandSizeMismatch;
	1495	}
	1496	const opc = getOpCode(tag, .oi, reg.size() == 8).?;
	1497	const encoder = try Encoder.init(code, 10);
	1498	if (reg.size() == 16) {
	1499	encoder.opcode_1byte(0x66);
	1500	}
	1501	encoder.rex(.{
	1502	.w = setRexWRegister(reg),
	1503	.b = reg.isExtended(),
	1504	});
	1505	opc.encodeWithReg(encoder, reg);
	1506	switch (reg.size()) {
	1507	8 => {
	1508	const imm8 = try math.cast(i8, imm);
	1509	encoder.imm8(imm8);
	1510	},
	1511	16 => {
	1512	const imm16 = try math.cast(i16, imm);
	1513	encoder.imm16(imm16);
	1514	},
	1515	32 => {
	1516	const imm32 = try math.cast(i32, imm);
	1517	encoder.imm32(imm32);
	1518	},
	1519	64 => {
	1520	encoder.imm64(@bitCast(u64, imm));
	1521	},
	1522	else => unreachable,
	1523	}
	1524	}
	1525
	1526	fn lowerToMiEnc(tag: Tag, reg_or_mem: RegisterOrMemory, imm: i32, code: *std.ArrayList(u8)) LoweringError!void {
	1527	const modrm_ext = getModRmExt(tag).?;
	1528	switch (reg_or_mem) {
	1529	.register => \|dst_reg\| {
	1530	const opc = getOpCode(tag, .mi, dst_reg.size() == 8).?;
	1531	const encoder = try Encoder.init(code, 7);
	1532	if (dst_reg.size() == 16) {
	1533	// 0x66 prefix switches to the non-default size; here we assume a switch from
	1534	// the default 32bits to 16bits operand-size.
	1535	// More info: https://www.cs.uni-potsdam.de/desn/lehre/ss15/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf#page=32&zoom=auto,-159,773
	1536	encoder.opcode_1byte(0x66);
	1537	}
	1538	encoder.rex(.{
	1539	.w = setRexWRegister(dst_reg),
	1540	.b = dst_reg.isExtended(),
	1541	});
	1542	opc.encode(encoder);
	1543	encoder.modRm_direct(modrm_ext, dst_reg.lowId());
	1544	switch (dst_reg.size()) {
	1545	8 => {
	1546	const imm8 = try math.cast(i8, imm);
	1547	encoder.imm8(imm8);
	1548	},
	1549	16 => {
	1550	const imm16 = try math.cast(i16, imm);
	1551	encoder.imm16(imm16);
	1552	},
	1553	32, 64 => encoder.imm32(imm),
	1554	else => unreachable,
	1555	}
	1556	},
	1557	.memory => \|dst_mem\| {
	1558	const opc = getOpCode(tag, .mi, dst_mem.ptr_size == .byte_ptr).?;
	1559	const encoder = try Encoder.init(code, 12);
	1560	if (dst_mem.ptr_size == .word_ptr) {
	1561	encoder.opcode_1byte(0x66);
	1562	}
	1563	if (dst_mem.reg) \|dst_reg\| {
	1564	if (dst_reg.size() != 64) {
	1565	return error.OperandSizeMismatch;
	1566	}
	1567	encoder.rex(.{
	1568	.w = dst_mem.ptr_size == .qword_ptr,
	1569	.b = dst_reg.isExtended(),
	1570	});
	1571	opc.encode(encoder);
	1572	if (dst_reg.lowId() == 4) {
	1573	if (dst_mem.disp == 0) {
	1574	encoder.modRm_SIBDisp0(modrm_ext);
	1575	encoder.sib_base(dst_reg.lowId());
	1576	} else if (immOpSize(dst_mem.disp) == 8) {
	1577	encoder.modRm_SIBDisp8(modrm_ext);
	1578	encoder.sib_baseDisp8(dst_reg.lowId());
	1579	encoder.disp8(@intCast(i8, dst_mem.disp));
	1580	} else {
	1581	encoder.modRm_SIBDisp32(modrm_ext);
	1582	encoder.sib_baseDisp32(dst_reg.lowId());
	1583	encoder.disp32(dst_mem.disp);
	1584	}
	1585	} else {
	1586	if (dst_mem.disp == 0) {
	1587	encoder.modRm_indirectDisp0(modrm_ext, dst_reg.lowId());
	1588	} else if (immOpSize(dst_mem.disp) == 8) {
	1589	encoder.modRm_indirectDisp8(modrm_ext, dst_reg.lowId());
	1590	encoder.disp8(@intCast(i8, dst_mem.disp));
	1591	} else {
	1592	encoder.modRm_indirectDisp32(modrm_ext, dst_reg.lowId());
	1593	encoder.disp32(dst_mem.disp);
	1594	}
	1595	}
	1596	} else {
	1597	opc.encode(encoder);
	1598	if (dst_mem.rip) {
	1599	encoder.modRm_RIPDisp32(modrm_ext);
	1600	} else {
	1601	encoder.modRm_SIBDisp0(modrm_ext);
	1602	encoder.sib_disp32();
	1603	}
	1604	encoder.disp32(dst_mem.disp);
	1605	}
	1606	switch (dst_mem.ptr_size) {
	1607	.byte_ptr => {
	1608	const imm8 = try math.cast(i8, imm);
	1609	encoder.imm8(imm8);
	1610	},
	1611	.word_ptr => {
	1612	const imm16 = try math.cast(i16, imm);
	1613	encoder.imm16(imm16);
	1614	},
	1615	.dword_ptr, .qword_ptr => {
	1616	encoder.imm32(imm);
	1617	},
	1618	}
	1619	},
	1620	}
	1621	}
	1622
	1623	fn lowerToRmEnc(
	1624	tag: Tag,
	1625	reg: Register,
	1626	reg_or_mem: RegisterOrMemory,
	1627	code: *std.ArrayList(u8),
	1628	) LoweringError!void {
	1629	const opc = getOpCode(tag, .rm, reg.size() == 8).?;
	1630	switch (reg_or_mem) {
	1631	.register => \|src_reg\| {
	1632	if (reg.size() != src_reg.size()) {
	1633	return error.OperandSizeMismatch;
	1634	}
	1635	const encoder = try Encoder.init(code, 3);
	1636	encoder.rex(.{
	1637	.w = setRexWRegister(reg) or setRexWRegister(src_reg),
	1638	.r = reg.isExtended(),
	1639	.b = src_reg.isExtended(),
	1640	});
	1641	opc.encode(encoder);
	1642	encoder.modRm_direct(reg.lowId(), src_reg.lowId());
	1643	},
	1644	.memory => \|src_mem\| {
	1645	if (reg.size() != src_mem.ptr_size.size()) {
	1646	return error.OperandSizeMismatch;
	1647	}
	1648	const encoder = try Encoder.init(code, 9);
	1649	if (reg.size() == 16) {
	1650	encoder.opcode_1byte(0x66);
	1651	}
	1652	if (src_mem.reg) \|src_reg\| {
	1653	// TODO handle 32-bit base register - requires prefix 0x67
	1654	// Intel Manual, Vol 1, chapter 3.6 and 3.6.1
	1655	if (src_reg.size() != 64) {
	1656	return error.OperandSizeMismatch;
	1657	}
	1658	encoder.rex(.{
	1659	.w = setRexWRegister(reg),
	1660	.r = reg.isExtended(),
	1661	.b = src_reg.isExtended(),
	1662	});
	1663	opc.encode(encoder);
	1664	if (src_reg.lowId() == 4) {
	1665	if (src_mem.disp == 0) {
	1666	encoder.modRm_SIBDisp0(reg.lowId());
	1667	encoder.sib_base(src_reg.lowId());
	1668	} else if (immOpSize(src_mem.disp) == 8) {
	1669	encoder.modRm_SIBDisp8(reg.lowId());
	1670	encoder.sib_baseDisp8(src_reg.lowId());
	1671	encoder.disp8(@intCast(i8, src_mem.disp));
	1672	} else {
	1673	encoder.modRm_SIBDisp32(reg.lowId());
	1674	encoder.sib_baseDisp32(src_reg.lowId());
	1675	encoder.disp32(src_mem.disp);
	1676	}
	1677	} else {
	1678	if (src_mem.disp == 0) {
	1679	encoder.modRm_indirectDisp0(reg.lowId(), src_reg.lowId());
	1680	} else if (immOpSize(src_mem.disp) == 8) {
	1681	encoder.modRm_indirectDisp8(reg.lowId(), src_reg.lowId());
	1682	encoder.disp8(@intCast(i8, src_mem.disp));
	1683	} else {
	1684	encoder.modRm_indirectDisp32(reg.lowId(), src_reg.lowId());
	1685	encoder.disp32(src_mem.disp);
	1686	}
	1687	}
	1688	} else {
	1689	encoder.rex(.{
	1690	.w = setRexWRegister(reg),
	1691	.r = reg.isExtended(),
	1692	});
	1693	opc.encode(encoder);
	1694	if (src_mem.rip) {
	1695	encoder.modRm_RIPDisp32(reg.lowId());
	1696	} else {
	1697	encoder.modRm_SIBDisp0(reg.lowId());
	1698	encoder.sib_disp32();
	1699	}
	1700	encoder.disp32(src_mem.disp);
	1701	}
	1702	},
	1703	}
	1704	}
	1705
	1706	fn lowerToMrEnc(
	1707	tag: Tag,
	1708	reg_or_mem: RegisterOrMemory,
	1709	reg: Register,
	1710	code: *std.ArrayList(u8),
	1711	) LoweringError!void {
	1712	const opc = getOpCode(tag, .mr, reg.size() == 8).?;
	1713	switch (reg_or_mem) {
	1714	.register => \|dst_reg\| {
	1715	if (dst_reg.size() != reg.size()) {
	1716	return error.OperandSizeMismatch;
	1717	}
	1718	const encoder = try Encoder.init(code, 3);
	1719	encoder.rex(.{
	1720	.w = setRexWRegister(dst_reg) or setRexWRegister(reg),
	1721	.r = reg.isExtended(),
	1722	.b = dst_reg.isExtended(),
	1723	});
	1724	opc.encode(encoder);
	1725	encoder.modRm_direct(reg.lowId(), dst_reg.lowId());
	1726	},
	1727	.memory => \|dst_mem\| {
	1728	if (dst_mem.ptr_size.size() != reg.size()) {
	1729	return error.OperandSizeMismatch;
	1730	}
	1731	const encoder = try Encoder.init(code, 9);
	1732	if (reg.size() == 16) {
	1733	encoder.opcode_1byte(0x66);
	1734	}
	1735	if (dst_mem.reg) \|dst_reg\| {
	1736	if (dst_reg.size() != 64) {
	1737	return error.OperandSizeMismatch;
	1738	}
	1739	encoder.rex(.{
	1740	.w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg),
	1741	.r = reg.isExtended(),
	1742	.b = dst_reg.isExtended(),
	1743	});
	1744	opc.encode(encoder);
	1745	if (dst_reg.lowId() == 4) {
	1746	if (dst_mem.disp == 0) {
	1747	encoder.modRm_SIBDisp0(reg.lowId());
	1748	encoder.sib_base(dst_reg.lowId());
	1749	} else if (immOpSize(dst_mem.disp) == 8) {
	1750	encoder.modRm_SIBDisp8(reg.lowId());
	1751	encoder.sib_baseDisp8(dst_reg.lowId());
	1752	encoder.disp8(@intCast(i8, dst_mem.disp));
	1753	} else {
	1754	encoder.modRm_SIBDisp32(reg.lowId());
	1755	encoder.sib_baseDisp32(dst_reg.lowId());
	1756	encoder.disp32(dst_mem.disp);
	1757	}
	1758	} else {
	1759	if (dst_mem.disp == 0) {
	1760	encoder.modRm_indirectDisp0(reg.lowId(), dst_reg.lowId());
	1761	} else if (immOpSize(dst_mem.disp) == 8) {
	1762	encoder.modRm_indirectDisp8(reg.lowId(), dst_reg.lowId());
	1763	encoder.disp8(@intCast(i8, dst_mem.disp));
	1764	} else {
	1765	encoder.modRm_indirectDisp32(reg.lowId(), dst_reg.lowId());
	1766	encoder.disp32(dst_mem.disp);
	1767	}
	1768	}
	1769	} else {
	1770	encoder.rex(.{
	1771	.w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg),
	1772	.r = reg.isExtended(),
	1773	});
	1774	opc.encode(encoder);
	1775	if (dst_mem.rip) {
	1776	encoder.modRm_RIPDisp32(reg.lowId());
	1777	} else {
	1778	encoder.modRm_SIBDisp0(reg.lowId());
	1779	encoder.sib_disp32();
	1780	}
	1781	encoder.disp32(dst_mem.disp);
	1782	}
	1783	},
	1784	}
	1785	}
	1786
	1787	fn lowerToRmiEnc(
	1788	tag: Tag,
	1789	reg: Register,
	1790	reg_or_mem: RegisterOrMemory,
	1791	imm: i32,
	1792	code: *std.ArrayList(u8),
	1793	) LoweringError!void {
	1794	if (reg.size() == 8) {
	1795	return error.OperandSizeMismatch;
	1796	}
	1797	const opc = getOpCode(tag, .rmi, false).?;
	1798	const encoder = try Encoder.init(code, 13);
	1799	if (reg.size() == 16) {
	1800	encoder.opcode_1byte(0x66);
	1801	}
	1802	switch (reg_or_mem) {
	1803	.register => \|src_reg\| {
	1804	if (reg.size() != src_reg.size()) {
	1805	return error.OperandSizeMismatch;
	1806	}
	1807	encoder.rex(.{
	1808	.w = setRexWRegister(reg) or setRexWRegister(src_reg),
	1809	.r = reg.isExtended(),
	1810	.b = src_reg.isExtended(),
	1811	});
	1812	opc.encode(encoder);
	1813	encoder.modRm_direct(reg.lowId(), src_reg.lowId());
	1814	},
	1815	.memory => \|src_mem\| {
	1816	if (src_mem.reg) \|src_reg\| {
	1817	// TODO handle 32-bit base register - requires prefix 0x67
	1818	// Intel Manual, Vol 1, chapter 3.6 and 3.6.1
	1819	if (src_reg.size() != 64) {
	1820	return error.OperandSizeMismatch;
	1821	}
	1822	if (src_mem.ptr_size == .byte_ptr) {
	1823	return error.OperandSizeMismatch;
	1824	}
	1825	encoder.rex(.{
	1826	.w = setRexWRegister(reg),
	1827	.r = reg.isExtended(),
	1828	.b = src_reg.isExtended(),
	1829	});
	1830	opc.encode(encoder);
	1831	if (src_reg.lowId() == 4) {
	1832	if (src_mem.disp == 0) {
	1833	encoder.modRm_SIBDisp0(reg.lowId());
	1834	encoder.sib_base(src_reg.lowId());
	1835	} else if (immOpSize(src_mem.disp) == 8) {
	1836	encoder.modRm_SIBDisp8(reg.lowId());
	1837	encoder.sib_baseDisp8(src_reg.lowId());
	1838	encoder.disp8(@intCast(i8, src_mem.disp));
	1839	} else {
	1840	encoder.modRm_SIBDisp32(reg.lowId());
	1841	encoder.sib_baseDisp32(src_reg.lowId());
	1842	encoder.disp32(src_mem.disp);
	1843	}
	1844	} else {
	1845	if (src_mem.disp == 0) {
	1846	encoder.modRm_indirectDisp0(reg.lowId(), src_reg.lowId());
	1847	} else if (immOpSize(src_mem.disp) == 8) {
	1848	encoder.modRm_indirectDisp8(reg.lowId(), src_reg.lowId());
	1849	encoder.disp8(@intCast(i8, src_mem.disp));
	1850	} else {
	1851	encoder.modRm_indirectDisp32(reg.lowId(), src_reg.lowId());
	1852	encoder.disp32(src_mem.disp);
	1853	}
	1854	}
	1855	} else {
	1856	encoder.rex(.{
	1857	.w = setRexWRegister(reg),
	1858	.r = reg.isExtended(),
	1859	});
	1860	opc.encode(encoder);
	1861	if (src_mem.rip) {
	1862	encoder.modRm_RIPDisp32(reg.lowId());
	1863	} else {
	1864	encoder.modRm_SIBDisp0(reg.lowId());
	1865	encoder.sib_disp32();
	1866	}
	1867	encoder.disp32(src_mem.disp);
	1868	}
	1869	},
	1870	}
	1871	switch (reg.size()) {
	1872	// TODO 8bit immediate
	1873	8 => unreachable,
	1874	16 => {
	1875	const imm16 = try math.cast(i16, imm);
	1876	encoder.imm16(imm16);
	1877	},
	1878	32, 64 => encoder.imm32(imm),
	1879	else => unreachable,
	1880	}
	1881	}
	1882
	1883	fn expectEqualHexStrings(expected: []const u8, given: []const u8, assembly: []const u8) !void {
	1884	assert(expected.len > 0);
	1885	if (mem.eql(u8, expected, given)) return;
	1886	const expected_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{std.fmt.fmtSliceHexLower(expected)});
	1887	defer testing.allocator.free(expected_fmt);
	1888	const given_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{std.fmt.fmtSliceHexLower(given)});
	1889	defer testing.allocator.free(given_fmt);
	1890	const idx = mem.indexOfDiff(u8, expected_fmt, given_fmt).?;
	1891	var padding = try testing.allocator.alloc(u8, idx + 5);
	1892	defer testing.allocator.free(padding);
	1893	mem.set(u8, padding, ' ');
	1894	std.debug.print("\nASM: {s}\nEXP: {s}\nGIV: {s}\n{s}^ -- first differing byte\n", .{
	1895	assembly,
	1896	expected_fmt,
	1897	given_fmt,
	1898	padding,
	1899	});
	1900	return error.TestFailed;
	1901	}
	1902
	1903	const TestIsel = struct {
	1904	code_buffer: std.ArrayList(u8),
	1905	next: usize = 0,
	1906
	1907	fn init() TestIsel {
	1908	return .{
	1909	.code_buffer = std.ArrayList(u8).init(testing.allocator),
	1910	};
	1911	}
	1912
	1913	fn deinit(isel: *TestIsel) void {
	1914	isel.code_buffer.deinit();
	1915	isel.next = undefined;
	1916	}
	1917
	1918	fn code(isel: TestIsel) std.ArrayList(u8) {
	1919	isel.next = isel.code_buffer.items.len;
	1920	return &isel.code_buffer;
	1921	}
	1922
	1923	fn lowered(isel: TestIsel) []const u8 {
	1924	return isel.code_buffer.items[isel.next..];
	1925	}
	1926	};
	1927
	1928	test "lower MI encoding" {
	1929	var isel = TestIsel.init();
	1930	defer isel.deinit();
	1931	try lowerToMiEnc(.mov, RegisterOrMemory.reg(.rax), 0x10, isel.code());
	1932	try expectEqualHexStrings("\x48\xc7\xc0\x10\x00\x00\x00", isel.lowered(), "mov rax, 0x10");
	1933	try lowerToMiEnc(.mov, RegisterOrMemory.mem(.r11, 0, .dword_ptr), 0x10, isel.code());
	1934	try expectEqualHexStrings("\x41\xc7\x03\x10\x00\x00\x00", isel.lowered(), "mov dword ptr [r11 + 0], 0x10");
	1935	try lowerToMiEnc(.add, RegisterOrMemory.mem(.rdx, -8, .dword_ptr), 0x10, isel.code());
	1936	try expectEqualHexStrings("\x81\x42\xF8\x10\x00\x00\x00", isel.lowered(), "add dword ptr [rdx - 8], 0x10");
	1937	try lowerToMiEnc(.sub, RegisterOrMemory.mem(.r11, 0x10000000, .dword_ptr), 0x10, isel.code());
	1938	try expectEqualHexStrings(
	1939	"\x41\x81\xab\x00\x00\x00\x10\x10\x00\x00\x00",
	1940	isel.lowered(),
	1941	"sub dword ptr [r11 + 0x10000000], 0x10",
	1942	);
	1943	try lowerToMiEnc(.@"and", RegisterOrMemory.mem(null, 0x10000000, .dword_ptr), 0x10, isel.code());
	1944	try expectEqualHexStrings(
	1945	"\x81\x24\x25\x00\x00\x00\x10\x10\x00\x00\x00",
	1946	isel.lowered(),
	1947	"and dword ptr [ds:0x10000000], 0x10",
	1948	);
	1949	try lowerToMiEnc(.@"and", RegisterOrMemory.mem(.r12, 0x10000000, .dword_ptr), 0x10, isel.code());
	1950	try expectEqualHexStrings(
	1951	"\x41\x81\xA4\x24\x00\x00\x00\x10\x10\x00\x00\x00",
	1952	isel.lowered(),
	1953	"and dword ptr [r12 + 0x10000000], 0x10",
	1954	);
	1955	try lowerToMiEnc(.mov, RegisterOrMemory.rip(0x10, .qword_ptr), 0x10, isel.code());
	1956	try expectEqualHexStrings(
	1957	"\xC7\x05\x10\x00\x00\x00\x10\x00\x00\x00",
	1958	isel.lowered(),
	1959	"mov qword ptr [rip + 0x10], 0x10",
	1960	);
	1961	try lowerToMiEnc(.mov, RegisterOrMemory.mem(.rbp, -8, .qword_ptr), 0x10, isel.code());
	1962	try expectEqualHexStrings(
	1963	"\x48\xc7\x45\xf8\x10\x00\x00\x00",
	1964	isel.lowered(),
	1965	"mov qword ptr [rbp - 8], 0x10",
	1966	);
	1967	try lowerToMiEnc(.mov, RegisterOrMemory.mem(.rbp, -2, .word_ptr), 0x10, isel.code());
	1968	try expectEqualHexStrings("\x66\xC7\x45\xFE\x10\x00", isel.lowered(), "mov word ptr [rbp - 2], 0x10");
	1969	try lowerToMiEnc(.mov, RegisterOrMemory.mem(.rbp, -1, .byte_ptr), 0x10, isel.code());
	1970	try expectEqualHexStrings("\xC6\x45\xFF\x10", isel.lowered(), "mov byte ptr [rbp - 1], 0x10");
	1971	}
	1972
	1973	test "lower RM encoding" {
	1974	var isel = TestIsel.init();
	1975	defer isel.deinit();
	1976	try lowerToRmEnc(.mov, .rax, RegisterOrMemory.reg(.rbx), isel.code());
	1977	try expectEqualHexStrings("\x48\x8b\xc3", isel.lowered(), "mov rax, rbx");
	1978	try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.r11, 0, .qword_ptr), isel.code());
	1979	try expectEqualHexStrings("\x49\x8b\x03", isel.lowered(), "mov rax, qword ptr [r11 + 0]");
	1980	try lowerToRmEnc(.add, .r11, RegisterOrMemory.mem(null, 0x10000000, .qword_ptr), isel.code());
	1981	try expectEqualHexStrings(
	1982	"\x4C\x03\x1C\x25\x00\x00\x00\x10",
	1983	isel.lowered(),
	1984	"add r11, qword ptr [ds:0x10000000]",
	1985	);
	1986	try lowerToRmEnc(.add, .r12b, RegisterOrMemory.mem(null, 0x10000000, .byte_ptr), isel.code());
	1987	try expectEqualHexStrings(
	1988	"\x44\x02\x24\x25\x00\x00\x00\x10",
	1989	isel.lowered(),
	1990	"add r11b, byte ptr [ds:0x10000000]",
	1991	);
	1992	try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.r13, 0x10000000, .qword_ptr), isel.code());
	1993	try expectEqualHexStrings(
	1994	"\x4D\x2B\x9D\x00\x00\x00\x10",
	1995	isel.lowered(),
	1996	"sub r11, qword ptr [r13 + 0x10000000]",
	1997	);
	1998	try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.r12, 0x10000000, .qword_ptr), isel.code());
	1999	try expectEqualHexStrings(
	2000	"\x4D\x2B\x9C\x24\x00\x00\x00\x10",
	2001	isel.lowered(),
	2002	"sub r11, qword ptr [r12 + 0x10000000]",
	2003	);
	2004	try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.rbp, -4, .qword_ptr), isel.code());
	2005	try expectEqualHexStrings("\x48\x8B\x45\xFC", isel.lowered(), "mov rax, qword ptr [rbp - 4]");
	2006	try lowerToRmEnc(.lea, .rax, RegisterOrMemory.rip(0x10, .qword_ptr), isel.code());
	2007	try expectEqualHexStrings("\x48\x8D\x05\x10\x00\x00\x00", isel.lowered(), "lea rax, [rip + 0x10]");
	2008	}
	2009
	2010	test "lower MR encoding" {
	2011	var isel = TestIsel.init();
	2012	defer isel.deinit();
	2013	try lowerToMrEnc(.mov, RegisterOrMemory.reg(.rax), .rbx, isel.code());
	2014	try expectEqualHexStrings("\x48\x89\xd8", isel.lowered(), "mov rax, rbx");
	2015	try lowerToMrEnc(.mov, RegisterOrMemory.mem(.rbp, -4, .qword_ptr), .r11, isel.code());
	2016	try expectEqualHexStrings("\x4c\x89\x5d\xfc", isel.lowered(), "mov qword ptr [rbp - 4], r11");
	2017	try lowerToMrEnc(.add, RegisterOrMemory.mem(null, 0x10000000, .byte_ptr), .r12b, isel.code());
	2018	try expectEqualHexStrings(
	2019	"\x44\x00\x24\x25\x00\x00\x00\x10",
	2020	isel.lowered(),
	2021	"add byte ptr [ds:0x10000000], r12b",
	2022	);
	2023	try lowerToMrEnc(.add, RegisterOrMemory.mem(null, 0x10000000, .dword_ptr), .r12d, isel.code());
	2024	try expectEqualHexStrings(
	2025	"\x44\x01\x24\x25\x00\x00\x00\x10",
	2026	isel.lowered(),
	2027	"add dword ptr [ds:0x10000000], r12d",
	2028	);
	2029	try lowerToMrEnc(.sub, RegisterOrMemory.mem(.r11, 0x10000000, .qword_ptr), .r12, isel.code());
	2030	try expectEqualHexStrings(
	2031	"\x4D\x29\xA3\x00\x00\x00\x10",
	2032	isel.lowered(),
	2033	"sub qword ptr [r11 + 0x10000000], r12",
	2034	);
	2035	try lowerToMrEnc(.mov, RegisterOrMemory.rip(0x10, .qword_ptr), .r12, isel.code());
	2036	try expectEqualHexStrings("\x4C\x89\x25\x10\x00\x00\x00", isel.lowered(), "mov qword ptr [rip + 0x10], r12");
	2037	}
	2038
	2039	test "lower OI encoding" {
	2040	var isel = TestIsel.init();
	2041	defer isel.deinit();
	2042	try lowerToOiEnc(.mov, .rax, 0x1000000000000000, isel.code());
	2043	try expectEqualHexStrings(
	2044	"\x48\xB8\x00\x00\x00\x00\x00\x00\x00\x10",
	2045	isel.lowered(),
	2046	"movabs rax, 0x1000000000000000",
	2047	);
	2048	try lowerToOiEnc(.mov, .r11, 0x1000000000000000, isel.code());
	2049	try expectEqualHexStrings(
	2050	"\x49\xBB\x00\x00\x00\x00\x00\x00\x00\x10",
	2051	isel.lowered(),
	2052	"movabs r11, 0x1000000000000000",
	2053	);
	2054	try lowerToOiEnc(.mov, .r11d, 0x10000000, isel.code());
	2055	try expectEqualHexStrings("\x41\xBB\x00\x00\x00\x10", isel.lowered(), "mov r11d, 0x10000000");
	2056	try lowerToOiEnc(.mov, .r11w, 0x1000, isel.code());
	2057	try expectEqualHexStrings("\x66\x41\xBB\x00\x10", isel.lowered(), "mov r11w, 0x1000");
	2058	try lowerToOiEnc(.mov, .r11b, 0x10, isel.code());
	2059	try expectEqualHexStrings("\x41\xB3\x10", isel.lowered(), "mov r11b, 0x10");
	2060	}
	2061
	2062	test "lower FD/TD encoding" {
	2063	var isel = TestIsel.init();
	2064	defer isel.deinit();
	2065	try lowerToFdEnc(.mov, .rax, 0x1000000000000000, isel.code());
	2066	try expectEqualHexStrings(
	2067	"\x48\xa1\x00\x00\x00\x00\x00\x00\x00\x10",
	2068	isel.lowered(),
	2069	"mov rax, ds:0x1000000000000000",
	2070	);
	2071	try lowerToFdEnc(.mov, .eax, 0x10000000, isel.code());
	2072	try expectEqualHexStrings("\xa1\x00\x00\x00\x10", isel.lowered(), "mov eax, ds:0x10000000");
	2073	try lowerToFdEnc(.mov, .ax, 0x1000, isel.code());
	2074	try expectEqualHexStrings("\x66\xa1\x00\x10", isel.lowered(), "mov ax, ds:0x1000");
	2075	try lowerToFdEnc(.mov, .al, 0x10, isel.code());
	2076	try expectEqualHexStrings("\xa0\x10", isel.lowered(), "mov al, ds:0x10");
	2077	}
	2078
	2079	test "lower M encoding" {
	2080	var isel = TestIsel.init();
	2081	defer isel.deinit();
	2082	try lowerToMEnc(.jmp_near, RegisterOrMemory.reg(.r12), isel.code());
	2083	try expectEqualHexStrings("\x41\xFF\xE4", isel.lowered(), "jmp r12");
	2084	try lowerToMEnc(.jmp_near, RegisterOrMemory.reg(.r12w), isel.code());
	2085	try expectEqualHexStrings("\x66\x41\xFF\xE4", isel.lowered(), "jmp r12w");
	2086	try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.r12, 0, .qword_ptr), isel.code());
	2087	try expectEqualHexStrings("\x41\xFF\x24\x24", isel.lowered(), "jmp qword ptr [r12]");
	2088	try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.r12, 0, .word_ptr), isel.code());
	2089	try expectEqualHexStrings("\x66\x41\xFF\x24\x24", isel.lowered(), "jmp word ptr [r12]");
	2090	try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.r12, 0x10, .qword_ptr), isel.code());
	2091	try expectEqualHexStrings("\x41\xFF\x64\x24\x10", isel.lowered(), "jmp qword ptr [r12 + 0x10]");
	2092	try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.r12, 0x1000, .qword_ptr), isel.code());
	2093	try expectEqualHexStrings(
	2094	"\x41\xFF\xA4\x24\x00\x10\x00\x00",
	2095	isel.lowered(),
	2096	"jmp qword ptr [r12 + 0x1000]",
	2097	);
	2098	try lowerToMEnc(.jmp_near, RegisterOrMemory.rip(0x10, .qword_ptr), isel.code());
	2099	try expectEqualHexStrings("\xFF\x25\x10\x00\x00\x00", isel.lowered(), "jmp qword ptr [rip + 0x10]");
	2100	try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(null, 0x10, .qword_ptr), isel.code());
	2101	try expectEqualHexStrings("\xFF\x24\x25\x10\x00\x00\x00", isel.lowered(), "jmp qword ptr [ds:0x10]");
	2102	try lowerToMEnc(.seta, RegisterOrMemory.reg(.r11b), isel.code());
	2103	try expectEqualHexStrings("\x41\x0F\x97\xC3", isel.lowered(), "seta r11b");
	2104	}
	2105
	2106	test "lower O encoding" {
	2107	var isel = TestIsel.init();
	2108	defer isel.deinit();
	2109	try lowerToOEnc(.pop, .r12, isel.code());
	2110	try expectEqualHexStrings("\x41\x5c", isel.lowered(), "pop r12");
	2111	try lowerToOEnc(.push, .r12w, isel.code());
	2112	try expectEqualHexStrings("\x66\x41\x54", isel.lowered(), "push r12w");
	2113	}
	2114
	2115	test "lower RMI encoding" {
	2116	var isel = TestIsel.init();
	2117	defer isel.deinit();
	2118	try lowerToRmiEnc(.imul, .rax, RegisterOrMemory.mem(.rbp, -8, .qword_ptr), 0x10, isel.code());
	2119	try expectEqualHexStrings(
	2120	"\x48\x69\x45\xF8\x10\x00\x00\x00",
	2121	isel.lowered(),
	2122	"imul rax, qword ptr [rbp - 8], 0x10",
	2123	);
	2124	try lowerToRmiEnc(.imul, .eax, RegisterOrMemory.mem(.rbp, -4, .dword_ptr), 0x10, isel.code());
	2125	try expectEqualHexStrings("\x69\x45\xFC\x10\x00\x00\x00", isel.lowered(), "imul eax, dword ptr [rbp - 4], 0x10");
	2126	try lowerToRmiEnc(.imul, .ax, RegisterOrMemory.mem(.rbp, -2, .word_ptr), 0x10, isel.code());
	2127	try expectEqualHexStrings("\x66\x69\x45\xFE\x10\x00", isel.lowered(), "imul ax, word ptr [rbp - 2], 0x10");
	2128	try lowerToRmiEnc(.imul, .r12, RegisterOrMemory.reg(.r12), 0x10, isel.code());
	2129	try expectEqualHexStrings("\x4D\x69\xE4\x10\x00\x00\x00", isel.lowered(), "imul r12, r12, 0x10");
	2130	try lowerToRmiEnc(.imul, .r12w, RegisterOrMemory.reg(.r12w), 0x10, isel.code());
	2131	try expectEqualHexStrings("\x66\x45\x69\xE4\x10\x00", isel.lowered(), "imul r12w, r12w, 0x10");
	2132	}