Commit 0049c7180f6b8df597c045d02189348f3be0cb0e | zig

author	Andrew Kelley <andrew@ziglang.org> 2021-12-23 16:39:51-08:00
committer	GitHub <noreply@github.com> 2021-12-23 16:39:51-08:00
log	0049c7180f6b8df597c045d02189348f3be0cb0e
tree	a015ea0254b44bc2aff3eac0e449a2fe5f5a70fc
parent	303bad9989866373042df9e80722185608bf1147
parent	4c119866507b244dc0658b5cb43e20295f1888d7
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #10394 from ziglang/stage2-x86_64-mir-intel-syntax

stage2: rewrite MIR -> Isel layer for x86_64

3 files changed, 1313 insertions(+), 584 deletions(-)

src
- arch
  - x86_64
    CodeGen.zig
    Emit.zig
    Mir.zig

src/arch/x86_64/CodeGen.zig+22-21

 @@ -275,7 +275,9 @@ pub fn generate(
         .stack_align = undefined,
         .end_di_line = module_fn.rbrace_line,
         .end_di_column = module_fn.rbrace_column,
-        .mir_to_air_map = if (builtin.mode == .Debug) std.AutoHashMap(Mir.Inst.Index, Air.Inst.Index).init(bin_file.allocator) else {},
+        .mir_to_air_map = if (builtin.mode == .Debug)
+            std.AutoHashMap(Mir.Inst.Index, Air.Inst.Index).init(bin_file.allocator)
+        else {},
     };
     defer function.stack.deinit(bin_file.allocator);
     defer function.blocks.deinit(bin_file.allocator);
 @@ -386,8 +388,8 @@ fn gen(self: *Self) InnerError!void {
         _ = try self.addInst(.{
             .tag = .mov,
             .ops = (Mir.Ops{
-                .reg1 = .rsp,
+                .reg1 = .rbp,
-                .reg2 = .rbp,
+                .reg2 = .rsp,
             }).encode(),
             .data = undefined,
         });
 @@ -1632,18 +1634,18 @@ fn genBinMathOpMir(
                     _ = try self.addInst(.{
                         .tag = mir_tag,
                         .ops = (Mir.Ops{
-                            .reg1 = src_reg,
+                            .reg1 = registerAlias(dst_reg, @divExact(src_reg.size(), 8)),
-                            .reg2 = dst_reg,
+                            .reg2 = src_reg,
-                            .flags = 0b11,
                         }).encode(),
                         .data = undefined,
                     });
                 },
                 .immediate => |imm| {
+                    // TODO I am not quite sure why we need to set the size of the register here...
                     _ = try self.addInst(.{
                         .tag = mir_tag,
                         .ops = (Mir.Ops{
-                            .reg1 = dst_reg,
+                            .reg1 = registerAlias(dst_reg, 4),
                         }).encode(),
                         .data = .{ .imm = @intCast(i32, imm) },
                     });
 @@ -1661,7 +1663,7 @@ fn genBinMathOpMir(
                         .tag = mir_tag,
                         .ops = (Mir.Ops{
                             .reg1 = registerAlias(dst_reg, @intCast(u32, abi_size)),
-                            .reg2 = registerAlias(.rbp, @intCast(u32, abi_size)),
+                            .reg2 = .rbp,
                             .flags = 0b01,
                         }).encode(),
                         .data = .{ .imm = -@intCast(i32, adj_off) },
 @@ -1691,8 +1693,8 @@ fn genBinMathOpMir(
                     _ = try self.addInst(.{
                         .tag = mir_tag,
                         .ops = (Mir.Ops{
-                            .reg1 = registerAlias(src_reg, @intCast(u32, abi_size)),
+                            .reg1 = .rbp,
-                            .reg2 = registerAlias(.rbp, @intCast(u32, abi_size)),
+                            .reg2 = registerAlias(src_reg, @intCast(u32, abi_size)),
                             .flags = 0b10,
                         }).encode(),
                         .data = .{ .imm = -@intCast(i32, adj_off) },
 @@ -1741,7 +1743,7 @@ fn genIMulOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !
                     _ = try self.addInst(.{
                         .tag = .imul_complex,
                         .ops = (Mir.Ops{
-                            .reg1 = dst_reg,
+                            .reg1 = registerAlias(dst_reg, @divExact(src_reg.size(), 8)),
                             .reg2 = src_reg,
                         }).encode(),
                         .data = undefined,
 @@ -1790,7 +1792,7 @@ fn genIMulOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !
                     _ = try self.addInst(.{
                         .tag = .imul_complex,
                         .ops = (Mir.Ops{
-                            .reg1 = dst_reg,
+                            .reg1 = registerAlias(dst_reg, @divExact(src_reg.size(), 8)),
                             .reg2 = src_reg,
                         }).encode(),
                         .data = undefined,
 @@ -2868,8 +2870,8 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
             _ = try self.addInst(.{
                 .tag = .mov,
                 .ops = (Mir.Ops{
-                    .reg1 = registerAlias(reg, @intCast(u32, abi_size)),
+                    .reg1 = .rbp,
-                    .reg2 = registerAlias(.rbp, @intCast(u32, abi_size)),
+                    .reg2 = registerAlias(reg, @intCast(u32, abi_size)),
                     .flags = 0b10,
                 }).encode(),
                 .data = .{ .imm = -@intCast(i32, adj_off) },
 @@ -2926,7 +2928,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
             _ = try self.addInst(.{
                 .tag = tag,
                 .ops = (Mir.Ops{
-                    .reg1 = reg,
+                    .reg1 = reg.to8(),
                     .flags = flags,
                 }).encode(),
                 .data = undefined,
 @@ -2952,10 +2954,11 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
             }
             if (x <= math.maxInt(i32)) {
                 // Next best case: if we set the lower four bytes, the upper four will be zeroed.
+                // TODO I am not quite sure why we need to set the size of the register here...
                 _ = try self.addInst(.{
                     .tag = .mov,
                     .ops = (Mir.Ops{
-                        .reg1 = reg,
+                        .reg1 = registerAlias(reg, 4),
                     }).encode(),
                     .data = .{ .imm = @intCast(i32, x) },
                 });
 @@ -2996,9 +2999,8 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
             _ = try self.addInst(.{
                 .tag = .mov,
                 .ops = (Mir.Ops{
-                    .reg1 = reg,
+                    .reg1 = registerAlias(reg, @divExact(src_reg.size(), 8)),
                     .reg2 = src_reg,
-                    .flags = 0b11,
                 }).encode(),
                 .data = undefined,
             });
 @@ -3085,15 +3087,14 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
             if (off < std.math.minInt(i32) or off > std.math.maxInt(i32)) {
                 return self.fail("stack offset too large", .{});
             }
-            const ioff = -@intCast(i32, off);
             _ = try self.addInst(.{
                 .tag = .mov,
                 .ops = (Mir.Ops{
                     .reg1 = registerAlias(reg, @intCast(u32, abi_size)),
-                    .reg2 = registerAlias(.rbp, @intCast(u32, abi_size)),
+                    .reg2 = .rbp,
                     .flags = 0b01,
                 }).encode(),
-                .data = .{ .imm = ioff },
+                .data = .{ .imm = -@intCast(i32, off) },
             });
         },
     }

src/arch/x86_64/Emit.zig+1290-563

...	@@ -11,8 +11,10 @@ const link = @import("../../link.zig");	...	@@ -11,8 +11,10 @@ const link = @import("../../link.zig");
11	const log = std.log.scoped(.codegen);	11	const log = std.log.scoped(.codegen);
12	const math = std.math;	12	const math = std.math;
13	const mem = std.mem;	13	const mem = std.mem;
		14	const testing = std.testing;
14		15
15	const Air = @import("../../Air.zig");	16	const Air = @import("../../Air.zig");
		17	const Allocator = mem.Allocator;
16	const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;	18	const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;
17	const DW = std.dwarf;	19	const DW = std.dwarf;
18	const Encoder = bits.Encoder;	20	const Encoder = bits.Encoder;
...	@@ -72,6 +74,7 @@ pub fn emitMir(emit: *Emit) InnerError!void {	...	@@ -72,6 +74,7 @@ pub fn emitMir(emit: *Emit) InnerError!void {
72	.@"or" => try emit.mirArith(.@"or", inst),	74	.@"or" => try emit.mirArith(.@"or", inst),
73	.sbb => try emit.mirArith(.sbb, inst),	75	.sbb => try emit.mirArith(.sbb, inst),
74	.cmp => try emit.mirArith(.cmp, inst),	76	.cmp => try emit.mirArith(.cmp, inst),
		77	.mov => try emit.mirArith(.mov, inst),
75		78
76	.adc_scale_src => try emit.mirArithScaleSrc(.adc, inst),	79	.adc_scale_src => try emit.mirArithScaleSrc(.adc, inst),
77	.add_scale_src => try emit.mirArithScaleSrc(.add, inst),	80	.add_scale_src => try emit.mirArithScaleSrc(.add, inst),
...	@@ -81,6 +84,7 @@ pub fn emitMir(emit: *Emit) InnerError!void {	...	@@ -81,6 +84,7 @@ pub fn emitMir(emit: *Emit) InnerError!void {
81	.or_scale_src => try emit.mirArithScaleSrc(.@"or", inst),	84	.or_scale_src => try emit.mirArithScaleSrc(.@"or", inst),
82	.sbb_scale_src => try emit.mirArithScaleSrc(.sbb, inst),	85	.sbb_scale_src => try emit.mirArithScaleSrc(.sbb, inst),
83	.cmp_scale_src => try emit.mirArithScaleSrc(.cmp, inst),	86	.cmp_scale_src => try emit.mirArithScaleSrc(.cmp, inst),
		87	.mov_scale_src => try emit.mirArithScaleSrc(.mov, inst),
84		88
85	.adc_scale_dst => try emit.mirArithScaleDst(.adc, inst),	89	.adc_scale_dst => try emit.mirArithScaleDst(.adc, inst),
86	.add_scale_dst => try emit.mirArithScaleDst(.add, inst),	90	.add_scale_dst => try emit.mirArithScaleDst(.add, inst),
...	@@ -90,6 +94,7 @@ pub fn emitMir(emit: *Emit) InnerError!void {	...	@@ -90,6 +94,7 @@ pub fn emitMir(emit: *Emit) InnerError!void {
90	.or_scale_dst => try emit.mirArithScaleDst(.@"or", inst),	94	.or_scale_dst => try emit.mirArithScaleDst(.@"or", inst),
91	.sbb_scale_dst => try emit.mirArithScaleDst(.sbb, inst),	95	.sbb_scale_dst => try emit.mirArithScaleDst(.sbb, inst),
92	.cmp_scale_dst => try emit.mirArithScaleDst(.cmp, inst),	96	.cmp_scale_dst => try emit.mirArithScaleDst(.cmp, inst),
		97	.mov_scale_dst => try emit.mirArithScaleDst(.mov, inst),
93		98
94	.adc_scale_imm => try emit.mirArithScaleImm(.adc, inst),	99	.adc_scale_imm => try emit.mirArithScaleImm(.adc, inst),
95	.add_scale_imm => try emit.mirArithScaleImm(.add, inst),	100	.add_scale_imm => try emit.mirArithScaleImm(.add, inst),
...	@@ -99,14 +104,8 @@ pub fn emitMir(emit: *Emit) InnerError!void {	...	@@ -99,14 +104,8 @@ pub fn emitMir(emit: *Emit) InnerError!void {
99	.or_scale_imm => try emit.mirArithScaleImm(.@"or", inst),	104	.or_scale_imm => try emit.mirArithScaleImm(.@"or", inst),
100	.sbb_scale_imm => try emit.mirArithScaleImm(.sbb, inst),	105	.sbb_scale_imm => try emit.mirArithScaleImm(.sbb, inst),
101	.cmp_scale_imm => try emit.mirArithScaleImm(.cmp, inst),	106	.cmp_scale_imm => try emit.mirArithScaleImm(.cmp, inst),
102
103	// Even though MOV is technically not an arithmetic op,
104	// its structure can be represented using the same set of
105	// opcode primitives.
106	.mov => try emit.mirArith(.mov, inst),
107	.mov_scale_src => try emit.mirArithScaleSrc(.mov, inst),
108	.mov_scale_dst => try emit.mirArithScaleDst(.mov, inst),
109	.mov_scale_imm => try emit.mirArithScaleImm(.mov, inst),	107	.mov_scale_imm => try emit.mirArithScaleImm(.mov, inst),
		108
110	.movabs => try emit.mirMovabs(inst),	109	.movabs => try emit.mirMovabs(inst),
111		110
112	.lea => try emit.mirLea(inst),	111	.lea => try emit.mirLea(inst),
...	@@ -117,16 +116,18 @@ pub fn emitMir(emit: *Emit) InnerError!void {	...	@@ -117,16 +116,18 @@ pub fn emitMir(emit: *Emit) InnerError!void {
117	.push => try emit.mirPushPop(.push, inst),	116	.push => try emit.mirPushPop(.push, inst),
118	.pop => try emit.mirPushPop(.pop, inst),	117	.pop => try emit.mirPushPop(.pop, inst),
119		118
120	.jmp => try emit.mirJmpCall(.jmp, inst),	119	.jmp => try emit.mirJmpCall(.jmp_near, inst),
121	.call => try emit.mirJmpCall(.call, inst),	120	.call => try emit.mirJmpCall(.call_near, inst),
122		121
123	.cond_jmp_greater_less => try emit.mirCondJmp(.cond_jmp_greater_less, inst),	122	.cond_jmp_greater_less,
124	.cond_jmp_above_below => try emit.mirCondJmp(.cond_jmp_above_below, inst),	123	.cond_jmp_above_below,
125	.cond_jmp_eq_ne => try emit.mirCondJmp(.cond_jmp_eq_ne, inst),	124	.cond_jmp_eq_ne,
		125	=> try emit.mirCondJmp(tag, inst),
126		126
127	.cond_set_byte_greater_less => try emit.mirCondSetByte(.cond_set_byte_greater_less, inst),	127	.cond_set_byte_greater_less,
128	.cond_set_byte_above_below => try emit.mirCondSetByte(.cond_set_byte_above_below, inst),	128	.cond_set_byte_above_below,
129	.cond_set_byte_eq_ne => try emit.mirCondSetByte(.cond_set_byte_eq_ne, inst),	129	.cond_set_byte_eq_ne,
		130	=> try emit.mirCondSetByte(tag, inst),
130		131
131	.ret => try emit.mirRet(inst),	132	.ret => try emit.mirRet(inst),
132		133
...	@@ -184,95 +185,45 @@ fn fixupRelocs(emit: *Emit) InnerError!void {	...	@@ -184,95 +185,45 @@ fn fixupRelocs(emit: *Emit) InnerError!void {
184	}	185	}
185		186
186	fn mirBrk(emit: *Emit) InnerError!void {	187	fn mirBrk(emit: *Emit) InnerError!void {
187	const encoder = try Encoder.init(emit.code, 1);	188	return lowerToZoEnc(.brk, emit.code);
188	encoder.opcode_1byte(0xcc);
189	}	189	}
190		190
191	fn mirNop(emit: *Emit) InnerError!void {	191	fn mirNop(emit: *Emit) InnerError!void {
192	const encoder = try Encoder.init(emit.code, 1);	192	return lowerToZoEnc(.nop, emit.code);
193	encoder.opcode_1byte(0x90);
194	}	193	}
195		194
196	fn mirSyscall(emit: *Emit) InnerError!void {	195	fn mirSyscall(emit: *Emit) InnerError!void {
197	const encoder = try Encoder.init(emit.code, 2);	196	return lowerToZoEnc(.syscall, emit.code);
198	encoder.opcode_2byte(0x0f, 0x05);
199	}	197	}
200		198
201	fn mirPushPop(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {	199	fn mirPushPop(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
202	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);	200	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
203	switch (ops.flags) {	201	switch (ops.flags) {
204	0b00 => {	202	0b00 => {
205	// PUSH/POP reg	203	// PUSH/POP reg
206	const opc: u8 = switch (tag) {	204	return lowerToOEnc(tag, ops.reg1, emit.code);
207	.push => 0x50,
208	.pop => 0x58,
209	else => unreachable,
210	};
211	const encoder = try Encoder.init(emit.code, 2);
212	encoder.rex(.{
213	.b = ops.reg1.isExtended(),
214	});
215	encoder.opcode_withReg(opc, ops.reg1.lowId());
216	},	205	},
217	0b01 => {	206	0b01 => {
218	// PUSH/POP r/m64	207	// PUSH/POP r/m64
219	const imm = emit.mir.instructions.items(.data)[inst].imm;	208	const imm = emit.mir.instructions.items(.data)[inst].imm;
220	const opc: u8 = switch (tag) {	209	return lowerToMEnc(tag, RegisterOrMemory.mem(ops.reg1, imm), emit.code);
221	.push => 0xff,
222	.pop => 0x8f,
223	else => unreachable,
224	};
225	const modrm_ext: u3 = switch (tag) {
226	.push => 0x6,
227	.pop => 0x0,
228	else => unreachable,
229	};
230	const encoder = try Encoder.init(emit.code, 6);
231	encoder.opcode_1byte(opc);
232	if (math.cast(i8, imm)) \|imm_i8\| {
233	encoder.modRm_indirectDisp8(modrm_ext, ops.reg1.lowId());
234	encoder.imm8(@intCast(i8, imm_i8));
235	} else \|_\| {
236	encoder.modRm_indirectDisp32(modrm_ext, ops.reg1.lowId());
237	encoder.imm32(imm);
238	}
239	},	210	},
240	0b10 => {	211	0b10 => {
241	// PUSH imm32	212	// PUSH imm32
242	assert(tag == .push);	213	assert(tag == .push);
243	const imm = emit.mir.instructions.items(.data)[inst].imm;	214	const imm = emit.mir.instructions.items(.data)[inst].imm;
244	const opc: u8 = if (imm <= math.maxInt(i8)) 0x6a else 0x6b;	215	return lowerToIEnc(.push, imm, emit.code);
245	const encoder = try Encoder.init(emit.code, 2);
246	encoder.opcode_1byte(opc);
247	if (imm <= math.maxInt(i8)) {
248	encoder.imm8(@intCast(i8, imm));
249	} else if (imm <= math.maxInt(i16)) {
250	encoder.imm16(@intCast(i16, imm));
251	} else {
252	encoder.imm32(imm);
253	}
254	},	216	},
255	0b11 => unreachable,	217	0b11 => unreachable,
256	}	218	}
257	}	219	}
258	fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {	220	fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
259	const callee_preserved_regs = bits.callee_preserved_regs;	221	const callee_preserved_regs = bits.callee_preserved_regs;
260	// PUSH/POP reg
261	const opc: u8 = switch (tag) {
262	.push => 0x50,
263	.pop => 0x58,
264	else => unreachable,
265	};
266
267	const regs = emit.mir.instructions.items(.data)[inst].regs_to_push_or_pop;	222	const regs = emit.mir.instructions.items(.data)[inst].regs_to_push_or_pop;
268	if (tag == .push) {	223	if (tag == .push) {
269	for (callee_preserved_regs) \|reg, i\| {	224	for (callee_preserved_regs) \|reg, i\| {
270	if ((regs >> @intCast(u5, i)) & 1 == 0) continue;	225	if ((regs >> @intCast(u5, i)) & 1 == 0) continue;
271	const encoder = try Encoder.init(emit.code, 2);	226	try lowerToOEnc(.push, reg, emit.code);
272	encoder.rex(.{
273	.b = reg.isExtended(),
274	});
275	encoder.opcode_withReg(opc, reg.lowId());
276	}	227	}
277	} else {	228	} else {
278	// pop in the reverse direction	229	// pop in the reverse direction
...	@@ -280,201 +231,89 @@ fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Mir.Inst.Tag, inst: M	...	@@ -280,201 +231,89 @@ fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Mir.Inst.Tag, inst: M
280	while (i > 0) : (i -= 1) {	231	while (i > 0) : (i -= 1) {
281	const reg = callee_preserved_regs[i - 1];	232	const reg = callee_preserved_regs[i - 1];
282	if ((regs >> @intCast(u5, i - 1)) & 1 == 0) continue;	233	if ((regs >> @intCast(u5, i - 1)) & 1 == 0) continue;
283	const encoder = try Encoder.init(emit.code, 2);	234	try lowerToOEnc(.pop, reg, emit.code);
284	encoder.rex(.{
285	.b = reg.isExtended(),
286	});
287	encoder.opcode_withReg(opc, reg.lowId());
288	}	235	}
289	}	236	}
290	}	237	}
291		238
292	fn mirJmpCall(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {	239	fn mirJmpCall(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
293	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);	240	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
294	const flag = @truncate(u1, ops.flags);	241	const flag = @truncate(u1, ops.flags);
295	if (flag == 0) {	242	if (flag == 0) {
296	const target = emit.mir.instructions.items(.data)[inst].inst;	243	const target = emit.mir.instructions.items(.data)[inst].inst;
297	const opc: u8 = switch (tag) {
298	.jmp => 0xe9,
299	.call => 0xe8,
300	else => unreachable,
301	};
302	const source = emit.code.items.len;	244	const source = emit.code.items.len;
303	const encoder = try Encoder.init(emit.code, 5);	245	try lowerToDEnc(tag, 0, emit.code);
304	encoder.opcode_1byte(opc);
305	try emit.relocs.append(emit.bin_file.allocator, .{	246	try emit.relocs.append(emit.bin_file.allocator, .{
306	.source = source,	247	.source = source,
307	.target = target,	248	.target = target,
308	.offset = emit.code.items.len,	249	.offset = emit.code.items.len - 4,
309	.length = 5,	250	.length = 5,
310	});	251	});
311	encoder.imm32(0x0);
312	return;	252	return;
313	}	253	}
314	const modrm_ext: u3 = switch (tag) {
315	.jmp => 0x4,
316	.call => 0x2,
317	else => unreachable,
318	};
319	if (ops.reg1 == .none) {	254	if (ops.reg1 == .none) {
320	// JMP/CALL [imm]	255	// JMP/CALL [imm]
321	const imm = emit.mir.instructions.items(.data)[inst].imm;	256	const imm = emit.mir.instructions.items(.data)[inst].imm;
322	const encoder = try Encoder.init(emit.code, 7);	257	return lowerToMEnc(tag, RegisterOrMemory.mem(null, imm), emit.code);
323	encoder.opcode_1byte(0xff);
324	encoder.modRm_SIBDisp0(modrm_ext);
325	encoder.sib_disp32();
326	encoder.imm32(imm);
327	return;
328	}	258	}
329	// JMP/CALL reg	259	// JMP/CALL reg
330	const encoder = try Encoder.init(emit.code, 2);	260	return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code);
331	encoder.opcode_1byte(0xff);
332	encoder.modRm_direct(modrm_ext, ops.reg1.lowId());
333	}	261	}
334		262
335	const CondType = enum {	263	fn mirCondJmp(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
336	/// greater than or equal	264	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
337	gte,	265	const target = emit.mir.instructions.items(.data)[inst].inst;
338		266	const tag = switch (mir_tag) {
339	/// greater than	267	.cond_jmp_greater_less => switch (ops.flags) {
340	gt,	268	0b00 => Tag.jge,
341		269	0b01 => Tag.jg,
342	/// less than	270	0b10 => Tag.jl,
343	lt,	271	0b11 => Tag.jle,
344
345	/// less than or equal
346	lte,
347
348	/// above or equal
349	ae,
350
351	/// above
352	a,
353
354	/// below
355	b,
356
357	/// below or equal
358	be,
359
360	/// not equal
361	ne,
362
363	/// equal
364	eq,
365
366	fn fromTagAndFlags(tag: Mir.Inst.Tag, flags: u2) CondType {
367	return switch (tag) {
368	.cond_jmp_greater_less,
369	.cond_set_byte_greater_less,
370	=> switch (flags) {
371	0b00 => CondType.gte,
372	0b01 => CondType.gt,
373	0b10 => CondType.lt,
374	0b11 => CondType.lte,
375	},
376	.cond_jmp_above_below,
377	.cond_set_byte_above_below,
378	=> switch (flags) {
379	0b00 => CondType.ae,
380	0b01 => CondType.a,
381	0b10 => CondType.b,
382	0b11 => CondType.be,
383	},
384	.cond_jmp_eq_ne,
385	.cond_set_byte_eq_ne,
386	=> switch (@truncate(u1, flags)) {
387	0b0 => CondType.ne,
388	0b1 => CondType.eq,
389	},
390	else => unreachable,
391	};
392	}
393	};
394
395	inline fn getCondOpCode(tag: Mir.Inst.Tag, cond: CondType) u8 {
396	switch (cond) {
397	.gte => return switch (tag) {
398	.cond_jmp_greater_less => 0x8d,
399	.cond_set_byte_greater_less => 0x9d,
400	else => unreachable,
401	},
402	.gt => return switch (tag) {
403	.cond_jmp_greater_less => 0x8f,
404	.cond_set_byte_greater_less => 0x9f,
405	else => unreachable,
406	},
407	.lt => return switch (tag) {
408	.cond_jmp_greater_less => 0x8c,
409	.cond_set_byte_greater_less => 0x9c,
410	else => unreachable,
411	},
412	.lte => return switch (tag) {
413	.cond_jmp_greater_less => 0x8e,
414	.cond_set_byte_greater_less => 0x9e,
415	else => unreachable,
416	},
417	.ae => return switch (tag) {
418	.cond_jmp_above_below => 0x83,
419	.cond_set_byte_above_below => 0x93,
420	else => unreachable,
421	},
422	.a => return switch (tag) {
423	.cond_jmp_above_below => 0x87,
424	.cond_set_byte_greater_less => 0x97,
425	else => unreachable,
426	},
427	.b => return switch (tag) {
428	.cond_jmp_above_below => 0x82,
429	.cond_set_byte_greater_less => 0x92,
430	else => unreachable,
431	},
432	.be => return switch (tag) {
433	.cond_jmp_above_below => 0x86,
434	.cond_set_byte_greater_less => 0x96,
435	else => unreachable,
436	},	272	},
437	.eq => return switch (tag) {	273	.cond_jmp_above_below => switch (ops.flags) {
438	.cond_jmp_eq_ne => 0x84,	274	0b00 => Tag.jae,
439	.cond_set_byte_eq_ne => 0x94,	275	0b01 => Tag.ja,
440	else => unreachable,	276	0b10 => Tag.jb,
		277	0b11 => Tag.jbe,
441	},	278	},
442	.ne => return switch (tag) {	279	.cond_jmp_eq_ne => switch (@truncate(u1, ops.flags)) {
443	.cond_jmp_eq_ne => 0x85,	280	0b0 => Tag.jne,
444	.cond_set_byte_eq_ne => 0x95,	281	0b1 => Tag.je,
445	else => unreachable,
446	},	282	},
447	}	283	else => unreachable,
448	}	284	};
449
450	fn mirCondJmp(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
451	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
452	const target = emit.mir.instructions.items(.data)[inst].inst;
453	const cond = CondType.fromTagAndFlags(tag, ops.flags);
454	const opc = getCondOpCode(tag, cond);
455	const source = emit.code.items.len;	285	const source = emit.code.items.len;
456	const encoder = try Encoder.init(emit.code, 6);	286	try lowerToDEnc(tag, 0, emit.code);
457	encoder.opcode_2byte(0x0f, opc);
458	try emit.relocs.append(emit.bin_file.allocator, .{	287	try emit.relocs.append(emit.bin_file.allocator, .{
459	.source = source,	288	.source = source,
460	.target = target,	289	.target = target,
461	.offset = emit.code.items.len,	290	.offset = emit.code.items.len - 4,
462	.length = 6,	291	.length = 6,
463	});	292	});
464	encoder.imm32(0);
465	}	293	}
466		294
467	fn mirCondSetByte(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {	295	fn mirCondSetByte(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
468	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);	296	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
469	const cond = CondType.fromTagAndFlags(tag, ops.flags);	297	const tag = switch (mir_tag) {
470	const opc = getCondOpCode(tag, cond);	298	.cond_set_byte_greater_less => switch (ops.flags) {
471	const encoder = try Encoder.init(emit.code, 4);	299	0b00 => Tag.setge,
472	encoder.rex(.{	300	0b01 => Tag.setg,
473	.w = true,	301	0b10 => Tag.setl,
474	.b = ops.reg1.isExtended(),	302	0b11 => Tag.setle,
475	});	303	},
476	encoder.opcode_2byte(0x0f, opc);	304	.cond_set_byte_above_below => switch (ops.flags) {
477	encoder.modRm_direct(0x0, ops.reg1.lowId());	305	0b00 => Tag.setae,
		306	0b01 => Tag.seta,
		307	0b10 => Tag.setb,
		308	0b11 => Tag.setbe,
		309	},
		310	.cond_set_byte_eq_ne => switch (@truncate(u1, ops.flags)) {
		311	0b0 => Tag.setne,
		312	0b1 => Tag.sete,
		313	},
		314	else => unreachable,
		315	};
		316	return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code);
478	}	317	}
479		318
480	fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {	319	fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
...	@@ -482,31 +321,17 @@ fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {	...	@@ -482,31 +321,17 @@ fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
482	assert(tag == .@"test");	321	assert(tag == .@"test");
483	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);	322	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
484	switch (ops.flags) {	323	switch (ops.flags) {
485	0b00 => blk: {	324	0b00 => {
486	if (ops.reg2 == .none) {	325	if (ops.reg2 == .none) {
487	// TEST r/m64, imm32	326	// TEST r/m64, imm32
		327	// MI
488	const imm = emit.mir.instructions.items(.data)[inst].imm;	328	const imm = emit.mir.instructions.items(.data)[inst].imm;
489	if (ops.reg1.to64() == .rax) {	329	if (ops.reg1.to64() == .rax) {
490	// TODO reduce the size of the instruction if the immediate	330	// TEST rax, imm32
491	// is smaller than 32 bits	331	// I
492	const encoder = try Encoder.init(emit.code, 6);	332	return lowerToIEnc(.@"test", imm, emit.code);
493	encoder.rex(.{
494	.w = true,
495	});
496	encoder.opcode_1byte(0xa9);
497	encoder.imm32(imm);
498	break :blk;
499	}	333	}
500	const opc: u8 = if (ops.reg1.size() == 8) 0xf6 else 0xf7;	334	return lowerToMiEnc(.@"test", RegisterOrMemory.reg(ops.reg1), imm, emit.code);
501	const encoder = try Encoder.init(emit.code, 7);
502	encoder.rex(.{
503	.w = true,
504	.b = ops.reg1.isExtended(),
505	});
506	encoder.opcode_1byte(opc);
507	encoder.modRm_direct(0, ops.reg1.lowId());
508	encoder.imm8(@intCast(i8, imm));
509	break :blk;
510	}	335	}
511	// TEST r/m64, r64	336	// TEST r/m64, r64
512	return emit.fail("TODO TEST r/m64, r64", .{});	337	return emit.fail("TODO TEST r/m64, r64", .{});
...	@@ -519,26 +344,161 @@ fn mirRet(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {	...	@@ -519,26 +344,161 @@ fn mirRet(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
519	const tag = emit.mir.instructions.items(.tag)[inst];	344	const tag = emit.mir.instructions.items(.tag)[inst];
520	assert(tag == .ret);	345	assert(tag == .ret);
521	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);	346	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
522	const encoder = try Encoder.init(emit.code, 3);
523	switch (ops.flags) {	347	switch (ops.flags) {
524	0b00 => {	348	0b00 => {
525	// RETF imm16	349	// RETF imm16
		350	// I
526	const imm = emit.mir.instructions.items(.data)[inst].imm;	351	const imm = emit.mir.instructions.items(.data)[inst].imm;
527	encoder.opcode_1byte(0xca);	352	return lowerToIEnc(.ret_far, imm, emit.code);
528	encoder.imm16(@intCast(i16, imm));
529	},	353	},
530	0b01 => encoder.opcode_1byte(0xcb), // RETF	354	0b01 => return lowerToZoEnc(.ret_far, emit.code),
531	0b10 => {	355	0b10 => {
532	// RET imm16	356	// RET imm16
		357	// I
533	const imm = emit.mir.instructions.items(.data)[inst].imm;	358	const imm = emit.mir.instructions.items(.data)[inst].imm;
534	encoder.opcode_1byte(0xc2);	359	return lowerToIEnc(.ret_near, imm, emit.code);
535	encoder.imm16(@intCast(i16, imm));
536	},	360	},
537	0b11 => encoder.opcode_1byte(0xc3), // RET	361	0b11 => return lowerToZoEnc(.ret_near, emit.code),
538	}	362	}
539	}	363	}
540		364
541	const EncType = enum {	365	const Tag = enum {
		366	adc,
		367	add,
		368	sub,
		369	xor,
		370	@"and",
		371	@"or",
		372	sbb,
		373	cmp,
		374	mov,
		375	lea,
		376	jmp_near,
		377	call_near,
		378	push,
		379	pop,
		380	@"test",
		381	brk,
		382	nop,
		383	imul,
		384	syscall,
		385	ret_near,
		386	ret_far,
		387	jo,
		388	jno,
		389	jb,
		390	jbe,
		391	jc,
		392	jnae,
		393	jnc,
		394	jae,
		395	je,
		396	jz,
		397	jne,
		398	jnz,
		399	jna,
		400	jnb,
		401	jnbe,
		402	ja,
		403	js,
		404	jns,
		405	jpe,
		406	jp,
		407	jpo,
		408	jnp,
		409	jnge,
		410	jl,
		411	jge,
		412	jnl,
		413	jle,
		414	jng,
		415	jg,
		416	jnle,
		417	seto,
		418	setno,
		419	setb,
		420	setc,
		421	setnae,
		422	setnb,
		423	setnc,
		424	setae,
		425	sete,
		426	setz,
		427	setne,
		428	setnz,
		429	setbe,
		430	setna,
		431	seta,
		432	setnbe,
		433	sets,
		434	setns,
		435	setp,
		436	setpe,
		437	setnp,
		438	setop,
		439	setl,
		440	setnge,
		441	setnl,
		442	setge,
		443	setle,
		444	setng,
		445	setnle,
		446	setg,
		447
		448	fn isSetCC(tag: Tag) bool {
		449	return switch (tag) {
		450	.seto,
		451	.setno,
		452	.setb,
		453	.setc,
		454	.setnae,
		455	.setnb,
		456	.setnc,
		457	.setae,
		458	.sete,
		459	.setz,
		460	.setne,
		461	.setnz,
		462	.setbe,
		463	.setna,
		464	.seta,
		465	.setnbe,
		466	.sets,
		467	.setns,
		468	.setp,
		469	.setpe,
		470	.setnp,
		471	.setop,
		472	.setl,
		473	.setnge,
		474	.setnl,
		475	.setge,
		476	.setle,
		477	.setng,
		478	.setnle,
		479	.setg,
		480	=> true,
		481	else => false,
		482	};
		483	}
		484	};
		485
		486	const Encoding = enum {
		487	/// OP
		488	zo,
		489
		490	/// OP rel32
		491	d,
		492
		493	/// OP r/m64
		494	m,
		495
		496	/// OP r64
		497	o,
		498
		499	/// OP imm32
		500	i,
		501
542	/// OP r/m64, imm32	502	/// OP r/m64, imm32
543	mi,	503	mi,
544		504
...	@@ -547,223 +507,853 @@ const EncType = enum {	...	@@ -547,223 +507,853 @@ const EncType = enum {
547		507
548	/// OP r64, r/m64	508	/// OP r64, r/m64
549	rm,	509	rm,
		510
		511	/// OP r64, imm64
		512	oi,
		513
		514	/// OP al/ax/eax/rax, moffs
		515	fd,
		516
		517	/// OP moffs, al/ax/eax/rax
		518	td,
		519
		520	/// OP r64, r/m64, imm32
		521	rmi,
550	};	522	};
551		523
552	const OpCode = struct {	524	const OpCode = union(enum) {
553	opc: u8,	525	one_byte: u8,
554	/// Only used if `EncType == .mi`.	526	two_byte: struct { _1: u8, _2: u8 },
555	modrm_ext: u3,	527
		528	fn oneByte(opc: u8) OpCode {
		529	return .{ .one_byte = opc };
		530	}
		531
		532	fn twoByte(opc1: u8, opc2: u8) OpCode {
		533	return .{ .two_byte = .{ ._1 = opc1, ._2 = opc2 } };
		534	}
		535
		536	fn encode(opc: OpCode, encoder: Encoder) void {
		537	switch (opc) {
		538	.one_byte => \|v\| encoder.opcode_1byte(v),
		539	.two_byte => \|v\| encoder.opcode_2byte(v._1, v._2),
		540	}
		541	}
		542
		543	fn encodeWithReg(opc: OpCode, encoder: Encoder, reg: Register) void {
		544	assert(opc == .one_byte);
		545	encoder.opcode_withReg(opc.one_byte, reg.lowId());
		546	}
556	};	547	};
557		548
558	inline fn getArithOpCode(tag: Mir.Inst.Tag, enc: EncType) OpCode {	549	inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode {
559	switch (enc) {	550	switch (enc) {
		551	.zo => return switch (tag) {
		552	.ret_near => OpCode.oneByte(0xc3),
		553	.ret_far => OpCode.oneByte(0xcb),
		554	.brk => OpCode.oneByte(0xcc),
		555	.nop => OpCode.oneByte(0x90),
		556	.syscall => OpCode.twoByte(0x0f, 0x05),
		557	else => null,
		558	},
		559	.d => return switch (tag) {
		560	.jmp_near => OpCode.oneByte(0xe9),
		561	.call_near => OpCode.oneByte(0xe8),
		562	.jo => if (is_one_byte) OpCode.oneByte(0x70) else OpCode.twoByte(0x0f, 0x80),
		563	.jno => if (is_one_byte) OpCode.oneByte(0x71) else OpCode.twoByte(0x0f, 0x81),
		564	.jb, .jc, .jnae => if (is_one_byte) OpCode.oneByte(0x72) else OpCode.twoByte(0x0f, 0x82),
		565	.jnb, .jnc, .jae => if (is_one_byte) OpCode.oneByte(0x73) else OpCode.twoByte(0x0f, 0x83),
		566	.je, .jz => if (is_one_byte) OpCode.oneByte(0x74) else OpCode.twoByte(0x0f, 0x84),
		567	.jne, .jnz => if (is_one_byte) OpCode.oneByte(0x75) else OpCode.twoByte(0x0f, 0x85),
		568	.jna, .jbe => if (is_one_byte) OpCode.oneByte(0x76) else OpCode.twoByte(0x0f, 0x86),
		569	.jnbe, .ja => if (is_one_byte) OpCode.oneByte(0x77) else OpCode.twoByte(0x0f, 0x87),
		570	.js => if (is_one_byte) OpCode.oneByte(0x78) else OpCode.twoByte(0x0f, 0x88),
		571	.jns => if (is_one_byte) OpCode.oneByte(0x79) else OpCode.twoByte(0x0f, 0x89),
		572	.jpe, .jp => if (is_one_byte) OpCode.oneByte(0x7a) else OpCode.twoByte(0x0f, 0x8a),
		573	.jpo, .jnp => if (is_one_byte) OpCode.oneByte(0x7b) else OpCode.twoByte(0x0f, 0x8b),
		574	.jnge, .jl => if (is_one_byte) OpCode.oneByte(0x7c) else OpCode.twoByte(0x0f, 0x8c),
		575	.jge, .jnl => if (is_one_byte) OpCode.oneByte(0x7d) else OpCode.twoByte(0x0f, 0x8d),
		576	.jle, .jng => if (is_one_byte) OpCode.oneByte(0x7e) else OpCode.twoByte(0x0f, 0x8e),
		577	.jg, .jnle => if (is_one_byte) OpCode.oneByte(0x7f) else OpCode.twoByte(0x0f, 0x8f),
		578	else => null,
		579	},
		580	.m => return switch (tag) {
		581	.jmp_near, .call_near, .push => OpCode.oneByte(0xff),
		582	.pop => OpCode.oneByte(0x8f),
		583	.seto => OpCode.twoByte(0x0f, 0x90),
		584	.setno => OpCode.twoByte(0x0f, 0x91),
		585	.setb, .setc, .setnae => OpCode.twoByte(0x0f, 0x92),
		586	.setnb, .setnc, .setae => OpCode.twoByte(0x0f, 0x93),
		587	.sete, .setz => OpCode.twoByte(0x0f, 0x94),
		588	.setne, .setnz => OpCode.twoByte(0x0f, 0x95),
		589	.setbe, .setna => OpCode.twoByte(0x0f, 0x96),
		590	.seta, .setnbe => OpCode.twoByte(0x0f, 0x97),
		591	.sets => OpCode.twoByte(0x0f, 0x98),
		592	.setns => OpCode.twoByte(0x0f, 0x99),
		593	.setp, .setpe => OpCode.twoByte(0x0f, 0x9a),
		594	.setnp, .setop => OpCode.twoByte(0x0f, 0x9b),
		595	.setl, .setnge => OpCode.twoByte(0x0f, 0x9c),
		596	.setnl, .setge => OpCode.twoByte(0x0f, 0x9d),
		597	.setle, .setng => OpCode.twoByte(0x0f, 0x9e),
		598	.setnle, .setg => OpCode.twoByte(0x0f, 0x9f),
		599	else => null,
		600	},
		601	.o => return switch (tag) {
		602	.push => OpCode.oneByte(0x50),
		603	.pop => OpCode.oneByte(0x58),
		604	else => null,
		605	},
		606	.i => return switch (tag) {
		607	.push => OpCode.oneByte(if (is_one_byte) 0x6a else 0x68),
		608	.@"test" => OpCode.oneByte(if (is_one_byte) 0xa8 else 0xa9),
		609	.ret_near => OpCode.oneByte(0xc2),
		610	.ret_far => OpCode.oneByte(0xca),
		611	else => null,
		612	},
560	.mi => return switch (tag) {	613	.mi => return switch (tag) {
561	.adc => .{ .opc = 0x81, .modrm_ext = 0x2 },	614	.adc, .add, .sub, .xor, .@"and", .@"or", .sbb, .cmp => OpCode.oneByte(if (is_one_byte) 0x80 else 0x81),
562	.add => .{ .opc = 0x81, .modrm_ext = 0x0 },	615	.mov => OpCode.oneByte(if (is_one_byte) 0xc6 else 0xc7),
563	.sub => .{ .opc = 0x81, .modrm_ext = 0x5 },	616	.@"test" => OpCode.oneByte(if (is_one_byte) 0xf6 else 0xf7),
564	.xor => .{ .opc = 0x81, .modrm_ext = 0x6 },	617	else => null,
565	.@"and" => .{ .opc = 0x81, .modrm_ext = 0x4 },
566	.@"or" => .{ .opc = 0x81, .modrm_ext = 0x1 },
567	.sbb => .{ .opc = 0x81, .modrm_ext = 0x3 },
568	.cmp => .{ .opc = 0x81, .modrm_ext = 0x7 },
569	.mov => .{ .opc = 0xc7, .modrm_ext = 0x0 },
570	else => unreachable,
571	},	618	},
572	.mr => {	619	.mr => return switch (tag) {
573	const opc: u8 = switch (tag) {	620	.adc => OpCode.oneByte(if (is_one_byte) 0x10 else 0x11),
574	.adc => 0x11,	621	.add => OpCode.oneByte(if (is_one_byte) 0x00 else 0x01),
575	.add => 0x01,	622	.sub => OpCode.oneByte(if (is_one_byte) 0x28 else 0x29),
576	.sub => 0x29,	623	.xor => OpCode.oneByte(if (is_one_byte) 0x30 else 0x31),
577	.xor => 0x31,	624	.@"and" => OpCode.oneByte(if (is_one_byte) 0x20 else 0x21),
578	.@"and" => 0x21,	625	.@"or" => OpCode.oneByte(if (is_one_byte) 0x08 else 0x09),
579	.@"or" => 0x09,	626	.sbb => OpCode.oneByte(if (is_one_byte) 0x18 else 0x19),
580	.sbb => 0x19,	627	.cmp => OpCode.oneByte(if (is_one_byte) 0x38 else 0x39),
581	.cmp => 0x39,	628	.mov => OpCode.oneByte(if (is_one_byte) 0x88 else 0x89),
582	.mov => 0x89,	629	else => null,
583	else => unreachable,
584	};
585	return .{ .opc = opc, .modrm_ext = undefined };
586	},	630	},
587	.rm => {	631	.rm => return switch (tag) {
588	const opc: u8 = switch (tag) {	632	.adc => OpCode.oneByte(if (is_one_byte) 0x12 else 0x13),
589	.adc => 0x13,	633	.add => OpCode.oneByte(if (is_one_byte) 0x02 else 0x03),
590	.add => 0x03,	634	.sub => OpCode.oneByte(if (is_one_byte) 0x2a else 0x2b),
591	.sub => 0x2b,	635	.xor => OpCode.oneByte(if (is_one_byte) 0x32 else 0x33),
592	.xor => 0x33,	636	.@"and" => OpCode.oneByte(if (is_one_byte) 0x22 else 0x23),
593	.@"and" => 0x23,	637	.@"or" => OpCode.oneByte(if (is_one_byte) 0x0b else 0x0b),
594	.@"or" => 0x0b,	638	.sbb => OpCode.oneByte(if (is_one_byte) 0x1a else 0x1b),
595	.sbb => 0x1b,	639	.cmp => OpCode.oneByte(if (is_one_byte) 0x3a else 0x3b),
596	.cmp => 0x3b,	640	.mov => OpCode.oneByte(if (is_one_byte) 0x8a else 0x8b),
597	.mov => 0x8b,	641	.lea => OpCode.oneByte(if (is_one_byte) 0x8c else 0x8d),
598	else => unreachable,	642	.imul => OpCode.twoByte(0x0f, 0xaf),
599	};	643	else => null,
600	return .{ .opc = opc, .modrm_ext = undefined };	644	},
		645	.oi => return switch (tag) {
		646	.mov => OpCode.oneByte(if (is_one_byte) 0xb0 else 0xb8),
		647	else => null,
		648	},
		649	.fd => return switch (tag) {
		650	.mov => OpCode.oneByte(if (is_one_byte) 0xa0 else 0xa1),
		651	else => null,
		652	},
		653	.td => return switch (tag) {
		654	.mov => OpCode.oneByte(if (is_one_byte) 0xa2 else 0xa3),
		655	else => null,
		656	},
		657	.rmi => return switch (tag) {
		658	.imul => OpCode.oneByte(if (is_one_byte) 0x6b else 0x69),
		659	else => null,
601	},	660	},
602	}	661	}
603	}	662	}
604		663
605	fn mirArith(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {	664	inline fn getModRmExt(tag: Tag) ?u3 {
606	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);	665	return switch (tag) {
607	switch (ops.flags) {	666	.adc => 0x2,
608	0b00 => blk: {	667	.add => 0x0,
609	if (ops.reg2 == .none) {	668	.sub => 0x5,
610	// OP reg1, imm32	669	.xor => 0x6,
611	// OP r/m64, imm32	670	.@"and" => 0x4,
612	const imm = emit.mir.instructions.items(.data)[inst].imm;	671	.@"or" => 0x1,
613	const opcode = getArithOpCode(tag, .mi);	672	.sbb => 0x3,
614	const encoder = try Encoder.init(emit.code, 7);	673	.cmp => 0x7,
		674	.mov => 0x0,
		675	.jmp_near => 0x4,
		676	.call_near => 0x2,
		677	.push => 0x6,
		678	.pop => 0x0,
		679	.@"test" => 0x0,
		680	.seto,
		681	.setno,
		682	.setb,
		683	.setc,
		684	.setnae,
		685	.setnb,
		686	.setnc,
		687	.setae,
		688	.sete,
		689	.setz,
		690	.setne,
		691	.setnz,
		692	.setbe,
		693	.setna,
		694	.seta,
		695	.setnbe,
		696	.sets,
		697	.setns,
		698	.setp,
		699	.setpe,
		700	.setnp,
		701	.setop,
		702	.setl,
		703	.setnge,
		704	.setnl,
		705	.setge,
		706	.setle,
		707	.setng,
		708	.setnle,
		709	.setg,
		710	=> 0x0,
		711	else => null,
		712	};
		713	}
		714
		715	const ScaleIndexBase = struct {
		716	scale: u2,
		717	index_reg: ?Register,
		718	base_reg: ?Register,
		719	};
		720
		721	const Memory = struct {
		722	reg: ?Register,
		723	rip: bool = false,
		724	disp: i32,
		725	sib: ?ScaleIndexBase = null,
		726	};
		727
		728	const RegisterOrMemory = union(enum) {
		729	register: Register,
		730	memory: Memory,
		731
		732	fn reg(register: Register) RegisterOrMemory {
		733	return .{ .register = register };
		734	}
		735
		736	fn mem(register: ?Register, disp: i32) RegisterOrMemory {
		737	return .{
		738	.memory = .{
		739	.reg = register,
		740	.disp = disp,
		741	},
		742	};
		743	}
		744
		745	fn rip(disp: i32) RegisterOrMemory {
		746	return .{
		747	.memory = .{
		748	.reg = null,
		749	.rip = true,
		750	.disp = disp,
		751	},
		752	};
		753	}
		754	};
		755
		756	fn lowerToZoEnc(tag: Tag, code: *std.ArrayList(u8)) InnerError!void {
		757	const opc = getOpCode(tag, .zo, false).?;
		758	const encoder = try Encoder.init(code, 1);
		759	opc.encode(encoder);
		760	}
		761
		762	fn lowerToIEnc(tag: Tag, imm: i32, code: *std.ArrayList(u8)) InnerError!void {
		763	if (tag == .ret_far or tag == .ret_near) {
		764	const encoder = try Encoder.init(code, 3);
		765	const opc = getOpCode(tag, .i, false).?;
		766	opc.encode(encoder);
		767	encoder.imm16(@intCast(i16, imm));
		768	return;
		769	}
		770	const opc = getOpCode(tag, .i, immOpSize(imm) == 8).?;
		771	const encoder = try Encoder.init(code, 5);
		772	if (immOpSize(imm) == 16) {
		773	encoder.opcode_1byte(0x66);
		774	}
		775	opc.encode(encoder);
		776	if (immOpSize(imm) == 8) {
		777	encoder.imm8(@intCast(i8, imm));
		778	} else if (immOpSize(imm) == 16) {
		779	encoder.imm16(@intCast(i16, imm));
		780	} else {
		781	encoder.imm32(imm);
		782	}
		783	}
		784
		785	fn lowerToOEnc(tag: Tag, reg: Register, code: *std.ArrayList(u8)) InnerError!void {
		786	if (reg.size() != 16 and reg.size() != 64) return error.EmitFail; // TODO correct for push/pop, but is it universal?
		787	const opc = getOpCode(tag, .o, false).?;
		788	const encoder = try Encoder.init(code, 3);
		789	if (reg.size() == 16) {
		790	encoder.opcode_1byte(0x66);
		791	}
		792	encoder.rex(.{
		793	.w = false,
		794	.b = reg.isExtended(),
		795	});
		796	opc.encodeWithReg(encoder, reg);
		797	}
		798
		799	fn lowerToDEnc(tag: Tag, imm: i32, code: *std.ArrayList(u8)) InnerError!void {
		800	const opc = getOpCode(tag, .d, false).?;
		801	const encoder = try Encoder.init(code, 6);
		802	opc.encode(encoder);
		803	encoder.imm32(imm);
		804	}
		805
		806	fn lowerToMEnc(tag: Tag, reg_or_mem: RegisterOrMemory, code: *std.ArrayList(u8)) InnerError!void {
		807	const opc = getOpCode(tag, .m, false).?;
		808	const modrm_ext = getModRmExt(tag).?;
		809	switch (reg_or_mem) {
		810	.register => \|reg\| {
		811	// TODO clean this up!
		812	if (reg.size() != 64) {
		813	if (reg.size() != 8 and !tag.isSetCC()) return error.EmitFail;
		814	}
		815	const encoder = try Encoder.init(code, 3);
		816	encoder.rex(.{
		817	.w = tag.isSetCC(),
		818	.b = reg.isExtended(),
		819	});
		820	opc.encode(encoder);
		821	encoder.modRm_direct(modrm_ext, reg.lowId());
		822	},
		823	.memory => \|mem_op\| {
		824	const encoder = try Encoder.init(code, 8);
		825	if (mem_op.reg) \|reg\| {
		826	// TODO clean this up!
		827	if (reg.size() != 64) {
		828	if (reg.size() != 8 and !tag.isSetCC()) return error.EmitFail;
		829	}
615	encoder.rex(.{	830	encoder.rex(.{
616	.w = ops.reg1.size() == 64,	831	.w = tag.isSetCC(),
617	.b = ops.reg1.isExtended(),	832	.b = reg.isExtended(),
618	});	833	});
619	if (tag != .mov and imm <= math.maxInt(i8)) {	834	opc.encode(encoder);
620	encoder.opcode_1byte(opcode.opc + 2);	835	if (reg.lowId() == 4) {
621	encoder.modRm_direct(opcode.modrm_ext, ops.reg1.lowId());	836	if (mem_op.disp == 0) {
622	encoder.imm8(@intCast(i8, imm));	837	encoder.modRm_SIBDisp0(modrm_ext);
		838	encoder.sib_base(reg.lowId());
		839	} else if (immOpSize(mem_op.disp) == 8) {
		840	encoder.modRm_SIBDisp8(modrm_ext);
		841	encoder.sib_baseDisp8(reg.lowId());
		842	encoder.disp8(@intCast(i8, mem_op.disp));
		843	} else {
		844	encoder.modRm_SIBDisp32(modrm_ext);
		845	encoder.sib_baseDisp32(reg.lowId());
		846	encoder.disp32(mem_op.disp);
		847	}
623	} else {	848	} else {
624	encoder.opcode_1byte(opcode.opc);	849	if (mem_op.disp == 0) {
625	encoder.modRm_direct(opcode.modrm_ext, ops.reg1.lowId());	850	encoder.modRm_indirectDisp0(modrm_ext, reg.lowId());
626	encoder.imm32(imm);	851	} else if (immOpSize(mem_op.disp) == 8) {
		852	encoder.modRm_indirectDisp8(modrm_ext, reg.lowId());
		853	encoder.disp8(@intCast(i8, mem_op.disp));
		854	} else {
		855	encoder.modRm_indirectDisp32(modrm_ext, reg.lowId());
		856	encoder.disp32(mem_op.disp);
		857	}
627	}	858	}
628	break :blk;	859	} else {
		860	opc.encode(encoder);
		861	if (mem_op.rip) {
		862	encoder.modRm_RIPDisp32(modrm_ext);
		863	} else {
		864	encoder.modRm_SIBDisp0(modrm_ext);
		865	encoder.sib_disp32();
		866	}
		867	encoder.disp32(mem_op.disp);
		868	}
		869	},
		870	}
		871	}
		872
		873	fn lowerToTdEnc(tag: Tag, moffs: i64, reg: Register, code: *std.ArrayList(u8)) InnerError!void {
		874	return lowerToTdFdEnc(tag, reg, moffs, code, true);
		875	}
		876
		877	fn lowerToFdEnc(tag: Tag, reg: Register, moffs: i64, code: *std.ArrayList(u8)) InnerError!void {
		878	return lowerToTdFdEnc(tag, reg, moffs, code, false);
		879	}
		880
		881	fn lowerToTdFdEnc(tag: Tag, reg: Register, moffs: i64, code: *std.ArrayList(u8), td: bool) InnerError!void {
		882	if (reg.lowId() != Register.rax.lowId()) return error.EmitFail;
		883	if (reg.size() != immOpSize(moffs)) return error.EmitFail;
		884	const opc = if (td)
		885	getOpCode(tag, .td, reg.size() == 8).?
		886	else
		887	getOpCode(tag, .fd, reg.size() == 8).?;
		888	const encoder = try Encoder.init(code, 10);
		889	if (reg.size() == 16) {
		890	encoder.opcode_1byte(0x66);
		891	}
		892	encoder.rex(.{
		893	.w = reg.size() == 64,
		894	});
		895	opc.encode(encoder);
		896	switch (reg.size()) {
		897	8 => {
		898	const moffs8 = try math.cast(i8, moffs);
		899	encoder.imm8(moffs8);
		900	},
		901	16 => {
		902	const moffs16 = try math.cast(i16, moffs);
		903	encoder.imm16(moffs16);
		904	},
		905	32 => {
		906	const moffs32 = try math.cast(i32, moffs);
		907	encoder.imm32(moffs32);
		908	},
		909	64 => {
		910	encoder.imm64(@bitCast(u64, moffs));
		911	},
		912	else => unreachable,
		913	}
		914	}
		915
		916	fn lowerToOiEnc(tag: Tag, reg: Register, imm: i64, code: *std.ArrayList(u8)) InnerError!void {
		917	if (reg.size() != immOpSize(imm)) return error.EmitFail;
		918	const opc = getOpCode(tag, .oi, reg.size() == 8).?;
		919	const encoder = try Encoder.init(code, 10);
		920	if (reg.size() == 16) {
		921	encoder.opcode_1byte(0x66);
		922	}
		923	encoder.rex(.{
		924	.w = reg.size() == 64,
		925	.b = reg.isExtended(),
		926	});
		927	opc.encodeWithReg(encoder, reg);
		928	switch (reg.size()) {
		929	8 => {
		930	const imm8 = try math.cast(i8, imm);
		931	encoder.imm8(imm8);
		932	},
		933	16 => {
		934	const imm16 = try math.cast(i16, imm);
		935	encoder.imm16(imm16);
		936	},
		937	32 => {
		938	const imm32 = try math.cast(i32, imm);
		939	encoder.imm32(imm32);
		940	},
		941	64 => {
		942	encoder.imm64(@bitCast(u64, imm));
		943	},
		944	else => unreachable,
		945	}
		946	}
		947
		948	fn lowerToMiEnc(tag: Tag, reg_or_mem: RegisterOrMemory, imm: i32, code: *std.ArrayList(u8)) InnerError!void {
		949	const modrm_ext = getModRmExt(tag).?;
		950	switch (reg_or_mem) {
		951	.register => \|dst_reg\| {
		952	const opc = getOpCode(tag, .mi, dst_reg.size() == 8).?;
		953	const encoder = try Encoder.init(code, 7);
		954	if (dst_reg.size() == 16) {
		955	// 0x66 prefix switches to the non-default size; here we assume a switch from
		956	// the default 32bits to 16bits operand-size.
		957	// More info: https://www.cs.uni-potsdam.de/desn/lehre/ss15/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf#page=32&zoom=auto,-159,773
		958	encoder.opcode_1byte(0x66);
629	}	959	}
630	// OP reg1, reg2
631	// OP r/m64, r64
632	const opcode = getArithOpCode(tag, .mr);
633	const opc = if (ops.reg1.size() == 8) opcode.opc - 1 else opcode.opc;
634	const encoder = try Encoder.init(emit.code, 3);
635	encoder.rex(.{	960	encoder.rex(.{
636	.w = ops.reg1.size() == 64 and ops.reg2.size() == 64,	961	.w = dst_reg.size() == 64,
637	.r = ops.reg1.isExtended(),	962	.b = dst_reg.isExtended(),
638	.b = ops.reg2.isExtended(),
639	});	963	});
640	encoder.opcode_1byte(opc);	964	opc.encode(encoder);
641	encoder.modRm_direct(ops.reg1.lowId(), ops.reg2.lowId());	965	encoder.modRm_direct(modrm_ext, dst_reg.lowId());
		966	switch (dst_reg.size()) {
		967	8 => {
		968	const imm8 = try math.cast(i8, imm);
		969	encoder.imm8(imm8);
		970	},
		971	16 => {
		972	const imm16 = try math.cast(i16, imm);
		973	encoder.imm16(imm16);
		974	},
		975	32, 64 => encoder.imm32(imm),
		976	else => unreachable,
		977	}
642	},	978	},
643	0b01 => blk: {	979	.memory => \|dst_mem\| {
644	const imm = emit.mir.instructions.items(.data)[inst].imm;	980	const opc = getOpCode(tag, .mi, false).?;
645	const opcode = getArithOpCode(tag, .rm);	981	const encoder = try Encoder.init(code, 12);
646	const opc = if (ops.reg1.size() == 8) opcode.opc - 1 else opcode.opc;	982	if (dst_mem.reg) \|dst_reg\| {
647	if (ops.reg2 == .none) {	983	// Register dst_reg can either be 64bit or 32bit in size.
648	// OP reg1, [imm32]	984	// TODO for memory operand, immediate operand pair, we currently
649	// OP r64, r/m64	985	// have no way of flagging whether the immediate can be 8-, 16- or
650	const encoder = try Encoder.init(emit.code, 8);	986	// 32-bit and whether the corresponding memory operand is respectively
		987	// a byte, word or dword ptr.
		988	// TODO we currently don't have a way to flag imm32 64bit sign extended
		989	if (dst_reg.size() != 64) return error.EmitFail;
651	encoder.rex(.{	990	encoder.rex(.{
652	.w = ops.reg1.size() == 64,	991	.w = false,
653	.b = ops.reg1.isExtended(),	992	.b = dst_reg.isExtended(),
654	});	993	});
655	encoder.opcode_1byte(opc);	994	opc.encode(encoder);
656	encoder.modRm_SIBDisp0(ops.reg1.lowId());	995	if (dst_reg.lowId() == 4) {
657	encoder.sib_disp32();	996	if (dst_mem.disp == 0) {
658	encoder.disp32(imm);	997	encoder.modRm_SIBDisp0(modrm_ext);
659	break :blk;	998	encoder.sib_base(dst_reg.lowId());
		999	} else if (immOpSize(dst_mem.disp) == 8) {
		1000	encoder.modRm_SIBDisp8(modrm_ext);
		1001	encoder.sib_baseDisp8(dst_reg.lowId());
		1002	encoder.disp8(@intCast(i8, dst_mem.disp));
		1003	} else {
		1004	encoder.modRm_SIBDisp32(modrm_ext);
		1005	encoder.sib_baseDisp32(dst_reg.lowId());
		1006	encoder.disp32(dst_mem.disp);
		1007	}
		1008	} else {
		1009	if (dst_mem.disp == 0) {
		1010	encoder.modRm_indirectDisp0(modrm_ext, dst_reg.lowId());
		1011	} else if (immOpSize(dst_mem.disp) == 8) {
		1012	encoder.modRm_indirectDisp8(modrm_ext, dst_reg.lowId());
		1013	encoder.disp8(@intCast(i8, dst_mem.disp));
		1014	} else {
		1015	encoder.modRm_indirectDisp32(modrm_ext, dst_reg.lowId());
		1016	encoder.disp32(dst_mem.disp);
		1017	}
		1018	}
		1019	} else {
		1020	opc.encode(encoder);
		1021	if (dst_mem.rip) {
		1022	encoder.modRm_RIPDisp32(modrm_ext);
		1023	} else {
		1024	encoder.modRm_SIBDisp0(modrm_ext);
		1025	encoder.sib_disp32();
		1026	}
		1027	encoder.disp32(dst_mem.disp);
660	}	1028	}
661	// OP reg1, [reg2 + imm32]	1029	encoder.imm32(imm);
662	// OP r64, r/m64	1030	},
663	const encoder = try Encoder.init(emit.code, 7);	1031	}
		1032	}
		1033
		1034	fn lowerToRmEnc(
		1035	tag: Tag,
		1036	reg: Register,
		1037	reg_or_mem: RegisterOrMemory,
		1038	code: *std.ArrayList(u8),
		1039	) InnerError!void {
		1040	const opc = getOpCode(tag, .rm, reg.size() == 8).?;
		1041	switch (reg_or_mem) {
		1042	.register => \|src_reg\| {
		1043	if (reg.size() != src_reg.size()) return error.EmitFail;
		1044	const encoder = try Encoder.init(code, 3);
664	encoder.rex(.{	1045	encoder.rex(.{
665	.w = ops.reg1.size() == 64,	1046	.w = reg.size() == 64,
666	.r = ops.reg1.isExtended(),	1047	.r = reg.isExtended(),
667	.b = ops.reg2.isExtended(),	1048	.b = src_reg.isExtended(),
668	});	1049	});
669	encoder.opcode_1byte(opc);	1050	opc.encode(encoder);
670	if (imm <= math.maxInt(i8)) {	1051	encoder.modRm_direct(reg.lowId(), src_reg.lowId());
671	encoder.modRm_indirectDisp8(ops.reg1.lowId(), ops.reg2.lowId());	1052	},
672	encoder.disp8(@intCast(i8, imm));	1053	.memory => \|src_mem\| {
		1054	const encoder = try Encoder.init(code, 9);
		1055	if (reg.size() == 16) {
		1056	encoder.opcode_1byte(0x66);
		1057	}
		1058	if (src_mem.reg) \|src_reg\| {
		1059	// TODO handle 32-bit base register - requires prefix 0x67
		1060	// Intel Manual, Vol 1, chapter 3.6 and 3.6.1
		1061	if (src_reg.size() != 64) return error.EmitFail;
		1062	encoder.rex(.{
		1063	.w = reg.size() == 64,
		1064	.r = reg.isExtended(),
		1065	.b = src_reg.isExtended(),
		1066	});
		1067	opc.encode(encoder);
		1068	if (src_reg.lowId() == 4) {
		1069	if (src_mem.disp == 0) {
		1070	encoder.modRm_SIBDisp0(reg.lowId());
		1071	encoder.sib_base(src_reg.lowId());
		1072	} else if (immOpSize(src_mem.disp) == 8) {
		1073	encoder.modRm_SIBDisp8(reg.lowId());
		1074	encoder.sib_baseDisp8(src_reg.lowId());
		1075	encoder.disp8(@intCast(i8, src_mem.disp));
		1076	} else {
		1077	encoder.modRm_SIBDisp32(reg.lowId());
		1078	encoder.sib_baseDisp32(src_reg.lowId());
		1079	encoder.disp32(src_mem.disp);
		1080	}
		1081	} else {
		1082	if (src_mem.disp == 0) {
		1083	encoder.modRm_indirectDisp0(reg.lowId(), src_reg.lowId());
		1084	} else if (immOpSize(src_mem.disp) == 8) {
		1085	encoder.modRm_indirectDisp8(reg.lowId(), src_reg.lowId());
		1086	encoder.disp8(@intCast(i8, src_mem.disp));
		1087	} else {
		1088	encoder.modRm_indirectDisp32(reg.lowId(), src_reg.lowId());
		1089	encoder.disp32(src_mem.disp);
		1090	}
		1091	}
673	} else {	1092	} else {
674	encoder.modRm_indirectDisp32(ops.reg1.lowId(), ops.reg2.lowId());	1093	encoder.rex(.{
675	encoder.disp32(imm);	1094	.w = reg.size() == 64,
		1095	.r = reg.isExtended(),
		1096	});
		1097	opc.encode(encoder);
		1098	if (src_mem.rip) {
		1099	encoder.modRm_RIPDisp32(reg.lowId());
		1100	} else {
		1101	encoder.modRm_SIBDisp0(reg.lowId());
		1102	encoder.sib_disp32();
		1103	}
		1104	encoder.disp32(src_mem.disp);
676	}	1105	}
677	},	1106	},
678	0b10 => blk: {	1107	}
679	if (ops.reg2 == .none) {	1108	}
680	// OP [reg1 + 0], imm32	1109
681	// OP r/m64, imm32	1110	fn lowerToMrEnc(
682	const imm = emit.mir.instructions.items(.data)[inst].imm;	1111	tag: Tag,
683	const opcode = getArithOpCode(tag, .mi);	1112	reg_or_mem: RegisterOrMemory,
684	const opc = if (ops.reg1.size() == 8) opcode.opc - 1 else opcode.opc;	1113	reg: Register,
685	const encoder = try Encoder.init(emit.code, 7);	1114	code: *std.ArrayList(u8),
		1115	) InnerError!void {
		1116	// We use size of source register reg to work out which
		1117	// variant of memory ptr to pick:
		1118	// * reg is 64bit - qword ptr
		1119	// * reg is 32bit - dword ptr
		1120	// * reg is 16bit - word ptr
		1121	// * reg is 8bit - byte ptr
		1122	const opc = getOpCode(tag, .mr, reg.size() == 8).?;
		1123	switch (reg_or_mem) {
		1124	.register => \|dst_reg\| {
		1125	if (dst_reg.size() != reg.size()) return error.EmitFail;
		1126	const encoder = try Encoder.init(code, 3);
		1127	encoder.rex(.{
		1128	.w = dst_reg.size() == 64,
		1129	.r = reg.isExtended(),
		1130	.b = dst_reg.isExtended(),
		1131	});
		1132	opc.encode(encoder);
		1133	encoder.modRm_direct(reg.lowId(), dst_reg.lowId());
		1134	},
		1135	.memory => \|dst_mem\| {
		1136	const encoder = try Encoder.init(code, 9);
		1137	if (reg.size() == 16) {
		1138	encoder.opcode_1byte(0x66);
		1139	}
		1140	if (dst_mem.reg) \|dst_reg\| {
		1141	if (dst_reg.size() != 64) return error.EmitFail;
		1142	encoder.rex(.{
		1143	.w = reg.size() == 64,
		1144	.r = reg.isExtended(),
		1145	.b = dst_reg.isExtended(),
		1146	});
		1147	opc.encode(encoder);
		1148	if (dst_reg.lowId() == 4) {
		1149	if (dst_mem.disp == 0) {
		1150	encoder.modRm_SIBDisp0(reg.lowId());
		1151	encoder.sib_base(dst_reg.lowId());
		1152	} else if (immOpSize(dst_mem.disp) == 8) {
		1153	encoder.modRm_SIBDisp8(reg.lowId());
		1154	encoder.sib_baseDisp8(dst_reg.lowId());
		1155	encoder.disp8(@intCast(i8, dst_mem.disp));
		1156	} else {
		1157	encoder.modRm_SIBDisp32(reg.lowId());
		1158	encoder.sib_baseDisp32(dst_reg.lowId());
		1159	encoder.disp32(dst_mem.disp);
		1160	}
		1161	} else {
		1162	if (dst_mem.disp == 0) {
		1163	encoder.modRm_indirectDisp0(reg.lowId(), dst_reg.lowId());
		1164	} else if (immOpSize(dst_mem.disp) == 8) {
		1165	encoder.modRm_indirectDisp8(reg.lowId(), dst_reg.lowId());
		1166	encoder.disp8(@intCast(i8, dst_mem.disp));
		1167	} else {
		1168	encoder.modRm_indirectDisp32(reg.lowId(), dst_reg.lowId());
		1169	encoder.disp32(dst_mem.disp);
		1170	}
		1171	}
		1172	} else {
686	encoder.rex(.{	1173	encoder.rex(.{
687	.w = ops.reg1.size() == 64,	1174	.w = reg.size() == 64,
688	.b = ops.reg1.isExtended(),	1175	.r = reg.isExtended(),
689	});	1176	});
690	encoder.opcode_1byte(opc);	1177	opc.encode(encoder);
691	encoder.modRm_indirectDisp0(opcode.modrm_ext, ops.reg1.lowId());	1178	if (dst_mem.rip) {
692	if (imm <= math.maxInt(i8)) {	1179	encoder.modRm_RIPDisp32(reg.lowId());
693	encoder.imm8(@intCast(i8, imm));
694	} else if (imm <= math.maxInt(i16)) {
695	encoder.imm16(@intCast(i16, imm));
696	} else {	1180	} else {
697	encoder.imm32(imm);	1181	encoder.modRm_SIBDisp0(reg.lowId());
		1182	encoder.sib_disp32();
698	}	1183	}
699	break :blk;	1184	encoder.disp32(dst_mem.disp);
700	}	1185	}
701	// OP [reg1 + imm32], reg2	1186	},
702	// OP r/m64, r64	1187	}
703	const imm = emit.mir.instructions.items(.data)[inst].imm;	1188	}
704	const opcode = getArithOpCode(tag, .mr);	1189
705	const opc = if (ops.reg1.size() == 8) opcode.opc - 1 else opcode.opc;	1190	fn lowerToRmiEnc(
706	const encoder = try Encoder.init(emit.code, 7);	1191	tag: Tag,
		1192	reg: Register,
		1193	reg_or_mem: RegisterOrMemory,
		1194	imm: i32,
		1195	code: *std.ArrayList(u8),
		1196	) InnerError!void {
		1197	const opc = getOpCode(tag, .rmi, reg.size() == 8).?;
		1198	switch (reg_or_mem) {
		1199	.register => \|src_reg\| {
		1200	if (reg.size() != src_reg.size()) return error.EmitFail;
		1201	const encoder = try Encoder.init(code, 7);
707	encoder.rex(.{	1202	encoder.rex(.{
708	.w = ops.reg2.size() == 64,	1203	.w = reg.size() == 64,
709	.r = ops.reg1.isExtended(),	1204	.r = reg.isExtended(),
710	.b = ops.reg2.isExtended(),	1205	.b = src_reg.isExtended(),
711	});	1206	});
712	encoder.opcode_1byte(opc);	1207	opc.encode(encoder);
713	if (imm <= math.maxInt(i8)) {	1208	encoder.modRm_direct(reg.lowId(), src_reg.lowId());
714	encoder.modRm_indirectDisp8(ops.reg1.lowId(), ops.reg2.lowId());	1209	switch (reg.size()) {
715	encoder.disp8(@intCast(i8, imm));	1210	8 => {
716	} else {	1211	const imm8 = try math.cast(i8, imm);
717	encoder.modRm_indirectDisp32(ops.reg1.lowId(), ops.reg2.lowId());	1212	encoder.imm8(imm8);
718	encoder.disp32(imm);	1213	},
		1214	16 => {
		1215	const imm16 = try math.cast(i16, imm);
		1216	encoder.imm16(imm16);
		1217	},
		1218	32, 64 => encoder.imm32(imm),
		1219	else => unreachable,
719	}	1220	}
720	},	1221	},
721	0b11 => blk: {	1222	.memory => \|src_mem\| {
722	if (ops.reg2 == .none) {	1223	const encoder = try Encoder.init(code, 13);
723	// OP [reg1 + imm32], imm32	1224	if (reg.size() == 16) {
724	// OP r/m64, imm32	1225	encoder.opcode_1byte(0x66);
725	const payload = emit.mir.instructions.items(.data)[inst].payload;	1226	}
726	const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;	1227	if (src_mem.reg) \|src_reg\| {
727	const opcode = getArithOpCode(tag, .mi);	1228	// TODO handle 32-bit base register - requires prefix 0x67
728	const opc = if (ops.reg1.size() == 8) opcode.opc - 1 else opcode.opc;	1229	// Intel Manual, Vol 1, chapter 3.6 and 3.6.1
729	const encoder = try Encoder.init(emit.code, 11);	1230	if (src_reg.size() != 64) return error.EmitFail;
730	encoder.rex(.{	1231	encoder.rex(.{
731	.w = false,	1232	.w = reg.size() == 64,
732	.b = ops.reg1.isExtended(),	1233	.r = reg.isExtended(),
		1234	.b = src_reg.isExtended(),
733	});	1235	});
734	encoder.opcode_1byte(opc);	1236	opc.encode(encoder);
735	if (imm_pair.dest_off <= math.maxInt(i8)) {	1237	if (src_reg.lowId() == 4) {
736	encoder.modRm_indirectDisp8(opcode.modrm_ext, ops.reg1.lowId());	1238	if (src_mem.disp == 0) {
737	encoder.disp8(@intCast(i8, imm_pair.dest_off));	1239	encoder.modRm_SIBDisp0(reg.lowId());
		1240	encoder.sib_base(src_reg.lowId());
		1241	} else if (immOpSize(src_mem.disp) == 8) {
		1242	encoder.modRm_SIBDisp8(reg.lowId());
		1243	encoder.sib_baseDisp8(src_reg.lowId());
		1244	encoder.disp8(@intCast(i8, src_mem.disp));
		1245	} else {
		1246	encoder.modRm_SIBDisp32(reg.lowId());
		1247	encoder.sib_baseDisp32(src_reg.lowId());
		1248	encoder.disp32(src_mem.disp);
		1249	}
738	} else {	1250	} else {
739	encoder.modRm_indirectDisp32(opcode.modrm_ext, ops.reg1.lowId());	1251	if (src_mem.disp == 0) {
740	encoder.disp32(imm_pair.dest_off);	1252	encoder.modRm_indirectDisp0(reg.lowId(), src_reg.lowId());
		1253	} else if (immOpSize(src_mem.disp) == 8) {
		1254	encoder.modRm_indirectDisp8(reg.lowId(), src_reg.lowId());
		1255	encoder.disp8(@intCast(i8, src_mem.disp));
		1256	} else {
		1257	encoder.modRm_indirectDisp32(reg.lowId(), src_reg.lowId());
		1258	encoder.disp32(src_mem.disp);
		1259	}
741	}	1260	}
742	encoder.imm32(imm_pair.operand);	1261	} else {
743	break :blk;	1262	encoder.rex(.{
		1263	.w = reg.size() == 64,
		1264	.r = reg.isExtended(),
		1265	});
		1266	opc.encode(encoder);
		1267	if (src_mem.rip) {
		1268	encoder.modRm_RIPDisp32(reg.lowId());
		1269	} else {
		1270	encoder.modRm_SIBDisp0(reg.lowId());
		1271	encoder.sib_disp32();
		1272	}
		1273	encoder.disp32(src_mem.disp);
744	}	1274	}
745	// TODO clearly mov doesn't belong here; for other, arithemtic ops,	1275	encoder.imm32(imm);
746	// this is the same as 0b00.
747	const opcode = getArithOpCode(tag, if (tag == .mov) .rm else .mr);
748	const opc = if (ops.reg1.size() == 8) opcode.opc - 1 else opcode.opc;
749	const encoder = try Encoder.init(emit.code, 3);
750	encoder.rex(.{
751	.w = ops.reg1.size() == 64 and ops.reg2.size() == 64,
752	.r = ops.reg1.isExtended(),
753	.b = ops.reg2.isExtended(),
754	});
755	encoder.opcode_1byte(opc);
756	encoder.modRm_direct(ops.reg1.lowId(), ops.reg2.lowId());
757	},	1276	},
758	}	1277	}
759	}	1278	}
760		1279
761	fn mirArithScaleSrc(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {	1280	fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
		1281	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
		1282	switch (ops.flags) {
		1283	0b00 => {
		1284	if (ops.reg2 == .none) {
		1285	// mov reg1, imm32
		1286	// MI
		1287	const imm = emit.mir.instructions.items(.data)[inst].imm;
		1288	return lowerToMiEnc(tag, RegisterOrMemory.reg(ops.reg1), imm, emit.code);
		1289	}
		1290	// mov reg1, reg2
		1291	// RM
		1292	return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
		1293	},
		1294	0b01 => {
		1295	const imm = emit.mir.instructions.items(.data)[inst].imm;
		1296	if (ops.reg2 == .none) {
		1297	// mov reg1, [imm32]
		1298	// RM
		1299	return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(null, imm), emit.code);
		1300	}
		1301	// mov reg1, [reg2 + imm32]
		1302	// RM
		1303	return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(ops.reg2, imm), emit.code);
		1304	},
		1305	0b10 => {
		1306	if (ops.reg2 == .none) {
		1307	// mov dword ptr [reg1 + 0], imm32
		1308	// MI
		1309	const imm = emit.mir.instructions.items(.data)[inst].imm;
		1310	return lowerToMiEnc(tag, RegisterOrMemory.mem(ops.reg1, 0), imm, emit.code);
		1311	}
		1312	// mov [reg1 + imm32], reg2
		1313	// MR
		1314	const imm = emit.mir.instructions.items(.data)[inst].imm;
		1315	return lowerToMrEnc(tag, RegisterOrMemory.mem(ops.reg1, imm), ops.reg2, emit.code);
		1316	},
		1317	0b11 => {
		1318	if (ops.reg2 == .none) {
		1319	// mov dword ptr [reg1 + imm32], imm32
		1320	// MI
		1321	const payload = emit.mir.instructions.items(.data)[inst].payload;
		1322	const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
		1323	return lowerToMiEnc(
		1324	tag,
		1325	RegisterOrMemory.mem(ops.reg1, imm_pair.dest_off),
		1326	imm_pair.operand,
		1327	emit.code,
		1328	);
		1329	}
		1330	return emit.fail("TODO unused variant: mov reg1, reg2, 0b11", .{});
		1331	},
		1332	}
		1333	}
		1334
		1335	fn immOpSize(imm: i64) u8 {
		1336	blk: {
		1337	_ = math.cast(i8, imm) catch break :blk;
		1338	return 8;
		1339	}
		1340	blk: {
		1341	_ = math.cast(i16, imm) catch break :blk;
		1342	return 16;
		1343	}
		1344	blk: {
		1345	_ = math.cast(i32, imm) catch break :blk;
		1346	return 32;
		1347	}
		1348	return 64;
		1349	}
		1350
		1351	// TODO
		1352	fn mirArithScaleSrc(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
762	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);	1353	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
763	const scale = ops.flags;	1354	const scale = ops.flags;
764	// OP reg1, [reg2 + scale*rcx + imm32]	1355	// OP reg1, [reg2 + scale*rcx + imm32]
765	const opcode = getArithOpCode(tag, .rm);	1356	const opc = getOpCode(tag, .rm, ops.reg1.size() == 8).?;
766	const opc = if (ops.reg1.size() == 8) opcode.opc - 1 else opcode.opc;
767	const imm = emit.mir.instructions.items(.data)[inst].imm;	1357	const imm = emit.mir.instructions.items(.data)[inst].imm;
768	const encoder = try Encoder.init(emit.code, 8);	1358	const encoder = try Encoder.init(emit.code, 8);
769	encoder.rex(.{	1359	encoder.rex(.{
...	@@ -771,7 +1361,7 @@ fn mirArithScaleSrc(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE	...	@@ -771,7 +1361,7 @@ fn mirArithScaleSrc(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
771	.r = ops.reg1.isExtended(),	1361	.r = ops.reg1.isExtended(),
772	.b = ops.reg2.isExtended(),	1362	.b = ops.reg2.isExtended(),
773	});	1363	});
774	encoder.opcode_1byte(opc);	1364	opc.encode(encoder);
775	if (imm <= math.maxInt(i8)) {	1365	if (imm <= math.maxInt(i8)) {
776	encoder.modRm_SIBDisp8(ops.reg1.lowId());	1366	encoder.modRm_SIBDisp8(ops.reg1.lowId());
777	encoder.sib_scaleIndexBaseDisp8(scale, Register.rcx.lowId(), ops.reg2.lowId());	1367	encoder.sib_scaleIndexBaseDisp8(scale, Register.rcx.lowId(), ops.reg2.lowId());
...	@@ -783,22 +1373,23 @@ fn mirArithScaleSrc(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE	...	@@ -783,22 +1373,23 @@ fn mirArithScaleSrc(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
783	}	1373	}
784	}	1374	}
785		1375
786	fn mirArithScaleDst(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {	1376	// TODO
		1377	fn mirArithScaleDst(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
787	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);	1378	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
788	const scale = ops.flags;	1379	const scale = ops.flags;
789	const imm = emit.mir.instructions.items(.data)[inst].imm;	1380	const imm = emit.mir.instructions.items(.data)[inst].imm;
790		1381
791	if (ops.reg2 == .none) {	1382	if (ops.reg2 == .none) {
792	// OP [reg1 + scale*rax + 0], imm32	1383	// OP [reg1 + scale*rax + 0], imm32
793	const opcode = getArithOpCode(tag, .mi);	1384	const opc = getOpCode(tag, .mi, ops.reg1.size() == 8).?;
794	const opc = if (ops.reg1.size() == 8) opcode.opc - 1 else opcode.opc;	1385	const modrm_ext = getModRmExt(tag).?;
795	const encoder = try Encoder.init(emit.code, 8);	1386	const encoder = try Encoder.init(emit.code, 8);
796	encoder.rex(.{	1387	encoder.rex(.{
797	.w = ops.reg1.size() == 64,	1388	.w = ops.reg1.size() == 64,
798	.b = ops.reg1.isExtended(),	1389	.b = ops.reg1.isExtended(),
799	});	1390	});
800	encoder.opcode_1byte(opc);	1391	opc.encode(encoder);
801	encoder.modRm_SIBDisp0(opcode.modrm_ext);	1392	encoder.modRm_SIBDisp0(modrm_ext);
802	encoder.sib_scaleIndexBase(scale, Register.rax.lowId(), ops.reg1.lowId());	1393	encoder.sib_scaleIndexBase(scale, Register.rax.lowId(), ops.reg1.lowId());
803	if (imm <= math.maxInt(i8)) {	1394	if (imm <= math.maxInt(i8)) {
804	encoder.imm8(@intCast(i8, imm));	1395	encoder.imm8(@intCast(i8, imm));
...	@@ -811,15 +1402,14 @@ fn mirArithScaleDst(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE	...	@@ -811,15 +1402,14 @@ fn mirArithScaleDst(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
811	}	1402	}
812		1403
813	// OP [reg1 + scale*rax + imm32], reg2	1404	// OP [reg1 + scale*rax + imm32], reg2
814	const opcode = getArithOpCode(tag, .mr);	1405	const opc = getOpCode(tag, .mr, ops.reg1.size() == 8).?;
815	const opc = if (ops.reg1.size() == 8) opcode.opc - 1 else opcode.opc;
816	const encoder = try Encoder.init(emit.code, 8);	1406	const encoder = try Encoder.init(emit.code, 8);
817	encoder.rex(.{	1407	encoder.rex(.{
818	.w = ops.reg1.size() == 64,	1408	.w = ops.reg1.size() == 64,
819	.r = ops.reg2.isExtended(),	1409	.r = ops.reg2.isExtended(),
820	.b = ops.reg1.isExtended(),	1410	.b = ops.reg1.isExtended(),
821	});	1411	});
822	encoder.opcode_1byte(opc);	1412	opc.encode(encoder);
823	if (imm <= math.maxInt(i8)) {	1413	if (imm <= math.maxInt(i8)) {
824	encoder.modRm_SIBDisp8(ops.reg2.lowId());	1414	encoder.modRm_SIBDisp8(ops.reg2.lowId());
825	encoder.sib_scaleIndexBaseDisp8(scale, Register.rax.lowId(), ops.reg1.lowId());	1415	encoder.sib_scaleIndexBaseDisp8(scale, Register.rax.lowId(), ops.reg1.lowId());
...	@@ -831,25 +1421,26 @@ fn mirArithScaleDst(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE	...	@@ -831,25 +1421,26 @@ fn mirArithScaleDst(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
831	}	1421	}
832	}	1422	}
833		1423
834	fn mirArithScaleImm(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {	1424	// TODO
		1425	fn mirArithScaleImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
835	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);	1426	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
836	const scale = ops.flags;	1427	const scale = ops.flags;
837	const payload = emit.mir.instructions.items(.data)[inst].payload;	1428	const payload = emit.mir.instructions.items(.data)[inst].payload;
838	const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;	1429	const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
839	const opcode = getArithOpCode(tag, .mi);	1430	const opc = getOpCode(tag, .mi, ops.reg1.size() == 8).?;
840	const opc = if (ops.reg1.size() == 8) opcode.opc - 1 else opcode.opc;	1431	const modrm_ext = getModRmExt(tag).?;
841	const encoder = try Encoder.init(emit.code, 2);	1432	const encoder = try Encoder.init(emit.code, 2);
842	encoder.rex(.{	1433	encoder.rex(.{
843	.w = ops.reg1.size() == 64,	1434	.w = ops.reg1.size() == 64,
844	.b = ops.reg1.isExtended(),	1435	.b = ops.reg1.isExtended(),
845	});	1436	});
846	encoder.opcode_1byte(opc);	1437	opc.encode(encoder);
847	if (imm_pair.dest_off <= math.maxInt(i8)) {	1438	if (imm_pair.dest_off <= math.maxInt(i8)) {
848	encoder.modRm_SIBDisp8(opcode.modrm_ext);	1439	encoder.modRm_SIBDisp8(modrm_ext);
849	encoder.sib_scaleIndexBaseDisp8(scale, Register.rax.lowId(), ops.reg1.lowId());	1440	encoder.sib_scaleIndexBaseDisp8(scale, Register.rax.lowId(), ops.reg1.lowId());
850	encoder.disp8(@intCast(i8, imm_pair.dest_off));	1441	encoder.disp8(@intCast(i8, imm_pair.dest_off));
851	} else {	1442	} else {
852	encoder.modRm_SIBDisp32(opcode.modrm_ext);	1443	encoder.modRm_SIBDisp32(modrm_ext);
853	encoder.sib_scaleIndexBaseDisp32(scale, Register.rax.lowId(), ops.reg1.lowId());	1444	encoder.sib_scaleIndexBaseDisp32(scale, Register.rax.lowId(), ops.reg1.lowId());
854	encoder.disp32(imm_pair.dest_off);	1445	encoder.disp32(imm_pair.dest_off);
855	}	1446	}
...	@@ -860,54 +1451,24 @@ fn mirMovabs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {	...	@@ -860,54 +1451,24 @@ fn mirMovabs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
860	const tag = emit.mir.instructions.items(.tag)[inst];	1451	const tag = emit.mir.instructions.items(.tag)[inst];
861	assert(tag == .movabs);	1452	assert(tag == .movabs);
862	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);	1453	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
863		1454	const imm: i64 = if (ops.reg1.size() == 64) blk: {
864	const encoder = try Encoder.init(emit.code, 10);
865	const is_64 = blk: {
866	if (ops.flags == 0b00) {
867	// movabs reg, imm64
868	const opc: u8 = if (ops.reg1.size() == 8) 0xb0 else 0xb8;
869	if (ops.reg1.size() == 64) {
870	encoder.rex(.{
871	.w = true,
872	.b = ops.reg1.isExtended(),
873	});
874	encoder.opcode_withReg(opc, ops.reg1.lowId());
875	break :blk true;
876	}
877	break :blk false;
878	}
879	if (ops.reg1 == .none) {
880	// movabs moffs64, rax
881	const opc: u8 = if (ops.reg2.size() == 8) 0xa2 else 0xa3;
882	encoder.rex(.{
883	.w = ops.reg2.size() == 64,
884	});
885	encoder.opcode_1byte(opc);
886	break :blk ops.reg2.size() == 64;
887	} else {
888	// movabs rax, moffs64
889	const opc: u8 = if (ops.reg2.size() == 8) 0xa0 else 0xa1;
890	encoder.rex(.{
891	.w = ops.reg1.size() == 64,
892	});
893	encoder.opcode_1byte(opc);
894	break :blk ops.reg1.size() == 64;
895	}
896	};
897
898	if (is_64) {
899	const payload = emit.mir.instructions.items(.data)[inst].payload;	1455	const payload = emit.mir.instructions.items(.data)[inst].payload;
900	const imm64 = emit.mir.extraData(Mir.Imm64, payload).data;	1456	const imm = emit.mir.extraData(Mir.Imm64, payload).data;
901	encoder.imm64(imm64.decode());	1457	break :blk @bitCast(i64, imm.decode());
		1458	} else emit.mir.instructions.items(.data)[inst].imm;
		1459	if (ops.flags == 0b00) {
		1460	// movabs reg, imm64
		1461	// OI
		1462	return lowerToOiEnc(.mov, ops.reg1, imm, emit.code);
		1463	}
		1464	if (ops.reg1 == .none) {
		1465	// movabs moffs64, rax
		1466	// TD
		1467	return lowerToTdEnc(.mov, imm, ops.reg2, emit.code);
902	} else {	1468	} else {
903	const imm = emit.mir.instructions.items(.data)[inst].imm;	1469	// movabs rax, moffs64
904	if (imm <= math.maxInt(i8)) {	1470	// FD
905	encoder.imm8(@intCast(i8, imm));	1471	return lowerToFdEnc(.mov, ops.reg1, imm, emit.code);
906	} else if (imm <= math.maxInt(i16)) {
907	encoder.imm16(@intCast(i16, imm));
908	} else {
909	encoder.imm32(imm);
910	}
911	}	1472	}
912	}	1473	}
913		1474
...	@@ -916,34 +1477,10 @@ fn mirIMulComplex(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {	...	@@ -916,34 +1477,10 @@ fn mirIMulComplex(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
916	assert(tag == .imul_complex);	1477	assert(tag == .imul_complex);
917	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);	1478	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
918	switch (ops.flags) {	1479	switch (ops.flags) {
919	0b00 => {	1480	0b00 => return lowerToRmEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code),
920	const encoder = try Encoder.init(emit.code, 4);
921	encoder.rex(.{
922	.w = ops.reg1.size() == 64,
923	.r = ops.reg1.isExtended(),
924	.b = ops.reg2.isExtended(),
925	});
926	encoder.opcode_2byte(0x0f, 0xaf);
927	encoder.modRm_direct(ops.reg1.lowId(), ops.reg2.lowId());
928	},
929	0b10 => {	1481	0b10 => {
930	const imm = emit.mir.instructions.items(.data)[inst].imm;	1482	const imm = emit.mir.instructions.items(.data)[inst].imm;
931	const opc: u8 = if (imm <= math.maxInt(i8)) 0x6b else 0x69;	1483	return lowerToRmiEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), imm, emit.code);
932	const encoder = try Encoder.init(emit.code, 7);
933	encoder.rex(.{
934	.w = ops.reg1.size() == 64,
935	.r = ops.reg1.isExtended(),
936	.b = ops.reg1.isExtended(),
937	});
938	encoder.opcode_1byte(opc);
939	encoder.modRm_direct(ops.reg1.lowId(), ops.reg2.lowId());
940	if (imm <= math.maxInt(i8)) {
941	encoder.imm8(@intCast(i8, imm));
942	} else if (imm <= math.maxInt(i16)) {
943	encoder.imm16(@intCast(i16, imm));
944	} else {
945	encoder.imm32(imm);
946	}
947	},	1484	},
948	else => return emit.fail("TODO implement imul", .{}),	1485	else => return emit.fail("TODO implement imul", .{}),
949	}	1486	}
...	@@ -955,37 +1492,7 @@ fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {	...	@@ -955,37 +1492,7 @@ fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
955	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);	1492	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
956	assert(ops.flags == 0b01);	1493	assert(ops.flags == 0b01);
957	const imm = emit.mir.instructions.items(.data)[inst].imm;	1494	const imm = emit.mir.instructions.items(.data)[inst].imm;
958		1495	return lowerToRmEnc(.lea, ops.reg1, RegisterOrMemory.mem(ops.reg2, imm), emit.code);
959	if (imm == 0) {
960	const encoder = try Encoder.init(emit.code, 3);
961	encoder.rex(.{
962	.w = ops.reg1.size() == 64,
963	.r = ops.reg1.isExtended(),
964	.b = ops.reg2.isExtended(),
965	});
966	encoder.opcode_1byte(0x8d);
967	encoder.modRm_indirectDisp0(ops.reg1.lowId(), ops.reg2.lowId());
968	} else if (imm <= math.maxInt(i8)) {
969	const encoder = try Encoder.init(emit.code, 4);
970	encoder.rex(.{
971	.w = ops.reg1.size() == 64,
972	.r = ops.reg1.isExtended(),
973	.b = ops.reg2.isExtended(),
974	});
975	encoder.opcode_1byte(0x8d);
976	encoder.modRm_indirectDisp8(ops.reg1.lowId(), ops.reg2.lowId());
977	encoder.disp8(@intCast(i8, imm));
978	} else {
979	const encoder = try Encoder.init(emit.code, 7);
980	encoder.rex(.{
981	.w = ops.reg1.size() == 64,
982	.r = ops.reg1.isExtended(),
983	.b = ops.reg2.isExtended(),
984	});
985	encoder.opcode_1byte(0x8d);
986	encoder.modRm_indirectDisp32(ops.reg1.lowId(), ops.reg2.lowId());
987	encoder.disp32(imm);
988	}
989	}	1496	}
990		1497
991	fn mirLeaRip(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {	1498	fn mirLeaRip(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
...	@@ -993,26 +1500,22 @@ fn mirLeaRip(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {	...	@@ -993,26 +1500,22 @@ fn mirLeaRip(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
993	assert(tag == .lea_rip);	1500	assert(tag == .lea_rip);
994	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);	1501	const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
995	const start_offset = emit.code.items.len;	1502	const start_offset = emit.code.items.len;
996	const encoder = try Encoder.init(emit.code, 7);	1503	try lowerToRmEnc(.lea, ops.reg1, RegisterOrMemory.rip(0), emit.code);
997	encoder.rex(.{
998	.w = ops.reg1.size() == 64,
999	.r = ops.reg1.isExtended(),
1000	});
1001	encoder.opcode_1byte(0x8d);
1002	encoder.modRm_RIPDisp32(ops.reg1.lowId());
1003	const end_offset = emit.code.items.len;	1504	const end_offset = emit.code.items.len;
1004	if (@truncate(u1, ops.flags) == 0b0) {	1505	if (@truncate(u1, ops.flags) == 0b0) {
		1506	// Backpatch the displacement
		1507	// TODO figure out if this can be simplified
1005	const payload = emit.mir.instructions.items(.data)[inst].payload;	1508	const payload = emit.mir.instructions.items(.data)[inst].payload;
1006	const imm = emit.mir.extraData(Mir.Imm64, payload).data.decode();	1509	const imm = emit.mir.extraData(Mir.Imm64, payload).data.decode();
1007	encoder.disp32(@intCast(i32, @intCast(i64, imm) - @intCast(i64, end_offset - start_offset + 4)));	1510	const disp = @intCast(i32, @intCast(i64, imm) - @intCast(i64, end_offset - start_offset));
		1511	mem.writeIntLittle(i32, emit.code.items[end_offset - 4 ..][0..4], disp);
1008	} else {	1512	} else {
1009	const got_entry = emit.mir.instructions.items(.data)[inst].got_entry;	1513	const got_entry = emit.mir.instructions.items(.data)[inst].got_entry;
1010	encoder.disp32(0);
1011	if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {	1514	if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
1012	// TODO I think the reloc might be in the wrong place.	1515	// TODO I think the reloc might be in the wrong place.
1013	const decl = macho_file.active_decl.?;	1516	const decl = macho_file.active_decl.?;
1014	try decl.link.macho.relocs.append(emit.bin_file.allocator, .{	1517	try decl.link.macho.relocs.append(emit.bin_file.allocator, .{
1015	.offset = @intCast(u32, end_offset),	1518	.offset = @intCast(u32, end_offset - 4),
1016	.target = .{ .local = got_entry },	1519	.target = .{ .local = got_entry },
1017	.addend = 0,	1520	.addend = 0,
1018	.subtractor = null,	1521	.subtractor = null,
...	@@ -1031,12 +1534,9 @@ fn mirCallExtern(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {	...	@@ -1031,12 +1534,9 @@ fn mirCallExtern(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1031	assert(tag == .call_extern);	1534	assert(tag == .call_extern);
1032	const n_strx = emit.mir.instructions.items(.data)[inst].extern_fn;	1535	const n_strx = emit.mir.instructions.items(.data)[inst].extern_fn;
1033	const offset = blk: {	1536	const offset = blk: {
1034	const offset = @intCast(u32, emit.code.items.len + 1);
1035	// callq	1537	// callq
1036	const encoder = try Encoder.init(emit.code, 5);	1538	try lowerToDEnc(.call_near, 0, emit.code);
1037	encoder.opcode_1byte(0xe8);	1539	break :blk @intCast(u32, emit.code.items.len) - 4;
1038	encoder.imm32(0x0);
1039	break :blk offset;
1040	};	1540	};
1041	if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {	1541	if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
1042	// Add relocation to the decl.	1542	// Add relocation to the decl.
...	@@ -1206,3 +1706,230 @@ fn addDbgInfoTypeReloc(emit: *Emit, ty: Type) !void {	...	@@ -1206,3 +1706,230 @@ fn addDbgInfoTypeReloc(emit: *Emit, ty: Type) !void {
1206	.none => {},	1706	.none => {},
1207	}	1707	}
1208	}	1708	}
		1709
		1710	fn expectEqualHexStrings(expected: []const u8, given: []const u8, assembly: []const u8) !void {
		1711	assert(expected.len > 0);
		1712	if (mem.eql(u8, expected, given)) return;
		1713	const expected_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{std.fmt.fmtSliceHexLower(expected)});
		1714	defer testing.allocator.free(expected_fmt);
		1715	const given_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{std.fmt.fmtSliceHexLower(given)});
		1716	defer testing.allocator.free(given_fmt);
		1717	const idx = mem.indexOfDiff(u8, expected_fmt, given_fmt).?;
		1718	var padding = try testing.allocator.alloc(u8, idx + 5);
		1719	defer testing.allocator.free(padding);
		1720	mem.set(u8, padding, ' ');
		1721	std.debug.print("\nASM: {s}\nEXP: {s}\nGIV: {s}\n{s}^ -- first differing byte\n", .{
		1722	assembly,
		1723	expected_fmt,
		1724	given_fmt,
		1725	padding,
		1726	});
		1727	return error.TestFailed;
		1728	}
		1729
		1730	const TestEmitCode = struct {
		1731	buf: std.ArrayList(u8),
		1732	next: usize = 0,
		1733
		1734	fn init() TestEmitCode {
		1735	return .{
		1736	.buf = std.ArrayList(u8).init(testing.allocator),
		1737	};
		1738	}
		1739
		1740	fn deinit(emit: *TestEmitCode) void {
		1741	emit.buf.deinit();
		1742	emit.next = undefined;
		1743	}
		1744
		1745	fn buffer(emit: TestEmitCode) std.ArrayList(u8) {
		1746	emit.next = emit.buf.items.len;
		1747	return &emit.buf;
		1748	}
		1749
		1750	fn emitted(emit: TestEmitCode) []const u8 {
		1751	return emit.buf.items[emit.next..];
		1752	}
		1753	};
		1754
		1755	test "lower MI encoding" {
		1756	var code = TestEmitCode.init();
		1757	defer code.deinit();
		1758	try lowerToMiEnc(.mov, RegisterOrMemory.reg(.rax), 0x10, code.buffer());
		1759	try expectEqualHexStrings("\x48\xc7\xc0\x10\x00\x00\x00", code.emitted(), "mov rax, 0x10");
		1760	try lowerToMiEnc(.mov, RegisterOrMemory.mem(.r11, 0), 0x10, code.buffer());
		1761	try expectEqualHexStrings("\x41\xc7\x03\x10\x00\x00\x00", code.emitted(), "mov dword ptr [r11 + 0], 0x10");
		1762	try lowerToMiEnc(.add, RegisterOrMemory.mem(.rdx, -8), 0x10, code.buffer());
		1763	try expectEqualHexStrings("\x81\x42\xF8\x10\x00\x00\x00", code.emitted(), "add dword ptr [rdx - 8], 0x10");
		1764	try lowerToMiEnc(.sub, RegisterOrMemory.mem(.r11, 0x10000000), 0x10, code.buffer());
		1765	try expectEqualHexStrings(
		1766	"\x41\x81\xab\x00\x00\x00\x10\x10\x00\x00\x00",
		1767	code.emitted(),
		1768	"sub dword ptr [r11 + 0x10000000], 0x10",
		1769	);
		1770	try lowerToMiEnc(.@"and", RegisterOrMemory.mem(null, 0x10000000), 0x10, code.buffer());
		1771	try expectEqualHexStrings(
		1772	"\x81\x24\x25\x00\x00\x00\x10\x10\x00\x00\x00",
		1773	code.emitted(),
		1774	"and dword ptr [ds:0x10000000], 0x10",
		1775	);
		1776	try lowerToMiEnc(.@"and", RegisterOrMemory.mem(.r12, 0x10000000), 0x10, code.buffer());
		1777	try expectEqualHexStrings(
		1778	"\x41\x81\xA4\x24\x00\x00\x00\x10\x10\x00\x00\x00",
		1779	code.emitted(),
		1780	"and dword ptr [r12 + 0x10000000], 0x10",
		1781	);
		1782	try lowerToMiEnc(.mov, RegisterOrMemory.rip(0x10), 0x10, code.buffer());
		1783	try expectEqualHexStrings(
		1784	"\xC7\x05\x10\x00\x00\x00\x10\x00\x00\x00",
		1785	code.emitted(),
		1786	"mov [rip + 0x10], 0x10",
		1787	);
		1788	}
		1789
		1790	test "lower RM encoding" {
		1791	var code = TestEmitCode.init();
		1792	defer code.deinit();
		1793	try lowerToRmEnc(.mov, .rax, RegisterOrMemory.reg(.rbx), code.buffer());
		1794	try expectEqualHexStrings("\x48\x8b\xc3", code.emitted(), "mov rax, rbx");
		1795	try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.r11, 0), code.buffer());
		1796	try expectEqualHexStrings("\x49\x8b\x03", code.emitted(), "mov rax, qword ptr [r11 + 0]");
		1797	try lowerToRmEnc(.add, .r11, RegisterOrMemory.mem(null, 0x10000000), code.buffer());
		1798	try expectEqualHexStrings(
		1799	"\x4C\x03\x1C\x25\x00\x00\x00\x10",
		1800	code.emitted(),
		1801	"add r11, qword ptr [ds:0x10000000]",
		1802	);
		1803	try lowerToRmEnc(.add, .r12b, RegisterOrMemory.mem(null, 0x10000000), code.buffer());
		1804	try expectEqualHexStrings(
		1805	"\x44\x02\x24\x25\x00\x00\x00\x10",
		1806	code.emitted(),
		1807	"add r11b, byte ptr [ds:0x10000000]",
		1808	);
		1809	try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.r13, 0x10000000), code.buffer());
		1810	try expectEqualHexStrings(
		1811	"\x4D\x2B\x9D\x00\x00\x00\x10",
		1812	code.emitted(),
		1813	"sub r11, qword ptr [r13 + 0x10000000]",
		1814	);
		1815	try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.r12, 0x10000000), code.buffer());
		1816	try expectEqualHexStrings(
		1817	"\x4D\x2B\x9C\x24\x00\x00\x00\x10",
		1818	code.emitted(),
		1819	"sub r11, qword ptr [r12 + 0x10000000]",
		1820	);
		1821	try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.rbp, -4), code.buffer());
		1822	try expectEqualHexStrings("\x48\x8B\x45\xFC", code.emitted(), "mov rax, qword ptr [rbp - 4]");
		1823	try lowerToRmEnc(.lea, .rax, RegisterOrMemory.rip(0x10), code.buffer());
		1824	try expectEqualHexStrings("\x48\x8D\x05\x10\x00\x00\x00", code.emitted(), "lea rax, [rip + 0x10]");
		1825	}
		1826
		1827	test "lower MR encoding" {
		1828	var code = TestEmitCode.init();
		1829	defer code.deinit();
		1830	try lowerToMrEnc(.mov, RegisterOrMemory.reg(.rax), .rbx, code.buffer());
		1831	try expectEqualHexStrings("\x48\x89\xd8", code.emitted(), "mov rax, rbx");
		1832	try lowerToMrEnc(.mov, RegisterOrMemory.mem(.rbp, -4), .r11, code.buffer());
		1833	try expectEqualHexStrings("\x4c\x89\x5d\xfc", code.emitted(), "mov qword ptr [rbp - 4], r11");
		1834	try lowerToMrEnc(.add, RegisterOrMemory.mem(null, 0x10000000), .r12b, code.buffer());
		1835	try expectEqualHexStrings(
		1836	"\x44\x00\x24\x25\x00\x00\x00\x10",
		1837	code.emitted(),
		1838	"add byte ptr [ds:0x10000000], r12b",
		1839	);
		1840	try lowerToMrEnc(.add, RegisterOrMemory.mem(null, 0x10000000), .r12d, code.buffer());
		1841	try expectEqualHexStrings(
		1842	"\x44\x01\x24\x25\x00\x00\x00\x10",
		1843	code.emitted(),
		1844	"add dword ptr [ds:0x10000000], r12d",
		1845	);
		1846	try lowerToMrEnc(.sub, RegisterOrMemory.mem(.r11, 0x10000000), .r12, code.buffer());
		1847	try expectEqualHexStrings(
		1848	"\x4D\x29\xA3\x00\x00\x00\x10",
		1849	code.emitted(),
		1850	"sub qword ptr [r11 + 0x10000000], r12",
		1851	);
		1852	try lowerToMrEnc(.mov, RegisterOrMemory.rip(0x10), .r12, code.buffer());
		1853	try expectEqualHexStrings("\x4C\x89\x25\x10\x00\x00\x00", code.emitted(), "mov qword ptr [rip + 0x10], r12");
		1854	}
		1855
		1856	test "lower OI encoding" {
		1857	var code = TestEmitCode.init();
		1858	defer code.deinit();
		1859	try lowerToOiEnc(.mov, .rax, 0x1000000000000000, code.buffer());
		1860	try expectEqualHexStrings(
		1861	"\x48\xB8\x00\x00\x00\x00\x00\x00\x00\x10",
		1862	code.emitted(),
		1863	"movabs rax, 0x1000000000000000",
		1864	);
		1865	try lowerToOiEnc(.mov, .r11, 0x1000000000000000, code.buffer());
		1866	try expectEqualHexStrings(
		1867	"\x49\xBB\x00\x00\x00\x00\x00\x00\x00\x10",
		1868	code.emitted(),
		1869	"movabs r11, 0x1000000000000000",
		1870	);
		1871	try lowerToOiEnc(.mov, .r11d, 0x10000000, code.buffer());
		1872	try expectEqualHexStrings("\x41\xBB\x00\x00\x00\x10", code.emitted(), "mov r11d, 0x10000000");
		1873	try lowerToOiEnc(.mov, .r11w, 0x1000, code.buffer());
		1874	try expectEqualHexStrings("\x66\x41\xBB\x00\x10", code.emitted(), "mov r11w, 0x1000");
		1875	try lowerToOiEnc(.mov, .r11b, 0x10, code.buffer());
		1876	try expectEqualHexStrings("\x41\xB3\x10", code.emitted(), "mov r11b, 0x10");
		1877	}
		1878
		1879	test "lower FD/TD encoding" {
		1880	var code = TestEmitCode.init();
		1881	defer code.deinit();
		1882	try lowerToFdEnc(.mov, .rax, 0x1000000000000000, code.buffer());
		1883	try expectEqualHexStrings(
		1884	"\x48\xa1\x00\x00\x00\x00\x00\x00\x00\x10",
		1885	code.emitted(),
		1886	"mov rax, ds:0x1000000000000000",
		1887	);
		1888	try lowerToFdEnc(.mov, .eax, 0x10000000, code.buffer());
		1889	try expectEqualHexStrings("\xa1\x00\x00\x00\x10", code.emitted(), "mov eax, ds:0x10000000");
		1890	try lowerToFdEnc(.mov, .ax, 0x1000, code.buffer());
		1891	try expectEqualHexStrings("\x66\xa1\x00\x10", code.emitted(), "mov ax, ds:0x1000");
		1892	try lowerToFdEnc(.mov, .al, 0x10, code.buffer());
		1893	try expectEqualHexStrings("\xa0\x10", code.emitted(), "mov al, ds:0x10");
		1894	}
		1895
		1896	test "lower M encoding" {
		1897	var code = TestEmitCode.init();
		1898	defer code.deinit();
		1899	try lowerToMEnc(.jmp_near, RegisterOrMemory.reg(.r12), code.buffer());
		1900	try expectEqualHexStrings("\x41\xFF\xE4", code.emitted(), "jmp r12");
		1901	try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.r12, 0), code.buffer());
		1902	try expectEqualHexStrings("\x41\xFF\x24\x24", code.emitted(), "jmp qword ptr [r12]");
		1903	try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.r12, 0x10), code.buffer());
		1904	try expectEqualHexStrings("\x41\xFF\x64\x24\x10", code.emitted(), "jmp qword ptr [r12 + 0x10]");
		1905	try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.r12, 0x1000), code.buffer());
		1906	try expectEqualHexStrings(
		1907	"\x41\xFF\xA4\x24\x00\x10\x00\x00",
		1908	code.emitted(),
		1909	"jmp qword ptr [r12 + 0x1000]",
		1910	);
		1911	try lowerToMEnc(.jmp_near, RegisterOrMemory.rip(0x10), code.buffer());
		1912	try expectEqualHexStrings("\xFF\x25\x10\x00\x00\x00", code.emitted(), "jmp qword ptr [rip + 0x10]");
		1913	try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(null, 0x10), code.buffer());
		1914	try expectEqualHexStrings("\xFF\x24\x25\x10\x00\x00\x00", code.emitted(), "jmp qword ptr [ds:0x10]");
		1915	try lowerToMEnc(.seta, RegisterOrMemory.reg(.r11b), code.buffer());
		1916	try expectEqualHexStrings("\x49\x0F\x97\xC3", code.emitted(), "seta r11b");
		1917	}
		1918
		1919	test "lower O encoding" {
		1920	var code = TestEmitCode.init();
		1921	defer code.deinit();
		1922	try lowerToOEnc(.pop, .r12, code.buffer());
		1923	try expectEqualHexStrings("\x41\x5c", code.emitted(), "pop r12");
		1924	try lowerToOEnc(.push, .r12w, code.buffer());
		1925	try expectEqualHexStrings("\x66\x41\x54", code.emitted(), "push r12w");
		1926	}
		1927
		1928	test "lower RMI encoding" {
		1929	var code = TestEmitCode.init();
		1930	defer code.deinit();
		1931	try lowerToRmiEnc(.imul, .rax, RegisterOrMemory.mem(.rbp, -8), 0x10, code.buffer());
		1932	try expectEqualHexStrings("\x48\x69\x45\xF8\x10\x00\x00\x00", code.emitted(), "imul rax, [rbp - 8], 0x10");
		1933	try lowerToRmiEnc(.imul, .r12, RegisterOrMemory.reg(.r12), 0x10, code.buffer());
		1934	try expectEqualHexStrings("\x4D\x69\xE4\x10\x00\x00\x00", code.emitted(), "imul r12, r12, 0x10");
		1935	}

src/arch/x86_64/Mir.zig+1

...	@@ -140,6 +140,7 @@ pub const Inst = struct {	...	@@ -140,6 +140,7 @@ pub const Inst = struct {
140	mov_scale_src,	140	mov_scale_src,
141	mov_scale_dst,	141	mov_scale_dst,
142	mov_scale_imm,	142	mov_scale_imm,
		143
143	lea,	144	lea,
144	lea_scale_src,	145	lea_scale_src,
145	lea_scale_dst,	146	lea_scale_dst,