Commit d525ecb523fed3c1496bf2f5315d06ea95716c8b | zig

author	Jakub Konka <kubkon@jakubkonka.com> 2023-03-13 20:48:41+01:00
committer	GitHub <noreply@github.com> 2023-03-13 20:48:41+01:00
log	d525ecb523fed3c1496bf2f5315d06ea95716c8b
tree	751784e96af8e3932f7da7c4346888dc2ed91ab0
parent	4942e4e8701b9a137d7f30fcb8ac1bc2d46f2a99
parent	e9fc0aba4c3b0855e580ff3afe7251920ae3dcf9
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #14886 from ziglang/x86_64-encoder

x86_64: add table-driven instruction encoder

8 files changed, 5333 insertions(+), 5340 deletions(-)

CMakeLists.txt
src
- arch
  - x86_64
    CodeGen.zig
    Emit.zig
    Encoding.zig
    Mir.zig
    bits.zig
    encoder.zig
    encodings.zig

CMakeLists.txt+4-1

 @@ -559,9 +559,12 @@ set(ZIG_STAGE2_SOURCES
     "${CMAKE_SOURCE_DIR}/src/arch/wasm/Mir.zig"
     "${CMAKE_SOURCE_DIR}/src/arch/x86_64/CodeGen.zig"
     "${CMAKE_SOURCE_DIR}/src/arch/x86_64/Emit.zig"
+    "${CMAKE_SOURCE_DIR}/src/arch/x86_64/Encoding.zig"
     "${CMAKE_SOURCE_DIR}/src/arch/x86_64/Mir.zig"
-    "${CMAKE_SOURCE_DIR}/src/arch/x86_64/bits.zig"
     "${CMAKE_SOURCE_DIR}/src/arch/x86_64/abi.zig"
+    "${CMAKE_SOURCE_DIR}/src/arch/x86_64/bits.zig"
+    "${CMAKE_SOURCE_DIR}/src/arch/x86_64/encoder.zig"
+    "${CMAKE_SOURCE_DIR}/src/arch/x86_64/encodings.zig"
     "${CMAKE_SOURCE_DIR}/src/clang.zig"
     "${CMAKE_SOURCE_DIR}/src/clang_options.zig"
     "${CMAKE_SOURCE_DIR}/src/clang_options_data.zig"

src/arch/x86_64/CodeGen.zig+803-1204

...	@@ -33,9 +33,11 @@ const errUnionPayloadOffset = codegen.errUnionPayloadOffset;	...	@@ -33,9 +33,11 @@ const errUnionPayloadOffset = codegen.errUnionPayloadOffset;
33	const errUnionErrorOffset = codegen.errUnionErrorOffset;	33	const errUnionErrorOffset = codegen.errUnionErrorOffset;
34		34
35	const Condition = bits.Condition;	35	const Condition = bits.Condition;
		36	const Immediate = bits.Immediate;
		37	const Memory = bits.Memory;
		38	const Register = bits.Register;
36	const RegisterManager = abi.RegisterManager;	39	const RegisterManager = abi.RegisterManager;
37	const RegisterLock = RegisterManager.RegisterLock;	40	const RegisterLock = RegisterManager.RegisterLock;
38	const Register = bits.Register;
39		41
40	const gp = abi.RegisterClass.gp;	42	const gp = abi.RegisterClass.gp;
41	const sse = abi.RegisterClass.sse;	43	const sse = abi.RegisterClass.sse;
...	@@ -303,7 +305,12 @@ pub fn generate(	...	@@ -303,7 +305,12 @@ pub fn generate(
303	var call_info = function.resolveCallingConventionValues(fn_type) catch \|err\| switch (err) {	305	var call_info = function.resolveCallingConventionValues(fn_type) catch \|err\| switch (err) {
304	error.CodegenFail => return Result{ .fail = function.err_msg.? },	306	error.CodegenFail => return Result{ .fail = function.err_msg.? },
305	error.OutOfRegisters => return Result{	307	error.OutOfRegisters => return Result{
306	.fail = try ErrorMsg.create(bin_file.allocator, src_loc, "CodeGen ran out of registers. This is a bug in the Zig compiler.", .{}),	308	.fail = try ErrorMsg.create(
		309	bin_file.allocator,
		310	src_loc,
		311	"CodeGen ran out of registers. This is a bug in the Zig compiler.",
		312	.{},
		313	),
307	},	314	},
308	else => \|e\| return e,	315	else => \|e\| return e,
309	};	316	};
...	@@ -342,6 +349,20 @@ pub fn generate(	...	@@ -342,6 +349,20 @@ pub fn generate(
342	defer emit.deinit();	349	defer emit.deinit();
343	emit.lowerMir() catch \|err\| switch (err) {	350	emit.lowerMir() catch \|err\| switch (err) {
344	error.EmitFail => return Result{ .fail = emit.err_msg.? },	351	error.EmitFail => return Result{ .fail = emit.err_msg.? },
		352	error.InvalidInstruction, error.CannotEncode => \|e\| {
		353	const msg = switch (e) {
		354	error.InvalidInstruction => "CodeGen failed to find a viable instruction.",
		355	error.CannotEncode => "CodeGen failed to encode the instruction.",
		356	};
		357	return Result{
		358	.fail = try ErrorMsg.create(
		359	bin_file.allocator,
		360	src_loc,
		361	"{s} This is a bug in the Zig compiler.",
		362	.{msg},
		363	),
		364	};
		365	},
345	else => \|e\| return e,	366	else => \|e\| return e,
346	};	367	};
347		368
...	@@ -355,52 +376,263 @@ pub fn generate(	...	@@ -355,52 +376,263 @@ pub fn generate(
355	fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {	376	fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {
356	const gpa = self.gpa;	377	const gpa = self.gpa;
357	try self.mir_instructions.ensureUnusedCapacity(gpa, 1);	378	try self.mir_instructions.ensureUnusedCapacity(gpa, 1);
358	const result_index = @intCast(Air.Inst.Index, self.mir_instructions.len);	379	const result_index = @intCast(Mir.Inst.Index, self.mir_instructions.len);
359	self.mir_instructions.appendAssumeCapacity(inst);	380	self.mir_instructions.appendAssumeCapacity(inst);
360	return result_index;	381	return result_index;
361	}	382	}
362		383
363	pub fn addExtra(self: *Self, extra: anytype) Allocator.Error!u32 {	384	fn addExtra(self: *Self, extra: anytype) Allocator.Error!u32 {
364	const fields = std.meta.fields(@TypeOf(extra));	385	const fields = std.meta.fields(@TypeOf(extra));
365	try self.mir_extra.ensureUnusedCapacity(self.gpa, fields.len);	386	try self.mir_extra.ensureUnusedCapacity(self.gpa, fields.len);
366	return self.addExtraAssumeCapacity(extra);	387	return self.addExtraAssumeCapacity(extra);
367	}	388	}
368		389
369	pub fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {	390	fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {
370	const fields = std.meta.fields(@TypeOf(extra));	391	const fields = std.meta.fields(@TypeOf(extra));
371	const result = @intCast(u32, self.mir_extra.items.len);	392	const result = @intCast(u32, self.mir_extra.items.len);
372	inline for (fields) \|field\| {	393	inline for (fields) \|field\| {
373	self.mir_extra.appendAssumeCapacity(switch (field.type) {	394	self.mir_extra.appendAssumeCapacity(switch (field.type) {
374	u32 => @field(extra, field.name),	395	u32 => @field(extra, field.name),
375	i32 => @bitCast(u32, @field(extra, field.name)),	396	i32 => @bitCast(u32, @field(extra, field.name)),
376	else => @compileError("bad field type"),	397	else => @compileError("bad field type: " ++ field.name ++ ": " ++ @typeName(field.type)),
377	});	398	});
378	}	399	}
379	return result;	400	return result;
380	}	401	}
381		402
		403	fn asmSetccRegister(self: *Self, reg: Register, cc: bits.Condition) !void {
		404	_ = try self.addInst(.{
		405	.tag = .setcc,
		406	.ops = .r_c,
		407	.data = .{ .r_c = .{
		408	.r1 = reg,
		409	.cc = cc,
		410	} },
		411	});
		412	}
		413
		414	fn asmCmovccRegisterRegister(self: *Self, reg1: Register, reg2: Register, cc: bits.Condition) !void {
		415	_ = try self.addInst(.{
		416	.tag = .cmovcc,
		417	.ops = .rr_c,
		418	.data = .{ .rr_c = .{
		419	.r1 = reg1,
		420	.r2 = reg2,
		421	.cc = cc,
		422	} },
		423	});
		424	}
		425
		426	fn asmJmpReloc(self: *Self, target: Mir.Inst.Index) !Mir.Inst.Index {
		427	return self.addInst(.{
		428	.tag = .jmp_reloc,
		429	.ops = undefined,
		430	.data = .{ .inst = target },
		431	});
		432	}
		433
		434	fn asmJccReloc(self: *Self, target: Mir.Inst.Index, cc: bits.Condition) !Mir.Inst.Index {
		435	return self.addInst(.{
		436	.tag = .jcc,
		437	.ops = .inst_cc,
		438	.data = .{ .inst_cc = .{
		439	.inst = target,
		440	.cc = cc,
		441	} },
		442	});
		443	}
		444
		445	fn asmOpOnly(self: *Self, tag: Mir.Inst.Tag) !void {
		446	_ = try self.addInst(.{
		447	.tag = tag,
		448	.ops = .none,
		449	.data = undefined,
		450	});
		451	}
		452
		453	fn asmRegister(self: *Self, tag: Mir.Inst.Tag, reg: Register) !void {
		454	_ = try self.addInst(.{
		455	.tag = tag,
		456	.ops = .r,
		457	.data = .{ .r = reg },
		458	});
		459	}
		460
		461	fn asmImmediate(self: *Self, tag: Mir.Inst.Tag, imm: Immediate) !void {
		462	const ops: Mir.Inst.Ops = if (imm == .signed) .imm_s else .imm_u;
		463	_ = try self.addInst(.{
		464	.tag = tag,
		465	.ops = ops,
		466	.data = .{ .imm = switch (imm) {
		467	.signed => \|x\| @bitCast(u32, x),
		468	.unsigned => \|x\| @intCast(u32, x),
		469	} },
		470	});
		471	}
		472
		473	fn asmRegisterRegister(self: *Self, tag: Mir.Inst.Tag, reg1: Register, reg2: Register) !void {
		474	_ = try self.addInst(.{
		475	.tag = tag,
		476	.ops = .rr,
		477	.data = .{ .rr = .{
		478	.r1 = reg1,
		479	.r2 = reg2,
		480	} },
		481	});
		482	}
		483
		484	fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.Tag, reg: Register, imm: Immediate) !void {
		485	const ops: Mir.Inst.Ops = switch (imm) {
		486	.signed => .ri_s,
		487	.unsigned => \|x\| if (x <= math.maxInt(u32)) .ri_u else .ri64,
		488	};
		489	const data: Mir.Inst.Data = switch (ops) {
		490	.ri_s => .{ .ri = .{
		491	.r1 = reg,
		492	.imm = @bitCast(u32, imm.signed),
		493	} },
		494	.ri_u => .{ .ri = .{
		495	.r1 = reg,
		496	.imm = @intCast(u32, imm.unsigned),
		497	} },
		498	.ri64 => .{ .rx = .{
		499	.r1 = reg,
		500	.payload = try self.addExtra(Mir.Imm64.encode(imm.unsigned)),
		501	} },
		502	else => unreachable,
		503	};
		504	_ = try self.addInst(.{
		505	.tag = tag,
		506	.ops = ops,
		507	.data = data,
		508	});
		509	}
		510
		511	fn asmRegisterRegisterImmediate(
		512	self: *Self,
		513	tag: Mir.Inst.Tag,
		514	reg1: Register,
		515	reg2: Register,
		516	imm: Immediate,
		517	) !void {
		518	const ops: Mir.Inst.Ops = switch (imm) {
		519	.signed => .rri_s,
		520	.unsigned => .rri_u,
		521	};
		522	const data: Mir.Inst.Data = switch (ops) {
		523	.rri_s => .{ .rri = .{
		524	.r1 = reg1,
		525	.r2 = reg2,
		526	.imm = @bitCast(u32, imm.signed),
		527	} },
		528	.rri_u => .{ .rri = .{
		529	.r1 = reg1,
		530	.r2 = reg2,
		531	.imm = @intCast(u32, imm.unsigned),
		532	} },
		533	else => unreachable,
		534	};
		535	_ = try self.addInst(.{
		536	.tag = tag,
		537	.ops = ops,
		538	.data = data,
		539	});
		540	}
		541
		542	fn asmMemory(self: *Self, tag: Mir.Inst.Tag, m: Memory) !void {
		543	const ops: Mir.Inst.Ops = switch (m) {
		544	.sib => .m_sib,
		545	.rip => .m_rip,
		546	else => unreachable,
		547	};
		548	const data: Mir.Inst.Data = .{ .payload = switch (ops) {
		549	.m_sib => try self.addExtra(Mir.MemorySib.encode(m)),
		550	.m_rip => try self.addExtra(Mir.MemoryRip.encode(m)),
		551	else => unreachable,
		552	} };
		553	_ = try self.addInst(.{
		554	.tag = tag,
		555	.ops = ops,
		556	.data = data,
		557	});
		558	}
		559
		560	fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.Tag, m: Memory, imm: Immediate) !void {
		561	const ops: Mir.Inst.Ops = switch (m) {
		562	.sib => if (imm == .signed) .mi_s_sib else .mi_u_sib,
		563	.rip => if (imm == .signed) .mi_s_rip else .mi_u_rip,
		564	else => unreachable,
		565	};
		566	const payload: u32 = switch (ops) {
		567	.mi_s_sib, .mi_u_sib => try self.addExtra(Mir.MemorySib.encode(m)),
		568	.mi_s_rip, .mi_u_rip => try self.addExtra(Mir.MemoryRip.encode(m)),
		569	else => unreachable,
		570	};
		571	const data: Mir.Inst.Data = .{
		572	.xi = .{ .payload = payload, .imm = switch (imm) {
		573	.signed => \|x\| @bitCast(u32, x),
		574	.unsigned => \|x\| @intCast(u32, x),
		575	} },
		576	};
		577	_ = try self.addInst(.{
		578	.tag = tag,
		579	.ops = ops,
		580	.data = data,
		581	});
		582	}
		583
		584	fn asmRegisterMemory(self: *Self, tag: Mir.Inst.Tag, reg: Register, m: Memory) !void {
		585	const ops: Mir.Inst.Ops = switch (m) {
		586	.sib => .rm_sib,
		587	.rip => .rm_rip,
		588	else => unreachable,
		589	};
		590	const data: Mir.Inst.Data = .{
		591	.rx = .{ .r1 = reg, .payload = switch (ops) {
		592	.rm_sib => try self.addExtra(Mir.MemorySib.encode(m)),
		593	.rm_rip => try self.addExtra(Mir.MemoryRip.encode(m)),
		594	else => unreachable,
		595	} },
		596	};
		597	_ = try self.addInst(.{
		598	.tag = tag,
		599	.ops = ops,
		600	.data = data,
		601	});
		602	}
		603
		604	fn asmMemoryRegister(self: *Self, tag: Mir.Inst.Tag, m: Memory, reg: Register) !void {
		605	const ops: Mir.Inst.Ops = switch (m) {
		606	.sib => .mr_sib,
		607	.rip => .mr_rip,
		608	else => unreachable,
		609	};
		610	const data: Mir.Inst.Data = .{
		611	.rx = .{ .r1 = reg, .payload = switch (ops) {
		612	.mr_sib => try self.addExtra(Mir.MemorySib.encode(m)),
		613	.mr_rip => try self.addExtra(Mir.MemoryRip.encode(m)),
		614	else => unreachable,
		615	} },
		616	};
		617	_ = try self.addInst(.{
		618	.tag = tag,
		619	.ops = ops,
		620	.data = data,
		621	});
		622	}
		623
382	fn gen(self: *Self) InnerError!void {	624	fn gen(self: *Self) InnerError!void {
383	const cc = self.fn_type.fnCallingConvention();	625	const cc = self.fn_type.fnCallingConvention();
384	if (cc != .Naked) {	626	if (cc != .Naked) {
385	_ = try self.addInst(.{	627	try self.asmRegister(.push, .rbp);
386	.tag = .push,	628	try self.asmRegisterRegister(.mov, .rbp, .rsp);
387	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),	629
388	.data = undefined, // unused for push reg,
389	});
390	_ = try self.addInst(.{
391	.tag = .mov,
392	.ops = Mir.Inst.Ops.encode(.{
393	.reg1 = .rbp,
394	.reg2 = .rsp,
395	}),
396	.data = undefined,
397	});
398	// We want to subtract the aligned stack frame size from rsp here, but we don't	630	// We want to subtract the aligned stack frame size from rsp here, but we don't
399	// yet know how big it will be, so we leave room for a 4-byte stack size.	631	// yet know how big it will be, so we leave room for a 4-byte stack size.
400	// TODO During semantic analysis, check if there are no function calls. If there	632	// TODO During semantic analysis, check if there are no function calls. If there
401	// are none, here we can omit the part where we subtract and then add rsp.	633	// are none, here we can omit the part where we subtract and then add rsp.
402	const backpatch_stack_sub = try self.addInst(.{	634	const backpatch_stack_sub = try self.addInst(.{
403	.tag = .nop,	635	.tag = .dead,
404	.ops = undefined,	636	.ops = undefined,
405	.data = undefined,	637	.data = undefined,
406	});	638	});
...	@@ -427,7 +659,7 @@ fn gen(self: *Self) InnerError!void {	...	@@ -427,7 +659,7 @@ fn gen(self: *Self) InnerError!void {
427		659
428	// Push callee-preserved regs that were used actually in use.	660	// Push callee-preserved regs that were used actually in use.
429	const backpatch_push_callee_preserved_regs = try self.addInst(.{	661	const backpatch_push_callee_preserved_regs = try self.addInst(.{
430	.tag = .nop,	662	.tag = .dead,
431	.ops = undefined,	663	.ops = undefined,
432	.data = undefined,	664	.data = undefined,
433	});	665	});
...	@@ -458,7 +690,7 @@ fn gen(self: *Self) InnerError!void {	...	@@ -458,7 +690,7 @@ fn gen(self: *Self) InnerError!void {
458		690
459	// Pop saved callee-preserved regs.	691	// Pop saved callee-preserved regs.
460	const backpatch_pop_callee_preserved_regs = try self.addInst(.{	692	const backpatch_pop_callee_preserved_regs = try self.addInst(.{
461	.tag = .nop,	693	.tag = .dead,
462	.ops = undefined,	694	.ops = undefined,
463	.data = undefined,	695	.data = undefined,
464	});	696	});
...	@@ -471,22 +703,13 @@ fn gen(self: *Self) InnerError!void {	...	@@ -471,22 +703,13 @@ fn gen(self: *Self) InnerError!void {
471		703
472	// Maybe add rsp, x if required. This is backpatched later.	704	// Maybe add rsp, x if required. This is backpatched later.
473	const backpatch_stack_add = try self.addInst(.{	705	const backpatch_stack_add = try self.addInst(.{
474	.tag = .nop,	706	.tag = .dead,
475	.ops = undefined,	707	.ops = undefined,
476	.data = undefined,	708	.data = undefined,
477	});	709	});
478		710
479	_ = try self.addInst(.{	711	try self.asmRegister(.pop, .rbp);
480	.tag = .pop,	712	try self.asmOpOnly(.ret);
481	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),
482	.data = undefined,
483	});
484
485	_ = try self.addInst(.{
486	.tag = .ret,
487	.ops = Mir.Inst.Ops.encode(.{ .flags = 0b11 }),
488	.data = undefined,
489	});
490		713
491	// Adjust the stack	714	// Adjust the stack
492	if (self.max_end_stack > math.maxInt(i32)) {	715	if (self.max_end_stack > math.maxInt(i32)) {
...	@@ -500,27 +723,34 @@ fn gen(self: *Self) InnerError!void {	...	@@ -500,27 +723,34 @@ fn gen(self: *Self) InnerError!void {
500	if (aligned_stack_end > 0) {	723	if (aligned_stack_end > 0) {
501	self.mir_instructions.set(backpatch_stack_sub, .{	724	self.mir_instructions.set(backpatch_stack_sub, .{
502	.tag = .sub,	725	.tag = .sub,
503	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }),	726	.ops = .ri_u,
504	.data = .{ .imm = aligned_stack_end },	727	.data = .{ .ri = .{
		728	.r1 = .rsp,
		729	.imm = aligned_stack_end,
		730	} },
505	});	731	});
506	self.mir_instructions.set(backpatch_stack_add, .{	732	self.mir_instructions.set(backpatch_stack_add, .{
507	.tag = .add,	733	.tag = .add,
508	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }),	734	.ops = .ri_u,
509	.data = .{ .imm = aligned_stack_end },	735	.data = .{ .ri = .{
		736	.r1 = .rsp,
		737	.imm = aligned_stack_end,
		738	} },
510	});	739	});
511		740
512	const save_reg_list = try self.addExtra(Mir.SaveRegisterList{	741	const save_reg_list = try self.addExtra(Mir.SaveRegisterList{
		742	.base_reg = @enumToInt(Register.rbp),
513	.register_list = reg_list.asInt(),	743	.register_list = reg_list.asInt(),
514	.stack_end = aligned_stack_end,	744	.stack_end = aligned_stack_end,
515	});	745	});
516	self.mir_instructions.set(backpatch_push_callee_preserved_regs, .{	746	self.mir_instructions.set(backpatch_push_callee_preserved_regs, .{
517	.tag = .push_regs,	747	.tag = .push_regs,
518	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),	748	.ops = undefined,
519	.data = .{ .payload = save_reg_list },	749	.data = .{ .payload = save_reg_list },
520	});	750	});
521	self.mir_instructions.set(backpatch_pop_callee_preserved_regs, .{	751	self.mir_instructions.set(backpatch_pop_callee_preserved_regs, .{
522	.tag = .pop_regs,	752	.tag = .pop_regs,
523	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }),	753	.ops = undefined,
524	.data = .{ .payload = save_reg_list },	754	.data = .{ .payload = save_reg_list },
525	});	755	});
526	}	756	}
...	@@ -815,10 +1045,10 @@ fn processDeath(self: *Self, inst: Air.Inst.Index) void {	...	@@ -815,10 +1045,10 @@ fn processDeath(self: *Self, inst: Air.Inst.Index) void {
815	branch.inst_table.putAssumeCapacity(inst, .dead);	1045	branch.inst_table.putAssumeCapacity(inst, .dead);
816	switch (prev_value) {	1046	switch (prev_value) {
817	.register => \|reg\| {	1047	.register => \|reg\| {
818	self.register_manager.freeReg(reg.to64());	1048	self.register_manager.freeReg(reg);
819	},	1049	},
820	.register_overflow => \|ro\| {	1050	.register_overflow => \|ro\| {
821	self.register_manager.freeReg(ro.reg.to64());	1051	self.register_manager.freeReg(ro.reg);
822	self.eflags_inst = null;	1052	self.eflags_inst = null;
823	},	1053	},
824	.eflags => {	1054	.eflags => {
...	@@ -1216,7 +1446,13 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1216,7 +1446,13 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void {
1216	};	1446	};
1217	defer if (dst_mcv_lock) \|lock\| self.register_manager.unlockReg(lock);	1447	defer if (dst_mcv_lock) \|lock\| self.register_manager.unlockReg(lock);
1218		1448
1219	const mask = ~@as(u64, 0);	1449	const mask = switch (operand_ty.abiSize(self.target.*)) {
		1450	1 => ~@as(u8, 0),
		1451	2 => ~@as(u16, 0),
		1452	4 => ~@as(u32, 0),
		1453	8 => ~@as(u64, 0),
		1454	else => unreachable,
		1455	};
1220	try self.genBinOpMir(.xor, operand_ty, dst_mcv, .{ .immediate = mask });	1456	try self.genBinOpMir(.xor, operand_ty, dst_mcv, .{ .immediate = mask });
1221		1457
1222	break :result dst_mcv;	1458	break :result dst_mcv;
...	@@ -1263,14 +1499,7 @@ fn airMin(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1263,14 +1499,7 @@ fn airMin(self: *Self, inst: Air.Inst.Index) !void {
1263	.unsigned => .b,	1499	.unsigned => .b,
1264	.signed => .l,	1500	.signed => .l,
1265	};	1501	};
1266	_ = try self.addInst(.{	1502	try self.asmCmovccRegisterRegister(dst_mcv.register, lhs_reg, cc);
1267	.tag = .cond_mov,
1268	.ops = Mir.Inst.Ops.encode(.{
1269	.reg1 = dst_mcv.register,
1270	.reg2 = lhs_reg,
1271	}),
1272	.data = .{ .cc = cc },
1273	});
1274		1503
1275	break :result dst_mcv;	1504	break :result dst_mcv;
1276	};	1505	};
...	@@ -1470,13 +1699,7 @@ fn genSetStackTruncatedOverflowCompare(	...	@@ -1470,13 +1699,7 @@ fn genSetStackTruncatedOverflowCompare(
1470	.signed => .o,	1699	.signed => .o,
1471	.unsigned => .c,	1700	.unsigned => .c,
1472	};	1701	};
1473	_ = try self.addInst(.{	1702	try self.asmSetccRegister(overflow_reg.to8(), cc);
1474	.tag = .cond_set_byte,
1475	.ops = Mir.Inst.Ops.encode(.{
1476	.reg1 = overflow_reg.to8(),
1477	}),
1478	.data = .{ .cc = cc },
1479	});
1480		1703
1481	const scratch_reg = temp_regs[1];	1704	const scratch_reg = temp_regs[1];
1482	try self.genSetReg(extended_ty, scratch_reg, .{ .register = reg });	1705	try self.genSetReg(extended_ty, scratch_reg, .{ .register = reg });
...	@@ -1489,12 +1712,7 @@ fn genSetStackTruncatedOverflowCompare(	...	@@ -1489,12 +1712,7 @@ fn genSetStackTruncatedOverflowCompare(
1489	);	1712	);
1490		1713
1491	const eq_reg = temp_regs[2];	1714	const eq_reg = temp_regs[2];
1492	_ = try self.addInst(.{	1715	try self.asmSetccRegister(eq_reg.to8(), .ne);
1493	.tag = .cond_set_byte,
1494	.ops = Mir.Inst.Ops.encode(.{ .reg1 = eq_reg.to8() }),
1495	.data = .{ .cc = .ne },
1496	});
1497
1498	try self.genBinOpMir(	1716	try self.genBinOpMir(
1499	.@"or",	1717	.@"or",
1500	Type.u8,	1718	Type.u8,
...	@@ -1638,23 +1856,8 @@ fn genIntMulDivOpMir(	...	@@ -1638,23 +1856,8 @@ fn genIntMulDivOpMir(
1638	}	1856	}
1639		1857
1640	switch (signedness) {	1858	switch (signedness) {
1641	.signed => {	1859	.signed => try self.asmOpOnly(.cqo),
1642	_ = try self.addInst(.{	1860	.unsigned => try self.asmRegisterRegister(.xor, .rdx, .rdx),
1643	.tag = .cwd,
1644	.ops = Mir.Inst.Ops.encode(.{ .flags = 0b11 }),
1645	.data = undefined,
1646	});
1647	},
1648	.unsigned => {
1649	_ = try self.addInst(.{
1650	.tag = .xor,
1651	.ops = Mir.Inst.Ops.encode(.{
1652	.reg1 = .rdx,
1653	.reg2 = .rdx,
1654	}),
1655	.data = undefined,
1656	});
1657	},
1658	}	1861	}
1659		1862
1660	const factor = switch (rhs) {	1863	const factor = switch (rhs) {
...	@@ -1667,29 +1870,11 @@ fn genIntMulDivOpMir(	...	@@ -1667,29 +1870,11 @@ fn genIntMulDivOpMir(
1667	};	1870	};
1668		1871
1669	switch (factor) {	1872	switch (factor) {
1670	.register => \|reg\| {	1873	.register => \|reg\| try self.asmRegister(tag, reg),
1671	_ = try self.addInst(.{	1874	.stack_offset => \|off\| try self.asmMemory(tag, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
1672	.tag = tag,	1875	.base = .rbp,
1673	.ops = Mir.Inst.Ops.encode(.{ .reg1 = reg }),	1876	.disp = -off,
1674	.data = undefined,	1877	})),
1675	});
1676	},
1677	.stack_offset => \|off\| {
1678	_ = try self.addInst(.{
1679	.tag = tag,
1680	.ops = Mir.Inst.Ops.encode(.{
1681	.reg2 = .rbp,
1682	.flags = switch (abi_size) {
1683	1 => 0b00,
1684	2 => 0b01,
1685	4 => 0b10,
1686	8 => 0b11,
1687	else => unreachable,
1688	},
1689	}),
1690	.data = .{ .imm = @bitCast(u32, -off) },
1691	});
1692	},
1693	else => unreachable,	1878	else => unreachable,
1694	}	1879	}
1695	}	1880	}
...	@@ -1717,38 +1902,10 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa	...	@@ -1717,38 +1902,10 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
1717	.unsigned => .div,	1902	.unsigned => .div,
1718	}, Type.isize, signedness, .{ .register = dividend }, .{ .register = divisor });	1903	}, Type.isize, signedness, .{ .register = dividend }, .{ .register = divisor });
1719		1904
1720	_ = try self.addInst(.{	1905	try self.asmRegisterRegister(.xor, divisor.to64(), dividend.to64());
1721	.tag = .xor,	1906	try self.asmRegisterImmediate(.sar, divisor.to64(), Immediate.u(63));
1722	.ops = Mir.Inst.Ops.encode(.{	1907	try self.asmRegisterRegister(.@"test", .rdx, .rdx);
1723	.reg1 = divisor.to64(),	1908	try self.asmCmovccRegisterRegister(divisor.to64(), .rdx, .e);
1724	.reg2 = dividend.to64(),
1725	}),
1726	.data = undefined,
1727	});
1728	_ = try self.addInst(.{
1729	.tag = .sar,
1730	.ops = Mir.Inst.Ops.encode(.{
1731	.reg1 = divisor.to64(),
1732	.flags = 0b10,
1733	}),
1734	.data = .{ .imm = 63 },
1735	});
1736	_ = try self.addInst(.{
1737	.tag = .@"test",
1738	.ops = Mir.Inst.Ops.encode(.{
1739	.reg1 = .rdx,
1740	.reg2 = .rdx,
1741	}),
1742	.data = undefined,
1743	});
1744	_ = try self.addInst(.{
1745	.tag = .cond_mov,
1746	.ops = Mir.Inst.Ops.encode(.{
1747	.reg1 = divisor.to64(),
1748	.reg2 = .rdx,
1749	}),
1750	.data = .{ .cc = .e },
1751	});
1752	try self.genBinOpMir(.add, Type.isize, .{ .register = divisor }, .{ .register = .rax });	1909	try self.genBinOpMir(.add, Type.isize, .{ .register = divisor }, .{ .register = .rax });
1753	return MCValue{ .register = divisor };	1910	return MCValue{ .register = divisor };
1754	}	1911	}
...	@@ -2182,18 +2339,10 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {	...	@@ -2182,18 +2339,10 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
2182		2339
2183	const addr_reg = try self.register_manager.allocReg(null, gp);	2340	const addr_reg = try self.register_manager.allocReg(null, gp);
2184	switch (slice_mcv) {	2341	switch (slice_mcv) {
2185	.stack_offset => \|off\| {	2342	.stack_offset => \|off\| try self.asmRegisterMemory(.mov, addr_reg.to64(), Memory.sib(.qword, .{
2186	// mov reg, [rbp - 8]	2343	.base = .rbp,
2187	_ = try self.addInst(.{	2344	.disp = -off,
2188	.tag = .mov,	2345	})),
2189	.ops = Mir.Inst.Ops.encode(.{
2190	.reg1 = addr_reg.to64(),
2191	.reg2 = .rbp,
2192	.flags = 0b01,
2193	}),
2194	.data = .{ .imm = @bitCast(u32, -@intCast(i32, off)) },
2195	});
2196	},
2197	else => return self.fail("TODO implement slice_elem_ptr when slice is {}", .{slice_mcv}),	2346	else => return self.fail("TODO implement slice_elem_ptr when slice is {}", .{slice_mcv}),
2198	}	2347	}
2199	// TODO we could allocate register here, but need to expect addr register and potentially	2348	// TODO we could allocate register here, but need to expect addr register and potentially
...	@@ -2268,26 +2417,16 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2268,26 +2417,16 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
2268	array_ty.abiAlignment(self.target.*),	2417	array_ty.abiAlignment(self.target.*),
2269	));	2418	));
2270	try self.genSetStack(array_ty, off, array, .{});	2419	try self.genSetStack(array_ty, off, array, .{});
2271	// lea reg, [rbp]	2420	try self.asmRegisterMemory(.lea, addr_reg.to64(), Memory.sib(.qword, .{
2272	_ = try self.addInst(.{	2421	.base = .rbp,
2273	.tag = .lea,	2422	.disp = -off,
2274	.ops = Mir.Inst.Ops.encode(.{	2423	}));
2275	.reg1 = addr_reg.to64(),
2276	.reg2 = .rbp,
2277	}),
2278	.data = .{ .imm = @bitCast(u32, -off) },
2279	});
2280	},	2424	},
2281	.stack_offset => \|off\| {	2425	.stack_offset => \|off\| {
2282	// lea reg, [rbp]	2426	try self.asmRegisterMemory(.lea, addr_reg.to64(), Memory.sib(.qword, .{
2283	_ = try self.addInst(.{	2427	.base = .rbp,
2284	.tag = .lea,	2428	.disp = -off,
2285	.ops = Mir.Inst.Ops.encode(.{	2429	}));
2286	.reg1 = addr_reg.to64(),
2287	.reg2 = .rbp,
2288	}),
2289	.data = .{ .imm = @bitCast(u32, -off) },
2290	});
2291	},	2430	},
2292	.memory, .linker_load => {	2431	.memory, .linker_load => {
2293	try self.loadMemPtrIntoRegister(addr_reg, Type.usize, array);	2432	try self.loadMemPtrIntoRegister(addr_reg, Type.usize, array);
...	@@ -2324,7 +2463,7 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2324,7 +2463,7 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
2324	defer if (ptr_lock) \|lock\| self.register_manager.unlockReg(lock);	2463	defer if (ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
2325		2464
2326	const elem_ty = ptr_ty.elemType2();	2465	const elem_ty = ptr_ty.elemType2();
2327	const elem_abi_size = elem_ty.abiSize(self.target.*);	2466	const elem_abi_size = @intCast(u32, elem_ty.abiSize(self.target.*));
2328	const index_ty = self.air.typeOf(bin_op.rhs);	2467	const index_ty = self.air.typeOf(bin_op.rhs);
2329	const index = try self.resolveInst(bin_op.rhs);	2468	const index = try self.resolveInst(bin_op.rhs);
2330	const index_lock: ?RegisterLock = switch (index) {	2469	const index_lock: ?RegisterLock = switch (index) {
...	@@ -2344,16 +2483,14 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2344,16 +2483,14 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
2344	if (elem_abi_size > 8) {	2483	if (elem_abi_size > 8) {
2345	return self.fail("TODO copy value with size {} from pointer", .{elem_abi_size});	2484	return self.fail("TODO copy value with size {} from pointer", .{elem_abi_size});
2346	} else {	2485	} else {
2347	// mov dst_mcv, [dst_mcv]	2486	try self.asmRegisterMemory(
2348	_ = try self.addInst(.{	2487	.mov,
2349	.tag = .mov,	2488	registerAlias(dst_mcv.register, elem_abi_size),
2350	.ops = Mir.Inst.Ops.encode(.{	2489	Memory.sib(Memory.PtrSize.fromSize(elem_abi_size), .{
2351	.reg1 = registerAlias(dst_mcv.register, @intCast(u32, elem_abi_size)),	2490	.base = dst_mcv.register,
2352	.reg2 = dst_mcv.register,	2491	.disp = 0,
2353	.flags = 0b01,
2354	}),	2492	}),
2355	.data = .{ .imm = 0 },	2493	);
2356	});
2357	break :result .{ .register = registerAlias(dst_mcv.register, @intCast(u32, elem_abi_size)) };	2494	break :result .{ .register = registerAlias(dst_mcv.register, @intCast(u32, elem_abi_size)) };
2358	}	2495	}
2359	};	2496	};
...	@@ -2580,7 +2717,7 @@ fn reuseOperand(	...	@@ -2580,7 +2717,7 @@ fn reuseOperand(
2580		2717
2581	fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!void {	2718	fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!void {
2582	const elem_ty = ptr_ty.elemType();	2719	const elem_ty = ptr_ty.elemType();
2583	const abi_size = elem_ty.abiSize(self.target.*);	2720	const abi_size = @intCast(u32, elem_ty.abiSize(self.target.*));
2584	switch (ptr) {	2721	switch (ptr) {
2585	.none => unreachable,	2722	.none => unreachable,
2586	.undef => unreachable,	2723	.undef => unreachable,
...	@@ -2607,16 +2744,11 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo	...	@@ -2607,16 +2744,11 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
2607	.undef => unreachable,	2744	.undef => unreachable,
2608	.eflags => unreachable,	2745	.eflags => unreachable,
2609	.register => \|dst_reg\| {	2746	.register => \|dst_reg\| {
2610	// mov dst_reg, [reg]	2747	try self.asmRegisterMemory(
2611	_ = try self.addInst(.{	2748	.mov,
2612	.tag = .mov,	2749	registerAlias(dst_reg, abi_size),
2613	.ops = Mir.Inst.Ops.encode(.{	2750	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg, .disp = 0 }),
2614	.reg1 = registerAlias(dst_reg, @intCast(u32, abi_size)),	2751	);
2615	.reg2 = reg,
2616	.flags = 0b01,
2617	}),
2618	.data = .{ .imm = 0 },
2619	});
2620	},	2752	},
2621	.stack_offset => \|off\| {	2753	.stack_offset => \|off\| {
2622	if (abi_size <= 8) {	2754	if (abi_size <= 8) {
...	@@ -2676,23 +2808,19 @@ fn loadMemPtrIntoRegister(self: *Self, reg: Register, ptr_ty: Type, ptr: MCValue	...	@@ -2676,23 +2808,19 @@ fn loadMemPtrIntoRegister(self: *Self, reg: Register, ptr_ty: Type, ptr: MCValue
2676	const atom = try coff_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl);	2808	const atom = try coff_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl);
2677	break :blk coff_file.getAtom(atom).getSymbolIndex().?;	2809	break :blk coff_file.getAtom(atom).getSymbolIndex().?;
2678	} else unreachable;	2810	} else unreachable;
2679	const flags: u2 = switch (load_struct.type) {	2811	const ops: Mir.Inst.Ops = switch (load_struct.type) {
2680	.got => 0b00,	2812	.got => .got_reloc,
2681	.direct => 0b01,	2813	.direct => .direct_reloc,
2682	.import => 0b10,	2814	.import => .import_reloc,
2683	};	2815	};
2684	_ = try self.addInst(.{	2816	_ = try self.addInst(.{
2685	.tag = .lea_pic,	2817	.tag = .lea_linker,
2686	.ops = Mir.Inst.Ops.encode(.{	2818	.ops = ops,
2687	.reg1 = registerAlias(reg, abi_size),	2819	.data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
2688	.flags = flags,	2820	.reg = @enumToInt(registerAlias(reg, abi_size)),
2689	}),	2821	.atom_index = atom_index,
2690	.data = .{	2822	.sym_index = load_struct.sym_index,
2691	.relocation = .{	2823	}) },
2692	.atom_index = atom_index,
2693	.sym_index = load_struct.sym_index,
2694	},
2695	},
2696	});	2824	});
2697	},	2825	},
2698	.memory => \|addr\| {	2826	.memory => \|addr\| {
...	@@ -2705,7 +2833,7 @@ fn loadMemPtrIntoRegister(self: *Self, reg: Register, ptr_ty: Type, ptr: MCValue	...	@@ -2705,7 +2833,7 @@ fn loadMemPtrIntoRegister(self: *Self, reg: Register, ptr_ty: Type, ptr: MCValue
2705	}	2833	}
2706		2834
2707	fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type) InnerError!void {	2835	fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type) InnerError!void {
2708	const abi_size = value_ty.abiSize(self.target.*);	2836	const abi_size = @intCast(u32, value_ty.abiSize(self.target.*));
2709	switch (ptr) {	2837	switch (ptr) {
2710	.none => unreachable,	2838	.none => unreachable,
2711	.undef => unreachable,	2839	.undef => unreachable,
...	@@ -2738,25 +2866,14 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type	...	@@ -2738,25 +2866,14 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2738	.immediate => \|imm\| {	2866	.immediate => \|imm\| {
2739	switch (abi_size) {	2867	switch (abi_size) {
2740	1, 2, 4 => {	2868	1, 2, 4 => {
2741	// TODO this is wasteful!	2869	const immediate = if (value_ty.isSignedInt())
2742	// introduce new MIR tag specifically for mov [reg + 0], imm	2870	Immediate.s(@intCast(i32, @bitCast(i64, imm)))
2743	const payload = try self.addExtra(Mir.ImmPair{	2871	else
2744	.dest_off = 0,	2872	Immediate.u(@truncate(u32, imm));
2745	.operand = @truncate(u32, imm),	2873	try self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
2746	});	2874	.base = reg.to64(),
2747	_ = try self.addInst(.{	2875	.disp = 0,
2748	.tag = .mov_mem_imm,	2876	}), immediate);
2749	.ops = Mir.Inst.Ops.encode(.{
2750	.reg1 = reg.to64(),
2751	.flags = switch (abi_size) {
2752	1 => 0b00,
2753	2 => 0b01,
2754	4 => 0b10,
2755	else => unreachable,
2756	},
2757	}),
2758	.data = .{ .payload = payload },
2759	});
2760	},	2877	},
2761	8 => {	2878	8 => {
2762	// TODO: optimization: if the imm is only using the lower	2879	// TODO: optimization: if the imm is only using the lower
...	@@ -2786,13 +2903,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type	...	@@ -2786,13 +2903,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2786	const overflow_bit_ty = value_ty.structFieldType(1);	2903	const overflow_bit_ty = value_ty.structFieldType(1);
2787	const overflow_bit_offset = value_ty.structFieldOffset(1, self.target.*);	2904	const overflow_bit_offset = value_ty.structFieldOffset(1, self.target.*);
2788	const tmp_reg = try self.register_manager.allocReg(null, gp);	2905	const tmp_reg = try self.register_manager.allocReg(null, gp);
2789	_ = try self.addInst(.{	2906	try self.asmSetccRegister(tmp_reg.to8(), ro.eflags);
2790	.tag = .cond_set_byte,
2791	.ops = Mir.Inst.Ops.encode(.{
2792	.reg1 = tmp_reg.to8(),
2793	}),
2794	.data = .{ .cc = ro.eflags },
2795	});
2796	try self.genInlineMemcpyRegisterRegister(	2907	try self.genInlineMemcpyRegisterRegister(
2797	overflow_bit_ty,	2908	overflow_bit_ty,
2798	reg,	2909	reg,
...	@@ -2833,17 +2944,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type	...	@@ -2833,17 +2944,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2833		2944
2834	try self.loadMemPtrIntoRegister(addr_reg, ptr_ty, ptr);	2945	try self.loadMemPtrIntoRegister(addr_reg, ptr_ty, ptr);
2835		2946
2836	// to get the actual address of the value we want to modify we have to go through the GOT	2947	// To get the actual address of the value we want to modify we have to go through the GOT
2837	// mov reg, [reg]	2948	try self.asmRegisterMemory(.mov, addr_reg.to64(), Memory.sib(.qword, .{
2838	_ = try self.addInst(.{	2949	.base = addr_reg.to64(),
2839	.tag = .mov,	2950	.disp = 0,
2840	.ops = Mir.Inst.Ops.encode(.{	2951	}));
2841	.reg1 = addr_reg.to64(),
2842	.reg2 = addr_reg.to64(),
2843	.flags = 0b01,
2844	}),
2845	.data = .{ .imm = 0 },
2846	});
2847		2952
2848	const new_ptr = MCValue{ .register = addr_reg.to64() };	2953	const new_ptr = MCValue{ .register = addr_reg.to64() };
2849		2954
...	@@ -2853,19 +2958,8 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type	...	@@ -2853,19 +2958,8 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2853	return self.fail("TODO saving imm to memory for abi_size {}", .{abi_size});	2958	return self.fail("TODO saving imm to memory for abi_size {}", .{abi_size});
2854	}	2959	}
2855		2960
2856	const payload = try self.addExtra(Mir.ImmPair{	2961	if (abi_size == 8) {
2857	.dest_off = 0,	2962	// TODO
2858	// TODO check if this logic is correct
2859	.operand = @truncate(u32, imm),
2860	});
2861	const flags: u2 = switch (abi_size) {
2862	1 => 0b00,
2863	2 => 0b01,
2864	4 => 0b10,
2865	8 => 0b11,
2866	else => unreachable,
2867	};
2868	if (flags == 0b11) {
2869	const top_bits: u32 = @intCast(u32, imm >> 32);	2963	const top_bits: u32 = @intCast(u32, imm >> 32);
2870	const can_extend = if (value_ty.isUnsignedInt())	2964	const can_extend = if (value_ty.isUnsignedInt())
2871	(top_bits == 0) and (imm & 0x8000_0000) == 0	2965	(top_bits == 0) and (imm & 0x8000_0000) == 0
...	@@ -2876,14 +2970,10 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type	...	@@ -2876,14 +2970,10 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2876	return self.fail("TODO imm64 would get incorrectly sign extended", .{});	2970	return self.fail("TODO imm64 would get incorrectly sign extended", .{});
2877	}	2971	}
2878	}	2972	}
2879	_ = try self.addInst(.{	2973	try self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
2880	.tag = .mov_mem_imm,	2974	.base = addr_reg.to64(),
2881	.ops = Mir.Inst.Ops.encode(.{	2975	.disp = 0,
2882	.reg1 = addr_reg.to64(),	2976	}), Immediate.u(@intCast(u32, imm)));
2883	.flags = flags,
2884	}),
2885	.data = .{ .payload = payload },
2886	});
2887	},	2977	},
2888	.register => {	2978	.register => {
2889	return self.store(new_ptr, value, ptr_ty, value_ty);	2979	return self.store(new_ptr, value, ptr_ty, value_ty);
...	@@ -2895,16 +2985,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type	...	@@ -2895,16 +2985,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2895	defer self.register_manager.unlockReg(tmp_reg_lock);	2985	defer self.register_manager.unlockReg(tmp_reg_lock);
2896		2986
2897	try self.loadMemPtrIntoRegister(tmp_reg, value_ty, value);	2987	try self.loadMemPtrIntoRegister(tmp_reg, value_ty, value);
		2988	try self.asmRegisterMemory(.mov, tmp_reg, Memory.sib(.qword, .{
		2989	.base = tmp_reg,
		2990	.disp = 0,
		2991	}));
2898		2992
2899	_ = try self.addInst(.{
2900	.tag = .mov,
2901	.ops = Mir.Inst.Ops.encode(.{
2902	.reg1 = tmp_reg,
2903	.reg2 = tmp_reg,
2904	.flags = 0b01,
2905	}),
2906	.data = .{ .imm = 0 },
2907	});
2908	return self.store(new_ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);	2993	return self.store(new_ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
2909	}	2994	}
2910		2995
...	@@ -3052,8 +3137,8 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3052,8 +3137,8 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
3052	const shift = @intCast(u8, struct_field_offset * @sizeOf(usize));	3137	const shift = @intCast(u8, struct_field_offset * @sizeOf(usize));
3053	try self.genShiftBinOpMir(.shr, Type.usize, dst_mcv.register, .{ .immediate = shift });	3138	try self.genShiftBinOpMir(.shr, Type.usize, dst_mcv.register, .{ .immediate = shift });
3054		3139
3055	// Mask with reg.size() - struct_field_size	3140	// Mask with reg.bitSize() - struct_field_size
3056	const max_reg_bit_width = Register.rax.size();	3141	const max_reg_bit_width = Register.rax.bitSize();
3057	const mask_shift = @intCast(u6, (max_reg_bit_width - struct_field_ty.bitSize(self.target.*)));	3142	const mask_shift = @intCast(u6, (max_reg_bit_width - struct_field_ty.bitSize(self.target.*)));
3058	const mask = (~@as(u64, 0)) >> mask_shift;	3143	const mask = (~@as(u64, 0)) >> mask_shift;
3059		3144
...	@@ -3066,14 +3151,11 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3066,14 +3151,11 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
3066	};	3151	};
3067	const field_size = @intCast(u32, struct_field_ty.abiSize(self.target.*));	3152	const field_size = @intCast(u32, struct_field_ty.abiSize(self.target.*));
3068	if (signedness == .signed and field_size < 8) {	3153	if (signedness == .signed and field_size < 8) {
3069	_ = try self.addInst(.{	3154	try self.asmRegisterRegister(
3070	.tag = .mov_sign_extend,	3155	.movsx,
3071	.ops = Mir.Inst.Ops.encode(.{	3156	dst_mcv.register,
3072	.reg1 = dst_mcv.register,	3157	registerAlias(dst_mcv.register, field_size),
3073	.reg2 = registerAlias(dst_mcv.register, field_size),	3158	);
3074	}),
3075	.data = undefined,
3076	});
3077	}	3159	}
3078		3160
3079	break :result dst_mcv;	3161	break :result dst_mcv;
...	@@ -3090,13 +3172,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3090,13 +3172,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
3090	defer self.register_manager.unlockReg(reg_lock);	3172	defer self.register_manager.unlockReg(reg_lock);
3091		3173
3092	const dst_reg = try self.register_manager.allocReg(inst, gp);	3174	const dst_reg = try self.register_manager.allocReg(inst, gp);
3093	_ = try self.addInst(.{	3175	try self.asmSetccRegister(dst_reg.to8(), ro.eflags);
3094	.tag = .cond_set_byte,
3095	.ops = Mir.Inst.Ops.encode(.{
3096	.reg1 = dst_reg.to8(),
3097	}),
3098	.data = .{ .cc = ro.eflags },
3099	});
3100	break :result MCValue{ .register = dst_reg.to8() };	3176	break :result MCValue{ .register = dst_reg.to8() };
3101	},	3177	},
3102	else => unreachable,	3178	else => unreachable,
...	@@ -3132,25 +3208,7 @@ fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shi	...	@@ -3132,25 +3208,7 @@ fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shi
3132	switch (shift) {	3208	switch (shift) {
3133	.immediate => \|imm\| switch (imm) {	3209	.immediate => \|imm\| switch (imm) {
3134	0 => return,	3210	0 => return,
3135	1 => {	3211	else => return self.asmRegisterImmediate(tag, registerAlias(reg, abi_size), Immediate.u(imm)),
3136	_ = try self.addInst(.{
3137	.tag = tag,
3138	.ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(reg, abi_size) }),
3139	.data = undefined,
3140	});
3141	return;
3142	},
3143	else => {
3144	_ = try self.addInst(.{
3145	.tag = tag,
3146	.ops = Mir.Inst.Ops.encode(.{
3147	.reg1 = registerAlias(reg, abi_size),
3148	.flags = 0b10,
3149	}),
3150	.data = .{ .imm = @intCast(u8, imm) },
3151	});
3152	return;
3153	},
3154	},	3212	},
3155	.register => \|shift_reg\| {	3213	.register => \|shift_reg\| {
3156	if (shift_reg == .rcx) break :blk;	3214	if (shift_reg == .rcx) break :blk;
...	@@ -3162,14 +3220,7 @@ fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shi	...	@@ -3162,14 +3220,7 @@ fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shi
3162	try self.genSetReg(Type.u8, .rcx, shift);	3220	try self.genSetReg(Type.u8, .rcx, shift);
3163	}	3221	}
3164		3222
3165	_ = try self.addInst(.{	3223	try self.asmRegisterRegister(tag, registerAlias(reg, abi_size), .cl);
3166	.tag = tag,
3167	.ops = Mir.Inst.Ops.encode(.{
3168	.reg1 = registerAlias(reg, abi_size),
3169	.flags = 0b01,
3170	}),
3171	.data = undefined,
3172	});
3173	}	3224	}
3174		3225
3175	/// Result is always a register.	3226	/// Result is always a register.
...	@@ -3542,59 +3593,44 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu	...	@@ -3542,59 +3593,44 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
3542	if (intrinsicsAllowed(self.target.*, dst_ty)) {	3593	if (intrinsicsAllowed(self.target.*, dst_ty)) {
3543	const actual_tag: Mir.Inst.Tag = switch (dst_ty.tag()) {	3594	const actual_tag: Mir.Inst.Tag = switch (dst_ty.tag()) {
3544	.f32 => switch (mir_tag) {	3595	.f32 => switch (mir_tag) {
3545	.add => if (hasAvxSupport(self.target.*))	3596	.add => .addss,
3546	Mir.Inst.Tag.add_f32_avx	3597	.cmp => .ucomiss,
3547	else	3598	else => return self.fail(
3548	Mir.Inst.Tag.add_f32_sse,	3599	"TODO genBinOpMir for f32 register-register with MIR tag {}",
3549	.cmp => if (hasAvxSupport(self.target.*))	3600	.{mir_tag},
3550	Mir.Inst.Tag.cmp_f32_avx	3601	),
3551	else
3552	Mir.Inst.Tag.cmp_f32_sse,
3553	else => return self.fail("TODO genBinOpMir for f32 register-register with MIR tag {}", .{mir_tag}),
3554	},	3602	},
3555	.f64 => switch (mir_tag) {	3603	.f64 => switch (mir_tag) {
3556	.add => if (hasAvxSupport(self.target.*))	3604	.add => .addsd,
3557	Mir.Inst.Tag.add_f64_avx	3605	.cmp => .ucomisd,
3558	else	3606	else => return self.fail(
3559	Mir.Inst.Tag.add_f64_sse,	3607	"TODO genBinOpMir for f64 register-register with MIR tag {}",
3560	.cmp => if (hasAvxSupport(self.target.*))	3608	.{mir_tag},
3561	Mir.Inst.Tag.cmp_f64_avx	3609	),
3562	else
3563	Mir.Inst.Tag.cmp_f64_sse,
3564	else => return self.fail("TODO genBinOpMir for f64 register-register with MIR tag {}", .{mir_tag}),
3565	},	3610	},
3566	else => return self.fail("TODO genBinOpMir for float register-register and type {}", .{dst_ty.fmtDebug()}),	3611	else => return self.fail(
		3612	"TODO genBinOpMir for float register-register and type {}",
		3613	.{dst_ty.fmtDebug()},
		3614	),
3567	};	3615	};
3568	_ = try self.addInst(.{	3616	return self.asmRegisterRegister(actual_tag, dst_reg.to128(), src_reg.to128());
3569	.tag = actual_tag,
3570	.ops = Mir.Inst.Ops.encode(.{
3571	.reg1 = dst_reg.to128(),
3572	.reg2 = src_reg.to128(),
3573	}),
3574	.data = undefined,
3575	});
3576	return;
3577	}	3617	}
3578		3618
3579	return self.fail("TODO genBinOpMir for float register-register and no intrinsics", .{});	3619	return self.fail("TODO genBinOpMir for float register-register and no intrinsics", .{});
3580	},	3620	},
3581	else => {	3621	else => try self.asmRegisterRegister(
3582	_ = try self.addInst(.{	3622	mir_tag,
3583	.tag = mir_tag,	3623	registerAlias(dst_reg, abi_size),
3584	.ops = Mir.Inst.Ops.encode(.{	3624	registerAlias(src_reg, abi_size),
3585	.reg1 = registerAlias(dst_reg, abi_size),	3625	),
3586	.reg2 = registerAlias(src_reg, abi_size),
3587	}),
3588	.data = undefined,
3589	});
3590	},
3591	},	3626	},
3592	.immediate => \|imm\| {	3627	.immediate => \|imm\| {
3593	_ = try self.addInst(.{	3628	// TODO
3594	.tag = mir_tag,	3629	try self.asmRegisterImmediate(
3595	.ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(dst_reg, abi_size) }),	3630	mir_tag,
3596	.data = .{ .imm = @truncate(u32, imm) },	3631	registerAlias(dst_reg, abi_size),
3597	});	3632	Immediate.u(@intCast(u32, imm)),
		3633	);
3598	},	3634	},
3599	.memory,	3635	.memory,
3600	.linker_load,	3636	.linker_load,
...	@@ -3611,15 +3647,11 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu	...	@@ -3611,15 +3647,11 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
3611	if (off > math.maxInt(i32)) {	3647	if (off > math.maxInt(i32)) {
3612	return self.fail("stack offset too large", .{});	3648	return self.fail("stack offset too large", .{});
3613	}	3649	}
3614	_ = try self.addInst(.{	3650	try self.asmRegisterMemory(
3615	.tag = mir_tag,	3651	mir_tag,
3616	.ops = Mir.Inst.Ops.encode(.{	3652	registerAlias(dst_reg, abi_size),
3617	.reg1 = registerAlias(dst_reg, abi_size),	3653	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -off }),
3618	.reg2 = .rbp,	3654	);
3619	.flags = 0b01,
3620	}),
3621	.data = .{ .imm = @bitCast(u32, -off) },
3622	});
3623	},	3655	},
3624	}	3656	}
3625	},	3657	},
...	@@ -3637,45 +3669,17 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu	...	@@ -3637,45 +3669,17 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
3637	.dead, .unreach => unreachable,	3669	.dead, .unreach => unreachable,
3638	.register_overflow => unreachable,	3670	.register_overflow => unreachable,
3639	.register => \|src_reg\| {	3671	.register => \|src_reg\| {
3640	_ = try self.addInst(.{	3672	try self.asmMemoryRegister(mir_tag, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
3641	.tag = mir_tag,	3673	.base = .rbp,
3642	.ops = Mir.Inst.Ops.encode(.{	3674	.disp = -off,
3643	.reg1 = .rbp,	3675	}), registerAlias(src_reg, abi_size));
3644	.reg2 = registerAlias(src_reg, abi_size),
3645	.flags = 0b10,
3646	}),
3647	.data = .{ .imm = @bitCast(u32, -off) },
3648	});
3649	},	3676	},
3650	.immediate => \|imm\| {	3677	.immediate => \|imm\| {
3651	const tag: Mir.Inst.Tag = switch (mir_tag) {	3678	// TODO
3652	.add => .add_mem_imm,	3679	try self.asmMemoryImmediate(mir_tag, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
3653	.@"or" => .or_mem_imm,	3680	.base = .rbp,
3654	.@"and" => .and_mem_imm,	3681	.disp = -off,
3655	.sub => .sub_mem_imm,	3682	}), Immediate.u(@intCast(u32, imm)));
3656	.xor => .xor_mem_imm,
3657	.cmp => .cmp_mem_imm,
3658	else => unreachable,
3659	};
3660	const flags: u2 = switch (abi_size) {
3661	1 => 0b00,
3662	2 => 0b01,
3663	4 => 0b10,
3664	8 => 0b11,
3665	else => unreachable,
3666	};
3667	const payload = try self.addExtra(Mir.ImmPair{
3668	.dest_off = @bitCast(u32, -off),
3669	.operand = @truncate(u32, imm),
3670	});
3671	_ = try self.addInst(.{
3672	.tag = tag,
3673	.ops = Mir.Inst.Ops.encode(.{
3674	.reg1 = .rbp,
3675	.flags = flags,
3676	}),
3677	.data = .{ .payload = payload },
3678	});
3679	},	3683	},
3680	.memory,	3684	.memory,
3681	.stack_offset,	3685	.stack_offset,
...	@@ -3718,30 +3722,21 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M	...	@@ -3718,30 +3722,21 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
3718	.dead, .unreach => unreachable,	3722	.dead, .unreach => unreachable,
3719	.ptr_stack_offset => unreachable,	3723	.ptr_stack_offset => unreachable,
3720	.register_overflow => unreachable,	3724	.register_overflow => unreachable,
3721	.register => \|src_reg\| {	3725	.register => \|src_reg\| try self.asmRegisterRegister(
3722	// register, register	3726	.imul,
3723	_ = try self.addInst(.{	3727	registerAlias(dst_reg, abi_size),
3724	.tag = .imul_complex,	3728	registerAlias(src_reg, abi_size),
3725	.ops = Mir.Inst.Ops.encode(.{	3729	),
3726	.reg1 = registerAlias(dst_reg, abi_size),
3727	.reg2 = registerAlias(src_reg, abi_size),
3728	}),
3729	.data = undefined,
3730	});
3731	},
3732	.immediate => \|imm\| {	3730	.immediate => \|imm\| {
3733	// TODO take into account the type's ABI size when selecting the register alias
3734	// register, immediate
3735	if (math.minInt(i32) <= imm and imm <= math.maxInt(i32)) {	3731	if (math.minInt(i32) <= imm and imm <= math.maxInt(i32)) {
3736	_ = try self.addInst(.{	3732	// TODO take into account the type's ABI size when selecting the register alias
3737	.tag = .imul_complex,	3733	// register, immediate
3738	.ops = Mir.Inst.Ops.encode(.{	3734	try self.asmRegisterRegisterImmediate(
3739	.reg1 = dst_reg.to32(),	3735	.imul,
3740	.reg2 = dst_reg.to32(),	3736	dst_reg.to32(),
3741	.flags = 0b10,	3737	dst_reg.to32(),
3742	}),	3738	Immediate.u(@intCast(u32, imm)),
3743	.data = .{ .imm = @truncate(u32, imm) },	3739	);
3744	});
3745	} else {	3740	} else {
3746	// TODO verify we don't spill and assign to the same register as dst_mcv	3741	// TODO verify we don't spill and assign to the same register as dst_mcv
3747	const src_reg = try self.copyToTmpRegister(dst_ty, src_mcv);	3742	const src_reg = try self.copyToTmpRegister(dst_ty, src_mcv);
...	@@ -3749,15 +3744,11 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M	...	@@ -3749,15 +3744,11 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
3749	}	3744	}
3750	},	3745	},
3751	.stack_offset => \|off\| {	3746	.stack_offset => \|off\| {
3752	_ = try self.addInst(.{	3747	try self.asmRegisterMemory(
3753	.tag = .imul_complex,	3748	.imul,
3754	.ops = Mir.Inst.Ops.encode(.{	3749	registerAlias(dst_reg, abi_size),
3755	.reg1 = registerAlias(dst_reg, abi_size),	3750	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -off }),
3756	.reg2 = .rbp,	3751	);
3757	.flags = 0b01,
3758	}),
3759	.data = .{ .imm = @bitCast(u32, -off) },
3760	});
3761	},	3752	},
3762	.memory => {	3753	.memory => {
3763	return self.fail("TODO implement x86 multiply source memory", .{});	3754	return self.fail("TODO implement x86 multiply source memory", .{});
...	@@ -3780,16 +3771,11 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M	...	@@ -3780,16 +3771,11 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
3780	.register => \|src_reg\| {	3771	.register => \|src_reg\| {
3781	// copy dst to a register	3772	// copy dst to a register
3782	const dst_reg = try self.copyToTmpRegister(dst_ty, dst_mcv);	3773	const dst_reg = try self.copyToTmpRegister(dst_ty, dst_mcv);
3783	// multiply into dst_reg	3774	try self.asmRegisterRegister(
3784	// register, register	3775	.imul,
3785	_ = try self.addInst(.{	3776	registerAlias(dst_reg, abi_size),
3786	.tag = .imul_complex,	3777	registerAlias(src_reg, abi_size),
3787	.ops = Mir.Inst.Ops.encode(.{	3778	);
3788	.reg1 = registerAlias(dst_reg, abi_size),
3789	.reg2 = registerAlias(src_reg, abi_size),
3790	}),
3791	.data = undefined,
3792	});
3793	// copy dst_reg back out	3779	// copy dst_reg back out
3794	return self.genSetStack(dst_ty, off, .{ .register = dst_reg }, .{});	3780	return self.genSetStack(dst_ty, off, .{ .register = dst_reg }, .{});
3795	},	3781	},
...	@@ -3915,20 +3901,12 @@ fn genVarDbgInfo(	...	@@ -3915,20 +3901,12 @@ fn genVarDbgInfo(
3915	}	3901	}
3916		3902
3917	fn airTrap(self: *Self) !void {	3903	fn airTrap(self: *Self) !void {
3918	_ = try self.addInst(.{	3904	try self.asmOpOnly(.ud2);
3919	.tag = .ud,
3920	.ops = Mir.Inst.Ops.encode(.{}),
3921	.data = undefined,
3922	});
3923	return self.finishAirBookkeeping();	3905	return self.finishAirBookkeeping();
3924	}	3906	}
3925		3907
3926	fn airBreakpoint(self: *Self) !void {	3908	fn airBreakpoint(self: *Self) !void {
3927	_ = try self.addInst(.{	3909	try self.asmOpOnly(.int3);
3928	.tag = .interrupt,
3929	.ops = Mir.Inst.Ops.encode(.{}),
3930	.data = undefined,
3931	});
3932	return self.finishAirBookkeeping();	3910	return self.finishAirBookkeeping();
3933	}	3911	}
3934		3912
...	@@ -4023,11 +4001,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier	...	@@ -4023,11 +4001,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
4023		4001
4024	if (info.stack_byte_count > 0) {	4002	if (info.stack_byte_count > 0) {
4025	// Adjust the stack	4003	// Adjust the stack
4026	_ = try self.addInst(.{	4004	try self.asmRegisterImmediate(.sub, .rsp, Immediate.u(info.stack_byte_count));
4027	.tag = .sub,
4028	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }),
4029	.data = .{ .imm = info.stack_byte_count },
4030	});
4031	}	4005	}
4032		4006
4033	// Due to incremental compilation, how function calls are generated depends	4007	// Due to incremental compilation, how function calls are generated depends
...	@@ -4040,12 +4014,11 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier	...	@@ -4040,12 +4014,11 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
4040	if (self.bin_file.cast(link.File.Elf)) \|elf_file\| {	4014	if (self.bin_file.cast(link.File.Elf)) \|elf_file\| {
4041	const atom_index = try elf_file.getOrCreateAtomForDecl(func.owner_decl);	4015	const atom_index = try elf_file.getOrCreateAtomForDecl(func.owner_decl);
4042	const atom = elf_file.getAtom(atom_index);	4016	const atom = elf_file.getAtom(atom_index);
4043	const got_addr = @intCast(u32, atom.getOffsetTableAddress(elf_file));	4017	const got_addr = atom.getOffsetTableAddress(elf_file);
4044	_ = try self.addInst(.{	4018	try self.asmMemory(.call, Memory.sib(.qword, .{
4045	.tag = .call,	4019	.base = .ds,
4046	.ops = Mir.Inst.Ops.encode(.{ .flags = 0b01 }),	4020	.disp = @intCast(i32, got_addr),
4047	.data = .{ .imm = got_addr },	4021	}));
4048	});
4049	} else if (self.bin_file.cast(link.File.Coff)) \|coff_file\| {	4022	} else if (self.bin_file.cast(link.File.Coff)) \|coff_file\| {
4050	const atom_index = try coff_file.getOrCreateAtomForDecl(func.owner_decl);	4023	const atom_index = try coff_file.getOrCreateAtomForDecl(func.owner_decl);
4051	const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;	4024	const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
...	@@ -4055,14 +4028,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier	...	@@ -4055,14 +4028,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
4055	.sym_index = sym_index,	4028	.sym_index = sym_index,
4056	},	4029	},
4057	});	4030	});
4058	_ = try self.addInst(.{	4031	try self.asmRegister(.call, .rax);
4059	.tag = .call,
4060	.ops = Mir.Inst.Ops.encode(.{
4061	.reg1 = .rax,
4062	.flags = 0b01,
4063	}),
4064	.data = undefined,
4065	});
4066	} else if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {	4032	} else if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {
4067	const atom_index = try macho_file.getOrCreateAtomForDecl(func.owner_decl);	4033	const atom_index = try macho_file.getOrCreateAtomForDecl(func.owner_decl);
4068	const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;	4034	const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
...	@@ -4072,14 +4038,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier	...	@@ -4072,14 +4038,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
4072	.sym_index = sym_index,	4038	.sym_index = sym_index,
4073	},	4039	},
4074	});	4040	});
4075	_ = try self.addInst(.{	4041	try self.asmRegister(.call, .rax);
4076	.tag = .call,
4077	.ops = Mir.Inst.Ops.encode(.{
4078	.reg1 = .rax,
4079	.flags = 0b01,
4080	}),
4081	.data = undefined,
4082	});
4083	} else if (self.bin_file.cast(link.File.Plan9)) \|p9\| {	4042	} else if (self.bin_file.cast(link.File.Plan9)) \|p9\| {
4084	const decl_block_index = try p9.seeDecl(func.owner_decl);	4043	const decl_block_index = try p9.seeDecl(func.owner_decl);
4085	const decl_block = p9.getDeclBlock(decl_block_index);	4044	const decl_block = p9.getDeclBlock(decl_block_index);
...	@@ -4088,11 +4047,10 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier	...	@@ -4088,11 +4047,10 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
4088	const got_addr = p9.bases.data;	4047	const got_addr = p9.bases.data;
4089	const got_index = decl_block.got_index.?;	4048	const got_index = decl_block.got_index.?;
4090	const fn_got_addr = got_addr + got_index * ptr_bytes;	4049	const fn_got_addr = got_addr + got_index * ptr_bytes;
4091	_ = try self.addInst(.{	4050	try self.asmMemory(.call, Memory.sib(.qword, .{
4092	.tag = .call,	4051	.base = .ds,
4093	.ops = Mir.Inst.Ops.encode(.{ .flags = 0b01 }),	4052	.disp = @intCast(i32, fn_got_addr),
4094	.data = .{ .imm = @intCast(u32, fn_got_addr) },	4053	}));
4095	});
4096	} else unreachable;	4054	} else unreachable;
4097	} else if (func_value.castTag(.extern_fn)) \|func_payload\| {	4055	} else if (func_value.castTag(.extern_fn)) \|func_payload\| {
4098	const extern_fn = func_payload.data;	4056	const extern_fn = func_payload.data;
...	@@ -4112,14 +4070,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier	...	@@ -4112,14 +4070,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
4112	.sym_index = sym_index,	4070	.sym_index = sym_index,
4113	},	4071	},
4114	});	4072	});
4115	_ = try self.addInst(.{	4073	try self.asmRegister(.call, .rax);
4116	.tag = .call,
4117	.ops = Mir.Inst.Ops.encode(.{
4118	.reg1 = .rax,
4119	.flags = 0b01,
4120	}),
4121	.data = undefined,
4122	});
4123	} else if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {	4074	} else if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {
4124	const sym_index = try macho_file.getGlobalSymbol(mem.sliceTo(decl_name, 0));	4075	const sym_index = try macho_file.getGlobalSymbol(mem.sliceTo(decl_name, 0));
4125	const atom = try macho_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl);	4076	const atom = try macho_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl);
...	@@ -4142,23 +4093,12 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier	...	@@ -4142,23 +4093,12 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
4142	assert(ty.zigTypeTag() == .Pointer);	4093	assert(ty.zigTypeTag() == .Pointer);
4143	const mcv = try self.resolveInst(callee);	4094	const mcv = try self.resolveInst(callee);
4144	try self.genSetReg(Type.initTag(.usize), .rax, mcv);	4095	try self.genSetReg(Type.initTag(.usize), .rax, mcv);
4145	_ = try self.addInst(.{	4096	try self.asmRegister(.call, .rax);
4146	.tag = .call,
4147	.ops = Mir.Inst.Ops.encode(.{
4148	.reg1 = .rax,
4149	.flags = 0b01,
4150	}),
4151	.data = undefined,
4152	});
4153	}	4097	}
4154		4098
4155	if (info.stack_byte_count > 0) {	4099	if (info.stack_byte_count > 0) {
4156	// Readjust the stack	4100	// Readjust the stack
4157	_ = try self.addInst(.{	4101	try self.asmRegisterImmediate(.add, .rsp, Immediate.u(info.stack_byte_count));
4158	.tag = .add,
4159	.ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }),
4160	.data = .{ .imm = info.stack_byte_count },
4161	});
4162	}	4102	}
4163		4103
4164	const result: MCValue = result: {	4104	const result: MCValue = result: {
...	@@ -4215,11 +4155,7 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4215,11 +4155,7 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {
4215	// TODO when implementing defer, this will need to jump to the appropriate defer expression.	4155	// TODO when implementing defer, this will need to jump to the appropriate defer expression.
4216	// TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction	4156	// TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
4217	// which is available if the jump is 127 bytes or less forward.	4157	// which is available if the jump is 127 bytes or less forward.
4218	const jmp_reloc = try self.addInst(.{	4158	const jmp_reloc = try self.asmJmpReloc(undefined);
4219	.tag = .jmp,
4220	.ops = Mir.Inst.Ops.encode(.{}),
4221	.data = .{ .inst = undefined },
4222	});
4223	try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);	4159	try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);
4224	return self.finishAir(inst, .dead, .{ un_op, .none, .none });	4160	return self.finishAir(inst, .dead, .{ un_op, .none, .none });
4225	}	4161	}
...	@@ -4251,11 +4187,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4251,11 +4187,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
4251	// TODO when implementing defer, this will need to jump to the appropriate defer expression.	4187	// TODO when implementing defer, this will need to jump to the appropriate defer expression.
4252	// TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction	4188	// TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
4253	// which is available if the jump is 127 bytes or less forward.	4189	// which is available if the jump is 127 bytes or less forward.
4254	const jmp_reloc = try self.addInst(.{	4190	const jmp_reloc = try self.asmJmpReloc(undefined);
4255	.tag = .jmp,
4256	.ops = Mir.Inst.Ops.encode(.{}),
4257	.data = .{ .inst = undefined },
4258	});
4259	try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);	4191	try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);
4260	return self.finishAir(inst, .dead, .{ un_op, .none, .none });	4192	return self.finishAir(inst, .dead, .{ un_op, .none, .none });
4261	}	4193	}
...	@@ -4430,33 +4362,13 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {	...	@@ -4430,33 +4362,13 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {
4430	const abi_size = ty.abiSize(self.target.*);	4362	const abi_size = ty.abiSize(self.target.*);
4431	switch (mcv) {	4363	switch (mcv) {
4432	.eflags => \|cc\| {	4364	.eflags => \|cc\| {
4433	return self.addInst(.{	4365	// Here we map the opposites since the jump is to the false branch.
4434	.tag = .cond_jmp,	4366	return self.asmJccReloc(undefined, cc.negate());
4435	.ops = Mir.Inst.Ops.encode(.{}),
4436	.data = .{
4437	.inst_cc = .{
4438	.inst = undefined,
4439	// Here we map the opposites since the jump is to the false branch.
4440	.cc = cc.negate(),
4441	},
4442	},
4443	});
4444	},	4367	},
4445	.register => \|reg\| {	4368	.register => \|reg\| {
4446	try self.spillEflagsIfOccupied();	4369	try self.spillEflagsIfOccupied();
4447	_ = try self.addInst(.{	4370	try self.asmRegisterImmediate(.@"test", reg, Immediate.u(1));
4448	.tag = .@"test",	4371	return self.asmJccReloc(undefined, .e);
4449	.ops = Mir.Inst.Ops.encode(.{ .reg1 = reg }),
4450	.data = .{ .imm = 1 },
4451	});
4452	return self.addInst(.{
4453	.tag = .cond_jmp,
4454	.ops = Mir.Inst.Ops.encode(.{}),
4455	.data = .{ .inst_cc = .{
4456	.inst = undefined,
4457	.cc = .e,
4458	} },
4459	});
4460	},	4372	},
4461	.immediate,	4373	.immediate,
4462	.stack_offset,	4374	.stack_offset,
...	@@ -4470,6 +4382,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {	...	@@ -4470,6 +4382,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {
4470	},	4382	},
4471	else => return self.fail("TODO implement condbr when condition is {s}", .{@tagName(mcv)}),	4383	else => return self.fail("TODO implement condbr when condition is {s}", .{@tagName(mcv)}),
4472	}	4384	}
		4385	return 0; // TODO
4473	}	4386	}
4474		4387
4475	fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {	4388	fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
...	@@ -4795,11 +4708,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4795,11 +4708,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
4795	const body = self.air.extra[loop.end..][0..loop.data.body_len];	4708	const body = self.air.extra[loop.end..][0..loop.data.body_len];
4796	const jmp_target = @intCast(u32, self.mir_instructions.len);	4709	const jmp_target = @intCast(u32, self.mir_instructions.len);
4797	try self.genBody(body);	4710	try self.genBody(body);
4798	_ = try self.addInst(.{	4711	_ = try self.asmJmpReloc(jmp_target);
4799	.tag = .jmp,
4800	.ops = Mir.Inst.Ops.encode(.{}),
4801	.data = .{ .inst = jmp_target },
4802	});
4803	return self.finishAirBookkeeping();	4712	return self.finishAirBookkeeping();
4804	}	4713	}
4805		4714
...	@@ -4844,23 +4753,16 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u	...	@@ -4844,23 +4753,16 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u
4844	.none => unreachable,	4753	.none => unreachable,
4845	.undef => unreachable,	4754	.undef => unreachable,
4846	.dead, .unreach => unreachable,	4755	.dead, .unreach => unreachable,
4847	.immediate => \|imm\| {	4756	.immediate => \|imm\| try self.asmRegisterImmediate(
4848	_ = try self.addInst(.{	4757	.xor,
4849	.tag = .xor,	4758	registerAlias(cond_reg, abi_size),
4850	.ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(cond_reg, abi_size) }),	4759	Immediate.u(imm),
4851	.data = .{ .imm = @intCast(u32, imm) },	4760	),
4852	});	4761	.register => \|reg\| try self.asmRegisterRegister(
4853	},	4762	.xor,
4854	.register => \|reg\| {	4763	registerAlias(cond_reg, abi_size),
4855	_ = try self.addInst(.{	4764	registerAlias(reg, abi_size),
4856	.tag = .xor,	4765	),
4857	.ops = Mir.Inst.Ops.encode(.{
4858	.reg1 = registerAlias(cond_reg, abi_size),
4859	.reg2 = registerAlias(reg, abi_size),
4860	}),
4861	.data = undefined,
4862	});
4863	},
4864	.stack_offset => {	4766	.stack_offset => {
4865	if (abi_size <= 8) {	4767	if (abi_size <= 8) {
4866	const reg = try self.copyToTmpRegister(ty, case);	4768	const reg = try self.copyToTmpRegister(ty, case);
...	@@ -4874,22 +4776,9 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u	...	@@ -4874,22 +4776,9 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u
4874	},	4776	},
4875	}	4777	}
4876		4778
4877	_ = try self.addInst(.{	4779	const aliased_reg = registerAlias(cond_reg, abi_size);
4878	.tag = .@"test",	4780	try self.asmRegisterRegister(.@"test", aliased_reg, aliased_reg);
4879	.ops = Mir.Inst.Ops.encode(.{	4781	return self.asmJccReloc(undefined, .ne);
4880	.reg1 = registerAlias(cond_reg, abi_size),
4881	.reg2 = registerAlias(cond_reg, abi_size),
4882	}),
4883	.data = undefined,
4884	});
4885	return self.addInst(.{
4886	.tag = .cond_jmp,
4887	.ops = Mir.Inst.Ops.encode(.{}),
4888	.data = .{ .inst_cc = .{
4889	.inst = undefined,
4890	.cc = .ne,
4891	} },
4892	});
4893	},	4782	},
4894	.stack_offset => {	4783	.stack_offset => {
4895	try self.spillEflagsIfOccupied();	4784	try self.spillEflagsIfOccupied();
...	@@ -4907,6 +4796,7 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u	...	@@ -4907,6 +4796,7 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u
4907	return self.fail("TODO implemenent switch mir when condition is {}", .{condition});	4796	return self.fail("TODO implemenent switch mir when condition is {}", .{condition});
4908	},	4797	},
4909	}	4798	}
		4799	return 0; // TODO
4910	}	4800	}
4911		4801
4912	fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {	4802	fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
...	@@ -5101,10 +4991,10 @@ fn canonicaliseBranches(self: Self, parent_branch: Branch, canon_branch: *Bran	...	@@ -5101,10 +4991,10 @@ fn canonicaliseBranches(self: Self, parent_branch: Branch, canon_branch: *Bran
5101	fn performReloc(self: *Self, reloc: Mir.Inst.Index) !void {	4991	fn performReloc(self: *Self, reloc: Mir.Inst.Index) !void {
5102	const next_inst = @intCast(u32, self.mir_instructions.len);	4992	const next_inst = @intCast(u32, self.mir_instructions.len);
5103	switch (self.mir_instructions.items(.tag)[reloc]) {	4993	switch (self.mir_instructions.items(.tag)[reloc]) {
5104	.cond_jmp => {	4994	.jcc => {
5105	self.mir_instructions.items(.data)[reloc].inst_cc.inst = next_inst;	4995	self.mir_instructions.items(.data)[reloc].inst_cc.inst = next_inst;
5106	},	4996	},
5107	.jmp => {	4997	.jmp_reloc => {
5108	self.mir_instructions.items(.data)[reloc].inst = next_inst;	4998	self.mir_instructions.items(.data)[reloc].inst = next_inst;
5109	},	4999	},
5110	else => unreachable,	5000	else => unreachable,
...	@@ -5146,11 +5036,7 @@ fn brVoid(self: *Self, block: Air.Inst.Index) !void {	...	@@ -5146,11 +5036,7 @@ fn brVoid(self: *Self, block: Air.Inst.Index) !void {
5146	// Emit a jump with a relocation. It will be patched up after the block ends.	5036	// Emit a jump with a relocation. It will be patched up after the block ends.
5147	try block_data.relocs.ensureUnusedCapacity(self.gpa, 1);	5037	try block_data.relocs.ensureUnusedCapacity(self.gpa, 1);
5148	// Leave the jump offset undefined	5038	// Leave the jump offset undefined
5149	const jmp_reloc = try self.addInst(.{	5039	const jmp_reloc = try self.asmJmpReloc(undefined);
5150	.tag = .jmp,
5151	.ops = Mir.Inst.Ops.encode(.{}),
5152	.data = .{ .inst = undefined },
5153	});
5154	block_data.relocs.appendAssumeCapacity(jmp_reloc);	5040	block_data.relocs.appendAssumeCapacity(jmp_reloc);
5155	}	5041	}
5156		5042
...	@@ -5223,31 +5109,19 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5223,31 +5109,19 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
5223	var iter = std.mem.tokenize(u8, asm_source, "\n\r");	5109	var iter = std.mem.tokenize(u8, asm_source, "\n\r");
5224	while (iter.next()) \|ins\| {	5110	while (iter.next()) \|ins\| {
5225	if (mem.eql(u8, ins, "syscall")) {	5111	if (mem.eql(u8, ins, "syscall")) {
5226	_ = try self.addInst(.{	5112	try self.asmOpOnly(.syscall);
5227	.tag = .syscall,
5228	.ops = undefined,
5229	.data = undefined,
5230	});
5231	} else if (mem.indexOf(u8, ins, "push")) \|_\| {	5113	} else if (mem.indexOf(u8, ins, "push")) \|_\| {
5232	const arg = ins[4..];	5114	const arg = ins[4..];
5233	if (mem.indexOf(u8, arg, "$")) \|l\| {	5115	if (mem.indexOf(u8, arg, "$")) \|l\| {
5234	const n = std.fmt.parseInt(u8, ins[4 + l + 1 ..], 10) catch {	5116	const n = std.fmt.parseInt(u8, ins[4 + l + 1 ..], 10) catch {
5235	return self.fail("TODO implement more inline asm int parsing", .{});	5117	return self.fail("TODO implement more inline asm int parsing", .{});
5236	};	5118	};
5237	_ = try self.addInst(.{	5119	try self.asmImmediate(.push, Immediate.u(n));
5238	.tag = .push,
5239	.ops = Mir.Inst.Ops.encode(.{ .flags = 0b10 }),
5240	.data = .{ .imm = n },
5241	});
5242	} else if (mem.indexOf(u8, arg, "%%")) \|l\| {	5120	} else if (mem.indexOf(u8, arg, "%%")) \|l\| {
5243	const reg_name = ins[4 + l + 2 ..];	5121	const reg_name = ins[4 + l + 2 ..];
5244	const reg = parseRegName(reg_name) orelse	5122	const reg = parseRegName(reg_name) orelse
5245	return self.fail("unrecognized register: '{s}'", .{reg_name});	5123	return self.fail("unrecognized register: '{s}'", .{reg_name});
5246	_ = try self.addInst(.{	5124	try self.asmRegister(.push, reg);
5247	.tag = .push,
5248	.ops = Mir.Inst.Ops.encode(.{ .reg1 = reg }),
5249	.data = undefined,
5250	});
5251	} else return self.fail("TODO more push operands", .{});	5125	} else return self.fail("TODO more push operands", .{});
5252	} else if (mem.indexOf(u8, ins, "pop")) \|_\| {	5126	} else if (mem.indexOf(u8, ins, "pop")) \|_\| {
5253	const arg = ins[3..];	5127	const arg = ins[3..];
...	@@ -5255,11 +5129,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -5255,11 +5129,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
5255	const reg_name = ins[3 + l + 2 ..];	5129	const reg_name = ins[3 + l + 2 ..];
5256	const reg = parseRegName(reg_name) orelse	5130	const reg = parseRegName(reg_name) orelse
5257	return self.fail("unrecognized register: '{s}'", .{reg_name});	5131	return self.fail("unrecognized register: '{s}'", .{reg_name});
5258	_ = try self.addInst(.{	5132	try self.asmRegister(.pop, reg);
5259	.tag = .pop,
5260	.ops = Mir.Inst.Ops.encode(.{ .reg1 = reg }),
5261	.data = undefined,
5262	});
5263	} else return self.fail("TODO more pop operands", .{});	5133	} else return self.fail("TODO more pop operands", .{});
5264	} else {	5134	} else {
5265	return self.fail("TODO implement support for more x86 assembly instructions", .{});	5135	return self.fail("TODO implement support for more x86 assembly instructions", .{});
...	@@ -5332,7 +5202,7 @@ fn setRegOrMem(self: *Self, ty: Type, loc: MCValue, val: MCValue) !void {	...	@@ -5332,7 +5202,7 @@ fn setRegOrMem(self: *Self, ty: Type, loc: MCValue, val: MCValue) !void {
5332	}	5202	}
5333		5203
5334	fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerError!void {	5204	fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerError!void {
5335	const abi_size = ty.abiSize(self.target.*);	5205	const abi_size = @intCast(u32, ty.abiSize(self.target.*));
5336	switch (mcv) {	5206	switch (mcv) {
5337	.dead => unreachable,	5207	.dead => unreachable,
5338	.unreach, .none => return,	5208	.unreach, .none => return,
...	@@ -5356,26 +5226,17 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE	...	@@ -5356,26 +5226,17 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
5356	.immediate => \|imm\| {	5226	.immediate => \|imm\| {
5357	switch (abi_size) {	5227	switch (abi_size) {
5358	1, 2, 4 => {	5228	1, 2, 4 => {
		5229	// TODO
5359	// We have a positive stack offset value but we want a twos complement negative	5230	// We have a positive stack offset value but we want a twos complement negative
5360	// offset from rbp, which is at the top of the stack frame.	5231	// offset from rbp, which is at the top of the stack frame.
5361	// mov [rbp+offset], immediate	5232	const immediate = if (ty.isSignedInt())
5362	const payload = try self.addExtra(Mir.ImmPair{	5233	Immediate.s(@intCast(i32, @bitCast(i64, imm)))
5363	.dest_off = @bitCast(u32, -stack_offset),	5234	else
5364	.operand = @truncate(u32, imm),	5235	Immediate.u(@intCast(u32, imm));
5365	});	5236	try self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
5366	_ = try self.addInst(.{	5237	.base = .rsp,
5367	.tag = .mov_mem_imm,	5238	.disp = -stack_offset,
5368	.ops = Mir.Inst.Ops.encode(.{	5239	}), immediate);
5369	.reg1 = .rsp,
5370	.flags = switch (abi_size) {
5371	1 => 0b00,
5372	2 => 0b01,
5373	4 => 0b10,
5374	else => unreachable,
5375	},
5376	}),
5377	.data = .{ .payload = payload },
5378	});
5379	},	5240	},
5380	8 => {	5241	8 => {
5381	const reg = try self.copyToTmpRegister(ty, mcv);	5242	const reg = try self.copyToTmpRegister(ty, mcv);
...	@@ -5400,44 +5261,32 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE	...	@@ -5400,44 +5261,32 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
5400	.Float => {	5261	.Float => {
5401	if (intrinsicsAllowed(self.target.*, ty)) {	5262	if (intrinsicsAllowed(self.target.*, ty)) {
5402	const tag: Mir.Inst.Tag = switch (ty.tag()) {	5263	const tag: Mir.Inst.Tag = switch (ty.tag()) {
5403	.f32 => if (hasAvxSupport(self.target.*))	5264	.f32 => .movss,
5404	Mir.Inst.Tag.mov_f32_avx	5265	.f64 => .movsd,
5405	else	5266	else => return self.fail(
5406	Mir.Inst.Tag.mov_f32_sse,	5267	"TODO genSetStackArg for register for type {}",
5407	.f64 => if (hasAvxSupport(self.target.*))	5268	.{ty.fmtDebug()},
5408	Mir.Inst.Tag.mov_f64_avx	5269	),
5409	else
5410	Mir.Inst.Tag.mov_f64_sse,
5411	else => return self.fail("TODO genSetStackArg for register for type {}", .{ty.fmtDebug()}),
5412	};	5270	};
5413	_ = try self.addInst(.{	5271	// TODO verify this
5414	.tag = tag,	5272	const ptr_size: Memory.PtrSize = switch (ty.tag()) {
5415	.ops = Mir.Inst.Ops.encode(.{	5273	.f32 => .dword,
5416	.reg1 = switch (ty.tag()) {	5274	.f64 => .qword,
5417	.f32 => .esp,	5275	else => unreachable,
5418	.f64 => .rsp,	5276	};
5419	else => unreachable,	5277	return self.asmMemoryRegister(tag, Memory.sib(ptr_size, .{
5420	},	5278	.base = .rsp,
5421	.reg2 = reg.to128(),	5279	.disp = -stack_offset,
5422	.flags = 0b01,	5280	}), reg.to128());
5423	}),
5424	.data = .{ .imm = @bitCast(u32, -stack_offset) },
5425	});
5426	return;
5427	}	5281	}
5428		5282
5429	return self.fail("TODO genSetStackArg for register with no intrinsics", .{});	5283	return self.fail("TODO genSetStackArg for register with no intrinsics", .{});
5430	},	5284	},
5431	else => {	5285	else => {
5432	_ = try self.addInst(.{	5286	try self.asmMemoryRegister(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
5433	.tag = .mov,	5287	.base = .rsp,
5434	.ops = Mir.Inst.Ops.encode(.{	5288	.disp = -stack_offset,
5435	.reg1 = .rsp,	5289	}), registerAlias(reg, abi_size));
5436	.reg2 = registerAlias(reg, @intCast(u32, abi_size)),
5437	.flags = 0b10,
5438	}),
5439	.data = .{ .imm = @bitCast(u32, -stack_offset) },
5440	});
5441	},	5290	},
5442	}	5291	}
5443	},	5292	},
...	@@ -5460,7 +5309,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE	...	@@ -5460,7 +5309,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
5460	}	5309	}
5461		5310
5462	fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: InlineMemcpyOpts) InnerError!void {	5311	fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: InlineMemcpyOpts) InnerError!void {
5463	const abi_size = ty.abiSize(self.target.*);	5312	const abi_size = @intCast(u32, ty.abiSize(self.target.*));
5464	switch (mcv) {	5313	switch (mcv) {
5465	.dead => unreachable,	5314	.dead => unreachable,
5466	.unreach, .none => return, // Nothing to do.	5315	.unreach, .none => return, // Nothing to do.
...	@@ -5468,10 +5317,19 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl	...	@@ -5468,10 +5317,19 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5468	if (!self.wantSafety())	5317	if (!self.wantSafety())
5469	return; // The already existing value will do just fine.	5318	return; // The already existing value will do just fine.
5470	// TODO Upgrade this to a memset call when we have that available.	5319	// TODO Upgrade this to a memset call when we have that available.
5471	switch (ty.abiSize(self.target.*)) {	5320	switch (abi_size) {
5472	1 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaa }, opts),	5321	1, 2, 4 => {
5473	2 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaa }, opts),	5322	const value: u64 = switch (abi_size) {
5474	4 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaa }, opts),	5323	1 => 0xaa,
		5324	2 => 0xaaaa,
		5325	4 => 0xaaaaaaaa,
		5326	else => unreachable,
		5327	};
		5328	return self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
		5329	.base = opts.dest_stack_base orelse .rbp,
		5330	.disp = -stack_offset,
		5331	}), Immediate.u(value));
		5332	},
5475	8 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaaaaaaaaaa }, opts),	5333	8 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaaaaaaaaaa }, opts),
5476	else => \|x\| return self.genInlineMemset(	5334	else => \|x\| return self.genInlineMemset(
5477	.{ .stack_offset = stack_offset },	5335	.{ .stack_offset = stack_offset },
...	@@ -5491,13 +5349,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl	...	@@ -5491,13 +5349,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5491	const overflow_bit_ty = ty.structFieldType(1);	5349	const overflow_bit_ty = ty.structFieldType(1);
5492	const overflow_bit_offset = ty.structFieldOffset(1, self.target.*);	5350	const overflow_bit_offset = ty.structFieldOffset(1, self.target.*);
5493	const tmp_reg = try self.register_manager.allocReg(null, gp);	5351	const tmp_reg = try self.register_manager.allocReg(null, gp);
5494	_ = try self.addInst(.{	5352	try self.asmSetccRegister(tmp_reg.to8(), ro.eflags);
5495	.tag = .cond_set_byte,
5496	.ops = Mir.Inst.Ops.encode(.{
5497	.reg1 = tmp_reg.to8(),
5498	}),
5499	.data = .{ .cc = ro.eflags },
5500	});
5501		5353
5502	return self.genSetStack(	5354	return self.genSetStack(
5503	overflow_bit_ty,	5355	overflow_bit_ty,
...	@@ -5512,72 +5364,36 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl	...	@@ -5512,72 +5364,36 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5512	},	5364	},
5513	.immediate => \|x_big\| {	5365	.immediate => \|x_big\| {
5514	const base_reg = opts.dest_stack_base orelse .rbp;	5366	const base_reg = opts.dest_stack_base orelse .rbp;
		5367	// TODO
5515	switch (abi_size) {	5368	switch (abi_size) {
5516	0 => {	5369	0 => {
5517	assert(ty.isError());	5370	assert(ty.isError());
5518	const payload = try self.addExtra(Mir.ImmPair{	5371	try self.asmMemoryImmediate(.mov, Memory.sib(.byte, .{
5519	.dest_off = @bitCast(u32, -stack_offset),	5372	.base = base_reg,
5520	.operand = @truncate(u32, x_big),	5373	.disp = -stack_offset,
5521	});	5374	}), Immediate.u(@truncate(u8, x_big)));
5522	_ = try self.addInst(.{
5523	.tag = .mov_mem_imm,
5524	.ops = Mir.Inst.Ops.encode(.{
5525	.reg1 = base_reg,
5526	.flags = 0b00,
5527	}),
5528	.data = .{ .payload = payload },
5529	});
5530	},	5375	},
5531	1, 2, 4 => {	5376	1, 2, 4 => {
5532	const payload = try self.addExtra(Mir.ImmPair{	5377	const immediate = if (ty.isSignedInt())
5533	.dest_off = @bitCast(u32, -stack_offset),	5378	Immediate.s(@truncate(i32, @bitCast(i64, x_big)))
5534	.operand = @truncate(u32, x_big),	5379	else
5535	});	5380	Immediate.u(@intCast(u32, x_big));
5536	_ = try self.addInst(.{	5381	try self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
5537	.tag = .mov_mem_imm,	5382	.base = base_reg,
5538	.ops = Mir.Inst.Ops.encode(.{	5383	.disp = -stack_offset,
5539	.reg1 = base_reg,	5384	}), immediate);
5540	.flags = switch (abi_size) {
5541	1 => 0b00,
5542	2 => 0b01,
5543	4 => 0b10,
5544	else => unreachable,
5545	},
5546	}),
5547	.data = .{ .payload = payload },
5548	});
5549	},	5385	},
5550	8 => {	5386	8 => {
5551	// 64 bit write to memory would take two mov's anyways so we	5387	// 64 bit write to memory would take two mov's anyways so we
5552	// insted just use two 32 bit writes to avoid register allocation	5388	// insted just use two 32 bit writes to avoid register allocation
5553	{	5389	try self.asmMemoryImmediate(.mov, Memory.sib(.dword, .{
5554	const payload = try self.addExtra(Mir.ImmPair{	5390	.base = base_reg,
5555	.dest_off = @bitCast(u32, -stack_offset + 4),	5391	.disp = -stack_offset + 4,
5556	.operand = @truncate(u32, x_big >> 32),	5392	}), Immediate.u(@truncate(u32, x_big >> 32)));
5557	});	5393	try self.asmMemoryImmediate(.mov, Memory.sib(.dword, .{
5558	_ = try self.addInst(.{	5394	.base = base_reg,
5559	.tag = .mov_mem_imm,	5395	.disp = -stack_offset,
5560	.ops = Mir.Inst.Ops.encode(.{	5396	}), Immediate.u(@truncate(u32, x_big)));
5561	.reg1 = base_reg,
5562	.flags = 0b10,
5563	}),
5564	.data = .{ .payload = payload },
5565	});
5566	}
5567	{
5568	const payload = try self.addExtra(Mir.ImmPair{
5569	.dest_off = @bitCast(u32, -stack_offset),
5570	.operand = @truncate(u32, x_big),
5571	});
5572	_ = try self.addInst(.{
5573	.tag = .mov_mem_imm,
5574	.ops = Mir.Inst.Ops.encode(.{
5575	.reg1 = base_reg,
5576	.flags = 0b10,
5577	}),
5578	.data = .{ .payload = payload },
5579	});
5580	}
5581	},	5397	},
5582	else => {	5398	else => {
5583	return self.fail("TODO implement set abi_size=large stack variable with immediate", .{});	5399	return self.fail("TODO implement set abi_size=large stack variable with immediate", .{});
...	@@ -5595,30 +5411,22 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl	...	@@ -5595,30 +5411,22 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5595	.Float => {	5411	.Float => {
5596	if (intrinsicsAllowed(self.target.*, ty)) {	5412	if (intrinsicsAllowed(self.target.*, ty)) {
5597	const tag: Mir.Inst.Tag = switch (ty.tag()) {	5413	const tag: Mir.Inst.Tag = switch (ty.tag()) {
5598	.f32 => if (hasAvxSupport(self.target.*))	5414	.f32 => .movss,
5599	Mir.Inst.Tag.mov_f32_avx	5415	.f64 => .movsd,
5600	else	5416	else => return self.fail(
5601	Mir.Inst.Tag.mov_f32_sse,	5417	"TODO genSetStack for register for type {}",
5602	.f64 => if (hasAvxSupport(self.target.*))	5418	.{ty.fmtDebug()},
5603	Mir.Inst.Tag.mov_f64_avx	5419	),
5604	else
5605	Mir.Inst.Tag.mov_f64_sse,
5606	else => return self.fail("TODO genSetStack for register for type {}", .{ty.fmtDebug()}),
5607	};	5420	};
5608	_ = try self.addInst(.{	5421	const ptr_size: Memory.PtrSize = switch (ty.tag()) {
5609	.tag = tag,	5422	.f32 => .dword,
5610	.ops = Mir.Inst.Ops.encode(.{	5423	.f64 => .qword,
5611	.reg1 = switch (ty.tag()) {	5424	else => unreachable,
5612	.f32 => base_reg.to32(),	5425	};
5613	.f64 => base_reg.to64(),	5426	return self.asmMemoryRegister(tag, Memory.sib(ptr_size, .{
5614	else => unreachable,	5427	.base = base_reg.to64(),
5615	},	5428	.disp = -stack_offset,
5616	.reg2 = reg.to128(),	5429	}), reg.to128());
5617	.flags = 0b01,
5618	}),
5619	.data = .{ .imm = @bitCast(u32, -stack_offset) },
5620	});
5621	return;
5622	}	5430	}
5623		5431
5624	return self.fail("TODO genSetStack for register for type float with no intrinsics", .{});	5432	return self.fail("TODO genSetStack for register for type float with no intrinsics", .{});
...	@@ -5666,7 +5474,7 @@ fn genInlineMemcpyRegisterRegister(	...	@@ -5666,7 +5474,7 @@ fn genInlineMemcpyRegisterRegister(
5666	src_reg: Register,	5474	src_reg: Register,
5667	offset: i32,	5475	offset: i32,
5668	) InnerError!void {	5476	) InnerError!void {
5669	assert(dst_reg.size() == 64);	5477	assert(dst_reg.bitSize() == 64);
5670		5478
5671	const dst_reg_lock = self.register_manager.lockReg(dst_reg);	5479	const dst_reg_lock = self.register_manager.lockReg(dst_reg);
5672	defer if (dst_reg_lock) \|lock\| self.register_manager.unlockReg(lock);	5480	defer if (dst_reg_lock) \|lock\| self.register_manager.unlockReg(lock);
...	@@ -5683,16 +5491,10 @@ fn genInlineMemcpyRegisterRegister(	...	@@ -5683,16 +5491,10 @@ fn genInlineMemcpyRegisterRegister(
5683	var remainder = abi_size;	5491	var remainder = abi_size;
5684	while (remainder > 0) {	5492	while (remainder > 0) {
5685	const nearest_power_of_two = @as(u6, 1) << math.log2_int(u3, @intCast(u3, remainder));	5493	const nearest_power_of_two = @as(u6, 1) << math.log2_int(u3, @intCast(u3, remainder));
5686		5494	try self.asmMemoryRegister(.mov, Memory.sib(Memory.PtrSize.fromSize(nearest_power_of_two), .{
5687	_ = try self.addInst(.{	5495	.base = dst_reg,
5688	.tag = .mov,	5496	.disp = -next_offset,
5689	.ops = Mir.Inst.Ops.encode(.{	5497	}), registerAlias(tmp_reg, nearest_power_of_two));
5690	.reg1 = dst_reg,
5691	.reg2 = registerAlias(tmp_reg, nearest_power_of_two),
5692	.flags = 0b10,
5693	}),
5694	.data = .{ .imm = @bitCast(u32, -next_offset) },
5695	});
5696		5498
5697	if (nearest_power_of_two > 1) {	5499	if (nearest_power_of_two > 1) {
5698	try self.genShiftBinOpMir(.shr, ty, tmp_reg, .{	5500	try self.genShiftBinOpMir(.shr, ty, tmp_reg, .{
...	@@ -5704,15 +5506,10 @@ fn genInlineMemcpyRegisterRegister(	...	@@ -5704,15 +5506,10 @@ fn genInlineMemcpyRegisterRegister(
5704	next_offset -= nearest_power_of_two;	5506	next_offset -= nearest_power_of_two;
5705	}	5507	}
5706	} else {	5508	} else {
5707	_ = try self.addInst(.{	5509	try self.asmMemoryRegister(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
5708	.tag = .mov,	5510	.base = dst_reg,
5709	.ops = Mir.Inst.Ops.encode(.{	5511	.disp = -offset,
5710	.reg1 = dst_reg,	5512	}), registerAlias(src_reg, abi_size));
5711	.reg2 = registerAlias(src_reg, @intCast(u32, abi_size)),
5712	.flags = 0b10,
5713	}),
5714	.data = .{ .imm = @bitCast(u32, -offset) },
5715	});
5716	}	5513	}
5717	}	5514	}
5718		5515
...	@@ -5752,24 +5549,17 @@ fn genInlineMemcpy(	...	@@ -5752,24 +5549,17 @@ fn genInlineMemcpy(
5752	try self.loadMemPtrIntoRegister(dst_addr_reg, Type.usize, dst_ptr);	5549	try self.loadMemPtrIntoRegister(dst_addr_reg, Type.usize, dst_ptr);
5753	},	5550	},
5754	.ptr_stack_offset, .stack_offset => \|off\| {	5551	.ptr_stack_offset, .stack_offset => \|off\| {
5755	_ = try self.addInst(.{	5552	try self.asmRegisterMemory(.lea, dst_addr_reg.to64(), Memory.sib(.qword, .{
5756	.tag = .lea,	5553	.base = opts.dest_stack_base orelse .rbp,
5757	.ops = Mir.Inst.Ops.encode(.{	5554	.disp = -off,
5758	.reg1 = dst_addr_reg.to64(),	5555	}));
5759	.reg2 = opts.dest_stack_base orelse .rbp,
5760	}),
5761	.data = .{ .imm = @bitCast(u32, -off) },
5762	});
5763	},	5556	},
5764	.register => \|reg\| {	5557	.register => \|reg\| {
5765	_ = try self.addInst(.{	5558	try self.asmRegisterRegister(
5766	.tag = .mov,	5559	.mov,
5767	.ops = Mir.Inst.Ops.encode(.{	5560	registerAlias(dst_addr_reg, @intCast(u32, @divExact(reg.bitSize(), 8))),
5768	.reg1 = registerAlias(dst_addr_reg, @divExact(reg.size(), 8)),	5561	reg,
5769	.reg2 = reg,	5562	);
5770	}),
5771	.data = undefined,
5772	});
5773	},	5563	},
5774	else => {	5564	else => {
5775	return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr});	5565	return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr});
...	@@ -5781,24 +5571,17 @@ fn genInlineMemcpy(	...	@@ -5781,24 +5571,17 @@ fn genInlineMemcpy(
5781	try self.loadMemPtrIntoRegister(src_addr_reg, Type.usize, src_ptr);	5571	try self.loadMemPtrIntoRegister(src_addr_reg, Type.usize, src_ptr);
5782	},	5572	},
5783	.ptr_stack_offset, .stack_offset => \|off\| {	5573	.ptr_stack_offset, .stack_offset => \|off\| {
5784	_ = try self.addInst(.{	5574	try self.asmRegisterMemory(.lea, src_addr_reg.to64(), Memory.sib(.qword, .{
5785	.tag = .lea,	5575	.base = opts.source_stack_base orelse .rbp,
5786	.ops = Mir.Inst.Ops.encode(.{	5576	.disp = -off,
5787	.reg1 = src_addr_reg.to64(),	5577	}));
5788	.reg2 = opts.source_stack_base orelse .rbp,
5789	}),
5790	.data = .{ .imm = @bitCast(u32, -off) },
5791	});
5792	},	5578	},
5793	.register => \|reg\| {	5579	.register => \|reg\| {
5794	_ = try self.addInst(.{	5580	try self.asmRegisterRegister(
5795	.tag = .mov,	5581	.mov,
5796	.ops = Mir.Inst.Ops.encode(.{	5582	registerAlias(src_addr_reg, @intCast(u32, @divExact(reg.bitSize(), 8))),
5797	.reg1 = registerAlias(src_addr_reg, @divExact(reg.size(), 8)),	5583	reg,
5798	.reg2 = reg,	5584	);
5799	}),
5800	.data = undefined,
5801	});
5802	},	5585	},
5803	else => {	5586	else => {
5804	return self.fail("TODO implement memcpy for setting stack when src is {}", .{src_ptr});	5587	return self.fail("TODO implement memcpy for setting stack when src is {}", .{src_ptr});
...	@@ -5806,74 +5589,35 @@ fn genInlineMemcpy(	...	@@ -5806,74 +5589,35 @@ fn genInlineMemcpy(
5806	}	5589	}
5807		5590
5808	try self.genSetReg(Type.usize, count_reg, len);	5591	try self.genSetReg(Type.usize, count_reg, len);
5809		5592	try self.asmRegisterImmediate(.mov, index_reg, Immediate.u(0));
5810	// mov index_reg, 0
5811	_ = try self.addInst(.{
5812	.tag = .mov,
5813	.ops = Mir.Inst.Ops.encode(.{ .reg1 = index_reg }),
5814	.data = .{ .imm = 0 },
5815	});
5816
5817	// loop:
5818	// cmp count, 0
5819	const loop_start = try self.addInst(.{	5593	const loop_start = try self.addInst(.{
5820	.tag = .cmp,	5594	.tag = .cmp,
5821	.ops = Mir.Inst.Ops.encode(.{ .reg1 = count_reg }),	5595	.ops = .ri_u,
5822	.data = .{ .imm = 0 },	5596	.data = .{ .ri = .{
5823	});	5597	.r1 = count_reg,
5824		5598	.imm = 0,
5825	// je end
5826	const loop_reloc = try self.addInst(.{
5827	.tag = .cond_jmp,
5828	.ops = Mir.Inst.Ops.encode(.{}),
5829	.data = .{ .inst_cc = .{
5830	.inst = undefined,
5831	.cc = .e,
5832	} },	5599	} },
5833	});	5600	});
5834		5601	const loop_reloc = try self.asmJccReloc(undefined, .e);
5835	// mov tmp, [addr + index_reg]	5602	try self.asmRegisterMemory(.mov, tmp_reg.to8(), Memory.sib(.byte, .{
5836	_ = try self.addInst(.{	5603	.base = src_addr_reg,
5837	.tag = .mov_scale_src,	5604	.scale_index = .{
5838	.ops = Mir.Inst.Ops.encode(.{	5605	.scale = 1,
5839	.reg1 = tmp_reg.to8(),	5606	.index = index_reg,
5840	.reg2 = src_addr_reg,	5607	},
5841	}),	5608	.disp = 0,
5842	.data = .{ .payload = try self.addExtra(Mir.IndexRegisterDisp.encode(index_reg, 0)) },	5609	}));
5843	});	5610	try self.asmMemoryRegister(.mov, Memory.sib(.byte, .{
5844		5611	.base = dst_addr_reg,
5845	// mov [stack_offset + index_reg], tmp	5612	.scale_index = .{
5846	_ = try self.addInst(.{	5613	.scale = 1,
5847	.tag = .mov_scale_dst,	5614	.index = index_reg,
5848	.ops = Mir.Inst.Ops.encode(.{	5615	},
5849	.reg1 = dst_addr_reg,	5616	.disp = 0,
5850	.reg2 = tmp_reg.to8(),	5617	}), tmp_reg.to8());
5851	}),	5618	try self.asmRegisterImmediate(.add, index_reg, Immediate.u(1));
5852	.data = .{ .payload = try self.addExtra(Mir.IndexRegisterDisp.encode(index_reg, 0)) },	5619	try self.asmRegisterImmediate(.sub, count_reg, Immediate.u(1));
5853	});	5620	_ = try self.asmJmpReloc(loop_start);
5854
5855	// add index_reg, 1
5856	_ = try self.addInst(.{
5857	.tag = .add,
5858	.ops = Mir.Inst.Ops.encode(.{ .reg1 = index_reg }),
5859	.data = .{ .imm = 1 },
5860	});
5861
5862	// sub count, 1
5863	_ = try self.addInst(.{
5864	.tag = .sub,
5865	.ops = Mir.Inst.Ops.encode(.{ .reg1 = count_reg }),
5866	.data = .{ .imm = 1 },
5867	});
5868
5869	// jmp loop
5870	_ = try self.addInst(.{
5871	.tag = .jmp,
5872	.ops = Mir.Inst.Ops.encode(.{}),
5873	.data = .{ .inst = loop_start },
5874	});
5875
5876	// end:
5877	try self.performReloc(loop_reloc);	5621	try self.performReloc(loop_reloc);
5878	}	5622	}
5879		5623
...	@@ -5905,24 +5649,17 @@ fn genInlineMemset(	...	@@ -5905,24 +5649,17 @@ fn genInlineMemset(
5905	try self.loadMemPtrIntoRegister(addr_reg, Type.usize, dst_ptr);	5649	try self.loadMemPtrIntoRegister(addr_reg, Type.usize, dst_ptr);
5906	},	5650	},
5907	.ptr_stack_offset, .stack_offset => \|off\| {	5651	.ptr_stack_offset, .stack_offset => \|off\| {
5908	_ = try self.addInst(.{	5652	try self.asmRegisterMemory(.lea, addr_reg.to64(), Memory.sib(.qword, .{
5909	.tag = .lea,	5653	.base = opts.dest_stack_base orelse .rbp,
5910	.ops = Mir.Inst.Ops.encode(.{	5654	.disp = -off,
5911	.reg1 = addr_reg.to64(),	5655	}));
5912	.reg2 = opts.dest_stack_base orelse .rbp,
5913	}),
5914	.data = .{ .imm = @bitCast(u32, -off) },
5915	});
5916	},	5656	},
5917	.register => \|reg\| {	5657	.register => \|reg\| {
5918	_ = try self.addInst(.{	5658	try self.asmRegisterRegister(
5919	.tag = .mov,	5659	.mov,
5920	.ops = Mir.Inst.Ops.encode(.{	5660	registerAlias(addr_reg, @intCast(u32, @divExact(reg.bitSize(), 8))),
5921	.reg1 = registerAlias(addr_reg, @divExact(reg.size(), 8)),	5661	reg,
5922	.reg2 = reg,	5662	);
5923	}),
5924	.data = undefined,
5925	});
5926	},	5663	},
5927	else => {	5664	else => {
5928	return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr});	5665	return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr});
...	@@ -5932,54 +5669,35 @@ fn genInlineMemset(	...	@@ -5932,54 +5669,35 @@ fn genInlineMemset(
5932	try self.genSetReg(Type.usize, index_reg, len);	5669	try self.genSetReg(Type.usize, index_reg, len);
5933	try self.genBinOpMir(.sub, Type.usize, .{ .register = index_reg }, .{ .immediate = 1 });	5670	try self.genBinOpMir(.sub, Type.usize, .{ .register = index_reg }, .{ .immediate = 1 });
5934		5671
5935	// loop:
5936	// cmp index_reg, -1
5937	const loop_start = try self.addInst(.{	5672	const loop_start = try self.addInst(.{
5938	.tag = .cmp,	5673	.tag = .cmp,
5939	.ops = Mir.Inst.Ops.encode(.{ .reg1 = index_reg }),	5674	.ops = .ri_s,
5940	.data = .{ .imm = @bitCast(u32, @as(i32, -1)) },	5675	.data = .{ .ri = .{
5941	});	5676	.r1 = index_reg,
5942		5677	.imm = @bitCast(u32, @as(i32, -1)),
5943	// je end
5944	const loop_reloc = try self.addInst(.{
5945	.tag = .cond_jmp,
5946	.ops = Mir.Inst.Ops.encode(.{}),
5947	.data = .{ .inst_cc = .{
5948	.inst = undefined,
5949	.cc = .e,
5950	} },	5678	} },
5951	});	5679	});
		5680	const loop_reloc = try self.asmJccReloc(undefined, .e);
5952		5681
5953	switch (value) {	5682	switch (value) {
5954	.immediate => \|x\| {	5683	.immediate => \|x\| {
5955	if (x > math.maxInt(i32)) {	5684	if (x > math.maxInt(i32)) {
5956	return self.fail("TODO inline memset for value immediate larger than 32bits", .{});	5685	return self.fail("TODO inline memset for value immediate larger than 32bits", .{});
5957	}	5686	}
5958	// mov byte ptr [rbp + index_reg + stack_offset], imm	5687	try self.asmMemoryImmediate(.mov, Memory.sib(.byte, .{
5959	_ = try self.addInst(.{	5688	.base = addr_reg,
5960	.tag = .mov_mem_index_imm,	5689	.scale_index = .{
5961	.ops = Mir.Inst.Ops.encode(.{ .reg1 = addr_reg }),	5690	.scale = 1,
5962	.data = .{ .payload = try self.addExtra(Mir.IndexRegisterDispImm.encode(index_reg, 0, @truncate(u32, x))) },	5691	.index = index_reg,
5963	});	5692	},
		5693	.disp = 0,
		5694	}), Immediate.u(@intCast(u8, x)));
5964	},	5695	},
5965	else => return self.fail("TODO inline memset for value of type {}", .{value}),	5696	else => return self.fail("TODO inline memset for value of type {}", .{value}),
5966	}	5697	}
5967		5698
5968	// sub index_reg, 1	5699	try self.asmRegisterImmediate(.sub, index_reg, Immediate.u(1));
5969	_ = try self.addInst(.{	5700	_ = try self.asmJmpReloc(loop_start);
5970	.tag = .sub,
5971	.ops = Mir.Inst.Ops.encode(.{ .reg1 = index_reg }),
5972	.data = .{ .imm = 1 },
5973	});
5974
5975	// jmp loop
5976	_ = try self.addInst(.{
5977	.tag = .jmp,
5978	.ops = Mir.Inst.Ops.encode(.{}),
5979	.data = .{ .inst = loop_start },
5980	});
5981
5982	// end:
5983	try self.performReloc(loop_reloc);	5701	try self.performReloc(loop_reloc);
5984	}	5702	}
5985		5703
...	@@ -5992,21 +5710,18 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void	...	@@ -5992,21 +5710,18 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
5992	if (off < std.math.minInt(i32) or off > std.math.maxInt(i32)) {	5710	if (off < std.math.minInt(i32) or off > std.math.maxInt(i32)) {
5993	return self.fail("stack offset too large", .{});	5711	return self.fail("stack offset too large", .{});
5994	}	5712	}
5995	_ = try self.addInst(.{	5713	try self.asmRegisterMemory(
5996	.tag = .lea,	5714	.lea,
5997	.ops = Mir.Inst.Ops.encode(.{	5715	registerAlias(reg, abi_size),
5998	.reg1 = registerAlias(reg, abi_size),	5716	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -off }),
5999	.reg2 = .rbp,	5717	);
6000	}),
6001	.data = .{ .imm = @bitCast(u32, -off) },
6002	});
6003	},	5718	},
6004	.unreach, .none => return, // Nothing to do.	5719	.unreach, .none => return, // Nothing to do.
6005	.undef => {	5720	.undef => {
6006	if (!self.wantSafety())	5721	if (!self.wantSafety())
6007	return; // The already existing value will do just fine.	5722	return; // The already existing value will do just fine.
6008	// Write the debug undefined value.	5723	// Write the debug undefined value.
6009	switch (registerAlias(reg, abi_size).size()) {	5724	switch (registerAlias(reg, abi_size).bitSize()) {
6010	8 => return self.genSetReg(ty, reg, .{ .immediate = 0xaa }),	5725	8 => return self.genSetReg(ty, reg, .{ .immediate = 0xaa }),
6011	16 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaa }),	5726	16 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaa }),
6012	32 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaa }),	5727	32 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaa }),
...	@@ -6015,50 +5730,29 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void	...	@@ -6015,50 +5730,29 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6015	}	5730	}
6016	},	5731	},
6017	.eflags => \|cc\| {	5732	.eflags => \|cc\| {
6018	_ = try self.addInst(.{	5733	return self.asmSetccRegister(reg.to8(), cc);
6019	.tag = .cond_set_byte,
6020	.ops = Mir.Inst.Ops.encode(.{
6021	.reg1 = reg.to8(),
6022	}),
6023	.data = .{ .cc = cc },
6024	});
6025	},	5734	},
6026	.immediate => \|x\| {	5735	.immediate => \|x\| {
6027	// 32-bit moves zero-extend to 64-bit, so xoring the 32-bit
6028	// register is the fastest way to zero a register.
6029	if (x == 0) {	5736	if (x == 0) {
6030	_ = try self.addInst(.{	5737	// 32-bit moves zero-extend to 64-bit, so xoring the 32-bit
6031	.tag = .xor,	5738	// register is the fastest way to zero a register.
6032	.ops = Mir.Inst.Ops.encode(.{	5739	return self.asmRegisterRegister(.xor, reg.to32(), reg.to32());
6033	.reg1 = reg.to32(),
6034	.reg2 = reg.to32(),
6035	}),
6036	.data = undefined,
6037	});
6038	return;
6039	}	5740	}
6040	if (x <= math.maxInt(i32)) {	5741	if (ty.isSignedInt()) {
6041	// Next best case: if we set the lower four bytes, the upper four will be zeroed.	5742	const signed_x = @bitCast(i64, x);
6042	_ = try self.addInst(.{	5743	if (math.minInt(i32) <= signed_x and signed_x <= math.maxInt(i32)) {
6043	.tag = .mov,	5744	return self.asmRegisterImmediate(
6044	.ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(reg, abi_size) }),	5745	.mov,
6045	.data = .{ .imm = @truncate(u32, x) },	5746	registerAlias(reg, abi_size),
6046	});	5747	Immediate.s(@intCast(i32, signed_x)),
6047	return;	5748	);
		5749	}
6048	}	5750	}
6049	// Worst case: we need to load the 64-bit register with the IMM. GNU's assemblers calls	5751	return self.asmRegisterImmediate(
6050	// this `movabs`, though this is officially just a different variant of the plain `mov`	5752	.mov,
6051	// instruction.	5753	registerAlias(reg, abi_size),
6052	//	5754	Immediate.u(x),
6053	// This encoding is, in fact, the same as the one used for 32-bit loads. The only	5755	);
6054	// difference is that we set REX.W before the instruction, which extends the load to
6055	// 64-bit and uses the full bit-width of the register.
6056	const payload = try self.addExtra(Mir.Imm64.encode(x));
6057	_ = try self.addInst(.{
6058	.tag = .movabs,
6059	.ops = Mir.Inst.Ops.encode(.{ .reg1 = reg.to64() }),
6060	.data = .{ .payload = payload },
6061	});
6062	},	5756	},
6063	.register => \|src_reg\| {	5757	.register => \|src_reg\| {
6064	// If the registers are the same, nothing to do.	5758	// If the registers are the same, nothing to do.
...	@@ -6069,69 +5763,38 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void	...	@@ -6069,69 +5763,38 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6069	.Int => switch (ty.intInfo(self.target.*).signedness) {	5763	.Int => switch (ty.intInfo(self.target.*).signedness) {
6070	.signed => {	5764	.signed => {
6071	if (abi_size <= 4) {	5765	if (abi_size <= 4) {
6072	_ = try self.addInst(.{	5766	return self.asmRegisterRegister(
6073	.tag = .mov_sign_extend,	5767	.movsx,
6074	.ops = Mir.Inst.Ops.encode(.{	5768	reg.to64(),
6075	.reg1 = reg.to64(),	5769	registerAlias(src_reg, abi_size),
6076	.reg2 = registerAlias(src_reg, abi_size),	5770	);
6077	}),
6078	.data = undefined,
6079	});
6080	return;
6081	}	5771	}
6082	},	5772	},
6083	.unsigned => {	5773	.unsigned => {
6084	if (abi_size <= 2) {	5774	if (abi_size <= 2) {
6085	_ = try self.addInst(.{	5775	return self.asmRegisterRegister(
6086	.tag = .mov_zero_extend,	5776	.movzx,
6087	.ops = Mir.Inst.Ops.encode(.{	5777	reg.to64(),
6088	.reg1 = reg.to64(),	5778	registerAlias(src_reg, abi_size),
6089	.reg2 = registerAlias(src_reg, abi_size),	5779	);
6090	}),
6091	.data = undefined,
6092	});
6093	return;
6094	}	5780	}
6095	},	5781	},
6096	},	5782	},
6097	.Float => {	5783	.Float => {
6098	if (intrinsicsAllowed(self.target.*, ty)) {	5784	if (intrinsicsAllowed(self.target.*, ty)) {
6099	const tag: Mir.Inst.Tag = switch (ty.tag()) {	5785	const tag: Mir.Inst.Tag = switch (ty.tag()) {
6100	.f32 => if (hasAvxSupport(self.target.*))	5786	.f32 => .movss,
6101	Mir.Inst.Tag.mov_f32_avx	5787	.f64 => .movsd,
6102	else
6103	Mir.Inst.Tag.mov_f32_sse,
6104	.f64 => if (hasAvxSupport(self.target.*))
6105	Mir.Inst.Tag.mov_f64_avx
6106	else
6107	Mir.Inst.Tag.mov_f64_sse,
6108	else => return self.fail("TODO genSetReg from register for {}", .{ty.fmtDebug()}),	5788	else => return self.fail("TODO genSetReg from register for {}", .{ty.fmtDebug()}),
6109	};	5789	};
6110	_ = try self.addInst(.{	5790	return self.asmRegisterRegister(tag, reg.to128(), src_reg.to128());
6111	.tag = tag,
6112	.ops = Mir.Inst.Ops.encode(.{
6113	.reg1 = reg.to128(),
6114	.reg2 = src_reg.to128(),
6115	.flags = 0b10,
6116	}),
6117	.data = undefined,
6118	});
6119	return;
6120	}	5791	}
6121
6122	return self.fail("TODO genSetReg from register for float with no intrinsics", .{});	5792	return self.fail("TODO genSetReg from register for float with no intrinsics", .{});
6123	},	5793	},
6124	else => {},	5794	else => {},
6125	}	5795	}
6126		5796
6127	_ = try self.addInst(.{	5797	try self.asmRegisterRegister(.mov, registerAlias(reg, abi_size), registerAlias(src_reg, abi_size));
6128	.tag = .mov,
6129	.ops = Mir.Inst.Ops.encode(.{
6130	.reg1 = registerAlias(reg, abi_size),
6131	.reg2 = registerAlias(src_reg, abi_size),
6132	}),
6133	.data = undefined,
6134	});
6135	},	5798	},
6136	.linker_load => {	5799	.linker_load => {
6137	switch (ty.zigTypeTag()) {	5800	switch (ty.zigTypeTag()) {
...	@@ -6141,45 +5804,30 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void	...	@@ -6141,45 +5804,30 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6141		5804
6142	if (intrinsicsAllowed(self.target.*, ty)) {	5805	if (intrinsicsAllowed(self.target.*, ty)) {
6143	const tag: Mir.Inst.Tag = switch (ty.tag()) {	5806	const tag: Mir.Inst.Tag = switch (ty.tag()) {
6144	.f32 => if (hasAvxSupport(self.target.*))	5807	.f32 => .movss,
6145	Mir.Inst.Tag.mov_f32_avx	5808	.f64 => .movsd,
6146	else
6147	Mir.Inst.Tag.mov_f32_sse,
6148	.f64 => if (hasAvxSupport(self.target.*))
6149	Mir.Inst.Tag.mov_f64_avx
6150	else
6151	Mir.Inst.Tag.mov_f64_sse,
6152	else => return self.fail("TODO genSetReg from memory for {}", .{ty.fmtDebug()}),	5809	else => return self.fail("TODO genSetReg from memory for {}", .{ty.fmtDebug()}),
6153	};	5810	};
6154		5811	const ptr_size: Memory.PtrSize = switch (ty.tag()) {
6155	_ = try self.addInst(.{	5812	.f32 => .dword,
6156	.tag = tag,	5813	.f64 => .qword,
6157	.ops = Mir.Inst.Ops.encode(.{	5814	else => unreachable,
6158	.reg1 = reg.to128(),	5815	};
6159	.reg2 = switch (ty.tag()) {	5816	return self.asmRegisterMemory(tag, reg.to128(), Memory.sib(ptr_size, .{
6160	.f32 => base_reg.to32(),	5817	.base = base_reg.to64(),
6161	.f64 => base_reg.to64(),	5818	.disp = 0,
6162	else => unreachable,	5819	}));
6163	},
6164	}),
6165	.data = .{ .imm = 0 },
6166	});
6167	return;
6168	}	5820	}
6169		5821
6170	return self.fail("TODO genSetReg from memory for float with no intrinsics", .{});	5822	return self.fail("TODO genSetReg from memory for float with no intrinsics", .{});
6171	},	5823	},
6172	else => {	5824	else => {
6173	try self.loadMemPtrIntoRegister(reg, Type.usize, mcv);	5825	try self.loadMemPtrIntoRegister(reg, Type.usize, mcv);
6174	_ = try self.addInst(.{	5826	try self.asmRegisterMemory(
6175	.tag = .mov,	5827	.mov,
6176	.ops = Mir.Inst.Ops.encode(.{	5828	registerAlias(reg, abi_size),
6177	.reg1 = registerAlias(reg, abi_size),	5829	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64(), .disp = 0 }),
6178	.reg2 = reg.to64(),	5830	);
6179	.flags = 0b01,
6180	}),
6181	.data = .{ .imm = 0 },
6182	});
6183	},	5831	},
6184	}	5832	}
6185	},	5833	},
...	@@ -6190,73 +5838,56 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void	...	@@ -6190,73 +5838,56 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6190		5838
6191	if (intrinsicsAllowed(self.target.*, ty)) {	5839	if (intrinsicsAllowed(self.target.*, ty)) {
6192	const tag: Mir.Inst.Tag = switch (ty.tag()) {	5840	const tag: Mir.Inst.Tag = switch (ty.tag()) {
6193	.f32 => if (hasAvxSupport(self.target.*))	5841	.f32 => .movss,
6194	Mir.Inst.Tag.mov_f32_avx	5842	.f64 => .movsd,
6195	else
6196	Mir.Inst.Tag.mov_f32_sse,
6197	.f64 => if (hasAvxSupport(self.target.*))
6198	Mir.Inst.Tag.mov_f64_avx
6199	else
6200	Mir.Inst.Tag.mov_f64_sse,
6201	else => return self.fail("TODO genSetReg from memory for {}", .{ty.fmtDebug()}),	5843	else => return self.fail("TODO genSetReg from memory for {}", .{ty.fmtDebug()}),
6202	};	5844	};
6203		5845	const ptr_size: Memory.PtrSize = switch (ty.tag()) {
6204	_ = try self.addInst(.{	5846	.f32 => .dword,
6205	.tag = tag,	5847	.f64 => .qword,
6206	.ops = Mir.Inst.Ops.encode(.{	5848	else => unreachable,
6207	.reg1 = reg.to128(),	5849	};
6208	.reg2 = switch (ty.tag()) {	5850	return self.asmRegisterMemory(tag, reg.to128(), Memory.sib(ptr_size, .{
6209	.f32 => base_reg.to32(),	5851	.base = base_reg.to64(),
6210	.f64 => base_reg.to64(),	5852	.disp = 0,
6211	else => unreachable,	5853	}));
6212	},
6213	}),
6214	.data = .{ .imm = 0 },
6215	});
6216	return;
6217	}	5854	}
6218		5855
6219	return self.fail("TODO genSetReg from memory for float with no intrinsics", .{});	5856	return self.fail("TODO genSetReg from memory for float with no intrinsics", .{});
6220	},	5857	},
6221	else => {	5858	else => {
6222	if (x <= math.maxInt(i32)) {	5859	if (x <= math.maxInt(i32)) {
6223	// mov reg, [ds:imm32]	5860	try self.asmRegisterMemory(
6224	_ = try self.addInst(.{	5861	.mov,
6225	.tag = .mov,	5862	registerAlias(reg, abi_size),
6226	.ops = Mir.Inst.Ops.encode(.{	5863	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
6227	.reg1 = registerAlias(reg, abi_size),	5864	.base = .ds,
6228	.flags = 0b01,	5865	.disp = @intCast(i32, x),
6229	}),	5866	}),
6230	.data = .{ .imm = @truncate(u32, x) },	5867	);
6231	});
6232	} else {	5868	} else {
6233	// If this is RAX, we can use a direct load.	5869	if (reg.to64() == .rax) {
6234	// Otherwise, we need to load the address, then indirectly load the value.	5870	// If this is RAX, we can use a direct load.
6235	if (reg.id() == 0) {	5871	// Otherwise, we need to load the address, then indirectly load the value.
6236	// movabs rax, ds:moffs64	5872	var moffs: Mir.MemoryMoffs = .{
6237	const payload = try self.addExtra(Mir.Imm64.encode(x));	5873	.seg = @enumToInt(Register.ds),
		5874	.msb = undefined,
		5875	.lsb = undefined,
		5876	};
		5877	moffs.encodeOffset(x);
6238	_ = try self.addInst(.{	5878	_ = try self.addInst(.{
6239	.tag = .movabs,	5879	.tag = .mov_moffs,
6240	.ops = Mir.Inst.Ops.encode(.{	5880	.ops = .rax_moffs,
6241	.reg1 = .rax,	5881	.data = .{ .payload = try self.addExtra(moffs) },
6242	.flags = 0b01, // imm64 will become moffs64
6243	}),
6244	.data = .{ .payload = payload },
6245	});	5882	});
6246	} else {	5883	} else {
6247	// Rather than duplicate the logic used for the move, we just use a self-call with a new MCValue.	5884	// Rather than duplicate the logic used for the move, we just use a self-call with a new MCValue.
6248	try self.genSetReg(ty, reg, MCValue{ .immediate = x });	5885	try self.genSetReg(ty, reg, MCValue{ .immediate = x });
6249		5886	try self.asmRegisterMemory(
6250	// mov reg, [reg + 0x0]	5887	.mov,
6251	_ = try self.addInst(.{	5888	registerAlias(reg, abi_size),
6252	.tag = .mov,	5889	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64(), .disp = 0 }),
6253	.ops = Mir.Inst.Ops.encode(.{	5890	);
6254	.reg1 = registerAlias(reg, abi_size),
6255	.reg2 = reg.to64(),
6256	.flags = 0b01,
6257	}),
6258	.data = .{ .imm = 0 },
6259	});
6260	}	5891	}
6261	}	5892	}
6262	},	5893	},
...	@@ -6270,85 +5901,59 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void	...	@@ -6270,85 +5901,59 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
6270	.Int => switch (ty.intInfo(self.target.*).signedness) {	5901	.Int => switch (ty.intInfo(self.target.*).signedness) {
6271	.signed => {	5902	.signed => {
6272	if (abi_size <= 4) {	5903	if (abi_size <= 4) {
6273	const flags: u2 = switch (abi_size) {	5904	return self.asmRegisterMemory(
6274	1 => 0b01,	5905	.movsx,
6275	2 => 0b10,	5906	reg.to64(),
6276	4 => 0b11,	5907	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
6277	else => unreachable,	5908	.base = .rbp,
6278	};	5909	.disp = -off,
6279	_ = try self.addInst(.{
6280	.tag = .mov_sign_extend,
6281	.ops = Mir.Inst.Ops.encode(.{
6282	.reg1 = reg.to64(),
6283	.reg2 = .rbp,
6284	.flags = flags,
6285	}),	5910	}),
6286	.data = .{ .imm = @bitCast(u32, -off) },	5911	);
6287	});
6288	return;
6289	}	5912	}
6290	},	5913	},
6291	.unsigned => {	5914	.unsigned => {
6292	if (abi_size <= 2) {	5915	if (abi_size <= 2) {
6293	const flags: u2 = switch (abi_size) {	5916	return self.asmRegisterMemory(
6294	1 => 0b01,	5917	.movzx,
6295	2 => 0b10,	5918	reg.to64(),
6296	else => unreachable,	5919	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
6297	};	5920	.base = .rbp,
6298	_ = try self.addInst(.{	5921	.disp = -off,
6299	.tag = .mov_zero_extend,
6300	.ops = Mir.Inst.Ops.encode(.{
6301	.reg1 = reg.to64(),
6302	.reg2 = .rbp,
6303	.flags = flags,
6304	}),	5922	}),
6305	.data = .{ .imm = @bitCast(u32, -off) },	5923	);
6306	});
6307	return;
6308	}	5924	}
6309	},	5925	},
6310	},	5926	},
6311	.Float => {	5927	.Float => {
6312	if (intrinsicsAllowed(self.target.*, ty)) {	5928	if (intrinsicsAllowed(self.target.*, ty)) {
6313	const tag: Mir.Inst.Tag = switch (ty.tag()) {	5929	const tag: Mir.Inst.Tag = switch (ty.tag()) {
6314	.f32 => if (hasAvxSupport(self.target.*))	5930	.f32 => .movss,
6315	Mir.Inst.Tag.mov_f32_avx	5931	.f64 => .movsd,
6316	else	5932	else => return self.fail(
6317	Mir.Inst.Tag.mov_f32_sse,	5933	"TODO genSetReg from stack offset for {}",
6318	.f64 => if (hasAvxSupport(self.target.*))	5934	.{ty.fmtDebug()},
6319	Mir.Inst.Tag.mov_f64_avx	5935	),
6320	else
6321	Mir.Inst.Tag.mov_f64_sse,
6322	else => return self.fail("TODO genSetReg from stack offset for {}", .{ty.fmtDebug()}),
6323	};	5936	};
6324	_ = try self.addInst(.{	5937	const ptr_size: Memory.PtrSize = switch (ty.tag()) {
6325	.tag = tag,	5938	.f32 => .dword,
6326	.ops = Mir.Inst.Ops.encode(.{	5939	.f64 => .qword,
6327	.reg1 = reg.to128(),	5940	else => unreachable,
6328	.reg2 = switch (ty.tag()) {	5941	};
6329	.f32 => .ebp,	5942	return self.asmRegisterMemory(tag, reg.to128(), Memory.sib(ptr_size, .{
6330	.f64 => .rbp,	5943	.base = .rbp,
6331	else => unreachable,	5944	.disp = -off,
6332	},	5945	}));
6333	}),
6334	.data = .{ .imm = @bitCast(u32, -off) },
6335	});
6336	return;
6337	}	5946	}
6338	return self.fail("TODO genSetReg from stack offset for float with no intrinsics", .{});	5947	return self.fail("TODO genSetReg from stack offset for float with no intrinsics", .{});
6339	},	5948	},
6340	else => {},	5949	else => {},
6341	}	5950	}
6342		5951
6343	_ = try self.addInst(.{	5952	try self.asmRegisterMemory(
6344	.tag = .mov,	5953	.mov,
6345	.ops = Mir.Inst.Ops.encode(.{	5954	registerAlias(reg, abi_size),
6346	.reg1 = registerAlias(reg, abi_size),	5955	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -off }),
6347	.reg2 = .rbp,	5956	);
6348	.flags = 0b01,
6349	}),
6350	.data = .{ .imm = @bitCast(u32, -off) },
6351	});
6352	},	5957	},
6353	}	5958	}
6354	}	5959	}
...	@@ -6412,6 +6017,16 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6412,6 +6017,16 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
6412	const src_ty = self.air.typeOf(ty_op.operand);	6017	const src_ty = self.air.typeOf(ty_op.operand);
6413	const dst_ty = self.air.typeOfIndex(inst);	6018	const dst_ty = self.air.typeOfIndex(inst);
6414	const operand = try self.resolveInst(ty_op.operand);	6019	const operand = try self.resolveInst(ty_op.operand);
		6020	const src_abi_size = @intCast(u32, src_ty.abiSize(self.target.*));
		6021	const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
		6022
		6023	switch (src_abi_size) {
		6024	4, 8 => {},
		6025	else => \|size\| return self.fail("TODO load ST(0) with abiSize={}", .{size}),
		6026	}
		6027	if (dst_abi_size > 8) {
		6028	return self.fail("TODO convert float with abiSize={}", .{dst_abi_size});
		6029	}
6415		6030
6416	// move float src to ST(0)	6031	// move float src to ST(0)
6417	const stack_offset = switch (operand) {	6032	const stack_offset = switch (operand) {
...	@@ -6419,41 +6034,24 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6419,41 +6034,24 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
6419	else => blk: {	6034	else => blk: {
6420	const offset = @intCast(i32, try self.allocMem(	6035	const offset = @intCast(i32, try self.allocMem(
6421	inst,	6036	inst,
6422	@intCast(u32, src_ty.abiSize(self.target.*)),	6037	src_abi_size,
6423	src_ty.abiAlignment(self.target.*),	6038	src_ty.abiAlignment(self.target.*),
6424	));	6039	));
6425	try self.genSetStack(src_ty, offset, operand, .{});	6040	try self.genSetStack(src_ty, offset, operand, .{});
6426	break :blk offset;	6041	break :blk offset;
6427	},	6042	},
6428	};	6043	};
6429	_ = try self.addInst(.{	6044	try self.asmMemory(.fld, Memory.sib(Memory.PtrSize.fromSize(src_abi_size), .{
6430	.tag = .fld,	6045	.base = .rbp,
6431	.ops = Mir.Inst.Ops.encode(.{	6046	.disp = -stack_offset,
6432	.reg1 = .rbp,	6047	}));
6433	.flags = switch (src_ty.abiSize(self.target.*)) {
6434	4 => 0b01,
6435	8 => 0b10,
6436	else => \|size\| return self.fail("TODO load ST(0) with abiSize={}", .{size}),
6437	},
6438	}),
6439	.data = .{ .imm = @bitCast(u32, -stack_offset) },
6440	});
6441		6048
6442	// convert	6049	// convert
6443	const stack_dst = try self.allocRegOrMem(inst, false);	6050	const stack_dst = try self.allocRegOrMem(inst, false);
6444	_ = try self.addInst(.{	6051	try self.asmMemory(.fisttp, Memory.sib(Memory.PtrSize.fromSize(dst_abi_size), .{
6445	.tag = .fisttp,	6052	.base = .rbp,
6446	.ops = Mir.Inst.Ops.encode(.{	6053	.disp = -stack_dst.stack_offset,
6447	.reg1 = .rbp,	6054	}));
6448	.flags = switch (dst_ty.abiSize(self.target.*)) {
6449	1...2 => 0b00,
6450	3...4 => 0b01,
6451	5...8 => 0b10,
6452	else => \|size\| return self.fail("TODO convert float with abiSize={}", .{size}),
6453	},
6454	}),
6455	.data = .{ .imm = @bitCast(u32, -stack_dst.stack_offset) },
6456	});
6457		6055
6458	return self.finishAir(inst, stack_dst, .{ ty_op.operand, .none, .none });	6056	return self.finishAir(inst, stack_dst, .{ ty_op.operand, .none, .none });
6459	}	6057	}
...	@@ -6544,15 +6142,10 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6544,15 +6142,10 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
6544	.linker_load, .memory => {	6142	.linker_load, .memory => {
6545	const reg = try self.register_manager.allocReg(null, gp);	6143	const reg = try self.register_manager.allocReg(null, gp);
6546	try self.loadMemPtrIntoRegister(reg, src_ty, src_ptr);	6144	try self.loadMemPtrIntoRegister(reg, src_ty, src_ptr);
6547	_ = try self.addInst(.{	6145	try self.asmRegisterMemory(.mov, reg, Memory.sib(.qword, .{
6548	.tag = .mov,	6146	.base = reg,
6549	.ops = Mir.Inst.Ops.encode(.{	6147	.disp = 0,
6550	.reg1 = reg,	6148	}));
6551	.reg2 = reg,
6552	.flags = 0b01,
6553	}),
6554	.data = .{ .imm = 0 },
6555	});
6556	break :blk MCValue{ .register = reg };	6149	break :blk MCValue{ .register = reg };
6557	},	6150	},
6558	else => break :blk src_ptr,	6151	else => break :blk src_ptr,
...	@@ -6903,7 +6496,7 @@ fn resolveCallingConventionValues(self: *Self, fn_ty: Type) !CallMCValues {	...	@@ -6903,7 +6496,7 @@ fn resolveCallingConventionValues(self: *Self, fn_ty: Type) !CallMCValues {
6903	if (ret_ty_size == 0) {	6496	if (ret_ty_size == 0) {
6904	assert(ret_ty.isError());	6497	assert(ret_ty.isError());
6905	result.return_value = .{ .immediate = 0 };	6498	result.return_value = .{ .immediate = 0 };
6906	} else if (ret_ty_size <= 8) {	6499	} else if (ret_ty_size <= 8 and !ret_ty.isRuntimeFloat()) {
6907	const aliased_reg = registerAlias(abi.getCAbiIntReturnRegs(self.target.*)[0], ret_ty_size);	6500	const aliased_reg = registerAlias(abi.getCAbiIntReturnRegs(self.target.*)[0], ret_ty_size);
6908	result.return_value = .{ .register = aliased_reg };	6501	result.return_value = .{ .register = aliased_reg };
6909	} else {	6502	} else {
...	@@ -6978,28 +6571,34 @@ fn parseRegName(name: []const u8) ?Register {	...	@@ -6978,28 +6571,34 @@ fn parseRegName(name: []const u8) ?Register {
6978		6571
6979	/// Returns register wide enough to hold at least `size_bytes`.	6572	/// Returns register wide enough to hold at least `size_bytes`.
6980	fn registerAlias(reg: Register, size_bytes: u32) Register {	6573	fn registerAlias(reg: Register, size_bytes: u32) Register {
6981	if (size_bytes == 0) {	6574	return switch (reg.class()) {
6982	unreachable; // should be comptime-known	6575	.general_purpose => if (size_bytes == 0)
6983	} else if (size_bytes <= 1) {	6576	unreachable // should be comptime-known
6984	return reg.to8();	6577	else if (size_bytes <= 1)
6985	} else if (size_bytes <= 2) {	6578	reg.to8()
6986	return reg.to16();	6579	else if (size_bytes <= 2)
6987	} else if (size_bytes <= 4) {	6580	reg.to16()
6988	return reg.to32();	6581	else if (size_bytes <= 4)
6989	} else if (size_bytes <= 8) {	6582	reg.to32()
6990	return reg.to64();	6583	else if (size_bytes <= 8)
6991	} else if (size_bytes <= 16) {	6584	reg.to64()
6992	return reg.to128();	6585	else
6993	} else if (size_bytes <= 32) {	6586	unreachable,
6994	return reg.to256();	6587	.floating_point => if (size_bytes <= 16)
6995	} else unreachable;	6588	reg.to128()
		6589	else if (size_bytes <= 32)
		6590	reg.to256()
		6591	else
		6592	unreachable,
		6593	.segment => unreachable,
		6594	};
6996	}	6595	}
6997		6596
6998	/// Truncates the value in the register in place.	6597	/// Truncates the value in the register in place.
6999	/// Clobbers any remaining bits.	6598	/// Clobbers any remaining bits.
7000	fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {	6599	fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
7001	const int_info = ty.intInfo(self.target.*);	6600	const int_info = ty.intInfo(self.target.*);
7002	const max_reg_bit_width = Register.rax.size();	6601	const max_reg_bit_width = Register.rax.bitSize();
7003	switch (int_info.signedness) {	6602	switch (int_info.signedness) {
7004	.signed => {	6603	.signed => {
7005	const shift = @intCast(u6, max_reg_bit_width - int_info.bits);	6604	const shift = @intCast(u6, max_reg_bit_width - int_info.bits);
...	@@ -7009,7 +6608,7 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {	...	@@ -7009,7 +6608,7 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
7009	.unsigned => {	6608	.unsigned => {
7010	const shift = @intCast(u6, max_reg_bit_width - int_info.bits);	6609	const shift = @intCast(u6, max_reg_bit_width - int_info.bits);
7011	const mask = (~@as(u64, 0)) >> shift;	6610	const mask = (~@as(u64, 0)) >> shift;
7012	if (int_info.bits <= 32) {	6611	if (int_info.bits < 32) {
7013	try self.genBinOpMir(.@"and", Type.usize, .{ .register = reg }, .{ .immediate = mask });	6612	try self.genBinOpMir(.@"and", Type.usize, .{ .register = reg }, .{ .immediate = mask });
7014	} else {	6613	} else {
7015	const tmp_reg = try self.copyToTmpRegister(Type.usize, .{ .immediate = mask });	6614	const tmp_reg = try self.copyToTmpRegister(Type.usize, .{ .immediate = mask });

src/arch/x86_64/Emit.zig+384-2836

...	@@ -7,6 +7,7 @@ const std = @import("std");	...	@@ -7,6 +7,7 @@ const std = @import("std");
7	const assert = std.debug.assert;	7	const assert = std.debug.assert;
8	const bits = @import("bits.zig");	8	const bits = @import("bits.zig");
9	const abi = @import("abi.zig");	9	const abi = @import("abi.zig");
		10	const encoder = @import("encoder.zig");
10	const link = @import("../../link.zig");	11	const link = @import("../../link.zig");
11	const log = std.log.scoped(.codegen);	12	const log = std.log.scoped(.codegen);
12	const math = std.math;	13	const math = std.math;
...	@@ -19,12 +20,14 @@ const CodeGen = @import("CodeGen.zig");	...	@@ -19,12 +20,14 @@ const CodeGen = @import("CodeGen.zig");
19	const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;	20	const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;
20	const Encoder = bits.Encoder;	21	const Encoder = bits.Encoder;
21	const ErrorMsg = Module.ErrorMsg;	22	const ErrorMsg = Module.ErrorMsg;
		23	const Immediate = bits.Immediate;
		24	const Instruction = encoder.Instruction;
22	const MCValue = @import("CodeGen.zig").MCValue;	25	const MCValue = @import("CodeGen.zig").MCValue;
		26	const Memory = bits.Memory;
23	const Mir = @import("Mir.zig");	27	const Mir = @import("Mir.zig");
24	const Module = @import("../../Module.zig");	28	const Module = @import("../../Module.zig");
25	const Instruction = bits.Instruction;
26	const Type = @import("../../type.zig").Type;
27	const Register = bits.Register;	29	const Register = bits.Register;
		30	const Type = @import("../../type.zig").Type;
28		31
29	mir: Mir,	32	mir: Mir,
30	bin_file: *link.File,	33	bin_file: *link.File,
...	@@ -45,6 +48,8 @@ relocs: std.ArrayListUnmanaged(Reloc) = .{},	...	@@ -45,6 +48,8 @@ relocs: std.ArrayListUnmanaged(Reloc) = .{},
45	const InnerError = error{	48	const InnerError = error{
46	OutOfMemory,	49	OutOfMemory,
47	EmitFail,	50	EmitFail,
		51	InvalidInstruction,
		52	CannotEncode,
48	};	53	};
49		54
50	const Reloc = struct {	55	const Reloc = struct {
...	@@ -65,133 +70,66 @@ pub fn lowerMir(emit: *Emit) InnerError!void {	...	@@ -65,133 +70,66 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
65	const inst = @intCast(u32, index);	70	const inst = @intCast(u32, index);
66	try emit.code_offset_mapping.putNoClobber(emit.bin_file.allocator, inst, emit.code.items.len);	71	try emit.code_offset_mapping.putNoClobber(emit.bin_file.allocator, inst, emit.code.items.len);
67	switch (tag) {	72	switch (tag) {
68	// GPR instructions	73	.adc,
69	.adc => try emit.mirArith(.adc, inst),	74	.add,
70	.add => try emit.mirArith(.add, inst),	75	.@"and",
71	.sub => try emit.mirArith(.sub, inst),	76	.call,
72	.xor => try emit.mirArith(.xor, inst),	77	.cbw,
73	.@"and" => try emit.mirArith(.@"and", inst),	78	.cwde,
74	.@"or" => try emit.mirArith(.@"or", inst),	79	.cdqe,
75	.sbb => try emit.mirArith(.sbb, inst),	80	.cwd,
76	.cmp => try emit.mirArith(.cmp, inst),	81	.cdq,
77	.mov => try emit.mirArith(.mov, inst),	82	.cqo,
78		83	.cmp,
79	.adc_mem_imm => try emit.mirArithMemImm(.adc, inst),	84	.div,
80	.add_mem_imm => try emit.mirArithMemImm(.add, inst),	85	.fisttp,
81	.sub_mem_imm => try emit.mirArithMemImm(.sub, inst),	86	.fld,
82	.xor_mem_imm => try emit.mirArithMemImm(.xor, inst),	87	.idiv,
83	.and_mem_imm => try emit.mirArithMemImm(.@"and", inst),	88	.imul,
84	.or_mem_imm => try emit.mirArithMemImm(.@"or", inst),	89	.int3,
85	.sbb_mem_imm => try emit.mirArithMemImm(.sbb, inst),	90	.jmp,
86	.cmp_mem_imm => try emit.mirArithMemImm(.cmp, inst),	91	.lea,
87	.mov_mem_imm => try emit.mirArithMemImm(.mov, inst),	92	.mov,
88		93	.movzx,
89	.adc_scale_src => try emit.mirArithScaleSrc(.adc, inst),	94	.mul,
90	.add_scale_src => try emit.mirArithScaleSrc(.add, inst),	95	.nop,
91	.sub_scale_src => try emit.mirArithScaleSrc(.sub, inst),	96	.@"or",
92	.xor_scale_src => try emit.mirArithScaleSrc(.xor, inst),	97	.pop,
93	.and_scale_src => try emit.mirArithScaleSrc(.@"and", inst),	98	.push,
94	.or_scale_src => try emit.mirArithScaleSrc(.@"or", inst),	99	.ret,
95	.sbb_scale_src => try emit.mirArithScaleSrc(.sbb, inst),	100	.sal,
96	.cmp_scale_src => try emit.mirArithScaleSrc(.cmp, inst),	101	.sar,
97	.mov_scale_src => try emit.mirArithScaleSrc(.mov, inst),	102	.sbb,
98		103	.shl,
99	.adc_scale_dst => try emit.mirArithScaleDst(.adc, inst),	104	.shr,
100	.add_scale_dst => try emit.mirArithScaleDst(.add, inst),	105	.sub,
101	.sub_scale_dst => try emit.mirArithScaleDst(.sub, inst),	106	.syscall,
102	.xor_scale_dst => try emit.mirArithScaleDst(.xor, inst),	107	.@"test",
103	.and_scale_dst => try emit.mirArithScaleDst(.@"and", inst),	108	.ud2,
104	.or_scale_dst => try emit.mirArithScaleDst(.@"or", inst),	109	.xor,
105	.sbb_scale_dst => try emit.mirArithScaleDst(.sbb, inst),
106	.cmp_scale_dst => try emit.mirArithScaleDst(.cmp, inst),
107	.mov_scale_dst => try emit.mirArithScaleDst(.mov, inst),
108
109	.adc_scale_imm => try emit.mirArithScaleImm(.adc, inst),
110	.add_scale_imm => try emit.mirArithScaleImm(.add, inst),
111	.sub_scale_imm => try emit.mirArithScaleImm(.sub, inst),
112	.xor_scale_imm => try emit.mirArithScaleImm(.xor, inst),
113	.and_scale_imm => try emit.mirArithScaleImm(.@"and", inst),
114	.or_scale_imm => try emit.mirArithScaleImm(.@"or", inst),
115	.sbb_scale_imm => try emit.mirArithScaleImm(.sbb, inst),
116	.cmp_scale_imm => try emit.mirArithScaleImm(.cmp, inst),
117	.mov_scale_imm => try emit.mirArithScaleImm(.mov, inst),
118
119	.adc_mem_index_imm => try emit.mirArithMemIndexImm(.adc, inst),
120	.add_mem_index_imm => try emit.mirArithMemIndexImm(.add, inst),
121	.sub_mem_index_imm => try emit.mirArithMemIndexImm(.sub, inst),
122	.xor_mem_index_imm => try emit.mirArithMemIndexImm(.xor, inst),
123	.and_mem_index_imm => try emit.mirArithMemIndexImm(.@"and", inst),
124	.or_mem_index_imm => try emit.mirArithMemIndexImm(.@"or", inst),
125	.sbb_mem_index_imm => try emit.mirArithMemIndexImm(.sbb, inst),
126	.cmp_mem_index_imm => try emit.mirArithMemIndexImm(.cmp, inst),
127	.mov_mem_index_imm => try emit.mirArithMemIndexImm(.mov, inst),
128
129	.mov_sign_extend => try emit.mirMovSignExtend(inst),
130	.mov_zero_extend => try emit.mirMovZeroExtend(inst),
131
132	.movabs => try emit.mirMovabs(inst),
133
134	.fisttp => try emit.mirFisttp(inst),
135	.fld => try emit.mirFld(inst),
136
137	.lea => try emit.mirLea(inst),
138	.lea_pic => try emit.mirLeaPic(inst),
139
140	.shl => try emit.mirShift(.shl, inst),
141	.sal => try emit.mirShift(.sal, inst),
142	.shr => try emit.mirShift(.shr, inst),
143	.sar => try emit.mirShift(.sar, inst),
144
145	.imul => try emit.mirMulDiv(.imul, inst),
146	.mul => try emit.mirMulDiv(.mul, inst),
147	.idiv => try emit.mirMulDiv(.idiv, inst),
148	.div => try emit.mirMulDiv(.div, inst),
149	.imul_complex => try emit.mirIMulComplex(inst),
150
151	.cwd => try emit.mirCwd(inst),
152
153	.push => try emit.mirPushPop(.push, inst),
154	.pop => try emit.mirPushPop(.pop, inst),
155
156	.jmp => try emit.mirJmpCall(.jmp_near, inst),
157	.call => try emit.mirJmpCall(.call_near, inst),
158
159	.cond_jmp => try emit.mirCondJmp(inst),
160	.cond_set_byte => try emit.mirCondSetByte(inst),
161	.cond_mov => try emit.mirCondMov(inst),
162
163	.ret => try emit.mirRet(inst),
164
165	.syscall => try emit.mirSyscall(),
166
167	.@"test" => try emit.mirTest(inst),
168
169	.ud => try emit.mirUndefinedInstruction(),
170	.interrupt => try emit.mirInterrupt(inst),
171	.nop => {}, // just skip it
172
173	// SSE instructions
174	.mov_f64_sse => try emit.mirMovFloatSse(.movsd, inst),
175	.mov_f32_sse => try emit.mirMovFloatSse(.movss, inst),
176		110
177	.add_f64_sse => try emit.mirAddFloatSse(.addsd, inst),	111	.addss,
178	.add_f32_sse => try emit.mirAddFloatSse(.addss, inst),	112	.cmpss,
		113	.movss,
		114	.ucomiss,
		115	.addsd,
		116	.cmpsd,
		117	.movsd,
		118	.ucomisd,
		119	=> try emit.mirEncodeGeneric(tag, inst),
179		120
180	.cmp_f64_sse => try emit.mirCmpFloatSse(.ucomisd, inst),	121	.jmp_reloc => try emit.mirJmpReloc(inst),
181	.cmp_f32_sse => try emit.mirCmpFloatSse(.ucomiss, inst),
182		122
183	// AVX instructions	123	.call_extern => try emit.mirCallExtern(inst),
184	.mov_f64_avx => try emit.mirMovFloatAvx(.vmovsd, inst),
185	.mov_f32_avx => try emit.mirMovFloatAvx(.vmovss, inst),
186		124
187	.add_f64_avx => try emit.mirAddFloatAvx(.vaddsd, inst),	125	.lea_linker => try emit.mirLeaLinker(inst),
188	.add_f32_avx => try emit.mirAddFloatAvx(.vaddss, inst),
189		126
190	.cmp_f64_avx => try emit.mirCmpFloatAvx(.vucomisd, inst),	127	.mov_moffs => try emit.mirMovMoffs(inst),
191	.cmp_f32_avx => try emit.mirCmpFloatAvx(.vucomiss, inst),
192		128
193	// Pseudo-instructions	129	.movsx => try emit.mirMovsx(inst),
194	.call_extern => try emit.mirCallExtern(inst),	130	.cmovcc => try emit.mirCmovcc(inst),
		131	.setcc => try emit.mirSetcc(inst),
		132	.jcc => try emit.mirJcc(inst),
195		133
196	.dbg_line => try emit.mirDbgLine(inst),	134	.dbg_line => try emit.mirDbgLine(inst),
197	.dbg_prologue_end => try emit.mirDbgPrologueEnd(inst),	135	.dbg_prologue_end => try emit.mirDbgPrologueEnd(inst),
...	@@ -200,9 +138,7 @@ pub fn lowerMir(emit: *Emit) InnerError!void {	...	@@ -200,9 +138,7 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
200	.push_regs => try emit.mirPushPopRegisterList(.push, inst),	138	.push_regs => try emit.mirPushPopRegisterList(.push, inst),
201	.pop_regs => try emit.mirPushPopRegisterList(.pop, inst),	139	.pop_regs => try emit.mirPushPopRegisterList(.pop, inst),
202		140
203	else => {	141	.dead => {},
204	return emit.fail("Implement MIR->Emit lowering for x86_64 for pseudo-inst: {}", .{tag});
205	},
206	}	142	}
207	}	143	}
208		144
...	@@ -235,798 +171,385 @@ fn fixupRelocs(emit: *Emit) InnerError!void {	...	@@ -235,798 +171,385 @@ fn fixupRelocs(emit: *Emit) InnerError!void {
235	}	171	}
236	}	172	}
237		173
238	fn mirUndefinedInstruction(emit: *Emit) InnerError!void {	174	fn encode(emit: *Emit, mnemonic: Instruction.Mnemonic, ops: struct {
239	return lowerToZoEnc(.ud2, emit.code);	175	op1: Instruction.Operand = .none,
		176	op2: Instruction.Operand = .none,
		177	op3: Instruction.Operand = .none,
		178	op4: Instruction.Operand = .none,
		179	}) InnerError!void {
		180	const inst = try Instruction.new(mnemonic, .{
		181	.op1 = ops.op1,
		182	.op2 = ops.op2,
		183	.op3 = ops.op3,
		184	.op4 = ops.op4,
		185	});
		186	return inst.encode(emit.code.writer());
240	}	187	}
241		188
242	fn mirInterrupt(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {	189	fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
243	const tag = emit.mir.instructions.items(.tag)[inst];	190	const mnemonic = inline for (@typeInfo(Instruction.Mnemonic).Enum.fields) \|field\| {
244	assert(tag == .interrupt);	191	if (mem.eql(u8, field.name, @tagName(tag))) break @field(Instruction.Mnemonic, field.name);
245	const ops = emit.mir.instructions.items(.ops)[inst].decode();	192	} else unreachable;
246	switch (ops.flags) {
247	0b00 => return lowerToZoEnc(.int3, emit.code),
248	else => return emit.fail("TODO handle variant 0b{b} of interrupt instruction", .{ops.flags}),
249	}
250	}
251		193
252	fn mirSyscall(emit: *Emit) InnerError!void {	194	const ops = emit.mir.instructions.items(.ops)[inst];
253	return lowerToZoEnc(.syscall, emit.code);	195	const data = emit.mir.instructions.items(.data)[inst];
254	}
255		196
256	fn mirPushPop(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {	197	var op1: Instruction.Operand = .none;
257	const ops = emit.mir.instructions.items(.ops)[inst].decode();	198	var op2: Instruction.Operand = .none;
258	switch (ops.flags) {	199	var op3: Instruction.Operand = .none;
259	0b00 => {	200	var op4: Instruction.Operand = .none;
260	// PUSH/POP reg
261	return lowerToOEnc(tag, ops.reg1, emit.code);
262	},
263	0b01 => {
264	// PUSH/POP r/m64
265	const imm = emit.mir.instructions.items(.data)[inst].imm;
266	const ptr_size: Memory.PtrSize = switch (immOpSize(imm)) {
267	16 => .word_ptr,
268	else => .qword_ptr,
269	};
270	return lowerToMEnc(tag, RegisterOrMemory.mem(ptr_size, .{
271	.disp = imm,
272	.base = ops.reg1,
273	}), emit.code);
274	},
275	0b10 => {
276	// PUSH imm32
277	assert(tag == .push);
278	const imm = emit.mir.instructions.items(.data)[inst].imm;
279	return lowerToIEnc(.push, imm, emit.code);
280	},
281	0b11 => unreachable,
282	}
283	}
284		201
285	fn mirPushPopRegisterList(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {	202	switch (ops) {
286	const ops = emit.mir.instructions.items(.ops)[inst].decode();	203	.none => {},
287	const payload = emit.mir.instructions.items(.data)[inst].payload;	204	.imm_s => op1 = .{ .imm = Immediate.s(@bitCast(i32, data.imm)) },
288	const save_reg_list = emit.mir.extraData(Mir.SaveRegisterList, payload).data;	205	.imm_u => op1 = .{ .imm = Immediate.u(data.imm) },
289	var disp: i32 = -@intCast(i32, save_reg_list.stack_end);	206	.r => op1 = .{ .reg = data.r },
290	const reg_list = Mir.RegisterList.fromInt(save_reg_list.register_list);	207	.rr => {
291	const callee_preserved_regs = abi.getCalleePreservedRegs(emit.target.*);	208	op1 = .{ .reg = data.rr.r1 };
292	for (callee_preserved_regs) \|reg\| {	209	op2 = .{ .reg = data.rr.r2 };
293	if (reg_list.isSet(callee_preserved_regs, reg)) {	210	},
294	switch (tag) {	211	.ri_s, .ri_u => {
295	.push => try lowerToMrEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{	212	const imm = switch (ops) {
296	.disp = @bitCast(u32, disp),	213	.ri_s => Immediate.s(@bitCast(i32, data.ri.imm)),
297	.base = ops.reg1,	214	.ri_u => Immediate.u(data.ri.imm),
298	}), reg, emit.code),	215	else => unreachable,
299	.pop => try lowerToRmEnc(.mov, reg, RegisterOrMemory.mem(.qword_ptr, .{	216	};
300	.disp = @bitCast(u32, disp),	217	op1 = .{ .reg = data.ri.r1 };
301	.base = ops.reg1,	218	op2 = .{ .imm = imm };
302	}), emit.code),	219	},
		220	.ri64 => {
		221	const imm64 = emit.mir.extraData(Mir.Imm64, data.rx.payload).data;
		222	op1 = .{ .reg = data.rx.r1 };
		223	op2 = .{ .imm = Immediate.u(Mir.Imm64.decode(imm64)) };
		224	},
		225	.rri_s, .rri_u => {
		226	const imm = switch (ops) {
		227	.rri_s => Immediate.s(@bitCast(i32, data.rri.imm)),
		228	.rri_u => Immediate.u(data.rri.imm),
		229	else => unreachable,
		230	};
		231	op1 = .{ .reg = data.rri.r1 };
		232	op2 = .{ .reg = data.rri.r2 };
		233	op3 = .{ .imm = imm };
		234	},
		235	.m_sib => {
		236	const msib = emit.mir.extraData(Mir.MemorySib, data.payload).data;
		237	op1 = .{ .mem = Mir.MemorySib.decode(msib) };
		238	},
		239	.m_rip => {
		240	const mrip = emit.mir.extraData(Mir.MemoryRip, data.payload).data;
		241	op1 = .{ .mem = Mir.MemoryRip.decode(mrip) };
		242	},
		243	.mi_s_sib, .mi_u_sib => {
		244	const msib = emit.mir.extraData(Mir.MemorySib, data.xi.payload).data;
		245	const imm = switch (ops) {
		246	.mi_s_sib => Immediate.s(@bitCast(i32, data.xi.imm)),
		247	.mi_u_sib => Immediate.u(data.xi.imm),
		248	else => unreachable,
		249	};
		250	op1 = .{ .mem = Mir.MemorySib.decode(msib) };
		251	op2 = .{ .imm = imm };
		252	},
		253	.mi_u_rip, .mi_s_rip => {
		254	const mrip = emit.mir.extraData(Mir.MemoryRip, data.xi.payload).data;
		255	const imm = switch (ops) {
		256	.mi_s_rip => Immediate.s(@bitCast(i32, data.xi.imm)),
		257	.mi_u_rip => Immediate.u(data.xi.imm),
		258	else => unreachable,
		259	};
		260	op1 = .{ .mem = Mir.MemoryRip.decode(mrip) };
		261	op2 = .{ .imm = imm };
		262	},
		263	.rm_sib, .mr_sib => {
		264	const msib = emit.mir.extraData(Mir.MemorySib, data.rx.payload).data;
		265	const op_r = .{ .reg = data.rx.r1 };
		266	const op_m = .{ .mem = Mir.MemorySib.decode(msib) };
		267	switch (ops) {
		268	.rm_sib => {
		269	op1 = op_r;
		270	op2 = op_m;
		271	},
		272	.mr_sib => {
		273	op1 = op_m;
		274	op2 = op_r;
		275	},
303	else => unreachable,	276	else => unreachable,
304	}	277	}
305	disp += 8;
306	}
307	}
308	}
309
310	fn mirJmpCall(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
311	const ops = emit.mir.instructions.items(.ops)[inst].decode();
312	switch (ops.flags) {
313	0b00 => {
314	const target = emit.mir.instructions.items(.data)[inst].inst;
315	const source = emit.code.items.len;
316	try lowerToDEnc(tag, 0, emit.code);
317	try emit.relocs.append(emit.bin_file.allocator, .{
318	.source = source,
319	.target = target,
320	.offset = emit.code.items.len - 4,
321	.length = 5,
322	});
323	},	278	},
324	0b01 => {	279	.rm_rip, .mr_rip => {
325	if (ops.reg1 == .none) {	280	const mrip = emit.mir.extraData(Mir.MemoryRip, data.rx.payload).data;
326	// JMP/CALL [imm]	281	const op_r = .{ .reg = data.rx.r1 };
327	const imm = emit.mir.instructions.items(.data)[inst].imm;	282	const op_m = .{ .mem = Mir.MemoryRip.decode(mrip) };
328	const ptr_size: Memory.PtrSize = switch (immOpSize(imm)) {	283	switch (ops) {
329	16 => .word_ptr,	284	.rm_sib => {
330	else => .qword_ptr,	285	op1 = op_r;
331	};	286	op2 = op_m;
332	return lowerToMEnc(tag, RegisterOrMemory.mem(ptr_size, .{ .disp = imm }), emit.code);	287	},
		288	.mr_sib => {
		289	op1 = op_m;
		290	op2 = op_r;
		291	},
		292	else => unreachable,
333	}	293	}
334	// JMP/CALL reg
335	return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code);
336	},	294	},
337	0b10 => {	295	else => return emit.fail("TODO handle generic encoding: {s}, {s}", .{
338	// JMP/CALL r/m64	296	@tagName(mnemonic),
339	const imm = emit.mir.instructions.items(.data)[inst].imm;	297	@tagName(ops),
340	return lowerToMEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{	298	}),
341	.disp = imm,
342	.base = ops.reg1,
343	}), emit.code);
344	},
345	0b11 => return emit.fail("TODO unused variant jmp/call 0b11", .{}),
346	}	299	}
347	}
348		300
349	fn mirCondJmp(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {	301	return emit.encode(mnemonic, .{
350	const mir_tag = emit.mir.instructions.items(.tag)[inst];	302	.op1 = op1,
351	assert(mir_tag == .cond_jmp);	303	.op2 = op2,
352	const inst_cc = emit.mir.instructions.items(.data)[inst].inst_cc;	304	.op3 = op3,
353	const tag: Tag = switch (inst_cc.cc) {	305	.op4 = op4,
354	.a => .ja,
355	.ae => .jae,
356	.b => .jb,
357	.be => .jbe,
358	.c => .jc,
359	.e => .je,
360	.g => .jg,
361	.ge => .jge,
362	.l => .jl,
363	.le => .jle,
364	.na => .jna,
365	.nae => .jnae,
366	.nb => .jnb,
367	.nbe => .jnbe,
368	.nc => .jnc,
369	.ne => .jne,
370	.ng => .jng,
371	.nge => .jnge,
372	.nl => .jnl,
373	.nle => .jnle,
374	.no => .jno,
375	.np => .jnp,
376	.ns => .jns,
377	.nz => .jnz,
378	.o => .jo,
379	.p => .jp,
380	.pe => .jpe,
381	.po => .jpo,
382	.s => .js,
383	.z => .jz,
384	};
385	const source = emit.code.items.len;
386	try lowerToDEnc(tag, 0, emit.code);
387	try emit.relocs.append(emit.bin_file.allocator, .{
388	.source = source,
389	.target = inst_cc.inst,
390	.offset = emit.code.items.len - 4,
391	.length = 6,
392	});	306	});
393	}	307	}
394		308
395	fn mirCondSetByte(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {	309	fn mirMovMoffs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
396	const mir_tag = emit.mir.instructions.items(.tag)[inst];	310	const ops = emit.mir.instructions.items(.ops)[inst];
397	assert(mir_tag == .cond_set_byte);	311	const payload = emit.mir.instructions.items(.data)[inst].payload;
398	const ops = emit.mir.instructions.items(.ops)[inst].decode();	312	const moffs = emit.mir.extraData(Mir.MemoryMoffs, payload).data;
399	const cc = emit.mir.instructions.items(.data)[inst].cc;	313	const seg = @intToEnum(Register, moffs.seg);
400	const tag: Tag = switch (cc) {	314	const offset = moffs.decodeOffset();
401	.a => .seta,	315	switch (ops) {
402	.ae => .setae,	316	.rax_moffs => {
403	.b => .setb,	317	try emit.encode(.mov, .{
404	.be => .setbe,	318	.op1 = .{ .reg = .rax },
405	.c => .setc,	319	.op2 = .{ .mem = Memory.moffs(seg, offset) },
406	.e => .sete,	320	});
407	.g => .setg,
408	.ge => .setge,
409	.l => .setl,
410	.le => .setle,
411	.na => .setna,
412	.nae => .setnae,
413	.nb => .setnb,
414	.nbe => .setnbe,
415	.nc => .setnc,
416	.ne => .setne,
417	.ng => .setng,
418	.nge => .setnge,
419	.nl => .setnl,
420	.nle => .setnle,
421	.no => .setno,
422	.np => .setnp,
423	.ns => .setns,
424	.nz => .setnz,
425	.o => .seto,
426	.p => .setp,
427	.pe => .setpe,
428	.po => .setpo,
429	.s => .sets,
430	.z => .setz,
431	};
432	return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1.to8()), emit.code);
433	}
434
435	fn mirCondMov(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
436	const mir_tag = emit.mir.instructions.items(.tag)[inst];
437	assert(mir_tag == .cond_mov);
438	const ops = emit.mir.instructions.items(.ops)[inst].decode();
439	const cc = emit.mir.instructions.items(.data)[inst].cc;
440	const tag: Tag = switch (cc) {
441	.a => .cmova,
442	.ae => .cmovae,
443	.b => .cmovb,
444	.be => .cmovbe,
445	.c => .cmovc,
446	.e => .cmove,
447	.g => .cmovg,
448	.ge => .cmovge,
449	.l => .cmovl,
450	.le => .cmovle,
451	.na => .cmovna,
452	.nae => .cmovnae,
453	.nb => .cmovnb,
454	.nbe => .cmovnbe,
455	.nc => .cmovnc,
456	.ne => .cmovne,
457	.ng => .cmovng,
458	.nge => .cmovnge,
459	.nl => .cmovnl,
460	.nle => .cmovnle,
461	.no => .cmovno,
462	.np => .cmovnp,
463	.ns => .cmovns,
464	.nz => .cmovnz,
465	.o => .cmovo,
466	.p => .cmovp,
467	.pe => .cmovpe,
468	.po => .cmovpo,
469	.s => .cmovs,
470	.z => .cmovz,
471	};
472
473	if (ops.flags == 0b00) {
474	return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
475	}
476	const imm = emit.mir.instructions.items(.data)[inst].imm;
477	const ptr_size: Memory.PtrSize = switch (ops.flags) {
478	0b00 => unreachable,
479	0b01 => .word_ptr,
480	0b10 => .dword_ptr,
481	0b11 => .qword_ptr,
482	};
483	return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(ptr_size, .{
484	.disp = imm,
485	.base = ops.reg2,
486	}), emit.code);
487	}
488
489	fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
490	const tag = emit.mir.instructions.items(.tag)[inst];
491	assert(tag == .@"test");
492	const ops = emit.mir.instructions.items(.ops)[inst].decode();
493	switch (ops.flags) {
494	0b00 => {
495	if (ops.reg2 == .none) {
496	// TEST r/m64, imm32
497	// MI
498	const imm = emit.mir.instructions.items(.data)[inst].imm;
499	if (ops.reg1.to64() == .rax) {
500	// TEST rax, imm32
501	// I
502	return lowerToIEnc(.@"test", imm, emit.code);
503	}
504	return lowerToMiEnc(.@"test", RegisterOrMemory.reg(ops.reg1), imm, emit.code);
505	}
506	// TEST r/m64, r64
507	return lowerToMrEnc(.@"test", RegisterOrMemory.reg(ops.reg1), ops.reg2, emit.code);
508	},
509	else => return emit.fail("TODO more TEST alternatives", .{}),
510	}
511	}
512
513	fn mirRet(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
514	const tag = emit.mir.instructions.items(.tag)[inst];
515	assert(tag == .ret);
516	const ops = emit.mir.instructions.items(.ops)[inst].decode();
517	switch (ops.flags) {
518	0b00 => {
519	// RETF imm16
520	// I
521	const imm = emit.mir.instructions.items(.data)[inst].imm;
522	return lowerToIEnc(.ret_far, imm, emit.code);
523	},
524	0b01 => {
525	return lowerToZoEnc(.ret_far, emit.code);
526	},
527	0b10 => {
528	// RET imm16
529	// I
530	const imm = emit.mir.instructions.items(.data)[inst].imm;
531	return lowerToIEnc(.ret_near, imm, emit.code);
532	},	321	},
533	0b11 => {	322	.moffs_rax => {
534	return lowerToZoEnc(.ret_near, emit.code);	323	try emit.encode(.mov, .{
		324	.op1 = .{ .mem = Memory.moffs(seg, offset) },
		325	.op2 = .{ .reg = .rax },
		326	});
535	},	327	},
		328	else => unreachable,
536	}	329	}
537	}	330	}
538		331
539	fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {	332	fn mirMovsx(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
540	const ops = emit.mir.instructions.items(.ops)[inst].decode();	333	const ops = emit.mir.instructions.items(.ops)[inst];
541	switch (ops.flags) {	334	const data = emit.mir.instructions.items(.data)[inst];
542	0b00 => {	335
543	if (ops.reg2 == .none) {	336	var op1: Instruction.Operand = .none;
544	// mov reg1, imm32	337	var op2: Instruction.Operand = .none;
545	// MI	338	switch (ops) {
546	const imm = emit.mir.instructions.items(.data)[inst].imm;	339	.rr => {
547	return lowerToMiEnc(tag, RegisterOrMemory.reg(ops.reg1), imm, emit.code);	340	op1 = .{ .reg = data.rr.r1 };
548	}	341	op2 = .{ .reg = data.rr.r2 };
549	// mov reg1, reg2
550	// RM
551	return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
552	},
553	0b01 => {
554	// mov reg1, [reg2 + imm32]
555	// RM
556	const imm = emit.mir.instructions.items(.data)[inst].imm;
557	const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null;
558	return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
559	.disp = imm,
560	.base = src_reg,
561	}), emit.code);
562	},	342	},
563	0b10 => {	343	.rm_sib => {
564	if (ops.reg2 == .none) {	344	const msib = emit.mir.extraData(Mir.MemorySib, data.rx.payload).data;
565	return emit.fail("TODO unused variant: mov reg1, none, 0b10", .{});	345	op1 = .{ .reg = data.rx.r1 };
566	}	346	op2 = .{ .mem = Mir.MemorySib.decode(msib) };
567	// mov [reg1 + imm32], reg2
568	// MR
569	const imm = emit.mir.instructions.items(.data)[inst].imm;
570	return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg2.size()), .{
571	.disp = imm,
572	.base = ops.reg1,
573	}), ops.reg2, emit.code);
574	},	347	},
575	0b11 => {	348	.rm_rip => {
576	return emit.fail("TODO unused variant: mov reg1, reg2, 0b11", .{});	349	const mrip = emit.mir.extraData(Mir.MemoryRip, data.rx.payload).data;
		350	op1 = .{ .reg = data.rx.r1 };
		351	op2 = .{ .mem = Mir.MemoryRip.decode(mrip) };
577	},	352	},
		353	else => unreachable, // TODO
578	}	354	}
579	}
580
581	fn mirArithMemImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
582	const ops = emit.mir.instructions.items(.ops)[inst].decode();
583	assert(ops.reg2 == .none);
584	const payload = emit.mir.instructions.items(.data)[inst].payload;
585	const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
586	const ptr_size: Memory.PtrSize = switch (ops.flags) {
587	0b00 => .byte_ptr,
588	0b01 => .word_ptr,
589	0b10 => .dword_ptr,
590	0b11 => .qword_ptr,
591	};
592	return lowerToMiEnc(tag, RegisterOrMemory.mem(ptr_size, .{
593	.disp = imm_pair.dest_off,
594	.base = ops.reg1,
595	}), imm_pair.operand, emit.code);
596	}
597
598	inline fn setRexWRegister(reg: Register) bool {
599	if (reg.size() > 64) return false;
600	if (reg.size() == 64) return true;
601	return switch (reg) {
602	.ah, .ch, .dh, .bh => true,
603	else => false,
604	};
605	}
606
607	inline fn immOpSize(u_imm: u32) u6 {
608	const imm = @bitCast(i32, u_imm);
609	if (math.minInt(i8) <= imm and imm <= math.maxInt(i8)) {
610	return 8;
611	}
612	if (math.minInt(i16) <= imm and imm <= math.maxInt(i16)) {
613	return 16;
614	}
615	return 32;
616	}
617		355
618	fn mirArithScaleSrc(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {	356	const mnemonic: Instruction.Mnemonic = switch (op1.bitSize()) {
619	const ops = emit.mir.instructions.items(.ops)[inst].decode();	357	32, 64 => if (op2.bitSize() == 32) .movsxd else .movsx,
620	const scale = ops.flags;	358	else => .movsx,
621	const payload = emit.mir.instructions.items(.data)[inst].payload;
622	const index_reg_disp = emit.mir.extraData(Mir.IndexRegisterDisp, payload).data.decode();
623	// OP reg1, [reg2 + scale*index + imm32]
624	const scale_index = ScaleIndex{
625	.scale = scale,
626	.index = index_reg_disp.index,
627	};
628	return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
629	.disp = index_reg_disp.disp,
630	.base = ops.reg2,
631	.scale_index = scale_index,
632	}), emit.code);
633	}
634
635	fn mirArithScaleDst(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
636	const ops = emit.mir.instructions.items(.ops)[inst].decode();
637	const scale = ops.flags;
638	const payload = emit.mir.instructions.items(.data)[inst].payload;
639	const index_reg_disp = emit.mir.extraData(Mir.IndexRegisterDisp, payload).data.decode();
640	const scale_index = ScaleIndex{
641	.scale = scale,
642	.index = index_reg_disp.index,
643	};	359	};
644	assert(ops.reg2 != .none);
645	// OP [reg1 + scale*index + imm32], reg2
646	return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg2.size()), .{
647	.disp = index_reg_disp.disp,
648	.base = ops.reg1,
649	.scale_index = scale_index,
650	}), ops.reg2, emit.code);
651	}
652		360
653	fn mirArithScaleImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {	361	return emit.encode(mnemonic, .{
654	const ops = emit.mir.instructions.items(.ops)[inst].decode();	362	.op1 = op1,
655	const scale = ops.flags;	363	.op2 = op2,
656	const payload = emit.mir.instructions.items(.data)[inst].payload;	364	});
657	const index_reg_disp_imm = emit.mir.extraData(Mir.IndexRegisterDispImm, payload).data.decode();
658	const scale_index = ScaleIndex{
659	.scale = scale,
660	.index = index_reg_disp_imm.index,
661	};
662	// OP qword ptr [reg1 + scale*index + imm32], imm32
663	return lowerToMiEnc(tag, RegisterOrMemory.mem(.qword_ptr, .{
664	.disp = index_reg_disp_imm.disp,
665	.base = ops.reg1,
666	.scale_index = scale_index,
667	}), index_reg_disp_imm.imm, emit.code);
668	}	365	}
669		366
670	fn mirArithMemIndexImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {	367	fn mnemonicFromConditionCode(comptime basename: []const u8, cc: bits.Condition) Instruction.Mnemonic {
671	const ops = emit.mir.instructions.items(.ops)[inst].decode();	368	inline for (@typeInfo(bits.Condition).Enum.fields) \|field\| {
672	assert(ops.reg2 == .none);	369	if (mem.eql(u8, field.name, @tagName(cc)))
673	const payload = emit.mir.instructions.items(.data)[inst].payload;	370	return @field(Instruction.Mnemonic, basename ++ field.name);
674	const index_reg_disp_imm = emit.mir.extraData(Mir.IndexRegisterDispImm, payload).data.decode();	371	} else unreachable;
675	const ptr_size: Memory.PtrSize = switch (ops.flags) {
676	0b00 => .byte_ptr,
677	0b01 => .word_ptr,
678	0b10 => .dword_ptr,
679	0b11 => .qword_ptr,
680	};
681	const scale_index = ScaleIndex{
682	.scale = 0,
683	.index = index_reg_disp_imm.index,
684	};
685	// OP ptr [reg1 + index + imm32], imm32
686	return lowerToMiEnc(tag, RegisterOrMemory.mem(ptr_size, .{
687	.disp = index_reg_disp_imm.disp,
688	.base = ops.reg1,
689	.scale_index = scale_index,
690	}), index_reg_disp_imm.imm, emit.code);
691	}	372	}
692		373
693	fn mirMovSignExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {	374	fn mirCmovcc(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
694	const mir_tag = emit.mir.instructions.items(.tag)[inst];	375	const ops = emit.mir.instructions.items(.ops)[inst];
695	assert(mir_tag == .mov_sign_extend);	376	switch (ops) {
696	const ops = emit.mir.instructions.items(.ops)[inst].decode();	377	.rr_c => {
697	const imm = if (ops.flags != 0b00) emit.mir.instructions.items(.data)[inst].imm else undefined;	378	const data = emit.mir.instructions.items(.data)[inst].rr_c;
698	switch (ops.flags) {	379	const mnemonic = mnemonicFromConditionCode("cmov", data.cc);
699	0b00 => {	380	return emit.encode(mnemonic, .{
700	const tag: Tag = if (ops.reg2.size() == 32) .movsxd else .movsx;	381	.op1 = .{ .reg = data.r1 },
701	return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);	382	.op2 = .{ .reg = data.r2 },
702	},	383	});
703	0b01 => {
704	return lowerToRmEnc(.movsx, ops.reg1, RegisterOrMemory.mem(.byte_ptr, .{
705	.disp = imm,
706	.base = ops.reg2,
707	}), emit.code);
708	},
709	0b10 => {
710	return lowerToRmEnc(.movsx, ops.reg1, RegisterOrMemory.mem(.word_ptr, .{
711	.disp = imm,
712	.base = ops.reg2,
713	}), emit.code);
714	},
715	0b11 => {
716	return lowerToRmEnc(.movsxd, ops.reg1, RegisterOrMemory.mem(.dword_ptr, .{
717	.disp = imm,
718	.base = ops.reg2,
719	}), emit.code);
720	},	384	},
		385	else => unreachable, // TODO
721	}	386	}
722	}	387	}
723		388
724	fn mirMovZeroExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {	389	fn mirSetcc(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
725	const mir_tag = emit.mir.instructions.items(.tag)[inst];	390	const ops = emit.mir.instructions.items(.ops)[inst];
726	assert(mir_tag == .mov_zero_extend);	391	switch (ops) {
727	const ops = emit.mir.instructions.items(.ops)[inst].decode();	392	.r_c => {
728	const imm = if (ops.flags != 0b00) emit.mir.instructions.items(.data)[inst].imm else undefined;	393	const data = emit.mir.instructions.items(.data)[inst].r_c;
729	switch (ops.flags) {	394	const mnemonic = mnemonicFromConditionCode("set", data.cc);
730	0b00 => {	395	return emit.encode(mnemonic, .{
731	return lowerToRmEnc(.movzx, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);	396	.op1 = .{ .reg = data.r1 },
732	},	397	});
733	0b01 => {
734	return lowerToRmEnc(.movzx, ops.reg1, RegisterOrMemory.mem(.byte_ptr, .{
735	.disp = imm,
736	.base = ops.reg2,
737	}), emit.code);
738	},
739	0b10 => {
740	return lowerToRmEnc(.movzx, ops.reg1, RegisterOrMemory.mem(.word_ptr, .{
741	.disp = imm,
742	.base = ops.reg2,
743	}), emit.code);
744	},
745	0b11 => {
746	return emit.fail("TODO unused variant: movzx 0b11", .{});
747	},	398	},
		399	else => unreachable, // TODO
748	}	400	}
749	}	401	}
750		402
751	fn mirMovabs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {	403	fn mirJcc(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
752	const tag = emit.mir.instructions.items(.tag)[inst];	404	const ops = emit.mir.instructions.items(.ops)[inst];
753	assert(tag == .movabs);	405	switch (ops) {
754	const ops = emit.mir.instructions.items(.ops)[inst].decode();	406	.inst_cc => {
755	switch (ops.flags) {	407	const data = emit.mir.instructions.items(.data)[inst].inst_cc;
756	0b00 => {	408	const mnemonic = mnemonicFromConditionCode("j", data.cc);
757	const imm: u64 = if (ops.reg1.size() == 64) blk: {	409	const source = emit.code.items.len;
758	const payload = emit.mir.instructions.items(.data)[inst].payload;	410	try emit.encode(mnemonic, .{
759	const imm = emit.mir.extraData(Mir.Imm64, payload).data;	411	.op1 = .{ .imm = Immediate.s(0) },
760	break :blk imm.decode();	412	});
761	} else emit.mir.instructions.items(.data)[inst].imm;	413	try emit.relocs.append(emit.bin_file.allocator, .{
762	// movabs reg, imm64	414	.source = source,
763	// OI	415	.target = data.inst,
764	return lowerToOiEnc(.mov, ops.reg1, imm, emit.code);	416	.offset = emit.code.items.len - 4,
765	},	417	.length = 6,
766	0b01 => {	418	});
767	if (ops.reg1 == .none) {
768	const imm: u64 = if (ops.reg2.size() == 64) blk: {
769	const payload = emit.mir.instructions.items(.data)[inst].payload;
770	const imm = emit.mir.extraData(Mir.Imm64, payload).data;
771	break :blk imm.decode();
772	} else emit.mir.instructions.items(.data)[inst].imm;
773	// movabs moffs64, rax
774	// TD
775	return lowerToTdEnc(.mov, imm, ops.reg2, emit.code);
776	}
777	const imm: u64 = if (ops.reg1.size() == 64) blk: {
778	const payload = emit.mir.instructions.items(.data)[inst].payload;
779	const imm = emit.mir.extraData(Mir.Imm64, payload).data;
780	break :blk imm.decode();
781	} else emit.mir.instructions.items(.data)[inst].imm;
782	// movabs rax, moffs64
783	// FD
784	return lowerToFdEnc(.mov, ops.reg1, imm, emit.code);
785	},	419	},
786	else => return emit.fail("TODO unused movabs variant", .{}),	420	else => unreachable, // TODO
787	}	421	}
788	}	422	}
789		423
790	fn mirFisttp(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {	424	fn mirJmpReloc(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
791	const tag = emit.mir.instructions.items(.tag)[inst];	425	const target = emit.mir.instructions.items(.data)[inst].inst;
792	assert(tag == .fisttp);	426	const source = emit.code.items.len;
793	const ops = emit.mir.instructions.items(.ops)[inst].decode();	427	try emit.encode(.jmp, .{
794		428	.op1 = .{ .imm = Immediate.s(0) },
795	// the selecting between operand sizes for this particular `fisttp` instruction	429	});
796	// is done via opcode instead of the usual prefixes.	430	try emit.relocs.append(emit.bin_file.allocator, .{
797		431	.source = source,
798	const opcode: Tag = switch (ops.flags) {	432	.target = target,
799	0b00 => .fisttp16,	433	.offset = emit.code.items.len - 4,
800	0b01 => .fisttp32,	434	.length = 5,
801	0b10 => .fisttp64,	435	});
802	else => unreachable,
803	};
804	const mem_or_reg = Memory{
805	.base = ops.reg1,
806	.disp = emit.mir.instructions.items(.data)[inst].imm,
807	.ptr_size = Memory.PtrSize.dword_ptr, // to prevent any prefix from being used
808	};
809	return lowerToMEnc(opcode, .{ .memory = mem_or_reg }, emit.code);
810	}
811
812	fn mirFld(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
813	const tag = emit.mir.instructions.items(.tag)[inst];
814	assert(tag == .fld);
815	const ops = emit.mir.instructions.items(.ops)[inst].decode();
816
817	// the selecting between operand sizes for this particular `fisttp` instruction
818	// is done via opcode instead of the usual prefixes.
819
820	const opcode: Tag = switch (ops.flags) {
821	0b01 => .fld32,
822	0b10 => .fld64,
823	else => unreachable,
824	};
825	const mem_or_reg = Memory{
826	.base = ops.reg1,
827	.disp = emit.mir.instructions.items(.data)[inst].imm,
828	.ptr_size = Memory.PtrSize.dword_ptr, // to prevent any prefix from being used
829	};
830	return lowerToMEnc(opcode, .{ .memory = mem_or_reg }, emit.code);
831	}	436	}
832		437
833	fn mirShift(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {	438	fn mirCallExtern(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
834	const ops = emit.mir.instructions.items(.ops)[inst].decode();	439	const relocation = emit.mir.instructions.items(.data)[inst].relocation;
835	switch (ops.flags) {
836	0b00 => {
837	// sal reg1, 1
838	// M1
839	return lowerToM1Enc(tag, RegisterOrMemory.reg(ops.reg1), emit.code);
840	},
841	0b01 => {
842	// sal reg1, .cl
843	// MC
844	return lowerToMcEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code);
845	},
846	0b10 => {
847	// sal reg1, imm8
848	// MI
849	const imm = @truncate(u8, emit.mir.instructions.items(.data)[inst].imm);
850	return lowerToMiImm8Enc(tag, RegisterOrMemory.reg(ops.reg1), imm, emit.code);
851	},
852	0b11 => {
853	return emit.fail("TODO unused variant: SHIFT reg1, 0b11", .{});
854	},
855	}
856	}
857		440
858	fn mirMulDiv(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {	441	const offset = blk: {
859	const ops = emit.mir.instructions.items(.ops)[inst].decode();	442	try emit.encode(.call, .{
860	if (ops.reg1 != .none) {	443	.op1 = .{ .imm = Immediate.s(0) },
861	assert(ops.reg2 == .none);	444	});
862	return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code);	445	break :blk @intCast(u32, emit.code.items.len) - 4;
863	}
864	assert(ops.reg2 != .none);
865	const imm = emit.mir.instructions.items(.data)[inst].imm;
866	const ptr_size: Memory.PtrSize = switch (ops.flags) {
867	0b00 => .byte_ptr,
868	0b01 => .word_ptr,
869	0b10 => .dword_ptr,
870	0b11 => .qword_ptr,
871	};	446	};
872	return lowerToMEnc(tag, RegisterOrMemory.mem(ptr_size, .{
873	.disp = imm,
874	.base = ops.reg2,
875	}), emit.code);
876	}
877		447
878	fn mirIMulComplex(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {	448	if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
879	const tag = emit.mir.instructions.items(.tag)[inst];	449	// Add relocation to the decl.
880	assert(tag == .imul_complex);	450	const atom_index = macho_file.getAtomIndexForSymbol(.{ .sym_index = relocation.atom_index, .file = null }).?;
881	const ops = emit.mir.instructions.items(.ops)[inst].decode();	451	const target = macho_file.getGlobalByIndex(relocation.sym_index);
882	switch (ops.flags) {	452	try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{
883	0b00 => {	453	.type = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_BRANCH),
884	return lowerToRmEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);	454	.target = target,
885	},	455	.offset = offset,
886	0b01 => {	456	.addend = 0,
887	const imm = emit.mir.instructions.items(.data)[inst].imm;	457	.pcrel = true,
888	const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null;	458	.length = 2,
889	return lowerToRmEnc(.imul, ops.reg1, RegisterOrMemory.mem(.qword_ptr, .{	459	});
890	.disp = imm,	460	} else if (emit.bin_file.cast(link.File.Coff)) \|coff_file\| {
891	.base = src_reg,	461	// Add relocation to the decl.
892	}), emit.code);	462	const atom_index = coff_file.getAtomIndexForSymbol(.{ .sym_index = relocation.atom_index, .file = null }).?;
893	},	463	const target = coff_file.getGlobalByIndex(relocation.sym_index);
894	0b10 => {	464	try link.File.Coff.Atom.addRelocation(coff_file, atom_index, .{
895	const imm = emit.mir.instructions.items(.data)[inst].imm;	465	.type = .direct,
896	return lowerToRmiEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), imm, emit.code);	466	.target = target,
897	},	467	.offset = offset,
898	0b11 => {	468	.addend = 0,
899	const payload = emit.mir.instructions.items(.data)[inst].payload;	469	.pcrel = true,
900	const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;	470	.length = 2,
901	return lowerToRmiEnc(.imul, ops.reg1, RegisterOrMemory.mem(.qword_ptr, .{	471	});
902	.disp = imm_pair.dest_off,	472	} else {
903	.base = ops.reg2,	473	return emit.fail("TODO implement call_extern for linking backends different than MachO and COFF", .{});
904	}), imm_pair.operand, emit.code);
905	},
906	}	474	}
907	}	475	}
908		476
909	fn mirCwd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {	477	fn mirPushPopRegisterList(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
910	const ops = emit.mir.instructions.items(.ops)[inst].decode();	478	const payload = emit.mir.instructions.items(.data)[inst].payload;
911	const tag: Tag = switch (ops.flags) {	479	const save_reg_list = emit.mir.extraData(Mir.SaveRegisterList, payload).data;
912	0b00 => .cbw,	480	const base = @intToEnum(Register, save_reg_list.base_reg);
913	0b01 => .cwd,	481	var disp: i32 = -@intCast(i32, save_reg_list.stack_end);
914	0b10 => .cdq,	482	const reg_list = Mir.RegisterList.fromInt(save_reg_list.register_list);
915	0b11 => .cqo,	483	const callee_preserved_regs = abi.getCalleePreservedRegs(emit.target.*);
916	};	484	for (callee_preserved_regs) \|reg\| {
917	return lowerToZoEnc(tag, emit.code);	485	if (reg_list.isSet(callee_preserved_regs, reg)) {
918	}	486	const op1: Instruction.Operand = .{ .mem = Memory.sib(.qword, .{
919		487	.base = base,
920	fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {	488	.disp = disp,
921	const tag = emit.mir.instructions.items(.tag)[inst];	489	}) };
922	assert(tag == .lea);	490	const op2: Instruction.Operand = .{ .reg = reg };
923	const ops = emit.mir.instructions.items(.ops)[inst].decode();	491	switch (tag) {
924	switch (ops.flags) {	492	.push => try emit.encode(.mov, .{
925	0b00 => {	493	.op1 = op1,
926	// lea reg1, [reg2 + imm32]	494	.op2 = op2,
927	// RM
928	const imm = emit.mir.instructions.items(.data)[inst].imm;
929	const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null;
930	return lowerToRmEnc(
931	.lea,
932	ops.reg1,
933	RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
934	.disp = imm,
935	.base = src_reg,
936	}),	495	}),
937	emit.code,	496	.pop => try emit.encode(.mov, .{
938	);	497	.op1 = op2,
939	},	498	.op2 = op1,
940	0b01 => {
941	// lea reg1, [rip + imm32]
942	// RM
943	const start_offset = emit.code.items.len;
944	try lowerToRmEnc(
945	.lea,
946	ops.reg1,
947	RegisterOrMemory.rip(Memory.PtrSize.new(ops.reg1.size()), 0),
948	emit.code,
949	);
950	const end_offset = emit.code.items.len;
951	// Backpatch the displacement
952	const payload = emit.mir.instructions.items(.data)[inst].payload;
953	const imm = emit.mir.extraData(Mir.Imm64, payload).data.decode();
954	const disp = @intCast(i32, @intCast(i64, imm) - @intCast(i64, end_offset - start_offset));
955	mem.writeIntLittle(i32, emit.code.items[end_offset - 4 ..][0..4], disp);
956	},
957	0b10 => {
958	// lea reg, [rbp + index + imm32]
959	const payload = emit.mir.instructions.items(.data)[inst].payload;
960	const index_reg_disp = emit.mir.extraData(Mir.IndexRegisterDisp, payload).data.decode();
961	const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null;
962	const scale_index = ScaleIndex{
963	.scale = 0,
964	.index = index_reg_disp.index,
965	};
966	return lowerToRmEnc(
967	.lea,
968	ops.reg1,
969	RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
970	.disp = index_reg_disp.disp,
971	.base = src_reg,
972	.scale_index = scale_index,
973	}),	499	}),
974	emit.code,	500	else => unreachable,
975	);	501	}
976	},	502	disp += 8;
977	0b11 => return emit.fail("TODO unused LEA variant 0b11", .{}),	503	}
978	}	504	}
979	}	505	}
980		506
981	fn mirLeaPic(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {	507	fn mirLeaLinker(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
982	const tag = emit.mir.instructions.items(.tag)[inst];	508	const ops = emit.mir.instructions.items(.ops)[inst];
983	assert(tag == .lea_pic);	509	const payload = emit.mir.instructions.items(.data)[inst].payload;
984	const ops = emit.mir.instructions.items(.ops)[inst].decode();	510	const metadata = emit.mir.extraData(Mir.LeaRegisterReloc, payload).data;
985	const relocation = emit.mir.instructions.items(.data)[inst].relocation;	511	const reg = @intToEnum(Register, metadata.reg);
986
987	switch (ops.flags) {
988	0b00, 0b01, 0b10 => {},
989	else => return emit.fail("TODO unused LEA PIC variant 0b11", .{}),
990	}
991		512
992	// lea reg1, [rip + reloc]	513	try emit.encode(.lea, .{
993	// RM	514	.op1 = .{ .reg = reg },
994	try lowerToRmEnc(	515	.op2 = .{ .mem = Memory.rip(Memory.PtrSize.fromBitSize(reg.bitSize()), 0) },
995	.lea,	516	});
996	ops.reg1,
997	RegisterOrMemory.rip(Memory.PtrSize.new(ops.reg1.size()), 0),
998	emit.code,
999	);
1000		517
1001	const end_offset = emit.code.items.len;	518	const end_offset = emit.code.items.len;
1002		519
1003	if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {	520	if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
1004	const reloc_type = switch (ops.flags) {	521	const reloc_type = switch (ops) {
1005	0b00 => @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_GOT),	522	.got_reloc => @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_GOT),
1006	0b01 => @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_SIGNED),	523	.direct_reloc => @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_SIGNED),
1007	else => unreachable,	524	else => unreachable,
1008	};	525	};
1009	const atom_index = macho_file.getAtomIndexForSymbol(.{ .sym_index = relocation.atom_index, .file = null }).?;	526	const atom_index = macho_file.getAtomIndexForSymbol(.{
		527	.sym_index = metadata.atom_index,
		528	.file = null,
		529	}).?;
1010	try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{	530	try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{
1011	.type = reloc_type,	531	.type = reloc_type,
1012	.target = .{ .sym_index = relocation.sym_index, .file = null },	532	.target = .{ .sym_index = metadata.sym_index, .file = null },
1013	.offset = @intCast(u32, end_offset - 4),	533	.offset = @intCast(u32, end_offset - 4),
1014	.addend = 0,	534	.addend = 0,
1015	.pcrel = true,	535	.pcrel = true,
1016	.length = 2,	536	.length = 2,
1017	});	537	});
1018	} else if (emit.bin_file.cast(link.File.Coff)) \|coff_file\| {	538	} else if (emit.bin_file.cast(link.File.Coff)) \|coff_file\| {
1019	const atom_index = coff_file.getAtomIndexForSymbol(.{ .sym_index = relocation.atom_index, .file = null }).?;	539	const atom_index = coff_file.getAtomIndexForSymbol(.{
		540	.sym_index = metadata.atom_index,
		541	.file = null,
		542	}).?;
1020	try link.File.Coff.Atom.addRelocation(coff_file, atom_index, .{	543	try link.File.Coff.Atom.addRelocation(coff_file, atom_index, .{
1021	.type = switch (ops.flags) {	544	.type = switch (ops) {
1022	0b00 => .got,	545	.got_reloc => .got,
1023	0b01 => .direct,	546	.direct_reloc => .direct,
1024	0b10 => .import,	547	.import_reloc => .import,
1025	else => unreachable,	548	else => unreachable,
1026	},	549	},
1027	.target = switch (ops.flags) {	550	.target = switch (ops) {
1028	0b00, 0b01 => .{ .sym_index = relocation.sym_index, .file = null },	551	.got_reloc, .direct_reloc => .{ .sym_index = metadata.sym_index, .file = null },
1029	0b10 => coff_file.getGlobalByIndex(relocation.sym_index),	552	.import_reloc => coff_file.getGlobalByIndex(metadata.sym_index),
1030	else => unreachable,	553	else => unreachable,
1031	},	554	},
1032	.offset = @intCast(u32, end_offset - 4),	555	.offset = @intCast(u32, end_offset - 4),
...	@@ -1039,158 +562,23 @@ fn mirLeaPic(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {	...	@@ -1039,158 +562,23 @@ fn mirLeaPic(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1039	}	562	}
1040	}	563	}
1041		564
1042	// SSE instructions	565	fn mirDbgLine(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1043		566	const payload = emit.mir.instructions.items(.data)[inst].payload;
1044	fn mirMovFloatSse(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {	567	const dbg_line_column = emit.mir.extraData(Mir.DbgLineColumn, payload).data;
1045	const ops = emit.mir.instructions.items(.ops)[inst].decode();	568	log.debug("mirDbgLine", .{});
1046	switch (ops.flags) {	569	try emit.dbgAdvancePCAndLine(dbg_line_column.line, dbg_line_column.column);
1047	0b00 => {
1048	const imm = emit.mir.instructions.items(.data)[inst].imm;
1049	return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg2.size()), .{
1050	.disp = imm,
1051	.base = ops.reg2,
1052	}), emit.code);
1053	},
1054	0b01 => {
1055	const imm = emit.mir.instructions.items(.data)[inst].imm;
1056	return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
1057	.disp = imm,
1058	.base = ops.reg1,
1059	}), ops.reg2, emit.code);
1060	},
1061	0b10 => {
1062	return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
1063	},
1064	else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, tag }),
1065	}
1066	}	570	}
1067		571
1068	fn mirAddFloatSse(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {	572	fn dbgAdvancePCAndLine(emit: *Emit, line: u32, column: u32) InnerError!void {
1069	const ops = emit.mir.instructions.items(.ops)[inst].decode();	573	const delta_line = @intCast(i32, line) - @intCast(i32, emit.prev_di_line);
1070	switch (ops.flags) {	574	const delta_pc: usize = emit.code.items.len - emit.prev_di_pc;
1071	0b00 => {	575	log.debug(" (advance pc={d} and line={d})", .{ delta_line, delta_pc });
1072	return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);	576	switch (emit.debug_output) {
1073	},	577	.dwarf => \|dw\| {
1074	else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, tag }),	578	try dw.advancePCAndLine(delta_line, delta_pc);
1075	}	579	emit.prev_di_line = line;
1076	}	580	emit.prev_di_column = column;
1077		581	emit.prev_di_pc = emit.code.items.len;
1078	fn mirCmpFloatSse(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
1079	const ops = emit.mir.instructions.items(.ops)[inst].decode();
1080	switch (ops.flags) {
1081	0b00 => {
1082	return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
1083	},
1084	else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, tag }),
1085	}
1086	}
1087	// AVX instructions
1088
1089	fn mirMovFloatAvx(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
1090	const ops = emit.mir.instructions.items(.ops)[inst].decode();
1091	switch (ops.flags) {
1092	0b00 => {
1093	const imm = emit.mir.instructions.items(.data)[inst].imm;
1094	return lowerToVmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg2.size()), .{
1095	.disp = imm,
1096	.base = ops.reg2,
1097	}), emit.code);
1098	},
1099	0b01 => {
1100	const imm = emit.mir.instructions.items(.data)[inst].imm;
1101	return lowerToMvEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{
1102	.disp = imm,
1103	.base = ops.reg1,
1104	}), ops.reg2, emit.code);
1105	},
1106	0b10 => {
1107	return lowerToRvmEnc(tag, ops.reg1, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
1108	},
1109	else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, tag }),
1110	}
1111	}
1112
1113	fn mirAddFloatAvx(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
1114	const ops = emit.mir.instructions.items(.ops)[inst].decode();
1115	switch (ops.flags) {
1116	0b00 => {
1117	return lowerToRvmEnc(tag, ops.reg1, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
1118	},
1119	else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, tag }),
1120	}
1121	}
1122
1123	fn mirCmpFloatAvx(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
1124	const ops = emit.mir.instructions.items(.ops)[inst].decode();
1125	switch (ops.flags) {
1126	0b00 => {
1127	return lowerToVmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
1128	},
1129	else => return emit.fail("TODO unused variant 0b{b} for mov_f64", .{ops.flags}),
1130	}
1131	}
1132
1133	// Pseudo-instructions
1134
1135	fn mirCallExtern(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1136	const tag = emit.mir.instructions.items(.tag)[inst];
1137	assert(tag == .call_extern);
1138	const relocation = emit.mir.instructions.items(.data)[inst].relocation;
1139
1140	const offset = blk: {
1141	// callq
1142	try lowerToDEnc(.call_near, 0, emit.code);
1143	break :blk @intCast(u32, emit.code.items.len) - 4;
1144	};
1145
1146	if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
1147	// Add relocation to the decl.
1148	const atom_index = macho_file.getAtomIndexForSymbol(.{ .sym_index = relocation.atom_index, .file = null }).?;
1149	const target = macho_file.getGlobalByIndex(relocation.sym_index);
1150	try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{
1151	.type = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_BRANCH),
1152	.target = target,
1153	.offset = offset,
1154	.addend = 0,
1155	.pcrel = true,
1156	.length = 2,
1157	});
1158	} else if (emit.bin_file.cast(link.File.Coff)) \|coff_file\| {
1159	// Add relocation to the decl.
1160	const atom_index = coff_file.getAtomIndexForSymbol(.{ .sym_index = relocation.atom_index, .file = null }).?;
1161	const target = coff_file.getGlobalByIndex(relocation.sym_index);
1162	try link.File.Coff.Atom.addRelocation(coff_file, atom_index, .{
1163	.type = .direct,
1164	.target = target,
1165	.offset = offset,
1166	.addend = 0,
1167	.pcrel = true,
1168	.length = 2,
1169	});
1170	} else {
1171	return emit.fail("TODO implement call_extern for linking backends different than MachO and COFF", .{});
1172	}
1173	}
1174
1175	fn mirDbgLine(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1176	const tag = emit.mir.instructions.items(.tag)[inst];
1177	assert(tag == .dbg_line);
1178	const payload = emit.mir.instructions.items(.data)[inst].payload;
1179	const dbg_line_column = emit.mir.extraData(Mir.DbgLineColumn, payload).data;
1180	log.debug("mirDbgLine", .{});
1181	try emit.dbgAdvancePCAndLine(dbg_line_column.line, dbg_line_column.column);
1182	}
1183
1184	fn dbgAdvancePCAndLine(emit: *Emit, line: u32, column: u32) InnerError!void {
1185	const delta_line = @intCast(i32, line) - @intCast(i32, emit.prev_di_line);
1186	const delta_pc: usize = emit.code.items.len - emit.prev_di_pc;
1187	log.debug(" (advance pc={d} and line={d})", .{ delta_line, delta_pc });
1188	switch (emit.debug_output) {
1189	.dwarf => \|dw\| {
1190	try dw.advancePCAndLine(delta_line, delta_pc);
1191	emit.prev_di_line = line;
1192	emit.prev_di_column = column;
1193	emit.prev_di_pc = emit.code.items.len;
1194	},	582	},
1195	.plan9 => \|dbg_out\| {	583	.plan9 => \|dbg_out\| {
1196	if (delta_pc <= 0) return; // only do this when the pc changes	584	if (delta_pc <= 0) return; // only do this when the pc changes
...	@@ -1232,8 +620,7 @@ fn dbgAdvancePCAndLine(emit: *Emit, line: u32, column: u32) InnerError!void {	...	@@ -1232,8 +620,7 @@ fn dbgAdvancePCAndLine(emit: *Emit, line: u32, column: u32) InnerError!void {
1232	}	620	}
1233		621
1234	fn mirDbgPrologueEnd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {	622	fn mirDbgPrologueEnd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1235	const tag = emit.mir.instructions.items(.tag)[inst];	623	_ = inst;
1236	assert(tag == .dbg_prologue_end);
1237	switch (emit.debug_output) {	624	switch (emit.debug_output) {
1238	.dwarf => \|dw\| {	625	.dwarf => \|dw\| {
1239	try dw.setPrologueEnd();	626	try dw.setPrologueEnd();
...	@@ -1249,8 +636,7 @@ fn mirDbgPrologueEnd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {	...	@@ -1249,8 +636,7 @@ fn mirDbgPrologueEnd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1249	}	636	}
1250		637
1251	fn mirDbgEpilogueBegin(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {	638	fn mirDbgEpilogueBegin(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1252	const tag = emit.mir.instructions.items(.tag)[inst];	639	_ = inst;
1253	assert(tag == .dbg_epilogue_begin);
1254	switch (emit.debug_output) {	640	switch (emit.debug_output) {
1255	.dwarf => \|dw\| {	641	.dwarf => \|dw\| {
1256	try dw.setEpilogueBegin();	642	try dw.setEpilogueBegin();
...	@@ -1264,1841 +650,3 @@ fn mirDbgEpilogueBegin(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {	...	@@ -1264,1841 +650,3 @@ fn mirDbgEpilogueBegin(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
1264	.none => {},	650	.none => {},
1265	}	651	}
1266	}	652	}
1267
1268	const Tag = enum {
1269	adc,
1270	add,
1271	sub,
1272	xor,
1273	@"and",
1274	@"or",
1275	sbb,
1276	cmp,
1277	mov,
1278	movsx,
1279	movsxd,
1280	movzx,
1281	lea,
1282	jmp_near,
1283	call_near,
1284	push,
1285	pop,
1286	@"test",
1287	ud2,
1288	int3,
1289	nop,
1290	imul,
1291	mul,
1292	idiv,
1293	div,
1294	syscall,
1295	ret_near,
1296	ret_far,
1297	fisttp16,
1298	fisttp32,
1299	fisttp64,
1300	fld32,
1301	fld64,
1302	jo,
1303	jno,
1304	jb,
1305	jbe,
1306	jc,
1307	jnae,
1308	jnc,
1309	jae,
1310	je,
1311	jz,
1312	jne,
1313	jnz,
1314	jna,
1315	jnb,
1316	jnbe,
1317	ja,
1318	js,
1319	jns,
1320	jpe,
1321	jp,
1322	jpo,
1323	jnp,
1324	jnge,
1325	jl,
1326	jge,
1327	jnl,
1328	jle,
1329	jng,
1330	jg,
1331	jnle,
1332	seto,
1333	setno,
1334	setb,
1335	setc,
1336	setnae,
1337	setnb,
1338	setnc,
1339	setae,
1340	sete,
1341	setz,
1342	setne,
1343	setnz,
1344	setbe,
1345	setna,
1346	seta,
1347	setnbe,
1348	sets,
1349	setns,
1350	setp,
1351	setpe,
1352	setnp,
1353	setpo,
1354	setl,
1355	setnge,
1356	setnl,
1357	setge,
1358	setle,
1359	setng,
1360	setnle,
1361	setg,
1362	cmovo,
1363	cmovno,
1364	cmovb,
1365	cmovc,
1366	cmovnae,
1367	cmovnb,
1368	cmovnc,
1369	cmovae,
1370	cmove,
1371	cmovz,
1372	cmovne,
1373	cmovnz,
1374	cmovbe,
1375	cmovna,
1376	cmova,
1377	cmovnbe,
1378	cmovs,
1379	cmovns,
1380	cmovp,
1381	cmovpe,
1382	cmovnp,
1383	cmovpo,
1384	cmovl,
1385	cmovnge,
1386	cmovnl,
1387	cmovge,
1388	cmovle,
1389	cmovng,
1390	cmovnle,
1391	cmovg,
1392	shl,
1393	sal,
1394	shr,
1395	sar,
1396	cbw,
1397	cwd,
1398	cdq,
1399	cqo,
1400	movsd,
1401	movss,
1402	addsd,
1403	addss,
1404	cmpsd,
1405	cmpss,
1406	ucomisd,
1407	ucomiss,
1408	vmovsd,
1409	vmovss,
1410	vaddsd,
1411	vaddss,
1412	vcmpsd,
1413	vcmpss,
1414	vucomisd,
1415	vucomiss,
1416
1417	fn isSse(tag: Tag) bool {
1418	return switch (tag) {
1419	.movsd,
1420	.movss,
1421	.addsd,
1422	.addss,
1423	.cmpsd,
1424	.cmpss,
1425	.ucomisd,
1426	.ucomiss,
1427	=> true,
1428
1429	else => false,
1430	};
1431	}
1432
1433	fn isAvx(tag: Tag) bool {
1434	return switch (tag) {
1435	.vmovsd,
1436	.vmovss,
1437	.vaddsd,
1438	.vaddss,
1439	.vcmpsd,
1440	.vcmpss,
1441	.vucomisd,
1442	.vucomiss,
1443	=> true,
1444
1445	else => false,
1446	};
1447	}
1448
1449	fn isSetCC(tag: Tag) bool {
1450	return switch (tag) {
1451	.seto,
1452	.setno,
1453	.setb,
1454	.setc,
1455	.setnae,
1456	.setnb,
1457	.setnc,
1458	.setae,
1459	.sete,
1460	.setz,
1461	.setne,
1462	.setnz,
1463	.setbe,
1464	.setna,
1465	.seta,
1466	.setnbe,
1467	.sets,
1468	.setns,
1469	.setp,
1470	.setpe,
1471	.setnp,
1472	.setpo,
1473	.setl,
1474	.setnge,
1475	.setnl,
1476	.setge,
1477	.setle,
1478	.setng,
1479	.setnle,
1480	.setg,
1481	=> true,
1482	else => false,
1483	};
1484	}
1485	};
1486
1487	const Encoding = enum {
1488	/// OP
1489	zo,
1490
1491	/// OP rel32
1492	d,
1493
1494	/// OP r/m64
1495	m,
1496
1497	/// OP r64
1498	o,
1499
1500	/// OP imm32
1501	i,
1502
1503	/// OP r/m64, 1
1504	m1,
1505
1506	/// OP r/m64, .cl
1507	mc,
1508
1509	/// OP r/m64, imm32
1510	mi,
1511
1512	/// OP r/m64, imm8
1513	mi8,
1514
1515	/// OP r/m64, r64
1516	mr,
1517
1518	/// OP r64, r/m64
1519	rm,
1520
1521	/// OP r64, imm64
1522	oi,
1523
1524	/// OP al/ax/eax/rax, moffs
1525	fd,
1526
1527	/// OP moffs, al/ax/eax/rax
1528	td,
1529
1530	/// OP r64, r/m64, imm32
1531	rmi,
1532
1533	/// OP xmm1, xmm2/m64
1534	vm,
1535
1536	/// OP m64, xmm1
1537	mv,
1538
1539	/// OP xmm1, xmm2, xmm3/m64
1540	rvm,
1541
1542	/// OP xmm1, xmm2, xmm3/m64, imm8
1543	rvmi,
1544	};
1545
1546	const OpCode = struct {
1547	bytes: [3]u8,
1548	count: usize,
1549
1550	fn init(comptime in_bytes: []const u8) OpCode {
1551	comptime assert(in_bytes.len <= 3);
1552	comptime var bytes: [3]u8 = undefined;
1553	inline for (in_bytes, 0..) \|x, i\| {
1554	bytes[i] = x;
1555	}
1556	return .{ .bytes = bytes, .count = in_bytes.len };
1557	}
1558
1559	fn encode(opc: OpCode, encoder: Encoder) void {
1560	switch (opc.count) {
1561	1 => encoder.opcode_1byte(opc.bytes[0]),
1562	2 => encoder.opcode_2byte(opc.bytes[0], opc.bytes[1]),
1563	3 => encoder.opcode_3byte(opc.bytes[0], opc.bytes[1], opc.bytes[2]),
1564	else => unreachable,
1565	}
1566	}
1567
1568	fn encodeWithReg(opc: OpCode, encoder: Encoder, reg: Register) void {
1569	assert(opc.count == 1);
1570	encoder.opcode_withReg(opc.bytes[0], reg.lowEnc());
1571	}
1572	};
1573
1574	inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) OpCode {
1575	// zig fmt: off
1576	switch (enc) {
1577	.zo => return switch (tag) {
1578	.ret_near => OpCode.init(&.{0xc3}),
1579	.ret_far => OpCode.init(&.{0xcb}),
1580	.ud2 => OpCode.init(&.{ 0x0F, 0x0B }),
1581	.int3 => OpCode.init(&.{0xcc}),
1582	.nop => OpCode.init(&.{0x90}),
1583	.syscall => OpCode.init(&.{ 0x0f, 0x05 }),
1584	.cbw => OpCode.init(&.{0x98}),
1585	.cwd,
1586	.cdq,
1587	.cqo => OpCode.init(&.{0x99}),
1588	else => unreachable,
1589	},
1590	.d => return switch (tag) {
1591	.jmp_near => OpCode.init(&.{0xe9}),
1592	.call_near => OpCode.init(&.{0xe8}),
1593
1594	.jo => if (is_one_byte) OpCode.init(&.{0x70}) else OpCode.init(&.{0x0f,0x80}),
1595
1596	.jno => if (is_one_byte) OpCode.init(&.{0x71}) else OpCode.init(&.{0x0f,0x81}),
1597
1598	.jb,
1599	.jc,
1600	.jnae => if (is_one_byte) OpCode.init(&.{0x72}) else OpCode.init(&.{0x0f,0x82}),
1601
1602	.jnb,
1603	.jnc,
1604	.jae => if (is_one_byte) OpCode.init(&.{0x73}) else OpCode.init(&.{0x0f,0x83}),
1605
1606	.je,
1607	.jz => if (is_one_byte) OpCode.init(&.{0x74}) else OpCode.init(&.{0x0f,0x84}),
1608
1609	.jne,
1610	.jnz => if (is_one_byte) OpCode.init(&.{0x75}) else OpCode.init(&.{0x0f,0x85}),
1611
1612	.jna,
1613	.jbe => if (is_one_byte) OpCode.init(&.{0x76}) else OpCode.init(&.{0x0f,0x86}),
1614
1615	.jnbe,
1616	.ja => if (is_one_byte) OpCode.init(&.{0x77}) else OpCode.init(&.{0x0f,0x87}),
1617
1618	.js => if (is_one_byte) OpCode.init(&.{0x78}) else OpCode.init(&.{0x0f,0x88}),
1619
1620	.jns => if (is_one_byte) OpCode.init(&.{0x79}) else OpCode.init(&.{0x0f,0x89}),
1621
1622	.jpe,
1623	.jp => if (is_one_byte) OpCode.init(&.{0x7a}) else OpCode.init(&.{0x0f,0x8a}),
1624
1625	.jpo,
1626	.jnp => if (is_one_byte) OpCode.init(&.{0x7b}) else OpCode.init(&.{0x0f,0x8b}),
1627
1628	.jnge,
1629	.jl => if (is_one_byte) OpCode.init(&.{0x7c}) else OpCode.init(&.{0x0f,0x8c}),
1630
1631	.jge,
1632	.jnl => if (is_one_byte) OpCode.init(&.{0x7d}) else OpCode.init(&.{0x0f,0x8d}),
1633
1634	.jle,
1635	.jng => if (is_one_byte) OpCode.init(&.{0x7e}) else OpCode.init(&.{0x0f,0x8e}),
1636
1637	.jg,
1638	.jnle => if (is_one_byte) OpCode.init(&.{0x7f}) else OpCode.init(&.{0x0f,0x8f}),
1639
1640	else => unreachable,
1641	},
1642	.m => return switch (tag) {
1643	.jmp_near,
1644	.call_near,
1645	.push => OpCode.init(&.{0xff}),
1646
1647	.pop => OpCode.init(&.{0x8f}),
1648	.seto => OpCode.init(&.{0x0f,0x90}),
1649	.setno => OpCode.init(&.{0x0f,0x91}),
1650
1651	.setb,
1652	.setc,
1653	.setnae => OpCode.init(&.{0x0f,0x92}),
1654
1655	.setnb,
1656	.setnc,
1657	.setae => OpCode.init(&.{0x0f,0x93}),
1658
1659	.sete,
1660	.setz => OpCode.init(&.{0x0f,0x94}),
1661
1662	.setne,
1663	.setnz => OpCode.init(&.{0x0f,0x95}),
1664
1665	.setbe,
1666	.setna => OpCode.init(&.{0x0f,0x96}),
1667
1668	.seta,
1669	.setnbe => OpCode.init(&.{0x0f,0x97}),
1670
1671	.sets => OpCode.init(&.{0x0f,0x98}),
1672	.setns => OpCode.init(&.{0x0f,0x99}),
1673
1674	.setp,
1675	.setpe => OpCode.init(&.{0x0f,0x9a}),
1676
1677	.setnp,
1678	.setpo => OpCode.init(&.{0x0f,0x9b}),
1679
1680	.setl,
1681	.setnge => OpCode.init(&.{0x0f,0x9c}),
1682
1683	.setnl,
1684	.setge => OpCode.init(&.{0x0f,0x9d}),
1685
1686	.setle,
1687	.setng => OpCode.init(&.{0x0f,0x9e}),
1688
1689	.setnle,
1690	.setg => OpCode.init(&.{0x0f,0x9f}),
1691
1692	.idiv,
1693	.div,
1694	.imul,
1695	.mul => if (is_one_byte) OpCode.init(&.{0xf6}) else OpCode.init(&.{0xf7}),
1696
1697	.fisttp16 => OpCode.init(&.{0xdf}),
1698	.fisttp32 => OpCode.init(&.{0xdb}),
1699	.fisttp64 => OpCode.init(&.{0xdd}),
1700	.fld32 => OpCode.init(&.{0xd9}),
1701	.fld64 => OpCode.init(&.{0xdd}),
1702	else => unreachable,
1703	},
1704	.o => return switch (tag) {
1705	.push => OpCode.init(&.{0x50}),
1706	.pop => OpCode.init(&.{0x58}),
1707	else => unreachable,
1708	},
1709	.i => return switch (tag) {
1710	.push => if (is_one_byte) OpCode.init(&.{0x6a}) else OpCode.init(&.{0x68}),
1711	.@"test" => if (is_one_byte) OpCode.init(&.{0xa8}) else OpCode.init(&.{0xa9}),
1712	.ret_near => OpCode.init(&.{0xc2}),
1713	.ret_far => OpCode.init(&.{0xca}),
1714	else => unreachable,
1715	},
1716	.m1 => return switch (tag) {
1717	.shl, .sal,
1718	.shr, .sar => if (is_one_byte) OpCode.init(&.{0xd0}) else OpCode.init(&.{0xd1}),
1719	else => unreachable,
1720	},
1721	.mc => return switch (tag) {
1722	.shl, .sal,
1723	.shr, .sar => if (is_one_byte) OpCode.init(&.{0xd2}) else OpCode.init(&.{0xd3}),
1724	else => unreachable,
1725	},
1726	.mi => return switch (tag) {
1727	.adc, .add,
1728	.sub, .xor,
1729	.@"and", .@"or",
1730	.sbb, .cmp => if (is_one_byte) OpCode.init(&.{0x80}) else OpCode.init(&.{0x81}),
1731	.mov => if (is_one_byte) OpCode.init(&.{0xc6}) else OpCode.init(&.{0xc7}),
1732	.@"test" => if (is_one_byte) OpCode.init(&.{0xf6}) else OpCode.init(&.{0xf7}),
1733	else => unreachable,
1734	},
1735	.mi8 => return switch (tag) {
1736	.adc, .add,
1737	.sub, .xor,
1738	.@"and", .@"or",
1739	.sbb, .cmp => OpCode.init(&.{0x83}),
1740	.shl, .sal,
1741	.shr, .sar => if (is_one_byte) OpCode.init(&.{0xc0}) else OpCode.init(&.{0xc1}),
1742	else => unreachable,
1743	},
1744	.mr => return switch (tag) {
1745	.adc => if (is_one_byte) OpCode.init(&.{0x10}) else OpCode.init(&.{0x11}),
1746	.add => if (is_one_byte) OpCode.init(&.{0x00}) else OpCode.init(&.{0x01}),
1747	.sub => if (is_one_byte) OpCode.init(&.{0x28}) else OpCode.init(&.{0x29}),
1748	.xor => if (is_one_byte) OpCode.init(&.{0x30}) else OpCode.init(&.{0x31}),
1749	.@"and" => if (is_one_byte) OpCode.init(&.{0x20}) else OpCode.init(&.{0x21}),
1750	.@"or" => if (is_one_byte) OpCode.init(&.{0x08}) else OpCode.init(&.{0x09}),
1751	.sbb => if (is_one_byte) OpCode.init(&.{0x18}) else OpCode.init(&.{0x19}),
1752	.cmp => if (is_one_byte) OpCode.init(&.{0x38}) else OpCode.init(&.{0x39}),
1753	.mov => if (is_one_byte) OpCode.init(&.{0x88}) else OpCode.init(&.{0x89}),
1754	.@"test" => if (is_one_byte) OpCode.init(&.{0x84}) else OpCode.init(&.{0x85}),
1755	.movsd => OpCode.init(&.{0xf2,0x0f,0x11}),
1756	.movss => OpCode.init(&.{0xf3,0x0f,0x11}),
1757	else => unreachable,
1758	},
1759	.rm => return switch (tag) {
1760	.adc => if (is_one_byte) OpCode.init(&.{0x12}) else OpCode.init(&.{0x13}),
1761	.add => if (is_one_byte) OpCode.init(&.{0x02}) else OpCode.init(&.{0x03}),
1762	.sub => if (is_one_byte) OpCode.init(&.{0x2a}) else OpCode.init(&.{0x2b}),
1763	.xor => if (is_one_byte) OpCode.init(&.{0x32}) else OpCode.init(&.{0x33}),
1764	.@"and" => if (is_one_byte) OpCode.init(&.{0x22}) else OpCode.init(&.{0x23}),
1765	.@"or" => if (is_one_byte) OpCode.init(&.{0x0a}) else OpCode.init(&.{0x0b}),
1766	.sbb => if (is_one_byte) OpCode.init(&.{0x1a}) else OpCode.init(&.{0x1b}),
1767	.cmp => if (is_one_byte) OpCode.init(&.{0x3a}) else OpCode.init(&.{0x3b}),
1768	.mov => if (is_one_byte) OpCode.init(&.{0x8a}) else OpCode.init(&.{0x8b}),
1769	.movsx => if (is_one_byte) OpCode.init(&.{0x0f,0xbe}) else OpCode.init(&.{0x0f,0xbf}),
1770	.movsxd => OpCode.init(&.{0x63}),
1771	.movzx => if (is_one_byte) OpCode.init(&.{0x0f,0xb6}) else OpCode.init(&.{0x0f,0xb7}),
1772	.lea => if (is_one_byte) OpCode.init(&.{0x8c}) else OpCode.init(&.{0x8d}),
1773	.imul => OpCode.init(&.{0x0f,0xaf}),
1774
1775	.cmova,
1776	.cmovnbe, => OpCode.init(&.{0x0f,0x47}),
1777
1778	.cmovae,
1779	.cmovnb, => OpCode.init(&.{0x0f,0x43}),
1780
1781	.cmovb,
1782	.cmovc,
1783	.cmovnae => OpCode.init(&.{0x0f,0x42}),
1784
1785	.cmovbe,
1786	.cmovna, => OpCode.init(&.{0x0f,0x46}),
1787
1788	.cmove,
1789	.cmovz, => OpCode.init(&.{0x0f,0x44}),
1790
1791	.cmovg,
1792	.cmovnle, => OpCode.init(&.{0x0f,0x4f}),
1793
1794	.cmovge,
1795	.cmovnl, => OpCode.init(&.{0x0f,0x4d}),
1796
1797	.cmovl,
1798	.cmovnge, => OpCode.init(&.{0x0f,0x4c}),
1799
1800	.cmovle,
1801	.cmovng, => OpCode.init(&.{0x0f,0x4e}),
1802
1803	.cmovne,
1804	.cmovnz, => OpCode.init(&.{0x0f,0x45}),
1805
1806	.cmovno => OpCode.init(&.{0x0f,0x41}),
1807
1808	.cmovnp,
1809	.cmovpo, => OpCode.init(&.{0x0f,0x4b}),
1810
1811	.cmovns => OpCode.init(&.{0x0f,0x49}),
1812
1813	.cmovo => OpCode.init(&.{0x0f,0x40}),
1814
1815	.cmovp,
1816	.cmovpe, => OpCode.init(&.{0x0f,0x4a}),
1817
1818	.cmovs => OpCode.init(&.{0x0f,0x48}),
1819
1820	.movsd => OpCode.init(&.{0xf2,0x0f,0x10}),
1821	.movss => OpCode.init(&.{0xf3,0x0f,0x10}),
1822	.addsd => OpCode.init(&.{0xf2,0x0f,0x58}),
1823	.addss => OpCode.init(&.{0xf3,0x0f,0x58}),
1824	.ucomisd => OpCode.init(&.{0x66,0x0f,0x2e}),
1825	.ucomiss => OpCode.init(&.{0x0f,0x2e}),
1826	else => unreachable,
1827	},
1828	.oi => return switch (tag) {
1829	.mov => if (is_one_byte) OpCode.init(&.{0xb0}) else OpCode.init(&.{0xb8}),
1830	else => unreachable,
1831	},
1832	.fd => return switch (tag) {
1833	.mov => if (is_one_byte) OpCode.init(&.{0xa0}) else OpCode.init(&.{0xa1}),
1834	else => unreachable,
1835	},
1836	.td => return switch (tag) {
1837	.mov => if (is_one_byte) OpCode.init(&.{0xa2}) else OpCode.init(&.{0xa3}),
1838	else => unreachable,
1839	},
1840	.rmi => return switch (tag) {
1841	.imul => if (is_one_byte) OpCode.init(&.{0x6b}) else OpCode.init(&.{0x69}),
1842	else => unreachable,
1843	},
1844	.mv => return switch (tag) {
1845	.vmovsd,
1846	.vmovss => OpCode.init(&.{0x11}),
1847	else => unreachable,
1848	},
1849	.vm => return switch (tag) {
1850	.vmovsd,
1851	.vmovss => OpCode.init(&.{0x10}),
1852	.vucomisd,
1853	.vucomiss => OpCode.init(&.{0x2e}),
1854	else => unreachable,
1855	},
1856	.rvm => return switch (tag) {
1857	.vaddsd,
1858	.vaddss => OpCode.init(&.{0x58}),
1859	.vmovsd,
1860	.vmovss => OpCode.init(&.{0x10}),
1861	else => unreachable,
1862	},
1863	.rvmi => return switch (tag) {
1864	.vcmpsd,
1865	.vcmpss => OpCode.init(&.{0xc2}),
1866	else => unreachable,
1867	},
1868	}
1869	// zig fmt: on
1870	}
1871
1872	inline fn getModRmExt(tag: Tag) u3 {
1873	return switch (tag) {
1874	.adc => 0x2,
1875	.add => 0x0,
1876	.sub => 0x5,
1877	.xor => 0x6,
1878	.@"and" => 0x4,
1879	.@"or" => 0x1,
1880	.sbb => 0x3,
1881	.cmp => 0x7,
1882	.mov => 0x0,
1883	.jmp_near => 0x4,
1884	.call_near => 0x2,
1885	.push => 0x6,
1886	.pop => 0x0,
1887	.@"test" => 0x0,
1888	.seto,
1889	.setno,
1890	.setb,
1891	.setc,
1892	.setnae,
1893	.setnb,
1894	.setnc,
1895	.setae,
1896	.sete,
1897	.setz,
1898	.setne,
1899	.setnz,
1900	.setbe,
1901	.setna,
1902	.seta,
1903	.setnbe,
1904	.sets,
1905	.setns,
1906	.setp,
1907	.setpe,
1908	.setnp,
1909	.setpo,
1910	.setl,
1911	.setnge,
1912	.setnl,
1913	.setge,
1914	.setle,
1915	.setng,
1916	.setnle,
1917	.setg,
1918	=> 0x0,
1919	.shl,
1920	.sal,
1921	=> 0x4,
1922	.shr => 0x5,
1923	.sar => 0x7,
1924	.mul => 0x4,
1925	.imul => 0x5,
1926	.div => 0x6,
1927	.idiv => 0x7,
1928	.fisttp16 => 0x1,
1929	.fisttp32 => 0x1,
1930	.fisttp64 => 0x1,
1931	.fld32 => 0x0,
1932	.fld64 => 0x0,
1933	else => unreachable,
1934	};
1935	}
1936
1937	const VexEncoding = struct {
1938	prefix: Encoder.Vex,
1939	reg: ?enum {
1940	ndd,
1941	nds,
1942	dds,
1943	},
1944	};
1945
1946	inline fn getVexEncoding(tag: Tag, enc: Encoding) VexEncoding {
1947	const desc: struct {
1948	reg: enum {
1949	none,
1950	ndd,
1951	nds,
1952	dds,
1953	} = .none,
1954	len_256: bool = false,
1955	wig: bool = false,
1956	lig: bool = false,
1957	lz: bool = false,
1958	lead_opc: enum {
1959	l_0f,
1960	l_0f_3a,
1961	l_0f_38,
1962	} = .l_0f,
1963	simd_prefix: enum {
1964	none,
1965	p_66,
1966	p_f2,
1967	p_f3,
1968	} = .none,
1969	} = blk: {
1970	switch (enc) {
1971	.mv => switch (tag) {
1972	.vmovsd => break :blk .{ .lig = true, .simd_prefix = .p_f2, .wig = true },
1973	.vmovss => break :blk .{ .lig = true, .simd_prefix = .p_f3, .wig = true },
1974	else => unreachable,
1975	},
1976	.vm => switch (tag) {
1977	.vmovsd => break :blk .{ .lig = true, .simd_prefix = .p_f2, .wig = true },
1978	.vmovss => break :blk .{ .lig = true, .simd_prefix = .p_f3, .wig = true },
1979	.vucomisd => break :blk .{ .lig = true, .simd_prefix = .p_66, .wig = true },
1980	.vucomiss => break :blk .{ .lig = true, .wig = true },
1981	else => unreachable,
1982	},
1983	.rvm => switch (tag) {
1984	.vaddsd => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f2, .wig = true },
1985	.vaddss => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f3, .wig = true },
1986	.vmovsd => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f2, .wig = true },
1987	.vmovss => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f3, .wig = true },
1988	else => unreachable,
1989	},
1990	.rvmi => switch (tag) {
1991	.vcmpsd => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f2, .wig = true },
1992	.vcmpss => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f3, .wig = true },
1993	else => unreachable,
1994	},
1995	else => unreachable,
1996	}
1997	};
1998
1999	var vex: Encoder.Vex = .{};
2000
2001	if (desc.len_256) vex.len_256();
2002	if (desc.wig) vex.wig();
2003	if (desc.lig) vex.lig();
2004	if (desc.lz) vex.lz();
2005
2006	switch (desc.lead_opc) {
2007	.l_0f => {},
2008	.l_0f_3a => vex.lead_opc_0f_3a(),
2009	.l_0f_38 => vex.lead_opc_0f_38(),
2010	}
2011
2012	switch (desc.simd_prefix) {
2013	.none => {},
2014	.p_66 => vex.simd_prefix_66(),
2015	.p_f2 => vex.simd_prefix_f2(),
2016	.p_f3 => vex.simd_prefix_f3(),
2017	}
2018
2019	return VexEncoding{ .prefix = vex, .reg = switch (desc.reg) {
2020	.none => null,
2021	.nds => .nds,
2022	.dds => .dds,
2023	.ndd => .ndd,
2024	} };
2025	}
2026
2027	const ScaleIndex = packed struct {
2028	scale: u2,
2029	index: Register,
2030	};
2031
2032	const Memory = struct {
2033	base: ?Register,
2034	rip: bool = false,
2035	disp: u32,
2036	ptr_size: PtrSize,
2037	scale_index: ?ScaleIndex = null,
2038
2039	const PtrSize = enum(u2) {
2040	byte_ptr = 0b00,
2041	word_ptr = 0b01,
2042	dword_ptr = 0b10,
2043	qword_ptr = 0b11,
2044
2045	fn new(bit_size: u64) PtrSize {
2046	return @intToEnum(PtrSize, math.log2_int(u4, @intCast(u4, @divExact(bit_size, 8))));
2047	}
2048
2049	/// Returns size in bits.
2050	fn size(ptr_size: PtrSize) u64 {
2051	return 8 * (math.powi(u8, 2, @enumToInt(ptr_size)) catch unreachable);
2052	}
2053	};
2054
2055	fn encode(mem_op: Memory, encoder: Encoder, operand: u3) void {
2056	if (mem_op.base) \|base\| {
2057	const dst = base.lowEnc();
2058	const src = operand;
2059	if (dst == 4 or mem_op.scale_index != null) {
2060	if (mem_op.disp == 0 and dst != 5) {
2061	encoder.modRm_SIBDisp0(src);
2062	if (mem_op.scale_index) \|si\| {
2063	encoder.sib_scaleIndexBase(si.scale, si.index.lowEnc(), dst);
2064	} else {
2065	encoder.sib_base(dst);
2066	}
2067	} else if (immOpSize(mem_op.disp) == 8) {
2068	encoder.modRm_SIBDisp8(src);
2069	if (mem_op.scale_index) \|si\| {
2070	encoder.sib_scaleIndexBaseDisp8(si.scale, si.index.lowEnc(), dst);
2071	} else {
2072	encoder.sib_baseDisp8(dst);
2073	}
2074	encoder.disp8(@bitCast(i8, @truncate(u8, mem_op.disp)));
2075	} else {
2076	encoder.modRm_SIBDisp32(src);
2077	if (mem_op.scale_index) \|si\| {
2078	encoder.sib_scaleIndexBaseDisp32(si.scale, si.index.lowEnc(), dst);
2079	} else {
2080	encoder.sib_baseDisp32(dst);
2081	}
2082	encoder.disp32(@bitCast(i32, mem_op.disp));
2083	}
2084	} else {
2085	if (mem_op.disp == 0 and dst != 5) {
2086	encoder.modRm_indirectDisp0(src, dst);
2087	} else if (immOpSize(mem_op.disp) == 8) {
2088	encoder.modRm_indirectDisp8(src, dst);
2089	encoder.disp8(@bitCast(i8, @truncate(u8, mem_op.disp)));
2090	} else {
2091	encoder.modRm_indirectDisp32(src, dst);
2092	encoder.disp32(@bitCast(i32, mem_op.disp));
2093	}
2094	}
2095	} else {
2096	if (mem_op.rip) {
2097	encoder.modRm_RIPDisp32(operand);
2098	} else {
2099	encoder.modRm_SIBDisp0(operand);
2100	if (mem_op.scale_index) \|si\| {
2101	encoder.sib_scaleIndexDisp32(si.scale, si.index.lowEnc());
2102	} else {
2103	encoder.sib_disp32();
2104	}
2105	}
2106	encoder.disp32(@bitCast(i32, mem_op.disp));
2107	}
2108	}
2109
2110	/// Returns size in bits.
2111	fn size(memory: Memory) u64 {
2112	return memory.ptr_size.size();
2113	}
2114	};
2115
2116	fn encodeImm(encoder: Encoder, imm: u32, size: u64) void {
2117	switch (size) {
2118	8 => encoder.imm8(@bitCast(i8, @truncate(u8, imm))),
2119	16 => encoder.imm16(@bitCast(i16, @truncate(u16, imm))),
2120	32, 64 => encoder.imm32(@bitCast(i32, imm)),
2121	else => unreachable,
2122	}
2123	}
2124
2125	const RegisterOrMemory = union(enum) {
2126	register: Register,
2127	memory: Memory,
2128
2129	fn reg(register: Register) RegisterOrMemory {
2130	return .{ .register = register };
2131	}
2132
2133	fn mem(ptr_size: Memory.PtrSize, args: struct {
2134	disp: u32,
2135	base: ?Register = null,
2136	scale_index: ?ScaleIndex = null,
2137	}) RegisterOrMemory {
2138	return .{
2139	.memory = .{
2140	.base = args.base,
2141	.disp = args.disp,
2142	.ptr_size = ptr_size,
2143	.scale_index = args.scale_index,
2144	},
2145	};
2146	}
2147
2148	fn rip(ptr_size: Memory.PtrSize, disp: u32) RegisterOrMemory {
2149	return .{
2150	.memory = .{
2151	.base = null,
2152	.rip = true,
2153	.disp = disp,
2154	.ptr_size = ptr_size,
2155	},
2156	};
2157	}
2158
2159	/// Returns size in bits.
2160	fn size(reg_or_mem: RegisterOrMemory) u64 {
2161	return switch (reg_or_mem) {
2162	.register => \|register\| register.size(),
2163	.memory => \|memory\| memory.size(),
2164	};
2165	}
2166	};
2167
2168	fn lowerToZoEnc(tag: Tag, code: *std.ArrayList(u8)) InnerError!void {
2169	assert(!tag.isAvx());
2170	const opc = getOpCode(tag, .zo, false);
2171	const encoder = try Encoder.init(code, 2);
2172	switch (tag) {
2173	.cqo => {
2174	encoder.rex(.{
2175	.w = true,
2176	});
2177	},
2178	else => {},
2179	}
2180	opc.encode(encoder);
2181	}
2182
2183	fn lowerToIEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) InnerError!void {
2184	assert(!tag.isAvx());
2185	if (tag == .ret_far or tag == .ret_near) {
2186	const encoder = try Encoder.init(code, 3);
2187	const opc = getOpCode(tag, .i, false);
2188	opc.encode(encoder);
2189	encoder.imm16(@bitCast(i16, @truncate(u16, imm)));
2190	return;
2191	}
2192	const opc = getOpCode(tag, .i, immOpSize(imm) == 8);
2193	const encoder = try Encoder.init(code, 5);
2194	if (immOpSize(imm) == 16) {
2195	encoder.prefix16BitMode();
2196	}
2197	opc.encode(encoder);
2198	encodeImm(encoder, imm, immOpSize(imm));
2199	}
2200
2201	fn lowerToOEnc(tag: Tag, reg: Register, code: *std.ArrayList(u8)) InnerError!void {
2202	assert(!tag.isAvx());
2203	const opc = getOpCode(tag, .o, false);
2204	const encoder = try Encoder.init(code, 3);
2205	if (reg.size() == 16) {
2206	encoder.prefix16BitMode();
2207	}
2208	encoder.rex(.{
2209	.w = false,
2210	.b = reg.isExtended(),
2211	});
2212	opc.encodeWithReg(encoder, reg);
2213	}
2214
2215	fn lowerToDEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) InnerError!void {
2216	assert(!tag.isAvx());
2217	const opc = getOpCode(tag, .d, false);
2218	const encoder = try Encoder.init(code, 6);
2219	opc.encode(encoder);
2220	encoder.imm32(@bitCast(i32, imm));
2221	}
2222
2223	fn lowerToMxEnc(tag: Tag, reg_or_mem: RegisterOrMemory, enc: Encoding, code: *std.ArrayList(u8)) InnerError!void {
2224	assert(!tag.isAvx());
2225	const opc = getOpCode(tag, enc, reg_or_mem.size() == 8);
2226	const modrm_ext = getModRmExt(tag);
2227	switch (reg_or_mem) {
2228	.register => \|reg\| {
2229	const encoder = try Encoder.init(code, 4);
2230	if (reg.size() == 16) {
2231	encoder.prefix16BitMode();
2232	}
2233	const wide = if (tag == .jmp_near) false else setRexWRegister(reg);
2234	encoder.rex(.{
2235	.w = wide,
2236	.b = reg.isExtended(),
2237	});
2238	opc.encode(encoder);
2239	encoder.modRm_direct(modrm_ext, reg.lowEnc());
2240	},
2241	.memory => \|mem_op\| {
2242	const encoder = try Encoder.init(code, 8);
2243	if (mem_op.ptr_size == .word_ptr) {
2244	encoder.prefix16BitMode();
2245	}
2246	if (mem_op.base) \|base\| {
2247	const wide = if (tag == .jmp_near) false else mem_op.ptr_size == .qword_ptr;
2248	encoder.rex(.{
2249	.w = wide,
2250	.b = base.isExtended(),
2251	.x = if (mem_op.scale_index) \|si\| si.index.isExtended() else false,
2252	});
2253	}
2254	opc.encode(encoder);
2255	mem_op.encode(encoder, modrm_ext);
2256	},
2257	}
2258	}
2259
2260	fn lowerToMEnc(tag: Tag, reg_or_mem: RegisterOrMemory, code: *std.ArrayList(u8)) InnerError!void {
2261	return lowerToMxEnc(tag, reg_or_mem, .m, code);
2262	}
2263
2264	fn lowerToM1Enc(tag: Tag, reg_or_mem: RegisterOrMemory, code: *std.ArrayList(u8)) InnerError!void {
2265	return lowerToMxEnc(tag, reg_or_mem, .m1, code);
2266	}
2267
2268	fn lowerToMcEnc(tag: Tag, reg_or_mem: RegisterOrMemory, code: *std.ArrayList(u8)) InnerError!void {
2269	return lowerToMxEnc(tag, reg_or_mem, .mc, code);
2270	}
2271
2272	fn lowerToTdEnc(tag: Tag, moffs: u64, reg: Register, code: *std.ArrayList(u8)) InnerError!void {
2273	return lowerToTdFdEnc(tag, reg, moffs, code, true);
2274	}
2275
2276	fn lowerToFdEnc(tag: Tag, reg: Register, moffs: u64, code: *std.ArrayList(u8)) InnerError!void {
2277	return lowerToTdFdEnc(tag, reg, moffs, code, false);
2278	}
2279
2280	fn lowerToTdFdEnc(tag: Tag, reg: Register, moffs: u64, code: *std.ArrayList(u8), td: bool) InnerError!void {
2281	assert(!tag.isAvx());
2282	const opc = if (td) getOpCode(tag, .td, reg.size() == 8) else getOpCode(tag, .fd, reg.size() == 8);
2283	const encoder = try Encoder.init(code, 10);
2284	if (reg.size() == 16) {
2285	encoder.prefix16BitMode();
2286	}
2287	encoder.rex(.{
2288	.w = setRexWRegister(reg),
2289	});
2290	opc.encode(encoder);
2291	switch (reg.size()) {
2292	8 => encoder.imm8(@bitCast(i8, @truncate(u8, moffs))),
2293	16 => encoder.imm16(@bitCast(i16, @truncate(u16, moffs))),
2294	32 => encoder.imm32(@bitCast(i32, @truncate(u32, moffs))),
2295	64 => encoder.imm64(moffs),
2296	else => unreachable,
2297	}
2298	}
2299
2300	fn lowerToOiEnc(tag: Tag, reg: Register, imm: u64, code: *std.ArrayList(u8)) InnerError!void {
2301	assert(!tag.isAvx());
2302	const opc = getOpCode(tag, .oi, reg.size() == 8);
2303	const encoder = try Encoder.init(code, 10);
2304	if (reg.size() == 16) {
2305	encoder.prefix16BitMode();
2306	}
2307	encoder.rex(.{
2308	.w = setRexWRegister(reg),
2309	.b = reg.isExtended(),
2310	});
2311	opc.encodeWithReg(encoder, reg);
2312	switch (reg.size()) {
2313	8 => encoder.imm8(@bitCast(i8, @truncate(u8, imm))),
2314	16 => encoder.imm16(@bitCast(i16, @truncate(u16, imm))),
2315	32 => encoder.imm32(@bitCast(i32, @truncate(u32, imm))),
2316	64 => encoder.imm64(imm),
2317	else => unreachable,
2318	}
2319	}
2320
2321	fn lowerToMiXEnc(
2322	tag: Tag,
2323	reg_or_mem: RegisterOrMemory,
2324	imm: u32,
2325	enc: Encoding,
2326	code: *std.ArrayList(u8),
2327	) InnerError!void {
2328	assert(!tag.isAvx());
2329	const modrm_ext = getModRmExt(tag);
2330	const opc = getOpCode(tag, enc, reg_or_mem.size() == 8);
2331	switch (reg_or_mem) {
2332	.register => \|dst_reg\| {
2333	const encoder = try Encoder.init(code, 7);
2334	if (dst_reg.size() == 16) {
2335	// 0x66 prefix switches to the non-default size; here we assume a switch from
2336	// the default 32bits to 16bits operand-size.
2337	// More info: https://www.cs.uni-potsdam.de/desn/lehre/ss15/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf#page=32&zoom=auto,-159,773
2338	encoder.prefix16BitMode();
2339	}
2340	encoder.rex(.{
2341	.w = setRexWRegister(dst_reg),
2342	.b = dst_reg.isExtended(),
2343	});
2344	opc.encode(encoder);
2345	encoder.modRm_direct(modrm_ext, dst_reg.lowEnc());
2346	encodeImm(encoder, imm, if (enc == .mi8) 8 else dst_reg.size());
2347	},
2348	.memory => \|dst_mem\| {
2349	const encoder = try Encoder.init(code, 12);
2350	if (dst_mem.ptr_size == .word_ptr) {
2351	encoder.prefix16BitMode();
2352	}
2353	if (dst_mem.base) \|base\| {
2354	encoder.rex(.{
2355	.w = dst_mem.ptr_size == .qword_ptr,
2356	.b = base.isExtended(),
2357	.x = if (dst_mem.scale_index) \|si\| si.index.isExtended() else false,
2358	});
2359	} else {
2360	encoder.rex(.{
2361	.w = dst_mem.ptr_size == .qword_ptr,
2362	.x = if (dst_mem.scale_index) \|si\| si.index.isExtended() else false,
2363	});
2364	}
2365	opc.encode(encoder);
2366	dst_mem.encode(encoder, modrm_ext);
2367	encodeImm(encoder, imm, if (enc == .mi8) 8 else dst_mem.ptr_size.size());
2368	},
2369	}
2370	}
2371
2372	fn lowerToMiImm8Enc(tag: Tag, reg_or_mem: RegisterOrMemory, imm: u8, code: *std.ArrayList(u8)) InnerError!void {
2373	return lowerToMiXEnc(tag, reg_or_mem, imm, .mi8, code);
2374	}
2375
2376	fn lowerToMiEnc(tag: Tag, reg_or_mem: RegisterOrMemory, imm: u32, code: *std.ArrayList(u8)) InnerError!void {
2377	return lowerToMiXEnc(tag, reg_or_mem, imm, .mi, code);
2378	}
2379
2380	fn lowerToRmEnc(
2381	tag: Tag,
2382	reg: Register,
2383	reg_or_mem: RegisterOrMemory,
2384	code: *std.ArrayList(u8),
2385	) InnerError!void {
2386	assert(!tag.isAvx());
2387	const opc = getOpCode(tag, .rm, reg.size() == 8 or reg_or_mem.size() == 8);
2388	switch (reg_or_mem) {
2389	.register => \|src_reg\| {
2390	const encoder = try Encoder.init(code, 5);
2391	if (reg.size() == 16) {
2392	encoder.prefix16BitMode();
2393	}
2394	encoder.rex(.{
2395	.w = setRexWRegister(reg) or setRexWRegister(src_reg),
2396	.r = reg.isExtended(),
2397	.b = src_reg.isExtended(),
2398	});
2399	opc.encode(encoder);
2400	encoder.modRm_direct(reg.lowEnc(), src_reg.lowEnc());
2401	},
2402	.memory => \|src_mem\| {
2403	const encoder = try Encoder.init(code, 9);
2404	if (reg.size() == 16) {
2405	encoder.prefix16BitMode();
2406	}
2407	if (src_mem.base) \|base\| {
2408	// TODO handle 32-bit base register - requires prefix 0x67
2409	// Intel Manual, Vol 1, chapter 3.6 and 3.6.1
2410	encoder.rex(.{
2411	.w = setRexWRegister(reg),
2412	.r = reg.isExtended(),
2413	.b = base.isExtended(),
2414	.x = if (src_mem.scale_index) \|si\| si.index.isExtended() else false,
2415	});
2416	} else {
2417	encoder.rex(.{
2418	.w = setRexWRegister(reg),
2419	.r = reg.isExtended(),
2420	.x = if (src_mem.scale_index) \|si\| si.index.isExtended() else false,
2421	});
2422	}
2423	opc.encode(encoder);
2424	src_mem.encode(encoder, reg.lowEnc());
2425	},
2426	}
2427	}
2428
2429	fn lowerToMrEnc(
2430	tag: Tag,
2431	reg_or_mem: RegisterOrMemory,
2432	reg: Register,
2433	code: *std.ArrayList(u8),
2434	) InnerError!void {
2435	assert(!tag.isAvx());
2436	const opc = getOpCode(tag, .mr, reg.size() == 8 or reg_or_mem.size() == 8);
2437	switch (reg_or_mem) {
2438	.register => \|dst_reg\| {
2439	const encoder = try Encoder.init(code, 4);
2440	if (dst_reg.size() == 16) {
2441	encoder.prefix16BitMode();
2442	}
2443	encoder.rex(.{
2444	.w = setRexWRegister(dst_reg) or setRexWRegister(reg),
2445	.r = reg.isExtended(),
2446	.b = dst_reg.isExtended(),
2447	});
2448	opc.encode(encoder);
2449	encoder.modRm_direct(reg.lowEnc(), dst_reg.lowEnc());
2450	},
2451	.memory => \|dst_mem\| {
2452	const encoder = try Encoder.init(code, 9);
2453	if (reg.size() == 16) {
2454	encoder.prefix16BitMode();
2455	}
2456	if (dst_mem.base) \|base\| {
2457	encoder.rex(.{
2458	.w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg),
2459	.r = reg.isExtended(),
2460	.b = base.isExtended(),
2461	.x = if (dst_mem.scale_index) \|si\| si.index.isExtended() else false,
2462	});
2463	} else {
2464	encoder.rex(.{
2465	.w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg),
2466	.r = reg.isExtended(),
2467	.x = if (dst_mem.scale_index) \|si\| si.index.isExtended() else false,
2468	});
2469	}
2470	opc.encode(encoder);
2471	dst_mem.encode(encoder, reg.lowEnc());
2472	},
2473	}
2474	}
2475
2476	fn lowerToRmiEnc(
2477	tag: Tag,
2478	reg: Register,
2479	reg_or_mem: RegisterOrMemory,
2480	imm: u32,
2481	code: *std.ArrayList(u8),
2482	) InnerError!void {
2483	assert(!tag.isAvx());
2484	const opc = getOpCode(tag, .rmi, false);
2485	const encoder = try Encoder.init(code, 13);
2486	if (reg.size() == 16) {
2487	encoder.prefix16BitMode();
2488	}
2489	switch (reg_or_mem) {
2490	.register => \|src_reg\| {
2491	encoder.rex(.{
2492	.w = setRexWRegister(reg) or setRexWRegister(src_reg),
2493	.r = reg.isExtended(),
2494	.b = src_reg.isExtended(),
2495	});
2496	opc.encode(encoder);
2497	encoder.modRm_direct(reg.lowEnc(), src_reg.lowEnc());
2498	},
2499	.memory => \|src_mem\| {
2500	if (src_mem.base) \|base\| {
2501	// TODO handle 32-bit base register - requires prefix 0x67
2502	// Intel Manual, Vol 1, chapter 3.6 and 3.6.1
2503	encoder.rex(.{
2504	.w = setRexWRegister(reg),
2505	.r = reg.isExtended(),
2506	.b = base.isExtended(),
2507	.x = if (src_mem.scale_index) \|si\| si.index.isExtended() else false,
2508	});
2509	} else {
2510	encoder.rex(.{
2511	.w = setRexWRegister(reg),
2512	.r = reg.isExtended(),
2513	.x = if (src_mem.scale_index) \|si\| si.index.isExtended() else false,
2514	});
2515	}
2516	opc.encode(encoder);
2517	src_mem.encode(encoder, reg.lowEnc());
2518	},
2519	}
2520	encodeImm(encoder, imm, reg.size());
2521	}
2522
2523	/// Also referred to as XM encoding in Intel manual.
2524	fn lowerToVmEnc(
2525	tag: Tag,
2526	reg: Register,
2527	reg_or_mem: RegisterOrMemory,
2528	code: *std.ArrayList(u8),
2529	) InnerError!void {
2530	const opc = getOpCode(tag, .vm, false);
2531	var enc = getVexEncoding(tag, .vm);
2532	const vex = &enc.prefix;
2533	switch (reg_or_mem) {
2534	.register => \|src_reg\| {
2535	const encoder = try Encoder.init(code, 5);
2536	vex.rex(.{
2537	.r = reg.isExtended(),
2538	.b = src_reg.isExtended(),
2539	});
2540	encoder.vex(enc.prefix);
2541	opc.encode(encoder);
2542	encoder.modRm_direct(reg.lowEnc(), src_reg.lowEnc());
2543	},
2544	.memory => \|src_mem\| {
2545	const encoder = try Encoder.init(code, 10);
2546	if (src_mem.base) \|base\| {
2547	vex.rex(.{
2548	.r = reg.isExtended(),
2549	.b = base.isExtended(),
2550	.x = if (src_mem.scale_index) \|si\| si.index.isExtended() else false,
2551	});
2552	} else {
2553	vex.rex(.{
2554	.r = reg.isExtended(),
2555	.x = if (src_mem.scale_index) \|si\| si.index.isExtended() else false,
2556	});
2557	}
2558	encoder.vex(enc.prefix);
2559	opc.encode(encoder);
2560	src_mem.encode(encoder, reg.lowEnc());
2561	},
2562	}
2563	}
2564
2565	/// Usually referred to as MR encoding with V/V in Intel manual.
2566	fn lowerToMvEnc(
2567	tag: Tag,
2568	reg_or_mem: RegisterOrMemory,
2569	reg: Register,
2570	code: *std.ArrayList(u8),
2571	) InnerError!void {
2572	const opc = getOpCode(tag, .mv, false);
2573	var enc = getVexEncoding(tag, .mv);
2574	const vex = &enc.prefix;
2575	switch (reg_or_mem) {
2576	.register => \|dst_reg\| {
2577	const encoder = try Encoder.init(code, 4);
2578	vex.rex(.{
2579	.r = reg.isExtended(),
2580	.b = dst_reg.isExtended(),
2581	});
2582	encoder.vex(enc.prefix);
2583	opc.encode(encoder);
2584	encoder.modRm_direct(reg.lowEnc(), dst_reg.lowEnc());
2585	},
2586	.memory => \|dst_mem\| {
2587	const encoder = try Encoder.init(code, 10);
2588	if (dst_mem.base) \|base\| {
2589	vex.rex(.{
2590	.r = reg.isExtended(),
2591	.b = base.isExtended(),
2592	.x = if (dst_mem.scale_index) \|si\| si.index.isExtended() else false,
2593	});
2594	} else {
2595	vex.rex(.{
2596	.r = reg.isExtended(),
2597	.x = if (dst_mem.scale_index) \|si\| si.index.isExtended() else false,
2598	});
2599	}
2600	encoder.vex(enc.prefix);
2601	opc.encode(encoder);
2602	dst_mem.encode(encoder, reg.lowEnc());
2603	},
2604	}
2605	}
2606
2607	fn lowerToRvmEnc(
2608	tag: Tag,
2609	reg1: Register,
2610	reg2: Register,
2611	reg_or_mem: RegisterOrMemory,
2612	code: *std.ArrayList(u8),
2613	) InnerError!void {
2614	const opc = getOpCode(tag, .rvm, false);
2615	var enc = getVexEncoding(tag, .rvm);
2616	const vex = &enc.prefix;
2617	switch (reg_or_mem) {
2618	.register => \|reg3\| {
2619	if (enc.reg) \|vvvv\| {
2620	switch (vvvv) {
2621	.nds => vex.reg(reg2.enc()),
2622	else => unreachable, // TODO
2623	}
2624	}
2625	const encoder = try Encoder.init(code, 5);
2626	vex.rex(.{
2627	.r = reg1.isExtended(),
2628	.b = reg3.isExtended(),
2629	});
2630	encoder.vex(enc.prefix);
2631	opc.encode(encoder);
2632	encoder.modRm_direct(reg1.lowEnc(), reg3.lowEnc());
2633	},
2634	.memory => \|dst_mem\| {
2635	_ = dst_mem;
2636	unreachable; // TODO
2637	},
2638	}
2639	}
2640
2641	fn lowerToRvmiEnc(
2642	tag: Tag,
2643	reg1: Register,
2644	reg2: Register,
2645	reg_or_mem: RegisterOrMemory,
2646	imm: u32,
2647	code: *std.ArrayList(u8),
2648	) InnerError!void {
2649	const opc = getOpCode(tag, .rvmi, false);
2650	var enc = getVexEncoding(tag, .rvmi);
2651	const vex = &enc.prefix;
2652	const encoder: Encoder = blk: {
2653	switch (reg_or_mem) {
2654	.register => \|reg3\| {
2655	if (enc.reg) \|vvvv\| {
2656	switch (vvvv) {
2657	.nds => vex.reg(reg2.enc()),
2658	else => unreachable, // TODO
2659	}
2660	}
2661	const encoder = try Encoder.init(code, 5);
2662	vex.rex(.{
2663	.r = reg1.isExtended(),
2664	.b = reg3.isExtended(),
2665	});
2666	encoder.vex(enc.prefix);
2667	opc.encode(encoder);
2668	encoder.modRm_direct(reg1.lowEnc(), reg3.lowEnc());
2669	break :blk encoder;
2670	},
2671	.memory => \|dst_mem\| {
2672	_ = dst_mem;
2673	unreachable; // TODO
2674	},
2675	}
2676	};
2677	encodeImm(encoder, imm, 8); // TODO
2678	}
2679
2680	fn expectEqualHexStrings(expected: []const u8, given: []const u8, assembly: []const u8) !void {
2681	assert(expected.len > 0);
2682	if (mem.eql(u8, expected, given)) return;
2683	const expected_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{std.fmt.fmtSliceHexLower(expected)});
2684	defer testing.allocator.free(expected_fmt);
2685	const given_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{std.fmt.fmtSliceHexLower(given)});
2686	defer testing.allocator.free(given_fmt);
2687	const idx = mem.indexOfDiff(u8, expected_fmt, given_fmt).?;
2688	var padding = try testing.allocator.alloc(u8, idx + 5);
2689	defer testing.allocator.free(padding);
2690	mem.set(u8, padding, ' ');
2691	std.debug.print("\nASM: {s}\nEXP: {s}\nGIV: {s}\n{s}^ -- first differing byte\n", .{
2692	assembly,
2693	expected_fmt,
2694	given_fmt,
2695	padding,
2696	});
2697	return error.TestFailed;
2698	}
2699
2700	const TestEmit = struct {
2701	code_buffer: std.ArrayList(u8),
2702	next: usize = 0,
2703
2704	fn init() TestEmit {
2705	return .{
2706	.code_buffer = std.ArrayList(u8).init(testing.allocator),
2707	};
2708	}
2709
2710	fn deinit(emit: *TestEmit) void {
2711	emit.code_buffer.deinit();
2712	emit.next = undefined;
2713	}
2714
2715	fn code(emit: TestEmit) std.ArrayList(u8) {
2716	emit.next = emit.code_buffer.items.len;
2717	return &emit.code_buffer;
2718	}
2719
2720	fn lowered(emit: TestEmit) []const u8 {
2721	return emit.code_buffer.items[emit.next..];
2722	}
2723	};
2724
2725	test "lower MI encoding" {
2726	var emit = TestEmit.init();
2727	defer emit.deinit();
2728	try lowerToMiEnc(.mov, RegisterOrMemory.reg(.rax), 0x10, emit.code());
2729	try expectEqualHexStrings("\x48\xc7\xc0\x10\x00\x00\x00", emit.lowered(), "mov rax, 0x10");
2730	try lowerToMiEnc(.mov, RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0, .base = .r11 }), 0x10, emit.code());
2731	try expectEqualHexStrings("\x41\xc7\x03\x10\x00\x00\x00", emit.lowered(), "mov dword ptr [r11 + 0], 0x10");
2732	try lowerToMiEnc(.add, RegisterOrMemory.mem(.dword_ptr, .{
2733	.disp = @bitCast(u32, @as(i32, -8)),
2734	.base = .rdx,
2735	}), 0x10, emit.code());
2736	try expectEqualHexStrings("\x81\x42\xF8\x10\x00\x00\x00", emit.lowered(), "add dword ptr [rdx - 8], 0x10");
2737	try lowerToMiEnc(.sub, RegisterOrMemory.mem(.dword_ptr, .{
2738	.disp = 0x10000000,
2739	.base = .r11,
2740	}), 0x10, emit.code());
2741	try expectEqualHexStrings(
2742	"\x41\x81\xab\x00\x00\x00\x10\x10\x00\x00\x00",
2743	emit.lowered(),
2744	"sub dword ptr [r11 + 0x10000000], 0x10",
2745	);
2746	try lowerToMiEnc(.@"and", RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0x10000000 }), 0x10, emit.code());
2747	try expectEqualHexStrings(
2748	"\x81\x24\x25\x00\x00\x00\x10\x10\x00\x00\x00",
2749	emit.lowered(),
2750	"and dword ptr [ds:0x10000000], 0x10",
2751	);
2752	try lowerToMiEnc(.@"and", RegisterOrMemory.mem(.dword_ptr, .{
2753	.disp = 0x10000000,
2754	.base = .r12,
2755	}), 0x10, emit.code());
2756	try expectEqualHexStrings(
2757	"\x41\x81\xA4\x24\x00\x00\x00\x10\x10\x00\x00\x00",
2758	emit.lowered(),
2759	"and dword ptr [r12 + 0x10000000], 0x10",
2760	);
2761	try lowerToMiEnc(.mov, RegisterOrMemory.rip(.qword_ptr, 0x10), 0x10, emit.code());
2762	try expectEqualHexStrings(
2763	"\x48\xC7\x05\x10\x00\x00\x00\x10\x00\x00\x00",
2764	emit.lowered(),
2765	"mov qword ptr [rip + 0x10], 0x10",
2766	);
2767	try lowerToMiEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
2768	.disp = @bitCast(u32, @as(i32, -8)),
2769	.base = .rbp,
2770	}), 0x10, emit.code());
2771	try expectEqualHexStrings(
2772	"\x48\xc7\x45\xf8\x10\x00\x00\x00",
2773	emit.lowered(),
2774	"mov qword ptr [rbp - 8], 0x10",
2775	);
2776	try lowerToMiEnc(.mov, RegisterOrMemory.mem(.word_ptr, .{
2777	.disp = @bitCast(u32, @as(i32, -2)),
2778	.base = .rbp,
2779	}), 0x10, emit.code());
2780	try expectEqualHexStrings("\x66\xC7\x45\xFE\x10\x00", emit.lowered(), "mov word ptr [rbp - 2], 0x10");
2781	try lowerToMiEnc(.mov, RegisterOrMemory.mem(.byte_ptr, .{
2782	.disp = @bitCast(u32, @as(i32, -1)),
2783	.base = .rbp,
2784	}), 0x10, emit.code());
2785	try expectEqualHexStrings("\xC6\x45\xFF\x10", emit.lowered(), "mov byte ptr [rbp - 1], 0x10");
2786	try lowerToMiEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
2787	.disp = 0x10000000,
2788	.scale_index = .{
2789	.scale = 1,
2790	.index = .rcx,
2791	},
2792	}), 0x10, emit.code());
2793	try expectEqualHexStrings(
2794	"\x48\xC7\x04\x4D\x00\x00\x00\x10\x10\x00\x00\x00",
2795	emit.lowered(),
2796	"mov qword ptr [rcx*2 + 0x10000000], 0x10",
2797	);
2798
2799	try lowerToMiImm8Enc(.add, RegisterOrMemory.reg(.rax), 0x10, emit.code());
2800	try expectEqualHexStrings("\x48\x83\xC0\x10", emit.lowered(), "add rax, 0x10");
2801	}
2802
2803	test "lower RM encoding" {
2804	var emit = TestEmit.init();
2805	defer emit.deinit();
2806	try lowerToRmEnc(.mov, .rax, RegisterOrMemory.reg(.rbx), emit.code());
2807	try expectEqualHexStrings("\x48\x8b\xc3", emit.lowered(), "mov rax, rbx");
2808	try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0, .base = .r11 }), emit.code());
2809	try expectEqualHexStrings("\x49\x8b\x03", emit.lowered(), "mov rax, qword ptr [r11 + 0]");
2810	try lowerToRmEnc(.add, .r11, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10000000 }), emit.code());
2811	try expectEqualHexStrings(
2812	"\x4C\x03\x1C\x25\x00\x00\x00\x10",
2813	emit.lowered(),
2814	"add r11, qword ptr [ds:0x10000000]",
2815	);
2816	try lowerToRmEnc(.add, .r12b, RegisterOrMemory.mem(.byte_ptr, .{ .disp = 0x10000000 }), emit.code());
2817	try expectEqualHexStrings(
2818	"\x44\x02\x24\x25\x00\x00\x00\x10",
2819	emit.lowered(),
2820	"add r11b, byte ptr [ds:0x10000000]",
2821	);
2822	try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.qword_ptr, .{
2823	.disp = 0x10000000,
2824	.base = .r13,
2825	}), emit.code());
2826	try expectEqualHexStrings(
2827	"\x4D\x2B\x9D\x00\x00\x00\x10",
2828	emit.lowered(),
2829	"sub r11, qword ptr [r13 + 0x10000000]",
2830	);
2831	try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.qword_ptr, .{
2832	.disp = 0x10000000,
2833	.base = .r12,
2834	}), emit.code());
2835	try expectEqualHexStrings(
2836	"\x4D\x2B\x9C\x24\x00\x00\x00\x10",
2837	emit.lowered(),
2838	"sub r11, qword ptr [r12 + 0x10000000]",
2839	);
2840	try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{
2841	.disp = @bitCast(u32, @as(i32, -4)),
2842	.base = .rbp,
2843	}), emit.code());
2844	try expectEqualHexStrings("\x48\x8B\x45\xFC", emit.lowered(), "mov rax, qword ptr [rbp - 4]");
2845	try lowerToRmEnc(.lea, .rax, RegisterOrMemory.rip(.qword_ptr, 0x10), emit.code());
2846	try expectEqualHexStrings("\x48\x8D\x05\x10\x00\x00\x00", emit.lowered(), "lea rax, [rip + 0x10]");
2847	try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{
2848	.disp = @bitCast(u32, @as(i32, -8)),
2849	.base = .rbp,
2850	.scale_index = .{
2851	.scale = 0,
2852	.index = .rcx,
2853	},
2854	}), emit.code());
2855	try expectEqualHexStrings("\x48\x8B\x44\x0D\xF8", emit.lowered(), "mov rax, qword ptr [rbp + rcx*1 - 8]");
2856	try lowerToRmEnc(.mov, .eax, RegisterOrMemory.mem(.dword_ptr, .{
2857	.disp = @bitCast(u32, @as(i32, -4)),
2858	.base = .rbp,
2859	.scale_index = .{
2860	.scale = 2,
2861	.index = .rdx,
2862	},
2863	}), emit.code());
2864	try expectEqualHexStrings("\x8B\x44\x95\xFC", emit.lowered(), "mov eax, dword ptr [rbp + rdx*4 - 4]");
2865	try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{
2866	.disp = @bitCast(u32, @as(i32, -8)),
2867	.base = .rbp,
2868	.scale_index = .{
2869	.scale = 3,
2870	.index = .rcx,
2871	},
2872	}), emit.code());
2873	try expectEqualHexStrings("\x48\x8B\x44\xCD\xF8", emit.lowered(), "mov rax, qword ptr [rbp + rcx*8 - 8]");
2874	try lowerToRmEnc(.mov, .r8b, RegisterOrMemory.mem(.byte_ptr, .{
2875	.disp = @bitCast(u32, @as(i32, -24)),
2876	.base = .rsi,
2877	.scale_index = .{
2878	.scale = 0,
2879	.index = .rcx,
2880	},
2881	}), emit.code());
2882	try expectEqualHexStrings("\x44\x8A\x44\x0E\xE8", emit.lowered(), "mov r8b, byte ptr [rsi + rcx*1 - 24]");
2883	try lowerToRmEnc(.lea, .rsi, RegisterOrMemory.mem(.qword_ptr, .{
2884	.disp = 0,
2885	.base = .rbp,
2886	.scale_index = .{
2887	.scale = 0,
2888	.index = .rcx,
2889	},
2890	}), emit.code());
2891	try expectEqualHexStrings("\x48\x8D\x74\x0D\x00", emit.lowered(), "lea rsi, qword ptr [rbp + rcx*1 + 0]");
2892	}
2893
2894	test "lower MR encoding" {
2895	var emit = TestEmit.init();
2896	defer emit.deinit();
2897	try lowerToMrEnc(.mov, RegisterOrMemory.reg(.rax), .rbx, emit.code());
2898	try expectEqualHexStrings("\x48\x89\xd8", emit.lowered(), "mov rax, rbx");
2899	try lowerToMrEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
2900	.disp = @bitCast(u32, @as(i32, -4)),
2901	.base = .rbp,
2902	}), .r11, emit.code());
2903	try expectEqualHexStrings("\x4c\x89\x5d\xfc", emit.lowered(), "mov qword ptr [rbp - 4], r11");
2904	try lowerToMrEnc(.add, RegisterOrMemory.mem(.byte_ptr, .{ .disp = 0x10000000 }), .r12b, emit.code());
2905	try expectEqualHexStrings(
2906	"\x44\x00\x24\x25\x00\x00\x00\x10",
2907	emit.lowered(),
2908	"add byte ptr [ds:0x10000000], r12b",
2909	);
2910	try lowerToMrEnc(.add, RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0x10000000 }), .r12d, emit.code());
2911	try expectEqualHexStrings(
2912	"\x44\x01\x24\x25\x00\x00\x00\x10",
2913	emit.lowered(),
2914	"add dword ptr [ds:0x10000000], r12d",
2915	);
2916	try lowerToMrEnc(.sub, RegisterOrMemory.mem(.qword_ptr, .{
2917	.disp = 0x10000000,
2918	.base = .r11,
2919	}), .r12, emit.code());
2920	try expectEqualHexStrings(
2921	"\x4D\x29\xA3\x00\x00\x00\x10",
2922	emit.lowered(),
2923	"sub qword ptr [r11 + 0x10000000], r12",
2924	);
2925	try lowerToMrEnc(.mov, RegisterOrMemory.rip(.qword_ptr, 0x10), .r12, emit.code());
2926	try expectEqualHexStrings("\x4C\x89\x25\x10\x00\x00\x00", emit.lowered(), "mov qword ptr [rip + 0x10], r12");
2927	}
2928
2929	test "lower OI encoding" {
2930	var emit = TestEmit.init();
2931	defer emit.deinit();
2932	try lowerToOiEnc(.mov, .rax, 0x1000000000000000, emit.code());
2933	try expectEqualHexStrings(
2934	"\x48\xB8\x00\x00\x00\x00\x00\x00\x00\x10",
2935	emit.lowered(),
2936	"movabs rax, 0x1000000000000000",
2937	);
2938	try lowerToOiEnc(.mov, .r11, 0x1000000000000000, emit.code());
2939	try expectEqualHexStrings(
2940	"\x49\xBB\x00\x00\x00\x00\x00\x00\x00\x10",
2941	emit.lowered(),
2942	"movabs r11, 0x1000000000000000",
2943	);
2944	try lowerToOiEnc(.mov, .r11d, 0x10000000, emit.code());
2945	try expectEqualHexStrings("\x41\xBB\x00\x00\x00\x10", emit.lowered(), "mov r11d, 0x10000000");
2946	try lowerToOiEnc(.mov, .r11w, 0x1000, emit.code());
2947	try expectEqualHexStrings("\x66\x41\xBB\x00\x10", emit.lowered(), "mov r11w, 0x1000");
2948	try lowerToOiEnc(.mov, .r11b, 0x10, emit.code());
2949	try expectEqualHexStrings("\x41\xB3\x10", emit.lowered(), "mov r11b, 0x10");
2950	}
2951
2952	test "lower FD/TD encoding" {
2953	var emit = TestEmit.init();
2954	defer emit.deinit();
2955	try lowerToFdEnc(.mov, .rax, 0x1000000000000000, emit.code());
2956	try expectEqualHexStrings(
2957	"\x48\xa1\x00\x00\x00\x00\x00\x00\x00\x10",
2958	emit.lowered(),
2959	"mov rax, ds:0x1000000000000000",
2960	);
2961	try lowerToFdEnc(.mov, .eax, 0x10000000, emit.code());
2962	try expectEqualHexStrings("\xa1\x00\x00\x00\x10", emit.lowered(), "mov eax, ds:0x10000000");
2963	try lowerToFdEnc(.mov, .ax, 0x1000, emit.code());
2964	try expectEqualHexStrings("\x66\xa1\x00\x10", emit.lowered(), "mov ax, ds:0x1000");
2965	try lowerToFdEnc(.mov, .al, 0x10, emit.code());
2966	try expectEqualHexStrings("\xa0\x10", emit.lowered(), "mov al, ds:0x10");
2967	}
2968
2969	test "lower M encoding" {
2970	var emit = TestEmit.init();
2971	defer emit.deinit();
2972	try lowerToMEnc(.jmp_near, RegisterOrMemory.reg(.r12), emit.code());
2973	try expectEqualHexStrings("\x41\xFF\xE4", emit.lowered(), "jmp r12");
2974	try lowerToMEnc(.jmp_near, RegisterOrMemory.reg(.r12w), emit.code());
2975	try expectEqualHexStrings("\x66\x41\xFF\xE4", emit.lowered(), "jmp r12w");
2976	try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0, .base = .r12 }), emit.code());
2977	try expectEqualHexStrings("\x41\xFF\x24\x24", emit.lowered(), "jmp qword ptr [r12]");
2978	try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.word_ptr, .{ .disp = 0, .base = .r12 }), emit.code());
2979	try expectEqualHexStrings("\x66\x41\xFF\x24\x24", emit.lowered(), "jmp word ptr [r12]");
2980	try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10, .base = .r12 }), emit.code());
2981	try expectEqualHexStrings("\x41\xFF\x64\x24\x10", emit.lowered(), "jmp qword ptr [r12 + 0x10]");
2982	try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{
2983	.disp = 0x1000,
2984	.base = .r12,
2985	}), emit.code());
2986	try expectEqualHexStrings(
2987	"\x41\xFF\xA4\x24\x00\x10\x00\x00",
2988	emit.lowered(),
2989	"jmp qword ptr [r12 + 0x1000]",
2990	);
2991	try lowerToMEnc(.jmp_near, RegisterOrMemory.rip(.qword_ptr, 0x10), emit.code());
2992	try expectEqualHexStrings("\xFF\x25\x10\x00\x00\x00", emit.lowered(), "jmp qword ptr [rip + 0x10]");
2993	try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10 }), emit.code());
2994	try expectEqualHexStrings("\xFF\x24\x25\x10\x00\x00\x00", emit.lowered(), "jmp qword ptr [ds:0x10]");
2995	try lowerToMEnc(.seta, RegisterOrMemory.reg(.r11b), emit.code());
2996	try expectEqualHexStrings("\x41\x0F\x97\xC3", emit.lowered(), "seta r11b");
2997	try lowerToMEnc(.idiv, RegisterOrMemory.reg(.rax), emit.code());
2998	try expectEqualHexStrings("\x48\xF7\xF8", emit.lowered(), "idiv rax");
2999	try lowerToMEnc(.imul, RegisterOrMemory.reg(.al), emit.code());
3000	try expectEqualHexStrings("\xF6\xE8", emit.lowered(), "imul al");
3001	}
3002
3003	test "lower M1 and MC encodings" {
3004	var emit = TestEmit.init();
3005	defer emit.deinit();
3006	try lowerToM1Enc(.sal, RegisterOrMemory.reg(.r12), emit.code());
3007	try expectEqualHexStrings("\x49\xD1\xE4", emit.lowered(), "sal r12, 1");
3008	try lowerToM1Enc(.sal, RegisterOrMemory.reg(.r12d), emit.code());
3009	try expectEqualHexStrings("\x41\xD1\xE4", emit.lowered(), "sal r12d, 1");
3010	try lowerToM1Enc(.sal, RegisterOrMemory.reg(.r12w), emit.code());
3011	try expectEqualHexStrings("\x66\x41\xD1\xE4", emit.lowered(), "sal r12w, 1");
3012	try lowerToM1Enc(.sal, RegisterOrMemory.reg(.r12b), emit.code());
3013	try expectEqualHexStrings("\x41\xD0\xE4", emit.lowered(), "sal r12b, 1");
3014	try lowerToM1Enc(.sal, RegisterOrMemory.reg(.rax), emit.code());
3015	try expectEqualHexStrings("\x48\xD1\xE0", emit.lowered(), "sal rax, 1");
3016	try lowerToM1Enc(.sal, RegisterOrMemory.reg(.eax), emit.code());
3017	try expectEqualHexStrings("\xD1\xE0", emit.lowered(), "sal eax, 1");
3018	try lowerToM1Enc(.sal, RegisterOrMemory.mem(.qword_ptr, .{
3019	.disp = @bitCast(u32, @as(i32, -0x10)),
3020	.base = .rbp,
3021	}), emit.code());
3022	try expectEqualHexStrings("\x48\xD1\x65\xF0", emit.lowered(), "sal qword ptr [rbp - 0x10], 1");
3023	try lowerToM1Enc(.sal, RegisterOrMemory.mem(.dword_ptr, .{
3024	.disp = @bitCast(u32, @as(i32, -0x10)),
3025	.base = .rbp,
3026	}), emit.code());
3027	try expectEqualHexStrings("\xD1\x65\xF0", emit.lowered(), "sal dword ptr [rbp - 0x10], 1");
3028
3029	try lowerToMcEnc(.shr, RegisterOrMemory.reg(.r12), emit.code());
3030	try expectEqualHexStrings("\x49\xD3\xEC", emit.lowered(), "shr r12, cl");
3031	try lowerToMcEnc(.shr, RegisterOrMemory.reg(.rax), emit.code());
3032	try expectEqualHexStrings("\x48\xD3\xE8", emit.lowered(), "shr rax, cl");
3033
3034	try lowerToMcEnc(.sar, RegisterOrMemory.reg(.rsi), emit.code());
3035	try expectEqualHexStrings("\x48\xD3\xFE", emit.lowered(), "sar rsi, cl");
3036	}
3037
3038	test "lower O encoding" {
3039	var emit = TestEmit.init();
3040	defer emit.deinit();
3041	try lowerToOEnc(.pop, .r12, emit.code());
3042	try expectEqualHexStrings("\x41\x5c", emit.lowered(), "pop r12");
3043	try lowerToOEnc(.push, .r12w, emit.code());
3044	try expectEqualHexStrings("\x66\x41\x54", emit.lowered(), "push r12w");
3045	}
3046
3047	test "lower RMI encoding" {
3048	var emit = TestEmit.init();
3049	defer emit.deinit();
3050	try lowerToRmiEnc(.imul, .rax, RegisterOrMemory.mem(.qword_ptr, .{
3051	.disp = @bitCast(u32, @as(i32, -8)),
3052	.base = .rbp,
3053	}), 0x10, emit.code());
3054	try expectEqualHexStrings(
3055	"\x48\x69\x45\xF8\x10\x00\x00\x00",
3056	emit.lowered(),
3057	"imul rax, qword ptr [rbp - 8], 0x10",
3058	);
3059	try lowerToRmiEnc(.imul, .eax, RegisterOrMemory.mem(.dword_ptr, .{
3060	.disp = @bitCast(u32, @as(i32, -4)),
3061	.base = .rbp,
3062	}), 0x10, emit.code());
3063	try expectEqualHexStrings("\x69\x45\xFC\x10\x00\x00\x00", emit.lowered(), "imul eax, dword ptr [rbp - 4], 0x10");
3064	try lowerToRmiEnc(.imul, .ax, RegisterOrMemory.mem(.word_ptr, .{
3065	.disp = @bitCast(u32, @as(i32, -2)),
3066	.base = .rbp,
3067	}), 0x10, emit.code());
3068	try expectEqualHexStrings("\x66\x69\x45\xFE\x10\x00", emit.lowered(), "imul ax, word ptr [rbp - 2], 0x10");
3069	try lowerToRmiEnc(.imul, .r12, RegisterOrMemory.reg(.r12), 0x10, emit.code());
3070	try expectEqualHexStrings("\x4D\x69\xE4\x10\x00\x00\x00", emit.lowered(), "imul r12, r12, 0x10");
3071	try lowerToRmiEnc(.imul, .r12w, RegisterOrMemory.reg(.r12w), 0x10, emit.code());
3072	try expectEqualHexStrings("\x66\x45\x69\xE4\x10\x00", emit.lowered(), "imul r12w, r12w, 0x10");
3073	}
3074
3075	test "lower MV encoding" {
3076	var emit = TestEmit.init();
3077	defer emit.deinit();
3078	try lowerToMvEnc(.vmovsd, RegisterOrMemory.rip(.qword_ptr, 0x10), .xmm1, emit.code());
3079	try expectEqualHexStrings(
3080	"\xC5\xFB\x11\x0D\x10\x00\x00\x00",
3081	emit.lowered(),
3082	"vmovsd qword ptr [rip + 0x10], xmm1",
3083	);
3084	}
3085
3086	test "lower VM encoding" {
3087	var emit = TestEmit.init();
3088	defer emit.deinit();
3089	try lowerToVmEnc(.vmovsd, .xmm1, RegisterOrMemory.rip(.qword_ptr, 0x10), emit.code());
3090	try expectEqualHexStrings(
3091	"\xC5\xFB\x10\x0D\x10\x00\x00\x00",
3092	emit.lowered(),
3093	"vmovsd xmm1, qword ptr [rip + 0x10]",
3094	);
3095	}
3096
3097	test "lower to RVM encoding" {
3098	var emit = TestEmit.init();
3099	defer emit.deinit();
3100	try lowerToRvmEnc(.vaddsd, .xmm0, .xmm1, RegisterOrMemory.reg(.xmm2), emit.code());
3101	try expectEqualHexStrings("\xC5\xF3\x58\xC2", emit.lowered(), "vaddsd xmm0, xmm1, xmm2");
3102	try lowerToRvmEnc(.vaddsd, .xmm0, .xmm0, RegisterOrMemory.reg(.xmm1), emit.code());
3103	try expectEqualHexStrings("\xC5\xFB\x58\xC1", emit.lowered(), "vaddsd xmm0, xmm0, xmm1");
3104	}

src/arch/x86_64/Encoding.zig created+587

...	@@ -0,0 +1,587 @@
1	const Encoding = @This();
2
3	const std = @import("std");
4	const assert = std.debug.assert;
5	const math = std.math;
6
7	const bits = @import("bits.zig");
8	const encoder = @import("encoder.zig");
9	const Instruction = encoder.Instruction;
10	const Register = bits.Register;
11	const Rex = encoder.Rex;
12	const LegacyPrefixes = encoder.LegacyPrefixes;
13
14	const table = @import("encodings.zig").table;
15
16	mnemonic: Mnemonic,
17	op_en: OpEn,
18	op1: Op,
19	op2: Op,
20	op3: Op,
21	op4: Op,
22	opc_len: u2,
23	opc: [3]u8,
24	modrm_ext: u3,
25	mode: Mode,
26
27	pub fn findByMnemonic(mnemonic: Mnemonic, args: struct {
28	op1: Instruction.Operand,
29	op2: Instruction.Operand,
30	op3: Instruction.Operand,
31	op4: Instruction.Operand,
32	}) !?Encoding {
33	const input_op1 = Op.fromOperand(args.op1);
34	const input_op2 = Op.fromOperand(args.op2);
35	const input_op3 = Op.fromOperand(args.op3);
36	const input_op4 = Op.fromOperand(args.op4);
37
38	const ops = &[_]Instruction.Operand{ args.op1, args.op2, args.op3, args.op4 };
39	const rex_required = for (ops) \|op\| switch (op) {
40	.reg => \|r\| switch (r) {
41	.spl, .bpl, .sil, .dil => break true,
42	else => {},
43	},
44	else => {},
45	} else false;
46	const rex_invalid = for (ops) \|op\| switch (op) {
47	.reg => \|r\| switch (r) {
48	.ah, .bh, .ch, .dh => break true,
49	else => {},
50	},
51	else => {},
52	} else false;
53	const rex_extended = for (ops) \|op\| switch (op) {
54	.reg => \|r\| if (r.isExtended()) break true,
55	.mem => \|m\| {
56	if (m.base()) \|base\| {
57	if (base.isExtended()) break true;
58	}
59	if (m.scaleIndex()) \|si\| {
60	if (si.index.isExtended()) break true;
61	}
62	},
63	else => {},
64	} else false;
65
66	if ((rex_required or rex_extended) and rex_invalid) return error.CannotEncode;
67
68	// TODO work out what is the maximum number of variants we can actually find in one swoop.
69	var candidates: [10]Encoding = undefined;
70	var count: usize = 0;
71	for (table) \|entry\| {
72	const enc = Encoding{
73	.mnemonic = entry[0],
74	.op_en = entry[1],
75	.op1 = entry[2],
76	.op2 = entry[3],
77	.op3 = entry[4],
78	.op4 = entry[5],
79	.opc_len = entry[6],
80	.opc = .{ entry[7], entry[8], entry[9] },
81	.modrm_ext = entry[10],
82	.mode = entry[11],
83	};
84	if (enc.mnemonic == mnemonic and
85	input_op1.isSubset(enc.op1, enc.mode) and
86	input_op2.isSubset(enc.op2, enc.mode) and
87	input_op3.isSubset(enc.op3, enc.mode) and
88	input_op4.isSubset(enc.op4, enc.mode))
89	{
90	if (rex_required) {
91	switch (enc.mode) {
92	.rex, .long => {
93	candidates[count] = enc;
94	count += 1;
95	},
96	else => {},
97	}
98	} else {
99	if (enc.mode != .rex) {
100	candidates[count] = enc;
101	count += 1;
102	}
103	}
104	}
105	}
106
107	if (count == 0) return null;
108	if (count == 1) return candidates[0];
109
110	const EncodingLength = struct {
111	fn estimate(encoding: Encoding, params: struct {
112	op1: Instruction.Operand,
113	op2: Instruction.Operand,
114	op3: Instruction.Operand,
115	op4: Instruction.Operand,
116	}) usize {
117	var inst = Instruction{
118	.op1 = params.op1,
119	.op2 = params.op2,
120	.op3 = params.op3,
121	.op4 = params.op4,
122	.encoding = encoding,
123	};
124	var cwriter = std.io.countingWriter(std.io.null_writer);
125	inst.encode(cwriter.writer()) catch unreachable; // Not allowed to fail here unless OOM.
126	return @intCast(usize, cwriter.bytes_written);
127	}
128	};
129
130	var shortest_encoding: ?struct {
131	index: usize,
132	len: usize,
133	} = null;
134	var i: usize = 0;
135	while (i < count) : (i += 1) {
136	const candidate = candidates[i];
137	switch (candidate.mode) {
138	.long, .rex => if (rex_invalid) return error.CannotEncode,
139	else => {},
140	}
141
142	const len = EncodingLength.estimate(candidate, .{
143	.op1 = args.op1,
144	.op2 = args.op2,
145	.op3 = args.op3,
146	.op4 = args.op4,
147	});
148	const current = shortest_encoding orelse {
149	shortest_encoding = .{ .index = i, .len = len };
150	continue;
151	};
152	if (len < current.len) {
153	shortest_encoding = .{ .index = i, .len = len };
154	}
155	}
156
157	return candidates[shortest_encoding.?.index];
158	}
159
160	/// Returns first matching encoding by opcode.
161	pub fn findByOpcode(opc: []const u8, prefixes: struct {
162	legacy: LegacyPrefixes,
163	rex: Rex,
164	}, modrm_ext: ?u3) ?Encoding {
165	for (table) \|entry\| {
166	const enc = Encoding{
167	.mnemonic = entry[0],
168	.op_en = entry[1],
169	.op1 = entry[2],
170	.op2 = entry[3],
171	.op3 = entry[4],
172	.op4 = entry[5],
173	.opc_len = entry[6],
174	.opc = .{ entry[7], entry[8], entry[9] },
175	.modrm_ext = entry[10],
176	.mode = entry[11],
177	};
178	const match = match: {
179	if (modrm_ext) \|ext\| {
180	break :match ext == enc.modrm_ext and std.mem.eql(u8, enc.opcode(), opc);
181	}
182	break :match std.mem.eql(u8, enc.opcode(), opc);
183	};
184	if (match) {
185	if (prefixes.rex.w) {
186	switch (enc.mode) {
187	.fpu, .sse, .sse2, .none => {},
188	.long, .rex => return enc,
189	}
190	} else if (prefixes.rex.present and !prefixes.rex.isSet()) {
191	if (enc.mode == .rex) return enc;
192	} else if (prefixes.legacy.prefix_66) {
193	switch (enc.operandBitSize()) {
194	16 => return enc,
195	else => {},
196	}
197	} else {
198	if (enc.mode == .none) {
199	switch (enc.operandBitSize()) {
200	16 => {},
201	else => return enc,
202	}
203	}
204	}
205	}
206	}
207	return null;
208	}
209
210	pub fn opcode(encoding: *const Encoding) []const u8 {
211	return encoding.opc[0..encoding.opc_len];
212	}
213
214	pub fn mandatoryPrefix(encoding: *const Encoding) ?u8 {
215	const prefix = encoding.opc[0];
216	return switch (prefix) {
217	0x66, 0xf2, 0xf3 => prefix,
218	else => null,
219	};
220	}
221
222	pub fn modRmExt(encoding: Encoding) u3 {
223	return switch (encoding.op_en) {
224	.m, .mi, .m1, .mc => encoding.modrm_ext,
225	else => unreachable,
226	};
227	}
228
229	pub fn operandBitSize(encoding: Encoding) u64 {
230	if (encoding.mode == .long) return 64;
231	const bit_size: u64 = switch (encoding.op_en) {
232	.np => switch (encoding.op1) {
233	.o16 => 16,
234	.o32 => 32,
235	.o64 => 64,
236	else => 32,
237	},
238	.td => encoding.op2.bitSize(),
239	else => encoding.op1.bitSize(),
240	};
241	return bit_size;
242	}
243
244	pub fn format(
245	encoding: Encoding,
246	comptime fmt: []const u8,
247	options: std.fmt.FormatOptions,
248	writer: anytype,
249	) !void {
250	_ = options;
251	_ = fmt;
252	switch (encoding.mode) {
253	.long => try writer.writeAll("REX.W + "),
254	else => {},
255	}
256
257	for (encoding.opcode()) \|byte\| {
258	try writer.print("{x:0>2} ", .{byte});
259	}
260
261	switch (encoding.op_en) {
262	.np, .fd, .td, .i, .zi, .d => {},
263	.o, .oi => {
264	const tag = switch (encoding.op1) {
265	.r8 => "rb",
266	.r16 => "rw",
267	.r32 => "rd",
268	.r64 => "rd",
269	else => unreachable,
270	};
271	try writer.print("+{s} ", .{tag});
272	},
273	.m, .mi, .m1, .mc => try writer.print("/{d} ", .{encoding.modRmExt()}),
274	.mr, .rm, .rmi => try writer.writeAll("/r "),
275	}
276
277	switch (encoding.op_en) {
278	.i, .d, .zi, .oi, .mi, .rmi => {
279	const op = switch (encoding.op_en) {
280	.i, .d => encoding.op1,
281	.zi, .oi, .mi => encoding.op2,
282	.rmi => encoding.op3,
283	else => unreachable,
284	};
285	const tag = switch (op) {
286	.imm8, .imm8s => "ib",
287	.imm16, .imm16s => "iw",
288	.imm32, .imm32s => "id",
289	.imm64 => "io",
290	.rel8 => "cb",
291	.rel16 => "cw",
292	.rel32 => "cd",
293	else => unreachable,
294	};
295	try writer.print("{s} ", .{tag});
296	},
297	.np, .fd, .td, .o, .m, .m1, .mc, .mr, .rm => {},
298	}
299
300	try writer.print("{s} ", .{@tagName(encoding.mnemonic)});
301
302	const ops = &[_]Op{ encoding.op1, encoding.op2, encoding.op3, encoding.op4 };
303	for (ops) \|op\| switch (op) {
304	.none, .o16, .o32, .o64 => break,
305	else => try writer.print("{s} ", .{@tagName(op)}),
306	};
307
308	const op_en = switch (encoding.op_en) {
309	.zi => .i,
310	else => \|op_en\| op_en,
311	};
312	try writer.print("{s}", .{@tagName(op_en)});
313	}
314
315	pub const Mnemonic = enum {
316	// zig fmt: off
317	// General-purpose
318	adc, add, @"and",
319	call, cbw, cwde, cdqe, cwd, cdq, cqo, cmp,
320	cmova, cmovae, cmovb, cmovbe, cmovc, cmove, cmovg, cmovge, cmovl, cmovle, cmovna,
321	cmovnae, cmovnb, cmovnbe, cmovnc, cmovne, cmovng, cmovnge, cmovnl, cmovnle, cmovno,
322	cmovnp, cmovns, cmovnz, cmovo, cmovp, cmovpe, cmovpo, cmovs, cmovz,
323	div,
324	fisttp, fld,
325	idiv, imul, int3,
326	ja, jae, jb, jbe, jc, jrcxz, je, jg, jge, jl, jle, jna, jnae, jnb, jnbe,
327	jnc, jne, jng, jnge, jnl, jnle, jno, jnp, jns, jnz, jo, jp, jpe, jpo, js, jz,
328	jmp,
329	lea,
330	mov, movsx, movsxd, movzx, mul,
331	nop,
332	@"or",
333	pop, push,
334	ret,
335	sal, sar, sbb, shl, shr, sub, syscall,
336	seta, setae, setb, setbe, setc, sete, setg, setge, setl, setle, setna, setnae,
337	setnb, setnbe, setnc, setne, setng, setnge, setnl, setnle, setno, setnp, setns,
338	setnz, seto, setp, setpe, setpo, sets, setz,
339	@"test",
340	ud2,
341	xor,
342	// SSE
343	addss,
344	cmpss,
345	movss,
346	ucomiss,
347	// SSE2
348	addsd,
349	cmpsd,
350	movq, movsd,
351	ucomisd,
352	// zig fmt: on
353	};
354
355	pub const OpEn = enum {
356	// zig fmt: off
357	np,
358	o, oi,
359	i, zi,
360	d, m,
361	fd, td,
362	m1, mc, mi, mr, rm, rmi,
363	// zig fmt: on
364	};
365
366	pub const Op = enum {
367	// zig fmt: off
368	none,
369	o16, o32, o64,
370	unity,
371	imm8, imm16, imm32, imm64,
372	imm8s, imm16s, imm32s,
373	al, ax, eax, rax,
374	cl,
375	r8, r16, r32, r64,
376	rm8, rm16, rm32, rm64,
377	m8, m16, m32, m64, m80,
378	rel8, rel16, rel32,
379	m,
380	moffs,
381	sreg,
382	xmm, xmm_m32, xmm_m64,
383	// zig fmt: on
384
385	pub fn fromOperand(operand: Instruction.Operand) Op {
386	switch (operand) {
387	.none => return .none,
388
389	.reg => \|reg\| {
390	switch (reg.class()) {
391	.segment => return .sreg,
392	.floating_point => return switch (reg.bitSize()) {
393	128 => .xmm,
394	else => unreachable,
395	},
396	.general_purpose => {
397	if (reg.to64() == .rax) return switch (reg) {
398	.al => .al,
399	.ax => .ax,
400	.eax => .eax,
401	.rax => .rax,
402	else => unreachable,
403	};
404	if (reg == .cl) return .cl;
405	return switch (reg.bitSize()) {
406	8 => .r8,
407	16 => .r16,
408	32 => .r32,
409	64 => .r64,
410	else => unreachable,
411	};
412	},
413	}
414	},
415
416	.mem => \|mem\| switch (mem) {
417	.moffs => return .moffs,
418	.sib, .rip => {
419	const bit_size = mem.bitSize();
420	return switch (bit_size) {
421	8 => .m8,
422	16 => .m16,
423	32 => .m32,
424	64 => .m64,
425	80 => .m80,
426	else => unreachable,
427	};
428	},
429	},
430
431	.imm => \|imm\| {
432	switch (imm) {
433	.signed => \|x\| {
434	if (x == 1) return .unity;
435	if (math.cast(i8, x)) \|_\| return .imm8s;
436	if (math.cast(i16, x)) \|_\| return .imm16s;
437	return .imm32s;
438	},
439	.unsigned => \|x\| {
440	if (x == 1) return .unity;
441	if (math.cast(i8, x)) \|_\| return .imm8s;
442	if (math.cast(u8, x)) \|_\| return .imm8;
443	if (math.cast(i16, x)) \|_\| return .imm16s;
444	if (math.cast(u16, x)) \|_\| return .imm16;
445	if (math.cast(i32, x)) \|_\| return .imm32s;
446	if (math.cast(u32, x)) \|_\| return .imm32;
447	return .imm64;
448	},
449	}
450	},
451	}
452	}
453
454	pub fn bitSize(op: Op) u64 {
455	return switch (op) {
456	.none, .o16, .o32, .o64, .moffs, .m, .sreg => unreachable,
457	.unity => 1,
458	.imm8, .imm8s, .al, .cl, .r8, .m8, .rm8, .rel8 => 8,
459	.imm16, .imm16s, .ax, .r16, .m16, .rm16, .rel16 => 16,
460	.imm32, .imm32s, .eax, .r32, .m32, .rm32, .rel32, .xmm_m32 => 32,
461	.imm64, .rax, .r64, .m64, .rm64, .xmm_m64 => 64,
462	.m80 => 80,
463	.xmm => 128,
464	};
465	}
466
467	pub fn isSigned(op: Op) bool {
468	return switch (op) {
469	.unity, .imm8, .imm16, .imm32, .imm64 => false,
470	.imm8s, .imm16s, .imm32s => true,
471	else => unreachable,
472	};
473	}
474
475	pub fn isUnsigned(op: Op) bool {
476	return !op.isSigned();
477	}
478
479	pub fn isRegister(op: Op) bool {
480	// zig fmt: off
481	return switch (op) {
482	.cl,
483	.al, .ax, .eax, .rax,
484	.r8, .r16, .r32, .r64,
485	.rm8, .rm16, .rm32, .rm64,
486	.xmm, .xmm_m32, .xmm_m64,
487	=> true,
488	else => false,
489	};
490	// zig fmt: on
491	}
492
493	pub fn isImmediate(op: Op) bool {
494	// zig fmt: off
495	return switch (op) {
496	.imm8, .imm16, .imm32, .imm64,
497	.imm8s, .imm16s, .imm32s,
498	.rel8, .rel16, .rel32,
499	.unity,
500	=> true,
501	else => false,
502	};
503	// zig fmt: on
504	}
505
506	pub fn isMemory(op: Op) bool {
507	// zig fmt: off
508	return switch (op) {
509	.rm8, .rm16, .rm32, .rm64,
510	.m8, .m16, .m32, .m64, .m80,
511	.m,
512	.xmm_m32, .xmm_m64,
513	=> true,
514	else => false,
515	};
516	// zig fmt: on
517	}
518
519	pub fn isSegmentRegister(op: Op) bool {
520	return switch (op) {
521	.moffs, .sreg => true,
522	else => false,
523	};
524	}
525
526	pub fn isFloatingPointRegister(op: Op) bool {
527	return switch (op) {
528	.xmm, .xmm_m32, .xmm_m64 => true,
529	else => false,
530	};
531	}
532
533	/// Given an operand `op` checks if `target` is a subset for the purposes of the encoding.
534	pub fn isSubset(op: Op, target: Op, mode: Mode) bool {
535	switch (op) {
536	.m, .o16, .o32, .o64 => unreachable,
537	.moffs, .sreg => return op == target,
538	.none => switch (target) {
539	.o16, .o32, .o64, .none => return true,
540	else => return false,
541	},
542	else => {
543	if (op.isRegister() and target.isRegister()) {
544	switch (mode) {
545	.sse, .sse2 => return op.isFloatingPointRegister() and target.isFloatingPointRegister(),
546	else => switch (target) {
547	.cl, .al, .ax, .eax, .rax => return op == target,
548	else => return op.bitSize() == target.bitSize(),
549	},
550	}
551	}
552	if (op.isMemory() and target.isMemory()) {
553	switch (target) {
554	.m => return true,
555	else => return op.bitSize() == target.bitSize(),
556	}
557	}
558	if (op.isImmediate() and target.isImmediate()) {
559	switch (target) {
560	.imm64 => if (op.bitSize() <= 64) return true,
561	.imm32s, .rel32 => if (op.bitSize() < 32 or (op.bitSize() == 32 and op.isSigned()))
562	return true,
563	.imm32 => if (op.bitSize() <= 32) return true,
564	.imm16s, .rel16 => if (op.bitSize() < 16 or (op.bitSize() == 16 and op.isSigned()))
565	return true,
566	.imm16 => if (op.bitSize() <= 16) return true,
567	.imm8s, .rel8 => if (op.bitSize() < 8 or (op.bitSize() == 8 and op.isSigned()))
568	return true,
569	.imm8 => if (op.bitSize() <= 8) return true,
570	else => {},
571	}
572	return op == target;
573	}
574	return false;
575	},
576	}
577	}
578	};
579
580	pub const Mode = enum {
581	none,
582	fpu,
583	rex,
584	long,
585	sse,
586	sse2,
587	};

src/arch/x86_64/Mir.zig+358-454

...	@@ -12,9 +12,12 @@ const builtin = @import("builtin");	...	@@ -12,9 +12,12 @@ const builtin = @import("builtin");
12	const assert = std.debug.assert;	12	const assert = std.debug.assert;
13		13
14	const bits = @import("bits.zig");	14	const bits = @import("bits.zig");
		15	const encoder = @import("encoder.zig");
		16
15	const Air = @import("../../Air.zig");	17	const Air = @import("../../Air.zig");
16	const CodeGen = @import("CodeGen.zig");	18	const CodeGen = @import("CodeGen.zig");
17	const IntegerBitSet = std.bit_set.IntegerBitSet;	19	const IntegerBitSet = std.bit_set.IntegerBitSet;
		20	const Memory = bits.Memory;
18	const Register = bits.Register;	21	const Register = bits.Register;
19		22
20	instructions: std.MultiArrayList(Inst).Slice,	23	instructions: std.MultiArrayList(Inst).Slice,
...	@@ -24,431 +27,300 @@ extra: []const u32,	...	@@ -24,431 +27,300 @@ extra: []const u32,
24	pub const Inst = struct {	27	pub const Inst = struct {
25	tag: Tag,	28	tag: Tag,
26	ops: Ops,	29	ops: Ops,
27	/// The meaning of this depends on `tag` and `ops`.
28	data: Data,	30	data: Data,
29		31
30	pub const Tag = enum(u16) {	32	pub const Index = u32;
31	/// ops flags: form:
32	/// 0b00 reg1, reg2
33	/// 0b00 reg1, imm32
34	/// 0b01 reg1, [reg2 + imm32]
35	/// 0b01 reg1, [ds:imm32]
36	/// 0b10 [reg1 + imm32], reg2
37	/// Notes:
38	/// * If reg2 is `none` then it means Data field `imm` is used as the immediate.
39	/// * When two imm32 values are required, Data field `payload` points at `ImmPair`.
40	adc,
41
42	/// ops flags: form:
43	/// 0b00 byte ptr [reg1 + imm32], imm8
44	/// 0b01 word ptr [reg1 + imm32], imm16
45	/// 0b10 dword ptr [reg1 + imm32], imm32
46	/// 0b11 qword ptr [reg1 + imm32], imm32 (sign-extended to imm64)
47	/// Notes:
48	/// * Uses `ImmPair` as payload
49	adc_mem_imm,
50
51	/// form: reg1, [reg2 + scale*index + imm32]
52	/// ops flags scale
53	/// 0b00 1
54	/// 0b01 2
55	/// 0b10 4
56	/// 0b11 8
57	/// Notes:
58	/// * Uses `IndexRegisterDisp` as payload
59	adc_scale_src,
60
61	/// form: [reg1 + scale*index + imm32], reg2
62	/// ops flags scale
63	/// 0b00 1
64	/// 0b01 2
65	/// 0b10 4
66	/// 0b11 8
67	/// Notes:
68	/// * Uses `IndexRegisterDisp` payload.
69	adc_scale_dst,
70
71	/// form: [reg1 + scale*rax + imm32], imm32
72	/// ops flags scale
73	/// 0b00 1
74	/// 0b01 2
75	/// 0b10 4
76	/// 0b11 8
77	/// Notes:
78	/// * Uses `IndexRegisterDispImm` payload.
79	adc_scale_imm,
80
81	/// ops flags: form:
82	/// 0b00 byte ptr [reg1 + index + imm32], imm8
83	/// 0b01 word ptr [reg1 + index + imm32], imm16
84	/// 0b10 dword ptr [reg1 + index + imm32], imm32
85	/// 0b11 qword ptr [reg1 + index + imm32], imm32 (sign-extended to imm64)
86	/// Notes:
87	/// * Uses `IndexRegisterDispImm` payload.
88	adc_mem_index_imm,
89
90	// The following instructions all have the same encoding as `adc`.
91		33
		34	pub const Tag = enum(u8) {
		35	/// Add with carry
		36	adc,
		37	/// Add
92	add,	38	add,
93	add_mem_imm,	39	/// Logical and
94	add_scale_src,
95	add_scale_dst,
96	add_scale_imm,
97	add_mem_index_imm,
98	sub,
99	sub_mem_imm,
100	sub_scale_src,
101	sub_scale_dst,
102	sub_scale_imm,
103	sub_mem_index_imm,
104	xor,
105	xor_mem_imm,
106	xor_scale_src,
107	xor_scale_dst,
108	xor_scale_imm,
109	xor_mem_index_imm,
110	@"and",	40	@"and",
111	and_mem_imm,	41	/// Call
112	and_scale_src,	42	call,
113	and_scale_dst,	43	/// Convert byte to word
114	and_scale_imm,	44	cbw,
115	and_mem_index_imm,	45	/// Convert word to doubleword
116	@"or",	46	cwde,
117	or_mem_imm,	47	/// Convert doubleword to quadword
118	or_scale_src,	48	cdqe,
119	or_scale_dst,	49	/// Convert word to doubleword
120	or_scale_imm,	50	cwd,
121	or_mem_index_imm,	51	/// Convert doubleword to quadword
122	rol,	52	cdq,
123	rol_mem_imm,	53	/// Convert doubleword to quadword
124	rol_scale_src,	54	cqo,
125	rol_scale_dst,	55	/// Logical compare
126	rol_scale_imm,
127	rol_mem_index_imm,
128	ror,
129	ror_mem_imm,
130	ror_scale_src,
131	ror_scale_dst,
132	ror_scale_imm,
133	ror_mem_index_imm,
134	rcl,
135	rcl_mem_imm,
136	rcl_scale_src,
137	rcl_scale_dst,
138	rcl_scale_imm,
139	rcl_mem_index_imm,
140	rcr,
141	rcr_mem_imm,
142	rcr_scale_src,
143	rcr_scale_dst,
144	rcr_scale_imm,
145	rcr_mem_index_imm,
146	sbb,
147	sbb_mem_imm,
148	sbb_scale_src,
149	sbb_scale_dst,
150	sbb_scale_imm,
151	sbb_mem_index_imm,
152	cmp,	56	cmp,
153	cmp_mem_imm,	57	/// Unsigned division
154	cmp_scale_src,
155	cmp_scale_dst,
156	cmp_scale_imm,
157	cmp_mem_index_imm,
158	mov,
159	mov_mem_imm,
160	mov_scale_src,
161	mov_scale_dst,
162	mov_scale_imm,
163	mov_mem_index_imm,
164
165	/// ops flags: form:
166	/// 0b00 reg1, reg2,
167	/// 0b01 reg1, byte ptr [reg2 + imm32]
168	/// 0b10 reg1, word ptr [reg2 + imm32]
169	/// 0b11 reg1, dword ptr [reg2 + imm32]
170	mov_sign_extend,
171
172	/// ops flags: form:
173	/// 0b00 reg1, reg2
174	/// 0b01 reg1, byte ptr [reg2 + imm32]
175	/// 0b10 reg1, word ptr [reg2 + imm32]
176	mov_zero_extend,
177
178	/// ops flags: form:
179	/// 0b00 reg1, [reg2 + imm32]
180	/// 0b00 reg1, [ds:imm32]
181	/// 0b01 reg1, [rip + imm32]
182	/// 0b10 reg1, [reg2 + index + imm32]
183	/// Notes:
184	/// * 0b10 uses `IndexRegisterDisp` payload
185	lea,
186
187	/// ops flags: form:
188	/// 0b00 reg1, [rip + reloc] // via GOT PIC
189	/// 0b01 reg1, [rip + reloc] // direct load PIC
190	/// 0b10 reg1, [rip + reloc] // via imports table PIC
191	/// Notes:
192	/// * `Data` contains `relocation`
193	lea_pic,
194
195	/// ops flags: form:
196	/// 0b00 reg1, 1
197	/// 0b01 reg1, .cl
198	/// 0b10 reg1, imm8
199	/// Notes:
200	/// * If flags == 0b10, uses `imm`.
201	shl,
202	shl_mem_imm,
203	shl_scale_src,
204	shl_scale_dst,
205	shl_scale_imm,
206	shl_mem_index_imm,
207	sal,
208	sal_mem_imm,
209	sal_scale_src,
210	sal_scale_dst,
211	sal_scale_imm,
212	sal_mem_index_imm,
213	shr,
214	shr_mem_imm,
215	shr_scale_src,
216	shr_scale_dst,
217	shr_scale_imm,
218	shr_mem_index_imm,
219	sar,
220	sar_mem_imm,
221	sar_scale_src,
222	sar_scale_dst,
223	sar_scale_imm,
224	sar_mem_index_imm,
225
226	/// ops flags: form:
227	/// 0b00 reg1
228	/// 0b00 byte ptr [reg2 + imm32]
229	/// 0b01 word ptr [reg2 + imm32]
230	/// 0b10 dword ptr [reg2 + imm32]
231	/// 0b11 qword ptr [reg2 + imm32]
232	imul,
233	idiv,
234	mul,
235	div,	58	div,
236		59	/// Store integer with truncation
237	/// ops flags: form:
238	/// 0b00 AX <- AL
239	/// 0b01 DX:AX <- AX
240	/// 0b10 EDX:EAX <- EAX
241	/// 0b11 RDX:RAX <- RAX
242	cwd,
243
244	/// ops flags: form:
245	/// 0b00 reg1, reg2
246	/// 0b01 reg1, [reg2 + imm32]
247	/// 0b01 reg1, [imm32] if reg2 is none
248	/// 0b10 reg1, reg2, imm32
249	/// 0b11 reg1, [reg2 + imm32], imm32
250	imul_complex,
251
252	/// ops flags: form:
253	/// 0b00 reg1, imm64
254	/// 0b01 rax, moffs64
255	/// Notes:
256	/// * If reg1 is 64-bit, the immediate is 64-bit and stored
257	/// within extra data `Imm64`.
258	/// * For 0b01, reg1 (or reg2) need to be
259	/// a version of rax. If reg1 == .none, then reg2 == .rax,
260	/// or vice versa.
261	movabs,
262
263	/// ops flags: form:
264	/// 0b00 word ptr [reg1 + imm32]
265	/// 0b01 dword ptr [reg1 + imm32]
266	/// 0b10 qword ptr [reg1 + imm32]
267	/// Notes:
268	/// * source is always ST(0)
269	/// * only supports memory operands as destination
270	fisttp,	60	fisttp,
271		61	/// Load floating-point value
272	/// ops flags: form:
273	/// 0b01 dword ptr [reg1 + imm32]
274	/// 0b10 qword ptr [reg1 + imm32]
275	fld,	62	fld,
276		63	/// Signed division
277	/// ops flags: form:	64	idiv,
278	/// 0b00 inst	65	/// Signed multiplication
279	/// 0b01 reg1	66	imul,
280	/// 0b01 [imm32] if reg1 is none	67	///
281	/// 0b10 [reg1 + imm32]	68	int3,
		69	/// Jump
282	jmp,	70	jmp,
283	call,	71	/// Load effective address
284		72	lea,
285	/// ops flags:	73	/// Move
286	/// unused	74	mov,
287	/// Notes:	75	/// Move with sign extension
288	/// * uses `inst_cc` in Data.	76	movsx,
289	cond_jmp,	77	/// Move with zero extension
290		78	movzx,
291	/// ops flags:	79	/// Multiply
292	/// 0b00 reg1	80	mul,
293	/// Notes:	81	/// No-op
294	/// * uses condition code (CC) stored as part of data	82	nop,
295	cond_set_byte,	83	/// Logical or
296		84	@"or",
297	/// ops flags:	85	/// Pop
298	/// 0b00 reg1, reg2,
299	/// 0b01 reg1, word ptr [reg2 + imm]
300	/// 0b10 reg1, dword ptr [reg2 + imm]
301	/// 0b11 reg1, qword ptr [reg2 + imm]
302	/// Notes:
303	/// * uses condition code (CC) stored as part of data
304	cond_mov,
305
306	/// ops flags: form:
307	/// 0b00 reg1
308	/// 0b01 [reg1 + imm32]
309	/// 0b10 imm32
310	/// Notes:
311	/// * If 0b10 is specified and the tag is push, pushes immediate onto the stack
312	/// using the mnemonic PUSH imm32.
313	push,
314	pop,	86	pop,
315		87	/// Push
316	/// ops flags: form:	88	push,
317	/// 0b00 retf imm16	89	/// Return
318	/// 0b01 retf
319	/// 0b10 retn imm16
320	/// 0b11 retn
321	ret,	90	ret,
322		91	/// Arithmetic shift left
323	/// Fast system call	92	sal,
		93	/// Arithmetic shift right
		94	sar,
		95	/// Integer subtraction with borrow
		96	sbb,
		97	/// Logical shift left
		98	shl,
		99	/// Logical shift right
		100	shr,
		101	/// Subtract
		102	sub,
		103	/// Syscall
324	syscall,	104	syscall,
325		105	/// Test condition
326	/// ops flags: form:
327	/// 0b00 reg1, imm32 if reg2 == .none
328	/// 0b00 reg1, reg2
329	/// TODO handle more cases
330	@"test",	106	@"test",
		107	/// Undefined instruction
		108	ud2,
		109	/// Logical exclusive-or
		110	xor,
331		111
332	/// Undefined Instruction	112	/// Add single precision floating point
333	ud,	113	addss,
334		114	/// Compare scalar single-precision floating-point values
335	/// Breakpoint form:	115	cmpss,
336	/// 0b00 int3	116	/// Move scalar single-precision floating-point value
337	interrupt,	117	movss,
338		118	/// Unordered compare scalar single-precision floating-point values
339	/// Nop	119	ucomiss,
340	nop,	120	/// Add double precision floating point
341		121	addsd,
342	/// SSE instructions	122	/// Compare scalar double-precision floating-point values
343	/// ops flags: form:	123	cmpsd,
344	/// 0b00 reg1, qword ptr [reg2 + imm32]	124	/// Move scalar double-precision floating-point value
345	/// 0b01 qword ptr [reg1 + imm32], reg2	125	movsd,
346	/// 0b10 reg1, reg2	126	/// Unordered compare scalar double-precision floating-point values
347	mov_f64_sse,	127	ucomisd,
348	mov_f32_sse,	128
349		129	/// Conditional move
350	/// ops flags: form:	130	cmovcc,
351	/// 0b00 reg1, reg2	131	/// Conditional jump
352	add_f64_sse,	132	jcc,
353	add_f32_sse,	133	/// Set byte on condition
354		134	setcc,
355	/// ops flags: form:	135
356	/// 0b00 reg1, reg2	136	/// Mov absolute to/from memory wrt segment register to/from rax
357	cmp_f64_sse,	137	mov_moffs,
358	cmp_f32_sse,	138
359		139	/// Jump with relocation to another local MIR instruction
360	/// AVX instructions	140	/// Uses `inst` payload.
361	/// ops flags: form:	141	jmp_reloc,
362	/// 0b00 reg1, qword ptr [reg2 + imm32]	142
363	/// 0b01 qword ptr [reg1 + imm32], reg2	143	/// Call to an extern symbol via linker relocation.
364	/// 0b10 reg1, reg1, reg2	144	/// Uses `relocation` payload.
365	mov_f64_avx,
366	mov_f32_avx,
367
368	/// ops flags: form:
369	/// 0b00 reg1, reg1, reg2
370	add_f64_avx,
371	add_f32_avx,
372
373	/// ops flags: form:
374	/// 0b00 reg1, reg1, reg2
375	cmp_f64_avx,
376	cmp_f32_avx,
377
378	/// Pseudo-instructions
379	/// call extern function
380	/// Notes:
381	/// * target of the call is stored as `relocation` in `Data` union.
382	call_extern,	145	call_extern,
383		146
384	/// end of prologue	147	/// Load effective address of a symbol not yet allocated in VM.
385	dbg_prologue_end,	148	lea_linker,
386		149
387	/// start of epilogue	150	/// End of prologue
		151	dbg_prologue_end,
		152	/// Start of epilogue
388	dbg_epilogue_begin,	153	dbg_epilogue_begin,
389		154	/// Update debug line
390	/// update debug line	155	/// Uses `payload` payload with data of type `DbgLineColumn`.
391	dbg_line,	156	dbg_line,
392		157	/// Push registers
393	/// push registers	158	/// Uses `payload` payload with data of type `SaveRegisterList`.
394	/// Uses `payload` field with `SaveRegisterList` as payload.
395	push_regs,	159	push_regs,
396		160	/// Pop registers
397	/// pop registers	161	/// Uses `payload` payload with data of type `SaveRegisterList`.
398	/// Uses `payload` field with `SaveRegisterList` as payload.
399	pop_regs,	162	pop_regs,
400	};
401	/// The position of an MIR instruction within the `Mir` instructions array.
402	pub const Index = u32;
403		163
404	pub const Ops = packed struct {	164	/// Tombstone
405	reg1: u7,	165	/// Emitter should skip this instruction.
406	reg2: u7,	166	dead,
407	flags: u2,	167	};
408
409	pub fn encode(vals: struct {
410	reg1: Register = .none,
411	reg2: Register = .none,
412	flags: u2 = 0b00,
413	}) Ops {
414	return .{
415	.reg1 = @enumToInt(vals.reg1),
416	.reg2 = @enumToInt(vals.reg2),
417	.flags = vals.flags,
418	};
419	}
420		168
421	pub fn decode(ops: Ops) struct {	169	pub const Ops = enum(u8) {
422	reg1: Register,	170	/// No data associated with this instruction (only mnemonic is used).
423	reg2: Register,	171	none,
424	flags: u2,	172	/// Single register operand.
425	} {	173	/// Uses `r` payload.
426	return .{	174	r,
427	.reg1 = @intToEnum(Register, ops.reg1),	175	/// Register, register operands.
428	.reg2 = @intToEnum(Register, ops.reg2),	176	/// Uses `rr` payload.
429	.flags = ops.flags,	177	rr,
430	};	178	/// Register, register, register operands.
431	}	179	/// Uses `rrr` payload.
		180	rrr,
		181	/// Register, register, immediate (sign-extended) operands.
		182	/// Uses `rri` payload.
		183	rri_s,
		184	/// Register, register, immediate (unsigned) operands.
		185	/// Uses `rri` payload.
		186	rri_u,
		187	/// Register with condition code (CC).
		188	/// Uses `r_c` payload.
		189	r_c,
		190	/// Register, register with condition code (CC).
		191	/// Uses `rr_c` payload.
		192	rr_c,
		193	/// Register, immediate (sign-extended) operands.
		194	/// Uses `ri` payload.
		195	ri_s,
		196	/// Register, immediate (unsigned) operands.
		197	/// Uses `ri` payload.
		198	ri_u,
		199	/// Register, 64-bit unsigned immediate operands.
		200	/// Uses `rx` payload with payload type `Imm64`.
		201	ri64,
		202	/// Immediate (sign-extended) operand.
		203	/// Uses `imm` payload.
		204	imm_s,
		205	/// Immediate (unsigned) operand.
		206	/// Uses `imm` payload.
		207	imm_u,
		208	/// Relative displacement operand.
		209	/// Uses `imm` payload.
		210	rel,
		211	/// Register, memory (SIB) operands.
		212	/// Uses `rx` payload.
		213	rm_sib,
		214	/// Register, memory (RIP) operands.
		215	/// Uses `rx` payload.
		216	rm_rip,
		217	/// Single memory (SIB) operand.
		218	/// Uses `payload` with extra data of type `MemorySib`.
		219	m_sib,
		220	/// Single memory (RIP) operand.
		221	/// Uses `payload` with extra data of type `MemoryRip`.
		222	m_rip,
		223	/// Memory (SIB), immediate (unsigned) operands.
		224	/// Uses `xi` payload with extra data of type `MemorySib`.
		225	mi_u_sib,
		226	/// Memory (RIP), immediate (unsigned) operands.
		227	/// Uses `xi` payload with extra data of type `MemoryRip`.
		228	mi_u_rip,
		229	/// Memory (SIB), immediate (sign-extend) operands.
		230	/// Uses `xi` payload with extra data of type `MemorySib`.
		231	mi_s_sib,
		232	/// Memory (RIP), immediate (sign-extend) operands.
		233	/// Uses `xi` payload with extra data of type `MemoryRip`.
		234	mi_s_rip,
		235	/// Memory (SIB), register operands.
		236	/// Uses `rx` payload with extra data of type `MemorySib`.
		237	mr_sib,
		238	/// Memory (RIP), register operands.
		239	/// Uses `rx` payload with extra data of type `MemoryRip`.
		240	mr_rip,
		241	/// Rax, Memory moffs.
		242	/// Uses `payload` with extra data of type `MemoryMoffs`.
		243	rax_moffs,
		244	/// Memory moffs, rax.
		245	/// Uses `payload` with extra data of type `MemoryMoffs`.
		246	moffs_rax,
		247	/// References another Mir instruction directly.
		248	/// Uses `inst` payload.
		249	inst,
		250	/// References another Mir instruction directly with condition code (CC).
		251	/// Uses `inst_cc` payload.
		252	inst_cc,
		253	/// Uses `payload` payload with data of type `MemoryConditionCode`.
		254	m_cc,
		255	/// Uses `rx` payload with extra data of type `MemoryConditionCode`.
		256	rm_cc,
		257	/// Uses `reloc` payload.
		258	reloc,
		259	/// Linker relocation - GOT indirection.
		260	/// Uses `payload` payload with extra data of type `LeaRegisterReloc`.
		261	got_reloc,
		262	/// Linker relocation - direct reference.
		263	/// Uses `payload` payload with extra data of type `LeaRegisterReloc`.
		264	direct_reloc,
		265	/// Linker relocation - imports table indirection (binding).
		266	/// Uses `payload` payload with extra data of type `LeaRegisterReloc`.
		267	import_reloc,
432	};	268	};
433		269
434	/// All instructions have a 4-byte payload, which is contained within
435	/// this union. `Tag` determines which union field is active, as well as
436	/// how to interpret the data within.
437	pub const Data = union {	270	pub const Data = union {
438	/// Another instruction.	271	/// References another Mir instruction.
439	inst: Index,	272	inst: Index,
440	/// A 32-bit immediate value.	273	/// Another instruction with condition code (CC).
441	imm: u32,	274	/// Used by `jcc`.
442	/// A condition code for use with EFLAGS register.
443	cc: bits.Condition,
444	/// Another instruction with condition code.
445	/// Used by `cond_jmp`.
446	inst_cc: struct {	275	inst_cc: struct {
447	/// Another instruction.	276	/// Another instruction.
448	inst: Index,	277	inst: Index,
449	/// A condition code for use with EFLAGS register.	278	/// A condition code for use with EFLAGS register.
450	cc: bits.Condition,	279	cc: bits.Condition,
451	},	280	},
		281	/// A 32-bit immediate value.
		282	imm: u32,
		283	r: Register,
		284	rr: struct {
		285	r1: Register,
		286	r2: Register,
		287	},
		288	rrr: struct {
		289	r1: Register,
		290	r2: Register,
		291	r3: Register,
		292	},
		293	rri: struct {
		294	r1: Register,
		295	r2: Register,
		296	imm: u32,
		297	},
		298	/// Register with condition code (CC).
		299	r_c: struct {
		300	r1: Register,
		301	cc: bits.Condition,
		302	},
		303	/// Register, register with condition code (CC).
		304	rr_c: struct {
		305	r1: Register,
		306	r2: Register,
		307	cc: bits.Condition,
		308	},
		309	/// Register, immediate.
		310	ri: struct {
		311	r1: Register,
		312	imm: u32,
		313	},
		314	/// Register, followed by custom payload found in extra.
		315	rx: struct {
		316	r1: Register,
		317	payload: u32,
		318	},
		319	/// Custom payload followed by an immediate.
		320	xi: struct {
		321	payload: u32,
		322	imm: u32,
		323	},
452	/// Relocation for the linker where:	324	/// Relocation for the linker where:
453	/// * `atom_index` is the index of the source	325	/// * `atom_index` is the index of the source
454	/// * `sym_index` is the index of the target	326	/// * `sym_index` is the index of the target
...	@@ -471,62 +343,13 @@ pub const Inst = struct {	...	@@ -471,62 +343,13 @@ pub const Inst = struct {
471	}	343	}
472	};	344	};
473		345
474	pub const IndexRegisterDisp = struct {	346	pub const LeaRegisterReloc = struct {
475	/// Index register to use with SIB-based encoding	347	/// Destination register.
476	index: u32,	348	reg: u32,
477		349	/// Index of the containing atom.
478	/// Displacement value	350	atom_index: u32,
479	disp: u32,	351	/// Index into the linker's symbol table.
480		352	sym_index: u32,
481	pub fn encode(index: Register, disp: u32) IndexRegisterDisp {
482	return .{
483	.index = @enumToInt(index),
484	.disp = disp,
485	};
486	}
487
488	pub fn decode(this: IndexRegisterDisp) struct {
489	index: Register,
490	disp: u32,
491	} {
492	return .{
493	.index = @intToEnum(Register, this.index),
494	.disp = this.disp,
495	};
496	}
497	};
498
499	/// TODO: would it be worth making `IndexRegisterDisp` and `IndexRegisterDispImm` a variable length list
500	/// instead of having two structs, one a superset of the other one?
501	pub const IndexRegisterDispImm = struct {
502	/// Index register to use with SIB-based encoding
503	index: u32,
504
505	/// Displacement value
506	disp: u32,
507
508	/// Immediate
509	imm: u32,
510
511	pub fn encode(index: Register, disp: u32, imm: u32) IndexRegisterDispImm {
512	return .{
513	.index = @enumToInt(index),
514	.disp = disp,
515	.imm = imm,
516	};
517	}
518
519	pub fn decode(this: IndexRegisterDispImm) struct {
520	index: Register,
521	disp: u32,
522	imm: u32,
523	} {
524	return .{
525	.index = @intToEnum(Register, this.index),
526	.disp = this.disp,
527	.imm = this.imm,
528	};
529	}
530	};	353	};
531		354
532	/// Used in conjunction with `SaveRegisterList` payload to transfer a list of used registers	355	/// Used in conjunction with `SaveRegisterList` payload to transfer a list of used registers
...	@@ -570,16 +393,13 @@ pub const RegisterList = struct {	...	@@ -570,16 +393,13 @@ pub const RegisterList = struct {
570	};	393	};
571		394
572	pub const SaveRegisterList = struct {	395	pub const SaveRegisterList = struct {
		396	/// Base register
		397	base_reg: u32,
573	/// Use `RegisterList` to populate.	398	/// Use `RegisterList` to populate.
574	register_list: u32,	399	register_list: u32,
575	stack_end: u32,	400	stack_end: u32,
576	};	401	};
577		402
578	pub const ImmPair = struct {
579	dest_off: u32,
580	operand: u32,
581	};
582
583	pub const Imm64 = struct {	403	pub const Imm64 = struct {
584	msb: u32,	404	msb: u32,
585	lsb: u32,	405	lsb: u32,
...	@@ -599,6 +419,90 @@ pub const Imm64 = struct {	...	@@ -599,6 +419,90 @@ pub const Imm64 = struct {
599	}	419	}
600	};	420	};
601		421
		422	// TODO this can be further compacted using packed struct
		423	pub const MemorySib = struct {
		424	/// Size of the pointer.
		425	ptr_size: u32,
		426	/// Base register. -1 means null, or no base register.
		427	base: i32,
		428	/// Scale for index register. -1 means null, or no scale.
		429	/// This has to be in sync with `index` field.
		430	scale: i32,
		431	/// Index register. -1 means null, or no index register.
		432	/// This has to be in sync with `scale` field.
		433	index: i32,
		434	/// Displacement value.
		435	disp: i32,
		436
		437	pub fn encode(mem: Memory) MemorySib {
		438	const sib = mem.sib;
		439	return .{
		440	.ptr_size = @enumToInt(sib.ptr_size),
		441	.base = if (sib.base) \|r\| @enumToInt(r) else -1,
		442	.scale = if (sib.scale_index) \|si\| si.scale else -1,
		443	.index = if (sib.scale_index) \|si\| @enumToInt(si.index) else -1,
		444	.disp = sib.disp,
		445	};
		446	}
		447
		448	pub fn decode(msib: MemorySib) Memory {
		449	const base: ?Register = if (msib.base == -1) null else @intToEnum(Register, msib.base);
		450	const scale_index: ?Memory.ScaleIndex = if (msib.index == -1) null else .{
		451	.scale = @intCast(u4, msib.scale),
		452	.index = @intToEnum(Register, msib.index),
		453	};
		454	const mem: Memory = .{ .sib = .{
		455	.ptr_size = @intToEnum(Memory.PtrSize, msib.ptr_size),
		456	.base = base,
		457	.scale_index = scale_index,
		458	.disp = msib.disp,
		459	} };
		460	return mem;
		461	}
		462	};
		463
		464	pub const MemoryRip = struct {
		465	/// Size of the pointer.
		466	ptr_size: u32,
		467	/// Displacement value.
		468	disp: i32,
		469
		470	pub fn encode(mem: Memory) MemoryRip {
		471	return .{
		472	.ptr_size = @enumToInt(mem.rip.ptr_size),
		473	.disp = mem.rip.disp,
		474	};
		475	}
		476
		477	pub fn decode(mrip: MemoryRip) Memory {
		478	return .{ .rip = .{
		479	.ptr_size = @intToEnum(Memory.PtrSize, mrip.ptr_size),
		480	.disp = mrip.disp,
		481	} };
		482	}
		483	};
		484
		485	pub const MemoryMoffs = struct {
		486	/// Segment register.
		487	seg: u32,
		488	/// Absolute offset wrt to the segment register split between MSB and LSB parts much like
		489	/// `Imm64` payload.
		490	msb: u32,
		491	lsb: u32,
		492
		493	pub fn encodeOffset(moffs: *MemoryMoffs, v: u64) void {
		494	moffs.msb = @truncate(u32, v >> 32);
		495	moffs.lsb = @truncate(u32, v);
		496	}
		497
		498	pub fn decodeOffset(moffs: *const MemoryMoffs) u64 {
		499	var res: u64 = 0;
		500	res \|= (@intCast(u64, moffs.msb) << 32);
		501	res \|= @intCast(u64, moffs.lsb);
		502	return res;
		503	}
		504	};
		505
602	pub const DbgLineColumn = struct {	506	pub const DbgLineColumn = struct {
603	line: u32,	507	line: u32,
604	column: u32,	508	column: u32,
...	@@ -610,9 +514,9 @@ pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void {	...	@@ -610,9 +514,9 @@ pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void {
610	mir.* = undefined;	514	mir.* = undefined;
611	}	515	}
612		516
613	pub fn extraData(mir: Mir, comptime T: type, index: usize) struct { data: T, end: usize } {	517	pub fn extraData(mir: Mir, comptime T: type, index: u32) struct { data: T, end: u32 } {
614	const fields = std.meta.fields(T);	518	const fields = std.meta.fields(T);
615	var i: usize = index;	519	var i: u32 = index;
616	var result: T = undefined;	520	var result: T = undefined;
617	inline for (fields) \|field\| {	521	inline for (fields) \|field\| {
618	@field(result, field.name) = switch (field.type) {	522	@field(result, field.name) = switch (field.type) {

src/arch/x86_64/bits.zig+301-845

...	@@ -1,9 +1,9 @@	...	@@ -1,9 +1,9 @@
1	const std = @import("std");	1	const std = @import("std");
2	const testing = std.testing;
3	const mem = std.mem;
4	const assert = std.debug.assert;	2	const assert = std.debug.assert;
5	const ArrayList = std.ArrayList;	3	const expect = std.testing.expect;
		4
6	const Allocator = std.mem.Allocator;	5	const Allocator = std.mem.Allocator;
		6	const ArrayList = std.ArrayList;
7	const DW = std.dwarf;	7	const DW = std.dwarf;
8		8
9	/// EFLAGS condition codes	9	/// EFLAGS condition codes
...	@@ -135,131 +135,199 @@ pub const Condition = enum(u5) {	...	@@ -135,131 +135,199 @@ pub const Condition = enum(u5) {
135	}	135	}
136	};	136	};
137		137
138	/// Definitions of all of the general purpose x64 registers. The order is semantically meaningful.
139	/// The registers are defined such that IDs go in descending order of 64-bit,
140	/// 32-bit, 16-bit, and then 8-bit, and each set contains exactly sixteen
141	/// registers. This results in some useful properties:
142	///
143	/// Any 64-bit register can be turned into its 32-bit form by adding 16, and
144	/// vice versa. This also works between 32-bit and 16-bit forms. With 8-bit, it
145	/// works for all except for sp, bp, si, and di, which do not have an 8-bit
146	/// form.
147	///
148	/// If (register & 8) is set, the register is extended.
149	///
150	/// The ID can be easily determined by figuring out what range the register is
151	/// in, and then subtracting the base.
152	pub const Register = enum(u7) {	138	pub const Register = enum(u7) {
153	// zig fmt: off	139	// zig fmt: off
154	// 0 through 15, 64-bit registers. 8-15 are extended.
155	// id is just the int value.
156	rax, rcx, rdx, rbx, rsp, rbp, rsi, rdi,	140	rax, rcx, rdx, rbx, rsp, rbp, rsi, rdi,
157	r8, r9, r10, r11, r12, r13, r14, r15,	141	r8, r9, r10, r11, r12, r13, r14, r15,
158		142
159	// 16 through 31, 32-bit registers. 24-31 are extended.
160	// id is int value - 16.
161	eax, ecx, edx, ebx, esp, ebp, esi, edi,	143	eax, ecx, edx, ebx, esp, ebp, esi, edi,
162	r8d, r9d, r10d, r11d, r12d, r13d, r14d, r15d,	144	r8d, r9d, r10d, r11d, r12d, r13d, r14d, r15d,
163		145
164	// 32-47, 16-bit registers. 40-47 are extended.
165	// id is int value - 32.
166	ax, cx, dx, bx, sp, bp, si, di,	146	ax, cx, dx, bx, sp, bp, si, di,
167	r8w, r9w, r10w, r11w, r12w, r13w, r14w, r15w,	147	r8w, r9w, r10w, r11w, r12w, r13w, r14w, r15w,
168		148
169	// 48-63, 8-bit registers. 56-63 are extended.	149	al, cl, dl, bl, spl, bpl, sil, dil,
170	// id is int value - 48.
171	al, cl, dl, bl, ah, ch, dh, bh,
172	r8b, r9b, r10b, r11b, r12b, r13b, r14b, r15b,	150	r8b, r9b, r10b, r11b, r12b, r13b, r14b, r15b,
173		151
174	// 64-79, 256-bit registers.	152	ah, ch, dh, bh,
175	// id is int value - 64.	153
176	ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7,	154	ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7,
177	ymm8, ymm9, ymm10, ymm11, ymm12, ymm13, ymm14, ymm15,	155	ymm8, ymm9, ymm10, ymm11, ymm12, ymm13, ymm14, ymm15,
178		156
179	// 80-95, 128-bit registers.
180	// id is int value - 80.
181	xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7,	157	xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7,
182	xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15,	158	xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15,
183		159
184	// Pseudo-value for MIR instructions.	160	es, cs, ss, ds, fs, gs,
		161
185	none,	162	none,
186	// zig fmt: on	163	// zig fmt: on
187		164
188	pub fn id(self: Register) u7 {	165	pub const Class = enum(u2) {
189	return switch (@enumToInt(self)) {	166	general_purpose,
190	0...63 => @as(u7, @truncate(u4, @enumToInt(self))),	167	floating_point,
191	64...79 => @enumToInt(self),	168	segment,
		169	};
		170
		171	pub fn class(reg: Register) Class {
		172	return switch (@enumToInt(reg)) {
		173	// zig fmt: off
		174	@enumToInt(Register.rax) ... @enumToInt(Register.r15) => .general_purpose,
		175	@enumToInt(Register.eax) ... @enumToInt(Register.r15d) => .general_purpose,
		176	@enumToInt(Register.ax) ... @enumToInt(Register.r15w) => .general_purpose,
		177	@enumToInt(Register.al) ... @enumToInt(Register.r15b) => .general_purpose,
		178	@enumToInt(Register.ah) ... @enumToInt(Register.bh) => .general_purpose,
		179
		180	@enumToInt(Register.ymm0) ... @enumToInt(Register.ymm15) => .floating_point,
		181	@enumToInt(Register.xmm0) ... @enumToInt(Register.xmm15) => .floating_point,
		182
		183	@enumToInt(Register.es) ... @enumToInt(Register.gs) => .segment,
		184
192	else => unreachable,	185	else => unreachable,
		186	// zig fmt: on
193	};	187	};
194	}	188	}
195		189
196	/// Returns the bit-width of the register.	190	pub fn id(reg: Register) u6 {
197	pub fn size(self: Register) u9 {	191	const base = switch (@enumToInt(reg)) {
198	return switch (@enumToInt(self)) {	192	// zig fmt: off
199	0...15 => 64,	193	@enumToInt(Register.rax) ... @enumToInt(Register.r15) => @enumToInt(Register.rax),
200	16...31 => 32,	194	@enumToInt(Register.eax) ... @enumToInt(Register.r15d) => @enumToInt(Register.eax),
201	32...47 => 16,	195	@enumToInt(Register.ax) ... @enumToInt(Register.r15w) => @enumToInt(Register.ax),
202	48...63 => 8,	196	@enumToInt(Register.al) ... @enumToInt(Register.r15b) => @enumToInt(Register.al),
203	64...79 => 256,	197	@enumToInt(Register.ah) ... @enumToInt(Register.bh) => @enumToInt(Register.ah) - 4,
204	80...95 => 128,	198
		199	@enumToInt(Register.ymm0) ... @enumToInt(Register.ymm15) => @enumToInt(Register.ymm0) - 16,
		200	@enumToInt(Register.xmm0) ... @enumToInt(Register.xmm15) => @enumToInt(Register.xmm0) - 16,
		201
		202	@enumToInt(Register.es) ... @enumToInt(Register.gs) => @enumToInt(Register.es) - 32,
		203
205	else => unreachable,	204	else => unreachable,
		205	// zig fmt: on
206	};	206	};
		207	return @intCast(u6, @enumToInt(reg) - base);
207	}	208	}
208		209
209	/// Returns whether the register is extended. Extended registers are the	210	pub fn bitSize(reg: Register) u64 {
210	/// new registers added with amd64, r8 through r15. This also includes any	211	return switch (@enumToInt(reg)) {
211	/// other variant of access to those registers, such as r8b, r15d, and so	212	// zig fmt: off
212	/// on. This is needed because access to these registers requires special	213	@enumToInt(Register.rax) ... @enumToInt(Register.r15) => 64,
213	/// handling via the REX prefix, via the B or R bits, depending on context.	214	@enumToInt(Register.eax) ... @enumToInt(Register.r15d) => 32,
214	pub fn isExtended(self: Register) bool {	215	@enumToInt(Register.ax) ... @enumToInt(Register.r15w) => 16,
215	return @enumToInt(self) & 0x08 != 0;	216	@enumToInt(Register.al) ... @enumToInt(Register.r15b) => 8,
		217	@enumToInt(Register.ah) ... @enumToInt(Register.bh) => 8,
		218
		219	@enumToInt(Register.ymm0) ... @enumToInt(Register.ymm15) => 256,
		220	@enumToInt(Register.xmm0) ... @enumToInt(Register.xmm15) => 128,
		221
		222	@enumToInt(Register.es) ... @enumToInt(Register.gs) => 16,
		223
		224	else => unreachable,
		225	// zig fmt: on
		226	};
216	}	227	}
217		228
218	/// This returns the 4-bit register ID, which is used in practically every	229	pub fn isExtended(reg: Register) bool {
219	/// opcode. Note that bit 3 (the highest bit) is never used directly in	230	return switch (@enumToInt(reg)) {
220	/// an instruction (@see isExtended), and requires special handling. The	231	// zig fmt: off
221	/// lower three bits are often embedded directly in instructions (such as	232	@enumToInt(Register.r8) ... @enumToInt(Register.r15) => true,
222	/// the B8 variant of moves), or used in R/M bytes.	233	@enumToInt(Register.r8d) ... @enumToInt(Register.r15d) => true,
223	pub fn enc(self: Register) u4 {	234	@enumToInt(Register.r8w) ... @enumToInt(Register.r15w) => true,
224	return @truncate(u4, @enumToInt(self));	235	@enumToInt(Register.r8b) ... @enumToInt(Register.r15b) => true,
		236
		237	@enumToInt(Register.ymm8) ... @enumToInt(Register.ymm15) => true,
		238	@enumToInt(Register.xmm8) ... @enumToInt(Register.xmm15) => true,
		239
		240	else => false,
		241	// zig fmt: on
		242	};
225	}	243	}
226		244
227	/// Like enc, but only returns the lower 3 bits.	245	pub fn enc(reg: Register) u4 {
228	pub fn lowEnc(self: Register) u3 {	246	const base = switch (@enumToInt(reg)) {
229	return @truncate(u3, @enumToInt(self));	247	// zig fmt: off
		248	@enumToInt(Register.rax) ... @enumToInt(Register.r15) => @enumToInt(Register.rax),
		249	@enumToInt(Register.eax) ... @enumToInt(Register.r15d) => @enumToInt(Register.eax),
		250	@enumToInt(Register.ax) ... @enumToInt(Register.r15w) => @enumToInt(Register.ax),
		251	@enumToInt(Register.al) ... @enumToInt(Register.r15b) => @enumToInt(Register.al),
		252	@enumToInt(Register.ah) ... @enumToInt(Register.bh) => @enumToInt(Register.ah) - 4,
		253
		254	@enumToInt(Register.ymm0) ... @enumToInt(Register.ymm15) => @enumToInt(Register.ymm0),
		255	@enumToInt(Register.xmm0) ... @enumToInt(Register.xmm15) => @enumToInt(Register.xmm0),
		256
		257	@enumToInt(Register.es) ... @enumToInt(Register.gs) => @enumToInt(Register.es),
		258
		259	else => unreachable,
		260	// zig fmt: on
		261	};
		262	return @truncate(u4, @enumToInt(reg) - base);
		263	}
		264
		265	pub fn lowEnc(reg: Register) u3 {
		266	return @truncate(u3, reg.enc());
		267	}
		268
		269	pub fn toBitSize(reg: Register, bit_size: u64) Register {
		270	return switch (bit_size) {
		271	8 => reg.to8(),
		272	16 => reg.to16(),
		273	32 => reg.to32(),
		274	64 => reg.to64(),
		275	128 => reg.to128(),
		276	256 => reg.to256(),
		277	else => unreachable,
		278	};
		279	}
		280
		281	fn gpBase(reg: Register) u7 {
		282	assert(reg.class() == .general_purpose);
		283	return switch (@enumToInt(reg)) {
		284	// zig fmt: off
		285	@enumToInt(Register.rax) ... @enumToInt(Register.r15) => @enumToInt(Register.rax),
		286	@enumToInt(Register.eax) ... @enumToInt(Register.r15d) => @enumToInt(Register.eax),
		287	@enumToInt(Register.ax) ... @enumToInt(Register.r15w) => @enumToInt(Register.ax),
		288	@enumToInt(Register.al) ... @enumToInt(Register.r15b) => @enumToInt(Register.al),
		289	@enumToInt(Register.ah) ... @enumToInt(Register.bh) => @enumToInt(Register.ah) - 4,
		290	else => unreachable,
		291	// zig fmt: on
		292	};
230	}	293	}
231		294
232	pub fn to256(self: Register) Register {	295	pub fn to64(reg: Register) Register {
233	return @intToEnum(Register, @as(u8, self.enc()) + 64);	296	return @intToEnum(Register, @enumToInt(reg) - reg.gpBase() + @enumToInt(Register.rax));
234	}	297	}
235		298
236	pub fn to128(self: Register) Register {	299	pub fn to32(reg: Register) Register {
237	return @intToEnum(Register, @as(u8, self.enc()) + 80);	300	return @intToEnum(Register, @enumToInt(reg) - reg.gpBase() + @enumToInt(Register.eax));
238	}	301	}
239		302
240	/// Convert from any register to its 64 bit alias.	303	pub fn to16(reg: Register) Register {
241	pub fn to64(self: Register) Register {	304	return @intToEnum(Register, @enumToInt(reg) - reg.gpBase() + @enumToInt(Register.ax));
242	return @intToEnum(Register, self.enc());
243	}	305	}
244		306
245	/// Convert from any register to its 32 bit alias.	307	pub fn to8(reg: Register) Register {
246	pub fn to32(self: Register) Register {	308	return @intToEnum(Register, @enumToInt(reg) - reg.gpBase() + @enumToInt(Register.al));
247	return @intToEnum(Register, @as(u8, self.enc()) + 16);
248	}	309	}
249		310
250	/// Convert from any register to its 16 bit alias.	311	fn fpBase(reg: Register) u7 {
251	pub fn to16(self: Register) Register {	312	assert(reg.class() == .floating_point);
252	return @intToEnum(Register, @as(u8, self.enc()) + 32);	313	return switch (@enumToInt(reg)) {
		314	@enumToInt(Register.ymm0)...@enumToInt(Register.ymm15) => @enumToInt(Register.ymm0),
		315	@enumToInt(Register.xmm0)...@enumToInt(Register.xmm15) => @enumToInt(Register.xmm0),
		316	else => unreachable,
		317	};
253	}	318	}
254		319
255	/// Convert from any register to its 8 bit alias.	320	pub fn to256(reg: Register) Register {
256	pub fn to8(self: Register) Register {	321	return @intToEnum(Register, @enumToInt(reg) - reg.fpBase() + @enumToInt(Register.ymm0));
257	return @intToEnum(Register, @as(u8, self.enc()) + 48);
258	}	322	}
259		323
260	pub fn dwarfLocOp(self: Register) u8 {	324	pub fn to128(reg: Register) Register {
261	switch (@enumToInt(self)) {	325	return @intToEnum(Register, @enumToInt(reg) - reg.fpBase() + @enumToInt(Register.xmm0));
262	0...63 => return switch (self.to64()) {	326	}
		327
		328	pub fn dwarfLocOp(reg: Register) u8 {
		329	return switch (reg.class()) {
		330	.general_purpose => switch (reg.to64()) {
263	.rax => DW.OP.reg0,	331	.rax => DW.OP.reg0,
264	.rdx => DW.OP.reg1,	332	.rdx => DW.OP.reg1,
265	.rcx => DW.OP.reg2,	333	.rcx => DW.OP.reg2,
...	@@ -268,31 +336,19 @@ pub const Register = enum(u7) {	...	@@ -268,31 +336,19 @@ pub const Register = enum(u7) {
268	.rdi => DW.OP.reg5,	336	.rdi => DW.OP.reg5,
269	.rbp => DW.OP.reg6,	337	.rbp => DW.OP.reg6,
270	.rsp => DW.OP.reg7,	338	.rsp => DW.OP.reg7,
271		339	else => @intCast(u8, @enumToInt(reg) - reg.gpBase()) + DW.OP.reg0,
272	.r8 => DW.OP.reg8,
273	.r9 => DW.OP.reg9,
274	.r10 => DW.OP.reg10,
275	.r11 => DW.OP.reg11,
276	.r12 => DW.OP.reg12,
277	.r13 => DW.OP.reg13,
278	.r14 => DW.OP.reg14,
279	.r15 => DW.OP.reg15,
280
281	else => unreachable,
282	},	340	},
283		341	.floating_point => @intCast(u8, @enumToInt(reg) - reg.fpBase()) + DW.OP.reg17,
284	64...79 => return @as(u8, self.enc()) + DW.OP.reg17,
285
286	else => unreachable,	342	else => unreachable,
287	}	343	};
288	}	344	}
289		345
290	/// DWARF encodings that push a value onto the DWARF stack that is either	346	/// DWARF encodings that push a value onto the DWARF stack that is either
291	/// the contents of a register or the result of adding the contents a given	347	/// the contents of a register or the result of adding the contents a given
292	/// register to a given signed offset.	348	/// register to a given signed offset.
293	pub fn dwarfLocOpDeref(self: Register) u8 {	349	pub fn dwarfLocOpDeref(reg: Register) u8 {
294	switch (@enumToInt(self)) {	350	return switch (reg.class()) {
295	0...63 => return switch (self.to64()) {	351	.general_purpose => switch (reg.to64()) {
296	.rax => DW.OP.breg0,	352	.rax => DW.OP.breg0,
297	.rdx => DW.OP.breg1,	353	.rdx => DW.OP.breg1,
298	.rcx => DW.OP.breg2,	354	.rcx => DW.OP.breg2,
...	@@ -300,795 +356,195 @@ pub const Register = enum(u7) {	...	@@ -300,795 +356,195 @@ pub const Register = enum(u7) {
300	.rsi => DW.OP.breg4,	356	.rsi => DW.OP.breg4,
301	.rdi => DW.OP.breg5,	357	.rdi => DW.OP.breg5,
302	.rbp => DW.OP.breg6,	358	.rbp => DW.OP.breg6,
303	.rsp => DW.OP.fbreg,	359	.rsp => DW.OP.breg7,
304		360	else => @intCast(u8, @enumToInt(reg) - reg.gpBase()) + DW.OP.breg0,
305	.r8 => DW.OP.breg8,
306	.r9 => DW.OP.breg9,
307	.r10 => DW.OP.breg10,
308	.r11 => DW.OP.breg11,
309	.r12 => DW.OP.breg12,
310	.r13 => DW.OP.breg13,
311	.r14 => DW.OP.breg14,
312	.r15 => DW.OP.breg15,
313
314	else => unreachable,
315	},	361	},
316		362	.floating_point => @intCast(u8, @enumToInt(reg) - reg.fpBase()) + DW.OP.breg17,
317	64...79 => return @as(u8, self.enc()) + DW.OP.breg17,
318
319	else => unreachable,	363	else => unreachable,
320	}	364	};
321	}	365	}
322	};	366	};
323		367
324	// zig fmt: on	368	test "Register id - different classes" {
325		369	try expect(Register.al.id() == Register.ax.id());
326	/// Encoding helper functions for x86_64 instructions	370	try expect(Register.ah.id() == Register.spl.id());
327	///	371	try expect(Register.ax.id() == Register.eax.id());
328	/// Many of these helpers do very little, but they can help make things	372	try expect(Register.eax.id() == Register.rax.id());
329	/// slightly more readable with more descriptive field names / function names.
330	///
331	/// Some of them also have asserts to ensure that we aren't doing dumb things.
332	/// For example, trying to use register 4 (esp) in an indirect modr/m byte is illegal,
333	/// you need to encode it with an SIB byte.
334	///
335	/// Note that ALL of these helper functions will assume capacity,
336	/// so ensure that the `code` has sufficient capacity before using them.
337	/// The `init` method is the recommended way to ensure capacity.
338	pub const Encoder = struct {
339	/// Non-owning reference to the code array
340	code: *ArrayList(u8),
341
342	const Self = @This();
343
344	/// Wrap `code` in Encoder to make it easier to call these helper functions
345	///
346	/// maximum_inst_size should contain the maximum number of bytes
347	/// that the encoded instruction will take.
348	/// This is because the helper functions will assume capacity
349	/// in order to avoid bounds checking.
350	pub fn init(code: *ArrayList(u8), maximum_inst_size: u8) !Self {
351	try code.ensureUnusedCapacity(maximum_inst_size);
352	return Self{ .code = code };
353	}
354
355	/// Directly write a number to the code array with big endianness
356	pub fn writeIntBig(self: Self, comptime T: type, value: T) void {
357	mem.writeIntBig(
358	T,
359	self.code.addManyAsArrayAssumeCapacity(@divExact(@typeInfo(T).Int.bits, 8)),
360	value,
361	);
362	}
363
364	/// Directly write a number to the code array with little endianness
365	pub fn writeIntLittle(self: Self, comptime T: type, value: T) void {
366	mem.writeIntLittle(
367	T,
368	self.code.addManyAsArrayAssumeCapacity(@divExact(@typeInfo(T).Int.bits, 8)),
369	value,
370	);
371	}
372		373
373	// --------	374	try expect(Register.ymm0.id() == 0b10000);
374	// Prefixes	375	try expect(Register.ymm0.id() != Register.rax.id());
375	// --------	376	try expect(Register.xmm0.id() == Register.ymm0.id());
376
377	pub const LegacyPrefixes = packed struct {
378	/// LOCK
379	prefix_f0: bool = false,
380	/// REPNZ, REPNE, REP, Scalar Double-precision
381	prefix_f2: bool = false,
382	/// REPZ, REPE, REP, Scalar Single-precision
383	prefix_f3: bool = false,
384
385	/// CS segment override or Branch not taken
386	prefix_2e: bool = false,
387	/// DS segment override
388	prefix_36: bool = false,
389	/// ES segment override
390	prefix_26: bool = false,
391	/// FS segment override
392	prefix_64: bool = false,
393	/// GS segment override
394	prefix_65: bool = false,
395
396	/// Branch taken
397	prefix_3e: bool = false,
398
399	/// Operand size override (enables 16 bit operation)
400	prefix_66: bool = false,
401
402	/// Address size override (enables 16 bit address size)
403	prefix_67: bool = false,
404
405	padding: u5 = 0,
406	};
407		377
408	/// Encodes legacy prefixes	378	try expect(Register.es.id() == 0b100000);
409	pub fn legacyPrefixes(self: Self, prefixes: LegacyPrefixes) void {	379	}
410	if (@bitCast(u16, prefixes) != 0) {
411	// Hopefully this path isn't taken very often, so we'll do it the slow way for now
412
413	// LOCK
414	if (prefixes.prefix_f0) self.code.appendAssumeCapacity(0xf0);
415	// REPNZ, REPNE, REP, Scalar Double-precision
416	if (prefixes.prefix_f2) self.code.appendAssumeCapacity(0xf2);
417	// REPZ, REPE, REP, Scalar Single-precision
418	if (prefixes.prefix_f3) self.code.appendAssumeCapacity(0xf3);
419
420	// CS segment override or Branch not taken
421	if (prefixes.prefix_2e) self.code.appendAssumeCapacity(0x2e);
422	// DS segment override
423	if (prefixes.prefix_36) self.code.appendAssumeCapacity(0x36);
424	// ES segment override
425	if (prefixes.prefix_26) self.code.appendAssumeCapacity(0x26);
426	// FS segment override
427	if (prefixes.prefix_64) self.code.appendAssumeCapacity(0x64);
428	// GS segment override
429	if (prefixes.prefix_65) self.code.appendAssumeCapacity(0x65);
430
431	// Branch taken
432	if (prefixes.prefix_3e) self.code.appendAssumeCapacity(0x3e);
433
434	// Operand size override
435	if (prefixes.prefix_66) self.code.appendAssumeCapacity(0x66);
436
437	// Address size override
438	if (prefixes.prefix_67) self.code.appendAssumeCapacity(0x67);
439	}
440	}
441
442	/// Use 16 bit operand size
443	///
444	/// Note that this flag is overridden by REX.W, if both are present.
445	pub fn prefix16BitMode(self: Self) void {
446	self.code.appendAssumeCapacity(0x66);
447	}
448
449	pub const Vex = struct {
450	rex_prefix: Rex = .{},
451	lead_opc: u5 = 0b0_0001,
452	register: u4 = 0b1111,
453	length: u1 = 0b0,
454	simd_prefix: u2 = 0b00,
455	wig_desc: bool = false,
456	lig_desc: bool = false,
457	lz_desc: bool = false,
458
459	pub fn rex(self: *Vex, r: Rex) void {
460	self.rex_prefix = r;
461	}
462
463	pub fn lead_opc_0f(self: *Vex) void {
464	self.lead_opc = 0b0_0001;
465	}
466
467	pub fn lead_opc_0f_38(self: *Vex) void {
468	self.lead_opc = 0b0_0010;
469	}
470
471	pub fn lead_opc_0f_3a(self: *Vex) void {
472	self.lead_opc = 0b0_0011;
473	}
474
475	pub fn reg(self: *Vex, register: u4) void {
476	self.register = ~register;
477	}
478
479	pub fn len_128(self: *Vex) void {
480	self.length = 0;
481	}
482
483	pub fn len_256(self: *Vex) void {
484	assert(!self.lz_desc);
485	self.length = 1;
486	}
487		380
488	pub fn simd_prefix_66(self: *Vex) void {	381	test "Register enc - different classes" {
489	self.simd_prefix = 0b01;	382	try expect(Register.al.enc() == Register.ax.enc());
490	}	383	try expect(Register.ax.enc() == Register.eax.enc());
		384	try expect(Register.eax.enc() == Register.rax.enc());
		385	try expect(Register.ymm0.enc() == Register.rax.enc());
		386	try expect(Register.xmm0.enc() == Register.ymm0.enc());
		387	try expect(Register.es.enc() == Register.rax.enc());
		388	}
491		389
492	pub fn simd_prefix_f3(self: *Vex) void {	390	test "Register classes" {
493	self.simd_prefix = 0b10;	391	try expect(Register.r11.class() == .general_purpose);
494	}	392	try expect(Register.ymm11.class() == .floating_point);
		393	try expect(Register.fs.class() == .segment);
		394	}
495		395
496	pub fn simd_prefix_f2(self: *Vex) void {	396	pub const Memory = union(enum) {
497	self.simd_prefix = 0b11;	397	sib: Sib,
498	}	398	rip: Rip,
		399	moffs: Moffs,
499		400
500	pub fn wig(self: *Vex) void {	401	pub const ScaleIndex = packed struct {
501	self.wig_desc = true;	402	scale: u4,
502	}	403	index: Register,
		404	};
503		405
504	pub fn lig(self: *Vex) void {	406	pub const PtrSize = enum {
505	self.lig_desc = true;	407	byte,
		408	word,
		409	dword,
		410	qword,
		411	tbyte,
		412
		413	pub fn fromSize(size: u32) PtrSize {
		414	return if (size <= 1)
		415	.byte
		416	else if (size <= 2)
		417	.word
		418	else if (size <= 4)
		419	.dword
		420	else if (size <= 8)
		421	.qword
		422	else if (size == 10)
		423	.tbyte
		424	else
		425	unreachable;
506	}	426	}
507		427
508	pub fn lz(self: *Vex) void {	428	pub fn fromBitSize(bit_size: u64) PtrSize {
509	self.lz_desc = true;	429	return switch (bit_size) {
		430	8 => .byte,
		431	16 => .word,
		432	32 => .dword,
		433	64 => .qword,
		434	80 => .tbyte,
		435	else => unreachable,
		436	};
510	}	437	}
511		438
512	pub fn write(self: Vex, writer: anytype) usize {	439	pub fn bitSize(s: PtrSize) u64 {
513	var buf: [3]u8 = .{0} ** 3;	440	return switch (s) {
514	const form_3byte: bool = blk: {	441	.byte => 8,
515	if (self.rex_prefix.w and !self.wig_desc) break :blk true;	442	.word => 16,
516	if (self.rex_prefix.x or self.rex_prefix.b) break :blk true;	443	.dword => 32,
517	break :blk self.lead_opc != 0b0_0001;	444	.qword => 64,
		445	.tbyte => 80,
518	};	446	};
519
520	if (self.lz_desc) {
521	assert(self.length == 0);
522	}
523
524	if (form_3byte) {
525	// First byte
526	buf[0] = 0xc4;
527	// Second byte
528	const rxb_mask: u3 = @intCast(u3, @boolToInt(!self.rex_prefix.r)) << 2 \|
529	@intCast(u2, @boolToInt(!self.rex_prefix.x)) << 1 \|
530	@boolToInt(!self.rex_prefix.b);
531	buf[1] \|= @intCast(u8, rxb_mask) << 5;
532	buf[1] \|= self.lead_opc;
533	// Third byte
534	buf[2] \|= @intCast(u8, @boolToInt(!self.rex_prefix.w)) << 7;
535	buf[2] \|= @intCast(u7, self.register) << 3;
536	buf[2] \|= @intCast(u3, self.length) << 2;
537	buf[2] \|= self.simd_prefix;
538	} else {
539	// First byte
540	buf[0] = 0xc5;
541	// Second byte
542	buf[1] \|= @intCast(u8, @boolToInt(!self.rex_prefix.r)) << 7;
543	buf[1] \|= @intCast(u7, self.register) << 3;
544	buf[1] \|= @intCast(u3, self.length) << 2;
545	buf[1] \|= self.simd_prefix;
546	}
547
548	const count: usize = if (form_3byte) 3 else 2;
549	_ = writer.writeAll(buf[0..count]) catch unreachable;
550	return count;
551	}	447	}
552	};	448	};
553		449
554	pub fn vex(self: Self, prefix: Vex) void {	450	pub const Sib = struct {
555	_ = prefix.write(self.code.writer());	451	ptr_size: PtrSize,
556	}	452	base: ?Register,
557		453	scale_index: ?ScaleIndex,
558	/// From section 2.2.1.2 of the manual, REX is encoded as b0100WRXB	454	disp: i32,
559	pub const Rex = struct {
560	/// Wide, enables 64-bit operation
561	w: bool = false,
562	/// Extends the reg field in the ModR/M byte
563	r: bool = false,
564	/// Extends the index field in the SIB byte
565	x: bool = false,
566	/// Extends the r/m field in the ModR/M byte,
567	/// or the base field in the SIB byte,
568	/// or the reg field in the Opcode byte
569	b: bool = false,
570	};	455	};
571		456
572	/// Encodes a REX prefix byte given all the fields	457	pub const Rip = struct {
573	///	458	ptr_size: PtrSize,
574	/// Use this byte whenever you need 64 bit operation,	459	disp: i32,
575	/// or one of reg, index, r/m, base, or opcode-reg might be extended.	460	};
576	///
577	/// See struct `Rex` for a description of each field.
578	///
579	/// Does not add a prefix byte if none of the fields are set!
580	pub fn rex(self: Self, byte: Rex) void {
581	var value: u8 = 0b0100_0000;
582
583	if (byte.w) value \|= 0b1000;
584	if (byte.r) value \|= 0b0100;
585	if (byte.x) value \|= 0b0010;
586	if (byte.b) value \|= 0b0001;
587
588	if (value != 0b0100_0000) {
589	self.code.appendAssumeCapacity(value);
590	}
591	}
592
593	// ------
594	// Opcode
595	// ------
596
597	/// Encodes a 1 byte opcode
598	pub fn opcode_1byte(self: Self, opcode: u8) void {
599	self.code.appendAssumeCapacity(opcode);
600	}
601
602	/// Encodes a 2 byte opcode
603	///
604	/// e.g. IMUL has the opcode 0x0f 0xaf, so you use
605	///
606	/// encoder.opcode_2byte(0x0f, 0xaf);
607	pub fn opcode_2byte(self: Self, prefix: u8, opcode: u8) void {
608	self.code.appendAssumeCapacity(prefix);
609	self.code.appendAssumeCapacity(opcode);
610	}
611
612	/// Encodes a 3 byte opcode
613	///
614	/// e.g. MOVSD has the opcode 0xf2 0x0f 0x10
615	///
616	/// encoder.opcode_3byte(0xf2, 0x0f, 0x10);
617	pub fn opcode_3byte(self: Self, prefix_1: u8, prefix_2: u8, opcode: u8) void {
618	self.code.appendAssumeCapacity(prefix_1);
619	self.code.appendAssumeCapacity(prefix_2);
620	self.code.appendAssumeCapacity(opcode);
621	}
622
623	/// Encodes a 1 byte opcode with a reg field
624	///
625	/// Remember to add a REX prefix byte if reg is extended!
626	pub fn opcode_withReg(self: Self, opcode: u8, reg: u3) void {
627	assert(opcode & 0b111 == 0);
628	self.code.appendAssumeCapacity(opcode \| reg);
629	}
630
631	// ------
632	// ModR/M
633	// ------
634
635	/// Construct a ModR/M byte given all the fields
636	///
637	/// Remember to add a REX prefix byte if reg or rm are extended!
638	pub fn modRm(self: Self, mod: u2, reg_or_opx: u3, rm: u3) void {
639	self.code.appendAssumeCapacity(
640	@as(u8, mod) << 6 \| @as(u8, reg_or_opx) << 3 \| rm,
641	);
642	}
643
644	/// Construct a ModR/M byte using direct r/m addressing
645	/// r/m effective address: r/m
646	///
647	/// Note reg's effective address is always just reg for the ModR/M byte.
648	/// Remember to add a REX prefix byte if reg or rm are extended!
649	pub fn modRm_direct(self: Self, reg_or_opx: u3, rm: u3) void {
650	self.modRm(0b11, reg_or_opx, rm);
651	}
652
653	/// Construct a ModR/M byte using indirect r/m addressing
654	/// r/m effective address: [r/m]
655	///
656	/// Note reg's effective address is always just reg for the ModR/M byte.
657	/// Remember to add a REX prefix byte if reg or rm are extended!
658	pub fn modRm_indirectDisp0(self: Self, reg_or_opx: u3, rm: u3) void {
659	assert(rm != 4 and rm != 5);
660	self.modRm(0b00, reg_or_opx, rm);
661	}
662
663	/// Construct a ModR/M byte using indirect SIB addressing
664	/// r/m effective address: [SIB]
665	///
666	/// Note reg's effective address is always just reg for the ModR/M byte.
667	/// Remember to add a REX prefix byte if reg or rm are extended!
668	pub fn modRm_SIBDisp0(self: Self, reg_or_opx: u3) void {
669	self.modRm(0b00, reg_or_opx, 0b100);
670	}
671
672	/// Construct a ModR/M byte using RIP-relative addressing
673	/// r/m effective address: [RIP + disp32]
674	///
675	/// Note reg's effective address is always just reg for the ModR/M byte.
676	/// Remember to add a REX prefix byte if reg or rm are extended!
677	pub fn modRm_RIPDisp32(self: Self, reg_or_opx: u3) void {
678	self.modRm(0b00, reg_or_opx, 0b101);
679	}
680
681	/// Construct a ModR/M byte using indirect r/m with a 8bit displacement
682	/// r/m effective address: [r/m + disp8]
683	///
684	/// Note reg's effective address is always just reg for the ModR/M byte.
685	/// Remember to add a REX prefix byte if reg or rm are extended!
686	pub fn modRm_indirectDisp8(self: Self, reg_or_opx: u3, rm: u3) void {
687	assert(rm != 4);
688	self.modRm(0b01, reg_or_opx, rm);
689	}
690
691	/// Construct a ModR/M byte using indirect SIB with a 8bit displacement
692	/// r/m effective address: [SIB + disp8]
693	///
694	/// Note reg's effective address is always just reg for the ModR/M byte.
695	/// Remember to add a REX prefix byte if reg or rm are extended!
696	pub fn modRm_SIBDisp8(self: Self, reg_or_opx: u3) void {
697	self.modRm(0b01, reg_or_opx, 0b100);
698	}
699
700	/// Construct a ModR/M byte using indirect r/m with a 32bit displacement
701	/// r/m effective address: [r/m + disp32]
702	///
703	/// Note reg's effective address is always just reg for the ModR/M byte.
704	/// Remember to add a REX prefix byte if reg or rm are extended!
705	pub fn modRm_indirectDisp32(self: Self, reg_or_opx: u3, rm: u3) void {
706	assert(rm != 4);
707	self.modRm(0b10, reg_or_opx, rm);
708	}
709
710	/// Construct a ModR/M byte using indirect SIB with a 32bit displacement
711	/// r/m effective address: [SIB + disp32]
712	///
713	/// Note reg's effective address is always just reg for the ModR/M byte.
714	/// Remember to add a REX prefix byte if reg or rm are extended!
715	pub fn modRm_SIBDisp32(self: Self, reg_or_opx: u3) void {
716	self.modRm(0b10, reg_or_opx, 0b100);
717	}
718
719	// ---
720	// SIB
721	// ---
722
723	/// Construct a SIB byte given all the fields
724	///
725	/// Remember to add a REX prefix byte if index or base are extended!
726	pub fn sib(self: Self, scale: u2, index: u3, base: u3) void {
727	self.code.appendAssumeCapacity(
728	@as(u8, scale) << 6 \| @as(u8, index) << 3 \| base,
729	);
730	}
731
732	/// Construct a SIB byte with scale * index + base, no frills.
733	/// r/m effective address: [base + scale * index]
734	///
735	/// Remember to add a REX prefix byte if index or base are extended!
736	pub fn sib_scaleIndexBase(self: Self, scale: u2, index: u3, base: u3) void {
737	assert(base != 5);
738
739	self.sib(scale, index, base);
740	}
741
742	/// Construct a SIB byte with scale * index + disp32
743	/// r/m effective address: [scale * index + disp32]
744	///
745	/// Remember to add a REX prefix byte if index or base are extended!
746	pub fn sib_scaleIndexDisp32(self: Self, scale: u2, index: u3) void {
747	assert(index != 4);
748
749	// scale is actually ignored
750	// index = 4 means no index
751	// base = 5 means no base, if mod == 0.
752	self.sib(scale, index, 5);
753	}
754
755	/// Construct a SIB byte with just base
756	/// r/m effective address: [base]
757	///
758	/// Remember to add a REX prefix byte if index or base are extended!
759	pub fn sib_base(self: Self, base: u3) void {
760	assert(base != 5);
761
762	// scale is actually ignored
763	// index = 4 means no index
764	self.sib(0, 4, base);
765	}
766
767	/// Construct a SIB byte with just disp32
768	/// r/m effective address: [disp32]
769	///
770	/// Remember to add a REX prefix byte if index or base are extended!
771	pub fn sib_disp32(self: Self) void {
772	// scale is actually ignored
773	// index = 4 means no index
774	// base = 5 means no base, if mod == 0.
775	self.sib(0, 4, 5);
776	}
777
778	/// Construct a SIB byte with scale * index + base + disp8
779	/// r/m effective address: [base + scale * index + disp8]
780	///
781	/// Remember to add a REX prefix byte if index or base are extended!
782	pub fn sib_scaleIndexBaseDisp8(self: Self, scale: u2, index: u3, base: u3) void {
783	self.sib(scale, index, base);
784	}
785
786	/// Construct a SIB byte with base + disp8, no index
787	/// r/m effective address: [base + disp8]
788	///
789	/// Remember to add a REX prefix byte if index or base are extended!
790	pub fn sib_baseDisp8(self: Self, base: u3) void {
791	// scale is ignored
792	// index = 4 means no index
793	self.sib(0, 4, base);
794	}
795		461
796	/// Construct a SIB byte with scale * index + base + disp32	462	pub const Moffs = struct {
797	/// r/m effective address: [base + scale * index + disp32]	463	seg: Register,
798	///	464	offset: u64,
799	/// Remember to add a REX prefix byte if index or base are extended!	465	};
800	pub fn sib_scaleIndexBaseDisp32(self: Self, scale: u2, index: u3, base: u3) void {
801	self.sib(scale, index, base);
802	}
803		466
804	/// Construct a SIB byte with base + disp32, no index	467	pub fn moffs(reg: Register, offset: u64) Memory {
805	/// r/m effective address: [base + disp32]	468	assert(reg.class() == .segment);
806	///	469	return .{ .moffs = .{ .seg = reg, .offset = offset } };
807	/// Remember to add a REX prefix byte if index or base are extended!
808	pub fn sib_baseDisp32(self: Self, base: u3) void {
809	// scale is ignored
810	// index = 4 means no index
811	self.sib(0, 4, base);
812	}	470	}
813		471
814	// -------------------------	472	pub fn sib(ptr_size: PtrSize, args: struct {
815	// Trivial (no bit fiddling)	473	disp: i32,
816	// -------------------------	474	base: ?Register = null,
817		475	scale_index: ?ScaleIndex = null,
818	/// Encode an 8 bit immediate	476	}) Memory {
819	///	477	return .{ .sib = .{
820	/// It is sign-extended to 64 bits by the cpu.	478	.base = args.base,
821	pub fn imm8(self: Self, imm: i8) void {	479	.disp = args.disp,
822	self.code.appendAssumeCapacity(@bitCast(u8, imm));	480	.ptr_size = ptr_size,
		481	.scale_index = args.scale_index,
		482	} };
823	}	483	}
824		484
825	/// Encode an 8 bit displacement	485	pub fn rip(ptr_size: PtrSize, disp: i32) Memory {
826	///	486	return .{ .rip = .{ .ptr_size = ptr_size, .disp = disp } };
827	/// It is sign-extended to 64 bits by the cpu.
828	pub fn disp8(self: Self, disp: i8) void {
829	self.code.appendAssumeCapacity(@bitCast(u8, disp));
830	}	487	}
831		488
832	/// Encode an 16 bit immediate	489	pub fn isSegmentRegister(mem: Memory) bool {
833	///	490	return switch (mem) {
834	/// It is sign-extended to 64 bits by the cpu.	491	.moffs => true,
835	pub fn imm16(self: Self, imm: i16) void {	492	.rip => false,
836	self.writeIntLittle(i16, imm);	493	.sib => \|s\| if (s.base) \|r\| r.class() == .segment else false,
		494	};
837	}	495	}
838		496
839	/// Encode an 32 bit immediate	497	pub fn base(mem: Memory) ?Register {
840	///	498	return switch (mem) {
841	/// It is sign-extended to 64 bits by the cpu.	499	.moffs => \|m\| m.seg,
842	pub fn imm32(self: Self, imm: i32) void {	500	.sib => \|s\| s.base,
843	self.writeIntLittle(i32, imm);	501	.rip => null,
		502	};
844	}	503	}
845		504
846	/// Encode an 32 bit displacement	505	pub fn scaleIndex(mem: Memory) ?ScaleIndex {
847	///	506	return switch (mem) {
848	/// It is sign-extended to 64 bits by the cpu.	507	.moffs, .rip => null,
849	pub fn disp32(self: Self, disp: i32) void {	508	.sib => \|s\| s.scale_index,
850	self.writeIntLittle(i32, disp);	509	};
851	}	510	}
852		511
853	/// Encode an 64 bit immediate	512	pub fn bitSize(mem: Memory) u64 {
854	///	513	return switch (mem) {
855	/// It is sign-extended to 64 bits by the cpu.	514	.rip => \|r\| r.ptr_size.bitSize(),
856	pub fn imm64(self: Self, imm: u64) void {	515	.sib => \|s\| s.ptr_size.bitSize(),
857	self.writeIntLittle(u64, imm);	516	.moffs => unreachable,
		517	};
858	}	518	}
859	};	519	};
860		520
861	test "Encoder helpers - general purpose registers" {	521	pub const Immediate = union(enum) {
862	var code = ArrayList(u8).init(testing.allocator);	522	signed: i32,
863	defer code.deinit();	523	unsigned: u64,
864
865	// simple integer multiplication
866
867	// imul eax,edi
868	// 0faf c7
869	{
870	try code.resize(0);
871	const encoder = try Encoder.init(&code, 4);
872	encoder.rex(.{
873	.r = Register.eax.isExtended(),
874	.b = Register.edi.isExtended(),
875	});
876	encoder.opcode_2byte(0x0f, 0xaf);
877	encoder.modRm_direct(
878	Register.eax.lowEnc(),
879	Register.edi.lowEnc(),
880	);
881
882	try testing.expectEqualSlices(u8, &[_]u8{ 0x0f, 0xaf, 0xc7 }, code.items);
883	}
884
885	// simple mov
886
887	// mov eax,edi
888	// 89 f8
889	{
890	try code.resize(0);
891	const encoder = try Encoder.init(&code, 3);
892	encoder.rex(.{
893	.r = Register.edi.isExtended(),
894	.b = Register.eax.isExtended(),
895	});
896	encoder.opcode_1byte(0x89);
897	encoder.modRm_direct(
898	Register.edi.lowEnc(),
899	Register.eax.lowEnc(),
900	);
901
902	try testing.expectEqualSlices(u8, &[_]u8{ 0x89, 0xf8 }, code.items);
903	}
904
905	// signed integer addition of 32-bit sign extended immediate to 64 bit register
906
907	// add rcx, 2147483647
908	//
909	// Using the following opcode: REX.W + 81 /0 id, we expect the following encoding
910	//
911	// 48 : REX.W set for 64 bit operand (rcx)
912	// 81 : opcode for "<arithmetic> with immediate"
913	// c1 : id = rcx,
914	// : c1 = 11 <-- mod = 11 indicates r/m is register (rcx)
915	// : 000 <-- opcode_extension = 0 because opcode extension is /0. /0 specifies ADD
916	// : 001 <-- 001 is rcx
917	// ffffff7f : 2147483647
918	{
919	try code.resize(0);
920	const encoder = try Encoder.init(&code, 7);
921	encoder.rex(.{ .w = true }); // use 64 bit operation
922	encoder.opcode_1byte(0x81);
923	encoder.modRm_direct(
924	0,
925	Register.rcx.lowEnc(),
926	);
927	encoder.imm32(2147483647);
928
929	try testing.expectEqualSlices(u8, &[_]u8{ 0x48, 0x81, 0xc1, 0xff, 0xff, 0xff, 0x7f }, code.items);
930	}
931	}
932
933	test "Encoder helpers - Vex prefix" {
934	var buf: [3]u8 = undefined;
935	var stream = std.io.fixedBufferStream(&buf);
936	const writer = stream.writer();
937
938	{
939	var vex_prefix = Encoder.Vex{};
940	vex_prefix.rex(.{
941	.r = true,
942	});
943	const nwritten = vex_prefix.write(writer);
944	try testing.expectEqualSlices(u8, &[_]u8{ 0xc5, 0x78 }, buf[0..nwritten]);
945	}
946
947	{
948	stream.reset();
949	var vex_prefix = Encoder.Vex{};
950	vex_prefix.reg(Register.xmm15.enc());
951	const nwritten = vex_prefix.write(writer);
952	try testing.expectEqualSlices(u8, &[_]u8{ 0xc5, 0x80 }, buf[0..nwritten]);
953	}
954		524
955	{	525	pub fn u(x: u64) Immediate {
956	stream.reset();	526	return .{ .unsigned = x };
957	var vex_prefix = Encoder.Vex{};
958	vex_prefix.rex(.{
959	.w = true,
960	.x = true,
961	});
962	const nwritten = vex_prefix.write(writer);
963	try testing.expectEqualSlices(u8, &[_]u8{ 0xc4, 0b101_0_0001, 0b0_1111_0_00 }, buf[0..nwritten]);
964	}	527	}
965		528
966	{	529	pub fn s(x: i32) Immediate {
967	stream.reset();	530	return .{ .signed = x };
968	var vex_prefix = Encoder.Vex{};
969	vex_prefix.rex(.{
970	.w = true,
971	.r = true,
972	});
973	vex_prefix.len_256();
974	vex_prefix.lead_opc_0f();
975	vex_prefix.simd_prefix_66();
976	const nwritten = vex_prefix.write(writer);
977	try testing.expectEqualSlices(u8, &[_]u8{ 0xc4, 0b011_0_0001, 0b0_1111_1_01 }, buf[0..nwritten]);
978	}	531	}
979		532
980	var code = ArrayList(u8).init(testing.allocator);	533	pub fn asUnsigned(imm: Immediate, bit_size: u64) u64 {
981	defer code.deinit();	534	return switch (imm) {
982		535	.signed => \|x\| switch (bit_size) {
983	{	536	1, 8 => @bitCast(u8, @intCast(i8, x)),
984	// vmovapd xmm1, xmm2	537	16 => @bitCast(u16, @intCast(i16, x)),
985	const encoder = try Encoder.init(&code, 4);	538	32, 64 => @bitCast(u32, x),
986	var vex = Encoder.Vex{};	539	else => unreachable,
987	vex.simd_prefix_66();	540	},
988	encoder.vex(vex); // use 64 bit operation	541	.unsigned => \|x\| switch (bit_size) {
989	encoder.opcode_1byte(0x28);	542	1, 8 => @intCast(u8, x),
990	encoder.modRm_direct(0, Register.xmm1.lowEnc());	543	16 => @intCast(u16, x),
991	try testing.expectEqualSlices(u8, &[_]u8{ 0xC5, 0xF9, 0x28, 0xC1 }, code.items);	544	32 => @intCast(u32, x),
992	}	545	64 => x,
993		546	else => unreachable,
994	{	547	},
995	try code.resize(0);	548	};
996
997	// vmovhpd xmm13, xmm1, qword ptr [rip]
998	const encoder = try Encoder.init(&code, 9);
999	var vex = Encoder.Vex{};
1000	vex.len_128();
1001	vex.simd_prefix_66();
1002	vex.lead_opc_0f();
1003	vex.rex(.{ .r = true });
1004	vex.reg(Register.xmm1.enc());
1005	encoder.vex(vex);
1006	encoder.opcode_1byte(0x16);
1007	encoder.modRm_RIPDisp32(Register.xmm13.lowEnc());
1008	encoder.disp32(0);
1009	try testing.expectEqualSlices(u8, &[_]u8{ 0xC5, 0x71, 0x16, 0x2D, 0x00, 0x00, 0x00, 0x00 }, code.items);
1010	}	549	}
1011	}	550	};
1012
1013	// TODO add these registers to the enum and populate dwarfLocOp
1014	// // Return Address register. This is stored in `0(%rsp, "")` and is not a physical register.
1015	// RA = (16, "RA"),
1016	//
1017	// XMM0 = (17, "xmm0"),
1018	// XMM1 = (18, "xmm1"),
1019	// XMM2 = (19, "xmm2"),
1020	// XMM3 = (20, "xmm3"),
1021	// XMM4 = (21, "xmm4"),
1022	// XMM5 = (22, "xmm5"),
1023	// XMM6 = (23, "xmm6"),
1024	// XMM7 = (24, "xmm7"),
1025	//
1026	// XMM8 = (25, "xmm8"),
1027	// XMM9 = (26, "xmm9"),
1028	// XMM10 = (27, "xmm10"),
1029	// XMM11 = (28, "xmm11"),
1030	// XMM12 = (29, "xmm12"),
1031	// XMM13 = (30, "xmm13"),
1032	// XMM14 = (31, "xmm14"),
1033	// XMM15 = (32, "xmm15"),
1034	//
1035	// ST0 = (33, "st0"),
1036	// ST1 = (34, "st1"),
1037	// ST2 = (35, "st2"),
1038	// ST3 = (36, "st3"),
1039	// ST4 = (37, "st4"),
1040	// ST5 = (38, "st5"),
1041	// ST6 = (39, "st6"),
1042	// ST7 = (40, "st7"),
1043	//
1044	// MM0 = (41, "mm0"),
1045	// MM1 = (42, "mm1"),
1046	// MM2 = (43, "mm2"),
1047	// MM3 = (44, "mm3"),
1048	// MM4 = (45, "mm4"),
1049	// MM5 = (46, "mm5"),
1050	// MM6 = (47, "mm6"),
1051	// MM7 = (48, "mm7"),
1052	//
1053	// RFLAGS = (49, "rFLAGS"),
1054	// ES = (50, "es"),
1055	// CS = (51, "cs"),
1056	// SS = (52, "ss"),
1057	// DS = (53, "ds"),
1058	// FS = (54, "fs"),
1059	// GS = (55, "gs"),
1060	//
1061	// FS_BASE = (58, "fs.base"),
1062	// GS_BASE = (59, "gs.base"),
1063	//
1064	// TR = (62, "tr"),
1065	// LDTR = (63, "ldtr"),
1066	// MXCSR = (64, "mxcsr"),
1067	// FCW = (65, "fcw"),
1068	// FSW = (66, "fsw"),
1069	//
1070	// XMM16 = (67, "xmm16"),
1071	// XMM17 = (68, "xmm17"),
1072	// XMM18 = (69, "xmm18"),
1073	// XMM19 = (70, "xmm19"),
1074	// XMM20 = (71, "xmm20"),
1075	// XMM21 = (72, "xmm21"),
1076	// XMM22 = (73, "xmm22"),
1077	// XMM23 = (74, "xmm23"),
1078	// XMM24 = (75, "xmm24"),
1079	// XMM25 = (76, "xmm25"),
1080	// XMM26 = (77, "xmm26"),
1081	// XMM27 = (78, "xmm27"),
1082	// XMM28 = (79, "xmm28"),
1083	// XMM29 = (80, "xmm29"),
1084	// XMM30 = (81, "xmm30"),
1085	// XMM31 = (82, "xmm31"),
1086	//
1087	// K0 = (118, "k0"),
1088	// K1 = (119, "k1"),
1089	// K2 = (120, "k2"),
1090	// K3 = (121, "k3"),
1091	// K4 = (122, "k4"),
1092	// K5 = (123, "k5"),
1093	// K6 = (124, "k6"),
1094	// K7 = (125, "k7"),

src/arch/x86_64/encoder.zig created+2275

...	@@ -0,0 +1,2275 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const log = std.log.scoped(.x86_64_encoder);
4	const math = std.math;
5	const testing = std.testing;
6
7	const bits = @import("bits.zig");
8	const Encoding = @import("Encoding.zig");
9	const Immediate = bits.Immediate;
10	const Memory = bits.Memory;
11	const Register = bits.Register;
12
13	pub const Instruction = struct {
14	op1: Operand = .none,
15	op2: Operand = .none,
16	op3: Operand = .none,
17	op4: Operand = .none,
18	encoding: Encoding,
19
20	pub const Mnemonic = Encoding.Mnemonic;
21
22	pub const Operand = union(enum) {
23	none,
24	reg: Register,
25	mem: Memory,
26	imm: Immediate,
27
28	/// Returns the bitsize of the operand.
29	pub fn bitSize(op: Operand) u64 {
30	return switch (op) {
31	.none => unreachable,
32	.reg => \|reg\| reg.bitSize(),
33	.mem => \|mem\| mem.bitSize(),
34	.imm => unreachable,
35	};
36	}
37
38	/// Returns true if the operand is a segment register.
39	/// Asserts the operand is either register or memory.
40	pub fn isSegmentRegister(op: Operand) bool {
41	return switch (op) {
42	.none => unreachable,
43	.reg => \|reg\| reg.class() == .segment,
44	.mem => \|mem\| mem.isSegmentRegister(),
45	.imm => unreachable,
46	};
47	}
48
49	pub fn fmtPrint(op: Operand, enc_op: Encoding.Op, writer: anytype) !void {
50	switch (op) {
51	.none => {},
52	.reg => \|reg\| try writer.writeAll(@tagName(reg)),
53	.mem => \|mem\| switch (mem) {
54	.rip => \|rip\| {
55	try writer.print("{s} ptr [rip", .{@tagName(rip.ptr_size)});
56	if (rip.disp != 0) {
57	const sign_bit = if (sign(rip.disp) < 0) "-" else "+";
58	const disp_abs = try std.math.absInt(rip.disp);
59	try writer.print(" {s} 0x{x}", .{ sign_bit, disp_abs });
60	}
61	try writer.writeByte(']');
62	},
63	.sib => \|sib\| {
64	try writer.print("{s} ptr ", .{@tagName(sib.ptr_size)});
65
66	if (mem.isSegmentRegister()) {
67	return writer.print("{s}:0x{x}", .{ @tagName(sib.base.?), sib.disp });
68	}
69
70	try writer.writeByte('[');
71
72	if (sib.base) \|base\| {
73	try writer.print("{s}", .{@tagName(base)});
74	}
75	if (sib.scale_index) \|si\| {
76	if (sib.base != null) {
77	try writer.writeAll(" + ");
78	}
79	try writer.print("{s} * {d}", .{ @tagName(si.index), si.scale });
80	}
81	if (sib.disp != 0) {
82	if (sib.base != null or sib.scale_index != null) {
83	try writer.writeByte(' ');
84	}
85	try writer.writeByte(if (sign(sib.disp) < 0) '-' else '+');
86	const disp_abs = try std.math.absInt(sib.disp);
87	try writer.print(" 0x{x}", .{disp_abs});
88	}
89
90	try writer.writeByte(']');
91	},
92	.moffs => \|moffs\| try writer.print("{s}:0x{x}", .{ @tagName(moffs.seg), moffs.offset }),
93	},
94	.imm => \|imm\| try writer.print("0x{x}", .{imm.asUnsigned(enc_op.bitSize())}),
95	}
96	}
97	};
98
99	pub fn new(mnemonic: Mnemonic, args: struct {
100	op1: Operand = .none,
101	op2: Operand = .none,
102	op3: Operand = .none,
103	op4: Operand = .none,
104	}) !Instruction {
105	const encoding = (try Encoding.findByMnemonic(mnemonic, .{
106	.op1 = args.op1,
107	.op2 = args.op2,
108	.op3 = args.op3,
109	.op4 = args.op4,
110	})) orelse {
111	log.debug("no encoding found for: {s} {s} {s} {s} {s}", .{
112	@tagName(mnemonic),
113	@tagName(Encoding.Op.fromOperand(args.op1)),
114	@tagName(Encoding.Op.fromOperand(args.op2)),
115	@tagName(Encoding.Op.fromOperand(args.op3)),
116	@tagName(Encoding.Op.fromOperand(args.op4)),
117	});
118	return error.InvalidInstruction;
119	};
120	log.debug("selected encoding: {}", .{encoding});
121	return .{
122	.op1 = args.op1,
123	.op2 = args.op2,
124	.op3 = args.op3,
125	.op4 = args.op4,
126	.encoding = encoding,
127	};
128	}
129
130	pub fn fmtPrint(inst: Instruction, writer: anytype) !void {
131	try writer.print("{s}", .{@tagName(inst.encoding.mnemonic)});
132	const ops = [_]struct { Operand, Encoding.Op }{
133	.{ inst.op1, inst.encoding.op1 },
134	.{ inst.op2, inst.encoding.op2 },
135	.{ inst.op3, inst.encoding.op3 },
136	.{ inst.op4, inst.encoding.op4 },
137	};
138	for (&ops, 0..) \|op, i\| {
139	if (op[0] == .none) break;
140	if (i > 0) {
141	try writer.writeByte(',');
142	}
143	try writer.writeByte(' ');
144	try op[0].fmtPrint(op[1], writer);
145	}
146	}
147
148	pub fn encode(inst: Instruction, writer: anytype) !void {
149	const encoder = Encoder(@TypeOf(writer)){ .writer = writer };
150	const encoding = inst.encoding;
151
152	try inst.encodeLegacyPrefixes(encoder);
153	try inst.encodeMandatoryPrefix(encoder);
154	try inst.encodeRexPrefix(encoder);
155	try inst.encodeOpcode(encoder);
156
157	switch (encoding.op_en) {
158	.np, .o => {},
159	.i, .d => try encodeImm(inst.op1.imm, encoding.op1, encoder),
160	.zi, .oi => try encodeImm(inst.op2.imm, encoding.op2, encoder),
161	.fd => try encoder.imm64(inst.op2.mem.moffs.offset),
162	.td => try encoder.imm64(inst.op1.mem.moffs.offset),
163	else => {
164	const mem_op = switch (encoding.op_en) {
165	.m, .mi, .m1, .mc, .mr => inst.op1,
166	.rm, .rmi => inst.op2,
167	else => unreachable,
168	};
169	switch (mem_op) {
170	.reg => \|reg\| {
171	const rm = switch (encoding.op_en) {
172	.m, .mi, .m1, .mc => encoding.modRmExt(),
173	.mr => inst.op2.reg.lowEnc(),
174	.rm, .rmi => inst.op1.reg.lowEnc(),
175	else => unreachable,
176	};
177	try encoder.modRm_direct(rm, reg.lowEnc());
178	},
179	.mem => \|mem\| {
180	const op = switch (encoding.op_en) {
181	.m, .mi, .m1, .mc => .none,
182	.mr => inst.op2,
183	.rm, .rmi => inst.op1,
184	else => unreachable,
185	};
186	try encodeMemory(encoding, mem, op, encoder);
187	},
188	else => unreachable,
189	}
190
191	switch (encoding.op_en) {
192	.mi => try encodeImm(inst.op2.imm, encoding.op2, encoder),
193	.rmi => try encodeImm(inst.op3.imm, encoding.op3, encoder),
194	else => {},
195	}
196	},
197	}
198	}
199
200	fn encodeOpcode(inst: Instruction, encoder: anytype) !void {
201	const opcode = inst.encoding.opcode();
202	switch (inst.encoding.op_en) {
203	.o, .oi => try encoder.opcode_withReg(opcode[0], inst.op1.reg.lowEnc()),
204	else => {
205	const index: usize = if (inst.encoding.mandatoryPrefix()) \|_\| 1 else 0;
206	for (opcode[index..]) \|byte\| {
207	try encoder.opcode_1byte(byte);
208	}
209	},
210	}
211	}
212
213	fn encodeLegacyPrefixes(inst: Instruction, encoder: anytype) !void {
214	const enc = inst.encoding;
215	const op_en = enc.op_en;
216
217	var legacy = LegacyPrefixes{};
218	if (enc.mode == .none) {
219	const bit_size = enc.operandBitSize();
220	if (bit_size == 16) {
221	legacy.set16BitOverride();
222	}
223	}
224
225	const segment_override: ?Register = switch (op_en) {
226	.i, .zi, .o, .oi, .d, .np => null,
227	.fd => inst.op2.mem.base().?,
228	.td => inst.op1.mem.base().?,
229	.rm, .rmi => if (inst.op2.isSegmentRegister()) blk: {
230	break :blk switch (inst.op2) {
231	.reg => \|r\| r,
232	.mem => \|m\| m.base().?,
233	else => unreachable,
234	};
235	} else null,
236	.m, .mi, .m1, .mc, .mr => if (inst.op1.isSegmentRegister()) blk: {
237	break :blk switch (inst.op1) {
238	.reg => \|r\| r,
239	.mem => \|m\| m.base().?,
240	else => unreachable,
241	};
242	} else null,
243	};
244	if (segment_override) \|seg\| {
245	legacy.setSegmentOverride(seg);
246	}
247
248	try encoder.legacyPrefixes(legacy);
249	}
250
251	fn encodeRexPrefix(inst: Instruction, encoder: anytype) !void {
252	const op_en = inst.encoding.op_en;
253
254	var rex = Rex{};
255	rex.present = inst.encoding.mode == .rex;
256	rex.w = inst.encoding.mode == .long;
257
258	switch (op_en) {
259	.np, .i, .zi, .fd, .td, .d => {},
260	.o, .oi => {
261	rex.b = inst.op1.reg.isExtended();
262	},
263	.m, .mi, .m1, .mc, .mr, .rm, .rmi => {
264	const r_op = switch (op_en) {
265	.rm, .rmi => inst.op1,
266	.mr => inst.op2,
267	else => null,
268	};
269	if (r_op) \|op\| {
270	rex.r = op.reg.isExtended();
271	}
272
273	const b_x_op = switch (op_en) {
274	.rm, .rmi => inst.op2,
275	.m, .mi, .m1, .mc, .mr => inst.op1,
276	else => unreachable,
277	};
278	switch (b_x_op) {
279	.reg => \|r\| {
280	rex.b = r.isExtended();
281	},
282	.mem => \|mem\| {
283	rex.b = if (mem.base()) \|base\| base.isExtended() else false;
284	rex.x = if (mem.scaleIndex()) \|si\| si.index.isExtended() else false;
285	},
286	else => unreachable,
287	}
288	},
289	}
290
291	try encoder.rex(rex);
292	}
293
294	fn encodeMandatoryPrefix(inst: Instruction, encoder: anytype) !void {
295	const prefix = inst.encoding.mandatoryPrefix() orelse return;
296	try encoder.opcode_1byte(prefix);
297	}
298
299	fn encodeMemory(encoding: Encoding, mem: Memory, operand: Operand, encoder: anytype) !void {
300	const operand_enc = switch (operand) {
301	.reg => \|reg\| reg.lowEnc(),
302	.none => encoding.modRmExt(),
303	else => unreachable,
304	};
305
306	switch (mem) {
307	.moffs => unreachable,
308	.sib => \|sib\| {
309	if (sib.base) \|base\| {
310	if (base.class() == .segment) {
311	// TODO audit this wrt SIB
312	try encoder.modRm_SIBDisp0(operand_enc);
313	if (sib.scale_index) \|si\| {
314	const scale = math.log2_int(u4, si.scale);
315	try encoder.sib_scaleIndexDisp32(scale, si.index.lowEnc());
316	} else {
317	try encoder.sib_disp32();
318	}
319	try encoder.disp32(sib.disp);
320	} else {
321	assert(base.class() == .general_purpose);
322	const dst = base.lowEnc();
323	const src = operand_enc;
324	if (dst == 4 or sib.scale_index != null) {
325	if (sib.disp == 0 and dst != 5) {
326	try encoder.modRm_SIBDisp0(src);
327	if (sib.scale_index) \|si\| {
328	const scale = math.log2_int(u4, si.scale);
329	try encoder.sib_scaleIndexBase(scale, si.index.lowEnc(), dst);
330	} else {
331	try encoder.sib_base(dst);
332	}
333	} else if (math.cast(i8, sib.disp)) \|_\| {
334	try encoder.modRm_SIBDisp8(src);
335	if (sib.scale_index) \|si\| {
336	const scale = math.log2_int(u4, si.scale);
337	try encoder.sib_scaleIndexBaseDisp8(scale, si.index.lowEnc(), dst);
338	} else {
339	try encoder.sib_baseDisp8(dst);
340	}
341	try encoder.disp8(@truncate(i8, sib.disp));
342	} else {
343	try encoder.modRm_SIBDisp32(src);
344	if (sib.scale_index) \|si\| {
345	const scale = math.log2_int(u4, si.scale);
346	try encoder.sib_scaleIndexBaseDisp32(scale, si.index.lowEnc(), dst);
347	} else {
348	try encoder.sib_baseDisp32(dst);
349	}
350	try encoder.disp32(sib.disp);
351	}
352	} else {
353	if (sib.disp == 0 and dst != 5) {
354	try encoder.modRm_indirectDisp0(src, dst);
355	} else if (math.cast(i8, sib.disp)) \|_\| {
356	try encoder.modRm_indirectDisp8(src, dst);
357	try encoder.disp8(@truncate(i8, sib.disp));
358	} else {
359	try encoder.modRm_indirectDisp32(src, dst);
360	try encoder.disp32(sib.disp);
361	}
362	}
363	}
364	} else {
365	try encoder.modRm_SIBDisp0(operand_enc);
366	if (sib.scale_index) \|si\| {
367	const scale = math.log2_int(u4, si.scale);
368	try encoder.sib_scaleIndexDisp32(scale, si.index.lowEnc());
369	} else {
370	try encoder.sib_disp32();
371	}
372	try encoder.disp32(sib.disp);
373	}
374	},
375	.rip => \|rip\| {
376	try encoder.modRm_RIPDisp32(operand_enc);
377	try encoder.disp32(rip.disp);
378	},
379	}
380	}
381
382	fn encodeImm(imm: Immediate, kind: Encoding.Op, encoder: anytype) !void {
383	const raw = imm.asUnsigned(kind.bitSize());
384	switch (kind.bitSize()) {
385	8 => try encoder.imm8(@intCast(u8, raw)),
386	16 => try encoder.imm16(@intCast(u16, raw)),
387	32 => try encoder.imm32(@intCast(u32, raw)),
388	64 => try encoder.imm64(raw),
389	else => unreachable,
390	}
391	}
392	};
393
394	inline fn sign(i: anytype) @TypeOf(i) {
395	return @as(@TypeOf(i), @boolToInt(i > 0)) - @boolToInt(i < 0);
396	}
397
398	pub const LegacyPrefixes = packed struct {
399	/// LOCK
400	prefix_f0: bool = false,
401	/// REPNZ, REPNE, REP, Scalar Double-precision
402	prefix_f2: bool = false,
403	/// REPZ, REPE, REP, Scalar Single-precision
404	prefix_f3: bool = false,
405
406	/// CS segment override or Branch not taken
407	prefix_2e: bool = false,
408	/// SS segment override
409	prefix_36: bool = false,
410	/// ES segment override
411	prefix_26: bool = false,
412	/// FS segment override
413	prefix_64: bool = false,
414	/// GS segment override
415	prefix_65: bool = false,
416
417	/// Branch taken
418	prefix_3e: bool = false,
419
420	/// Address size override (enables 16 bit address size)
421	prefix_67: bool = false,
422
423	/// Operand size override (enables 16 bit operation)
424	prefix_66: bool = false,
425
426	padding: u5 = 0,
427
428	pub fn setSegmentOverride(self: *LegacyPrefixes, reg: Register) void {
429	assert(reg.class() == .segment);
430	switch (reg) {
431	.cs => self.prefix_2e = true,
432	.ss => self.prefix_36 = true,
433	.es => self.prefix_26 = true,
434	.fs => self.prefix_64 = true,
435	.gs => self.prefix_65 = true,
436	.ds => {},
437	else => unreachable,
438	}
439	}
440
441	pub fn set16BitOverride(self: *LegacyPrefixes) void {
442	self.prefix_66 = true;
443	}
444	};
445
446	fn Encoder(comptime T: type) type {
447	return struct {
448	writer: T,
449
450	const Self = @This();
451
452	// --------
453	// Prefixes
454	// --------
455
456	/// Encodes legacy prefixes
457	pub fn legacyPrefixes(self: Self, prefixes: LegacyPrefixes) !void {
458	if (@bitCast(u16, prefixes) != 0) {
459	// Hopefully this path isn't taken very often, so we'll do it the slow way for now
460
461	// LOCK
462	if (prefixes.prefix_f0) try self.writer.writeByte(0xf0);
463	// REPNZ, REPNE, REP, Scalar Double-precision
464	if (prefixes.prefix_f2) try self.writer.writeByte(0xf2);
465	// REPZ, REPE, REP, Scalar Single-precision
466	if (prefixes.prefix_f3) try self.writer.writeByte(0xf3);
467
468	// CS segment override or Branch not taken
469	if (prefixes.prefix_2e) try self.writer.writeByte(0x2e);
470	// DS segment override
471	if (prefixes.prefix_36) try self.writer.writeByte(0x36);
472	// ES segment override
473	if (prefixes.prefix_26) try self.writer.writeByte(0x26);
474	// FS segment override
475	if (prefixes.prefix_64) try self.writer.writeByte(0x64);
476	// GS segment override
477	if (prefixes.prefix_65) try self.writer.writeByte(0x65);
478
479	// Branch taken
480	if (prefixes.prefix_3e) try self.writer.writeByte(0x3e);
481
482	// Operand size override
483	if (prefixes.prefix_66) try self.writer.writeByte(0x66);
484
485	// Address size override
486	if (prefixes.prefix_67) try self.writer.writeByte(0x67);
487	}
488	}
489
490	/// Use 16 bit operand size
491	///
492	/// Note that this flag is overridden by REX.W, if both are present.
493	pub fn prefix16BitMode(self: Self) !void {
494	try self.writer.writeByte(0x66);
495	}
496
497	/// Encodes a REX prefix byte given all the fields
498	///
499	/// Use this byte whenever you need 64 bit operation,
500	/// or one of reg, index, r/m, base, or opcode-reg might be extended.
501	///
502	/// See struct `Rex` for a description of each field.
503	pub fn rex(self: Self, byte: Rex) !void {
504	if (!byte.present and !byte.isSet()) return;
505
506	var value: u8 = 0b0100_0000;
507
508	if (byte.w) value \|= 0b1000;
509	if (byte.r) value \|= 0b0100;
510	if (byte.x) value \|= 0b0010;
511	if (byte.b) value \|= 0b0001;
512
513	try self.writer.writeByte(value);
514	}
515
516	// ------
517	// Opcode
518	// ------
519
520	/// Encodes a 1 byte opcode
521	pub fn opcode_1byte(self: Self, opcode: u8) !void {
522	try self.writer.writeByte(opcode);
523	}
524
525	/// Encodes a 2 byte opcode
526	///
527	/// e.g. IMUL has the opcode 0x0f 0xaf, so you use
528	///
529	/// encoder.opcode_2byte(0x0f, 0xaf);
530	pub fn opcode_2byte(self: Self, prefix: u8, opcode: u8) !void {
531	try self.writer.writeAll(&.{ prefix, opcode });
532	}
533
534	/// Encodes a 3 byte opcode
535	///
536	/// e.g. MOVSD has the opcode 0xf2 0x0f 0x10
537	///
538	/// encoder.opcode_3byte(0xf2, 0x0f, 0x10);
539	pub fn opcode_3byte(self: Self, prefix_1: u8, prefix_2: u8, opcode: u8) !void {
540	try self.writer.writeAll(&.{ prefix_1, prefix_2, opcode });
541	}
542
543	/// Encodes a 1 byte opcode with a reg field
544	///
545	/// Remember to add a REX prefix byte if reg is extended!
546	pub fn opcode_withReg(self: Self, opcode: u8, reg: u3) !void {
547	assert(opcode & 0b111 == 0);
548	try self.writer.writeByte(opcode \| reg);
549	}
550
551	// ------
552	// ModR/M
553	// ------
554
555	/// Construct a ModR/M byte given all the fields
556	///
557	/// Remember to add a REX prefix byte if reg or rm are extended!
558	pub fn modRm(self: Self, mod: u2, reg_or_opx: u3, rm: u3) !void {
559	try self.writer.writeByte(@as(u8, mod) << 6 \| @as(u8, reg_or_opx) << 3 \| rm);
560	}
561
562	/// Construct a ModR/M byte using direct r/m addressing
563	/// r/m effective address: r/m
564	///
565	/// Note reg's effective address is always just reg for the ModR/M byte.
566	/// Remember to add a REX prefix byte if reg or rm are extended!
567	pub fn modRm_direct(self: Self, reg_or_opx: u3, rm: u3) !void {
568	try self.modRm(0b11, reg_or_opx, rm);
569	}
570
571	/// Construct a ModR/M byte using indirect r/m addressing
572	/// r/m effective address: [r/m]
573	///
574	/// Note reg's effective address is always just reg for the ModR/M byte.
575	/// Remember to add a REX prefix byte if reg or rm are extended!
576	pub fn modRm_indirectDisp0(self: Self, reg_or_opx: u3, rm: u3) !void {
577	assert(rm != 4 and rm != 5);
578	try self.modRm(0b00, reg_or_opx, rm);
579	}
580
581	/// Construct a ModR/M byte using indirect SIB addressing
582	/// r/m effective address: [SIB]
583	///
584	/// Note reg's effective address is always just reg for the ModR/M byte.
585	/// Remember to add a REX prefix byte if reg or rm are extended!
586	pub fn modRm_SIBDisp0(self: Self, reg_or_opx: u3) !void {
587	try self.modRm(0b00, reg_or_opx, 0b100);
588	}
589
590	/// Construct a ModR/M byte using RIP-relative addressing
591	/// r/m effective address: [RIP + disp32]
592	///
593	/// Note reg's effective address is always just reg for the ModR/M byte.
594	/// Remember to add a REX prefix byte if reg or rm are extended!
595	pub fn modRm_RIPDisp32(self: Self, reg_or_opx: u3) !void {
596	try self.modRm(0b00, reg_or_opx, 0b101);
597	}
598
599	/// Construct a ModR/M byte using indirect r/m with a 8bit displacement
600	/// r/m effective address: [r/m + disp8]
601	///
602	/// Note reg's effective address is always just reg for the ModR/M byte.
603	/// Remember to add a REX prefix byte if reg or rm are extended!
604	pub fn modRm_indirectDisp8(self: Self, reg_or_opx: u3, rm: u3) !void {
605	assert(rm != 4);
606	try self.modRm(0b01, reg_or_opx, rm);
607	}
608
609	/// Construct a ModR/M byte using indirect SIB with a 8bit displacement
610	/// r/m effective address: [SIB + disp8]
611	///
612	/// Note reg's effective address is always just reg for the ModR/M byte.
613	/// Remember to add a REX prefix byte if reg or rm are extended!
614	pub fn modRm_SIBDisp8(self: Self, reg_or_opx: u3) !void {
615	try self.modRm(0b01, reg_or_opx, 0b100);
616	}
617
618	/// Construct a ModR/M byte using indirect r/m with a 32bit displacement
619	/// r/m effective address: [r/m + disp32]
620	///
621	/// Note reg's effective address is always just reg for the ModR/M byte.
622	/// Remember to add a REX prefix byte if reg or rm are extended!
623	pub fn modRm_indirectDisp32(self: Self, reg_or_opx: u3, rm: u3) !void {
624	assert(rm != 4);
625	try self.modRm(0b10, reg_or_opx, rm);
626	}
627
628	/// Construct a ModR/M byte using indirect SIB with a 32bit displacement
629	/// r/m effective address: [SIB + disp32]
630	///
631	/// Note reg's effective address is always just reg for the ModR/M byte.
632	/// Remember to add a REX prefix byte if reg or rm are extended!
633	pub fn modRm_SIBDisp32(self: Self, reg_or_opx: u3) !void {
634	try self.modRm(0b10, reg_or_opx, 0b100);
635	}
636
637	// ---
638	// SIB
639	// ---
640
641	/// Construct a SIB byte given all the fields
642	///
643	/// Remember to add a REX prefix byte if index or base are extended!
644	pub fn sib(self: Self, scale: u2, index: u3, base: u3) !void {
645	try self.writer.writeByte(@as(u8, scale) << 6 \| @as(u8, index) << 3 \| base);
646	}
647
648	/// Construct a SIB byte with scale * index + base, no frills.
649	/// r/m effective address: [base + scale * index]
650	///
651	/// Remember to add a REX prefix byte if index or base are extended!
652	pub fn sib_scaleIndexBase(self: Self, scale: u2, index: u3, base: u3) !void {
653	assert(base != 5);
654
655	try self.sib(scale, index, base);
656	}
657
658	/// Construct a SIB byte with scale * index + disp32
659	/// r/m effective address: [scale * index + disp32]
660	///
661	/// Remember to add a REX prefix byte if index or base are extended!
662	pub fn sib_scaleIndexDisp32(self: Self, scale: u2, index: u3) !void {
663	// scale is actually ignored
664	// index = 4 means no index if and only if we haven't extended the register
665	// TODO enforce this
666	// base = 5 means no base, if mod == 0.
667	try self.sib(scale, index, 5);
668	}
669
670	/// Construct a SIB byte with just base
671	/// r/m effective address: [base]
672	///
673	/// Remember to add a REX prefix byte if index or base are extended!
674	pub fn sib_base(self: Self, base: u3) !void {
675	assert(base != 5);
676
677	// scale is actually ignored
678	// index = 4 means no index
679	try self.sib(0, 4, base);
680	}
681
682	/// Construct a SIB byte with just disp32
683	/// r/m effective address: [disp32]
684	///
685	/// Remember to add a REX prefix byte if index or base are extended!
686	pub fn sib_disp32(self: Self) !void {
687	// scale is actually ignored
688	// index = 4 means no index
689	// base = 5 means no base, if mod == 0.
690	try self.sib(0, 4, 5);
691	}
692
693	/// Construct a SIB byte with scale * index + base + disp8
694	/// r/m effective address: [base + scale * index + disp8]
695	///
696	/// Remember to add a REX prefix byte if index or base are extended!
697	pub fn sib_scaleIndexBaseDisp8(self: Self, scale: u2, index: u3, base: u3) !void {
698	try self.sib(scale, index, base);
699	}
700
701	/// Construct a SIB byte with base + disp8, no index
702	/// r/m effective address: [base + disp8]
703	///
704	/// Remember to add a REX prefix byte if index or base are extended!
705	pub fn sib_baseDisp8(self: Self, base: u3) !void {
706	// scale is ignored
707	// index = 4 means no index
708	try self.sib(0, 4, base);
709	}
710
711	/// Construct a SIB byte with scale * index + base + disp32
712	/// r/m effective address: [base + scale * index + disp32]
713	///
714	/// Remember to add a REX prefix byte if index or base are extended!
715	pub fn sib_scaleIndexBaseDisp32(self: Self, scale: u2, index: u3, base: u3) !void {
716	try self.sib(scale, index, base);
717	}
718
719	/// Construct a SIB byte with base + disp32, no index
720	/// r/m effective address: [base + disp32]
721	///
722	/// Remember to add a REX prefix byte if index or base are extended!
723	pub fn sib_baseDisp32(self: Self, base: u3) !void {
724	// scale is ignored
725	// index = 4 means no index
726	try self.sib(0, 4, base);
727	}
728
729	// -------------------------
730	// Trivial (no bit fiddling)
731	// -------------------------
732
733	/// Encode an 8 bit displacement
734	///
735	/// It is sign-extended to 64 bits by the cpu.
736	pub fn disp8(self: Self, disp: i8) !void {
737	try self.writer.writeByte(@bitCast(u8, disp));
738	}
739
740	/// Encode an 32 bit displacement
741	///
742	/// It is sign-extended to 64 bits by the cpu.
743	pub fn disp32(self: Self, disp: i32) !void {
744	try self.writer.writeIntLittle(i32, disp);
745	}
746
747	/// Encode an 8 bit immediate
748	///
749	/// It is sign-extended to 64 bits by the cpu.
750	pub fn imm8(self: Self, imm: u8) !void {
751	try self.writer.writeByte(imm);
752	}
753
754	/// Encode an 16 bit immediate
755	///
756	/// It is sign-extended to 64 bits by the cpu.
757	pub fn imm16(self: Self, imm: u16) !void {
758	try self.writer.writeIntLittle(u16, imm);
759	}
760
761	/// Encode an 32 bit immediate
762	///
763	/// It is sign-extended to 64 bits by the cpu.
764	pub fn imm32(self: Self, imm: u32) !void {
765	try self.writer.writeIntLittle(u32, imm);
766	}
767
768	/// Encode an 64 bit immediate
769	///
770	/// It is sign-extended to 64 bits by the cpu.
771	pub fn imm64(self: Self, imm: u64) !void {
772	try self.writer.writeIntLittle(u64, imm);
773	}
774	};
775	}
776
777	pub const Rex = struct {
778	w: bool = false,
779	r: bool = false,
780	x: bool = false,
781	b: bool = false,
782	present: bool = false,
783
784	pub fn isSet(rex: Rex) bool {
785	return rex.w or rex.r or rex.x or rex.b;
786	}
787	};
788
789	// Tests
790	fn expectEqualHexStrings(expected: []const u8, given: []const u8, assembly: []const u8) !void {
791	assert(expected.len > 0);
792	if (std.mem.eql(u8, expected, given)) return;
793	const expected_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{std.fmt.fmtSliceHexLower(expected)});
794	defer testing.allocator.free(expected_fmt);
795	const given_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{std.fmt.fmtSliceHexLower(given)});
796	defer testing.allocator.free(given_fmt);
797	const idx = std.mem.indexOfDiff(u8, expected_fmt, given_fmt).?;
798	var padding = try testing.allocator.alloc(u8, idx + 5);
799	defer testing.allocator.free(padding);
800	std.mem.set(u8, padding, ' ');
801	std.debug.print("\nASM: {s}\nEXP: {s}\nGIV: {s}\n{s}^ -- first differing byte\n", .{
802	assembly,
803	expected_fmt,
804	given_fmt,
805	padding,
806	});
807	return error.TestFailed;
808	}
809
810	const TestEncode = struct {
811	buffer: [32]u8 = undefined,
812	index: usize = 0,
813
814	fn encode(enc: *TestEncode, mnemonic: Instruction.Mnemonic, args: struct {
815	op1: Instruction.Operand = .none,
816	op2: Instruction.Operand = .none,
817	op3: Instruction.Operand = .none,
818	op4: Instruction.Operand = .none,
819	}) !void {
820	var stream = std.io.fixedBufferStream(&enc.buffer);
821	var count_writer = std.io.countingWriter(stream.writer());
822	const inst = try Instruction.new(mnemonic, .{
823	.op1 = args.op1,
824	.op2 = args.op2,
825	.op3 = args.op3,
826	.op4 = args.op4,
827	});
828	try inst.encode(count_writer.writer());
829	enc.index = count_writer.bytes_written;
830	}
831
832	fn code(enc: TestEncode) []const u8 {
833	return enc.buffer[0..enc.index];
834	}
835	};
836
837	test "encode" {
838	var buf = std.ArrayList(u8).init(testing.allocator);
839	defer buf.deinit();
840
841	const inst = try Instruction.new(.mov, .{
842	.op1 = .{ .reg = .rbx },
843	.op2 = .{ .imm = Immediate.u(4) },
844	});
845	try inst.encode(buf.writer());
846	try testing.expectEqualSlices(u8, &.{ 0x48, 0xc7, 0xc3, 0x4, 0x0, 0x0, 0x0 }, buf.items);
847	}
848
849	test "lower I encoding" {
850	var enc = TestEncode{};
851
852	try enc.encode(.push, .{ .op1 = .{ .imm = Immediate.u(0x10) } });
853	try expectEqualHexStrings("\x6A\x10", enc.code(), "push 0x10");
854
855	try enc.encode(.push, .{ .op1 = .{ .imm = Immediate.u(0x1000) } });
856	try expectEqualHexStrings("\x66\x68\x00\x10", enc.code(), "push 0x1000");
857
858	try enc.encode(.push, .{ .op1 = .{ .imm = Immediate.u(0x10000000) } });
859	try expectEqualHexStrings("\x68\x00\x00\x00\x10", enc.code(), "push 0x10000000");
860
861	try enc.encode(.adc, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .imm = Immediate.u(0x10000000) } });
862	try expectEqualHexStrings("\x48\x15\x00\x00\x00\x10", enc.code(), "adc rax, 0x10000000");
863
864	try enc.encode(.add, .{ .op1 = .{ .reg = .al }, .op2 = .{ .imm = Immediate.u(0x10) } });
865	try expectEqualHexStrings("\x04\x10", enc.code(), "add al, 0x10");
866
867	try enc.encode(.add, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .imm = Immediate.u(0x10) } });
868	try expectEqualHexStrings("\x48\x83\xC0\x10", enc.code(), "add rax, 0x10");
869
870	try enc.encode(.sbb, .{ .op1 = .{ .reg = .ax }, .op2 = .{ .imm = Immediate.u(0x10) } });
871	try expectEqualHexStrings("\x66\x1D\x10\x00", enc.code(), "sbb ax, 0x10");
872
873	try enc.encode(.xor, .{ .op1 = .{ .reg = .al }, .op2 = .{ .imm = Immediate.u(0x10) } });
874	try expectEqualHexStrings("\x34\x10", enc.code(), "xor al, 0x10");
875	}
876
877	test "lower MI encoding" {
878	var enc = TestEncode{};
879
880	try enc.encode(.mov, .{ .op1 = .{ .reg = .r12 }, .op2 = .{ .imm = Immediate.u(0x1000) } });
881	try expectEqualHexStrings("\x49\xC7\xC4\x00\x10\x00\x00", enc.code(), "mov r12, 0x1000");
882
883	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.sib(.byte, .{
884	.base = .r12,
885	.disp = 0,
886	}) }, .op2 = .{ .imm = Immediate.u(0x10) } });
887	try expectEqualHexStrings("\x41\xC6\x04\x24\x10", enc.code(), "mov BYTE PTR [r12], 0x10");
888
889	try enc.encode(.mov, .{ .op1 = .{ .reg = .r12 }, .op2 = .{ .imm = Immediate.u(0x1000) } });
890	try expectEqualHexStrings("\x49\xC7\xC4\x00\x10\x00\x00", enc.code(), "mov r12, 0x1000");
891
892	try enc.encode(.mov, .{ .op1 = .{ .reg = .r12 }, .op2 = .{ .imm = Immediate.u(0x1000) } });
893	try expectEqualHexStrings("\x49\xC7\xC4\x00\x10\x00\x00", enc.code(), "mov r12, 0x1000");
894
895	try enc.encode(.mov, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .imm = Immediate.u(0x10) } });
896	try expectEqualHexStrings("\x48\xc7\xc0\x10\x00\x00\x00", enc.code(), "mov rax, 0x10");
897
898	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.sib(.dword, .{
899	.base = .r11,
900	.disp = 0,
901	}) }, .op2 = .{ .imm = Immediate.u(0x10) } });
902	try expectEqualHexStrings("\x41\xc7\x03\x10\x00\x00\x00", enc.code(), "mov DWORD PTR [r11], 0x10");
903
904	try enc.encode(.mov, .{
905	.op1 = .{ .mem = Memory.rip(.qword, 0x10) },
906	.op2 = .{ .imm = Immediate.u(0x10) },
907	});
908	try expectEqualHexStrings(
909	"\x48\xC7\x05\x10\x00\x00\x00\x10\x00\x00\x00",
910	enc.code(),
911	"mov QWORD PTR [rip + 0x10], 0x10",
912	);
913
914	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
915	.base = .rbp,
916	.disp = -8,
917	}) }, .op2 = .{ .imm = Immediate.u(0x10) } });
918	try expectEqualHexStrings("\x48\xc7\x45\xf8\x10\x00\x00\x00", enc.code(), "mov QWORD PTR [rbp - 8], 0x10");
919
920	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.sib(.word, .{
921	.base = .rbp,
922	.disp = -2,
923	}) }, .op2 = .{ .imm = Immediate.s(-16) } });
924	try expectEqualHexStrings("\x66\xC7\x45\xFE\xF0\xFF", enc.code(), "mov WORD PTR [rbp - 2], -16");
925
926	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.sib(.byte, .{
927	.base = .rbp,
928	.disp = -1,
929	}) }, .op2 = .{ .imm = Immediate.u(0x10) } });
930	try expectEqualHexStrings("\xC6\x45\xFF\x10", enc.code(), "mov BYTE PTR [rbp - 1], 0x10");
931
932	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
933	.base = .ds,
934	.disp = 0x10000000,
935	.scale_index = .{
936	.scale = 2,
937	.index = .rcx,
938	},
939	}) }, .op2 = .{ .imm = Immediate.u(0x10) } });
940	try expectEqualHexStrings(
941	"\x48\xC7\x04\x4D\x00\x00\x00\x10\x10\x00\x00\x00",
942	enc.code(),
943	"mov QWORD PTR [rcx*2 + 0x10000000], 0x10",
944	);
945
946	try enc.encode(.adc, .{ .op1 = .{ .mem = Memory.sib(.byte, .{
947	.base = .rbp,
948	.disp = -0x10,
949	}) }, .op2 = .{ .imm = Immediate.u(0x10) } });
950	try expectEqualHexStrings("\x80\x55\xF0\x10", enc.code(), "adc BYTE PTR [rbp - 0x10], 0x10");
951
952	try enc.encode(.adc, .{ .op1 = .{ .mem = Memory.rip(.qword, 0) }, .op2 = .{ .imm = Immediate.u(0x10) } });
953	try expectEqualHexStrings("\x48\x83\x15\x00\x00\x00\x00\x10", enc.code(), "adc QWORD PTR [rip], 0x10");
954
955	try enc.encode(.adc, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .imm = Immediate.u(0x10) } });
956	try expectEqualHexStrings("\x48\x83\xD0\x10", enc.code(), "adc rax, 0x10");
957
958	try enc.encode(.add, .{ .op1 = .{ .mem = Memory.sib(.dword, .{
959	.base = .rdx,
960	.disp = -8,
961	}) }, .op2 = .{ .imm = Immediate.u(0x10) } });
962	try expectEqualHexStrings("\x83\x42\xF8\x10", enc.code(), "add DWORD PTR [rdx - 8], 0x10");
963
964	try enc.encode(.add, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .imm = Immediate.u(0x10) } });
965	try expectEqualHexStrings("\x48\x83\xC0\x10", enc.code(), "add rax, 0x10");
966
967	try enc.encode(.add, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
968	.base = .rbp,
969	.disp = -0x10,
970	}) }, .op2 = .{ .imm = Immediate.s(-0x10) } });
971	try expectEqualHexStrings("\x48\x83\x45\xF0\xF0", enc.code(), "add QWORD PTR [rbp - 0x10], -0x10");
972
973	try enc.encode(.@"and", .{ .op1 = .{ .mem = Memory.sib(.dword, .{
974	.base = .ds,
975	.disp = 0x10000000,
976	}) }, .op2 = .{ .imm = Immediate.u(0x10) } });
977	try expectEqualHexStrings(
978	"\x83\x24\x25\x00\x00\x00\x10\x10",
979	enc.code(),
980	"and DWORD PTR ds:0x10000000, 0x10",
981	);
982
983	try enc.encode(.@"and", .{ .op1 = .{ .mem = Memory.sib(.dword, .{
984	.base = .es,
985	.disp = 0x10000000,
986	}) }, .op2 = .{ .imm = Immediate.u(0x10) } });
987	try expectEqualHexStrings(
988	"\x26\x83\x24\x25\x00\x00\x00\x10\x10",
989	enc.code(),
990	"and DWORD PTR es:0x10000000, 0x10",
991	);
992
993	try enc.encode(.@"and", .{ .op1 = .{ .mem = Memory.sib(.dword, .{
994	.base = .r12,
995	.disp = 0x10000000,
996	}) }, .op2 = .{ .imm = Immediate.u(0x10) } });
997	try expectEqualHexStrings(
998	"\x41\x83\xA4\x24\x00\x00\x00\x10\x10",
999	enc.code(),
1000	"and DWORD PTR [r12 + 0x10000000], 0x10",
1001	);
1002
1003	try enc.encode(.sub, .{ .op1 = .{ .mem = Memory.sib(.dword, .{
1004	.base = .r11,
1005	.disp = 0x10000000,
1006	}) }, .op2 = .{ .imm = Immediate.u(0x10) } });
1007	try expectEqualHexStrings(
1008	"\x41\x83\xAB\x00\x00\x00\x10\x10",
1009	enc.code(),
1010	"sub DWORD PTR [r11 + 0x10000000], 0x10",
1011	);
1012	}
1013
1014	test "lower RM encoding" {
1015	var enc = TestEncode{};
1016
1017	try enc.encode(.mov, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .mem = Memory.sib(.qword, .{
1018	.base = .r11,
1019	.disp = 0,
1020	}) } });
1021	try expectEqualHexStrings("\x49\x8b\x03", enc.code(), "mov rax, QWORD PTR [r11]");
1022
1023	try enc.encode(.mov, .{ .op1 = .{ .reg = .rbx }, .op2 = .{ .mem = Memory.sib(.qword, .{
1024	.base = .ds,
1025	.disp = 0x10,
1026	}) } });
1027	try expectEqualHexStrings("\x48\x8B\x1C\x25\x10\x00\x00\x00", enc.code(), "mov rbx, QWORD PTR ds:0x10");
1028
1029	try enc.encode(.mov, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .mem = Memory.sib(.qword, .{
1030	.base = .rbp,
1031	.disp = -4,
1032	}) } });
1033	try expectEqualHexStrings("\x48\x8B\x45\xFC", enc.code(), "mov rax, QWORD PTR [rbp - 4]");
1034
1035	try enc.encode(.mov, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .mem = Memory.sib(.qword, .{
1036	.base = .rbp,
1037	.scale_index = .{
1038	.scale = 1,
1039	.index = .rcx,
1040	},
1041	.disp = -8,
1042	}) } });
1043	try expectEqualHexStrings("\x48\x8B\x44\x0D\xF8", enc.code(), "mov rax, QWORD PTR [rbp + rcx*1 - 8]");
1044
1045	try enc.encode(.mov, .{ .op1 = .{ .reg = .eax }, .op2 = .{ .mem = Memory.sib(.dword, .{
1046	.base = .rbp,
1047	.scale_index = .{
1048	.scale = 4,
1049	.index = .rdx,
1050	},
1051	.disp = -4,
1052	}) } });
1053	try expectEqualHexStrings("\x8B\x44\x95\xFC", enc.code(), "mov eax, dword ptr [rbp + rdx*4 - 4]");
1054
1055	try enc.encode(.mov, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .mem = Memory.sib(.qword, .{
1056	.base = .rbp,
1057	.scale_index = .{
1058	.scale = 8,
1059	.index = .rcx,
1060	},
1061	.disp = -8,
1062	}) } });
1063	try expectEqualHexStrings("\x48\x8B\x44\xCD\xF8", enc.code(), "mov rax, QWORD PTR [rbp + rcx*8 - 8]");
1064
1065	try enc.encode(.mov, .{ .op1 = .{ .reg = .r8b }, .op2 = .{ .mem = Memory.sib(.byte, .{
1066	.base = .rsi,
1067	.scale_index = .{
1068	.scale = 1,
1069	.index = .rcx,
1070	},
1071	.disp = -24,
1072	}) } });
1073	try expectEqualHexStrings("\x44\x8A\x44\x0E\xE8", enc.code(), "mov r8b, BYTE PTR [rsi + rcx*1 - 24]");
1074
1075	// TODO this mnemonic needs cleanup as some prefixes are obsolete.
1076	try enc.encode(.mov, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .reg = .cs } });
1077	try expectEqualHexStrings("\x48\x8C\xC8", enc.code(), "mov rax, cs");
1078
1079	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
1080	.base = .rbp,
1081	.disp = -16,
1082	}) }, .op2 = .{ .reg = .fs } });
1083	try expectEqualHexStrings("\x48\x8C\x65\xF0", enc.code(), "mov QWORD PTR [rbp - 16], fs");
1084
1085	try enc.encode(.mov, .{ .op1 = .{ .reg = .r12w }, .op2 = .{ .reg = .cs } });
1086	try expectEqualHexStrings("\x66\x41\x8C\xCC", enc.code(), "mov r12w, cs");
1087
1088	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.sib(.word, .{
1089	.base = .rbp,
1090	.disp = -16,
1091	}) }, .op2 = .{ .reg = .fs } });
1092	try expectEqualHexStrings("\x66\x8C\x65\xF0", enc.code(), "mov WORD PTR [rbp - 16], fs");
1093
1094	try enc.encode(.movsx, .{ .op1 = .{ .reg = .eax }, .op2 = .{ .reg = .bx } });
1095	try expectEqualHexStrings("\x0F\xBF\xC3", enc.code(), "movsx eax, bx");
1096
1097	try enc.encode(.movsx, .{ .op1 = .{ .reg = .eax }, .op2 = .{ .reg = .bl } });
1098	try expectEqualHexStrings("\x0F\xBE\xC3", enc.code(), "movsx eax, bl");
1099
1100	try enc.encode(.movsx, .{ .op1 = .{ .reg = .ax }, .op2 = .{ .reg = .bl } });
1101	try expectEqualHexStrings("\x66\x0F\xBE\xC3", enc.code(), "movsx ax, bl");
1102
1103	try enc.encode(.movsx, .{ .op1 = .{ .reg = .eax }, .op2 = .{ .mem = Memory.sib(.word, .{
1104	.base = .rbp,
1105	.disp = 0,
1106	}) } });
1107	try expectEqualHexStrings("\x0F\xBF\x45\x00", enc.code(), "movsx eax, BYTE PTR [rbp]");
1108
1109	try enc.encode(.movsx, .{ .op1 = .{ .reg = .eax }, .op2 = .{ .mem = Memory.sib(.byte, .{
1110	.base = null,
1111	.scale_index = .{
1112	.index = .rax,
1113	.scale = 2,
1114	},
1115	.disp = 0,
1116	}) } });
1117	try expectEqualHexStrings("\x0F\xBE\x04\x45\x00\x00\x00\x00", enc.code(), "movsx eax, BYTE PTR [rax * 2]");
1118
1119	try enc.encode(.movsx, .{ .op1 = .{ .reg = .ax }, .op2 = .{ .mem = Memory.rip(.byte, 0x10) } });
1120	try expectEqualHexStrings("\x66\x0F\xBE\x05\x10\x00\x00\x00", enc.code(), "movsx ax, BYTE PTR [rip + 0x10]");
1121
1122	try enc.encode(.movsx, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .reg = .bx } });
1123	try expectEqualHexStrings("\x48\x0F\xBF\xC3", enc.code(), "movsx rax, bx");
1124
1125	try enc.encode(.movsxd, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .reg = .ebx } });
1126	try expectEqualHexStrings("\x48\x63\xC3", enc.code(), "movsxd rax, ebx");
1127
1128	try enc.encode(.lea, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .mem = Memory.rip(.qword, 0x10) } });
1129	try expectEqualHexStrings("\x48\x8D\x05\x10\x00\x00\x00", enc.code(), "lea rax, QWORD PTR [rip + 0x10]");
1130
1131	try enc.encode(.lea, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .mem = Memory.rip(.dword, 0x10) } });
1132	try expectEqualHexStrings("\x48\x8D\x05\x10\x00\x00\x00", enc.code(), "lea rax, DWORD PTR [rip + 0x10]");
1133
1134	try enc.encode(.lea, .{ .op1 = .{ .reg = .eax }, .op2 = .{ .mem = Memory.rip(.dword, 0x10) } });
1135	try expectEqualHexStrings("\x8D\x05\x10\x00\x00\x00", enc.code(), "lea eax, DWORD PTR [rip + 0x10]");
1136
1137	try enc.encode(.lea, .{ .op1 = .{ .reg = .eax }, .op2 = .{ .mem = Memory.rip(.word, 0x10) } });
1138	try expectEqualHexStrings("\x8D\x05\x10\x00\x00\x00", enc.code(), "lea eax, WORD PTR [rip + 0x10]");
1139
1140	try enc.encode(.lea, .{ .op1 = .{ .reg = .ax }, .op2 = .{ .mem = Memory.rip(.byte, 0x10) } });
1141	try expectEqualHexStrings("\x66\x8D\x05\x10\x00\x00\x00", enc.code(), "lea ax, BYTE PTR [rip + 0x10]");
1142
1143	try enc.encode(.lea, .{ .op1 = .{ .reg = .rsi }, .op2 = .{ .mem = Memory.sib(.qword, .{
1144	.base = .rbp,
1145	.scale_index = .{
1146	.scale = 1,
1147	.index = .rcx,
1148	},
1149	.disp = 0,
1150	}) } });
1151	try expectEqualHexStrings("\x48\x8D\x74\x0D\x00", enc.code(), "lea rsi, QWORD PTR [rbp + rcx*1 + 0]");
1152
1153	try enc.encode(.add, .{ .op1 = .{ .reg = .r11 }, .op2 = .{ .mem = Memory.sib(.qword, .{
1154	.base = .ds,
1155	.disp = 0x10000000,
1156	}) } });
1157	try expectEqualHexStrings("\x4C\x03\x1C\x25\x00\x00\x00\x10", enc.code(), "add r11, QWORD PTR ds:0x10000000");
1158
1159	try enc.encode(.add, .{ .op1 = .{ .reg = .r12b }, .op2 = .{ .mem = Memory.sib(.byte, .{
1160	.base = .ds,
1161	.disp = 0x10000000,
1162	}) } });
1163	try expectEqualHexStrings("\x44\x02\x24\x25\x00\x00\x00\x10", enc.code(), "add r11b, BYTE PTR ds:0x10000000");
1164
1165	try enc.encode(.add, .{ .op1 = .{ .reg = .r12b }, .op2 = .{ .mem = Memory.sib(.byte, .{
1166	.base = .fs,
1167	.disp = 0x10000000,
1168	}) } });
1169	try expectEqualHexStrings("\x64\x44\x02\x24\x25\x00\x00\x00\x10", enc.code(), "add r11b, BYTE PTR fs:0x10000000");
1170
1171	try enc.encode(.sub, .{ .op1 = .{ .reg = .r11 }, .op2 = .{ .mem = Memory.sib(.qword, .{
1172	.base = .r13,
1173	.disp = 0x10000000,
1174	}) } });
1175	try expectEqualHexStrings("\x4D\x2B\x9D\x00\x00\x00\x10", enc.code(), "sub r11, QWORD PTR [r13 + 0x10000000]");
1176
1177	try enc.encode(.sub, .{ .op1 = .{ .reg = .r11 }, .op2 = .{ .mem = Memory.sib(.qword, .{
1178	.base = .r12,
1179	.disp = 0x10000000,
1180	}) } });
1181	try expectEqualHexStrings("\x4D\x2B\x9C\x24\x00\x00\x00\x10", enc.code(), "sub r11, QWORD PTR [r12 + 0x10000000]");
1182
1183	try enc.encode(.imul, .{ .op1 = .{ .reg = .r11 }, .op2 = .{ .reg = .r12 } });
1184	try expectEqualHexStrings("\x4D\x0F\xAF\xDC", enc.code(), "mov r11, r12");
1185	}
1186
1187	test "lower RMI encoding" {
1188	var enc = TestEncode{};
1189
1190	try enc.encode(.imul, .{
1191	.op1 = .{ .reg = .r11 },
1192	.op2 = .{ .reg = .r12 },
1193	.op3 = .{ .imm = Immediate.s(-2) },
1194	});
1195	try expectEqualHexStrings("\x4D\x6B\xDC\xFE", enc.code(), "imul r11, r12, -2");
1196
1197	try enc.encode(.imul, .{
1198	.op1 = .{ .reg = .r11 },
1199	.op2 = .{ .mem = Memory.rip(.qword, -16) },
1200	.op3 = .{ .imm = Immediate.s(-1024) },
1201	});
1202	try expectEqualHexStrings(
1203	"\x4C\x69\x1D\xF0\xFF\xFF\xFF\x00\xFC\xFF\xFF",
1204	enc.code(),
1205	"imul r11, QWORD PTR [rip - 16], -1024",
1206	);
1207
1208	try enc.encode(.imul, .{
1209	.op1 = .{ .reg = .bx },
1210	.op2 = .{ .mem = Memory.sib(.word, .{
1211	.base = .rbp,
1212	.disp = -16,
1213	}) },
1214	.op3 = .{ .imm = Immediate.s(-1024) },
1215	});
1216	try expectEqualHexStrings(
1217	"\x66\x69\x5D\xF0\x00\xFC",
1218	enc.code(),
1219	"imul bx, WORD PTR [rbp - 16], -1024",
1220	);
1221
1222	try enc.encode(.imul, .{
1223	.op1 = .{ .reg = .bx },
1224	.op2 = .{ .mem = Memory.sib(.word, .{
1225	.base = .rbp,
1226	.disp = -16,
1227	}) },
1228	.op3 = .{ .imm = Immediate.u(1024) },
1229	});
1230	try expectEqualHexStrings(
1231	"\x66\x69\x5D\xF0\x00\x04",
1232	enc.code(),
1233	"imul bx, WORD PTR [rbp - 16], 1024",
1234	);
1235	}
1236
1237	test "lower MR encoding" {
1238	var enc = TestEncode{};
1239
1240	try enc.encode(.mov, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .reg = .rbx } });
1241	try expectEqualHexStrings("\x48\x89\xD8", enc.code(), "mov rax, rbx");
1242
1243	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
1244	.base = .rbp,
1245	.disp = -4,
1246	}) }, .op2 = .{ .reg = .r11 } });
1247	try expectEqualHexStrings("\x4c\x89\x5d\xfc", enc.code(), "mov QWORD PTR [rbp - 4], r11");
1248
1249	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.rip(.qword, 0x10) }, .op2 = .{ .reg = .r12 } });
1250	try expectEqualHexStrings("\x4C\x89\x25\x10\x00\x00\x00", enc.code(), "mov QWORD PTR [rip + 0x10], r12");
1251
1252	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
1253	.base = .r11,
1254	.scale_index = .{
1255	.scale = 2,
1256	.index = .r12,
1257	},
1258	.disp = 0x10,
1259	}) }, .op2 = .{ .reg = .r13 } });
1260	try expectEqualHexStrings("\x4F\x89\x6C\x63\x10", enc.code(), "mov QWORD PTR [r11 + 2 * r12 + 0x10], r13");
1261
1262	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.rip(.word, -0x10) }, .op2 = .{ .reg = .r12w } });
1263	try expectEqualHexStrings("\x66\x44\x89\x25\xF0\xFF\xFF\xFF", enc.code(), "mov WORD PTR [rip - 0x10], r12w");
1264
1265	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.sib(.byte, .{
1266	.base = .r11,
1267	.scale_index = .{
1268	.scale = 2,
1269	.index = .r12,
1270	},
1271	.disp = 0x10,
1272	}) }, .op2 = .{ .reg = .r13b } });
1273	try expectEqualHexStrings("\x47\x88\x6C\x63\x10", enc.code(), "mov BYTE PTR [r11 + 2 * r12 + 0x10], r13b");
1274
1275	try enc.encode(.add, .{ .op1 = .{ .mem = Memory.sib(.byte, .{
1276	.base = .ds,
1277	.disp = 0x10000000,
1278	}) }, .op2 = .{ .reg = .r12b } });
1279	try expectEqualHexStrings("\x44\x00\x24\x25\x00\x00\x00\x10", enc.code(), "add BYTE PTR ds:0x10000000, r12b");
1280
1281	try enc.encode(.add, .{ .op1 = .{ .mem = Memory.sib(.dword, .{
1282	.base = .ds,
1283	.disp = 0x10000000,
1284	}) }, .op2 = .{ .reg = .r12d } });
1285	try expectEqualHexStrings("\x44\x01\x24\x25\x00\x00\x00\x10", enc.code(), "add DWORD PTR [ds:0x10000000], r12d");
1286
1287	try enc.encode(.add, .{ .op1 = .{ .mem = Memory.sib(.dword, .{
1288	.base = .gs,
1289	.disp = 0x10000000,
1290	}) }, .op2 = .{ .reg = .r12d } });
1291	try expectEqualHexStrings("\x65\x44\x01\x24\x25\x00\x00\x00\x10", enc.code(), "add DWORD PTR [gs:0x10000000], r12d");
1292
1293	try enc.encode(.sub, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
1294	.base = .r11,
1295	.disp = 0x10000000,
1296	}) }, .op2 = .{ .reg = .r12 } });
1297	try expectEqualHexStrings("\x4D\x29\xA3\x00\x00\x00\x10", enc.code(), "sub QWORD PTR [r11 + 0x10000000], r12");
1298	}
1299
1300	test "lower M encoding" {
1301	var enc = TestEncode{};
1302
1303	try enc.encode(.call, .{ .op1 = .{ .reg = .r12 } });
1304	try expectEqualHexStrings("\x41\xFF\xD4", enc.code(), "call r12");
1305
1306	try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
1307	.base = .r12,
1308	.disp = 0,
1309	}) } });
1310	try expectEqualHexStrings("\x41\xFF\x14\x24", enc.code(), "call QWORD PTR [r12]");
1311
1312	try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
1313	.base = null,
1314	.scale_index = .{
1315	.index = .r11,
1316	.scale = 2,
1317	},
1318	.disp = 0,
1319	}) } });
1320	try expectEqualHexStrings("\x42\xFF\x14\x5D\x00\x00\x00\x00", enc.code(), "call QWORD PTR [r11 * 2]");
1321
1322	try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
1323	.base = null,
1324	.scale_index = .{
1325	.index = .r12,
1326	.scale = 2,
1327	},
1328	.disp = 0,
1329	}) } });
1330	try expectEqualHexStrings("\x42\xFF\x14\x65\x00\x00\x00\x00", enc.code(), "call QWORD PTR [r12 * 2]");
1331
1332	try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
1333	.base = .gs,
1334	.disp = 0,
1335	}) } });
1336	try expectEqualHexStrings("\x65\xFF\x14\x25\x00\x00\x00\x00", enc.code(), "call gs:0x0");
1337
1338	try enc.encode(.call, .{ .op1 = .{ .imm = Immediate.s(0) } });
1339	try expectEqualHexStrings("\xE8\x00\x00\x00\x00", enc.code(), "call 0x0");
1340
1341	try enc.encode(.push, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
1342	.base = .rbp,
1343	.disp = 0,
1344	}) } });
1345	try expectEqualHexStrings("\xFF\x75\x00", enc.code(), "push QWORD PTR [rbp]");
1346
1347	try enc.encode(.push, .{ .op1 = .{ .mem = Memory.sib(.word, .{
1348	.base = .rbp,
1349	.disp = 0,
1350	}) } });
1351	try expectEqualHexStrings("\x66\xFF\x75\x00", enc.code(), "push QWORD PTR [rbp]");
1352
1353	try enc.encode(.pop, .{ .op1 = .{ .mem = Memory.rip(.qword, 0) } });
1354	try expectEqualHexStrings("\x8F\x05\x00\x00\x00\x00", enc.code(), "pop QWORD PTR [rip]");
1355
1356	try enc.encode(.pop, .{ .op1 = .{ .mem = Memory.rip(.word, 0) } });
1357	try expectEqualHexStrings("\x66\x8F\x05\x00\x00\x00\x00", enc.code(), "pop WORD PTR [rbp]");
1358
1359	try enc.encode(.imul, .{ .op1 = .{ .reg = .rax } });
1360	try expectEqualHexStrings("\x48\xF7\xE8", enc.code(), "imul rax");
1361
1362	try enc.encode(.imul, .{ .op1 = .{ .reg = .r12 } });
1363	try expectEqualHexStrings("\x49\xF7\xEC", enc.code(), "imul r12");
1364	}
1365
1366	test "lower O encoding" {
1367	var enc = TestEncode{};
1368
1369	try enc.encode(.push, .{ .op1 = .{ .reg = .rax } });
1370	try expectEqualHexStrings("\x50", enc.code(), "push rax");
1371
1372	try enc.encode(.push, .{ .op1 = .{ .reg = .r12w } });
1373	try expectEqualHexStrings("\x66\x41\x54", enc.code(), "push r12w");
1374
1375	try enc.encode(.pop, .{ .op1 = .{ .reg = .r12 } });
1376	try expectEqualHexStrings("\x41\x5c", enc.code(), "pop r12");
1377	}
1378
1379	test "lower OI encoding" {
1380	var enc = TestEncode{};
1381
1382	try enc.encode(.mov, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .imm = Immediate.u(0x1000000000000000) } });
1383	try expectEqualHexStrings(
1384	"\x48\xB8\x00\x00\x00\x00\x00\x00\x00\x10",
1385	enc.code(),
1386	"movabs rax, 0x1000000000000000",
1387	);
1388
1389	try enc.encode(.mov, .{ .op1 = .{ .reg = .r11 }, .op2 = .{ .imm = Immediate.u(0x1000000000000000) } });
1390	try expectEqualHexStrings(
1391	"\x49\xBB\x00\x00\x00\x00\x00\x00\x00\x10",
1392	enc.code(),
1393	"movabs r11, 0x1000000000000000",
1394	);
1395
1396	try enc.encode(.mov, .{ .op1 = .{ .reg = .r11d }, .op2 = .{ .imm = Immediate.u(0x10000000) } });
1397	try expectEqualHexStrings("\x41\xBB\x00\x00\x00\x10", enc.code(), "mov r11d, 0x10000000");
1398
1399	try enc.encode(.mov, .{ .op1 = .{ .reg = .r11w }, .op2 = .{ .imm = Immediate.u(0x1000) } });
1400	try expectEqualHexStrings("\x66\x41\xBB\x00\x10", enc.code(), "mov r11w, 0x1000");
1401
1402	try enc.encode(.mov, .{ .op1 = .{ .reg = .r11b }, .op2 = .{ .imm = Immediate.u(0x10) } });
1403	try expectEqualHexStrings("\x41\xB3\x10", enc.code(), "mov r11b, 0x10");
1404	}
1405
1406	test "lower FD/TD encoding" {
1407	var enc = TestEncode{};
1408
1409	try enc.encode(.mov, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .mem = Memory.moffs(.cs, 0x10) } });
1410	try expectEqualHexStrings("\x2E\x48\xA1\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs rax, cs:0x10");
1411
1412	try enc.encode(.mov, .{ .op1 = .{ .reg = .eax }, .op2 = .{ .mem = Memory.moffs(.fs, 0x10) } });
1413	try expectEqualHexStrings("\x64\xA1\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs eax, fs:0x10");
1414
1415	try enc.encode(.mov, .{ .op1 = .{ .reg = .ax }, .op2 = .{ .mem = Memory.moffs(.gs, 0x10) } });
1416	try expectEqualHexStrings("\x65\x66\xA1\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs ax, gs:0x10");
1417
1418	try enc.encode(.mov, .{ .op1 = .{ .reg = .al }, .op2 = .{ .mem = Memory.moffs(.ds, 0x10) } });
1419	try expectEqualHexStrings("\xA0\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs al, ds:0x10");
1420
1421	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.moffs(.cs, 0x10) }, .op2 = .{ .reg = .rax } });
1422	try expectEqualHexStrings("\x2E\x48\xA3\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs cs:0x10, rax");
1423
1424	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.moffs(.fs, 0x10) }, .op2 = .{ .reg = .eax } });
1425	try expectEqualHexStrings("\x64\xA3\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs fs:0x10, eax");
1426
1427	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.moffs(.gs, 0x10) }, .op2 = .{ .reg = .ax } });
1428	try expectEqualHexStrings("\x65\x66\xA3\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs gs:0x10, ax");
1429
1430	try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.moffs(.ds, 0x10) }, .op2 = .{ .reg = .al } });
1431	try expectEqualHexStrings("\xA2\x10\x00\x00\x00\x00\x00\x00\x00", enc.code(), "movabs ds:0x10, al");
1432	}
1433
1434	test "lower NP encoding" {
1435	var enc = TestEncode{};
1436
1437	try enc.encode(.int3, .{});
1438	try expectEqualHexStrings("\xCC", enc.code(), "int3");
1439
1440	try enc.encode(.nop, .{});
1441	try expectEqualHexStrings("\x90", enc.code(), "nop");
1442
1443	try enc.encode(.ret, .{});
1444	try expectEqualHexStrings("\xC3", enc.code(), "ret");
1445
1446	try enc.encode(.syscall, .{});
1447	try expectEqualHexStrings("\x0f\x05", enc.code(), "syscall");
1448	}
1449
1450	fn invalidInstruction(mnemonic: Instruction.Mnemonic, args: struct {
1451	op1: Instruction.Operand = .none,
1452	op2: Instruction.Operand = .none,
1453	op3: Instruction.Operand = .none,
1454	op4: Instruction.Operand = .none,
1455	}) !void {
1456	const err = Instruction.new(mnemonic, .{
1457	.op1 = args.op1,
1458	.op2 = args.op2,
1459	.op3 = args.op3,
1460	.op4 = args.op4,
1461	});
1462	try testing.expectError(error.InvalidInstruction, err);
1463	}
1464
1465	test "invalid instruction" {
1466	try invalidInstruction(.call, .{ .op1 = .{ .reg = .eax } });
1467	try invalidInstruction(.call, .{ .op1 = .{ .reg = .ax } });
1468	try invalidInstruction(.call, .{ .op1 = .{ .reg = .al } });
1469	try invalidInstruction(.call, .{ .op1 = .{ .mem = Memory.rip(.dword, 0) } });
1470	try invalidInstruction(.call, .{ .op1 = .{ .mem = Memory.rip(.word, 0) } });
1471	try invalidInstruction(.call, .{ .op1 = .{ .mem = Memory.rip(.byte, 0) } });
1472	try invalidInstruction(.mov, .{ .op1 = .{ .mem = Memory.rip(.word, 0x10) }, .op2 = .{ .reg = .r12 } });
1473	try invalidInstruction(.lea, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .reg = .rbx } });
1474	try invalidInstruction(.lea, .{ .op1 = .{ .reg = .al }, .op2 = .{ .mem = Memory.rip(.byte, 0) } });
1475	try invalidInstruction(.pop, .{ .op1 = .{ .reg = .r12b } });
1476	try invalidInstruction(.pop, .{ .op1 = .{ .reg = .r12d } });
1477	try invalidInstruction(.push, .{ .op1 = .{ .reg = .r12b } });
1478	try invalidInstruction(.push, .{ .op1 = .{ .reg = .r12d } });
1479	try invalidInstruction(.push, .{ .op1 = .{ .imm = Immediate.u(0x1000000000000000) } });
1480	}
1481
1482	fn cannotEncode(mnemonic: Instruction.Mnemonic, args: struct {
1483	op1: Instruction.Operand = .none,
1484	op2: Instruction.Operand = .none,
1485	op3: Instruction.Operand = .none,
1486	op4: Instruction.Operand = .none,
1487	}) !void {
1488	try testing.expectError(error.CannotEncode, Instruction.new(mnemonic, .{
1489	.op1 = args.op1,
1490	.op2 = args.op2,
1491	.op3 = args.op3,
1492	.op4 = args.op4,
1493	}));
1494	}
1495
1496	test "cannot encode" {
1497	try cannotEncode(.@"test", .{
1498	.op1 = .{ .mem = Memory.sib(.byte, .{ .base = .r12, .disp = 0 }) },
1499	.op2 = .{ .reg = .ah },
1500	});
1501	try cannotEncode(.@"test", .{
1502	.op1 = .{ .reg = .r11b },
1503	.op2 = .{ .reg = .bh },
1504	});
1505	try cannotEncode(.mov, .{
1506	.op1 = .{ .reg = .sil },
1507	.op2 = .{ .reg = .ah },
1508	});
1509	}
1510
1511	const Assembler = struct {
1512	it: Tokenizer,
1513
1514	const Tokenizer = struct {
1515	input: []const u8,
1516	pos: usize = 0,
1517
1518	const Error = error{InvalidToken};
1519
1520	const Token = struct {
1521	id: Id,
1522	start: usize,
1523	end: usize,
1524
1525	const Id = enum {
1526	eof,
1527
1528	space,
1529	new_line,
1530
1531	colon,
1532	comma,
1533	open_br,
1534	close_br,
1535	plus,
1536	minus,
1537	star,
1538
1539	string,
1540	numeral,
1541	};
1542	};
1543
1544	const Iterator = struct {};
1545
1546	fn next(it: *Tokenizer) !Token {
1547	var result = Token{
1548	.id = .eof,
1549	.start = it.pos,
1550	.end = it.pos,
1551	};
1552
1553	var state: enum {
1554	start,
1555	space,
1556	new_line,
1557	string,
1558	numeral,
1559	numeral_hex,
1560	} = .start;
1561
1562	while (it.pos < it.input.len) : (it.pos += 1) {
1563	const ch = it.input[it.pos];
1564	switch (state) {
1565	.start => switch (ch) {
1566	',' => {
1567	result.id = .comma;
1568	it.pos += 1;
1569	break;
1570	},
1571	':' => {
1572	result.id = .colon;
1573	it.pos += 1;
1574	break;
1575	},
1576	'[' => {
1577	result.id = .open_br;
1578	it.pos += 1;
1579	break;
1580	},
1581	']' => {
1582	result.id = .close_br;
1583	it.pos += 1;
1584	break;
1585	},
1586	'+' => {
1587	result.id = .plus;
1588	it.pos += 1;
1589	break;
1590	},
1591	'-' => {
1592	result.id = .minus;
1593	it.pos += 1;
1594	break;
1595	},
1596	'*' => {
1597	result.id = .star;
1598	it.pos += 1;
1599	break;
1600	},
1601	' ', '\t' => state = .space,
1602	'\n', '\r' => state = .new_line,
1603	'a'...'z', 'A'...'Z' => state = .string,
1604	'0'...'9' => state = .numeral,
1605	else => return error.InvalidToken,
1606	},
1607
1608	.space => switch (ch) {
1609	' ', '\t' => {},
1610	else => {
1611	result.id = .space;
1612	break;
1613	},
1614	},
1615
1616	.new_line => switch (ch) {
1617	'\n', '\r', ' ', '\t' => {},
1618	else => {
1619	result.id = .new_line;
1620	break;
1621	},
1622	},
1623
1624	.string => switch (ch) {
1625	'a'...'z', 'A'...'Z', '0'...'9' => {},
1626	else => {
1627	result.id = .string;
1628	break;
1629	},
1630	},
1631
1632	.numeral => switch (ch) {
1633	'x' => state = .numeral_hex,
1634	'0'...'9' => {},
1635	else => {
1636	result.id = .numeral;
1637	break;
1638	},
1639	},
1640
1641	.numeral_hex => switch (ch) {
1642	'a'...'f' => {},
1643	'0'...'9' => {},
1644	else => {
1645	result.id = .numeral;
1646	break;
1647	},
1648	},
1649	}
1650	}
1651
1652	if (it.pos >= it.input.len) {
1653	switch (state) {
1654	.string => result.id = .string,
1655	.numeral, .numeral_hex => result.id = .numeral,
1656	else => {},
1657	}
1658	}
1659
1660	result.end = it.pos;
1661	return result;
1662	}
1663
1664	fn seekTo(it: *Tokenizer, pos: usize) void {
1665	it.pos = pos;
1666	}
1667	};
1668
1669	pub fn init(input: []const u8) Assembler {
1670	return .{
1671	.it = Tokenizer{ .input = input },
1672	};
1673	}
1674
1675	pub fn assemble(as: *Assembler, writer: anytype) !void {
1676	while (try as.next()) \|parsed_inst\| {
1677	const inst = try Instruction.new(parsed_inst.mnemonic, .{
1678	.op1 = parsed_inst.ops[0],
1679	.op2 = parsed_inst.ops[1],
1680	.op3 = parsed_inst.ops[2],
1681	.op4 = parsed_inst.ops[3],
1682	});
1683	try inst.encode(writer);
1684	}
1685	}
1686
1687	const ParseResult = struct {
1688	mnemonic: Instruction.Mnemonic,
1689	ops: [4]Instruction.Operand,
1690	};
1691
1692	const ParseError = error{
1693	UnexpectedToken,
1694	InvalidMnemonic,
1695	InvalidOperand,
1696	InvalidRegister,
1697	InvalidPtrSize,
1698	InvalidMemoryOperand,
1699	InvalidScaleIndex,
1700	} \|\| Tokenizer.Error \|\| std.fmt.ParseIntError;
1701
1702	fn next(as: *Assembler) ParseError!?ParseResult {
1703	try as.skip(2, .{ .space, .new_line });
1704	const mnemonic_tok = as.expect(.string) catch \|err\| switch (err) {
1705	error.UnexpectedToken => return if (try as.peek() == .eof) null else err,
1706	else => return err,
1707	};
1708	const mnemonic = mnemonicFromString(as.source(mnemonic_tok)) orelse
1709	return error.InvalidMnemonic;
1710	try as.skip(1, .{.space});
1711
1712	const rules = .{
1713	.{},
1714	.{.register},
1715	.{.memory},
1716	.{.immediate},
1717	.{ .register, .register },
1718	.{ .register, .memory },
1719	.{ .memory, .register },
1720	.{ .register, .immediate },
1721	.{ .memory, .immediate },
1722	.{ .register, .register, .immediate },
1723	.{ .register, .memory, .immediate },
1724	};
1725
1726	const pos = as.it.pos;
1727	inline for (rules) \|rule\| {
1728	var ops = [4]Instruction.Operand{ .none, .none, .none, .none };
1729	if (as.parseOperandRule(rule, &ops)) {
1730	return .{
1731	.mnemonic = mnemonic,
1732	.ops = ops,
1733	};
1734	} else \|_\| {
1735	as.it.seekTo(pos);
1736	}
1737	}
1738
1739	return error.InvalidOperand;
1740	}
1741
1742	fn source(as: *Assembler, token: Tokenizer.Token) []const u8 {
1743	return as.it.input[token.start..token.end];
1744	}
1745
1746	fn peek(as: *Assembler) Tokenizer.Error!Tokenizer.Token.Id {
1747	const pos = as.it.pos;
1748	const next_tok = try as.it.next();
1749	const id = next_tok.id;
1750	as.it.seekTo(pos);
1751	return id;
1752	}
1753
1754	fn expect(as: *Assembler, id: Tokenizer.Token.Id) ParseError!Tokenizer.Token {
1755	const next_tok_id = try as.peek();
1756	if (next_tok_id == id) return as.it.next();
1757	return error.UnexpectedToken;
1758	}
1759
1760	fn skip(as: *Assembler, comptime num: comptime_int, tok_ids: [num]Tokenizer.Token.Id) Tokenizer.Error!void {
1761	outer: while (true) {
1762	const pos = as.it.pos;
1763	const next_tok = try as.it.next();
1764	inline for (tok_ids) \|tok_id\| {
1765	if (next_tok.id == tok_id) continue :outer;
1766	}
1767	as.it.seekTo(pos);
1768	break;
1769	}
1770	}
1771
1772	fn mnemonicFromString(bytes: []const u8) ?Instruction.Mnemonic {
1773	const ti = @typeInfo(Instruction.Mnemonic).Enum;
1774	inline for (ti.fields) \|field\| {
1775	if (std.mem.eql(u8, bytes, field.name)) {
1776	return @field(Instruction.Mnemonic, field.name);
1777	}
1778	}
1779	return null;
1780	}
1781
1782	fn parseOperandRule(as: Assembler, rule: anytype, ops: [4]Instruction.Operand) ParseError!void {
1783	inline for (rule, 0..) \|cond, i\| {
1784	comptime assert(i < 4);
1785	if (i > 0) {
1786	_ = try as.expect(.comma);
1787	try as.skip(1, .{.space});
1788	}
1789	if (@typeInfo(@TypeOf(cond)) != .EnumLiteral) {
1790	@compileError("invalid condition in the rule: " ++ @typeName(@TypeOf(cond)));
1791	}
1792	switch (cond) {
1793	.register => {
1794	const reg_tok = try as.expect(.string);
1795	const reg = registerFromString(as.source(reg_tok)) orelse
1796	return error.InvalidOperand;
1797	ops[i] = .{ .reg = reg };
1798	},
1799	.memory => {
1800	const mem = try as.parseMemory();
1801	ops[i] = .{ .mem = mem };
1802	},
1803	.immediate => {
1804	const is_neg = if (as.expect(.minus)) \|_\| true else \|_\| false;
1805	const imm_tok = try as.expect(.numeral);
1806	const imm: Immediate = if (is_neg) blk: {
1807	const imm = try std.fmt.parseInt(i32, as.source(imm_tok), 0);
1808	break :blk .{ .signed = imm * -1 };
1809	} else .{ .unsigned = try std.fmt.parseInt(u64, as.source(imm_tok), 0) };
1810	ops[i] = .{ .imm = imm };
1811	},
1812	else => @compileError("unhandled enum literal " ++ @tagName(cond)),
1813	}
1814	try as.skip(1, .{.space});
1815	}
1816
1817	try as.skip(1, .{.space});
1818	const tok = try as.it.next();
1819	switch (tok.id) {
1820	.new_line, .eof => {},
1821	else => return error.InvalidOperand,
1822	}
1823	}
1824
1825	fn registerFromString(bytes: []const u8) ?Register {
1826	const ti = @typeInfo(Register).Enum;
1827	inline for (ti.fields) \|field\| {
1828	if (std.mem.eql(u8, bytes, field.name)) {
1829	return @field(Register, field.name);
1830	}
1831	}
1832	return null;
1833	}
1834
1835	fn parseMemory(as: *Assembler) ParseError!Memory {
1836	const ptr_size: ?Memory.PtrSize = blk: {
1837	const pos = as.it.pos;
1838	const ptr_size = as.parsePtrSize() catch \|err\| switch (err) {
1839	error.UnexpectedToken => {
1840	as.it.seekTo(pos);
1841	break :blk null;
1842	},
1843	else => return err,
1844	};
1845	break :blk ptr_size;
1846	};
1847
1848	try as.skip(1, .{.space});
1849
1850	// Supported rules and orderings.
1851	const rules = .{
1852	.{ .open_br, .base, .close_br }, // [ base ]
1853	.{ .open_br, .base, .plus, .disp, .close_br }, // [ base + disp ]
1854	.{ .open_br, .base, .minus, .disp, .close_br }, // [ base - disp ]
1855	.{ .open_br, .disp, .plus, .base, .close_br }, // [ disp + base ]
1856	.{ .open_br, .base, .plus, .index, .close_br }, // [ base + index ]
1857	.{ .open_br, .base, .plus, .index, .star, .scale, .close_br }, // [ base + index * scale ]
1858	.{ .open_br, .index, .star, .scale, .plus, .base, .close_br }, // [ index * scale + base ]
1859	.{ .open_br, .base, .plus, .index, .star, .scale, .plus, .disp, .close_br }, // [ base + index * scale + disp ]
1860	.{ .open_br, .base, .plus, .index, .star, .scale, .minus, .disp, .close_br }, // [ base + index * scale - disp ]
1861	.{ .open_br, .index, .star, .scale, .plus, .base, .plus, .disp, .close_br }, // [ index * scale + base + disp ]
1862	.{ .open_br, .index, .star, .scale, .plus, .base, .minus, .disp, .close_br }, // [ index * scale + base - disp ]
1863	.{ .open_br, .disp, .plus, .index, .star, .scale, .plus, .base, .close_br }, // [ disp + index * scale + base ]
1864	.{ .open_br, .disp, .plus, .base, .plus, .index, .star, .scale, .close_br }, // [ disp + base + index * scale ]
1865	.{ .open_br, .base, .plus, .disp, .plus, .index, .star, .scale, .close_br }, // [ base + disp + index * scale ]
1866	.{ .open_br, .base, .minus, .disp, .plus, .index, .star, .scale, .close_br }, // [ base - disp + index * scale ]
1867	.{ .open_br, .base, .plus, .disp, .plus, .scale, .star, .index, .close_br }, // [ base + disp + scale * index ]
1868	.{ .open_br, .base, .minus, .disp, .plus, .scale, .star, .index, .close_br }, // [ base - disp + scale * index ]
1869	.{ .open_br, .rip, .plus, .disp, .close_br }, // [ rip + disp ]
1870	.{ .open_br, .rip, .minus, .disp, .close_br }, // [ rig - disp ]
1871	.{ .base, .colon, .disp }, // seg:disp
1872	};
1873
1874	const pos = as.it.pos;
1875	inline for (rules) \|rule\| {
1876	if (as.parseMemoryRule(rule)) \|res\| {
1877	if (res.rip) {
1878	if (res.base != null or res.scale_index != null or res.offset != null)
1879	return error.InvalidMemoryOperand;
1880	return Memory.rip(ptr_size orelse .qword, res.disp orelse 0);
1881	}
1882	if (res.base) \|base\| {
1883	if (res.rip)
1884	return error.InvalidMemoryOperand;
1885	if (res.offset) \|offset\| {
1886	if (res.scale_index != null or res.disp != null)
1887	return error.InvalidMemoryOperand;
1888	return Memory.moffs(base, offset);
1889	}
1890	return Memory.sib(ptr_size orelse .qword, .{
1891	.base = base,
1892	.scale_index = res.scale_index,
1893	.disp = res.disp orelse 0,
1894	});
1895	}
1896	return error.InvalidMemoryOperand;
1897	} else \|_\| {
1898	as.it.seekTo(pos);
1899	}
1900	}
1901
1902	return error.InvalidOperand;
1903	}
1904
1905	const MemoryParseResult = struct {
1906	rip: bool = false,
1907	base: ?Register = null,
1908	scale_index: ?Memory.ScaleIndex = null,
1909	disp: ?i32 = null,
1910	offset: ?u64 = null,
1911	};
1912
1913	fn parseMemoryRule(as: *Assembler, rule: anytype) ParseError!MemoryParseResult {
1914	var res: MemoryParseResult = .{};
1915	inline for (rule, 0..) \|cond, i\| {
1916	if (@typeInfo(@TypeOf(cond)) != .EnumLiteral) {
1917	@compileError("unsupported condition type in the rule: " ++ @typeName(@TypeOf(cond)));
1918	}
1919	switch (cond) {
1920	.open_br, .close_br, .plus, .minus, .star, .colon => {
1921	_ = try as.expect(cond);
1922	},
1923	.base => {
1924	const tok = try as.expect(.string);
1925	res.base = registerFromString(as.source(tok)) orelse return error.InvalidMemoryOperand;
1926	},
1927	.rip => {
1928	const tok = try as.expect(.string);
1929	if (!std.mem.eql(u8, as.source(tok), "rip")) return error.InvalidMemoryOperand;
1930	res.rip = true;
1931	},
1932	.index => {
1933	const tok = try as.expect(.string);
1934	const index = registerFromString(as.source(tok)) orelse
1935	return error.InvalidMemoryOperand;
1936	if (res.scale_index) \|*si\| {
1937	si.index = index;
1938	} else {
1939	res.scale_index = .{ .scale = 1, .index = index };
1940	}
1941	},
1942	.scale => {
1943	const tok = try as.expect(.numeral);
1944	const scale = try std.fmt.parseInt(u2, as.source(tok), 0);
1945	if (res.scale_index) \|*si\| {
1946	si.scale = scale;
1947	} else {
1948	res.scale_index = .{ .scale = scale, .index = undefined };
1949	}
1950	},
1951	.disp => {
1952	const tok = try as.expect(.numeral);
1953	const is_neg = blk: {
1954	if (i > 0) {
1955	if (rule[i - 1] == .minus) break :blk true;
1956	}
1957	break :blk false;
1958	};
1959	if (std.fmt.parseInt(i32, as.source(tok), 0)) \|disp\| {
1960	res.disp = if (is_neg) -1 * disp else disp;
1961	} else \|err\| switch (err) {
1962	error.Overflow => {
1963	if (is_neg) return err;
1964	if (res.base) \|base\| {
1965	if (base.class() != .segment) return err;
1966	}
1967	const offset = try std.fmt.parseInt(u64, as.source(tok), 0);
1968	res.offset = offset;
1969	},
1970	else => return err,
1971	}
1972	},
1973	else => @compileError("unhandled operand output type: " ++ @tagName(cond)),
1974	}
1975	try as.skip(1, .{.space});
1976	}
1977	return res;
1978	}
1979
1980	fn parsePtrSize(as: *Assembler) ParseError!Memory.PtrSize {
1981	const size = try as.expect(.string);
1982	try as.skip(1, .{.space});
1983	const ptr = try as.expect(.string);
1984
1985	const size_raw = as.source(size);
1986	const ptr_raw = as.source(ptr);
1987	const len = size_raw.len + ptr_raw.len + 1;
1988	var buf: ["qword ptr".len]u8 = undefined;
1989	if (len > buf.len) return error.InvalidPtrSize;
1990
1991	for (size_raw, 0..) \|c, i\| {
1992	buf[i] = std.ascii.toLower(c);
1993	}
1994	buf[size_raw.len] = ' ';
1995	for (ptr_raw, 0..) \|c, i\| {
1996	buf[size_raw.len + i + 1] = std.ascii.toLower(c);
1997	}
1998
1999	const slice = buf[0..len];
2000	if (std.mem.eql(u8, slice, "qword ptr")) return .qword;
2001	if (std.mem.eql(u8, slice, "dword ptr")) return .dword;
2002	if (std.mem.eql(u8, slice, "word ptr")) return .word;
2003	if (std.mem.eql(u8, slice, "byte ptr")) return .byte;
2004	if (std.mem.eql(u8, slice, "tbyte ptr")) return .tbyte;
2005	return error.InvalidPtrSize;
2006	}
2007	};
2008
2009	test "assemble" {
2010	const input =
2011	\\int3
2012	\\mov rax, rbx
2013	\\mov qword ptr [rbp], rax
2014	\\mov qword ptr [rbp - 16], rax
2015	\\mov qword ptr [16 + rbp], rax
2016	\\mov rax, 0x10
2017	\\mov byte ptr [rbp - 0x10], 0x10
2018	\\mov word ptr [rbp + r12], r11w
2019	\\mov word ptr [rbp + r12 * 2], r11w
2020	\\mov word ptr [rbp + r12 * 2 - 16], r11w
2021	\\mov dword ptr [rip - 16], r12d
2022	\\mov rax, fs:0x0
2023	\\mov rax, gs:0x1000000000000000
2024	\\movzx r12, al
2025	\\imul r12, qword ptr [rbp - 16], 6
2026	\\jmp 0x0
2027	\\jc 0x0
2028	\\jb 0x0
2029	\\sal rax, 1
2030	\\sal rax, 63
2031	\\shl rax, 63
2032	\\sar rax, 63
2033	\\shr rax, 63
2034	\\test byte ptr [rbp - 16], r12b
2035	\\sal r12, cl
2036	\\mul qword ptr [rip - 16]
2037	\\div r12
2038	\\idiv byte ptr [rbp - 16]
2039	\\cwde
2040	\\cbw
2041	\\cdqe
2042	\\test byte ptr [rbp], ah
2043	\\test byte ptr [r12], spl
2044	\\cdq
2045	\\cwd
2046	\\cqo
2047	\\test bl, 0x1
2048	\\mov rbx,0x8000000000000000
2049	\\movss xmm0, dword ptr [rbp]
2050	\\movss xmm0, xmm1
2051	\\movss dword ptr [rbp - 16 + rax * 2], xmm7
2052	\\movss dword ptr [rbp - 16 + rax * 2], xmm8
2053	\\movss xmm15, xmm9
2054	\\movsd xmm8, qword ptr [rbp - 16]
2055	\\movsd qword ptr [rbp - 8], xmm0
2056	\\movq xmm8, qword ptr [rbp - 16]
2057	\\movq qword ptr [rbp - 16], xmm8
2058	\\ucomisd xmm0, qword ptr [rbp - 16]
2059	\\fisttp qword ptr [rbp - 16]
2060	\\fisttp word ptr [rip + 32]
2061	\\fisttp dword ptr [rax]
2062	\\fld tbyte ptr [rbp]
2063	\\fld dword ptr [rbp]
2064	\\xor bl, 0xff
2065	\\ud2
2066	\\add rsp, -1
2067	\\add rsp, 0xff
2068	\\mov sil, byte ptr [rax + rcx * 1]
2069	\\
2070	;
2071
2072	// zig fmt: off
2073	const expected = &[_]u8{
2074	0xCC,
2075	0x48, 0x89, 0xD8,
2076	0x48, 0x89, 0x45, 0x00,
2077	0x48, 0x89, 0x45, 0xF0,
2078	0x48, 0x89, 0x45, 0x10,
2079	0x48, 0xC7, 0xC0, 0x10, 0x00, 0x00, 0x00,
2080	0xC6, 0x45, 0xF0, 0x10,
2081	0x66, 0x46, 0x89, 0x5C, 0x25, 0x00,
2082	0x66, 0x46, 0x89, 0x5C, 0x65, 0x00,
2083	0x66, 0x46, 0x89, 0x5C, 0x65, 0xF0,
2084	0x44, 0x89, 0x25, 0xF0, 0xFF, 0xFF, 0xFF,
2085	0x64, 0x48, 0x8B, 0x04, 0x25, 0x00, 0x00, 0x00, 0x00,
2086	0x65, 0x48, 0xA1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10,
2087	0x4C, 0x0F, 0xB6, 0xE0,
2088	0x4C, 0x6B, 0x65, 0xF0, 0x06,
2089	0xE9, 0x00, 0x00, 0x00, 0x00,
2090	0x0F, 0x82, 0x00, 0x00, 0x00, 0x00,
2091	0x0F, 0x82, 0x00, 0x00, 0x00, 0x00,
2092	0x48, 0xD1, 0xE0,
2093	0x48, 0xC1, 0xE0, 0x3F,
2094	0x48, 0xC1, 0xE0, 0x3F,
2095	0x48, 0xC1, 0xF8, 0x3F,
2096	0x48, 0xC1, 0xE8, 0x3F,
2097	0x44, 0x84, 0x65, 0xF0,
2098	0x49, 0xD3, 0xE4,
2099	0x48, 0xF7, 0x25, 0xF0, 0xFF, 0xFF, 0xFF,
2100	0x49, 0xF7, 0xF4,
2101	0xF6, 0x7D, 0xF0,
2102	0x98,
2103	0x66, 0x98,
2104	0x48, 0x98,
2105	0x84, 0x65, 0x00,
2106	0x41, 0x84, 0x24, 0x24,
2107	0x99,
2108	0x66, 0x99,
2109	0x48, 0x99,
2110	0xF6, 0xC3, 0x01,
2111	0x48, 0xBB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80,
2112	0xF3, 0x0F, 0x10, 0x45, 0x00,
2113	0xF3, 0x0F, 0x10, 0xC1,
2114	0xF3, 0x0F, 0x11, 0x7C, 0x45, 0xF0,
2115	0xF3, 0x44, 0x0F, 0x11, 0x44, 0x45, 0xF0,
2116	0xF3, 0x45, 0x0F, 0x10, 0xF9,
2117	0xF2, 0x44, 0x0F, 0x10, 0x45, 0xF0,
2118	0xF2, 0x0F, 0x11, 0x45, 0xF8,
2119	0xF3, 0x44, 0x0F, 0x7E, 0x45, 0xF0,
2120	0x66, 0x44, 0x0F, 0xD6, 0x45, 0xF0,
2121	0x66, 0x0F, 0x2E, 0x45, 0xF0,
2122	0xDD, 0x4D, 0xF0,
2123	0xDF, 0x0D, 0x20, 0x00, 0x00, 0x00,
2124	0xDB, 0x08,
2125	0xDB, 0x6D, 0x00,
2126	0xD9, 0x45, 0x00,
2127	0x80, 0xF3, 0xFF,
2128	0x0F, 0x0B,
2129	0x48, 0x83, 0xC4, 0xFF,
2130	0x48, 0x81, 0xC4, 0xFF, 0x00, 0x00, 0x00,
2131	0x40, 0x8A, 0x34, 0x08,
2132	};
2133	// zig fmt: on
2134
2135	var as = Assembler.init(input);
2136	var output = std.ArrayList(u8).init(testing.allocator);
2137	defer output.deinit();
2138	try as.assemble(output.writer());
2139	try expectEqualHexStrings(expected, output.items, input);
2140	}
2141
2142	test "assemble - Jcc" {
2143	const mnemonics = [_]struct { Instruction.Mnemonic, u8 }{
2144	.{ .ja, 0x87 },
2145	.{ .jae, 0x83 },
2146	.{ .jb, 0x82 },
2147	.{ .jbe, 0x86 },
2148	.{ .jc, 0x82 },
2149	.{ .je, 0x84 },
2150	.{ .jg, 0x8f },
2151	.{ .jge, 0x8d },
2152	.{ .jl, 0x8c },
2153	.{ .jle, 0x8e },
2154	.{ .jna, 0x86 },
2155	.{ .jnae, 0x82 },
2156	.{ .jnb, 0x83 },
2157	.{ .jnbe, 0x87 },
2158	.{ .jnc, 0x83 },
2159	.{ .jne, 0x85 },
2160	.{ .jng, 0x8e },
2161	.{ .jnge, 0x8c },
2162	.{ .jnl, 0x8d },
2163	.{ .jnle, 0x8f },
2164	.{ .jno, 0x81 },
2165	.{ .jnp, 0x8b },
2166	.{ .jns, 0x89 },
2167	.{ .jnz, 0x85 },
2168	.{ .jo, 0x80 },
2169	.{ .jp, 0x8a },
2170	.{ .jpe, 0x8a },
2171	.{ .jpo, 0x8b },
2172	.{ .js, 0x88 },
2173	.{ .jz, 0x84 },
2174	};
2175
2176	inline for (&mnemonics) \|mnemonic\| {
2177	const input = @tagName(mnemonic[0]) ++ " 0x0";
2178	const expected = [_]u8{ 0x0f, mnemonic[1], 0x0, 0x0, 0x0, 0x0 };
2179	var as = Assembler.init(input);
2180	var output = std.ArrayList(u8).init(testing.allocator);
2181	defer output.deinit();
2182	try as.assemble(output.writer());
2183	try expectEqualHexStrings(&expected, output.items, input);
2184	}
2185	}
2186
2187	test "assemble - SETcc" {
2188	const mnemonics = [_]struct { Instruction.Mnemonic, u8 }{
2189	.{ .seta, 0x97 },
2190	.{ .setae, 0x93 },
2191	.{ .setb, 0x92 },
2192	.{ .setbe, 0x96 },
2193	.{ .setc, 0x92 },
2194	.{ .sete, 0x94 },
2195	.{ .setg, 0x9f },
2196	.{ .setge, 0x9d },
2197	.{ .setl, 0x9c },
2198	.{ .setle, 0x9e },
2199	.{ .setna, 0x96 },
2200	.{ .setnae, 0x92 },
2201	.{ .setnb, 0x93 },
2202	.{ .setnbe, 0x97 },
2203	.{ .setnc, 0x93 },
2204	.{ .setne, 0x95 },
2205	.{ .setng, 0x9e },
2206	.{ .setnge, 0x9c },
2207	.{ .setnl, 0x9d },
2208	.{ .setnle, 0x9f },
2209	.{ .setno, 0x91 },
2210	.{ .setnp, 0x9b },
2211	.{ .setns, 0x99 },
2212	.{ .setnz, 0x95 },
2213	.{ .seto, 0x90 },
2214	.{ .setp, 0x9a },
2215	.{ .setpe, 0x9a },
2216	.{ .setpo, 0x9b },
2217	.{ .sets, 0x98 },
2218	.{ .setz, 0x94 },
2219	};
2220
2221	inline for (&mnemonics) \|mnemonic\| {
2222	const input = @tagName(mnemonic[0]) ++ " al";
2223	const expected = [_]u8{ 0x0f, mnemonic[1], 0xC0 };
2224	var as = Assembler.init(input);
2225	var output = std.ArrayList(u8).init(testing.allocator);
2226	defer output.deinit();
2227	try as.assemble(output.writer());
2228	try expectEqualHexStrings(&expected, output.items, input);
2229	}
2230	}
2231
2232	test "assemble - CMOVcc" {
2233	const mnemonics = [_]struct { Instruction.Mnemonic, u8 }{
2234	.{ .cmova, 0x47 },
2235	.{ .cmovae, 0x43 },
2236	.{ .cmovb, 0x42 },
2237	.{ .cmovbe, 0x46 },
2238	.{ .cmovc, 0x42 },
2239	.{ .cmove, 0x44 },
2240	.{ .cmovg, 0x4f },
2241	.{ .cmovge, 0x4d },
2242	.{ .cmovl, 0x4c },
2243	.{ .cmovle, 0x4e },
2244	.{ .cmovna, 0x46 },
2245	.{ .cmovnae, 0x42 },
2246	.{ .cmovnb, 0x43 },
2247	.{ .cmovnbe, 0x47 },
2248	.{ .cmovnc, 0x43 },
2249	.{ .cmovne, 0x45 },
2250	.{ .cmovng, 0x4e },
2251	.{ .cmovnge, 0x4c },
2252	.{ .cmovnl, 0x4d },
2253	.{ .cmovnle, 0x4f },
2254	.{ .cmovno, 0x41 },
2255	.{ .cmovnp, 0x4b },
2256	.{ .cmovns, 0x49 },
2257	.{ .cmovnz, 0x45 },
2258	.{ .cmovo, 0x40 },
2259	.{ .cmovp, 0x4a },
2260	.{ .cmovpe, 0x4a },
2261	.{ .cmovpo, 0x4b },
2262	.{ .cmovs, 0x48 },
2263	.{ .cmovz, 0x44 },
2264	};
2265
2266	inline for (&mnemonics) \|mnemonic\| {
2267	const input = @tagName(mnemonic[0]) ++ " rax, rbx";
2268	const expected = [_]u8{ 0x48, 0x0f, mnemonic[1], 0xC3 };
2269	var as = Assembler.init(input);
2270	var output = std.ArrayList(u8).init(testing.allocator);
2271	defer output.deinit();
2272	try as.assemble(output.writer());
2273	try expectEqualHexStrings(&expected, output.items, input);
2274	}
2275	}

src/arch/x86_64/encodings.zig created+621

...	@@ -0,0 +1,621 @@
1	const Encoding = @import("Encoding.zig");
2	const Mnemonic = Encoding.Mnemonic;
3	const OpEn = Encoding.OpEn;
4	const Op = Encoding.Op;
5	const Mode = Encoding.Mode;
6
7	const opcode_len = u2;
8	const modrm_ext = u3;
9
10	const Entry = struct { Mnemonic, OpEn, Op, Op, Op, Op, opcode_len, u8, u8, u8, modrm_ext, Mode };
11
12	// TODO move this into a .zon file when Zig is capable of importing .zon files
13	// zig fmt: off
14	pub const table = &[_]Entry{
15	// General-purpose
16	.{ .adc, .zi, .al, .imm8, .none, .none, 1, 0x14, 0x00, 0x00, 0, .none },
17	.{ .adc, .zi, .ax, .imm16, .none, .none, 1, 0x15, 0x00, 0x00, 0, .none },
18	.{ .adc, .zi, .eax, .imm32, .none, .none, 1, 0x15, 0x00, 0x00, 0, .none },
19	.{ .adc, .zi, .rax, .imm32s, .none, .none, 1, 0x15, 0x00, 0x00, 0, .long },
20	.{ .adc, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 2, .none },
21	.{ .adc, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 2, .rex },
22	.{ .adc, .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 2, .none },
23	.{ .adc, .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 2, .none },
24	.{ .adc, .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 2, .long },
25	.{ .adc, .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 2, .none },
26	.{ .adc, .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 2, .none },
27	.{ .adc, .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 2, .long },
28	.{ .adc, .mr, .rm8, .r8, .none, .none, 1, 0x10, 0x00, 0x00, 0, .none },
29	.{ .adc, .mr, .rm8, .r8, .none, .none, 1, 0x10, 0x00, 0x00, 0, .rex },
30	.{ .adc, .mr, .rm16, .r16, .none, .none, 1, 0x11, 0x00, 0x00, 0, .none },
31	.{ .adc, .mr, .rm32, .r32, .none, .none, 1, 0x11, 0x00, 0x00, 0, .none },
32	.{ .adc, .mr, .rm64, .r64, .none, .none, 1, 0x11, 0x00, 0x00, 0, .long },
33	.{ .adc, .rm, .r8, .rm8, .none, .none, 1, 0x12, 0x00, 0x00, 0, .none },
34	.{ .adc, .rm, .r8, .rm8, .none, .none, 1, 0x12, 0x00, 0x00, 0, .rex },
35	.{ .adc, .rm, .r16, .rm16, .none, .none, 1, 0x13, 0x00, 0x00, 0, .none },
36	.{ .adc, .rm, .r32, .rm32, .none, .none, 1, 0x13, 0x00, 0x00, 0, .none },
37	.{ .adc, .rm, .r64, .rm64, .none, .none, 1, 0x13, 0x00, 0x00, 0, .long },
38
39	.{ .add, .zi, .al, .imm8, .none, .none, 1, 0x04, 0x00, 0x00, 0, .none },
40	.{ .add, .zi, .ax, .imm16, .none, .none, 1, 0x05, 0x00, 0x00, 0, .none },
41	.{ .add, .zi, .eax, .imm32, .none, .none, 1, 0x05, 0x00, 0x00, 0, .none },
42	.{ .add, .zi, .rax, .imm32s, .none, .none, 1, 0x05, 0x00, 0x00, 0, .long },
43	.{ .add, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 0, .none },
44	.{ .add, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 0, .rex },
45	.{ .add, .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 0, .none },
46	.{ .add, .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 0, .none },
47	.{ .add, .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 0, .long },
48	.{ .add, .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 0, .none },
49	.{ .add, .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 0, .none },
50	.{ .add, .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 0, .long },
51	.{ .add, .mr, .rm8, .r8, .none, .none, 1, 0x00, 0x00, 0x00, 0, .none },
52	.{ .add, .mr, .rm8, .r8, .none, .none, 1, 0x00, 0x00, 0x00, 0, .rex },
53	.{ .add, .mr, .rm16, .r16, .none, .none, 1, 0x01, 0x00, 0x00, 0, .none },
54	.{ .add, .mr, .rm32, .r32, .none, .none, 1, 0x01, 0x00, 0x00, 0, .none },
55	.{ .add, .mr, .rm64, .r64, .none, .none, 1, 0x01, 0x00, 0x00, 0, .long },
56	.{ .add, .rm, .r8, .rm8, .none, .none, 1, 0x02, 0x00, 0x00, 0, .none },
57	.{ .add, .rm, .r8, .rm8, .none, .none, 1, 0x02, 0x00, 0x00, 0, .rex },
58	.{ .add, .rm, .r16, .rm16, .none, .none, 1, 0x03, 0x00, 0x00, 0, .none },
59	.{ .add, .rm, .r32, .rm32, .none, .none, 1, 0x03, 0x00, 0x00, 0, .none },
60	.{ .add, .rm, .r64, .rm64, .none, .none, 1, 0x03, 0x00, 0x00, 0, .long },
61
62	.{ .@"and", .zi, .al, .imm8, .none, .none, 1, 0x24, 0x00, 0x00, 0, .none },
63	.{ .@"and", .zi, .ax, .imm16, .none, .none, 1, 0x25, 0x00, 0x00, 0, .none },
64	.{ .@"and", .zi, .eax, .imm32, .none, .none, 1, 0x25, 0x00, 0x00, 0, .none },
65	.{ .@"and", .zi, .rax, .imm32s, .none, .none, 1, 0x25, 0x00, 0x00, 0, .long },
66	.{ .@"and", .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 4, .none },
67	.{ .@"and", .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 4, .rex },
68	.{ .@"and", .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 4, .none },
69	.{ .@"and", .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 4, .none },
70	.{ .@"and", .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 4, .long },
71	.{ .@"and", .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 4, .none },
72	.{ .@"and", .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 4, .none },
73	.{ .@"and", .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 4, .long },
74	.{ .@"and", .mr, .rm8, .r8, .none, .none, 1, 0x20, 0x00, 0x00, 0, .none },
75	.{ .@"and", .mr, .rm8, .r8, .none, .none, 1, 0x20, 0x00, 0x00, 0, .rex },
76	.{ .@"and", .mr, .rm16, .r16, .none, .none, 1, 0x21, 0x00, 0x00, 0, .none },
77	.{ .@"and", .mr, .rm32, .r32, .none, .none, 1, 0x21, 0x00, 0x00, 0, .none },
78	.{ .@"and", .mr, .rm64, .r64, .none, .none, 1, 0x21, 0x00, 0x00, 0, .long },
79	.{ .@"and", .rm, .r8, .rm8, .none, .none, 1, 0x22, 0x00, 0x00, 0, .none },
80	.{ .@"and", .rm, .r8, .rm8, .none, .none, 1, 0x22, 0x00, 0x00, 0, .rex },
81	.{ .@"and", .rm, .r16, .rm16, .none, .none, 1, 0x23, 0x00, 0x00, 0, .none },
82	.{ .@"and", .rm, .r32, .rm32, .none, .none, 1, 0x23, 0x00, 0x00, 0, .none },
83	.{ .@"and", .rm, .r64, .rm64, .none, .none, 1, 0x23, 0x00, 0x00, 0, .long },
84
85	// This is M encoding according to Intel, but D makes more sense here.
86	.{ .call, .d, .rel32, .none, .none, .none, 1, 0xe8, 0x00, 0x00, 0, .none },
87	.{ .call, .m, .rm64, .none, .none, .none, 1, 0xff, 0x00, 0x00, 2, .none },
88
89	.{ .cbw, .np, .o16, .none, .none, .none, 1, 0x98, 0x00, 0x00, 0, .none },
90	.{ .cwde, .np, .o32, .none, .none, .none, 1, 0x98, 0x00, 0x00, 0, .none },
91	.{ .cdqe, .np, .o64, .none, .none, .none, 1, 0x98, 0x00, 0x00, 0, .long },
92
93	.{ .cwd, .np, .o16, .none, .none, .none, 1, 0x99, 0x00, 0x00, 0, .none },
94	.{ .cdq, .np, .o32, .none, .none, .none, 1, 0x99, 0x00, 0x00, 0, .none },
95	.{ .cqo, .np, .o64, .none, .none, .none, 1, 0x99, 0x00, 0x00, 0, .long },
96
97	.{ .cmova, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .none },
98	.{ .cmova, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .none },
99	.{ .cmova, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .long },
100	.{ .cmovae, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none },
101	.{ .cmovae, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none },
102	.{ .cmovae, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .long },
103	.{ .cmovb, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none },
104	.{ .cmovb, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none },
105	.{ .cmovb, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .long },
106	.{ .cmovbe, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .none },
107	.{ .cmovbe, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .none },
108	.{ .cmovbe, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .long },
109	.{ .cmovc, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none },
110	.{ .cmovc, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none },
111	.{ .cmovc, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .long },
112	.{ .cmove, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .none },
113	.{ .cmove, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .none },
114	.{ .cmove, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .long },
115	.{ .cmovg, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .none },
116	.{ .cmovg, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .none },
117	.{ .cmovg, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .long },
118	.{ .cmovge, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .none },
119	.{ .cmovge, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .none },
120	.{ .cmovge, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .long },
121	.{ .cmovl, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .none },
122	.{ .cmovl, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .none },
123	.{ .cmovl, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .long },
124	.{ .cmovle, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .none },
125	.{ .cmovle, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .none },
126	.{ .cmovle, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .long },
127	.{ .cmovna, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .none },
128	.{ .cmovna, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .none },
129	.{ .cmovna, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .long },
130	.{ .cmovnae, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none },
131	.{ .cmovnae, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none },
132	.{ .cmovnae, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .long },
133	.{ .cmovnb, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none },
134	.{ .cmovnb, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none },
135	.{ .cmovnb, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .long },
136	.{ .cmovnbe, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .none },
137	.{ .cmovnbe, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .none },
138	.{ .cmovnbe, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .long },
139	.{ .cmovnc, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none },
140	.{ .cmovnc, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none },
141	.{ .cmovnc, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .long },
142	.{ .cmovne, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .none },
143	.{ .cmovne, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .none },
144	.{ .cmovne, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .long },
145	.{ .cmovng, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .none },
146	.{ .cmovng, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .none },
147	.{ .cmovng, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .long },
148	.{ .cmovnge, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .none },
149	.{ .cmovnge, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .none },
150	.{ .cmovnge, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .long },
151	.{ .cmovnl, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .none },
152	.{ .cmovnl, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .none },
153	.{ .cmovnl, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .long },
154	.{ .cmovnle, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .none },
155	.{ .cmovnle, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .none },
156	.{ .cmovnle, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .long },
157	.{ .cmovno, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x41, 0x00, 0, .none },
158	.{ .cmovno, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x41, 0x00, 0, .none },
159	.{ .cmovno, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x41, 0x00, 0, .long },
160	.{ .cmovnp, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .none },
161	.{ .cmovnp, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .none },
162	.{ .cmovnp, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .long },
163	.{ .cmovns, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x49, 0x00, 0, .none },
164	.{ .cmovns, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x49, 0x00, 0, .none },
165	.{ .cmovns, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x49, 0x00, 0, .long },
166	.{ .cmovnz, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .none },
167	.{ .cmovnz, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .none },
168	.{ .cmovnz, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .long },
169	.{ .cmovo, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x40, 0x00, 0, .none },
170	.{ .cmovo, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x40, 0x00, 0, .none },
171	.{ .cmovo, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x40, 0x00, 0, .long },
172	.{ .cmovp, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .none },
173	.{ .cmovp, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .none },
174	.{ .cmovp, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .long },
175	.{ .cmovpe, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .none },
176	.{ .cmovpe, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .none },
177	.{ .cmovpe, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .long },
178	.{ .cmovpo, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .none },
179	.{ .cmovpo, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .none },
180	.{ .cmovpo, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .long },
181	.{ .cmovs, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x48, 0x00, 0, .none },
182	.{ .cmovs, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x48, 0x00, 0, .none },
183	.{ .cmovs, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x48, 0x00, 0, .long },
184	.{ .cmovz, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .none },
185	.{ .cmovz, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .none },
186	.{ .cmovz, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .long },
187
188	.{ .cmp, .zi, .al, .imm8, .none, .none, 1, 0x3c, 0x00, 0x00, 0, .none },
189	.{ .cmp, .zi, .ax, .imm16, .none, .none, 1, 0x3d, 0x00, 0x00, 0, .none },
190	.{ .cmp, .zi, .eax, .imm32, .none, .none, 1, 0x3d, 0x00, 0x00, 0, .none },
191	.{ .cmp, .zi, .rax, .imm32s, .none, .none, 1, 0x3d, 0x00, 0x00, 0, .long },
192	.{ .cmp, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 7, .none },
193	.{ .cmp, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 7, .rex },
194	.{ .cmp, .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 7, .none },
195	.{ .cmp, .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 7, .none },
196	.{ .cmp, .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 7, .long },
197	.{ .cmp, .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 7, .none },
198	.{ .cmp, .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 7, .none },
199	.{ .cmp, .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 7, .long },
200	.{ .cmp, .mr, .rm8, .r8, .none, .none, 1, 0x38, 0x00, 0x00, 0, .none },
201	.{ .cmp, .mr, .rm8, .r8, .none, .none, 1, 0x38, 0x00, 0x00, 0, .rex },
202	.{ .cmp, .mr, .rm16, .r16, .none, .none, 1, 0x39, 0x00, 0x00, 0, .none },
203	.{ .cmp, .mr, .rm32, .r32, .none, .none, 1, 0x39, 0x00, 0x00, 0, .none },
204	.{ .cmp, .mr, .rm64, .r64, .none, .none, 1, 0x39, 0x00, 0x00, 0, .long },
205	.{ .cmp, .rm, .r8, .rm8, .none, .none, 1, 0x3a, 0x00, 0x00, 0, .none },
206	.{ .cmp, .rm, .r8, .rm8, .none, .none, 1, 0x3a, 0x00, 0x00, 0, .rex },
207	.{ .cmp, .rm, .r16, .rm16, .none, .none, 1, 0x3b, 0x00, 0x00, 0, .none },
208	.{ .cmp, .rm, .r32, .rm32, .none, .none, 1, 0x3b, 0x00, 0x00, 0, .none },
209	.{ .cmp, .rm, .r64, .rm64, .none, .none, 1, 0x3b, 0x00, 0x00, 0, .long },
210
211	.{ .div, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 6, .none },
212	.{ .div, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 6, .rex },
213	.{ .div, .m, .rm16, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 6, .none },
214	.{ .div, .m, .rm32, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 6, .none },
215	.{ .div, .m, .rm64, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 6, .long },
216
217	.{ .fisttp, .m, .m16, .none, .none, .none, 1, 0xdf, 0x00, 0x00, 1, .fpu },
218	.{ .fisttp, .m, .m32, .none, .none, .none, 1, 0xdb, 0x00, 0x00, 1, .fpu },
219	.{ .fisttp, .m, .m64, .none, .none, .none, 1, 0xdd, 0x00, 0x00, 1, .fpu },
220
221	.{ .fld, .m, .m32, .none, .none, .none, 1, 0xd9, 0x00, 0x00, 0, .fpu },
222	.{ .fld, .m, .m64, .none, .none, .none, 1, 0xdd, 0x00, 0x00, 0, .fpu },
223	.{ .fld, .m, .m80, .none, .none, .none, 1, 0xdb, 0x00, 0x00, 5, .fpu },
224
225	.{ .idiv, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 7, .none },
226	.{ .idiv, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 7, .rex },
227	.{ .idiv, .m, .rm16, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 7, .none },
228	.{ .idiv, .m, .rm32, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 7, .none },
229	.{ .idiv, .m, .rm64, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 7, .long },
230
231	.{ .imul, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 5, .none },
232	.{ .imul, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 5, .rex },
233	.{ .imul, .m, .rm16, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 5, .none },
234	.{ .imul, .m, .rm32, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 5, .none },
235	.{ .imul, .m, .rm64, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 5, .long },
236	.{ .imul, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0xaf, 0x00, 0, .none },
237	.{ .imul, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0xaf, 0x00, 0, .none },
238	.{ .imul, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0xaf, 0x00, 0, .long },
239	.{ .imul, .rmi, .r16, .rm16, .imm8s, .none, 1, 0x6b, 0x00, 0x00, 0, .none },
240	.{ .imul, .rmi, .r32, .rm32, .imm8s, .none, 1, 0x6b, 0x00, 0x00, 0, .none },
241	.{ .imul, .rmi, .r64, .rm64, .imm8s, .none, 1, 0x6b, 0x00, 0x00, 0, .long },
242	.{ .imul, .rmi, .r16, .rm16, .imm16, .none, 1, 0x69, 0x00, 0x00, 0, .none },
243	.{ .imul, .rmi, .r32, .rm32, .imm32, .none, 1, 0x69, 0x00, 0x00, 0, .none },
244	.{ .imul, .rmi, .r64, .rm64, .imm32, .none, 1, 0x69, 0x00, 0x00, 0, .long },
245
246	.{ .int3, .np, .none, .none, .none, .none, 1, 0xcc, 0x00, 0x00, 0, .none },
247
248	.{ .ja, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x87, 0x00, 0, .none },
249	.{ .jae, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x83, 0x00, 0, .none },
250	.{ .jb, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x82, 0x00, 0, .none },
251	.{ .jbe, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x86, 0x00, 0, .none },
252	.{ .jc, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x82, 0x00, 0, .none },
253	.{ .jrcxz, .d, .rel32, .none, .none, .none, 1, 0xe3, 0x00, 0x00, 0, .none },
254	.{ .je, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x84, 0x00, 0, .none },
255	.{ .jg, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8f, 0x00, 0, .none },
256	.{ .jge, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8d, 0x00, 0, .none },
257	.{ .jl, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8c, 0x00, 0, .none },
258	.{ .jle, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8e, 0x00, 0, .none },
259	.{ .jna, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x86, 0x00, 0, .none },
260	.{ .jnae, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x82, 0x00, 0, .none },
261	.{ .jnb, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x83, 0x00, 0, .none },
262	.{ .jnbe, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x87, 0x00, 0, .none },
263	.{ .jnc, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x83, 0x00, 0, .none },
264	.{ .jne, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x85, 0x00, 0, .none },
265	.{ .jng, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8e, 0x00, 0, .none },
266	.{ .jnge, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8c, 0x00, 0, .none },
267	.{ .jnl, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8d, 0x00, 0, .none },
268	.{ .jnle, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8f, 0x00, 0, .none },
269	.{ .jno, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x81, 0x00, 0, .none },
270	.{ .jnp, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8b, 0x00, 0, .none },
271	.{ .jns, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x89, 0x00, 0, .none },
272	.{ .jnz, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x85, 0x00, 0, .none },
273	.{ .jo, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x80, 0x00, 0, .none },
274	.{ .jp, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8a, 0x00, 0, .none },
275	.{ .jpe, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8a, 0x00, 0, .none },
276	.{ .jpo, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8b, 0x00, 0, .none },
277	.{ .js, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x88, 0x00, 0, .none },
278	.{ .jz, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x84, 0x00, 0, .none },
279
280	.{ .jmp, .d, .rel32, .none, .none, .none, 1, 0xe9, 0x00, 0x00, 0, .none },
281	.{ .jmp, .m, .rm64, .none, .none, .none, 1, 0xff, 0x00, 0x00, 4, .none },
282
283	.{ .lea, .rm, .r16, .m, .none, .none, 1, 0x8d, 0x00, 0x00, 0, .none },
284	.{ .lea, .rm, .r32, .m, .none, .none, 1, 0x8d, 0x00, 0x00, 0, .none },
285	.{ .lea, .rm, .r64, .m, .none, .none, 1, 0x8d, 0x00, 0x00, 0, .long },
286
287	.{ .mov, .mr, .rm8, .r8, .none, .none, 1, 0x88, 0x00, 0x00, 0, .none },
288	.{ .mov, .mr, .rm8, .r8, .none, .none, 1, 0x88, 0x00, 0x00, 0, .rex },
289	.{ .mov, .mr, .rm16, .r16, .none, .none, 1, 0x89, 0x00, 0x00, 0, .none },
290	.{ .mov, .mr, .rm32, .r32, .none, .none, 1, 0x89, 0x00, 0x00, 0, .none },
291	.{ .mov, .mr, .rm64, .r64, .none, .none, 1, 0x89, 0x00, 0x00, 0, .long },
292	.{ .mov, .rm, .r8, .rm8, .none, .none, 1, 0x8a, 0x00, 0x00, 0, .none },
293	.{ .mov, .rm, .r8, .rm8, .none, .none, 1, 0x8a, 0x00, 0x00, 0, .rex },
294	.{ .mov, .rm, .r16, .rm16, .none, .none, 1, 0x8b, 0x00, 0x00, 0, .none },
295	.{ .mov, .rm, .r32, .rm32, .none, .none, 1, 0x8b, 0x00, 0x00, 0, .none },
296	.{ .mov, .rm, .r64, .rm64, .none, .none, 1, 0x8b, 0x00, 0x00, 0, .long },
297	.{ .mov, .mr, .rm16, .sreg, .none, .none, 1, 0x8c, 0x00, 0x00, 0, .none },
298	.{ .mov, .mr, .rm64, .sreg, .none, .none, 1, 0x8c, 0x00, 0x00, 0, .long },
299	.{ .mov, .rm, .sreg, .rm16, .none, .none, 1, 0x8e, 0x00, 0x00, 0, .none },
300	.{ .mov, .rm, .sreg, .rm64, .none, .none, 1, 0x8e, 0x00, 0x00, 0, .long },
301	.{ .mov, .fd, .al, .moffs, .none, .none, 1, 0xa0, 0x00, 0x00, 0, .none },
302	.{ .mov, .fd, .ax, .moffs, .none, .none, 1, 0xa1, 0x00, 0x00, 0, .none },
303	.{ .mov, .fd, .eax, .moffs, .none, .none, 1, 0xa1, 0x00, 0x00, 0, .none },
304	.{ .mov, .fd, .rax, .moffs, .none, .none, 1, 0xa1, 0x00, 0x00, 0, .long },
305	.{ .mov, .td, .moffs, .al, .none, .none, 1, 0xa2, 0x00, 0x00, 0, .none },
306	.{ .mov, .td, .moffs, .ax, .none, .none, 1, 0xa3, 0x00, 0x00, 0, .none },
307	.{ .mov, .td, .moffs, .eax, .none, .none, 1, 0xa3, 0x00, 0x00, 0, .none },
308	.{ .mov, .td, .moffs, .rax, .none, .none, 1, 0xa3, 0x00, 0x00, 0, .long },
309	.{ .mov, .oi, .r8, .imm8, .none, .none, 1, 0xb0, 0x00, 0x00, 0, .none },
310	.{ .mov, .oi, .r8, .imm8, .none, .none, 1, 0xb0, 0x00, 0x00, 0, .rex },
311	.{ .mov, .oi, .r16, .imm16, .none, .none, 1, 0xb8, 0x00, 0x00, 0, .none },
312	.{ .mov, .oi, .r32, .imm32, .none, .none, 1, 0xb8, 0x00, 0x00, 0, .none },
313	.{ .mov, .oi, .r64, .imm64, .none, .none, 1, 0xb8, 0x00, 0x00, 0, .long },
314	.{ .mov, .mi, .rm8, .imm8, .none, .none, 1, 0xc6, 0x00, 0x00, 0, .none },
315	.{ .mov, .mi, .rm8, .imm8, .none, .none, 1, 0xc6, 0x00, 0x00, 0, .rex },
316	.{ .mov, .mi, .rm16, .imm16, .none, .none, 1, 0xc7, 0x00, 0x00, 0, .none },
317	.{ .mov, .mi, .rm32, .imm32, .none, .none, 1, 0xc7, 0x00, 0x00, 0, .none },
318	.{ .mov, .mi, .rm64, .imm32s, .none, .none, 1, 0xc7, 0x00, 0x00, 0, .long },
319
320	.{ .movsx, .rm, .r16, .rm8, .none, .none, 2, 0x0f, 0xbe, 0x00, 0, .none },
321	.{ .movsx, .rm, .r16, .rm8, .none, .none, 2, 0x0f, 0xbe, 0x00, 0, .rex },
322	.{ .movsx, .rm, .r32, .rm8, .none, .none, 2, 0x0f, 0xbe, 0x00, 0, .none },
323	.{ .movsx, .rm, .r32, .rm8, .none, .none, 2, 0x0f, 0xbe, 0x00, 0, .rex },
324	.{ .movsx, .rm, .r64, .rm8, .none, .none, 2, 0x0f, 0xbe, 0x00, 0, .long },
325	.{ .movsx, .rm, .r32, .rm16, .none, .none, 2, 0x0f, 0xbf, 0x00, 0, .none },
326	.{ .movsx, .rm, .r64, .rm16, .none, .none, 2, 0x0f, 0xbf, 0x00, 0, .long },
327
328	// This instruction is discouraged.
329	.{ .movsxd, .rm, .r32, .rm32, .none, .none, 1, 0x63, 0x00, 0x00, 0, .none },
330	.{ .movsxd, .rm, .r64, .rm32, .none, .none, 1, 0x63, 0x00, 0x00, 0, .long },
331
332	.{ .movzx, .rm, .r16, .rm8, .none, .none, 2, 0x0f, 0xb6, 0x00, 0, .none },
333	.{ .movzx, .rm, .r32, .rm8, .none, .none, 2, 0x0f, 0xb6, 0x00, 0, .none },
334	.{ .movzx, .rm, .r64, .rm8, .none, .none, 2, 0x0f, 0xb6, 0x00, 0, .long },
335	.{ .movzx, .rm, .r32, .rm16, .none, .none, 2, 0x0f, 0xb7, 0x00, 0, .none },
336	.{ .movzx, .rm, .r64, .rm16, .none, .none, 2, 0x0f, 0xb7, 0x00, 0, .long },
337
338	.{ .mul, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 4, .none },
339	.{ .mul, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 4, .rex },
340	.{ .mul, .m, .rm16, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 4, .none },
341	.{ .mul, .m, .rm32, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 4, .none },
342	.{ .mul, .m, .rm64, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 4, .long },
343
344	.{ .nop, .np, .none, .none, .none, .none, 1, 0x90, 0x00, 0x00, 0, .none },
345
346	.{ .@"or", .zi, .al, .imm8, .none, .none, 1, 0x0c, 0x00, 0x00, 0, .none },
347	.{ .@"or", .zi, .ax, .imm16, .none, .none, 1, 0x0d, 0x00, 0x00, 0, .none },
348	.{ .@"or", .zi, .eax, .imm32, .none, .none, 1, 0x0d, 0x00, 0x00, 0, .none },
349	.{ .@"or", .zi, .rax, .imm32s, .none, .none, 1, 0x0d, 0x00, 0x00, 0, .long },
350	.{ .@"or", .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 1, .none },
351	.{ .@"or", .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 1, .rex },
352	.{ .@"or", .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 1, .none },
353	.{ .@"or", .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 1, .none },
354	.{ .@"or", .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 1, .long },
355	.{ .@"or", .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 1, .none },
356	.{ .@"or", .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 1, .none },
357	.{ .@"or", .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 1, .long },
358	.{ .@"or", .mr, .rm8, .r8, .none, .none, 1, 0x08, 0x00, 0x00, 0, .none },
359	.{ .@"or", .mr, .rm8, .r8, .none, .none, 1, 0x08, 0x00, 0x00, 0, .rex },
360	.{ .@"or", .mr, .rm16, .r16, .none, .none, 1, 0x09, 0x00, 0x00, 0, .none },
361	.{ .@"or", .mr, .rm32, .r32, .none, .none, 1, 0x09, 0x00, 0x00, 0, .none },
362	.{ .@"or", .mr, .rm64, .r64, .none, .none, 1, 0x09, 0x00, 0x00, 0, .long },
363	.{ .@"or", .rm, .r8, .rm8, .none, .none, 1, 0x0a, 0x00, 0x00, 0, .none },
364	.{ .@"or", .rm, .r8, .rm8, .none, .none, 1, 0x0a, 0x00, 0x00, 0, .rex },
365	.{ .@"or", .rm, .r16, .rm16, .none, .none, 1, 0x0b, 0x00, 0x00, 0, .none },
366	.{ .@"or", .rm, .r32, .rm32, .none, .none, 1, 0x0b, 0x00, 0x00, 0, .none },
367	.{ .@"or", .rm, .r64, .rm64, .none, .none, 1, 0x0b, 0x00, 0x00, 0, .long },
368
369	.{ .pop, .o, .r16, .none, .none, .none, 1, 0x58, 0x00, 0x00, 0, .none },
370	.{ .pop, .o, .r64, .none, .none, .none, 1, 0x58, 0x00, 0x00, 0, .none },
371	.{ .pop, .m, .rm16, .none, .none, .none, 1, 0x8f, 0x00, 0x00, 0, .none },
372	.{ .pop, .m, .rm64, .none, .none, .none, 1, 0x8f, 0x00, 0x00, 0, .none },
373
374	.{ .push, .o, .r16, .none, .none, .none, 1, 0x50, 0x00, 0x00, 0, .none },
375	.{ .push, .o, .r64, .none, .none, .none, 1, 0x50, 0x00, 0x00, 0, .none },
376	.{ .push, .m, .rm16, .none, .none, .none, 1, 0xff, 0x00, 0x00, 6, .none },
377	.{ .push, .m, .rm64, .none, .none, .none, 1, 0xff, 0x00, 0x00, 6, .none },
378	.{ .push, .i, .imm8, .none, .none, .none, 1, 0x6a, 0x00, 0x00, 0, .none },
379	.{ .push, .i, .imm16, .none, .none, .none, 1, 0x68, 0x00, 0x00, 0, .none },
380	.{ .push, .i, .imm32, .none, .none, .none, 1, 0x68, 0x00, 0x00, 0, .none },
381
382	.{ .ret, .np, .none, .none, .none, .none, 1, 0xc3, 0x00, 0x00, 0, .none },
383
384	.{ .sal, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 4, .none },
385	.{ .sal, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 4, .rex },
386	.{ .sal, .m1, .rm16, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .none },
387	.{ .sal, .m1, .rm32, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .none },
388	.{ .sal, .m1, .rm64, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .long },
389	.{ .sal, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 4, .none },
390	.{ .sal, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 4, .rex },
391	.{ .sal, .mc, .rm16, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 4, .none },
392	.{ .sal, .mc, .rm32, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 4, .none },
393	.{ .sal, .mc, .rm64, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 4, .long },
394	.{ .sal, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 4, .none },
395	.{ .sal, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 4, .rex },
396	.{ .sal, .mi, .rm16, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 4, .none },
397	.{ .sal, .mi, .rm32, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 4, .none },
398	.{ .sal, .mi, .rm64, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 4, .long },
399
400	.{ .sar, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 7, .none },
401	.{ .sar, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 7, .rex },
402	.{ .sar, .m1, .rm16, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 7, .none },
403	.{ .sar, .m1, .rm32, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 7, .none },
404	.{ .sar, .m1, .rm64, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 7, .long },
405	.{ .sar, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 7, .none },
406	.{ .sar, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 7, .rex },
407	.{ .sar, .mc, .rm16, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 7, .none },
408	.{ .sar, .mc, .rm32, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 7, .none },
409	.{ .sar, .mc, .rm64, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 7, .long },
410	.{ .sar, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 7, .none },
411	.{ .sar, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 7, .rex },
412	.{ .sar, .mi, .rm16, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 7, .none },
413	.{ .sar, .mi, .rm32, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 7, .none },
414	.{ .sar, .mi, .rm64, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 7, .long },
415
416	.{ .sbb, .zi, .al, .imm8, .none, .none, 1, 0x1c, 0x00, 0x00, 0, .none },
417	.{ .sbb, .zi, .ax, .imm16, .none, .none, 1, 0x1d, 0x00, 0x00, 0, .none },
418	.{ .sbb, .zi, .eax, .imm32, .none, .none, 1, 0x1d, 0x00, 0x00, 0, .none },
419	.{ .sbb, .zi, .rax, .imm32s, .none, .none, 1, 0x1d, 0x00, 0x00, 0, .long },
420	.{ .sbb, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 3, .none },
421	.{ .sbb, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 3, .rex },
422	.{ .sbb, .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 3, .none },
423	.{ .sbb, .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 3, .none },
424	.{ .sbb, .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 3, .long },
425	.{ .sbb, .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 3, .none },
426	.{ .sbb, .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 3, .none },
427	.{ .sbb, .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 3, .long },
428	.{ .sbb, .mr, .rm8, .r8, .none, .none, 1, 0x18, 0x00, 0x00, 0, .none },
429	.{ .sbb, .mr, .rm8, .r8, .none, .none, 1, 0x18, 0x00, 0x00, 0, .rex },
430	.{ .sbb, .mr, .rm16, .r16, .none, .none, 1, 0x19, 0x00, 0x00, 0, .none },
431	.{ .sbb, .mr, .rm32, .r32, .none, .none, 1, 0x19, 0x00, 0x00, 0, .none },
432	.{ .sbb, .mr, .rm64, .r64, .none, .none, 1, 0x19, 0x00, 0x00, 0, .long },
433	.{ .sbb, .rm, .r8, .rm8, .none, .none, 1, 0x1a, 0x00, 0x00, 0, .none },
434	.{ .sbb, .rm, .r8, .rm8, .none, .none, 1, 0x1a, 0x00, 0x00, 0, .rex },
435	.{ .sbb, .rm, .r16, .rm16, .none, .none, 1, 0x1b, 0x00, 0x00, 0, .none },
436	.{ .sbb, .rm, .r32, .rm32, .none, .none, 1, 0x1b, 0x00, 0x00, 0, .none },
437	.{ .sbb, .rm, .r64, .rm64, .none, .none, 1, 0x1b, 0x00, 0x00, 0, .long },
438
439	.{ .seta, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x97, 0x00, 0, .none },
440	.{ .seta, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x97, 0x00, 0, .rex },
441	.{ .setae, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .none },
442	.{ .setae, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .rex },
443	.{ .setb, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .none },
444	.{ .setb, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .rex },
445	.{ .setbe, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x96, 0x00, 0, .none },
446	.{ .setbe, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x96, 0x00, 0, .rex },
447	.{ .setc, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .none },
448	.{ .setc, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .rex },
449	.{ .sete, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x94, 0x00, 0, .none },
450	.{ .sete, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x94, 0x00, 0, .rex },
451	.{ .setg, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9f, 0x00, 0, .none },
452	.{ .setg, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9f, 0x00, 0, .rex },
453	.{ .setge, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9d, 0x00, 0, .none },
454	.{ .setge, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9d, 0x00, 0, .rex },
455	.{ .setl, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9c, 0x00, 0, .none },
456	.{ .setl, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9c, 0x00, 0, .rex },
457	.{ .setle, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9e, 0x00, 0, .none },
458	.{ .setle, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9e, 0x00, 0, .rex },
459	.{ .setna, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x96, 0x00, 0, .none },
460	.{ .setna, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x96, 0x00, 0, .rex },
461	.{ .setnae, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .none },
462	.{ .setnae, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .rex },
463	.{ .setnb, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .none },
464	.{ .setnb, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .rex },
465	.{ .setnbe, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x97, 0x00, 0, .none },
466	.{ .setnbe, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x97, 0x00, 0, .rex },
467	.{ .setnc, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .none },
468	.{ .setnc, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .rex },
469	.{ .setne, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x95, 0x00, 0, .none },
470	.{ .setne, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x95, 0x00, 0, .rex },
471	.{ .setng, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9e, 0x00, 0, .none },
472	.{ .setng, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9e, 0x00, 0, .rex },
473	.{ .setnge, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9c, 0x00, 0, .none },
474	.{ .setnge, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9c, 0x00, 0, .rex },
475	.{ .setnl, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9d, 0x00, 0, .none },
476	.{ .setnl, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9d, 0x00, 0, .rex },
477	.{ .setnle, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9f, 0x00, 0, .none },
478	.{ .setnle, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9f, 0x00, 0, .rex },
479	.{ .setno, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x91, 0x00, 0, .none },
480	.{ .setno, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x91, 0x00, 0, .rex },
481	.{ .setnp, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9b, 0x00, 0, .none },
482	.{ .setnp, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9b, 0x00, 0, .rex },
483	.{ .setns, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x99, 0x00, 0, .none },
484	.{ .setns, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x99, 0x00, 0, .rex },
485	.{ .setnz, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x95, 0x00, 0, .none },
486	.{ .setnz, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x95, 0x00, 0, .rex },
487	.{ .seto, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x90, 0x00, 0, .none },
488	.{ .seto, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x90, 0x00, 0, .rex },
489	.{ .setp, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9a, 0x00, 0, .none },
490	.{ .setp, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9a, 0x00, 0, .rex },
491	.{ .setpe, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9a, 0x00, 0, .none },
492	.{ .setpe, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9a, 0x00, 0, .rex },
493	.{ .setpo, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9b, 0x00, 0, .none },
494	.{ .setpo, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9b, 0x00, 0, .rex },
495	.{ .sets, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x98, 0x00, 0, .none },
496	.{ .sets, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x98, 0x00, 0, .rex },
497	.{ .setz, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x94, 0x00, 0, .none },
498	.{ .setz, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x94, 0x00, 0, .rex },
499
500	.{ .shl, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 4, .none },
501	.{ .shl, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 4, .rex },
502	.{ .shl, .m1, .rm16, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .none },
503	.{ .shl, .m1, .rm32, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .none },
504	.{ .shl, .m1, .rm64, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .long },
505	.{ .shl, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 4, .none },
506	.{ .shl, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 4, .rex },
507	.{ .shl, .mc, .rm16, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 4, .none },
508	.{ .shl, .mc, .rm32, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 4, .none },
509	.{ .shl, .mc, .rm64, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 4, .long },
510	.{ .shl, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 4, .none },
511	.{ .shl, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 4, .rex },
512	.{ .shl, .mi, .rm16, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 4, .none },
513	.{ .shl, .mi, .rm32, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 4, .none },
514	.{ .shl, .mi, .rm64, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 4, .long },
515
516	.{ .shr, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 5, .none },
517	.{ .shr, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 5, .rex },
518	.{ .shr, .m1, .rm16, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 5, .none },
519	.{ .shr, .m1, .rm32, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 5, .none },
520	.{ .shr, .m1, .rm64, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 5, .long },
521	.{ .shr, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 5, .none },
522	.{ .shr, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 5, .rex },
523	.{ .shr, .mc, .rm16, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 5, .none },
524	.{ .shr, .mc, .rm32, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 5, .none },
525	.{ .shr, .mc, .rm64, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 5, .long },
526	.{ .shr, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 5, .none },
527	.{ .shr, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 5, .rex },
528	.{ .shr, .mi, .rm16, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 5, .none },
529	.{ .shr, .mi, .rm32, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 5, .none },
530	.{ .shr, .mi, .rm64, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 5, .long },
531
532	.{ .sub, .zi, .al, .imm8, .none, .none, 1, 0x2c, 0x00, 0x00, 0, .none },
533	.{ .sub, .zi, .ax, .imm16, .none, .none, 1, 0x2d, 0x00, 0x00, 0, .none },
534	.{ .sub, .zi, .eax, .imm32, .none, .none, 1, 0x2d, 0x00, 0x00, 0, .none },
535	.{ .sub, .zi, .rax, .imm32s, .none, .none, 1, 0x2d, 0x00, 0x00, 0, .long },
536	.{ .sub, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 5, .none },
537	.{ .sub, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 5, .rex },
538	.{ .sub, .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 5, .none },
539	.{ .sub, .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 5, .none },
540	.{ .sub, .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 5, .long },
541	.{ .sub, .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 5, .none },
542	.{ .sub, .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 5, .none },
543	.{ .sub, .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 5, .long },
544	.{ .sub, .mr, .rm8, .r8, .none, .none, 1, 0x28, 0x00, 0x00, 0, .none },
545	.{ .sub, .mr, .rm8, .r8, .none, .none, 1, 0x28, 0x00, 0x00, 0, .rex },
546	.{ .sub, .mr, .rm16, .r16, .none, .none, 1, 0x29, 0x00, 0x00, 0, .none },
547	.{ .sub, .mr, .rm32, .r32, .none, .none, 1, 0x29, 0x00, 0x00, 0, .none },
548	.{ .sub, .mr, .rm64, .r64, .none, .none, 1, 0x29, 0x00, 0x00, 0, .long },
549	.{ .sub, .rm, .r8, .rm8, .none, .none, 1, 0x2a, 0x00, 0x00, 0, .none },
550	.{ .sub, .rm, .r8, .rm8, .none, .none, 1, 0x2a, 0x00, 0x00, 0, .rex },
551	.{ .sub, .rm, .r16, .rm16, .none, .none, 1, 0x2b, 0x00, 0x00, 0, .none },
552	.{ .sub, .rm, .r32, .rm32, .none, .none, 1, 0x2b, 0x00, 0x00, 0, .none },
553	.{ .sub, .rm, .r64, .rm64, .none, .none, 1, 0x2b, 0x00, 0x00, 0, .long },
554
555	.{ .syscall, .np, .none, .none, .none, .none, 2, 0x0f, 0x05, 0x00, 0, .none },
556
557	.{ .@"test", .zi, .al, .imm8, .none, .none, 1, 0xa8, 0x00, 0x00, 0, .none },
558	.{ .@"test", .zi, .ax, .imm16, .none, .none, 1, 0xa9, 0x00, 0x00, 0, .none },
559	.{ .@"test", .zi, .eax, .imm32, .none, .none, 1, 0xa9, 0x00, 0x00, 0, .none },
560	.{ .@"test", .zi, .rax, .imm32s, .none, .none, 1, 0xa9, 0x00, 0x00, 0, .long },
561	.{ .@"test", .mi, .rm8, .imm8, .none, .none, 1, 0xf6, 0x00, 0x00, 0, .none },
562	.{ .@"test", .mi, .rm8, .imm8, .none, .none, 1, 0xf6, 0x00, 0x00, 0, .rex },
563	.{ .@"test", .mi, .rm16, .imm16, .none, .none, 1, 0xf7, 0x00, 0x00, 0, .none },
564	.{ .@"test", .mi, .rm32, .imm32, .none, .none, 1, 0xf7, 0x00, 0x00, 0, .none },
565	.{ .@"test", .mi, .rm64, .imm32s, .none, .none, 1, 0xf7, 0x00, 0x00, 0, .long },
566	.{ .@"test", .mr, .rm8, .r8, .none, .none, 1, 0x84, 0x00, 0x00, 0, .none },
567	.{ .@"test", .mr, .rm8, .r8, .none, .none, 1, 0x84, 0x00, 0x00, 0, .rex },
568	.{ .@"test", .mr, .rm16, .r16, .none, .none, 1, 0x85, 0x00, 0x00, 0, .none },
569	.{ .@"test", .mr, .rm32, .r32, .none, .none, 1, 0x85, 0x00, 0x00, 0, .none },
570	.{ .@"test", .mr, .rm64, .r64, .none, .none, 1, 0x85, 0x00, 0x00, 0, .long },
571
572	.{ .ud2, .np, .none, .none, .none, .none, 2, 0x0f, 0x0b, 0x00, 0, .none },
573
574	.{ .xor, .zi, .al, .imm8, .none, .none, 1, 0x34, 0x00, 0x00, 0, .none },
575	.{ .xor, .zi, .ax, .imm16, .none, .none, 1, 0x35, 0x00, 0x00, 0, .none },
576	.{ .xor, .zi, .eax, .imm32, .none, .none, 1, 0x35, 0x00, 0x00, 0, .none },
577	.{ .xor, .zi, .rax, .imm32s, .none, .none, 1, 0x35, 0x00, 0x00, 0, .long },
578	.{ .xor, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 6, .none },
579	.{ .xor, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 6, .rex },
580	.{ .xor, .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 6, .none },
581	.{ .xor, .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 6, .none },
582	.{ .xor, .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 6, .long },
583	.{ .xor, .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 6, .none },
584	.{ .xor, .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 6, .none },
585	.{ .xor, .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 6, .long },
586	.{ .xor, .mr, .rm8, .r8, .none, .none, 1, 0x30, 0x00, 0x00, 0, .none },
587	.{ .xor, .mr, .rm8, .r8, .none, .none, 1, 0x30, 0x00, 0x00, 0, .rex },
588	.{ .xor, .mr, .rm16, .r16, .none, .none, 1, 0x31, 0x00, 0x00, 0, .none },
589	.{ .xor, .mr, .rm32, .r32, .none, .none, 1, 0x31, 0x00, 0x00, 0, .none },
590	.{ .xor, .mr, .rm64, .r64, .none, .none, 1, 0x31, 0x00, 0x00, 0, .long },
591	.{ .xor, .rm, .r8, .rm8, .none, .none, 1, 0x32, 0x00, 0x00, 0, .none },
592	.{ .xor, .rm, .r8, .rm8, .none, .none, 1, 0x32, 0x00, 0x00, 0, .rex },
593	.{ .xor, .rm, .r16, .rm16, .none, .none, 1, 0x33, 0x00, 0x00, 0, .none },
594	.{ .xor, .rm, .r32, .rm32, .none, .none, 1, 0x33, 0x00, 0x00, 0, .none },
595	.{ .xor, .rm, .r64, .rm64, .none, .none, 1, 0x33, 0x00, 0x00, 0, .long },
596
597	// SSE
598	.{ .addss, .rm, .xmm, .xmm_m32, .none, .none, 3, 0xf3, 0x0f, 0x58, 0, .sse },
599
600	.{ .cmpss, .rmi, .xmm, .xmm_m32, .imm8, .none, 3, 0xf3, 0x0f, 0xc2, 0, .sse },
601
602	.{ .movss, .rm, .xmm, .xmm_m32, .none, .none, 3, 0xf3, 0x0f, 0x10, 0, .sse },
603	.{ .movss, .mr, .xmm_m32, .xmm, .none, .none, 3, 0xf3, 0x0f, 0x11, 0, .sse },
604
605	.{ .ucomiss, .rm, .xmm, .xmm_m32, .none, .none, 2, 0x0f, 0x2e, 0x00, 0, .sse },
606
607	// SSE2
608	.{ .addsd, .rm, .xmm, .xmm_m64, .none, .none, 3, 0xf2, 0x0f, 0x58, 0, .sse2 },
609
610	.{ .cmpsd, .rmi, .xmm, .xmm_m64, .imm8, .none, 3, 0xf2, 0x0f, 0xc2, 0, .sse2 },
611
612	.{ .movq, .rm, .xmm, .xmm_m64, .none, .none, 3, 0xf3, 0x0f, 0x7e, 0, .sse2 },
613	.{ .movq, .mr, .xmm_m64, .xmm, .none, .none, 3, 0x66, 0x0f, 0xd6, 0, .sse2 },
614
615	.{ .movsd, .rm, .xmm, .xmm_m64, .none, .none, 3, 0xf2, 0x0f, 0x10, 0, .sse2 },
616	.{ .movsd, .mr, .xmm_m64, .xmm, .none, .none, 3, 0xf2, 0x0f, 0x11, 0, .sse2 },
617
618	.{ .ucomisd, .rm, .xmm, .xmm_m64, .none, .none, 3, 0x66, 0x0f, 0x2e, 0, .sse2 },
619	};
620	// zig fmt: on
621