Commit 515e1c93e18d81435410f2cb45f3788c6be13fbf | zig

author	Jakub Konka <kubkon@jakubkonka.com> 2023-03-21 11:16:29+01:00
committer	GitHub <noreply@github.com> 2023-03-21 11:16:29+01:00
log	515e1c93e18d81435410f2cb45f3788c6be13fbf
tree	10dbb755d5323d9ca4fd3ddfb88c78a8bd980ce7
parent	dff4bbfd2426ce4943972782d2bcffc89b3fd26d
parent	c58b5732f381fe4f145537501e29347be01e2a44
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #14993 from jacobly0/x86_64

x86_64: implement more operations

30 files changed, 2921 insertions(+), 1576 deletions(-)

src/arch/aarch64/CodeGen.zig+46-36

...	...	@@ -3011,41 +3011,16 @@ fn optionalPayload(self: *Self, inst: Air.Inst.Index, mcv: MCValue, optional_ty:
3011	3011	return MCValue{ .register = reg };
3012	3012	}
3013	3013
3014		const offset = @intCast(u32, optional_ty.abiSize(self.target.) - payload_ty.abiSize(self.target.));
3015	3014	switch (mcv) {
3016		.register => \|source_reg\| {
	3015	.register => {
3017	3016	// TODO should we reuse the operand here?
3018	3017	const raw_reg = try self.register_manager.allocReg(inst, gp);
3019	3018	const dest_reg = raw_reg.toX();
3020	3019
3021		const shift = @intCast(u6, offset * 8);
3022		if (shift == 0) {
3023		try self.genSetReg(payload_ty, dest_reg, mcv);
3024		} else {
3025		_ = try self.addInst(.{
3026		.tag = if (payload_ty.isSignedInt())
3027		Mir.Inst.Tag.asr_immediate
3028		else
3029		Mir.Inst.Tag.lsr_immediate,
3030		.data = .{ .rr_shift = .{
3031		.rd = dest_reg,
3032		.rn = source_reg.toX(),
3033		.shift = shift,
3034		} },
3035		});
3036		}
3037
	3020	try self.genSetReg(payload_ty, dest_reg, mcv);
3038	3021	return MCValue{ .register = self.registerAlias(dest_reg, payload_ty) };
3039	3022	},
3040		.stack_argument_offset => \|off\| {
3041		return MCValue{ .stack_argument_offset = off + offset };
3042		},
3043		.stack_offset => \|off\| {
3044		return MCValue{ .stack_offset = off - offset };
3045		},
3046		.memory => \|addr\| {
3047		return MCValue{ .memory = addr + offset };
3048		},
	3023	.stack_argument_offset, .stack_offset, .memory => return mcv,
3049	3024	else => unreachable, // invalid MCValue for an error union
3050	3025	}
3051	3026	}
...	...	@@ -3289,12 +3264,11 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
3289	3264
3290	3265	const optional_abi_size = @intCast(u32, optional_ty.abiSize(self.target.*));
3291	3266	const optional_abi_align = optional_ty.abiAlignment(self.target.*);
3292		const payload_abi_size = @intCast(u32, payload_ty.abiSize(self.target.*));
3293		const offset = optional_abi_size - payload_abi_size;
	3267	const offset = @intCast(u32, payload_ty.abiSize(self.target.*));
3294	3268
3295	3269	const stack_offset = try self.allocMem(optional_abi_size, optional_abi_align, inst);
3296		try self.genSetStack(Type.bool, stack_offset, .{ .immediate = 1 });
3297		try self.genSetStack(payload_ty, stack_offset - @intCast(u32, offset), operand);
	3270	try self.genSetStack(payload_ty, stack_offset, operand);
	3271	try self.genSetStack(Type.bool, stack_offset - offset, .{ .immediate = 1 });
3298	3272
3299	3273	break :result MCValue{ .stack_offset = stack_offset };
3300	3274	};
...	...	@@ -4834,13 +4808,49 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
4834	4808	}
4835	4809
4836	4810	fn isNull(self: *Self, operand_bind: ReadArg.Bind, operand_ty: Type) !MCValue {
4837		const sentinel_ty: Type = if (!operand_ty.isPtrLikeOptional()) blk: {
	4811	const sentinel: struct { ty: Type, bind: ReadArg.Bind } = if (!operand_ty.isPtrLikeOptional()) blk: {
4838	4812	var buf: Type.Payload.ElemType = undefined;
4839	4813	const payload_ty = operand_ty.optionalChild(&buf);
4840		break :blk if (payload_ty.hasRuntimeBitsIgnoreComptime()) Type.bool else operand_ty;
4841		} else operand_ty;
	4814	if (!payload_ty.hasRuntimeBitsIgnoreComptime())
	4815	break :blk .{ .ty = operand_ty, .bind = operand_bind };
	4816
	4817	const offset = @intCast(u32, payload_ty.abiSize(self.target.*));
	4818	const operand_mcv = try operand_bind.resolveToMcv(self);
	4819	const new_mcv: MCValue = switch (operand_mcv) {
	4820	.register => \|source_reg\| new: {
	4821	// TODO should we reuse the operand here?
	4822	const raw_reg = try self.register_manager.allocReg(null, gp);
	4823	const dest_reg = raw_reg.toX();
	4824
	4825	const shift = @intCast(u6, offset * 8);
	4826	if (shift == 0) {
	4827	try self.genSetReg(payload_ty, dest_reg, operand_mcv);
	4828	} else {
	4829	_ = try self.addInst(.{
	4830	.tag = if (payload_ty.isSignedInt())
	4831	Mir.Inst.Tag.asr_immediate
	4832	else
	4833	Mir.Inst.Tag.lsr_immediate,
	4834	.data = .{ .rr_shift = .{
	4835	.rd = dest_reg,
	4836	.rn = source_reg.toX(),
	4837	.shift = shift,
	4838	} },
	4839	});
	4840	}
	4841
	4842	break :new .{ .register = self.registerAlias(dest_reg, payload_ty) };
	4843	},
	4844	.stack_argument_offset => \|off\| .{ .stack_argument_offset = off + offset },
	4845	.stack_offset => \|off\| .{ .stack_offset = off - offset },
	4846	.memory => \|addr\| .{ .memory = addr + offset },
	4847	else => unreachable, // invalid MCValue for an optional
	4848	};
	4849
	4850	break :blk .{ .ty = Type.bool, .bind = .{ .mcv = new_mcv } };
	4851	} else .{ .ty = operand_ty, .bind = operand_bind };
4842	4852	const imm_bind: ReadArg.Bind = .{ .mcv = .{ .immediate = 0 } };
4843		return self.cmp(operand_bind, imm_bind, sentinel_ty, .eq);
	4853	return self.cmp(sentinel.bind, imm_bind, sentinel.ty, .eq);
4844	4854	}
4845	4855
4846	4856	fn isNonNull(self: *Self, operand_bind: ReadArg.Bind, operand_ty: Type) !MCValue {

src/arch/wasm/CodeGen.zig+18-31

...	...	@@ -2715,20 +2715,7 @@ fn lowerParentPtr(func: *CodeGen, ptr_val: Value, ptr_child_ty: Type) InnerError
2715	2715	},
2716	2716	.opt_payload_ptr => {
2717	2717	const payload_ptr = ptr_val.castTag(.opt_payload_ptr).?.data;
2718		const parent_ptr = try func.lowerParentPtr(payload_ptr.container_ptr, payload_ptr.container_ty);
2719		var buf: Type.Payload.ElemType = undefined;
2720		const payload_ty = payload_ptr.container_ty.optionalChild(&buf);
2721		if (!payload_ty.hasRuntimeBitsIgnoreComptime() or payload_ty.optionalReprIsPayload()) {
2722		return parent_ptr;
2723		}
2724
2725		const abi_size = payload_ptr.container_ty.abiSize(func.target);
2726		const offset = abi_size - payload_ty.abiSize(func.target);
2727
2728		return WValue{ .memory_offset = .{
2729		.pointer = parent_ptr.memory,
2730		.offset = @intCast(u32, offset),
2731		} };
	2718	return func.lowerParentPtr(payload_ptr.container_ptr, payload_ptr.container_ty);
2732	2719	},
2733	2720	else => \|tag\| return func.fail("TODO: Implement lowerParentPtr for tag: {}", .{tag}),
2734	2721	}
...	...	@@ -2889,7 +2876,7 @@ fn lowerConstant(func: *CodeGen, arg_val: Value, ty: Type) InnerError!WValue {
2889	2876	}
2890	2877	} else {
2891	2878	const is_pl = val.tag() == .opt_payload;
2892		return WValue{ .imm32 = if (is_pl) @as(u32, 1) else 0 };
	2879	return WValue{ .imm32 = @boolToInt(is_pl) };
2893	2880	},
2894	2881	.Struct => {
2895	2882	const struct_obj = ty.castTag(.@"struct").?.data;
...	...	@@ -3882,7 +3869,11 @@ fn isNull(func: *CodeGen, operand: WValue, optional_ty: Type, opcode: wasm.Opcod
3882	3869	// When payload is zero-bits, we can treat operand as a value, rather than
3883	3870	// a pointer to the stack value
3884	3871	if (payload_ty.hasRuntimeBitsIgnoreComptime()) {
3885		try func.addMemArg(.i32_load8_u, .{ .offset = operand.offset(), .alignment = 1 });
	3872	const offset = std.math.cast(u32, payload_ty.abiSize(func.target)) orelse {
	3873	const module = func.bin_file.base.options.module.?;
	3874	return func.fail("Optional type {} too big to fit into stack frame", .{optional_ty.fmt(module)});
	3875	};
	3876	try func.addMemArg(.i32_load8_u, .{ .offset = operand.offset() + offset, .alignment = 1 });
3886	3877	}
3887	3878	} else if (payload_ty.isSlice()) {
3888	3879	switch (func.arch()) {
...	...	@@ -3911,13 +3902,11 @@ fn airOptionalPayload(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
3911	3902	const operand = try func.resolveInst(ty_op.operand);
3912	3903	if (opt_ty.optionalReprIsPayload()) break :result func.reuseOperand(ty_op.operand, operand);
3913	3904
3914		const offset = opt_ty.abiSize(func.target) - payload_ty.abiSize(func.target);
3915
3916	3905	if (isByRef(payload_ty, func.target)) {
3917		break :result try func.buildPointerOffset(operand, offset, .new);
	3906	break :result try func.buildPointerOffset(operand, 0, .new);
3918	3907	}
3919	3908
3920		const payload = try func.load(operand, payload_ty, @intCast(u32, offset));
	3909	const payload = try func.load(operand, payload_ty, 0);
3921	3910	break :result try payload.toLocal(func, payload_ty);
3922	3911	};
3923	3912	func.finishAir(inst, result, &.{ty_op.operand});
...	...	@@ -3936,8 +3925,7 @@ fn airOptionalPayloadPtr(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
3936	3925	break :result func.reuseOperand(ty_op.operand, operand);
3937	3926	}
3938	3927
3939		const offset = opt_ty.abiSize(func.target) - payload_ty.abiSize(func.target);
3940		break :result try func.buildPointerOffset(operand, offset, .new);
	3928	break :result try func.buildPointerOffset(operand, 0, .new);
3941	3929	};
3942	3930	func.finishAir(inst, result, &.{ty_op.operand});
3943	3931	}
...	...	@@ -3956,16 +3944,16 @@ fn airOptionalPayloadPtrSet(func: *CodeGen, inst: Air.Inst.Index) InnerError!voi
3956	3944	return func.finishAir(inst, operand, &.{ty_op.operand});
3957	3945	}
3958	3946
3959		const offset = std.math.cast(u32, opt_ty.abiSize(func.target) - payload_ty.abiSize(func.target)) orelse {
	3947	const offset = std.math.cast(u32, payload_ty.abiSize(func.target)) orelse {
3960	3948	const module = func.bin_file.base.options.module.?;
3961	3949	return func.fail("Optional type {} too big to fit into stack frame", .{opt_ty.fmt(module)});
3962	3950	};
3963	3951
3964	3952	try func.emitWValue(operand);
3965	3953	try func.addImm32(1);
3966		try func.addMemArg(.i32_store8, .{ .offset = operand.offset(), .alignment = 1 });
	3954	try func.addMemArg(.i32_store8, .{ .offset = operand.offset() + offset, .alignment = 1 });
3967	3955
3968		const result = try func.buildPointerOffset(operand, offset, .new);
	3956	const result = try func.buildPointerOffset(operand, 0, .new);
3969	3957	return func.finishAir(inst, result, &.{ty_op.operand});
3970	3958	}
3971	3959
...	...	@@ -3988,7 +3976,7 @@ fn airWrapOptional(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
3988	3976	if (op_ty.optionalReprIsPayload()) {
3989	3977	break :result func.reuseOperand(ty_op.operand, operand);
3990	3978	}
3991		const offset = std.math.cast(u32, op_ty.abiSize(func.target) - payload_ty.abiSize(func.target)) orelse {
	3979	const offset = std.math.cast(u32, payload_ty.abiSize(func.target)) orelse {
3992	3980	const module = func.bin_file.base.options.module.?;
3993	3981	return func.fail("Optional type {} too big to fit into stack frame", .{op_ty.fmt(module)});
3994	3982	};
...	...	@@ -3997,9 +3985,9 @@ fn airWrapOptional(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
3997	3985	const result_ptr = try func.allocStack(op_ty);
3998	3986	try func.emitWValue(result_ptr);
3999	3987	try func.addImm32(1);
4000		try func.addMemArg(.i32_store8, .{ .offset = result_ptr.offset(), .alignment = 1 });
	3988	try func.addMemArg(.i32_store8, .{ .offset = result_ptr.offset() + offset, .alignment = 1 });
4001	3989
4002		const payload_ptr = try func.buildPointerOffset(result_ptr, offset, .new);
	3990	const payload_ptr = try func.buildPointerOffset(result_ptr, 0, .new);
4003	3991	try func.store(payload_ptr, operand, payload_ty, 0);
4004	3992	break :result result_ptr;
4005	3993	};
...	...	@@ -4719,7 +4707,6 @@ fn cmpOptionals(func: *CodeGen, lhs: WValue, rhs: WValue, operand_ty: Type, op:
4719	4707	assert(op == .eq or op == .neq);
4720	4708	var buf: Type.Payload.ElemType = undefined;
4721	4709	const payload_ty = operand_ty.optionalChild(&buf);
4722		const offset = @intCast(u32, operand_ty.abiSize(func.target) - payload_ty.abiSize(func.target));
4723	4710
4724	4711	// We store the final result in here that will be validated
4725	4712	// if the optional is truly equal.
...	...	@@ -4732,8 +4719,8 @@ fn cmpOptionals(func: *CodeGen, lhs: WValue, rhs: WValue, operand_ty: Type, op:
4732	4719	try func.addTag(.i32_ne); // inverse so we can exit early
4733	4720	try func.addLabel(.br_if, 0);
4734	4721
4735		_ = try func.load(lhs, payload_ty, offset);
4736		_ = try func.load(rhs, payload_ty, offset);
	4722	_ = try func.load(lhs, payload_ty, 0);
	4723	_ = try func.load(rhs, payload_ty, 0);
4737	4724	const opcode = buildOpcode(.{ .op = .ne, .valtype1 = typeToValtype(payload_ty, func.target) });
4738	4725	try func.addTag(Mir.Inst.Tag.fromOpcode(opcode));
4739	4726	try func.addLabel(.br_if, 0);

src/arch/x86_64/CodeGen.zig+1529-647

...	...	@@ -402,19 +402,38 @@ fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {
402	402	fn asmSetccRegister(self: *Self, reg: Register, cc: bits.Condition) !void {
403	403	_ = try self.addInst(.{
404	404	.tag = .setcc,
405		.ops = .r_c,
406		.data = .{ .r_c = .{
	405	.ops = .r_cc,
	406	.data = .{ .r_cc = .{
407	407	.r1 = reg,
408	408	.cc = cc,
409	409	} },
410	410	});
411	411	}
412	412
	413	fn asmSetccMemory(self: *Self, m: Memory, cc: bits.Condition) !void {
	414	_ = try self.addInst(.{
	415	.tag = .setcc,
	416	.ops = switch (m) {
	417	.sib => .m_sib_cc,
	418	.rip => .m_rip_cc,
	419	else => unreachable,
	420	},
	421	.data = .{ .x_cc = .{
	422	.payload = switch (m) {
	423	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	424	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	425	else => unreachable,
	426	},
	427	.cc = cc,
	428	} },
	429	});
	430	}
	431
413	432	fn asmCmovccRegisterRegister(self: *Self, reg1: Register, reg2: Register, cc: bits.Condition) !void {
414	433	_ = try self.addInst(.{
415	434	.tag = .cmovcc,
416		.ops = .rr_c,
417		.data = .{ .rr_c = .{
	435	.ops = .rr_cc,
	436	.data = .{ .rr_cc = .{
418	437	.r1 = reg1,
419	438	.r2 = reg2,
420	439	.cc = cc,
...	...	@@ -422,6 +441,26 @@ fn asmCmovccRegisterRegister(self: *Self, reg1: Register, reg2: Register, cc: bi
422	441	});
423	442	}
424	443
	444	fn asmCmovccRegisterMemory(self: *Self, reg: Register, m: Memory, cc: bits.Condition) !void {
	445	_ = try self.addInst(.{
	446	.tag = .cmovcc,
	447	.ops = switch (m) {
	448	.sib => .rm_sib_cc,
	449	.rip => .rm_rip_cc,
	450	else => unreachable,
	451	},
	452	.data = .{ .rx_cc = .{
	453	.r1 = reg,
	454	.cc = cc,
	455	.payload = switch (m) {
	456	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	457	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	458	else => unreachable,
	459	},
	460	} },
	461	});
	462	}
	463
425	464	fn asmJmpReloc(self: *Self, target: Mir.Inst.Index) !Mir.Inst.Index {
426	465	return self.addInst(.{
427	466	.tag = .jmp_reloc,
...	...	@@ -793,18 +832,23 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
793	832
794	833	switch (air_tags[inst]) {
795	834	// zig fmt: off
796		.add => try self.airBinOp(inst, .add),
797		.addwrap => try self.airBinOp(inst, .addwrap),
798		.sub => try self.airBinOp(inst, .sub),
799		.subwrap => try self.airBinOp(inst, .subwrap),
800		.bool_and => try self.airBinOp(inst, .bool_and),
801		.bool_or => try self.airBinOp(inst, .bool_or),
802		.bit_and => try self.airBinOp(inst, .bit_and),
803		.bit_or => try self.airBinOp(inst, .bit_or),
804		.xor => try self.airBinOp(inst, .xor),
805
806		.ptr_add => try self.airPtrArithmetic(inst, .ptr_add),
807		.ptr_sub => try self.airPtrArithmetic(inst, .ptr_sub),
	835	.not,
	836	=> \|tag\| try self.airUnOp(inst, tag),
	837
	838	.add,
	839	.addwrap,
	840	.sub,
	841	.subwrap,
	842	.bool_and,
	843	.bool_or,
	844	.bit_and,
	845	.bit_or,
	846	.xor,
	847	.min,
	848	.max,
	849	=> \|tag\| try self.airBinOp(inst, tag),
	850
	851	.ptr_add, .ptr_sub => \|tag\| try self.airPtrArithmetic(inst, tag),
808	852
809	853	.shr, .shr_exact => try self.airShlShrBinOp(inst),
810	854	.shl, .shl_exact => try self.airShlShrBinOp(inst),
...	...	@@ -818,8 +862,6 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
818	862	.sub_sat => try self.airSubSat(inst),
819	863	.mul_sat => try self.airMulSat(inst),
820	864	.shl_sat => try self.airShlSat(inst),
821		.min => try self.airMin(inst),
822		.max => try self.airMax(inst),
823	865	.slice => try self.airSlice(inst),
824	866
825	867	.sqrt,
...	...	@@ -867,7 +909,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
867	909	.breakpoint => try self.airBreakpoint(),
868	910	.ret_addr => try self.airRetAddr(inst),
869	911	.frame_addr => try self.airFrameAddress(inst),
870		.fence => try self.airFence(),
	912	.fence => try self.airFence(inst),
871	913	.cond_br => try self.airCondBr(inst),
872	914	.dbg_stmt => try self.airDbgStmt(inst),
873	915	.fptrunc => try self.airFptrunc(inst),
...	...	@@ -885,7 +927,6 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
885	927	.is_err_ptr => try self.airIsErrPtr(inst),
886	928	.load => try self.airLoad(inst),
887	929	.loop => try self.airLoop(inst),
888		.not => try self.airNot(inst),
889	930	.ptrtoint => try self.airPtrToInt(inst),
890	931	.ret => try self.airRet(inst),
891	932	.ret_load => try self.airRetLoad(inst),
...	...	@@ -972,10 +1013,10 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
972	1013	.optional_payload => try self.airOptionalPayload(inst),
973	1014	.optional_payload_ptr => try self.airOptionalPayloadPtr(inst),
974	1015	.optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst),
975		.unwrap_errunion_err => try self.airUnwrapErrErr(inst),
976		.unwrap_errunion_payload => try self.airUnwrapErrPayload(inst),
977		.unwrap_errunion_err_ptr => try self.airUnwrapErrErrPtr(inst),
978		.unwrap_errunion_payload_ptr=> try self.airUnwrapErrPayloadPtr(inst),
	1016	.unwrap_errunion_err => try self.airUnwrapErrUnionErr(inst),
	1017	.unwrap_errunion_payload => try self.airUnwrapErrUnionPayload(inst),
	1018	.unwrap_errunion_err_ptr => try self.airUnwrapErrUnionErrPtr(inst),
	1019	.unwrap_errunion_payload_ptr=> try self.airUnwrapErrUnionPayloadPtr(inst),
979	1020	.errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst),
980	1021	.err_return_trace => try self.airErrReturnTrace(inst),
981	1022	.set_err_return_trace => try self.airSetErrReturnTrace(inst),
...	...	@@ -1266,7 +1307,7 @@ pub fn spillEflagsIfOccupied(self: *Self) !void {
1266	1307	}
1267	1308	}
1268	1309
1269		pub fn spillRegisters(self: *Self, comptime count: comptime_int, registers: [count]Register) !void {
	1310	pub fn spillRegisters(self: *Self, registers: []const Register) !void {
1270	1311	for (registers) \|reg\| {
1271	1312	try self.register_manager.getReg(reg, null);
1272	1313	}
...	...	@@ -1399,8 +1440,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
1399	1440
1400	1441	// when truncating a `u16` to `u5`, for example, those top 3 bits in the result
1401	1442	// have to be removed. this only happens if the dst if not a power-of-two size.
1402		const dst_bit_size = dst_ty.bitSize(self.target.*);
1403		if (!math.isPowerOfTwo(dst_bit_size) or dst_bit_size < 8) {
	1443	if (self.regExtraBits(dst_ty) > 0) {
1404	1444	try self.truncateRegister(dst_ty, reg);
1405	1445	}
1406	1446
...	...	@@ -1414,113 +1454,6 @@ fn airBoolToInt(self: *Self, inst: Air.Inst.Index) !void {
1414	1454	return self.finishAir(inst, result, .{ un_op, .none, .none });
1415	1455	}
1416	1456
1417		fn airNot(self: *Self, inst: Air.Inst.Index) !void {
1418		const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1419
1420		if (self.liveness.isUnused(inst)) {
1421		return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
1422		}
1423
1424		const operand_ty = self.air.typeOf(ty_op.operand);
1425		const operand = try self.resolveInst(ty_op.operand);
1426
1427		const result: MCValue = result: {
1428		switch (operand) {
1429		.dead => unreachable,
1430		.unreach => unreachable,
1431		.eflags => \|cc\| {
1432		break :result MCValue{ .eflags = cc.negate() };
1433		},
1434		else => {},
1435		}
1436
1437		const operand_lock: ?RegisterLock = switch (operand) {
1438		.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
1439		else => null,
1440		};
1441		defer if (operand_lock) \|lock\| self.register_manager.unlockReg(lock);
1442
1443		const dst_mcv: MCValue = blk: {
1444		if (self.reuseOperand(inst, ty_op.operand, 0, operand) and operand.isRegister()) {
1445		break :blk operand;
1446		}
1447		break :blk try self.copyToRegisterWithInstTracking(inst, operand_ty, operand);
1448		};
1449		const dst_mcv_lock: ?RegisterLock = switch (dst_mcv) {
1450		.register => \|reg\| self.register_manager.lockReg(reg),
1451		else => null,
1452		};
1453		defer if (dst_mcv_lock) \|lock\| self.register_manager.unlockReg(lock);
1454
1455		const mask = switch (operand_ty.abiSize(self.target.*)) {
1456		1 => ~@as(u8, 0),
1457		2 => ~@as(u16, 0),
1458		4 => ~@as(u32, 0),
1459		8 => ~@as(u64, 0),
1460		else => unreachable,
1461		};
1462		try self.genBinOpMir(.xor, operand_ty, dst_mcv, .{ .immediate = mask });
1463
1464		break :result dst_mcv;
1465		};
1466		return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1467		}
1468
1469		fn airMin(self: *Self, inst: Air.Inst.Index) !void {
1470		const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1471		if (self.liveness.isUnused(inst)) {
1472		return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
1473		}
1474
1475		const ty = self.air.typeOfIndex(inst);
1476		if (ty.zigTypeTag() != .Int) {
1477		return self.fail("TODO implement min for type {}", .{ty.fmtDebug()});
1478		}
1479		const signedness = ty.intInfo(self.target.*).signedness;
1480		const result: MCValue = result: {
1481		// TODO improve by checking if any operand can be reused.
1482		// TODO audit register allocation
1483		const lhs = try self.resolveInst(bin_op.lhs);
1484		const lhs_lock: ?RegisterLock = switch (lhs) {
1485		.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
1486		else => null,
1487		};
1488		defer if (lhs_lock) \|lock\| self.register_manager.unlockReg(lock);
1489
1490		const lhs_reg = try self.copyToTmpRegister(ty, lhs);
1491		const lhs_reg_lock = self.register_manager.lockRegAssumeUnused(lhs_reg);
1492		defer self.register_manager.unlockReg(lhs_reg_lock);
1493
1494		const rhs_mcv = try self.limitImmediateType(bin_op.rhs, i32);
1495		const rhs_lock: ?RegisterLock = switch (rhs_mcv) {
1496		.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
1497		else => null,
1498		};
1499		defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
1500
1501		try self.genBinOpMir(.cmp, ty, .{ .register = lhs_reg }, rhs_mcv);
1502
1503		const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ty, rhs_mcv);
1504		const cc: Condition = switch (signedness) {
1505		.unsigned => .b,
1506		.signed => .l,
1507		};
1508		try self.asmCmovccRegisterRegister(dst_mcv.register, lhs_reg, cc);
1509
1510		break :result dst_mcv;
1511		};
1512		return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1513		}
1514
1515		fn airMax(self: *Self, inst: Air.Inst.Index) !void {
1516		const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1517		const result: MCValue = if (self.liveness.isUnused(inst))
1518		.dead
1519		else
1520		return self.fail("TODO implement max for {}", .{self.target.cpu.arch});
1521		return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1522		}
1523
1524	1457	fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
1525	1458	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1526	1459	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
...	...	@@ -1542,14 +1475,23 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
1542	1475	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1543	1476	}
1544	1477
	1478	fn airUnOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
	1479	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	1480
	1481	const result = if (self.liveness.isUnused(inst))
	1482	.dead
	1483	else
	1484	try self.genUnOp(inst, tag, ty_op.operand);
	1485	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	1486	}
	1487
1545	1488	fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
1546	1489	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1547	1490
1548		if (self.liveness.isUnused(inst)) {
1549		return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
1550		}
1551
1552		const result = try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs);
	1491	const result = if (self.liveness.isUnused(inst))
	1492	.dead
	1493	else
	1494	try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs);
1553	1495	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1554	1496	}
1555	1497
...	...	@@ -1557,31 +1499,32 @@ fn airPtrArithmetic(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void
1557	1499	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1558	1500	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
1559	1501
1560		if (self.liveness.isUnused(inst)) {
1561		return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
1562		}
1563
1564		const result = try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs);
	1502	const result = if (self.liveness.isUnused(inst))
	1503	.dead
	1504	else
	1505	try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs);
1565	1506	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1566	1507	}
1567	1508
1568	1509	fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
1569	1510	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	1511	const result = result: {
	1512	if (self.liveness.isUnused(inst)) break :result .dead;
1570	1513
1571		if (self.liveness.isUnused(inst)) {
1572		return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
1573		}
1574
1575		const tag = self.air.instructions.items(.tag)[inst];
1576		const ty = self.air.typeOfIndex(inst);
	1514	const tag = self.air.instructions.items(.tag)[inst];
	1515	const ty = self.air.typeOfIndex(inst);
1577	1516
1578		try self.spillRegisters(2, .{ .rax, .rdx });
	1517	if (ty.zigTypeTag() == .Float) {
	1518	break :result try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs);
	1519	}
1579	1520
1580		const lhs = try self.resolveInst(bin_op.lhs);
1581		const rhs = try self.resolveInst(bin_op.rhs);
	1521	try self.spillRegisters(&.{ .rax, .rdx });
1582	1522
1583		const result = try self.genMulDivBinOp(tag, inst, ty, lhs, rhs);
	1523	const lhs = try self.resolveInst(bin_op.lhs);
	1524	const rhs = try self.resolveInst(bin_op.rhs);
1584	1525
	1526	break :result try self.genMulDivBinOp(tag, inst, ty, lhs, rhs);
	1527	};
1585	1528	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1586	1529	}
1587	1530
...	...	@@ -1629,7 +1572,7 @@ fn airAddSubShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1629	1572	try self.spillEflagsIfOccupied();
1630	1573
1631	1574	if (tag == .shl_with_overflow) {
1632		try self.spillRegisters(1, .{.rcx});
	1575	try self.spillRegisters(&.{.rcx});
1633	1576	}
1634	1577
1635	1578	const partial: MCValue = switch (tag) {
...	...	@@ -1756,7 +1699,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1756	1699	try self.spillEflagsIfOccupied();
1757	1700	self.eflags_inst = inst;
1758	1701
1759		try self.spillRegisters(2, .{ .rax, .rdx });
	1702	try self.spillRegisters(&.{ .rax, .rdx });
1760	1703
1761	1704	const lhs = try self.resolveInst(bin_op.lhs);
1762	1705	const rhs = try self.resolveInst(bin_op.rhs);
...	...	@@ -1809,7 +1752,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1809	1752	break :dst_reg dst_reg;
1810	1753	},
1811	1754	.unsigned => {
1812		try self.spillRegisters(2, .{ .rax, .rdx });
	1755	try self.spillRegisters(&.{ .rax, .rdx });
1813	1756
1814	1757	const lhs = try self.resolveInst(bin_op.lhs);
1815	1758	const rhs = try self.resolveInst(bin_op.rhs);
...	...	@@ -1923,7 +1866,7 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
1923	1866	return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
1924	1867	}
1925	1868
1926		try self.spillRegisters(1, .{.rcx});
	1869	try self.spillRegisters(&.{.rcx});
1927	1870
1928	1871	const tag = self.air.instructions.items(.tag)[inst];
1929	1872	const lhs = try self.resolveInst(bin_op.lhs);
...	...	@@ -1947,59 +1890,82 @@ fn airShlSat(self: *Self, inst: Air.Inst.Index) !void {
1947	1890
1948	1891	fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {
1949	1892	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1950		if (self.liveness.isUnused(inst)) {
1951		return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
1952		}
1953
1954		const payload_ty = self.air.typeOfIndex(inst);
1955		const optional_ty = self.air.typeOf(ty_op.operand);
1956		const operand = try self.resolveInst(ty_op.operand);
1957	1893	const result: MCValue = result: {
1958		if (!payload_ty.hasRuntimeBits()) break :result MCValue.none;
1959		if (optional_ty.isPtrLikeOptional()) {
1960		if (self.reuseOperand(inst, ty_op.operand, 0, operand)) {
1961		break :result operand;
	1894	if (self.liveness.isUnused(inst)) break :result .none;
	1895
	1896	const pl_ty = self.air.typeOfIndex(inst);
	1897	const opt_mcv = try self.resolveInst(ty_op.operand);
	1898
	1899	if (self.reuseOperand(inst, ty_op.operand, 0, opt_mcv)) {
	1900	switch (opt_mcv) {
	1901	.register => \|reg\| try self.truncateRegister(pl_ty, reg),
	1902	.register_overflow => \|ro\| try self.truncateRegister(pl_ty, ro.reg),
	1903	else => {},
1962	1904	}
1963		break :result try self.copyToRegisterWithInstTracking(inst, payload_ty, operand);
	1905	break :result opt_mcv;
1964	1906	}
1965	1907
1966		const offset = optional_ty.abiSize(self.target.) - payload_ty.abiSize(self.target.);
1967		switch (operand) {
1968		.stack_offset => \|off\| {
1969		break :result MCValue{ .stack_offset = off - @intCast(i32, offset) };
1970		},
1971		.register => {
1972		// TODO reuse the operand
1973		const result = try self.copyToRegisterWithInstTracking(inst, optional_ty, operand);
1974		const shift = @intCast(u8, offset * @sizeOf(usize));
1975		try self.genShiftBinOpMir(.shr, optional_ty, result.register, .{ .immediate = @intCast(u8, shift) });
1976		break :result result;
1977		},
1978		else => return self.fail("TODO implement optional_payload when operand is {}", .{operand}),
1979		}
	1908	const pl_mcv = try self.allocRegOrMem(inst, true);
	1909	try self.setRegOrMem(pl_ty, pl_mcv, switch (opt_mcv) {
	1910	else => opt_mcv,
	1911	.register_overflow => \|ro\| .{ .register = ro.reg },
	1912	});
	1913	break :result pl_mcv;
1980	1914	};
1981	1915	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1982	1916	}
1983	1917
1984	1918	fn airOptionalPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
1985	1919	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1986		const result: MCValue = if (self.liveness.isUnused(inst))
1987		.dead
1988		else
1989		return self.fail("TODO implement .optional_payload_ptr for {}", .{self.target.cpu.arch});
	1920	const result: MCValue = result: {
	1921	if (self.liveness.isUnused(inst)) break :result .dead;
	1922
	1923	const dst_ty = self.air.typeOfIndex(inst);
	1924	const opt_mcv = try self.resolveInst(ty_op.operand);
	1925
	1926	break :result if (self.reuseOperand(inst, ty_op.operand, 0, opt_mcv))
	1927	opt_mcv
	1928	else
	1929	try self.copyToRegisterWithInstTracking(inst, dst_ty, opt_mcv);
	1930	};
1990	1931	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
1991	1932	}
1992	1933
1993	1934	fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
1994	1935	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1995		const result: MCValue = if (self.liveness.isUnused(inst))
1996		.dead
1997		else
1998		return self.fail("TODO implement .optional_payload_ptr_set for {}", .{self.target.cpu.arch});
	1936	const result = result: {
	1937	const dst_ty = self.air.typeOfIndex(inst);
	1938	const src_ty = self.air.typeOf(ty_op.operand);
	1939	const opt_ty = src_ty.childType();
	1940	const src_mcv = try self.resolveInst(ty_op.operand);
	1941
	1942	if (opt_ty.optionalReprIsPayload()) {
	1943	break :result if (self.liveness.isUnused(inst))
	1944	.dead
	1945	else if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
	1946	src_mcv
	1947	else
	1948	try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);
	1949	}
	1950
	1951	const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
	1952	src_mcv
	1953	else
	1954	try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);
	1955
	1956	const pl_ty = dst_ty.childType();
	1957	const pl_abi_size = @intCast(i32, pl_ty.abiSize(self.target.*));
	1958	try self.asmMemoryImmediate(
	1959	.mov,
	1960	Memory.sib(.byte, .{ .base = dst_mcv.register, .disp = pl_abi_size }),
	1961	Immediate.u(1),
	1962	);
	1963	break :result if (self.liveness.isUnused(inst)) .dead else dst_mcv;
	1964	};
1999	1965	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2000	1966	}
2001	1967
2002		fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) !void {
	1968	fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
2003	1969	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2004	1970	if (self.liveness.isUnused(inst)) {
2005	1971	return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
...	...	@@ -2026,8 +1992,14 @@ fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) !void {
2026	1992	},
2027	1993	.register => \|reg\| {
2028	1994	// TODO reuse operand
2029		const lock = self.register_manager.lockRegAssumeUnused(reg);
2030		defer self.register_manager.unlockReg(lock);
	1995	self.register_manager.getRegAssumeFree(.rcx, null);
	1996	const rcx_lock =
	1997	if (err_off > 0) self.register_manager.lockRegAssumeUnused(.rcx) else null;
	1998	defer if (rcx_lock) \|lock\| self.register_manager.unlockReg(lock);
	1999
	2000	const eu_lock = self.register_manager.lockReg(reg);
	2001	defer if (eu_lock) \|lock\| self.register_manager.unlockReg(lock);
	2002
2031	2003	const result = try self.copyToRegisterWithInstTracking(inst, err_union_ty, operand);
2032	2004	if (err_off > 0) {
2033	2005	const shift = @intCast(u6, err_off * 8);
...	...	@@ -2043,7 +2015,7 @@ fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) !void {
2043	2015	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2044	2016	}
2045	2017
2046		fn airUnwrapErrPayload(self: *Self, inst: Air.Inst.Index) !void {
	2018	fn airUnwrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
2047	2019	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2048	2020	if (self.liveness.isUnused(inst)) {
2049	2021	return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
...	...	@@ -2075,8 +2047,14 @@ fn genUnwrapErrorUnionPayloadMir(
2075	2047	},
2076	2048	.register => \|reg\| {
2077	2049	// TODO reuse operand
2078		const lock = self.register_manager.lockRegAssumeUnused(reg);
2079		defer self.register_manager.unlockReg(lock);
	2050	self.register_manager.getRegAssumeFree(.rcx, null);
	2051	const rcx_lock =
	2052	if (payload_off > 0) self.register_manager.lockRegAssumeUnused(.rcx) else null;
	2053	defer if (rcx_lock) \|lock\| self.register_manager.unlockReg(lock);
	2054
	2055	const eu_lock = self.register_manager.lockReg(reg);
	2056	defer if (eu_lock) \|lock\| self.register_manager.unlockReg(lock);
	2057
2080	2058	const result_reg: Register = if (maybe_inst) \|inst\|
2081	2059	(try self.copyToRegisterWithInstTracking(inst, err_union_ty, err_union)).register
2082	2060	else
...	...	@@ -2097,31 +2075,118 @@ fn genUnwrapErrorUnionPayloadMir(
2097	2075	}
2098	2076
2099	2077	// *(E!T) -> E
2100		fn airUnwrapErrErrPtr(self: *Self, inst: Air.Inst.Index) !void {
	2078	fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
2101	2079	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2102		const result: MCValue = if (self.liveness.isUnused(inst))
2103		.dead
2104		else
2105		return self.fail("TODO implement unwrap error union error ptr for {}", .{self.target.cpu.arch});
	2080	const result: MCValue = result: {
	2081	if (self.liveness.isUnused(inst)) break :result .dead;
	2082
	2083	const src_ty = self.air.typeOf(ty_op.operand);
	2084	const src_mcv = try self.resolveInst(ty_op.operand);
	2085	const src_reg = switch (src_mcv) {
	2086	.register => \|reg\| reg,
	2087	else => try self.copyToTmpRegister(src_ty, src_mcv),
	2088	};
	2089	const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
	2090	defer self.register_manager.unlockReg(src_lock);
	2091
	2092	const dst_reg = try self.register_manager.allocReg(inst, gp);
	2093	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
	2094	defer self.register_manager.unlockReg(dst_lock);
	2095
	2096	const eu_ty = src_ty.childType();
	2097	const pl_ty = eu_ty.errorUnionPayload();
	2098	const err_ty = eu_ty.errorUnionSet();
	2099	const err_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*));
	2100	const err_abi_size = @intCast(u32, err_ty.abiSize(self.target.*));
	2101	try self.asmRegisterMemory(
	2102	.mov,
	2103	registerAlias(dst_reg, err_abi_size),
	2104	Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{ .base = src_reg, .disp = err_off }),
	2105	);
	2106	break :result .{ .register = dst_reg };
	2107	};
2106	2108	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2107	2109	}
2108	2110
2109	2111	// (E!T) -> T
2110		fn airUnwrapErrPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
	2112	fn airUnwrapErrUnionPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
2111	2113	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2112		const result: MCValue = if (self.liveness.isUnused(inst))
2113		.dead
2114		else
2115		return self.fail("TODO implement unwrap error union payload ptr for {}", .{self.target.cpu.arch});
	2114	const result: MCValue = result: {
	2115	if (self.liveness.isUnused(inst)) break :result .dead;
	2116
	2117	const src_ty = self.air.typeOf(ty_op.operand);
	2118	const src_mcv = try self.resolveInst(ty_op.operand);
	2119	const src_reg = switch (src_mcv) {
	2120	.register => \|reg\| reg,
	2121	else => try self.copyToTmpRegister(src_ty, src_mcv),
	2122	};
	2123	const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
	2124	defer self.register_manager.unlockReg(src_lock);
	2125
	2126	const dst_ty = self.air.typeOfIndex(inst);
	2127	const dst_reg = if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
	2128	src_reg
	2129	else
	2130	try self.register_manager.allocReg(inst, gp);
	2131	const dst_lock = self.register_manager.lockReg(dst_reg);
	2132	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
	2133
	2134	const eu_ty = src_ty.childType();
	2135	const pl_ty = eu_ty.errorUnionPayload();
	2136	const pl_off = @intCast(i32, errUnionPayloadOffset(pl_ty, self.target.*));
	2137	const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
	2138	try self.asmRegisterMemory(
	2139	.lea,
	2140	registerAlias(dst_reg, dst_abi_size),
	2141	Memory.sib(.qword, .{ .base = src_reg, .disp = pl_off }),
	2142	);
	2143	break :result .{ .register = dst_reg };
	2144	};
2116	2145	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2117	2146	}
2118	2147
2119	2148	fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
2120	2149	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2121		const result: MCValue = if (self.liveness.isUnused(inst))
2122		.dead
2123		else
2124		return self.fail("TODO implement .errunion_payload_ptr_set for {}", .{self.target.cpu.arch});
	2150	const result: MCValue = result: {
	2151	const src_ty = self.air.typeOf(ty_op.operand);
	2152	const src_mcv = try self.resolveInst(ty_op.operand);
	2153	const src_reg = switch (src_mcv) {
	2154	.register => \|reg\| reg,
	2155	else => try self.copyToTmpRegister(src_ty, src_mcv),
	2156	};
	2157	const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
	2158	defer self.register_manager.unlockReg(src_lock);
	2159
	2160	const eu_ty = src_ty.childType();
	2161	const pl_ty = eu_ty.errorUnionPayload();
	2162	const err_ty = eu_ty.errorUnionSet();
	2163	const err_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*));
	2164	const err_abi_size = @intCast(u32, err_ty.abiSize(self.target.*));
	2165	try self.asmMemoryImmediate(
	2166	.mov,
	2167	Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{ .base = src_reg, .disp = err_off }),
	2168	Immediate.u(0),
	2169	);
	2170
	2171	if (self.liveness.isUnused(inst)) break :result .dead;
	2172
	2173	const dst_ty = self.air.typeOfIndex(inst);
	2174	const dst_reg = if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
	2175	src_reg
	2176	else
	2177	try self.register_manager.allocReg(inst, gp);
	2178	const dst_lock = self.register_manager.lockReg(dst_reg);
	2179	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
	2180
	2181	const pl_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*));
	2182	const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
	2183	try self.asmRegisterMemory(
	2184	.lea,
	2185	registerAlias(dst_reg, dst_abi_size),
	2186	Memory.sib(.qword, .{ .base = src_reg, .disp = pl_off }),
	2187	);
	2188	break :result .{ .register = dst_reg };
	2189	};
2125	2190	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2126	2191	}
2127	2192
...	...	@@ -2145,41 +2210,45 @@ fn airSaveErrReturnTraceIndex(self: *Self, inst: Air.Inst.Index) !void {
2145	2210
2146	2211	fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
2147	2212	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2148		if (self.liveness.isUnused(inst)) {
2149		return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
2150		}
2151
2152		const payload_ty = self.air.typeOf(ty_op.operand);
2153	2213	const result: MCValue = result: {
2154		if (!payload_ty.hasRuntimeBits()) {
2155		break :result MCValue{ .immediate = 1 };
2156		}
	2214	if (self.liveness.isUnused(inst)) break :result .dead;
2157	2215
2158		const optional_ty = self.air.typeOfIndex(inst);
2159		const operand = try self.resolveInst(ty_op.operand);
2160		const operand_lock: ?RegisterLock = switch (operand) {
	2216	const pl_ty = self.air.typeOf(ty_op.operand);
	2217	if (!pl_ty.hasRuntimeBits()) break :result .{ .immediate = 1 };
	2218
	2219	const opt_ty = self.air.typeOfIndex(inst);
	2220	const pl_mcv = try self.resolveInst(ty_op.operand);
	2221	const same_repr = opt_ty.optionalReprIsPayload();
	2222	if (same_repr and self.reuseOperand(inst, ty_op.operand, 0, pl_mcv)) break :result pl_mcv;
	2223
	2224	const pl_lock: ?RegisterLock = switch (pl_mcv) {
2161	2225	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
2162	2226	else => null,
2163	2227	};
2164		defer if (operand_lock) \|lock\| self.register_manager.unlockReg(lock);
	2228	defer if (pl_lock) \|lock\| self.register_manager.unlockReg(lock);
2165	2229
2166		if (optional_ty.isPtrLikeOptional()) {
2167		// TODO should we check if we can reuse the operand?
2168		if (self.reuseOperand(inst, ty_op.operand, 0, operand)) {
2169		break :result operand;
2170		}
2171		break :result try self.copyToRegisterWithInstTracking(inst, payload_ty, operand);
2172		}
	2230	const opt_mcv = try self.allocRegOrMem(inst, true);
	2231	try self.setRegOrMem(pl_ty, opt_mcv, pl_mcv);
2173	2232
2174		const optional_abi_size = @intCast(u32, optional_ty.abiSize(self.target.*));
2175		const optional_abi_align = optional_ty.abiAlignment(self.target.*);
2176		const payload_abi_size = @intCast(u32, payload_ty.abiSize(self.target.*));
2177		const offset = optional_abi_size - payload_abi_size;
	2233	if (!same_repr) {
	2234	const pl_abi_size = @intCast(i32, pl_ty.abiSize(self.target.*));
	2235	switch (opt_mcv) {
	2236	else => unreachable,
2178	2237
2179		const stack_offset = @intCast(i32, try self.allocMem(inst, optional_abi_size, optional_abi_align));
2180		try self.genSetStack(Type.bool, stack_offset, .{ .immediate = 1 }, .{});
2181		try self.genSetStack(payload_ty, stack_offset - @intCast(i32, offset), operand, .{});
2182		break :result MCValue{ .stack_offset = stack_offset };
	2238	.register => \|opt_reg\| try self.asmRegisterImmediate(
	2239	.bts,
	2240	opt_reg,
	2241	Immediate.u(@intCast(u6, pl_abi_size * 8)),
	2242	),
	2243
	2244	.stack_offset => \|off\| try self.asmMemoryImmediate(
	2245	.mov,
	2246	Memory.sib(.byte, .{ .base = .rsp, .disp = pl_abi_size - off }),
	2247	Immediate.u(0),
	2248	),
	2249	}
	2250	}
	2251	break :result opt_mcv;
2183	2252	};
2184	2253	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2185	2254	}
...	...	@@ -2281,19 +2350,53 @@ fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {
2281	2350
2282	2351	fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
2283	2352	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2284		const result: MCValue = if (self.liveness.isUnused(inst))
2285		.dead
2286		else
2287		return self.fail("TODO implement ptr_slice_len_ptr for {}", .{self.target.cpu.arch});
	2353	const result: MCValue = result: {
	2354	if (self.liveness.isUnused(inst)) break :result .dead;
	2355
	2356	const src_ty = self.air.typeOf(ty_op.operand);
	2357	const src_mcv = try self.resolveInst(ty_op.operand);
	2358	const src_reg = switch (src_mcv) {
	2359	.register => \|reg\| reg,
	2360	else => try self.copyToTmpRegister(src_ty, src_mcv),
	2361	};
	2362	const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
	2363	defer self.register_manager.unlockReg(src_lock);
	2364
	2365	const dst_ty = self.air.typeOfIndex(inst);
	2366	const dst_reg = if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
	2367	src_reg
	2368	else
	2369	try self.register_manager.allocReg(inst, gp);
	2370	const dst_lock = self.register_manager.lockReg(dst_reg);
	2371	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
	2372
	2373	const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
	2374	try self.asmRegisterMemory(
	2375	.lea,
	2376	registerAlias(dst_reg, dst_abi_size),
	2377	Memory.sib(.qword, .{
	2378	.base = src_reg,
	2379	.disp = @divExact(self.target.cpu.arch.ptrBitWidth(), 8),
	2380	}),
	2381	);
	2382	break :result .{ .register = dst_reg };
	2383	};
2288	2384	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2289	2385	}
2290	2386
2291	2387	fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void {
2292	2388	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2293		const result: MCValue = if (self.liveness.isUnused(inst))
2294		.dead
2295		else
2296		return self.fail("TODO implement ptr_slice_ptr_ptr for {}", .{self.target.cpu.arch});
	2389	const result: MCValue = result: {
	2390	if (self.liveness.isUnused(inst)) break :result .dead;
	2391
	2392	const dst_ty = self.air.typeOfIndex(inst);
	2393	const opt_mcv = try self.resolveInst(ty_op.operand);
	2394
	2395	break :result if (self.reuseOperand(inst, ty_op.operand, 0, opt_mcv))
	2396	opt_mcv
	2397	else
	2398	try self.copyToRegisterWithInstTracking(inst, dst_ty, opt_mcv);
	2399	};
2297	2400	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2298	2401	}
2299	2402
...	...	@@ -2492,10 +2595,7 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
2492	2595	try self.asmRegisterMemory(
2493	2596	.mov,
2494	2597	registerAlias(dst_mcv.register, elem_abi_size),
2495		Memory.sib(Memory.PtrSize.fromSize(elem_abi_size), .{
2496		.base = dst_mcv.register,
2497		.disp = 0,
2498		}),
	2598	Memory.sib(Memory.PtrSize.fromSize(elem_abi_size), .{ .base = dst_mcv.register }),
2499	2599	);
2500	2600	break :result .{ .register = registerAlias(dst_mcv.register, @intCast(u32, elem_abi_size)) };
2501	2601	}
...	...	@@ -2614,7 +2714,7 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
2614	2714	},
2615	2715	.register => {
2616	2716	const shift: u6 = if (layout.tag_align < layout.payload_align)
2617		@intCast(u6, layout.payload_size * @sizeOf(usize))
	2717	@intCast(u6, layout.payload_size * 8)
2618	2718	else
2619	2719	0;
2620	2720	const result = try self.copyToRegisterWithInstTracking(inst, union_ty, operand);
...	...	@@ -2632,46 +2732,418 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
2632	2732
2633	2733	fn airClz(self: *Self, inst: Air.Inst.Index) !void {
2634	2734	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2635		const result: MCValue = if (self.liveness.isUnused(inst))
2636		.dead
2637		else
2638		return self.fail("TODO implement airClz for {}", .{self.target.cpu.arch});
	2735	const result = result: {
	2736	if (self.liveness.isUnused(inst)) break :result .dead;
	2737
	2738	const dst_ty = self.air.typeOfIndex(inst);
	2739	const src_ty = self.air.typeOf(ty_op.operand);
	2740
	2741	const src_mcv = try self.resolveInst(ty_op.operand);
	2742	const mat_src_mcv = switch (src_mcv) {
	2743	.immediate => MCValue{ .register = try self.copyToTmpRegister(src_ty, src_mcv) },
	2744	else => src_mcv,
	2745	};
	2746	const mat_src_lock = switch (mat_src_mcv) {
	2747	.register => \|reg\| self.register_manager.lockReg(reg),
	2748	else => null,
	2749	};
	2750	defer if (mat_src_lock) \|lock\| self.register_manager.unlockReg(lock);
	2751
	2752	const dst_reg = try self.register_manager.allocReg(inst, gp);
	2753	const dst_mcv = MCValue{ .register = dst_reg };
	2754	const dst_lock = self.register_manager.lockReg(dst_reg);
	2755	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
	2756
	2757	if (Target.x86.featureSetHas(self.target.cpu.features, .lzcnt)) {
	2758	try self.genBinOpMir(.lzcnt, src_ty, dst_mcv, mat_src_mcv);
	2759	const extra_bits = self.regExtraBits(src_ty);
	2760	if (extra_bits > 0) {
	2761	try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .immediate = extra_bits });
	2762	}
	2763	break :result dst_mcv;
	2764	}
	2765
	2766	const src_bits = src_ty.bitSize(self.target.*);
	2767	const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits });
	2768	const width_mcv = MCValue{ .register = width_reg };
	2769	try self.genBinOpMir(.bsr, src_ty, dst_mcv, mat_src_mcv);
	2770
	2771	const dst_abi_size = @intCast(u32, @max(dst_ty.abiSize(self.target.*), 2));
	2772	try self.asmCmovccRegisterRegister(
	2773	registerAlias(dst_reg, dst_abi_size),
	2774	registerAlias(width_reg, dst_abi_size),
	2775	.z,
	2776	);
	2777
	2778	try self.genBinOpMir(.sub, dst_ty, width_mcv, dst_mcv);
	2779	break :result width_mcv;
	2780	};
2639	2781	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2640	2782	}
2641	2783
2642	2784	fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
2643	2785	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2644		const result: MCValue = if (self.liveness.isUnused(inst))
2645		.dead
2646		else
2647		return self.fail("TODO implement airCtz for {}", .{self.target.cpu.arch});
2648		return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2649		}
	2786	const result = result: {
	2787	if (self.liveness.isUnused(inst)) break :result .dead;
2650	2788
2651		fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
	2789	const dst_ty = self.air.typeOfIndex(inst);
	2790	const src_ty = self.air.typeOf(ty_op.operand);
	2791	const src_bits = src_ty.bitSize(self.target.*);
	2792
	2793	const src_mcv = try self.resolveInst(ty_op.operand);
	2794	const mat_src_mcv = switch (src_mcv) {
	2795	.immediate => MCValue{ .register = try self.copyToTmpRegister(src_ty, src_mcv) },
	2796	else => src_mcv,
	2797	};
	2798	const mat_src_lock = switch (mat_src_mcv) {
	2799	.register => \|reg\| self.register_manager.lockReg(reg),
	2800	else => null,
	2801	};
	2802	defer if (mat_src_lock) \|lock\| self.register_manager.unlockReg(lock);
	2803
	2804	const dst_reg = try self.register_manager.allocReg(inst, gp);
	2805	const dst_mcv = MCValue{ .register = dst_reg };
	2806	const dst_lock = self.register_manager.lockReg(dst_reg);
	2807	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
	2808
	2809	if (Target.x86.featureSetHas(self.target.cpu.features, .bmi)) {
	2810	const extra_bits = self.regExtraBits(src_ty);
	2811	const masked_mcv = if (extra_bits > 0) masked: {
	2812	const mask_mcv = MCValue{
	2813	.immediate = ((@as(u64, 1) << @intCast(u6, extra_bits)) - 1) << @intCast(u6, src_bits),
	2814	};
	2815	const tmp_mcv = tmp: {
	2816	if (src_mcv.isImmediate() or self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) {
	2817	break :tmp src_mcv;
	2818	}
	2819	try self.genSetReg(src_ty, dst_reg, src_mcv);
	2820	break :tmp dst_mcv;
	2821	};
	2822	try self.genBinOpMir(.@"or", src_ty, tmp_mcv, mask_mcv);
	2823	break :masked tmp_mcv;
	2824	} else mat_src_mcv;
	2825	try self.genBinOpMir(.tzcnt, src_ty, dst_mcv, masked_mcv);
	2826	break :result dst_mcv;
	2827	}
	2828
	2829	const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits });
	2830	try self.genBinOpMir(.bsf, src_ty, dst_mcv, mat_src_mcv);
	2831
	2832	const abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
	2833	try self.asmCmovccRegisterRegister(
	2834	registerAlias(dst_reg, abi_size),
	2835	registerAlias(width_reg, abi_size),
	2836	.z,
	2837	);
	2838
	2839	break :result dst_mcv;
	2840	};
	2841	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	2842	}
	2843
	2844	fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
2652	2845	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2653		const result: MCValue = if (self.liveness.isUnused(inst))
2654		.dead
2655		else
2656		return self.fail("TODO implement airPopcount for {}", .{self.target.cpu.arch});
	2846	const result: MCValue = result: {
	2847	if (self.liveness.isUnused(inst)) break :result .dead;
	2848
	2849	const src_ty = self.air.typeOf(ty_op.operand);
	2850	const src_abi_size = @intCast(u32, src_ty.abiSize(self.target.*));
	2851	const src_mcv = try self.resolveInst(ty_op.operand);
	2852
	2853	if (Target.x86.featureSetHas(self.target.cpu.features, .popcnt)) {
	2854	const mat_src_mcv = switch (src_mcv) {
	2855	.immediate => MCValue{ .register = try self.copyToTmpRegister(src_ty, src_mcv) },
	2856	else => src_mcv,
	2857	};
	2858	const mat_src_lock = switch (mat_src_mcv) {
	2859	.register => \|reg\| self.register_manager.lockReg(reg),
	2860	else => null,
	2861	};
	2862	defer if (mat_src_lock) \|lock\| self.register_manager.unlockReg(lock);
	2863
	2864	const dst_mcv: MCValue = if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
	2865	src_mcv
	2866	else
	2867	.{ .register = try self.register_manager.allocReg(inst, gp) };
	2868
	2869	const popcnt_ty = if (src_abi_size > 1) src_ty else Type.u16;
	2870	try self.genBinOpMir(.popcnt, popcnt_ty, dst_mcv, mat_src_mcv);
	2871	break :result dst_mcv;
	2872	}
	2873
	2874	const mask = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - src_abi_size * 8);
	2875	const imm_0_1 = Immediate.u(mask / 0b1_1);
	2876	const imm_00_11 = Immediate.u(mask / 0b01_01);
	2877	const imm_0000_1111 = Immediate.u(mask / 0b0001_0001);
	2878	const imm_0000_0001 = Immediate.u(mask / 0b1111_1111);
	2879
	2880	const tmp_reg = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
	2881	src_mcv.register
	2882	else
	2883	try self.copyToTmpRegister(src_ty, src_mcv);
	2884	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
	2885	defer self.register_manager.unlockReg(tmp_lock);
	2886
	2887	const dst_reg = try self.register_manager.allocReg(inst, gp);
	2888	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
	2889	defer self.register_manager.unlockReg(dst_lock);
	2890
	2891	{
	2892	const dst = registerAlias(dst_reg, src_abi_size);
	2893	const tmp = registerAlias(tmp_reg, src_abi_size);
	2894	const imm = if (src_abi_size > 4)
	2895	try self.register_manager.allocReg(null, gp)
	2896	else
	2897	undefined;
	2898
	2899	// tmp = operand
	2900	try self.asmRegisterRegister(.mov, dst, tmp);
	2901	// dst = operand
	2902	try self.asmRegisterImmediate(.shr, tmp, Immediate.u(1));
	2903	// tmp = operand >> 1
	2904	if (src_abi_size > 4) {
	2905	try self.asmRegisterImmediate(.mov, imm, imm_0_1);
	2906	try self.asmRegisterRegister(.@"and", tmp, imm);
	2907	} else try self.asmRegisterImmediate(.@"and", tmp, imm_0_1);
	2908	// tmp = (operand >> 1) & 0x55...55
	2909	try self.asmRegisterRegister(.sub, dst, tmp);
	2910	// dst = temp1 = operand - ((operand >> 1) & 0x55...55)
	2911	try self.asmRegisterRegister(.mov, tmp, dst);
	2912	// tmp = temp1
	2913	try self.asmRegisterImmediate(.shr, dst, Immediate.u(2));
	2914	// dst = temp1 >> 2
	2915	if (src_abi_size > 4) {
	2916	try self.asmRegisterImmediate(.mov, imm, imm_00_11);
	2917	try self.asmRegisterRegister(.@"and", tmp, imm);
	2918	try self.asmRegisterRegister(.@"and", dst, imm);
	2919	} else {
	2920	try self.asmRegisterImmediate(.@"and", tmp, imm_00_11);
	2921	try self.asmRegisterImmediate(.@"and", dst, imm_00_11);
	2922	}
	2923	// tmp = temp1 & 0x33...33
	2924	// dst = (temp1 >> 2) & 0x33...33
	2925	try self.asmRegisterRegister(.add, tmp, dst);
	2926	// tmp = temp2 = (temp1 & 0x33...33) + ((temp1 >> 2) & 0x33...33)
	2927	try self.asmRegisterRegister(.mov, dst, tmp);
	2928	// dst = temp2
	2929	try self.asmRegisterImmediate(.shr, tmp, Immediate.u(4));
	2930	// tmp = temp2 >> 4
	2931	try self.asmRegisterRegister(.add, dst, tmp);
	2932	// dst = temp2 + (temp2 >> 4)
	2933	if (src_abi_size > 4) {
	2934	try self.asmRegisterImmediate(.mov, imm, imm_0000_1111);
	2935	try self.asmRegisterImmediate(.mov, tmp, imm_0000_0001);
	2936	try self.asmRegisterRegister(.@"and", dst, imm);
	2937	try self.asmRegisterRegister(.imul, dst, tmp);
	2938	} else {
	2939	try self.asmRegisterImmediate(.@"and", dst, imm_0000_1111);
	2940	if (src_abi_size > 1) {
	2941	try self.asmRegisterRegisterImmediate(.imul, dst, dst, imm_0000_0001);
	2942	}
	2943	}
	2944	// dst = temp3 = (temp2 + (temp2 >> 4)) & 0x0f...0f
	2945	// dst = temp3 * 0x01...01
	2946	if (src_abi_size > 1) {
	2947	try self.asmRegisterImmediate(.shr, dst, Immediate.u((src_abi_size - 1) * 8));
	2948	}
	2949	// dst = (temp3 * 0x01...01) >> (bits - 8)
	2950	}
	2951	break :result .{ .register = dst_reg };
	2952	};
2657	2953	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2658	2954	}
2659	2955
	2956	fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, mem_ok: bool) !MCValue {
	2957	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	2958
	2959	const src_bits = self.regBitSize(src_ty);
	2960	const src_lock = switch (src_mcv) {
	2961	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
	2962	else => null,
	2963	};
	2964	defer if (src_lock) \|lock\| self.register_manager.unlockReg(lock);
	2965
	2966	switch (src_bits) {
	2967	else => unreachable,
	2968	8 => return if ((mem_ok or src_mcv.isRegister()) and
	2969	self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
	2970	src_mcv
	2971	else
	2972	try self.copyToRegisterWithInstTracking(inst, src_ty, src_mcv),
	2973	16 => if ((mem_ok or src_mcv.isRegister()) and
	2974	self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
	2975	{
	2976	try self.genBinOpMir(.rol, src_ty, src_mcv, .{ .immediate = 8 });
	2977	return src_mcv;
	2978	},
	2979	32, 64 => if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) {
	2980	try self.genUnOpMir(.bswap, src_ty, src_mcv);
	2981	return src_mcv;
	2982	},
	2983	}
	2984
	2985	if (src_mcv.isRegister()) {
	2986	const dst_mcv: MCValue = if (mem_ok)
	2987	try self.allocRegOrMem(inst, true)
	2988	else
	2989	.{ .register = try self.register_manager.allocReg(inst, gp) };
	2990	if (dst_mcv.isRegister()) {
	2991	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_mcv.register);
	2992	defer self.register_manager.unlockReg(dst_lock);
	2993
	2994	try self.genSetReg(src_ty, dst_mcv.register, src_mcv);
	2995	switch (src_bits) {
	2996	else => unreachable,
	2997	16 => try self.genBinOpMir(.rol, src_ty, dst_mcv, .{ .immediate = 8 }),
	2998	32, 64 => try self.genUnOpMir(.bswap, src_ty, dst_mcv),
	2999	}
	3000	} else try self.genBinOpMir(.movbe, src_ty, dst_mcv, src_mcv);
	3001	return dst_mcv;
	3002	}
	3003
	3004	const dst_reg = try self.register_manager.allocReg(inst, gp);
	3005	const dst_mcv = MCValue{ .register = dst_reg };
	3006	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
	3007	defer self.register_manager.unlockReg(dst_lock);
	3008
	3009	try self.genBinOpMir(.movbe, src_ty, dst_mcv, src_mcv);
	3010	return dst_mcv;
	3011	}
	3012
2660	3013	fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {
2661	3014	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2662		const result: MCValue = if (self.liveness.isUnused(inst))
2663		.dead
2664		else
2665		return self.fail("TODO implement airByteSwap for {}", .{self.target.cpu.arch});
	3015	const result = result: {
	3016	if (self.liveness.isUnused(inst)) break :result .dead;
	3017
	3018	const src_ty = self.air.typeOf(ty_op.operand);
	3019	const src_mcv = try self.resolveInst(ty_op.operand);
	3020
	3021	const dst_mcv = try self.byteSwap(inst, src_ty, src_mcv, true);
	3022	switch (self.regExtraBits(src_ty)) {
	3023	0 => {},
	3024	else => \|extra\| try self.genBinOpMir(
	3025	if (src_ty.isSignedInt()) .sar else .shr,
	3026	src_ty,
	3027	dst_mcv,
	3028	.{ .immediate = extra },
	3029	),
	3030	}
	3031	break :result dst_mcv;
	3032	};
	3033
2666	3034	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2667	3035	}
2668	3036
2669	3037	fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
2670	3038	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2671		const result: MCValue = if (self.liveness.isUnused(inst))
2672		.dead
2673		else
2674		return self.fail("TODO implement airBitReverse for {}", .{self.target.cpu.arch});
	3039	const result = result: {
	3040	if (self.liveness.isUnused(inst)) break :result .dead;
	3041
	3042	const src_ty = self.air.typeOf(ty_op.operand);
	3043	const src_abi_size = @intCast(u32, src_ty.abiSize(self.target.*));
	3044	const src_mcv = try self.resolveInst(ty_op.operand);
	3045
	3046	const dst_mcv = try self.byteSwap(inst, src_ty, src_mcv, false);
	3047	const dst_reg = dst_mcv.register;
	3048	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
	3049	defer self.register_manager.unlockReg(dst_lock);
	3050
	3051	const tmp_reg = try self.register_manager.allocReg(null, gp);
	3052	const tmp_lock = self.register_manager.lockReg(tmp_reg);
	3053	defer if (tmp_lock) \|lock\| self.register_manager.unlockReg(lock);
	3054
	3055	{
	3056	const dst = registerAlias(dst_reg, src_abi_size);
	3057	const tmp = registerAlias(tmp_reg, src_abi_size);
	3058	const imm = if (src_abi_size > 4)
	3059	try self.register_manager.allocReg(null, gp)
	3060	else
	3061	undefined;
	3062
	3063	const mask = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - src_abi_size * 8);
	3064	const imm_0000_1111 = Immediate.u(mask / 0b0001_0001);
	3065	const imm_00_11 = Immediate.u(mask / 0b01_01);
	3066	const imm_0_1 = Immediate.u(mask / 0b1_1);
	3067
	3068	// dst = temp1 = bswap(operand)
	3069	try self.asmRegisterRegister(.mov, tmp, dst);
	3070	// tmp = temp1
	3071	try self.asmRegisterImmediate(.shr, dst, Immediate.u(4));
	3072	// dst = temp1 >> 4
	3073	if (src_abi_size > 4) {
	3074	try self.asmRegisterImmediate(.mov, imm, imm_0000_1111);
	3075	try self.asmRegisterRegister(.@"and", tmp, imm);
	3076	try self.asmRegisterRegister(.@"and", dst, imm);
	3077	} else {
	3078	try self.asmRegisterImmediate(.@"and", tmp, imm_0000_1111);
	3079	try self.asmRegisterImmediate(.@"and", dst, imm_0000_1111);
	3080	}
	3081	// tmp = temp1 & 0x0F...0F
	3082	// dst = (temp1 >> 4) & 0x0F...0F
	3083	try self.asmRegisterImmediate(.shl, tmp, Immediate.u(4));
	3084	// tmp = (temp1 & 0x0F...0F) << 4
	3085	try self.asmRegisterRegister(.@"or", dst, tmp);
	3086	// dst = temp2 = ((temp1 >> 4) & 0x0F...0F) \| ((temp1 & 0x0F...0F) << 4)
	3087	try self.asmRegisterRegister(.mov, tmp, dst);
	3088	// tmp = temp2
	3089	try self.asmRegisterImmediate(.shr, dst, Immediate.u(2));
	3090	// dst = temp2 >> 2
	3091	if (src_abi_size > 4) {
	3092	try self.asmRegisterImmediate(.mov, imm, imm_00_11);
	3093	try self.asmRegisterRegister(.@"and", tmp, imm);
	3094	try self.asmRegisterRegister(.@"and", dst, imm);
	3095	} else {
	3096	try self.asmRegisterImmediate(.@"and", tmp, imm_00_11);
	3097	try self.asmRegisterImmediate(.@"and", dst, imm_00_11);
	3098	}
	3099	// tmp = temp2 & 0x33...33
	3100	// dst = (temp2 >> 2) & 0x33...33
	3101	try self.asmRegisterMemory(
	3102	.lea,
	3103	if (src_abi_size > 4) tmp.to64() else tmp.to32(),
	3104	Memory.sib(.qword, .{
	3105	.base = dst.to64(),
	3106	.scale_index = .{ .index = tmp.to64(), .scale = 1 << 2 },
	3107	}),
	3108	);
	3109	// tmp = temp3 = ((temp2 >> 2) & 0x33...33) + ((temp2 & 0x33...33) << 2)
	3110	try self.asmRegisterRegister(.mov, dst, tmp);
	3111	// dst = temp3
	3112	try self.asmRegisterImmediate(.shr, tmp, Immediate.u(1));
	3113	// tmp = temp3 >> 1
	3114	if (src_abi_size > 4) {
	3115	try self.asmRegisterImmediate(.mov, imm, imm_0_1);
	3116	try self.asmRegisterRegister(.@"and", dst, imm);
	3117	try self.asmRegisterRegister(.@"and", tmp, imm);
	3118	} else {
	3119	try self.asmRegisterImmediate(.@"and", dst, imm_0_1);
	3120	try self.asmRegisterImmediate(.@"and", tmp, imm_0_1);
	3121	}
	3122	// dst = temp3 & 0x55...55
	3123	// tmp = (temp3 >> 1) & 0x55...55
	3124	try self.asmRegisterMemory(
	3125	.lea,
	3126	if (src_abi_size > 4) dst.to64() else dst.to32(),
	3127	Memory.sib(.qword, .{
	3128	.base = tmp.to64(),
	3129	.scale_index = .{ .index = dst.to64(), .scale = 1 << 1 },
	3130	}),
	3131	);
	3132	// dst = ((temp3 >> 1) & 0x55...55) + ((temp3 & 0x55...55) << 1)
	3133	}
	3134
	3135	switch (self.regExtraBits(src_ty)) {
	3136	0 => {},
	3137	else => \|extra\| try self.genBinOpMir(
	3138	if (src_ty.isSignedInt()) .sar else .shr,
	3139	src_ty,
	3140	dst_mcv,
	3141	.{ .immediate = extra },
	3142	),
	3143	}
	3144	break :result dst_mcv;
	3145	};
	3146
2675	3147	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2676	3148	}
2677	3149
...	...	@@ -2753,7 +3225,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
2753	3225	try self.asmRegisterMemory(
2754	3226	.mov,
2755	3227	registerAlias(dst_reg, abi_size),
2756		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg, .disp = 0 }),
	3228	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg }),
2757	3229	);
2758	3230	},
2759	3231	.stack_offset => \|off\| {
...	...	@@ -2865,8 +3337,14 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2865	3337	.none => unreachable,
2866	3338	.dead => unreachable,
2867	3339	.unreach => unreachable,
2868		.eflags => unreachable,
	3340	.eflags => \|cc\| {
	3341	try self.asmSetccMemory(Memory.sib(
	3342	Memory.PtrSize.fromSize(abi_size),
	3343	.{ .base = reg.to64() },
	3344	), cc);
	3345	},
2869	3346	.undef => {
	3347	if (!self.wantSafety()) return; // The already existing value will do just fine.
2870	3348	switch (abi_size) {
2871	3349	1 => try self.store(ptr, .{ .immediate = 0xaa }, ptr_ty, value_ty),
2872	3350	2 => try self.store(ptr, .{ .immediate = 0xaaaa }, ptr_ty, value_ty),
...	...	@@ -2882,10 +3360,10 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2882	3360	Immediate.s(@intCast(i32, @bitCast(i64, imm)))
2883	3361	else
2884	3362	Immediate.u(@truncate(u32, imm));
2885		try self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
2886		.base = reg.to64(),
2887		.disp = 0,
2888		}), immediate);
	3363	try self.asmMemoryImmediate(.mov, Memory.sib(
	3364	Memory.PtrSize.fromSize(abi_size),
	3365	.{ .base = reg.to64() },
	3366	), immediate);
2889	3367	},
2890	3368	8 => {
2891	3369	// TODO: optimization: if the imm is only using the lower
...	...	@@ -2957,10 +3435,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2957	3435	try self.loadMemPtrIntoRegister(addr_reg, ptr_ty, ptr);
2958	3436
2959	3437	// To get the actual address of the value we want to modify we have to go through the GOT
2960		try self.asmRegisterMemory(.mov, addr_reg.to64(), Memory.sib(.qword, .{
2961		.base = addr_reg.to64(),
2962		.disp = 0,
2963		}));
	3438	try self.asmRegisterMemory(
	3439	.mov,
	3440	addr_reg.to64(),
	3441	Memory.sib(.qword, .{ .base = addr_reg.to64() }),
	3442	);
2964	3443
2965	3444	const new_ptr = MCValue{ .register = addr_reg.to64() };
2966	3445
...	...	@@ -2982,10 +3461,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2982	3461	return self.fail("TODO imm64 would get incorrectly sign extended", .{});
2983	3462	}
2984	3463	}
2985		try self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
2986		.base = addr_reg.to64(),
2987		.disp = 0,
2988		}), Immediate.u(@intCast(u32, imm)));
	3464	try self.asmMemoryImmediate(
	3465	.mov,
	3466	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = addr_reg.to64() }),
	3467	Immediate.u(@intCast(u32, imm)),
	3468	);
2989	3469	},
2990	3470	.register => {
2991	3471	return self.store(new_ptr, value, ptr_ty, value_ty);
...	...	@@ -2997,10 +3477,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
2997	3477	defer self.register_manager.unlockReg(tmp_reg_lock);
2998	3478
2999	3479	try self.loadMemPtrIntoRegister(tmp_reg, value_ty, value);
3000		try self.asmRegisterMemory(.mov, tmp_reg, Memory.sib(.qword, .{
3001		.base = tmp_reg,
3002		.disp = 0,
3003		}));
	3480	try self.asmRegisterMemory(
	3481	.mov,
	3482	tmp_reg,
	3483	Memory.sib(.qword, .{ .base = tmp_reg }),
	3484	);
3004	3485
3005	3486	return self.store(new_ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
3006	3487	}
...	...	@@ -3152,7 +3633,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
3152	3633	defer if (dst_mcv_lock) \|lock\| self.register_manager.unlockReg(lock);
3153	3634
3154	3635	// Shift by struct_field_offset.
3155		const shift = @intCast(u8, struct_field_offset * @sizeOf(usize));
	3636	const shift = @intCast(u8, struct_field_offset * 8);
3156	3637	try self.genShiftBinOpMir(.shr, Type.usize, dst_mcv.register, .{ .immediate = shift });
3157	3638
3158	3639	// Mask with reg.bitSize() - struct_field_size
...	...	@@ -3212,6 +3693,107 @@ fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void {
3212	3693	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3213	3694	}
3214	3695
	3696	fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air: Air.Inst.Ref) !MCValue {
	3697	const src_ty = self.air.typeOf(src_air);
	3698	const src_mcv = try self.resolveInst(src_air);
	3699	if (src_ty.zigTypeTag() == .Vector) {
	3700	return self.fail("TODO implement genBinOp for {}", .{src_ty.fmt(self.bin_file.options.module.?)});
	3701	}
	3702	if (src_ty.abiSize(self.target.*) > 8) {
	3703	return self.fail("TODO implement genBinOp for {}", .{src_ty.fmt(self.bin_file.options.module.?)});
	3704	}
	3705
	3706	switch (src_mcv) {
	3707	.eflags => \|cc\| switch (tag) {
	3708	.not => return .{ .eflags = cc.negate() },
	3709	else => {},
	3710	},
	3711	else => {},
	3712	}
	3713
	3714	const src_lock = switch (src_mcv) {
	3715	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
	3716	else => null,
	3717	};
	3718	defer if (src_lock) \|lock\| self.register_manager.unlockReg(lock);
	3719
	3720	const dst_mcv: MCValue = if (maybe_inst) \|inst\|
	3721	if (self.reuseOperand(inst, src_air, 0, src_mcv))
	3722	src_mcv
	3723	else
	3724	try self.copyToRegisterWithInstTracking(inst, src_ty, src_mcv)
	3725	else
	3726	.{ .register = try self.copyToTmpRegister(src_ty, src_mcv) };
	3727	const dst_lock = switch (dst_mcv) {
	3728	.register => \|reg\| self.register_manager.lockReg(reg),
	3729	else => null,
	3730	};
	3731	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
	3732
	3733	switch (tag) {
	3734	.not => {
	3735	const int_info = if (src_ty.tag() == .bool)
	3736	std.builtin.Type.Int{ .signedness = .unsigned, .bits = 1 }
	3737	else
	3738	src_ty.intInfo(self.target.*);
	3739	const extra_bits = self.regExtraBits(src_ty);
	3740	if (int_info.signedness == .unsigned and extra_bits > 0) {
	3741	const mask = (@as(u64, 1) << @intCast(u6, src_ty.bitSize(self.target.*))) - 1;
	3742	try self.genBinOpMir(.xor, src_ty, dst_mcv, .{ .immediate = mask });
	3743	} else try self.genUnOpMir(.not, src_ty, dst_mcv);
	3744	},
	3745
	3746	.neg => try self.genUnOpMir(.neg, src_ty, dst_mcv),
	3747
	3748	else => unreachable,
	3749	}
	3750	return dst_mcv;
	3751	}
	3752
	3753	fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue) !void {
	3754	const abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
	3755	switch (dst_mcv) {
	3756	.none => unreachable,
	3757	.undef => unreachable,
	3758	.dead, .unreach, .immediate => unreachable,
	3759	.eflags => unreachable,
	3760	.register_overflow => unreachable,
	3761	.register => \|dst_reg\| try self.asmRegister(mir_tag, registerAlias(dst_reg, abi_size)),
	3762	.ptr_stack_offset, .stack_offset => \|off\| {
	3763	if (off > math.maxInt(i32)) {
	3764	return self.fail("stack offset too large", .{});
	3765	}
	3766	if (abi_size > 8) {
	3767	return self.fail("TODO implement {} for stack dst with large ABI", .{mir_tag});
	3768	}
	3769
	3770	try self.asmMemory(mir_tag, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
	3771	.base = .rbp,
	3772	.disp = -off,
	3773	}));
	3774	},
	3775	.memory, .linker_load => {
	3776	const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
	3777	const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
	3778	defer self.register_manager.unlockReg(addr_reg_lock);
	3779
	3780	try self.loadMemPtrIntoRegister(addr_reg, Type.usize, dst_mcv);
	3781
	3782	// To get the actual address of the value we want to modify we have to go through the GOT
	3783	try self.asmRegisterMemory(
	3784	.mov,
	3785	addr_reg,
	3786	Memory.sib(.qword, .{ .base = addr_reg }),
	3787	);
	3788
	3789	try self.asmMemory(
	3790	mir_tag,
	3791	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = addr_reg }),
	3792	);
	3793	},
	3794	}
	3795	}
	3796
3215	3797	/// Clobbers .rcx for non-immediate shift value.
3216	3798	fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shift: MCValue) !void {
3217	3799	assert(reg.to64() != .rcx);
...	...	@@ -3233,8 +3815,7 @@ fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shi
3233	3815	},
3234	3816	else => {},
3235	3817	}
3236		assert(self.register_manager.isRegFree(.rcx));
3237		try self.register_manager.getReg(.rcx, null);
	3818	self.register_manager.getRegAssumeFree(.rcx, null);
3238	3819	try self.genSetReg(Type.u8, .rcx, shift);
3239	3820	}
3240	3821
...	...	@@ -3274,8 +3855,7 @@ fn genShiftBinOp(
3274	3855	};
3275	3856	defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
3276	3857
3277		assert(self.register_manager.isRegFree(.rcx));
3278		try self.register_manager.getReg(.rcx, null);
	3858	self.register_manager.getRegAssumeFree(.rcx, null);
3279	3859	const rcx_lock = self.register_manager.lockRegAssumeUnused(.rcx);
3280	3860	defer self.register_manager.unlockReg(rcx_lock);
3281	3861
...	...	@@ -3325,10 +3905,10 @@ fn genMulDivBinOp(
3325	3905	rhs: MCValue,
3326	3906	) !MCValue {
3327	3907	if (ty.zigTypeTag() == .Vector or ty.zigTypeTag() == .Float) {
3328		return self.fail("TODO implement genBinOp for {}", .{ty.fmtDebug()});
	3908	return self.fail("TODO implement genMulDivBinOp for {}", .{ty.fmtDebug()});
3329	3909	}
3330	3910	if (ty.abiSize(self.target.*) > 8) {
3331		return self.fail("TODO implement genBinOp for {}", .{ty.fmtDebug()});
	3911	return self.fail("TODO implement genMulDivBinOp for {}", .{ty.fmtDebug()});
3332	3912	}
3333	3913	if (tag == .div_float) {
3334	3914	return self.fail("TODO implement genMulDivBinOp for div_float", .{});
...	...	@@ -3486,10 +4066,21 @@ fn genBinOp(
3486	4066	const lhs_ty = self.air.typeOf(lhs_air);
3487	4067	const rhs_ty = self.air.typeOf(rhs_air);
3488	4068	if (lhs_ty.zigTypeTag() == .Vector) {
3489		return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmtDebug()});
	4069	return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmt(self.bin_file.options.module.?)});
3490	4070	}
3491	4071	if (lhs_ty.abiSize(self.target.*) > 8) {
3492		return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmtDebug()});
	4072	return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmt(self.bin_file.options.module.?)});
	4073	}
	4074
	4075	switch (lhs) {
	4076	.immediate => \|imm\| switch (imm) {
	4077	0 => switch (tag) {
	4078	.sub, .subwrap => return self.genUnOp(maybe_inst, .neg, rhs_air),
	4079	else => {},
	4080	},
	4081	else => {},
	4082	},
	4083	else => {},
3493	4084	}
3494	4085
3495	4086	const is_commutative: bool = switch (tag) {
...	...	@@ -3500,6 +4091,8 @@ fn genBinOp(
3500	4091	.bool_and,
3501	4092	.bit_and,
3502	4093	.xor,
	4094	.min,
	4095	.max,
3503	4096	=> true,
3504	4097
3505	4098	else => false,
...	...	@@ -3512,7 +4105,7 @@ fn genBinOp(
3512	4105	defer if (lhs_lock) \|lock\| self.register_manager.unlockReg(lock);
3513	4106
3514	4107	const rhs_lock: ?RegisterLock = switch (rhs) {
3515		.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
	4108	.register => \|reg\| self.register_manager.lockReg(reg),
3516	4109	else => null,
3517	4110	};
3518	4111	defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
...	...	@@ -3520,10 +4113,10 @@ fn genBinOp(
3520	4113	var flipped: bool = false;
3521	4114	const dst_mcv: MCValue = blk: {
3522	4115	if (maybe_inst) \|inst\| {
3523		if (self.reuseOperand(inst, lhs_air, 0, lhs) and lhs.isRegister()) {
	4116	if (lhs.isRegister() and self.reuseOperand(inst, lhs_air, 0, lhs)) {
3524	4117	break :blk lhs;
3525	4118	}
3526		if (is_commutative and self.reuseOperand(inst, rhs_air, 1, rhs) and rhs.isRegister()) {
	4119	if (is_commutative and rhs.isRegister() and self.reuseOperand(inst, rhs_air, 1, rhs)) {
3527	4120	flipped = true;
3528	4121	break :blk rhs;
3529	4122	}
...	...	@@ -3551,11 +4144,56 @@ fn genBinOp(
3551	4144	switch (tag) {
3552	4145	.add,
3553	4146	.addwrap,
3554		=> try self.genBinOpMir(.add, lhs_ty, dst_mcv, src_mcv),
	4147	=> try self.genBinOpMir(switch (lhs_ty.tag()) {
	4148	else => .add,
	4149	.f32 => .addss,
	4150	.f64 => .addsd,
	4151	}, lhs_ty, dst_mcv, src_mcv),
3555	4152
3556	4153	.sub,
3557	4154	.subwrap,
3558		=> try self.genBinOpMir(.sub, lhs_ty, dst_mcv, src_mcv),
	4155	=> try self.genBinOpMir(switch (lhs_ty.tag()) {
	4156	else => .sub,
	4157	.f32 => .subss,
	4158	.f64 => .subsd,
	4159	}, lhs_ty, dst_mcv, src_mcv),
	4160
	4161	.mul => try self.genBinOpMir(switch (lhs_ty.tag()) {
	4162	.f32 => .mulss,
	4163	.f64 => .mulsd,
	4164	else => return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?) }),
	4165	}, lhs_ty, dst_mcv, src_mcv),
	4166
	4167	.div_float,
	4168	.div_exact,
	4169	=> try self.genBinOpMir(switch (lhs_ty.tag()) {
	4170	.f32 => .divss,
	4171	.f64 => .divsd,
	4172	else => return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?) }),
	4173	}, lhs_ty, dst_mcv, src_mcv),
	4174
	4175	.div_trunc,
	4176	.div_floor,
	4177	=> {
	4178	try self.genBinOpMir(switch (lhs_ty.tag()) {
	4179	.f32 => .divss,
	4180	.f64 => .divsd,
	4181	else => return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?) }),
	4182	}, lhs_ty, dst_mcv, src_mcv);
	4183	if (Target.x86.featureSetHas(self.target.cpu.features, .sse4_1)) {
	4184	const abi_size = @intCast(u32, lhs_ty.abiSize(self.target.*));
	4185	const dst_alias = registerAlias(dst_mcv.register, abi_size);
	4186	try self.asmRegisterRegisterImmediate(switch (lhs_ty.tag()) {
	4187	.f32 => .roundss,
	4188	.f64 => .roundsd,
	4189	else => unreachable,
	4190	}, dst_alias, dst_alias, Immediate.u(switch (tag) {
	4191	.div_trunc => 0b1_0_11,
	4192	.div_floor => 0b1_0_01,
	4193	else => unreachable,
	4194	}));
	4195	} else return self.fail("TODO implement round without sse4_1", .{});
	4196	},
3559	4197
3560	4198	.ptr_add,
3561	4199	.ptr_sub,
...	...	@@ -3580,6 +4218,113 @@ fn genBinOp(
3580	4218
3581	4219	.xor => try self.genBinOpMir(.xor, lhs_ty, dst_mcv, src_mcv),
3582	4220
	4221	.min,
	4222	.max,
	4223	=> switch (lhs_ty.zigTypeTag()) {
	4224	.Int => {
	4225	const mat_src_mcv = switch (src_mcv) {
	4226	.immediate => MCValue{ .register = try self.copyToTmpRegister(rhs_ty, src_mcv) },
	4227	else => src_mcv,
	4228	};
	4229	const mat_mcv_lock = switch (mat_src_mcv) {
	4230	.register => \|reg\| self.register_manager.lockReg(reg),
	4231	else => null,
	4232	};
	4233	defer if (mat_mcv_lock) \|lock\| self.register_manager.unlockReg(lock);
	4234
	4235	try self.genBinOpMir(.cmp, lhs_ty, dst_mcv, mat_src_mcv);
	4236
	4237	const int_info = lhs_ty.intInfo(self.target.*);
	4238	const cc: Condition = switch (int_info.signedness) {
	4239	.unsigned => switch (tag) {
	4240	.min => .a,
	4241	.max => .b,
	4242	else => unreachable,
	4243	},
	4244	.signed => switch (tag) {
	4245	.min => .g,
	4246	.max => .l,
	4247	else => unreachable,
	4248	},
	4249	};
	4250
	4251	const abi_size = @intCast(u32, lhs_ty.abiSize(self.target.*));
	4252	switch (dst_mcv) {
	4253	.none,
	4254	.undef,
	4255	.dead,
	4256	.unreach,
	4257	.immediate,
	4258	.eflags,
	4259	.register_overflow,
	4260	.stack_offset,
	4261	.ptr_stack_offset,
	4262	.memory,
	4263	.linker_load,
	4264	=> unreachable,
	4265	.register => \|dst_reg\| switch (mat_src_mcv) {
	4266	.none,
	4267	.undef,
	4268	.dead,
	4269	.unreach,
	4270	.immediate,
	4271	.eflags,
	4272	.register_overflow,
	4273	.ptr_stack_offset,
	4274	=> unreachable,
	4275	.register => \|src_reg\| try self.asmCmovccRegisterRegister(
	4276	registerAlias(dst_reg, abi_size),
	4277	registerAlias(src_reg, abi_size),
	4278	cc,
	4279	),
	4280	.stack_offset => \|off\| try self.asmCmovccRegisterMemory(
	4281	registerAlias(dst_reg, abi_size),
	4282	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
	4283	.base = .rbp,
	4284	.disp = -off,
	4285	}),
	4286	cc,
	4287	),
	4288	.memory, .linker_load => {
	4289	const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
	4290	const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
	4291	defer self.register_manager.unlockReg(addr_reg_lock);
	4292
	4293	try self.loadMemPtrIntoRegister(addr_reg, Type.usize, dst_mcv);
	4294
	4295	// To get the actual address of the value we want to modify we
	4296	// we have to go through the GOT
	4297	try self.asmRegisterMemory(
	4298	.mov,
	4299	addr_reg,
	4300	Memory.sib(.qword, .{ .base = addr_reg }),
	4301	);
	4302
	4303	try self.asmCmovccRegisterMemory(
	4304	registerAlias(dst_reg, abi_size),
	4305	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = addr_reg }),
	4306	cc,
	4307	);
	4308	},
	4309	},
	4310	}
	4311	},
	4312	.Float => try self.genBinOpMir(switch (lhs_ty.tag()) {
	4313	.f32 => switch (tag) {
	4314	.min => .minss,
	4315	.max => .maxss,
	4316	else => unreachable,
	4317	},
	4318	.f64 => switch (tag) {
	4319	.min => .minsd,
	4320	.max => .maxsd,
	4321	else => unreachable,
	4322	},
	4323	else => return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?) }),
	4324	}, lhs_ty, dst_mcv, src_mcv),
	4325	else => return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?) }),
	4326	},
	4327
3583	4328	else => unreachable,
3584	4329	}
3585	4330	return dst_mcv;
...	...	@@ -3609,29 +4354,7 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
3609	4354	.register => \|src_reg\| switch (dst_ty.zigTypeTag()) {
3610	4355	.Float => {
3611	4356	if (intrinsicsAllowed(self.target.*, dst_ty)) {
3612		const actual_tag: Mir.Inst.Tag = switch (dst_ty.tag()) {
3613		.f32 => switch (mir_tag) {
3614		.add => .addss,
3615		.cmp => .ucomiss,
3616		else => return self.fail(
3617		"TODO genBinOpMir for f32 register-register with MIR tag {}",
3618		.{mir_tag},
3619		),
3620		},
3621		.f64 => switch (mir_tag) {
3622		.add => .addsd,
3623		.cmp => .ucomisd,
3624		else => return self.fail(
3625		"TODO genBinOpMir for f64 register-register with MIR tag {}",
3626		.{mir_tag},
3627		),
3628		},
3629		else => return self.fail(
3630		"TODO genBinOpMir for float register-register and type {}",
3631		.{dst_ty.fmtDebug()},
3632		),
3633		};
3634		return self.asmRegisterRegister(actual_tag, dst_reg.to128(), src_reg.to128());
	4357	return self.asmRegisterRegister(mir_tag, dst_reg.to128(), src_reg.to128());
3635	4358	}
3636	4359
3637	4360	return self.fail("TODO genBinOpMir for float register-register and no intrinsics", .{});
...	...	@@ -3643,16 +4366,15 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
3643	4366	),
3644	4367	},
3645	4368	.immediate => \|imm\| {
3646		switch (abi_size) {
3647		0 => unreachable,
3648		1...4 => {
	4369	switch (self.regBitSize(dst_ty)) {
	4370	8, 16, 32 => {
3649	4371	try self.asmRegisterImmediate(
3650	4372	mir_tag,
3651	4373	registerAlias(dst_reg, abi_size),
3652	4374	Immediate.u(@intCast(u32, imm)),
3653	4375	);
3654	4376	},
3655		5...8 => {
	4377	64 => {
3656	4378	if (math.cast(i32, @bitCast(i64, imm))) \|small\| {
3657	4379	try self.asmRegisterImmediate(
3658	4380	mir_tag,
...	...	@@ -3660,13 +4382,10 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
3660	4382	Immediate.s(small),
3661	4383	);
3662	4384	} else {
3663		const tmp_reg = try self.register_manager.allocReg(null, gp);
3664		const tmp_alias = registerAlias(tmp_reg, abi_size);
3665		try self.asmRegisterImmediate(.mov, tmp_alias, Immediate.u(imm));
3666	4385	try self.asmRegisterRegister(
3667	4386	mir_tag,
3668	4387	registerAlias(dst_reg, abi_size),
3669		tmp_alias,
	4388	registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size),
3670	4389	);
3671	4390	}
3672	4391	},
...	...	@@ -3716,11 +4435,43 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
3716	4435	}), registerAlias(src_reg, abi_size));
3717	4436	},
3718	4437	.immediate => \|imm\| {
3719		// TODO
3720		try self.asmMemoryImmediate(mir_tag, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
3721		.base = .rbp,
3722		.disp = -off,
3723		}), Immediate.u(@intCast(u32, imm)));
	4438	switch (self.regBitSize(dst_ty)) {
	4439	8, 16, 32 => {
	4440	try self.asmMemoryImmediate(
	4441	mir_tag,
	4442	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
	4443	.base = .rbp,
	4444	.disp = -off,
	4445	}),
	4446	if (math.cast(i32, @bitCast(i64, imm))) \|small\|
	4447	Immediate.s(small)
	4448	else
	4449	Immediate.u(@intCast(u32, imm)),
	4450	);
	4451	},
	4452	64 => {
	4453	if (math.cast(i32, @bitCast(i64, imm))) \|small\| {
	4454	try self.asmMemoryImmediate(
	4455	mir_tag,
	4456	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
	4457	.base = .rbp,
	4458	.disp = -off,
	4459	}),
	4460	Immediate.s(small),
	4461	);
	4462	} else {
	4463	try self.asmMemoryRegister(
	4464	mir_tag,
	4465	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
	4466	.base = .rbp,
	4467	.disp = -off,
	4468	}),
	4469	registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size),
	4470	);
	4471	}
	4472	},
	4473	else => return self.fail("TODO getBinOpMir implement large immediate ABI", .{}),
	4474	}
3724	4475	},
3725	4476	.memory,
3726	4477	.stack_offset,
...	...	@@ -3952,18 +4703,33 @@ fn airBreakpoint(self: *Self) !void {
3952	4703	}
3953	4704
3954	4705	fn airRetAddr(self: *Self, inst: Air.Inst.Index) !void {
3955		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airRetAddr for x86_64", .{});
	4706	const result = if (self.liveness.isUnused(inst)) .dead else result: {
	4707	const dst_mcv = try self.allocRegOrMem(inst, true);
	4708	try self.setRegOrMem(Type.usize, dst_mcv, .{
	4709	.stack_offset = -@as(i32, @divExact(self.target.cpu.arch.ptrBitWidth(), 8)),
	4710	});
	4711	break :result dst_mcv;
	4712	};
3956	4713	return self.finishAir(inst, result, .{ .none, .none, .none });
3957	4714	}
3958	4715
3959	4716	fn airFrameAddress(self: *Self, inst: Air.Inst.Index) !void {
3960		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airFrameAddress for x86_64", .{});
	4717	const result = if (self.liveness.isUnused(inst)) .dead else result: {
	4718	const dst_mcv = try self.allocRegOrMem(inst, true);
	4719	try self.setRegOrMem(Type.usize, dst_mcv, .{ .register = .rbp });
	4720	break :result dst_mcv;
	4721	};
3961	4722	return self.finishAir(inst, result, .{ .none, .none, .none });
3962	4723	}
3963	4724
3964		fn airFence(self: *Self) !void {
3965		return self.fail("TODO implement fence() for {}", .{self.target.cpu.arch});
3966		//return self.finishAirBookkeeping();
	4725	fn airFence(self: *Self, inst: Air.Inst.Index) !void {
	4726	const order = self.air.instructions.items(.data)[inst].fence;
	4727	switch (order) {
	4728	.Unordered, .Monotonic => unreachable,
	4729	.Acquire, .Release, .AcqRel => {},
	4730	.SeqCst => try self.asmOpOnly(.mfence),
	4731	}
	4732	return self.finishAirBookkeeping();
3967	4733	}
3968	4734
3969	4735	fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !void {
...	...	@@ -3984,11 +4750,40 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
3984	4750	defer info.deinit(self);
3985	4751
3986	4752	try self.spillEflagsIfOccupied();
	4753	try self.spillRegisters(abi.getCallerPreservedRegs(self.target.*));
3987	4754
3988		for (abi.getCallerPreservedRegs(self.target.*)) \|reg\| {
3989		try self.register_manager.getReg(reg, null);
	4755	// set stack arguments first because this can clobber registers
	4756	// also clobber spill arguments as we go
	4757	if (info.return_value == .stack_offset) {
	4758	try self.spillRegisters(&.{abi.getCAbiIntParamRegs(self.target.*)[0]});
	4759	}
	4760	for (args, info.args) \|arg, mc_arg\| {
	4761	const arg_ty = self.air.typeOf(arg);
	4762	const arg_mcv = try self.resolveInst(arg);
	4763	// Here we do not use setRegOrMem even though the logic is similar, because
	4764	// the function call will move the stack pointer, so the offsets are different.
	4765	switch (mc_arg) {
	4766	.none => {},
	4767	.register => \|reg\| try self.spillRegisters(&.{reg}),
	4768	.stack_offset => \|off\| {
	4769	// TODO rewrite using `genSetStack`
	4770	try self.genSetStackArg(arg_ty, off, arg_mcv);
	4771	},
	4772	.ptr_stack_offset => {
	4773	return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{});
	4774	},
	4775	.undef => unreachable,
	4776	.immediate => unreachable,
	4777	.unreach => unreachable,
	4778	.dead => unreachable,
	4779	.memory => unreachable,
	4780	.linker_load => unreachable,
	4781	.eflags => unreachable,
	4782	.register_overflow => unreachable,
	4783	}
3990	4784	}
3991	4785
	4786	// now we are free to set register arguments
3992	4787	const ret_reg_lock: ?RegisterLock = blk: {
3993	4788	if (info.return_value == .stack_offset) {
3994	4789	const ret_ty = fn_ty.fnReturnType();
...	...	@@ -3998,7 +4793,6 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
3998	4793	log.debug("airCall: return value on stack at offset {}", .{stack_offset});
3999	4794
4000	4795	const ret_reg = abi.getCAbiIntParamRegs(self.target.*)[0];
4001		try self.register_manager.getReg(ret_reg, null);
4002	4796	try self.genSetReg(Type.usize, ret_reg, .{ .ptr_stack_offset = stack_offset });
4003	4797	const ret_reg_lock = self.register_manager.lockRegAssumeUnused(ret_reg);
4004	4798
...	...	@@ -4010,25 +4804,12 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
4010	4804	};
4011	4805	defer if (ret_reg_lock) \|lock\| self.register_manager.unlockReg(lock);
4012	4806
4013		for (args, info.args) \|arg, info_arg\| {
4014		const mc_arg = info_arg;
	4807	for (args, info.args) \|arg, mc_arg\| {
4015	4808	const arg_ty = self.air.typeOf(arg);
4016	4809	const arg_mcv = try self.resolveInst(arg);
4017		// Here we do not use setRegOrMem even though the logic is similar, because
4018		// the function call will move the stack pointer, so the offsets are different.
4019	4810	switch (mc_arg) {
4020		.none => continue,
4021		.register => \|reg\| {
4022		try self.register_manager.getReg(reg, null);
4023		try self.genSetReg(arg_ty, reg, arg_mcv);
4024		},
4025		.stack_offset => \|off\| {
4026		// TODO rewrite using `genSetStack`
4027		try self.genSetStackArg(arg_ty, off, arg_mcv);
4028		},
4029		.ptr_stack_offset => {
4030		return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{});
4031		},
	4811	.none, .stack_offset, .ptr_stack_offset => {},
	4812	.register => \|reg\| try self.genSetReg(arg_ty, reg, arg_mcv),
4032	4813	.undef => unreachable,
4033	4814	.immediate => unreachable,
4034	4815	.unreach => unreachable,
...	...	@@ -4293,7 +5074,11 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
4293	5074	};
4294	5075	defer if (src_lock) \|lock\| self.register_manager.unlockReg(lock);
4295	5076
4296		try self.genBinOpMir(.cmp, ty, dst_mcv, src_mcv);
	5077	try self.genBinOpMir(switch (ty.tag()) {
	5078	else => .cmp,
	5079	.f32 => .ucomiss,
	5080	.f64 => .ucomisd,
	5081	}, ty, dst_mcv, src_mcv);
4297	5082
4298	5083	break :result switch (signedness) {
4299	5084	.signed => MCValue{ .eflags = Condition.fromCompareOperatorSigned(op) },
...	...	@@ -4510,25 +5295,113 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
4510	5295	return self.finishAir(inst, .unreach, .{ .none, .none, .none });
4511	5296	}
4512	5297
4513		fn isNull(self: *Self, inst: Air.Inst.Index, ty: Type, operand: MCValue) !MCValue {
	5298	fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MCValue {
	5299	switch (opt_mcv) {
	5300	.register_overflow => \|ro\| return .{ .eflags = ro.eflags.negate() },
	5301	else => {},
	5302	}
	5303
4514	5304	try self.spillEflagsIfOccupied();
4515	5305	self.eflags_inst = inst;
4516	5306
4517		const cmp_ty: Type = if (!ty.isPtrLikeOptional()) blk: {
4518		var buf: Type.Payload.ElemType = undefined;
4519		const payload_ty = ty.optionalChild(&buf);
4520		break :blk if (payload_ty.hasRuntimeBitsIgnoreComptime()) Type.bool else ty;
4521		} else ty;
	5307	var pl_buf: Type.Payload.ElemType = undefined;
	5308	const pl_ty = opt_ty.optionalChild(&pl_buf);
	5309
	5310	var ptr_buf: Type.SlicePtrFieldTypeBuffer = undefined;
	5311	const some_info: struct { off: i32, ty: Type } = if (opt_ty.optionalReprIsPayload())
	5312	.{ .off = 0, .ty = if (pl_ty.isSlice()) pl_ty.slicePtrFieldType(&ptr_buf) else pl_ty }
	5313	else
	5314	.{ .off = @intCast(i32, pl_ty.abiSize(self.target.*)), .ty = Type.bool };
	5315
	5316	switch (opt_mcv) {
	5317	.none,
	5318	.unreach,
	5319	.dead,
	5320	.undef,
	5321	.immediate,
	5322	.register_overflow,
	5323	.ptr_stack_offset,
	5324	.eflags,
	5325	=> unreachable,
	5326
	5327	.register => \|opt_reg\| {
	5328	if (some_info.off == 0) {
	5329	const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
	5330	const alias_reg = registerAlias(opt_reg, some_abi_size);
	5331	assert(some_abi_size * 8 == alias_reg.bitSize());
	5332	try self.asmRegisterRegister(.@"test", alias_reg, alias_reg);
	5333	return .{ .eflags = .z };
	5334	}
	5335	assert(some_info.ty.tag() == .bool);
	5336	const opt_abi_size = @intCast(u32, opt_ty.abiSize(self.target.*));
	5337	try self.asmRegisterImmediate(
	5338	.bt,
	5339	registerAlias(opt_reg, opt_abi_size),
	5340	Immediate.u(@intCast(u6, some_info.off * 8)),
	5341	);
	5342	return .{ .eflags = .nc };
	5343	},
	5344
	5345	.memory, .linker_load => {
	5346	const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
	5347	const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
	5348	defer self.register_manager.unlockReg(addr_reg_lock);
	5349
	5350	try self.loadMemPtrIntoRegister(addr_reg, Type.usize, opt_mcv);
	5351
	5352	// To get the actual address of the value we want to modify we have to go through the GOT
	5353	try self.asmRegisterMemory(
	5354	.mov,
	5355	addr_reg,
	5356	Memory.sib(.qword, .{ .base = addr_reg }),
	5357	);
4522	5358
4523		try self.genBinOpMir(.cmp, cmp_ty, operand, MCValue{ .immediate = 0 });
	5359	const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
	5360	try self.asmMemoryImmediate(.cmp, Memory.sib(
	5361	Memory.PtrSize.fromSize(some_abi_size),
	5362	.{ .base = addr_reg, .disp = some_info.off },
	5363	), Immediate.u(0));
	5364	return .{ .eflags = .e };
	5365	},
4524	5366
4525		return MCValue{ .eflags = .e };
	5367	.stack_offset => \|off\| {
	5368	const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
	5369	try self.asmMemoryImmediate(.cmp, Memory.sib(
	5370	Memory.PtrSize.fromSize(some_abi_size),
	5371	.{ .base = .rbp, .disp = some_info.off - off },
	5372	), Immediate.u(0));
	5373	return .{ .eflags = .e };
	5374	},
	5375	}
4526	5376	}
4527	5377
4528		fn isNonNull(self: *Self, inst: Air.Inst.Index, ty: Type, operand: MCValue) !MCValue {
4529		const is_null_res = try self.isNull(inst, ty, operand);
4530		assert(is_null_res.eflags == .e);
4531		return MCValue{ .eflags = is_null_res.eflags.negate() };
	5378	fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue) !MCValue {
	5379	try self.spillEflagsIfOccupied();
	5380	self.eflags_inst = inst;
	5381
	5382	const opt_ty = ptr_ty.childType();
	5383	var pl_buf: Type.Payload.ElemType = undefined;
	5384	const pl_ty = opt_ty.optionalChild(&pl_buf);
	5385
	5386	var ptr_buf: Type.SlicePtrFieldTypeBuffer = undefined;
	5387	const some_info: struct { off: i32, ty: Type } = if (opt_ty.optionalReprIsPayload())
	5388	.{ .off = 0, .ty = if (pl_ty.isSlice()) pl_ty.slicePtrFieldType(&ptr_buf) else pl_ty }
	5389	else
	5390	.{ .off = @intCast(i32, pl_ty.abiSize(self.target.*)), .ty = Type.bool };
	5391
	5392	const ptr_reg = switch (ptr_mcv) {
	5393	.register => \|reg\| reg,
	5394	else => try self.copyToTmpRegister(ptr_ty, ptr_mcv),
	5395	};
	5396	const ptr_lock = self.register_manager.lockReg(ptr_reg);
	5397	defer if (ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
	5398
	5399	const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
	5400	try self.asmMemoryImmediate(.cmp, Memory.sib(
	5401	Memory.PtrSize.fromSize(some_abi_size),
	5402	.{ .base = ptr_reg, .disp = some_info.off },
	5403	), Immediate.u(0));
	5404	return .{ .eflags = .e };
4532	5405	}
4533	5406
4534	5407	fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) !MCValue {
...	...	@@ -4550,8 +5423,13 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) !
4550	5423	try self.genBinOpMir(.cmp, Type.anyerror, .{ .stack_offset = offset }, .{ .immediate = 0 });
4551	5424	},
4552	5425	.register => \|reg\| {
4553		const maybe_lock = self.register_manager.lockReg(reg);
4554		defer if (maybe_lock) \|lock\| self.register_manager.unlockReg(lock);
	5426	self.register_manager.getRegAssumeFree(.rcx, null);
	5427	const rcx_lock = if (err_off > 0) self.register_manager.lockRegAssumeUnused(.rcx) else null;
	5428	defer if (rcx_lock) \|lock\| self.register_manager.unlockReg(lock);
	5429
	5430	const eu_lock = self.register_manager.lockReg(reg);
	5431	defer if (eu_lock) \|lock\| self.register_manager.unlockReg(lock);
	5432
4555	5433	const tmp_reg = try self.copyToTmpRegister(ty, operand);
4556	5434	if (err_off > 0) {
4557	5435	const shift = @intCast(u6, err_off * 8);
...	...	@@ -4594,29 +5472,11 @@ fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {
4594	5472
4595	5473	fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {
4596	5474	const un_op = self.air.instructions.items(.data)[inst].un_op;
4597
4598		if (self.liveness.isUnused(inst)) {
4599		return self.finishAir(inst, .dead, .{ un_op, .none, .none });
4600		}
4601
4602		const operand_ptr = try self.resolveInst(un_op);
4603		const operand_ptr_lock: ?RegisterLock = switch (operand_ptr) {
4604		.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
4605		else => null,
	5475	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	5476	const operand = try self.resolveInst(un_op);
	5477	const ty = self.air.typeOf(un_op);
	5478	break :result try self.isNullPtr(inst, ty, operand);
4606	5479	};
4607		defer if (operand_ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
4608
4609		const ptr_ty = self.air.typeOf(un_op);
4610		const elem_ty = ptr_ty.childType();
4611		const operand = if (elem_ty.isPtrLikeOptional() and self.reuseOperand(inst, un_op, 0, operand_ptr))
4612		// The MCValue that holds the pointer can be re-used as the value.
4613		operand_ptr
4614		else
4615		try self.allocTempRegOrMem(elem_ty, true);
4616		try self.load(operand, operand_ptr, ptr_ty);
4617
4618		const result = try self.isNull(inst, elem_ty, operand);
4619
4620	5480	return self.finishAir(inst, result, .{ un_op, .none, .none });
4621	5481	}
4622	5482
...	...	@@ -4625,36 +5485,24 @@ fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {
4625	5485	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
4626	5486	const operand = try self.resolveInst(un_op);
4627	5487	const ty = self.air.typeOf(un_op);
4628		break :result try self.isNonNull(inst, ty, operand);
	5488	break :result switch (try self.isNull(inst, ty, operand)) {
	5489	.eflags => \|cc\| .{ .eflags = cc.negate() },
	5490	else => unreachable,
	5491	};
4629	5492	};
4630	5493	return self.finishAir(inst, result, .{ un_op, .none, .none });
4631	5494	}
4632	5495
4633	5496	fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {
4634	5497	const un_op = self.air.instructions.items(.data)[inst].un_op;
4635
4636		if (self.liveness.isUnused(inst)) {
4637		return self.finishAir(inst, .dead, .{ un_op, .none, .none });
4638		}
4639
4640		const operand_ptr = try self.resolveInst(un_op);
4641		const operand_ptr_lock: ?RegisterLock = switch (operand_ptr) {
4642		.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
4643		else => null,
	5498	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
	5499	const operand = try self.resolveInst(un_op);
	5500	const ty = self.air.typeOf(un_op);
	5501	break :result switch (try self.isNullPtr(inst, ty, operand)) {
	5502	.eflags => \|cc\| .{ .eflags = cc.negate() },
	5503	else => unreachable,
	5504	};
4644	5505	};
4645		defer if (operand_ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
4646
4647		const ptr_ty = self.air.typeOf(un_op);
4648		const elem_ty = ptr_ty.childType();
4649		const operand = if (elem_ty.isPtrLikeOptional() and self.reuseOperand(inst, un_op, 0, operand_ptr))
4650		// The MCValue that holds the pointer can be re-used as the value.
4651		operand_ptr
4652		else
4653		try self.allocTempRegOrMem(elem_ty, true);
4654		try self.load(operand, operand_ptr, ptr_ty);
4655
4656		const result = try self.isNonNull(inst, ptr_ty.elemType(), operand);
4657
4658	5506	return self.finishAir(inst, result, .{ un_op, .none, .none });
4659	5507	}
4660	5508
...	...	@@ -4773,69 +5621,6 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
4773	5621	return self.finishAir(inst, result, .{ .none, .none, .none });
4774	5622	}
4775	5623
4776		fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u32 {
4777		const abi_size = @intCast(u32, ty.abiSize(self.target.*));
4778		switch (condition) {
4779		.none => unreachable,
4780		.undef => unreachable,
4781		.dead, .unreach => unreachable,
4782		.eflags => unreachable,
4783		.register => \|cond_reg\| {
4784		try self.spillEflagsIfOccupied();
4785
4786		const cond_reg_lock = self.register_manager.lockReg(cond_reg);
4787		defer if (cond_reg_lock) \|lock\| self.register_manager.unlockReg(lock);
4788
4789		switch (case) {
4790		.none => unreachable,
4791		.undef => unreachable,
4792		.dead, .unreach => unreachable,
4793		.immediate => \|imm\| try self.asmRegisterImmediate(
4794		.xor,
4795		registerAlias(cond_reg, abi_size),
4796		Immediate.u(imm),
4797		),
4798		.register => \|reg\| try self.asmRegisterRegister(
4799		.xor,
4800		registerAlias(cond_reg, abi_size),
4801		registerAlias(reg, abi_size),
4802		),
4803		.stack_offset => {
4804		if (abi_size <= 8) {
4805		const reg = try self.copyToTmpRegister(ty, case);
4806		return self.genCondSwitchMir(ty, condition, .{ .register = reg });
4807		}
4808
4809		return self.fail("TODO implement switch mir when case is stack offset with abi larger than 8 bytes", .{});
4810		},
4811		else => {
4812		return self.fail("TODO implement switch mir when case is {}", .{case});
4813		},
4814		}
4815
4816		const aliased_reg = registerAlias(cond_reg, abi_size);
4817		try self.asmRegisterRegister(.@"test", aliased_reg, aliased_reg);
4818		return self.asmJccReloc(undefined, .ne);
4819		},
4820		.stack_offset => {
4821		try self.spillEflagsIfOccupied();
4822
4823		if (abi_size <= 8) {
4824		const reg = try self.copyToTmpRegister(ty, condition);
4825		const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
4826		defer self.register_manager.unlockReg(reg_lock);
4827		return self.genCondSwitchMir(ty, .{ .register = reg }, case);
4828		}
4829
4830		return self.fail("TODO implement switch mir when condition is stack offset with abi larger than 8 bytes", .{});
4831		},
4832		else => {
4833		return self.fail("TODO implemenent switch mir when condition is {}", .{condition});
4834		},
4835		}
4836		return 0; // TODO
4837		}
4838
4839	5624	fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
4840	5625	const pl_op = self.air.instructions.items(.data)[inst].pl_op;
4841	5626	const condition = try self.resolveInst(pl_op.operand);
...	...	@@ -4880,8 +5665,10 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
4880	5665	defer self.gpa.free(relocs);
4881	5666
4882	5667	for (items, relocs) \|item, *reloc\| {
	5668	try self.spillEflagsIfOccupied();
4883	5669	const item_mcv = try self.resolveInst(item);
4884		reloc.* = try self.genCondSwitchMir(condition_ty, condition, item_mcv);
	5670	try self.genBinOpMir(.cmp, condition_ty, condition, item_mcv);
	5671	reloc.* = try self.asmJccReloc(undefined, .ne);
4885	5672	}
4886	5673
4887	5674	// Capture the state of register and stack allocation state so that we can revert to it.
...	...	@@ -4976,14 +5763,8 @@ fn canonicaliseBranches(self: Self, parent_branch: Branch, canon_branch: *Bran
4976	5763	if (target_value == .dead)
4977	5764	continue;
4978	5765	// The instruction is only overridden in the else branch.
4979		var i: usize = self.branch_stack.items.len - 1;
4980		while (true) {
4981		i -= 1; // If this overflows, the question is: why wasn't the instruction marked dead?
4982		if (self.branch_stack.items[i].inst_table.get(target_key)) \|mcv\| {
4983		assert(mcv != .dead);
4984		break :blk mcv;
4985		}
4986		}
	5766	// If integer overflows occurs, the question is: why wasn't the instruction marked dead?
	5767	break :blk self.getResolvedInstValue(target_key).?;
4987	5768	};
4988	5769	log.debug("consolidating target_entry {d} {}=>{}", .{ target_key, target_value, canon_mcv });
4989	5770	// TODO make sure the destination stack offset / register does not already have something
...	...	@@ -5000,16 +5781,7 @@ fn canonicaliseBranches(self: Self, parent_branch: Branch, canon_branch: *Bran
5000	5781	log.debug("canon_value = {}", .{canon_value});
5001	5782	if (canon_value == .dead)
5002	5783	continue;
5003		const parent_mcv = blk: {
5004		var i: usize = self.branch_stack.items.len - 1;
5005		while (true) {
5006		i -= 1;
5007		if (self.branch_stack.items[i].inst_table.get(canon_key)) \|mcv\| {
5008		assert(mcv != .dead);
5009		break :blk mcv;
5010		}
5011		}
5012		};
	5784	const parent_mcv = self.getResolvedInstValue(canon_key).?;
5013	5785	log.debug("consolidating canon_entry {d} {}=>{}", .{ canon_key, parent_mcv, canon_value });
5014	5786	// TODO make sure the destination stack offset / register does not already have something
5015	5787	// going on there.
...	...	@@ -5237,6 +6009,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
5237	6009	.dead => unreachable,
5238	6010	.unreach, .none => return,
5239	6011	.undef => {
	6012	if (!self.wantSafety()) return; // The already existing value will do just fine.
5240	6013	if (abi_size <= 8) {
5241	6014	const reg = try self.copyToTmpRegister(ty, mcv);
5242	6015	return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
...	...	@@ -5344,8 +6117,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
5344	6117	.dead => unreachable,
5345	6118	.unreach, .none => return, // Nothing to do.
5346	6119	.undef => {
5347		if (!self.wantSafety())
5348		return; // The already existing value will do just fine.
	6120	if (!self.wantSafety()) return; // The already existing value will do just fine.
5349	6121	// TODO Upgrade this to a memset call when we have that available.
5350	6122	switch (abi_size) {
5351	6123	1, 2, 4 => {
...	...	@@ -5567,19 +6339,14 @@ fn genInlineMemcpy(
5567	6339	null;
5568	6340	defer if (dsbase_lock) \|lock\| self.register_manager.unlockReg(lock);
5569	6341
5570		const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null }, gp);
5571		const dst_addr_reg = regs[0];
5572		const src_addr_reg = regs[1];
5573		const index_reg = regs[2].to64();
5574		const count_reg = regs[3].to64();
5575		const tmp_reg = regs[4].to8();
	6342	try self.spillRegisters(&.{ .rdi, .rsi, .rcx });
5576	6343
5577	6344	switch (dst_ptr) {
5578	6345	.memory, .linker_load => {
5579		try self.loadMemPtrIntoRegister(dst_addr_reg, Type.usize, dst_ptr);
	6346	try self.loadMemPtrIntoRegister(.rdi, Type.usize, dst_ptr);
5580	6347	},
5581	6348	.ptr_stack_offset, .stack_offset => \|off\| {
5582		try self.asmRegisterMemory(.lea, dst_addr_reg.to64(), Memory.sib(.qword, .{
	6349	try self.asmRegisterMemory(.lea, .rdi, Memory.sib(.qword, .{
5583	6350	.base = opts.dest_stack_base orelse .rbp,
5584	6351	.disp = -off,
5585	6352	}));
...	...	@@ -5587,7 +6354,7 @@ fn genInlineMemcpy(
5587	6354	.register => \|reg\| {
5588	6355	try self.asmRegisterRegister(
5589	6356	.mov,
5590		registerAlias(dst_addr_reg, @intCast(u32, @divExact(reg.bitSize(), 8))),
	6357	registerAlias(.rdi, @intCast(u32, @divExact(reg.bitSize(), 8))),
5591	6358	reg,
5592	6359	);
5593	6360	},
...	...	@@ -5598,10 +6365,10 @@ fn genInlineMemcpy(
5598	6365
5599	6366	switch (src_ptr) {
5600	6367	.memory, .linker_load => {
5601		try self.loadMemPtrIntoRegister(src_addr_reg, Type.usize, src_ptr);
	6368	try self.loadMemPtrIntoRegister(.rsi, Type.usize, src_ptr);
5602	6369	},
5603	6370	.ptr_stack_offset, .stack_offset => \|off\| {
5604		try self.asmRegisterMemory(.lea, src_addr_reg.to64(), Memory.sib(.qword, .{
	6371	try self.asmRegisterMemory(.lea, .rsi, Memory.sib(.qword, .{
5605	6372	.base = opts.source_stack_base orelse .rbp,
5606	6373	.disp = -off,
5607	6374	}));
...	...	@@ -5609,7 +6376,7 @@ fn genInlineMemcpy(
5609	6376	.register => \|reg\| {
5610	6377	try self.asmRegisterRegister(
5611	6378	.mov,
5612		registerAlias(src_addr_reg, @intCast(u32, @divExact(reg.bitSize(), 8))),
	6379	registerAlias(.rsi, @intCast(u32, @divExact(reg.bitSize(), 8))),
5613	6380	reg,
5614	6381	);
5615	6382	},
...	...	@@ -5618,37 +6385,12 @@ fn genInlineMemcpy(
5618	6385	},
5619	6386	}
5620	6387
5621		try self.genSetReg(Type.usize, count_reg, len);
5622		try self.asmRegisterImmediate(.mov, index_reg, Immediate.u(0));
5623		const loop_start = try self.addInst(.{
5624		.tag = .cmp,
5625		.ops = .ri_u,
5626		.data = .{ .ri = .{
5627		.r1 = count_reg,
5628		.imm = 0,
5629		} },
	6388	try self.genSetReg(Type.usize, .rcx, len);
	6389	_ = try self.addInst(.{
	6390	.tag = .movs,
	6391	.ops = .string,
	6392	.data = .{ .string = .{ .repeat = .rep, .width = .b } },
5630	6393	});
5631		const loop_reloc = try self.asmJccReloc(undefined, .e);
5632		try self.asmRegisterMemory(.mov, tmp_reg.to8(), Memory.sib(.byte, .{
5633		.base = src_addr_reg,
5634		.scale_index = .{
5635		.scale = 1,
5636		.index = index_reg,
5637		},
5638		.disp = 0,
5639		}));
5640		try self.asmMemoryRegister(.mov, Memory.sib(.byte, .{
5641		.base = dst_addr_reg,
5642		.scale_index = .{
5643		.scale = 1,
5644		.index = index_reg,
5645		},
5646		.disp = 0,
5647		}), tmp_reg.to8());
5648		try self.asmRegisterImmediate(.add, index_reg, Immediate.u(1));
5649		try self.asmRegisterImmediate(.sub, count_reg, Immediate.u(1));
5650		_ = try self.asmJmpReloc(loop_start);
5651		try self.performReloc(loop_reloc);
5652	6394	}
5653	6395
5654	6396	fn genInlineMemset(
...	...	@@ -5658,28 +6400,20 @@ fn genInlineMemset(
5658	6400	len: MCValue,
5659	6401	opts: InlineMemcpyOpts,
5660	6402	) InnerError!void {
5661		const ssbase_lock: ?RegisterLock = if (opts.source_stack_base) \|reg\|
5662		self.register_manager.lockReg(reg)
5663		else
5664		null;
5665		defer if (ssbase_lock) \|reg\| self.register_manager.unlockReg(reg);
5666
5667	6403	const dsbase_lock: ?RegisterLock = if (opts.dest_stack_base) \|reg\|
5668	6404	self.register_manager.lockReg(reg)
5669	6405	else
5670	6406	null;
5671	6407	defer if (dsbase_lock) \|lock\| self.register_manager.unlockReg(lock);
5672	6408
5673		const regs = try self.register_manager.allocRegs(2, .{ null, null }, gp);
5674		const addr_reg = regs[0];
5675		const index_reg = regs[1].to64();
	6409	try self.spillRegisters(&.{ .rdi, .al, .rcx });
5676	6410
5677	6411	switch (dst_ptr) {
5678	6412	.memory, .linker_load => {
5679		try self.loadMemPtrIntoRegister(addr_reg, Type.usize, dst_ptr);
	6413	try self.loadMemPtrIntoRegister(.rdi, Type.usize, dst_ptr);
5680	6414	},
5681	6415	.ptr_stack_offset, .stack_offset => \|off\| {
5682		try self.asmRegisterMemory(.lea, addr_reg.to64(), Memory.sib(.qword, .{
	6416	try self.asmRegisterMemory(.lea, .rdi, Memory.sib(.qword, .{
5683	6417	.base = opts.dest_stack_base orelse .rbp,
5684	6418	.disp = -off,
5685	6419	}));
...	...	@@ -5687,48 +6421,22 @@ fn genInlineMemset(
5687	6421	.register => \|reg\| {
5688	6422	try self.asmRegisterRegister(
5689	6423	.mov,
5690		registerAlias(addr_reg, @intCast(u32, @divExact(reg.bitSize(), 8))),
	6424	registerAlias(.rdi, @intCast(u32, @divExact(reg.bitSize(), 8))),
5691	6425	reg,
5692	6426	);
5693	6427	},
5694	6428	else => {
5695		return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr});
	6429	return self.fail("TODO implement memset for setting stack when dest is {}", .{dst_ptr});
5696	6430	},
5697	6431	}
5698	6432
5699		try self.genSetReg(Type.usize, index_reg, len);
5700		try self.genBinOpMir(.sub, Type.usize, .{ .register = index_reg }, .{ .immediate = 1 });
5701
5702		const loop_start = try self.addInst(.{
5703		.tag = .cmp,
5704		.ops = .ri_s,
5705		.data = .{ .ri = .{
5706		.r1 = index_reg,
5707		.imm = @bitCast(u32, @as(i32, -1)),
5708		} },
	6433	try self.genSetReg(Type.u8, .al, value);
	6434	try self.genSetReg(Type.usize, .rcx, len);
	6435	_ = try self.addInst(.{
	6436	.tag = .stos,
	6437	.ops = .string,
	6438	.data = .{ .string = .{ .repeat = .rep, .width = .b } },
5709	6439	});
5710		const loop_reloc = try self.asmJccReloc(undefined, .e);
5711
5712		switch (value) {
5713		.immediate => \|x\| {
5714		if (x > math.maxInt(i32)) {
5715		return self.fail("TODO inline memset for value immediate larger than 32bits", .{});
5716		}
5717		try self.asmMemoryImmediate(.mov, Memory.sib(.byte, .{
5718		.base = addr_reg,
5719		.scale_index = .{
5720		.scale = 1,
5721		.index = index_reg,
5722		},
5723		.disp = 0,
5724		}), Immediate.u(@intCast(u8, x)));
5725		},
5726		else => return self.fail("TODO inline memset for value of type {}", .{value}),
5727		}
5728
5729		try self.asmRegisterImmediate(.sub, index_reg, Immediate.u(1));
5730		_ = try self.asmJmpReloc(loop_start);
5731		try self.performReloc(loop_reloc);
5732	6440	}
5733	6441
5734	6442	fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void {
...	...	@@ -5748,10 +6456,9 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
5748	6456	},
5749	6457	.unreach, .none => return, // Nothing to do.
5750	6458	.undef => {
5751		if (!self.wantSafety())
5752		return; // The already existing value will do just fine.
	6459	if (!self.wantSafety()) return; // The already existing value will do just fine.
5753	6460	// Write the debug undefined value.
5754		switch (registerAlias(reg, abi_size).bitSize()) {
	6461	switch (self.regBitSize(ty)) {
5755	6462	8 => return self.genSetReg(ty, reg, .{ .immediate = 0xaa }),
5756	6463	16 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaa }),
5757	6464	32 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaa }),
...	...	@@ -5762,27 +6469,27 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
5762	6469	.eflags => \|cc\| {
5763	6470	return self.asmSetccRegister(reg.to8(), cc);
5764	6471	},
5765		.immediate => \|x\| {
5766		if (x == 0) {
	6472	.immediate => \|imm\| {
	6473	if (imm == 0) {
5767	6474	// 32-bit moves zero-extend to 64-bit, so xoring the 32-bit
5768	6475	// register is the fastest way to zero a register.
5769		return self.asmRegisterRegister(.xor, reg.to32(), reg.to32());
5770		}
5771		if (ty.isSignedInt()) {
5772		const signed_x = @bitCast(i64, x);
5773		if (math.minInt(i32) <= signed_x and signed_x <= math.maxInt(i32)) {
5774		return self.asmRegisterImmediate(
5775		.mov,
5776		registerAlias(reg, abi_size),
5777		Immediate.s(@intCast(i32, signed_x)),
5778		);
5779		}
	6476	try self.asmRegisterRegister(.xor, reg.to32(), reg.to32());
	6477	} else if (abi_size > 4 and math.cast(u32, imm) != null) {
	6478	// 32-bit moves zero-extend to 64-bit.
	6479	try self.asmRegisterImmediate(.mov, reg.to32(), Immediate.u(imm));
	6480	} else if (abi_size <= 4 and @bitCast(i64, imm) < 0) {
	6481	try self.asmRegisterImmediate(
	6482	.mov,
	6483	registerAlias(reg, abi_size),
	6484	Immediate.s(@intCast(i32, @bitCast(i64, imm))),
	6485	);
	6486	} else {
	6487	try self.asmRegisterImmediate(
	6488	.mov,
	6489	registerAlias(reg, abi_size),
	6490	Immediate.u(imm),
	6491	);
5780	6492	}
5781		return self.asmRegisterImmediate(
5782		.mov,
5783		registerAlias(reg, abi_size),
5784		Immediate.u(x),
5785		);
5786	6493	},
5787	6494	.register => \|src_reg\| {
5788	6495	// If the registers are the same, nothing to do.
...	...	@@ -5843,10 +6550,11 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
5843	6550	.f64 => .qword,
5844	6551	else => unreachable,
5845	6552	};
5846		return self.asmRegisterMemory(tag, reg.to128(), Memory.sib(ptr_size, .{
5847		.base = base_reg.to64(),
5848		.disp = 0,
5849		}));
	6553	return self.asmRegisterMemory(
	6554	tag,
	6555	reg.to128(),
	6556	Memory.sib(ptr_size, .{ .base = base_reg.to64() }),
	6557	);
5850	6558	}
5851	6559
5852	6560	return self.fail("TODO genSetReg from memory for float with no intrinsics", .{});
...	...	@@ -5856,7 +6564,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
5856	6564	try self.asmRegisterMemory(
5857	6565	.mov,
5858	6566	registerAlias(reg, abi_size),
5859		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64(), .disp = 0 }),
	6567	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
5860	6568	);
5861	6569	},
5862	6570	}
...	...	@@ -5877,10 +6585,11 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
5877	6585	.f64 => .qword,
5878	6586	else => unreachable,
5879	6587	};
5880		return self.asmRegisterMemory(tag, reg.to128(), Memory.sib(ptr_size, .{
5881		.base = base_reg.to64(),
5882		.disp = 0,
5883		}));
	6588	return self.asmRegisterMemory(
	6589	tag,
	6590	reg.to128(),
	6591	Memory.sib(ptr_size, .{ .base = base_reg.to64() }),
	6592	);
5884	6593	}
5885	6594
5886	6595	return self.fail("TODO genSetReg from memory for float with no intrinsics", .{});
...	...	@@ -5916,7 +6625,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
5916	6625	try self.asmRegisterMemory(
5917	6626	.mov,
5918	6627	registerAlias(reg, abi_size),
5919		Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64(), .disp = 0 }),
	6628	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
5920	6629	);
5921	6630	}
5922	6631	}
...	...	@@ -6088,26 +6797,178 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
6088	6797
6089	6798	fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
6090	6799	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
6091		const extra = self.air.extraData(Air.Block, ty_pl.payload);
6092		_ = extra;
6093		return self.fail("TODO implement x86 airCmpxchg", .{});
6094		// return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });
	6800	const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
	6801
	6802	const ptr_ty = self.air.typeOf(extra.ptr);
	6803	const ptr_mcv = try self.resolveInst(extra.ptr);
	6804	const val_ty = self.air.typeOf(extra.expected_value);
	6805
	6806	const exp_mcv = try self.resolveInst(extra.expected_value);
	6807	try self.genSetReg(val_ty, .rax, exp_mcv);
	6808	const rax_lock = self.register_manager.lockRegAssumeUnused(.rax);
	6809	defer self.register_manager.unlockReg(rax_lock);
	6810
	6811	const new_mcv = try self.resolveInst(extra.new_value);
	6812	const new_reg = try self.copyToTmpRegister(val_ty, new_mcv);
	6813	const new_lock = self.register_manager.lockRegAssumeUnused(new_reg);
	6814	defer self.register_manager.unlockReg(new_lock);
	6815
	6816	const val_abi_size = @intCast(u32, val_ty.abiSize(self.target.*));
	6817	const ptr_size = Memory.PtrSize.fromSize(val_abi_size);
	6818	const ptr_mem: Memory = switch (ptr_mcv) {
	6819	.register => \|reg\| Memory.sib(ptr_size, .{ .base = reg }),
	6820	.ptr_stack_offset => \|off\| Memory.sib(ptr_size, .{ .base = .rbp, .disp = -off }),
	6821	else => Memory.sib(ptr_size, .{ .base = try self.copyToTmpRegister(ptr_ty, ptr_mcv) }),
	6822	};
	6823	const mem_lock = if (ptr_mem.base()) \|reg\| self.register_manager.lockReg(reg) else null;
	6824	defer if (mem_lock) \|lock\| self.register_manager.unlockReg(lock);
	6825
	6826	try self.spillEflagsIfOccupied();
	6827	_ = try self.addInst(.{ .tag = .cmpxchg, .ops = .lock_mr_sib, .data = .{ .rx = .{
	6828	.r1 = new_reg,
	6829	.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
	6830	} } });
	6831
	6832	const result: MCValue = result: {
	6833	if (self.liveness.isUnused(inst)) break :result .dead;
	6834
	6835	self.eflags_inst = inst;
	6836	break :result .{ .register_overflow = .{ .reg = .rax, .eflags = .ne } };
	6837	};
	6838	return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });
	6839	}
	6840
	6841	fn atomicOp(
	6842	self: *Self,
	6843	dst_reg: Register,
	6844	ptr_mcv: MCValue,
	6845	val_mcv: MCValue,
	6846	ptr_ty: Type,
	6847	val_ty: Type,
	6848	unused: bool,
	6849	op: ?std.builtin.AtomicRmwOp,
	6850	order: std.builtin.AtomicOrder,
	6851	) InnerError!void {
	6852	const dst_lock = self.register_manager.lockReg(dst_reg);
	6853	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
	6854
	6855	const ptr_lock = switch (ptr_mcv) {
	6856	.register => \|reg\| self.register_manager.lockReg(reg),
	6857	else => null,
	6858	};
	6859	defer if (ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
	6860
	6861	const val_lock = switch (val_mcv) {
	6862	.register => \|reg\| self.register_manager.lockReg(reg),
	6863	else => null,
	6864	};
	6865	defer if (val_lock) \|lock\| self.register_manager.unlockReg(lock);
	6866
	6867	const val_abi_size = @intCast(u32, val_ty.abiSize(self.target.*));
	6868	const ptr_size = Memory.PtrSize.fromSize(val_abi_size);
	6869	const ptr_mem: Memory = switch (ptr_mcv) {
	6870	.register => \|reg\| Memory.sib(ptr_size, .{ .base = reg }),
	6871	.ptr_stack_offset => \|off\| Memory.sib(ptr_size, .{ .base = .rbp, .disp = -off }),
	6872	else => Memory.sib(ptr_size, .{ .base = try self.copyToTmpRegister(ptr_ty, ptr_mcv) }),
	6873	};
	6874	const mem_lock = if (ptr_mem.base()) \|reg\| self.register_manager.lockReg(reg) else null;
	6875	defer if (mem_lock) \|lock\| self.register_manager.unlockReg(lock);
	6876
	6877	try self.genSetReg(val_ty, dst_reg, val_mcv);
	6878
	6879	const need_loop = val_ty.isRuntimeFloat() or if (op) \|rmw\| switch (rmw) {
	6880	.Xchg, .Add, .Sub => false,
	6881	.And, .Or, .Xor => !unused,
	6882	.Nand, .Max, .Min => true,
	6883	} else false;
	6884	if (!need_loop) {
	6885	const tag: Mir.Inst.Tag = if (op) \|rmw\| switch (rmw) {
	6886	.Xchg => if (unused) .mov else .xchg,
	6887	.Add => if (unused) .add else .xadd,
	6888	.Sub => if (unused) .sub else .xadd,
	6889	.And => .@"and",
	6890	.Or => .@"or",
	6891	.Xor => .xor,
	6892	else => unreachable,
	6893	} else switch (order) {
	6894	.Unordered, .Monotonic, .Release, .AcqRel => .mov,
	6895	.Acquire => unreachable,
	6896	.SeqCst => .xchg,
	6897	};
	6898	if (op == std.builtin.AtomicRmwOp.Sub and tag == .xadd) {
	6899	try self.genUnOpMir(.neg, val_ty, .{ .register = dst_reg });
	6900	}
	6901	_ = try self.addInst(.{ .tag = tag, .ops = switch (tag) {
	6902	.mov, .xchg => .mr_sib,
	6903	.xadd, .add, .sub, .@"and", .@"or", .xor => .lock_mr_sib,
	6904	else => unreachable,
	6905	}, .data = .{ .rx = .{
	6906	.r1 = registerAlias(dst_reg, val_abi_size),
	6907	.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
	6908	} } });
	6909	return;
	6910	}
	6911
	6912	return self.fail("TODO implement x86 atomic loop", .{});
6095	6913	}
6096	6914
6097	6915	fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void {
6098		_ = inst;
6099		return self.fail("TODO implement x86 airAtomicRaw", .{});
	6916	const pl_op = self.air.instructions.items(.data)[inst].pl_op;
	6917	const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data;
	6918
	6919	const dst_reg = try self.register_manager.allocReg(inst, gp);
	6920
	6921	const ptr_ty = self.air.typeOf(pl_op.operand);
	6922	const ptr_mcv = try self.resolveInst(pl_op.operand);
	6923
	6924	const val_ty = self.air.typeOf(extra.operand);
	6925	const val_mcv = try self.resolveInst(extra.operand);
	6926
	6927	const unused = self.liveness.isUnused(inst);
	6928	try self.atomicOp(dst_reg, ptr_mcv, val_mcv, ptr_ty, val_ty, unused, extra.op(), extra.ordering());
	6929	const result: MCValue = if (unused) .dead else .{ .register = dst_reg };
	6930	return self.finishAir(inst, result, .{ pl_op.operand, extra.operand, .none });
6100	6931	}
6101	6932
6102	6933	fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {
6103		_ = inst;
6104		return self.fail("TODO implement airAtomicLoad for {}", .{self.target.cpu.arch});
	6934	const atomic_load = self.air.instructions.items(.data)[inst].atomic_load;
	6935
	6936	const result: MCValue = result: {
	6937	if (self.liveness.isUnused(inst)) break :result .dead;
	6938
	6939	const ptr_ty = self.air.typeOf(atomic_load.ptr);
	6940	const ptr_mcv = try self.resolveInst(atomic_load.ptr);
	6941	const ptr_lock = switch (ptr_mcv) {
	6942	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
	6943	else => null,
	6944	};
	6945	defer if (ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
	6946
	6947	const dst_mcv =
	6948	if (self.reuseOperand(inst, atomic_load.ptr, 0, ptr_mcv))
	6949	ptr_mcv
	6950	else
	6951	try self.allocRegOrMem(inst, true);
	6952
	6953	try self.load(dst_mcv, ptr_mcv, ptr_ty);
	6954	break :result dst_mcv;
	6955	};
	6956	return self.finishAir(inst, result, .{ atomic_load.ptr, .none, .none });
6105	6957	}
6106	6958
6107	6959	fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void {
6108		_ = inst;
6109		_ = order;
6110		return self.fail("TODO implement airAtomicStore for {}", .{self.target.cpu.arch});
	6960	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
	6961
	6962	const dst_reg = try self.register_manager.allocReg(null, gp);
	6963
	6964	const ptr_ty = self.air.typeOf(bin_op.lhs);
	6965	const ptr_mcv = try self.resolveInst(bin_op.lhs);
	6966
	6967	const val_ty = self.air.typeOf(bin_op.rhs);
	6968	const val_mcv = try self.resolveInst(bin_op.rhs);
	6969
	6970	try self.atomicOp(dst_reg, ptr_mcv, val_mcv, ptr_ty, val_ty, true, null, order);
	6971	return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });
6111	6972	}
6112	6973
6113	6974	fn airMemset(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -6137,7 +6998,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index) !void {
6137	6998
6138	6999	try self.genInlineMemset(dst_ptr, src_val, len, .{});
6139	7000
6140		return self.finishAir(inst, .none, .{ pl_op.operand, .none, .none });
	7001	return self.finishAir(inst, .none, .{ pl_op.operand, extra.lhs, extra.rhs });
6141	7002	}
6142	7003
6143	7004	fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -6172,10 +7033,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
6172	7033	.linker_load, .memory => {
6173	7034	const reg = try self.register_manager.allocReg(null, gp);
6174	7035	try self.loadMemPtrIntoRegister(reg, src_ty, src_ptr);
6175		try self.asmRegisterMemory(.mov, reg, Memory.sib(.qword, .{
6176		.base = reg,
6177		.disp = 0,
6178		}));
	7036	try self.asmRegisterMemory(.mov, reg, Memory.sib(.qword, .{ .base = reg }));
6179	7037	break :blk MCValue{ .register = reg };
6180	7038	},
6181	7039	else => break :blk src_ptr,
...	...	@@ -6189,7 +7047,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
6189	7047
6190	7048	try self.genInlineMemcpy(dst_ptr, src, len, .{});
6191	7049
6192		return self.finishAir(inst, .none, .{ pl_op.operand, .none, .none });
	7050	return self.finishAir(inst, .none, .{ pl_op.operand, extra.lhs, extra.rhs });
6193	7051	}
6194	7052
6195	7053	fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -6629,7 +7487,10 @@ fn registerAlias(reg: Register, size_bytes: u32) Register {
6629	7487	/// Truncates the value in the register in place.
6630	7488	/// Clobbers any remaining bits.
6631	7489	fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
6632		const int_info = ty.intInfo(self.target.*);
	7490	const int_info = if (ty.isAbiInt()) ty.intInfo(self.target.*) else std.builtin.Type.Int{
	7491	.signedness = .unsigned,
	7492	.bits = @intCast(u16, ty.bitSize(self.target.*)),
	7493	};
6633	7494	const max_reg_bit_width = Register.rax.bitSize();
6634	7495	switch (int_info.signedness) {
6635	7496	.signed => {
...	...	@@ -6650,6 +7511,27 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
6650	7511	}
6651	7512	}
6652	7513
	7514	fn regBitSize(self: *Self, ty: Type) u64 {
	7515	return switch (ty.zigTypeTag()) {
	7516	else => switch (ty.abiSize(self.target.*)) {
	7517	1 => 8,
	7518	2 => 16,
	7519	3...4 => 32,
	7520	5...8 => 64,
	7521	else => unreachable,
	7522	},
	7523	.Float => switch (ty.abiSize(self.target.*)) {
	7524	1...16 => 128,
	7525	17...32 => 256,
	7526	else => unreachable,
	7527	},
	7528	};
	7529	}
	7530
	7531	fn regExtraBits(self: *Self, ty: Type) u64 {
	7532	return self.regBitSize(ty) - ty.bitSize(self.target.*);
	7533	}
	7534
6653	7535	fn intrinsicsAllowed(target: Target, ty: Type) bool {
6654	7536	return switch (ty.tag()) {
6655	7537	.f32,

src/arch/x86_64/Emit.zig+177-35

...	...	@@ -73,6 +73,13 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
73	73	.adc,
74	74	.add,
75	75	.@"and",
	76	.bsf,
	77	.bsr,
	78	.bswap,
	79	.bt,
	80	.btc,
	81	.btr,
	82	.bts,
76	83	.call,
77	84	.cbw,
78	85	.cwde,
...	...	@@ -81,6 +88,7 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
81	88	.cdq,
82	89	.cqo,
83	90	.cmp,
	91	.cmpxchg,
84	92	.div,
85	93	.fisttp,
86	94	.fld,
...	...	@@ -89,35 +97,71 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
89	97	.int3,
90	98	.jmp,
91	99	.lea,
	100	.lfence,
	101	.lzcnt,
	102	.mfence,
92	103	.mov,
	104	.movbe,
93	105	.movzx,
94	106	.mul,
	107	.neg,
95	108	.nop,
	109	.not,
96	110	.@"or",
97	111	.pop,
	112	.popcnt,
98	113	.push,
	114	.rcl,
	115	.rcr,
99	116	.ret,
	117	.rol,
	118	.ror,
100	119	.sal,
101	120	.sar,
102	121	.sbb,
	122	.sfence,
103	123	.shl,
104	124	.shr,
105	125	.sub,
106	126	.syscall,
107	127	.@"test",
	128	.tzcnt,
108	129	.ud2,
	130	.xadd,
	131	.xchg,
109	132	.xor,
110	133
111	134	.addss,
112	135	.cmpss,
	136	.divss,
	137	.maxss,
	138	.minss,
113	139	.movss,
	140	.mulss,
	141	.roundss,
	142	.subss,
114	143	.ucomiss,
115	144	.addsd,
116	145	.cmpsd,
	146	.divsd,
	147	.maxsd,
	148	.minsd,
117	149	.movsd,
	150	.mulsd,
	151	.roundsd,
	152	.subsd,
118	153	.ucomisd,
119	154	=> try emit.mirEncodeGeneric(tag, inst),
120	155
	156	.cmps,
	157	.lods,
	158	.movs,
	159	.scas,
	160	.stos,
	161	=> try emit.mirString(tag, inst),
	162
	163	.cmpxchgb => try emit.mirCmpxchgBytes(inst),
	164
121	165	.jmp_reloc => try emit.mirJmpReloc(inst),
122	166
123	167	.call_extern => try emit.mirCallExtern(inst),
...	...	@@ -171,18 +215,8 @@ fn fixupRelocs(emit: *Emit) InnerError!void {
171	215	}
172	216	}
173	217
174		fn encode(emit: *Emit, mnemonic: Instruction.Mnemonic, ops: struct {
175		op1: Instruction.Operand = .none,
176		op2: Instruction.Operand = .none,
177		op3: Instruction.Operand = .none,
178		op4: Instruction.Operand = .none,
179		}) InnerError!void {
180		const inst = try Instruction.new(mnemonic, .{
181		.op1 = ops.op1,
182		.op2 = ops.op2,
183		.op3 = ops.op3,
184		.op4 = ops.op4,
185		});
	218	fn encode(emit: *Emit, mnemonic: Instruction.Mnemonic, ops: Instruction.Init) InnerError!void {
	219	const inst = try Instruction.new(mnemonic, ops);
186	220	return inst.encode(emit.code.writer());
187	221	}
188	222
...	...	@@ -194,6 +228,20 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
194	228	const ops = emit.mir.instructions.items(.ops)[inst];
195	229	const data = emit.mir.instructions.items(.data)[inst];
196	230
	231	const prefix: Instruction.Prefix = switch (ops) {
	232	.lock_m_sib,
	233	.lock_m_rip,
	234	.lock_mi_u_sib,
	235	.lock_mi_u_rip,
	236	.lock_mi_s_sib,
	237	.lock_mi_s_rip,
	238	.lock_mr_sib,
	239	.lock_mr_rip,
	240	.lock_moffs_rax,
	241	=> .lock,
	242	else => .none,
	243	};
	244
197	245	var op1: Instruction.Operand = .none;
198	246	var op2: Instruction.Operand = .none;
199	247	var op3: Instruction.Operand = .none;
...	...	@@ -232,35 +280,35 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
232	280	op2 = .{ .reg = data.rri.r2 };
233	281	op3 = .{ .imm = imm };
234	282	},
235		.m_sib => {
	283	.m_sib, .lock_m_sib => {
236	284	const msib = emit.mir.extraData(Mir.MemorySib, data.payload).data;
237	285	op1 = .{ .mem = Mir.MemorySib.decode(msib) };
238	286	},
239		.m_rip => {
	287	.m_rip, .lock_m_rip => {
240	288	const mrip = emit.mir.extraData(Mir.MemoryRip, data.payload).data;
241	289	op1 = .{ .mem = Mir.MemoryRip.decode(mrip) };
242	290	},
243		.mi_s_sib, .mi_u_sib => {
	291	.mi_s_sib, .mi_u_sib, .lock_mi_s_sib, .lock_mi_u_sib => {
244	292	const msib = emit.mir.extraData(Mir.MemorySib, data.xi.payload).data;
245	293	const imm = switch (ops) {
246		.mi_s_sib => Immediate.s(@bitCast(i32, data.xi.imm)),
247		.mi_u_sib => Immediate.u(data.xi.imm),
	294	.mi_s_sib, .lock_mi_s_sib => Immediate.s(@bitCast(i32, data.xi.imm)),
	295	.mi_u_sib, .lock_mi_u_sib => Immediate.u(data.xi.imm),
248	296	else => unreachable,
249	297	};
250	298	op1 = .{ .mem = Mir.MemorySib.decode(msib) };
251	299	op2 = .{ .imm = imm };
252	300	},
253		.mi_u_rip, .mi_s_rip => {
	301	.mi_u_rip, .mi_s_rip, .lock_mi_u_rip, .lock_mi_s_rip => {
254	302	const mrip = emit.mir.extraData(Mir.MemoryRip, data.xi.payload).data;
255	303	const imm = switch (ops) {
256		.mi_s_rip => Immediate.s(@bitCast(i32, data.xi.imm)),
257		.mi_u_rip => Immediate.u(data.xi.imm),
	304	.mi_s_rip, .lock_mi_s_rip => Immediate.s(@bitCast(i32, data.xi.imm)),
	305	.mi_u_rip, .lock_mi_u_rip => Immediate.u(data.xi.imm),
258	306	else => unreachable,
259	307	};
260	308	op1 = .{ .mem = Mir.MemoryRip.decode(mrip) };
261	309	op2 = .{ .imm = imm };
262	310	},
263		.rm_sib, .mr_sib => {
	311	.rm_sib, .mr_sib, .lock_mr_sib => {
264	312	const msib = emit.mir.extraData(Mir.MemorySib, data.rx.payload).data;
265	313	const op_r = .{ .reg = data.rx.r1 };
266	314	const op_m = .{ .mem = Mir.MemorySib.decode(msib) };
...	...	@@ -269,23 +317,23 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
269	317	op1 = op_r;
270	318	op2 = op_m;
271	319	},
272		.mr_sib => {
	320	.mr_sib, .lock_mr_sib => {
273	321	op1 = op_m;
274	322	op2 = op_r;
275	323	},
276	324	else => unreachable,
277	325	}
278	326	},
279		.rm_rip, .mr_rip => {
	327	.rm_rip, .mr_rip, .lock_mr_rip => {
280	328	const mrip = emit.mir.extraData(Mir.MemoryRip, data.rx.payload).data;
281	329	const op_r = .{ .reg = data.rx.r1 };
282	330	const op_m = .{ .mem = Mir.MemoryRip.decode(mrip) };
283	331	switch (ops) {
284		.rm_sib => {
	332	.rm_rip => {
285	333	op1 = op_r;
286	334	op2 = op_m;
287	335	},
288		.mr_sib => {
	336	.mr_rip, .lock_mr_rip => {
289	337	op1 = op_m;
290	338	op2 = op_r;
291	339	},
...	...	@@ -299,6 +347,7 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
299	347	}
300	348
301	349	return emit.encode(mnemonic, .{
	350	.prefix = prefix,
302	351	.op1 = op1,
303	352	.op2 = op2,
304	353	.op3 = op3,
...	...	@@ -306,6 +355,61 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
306	355	});
307	356	}
308	357
	358	fn mirString(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
	359	const ops = emit.mir.instructions.items(.ops)[inst];
	360	switch (ops) {
	361	.string => {
	362	const data = emit.mir.instructions.items(.data)[inst].string;
	363	const mnemonic = switch (tag) {
	364	inline .cmps, .lods, .movs, .scas, .stos => \|comptime_tag\| switch (data.width) {
	365	inline else => \|comptime_width\| @field(
	366	Instruction.Mnemonic,
	367	@tagName(comptime_tag) ++ @tagName(comptime_width),
	368	),
	369	},
	370	else => unreachable,
	371	};
	372	return emit.encode(mnemonic, .{ .prefix = switch (data.repeat) {
	373	inline else => \|comptime_repeat\| @field(Instruction.Prefix, @tagName(comptime_repeat)),
	374	} });
	375	},
	376	else => unreachable,
	377	}
	378	}
	379
	380	fn mirCmpxchgBytes(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
	381	const ops = emit.mir.instructions.items(.ops)[inst];
	382	const data = emit.mir.instructions.items(.data)[inst];
	383
	384	var op1: Instruction.Operand = .none;
	385	switch (ops) {
	386	.m_sib, .lock_m_sib => {
	387	const sib = emit.mir.extraData(Mir.MemorySib, data.payload).data;
	388	op1 = .{ .mem = Mir.MemorySib.decode(sib) };
	389	},
	390	.m_rip, .lock_m_rip => {
	391	const rip = emit.mir.extraData(Mir.MemoryRip, data.payload).data;
	392	op1 = .{ .mem = Mir.MemoryRip.decode(rip) };
	393	},
	394	else => unreachable,
	395	}
	396
	397	const mnemonic: Instruction.Mnemonic = switch (op1.mem.bitSize()) {
	398	64 => .cmpxchg8b,
	399	128 => .cmpxchg16b,
	400	else => unreachable,
	401	};
	402
	403	return emit.encode(mnemonic, .{
	404	.prefix = switch (ops) {
	405	.m_sib, .m_rip => .none,
	406	.lock_m_sib, .lock_m_rip => .lock,
	407	else => unreachable,
	408	},
	409	.op1 = op1,
	410	});
	411	}
	412
309	413	fn mirMovMoffs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
310	414	const ops = emit.mir.instructions.items(.ops)[inst];
311	415	const payload = emit.mir.instructions.items(.data)[inst].payload;
...	...	@@ -319,8 +423,13 @@ fn mirMovMoffs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
319	423	.op2 = .{ .mem = Memory.moffs(seg, offset) },
320	424	});
321	425	},
322		.moffs_rax => {
	426	.moffs_rax, .lock_moffs_rax => {
323	427	try emit.encode(.mov, .{
	428	.prefix = switch (ops) {
	429	.moffs_rax => .none,
	430	.lock_moffs_rax => .lock,
	431	else => unreachable,
	432	},
324	433	.op1 = .{ .mem = Memory.moffs(seg, offset) },
325	434	.op2 = .{ .reg = .rax },
326	435	});
...	...	@@ -365,37 +474,70 @@ fn mirMovsx(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
365	474	}
366	475
367	476	fn mnemonicFromConditionCode(comptime basename: []const u8, cc: bits.Condition) Instruction.Mnemonic {
368		inline for (@typeInfo(bits.Condition).Enum.fields) \|field\| {
369		if (mem.eql(u8, field.name, @tagName(cc)))
370		return @field(Instruction.Mnemonic, basename ++ field.name);
371		} else unreachable;
	477	return switch (cc) {
	478	inline else => \|comptime_cc\| @field(Instruction.Mnemonic, basename ++ @tagName(comptime_cc)),
	479	};
372	480	}
373	481
374	482	fn mirCmovcc(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
375	483	const ops = emit.mir.instructions.items(.ops)[inst];
376	484	switch (ops) {
377		.rr_c => {
378		const data = emit.mir.instructions.items(.data)[inst].rr_c;
	485	.rr_cc => {
	486	const data = emit.mir.instructions.items(.data)[inst].rr_cc;
379	487	const mnemonic = mnemonicFromConditionCode("cmov", data.cc);
380	488	return emit.encode(mnemonic, .{
381	489	.op1 = .{ .reg = data.r1 },
382	490	.op2 = .{ .reg = data.r2 },
383	491	});
384	492	},
385		else => unreachable, // TODO
	493	.rm_sib_cc => {
	494	const data = emit.mir.instructions.items(.data)[inst].rx_cc;
	495	const extra = emit.mir.extraData(Mir.MemorySib, data.payload).data;
	496	const mnemonic = mnemonicFromConditionCode("cmov", data.cc);
	497	return emit.encode(mnemonic, .{
	498	.op1 = .{ .reg = data.r1 },
	499	.op2 = .{ .mem = Mir.MemorySib.decode(extra) },
	500	});
	501	},
	502	.rm_rip_cc => {
	503	const data = emit.mir.instructions.items(.data)[inst].rx_cc;
	504	const extra = emit.mir.extraData(Mir.MemoryRip, data.payload).data;
	505	const mnemonic = mnemonicFromConditionCode("cmov", data.cc);
	506	return emit.encode(mnemonic, .{
	507	.op1 = .{ .reg = data.r1 },
	508	.op2 = .{ .mem = Mir.MemoryRip.decode(extra) },
	509	});
	510	},
	511	else => unreachable,
386	512	}
387	513	}
388	514
389	515	fn mirSetcc(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
390	516	const ops = emit.mir.instructions.items(.ops)[inst];
391	517	switch (ops) {
392		.r_c => {
393		const data = emit.mir.instructions.items(.data)[inst].r_c;
	518	.r_cc => {
	519	const data = emit.mir.instructions.items(.data)[inst].r_cc;
394	520	const mnemonic = mnemonicFromConditionCode("set", data.cc);
395	521	return emit.encode(mnemonic, .{
396	522	.op1 = .{ .reg = data.r1 },
397	523	});
398	524	},
	525	.m_sib_cc => {
	526	const data = emit.mir.instructions.items(.data)[inst].x_cc;
	527	const extra = emit.mir.extraData(Mir.MemorySib, data.payload).data;
	528	const mnemonic = mnemonicFromConditionCode("set", data.cc);
	529	return emit.encode(mnemonic, .{
	530	.op1 = .{ .mem = Mir.MemorySib.decode(extra) },
	531	});
	532	},
	533	.m_rip_cc => {
	534	const data = emit.mir.instructions.items(.data)[inst].x_cc;
	535	const extra = emit.mir.extraData(Mir.MemoryRip, data.payload).data;
	536	const mnemonic = mnemonicFromConditionCode("set", data.cc);
	537	return emit.encode(mnemonic, .{
	538	.op1 = .{ .mem = Mir.MemoryRip.decode(extra) },
	539	});
	540	},
399	541	else => unreachable, // TODO
400	542	}
401	543	}

src/arch/x86_64/Encoding.zig+60-42

...	...	@@ -19,17 +19,12 @@ op1: Op,
19	19	op2: Op,
20	20	op3: Op,
21	21	op4: Op,
22		opc_len: u2,
23		opc: [3]u8,
	22	opc_len: u3,
	23	opc: [7]u8,
24	24	modrm_ext: u3,
25	25	mode: Mode,
26	26
27		pub fn findByMnemonic(mnemonic: Mnemonic, args: struct {
28		op1: Instruction.Operand,
29		op2: Instruction.Operand,
30		op3: Instruction.Operand,
31		op4: Instruction.Operand,
32		}) !?Encoding {
	27	pub fn findByMnemonic(mnemonic: Mnemonic, args: Instruction.Init) !?Encoding {
33	28	const input_op1 = Op.fromOperand(args.op1);
34	29	const input_op2 = Op.fromOperand(args.op2);
35	30	const input_op3 = Op.fromOperand(args.op3);
...	...	@@ -69,18 +64,19 @@ pub fn findByMnemonic(mnemonic: Mnemonic, args: struct {
69	64	var candidates: [10]Encoding = undefined;
70	65	var count: usize = 0;
71	66	for (table) \|entry\| {
72		const enc = Encoding{
	67	var enc = Encoding{
73	68	.mnemonic = entry[0],
74	69	.op_en = entry[1],
75	70	.op1 = entry[2],
76	71	.op2 = entry[3],
77	72	.op3 = entry[4],
78	73	.op4 = entry[5],
79		.opc_len = entry[6],
80		.opc = .{ entry[7], entry[8], entry[9] },
81		.modrm_ext = entry[10],
82		.mode = entry[11],
	74	.opc_len = @intCast(u3, entry[6].len),
	75	.opc = undefined,
	76	.modrm_ext = entry[7],
	77	.mode = entry[8],
83	78	};
	79	std.mem.copy(u8, &enc.opc, entry[6]);
84	80	if (enc.mnemonic == mnemonic and
85	81	input_op1.isSubset(enc.op1, enc.mode) and
86	82	input_op2.isSubset(enc.op2, enc.mode) and
...	...	@@ -108,17 +104,13 @@ pub fn findByMnemonic(mnemonic: Mnemonic, args: struct {
108	104	if (count == 1) return candidates[0];
109	105
110	106	const EncodingLength = struct {
111		fn estimate(encoding: Encoding, params: struct {
112		op1: Instruction.Operand,
113		op2: Instruction.Operand,
114		op3: Instruction.Operand,
115		op4: Instruction.Operand,
116		}) usize {
	107	fn estimate(encoding: Encoding, params: Instruction.Init) usize {
117	108	var inst = Instruction{
118	109	.op1 = params.op1,
119	110	.op2 = params.op2,
120	111	.op3 = params.op3,
121	112	.op4 = params.op4,
	113	.prefix = params.prefix,
122	114	.encoding = encoding,
123	115	};
124	116	var cwriter = std.io.countingWriter(std.io.null_writer);
...	...	@@ -139,12 +131,7 @@ pub fn findByMnemonic(mnemonic: Mnemonic, args: struct {
139	131	else => {},
140	132	}
141	133
142		const len = EncodingLength.estimate(candidate, .{
143		.op1 = args.op1,
144		.op2 = args.op2,
145		.op3 = args.op3,
146		.op4 = args.op4,
147		});
	134	const len = EncodingLength.estimate(candidate, args);
148	135	const current = shortest_encoding orelse {
149	136	shortest_encoding = .{ .index = i, .len = len };
150	137	continue;
...	...	@@ -184,7 +171,7 @@ pub fn findByOpcode(opc: []const u8, prefixes: struct {
184	171	if (match) {
185	172	if (prefixes.rex.w) {
186	173	switch (enc.mode) {
187		.fpu, .sse, .sse2, .none => {},
	174	.fpu, .sse, .sse2, .sse4_1, .none => {},
188	175	.long, .rex => return enc,
189	176	}
190	177	} else if (prefixes.rex.present and !prefixes.rex.isSet()) {
...	...	@@ -227,7 +214,11 @@ pub fn modRmExt(encoding: Encoding) u3 {
227	214	}
228	215
229	216	pub fn operandBitSize(encoding: Encoding) u64 {
230		if (encoding.mode == .long) return 64;
	217	switch (encoding.mode) {
	218	.short => return 16,
	219	.long => return 64,
	220	else => {},
	221	}
231	222	const bit_size: u64 = switch (encoding.op_en) {
232	223	.np => switch (encoding.op1) {
233	224	.o16 => 16,
...	...	@@ -316,39 +307,63 @@ pub const Mnemonic = enum {
316	307	// zig fmt: off
317	308	// General-purpose
318	309	adc, add, @"and",
319		call, cbw, cwde, cdqe, cwd, cdq, cqo, cmp,
	310	bsf, bsr, bswap, bt, btc, btr, bts,
	311	call, cbw, cdq, cdqe,
320	312	cmova, cmovae, cmovb, cmovbe, cmovc, cmove, cmovg, cmovge, cmovl, cmovle, cmovna,
321	313	cmovnae, cmovnb, cmovnbe, cmovnc, cmovne, cmovng, cmovnge, cmovnl, cmovnle, cmovno,
322	314	cmovnp, cmovns, cmovnz, cmovo, cmovp, cmovpe, cmovpo, cmovs, cmovz,
	315	cmp,
	316	cmps, cmpsb, cmpsd, cmpsq, cmpsw,
	317	cmpxchg, cmpxchg8b, cmpxchg16b,
	318	cqo, cwd, cwde,
323	319	div,
324	320	fisttp, fld,
325	321	idiv, imul, int3,
326	322	ja, jae, jb, jbe, jc, jrcxz, je, jg, jge, jl, jle, jna, jnae, jnb, jnbe,
327	323	jnc, jne, jng, jnge, jnl, jnle, jno, jnp, jns, jnz, jo, jp, jpe, jpo, js, jz,
328	324	jmp,
329		lea,
330		mov, movsx, movsxd, movzx, mul,
331		nop,
	325	lea, lfence,
	326	lods, lodsb, lodsd, lodsq, lodsw,
	327	lzcnt,
	328	mfence, mov, movbe,
	329	movs, movsb, movsd, movsq, movsw,
	330	movsx, movsxd, movzx, mul,
	331	neg, nop, not,
332	332	@"or",
333		pop, push,
334		ret,
335		sal, sar, sbb, shl, shr, sub, syscall,
	333	pop, popcnt, push,
	334	rcl, rcr, ret, rol, ror,
	335	sal, sar, sbb,
	336	scas, scasb, scasd, scasq, scasw,
	337	shl, shr, sub, syscall,
336	338	seta, setae, setb, setbe, setc, sete, setg, setge, setl, setle, setna, setnae,
337	339	setnb, setnbe, setnc, setne, setng, setnge, setnl, setnle, setno, setnp, setns,
338	340	setnz, seto, setp, setpe, setpo, sets, setz,
339		@"test",
	341	sfence,
	342	stos, stosb, stosd, stosq, stosw,
	343	@"test", tzcnt,
340	344	ud2,
341		xor,
	345	xadd, xchg, xor,
342	346	// SSE
343	347	addss,
344	348	cmpss,
	349	divss,
	350	maxss, minss,
345	351	movss,
	352	mulss,
	353	subss,
346	354	ucomiss,
347	355	// SSE2
348	356	addsd,
349		cmpsd,
350		movq, movsd,
	357	//cmpsd,
	358	divsd,
	359	maxsd, minsd,
	360	movq, //movsd,
	361	mulsd,
	362	subsd,
351	363	ucomisd,
	364	// SSE4.1
	365	roundss,
	366	roundsd,
352	367	// zig fmt: on
353	368	};
354	369
...	...	@@ -374,7 +389,7 @@ pub const Op = enum {
374	389	cl,
375	390	r8, r16, r32, r64,
376	391	rm8, rm16, rm32, rm64,
377		m8, m16, m32, m64, m80,
	392	m8, m16, m32, m64, m80, m128,
378	393	rel8, rel16, rel32,
379	394	m,
380	395	moffs,
...	...	@@ -423,6 +438,7 @@ pub const Op = enum {
423	438	32 => .m32,
424	439	64 => .m64,
425	440	80 => .m80,
	441	128 => .m128,
426	442	else => unreachable,
427	443	};
428	444	},
...	...	@@ -460,7 +476,7 @@ pub const Op = enum {
460	476	.imm32, .imm32s, .eax, .r32, .m32, .rm32, .rel32, .xmm_m32 => 32,
461	477	.imm64, .rax, .r64, .m64, .rm64, .xmm_m64 => 64,
462	478	.m80 => 80,
463		.xmm => 128,
	479	.m128, .xmm => 128,
464	480	};
465	481	}
466	482
...	...	@@ -507,7 +523,7 @@ pub const Op = enum {
507	523	// zig fmt: off
508	524	return switch (op) {
509	525	.rm8, .rm16, .rm32, .rm64,
510		.m8, .m16, .m32, .m64, .m80,
	526	.m8, .m16, .m32, .m64, .m80, .m128,
511	527	.m,
512	528	.xmm_m32, .xmm_m64,
513	529	=> true,
...	...	@@ -542,7 +558,7 @@ pub const Op = enum {
542	558	else => {
543	559	if (op.isRegister() and target.isRegister()) {
544	560	switch (mode) {
545		.sse, .sse2 => return op.isFloatingPointRegister() and target.isFloatingPointRegister(),
	561	.sse, .sse2, .sse4_1 => return op.isFloatingPointRegister() and target.isFloatingPointRegister(),
546	562	else => switch (target) {
547	563	.cl, .al, .ax, .eax, .rax => return op == target,
548	564	else => return op.bitSize() == target.bitSize(),
...	...	@@ -579,9 +595,11 @@ pub const Op = enum {
579	595
580	596	pub const Mode = enum {
581	597	none,
	598	short,
582	599	fpu,
583	600	rex,
584	601	long,
585	602	sse,
586	603	sse2,
	604	sse4_1,
587	605	};

src/arch/x86_64/Mir.zig+149-12

...	...	@@ -38,6 +38,20 @@ pub const Inst = struct {
38	38	add,
39	39	/// Logical and
40	40	@"and",
	41	/// Bit scan forward
	42	bsf,
	43	/// Bit scan reverse
	44	bsr,
	45	/// Byte swap
	46	bswap,
	47	/// Bit test
	48	bt,
	49	/// Bit test and complement
	50	btc,
	51	/// Bit test and reset
	52	btr,
	53	/// Bit test and set
	54	bts,
41	55	/// Call
42	56	call,
43	57	/// Convert byte to word
...	...	@@ -54,6 +68,10 @@ pub const Inst = struct {
54	68	cqo,
55	69	/// Logical compare
56	70	cmp,
	71	/// Compare and exchange
	72	cmpxchg,
	73	/// Compare and exchange bytes
	74	cmpxchgb,
57	75	/// Unsigned division
58	76	div,
59	77	/// Store integer with truncation
...	...	@@ -70,30 +88,54 @@ pub const Inst = struct {
70	88	jmp,
71	89	/// Load effective address
72	90	lea,
	91	/// Load fence
	92	lfence,
	93	/// Count the number of leading zero bits
	94	lzcnt,
	95	/// Memory fence
	96	mfence,
73	97	/// Move
74	98	mov,
	99	/// Move data after swapping bytes
	100	movbe,
75	101	/// Move with sign extension
76	102	movsx,
77	103	/// Move with zero extension
78	104	movzx,
79	105	/// Multiply
80	106	mul,
	107	/// Two's complement negation
	108	neg,
81	109	/// No-op
82	110	nop,
	111	/// One's complement negation
	112	not,
83	113	/// Logical or
84	114	@"or",
85	115	/// Pop
86	116	pop,
	117	/// Return the count of number of bits set to 1
	118	popcnt,
87	119	/// Push
88	120	push,
	121	/// Rotate left through carry
	122	rcl,
	123	/// Rotate right through carry
	124	rcr,
89	125	/// Return
90	126	ret,
	127	/// Rotate left
	128	rol,
	129	/// Rotate right
	130	ror,
91	131	/// Arithmetic shift left
92	132	sal,
93	133	/// Arithmetic shift right
94	134	sar,
95	135	/// Integer subtraction with borrow
96	136	sbb,
	137	/// Store fence
	138	sfence,
97	139	/// Logical shift left
98	140	shl,
99	141	/// Logical shift right
...	...	@@ -104,28 +146,69 @@ pub const Inst = struct {
104	146	syscall,
105	147	/// Test condition
106	148	@"test",
	149	/// Count the number of trailing zero bits
	150	tzcnt,
107	151	/// Undefined instruction
108	152	ud2,
	153	/// Exchange and add
	154	xadd,
	155	/// Exchange register/memory with register
	156	xchg,
109	157	/// Logical exclusive-or
110	158	xor,
111	159
112		/// Add single precision floating point
	160	/// Add single precision floating point values
113	161	addss,
114	162	/// Compare scalar single-precision floating-point values
115	163	cmpss,
	164	/// Divide scalar single-precision floating-point values
	165	divss,
	166	/// Return maximum single-precision floating-point value
	167	maxss,
	168	/// Return minimum single-precision floating-point value
	169	minss,
116	170	/// Move scalar single-precision floating-point value
117	171	movss,
	172	/// Multiply scalar single-precision floating-point values
	173	mulss,
	174	/// Round scalar single-precision floating-point values
	175	roundss,
	176	/// Subtract scalar single-precision floating-point values
	177	subss,
118	178	/// Unordered compare scalar single-precision floating-point values
119	179	ucomiss,
120		/// Add double precision floating point
	180	/// Add double precision floating point values
121	181	addsd,
122	182	/// Compare scalar double-precision floating-point values
123	183	cmpsd,
	184	/// Divide scalar double-precision floating-point values
	185	divsd,
	186	/// Return maximum double-precision floating-point value
	187	maxsd,
	188	/// Return minimum double-precision floating-point value
	189	minsd,
124	190	/// Move scalar double-precision floating-point value
125	191	movsd,
	192	/// Multiply scalar double-precision floating-point values
	193	mulsd,
	194	/// Round scalar double-precision floating-point values
	195	roundsd,
	196	/// Subtract scalar double-precision floating-point values
	197	subsd,
126	198	/// Unordered compare scalar double-precision floating-point values
127	199	ucomisd,
128	200
	201	/// Compare string operands
	202	cmps,
	203	/// Load string
	204	lods,
	205	/// Move data from string to string
	206	movs,
	207	/// Scan string
	208	scas,
	209	/// Store string
	210	stos,
	211
129	212	/// Conditional move
130	213	cmovcc,
131	214	/// Conditional jump
...	...	@@ -185,11 +268,11 @@ pub const Inst = struct {
185	268	/// Uses `rri` payload.
186	269	rri_u,
187	270	/// Register with condition code (CC).
188		/// Uses `r_c` payload.
189		r_c,
	271	/// Uses `r_cc` payload.
	272	r_cc,
190	273	/// Register, register with condition code (CC).
191		/// Uses `rr_c` payload.
192		rr_c,
	274	/// Uses `rr_cc` payload.
	275	rr_cc,
193	276	/// Register, immediate (sign-extended) operands.
194	277	/// Uses `ri` payload.
195	278	ri_s,
...	...	@@ -214,12 +297,24 @@ pub const Inst = struct {
214	297	/// Register, memory (RIP) operands.
215	298	/// Uses `rx` payload.
216	299	rm_rip,
	300	/// Register, memory (SIB) operands with condition code (CC).
	301	/// Uses `rx_cc` payload.
	302	rm_sib_cc,
	303	/// Register, memory (RIP) operands with condition code (CC).
	304	/// Uses `rx_cc` payload.
	305	rm_rip_cc,
217	306	/// Single memory (SIB) operand.
218	307	/// Uses `payload` with extra data of type `MemorySib`.
219	308	m_sib,
220	309	/// Single memory (RIP) operand.
221	310	/// Uses `payload` with extra data of type `MemoryRip`.
222	311	m_rip,
	312	/// Single memory (SIB) operand with condition code (CC).
	313	/// Uses `x_cc` with extra data of type `MemorySib`.
	314	m_sib_cc,
	315	/// Single memory (RIP) operand with condition code (CC).
	316	/// Uses `x_cc` with extra data of type `MemoryRip`.
	317	m_rip_cc,
223	318	/// Memory (SIB), immediate (unsigned) operands.
224	319	/// Uses `xi` payload with extra data of type `MemorySib`.
225	320	mi_u_sib,
...	...	@@ -244,16 +339,42 @@ pub const Inst = struct {
244	339	/// Memory moffs, rax.
245	340	/// Uses `payload` with extra data of type `MemoryMoffs`.
246	341	moffs_rax,
	342	/// Single memory (SIB) operand with lock prefix.
	343	/// Uses `payload` with extra data of type `MemorySib`.
	344	lock_m_sib,
	345	/// Single memory (RIP) operand with lock prefix.
	346	/// Uses `payload` with extra data of type `MemoryRip`.
	347	lock_m_rip,
	348	/// Memory (SIB), immediate (unsigned) operands with lock prefix.
	349	/// Uses `xi` payload with extra data of type `MemorySib`.
	350	lock_mi_u_sib,
	351	/// Memory (RIP), immediate (unsigned) operands with lock prefix.
	352	/// Uses `xi` payload with extra data of type `MemoryRip`.
	353	lock_mi_u_rip,
	354	/// Memory (SIB), immediate (sign-extend) operands with lock prefix.
	355	/// Uses `xi` payload with extra data of type `MemorySib`.
	356	lock_mi_s_sib,
	357	/// Memory (RIP), immediate (sign-extend) operands with lock prefix.
	358	/// Uses `xi` payload with extra data of type `MemoryRip`.
	359	lock_mi_s_rip,
	360	/// Memory (SIB), register operands with lock prefix.
	361	/// Uses `rx` payload with extra data of type `MemorySib`.
	362	lock_mr_sib,
	363	/// Memory (RIP), register operands with lock prefix.
	364	/// Uses `rx` payload with extra data of type `MemoryRip`.
	365	lock_mr_rip,
	366	/// Memory moffs, rax with lock prefix.
	367	/// Uses `payload` with extra data of type `MemoryMoffs`.
	368	lock_moffs_rax,
247	369	/// References another Mir instruction directly.
248	370	/// Uses `inst` payload.
249	371	inst,
250	372	/// References another Mir instruction directly with condition code (CC).
251	373	/// Uses `inst_cc` payload.
252	374	inst_cc,
253		/// Uses `payload` payload with data of type `MemoryConditionCode`.
254		m_cc,
255		/// Uses `rx` payload with extra data of type `MemoryConditionCode`.
256		rm_cc,
	375	/// String repeat and width
	376	/// Uses `string` payload.
	377	string,
257	378	/// Uses `reloc` payload.
258	379	reloc,
259	380	/// Linker relocation - GOT indirection.
...	...	@@ -295,13 +416,18 @@ pub const Inst = struct {
295	416	r2: Register,
296	417	imm: u32,
297	418	},
	419	/// Condition code (CC), followed by custom payload found in extra.
	420	x_cc: struct {
	421	payload: u32,
	422	cc: bits.Condition,
	423	},
298	424	/// Register with condition code (CC).
299		r_c: struct {
	425	r_cc: struct {
300	426	r1: Register,
301	427	cc: bits.Condition,
302	428	},
303	429	/// Register, register with condition code (CC).
304		rr_c: struct {
	430	rr_cc: struct {
305	431	r1: Register,
306	432	r2: Register,
307	433	cc: bits.Condition,
...	...	@@ -316,11 +442,22 @@ pub const Inst = struct {
316	442	r1: Register,
317	443	payload: u32,
318	444	},
	445	/// Register with condition code (CC), followed by custom payload found in extra.
	446	rx_cc: struct {
	447	r1: Register,
	448	cc: bits.Condition,
	449	payload: u32,
	450	},
319	451	/// Custom payload followed by an immediate.
320	452	xi: struct {
321	453	payload: u32,
322	454	imm: u32,
323	455	},
	456	/// String instruction prefix and width.
	457	string: struct {
	458	repeat: bits.StringRepeat,
	459	width: bits.StringWidth,
	460	},
324	461	/// Relocation for the linker where:
325	462	/// * `atom_index` is the index of the source
326	463	/// * `sym_index` is the index of the target

src/arch/x86_64/bits.zig+8-5

...	...	@@ -6,6 +6,9 @@ const Allocator = std.mem.Allocator;
6	6	const ArrayList = std.ArrayList;
7	7	const DW = std.dwarf;
8	8
	9	pub const StringRepeat = enum(u3) { none, rep, repe, repz, repne, repnz };
	10	pub const StringWidth = enum(u2) { b, w, d, q };
	11
9	12	/// EFLAGS condition codes
10	13	pub const Condition = enum(u5) {
11	14	/// above
...	...	@@ -97,8 +100,7 @@ pub const Condition = enum(u5) {
97	100	};
98	101	}
99	102
100		/// Returns the condition which is true iff the given condition is
101		/// false (if such a condition exists)
	103	/// Returns the condition which is true iff the given condition is false
102	104	pub fn negate(cond: Condition) Condition {
103	105	return switch (cond) {
104	106	.a => .na,
...	...	@@ -127,8 +129,8 @@ pub const Condition = enum(u5) {
127	129	.nz => .z,
128	130	.o => .no,
129	131	.p => .np,
130		.pe => unreachable,
131		.po => unreachable,
	132	.pe => .po,
	133	.po => .pe,
132	134	.s => .ns,
133	135	.z => .nz,
134	136	};
...	...	@@ -470,10 +472,11 @@ pub const Memory = union(enum) {
470	472	}
471	473
472	474	pub fn sib(ptr_size: PtrSize, args: struct {
473		disp: i32,
	475	disp: i32 = 0,
474	476	base: ?Register = null,
475	477	scale_index: ?ScaleIndex = null,
476	478	}) Memory {
	479	if (args.scale_index) \|si\| assert(std.math.isPowerOfTwo(si.scale));
477	480	return .{ .sib = .{
478	481	.base = args.base,
479	482	.disp = args.disp,

src/arch/x86_64/encoder.zig+84-111

...	...	@@ -15,10 +15,21 @@ pub const Instruction = struct {
15	15	op2: Operand = .none,
16	16	op3: Operand = .none,
17	17	op4: Operand = .none,
	18	prefix: Prefix = .none,
18	19	encoding: Encoding,
19	20
20	21	pub const Mnemonic = Encoding.Mnemonic;
21	22
	23	pub const Prefix = enum(u3) {
	24	none,
	25	lock,
	26	rep,
	27	repe,
	28	repz,
	29	repne,
	30	repnz,
	31	};
	32
22	33	pub const Operand = union(enum) {
23	34	none,
24	35	reg: Register,
...	...	@@ -96,19 +107,18 @@ pub const Instruction = struct {
96	107	}
97	108	};
98	109
99		pub fn new(mnemonic: Mnemonic, args: struct {
	110	pub const Init = struct {
	111	prefix: Prefix = .none,
100	112	op1: Operand = .none,
101	113	op2: Operand = .none,
102	114	op3: Operand = .none,
103	115	op4: Operand = .none,
104		}) !Instruction {
105		const encoding = (try Encoding.findByMnemonic(mnemonic, .{
106		.op1 = args.op1,
107		.op2 = args.op2,
108		.op3 = args.op3,
109		.op4 = args.op4,
110		})) orelse {
111		log.debug("no encoding found for: {s} {s} {s} {s} {s}", .{
	116	};
	117
	118	pub fn new(mnemonic: Mnemonic, args: Init) !Instruction {
	119	const encoding = (try Encoding.findByMnemonic(mnemonic, args)) orelse {
	120	log.debug("no encoding found for: {s} {s} {s} {s} {s} {s}", .{
	121	@tagName(args.prefix),
112	122	@tagName(mnemonic),
113	123	@tagName(Encoding.Op.fromOperand(args.op1)),
114	124	@tagName(Encoding.Op.fromOperand(args.op2)),
...	...	@@ -119,6 +129,7 @@ pub const Instruction = struct {
119	129	};
120	130	log.debug("selected encoding: {}", .{encoding});
121	131	return .{
	132	.prefix = args.prefix,
122	133	.op1 = args.op1,
123	134	.op2 = args.op2,
124	135	.op3 = args.op3,
...	...	@@ -128,6 +139,7 @@ pub const Instruction = struct {
128	139	}
129	140
130	141	pub fn fmtPrint(inst: Instruction, writer: anytype) !void {
	142	if (inst.prefix != .none) try writer.print("{s} ", .{@tagName(inst.prefix)});
131	143	try writer.print("{s}", .{@tagName(inst.encoding.mnemonic)});
132	144	const ops = [_]struct { Operand, Encoding.Op }{
133	145	.{ inst.op1, inst.encoding.op1 },
...	...	@@ -199,14 +211,12 @@ pub const Instruction = struct {
199	211
200	212	fn encodeOpcode(inst: Instruction, encoder: anytype) !void {
201	213	const opcode = inst.encoding.opcode();
	214	const first = @boolToInt(inst.encoding.mandatoryPrefix() != null);
	215	const final = opcode.len - 1;
	216	for (opcode[first..final]) \|byte\| try encoder.opcode_1byte(byte);
202	217	switch (inst.encoding.op_en) {
203		.o, .oi => try encoder.opcode_withReg(opcode[0], inst.op1.reg.lowEnc()),
204		else => {
205		const index: usize = if (inst.encoding.mandatoryPrefix()) \|_\| 1 else 0;
206		for (opcode[index..]) \|byte\| {
207		try encoder.opcode_1byte(byte);
208		}
209		},
	218	.o, .oi => try encoder.opcode_withReg(opcode[final], inst.op1.reg.lowEnc()),
	219	else => try encoder.opcode_1byte(opcode[final]),
210	220	}
211	221	}
212	222
...	...	@@ -215,6 +225,14 @@ pub const Instruction = struct {
215	225	const op_en = enc.op_en;
216	226
217	227	var legacy = LegacyPrefixes{};
	228
	229	switch (inst.prefix) {
	230	.none => {},
	231	.lock => legacy.prefix_f0 = true,
	232	.repne, .repnz => legacy.prefix_f2 = true,
	233	.rep, .repe, .repz => legacy.prefix_f3 = true,
	234	}
	235
218	236	if (enc.mode == .none) {
219	237	const bit_size = enc.operandBitSize();
220	238	if (bit_size == 16) {
...	...	@@ -811,15 +829,11 @@ const TestEncode = struct {
811	829	buffer: [32]u8 = undefined,
812	830	index: usize = 0,
813	831
814		fn encode(enc: *TestEncode, mnemonic: Instruction.Mnemonic, args: struct {
815		op1: Instruction.Operand = .none,
816		op2: Instruction.Operand = .none,
817		op3: Instruction.Operand = .none,
818		op4: Instruction.Operand = .none,
819		}) !void {
	832	fn encode(enc: *TestEncode, mnemonic: Instruction.Mnemonic, args: Instruction.Init) !void {
820	833	var stream = std.io.fixedBufferStream(&enc.buffer);
821	834	var count_writer = std.io.countingWriter(stream.writer());
822	835	const inst = try Instruction.new(mnemonic, .{
	836	.prefix = args.prefix,
823	837	.op1 = args.op1,
824	838	.op2 = args.op2,
825	839	.op3 = args.op3,
...	...	@@ -880,10 +894,10 @@ test "lower MI encoding" {
880	894	try enc.encode(.mov, .{ .op1 = .{ .reg = .r12 }, .op2 = .{ .imm = Immediate.u(0x1000) } });
881	895	try expectEqualHexStrings("\x49\xC7\xC4\x00\x10\x00\x00", enc.code(), "mov r12, 0x1000");
882	896
883		try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.sib(.byte, .{
884		.base = .r12,
885		.disp = 0,
886		}) }, .op2 = .{ .imm = Immediate.u(0x10) } });
	897	try enc.encode(.mov, .{
	898	.op1 = .{ .mem = Memory.sib(.byte, .{ .base = .r12 }) },
	899	.op2 = .{ .imm = Immediate.u(0x10) },
	900	});
887	901	try expectEqualHexStrings("\x41\xC6\x04\x24\x10", enc.code(), "mov BYTE PTR [r12], 0x10");
888	902
889	903	try enc.encode(.mov, .{ .op1 = .{ .reg = .r12 }, .op2 = .{ .imm = Immediate.u(0x1000) } });
...	...	@@ -895,10 +909,10 @@ test "lower MI encoding" {
895	909	try enc.encode(.mov, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .imm = Immediate.u(0x10) } });
896	910	try expectEqualHexStrings("\x48\xc7\xc0\x10\x00\x00\x00", enc.code(), "mov rax, 0x10");
897	911
898		try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.sib(.dword, .{
899		.base = .r11,
900		.disp = 0,
901		}) }, .op2 = .{ .imm = Immediate.u(0x10) } });
	912	try enc.encode(.mov, .{
	913	.op1 = .{ .mem = Memory.sib(.dword, .{ .base = .r11 }) },
	914	.op2 = .{ .imm = Immediate.u(0x10) },
	915	});
902	916	try expectEqualHexStrings("\x41\xc7\x03\x10\x00\x00\x00", enc.code(), "mov DWORD PTR [r11], 0x10");
903	917
904	918	try enc.encode(.mov, .{
...	...	@@ -1014,10 +1028,10 @@ test "lower MI encoding" {
1014	1028	test "lower RM encoding" {
1015	1029	var enc = TestEncode{};
1016	1030
1017		try enc.encode(.mov, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .mem = Memory.sib(.qword, .{
1018		.base = .r11,
1019		.disp = 0,
1020		}) } });
	1031	try enc.encode(.mov, .{
	1032	.op1 = .{ .reg = .rax },
	1033	.op2 = .{ .mem = Memory.sib(.qword, .{ .base = .r11 }) },
	1034	});
1021	1035	try expectEqualHexStrings("\x49\x8b\x03", enc.code(), "mov rax, QWORD PTR [r11]");
1022	1036
1023	1037	try enc.encode(.mov, .{ .op1 = .{ .reg = .rbx }, .op2 = .{ .mem = Memory.sib(.qword, .{
...	...	@@ -1100,20 +1114,16 @@ test "lower RM encoding" {
1100	1114	try enc.encode(.movsx, .{ .op1 = .{ .reg = .ax }, .op2 = .{ .reg = .bl } });
1101	1115	try expectEqualHexStrings("\x66\x0F\xBE\xC3", enc.code(), "movsx ax, bl");
1102	1116
1103		try enc.encode(.movsx, .{ .op1 = .{ .reg = .eax }, .op2 = .{ .mem = Memory.sib(.word, .{
1104		.base = .rbp,
1105		.disp = 0,
1106		}) } });
	1117	try enc.encode(.movsx, .{
	1118	.op1 = .{ .reg = .eax },
	1119	.op2 = .{ .mem = Memory.sib(.word, .{ .base = .rbp }) },
	1120	});
1107	1121	try expectEqualHexStrings("\x0F\xBF\x45\x00", enc.code(), "movsx eax, BYTE PTR [rbp]");
1108	1122
1109		try enc.encode(.movsx, .{ .op1 = .{ .reg = .eax }, .op2 = .{ .mem = Memory.sib(.byte, .{
1110		.base = null,
1111		.scale_index = .{
1112		.index = .rax,
1113		.scale = 2,
1114		},
1115		.disp = 0,
1116		}) } });
	1123	try enc.encode(.movsx, .{
	1124	.op1 = .{ .reg = .eax },
	1125	.op2 = .{ .mem = Memory.sib(.byte, .{ .scale_index = .{ .index = .rax, .scale = 2 } }) },
	1126	});
1117	1127	try expectEqualHexStrings("\x0F\xBE\x04\x45\x00\x00\x00\x00", enc.code(), "movsx eax, BYTE PTR [rax * 2]");
1118	1128
1119	1129	try enc.encode(.movsx, .{ .op1 = .{ .reg = .ax }, .op2 = .{ .mem = Memory.rip(.byte, 0x10) } });
...	...	@@ -1140,14 +1150,13 @@ test "lower RM encoding" {
1140	1150	try enc.encode(.lea, .{ .op1 = .{ .reg = .ax }, .op2 = .{ .mem = Memory.rip(.byte, 0x10) } });
1141	1151	try expectEqualHexStrings("\x66\x8D\x05\x10\x00\x00\x00", enc.code(), "lea ax, BYTE PTR [rip + 0x10]");
1142	1152
1143		try enc.encode(.lea, .{ .op1 = .{ .reg = .rsi }, .op2 = .{ .mem = Memory.sib(.qword, .{
1144		.base = .rbp,
1145		.scale_index = .{
1146		.scale = 1,
1147		.index = .rcx,
1148		},
1149		.disp = 0,
1150		}) } });
	1153	try enc.encode(.lea, .{
	1154	.op1 = .{ .reg = .rsi },
	1155	.op2 = .{ .mem = Memory.sib(.qword, .{
	1156	.base = .rbp,
	1157	.scale_index = .{ .scale = 1, .index = .rcx },
	1158	}) },
	1159	});
1151	1160	try expectEqualHexStrings("\x48\x8D\x74\x0D\x00", enc.code(), "lea rsi, QWORD PTR [rbp + rcx*1 + 0]");
1152	1161
1153	1162	try enc.encode(.add, .{ .op1 = .{ .reg = .r11 }, .op2 = .{ .mem = Memory.sib(.qword, .{
...	...	@@ -1303,51 +1312,35 @@ test "lower M encoding" {
1303	1312	try enc.encode(.call, .{ .op1 = .{ .reg = .r12 } });
1304	1313	try expectEqualHexStrings("\x41\xFF\xD4", enc.code(), "call r12");
1305	1314
1306		try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
1307		.base = .r12,
1308		.disp = 0,
1309		}) } });
	1315	try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{ .base = .r12 }) } });
1310	1316	try expectEqualHexStrings("\x41\xFF\x14\x24", enc.code(), "call QWORD PTR [r12]");
1311	1317
1312		try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
1313		.base = null,
1314		.scale_index = .{
1315		.index = .r11,
1316		.scale = 2,
1317		},
1318		.disp = 0,
1319		}) } });
	1318	try enc.encode(.call, .{
	1319	.op1 = .{ .mem = Memory.sib(.qword, .{
	1320	.base = null,
	1321	.scale_index = .{ .index = .r11, .scale = 2 },
	1322	}) },
	1323	});
1320	1324	try expectEqualHexStrings("\x42\xFF\x14\x5D\x00\x00\x00\x00", enc.code(), "call QWORD PTR [r11 * 2]");
1321	1325
1322		try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
1323		.base = null,
1324		.scale_index = .{
1325		.index = .r12,
1326		.scale = 2,
1327		},
1328		.disp = 0,
1329		}) } });
	1326	try enc.encode(.call, .{
	1327	.op1 = .{ .mem = Memory.sib(.qword, .{
	1328	.base = null,
	1329	.scale_index = .{ .index = .r12, .scale = 2 },
	1330	}) },
	1331	});
1330	1332	try expectEqualHexStrings("\x42\xFF\x14\x65\x00\x00\x00\x00", enc.code(), "call QWORD PTR [r12 * 2]");
1331	1333
1332		try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
1333		.base = .gs,
1334		.disp = 0,
1335		}) } });
	1334	try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{ .base = .gs }) } });
1336	1335	try expectEqualHexStrings("\x65\xFF\x14\x25\x00\x00\x00\x00", enc.code(), "call gs:0x0");
1337	1336
1338	1337	try enc.encode(.call, .{ .op1 = .{ .imm = Immediate.s(0) } });
1339	1338	try expectEqualHexStrings("\xE8\x00\x00\x00\x00", enc.code(), "call 0x0");
1340	1339
1341		try enc.encode(.push, .{ .op1 = .{ .mem = Memory.sib(.qword, .{
1342		.base = .rbp,
1343		.disp = 0,
1344		}) } });
	1340	try enc.encode(.push, .{ .op1 = .{ .mem = Memory.sib(.qword, .{ .base = .rbp }) } });
1345	1341	try expectEqualHexStrings("\xFF\x75\x00", enc.code(), "push QWORD PTR [rbp]");
1346	1342
1347		try enc.encode(.push, .{ .op1 = .{ .mem = Memory.sib(.word, .{
1348		.base = .rbp,
1349		.disp = 0,
1350		}) } });
	1343	try enc.encode(.push, .{ .op1 = .{ .mem = Memory.sib(.word, .{ .base = .rbp }) } });
1351	1344	try expectEqualHexStrings("\x66\xFF\x75\x00", enc.code(), "push QWORD PTR [rbp]");
1352	1345
1353	1346	try enc.encode(.pop, .{ .op1 = .{ .mem = Memory.rip(.qword, 0) } });
...	...	@@ -1447,18 +1440,8 @@ test "lower NP encoding" {
1447	1440	try expectEqualHexStrings("\x0f\x05", enc.code(), "syscall");
1448	1441	}
1449	1442
1450		fn invalidInstruction(mnemonic: Instruction.Mnemonic, args: struct {
1451		op1: Instruction.Operand = .none,
1452		op2: Instruction.Operand = .none,
1453		op3: Instruction.Operand = .none,
1454		op4: Instruction.Operand = .none,
1455		}) !void {
1456		const err = Instruction.new(mnemonic, .{
1457		.op1 = args.op1,
1458		.op2 = args.op2,
1459		.op3 = args.op3,
1460		.op4 = args.op4,
1461		});
	1443	fn invalidInstruction(mnemonic: Instruction.Mnemonic, args: Instruction.Init) !void {
	1444	const err = Instruction.new(mnemonic, args);
1462	1445	try testing.expectError(error.InvalidInstruction, err);
1463	1446	}
1464	1447
...	...	@@ -1479,23 +1462,13 @@ test "invalid instruction" {
1479	1462	try invalidInstruction(.push, .{ .op1 = .{ .imm = Immediate.u(0x1000000000000000) } });
1480	1463	}
1481	1464
1482		fn cannotEncode(mnemonic: Instruction.Mnemonic, args: struct {
1483		op1: Instruction.Operand = .none,
1484		op2: Instruction.Operand = .none,
1485		op3: Instruction.Operand = .none,
1486		op4: Instruction.Operand = .none,
1487		}) !void {
1488		try testing.expectError(error.CannotEncode, Instruction.new(mnemonic, .{
1489		.op1 = args.op1,
1490		.op2 = args.op2,
1491		.op3 = args.op3,
1492		.op4 = args.op4,
1493		}));
	1465	fn cannotEncode(mnemonic: Instruction.Mnemonic, args: Instruction.Init) !void {
	1466	try testing.expectError(error.CannotEncode, Instruction.new(mnemonic, args));
1494	1467	}
1495	1468
1496	1469	test "cannot encode" {
1497	1470	try cannotEncode(.@"test", .{
1498		.op1 = .{ .mem = Memory.sib(.byte, .{ .base = .r12, .disp = 0 }) },
	1471	.op1 = .{ .mem = Memory.sib(.byte, .{ .base = .r12 }) },
1499	1472	.op2 = .{ .reg = .ah },
1500	1473	});
1501	1474	try cannotEncode(.@"test", .{

src/arch/x86_64/encodings.zig+830-591

...	...	@@ -4,618 +4,857 @@ const OpEn = Encoding.OpEn;
4	4	const Op = Encoding.Op;
5	5	const Mode = Encoding.Mode;
6	6
7		const opcode_len = u2;
8	7	const modrm_ext = u3;
9	8
10		const Entry = struct { Mnemonic, OpEn, Op, Op, Op, Op, opcode_len, u8, u8, u8, modrm_ext, Mode };
	9	const Entry = struct { Mnemonic, OpEn, Op, Op, Op, Op, []const u8, modrm_ext, Mode };
11	10
12	11	// TODO move this into a .zon file when Zig is capable of importing .zon files
13	12	// zig fmt: off
14	13	pub const table = &[_]Entry{
15	14	// General-purpose
16		.{ .adc, .zi, .al, .imm8, .none, .none, 1, 0x14, 0x00, 0x00, 0, .none },
17		.{ .adc, .zi, .ax, .imm16, .none, .none, 1, 0x15, 0x00, 0x00, 0, .none },
18		.{ .adc, .zi, .eax, .imm32, .none, .none, 1, 0x15, 0x00, 0x00, 0, .none },
19		.{ .adc, .zi, .rax, .imm32s, .none, .none, 1, 0x15, 0x00, 0x00, 0, .long },
20		.{ .adc, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 2, .none },
21		.{ .adc, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 2, .rex },
22		.{ .adc, .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 2, .none },
23		.{ .adc, .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 2, .none },
24		.{ .adc, .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 2, .long },
25		.{ .adc, .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 2, .none },
26		.{ .adc, .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 2, .none },
27		.{ .adc, .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 2, .long },
28		.{ .adc, .mr, .rm8, .r8, .none, .none, 1, 0x10, 0x00, 0x00, 0, .none },
29		.{ .adc, .mr, .rm8, .r8, .none, .none, 1, 0x10, 0x00, 0x00, 0, .rex },
30		.{ .adc, .mr, .rm16, .r16, .none, .none, 1, 0x11, 0x00, 0x00, 0, .none },
31		.{ .adc, .mr, .rm32, .r32, .none, .none, 1, 0x11, 0x00, 0x00, 0, .none },
32		.{ .adc, .mr, .rm64, .r64, .none, .none, 1, 0x11, 0x00, 0x00, 0, .long },
33		.{ .adc, .rm, .r8, .rm8, .none, .none, 1, 0x12, 0x00, 0x00, 0, .none },
34		.{ .adc, .rm, .r8, .rm8, .none, .none, 1, 0x12, 0x00, 0x00, 0, .rex },
35		.{ .adc, .rm, .r16, .rm16, .none, .none, 1, 0x13, 0x00, 0x00, 0, .none },
36		.{ .adc, .rm, .r32, .rm32, .none, .none, 1, 0x13, 0x00, 0x00, 0, .none },
37		.{ .adc, .rm, .r64, .rm64, .none, .none, 1, 0x13, 0x00, 0x00, 0, .long },
38
39		.{ .add, .zi, .al, .imm8, .none, .none, 1, 0x04, 0x00, 0x00, 0, .none },
40		.{ .add, .zi, .ax, .imm16, .none, .none, 1, 0x05, 0x00, 0x00, 0, .none },
41		.{ .add, .zi, .eax, .imm32, .none, .none, 1, 0x05, 0x00, 0x00, 0, .none },
42		.{ .add, .zi, .rax, .imm32s, .none, .none, 1, 0x05, 0x00, 0x00, 0, .long },
43		.{ .add, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 0, .none },
44		.{ .add, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 0, .rex },
45		.{ .add, .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 0, .none },
46		.{ .add, .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 0, .none },
47		.{ .add, .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 0, .long },
48		.{ .add, .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 0, .none },
49		.{ .add, .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 0, .none },
50		.{ .add, .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 0, .long },
51		.{ .add, .mr, .rm8, .r8, .none, .none, 1, 0x00, 0x00, 0x00, 0, .none },
52		.{ .add, .mr, .rm8, .r8, .none, .none, 1, 0x00, 0x00, 0x00, 0, .rex },
53		.{ .add, .mr, .rm16, .r16, .none, .none, 1, 0x01, 0x00, 0x00, 0, .none },
54		.{ .add, .mr, .rm32, .r32, .none, .none, 1, 0x01, 0x00, 0x00, 0, .none },
55		.{ .add, .mr, .rm64, .r64, .none, .none, 1, 0x01, 0x00, 0x00, 0, .long },
56		.{ .add, .rm, .r8, .rm8, .none, .none, 1, 0x02, 0x00, 0x00, 0, .none },
57		.{ .add, .rm, .r8, .rm8, .none, .none, 1, 0x02, 0x00, 0x00, 0, .rex },
58		.{ .add, .rm, .r16, .rm16, .none, .none, 1, 0x03, 0x00, 0x00, 0, .none },
59		.{ .add, .rm, .r32, .rm32, .none, .none, 1, 0x03, 0x00, 0x00, 0, .none },
60		.{ .add, .rm, .r64, .rm64, .none, .none, 1, 0x03, 0x00, 0x00, 0, .long },
61
62		.{ .@"and", .zi, .al, .imm8, .none, .none, 1, 0x24, 0x00, 0x00, 0, .none },
63		.{ .@"and", .zi, .ax, .imm16, .none, .none, 1, 0x25, 0x00, 0x00, 0, .none },
64		.{ .@"and", .zi, .eax, .imm32, .none, .none, 1, 0x25, 0x00, 0x00, 0, .none },
65		.{ .@"and", .zi, .rax, .imm32s, .none, .none, 1, 0x25, 0x00, 0x00, 0, .long },
66		.{ .@"and", .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 4, .none },
67		.{ .@"and", .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 4, .rex },
68		.{ .@"and", .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 4, .none },
69		.{ .@"and", .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 4, .none },
70		.{ .@"and", .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 4, .long },
71		.{ .@"and", .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 4, .none },
72		.{ .@"and", .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 4, .none },
73		.{ .@"and", .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 4, .long },
74		.{ .@"and", .mr, .rm8, .r8, .none, .none, 1, 0x20, 0x00, 0x00, 0, .none },
75		.{ .@"and", .mr, .rm8, .r8, .none, .none, 1, 0x20, 0x00, 0x00, 0, .rex },
76		.{ .@"and", .mr, .rm16, .r16, .none, .none, 1, 0x21, 0x00, 0x00, 0, .none },
77		.{ .@"and", .mr, .rm32, .r32, .none, .none, 1, 0x21, 0x00, 0x00, 0, .none },
78		.{ .@"and", .mr, .rm64, .r64, .none, .none, 1, 0x21, 0x00, 0x00, 0, .long },
79		.{ .@"and", .rm, .r8, .rm8, .none, .none, 1, 0x22, 0x00, 0x00, 0, .none },
80		.{ .@"and", .rm, .r8, .rm8, .none, .none, 1, 0x22, 0x00, 0x00, 0, .rex },
81		.{ .@"and", .rm, .r16, .rm16, .none, .none, 1, 0x23, 0x00, 0x00, 0, .none },
82		.{ .@"and", .rm, .r32, .rm32, .none, .none, 1, 0x23, 0x00, 0x00, 0, .none },
83		.{ .@"and", .rm, .r64, .rm64, .none, .none, 1, 0x23, 0x00, 0x00, 0, .long },
	15	.{ .adc, .zi, .al, .imm8, .none, .none, &.{ 0x14 }, 0, .none },
	16	.{ .adc, .zi, .ax, .imm16, .none, .none, &.{ 0x15 }, 0, .none },
	17	.{ .adc, .zi, .eax, .imm32, .none, .none, &.{ 0x15 }, 0, .none },
	18	.{ .adc, .zi, .rax, .imm32s, .none, .none, &.{ 0x15 }, 0, .long },
	19	.{ .adc, .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 2, .none },
	20	.{ .adc, .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 2, .rex },
	21	.{ .adc, .mi, .rm16, .imm16, .none, .none, &.{ 0x81 }, 2, .none },
	22	.{ .adc, .mi, .rm32, .imm32, .none, .none, &.{ 0x81 }, 2, .none },
	23	.{ .adc, .mi, .rm64, .imm32s, .none, .none, &.{ 0x81 }, 2, .long },
	24	.{ .adc, .mi, .rm16, .imm8s, .none, .none, &.{ 0x83 }, 2, .none },
	25	.{ .adc, .mi, .rm32, .imm8s, .none, .none, &.{ 0x83 }, 2, .none },
	26	.{ .adc, .mi, .rm64, .imm8s, .none, .none, &.{ 0x83 }, 2, .long },
	27	.{ .adc, .mr, .rm8, .r8, .none, .none, &.{ 0x10 }, 0, .none },
	28	.{ .adc, .mr, .rm8, .r8, .none, .none, &.{ 0x10 }, 0, .rex },
	29	.{ .adc, .mr, .rm16, .r16, .none, .none, &.{ 0x11 }, 0, .none },
	30	.{ .adc, .mr, .rm32, .r32, .none, .none, &.{ 0x11 }, 0, .none },
	31	.{ .adc, .mr, .rm64, .r64, .none, .none, &.{ 0x11 }, 0, .long },
	32	.{ .adc, .rm, .r8, .rm8, .none, .none, &.{ 0x12 }, 0, .none },
	33	.{ .adc, .rm, .r8, .rm8, .none, .none, &.{ 0x12 }, 0, .rex },
	34	.{ .adc, .rm, .r16, .rm16, .none, .none, &.{ 0x13 }, 0, .none },
	35	.{ .adc, .rm, .r32, .rm32, .none, .none, &.{ 0x13 }, 0, .none },
	36	.{ .adc, .rm, .r64, .rm64, .none, .none, &.{ 0x13 }, 0, .long },
	37
	38	.{ .add, .zi, .al, .imm8, .none, .none, &.{ 0x04 }, 0, .none },
	39	.{ .add, .zi, .ax, .imm16, .none, .none, &.{ 0x05 }, 0, .none },
	40	.{ .add, .zi, .eax, .imm32, .none, .none, &.{ 0x05 }, 0, .none },
	41	.{ .add, .zi, .rax, .imm32s, .none, .none, &.{ 0x05 }, 0, .long },
	42	.{ .add, .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 0, .none },
	43	.{ .add, .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 0, .rex },
	44	.{ .add, .mi, .rm16, .imm16, .none, .none, &.{ 0x81 }, 0, .none },
	45	.{ .add, .mi, .rm32, .imm32, .none, .none, &.{ 0x81 }, 0, .none },
	46	.{ .add, .mi, .rm64, .imm32s, .none, .none, &.{ 0x81 }, 0, .long },
	47	.{ .add, .mi, .rm16, .imm8s, .none, .none, &.{ 0x83 }, 0, .none },
	48	.{ .add, .mi, .rm32, .imm8s, .none, .none, &.{ 0x83 }, 0, .none },
	49	.{ .add, .mi, .rm64, .imm8s, .none, .none, &.{ 0x83 }, 0, .long },
	50	.{ .add, .mr, .rm8, .r8, .none, .none, &.{ 0x00 }, 0, .none },
	51	.{ .add, .mr, .rm8, .r8, .none, .none, &.{ 0x00 }, 0, .rex },
	52	.{ .add, .mr, .rm16, .r16, .none, .none, &.{ 0x01 }, 0, .none },
	53	.{ .add, .mr, .rm32, .r32, .none, .none, &.{ 0x01 }, 0, .none },
	54	.{ .add, .mr, .rm64, .r64, .none, .none, &.{ 0x01 }, 0, .long },
	55	.{ .add, .rm, .r8, .rm8, .none, .none, &.{ 0x02 }, 0, .none },
	56	.{ .add, .rm, .r8, .rm8, .none, .none, &.{ 0x02 }, 0, .rex },
	57	.{ .add, .rm, .r16, .rm16, .none, .none, &.{ 0x03 }, 0, .none },
	58	.{ .add, .rm, .r32, .rm32, .none, .none, &.{ 0x03 }, 0, .none },
	59	.{ .add, .rm, .r64, .rm64, .none, .none, &.{ 0x03 }, 0, .long },
	60
	61	.{ .@"and", .zi, .al, .imm8, .none, .none, &.{ 0x24 }, 0, .none },
	62	.{ .@"and", .zi, .ax, .imm16, .none, .none, &.{ 0x25 }, 0, .none },
	63	.{ .@"and", .zi, .eax, .imm32, .none, .none, &.{ 0x25 }, 0, .none },
	64	.{ .@"and", .zi, .rax, .imm32s, .none, .none, &.{ 0x25 }, 0, .long },
	65	.{ .@"and", .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 4, .none },
	66	.{ .@"and", .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 4, .rex },
	67	.{ .@"and", .mi, .rm16, .imm16, .none, .none, &.{ 0x81 }, 4, .none },
	68	.{ .@"and", .mi, .rm32, .imm32, .none, .none, &.{ 0x81 }, 4, .none },
	69	.{ .@"and", .mi, .rm64, .imm32s, .none, .none, &.{ 0x81 }, 4, .long },
	70	.{ .@"and", .mi, .rm16, .imm8s, .none, .none, &.{ 0x83 }, 4, .none },
	71	.{ .@"and", .mi, .rm32, .imm8s, .none, .none, &.{ 0x83 }, 4, .none },
	72	.{ .@"and", .mi, .rm64, .imm8s, .none, .none, &.{ 0x83 }, 4, .long },
	73	.{ .@"and", .mr, .rm8, .r8, .none, .none, &.{ 0x20 }, 0, .none },
	74	.{ .@"and", .mr, .rm8, .r8, .none, .none, &.{ 0x20 }, 0, .rex },
	75	.{ .@"and", .mr, .rm16, .r16, .none, .none, &.{ 0x21 }, 0, .none },
	76	.{ .@"and", .mr, .rm32, .r32, .none, .none, &.{ 0x21 }, 0, .none },
	77	.{ .@"and", .mr, .rm64, .r64, .none, .none, &.{ 0x21 }, 0, .long },
	78	.{ .@"and", .rm, .r8, .rm8, .none, .none, &.{ 0x22 }, 0, .none },
	79	.{ .@"and", .rm, .r8, .rm8, .none, .none, &.{ 0x22 }, 0, .rex },
	80	.{ .@"and", .rm, .r16, .rm16, .none, .none, &.{ 0x23 }, 0, .none },
	81	.{ .@"and", .rm, .r32, .rm32, .none, .none, &.{ 0x23 }, 0, .none },
	82	.{ .@"and", .rm, .r64, .rm64, .none, .none, &.{ 0x23 }, 0, .long },
	83
	84	.{ .bsf, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0xbc }, 0, .none },
	85	.{ .bsf, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0xbc }, 0, .none },
	86	.{ .bsf, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0xbc }, 0, .long },
	87
	88	.{ .bsr, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0xbd }, 0, .none },
	89	.{ .bsr, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0xbd }, 0, .none },
	90	.{ .bsr, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0xbd }, 0, .long },
	91
	92	.{ .bswap, .o, .r32, .none, .none, .none, &.{ 0x0f, 0xc8 }, 0, .none },
	93	.{ .bswap, .o, .r64, .none, .none, .none, &.{ 0x0f, 0xc8 }, 0, .long },
	94
	95	.{ .bt, .mr, .rm16, .r16, .none, .none, &.{ 0x0f, 0xa3 }, 0, .none },
	96	.{ .bt, .mr, .rm32, .r32, .none, .none, &.{ 0x0f, 0xa3 }, 0, .none },
	97	.{ .bt, .mr, .rm64, .r64, .none, .none, &.{ 0x0f, 0xa3 }, 0, .long },
	98	.{ .bt, .mi, .rm16, .imm8, .none, .none, &.{ 0x0f, 0xba }, 4, .none },
	99	.{ .bt, .mi, .rm32, .imm8, .none, .none, &.{ 0x0f, 0xba }, 4, .none },
	100	.{ .bt, .mi, .rm64, .imm8, .none, .none, &.{ 0x0f, 0xba }, 4, .long },
	101
	102	.{ .btc, .mr, .rm16, .r16, .none, .none, &.{ 0x0f, 0xbb }, 0, .none },
	103	.{ .btc, .mr, .rm32, .r32, .none, .none, &.{ 0x0f, 0xbb }, 0, .none },
	104	.{ .btc, .mr, .rm64, .r64, .none, .none, &.{ 0x0f, 0xbb }, 0, .long },
	105	.{ .btc, .mi, .rm16, .imm8, .none, .none, &.{ 0x0f, 0xba }, 7, .none },
	106	.{ .btc, .mi, .rm32, .imm8, .none, .none, &.{ 0x0f, 0xba }, 7, .none },
	107	.{ .btc, .mi, .rm64, .imm8, .none, .none, &.{ 0x0f, 0xba }, 7, .long },
	108
	109	.{ .btr, .mr, .rm16, .r16, .none, .none, &.{ 0x0f, 0xb3 }, 0, .none },
	110	.{ .btr, .mr, .rm32, .r32, .none, .none, &.{ 0x0f, 0xb3 }, 0, .none },
	111	.{ .btr, .mr, .rm64, .r64, .none, .none, &.{ 0x0f, 0xb3 }, 0, .long },
	112	.{ .btr, .mi, .rm16, .imm8, .none, .none, &.{ 0x0f, 0xba }, 6, .none },
	113	.{ .btr, .mi, .rm32, .imm8, .none, .none, &.{ 0x0f, 0xba }, 6, .none },
	114	.{ .btr, .mi, .rm64, .imm8, .none, .none, &.{ 0x0f, 0xba }, 6, .long },
	115
	116	.{ .bts, .mr, .rm16, .r16, .none, .none, &.{ 0x0f, 0xab }, 0, .none },
	117	.{ .bts, .mr, .rm32, .r32, .none, .none, &.{ 0x0f, 0xab }, 0, .none },
	118	.{ .bts, .mr, .rm64, .r64, .none, .none, &.{ 0x0f, 0xab }, 0, .long },
	119	.{ .bts, .mi, .rm16, .imm8, .none, .none, &.{ 0x0f, 0xba }, 5, .none },
	120	.{ .bts, .mi, .rm32, .imm8, .none, .none, &.{ 0x0f, 0xba }, 5, .none },
	121	.{ .bts, .mi, .rm64, .imm8, .none, .none, &.{ 0x0f, 0xba }, 5, .long },
84	122
85	123	// This is M encoding according to Intel, but D makes more sense here.
86		.{ .call, .d, .rel32, .none, .none, .none, 1, 0xe8, 0x00, 0x00, 0, .none },
87		.{ .call, .m, .rm64, .none, .none, .none, 1, 0xff, 0x00, 0x00, 2, .none },
88
89		.{ .cbw, .np, .o16, .none, .none, .none, 1, 0x98, 0x00, 0x00, 0, .none },
90		.{ .cwde, .np, .o32, .none, .none, .none, 1, 0x98, 0x00, 0x00, 0, .none },
91		.{ .cdqe, .np, .o64, .none, .none, .none, 1, 0x98, 0x00, 0x00, 0, .long },
92
93		.{ .cwd, .np, .o16, .none, .none, .none, 1, 0x99, 0x00, 0x00, 0, .none },
94		.{ .cdq, .np, .o32, .none, .none, .none, 1, 0x99, 0x00, 0x00, 0, .none },
95		.{ .cqo, .np, .o64, .none, .none, .none, 1, 0x99, 0x00, 0x00, 0, .long },
96
97		.{ .cmova, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .none },
98		.{ .cmova, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .none },
99		.{ .cmova, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .long },
100		.{ .cmovae, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none },
101		.{ .cmovae, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none },
102		.{ .cmovae, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .long },
103		.{ .cmovb, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none },
104		.{ .cmovb, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none },
105		.{ .cmovb, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .long },
106		.{ .cmovbe, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .none },
107		.{ .cmovbe, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .none },
108		.{ .cmovbe, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .long },
109		.{ .cmovc, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none },
110		.{ .cmovc, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none },
111		.{ .cmovc, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .long },
112		.{ .cmove, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .none },
113		.{ .cmove, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .none },
114		.{ .cmove, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .long },
115		.{ .cmovg, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .none },
116		.{ .cmovg, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .none },
117		.{ .cmovg, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .long },
118		.{ .cmovge, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .none },
119		.{ .cmovge, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .none },
120		.{ .cmovge, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .long },
121		.{ .cmovl, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .none },
122		.{ .cmovl, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .none },
123		.{ .cmovl, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .long },
124		.{ .cmovle, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .none },
125		.{ .cmovle, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .none },
126		.{ .cmovle, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .long },
127		.{ .cmovna, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .none },
128		.{ .cmovna, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .none },
129		.{ .cmovna, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .long },
130		.{ .cmovnae, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none },
131		.{ .cmovnae, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none },
132		.{ .cmovnae, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .long },
133		.{ .cmovnb, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none },
134		.{ .cmovnb, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none },
135		.{ .cmovnb, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .long },
136		.{ .cmovnbe, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .none },
137		.{ .cmovnbe, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .none },
138		.{ .cmovnbe, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .long },
139		.{ .cmovnc, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none },
140		.{ .cmovnc, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none },
141		.{ .cmovnc, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .long },
142		.{ .cmovne, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .none },
143		.{ .cmovne, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .none },
144		.{ .cmovne, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .long },
145		.{ .cmovng, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .none },
146		.{ .cmovng, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .none },
147		.{ .cmovng, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .long },
148		.{ .cmovnge, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .none },
149		.{ .cmovnge, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .none },
150		.{ .cmovnge, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .long },
151		.{ .cmovnl, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .none },
152		.{ .cmovnl, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .none },
153		.{ .cmovnl, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .long },
154		.{ .cmovnle, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .none },
155		.{ .cmovnle, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .none },
156		.{ .cmovnle, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .long },
157		.{ .cmovno, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x41, 0x00, 0, .none },
158		.{ .cmovno, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x41, 0x00, 0, .none },
159		.{ .cmovno, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x41, 0x00, 0, .long },
160		.{ .cmovnp, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .none },
161		.{ .cmovnp, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .none },
162		.{ .cmovnp, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .long },
163		.{ .cmovns, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x49, 0x00, 0, .none },
164		.{ .cmovns, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x49, 0x00, 0, .none },
165		.{ .cmovns, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x49, 0x00, 0, .long },
166		.{ .cmovnz, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .none },
167		.{ .cmovnz, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .none },
168		.{ .cmovnz, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .long },
169		.{ .cmovo, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x40, 0x00, 0, .none },
170		.{ .cmovo, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x40, 0x00, 0, .none },
171		.{ .cmovo, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x40, 0x00, 0, .long },
172		.{ .cmovp, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .none },
173		.{ .cmovp, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .none },
174		.{ .cmovp, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .long },
175		.{ .cmovpe, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .none },
176		.{ .cmovpe, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .none },
177		.{ .cmovpe, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .long },
178		.{ .cmovpo, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .none },
179		.{ .cmovpo, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .none },
180		.{ .cmovpo, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .long },
181		.{ .cmovs, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x48, 0x00, 0, .none },
182		.{ .cmovs, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x48, 0x00, 0, .none },
183		.{ .cmovs, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x48, 0x00, 0, .long },
184		.{ .cmovz, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .none },
185		.{ .cmovz, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .none },
186		.{ .cmovz, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .long },
187
188		.{ .cmp, .zi, .al, .imm8, .none, .none, 1, 0x3c, 0x00, 0x00, 0, .none },
189		.{ .cmp, .zi, .ax, .imm16, .none, .none, 1, 0x3d, 0x00, 0x00, 0, .none },
190		.{ .cmp, .zi, .eax, .imm32, .none, .none, 1, 0x3d, 0x00, 0x00, 0, .none },
191		.{ .cmp, .zi, .rax, .imm32s, .none, .none, 1, 0x3d, 0x00, 0x00, 0, .long },
192		.{ .cmp, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 7, .none },
193		.{ .cmp, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 7, .rex },
194		.{ .cmp, .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 7, .none },
195		.{ .cmp, .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 7, .none },
196		.{ .cmp, .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 7, .long },
197		.{ .cmp, .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 7, .none },
198		.{ .cmp, .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 7, .none },
199		.{ .cmp, .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 7, .long },
200		.{ .cmp, .mr, .rm8, .r8, .none, .none, 1, 0x38, 0x00, 0x00, 0, .none },
201		.{ .cmp, .mr, .rm8, .r8, .none, .none, 1, 0x38, 0x00, 0x00, 0, .rex },
202		.{ .cmp, .mr, .rm16, .r16, .none, .none, 1, 0x39, 0x00, 0x00, 0, .none },
203		.{ .cmp, .mr, .rm32, .r32, .none, .none, 1, 0x39, 0x00, 0x00, 0, .none },
204		.{ .cmp, .mr, .rm64, .r64, .none, .none, 1, 0x39, 0x00, 0x00, 0, .long },
205		.{ .cmp, .rm, .r8, .rm8, .none, .none, 1, 0x3a, 0x00, 0x00, 0, .none },
206		.{ .cmp, .rm, .r8, .rm8, .none, .none, 1, 0x3a, 0x00, 0x00, 0, .rex },
207		.{ .cmp, .rm, .r16, .rm16, .none, .none, 1, 0x3b, 0x00, 0x00, 0, .none },
208		.{ .cmp, .rm, .r32, .rm32, .none, .none, 1, 0x3b, 0x00, 0x00, 0, .none },
209		.{ .cmp, .rm, .r64, .rm64, .none, .none, 1, 0x3b, 0x00, 0x00, 0, .long },
210
211		.{ .div, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 6, .none },
212		.{ .div, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 6, .rex },
213		.{ .div, .m, .rm16, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 6, .none },
214		.{ .div, .m, .rm32, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 6, .none },
215		.{ .div, .m, .rm64, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 6, .long },
216
217		.{ .fisttp, .m, .m16, .none, .none, .none, 1, 0xdf, 0x00, 0x00, 1, .fpu },
218		.{ .fisttp, .m, .m32, .none, .none, .none, 1, 0xdb, 0x00, 0x00, 1, .fpu },
219		.{ .fisttp, .m, .m64, .none, .none, .none, 1, 0xdd, 0x00, 0x00, 1, .fpu },
220
221		.{ .fld, .m, .m32, .none, .none, .none, 1, 0xd9, 0x00, 0x00, 0, .fpu },
222		.{ .fld, .m, .m64, .none, .none, .none, 1, 0xdd, 0x00, 0x00, 0, .fpu },
223		.{ .fld, .m, .m80, .none, .none, .none, 1, 0xdb, 0x00, 0x00, 5, .fpu },
224
225		.{ .idiv, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 7, .none },
226		.{ .idiv, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 7, .rex },
227		.{ .idiv, .m, .rm16, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 7, .none },
228		.{ .idiv, .m, .rm32, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 7, .none },
229		.{ .idiv, .m, .rm64, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 7, .long },
230
231		.{ .imul, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 5, .none },
232		.{ .imul, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 5, .rex },
233		.{ .imul, .m, .rm16, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 5, .none },
234		.{ .imul, .m, .rm32, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 5, .none },
235		.{ .imul, .m, .rm64, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 5, .long },
236		.{ .imul, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0xaf, 0x00, 0, .none },
237		.{ .imul, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0xaf, 0x00, 0, .none },
238		.{ .imul, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0xaf, 0x00, 0, .long },
239		.{ .imul, .rmi, .r16, .rm16, .imm8s, .none, 1, 0x6b, 0x00, 0x00, 0, .none },
240		.{ .imul, .rmi, .r32, .rm32, .imm8s, .none, 1, 0x6b, 0x00, 0x00, 0, .none },
241		.{ .imul, .rmi, .r64, .rm64, .imm8s, .none, 1, 0x6b, 0x00, 0x00, 0, .long },
242		.{ .imul, .rmi, .r16, .rm16, .imm16, .none, 1, 0x69, 0x00, 0x00, 0, .none },
243		.{ .imul, .rmi, .r32, .rm32, .imm32, .none, 1, 0x69, 0x00, 0x00, 0, .none },
244		.{ .imul, .rmi, .r64, .rm64, .imm32, .none, 1, 0x69, 0x00, 0x00, 0, .long },
245
246		.{ .int3, .np, .none, .none, .none, .none, 1, 0xcc, 0x00, 0x00, 0, .none },
247
248		.{ .ja, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x87, 0x00, 0, .none },
249		.{ .jae, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x83, 0x00, 0, .none },
250		.{ .jb, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x82, 0x00, 0, .none },
251		.{ .jbe, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x86, 0x00, 0, .none },
252		.{ .jc, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x82, 0x00, 0, .none },
253		.{ .jrcxz, .d, .rel32, .none, .none, .none, 1, 0xe3, 0x00, 0x00, 0, .none },
254		.{ .je, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x84, 0x00, 0, .none },
255		.{ .jg, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8f, 0x00, 0, .none },
256		.{ .jge, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8d, 0x00, 0, .none },
257		.{ .jl, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8c, 0x00, 0, .none },
258		.{ .jle, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8e, 0x00, 0, .none },
259		.{ .jna, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x86, 0x00, 0, .none },
260		.{ .jnae, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x82, 0x00, 0, .none },
261		.{ .jnb, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x83, 0x00, 0, .none },
262		.{ .jnbe, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x87, 0x00, 0, .none },
263		.{ .jnc, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x83, 0x00, 0, .none },
264		.{ .jne, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x85, 0x00, 0, .none },
265		.{ .jng, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8e, 0x00, 0, .none },
266		.{ .jnge, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8c, 0x00, 0, .none },
267		.{ .jnl, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8d, 0x00, 0, .none },
268		.{ .jnle, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8f, 0x00, 0, .none },
269		.{ .jno, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x81, 0x00, 0, .none },
270		.{ .jnp, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8b, 0x00, 0, .none },
271		.{ .jns, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x89, 0x00, 0, .none },
272		.{ .jnz, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x85, 0x00, 0, .none },
273		.{ .jo, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x80, 0x00, 0, .none },
274		.{ .jp, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8a, 0x00, 0, .none },
275		.{ .jpe, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8a, 0x00, 0, .none },
276		.{ .jpo, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8b, 0x00, 0, .none },
277		.{ .js, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x88, 0x00, 0, .none },
278		.{ .jz, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x84, 0x00, 0, .none },
279
280		.{ .jmp, .d, .rel32, .none, .none, .none, 1, 0xe9, 0x00, 0x00, 0, .none },
281		.{ .jmp, .m, .rm64, .none, .none, .none, 1, 0xff, 0x00, 0x00, 4, .none },
282
283		.{ .lea, .rm, .r16, .m, .none, .none, 1, 0x8d, 0x00, 0x00, 0, .none },
284		.{ .lea, .rm, .r32, .m, .none, .none, 1, 0x8d, 0x00, 0x00, 0, .none },
285		.{ .lea, .rm, .r64, .m, .none, .none, 1, 0x8d, 0x00, 0x00, 0, .long },
286
287		.{ .mov, .mr, .rm8, .r8, .none, .none, 1, 0x88, 0x00, 0x00, 0, .none },
288		.{ .mov, .mr, .rm8, .r8, .none, .none, 1, 0x88, 0x00, 0x00, 0, .rex },
289		.{ .mov, .mr, .rm16, .r16, .none, .none, 1, 0x89, 0x00, 0x00, 0, .none },
290		.{ .mov, .mr, .rm32, .r32, .none, .none, 1, 0x89, 0x00, 0x00, 0, .none },
291		.{ .mov, .mr, .rm64, .r64, .none, .none, 1, 0x89, 0x00, 0x00, 0, .long },
292		.{ .mov, .rm, .r8, .rm8, .none, .none, 1, 0x8a, 0x00, 0x00, 0, .none },
293		.{ .mov, .rm, .r8, .rm8, .none, .none, 1, 0x8a, 0x00, 0x00, 0, .rex },
294		.{ .mov, .rm, .r16, .rm16, .none, .none, 1, 0x8b, 0x00, 0x00, 0, .none },
295		.{ .mov, .rm, .r32, .rm32, .none, .none, 1, 0x8b, 0x00, 0x00, 0, .none },
296		.{ .mov, .rm, .r64, .rm64, .none, .none, 1, 0x8b, 0x00, 0x00, 0, .long },
297		.{ .mov, .mr, .rm16, .sreg, .none, .none, 1, 0x8c, 0x00, 0x00, 0, .none },
298		.{ .mov, .mr, .rm64, .sreg, .none, .none, 1, 0x8c, 0x00, 0x00, 0, .long },
299		.{ .mov, .rm, .sreg, .rm16, .none, .none, 1, 0x8e, 0x00, 0x00, 0, .none },
300		.{ .mov, .rm, .sreg, .rm64, .none, .none, 1, 0x8e, 0x00, 0x00, 0, .long },
301		.{ .mov, .fd, .al, .moffs, .none, .none, 1, 0xa0, 0x00, 0x00, 0, .none },
302		.{ .mov, .fd, .ax, .moffs, .none, .none, 1, 0xa1, 0x00, 0x00, 0, .none },
303		.{ .mov, .fd, .eax, .moffs, .none, .none, 1, 0xa1, 0x00, 0x00, 0, .none },
304		.{ .mov, .fd, .rax, .moffs, .none, .none, 1, 0xa1, 0x00, 0x00, 0, .long },
305		.{ .mov, .td, .moffs, .al, .none, .none, 1, 0xa2, 0x00, 0x00, 0, .none },
306		.{ .mov, .td, .moffs, .ax, .none, .none, 1, 0xa3, 0x00, 0x00, 0, .none },
307		.{ .mov, .td, .moffs, .eax, .none, .none, 1, 0xa3, 0x00, 0x00, 0, .none },
308		.{ .mov, .td, .moffs, .rax, .none, .none, 1, 0xa3, 0x00, 0x00, 0, .long },
309		.{ .mov, .oi, .r8, .imm8, .none, .none, 1, 0xb0, 0x00, 0x00, 0, .none },
310		.{ .mov, .oi, .r8, .imm8, .none, .none, 1, 0xb0, 0x00, 0x00, 0, .rex },
311		.{ .mov, .oi, .r16, .imm16, .none, .none, 1, 0xb8, 0x00, 0x00, 0, .none },
312		.{ .mov, .oi, .r32, .imm32, .none, .none, 1, 0xb8, 0x00, 0x00, 0, .none },
313		.{ .mov, .oi, .r64, .imm64, .none, .none, 1, 0xb8, 0x00, 0x00, 0, .long },
314		.{ .mov, .mi, .rm8, .imm8, .none, .none, 1, 0xc6, 0x00, 0x00, 0, .none },
315		.{ .mov, .mi, .rm8, .imm8, .none, .none, 1, 0xc6, 0x00, 0x00, 0, .rex },
316		.{ .mov, .mi, .rm16, .imm16, .none, .none, 1, 0xc7, 0x00, 0x00, 0, .none },
317		.{ .mov, .mi, .rm32, .imm32, .none, .none, 1, 0xc7, 0x00, 0x00, 0, .none },
318		.{ .mov, .mi, .rm64, .imm32s, .none, .none, 1, 0xc7, 0x00, 0x00, 0, .long },
319
320		.{ .movsx, .rm, .r16, .rm8, .none, .none, 2, 0x0f, 0xbe, 0x00, 0, .none },
321		.{ .movsx, .rm, .r16, .rm8, .none, .none, 2, 0x0f, 0xbe, 0x00, 0, .rex },
322		.{ .movsx, .rm, .r32, .rm8, .none, .none, 2, 0x0f, 0xbe, 0x00, 0, .none },
323		.{ .movsx, .rm, .r32, .rm8, .none, .none, 2, 0x0f, 0xbe, 0x00, 0, .rex },
324		.{ .movsx, .rm, .r64, .rm8, .none, .none, 2, 0x0f, 0xbe, 0x00, 0, .long },
325		.{ .movsx, .rm, .r32, .rm16, .none, .none, 2, 0x0f, 0xbf, 0x00, 0, .none },
326		.{ .movsx, .rm, .r64, .rm16, .none, .none, 2, 0x0f, 0xbf, 0x00, 0, .long },
	124	.{ .call, .d, .rel32, .none, .none, .none, &.{ 0xe8 }, 0, .none },
	125	.{ .call, .m, .rm64, .none, .none, .none, &.{ 0xff }, 2, .none },
	126
	127	.{ .cbw, .np, .o16, .none, .none, .none, &.{ 0x98 }, 0, .none },
	128	.{ .cwde, .np, .o32, .none, .none, .none, &.{ 0x98 }, 0, .none },
	129	.{ .cdqe, .np, .o64, .none, .none, .none, &.{ 0x98 }, 0, .long },
	130
	131	.{ .cwd, .np, .o16, .none, .none, .none, &.{ 0x99 }, 0, .none },
	132	.{ .cdq, .np, .o32, .none, .none, .none, &.{ 0x99 }, 0, .none },
	133	.{ .cqo, .np, .o64, .none, .none, .none, &.{ 0x99 }, 0, .long },
	134
	135	.{ .cmova, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x47 }, 0, .none },
	136	.{ .cmova, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x47 }, 0, .none },
	137	.{ .cmova, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x47 }, 0, .long },
	138	.{ .cmovae, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x43 }, 0, .none },
	139	.{ .cmovae, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x43 }, 0, .none },
	140	.{ .cmovae, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x43 }, 0, .long },
	141	.{ .cmovb, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x42 }, 0, .none },
	142	.{ .cmovb, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x42 }, 0, .none },
	143	.{ .cmovb, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x42 }, 0, .long },
	144	.{ .cmovbe, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x46 }, 0, .none },
	145	.{ .cmovbe, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x46 }, 0, .none },
	146	.{ .cmovbe, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x46 }, 0, .long },
	147	.{ .cmovc, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x42 }, 0, .none },
	148	.{ .cmovc, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x42 }, 0, .none },
	149	.{ .cmovc, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x42 }, 0, .long },
	150	.{ .cmove, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x44 }, 0, .none },
	151	.{ .cmove, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x44 }, 0, .none },
	152	.{ .cmove, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x44 }, 0, .long },
	153	.{ .cmovg, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4f }, 0, .none },
	154	.{ .cmovg, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4f }, 0, .none },
	155	.{ .cmovg, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4f }, 0, .long },
	156	.{ .cmovge, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4d }, 0, .none },
	157	.{ .cmovge, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4d }, 0, .none },
	158	.{ .cmovge, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4d }, 0, .long },
	159	.{ .cmovl, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4c }, 0, .none },
	160	.{ .cmovl, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4c }, 0, .none },
	161	.{ .cmovl, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4c }, 0, .long },
	162	.{ .cmovle, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4e }, 0, .none },
	163	.{ .cmovle, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4e }, 0, .none },
	164	.{ .cmovle, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4e }, 0, .long },
	165	.{ .cmovna, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x46 }, 0, .none },
	166	.{ .cmovna, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x46 }, 0, .none },
	167	.{ .cmovna, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x46 }, 0, .long },
	168	.{ .cmovnae, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x42 }, 0, .none },
	169	.{ .cmovnae, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x42 }, 0, .none },
	170	.{ .cmovnae, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x42 }, 0, .long },
	171	.{ .cmovnb, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x43 }, 0, .none },
	172	.{ .cmovnb, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x43 }, 0, .none },
	173	.{ .cmovnb, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x43 }, 0, .long },
	174	.{ .cmovnbe, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x47 }, 0, .none },
	175	.{ .cmovnbe, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x47 }, 0, .none },
	176	.{ .cmovnbe, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x47 }, 0, .long },
	177	.{ .cmovnc, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x43 }, 0, .none },
	178	.{ .cmovnc, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x43 }, 0, .none },
	179	.{ .cmovnc, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x43 }, 0, .long },
	180	.{ .cmovne, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x45 }, 0, .none },
	181	.{ .cmovne, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x45 }, 0, .none },
	182	.{ .cmovne, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x45 }, 0, .long },
	183	.{ .cmovng, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4e }, 0, .none },
	184	.{ .cmovng, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4e }, 0, .none },
	185	.{ .cmovng, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4e }, 0, .long },
	186	.{ .cmovnge, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4c }, 0, .none },
	187	.{ .cmovnge, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4c }, 0, .none },
	188	.{ .cmovnge, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4c }, 0, .long },
	189	.{ .cmovnl, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4d }, 0, .none },
	190	.{ .cmovnl, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4d }, 0, .none },
	191	.{ .cmovnl, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4d }, 0, .long },
	192	.{ .cmovnle, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4f }, 0, .none },
	193	.{ .cmovnle, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4f }, 0, .none },
	194	.{ .cmovnle, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4f }, 0, .long },
	195	.{ .cmovno, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x41 }, 0, .none },
	196	.{ .cmovno, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x41 }, 0, .none },
	197	.{ .cmovno, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x41 }, 0, .long },
	198	.{ .cmovnp, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4b }, 0, .none },
	199	.{ .cmovnp, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4b }, 0, .none },
	200	.{ .cmovnp, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4b }, 0, .long },
	201	.{ .cmovns, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x49 }, 0, .none },
	202	.{ .cmovns, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x49 }, 0, .none },
	203	.{ .cmovns, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x49 }, 0, .long },
	204	.{ .cmovnz, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x45 }, 0, .none },
	205	.{ .cmovnz, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x45 }, 0, .none },
	206	.{ .cmovnz, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x45 }, 0, .long },
	207	.{ .cmovo, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x40 }, 0, .none },
	208	.{ .cmovo, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x40 }, 0, .none },
	209	.{ .cmovo, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x40 }, 0, .long },
	210	.{ .cmovp, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4a }, 0, .none },
	211	.{ .cmovp, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4a }, 0, .none },
	212	.{ .cmovp, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4a }, 0, .long },
	213	.{ .cmovpe, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4a }, 0, .none },
	214	.{ .cmovpe, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4a }, 0, .none },
	215	.{ .cmovpe, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4a }, 0, .long },
	216	.{ .cmovpo, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4b }, 0, .none },
	217	.{ .cmovpo, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4b }, 0, .none },
	218	.{ .cmovpo, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4b }, 0, .long },
	219	.{ .cmovs, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x48 }, 0, .none },
	220	.{ .cmovs, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x48 }, 0, .none },
	221	.{ .cmovs, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x48 }, 0, .long },
	222	.{ .cmovz, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x44 }, 0, .none },
	223	.{ .cmovz, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x44 }, 0, .none },
	224	.{ .cmovz, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x44 }, 0, .long },
	225
	226	.{ .cmp, .zi, .al, .imm8, .none, .none, &.{ 0x3c }, 0, .none },
	227	.{ .cmp, .zi, .ax, .imm16, .none, .none, &.{ 0x3d }, 0, .none },
	228	.{ .cmp, .zi, .eax, .imm32, .none, .none, &.{ 0x3d }, 0, .none },
	229	.{ .cmp, .zi, .rax, .imm32s, .none, .none, &.{ 0x3d }, 0, .long },
	230	.{ .cmp, .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 7, .none },
	231	.{ .cmp, .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 7, .rex },
	232	.{ .cmp, .mi, .rm16, .imm16, .none, .none, &.{ 0x81 }, 7, .none },
	233	.{ .cmp, .mi, .rm32, .imm32, .none, .none, &.{ 0x81 }, 7, .none },
	234	.{ .cmp, .mi, .rm64, .imm32s, .none, .none, &.{ 0x81 }, 7, .long },
	235	.{ .cmp, .mi, .rm16, .imm8s, .none, .none, &.{ 0x83 }, 7, .none },
	236	.{ .cmp, .mi, .rm32, .imm8s, .none, .none, &.{ 0x83 }, 7, .none },
	237	.{ .cmp, .mi, .rm64, .imm8s, .none, .none, &.{ 0x83 }, 7, .long },
	238	.{ .cmp, .mr, .rm8, .r8, .none, .none, &.{ 0x38 }, 0, .none },
	239	.{ .cmp, .mr, .rm8, .r8, .none, .none, &.{ 0x38 }, 0, .rex },
	240	.{ .cmp, .mr, .rm16, .r16, .none, .none, &.{ 0x39 }, 0, .none },
	241	.{ .cmp, .mr, .rm32, .r32, .none, .none, &.{ 0x39 }, 0, .none },
	242	.{ .cmp, .mr, .rm64, .r64, .none, .none, &.{ 0x39 }, 0, .long },
	243	.{ .cmp, .rm, .r8, .rm8, .none, .none, &.{ 0x3a }, 0, .none },
	244	.{ .cmp, .rm, .r8, .rm8, .none, .none, &.{ 0x3a }, 0, .rex },
	245	.{ .cmp, .rm, .r16, .rm16, .none, .none, &.{ 0x3b }, 0, .none },
	246	.{ .cmp, .rm, .r32, .rm32, .none, .none, &.{ 0x3b }, 0, .none },
	247	.{ .cmp, .rm, .r64, .rm64, .none, .none, &.{ 0x3b }, 0, .long },
	248
	249	.{ .cmps, .np, .m8, .m8, .none, .none, &.{ 0xa6 }, 0, .none },
	250	.{ .cmps, .np, .m16, .m16, .none, .none, &.{ 0xa7 }, 0, .none },
	251	.{ .cmps, .np, .m32, .m32, .none, .none, &.{ 0xa7 }, 0, .none },
	252	.{ .cmps, .np, .m64, .m64, .none, .none, &.{ 0xa7 }, 0, .long },
	253	.{ .cmpsb, .np, .none, .none, .none, .none, &.{ 0xa6 }, 0, .none },
	254	.{ .cmpsw, .np, .none, .none, .none, .none, &.{ 0xa7 }, 0, .short },
	255	.{ .cmpsd, .np, .none, .none, .none, .none, &.{ 0xa7 }, 0, .none },
	256	.{ .cmpsq, .np, .none, .none, .none, .none, &.{ 0xa7 }, 0, .long },
	257
	258	.{ .cmpxchg, .mr, .rm8, .r8, .none, .none, &.{ 0x0f, 0xb0 }, 0, .none },
	259	.{ .cmpxchg, .mr, .rm8, .r8, .none, .none, &.{ 0x0f, 0xb0 }, 0, .rex },
	260	.{ .cmpxchg, .mr, .rm16, .r16, .none, .none, &.{ 0x0f, 0xb1 }, 0, .rex },
	261	.{ .cmpxchg, .mr, .rm32, .r32, .none, .none, &.{ 0x0f, 0xb1 }, 0, .rex },
	262	.{ .cmpxchg, .mr, .rm64, .r64, .none, .none, &.{ 0x0f, 0xb1 }, 0, .long },
	263
	264	.{ .cmpxchg8b , .m, .m64, .none, .none, .none, &.{ 0x0f, 0xc7 }, 1, .none },
	265	.{ .cmpxchg16b, .m, .m128, .none, .none, .none, &.{ 0x0f, 0xc7 }, 1, .long },
	266
	267	.{ .div, .m, .rm8, .none, .none, .none, &.{ 0xf6 }, 6, .none },
	268	.{ .div, .m, .rm8, .none, .none, .none, &.{ 0xf6 }, 6, .rex },
	269	.{ .div, .m, .rm16, .none, .none, .none, &.{ 0xf7 }, 6, .none },
	270	.{ .div, .m, .rm32, .none, .none, .none, &.{ 0xf7 }, 6, .none },
	271	.{ .div, .m, .rm64, .none, .none, .none, &.{ 0xf7 }, 6, .long },
	272
	273	.{ .fisttp, .m, .m16, .none, .none, .none, &.{ 0xdf }, 1, .fpu },
	274	.{ .fisttp, .m, .m32, .none, .none, .none, &.{ 0xdb }, 1, .fpu },
	275	.{ .fisttp, .m, .m64, .none, .none, .none, &.{ 0xdd }, 1, .fpu },
	276
	277	.{ .fld, .m, .m32, .none, .none, .none, &.{ 0xd9 }, 0, .fpu },
	278	.{ .fld, .m, .m64, .none, .none, .none, &.{ 0xdd }, 0, .fpu },
	279	.{ .fld, .m, .m80, .none, .none, .none, &.{ 0xdb }, 5, .fpu },
	280
	281	.{ .idiv, .m, .rm8, .none, .none, .none, &.{ 0xf6 }, 7, .none },
	282	.{ .idiv, .m, .rm8, .none, .none, .none, &.{ 0xf6 }, 7, .rex },
	283	.{ .idiv, .m, .rm16, .none, .none, .none, &.{ 0xf7 }, 7, .none },
	284	.{ .idiv, .m, .rm32, .none, .none, .none, &.{ 0xf7 }, 7, .none },
	285	.{ .idiv, .m, .rm64, .none, .none, .none, &.{ 0xf7 }, 7, .long },
	286
	287	.{ .imul, .m, .rm8, .none, .none, .none, &.{ 0xf6 }, 5, .none },
	288	.{ .imul, .m, .rm8, .none, .none, .none, &.{ 0xf6 }, 5, .rex },
	289	.{ .imul, .m, .rm16, .none, .none, .none, &.{ 0xf7 }, 5, .none },
	290	.{ .imul, .m, .rm32, .none, .none, .none, &.{ 0xf7 }, 5, .none },
	291	.{ .imul, .m, .rm64, .none, .none, .none, &.{ 0xf7 }, 5, .long },
	292	.{ .imul, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0xaf }, 0, .none },
	293	.{ .imul, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0xaf }, 0, .none },
	294	.{ .imul, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0xaf }, 0, .long },
	295	.{ .imul, .rmi, .r16, .rm16, .imm8s, .none, &.{ 0x6b }, 0, .none },
	296	.{ .imul, .rmi, .r32, .rm32, .imm8s, .none, &.{ 0x6b }, 0, .none },
	297	.{ .imul, .rmi, .r64, .rm64, .imm8s, .none, &.{ 0x6b }, 0, .long },
	298	.{ .imul, .rmi, .r16, .rm16, .imm16, .none, &.{ 0x69 }, 0, .none },
	299	.{ .imul, .rmi, .r32, .rm32, .imm32, .none, &.{ 0x69 }, 0, .none },
	300	.{ .imul, .rmi, .r64, .rm64, .imm32, .none, &.{ 0x69 }, 0, .long },
	301
	302	.{ .int3, .np, .none, .none, .none, .none, &.{ 0xcc }, 0, .none },
	303
	304	.{ .ja, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x87 }, 0, .none },
	305	.{ .jae, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x83 }, 0, .none },
	306	.{ .jb, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x82 }, 0, .none },
	307	.{ .jbe, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x86 }, 0, .none },
	308	.{ .jc, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x82 }, 0, .none },
	309	.{ .jrcxz, .d, .rel32, .none, .none, .none, &.{ 0xe3 }, 0, .none },
	310	.{ .je, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x84 }, 0, .none },
	311	.{ .jg, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8f }, 0, .none },
	312	.{ .jge, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8d }, 0, .none },
	313	.{ .jl, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8c }, 0, .none },
	314	.{ .jle, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8e }, 0, .none },
	315	.{ .jna, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x86 }, 0, .none },
	316	.{ .jnae, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x82 }, 0, .none },
	317	.{ .jnb, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x83 }, 0, .none },
	318	.{ .jnbe, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x87 }, 0, .none },
	319	.{ .jnc, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x83 }, 0, .none },
	320	.{ .jne, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x85 }, 0, .none },
	321	.{ .jng, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8e }, 0, .none },
	322	.{ .jnge, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8c }, 0, .none },
	323	.{ .jnl, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8d }, 0, .none },
	324	.{ .jnle, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8f }, 0, .none },
	325	.{ .jno, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x81 }, 0, .none },
	326	.{ .jnp, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8b }, 0, .none },
	327	.{ .jns, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x89 }, 0, .none },
	328	.{ .jnz, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x85 }, 0, .none },
	329	.{ .jo, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x80 }, 0, .none },
	330	.{ .jp, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8a }, 0, .none },
	331	.{ .jpe, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8a }, 0, .none },
	332	.{ .jpo, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8b }, 0, .none },
	333	.{ .js, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x88 }, 0, .none },
	334	.{ .jz, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x84 }, 0, .none },
	335
	336	.{ .jmp, .d, .rel32, .none, .none, .none, &.{ 0xe9 }, 0, .none },
	337	.{ .jmp, .m, .rm64, .none, .none, .none, &.{ 0xff }, 4, .none },
	338
	339	.{ .lea, .rm, .r16, .m, .none, .none, &.{ 0x8d }, 0, .none },
	340	.{ .lea, .rm, .r32, .m, .none, .none, &.{ 0x8d }, 0, .none },
	341	.{ .lea, .rm, .r64, .m, .none, .none, &.{ 0x8d }, 0, .long },
	342
	343	.{ .lfence, .np, .none, .none, .none, .none, &.{ 0x0f, 0xae, 0xe8 }, 0, .none },
	344
	345	.{ .lods, .np, .m8, .none, .none, .none, &.{ 0xac }, 0, .none },
	346	.{ .lods, .np, .m16, .none, .none, .none, &.{ 0xad }, 0, .none },
	347	.{ .lods, .np, .m32, .none, .none, .none, &.{ 0xad }, 0, .none },
	348	.{ .lods, .np, .m64, .none, .none, .none, &.{ 0xad }, 0, .long },
	349	.{ .lodsb, .np, .none, .none, .none, .none, &.{ 0xac }, 0, .none },
	350	.{ .lodsw, .np, .none, .none, .none, .none, &.{ 0xad }, 0, .short },
	351	.{ .lodsd, .np, .none, .none, .none, .none, &.{ 0xad }, 0, .none },
	352	.{ .lodsq, .np, .none, .none, .none, .none, &.{ 0xad }, 0, .long },
	353
	354	.{ .lzcnt, .rm, .r16, .rm16, .none, .none, &.{ 0xf3, 0x0f, 0xbd }, 0, .none },
	355	.{ .lzcnt, .rm, .r32, .rm32, .none, .none, &.{ 0xf3, 0x0f, 0xbd }, 0, .none },
	356	.{ .lzcnt, .rm, .r64, .rm64, .none, .none, &.{ 0xf3, 0x0f, 0xbd }, 0, .long },
	357
	358	.{ .mfence, .np, .none, .none, .none, .none, &.{ 0x0f, 0xae, 0xf0 }, 0, .none },
	359
	360	.{ .mov, .mr, .rm8, .r8, .none, .none, &.{ 0x88 }, 0, .none },
	361	.{ .mov, .mr, .rm8, .r8, .none, .none, &.{ 0x88 }, 0, .rex },
	362	.{ .mov, .mr, .rm16, .r16, .none, .none, &.{ 0x89 }, 0, .none },
	363	.{ .mov, .mr, .rm32, .r32, .none, .none, &.{ 0x89 }, 0, .none },
	364	.{ .mov, .mr, .rm64, .r64, .none, .none, &.{ 0x89 }, 0, .long },
	365	.{ .mov, .rm, .r8, .rm8, .none, .none, &.{ 0x8a }, 0, .none },
	366	.{ .mov, .rm, .r8, .rm8, .none, .none, &.{ 0x8a }, 0, .rex },
	367	.{ .mov, .rm, .r16, .rm16, .none, .none, &.{ 0x8b }, 0, .none },
	368	.{ .mov, .rm, .r32, .rm32, .none, .none, &.{ 0x8b }, 0, .none },
	369	.{ .mov, .rm, .r64, .rm64, .none, .none, &.{ 0x8b }, 0, .long },
	370	.{ .mov, .mr, .rm16, .sreg, .none, .none, &.{ 0x8c }, 0, .none },
	371	.{ .mov, .mr, .rm64, .sreg, .none, .none, &.{ 0x8c }, 0, .long },
	372	.{ .mov, .rm, .sreg, .rm16, .none, .none, &.{ 0x8e }, 0, .none },
	373	.{ .mov, .rm, .sreg, .rm64, .none, .none, &.{ 0x8e }, 0, .long },
	374	.{ .mov, .fd, .al, .moffs, .none, .none, &.{ 0xa0 }, 0, .none },
	375	.{ .mov, .fd, .ax, .moffs, .none, .none, &.{ 0xa1 }, 0, .none },
	376	.{ .mov, .fd, .eax, .moffs, .none, .none, &.{ 0xa1 }, 0, .none },
	377	.{ .mov, .fd, .rax, .moffs, .none, .none, &.{ 0xa1 }, 0, .long },
	378	.{ .mov, .td, .moffs, .al, .none, .none, &.{ 0xa2 }, 0, .none },
	379	.{ .mov, .td, .moffs, .ax, .none, .none, &.{ 0xa3 }, 0, .none },
	380	.{ .mov, .td, .moffs, .eax, .none, .none, &.{ 0xa3 }, 0, .none },
	381	.{ .mov, .td, .moffs, .rax, .none, .none, &.{ 0xa3 }, 0, .long },
	382	.{ .mov, .oi, .r8, .imm8, .none, .none, &.{ 0xb0 }, 0, .none },
	383	.{ .mov, .oi, .r8, .imm8, .none, .none, &.{ 0xb0 }, 0, .rex },
	384	.{ .mov, .oi, .r16, .imm16, .none, .none, &.{ 0xb8 }, 0, .none },
	385	.{ .mov, .oi, .r32, .imm32, .none, .none, &.{ 0xb8 }, 0, .none },
	386	.{ .mov, .oi, .r64, .imm64, .none, .none, &.{ 0xb8 }, 0, .long },
	387	.{ .mov, .mi, .rm8, .imm8, .none, .none, &.{ 0xc6 }, 0, .none },
	388	.{ .mov, .mi, .rm8, .imm8, .none, .none, &.{ 0xc6 }, 0, .rex },
	389	.{ .mov, .mi, .rm16, .imm16, .none, .none, &.{ 0xc7 }, 0, .none },
	390	.{ .mov, .mi, .rm32, .imm32, .none, .none, &.{ 0xc7 }, 0, .none },
	391	.{ .mov, .mi, .rm64, .imm32s, .none, .none, &.{ 0xc7 }, 0, .long },
	392
	393	.{ .movbe, .rm, .r16, .m16, .none, .none, &.{ 0x0f, 0x38, 0xf0 }, 0, .none },
	394	.{ .movbe, .rm, .r32, .m32, .none, .none, &.{ 0x0f, 0x38, 0xf0 }, 0, .none },
	395	.{ .movbe, .rm, .r64, .m64, .none, .none, &.{ 0x0f, 0x38, 0xf0 }, 0, .long },
	396	.{ .movbe, .mr, .m16, .r16, .none, .none, &.{ 0x0f, 0x38, 0xf1 }, 0, .none },
	397	.{ .movbe, .mr, .m32, .r32, .none, .none, &.{ 0x0f, 0x38, 0xf1 }, 0, .none },
	398	.{ .movbe, .mr, .m64, .r64, .none, .none, &.{ 0x0f, 0x38, 0xf1 }, 0, .long },
	399
	400	.{ .movs, .np, .m8, .m8, .none, .none, &.{ 0xa4 }, 0, .none },
	401	.{ .movs, .np, .m16, .m16, .none, .none, &.{ 0xa5 }, 0, .none },
	402	.{ .movs, .np, .m32, .m32, .none, .none, &.{ 0xa5 }, 0, .none },
	403	.{ .movs, .np, .m64, .m64, .none, .none, &.{ 0xa5 }, 0, .long },
	404	.{ .movsb, .np, .none, .none, .none, .none, &.{ 0xa4 }, 0, .none },
	405	.{ .movsw, .np, .none, .none, .none, .none, &.{ 0xa5 }, 0, .short },
	406	.{ .movsd, .np, .none, .none, .none, .none, &.{ 0xa5 }, 0, .none },
	407	.{ .movsq, .np, .none, .none, .none, .none, &.{ 0xa5 }, 0, .long },
	408
	409	.{ .movsx, .rm, .r16, .rm8, .none, .none, &.{ 0x0f, 0xbe }, 0, .none },
	410	.{ .movsx, .rm, .r16, .rm8, .none, .none, &.{ 0x0f, 0xbe }, 0, .rex },
	411	.{ .movsx, .rm, .r32, .rm8, .none, .none, &.{ 0x0f, 0xbe }, 0, .none },
	412	.{ .movsx, .rm, .r32, .rm8, .none, .none, &.{ 0x0f, 0xbe }, 0, .rex },
	413	.{ .movsx, .rm, .r64, .rm8, .none, .none, &.{ 0x0f, 0xbe }, 0, .long },
	414	.{ .movsx, .rm, .r32, .rm16, .none, .none, &.{ 0x0f, 0xbf }, 0, .none },
	415	.{ .movsx, .rm, .r64, .rm16, .none, .none, &.{ 0x0f, 0xbf }, 0, .long },
327	416
328	417	// This instruction is discouraged.
329		.{ .movsxd, .rm, .r32, .rm32, .none, .none, 1, 0x63, 0x00, 0x00, 0, .none },
330		.{ .movsxd, .rm, .r64, .rm32, .none, .none, 1, 0x63, 0x00, 0x00, 0, .long },
331
332		.{ .movzx, .rm, .r16, .rm8, .none, .none, 2, 0x0f, 0xb6, 0x00, 0, .none },
333		.{ .movzx, .rm, .r32, .rm8, .none, .none, 2, 0x0f, 0xb6, 0x00, 0, .none },
334		.{ .movzx, .rm, .r64, .rm8, .none, .none, 2, 0x0f, 0xb6, 0x00, 0, .long },
335		.{ .movzx, .rm, .r32, .rm16, .none, .none, 2, 0x0f, 0xb7, 0x00, 0, .none },
336		.{ .movzx, .rm, .r64, .rm16, .none, .none, 2, 0x0f, 0xb7, 0x00, 0, .long },
337
338		.{ .mul, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 4, .none },
339		.{ .mul, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 4, .rex },
340		.{ .mul, .m, .rm16, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 4, .none },
341		.{ .mul, .m, .rm32, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 4, .none },
342		.{ .mul, .m, .rm64, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 4, .long },
343
344		.{ .nop, .np, .none, .none, .none, .none, 1, 0x90, 0x00, 0x00, 0, .none },
345
346		.{ .@"or", .zi, .al, .imm8, .none, .none, 1, 0x0c, 0x00, 0x00, 0, .none },
347		.{ .@"or", .zi, .ax, .imm16, .none, .none, 1, 0x0d, 0x00, 0x00, 0, .none },
348		.{ .@"or", .zi, .eax, .imm32, .none, .none, 1, 0x0d, 0x00, 0x00, 0, .none },
349		.{ .@"or", .zi, .rax, .imm32s, .none, .none, 1, 0x0d, 0x00, 0x00, 0, .long },
350		.{ .@"or", .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 1, .none },
351		.{ .@"or", .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 1, .rex },
352		.{ .@"or", .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 1, .none },
353		.{ .@"or", .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 1, .none },
354		.{ .@"or", .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 1, .long },
355		.{ .@"or", .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 1, .none },
356		.{ .@"or", .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 1, .none },
357		.{ .@"or", .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 1, .long },
358		.{ .@"or", .mr, .rm8, .r8, .none, .none, 1, 0x08, 0x00, 0x00, 0, .none },
359		.{ .@"or", .mr, .rm8, .r8, .none, .none, 1, 0x08, 0x00, 0x00, 0, .rex },
360		.{ .@"or", .mr, .rm16, .r16, .none, .none, 1, 0x09, 0x00, 0x00, 0, .none },
361		.{ .@"or", .mr, .rm32, .r32, .none, .none, 1, 0x09, 0x00, 0x00, 0, .none },
362		.{ .@"or", .mr, .rm64, .r64, .none, .none, 1, 0x09, 0x00, 0x00, 0, .long },
363		.{ .@"or", .rm, .r8, .rm8, .none, .none, 1, 0x0a, 0x00, 0x00, 0, .none },
364		.{ .@"or", .rm, .r8, .rm8, .none, .none, 1, 0x0a, 0x00, 0x00, 0, .rex },
365		.{ .@"or", .rm, .r16, .rm16, .none, .none, 1, 0x0b, 0x00, 0x00, 0, .none },
366		.{ .@"or", .rm, .r32, .rm32, .none, .none, 1, 0x0b, 0x00, 0x00, 0, .none },
367		.{ .@"or", .rm, .r64, .rm64, .none, .none, 1, 0x0b, 0x00, 0x00, 0, .long },
368
369		.{ .pop, .o, .r16, .none, .none, .none, 1, 0x58, 0x00, 0x00, 0, .none },
370		.{ .pop, .o, .r64, .none, .none, .none, 1, 0x58, 0x00, 0x00, 0, .none },
371		.{ .pop, .m, .rm16, .none, .none, .none, 1, 0x8f, 0x00, 0x00, 0, .none },
372		.{ .pop, .m, .rm64, .none, .none, .none, 1, 0x8f, 0x00, 0x00, 0, .none },
373
374		.{ .push, .o, .r16, .none, .none, .none, 1, 0x50, 0x00, 0x00, 0, .none },
375		.{ .push, .o, .r64, .none, .none, .none, 1, 0x50, 0x00, 0x00, 0, .none },
376		.{ .push, .m, .rm16, .none, .none, .none, 1, 0xff, 0x00, 0x00, 6, .none },
377		.{ .push, .m, .rm64, .none, .none, .none, 1, 0xff, 0x00, 0x00, 6, .none },
378		.{ .push, .i, .imm8, .none, .none, .none, 1, 0x6a, 0x00, 0x00, 0, .none },
379		.{ .push, .i, .imm16, .none, .none, .none, 1, 0x68, 0x00, 0x00, 0, .none },
380		.{ .push, .i, .imm32, .none, .none, .none, 1, 0x68, 0x00, 0x00, 0, .none },
381
382		.{ .ret, .np, .none, .none, .none, .none, 1, 0xc3, 0x00, 0x00, 0, .none },
383
384		.{ .sal, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 4, .none },
385		.{ .sal, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 4, .rex },
386		.{ .sal, .m1, .rm16, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .none },
387		.{ .sal, .m1, .rm32, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .none },
388		.{ .sal, .m1, .rm64, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .long },
389		.{ .sal, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 4, .none },
390		.{ .sal, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 4, .rex },
391		.{ .sal, .mc, .rm16, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 4, .none },
392		.{ .sal, .mc, .rm32, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 4, .none },
393		.{ .sal, .mc, .rm64, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 4, .long },
394		.{ .sal, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 4, .none },
395		.{ .sal, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 4, .rex },
396		.{ .sal, .mi, .rm16, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 4, .none },
397		.{ .sal, .mi, .rm32, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 4, .none },
398		.{ .sal, .mi, .rm64, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 4, .long },
399
400		.{ .sar, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 7, .none },
401		.{ .sar, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 7, .rex },
402		.{ .sar, .m1, .rm16, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 7, .none },
403		.{ .sar, .m1, .rm32, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 7, .none },
404		.{ .sar, .m1, .rm64, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 7, .long },
405		.{ .sar, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 7, .none },
406		.{ .sar, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 7, .rex },
407		.{ .sar, .mc, .rm16, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 7, .none },
408		.{ .sar, .mc, .rm32, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 7, .none },
409		.{ .sar, .mc, .rm64, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 7, .long },
410		.{ .sar, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 7, .none },
411		.{ .sar, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 7, .rex },
412		.{ .sar, .mi, .rm16, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 7, .none },
413		.{ .sar, .mi, .rm32, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 7, .none },
414		.{ .sar, .mi, .rm64, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 7, .long },
415
416		.{ .sbb, .zi, .al, .imm8, .none, .none, 1, 0x1c, 0x00, 0x00, 0, .none },
417		.{ .sbb, .zi, .ax, .imm16, .none, .none, 1, 0x1d, 0x00, 0x00, 0, .none },
418		.{ .sbb, .zi, .eax, .imm32, .none, .none, 1, 0x1d, 0x00, 0x00, 0, .none },
419		.{ .sbb, .zi, .rax, .imm32s, .none, .none, 1, 0x1d, 0x00, 0x00, 0, .long },
420		.{ .sbb, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 3, .none },
421		.{ .sbb, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 3, .rex },
422		.{ .sbb, .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 3, .none },
423		.{ .sbb, .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 3, .none },
424		.{ .sbb, .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 3, .long },
425		.{ .sbb, .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 3, .none },
426		.{ .sbb, .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 3, .none },
427		.{ .sbb, .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 3, .long },
428		.{ .sbb, .mr, .rm8, .r8, .none, .none, 1, 0x18, 0x00, 0x00, 0, .none },
429		.{ .sbb, .mr, .rm8, .r8, .none, .none, 1, 0x18, 0x00, 0x00, 0, .rex },
430		.{ .sbb, .mr, .rm16, .r16, .none, .none, 1, 0x19, 0x00, 0x00, 0, .none },
431		.{ .sbb, .mr, .rm32, .r32, .none, .none, 1, 0x19, 0x00, 0x00, 0, .none },
432		.{ .sbb, .mr, .rm64, .r64, .none, .none, 1, 0x19, 0x00, 0x00, 0, .long },
433		.{ .sbb, .rm, .r8, .rm8, .none, .none, 1, 0x1a, 0x00, 0x00, 0, .none },
434		.{ .sbb, .rm, .r8, .rm8, .none, .none, 1, 0x1a, 0x00, 0x00, 0, .rex },
435		.{ .sbb, .rm, .r16, .rm16, .none, .none, 1, 0x1b, 0x00, 0x00, 0, .none },
436		.{ .sbb, .rm, .r32, .rm32, .none, .none, 1, 0x1b, 0x00, 0x00, 0, .none },
437		.{ .sbb, .rm, .r64, .rm64, .none, .none, 1, 0x1b, 0x00, 0x00, 0, .long },
438
439		.{ .seta, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x97, 0x00, 0, .none },
440		.{ .seta, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x97, 0x00, 0, .rex },
441		.{ .setae, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .none },
442		.{ .setae, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .rex },
443		.{ .setb, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .none },
444		.{ .setb, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .rex },
445		.{ .setbe, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x96, 0x00, 0, .none },
446		.{ .setbe, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x96, 0x00, 0, .rex },
447		.{ .setc, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .none },
448		.{ .setc, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .rex },
449		.{ .sete, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x94, 0x00, 0, .none },
450		.{ .sete, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x94, 0x00, 0, .rex },
451		.{ .setg, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9f, 0x00, 0, .none },
452		.{ .setg, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9f, 0x00, 0, .rex },
453		.{ .setge, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9d, 0x00, 0, .none },
454		.{ .setge, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9d, 0x00, 0, .rex },
455		.{ .setl, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9c, 0x00, 0, .none },
456		.{ .setl, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9c, 0x00, 0, .rex },
457		.{ .setle, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9e, 0x00, 0, .none },
458		.{ .setle, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9e, 0x00, 0, .rex },
459		.{ .setna, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x96, 0x00, 0, .none },
460		.{ .setna, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x96, 0x00, 0, .rex },
461		.{ .setnae, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .none },
462		.{ .setnae, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .rex },
463		.{ .setnb, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .none },
464		.{ .setnb, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .rex },
465		.{ .setnbe, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x97, 0x00, 0, .none },
466		.{ .setnbe, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x97, 0x00, 0, .rex },
467		.{ .setnc, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .none },
468		.{ .setnc, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .rex },
469		.{ .setne, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x95, 0x00, 0, .none },
470		.{ .setne, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x95, 0x00, 0, .rex },
471		.{ .setng, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9e, 0x00, 0, .none },
472		.{ .setng, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9e, 0x00, 0, .rex },
473		.{ .setnge, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9c, 0x00, 0, .none },
474		.{ .setnge, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9c, 0x00, 0, .rex },
475		.{ .setnl, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9d, 0x00, 0, .none },
476		.{ .setnl, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9d, 0x00, 0, .rex },
477		.{ .setnle, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9f, 0x00, 0, .none },
478		.{ .setnle, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9f, 0x00, 0, .rex },
479		.{ .setno, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x91, 0x00, 0, .none },
480		.{ .setno, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x91, 0x00, 0, .rex },
481		.{ .setnp, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9b, 0x00, 0, .none },
482		.{ .setnp, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9b, 0x00, 0, .rex },
483		.{ .setns, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x99, 0x00, 0, .none },
484		.{ .setns, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x99, 0x00, 0, .rex },
485		.{ .setnz, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x95, 0x00, 0, .none },
486		.{ .setnz, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x95, 0x00, 0, .rex },
487		.{ .seto, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x90, 0x00, 0, .none },
488		.{ .seto, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x90, 0x00, 0, .rex },
489		.{ .setp, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9a, 0x00, 0, .none },
490		.{ .setp, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9a, 0x00, 0, .rex },
491		.{ .setpe, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9a, 0x00, 0, .none },
492		.{ .setpe, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9a, 0x00, 0, .rex },
493		.{ .setpo, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9b, 0x00, 0, .none },
494		.{ .setpo, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9b, 0x00, 0, .rex },
495		.{ .sets, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x98, 0x00, 0, .none },
496		.{ .sets, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x98, 0x00, 0, .rex },
497		.{ .setz, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x94, 0x00, 0, .none },
498		.{ .setz, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x94, 0x00, 0, .rex },
499
500		.{ .shl, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 4, .none },
501		.{ .shl, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 4, .rex },
502		.{ .shl, .m1, .rm16, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .none },
503		.{ .shl, .m1, .rm32, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .none },
504		.{ .shl, .m1, .rm64, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .long },
505		.{ .shl, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 4, .none },
506		.{ .shl, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 4, .rex },
507		.{ .shl, .mc, .rm16, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 4, .none },
508		.{ .shl, .mc, .rm32, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 4, .none },
509		.{ .shl, .mc, .rm64, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 4, .long },
510		.{ .shl, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 4, .none },
511		.{ .shl, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 4, .rex },
512		.{ .shl, .mi, .rm16, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 4, .none },
513		.{ .shl, .mi, .rm32, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 4, .none },
514		.{ .shl, .mi, .rm64, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 4, .long },
515
516		.{ .shr, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 5, .none },
517		.{ .shr, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 5, .rex },
518		.{ .shr, .m1, .rm16, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 5, .none },
519		.{ .shr, .m1, .rm32, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 5, .none },
520		.{ .shr, .m1, .rm64, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 5, .long },
521		.{ .shr, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 5, .none },
522		.{ .shr, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 5, .rex },
523		.{ .shr, .mc, .rm16, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 5, .none },
524		.{ .shr, .mc, .rm32, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 5, .none },
525		.{ .shr, .mc, .rm64, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 5, .long },
526		.{ .shr, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 5, .none },
527		.{ .shr, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 5, .rex },
528		.{ .shr, .mi, .rm16, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 5, .none },
529		.{ .shr, .mi, .rm32, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 5, .none },
530		.{ .shr, .mi, .rm64, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 5, .long },
531
532		.{ .sub, .zi, .al, .imm8, .none, .none, 1, 0x2c, 0x00, 0x00, 0, .none },
533		.{ .sub, .zi, .ax, .imm16, .none, .none, 1, 0x2d, 0x00, 0x00, 0, .none },
534		.{ .sub, .zi, .eax, .imm32, .none, .none, 1, 0x2d, 0x00, 0x00, 0, .none },
535		.{ .sub, .zi, .rax, .imm32s, .none, .none, 1, 0x2d, 0x00, 0x00, 0, .long },
536		.{ .sub, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 5, .none },
537		.{ .sub, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 5, .rex },
538		.{ .sub, .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 5, .none },
539		.{ .sub, .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 5, .none },
540		.{ .sub, .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 5, .long },
541		.{ .sub, .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 5, .none },
542		.{ .sub, .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 5, .none },
543		.{ .sub, .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 5, .long },
544		.{ .sub, .mr, .rm8, .r8, .none, .none, 1, 0x28, 0x00, 0x00, 0, .none },
545		.{ .sub, .mr, .rm8, .r8, .none, .none, 1, 0x28, 0x00, 0x00, 0, .rex },
546		.{ .sub, .mr, .rm16, .r16, .none, .none, 1, 0x29, 0x00, 0x00, 0, .none },
547		.{ .sub, .mr, .rm32, .r32, .none, .none, 1, 0x29, 0x00, 0x00, 0, .none },
548		.{ .sub, .mr, .rm64, .r64, .none, .none, 1, 0x29, 0x00, 0x00, 0, .long },
549		.{ .sub, .rm, .r8, .rm8, .none, .none, 1, 0x2a, 0x00, 0x00, 0, .none },
550		.{ .sub, .rm, .r8, .rm8, .none, .none, 1, 0x2a, 0x00, 0x00, 0, .rex },
551		.{ .sub, .rm, .r16, .rm16, .none, .none, 1, 0x2b, 0x00, 0x00, 0, .none },
552		.{ .sub, .rm, .r32, .rm32, .none, .none, 1, 0x2b, 0x00, 0x00, 0, .none },
553		.{ .sub, .rm, .r64, .rm64, .none, .none, 1, 0x2b, 0x00, 0x00, 0, .long },
554
555		.{ .syscall, .np, .none, .none, .none, .none, 2, 0x0f, 0x05, 0x00, 0, .none },
556
557		.{ .@"test", .zi, .al, .imm8, .none, .none, 1, 0xa8, 0x00, 0x00, 0, .none },
558		.{ .@"test", .zi, .ax, .imm16, .none, .none, 1, 0xa9, 0x00, 0x00, 0, .none },
559		.{ .@"test", .zi, .eax, .imm32, .none, .none, 1, 0xa9, 0x00, 0x00, 0, .none },
560		.{ .@"test", .zi, .rax, .imm32s, .none, .none, 1, 0xa9, 0x00, 0x00, 0, .long },
561		.{ .@"test", .mi, .rm8, .imm8, .none, .none, 1, 0xf6, 0x00, 0x00, 0, .none },
562		.{ .@"test", .mi, .rm8, .imm8, .none, .none, 1, 0xf6, 0x00, 0x00, 0, .rex },
563		.{ .@"test", .mi, .rm16, .imm16, .none, .none, 1, 0xf7, 0x00, 0x00, 0, .none },
564		.{ .@"test", .mi, .rm32, .imm32, .none, .none, 1, 0xf7, 0x00, 0x00, 0, .none },
565		.{ .@"test", .mi, .rm64, .imm32s, .none, .none, 1, 0xf7, 0x00, 0x00, 0, .long },
566		.{ .@"test", .mr, .rm8, .r8, .none, .none, 1, 0x84, 0x00, 0x00, 0, .none },
567		.{ .@"test", .mr, .rm8, .r8, .none, .none, 1, 0x84, 0x00, 0x00, 0, .rex },
568		.{ .@"test", .mr, .rm16, .r16, .none, .none, 1, 0x85, 0x00, 0x00, 0, .none },
569		.{ .@"test", .mr, .rm32, .r32, .none, .none, 1, 0x85, 0x00, 0x00, 0, .none },
570		.{ .@"test", .mr, .rm64, .r64, .none, .none, 1, 0x85, 0x00, 0x00, 0, .long },
571
572		.{ .ud2, .np, .none, .none, .none, .none, 2, 0x0f, 0x0b, 0x00, 0, .none },
573
574		.{ .xor, .zi, .al, .imm8, .none, .none, 1, 0x34, 0x00, 0x00, 0, .none },
575		.{ .xor, .zi, .ax, .imm16, .none, .none, 1, 0x35, 0x00, 0x00, 0, .none },
576		.{ .xor, .zi, .eax, .imm32, .none, .none, 1, 0x35, 0x00, 0x00, 0, .none },
577		.{ .xor, .zi, .rax, .imm32s, .none, .none, 1, 0x35, 0x00, 0x00, 0, .long },
578		.{ .xor, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 6, .none },
579		.{ .xor, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 6, .rex },
580		.{ .xor, .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 6, .none },
581		.{ .xor, .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 6, .none },
582		.{ .xor, .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 6, .long },
583		.{ .xor, .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 6, .none },
584		.{ .xor, .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 6, .none },
585		.{ .xor, .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 6, .long },
586		.{ .xor, .mr, .rm8, .r8, .none, .none, 1, 0x30, 0x00, 0x00, 0, .none },
587		.{ .xor, .mr, .rm8, .r8, .none, .none, 1, 0x30, 0x00, 0x00, 0, .rex },
588		.{ .xor, .mr, .rm16, .r16, .none, .none, 1, 0x31, 0x00, 0x00, 0, .none },
589		.{ .xor, .mr, .rm32, .r32, .none, .none, 1, 0x31, 0x00, 0x00, 0, .none },
590		.{ .xor, .mr, .rm64, .r64, .none, .none, 1, 0x31, 0x00, 0x00, 0, .long },
591		.{ .xor, .rm, .r8, .rm8, .none, .none, 1, 0x32, 0x00, 0x00, 0, .none },
592		.{ .xor, .rm, .r8, .rm8, .none, .none, 1, 0x32, 0x00, 0x00, 0, .rex },
593		.{ .xor, .rm, .r16, .rm16, .none, .none, 1, 0x33, 0x00, 0x00, 0, .none },
594		.{ .xor, .rm, .r32, .rm32, .none, .none, 1, 0x33, 0x00, 0x00, 0, .none },
595		.{ .xor, .rm, .r64, .rm64, .none, .none, 1, 0x33, 0x00, 0x00, 0, .long },
	418	.{ .movsxd, .rm, .r32, .rm32, .none, .none, &.{ 0x63 }, 0, .none },
	419	.{ .movsxd, .rm, .r64, .rm32, .none, .none, &.{ 0x63 }, 0, .long },
	420
	421	.{ .movzx, .rm, .r16, .rm8, .none, .none, &.{ 0x0f, 0xb6 }, 0, .none },
	422	.{ .movzx, .rm, .r32, .rm8, .none, .none, &.{ 0x0f, 0xb6 }, 0, .none },
	423	.{ .movzx, .rm, .r64, .rm8, .none, .none, &.{ 0x0f, 0xb6 }, 0, .long },
	424	.{ .movzx, .rm, .r32, .rm16, .none, .none, &.{ 0x0f, 0xb7 }, 0, .none },
	425	.{ .movzx, .rm, .r64, .rm16, .none, .none, &.{ 0x0f, 0xb7 }, 0, .long },
	426
	427	.{ .mul, .m, .rm8, .none, .none, .none, &.{ 0xf6 }, 4, .none },
	428	.{ .mul, .m, .rm8, .none, .none, .none, &.{ 0xf6 }, 4, .rex },
	429	.{ .mul, .m, .rm16, .none, .none, .none, &.{ 0xf7 }, 4, .none },
	430	.{ .mul, .m, .rm32, .none, .none, .none, &.{ 0xf7 }, 4, .none },
	431	.{ .mul, .m, .rm64, .none, .none, .none, &.{ 0xf7 }, 4, .long },
	432
	433	.{ .neg, .m, .rm8, .none, .none, .none, &.{ 0xf6 }, 3, .none },
	434	.{ .neg, .m, .rm8, .none, .none, .none, &.{ 0xf6 }, 3, .rex },
	435	.{ .neg, .m, .rm16, .none, .none, .none, &.{ 0xf7 }, 3, .none },
	436	.{ .neg, .m, .rm32, .none, .none, .none, &.{ 0xf7 }, 3, .none },
	437	.{ .neg, .m, .rm64, .none, .none, .none, &.{ 0xf7 }, 3, .long },
	438
	439	.{ .nop, .np, .none, .none, .none, .none, &.{ 0x90 }, 0, .none },
	440
	441	.{ .not, .m, .rm8, .none, .none, .none, &.{ 0xf6 }, 2, .none },
	442	.{ .not, .m, .rm8, .none, .none, .none, &.{ 0xf6 }, 2, .rex },
	443	.{ .not, .m, .rm16, .none, .none, .none, &.{ 0xf7 }, 2, .none },
	444	.{ .not, .m, .rm32, .none, .none, .none, &.{ 0xf7 }, 2, .none },
	445	.{ .not, .m, .rm64, .none, .none, .none, &.{ 0xf7 }, 2, .long },
	446
	447	.{ .@"or", .zi, .al, .imm8, .none, .none, &.{ 0x0c }, 0, .none },
	448	.{ .@"or", .zi, .ax, .imm16, .none, .none, &.{ 0x0d }, 0, .none },
	449	.{ .@"or", .zi, .eax, .imm32, .none, .none, &.{ 0x0d }, 0, .none },
	450	.{ .@"or", .zi, .rax, .imm32s, .none, .none, &.{ 0x0d }, 0, .long },
	451	.{ .@"or", .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 1, .none },
	452	.{ .@"or", .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 1, .rex },
	453	.{ .@"or", .mi, .rm16, .imm16, .none, .none, &.{ 0x81 }, 1, .none },
	454	.{ .@"or", .mi, .rm32, .imm32, .none, .none, &.{ 0x81 }, 1, .none },
	455	.{ .@"or", .mi, .rm64, .imm32s, .none, .none, &.{ 0x81 }, 1, .long },
	456	.{ .@"or", .mi, .rm16, .imm8s, .none, .none, &.{ 0x83 }, 1, .none },
	457	.{ .@"or", .mi, .rm32, .imm8s, .none, .none, &.{ 0x83 }, 1, .none },
	458	.{ .@"or", .mi, .rm64, .imm8s, .none, .none, &.{ 0x83 }, 1, .long },
	459	.{ .@"or", .mr, .rm8, .r8, .none, .none, &.{ 0x08 }, 0, .none },
	460	.{ .@"or", .mr, .rm8, .r8, .none, .none, &.{ 0x08 }, 0, .rex },
	461	.{ .@"or", .mr, .rm16, .r16, .none, .none, &.{ 0x09 }, 0, .none },
	462	.{ .@"or", .mr, .rm32, .r32, .none, .none, &.{ 0x09 }, 0, .none },
	463	.{ .@"or", .mr, .rm64, .r64, .none, .none, &.{ 0x09 }, 0, .long },
	464	.{ .@"or", .rm, .r8, .rm8, .none, .none, &.{ 0x0a }, 0, .none },
	465	.{ .@"or", .rm, .r8, .rm8, .none, .none, &.{ 0x0a }, 0, .rex },
	466	.{ .@"or", .rm, .r16, .rm16, .none, .none, &.{ 0x0b }, 0, .none },
	467	.{ .@"or", .rm, .r32, .rm32, .none, .none, &.{ 0x0b }, 0, .none },
	468	.{ .@"or", .rm, .r64, .rm64, .none, .none, &.{ 0x0b }, 0, .long },
	469
	470	.{ .pop, .o, .r16, .none, .none, .none, &.{ 0x58 }, 0, .none },
	471	.{ .pop, .o, .r64, .none, .none, .none, &.{ 0x58 }, 0, .none },
	472	.{ .pop, .m, .rm16, .none, .none, .none, &.{ 0x8f }, 0, .none },
	473	.{ .pop, .m, .rm64, .none, .none, .none, &.{ 0x8f }, 0, .none },
	474
	475	.{ .popcnt, .rm, .r16, .rm16, .none, .none, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none },
	476	.{ .popcnt, .rm, .r32, .rm32, .none, .none, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none },
	477	.{ .popcnt, .rm, .r64, .rm64, .none, .none, &.{ 0xf3, 0x0f, 0xb8 }, 0, .long },
	478
	479	.{ .push, .o, .r16, .none, .none, .none, &.{ 0x50 }, 0, .none },
	480	.{ .push, .o, .r64, .none, .none, .none, &.{ 0x50 }, 0, .none },
	481	.{ .push, .m, .rm16, .none, .none, .none, &.{ 0xff }, 6, .none },
	482	.{ .push, .m, .rm64, .none, .none, .none, &.{ 0xff }, 6, .none },
	483	.{ .push, .i, .imm8, .none, .none, .none, &.{ 0x6a }, 0, .none },
	484	.{ .push, .i, .imm16, .none, .none, .none, &.{ 0x68 }, 0, .none },
	485	.{ .push, .i, .imm32, .none, .none, .none, &.{ 0x68 }, 0, .none },
	486
	487	.{ .ret, .np, .none, .none, .none, .none, &.{ 0xc3 }, 0, .none },
	488
	489	.{ .rcl, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 2, .none },
	490	.{ .rcl, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 2, .rex },
	491	.{ .rcl, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 2, .none },
	492	.{ .rcl, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 2, .rex },
	493	.{ .rcl, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 2, .none },
	494	.{ .rcl, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 2, .rex },
	495	.{ .rcl, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 2, .none },
	496	.{ .rcl, .mc, .rm16, .cl, .none, .none, &.{ 0xd3 }, 2, .none },
	497	.{ .rcl, .mi, .rm16, .imm8, .none, .none, &.{ 0xc1 }, 2, .none },
	498	.{ .rcl, .m1, .rm32, .unity, .none, .none, &.{ 0xd1 }, 2, .none },
	499	.{ .rcl, .m1, .rm64, .unity, .none, .none, &.{ 0xd1 }, 2, .long },
	500	.{ .rcl, .mc, .rm32, .cl, .none, .none, &.{ 0xd3 }, 2, .none },
	501	.{ .rcl, .mc, .rm64, .cl, .none, .none, &.{ 0xd3 }, 2, .long },
	502	.{ .rcl, .mi, .rm32, .imm8, .none, .none, &.{ 0xc1 }, 2, .none },
	503	.{ .rcl, .mi, .rm64, .imm8, .none, .none, &.{ 0xc1 }, 2, .long },
	504
	505	.{ .rcr, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 3, .none },
	506	.{ .rcr, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 3, .rex },
	507	.{ .rcr, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 3, .none },
	508	.{ .rcr, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 3, .rex },
	509	.{ .rcr, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 3, .none },
	510	.{ .rcr, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 3, .rex },
	511	.{ .rcr, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 3, .none },
	512	.{ .rcr, .mc, .rm16, .cl, .none, .none, &.{ 0xd3 }, 3, .none },
	513	.{ .rcr, .mi, .rm16, .imm8, .none, .none, &.{ 0xc1 }, 3, .none },
	514	.{ .rcr, .m1, .rm32, .unity, .none, .none, &.{ 0xd1 }, 3, .none },
	515	.{ .rcr, .m1, .rm64, .unity, .none, .none, &.{ 0xd1 }, 3, .long },
	516	.{ .rcr, .mc, .rm32, .cl, .none, .none, &.{ 0xd3 }, 3, .none },
	517	.{ .rcr, .mc, .rm64, .cl, .none, .none, &.{ 0xd3 }, 3, .long },
	518	.{ .rcr, .mi, .rm32, .imm8, .none, .none, &.{ 0xc1 }, 3, .none },
	519	.{ .rcr, .mi, .rm64, .imm8, .none, .none, &.{ 0xc1 }, 3, .long },
	520
	521	.{ .rol, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 0, .none },
	522	.{ .rol, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 0, .rex },
	523	.{ .rol, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 0, .none },
	524	.{ .rol, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 0, .rex },
	525	.{ .rol, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 0, .none },
	526	.{ .rol, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 0, .rex },
	527	.{ .rol, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 0, .none },
	528	.{ .rol, .mc, .rm16, .cl, .none, .none, &.{ 0xd3 }, 0, .none },
	529	.{ .rol, .mi, .rm16, .imm8, .none, .none, &.{ 0xc1 }, 0, .none },
	530	.{ .rol, .m1, .rm32, .unity, .none, .none, &.{ 0xd1 }, 0, .none },
	531	.{ .rol, .m1, .rm64, .unity, .none, .none, &.{ 0xd1 }, 0, .long },
	532	.{ .rol, .mc, .rm32, .cl, .none, .none, &.{ 0xd3 }, 0, .none },
	533	.{ .rol, .mc, .rm64, .cl, .none, .none, &.{ 0xd3 }, 0, .long },
	534	.{ .rol, .mi, .rm32, .imm8, .none, .none, &.{ 0xc1 }, 0, .none },
	535	.{ .rol, .mi, .rm64, .imm8, .none, .none, &.{ 0xc1 }, 0, .long },
	536
	537	.{ .ror, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 1, .none },
	538	.{ .ror, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 1, .rex },
	539	.{ .ror, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 1, .none },
	540	.{ .ror, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 1, .rex },
	541	.{ .ror, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 1, .none },
	542	.{ .ror, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 1, .rex },
	543	.{ .ror, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 1, .none },
	544	.{ .ror, .mc, .rm16, .cl, .none, .none, &.{ 0xd3 }, 1, .none },
	545	.{ .ror, .mi, .rm16, .imm8, .none, .none, &.{ 0xc1 }, 1, .none },
	546	.{ .ror, .m1, .rm32, .unity, .none, .none, &.{ 0xd1 }, 1, .none },
	547	.{ .ror, .m1, .rm64, .unity, .none, .none, &.{ 0xd1 }, 1, .long },
	548	.{ .ror, .mc, .rm32, .cl, .none, .none, &.{ 0xd3 }, 1, .none },
	549	.{ .ror, .mc, .rm64, .cl, .none, .none, &.{ 0xd3 }, 1, .long },
	550	.{ .ror, .mi, .rm32, .imm8, .none, .none, &.{ 0xc1 }, 1, .none },
	551	.{ .ror, .mi, .rm64, .imm8, .none, .none, &.{ 0xc1 }, 1, .long },
	552
	553	.{ .sal, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 4, .none },
	554	.{ .sal, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 4, .rex },
	555	.{ .sal, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 4, .none },
	556	.{ .sal, .m1, .rm32, .unity, .none, .none, &.{ 0xd1 }, 4, .none },
	557	.{ .sal, .m1, .rm64, .unity, .none, .none, &.{ 0xd1 }, 4, .long },
	558	.{ .sal, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 4, .none },
	559	.{ .sal, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 4, .rex },
	560	.{ .sal, .mc, .rm16, .cl, .none, .none, &.{ 0xd3 }, 4, .none },
	561	.{ .sal, .mc, .rm32, .cl, .none, .none, &.{ 0xd3 }, 4, .none },
	562	.{ .sal, .mc, .rm64, .cl, .none, .none, &.{ 0xd3 }, 4, .long },
	563	.{ .sal, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 4, .none },
	564	.{ .sal, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 4, .rex },
	565	.{ .sal, .mi, .rm16, .imm8, .none, .none, &.{ 0xc1 }, 4, .none },
	566	.{ .sal, .mi, .rm32, .imm8, .none, .none, &.{ 0xc1 }, 4, .none },
	567	.{ .sal, .mi, .rm64, .imm8, .none, .none, &.{ 0xc1 }, 4, .long },
	568
	569	.{ .sar, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 7, .none },
	570	.{ .sar, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 7, .rex },
	571	.{ .sar, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 7, .none },
	572	.{ .sar, .m1, .rm32, .unity, .none, .none, &.{ 0xd1 }, 7, .none },
	573	.{ .sar, .m1, .rm64, .unity, .none, .none, &.{ 0xd1 }, 7, .long },
	574	.{ .sar, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 7, .none },
	575	.{ .sar, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 7, .rex },
	576	.{ .sar, .mc, .rm16, .cl, .none, .none, &.{ 0xd3 }, 7, .none },
	577	.{ .sar, .mc, .rm32, .cl, .none, .none, &.{ 0xd3 }, 7, .none },
	578	.{ .sar, .mc, .rm64, .cl, .none, .none, &.{ 0xd3 }, 7, .long },
	579	.{ .sar, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 7, .none },
	580	.{ .sar, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 7, .rex },
	581	.{ .sar, .mi, .rm16, .imm8, .none, .none, &.{ 0xc1 }, 7, .none },
	582	.{ .sar, .mi, .rm32, .imm8, .none, .none, &.{ 0xc1 }, 7, .none },
	583	.{ .sar, .mi, .rm64, .imm8, .none, .none, &.{ 0xc1 }, 7, .long },
	584
	585	.{ .sbb, .zi, .al, .imm8, .none, .none, &.{ 0x1c }, 0, .none },
	586	.{ .sbb, .zi, .ax, .imm16, .none, .none, &.{ 0x1d }, 0, .none },
	587	.{ .sbb, .zi, .eax, .imm32, .none, .none, &.{ 0x1d }, 0, .none },
	588	.{ .sbb, .zi, .rax, .imm32s, .none, .none, &.{ 0x1d }, 0, .long },
	589	.{ .sbb, .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 3, .none },
	590	.{ .sbb, .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 3, .rex },
	591	.{ .sbb, .mi, .rm16, .imm16, .none, .none, &.{ 0x81 }, 3, .none },
	592	.{ .sbb, .mi, .rm32, .imm32, .none, .none, &.{ 0x81 }, 3, .none },
	593	.{ .sbb, .mi, .rm64, .imm32s, .none, .none, &.{ 0x81 }, 3, .long },
	594	.{ .sbb, .mi, .rm16, .imm8s, .none, .none, &.{ 0x83 }, 3, .none },
	595	.{ .sbb, .mi, .rm32, .imm8s, .none, .none, &.{ 0x83 }, 3, .none },
	596	.{ .sbb, .mi, .rm64, .imm8s, .none, .none, &.{ 0x83 }, 3, .long },
	597	.{ .sbb, .mr, .rm8, .r8, .none, .none, &.{ 0x18 }, 0, .none },
	598	.{ .sbb, .mr, .rm8, .r8, .none, .none, &.{ 0x18 }, 0, .rex },
	599	.{ .sbb, .mr, .rm16, .r16, .none, .none, &.{ 0x19 }, 0, .none },
	600	.{ .sbb, .mr, .rm32, .r32, .none, .none, &.{ 0x19 }, 0, .none },
	601	.{ .sbb, .mr, .rm64, .r64, .none, .none, &.{ 0x19 }, 0, .long },
	602	.{ .sbb, .rm, .r8, .rm8, .none, .none, &.{ 0x1a }, 0, .none },
	603	.{ .sbb, .rm, .r8, .rm8, .none, .none, &.{ 0x1a }, 0, .rex },
	604	.{ .sbb, .rm, .r16, .rm16, .none, .none, &.{ 0x1b }, 0, .none },
	605	.{ .sbb, .rm, .r32, .rm32, .none, .none, &.{ 0x1b }, 0, .none },
	606	.{ .sbb, .rm, .r64, .rm64, .none, .none, &.{ 0x1b }, 0, .long },
	607
	608	.{ .scas, .np, .m8, .none, .none, .none, &.{ 0xae }, 0, .none },
	609	.{ .scas, .np, .m16, .none, .none, .none, &.{ 0xaf }, 0, .none },
	610	.{ .scas, .np, .m32, .none, .none, .none, &.{ 0xaf }, 0, .none },
	611	.{ .scas, .np, .m64, .none, .none, .none, &.{ 0xaf }, 0, .long },
	612	.{ .scasb, .np, .none, .none, .none, .none, &.{ 0xae }, 0, .none },
	613	.{ .scasw, .np, .none, .none, .none, .none, &.{ 0xaf }, 0, .short },
	614	.{ .scasd, .np, .none, .none, .none, .none, &.{ 0xaf }, 0, .none },
	615	.{ .scasq, .np, .none, .none, .none, .none, &.{ 0xaf }, 0, .long },
	616
	617	.{ .seta, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x97 }, 0, .none },
	618	.{ .seta, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x97 }, 0, .rex },
	619	.{ .setae, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x93 }, 0, .none },
	620	.{ .setae, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x93 }, 0, .rex },
	621	.{ .setb, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x92 }, 0, .none },
	622	.{ .setb, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x92 }, 0, .rex },
	623	.{ .setbe, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x96 }, 0, .none },
	624	.{ .setbe, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x96 }, 0, .rex },
	625	.{ .setc, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x92 }, 0, .none },
	626	.{ .setc, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x92 }, 0, .rex },
	627	.{ .sete, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x94 }, 0, .none },
	628	.{ .sete, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x94 }, 0, .rex },
	629	.{ .setg, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9f }, 0, .none },
	630	.{ .setg, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9f }, 0, .rex },
	631	.{ .setge, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9d }, 0, .none },
	632	.{ .setge, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9d }, 0, .rex },
	633	.{ .setl, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9c }, 0, .none },
	634	.{ .setl, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9c }, 0, .rex },
	635	.{ .setle, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9e }, 0, .none },
	636	.{ .setle, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9e }, 0, .rex },
	637	.{ .setna, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x96 }, 0, .none },
	638	.{ .setna, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x96 }, 0, .rex },
	639	.{ .setnae, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x92 }, 0, .none },
	640	.{ .setnae, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x92 }, 0, .rex },
	641	.{ .setnb, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x93 }, 0, .none },
	642	.{ .setnb, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x93 }, 0, .rex },
	643	.{ .setnbe, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x97 }, 0, .none },
	644	.{ .setnbe, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x97 }, 0, .rex },
	645	.{ .setnc, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x93 }, 0, .none },
	646	.{ .setnc, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x93 }, 0, .rex },
	647	.{ .setne, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x95 }, 0, .none },
	648	.{ .setne, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x95 }, 0, .rex },
	649	.{ .setng, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9e }, 0, .none },
	650	.{ .setng, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9e }, 0, .rex },
	651	.{ .setnge, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9c }, 0, .none },
	652	.{ .setnge, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9c }, 0, .rex },
	653	.{ .setnl, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9d }, 0, .none },
	654	.{ .setnl, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9d }, 0, .rex },
	655	.{ .setnle, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9f }, 0, .none },
	656	.{ .setnle, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9f }, 0, .rex },
	657	.{ .setno, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x91 }, 0, .none },
	658	.{ .setno, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x91 }, 0, .rex },
	659	.{ .setnp, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9b }, 0, .none },
	660	.{ .setnp, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9b }, 0, .rex },
	661	.{ .setns, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x99 }, 0, .none },
	662	.{ .setns, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x99 }, 0, .rex },
	663	.{ .setnz, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x95 }, 0, .none },
	664	.{ .setnz, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x95 }, 0, .rex },
	665	.{ .seto, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x90 }, 0, .none },
	666	.{ .seto, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x90 }, 0, .rex },
	667	.{ .setp, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9a }, 0, .none },
	668	.{ .setp, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9a }, 0, .rex },
	669	.{ .setpe, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9a }, 0, .none },
	670	.{ .setpe, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9a }, 0, .rex },
	671	.{ .setpo, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9b }, 0, .none },
	672	.{ .setpo, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9b }, 0, .rex },
	673	.{ .sets, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x98 }, 0, .none },
	674	.{ .sets, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x98 }, 0, .rex },
	675	.{ .setz, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x94 }, 0, .none },
	676	.{ .setz, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x94 }, 0, .rex },
	677
	678	.{ .sfence, .np, .none, .none, .none, .none, &.{ 0x0f, 0xae, 0xf8 }, 0, .none },
	679
	680	.{ .shl, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 4, .none },
	681	.{ .shl, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 4, .rex },
	682	.{ .shl, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 4, .none },
	683	.{ .shl, .m1, .rm32, .unity, .none, .none, &.{ 0xd1 }, 4, .none },
	684	.{ .shl, .m1, .rm64, .unity, .none, .none, &.{ 0xd1 }, 4, .long },
	685	.{ .shl, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 4, .none },
	686	.{ .shl, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 4, .rex },
	687	.{ .shl, .mc, .rm16, .cl, .none, .none, &.{ 0xd3 }, 4, .none },
	688	.{ .shl, .mc, .rm32, .cl, .none, .none, &.{ 0xd3 }, 4, .none },
	689	.{ .shl, .mc, .rm64, .cl, .none, .none, &.{ 0xd3 }, 4, .long },
	690	.{ .shl, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 4, .none },
	691	.{ .shl, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 4, .rex },
	692	.{ .shl, .mi, .rm16, .imm8, .none, .none, &.{ 0xc1 }, 4, .none },
	693	.{ .shl, .mi, .rm32, .imm8, .none, .none, &.{ 0xc1 }, 4, .none },
	694	.{ .shl, .mi, .rm64, .imm8, .none, .none, &.{ 0xc1 }, 4, .long },
	695
	696	.{ .shr, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 5, .none },
	697	.{ .shr, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 5, .rex },
	698	.{ .shr, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 5, .none },
	699	.{ .shr, .m1, .rm32, .unity, .none, .none, &.{ 0xd1 }, 5, .none },
	700	.{ .shr, .m1, .rm64, .unity, .none, .none, &.{ 0xd1 }, 5, .long },
	701	.{ .shr, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 5, .none },
	702	.{ .shr, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 5, .rex },
	703	.{ .shr, .mc, .rm16, .cl, .none, .none, &.{ 0xd3 }, 5, .none },
	704	.{ .shr, .mc, .rm32, .cl, .none, .none, &.{ 0xd3 }, 5, .none },
	705	.{ .shr, .mc, .rm64, .cl, .none, .none, &.{ 0xd3 }, 5, .long },
	706	.{ .shr, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 5, .none },
	707	.{ .shr, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 5, .rex },
	708	.{ .shr, .mi, .rm16, .imm8, .none, .none, &.{ 0xc1 }, 5, .none },
	709	.{ .shr, .mi, .rm32, .imm8, .none, .none, &.{ 0xc1 }, 5, .none },
	710	.{ .shr, .mi, .rm64, .imm8, .none, .none, &.{ 0xc1 }, 5, .long },
	711
	712	.{ .stos, .np, .m8, .none, .none, .none, &.{ 0xaa }, 0, .none },
	713	.{ .stos, .np, .m16, .none, .none, .none, &.{ 0xab }, 0, .none },
	714	.{ .stos, .np, .m32, .none, .none, .none, &.{ 0xab }, 0, .none },
	715	.{ .stos, .np, .m64, .none, .none, .none, &.{ 0xab }, 0, .long },
	716	.{ .stosb, .np, .none, .none, .none, .none, &.{ 0xaa }, 0, .none },
	717	.{ .stosw, .np, .none, .none, .none, .none, &.{ 0xab }, 0, .short },
	718	.{ .stosd, .np, .none, .none, .none, .none, &.{ 0xab }, 0, .none },
	719	.{ .stosq, .np, .none, .none, .none, .none, &.{ 0xab }, 0, .long },
	720
	721	.{ .sub, .zi, .al, .imm8, .none, .none, &.{ 0x2c }, 0, .none },
	722	.{ .sub, .zi, .ax, .imm16, .none, .none, &.{ 0x2d }, 0, .none },
	723	.{ .sub, .zi, .eax, .imm32, .none, .none, &.{ 0x2d }, 0, .none },
	724	.{ .sub, .zi, .rax, .imm32s, .none, .none, &.{ 0x2d }, 0, .long },
	725	.{ .sub, .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 5, .none },
	726	.{ .sub, .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 5, .rex },
	727	.{ .sub, .mi, .rm16, .imm16, .none, .none, &.{ 0x81 }, 5, .none },
	728	.{ .sub, .mi, .rm32, .imm32, .none, .none, &.{ 0x81 }, 5, .none },
	729	.{ .sub, .mi, .rm64, .imm32s, .none, .none, &.{ 0x81 }, 5, .long },
	730	.{ .sub, .mi, .rm16, .imm8s, .none, .none, &.{ 0x83 }, 5, .none },
	731	.{ .sub, .mi, .rm32, .imm8s, .none, .none, &.{ 0x83 }, 5, .none },
	732	.{ .sub, .mi, .rm64, .imm8s, .none, .none, &.{ 0x83 }, 5, .long },
	733	.{ .sub, .mr, .rm8, .r8, .none, .none, &.{ 0x28 }, 0, .none },
	734	.{ .sub, .mr, .rm8, .r8, .none, .none, &.{ 0x28 }, 0, .rex },
	735	.{ .sub, .mr, .rm16, .r16, .none, .none, &.{ 0x29 }, 0, .none },
	736	.{ .sub, .mr, .rm32, .r32, .none, .none, &.{ 0x29 }, 0, .none },
	737	.{ .sub, .mr, .rm64, .r64, .none, .none, &.{ 0x29 }, 0, .long },
	738	.{ .sub, .rm, .r8, .rm8, .none, .none, &.{ 0x2a }, 0, .none },
	739	.{ .sub, .rm, .r8, .rm8, .none, .none, &.{ 0x2a }, 0, .rex },
	740	.{ .sub, .rm, .r16, .rm16, .none, .none, &.{ 0x2b }, 0, .none },
	741	.{ .sub, .rm, .r32, .rm32, .none, .none, &.{ 0x2b }, 0, .none },
	742	.{ .sub, .rm, .r64, .rm64, .none, .none, &.{ 0x2b }, 0, .long },
	743
	744	.{ .syscall, .np, .none, .none, .none, .none, &.{ 0x0f, 0x05 }, 0, .none }
	745	,
	746	.{ .@"test", .zi, .al, .imm8, .none, .none, &.{ 0xa8 }, 0, .none },
	747	.{ .@"test", .zi, .ax, .imm16, .none, .none, &.{ 0xa9 }, 0, .none },
	748	.{ .@"test", .zi, .eax, .imm32, .none, .none, &.{ 0xa9 }, 0, .none },
	749	.{ .@"test", .zi, .rax, .imm32s, .none, .none, &.{ 0xa9 }, 0, .long },
	750	.{ .@"test", .mi, .rm8, .imm8, .none, .none, &.{ 0xf6 }, 0, .none },
	751	.{ .@"test", .mi, .rm8, .imm8, .none, .none, &.{ 0xf6 }, 0, .rex },
	752	.{ .@"test", .mi, .rm16, .imm16, .none, .none, &.{ 0xf7 }, 0, .none },
	753	.{ .@"test", .mi, .rm32, .imm32, .none, .none, &.{ 0xf7 }, 0, .none },
	754	.{ .@"test", .mi, .rm64, .imm32s, .none, .none, &.{ 0xf7 }, 0, .long },
	755	.{ .@"test", .mr, .rm8, .r8, .none, .none, &.{ 0x84 }, 0, .none },
	756	.{ .@"test", .mr, .rm8, .r8, .none, .none, &.{ 0x84 }, 0, .rex },
	757	.{ .@"test", .mr, .rm16, .r16, .none, .none, &.{ 0x85 }, 0, .none },
	758	.{ .@"test", .mr, .rm32, .r32, .none, .none, &.{ 0x85 }, 0, .none },
	759	.{ .@"test", .mr, .rm64, .r64, .none, .none, &.{ 0x85 }, 0, .long },
	760
	761	.{ .tzcnt, .rm, .r16, .rm16, .none, .none, &.{ 0xf3, 0x0f, 0xbc }, 0, .none },
	762	.{ .tzcnt, .rm, .r32, .rm32, .none, .none, &.{ 0xf3, 0x0f, 0xbc }, 0, .none },
	763	.{ .tzcnt, .rm, .r64, .rm64, .none, .none, &.{ 0xf3, 0x0f, 0xbc }, 0, .long },
	764
	765	.{ .ud2, .np, .none, .none, .none, .none, &.{ 0x0f, 0x0b }, 0, .none },
	766
	767	.{ .xadd, .mr, .rm8, .r8, .none, .none, &.{ 0x0f, 0xc0 }, 0, .none },
	768	.{ .xadd, .mr, .rm8, .r8, .none, .none, &.{ 0x0f, 0xc0 }, 0, .rex },
	769	.{ .xadd, .mr, .rm16, .r16, .none, .none, &.{ 0x0f, 0xc1 }, 0, .none },
	770	.{ .xadd, .mr, .rm32, .r32, .none, .none, &.{ 0x0f, 0xc1 }, 0, .none },
	771	.{ .xadd, .mr, .rm64, .r64, .none, .none, &.{ 0x0f, 0xc1 }, 0, .long },
	772
	773	.{ .xchg, .o, .ax, .r16, .none, .none, &.{ 0x90 }, 0, .none },
	774	.{ .xchg, .o, .r16, .ax, .none, .none, &.{ 0x90 }, 0, .none },
	775	.{ .xchg, .o, .eax, .r32, .none, .none, &.{ 0x90 }, 0, .none },
	776	.{ .xchg, .o, .rax, .r64, .none, .none, &.{ 0x90 }, 0, .long },
	777	.{ .xchg, .o, .r32, .eax, .none, .none, &.{ 0x90 }, 0, .none },
	778	.{ .xchg, .o, .r64, .rax, .none, .none, &.{ 0x90 }, 0, .long },
	779	.{ .xchg, .mr, .rm8, .r8, .none, .none, &.{ 0x86 }, 0, .none },
	780	.{ .xchg, .mr, .rm8, .r8, .none, .none, &.{ 0x86 }, 0, .rex },
	781	.{ .xchg, .rm, .r8, .rm8, .none, .none, &.{ 0x86 }, 0, .none },
	782	.{ .xchg, .rm, .r8, .rm8, .none, .none, &.{ 0x86 }, 0, .rex },
	783	.{ .xchg, .mr, .rm16, .r16, .none, .none, &.{ 0x87 }, 0, .none },
	784	.{ .xchg, .rm, .r16, .rm16, .none, .none, &.{ 0x87 }, 0, .none },
	785	.{ .xchg, .mr, .rm32, .r32, .none, .none, &.{ 0x87 }, 0, .none },
	786	.{ .xchg, .mr, .rm64, .r64, .none, .none, &.{ 0x87 }, 0, .long },
	787	.{ .xchg, .rm, .r32, .rm32, .none, .none, &.{ 0x87 }, 0, .none },
	788	.{ .xchg, .rm, .r64, .rm64, .none, .none, &.{ 0x87 }, 0, .long },
	789
	790	.{ .xor, .zi, .al, .imm8, .none, .none, &.{ 0x34 }, 0, .none },
	791	.{ .xor, .zi, .ax, .imm16, .none, .none, &.{ 0x35 }, 0, .none },
	792	.{ .xor, .zi, .eax, .imm32, .none, .none, &.{ 0x35 }, 0, .none },
	793	.{ .xor, .zi, .rax, .imm32s, .none, .none, &.{ 0x35 }, 0, .long },
	794	.{ .xor, .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 6, .none },
	795	.{ .xor, .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 6, .rex },
	796	.{ .xor, .mi, .rm16, .imm16, .none, .none, &.{ 0x81 }, 6, .none },
	797	.{ .xor, .mi, .rm32, .imm32, .none, .none, &.{ 0x81 }, 6, .none },
	798	.{ .xor, .mi, .rm64, .imm32s, .none, .none, &.{ 0x81 }, 6, .long },
	799	.{ .xor, .mi, .rm16, .imm8s, .none, .none, &.{ 0x83 }, 6, .none },
	800	.{ .xor, .mi, .rm32, .imm8s, .none, .none, &.{ 0x83 }, 6, .none },
	801	.{ .xor, .mi, .rm64, .imm8s, .none, .none, &.{ 0x83 }, 6, .long },
	802	.{ .xor, .mr, .rm8, .r8, .none, .none, &.{ 0x30 }, 0, .none },
	803	.{ .xor, .mr, .rm8, .r8, .none, .none, &.{ 0x30 }, 0, .rex },
	804	.{ .xor, .mr, .rm16, .r16, .none, .none, &.{ 0x31 }, 0, .none },
	805	.{ .xor, .mr, .rm32, .r32, .none, .none, &.{ 0x31 }, 0, .none },
	806	.{ .xor, .mr, .rm64, .r64, .none, .none, &.{ 0x31 }, 0, .long },
	807	.{ .xor, .rm, .r8, .rm8, .none, .none, &.{ 0x32 }, 0, .none },
	808	.{ .xor, .rm, .r8, .rm8, .none, .none, &.{ 0x32 }, 0, .rex },
	809	.{ .xor, .rm, .r16, .rm16, .none, .none, &.{ 0x33 }, 0, .none },
	810	.{ .xor, .rm, .r32, .rm32, .none, .none, &.{ 0x33 }, 0, .none },
	811	.{ .xor, .rm, .r64, .rm64, .none, .none, &.{ 0x33 }, 0, .long },
596	812
597	813	// SSE
598		.{ .addss, .rm, .xmm, .xmm_m32, .none, .none, 3, 0xf3, 0x0f, 0x58, 0, .sse },
	814	.{ .addss, .rm, .xmm, .xmm_m32, .none, .none, &.{ 0xf3, 0x0f, 0x58 }, 0, .sse },
599	815
600		.{ .cmpss, .rmi, .xmm, .xmm_m32, .imm8, .none, 3, 0xf3, 0x0f, 0xc2, 0, .sse },
	816	.{ .cmpss, .rmi, .xmm, .xmm_m32, .imm8, .none, &.{ 0xf3, 0x0f, 0xc2 }, 0, .sse },
601	817
602		.{ .movss, .rm, .xmm, .xmm_m32, .none, .none, 3, 0xf3, 0x0f, 0x10, 0, .sse },
603		.{ .movss, .mr, .xmm_m32, .xmm, .none, .none, 3, 0xf3, 0x0f, 0x11, 0, .sse },
	818	.{ .divss, .rm, .xmm, .xmm_m32, .none, .none, &.{ 0xf3, 0x0f, 0x5e }, 0, .sse },
604	819
605		.{ .ucomiss, .rm, .xmm, .xmm_m32, .none, .none, 2, 0x0f, 0x2e, 0x00, 0, .sse },
	820	.{ .maxss, .rm, .xmm, .xmm_m32, .none, .none, &.{ 0xf3, 0x0f, 0x5f }, 0, .sse },
	821
	822	.{ .minss, .rm, .xmm, .xmm_m32, .none, .none, &.{ 0xf3, 0x0f, 0x5d }, 0, .sse },
	823
	824	.{ .movss, .rm, .xmm, .xmm_m32, .none, .none, &.{ 0xf3, 0x0f, 0x10 }, 0, .sse },
	825	.{ .movss, .mr, .xmm_m32, .xmm, .none, .none, &.{ 0xf3, 0x0f, 0x11 }, 0, .sse },
	826
	827	.{ .mulss, .rm, .xmm, .xmm_m32, .none, .none, &.{ 0xf3, 0x0f, 0x59 }, 0, .sse },
	828
	829	.{ .subss, .rm, .xmm, .xmm_m32, .none, .none, &.{ 0xf3, 0x0f, 0x5c }, 0, .sse },
	830
	831	.{ .ucomiss, .rm, .xmm, .xmm_m32, .none, .none, &.{ 0x0f, 0x2e }, 0, .sse },
606	832
607	833	// SSE2
608		.{ .addsd, .rm, .xmm, .xmm_m64, .none, .none, 3, 0xf2, 0x0f, 0x58, 0, .sse2 },
	834	.{ .addsd, .rm, .xmm, .xmm_m64, .none, .none, &.{ 0xf2, 0x0f, 0x58 }, 0, .sse2 },
	835
	836	.{ .cmpsd, .rmi, .xmm, .xmm_m64, .imm8, .none, &.{ 0xf2, 0x0f, 0xc2 }, 0, .sse2 },
609	837
610		.{ .cmpsd, .rmi, .xmm, .xmm_m64, .imm8, .none, 3, 0xf2, 0x0f, 0xc2, 0, .sse2 },
	838	.{ .divsd, .rm, .xmm, .xmm_m64, .none, .none, &.{ 0xf2, 0x0f, 0x5e }, 0, .sse2 },
611	839
612		.{ .movq, .rm, .xmm, .xmm_m64, .none, .none, 3, 0xf3, 0x0f, 0x7e, 0, .sse2 },
613		.{ .movq, .mr, .xmm_m64, .xmm, .none, .none, 3, 0x66, 0x0f, 0xd6, 0, .sse2 },
	840	.{ .maxsd, .rm, .xmm, .xmm_m64, .none, .none, &.{ 0xf2, 0x0f, 0x5f }, 0, .sse2 },
614	841
615		.{ .movsd, .rm, .xmm, .xmm_m64, .none, .none, 3, 0xf2, 0x0f, 0x10, 0, .sse2 },
616		.{ .movsd, .mr, .xmm_m64, .xmm, .none, .none, 3, 0xf2, 0x0f, 0x11, 0, .sse2 },
	842	.{ .minsd, .rm, .xmm, .xmm_m64, .none, .none, &.{ 0xf2, 0x0f, 0x5d }, 0, .sse2 },
617	843
618		.{ .ucomisd, .rm, .xmm, .xmm_m64, .none, .none, 3, 0x66, 0x0f, 0x2e, 0, .sse2 },
	844	.{ .movq, .rm, .xmm, .xmm_m64, .none, .none, &.{ 0xf3, 0x0f, 0x7e }, 0, .sse2 },
	845	.{ .movq, .mr, .xmm_m64, .xmm, .none, .none, &.{ 0x66, 0x0f, 0xd6 }, 0, .sse2 },
	846
	847	.{ .mulsd, .rm, .xmm, .xmm_m64, .none, .none, &.{ 0xf2, 0x0f, 0x59 }, 0, .sse2 },
	848
	849	.{ .subsd, .rm, .xmm, .xmm_m64, .none, .none, &.{ 0xf2, 0x0f, 0x5c }, 0, .sse2 },
	850
	851	.{ .movsd, .rm, .xmm, .xmm_m64, .none, .none, &.{ 0xf2, 0x0f, 0x10 }, 0, .sse2 },
	852	.{ .movsd, .mr, .xmm_m64, .xmm, .none, .none, &.{ 0xf2, 0x0f, 0x11 }, 0, .sse2 },
	853
	854	.{ .ucomisd, .rm, .xmm, .xmm_m64, .none, .none, &.{ 0x66, 0x0f, 0x2e }, 0, .sse2 },
	855
	856	// SSE4.1
	857	.{ .roundss, .rmi, .xmm, .xmm_m32, .imm8, .none, &.{ 0x66, 0x0f, 0x3a, 0x0a }, 0, .sse4_1 },
	858	.{ .roundsd, .rmi, .xmm, .xmm_m64, .imm8, .none, &.{ 0x66, 0x0f, 0x3a, 0x0b }, 0, .sse4_1 },
619	859	};
620	860	// zig fmt: on
621

src/codegen.zig+3-2

...	...	@@ -608,7 +608,6 @@ pub fn generateSymbol(
608	608	const payload_type = typed_value.ty.optionalChild(&opt_buf);
609	609	const is_pl = !typed_value.val.isNull();
610	610	const abi_size = math.cast(usize, typed_value.ty.abiSize(target)) orelse return error.Overflow;
611		const offset = abi_size - (math.cast(usize, payload_type.abiSize(target)) orelse return error.Overflow);
612	611
613	612	if (!payload_type.hasRuntimeBits()) {
614	613	try code.writer().writeByteNTimes(@boolToInt(is_pl), abi_size);
...	...	@@ -639,8 +638,8 @@ pub fn generateSymbol(
639	638	return Result.ok;
640	639	}
641	640
	641	const padding = abi_size - (math.cast(usize, payload_type.abiSize(target)) orelse return error.Overflow) - 1;
642	642	const value = if (typed_value.val.castTag(.opt_payload)) \|payload\| payload.data else Value.initTag(.undef);
643		try code.writer().writeByteNTimes(@boolToInt(is_pl), offset);
644	643	switch (try generateSymbol(bin_file, src_loc, .{
645	644	.ty = payload_type,
646	645	.val = value,
...	...	@@ -648,6 +647,8 @@ pub fn generateSymbol(
648	647	.ok => {},
649	648	.fail => \|em\| return Result{ .fail = em },
650	649	}
	650	try code.writer().writeByte(@boolToInt(is_pl));
	651	try code.writer().writeByteNTimes(0, padding);
651	652
652	653	return Result.ok;
653	654	},

src/register_manager.zig+16-24

...	...	@@ -305,40 +305,32 @@ pub fn RegisterManager(
305	305	pub fn getReg(self: *Self, reg: Register, inst: ?Air.Inst.Index) AllocateRegistersError!void {
306	306	const index = indexOfRegIntoTracked(reg) orelse return;
307	307	log.debug("getReg {} for inst {?}", .{ reg, inst });
308		self.markRegAllocated(reg);
309	308
310		if (inst) \|tracked_inst\|
311		if (!self.isRegFree(reg)) {
312		// Move the instruction that was previously there to a
313		// stack allocation.
314		const spilled_inst = self.registers[index];
315		self.registers[index] = tracked_inst;
316		try self.getFunction().spillInstruction(reg, spilled_inst);
317		} else {
318		self.getRegAssumeFree(reg, tracked_inst);
319		}
320		else {
321		if (!self.isRegFree(reg)) {
322		// Move the instruction that was previously there to a
323		// stack allocation.
324		const spilled_inst = self.registers[index];
325		try self.getFunction().spillInstruction(reg, spilled_inst);
326		self.freeReg(reg);
327		}
328		}
	309	if (!self.isRegFree(reg)) {
	310	self.markRegAllocated(reg);
	311
	312	// Move the instruction that was previously there to a
	313	// stack allocation.
	314	const spilled_inst = self.registers[index];
	315	if (inst) \|tracked_inst\| self.registers[index] = tracked_inst;
	316	try self.getFunction().spillInstruction(reg, spilled_inst);
	317	if (inst == null) self.freeReg(reg);
	318	} else self.getRegAssumeFree(reg, inst);
329	319	}
330	320
331	321	/// Allocates the specified register with the specified
332	322	/// instruction. Asserts that the register is free and no
333	323	/// spilling is necessary.
334		pub fn getRegAssumeFree(self: *Self, reg: Register, inst: Air.Inst.Index) void {
	324	pub fn getRegAssumeFree(self: *Self, reg: Register, inst: ?Air.Inst.Index) void {
335	325	const index = indexOfRegIntoTracked(reg) orelse return;
336		log.debug("getRegAssumeFree {} for inst {}", .{ reg, inst });
	326	log.debug("getRegAssumeFree {} for inst {?}", .{ reg, inst });
337	327	self.markRegAllocated(reg);
338	328
339	329	assert(self.isRegFree(reg));
340		self.registers[index] = inst;
341		self.markRegUsed(reg);
	330	if (inst) \|tracked_inst\| {
	331	self.registers[index] = tracked_inst;
	332	self.markRegUsed(reg);
	333	}
342	334	}
343	335
344	336	/// Marks the specified register as free

test/behavior/atomics.zig-7

...	...	@@ -33,7 +33,6 @@ fn testCmpxchg() !void {
33	33
34	34	test "fence" {
35	35	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
36		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
37	36	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
38	37	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
39	38
...	...	@@ -44,7 +43,6 @@ test "fence" {
44	43
45	44	test "atomicrmw and atomicload" {
46	45	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
47		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
48	46	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
49	47	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
50	48	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
...	...	@@ -73,7 +71,6 @@ fn testAtomicLoad(ptr: *u8) !void {
73	71
74	72	test "cmpxchg with ptr" {
75	73	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
76		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
77	74	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
78	75	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
79	76	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
...	...	@@ -162,7 +159,6 @@ test "cmpxchg on a global variable" {
162	159
163	160	test "atomic load and rmw with enum" {
164	161	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
165		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
166	162	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
167	163	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
168	164	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
...	...	@@ -180,7 +176,6 @@ test "atomic load and rmw with enum" {
180	176
181	177	test "atomic store" {
182	178	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
183		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
184	179	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
185	180	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
186	181	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
...	...	@@ -194,7 +189,6 @@ test "atomic store" {
194	189
195	190	test "atomic store comptime" {
196	191	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
197		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
198	192	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
199	193	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
200	194	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
...	...	@@ -424,7 +418,6 @@ fn testAtomicsWithType(comptime T: type, a: T, b: T) !void {
424	418
425	419	test "return @atomicStore, using it as a void value" {
426	420	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
427		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
428	421	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
429	422	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
430	423	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO

test/behavior/bugs/12984.zig-1

...	...	@@ -14,7 +14,6 @@ pub const CustomDraw = DeleagateWithContext(fn (?OnConfirm) void);
14	14	test "simple test" {
15	15	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
16	16	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
17		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
18	17
19	18	var c: CustomDraw = undefined;
20	19	_ = c;

test/behavior/bugs/13068.zig-1

...	...	@@ -8,7 +8,6 @@ test {
8	8	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
9	9	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
10	10	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
11		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
12	11	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
13	12
14	13	list.items.len = 0;

test/behavior/bugs/13785.zig-1

...	...	@@ -3,7 +3,6 @@ const std = @import("std");
3	3
4	4	const S = packed struct { a: u0 = 0 };
5	5	test {
6		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
7	6	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
8	7	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
9	8	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/call.zig-1

...	...	@@ -329,7 +329,6 @@ test "inline call preserves tail call" {
329	329	test "inline call doesn't re-evaluate non generic struct" {
330	330	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
331	331	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
332		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
333	332
334	333	const S = struct {
335	334	fn foo(f: struct { a: u8, b: u8 }) !void {

test/behavior/cast.zig-2

...	...	@@ -655,7 +655,6 @@ test "@floatCast cast down" {
655	655	}
656	656
657	657	test "peer type resolution: unreachable, error set, unreachable" {
658		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
659	658	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
660	659	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
661	660
...	...	@@ -1206,7 +1205,6 @@ fn cast128Float(x: u128) f128 {
1206	1205	test "implicit cast from [N]T to ?[]T" {
1207	1206	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
1208	1207	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1209		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1210	1208	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1211	1209
1212	1210	var x: ?[*]u16 = null;

test/behavior/error.zig+1-2

...	...	@@ -451,7 +451,6 @@ test "optional error set is the same size as error set" {
451	451	}
452	452
453	453	test "nested catch" {
454		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
455	454	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
456	455	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
457	456
...	...	@@ -731,7 +730,6 @@ test "pointer to error union payload" {
731	730	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
732	731	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
733	732	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
734		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
735	733	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
736	734
737	735	var err_union: anyerror!u8 = 15;
...	...	@@ -876,6 +874,7 @@ test "field access of anyerror results in smaller error set" {
876	874	}
877	875
878	876	test "optional error union return type" {
	877	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
879	878	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
880	879
881	880	const S = struct {

test/behavior/if.zig-1

...	...	@@ -130,7 +130,6 @@ test "if peer expressions inferred optional type" {
130	130	}
131	131
132	132	test "if-else expression with runtime condition result location is inferred optional" {
133		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
134	133	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
135	134	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
136	135	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/maximum_minimum.zig-2

...	...	@@ -5,7 +5,6 @@ const expect = std.testing.expect;
5	5	const expectEqual = std.testing.expectEqual;
6	6
7	7	test "@max" {
8		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
9	8	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
10	9	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
11	10	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -52,7 +51,6 @@ test "@max on vectors" {
52	51	}
53	52
54	53	test "@min" {
55		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
56	54	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
57	55	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
58	56	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/merge_error_sets.zig-1

...	...	@@ -12,7 +12,6 @@ fn foo() C!void {
12	12	}
13	13
14	14	test "merge error sets" {
15		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
16	15	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
17	16
18	17	if (foo()) {

test/behavior/null.zig-2

...	...	@@ -29,7 +29,6 @@ test "optional type" {
29	29	}
30	30
31	31	test "test maybe object and get a pointer to the inner value" {
32		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
33	32	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
34	33	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
35	34	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -138,7 +137,6 @@ test "optional pointer to 0 bit type null value at runtime" {
138	137	}
139	138
140	139	test "if var maybe pointer" {
141		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
142	140	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
143	141	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
144	142	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/optional.zig-7

...	...	@@ -91,7 +91,6 @@ test "address of unwrap optional" {
91	91	test "nested optional field in struct" {
92	92	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
93	93	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
94		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
95	94	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
96	95
97	96	const S2 = struct {
...	...	@@ -109,7 +108,6 @@ test "nested optional field in struct" {
109	108	test "equality compare optional with non-optional" {
110	109	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
111	110	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
112		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
113	111	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
114	112
115	113	try test_cmp_optional_non_optional();
...	...	@@ -227,7 +225,6 @@ test "assigning to an unwrapped optional field in an inline loop" {
227	225	}
228	226
229	227	test "coerce an anon struct literal to optional struct" {
230		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
231	228	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
232	229	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
233	230	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -247,7 +244,6 @@ test "coerce an anon struct literal to optional struct" {
247	244	}
248	245
249	246	test "0-bit child type coerced to optional return ptr result location" {
250		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
251	247	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
252	248	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
253	249	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -299,7 +295,6 @@ test "0-bit child type coerced to optional" {
299	295	}
300	296
301	297	test "array of optional unaligned types" {
302		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
303	298	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
304	299	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
305	300	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -336,7 +331,6 @@ test "array of optional unaligned types" {
336	331	}
337	332
338	333	test "optional pointer to zero bit optional payload" {
339		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
340	334	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
341	335	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
342	336	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -450,7 +444,6 @@ test "Optional slice size is optimized" {
450	444	test "peer type resolution in nested if expressions" {
451	445	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
452	446	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
453		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
454	447
455	448	const Thing = struct { n: i32 };
456	449	var a = false;

test/behavior/ptrcast.zig-1

...	...	@@ -18,7 +18,6 @@ fn testReinterpretBytesAsInteger() !void {
18	18	}
19	19
20	20	test "reinterpret an array over multiple elements, with no well-defined layout" {
21		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
22	21	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
23	22	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
24	23	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/return_address.zig-1

...	...	@@ -7,7 +7,6 @@ fn retAddr() usize {
7	7
8	8	test "return address" {
9	9	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
10		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
11	10	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
12	11	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
13	12	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/struct.zig-3

...	...	@@ -1149,7 +1149,6 @@ test "anon init through error unions and optionals" {
1149	1149	}
1150	1150
1151	1151	test "anon init through optional" {
1152		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1153	1152	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1154	1153	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1155	1154	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -1456,7 +1455,6 @@ test "struct has only one reference" {
1456	1455	test "no dependency loop on pointer to optional struct" {
1457	1456	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
1458	1457	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1459		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1460	1458
1461	1459	const S = struct {
1462	1460	const A = struct { b: B };
...	...	@@ -1509,7 +1507,6 @@ test "no dependency loop on optional field wrapped in generic function" {
1509	1507	}
1510	1508
1511	1509	test "optional field init with tuple" {
1512		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1513	1510	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1514	1511	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1515	1512

test/behavior/switch.zig-1

...	...	@@ -228,7 +228,6 @@ const SwitchProngWithVarEnum = union(enum) {
228	228	};
229	229
230	230	test "switch prong with variable" {
231		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
232	231	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
233	232	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
234	233	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/tuple.zig-4

...	...	@@ -263,7 +263,6 @@ test "initializing anon struct with mixed comptime-runtime fields" {
263	263	test "tuple in tuple passed to generic function" {
264	264	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
265	265	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
266		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
267	266	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
268	267
269	268	const S = struct {
...	...	@@ -283,7 +282,6 @@ test "tuple in tuple passed to generic function" {
283	282	test "coerce tuple to tuple" {
284	283	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
285	284	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
286		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
287	285	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
288	286
289	287	const T = std.meta.Tuple(&.{u8});
...	...	@@ -298,7 +296,6 @@ test "coerce tuple to tuple" {
298	296	test "tuple type with void field" {
299	297	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
300	298	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
301		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
302	299
303	300	const T = std.meta.Tuple(&[_]type{void});
304	301	const x = T{{}};
...	...	@@ -335,7 +332,6 @@ test "zero sized struct in tuple handled correctly" {
335	332	test "tuple type with void field and a runtime field" {
336	333	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
337	334	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
338		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
339	335
340	336	const T = std.meta.Tuple(&[_]type{ usize, void });
341	337	var t: T = .{ 5, {} };

test/behavior/union.zig-1

...	...	@@ -1227,7 +1227,6 @@ test "union tag is set when initiated as a temporary value at runtime" {
1227	1227	}
1228	1228
1229	1229	test "extern union most-aligned field is smaller" {
1230		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1231	1230	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1232	1231	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1233	1232	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/while.zig-1

...	...	@@ -341,7 +341,6 @@ test "else continue outer while" {
341	341	}
342	342
343	343	test "try terminating an infinite loop" {
344		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
345	344	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
346	345	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
347	346