Commit f40539e5d83365412bf8c6973ce867125ea36faf | zig

author	Jakub Konka <kubkon@jakubkonka.com> 2023-05-08 19:04:05+02:00
committer	GitHub <noreply@github.com> 2023-05-08 19:04:05+02:00
log	f40539e5d83365412bf8c6973ce867125ea36faf
tree	c8b6c458a2a20c1846d57721de33d2befed750e7
parent	16314e0e199051289075c9e3dab425ce3fb52719
parent	1f5aa7747f5710e281cd2190508ce562a4bfd35f
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #15597 from jacobly0/x86_64-behavior

x86_64: more behavior than ever before

18 files changed, 4760 insertions(+), 2753 deletions(-)

lib
- std
  - target
    x86.zig
src
- arch
  - x86_64
    CodeGen.zig
    Emit.zig
    Encoding.zig
    Lower.zig
    Mir.zig
    bits.zig
    encoder.zig
    encodings.zig
- link
  - Dwarf.zig
test
- behavior

lib/std/target/x86.zig+3-3

src/arch/x86_64/CodeGen.zig+2140-944

...	@@ -205,16 +205,7 @@ pub const MCValue = union(enum) {	...	@@ -205,16 +205,7 @@ pub const MCValue = union(enum) {
205		205
206	fn isMemory(mcv: MCValue) bool {	206	fn isMemory(mcv: MCValue) bool {
207	return switch (mcv) {	207	return switch (mcv) {
208	.memory,	208	.memory, .indirect, .load_frame => true,
209	.load_direct,
210	.lea_direct,
211	.load_got,
212	.lea_got,
213	.load_tlv,
214	.lea_tlv,
215	.load_frame,
216	.lea_frame,
217	=> true,
218	else => false,	209	else => false,
219	};	210	};
220	}	211	}
...	@@ -937,7 +928,7 @@ fn formatWipMir(	...	@@ -937,7 +928,7 @@ fn formatWipMir(
937	.target = data.self.target,	928	.target = data.self.target,
938	.src_loc = data.self.src_loc,	929	.src_loc = data.self.src_loc,
939	};	930	};
940	for (lower.lowerMir(data.self.mir_instructions.get(data.inst)) catch \|err\| switch (err) {	931	for ((lower.lowerMir(data.inst) catch \|err\| switch (err) {
941	error.LowerFail => {	932	error.LowerFail => {
942	defer {	933	defer {
943	lower.err_msg.?.deinit(data.self.gpa);	934	lower.err_msg.?.deinit(data.self.gpa);
...	@@ -955,7 +946,7 @@ fn formatWipMir(	...	@@ -955,7 +946,7 @@ fn formatWipMir(
955	return;	946	return;
956	},	947	},
957	else => \|e\| return e,	948	else => \|e\| return e,
958	}) \|lower_inst\| try writer.print(" \| {}", .{lower_inst});	949	}).insts) \|lowered_inst\| try writer.print(" \| {}", .{lowered_inst});
959	}	950	}
960	fn fmtWipMir(self: *Self, inst: Mir.Inst.Index) std.fmt.Formatter(formatWipMir) {	951	fn fmtWipMir(self: *Self, inst: Mir.Inst.Index) std.fmt.Formatter(formatWipMir) {
961	return .{ .data = .{ .self = self, .inst = inst } };	952	return .{ .data = .{ .self = self, .inst = inst } };
...	@@ -982,14 +973,14 @@ fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {	...	@@ -982,14 +973,14 @@ fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {
982	try self.mir_instructions.ensureUnusedCapacity(gpa, 1);	973	try self.mir_instructions.ensureUnusedCapacity(gpa, 1);
983	const result_index = @intCast(Mir.Inst.Index, self.mir_instructions.len);	974	const result_index = @intCast(Mir.Inst.Index, self.mir_instructions.len);
984	self.mir_instructions.appendAssumeCapacity(inst);	975	self.mir_instructions.appendAssumeCapacity(inst);
985	switch (inst.tag) {	976	if (inst.tag != .pseudo or switch (inst.ops) {
986	else => wip_mir_log.debug("{}", .{self.fmtWipMir(result_index)}),	977	else => true,
987	.dbg_line,	978	.pseudo_dbg_prologue_end_none,
988	.dbg_prologue_end,	979	.pseudo_dbg_line_line_column,
989	.dbg_epilogue_begin,	980	.pseudo_dbg_epilogue_begin_none,
990	.dead,	981	.pseudo_dead_none,
991	=> {},	982	=> false,
992	}	983	}) wip_mir_log.debug("{}", .{self.fmtWipMir(result_index)});
993	return result_index;	984	return result_index;
994	}	985	}
995		986
...	@@ -1012,131 +1003,248 @@ fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {	...	@@ -1012,131 +1003,248 @@ fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {
1012	return result;	1003	return result;
1013	}	1004	}
1014		1005
1015	fn asmSetccRegister(self: *Self, reg: Register, cc: bits.Condition) !void {	1006	/// A `cc` of `.z_and_np` clobbers `reg2`!
		1007	fn asmCmovccRegisterRegister(self: *Self, reg1: Register, reg2: Register, cc: bits.Condition) !void {
1016	_ = try self.addInst(.{	1008	_ = try self.addInst(.{
1017	.tag = .setcc,	1009	.tag = switch (cc) {
1018	.ops = .r_cc,	1010	else => .cmov,
1019	.data = .{ .r_cc = .{ .r = reg, .cc = cc } },	1011	.z_and_np, .nz_or_p => .pseudo,
		1012	},
		1013	.ops = switch (cc) {
		1014	else => .rr,
		1015	.z_and_np => .pseudo_cmov_z_and_np_rr,
		1016	.nz_or_p => .pseudo_cmov_nz_or_p_rr,
		1017	},
		1018	.data = .{ .rr = .{
		1019	.fixes = switch (cc) {
		1020	else => Mir.Inst.Fixes.fromCondition(cc),
		1021	.z_and_np, .nz_or_p => ._,
		1022	},
		1023	.r1 = reg1,
		1024	.r2 = reg2,
		1025	} },
1020	});	1026	});
1021	}	1027	}
1022		1028
1023	fn asmSetccMemory(self: *Self, m: Memory, cc: bits.Condition) !void {	1029	/// A `cc` of `.z_and_np` is not supported by this encoding!
		1030	fn asmCmovccRegisterMemory(self: *Self, reg: Register, m: Memory, cc: bits.Condition) !void {
1024	_ = try self.addInst(.{	1031	_ = try self.addInst(.{
1025	.tag = .setcc,	1032	.tag = switch (cc) {
1026	.ops = switch (m) {	1033	else => .cmov,
1027	.sib => .m_sib_cc,	1034	.z_and_np => unreachable,
1028	.rip => .m_rip_cc,	1035	.nz_or_p => .pseudo,
1029	else => unreachable,
1030	},	1036	},
1031	.data = .{ .x_cc = .{ .cc = cc, .payload = switch (m) {	1037	.ops = switch (cc) {
1032	.sib => try self.addExtra(Mir.MemorySib.encode(m)),	1038	else => switch (m) {
1033	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),	1039	.sib => .rm_sib,
1034	else => unreachable,	1040	.rip => .rm_rip,
1035	} } },	1041	else => unreachable,
		1042	},
		1043	.z_and_np => unreachable,
		1044	.nz_or_p => switch (m) {
		1045	.sib => .pseudo_cmov_nz_or_p_rm_sib,
		1046	.rip => .pseudo_cmov_nz_or_p_rm_rip,
		1047	else => unreachable,
		1048	},
		1049	},
		1050	.data = .{ .rx = .{
		1051	.fixes = switch (cc) {
		1052	else => Mir.Inst.Fixes.fromCondition(cc),
		1053	.z_and_np => unreachable,
		1054	.nz_or_p => ._,
		1055	},
		1056	.r1 = reg,
		1057	.payload = switch (m) {
		1058	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
		1059	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
		1060	else => unreachable,
		1061	},
		1062	} },
1036	});	1063	});
1037	}	1064	}
1038		1065
1039	fn asmCmovccRegisterRegister(self: *Self, reg1: Register, reg2: Register, cc: bits.Condition) !void {	1066	fn asmSetccRegister(self: *Self, reg: Register, cc: bits.Condition) !void {
1040	_ = try self.addInst(.{	1067	_ = try self.addInst(.{
1041	.tag = .cmovcc,	1068	.tag = switch (cc) {
1042	.ops = .rr_cc,	1069	else => .set,
1043	.data = .{ .rr_cc = .{ .r1 = reg1, .r2 = reg2, .cc = cc } },	1070	.z_and_np, .nz_or_p => .pseudo,
		1071	},
		1072	.ops = switch (cc) {
		1073	else => .r,
		1074	.z_and_np => .pseudo_set_z_and_np_r,
		1075	.nz_or_p => .pseudo_set_nz_or_p_r,
		1076	},
		1077	.data = switch (cc) {
		1078	else => .{ .r = .{
		1079	.fixes = Mir.Inst.Fixes.fromCondition(cc),
		1080	.r1 = reg,
		1081	} },
		1082	.z_and_np, .nz_or_p => .{ .r_scratch = .{
		1083	.r1 = reg,
		1084	.scratch_reg = (try self.register_manager.allocReg(null, gp)).to8(),
		1085	} },
		1086	},
1044	});	1087	});
1045	}	1088	}
1046		1089
1047	fn asmCmovccRegisterMemory(self: *Self, reg: Register, m: Memory, cc: bits.Condition) !void {	1090	fn asmSetccMemory(self: *Self, m: Memory, cc: bits.Condition) !void {
		1091	const payload = switch (m) {
		1092	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
		1093	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
		1094	else => unreachable,
		1095	};
1048	_ = try self.addInst(.{	1096	_ = try self.addInst(.{
1049	.tag = .cmovcc,	1097	.tag = switch (cc) {
1050	.ops = switch (m) {	1098	else => .set,
1051	.sib => .rm_sib_cc,	1099	.z_and_np, .nz_or_p => .pseudo,
1052	.rip => .rm_rip_cc,	1100	},
1053	else => unreachable,	1101	.ops = switch (cc) {
		1102	else => switch (m) {
		1103	.sib => .m_sib,
		1104	.rip => .m_rip,
		1105	else => unreachable,
		1106	},
		1107	.z_and_np => switch (m) {
		1108	.sib => .pseudo_set_z_and_np_m_sib,
		1109	.rip => .pseudo_set_z_and_np_m_rip,
		1110	else => unreachable,
		1111	},
		1112	.nz_or_p => switch (m) {
		1113	.sib => .pseudo_set_nz_or_p_m_sib,
		1114	.rip => .pseudo_set_nz_or_p_m_rip,
		1115	else => unreachable,
		1116	},
		1117	},
		1118	.data = switch (cc) {
		1119	else => .{ .x = .{
		1120	.fixes = Mir.Inst.Fixes.fromCondition(cc),
		1121	.payload = payload,
		1122	} },
		1123	.z_and_np, .nz_or_p => .{ .x_scratch = .{
		1124	.scratch_reg = (try self.register_manager.allocReg(null, gp)).to8(),
		1125	.payload = payload,
		1126	} },
1054	},	1127	},
1055	.data = .{ .rx_cc = .{ .r = reg, .cc = cc, .payload = switch (m) {
1056	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
1057	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
1058	else => unreachable,
1059	} } },
1060	});	1128	});
1061	}	1129	}
1062		1130
1063	fn asmJmpReloc(self: *Self, target: Mir.Inst.Index) !Mir.Inst.Index {	1131	fn asmJmpReloc(self: *Self, target: Mir.Inst.Index) !Mir.Inst.Index {
1064	return self.addInst(.{	1132	return self.addInst(.{
1065	.tag = .jmp_reloc,	1133	.tag = .jmp,
1066	.ops = undefined,	1134	.ops = .inst,
1067	.data = .{ .inst = target },	1135	.data = .{ .inst = .{
		1136	.inst = target,
		1137	} },
1068	});	1138	});
1069	}	1139	}
1070		1140
1071	fn asmJccReloc(self: *Self, target: Mir.Inst.Index, cc: bits.Condition) !Mir.Inst.Index {	1141	fn asmJccReloc(self: *Self, target: Mir.Inst.Index, cc: bits.Condition) !Mir.Inst.Index {
1072	return self.addInst(.{	1142	return self.addInst(.{
1073	.tag = .jcc,	1143	.tag = switch (cc) {
1074	.ops = .inst_cc,	1144	else => .j,
1075	.data = .{ .inst_cc = .{ .inst = target, .cc = cc } },	1145	.z_and_np, .nz_or_p => .pseudo,
		1146	},
		1147	.ops = switch (cc) {
		1148	else => .inst,
		1149	.z_and_np => .pseudo_j_z_and_np_inst,
		1150	.nz_or_p => .pseudo_j_nz_or_p_inst,
		1151	},
		1152	.data = .{ .inst = .{
		1153	.fixes = switch (cc) {
		1154	else => Mir.Inst.Fixes.fromCondition(cc),
		1155	.z_and_np, .nz_or_p => ._,
		1156	},
		1157	.inst = target,
		1158	} },
1076	});	1159	});
1077	}	1160	}
1078		1161
1079	fn asmPlaceholder(self: *Self) !Mir.Inst.Index {	1162	fn asmPlaceholder(self: *Self) !Mir.Inst.Index {
1080	return self.addInst(.{	1163	return self.addInst(.{
1081	.tag = .dead,	1164	.tag = .pseudo,
1082	.ops = undefined,	1165	.ops = .pseudo_dead_none,
1083	.data = undefined,	1166	.data = undefined,
1084	});	1167	});
1085	}	1168	}
1086		1169
1087	fn asmOpOnly(self: *Self, tag: Mir.Inst.Tag) !void {	1170	fn asmOpOnly(self: *Self, tag: Mir.Inst.FixedTag) !void {
1088	_ = try self.addInst(.{	1171	_ = try self.addInst(.{
1089	.tag = tag,	1172	.tag = tag[1],
1090	.ops = .none,	1173	.ops = .none,
		1174	.data = .{ .none = .{
		1175	.fixes = tag[0],
		1176	} },
		1177	});
		1178	}
		1179
		1180	fn asmPseudo(self: *Self, ops: Mir.Inst.Ops) !void {
		1181	_ = try self.addInst(.{
		1182	.tag = .pseudo,
		1183	.ops = ops,
1091	.data = undefined,	1184	.data = undefined,
1092	});	1185	});
1093	}	1186	}
1094		1187
1095	fn asmRegister(self: *Self, tag: Mir.Inst.Tag, reg: Register) !void {	1188	fn asmRegister(self: *Self, tag: Mir.Inst.FixedTag, reg: Register) !void {
1096	_ = try self.addInst(.{	1189	_ = try self.addInst(.{
1097	.tag = tag,	1190	.tag = tag[1],
1098	.ops = .r,	1191	.ops = .r,
1099	.data = .{ .r = reg },	1192	.data = .{ .r = .{
		1193	.fixes = tag[0],
		1194	.r1 = reg,
		1195	} },
1100	});	1196	});
1101	}	1197	}
1102		1198
1103	fn asmImmediate(self: *Self, tag: Mir.Inst.Tag, imm: Immediate) !void {	1199	fn asmImmediate(self: *Self, tag: Mir.Inst.FixedTag, imm: Immediate) !void {
1104	_ = try self.addInst(.{	1200	_ = try self.addInst(.{
1105	.tag = tag,	1201	.tag = tag[1],
1106	.ops = switch (imm) {	1202	.ops = switch (imm) {
1107	.signed => .i_s,	1203	.signed => .i_s,
1108	.unsigned => .i_u,	1204	.unsigned => .i_u,
1109	},	1205	},
1110	.data = .{ .i = switch (imm) {	1206	.data = .{ .i = .{
1111	.signed => \|s\| @bitCast(u32, s),	1207	.fixes = tag[0],
1112	.unsigned => \|u\| @intCast(u32, u),	1208	.i = switch (imm) {
		1209	.signed => \|s\| @bitCast(u32, s),
		1210	.unsigned => \|u\| @intCast(u32, u),
		1211	},
1113	} },	1212	} },
1114	});	1213	});
1115	}	1214	}
1116		1215
1117	fn asmRegisterRegister(self: *Self, tag: Mir.Inst.Tag, reg1: Register, reg2: Register) !void {	1216	fn asmRegisterRegister(self: *Self, tag: Mir.Inst.FixedTag, reg1: Register, reg2: Register) !void {
1118	_ = try self.addInst(.{	1217	_ = try self.addInst(.{
1119	.tag = tag,	1218	.tag = tag[1],
1120	.ops = .rr,	1219	.ops = .rr,
1121	.data = .{ .rr = .{ .r1 = reg1, .r2 = reg2 } },	1220	.data = .{ .rr = .{
		1221	.fixes = tag[0],
		1222	.r1 = reg1,
		1223	.r2 = reg2,
		1224	} },
1122	});	1225	});
1123	}	1226	}
1124		1227
1125	fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.Tag, reg: Register, imm: Immediate) !void {	1228	fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, imm: Immediate) !void {
1126	const ops: Mir.Inst.Ops = switch (imm) {	1229	const ops: Mir.Inst.Ops = switch (imm) {
1127	.signed => .ri_s,	1230	.signed => .ri_s,
1128	.unsigned => \|u\| if (math.cast(u32, u)) \|_\| .ri_u else .ri64,	1231	.unsigned => \|u\| if (math.cast(u32, u)) \|_\| .ri_u else .ri64,
1129	};	1232	};
1130	_ = try self.addInst(.{	1233	_ = try self.addInst(.{
1131	.tag = tag,	1234	.tag = tag[1],
1132	.ops = ops,	1235	.ops = ops,
1133	.data = switch (ops) {	1236	.data = switch (ops) {
1134	.ri_s, .ri_u => .{ .ri = .{ .r = reg, .i = switch (imm) {	1237	.ri_s, .ri_u => .{ .ri = .{
1135	.signed => \|s\| @bitCast(u32, s),	1238	.fixes = tag[0],
1136	.unsigned => \|u\| @intCast(u32, u),	1239	.r1 = reg,
1137	} } },	1240	.i = switch (imm) {
		1241	.signed => \|s\| @bitCast(u32, s),
		1242	.unsigned => \|u\| @intCast(u32, u),
		1243	},
		1244	} },
1138	.ri64 => .{ .rx = .{	1245	.ri64 => .{ .rx = .{
1139	.r = reg,	1246	.fixes = tag[0],
		1247	.r1 = reg,
1140	.payload = try self.addExtra(Mir.Imm64.encode(imm.unsigned)),	1248	.payload = try self.addExtra(Mir.Imm64.encode(imm.unsigned)),
1141	} },	1249	} },
1142	else => unreachable,	1250	else => unreachable,
...	@@ -1146,111 +1254,214 @@ fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.Tag, reg: Register, imm: Imme	...	@@ -1146,111 +1254,214 @@ fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.Tag, reg: Register, imm: Imme
1146		1254
1147	fn asmRegisterRegisterRegister(	1255	fn asmRegisterRegisterRegister(
1148	self: *Self,	1256	self: *Self,
1149	tag: Mir.Inst.Tag,	1257	tag: Mir.Inst.FixedTag,
1150	reg1: Register,	1258	reg1: Register,
1151	reg2: Register,	1259	reg2: Register,
1152	reg3: Register,	1260	reg3: Register,
1153	) !void {	1261	) !void {
1154	_ = try self.addInst(.{	1262	_ = try self.addInst(.{
1155	.tag = tag,	1263	.tag = tag[1],
1156	.ops = .rrr,	1264	.ops = .rrr,
1157	.data = .{ .rrr = .{ .r1 = reg1, .r2 = reg2, .r3 = reg3 } },	1265	.data = .{ .rrr = .{
		1266	.fixes = tag[0],
		1267	.r1 = reg1,
		1268	.r2 = reg2,
		1269	.r3 = reg3,
		1270	} },
		1271	});
		1272	}
		1273
		1274	fn asmRegisterRegisterRegisterImmediate(
		1275	self: *Self,
		1276	tag: Mir.Inst.FixedTag,
		1277	reg1: Register,
		1278	reg2: Register,
		1279	reg3: Register,
		1280	imm: Immediate,
		1281	) !void {
		1282	_ = try self.addInst(.{
		1283	.tag = tag[1],
		1284	.ops = .rrri,
		1285	.data = .{ .rrri = .{
		1286	.fixes = tag[0],
		1287	.r1 = reg1,
		1288	.r2 = reg2,
		1289	.r3 = reg3,
		1290	.i = @intCast(u8, imm.unsigned),
		1291	} },
1158	});	1292	});
1159	}	1293	}
1160		1294
1161	fn asmRegisterRegisterImmediate(	1295	fn asmRegisterRegisterImmediate(
1162	self: *Self,	1296	self: *Self,
1163	tag: Mir.Inst.Tag,	1297	tag: Mir.Inst.FixedTag,
1164	reg1: Register,	1298	reg1: Register,
1165	reg2: Register,	1299	reg2: Register,
1166	imm: Immediate,	1300	imm: Immediate,
1167	) !void {	1301	) !void {
1168	_ = try self.addInst(.{	1302	_ = try self.addInst(.{
1169	.tag = tag,	1303	.tag = tag[1],
1170	.ops = switch (imm) {	1304	.ops = switch (imm) {
1171	.signed => .rri_s,	1305	.signed => .rri_s,
1172	.unsigned => .rri_u,	1306	.unsigned => .rri_u,
1173	},	1307	},
1174	.data = .{ .rri = .{ .r1 = reg1, .r2 = reg2, .i = switch (imm) {	1308	.data = .{ .rri = .{
1175	.signed => \|s\| @bitCast(u32, s),	1309	.fixes = tag[0],
1176	.unsigned => \|u\| @intCast(u32, u),	1310	.r1 = reg1,
1177	} } },	1311	.r2 = reg2,
		1312	.i = switch (imm) {
		1313	.signed => \|s\| @bitCast(u32, s),
		1314	.unsigned => \|u\| @intCast(u32, u),
		1315	},
		1316	} },
		1317	});
		1318	}
		1319
		1320	fn asmRegisterRegisterMemory(
		1321	self: *Self,
		1322	tag: Mir.Inst.FixedTag,
		1323	reg1: Register,
		1324	reg2: Register,
		1325	m: Memory,
		1326	) !void {
		1327	_ = try self.addInst(.{
		1328	.tag = tag[1],
		1329	.ops = switch (m) {
		1330	.sib => .rrm_sib,
		1331	.rip => .rrm_rip,
		1332	else => unreachable,
		1333	},
		1334	.data = .{ .rrx = .{
		1335	.fixes = tag[0],
		1336	.r1 = reg1,
		1337	.r2 = reg2,
		1338	.payload = switch (m) {
		1339	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
		1340	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
		1341	else => unreachable,
		1342	},
		1343	} },
1178	});	1344	});
1179	}	1345	}
1180		1346
1181	fn asmMemory(self: *Self, tag: Mir.Inst.Tag, m: Memory) !void {	1347	fn asmMemory(self: *Self, tag: Mir.Inst.FixedTag, m: Memory) !void {
1182	_ = try self.addInst(.{	1348	_ = try self.addInst(.{
1183	.tag = tag,	1349	.tag = tag[1],
1184	.ops = switch (m) {	1350	.ops = switch (m) {
1185	.sib => .m_sib,	1351	.sib => .m_sib,
1186	.rip => .m_rip,	1352	.rip => .m_rip,
1187	else => unreachable,	1353	else => unreachable,
1188	},	1354	},
1189	.data = .{ .payload = switch (m) {	1355	.data = .{ .x = .{
1190	.sib => try self.addExtra(Mir.MemorySib.encode(m)),	1356	.fixes = tag[0],
1191	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),	1357	.payload = switch (m) {
1192	else => unreachable,	1358	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
		1359	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
		1360	else => unreachable,
		1361	},
1193	} },	1362	} },
1194	});	1363	});
1195	}	1364	}
1196		1365
1197	fn asmRegisterMemory(self: *Self, tag: Mir.Inst.Tag, reg: Register, m: Memory) !void {	1366	fn asmRegisterMemory(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, m: Memory) !void {
1198	_ = try self.addInst(.{	1367	_ = try self.addInst(.{
1199	.tag = tag,	1368	.tag = tag[1],
1200	.ops = switch (m) {	1369	.ops = switch (m) {
1201	.sib => .rm_sib,	1370	.sib => .rm_sib,
1202	.rip => .rm_rip,	1371	.rip => .rm_rip,
1203	else => unreachable,	1372	else => unreachable,
1204	},	1373	},
1205	.data = .{ .rx = .{ .r = reg, .payload = switch (m) {	1374	.data = .{ .rx = .{
1206	.sib => try self.addExtra(Mir.MemorySib.encode(m)),	1375	.fixes = tag[0],
1207	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),	1376	.r1 = reg,
1208	else => unreachable,	1377	.payload = switch (m) {
1209	} } },	1378	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
		1379	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
		1380	else => unreachable,
		1381	},
		1382	} },
1210	});	1383	});
1211	}	1384	}
1212		1385
1213	fn asmRegisterMemoryImmediate(	1386	fn asmRegisterMemoryImmediate(
1214	self: *Self,	1387	self: *Self,
1215	tag: Mir.Inst.Tag,	1388	tag: Mir.Inst.FixedTag,
1216	reg: Register,	1389	reg: Register,
1217	m: Memory,	1390	m: Memory,
1218	imm: Immediate,	1391	imm: Immediate,
1219	) !void {	1392	) !void {
1220	_ = try self.addInst(.{	1393	_ = try self.addInst(.{
1221	.tag = tag,	1394	.tag = tag[1],
1222	.ops = switch (m) {	1395	.ops = switch (m) {
1223	.sib => .rmi_sib,	1396	.sib => .rmi_sib,
1224	.rip => .rmi_rip,	1397	.rip => .rmi_rip,
1225	else => unreachable,	1398	else => unreachable,
1226	},	1399	},
1227	.data = .{ .rix = .{ .r = reg, .i = @intCast(u8, imm.unsigned), .payload = switch (m) {	1400	.data = .{ .rix = .{
1228	.sib => try self.addExtra(Mir.MemorySib.encode(m)),	1401	.fixes = tag[0],
1229	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),	1402	.r1 = reg,
		1403	.i = @intCast(u8, imm.unsigned),
		1404	.payload = switch (m) {
		1405	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
		1406	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
		1407	else => unreachable,
		1408	},
		1409	} },
		1410	});
		1411	}
		1412
		1413	fn asmRegisterRegisterMemoryImmediate(
		1414	self: *Self,
		1415	tag: Mir.Inst.FixedTag,
		1416	reg1: Register,
		1417	reg2: Register,
		1418	m: Memory,
		1419	imm: Immediate,
		1420	) !void {
		1421	_ = try self.addInst(.{
		1422	.tag = tag[1],
		1423	.ops = switch (m) {
		1424	.sib => .rrmi_sib,
		1425	.rip => .rrmi_rip,
1230	else => unreachable,	1426	else => unreachable,
1231	} } },	1427	},
		1428	.data = .{ .rrix = .{
		1429	.fixes = tag[0],
		1430	.r1 = reg1,
		1431	.r2 = reg2,
		1432	.i = @intCast(u8, imm.unsigned),
		1433	.payload = switch (m) {
		1434	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
		1435	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
		1436	else => unreachable,
		1437	},
		1438	} },
1232	});	1439	});
1233	}	1440	}
1234		1441
1235	fn asmMemoryRegister(self: *Self, tag: Mir.Inst.Tag, m: Memory, reg: Register) !void {	1442	fn asmMemoryRegister(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, reg: Register) !void {
1236	_ = try self.addInst(.{	1443	_ = try self.addInst(.{
1237	.tag = tag,	1444	.tag = tag[1],
1238	.ops = switch (m) {	1445	.ops = switch (m) {
1239	.sib => .mr_sib,	1446	.sib => .mr_sib,
1240	.rip => .mr_rip,	1447	.rip => .mr_rip,
1241	else => unreachable,	1448	else => unreachable,
1242	},	1449	},
1243	.data = .{ .rx = .{ .r = reg, .payload = switch (m) {	1450	.data = .{ .rx = .{
1244	.sib => try self.addExtra(Mir.MemorySib.encode(m)),	1451	.fixes = tag[0],
1245	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),	1452	.r1 = reg,
1246	else => unreachable,	1453	.payload = switch (m) {
1247	} } },	1454	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
		1455	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
		1456	else => unreachable,
		1457	},
		1458	} },
1248	});	1459	});
1249	}	1460	}
1250		1461
1251	fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.Tag, m: Memory, imm: Immediate) !void {	1462	fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, imm: Immediate) !void {
1252	_ = try self.addInst(.{	1463	_ = try self.addInst(.{
1253	.tag = tag,	1464	.tag = tag[1],
1254	.ops = switch (m) {	1465	.ops = switch (m) {
1255	.sib => switch (imm) {	1466	.sib => switch (imm) {
1256	.signed => .mi_sib_s,	1467	.signed => .mi_sib_s,
...	@@ -1262,67 +1473,81 @@ fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.Tag, m: Memory, imm: Immediate)	...	@@ -1262,67 +1473,81 @@ fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.Tag, m: Memory, imm: Immediate)
1262	},	1473	},
1263	else => unreachable,	1474	else => unreachable,
1264	},	1475	},
1265	.data = .{ .ix = .{ .i = switch (imm) {	1476	.data = .{ .x = .{
1266	.signed => \|s\| @bitCast(u32, s),	1477	.fixes = tag[0],
1267	.unsigned => \|u\| @intCast(u32, u),	1478	.payload = try self.addExtra(Mir.Imm32{ .imm = switch (imm) {
1268	}, .payload = switch (m) {	1479	.signed => \|s\| @bitCast(u32, s),
1269	.sib => try self.addExtra(Mir.MemorySib.encode(m)),	1480	.unsigned => \|u\| @intCast(u32, u),
1270	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),	1481	} }),
1271	else => unreachable,	1482	} },
1272	} } },
1273	});	1483	});
		1484	_ = switch (m) {
		1485	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
		1486	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
		1487	else => unreachable,
		1488	};
1274	}	1489	}
1275		1490
1276	fn asmMemoryRegisterRegister(	1491	fn asmMemoryRegisterRegister(
1277	self: *Self,	1492	self: *Self,
1278	tag: Mir.Inst.Tag,	1493	tag: Mir.Inst.FixedTag,
1279	m: Memory,	1494	m: Memory,
1280	reg1: Register,	1495	reg1: Register,
1281	reg2: Register,	1496	reg2: Register,
1282	) !void {	1497	) !void {
1283	_ = try self.addInst(.{	1498	_ = try self.addInst(.{
1284	.tag = tag,	1499	.tag = tag[1],
1285	.ops = switch (m) {	1500	.ops = switch (m) {
1286	.sib => .mrr_sib,	1501	.sib => .mrr_sib,
1287	.rip => .mrr_rip,	1502	.rip => .mrr_rip,
1288	else => unreachable,	1503	else => unreachable,
1289	},	1504	},
1290	.data = .{ .rrx = .{ .r1 = reg1, .r2 = reg2, .payload = switch (m) {	1505	.data = .{ .rrx = .{
1291	.sib => try self.addExtra(Mir.MemorySib.encode(m)),	1506	.fixes = tag[0],
1292	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),	1507	.r1 = reg1,
1293	else => unreachable,	1508	.r2 = reg2,
1294	} } },	1509	.payload = switch (m) {
		1510	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
		1511	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
		1512	else => unreachable,
		1513	},
		1514	} },
1295	});	1515	});
1296	}	1516	}
1297		1517
1298	fn asmMemoryRegisterImmediate(	1518	fn asmMemoryRegisterImmediate(
1299	self: *Self,	1519	self: *Self,
1300	tag: Mir.Inst.Tag,	1520	tag: Mir.Inst.FixedTag,
1301	m: Memory,	1521	m: Memory,
1302	reg: Register,	1522	reg: Register,
1303	imm: Immediate,	1523	imm: Immediate,
1304	) !void {	1524	) !void {
1305	_ = try self.addInst(.{	1525	_ = try self.addInst(.{
1306	.tag = tag,	1526	.tag = tag[1],
1307	.ops = switch (m) {	1527	.ops = switch (m) {
1308	.sib => .mri_sib,	1528	.sib => .mri_sib,
1309	.rip => .mri_rip,	1529	.rip => .mri_rip,
1310	else => unreachable,	1530	else => unreachable,
1311	},	1531	},
1312	.data = .{ .rix = .{ .r = reg, .i = @intCast(u8, imm.unsigned), .payload = switch (m) {	1532	.data = .{ .rix = .{
1313	.sib => try self.addExtra(Mir.MemorySib.encode(m)),	1533	.fixes = tag[0],
1314	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),	1534	.r1 = reg,
1315	else => unreachable,	1535	.i = @intCast(u8, imm.unsigned),
1316	} } },	1536	.payload = switch (m) {
		1537	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
		1538	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
		1539	else => unreachable,
		1540	},
		1541	} },
1317	});	1542	});
1318	}	1543	}
1319		1544
1320	fn gen(self: *Self) InnerError!void {	1545	fn gen(self: *Self) InnerError!void {
1321	const cc = self.fn_type.fnCallingConvention();	1546	const cc = self.fn_type.fnCallingConvention();
1322	if (cc != .Naked) {	1547	if (cc != .Naked) {
1323	try self.asmRegister(.push, .rbp);	1548	try self.asmRegister(.{ ._, .push }, .rbp);
1324	const backpatch_push_callee_preserved_regs = try self.asmPlaceholder();	1549	const backpatch_push_callee_preserved_regs = try self.asmPlaceholder();
1325	try self.asmRegisterRegister(.mov, .rbp, .rsp);	1550	try self.asmRegisterRegister(.{ ._, .mov }, .rbp, .rsp);
1326	const backpatch_frame_align = try self.asmPlaceholder();	1551	const backpatch_frame_align = try self.asmPlaceholder();
1327	const backpatch_stack_alloc = try self.asmPlaceholder();	1552	const backpatch_stack_alloc = try self.asmPlaceholder();
1328		1553
...	@@ -1346,7 +1571,7 @@ fn gen(self: *Self) InnerError!void {	...	@@ -1346,7 +1571,7 @@ fn gen(self: *Self) InnerError!void {
1346	else => unreachable,	1571	else => unreachable,
1347	}	1572	}
1348		1573
1349	try self.asmOpOnly(.dbg_prologue_end);	1574	try self.asmPseudo(.pseudo_dbg_prologue_end_none);
1350		1575
1351	try self.genBody(self.air.getMainBody());	1576	try self.genBody(self.air.getMainBody());
1352		1577
...	@@ -1358,15 +1583,15 @@ fn gen(self: *Self) InnerError!void {	...	@@ -1358,15 +1583,15 @@ fn gen(self: *Self) InnerError!void {
1358	// }	1583	// }
1359	// Eliding the reloc will cause a miscompilation in this case.	1584	// Eliding the reloc will cause a miscompilation in this case.
1360	for (self.exitlude_jump_relocs.items) \|jmp_reloc\| {	1585	for (self.exitlude_jump_relocs.items) \|jmp_reloc\| {
1361	self.mir_instructions.items(.data)[jmp_reloc].inst =	1586	self.mir_instructions.items(.data)[jmp_reloc].inst.inst =
1362	@intCast(u32, self.mir_instructions.len);	1587	@intCast(u32, self.mir_instructions.len);
1363	}	1588	}
1364		1589
1365	try self.asmOpOnly(.dbg_epilogue_begin);	1590	try self.asmPseudo(.pseudo_dbg_epilogue_begin_none);
1366	const backpatch_stack_dealloc = try self.asmPlaceholder();	1591	const backpatch_stack_dealloc = try self.asmPlaceholder();
1367	const backpatch_pop_callee_preserved_regs = try self.asmPlaceholder();	1592	const backpatch_pop_callee_preserved_regs = try self.asmPlaceholder();
1368	try self.asmRegister(.pop, .rbp);	1593	try self.asmRegister(.{ ._, .pop }, .rbp);
1369	try self.asmOpOnly(.ret);	1594	try self.asmOpOnly(.{ ._, .ret });
1370		1595
1371	const frame_layout = try self.computeFrameLayout();	1596	const frame_layout = try self.computeFrameLayout();
1372	const need_frame_align = frame_layout.stack_mask != math.maxInt(u32);	1597	const need_frame_align = frame_layout.stack_mask != math.maxInt(u32);
...	@@ -1376,46 +1601,54 @@ fn gen(self: *Self) InnerError!void {	...	@@ -1376,46 +1601,54 @@ fn gen(self: *Self) InnerError!void {
1376	self.mir_instructions.set(backpatch_frame_align, .{	1601	self.mir_instructions.set(backpatch_frame_align, .{
1377	.tag = .@"and",	1602	.tag = .@"and",
1378	.ops = .ri_s,	1603	.ops = .ri_s,
1379	.data = .{ .ri = .{ .r = .rsp, .i = frame_layout.stack_mask } },	1604	.data = .{ .ri = .{
		1605	.r1 = .rsp,
		1606	.i = frame_layout.stack_mask,
		1607	} },
1380	});	1608	});
1381	}	1609	}
1382	if (need_stack_adjust) {	1610	if (need_stack_adjust) {
1383	self.mir_instructions.set(backpatch_stack_alloc, .{	1611	self.mir_instructions.set(backpatch_stack_alloc, .{
1384	.tag = .sub,	1612	.tag = .sub,
1385	.ops = .ri_s,	1613	.ops = .ri_s,
1386	.data = .{ .ri = .{ .r = .rsp, .i = frame_layout.stack_adjust } },	1614	.data = .{ .ri = .{
		1615	.r1 = .rsp,
		1616	.i = frame_layout.stack_adjust,
		1617	} },
1387	});	1618	});
1388	}	1619	}
1389	if (need_frame_align or need_stack_adjust) {	1620	if (need_frame_align or need_stack_adjust) {
1390	self.mir_instructions.set(backpatch_stack_dealloc, .{	1621	self.mir_instructions.set(backpatch_stack_dealloc, .{
1391	.tag = .mov,	1622	.tag = .mov,
1392	.ops = .rr,	1623	.ops = .rr,
1393	.data = .{ .rr = .{ .r1 = .rsp, .r2 = .rbp } },	1624	.data = .{ .rr = .{
		1625	.r1 = .rsp,
		1626	.r2 = .rbp,
		1627	} },
1394	});	1628	});
1395	}	1629	}
1396	if (need_save_reg) {	1630	if (need_save_reg) {
1397	const save_reg_list = frame_layout.save_reg_list.asInt();
1398	self.mir_instructions.set(backpatch_push_callee_preserved_regs, .{	1631	self.mir_instructions.set(backpatch_push_callee_preserved_regs, .{
1399	.tag = .push_regs,	1632	.tag = .pseudo,
1400	.ops = undefined,	1633	.ops = .pseudo_push_reg_list,
1401	.data = .{ .payload = save_reg_list },	1634	.data = .{ .reg_list = frame_layout.save_reg_list },
1402	});	1635	});
1403	self.mir_instructions.set(backpatch_pop_callee_preserved_regs, .{	1636	self.mir_instructions.set(backpatch_pop_callee_preserved_regs, .{
1404	.tag = .pop_regs,	1637	.tag = .pseudo,
1405	.ops = undefined,	1638	.ops = .pseudo_pop_reg_list,
1406	.data = .{ .payload = save_reg_list },	1639	.data = .{ .reg_list = frame_layout.save_reg_list },
1407	});	1640	});
1408	}	1641	}
1409	} else {	1642	} else {
1410	try self.asmOpOnly(.dbg_prologue_end);	1643	try self.asmPseudo(.pseudo_dbg_prologue_end_none);
1411	try self.genBody(self.air.getMainBody());	1644	try self.genBody(self.air.getMainBody());
1412	try self.asmOpOnly(.dbg_epilogue_begin);	1645	try self.asmPseudo(.pseudo_dbg_epilogue_begin_none);
1413	}	1646	}
1414		1647
1415	// Drop them off at the rbrace.	1648	// Drop them off at the rbrace.
1416	_ = try self.addInst(.{	1649	_ = try self.addInst(.{
1417	.tag = .dbg_line,	1650	.tag = .pseudo,
1418	.ops = undefined,	1651	.ops = .pseudo_dbg_line_line_column,
1419	.data = .{ .line_column = .{	1652	.data = .{ .line_column = .{
1420	.line = self.end_di_line,	1653	.line = self.end_di_line,
1421	.column = self.end_di_column,	1654	.column = self.end_di_column,
...	@@ -1480,12 +1713,12 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {	...	@@ -1480,12 +1713,12 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
1480	.log,	1713	.log,
1481	.log2,	1714	.log2,
1482	.log10,	1715	.log10,
1483	.floor,
1484	.ceil,
1485	.round,	1716	.round,
1486	.trunc_float,
1487	=> try self.airUnaryMath(inst),	1717	=> try self.airUnaryMath(inst),
1488		1718
		1719	.floor => try self.airRound(inst, 0b1_0_01),
		1720	.ceil => try self.airRound(inst, 0b1_0_10),
		1721	.trunc_float => try self.airRound(inst, 0b1_0_11),
1489	.sqrt => try self.airSqrt(inst),	1722	.sqrt => try self.airSqrt(inst),
1490	.neg, .fabs => try self.airFloatSign(inst),	1723	.neg, .fabs => try self.airFloatSign(inst),
1491		1724
...	@@ -1731,7 +1964,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {	...	@@ -1731,7 +1964,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
1731	};	1964	};
1732	const tag_val = Value.initPayload(&tag_pl.base);	1965	const tag_val = Value.initPayload(&tag_pl.base);
1733	const tag_mcv = try self.genTypedValue(.{ .ty = enum_ty, .val = tag_val });	1966	const tag_mcv = try self.genTypedValue(.{ .ty = enum_ty, .val = tag_val });
1734	try self.genBinOpMir(.cmp, enum_ty, enum_mcv, tag_mcv);	1967	try self.genBinOpMir(.{ ._, .cmp }, enum_ty, enum_mcv, tag_mcv);
1735	const skip_reloc = try self.asmJccReloc(undefined, .ne);	1968	const skip_reloc = try self.asmJccReloc(undefined, .ne);
1736		1969
1737	try self.genSetMem(	1970	try self.genSetMem(
...	@@ -1751,7 +1984,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {	...	@@ -1751,7 +1984,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
1751	try self.airTrap();	1984	try self.airTrap();
1752		1985
1753	for (exitlude_jump_relocs) \|reloc\| try self.performReloc(reloc);	1986	for (exitlude_jump_relocs) \|reloc\| try self.performReloc(reloc);
1754	try self.asmOpOnly(.ret);	1987	try self.asmOpOnly(.{ ._, .ret });
1755	},	1988	},
1756	else => return self.fail(	1989	else => return self.fail(
1757	"TODO implement {s} for {}",	1990	"TODO implement {s} for {}",
...	@@ -1919,6 +2152,11 @@ fn computeFrameLayout(self: *Self) !FrameLayout {	...	@@ -1919,6 +2152,11 @@ fn computeFrameLayout(self: *Self) !FrameLayout {
1919	};	2152	};
1920	}	2153	}
1921		2154
		2155	fn getFrameAddrAlignment(self: *Self, frame_addr: FrameAddr) u32 {
		2156	const alloc_align = @as(u32, 1) << self.frame_allocs.get(@enumToInt(frame_addr.index)).abi_align;
		2157	return @min(alloc_align, @bitCast(u32, frame_addr.off) & (alloc_align - 1));
		2158	}
		2159
1922	fn allocFrameIndex(self: *Self, alloc: FrameAlloc) !FrameIndex {	2160	fn allocFrameIndex(self: *Self, alloc: FrameAlloc) !FrameIndex {
1923	const frame_allocs_slice = self.frame_allocs.slice();	2161	const frame_allocs_slice = self.frame_allocs.slice();
1924	const frame_size = frame_allocs_slice.items(.abi_size);	2162	const frame_size = frame_allocs_slice.items(.abi_size);
...	@@ -1962,24 +2200,36 @@ fn allocTempRegOrMem(self: *Self, elem_ty: Type, reg_ok: bool) !MCValue {	...	@@ -1962,24 +2200,36 @@ fn allocTempRegOrMem(self: *Self, elem_ty: Type, reg_ok: bool) !MCValue {
1962	return self.allocRegOrMemAdvanced(elem_ty, null, reg_ok);	2200	return self.allocRegOrMemAdvanced(elem_ty, null, reg_ok);
1963	}	2201	}
1964		2202
1965	fn allocRegOrMemAdvanced(self: *Self, elem_ty: Type, inst: ?Air.Inst.Index, reg_ok: bool) !MCValue {	2203	fn allocRegOrMemAdvanced(self: *Self, ty: Type, inst: ?Air.Inst.Index, reg_ok: bool) !MCValue {
1966	const abi_size = math.cast(u32, elem_ty.abiSize(self.target.*)) orelse {	2204	const abi_size = math.cast(u32, ty.abiSize(self.target.*)) orelse {
1967	const mod = self.bin_file.options.module.?;	2205	const mod = self.bin_file.options.module.?;
1968	return self.fail("type '{}' too big to fit into stack frame", .{elem_ty.fmt(mod)});	2206	return self.fail("type '{}' too big to fit into stack frame", .{ty.fmt(mod)});
1969	};	2207	};
1970		2208
1971	if (reg_ok) {	2209	if (reg_ok) need_mem: {
1972	// Make sure the type can fit in a register before we try to allocate one.	2210	if (abi_size <= @as(u32, switch (ty.zigTypeTag()) {
1973	const ptr_bits = self.target.cpu.arch.ptrBitWidth();	2211	.Float => switch (ty.floatBits(self.target.*)) {
1974	const ptr_bytes: u64 = @divExact(ptr_bits, 8);	2212	16, 32, 64, 128 => 16,
1975	if (abi_size <= ptr_bytes) {	2213	80 => break :need_mem,
1976	if (self.register_manager.tryAllocReg(inst, regClassForType(elem_ty))) \|reg\| {	2214	else => unreachable,
		2215	},
		2216	.Vector => switch (ty.childType().zigTypeTag()) {
		2217	.Float => switch (ty.childType().floatBits(self.target.*)) {
		2218	16, 32, 64 => if (self.hasFeature(.avx)) 32 else 16,
		2219	80, 128 => break :need_mem,
		2220	else => unreachable,
		2221	},
		2222	else => break :need_mem,
		2223	},
		2224	else => 8,
		2225	})) {
		2226	if (self.register_manager.tryAllocReg(inst, regClassForType(ty))) \|reg\| {
1977	return MCValue{ .register = registerAlias(reg, abi_size) };	2227	return MCValue{ .register = registerAlias(reg, abi_size) };
1978	}	2228	}
1979	}	2229	}
1980	}	2230	}
1981		2231
1982	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(elem_ty, self.target.*));	2232	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(ty, self.target.*));
1983	return .{ .load_frame = .{ .index = frame_index } };	2233	return .{ .load_frame = .{ .index = frame_index } };
1984	}	2234	}
1985		2235
...	@@ -2172,44 +2422,127 @@ fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2172,44 +2422,127 @@ fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {
2172	fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {	2422	fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
2173	const ty_op = self.air.instructions.items(.data)[inst].ty_op;	2423	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2174	const dst_ty = self.air.typeOfIndex(inst);	2424	const dst_ty = self.air.typeOfIndex(inst);
		2425	const dst_bits = dst_ty.floatBits(self.target.*);
2175	const src_ty = self.air.typeOf(ty_op.operand);	2426	const src_ty = self.air.typeOf(ty_op.operand);
2176	if (dst_ty.floatBits(self.target.) != 32 or src_ty.floatBits(self.target.) != 64 or	2427	const src_bits = src_ty.floatBits(self.target.*);
2177	!Target.x86.featureSetHas(self.target.cpu.features, .sse2))
2178	return self.fail("TODO implement airFptrunc from {} to {}", .{
2179	src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
2180	});
2181		2428
2182	const src_mcv = try self.resolveInst(ty_op.operand);	2429	const src_mcv = try self.resolveInst(ty_op.operand);
2183	const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))	2430	const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
2184	src_mcv	2431	src_mcv
2185	else	2432	else
2186	try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);	2433	try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);
2187	const dst_lock = self.register_manager.lockReg(dst_mcv.register);	2434	const dst_reg = dst_mcv.getReg().?.to128();
		2435	const dst_lock = self.register_manager.lockReg(dst_reg);
2188	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);	2436	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
2189		2437
2190	try self.genBinOpMir(.cvtsd2ss, src_ty, dst_mcv, src_mcv);	2438	if (dst_bits == 16 and self.hasFeature(.f16c)) {
		2439	switch (src_bits) {
		2440	32 => {
		2441	const mat_src_reg = if (src_mcv.isRegister())
		2442	src_mcv.getReg().?
		2443	else
		2444	try self.copyToTmpRegister(src_ty, src_mcv);
		2445	try self.asmRegisterRegisterImmediate(
		2446	.{ .v_, .cvtps2ph },
		2447	dst_reg,
		2448	mat_src_reg.to128(),
		2449	Immediate.u(0b1_00),
		2450	);
		2451	},
		2452	else => return self.fail("TODO implement airFptrunc from {} to {}", .{
		2453	src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
		2454	}),
		2455	}
		2456	} else if (src_bits == 64 and dst_bits == 32) {
		2457	if (self.hasFeature(.avx)) if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
		2458	.{ .v_, .cvtsd2ss },
		2459	dst_reg,
		2460	dst_reg,
		2461	src_mcv.mem(.qword),
		2462	) else try self.asmRegisterRegisterRegister(
		2463	.{ .v_, .cvtsd2ss },
		2464	dst_reg,
		2465	dst_reg,
		2466	(if (src_mcv.isRegister())
		2467	src_mcv.getReg().?
		2468	else
		2469	try self.copyToTmpRegister(src_ty, src_mcv)).to128(),
		2470	) else if (src_mcv.isMemory()) try self.asmRegisterMemory(
		2471	.{ ._, .cvtsd2ss },
		2472	dst_reg,
		2473	src_mcv.mem(.qword),
		2474	) else try self.asmRegisterRegister(
		2475	.{ ._, .cvtsd2ss },
		2476	dst_reg,
		2477	(if (src_mcv.isRegister())
		2478	src_mcv.getReg().?
		2479	else
		2480	try self.copyToTmpRegister(src_ty, src_mcv)).to128(),
		2481	);
		2482	} else return self.fail("TODO implement airFptrunc from {} to {}", .{
		2483	src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
		2484	});
2191	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });	2485	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
2192	}	2486	}
2193		2487
2194	fn airFpext(self: *Self, inst: Air.Inst.Index) !void {	2488	fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
2195	const ty_op = self.air.instructions.items(.data)[inst].ty_op;	2489	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2196	const dst_ty = self.air.typeOfIndex(inst);	2490	const dst_ty = self.air.typeOfIndex(inst);
		2491	const dst_bits = dst_ty.floatBits(self.target.*);
2197	const src_ty = self.air.typeOf(ty_op.operand);	2492	const src_ty = self.air.typeOf(ty_op.operand);
2198	if (dst_ty.floatBits(self.target.) != 64 or src_ty.floatBits(self.target.) != 32 or	2493	const src_bits = src_ty.floatBits(self.target.*);
2199	!Target.x86.featureSetHas(self.target.cpu.features, .sse2))
2200	return self.fail("TODO implement airFpext from {} to {}", .{
2201	src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
2202	});
2203		2494
2204	const src_mcv = try self.resolveInst(ty_op.operand);	2495	const src_mcv = try self.resolveInst(ty_op.operand);
2205	const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))	2496	const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
2206	src_mcv	2497	src_mcv
2207	else	2498	else
2208	try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);	2499	try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);
2209	const dst_lock = self.register_manager.lockReg(dst_mcv.register);	2500	const dst_reg = dst_mcv.getReg().?.to128();
		2501	const dst_lock = self.register_manager.lockReg(dst_reg);
2210	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);	2502	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
2211		2503
2212	try self.genBinOpMir(.cvtss2sd, src_ty, dst_mcv, src_mcv);	2504	if (src_bits == 16 and self.hasFeature(.f16c)) {
		2505	const mat_src_reg = if (src_mcv.isRegister())
		2506	src_mcv.getReg().?
		2507	else
		2508	try self.copyToTmpRegister(src_ty, src_mcv);
		2509	try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg, mat_src_reg.to128());
		2510	switch (dst_bits) {
		2511	32 => {},
		2512	64 => try self.asmRegisterRegisterRegister(.{ .v_, .cvtss2sd }, dst_reg, dst_reg, dst_reg),
		2513	else => return self.fail("TODO implement airFpext from {} to {}", .{
		2514	src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
		2515	}),
		2516	}
		2517	} else if (src_bits == 32 and dst_bits == 64) {
		2518	if (self.hasFeature(.avx)) if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
		2519	.{ .v_, .cvtss2sd },
		2520	dst_reg,
		2521	dst_reg,
		2522	src_mcv.mem(.dword),
		2523	) else try self.asmRegisterRegisterRegister(
		2524	.{ .v_, .cvtss2sd },
		2525	dst_reg,
		2526	dst_reg,
		2527	(if (src_mcv.isRegister())
		2528	src_mcv.getReg().?
		2529	else
		2530	try self.copyToTmpRegister(src_ty, src_mcv)).to128(),
		2531	) else if (src_mcv.isMemory()) try self.asmRegisterMemory(
		2532	.{ ._, .cvtss2sd },
		2533	dst_reg,
		2534	src_mcv.mem(.dword),
		2535	) else try self.asmRegisterRegister(
		2536	.{ ._, .cvtss2sd },
		2537	dst_reg,
		2538	(if (src_mcv.isRegister())
		2539	src_mcv.getReg().?
		2540	else
		2541	try self.copyToTmpRegister(src_ty, src_mcv)).to128(),
		2542	);
		2543	} else return self.fail("TODO implement airFpext from {} to {}", .{
		2544	src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
		2545	});
2213	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });	2546	return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
2214	}	2547	}
2215		2548
...	@@ -2241,11 +2574,11 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2241,11 +2574,11 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
2241	switch (dst_mcv) {	2574	switch (dst_mcv) {
2242	.register => \|dst_reg\| {	2575	.register => \|dst_reg\| {
2243	const min_abi_size = @min(dst_abi_size, src_abi_size);	2576	const min_abi_size = @min(dst_abi_size, src_abi_size);
2244	const tag: Mir.Inst.Tag = switch (signedness) {	2577	const tag: Mir.Inst.FixedTag = switch (signedness) {
2245	.signed => .movsx,	2578	.signed => if (min_abi_size >= 4) .{ ._d, .movsx } else .{ ._, .movsx },
2246	.unsigned => if (min_abi_size > 2) .mov else .movzx,	2579	.unsigned => if (min_abi_size >= 4) .{ ._, .mov } else .{ ._, .movzx },
2247	};	2580	};
2248	const dst_alias = switch (tag) {	2581	const dst_alias = switch (tag[1]) {
2249	.movsx => dst_reg.to64(),	2582	.movsx => dst_reg.to64(),
2250	.mov, .movzx => if (min_abi_size > 4) dst_reg.to64() else dst_reg.to32(),	2583	.mov, .movzx => if (min_abi_size > 4) dst_reg.to64() else dst_reg.to32(),
2251	else => unreachable,	2584	else => unreachable,
...	@@ -2274,14 +2607,24 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2274,14 +2607,24 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
2274	try self.genCopy(min_ty, dst_mcv, src_mcv);	2607	try self.genCopy(min_ty, dst_mcv, src_mcv);
2275	const extra = dst_abi_size * 8 - dst_int_info.bits;	2608	const extra = dst_abi_size * 8 - dst_int_info.bits;
2276	if (extra > 0) {	2609	if (extra > 0) {
2277	try self.genShiftBinOpMir(switch (signedness) {	2610	try self.genShiftBinOpMir(
2278	.signed => .sal,	2611	switch (signedness) {
2279	.unsigned => .shl,	2612	.signed => .{ ._l, .sa },
2280	}, dst_ty, dst_mcv, .{ .immediate = extra });	2613	.unsigned => .{ ._l, .sh },
2281	try self.genShiftBinOpMir(switch (signedness) {	2614	},
2282	.signed => .sar,	2615	dst_ty,
2283	.unsigned => .shr,	2616	dst_mcv,
2284	}, dst_ty, dst_mcv, .{ .immediate = extra });	2617	.{ .immediate = extra },
		2618	);
		2619	try self.genShiftBinOpMir(
		2620	switch (signedness) {
		2621	.signed => .{ ._r, .sa },
		2622	.unsigned => .{ ._r, .sh },
		2623	},
		2624	dst_ty,
		2625	dst_mcv,
		2626	.{ .immediate = extra },
		2627	);
2285	}	2628	}
2286	},	2629	},
2287	}	2630	}
...	@@ -2466,8 +2809,8 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2466,8 +2809,8 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
2466	const reg_bits = self.regBitSize(ty);	2809	const reg_bits = self.regBitSize(ty);
2467	const cc: Condition = if (ty.isSignedInt()) cc: {	2810	const cc: Condition = if (ty.isSignedInt()) cc: {
2468	try self.genSetReg(limit_reg, ty, dst_mcv);	2811	try self.genSetReg(limit_reg, ty, dst_mcv);
2469	try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });	2812	try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
2470	try self.genBinOpMir(.xor, ty, limit_mcv, .{	2813	try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
2471	.immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,	2814	.immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
2472	});	2815	});
2473	break :cc .o;	2816	break :cc .o;
...	@@ -2477,7 +2820,7 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2477,7 +2820,7 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
2477	});	2820	});
2478	break :cc .c;	2821	break :cc .c;
2479	};	2822	};
2480	try self.genBinOpMir(.add, ty, dst_mcv, rhs_mcv);	2823	try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, rhs_mcv);
2481		2824
2482	const cmov_abi_size = @max(@intCast(u32, ty.abiSize(self.target.*)), 2);	2825	const cmov_abi_size = @max(@intCast(u32, ty.abiSize(self.target.*)), 2);
2483	try self.asmCmovccRegisterRegister(	2826	try self.asmCmovccRegisterRegister(
...	@@ -2517,8 +2860,8 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2517,8 +2860,8 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
2517	const reg_bits = self.regBitSize(ty);	2860	const reg_bits = self.regBitSize(ty);
2518	const cc: Condition = if (ty.isSignedInt()) cc: {	2861	const cc: Condition = if (ty.isSignedInt()) cc: {
2519	try self.genSetReg(limit_reg, ty, dst_mcv);	2862	try self.genSetReg(limit_reg, ty, dst_mcv);
2520	try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });	2863	try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
2521	try self.genBinOpMir(.xor, ty, limit_mcv, .{	2864	try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
2522	.immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,	2865	.immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
2523	});	2866	});
2524	break :cc .o;	2867	break :cc .o;
...	@@ -2526,7 +2869,7 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2526,7 +2869,7 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
2526	try self.genSetReg(limit_reg, ty, .{ .immediate = 0 });	2869	try self.genSetReg(limit_reg, ty, .{ .immediate = 0 });
2527	break :cc .c;	2870	break :cc .c;
2528	};	2871	};
2529	try self.genBinOpMir(.sub, ty, dst_mcv, rhs_mcv);	2872	try self.genBinOpMir(.{ ._, .sub }, ty, dst_mcv, rhs_mcv);
2530		2873
2531	const cmov_abi_size = @max(@intCast(u32, ty.abiSize(self.target.*)), 2);	2874	const cmov_abi_size = @max(@intCast(u32, ty.abiSize(self.target.*)), 2);
2532	try self.asmCmovccRegisterRegister(	2875	try self.asmCmovccRegisterRegister(
...	@@ -2568,9 +2911,9 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2568,9 +2911,9 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
2568	const reg_bits = self.regBitSize(ty);	2911	const reg_bits = self.regBitSize(ty);
2569	const cc: Condition = if (ty.isSignedInt()) cc: {	2912	const cc: Condition = if (ty.isSignedInt()) cc: {
2570	try self.genSetReg(limit_reg, ty, lhs_mcv);	2913	try self.genSetReg(limit_reg, ty, lhs_mcv);
2571	try self.genBinOpMir(.xor, ty, limit_mcv, rhs_mcv);	2914	try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, rhs_mcv);
2572	try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });	2915	try self.genShiftBinOpMir(.{ ._, .sa }, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
2573	try self.genBinOpMir(.xor, ty, limit_mcv, .{	2916	try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
2574	.immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,	2917	.immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
2575	});	2918	});
2576	break :cc .o;	2919	break :cc .o;
...	@@ -2683,7 +3026,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2683,7 +3026,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2683	};	3026	};
2684	defer if (tmp_lock) \|lock\| self.register_manager.unlockReg(lock);	3027	defer if (tmp_lock) \|lock\| self.register_manager.unlockReg(lock);
2685		3028
2686	try self.genBinOpMir(.cmp, lhs_ty, tmp_mcv, lhs);	3029	try self.genBinOpMir(.{ ._, .cmp }, lhs_ty, tmp_mcv, lhs);
2687	const cc = Condition.ne;	3030	const cc = Condition.ne;
2688		3031
2689	const tuple_ty = self.air.typeOfIndex(inst);	3032	const tuple_ty = self.air.typeOfIndex(inst);
...	@@ -2770,12 +3113,17 @@ fn genSetFrameTruncatedOverflowCompare(	...	@@ -2770,12 +3113,17 @@ fn genSetFrameTruncatedOverflowCompare(
2770	src_mcv;	3113	src_mcv;
2771	try self.genSetReg(scratch_reg, hi_limb_ty, hi_limb_mcv);	3114	try self.genSetReg(scratch_reg, hi_limb_ty, hi_limb_mcv);
2772	try self.truncateRegister(hi_limb_ty, scratch_reg);	3115	try self.truncateRegister(hi_limb_ty, scratch_reg);
2773	try self.genBinOpMir(.cmp, hi_limb_ty, .{ .register = scratch_reg }, hi_limb_mcv);	3116	try self.genBinOpMir(.{ ._, .cmp }, hi_limb_ty, .{ .register = scratch_reg }, hi_limb_mcv);
2774		3117
2775	const eq_reg = temp_regs[2];	3118	const eq_reg = temp_regs[2];
2776	if (overflow_cc) \|_\| {	3119	if (overflow_cc) \|_\| {
2777	try self.asmSetccRegister(eq_reg.to8(), .ne);	3120	try self.asmSetccRegister(eq_reg.to8(), .ne);
2778	try self.genBinOpMir(.@"or", Type.u8, .{ .register = overflow_reg }, .{ .register = eq_reg });	3121	try self.genBinOpMir(
		3122	.{ ._, .@"or" },
		3123	Type.u8,
		3124	.{ .register = overflow_reg },
		3125	.{ .register = eq_reg },
		3126	);
2779	}	3127	}
2780		3128
2781	const payload_off = @intCast(i32, tuple_ty.structFieldOffset(0, self.target.*));	3129	const payload_off = @intCast(i32, tuple_ty.structFieldOffset(0, self.target.*));
...	@@ -2904,28 +3252,25 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2904,28 +3252,25 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2904	/// Generates signed or unsigned integer multiplication/division.	3252	/// Generates signed or unsigned integer multiplication/division.
2905	/// Clobbers .rax and .rdx registers.	3253	/// Clobbers .rax and .rdx registers.
2906	/// Quotient is saved in .rax and remainder in .rdx.	3254	/// Quotient is saved in .rax and remainder in .rdx.
2907	fn genIntMulDivOpMir(	3255	fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue, rhs: MCValue) !void {
2908	self: *Self,
2909	tag: Mir.Inst.Tag,
2910	ty: Type,
2911	lhs: MCValue,
2912	rhs: MCValue,
2913	) !void {
2914	const abi_size = @intCast(u32, ty.abiSize(self.target.*));	3256	const abi_size = @intCast(u32, ty.abiSize(self.target.*));
2915	if (abi_size > 8) {	3257	if (abi_size > 8) {
2916	return self.fail("TODO implement genIntMulDivOpMir for ABI size larger than 8", .{});	3258	return self.fail("TODO implement genIntMulDivOpMir for ABI size larger than 8", .{});
2917	}	3259	}
2918		3260
2919	try self.genSetReg(.rax, ty, lhs);	3261	try self.genSetReg(.rax, ty, lhs);
2920	switch (tag) {	3262	switch (tag[1]) {
2921	else => unreachable,	3263	else => unreachable,
2922	.mul, .imul => {},	3264	.mul => {},
2923	.div => try self.asmRegisterRegister(.xor, .edx, .edx),	3265	.div => switch (tag[0]) {
2924	.idiv => switch (self.regBitSize(ty)) {	3266	._ => try self.asmRegisterRegister(.{ ._, .xor }, .edx, .edx),
2925	8 => try self.asmOpOnly(.cbw),	3267	.i_ => switch (self.regBitSize(ty)) {
2926	16 => try self.asmOpOnly(.cwd),	3268	8 => try self.asmOpOnly(.{ ._, .cbw }),
2927	32 => try self.asmOpOnly(.cdq),	3269	16 => try self.asmOpOnly(.{ ._, .cwd }),
2928	64 => try self.asmOpOnly(.cqo),	3270	32 => try self.asmOpOnly(.{ ._, .cdq }),
		3271	64 => try self.asmOpOnly(.{ ._, .cqo }),
		3272	else => unreachable,
		3273	},
2929	else => unreachable,	3274	else => unreachable,
2930	},	3275	},
2931	}	3276	}
...	@@ -2963,23 +3308,28 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa	...	@@ -2963,23 +3308,28 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
2963	const divisor_lock = self.register_manager.lockReg(divisor);	3308	const divisor_lock = self.register_manager.lockReg(divisor);
2964	defer if (divisor_lock) \|lock\| self.register_manager.unlockReg(lock);	3309	defer if (divisor_lock) \|lock\| self.register_manager.unlockReg(lock);
2965		3310
2966	try self.genIntMulDivOpMir(switch (int_info.signedness) {	3311	try self.genIntMulDivOpMir(
2967	.signed => .idiv,	3312	switch (int_info.signedness) {
2968	.unsigned => .div,	3313	.signed => .{ .i_, .div },
2969	}, ty, .{ .register = dividend }, .{ .register = divisor });	3314	.unsigned => .{ ._, .div },
		3315	},
		3316	ty,
		3317	.{ .register = dividend },
		3318	.{ .register = divisor },
		3319	);
2970		3320
2971	try self.asmRegisterRegister(	3321	try self.asmRegisterRegister(
2972	.xor,	3322	.{ ._, .xor },
2973	registerAlias(divisor, abi_size),	3323	registerAlias(divisor, abi_size),
2974	registerAlias(dividend, abi_size),	3324	registerAlias(dividend, abi_size),
2975	);	3325	);
2976	try self.asmRegisterImmediate(	3326	try self.asmRegisterImmediate(
2977	.sar,	3327	.{ ._r, .sa },
2978	registerAlias(divisor, abi_size),	3328	registerAlias(divisor, abi_size),
2979	Immediate.u(int_info.bits - 1),	3329	Immediate.u(int_info.bits - 1),
2980	);	3330	);
2981	try self.asmRegisterRegister(	3331	try self.asmRegisterRegister(
2982	.@"test",	3332	.{ ._, .@"test" },
2983	registerAlias(.rdx, abi_size),	3333	registerAlias(.rdx, abi_size),
2984	registerAlias(.rdx, abi_size),	3334	registerAlias(.rdx, abi_size),
2985	);	3335	);
...	@@ -2988,7 +3338,7 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa	...	@@ -2988,7 +3338,7 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
2988	registerAlias(.rdx, abi_size),	3338	registerAlias(.rdx, abi_size),
2989	.z,	3339	.z,
2990	);	3340	);
2991	try self.genBinOpMir(.add, ty, .{ .register = divisor }, .{ .register = .rax });	3341	try self.genBinOpMir(.{ ._, .add }, ty, .{ .register = divisor }, .{ .register = .rax });
2992	return MCValue{ .register = divisor };	3342	return MCValue{ .register = divisor };
2993	}	3343	}
2994		3344
...	@@ -3110,7 +3460,12 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3110,7 +3460,12 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
3110	const result = try self.copyToRegisterWithInstTracking(inst, err_union_ty, operand);	3460	const result = try self.copyToRegisterWithInstTracking(inst, err_union_ty, operand);
3111	if (err_off > 0) {	3461	if (err_off > 0) {
3112	const shift = @intCast(u6, err_off * 8);	3462	const shift = @intCast(u6, err_off * 8);
3113	try self.genShiftBinOpMir(.shr, err_union_ty, result, .{ .immediate = shift });	3463	try self.genShiftBinOpMir(
		3464	.{ ._r, .sh },
		3465	err_union_ty,
		3466	result,
		3467	.{ .immediate = shift },
		3468	);
3114	} else {	3469	} else {
3115	try self.truncateRegister(Type.anyerror, result.register);	3470	try self.truncateRegister(Type.anyerror, result.register);
3116	}	3471	}
...	@@ -3162,7 +3517,12 @@ fn genUnwrapErrorUnionPayloadMir(	...	@@ -3162,7 +3517,12 @@ fn genUnwrapErrorUnionPayloadMir(
3162	.{ .register = try self.copyToTmpRegister(err_union_ty, err_union) };	3517	.{ .register = try self.copyToTmpRegister(err_union_ty, err_union) };
3163	if (payload_off > 0) {	3518	if (payload_off > 0) {
3164	const shift = @intCast(u6, payload_off * 8);	3519	const shift = @intCast(u6, payload_off * 8);
3165	try self.genShiftBinOpMir(.shr, err_union_ty, result_mcv, .{ .immediate = shift });	3520	try self.genShiftBinOpMir(
		3521	.{ ._r, .sh },
		3522	err_union_ty,
		3523	result_mcv,
		3524	.{ .immediate = shift },
		3525	);
3166	} else {	3526	} else {
3167	try self.truncateRegister(payload_ty, result_mcv.register);	3527	try self.truncateRegister(payload_ty, result_mcv.register);
3168	}	3528	}
...	@@ -3199,7 +3559,7 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3199,7 +3559,7 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
3199	const err_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*));	3559	const err_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*));
3200	const err_abi_size = @intCast(u32, err_ty.abiSize(self.target.*));	3560	const err_abi_size = @intCast(u32, err_ty.abiSize(self.target.*));
3201	try self.asmRegisterMemory(	3561	try self.asmRegisterMemory(
3202	.mov,	3562	.{ ._, .mov },
3203	registerAlias(dst_reg, err_abi_size),	3563	registerAlias(dst_reg, err_abi_size),
3204	Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{	3564	Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{
3205	.base = .{ .reg = src_reg },	3565	.base = .{ .reg = src_reg },
...	@@ -3237,7 +3597,7 @@ fn airUnwrapErrUnionPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3237,7 +3597,7 @@ fn airUnwrapErrUnionPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
3237	const pl_off = @intCast(i32, errUnionPayloadOffset(pl_ty, self.target.*));	3597	const pl_off = @intCast(i32, errUnionPayloadOffset(pl_ty, self.target.*));
3238	const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));	3598	const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
3239	try self.asmRegisterMemory(	3599	try self.asmRegisterMemory(
3240	.lea,	3600	.{ ._, .lea },
3241	registerAlias(dst_reg, dst_abi_size),	3601	registerAlias(dst_reg, dst_abi_size),
3242	Memory.sib(.qword, .{ .base = .{ .reg = src_reg }, .disp = pl_off }),	3602	Memory.sib(.qword, .{ .base = .{ .reg = src_reg }, .disp = pl_off }),
3243	);	3603	);
...	@@ -3263,7 +3623,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3263,7 +3623,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
3263	const err_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*));	3623	const err_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*));
3264	const err_abi_size = @intCast(u32, err_ty.abiSize(self.target.*));	3624	const err_abi_size = @intCast(u32, err_ty.abiSize(self.target.*));
3265	try self.asmMemoryImmediate(	3625	try self.asmMemoryImmediate(
3266	.mov,	3626	.{ ._, .mov },
3267	Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{	3627	Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{
3268	.base = .{ .reg = src_reg },	3628	.base = .{ .reg = src_reg },
3269	.disp = err_off,	3629	.disp = err_off,
...	@@ -3284,7 +3644,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3284,7 +3644,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
3284	const pl_off = @intCast(i32, errUnionPayloadOffset(pl_ty, self.target.*));	3644	const pl_off = @intCast(i32, errUnionPayloadOffset(pl_ty, self.target.*));
3285	const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));	3645	const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
3286	try self.asmRegisterMemory(	3646	try self.asmRegisterMemory(
3287	.lea,	3647	.{ ._, .lea },
3288	registerAlias(dst_reg, dst_abi_size),	3648	registerAlias(dst_reg, dst_abi_size),
3289	Memory.sib(.qword, .{ .base = .{ .reg = src_reg }, .disp = pl_off }),	3649	Memory.sib(.qword, .{ .base = .{ .reg = src_reg }, .disp = pl_off }),
3290	);	3650	);
...	@@ -3335,13 +3695,13 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3335,13 +3695,13 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
3335	else => unreachable,	3695	else => unreachable,
3336		3696
3337	.register => \|opt_reg\| try self.asmRegisterImmediate(	3697	.register => \|opt_reg\| try self.asmRegisterImmediate(
3338	.bts,	3698	.{ ._s, .bt },
3339	opt_reg,	3699	opt_reg,
3340	Immediate.u(@intCast(u6, pl_abi_size * 8)),	3700	Immediate.u(@intCast(u6, pl_abi_size * 8)),
3341	),	3701	),
3342		3702
3343	.load_frame => \|frame_addr\| try self.asmMemoryImmediate(	3703	.load_frame => \|frame_addr\| try self.asmMemoryImmediate(
3344	.mov,	3704	.{ ._, .mov },
3345	Memory.sib(.byte, .{	3705	Memory.sib(.byte, .{
3346	.base = .{ .frame = frame_addr.index },	3706	.base = .{ .frame = frame_addr.index },
3347	.disp = frame_addr.off + pl_abi_size,	3707	.disp = frame_addr.off + pl_abi_size,
...	@@ -3453,7 +3813,7 @@ fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3453,7 +3813,7 @@ fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
3453		3813
3454	const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));	3814	const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
3455	try self.asmRegisterMemory(	3815	try self.asmRegisterMemory(
3456	.lea,	3816	.{ ._, .lea },
3457	registerAlias(dst_reg, dst_abi_size),	3817	registerAlias(dst_reg, dst_abi_size),
3458	Memory.sib(.qword, .{	3818	Memory.sib(.qword, .{
3459	.base = .{ .reg = src_reg },	3819	.base = .{ .reg = src_reg },
...	@@ -3527,7 +3887,7 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {	...	@@ -3527,7 +3887,7 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
3527	try self.genSetReg(addr_reg, Type.usize, slice_mcv);	3887	try self.genSetReg(addr_reg, Type.usize, slice_mcv);
3528	// TODO we could allocate register here, but need to expect addr register and potentially	3888	// TODO we could allocate register here, but need to expect addr register and potentially
3529	// offset register.	3889	// offset register.
3530	try self.genBinOpMir(.add, slice_ptr_field_type, .{ .register = addr_reg }, .{	3890	try self.genBinOpMir(.{ ._, .add }, slice_ptr_field_type, .{ .register = addr_reg }, .{
3531	.register = offset_reg,	3891	.register = offset_reg,
3532	});	3892	});
3533	return MCValue{ .register = addr_reg.to64() };	3893	return MCValue{ .register = addr_reg.to64() };
...	@@ -3585,13 +3945,13 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3585,13 +3945,13 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
3585	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, self.target.*));	3945	const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, self.target.*));
3586	try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array);	3946	try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array);
3587	try self.asmRegisterMemory(	3947	try self.asmRegisterMemory(
3588	.lea,	3948	.{ ._, .lea },
3589	addr_reg,	3949	addr_reg,
3590	Memory.sib(.qword, .{ .base = .{ .frame = frame_index } }),	3950	Memory.sib(.qword, .{ .base = .{ .frame = frame_index } }),
3591	);	3951	);
3592	},	3952	},
3593	.load_frame => \|frame_addr\| try self.asmRegisterMemory(	3953	.load_frame => \|frame_addr\| try self.asmRegisterMemory(
3594	.lea,	3954	.{ ._, .lea },
3595	addr_reg,	3955	addr_reg,
3596	Memory.sib(.qword, .{ .base = .{ .frame = frame_addr.index }, .disp = frame_addr.off }),	3956	Memory.sib(.qword, .{ .base = .{ .frame = frame_addr.index }, .disp = frame_addr.off }),
3597	),	3957	),
...	@@ -3607,7 +3967,12 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3607,7 +3967,12 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
3607	// TODO we could allocate register here, but need to expect addr register and potentially	3967	// TODO we could allocate register here, but need to expect addr register and potentially
3608	// offset register.	3968	// offset register.
3609	const dst_mcv = try self.allocRegOrMem(inst, false);	3969	const dst_mcv = try self.allocRegOrMem(inst, false);
3610	try self.genBinOpMir(.add, Type.usize, .{ .register = addr_reg }, .{ .register = offset_reg });	3970	try self.genBinOpMir(
		3971	.{ ._, .add },
		3972	Type.usize,
		3973	.{ .register = addr_reg },
		3974	.{ .register = offset_reg },
		3975	);
3611	try self.genCopy(elem_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } });	3976	try self.genCopy(elem_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } });
3612		3977
3613	return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });	3978	return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
...	@@ -3641,7 +4006,11 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3641,7 +4006,11 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
3641	try self.copyToTmpRegister(ptr_ty, ptr_mcv);	4006	try self.copyToTmpRegister(ptr_ty, ptr_mcv);
3642	const elem_ptr_lock = self.register_manager.lockRegAssumeUnused(elem_ptr_reg);	4007	const elem_ptr_lock = self.register_manager.lockRegAssumeUnused(elem_ptr_reg);
3643	defer self.register_manager.unlockReg(elem_ptr_lock);	4008	defer self.register_manager.unlockReg(elem_ptr_lock);
3644	try self.asmRegisterRegister(.add, elem_ptr_reg, offset_reg);	4009	try self.asmRegisterRegister(
		4010	.{ ._, .add },
		4011	elem_ptr_reg,
		4012	offset_reg,
		4013	);
3645		4014
3646	const dst_mcv = try self.allocRegOrMem(inst, true);	4015	const dst_mcv = try self.allocRegOrMem(inst, true);
3647	const dst_lock = switch (dst_mcv) {	4016	const dst_lock = switch (dst_mcv) {
...	@@ -3681,7 +4050,7 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3681,7 +4050,7 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
3681	defer self.register_manager.unlockReg(offset_reg_lock);	4050	defer self.register_manager.unlockReg(offset_reg_lock);
3682		4051
3683	const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr);	4052	const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr);
3684	try self.genBinOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg });	4053	try self.genBinOpMir(.{ ._, .add }, ptr_ty, dst_mcv, .{ .register = offset_reg });
3685		4054
3686	return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none });	4055	return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none });
3687	}	4056	}
...	@@ -3714,7 +4083,12 @@ fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3714,7 +4083,12 @@ fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
3714	const adjusted_ptr: MCValue = if (layout.payload_size > 0 and layout.tag_align < layout.payload_align) blk: {	4083	const adjusted_ptr: MCValue = if (layout.payload_size > 0 and layout.tag_align < layout.payload_align) blk: {
3715	// TODO reusing the operand	4084	// TODO reusing the operand
3716	const reg = try self.copyToTmpRegister(ptr_union_ty, ptr);	4085	const reg = try self.copyToTmpRegister(ptr_union_ty, ptr);
3717	try self.genBinOpMir(.add, ptr_union_ty, .{ .register = reg }, .{ .immediate = layout.payload_size });	4086	try self.genBinOpMir(
		4087	.{ ._, .add },
		4088	ptr_union_ty,
		4089	.{ .register = reg },
		4090	.{ .immediate = layout.payload_size },
		4091	);
3718	break :blk MCValue{ .register = reg };	4092	break :blk MCValue{ .register = reg };
3719	} else ptr;	4093	} else ptr;
3720		4094
...	@@ -3767,7 +4141,7 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3767,7 +4141,7 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
3767	else	4141	else
3768	0;	4142	0;
3769	const result = try self.copyToRegisterWithInstTracking(inst, union_ty, operand);	4143	const result = try self.copyToRegisterWithInstTracking(inst, union_ty, operand);
3770	try self.genShiftBinOpMir(.shr, Type.usize, result, .{ .immediate = shift });	4144	try self.genShiftBinOpMir(.{ ._r, .sh }, Type.usize, result, .{ .immediate = shift });
3771	break :blk MCValue{	4145	break :blk MCValue{
3772	.register = registerAlias(result.register, @intCast(u32, layout.tag_size)),	4146	.register = registerAlias(result.register, @intCast(u32, layout.tag_size)),
3773	};	4147	};
...	@@ -3798,24 +4172,53 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3798,24 +4172,53 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
3798		4172
3799	const dst_reg = try self.register_manager.allocReg(inst, gp);	4173	const dst_reg = try self.register_manager.allocReg(inst, gp);
3800	const dst_mcv = MCValue{ .register = dst_reg };	4174	const dst_mcv = MCValue{ .register = dst_reg };
3801	const dst_lock = self.register_manager.lockReg(dst_reg);	4175	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
3802	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);	4176	defer self.register_manager.unlockReg(dst_lock);
3803		4177
3804	if (Target.x86.featureSetHas(self.target.cpu.features, .lzcnt)) {	4178	const src_bits = src_ty.bitSize(self.target.*);
3805	try self.genBinOpMir(.lzcnt, src_ty, dst_mcv, mat_src_mcv);	4179	if (self.hasFeature(.lzcnt)) {
3806	const extra_bits = self.regExtraBits(src_ty);	4180	if (src_bits <= 64) {
3807	if (extra_bits > 0) {	4181	try self.genBinOpMir(.{ ._, .lzcnt }, src_ty, dst_mcv, mat_src_mcv);
3808	try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .immediate = extra_bits });	4182
3809	}	4183	const extra_bits = self.regExtraBits(src_ty);
		4184	if (extra_bits > 0) {
		4185	try self.genBinOpMir(.{ ._, .sub }, dst_ty, dst_mcv, .{ .immediate = extra_bits });
		4186	}
		4187	} else if (src_bits <= 128) {
		4188	const tmp_reg = try self.register_manager.allocReg(null, gp);
		4189	const tmp_mcv = MCValue{ .register = tmp_reg };
		4190	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
		4191	defer self.register_manager.unlockReg(tmp_lock);
		4192
		4193	try self.genBinOpMir(.{ ._, .lzcnt }, Type.u64, dst_mcv, mat_src_mcv);
		4194	try self.genBinOpMir(.{ ._, .add }, dst_ty, dst_mcv, .{ .immediate = 64 });
		4195	try self.genBinOpMir(
		4196	.{ ._, .lzcnt },
		4197	Type.u64,
		4198	tmp_mcv,
		4199	mat_src_mcv.address().offset(8).deref(),
		4200	);
		4201	try self.asmCmovccRegisterRegister(dst_reg.to32(), tmp_reg.to32(), .nc);
		4202
		4203	if (src_bits < 128) {
		4204	try self.genBinOpMir(
		4205	.{ ._, .sub },
		4206	dst_ty,
		4207	dst_mcv,
		4208	.{ .immediate = 128 - src_bits },
		4209	);
		4210	}
		4211	} else return self.fail("TODO airClz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
3810	break :result dst_mcv;	4212	break :result dst_mcv;
3811	}	4213	}
3812		4214
3813	const src_bits = src_ty.bitSize(self.target.*);	4215	if (src_bits > 64)
		4216	return self.fail("TODO airClz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
3814	if (math.isPowerOfTwo(src_bits)) {	4217	if (math.isPowerOfTwo(src_bits)) {
3815	const imm_reg = try self.copyToTmpRegister(dst_ty, .{	4218	const imm_reg = try self.copyToTmpRegister(dst_ty, .{
3816	.immediate = src_bits ^ (src_bits - 1),	4219	.immediate = src_bits ^ (src_bits - 1),
3817	});	4220	});
3818	try self.genBinOpMir(.bsr, src_ty, dst_mcv, mat_src_mcv);	4221	try self.genBinOpMir(.{ ._, .bsr }, src_ty, dst_mcv, mat_src_mcv);
3819		4222
3820	const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);	4223	const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
3821	try self.asmCmovccRegisterRegister(	4224	try self.asmCmovccRegisterRegister(
...	@@ -3824,12 +4227,12 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3824,12 +4227,12 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
3824	.z,	4227	.z,
3825	);	4228	);
3826		4229
3827	try self.genBinOpMir(.xor, dst_ty, dst_mcv, .{ .immediate = src_bits - 1 });	4230	try self.genBinOpMir(.{ ._, .xor }, dst_ty, dst_mcv, .{ .immediate = src_bits - 1 });
3828	} else {	4231	} else {
3829	const imm_reg = try self.copyToTmpRegister(dst_ty, .{	4232	const imm_reg = try self.copyToTmpRegister(dst_ty, .{
3830	.immediate = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - self.regBitSize(dst_ty)),	4233	.immediate = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - self.regBitSize(dst_ty)),
3831	});	4234	});
3832	try self.genBinOpMir(.bsr, src_ty, dst_mcv, mat_src_mcv);	4235	try self.genBinOpMir(.{ ._, .bsr }, src_ty, dst_mcv, mat_src_mcv);
3833		4236
3834	const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);	4237	const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
3835	try self.asmCmovccRegisterRegister(	4238	try self.asmCmovccRegisterRegister(
...	@@ -3839,7 +4242,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3839,7 +4242,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
3839	);	4242	);
3840		4243
3841	try self.genSetReg(dst_reg, dst_ty, .{ .immediate = src_bits - 1 });	4244	try self.genSetReg(dst_reg, dst_ty, .{ .immediate = src_bits - 1 });
3842	try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .register = imm_reg });	4245	try self.genBinOpMir(.{ ._, .sub }, dst_ty, dst_mcv, .{ .register = imm_reg });
3843	}	4246	}
3844	break :result dst_mcv;	4247	break :result dst_mcv;
3845	};	4248	};
...	@@ -3869,27 +4272,55 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3869,27 +4272,55 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
3869	const dst_lock = self.register_manager.lockReg(dst_reg);	4272	const dst_lock = self.register_manager.lockReg(dst_reg);
3870	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);	4273	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
3871		4274
3872	if (Target.x86.featureSetHas(self.target.cpu.features, .bmi)) {	4275	if (self.hasFeature(.bmi)) {
3873	const extra_bits = self.regExtraBits(src_ty);	4276	if (src_bits <= 64) {
3874	const masked_mcv = if (extra_bits > 0) masked: {	4277	const extra_bits = self.regExtraBits(src_ty);
3875	const mask_mcv = MCValue{	4278	const masked_mcv = if (extra_bits > 0) masked: {
3876	.immediate = ((@as(u64, 1) << @intCast(u6, extra_bits)) - 1) <<	4279	const tmp_mcv = tmp: {
3877	@intCast(u6, src_bits),	4280	if (src_mcv.isImmediate() or self.liveness.operandDies(inst, 0))
3878	};	4281	break :tmp src_mcv;
3879	const tmp_mcv = tmp: {	4282	try self.genSetReg(dst_reg, src_ty, src_mcv);
3880	if (src_mcv.isImmediate() or self.liveness.operandDies(inst, 0)) break :tmp src_mcv;	4283	break :tmp dst_mcv;
3881	try self.genSetReg(dst_reg, src_ty, src_mcv);	4284	};
3882	break :tmp dst_mcv;	4285	try self.genBinOpMir(
3883	};	4286	.{ ._, .@"or" },
3884	try self.genBinOpMir(.@"or", src_ty, tmp_mcv, mask_mcv);	4287	src_ty,
3885	break :masked tmp_mcv;	4288	tmp_mcv,
3886	} else mat_src_mcv;	4289	.{ .immediate = (@as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - extra_bits)) <<
3887	try self.genBinOpMir(.tzcnt, src_ty, dst_mcv, masked_mcv);	4290	@intCast(u6, src_bits) },
		4291	);
		4292	break :masked tmp_mcv;
		4293	} else mat_src_mcv;
		4294	try self.genBinOpMir(.{ ._, .tzcnt }, src_ty, dst_mcv, masked_mcv);
		4295	} else if (src_bits <= 128) {
		4296	const tmp_reg = try self.register_manager.allocReg(null, gp);
		4297	const tmp_mcv = MCValue{ .register = tmp_reg };
		4298	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
		4299	defer self.register_manager.unlockReg(tmp_lock);
		4300
		4301	const masked_mcv = if (src_bits < 128) masked: {
		4302	try self.genCopy(Type.u64, dst_mcv, mat_src_mcv.address().offset(8).deref());
		4303	try self.genBinOpMir(
		4304	.{ ._, .@"or" },
		4305	Type.u64,
		4306	dst_mcv,
		4307	.{ .immediate = @as(u64, math.maxInt(u64)) << @intCast(u6, src_bits - 64) },
		4308	);
		4309	break :masked dst_mcv;
		4310	} else mat_src_mcv.address().offset(8).deref();
		4311	try self.genBinOpMir(.{ ._, .tzcnt }, Type.u64, dst_mcv, masked_mcv);
		4312	try self.genBinOpMir(.{ ._, .add }, dst_ty, dst_mcv, .{ .immediate = 64 });
		4313	try self.genBinOpMir(.{ ._, .tzcnt }, Type.u64, tmp_mcv, mat_src_mcv);
		4314	try self.asmCmovccRegisterRegister(dst_reg.to32(), tmp_reg.to32(), .nc);
		4315	} else return self.fail("TODO airCtz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
3888	break :result dst_mcv;	4316	break :result dst_mcv;
3889	}	4317	}
3890		4318
		4319	if (src_bits > 64)
		4320	return self.fail("TODO airCtz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
		4321
3891	const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits });	4322	const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits });
3892	try self.genBinOpMir(.bsf, src_ty, dst_mcv, mat_src_mcv);	4323	try self.genBinOpMir(.{ ._, .bsf }, src_ty, dst_mcv, mat_src_mcv);
3893		4324
3894	const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);	4325	const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
3895	try self.asmCmovccRegisterRegister(	4326	try self.asmCmovccRegisterRegister(
...	@@ -3909,7 +4340,7 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3909,7 +4340,7 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
3909	const src_abi_size = @intCast(u32, src_ty.abiSize(self.target.*));	4340	const src_abi_size = @intCast(u32, src_ty.abiSize(self.target.*));
3910	const src_mcv = try self.resolveInst(ty_op.operand);	4341	const src_mcv = try self.resolveInst(ty_op.operand);
3911		4342
3912	if (Target.x86.featureSetHas(self.target.cpu.features, .popcnt)) {	4343	if (self.hasFeature(.popcnt)) {
3913	const mat_src_mcv = switch (src_mcv) {	4344	const mat_src_mcv = switch (src_mcv) {
3914	.immediate => MCValue{ .register = try self.copyToTmpRegister(src_ty, src_mcv) },	4345	.immediate => MCValue{ .register = try self.copyToTmpRegister(src_ty, src_mcv) },
3915	else => src_mcv,	4346	else => src_mcv,
...	@@ -3927,7 +4358,7 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3927,7 +4358,7 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
3927	.{ .register = try self.register_manager.allocReg(inst, gp) };	4358	.{ .register = try self.register_manager.allocReg(inst, gp) };
3928		4359
3929	const popcnt_ty = if (src_abi_size > 1) src_ty else Type.u16;	4360	const popcnt_ty = if (src_abi_size > 1) src_ty else Type.u16;
3930	try self.genBinOpMir(.popcnt, popcnt_ty, dst_mcv, mat_src_mcv);	4361	try self.genBinOpMir(.{ ._, .popcnt }, popcnt_ty, dst_mcv, mat_src_mcv);
3931	break :result dst_mcv;	4362	break :result dst_mcv;
3932	}	4363	}
3933		4364
...	@@ -3958,54 +4389,54 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3958,54 +4389,54 @@ fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
3958	undefined;	4389	undefined;
3959		4390
3960	// dst = operand	4391	// dst = operand
3961	try self.asmRegisterRegister(.mov, tmp, dst);	4392	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
3962	// tmp = operand	4393	// tmp = operand
3963	try self.asmRegisterImmediate(.shr, tmp, Immediate.u(1));	4394	try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, Immediate.u(1));
3964	// tmp = operand >> 1	4395	// tmp = operand >> 1
3965	if (src_abi_size > 4) {	4396	if (src_abi_size > 4) {
3966	try self.asmRegisterImmediate(.mov, imm, imm_0_1);	4397	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0_1);
3967	try self.asmRegisterRegister(.@"and", tmp, imm);	4398	try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm);
3968	} else try self.asmRegisterImmediate(.@"and", tmp, imm_0_1);	4399	} else try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_0_1);
3969	// tmp = (operand >> 1) & 0x55...55	4400	// tmp = (operand >> 1) & 0x55...55
3970	try self.asmRegisterRegister(.sub, dst, tmp);	4401	try self.asmRegisterRegister(.{ ._, .sub }, dst, tmp);
3971	// dst = temp1 = operand - ((operand >> 1) & 0x55...55)	4402	// dst = temp1 = operand - ((operand >> 1) & 0x55...55)
3972	try self.asmRegisterRegister(.mov, tmp, dst);	4403	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
3973	// tmp = temp1	4404	// tmp = temp1
3974	try self.asmRegisterImmediate(.shr, dst, Immediate.u(2));	4405	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u(2));
3975	// dst = temp1 >> 2	4406	// dst = temp1 >> 2
3976	if (src_abi_size > 4) {	4407	if (src_abi_size > 4) {
3977	try self.asmRegisterImmediate(.mov, imm, imm_00_11);	4408	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_00_11);
3978	try self.asmRegisterRegister(.@"and", tmp, imm);	4409	try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm);
3979	try self.asmRegisterRegister(.@"and", dst, imm);	4410	try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm);
3980	} else {	4411	} else {
3981	try self.asmRegisterImmediate(.@"and", tmp, imm_00_11);	4412	try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_00_11);
3982	try self.asmRegisterImmediate(.@"and", dst, imm_00_11);	4413	try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_00_11);
3983	}	4414	}
3984	// tmp = temp1 & 0x33...33	4415	// tmp = temp1 & 0x33...33
3985	// dst = (temp1 >> 2) & 0x33...33	4416	// dst = (temp1 >> 2) & 0x33...33
3986	try self.asmRegisterRegister(.add, tmp, dst);	4417	try self.asmRegisterRegister(.{ ._, .add }, tmp, dst);
3987	// tmp = temp2 = (temp1 & 0x33...33) + ((temp1 >> 2) & 0x33...33)	4418	// tmp = temp2 = (temp1 & 0x33...33) + ((temp1 >> 2) & 0x33...33)
3988	try self.asmRegisterRegister(.mov, dst, tmp);	4419	try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp);
3989	// dst = temp2	4420	// dst = temp2
3990	try self.asmRegisterImmediate(.shr, tmp, Immediate.u(4));	4421	try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, Immediate.u(4));
3991	// tmp = temp2 >> 4	4422	// tmp = temp2 >> 4
3992	try self.asmRegisterRegister(.add, dst, tmp);	4423	try self.asmRegisterRegister(.{ ._, .add }, dst, tmp);
3993	// dst = temp2 + (temp2 >> 4)	4424	// dst = temp2 + (temp2 >> 4)
3994	if (src_abi_size > 4) {	4425	if (src_abi_size > 4) {
3995	try self.asmRegisterImmediate(.mov, imm, imm_0000_1111);	4426	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0000_1111);
3996	try self.asmRegisterImmediate(.mov, tmp, imm_0000_0001);	4427	try self.asmRegisterImmediate(.{ ._, .mov }, tmp, imm_0000_0001);
3997	try self.asmRegisterRegister(.@"and", dst, imm);	4428	try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm);
3998	try self.asmRegisterRegister(.imul, dst, tmp);	4429	try self.asmRegisterRegister(.{ .i_, .mul }, dst, tmp);
3999	} else {	4430	} else {
4000	try self.asmRegisterImmediate(.@"and", dst, imm_0000_1111);	4431	try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_0000_1111);
4001	if (src_abi_size > 1) {	4432	if (src_abi_size > 1) {
4002	try self.asmRegisterRegisterImmediate(.imul, dst, dst, imm_0000_0001);	4433	try self.asmRegisterRegisterImmediate(.{ .i_, .mul }, dst, dst, imm_0000_0001);
4003	}	4434	}
4004	}	4435	}
4005	// dst = temp3 = (temp2 + (temp2 >> 4)) & 0x0f...0f	4436	// dst = temp3 = (temp2 + (temp2 >> 4)) & 0x0f...0f
4006	// dst = temp3 * 0x01...01	4437	// dst = temp3 * 0x01...01
4007	if (src_abi_size > 1) {	4438	if (src_abi_size > 1) {
4008	try self.asmRegisterImmediate(.shr, dst, Immediate.u((src_abi_size - 1) * 8));	4439	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u((src_abi_size - 1) * 8));
4009	}	4440	}
4010	// dst = (temp3 * 0x01...01) >> (bits - 8)	4441	// dst = (temp3 * 0x01...01) >> (bits - 8)
4011	}	4442	}
...	@@ -4034,11 +4465,11 @@ fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, m	...	@@ -4034,11 +4465,11 @@ fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, m
4034	16 => if ((mem_ok or src_mcv.isRegister()) and	4465	16 => if ((mem_ok or src_mcv.isRegister()) and
4035	self.reuseOperand(inst, ty_op.operand, 0, src_mcv))	4466	self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
4036	{	4467	{
4037	try self.genBinOpMir(.rol, src_ty, src_mcv, .{ .immediate = 8 });	4468	try self.genBinOpMir(.{ ._l, .ro }, src_ty, src_mcv, .{ .immediate = 8 });
4038	return src_mcv;	4469	return src_mcv;
4039	},	4470	},
4040	32, 64 => if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) {	4471	32, 64 => if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) {
4041	try self.genUnOpMir(.bswap, src_ty, src_mcv);	4472	try self.genUnOpMir(.{ ._, .bswap }, src_ty, src_mcv);
4042	return src_mcv;	4473	return src_mcv;
4043	},	4474	},
4044	}	4475	}
...	@@ -4055,10 +4486,10 @@ fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, m	...	@@ -4055,10 +4486,10 @@ fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, m
4055	try self.genSetReg(dst_mcv.register, src_ty, src_mcv);	4486	try self.genSetReg(dst_mcv.register, src_ty, src_mcv);
4056	switch (src_bits) {	4487	switch (src_bits) {
4057	else => unreachable,	4488	else => unreachable,
4058	16 => try self.genBinOpMir(.rol, src_ty, dst_mcv, .{ .immediate = 8 }),	4489	16 => try self.genBinOpMir(.{ ._l, .ro }, src_ty, dst_mcv, .{ .immediate = 8 }),
4059	32, 64 => try self.genUnOpMir(.bswap, src_ty, dst_mcv),	4490	32, 64 => try self.genUnOpMir(.{ ._, .bswap }, src_ty, dst_mcv),
4060	}	4491	}
4061	} else try self.genBinOpMir(.movbe, src_ty, dst_mcv, src_mcv);	4492	} else try self.genBinOpMir(.{ ._, .movbe }, src_ty, dst_mcv, src_mcv);
4062	return dst_mcv;	4493	return dst_mcv;
4063	}	4494	}
4064		4495
...	@@ -4067,7 +4498,7 @@ fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, m	...	@@ -4067,7 +4498,7 @@ fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, m
4067	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);	4498	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
4068	defer self.register_manager.unlockReg(dst_lock);	4499	defer self.register_manager.unlockReg(dst_lock);
4069		4500
4070	try self.genBinOpMir(.movbe, src_ty, dst_mcv, src_mcv);	4501	try self.genBinOpMir(.{ ._, .movbe }, src_ty, dst_mcv, src_mcv);
4071	return dst_mcv;	4502	return dst_mcv;
4072	}	4503	}
4073		4504
...	@@ -4081,7 +4512,7 @@ fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4081,7 +4512,7 @@ fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {
4081	switch (self.regExtraBits(src_ty)) {	4512	switch (self.regExtraBits(src_ty)) {
4082	0 => {},	4513	0 => {},
4083	else => \|extra\| try self.genBinOpMir(	4514	else => \|extra\| try self.genBinOpMir(
4084	if (src_ty.isSignedInt()) .sar else .shr,	4515	if (src_ty.isSignedInt()) .{ ._r, .sa } else .{ ._r, .sh },
4085	src_ty,	4516	src_ty,
4086	dst_mcv,	4517	dst_mcv,
4087	.{ .immediate = extra },	4518	.{ .immediate = extra },
...	@@ -4121,40 +4552,40 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4121,40 +4552,40 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
4121	const imm_0_1 = Immediate.u(mask / 0b1_1);	4552	const imm_0_1 = Immediate.u(mask / 0b1_1);
4122		4553
4123	// dst = temp1 = bswap(operand)	4554	// dst = temp1 = bswap(operand)
4124	try self.asmRegisterRegister(.mov, tmp, dst);	4555	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
4125	// tmp = temp1	4556	// tmp = temp1
4126	try self.asmRegisterImmediate(.shr, dst, Immediate.u(4));	4557	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u(4));
4127	// dst = temp1 >> 4	4558	// dst = temp1 >> 4
4128	if (src_abi_size > 4) {	4559	if (src_abi_size > 4) {
4129	try self.asmRegisterImmediate(.mov, imm, imm_0000_1111);	4560	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0000_1111);
4130	try self.asmRegisterRegister(.@"and", tmp, imm);	4561	try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm);
4131	try self.asmRegisterRegister(.@"and", dst, imm);	4562	try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm);
4132	} else {	4563	} else {
4133	try self.asmRegisterImmediate(.@"and", tmp, imm_0000_1111);	4564	try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_0000_1111);
4134	try self.asmRegisterImmediate(.@"and", dst, imm_0000_1111);	4565	try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_0000_1111);
4135	}	4566	}
4136	// tmp = temp1 & 0x0F...0F	4567	// tmp = temp1 & 0x0F...0F
4137	// dst = (temp1 >> 4) & 0x0F...0F	4568	// dst = (temp1 >> 4) & 0x0F...0F
4138	try self.asmRegisterImmediate(.shl, tmp, Immediate.u(4));	4569	try self.asmRegisterImmediate(.{ ._l, .sh }, tmp, Immediate.u(4));
4139	// tmp = (temp1 & 0x0F...0F) << 4	4570	// tmp = (temp1 & 0x0F...0F) << 4
4140	try self.asmRegisterRegister(.@"or", dst, tmp);	4571	try self.asmRegisterRegister(.{ ._, .@"or" }, dst, tmp);
4141	// dst = temp2 = ((temp1 >> 4) & 0x0F...0F) \| ((temp1 & 0x0F...0F) << 4)	4572	// dst = temp2 = ((temp1 >> 4) & 0x0F...0F) \| ((temp1 & 0x0F...0F) << 4)
4142	try self.asmRegisterRegister(.mov, tmp, dst);	4573	try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
4143	// tmp = temp2	4574	// tmp = temp2
4144	try self.asmRegisterImmediate(.shr, dst, Immediate.u(2));	4575	try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u(2));
4145	// dst = temp2 >> 2	4576	// dst = temp2 >> 2
4146	if (src_abi_size > 4) {	4577	if (src_abi_size > 4) {
4147	try self.asmRegisterImmediate(.mov, imm, imm_00_11);	4578	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_00_11);
4148	try self.asmRegisterRegister(.@"and", tmp, imm);	4579	try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm);
4149	try self.asmRegisterRegister(.@"and", dst, imm);	4580	try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm);
4150	} else {	4581	} else {
4151	try self.asmRegisterImmediate(.@"and", tmp, imm_00_11);	4582	try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_00_11);
4152	try self.asmRegisterImmediate(.@"and", dst, imm_00_11);	4583	try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_00_11);
4153	}	4584	}
4154	// tmp = temp2 & 0x33...33	4585	// tmp = temp2 & 0x33...33
4155	// dst = (temp2 >> 2) & 0x33...33	4586	// dst = (temp2 >> 2) & 0x33...33
4156	try self.asmRegisterMemory(	4587	try self.asmRegisterMemory(
4157	.lea,	4588	.{ ._, .lea },
4158	if (src_abi_size > 4) tmp.to64() else tmp.to32(),	4589	if (src_abi_size > 4) tmp.to64() else tmp.to32(),
4159	Memory.sib(.qword, .{	4590	Memory.sib(.qword, .{
4160	.base = .{ .reg = dst.to64() },	4591	.base = .{ .reg = dst.to64() },
...	@@ -4162,22 +4593,22 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4162,22 +4593,22 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
4162	}),	4593	}),
4163	);	4594	);
4164	// tmp = temp3 = ((temp2 >> 2) & 0x33...33) + ((temp2 & 0x33...33) << 2)	4595	// tmp = temp3 = ((temp2 >> 2) & 0x33...33) + ((temp2 & 0x33...33) << 2)
4165	try self.asmRegisterRegister(.mov, dst, tmp);	4596	try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp);
4166	// dst = temp3	4597	// dst = temp3
4167	try self.asmRegisterImmediate(.shr, tmp, Immediate.u(1));	4598	try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, Immediate.u(1));
4168	// tmp = temp3 >> 1	4599	// tmp = temp3 >> 1
4169	if (src_abi_size > 4) {	4600	if (src_abi_size > 4) {
4170	try self.asmRegisterImmediate(.mov, imm, imm_0_1);	4601	try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0_1);
4171	try self.asmRegisterRegister(.@"and", dst, imm);	4602	try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm);
4172	try self.asmRegisterRegister(.@"and", tmp, imm);	4603	try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm);
4173	} else {	4604	} else {
4174	try self.asmRegisterImmediate(.@"and", dst, imm_0_1);	4605	try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_0_1);
4175	try self.asmRegisterImmediate(.@"and", tmp, imm_0_1);	4606	try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_0_1);
4176	}	4607	}
4177	// dst = temp3 & 0x55...55	4608	// dst = temp3 & 0x55...55
4178	// tmp = (temp3 >> 1) & 0x55...55	4609	// tmp = (temp3 >> 1) & 0x55...55
4179	try self.asmRegisterMemory(	4610	try self.asmRegisterMemory(
4180	.lea,	4611	.{ ._, .lea },
4181	if (src_abi_size > 4) dst.to64() else dst.to32(),	4612	if (src_abi_size > 4) dst.to64() else dst.to32(),
4182	Memory.sib(.qword, .{	4613	Memory.sib(.qword, .{
4183	.base = .{ .reg = tmp.to64() },	4614	.base = .{ .reg = tmp.to64() },
...	@@ -4190,7 +4621,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4190,7 +4621,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
4190	switch (self.regExtraBits(src_ty)) {	4621	switch (self.regExtraBits(src_ty)) {
4191	0 => {},	4622	0 => {},
4192	else => \|extra\| try self.genBinOpMir(	4623	else => \|extra\| try self.genBinOpMir(
4193	if (src_ty.isSignedInt()) .sar else .shr,	4624	if (src_ty.isSignedInt()) .{ ._r, .sa } else .{ ._r, .sh },
4194	src_ty,	4625	src_ty,
4195	dst_mcv,	4626	dst_mcv,
4196	.{ .immediate = extra },	4627	.{ .immediate = extra },
...	@@ -4246,19 +4677,20 @@ fn airFloatSign(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4246,19 +4677,20 @@ fn airFloatSign(self: *Self, inst: Air.Inst.Index) !void {
4246	const tag = self.air.instructions.items(.tag)[inst];	4677	const tag = self.air.instructions.items(.tag)[inst];
4247	try self.genBinOpMir(switch (ty_bits) {	4678	try self.genBinOpMir(switch (ty_bits) {
4248	// No point using an extra prefix byte for *pd which performs the same operation.	4679	// No point using an extra prefix byte for *pd which performs the same operation.
4249	32, 64 => switch (tag) {	4680	16, 32, 64, 128 => switch (tag) {
4250	.neg => .xorps,	4681	.neg => .{ ._ps, .xor },
4251	.fabs => .andnps,	4682	.fabs => .{ ._ps, .andn },
4252	else => unreachable,	4683	else => unreachable,
4253	},	4684	},
4254	else => return self.fail("TODO implement airFloatSign for {}", .{	4685	80 => return self.fail("TODO implement airFloatSign for {}", .{
4255	ty.fmt(self.bin_file.options.module.?),	4686	ty.fmt(self.bin_file.options.module.?),
4256	}),	4687	}),
		4688	else => unreachable,
4257	}, vec_ty, dst_mcv, sign_mcv);	4689	}, vec_ty, dst_mcv, sign_mcv);
4258	return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });	4690	return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
4259	}	4691	}
4260		4692
4261	fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {	4693	fn airRound(self: *Self, inst: Air.Inst.Index, mode: u4) !void {
4262	const un_op = self.air.instructions.items(.data)[inst].un_op;	4694	const un_op = self.air.instructions.items(.data)[inst].un_op;
4263	const ty = self.air.typeOf(un_op);	4695	const ty = self.air.typeOf(un_op);
4264		4696
...	@@ -4267,40 +4699,246 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4267,40 +4699,246 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
4267	src_mcv	4699	src_mcv
4268	else	4700	else
4269	try self.copyToRegisterWithInstTracking(inst, ty, src_mcv);	4701	try self.copyToRegisterWithInstTracking(inst, ty, src_mcv);
4270		4702	const dst_reg = dst_mcv.getReg().?;
4271	try self.genBinOpMir(switch (ty.zigTypeTag()) {	4703	const dst_lock = self.register_manager.lockReg(dst_reg);
4272	.Float => switch (ty.floatBits(self.target.*)) {	4704	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
4273	32 => .sqrtss,	4705	try self.genRound(ty, dst_reg, src_mcv, mode);
4274	64 => .sqrtsd,
4275	else => return self.fail("TODO implement airSqrt for {}", .{
4276	ty.fmt(self.bin_file.options.module.?),
4277	}),
4278	},
4279	else => return self.fail("TODO implement airSqrt for {}", .{
4280	ty.fmt(self.bin_file.options.module.?),
4281	}),
4282	}, ty, dst_mcv, src_mcv);
4283	return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });	4706	return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
4284	}	4707	}
4285		4708
4286	fn airUnaryMath(self: *Self, inst: Air.Inst.Index) !void {	4709	fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: u4) !void {
4287	const un_op = self.air.instructions.items(.data)[inst].un_op;	4710	if (!self.hasFeature(.sse4_1))
4288	_ = un_op;	4711	return self.fail("TODO implement genRound without sse4_1 feature", .{});
4289	return self.fail("TODO implement airUnaryMath for {}", .{
4290	self.air.instructions.items(.tag)[inst],
4291	});
4292	//return self.finishAir(inst, result, .{ un_op, .none, .none });
4293	}
4294		4712
4295	fn reuseOperand(	4713	const mir_tag = if (@as(?Mir.Inst.FixedTag, switch (ty.zigTypeTag()) {
4296	self: *Self,	4714	.Float => switch (ty.floatBits(self.target.*)) {
4297	inst: Air.Inst.Index,	4715	32 => if (self.hasFeature(.avx)) .{ .v_ss, .round } else .{ ._ss, .round },
4298	operand: Air.Inst.Ref,	4716	64 => if (self.hasFeature(.avx)) .{ .v_sd, .round } else .{ ._sd, .round },
4299	op_index: Liveness.OperandInt,	4717	16, 80, 128 => null,
4300	mcv: MCValue,	4718	else => unreachable,
4301	) bool {	4719	},
4302	return self.reuseOperandAdvanced(inst, operand, op_index, mcv, inst);	4720	.Vector => switch (ty.childType().zigTypeTag()) {
4303	}	4721	.Float => switch (ty.childType().floatBits(self.target.*)) {
		4722	32 => switch (ty.vectorLen()) {
		4723	1 => if (self.hasFeature(.avx)) .{ .v_ss, .round } else .{ ._ss, .round },
		4724	2...4 => if (self.hasFeature(.avx)) .{ .v_ps, .round } else .{ ._ps, .round },
		4725	5...8 => if (self.hasFeature(.avx)) .{ .v_ps, .round } else null,
		4726	else => null,
		4727	},
		4728	64 => switch (ty.vectorLen()) {
		4729	1 => if (self.hasFeature(.avx)) .{ .v_sd, .round } else .{ ._sd, .round },
		4730	2 => if (self.hasFeature(.avx)) .{ .v_pd, .round } else .{ ._pd, .round },
		4731	3...4 => if (self.hasFeature(.avx)) .{ .v_pd, .round } else null,
		4732	else => null,
		4733	},
		4734	16, 80, 128 => null,
		4735	else => unreachable,
		4736	},
		4737	else => null,
		4738	},
		4739	else => unreachable,
		4740	})) \|tag\| tag else return self.fail("TODO implement genRound for {}", .{
		4741	ty.fmt(self.bin_file.options.module.?),
		4742	});
		4743
		4744	const abi_size = @intCast(u32, ty.abiSize(self.target.*));
		4745	const dst_alias = registerAlias(dst_reg, abi_size);
		4746	switch (mir_tag[0]) {
		4747	.v_ss, .v_sd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
		4748	mir_tag,
		4749	dst_alias,
		4750	dst_alias,
		4751	src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
		4752	Immediate.u(mode),
		4753	) else try self.asmRegisterRegisterRegisterImmediate(
		4754	mir_tag,
		4755	dst_alias,
		4756	dst_alias,
		4757	registerAlias(if (src_mcv.isRegister())
		4758	src_mcv.getReg().?
		4759	else
		4760	try self.copyToTmpRegister(ty, src_mcv), abi_size),
		4761	Immediate.u(mode),
		4762	),
		4763	else => if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
		4764	mir_tag,
		4765	dst_alias,
		4766	src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
		4767	Immediate.u(mode),
		4768	) else try self.asmRegisterRegisterImmediate(
		4769	mir_tag,
		4770	dst_alias,
		4771	registerAlias(if (src_mcv.isRegister())
		4772	src_mcv.getReg().?
		4773	else
		4774	try self.copyToTmpRegister(ty, src_mcv), abi_size),
		4775	Immediate.u(mode),
		4776	),
		4777	}
		4778	}
		4779
		4780	fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
		4781	const un_op = self.air.instructions.items(.data)[inst].un_op;
		4782	const ty = self.air.typeOf(un_op);
		4783	const abi_size = @intCast(u32, ty.abiSize(self.target.*));
		4784
		4785	const src_mcv = try self.resolveInst(un_op);
		4786	const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, un_op, 0, src_mcv))
		4787	src_mcv
		4788	else
		4789	try self.copyToRegisterWithInstTracking(inst, ty, src_mcv);
		4790	const dst_reg = registerAlias(dst_mcv.getReg().?, abi_size);
		4791	const dst_lock = self.register_manager.lockReg(dst_reg);
		4792	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
		4793
		4794	const result: MCValue = result: {
		4795	const mir_tag = if (@as(?Mir.Inst.FixedTag, switch (ty.zigTypeTag()) {
		4796	.Float => switch (ty.floatBits(self.target.*)) {
		4797	16 => if (self.hasFeature(.f16c)) {
		4798	const mat_src_reg = if (src_mcv.isRegister())
		4799	src_mcv.getReg().?
		4800	else
		4801	try self.copyToTmpRegister(ty, src_mcv);
		4802	try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg, mat_src_reg.to128());
		4803	try self.asmRegisterRegisterRegister(.{ .v_ss, .sqrt }, dst_reg, dst_reg, dst_reg);
		4804	try self.asmRegisterRegisterImmediate(
		4805	.{ .v_, .cvtps2ph },
		4806	dst_reg,
		4807	dst_reg,
		4808	Immediate.u(0b1_00),
		4809	);
		4810	break :result dst_mcv;
		4811	} else null,
		4812	32 => if (self.hasFeature(.avx)) .{ .v_ss, .sqrt } else .{ ._ss, .sqrt },
		4813	64 => if (self.hasFeature(.avx)) .{ .v_sd, .sqrt } else .{ ._sd, .sqrt },
		4814	80, 128 => null,
		4815	else => unreachable,
		4816	},
		4817	.Vector => switch (ty.childType().zigTypeTag()) {
		4818	.Float => switch (ty.childType().floatBits(self.target.*)) {
		4819	16 => if (self.hasFeature(.f16c)) switch (ty.vectorLen()) {
		4820	1 => {
		4821	try self.asmRegisterRegister(
		4822	.{ .v_, .cvtph2ps },
		4823	dst_reg,
		4824	(if (src_mcv.isRegister())
		4825	src_mcv.getReg().?
		4826	else
		4827	try self.copyToTmpRegister(ty, src_mcv)).to128(),
		4828	);
		4829	try self.asmRegisterRegisterRegister(
		4830	.{ .v_ss, .sqrt },
		4831	dst_reg,
		4832	dst_reg,
		4833	dst_reg,
		4834	);
		4835	try self.asmRegisterRegisterImmediate(
		4836	.{ .v_, .cvtps2ph },
		4837	dst_reg,
		4838	dst_reg,
		4839	Immediate.u(0b1_00),
		4840	);
		4841	break :result dst_mcv;
		4842	},
		4843	2...8 => {
		4844	const wide_reg = registerAlias(dst_reg, abi_size * 2);
		4845	if (src_mcv.isMemory()) try self.asmRegisterMemory(
		4846	.{ .v_, .cvtph2ps },
		4847	wide_reg,
		4848	src_mcv.mem(Memory.PtrSize.fromSize(
		4849	@intCast(u32, @divExact(wide_reg.bitSize(), 16)),
		4850	)),
		4851	) else try self.asmRegisterRegister(
		4852	.{ .v_, .cvtph2ps },
		4853	wide_reg,
		4854	(if (src_mcv.isRegister())
		4855	src_mcv.getReg().?
		4856	else
		4857	try self.copyToTmpRegister(ty, src_mcv)).to128(),
		4858	);
		4859	try self.asmRegisterRegister(.{ .v_ps, .sqrt }, wide_reg, wide_reg);
		4860	try self.asmRegisterRegisterImmediate(
		4861	.{ .v_, .cvtps2ph },
		4862	dst_reg,
		4863	wide_reg,
		4864	Immediate.u(0b1_00),
		4865	);
		4866	break :result dst_mcv;
		4867	},
		4868	else => null,
		4869	} else null,
		4870	32 => switch (ty.vectorLen()) {
		4871	1 => if (self.hasFeature(.avx)) .{ .v_ss, .sqrt } else .{ ._ss, .sqrt },
		4872	2...4 => if (self.hasFeature(.avx)) .{ .v_ps, .sqrt } else .{ ._ps, .sqrt },
		4873	5...8 => if (self.hasFeature(.avx)) .{ .v_ps, .sqrt } else null,
		4874	else => null,
		4875	},
		4876	64 => switch (ty.vectorLen()) {
		4877	1 => if (self.hasFeature(.avx)) .{ .v_sd, .sqrt } else .{ ._sd, .sqrt },
		4878	2 => if (self.hasFeature(.avx)) .{ .v_pd, .sqrt } else .{ ._pd, .sqrt },
		4879	3...4 => if (self.hasFeature(.avx)) .{ .v_pd, .sqrt } else null,
		4880	else => null,
		4881	},
		4882	80, 128 => null,
		4883	else => unreachable,
		4884	},
		4885	else => unreachable,
		4886	},
		4887	else => unreachable,
		4888	})) \|tag\| tag else return self.fail("TODO implement airSqrt for {}", .{
		4889	ty.fmt(self.bin_file.options.module.?),
		4890	});
		4891	switch (mir_tag[0]) {
		4892	.v_ss, .v_sd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
		4893	mir_tag,
		4894	dst_reg,
		4895	dst_reg,
		4896	src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
		4897	) else try self.asmRegisterRegisterRegister(
		4898	mir_tag,
		4899	dst_reg,
		4900	dst_reg,
		4901	registerAlias(if (src_mcv.isRegister())
		4902	src_mcv.getReg().?
		4903	else
		4904	try self.copyToTmpRegister(ty, src_mcv), abi_size),
		4905	),
		4906	else => if (src_mcv.isMemory()) try self.asmRegisterMemory(
		4907	mir_tag,
		4908	dst_reg,
		4909	src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
		4910	) else try self.asmRegisterRegister(
		4911	mir_tag,
		4912	dst_reg,
		4913	registerAlias(if (src_mcv.isRegister())
		4914	src_mcv.getReg().?
		4915	else
		4916	try self.copyToTmpRegister(ty, src_mcv), abi_size),
		4917	),
		4918	}
		4919	break :result dst_mcv;
		4920	};
		4921	return self.finishAir(inst, result, .{ un_op, .none, .none });
		4922	}
		4923
		4924	fn airUnaryMath(self: *Self, inst: Air.Inst.Index) !void {
		4925	const un_op = self.air.instructions.items(.data)[inst].un_op;
		4926	_ = un_op;
		4927	return self.fail("TODO implement airUnaryMath for {}", .{
		4928	self.air.instructions.items(.tag)[inst],
		4929	});
		4930	//return self.finishAir(inst, result, .{ un_op, .none, .none });
		4931	}
		4932
		4933	fn reuseOperand(
		4934	self: *Self,
		4935	inst: Air.Inst.Index,
		4936	operand: Air.Inst.Ref,
		4937	op_index: Liveness.OperandInt,
		4938	mcv: MCValue,
		4939	) bool {
		4940	return self.reuseOperandAdvanced(inst, operand, op_index, mcv, inst);
		4941	}
4304		4942
4305	fn reuseOperandAdvanced(	4943	fn reuseOperandAdvanced(
4306	self: *Self,	4944	self: *Self,
...	@@ -4366,14 +5004,14 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn	...	@@ -4366,14 +5004,14 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
4366	if (load_abi_size <= 8) {	5004	if (load_abi_size <= 8) {
4367	const load_reg = registerAlias(dst_reg, load_abi_size);	5005	const load_reg = registerAlias(dst_reg, load_abi_size);
4368	try self.asmRegisterMemory(	5006	try self.asmRegisterMemory(
4369	.mov,	5007	.{ ._, .mov },
4370	load_reg,	5008	load_reg,
4371	Memory.sib(Memory.PtrSize.fromSize(load_abi_size), .{	5009	Memory.sib(Memory.PtrSize.fromSize(load_abi_size), .{
4372	.base = .{ .reg = ptr_reg },	5010	.base = .{ .reg = ptr_reg },
4373	.disp = val_byte_off,	5011	.disp = val_byte_off,
4374	}),	5012	}),
4375	);	5013	);
4376	try self.asmRegisterImmediate(.shr, load_reg, Immediate.u(val_bit_off));	5014	try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(val_bit_off));
4377	} else {	5015	} else {
4378	const tmp_reg = registerAlias(try self.register_manager.allocReg(null, gp), val_abi_size);	5016	const tmp_reg = registerAlias(try self.register_manager.allocReg(null, gp), val_abi_size);
4379	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);	5017	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
...	@@ -4381,7 +5019,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn	...	@@ -4381,7 +5019,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
4381		5019
4382	const dst_alias = registerAlias(dst_reg, val_abi_size);	5020	const dst_alias = registerAlias(dst_reg, val_abi_size);
4383	try self.asmRegisterMemory(	5021	try self.asmRegisterMemory(
4384	.mov,	5022	.{ ._, .mov },
4385	dst_alias,	5023	dst_alias,
4386	Memory.sib(Memory.PtrSize.fromSize(val_abi_size), .{	5024	Memory.sib(Memory.PtrSize.fromSize(val_abi_size), .{
4387	.base = .{ .reg = ptr_reg },	5025	.base = .{ .reg = ptr_reg },
...	@@ -4389,14 +5027,19 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn	...	@@ -4389,14 +5027,19 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
4389	}),	5027	}),
4390	);	5028	);
4391	try self.asmRegisterMemory(	5029	try self.asmRegisterMemory(
4392	.mov,	5030	.{ ._, .mov },
4393	tmp_reg,	5031	tmp_reg,
4394	Memory.sib(Memory.PtrSize.fromSize(val_abi_size), .{	5032	Memory.sib(Memory.PtrSize.fromSize(val_abi_size), .{
4395	.base = .{ .reg = ptr_reg },	5033	.base = .{ .reg = ptr_reg },
4396	.disp = val_byte_off + 1,	5034	.disp = val_byte_off + 1,
4397	}),	5035	}),
4398	);	5036	);
4399	try self.asmRegisterRegisterImmediate(.shrd, dst_alias, tmp_reg, Immediate.u(val_bit_off));	5037	try self.asmRegisterRegisterImmediate(
		5038	.{ ._rd, .sh },
		5039	dst_alias,
		5040	tmp_reg,
		5041	Immediate.u(val_bit_off),
		5042	);
4400	}	5043	}
4401		5044
4402	if (val_extra_bits > 0) try self.truncateRegister(val_ty, dst_reg);	5045	if (val_extra_bits > 0) try self.truncateRegister(val_ty, dst_reg);
...	@@ -4502,13 +5145,13 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In	...	@@ -4502,13 +5145,13 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
4502	const part_mask_not = part_mask ^	5145	const part_mask_not = part_mask ^
4503	(@as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - limb_abi_bits));	5146	(@as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - limb_abi_bits));
4504	if (limb_abi_size <= 4) {	5147	if (limb_abi_size <= 4) {
4505	try self.asmMemoryImmediate(.@"and", limb_mem, Immediate.u(part_mask_not));	5148	try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, Immediate.u(part_mask_not));
4506	} else if (math.cast(i32, @bitCast(i64, part_mask_not))) \|small\| {	5149	} else if (math.cast(i32, @bitCast(i64, part_mask_not))) \|small\| {
4507	try self.asmMemoryImmediate(.@"and", limb_mem, Immediate.s(small));	5150	try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, Immediate.s(small));
4508	} else {	5151	} else {
4509	const part_mask_reg = try self.register_manager.allocReg(null, gp);	5152	const part_mask_reg = try self.register_manager.allocReg(null, gp);
4510	try self.asmRegisterImmediate(.mov, part_mask_reg, Immediate.u(part_mask_not));	5153	try self.asmRegisterImmediate(.{ ._, .mov }, part_mask_reg, Immediate.u(part_mask_not));
4511	try self.asmMemoryRegister(.@"and", limb_mem, part_mask_reg);	5154	try self.asmMemoryRegister(.{ ._, .@"and" }, limb_mem, part_mask_reg);
4512	}	5155	}
4513		5156
4514	if (src_bit_size <= 64) {	5157	if (src_bit_size <= 64) {
...	@@ -4519,14 +5162,26 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In	...	@@ -4519,14 +5162,26 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
4519		5162
4520	try self.genSetReg(tmp_reg, src_ty, src_mcv);	5163	try self.genSetReg(tmp_reg, src_ty, src_mcv);
4521	switch (limb_i) {	5164	switch (limb_i) {
4522	0 => try self.genShiftBinOpMir(.shl, src_ty, tmp_mcv, .{ .immediate = src_bit_off }),	5165	0 => try self.genShiftBinOpMir(
4523	1 => try self.genShiftBinOpMir(.shr, src_ty, tmp_mcv, .{	5166	.{ ._l, .sh },
4524	.immediate = limb_abi_bits - src_bit_off,	5167	src_ty,
4525	}),	5168	tmp_mcv,
		5169	.{ .immediate = src_bit_off },
		5170	),
		5171	1 => try self.genShiftBinOpMir(
		5172	.{ ._r, .sh },
		5173	src_ty,
		5174	tmp_mcv,
		5175	.{ .immediate = limb_abi_bits - src_bit_off },
		5176	),
4526	else => unreachable,	5177	else => unreachable,
4527	}	5178	}
4528	try self.genBinOpMir(.@"and", src_ty, tmp_mcv, .{ .immediate = part_mask });	5179	try self.genBinOpMir(.{ ._, .@"and" }, src_ty, tmp_mcv, .{ .immediate = part_mask });
4529	try self.asmMemoryRegister(.@"or", limb_mem, registerAlias(tmp_reg, limb_abi_size));	5180	try self.asmMemoryRegister(
		5181	.{ ._, .@"or" },
		5182	limb_mem,
		5183	registerAlias(tmp_reg, limb_abi_size),
		5184	);
4530	} else return self.fail("TODO: implement packed store of {}", .{	5185	} else return self.fail("TODO: implement packed store of {}", .{
4531	src_ty.fmt(self.bin_file.options.module.?),	5186	src_ty.fmt(self.bin_file.options.module.?),
4532	});	5187	});
...	@@ -4626,7 +5281,7 @@ fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32	...	@@ -4626,7 +5281,7 @@ fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32
4626	.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },	5281	.load_tlv => \|sym_index\| .{ .lea_tlv = sym_index },
4627	else => mcv,	5282	else => mcv,
4628	});	5283	});
4629	try self.genBinOpMir(.add, Type.usize, dst_mcv, .{ .register = offset_reg });	5284	try self.genBinOpMir(.{ ._, .add }, Type.usize, dst_mcv, .{ .register = offset_reg });
4630	break :result dst_mcv;	5285	break :result dst_mcv;
4631	},	5286	},
4632	.indirect => \|reg_off\| break :result .{ .indirect = .{	5287	.indirect => \|reg_off\| break :result .{ .indirect = .{
...	@@ -4710,14 +5365,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4710,14 +5365,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
4710	if (load_abi_size <= 8) {	5365	if (load_abi_size <= 8) {
4711	const load_reg = registerAlias(dst_reg, load_abi_size);	5366	const load_reg = registerAlias(dst_reg, load_abi_size);
4712	try self.asmRegisterMemory(	5367	try self.asmRegisterMemory(
4713	.mov,	5368	.{ ._, .mov },
4714	load_reg,	5369	load_reg,
4715	Memory.sib(Memory.PtrSize.fromSize(load_abi_size), .{	5370	Memory.sib(Memory.PtrSize.fromSize(load_abi_size), .{
4716	.base = .{ .frame = frame_addr.index },	5371	.base = .{ .frame = frame_addr.index },
4717	.disp = frame_addr.off + field_byte_off,	5372	.disp = frame_addr.off + field_byte_off,
4718	}),	5373	}),
4719	);	5374	);
4720	try self.asmRegisterImmediate(.shr, load_reg, Immediate.u(field_bit_off));	5375	try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(field_bit_off));
4721	} else {	5376	} else {
4722	const tmp_reg = registerAlias(	5377	const tmp_reg = registerAlias(
4723	try self.register_manager.allocReg(null, gp),	5378	try self.register_manager.allocReg(null, gp),
...	@@ -4728,7 +5383,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4728,7 +5383,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
4728		5383
4729	const dst_alias = registerAlias(dst_reg, field_abi_size);	5384	const dst_alias = registerAlias(dst_reg, field_abi_size);
4730	try self.asmRegisterMemory(	5385	try self.asmRegisterMemory(
4731	.mov,	5386	.{ ._, .mov },
4732	dst_alias,	5387	dst_alias,
4733	Memory.sib(Memory.PtrSize.fromSize(field_abi_size), .{	5388	Memory.sib(Memory.PtrSize.fromSize(field_abi_size), .{
4734	.base = .{ .frame = frame_addr.index },	5389	.base = .{ .frame = frame_addr.index },
...	@@ -4736,7 +5391,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4736,7 +5391,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
4736	}),	5391	}),
4737	);	5392	);
4738	try self.asmRegisterMemory(	5393	try self.asmRegisterMemory(
4739	.mov,	5394	.{ ._, .mov },
4740	tmp_reg,	5395	tmp_reg,
4741	Memory.sib(Memory.PtrSize.fromSize(field_abi_size), .{	5396	Memory.sib(Memory.PtrSize.fromSize(field_abi_size), .{
4742	.base = .{ .frame = frame_addr.index },	5397	.base = .{ .frame = frame_addr.index },
...	@@ -4744,7 +5399,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4744,7 +5399,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
4744	}),	5399	}),
4745	);	5400	);
4746	try self.asmRegisterRegisterImmediate(	5401	try self.asmRegisterRegisterImmediate(
4747	.shrd,	5402	.{ ._rd, .sh },
4748	dst_alias,	5403	dst_alias,
4749	tmp_reg,	5404	tmp_reg,
4750	Immediate.u(field_bit_off),	5405	Immediate.u(field_bit_off),
...	@@ -4752,7 +5407,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4752,7 +5407,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
4752	}	5407	}
4753		5408
4754	if (field_extra_bits > 0) try self.truncateRegister(field_ty, dst_reg);	5409	if (field_extra_bits > 0) try self.truncateRegister(field_ty, dst_reg);
4755	break :result .{ .register = dst_reg };	5410
		5411	const dst_mcv = MCValue{ .register = dst_reg };
		5412	const dst_rc = regClassForType(field_ty);
		5413	if (dst_rc.eql(gp)) break :result dst_mcv;
		5414
		5415	const result_reg = try self.register_manager.allocReg(inst, dst_rc);
		5416	try self.genSetReg(result_reg, field_ty, dst_mcv);
		5417	break :result .{ .register = result_reg };
4756	},	5418	},
4757	.register => \|reg\| {	5419	.register => \|reg\| {
4758	const reg_lock = self.register_manager.lockRegAssumeUnused(reg);	5420	const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
...	@@ -4773,21 +5435,26 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4773,21 +5435,26 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
4773	defer if (dst_mcv_lock) \|lock\| self.register_manager.unlockReg(lock);	5435	defer if (dst_mcv_lock) \|lock\| self.register_manager.unlockReg(lock);
4774		5436
4775	// Shift by struct_field_offset.	5437	// Shift by struct_field_offset.
4776	try self.genShiftBinOpMir(.shr, Type.usize, dst_mcv, .{ .immediate = field_off });	5438	try self.genShiftBinOpMir(
		5439	.{ ._r, .sh },
		5440	Type.usize,
		5441	dst_mcv,
		5442	.{ .immediate = field_off },
		5443	);
4777		5444
4778	// Mask to field_bit_size bits	5445	// Mask to field_bit_size bits
4779	const field_bit_size = field_ty.bitSize(self.target.*);	5446	const field_bit_size = field_ty.bitSize(self.target.*);
4780	const mask = ~@as(u64, 0) >> @intCast(u6, 64 - field_bit_size);	5447	const mask = ~@as(u64, 0) >> @intCast(u6, 64 - field_bit_size);
4781		5448
4782	const tmp_reg = try self.copyToTmpRegister(Type.usize, .{ .immediate = mask });	5449	const tmp_reg = try self.copyToTmpRegister(Type.usize, .{ .immediate = mask });
4783	try self.genBinOpMir(.@"and", Type.usize, dst_mcv, .{ .register = tmp_reg });	5450	try self.genBinOpMir(.{ ._, .@"and" }, Type.usize, dst_mcv, .{ .register = tmp_reg });
4784		5451
4785	const signedness =	5452	const signedness =
4786	if (field_ty.isAbiInt()) field_ty.intInfo(self.target.*).signedness else .unsigned;	5453	if (field_ty.isAbiInt()) field_ty.intInfo(self.target.*).signedness else .unsigned;
4787	const field_byte_size = @intCast(u32, field_ty.abiSize(self.target.*));	5454	const field_byte_size = @intCast(u32, field_ty.abiSize(self.target.*));
4788	if (signedness == .signed and field_byte_size < 8) {	5455	if (signedness == .signed and field_byte_size < 8) {
4789	try self.asmRegisterRegister(	5456	try self.asmRegisterRegister(
4790	.movsx,	5457	if (field_byte_size >= 4) .{ ._d, .movsx } else .{ ._, .movsx },
4791	dst_mcv.register,	5458	dst_mcv.register,
4792	registerAlias(dst_mcv.register, field_byte_size),	5459	registerAlias(dst_mcv.register, field_byte_size),
4793	);	5460	);
...	@@ -4899,17 +5566,17 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:	...	@@ -4899,17 +5566,17 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
4899		5566
4900	if (limb_pl.base.tag == .int_unsigned and self.regExtraBits(limb_ty) > 0) {	5567	if (limb_pl.base.tag == .int_unsigned and self.regExtraBits(limb_ty) > 0) {
4901	const mask = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - limb_pl.data);	5568	const mask = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - limb_pl.data);
4902	try self.genBinOpMir(.xor, limb_ty, limb_mcv, .{ .immediate = mask });	5569	try self.genBinOpMir(.{ ._, .xor }, limb_ty, limb_mcv, .{ .immediate = mask });
4903	} else try self.genUnOpMir(.not, limb_ty, limb_mcv);	5570	} else try self.genUnOpMir(.{ ._, .not }, limb_ty, limb_mcv);
4904	}	5571	}
4905	},	5572	},
4906	.neg => try self.genUnOpMir(.neg, src_ty, dst_mcv),	5573	.neg => try self.genUnOpMir(.{ ._, .neg }, src_ty, dst_mcv),
4907	else => unreachable,	5574	else => unreachable,
4908	}	5575	}
4909	return dst_mcv;	5576	return dst_mcv;
4910	}	5577	}
4911		5578
4912	fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue) !void {	5579	fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MCValue) !void {
4913	const abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));	5580	const abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
4914	if (abi_size > 8) return self.fail("TODO implement {} for {}", .{	5581	if (abi_size > 8) return self.fail("TODO implement {} for {}", .{
4915	mir_tag,	5582	mir_tag,
...	@@ -4952,7 +5619,7 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue	...	@@ -4952,7 +5619,7 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue
4952	/// Clobbers .rcx for non-immediate shift value.	5619	/// Clobbers .rcx for non-immediate shift value.
4953	fn genShiftBinOpMir(	5620	fn genShiftBinOpMir(
4954	self: *Self,	5621	self: *Self,
4955	tag: Mir.Inst.Tag,	5622	tag: Mir.Inst.FixedTag,
4956	ty: Type,	5623	ty: Type,
4957	lhs_mcv: MCValue,	5624	lhs_mcv: MCValue,
4958	shift_mcv: MCValue,	5625	shift_mcv: MCValue,
...	@@ -5037,16 +5704,16 @@ fn genShiftBinOpMir(	...	@@ -5037,16 +5704,16 @@ fn genShiftBinOpMir(
5037	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);	5704	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
5038	defer self.register_manager.unlockReg(tmp_lock);	5705	defer self.register_manager.unlockReg(tmp_lock);
5039		5706
5040	const info: struct { offsets: [2]i32, double_tag: Mir.Inst.Tag } = switch (tag) {	5707	const info: struct { offsets: [2]i32, double_tag: Mir.Inst.FixedTag } = switch (tag[0]) {
5041	.shl, .sal => .{ .offsets = .{ 0, 8 }, .double_tag = .shld },	5708	._l => .{ .offsets = .{ 0, 8 }, .double_tag = .{ ._ld, .sh } },
5042	.shr, .sar => .{ .offsets = .{ 8, 0 }, .double_tag = .shrd },	5709	._r => .{ .offsets = .{ 8, 0 }, .double_tag = .{ ._rd, .sh } },
5043	else => unreachable,	5710	else => unreachable,
5044	};	5711	};
5045	switch (lhs_mcv) {	5712	switch (lhs_mcv) {
5046	.load_frame => \|dst_frame_addr\| switch (rhs_mcv) {	5713	.load_frame => \|dst_frame_addr\| switch (rhs_mcv) {
5047	.immediate => \|rhs_imm\| if (rhs_imm == 0) {} else if (rhs_imm < 64) {	5714	.immediate => \|rhs_imm\| if (rhs_imm == 0) {} else if (rhs_imm < 64) {
5048	try self.asmRegisterMemory(	5715	try self.asmRegisterMemory(
5049	.mov,	5716	.{ ._, .mov },
5050	tmp_reg,	5717	tmp_reg,
5051	Memory.sib(.qword, .{	5718	Memory.sib(.qword, .{
5052	.base = .{ .frame = dst_frame_addr.index },	5719	.base = .{ .frame = dst_frame_addr.index },
...	@@ -5073,7 +5740,7 @@ fn genShiftBinOpMir(	...	@@ -5073,7 +5740,7 @@ fn genShiftBinOpMir(
5073	} else {	5740	} else {
5074	assert(rhs_imm < 128);	5741	assert(rhs_imm < 128);
5075	try self.asmRegisterMemory(	5742	try self.asmRegisterMemory(
5076	.mov,	5743	.{ ._, .mov },
5077	tmp_reg,	5744	tmp_reg,
5078	Memory.sib(.qword, .{	5745	Memory.sib(.qword, .{
5079	.base = .{ .frame = dst_frame_addr.index },	5746	.base = .{ .frame = dst_frame_addr.index },
...	@@ -5084,34 +5751,30 @@ fn genShiftBinOpMir(	...	@@ -5084,34 +5751,30 @@ fn genShiftBinOpMir(
5084	try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(rhs_imm - 64));	5751	try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(rhs_imm - 64));
5085	}	5752	}
5086	try self.asmMemoryRegister(	5753	try self.asmMemoryRegister(
5087	.mov,	5754	.{ ._, .mov },
5088	Memory.sib(.qword, .{	5755	Memory.sib(.qword, .{
5089	.base = .{ .frame = dst_frame_addr.index },	5756	.base = .{ .frame = dst_frame_addr.index },
5090	.disp = dst_frame_addr.off + info.offsets[1],	5757	.disp = dst_frame_addr.off + info.offsets[1],
5091	}),	5758	}),
5092	tmp_reg,	5759	tmp_reg,
5093	);	5760	);
5094	switch (tag) {	5761	if (tag[0] == ._r and tag[1] == .sa) try self.asmMemoryImmediate(
5095	.shl, .sal, .shr => {	5762	tag,
5096	try self.asmRegisterRegister(.xor, tmp_reg.to32(), tmp_reg.to32());	5763	Memory.sib(.qword, .{
5097	try self.asmMemoryRegister(	5764	.base = .{ .frame = dst_frame_addr.index },
5098	.mov,	5765	.disp = dst_frame_addr.off + info.offsets[0],
5099	Memory.sib(.qword, .{	5766	}),
5100	.base = .{ .frame = dst_frame_addr.index },	5767	Immediate.u(63),
5101	.disp = dst_frame_addr.off + info.offsets[0],	5768	) else {
5102	}),	5769	try self.asmRegisterRegister(.{ ._, .xor }, tmp_reg.to32(), tmp_reg.to32());
5103	tmp_reg,	5770	try self.asmMemoryRegister(
5104	);	5771	.{ ._, .mov },
5105	},
5106	.sar => try self.asmMemoryImmediate(
5107	tag,
5108	Memory.sib(.qword, .{	5772	Memory.sib(.qword, .{
5109	.base = .{ .frame = dst_frame_addr.index },	5773	.base = .{ .frame = dst_frame_addr.index },
5110	.disp = dst_frame_addr.off + info.offsets[0],	5774	.disp = dst_frame_addr.off + info.offsets[0],
5111	}),	5775	}),
5112	Immediate.u(63),	5776	tmp_reg,
5113	),	5777	);
5114	else => unreachable,
5115	}	5778	}
5116	},	5779	},
5117	else => {	5780	else => {
...	@@ -5125,7 +5788,7 @@ fn genShiftBinOpMir(	...	@@ -5125,7 +5788,7 @@ fn genShiftBinOpMir(
5125		5788
5126	try self.genSetReg(.cl, Type.u8, rhs_mcv);	5789	try self.genSetReg(.cl, Type.u8, rhs_mcv);
5127	try self.asmRegisterMemory(	5790	try self.asmRegisterMemory(
5128	.mov,	5791	.{ ._, .mov },
5129	first_reg,	5792	first_reg,
5130	Memory.sib(.qword, .{	5793	Memory.sib(.qword, .{
5131	.base = .{ .frame = dst_frame_addr.index },	5794	.base = .{ .frame = dst_frame_addr.index },
...	@@ -5133,32 +5796,28 @@ fn genShiftBinOpMir(	...	@@ -5133,32 +5796,28 @@ fn genShiftBinOpMir(
5133	}),	5796	}),
5134	);	5797	);
5135	try self.asmRegisterMemory(	5798	try self.asmRegisterMemory(
5136	.mov,	5799	.{ ._, .mov },
5137	second_reg,	5800	second_reg,
5138	Memory.sib(.qword, .{	5801	Memory.sib(.qword, .{
5139	.base = .{ .frame = dst_frame_addr.index },	5802	.base = .{ .frame = dst_frame_addr.index },
5140	.disp = dst_frame_addr.off + info.offsets[1],	5803	.disp = dst_frame_addr.off + info.offsets[1],
5141	}),	5804	}),
5142	);	5805	);
5143	switch (tag) {	5806	if (tag[0] == ._r and tag[1] == .sa) {
5144	.shl, .sal, .shr => try self.asmRegisterRegister(	5807	try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, first_reg);
5145	.xor,	5808	try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(63));
5146	tmp_reg.to32(),	5809	} else try self.asmRegisterRegister(
5147	tmp_reg.to32(),	5810	.{ ._, .xor },
5148	),	5811	tmp_reg.to32(),
5149	.sar => {	5812	tmp_reg.to32(),
5150	try self.asmRegisterRegister(.mov, tmp_reg, first_reg);	5813	);
5151	try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(63));
5152	},
5153	else => unreachable,
5154	}
5155	try self.asmRegisterRegisterRegister(info.double_tag, second_reg, first_reg, .cl);	5814	try self.asmRegisterRegisterRegister(info.double_tag, second_reg, first_reg, .cl);
5156	try self.asmRegisterRegister(tag, first_reg, .cl);	5815	try self.asmRegisterRegister(tag, first_reg, .cl);
5157	try self.asmRegisterImmediate(.cmp, .cl, Immediate.u(64));	5816	try self.asmRegisterImmediate(.{ ._, .cmp }, .cl, Immediate.u(64));
5158	try self.asmCmovccRegisterRegister(second_reg, first_reg, .ae);	5817	try self.asmCmovccRegisterRegister(second_reg, first_reg, .ae);
5159	try self.asmCmovccRegisterRegister(first_reg, tmp_reg, .ae);	5818	try self.asmCmovccRegisterRegister(first_reg, tmp_reg, .ae);
5160	try self.asmMemoryRegister(	5819	try self.asmMemoryRegister(
5161	.mov,	5820	.{ ._, .mov },
5162	Memory.sib(.qword, .{	5821	Memory.sib(.qword, .{
5163	.base = .{ .frame = dst_frame_addr.index },	5822	.base = .{ .frame = dst_frame_addr.index },
5164	.disp = dst_frame_addr.off + info.offsets[1],	5823	.disp = dst_frame_addr.off + info.offsets[1],
...	@@ -5166,7 +5825,7 @@ fn genShiftBinOpMir(	...	@@ -5166,7 +5825,7 @@ fn genShiftBinOpMir(
5166	second_reg,	5825	second_reg,
5167	);	5826	);
5168	try self.asmMemoryRegister(	5827	try self.asmMemoryRegister(
5169	.mov,	5828	.{ ._, .mov },
5170	Memory.sib(.qword, .{	5829	Memory.sib(.qword, .{
5171	.base = .{ .frame = dst_frame_addr.index },	5830	.base = .{ .frame = dst_frame_addr.index },
5172	.disp = dst_frame_addr.off + info.offsets[0],	5831	.disp = dst_frame_addr.off + info.offsets[0],
...	@@ -5191,7 +5850,7 @@ fn genShiftBinOpMir(	...	@@ -5191,7 +5850,7 @@ fn genShiftBinOpMir(
5191	/// Asserts .rcx is free.	5850	/// Asserts .rcx is free.
5192	fn genShiftBinOp(	5851	fn genShiftBinOp(
5193	self: *Self,	5852	self: *Self,
5194	tag: Air.Inst.Tag,	5853	air_tag: Air.Inst.Tag,
5195	maybe_inst: ?Air.Inst.Index,	5854	maybe_inst: ?Air.Inst.Index,
5196	lhs_mcv: MCValue,	5855	lhs_mcv: MCValue,
5197	rhs_mcv: MCValue,	5856	rhs_mcv: MCValue,
...	@@ -5236,14 +5895,14 @@ fn genShiftBinOp(	...	@@ -5236,14 +5895,14 @@ fn genShiftBinOp(
5236	};	5895	};
5237		5896
5238	const signedness = lhs_ty.intInfo(self.target.*).signedness;	5897	const signedness = lhs_ty.intInfo(self.target.*).signedness;
5239	try self.genShiftBinOpMir(switch (tag) {	5898	try self.genShiftBinOpMir(switch (air_tag) {
5240	.shl, .shl_exact => switch (signedness) {	5899	.shl, .shl_exact => switch (signedness) {
5241	.signed => .sal,	5900	.signed => .{ ._l, .sa },
5242	.unsigned => .shl,	5901	.unsigned => .{ ._l, .sh },
5243	},	5902	},
5244	.shr, .shr_exact => switch (signedness) {	5903	.shr, .shr_exact => switch (signedness) {
5245	.signed => .sar,	5904	.signed => .{ ._r, .sa },
5246	.unsigned => .shr,	5905	.unsigned => .{ ._r, .sh },
5247	},	5906	},
5248	else => unreachable,	5907	else => unreachable,
5249	}, lhs_ty, dst_mcv, rhs_mcv);	5908	}, lhs_ty, dst_mcv, rhs_mcv);
...	@@ -5303,20 +5962,18 @@ fn genMulDivBinOp(	...	@@ -5303,20 +5962,18 @@ fn genMulDivBinOp(
5303	try self.register_manager.getReg(.rax, track_inst_rax);	5962	try self.register_manager.getReg(.rax, track_inst_rax);
5304	try self.register_manager.getReg(.rdx, track_inst_rdx);	5963	try self.register_manager.getReg(.rdx, track_inst_rdx);
5305		5964
5306	const mir_tag: Mir.Inst.Tag = switch (signedness) {	5965	try self.genIntMulDivOpMir(switch (signedness) {
5307	.signed => switch (tag) {	5966	.signed => switch (tag) {
5308	.mul, .mulwrap => .imul,	5967	.mul, .mulwrap => .{ .i_, .mul },
5309	.div_trunc, .div_exact, .rem => .idiv,	5968	.div_trunc, .div_exact, .rem => .{ .i_, .div },
5310	else => unreachable,	5969	else => unreachable,
5311	},	5970	},
5312	.unsigned => switch (tag) {	5971	.unsigned => switch (tag) {
5313	.mul, .mulwrap => .mul,	5972	.mul, .mulwrap => .{ ._, .mul },
5314	.div_trunc, .div_exact, .rem => .div,	5973	.div_trunc, .div_exact, .rem => .{ ._, .div },
5315	else => unreachable,	5974	else => unreachable,
5316	},	5975	},
5317	};	5976	}, ty, lhs, rhs);
5318
5319	try self.genIntMulDivOpMir(mir_tag, ty, lhs, rhs);
5320		5977
5321	if (dst_abi_size <= 8) return .{ .register = registerAlias(switch (tag) {	5978	if (dst_abi_size <= 8) return .{ .register = registerAlias(switch (tag) {
5322	.mul, .mulwrap, .div_trunc, .div_exact => .rax,	5979	.mul, .mulwrap, .div_trunc, .div_exact => .rax,
...	@@ -5326,7 +5983,7 @@ fn genMulDivBinOp(	...	@@ -5326,7 +5983,7 @@ fn genMulDivBinOp(
5326		5983
5327	const dst_mcv = try self.allocRegOrMemAdvanced(dst_ty, maybe_inst, false);	5984	const dst_mcv = try self.allocRegOrMemAdvanced(dst_ty, maybe_inst, false);
5328	try self.asmMemoryRegister(	5985	try self.asmMemoryRegister(
5329	.mov,	5986	.{ ._, .mov },
5330	Memory.sib(.qword, .{	5987	Memory.sib(.qword, .{
5331	.base = .{ .frame = dst_mcv.load_frame.index },	5988	.base = .{ .frame = dst_mcv.load_frame.index },
5332	.disp = dst_mcv.load_frame.off,	5989	.disp = dst_mcv.load_frame.off,
...	@@ -5334,7 +5991,7 @@ fn genMulDivBinOp(	...	@@ -5334,7 +5991,7 @@ fn genMulDivBinOp(
5334	.rax,	5991	.rax,
5335	);	5992	);
5336	try self.asmMemoryRegister(	5993	try self.asmMemoryRegister(
5337	.mov,	5994	.{ ._, .mov },
5338	Memory.sib(.qword, .{	5995	Memory.sib(.qword, .{
5339	.base = .{ .frame = dst_mcv.load_frame.index },	5996	.base = .{ .frame = dst_mcv.load_frame.index },
5340	.disp = dst_mcv.load_frame.off + 8,	5997	.disp = dst_mcv.load_frame.off + 8,
...	@@ -5375,12 +6032,12 @@ fn genMulDivBinOp(	...	@@ -5375,12 +6032,12 @@ fn genMulDivBinOp(
5375	try self.copyToRegisterWithInstTracking(inst, ty, lhs)	6032	try self.copyToRegisterWithInstTracking(inst, ty, lhs)
5376	else	6033	else
5377	.{ .register = try self.copyToTmpRegister(ty, lhs) };	6034	.{ .register = try self.copyToTmpRegister(ty, lhs) };
5378	try self.genBinOpMir(.sub, ty, result, div_floor);	6035	try self.genBinOpMir(.{ ._, .sub }, ty, result, div_floor);
5379		6036
5380	return result;	6037	return result;
5381	},	6038	},
5382	.unsigned => {	6039	.unsigned => {
5383	try self.genIntMulDivOpMir(.div, ty, lhs, rhs);	6040	try self.genIntMulDivOpMir(.{ ._, .div }, ty, lhs, rhs);
5384	return .{ .register = registerAlias(.rdx, abi_size) };	6041	return .{ .register = registerAlias(.rdx, abi_size) };
5385	},	6042	},
5386	}	6043	}
...	@@ -5422,7 +6079,7 @@ fn genMulDivBinOp(	...	@@ -5422,7 +6079,7 @@ fn genMulDivBinOp(
5422	switch (signedness) {	6079	switch (signedness) {
5423	.signed => return try self.genInlineIntDivFloor(ty, lhs, actual_rhs),	6080	.signed => return try self.genInlineIntDivFloor(ty, lhs, actual_rhs),
5424	.unsigned => {	6081	.unsigned => {
5425	try self.genIntMulDivOpMir(.div, ty, lhs, actual_rhs);	6082	try self.genIntMulDivOpMir(.{ ._, .div }, ty, lhs, actual_rhs);
5426	return .{ .register = registerAlias(.rax, abi_size) };	6083	return .{ .register = registerAlias(.rax, abi_size) };
5427	},	6084	},
5428	}	6085	}
...	@@ -5432,25 +6089,22 @@ fn genMulDivBinOp(	...	@@ -5432,25 +6089,22 @@ fn genMulDivBinOp(
5432	}	6089	}
5433	}	6090	}
5434		6091
5435	/// Result is always a register.
5436	fn genBinOp(	6092	fn genBinOp(
5437	self: *Self,	6093	self: *Self,
5438	maybe_inst: ?Air.Inst.Index,	6094	maybe_inst: ?Air.Inst.Index,
5439	tag: Air.Inst.Tag,	6095	air_tag: Air.Inst.Tag,
5440	lhs_air: Air.Inst.Ref,	6096	lhs_air: Air.Inst.Ref,
5441	rhs_air: Air.Inst.Ref,	6097	rhs_air: Air.Inst.Ref,
5442	) !MCValue {	6098	) !MCValue {
5443	const lhs = try self.resolveInst(lhs_air);	6099	const lhs_mcv = try self.resolveInst(lhs_air);
5444	const rhs = try self.resolveInst(rhs_air);	6100	const rhs_mcv = try self.resolveInst(rhs_air);
5445	const lhs_ty = self.air.typeOf(lhs_air);	6101	const lhs_ty = self.air.typeOf(lhs_air);
5446	const rhs_ty = self.air.typeOf(rhs_air);	6102	const rhs_ty = self.air.typeOf(rhs_air);
5447	if (lhs_ty.zigTypeTag() == .Vector) {	6103	const abi_size = @intCast(u32, lhs_ty.abiSize(self.target.*));
5448	return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmt(self.bin_file.options.module.?)});
5449	}
5450		6104
5451	switch (lhs) {	6105	switch (lhs_mcv) {
5452	.immediate => \|imm\| switch (imm) {	6106	.immediate => \|imm\| switch (imm) {
5453	0 => switch (tag) {	6107	0 => switch (air_tag) {
5454	.sub, .subwrap => return self.genUnOp(maybe_inst, .neg, rhs_air),	6108	.sub, .subwrap => return self.genUnOp(maybe_inst, .neg, rhs_air),
5455	else => {},	6109	else => {},
5456	},	6110	},
...	@@ -5459,9 +6113,10 @@ fn genBinOp(	...	@@ -5459,9 +6113,10 @@ fn genBinOp(
5459	else => {},	6113	else => {},
5460	}	6114	}
5461		6115
5462	const is_commutative = switch (tag) {	6116	const is_commutative = switch (air_tag) {
5463	.add,	6117	.add,
5464	.addwrap,	6118	.addwrap,
		6119	.mul,
5465	.bool_or,	6120	.bool_or,
5466	.bit_or,	6121	.bit_or,
5467	.bool_and,	6122	.bool_and,
...	@@ -5473,48 +6128,42 @@ fn genBinOp(	...	@@ -5473,48 +6128,42 @@ fn genBinOp(
5473		6128
5474	else => false,	6129	else => false,
5475	};	6130	};
5476	const dst_mem_ok = switch (tag) {	6131	const vec_op = switch (lhs_ty.zigTypeTag()) {
5477	.add,	6132	else => false,
5478	.addwrap,	6133	.Float, .Vector => true,
5479	.sub,
5480	.subwrap,
5481	.mul,
5482	.div_float,
5483	.div_exact,
5484	.div_trunc,
5485	.div_floor,
5486	=> !lhs_ty.isRuntimeFloat(),
5487
5488	else => true,
5489	};	6134	};
5490		6135
5491	const lhs_lock: ?RegisterLock = switch (lhs) {	6136	const lhs_lock: ?RegisterLock = switch (lhs_mcv) {
5492	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),	6137	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
5493	else => null,	6138	else => null,
5494	};	6139	};
5495	defer if (lhs_lock) \|lock\| self.register_manager.unlockReg(lock);	6140	defer if (lhs_lock) \|lock\| self.register_manager.unlockReg(lock);
5496		6141
5497	const rhs_lock: ?RegisterLock = switch (rhs) {	6142	const rhs_lock: ?RegisterLock = switch (rhs_mcv) {
5498	.register => \|reg\| self.register_manager.lockReg(reg),	6143	.register => \|reg\| self.register_manager.lockReg(reg),
5499	else => null,	6144	else => null,
5500	};	6145	};
5501	defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);	6146	defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
5502		6147
5503	var flipped: bool = false;	6148	var flipped = false;
		6149	var copied_to_dst = true;
5504	const dst_mcv: MCValue = dst: {	6150	const dst_mcv: MCValue = dst: {
5505	if (maybe_inst) \|inst\| {	6151	if (maybe_inst) \|inst\| {
5506	if ((dst_mem_ok or lhs.isRegister()) and self.reuseOperand(inst, lhs_air, 0, lhs)) {	6152	if ((!vec_op or lhs_mcv.isRegister()) and self.reuseOperand(inst, lhs_air, 0, lhs_mcv)) {
5507	break :dst lhs;	6153	break :dst lhs_mcv;
5508	}	6154	}
5509	if (is_commutative and (dst_mem_ok or rhs.isRegister()) and	6155	if (is_commutative and (!vec_op or rhs_mcv.isRegister()) and
5510	self.reuseOperand(inst, rhs_air, 1, rhs))	6156	self.reuseOperand(inst, rhs_air, 1, rhs_mcv))
5511	{	6157	{
5512	flipped = true;	6158	flipped = true;
5513	break :dst rhs;	6159	break :dst rhs_mcv;
5514	}	6160	}
5515	}	6161	}
5516	const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, maybe_inst, true);	6162	const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, maybe_inst, true);
5517	try self.genCopy(lhs_ty, dst_mcv, lhs);	6163	if (vec_op and lhs_mcv.isRegister() and self.hasFeature(.avx))
		6164	copied_to_dst = false
		6165	else
		6166	try self.genCopy(lhs_ty, dst_mcv, lhs_mcv);
5518	break :dst dst_mcv;	6167	break :dst dst_mcv;
5519	};	6168	};
5520	const dst_lock: ?RegisterLock = switch (dst_mcv) {	6169	const dst_lock: ?RegisterLock = switch (dst_mcv) {
...	@@ -5523,160 +6172,52 @@ fn genBinOp(	...	@@ -5523,160 +6172,52 @@ fn genBinOp(
5523	};	6172	};
5524	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);	6173	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
5525		6174
5526	const src_mcv = if (flipped) lhs else rhs;	6175	const src_mcv = if (flipped) lhs_mcv else rhs_mcv;
5527	switch (tag) {	6176	if (!vec_op) {
5528	.add,	6177	switch (air_tag) {
5529	.addwrap,	6178	.add,
5530	=> try self.genBinOpMir(switch (lhs_ty.zigTypeTag()) {	6179	.addwrap,
5531	else => .add,	6180	=> try self.genBinOpMir(.{ ._, .add }, lhs_ty, dst_mcv, src_mcv),
5532	.Float => switch (lhs_ty.floatBits(self.target.*)) {
5533	32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse))
5534	.addss
5535	else
5536	return self.fail("TODO implement genBinOp for {s} {} without sse", .{
5537	@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
5538	}),
5539	64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2))
5540	.addsd
5541	else
5542	return self.fail("TODO implement genBinOp for {s} {} without sse2", .{
5543	@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
5544	}),
5545	else => return self.fail("TODO implement genBinOp for {s} {}", .{
5546	@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
5547	}),
5548	},
5549	}, lhs_ty, dst_mcv, src_mcv),
5550
5551	.sub,
5552	.subwrap,
5553	=> try self.genBinOpMir(switch (lhs_ty.zigTypeTag()) {
5554	else => .sub,
5555	.Float => switch (lhs_ty.floatBits(self.target.*)) {
5556	32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse))
5557	.subss
5558	else
5559	return self.fail("TODO implement genBinOp for {s} {} without sse", .{
5560	@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
5561	}),
5562	64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2))
5563	.subsd
5564	else
5565	return self.fail("TODO implement genBinOp for {s} {} without sse2", .{
5566	@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
5567	}),
5568	else => return self.fail("TODO implement genBinOp for {s} {}", .{
5569	@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
5570	}),
5571	},
5572	}, lhs_ty, dst_mcv, src_mcv),
5573
5574	.mul => try self.genBinOpMir(switch (lhs_ty.zigTypeTag()) {
5575	else => return self.fail("TODO implement genBinOp for {s} {}", .{
5576	@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
5577	}),
5578	.Float => switch (lhs_ty.floatBits(self.target.*)) {
5579	32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse))
5580	.mulss
5581	else
5582	return self.fail("TODO implement genBinOp for {s} {} without sse", .{
5583	@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
5584	}),
5585	64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2))
5586	.mulsd
5587	else
5588	return self.fail("TODO implement genBinOp for {s} {} without sse2", .{
5589	@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
5590	}),
5591	else => return self.fail("TODO implement genBinOp for {s} {}", .{
5592	@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
5593	}),
5594	},
5595	}, lhs_ty, dst_mcv, src_mcv),
5596		6181
5597	.div_float,	6182	.sub,
5598	.div_exact,	6183	.subwrap,
5599	.div_trunc,	6184	=> try self.genBinOpMir(.{ ._, .sub }, lhs_ty, dst_mcv, src_mcv),
5600	.div_floor,	6185
5601	=> {	6186	.ptr_add,
5602	try self.genBinOpMir(switch (lhs_ty.zigTypeTag()) {	6187	.ptr_sub,
5603	else => return self.fail("TODO implement genBinOp for {s} {}", .{	6188	=> {
5604	@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),	6189	const tmp_reg = try self.copyToTmpRegister(rhs_ty, src_mcv);
5605	}),	6190	const tmp_mcv = MCValue{ .register = tmp_reg };
5606	.Float => switch (lhs_ty.floatBits(self.target.*)) {	6191	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
5607	32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse))	6192	defer self.register_manager.unlockReg(tmp_lock);
5608	.divss	6193
5609	else	6194	const elem_size = lhs_ty.elemType2().abiSize(self.target.*);
5610	return self.fail("TODO implement genBinOp for {s} {} without sse", .{	6195	try self.genIntMulComplexOpMir(rhs_ty, tmp_mcv, .{ .immediate = elem_size });
5611	@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),	6196	try self.genBinOpMir(
5612	}),	6197	switch (air_tag) {
5613	64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2))	6198	.ptr_add => .{ ._, .add },
5614	.divsd	6199	.ptr_sub => .{ ._, .sub },
5615	else
5616	return self.fail("TODO implement genBinOp for {s} {} without sse2", .{
5617	@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
5618	}),
5619	else => return self.fail("TODO implement genBinOp for {s} {}", .{
5620	@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
5621	}),
5622	},
5623	}, lhs_ty, dst_mcv, src_mcv);
5624	switch (tag) {
5625	.div_float,
5626	.div_exact,
5627	=> {},
5628	.div_trunc,
5629	.div_floor,
5630	=> if (Target.x86.featureSetHas(self.target.cpu.features, .sse4_1)) {
5631	const abi_size = @intCast(u32, lhs_ty.abiSize(self.target.*));
5632	const dst_alias = registerAlias(dst_mcv.register, abi_size);
5633	try self.asmRegisterRegisterImmediate(switch (lhs_ty.floatBits(self.target.*)) {
5634	32 => .roundss,
5635	64 => .roundsd,
5636	else => unreachable,
5637	}, dst_alias, dst_alias, Immediate.u(switch (tag) {
5638	.div_trunc => 0b1_0_11,
5639	.div_floor => 0b1_0_01,
5640	else => unreachable,	6200	else => unreachable,
5641	}));	6201	},
5642	} else return self.fail("TODO implement genBinOp for {s} {} without sse4_1", .{	6202	lhs_ty,
5643	@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),	6203	dst_mcv,
5644	}),	6204	tmp_mcv,
5645	else => unreachable,	6205	);
5646	}	6206	},
5647	},
5648
5649	.ptr_add,
5650	.ptr_sub,
5651	=> {
5652	const tmp_reg = try self.copyToTmpRegister(rhs_ty, src_mcv);
5653	const tmp_mcv = MCValue{ .register = tmp_reg };
5654	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
5655	defer self.register_manager.unlockReg(tmp_lock);
5656
5657	const elem_size = lhs_ty.elemType2().abiSize(self.target.*);
5658	try self.genIntMulComplexOpMir(rhs_ty, tmp_mcv, .{ .immediate = elem_size });
5659	try self.genBinOpMir(switch (tag) {
5660	.ptr_add => .add,
5661	.ptr_sub => .sub,
5662	else => unreachable,
5663	}, lhs_ty, dst_mcv, tmp_mcv);
5664	},
5665		6207
5666	.bool_or,	6208	.bool_or,
5667	.bit_or,	6209	.bit_or,
5668	=> try self.genBinOpMir(.@"or", lhs_ty, dst_mcv, src_mcv),	6210	=> try self.genBinOpMir(.{ ._, .@"or" }, lhs_ty, dst_mcv, src_mcv),
5669		6211
5670	.bool_and,	6212	.bool_and,
5671	.bit_and,	6213	.bit_and,
5672	=> try self.genBinOpMir(.@"and", lhs_ty, dst_mcv, src_mcv),	6214	=> try self.genBinOpMir(.{ ._, .@"and" }, lhs_ty, dst_mcv, src_mcv),
5673		6215
5674	.xor => try self.genBinOpMir(.xor, lhs_ty, dst_mcv, src_mcv),	6216	.xor => try self.genBinOpMir(.{ ._, .xor }, lhs_ty, dst_mcv, src_mcv),
5675		6217
5676	.min,	6218	.min,
5677	.max,	6219	.max,
5678	=> switch (lhs_ty.zigTypeTag()) {	6220	=> {
5679	.Int => {
5680	const mat_src_mcv: MCValue = if (switch (src_mcv) {	6221	const mat_src_mcv: MCValue = if (switch (src_mcv) {
5681	.immediate,	6222	.immediate,
5682	.eflags,	6223	.eflags,
...	@@ -5698,16 +6239,16 @@ fn genBinOp(	...	@@ -5698,16 +6239,16 @@ fn genBinOp(
5698	};	6239	};
5699	defer if (mat_mcv_lock) \|lock\| self.register_manager.unlockReg(lock);	6240	defer if (mat_mcv_lock) \|lock\| self.register_manager.unlockReg(lock);
5700		6241
5701	try self.genBinOpMir(.cmp, lhs_ty, dst_mcv, mat_src_mcv);	6242	try self.genBinOpMir(.{ ._, .cmp }, lhs_ty, dst_mcv, mat_src_mcv);
5702		6243
5703	const int_info = lhs_ty.intInfo(self.target.*);	6244	const int_info = lhs_ty.intInfo(self.target.*);
5704	const cc: Condition = switch (int_info.signedness) {	6245	const cc: Condition = switch (int_info.signedness) {
5705	.unsigned => switch (tag) {	6246	.unsigned => switch (air_tag) {
5706	.min => .a,	6247	.min => .a,
5707	.max => .b,	6248	.max => .b,
5708	else => unreachable,	6249	else => unreachable,
5709	},	6250	},
5710	.signed => switch (tag) {	6251	.signed => switch (air_tag) {
5711	.min => .g,	6252	.min => .g,
5712	.max => .l,	6253	.max => .l,
5713	else => unreachable,	6254	else => unreachable,
...	@@ -5766,32 +6307,404 @@ fn genBinOp(	...	@@ -5766,32 +6307,404 @@ fn genBinOp(
5766	}	6307	}
5767	try self.genCopy(lhs_ty, dst_mcv, .{ .register = tmp_reg });	6308	try self.genCopy(lhs_ty, dst_mcv, .{ .register = tmp_reg });
5768	},	6309	},
5769	.Float => try self.genBinOpMir(switch (lhs_ty.floatBits(self.target.*)) {	6310
5770	32 => switch (tag) {
5771	.min => .minss,
5772	.max => .maxss,
5773	else => unreachable,
5774	},
5775	64 => switch (tag) {
5776	.min => .minsd,
5777	.max => .maxsd,
5778	else => unreachable,
5779	},
5780	else => return self.fail("TODO implement genBinOp for {s} {}", .{
5781	@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
5782	}),
5783	}, lhs_ty, dst_mcv, src_mcv),
5784	else => return self.fail("TODO implement genBinOp for {s} {}", .{	6311	else => return self.fail("TODO implement genBinOp for {s} {}", .{
5785	@tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),	6312	@tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
5786	}),	6313	}),
5787	},	6314	}
		6315	return dst_mcv;
		6316	}
5788		6317
		6318	const dst_reg = registerAlias(dst_mcv.getReg().?, abi_size);
		6319	const mir_tag = if (@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag()) {
		6320	else => unreachable,
		6321	.Float => switch (lhs_ty.floatBits(self.target.*)) {
		6322	16 => if (self.hasFeature(.f16c)) {
		6323	const tmp_reg = (try self.register_manager.allocReg(null, sse)).to128();
		6324	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
		6325	defer self.register_manager.unlockReg(tmp_lock);
		6326
		6327	if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
		6328	.{ .vp_w, .insr },
		6329	dst_reg,
		6330	dst_reg,
		6331	src_mcv.mem(.word),
		6332	Immediate.u(1),
		6333	) else try self.asmRegisterRegisterRegister(
		6334	.{ .vp_, .unpcklwd },
		6335	dst_reg,
		6336	dst_reg,
		6337	(if (src_mcv.isRegister())
		6338	src_mcv.getReg().?
		6339	else
		6340	try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
		6341	);
		6342	try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg, dst_reg);
		6343	try self.asmRegisterRegister(.{ .v_, .movshdup }, tmp_reg, dst_reg);
		6344	try self.asmRegisterRegisterRegister(
		6345	switch (air_tag) {
		6346	.add => .{ .v_ss, .add },
		6347	.sub => .{ .v_ss, .sub },
		6348	.div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ss, .div },
		6349	.max => .{ .v_ss, .max },
		6350	.min => .{ .v_ss, .max },
		6351	else => unreachable,
		6352	},
		6353	dst_reg,
		6354	dst_reg,
		6355	tmp_reg,
		6356	);
		6357	try self.asmRegisterRegisterImmediate(
		6358	.{ .v_, .cvtps2ph },
		6359	dst_reg,
		6360	dst_reg,
		6361	Immediate.u(0b1_00),
		6362	);
		6363	return dst_mcv;
		6364	} else null,
		6365	32 => switch (air_tag) {
		6366	.add => if (self.hasFeature(.avx)) .{ .v_ss, .add } else .{ ._ss, .add },
		6367	.sub => if (self.hasFeature(.avx)) .{ .v_ss, .sub } else .{ ._ss, .sub },
		6368	.mul => if (self.hasFeature(.avx)) .{ .v_ss, .mul } else .{ ._ss, .mul },
		6369	.div_float,
		6370	.div_trunc,
		6371	.div_floor,
		6372	.div_exact,
		6373	=> if (self.hasFeature(.avx)) .{ .v_ss, .div } else .{ ._ss, .div },
		6374	.max => if (self.hasFeature(.avx)) .{ .v_ss, .max } else .{ ._ss, .max },
		6375	.min => if (self.hasFeature(.avx)) .{ .v_ss, .min } else .{ ._ss, .min },
		6376	else => unreachable,
		6377	},
		6378	64 => switch (air_tag) {
		6379	.add => if (self.hasFeature(.avx)) .{ .v_sd, .add } else .{ ._sd, .add },
		6380	.sub => if (self.hasFeature(.avx)) .{ .v_sd, .sub } else .{ ._sd, .sub },
		6381	.mul => if (self.hasFeature(.avx)) .{ .v_sd, .mul } else .{ ._sd, .mul },
		6382	.div_float,
		6383	.div_trunc,
		6384	.div_floor,
		6385	.div_exact,
		6386	=> if (self.hasFeature(.avx)) .{ .v_sd, .div } else .{ ._sd, .div },
		6387	.max => if (self.hasFeature(.avx)) .{ .v_sd, .max } else .{ ._sd, .max },
		6388	.min => if (self.hasFeature(.avx)) .{ .v_sd, .min } else .{ ._sd, .min },
		6389	else => unreachable,
		6390	},
		6391	80, 128 => null,
		6392	else => unreachable,
		6393	},
		6394	.Vector => switch (lhs_ty.childType().zigTypeTag()) {
		6395	else => null,
		6396	.Float => switch (lhs_ty.childType().floatBits(self.target.*)) {
		6397	16 => if (self.hasFeature(.f16c)) switch (lhs_ty.vectorLen()) {
		6398	1 => {
		6399	const tmp_reg = (try self.register_manager.allocReg(null, sse)).to128();
		6400	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
		6401	defer self.register_manager.unlockReg(tmp_lock);
		6402
		6403	if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
		6404	.{ .vp_w, .insr },
		6405	dst_reg,
		6406	dst_reg,
		6407	src_mcv.mem(.word),
		6408	Immediate.u(1),
		6409	) else try self.asmRegisterRegisterRegister(
		6410	.{ .vp_, .unpcklwd },
		6411	dst_reg,
		6412	dst_reg,
		6413	(if (src_mcv.isRegister())
		6414	src_mcv.getReg().?
		6415	else
		6416	try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
		6417	);
		6418	try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg, dst_reg);
		6419	try self.asmRegisterRegister(.{ .v_, .movshdup }, tmp_reg, dst_reg);
		6420	try self.asmRegisterRegisterRegister(
		6421	switch (air_tag) {
		6422	.add => .{ .v_ss, .add },
		6423	.sub => .{ .v_ss, .sub },
		6424	.div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ss, .div },
		6425	.max => .{ .v_ss, .max },
		6426	.min => .{ .v_ss, .max },
		6427	else => unreachable,
		6428	},
		6429	dst_reg,
		6430	dst_reg,
		6431	tmp_reg,
		6432	);
		6433	try self.asmRegisterRegisterImmediate(
		6434	.{ .v_, .cvtps2ph },
		6435	dst_reg,
		6436	dst_reg,
		6437	Immediate.u(0b1_00),
		6438	);
		6439	return dst_mcv;
		6440	},
		6441	2 => {
		6442	const tmp_reg = (try self.register_manager.allocReg(null, sse)).to128();
		6443	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
		6444	defer self.register_manager.unlockReg(tmp_lock);
		6445
		6446	if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
		6447	.{ .vp_d, .insr },
		6448	dst_reg,
		6449	src_mcv.mem(.dword),
		6450	Immediate.u(1),
		6451	) else try self.asmRegisterRegisterRegister(
		6452	.{ .v_ps, .unpckl },
		6453	dst_reg,
		6454	dst_reg,
		6455	(if (src_mcv.isRegister())
		6456	src_mcv.getReg().?
		6457	else
		6458	try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
		6459	);
		6460	try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg, dst_reg);
		6461	try self.asmRegisterRegisterRegister(
		6462	.{ .v_ps, .movhl },
		6463	tmp_reg,
		6464	dst_reg,
		6465	dst_reg,
		6466	);
		6467	try self.asmRegisterRegisterRegister(
		6468	switch (air_tag) {
		6469	.add => .{ .v_ps, .add },
		6470	.sub => .{ .v_ps, .sub },
		6471	.div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div },
		6472	.max => .{ .v_ps, .max },
		6473	.min => .{ .v_ps, .max },
		6474	else => unreachable,
		6475	},
		6476	dst_reg,
		6477	dst_reg,
		6478	tmp_reg,
		6479	);
		6480	try self.asmRegisterRegisterImmediate(
		6481	.{ .v_, .cvtps2ph },
		6482	dst_reg,
		6483	dst_reg,
		6484	Immediate.u(0b1_00),
		6485	);
		6486	return dst_mcv;
		6487	},
		6488	3...4 => {
		6489	const tmp_reg = (try self.register_manager.allocReg(null, sse)).to128();
		6490	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
		6491	defer self.register_manager.unlockReg(tmp_lock);
		6492
		6493	try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg, dst_reg);
		6494	if (src_mcv.isMemory()) try self.asmRegisterMemory(
		6495	.{ .v_, .cvtph2ps },
		6496	tmp_reg,
		6497	src_mcv.mem(.qword),
		6498	) else try self.asmRegisterRegister(
		6499	.{ .v_, .cvtph2ps },
		6500	tmp_reg,
		6501	(if (src_mcv.isRegister())
		6502	src_mcv.getReg().?
		6503	else
		6504	try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
		6505	);
		6506	try self.asmRegisterRegisterRegister(
		6507	switch (air_tag) {
		6508	.add => .{ .v_ps, .add },
		6509	.sub => .{ .v_ps, .sub },
		6510	.div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div },
		6511	.max => .{ .v_ps, .max },
		6512	.min => .{ .v_ps, .max },
		6513	else => unreachable,
		6514	},
		6515	dst_reg,
		6516	dst_reg,
		6517	tmp_reg,
		6518	);
		6519	try self.asmRegisterRegisterImmediate(
		6520	.{ .v_, .cvtps2ph },
		6521	dst_reg,
		6522	dst_reg,
		6523	Immediate.u(0b1_00),
		6524	);
		6525	return dst_mcv;
		6526	},
		6527	5...8 => {
		6528	const tmp_reg = (try self.register_manager.allocReg(null, sse)).to256();
		6529	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
		6530	defer self.register_manager.unlockReg(tmp_lock);
		6531
		6532	try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, dst_reg.to256(), dst_reg);
		6533	if (src_mcv.isMemory()) try self.asmRegisterMemory(
		6534	.{ .v_, .cvtph2ps },
		6535	tmp_reg,
		6536	src_mcv.mem(.xword),
		6537	) else try self.asmRegisterRegister(
		6538	.{ .v_, .cvtph2ps },
		6539	tmp_reg,
		6540	(if (src_mcv.isRegister())
		6541	src_mcv.getReg().?
		6542	else
		6543	try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
		6544	);
		6545	try self.asmRegisterRegisterRegister(
		6546	switch (air_tag) {
		6547	.add => .{ .v_ps, .add },
		6548	.sub => .{ .v_ps, .sub },
		6549	.div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div },
		6550	.max => .{ .v_ps, .max },
		6551	.min => .{ .v_ps, .max },
		6552	else => unreachable,
		6553	},
		6554	dst_reg.to256(),
		6555	dst_reg.to256(),
		6556	tmp_reg,
		6557	);
		6558	try self.asmRegisterRegisterImmediate(
		6559	.{ .v_, .cvtps2ph },
		6560	dst_reg,
		6561	dst_reg.to256(),
		6562	Immediate.u(0b1_00),
		6563	);
		6564	return dst_mcv;
		6565	},
		6566	else => null,
		6567	} else null,
		6568	32 => switch (lhs_ty.vectorLen()) {
		6569	1 => switch (air_tag) {
		6570	.add => if (self.hasFeature(.avx)) .{ .v_ss, .add } else .{ ._ss, .add },
		6571	.sub => if (self.hasFeature(.avx)) .{ .v_ss, .sub } else .{ ._ss, .sub },
		6572	.mul => if (self.hasFeature(.avx)) .{ .v_ss, .mul } else .{ ._ss, .mul },
		6573	.div_float,
		6574	.div_trunc,
		6575	.div_floor,
		6576	.div_exact,
		6577	=> if (self.hasFeature(.avx)) .{ .v_ss, .div } else .{ ._ss, .div },
		6578	.max => if (self.hasFeature(.avx)) .{ .v_ss, .max } else .{ ._ss, .max },
		6579	.min => if (self.hasFeature(.avx)) .{ .v_ss, .min } else .{ ._ss, .min },
		6580	else => unreachable,
		6581	},
		6582	2...4 => switch (air_tag) {
		6583	.add => if (self.hasFeature(.avx)) .{ .v_ps, .add } else .{ ._ps, .add },
		6584	.sub => if (self.hasFeature(.avx)) .{ .v_ps, .sub } else .{ ._ps, .sub },
		6585	.mul => if (self.hasFeature(.avx)) .{ .v_ps, .mul } else .{ ._ps, .mul },
		6586	.div_float,
		6587	.div_trunc,
		6588	.div_floor,
		6589	.div_exact,
		6590	=> if (self.hasFeature(.avx)) .{ .v_ps, .div } else .{ ._ps, .div },
		6591	.max => if (self.hasFeature(.avx)) .{ .v_ps, .max } else .{ ._ps, .max },
		6592	.min => if (self.hasFeature(.avx)) .{ .v_ps, .min } else .{ ._ps, .min },
		6593	else => unreachable,
		6594	},
		6595	5...8 => if (self.hasFeature(.avx)) switch (air_tag) {
		6596	.add => .{ .v_ps, .add },
		6597	.sub => .{ .v_ps, .sub },
		6598	.mul => .{ .v_ps, .mul },
		6599	.div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div },
		6600	.max => .{ .v_ps, .max },
		6601	.min => .{ .v_ps, .min },
		6602	else => unreachable,
		6603	} else null,
		6604	else => null,
		6605	},
		6606	64 => switch (lhs_ty.vectorLen()) {
		6607	1 => switch (air_tag) {
		6608	.add => if (self.hasFeature(.avx)) .{ .v_sd, .add } else .{ ._sd, .add },
		6609	.sub => if (self.hasFeature(.avx)) .{ .v_sd, .sub } else .{ ._sd, .sub },
		6610	.mul => if (self.hasFeature(.avx)) .{ .v_sd, .mul } else .{ ._sd, .mul },
		6611	.div_float,
		6612	.div_trunc,
		6613	.div_floor,
		6614	.div_exact,
		6615	=> if (self.hasFeature(.avx)) .{ .v_sd, .div } else .{ ._sd, .div },
		6616	.max => if (self.hasFeature(.avx)) .{ .v_sd, .max } else .{ ._sd, .max },
		6617	.min => if (self.hasFeature(.avx)) .{ .v_sd, .min } else .{ ._sd, .min },
		6618	else => unreachable,
		6619	},
		6620	2 => switch (air_tag) {
		6621	.add => if (self.hasFeature(.avx)) .{ .v_pd, .add } else .{ ._pd, .add },
		6622	.sub => if (self.hasFeature(.avx)) .{ .v_pd, .sub } else .{ ._pd, .sub },
		6623	.mul => if (self.hasFeature(.avx)) .{ .v_pd, .mul } else .{ ._pd, .mul },
		6624	.div_float,
		6625	.div_trunc,
		6626	.div_floor,
		6627	.div_exact,
		6628	=> if (self.hasFeature(.avx)) .{ .v_pd, .div } else .{ ._pd, .div },
		6629	.max => if (self.hasFeature(.avx)) .{ .v_pd, .max } else .{ ._pd, .max },
		6630	.min => if (self.hasFeature(.avx)) .{ .v_pd, .min } else .{ ._pd, .min },
		6631	else => unreachable,
		6632	},
		6633	3...4 => if (self.hasFeature(.avx)) switch (air_tag) {
		6634	.add => .{ .v_pd, .add },
		6635	.sub => .{ .v_pd, .sub },
		6636	.mul => .{ .v_pd, .mul },
		6637	.div_float, .div_trunc, .div_floor, .div_exact => .{ .v_pd, .div },
		6638	.max => .{ .v_pd, .max },
		6639	.min => .{ .v_pd, .min },
		6640	else => unreachable,
		6641	} else null,
		6642	else => null,
		6643	},
		6644	80, 128 => null,
		6645	else => unreachable,
		6646	},
		6647	},
		6648	})) \|tag\| tag else return self.fail("TODO implement genBinOp for {s} {}", .{
		6649	@tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
		6650	});
		6651	if (self.hasFeature(.avx)) {
		6652	const src1_alias =
		6653	if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);
		6654	if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
		6655	mir_tag,
		6656	dst_reg,
		6657	src1_alias,
		6658	src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
		6659	) else try self.asmRegisterRegisterRegister(
		6660	mir_tag,
		6661	dst_reg,
		6662	src1_alias,
		6663	registerAlias(if (src_mcv.isRegister())
		6664	src_mcv.getReg().?
		6665	else
		6666	try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
		6667	);
		6668	} else {
		6669	assert(copied_to_dst);
		6670	if (src_mcv.isMemory()) try self.asmRegisterMemory(
		6671	mir_tag,
		6672	dst_reg,
		6673	src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
		6674	) else try self.asmRegisterRegister(
		6675	mir_tag,
		6676	dst_reg,
		6677	registerAlias(if (src_mcv.isRegister())
		6678	src_mcv.getReg().?
		6679	else
		6680	try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
		6681	);
		6682	}
		6683	switch (air_tag) {
		6684	.add, .sub, .mul, .div_float, .div_exact => {},
		6685	.div_trunc, .div_floor => try self.genRound(
		6686	lhs_ty,
		6687	dst_reg,
		6688	.{ .register = dst_reg },
		6689	switch (air_tag) {
		6690	.div_trunc => 0b1_0_11,
		6691	.div_floor => 0b1_0_01,
		6692	else => unreachable,
		6693	},
		6694	),
		6695	.max, .min => {}, // TODO: unordered select
5789	else => unreachable,	6696	else => unreachable,
5790	}	6697	}
5791	return dst_mcv;	6698	return dst_mcv;
5792	}	6699	}
5793		6700
5794	fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void {	6701	fn genBinOpMir(
		6702	self: *Self,
		6703	mir_tag: Mir.Inst.FixedTag,
		6704	ty: Type,
		6705	dst_mcv: MCValue,
		6706	src_mcv: MCValue,
		6707	) !void {
5795	const abi_size = @intCast(u32, ty.abiSize(self.target.*));	6708	const abi_size = @intCast(u32, ty.abiSize(self.target.*));
5796	switch (dst_mcv) {	6709	switch (dst_mcv) {
5797	.none,	6710	.none,
...	@@ -5818,20 +6731,11 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s	...	@@ -5818,20 +6731,11 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
5818	.register_overflow,	6731	.register_overflow,
5819	.reserved_frame,	6732	.reserved_frame,
5820	=> unreachable,	6733	=> unreachable,
5821	.register => \|src_reg\| switch (ty.zigTypeTag()) {	6734	.register => \|src_reg\| try self.asmRegisterRegister(
5822	.Float => {	6735	mir_tag,
5823	if (!Target.x86.featureSetHas(self.target.cpu.features, .sse))	6736	dst_alias,
5824	return self.fail("TODO genBinOpMir for {s} {} without sse", .{	6737	registerAlias(src_reg, abi_size),
5825	@tagName(mir_tag), ty.fmt(self.bin_file.options.module.?),	6738	),
5826	});
5827	return self.asmRegisterRegister(mir_tag, dst_reg.to128(), src_reg.to128());
5828	},
5829	else => try self.asmRegisterRegister(
5830	mir_tag,
5831	dst_alias,
5832	registerAlias(src_reg, abi_size),
5833	),
5834	},
5835	.immediate => \|imm\| switch (self.regBitSize(ty)) {	6739	.immediate => \|imm\| switch (self.regBitSize(ty)) {
5836	8 => try self.asmRegisterImmediate(	6740	8 => try self.asmRegisterImmediate(
5837	mir_tag,	6741	mir_tag,
...	@@ -6005,14 +6909,14 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s	...	@@ -6005,14 +6909,14 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
6005	};	6909	};
6006	var off: i32 = 0;	6910	var off: i32 = 0;
6007	while (off < abi_size) : (off += 8) {	6911	while (off < abi_size) : (off += 8) {
6008	const mir_limb_tag = switch (off) {	6912	const mir_limb_tag: Mir.Inst.FixedTag = switch (off) {
6009	0 => mir_tag,	6913	0 => mir_tag,
6010	else => switch (mir_tag) {	6914	else => switch (mir_tag[1]) {
6011	.add => .adc,	6915	.add => .{ ._, .adc },
6012	.sub, .cmp => .sbb,	6916	.sub, .cmp => .{ ._, .sbb },
6013	.@"or", .@"and", .xor => mir_tag,	6917	.@"or", .@"and", .xor => mir_tag,
6014	else => return self.fail("TODO genBinOpMir implement large ABI for {s}", .{	6918	else => return self.fail("TODO genBinOpMir implement large ABI for {s}", .{
6015	@tagName(mir_tag),	6919	@tagName(mir_tag[1]),
6016	}),	6920	}),
6017	},	6921	},
6018	};	6922	};
...	@@ -6184,14 +7088,14 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M	...	@@ -6184,14 +7088,14 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
6184	.reserved_frame,	7088	.reserved_frame,
6185	=> unreachable,	7089	=> unreachable,
6186	.register => \|src_reg\| try self.asmRegisterRegister(	7090	.register => \|src_reg\| try self.asmRegisterRegister(
6187	.imul,	7091	.{ .i_, .mul },
6188	dst_alias,	7092	dst_alias,
6189	registerAlias(src_reg, abi_size),	7093	registerAlias(src_reg, abi_size),
6190	),	7094	),
6191	.immediate => \|imm\| {	7095	.immediate => \|imm\| {
6192	if (math.cast(i32, imm)) \|small\| {	7096	if (math.cast(i32, imm)) \|small\| {
6193	try self.asmRegisterRegisterImmediate(	7097	try self.asmRegisterRegisterImmediate(
6194	.imul,	7098	.{ .i_, .mul },
6195	dst_alias,	7099	dst_alias,
6196	dst_alias,	7100	dst_alias,
6197	Immediate.s(small),	7101	Immediate.s(small),
...	@@ -6211,19 +7115,19 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M	...	@@ -6211,19 +7115,19 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
6211	.lea_tlv,	7115	.lea_tlv,
6212	.lea_frame,	7116	.lea_frame,
6213	=> try self.asmRegisterRegister(	7117	=> try self.asmRegisterRegister(
6214	.imul,	7118	.{ .i_, .mul },
6215	dst_alias,	7119	dst_alias,
6216	registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size),	7120	registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size),
6217	),	7121	),
6218	.memory, .indirect, .load_frame => try self.asmRegisterMemory(	7122	.memory, .indirect, .load_frame => try self.asmRegisterMemory(
6219	.imul,	7123	.{ .i_, .mul },
6220	dst_alias,	7124	dst_alias,
6221	Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (src_mcv) {	7125	Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (src_mcv) {
6222	.memory => \|addr\| .{	7126	.memory => \|addr\| .{
6223	.base = .{ .reg = .ds },	7127	.base = .{ .reg = .ds },
6224	.disp = math.cast(i32, @bitCast(i64, addr)) orelse	7128	.disp = math.cast(i32, @bitCast(i64, addr)) orelse
6225	return self.asmRegisterRegister(	7129	return self.asmRegisterRegister(
6226	.imul,	7130	.{ .i_, .mul },
6227	dst_alias,	7131	dst_alias,
6228	registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size),	7132	registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size),
6229	),	7133	),
...	@@ -6348,12 +7252,12 @@ fn genVarDbgInfo(	...	@@ -6348,12 +7252,12 @@ fn genVarDbgInfo(
6348	}	7252	}
6349		7253
6350	fn airTrap(self: *Self) !void {	7254	fn airTrap(self: *Self) !void {
6351	try self.asmOpOnly(.ud2);	7255	try self.asmOpOnly(.{ ._, .ud2 });
6352	return self.finishAirBookkeeping();	7256	return self.finishAirBookkeeping();
6353	}	7257	}
6354		7258
6355	fn airBreakpoint(self: *Self) !void {	7259	fn airBreakpoint(self: *Self) !void {
6356	try self.asmOpOnly(.int3);	7260	try self.asmOpOnly(.{ ._, .int3 });
6357	return self.finishAirBookkeeping();	7261	return self.finishAirBookkeeping();
6358	}	7262	}
6359		7263
...	@@ -6374,7 +7278,7 @@ fn airFence(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6374,7 +7278,7 @@ fn airFence(self: *Self, inst: Air.Inst.Index) !void {
6374	switch (order) {	7278	switch (order) {
6375	.Unordered, .Monotonic => unreachable,	7279	.Unordered, .Monotonic => unreachable,
6376	.Acquire, .Release, .AcqRel => {},	7280	.Acquire, .Release, .AcqRel => {},
6377	.SeqCst => try self.asmOpOnly(.mfence),	7281	.SeqCst => try self.asmOpOnly(.{ ._, .mfence }),
6378	}	7282	}
6379	return self.finishAirBookkeeping();	7283	return self.finishAirBookkeeping();
6380	}	7284	}
...	@@ -6468,7 +7372,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier	...	@@ -6468,7 +7372,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
6468	const atom = elf_file.getAtom(atom_index);	7372	const atom = elf_file.getAtom(atom_index);
6469	_ = try atom.getOrCreateOffsetTableEntry(elf_file);	7373	_ = try atom.getOrCreateOffsetTableEntry(elf_file);
6470	const got_addr = atom.getOffsetTableAddress(elf_file);	7374	const got_addr = atom.getOffsetTableAddress(elf_file);
6471	try self.asmMemory(.call, Memory.sib(.qword, .{	7375	try self.asmMemory(.{ ._, .call }, Memory.sib(.qword, .{
6472	.base = .{ .reg = .ds },	7376	.base = .{ .reg = .ds },
6473	.disp = @intCast(i32, got_addr),	7377	.disp = @intCast(i32, got_addr),
6474	}));	7378	}));
...	@@ -6476,12 +7380,12 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier	...	@@ -6476,12 +7380,12 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
6476	const atom = try coff_file.getOrCreateAtomForDecl(func.owner_decl);	7380	const atom = try coff_file.getOrCreateAtomForDecl(func.owner_decl);
6477	const sym_index = coff_file.getAtom(atom).getSymbolIndex().?;	7381	const sym_index = coff_file.getAtom(atom).getSymbolIndex().?;
6478	try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });	7382	try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });
6479	try self.asmRegister(.call, .rax);	7383	try self.asmRegister(.{ ._, .call }, .rax);
6480	} else if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {	7384	} else if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {
6481	const atom = try macho_file.getOrCreateAtomForDecl(func.owner_decl);	7385	const atom = try macho_file.getOrCreateAtomForDecl(func.owner_decl);
6482	const sym_index = macho_file.getAtom(atom).getSymbolIndex().?;	7386	const sym_index = macho_file.getAtom(atom).getSymbolIndex().?;
6483	try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });	7387	try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });
6484	try self.asmRegister(.call, .rax);	7388	try self.asmRegister(.{ ._, .call }, .rax);
6485	} else if (self.bin_file.cast(link.File.Plan9)) \|p9\| {	7389	} else if (self.bin_file.cast(link.File.Plan9)) \|p9\| {
6486	const decl_block_index = try p9.seeDecl(func.owner_decl);	7390	const decl_block_index = try p9.seeDecl(func.owner_decl);
6487	const decl_block = p9.getDeclBlock(decl_block_index);	7391	const decl_block = p9.getDeclBlock(decl_block_index);
...	@@ -6490,7 +7394,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier	...	@@ -6490,7 +7394,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
6490	const got_addr = p9.bases.data;	7394	const got_addr = p9.bases.data;
6491	const got_index = decl_block.got_index.?;	7395	const got_index = decl_block.got_index.?;
6492	const fn_got_addr = got_addr + got_index * ptr_bytes;	7396	const fn_got_addr = got_addr + got_index * ptr_bytes;
6493	try self.asmMemory(.call, Memory.sib(.qword, .{	7397	try self.asmMemory(.{ ._, .call }, Memory.sib(.qword, .{
6494	.base = .{ .reg = .ds },	7398	.base = .{ .reg = .ds },
6495	.disp = @intCast(i32, fn_got_addr),	7399	.disp = @intCast(i32, fn_got_addr),
6496	}));	7400	}));
...	@@ -6503,22 +7407,24 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier	...	@@ -6503,22 +7407,24 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
6503	const atom_index = try self.owner.getSymbolIndex(self);	7407	const atom_index = try self.owner.getSymbolIndex(self);
6504	const sym_index = try coff_file.getGlobalSymbol(decl_name, lib_name);	7408	const sym_index = try coff_file.getGlobalSymbol(decl_name, lib_name);
6505	_ = try self.addInst(.{	7409	_ = try self.addInst(.{
6506	.tag = .mov_linker,	7410	.tag = .mov,
6507	.ops = .import_reloc,	7411	.ops = .import_reloc,
6508	.data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{	7412	.data = .{ .rx = .{
6509	.reg = @enumToInt(Register.rax),	7413	.r1 = .rax,
6510	.atom_index = atom_index,	7414	.payload = try self.addExtra(Mir.Reloc{
6511	.sym_index = sym_index,	7415	.atom_index = atom_index,
6512	}) },	7416	.sym_index = sym_index,
		7417	}),
		7418	} },
6513	});	7419	});
6514	try self.asmRegister(.call, .rax);	7420	try self.asmRegister(.{ ._, .call }, .rax);
6515	} else if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {	7421	} else if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {
6516	const atom_index = try self.owner.getSymbolIndex(self);	7422	const atom_index = try self.owner.getSymbolIndex(self);
6517	const sym_index = try macho_file.getGlobalSymbol(decl_name, lib_name);	7423	const sym_index = try macho_file.getGlobalSymbol(decl_name, lib_name);
6518	_ = try self.addInst(.{	7424	_ = try self.addInst(.{
6519	.tag = .call_extern,	7425	.tag = .call,
6520	.ops = undefined,	7426	.ops = .extern_fn_reloc,
6521	.data = .{ .relocation = .{	7427	.data = .{ .reloc = .{
6522	.atom_index = atom_index,	7428	.atom_index = atom_index,
6523	.sym_index = sym_index,	7429	.sym_index = sym_index,
6524	} },	7430	} },
...	@@ -6533,7 +7439,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier	...	@@ -6533,7 +7439,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
6533	assert(ty.zigTypeTag() == .Pointer);	7439	assert(ty.zigTypeTag() == .Pointer);
6534	const mcv = try self.resolveInst(callee);	7440	const mcv = try self.resolveInst(callee);
6535	try self.genSetReg(.rax, Type.usize, mcv);	7441	try self.genSetReg(.rax, Type.usize, mcv);
6536	try self.asmRegister(.call, .rax);	7442	try self.asmRegister(.{ ._, .call }, .rax);
6537	}	7443	}
6538		7444
6539	var bt = self.liveness.iterateBigTomb(inst);	7445	var bt = self.liveness.iterateBigTomb(inst);
...	@@ -6588,8 +7494,6 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6588,8 +7494,6 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
6588	fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	7494	fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
6589	const bin_op = self.air.instructions.items(.data)[inst].bin_op;	7495	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
6590	const ty = self.air.typeOf(bin_op.lhs);	7496	const ty = self.air.typeOf(bin_op.lhs);
6591	const ty_abi_size = ty.abiSize(self.target.*);
6592	const can_reuse = ty_abi_size <= 8;
6593		7497
6594	try self.spillEflagsIfOccupied();	7498	try self.spillEflagsIfOccupied();
6595	self.eflags_inst = inst;	7499	self.eflags_inst = inst;
...	@@ -6608,52 +7512,103 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	...	@@ -6608,52 +7512,103 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
6608	};	7512	};
6609	defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);	7513	defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
6610		7514
6611	const dst_mem_ok = !ty.isRuntimeFloat();	7515	const result = MCValue{
6612	var flipped = false;	7516	.eflags = switch (ty.zigTypeTag()) {
6613	const dst_mcv: MCValue = if (can_reuse and !lhs_mcv.isImmediate() and	7517	else => result: {
6614	(dst_mem_ok or lhs_mcv.isRegister()) and self.liveness.operandDies(inst, 0))	7518	var flipped = false;
6615	lhs_mcv	7519	const dst_mcv: MCValue = if (lhs_mcv.isRegister() or lhs_mcv.isMemory())
6616	else if (can_reuse and !rhs_mcv.isImmediate() and	7520	lhs_mcv
6617	(dst_mem_ok or rhs_mcv.isRegister()) and self.liveness.operandDies(inst, 1))	7521	else if (rhs_mcv.isRegister() or rhs_mcv.isMemory()) dst: {
6618	dst: {	7522	flipped = true;
6619	flipped = true;	7523	break :dst rhs_mcv;
6620	break :dst rhs_mcv;	7524	} else .{ .register = try self.copyToTmpRegister(ty, lhs_mcv) };
6621	} else if (dst_mem_ok) dst: {	7525	const dst_lock = switch (dst_mcv) {
6622	const dst_mcv = try self.allocTempRegOrMem(ty, true);	7526	.register => \|reg\| self.register_manager.lockReg(reg),
6623	try self.genCopy(ty, dst_mcv, lhs_mcv);	7527	else => null,
6624	break :dst dst_mcv;	7528	};
6625	} else .{ .register = try self.copyToTmpRegister(ty, lhs_mcv) };	7529	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
6626	const dst_lock = switch (dst_mcv) {	7530	const src_mcv = if (flipped) lhs_mcv else rhs_mcv;
6627	.register => \|reg\| self.register_manager.lockReg(reg),
6628	else => null,
6629	};
6630	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
6631		7531
6632	const src_mcv = if (flipped) lhs_mcv else rhs_mcv;	7532	try self.genBinOpMir(.{ ._, .cmp }, ty, dst_mcv, src_mcv);
6633	try self.genBinOpMir(switch (ty.zigTypeTag()) {	7533	break :result Condition.fromCompareOperator(
6634	else => .cmp,	7534	if (ty.isAbiInt()) ty.intInfo(self.target.*).signedness else .unsigned,
6635	.Float => switch (ty.floatBits(self.target.*)) {	7535	if (flipped) op.reverse() else op,
6636	32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse))	7536	);
6637	.ucomiss	7537	},
6638	else	7538	.Float => result: {
6639	return self.fail("TODO implement airCmp for {} without sse", .{	7539	const flipped = switch (op) {
6640	ty.fmt(self.bin_file.options.module.?),	7540	.lt, .lte => true,
6641	}),	7541	.eq, .gte, .gt, .neq => false,
6642	64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2))	7542	};
6643	.ucomisd
6644	else
6645	return self.fail("TODO implement airCmp for {} without sse2", .{
6646	ty.fmt(self.bin_file.options.module.?),
6647	}),
6648	else => return self.fail("TODO implement airCmp for {}", .{
6649	ty.fmt(self.bin_file.options.module.?),
6650	}),
6651	},
6652	}, ty, dst_mcv, src_mcv);
6653		7543
6654	const signedness = if (ty.isAbiInt()) ty.intInfo(self.target.*).signedness else .unsigned;	7544	const dst_mcv = if (flipped) rhs_mcv else lhs_mcv;
6655	const result = MCValue{	7545	const dst_reg = if (dst_mcv.isRegister())
6656	.eflags = Condition.fromCompareOperator(signedness, if (flipped) op.reverse() else op),	7546	dst_mcv.getReg().?
		7547	else
		7548	try self.copyToTmpRegister(ty, dst_mcv);
		7549	const dst_lock = self.register_manager.lockReg(dst_reg);
		7550	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
		7551	const src_mcv = if (flipped) lhs_mcv else rhs_mcv;
		7552
		7553	switch (ty.floatBits(self.target.*)) {
		7554	16 => if (self.hasFeature(.f16c)) {
		7555	const tmp1_reg = (try self.register_manager.allocReg(null, sse)).to128();
		7556	const tmp1_mcv = MCValue{ .register = tmp1_reg };
		7557	const tmp1_lock = self.register_manager.lockRegAssumeUnused(tmp1_reg);
		7558	defer self.register_manager.unlockReg(tmp1_lock);
		7559
		7560	const tmp2_reg = (try self.register_manager.allocReg(null, sse)).to128();
		7561	const tmp2_mcv = MCValue{ .register = tmp2_reg };
		7562	const tmp2_lock = self.register_manager.lockRegAssumeUnused(tmp2_reg);
		7563	defer self.register_manager.unlockReg(tmp2_lock);
		7564
		7565	if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
		7566	.{ .vp_w, .insr },
		7567	tmp1_reg,
		7568	dst_reg.to128(),
		7569	src_mcv.mem(.word),
		7570	Immediate.u(1),
		7571	) else try self.asmRegisterRegisterRegister(
		7572	.{ .vp_, .unpcklwd },
		7573	tmp1_reg,
		7574	dst_reg.to128(),
		7575	(if (src_mcv.isRegister())
		7576	src_mcv.getReg().?
		7577	else
		7578	try self.copyToTmpRegister(ty, src_mcv)).to128(),
		7579	);
		7580	try self.asmRegisterRegister(.{ .v_, .cvtph2ps }, tmp1_reg, tmp1_reg);
		7581	try self.asmRegisterRegister(.{ .v_, .movshdup }, tmp2_reg, tmp1_reg);
		7582	try self.genBinOpMir(.{ ._ss, .ucomi }, ty, tmp1_mcv, tmp2_mcv);
		7583	} else return self.fail("TODO implement airCmp for {}", .{
		7584	ty.fmt(self.bin_file.options.module.?),
		7585	}),
		7586	32 => try self.genBinOpMir(
		7587	.{ ._ss, .ucomi },
		7588	ty,
		7589	.{ .register = dst_reg },
		7590	src_mcv,
		7591	),
		7592	64 => try self.genBinOpMir(
		7593	.{ ._sd, .ucomi },
		7594	ty,
		7595	.{ .register = dst_reg },
		7596	src_mcv,
		7597	),
		7598	else => return self.fail("TODO implement airCmp for {}", .{
		7599	ty.fmt(self.bin_file.options.module.?),
		7600	}),
		7601	}
		7602
		7603	break :result switch (if (flipped) op.reverse() else op) {
		7604	.lt, .lte => unreachable, // required to have been canonicalized to gt(e)
		7605	.gt => .a,
		7606	.gte => .ae,
		7607	.eq => .z_and_np,
		7608	.neq => .nz_or_p,
		7609	};
		7610	},
		7611	},
6657	};	7612	};
6658	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });	7613	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
6659	}	7614	}
...	@@ -6683,7 +7638,7 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6683,7 +7638,7 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
6683	else => try self.copyToTmpRegister(op_ty, op_mcv),	7638	else => try self.copyToTmpRegister(op_ty, op_mcv),
6684	};	7639	};
6685	try self.asmRegisterMemory(	7640	try self.asmRegisterMemory(
6686	.cmp,	7641	.{ ._, .cmp },
6687	registerAlias(dst_reg, op_abi_size),	7642	registerAlias(dst_reg, op_abi_size),
6688	Memory.sib(Memory.PtrSize.fromSize(op_abi_size), .{ .base = .{ .reg = addr_reg } }),	7643	Memory.sib(Memory.PtrSize.fromSize(op_abi_size), .{ .base = .{ .reg = addr_reg } }),
6689	);	7644	);
...	@@ -6757,8 +7712,8 @@ fn genTry(	...	@@ -6757,8 +7712,8 @@ fn genTry(
6757	fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {	7712	fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
6758	const dbg_stmt = self.air.instructions.items(.data)[inst].dbg_stmt;	7713	const dbg_stmt = self.air.instructions.items(.data)[inst].dbg_stmt;
6759	_ = try self.addInst(.{	7714	_ = try self.addInst(.{
6760	.tag = .dbg_line,	7715	.tag = .pseudo,
6761	.ops = undefined,	7716	.ops = .pseudo_dbg_line_line_column,
6762	.data = .{ .line_column = .{	7717	.data = .{ .line_column = .{
6763	.line = dbg_stmt.line,	7718	.line = dbg_stmt.line,
6764	.column = dbg_stmt.column,	7719	.column = dbg_stmt.column,
...	@@ -6803,7 +7758,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {	...	@@ -6803,7 +7758,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {
6803	},	7758	},
6804	.register => \|reg\| {	7759	.register => \|reg\| {
6805	try self.spillEflagsIfOccupied();	7760	try self.spillEflagsIfOccupied();
6806	try self.asmRegisterImmediate(.@"test", reg, Immediate.u(1));	7761	try self.asmRegisterImmediate(.{ ._, .@"test" }, reg, Immediate.u(1));
6807	return self.asmJccReloc(undefined, .e);	7762	return self.asmJccReloc(undefined, .e);
6808	},	7763	},
6809	.immediate,	7764	.immediate,
...	@@ -6906,13 +7861,13 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC	...	@@ -6906,13 +7861,13 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
6906	const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));	7861	const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
6907	const alias_reg = registerAlias(opt_reg, some_abi_size);	7862	const alias_reg = registerAlias(opt_reg, some_abi_size);
6908	assert(some_abi_size * 8 == alias_reg.bitSize());	7863	assert(some_abi_size * 8 == alias_reg.bitSize());
6909	try self.asmRegisterRegister(.@"test", alias_reg, alias_reg);	7864	try self.asmRegisterRegister(.{ ._, .@"test" }, alias_reg, alias_reg);
6910	return .{ .eflags = .z };	7865	return .{ .eflags = .z };
6911	}	7866	}
6912	assert(some_info.ty.tag() == .bool);	7867	assert(some_info.ty.tag() == .bool);
6913	const opt_abi_size = @intCast(u32, opt_ty.abiSize(self.target.*));	7868	const opt_abi_size = @intCast(u32, opt_ty.abiSize(self.target.*));
6914	try self.asmRegisterImmediate(	7869	try self.asmRegisterImmediate(
6915	.bt,	7870	.{ ._, .bt },
6916	registerAlias(opt_reg, opt_abi_size),	7871	registerAlias(opt_reg, opt_abi_size),
6917	Immediate.u(@intCast(u6, some_info.off * 8)),	7872	Immediate.u(@intCast(u6, some_info.off * 8)),
6918	);	7873	);
...	@@ -6931,7 +7886,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC	...	@@ -6931,7 +7886,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
6931	try self.genSetReg(addr_reg, Type.usize, opt_mcv.address());	7886	try self.genSetReg(addr_reg, Type.usize, opt_mcv.address());
6932	const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));	7887	const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
6933	try self.asmMemoryImmediate(	7888	try self.asmMemoryImmediate(
6934	.cmp,	7889	.{ ._, .cmp },
6935	Memory.sib(Memory.PtrSize.fromSize(some_abi_size), .{	7890	Memory.sib(Memory.PtrSize.fromSize(some_abi_size), .{
6936	.base = .{ .reg = addr_reg },	7891	.base = .{ .reg = addr_reg },
6937	.disp = some_info.off,	7892	.disp = some_info.off,
...	@@ -6944,7 +7899,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC	...	@@ -6944,7 +7899,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
6944	.indirect, .load_frame => {	7899	.indirect, .load_frame => {
6945	const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));	7900	const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
6946	try self.asmMemoryImmediate(	7901	try self.asmMemoryImmediate(
6947	.cmp,	7902	.{ ._, .cmp },
6948	Memory.sib(Memory.PtrSize.fromSize(some_abi_size), switch (opt_mcv) {	7903	Memory.sib(Memory.PtrSize.fromSize(some_abi_size), switch (opt_mcv) {
6949	.indirect => \|reg_off\| .{	7904	.indirect => \|reg_off\| .{
6950	.base = .{ .reg = reg_off.reg },	7905	.base = .{ .reg = reg_off.reg },
...	@@ -6986,7 +7941,7 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)	...	@@ -6986,7 +7941,7 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)
6986		7941
6987	const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));	7942	const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
6988	try self.asmMemoryImmediate(	7943	try self.asmMemoryImmediate(
6989	.cmp,	7944	.{ ._, .cmp },
6990	Memory.sib(Memory.PtrSize.fromSize(some_abi_size), .{	7945	Memory.sib(Memory.PtrSize.fromSize(some_abi_size), .{
6991	.base = .{ .reg = ptr_reg },	7946	.base = .{ .reg = ptr_reg },
6992	.disp = some_info.off,	7947	.disp = some_info.off,
...	@@ -7017,14 +7972,24 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) !	...	@@ -7017,14 +7972,24 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) !
7017	const tmp_reg = try self.copyToTmpRegister(ty, operand);	7972	const tmp_reg = try self.copyToTmpRegister(ty, operand);
7018	if (err_off > 0) {	7973	if (err_off > 0) {
7019	const shift = @intCast(u6, err_off * 8);	7974	const shift = @intCast(u6, err_off * 8);
7020	try self.genShiftBinOpMir(.shr, ty, .{ .register = tmp_reg }, .{ .immediate = shift });	7975	try self.genShiftBinOpMir(
		7976	.{ ._r, .sh },
		7977	ty,
		7978	.{ .register = tmp_reg },
		7979	.{ .immediate = shift },
		7980	);
7021	} else {	7981	} else {
7022	try self.truncateRegister(Type.anyerror, tmp_reg);	7982	try self.truncateRegister(Type.anyerror, tmp_reg);
7023	}	7983	}
7024	try self.genBinOpMir(.cmp, Type.anyerror, .{ .register = tmp_reg }, .{ .immediate = 0 });	7984	try self.genBinOpMir(
		7985	.{ ._, .cmp },
		7986	Type.anyerror,
		7987	.{ .register = tmp_reg },
		7988	.{ .immediate = 0 },
		7989	);
7025	},	7990	},
7026	.load_frame => \|frame_addr\| try self.genBinOpMir(	7991	.load_frame => \|frame_addr\| try self.genBinOpMir(
7027	.cmp,	7992	.{ ._, .cmp },
7028	Type.anyerror,	7993	Type.anyerror,
7029	.{ .load_frame = .{	7994	.{ .load_frame = .{
7030	.index = frame_addr.index,	7995	.index = frame_addr.index,
...	@@ -7249,7 +8214,7 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7249,7 +8214,7 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
7249	try self.spillEflagsIfOccupied();	8214	try self.spillEflagsIfOccupied();
7250	for (items, relocs, 0..) \|item, *reloc, i\| {	8215	for (items, relocs, 0..) \|item, *reloc, i\| {
7251	const item_mcv = try self.resolveInst(item);	8216	const item_mcv = try self.resolveInst(item);
7252	try self.genBinOpMir(.cmp, condition_ty, condition, item_mcv);	8217	try self.genBinOpMir(.{ ._, .cmp }, condition_ty, condition, item_mcv);
7253	reloc.* = try self.asmJccReloc(undefined, if (i < relocs.len - 1) .e else .ne);	8218	reloc.* = try self.asmJccReloc(undefined, if (i < relocs.len - 1) .e else .ne);
7254	}	8219	}
7255		8220
...	@@ -7289,14 +8254,14 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7289,14 +8254,14 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
7289	fn performReloc(self: *Self, reloc: Mir.Inst.Index) !void {	8254	fn performReloc(self: *Self, reloc: Mir.Inst.Index) !void {
7290	const next_inst = @intCast(u32, self.mir_instructions.len);	8255	const next_inst = @intCast(u32, self.mir_instructions.len);
7291	switch (self.mir_instructions.items(.tag)[reloc]) {	8256	switch (self.mir_instructions.items(.tag)[reloc]) {
7292	.jcc => {	8257	.j, .jmp => {},
7293	self.mir_instructions.items(.data)[reloc].inst_cc.inst = next_inst;	8258	.pseudo => switch (self.mir_instructions.items(.ops)[reloc]) {
7294	},	8259	.pseudo_j_z_and_np_inst, .pseudo_j_nz_or_p_inst => {},
7295	.jmp_reloc => {	8260	else => unreachable,
7296	self.mir_instructions.items(.data)[reloc].inst = next_inst;
7297	},	8261	},
7298	else => unreachable,	8262	else => unreachable,
7299	}	8263	}
		8264	self.mir_instructions.items(.data)[reloc].inst.inst = next_inst;
7300	}	8265	}
7301		8266
7302	fn airBr(self: *Self, inst: Air.Inst.Index) !void {	8267	fn airBr(self: *Self, inst: Air.Inst.Index) !void {
...	@@ -7460,7 +8425,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7460,7 +8425,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
7460	.qword	8425	.qword
7461	else	8426	else
7462	null;	8427	null;
7463	const mnem = mnem: {	8428	const mnem_tag = Mir.Inst.FixedTag{ ._, mnem: {
7464	if (mnem_size) \|_\| {	8429	if (mnem_size) \|_\| {
7465	if (std.meta.stringToEnum(Mir.Inst.Tag, mnem_str[0 .. mnem_str.len - 1])) \|mnem\| {	8430	if (std.meta.stringToEnum(Mir.Inst.Tag, mnem_str[0 .. mnem_str.len - 1])) \|mnem\| {
7466	break :mnem mnem;	8431	break :mnem mnem;
...	@@ -7468,7 +8433,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7468,7 +8433,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
7468	}	8433	}
7469	break :mnem std.meta.stringToEnum(Mir.Inst.Tag, mnem_str) orelse	8434	break :mnem std.meta.stringToEnum(Mir.Inst.Tag, mnem_str) orelse
7470	return self.fail("Invalid mnemonic: '{s}'", .{mnem_str});	8435	return self.fail("Invalid mnemonic: '{s}'", .{mnem_str});
7471	};	8436	} };
7472		8437
7473	var op_it = mem.tokenize(u8, mnem_it.rest(), ",");	8438	var op_it = mem.tokenize(u8, mnem_it.rest(), ",");
7474	var ops = [1]encoder.Instruction.Operand{.none} ** 4;	8439	var ops = [1]encoder.Instruction.Operand{.none} ** 4;
...	@@ -7519,51 +8484,51 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7519,51 +8484,51 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
7519	} else if (op_it.next()) \|op_str\| return self.fail("Extra operand: '{s}'", .{op_str});	8484	} else if (op_it.next()) \|op_str\| return self.fail("Extra operand: '{s}'", .{op_str});
7520		8485
7521	(switch (ops[0]) {	8486	(switch (ops[0]) {
7522	.none => self.asmOpOnly(mnem),	8487	.none => self.asmOpOnly(mnem_tag),
7523	.reg => \|reg0\| switch (ops[1]) {	8488	.reg => \|reg0\| switch (ops[1]) {
7524	.none => self.asmRegister(mnem, reg0),	8489	.none => self.asmRegister(mnem_tag, reg0),
7525	.reg => \|reg1\| switch (ops[2]) {	8490	.reg => \|reg1\| switch (ops[2]) {
7526	.none => self.asmRegisterRegister(mnem, reg1, reg0),	8491	.none => self.asmRegisterRegister(mnem_tag, reg1, reg0),
7527	.reg => \|reg2\| switch (ops[3]) {	8492	.reg => \|reg2\| switch (ops[3]) {
7528	.none => self.asmRegisterRegisterRegister(mnem, reg2, reg1, reg0),	8493	.none => self.asmRegisterRegisterRegister(mnem_tag, reg2, reg1, reg0),
7529	else => error.InvalidInstruction,	8494	else => error.InvalidInstruction,
7530	},	8495	},
7531	.mem => \|mem2\| switch (ops[3]) {	8496	.mem => \|mem2\| switch (ops[3]) {
7532	.none => self.asmMemoryRegisterRegister(mnem, mem2, reg1, reg0),	8497	.none => self.asmMemoryRegisterRegister(mnem_tag, mem2, reg1, reg0),
7533	else => error.InvalidInstruction,	8498	else => error.InvalidInstruction,
7534	},	8499	},
7535	else => error.InvalidInstruction,	8500	else => error.InvalidInstruction,
7536	},	8501	},
7537	.mem => \|mem1\| switch (ops[2]) {	8502	.mem => \|mem1\| switch (ops[2]) {
7538	.none => self.asmMemoryRegister(mnem, mem1, reg0),	8503	.none => self.asmMemoryRegister(mnem_tag, mem1, reg0),
7539	else => error.InvalidInstruction,	8504	else => error.InvalidInstruction,
7540	},	8505	},
7541	else => error.InvalidInstruction,	8506	else => error.InvalidInstruction,
7542	},	8507	},
7543	.mem => \|mem0\| switch (ops[1]) {	8508	.mem => \|mem0\| switch (ops[1]) {
7544	.none => self.asmMemory(mnem, mem0),	8509	.none => self.asmMemory(mnem_tag, mem0),
7545	.reg => \|reg1\| switch (ops[2]) {	8510	.reg => \|reg1\| switch (ops[2]) {
7546	.none => self.asmRegisterMemory(mnem, reg1, mem0),	8511	.none => self.asmRegisterMemory(mnem_tag, reg1, mem0),
7547	else => error.InvalidInstruction,	8512	else => error.InvalidInstruction,
7548	},	8513	},
7549	else => error.InvalidInstruction,	8514	else => error.InvalidInstruction,
7550	},	8515	},
7551	.imm => \|imm0\| switch (ops[1]) {	8516	.imm => \|imm0\| switch (ops[1]) {
7552	.none => self.asmImmediate(mnem, imm0),	8517	.none => self.asmImmediate(mnem_tag, imm0),
7553	.reg => \|reg1\| switch (ops[2]) {	8518	.reg => \|reg1\| switch (ops[2]) {
7554	.none => self.asmRegisterImmediate(mnem, reg1, imm0),	8519	.none => self.asmRegisterImmediate(mnem_tag, reg1, imm0),
7555	.reg => \|reg2\| switch (ops[3]) {	8520	.reg => \|reg2\| switch (ops[3]) {
7556	.none => self.asmRegisterRegisterImmediate(mnem, reg2, reg1, imm0),	8521	.none => self.asmRegisterRegisterImmediate(mnem_tag, reg2, reg1, imm0),
7557	else => error.InvalidInstruction,	8522	else => error.InvalidInstruction,
7558	},	8523	},
7559	.mem => \|mem2\| switch (ops[3]) {	8524	.mem => \|mem2\| switch (ops[3]) {
7560	.none => self.asmMemoryRegisterImmediate(mnem, mem2, reg1, imm0),	8525	.none => self.asmMemoryRegisterImmediate(mnem_tag, mem2, reg1, imm0),
7561	else => error.InvalidInstruction,	8526	else => error.InvalidInstruction,
7562	},	8527	},
7563	else => error.InvalidInstruction,	8528	else => error.InvalidInstruction,
7564	},	8529	},
7565	.mem => \|mem1\| switch (ops[2]) {	8530	.mem => \|mem1\| switch (ops[2]) {
7566	.none => self.asmMemoryImmediate(mnem, mem1, imm0),	8531	.none => self.asmMemoryImmediate(mnem_tag, mem1, imm0),
7567	else => error.InvalidInstruction,	8532	else => error.InvalidInstruction,
7568	},	8533	},
7569	else => error.InvalidInstruction,	8534	else => error.InvalidInstruction,
...	@@ -7572,7 +8537,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7572,7 +8537,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
7572	error.InvalidInstruction => return self.fail(	8537	error.InvalidInstruction => return self.fail(
7573	"Invalid instruction: '{s} {s} {s} {s} {s}'",	8538	"Invalid instruction: '{s} {s} {s} {s} {s}'",
7574	.{	8539	.{
7575	@tagName(mnem),	8540	@tagName(mnem_tag[1]),
7576	@tagName(ops[0]),	8541	@tagName(ops[0]),
7577	@tagName(ops[1]),	8542	@tagName(ops[1]),
7578	@tagName(ops[2]),	8543	@tagName(ops[2]),
...	@@ -7603,19 +8568,55 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7603,19 +8568,55 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
7603	return self.finishAirResult(inst, result);	8568	return self.finishAirResult(inst, result);
7604	}	8569	}
7605		8570
7606	fn movMirTag(self: *Self, ty: Type) !Mir.Inst.Tag {	8571	fn movMirTag(self: *Self, ty: Type, aligned: bool) !Mir.Inst.FixedTag {
7607	return switch (ty.zigTypeTag()) {	8572	switch (ty.zigTypeTag()) {
7608	else => .mov,	8573	else => return .{ ._, .mov },
7609	.Float => switch (ty.floatBits(self.target.*)) {	8574	.Float => switch (ty.floatBits(self.target.*)) {
7610	16 => unreachable, // needs special handling	8575	16 => unreachable, // needs special handling
7611	32 => .movss,	8576	32 => return if (self.hasFeature(.avx)) .{ .v_ss, .mov } else .{ ._ss, .mov },
7612	64 => .movsd,	8577	64 => return if (self.hasFeature(.avx)) .{ .v_sd, .mov } else .{ ._sd, .mov },
7613	128 => .movaps,	8578	128 => return if (self.hasFeature(.avx))
7614	else => return self.fail("TODO movMirTag from {}", .{	8579	if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
7615	ty.fmt(self.bin_file.options.module.?),	8580	else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu },
7616	}),	8581	else => {},
7617	},	8582	},
7618	};	8583	.Vector => switch (ty.childType().zigTypeTag()) {
		8584	.Float => switch (ty.childType().floatBits(self.target.*)) {
		8585	16 => switch (ty.vectorLen()) {
		8586	1 => unreachable, // needs special handling
		8587	2 => return if (self.hasFeature(.avx)) .{ .v_ss, .mov } else .{ ._ss, .mov },
		8588	3...4 => return if (self.hasFeature(.avx)) .{ .v_sd, .mov } else .{ ._sd, .mov },
		8589	5...8 => return if (self.hasFeature(.avx))
		8590	if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
		8591	else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu },
		8592	9...16 => if (self.hasFeature(.avx))
		8593	return if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu },
		8594	else => {},
		8595	},
		8596	32 => switch (ty.vectorLen()) {
		8597	1 => return if (self.hasFeature(.avx)) .{ .v_ss, .mov } else .{ ._ss, .mov },
		8598	2...4 => return if (self.hasFeature(.avx))
		8599	if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
		8600	else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu },
		8601	5...8 => if (self.hasFeature(.avx))
		8602	return if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu },
		8603	else => {},
		8604	},
		8605	64 => switch (ty.vectorLen()) {
		8606	1 => return if (self.hasFeature(.avx)) .{ .v_sd, .mov } else .{ ._sd, .mov },
		8607	2 => return if (self.hasFeature(.avx))
		8608	if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
		8609	else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu },
		8610	3...4 => if (self.hasFeature(.avx))
		8611	return if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu },
		8612	else => {},
		8613	},
		8614	else => {},
		8615	},
		8616	else => {},
		8617	},
		8618	}
		8619	return self.fail("TODO movMirTag for {}", .{ty.fmt(self.bin_file.options.module.?)});
7619	}	8620	}
7620		8621
7621	fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void {	8622	fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void {
...	@@ -7685,7 +8686,8 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError	...	@@ -7685,7 +8686,8 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError
7685		8686
7686	fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerError!void {	8687	fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerError!void {
7687	const abi_size = @intCast(u32, ty.abiSize(self.target.*));	8688	const abi_size = @intCast(u32, ty.abiSize(self.target.*));
7688	if (abi_size > 8) return self.fail("genSetReg called with a value larger than one register", .{});	8689	if (abi_size * 8 > dst_reg.bitSize())
		8690	return self.fail("genSetReg called with a value larger than dst_reg", .{});
7689	switch (src_mcv) {	8691	switch (src_mcv) {
7690	.none,	8692	.none,
7691	.unreach,	8693	.unreach,
...	@@ -7700,19 +8702,19 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr	...	@@ -7700,19 +8702,19 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
7700	if (imm == 0) {	8702	if (imm == 0) {
7701	// 32-bit moves zero-extend to 64-bit, so xoring the 32-bit	8703	// 32-bit moves zero-extend to 64-bit, so xoring the 32-bit
7702	// register is the fastest way to zero a register.	8704	// register is the fastest way to zero a register.
7703	try self.asmRegisterRegister(.xor, dst_reg.to32(), dst_reg.to32());	8705	try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32());
7704	} else if (abi_size > 4 and math.cast(u32, imm) != null) {	8706	} else if (abi_size > 4 and math.cast(u32, imm) != null) {
7705	// 32-bit moves zero-extend to 64-bit.	8707	// 32-bit moves zero-extend to 64-bit.
7706	try self.asmRegisterImmediate(.mov, dst_reg.to32(), Immediate.u(imm));	8708	try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), Immediate.u(imm));
7707	} else if (abi_size <= 4 and @bitCast(i64, imm) < 0) {	8709	} else if (abi_size <= 4 and @bitCast(i64, imm) < 0) {
7708	try self.asmRegisterImmediate(	8710	try self.asmRegisterImmediate(
7709	.mov,	8711	.{ ._, .mov },
7710	registerAlias(dst_reg, abi_size),	8712	registerAlias(dst_reg, abi_size),
7711	Immediate.s(@intCast(i32, @bitCast(i64, imm))),	8713	Immediate.s(@intCast(i32, @bitCast(i64, imm))),
7712	);	8714	);
7713	} else {	8715	} else {
7714	try self.asmRegisterImmediate(	8716	try self.asmRegisterImmediate(
7715	.mov,	8717	.{ ._, .mov },
7716	registerAlias(dst_reg, abi_size),	8718	registerAlias(dst_reg, abi_size),
7717	Immediate.u(imm),	8719	Immediate.u(imm),
7718	);	8720	);
...	@@ -7721,18 +8723,18 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr	...	@@ -7721,18 +8723,18 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
7721	.register => \|src_reg\| if (dst_reg.id() != src_reg.id()) try self.asmRegisterRegister(	8723	.register => \|src_reg\| if (dst_reg.id() != src_reg.id()) try self.asmRegisterRegister(
7722	if ((dst_reg.class() == .floating_point) == (src_reg.class() == .floating_point))	8724	if ((dst_reg.class() == .floating_point) == (src_reg.class() == .floating_point))
7723	switch (ty.zigTypeTag()) {	8725	switch (ty.zigTypeTag()) {
7724	else => .mov,	8726	else => .{ ._, .mov },
7725	.Float, .Vector => .movaps,	8727	.Float, .Vector => .{ ._ps, .mova },
7726	}	8728	}
7727	else switch (abi_size) {	8729	else switch (abi_size) {
7728	2 => return try self.asmRegisterRegisterImmediate(	8730	2 => return try self.asmRegisterRegisterImmediate(
7729	if (dst_reg.class() == .floating_point) .pinsrw else .pextrw,	8731	if (dst_reg.class() == .floating_point) .{ .p_w, .insr } else .{ .p_w, .extr },
7730	registerAlias(dst_reg, abi_size),	8732	registerAlias(dst_reg, 4),
7731	registerAlias(src_reg, abi_size),	8733	registerAlias(src_reg, 4),
7732	Immediate.u(0),	8734	Immediate.u(0),
7733	),	8735	),
7734	4 => .movd,	8736	4 => .{ ._d, .mov },
7735	8 => .movq,	8737	8 => .{ ._q, .mov },
7736	else => return self.fail(	8738	else => return self.fail(
7737	"unsupported register copy from {s} to {s}",	8739	"unsupported register copy from {s} to {s}",
7738	.{ @tagName(src_reg), @tagName(dst_reg) },	8740	.{ @tagName(src_reg), @tagName(dst_reg) },
...	@@ -7759,7 +8761,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr	...	@@ -7759,7 +8761,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
7759	});	8761	});
7760	if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)	8762	if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
7761	try self.asmRegisterMemoryImmediate(	8763	try self.asmRegisterMemoryImmediate(
7762	.pinsrw,	8764	.{ .p_w, .insr },
7763	registerAlias(dst_reg, abi_size),	8765	registerAlias(dst_reg, abi_size),
7764	src_mem,	8766	src_mem,
7765	Immediate.u(0),	8767	Immediate.u(0),
...	@@ -7769,10 +8771,14 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr	...	@@ -7769,10 +8771,14 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
7769	switch (src_mcv) {	8771	switch (src_mcv) {
7770	.register_offset => \|reg_off\| switch (reg_off.off) {	8772	.register_offset => \|reg_off\| switch (reg_off.off) {
7771	0 => return self.genSetReg(dst_reg, ty, .{ .register = reg_off.reg }),	8773	0 => return self.genSetReg(dst_reg, ty, .{ .register = reg_off.reg }),
7772	else => .lea,	8774	else => .{ ._, .lea },
7773	},	8775	},
7774	.indirect, .load_frame => try self.movMirTag(ty),	8776	.indirect => try self.movMirTag(ty, false),
7775	.lea_frame => .lea,	8777	.load_frame => \|frame_addr\| try self.movMirTag(
		8778	ty,
		8779	self.getFrameAddrAlignment(frame_addr) >= ty.abiAlignment(self.target.*),
		8780	),
		8781	.lea_frame => .{ ._, .lea },
7776	else => unreachable,	8782	else => unreachable,
7777	},	8783	},
7778	registerAlias(dst_reg, abi_size),	8784	registerAlias(dst_reg, abi_size),
...	@@ -7788,14 +8794,18 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr	...	@@ -7788,14 +8794,18 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
7788	});	8794	});
7789	return if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)	8795	return if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
7790	self.asmRegisterMemoryImmediate(	8796	self.asmRegisterMemoryImmediate(
7791	.pinsrw,	8797	.{ .p_w, .insr },
7792	registerAlias(dst_reg, abi_size),	8798	registerAlias(dst_reg, abi_size),
7793	src_mem,	8799	src_mem,
7794	Immediate.u(0),	8800	Immediate.u(0),
7795	)	8801	)
7796	else	8802	else
7797	self.asmRegisterMemory(	8803	self.asmRegisterMemory(
7798	try self.movMirTag(ty),	8804	try self.movMirTag(ty, mem.isAlignedGeneric(
		8805	u32,
		8806	@bitCast(u32, small_addr),
		8807	ty.abiAlignment(self.target.*),
		8808	)),
7799	registerAlias(dst_reg, abi_size),	8809	registerAlias(dst_reg, abi_size),
7800	src_mem,	8810	src_mem,
7801	);	8811	);
...	@@ -7803,13 +8813,15 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr	...	@@ -7803,13 +8813,15 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
7803	.load_direct => \|sym_index\| if (!ty.isRuntimeFloat()) {	8813	.load_direct => \|sym_index\| if (!ty.isRuntimeFloat()) {
7804	const atom_index = try self.owner.getSymbolIndex(self);	8814	const atom_index = try self.owner.getSymbolIndex(self);
7805	_ = try self.addInst(.{	8815	_ = try self.addInst(.{
7806	.tag = .mov_linker,	8816	.tag = .mov,
7807	.ops = .direct_reloc,	8817	.ops = .direct_reloc,
7808	.data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{	8818	.data = .{ .rx = .{
7809	.reg = @enumToInt(dst_reg.to64()),	8819	.r1 = dst_reg.to64(),
7810	.atom_index = atom_index,	8820	.payload = try self.addExtra(Mir.Reloc{
7811	.sym_index = sym_index,	8821	.atom_index = atom_index,
7812	}) },	8822	.sym_index = sym_index,
		8823	}),
		8824	} },
7813	});	8825	});
7814	return;	8826	return;
7815	},	8827	},
...	@@ -7826,14 +8838,14 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr	...	@@ -7826,14 +8838,14 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
7826	});	8838	});
7827	if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)	8839	if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
7828	try self.asmRegisterMemoryImmediate(	8840	try self.asmRegisterMemoryImmediate(
7829	.pinsrw,	8841	.{ .p_w, .insr },
7830	registerAlias(dst_reg, abi_size),	8842	registerAlias(dst_reg, abi_size),
7831	src_mem,	8843	src_mem,
7832	Immediate.u(0),	8844	Immediate.u(0),
7833	)	8845	)
7834	else	8846	else
7835	try self.asmRegisterMemory(	8847	try self.asmRegisterMemory(
7836	try self.movMirTag(ty),	8848	try self.movMirTag(ty, false),
7837	registerAlias(dst_reg, abi_size),	8849	registerAlias(dst_reg, abi_size),
7838	src_mem,	8850	src_mem,
7839	);	8851	);
...	@@ -7842,8 +8854,8 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr	...	@@ -7842,8 +8854,8 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
7842	const atom_index = try self.owner.getSymbolIndex(self);	8854	const atom_index = try self.owner.getSymbolIndex(self);
7843	_ = try self.addInst(.{	8855	_ = try self.addInst(.{
7844	.tag = switch (src_mcv) {	8856	.tag = switch (src_mcv) {
7845	.lea_direct => .lea_linker,	8857	.lea_direct => .lea,
7846	.lea_got => .mov_linker,	8858	.lea_got => .mov,
7847	else => unreachable,	8859	else => unreachable,
7848	},	8860	},
7849	.ops = switch (src_mcv) {	8861	.ops = switch (src_mcv) {
...	@@ -7851,27 +8863,31 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr	...	@@ -7851,27 +8863,31 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
7851	.lea_got => .got_reloc,	8863	.lea_got => .got_reloc,
7852	else => unreachable,	8864	else => unreachable,
7853	},	8865	},
7854	.data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{	8866	.data = .{ .rx = .{
7855	.reg = @enumToInt(dst_reg.to64()),	8867	.r1 = dst_reg.to64(),
7856	.atom_index = atom_index,	8868	.payload = try self.addExtra(Mir.Reloc{
7857	.sym_index = sym_index,	8869	.atom_index = atom_index,
7858	}) },	8870	.sym_index = sym_index,
		8871	}),
		8872	} },
7859	});	8873	});
7860	},	8874	},
7861	.lea_tlv => \|sym_index\| {	8875	.lea_tlv => \|sym_index\| {
7862	const atom_index = try self.owner.getSymbolIndex(self);	8876	const atom_index = try self.owner.getSymbolIndex(self);
7863	if (self.bin_file.cast(link.File.MachO)) \|_\| {	8877	if (self.bin_file.cast(link.File.MachO)) \|_\| {
7864	_ = try self.addInst(.{	8878	_ = try self.addInst(.{
7865	.tag = .lea_linker,	8879	.tag = .lea,
7866	.ops = .tlv_reloc,	8880	.ops = .tlv_reloc,
7867	.data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{	8881	.data = .{ .rx = .{
7868	.reg = @enumToInt(Register.rdi),	8882	.r1 = .rdi,
7869	.atom_index = atom_index,	8883	.payload = try self.addExtra(Mir.Reloc{
7870	.sym_index = sym_index,	8884	.atom_index = atom_index,
7871	}) },	8885	.sym_index = sym_index,
		8886	}),
		8887	} },
7872	});	8888	});
7873	// TODO: spill registers before calling	8889	// TODO: spill registers before calling
7874	try self.asmMemory(.call, Memory.sib(.qword, .{ .base = .{ .reg = .rdi } }));	8890	try self.asmMemory(.{ ._, .call }, Memory.sib(.qword, .{ .base = .{ .reg = .rdi } }));
7875	try self.genSetReg(dst_reg.to64(), Type.usize, .{ .register = .rax });	8891	try self.genSetReg(dst_reg.to64(), Type.usize, .{ .register = .rax });
7876	} else return self.fail("TODO emit ptr to TLV sequence on {s}", .{	8892	} else return self.fail("TODO emit ptr to TLV sequence on {s}", .{
7877	@tagName(self.bin_file.tag),	8893	@tagName(self.bin_file.tag),
...	@@ -7898,7 +8914,7 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal	...	@@ -7898,7 +8914,7 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
7898	else	8914	else
7899	Immediate.u(@intCast(u32, imm));	8915	Immediate.u(@intCast(u32, imm));
7900	try self.asmMemoryImmediate(	8916	try self.asmMemoryImmediate(
7901	.mov,	8917	.{ ._, .mov },
7902	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),	8918	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),
7903	immediate,	8919	immediate,
7904	);	8920	);
...	@@ -7906,14 +8922,14 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal	...	@@ -7906,14 +8922,14 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
7906	3, 5...7 => unreachable,	8922	3, 5...7 => unreachable,
7907	else => if (math.cast(i32, @bitCast(i64, imm))) \|small\| {	8923	else => if (math.cast(i32, @bitCast(i64, imm))) \|small\| {
7908	try self.asmMemoryImmediate(	8924	try self.asmMemoryImmediate(
7909	.mov,	8925	.{ ._, .mov },
7910	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),	8926	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),
7911	Immediate.s(small),	8927	Immediate.s(small),
7912	);	8928	);
7913	} else {	8929	} else {
7914	var offset: i32 = 0;	8930	var offset: i32 = 0;
7915	while (offset < abi_size) : (offset += 4) try self.asmMemoryImmediate(	8931	while (offset < abi_size) : (offset += 4) try self.asmMemoryImmediate(
7916	.mov,	8932	.{ ._, .mov },
7917	Memory.sib(.dword, .{ .base = base, .disp = disp + offset }),	8933	Memory.sib(.dword, .{ .base = base, .disp = disp + offset }),
7918	if (ty.isSignedInt())	8934	if (ty.isSignedInt())
7919	Immediate.s(@truncate(	8935	Immediate.s(@truncate(
...	@@ -7936,14 +8952,31 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal	...	@@ -7936,14 +8952,31 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
7936	);	8952	);
7937	if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)	8953	if (ty.isRuntimeFloat() and ty.floatBits(self.target.*) == 16)
7938	try self.asmMemoryRegisterImmediate(	8954	try self.asmMemoryRegisterImmediate(
7939	.pextrw,	8955	.{ .p_w, .extr },
7940	dst_mem,	8956	dst_mem,
7941	registerAlias(src_reg, abi_size),	8957	src_reg.to128(),
7942	Immediate.u(0),	8958	Immediate.u(0),
7943	)	8959	)
7944	else	8960	else
7945	try self.asmMemoryRegister(	8961	try self.asmMemoryRegister(
7946	try self.movMirTag(ty),	8962	try self.movMirTag(ty, switch (base) {
		8963	.none => mem.isAlignedGeneric(
		8964	u32,
		8965	@bitCast(u32, disp),
		8966	ty.abiAlignment(self.target.*),
		8967	),
		8968	.reg => \|reg\| switch (reg) {
		8969	.es, .cs, .ss, .ds => mem.isAlignedGeneric(
		8970	u32,
		8971	@bitCast(u32, disp),
		8972	ty.abiAlignment(self.target.*),
		8973	),
		8974	else => false,
		8975	},
		8976	.frame => \|frame_index\| self.getFrameAddrAlignment(
		8977	.{ .index = frame_index, .off = disp },
		8978	) >= ty.abiAlignment(self.target.*),
		8979	}),
7947	dst_mem,	8980	dst_mem,
7948	registerAlias(src_reg, abi_size),	8981	registerAlias(src_reg, abi_size),
7949	);	8982	);
...	@@ -8015,7 +9048,7 @@ fn genInlineMemcpyRegisterRegister(	...	@@ -8015,7 +9048,7 @@ fn genInlineMemcpyRegisterRegister(
8015	while (remainder > 0) {	9048	while (remainder > 0) {
8016	const nearest_power_of_two = @as(u6, 1) << math.log2_int(u3, @intCast(u3, remainder));	9049	const nearest_power_of_two = @as(u6, 1) << math.log2_int(u3, @intCast(u3, remainder));
8017	try self.asmMemoryRegister(	9050	try self.asmMemoryRegister(
8018	.mov,	9051	.{ ._, .mov },
8019	Memory.sib(Memory.PtrSize.fromSize(nearest_power_of_two), .{	9052	Memory.sib(Memory.PtrSize.fromSize(nearest_power_of_two), .{
8020	.base = dst_reg,	9053	.base = dst_reg,
8021	.disp = -next_offset,	9054	.disp = -next_offset,
...	@@ -8024,7 +9057,7 @@ fn genInlineMemcpyRegisterRegister(	...	@@ -8024,7 +9057,7 @@ fn genInlineMemcpyRegisterRegister(
8024	);	9057	);
8025		9058
8026	if (nearest_power_of_two > 1) {	9059	if (nearest_power_of_two > 1) {
8027	try self.genShiftBinOpMir(.shr, ty, .{ .register = tmp_reg }, .{	9060	try self.genShiftBinOpMir(.{ ._r, .sh }, ty, .{ .register = tmp_reg }, .{
8028	.immediate = nearest_power_of_two * 8,	9061	.immediate = nearest_power_of_two * 8,
8029	});	9062	});
8030	}	9063	}
...	@@ -8035,8 +9068,8 @@ fn genInlineMemcpyRegisterRegister(	...	@@ -8035,8 +9068,8 @@ fn genInlineMemcpyRegisterRegister(
8035	} else {	9068	} else {
8036	try self.asmMemoryRegister(	9069	try self.asmMemoryRegister(
8037	switch (src_reg.class()) {	9070	switch (src_reg.class()) {
8038	.general_purpose, .segment => .mov,	9071	.general_purpose, .segment => .{ ._, .mov },
8039	.floating_point => .movss,	9072	.floating_point => .{ ._ss, .mov },
8040	},	9073	},
8041	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = dst_reg, .disp = -offset }),	9074	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = dst_reg, .disp = -offset }),
8042	registerAlias(src_reg, abi_size),	9075	registerAlias(src_reg, abi_size),
...	@@ -8049,11 +9082,7 @@ fn genInlineMemcpy(self: *Self, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue	...	@@ -8049,11 +9082,7 @@ fn genInlineMemcpy(self: *Self, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue
8049	try self.genSetReg(.rdi, Type.usize, dst_ptr);	9082	try self.genSetReg(.rdi, Type.usize, dst_ptr);
8050	try self.genSetReg(.rsi, Type.usize, src_ptr);	9083	try self.genSetReg(.rsi, Type.usize, src_ptr);
8051	try self.genSetReg(.rcx, Type.usize, len);	9084	try self.genSetReg(.rcx, Type.usize, len);
8052	_ = try self.addInst(.{	9085	try self.asmOpOnly(.{ .@"rep _sb", .mov });
8053	.tag = .movs,
8054	.ops = .string,
8055	.data = .{ .string = .{ .repeat = .rep, .width = .b } },
8056	});
8057	}	9086	}
8058		9087
8059	fn genInlineMemset(self: *Self, dst_ptr: MCValue, value: MCValue, len: MCValue) InnerError!void {	9088	fn genInlineMemset(self: *Self, dst_ptr: MCValue, value: MCValue, len: MCValue) InnerError!void {
...	@@ -8061,11 +9090,7 @@ fn genInlineMemset(self: *Self, dst_ptr: MCValue, value: MCValue, len: MCValue)	...	@@ -8061,11 +9090,7 @@ fn genInlineMemset(self: *Self, dst_ptr: MCValue, value: MCValue, len: MCValue)
8061	try self.genSetReg(.rdi, Type.usize, dst_ptr);	9090	try self.genSetReg(.rdi, Type.usize, dst_ptr);
8062	try self.genSetReg(.al, Type.u8, value);	9091	try self.genSetReg(.al, Type.u8, value);
8063	try self.genSetReg(.rcx, Type.usize, len);	9092	try self.genSetReg(.rcx, Type.usize, len);
8064	_ = try self.addInst(.{	9093	try self.asmOpOnly(.{ .@"rep _sb", .sto });
8065	.tag = .stos,
8066	.ops = .string,
8067	.data = .{ .string = .{ .repeat = .rep, .width = .b } },
8068	});
8069	}	9094	}
8070		9095
8071	fn genLazySymbolRef(	9096	fn genLazySymbolRef(
...	@@ -8083,14 +9108,14 @@ fn genLazySymbolRef(	...	@@ -8083,14 +9108,14 @@ fn genLazySymbolRef(
8083	const got_mem =	9108	const got_mem =
8084	Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) });	9109	Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) });
8085	switch (tag) {	9110	switch (tag) {
8086	.lea, .mov => try self.asmRegisterMemory(.mov, reg.to64(), got_mem),	9111	.lea, .mov => try self.asmRegisterMemory(.{ ._, .mov }, reg.to64(), got_mem),
8087	.call => try self.asmMemory(.call, got_mem),	9112	.call => try self.asmMemory(.{ ._, .call }, got_mem),
8088	else => unreachable,	9113	else => unreachable,
8089	}	9114	}
8090	switch (tag) {	9115	switch (tag) {
8091	.lea, .call => {},	9116	.lea, .call => {},
8092	.mov => try self.asmRegisterMemory(	9117	.mov => try self.asmRegisterMemory(
8093	tag,	9118	.{ ._, tag },
8094	reg.to64(),	9119	reg.to64(),
8095	Memory.sib(.qword, .{ .base = .{ .reg = reg.to64() } }),	9120	Memory.sib(.qword, .{ .base = .{ .reg = reg.to64() } }),
8096	),	9121	),
...	@@ -8107,7 +9132,7 @@ fn genLazySymbolRef(	...	@@ -8107,7 +9132,7 @@ fn genLazySymbolRef(
8107	}	9132	}
8108	switch (tag) {	9133	switch (tag) {
8109	.lea, .mov => {},	9134	.lea, .mov => {},
8110	.call => try self.asmRegister(.call, reg),	9135	.call => try self.asmRegister(.{ ._, .call }, reg),
8111	else => unreachable,	9136	else => unreachable,
8112	}	9137	}
8113	} else if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {	9138	} else if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {
...	@@ -8121,7 +9146,7 @@ fn genLazySymbolRef(	...	@@ -8121,7 +9146,7 @@ fn genLazySymbolRef(
8121	}	9146	}
8122	switch (tag) {	9147	switch (tag) {
8123	.lea, .mov => {},	9148	.lea, .mov => {},
8124	.call => try self.asmRegister(.call, reg),	9149	.call => try self.asmRegister(.{ ._, .call }, reg),
8125	else => unreachable,	9150	else => unreachable,
8126	}	9151	}
8127	} else {	9152	} else {
...	@@ -8164,7 +9189,7 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -8164,7 +9189,7 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
8164	defer if (operand_lock) \|lock\| self.register_manager.unlockReg(lock);	9189	defer if (operand_lock) \|lock\| self.register_manager.unlockReg(lock);
8165		9190
8166	const dest = try self.allocRegOrMem(inst, true);	9191	const dest = try self.allocRegOrMem(inst, true);
8167	try self.genCopy(self.air.typeOfIndex(inst), dest, operand);	9192	try self.genCopy(if (!dest.isMemory() or operand.isMemory()) dst_ty else src_ty, dest, operand);
8168	break :result dest;	9193	break :result dest;
8169	};	9194	};
8170	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	9195	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
...	@@ -8226,13 +9251,13 @@ fn airIntToFloat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -8226,13 +9251,13 @@ fn airIntToFloat(self: *Self, inst: Air.Inst.Index) !void {
8226		9251
8227	try self.asmRegisterRegister(switch (dst_ty.floatBits(self.target.*)) {	9252	try self.asmRegisterRegister(switch (dst_ty.floatBits(self.target.*)) {
8228	32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse))	9253	32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse))
8229	.cvtsi2ss	9254	.{ ._, .cvtsi2ss }
8230	else	9255	else
8231	return self.fail("TODO implement airIntToFloat from {} to {} without sse", .{	9256	return self.fail("TODO implement airIntToFloat from {} to {} without sse", .{
8232	src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),	9257	src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
8233	}),	9258	}),
8234	64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2))	9259	64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2))
8235	.cvtsi2sd	9260	.{ ._, .cvtsi2sd }
8236	else	9261	else
8237	return self.fail("TODO implement airIntToFloat from {} to {} without sse2", .{	9262	return self.fail("TODO implement airIntToFloat from {} to {} without sse2", .{
8238	src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),	9263	src_ty.fmt(self.bin_file.options.module.?), dst_ty.fmt(self.bin_file.options.module.?),
...	@@ -8272,7 +9297,7 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -8272,7 +9297,7 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
8272	},	9297	},
8273	};	9298	};
8274	try self.asmMemory(	9299	try self.asmMemory(
8275	.fld,	9300	.{ .f_, .ld },
8276	Memory.sib(Memory.PtrSize.fromSize(src_abi_size), .{	9301	Memory.sib(Memory.PtrSize.fromSize(src_abi_size), .{
8277	.base = .{ .frame = frame_addr.index },	9302	.base = .{ .frame = frame_addr.index },
8278	.disp = frame_addr.off,	9303	.disp = frame_addr.off,
...	@@ -8282,7 +9307,7 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -8282,7 +9307,7 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
8282	// convert	9307	// convert
8283	const stack_dst = try self.allocRegOrMem(inst, false);	9308	const stack_dst = try self.allocRegOrMem(inst, false);
8284	try self.asmMemory(	9309	try self.asmMemory(
8285	.fisttp,	9310	.{ .f_p, .istt },
8286	Memory.sib(Memory.PtrSize.fromSize(dst_abi_size), .{	9311	Memory.sib(Memory.PtrSize.fromSize(dst_abi_size), .{
8287	.base = .{ .frame = stack_dst.load_frame.index },	9312	.base = .{ .frame = stack_dst.load_frame.index },
8288	.disp = stack_dst.load_frame.off,	9313	.disp = stack_dst.load_frame.off,
...	@@ -8338,16 +9363,11 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -8338,16 +9363,11 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
8338	defer if (ptr_lock) \|lock\| self.register_manager.unlockReg(lock);	9363	defer if (ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
8339		9364
8340	try self.spillEflagsIfOccupied();	9365	try self.spillEflagsIfOccupied();
8341	if (val_abi_size <= 8) {	9366	if (val_abi_size <= 8) try self.asmMemoryRegister(
8342	_ = try self.addInst(.{ .tag = .cmpxchg, .ops = .lock_mr_sib, .data = .{ .rx = .{	9367	.{ .@"lock _", .cmpxchg },
8343	.r = registerAlias(new_reg.?, val_abi_size),	9368	ptr_mem,
8344	.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),	9369	registerAlias(new_reg.?, val_abi_size),
8345	} } });	9370	) else try self.asmMemory(.{ .@"lock _16b", .cmpxchg }, ptr_mem);
8346	} else {
8347	_ = try self.addInst(.{ .tag = .cmpxchgb, .ops = .lock_m_sib, .data = .{
8348	.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
8349	} });
8350	}
8351		9371
8352	const result: MCValue = result: {	9372	const result: MCValue = result: {
8353	if (self.liveness.isUnused(inst)) break :result .unreach;	9373	if (self.liveness.isUnused(inst)) break :result .unreach;
...	@@ -8445,16 +9465,17 @@ fn atomicOp(	...	@@ -8445,16 +9465,17 @@ fn atomicOp(
8445		9465
8446	try self.genSetReg(dst_reg, val_ty, val_mcv);	9466	try self.genSetReg(dst_reg, val_ty, val_mcv);
8447	if (rmw_op == std.builtin.AtomicRmwOp.Sub and tag == .xadd) {	9467	if (rmw_op == std.builtin.AtomicRmwOp.Sub and tag == .xadd) {
8448	try self.genUnOpMir(.neg, val_ty, dst_mcv);	9468	try self.genUnOpMir(.{ ._, .neg }, val_ty, dst_mcv);
8449	}	9469	}
8450	_ = try self.addInst(.{ .tag = tag, .ops = switch (tag) {	9470	try self.asmMemoryRegister(
8451	.mov, .xchg => .mr_sib,	9471	switch (tag) {
8452	.xadd, .add, .sub, .@"and", .@"or", .xor => .lock_mr_sib,	9472	.mov, .xchg => .{ ._, tag },
8453	else => unreachable,	9473	.xadd, .add, .sub, .@"and", .@"or", .xor => .{ .@"lock _", tag },
8454	}, .data = .{ .rx = .{	9474	else => unreachable,
8455	.r = registerAlias(dst_reg, val_abi_size),	9475	},
8456	.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),	9476	ptr_mem,
8457	} } });	9477	registerAlias(dst_reg, val_abi_size),
		9478	);
8458		9479
8459	return if (unused) .unreach else dst_mcv;	9480	return if (unused) .unreach else dst_mcv;
8460	},	9481	},
...	@@ -8464,22 +9485,22 @@ fn atomicOp(	...	@@ -8464,22 +9485,22 @@ fn atomicOp(
8464	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);	9485	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
8465	defer self.register_manager.unlockReg(tmp_lock);	9486	defer self.register_manager.unlockReg(tmp_lock);
8466		9487
8467	try self.asmRegisterMemory(.mov, registerAlias(.rax, val_abi_size), ptr_mem);	9488	try self.asmRegisterMemory(.{ ._, .mov }, registerAlias(.rax, val_abi_size), ptr_mem);
8468	const loop = @intCast(u32, self.mir_instructions.len);	9489	const loop = @intCast(u32, self.mir_instructions.len);
8469	if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {	9490	if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
8470	try self.genSetReg(tmp_reg, val_ty, .{ .register = .rax });	9491	try self.genSetReg(tmp_reg, val_ty, .{ .register = .rax });
8471	}	9492	}
8472	if (rmw_op) \|op\| switch (op) {	9493	if (rmw_op) \|op\| switch (op) {
8473	.Xchg => try self.genSetReg(tmp_reg, val_ty, val_mcv),	9494	.Xchg => try self.genSetReg(tmp_reg, val_ty, val_mcv),
8474	.Add => try self.genBinOpMir(.add, val_ty, tmp_mcv, val_mcv),	9495	.Add => try self.genBinOpMir(.{ ._, .add }, val_ty, tmp_mcv, val_mcv),
8475	.Sub => try self.genBinOpMir(.sub, val_ty, tmp_mcv, val_mcv),	9496	.Sub => try self.genBinOpMir(.{ ._, .sub }, val_ty, tmp_mcv, val_mcv),
8476	.And => try self.genBinOpMir(.@"and", val_ty, tmp_mcv, val_mcv),	9497	.And => try self.genBinOpMir(.{ ._, .@"and" }, val_ty, tmp_mcv, val_mcv),
8477	.Nand => {	9498	.Nand => {
8478	try self.genBinOpMir(.@"and", val_ty, tmp_mcv, val_mcv);	9499	try self.genBinOpMir(.{ ._, .@"and" }, val_ty, tmp_mcv, val_mcv);
8479	try self.genUnOpMir(.not, val_ty, tmp_mcv);	9500	try self.genUnOpMir(.{ ._, .not }, val_ty, tmp_mcv);
8480	},	9501	},
8481	.Or => try self.genBinOpMir(.@"or", val_ty, tmp_mcv, val_mcv),	9502	.Or => try self.genBinOpMir(.{ ._, .@"or" }, val_ty, tmp_mcv, val_mcv),
8482	.Xor => try self.genBinOpMir(.xor, val_ty, tmp_mcv, val_mcv),	9503	.Xor => try self.genBinOpMir(.{ ._, .xor }, val_ty, tmp_mcv, val_mcv),
8483	.Min, .Max => {	9504	.Min, .Max => {
8484	const cc: Condition = switch (if (val_ty.isAbiInt())	9505	const cc: Condition = switch (if (val_ty.isAbiInt())
8485	val_ty.intInfo(self.target.*).signedness	9506	val_ty.intInfo(self.target.*).signedness
...	@@ -8497,7 +9518,7 @@ fn atomicOp(	...	@@ -8497,7 +9518,7 @@ fn atomicOp(
8497	},	9518	},
8498	};	9519	};
8499		9520
8500	try self.genBinOpMir(.cmp, val_ty, tmp_mcv, val_mcv);	9521	try self.genBinOpMir(.{ ._, .cmp }, val_ty, tmp_mcv, val_mcv);
8501	const cmov_abi_size = @max(val_abi_size, 2);	9522	const cmov_abi_size = @max(val_abi_size, 2);
8502	switch (val_mcv) {	9523	switch (val_mcv) {
8503	.register => \|val_reg\| try self.asmCmovccRegisterRegister(	9524	.register => \|val_reg\| try self.asmCmovccRegisterRegister(
...	@@ -8521,19 +9542,20 @@ fn atomicOp(	...	@@ -8521,19 +9542,20 @@ fn atomicOp(
8521	}	9542	}
8522	},	9543	},
8523	};	9544	};
8524	_ = try self.addInst(.{ .tag = .cmpxchg, .ops = .lock_mr_sib, .data = .{ .rx = .{	9545	try self.asmMemoryRegister(
8525	.r = registerAlias(tmp_reg, val_abi_size),	9546	.{ .@"lock _", .cmpxchg },
8526	.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),	9547	ptr_mem,
8527	} } });	9548	registerAlias(tmp_reg, val_abi_size),
		9549	);
8528	_ = try self.asmJccReloc(loop, .ne);	9550	_ = try self.asmJccReloc(loop, .ne);
8529	return if (unused) .unreach else .{ .register = .rax };	9551	return if (unused) .unreach else .{ .register = .rax };
8530	} else {	9552	} else {
8531	try self.asmRegisterMemory(.mov, .rax, Memory.sib(.qword, .{	9553	try self.asmRegisterMemory(.{ ._, .mov }, .rax, Memory.sib(.qword, .{
8532	.base = ptr_mem.sib.base,	9554	.base = ptr_mem.sib.base,
8533	.scale_index = ptr_mem.scaleIndex(),	9555	.scale_index = ptr_mem.scaleIndex(),
8534	.disp = ptr_mem.sib.disp + 0,	9556	.disp = ptr_mem.sib.disp + 0,
8535	}));	9557	}));
8536	try self.asmRegisterMemory(.mov, .rdx, Memory.sib(.qword, .{	9558	try self.asmRegisterMemory(.{ ._, .mov }, .rdx, Memory.sib(.qword, .{
8537	.base = ptr_mem.sib.base,	9559	.base = ptr_mem.sib.base,
8538	.scale_index = ptr_mem.scaleIndex(),	9560	.scale_index = ptr_mem.scaleIndex(),
8539	.disp = ptr_mem.sib.disp + 8,	9561	.disp = ptr_mem.sib.disp + 8,
...	@@ -8548,53 +9570,51 @@ fn atomicOp(	...	@@ -8548,53 +9570,51 @@ fn atomicOp(
8548	const val_lo_mem = val_mem_mcv.mem(.qword);	9570	const val_lo_mem = val_mem_mcv.mem(.qword);
8549	const val_hi_mem = val_mem_mcv.address().offset(8).deref().mem(.qword);	9571	const val_hi_mem = val_mem_mcv.address().offset(8).deref().mem(.qword);
8550	if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {	9572	if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
8551	try self.asmRegisterRegister(.mov, .rbx, .rax);	9573	try self.asmRegisterRegister(.{ ._, .mov }, .rbx, .rax);
8552	try self.asmRegisterRegister(.mov, .rcx, .rdx);	9574	try self.asmRegisterRegister(.{ ._, .mov }, .rcx, .rdx);
8553	}	9575	}
8554	if (rmw_op) \|op\| switch (op) {	9576	if (rmw_op) \|op\| switch (op) {
8555	.Xchg => {	9577	.Xchg => {
8556	try self.asmRegisterMemory(.mov, .rbx, val_lo_mem);	9578	try self.asmRegisterMemory(.{ ._, .mov }, .rbx, val_lo_mem);
8557	try self.asmRegisterMemory(.mov, .rcx, val_hi_mem);	9579	try self.asmRegisterMemory(.{ ._, .mov }, .rcx, val_hi_mem);
8558	},	9580	},
8559	.Add => {	9581	.Add => {
8560	try self.asmRegisterMemory(.add, .rbx, val_lo_mem);	9582	try self.asmRegisterMemory(.{ ._, .add }, .rbx, val_lo_mem);
8561	try self.asmRegisterMemory(.adc, .rcx, val_hi_mem);	9583	try self.asmRegisterMemory(.{ ._, .adc }, .rcx, val_hi_mem);
8562	},	9584	},
8563	.Sub => {	9585	.Sub => {
8564	try self.asmRegisterMemory(.sub, .rbx, val_lo_mem);	9586	try self.asmRegisterMemory(.{ ._, .sub }, .rbx, val_lo_mem);
8565	try self.asmRegisterMemory(.sbb, .rcx, val_hi_mem);	9587	try self.asmRegisterMemory(.{ ._, .sbb }, .rcx, val_hi_mem);
8566	},	9588	},
8567	.And => {	9589	.And => {
8568	try self.asmRegisterMemory(.@"and", .rbx, val_lo_mem);	9590	try self.asmRegisterMemory(.{ ._, .@"and" }, .rbx, val_lo_mem);
8569	try self.asmRegisterMemory(.@"and", .rcx, val_hi_mem);	9591	try self.asmRegisterMemory(.{ ._, .@"and" }, .rcx, val_hi_mem);
8570	},	9592	},
8571	.Nand => {	9593	.Nand => {
8572	try self.asmRegisterMemory(.@"and", .rbx, val_lo_mem);	9594	try self.asmRegisterMemory(.{ ._, .@"and" }, .rbx, val_lo_mem);
8573	try self.asmRegisterMemory(.@"and", .rcx, val_hi_mem);	9595	try self.asmRegisterMemory(.{ ._, .@"and" }, .rcx, val_hi_mem);
8574	try self.asmRegister(.not, .rbx);	9596	try self.asmRegister(.{ ._, .not }, .rbx);
8575	try self.asmRegister(.not, .rcx);	9597	try self.asmRegister(.{ ._, .not }, .rcx);
8576	},	9598	},
8577	.Or => {	9599	.Or => {
8578	try self.asmRegisterMemory(.@"or", .rbx, val_lo_mem);	9600	try self.asmRegisterMemory(.{ ._, .@"or" }, .rbx, val_lo_mem);
8579	try self.asmRegisterMemory(.@"or", .rcx, val_hi_mem);	9601	try self.asmRegisterMemory(.{ ._, .@"or" }, .rcx, val_hi_mem);
8580	},	9602	},
8581	.Xor => {	9603	.Xor => {
8582	try self.asmRegisterMemory(.xor, .rbx, val_lo_mem);	9604	try self.asmRegisterMemory(.{ ._, .xor }, .rbx, val_lo_mem);
8583	try self.asmRegisterMemory(.xor, .rcx, val_hi_mem);	9605	try self.asmRegisterMemory(.{ ._, .xor }, .rcx, val_hi_mem);
8584	},	9606	},
8585	else => return self.fail("TODO implement x86 atomic loop for {} {s}", .{	9607	else => return self.fail("TODO implement x86 atomic loop for {} {s}", .{
8586	val_ty.fmt(self.bin_file.options.module.?), @tagName(op),	9608	val_ty.fmt(self.bin_file.options.module.?), @tagName(op),
8587	}),	9609	}),
8588	};	9610	};
8589	_ = try self.addInst(.{ .tag = .cmpxchgb, .ops = .lock_m_sib, .data = .{	9611	try self.asmMemory(.{ .@"lock _16b", .cmpxchg }, ptr_mem);
8590	.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
8591	} });
8592	_ = try self.asmJccReloc(loop, .ne);	9612	_ = try self.asmJccReloc(loop, .ne);
8593		9613
8594	if (unused) return .unreach;	9614	if (unused) return .unreach;
8595	const dst_mcv = try self.allocTempRegOrMem(val_ty, false);	9615	const dst_mcv = try self.allocTempRegOrMem(val_ty, false);
8596	try self.asmMemoryRegister(	9616	try self.asmMemoryRegister(
8597	.mov,	9617	.{ ._, .mov },
8598	Memory.sib(.qword, .{	9618	Memory.sib(.qword, .{
8599	.base = .{ .frame = dst_mcv.load_frame.index },	9619	.base = .{ .frame = dst_mcv.load_frame.index },
8600	.disp = dst_mcv.load_frame.off + 0,	9620	.disp = dst_mcv.load_frame.off + 0,
...	@@ -8602,7 +9622,7 @@ fn atomicOp(	...	@@ -8602,7 +9622,7 @@ fn atomicOp(
8602	.rax,	9622	.rax,
8603	);	9623	);
8604	try self.asmMemoryRegister(	9624	try self.asmMemoryRegister(
8605	.mov,	9625	.{ ._, .mov },
8606	Memory.sib(.qword, .{	9626	Memory.sib(.qword, .{
8607	.base = .{ .frame = dst_mcv.load_frame.index },	9627	.base = .{ .frame = dst_mcv.load_frame.index },
8608	.disp = dst_mcv.load_frame.off + 8,	9628	.disp = dst_mcv.load_frame.off + 8,
...	@@ -8754,8 +9774,13 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {	...	@@ -8754,8 +9774,13 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
8754	.off = elem_abi_size,	9774	.off = elem_abi_size,
8755	} });	9775	} });
8756		9776
8757	try self.genBinOpMir(.sub, Type.usize, len_mcv, .{ .immediate = 1 });	9777	try self.genBinOpMir(.{ ._, .sub }, Type.usize, len_mcv, .{ .immediate = 1 });
8758	try self.asmRegisterRegisterImmediate(.imul, len_reg, len_reg, Immediate.u(elem_abi_size));	9778	try self.asmRegisterRegisterImmediate(
		9779	.{ .i_, .mul },
		9780	len_reg,
		9781	len_reg,
		9782	Immediate.u(elem_abi_size),
		9783	);
8759	try self.genInlineMemcpy(second_elem_ptr_mcv, ptr, len_mcv);	9784	try self.genInlineMemcpy(second_elem_ptr_mcv, ptr, len_mcv);
8760		9785
8761	try self.performReloc(skip_reloc);	9786	try self.performReloc(skip_reloc);
...	@@ -8893,7 +9918,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -8893,7 +9918,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
8893	try self.truncateRegister(err_ty, err_reg.to32());	9918	try self.truncateRegister(err_ty, err_reg.to32());
8894		9919
8895	try self.asmRegisterMemory(	9920	try self.asmRegisterMemory(
8896	.mov,	9921	.{ ._, .mov },
8897	start_reg.to32(),	9922	start_reg.to32(),
8898	Memory.sib(.dword, .{	9923	Memory.sib(.dword, .{
8899	.base = .{ .reg = addr_reg.to64() },	9924	.base = .{ .reg = addr_reg.to64() },
...	@@ -8902,7 +9927,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -8902,7 +9927,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
8902	}),	9927	}),
8903	);	9928	);
8904	try self.asmRegisterMemory(	9929	try self.asmRegisterMemory(
8905	.mov,	9930	.{ ._, .mov },
8906	end_reg.to32(),	9931	end_reg.to32(),
8907	Memory.sib(.dword, .{	9932	Memory.sib(.dword, .{
8908	.base = .{ .reg = addr_reg.to64() },	9933	.base = .{ .reg = addr_reg.to64() },
...	@@ -8910,9 +9935,9 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -8910,9 +9935,9 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
8910	.disp = 8,	9935	.disp = 8,
8911	}),	9936	}),
8912	);	9937	);
8913	try self.asmRegisterRegister(.sub, end_reg.to32(), start_reg.to32());	9938	try self.asmRegisterRegister(.{ ._, .sub }, end_reg.to32(), start_reg.to32());
8914	try self.asmRegisterMemory(	9939	try self.asmRegisterMemory(
8915	.lea,	9940	.{ ._, .lea },
8916	start_reg.to64(),	9941	start_reg.to64(),
8917	Memory.sib(.byte, .{	9942	Memory.sib(.byte, .{
8918	.base = .{ .reg = addr_reg.to64() },	9943	.base = .{ .reg = addr_reg.to64() },
...	@@ -8921,7 +9946,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -8921,7 +9946,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
8921	}),	9946	}),
8922	);	9947	);
8923	try self.asmRegisterMemory(	9948	try self.asmRegisterMemory(
8924	.lea,	9949	.{ ._, .lea },
8925	end_reg.to32(),	9950	end_reg.to32(),
8926	Memory.sib(.byte, .{	9951	Memory.sib(.byte, .{
8927	.base = .{ .reg = end_reg.to64() },	9952	.base = .{ .reg = end_reg.to64() },
...	@@ -8931,7 +9956,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -8931,7 +9956,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
8931		9956
8932	const dst_mcv = try self.allocRegOrMem(inst, false);	9957	const dst_mcv = try self.allocRegOrMem(inst, false);
8933	try self.asmMemoryRegister(	9958	try self.asmMemoryRegister(
8934	.mov,	9959	.{ ._, .mov },
8935	Memory.sib(.qword, .{	9960	Memory.sib(.qword, .{
8936	.base = .{ .frame = dst_mcv.load_frame.index },	9961	.base = .{ .frame = dst_mcv.load_frame.index },
8937	.disp = dst_mcv.load_frame.off,	9962	.disp = dst_mcv.load_frame.off,
...	@@ -8939,7 +9964,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -8939,7 +9964,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
8939	start_reg.to64(),	9964	start_reg.to64(),
8940	);	9965	);
8941	try self.asmMemoryRegister(	9966	try self.asmMemoryRegister(
8942	.mov,	9967	.{ ._, .mov },
8943	Memory.sib(.qword, .{	9968	Memory.sib(.qword, .{
8944	.base = .{ .frame = dst_mcv.load_frame.index },	9969	.base = .{ .frame = dst_mcv.load_frame.index },
8945	.disp = dst_mcv.load_frame.off + 8,	9970	.disp = dst_mcv.load_frame.off + 8,
...	@@ -9035,13 +10060,13 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -9035,13 +10060,13 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
9035	try self.truncateRegister(elem_ty, elem_reg);	10060	try self.truncateRegister(elem_ty, elem_reg);
9036	}	10061	}
9037	if (elem_bit_off > 0) try self.genShiftBinOpMir(	10062	if (elem_bit_off > 0) try self.genShiftBinOpMir(
9038	.shl,	10063	.{ ._l, .sh },
9039	elem_ty,	10064	elem_ty,
9040	.{ .register = elem_reg },	10065	.{ .register = elem_reg },
9041	.{ .immediate = elem_bit_off },	10066	.{ .immediate = elem_bit_off },
9042	);	10067	);
9043	try self.genBinOpMir(	10068	try self.genBinOpMir(
9044	.@"or",	10069	.{ ._, .@"or" },
9045	elem_ty,	10070	elem_ty,
9046	.{ .load_frame = .{ .index = frame_index, .off = elem_byte_off } },	10071	.{ .load_frame = .{ .index = frame_index, .off = elem_byte_off } },
9047	.{ .register = elem_reg },	10072	.{ .register = elem_reg },
...	@@ -9052,13 +10077,13 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -9052,13 +10077,13 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
9052	try self.truncateRegister(elem_ty, registerAlias(reg, elem_abi_size));	10077	try self.truncateRegister(elem_ty, registerAlias(reg, elem_abi_size));
9053	}	10078	}
9054	try self.genShiftBinOpMir(	10079	try self.genShiftBinOpMir(
9055	.shr,	10080	.{ ._r, .sh },
9056	elem_ty,	10081	elem_ty,
9057	.{ .register = reg },	10082	.{ .register = reg },
9058	.{ .immediate = elem_abi_bits - elem_bit_off },	10083	.{ .immediate = elem_abi_bits - elem_bit_off },
9059	);	10084	);
9060	try self.genBinOpMir(	10085	try self.genBinOpMir(
9061	.@"or",	10086	.{ ._, .@"or" },
9062	elem_ty,	10087	elem_ty,
9063	.{ .load_frame = .{	10088	.{ .load_frame = .{
9064	.index = frame_index,	10089	.index = frame_index,
...	@@ -9130,9 +10155,150 @@ fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -9130,9 +10155,150 @@ fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void {
9130	fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {	10155	fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
9131	const pl_op = self.air.instructions.items(.data)[inst].pl_op;	10156	const pl_op = self.air.instructions.items(.data)[inst].pl_op;
9132	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;	10157	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
9133	_ = extra;	10158	const ty = self.air.typeOfIndex(inst);
9134	return self.fail("TODO implement airMulAdd for x86_64", .{});	10159
9135	//return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, pl_op.operand });	10160	if (!self.hasFeature(.fma)) return self.fail("TODO implement airMulAdd for {}", .{
		10161	ty.fmt(self.bin_file.options.module.?),
		10162	});
		10163
		10164	const ops = [3]Air.Inst.Ref{ extra.lhs, extra.rhs, pl_op.operand };
		10165	var mcvs: [3]MCValue = undefined;
		10166	var locks = [1]?RegisterManager.RegisterLock{null} ** 3;
		10167	defer for (locks) \|reg_lock\| if (reg_lock) \|lock\| self.register_manager.unlockReg(lock);
		10168	var order = [1]u2{0} ** 3;
		10169	var unused = std.StaticBitSet(3).initFull();
		10170	for (ops, &mcvs, &locks, 0..) \|op, mcv, lock, op_i\| {
		10171	const op_index = @intCast(u2, op_i);
		10172	mcv.* = try self.resolveInst(op);
		10173	if (unused.isSet(0) and mcv.isRegister() and self.reuseOperand(inst, op, op_index, mcv.*)) {
		10174	order[op_index] = 1;
		10175	unused.unset(0);
		10176	} else if (unused.isSet(2) and mcv.isMemory()) {
		10177	order[op_index] = 3;
		10178	unused.unset(2);
		10179	}
		10180	switch (mcv.*) {
		10181	.register => \|reg\| lock.* = self.register_manager.lockReg(reg),
		10182	else => {},
		10183	}
		10184	}
		10185	for (&order, &mcvs, &locks) \|mop_index, mcv, *lock\| {
		10186	if (mop_index.* != 0) continue;
		10187	mop_index.* = 1 + @intCast(u2, unused.toggleFirstSet().?);
		10188	if (mop_index.* > 1 and mcv.isRegister()) continue;
		10189	const reg = try self.copyToTmpRegister(ty, mcv.*);
		10190	mcv.* = .{ .register = reg };
		10191	if (lock.*) \|old_lock\| self.register_manager.unlockReg(old_lock);
		10192	lock.* = self.register_manager.lockRegAssumeUnused(reg);
		10193	}
		10194
		10195	const mir_tag = if (@as(
		10196	?Mir.Inst.FixedTag,
		10197	if (mem.eql(u2, &order, &.{ 1, 3, 2 }) or mem.eql(u2, &order, &.{ 3, 1, 2 }))
		10198	switch (ty.zigTypeTag()) {
		10199	.Float => switch (ty.floatBits(self.target.*)) {
		10200	32 => .{ .v_ss, .fmadd132 },
		10201	64 => .{ .v_sd, .fmadd132 },
		10202	16, 80, 128 => null,
		10203	else => unreachable,
		10204	},
		10205	.Vector => switch (ty.childType().zigTypeTag()) {
		10206	.Float => switch (ty.childType().floatBits(self.target.*)) {
		10207	32 => switch (ty.vectorLen()) {
		10208	1 => .{ .v_ss, .fmadd132 },
		10209	2...8 => .{ .v_ps, .fmadd132 },
		10210	else => null,
		10211	},
		10212	64 => switch (ty.vectorLen()) {
		10213	1 => .{ .v_sd, .fmadd132 },
		10214	2...4 => .{ .v_pd, .fmadd132 },
		10215	else => null,
		10216	},
		10217	16, 80, 128 => null,
		10218	else => unreachable,
		10219	},
		10220	else => unreachable,
		10221	},
		10222	else => unreachable,
		10223	}
		10224	else if (mem.eql(u2, &order, &.{ 2, 1, 3 }) or mem.eql(u2, &order, &.{ 1, 2, 3 }))
		10225	switch (ty.zigTypeTag()) {
		10226	.Float => switch (ty.floatBits(self.target.*)) {
		10227	32 => .{ .v_ss, .fmadd213 },
		10228	64 => .{ .v_sd, .fmadd213 },
		10229	16, 80, 128 => null,
		10230	else => unreachable,
		10231	},
		10232	.Vector => switch (ty.childType().zigTypeTag()) {
		10233	.Float => switch (ty.childType().floatBits(self.target.*)) {
		10234	32 => switch (ty.vectorLen()) {
		10235	1 => .{ .v_ss, .fmadd213 },
		10236	2...8 => .{ .v_ps, .fmadd213 },
		10237	else => null,
		10238	},
		10239	64 => switch (ty.vectorLen()) {
		10240	1 => .{ .v_sd, .fmadd213 },
		10241	2...4 => .{ .v_pd, .fmadd213 },
		10242	else => null,
		10243	},
		10244	16, 80, 128 => null,
		10245	else => unreachable,
		10246	},
		10247	else => unreachable,
		10248	},
		10249	else => unreachable,
		10250	}
		10251	else if (mem.eql(u2, &order, &.{ 2, 3, 1 }) or mem.eql(u2, &order, &.{ 3, 2, 1 }))
		10252	switch (ty.zigTypeTag()) {
		10253	.Float => switch (ty.floatBits(self.target.*)) {
		10254	32 => .{ .v_ss, .fmadd231 },
		10255	64 => .{ .v_sd, .fmadd231 },
		10256	16, 80, 128 => null,
		10257	else => unreachable,
		10258	},
		10259	.Vector => switch (ty.childType().zigTypeTag()) {
		10260	.Float => switch (ty.childType().floatBits(self.target.*)) {
		10261	32 => switch (ty.vectorLen()) {
		10262	1 => .{ .v_ss, .fmadd231 },
		10263	2...8 => .{ .v_ps, .fmadd231 },
		10264	else => null,
		10265	},
		10266	64 => switch (ty.vectorLen()) {
		10267	1 => .{ .v_sd, .fmadd231 },
		10268	2...4 => .{ .v_pd, .fmadd231 },
		10269	else => null,
		10270	},
		10271	16, 80, 128 => null,
		10272	else => unreachable,
		10273	},
		10274	else => unreachable,
		10275	},
		10276	else => unreachable,
		10277	}
		10278	else
		10279	unreachable,
		10280	)) \|tag\| tag else return self.fail("TODO implement airMulAdd for {}", .{
		10281	ty.fmt(self.bin_file.options.module.?),
		10282	});
		10283
		10284	var mops: [3]MCValue = undefined;
		10285	for (order, mcvs) \|mop_index, mcv\| mops[mop_index - 1] = mcv;
		10286
		10287	const abi_size = @intCast(u32, ty.abiSize(self.target.*));
		10288	const mop1_reg = registerAlias(mops[0].getReg().?, abi_size);
		10289	const mop2_reg = registerAlias(mops[1].getReg().?, abi_size);
		10290	if (mops[2].isRegister()) try self.asmRegisterRegisterRegister(
		10291	mir_tag,
		10292	mop1_reg,
		10293	mop2_reg,
		10294	registerAlias(mops[2].getReg().?, abi_size),
		10295	) else try self.asmRegisterRegisterMemory(
		10296	mir_tag,
		10297	mop1_reg,
		10298	mop2_reg,
		10299	mops[2].mem(Memory.PtrSize.fromSize(abi_size)),
		10300	);
		10301	return self.finishAir(inst, mops[0], ops);
9136	}	10302	}
9137		10303
9138	fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {	10304	fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {
...	@@ -9471,17 +10637,37 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {	...	@@ -9471,17 +10637,37 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
9471	switch (int_info.signedness) {	10637	switch (int_info.signedness) {
9472	.signed => {	10638	.signed => {
9473	const shift = @intCast(u6, max_reg_bit_width - int_info.bits);	10639	const shift = @intCast(u6, max_reg_bit_width - int_info.bits);
9474	try self.genShiftBinOpMir(.sal, Type.isize, .{ .register = reg }, .{ .immediate = shift });	10640	try self.genShiftBinOpMir(
9475	try self.genShiftBinOpMir(.sar, Type.isize, .{ .register = reg }, .{ .immediate = shift });	10641	.{ ._l, .sa },
		10642	Type.isize,
		10643	.{ .register = reg },
		10644	.{ .immediate = shift },
		10645	);
		10646	try self.genShiftBinOpMir(
		10647	.{ ._r, .sa },
		10648	Type.isize,
		10649	.{ .register = reg },
		10650	.{ .immediate = shift },
		10651	);
9476	},	10652	},
9477	.unsigned => {	10653	.unsigned => {
9478	const shift = @intCast(u6, max_reg_bit_width - int_info.bits);	10654	const shift = @intCast(u6, max_reg_bit_width - int_info.bits);
9479	const mask = (~@as(u64, 0)) >> shift;	10655	const mask = (~@as(u64, 0)) >> shift;
9480	if (int_info.bits <= 32) {	10656	if (int_info.bits <= 32) {
9481	try self.genBinOpMir(.@"and", Type.u32, .{ .register = reg }, .{ .immediate = mask });	10657	try self.genBinOpMir(
		10658	.{ ._, .@"and" },
		10659	Type.u32,
		10660	.{ .register = reg },
		10661	.{ .immediate = mask },
		10662	);
9482	} else {	10663	} else {
9483	const tmp_reg = try self.copyToTmpRegister(Type.usize, .{ .immediate = mask });	10664	const tmp_reg = try self.copyToTmpRegister(Type.usize, .{ .immediate = mask });
9484	try self.genBinOpMir(.@"and", Type.usize, .{ .register = reg }, .{ .register = tmp_reg });	10665	try self.genBinOpMir(
		10666	.{ ._, .@"and" },
		10667	Type.usize,
		10668	.{ .register = reg },
		10669	.{ .register = tmp_reg },
		10670	);
9485	}	10671	}
9486	},	10672	},
9487	}	10673	}
...	@@ -9508,3 +10694,13 @@ fn regBitSize(self: *Self, ty: Type) u64 {	...	@@ -9508,3 +10694,13 @@ fn regBitSize(self: *Self, ty: Type) u64 {
9508	fn regExtraBits(self: *Self, ty: Type) u64 {	10694	fn regExtraBits(self: *Self, ty: Type) u64 {
9509	return self.regBitSize(ty) - ty.bitSize(self.target.*);	10695	return self.regBitSize(ty) - ty.bitSize(self.target.*);
9510	}	10696	}
		10697
		10698	fn hasFeature(self: *Self, feature: Target.x86.Feature) bool {
		10699	return Target.x86.featureSetHas(self.target.cpu.features, feature);
		10700	}
		10701	fn hasAnyFeatures(self: *Self, features: anytype) bool {
		10702	return Target.x86.featureSetHasAny(self.target.cpu.features, features);
		10703	}
		10704	fn hasAllFeatures(self: *Self, features: anytype) bool {
		10705	return Target.x86.featureSetHasAll(self.target.cpu.features, features);
		10706	}

src/arch/x86_64/Emit.zig+141-131

...	@@ -18,142 +18,152 @@ pub const Error = Lower.Error \|\| error{	...	@@ -18,142 +18,152 @@ pub const Error = Lower.Error \|\| error{
18	};	18	};
19		19
20	pub fn emitMir(emit: *Emit) Error!void {	20	pub fn emitMir(emit: *Emit) Error!void {
21	for (0..emit.lower.mir.instructions.len) \|i\| {	21	for (0..emit.lower.mir.instructions.len) \|mir_i\| {
22	const index = @intCast(Mir.Inst.Index, i);	22	const mir_index = @intCast(Mir.Inst.Index, mir_i);
23	const inst = emit.lower.mir.instructions.get(index);	23	try emit.code_offset_mapping.putNoClobber(
24		24	emit.lower.allocator,
25	const start_offset = @intCast(u32, emit.code.items.len);	25	mir_index,
26	try emit.code_offset_mapping.putNoClobber(emit.lower.allocator, index, start_offset);	26	@intCast(u32, emit.code.items.len),
27	for (try emit.lower.lowerMir(inst)) \|lower_inst\| try lower_inst.encode(emit.code.writer(), .{});	27	);
28	const end_offset = @intCast(u32, emit.code.items.len);	28	const lowered = try emit.lower.lowerMir(mir_index);
29		29	var lowered_relocs = lowered.relocs;
30	switch (inst.tag) {	30	for (lowered.insts, 0..) \|lowered_inst, lowered_index\| {
31	else => {},	31	const start_offset = @intCast(u32, emit.code.items.len);
32		32	try lowered_inst.encode(emit.code.writer(), .{});
33	.jmp_reloc => try emit.relocs.append(emit.lower.allocator, .{	33	const end_offset = @intCast(u32, emit.code.items.len);
34	.source = start_offset,	34	while (lowered_relocs.len > 0 and
35	.target = inst.data.inst,	35	lowered_relocs[0].lowered_inst_index == lowered_index) : ({
36	.offset = end_offset - 4,	36	lowered_relocs = lowered_relocs[1..];
37	.length = 5,	37	}) switch (lowered_relocs[0].target) {
38	}),	38	.inst => \|target\| try emit.relocs.append(emit.lower.allocator, .{
39		39	.source = start_offset,
40	.call_extern => if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
41	// Add relocation to the decl.
42	const atom_index = macho_file.getAtomIndexForSymbol(
43	.{ .sym_index = inst.data.relocation.atom_index, .file = null },
44	).?;
45	const target = macho_file.getGlobalByIndex(inst.data.relocation.sym_index);
46	try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{
47	.type = .branch,
48	.target = target,	40	.target = target,
49	.offset = end_offset - 4,	41	.offset = end_offset - 4,
50	.addend = 0,	42	.length = @intCast(u5, end_offset - start_offset),
51	.pcrel = true,	43	}),
52	.length = 2,	44	.linker_extern_fn => \|symbol\| if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
53	});	45	// Add relocation to the decl.
54	} else if (emit.bin_file.cast(link.File.Coff)) \|coff_file\| {	46	const atom_index = macho_file.getAtomIndexForSymbol(
55	// Add relocation to the decl.	47	.{ .sym_index = symbol.atom_index, .file = null },
56	const atom_index = coff_file.getAtomIndexForSymbol(	48	).?;
57	.{ .sym_index = inst.data.relocation.atom_index, .file = null },	49	const target = macho_file.getGlobalByIndex(symbol.sym_index);
58	).?;	50	try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{
59	const target = coff_file.getGlobalByIndex(inst.data.relocation.sym_index);	51	.type = .branch,
60	try link.File.Coff.Atom.addRelocation(coff_file, atom_index, .{	52	.target = target,
61	.type = .direct,	53	.offset = end_offset - 4,
62	.target = target,	54	.addend = 0,
63	.offset = end_offset - 4,	55	.pcrel = true,
64	.addend = 0,	56	.length = 2,
65	.pcrel = true,	57	});
66	.length = 2,	58	} else if (emit.bin_file.cast(link.File.Coff)) \|coff_file\| {
67	});	59	// Add relocation to the decl.
68	} else return emit.fail("TODO implement {} for {}", .{ inst.tag, emit.bin_file.tag }),	60	const atom_index = coff_file.getAtomIndexForSymbol(
69		61	.{ .sym_index = symbol.atom_index, .file = null },
70	.mov_linker, .lea_linker => if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {	62	).?;
71	const metadata =	63	const target = coff_file.getGlobalByIndex(symbol.sym_index);
72	emit.lower.mir.extraData(Mir.LeaRegisterReloc, inst.data.payload).data;	64	try link.File.Coff.Atom.addRelocation(coff_file, atom_index, .{
73	const atom_index = macho_file.getAtomIndexForSymbol(.{	65	.type = .direct,
74	.sym_index = metadata.atom_index,	66	.target = target,
75	.file = null,	67	.offset = end_offset - 4,
76	}).?;	68	.addend = 0,
77	try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{	69	.pcrel = true,
78	.type = switch (inst.ops) {	70	.length = 2,
79	.got_reloc => .got,	71	});
80	.direct_reloc => .signed,	72	} else return emit.fail("TODO implement extern reloc for {s}", .{
81	.tlv_reloc => .tlv,	73	@tagName(emit.bin_file.tag),
82	else => unreachable,	74	}),
83	},	75	.linker_got,
84	.target = .{ .sym_index = metadata.sym_index, .file = null },	76	.linker_direct,
85	.offset = @intCast(u32, end_offset - 4),	77	.linker_import,
86	.addend = 0,	78	.linker_tlv,
87	.pcrel = true,	79	=> \|symbol\| if (emit.bin_file.cast(link.File.MachO)) \|macho_file\| {
88	.length = 2,	80	const atom_index = macho_file.getAtomIndexForSymbol(.{
89	});	81	.sym_index = symbol.atom_index,
90	} else if (emit.bin_file.cast(link.File.Coff)) \|coff_file\| {	82	.file = null,
91	const metadata =	83	}).?;
92	emit.lower.mir.extraData(Mir.LeaRegisterReloc, inst.data.payload).data;	84	try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{
93	const atom_index = coff_file.getAtomIndexForSymbol(.{	85	.type = switch (lowered_relocs[0].target) {
94	.sym_index = metadata.atom_index,	86	.linker_got => .got,
95	.file = null,	87	.linker_direct => .signed,
96	}).?;	88	.linker_tlv => .tlv,
97	try link.File.Coff.Atom.addRelocation(coff_file, atom_index, .{	89	else => unreachable,
98	.type = switch (inst.ops) {	90	},
99	.got_reloc => .got,	91	.target = .{ .sym_index = symbol.sym_index, .file = null },
100	.direct_reloc => .direct,	92	.offset = @intCast(u32, end_offset - 4),
101	.import_reloc => .import,	93	.addend = 0,
102	else => unreachable,	94	.pcrel = true,
103	},	95	.length = 2,
104	.target = switch (inst.ops) {	96	});
105	.got_reloc,	97	} else if (emit.bin_file.cast(link.File.Coff)) \|coff_file\| {
106	.direct_reloc,	98	const atom_index = coff_file.getAtomIndexForSymbol(.{
107	=> .{ .sym_index = metadata.sym_index, .file = null },	99	.sym_index = symbol.atom_index,
108	.import_reloc => coff_file.getGlobalByIndex(metadata.sym_index),	100	.file = null,
109	else => unreachable,	101	}).?;
110	},	102	try link.File.Coff.Atom.addRelocation(coff_file, atom_index, .{
111	.offset = @intCast(u32, end_offset - 4),	103	.type = switch (lowered_relocs[0].target) {
112	.addend = 0,	104	.linker_got => .got,
113	.pcrel = true,	105	.linker_direct => .direct,
114	.length = 2,	106	.linker_import => .import,
115	});	107	else => unreachable,
116	} else return emit.fail("TODO implement {} for {}", .{ inst.tag, emit.bin_file.tag }),	108	},
117		109	.target = switch (lowered_relocs[0].target) {
118	.jcc => try emit.relocs.append(emit.lower.allocator, .{	110	.linker_got,
119	.source = start_offset,	111	.linker_direct,
120	.target = inst.data.inst_cc.inst,	112	=> .{ .sym_index = symbol.sym_index, .file = null },
121	.offset = end_offset - 4,	113	.linker_import => coff_file.getGlobalByIndex(symbol.sym_index),
122	.length = 6,	114	else => unreachable,
123	}),	115	},
124		116	.offset = @intCast(u32, end_offset - 4),
125	.dbg_line => try emit.dbgAdvancePCAndLine(	117	.addend = 0,
126	inst.data.line_column.line,	118	.pcrel = true,
127	inst.data.line_column.column,	119	.length = 2,
128	),	120	});
		121	} else return emit.fail("TODO implement linker reloc for {s}", .{
		122	@tagName(emit.bin_file.tag),
		123	}),
		124	};
		125	}
		126	std.debug.assert(lowered_relocs.len == 0);
129		127
130	.dbg_prologue_end => {	128	if (lowered.insts.len == 0) {
131	switch (emit.debug_output) {	129	const mir_inst = emit.lower.mir.instructions.get(mir_index);
132	.dwarf => \|dw\| {	130	switch (mir_inst.tag) {
133	try dw.setPrologueEnd();	131	else => unreachable,
134	log.debug("mirDbgPrologueEnd (line={d}, col={d})", .{	132	.pseudo => switch (mir_inst.ops) {
135	emit.prev_di_line, emit.prev_di_column,	133	else => unreachable,
136	});	134	.pseudo_dbg_prologue_end_none => {
137	try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);	135	switch (emit.debug_output) {
		136	.dwarf => \|dw\| {
		137	try dw.setPrologueEnd();
		138	log.debug("mirDbgPrologueEnd (line={d}, col={d})", .{
		139	emit.prev_di_line, emit.prev_di_column,
		140	});
		141	try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
		142	},
		143	.plan9 => {},
		144	.none => {},
		145	}
138	},	146	},
139	.plan9 => {},	147	.pseudo_dbg_line_line_column => try emit.dbgAdvancePCAndLine(
140	.none => {},	148	mir_inst.data.line_column.line,
141	}	149	mir_inst.data.line_column.column,
142	},	150	),
143		151	.pseudo_dbg_epilogue_begin_none => {
144	.dbg_epilogue_begin => {	152	switch (emit.debug_output) {
145	switch (emit.debug_output) {	153	.dwarf => \|dw\| {
146	.dwarf => \|dw\| {	154	try dw.setEpilogueBegin();
147	try dw.setEpilogueBegin();	155	log.debug("mirDbgEpilogueBegin (line={d}, col={d})", .{
148	log.debug("mirDbgEpilogueBegin (line={d}, col={d})", .{	156	emit.prev_di_line, emit.prev_di_column,
149	emit.prev_di_line, emit.prev_di_column,	157	});
150	});	158	try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
151	try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);	159	},
		160	.plan9 => {},
		161	.none => {},
		162	}
152	},	163	},
153	.plan9 => {},	164	.pseudo_dead_none => {},
154	.none => {},	165	},
155	}	166	}
156	},
157	}	167	}
158	}	168	}
159	try emit.fixupRelocs();	169	try emit.fixupRelocs();

src/arch/x86_64/Encoding.zig+240-107

 @@ -23,6 +23,7 @@ const Data = struct {
     opc: [7]u8,
     modrm_ext: u3,
     mode: Mode,
+    feature: Feature,
 };
 pub fn findByMnemonic(
 @@ -57,9 +58,9 @@ pub fn findByMnemonic(
     var shortest_len: ?usize = null;
     next: for (mnemonic_to_encodings_map[@enumToInt(mnemonic)]) |data| {
         switch (data.mode) {
-            .rex => if (!rex_required) continue,
+            .none, .short => if (rex_required) continue,
-            .long, .sse_long, .sse2_long => {},
+            .rex, .rex_short => if (!rex_required) continue,
-            else => if (rex_required) continue,
+            else => {},
         }
         for (input_ops, data.ops) |input_op, data_op|
             if (!input_op.isSubset(data_op)) continue :next;
 @@ -88,28 +89,13 @@ pub fn findByOpcode(opc: []const u8, prefixes: struct {
         if (modrm_ext) |ext| if (ext != data.modrm_ext) continue;
         if (!std.mem.eql(u8, opc, enc.opcode())) continue;
         if (prefixes.rex.w) {
-            switch (data.mode) {
+            if (!data.mode.isLong()) continue;
-                .short, .fpu, .sse, .sse2, .sse4_1, .none => continue,
-                .long, .sse_long, .sse2_long, .rex => {},
-            }
         } else if (prefixes.rex.present and !prefixes.rex.isSet()) {
-            switch (data.mode) {
+            if (!data.mode.isRex()) continue;
-                .rex => {},
-                else => continue,
-            }
         } else if (prefixes.legacy.prefix_66) {
-            switch (enc.operandBitSize()) {
+            if (!data.mode.isShort()) continue;
-                16 => {},
-                else => continue,
-            }
         } else {
-            switch (data.mode) {
+            if (data.mode.isShort()) continue;
-                .none => switch (enc.operandBitSize()) {
-                    16 => continue,
-                    else => {},
-                },
-                else => continue,
-            }
         }
         return enc;
     };
 @@ -130,30 +116,11 @@ pub fn mandatoryPrefix(encoding: *const Encoding) ?u8 {
 pub fn modRmExt(encoding: Encoding) u3 {
     return switch (encoding.data.op_en) {
-        .m, .mi, .m1, .mc => encoding.data.modrm_ext,
+        .m, .mi, .m1, .mc, .vmi => encoding.data.modrm_ext,
         else => unreachable,
     };
 }
-pub fn operandBitSize(encoding: Encoding) u64 {
-    switch (encoding.data.mode) {
-        .short => return 16,
-        .long, .sse_long, .sse2_long => return 64,
-        else => {},
-    }
-    const bit_size: u64 = switch (encoding.data.op_en) {
-        .np => switch (encoding.data.ops[0]) {
-            .o16 => 16,
-            .o32 => 32,
-            .o64 => 64,
-            else => 32,
-        },
-        .td => encoding.data.ops[1].bitSize(),
-        else => encoding.data.ops[0].bitSize(),
-    };
-    return bit_size;
-}
 pub fn format(
     encoding: Encoding,
     comptime fmt: []const u8,
 @@ -162,14 +129,41 @@ pub fn format(
 ) !void {
     _ = options;
     _ = fmt;
-    switch (encoding.data.mode) {
-        .long, .sse_long, .sse2_long => try writer.writeAll("REX.W + "),
-        else => {},
-    }
-    for (encoding.opcode()) |byte| {
+    var opc = encoding.opcode();
-        try writer.print("{x:0>2} ", .{byte});
+    if (encoding.data.mode.isVex()) {
-    }
+        try writer.writeAll("VEX.");
+        try writer.writeAll(switch (encoding.data.mode) {
+            .vex_128_w0, .vex_128_w1, .vex_128_wig => "128",
+            .vex_256_w0, .vex_256_w1, .vex_256_wig => "256",
+            .vex_lig_w0, .vex_lig_w1, .vex_lig_wig => "LIG",
+            .vex_lz_w0, .vex_lz_w1, .vex_lz_wig => "LZ",
+            else => unreachable,
+        });
+        switch (opc[0]) {
+            else => {},
+            0x66, 0xf3, 0xf2 => {
+                try writer.print(".{X:0>2}", .{opc[0]});
+                opc = opc[1..];
+            },
+        }
+        try writer.print(".{}", .{std.fmt.fmtSliceHexUpper(opc[0 .. opc.len - 1])});
+        opc = opc[opc.len - 1 ..];
+        try writer.writeAll(".W");
+        try writer.writeAll(switch (encoding.data.mode) {
+            .vex_128_w0, .vex_256_w0, .vex_lig_w0, .vex_lz_w0 => "0",
+            .vex_128_w1, .vex_256_w1, .vex_lig_w1, .vex_lz_w1 => "1",
+            .vex_128_wig, .vex_256_wig, .vex_lig_wig, .vex_lz_wig => "IG",
+            else => unreachable,
+        });
+        try writer.writeByte(' ');
+    } else if (encoding.data.mode.isLong()) try writer.writeAll("REX.W + ");
+    for (opc) |byte| try writer.print("{x:0>2} ", .{byte});
     switch (encoding.data.op_en) {
         .np, .fd, .td, .i, .zi, .d => {},
 @@ -183,16 +177,17 @@ pub fn format(
             };
             try writer.print("+{s} ", .{tag});
         },
-        .m, .mi, .m1, .mc => try writer.print("/{d} ", .{encoding.modRmExt()}),
+        .m, .mi, .m1, .mc, .vmi => try writer.print("/{d} ", .{encoding.modRmExt()}),
-        .mr, .rm, .rmi, .mri, .mrc => try writer.writeAll("/r "),
+        .mr, .rm, .rmi, .mri, .mrc, .rvm, .rvmi, .mvr => try writer.writeAll("/r "),
     }
     switch (encoding.data.op_en) {
-        .i, .d, .zi, .oi, .mi, .rmi, .mri => {
+        .i, .d, .zi, .oi, .mi, .rmi, .mri, .vmi, .rvmi => {
             const op = switch (encoding.data.op_en) {
                 .i, .d => encoding.data.ops[0],
                 .zi, .oi, .mi => encoding.data.ops[1],
-                .rmi, .mri => encoding.data.ops[2],
+                .rmi, .mri, .vmi => encoding.data.ops[2],
+                .rvmi => encoding.data.ops[3],
                 else => unreachable,
             };
             const tag = switch (op) {
 @@ -207,7 +202,7 @@ pub fn format(
             };
             try writer.print("{s} ", .{tag});
         },
-        .np, .fd, .td, .o, .m, .m1, .mc, .mr, .rm, .mrc => {},
+        .np, .fd, .td, .o, .m, .m1, .mc, .mr, .rm, .mrc, .rvm, .mvr => {},
     }
     try writer.print("{s} ", .{@tagName(encoding.mnemonic)});
 @@ -267,44 +262,79 @@ pub const Mnemonic = enum {
     // MMX
     movd,
     // SSE
-    addss,
+    addps, addss,
     andps,
     andnps,
     cmpss,
     cvtsi2ss,
-    divss,
+    divps, divss,
-    maxss, minss,
+    maxps, maxss,
-    movaps, movss, movups,
+    minps, minss,
-    mulss,
+    movaps, movhlps, movss, movups,
+    mulps, mulss,
     orps,
-    pextrw,
+    pextrw, pinsrw,
-    pinsrw,
+    sqrtps, sqrtss,
-    sqrtps,
+    subps, subss,
-    sqrtss,
-    subss,
     ucomiss,
     xorps,
     // SSE2
-    addsd,
+    addpd, addsd,
     andpd,
     andnpd,
     //cmpsd,
     cvtsd2ss, cvtsi2sd, cvtss2sd,
-    divsd,
+    divpd, divsd,
-    maxsd, minsd,
+    maxpd, maxsd,
+    minpd, minsd,
     movapd,
     movq, //movd, movsd,
     movupd,
-    mulsd,
+    mulpd, mulsd,
     orpd,
-    sqrtpd,
+    pshufhw, pshuflw,
-    sqrtsd,
+    psrld, psrlq, psrlw,
-    subsd,
+    punpckhbw, punpckhdq, punpckhqdq, punpckhwd,
+    punpcklbw, punpckldq, punpcklqdq, punpcklwd,
+    sqrtpd, sqrtsd,
+    subpd, subsd,
     ucomisd,
     xorpd,
+    // SSE3
+    movddup, movshdup, movsldup,
     // SSE4.1
-    roundss,
+    pextrb, pextrd, pextrq,
-    roundsd,
+    pinsrb, pinsrd, pinsrq,
+    roundpd, roundps, roundsd, roundss,
+    // AVX
+    vaddpd, vaddps, vaddsd, vaddss,
+    vcvtsd2ss, vcvtsi2sd, vcvtsi2ss, vcvtss2sd,
+    vdivpd, vdivps, vdivsd, vdivss,
+    vmaxpd, vmaxps, vmaxsd, vmaxss,
+    vminpd, vminps, vminsd, vminss,
+    vmovapd, vmovaps,
+    vmovddup, vmovhlps,
+    vmovsd,
+    vmovshdup, vmovsldup,
+    vmovss,
+    vmovupd, vmovups,
+    vmulpd, vmulps, vmulsd, vmulss,
+    vpextrb, vpextrd, vpextrq, vpextrw,
+    vpinsrb, vpinsrd, vpinsrq, vpinsrw,
+    vpshufhw, vpshuflw,
+    vpsrld, vpsrlq, vpsrlw,
+    vpunpckhbw, vpunpckhdq, vpunpckhqdq, vpunpckhwd,
+    vpunpcklbw, vpunpckldq, vpunpcklqdq, vpunpcklwd,
+    vroundpd, vroundps, vroundsd, vroundss,
+    vsqrtpd, vsqrtps, vsqrtsd, vsqrtss,
+    vsubpd, vsubps, vsubsd, vsubss,
+    // F16C
+    vcvtph2ps, vcvtps2ph,
+    // FMA
+    vfmadd132pd, vfmadd213pd, vfmadd231pd,
+    vfmadd132ps, vfmadd213ps, vfmadd231ps,
+    vfmadd132sd, vfmadd213sd, vfmadd231sd,
+    vfmadd132ss, vfmadd213ss, vfmadd231ss,
     // zig fmt: on
 };
 @@ -317,6 +347,7 @@ pub const OpEn = enum {
     fd, td,
     m1, mc, mi, mr, rm,
     rmi, mri, mrc,
+    vmi, rvm, rvmi, mvr,
     // zig fmt: on
 };
 @@ -331,12 +362,14 @@ pub const Op = enum {
     cl,
     r8, r16, r32, r64,
     rm8, rm16, rm32, rm64,
-    m8, m16, m32, m64, m80, m128,
+    r32_m8, r32_m16, r64_m16,
+    m8, m16, m32, m64, m80, m128, m256,
     rel8, rel16, rel32,
     m,
     moffs,
     sreg,
     xmm, xmm_m32, xmm_m64, xmm_m128,
+    ymm, ymm_m256,
     // zig fmt: on
     pub fn fromOperand(operand: Instruction.Operand) Op {
 @@ -348,6 +381,7 @@ pub const Op = enum {
                     .segment => return .sreg,
                     .floating_point => return switch (reg.bitSize()) {
                         128 => .xmm,
+                        256 => .ymm,
                         else => unreachable,
                     },
                     .general_purpose => {
 @@ -381,6 +415,7 @@ pub const Op = enum {
                         64 => .m64,
                         80 => .m80,
                         128 => .m128,
+                        256 => .m256,
                         else => unreachable,
                     };
                 },
 @@ -409,16 +444,52 @@ pub const Op = enum {
         }
     }
-    pub fn bitSize(op: Op) u64 {
+    pub fn immBitSize(op: Op) u64 {
         return switch (op) {
             .none, .o16, .o32, .o64, .moffs, .m, .sreg => unreachable,
+            .al, .cl, .r8, .rm8, .r32_m8 => unreachable,
+            .ax, .r16, .rm16 => unreachable,
+            .eax, .r32, .rm32, .r32_m16 => unreachable,
+            .rax, .r64, .rm64, .r64_m16 => unreachable,
+            .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => unreachable,
+            .ymm, .ymm_m256 => unreachable,
+            .m8, .m16, .m32, .m64, .m80, .m128, .m256 => unreachable,
             .unity => 1,
-            .imm8, .imm8s, .al, .cl, .r8, .m8, .rm8, .rel8 => 8,
+            .imm8, .imm8s, .rel8 => 8,
-            .imm16, .imm16s, .ax, .r16, .m16, .rm16, .rel16 => 16,
+            .imm16, .imm16s, .rel16 => 16,
-            .imm32, .imm32s, .eax, .r32, .m32, .rm32, .rel32, .xmm_m32 => 32,
+            .imm32, .imm32s, .rel32 => 32,
-            .imm64, .rax, .r64, .m64, .rm64, .xmm_m64 => 64,
+            .imm64 => 64,
+        };
+    }
+    pub fn regBitSize(op: Op) u64 {
+        return switch (op) {
+            .none, .o16, .o32, .o64, .moffs, .m, .sreg => unreachable,
+            .unity, .imm8, .imm8s, .imm16, .imm16s, .imm32, .imm32s, .imm64 => unreachable,
+            .rel8, .rel16, .rel32 => unreachable,
+            .m8, .m16, .m32, .m64, .m80, .m128, .m256 => unreachable,
+            .al, .cl, .r8, .rm8 => 8,
+            .ax, .r16, .rm16 => 16,
+            .eax, .r32, .rm32, .r32_m8, .r32_m16 => 32,
+            .rax, .r64, .rm64, .r64_m16 => 64,
+            .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => 128,
+            .ymm, .ymm_m256 => 256,
+        };
+    }
+    pub fn memBitSize(op: Op) u64 {
+        return switch (op) {
+            .none, .o16, .o32, .o64, .moffs, .m, .sreg => unreachable,
+            .unity, .imm8, .imm8s, .imm16, .imm16s, .imm32, .imm32s, .imm64 => unreachable,
+            .rel8, .rel16, .rel32 => unreachable,
+            .al, .cl, .r8, .ax, .r16, .eax, .r32, .rax, .r64, .xmm, .ymm => unreachable,
+            .m8, .rm8, .r32_m8 => 8,
+            .m16, .rm16, .r32_m16, .r64_m16 => 16,
+            .m32, .rm32, .xmm_m32 => 32,
+            .m64, .rm64, .xmm_m64 => 64,
             .m80 => 80,
-            .m128, .xmm, .xmm_m128 => 128,
+            .m128, .xmm_m128 => 128,
+            .m256, .ymm_m256 => 256,
         };
     }
 @@ -441,7 +512,9 @@ pub const Op = enum {
             .al, .ax, .eax, .rax,
             .r8, .r16, .r32, .r64,
             .rm8, .rm16, .rm32, .rm64,
+            .r32_m8, .r32_m16, .r64_m16,
             .xmm, .xmm_m32, .xmm_m64, .xmm_m128,
+            .ymm, .ymm_m256,
             => true,
             else => false,
         };
 @@ -465,9 +538,11 @@ pub const Op = enum {
         // zig fmt: off
         return switch (op) {
             .rm8, .rm16, .rm32, .rm64,
-            .m8, .m16, .m32, .m64, .m80, .m128,
+            .r32_m8, .r32_m16, .r64_m16,
+            .m8, .m16, .m32, .m64, .m80, .m128, .m256,
             .m,
             .xmm_m32, .xmm_m64, .xmm_m128,
+            .ymm_m256,
             =>  true,
             else => false,
         };
 @@ -487,15 +562,10 @@ pub const Op = enum {
             .al, .ax, .eax, .rax, .cl => .general_purpose,
             .r8, .r16, .r32, .r64 => .general_purpose,
             .rm8, .rm16, .rm32, .rm64 => .general_purpose,
+            .r32_m8, .r32_m16, .r64_m16 => .general_purpose,
             .sreg => .segment,
             .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => .floating_point,
-        };
+            .ymm, .ymm_m256 => .floating_point,
-    }
-    pub fn isFloatingPointRegister(op: Op) bool {
-        return switch (op) {
-            .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => true,
-            else => false,
         };
     }
 @@ -512,30 +582,27 @@ pub const Op = enum {
                 if (op.isRegister() and target.isRegister()) {
                     return switch (target) {
                         .cl, .al, .ax, .eax, .rax => op == target,
-                        else => op.class() == target.class() and switch (target.class()) {
+                        else => op.class() == target.class() and op.regBitSize() == target.regBitSize(),
-                            .floating_point => true,
-                            else => op.bitSize() == target.bitSize(),
-                        },
                     };
                 }
                 if (op.isMemory() and target.isMemory()) {
                     switch (target) {
                         .m => return true,
-                        else => return op.bitSize() == target.bitSize(),
+                        else => return op.memBitSize() == target.memBitSize(),
                     }
                 }
                 if (op.isImmediate() and target.isImmediate()) {
                     switch (target) {
-                        .imm64 => if (op.bitSize() <= 64) return true,
+                        .imm64 => if (op.immBitSize() <= 64) return true,
-                        .imm32s, .rel32 => if (op.bitSize() < 32 or (op.bitSize() == 32 and op.isSigned()))
+                        .imm32s, .rel32 => if (op.immBitSize() < 32 or (op.immBitSize() == 32 and op.isSigned()))
                             return true,
-                        .imm32 => if (op.bitSize() <= 32) return true,
+                        .imm32 => if (op.immBitSize() <= 32) return true,
-                        .imm16s, .rel16 => if (op.bitSize() < 16 or (op.bitSize() == 16 and op.isSigned()))
+                        .imm16s, .rel16 => if (op.immBitSize() < 16 or (op.immBitSize() == 16 and op.isSigned()))
                             return true,
-                        .imm16 => if (op.bitSize() <= 16) return true,
+                        .imm16 => if (op.immBitSize() <= 16) return true,
-                        .imm8s, .rel8 => if (op.bitSize() < 8 or (op.bitSize() == 8 and op.isSigned()))
+                        .imm8s, .rel8 => if (op.immBitSize() < 8 or (op.immBitSize() == 8 and op.isSigned()))
                             return true,
-                        .imm8 => if (op.bitSize() <= 8) return true,
+                        .imm8 => if (op.immBitSize() <= 8) return true,
                         else => {},
                     }
                     return op == target;
 @@ -547,16 +614,81 @@ pub const Op = enum {
 };
 pub const Mode = enum {
+    // zig fmt: off
+    none,
+    short, long,
+    rex, rex_short,
+    vex_128_w0, vex_128_w1, vex_128_wig,
+    vex_256_w0, vex_256_w1, vex_256_wig,
+    vex_lig_w0, vex_lig_w1, vex_lig_wig,
+    vex_lz_w0,  vex_lz_w1,  vex_lz_wig,
+    // zig fmt: on
+    pub fn isShort(mode: Mode) bool {
+        return switch (mode) {
+            .short, .rex_short => true,
+            else => false,
+        };
+    }
+    pub fn isLong(mode: Mode) bool {
+        return switch (mode) {
+            .long,
+            .vex_128_w1,
+            .vex_256_w1,
+            .vex_lig_w1,
+            .vex_lz_w1,
+            => true,
+            else => false,
+        };
+    }
+    pub fn isRex(mode: Mode) bool {
+        return switch (mode) {
+            else => false,
+            .rex, .rex_short => true,
+        };
+    }
+    pub fn isVex(mode: Mode) bool {
+        return switch (mode) {
+            // zig fmt: off
+            else => false,
+            .vex_128_w0, .vex_128_w1, .vex_128_wig,
+            .vex_256_w0, .vex_256_w1, .vex_256_wig,
+            .vex_lig_w0, .vex_lig_w1, .vex_lig_wig,
+            .vex_lz_w0,  .vex_lz_w1,  .vex_lz_wig,
+            => true,
+            // zig fmt: on
+        };
+    }
+    pub fn isVecLong(mode: Mode) bool {
+        return switch (mode) {
+            // zig fmt: off
+            else => unreachable,
+            .vex_128_w0, .vex_128_w1, .vex_128_wig,
+            .vex_lig_w0, .vex_lig_w1, .vex_lig_wig,
+            .vex_lz_w0,  .vex_lz_w1,  .vex_lz_wig,
+            => false,
+            .vex_256_w0, .vex_256_w1, .vex_256_wig,
+            => true,
+            // zig fmt: on
+        };
+    }
+};
+pub const Feature = enum {
     none,
-    short,
+    avx,
-    fpu,
+    avx2,
-    rex,
+    f16c,
-    long,
+    fma,
     sse,
-    sse_long,
     sse2,
-    sse2_long,
+    sse3,
     sse4_1,
+    x87,
 };
 fn estimateInstructionLength(prefix: Prefix, encoding: Encoding, ops: []const Operand) usize {
 @@ -573,7 +705,7 @@ fn estimateInstructionLength(prefix: Prefix, encoding: Encoding, ops: []const Op
 }
 const mnemonic_to_encodings_map = init: {
-    @setEvalBranchQuota(100_000);
+    @setEvalBranchQuota(20_000);
     const encodings = @import("encodings.zig");
     var entries = encodings.table;
     std.sort.sort(encodings.Entry, &entries, {}, struct {
 @@ -593,6 +725,7 @@ const mnemonic_to_encodings_map = init: {
             .opc = undefined,
             .modrm_ext = entry[4],
             .mode = entry[5],
+            .feature = entry[6],
         };
         // TODO: use `@memcpy` for these. When I did that, I got a false positive
         // compile error for this copy happening at compile time.

src/arch/x86_64/Lower.zig+283-323

...	@@ -5,13 +5,22 @@ mir: Mir,	...	@@ -5,13 +5,22 @@ mir: Mir,
5	target: *const std.Target,	5	target: *const std.Target,
6	err_msg: ?*ErrorMsg = null,	6	err_msg: ?*ErrorMsg = null,
7	src_loc: Module.SrcLoc,	7	src_loc: Module.SrcLoc,
8	result: [	8	result_insts_len: u8 = undefined,
		9	result_relocs_len: u8 = undefined,
		10	result_insts: [
9	std.mem.max(usize, &.{	11	std.mem.max(usize, &.{
10	abi.Win64.callee_preserved_regs.len,	12	2, // cmovcc: cmovcc \ cmovcc
11	abi.SysV.callee_preserved_regs.len,	13	3, // setcc: setcc \ setcc \ logicop
		14	2, // jcc: jcc \ jcc
		15	abi.Win64.callee_preserved_regs.len, // push_regs/pop_regs
		16	abi.SysV.callee_preserved_regs.len, // push_regs/pop_regs
12	})	17	})
13	]Instruction = undefined,	18	]Instruction = undefined,
14	result_len: usize = undefined,	19	result_relocs: [
		20	std.mem.max(usize, &.{
		21	2, // jcc: jcc \ jcc
		22	})
		23	]Reloc = undefined,
15		24
16	pub const Error = error{	25	pub const Error = error{
17	OutOfMemory,	26	OutOfMemory,
...	@@ -20,155 +29,155 @@ pub const Error = error{	...	@@ -20,155 +29,155 @@ pub const Error = error{
20	CannotEncode,	29	CannotEncode,
21	};	30	};
22		31
23	/// The returned slice is overwritten by the next call to lowerMir.	32	pub const Reloc = struct {
24	pub fn lowerMir(lower: *Lower, inst: Mir.Inst) Error![]const Instruction {	33	lowered_inst_index: u8,
25	lower.result = undefined;	34	target: Target,
26	errdefer lower.result = undefined;	35
27	lower.result_len = 0;	36	const Target = union(enum) {
28	defer lower.result_len = undefined;	37	inst: Mir.Inst.Index,
		38	linker_extern_fn: Mir.Reloc,
		39	linker_got: Mir.Reloc,
		40	linker_direct: Mir.Reloc,
		41	linker_import: Mir.Reloc,
		42	linker_tlv: Mir.Reloc,
		43	};
		44	};
29		45
		46	/// The returned slice is overwritten by the next call to lowerMir.
		47	pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct {
		48	insts: []const Instruction,
		49	relocs: []const Reloc,
		50	} {
		51	lower.result_insts = undefined;
		52	lower.result_relocs = undefined;
		53	errdefer lower.result_insts = undefined;
		54	errdefer lower.result_relocs = undefined;
		55	lower.result_insts_len = 0;
		56	lower.result_relocs_len = 0;
		57	defer lower.result_insts_len = undefined;
		58	defer lower.result_relocs_len = undefined;
		59
		60	const inst = lower.mir.instructions.get(index);
30	switch (inst.tag) {	61	switch (inst.tag) {
31	.adc,	62	else => try lower.generic(inst),
32	.add,	63	.pseudo => switch (inst.ops) {
33	.@"and",	64	.pseudo_cmov_z_and_np_rr => {
34	.bsf,	65	try lower.emit(.none, .cmovnz, &.{
35	.bsr,	66	.{ .reg = inst.data.rr.r2 },
36	.bswap,	67	.{ .reg = inst.data.rr.r1 },
37	.bt,	68	});
38	.btc,	69	try lower.emit(.none, .cmovnp, &.{
39	.btr,	70	.{ .reg = inst.data.rr.r1 },
40	.bts,	71	.{ .reg = inst.data.rr.r2 },
41	.call,	72	});
42	.cbw,	73	},
43	.cwde,	74	.pseudo_cmov_nz_or_p_rr => {
44	.cdqe,	75	try lower.emit(.none, .cmovnz, &.{
45	.cwd,	76	.{ .reg = inst.data.rr.r1 },
46	.cdq,	77	.{ .reg = inst.data.rr.r2 },
47	.cqo,	78	});
48	.cmp,	79	try lower.emit(.none, .cmovp, &.{
49	.cmpxchg,	80	.{ .reg = inst.data.rr.r1 },
50	.div,	81	.{ .reg = inst.data.rr.r2 },
51	.fisttp,	82	});
52	.fld,	83	},
53	.idiv,	84	.pseudo_cmov_nz_or_p_rm_sib,
54	.imul,	85	.pseudo_cmov_nz_or_p_rm_rip,
55	.int3,	86	=> {
56	.jmp,	87	try lower.emit(.none, .cmovnz, &.{
57	.lea,	88	.{ .reg = inst.data.rx.r1 },
58	.lfence,	89	.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
59	.lzcnt,	90	});
60	.mfence,	91	try lower.emit(.none, .cmovp, &.{
61	.mov,	92	.{ .reg = inst.data.rx.r1 },
62	.movbe,	93	.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
63	.movd,	94	});
64	.movq,	95	},
65	.movzx,	96	.pseudo_set_z_and_np_r => {
66	.mul,	97	try lower.emit(.none, .setz, &.{
67	.neg,	98	.{ .reg = inst.data.r_scratch.r1 },
68	.nop,	99	});
69	.not,	100	try lower.emit(.none, .setnp, &.{
70	.@"or",	101	.{ .reg = inst.data.r_scratch.scratch_reg },
71	.pop,	102	});
72	.popcnt,	103	try lower.emit(.none, .@"and", &.{
73	.push,	104	.{ .reg = inst.data.r_scratch.r1 },
74	.rcl,	105	.{ .reg = inst.data.r_scratch.scratch_reg },
75	.rcr,	106	});
76	.ret,	107	},
77	.rol,	108	.pseudo_set_z_and_np_m_sib,
78	.ror,	109	.pseudo_set_z_and_np_m_rip,
79	.sal,	110	=> {
80	.sar,	111	try lower.emit(.none, .setz, &.{
81	.sbb,	112	.{ .mem = lower.mem(inst.ops, inst.data.x_scratch.payload) },
82	.sfence,	113	});
83	.shl,	114	try lower.emit(.none, .setnp, &.{
84	.shld,	115	.{ .reg = inst.data.x_scratch.scratch_reg },
85	.shr,	116	});
86	.shrd,	117	try lower.emit(.none, .@"and", &.{
87	.sub,	118	.{ .mem = lower.mem(inst.ops, inst.data.x_scratch.payload) },
88	.syscall,	119	.{ .reg = inst.data.x_scratch.scratch_reg },
89	.@"test",	120	});
90	.tzcnt,	121	},
91	.ud2,	122	.pseudo_set_nz_or_p_r => {
92	.xadd,	123	try lower.emit(.none, .setnz, &.{
93	.xchg,	124	.{ .reg = inst.data.r_scratch.r1 },
94	.xor,	125	});
95		126	try lower.emit(.none, .setp, &.{
96	.addss,	127	.{ .reg = inst.data.r_scratch.scratch_reg },
97	.andnps,	128	});
98	.andps,	129	try lower.emit(.none, .@"or", &.{
99	.cmpss,	130	.{ .reg = inst.data.r_scratch.r1 },
100	.cvtsi2ss,	131	.{ .reg = inst.data.r_scratch.scratch_reg },
101	.divss,	132	});
102	.maxss,	133	},
103	.minss,	134	.pseudo_set_nz_or_p_m_sib,
104	.movaps,	135	.pseudo_set_nz_or_p_m_rip,
105	.movss,	136	=> {
106	.movups,	137	try lower.emit(.none, .setnz, &.{
107	.mulss,	138	.{ .mem = lower.mem(inst.ops, inst.data.x_scratch.payload) },
108	.orps,	139	});
109	.pextrw,	140	try lower.emit(.none, .setp, &.{
110	.pinsrw,	141	.{ .reg = inst.data.x_scratch.scratch_reg },
111	.roundss,	142	});
112	.sqrtps,	143	try lower.emit(.none, .@"or", &.{
113	.sqrtss,	144	.{ .mem = lower.mem(inst.ops, inst.data.x_scratch.payload) },
114	.subss,	145	.{ .reg = inst.data.x_scratch.scratch_reg },
115	.ucomiss,	146	});
116	.xorps,	147	},
117	.addsd,	148	.pseudo_j_z_and_np_inst => {
118	.andnpd,	149	try lower.emit(.none, .jnz, &.{
119	.andpd,	150	.{ .imm = lower.reloc(.{ .inst = index + 1 }) },
120	.cmpsd,	151	});
121	.cvtsd2ss,	152	try lower.emit(.none, .jnp, &.{
122	.cvtsi2sd,	153	.{ .imm = lower.reloc(.{ .inst = inst.data.inst.inst }) },
123	.cvtss2sd,	154	});
124	.divsd,	155	},
125	.maxsd,	156	.pseudo_j_nz_or_p_inst => {
126	.minsd,	157	try lower.emit(.none, .jnz, &.{
127	.movsd,	158	.{ .imm = lower.reloc(.{ .inst = inst.data.inst.inst }) },
128	.mulsd,	159	});
129	.orpd,	160	try lower.emit(.none, .jp, &.{
130	.roundsd,	161	.{ .imm = lower.reloc(.{ .inst = inst.data.inst.inst }) },
131	.sqrtpd,	162	});
132	.sqrtsd,	163	},
133	.subsd,
134	.ucomisd,
135	.xorpd,
136	=> try lower.mirGeneric(inst),
137
138	.cmps,
139	.lods,
140	.movs,
141	.scas,
142	.stos,
143	=> try lower.mirString(inst),
144
145	.cmpxchgb => try lower.mirCmpxchgBytes(inst),
146
147	.jmp_reloc => try lower.emit(.none, .jmp, &.{.{ .imm = Immediate.s(0) }}),
148
149	.call_extern => try lower.emit(.none, .call, &.{.{ .imm = Immediate.s(0) }}),
150
151	.lea_linker => try lower.mirLeaLinker(inst),
152	.mov_linker => try lower.mirMovLinker(inst),
153
154	.mov_moffs => try lower.mirMovMoffs(inst),
155
156	.movsx => try lower.mirMovsx(inst),
157	.cmovcc => try lower.mirCmovcc(inst),
158	.setcc => try lower.mirSetcc(inst),
159	.jcc => try lower.emit(.none, mnem_cc(.j, inst.data.inst_cc.cc), &.{.{ .imm = Immediate.s(0) }}),
160		164
161	.push_regs => try lower.mirPushPopRegisterList(inst, .push),	165	.pseudo_push_reg_list => try lower.pushPopRegList(.push, inst),
162	.pop_regs => try lower.mirPushPopRegisterList(inst, .pop),	166	.pseudo_pop_reg_list => try lower.pushPopRegList(.pop, inst),
163		167
164	.dbg_line,	168	.pseudo_dbg_prologue_end_none,
165	.dbg_prologue_end,	169	.pseudo_dbg_line_line_column,
166	.dbg_epilogue_begin,	170	.pseudo_dbg_epilogue_begin_none,
167	.dead,	171	.pseudo_dead_none,
168	=> {},	172	=> {},
		173	else => unreachable,
		174	},
169	}	175	}
170		176
171	return lower.result[0..lower.result_len];	177	return .{
		178	.insts = lower.result_insts[0..lower.result_insts_len],
		179	.relocs = lower.result_relocs[0..lower.result_relocs_len],
		180	};
172	}	181	}
173		182
174	pub fn fail(lower: *Lower, comptime format: []const u8, args: anytype) Error {	183	pub fn fail(lower: *Lower, comptime format: []const u8, args: anytype) Error {
...	@@ -178,12 +187,6 @@ pub fn fail(lower: *Lower, comptime format: []const u8, args: anytype) Error {	...	@@ -178,12 +187,6 @@ pub fn fail(lower: *Lower, comptime format: []const u8, args: anytype) Error {
178	return error.LowerFail;	187	return error.LowerFail;
179	}	188	}
180		189
181	fn mnem_cc(comptime base: @Type(.EnumLiteral), cc: bits.Condition) Mnemonic {
182	return switch (cc) {
183	inline else => \|c\| @field(Mnemonic, @tagName(base) ++ @tagName(c)),
184	};
185	}
186
187	fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {	190	fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
188	return switch (ops) {	191	return switch (ops) {
189	.rri_s,	192	.rri_s,
...	@@ -191,21 +194,22 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {	...	@@ -191,21 +194,22 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
191	.i_s,	194	.i_s,
192	.mi_sib_s,	195	.mi_sib_s,
193	.mi_rip_s,	196	.mi_rip_s,
194	.lock_mi_sib_s,
195	.lock_mi_rip_s,
196	=> Immediate.s(@bitCast(i32, i)),	197	=> Immediate.s(@bitCast(i32, i)),
197		198
		199	.rrri,
198	.rri_u,	200	.rri_u,
199	.ri_u,	201	.ri_u,
200	.i_u,	202	.i_u,
201	.mi_sib_u,	203	.mi_sib_u,
202	.mi_rip_u,	204	.mi_rip_u,
203	.lock_mi_sib_u,
204	.lock_mi_rip_u,
205	.rmi_sib,	205	.rmi_sib,
206	.rmi_rip,	206	.rmi_rip,
207	.mri_sib,	207	.mri_sib,
208	.mri_rip,	208	.mri_rip,
		209	.rrm_sib,
		210	.rrm_rip,
		211	.rrmi_sib,
		212	.rrmi_rip,
209	=> Immediate.u(i),	213	=> Immediate.u(i),
210		214
211	.ri64 => Immediate.u(lower.mir.extraData(Mir.Imm64, i).data.decode()),	215	.ri64 => Immediate.u(lower.mir.extraData(Mir.Imm64, i).data.decode()),
...	@@ -217,76 +221,108 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {	...	@@ -217,76 +221,108 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
217	fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {	221	fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
218	return lower.mir.resolveFrameLoc(switch (ops) {	222	return lower.mir.resolveFrameLoc(switch (ops) {
219	.rm_sib,	223	.rm_sib,
220	.rm_sib_cc,
221	.rmi_sib,	224	.rmi_sib,
222	.m_sib,	225	.m_sib,
223	.m_sib_cc,
224	.mi_sib_u,	226	.mi_sib_u,
225	.mi_sib_s,	227	.mi_sib_s,
226	.mr_sib,	228	.mr_sib,
227	.mrr_sib,	229	.mrr_sib,
228	.mri_sib,	230	.mri_sib,
229	.lock_m_sib,	231	.rrm_sib,
230	.lock_mi_sib_u,	232	.rrmi_sib,
231	.lock_mi_sib_s,	233
232	.lock_mr_sib,	234	.pseudo_cmov_nz_or_p_rm_sib,
		235	.pseudo_set_z_and_np_m_sib,
		236	.pseudo_set_nz_or_p_m_sib,
233	=> lower.mir.extraData(Mir.MemorySib, payload).data.decode(),	237	=> lower.mir.extraData(Mir.MemorySib, payload).data.decode(),
234		238
235	.rm_rip,	239	.rm_rip,
236	.rm_rip_cc,
237	.rmi_rip,	240	.rmi_rip,
238	.m_rip,	241	.m_rip,
239	.m_rip_cc,
240	.mi_rip_u,	242	.mi_rip_u,
241	.mi_rip_s,	243	.mi_rip_s,
242	.mr_rip,	244	.mr_rip,
243	.mrr_rip,	245	.mrr_rip,
244	.mri_rip,	246	.mri_rip,
245	.lock_m_rip,	247	.rrm_rip,
246	.lock_mi_rip_u,	248	.rrmi_rip,
247	.lock_mi_rip_s,	249
248	.lock_mr_rip,	250	.pseudo_cmov_nz_or_p_rm_rip,
		251	.pseudo_set_z_and_np_m_rip,
		252	.pseudo_set_nz_or_p_m_rip,
249	=> lower.mir.extraData(Mir.MemoryRip, payload).data.decode(),	253	=> lower.mir.extraData(Mir.MemoryRip, payload).data.decode(),
250		254
251	.rax_moffs,	255	.rax_moffs,
252	.moffs_rax,	256	.moffs_rax,
253	.lock_moffs_rax,
254	=> lower.mir.extraData(Mir.MemoryMoffs, payload).data.decode(),	257	=> lower.mir.extraData(Mir.MemoryMoffs, payload).data.decode(),
255		258
256	else => unreachable,	259	else => unreachable,
257	});	260	});
258	}	261	}
259		262
		263	fn reloc(lower: *Lower, target: Reloc.Target) Immediate {
		264	lower.result_relocs[lower.result_relocs_len] = .{
		265	.lowered_inst_index = lower.result_insts_len,
		266	.target = target,
		267	};
		268	lower.result_relocs_len += 1;
		269	return Immediate.s(0);
		270	}
		271
260	fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) Error!void {	272	fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) Error!void {
261	lower.result[lower.result_len] = try Instruction.new(prefix, mnemonic, ops);	273	lower.result_insts[lower.result_insts_len] = try Instruction.new(prefix, mnemonic, ops);
262	lower.result_len += 1;	274	lower.result_insts_len += 1;
263	}	275	}
264		276
265	fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {	277	fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
266	try lower.emit(switch (inst.ops) {	278	const fixes = switch (inst.ops) {
267	else => .none,	279	.none => inst.data.none.fixes,
268	.lock_m_sib,	280	.inst => inst.data.inst.fixes,
269	.lock_m_rip,	281	.i_s, .i_u => inst.data.i.fixes,
270	.lock_mi_sib_u,	282	.r => inst.data.r.fixes,
271	.lock_mi_rip_u,	283	.rr => inst.data.rr.fixes,
272	.lock_mi_sib_s,	284	.rrr => inst.data.rrr.fixes,
273	.lock_mi_rip_s,	285	.rrri => inst.data.rrri.fixes,
274	.lock_mr_sib,	286	.rri_s, .rri_u => inst.data.rri.fixes,
275	.lock_mr_rip,	287	.ri_s, .ri_u => inst.data.ri.fixes,
276	.lock_moffs_rax,	288	.ri64, .rm_sib, .rm_rip, .mr_sib, .mr_rip => inst.data.rx.fixes,
277	=> .lock,	289	.mrr_sib, .mrr_rip, .rrm_sib, .rrm_rip => inst.data.rrx.fixes,
278	}, switch (inst.tag) {	290	.rmi_sib, .rmi_rip, .mri_sib, .mri_rip => inst.data.rix.fixes,
279	inline else => \|tag\| if (@hasField(Mnemonic, @tagName(tag)))	291	.rrmi_sib, .rrmi_rip => inst.data.rrix.fixes,
280	@field(Mnemonic, @tagName(tag))	292	.mi_sib_u, .mi_rip_u, .mi_sib_s, .mi_rip_s => inst.data.x.fixes,
		293	.m_sib, .m_rip, .rax_moffs, .moffs_rax => inst.data.x.fixes,
		294	.extern_fn_reloc, .got_reloc, .direct_reloc, .import_reloc, .tlv_reloc => ._,
		295	else => return lower.fail("TODO lower .{s}", .{@tagName(inst.ops)}),
		296	};
		297	try lower.emit(switch (fixes) {
		298	inline else => \|tag\| comptime if (std.mem.indexOfScalar(u8, @tagName(tag), ' ')) \|space\|
		299	@field(Prefix, @tagName(tag)[0..space])
281	else	300	else
282	unreachable,	301	.none,
		302	}, mnemonic: {
		303	comptime var max_len = 0;
		304	inline for (@typeInfo(Mnemonic).Enum.fields) \|field\| max_len = @max(field.name.len, max_len);
		305	var buf: [max_len]u8 = undefined;
		306
		307	const fixes_name = @tagName(fixes);
		308	const pattern = fixes_name[if (std.mem.indexOfScalar(u8, fixes_name, ' ')) \|i\| i + 1 else 0..];
		309	const wildcard_i = std.mem.indexOfScalar(u8, pattern, '_').?;
		310	const parts = .{ pattern[0..wildcard_i], @tagName(inst.tag), pattern[wildcard_i + 1 ..] };
		311	const err_msg = "unsupported mnemonic: ";
		312	const mnemonic = std.fmt.bufPrint(&buf, "{s}{s}{s}", parts) catch
		313	return lower.fail(err_msg ++ "'{s}{s}{s}'", parts);
		314	break :mnemonic std.meta.stringToEnum(Mnemonic, mnemonic) orelse
		315	return lower.fail(err_msg ++ "'{s}'", .{mnemonic});
283	}, switch (inst.ops) {	316	}, switch (inst.ops) {
284	.none => &.{},	317	.none => &.{},
		318	.inst => &.{
		319	.{ .imm = lower.reloc(.{ .inst = inst.data.inst.inst }) },
		320	},
285	.i_s, .i_u => &.{	321	.i_s, .i_u => &.{
286	.{ .imm = lower.imm(inst.ops, inst.data.i) },	322	.{ .imm = lower.imm(inst.ops, inst.data.i.i) },
287	},	323	},
288	.r => &.{	324	.r => &.{
289	.{ .reg = inst.data.r },	325	.{ .reg = inst.data.r.r1 },
290	},	326	},
291	.rr => &.{	327	.rr => &.{
292	.{ .reg = inst.data.rr.r1 },	328	.{ .reg = inst.data.rr.r1 },
...	@@ -297,12 +333,18 @@ fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {	...	@@ -297,12 +333,18 @@ fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {
297	.{ .reg = inst.data.rrr.r2 },	333	.{ .reg = inst.data.rrr.r2 },
298	.{ .reg = inst.data.rrr.r3 },	334	.{ .reg = inst.data.rrr.r3 },
299	},	335	},
		336	.rrri => &.{
		337	.{ .reg = inst.data.rrri.r1 },
		338	.{ .reg = inst.data.rrri.r2 },
		339	.{ .reg = inst.data.rrri.r3 },
		340	.{ .imm = lower.imm(inst.ops, inst.data.rrri.i) },
		341	},
300	.ri_s, .ri_u => &.{	342	.ri_s, .ri_u => &.{
301	.{ .reg = inst.data.ri.r },	343	.{ .reg = inst.data.ri.r1 },
302	.{ .imm = lower.imm(inst.ops, inst.data.ri.i) },	344	.{ .imm = lower.imm(inst.ops, inst.data.ri.i) },
303	},	345	},
304	.ri64 => &.{	346	.ri64 => &.{
305	.{ .reg = inst.data.rx.r },	347	.{ .reg = inst.data.rx.r1 },
306	.{ .imm = lower.imm(inst.ops, inst.data.rx.payload) },	348	.{ .imm = lower.imm(inst.ops, inst.data.rx.payload) },
307	},	349	},
308	.rri_s, .rri_u => &.{	350	.rri_s, .rri_u => &.{
...	@@ -310,33 +352,28 @@ fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {	...	@@ -310,33 +352,28 @@ fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {
310	.{ .reg = inst.data.rri.r2 },	352	.{ .reg = inst.data.rri.r2 },
311	.{ .imm = lower.imm(inst.ops, inst.data.rri.i) },	353	.{ .imm = lower.imm(inst.ops, inst.data.rri.i) },
312	},	354	},
313	.m_sib, .lock_m_sib, .m_rip, .lock_m_rip => &.{	355	.m_sib, .m_rip => &.{
314	.{ .mem = lower.mem(inst.ops, inst.data.payload) },	356	.{ .mem = lower.mem(inst.ops, inst.data.x.payload) },
315	},	357	},
316	.mi_sib_s,	358	.mi_sib_s, .mi_sib_u, .mi_rip_u, .mi_rip_s => &.{
317	.lock_mi_sib_s,	359	.{ .mem = lower.mem(inst.ops, inst.data.x.payload + 1) },
318	.mi_sib_u,	360	.{ .imm = lower.imm(
319	.lock_mi_sib_u,	361	inst.ops,
320	.mi_rip_u,	362	lower.mir.extraData(Mir.Imm32, inst.data.x.payload).data.imm,
321	.lock_mi_rip_u,	363	) },
322	.mi_rip_s,
323	.lock_mi_rip_s,
324	=> &.{
325	.{ .mem = lower.mem(inst.ops, inst.data.ix.payload) },
326	.{ .imm = lower.imm(inst.ops, inst.data.ix.i) },
327	},	364	},
328	.rm_sib, .rm_rip => &.{	365	.rm_sib, .rm_rip => &.{
329	.{ .reg = inst.data.rx.r },	366	.{ .reg = inst.data.rx.r1 },
330	.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },	367	.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
331	},	368	},
332	.rmi_sib, .rmi_rip => &.{	369	.rmi_sib, .rmi_rip => &.{
333	.{ .reg = inst.data.rix.r },	370	.{ .reg = inst.data.rix.r1 },
334	.{ .mem = lower.mem(inst.ops, inst.data.rix.payload) },	371	.{ .mem = lower.mem(inst.ops, inst.data.rix.payload) },
335	.{ .imm = lower.imm(inst.ops, inst.data.rix.i) },	372	.{ .imm = lower.imm(inst.ops, inst.data.rix.i) },
336	},	373	},
337	.mr_sib, .lock_mr_sib, .mr_rip, .lock_mr_rip => &.{	374	.mr_sib, .mr_rip => &.{
338	.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },	375	.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
339	.{ .reg = inst.data.rx.r },	376	.{ .reg = inst.data.rx.r1 },
340	},	377	},
341	.mrr_sib, .mrr_rip => &.{	378	.mrr_sib, .mrr_rip => &.{
342	.{ .mem = lower.mem(inst.ops, inst.data.rrx.payload) },	379	.{ .mem = lower.mem(inst.ops, inst.data.rrx.payload) },
...	@@ -345,137 +382,60 @@ fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {	...	@@ -345,137 +382,60 @@ fn mirGeneric(lower: *Lower, inst: Mir.Inst) Error!void {
345	},	382	},
346	.mri_sib, .mri_rip => &.{	383	.mri_sib, .mri_rip => &.{
347	.{ .mem = lower.mem(inst.ops, inst.data.rix.payload) },	384	.{ .mem = lower.mem(inst.ops, inst.data.rix.payload) },
348	.{ .reg = inst.data.rix.r },	385	.{ .reg = inst.data.rix.r1 },
349	.{ .imm = lower.imm(inst.ops, inst.data.rix.i) },	386	.{ .imm = lower.imm(inst.ops, inst.data.rix.i) },
350	},	387	},
351	else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),	388	.rrm_sib, .rrm_rip => &.{
352	});	389	.{ .reg = inst.data.rrx.r1 },
353	}	390	.{ .reg = inst.data.rrx.r2 },
354		391	.{ .mem = lower.mem(inst.ops, inst.data.rrx.payload) },
355	fn mirString(lower: *Lower, inst: Mir.Inst) Error!void {	392	},
356	switch (inst.ops) {	393	.rrmi_sib, .rrmi_rip => &.{
357	.string => try lower.emit(switch (inst.data.string.repeat) {	394	.{ .reg = inst.data.rrix.r1 },
358	inline else => \|repeat\| @field(Prefix, @tagName(repeat)),	395	.{ .reg = inst.data.rrix.r2 },
359	}, switch (inst.tag) {	396	.{ .mem = lower.mem(inst.ops, inst.data.rrix.payload) },
360	inline .cmps, .lods, .movs, .scas, .stos => \|tag\| switch (inst.data.string.width) {	397	.{ .imm = lower.imm(inst.ops, inst.data.rrix.i) },
361	inline else => \|width\| @field(Mnemonic, @tagName(tag) ++ @tagName(width)),
362	},
363	else => unreachable,
364	}, &.{}),
365	else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
366	}
367	}
368
369	fn mirCmpxchgBytes(lower: *Lower, inst: Mir.Inst) Error!void {
370	const ops: [1]Operand = switch (inst.ops) {
371	.m_sib, .lock_m_sib, .m_rip, .lock_m_rip => .{
372	.{ .mem = lower.mem(inst.ops, inst.data.payload) },
373	},	398	},
374	else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
375	};
376	try lower.emit(switch (inst.ops) {
377	.m_sib, .m_rip => .none,
378	.lock_m_sib, .lock_m_rip => .lock,
379	else => unreachable,
380	}, switch (@divExact(ops[0].bitSize(), 8)) {
381	8 => .cmpxchg8b,
382	16 => .cmpxchg16b,
383	else => return lower.fail("invalid operand for {s}", .{@tagName(inst.tag)}),
384	}, &ops);
385	}
386
387	fn mirMovMoffs(lower: *Lower, inst: Mir.Inst) Error!void {
388	try lower.emit(switch (inst.ops) {
389	.rax_moffs, .moffs_rax => .none,
390	.lock_moffs_rax => .lock,
391	else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
392	}, .mov, switch (inst.ops) {
393	.rax_moffs => &.{	399	.rax_moffs => &.{
394	.{ .reg = .rax },	400	.{ .reg = .rax },
395	.{ .mem = lower.mem(inst.ops, inst.data.payload) },	401	.{ .mem = lower.mem(inst.ops, inst.data.x.payload) },
396	},	402	},
397	.moffs_rax, .lock_moffs_rax => &.{	403	.moffs_rax => &.{
398	.{ .mem = lower.mem(inst.ops, inst.data.payload) },	404	.{ .mem = lower.mem(inst.ops, inst.data.x.payload) },
399	.{ .reg = .rax },	405	.{ .reg = .rax },
400	},	406	},
401	else => unreachable,	407	.extern_fn_reloc => &.{
402	});	408	.{ .imm = lower.reloc(.{ .linker_extern_fn = inst.data.reloc }) },
403	}
404
405	fn mirMovsx(lower: *Lower, inst: Mir.Inst) Error!void {
406	const ops: [2]Operand = switch (inst.ops) {
407	.rr => .{
408	.{ .reg = inst.data.rr.r1 },
409	.{ .reg = inst.data.rr.r2 },
410	},
411	.rm_sib, .rm_rip => .{
412	.{ .reg = inst.data.rx.r },
413	.{ .mem = lower.mem(inst.ops, inst.data.rx.payload) },
414	},	409	},
415	else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),	410	.got_reloc, .direct_reloc, .import_reloc, .tlv_reloc => ops: {
416	};	411	const reg = inst.data.rx.r1;
417	try lower.emit(.none, switch (ops[0].bitSize()) {	412	const extra = lower.mir.extraData(Mir.Reloc, inst.data.rx.payload).data;
418	32, 64 => switch (ops[1].bitSize()) {	413	_ = lower.reloc(switch (inst.ops) {
419	32 => .movsxd,	414	.got_reloc => .{ .linker_got = extra },
420	else => .movsx,	415	.direct_reloc => .{ .linker_direct = extra },
		416	.import_reloc => .{ .linker_import = extra },
		417	.tlv_reloc => .{ .linker_tlv = extra },
		418	else => unreachable,
		419	});
		420	break :ops &.{
		421	.{ .reg = reg },
		422	.{ .mem = Memory.rip(Memory.PtrSize.fromBitSize(reg.bitSize()), 0) },
		423	};
421	},	424	},
422	else => .movsx,
423	}, &ops);
424	}
425
426	fn mirCmovcc(lower: *Lower, inst: Mir.Inst) Error!void {
427	switch (inst.ops) {
428	.rr_cc => try lower.emit(.none, mnem_cc(.cmov, inst.data.rr_cc.cc), &.{
429	.{ .reg = inst.data.rr_cc.r1 },
430	.{ .reg = inst.data.rr_cc.r2 },
431	}),
432	.rm_sib_cc, .rm_rip_cc => try lower.emit(.none, mnem_cc(.cmov, inst.data.rx_cc.cc), &.{
433	.{ .reg = inst.data.rx_cc.r },
434	.{ .mem = lower.mem(inst.ops, inst.data.rx_cc.payload) },
435	}),
436	else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),	425	else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
437	}	426	});
438	}
439
440	fn mirSetcc(lower: *Lower, inst: Mir.Inst) Error!void {
441	switch (inst.ops) {
442	.r_cc => try lower.emit(.none, mnem_cc(.set, inst.data.r_cc.cc), &.{
443	.{ .reg = inst.data.r_cc.r },
444	}),
445	.m_sib_cc, .m_rip_cc => try lower.emit(.none, mnem_cc(.set, inst.data.x_cc.cc), &.{
446	.{ .mem = lower.mem(inst.ops, inst.data.x_cc.payload) },
447	}),
448	else => return lower.fail("TODO lower {s} {s}", .{ @tagName(inst.tag), @tagName(inst.ops) }),
449	}
450	}	427	}
451		428
452	fn mirPushPopRegisterList(lower: *Lower, inst: Mir.Inst, comptime mnemonic: Mnemonic) Error!void {	429	fn pushPopRegList(lower: *Lower, comptime mnemonic: Mnemonic, inst: Mir.Inst) Error!void {
453	const reg_list = Mir.RegisterList.fromInt(inst.data.payload);
454	const callee_preserved_regs = abi.getCalleePreservedRegs(lower.target.*);	430	const callee_preserved_regs = abi.getCalleePreservedRegs(lower.target.*);
455	var it = reg_list.iterator(.{ .direction = switch (mnemonic) {	431	var it = inst.data.reg_list.iterator(.{ .direction = switch (mnemonic) {
456	.push => .reverse,	432	.push => .reverse,
457	.pop => .forward,	433	.pop => .forward,
458	else => unreachable,	434	else => unreachable,
459	} });	435	} });
460	while (it.next()) \|i\| try lower.emit(.none, mnemonic, &.{.{ .reg = callee_preserved_regs[i] }});	436	while (it.next()) \|i\| try lower.emit(.none, mnemonic, &.{.{
461	}	437	.reg = callee_preserved_regs[i],
462		438	}});
463	fn mirLeaLinker(lower: *Lower, inst: Mir.Inst) Error!void {
464	const metadata = lower.mir.extraData(Mir.LeaRegisterReloc, inst.data.payload).data;
465	const reg = @intToEnum(Register, metadata.reg);
466	try lower.emit(.none, .lea, &.{
467	.{ .reg = reg },
468	.{ .mem = Memory.rip(Memory.PtrSize.fromBitSize(reg.bitSize()), 0) },
469	});
470	}
471
472	fn mirMovLinker(lower: *Lower, inst: Mir.Inst) Error!void {
473	const metadata = lower.mir.extraData(Mir.LeaRegisterReloc, inst.data.payload).data;
474	const reg = @intToEnum(Register, metadata.reg);
475	try lower.emit(.none, .mov, &.{
476	.{ .reg = reg },
477	.{ .mem = Memory.rip(Memory.PtrSize.fromBitSize(reg.bitSize()), 0) },
478	});
479	}	439	}
480		440
481	const abi = @import("abi.zig");	441	const abi = @import("abi.zig");

src/arch/x86_64/Mir.zig+560-276

...	@@ -32,12 +32,260 @@ pub const Inst = struct {	...	@@ -32,12 +32,260 @@ pub const Inst = struct {
32		32
33	pub const Index = u32;	33	pub const Index = u32;
34		34
		35	pub const Fixes = enum(u8) {
		36	/// ___
		37	@"_",
		38
		39	/// Integer __
		40	i_,
		41
		42	/// ___ Left
		43	_l,
		44	/// ___ Left Double
		45	_ld,
		46	/// ___ Right
		47	_r,
		48	/// ___ Right Double
		49	_rd,
		50
		51	/// ___ Above
		52	_a,
		53	/// ___ Above Or Equal
		54	_ae,
		55	/// ___ Below
		56	_b,
		57	/// ___ Below Or Equal
		58	_be,
		59	/// ___ Carry
		60	_c,
		61	/// ___ Equal
		62	_e,
		63	/// ___ Greater
		64	_g,
		65	/// ___ Greater Or Equal
		66	_ge,
		67	/// ___ Less
		68	//_l,
		69	/// ___ Less Or Equal
		70	_le,
		71	/// ___ Not Above
		72	_na,
		73	/// ___ Not Above Or Equal
		74	_nae,
		75	/// ___ Not Below
		76	_nb,
		77	/// ___ Not Below Or Equal
		78	_nbe,
		79	/// ___ Not Carry
		80	_nc,
		81	/// ___ Not Equal
		82	_ne,
		83	/// ___ Not Greater
		84	_ng,
		85	/// ___ Not Greater Or Equal
		86	_nge,
		87	/// ___ Not Less
		88	_nl,
		89	/// ___ Not Less Or Equal
		90	_nle,
		91	/// ___ Not Overflow
		92	_no,
		93	/// ___ Not Parity
		94	_np,
		95	/// ___ Not Sign
		96	_ns,
		97	/// ___ Not Zero
		98	_nz,
		99	/// ___ Overflow
		100	_o,
		101	/// ___ Parity
		102	_p,
		103	/// ___ Parity Even
		104	_pe,
		105	/// ___ Parity Odd
		106	_po,
		107	/// ___ Sign
		108	_s,
		109	/// ___ Zero
		110	_z,
		111
		112	/// ___ Byte
		113	//_b,
		114	/// ___ Word
		115	_w,
		116	/// ___ Doubleword
		117	_d,
		118	/// ___ QuadWord
		119	_q,
		120
		121	/// ___ String
		122	//_s,
		123	/// ___ String Byte
		124	_sb,
		125	/// ___ String Word
		126	_sw,
		127	/// ___ String Doubleword
		128	_sd,
		129	/// ___ String Quadword
		130	_sq,
		131
		132	/// Repeat ___ String
		133	@"rep _s",
		134	/// Repeat ___ String Byte
		135	@"rep _sb",
		136	/// Repeat ___ String Word
		137	@"rep _sw",
		138	/// Repeat ___ String Doubleword
		139	@"rep _sd",
		140	/// Repeat ___ String Quadword
		141	@"rep _sq",
		142
		143	/// Repeat Equal ___ String
		144	@"repe _s",
		145	/// Repeat Equal ___ String Byte
		146	@"repe _sb",
		147	/// Repeat Equal ___ String Word
		148	@"repe _sw",
		149	/// Repeat Equal ___ String Doubleword
		150	@"repe _sd",
		151	/// Repeat Equal ___ String Quadword
		152	@"repe _sq",
		153
		154	/// Repeat Not Equal ___ String
		155	@"repne _s",
		156	/// Repeat Not Equal ___ String Byte
		157	@"repne _sb",
		158	/// Repeat Not Equal ___ String Word
		159	@"repne _sw",
		160	/// Repeat Not Equal ___ String Doubleword
		161	@"repne _sd",
		162	/// Repeat Not Equal ___ String Quadword
		163	@"repne _sq",
		164
		165	/// Repeat Not Zero ___ String
		166	@"repnz _s",
		167	/// Repeat Not Zero ___ String Byte
		168	@"repnz _sb",
		169	/// Repeat Not Zero ___ String Word
		170	@"repnz _sw",
		171	/// Repeat Not Zero ___ String Doubleword
		172	@"repnz _sd",
		173	/// Repeat Not Zero ___ String Quadword
		174	@"repnz _sq",
		175
		176	/// Repeat Zero ___ String
		177	@"repz _s",
		178	/// Repeat Zero ___ String Byte
		179	@"repz _sb",
		180	/// Repeat Zero ___ String Word
		181	@"repz _sw",
		182	/// Repeat Zero ___ String Doubleword
		183	@"repz _sd",
		184	/// Repeat Zero ___ String Quadword
		185	@"repz _sq",
		186
		187	/// Locked ___
		188	@"lock _",
		189	/// ___ And Complement
		190	//_c,
		191	/// Locked ___ And Complement
		192	@"lock _c",
		193	/// ___ And Reset
		194	//_r,
		195	/// Locked ___ And Reset
		196	@"lock _r",
		197	/// ___ And Set
		198	//_s,
		199	/// Locked ___ And Set
		200	@"lock _s",
		201	/// ___ 8 Bytes
		202	_8b,
		203	/// Locked ___ 8 Bytes
		204	@"lock _8b",
		205	/// ___ 16 Bytes
		206	_16b,
		207	/// Locked ___ 16 Bytes
		208	@"lock _16b",
		209
		210	/// Float ___
		211	f_,
		212	/// Float ___ Pop
		213	f_p,
		214
		215	/// Packed ___
		216	p_,
		217	/// Packed ___ Byte
		218	p_b,
		219	/// Packed ___ Word
		220	p_w,
		221	/// Packed ___ Doubleword
		222	p_d,
		223	/// Packed ___ Quadword
		224	p_q,
		225	/// Packed ___ Double Quadword
		226	p_dq,
		227
		228	/// ___ Scalar Single-Precision Values
		229	_ss,
		230	/// ___ Packed Single-Precision Values
		231	_ps,
		232	/// ___ Scalar Double-Precision Values
		233	//_sd,
		234	/// ___ Packed Double-Precision Values
		235	_pd,
		236
		237	/// VEX-Encoded ___
		238	v_,
		239	/// VEX-Encoded Packed ___
		240	vp_,
		241	/// VEX-Encoded Packed ___ Byte
		242	vp_b,
		243	/// VEX-Encoded Packed ___ Word
		244	vp_w,
		245	/// VEX-Encoded Packed ___ Doubleword
		246	vp_d,
		247	/// VEX-Encoded Packed ___ Quadword
		248	vp_q,
		249	/// VEX-Encoded Packed ___ Double Quadword
		250	vp_dq,
		251	/// VEX-Encoded ___ Scalar Single-Precision Values
		252	v_ss,
		253	/// VEX-Encoded ___ Packed Single-Precision Values
		254	v_ps,
		255	/// VEX-Encoded ___ Scalar Double-Precision Values
		256	v_sd,
		257	/// VEX-Encoded ___ Packed Double-Precision Values
		258	v_pd,
		259
		260	/// Mask ___ Byte
		261	k_b,
		262	/// Mask ___ Word
		263	k_w,
		264	/// Mask ___ Doubleword
		265	k_d,
		266	/// Mask ___ Quadword
		267	k_q,
		268
		269	pub fn fromCondition(cc: bits.Condition) Fixes {
		270	return switch (cc) {
		271	inline else => \|cc_tag\| @field(Fixes, "_" ++ @tagName(cc_tag)),
		272	.z_and_np, .nz_or_p => unreachable,
		273	};
		274	}
		275	};
		276
35	pub const Tag = enum(u8) {	277	pub const Tag = enum(u8) {
36	/// Add with carry	278	/// Add with carry
37	adc,	279	adc,
38	/// Add	280	/// Add
		281	/// Add packed single-precision floating-point values
		282	/// Add scalar single-precision floating-point values
		283	/// Add packed double-precision floating-point values
		284	/// Add scalar double-precision floating-point values
39	add,	285	add,
40	/// Logical and	286	/// Logical and
		287	/// Bitwise logical and of packed single-precision floating-point values
		288	/// Bitwise logical and of packed double-precision floating-point values
41	@"and",	289	@"and",
42	/// Bit scan forward	290	/// Bit scan forward
43	bsf,	291	bsf,
...	@@ -46,49 +294,55 @@ pub const Inst = struct {	...	@@ -46,49 +294,55 @@ pub const Inst = struct {
46	/// Byte swap	294	/// Byte swap
47	bswap,	295	bswap,
48	/// Bit test	296	/// Bit test
49	bt,
50	/// Bit test and complement	297	/// Bit test and complement
51	btc,
52	/// Bit test and reset	298	/// Bit test and reset
53	btr,
54	/// Bit test and set	299	/// Bit test and set
55	bts,	300	bt,
56	/// Call	301	/// Call
57	call,	302	call,
58	/// Convert byte to word	303	/// Convert byte to word
59	cbw,	304	cbw,
60	/// Convert word to doubleword
61	cwde,
62	/// Convert doubleword to quadword
63	cdqe,
64	/// Convert word to doubleword
65	cwd,
66	/// Convert doubleword to quadword	305	/// Convert doubleword to quadword
67	cdq,	306	cdq,
68	/// Convert doubleword to quadword	307	/// Convert doubleword to quadword
69	cqo,	308	cdqe,
		309	/// Conditional move
		310	cmov,
70	/// Logical compare	311	/// Logical compare
		312	/// Compare string
		313	/// Compare scalar single-precision floating-point values
		314	/// Compare scalar double-precision floating-point values
71	cmp,	315	cmp,
72	/// Compare and exchange	316	/// Compare and exchange
73	cmpxchg,
74	/// Compare and exchange bytes	317	/// Compare and exchange bytes
75	cmpxchgb,	318	cmpxchg,
		319	/// Convert doubleword to quadword
		320	cqo,
		321	/// Convert word to doubleword
		322	cwd,
		323	/// Convert word to doubleword
		324	cwde,
76	/// Unsigned division	325	/// Unsigned division
77	div,
78	/// Store integer with truncation
79	fisttp,
80	/// Load floating-point value
81	fld,
82	/// Signed division	326	/// Signed division
83	idiv,	327	/// Divide packed single-precision floating-point values
84	/// Signed multiplication	328	/// Divide scalar single-precision floating-point values
85	imul,	329	/// Divide packed double-precision floating-point values
		330	/// Divide scalar double-precision floating-point values
		331	div,
86	///	332	///
87	int3,	333	int3,
		334	/// Store integer with truncation
		335	istt,
		336	/// Conditional jump
		337	j,
88	/// Jump	338	/// Jump
89	jmp,	339	jmp,
		340	/// Load floating-point value
		341	ld,
90	/// Load effective address	342	/// Load effective address
91	lea,	343	lea,
		344	/// Load string
		345	lod,
92	/// Load fence	346	/// Load fence
93	lfence,	347	lfence,
94	/// Count the number of leading zero bits	348	/// Count the number of leading zero bits
...	@@ -96,18 +350,24 @@ pub const Inst = struct {	...	@@ -96,18 +350,24 @@ pub const Inst = struct {
96	/// Memory fence	350	/// Memory fence
97	mfence,	351	mfence,
98	/// Move	352	/// Move
		353	/// Move data from string to string
		354	/// Move scalar single-precision floating-point value
		355	/// Move scalar double-precision floating-point value
		356	/// Move doubleword
		357	/// Move quadword
99	mov,	358	mov,
100	/// Move data after swapping bytes	359	/// Move data after swapping bytes
101	movbe,	360	movbe,
102	/// Move doubleword
103	movd,
104	/// Move quadword
105	movq,
106	/// Move with sign extension	361	/// Move with sign extension
107	movsx,	362	movsx,
108	/// Move with zero extension	363	/// Move with zero extension
109	movzx,	364	movzx,
110	/// Multiply	365	/// Multiply
		366	/// Signed multiplication
		367	/// Multiply packed single-precision floating-point values
		368	/// Multiply scalar single-precision floating-point values
		369	/// Multiply packed double-precision floating-point values
		370	/// Multiply scalar double-precision floating-point values
111	mul,	371	mul,
112	/// Two's complement negation	372	/// Two's complement negation
113	neg,	373	neg,
...	@@ -116,6 +376,8 @@ pub const Inst = struct {	...	@@ -116,6 +376,8 @@ pub const Inst = struct {
116	/// One's complement negation	376	/// One's complement negation
117	not,	377	not,
118	/// Logical or	378	/// Logical or
		379	/// Bitwise logical or of packed single-precision floating-point values
		380	/// Bitwise logical or of packed double-precision floating-point values
119	@"or",	381	@"or",
120	/// Pop	382	/// Pop
121	pop,	383	pop,
...	@@ -124,33 +386,37 @@ pub const Inst = struct {	...	@@ -124,33 +386,37 @@ pub const Inst = struct {
124	/// Push	386	/// Push
125	push,	387	push,
126	/// Rotate left through carry	388	/// Rotate left through carry
127	rcl,
128	/// Rotate right through carry	389	/// Rotate right through carry
129	rcr,	390	rc,
130	/// Return	391	/// Return
131	ret,	392	ret,
132	/// Rotate left	393	/// Rotate left
133	rol,
134	/// Rotate right	394	/// Rotate right
135	ror,	395	ro,
136	/// Arithmetic shift left	396	/// Arithmetic shift left
137	sal,
138	/// Arithmetic shift right	397	/// Arithmetic shift right
139	sar,	398	sa,
140	/// Integer subtraction with borrow	399	/// Integer subtraction with borrow
141	sbb,	400	sbb,
		401	/// Scan string
		402	sca,
		403	/// Set byte on condition
		404	set,
142	/// Store fence	405	/// Store fence
143	sfence,	406	sfence,
144	/// Logical shift left	407	/// Logical shift left
145	shl,
146	/// Double precision shift left	408	/// Double precision shift left
147	shld,
148	/// Logical shift right	409	/// Logical shift right
149	shr,
150	/// Double precision shift right	410	/// Double precision shift right
151	shrd,	411	sh,
152	/// Subtract	412	/// Subtract
		413	/// Subtract packed single-precision floating-point values
		414	/// Subtract scalar single-precision floating-point values
		415	/// Subtract packed double-precision floating-point values
		416	/// Subtract scalar double-precision floating-point values
153	sub,	417	sub,
		418	/// Store string
		419	sto,
154	/// Syscall	420	/// Syscall
155	syscall,	421	syscall,
156	/// Test condition	422	/// Test condition
...	@@ -164,142 +430,131 @@ pub const Inst = struct {	...	@@ -164,142 +430,131 @@ pub const Inst = struct {
164	/// Exchange register/memory with register	430	/// Exchange register/memory with register
165	xchg,	431	xchg,
166	/// Logical exclusive-or	432	/// Logical exclusive-or
		433	/// Bitwise logical xor of packed single-precision floating-point values
		434	/// Bitwise logical xor of packed double-precision floating-point values
167	xor,	435	xor,
168		436
169	/// Add single precision floating point values	437	/// Bitwise logical and not of packed single-precision floating-point values
170	addss,	438	/// Bitwise logical and not of packed double-precision floating-point values
171	/// Bitwise logical and of packed single precision floating-point values	439	andn,
172	andps,
173	/// Bitwise logical and not of packed single precision floating-point values
174	andnps,
175	/// Compare scalar single-precision floating-point values
176	cmpss,
177	/// Convert doubleword integer to scalar single-precision floating-point value	440	/// Convert doubleword integer to scalar single-precision floating-point value
178	cvtsi2ss,	441	cvtsi2ss,
179	/// Divide scalar single-precision floating-point values	442	/// Maximum of packed single-precision floating-point values
180	divss,	443	/// Maximum of scalar single-precision floating-point values
181	/// Return maximum single-precision floating-point value	444	/// Maximum of packed double-precision floating-point values
182	maxss,	445	/// Maximum of scalar double-precision floating-point values
183	/// Return minimum single-precision floating-point value	446	max,
184	minss,	447	/// Minimum of packed single-precision floating-point values
		448	/// Minimum of scalar single-precision floating-point values
		449	/// Minimum of packed double-precision floating-point values
		450	/// Minimum of scalar double-precision floating-point values
		451	min,
185	/// Move aligned packed single-precision floating-point values	452	/// Move aligned packed single-precision floating-point values
186	movaps,	453	/// Move aligned packed double-precision floating-point values
187	/// Move scalar single-precision floating-point value	454	mova,
188	movss,	455	/// Move packed single-precision floating-point values high to low
		456	movhl,
189	/// Move unaligned packed single-precision floating-point values	457	/// Move unaligned packed single-precision floating-point values
190	movups,	458	/// Move unaligned packed double-precision floating-point values
191	/// Multiply scalar single-precision floating-point values	459	movu,
192	mulss,	460	/// Extract byte
193	/// Bitwise logical or of packed single precision floating-point values
194	orps,
195	/// Extract word	461	/// Extract word
196	pextrw,	462	/// Extract doubleword
		463	/// Extract quadword
		464	extr,
		465	/// Insert byte
197	/// Insert word	466	/// Insert word
198	pinsrw,	467	/// Insert doubleword
199	/// Round scalar single-precision floating-point values	468	/// Insert quadword
200	roundss,	469	insr,
201	/// Square root of scalar single precision floating-point value	470	/// Square root of packed single-precision floating-point values
202	sqrtps,	471	/// Square root of scalar single-precision floating-point value
203	/// Subtract scalar single-precision floating-point values	472	/// Square root of packed double-precision floating-point values
204	sqrtss,	473	/// Square root of scalar double-precision floating-point value
205	/// Square root of single precision floating-point values	474	sqrt,
206	subss,
207	/// Unordered compare scalar single-precision floating-point values	475	/// Unordered compare scalar single-precision floating-point values
208	ucomiss,	476	/// Unordered compare scalar double-precision floating-point values
209	/// Bitwise logical xor of packed single precision floating-point values	477	ucomi,
210	xorps,	478	/// Unpack and interleave high packed single-precision floating-point values
211	/// Add double precision floating point values	479	/// Unpack and interleave high packed double-precision floating-point values
212	addsd,	480	unpckh,
213	/// Bitwise logical and not of packed double precision floating-point values	481	/// Unpack and interleave low packed single-precision floating-point values
214	andnpd,	482	/// Unpack and interleave low packed double-precision floating-point values
215	/// Bitwise logical and of packed double precision floating-point values	483	unpckl,
216	andpd,	484
217	/// Compare scalar double-precision floating-point values
218	cmpsd,
219	/// Convert scalar double-precision floating-point value to scalar single-precision floating-point value	485	/// Convert scalar double-precision floating-point value to scalar single-precision floating-point value
220	cvtsd2ss,	486	cvtsd2ss,
221	/// Convert doubleword integer to scalar double-precision floating-point value	487	/// Convert doubleword integer to scalar double-precision floating-point value
222	cvtsi2sd,	488	cvtsi2sd,
223	/// Convert scalar single-precision floating-point value to scalar double-precision floating-point value	489	/// Convert scalar single-precision floating-point value to scalar double-precision floating-point value
224	cvtss2sd,	490	cvtss2sd,
225	/// Divide scalar double-precision floating-point values	491	/// Shuffle packed high words
226	divsd,	492	shufh,
227	/// Return maximum double-precision floating-point value	493	/// Shuffle packed low words
228	maxsd,	494	shufl,
229	/// Return minimum double-precision floating-point value	495	/// Shift packed data right logical
230	minsd,	496	/// Shift packed data right logical
231	/// Move scalar double-precision floating-point value	497	/// Shift packed data right logical
232	movsd,	498	srl,
233	/// Multiply scalar double-precision floating-point values	499	/// Unpack high data
234	mulsd,	500	unpckhbw,
235	/// Bitwise logical or of packed double precision floating-point values	501	/// Unpack high data
236	orpd,	502	unpckhdq,
237	/// Round scalar double-precision floating-point values	503	/// Unpack high data
238	roundsd,	504	unpckhqdq,
239	/// Square root of double precision floating-point values	505	/// Unpack high data
240	sqrtpd,	506	unpckhwd,
241	/// Square root of scalar double precision floating-point value	507	/// Unpack low data
242	sqrtsd,	508	unpcklbw,
243	/// Subtract scalar double-precision floating-point values	509	/// Unpack low data
244	subsd,	510	unpckldq,
245	/// Unordered compare scalar double-precision floating-point values	511	/// Unpack low data
246	ucomisd,	512	unpcklqdq,
247	/// Bitwise logical xor of packed double precision floating-point values	513	/// Unpack low data
248	xorpd,	514	unpcklwd,
249		515
250	/// Compare string operands	516	/// Replicate double floating-point values
251	cmps,	517	movddup,
252	/// Load string	518	/// Replicate single floating-point values
253	lods,	519	movshdup,
254	/// Move data from string to string	520	/// Replicate single floating-point values
255	movs,	521	movsldup,
256	/// Scan string
257	scas,
258	/// Store string
259	stos,
260
261	/// Conditional move
262	cmovcc,
263	/// Conditional jump
264	jcc,
265	/// Set byte on condition
266	setcc,
267
268	/// Mov absolute to/from memory wrt segment register to/from rax
269	mov_moffs,
270
271	/// Jump with relocation to another local MIR instruction
272	/// Uses `inst` payload.
273	jmp_reloc,
274		522
275	/// Call to an extern symbol via linker relocation.	523	/// Round packed single-precision floating-point values
276	/// Uses `relocation` payload.	524	/// Round scalar single-precision floating-point value
277	call_extern,	525	/// Round packed double-precision floating-point values
		526	/// Round scalar double-precision floating-point value
		527	round,
278		528
279	/// Load effective address of a symbol not yet allocated in VM.	529	/// Convert 16-bit floating-point values to single-precision floating-point values
280	lea_linker,	530	cvtph2ps,
281	/// Move address of a symbol not yet allocated in VM.	531	/// Convert single-precision floating-point values to 16-bit floating-point values
282	mov_linker,	532	cvtps2ph,
283		533
284	/// End of prologue	534	/// Fused multiply-add of packed single-precision floating-point values
285	dbg_prologue_end,	535	/// Fused multiply-add of scalar single-precision floating-point values
286	/// Start of epilogue	536	/// Fused multiply-add of packed double-precision floating-point values
287	dbg_epilogue_begin,	537	/// Fused multiply-add of scalar double-precision floating-point values
288	/// Update debug line	538	fmadd132,
289	/// Uses `line_column` payload containing the line and column.	539	/// Fused multiply-add of packed single-precision floating-point values
290	dbg_line,	540	/// Fused multiply-add of scalar single-precision floating-point values
291	/// Push registers	541	/// Fused multiply-add of packed double-precision floating-point values
292	/// Uses `payload` payload containing `RegisterList.asInt` directly.	542	/// Fused multiply-add of scalar double-precision floating-point values
293	push_regs,	543	fmadd213,
294	/// Pop registers	544	/// Fused multiply-add of packed single-precision floating-point values
295	/// Uses `payload` payload containing `RegisterList.asInt` directly.	545	/// Fused multiply-add of scalar single-precision floating-point values
296	pop_regs,	546	/// Fused multiply-add of packed double-precision floating-point values
		547	/// Fused multiply-add of scalar double-precision floating-point values
		548	fmadd231,
297		549
298	/// Tombstone	550	/// A pseudo instruction that requires special lowering.
299	/// Emitter should skip this instruction.	551	/// This should be the only tag in this enum that doesn't
300	dead,	552	/// directly correspond to one or more instruction mnemonics.
		553	pseudo,
301	};	554	};
302		555
		556	pub const FixedTag = struct { Fixes, Tag };
		557
303	pub const Ops = enum(u8) {	558	pub const Ops = enum(u8) {
304	/// No data associated with this instruction (only mnemonic is used).	559	/// No data associated with this instruction (only mnemonic is used).
305	none,	560	none,
...	@@ -312,18 +567,15 @@ pub const Inst = struct {	...	@@ -312,18 +567,15 @@ pub const Inst = struct {
312	/// Register, register, register operands.	567	/// Register, register, register operands.
313	/// Uses `rrr` payload.	568	/// Uses `rrr` payload.
314	rrr,	569	rrr,
		570	/// Register, register, register, immediate (byte) operands.
		571	/// Uses `rrri` payload.
		572	rrri,
315	/// Register, register, immediate (sign-extended) operands.	573	/// Register, register, immediate (sign-extended) operands.
316	/// Uses `rri` payload.	574	/// Uses `rri` payload.
317	rri_s,	575	rri_s,
318	/// Register, register, immediate (unsigned) operands.	576	/// Register, register, immediate (unsigned) operands.
319	/// Uses `rri` payload.	577	/// Uses `rri` payload.
320	rri_u,	578	rri_u,
321	/// Register with condition code (CC).
322	/// Uses `r_cc` payload.
323	r_cc,
324	/// Register, register with condition code (CC).
325	/// Uses `rr_cc` payload.
326	rr_cc,
327	/// Register, immediate (sign-extended) operands.	579	/// Register, immediate (sign-extended) operands.
328	/// Uses `ri` payload.	580	/// Uses `ri` payload.
329	ri_s,	581	ri_s,
...	@@ -348,41 +600,41 @@ pub const Inst = struct {	...	@@ -348,41 +600,41 @@ pub const Inst = struct {
348	/// Register, memory (RIP) operands.	600	/// Register, memory (RIP) operands.
349	/// Uses `rx` payload.	601	/// Uses `rx` payload.
350	rm_rip,	602	rm_rip,
351	/// Register, memory (SIB) operands with condition code (CC).
352	/// Uses `rx_cc` payload.
353	rm_sib_cc,
354	/// Register, memory (RIP) operands with condition code (CC).
355	/// Uses `rx_cc` payload.
356	rm_rip_cc,
357	/// Register, memory (SIB), immediate (byte) operands.	603	/// Register, memory (SIB), immediate (byte) operands.
358	/// Uses `rix` payload with extra data of type `MemorySib`.	604	/// Uses `rix` payload with extra data of type `MemorySib`.
359	rmi_sib,	605	rmi_sib,
		606	/// Register, register, memory (RIP).
		607	/// Uses `rrix` payload with extra data of type `MemoryRip`.
		608	rrm_rip,
		609	/// Register, register, memory (SIB).
		610	/// Uses `rrix` payload with extra data of type `MemorySib`.
		611	rrm_sib,
		612	/// Register, register, memory (RIP), immediate (byte) operands.
		613	/// Uses `rrix` payload with extra data of type `MemoryRip`.
		614	rrmi_rip,
		615	/// Register, register, memory (SIB), immediate (byte) operands.
		616	/// Uses `rrix` payload with extra data of type `MemorySib`.
		617	rrmi_sib,
360	/// Register, memory (RIP), immediate (byte) operands.	618	/// Register, memory (RIP), immediate (byte) operands.
361	/// Uses `rix` payload with extra data of type `MemoryRip`.	619	/// Uses `rix` payload with extra data of type `MemoryRip`.
362	rmi_rip,	620	rmi_rip,
363	/// Single memory (SIB) operand.	621	/// Single memory (SIB) operand.
364	/// Uses `payload` with extra data of type `MemorySib`.	622	/// Uses `x` with extra data of type `MemorySib`.
365	m_sib,	623	m_sib,
366	/// Single memory (RIP) operand.	624	/// Single memory (RIP) operand.
367	/// Uses `payload` with extra data of type `MemoryRip`.	625	/// Uses `x` with extra data of type `MemoryRip`.
368	m_rip,	626	m_rip,
369	/// Single memory (SIB) operand with condition code (CC).
370	/// Uses `x_cc` with extra data of type `MemorySib`.
371	m_sib_cc,
372	/// Single memory (RIP) operand with condition code (CC).
373	/// Uses `x_cc` with extra data of type `MemoryRip`.
374	m_rip_cc,
375	/// Memory (SIB), immediate (unsigned) operands.	627	/// Memory (SIB), immediate (unsigned) operands.
376	/// Uses `ix` payload with extra data of type `MemorySib`.	628	/// Uses `x` payload with extra data of type `Imm32` followed by `MemorySib`.
377	mi_sib_u,	629	mi_sib_u,
378	/// Memory (RIP), immediate (unsigned) operands.	630	/// Memory (RIP), immediate (unsigned) operands.
379	/// Uses `ix` payload with extra data of type `MemoryRip`.	631	/// Uses `x` payload with extra data of type `Imm32` followed by `MemoryRip`.
380	mi_rip_u,	632	mi_rip_u,
381	/// Memory (SIB), immediate (sign-extend) operands.	633	/// Memory (SIB), immediate (sign-extend) operands.
382	/// Uses `ix` payload with extra data of type `MemorySib`.	634	/// Uses `x` payload with extra data of type `Imm32` followed by `MemorySib`.
383	mi_sib_s,	635	mi_sib_s,
384	/// Memory (RIP), immediate (sign-extend) operands.	636	/// Memory (RIP), immediate (sign-extend) operands.
385	/// Uses `ix` payload with extra data of type `MemoryRip`.	637	/// Uses `x` payload with extra data of type `Imm32` followed by `MemoryRip`.
386	mi_rip_s,	638	mi_rip_s,
387	/// Memory (SIB), register operands.	639	/// Memory (SIB), register operands.
388	/// Uses `rx` payload with extra data of type `MemorySib`.	640	/// Uses `rx` payload with extra data of type `MemorySib`.
...	@@ -403,161 +655,200 @@ pub const Inst = struct {	...	@@ -403,161 +655,200 @@ pub const Inst = struct {
403	/// Uses `rix` payload with extra data of type `MemoryRip`.	655	/// Uses `rix` payload with extra data of type `MemoryRip`.
404	mri_rip,	656	mri_rip,
405	/// Rax, Memory moffs.	657	/// Rax, Memory moffs.
406	/// Uses `payload` with extra data of type `MemoryMoffs`.	658	/// Uses `x` with extra data of type `MemoryMoffs`.
407	rax_moffs,	659	rax_moffs,
408	/// Memory moffs, rax.	660	/// Memory moffs, rax.
409	/// Uses `payload` with extra data of type `MemoryMoffs`.	661	/// Uses `x` with extra data of type `MemoryMoffs`.
410	moffs_rax,	662	moffs_rax,
411	/// Single memory (SIB) operand with lock prefix.
412	/// Uses `payload` with extra data of type `MemorySib`.
413	lock_m_sib,
414	/// Single memory (RIP) operand with lock prefix.
415	/// Uses `payload` with extra data of type `MemoryRip`.
416	lock_m_rip,
417	/// Memory (SIB), immediate (unsigned) operands with lock prefix.
418	/// Uses `xi` payload with extra data of type `MemorySib`.
419	lock_mi_sib_u,
420	/// Memory (RIP), immediate (unsigned) operands with lock prefix.
421	/// Uses `xi` payload with extra data of type `MemoryRip`.
422	lock_mi_rip_u,
423	/// Memory (SIB), immediate (sign-extend) operands with lock prefix.
424	/// Uses `xi` payload with extra data of type `MemorySib`.
425	lock_mi_sib_s,
426	/// Memory (RIP), immediate (sign-extend) operands with lock prefix.
427	/// Uses `xi` payload with extra data of type `MemoryRip`.
428	lock_mi_rip_s,
429	/// Memory (SIB), register operands with lock prefix.
430	/// Uses `rx` payload with extra data of type `MemorySib`.
431	lock_mr_sib,
432	/// Memory (RIP), register operands with lock prefix.
433	/// Uses `rx` payload with extra data of type `MemoryRip`.
434	lock_mr_rip,
435	/// Memory moffs, rax with lock prefix.
436	/// Uses `payload` with extra data of type `MemoryMoffs`.
437	lock_moffs_rax,
438	/// References another Mir instruction directly.	663	/// References another Mir instruction directly.
439	/// Uses `inst` payload.	664	/// Uses `inst` payload.
440	inst,	665	inst,
441	/// References another Mir instruction directly with condition code (CC).	666	/// Linker relocation - external function.
442	/// Uses `inst_cc` payload.
443	inst_cc,
444	/// String repeat and width
445	/// Uses `string` payload.
446	string,
447	/// Uses `reloc` payload.	667	/// Uses `reloc` payload.
448	reloc,	668	extern_fn_reloc,
449	/// Linker relocation - GOT indirection.	669	/// Linker relocation - GOT indirection.
450	/// Uses `payload` payload with extra data of type `LeaRegisterReloc`.	670	/// Uses `rx` payload with extra data of type `Reloc`.
451	got_reloc,	671	got_reloc,
452	/// Linker relocation - direct reference.	672	/// Linker relocation - direct reference.
453	/// Uses `payload` payload with extra data of type `LeaRegisterReloc`.	673	/// Uses `rx` payload with extra data of type `Reloc`.
454	direct_reloc,	674	direct_reloc,
455	/// Linker relocation - imports table indirection (binding).	675	/// Linker relocation - imports table indirection (binding).
456	/// Uses `payload` payload with extra data of type `LeaRegisterReloc`.	676	/// Uses `rx` payload with extra data of type `Reloc`.
457	import_reloc,	677	import_reloc,
458	/// Linker relocation - threadlocal variable via GOT indirection.	678	/// Linker relocation - threadlocal variable via GOT indirection.
459	/// Uses `payload` payload with extra data of type `LeaRegisterReloc`.	679	/// Uses `rx` payload with extra data of type `Reloc`.
460	tlv_reloc,	680	tlv_reloc,
		681
		682	// Pseudo instructions:
		683
		684	/// Conditional move if zero flag set and parity flag not set
		685	/// Clobbers the source operand!
		686	/// Uses `rr` payload.
		687	pseudo_cmov_z_and_np_rr,
		688	/// Conditional move if zero flag not set or parity flag set
		689	/// Uses `rr` payload.
		690	pseudo_cmov_nz_or_p_rr,
		691	/// Conditional move if zero flag not set or parity flag set
		692	/// Uses `rx` payload.
		693	pseudo_cmov_nz_or_p_rm_sib,
		694	/// Conditional move if zero flag not set or parity flag set
		695	/// Uses `rx` payload.
		696	pseudo_cmov_nz_or_p_rm_rip,
		697	/// Set byte if zero flag set and parity flag not set
		698	/// Requires a scratch register!
		699	/// Uses `r_scratch` payload.
		700	pseudo_set_z_and_np_r,
		701	/// Set byte if zero flag set and parity flag not set
		702	/// Requires a scratch register!
		703	/// Uses `x_scratch` payload.
		704	pseudo_set_z_and_np_m_sib,
		705	/// Set byte if zero flag set and parity flag not set
		706	/// Requires a scratch register!
		707	/// Uses `x_scratch` payload.
		708	pseudo_set_z_and_np_m_rip,
		709	/// Set byte if zero flag not set or parity flag set
		710	/// Requires a scratch register!
		711	/// Uses `r_scratch` payload.
		712	pseudo_set_nz_or_p_r,
		713	/// Set byte if zero flag not set or parity flag set
		714	/// Requires a scratch register!
		715	/// Uses `x_scratch` payload.
		716	pseudo_set_nz_or_p_m_sib,
		717	/// Set byte if zero flag not set or parity flag set
		718	/// Requires a scratch register!
		719	/// Uses `x_scratch` payload.
		720	pseudo_set_nz_or_p_m_rip,
		721	/// Jump if zero flag set and parity flag not set
		722	/// Uses `inst` payload.
		723	pseudo_j_z_and_np_inst,
		724	/// Jump if zero flag not set or parity flag set
		725	/// Uses `inst` payload.
		726	pseudo_j_nz_or_p_inst,
		727
		728	/// Push registers
		729	/// Uses `reg_list` payload.
		730	pseudo_push_reg_list,
		731	/// Pop registers
		732	/// Uses `reg_list` payload.
		733	pseudo_pop_reg_list,
		734
		735	/// End of prologue
		736	pseudo_dbg_prologue_end_none,
		737	/// Update debug line
		738	/// Uses `line_column` payload.
		739	pseudo_dbg_line_line_column,
		740	/// Start of epilogue
		741	pseudo_dbg_epilogue_begin_none,
		742
		743	/// Tombstone
		744	/// Emitter should skip this instruction.
		745	pseudo_dead_none,
461	};	746	};
462		747
463	pub const Data = union {	748	pub const Data = union {
		749	none: struct {
		750	fixes: Fixes = ._,
		751	},
464	/// References another Mir instruction.	752	/// References another Mir instruction.
465	inst: Index,	753	inst: struct {
466	/// Another instruction with condition code (CC).	754	fixes: Fixes = ._,
467	/// Used by `jcc`.
468	inst_cc: struct {
469	/// Another instruction.
470	inst: Index,	755	inst: Index,
471	/// A condition code for use with EFLAGS register.
472	cc: bits.Condition,
473	},	756	},
474	/// A 32-bit immediate value.	757	/// A 32-bit immediate value.
475	i: u32,	758	i: struct {
476	r: Register,	759	fixes: Fixes = ._,
		760	i: u32,
		761	},
		762	r: struct {
		763	fixes: Fixes = ._,
		764	r1: Register,
		765	},
477	rr: struct {	766	rr: struct {
		767	fixes: Fixes = ._,
478	r1: Register,	768	r1: Register,
479	r2: Register,	769	r2: Register,
480	},	770	},
481	rrr: struct {	771	rrr: struct {
		772	fixes: Fixes = ._,
482	r1: Register,	773	r1: Register,
483	r2: Register,	774	r2: Register,
484	r3: Register,	775	r3: Register,
485	},	776	},
486	rri: struct {	777	rrri: struct {
		778	fixes: Fixes = ._,
487	r1: Register,	779	r1: Register,
488	r2: Register,	780	r2: Register,
489	i: u32,	781	r3: Register,
490	},	782	i: u8,
491	/// Condition code (CC), followed by custom payload found in extra.
492	x_cc: struct {
493	cc: bits.Condition,
494	payload: u32,
495	},
496	/// Register with condition code (CC).
497	r_cc: struct {
498	r: Register,
499	cc: bits.Condition,
500	},	783	},
501	/// Register, register with condition code (CC).	784	rri: struct {
502	rr_cc: struct {	785	fixes: Fixes = ._,
503	r1: Register,	786	r1: Register,
504	r2: Register,	787	r2: Register,
505	cc: bits.Condition,	788	i: u32,
506	},	789	},
507	/// Register, immediate.	790	/// Register, immediate.
508	ri: struct {	791	ri: struct {
509	r: Register,	792	fixes: Fixes = ._,
		793	r1: Register,
510	i: u32,	794	i: u32,
511	},	795	},
512	/// Register, followed by custom payload found in extra.	796	/// Register, followed by custom payload found in extra.
513	rx: struct {	797	rx: struct {
514	r: Register,	798	fixes: Fixes = ._,
515	payload: u32,	799	r1: Register,
516	},
517	/// Register with condition code (CC), followed by custom payload found in extra.
518	rx_cc: struct {
519	r: Register,
520	cc: bits.Condition,
521	payload: u32,
522	},
523	/// Immediate, followed by Custom payload found in extra.
524	ix: struct {
525	i: u32,
526	payload: u32,	800	payload: u32,
527	},	801	},
528	/// Register, register, followed by Custom payload found in extra.	802	/// Register, register, followed by Custom payload found in extra.
529	rrx: struct {	803	rrx: struct {
		804	fixes: Fixes = ._,
530	r1: Register,	805	r1: Register,
531	r2: Register,	806	r2: Register,
532	payload: u32,	807	payload: u32,
533	},	808	},
534	/// Register, byte immediate, followed by Custom payload found in extra.	809	/// Register, byte immediate, followed by Custom payload found in extra.
535	rix: struct {	810	rix: struct {
536	r: Register,	811	fixes: Fixes = ._,
		812	r1: Register,
		813	i: u8,
		814	payload: u32,
		815	},
		816	/// Register, register, byte immediate, followed by Custom payload found in extra.
		817	rrix: struct {
		818	fixes: Fixes = ._,
		819	r1: Register,
		820	r2: Register,
537	i: u8,	821	i: u8,
538	payload: u32,	822	payload: u32,
539	},	823	},
540	/// String instruction prefix and width.	824	/// Register, scratch register
541	string: struct {	825	r_scratch: struct {
542	repeat: bits.StringRepeat,	826	fixes: Fixes = ._,
543	width: bits.StringWidth,	827	r1: Register,
		828	scratch_reg: Register,
		829	},
		830	/// Scratch register, followed by Custom payload found in extra.
		831	x_scratch: struct {
		832	fixes: Fixes = ._,
		833	scratch_reg: Register,
		834	payload: u32,
		835	},
		836	/// Custom payload found in extra.
		837	x: struct {
		838	fixes: Fixes = ._,
		839	payload: u32,
544	},	840	},
545	/// Relocation for the linker where:	841	/// Relocation for the linker where:
546	/// * `atom_index` is the index of the source	842	/// * `atom_index` is the index of the source
547	/// * `sym_index` is the index of the target	843	/// * `sym_index` is the index of the target
548	relocation: struct {	844	reloc: Reloc,
549	/// Index of the containing atom.
550	atom_index: u32,
551	/// Index into the linker's symbol table.
552	sym_index: u32,
553	},
554	/// Debug line and column position	845	/// Debug line and column position
555	line_column: struct {	846	line_column: struct {
556	line: u32,	847	line: u32,
557	column: u32,	848	column: u32,
558	},	849	},
559	/// Index into `extra`. Meaning of what can be found there is context-dependent.	850	/// Register list
560	payload: u32,	851	reg_list: RegisterList,
561	};	852	};
562		853
563	// Make sure we don't accidentally make instructions bigger than expected.	854	// Make sure we don't accidentally make instructions bigger than expected.
...	@@ -569,9 +860,8 @@ pub const Inst = struct {	...	@@ -569,9 +860,8 @@ pub const Inst = struct {
569	}	860	}
570	};	861	};
571		862
572	pub const LeaRegisterReloc = struct {	863	/// A linker symbol not yet allocated in VM.
573	/// Destination register.	864	pub const Reloc = struct {
574	reg: u32,
575	/// Index of the containing atom.	865	/// Index of the containing atom.
576	atom_index: u32,	866	atom_index: u32,
577	/// Index into the linker's symbol table.	867	/// Index into the linker's symbol table.
...	@@ -606,21 +896,15 @@ pub const RegisterList = struct {	...	@@ -606,21 +896,15 @@ pub const RegisterList = struct {
606	return self.bitset.iterator(options);	896	return self.bitset.iterator(options);
607	}	897	}
608		898
609	pub fn asInt(self: Self) u32 {
610	return self.bitset.mask;
611	}
612
613	pub fn fromInt(mask: u32) Self {
614	return .{
615	.bitset = BitSet{ .mask = @intCast(BitSet.MaskInt, mask) },
616	};
617	}
618
619	pub fn count(self: Self) u32 {	899	pub fn count(self: Self) u32 {
620	return @intCast(u32, self.bitset.count());	900	return @intCast(u32, self.bitset.count());
621	}	901	}
622	};	902	};
623		903
		904	pub const Imm32 = struct {
		905	imm: u32,
		906	};
		907
624	pub const Imm64 = struct {	908	pub const Imm64 = struct {
625	msb: u32,	909	msb: u32,
626	lsb: u32,	910	lsb: u32,

src/arch/x86_64/bits.zig+21-7

 @@ -6,9 +6,6 @@ const Allocator = std.mem.Allocator;
 const ArrayList = std.ArrayList;
 const DW = std.dwarf;
-pub const StringRepeat = enum(u3) { none, rep, repe, repz, repne, repnz };
-pub const StringWidth = enum(u2) { b, w, d, q };
 /// EFLAGS condition codes
 pub const Condition = enum(u5) {
     /// above
 @@ -72,6 +69,12 @@ pub const Condition = enum(u5) {
     /// zero
     z,
+    // Pseudo conditions
+    /// zero and not parity
+    z_and_np,
+    /// not zero or parity
+    nz_or_p,
     /// Converts a std.math.CompareOperator into a condition flag,
     /// i.e. returns the condition that is true iff the result of the
     /// comparison is true. Assumes signed comparison
 @@ -143,6 +146,9 @@ pub const Condition = enum(u5) {
             .po => .pe,
             .s => .ns,
             .z => .nz,
+            .z_and_np => .nz_or_p,
+            .nz_or_p => .z_and_np,
         };
     }
 };
 @@ -476,7 +482,9 @@ pub const Memory = union(enum) {
         dword,
         qword,
         tbyte,
-        dqword,
+        xword,
+        yword,
+        zword,
         pub fn fromSize(size: u32) PtrSize {
             return switch (size) {
 @@ -484,7 +492,9 @@ pub const Memory = union(enum) {
                 2...2 => .word,
                 3...4 => .dword,
                 5...8 => .qword,
-                9...16 => .dqword,
+                9...16 => .xword,
+                17...32 => .yword,
+                33...64 => .zword,
                 else => unreachable,
             };
         }
 @@ -496,7 +506,9 @@ pub const Memory = union(enum) {
                 32 => .dword,
                 64 => .qword,
                 80 => .tbyte,
-                128 => .dqword,
+                128 => .xword,
+                256 => .yword,
+                512 => .zword,
                 else => unreachable,
             };
         }
 @@ -508,7 +520,9 @@ pub const Memory = union(enum) {
                 .dword => 32,
                 .qword => 64,
                 .tbyte => 80,
-                .dqword => 128,
+                .xword => 128,
+                .yword => 256,
+                .zword => 512,
             };
         }
     };

src/arch/x86_64/encoder.zig+154-35

...	@@ -151,15 +151,12 @@ pub const Instruction = struct {	...	@@ -151,15 +151,12 @@ pub const Instruction = struct {
151	moffs.offset,	151	moffs.offset,
152	}),	152	}),
153	},	153	},
154	.imm => \|imm\| try writer.print("0x{x}", .{imm.asUnsigned(enc_op.bitSize())}),	154	.imm => \|imm\| try writer.print("0x{x}", .{imm.asUnsigned(enc_op.immBitSize())}),
155	}	155	}
156	}	156	}
157		157
158	pub fn fmtPrint(op: Operand, enc_op: Encoding.Op) std.fmt.Formatter(fmt) {	158	pub fn fmtPrint(op: Operand, enc_op: Encoding.Op) std.fmt.Formatter(fmt) {
159	return .{ .data = .{	159	return .{ .data = .{ .op = op, .enc_op = enc_op } };
160	.op = op,
161	.enc_op = enc_op,
162	} };
163	}	160	}
164	};	161	};
165		162
...	@@ -209,10 +206,16 @@ pub const Instruction = struct {	...	@@ -209,10 +206,16 @@ pub const Instruction = struct {
209	const enc = inst.encoding;	206	const enc = inst.encoding;
210	const data = enc.data;	207	const data = enc.data;
211		208
212	try inst.encodeLegacyPrefixes(encoder);	209	if (data.mode.isVex()) {
213	try inst.encodeMandatoryPrefix(encoder);	210	try inst.encodeVexPrefix(encoder);
214	try inst.encodeRexPrefix(encoder);	211	const opc = inst.encoding.opcode();
215	try inst.encodeOpcode(encoder);	212	try encoder.opcode_1byte(opc[opc.len - 1]);
		213	} else {
		214	try inst.encodeLegacyPrefixes(encoder);
		215	try inst.encodeMandatoryPrefix(encoder);
		216	try inst.encodeRexPrefix(encoder);
		217	try inst.encodeOpcode(encoder);
		218	}
216		219
217	switch (data.op_en) {	220	switch (data.op_en) {
218	.np, .o => {},	221	.np, .o => {},
...	@@ -222,25 +225,28 @@ pub const Instruction = struct {	...	@@ -222,25 +225,28 @@ pub const Instruction = struct {
222	.td => try encoder.imm64(inst.ops[0].mem.moffs.offset),	225	.td => try encoder.imm64(inst.ops[0].mem.moffs.offset),
223	else => {	226	else => {
224	const mem_op = switch (data.op_en) {	227	const mem_op = switch (data.op_en) {
225	.m, .mi, .m1, .mc, .mr, .mri, .mrc => inst.ops[0],	228	.m, .mi, .m1, .mc, .mr, .mri, .mrc, .mvr => inst.ops[0],
226	.rm, .rmi => inst.ops[1],	229	.rm, .rmi, .vmi => inst.ops[1],
		230	.rvm, .rvmi => inst.ops[2],
227	else => unreachable,	231	else => unreachable,
228	};	232	};
229	switch (mem_op) {	233	switch (mem_op) {
230	.reg => \|reg\| {	234	.reg => \|reg\| {
231	const rm = switch (data.op_en) {	235	const rm = switch (data.op_en) {
232	.m, .mi, .m1, .mc => enc.modRmExt(),	236	.m, .mi, .m1, .mc, .vmi => enc.modRmExt(),
233	.mr, .mri, .mrc => inst.ops[1].reg.lowEnc(),	237	.mr, .mri, .mrc => inst.ops[1].reg.lowEnc(),
234	.rm, .rmi => inst.ops[0].reg.lowEnc(),	238	.rm, .rmi, .rvm, .rvmi => inst.ops[0].reg.lowEnc(),
		239	.mvr => inst.ops[2].reg.lowEnc(),
235	else => unreachable,	240	else => unreachable,
236	};	241	};
237	try encoder.modRm_direct(rm, reg.lowEnc());	242	try encoder.modRm_direct(rm, reg.lowEnc());
238	},	243	},
239	.mem => \|mem\| {	244	.mem => \|mem\| {
240	const op = switch (data.op_en) {	245	const op = switch (data.op_en) {
241	.m, .mi, .m1, .mc => .none,	246	.m, .mi, .m1, .mc, .vmi => .none,
242	.mr, .mri, .mrc => inst.ops[1],	247	.mr, .mri, .mrc => inst.ops[1],
243	.rm, .rmi => inst.ops[0],	248	.rm, .rmi, .rvm, .rvmi => inst.ops[0],
		249	.mvr => inst.ops[2],
244	else => unreachable,	250	else => unreachable,
245	};	251	};
246	try encodeMemory(enc, mem, op, encoder);	252	try encodeMemory(enc, mem, op, encoder);
...	@@ -250,7 +256,8 @@ pub const Instruction = struct {	...	@@ -250,7 +256,8 @@ pub const Instruction = struct {
250		256
251	switch (data.op_en) {	257	switch (data.op_en) {
252	.mi => try encodeImm(inst.ops[1].imm, data.ops[1], encoder),	258	.mi => try encodeImm(inst.ops[1].imm, data.ops[1], encoder),
253	.rmi, .mri => try encodeImm(inst.ops[2].imm, data.ops[2], encoder),	259	.rmi, .mri, .vmi => try encodeImm(inst.ops[2].imm, data.ops[2], encoder),
		260	.rvmi => try encodeImm(inst.ops[3].imm, data.ops[3], encoder),
254	else => {},	261	else => {},
255	}	262	}
256	},	263	},
...	@@ -282,11 +289,9 @@ pub const Instruction = struct {	...	@@ -282,11 +289,9 @@ pub const Instruction = struct {
282	.rep, .repe, .repz => legacy.prefix_f3 = true,	289	.rep, .repe, .repz => legacy.prefix_f3 = true,
283	}	290	}
284		291
285	if (data.mode == .none) {	292	switch (data.mode) {
286	const bit_size = enc.operandBitSize();	293	.short, .rex_short => legacy.set16BitOverride(),
287	if (bit_size == 16) {	294	else => {},
288	legacy.set16BitOverride();
289	}
290	}	295	}
291		296
292	const segment_override: ?Register = switch (op_en) {	297	const segment_override: ?Register = switch (op_en) {
...	@@ -309,6 +314,7 @@ pub const Instruction = struct {	...	@@ -309,6 +314,7 @@ pub const Instruction = struct {
309	}	314	}
310	else	315	else
311	null,	316	null,
		317	.vmi, .rvm, .rvmi, .mvr => unreachable,
312	};	318	};
313	if (segment_override) \|seg\| {	319	if (segment_override) \|seg\| {
314	legacy.setSegmentOverride(seg);	320	legacy.setSegmentOverride(seg);
...	@@ -322,10 +328,7 @@ pub const Instruction = struct {	...	@@ -322,10 +328,7 @@ pub const Instruction = struct {
322		328
323	var rex = Rex{};	329	var rex = Rex{};
324	rex.present = inst.encoding.data.mode == .rex;	330	rex.present = inst.encoding.data.mode == .rex;
325	switch (inst.encoding.data.mode) {	331	rex.w = inst.encoding.data.mode == .long;
326	.long, .sse_long, .sse2_long => rex.w = true,
327	else => {},
328	}
329		332
330	switch (op_en) {	333	switch (op_en) {
331	.np, .i, .zi, .fd, .td, .d => {},	334	.np, .i, .zi, .fd, .td, .d => {},
...	@@ -346,11 +349,71 @@ pub const Instruction = struct {	...	@@ -346,11 +349,71 @@ pub const Instruction = struct {
346	rex.b = b_x_op.isBaseExtended();	349	rex.b = b_x_op.isBaseExtended();
347	rex.x = b_x_op.isIndexExtended();	350	rex.x = b_x_op.isIndexExtended();
348	},	351	},
		352	.vmi, .rvm, .rvmi, .mvr => unreachable,
349	}	353	}
350		354
351	try encoder.rex(rex);	355	try encoder.rex(rex);
352	}	356	}
353		357
		358	fn encodeVexPrefix(inst: Instruction, encoder: anytype) !void {
		359	const op_en = inst.encoding.data.op_en;
		360	const opc = inst.encoding.opcode();
		361	const mand_pre = inst.encoding.mandatoryPrefix();
		362
		363	var vex = Vex{};
		364
		365	vex.w = inst.encoding.data.mode.isLong();
		366
		367	switch (op_en) {
		368	.np, .i, .zi, .fd, .td, .d => {},
		369	.o, .oi => vex.b = inst.ops[0].reg.isExtended(),
		370	.m, .mi, .m1, .mc, .mr, .rm, .rmi, .mri, .mrc, .vmi, .rvm, .rvmi, .mvr => {
		371	const r_op = switch (op_en) {
		372	.rm, .rmi, .rvm, .rvmi => inst.ops[0],
		373	.mr, .mri, .mrc => inst.ops[1],
		374	.mvr => inst.ops[2],
		375	.m, .mi, .m1, .mc, .vmi => .none,
		376	else => unreachable,
		377	};
		378	vex.r = r_op.isBaseExtended();
		379
		380	const b_x_op = switch (op_en) {
		381	.rm, .rmi, .vmi => inst.ops[1],
		382	.m, .mi, .m1, .mc, .mr, .mri, .mrc, .mvr => inst.ops[0],
		383	.rvm, .rvmi => inst.ops[2],
		384	else => unreachable,
		385	};
		386	vex.b = b_x_op.isBaseExtended();
		387	vex.x = b_x_op.isIndexExtended();
		388	},
		389	}
		390
		391	vex.l = inst.encoding.data.mode.isVecLong();
		392
		393	vex.p = if (mand_pre) \|mand\| switch (mand) {
		394	0x66 => .@"66",
		395	0xf2 => .f2,
		396	0xf3 => .f3,
		397	else => unreachable,
		398	} else .none;
		399
		400	const leading: usize = if (mand_pre) \|_\| 1 else 0;
		401	assert(opc[leading] == 0x0f);
		402	vex.m = switch (opc[leading + 1]) {
		403	else => .@"0f",
		404	0x38 => .@"0f38",
		405	0x3a => .@"0f3a",
		406	};
		407
		408	switch (op_en) {
		409	else => {},
		410	.vmi => vex.v = inst.ops[0].reg,
		411	.rvm, .rvmi => vex.v = inst.ops[1].reg,
		412	}
		413
		414	try encoder.vex(vex);
		415	}
		416
354	fn encodeMandatoryPrefix(inst: Instruction, encoder: anytype) !void {	417	fn encodeMandatoryPrefix(inst: Instruction, encoder: anytype) !void {
355	const prefix = inst.encoding.mandatoryPrefix() orelse return;	418	const prefix = inst.encoding.mandatoryPrefix() orelse return;
356	try encoder.opcode_1byte(prefix);	419	try encoder.opcode_1byte(prefix);
...	@@ -443,8 +506,8 @@ pub const Instruction = struct {	...	@@ -443,8 +506,8 @@ pub const Instruction = struct {
443	}	506	}
444		507
445	fn encodeImm(imm: Immediate, kind: Encoding.Op, encoder: anytype) !void {	508	fn encodeImm(imm: Immediate, kind: Encoding.Op, encoder: anytype) !void {
446	const raw = imm.asUnsigned(kind.bitSize());	509	const raw = imm.asUnsigned(kind.immBitSize());
447	switch (kind.bitSize()) {	510	switch (kind.immBitSize()) {
448	8 => try encoder.imm8(@intCast(u8, raw)),	511	8 => try encoder.imm8(@intCast(u8, raw)),
449	16 => try encoder.imm16(@intCast(u16, raw)),	512	16 => try encoder.imm16(@intCast(u16, raw)),
450	32 => try encoder.imm32(@intCast(u32, raw)),	513	32 => try encoder.imm32(@intCast(u32, raw)),
...	@@ -562,17 +625,48 @@ fn Encoder(comptime T: type, comptime opts: Options) type {	...	@@ -562,17 +625,48 @@ fn Encoder(comptime T: type, comptime opts: Options) type {
562	/// or one of reg, index, r/m, base, or opcode-reg might be extended.	625	/// or one of reg, index, r/m, base, or opcode-reg might be extended.
563	///	626	///
564	/// See struct `Rex` for a description of each field.	627	/// See struct `Rex` for a description of each field.
565	pub fn rex(self: Self, byte: Rex) !void {	628	pub fn rex(self: Self, fields: Rex) !void {
566	if (!byte.present and !byte.isSet()) return;	629	if (!fields.present and !fields.isSet()) return;
567		630
568	var value: u8 = 0b0100_0000;	631	var byte: u8 = 0b0100_0000;
569		632
570	if (byte.w) value \|= 0b1000;	633	if (fields.w) byte \|= 0b1000;
571	if (byte.r) value \|= 0b0100;	634	if (fields.r) byte \|= 0b0100;
572	if (byte.x) value \|= 0b0010;	635	if (fields.x) byte \|= 0b0010;
573	if (byte.b) value \|= 0b0001;	636	if (fields.b) byte \|= 0b0001;
574		637
575	try self.writer.writeByte(value);	638	try self.writer.writeByte(byte);
		639	}
		640
		641	/// Encodes a VEX prefix given all the fields
		642	///
		643	/// See struct `Vex` for a description of each field.
		644	pub fn vex(self: Self, fields: Vex) !void {
		645	if (fields.is3Byte()) {
		646	try self.writer.writeByte(0b1100_0100);
		647
		648	try self.writer.writeByte(
		649	@as(u8, ~@boolToInt(fields.r)) << 7 \|
		650	@as(u8, ~@boolToInt(fields.x)) << 6 \|
		651	@as(u8, ~@boolToInt(fields.b)) << 5 \|
		652	@as(u8, @enumToInt(fields.m)) << 0,
		653	);
		654
		655	try self.writer.writeByte(
		656	@as(u8, @boolToInt(fields.w)) << 7 \|
		657	@as(u8, ~fields.v.enc()) << 3 \|
		658	@as(u8, @boolToInt(fields.l)) << 2 \|
		659	@as(u8, @enumToInt(fields.p)) << 0,
		660	);
		661	} else {
		662	try self.writer.writeByte(0b1100_0101);
		663	try self.writer.writeByte(
		664	@as(u8, ~@boolToInt(fields.r)) << 7 \|
		665	@as(u8, ~fields.v.enc()) << 3 \|
		666	@as(u8, @boolToInt(fields.l)) << 2 \|
		667	@as(u8, @enumToInt(fields.p)) << 0,
		668	);
		669	}
576	}	670	}
577		671
578	// ------	672	// ------
...	@@ -848,6 +942,31 @@ pub const Rex = struct {	...	@@ -848,6 +942,31 @@ pub const Rex = struct {
848	}	942	}
849	};	943	};
850		944
		945	pub const Vex = struct {
		946	w: bool = false,
		947	r: bool = false,
		948	x: bool = false,
		949	b: bool = false,
		950	l: bool = false,
		951	p: enum(u2) {
		952	none = 0b00,
		953	@"66" = 0b01,
		954	f3 = 0b10,
		955	f2 = 0b11,
		956	} = .none,
		957	m: enum(u5) {
		958	@"0f" = 0b0_0001,
		959	@"0f38" = 0b0_0010,
		960	@"0f3a" = 0b0_0011,
		961	_,
		962	} = .@"0f",
		963	v: Register = .ymm0,
		964
		965	pub fn is3Byte(vex: Vex) bool {
		966	return vex.w or vex.x or vex.b or vex.m != .@"0f";
		967	}
		968	};
		969
851	// Tests	970	// Tests
852	fn expectEqualHexStrings(expected: []const u8, given: []const u8, assembly: []const u8) !void {	971	fn expectEqualHexStrings(expected: []const u8, given: []const u8, assembly: []const u8) !void {
853	assert(expected.len > 0);	972	assert(expected.len > 0);

src/arch/x86_64/encodings.zig+1156-870

...	@@ -3,933 +3,1219 @@ const Mnemonic = Encoding.Mnemonic;	...	@@ -3,933 +3,1219 @@ const Mnemonic = Encoding.Mnemonic;
3	const OpEn = Encoding.OpEn;	3	const OpEn = Encoding.OpEn;
4	const Op = Encoding.Op;	4	const Op = Encoding.Op;
5	const Mode = Encoding.Mode;	5	const Mode = Encoding.Mode;
		6	const Feature = Encoding.Feature;
6		7
7	const modrm_ext = u3;	8	const modrm_ext = u3;
8		9
9	pub const Entry = struct { Mnemonic, OpEn, []const Op, []const u8, modrm_ext, Mode };	10	pub const Entry = struct { Mnemonic, OpEn, []const Op, []const u8, modrm_ext, Mode, Feature };
10		11
11	// TODO move this into a .zon file when Zig is capable of importing .zon files	12	// TODO move this into a .zon file when Zig is capable of importing .zon files
12	// zig fmt: off	13	// zig fmt: off
13	pub const table = [_]Entry{	14	pub const table = [_]Entry{
14	// General-purpose	15	// General-purpose
15	.{ .adc, .zi, &.{ .al, .imm8 }, &.{ 0x14 }, 0, .none },	16	.{ .adc, .zi, &.{ .al, .imm8 }, &.{ 0x14 }, 0, .none, .none },
16	.{ .adc, .zi, &.{ .ax, .imm16 }, &.{ 0x15 }, 0, .none },	17	.{ .adc, .zi, &.{ .ax, .imm16 }, &.{ 0x15 }, 0, .short, .none },
17	.{ .adc, .zi, &.{ .eax, .imm32 }, &.{ 0x15 }, 0, .none },	18	.{ .adc, .zi, &.{ .eax, .imm32 }, &.{ 0x15 }, 0, .none, .none },
18	.{ .adc, .zi, &.{ .rax, .imm32s }, &.{ 0x15 }, 0, .long },	19	.{ .adc, .zi, &.{ .rax, .imm32s }, &.{ 0x15 }, 0, .long, .none },
19	.{ .adc, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 2, .none },	20	.{ .adc, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 2, .none, .none },
20	.{ .adc, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 2, .rex },	21	.{ .adc, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 2, .rex, .none },
21	.{ .adc, .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 2, .none },	22	.{ .adc, .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 2, .short, .none },
22	.{ .adc, .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 2, .none },	23	.{ .adc, .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 2, .none, .none },
23	.{ .adc, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 2, .long },	24	.{ .adc, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 2, .long, .none },
24	.{ .adc, .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 2, .none },	25	.{ .adc, .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 2, .short, .none },
25	.{ .adc, .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 2, .none },	26	.{ .adc, .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 2, .none, .none },
26	.{ .adc, .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 2, .long },	27	.{ .adc, .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 2, .long, .none },
27	.{ .adc, .mr, &.{ .rm8, .r8 }, &.{ 0x10 }, 0, .none },	28	.{ .adc, .mr, &.{ .rm8, .r8 }, &.{ 0x10 }, 0, .none, .none },
28	.{ .adc, .mr, &.{ .rm8, .r8 }, &.{ 0x10 }, 0, .rex },	29	.{ .adc, .mr, &.{ .rm8, .r8 }, &.{ 0x10 }, 0, .rex, .none },
29	.{ .adc, .mr, &.{ .rm16, .r16 }, &.{ 0x11 }, 0, .none },	30	.{ .adc, .mr, &.{ .rm16, .r16 }, &.{ 0x11 }, 0, .short, .none },
30	.{ .adc, .mr, &.{ .rm32, .r32 }, &.{ 0x11 }, 0, .none },	31	.{ .adc, .mr, &.{ .rm32, .r32 }, &.{ 0x11 }, 0, .none, .none },
31	.{ .adc, .mr, &.{ .rm64, .r64 }, &.{ 0x11 }, 0, .long },	32	.{ .adc, .mr, &.{ .rm64, .r64 }, &.{ 0x11 }, 0, .long, .none },
32	.{ .adc, .rm, &.{ .r8, .rm8 }, &.{ 0x12 }, 0, .none },	33	.{ .adc, .rm, &.{ .r8, .rm8 }, &.{ 0x12 }, 0, .none, .none },
33	.{ .adc, .rm, &.{ .r8, .rm8 }, &.{ 0x12 }, 0, .rex },	34	.{ .adc, .rm, &.{ .r8, .rm8 }, &.{ 0x12 }, 0, .rex, .none },
34	.{ .adc, .rm, &.{ .r16, .rm16 }, &.{ 0x13 }, 0, .none },	35	.{ .adc, .rm, &.{ .r16, .rm16 }, &.{ 0x13 }, 0, .short, .none },
35	.{ .adc, .rm, &.{ .r32, .rm32 }, &.{ 0x13 }, 0, .none },	36	.{ .adc, .rm, &.{ .r32, .rm32 }, &.{ 0x13 }, 0, .none, .none },
36	.{ .adc, .rm, &.{ .r64, .rm64 }, &.{ 0x13 }, 0, .long },	37	.{ .adc, .rm, &.{ .r64, .rm64 }, &.{ 0x13 }, 0, .long, .none },
37		38
38	.{ .add, .zi, &.{ .al, .imm8 }, &.{ 0x04 }, 0, .none },	39	.{ .add, .zi, &.{ .al, .imm8 }, &.{ 0x04 }, 0, .none, .none },
39	.{ .add, .zi, &.{ .ax, .imm16 }, &.{ 0x05 }, 0, .none },	40	.{ .add, .zi, &.{ .ax, .imm16 }, &.{ 0x05 }, 0, .short, .none },
40	.{ .add, .zi, &.{ .eax, .imm32 }, &.{ 0x05 }, 0, .none },	41	.{ .add, .zi, &.{ .eax, .imm32 }, &.{ 0x05 }, 0, .none, .none },
41	.{ .add, .zi, &.{ .rax, .imm32s }, &.{ 0x05 }, 0, .long },	42	.{ .add, .zi, &.{ .rax, .imm32s }, &.{ 0x05 }, 0, .long, .none },
42	.{ .add, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 0, .none },	43	.{ .add, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 0, .none, .none },
43	.{ .add, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 0, .rex },	44	.{ .add, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 0, .rex, .none },
44	.{ .add, .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 0, .none },	45	.{ .add, .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 0, .short, .none },
45	.{ .add, .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 0, .none },	46	.{ .add, .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 0, .none, .none },
46	.{ .add, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 0, .long },	47	.{ .add, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 0, .long, .none },
47	.{ .add, .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 0, .none },	48	.{ .add, .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 0, .short, .none },
48	.{ .add, .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 0, .none },	49	.{ .add, .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 0, .none, .none },
49	.{ .add, .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 0, .long },	50	.{ .add, .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 0, .long, .none },
50	.{ .add, .mr, &.{ .rm8, .r8 }, &.{ 0x00 }, 0, .none },	51	.{ .add, .mr, &.{ .rm8, .r8 }, &.{ 0x00 }, 0, .none, .none },
51	.{ .add, .mr, &.{ .rm8, .r8 }, &.{ 0x00 }, 0, .rex },	52	.{ .add, .mr, &.{ .rm8, .r8 }, &.{ 0x00 }, 0, .rex, .none },
52	.{ .add, .mr, &.{ .rm16, .r16 }, &.{ 0x01 }, 0, .none },	53	.{ .add, .mr, &.{ .rm16, .r16 }, &.{ 0x01 }, 0, .short, .none },
53	.{ .add, .mr, &.{ .rm32, .r32 }, &.{ 0x01 }, 0, .none },	54	.{ .add, .mr, &.{ .rm32, .r32 }, &.{ 0x01 }, 0, .none, .none },
54	.{ .add, .mr, &.{ .rm64, .r64 }, &.{ 0x01 }, 0, .long },	55	.{ .add, .mr, &.{ .rm64, .r64 }, &.{ 0x01 }, 0, .long, .none },
55	.{ .add, .rm, &.{ .r8, .rm8 }, &.{ 0x02 }, 0, .none },	56	.{ .add, .rm, &.{ .r8, .rm8 }, &.{ 0x02 }, 0, .none, .none },
56	.{ .add, .rm, &.{ .r8, .rm8 }, &.{ 0x02 }, 0, .rex },	57	.{ .add, .rm, &.{ .r8, .rm8 }, &.{ 0x02 }, 0, .rex, .none },
57	.{ .add, .rm, &.{ .r16, .rm16 }, &.{ 0x03 }, 0, .none },	58	.{ .add, .rm, &.{ .r16, .rm16 }, &.{ 0x03 }, 0, .short, .none },
58	.{ .add, .rm, &.{ .r32, .rm32 }, &.{ 0x03 }, 0, .none },	59	.{ .add, .rm, &.{ .r32, .rm32 }, &.{ 0x03 }, 0, .none, .none },
59	.{ .add, .rm, &.{ .r64, .rm64 }, &.{ 0x03 }, 0, .long },	60	.{ .add, .rm, &.{ .r64, .rm64 }, &.{ 0x03 }, 0, .long, .none },
60		61
61	.{ .@"and", .zi, &.{ .al, .imm8 }, &.{ 0x24 }, 0, .none },	62	.{ .@"and", .zi, &.{ .al, .imm8 }, &.{ 0x24 }, 0, .none, .none },
62	.{ .@"and", .zi, &.{ .ax, .imm16 }, &.{ 0x25 }, 0, .none },	63	.{ .@"and", .zi, &.{ .ax, .imm16 }, &.{ 0x25 }, 0, .short, .none },
63	.{ .@"and", .zi, &.{ .eax, .imm32 }, &.{ 0x25 }, 0, .none },	64	.{ .@"and", .zi, &.{ .eax, .imm32 }, &.{ 0x25 }, 0, .none, .none },
64	.{ .@"and", .zi, &.{ .rax, .imm32s }, &.{ 0x25 }, 0, .long },	65	.{ .@"and", .zi, &.{ .rax, .imm32s }, &.{ 0x25 }, 0, .long, .none },
65	.{ .@"and", .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 4, .none },	66	.{ .@"and", .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 4, .none, .none },
66	.{ .@"and", .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 4, .rex },	67	.{ .@"and", .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 4, .rex, .none },
67	.{ .@"and", .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 4, .none },	68	.{ .@"and", .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 4, .short, .none },
68	.{ .@"and", .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 4, .none },	69	.{ .@"and", .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 4, .none, .none },
69	.{ .@"and", .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 4, .long },	70	.{ .@"and", .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 4, .long, .none },
70	.{ .@"and", .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 4, .none },	71	.{ .@"and", .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 4, .short, .none },
71	.{ .@"and", .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 4, .none },	72	.{ .@"and", .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 4, .none, .none },
72	.{ .@"and", .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 4, .long },	73	.{ .@"and", .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 4, .long, .none },
73	.{ .@"and", .mr, &.{ .rm8, .r8 }, &.{ 0x20 }, 0, .none },	74	.{ .@"and", .mr, &.{ .rm8, .r8 }, &.{ 0x20 }, 0, .none, .none },
74	.{ .@"and", .mr, &.{ .rm8, .r8 }, &.{ 0x20 }, 0, .rex },	75	.{ .@"and", .mr, &.{ .rm8, .r8 }, &.{ 0x20 }, 0, .rex, .none },
75	.{ .@"and", .mr, &.{ .rm16, .r16 }, &.{ 0x21 }, 0, .none },	76	.{ .@"and", .mr, &.{ .rm16, .r16 }, &.{ 0x21 }, 0, .short, .none },
76	.{ .@"and", .mr, &.{ .rm32, .r32 }, &.{ 0x21 }, 0, .none },	77	.{ .@"and", .mr, &.{ .rm32, .r32 }, &.{ 0x21 }, 0, .none, .none },
77	.{ .@"and", .mr, &.{ .rm64, .r64 }, &.{ 0x21 }, 0, .long },	78	.{ .@"and", .mr, &.{ .rm64, .r64 }, &.{ 0x21 }, 0, .long, .none },
78	.{ .@"and", .rm, &.{ .r8, .rm8 }, &.{ 0x22 }, 0, .none },	79	.{ .@"and", .rm, &.{ .r8, .rm8 }, &.{ 0x22 }, 0, .none, .none },
79	.{ .@"and", .rm, &.{ .r8, .rm8 }, &.{ 0x22 }, 0, .rex },	80	.{ .@"and", .rm, &.{ .r8, .rm8 }, &.{ 0x22 }, 0, .rex, .none },
80	.{ .@"and", .rm, &.{ .r16, .rm16 }, &.{ 0x23 }, 0, .none },	81	.{ .@"and", .rm, &.{ .r16, .rm16 }, &.{ 0x23 }, 0, .short, .none },
81	.{ .@"and", .rm, &.{ .r32, .rm32 }, &.{ 0x23 }, 0, .none },	82	.{ .@"and", .rm, &.{ .r32, .rm32 }, &.{ 0x23 }, 0, .none, .none },
82	.{ .@"and", .rm, &.{ .r64, .rm64 }, &.{ 0x23 }, 0, .long },	83	.{ .@"and", .rm, &.{ .r64, .rm64 }, &.{ 0x23 }, 0, .long, .none },
83		84
84	.{ .bsf, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0xbc }, 0, .none },	85	.{ .bsf, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0xbc }, 0, .short, .none },
85	.{ .bsf, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0xbc }, 0, .none },	86	.{ .bsf, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0xbc }, 0, .none, .none },
86	.{ .bsf, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0xbc }, 0, .long },	87	.{ .bsf, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0xbc }, 0, .long, .none },
87		88
88	.{ .bsr, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0xbd }, 0, .none },	89	.{ .bsr, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0xbd }, 0, .short, .none },
89	.{ .bsr, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0xbd }, 0, .none },	90	.{ .bsr, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0xbd }, 0, .none, .none },
90	.{ .bsr, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0xbd }, 0, .long },	91	.{ .bsr, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0xbd }, 0, .long, .none },
91		92
92	.{ .bswap, .o, &.{ .r32 }, &.{ 0x0f, 0xc8 }, 0, .none },	93	.{ .bswap, .o, &.{ .r32 }, &.{ 0x0f, 0xc8 }, 0, .none, .none },
93	.{ .bswap, .o, &.{ .r64 }, &.{ 0x0f, 0xc8 }, 0, .long },	94	.{ .bswap, .o, &.{ .r64 }, &.{ 0x0f, 0xc8 }, 0, .long, .none },
94		95
95	.{ .bt, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xa3 }, 0, .none },	96	.{ .bt, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xa3 }, 0, .short, .none },
96	.{ .bt, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xa3 }, 0, .none },	97	.{ .bt, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xa3 }, 0, .none, .none },
97	.{ .bt, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xa3 }, 0, .long },	98	.{ .bt, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xa3 }, 0, .long, .none },
98	.{ .bt, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 4, .none },	99	.{ .bt, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 4, .short, .none },
99	.{ .bt, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 4, .none },	100	.{ .bt, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 4, .none, .none },
100	.{ .bt, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 4, .long },	101	.{ .bt, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 4, .long, .none },
101		102
102	.{ .btc, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xbb }, 0, .none },	103	.{ .btc, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xbb }, 0, .short, .none },
103	.{ .btc, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xbb }, 0, .none },	104	.{ .btc, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xbb }, 0, .none, .none },
104	.{ .btc, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xbb }, 0, .long },	105	.{ .btc, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xbb }, 0, .long, .none },
105	.{ .btc, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 7, .none },	106	.{ .btc, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 7, .short, .none },
106	.{ .btc, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 7, .none },	107	.{ .btc, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 7, .none, .none },
107	.{ .btc, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 7, .long },	108	.{ .btc, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 7, .long, .none },
108		109
109	.{ .btr, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xb3 }, 0, .none },	110	.{ .btr, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xb3 }, 0, .short, .none },
110	.{ .btr, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xb3 }, 0, .none },	111	.{ .btr, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xb3 }, 0, .none, .none },
111	.{ .btr, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xb3 }, 0, .long },	112	.{ .btr, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xb3 }, 0, .long, .none },
112	.{ .btr, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 6, .none },	113	.{ .btr, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 6, .short, .none },
113	.{ .btr, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 6, .none },	114	.{ .btr, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 6, .none, .none },
114	.{ .btr, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 6, .long },	115	.{ .btr, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 6, .long, .none },
115		116
116	.{ .bts, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xab }, 0, .none },	117	.{ .bts, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xab }, 0, .short, .none },
117	.{ .bts, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xab }, 0, .none },	118	.{ .bts, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xab }, 0, .none, .none },
118	.{ .bts, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xab }, 0, .long },	119	.{ .bts, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xab }, 0, .long, .none },
119	.{ .bts, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 5, .none },	120	.{ .bts, .mi, &.{ .rm16, .imm8 }, &.{ 0x0f, 0xba }, 5, .short, .none },
120	.{ .bts, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 5, .none },	121	.{ .bts, .mi, &.{ .rm32, .imm8 }, &.{ 0x0f, 0xba }, 5, .none, .none },
121	.{ .bts, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 5, .long },	122	.{ .bts, .mi, &.{ .rm64, .imm8 }, &.{ 0x0f, 0xba }, 5, .long, .none },
122		123
123	// This is M encoding according to Intel, but D makes more sense here.	124	// This is M encoding according to Intel, but D makes more sense here.
124	.{ .call, .d, &.{ .rel32 }, &.{ 0xe8 }, 0, .none },	125	.{ .call, .d, &.{ .rel32 }, &.{ 0xe8 }, 0, .none, .none },
125	.{ .call, .m, &.{ .rm64 }, &.{ 0xff }, 2, .none },	126	.{ .call, .m, &.{ .rm64 }, &.{ 0xff }, 2, .none, .none },
126		127
127	.{ .cbw, .np, &.{ .o16 }, &.{ 0x98 }, 0, .none },	128	.{ .cbw, .np, &.{ .o16 }, &.{ 0x98 }, 0, .short, .none },
128	.{ .cwde, .np, &.{ .o32 }, &.{ 0x98 }, 0, .none },	129	.{ .cwde, .np, &.{ .o32 }, &.{ 0x98 }, 0, .none, .none },
129	.{ .cdqe, .np, &.{ .o64 }, &.{ 0x98 }, 0, .long },	130	.{ .cdqe, .np, &.{ .o64 }, &.{ 0x98 }, 0, .long, .none },
130		131
131	.{ .cwd, .np, &.{ .o16 }, &.{ 0x99 }, 0, .none },	132	.{ .cwd, .np, &.{ .o16 }, &.{ 0x99 }, 0, .short, .none },
132	.{ .cdq, .np, &.{ .o32 }, &.{ 0x99 }, 0, .none },	133	.{ .cdq, .np, &.{ .o32 }, &.{ 0x99 }, 0, .none, .none },
133	.{ .cqo, .np, &.{ .o64 }, &.{ 0x99 }, 0, .long },	134	.{ .cqo, .np, &.{ .o64 }, &.{ 0x99 }, 0, .long, .none },
134		135
135	.{ .cmova, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x47 }, 0, .none },	136	.{ .cmova, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x47 }, 0, .short, .none },
136	.{ .cmova, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x47 }, 0, .none },	137	.{ .cmova, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x47 }, 0, .none, .none },
137	.{ .cmova, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x47 }, 0, .long },	138	.{ .cmova, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x47 }, 0, .long, .none },
138	.{ .cmovae, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .none },	139	.{ .cmovae, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .short, .none },
139	.{ .cmovae, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none },	140	.{ .cmovae, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none, .none },
140	.{ .cmovae, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long },	141	.{ .cmovae, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long, .none },
141	.{ .cmovb, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .none },	142	.{ .cmovb, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .short, .none },
142	.{ .cmovb, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none },	143	.{ .cmovb, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none, .none },
143	.{ .cmovb, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long },	144	.{ .cmovb, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long, .none },
144	.{ .cmovbe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x46 }, 0, .none },	145	.{ .cmovbe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x46 }, 0, .short, .none },
145	.{ .cmovbe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x46 }, 0, .none },	146	.{ .cmovbe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x46 }, 0, .none, .none },
146	.{ .cmovbe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x46 }, 0, .long },	147	.{ .cmovbe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x46 }, 0, .long, .none },
147	.{ .cmovc, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .none },	148	.{ .cmovc, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .short, .none },
148	.{ .cmovc, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none },	149	.{ .cmovc, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none, .none },
149	.{ .cmovc, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long },	150	.{ .cmovc, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long, .none },
150	.{ .cmove, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x44 }, 0, .none },	151	.{ .cmove, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x44 }, 0, .short, .none },
151	.{ .cmove, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x44 }, 0, .none },	152	.{ .cmove, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x44 }, 0, .none, .none },
152	.{ .cmove, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x44 }, 0, .long },	153	.{ .cmove, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x44 }, 0, .long, .none },
153	.{ .cmovg, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4f }, 0, .none },	154	.{ .cmovg, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4f }, 0, .short, .none },
154	.{ .cmovg, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4f }, 0, .none },	155	.{ .cmovg, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4f }, 0, .none, .none },
155	.{ .cmovg, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4f }, 0, .long },	156	.{ .cmovg, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4f }, 0, .long, .none },
156	.{ .cmovge, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4d }, 0, .none },	157	.{ .cmovge, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4d }, 0, .short, .none },
157	.{ .cmovge, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4d }, 0, .none },	158	.{ .cmovge, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4d }, 0, .none, .none },
158	.{ .cmovge, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4d }, 0, .long },	159	.{ .cmovge, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4d }, 0, .long, .none },
159	.{ .cmovl, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4c }, 0, .none },	160	.{ .cmovl, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4c }, 0, .short, .none },
160	.{ .cmovl, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4c }, 0, .none },	161	.{ .cmovl, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4c }, 0, .none, .none },
161	.{ .cmovl, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4c }, 0, .long },	162	.{ .cmovl, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4c }, 0, .long, .none },
162	.{ .cmovle, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4e }, 0, .none },	163	.{ .cmovle, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4e }, 0, .short, .none },
163	.{ .cmovle, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4e }, 0, .none },	164	.{ .cmovle, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4e }, 0, .none, .none },
164	.{ .cmovle, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4e }, 0, .long },	165	.{ .cmovle, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4e }, 0, .long, .none },
165	.{ .cmovna, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x46 }, 0, .none },	166	.{ .cmovna, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x46 }, 0, .short, .none },
166	.{ .cmovna, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x46 }, 0, .none },	167	.{ .cmovna, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x46 }, 0, .none, .none },
167	.{ .cmovna, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x46 }, 0, .long },	168	.{ .cmovna, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x46 }, 0, .long, .none },
168	.{ .cmovnae, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .none },	169	.{ .cmovnae, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x42 }, 0, .short, .none },
169	.{ .cmovnae, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none },	170	.{ .cmovnae, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x42 }, 0, .none, .none },
170	.{ .cmovnae, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long },	171	.{ .cmovnae, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x42 }, 0, .long, .none },
171	.{ .cmovnb, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .none },	172	.{ .cmovnb, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .short, .none },
172	.{ .cmovnb, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none },	173	.{ .cmovnb, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none, .none },
173	.{ .cmovnb, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long },	174	.{ .cmovnb, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long, .none },
174	.{ .cmovnbe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x47 }, 0, .none },	175	.{ .cmovnbe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x47 }, 0, .short, .none },
175	.{ .cmovnbe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x47 }, 0, .none },	176	.{ .cmovnbe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x47 }, 0, .none, .none },
176	.{ .cmovnbe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x47 }, 0, .long },	177	.{ .cmovnbe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x47 }, 0, .long, .none },
177	.{ .cmovnc, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .none },	178	.{ .cmovnc, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x43 }, 0, .short, .none },
178	.{ .cmovnc, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none },	179	.{ .cmovnc, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x43 }, 0, .none, .none },
179	.{ .cmovnc, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long },	180	.{ .cmovnc, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x43 }, 0, .long, .none },
180	.{ .cmovne, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x45 }, 0, .none },	181	.{ .cmovne, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x45 }, 0, .short, .none },
181	.{ .cmovne, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x45 }, 0, .none },	182	.{ .cmovne, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x45 }, 0, .none, .none },
182	.{ .cmovne, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x45 }, 0, .long },	183	.{ .cmovne, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x45 }, 0, .long, .none },
183	.{ .cmovng, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4e }, 0, .none },	184	.{ .cmovng, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4e }, 0, .short, .none },
184	.{ .cmovng, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4e }, 0, .none },	185	.{ .cmovng, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4e }, 0, .none, .none },
185	.{ .cmovng, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4e }, 0, .long },	186	.{ .cmovng, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4e }, 0, .long, .none },
186	.{ .cmovnge, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4c }, 0, .none },	187	.{ .cmovnge, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4c }, 0, .short, .none },
187	.{ .cmovnge, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4c }, 0, .none },	188	.{ .cmovnge, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4c }, 0, .none, .none },
188	.{ .cmovnge, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4c }, 0, .long },	189	.{ .cmovnge, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4c }, 0, .long, .none },
189	.{ .cmovnl, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4d }, 0, .none },	190	.{ .cmovnl, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4d }, 0, .short, .none },
190	.{ .cmovnl, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4d }, 0, .none },	191	.{ .cmovnl, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4d }, 0, .none, .none },
191	.{ .cmovnl, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4d }, 0, .long },	192	.{ .cmovnl, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4d }, 0, .long, .none },
192	.{ .cmovnle, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4f }, 0, .none },	193	.{ .cmovnle, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4f }, 0, .short, .none },
193	.{ .cmovnle, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4f }, 0, .none },	194	.{ .cmovnle, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4f }, 0, .none, .none },
194	.{ .cmovnle, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4f }, 0, .long },	195	.{ .cmovnle, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4f }, 0, .long, .none },
195	.{ .cmovno, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x41 }, 0, .none },	196	.{ .cmovno, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x41 }, 0, .short, .none },
196	.{ .cmovno, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x41 }, 0, .none },	197	.{ .cmovno, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x41 }, 0, .none, .none },
197	.{ .cmovno, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x41 }, 0, .long },	198	.{ .cmovno, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x41 }, 0, .long, .none },
198	.{ .cmovnp, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4b }, 0, .none },	199	.{ .cmovnp, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4b }, 0, .short, .none },
199	.{ .cmovnp, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4b }, 0, .none },	200	.{ .cmovnp, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4b }, 0, .none, .none },
200	.{ .cmovnp, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4b }, 0, .long },	201	.{ .cmovnp, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4b }, 0, .long, .none },
201	.{ .cmovns, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x49 }, 0, .none },	202	.{ .cmovns, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x49 }, 0, .short, .none },
202	.{ .cmovns, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x49 }, 0, .none },	203	.{ .cmovns, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x49 }, 0, .none, .none },
203	.{ .cmovns, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x49 }, 0, .long },	204	.{ .cmovns, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x49 }, 0, .long, .none },
204	.{ .cmovnz, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x45 }, 0, .none },	205	.{ .cmovnz, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x45 }, 0, .short, .none },
205	.{ .cmovnz, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x45 }, 0, .none },	206	.{ .cmovnz, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x45 }, 0, .none, .none },
206	.{ .cmovnz, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x45 }, 0, .long },	207	.{ .cmovnz, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x45 }, 0, .long, .none },
207	.{ .cmovo, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x40 }, 0, .none },	208	.{ .cmovo, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x40 }, 0, .short, .none },
208	.{ .cmovo, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x40 }, 0, .none },	209	.{ .cmovo, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x40 }, 0, .none, .none },
209	.{ .cmovo, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x40 }, 0, .long },	210	.{ .cmovo, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x40 }, 0, .long, .none },
210	.{ .cmovp, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4a }, 0, .none },	211	.{ .cmovp, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4a }, 0, .short, .none },
211	.{ .cmovp, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4a }, 0, .none },	212	.{ .cmovp, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4a }, 0, .none, .none },
212	.{ .cmovp, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4a }, 0, .long },	213	.{ .cmovp, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4a }, 0, .long, .none },
213	.{ .cmovpe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4a }, 0, .none },	214	.{ .cmovpe, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4a }, 0, .short, .none },
214	.{ .cmovpe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4a }, 0, .none },	215	.{ .cmovpe, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4a }, 0, .none, .none },
215	.{ .cmovpe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4a }, 0, .long },	216	.{ .cmovpe, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4a }, 0, .long, .none },
216	.{ .cmovpo, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4b }, 0, .none },	217	.{ .cmovpo, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x4b }, 0, .short, .none },
217	.{ .cmovpo, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4b }, 0, .none },	218	.{ .cmovpo, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x4b }, 0, .none, .none },
218	.{ .cmovpo, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4b }, 0, .long },	219	.{ .cmovpo, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x4b }, 0, .long, .none },
219	.{ .cmovs, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x48 }, 0, .none },	220	.{ .cmovs, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x48 }, 0, .short, .none },
220	.{ .cmovs, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x48 }, 0, .none },	221	.{ .cmovs, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x48 }, 0, .none, .none },
221	.{ .cmovs, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x48 }, 0, .long },	222	.{ .cmovs, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x48 }, 0, .long, .none },
222	.{ .cmovz, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x44 }, 0, .none },	223	.{ .cmovz, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x44 }, 0, .short, .none },
223	.{ .cmovz, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x44 }, 0, .none },	224	.{ .cmovz, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x44 }, 0, .none, .none },
224	.{ .cmovz, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x44 }, 0, .long },	225	.{ .cmovz, .rm, &.{ .r64, .rm64 }, &.{ 0x0f, 0x44 }, 0, .long, .none },
225		226
226	.{ .cmp, .zi, &.{ .al, .imm8 }, &.{ 0x3c }, 0, .none },	227	.{ .cmp, .zi, &.{ .al, .imm8 }, &.{ 0x3c }, 0, .none, .none },
227	.{ .cmp, .zi, &.{ .ax, .imm16 }, &.{ 0x3d }, 0, .none },	228	.{ .cmp, .zi, &.{ .ax, .imm16 }, &.{ 0x3d }, 0, .short, .none },
228	.{ .cmp, .zi, &.{ .eax, .imm32 }, &.{ 0x3d }, 0, .none },	229	.{ .cmp, .zi, &.{ .eax, .imm32 }, &.{ 0x3d }, 0, .none, .none },
229	.{ .cmp, .zi, &.{ .rax, .imm32s }, &.{ 0x3d }, 0, .long },	230	.{ .cmp, .zi, &.{ .rax, .imm32s }, &.{ 0x3d }, 0, .long, .none },
230	.{ .cmp, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 7, .none },	231	.{ .cmp, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 7, .none, .none },
231	.{ .cmp, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 7, .rex },	232	.{ .cmp, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 7, .rex, .none },
232	.{ .cmp, .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 7, .none },	233	.{ .cmp, .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 7, .short, .none },
233	.{ .cmp, .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 7, .none },	234	.{ .cmp, .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 7, .none, .none },
234	.{ .cmp, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 7, .long },	235	.{ .cmp, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 7, .long, .none },
235	.{ .cmp, .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 7, .none },	236	.{ .cmp, .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 7, .short, .none },
236	.{ .cmp, .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 7, .none },	237	.{ .cmp, .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 7, .none, .none },
237	.{ .cmp, .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 7, .long },	238	.{ .cmp, .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 7, .long, .none },
238	.{ .cmp, .mr, &.{ .rm8, .r8 }, &.{ 0x38 }, 0, .none },	239	.{ .cmp, .mr, &.{ .rm8, .r8 }, &.{ 0x38 }, 0, .none, .none },
239	.{ .cmp, .mr, &.{ .rm8, .r8 }, &.{ 0x38 }, 0, .rex },	240	.{ .cmp, .mr, &.{ .rm8, .r8 }, &.{ 0x38 }, 0, .rex, .none },
240	.{ .cmp, .mr, &.{ .rm16, .r16 }, &.{ 0x39 }, 0, .none },	241	.{ .cmp, .mr, &.{ .rm16, .r16 }, &.{ 0x39 }, 0, .short, .none },
241	.{ .cmp, .mr, &.{ .rm32, .r32 }, &.{ 0x39 }, 0, .none },	242	.{ .cmp, .mr, &.{ .rm32, .r32 }, &.{ 0x39 }, 0, .none, .none },
242	.{ .cmp, .mr, &.{ .rm64, .r64 }, &.{ 0x39 }, 0, .long },	243	.{ .cmp, .mr, &.{ .rm64, .r64 }, &.{ 0x39 }, 0, .long, .none },
243	.{ .cmp, .rm, &.{ .r8, .rm8 }, &.{ 0x3a }, 0, .none },	244	.{ .cmp, .rm, &.{ .r8, .rm8 }, &.{ 0x3a }, 0, .none, .none },
244	.{ .cmp, .rm, &.{ .r8, .rm8 }, &.{ 0x3a }, 0, .rex },	245	.{ .cmp, .rm, &.{ .r8, .rm8 }, &.{ 0x3a }, 0, .rex, .none },
245	.{ .cmp, .rm, &.{ .r16, .rm16 }, &.{ 0x3b }, 0, .none },	246	.{ .cmp, .rm, &.{ .r16, .rm16 }, &.{ 0x3b }, 0, .short, .none },
246	.{ .cmp, .rm, &.{ .r32, .rm32 }, &.{ 0x3b }, 0, .none },	247	.{ .cmp, .rm, &.{ .r32, .rm32 }, &.{ 0x3b }, 0, .none, .none },
247	.{ .cmp, .rm, &.{ .r64, .rm64 }, &.{ 0x3b }, 0, .long },	248	.{ .cmp, .rm, &.{ .r64, .rm64 }, &.{ 0x3b }, 0, .long, .none },
248		249
249	.{ .cmps, .np, &.{ .m8, .m8 }, &.{ 0xa6 }, 0, .none },	250	.{ .cmps, .np, &.{ .m8, .m8 }, &.{ 0xa6 }, 0, .none, .none },
250	.{ .cmps, .np, &.{ .m16, .m16 }, &.{ 0xa7 }, 0, .none },	251	.{ .cmps, .np, &.{ .m16, .m16 }, &.{ 0xa7 }, 0, .short, .none },
251	.{ .cmps, .np, &.{ .m32, .m32 }, &.{ 0xa7 }, 0, .none },	252	.{ .cmps, .np, &.{ .m32, .m32 }, &.{ 0xa7 }, 0, .none, .none },
252	.{ .cmps, .np, &.{ .m64, .m64 }, &.{ 0xa7 }, 0, .long },	253	.{ .cmps, .np, &.{ .m64, .m64 }, &.{ 0xa7 }, 0, .long, .none },
253		254
254	.{ .cmpsb, .np, &.{}, &.{ 0xa6 }, 0, .none },	255	.{ .cmpsb, .np, &.{}, &.{ 0xa6 }, 0, .none, .none },
255	.{ .cmpsw, .np, &.{}, &.{ 0xa7 }, 0, .short },	256	.{ .cmpsw, .np, &.{}, &.{ 0xa7 }, 0, .short, .none },
256	.{ .cmpsd, .np, &.{}, &.{ 0xa7 }, 0, .none },	257	.{ .cmpsd, .np, &.{}, &.{ 0xa7 }, 0, .none, .none },
257	.{ .cmpsq, .np, &.{}, &.{ 0xa7 }, 0, .long },	258	.{ .cmpsq, .np, &.{}, &.{ 0xa7 }, 0, .long, .none },
258		259
259	.{ .cmpxchg, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xb0 }, 0, .none },	260	.{ .cmpxchg, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xb0 }, 0, .none, .none },
260	.{ .cmpxchg, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xb0 }, 0, .rex },	261	.{ .cmpxchg, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xb0 }, 0, .rex, .none },
261	.{ .cmpxchg, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xb1 }, 0, .none },	262	.{ .cmpxchg, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xb1 }, 0, .short, .none },
262	.{ .cmpxchg, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xb1 }, 0, .none },	263	.{ .cmpxchg, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xb1 }, 0, .none, .none },
263	.{ .cmpxchg, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xb1 }, 0, .long },	264	.{ .cmpxchg, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xb1 }, 0, .long, .none },
264		265
265	.{ .cmpxchg8b , .m, &.{ .m64 }, &.{ 0x0f, 0xc7 }, 1, .none },	266	.{ .cmpxchg8b, .m, &.{ .m64 }, &.{ 0x0f, 0xc7 }, 1, .none, .none },
266	.{ .cmpxchg16b, .m, &.{ .m128 }, &.{ 0x0f, 0xc7 }, 1, .long },	267	.{ .cmpxchg16b, .m, &.{ .m128 }, &.{ 0x0f, 0xc7 }, 1, .long, .none },
267		268
268	.{ .div, .m, &.{ .rm8 }, &.{ 0xf6 }, 6, .none },	269	.{ .div, .m, &.{ .rm8 }, &.{ 0xf6 }, 6, .none, .none },
269	.{ .div, .m, &.{ .rm8 }, &.{ 0xf6 }, 6, .rex },	270	.{ .div, .m, &.{ .rm8 }, &.{ 0xf6 }, 6, .rex, .none },
270	.{ .div, .m, &.{ .rm16 }, &.{ 0xf7 }, 6, .none },	271	.{ .div, .m, &.{ .rm16 }, &.{ 0xf7 }, 6, .short, .none },
271	.{ .div, .m, &.{ .rm32 }, &.{ 0xf7 }, 6, .none },	272	.{ .div, .m, &.{ .rm32 }, &.{ 0xf7 }, 6, .none, .none },
272	.{ .div, .m, &.{ .rm64 }, &.{ 0xf7 }, 6, .long },	273	.{ .div, .m, &.{ .rm64 }, &.{ 0xf7 }, 6, .long, .none },
273		274
274	.{ .fisttp, .m, &.{ .m16 }, &.{ 0xdf }, 1, .fpu },	275	.{ .fisttp, .m, &.{ .m16 }, &.{ 0xdf }, 1, .none, .x87 },
275	.{ .fisttp, .m, &.{ .m32 }, &.{ 0xdb }, 1, .fpu },	276	.{ .fisttp, .m, &.{ .m32 }, &.{ 0xdb }, 1, .none, .x87 },
276	.{ .fisttp, .m, &.{ .m64 }, &.{ 0xdd }, 1, .fpu },	277	.{ .fisttp, .m, &.{ .m64 }, &.{ 0xdd }, 1, .none, .x87 },
277		278
278	.{ .fld, .m, &.{ .m32 }, &.{ 0xd9 }, 0, .fpu },	279	.{ .fld, .m, &.{ .m32 }, &.{ 0xd9 }, 0, .none, .x87 },
279	.{ .fld, .m, &.{ .m64 }, &.{ 0xdd }, 0, .fpu },	280	.{ .fld, .m, &.{ .m64 }, &.{ 0xdd }, 0, .none, .x87 },
280	.{ .fld, .m, &.{ .m80 }, &.{ 0xdb }, 5, .fpu },	281	.{ .fld, .m, &.{ .m80 }, &.{ 0xdb }, 5, .none, .x87 },
281		282
282	.{ .idiv, .m, &.{ .rm8 }, &.{ 0xf6 }, 7, .none },	283	.{ .idiv, .m, &.{ .rm8 }, &.{ 0xf6 }, 7, .none, .none },
283	.{ .idiv, .m, &.{ .rm8 }, &.{ 0xf6 }, 7, .rex },	284	.{ .idiv, .m, &.{ .rm8 }, &.{ 0xf6 }, 7, .rex, .none },
284	.{ .idiv, .m, &.{ .rm16 }, &.{ 0xf7 }, 7, .none },	285	.{ .idiv, .m, &.{ .rm16 }, &.{ 0xf7 }, 7, .short, .none },
285	.{ .idiv, .m, &.{ .rm32 }, &.{ 0xf7 }, 7, .none },	286	.{ .idiv, .m, &.{ .rm32 }, &.{ 0xf7 }, 7, .none, .none },
286	.{ .idiv, .m, &.{ .rm64 }, &.{ 0xf7 }, 7, .long },	287	.{ .idiv, .m, &.{ .rm64 }, &.{ 0xf7 }, 7, .long, .none },
287		288
288	.{ .imul, .m, &.{ .rm8 }, &.{ 0xf6 }, 5, .none },	289	.{ .imul, .m, &.{ .rm8 }, &.{ 0xf6 }, 5, .none, .none },
289	.{ .imul, .m, &.{ .rm8 }, &.{ 0xf6 }, 5, .rex },	290	.{ .imul, .m, &.{ .rm8 }, &.{ 0xf6 }, 5, .rex, .none },
290	.{ .imul, .m, &.{ .rm16, }, &.{ 0xf7 }, 5, .none },	291	.{ .imul, .m, &.{ .rm16, }, &.{ 0xf7 }, 5, .short, .none },
291	.{ .imul, .m, &.{ .rm32, }, &.{ 0xf7 }, 5, .none },	292	.{ .imul, .m, &.{ .rm32, }, &.{ 0xf7 }, 5, .none, .none },
292	.{ .imul, .m, &.{ .rm64, }, &.{ 0xf7 }, 5, .long },	293	.{ .imul, .m, &.{ .rm64, }, &.{ 0xf7 }, 5, .long, .none },
293	.{ .imul, .rm, &.{ .r16, .rm16, }, &.{ 0x0f, 0xaf }, 0, .none },	294	.{ .imul, .rm, &.{ .r16, .rm16, }, &.{ 0x0f, 0xaf }, 0, .short, .none },
294	.{ .imul, .rm, &.{ .r32, .rm32, }, &.{ 0x0f, 0xaf }, 0, .none },	295	.{ .imul, .rm, &.{ .r32, .rm32, }, &.{ 0x0f, 0xaf }, 0, .none, .none },
295	.{ .imul, .rm, &.{ .r64, .rm64, }, &.{ 0x0f, 0xaf }, 0, .long },	296	.{ .imul, .rm, &.{ .r64, .rm64, }, &.{ 0x0f, 0xaf }, 0, .long, .none },
296	.{ .imul, .rmi, &.{ .r16, .rm16, .imm8s }, &.{ 0x6b }, 0, .none },	297	.{ .imul, .rmi, &.{ .r16, .rm16, .imm8s }, &.{ 0x6b }, 0, .short, .none },
297	.{ .imul, .rmi, &.{ .r32, .rm32, .imm8s }, &.{ 0x6b }, 0, .none },	298	.{ .imul, .rmi, &.{ .r32, .rm32, .imm8s }, &.{ 0x6b }, 0, .none, .none },
298	.{ .imul, .rmi, &.{ .r64, .rm64, .imm8s }, &.{ 0x6b }, 0, .long },	299	.{ .imul, .rmi, &.{ .r64, .rm64, .imm8s }, &.{ 0x6b }, 0, .long, .none },
299	.{ .imul, .rmi, &.{ .r16, .rm16, .imm16 }, &.{ 0x69 }, 0, .none },	300	.{ .imul, .rmi, &.{ .r16, .rm16, .imm16 }, &.{ 0x69 }, 0, .short, .none },
300	.{ .imul, .rmi, &.{ .r32, .rm32, .imm32 }, &.{ 0x69 }, 0, .none },	301	.{ .imul, .rmi, &.{ .r32, .rm32, .imm32 }, &.{ 0x69 }, 0, .none, .none },
301	.{ .imul, .rmi, &.{ .r64, .rm64, .imm32 }, &.{ 0x69 }, 0, .long },	302	.{ .imul, .rmi, &.{ .r64, .rm64, .imm32 }, &.{ 0x69 }, 0, .long, .none },
302		303
303	.{ .int3, .np, &.{}, &.{ 0xcc }, 0, .none },	304	.{ .int3, .np, &.{}, &.{ 0xcc }, 0, .none, .none },
304		305
305	.{ .ja, .d, &.{ .rel32 }, &.{ 0x0f, 0x87 }, 0, .none },	306	.{ .ja, .d, &.{ .rel32 }, &.{ 0x0f, 0x87 }, 0, .none, .none },
306	.{ .jae, .d, &.{ .rel32 }, &.{ 0x0f, 0x83 }, 0, .none },	307	.{ .jae, .d, &.{ .rel32 }, &.{ 0x0f, 0x83 }, 0, .none, .none },
307	.{ .jb, .d, &.{ .rel32 }, &.{ 0x0f, 0x82 }, 0, .none },	308	.{ .jb, .d, &.{ .rel32 }, &.{ 0x0f, 0x82 }, 0, .none, .none },
308	.{ .jbe, .d, &.{ .rel32 }, &.{ 0x0f, 0x86 }, 0, .none },	309	.{ .jbe, .d, &.{ .rel32 }, &.{ 0x0f, 0x86 }, 0, .none, .none },
309	.{ .jc, .d, &.{ .rel32 }, &.{ 0x0f, 0x82 }, 0, .none },	310	.{ .jc, .d, &.{ .rel32 }, &.{ 0x0f, 0x82 }, 0, .none, .none },
310	.{ .jrcxz, .d, &.{ .rel32 }, &.{ 0xe3 }, 0, .none },	311	.{ .jrcxz, .d, &.{ .rel32 }, &.{ 0xe3 }, 0, .none, .none },
311	.{ .je, .d, &.{ .rel32 }, &.{ 0x0f, 0x84 }, 0, .none },	312	.{ .je, .d, &.{ .rel32 }, &.{ 0x0f, 0x84 }, 0, .none, .none },
312	.{ .jg, .d, &.{ .rel32 }, &.{ 0x0f, 0x8f }, 0, .none },	313	.{ .jg, .d, &.{ .rel32 }, &.{ 0x0f, 0x8f }, 0, .none, .none },
313	.{ .jge, .d, &.{ .rel32 }, &.{ 0x0f, 0x8d }, 0, .none },	314	.{ .jge, .d, &.{ .rel32 }, &.{ 0x0f, 0x8d }, 0, .none, .none },
314	.{ .jl, .d, &.{ .rel32 }, &.{ 0x0f, 0x8c }, 0, .none },	315	.{ .jl, .d, &.{ .rel32 }, &.{ 0x0f, 0x8c }, 0, .none, .none },
315	.{ .jle, .d, &.{ .rel32 }, &.{ 0x0f, 0x8e }, 0, .none },	316	.{ .jle, .d, &.{ .rel32 }, &.{ 0x0f, 0x8e }, 0, .none, .none },
316	.{ .jna, .d, &.{ .rel32 }, &.{ 0x0f, 0x86 }, 0, .none },	317	.{ .jna, .d, &.{ .rel32 }, &.{ 0x0f, 0x86 }, 0, .none, .none },
317	.{ .jnae, .d, &.{ .rel32 }, &.{ 0x0f, 0x82 }, 0, .none },	318	.{ .jnae, .d, &.{ .rel32 }, &.{ 0x0f, 0x82 }, 0, .none, .none },
318	.{ .jnb, .d, &.{ .rel32 }, &.{ 0x0f, 0x83 }, 0, .none },	319	.{ .jnb, .d, &.{ .rel32 }, &.{ 0x0f, 0x83 }, 0, .none, .none },
319	.{ .jnbe, .d, &.{ .rel32 }, &.{ 0x0f, 0x87 }, 0, .none },	320	.{ .jnbe, .d, &.{ .rel32 }, &.{ 0x0f, 0x87 }, 0, .none, .none },
320	.{ .jnc, .d, &.{ .rel32 }, &.{ 0x0f, 0x83 }, 0, .none },	321	.{ .jnc, .d, &.{ .rel32 }, &.{ 0x0f, 0x83 }, 0, .none, .none },
321	.{ .jne, .d, &.{ .rel32 }, &.{ 0x0f, 0x85 }, 0, .none },	322	.{ .jne, .d, &.{ .rel32 }, &.{ 0x0f, 0x85 }, 0, .none, .none },
322	.{ .jng, .d, &.{ .rel32 }, &.{ 0x0f, 0x8e }, 0, .none },	323	.{ .jng, .d, &.{ .rel32 }, &.{ 0x0f, 0x8e }, 0, .none, .none },
323	.{ .jnge, .d, &.{ .rel32 }, &.{ 0x0f, 0x8c }, 0, .none },	324	.{ .jnge, .d, &.{ .rel32 }, &.{ 0x0f, 0x8c }, 0, .none, .none },
324	.{ .jnl, .d, &.{ .rel32 }, &.{ 0x0f, 0x8d }, 0, .none },	325	.{ .jnl, .d, &.{ .rel32 }, &.{ 0x0f, 0x8d }, 0, .none, .none },
325	.{ .jnle, .d, &.{ .rel32 }, &.{ 0x0f, 0x8f }, 0, .none },	326	.{ .jnle, .d, &.{ .rel32 }, &.{ 0x0f, 0x8f }, 0, .none, .none },
326	.{ .jno, .d, &.{ .rel32 }, &.{ 0x0f, 0x81 }, 0, .none },	327	.{ .jno, .d, &.{ .rel32 }, &.{ 0x0f, 0x81 }, 0, .none, .none },
327	.{ .jnp, .d, &.{ .rel32 }, &.{ 0x0f, 0x8b }, 0, .none },	328	.{ .jnp, .d, &.{ .rel32 }, &.{ 0x0f, 0x8b }, 0, .none, .none },
328	.{ .jns, .d, &.{ .rel32 }, &.{ 0x0f, 0x89 }, 0, .none },	329	.{ .jns, .d, &.{ .rel32 }, &.{ 0x0f, 0x89 }, 0, .none, .none },
329	.{ .jnz, .d, &.{ .rel32 }, &.{ 0x0f, 0x85 }, 0, .none },	330	.{ .jnz, .d, &.{ .rel32 }, &.{ 0x0f, 0x85 }, 0, .none, .none },
330	.{ .jo, .d, &.{ .rel32 }, &.{ 0x0f, 0x80 }, 0, .none },	331	.{ .jo, .d, &.{ .rel32 }, &.{ 0x0f, 0x80 }, 0, .none, .none },
331	.{ .jp, .d, &.{ .rel32 }, &.{ 0x0f, 0x8a }, 0, .none },	332	.{ .jp, .d, &.{ .rel32 }, &.{ 0x0f, 0x8a }, 0, .none, .none },
332	.{ .jpe, .d, &.{ .rel32 }, &.{ 0x0f, 0x8a }, 0, .none },	333	.{ .jpe, .d, &.{ .rel32 }, &.{ 0x0f, 0x8a }, 0, .none, .none },
333	.{ .jpo, .d, &.{ .rel32 }, &.{ 0x0f, 0x8b }, 0, .none },	334	.{ .jpo, .d, &.{ .rel32 }, &.{ 0x0f, 0x8b }, 0, .none, .none },
334	.{ .js, .d, &.{ .rel32 }, &.{ 0x0f, 0x88 }, 0, .none },	335	.{ .js, .d, &.{ .rel32 }, &.{ 0x0f, 0x88 }, 0, .none, .none },
335	.{ .jz, .d, &.{ .rel32 }, &.{ 0x0f, 0x84 }, 0, .none },	336	.{ .jz, .d, &.{ .rel32 }, &.{ 0x0f, 0x84 }, 0, .none, .none },
336		337
337	.{ .jmp, .d, &.{ .rel32 }, &.{ 0xe9 }, 0, .none },	338	.{ .jmp, .d, &.{ .rel32 }, &.{ 0xe9 }, 0, .none, .none },
338	.{ .jmp, .m, &.{ .rm64 }, &.{ 0xff }, 4, .none },	339	.{ .jmp, .m, &.{ .rm64 }, &.{ 0xff }, 4, .none, .none },
339		340
340	.{ .lea, .rm, &.{ .r16, .m }, &.{ 0x8d }, 0, .none },	341	.{ .lea, .rm, &.{ .r16, .m }, &.{ 0x8d }, 0, .short, .none },
341	.{ .lea, .rm, &.{ .r32, .m }, &.{ 0x8d }, 0, .none },	342	.{ .lea, .rm, &.{ .r32, .m }, &.{ 0x8d }, 0, .none, .none },
342	.{ .lea, .rm, &.{ .r64, .m }, &.{ 0x8d }, 0, .long },	343	.{ .lea, .rm, &.{ .r64, .m }, &.{ 0x8d }, 0, .long, .none },
343		344
344	.{ .lfence, .np, &.{}, &.{ 0x0f, 0xae, 0xe8 }, 0, .none },	345	.{ .lfence, .np, &.{}, &.{ 0x0f, 0xae, 0xe8 }, 0, .none, .none },
345		346
346	.{ .lods, .np, &.{ .m8 }, &.{ 0xac }, 0, .none },	347	.{ .lods, .np, &.{ .m8 }, &.{ 0xac }, 0, .none, .none },
347	.{ .lods, .np, &.{ .m16 }, &.{ 0xad }, 0, .none },	348	.{ .lods, .np, &.{ .m16 }, &.{ 0xad }, 0, .short, .none },
348	.{ .lods, .np, &.{ .m32 }, &.{ 0xad }, 0, .none },	349	.{ .lods, .np, &.{ .m32 }, &.{ 0xad }, 0, .none, .none },
349	.{ .lods, .np, &.{ .m64 }, &.{ 0xad }, 0, .long },	350	.{ .lods, .np, &.{ .m64 }, &.{ 0xad }, 0, .long, .none },
350		351
351	.{ .lodsb, .np, &.{}, &.{ 0xac }, 0, .none },	352	.{ .lodsb, .np, &.{}, &.{ 0xac }, 0, .none, .none },
352	.{ .lodsw, .np, &.{}, &.{ 0xad }, 0, .short },	353	.{ .lodsw, .np, &.{}, &.{ 0xad }, 0, .short, .none },
353	.{ .lodsd, .np, &.{}, &.{ 0xad }, 0, .none },	354	.{ .lodsd, .np, &.{}, &.{ 0xad }, 0, .none, .none },
354	.{ .lodsq, .np, &.{}, &.{ 0xad }, 0, .long },	355	.{ .lodsq, .np, &.{}, &.{ 0xad }, 0, .long, .none },
355		356
356	.{ .lzcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .none },	357	.{ .lzcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .short, .none },
357	.{ .lzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .none },	358	.{ .lzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .none, .none },
358	.{ .lzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .long },	359	.{ .lzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .long, .none },
359		360
360	.{ .mfence, .np, &.{}, &.{ 0x0f, 0xae, 0xf0 }, 0, .none },	361	.{ .mfence, .np, &.{}, &.{ 0x0f, 0xae, 0xf0 }, 0, .none, .none },
361		362
362	.{ .mov, .mr, &.{ .rm8, .r8 }, &.{ 0x88 }, 0, .none },	363	.{ .mov, .mr, &.{ .rm8, .r8 }, &.{ 0x88 }, 0, .none, .none },
363	.{ .mov, .mr, &.{ .rm8, .r8 }, &.{ 0x88 }, 0, .rex },	364	.{ .mov, .mr, &.{ .rm8, .r8 }, &.{ 0x88 }, 0, .rex, .none },
364	.{ .mov, .mr, &.{ .rm16, .r16 }, &.{ 0x89 }, 0, .none },	365	.{ .mov, .mr, &.{ .rm16, .r16 }, &.{ 0x89 }, 0, .short, .none },
365	.{ .mov, .mr, &.{ .rm32, .r32 }, &.{ 0x89 }, 0, .none },	366	.{ .mov, .mr, &.{ .rm32, .r32 }, &.{ 0x89 }, 0, .none, .none },
366	.{ .mov, .mr, &.{ .rm64, .r64 }, &.{ 0x89 }, 0, .long },	367	.{ .mov, .mr, &.{ .rm64, .r64 }, &.{ 0x89 }, 0, .long, .none },
367	.{ .mov, .rm, &.{ .r8, .rm8 }, &.{ 0x8a }, 0, .none },	368	.{ .mov, .rm, &.{ .r8, .rm8 }, &.{ 0x8a }, 0, .none, .none },
368	.{ .mov, .rm, &.{ .r8, .rm8 }, &.{ 0x8a }, 0, .rex },	369	.{ .mov, .rm, &.{ .r8, .rm8 }, &.{ 0x8a }, 0, .rex, .none },
369	.{ .mov, .rm, &.{ .r16, .rm16 }, &.{ 0x8b }, 0, .none },	370	.{ .mov, .rm, &.{ .r16, .rm16 }, &.{ 0x8b }, 0, .short, .none },
370	.{ .mov, .rm, &.{ .r32, .rm32 }, &.{ 0x8b }, 0, .none },	371	.{ .mov, .rm, &.{ .r32, .rm32 }, &.{ 0x8b }, 0, .none, .none },
371	.{ .mov, .rm, &.{ .r64, .rm64 }, &.{ 0x8b }, 0, .long },	372	.{ .mov, .rm, &.{ .r64, .rm64 }, &.{ 0x8b }, 0, .long, .none },
372	.{ .mov, .mr, &.{ .rm16, .sreg }, &.{ 0x8c }, 0, .none },	373	.{ .mov, .mr, &.{ .rm16, .sreg }, &.{ 0x8c }, 0, .short, .none },
373	.{ .mov, .mr, &.{ .rm64, .sreg }, &.{ 0x8c }, 0, .long },	374	.{ .mov, .mr, &.{ .r32_m16, .sreg }, &.{ 0x8c }, 0, .none, .none },
374	.{ .mov, .rm, &.{ .sreg, .rm16 }, &.{ 0x8e }, 0, .none },	375	.{ .mov, .mr, &.{ .r64_m16, .sreg }, &.{ 0x8c }, 0, .long, .none },
375	.{ .mov, .rm, &.{ .sreg, .rm64 }, &.{ 0x8e }, 0, .long },	376	.{ .mov, .rm, &.{ .sreg, .rm16 }, &.{ 0x8e }, 0, .short, .none },
376	.{ .mov, .fd, &.{ .al, .moffs }, &.{ 0xa0 }, 0, .none },	377	.{ .mov, .rm, &.{ .sreg, .r32_m16 }, &.{ 0x8e }, 0, .none, .none },
377	.{ .mov, .fd, &.{ .ax, .moffs }, &.{ 0xa1 }, 0, .none },	378	.{ .mov, .rm, &.{ .sreg, .r64_m16 }, &.{ 0x8e }, 0, .long, .none },
378	.{ .mov, .fd, &.{ .eax, .moffs }, &.{ 0xa1 }, 0, .none },	379	.{ .mov, .fd, &.{ .al, .moffs }, &.{ 0xa0 }, 0, .none, .none },
379	.{ .mov, .fd, &.{ .rax, .moffs }, &.{ 0xa1 }, 0, .long },	380	.{ .mov, .fd, &.{ .ax, .moffs }, &.{ 0xa1 }, 0, .none, .none },
380	.{ .mov, .td, &.{ .moffs, .al }, &.{ 0xa2 }, 0, .none },	381	.{ .mov, .fd, &.{ .eax, .moffs }, &.{ 0xa1 }, 0, .none, .none },
381	.{ .mov, .td, &.{ .moffs, .ax }, &.{ 0xa3 }, 0, .none },	382	.{ .mov, .fd, &.{ .rax, .moffs }, &.{ 0xa1 }, 0, .long, .none },
382	.{ .mov, .td, &.{ .moffs, .eax }, &.{ 0xa3 }, 0, .none },	383	.{ .mov, .td, &.{ .moffs, .al }, &.{ 0xa2 }, 0, .none, .none },
383	.{ .mov, .td, &.{ .moffs, .rax }, &.{ 0xa3 }, 0, .long },	384	.{ .mov, .td, &.{ .moffs, .ax }, &.{ 0xa3 }, 0, .none, .none },
384	.{ .mov, .oi, &.{ .r8, .imm8 }, &.{ 0xb0 }, 0, .none },	385	.{ .mov, .td, &.{ .moffs, .eax }, &.{ 0xa3 }, 0, .none, .none },
385	.{ .mov, .oi, &.{ .r8, .imm8 }, &.{ 0xb0 }, 0, .rex },	386	.{ .mov, .td, &.{ .moffs, .rax }, &.{ 0xa3 }, 0, .long, .none },
386	.{ .mov, .oi, &.{ .r16, .imm16 }, &.{ 0xb8 }, 0, .none },	387	.{ .mov, .oi, &.{ .r8, .imm8 }, &.{ 0xb0 }, 0, .none, .none },
387	.{ .mov, .oi, &.{ .r32, .imm32 }, &.{ 0xb8 }, 0, .none },	388	.{ .mov, .oi, &.{ .r8, .imm8 }, &.{ 0xb0 }, 0, .rex, .none },
388	.{ .mov, .oi, &.{ .r64, .imm64 }, &.{ 0xb8 }, 0, .long },	389	.{ .mov, .oi, &.{ .r16, .imm16 }, &.{ 0xb8 }, 0, .short, .none },
389	.{ .mov, .mi, &.{ .rm8, .imm8 }, &.{ 0xc6 }, 0, .none },	390	.{ .mov, .oi, &.{ .r32, .imm32 }, &.{ 0xb8 }, 0, .none, .none },
390	.{ .mov, .mi, &.{ .rm8, .imm8 }, &.{ 0xc6 }, 0, .rex },	391	.{ .mov, .oi, &.{ .r64, .imm64 }, &.{ 0xb8 }, 0, .long, .none },
391	.{ .mov, .mi, &.{ .rm16, .imm16 }, &.{ 0xc7 }, 0, .none },	392	.{ .mov, .mi, &.{ .rm8, .imm8 }, &.{ 0xc6 }, 0, .none, .none },
392	.{ .mov, .mi, &.{ .rm32, .imm32 }, &.{ 0xc7 }, 0, .none },	393	.{ .mov, .mi, &.{ .rm8, .imm8 }, &.{ 0xc6 }, 0, .rex, .none },
393	.{ .mov, .mi, &.{ .rm64, .imm32s }, &.{ 0xc7 }, 0, .long },	394	.{ .mov, .mi, &.{ .rm16, .imm16 }, &.{ 0xc7 }, 0, .short, .none },
394		395	.{ .mov, .mi, &.{ .rm32, .imm32 }, &.{ 0xc7 }, 0, .none, .none },
395	.{ .movbe, .rm, &.{ .r16, .m16 }, &.{ 0x0f, 0x38, 0xf0 }, 0, .none },	396	.{ .mov, .mi, &.{ .rm64, .imm32s }, &.{ 0xc7 }, 0, .long, .none },
396	.{ .movbe, .rm, &.{ .r32, .m32 }, &.{ 0x0f, 0x38, 0xf0 }, 0, .none },	397
397	.{ .movbe, .rm, &.{ .r64, .m64 }, &.{ 0x0f, 0x38, 0xf0 }, 0, .long },	398	.{ .movbe, .rm, &.{ .r16, .m16 }, &.{ 0x0f, 0x38, 0xf0 }, 0, .short, .none },
398	.{ .movbe, .mr, &.{ .m16, .r16 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .none },	399	.{ .movbe, .rm, &.{ .r32, .m32 }, &.{ 0x0f, 0x38, 0xf0 }, 0, .none, .none },
399	.{ .movbe, .mr, &.{ .m32, .r32 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .none },	400	.{ .movbe, .rm, &.{ .r64, .m64 }, &.{ 0x0f, 0x38, 0xf0 }, 0, .long, .none },
400	.{ .movbe, .mr, &.{ .m64, .r64 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .long },	401	.{ .movbe, .mr, &.{ .m16, .r16 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .short, .none },
401		402	.{ .movbe, .mr, &.{ .m32, .r32 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .none, .none },
402	.{ .movs, .np, &.{ .m8, .m8 }, &.{ 0xa4 }, 0, .none },	403	.{ .movbe, .mr, &.{ .m64, .r64 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .long, .none },
403	.{ .movs, .np, &.{ .m16, .m16 }, &.{ 0xa5 }, 0, .none },	404
404	.{ .movs, .np, &.{ .m32, .m32 }, &.{ 0xa5 }, 0, .none },	405	.{ .movs, .np, &.{ .m8, .m8 }, &.{ 0xa4 }, 0, .none, .none },
405	.{ .movs, .np, &.{ .m64, .m64 }, &.{ 0xa5 }, 0, .long },	406	.{ .movs, .np, &.{ .m16, .m16 }, &.{ 0xa5 }, 0, .short, .none },
406		407	.{ .movs, .np, &.{ .m32, .m32 }, &.{ 0xa5 }, 0, .none, .none },
407	.{ .movsb, .np, &.{}, &.{ 0xa4 }, 0, .none },	408	.{ .movs, .np, &.{ .m64, .m64 }, &.{ 0xa5 }, 0, .long, .none },
408	.{ .movsw, .np, &.{}, &.{ 0xa5 }, 0, .short },	409
409	.{ .movsd, .np, &.{}, &.{ 0xa5 }, 0, .none },	410	.{ .movsb, .np, &.{}, &.{ 0xa4 }, 0, .none, .none },
410	.{ .movsq, .np, &.{}, &.{ 0xa5 }, 0, .long },	411	.{ .movsw, .np, &.{}, &.{ 0xa5 }, 0, .short, .none },
411		412	.{ .movsd, .np, &.{}, &.{ 0xa5 }, 0, .none, .none },
412	.{ .movsx, .rm, &.{ .r16, .rm8 }, &.{ 0x0f, 0xbe }, 0, .none },	413	.{ .movsq, .np, &.{}, &.{ 0xa5 }, 0, .long, .none },
413	.{ .movsx, .rm, &.{ .r16, .rm8 }, &.{ 0x0f, 0xbe }, 0, .rex },	414
414	.{ .movsx, .rm, &.{ .r32, .rm8 }, &.{ 0x0f, 0xbe }, 0, .none },	415	.{ .movsx, .rm, &.{ .r16, .rm8 }, &.{ 0x0f, 0xbe }, 0, .short, .none },
415	.{ .movsx, .rm, &.{ .r32, .rm8 }, &.{ 0x0f, 0xbe }, 0, .rex },	416	.{ .movsx, .rm, &.{ .r16, .rm8 }, &.{ 0x0f, 0xbe }, 0, .rex_short, .none },
416	.{ .movsx, .rm, &.{ .r64, .rm8 }, &.{ 0x0f, 0xbe }, 0, .long },	417	.{ .movsx, .rm, &.{ .r32, .rm8 }, &.{ 0x0f, 0xbe }, 0, .none, .none },
417	.{ .movsx, .rm, &.{ .r32, .rm16 }, &.{ 0x0f, 0xbf }, 0, .none },	418	.{ .movsx, .rm, &.{ .r32, .rm8 }, &.{ 0x0f, 0xbe }, 0, .rex, .none },
418	.{ .movsx, .rm, &.{ .r64, .rm16 }, &.{ 0x0f, 0xbf }, 0, .long },	419	.{ .movsx, .rm, &.{ .r64, .rm8 }, &.{ 0x0f, 0xbe }, 0, .long, .none },
		420	.{ .movsx, .rm, &.{ .r32, .rm16 }, &.{ 0x0f, 0xbf }, 0, .none, .none },
		421	.{ .movsx, .rm, &.{ .r32, .rm16 }, &.{ 0x0f, 0xbf }, 0, .rex, .none },
		422	.{ .movsx, .rm, &.{ .r64, .rm16 }, &.{ 0x0f, 0xbf }, 0, .long, .none },
419		423
420	// This instruction is discouraged.	424	// This instruction is discouraged.
421	.{ .movsxd, .rm, &.{ .r32, .rm32 }, &.{ 0x63 }, 0, .none },	425	.{ .movsxd, .rm, &.{ .r32, .rm32 }, &.{ 0x63 }, 0, .none, .none },
422	.{ .movsxd, .rm, &.{ .r64, .rm32 }, &.{ 0x63 }, 0, .long },	426	.{ .movsxd, .rm, &.{ .r64, .rm32 }, &.{ 0x63 }, 0, .long, .none },
423		427
424	.{ .movzx, .rm, &.{ .r16, .rm8 }, &.{ 0x0f, 0xb6 }, 0, .none },	428	.{ .movzx, .rm, &.{ .r16, .rm8 }, &.{ 0x0f, 0xb6 }, 0, .short, .none },
425	.{ .movzx, .rm, &.{ .r32, .rm8 }, &.{ 0x0f, 0xb6 }, 0, .none },	429	.{ .movzx, .rm, &.{ .r16, .rm8 }, &.{ 0x0f, 0xb6 }, 0, .rex_short, .none },
426	.{ .movzx, .rm, &.{ .r64, .rm8 }, &.{ 0x0f, 0xb6 }, 0, .long },	430	.{ .movzx, .rm, &.{ .r32, .rm8 }, &.{ 0x0f, 0xb6 }, 0, .none, .none },
427	.{ .movzx, .rm, &.{ .r32, .rm16 }, &.{ 0x0f, 0xb7 }, 0, .none },	431	.{ .movzx, .rm, &.{ .r32, .rm8 }, &.{ 0x0f, 0xb6 }, 0, .rex, .none },
428	.{ .movzx, .rm, &.{ .r64, .rm16 }, &.{ 0x0f, 0xb7 }, 0, .long },	432	.{ .movzx, .rm, &.{ .r64, .rm8 }, &.{ 0x0f, 0xb6 }, 0, .long, .none },
429		433	.{ .movzx, .rm, &.{ .r32, .rm16 }, &.{ 0x0f, 0xb7 }, 0, .none, .none },
430	.{ .mul, .m, &.{ .rm8 }, &.{ 0xf6 }, 4, .none },	434	.{ .movzx, .rm, &.{ .r32, .rm16 }, &.{ 0x0f, 0xb7 }, 0, .rex, .none },
431	.{ .mul, .m, &.{ .rm8 }, &.{ 0xf6 }, 4, .rex },	435	.{ .movzx, .rm, &.{ .r64, .rm16 }, &.{ 0x0f, 0xb7 }, 0, .long, .none },
432	.{ .mul, .m, &.{ .rm16 }, &.{ 0xf7 }, 4, .none },	436
433	.{ .mul, .m, &.{ .rm32 }, &.{ 0xf7 }, 4, .none },	437	.{ .mul, .m, &.{ .rm8 }, &.{ 0xf6 }, 4, .none, .none },
434	.{ .mul, .m, &.{ .rm64 }, &.{ 0xf7 }, 4, .long },	438	.{ .mul, .m, &.{ .rm8 }, &.{ 0xf6 }, 4, .rex, .none },
435		439	.{ .mul, .m, &.{ .rm16 }, &.{ 0xf7 }, 4, .short, .none },
436	.{ .neg, .m, &.{ .rm8 }, &.{ 0xf6 }, 3, .none },	440	.{ .mul, .m, &.{ .rm32 }, &.{ 0xf7 }, 4, .none, .none },
437	.{ .neg, .m, &.{ .rm8 }, &.{ 0xf6 }, 3, .rex },	441	.{ .mul, .m, &.{ .rm64 }, &.{ 0xf7 }, 4, .long, .none },
438	.{ .neg, .m, &.{ .rm16 }, &.{ 0xf7 }, 3, .none },	442
439	.{ .neg, .m, &.{ .rm32 }, &.{ 0xf7 }, 3, .none },	443	.{ .neg, .m, &.{ .rm8 }, &.{ 0xf6 }, 3, .none, .none },
440	.{ .neg, .m, &.{ .rm64 }, &.{ 0xf7 }, 3, .long },	444	.{ .neg, .m, &.{ .rm8 }, &.{ 0xf6 }, 3, .rex, .none },
441		445	.{ .neg, .m, &.{ .rm16 }, &.{ 0xf7 }, 3, .short, .none },
442	.{ .nop, .np, &.{}, &.{ 0x90 }, 0, .none },	446	.{ .neg, .m, &.{ .rm32 }, &.{ 0xf7 }, 3, .none, .none },
443		447	.{ .neg, .m, &.{ .rm64 }, &.{ 0xf7 }, 3, .long, .none },
444	.{ .not, .m, &.{ .rm8 }, &.{ 0xf6 }, 2, .none },	448
445	.{ .not, .m, &.{ .rm8 }, &.{ 0xf6 }, 2, .rex },	449	.{ .nop, .np, &.{}, &.{ 0x90 }, 0, .none, .none },
446	.{ .not, .m, &.{ .rm16 }, &.{ 0xf7 }, 2, .none },	450
447	.{ .not, .m, &.{ .rm32 }, &.{ 0xf7 }, 2, .none },	451	.{ .not, .m, &.{ .rm8 }, &.{ 0xf6 }, 2, .none, .none },
448	.{ .not, .m, &.{ .rm64 }, &.{ 0xf7 }, 2, .long },	452	.{ .not, .m, &.{ .rm8 }, &.{ 0xf6 }, 2, .rex, .none },
449		453	.{ .not, .m, &.{ .rm16 }, &.{ 0xf7 }, 2, .short, .none },
450	.{ .@"or", .zi, &.{ .al, .imm8 }, &.{ 0x0c }, 0, .none },	454	.{ .not, .m, &.{ .rm32 }, &.{ 0xf7 }, 2, .none, .none },
451	.{ .@"or", .zi, &.{ .ax, .imm16 }, &.{ 0x0d }, 0, .none },	455	.{ .not, .m, &.{ .rm64 }, &.{ 0xf7 }, 2, .long, .none },
452	.{ .@"or", .zi, &.{ .eax, .imm32 }, &.{ 0x0d }, 0, .none },	456
453	.{ .@"or", .zi, &.{ .rax, .imm32s }, &.{ 0x0d }, 0, .long },	457	.{ .@"or", .zi, &.{ .al, .imm8 }, &.{ 0x0c }, 0, .none, .none },
454	.{ .@"or", .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 1, .none },	458	.{ .@"or", .zi, &.{ .ax, .imm16 }, &.{ 0x0d }, 0, .short, .none },
455	.{ .@"or", .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 1, .rex },	459	.{ .@"or", .zi, &.{ .eax, .imm32 }, &.{ 0x0d }, 0, .none, .none },
456	.{ .@"or", .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 1, .none },	460	.{ .@"or", .zi, &.{ .rax, .imm32s }, &.{ 0x0d }, 0, .long, .none },
457	.{ .@"or", .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 1, .none },	461	.{ .@"or", .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 1, .none, .none },
458	.{ .@"or", .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 1, .long },	462	.{ .@"or", .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 1, .rex, .none },
459	.{ .@"or", .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 1, .none },	463	.{ .@"or", .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 1, .short, .none },
460	.{ .@"or", .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 1, .none },	464	.{ .@"or", .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 1, .none, .none },
461	.{ .@"or", .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 1, .long },	465	.{ .@"or", .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 1, .long, .none },
462	.{ .@"or", .mr, &.{ .rm8, .r8 }, &.{ 0x08 }, 0, .none },	466	.{ .@"or", .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 1, .short, .none },
463	.{ .@"or", .mr, &.{ .rm8, .r8 }, &.{ 0x08 }, 0, .rex },	467	.{ .@"or", .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 1, .none, .none },
464	.{ .@"or", .mr, &.{ .rm16, .r16 }, &.{ 0x09 }, 0, .none },	468	.{ .@"or", .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 1, .long, .none },
465	.{ .@"or", .mr, &.{ .rm32, .r32 }, &.{ 0x09 }, 0, .none },	469	.{ .@"or", .mr, &.{ .rm8, .r8 }, &.{ 0x08 }, 0, .none, .none },
466	.{ .@"or", .mr, &.{ .rm64, .r64 }, &.{ 0x09 }, 0, .long },	470	.{ .@"or", .mr, &.{ .rm8, .r8 }, &.{ 0x08 }, 0, .rex, .none },
467	.{ .@"or", .rm, &.{ .r8, .rm8 }, &.{ 0x0a }, 0, .none },	471	.{ .@"or", .mr, &.{ .rm16, .r16 }, &.{ 0x09 }, 0, .short, .none },
468	.{ .@"or", .rm, &.{ .r8, .rm8 }, &.{ 0x0a }, 0, .rex },	472	.{ .@"or", .mr, &.{ .rm32, .r32 }, &.{ 0x09 }, 0, .none, .none },
469	.{ .@"or", .rm, &.{ .r16, .rm16 }, &.{ 0x0b }, 0, .none },	473	.{ .@"or", .mr, &.{ .rm64, .r64 }, &.{ 0x09 }, 0, .long, .none },
470	.{ .@"or", .rm, &.{ .r32, .rm32 }, &.{ 0x0b }, 0, .none },	474	.{ .@"or", .rm, &.{ .r8, .rm8 }, &.{ 0x0a }, 0, .none, .none },
471	.{ .@"or", .rm, &.{ .r64, .rm64 }, &.{ 0x0b }, 0, .long },	475	.{ .@"or", .rm, &.{ .r8, .rm8 }, &.{ 0x0a }, 0, .rex, .none },
472		476	.{ .@"or", .rm, &.{ .r16, .rm16 }, &.{ 0x0b }, 0, .short, .none },
473	.{ .pop, .o, &.{ .r16 }, &.{ 0x58 }, 0, .none },	477	.{ .@"or", .rm, &.{ .r32, .rm32 }, &.{ 0x0b }, 0, .none, .none },
474	.{ .pop, .o, &.{ .r64 }, &.{ 0x58 }, 0, .none },	478	.{ .@"or", .rm, &.{ .r64, .rm64 }, &.{ 0x0b }, 0, .long, .none },
475	.{ .pop, .m, &.{ .rm16 }, &.{ 0x8f }, 0, .none },	479
476	.{ .pop, .m, &.{ .rm64 }, &.{ 0x8f }, 0, .none },	480	.{ .pop, .o, &.{ .r16 }, &.{ 0x58 }, 0, .short, .none },
477		481	.{ .pop, .o, &.{ .r64 }, &.{ 0x58 }, 0, .none, .none },
478	.{ .popcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none },	482	.{ .pop, .m, &.{ .rm16 }, &.{ 0x8f }, 0, .short, .none },
479	.{ .popcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none },	483	.{ .pop, .m, &.{ .rm64 }, &.{ 0x8f }, 0, .none, .none },
480	.{ .popcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .long },	484
481		485	.{ .popcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .short, .none },
482	.{ .push, .o, &.{ .r16 }, &.{ 0x50 }, 0, .none },	486	.{ .popcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none, .none },
483	.{ .push, .o, &.{ .r64 }, &.{ 0x50 }, 0, .none },	487	.{ .popcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .long, .none },
484	.{ .push, .m, &.{ .rm16 }, &.{ 0xff }, 6, .none },	488
485	.{ .push, .m, &.{ .rm64 }, &.{ 0xff }, 6, .none },	489	.{ .push, .o, &.{ .r16 }, &.{ 0x50 }, 0, .short, .none },
486	.{ .push, .i, &.{ .imm8 }, &.{ 0x6a }, 0, .none },	490	.{ .push, .o, &.{ .r64 }, &.{ 0x50 }, 0, .none, .none },
487	.{ .push, .i, &.{ .imm16 }, &.{ 0x68 }, 0, .none },	491	.{ .push, .m, &.{ .rm16 }, &.{ 0xff }, 6, .short, .none },
488	.{ .push, .i, &.{ .imm32 }, &.{ 0x68 }, 0, .none },	492	.{ .push, .m, &.{ .rm64 }, &.{ 0xff }, 6, .none, .none },
489		493	.{ .push, .i, &.{ .imm8 }, &.{ 0x6a }, 0, .none, .none },
490	.{ .ret, .np, &.{}, &.{ 0xc3 }, 0, .none },	494	.{ .push, .i, &.{ .imm16 }, &.{ 0x68 }, 0, .short, .none },
491		495	.{ .push, .i, &.{ .imm32 }, &.{ 0x68 }, 0, .none, .none },
492	.{ .rcl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 2, .none },	496
493	.{ .rcl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 2, .rex },	497	.{ .ret, .np, &.{}, &.{ 0xc3 }, 0, .none, .none },
494	.{ .rcl, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 2, .none },	498
495	.{ .rcl, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 2, .rex },	499	.{ .rcl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 2, .none, .none },
496	.{ .rcl, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 2, .none },	500	.{ .rcl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 2, .rex, .none },
497	.{ .rcl, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 2, .rex },	501	.{ .rcl, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 2, .none, .none },
498	.{ .rcl, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 2, .none },	502	.{ .rcl, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 2, .rex, .none },
499	.{ .rcl, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 2, .none },	503	.{ .rcl, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 2, .none, .none },
500	.{ .rcl, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 2, .none },	504	.{ .rcl, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 2, .rex, .none },
501	.{ .rcl, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 2, .none },	505	.{ .rcl, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 2, .short, .none },
502	.{ .rcl, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 2, .long },	506	.{ .rcl, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 2, .short, .none },
503	.{ .rcl, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 2, .none },	507	.{ .rcl, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 2, .short, .none },
504	.{ .rcl, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 2, .long },	508	.{ .rcl, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 2, .none, .none },
505	.{ .rcl, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 2, .none },	509	.{ .rcl, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 2, .long, .none },
506	.{ .rcl, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 2, .long },	510	.{ .rcl, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 2, .none, .none },
507		511	.{ .rcl, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 2, .long, .none },
508	.{ .rcr, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 3, .none },	512	.{ .rcl, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 2, .none, .none },
509	.{ .rcr, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 3, .rex },	513	.{ .rcl, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 2, .long, .none },
510	.{ .rcr, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 3, .none },	514
511	.{ .rcr, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 3, .rex },	515	.{ .rcr, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 3, .none, .none },
512	.{ .rcr, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 3, .none },	516	.{ .rcr, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 3, .rex, .none },
513	.{ .rcr, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 3, .rex },	517	.{ .rcr, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 3, .none, .none },
514	.{ .rcr, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 3, .none },	518	.{ .rcr, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 3, .rex, .none },
515	.{ .rcr, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 3, .none },	519	.{ .rcr, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 3, .none, .none },
516	.{ .rcr, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 3, .none },	520	.{ .rcr, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 3, .rex, .none },
517	.{ .rcr, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 3, .none },	521	.{ .rcr, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 3, .short, .none },
518	.{ .rcr, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 3, .long },	522	.{ .rcr, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 3, .short, .none },
519	.{ .rcr, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 3, .none },	523	.{ .rcr, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 3, .short, .none },
520	.{ .rcr, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 3, .long },	524	.{ .rcr, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 3, .none, .none },
521	.{ .rcr, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 3, .none },	525	.{ .rcr, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 3, .long, .none },
522	.{ .rcr, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 3, .long },	526	.{ .rcr, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 3, .none, .none },
523		527	.{ .rcr, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 3, .long, .none },
524	.{ .rol, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 0, .none },	528	.{ .rcr, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 3, .none, .none },
525	.{ .rol, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 0, .rex },	529	.{ .rcr, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 3, .long, .none },
526	.{ .rol, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 0, .none },	530
527	.{ .rol, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 0, .rex },	531	.{ .rol, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 0, .none, .none },
528	.{ .rol, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 0, .none },	532	.{ .rol, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 0, .rex, .none },
529	.{ .rol, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 0, .rex },	533	.{ .rol, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 0, .none, .none },
530	.{ .rol, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 0, .none },	534	.{ .rol, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 0, .rex, .none },
531	.{ .rol, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 0, .none },	535	.{ .rol, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 0, .none, .none },
532	.{ .rol, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 0, .none },	536	.{ .rol, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 0, .rex, .none },
533	.{ .rol, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 0, .none },	537	.{ .rol, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 0, .short, .none },
534	.{ .rol, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 0, .long },	538	.{ .rol, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 0, .short, .none },
535	.{ .rol, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 0, .none },	539	.{ .rol, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 0, .short, .none },
536	.{ .rol, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 0, .long },	540	.{ .rol, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 0, .none, .none },
537	.{ .rol, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 0, .none },	541	.{ .rol, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 0, .long, .none },
538	.{ .rol, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 0, .long },	542	.{ .rol, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 0, .none, .none },
539		543	.{ .rol, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 0, .long, .none },
540	.{ .ror, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 1, .none },	544	.{ .rol, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 0, .none, .none },
541	.{ .ror, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 1, .rex },	545	.{ .rol, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 0, .long, .none },
542	.{ .ror, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 1, .none },	546
543	.{ .ror, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 1, .rex },	547	.{ .ror, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 1, .none, .none },
544	.{ .ror, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 1, .none },	548	.{ .ror, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 1, .rex, .none },
545	.{ .ror, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 1, .rex },	549	.{ .ror, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 1, .none, .none },
546	.{ .ror, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 1, .none },	550	.{ .ror, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 1, .rex, .none },
547	.{ .ror, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 1, .none },	551	.{ .ror, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 1, .none, .none },
548	.{ .ror, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 1, .none },	552	.{ .ror, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 1, .rex, .none },
549	.{ .ror, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 1, .none },	553	.{ .ror, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 1, .short, .none },
550	.{ .ror, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 1, .long },	554	.{ .ror, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 1, .short, .none },
551	.{ .ror, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 1, .none },	555	.{ .ror, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 1, .short, .none },
552	.{ .ror, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 1, .long },	556	.{ .ror, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 1, .none, .none },
553	.{ .ror, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 1, .none },	557	.{ .ror, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 1, .long, .none },
554	.{ .ror, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 1, .long },	558	.{ .ror, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 1, .none, .none },
555		559	.{ .ror, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 1, .long, .none },
556	.{ .sal, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 4, .none },	560	.{ .ror, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 1, .none, .none },
557	.{ .sal, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 4, .rex },	561	.{ .ror, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 1, .long, .none },
558	.{ .sal, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 4, .none },	562
559	.{ .sal, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 4, .none },	563	.{ .sal, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 4, .none, .none },
560	.{ .sal, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 4, .long },	564	.{ .sal, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 4, .rex, .none },
561	.{ .sal, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 4, .none },	565	.{ .sal, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 4, .short, .none },
562	.{ .sal, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 4, .rex },	566	.{ .sal, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 4, .none, .none },
563	.{ .sal, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 4, .none },	567	.{ .sal, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 4, .long, .none },
564	.{ .sal, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 4, .none },	568	.{ .sal, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 4, .none, .none },
565	.{ .sal, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 4, .long },	569	.{ .sal, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 4, .rex, .none },
566	.{ .sal, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 4, .none },	570	.{ .sal, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 4, .short, .none },
567	.{ .sal, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 4, .rex },	571	.{ .sal, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 4, .none, .none },
568	.{ .sal, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 4, .none },	572	.{ .sal, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 4, .long, .none },
569	.{ .sal, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 4, .none },	573	.{ .sal, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 4, .none, .none },
570	.{ .sal, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 4, .long },	574	.{ .sal, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 4, .rex, .none },
571		575	.{ .sal, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 4, .short, .none },
572	.{ .sar, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 7, .none },	576	.{ .sal, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 4, .none, .none },
573	.{ .sar, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 7, .rex },	577	.{ .sal, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 4, .long, .none },
574	.{ .sar, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 7, .none },	578
575	.{ .sar, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 7, .none },	579	.{ .sar, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 7, .none, .none },
576	.{ .sar, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 7, .long },	580	.{ .sar, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 7, .rex, .none },
577	.{ .sar, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 7, .none },	581	.{ .sar, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 7, .short, .none },
578	.{ .sar, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 7, .rex },	582	.{ .sar, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 7, .none, .none },
579	.{ .sar, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 7, .none },	583	.{ .sar, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 7, .long, .none },
580	.{ .sar, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 7, .none },	584	.{ .sar, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 7, .none, .none },
581	.{ .sar, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 7, .long },	585	.{ .sar, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 7, .rex, .none },
582	.{ .sar, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 7, .none },	586	.{ .sar, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 7, .short, .none },
583	.{ .sar, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 7, .rex },	587	.{ .sar, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 7, .none, .none },
584	.{ .sar, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 7, .none },	588	.{ .sar, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 7, .long, .none },
585	.{ .sar, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 7, .none },	589	.{ .sar, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 7, .none, .none },
586	.{ .sar, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 7, .long },	590	.{ .sar, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 7, .rex, .none },
587		591	.{ .sar, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 7, .short, .none },
588	.{ .sbb, .zi, &.{ .al, .imm8 }, &.{ 0x1c }, 0, .none },	592	.{ .sar, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 7, .none, .none },
589	.{ .sbb, .zi, &.{ .ax, .imm16 }, &.{ 0x1d }, 0, .none },	593	.{ .sar, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 7, .long, .none },
590	.{ .sbb, .zi, &.{ .eax, .imm32 }, &.{ 0x1d }, 0, .none },	594
591	.{ .sbb, .zi, &.{ .rax, .imm32s }, &.{ 0x1d }, 0, .long },	595	.{ .sbb, .zi, &.{ .al, .imm8 }, &.{ 0x1c }, 0, .none, .none },
592	.{ .sbb, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 3, .none },	596	.{ .sbb, .zi, &.{ .ax, .imm16 }, &.{ 0x1d }, 0, .short, .none },
593	.{ .sbb, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 3, .rex },	597	.{ .sbb, .zi, &.{ .eax, .imm32 }, &.{ 0x1d }, 0, .none, .none },
594	.{ .sbb, .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 3, .none },	598	.{ .sbb, .zi, &.{ .rax, .imm32s }, &.{ 0x1d }, 0, .long, .none },
595	.{ .sbb, .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 3, .none },	599	.{ .sbb, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 3, .none, .none },
596	.{ .sbb, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 3, .long },	600	.{ .sbb, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 3, .rex, .none },
597	.{ .sbb, .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 3, .none },	601	.{ .sbb, .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 3, .short, .none },
598	.{ .sbb, .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 3, .none },	602	.{ .sbb, .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 3, .none, .none },
599	.{ .sbb, .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 3, .long },	603	.{ .sbb, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 3, .long, .none },
600	.{ .sbb, .mr, &.{ .rm8, .r8 }, &.{ 0x18 }, 0, .none },	604	.{ .sbb, .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 3, .short, .none },
601	.{ .sbb, .mr, &.{ .rm8, .r8 }, &.{ 0x18 }, 0, .rex },	605	.{ .sbb, .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 3, .none, .none },
602	.{ .sbb, .mr, &.{ .rm16, .r16 }, &.{ 0x19 }, 0, .none },	606	.{ .sbb, .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 3, .long, .none },
603	.{ .sbb, .mr, &.{ .rm32, .r32 }, &.{ 0x19 }, 0, .none },	607	.{ .sbb, .mr, &.{ .rm8, .r8 }, &.{ 0x18 }, 0, .none, .none },
604	.{ .sbb, .mr, &.{ .rm64, .r64 }, &.{ 0x19 }, 0, .long },	608	.{ .sbb, .mr, &.{ .rm8, .r8 }, &.{ 0x18 }, 0, .rex, .none },
605	.{ .sbb, .rm, &.{ .r8, .rm8 }, &.{ 0x1a }, 0, .none },	609	.{ .sbb, .mr, &.{ .rm16, .r16 }, &.{ 0x19 }, 0, .short, .none },
606	.{ .sbb, .rm, &.{ .r8, .rm8 }, &.{ 0x1a }, 0, .rex },	610	.{ .sbb, .mr, &.{ .rm32, .r32 }, &.{ 0x19 }, 0, .none, .none },
607	.{ .sbb, .rm, &.{ .r16, .rm16 }, &.{ 0x1b }, 0, .none },	611	.{ .sbb, .mr, &.{ .rm64, .r64 }, &.{ 0x19 }, 0, .long, .none },
608	.{ .sbb, .rm, &.{ .r32, .rm32 }, &.{ 0x1b }, 0, .none },	612	.{ .sbb, .rm, &.{ .r8, .rm8 }, &.{ 0x1a }, 0, .none, .none },
609	.{ .sbb, .rm, &.{ .r64, .rm64 }, &.{ 0x1b }, 0, .long },	613	.{ .sbb, .rm, &.{ .r8, .rm8 }, &.{ 0x1a }, 0, .rex, .none },
610		614	.{ .sbb, .rm, &.{ .r16, .rm16 }, &.{ 0x1b }, 0, .short, .none },
611	.{ .scas, .np, &.{ .m8 }, &.{ 0xae }, 0, .none },	615	.{ .sbb, .rm, &.{ .r32, .rm32 }, &.{ 0x1b }, 0, .none, .none },
612	.{ .scas, .np, &.{ .m16 }, &.{ 0xaf }, 0, .none },	616	.{ .sbb, .rm, &.{ .r64, .rm64 }, &.{ 0x1b }, 0, .long, .none },
613	.{ .scas, .np, &.{ .m32 }, &.{ 0xaf }, 0, .none },	617
614	.{ .scas, .np, &.{ .m64 }, &.{ 0xaf }, 0, .long },	618	.{ .scas, .np, &.{ .m8 }, &.{ 0xae }, 0, .none, .none },
615		619	.{ .scas, .np, &.{ .m16 }, &.{ 0xaf }, 0, .short, .none },
616	.{ .scasb, .np, &.{}, &.{ 0xae }, 0, .none },	620	.{ .scas, .np, &.{ .m32 }, &.{ 0xaf }, 0, .none, .none },
617	.{ .scasw, .np, &.{}, &.{ 0xaf }, 0, .short },	621	.{ .scas, .np, &.{ .m64 }, &.{ 0xaf }, 0, .long, .none },
618	.{ .scasd, .np, &.{}, &.{ 0xaf }, 0, .none },	622
619	.{ .scasq, .np, &.{}, &.{ 0xaf }, 0, .long },	623	.{ .scasb, .np, &.{}, &.{ 0xae }, 0, .none, .none },
620		624	.{ .scasw, .np, &.{}, &.{ 0xaf }, 0, .short, .none },
621	.{ .seta, .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .none },	625	.{ .scasd, .np, &.{}, &.{ 0xaf }, 0, .none, .none },
622	.{ .seta, .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .rex },	626	.{ .scasq, .np, &.{}, &.{ 0xaf }, 0, .long, .none },
623	.{ .setae, .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .none },	627
624	.{ .setae, .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .rex },	628	.{ .seta, .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .none, .none },
625	.{ .setb, .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .none },	629	.{ .seta, .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .rex, .none },
626	.{ .setb, .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .rex },	630	.{ .setae, .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .none, .none },
627	.{ .setbe, .m, &.{ .rm8 }, &.{ 0x0f, 0x96 }, 0, .none },	631	.{ .setae, .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .rex, .none },
628	.{ .setbe, .m, &.{ .rm8 }, &.{ 0x0f, 0x96 }, 0, .rex },	632	.{ .setb, .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .none, .none },
629	.{ .setc, .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .none },	633	.{ .setb, .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .rex, .none },
630	.{ .setc, .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .rex },	634	.{ .setbe, .m, &.{ .rm8 }, &.{ 0x0f, 0x96 }, 0, .none, .none },
631	.{ .sete, .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .none },	635	.{ .setbe, .m, &.{ .rm8 }, &.{ 0x0f, 0x96 }, 0, .rex, .none },
632	.{ .sete, .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .rex },	636	.{ .setc, .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .none, .none },
633	.{ .setg, .m, &.{ .rm8 }, &.{ 0x0f, 0x9f }, 0, .none },	637	.{ .setc, .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .rex, .none },
634	.{ .setg, .m, &.{ .rm8 }, &.{ 0x0f, 0x9f }, 0, .rex },	638	.{ .sete, .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .none, .none },
635	.{ .setge, .m, &.{ .rm8 }, &.{ 0x0f, 0x9d }, 0, .none },	639	.{ .sete, .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .rex, .none },
636	.{ .setge, .m, &.{ .rm8 }, &.{ 0x0f, 0x9d }, 0, .rex },	640	.{ .setg, .m, &.{ .rm8 }, &.{ 0x0f, 0x9f }, 0, .none, .none },
637	.{ .setl, .m, &.{ .rm8 }, &.{ 0x0f, 0x9c }, 0, .none },	641	.{ .setg, .m, &.{ .rm8 }, &.{ 0x0f, 0x9f }, 0, .rex, .none },
638	.{ .setl, .m, &.{ .rm8 }, &.{ 0x0f, 0x9c }, 0, .rex },	642	.{ .setge, .m, &.{ .rm8 }, &.{ 0x0f, 0x9d }, 0, .none, .none },
639	.{ .setle, .m, &.{ .rm8 }, &.{ 0x0f, 0x9e }, 0, .none },	643	.{ .setge, .m, &.{ .rm8 }, &.{ 0x0f, 0x9d }, 0, .rex, .none },
640	.{ .setle, .m, &.{ .rm8 }, &.{ 0x0f, 0x9e }, 0, .rex },	644	.{ .setl, .m, &.{ .rm8 }, &.{ 0x0f, 0x9c }, 0, .none, .none },
641	.{ .setna, .m, &.{ .rm8 }, &.{ 0x0f, 0x96 }, 0, .none },	645	.{ .setl, .m, &.{ .rm8 }, &.{ 0x0f, 0x9c }, 0, .rex, .none },
642	.{ .setna, .m, &.{ .rm8 }, &.{ 0x0f, 0x96 }, 0, .rex },	646	.{ .setle, .m, &.{ .rm8 }, &.{ 0x0f, 0x9e }, 0, .none, .none },
643	.{ .setnae, .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .none },	647	.{ .setle, .m, &.{ .rm8 }, &.{ 0x0f, 0x9e }, 0, .rex, .none },
644	.{ .setnae, .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .rex },	648	.{ .setna, .m, &.{ .rm8 }, &.{ 0x0f, 0x96 }, 0, .none, .none },
645	.{ .setnb, .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .none },	649	.{ .setna, .m, &.{ .rm8 }, &.{ 0x0f, 0x96 }, 0, .rex, .none },
646	.{ .setnb, .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .rex },	650	.{ .setnae, .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .none, .none },
647	.{ .setnbe, .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .none },	651	.{ .setnae, .m, &.{ .rm8 }, &.{ 0x0f, 0x92 }, 0, .rex, .none },
648	.{ .setnbe, .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .rex },	652	.{ .setnb, .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .none, .none },
649	.{ .setnc, .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .none },	653	.{ .setnb, .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .rex, .none },
650	.{ .setnc, .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .rex },	654	.{ .setnbe, .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .none, .none },
651	.{ .setne, .m, &.{ .rm8 }, &.{ 0x0f, 0x95 }, 0, .none },	655	.{ .setnbe, .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .rex, .none },
652	.{ .setne, .m, &.{ .rm8 }, &.{ 0x0f, 0x95 }, 0, .rex },	656	.{ .setnc, .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .none, .none },
653	.{ .setng, .m, &.{ .rm8 }, &.{ 0x0f, 0x9e }, 0, .none },	657	.{ .setnc, .m, &.{ .rm8 }, &.{ 0x0f, 0x93 }, 0, .rex, .none },
654	.{ .setng, .m, &.{ .rm8 }, &.{ 0x0f, 0x9e }, 0, .rex },	658	.{ .setne, .m, &.{ .rm8 }, &.{ 0x0f, 0x95 }, 0, .none, .none },
655	.{ .setnge, .m, &.{ .rm8 }, &.{ 0x0f, 0x9c }, 0, .none },	659	.{ .setne, .m, &.{ .rm8 }, &.{ 0x0f, 0x95 }, 0, .rex, .none },
656	.{ .setnge, .m, &.{ .rm8 }, &.{ 0x0f, 0x9c }, 0, .rex },	660	.{ .setng, .m, &.{ .rm8 }, &.{ 0x0f, 0x9e }, 0, .none, .none },
657	.{ .setnl, .m, &.{ .rm8 }, &.{ 0x0f, 0x9d }, 0, .none },	661	.{ .setng, .m, &.{ .rm8 }, &.{ 0x0f, 0x9e }, 0, .rex, .none },
658	.{ .setnl, .m, &.{ .rm8 }, &.{ 0x0f, 0x9d }, 0, .rex },	662	.{ .setnge, .m, &.{ .rm8 }, &.{ 0x0f, 0x9c }, 0, .none, .none },
659	.{ .setnle, .m, &.{ .rm8 }, &.{ 0x0f, 0x9f }, 0, .none },	663	.{ .setnge, .m, &.{ .rm8 }, &.{ 0x0f, 0x9c }, 0, .rex, .none },
660	.{ .setnle, .m, &.{ .rm8 }, &.{ 0x0f, 0x9f }, 0, .rex },	664	.{ .setnl, .m, &.{ .rm8 }, &.{ 0x0f, 0x9d }, 0, .none, .none },
661	.{ .setno, .m, &.{ .rm8 }, &.{ 0x0f, 0x91 }, 0, .none },	665	.{ .setnl, .m, &.{ .rm8 }, &.{ 0x0f, 0x9d }, 0, .rex, .none },
662	.{ .setno, .m, &.{ .rm8 }, &.{ 0x0f, 0x91 }, 0, .rex },	666	.{ .setnle, .m, &.{ .rm8 }, &.{ 0x0f, 0x9f }, 0, .none, .none },
663	.{ .setnp, .m, &.{ .rm8 }, &.{ 0x0f, 0x9b }, 0, .none },	667	.{ .setnle, .m, &.{ .rm8 }, &.{ 0x0f, 0x9f }, 0, .rex, .none },
664	.{ .setnp, .m, &.{ .rm8 }, &.{ 0x0f, 0x9b }, 0, .rex },	668	.{ .setno, .m, &.{ .rm8 }, &.{ 0x0f, 0x91 }, 0, .none, .none },
665	.{ .setns, .m, &.{ .rm8 }, &.{ 0x0f, 0x99 }, 0, .none },	669	.{ .setno, .m, &.{ .rm8 }, &.{ 0x0f, 0x91 }, 0, .rex, .none },
666	.{ .setns, .m, &.{ .rm8 }, &.{ 0x0f, 0x99 }, 0, .rex },	670	.{ .setnp, .m, &.{ .rm8 }, &.{ 0x0f, 0x9b }, 0, .none, .none },
667	.{ .setnz, .m, &.{ .rm8 }, &.{ 0x0f, 0x95 }, 0, .none },	671	.{ .setnp, .m, &.{ .rm8 }, &.{ 0x0f, 0x9b }, 0, .rex, .none },
668	.{ .setnz, .m, &.{ .rm8 }, &.{ 0x0f, 0x95 }, 0, .rex },	672	.{ .setns, .m, &.{ .rm8 }, &.{ 0x0f, 0x99 }, 0, .none, .none },
669	.{ .seto, .m, &.{ .rm8 }, &.{ 0x0f, 0x90 }, 0, .none },	673	.{ .setns, .m, &.{ .rm8 }, &.{ 0x0f, 0x99 }, 0, .rex, .none },
670	.{ .seto, .m, &.{ .rm8 }, &.{ 0x0f, 0x90 }, 0, .rex },	674	.{ .setnz, .m, &.{ .rm8 }, &.{ 0x0f, 0x95 }, 0, .none, .none },
671	.{ .setp, .m, &.{ .rm8 }, &.{ 0x0f, 0x9a }, 0, .none },	675	.{ .setnz, .m, &.{ .rm8 }, &.{ 0x0f, 0x95 }, 0, .rex, .none },
672	.{ .setp, .m, &.{ .rm8 }, &.{ 0x0f, 0x9a }, 0, .rex },	676	.{ .seto, .m, &.{ .rm8 }, &.{ 0x0f, 0x90 }, 0, .none, .none },
673	.{ .setpe, .m, &.{ .rm8 }, &.{ 0x0f, 0x9a }, 0, .none },	677	.{ .seto, .m, &.{ .rm8 }, &.{ 0x0f, 0x90 }, 0, .rex, .none },
674	.{ .setpe, .m, &.{ .rm8 }, &.{ 0x0f, 0x9a }, 0, .rex },	678	.{ .setp, .m, &.{ .rm8 }, &.{ 0x0f, 0x9a }, 0, .none, .none },
675	.{ .setpo, .m, &.{ .rm8 }, &.{ 0x0f, 0x9b }, 0, .none },	679	.{ .setp, .m, &.{ .rm8 }, &.{ 0x0f, 0x9a }, 0, .rex, .none },
676	.{ .setpo, .m, &.{ .rm8 }, &.{ 0x0f, 0x9b }, 0, .rex },	680	.{ .setpe, .m, &.{ .rm8 }, &.{ 0x0f, 0x9a }, 0, .none, .none },
677	.{ .sets, .m, &.{ .rm8 }, &.{ 0x0f, 0x98 }, 0, .none },	681	.{ .setpe, .m, &.{ .rm8 }, &.{ 0x0f, 0x9a }, 0, .rex, .none },
678	.{ .sets, .m, &.{ .rm8 }, &.{ 0x0f, 0x98 }, 0, .rex },	682	.{ .setpo, .m, &.{ .rm8 }, &.{ 0x0f, 0x9b }, 0, .none, .none },
679	.{ .setz, .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .none },	683	.{ .setpo, .m, &.{ .rm8 }, &.{ 0x0f, 0x9b }, 0, .rex, .none },
680	.{ .setz, .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .rex },	684	.{ .sets, .m, &.{ .rm8 }, &.{ 0x0f, 0x98 }, 0, .none, .none },
681		685	.{ .sets, .m, &.{ .rm8 }, &.{ 0x0f, 0x98 }, 0, .rex, .none },
682	.{ .sfence, .np, &.{}, &.{ 0x0f, 0xae, 0xf8 }, 0, .none },	686	.{ .setz, .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .none, .none },
683		687	.{ .setz, .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .rex, .none },
684	.{ .shl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 4, .none },	688
685	.{ .shl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 4, .rex },	689	.{ .sfence, .np, &.{}, &.{ 0x0f, 0xae, 0xf8 }, 0, .none, .none },
686	.{ .shl, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 4, .none },	690
687	.{ .shl, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 4, .none },	691	.{ .shl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 4, .none, .none },
688	.{ .shl, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 4, .long },	692	.{ .shl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 4, .rex, .none },
689	.{ .shl, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 4, .none },	693	.{ .shl, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 4, .short, .none },
690	.{ .shl, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 4, .rex },	694	.{ .shl, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 4, .none, .none },
691	.{ .shl, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 4, .none },	695	.{ .shl, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 4, .long, .none },
692	.{ .shl, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 4, .none },	696	.{ .shl, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 4, .none, .none },
693	.{ .shl, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 4, .long },	697	.{ .shl, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 4, .rex, .none },
694	.{ .shl, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 4, .none },	698	.{ .shl, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 4, .short, .none },
695	.{ .shl, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 4, .rex },	699	.{ .shl, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 4, .none, .none },
696	.{ .shl, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 4, .none },	700	.{ .shl, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 4, .long, .none },
697	.{ .shl, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 4, .none },	701	.{ .shl, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 4, .none, .none },
698	.{ .shl, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 4, .long },	702	.{ .shl, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 4, .rex, .none },
699		703	.{ .shl, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 4, .short, .none },
700	.{ .shld, .mri, &.{ .rm16, .r16, .imm8 }, &.{ 0x0f, 0xa4 }, 0, .none },	704	.{ .shl, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 4, .none, .none },
701	.{ .shld, .mrc, &.{ .rm16, .r16, .cl }, &.{ 0x0f, 0xa5 }, 0, .none },	705	.{ .shl, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 4, .long, .none },
702	.{ .shld, .mri, &.{ .rm32, .r32, .imm8 }, &.{ 0x0f, 0xa4 }, 0, .none },	706
703	.{ .shld, .mri, &.{ .rm64, .r64, .imm8 }, &.{ 0x0f, 0xa4 }, 0, .long },	707	.{ .shld, .mri, &.{ .rm16, .r16, .imm8 }, &.{ 0x0f, 0xa4 }, 0, .short, .none },
704	.{ .shld, .mrc, &.{ .rm32, .r32, .cl }, &.{ 0x0f, 0xa5 }, 0, .none },	708	.{ .shld, .mrc, &.{ .rm16, .r16, .cl }, &.{ 0x0f, 0xa5 }, 0, .short, .none },
705	.{ .shld, .mrc, &.{ .rm64, .r64, .cl }, &.{ 0x0f, 0xa5 }, 0, .long },	709	.{ .shld, .mri, &.{ .rm32, .r32, .imm8 }, &.{ 0x0f, 0xa4 }, 0, .none, .none },
706		710	.{ .shld, .mri, &.{ .rm64, .r64, .imm8 }, &.{ 0x0f, 0xa4 }, 0, .long, .none },
707	.{ .shr, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 5, .none },	711	.{ .shld, .mrc, &.{ .rm32, .r32, .cl }, &.{ 0x0f, 0xa5 }, 0, .none, .none },
708	.{ .shr, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 5, .rex },	712	.{ .shld, .mrc, &.{ .rm64, .r64, .cl }, &.{ 0x0f, 0xa5 }, 0, .long, .none },
709	.{ .shr, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 5, .none },	713
710	.{ .shr, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 5, .none },	714	.{ .shr, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 5, .none, .none },
711	.{ .shr, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 5, .long },	715	.{ .shr, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 5, .rex, .none },
712	.{ .shr, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 5, .none },	716	.{ .shr, .m1, &.{ .rm16, .unity }, &.{ 0xd1 }, 5, .short, .none },
713	.{ .shr, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 5, .rex },	717	.{ .shr, .m1, &.{ .rm32, .unity }, &.{ 0xd1 }, 5, .none, .none },
714	.{ .shr, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 5, .none },	718	.{ .shr, .m1, &.{ .rm64, .unity }, &.{ 0xd1 }, 5, .long, .none },
715	.{ .shr, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 5, .none },	719	.{ .shr, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 5, .none, .none },
716	.{ .shr, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 5, .long },	720	.{ .shr, .mc, &.{ .rm8, .cl }, &.{ 0xd2 }, 5, .rex, .none },
717	.{ .shr, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 5, .none },	721	.{ .shr, .mc, &.{ .rm16, .cl }, &.{ 0xd3 }, 5, .short, .none },
718	.{ .shr, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 5, .rex },	722	.{ .shr, .mc, &.{ .rm32, .cl }, &.{ 0xd3 }, 5, .none, .none },
719	.{ .shr, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 5, .none },	723	.{ .shr, .mc, &.{ .rm64, .cl }, &.{ 0xd3 }, 5, .long, .none },
720	.{ .shr, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 5, .none },	724	.{ .shr, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 5, .none, .none },
721	.{ .shr, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 5, .long },	725	.{ .shr, .mi, &.{ .rm8, .imm8 }, &.{ 0xc0 }, 5, .rex, .none },
722		726	.{ .shr, .mi, &.{ .rm16, .imm8 }, &.{ 0xc1 }, 5, .short, .none },
723	.{ .shrd, .mri, &.{ .rm16, .r16, .imm8 }, &.{ 0x0f, 0xac }, 0, .none },	727	.{ .shr, .mi, &.{ .rm32, .imm8 }, &.{ 0xc1 }, 5, .none, .none },
724	.{ .shrd, .mrc, &.{ .rm16, .r16, .cl }, &.{ 0x0f, 0xad }, 0, .none },	728	.{ .shr, .mi, &.{ .rm64, .imm8 }, &.{ 0xc1 }, 5, .long, .none },
725	.{ .shrd, .mri, &.{ .rm32, .r32, .imm8 }, &.{ 0x0f, 0xac }, 0, .none },	729
726	.{ .shrd, .mri, &.{ .rm64, .r64, .imm8 }, &.{ 0x0f, 0xac }, 0, .long },	730	.{ .shrd, .mri, &.{ .rm16, .r16, .imm8 }, &.{ 0x0f, 0xac }, 0, .short, .none },
727	.{ .shrd, .mrc, &.{ .rm32, .r32, .cl }, &.{ 0x0f, 0xad }, 0, .none },	731	.{ .shrd, .mrc, &.{ .rm16, .r16, .cl }, &.{ 0x0f, 0xad }, 0, .short, .none },
728	.{ .shrd, .mrc, &.{ .rm64, .r64, .cl }, &.{ 0x0f, 0xad }, 0, .long },	732	.{ .shrd, .mri, &.{ .rm32, .r32, .imm8 }, &.{ 0x0f, 0xac }, 0, .none, .none },
729		733	.{ .shrd, .mri, &.{ .rm64, .r64, .imm8 }, &.{ 0x0f, 0xac }, 0, .long, .none },
730	.{ .stos, .np, &.{ .m8 }, &.{ 0xaa }, 0, .none },	734	.{ .shrd, .mrc, &.{ .rm32, .r32, .cl }, &.{ 0x0f, 0xad }, 0, .none, .none },
731	.{ .stos, .np, &.{ .m16 }, &.{ 0xab }, 0, .none },	735	.{ .shrd, .mrc, &.{ .rm64, .r64, .cl }, &.{ 0x0f, 0xad }, 0, .long, .none },
732	.{ .stos, .np, &.{ .m32 }, &.{ 0xab }, 0, .none },	736
733	.{ .stos, .np, &.{ .m64 }, &.{ 0xab }, 0, .long },	737	.{ .stos, .np, &.{ .m8 }, &.{ 0xaa }, 0, .none, .none },
734		738	.{ .stos, .np, &.{ .m16 }, &.{ 0xab }, 0, .short, .none },
735	.{ .stosb, .np, &.{}, &.{ 0xaa }, 0, .none },	739	.{ .stos, .np, &.{ .m32 }, &.{ 0xab }, 0, .none, .none },
736	.{ .stosw, .np, &.{}, &.{ 0xab }, 0, .short },	740	.{ .stos, .np, &.{ .m64 }, &.{ 0xab }, 0, .long, .none },
737	.{ .stosd, .np, &.{}, &.{ 0xab }, 0, .none },	741
738	.{ .stosq, .np, &.{}, &.{ 0xab }, 0, .long },	742	.{ .stosb, .np, &.{}, &.{ 0xaa }, 0, .none, .none },
739		743	.{ .stosw, .np, &.{}, &.{ 0xab }, 0, .short, .none },
740	.{ .sub, .zi, &.{ .al, .imm8 }, &.{ 0x2c }, 0, .none },	744	.{ .stosd, .np, &.{}, &.{ 0xab }, 0, .none, .none },
741	.{ .sub, .zi, &.{ .ax, .imm16 }, &.{ 0x2d }, 0, .none },	745	.{ .stosq, .np, &.{}, &.{ 0xab }, 0, .long, .none },
742	.{ .sub, .zi, &.{ .eax, .imm32 }, &.{ 0x2d }, 0, .none },	746
743	.{ .sub, .zi, &.{ .rax, .imm32s }, &.{ 0x2d }, 0, .long },	747	.{ .sub, .zi, &.{ .al, .imm8 }, &.{ 0x2c }, 0, .none, .none },
744	.{ .sub, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 5, .none },	748	.{ .sub, .zi, &.{ .ax, .imm16 }, &.{ 0x2d }, 0, .short, .none },
745	.{ .sub, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 5, .rex },	749	.{ .sub, .zi, &.{ .eax, .imm32 }, &.{ 0x2d }, 0, .none, .none },
746	.{ .sub, .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 5, .none },	750	.{ .sub, .zi, &.{ .rax, .imm32s }, &.{ 0x2d }, 0, .long, .none },
747	.{ .sub, .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 5, .none },	751	.{ .sub, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 5, .none, .none },
748	.{ .sub, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 5, .long },	752	.{ .sub, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 5, .rex, .none },
749	.{ .sub, .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 5, .none },	753	.{ .sub, .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 5, .short, .none },
750	.{ .sub, .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 5, .none },	754	.{ .sub, .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 5, .none, .none },
751	.{ .sub, .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 5, .long },	755	.{ .sub, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 5, .long, .none },
752	.{ .sub, .mr, &.{ .rm8, .r8 }, &.{ 0x28 }, 0, .none },	756	.{ .sub, .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 5, .short, .none },
753	.{ .sub, .mr, &.{ .rm8, .r8 }, &.{ 0x28 }, 0, .rex },	757	.{ .sub, .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 5, .none, .none },
754	.{ .sub, .mr, &.{ .rm16, .r16 }, &.{ 0x29 }, 0, .none },	758	.{ .sub, .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 5, .long, .none },
755	.{ .sub, .mr, &.{ .rm32, .r32 }, &.{ 0x29 }, 0, .none },	759	.{ .sub, .mr, &.{ .rm8, .r8 }, &.{ 0x28 }, 0, .none, .none },
756	.{ .sub, .mr, &.{ .rm64, .r64 }, &.{ 0x29 }, 0, .long },	760	.{ .sub, .mr, &.{ .rm8, .r8 }, &.{ 0x28 }, 0, .rex, .none },
757	.{ .sub, .rm, &.{ .r8, .rm8 }, &.{ 0x2a }, 0, .none },	761	.{ .sub, .mr, &.{ .rm16, .r16 }, &.{ 0x29 }, 0, .short, .none },
758	.{ .sub, .rm, &.{ .r8, .rm8 }, &.{ 0x2a }, 0, .rex },	762	.{ .sub, .mr, &.{ .rm32, .r32 }, &.{ 0x29 }, 0, .none, .none },
759	.{ .sub, .rm, &.{ .r16, .rm16 }, &.{ 0x2b }, 0, .none },	763	.{ .sub, .mr, &.{ .rm64, .r64 }, &.{ 0x29 }, 0, .long, .none },
760	.{ .sub, .rm, &.{ .r32, .rm32 }, &.{ 0x2b }, 0, .none },	764	.{ .sub, .rm, &.{ .r8, .rm8 }, &.{ 0x2a }, 0, .none, .none },
761	.{ .sub, .rm, &.{ .r64, .rm64 }, &.{ 0x2b }, 0, .long },	765	.{ .sub, .rm, &.{ .r8, .rm8 }, &.{ 0x2a }, 0, .rex, .none },
762		766	.{ .sub, .rm, &.{ .r16, .rm16 }, &.{ 0x2b }, 0, .short, .none },
763	.{ .syscall, .np, &.{}, &.{ 0x0f, 0x05 }, 0, .none }	767	.{ .sub, .rm, &.{ .r32, .rm32 }, &.{ 0x2b }, 0, .none, .none },
764	,	768	.{ .sub, .rm, &.{ .r64, .rm64 }, &.{ 0x2b }, 0, .long, .none },
765	.{ .@"test", .zi, &.{ .al, .imm8 }, &.{ 0xa8 }, 0, .none },	769
766	.{ .@"test", .zi, &.{ .ax, .imm16 }, &.{ 0xa9 }, 0, .none },	770	.{ .syscall, .np, &.{}, &.{ 0x0f, 0x05 }, 0, .none, .none },
767	.{ .@"test", .zi, &.{ .eax, .imm32 }, &.{ 0xa9 }, 0, .none },	771
768	.{ .@"test", .zi, &.{ .rax, .imm32s }, &.{ 0xa9 }, 0, .long },	772	.{ .@"test", .zi, &.{ .al, .imm8 }, &.{ 0xa8 }, 0, .none, .none },
769	.{ .@"test", .mi, &.{ .rm8, .imm8 }, &.{ 0xf6 }, 0, .none },	773	.{ .@"test", .zi, &.{ .ax, .imm16 }, &.{ 0xa9 }, 0, .short, .none },
770	.{ .@"test", .mi, &.{ .rm8, .imm8 }, &.{ 0xf6 }, 0, .rex },	774	.{ .@"test", .zi, &.{ .eax, .imm32 }, &.{ 0xa9 }, 0, .none, .none },
771	.{ .@"test", .mi, &.{ .rm16, .imm16 }, &.{ 0xf7 }, 0, .none },	775	.{ .@"test", .zi, &.{ .rax, .imm32s }, &.{ 0xa9 }, 0, .long, .none },
772	.{ .@"test", .mi, &.{ .rm32, .imm32 }, &.{ 0xf7 }, 0, .none },	776	.{ .@"test", .mi, &.{ .rm8, .imm8 }, &.{ 0xf6 }, 0, .none, .none },
773	.{ .@"test", .mi, &.{ .rm64, .imm32s }, &.{ 0xf7 }, 0, .long },	777	.{ .@"test", .mi, &.{ .rm8, .imm8 }, &.{ 0xf6 }, 0, .rex, .none },
774	.{ .@"test", .mr, &.{ .rm8, .r8 }, &.{ 0x84 }, 0, .none },	778	.{ .@"test", .mi, &.{ .rm16, .imm16 }, &.{ 0xf7 }, 0, .short, .none },
775	.{ .@"test", .mr, &.{ .rm8, .r8 }, &.{ 0x84 }, 0, .rex },	779	.{ .@"test", .mi, &.{ .rm32, .imm32 }, &.{ 0xf7 }, 0, .none, .none },
776	.{ .@"test", .mr, &.{ .rm16, .r16 }, &.{ 0x85 }, 0, .none },	780	.{ .@"test", .mi, &.{ .rm64, .imm32s }, &.{ 0xf7 }, 0, .long, .none },
777	.{ .@"test", .mr, &.{ .rm32, .r32 }, &.{ 0x85 }, 0, .none },	781	.{ .@"test", .mr, &.{ .rm8, .r8 }, &.{ 0x84 }, 0, .none, .none },
778	.{ .@"test", .mr, &.{ .rm64, .r64 }, &.{ 0x85 }, 0, .long },	782	.{ .@"test", .mr, &.{ .rm8, .r8 }, &.{ 0x84 }, 0, .rex, .none },
779		783	.{ .@"test", .mr, &.{ .rm16, .r16 }, &.{ 0x85 }, 0, .short, .none },
780	.{ .tzcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .none },	784	.{ .@"test", .mr, &.{ .rm32, .r32 }, &.{ 0x85 }, 0, .none, .none },
781	.{ .tzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .none },	785	.{ .@"test", .mr, &.{ .rm64, .r64 }, &.{ 0x85 }, 0, .long, .none },
782	.{ .tzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .long },	786
783		787	.{ .tzcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .short, .none },
784	.{ .ud2, .np, &.{}, &.{ 0x0f, 0x0b }, 0, .none },	788	.{ .tzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .none, .none },
785		789	.{ .tzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .long, .none },
786	.{ .xadd, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xc0 }, 0, .none },	790
787	.{ .xadd, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xc0 }, 0, .rex },	791	.{ .ud2, .np, &.{}, &.{ 0x0f, 0x0b }, 0, .none, .none },
788	.{ .xadd, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xc1 }, 0, .none },	792
789	.{ .xadd, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xc1 }, 0, .none },	793	.{ .xadd, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xc0 }, 0, .none, .none },
790	.{ .xadd, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xc1 }, 0, .long },	794	.{ .xadd, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xc0 }, 0, .rex, .none },
791		795	.{ .xadd, .mr, &.{ .rm16, .r16 }, &.{ 0x0f, 0xc1 }, 0, .short, .none },
792	.{ .xchg, .o, &.{ .ax, .r16 }, &.{ 0x90 }, 0, .none },	796	.{ .xadd, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xc1 }, 0, .none, .none },
793	.{ .xchg, .o, &.{ .r16, .ax }, &.{ 0x90 }, 0, .none },	797	.{ .xadd, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xc1 }, 0, .long, .none },
794	.{ .xchg, .o, &.{ .eax, .r32 }, &.{ 0x90 }, 0, .none },	798
795	.{ .xchg, .o, &.{ .rax, .r64 }, &.{ 0x90 }, 0, .long },	799	.{ .xchg, .o, &.{ .ax, .r16 }, &.{ 0x90 }, 0, .short, .none },
796	.{ .xchg, .o, &.{ .r32, .eax }, &.{ 0x90 }, 0, .none },	800	.{ .xchg, .o, &.{ .r16, .ax }, &.{ 0x90 }, 0, .short, .none },
797	.{ .xchg, .o, &.{ .r64, .rax }, &.{ 0x90 }, 0, .long },	801	.{ .xchg, .o, &.{ .eax, .r32 }, &.{ 0x90 }, 0, .none, .none },
798	.{ .xchg, .mr, &.{ .rm8, .r8 }, &.{ 0x86 }, 0, .none },	802	.{ .xchg, .o, &.{ .rax, .r64 }, &.{ 0x90 }, 0, .long, .none },
799	.{ .xchg, .mr, &.{ .rm8, .r8 }, &.{ 0x86 }, 0, .rex },	803	.{ .xchg, .o, &.{ .r32, .eax }, &.{ 0x90 }, 0, .none, .none },
800	.{ .xchg, .rm, &.{ .r8, .rm8 }, &.{ 0x86 }, 0, .none },	804	.{ .xchg, .o, &.{ .r64, .rax }, &.{ 0x90 }, 0, .long, .none },
801	.{ .xchg, .rm, &.{ .r8, .rm8 }, &.{ 0x86 }, 0, .rex },	805	.{ .xchg, .mr, &.{ .rm8, .r8 }, &.{ 0x86 }, 0, .none, .none },
802	.{ .xchg, .mr, &.{ .rm16, .r16 }, &.{ 0x87 }, 0, .none },	806	.{ .xchg, .mr, &.{ .rm8, .r8 }, &.{ 0x86 }, 0, .rex, .none },
803	.{ .xchg, .rm, &.{ .r16, .rm16 }, &.{ 0x87 }, 0, .none },	807	.{ .xchg, .rm, &.{ .r8, .rm8 }, &.{ 0x86 }, 0, .none, .none },
804	.{ .xchg, .mr, &.{ .rm32, .r32 }, &.{ 0x87 }, 0, .none },	808	.{ .xchg, .rm, &.{ .r8, .rm8 }, &.{ 0x86 }, 0, .rex, .none },
805	.{ .xchg, .mr, &.{ .rm64, .r64 }, &.{ 0x87 }, 0, .long },	809	.{ .xchg, .mr, &.{ .rm16, .r16 }, &.{ 0x87 }, 0, .short, .none },
806	.{ .xchg, .rm, &.{ .r32, .rm32 }, &.{ 0x87 }, 0, .none },	810	.{ .xchg, .rm, &.{ .r16, .rm16 }, &.{ 0x87 }, 0, .short, .none },
807	.{ .xchg, .rm, &.{ .r64, .rm64 }, &.{ 0x87 }, 0, .long },	811	.{ .xchg, .mr, &.{ .rm32, .r32 }, &.{ 0x87 }, 0, .none, .none },
808		812	.{ .xchg, .mr, &.{ .rm64, .r64 }, &.{ 0x87 }, 0, .long, .none },
809	.{ .xor, .zi, &.{ .al, .imm8 }, &.{ 0x34 }, 0, .none },	813	.{ .xchg, .rm, &.{ .r32, .rm32 }, &.{ 0x87 }, 0, .none, .none },
810	.{ .xor, .zi, &.{ .ax, .imm16 }, &.{ 0x35 }, 0, .none },	814	.{ .xchg, .rm, &.{ .r64, .rm64 }, &.{ 0x87 }, 0, .long, .none },
811	.{ .xor, .zi, &.{ .eax, .imm32 }, &.{ 0x35 }, 0, .none },	815
812	.{ .xor, .zi, &.{ .rax, .imm32s }, &.{ 0x35 }, 0, .long },	816	.{ .xor, .zi, &.{ .al, .imm8 }, &.{ 0x34 }, 0, .none, .none },
813	.{ .xor, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 6, .none },	817	.{ .xor, .zi, &.{ .ax, .imm16 }, &.{ 0x35 }, 0, .short, .none },
814	.{ .xor, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 6, .rex },	818	.{ .xor, .zi, &.{ .eax, .imm32 }, &.{ 0x35 }, 0, .none, .none },
815	.{ .xor, .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 6, .none },	819	.{ .xor, .zi, &.{ .rax, .imm32s }, &.{ 0x35 }, 0, .long, .none },
816	.{ .xor, .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 6, .none },	820	.{ .xor, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 6, .none, .none },
817	.{ .xor, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 6, .long },	821	.{ .xor, .mi, &.{ .rm8, .imm8 }, &.{ 0x80 }, 6, .rex, .none },
818	.{ .xor, .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 6, .none },	822	.{ .xor, .mi, &.{ .rm16, .imm16 }, &.{ 0x81 }, 6, .short, .none },
819	.{ .xor, .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 6, .none },	823	.{ .xor, .mi, &.{ .rm32, .imm32 }, &.{ 0x81 }, 6, .none, .none },
820	.{ .xor, .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 6, .long },	824	.{ .xor, .mi, &.{ .rm64, .imm32s }, &.{ 0x81 }, 6, .long, .none },
821	.{ .xor, .mr, &.{ .rm8, .r8 }, &.{ 0x30 }, 0, .none },	825	.{ .xor, .mi, &.{ .rm16, .imm8s }, &.{ 0x83 }, 6, .short, .none },
822	.{ .xor, .mr, &.{ .rm8, .r8 }, &.{ 0x30 }, 0, .rex },	826	.{ .xor, .mi, &.{ .rm32, .imm8s }, &.{ 0x83 }, 6, .none, .none },
823	.{ .xor, .mr, &.{ .rm16, .r16 }, &.{ 0x31 }, 0, .none },	827	.{ .xor, .mi, &.{ .rm64, .imm8s }, &.{ 0x83 }, 6, .long, .none },
824	.{ .xor, .mr, &.{ .rm32, .r32 }, &.{ 0x31 }, 0, .none },	828	.{ .xor, .mr, &.{ .rm8, .r8 }, &.{ 0x30 }, 0, .none, .none },
825	.{ .xor, .mr, &.{ .rm64, .r64 }, &.{ 0x31 }, 0, .long },	829	.{ .xor, .mr, &.{ .rm8, .r8 }, &.{ 0x30 }, 0, .rex, .none },
826	.{ .xor, .rm, &.{ .r8, .rm8 }, &.{ 0x32 }, 0, .none },	830	.{ .xor, .mr, &.{ .rm16, .r16 }, &.{ 0x31 }, 0, .short, .none },
827	.{ .xor, .rm, &.{ .r8, .rm8 }, &.{ 0x32 }, 0, .rex },	831	.{ .xor, .mr, &.{ .rm32, .r32 }, &.{ 0x31 }, 0, .none, .none },
828	.{ .xor, .rm, &.{ .r16, .rm16 }, &.{ 0x33 }, 0, .none },	832	.{ .xor, .mr, &.{ .rm64, .r64 }, &.{ 0x31 }, 0, .long, .none },
829	.{ .xor, .rm, &.{ .r32, .rm32 }, &.{ 0x33 }, 0, .none },	833	.{ .xor, .rm, &.{ .r8, .rm8 }, &.{ 0x32 }, 0, .none, .none },
830	.{ .xor, .rm, &.{ .r64, .rm64 }, &.{ 0x33 }, 0, .long },	834	.{ .xor, .rm, &.{ .r8, .rm8 }, &.{ 0x32 }, 0, .rex, .none },
		835	.{ .xor, .rm, &.{ .r16, .rm16 }, &.{ 0x33 }, 0, .short, .none },
		836	.{ .xor, .rm, &.{ .r32, .rm32 }, &.{ 0x33 }, 0, .none, .none },
		837	.{ .xor, .rm, &.{ .r64, .rm64 }, &.{ 0x33 }, 0, .long, .none },
831		838
832	// SSE	839	// SSE
833	.{ .addss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x58 }, 0, .sse },	840	.{ .addps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x58 }, 0, .none, .sse },
834		841
835	.{ .andnps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x55 }, 0, .sse },	842	.{ .addss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x58 }, 0, .none, .sse },
836		843
837	.{ .andps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x54 }, 0, .sse },	844	.{ .andnps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x55 }, 0, .none, .sse },
838		845
839	.{ .cmpss, .rmi, &.{ .xmm, .xmm_m32, .imm8 }, &.{ 0xf3, 0x0f, 0xc2 }, 0, .sse },	846	.{ .andps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x54 }, 0, .none, .sse },
840		847
841	.{ .cvtsi2ss, .rm, &.{ .xmm, .rm32 }, &.{ 0xf3, 0x0f, 0x2a }, 0, .sse },	848	.{ .cmpss, .rmi, &.{ .xmm, .xmm_m32, .imm8 }, &.{ 0xf3, 0x0f, 0xc2 }, 0, .none, .sse },
842	.{ .cvtsi2ss, .rm, &.{ .xmm, .rm64 }, &.{ 0xf3, 0x0f, 0x2a }, 0, .sse_long },
843		849
844	.{ .divss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5e }, 0, .sse },	850	.{ .cvtsi2ss, .rm, &.{ .xmm, .rm32 }, &.{ 0xf3, 0x0f, 0x2a }, 0, .none, .sse },
		851	.{ .cvtsi2ss, .rm, &.{ .xmm, .rm64 }, &.{ 0xf3, 0x0f, 0x2a }, 0, .long, .sse },
845		852
846	.{ .maxss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5f }, 0, .sse },	853	.{ .divps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x5e }, 0, .none, .sse },
847		854
848	.{ .minss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5d }, 0, .sse },	855	.{ .divss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5e }, 0, .none, .sse },
849		856
850	.{ .movaps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x28 }, 0, .sse },	857	.{ .maxps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x5f }, 0, .none, .sse },
851	.{ .movaps, .mr, &.{ .xmm_m128, .xmm }, &.{ 0x0f, 0x29 }, 0, .sse },
852		858
853	.{ .movss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x10 }, 0, .sse },	859	.{ .maxss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5f }, 0, .none, .sse },
854	.{ .movss, .mr, &.{ .xmm_m32, .xmm }, &.{ 0xf3, 0x0f, 0x11 }, 0, .sse },
855		860
856	.{ .movups, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x10 }, 0, .sse },	861	.{ .minps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x5d }, 0, .none, .sse },
857	.{ .movups, .mr, &.{ .xmm_m128, .xmm }, &.{ 0x0f, 0x11 }, 0, .sse },
858		862
859	.{ .mulss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x59 }, 0, .sse },	863	.{ .minss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5d }, 0, .none, .sse },
860		864
861	.{ .orps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x56 }, 0, .sse },	865	.{ .movaps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x28 }, 0, .none, .sse },
		866	.{ .movaps, .mr, &.{ .xmm_m128, .xmm }, &.{ 0x0f, 0x29 }, 0, .none, .sse },
862		867
863	.{ .subss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5c }, 0, .sse },	868	.{ .movhlps, .rm, &.{ .xmm, .xmm }, &.{ 0x0f, 0x12 }, 0, .none, .sse },
864		869
865	.{ .sqrtps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x51 }, 0, .sse },	870	.{ .movss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x10 }, 0, .none, .sse },
866	.{ .sqrtss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x51 }, 0, .sse },	871	.{ .movss, .mr, &.{ .xmm_m32, .xmm }, &.{ 0xf3, 0x0f, 0x11 }, 0, .none, .sse },
867		872
868	.{ .ucomiss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x0f, 0x2e }, 0, .sse },	873	.{ .movups, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x10 }, 0, .none, .sse },
		874	.{ .movups, .mr, &.{ .xmm_m128, .xmm }, &.{ 0x0f, 0x11 }, 0, .none, .sse },
869		875
870	.{ .xorps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x57 }, 0, .sse },	876	.{ .mulps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x59 }, 0, .none, .sse },
		877
		878	.{ .mulss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x59 }, 0, .none, .sse },
		879
		880	.{ .orps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x56 }, 0, .none, .sse },
		881
		882	.{ .subps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x5c }, 0, .none, .sse },
		883
		884	.{ .subss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5c }, 0, .none, .sse },
		885
		886	.{ .sqrtps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x51 }, 0, .none, .sse },
		887
		888	.{ .sqrtss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x51 }, 0, .none, .sse },
		889
		890	.{ .ucomiss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x0f, 0x2e }, 0, .none, .sse },
		891
		892	.{ .xorps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x57 }, 0, .none, .sse },
871		893
872	// SSE2	894	// SSE2
873	.{ .addsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x58 }, 0, .sse2 },	895	.{ .addpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x58 }, 0, .none, .sse2 },
		896
		897	.{ .addsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x58 }, 0, .none, .sse2 },
		898
		899	.{ .andnpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x55 }, 0, .none, .sse2 },
		900
		901	.{ .andpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x54 }, 0, .none, .sse2 },
		902
		903	.{ .cmpsd, .rmi, &.{ .xmm, .xmm_m64, .imm8 }, &.{ 0xf2, 0x0f, 0xc2 }, 0, .none, .sse2 },
		904
		905	.{ .cvtsd2ss, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5a }, 0, .none, .sse2 },
		906
		907	.{ .cvtsi2sd, .rm, &.{ .xmm, .rm32 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .none, .sse2 },
		908	.{ .cvtsi2sd, .rm, &.{ .xmm, .rm64 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .long, .sse2 },
		909
		910	.{ .cvtss2sd, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5a }, 0, .none, .sse2 },
		911
		912	.{ .divpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5e }, 0, .none, .sse2 },
		913
		914	.{ .divsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5e }, 0, .none, .sse2 },
		915
		916	.{ .maxpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5f }, 0, .none, .sse2 },
		917
		918	.{ .maxsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5f }, 0, .none, .sse2 },
		919
		920	.{ .minpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5d }, 0, .none, .sse2 },
		921
		922	.{ .minsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5d }, 0, .none, .sse2 },
		923
		924	.{ .movapd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x28 }, 0, .none, .sse2 },
		925	.{ .movapd, .mr, &.{ .xmm_m128, .xmm }, &.{ 0x66, 0x0f, 0x29 }, 0, .none, .sse2 },
		926
		927	.{ .movd, .rm, &.{ .xmm, .rm32 }, &.{ 0x66, 0x0f, 0x6e }, 0, .none, .sse2 },
		928	.{ .movd, .mr, &.{ .rm32, .xmm }, &.{ 0x66, 0x0f, 0x7e }, 0, .none, .sse2 },
874		929
875	.{ .andnpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x55 }, 0, .sse2 },	930	.{ .movq, .rm, &.{ .xmm, .rm64 }, &.{ 0x66, 0x0f, 0x6e }, 0, .long, .sse2 },
		931	.{ .movq, .mr, &.{ .rm64, .xmm }, &.{ 0x66, 0x0f, 0x7e }, 0, .long, .sse2 },
876		932
877	.{ .andpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x54 }, 0, .sse2 },	933	.{ .movq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf3, 0x0f, 0x7e }, 0, .none, .sse2 },
		934	.{ .movq, .mr, &.{ .xmm_m64, .xmm }, &.{ 0x66, 0x0f, 0xd6 }, 0, .none, .sse2 },
878		935
879	.{ .cmpsd, .rmi, &.{ .xmm, .xmm_m64, .imm8 }, &.{ 0xf2, 0x0f, 0xc2 }, 0, .sse2 },	936	.{ .movupd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x10 }, 0, .none, .sse2 },
		937	.{ .movupd, .mr, &.{ .xmm_m128, .xmm }, &.{ 0x66, 0x0f, 0x11 }, 0, .none, .sse2 },
880		938
881	.{ .cvtsd2ss, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5a }, 0, .sse2 },	939	.{ .mulpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x59 }, 0, .none, .sse2 },
882		940
883	.{ .cvtsi2sd, .rm, &.{ .xmm, .rm32 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .sse2 },	941	.{ .mulsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x59 }, 0, .none, .sse2 },
884	.{ .cvtsi2sd, .rm, &.{ .xmm, .rm64 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .sse2_long },
885		942
886	.{ .cvtss2sd, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5a }, 0, .sse2 },	943	.{ .orpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x56 }, 0, .none, .sse2 },
887		944
888	.{ .divsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5e }, 0, .sse2 },	945	.{ .pextrw, .rmi, &.{ .r32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0xc5 }, 0, .none, .sse2 },
889		946
890	.{ .maxsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5f }, 0, .sse2 },	947	.{ .pinsrw, .rmi, &.{ .xmm, .r32_m16, .imm8 }, &.{ 0x66, 0x0f, 0xc4 }, 0, .none, .sse2 },
891		948
892	.{ .minsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5d }, 0, .sse2 },	949	.{ .pshufhw, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0xf3, 0x0f, 0x70 }, 0, .none, .sse2 },
893		950
894	.{ .movapd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x28 }, 0, .sse2 },	951	.{ .pshuflw, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0xf2, 0x0f, 0x70 }, 0, .none, .sse2 },
895	.{ .movapd, .mr, &.{ .xmm_m128, .xmm }, &.{ 0x66, 0x0f, 0x29 }, 0, .sse2 },
896		952
897	.{ .movd, .rm, &.{ .xmm, .rm32 }, &.{ 0x66, 0x0f, 0x6e }, 0, .sse2 },	953	.{ .psrlw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd1 }, 0, .none, .sse2 },
898	.{ .movd, .mr, &.{ .rm32, .xmm }, &.{ 0x66, 0x0f, 0x7e }, 0, .sse2 },	954	.{ .psrlw, .mi, &.{ .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x71 }, 2, .none, .sse2 },
		955	.{ .psrld, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd2 }, 0, .none, .sse2 },
		956	.{ .psrld, .mi, &.{ .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x72 }, 2, .none, .sse2 },
		957	.{ .psrlq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd3 }, 0, .none, .sse2 },
		958	.{ .psrlq, .mi, &.{ .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x73 }, 2, .none, .sse2 },
899		959
900	.{ .movq, .rm, &.{ .xmm, .rm64 }, &.{ 0x66, 0x0f, 0x6e }, 0, .sse2_long },	960	.{ .punpckhbw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x68 }, 0, .none, .sse2 },
901	.{ .movq, .mr, &.{ .rm64, .xmm }, &.{ 0x66, 0x0f, 0x7e }, 0, .sse2_long },	961	.{ .punpckhwd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x69 }, 0, .none, .sse2 },
		962	.{ .punpckhdq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6a }, 0, .none, .sse2 },
		963	.{ .punpckhqdq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6d }, 0, .none, .sse2 },
902		964
903	.{ .movq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf3, 0x0f, 0x7e }, 0, .sse2 },	965	.{ .punpcklbw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x60 }, 0, .none, .sse2 },
904	.{ .movq, .mr, &.{ .xmm_m64, .xmm }, &.{ 0x66, 0x0f, 0xd6 }, 0, .sse2 },	966	.{ .punpcklwd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x61 }, 0, .none, .sse2 },
		967	.{ .punpckldq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x62 }, 0, .none, .sse2 },
		968	.{ .punpcklqdq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6c }, 0, .none, .sse2 },
905		969
906	.{ .movupd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x10 }, 0, .sse2 },	970	.{ .sqrtpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x51 }, 0, .none, .sse2 },
907	.{ .movupd, .mr, &.{ .xmm_m128, .xmm }, &.{ 0x66, 0x0f, 0x11 }, 0, .sse2 },
908		971
909	.{ .mulsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x59 }, 0, .sse2 },	972	.{ .sqrtsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x51 }, 0, .none, .sse2 },
910		973
911	.{ .orpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x56 }, 0, .sse2 },	974	.{ .subpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5c }, 0, .none, .sse2 },
912		975
913	.{ .pextrw, .mri, &.{ .r16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0xc5 }, 0, .sse2 },	976	.{ .subsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5c }, 0, .none, .sse2 },
914		977
915	.{ .pinsrw, .rmi, &.{ .xmm, .rm16, .imm8 }, &.{ 0x66, 0x0f, 0xc4 }, 0, .sse2 },	978	.{ .movsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x10 }, 0, .none, .sse2 },
		979	.{ .movsd, .mr, &.{ .xmm_m64, .xmm }, &.{ 0xf2, 0x0f, 0x11 }, 0, .none, .sse2 },
916		980
917	.{ .sqrtpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x51 }, 0, .sse2 },	981	.{ .ucomisd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x2e }, 0, .none, .sse2 },
918	.{ .sqrtsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x51 }, 0, .sse2 },
919		982
920	.{ .subsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5c }, 0, .sse2 },	983	.{ .xorpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x57 }, 0, .none, .sse2 },
921		984
922	.{ .movsd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x10 }, 0, .sse2 },	985	// SSE3
923	.{ .movsd, .mr, &.{ .xmm_m64, .xmm }, &.{ 0xf2, 0x0f, 0x11 }, 0, .sse2 },	986	.{ .movddup, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x12 }, 0, .none, .sse3 },
924		987
925	.{ .ucomisd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x2e }, 0, .sse2 },	988	.{ .movshdup, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x16 }, 0, .none, .sse3 },
926		989
927	.{ .xorpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x57 }, 0, .sse2 },	990	.{ .movsldup, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x12 }, 0, .none, .sse3 },
928		991
929	// SSE4.1	992	// SSE4.1
930	.{ .pextrw, .mri, &.{ .rm16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .sse4_1 },	993	.{ .pextrb, .mri, &.{ .r32_m8, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x14 }, 0, .none, .sse4_1 },
		994	.{ .pextrd, .mri, &.{ .rm32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .none, .sse4_1 },
		995	.{ .pextrq, .mri, &.{ .rm64, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .long, .sse4_1 },
		996
		997	.{ .pextrw, .mri, &.{ .r32_m16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .none, .sse4_1 },
		998
		999	.{ .pinsrb, .rmi, &.{ .xmm, .r32_m8, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x20 }, 0, .none, .sse4_1 },
		1000	.{ .pinsrd, .rmi, &.{ .xmm, .rm32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .none, .sse4_1 },
		1001	.{ .pinsrq, .rmi, &.{ .xmm, .rm64, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .long, .sse4_1 },
		1002
		1003	.{ .roundpd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x09 }, 0, .none, .sse4_1 },
		1004
		1005	.{ .roundps, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x08 }, 0, .none, .sse4_1 },
		1006
		1007	.{ .roundsd, .rmi, &.{ .xmm, .xmm_m64, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0b }, 0, .none, .sse4_1 },
		1008
		1009	.{ .roundss, .rmi, &.{ .xmm, .xmm_m32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0a }, 0, .none, .sse4_1 },
		1010
		1011	// AVX
		1012	.{ .vaddpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x58 }, 0, .vex_128_wig, .avx },
		1013	.{ .vaddpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x58 }, 0, .vex_256_wig, .avx },
		1014
		1015	.{ .vaddps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x0f, 0x58 }, 0, .vex_128_wig, .avx },
		1016	.{ .vaddps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x0f, 0x58 }, 0, .vex_256_wig, .avx },
		1017
		1018	.{ .vaddsd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x58 }, 0, .vex_lig_wig, .avx },
		1019
		1020	.{ .vaddss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x58 }, 0, .vex_lig_wig, .avx },
		1021
		1022	.{ .vcvtsd2ss, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5a }, 0, .vex_lig_wig, .avx },
		1023
		1024	.{ .vcvtsi2sd, .rvm, &.{ .xmm, .xmm, .rm32 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .vex_lig_w0, .avx },
		1025	.{ .vcvtsi2sd, .rvm, &.{ .xmm, .xmm, .rm64 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .vex_lig_w1, .avx },
		1026
		1027	.{ .vcvtsi2ss, .rvm, &.{ .xmm, .xmm, .rm32 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .vex_lig_w0, .avx },
		1028	.{ .vcvtsi2ss, .rvm, &.{ .xmm, .xmm, .rm64 }, &.{ 0xf2, 0x0f, 0x2a }, 0, .vex_lig_w1, .avx },
		1029
		1030	.{ .vcvtss2sd, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf2, 0x0f, 0x5a }, 0, .vex_lig_wig, .avx },
		1031
		1032	.{ .vdivpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5e }, 0, .vex_128_wig, .avx },
		1033	.{ .vdivpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x5e }, 0, .vex_256_wig, .avx },
		1034
		1035	.{ .vdivps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x0f, 0x5e }, 0, .vex_128_wig, .avx },
		1036	.{ .vdivps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x0f, 0x5e }, 0, .vex_256_wig, .avx },
		1037
		1038	.{ .vdivsd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5e }, 0, .vex_lig_wig, .avx },
		1039
		1040	.{ .vdivss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5e }, 0, .vex_lig_wig, .avx },
		1041
		1042	.{ .vmaxpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5f }, 0, .vex_128_wig, .avx },
		1043	.{ .vmaxpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x5f }, 0, .vex_256_wig, .avx },
		1044
		1045	.{ .vmaxps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x0f, 0x5f }, 0, .vex_128_wig, .avx },
		1046	.{ .vmaxps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x0f, 0x5f }, 0, .vex_256_wig, .avx },
		1047
		1048	.{ .vmaxsd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5f }, 0, .vex_lig_wig, .avx },
		1049
		1050	.{ .vmaxss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5f }, 0, .vex_lig_wig, .avx },
		1051
		1052	.{ .vminpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5d }, 0, .vex_128_wig, .avx },
		1053	.{ .vminpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x5d }, 0, .vex_256_wig, .avx },
		1054
		1055	.{ .vminps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x0f, 0x5d }, 0, .vex_128_wig, .avx },
		1056	.{ .vminps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x0f, 0x5d }, 0, .vex_256_wig, .avx },
		1057
		1058	.{ .vminsd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5d }, 0, .vex_lig_wig, .avx },
		1059
		1060	.{ .vminss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5d }, 0, .vex_lig_wig, .avx },
		1061
		1062	.{ .vmovapd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x28 }, 0, .vex_128_wig, .avx },
		1063	.{ .vmovapd, .mr, &.{ .xmm_m128, .xmm }, &.{ 0x66, 0x0f, 0x29 }, 0, .vex_128_wig, .avx },
		1064	.{ .vmovapd, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x28 }, 0, .vex_256_wig, .avx },
		1065	.{ .vmovapd, .mr, &.{ .ymm_m256, .ymm }, &.{ 0x66, 0x0f, 0x29 }, 0, .vex_256_wig, .avx },
		1066
		1067	.{ .vmovaps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x28 }, 0, .vex_128_wig, .avx },
		1068	.{ .vmovaps, .mr, &.{ .xmm_m128, .xmm }, &.{ 0x0f, 0x29 }, 0, .vex_128_wig, .avx },
		1069	.{ .vmovaps, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x0f, 0x28 }, 0, .vex_256_wig, .avx },
		1070	.{ .vmovaps, .mr, &.{ .ymm_m256, .ymm }, &.{ 0x0f, 0x29 }, 0, .vex_256_wig, .avx },
		1071
		1072	.{ .vmovddup, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x12 }, 0, .vex_128_wig, .avx },
		1073	.{ .vmovddup, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0xf2, 0x0f, 0x12 }, 0, .vex_256_wig, .avx },
		1074
		1075	.{ .vmovhlps, .rvm, &.{ .xmm, .xmm, .xmm }, &.{ 0x0f, 0x12 }, 0, .vex_128_wig, .avx },
		1076
		1077	.{ .vmovsd, .rvm, &.{ .xmm, .xmm, .xmm }, &.{ 0xf2, 0x0f, 0x10 }, 0, .vex_lig_wig, .avx },
		1078	.{ .vmovsd, .rm, &.{ .xmm, .m64 }, &.{ 0xf2, 0x0f, 0x10 }, 0, .vex_lig_wig, .avx },
		1079	.{ .vmovsd, .mvr, &.{ .xmm, .xmm, .xmm }, &.{ 0xf2, 0x0f, 0x11 }, 0, .vex_lig_wig, .avx },
		1080	.{ .vmovsd, .mr, &.{ .m64, .xmm }, &.{ 0xf2, 0x0f, 0x11 }, 0, .vex_lig_wig, .avx },
		1081
		1082	.{ .vmovshdup, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x16 }, 0, .vex_128_wig, .avx },
		1083	.{ .vmovshdup, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0xf3, 0x0f, 0x16 }, 0, .vex_256_wig, .avx },
		1084
		1085	.{ .vmovsldup, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x12 }, 0, .vex_128_wig, .avx },
		1086	.{ .vmovsldup, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0xf3, 0x0f, 0x12 }, 0, .vex_256_wig, .avx },
		1087
		1088	.{ .vmovss, .rvm, &.{ .xmm, .xmm, .xmm }, &.{ 0xf3, 0x0f, 0x10 }, 0, .vex_lig_wig, .avx },
		1089	.{ .vmovss, .rm, &.{ .xmm, .m32 }, &.{ 0xf3, 0x0f, 0x10 }, 0, .vex_lig_wig, .avx },
		1090	.{ .vmovss, .mvr, &.{ .xmm, .xmm, .xmm }, &.{ 0xf3, 0x0f, 0x11 }, 0, .vex_lig_wig, .avx },
		1091	.{ .vmovss, .mr, &.{ .m32, .xmm }, &.{ 0xf3, 0x0f, 0x11 }, 0, .vex_lig_wig, .avx },
		1092
		1093	.{ .vmovupd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x10 }, 0, .vex_128_wig, .avx },
		1094	.{ .vmovupd, .mr, &.{ .xmm_m128, .xmm }, &.{ 0x66, 0x0f, 0x11 }, 0, .vex_128_wig, .avx },
		1095	.{ .vmovupd, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x10 }, 0, .vex_256_wig, .avx },
		1096	.{ .vmovupd, .mr, &.{ .ymm_m256, .ymm }, &.{ 0x66, 0x0f, 0x11 }, 0, .vex_256_wig, .avx },
		1097
		1098	.{ .vmovups, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x10 }, 0, .vex_128_wig, .avx },
		1099	.{ .vmovups, .mr, &.{ .xmm_m128, .xmm }, &.{ 0x0f, 0x11 }, 0, .vex_128_wig, .avx },
		1100	.{ .vmovups, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x0f, 0x10 }, 0, .vex_256_wig, .avx },
		1101	.{ .vmovups, .mr, &.{ .ymm_m256, .ymm }, &.{ 0x0f, 0x11 }, 0, .vex_256_wig, .avx },
		1102
		1103	.{ .vmulpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x59 }, 0, .vex_128_wig, .avx },
		1104	.{ .vmulpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x59 }, 0, .vex_256_wig, .avx },
		1105
		1106	.{ .vmulps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x0f, 0x59 }, 0, .vex_128_wig, .avx },
		1107	.{ .vmulps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x0f, 0x59 }, 0, .vex_256_wig, .avx },
		1108
		1109	.{ .vmulsd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x59 }, 0, .vex_lig_wig, .avx },
		1110
		1111	.{ .vmulss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x59 }, 0, .vex_lig_wig, .avx },
		1112
		1113	.{ .vpextrb, .mri, &.{ .r32_m8, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x14 }, 0, .vex_128_w0, .avx },
		1114	.{ .vpextrd, .mri, &.{ .rm32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .vex_128_w0, .avx },
		1115	.{ .vpextrq, .mri, &.{ .rm64, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .vex_128_w1, .avx },
		1116
		1117	.{ .vpextrw, .rmi, &.{ .r32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x15 }, 0, .vex_128_wig, .avx },
		1118	.{ .vpextrw, .mri, &.{ .r32_m16, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x15 }, 0, .vex_128_wig, .avx },
		1119
		1120	.{ .vpinsrb, .rmi, &.{ .xmm, .r32_m8, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x20 }, 0, .vex_128_w0, .avx },
		1121	.{ .vpinsrd, .rmi, &.{ .xmm, .rm32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .vex_128_w0, .avx },
		1122	.{ .vpinsrq, .rmi, &.{ .xmm, .rm64, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x22 }, 0, .vex_128_w1, .avx },
		1123
		1124	.{ .vpinsrw, .rvmi, &.{ .xmm, .xmm, .r32_m16, .imm8 }, &.{ 0x66, 0x0f, 0xc4 }, 0, .vex_128_wig, .avx },
		1125
		1126	.{ .vpsrlw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd1 }, 0, .vex_128_wig, .avx },
		1127	.{ .vpsrlw, .vmi, &.{ .xmm, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x71 }, 2, .vex_128_wig, .avx },
		1128	.{ .vpsrld, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd2 }, 0, .vex_128_wig, .avx },
		1129	.{ .vpsrld, .vmi, &.{ .xmm, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x72 }, 2, .vex_128_wig, .avx },
		1130	.{ .vpsrlq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd3 }, 0, .vex_128_wig, .avx },
		1131	.{ .vpsrlq, .vmi, &.{ .xmm, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x73 }, 2, .vex_128_wig, .avx },
		1132
		1133	.{ .vpunpckhbw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x68 }, 0, .vex_128_wig, .avx },
		1134	.{ .vpunpckhwd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x69 }, 0, .vex_128_wig, .avx },
		1135	.{ .vpunpckhdq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6a }, 0, .vex_128_wig, .avx },
		1136	.{ .vpunpckhqdq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6d }, 0, .vex_128_wig, .avx },
		1137
		1138	.{ .vpunpcklbw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x60 }, 0, .vex_128_wig, .avx },
		1139	.{ .vpunpcklwd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x61 }, 0, .vex_128_wig, .avx },
		1140	.{ .vpunpckldq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x62 }, 0, .vex_128_wig, .avx },
		1141	.{ .vpunpcklqdq, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6c }, 0, .vex_128_wig, .avx },
		1142
		1143	.{ .vroundpd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x09 }, 0, .vex_128_wig, .avx },
		1144	.{ .vroundpd, .rmi, &.{ .ymm, .ymm_m256, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x09 }, 0, .vex_256_wig, .avx },
		1145
		1146	.{ .vroundps, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x08 }, 0, .vex_128_wig, .avx },
		1147	.{ .vroundps, .rmi, &.{ .ymm, .ymm_m256, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x08 }, 0, .vex_256_wig, .avx },
		1148
		1149	.{ .vroundsd, .rvmi, &.{ .xmm, .xmm, .xmm_m64, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0b }, 0, .vex_lig_wig, .avx },
		1150
		1151	.{ .vroundss, .rvmi, &.{ .xmm, .xmm, .xmm_m32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0a }, 0, .vex_lig_wig, .avx },
		1152
		1153	.{ .vsqrtpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x51 }, 0, .vex_128_wig, .avx },
		1154	.{ .vsqrtpd, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x51 }, 0, .vex_256_wig, .avx },
		1155
		1156	.{ .vsqrtps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x51 }, 0, .vex_128_wig, .avx },
		1157	.{ .vsqrtps, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x0f, 0x51 }, 0, .vex_256_wig, .avx },
		1158
		1159	.{ .vsqrtsd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x51 }, 0, .vex_lig_wig, .avx },
		1160
		1161	.{ .vsqrtss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x51 }, 0, .vex_lig_wig, .avx },
		1162
		1163	.{ .vsubpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x5c }, 0, .vex_128_wig, .avx },
		1164	.{ .vsubpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x5c }, 0, .vex_256_wig, .avx },
		1165
		1166	.{ .vsubps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x0f, 0x5c }, 0, .vex_128_wig, .avx },
		1167	.{ .vsubps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x0f, 0x5c }, 0, .vex_256_wig, .avx },
		1168
		1169	.{ .vsubsd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0xf2, 0x0f, 0x5c }, 0, .vex_lig_wig, .avx },
		1170
		1171	.{ .vsubss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0xf3, 0x0f, 0x5c }, 0, .vex_lig_wig, .avx },
		1172
		1173	// F16C
		1174	.{ .vcvtph2ps, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x13 }, 0, .vex_128_w0, .f16c },
		1175	.{ .vcvtph2ps, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x13 }, 0, .vex_256_w0, .f16c },
		1176
		1177	.{ .vcvtps2ph, .mri, &.{ .xmm_m64, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x1d }, 0, .vex_128_w0, .f16c },
		1178	.{ .vcvtps2ph, .mri, &.{ .xmm_m128, .ymm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x1d }, 0, .vex_256_w0, .f16c },
		1179
		1180	// FMA
		1181	.{ .vfmadd132pd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x98 }, 0, .vex_128_w1, .fma },
		1182	.{ .vfmadd213pd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0xa8 }, 0, .vex_128_w1, .fma },
		1183	.{ .vfmadd231pd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0xb8 }, 0, .vex_128_w1, .fma },
		1184	.{ .vfmadd132pd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x98 }, 0, .vex_256_w1, .fma },
		1185	.{ .vfmadd213pd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0xa8 }, 0, .vex_256_w1, .fma },
		1186	.{ .vfmadd231pd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0xb8 }, 0, .vex_256_w1, .fma },
		1187
		1188	.{ .vfmadd132ps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x98 }, 0, .vex_128_w0, .fma },
		1189	.{ .vfmadd213ps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0xa8 }, 0, .vex_128_w0, .fma },
		1190	.{ .vfmadd231ps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0xb8 }, 0, .vex_128_w0, .fma },
		1191	.{ .vfmadd132ps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x98 }, 0, .vex_256_w0, .fma },
		1192	.{ .vfmadd213ps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0xa8 }, 0, .vex_256_w0, .fma },
		1193	.{ .vfmadd231ps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0xb8 }, 0, .vex_256_w0, .fma },
		1194
		1195	.{ .vfmadd132sd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x99 }, 0, .vex_lig_w1, .fma },
		1196	.{ .vfmadd213sd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0xa9 }, 0, .vex_lig_w1, .fma },
		1197	.{ .vfmadd231sd, .rvm, &.{ .xmm, .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0xb9 }, 0, .vex_lig_w1, .fma },
		1198
		1199	.{ .vfmadd132ss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x99 }, 0, .vex_lig_w0, .fma },
		1200	.{ .vfmadd213ss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0xa9 }, 0, .vex_lig_w0, .fma },
		1201	.{ .vfmadd231ss, .rvm, &.{ .xmm, .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0xb9 }, 0, .vex_lig_w0, .fma },
		1202
		1203	// AVX2
		1204	.{ .vpsrlw, .rvm, &.{ .ymm, .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd1 }, 0, .vex_256_wig, .avx2 },
		1205	.{ .vpsrlw, .vmi, &.{ .ymm, .ymm, .imm8 }, &.{ 0x66, 0x0f, 0x71 }, 2, .vex_256_wig, .avx2 },
		1206	.{ .vpsrld, .rvm, &.{ .ymm, .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd2 }, 0, .vex_256_wig, .avx2 },
		1207	.{ .vpsrld, .vmi, &.{ .ymm, .ymm, .imm8 }, &.{ 0x66, 0x0f, 0x72 }, 2, .vex_256_wig, .avx2 },
		1208	.{ .vpsrlq, .rvm, &.{ .ymm, .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xd3 }, 0, .vex_256_wig, .avx2 },
		1209	.{ .vpsrlq, .vmi, &.{ .ymm, .ymm, .imm8 }, &.{ 0x66, 0x0f, 0x73 }, 2, .vex_256_wig, .avx2 },
		1210
		1211	.{ .vpunpckhbw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x68 }, 0, .vex_256_wig, .avx2 },
		1212	.{ .vpunpckhwd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x69 }, 0, .vex_256_wig, .avx2 },
		1213	.{ .vpunpckhdq, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x6a }, 0, .vex_256_wig, .avx2 },
		1214	.{ .vpunpckhqdq, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x6d }, 0, .vex_256_wig, .avx2 },
931		1215
932	.{ .roundss, .rmi, &.{ .xmm, .xmm_m32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0a }, 0, .sse4_1 },	1216	.{ .vpunpcklbw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x60 }, 0, .vex_256_wig, .avx2 },
933	.{ .roundsd, .rmi, &.{ .xmm, .xmm_m64, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0b }, 0, .sse4_1 },	1217	.{ .vpunpcklwd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x61 }, 0, .vex_256_wig, .avx2 },
		1218	.{ .vpunpckldq, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x62 }, 0, .vex_256_wig, .avx2 },
		1219	.{ .vpunpcklqdq, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x6c }, 0, .vex_256_wig, .avx2 },
934	};	1220	};
935	// zig fmt: on	1221	// zig fmt: on

src/link/Dwarf.zig+29-25

...	@@ -526,7 +526,7 @@ pub const DeclState = struct {	...	@@ -526,7 +526,7 @@ pub const DeclState = struct {
526	.ErrorUnion => {	526	.ErrorUnion => {
527	const error_ty = ty.errorUnionSet();	527	const error_ty = ty.errorUnionSet();
528	const payload_ty = ty.errorUnionPayload();	528	const payload_ty = ty.errorUnionPayload();
529	const payload_align = payload_ty.abiAlignment(target);	529	const payload_align = if (payload_ty.isNoReturn()) 0 else payload_ty.abiAlignment(target);
530	const error_align = Type.anyerror.abiAlignment(target);	530	const error_align = Type.anyerror.abiAlignment(target);
531	const abi_size = ty.abiSize(target);	531	const abi_size = ty.abiSize(target);
532	const payload_off = if (error_align >= payload_align) Type.anyerror.abiSize(target) else 0;	532	const payload_off = if (error_align >= payload_align) Type.anyerror.abiSize(target) else 0;
...	@@ -540,31 +540,35 @@ pub const DeclState = struct {	...	@@ -540,31 +540,35 @@ pub const DeclState = struct {
540	const name = try ty.nameAllocArena(arena, module);	540	const name = try ty.nameAllocArena(arena, module);
541	try dbg_info_buffer.writer().print("{s}\x00", .{name});	541	try dbg_info_buffer.writer().print("{s}\x00", .{name});
542		542
543	// DW.AT.member	543	if (!payload_ty.isNoReturn()) {
544	try dbg_info_buffer.ensureUnusedCapacity(7);	544	// DW.AT.member
545	dbg_info_buffer.appendAssumeCapacity(@enumToInt(AbbrevKind.struct_member));	545	try dbg_info_buffer.ensureUnusedCapacity(7);
546	// DW.AT.name, DW.FORM.string	546	dbg_info_buffer.appendAssumeCapacity(@enumToInt(AbbrevKind.struct_member));
547	dbg_info_buffer.appendSliceAssumeCapacity("value");	547	// DW.AT.name, DW.FORM.string
548	dbg_info_buffer.appendAssumeCapacity(0);	548	dbg_info_buffer.appendSliceAssumeCapacity("value");
549	// DW.AT.type, DW.FORM.ref4	549	dbg_info_buffer.appendAssumeCapacity(0);
550	var index = dbg_info_buffer.items.len;	550	// DW.AT.type, DW.FORM.ref4
551	try dbg_info_buffer.resize(index + 4);	551	const index = dbg_info_buffer.items.len;
552	try self.addTypeRelocGlobal(atom_index, payload_ty, @intCast(u32, index));	552	try dbg_info_buffer.resize(index + 4);
553	// DW.AT.data_member_location, DW.FORM.sdata	553	try self.addTypeRelocGlobal(atom_index, payload_ty, @intCast(u32, index));
554	try leb128.writeULEB128(dbg_info_buffer.writer(), payload_off);	554	// DW.AT.data_member_location, DW.FORM.sdata
		555	try leb128.writeULEB128(dbg_info_buffer.writer(), payload_off);
		556	}
555		557
556	// DW.AT.member	558	{
557	try dbg_info_buffer.ensureUnusedCapacity(5);	559	// DW.AT.member
558	dbg_info_buffer.appendAssumeCapacity(@enumToInt(AbbrevKind.struct_member));	560	try dbg_info_buffer.ensureUnusedCapacity(5);
559	// DW.AT.name, DW.FORM.string	561	dbg_info_buffer.appendAssumeCapacity(@enumToInt(AbbrevKind.struct_member));
560	dbg_info_buffer.appendSliceAssumeCapacity("err");	562	// DW.AT.name, DW.FORM.string
561	dbg_info_buffer.appendAssumeCapacity(0);	563	dbg_info_buffer.appendSliceAssumeCapacity("err");
562	// DW.AT.type, DW.FORM.ref4	564	dbg_info_buffer.appendAssumeCapacity(0);
563	index = dbg_info_buffer.items.len;	565	// DW.AT.type, DW.FORM.ref4
564	try dbg_info_buffer.resize(index + 4);	566	const index = dbg_info_buffer.items.len;
565	try self.addTypeRelocGlobal(atom_index, error_ty, @intCast(u32, index));	567	try dbg_info_buffer.resize(index + 4);
566	// DW.AT.data_member_location, DW.FORM.sdata	568	try self.addTypeRelocGlobal(atom_index, error_ty, @intCast(u32, index));
567	try leb128.writeULEB128(dbg_info_buffer.writer(), error_off);	569	// DW.AT.data_member_location, DW.FORM.sdata
		570	try leb128.writeULEB128(dbg_info_buffer.writer(), error_off);
		571	}
568		572
569	// DW.AT.structure_type delimit children	573	// DW.AT.structure_type delimit children
570	try dbg_info_buffer.append(0);	574	try dbg_info_buffer.append(0);

test/behavior/bugs/12891.zig-6

test/behavior/bugs/2114.zig+2-1

...	@@ -9,7 +9,8 @@ fn ctz(x: anytype) usize {	...	@@ -9,7 +9,8 @@ fn ctz(x: anytype) usize {
9		9
10	test "fixed" {	10	test "fixed" {
11	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO	11	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
12	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO	12	if (builtin.zig_backend == .stage2_x86_64 and
		13	!comptime std.Target.x86.featureSetHas(builtin.cpu.features, .bmi)) return error.SkipZigTest; // TODO
13	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	14	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
14	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	15	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
15	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	16	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/error.zig-3

test/behavior/field_parent_ptr.zig-1

...	@@ -2,7 +2,6 @@ const expect = @import("std").testing.expect;	...	@@ -2,7 +2,6 @@ const expect = @import("std").testing.expect;
2	const builtin = @import("builtin");	2	const builtin = @import("builtin");
3		3
4	test "@fieldParentPtr non-first field" {	4	test "@fieldParentPtr non-first field" {
5	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
6	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;	5	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
7	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	6	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
8	try testParentFieldPtr(&foo.c);	7	try testParentFieldPtr(&foo.c);

test/behavior/floatop.zig+17-14

test/behavior/math.zig+4-2

test/behavior/muladd.zig+8-4

...	@@ -1,8 +1,12 @@	...	@@ -1,8 +1,12 @@
		1	const std = @import("std");
1	const builtin = @import("builtin");	2	const builtin = @import("builtin");
2	const expect = @import("std").testing.expect;	3	const expect = std.testing.expect;
		4
		5	const no_x86_64_hardware_fma_support = builtin.zig_backend == .stage2_x86_64 and
		6	!std.Target.x86.featureSetHas(builtin.cpu.features, .fma);
3		7
4	test "@mulAdd" {	8	test "@mulAdd" {
5	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO	9	if (no_x86_64_hardware_fma_support) return error.SkipZigTest; // TODO
6	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	10	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
7	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	11	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
8	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	12	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	@@ -116,7 +120,7 @@ fn vector32() !void {	...	@@ -116,7 +120,7 @@ fn vector32() !void {
116		120
117	test "vector f32" {	121	test "vector f32" {
118	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO	122	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
119	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO	123	if (no_x86_64_hardware_fma_support) return error.SkipZigTest; // TODO
120	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	124	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
121	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	125	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
122	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	126	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	@@ -139,7 +143,7 @@ fn vector64() !void {	...	@@ -139,7 +143,7 @@ fn vector64() !void {
139		143
140	test "vector f64" {	144	test "vector f64" {
141	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO	145	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
142	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO	146	if (no_x86_64_hardware_fma_support) return error.SkipZigTest; // TODO
143	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	147	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
144	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	148	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
145	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	149	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

test/behavior/vector.zig+2-1

...	@@ -168,7 +168,8 @@ test "array to vector" {	...	@@ -168,7 +168,8 @@ test "array to vector" {
168		168
169	test "array to vector with element type coercion" {	169	test "array to vector with element type coercion" {
170	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO	170	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
171	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO	171	if (builtin.zig_backend == .stage2_x86_64 and
		172	!comptime std.Target.x86.featureSetHas(builtin.cpu.features, .f16c)) return error.SkipZigTest; // TODO
172	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	173	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
173	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	174	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
174	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	175	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

...	@@ -326,7 +326,7 @@ pub const all_features = blk: {	...	@@ -326,7 +326,7 @@ pub const all_features = blk: {
326	};	326	};
327	result[@enumToInt(Feature.avx512ifma)] = .{	327	result[@enumToInt(Feature.avx512ifma)] = .{
328	.llvm_name = "avx512ifma",	328	.llvm_name = "avx512ifma",
329	.description = "Enable AVX-512 Integer Fused Multiple-Add",	329	.description = "Enable AVX-512 Integer Fused Multiply-Add",
330	.dependencies = featureSet(&[_]Feature{	330	.dependencies = featureSet(&[_]Feature{
331	.avx512f,	331	.avx512f,
332	}),	332	}),
...	@@ -599,14 +599,14 @@ pub const all_features = blk: {	...	@@ -599,14 +599,14 @@ pub const all_features = blk: {
599	};	599	};
600	result[@enumToInt(Feature.fma)] = .{	600	result[@enumToInt(Feature.fma)] = .{
601	.llvm_name = "fma",	601	.llvm_name = "fma",
602	.description = "Enable three-operand fused multiple-add",	602	.description = "Enable three-operand fused multiply-add",
603	.dependencies = featureSet(&[_]Feature{	603	.dependencies = featureSet(&[_]Feature{
604	.avx,	604	.avx,
605	}),	605	}),
606	};	606	};
607	result[@enumToInt(Feature.fma4)] = .{	607	result[@enumToInt(Feature.fma4)] = .{
608	.llvm_name = "fma4",	608	.llvm_name = "fma4",
609	.description = "Enable four-operand fused multiple-add",	609	.description = "Enable four-operand fused multiply-add",
610	.dependencies = featureSet(&[_]Feature{	610	.dependencies = featureSet(&[_]Feature{
611	.avx,	611	.avx,
612	.sse4a,	612	.sse4a,

...	@@ -29,7 +29,6 @@ test "inf >= 1" {	...	@@ -29,7 +29,6 @@ test "inf >= 1" {
29	test "isNan(nan * 1)" {	29	test "isNan(nan * 1)" {
30	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	30	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
31	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	31	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
32	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
33		32
34	const nan_times_one = comptime std.math.nan(f64) * 1;	33	const nan_times_one = comptime std.math.nan(f64) * 1;
35	try std.testing.expect(std.math.isNan(nan_times_one));	34	try std.testing.expect(std.math.isNan(nan_times_one));
...	@@ -37,7 +36,6 @@ test "isNan(nan * 1)" {	...	@@ -37,7 +36,6 @@ test "isNan(nan * 1)" {
37	test "runtime isNan(nan * 1)" {	36	test "runtime isNan(nan * 1)" {
38	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	37	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
39	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	38	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
40	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
41		39
42	const nan_times_one = std.math.nan(f64) * 1;	40	const nan_times_one = std.math.nan(f64) * 1;
43	try std.testing.expect(std.math.isNan(nan_times_one));	41	try std.testing.expect(std.math.isNan(nan_times_one));
...	@@ -45,7 +43,6 @@ test "runtime isNan(nan * 1)" {	...	@@ -45,7 +43,6 @@ test "runtime isNan(nan * 1)" {
45	test "isNan(nan * 0)" {	43	test "isNan(nan * 0)" {
46	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	44	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
47	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	45	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
48	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
49		46
50	const nan_times_zero = comptime std.math.nan(f64) * 0;	47	const nan_times_zero = comptime std.math.nan(f64) * 0;
51	try std.testing.expect(std.math.isNan(nan_times_zero));	48	try std.testing.expect(std.math.isNan(nan_times_zero));
...	@@ -55,7 +52,6 @@ test "isNan(nan * 0)" {	...	@@ -55,7 +52,6 @@ test "isNan(nan * 0)" {
55	test "isNan(inf * 0)" {	52	test "isNan(inf * 0)" {
56	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	53	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
57	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	54	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
58	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
59		55
60	const inf_times_zero = comptime std.math.inf(f64) * 0;	56	const inf_times_zero = comptime std.math.inf(f64) * 0;
61	try std.testing.expect(std.math.isNan(inf_times_zero));	57	try std.testing.expect(std.math.isNan(inf_times_zero));
...	@@ -65,7 +61,6 @@ test "isNan(inf * 0)" {	...	@@ -65,7 +61,6 @@ test "isNan(inf * 0)" {
65	test "runtime isNan(nan * 0)" {	61	test "runtime isNan(nan * 0)" {
66	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	62	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
67	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	63	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
68	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
69		64
70	const nan_times_zero = std.math.nan(f64) * 0;	65	const nan_times_zero = std.math.nan(f64) * 0;
71	try std.testing.expect(std.math.isNan(nan_times_zero));	66	try std.testing.expect(std.math.isNan(nan_times_zero));
...	@@ -75,7 +70,6 @@ test "runtime isNan(nan * 0)" {	...	@@ -75,7 +70,6 @@ test "runtime isNan(nan * 0)" {
75	test "runtime isNan(inf * 0)" {	70	test "runtime isNan(inf * 0)" {
76	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	71	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
77	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	72	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
78	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
79		73
80	const inf_times_zero = std.math.inf(f64) * 0;	74	const inf_times_zero = std.math.inf(f64) * 0;
81	try std.testing.expect(std.math.isNan(inf_times_zero));	75	try std.testing.expect(std.math.isNan(inf_times_zero));

...	@@ -757,7 +757,6 @@ test "error union of noreturn used with if" {	...	@@ -757,7 +757,6 @@ test "error union of noreturn used with if" {
757	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	757	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
758	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	758	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
759	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO	759	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
760	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
761	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	760	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
762		761
763	NoReturn.a = 64;	762	NoReturn.a = 64;
...	@@ -772,7 +771,6 @@ test "error union of noreturn used with try" {	...	@@ -772,7 +771,6 @@ test "error union of noreturn used with try" {
772	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	771	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
773	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	772	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
774	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO	773	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
775	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
776	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	774	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
777		775
778	NoReturn.a = 64;	776	NoReturn.a = 64;
...	@@ -784,7 +782,6 @@ test "error union of noreturn used with catch" {	...	@@ -784,7 +782,6 @@ test "error union of noreturn used with catch" {
784	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	782	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
785	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	783	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
786	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO	784	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
787	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
788	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	785	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
789		786
790	NoReturn.a = 64;	787	NoReturn.a = 64;

...	@@ -8,6 +8,8 @@ const has_f80_rt = switch (builtin.cpu.arch) {	...	@@ -8,6 +8,8 @@ const has_f80_rt = switch (builtin.cpu.arch) {
8	.x86_64, .x86 => true,	8	.x86_64, .x86 => true,
9	else => false,	9	else => false,
10	};	10	};
		11	const no_x86_64_hardware_f16_support = builtin.zig_backend == .stage2_x86_64 and
		12	!std.Target.x86.featureSetHas(builtin.cpu.features, .f16c);
11		13
12	const epsilon_16 = 0.001;	14	const epsilon_16 = 0.001;
13	const epsilon = 0.000001;	15	const epsilon = 0.000001;
...	@@ -52,7 +54,7 @@ fn testFloatComparisons() !void {	...	@@ -52,7 +54,7 @@ fn testFloatComparisons() !void {
52	}	54	}
53		55
54	test "different sized float comparisons" {	56	test "different sized float comparisons" {
55	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO	57	if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
56	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	58	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
57	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	59	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
58	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	60	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	@@ -134,7 +136,6 @@ fn testSqrt() !void {	...	@@ -134,7 +136,6 @@ fn testSqrt() !void {
134		136
135	test "@sqrt with vectors" {	137	test "@sqrt with vectors" {
136	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO	138	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
137	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
138	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	139	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
139	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	140	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
140		141
...	@@ -152,7 +153,7 @@ fn testSqrtWithVectors() !void {	...	@@ -152,7 +153,7 @@ fn testSqrtWithVectors() !void {
152	}	153	}
153		154
154	test "more @sqrt f16 tests" {	155	test "more @sqrt f16 tests" {
155	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO	156	if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
156	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	157	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
157	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	158	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
158	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	159	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	@@ -202,7 +203,7 @@ fn testSqrtLegacy(comptime T: type, x: T) !void {	...	@@ -202,7 +203,7 @@ fn testSqrtLegacy(comptime T: type, x: T) !void {
202	}	203	}
203		204
204	test "@sin" {	205	test "@sin" {
205	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO	206	if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
206	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	207	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
207	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	208	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
208	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	209	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	@@ -241,7 +242,7 @@ fn testSinWithVectors() !void {	...	@@ -241,7 +242,7 @@ fn testSinWithVectors() !void {
241	}	242	}
242		243
243	test "@cos" {	244	test "@cos" {
244	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO	245	if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
245	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	246	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
246	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	247	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
247	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	248	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	@@ -280,7 +281,7 @@ fn testCosWithVectors() !void {	...	@@ -280,7 +281,7 @@ fn testCosWithVectors() !void {
280	}	281	}
281		282
282	test "@exp" {	283	test "@exp" {
283	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO	284	if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
284	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	285	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
285	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	286	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
286	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	287	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	@@ -318,7 +319,7 @@ fn testExpWithVectors() !void {	...	@@ -318,7 +319,7 @@ fn testExpWithVectors() !void {
318	}	319	}
319		320
320	test "@exp2" {	321	test "@exp2" {
321	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO	322	if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
322	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	323	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
323	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	324	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
324	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	325	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	@@ -403,7 +404,7 @@ test "@log with @vectors" {	...	@@ -403,7 +404,7 @@ test "@log with @vectors" {
403	}	404	}
404		405
405	test "@log2" {	406	test "@log2" {
406	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO	407	if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
407	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	408	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
408	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	409	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
409	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	410	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	@@ -445,7 +446,7 @@ fn testLog2WithVectors() !void {	...	@@ -445,7 +446,7 @@ fn testLog2WithVectors() !void {
445	}	446	}
446		447
447	test "@log10" {	448	test "@log10" {
448	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO	449	if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
449	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	450	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
450	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	451	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
451	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	452	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	@@ -617,7 +618,8 @@ fn testFloor() !void {	...	@@ -617,7 +618,8 @@ fn testFloor() !void {
617		618
618	test "@floor with vectors" {	619	test "@floor with vectors" {
619	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO	620	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
620	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO	621	if (builtin.zig_backend == .stage2_x86_64 and
		622	!comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest; // TODO
621	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	623	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
622	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	624	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
623		625
...	@@ -707,7 +709,8 @@ fn testCeil() !void {	...	@@ -707,7 +709,8 @@ fn testCeil() !void {
707		709
708	test "@ceil with vectors" {	710	test "@ceil with vectors" {
709	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO	711	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
710	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO	712	if (builtin.zig_backend == .stage2_x86_64 and
		713	!comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest; // TODO
711	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	714	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
712	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	715	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
713		716
...	@@ -797,7 +800,8 @@ fn testTrunc() !void {	...	@@ -797,7 +800,8 @@ fn testTrunc() !void {
797		800
798	test "@trunc with vectors" {	801	test "@trunc with vectors" {
799	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO	802	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
800	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO	803	if (builtin.zig_backend == .stage2_x86_64 and
		804	!comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest; // TODO
801	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	805	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
802	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	806	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
803		807
...	@@ -878,7 +882,7 @@ fn testTruncLegacy(comptime T: type, x: T) !void {	...	@@ -878,7 +882,7 @@ fn testTruncLegacy(comptime T: type, x: T) !void {
878	}	882	}
879		883
880	test "negation f16" {	884	test "negation f16" {
881	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO	885	if (no_x86_64_hardware_f16_support) return error.SkipZigTest; // TODO
882	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	886	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
883	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	887	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
884	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	888	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	@@ -1037,7 +1041,6 @@ test "comptime_float zero divided by zero produces zero" {	...	@@ -1037,7 +1041,6 @@ test "comptime_float zero divided by zero produces zero" {
1037	}	1041	}
1038		1042
1039	test "nan negation f16" {	1043	test "nan negation f16" {
1040	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1041	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	1044	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1042	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	1045	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1043	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	1046	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO

...	@@ -77,7 +77,8 @@ fn testClz() !void {	...	@@ -77,7 +77,8 @@ fn testClz() !void {
77	}	77	}
78		78
79	test "@clz big ints" {	79	test "@clz big ints" {
80	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO	80	if (builtin.zig_backend == .stage2_x86_64 and
		81	!comptime std.Target.x86.featureSetHas(builtin.cpu.features, .lzcnt)) return error.SkipZigTest; // TODO
81	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	82	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
82	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	83	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
83	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	84	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	@@ -398,7 +399,8 @@ fn testBinaryNot128(comptime Type: type, x: Type) !void {	...	@@ -398,7 +399,8 @@ fn testBinaryNot128(comptime Type: type, x: Type) !void {
398		399
399	test "division" {	400	test "division" {
400	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO	401	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
401	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO	402	if (builtin.zig_backend == .stage2_x86_64 and
		403	!comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest; // TODO
402	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO	404	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
403	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	405	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
404	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	406	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO