authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-01-03 03:56:43-05:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-01-16 20:47:30-05:00
logb7acd977896a13d37d3f592627e55d372aeedc6a
tree0ea1f74df3abf9bd88b298464617e18d671e87c1
parent074232b3e53f6a6bbf0851a97c733a848e739bd8

x86_64: fix hazards exposed by new calling convention

5 files changed, 196 insertions(+), 132 deletions(-)

src/arch/x86_64/CodeGen.zig+48-25
......@@ -22303,8 +22303,8 @@ fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: C
2230322303 .off = -dst_reg_off.off,
2230422304 } },
2230522305 }, opts),
22306 inline .register_pair, .register_triple, .register_quadruple => |dst_regs| {
22307 const src_info: ?struct { addr_reg: Register, addr_lock: RegisterLock } = switch (src_mcv) {
22306 inline .register_pair, .register_triple, .register_quadruple => |dst_regs, dst_tag| {
22307 const src_info: ?struct { addr_reg: Register, addr_lock: RegisterLock } = src_info: switch (src_mcv) {
2230822308 .register => |src_reg| switch (dst_regs[0].class()) {
2230922309 .general_purpose => switch (src_reg.class()) {
2231022310 else => unreachable,
......@@ -22329,43 +22329,66 @@ fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: C
2232922329 },
2233022330 else => unreachable,
2233122331 },
22332 .register_pair, .memory, .indirect, .load_frame => null,
22333 .load_symbol, .load_direct, .load_got, .load_tlv => src: {
22332 dst_tag => |src_regs| {
22333 var hazard_regs = src_regs;
22334 for (dst_regs, &hazard_regs, 1..) |dst_reg, src_reg, hazard_index| {
22335 const dst_id = dst_reg.id();
22336 if (dst_id == src_reg.id()) continue;
22337 var mir_tag: Mir.Inst.Tag = .mov;
22338 for (hazard_regs[hazard_index..]) |*hazard_reg| {
22339 if (dst_id != hazard_reg.id()) continue;
22340 mir_tag = .xchg;
22341 hazard_reg.* = src_reg;
22342 }
22343 try self.asmRegisterRegister(.{ ._, mir_tag }, dst_reg.to64(), src_reg.to64());
22344 }
22345 return;
22346 },
22347 .memory, .indirect, .load_frame => null,
22348 .load_symbol, .load_direct, .load_got, .load_tlv => {
2233422349 const src_addr_reg =
2233522350 (try self.register_manager.allocReg(null, abi.RegisterClass.gp)).to64();
2233622351 const src_addr_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg);
2233722352 errdefer self.register_manager.unlockReg(src_addr_lock);
2233822353
2233922354 try self.genSetReg(src_addr_reg, .usize, src_mcv.address(), opts);
22340 break :src .{ .addr_reg = src_addr_reg, .addr_lock = src_addr_lock };
22355 break :src_info .{ .addr_reg = src_addr_reg, .addr_lock = src_addr_lock };
2234122356 },
22342 .air_ref => |src_ref| return self.genCopy(
22343 ty,
22344 dst_mcv,
22345 try self.resolveInst(src_ref),
22346 opts,
22347 ),
22357 .air_ref => |src_ref| return self.genCopy(ty, dst_mcv, try self.resolveInst(src_ref), opts),
2234822358 else => return self.fail("TODO implement genCopy for {s} of {}", .{
2234922359 @tagName(src_mcv), ty.fmt(pt),
2235022360 }),
2235122361 };
2235222362 defer if (src_info) |info| self.register_manager.unlockReg(info.addr_lock);
2235322363
22354 var part_disp: i32 = 0;
22355 for (dst_regs, try self.splitType(dst_regs.len, ty), 0..) |dst_reg, dst_ty, part_i| {
22356 try self.genSetReg(dst_reg, dst_ty, switch (src_mcv) {
22357 inline .register_pair,
22358 .register_triple,
22359 .register_quadruple,
22360 => |src_regs| .{ .register = src_regs[part_i] },
22361 .memory, .indirect, .load_frame => src_mcv.address().offset(part_disp).deref(),
22362 .load_symbol, .load_direct, .load_got, .load_tlv => .{ .indirect = .{
22363 .reg = src_info.?.addr_reg,
22364 .off = part_disp,
22365 } },
22364 for ([_]bool{ false, true }) |emit_hazard| {
22365 var hazard_count: u3 = 0;
22366 var part_disp: i32 = 0;
22367 for (dst_regs, try self.splitType(dst_regs.len, ty), 0..) |dst_reg, dst_ty, part_i| {
22368 defer part_disp += @intCast(dst_ty.abiSize(pt.zcu));
22369 const is_hazard = if (src_mcv.getReg()) |src_reg|
22370 dst_reg.id() == src_reg.id()
22371 else if (src_info) |info|
22372 dst_reg.id() == info.addr_reg.id()
22373 else
22374 false;
22375 if (is_hazard) hazard_count += 1;
22376 if (is_hazard != emit_hazard) continue;
22377 try self.genSetReg(dst_reg, dst_ty, switch (src_mcv) {
22378 dst_tag => |src_regs| .{ .register = src_regs[part_i] },
22379 .memory, .indirect, .load_frame => src_mcv.address().offset(part_disp).deref(),
22380 .load_symbol, .load_direct, .load_got, .load_tlv => .{ .indirect = .{
22381 .reg = src_info.?.addr_reg,
22382 .off = part_disp,
22383 } },
22384 else => unreachable,
22385 }, opts);
22386 }
22387 switch (hazard_count) {
22388 0 => break,
22389 1 => continue,
2236622390 else => unreachable,
22367 }, opts);
22368 part_disp += @intCast(dst_ty.abiSize(pt.zcu));
22391 }
2236922392 }
2237022393 },
2237122394 .indirect => |reg_off| try self.genSetMem(
src/arch/x86_64/Disassembler.zig+33-15
......@@ -38,28 +38,46 @@ pub fn next(dis: *Disassembler) Error!?Instruction {
3838
3939 const enc = try dis.parseEncoding(prefixes) orelse return error.UnknownOpcode;
4040 switch (enc.data.op_en) {
41 .zo => return inst(enc, .{}),
42 .d, .i => {
43 const imm = try dis.parseImm(enc.data.ops[0]);
41 .z => return inst(enc, .{}),
42 .o => {
43 const reg_low_enc: u3 = @truncate(dis.code[dis.pos - 1]);
4444 return inst(enc, .{
45 .op1 = .{ .imm = imm },
45 .op1 = .{ .reg = parseGpRegister(reg_low_enc, prefixes.rex.b, prefixes.rex, enc.data.ops[0].regBitSize()) },
4646 });
4747 },
48 .zi => {
48 .zo => {
49 const reg_low_enc: u3 = @truncate(dis.code[dis.pos - 1]);
50 return inst(enc, .{
51 .op1 = .{ .reg = enc.data.ops[0].toReg() },
52 .op2 = .{ .reg = parseGpRegister(reg_low_enc, prefixes.rex.b, prefixes.rex, enc.data.ops[1].regBitSize()) },
53 });
54 },
55 .oz => {
56 const reg_low_enc: u3 = @truncate(dis.code[dis.pos - 1]);
57 return inst(enc, .{
58 .op1 = .{ .reg = parseGpRegister(reg_low_enc, prefixes.rex.b, prefixes.rex, enc.data.ops[0].regBitSize()) },
59 .op2 = .{ .reg = enc.data.ops[1].toReg() },
60 });
61 },
62 .oi => {
63 const reg_low_enc: u3 = @truncate(dis.code[dis.pos - 1]);
4964 const imm = try dis.parseImm(enc.data.ops[1]);
5065 return inst(enc, .{
51 .op1 = .{ .reg = Register.rax.toBitSize(enc.data.ops[0].regBitSize()) },
66 .op1 = .{ .reg = parseGpRegister(reg_low_enc, prefixes.rex.b, prefixes.rex, enc.data.ops[0].regBitSize()) },
5267 .op2 = .{ .imm = imm },
5368 });
5469 },
55 .o, .oi => {
56 const reg_low_enc = @as(u3, @truncate(dis.code[dis.pos - 1]));
57 const op2: Instruction.Operand = if (enc.data.op_en == .oi) .{
58 .imm = try dis.parseImm(enc.data.ops[1]),
59 } else .none;
70 .i, .d => {
71 const imm = try dis.parseImm(enc.data.ops[0]);
6072 return inst(enc, .{
61 .op1 = .{ .reg = parseGpRegister(reg_low_enc, prefixes.rex.b, prefixes.rex, enc.data.ops[0].regBitSize()) },
62 .op2 = op2,
73 .op1 = .{ .imm = imm },
74 });
75 },
76 .zi => {
77 const imm = try dis.parseImm(enc.data.ops[1]);
78 return inst(enc, .{
79 .op1 = .{ .reg = enc.data.ops[0].toReg() },
80 .op2 = .{ .imm = imm },
6381 });
6482 },
6583 .m, .mi, .m1, .mc => {
......@@ -118,7 +136,7 @@ pub fn next(dis: *Disassembler) Error!?Instruction {
118136 const seg = segmentRegister(prefixes.legacy);
119137 const offset = try dis.parseOffset();
120138 return inst(enc, .{
121 .op1 = .{ .reg = Register.rax.toBitSize(enc.data.ops[0].regBitSize()) },
139 .op1 = .{ .reg = enc.data.ops[0].toReg() },
122140 .op2 = .{ .mem = Memory.initMoffs(seg, offset) },
123141 });
124142 },
......@@ -127,7 +145,7 @@ pub fn next(dis: *Disassembler) Error!?Instruction {
127145 const offset = try dis.parseOffset();
128146 return inst(enc, .{
129147 .op1 = .{ .mem = Memory.initMoffs(seg, offset) },
130 .op2 = .{ .reg = Register.rax.toBitSize(enc.data.ops[1].regBitSize()) },
148 .op2 = .{ .reg = enc.data.ops[1].toReg() },
131149 });
132150 },
133151 .mr, .mri, .mrc => {
src/arch/x86_64/Encoding.zig+26-6
......@@ -176,9 +176,14 @@ pub fn format(
176176 for (opc) |byte| try writer.print("{x:0>2} ", .{byte});
177177
178178 switch (encoding.data.op_en) {
179 .zo, .fd, .td, .i, .zi, .d => {},
180 .o, .oi => {
181 const tag = switch (encoding.data.ops[0]) {
179 .z, .fd, .td, .i, .zi, .d => {},
180 .o, .zo, .oz, .oi => {
181 const op = switch (encoding.data.op_en) {
182 .o, .oz, .oi => encoding.data.ops[0],
183 .zo => encoding.data.ops[1],
184 else => unreachable,
185 };
186 const tag = switch (op) {
182187 .r8 => "rb",
183188 .r16 => "rw",
184189 .r32 => "rd",
......@@ -213,7 +218,7 @@ pub fn format(
213218 try writer.print("{s} ", .{tag});
214219 },
215220 .rvmr => try writer.writeAll("/is4 "),
216 .zo, .fd, .td, .o, .m, .m1, .mc, .mr, .rm, .mrc, .rm0, .rvm, .mvr, .rmv => {},
221 .z, .fd, .td, .o, .zo, .oz, .m, .m1, .mc, .mr, .rm, .mrc, .rm0, .rvm, .mvr, .rmv => {},
217222 }
218223
219224 try writer.print("{s} ", .{@tagName(encoding.mnemonic)});
......@@ -455,8 +460,8 @@ pub const Mnemonic = enum {
455460
456461pub const OpEn = enum {
457462 // zig fmt: off
458 zo,
459 o, oi,
463 z,
464 o, zo, oz, oi,
460465 i, zi,
461466 d, m,
462467 fd, td,
......@@ -575,6 +580,21 @@ pub const Op = enum {
575580 };
576581 }
577582
583 pub fn toReg(op: Op) Register {
584 return switch (op) {
585 else => .none,
586 .al => .al,
587 .ax => .ax,
588 .eax => .eax,
589 .rax => .rax,
590 .cl => .cl,
591 .rip => .rip,
592 .eip => .eip,
593 .ip => .ip,
594 .xmm0 => .xmm0,
595 };
596 }
597
578598 pub fn immBitSize(op: Op) u64 {
579599 return switch (op) {
580600 .none, .o16, .o32, .o64, .moffs, .m, .sreg => unreachable,
src/arch/x86_64/encoder.zig+11-8
......@@ -336,7 +336,7 @@ pub const Instruction = struct {
336336 .directive => .{
337337 .mnemonic = mnemonic,
338338 .data = .{
339 .op_en = .zo,
339 .op_en = .z,
340340 .ops = .{
341341 if (ops.len > 0) Encoding.Op.fromOperand(ops[0], target) else .none,
342342 if (ops.len > 1) Encoding.Op.fromOperand(ops[1], target) else .none,
......@@ -401,7 +401,7 @@ pub const Instruction = struct {
401401 }
402402
403403 switch (data.op_en) {
404 .zo, .o => {},
404 .z, .o, .zo, .oz => {},
405405 .i, .d => try encodeImm(inst.ops[0].imm, data.ops[0], encoder),
406406 .zi, .oi => try encodeImm(inst.ops[1].imm, data.ops[1], encoder),
407407 .fd => try encoder.imm64(inst.ops[1].mem.moffs.offset),
......@@ -454,7 +454,8 @@ pub const Instruction = struct {
454454 const final = opcode.len - 1;
455455 for (opcode[first..final]) |byte| try encoder.opcode_1byte(byte);
456456 switch (inst.encoding.data.op_en) {
457 .o, .oi => try encoder.opcode_withReg(opcode[final], inst.ops[0].reg.lowEnc()),
457 .o, .oz, .oi => try encoder.opcode_withReg(opcode[final], inst.ops[0].reg.lowEnc()),
458 .zo => try encoder.opcode_withReg(opcode[final], inst.ops[1].reg.lowEnc()),
458459 else => try encoder.opcode_1byte(opcode[final]),
459460 }
460461 }
......@@ -480,7 +481,7 @@ pub const Instruction = struct {
480481 }
481482
482483 const segment_override: ?Register = switch (op_en) {
483 .zo, .i, .zi, .o, .oi, .d => null,
484 .z, .i, .zi, .o, .zo, .oz, .oi, .d => null,
484485 .fd => inst.ops[1].mem.base().reg,
485486 .td => inst.ops[0].mem.base().reg,
486487 .rm, .rmi, .rm0 => if (inst.ops[1].isSegmentRegister())
......@@ -516,8 +517,9 @@ pub const Instruction = struct {
516517 rex.w = inst.encoding.data.mode == .long;
517518
518519 switch (op_en) {
519 .zo, .i, .zi, .fd, .td, .d => {},
520 .o, .oi => rex.b = inst.ops[0].reg.isExtended(),
520 .z, .i, .zi, .fd, .td, .d => {},
521 .o, .oz, .oi => rex.b = inst.ops[0].reg.isExtended(),
522 .zo => rex.b = inst.ops[1].reg.isExtended(),
521523 .m, .mi, .m1, .mc, .mr, .rm, .rmi, .mri, .mrc, .rm0, .rmv => {
522524 const r_op = switch (op_en) {
523525 .rm, .rmi, .rm0, .rmv => inst.ops[0],
......@@ -550,8 +552,9 @@ pub const Instruction = struct {
550552 vex.w = inst.encoding.data.mode.isLong();
551553
552554 switch (op_en) {
553 .zo, .i, .zi, .fd, .td, .d => {},
554 .o, .oi => vex.b = inst.ops[0].reg.isExtended(),
555 .z, .i, .zi, .fd, .td, .d => {},
556 .o, .oz, .oi => vex.b = inst.ops[0].reg.isExtended(),
557 .zo => vex.b = inst.ops[1].reg.isExtended(),
555558 .m, .mi, .m1, .mc, .mr, .rm, .rmi, .mri, .mrc, .rm0, .vmi, .rvm, .rvmr, .rvmi, .mvr, .rmv => {
556559 const r_op = switch (op_en) {
557560 .rm, .rmi, .rm0, .rvm, .rvmr, .rvmi, .rmv => inst.ops[0],
src/arch/x86_64/encodings.zig+78-78
......@@ -124,27 +124,27 @@ pub const table = [_]Entry{
124124 .{ .call, .d, &.{ .rel32 }, &.{ 0xe8 }, 0, .none, .none },
125125 .{ .call, .m, &.{ .rm64 }, &.{ 0xff }, 2, .none, .none },
126126
127 .{ .cbw, .zo, &.{ .o16 }, &.{ 0x98 }, 0, .short, .none },
128 .{ .cwde, .zo, &.{ .o32 }, &.{ 0x98 }, 0, .none, .none },
129 .{ .cdqe, .zo, &.{ .o64 }, &.{ 0x98 }, 0, .long, .none },
127 .{ .cbw, .z, &.{ .o16 }, &.{ 0x98 }, 0, .short, .none },
128 .{ .cwde, .z, &.{ .o32 }, &.{ 0x98 }, 0, .none, .none },
129 .{ .cdqe, .z, &.{ .o64 }, &.{ 0x98 }, 0, .long, .none },
130130
131 .{ .cwd, .zo, &.{ .o16 }, &.{ 0x99 }, 0, .short, .none },
132 .{ .cdq, .zo, &.{ .o32 }, &.{ 0x99 }, 0, .none, .none },
133 .{ .cqo, .zo, &.{ .o64 }, &.{ 0x99 }, 0, .long, .none },
131 .{ .cwd, .z, &.{ .o16 }, &.{ 0x99 }, 0, .short, .none },
132 .{ .cdq, .z, &.{ .o32 }, &.{ 0x99 }, 0, .none, .none },
133 .{ .cqo, .z, &.{ .o64 }, &.{ 0x99 }, 0, .long, .none },
134134
135 .{ .clac, .zo, &.{}, &.{ 0x0f, 0x01, 0xca }, 0, .none, .smap },
135 .{ .clac, .z, &.{}, &.{ 0x0f, 0x01, 0xca }, 0, .none, .smap },
136136
137 .{ .clc, .zo, &.{}, &.{ 0xf8 }, 0, .none, .none },
137 .{ .clc, .z, &.{}, &.{ 0xf8 }, 0, .none, .none },
138138
139 .{ .cld, .zo, &.{}, &.{ 0xfc }, 0, .none, .none },
139 .{ .cld, .z, &.{}, &.{ 0xfc }, 0, .none, .none },
140140
141141 .{ .clflush, .m, &.{ .m8 }, &.{ 0x0f, 0xae }, 7, .none, .none },
142142
143 .{ .cli, .zo, &.{}, &.{ 0xfa }, 0, .none, .none },
143 .{ .cli, .z, &.{}, &.{ 0xfa }, 0, .none, .none },
144144
145 .{ .clts, .zo, &.{}, &.{ 0x0f, 0x06 }, 0, .none, .none },
145 .{ .clts, .z, &.{}, &.{ 0x0f, 0x06 }, 0, .none, .none },
146146
147 .{ .clui, .zo, &.{}, &.{ 0xf3, 0x0f, 0x01, 0xee }, 0, .none, .uintr },
147 .{ .clui, .z, &.{}, &.{ 0xf3, 0x0f, 0x01, 0xee }, 0, .none, .uintr },
148148
149149 .{ .cmova, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x47 }, 0, .short, .cmov },
150150 .{ .cmova, .rm, &.{ .r32, .rm32 }, &.{ 0x0f, 0x47 }, 0, .none, .cmov },
......@@ -260,15 +260,15 @@ pub const table = [_]Entry{
260260 .{ .cmp, .rm, &.{ .r32, .rm32 }, &.{ 0x3b }, 0, .none, .none },
261261 .{ .cmp, .rm, &.{ .r64, .rm64 }, &.{ 0x3b }, 0, .long, .none },
262262
263 .{ .cmps, .zo, &.{ .m8, .m8 }, &.{ 0xa6 }, 0, .none, .none },
264 .{ .cmps, .zo, &.{ .m16, .m16 }, &.{ 0xa7 }, 0, .short, .none },
265 .{ .cmps, .zo, &.{ .m32, .m32 }, &.{ 0xa7 }, 0, .none, .none },
266 .{ .cmps, .zo, &.{ .m64, .m64 }, &.{ 0xa7 }, 0, .long, .none },
263 .{ .cmps, .z, &.{ .m8, .m8 }, &.{ 0xa6 }, 0, .none, .none },
264 .{ .cmps, .z, &.{ .m16, .m16 }, &.{ 0xa7 }, 0, .short, .none },
265 .{ .cmps, .z, &.{ .m32, .m32 }, &.{ 0xa7 }, 0, .none, .none },
266 .{ .cmps, .z, &.{ .m64, .m64 }, &.{ 0xa7 }, 0, .long, .none },
267267
268 .{ .cmpsb, .zo, &.{}, &.{ 0xa6 }, 0, .none, .none },
269 .{ .cmpsw, .zo, &.{}, &.{ 0xa7 }, 0, .short, .none },
270 .{ .cmpsd, .zo, &.{}, &.{ 0xa7 }, 0, .none, .none },
271 .{ .cmpsq, .zo, &.{}, &.{ 0xa7 }, 0, .long, .none },
268 .{ .cmpsb, .z, &.{}, &.{ 0xa6 }, 0, .none, .none },
269 .{ .cmpsw, .z, &.{}, &.{ 0xa7 }, 0, .short, .none },
270 .{ .cmpsd, .z, &.{}, &.{ 0xa7 }, 0, .none, .none },
271 .{ .cmpsq, .z, &.{}, &.{ 0xa7 }, 0, .long, .none },
272272
273273 .{ .cmpxchg, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xb0 }, 0, .none, .none },
274274 .{ .cmpxchg, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xb0 }, 0, .rex, .none },
......@@ -279,7 +279,7 @@ pub const table = [_]Entry{
279279 .{ .cmpxchg8b, .m, &.{ .m64 }, &.{ 0x0f, 0xc7 }, 1, .none, .none },
280280 .{ .cmpxchg16b, .m, &.{ .m128 }, &.{ 0x0f, 0xc7 }, 1, .long, .none },
281281
282 .{ .cpuid, .zo, &.{}, &.{ 0x0f, 0xa2 }, 0, .none, .none },
282 .{ .cpuid, .z, &.{}, &.{ 0x0f, 0xa2 }, 0, .none, .none },
283283
284284 .{ .dec, .m, &.{ .rm8 }, &.{ 0xfe }, 1, .none, .none },
285285 .{ .dec, .m, &.{ .rm8 }, &.{ 0xfe }, 1, .rex, .none },
......@@ -320,7 +320,7 @@ pub const table = [_]Entry{
320320 .{ .inc, .m, &.{ .rm32 }, &.{ 0xff }, 0, .none, .none },
321321 .{ .inc, .m, &.{ .rm64 }, &.{ 0xff }, 0, .long, .none },
322322
323 .{ .int3, .zo, &.{}, &.{ 0xcc }, 0, .none, .none },
323 .{ .int3, .z, &.{}, &.{ 0xcc }, 0, .none, .none },
324324
325325 .{ .ja, .d, &.{ .rel32 }, &.{ 0x0f, 0x87 }, 0, .none, .none },
326326 .{ .jae, .d, &.{ .rel32 }, &.{ 0x0f, 0x83 }, 0, .none, .none },
......@@ -361,23 +361,23 @@ pub const table = [_]Entry{
361361 .{ .lea, .rm, &.{ .r32, .m }, &.{ 0x8d }, 0, .none, .none },
362362 .{ .lea, .rm, &.{ .r64, .m }, &.{ 0x8d }, 0, .long, .none },
363363
364 .{ .lfence, .zo, &.{}, &.{ 0x0f, 0xae, 0xe8 }, 0, .none, .none },
364 .{ .lfence, .z, &.{}, &.{ 0x0f, 0xae, 0xe8 }, 0, .none, .none },
365365
366 .{ .lods, .zo, &.{ .m8 }, &.{ 0xac }, 0, .none, .none },
367 .{ .lods, .zo, &.{ .m16 }, &.{ 0xad }, 0, .short, .none },
368 .{ .lods, .zo, &.{ .m32 }, &.{ 0xad }, 0, .none, .none },
369 .{ .lods, .zo, &.{ .m64 }, &.{ 0xad }, 0, .long, .none },
366 .{ .lods, .z, &.{ .m8 }, &.{ 0xac }, 0, .none, .none },
367 .{ .lods, .z, &.{ .m16 }, &.{ 0xad }, 0, .short, .none },
368 .{ .lods, .z, &.{ .m32 }, &.{ 0xad }, 0, .none, .none },
369 .{ .lods, .z, &.{ .m64 }, &.{ 0xad }, 0, .long, .none },
370370
371 .{ .lodsb, .zo, &.{}, &.{ 0xac }, 0, .none, .none },
372 .{ .lodsw, .zo, &.{}, &.{ 0xad }, 0, .short, .none },
373 .{ .lodsd, .zo, &.{}, &.{ 0xad }, 0, .none, .none },
374 .{ .lodsq, .zo, &.{}, &.{ 0xad }, 0, .long, .none },
371 .{ .lodsb, .z, &.{}, &.{ 0xac }, 0, .none, .none },
372 .{ .lodsw, .z, &.{}, &.{ 0xad }, 0, .short, .none },
373 .{ .lodsd, .z, &.{}, &.{ 0xad }, 0, .none, .none },
374 .{ .lodsq, .z, &.{}, &.{ 0xad }, 0, .long, .none },
375375
376376 .{ .lzcnt, .rm, &.{ .r16, .rm16 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .short, .lzcnt },
377377 .{ .lzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .none, .lzcnt },
378378 .{ .lzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbd }, 0, .long, .lzcnt },
379379
380 .{ .mfence, .zo, &.{}, &.{ 0x0f, 0xae, 0xf0 }, 0, .none, .none },
380 .{ .mfence, .z, &.{}, &.{ 0x0f, 0xae, 0xf0 }, 0, .none, .none },
381381
382382 .{ .mov, .mr, &.{ .rm8, .r8 }, &.{ 0x88 }, 0, .none, .none },
383383 .{ .mov, .mr, &.{ .rm8, .r8 }, &.{ 0x88 }, 0, .rex, .none },
......@@ -421,15 +421,15 @@ pub const table = [_]Entry{
421421 .{ .movbe, .mr, &.{ .m32, .r32 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .none, .movbe },
422422 .{ .movbe, .mr, &.{ .m64, .r64 }, &.{ 0x0f, 0x38, 0xf1 }, 0, .long, .movbe },
423423
424 .{ .movs, .zo, &.{ .m8, .m8 }, &.{ 0xa4 }, 0, .none, .none },
425 .{ .movs, .zo, &.{ .m16, .m16 }, &.{ 0xa5 }, 0, .short, .none },
426 .{ .movs, .zo, &.{ .m32, .m32 }, &.{ 0xa5 }, 0, .none, .none },
427 .{ .movs, .zo, &.{ .m64, .m64 }, &.{ 0xa5 }, 0, .long, .none },
424 .{ .movs, .z, &.{ .m8, .m8 }, &.{ 0xa4 }, 0, .none, .none },
425 .{ .movs, .z, &.{ .m16, .m16 }, &.{ 0xa5 }, 0, .short, .none },
426 .{ .movs, .z, &.{ .m32, .m32 }, &.{ 0xa5 }, 0, .none, .none },
427 .{ .movs, .z, &.{ .m64, .m64 }, &.{ 0xa5 }, 0, .long, .none },
428428
429 .{ .movsb, .zo, &.{}, &.{ 0xa4 }, 0, .none, .none },
430 .{ .movsw, .zo, &.{}, &.{ 0xa5 }, 0, .short, .none },
431 .{ .movsd, .zo, &.{}, &.{ 0xa5 }, 0, .none, .none },
432 .{ .movsq, .zo, &.{}, &.{ 0xa5 }, 0, .long, .none },
429 .{ .movsb, .z, &.{}, &.{ 0xa4 }, 0, .none, .none },
430 .{ .movsw, .z, &.{}, &.{ 0xa5 }, 0, .short, .none },
431 .{ .movsd, .z, &.{}, &.{ 0xa5 }, 0, .none, .none },
432 .{ .movsq, .z, &.{}, &.{ 0xa5 }, 0, .long, .none },
433433
434434 .{ .movsx, .rm, &.{ .r16, .rm8 }, &.{ 0x0f, 0xbe }, 0, .short, .none },
435435 .{ .movsx, .rm, &.{ .r16, .rm8 }, &.{ 0x0f, 0xbe }, 0, .rex_short, .none },
......@@ -465,7 +465,7 @@ pub const table = [_]Entry{
465465 .{ .neg, .m, &.{ .rm32 }, &.{ 0xf7 }, 3, .none, .none },
466466 .{ .neg, .m, &.{ .rm64 }, &.{ 0xf7 }, 3, .long, .none },
467467
468 .{ .nop, .zo, &.{}, &.{ 0x90 }, 0, .none, .none },
468 .{ .nop, .z, &.{}, &.{ 0x90 }, 0, .none, .none },
469469
470470 .{ .not, .m, &.{ .rm8 }, &.{ 0xf6 }, 2, .none, .none },
471471 .{ .not, .m, &.{ .rm8 }, &.{ 0xf6 }, 2, .rex, .none },
......@@ -496,7 +496,7 @@ pub const table = [_]Entry{
496496 .{ .@"or", .rm, &.{ .r32, .rm32 }, &.{ 0x0b }, 0, .none, .none },
497497 .{ .@"or", .rm, &.{ .r64, .rm64 }, &.{ 0x0b }, 0, .long, .none },
498498
499 .{ .pause, .zo, &.{}, &.{ 0xf3, 0x90 }, 0, .none, .none },
499 .{ .pause, .z, &.{}, &.{ 0xf3, 0x90 }, 0, .none, .none },
500500
501501 .{ .pop, .o, &.{ .r16 }, &.{ 0x58 }, 0, .short, .none },
502502 .{ .pop, .o, &.{ .r64 }, &.{ 0x58 }, 0, .none, .none },
......@@ -507,7 +507,7 @@ pub const table = [_]Entry{
507507 .{ .popcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none, .popcnt },
508508 .{ .popcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xb8 }, 0, .long, .popcnt },
509509
510 .{ .popfq, .zo, &.{}, &.{ 0x9d }, 0, .none, .none },
510 .{ .popfq, .z, &.{}, &.{ 0x9d }, 0, .none, .none },
511511
512512 .{ .push, .o, &.{ .r16 }, &.{ 0x50 }, 0, .short, .none },
513513 .{ .push, .o, &.{ .r64 }, &.{ 0x50 }, 0, .none, .none },
......@@ -517,9 +517,9 @@ pub const table = [_]Entry{
517517 .{ .push, .i, &.{ .imm16 }, &.{ 0x68 }, 0, .short, .none },
518518 .{ .push, .i, &.{ .imm32 }, &.{ 0x68 }, 0, .none, .none },
519519
520 .{ .pushfq, .zo, &.{}, &.{ 0x9c }, 0, .none, .none },
520 .{ .pushfq, .z, &.{}, &.{ 0x9c }, 0, .none, .none },
521521
522 .{ .ret, .zo, &.{}, &.{ 0xc3 }, 0, .none, .none },
522 .{ .ret, .z, &.{}, &.{ 0xc3 }, 0, .none, .none },
523523
524524 .{ .rcl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 2, .none, .none },
525525 .{ .rcl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 2, .rex, .none },
......@@ -640,15 +640,15 @@ pub const table = [_]Entry{
640640 .{ .sbb, .rm, &.{ .r32, .rm32 }, &.{ 0x1b }, 0, .none, .none },
641641 .{ .sbb, .rm, &.{ .r64, .rm64 }, &.{ 0x1b }, 0, .long, .none },
642642
643 .{ .scas, .zo, &.{ .m8 }, &.{ 0xae }, 0, .none, .none },
644 .{ .scas, .zo, &.{ .m16 }, &.{ 0xaf }, 0, .short, .none },
645 .{ .scas, .zo, &.{ .m32 }, &.{ 0xaf }, 0, .none, .none },
646 .{ .scas, .zo, &.{ .m64 }, &.{ 0xaf }, 0, .long, .none },
643 .{ .scas, .z, &.{ .m8 }, &.{ 0xae }, 0, .none, .none },
644 .{ .scas, .z, &.{ .m16 }, &.{ 0xaf }, 0, .short, .none },
645 .{ .scas, .z, &.{ .m32 }, &.{ 0xaf }, 0, .none, .none },
646 .{ .scas, .z, &.{ .m64 }, &.{ 0xaf }, 0, .long, .none },
647647
648 .{ .scasb, .zo, &.{}, &.{ 0xae }, 0, .none, .none },
649 .{ .scasw, .zo, &.{}, &.{ 0xaf }, 0, .short, .none },
650 .{ .scasd, .zo, &.{}, &.{ 0xaf }, 0, .none, .none },
651 .{ .scasq, .zo, &.{}, &.{ 0xaf }, 0, .long, .none },
648 .{ .scasb, .z, &.{}, &.{ 0xae }, 0, .none, .none },
649 .{ .scasw, .z, &.{}, &.{ 0xaf }, 0, .short, .none },
650 .{ .scasd, .z, &.{}, &.{ 0xaf }, 0, .none, .none },
651 .{ .scasq, .z, &.{}, &.{ 0xaf }, 0, .long, .none },
652652
653653 .{ .seta, .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .none, .none },
654654 .{ .seta, .m, &.{ .rm8 }, &.{ 0x0f, 0x97 }, 0, .rex, .none },
......@@ -711,7 +711,7 @@ pub const table = [_]Entry{
711711 .{ .setz, .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .none, .none },
712712 .{ .setz, .m, &.{ .rm8 }, &.{ 0x0f, 0x94 }, 0, .rex, .none },
713713
714 .{ .sfence, .zo, &.{}, &.{ 0x0f, 0xae, 0xf8 }, 0, .none, .none },
714 .{ .sfence, .z, &.{}, &.{ 0x0f, 0xae, 0xf8 }, 0, .none, .none },
715715
716716 .{ .shl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 4, .none, .none },
717717 .{ .shl, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 4, .rex, .none },
......@@ -759,25 +759,25 @@ pub const table = [_]Entry{
759759 .{ .shrd, .mrc, &.{ .rm32, .r32, .cl }, &.{ 0x0f, 0xad }, 0, .none, .none },
760760 .{ .shrd, .mrc, &.{ .rm64, .r64, .cl }, &.{ 0x0f, 0xad }, 0, .long, .none },
761761
762 .{ .stac, .zo, &.{}, &.{ 0x0f, 0x01, 0xcb }, 0, .none, .smap },
762 .{ .stac, .z, &.{}, &.{ 0x0f, 0x01, 0xcb }, 0, .none, .smap },
763763
764 .{ .stc, .zo, &.{}, &.{ 0xf9 }, 0, .none, .none },
764 .{ .stc, .z, &.{}, &.{ 0xf9 }, 0, .none, .none },
765765
766 .{ .std, .zo, &.{}, &.{ 0xfd }, 0, .none, .none },
766 .{ .std, .z, &.{}, &.{ 0xfd }, 0, .none, .none },
767767
768 .{ .sti, .zo, &.{}, &.{ 0xfb }, 0, .none, .none },
768 .{ .sti, .z, &.{}, &.{ 0xfb }, 0, .none, .none },
769769
770 .{ .stui, .zo, &.{}, &.{ 0xf3, 0x0f, 0x01, 0xef }, 0, .none, .uintr },
770 .{ .stui, .z, &.{}, &.{ 0xf3, 0x0f, 0x01, 0xef }, 0, .none, .uintr },
771771
772 .{ .stos, .zo, &.{ .m8 }, &.{ 0xaa }, 0, .none, .none },
773 .{ .stos, .zo, &.{ .m16 }, &.{ 0xab }, 0, .short, .none },
774 .{ .stos, .zo, &.{ .m32 }, &.{ 0xab }, 0, .none, .none },
775 .{ .stos, .zo, &.{ .m64 }, &.{ 0xab }, 0, .long, .none },
772 .{ .stos, .z, &.{ .m8 }, &.{ 0xaa }, 0, .none, .none },
773 .{ .stos, .z, &.{ .m16 }, &.{ 0xab }, 0, .short, .none },
774 .{ .stos, .z, &.{ .m32 }, &.{ 0xab }, 0, .none, .none },
775 .{ .stos, .z, &.{ .m64 }, &.{ 0xab }, 0, .long, .none },
776776
777 .{ .stosb, .zo, &.{}, &.{ 0xaa }, 0, .none, .none },
778 .{ .stosw, .zo, &.{}, &.{ 0xab }, 0, .short, .none },
779 .{ .stosd, .zo, &.{}, &.{ 0xab }, 0, .none, .none },
780 .{ .stosq, .zo, &.{}, &.{ 0xab }, 0, .long, .none },
777 .{ .stosb, .z, &.{}, &.{ 0xaa }, 0, .none, .none },
778 .{ .stosw, .z, &.{}, &.{ 0xab }, 0, .short, .none },
779 .{ .stosd, .z, &.{}, &.{ 0xab }, 0, .none, .none },
780 .{ .stosq, .z, &.{}, &.{ 0xab }, 0, .long, .none },
781781
782782 .{ .sub, .zi, &.{ .al, .imm8 }, &.{ 0x2c }, 0, .none, .none },
783783 .{ .sub, .zi, &.{ .ax, .imm16 }, &.{ 0x2d }, 0, .short, .none },
......@@ -802,7 +802,7 @@ pub const table = [_]Entry{
802802 .{ .sub, .rm, &.{ .r32, .rm32 }, &.{ 0x2b }, 0, .none, .none },
803803 .{ .sub, .rm, &.{ .r64, .rm64 }, &.{ 0x2b }, 0, .long, .none },
804804
805 .{ .syscall, .zo, &.{}, &.{ 0x0f, 0x05 }, 0, .none, .none },
805 .{ .syscall, .z, &.{}, &.{ 0x0f, 0x05 }, 0, .none, .none },
806806
807807 .{ .@"test", .zi, &.{ .al, .imm8 }, &.{ 0xa8 }, 0, .none, .none },
808808 .{ .@"test", .zi, &.{ .ax, .imm16 }, &.{ 0xa9 }, 0, .short, .none },
......@@ -823,7 +823,7 @@ pub const table = [_]Entry{
823823 .{ .tzcnt, .rm, &.{ .r32, .rm32 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .none, .bmi },
824824 .{ .tzcnt, .rm, &.{ .r64, .rm64 }, &.{ 0xf3, 0x0f, 0xbc }, 0, .long, .bmi },
825825
826 .{ .ud2, .zo, &.{}, &.{ 0x0f, 0x0b }, 0, .none, .none },
826 .{ .ud2, .z, &.{}, &.{ 0x0f, 0x0b }, 0, .none, .none },
827827
828828 .{ .xadd, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xc0 }, 0, .none, .none },
829829 .{ .xadd, .mr, &.{ .rm8, .r8 }, &.{ 0x0f, 0xc0 }, 0, .rex, .none },
......@@ -831,12 +831,12 @@ pub const table = [_]Entry{
831831 .{ .xadd, .mr, &.{ .rm32, .r32 }, &.{ 0x0f, 0xc1 }, 0, .none, .none },
832832 .{ .xadd, .mr, &.{ .rm64, .r64 }, &.{ 0x0f, 0xc1 }, 0, .long, .none },
833833
834 .{ .xchg, .o, &.{ .ax, .r16 }, &.{ 0x90 }, 0, .short, .none },
835 .{ .xchg, .o, &.{ .r16, .ax }, &.{ 0x90 }, 0, .short, .none },
836 .{ .xchg, .o, &.{ .eax, .r32 }, &.{ 0x90 }, 0, .none, .none },
837 .{ .xchg, .o, &.{ .rax, .r64 }, &.{ 0x90 }, 0, .long, .none },
838 .{ .xchg, .o, &.{ .r32, .eax }, &.{ 0x90 }, 0, .none, .none },
839 .{ .xchg, .o, &.{ .r64, .rax }, &.{ 0x90 }, 0, .long, .none },
834 .{ .xchg, .zo, &.{ .ax, .r16 }, &.{ 0x90 }, 0, .short, .none },
835 .{ .xchg, .oz, &.{ .r16, .ax }, &.{ 0x90 }, 0, .short, .none },
836 .{ .xchg, .zo, &.{ .eax, .r32 }, &.{ 0x90 }, 0, .none, .none },
837 .{ .xchg, .zo, &.{ .rax, .r64 }, &.{ 0x90 }, 0, .long, .none },
838 .{ .xchg, .oz, &.{ .r32, .eax }, &.{ 0x90 }, 0, .none, .none },
839 .{ .xchg, .oz, &.{ .r64, .rax }, &.{ 0x90 }, 0, .long, .none },
840840 .{ .xchg, .mr, &.{ .rm8, .r8 }, &.{ 0x86 }, 0, .none, .none },
841841 .{ .xchg, .mr, &.{ .rm8, .r8 }, &.{ 0x86 }, 0, .rex, .none },
842842 .{ .xchg, .rm, &.{ .r8, .rm8 }, &.{ 0x86 }, 0, .none, .none },
......@@ -848,7 +848,7 @@ pub const table = [_]Entry{
848848 .{ .xchg, .rm, &.{ .r32, .rm32 }, &.{ 0x87 }, 0, .none, .none },
849849 .{ .xchg, .rm, &.{ .r64, .rm64 }, &.{ 0x87 }, 0, .long, .none },
850850
851 .{ .xgetbv, .zo, &.{}, &.{ 0x0f, 0x01, 0xd0 }, 0, .none, .none },
851 .{ .xgetbv, .z, &.{}, &.{ 0x0f, 0x01, 0xd0 }, 0, .none, .none },
852852
853853 .{ .xor, .zi, &.{ .al, .imm8 }, &.{ 0x34 }, 0, .none, .none },
854854 .{ .xor, .zi, &.{ .ax, .imm16 }, &.{ 0x35 }, 0, .short, .none },
......@@ -874,9 +874,9 @@ pub const table = [_]Entry{
874874 .{ .xor, .rm, &.{ .r64, .rm64 }, &.{ 0x33 }, 0, .long, .none },
875875
876876 // X87
877 .{ .fabs, .zo, &.{}, &.{ 0xd9, 0xe1 }, 0, .none, .x87 },
877 .{ .fabs, .z, &.{}, &.{ 0xd9, 0xe1 }, 0, .none, .x87 },
878878
879 .{ .fchs, .zo, &.{}, &.{ 0xd9, 0xe0 }, 0, .none, .x87 },
879 .{ .fchs, .z, &.{}, &.{ 0xd9, 0xe0 }, 0, .none, .x87 },
880880
881881 .{ .ffree, .o, &.{ .st }, &.{ 0xdd, 0xc0 }, 0, .none, .x87 },
882882