Commit d064cf639f0f05f0f5dda84c228783c37db010b8 | zig

author	Jacob Young <jacobly@ziglang.org> 2023-03-25 08:58:00-04:00
committer	Jacob Young <jacobly@ziglang.org> 2023-03-25 16:23:55-04:00
log	d064cf639f0f05f0f5dda84c228783c37db010b8
tree	c52cdd96e3198623a07dec5709bb9deeb16a7b1e
parent	77300c02d65af81da6019995e4f1a156ef364df4

x86_64: implement 128-bit shifts

6 files changed, 533 insertions(+), 282 deletions(-)

src
- arch
  - x86_64
    CodeGen.zig
    Emit.zig
    Encoding.zig
    Mir.zig
    encoder.zig
    encodings.zig

src/arch/x86_64/CodeGen.zig+376-213

...	@@ -409,10 +409,7 @@ fn asmSetccRegister(self: *Self, reg: Register, cc: bits.Condition) !void {	...	@@ -409,10 +409,7 @@ fn asmSetccRegister(self: *Self, reg: Register, cc: bits.Condition) !void {
409	_ = try self.addInst(.{	409	_ = try self.addInst(.{
410	.tag = .setcc,	410	.tag = .setcc,
411	.ops = .r_cc,	411	.ops = .r_cc,
412	.data = .{ .r_cc = .{	412	.data = .{ .r_cc = .{ .r = reg, .cc = cc } },
413	.r1 = reg,
414	.cc = cc,
415	} },
416	});	413	});
417	}	414	}
418		415
...	@@ -424,14 +421,11 @@ fn asmSetccMemory(self: *Self, m: Memory, cc: bits.Condition) !void {	...	@@ -424,14 +421,11 @@ fn asmSetccMemory(self: *Self, m: Memory, cc: bits.Condition) !void {
424	.rip => .m_rip_cc,	421	.rip => .m_rip_cc,
425	else => unreachable,	422	else => unreachable,
426	},	423	},
427	.data = .{ .x_cc = .{	424	.data = .{ .x_cc = .{ .cc = cc, .payload = switch (m) {
428	.payload = switch (m) {	425	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
429	.sib => try self.addExtra(Mir.MemorySib.encode(m)),	426	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
430	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),	427	else => unreachable,
431	else => unreachable,	428	} } },
432	},
433	.cc = cc,
434	} },
435	});	429	});
436	}	430	}
437		431
...	@@ -439,11 +433,7 @@ fn asmCmovccRegisterRegister(self: *Self, reg1: Register, reg2: Register, cc: bi	...	@@ -439,11 +433,7 @@ fn asmCmovccRegisterRegister(self: *Self, reg1: Register, reg2: Register, cc: bi
439	_ = try self.addInst(.{	433	_ = try self.addInst(.{
440	.tag = .cmovcc,	434	.tag = .cmovcc,
441	.ops = .rr_cc,	435	.ops = .rr_cc,
442	.data = .{ .rr_cc = .{	436	.data = .{ .rr_cc = .{ .r1 = reg1, .r2 = reg2, .cc = cc } },
443	.r1 = reg1,
444	.r2 = reg2,
445	.cc = cc,
446	} },
447	});	437	});
448	}	438	}
449		439
...	@@ -455,15 +445,11 @@ fn asmCmovccRegisterMemory(self: *Self, reg: Register, m: Memory, cc: bits.Condi	...	@@ -455,15 +445,11 @@ fn asmCmovccRegisterMemory(self: *Self, reg: Register, m: Memory, cc: bits.Condi
455	.rip => .rm_rip_cc,	445	.rip => .rm_rip_cc,
456	else => unreachable,	446	else => unreachable,
457	},	447	},
458	.data = .{ .rx_cc = .{	448	.data = .{ .rx_cc = .{ .r = reg, .cc = cc, .payload = switch (m) {
459	.r1 = reg,	449	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
460	.cc = cc,	450	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
461	.payload = switch (m) {	451	else => unreachable,
462	.sib => try self.addExtra(Mir.MemorySib.encode(m)),	452	} } },
463	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
464	else => unreachable,
465	},
466	} },
467	});	453	});
468	}	454	}
469		455
...	@@ -479,10 +465,7 @@ fn asmJccReloc(self: *Self, target: Mir.Inst.Index, cc: bits.Condition) !Mir.Ins	...	@@ -479,10 +465,7 @@ fn asmJccReloc(self: *Self, target: Mir.Inst.Index, cc: bits.Condition) !Mir.Ins
479	return self.addInst(.{	465	return self.addInst(.{
480	.tag = .jcc,	466	.tag = .jcc,
481	.ops = .inst_cc,	467	.ops = .inst_cc,
482	.data = .{ .inst_cc = .{	468	.data = .{ .inst_cc = .{ .inst = target, .cc = cc } },
483	.inst = target,
484	.cc = cc,
485	} },
486	});	469	});
487	}	470	}
488		471
...	@@ -503,13 +486,15 @@ fn asmRegister(self: *Self, tag: Mir.Inst.Tag, reg: Register) !void {	...	@@ -503,13 +486,15 @@ fn asmRegister(self: *Self, tag: Mir.Inst.Tag, reg: Register) !void {
503	}	486	}
504		487
505	fn asmImmediate(self: *Self, tag: Mir.Inst.Tag, imm: Immediate) !void {	488	fn asmImmediate(self: *Self, tag: Mir.Inst.Tag, imm: Immediate) !void {
506	const ops: Mir.Inst.Ops = if (imm == .signed) .imm_s else .imm_u;
507	_ = try self.addInst(.{	489	_ = try self.addInst(.{
508	.tag = tag,	490	.tag = tag,
509	.ops = ops,	491	.ops = switch (imm) {
510	.data = .{ .imm = switch (imm) {	492	.signed => .i_s,
511	.signed => \|x\| @bitCast(u32, x),	493	.unsigned => .i_u,
512	.unsigned => \|x\| @intCast(u32, x),	494	},
		495	.data = .{ .i = switch (imm) {
		496	.signed => \|s\| @bitCast(u32, s),
		497	.unsigned => \|u\| @intCast(u32, u),
513	} },	498	} },
514	});	499	});
515	}	500	}
...	@@ -518,37 +503,43 @@ fn asmRegisterRegister(self: *Self, tag: Mir.Inst.Tag, reg1: Register, reg2: Reg	...	@@ -518,37 +503,43 @@ fn asmRegisterRegister(self: *Self, tag: Mir.Inst.Tag, reg1: Register, reg2: Reg
518	_ = try self.addInst(.{	503	_ = try self.addInst(.{
519	.tag = tag,	504	.tag = tag,
520	.ops = .rr,	505	.ops = .rr,
521	.data = .{ .rr = .{	506	.data = .{ .rr = .{ .r1 = reg1, .r2 = reg2 } },
522	.r1 = reg1,
523	.r2 = reg2,
524	} },
525	});	507	});
526	}	508	}
527		509
528	fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.Tag, reg: Register, imm: Immediate) !void {	510	fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.Tag, reg: Register, imm: Immediate) !void {
529	const ops: Mir.Inst.Ops = switch (imm) {	511	const ops: Mir.Inst.Ops = switch (imm) {
530	.signed => .ri_s,	512	.signed => .ri_s,
531	.unsigned => \|x\| if (x <= math.maxInt(u32)) .ri_u else .ri64,	513	.unsigned => \|u\| if (math.cast(u32, u)) \|_\| .ri_u else .ri64,
532	};
533	const data: Mir.Inst.Data = switch (ops) {
534	.ri_s => .{ .ri = .{
535	.r1 = reg,
536	.imm = @bitCast(u32, imm.signed),
537	} },
538	.ri_u => .{ .ri = .{
539	.r1 = reg,
540	.imm = @intCast(u32, imm.unsigned),
541	} },
542	.ri64 => .{ .rx = .{
543	.r1 = reg,
544	.payload = try self.addExtra(Mir.Imm64.encode(imm.unsigned)),
545	} },
546	else => unreachable,
547	};	514	};
548	_ = try self.addInst(.{	515	_ = try self.addInst(.{
549	.tag = tag,	516	.tag = tag,
550	.ops = ops,	517	.ops = ops,
551	.data = data,	518	.data = switch (ops) {
		519	.ri_s, .ri_u => .{ .ri = .{ .r = reg, .i = switch (imm) {
		520	.signed => \|s\| @bitCast(u32, s),
		521	.unsigned => \|u\| @intCast(u32, u),
		522	} } },
		523	.ri64 => .{ .rx = .{
		524	.r = reg,
		525	.payload = try self.addExtra(Mir.Imm64.encode(imm.unsigned)),
		526	} },
		527	else => unreachable,
		528	},
		529	});
		530	}
		531
		532	fn asmRegisterRegisterRegister(
		533	self: *Self,
		534	tag: Mir.Inst.Tag,
		535	reg1: Register,
		536	reg2: Register,
		537	reg3: Register,
		538	) !void {
		539	_ = try self.addInst(.{
		540	.tag = tag,
		541	.ops = .rrr,
		542	.data = .{ .rrr = .{ .r1 = reg1, .r2 = reg2, .r3 = reg3 } },
552	});	543	});
553	}	544	}
554		545
...	@@ -559,109 +550,142 @@ fn asmRegisterRegisterImmediate(	...	@@ -559,109 +550,142 @@ fn asmRegisterRegisterImmediate(
559	reg2: Register,	550	reg2: Register,
560	imm: Immediate,	551	imm: Immediate,
561	) !void {	552	) !void {
562	const ops: Mir.Inst.Ops = switch (imm) {
563	.signed => .rri_s,
564	.unsigned => .rri_u,
565	};
566	const data: Mir.Inst.Data = switch (ops) {
567	.rri_s => .{ .rri = .{
568	.r1 = reg1,
569	.r2 = reg2,
570	.imm = @bitCast(u32, imm.signed),
571	} },
572	.rri_u => .{ .rri = .{
573	.r1 = reg1,
574	.r2 = reg2,
575	.imm = @intCast(u32, imm.unsigned),
576	} },
577	else => unreachable,
578	};
579	_ = try self.addInst(.{	553	_ = try self.addInst(.{
580	.tag = tag,	554	.tag = tag,
581	.ops = ops,	555	.ops = switch (imm) {
582	.data = data,	556	.signed => .rri_s,
		557	.unsigned => .rri_u,
		558	},
		559	.data = .{ .rri = .{ .r1 = reg1, .r2 = reg2, .i = switch (imm) {
		560	.signed => \|s\| @bitCast(u32, s),
		561	.unsigned => \|u\| @intCast(u32, u),
		562	} } },
583	});	563	});
584	}	564	}
585		565
586	fn asmMemory(self: *Self, tag: Mir.Inst.Tag, m: Memory) !void {	566	fn asmMemory(self: *Self, tag: Mir.Inst.Tag, m: Memory) !void {
587	const ops: Mir.Inst.Ops = switch (m) {
588	.sib => .m_sib,
589	.rip => .m_rip,
590	else => unreachable,
591	};
592	const data: Mir.Inst.Data = .{ .payload = switch (ops) {
593	.m_sib => try self.addExtra(Mir.MemorySib.encode(m)),
594	.m_rip => try self.addExtra(Mir.MemoryRip.encode(m)),
595	else => unreachable,
596	} };
597	_ = try self.addInst(.{	567	_ = try self.addInst(.{
598	.tag = tag,	568	.tag = tag,
599	.ops = ops,	569	.ops = switch (m) {
600	.data = data,	570	.sib => .m_sib,
		571	.rip => .m_rip,
		572	else => unreachable,
		573	},
		574	.data = .{ .payload = switch (m) {
		575	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
		576	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
		577	else => unreachable,
		578	} },
601	});	579	});
602	}	580	}
603		581
604	fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.Tag, m: Memory, imm: Immediate) !void {	582	fn asmRegisterMemory(self: *Self, tag: Mir.Inst.Tag, reg: Register, m: Memory) !void {
605	const ops: Mir.Inst.Ops = switch (m) {
606	.sib => if (imm == .signed) .mi_s_sib else .mi_u_sib,
607	.rip => if (imm == .signed) .mi_s_rip else .mi_u_rip,
608	else => unreachable,
609	};
610	const payload: u32 = switch (ops) {
611	.mi_s_sib, .mi_u_sib => try self.addExtra(Mir.MemorySib.encode(m)),
612	.mi_s_rip, .mi_u_rip => try self.addExtra(Mir.MemoryRip.encode(m)),
613	else => unreachable,
614	};
615	const data: Mir.Inst.Data = .{
616	.xi = .{ .payload = payload, .imm = switch (imm) {
617	.signed => \|x\| @bitCast(u32, x),
618	.unsigned => \|x\| @intCast(u32, x),
619	} },
620	};
621	_ = try self.addInst(.{	583	_ = try self.addInst(.{
622	.tag = tag,	584	.tag = tag,
623	.ops = ops,	585	.ops = switch (m) {
624	.data = data,	586	.sib => .rm_sib,
		587	.rip => .rm_rip,
		588	else => unreachable,
		589	},
		590	.data = .{ .rx = .{ .r = reg, .payload = switch (m) {
		591	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
		592	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
		593	else => unreachable,
		594	} } },
625	});	595	});
626	}	596	}
627		597
628	fn asmRegisterMemory(self: *Self, tag: Mir.Inst.Tag, reg: Register, m: Memory) !void {	598	fn asmMemoryRegister(self: *Self, tag: Mir.Inst.Tag, m: Memory, reg: Register) !void {
629	const ops: Mir.Inst.Ops = switch (m) {	599	_ = try self.addInst(.{
630	.sib => .rm_sib,	600	.tag = tag,
631	.rip => .rm_rip,	601	.ops = switch (m) {
632	else => unreachable,	602	.sib => .mr_sib,
633	};	603	.rip => .mr_rip,
634	const data: Mir.Inst.Data = .{
635	.rx = .{ .r1 = reg, .payload = switch (ops) {
636	.rm_sib => try self.addExtra(Mir.MemorySib.encode(m)),
637	.rm_rip => try self.addExtra(Mir.MemoryRip.encode(m)),
638	else => unreachable,	604	else => unreachable,
639	} },	605	},
640	};	606	.data = .{ .rx = .{ .r = reg, .payload = switch (m) {
		607	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
		608	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
		609	else => unreachable,
		610	} } },
		611	});
		612	}
		613
		614	fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.Tag, m: Memory, imm: Immediate) !void {
641	_ = try self.addInst(.{	615	_ = try self.addInst(.{
642	.tag = tag,	616	.tag = tag,
643	.ops = ops,	617	.ops = switch (m) {
644	.data = data,	618	.sib => switch (imm) {
		619	.signed => .mi_sib_s,
		620	.unsigned => .mi_sib_u,
		621	},
		622	.rip => switch (imm) {
		623	.signed => .mi_rip_s,
		624	.unsigned => .mi_rip_u,
		625	},
		626	else => unreachable,
		627	},
		628	.data = .{ .ix = .{ .i = switch (imm) {
		629	.signed => \|s\| @bitCast(u32, s),
		630	.unsigned => \|u\| @intCast(u32, u),
		631	}, .payload = switch (m) {
		632	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
		633	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
		634	else => unreachable,
		635	} } },
645	});	636	});
646	}	637	}
647		638
648	fn asmMemoryRegister(self: *Self, tag: Mir.Inst.Tag, m: Memory, reg: Register) !void {	639	fn asmMemoryRegisterRegister(
649	const ops: Mir.Inst.Ops = switch (m) {	640	self: *Self,
650	.sib => .mr_sib,	641	tag: Mir.Inst.Tag,
651	.rip => .mr_rip,	642	m: Memory,
652	else => unreachable,	643	reg1: Register,
653	};	644	reg2: Register,
654	const data: Mir.Inst.Data = .{	645	) !void {
655	.rx = .{ .r1 = reg, .payload = switch (ops) {	646	_ = try self.addInst(.{
656	.mr_sib => try self.addExtra(Mir.MemorySib.encode(m)),	647	.tag = tag,
657	.mr_rip => try self.addExtra(Mir.MemoryRip.encode(m)),	648	.ops = switch (m) {
		649	.sib => .mrr_sib,
		650	.rip => .mrr_rip,
658	else => unreachable,	651	else => unreachable,
659	} },	652	},
660	};	653	.data = .{ .rrx = .{ .r1 = reg1, .r2 = reg2, .payload = switch (m) {
		654	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
		655	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
		656	else => unreachable,
		657	} } },
		658	});
		659	}
		660
		661	fn asmMemoryRegisterImmediate(
		662	self: *Self,
		663	tag: Mir.Inst.Tag,
		664	m: Memory,
		665	reg: Register,
		666	imm: Immediate,
		667	) !void {
661	_ = try self.addInst(.{	668	_ = try self.addInst(.{
662	.tag = tag,	669	.tag = tag,
663	.ops = ops,	670	.ops = switch (m) {
664	.data = data,	671	.sib => switch (imm) {
		672	.signed => .mri_sib_s,
		673	.unsigned => .mri_sib_u,
		674	},
		675	.rip => switch (imm) {
		676	.signed => .mri_rip_s,
		677	.unsigned => .mri_sib_u,
		678	},
		679	else => unreachable,
		680	},
		681	.data = .{ .rix = .{ .r = reg, .i = switch (imm) {
		682	.signed => \|s\| @bitCast(u32, s),
		683	.unsigned => \|u\| @intCast(u32, u),
		684	}, .payload = switch (m) {
		685	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
		686	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
		687	else => unreachable,
		688	} } },
665	});	689	});
666	}	690	}
667		691
...	@@ -768,18 +792,12 @@ fn gen(self: *Self) InnerError!void {	...	@@ -768,18 +792,12 @@ fn gen(self: *Self) InnerError!void {
768	self.mir_instructions.set(backpatch_stack_sub, .{	792	self.mir_instructions.set(backpatch_stack_sub, .{
769	.tag = .sub,	793	.tag = .sub,
770	.ops = .ri_u,	794	.ops = .ri_u,
771	.data = .{ .ri = .{	795	.data = .{ .ri = .{ .r = .rsp, .i = aligned_stack_end } },
772	.r1 = .rsp,
773	.imm = aligned_stack_end,
774	} },
775	});	796	});
776	self.mir_instructions.set(backpatch_stack_add, .{	797	self.mir_instructions.set(backpatch_stack_add, .{
777	.tag = .add,	798	.tag = .add,
778	.ops = .ri_u,	799	.ops = .ri_u,
779	.data = .{ .ri = .{	800	.data = .{ .ri = .{ .r = .rsp, .i = aligned_stack_end } },
780	.r1 = .rsp,
781	.imm = aligned_stack_end,
782	} },
783	});	801	});
784		802
785	const save_reg_list = try self.addExtra(Mir.SaveRegisterList{	803	const save_reg_list = try self.addExtra(Mir.SaveRegisterList{
...	@@ -1732,6 +1750,7 @@ fn airAddSubShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -1732,6 +1750,7 @@ fn airAddSubShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1732	.add_with_overflow => try self.genBinOp(null, .add, bin_op.lhs, bin_op.rhs),	1750	.add_with_overflow => try self.genBinOp(null, .add, bin_op.lhs, bin_op.rhs),
1733	.sub_with_overflow => try self.genBinOp(null, .sub, bin_op.lhs, bin_op.rhs),	1751	.sub_with_overflow => try self.genBinOp(null, .sub, bin_op.lhs, bin_op.rhs),
1734	.shl_with_overflow => blk: {	1752	.shl_with_overflow => blk: {
		1753	try self.register_manager.getReg(.rcx, null);
1735	const lhs = try self.resolveInst(bin_op.lhs);	1754	const lhs = try self.resolveInst(bin_op.lhs);
1736	const rhs = try self.resolveInst(bin_op.rhs);	1755	const rhs = try self.resolveInst(bin_op.rhs);
1737	const shift_ty = self.air.typeOf(bin_op.rhs);	1756	const shift_ty = self.air.typeOf(bin_op.rhs);
...	@@ -2022,6 +2041,7 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2022,6 +2041,7 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
2022	try self.spillRegisters(&.{.rcx});	2041	try self.spillRegisters(&.{.rcx});
2023		2042
2024	const tag = self.air.instructions.items(.tag)[inst];	2043	const tag = self.air.instructions.items(.tag)[inst];
		2044	try self.register_manager.getReg(.rcx, null);
2025	const lhs = try self.resolveInst(bin_op.lhs);	2045	const lhs = try self.resolveInst(bin_op.lhs);
2026	const rhs = try self.resolveInst(bin_op.rhs);	2046	const rhs = try self.resolveInst(bin_op.rhs);
2027	const lhs_ty = self.air.typeOf(bin_op.lhs);	2047	const lhs_ty = self.air.typeOf(bin_op.lhs);
...	@@ -2151,7 +2171,7 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2151,7 +2171,7 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
2151	const result = try self.copyToRegisterWithInstTracking(inst, err_union_ty, operand);	2171	const result = try self.copyToRegisterWithInstTracking(inst, err_union_ty, operand);
2152	if (err_off > 0) {	2172	if (err_off > 0) {
2153	const shift = @intCast(u6, err_off * 8);	2173	const shift = @intCast(u6, err_off * 8);
2154	try self.genShiftBinOpMir(.shr, err_union_ty, result.register, .{ .immediate = shift });	2174	try self.genShiftBinOpMir(.shr, err_union_ty, result, .{ .immediate = shift });
2155	} else {	2175	} else {
2156	try self.truncateRegister(Type.anyerror, result.register);	2176	try self.truncateRegister(Type.anyerror, result.register);
2157	}	2177	}
...	@@ -2183,9 +2203,7 @@ fn genUnwrapErrorUnionPayloadMir(	...	@@ -2183,9 +2203,7 @@ fn genUnwrapErrorUnionPayloadMir(
2183	const payload_ty = err_union_ty.errorUnionPayload();	2203	const payload_ty = err_union_ty.errorUnionPayload();
2184		2204
2185	const result: MCValue = result: {	2205	const result: MCValue = result: {
2186	if (!payload_ty.hasRuntimeBitsIgnoreComptime()) {	2206	if (!payload_ty.hasRuntimeBitsIgnoreComptime()) break :result .none;
2187	break :result MCValue.none;
2188	}
2189		2207
2190	const payload_off = errUnionPayloadOffset(payload_ty, self.target.*);	2208	const payload_off = errUnionPayloadOffset(payload_ty, self.target.*);
2191	switch (err_union) {	2209	switch (err_union) {
...	@@ -2198,17 +2216,17 @@ fn genUnwrapErrorUnionPayloadMir(	...	@@ -2198,17 +2216,17 @@ fn genUnwrapErrorUnionPayloadMir(
2198	const eu_lock = self.register_manager.lockReg(reg);	2216	const eu_lock = self.register_manager.lockReg(reg);
2199	defer if (eu_lock) \|lock\| self.register_manager.unlockReg(lock);	2217	defer if (eu_lock) \|lock\| self.register_manager.unlockReg(lock);
2200		2218
2201	const result_reg: Register = if (maybe_inst) \|inst\|	2219	const result_mcv: MCValue = if (maybe_inst) \|inst\|
2202	(try self.copyToRegisterWithInstTracking(inst, err_union_ty, err_union)).register	2220	try self.copyToRegisterWithInstTracking(inst, err_union_ty, err_union)
2203	else	2221	else
2204	try self.copyToTmpRegister(err_union_ty, err_union);	2222	.{ .register = try self.copyToTmpRegister(err_union_ty, err_union) };
2205	if (payload_off > 0) {	2223	if (payload_off > 0) {
2206	const shift = @intCast(u6, payload_off * 8);	2224	const shift = @intCast(u6, payload_off * 8);
2207	try self.genShiftBinOpMir(.shr, err_union_ty, result_reg, .{ .immediate = shift });	2225	try self.genShiftBinOpMir(.shr, err_union_ty, result_mcv, .{ .immediate = shift });
2208	} else {	2226	} else {
2209	try self.truncateRegister(payload_ty, result_reg);	2227	try self.truncateRegister(payload_ty, result_mcv.register);
2210	}	2228	}
2211	break :result MCValue{ .register = result_reg };	2229	break :result result_mcv;
2212	},	2230	},
2213	else => return self.fail("TODO implement genUnwrapErrorUnionPayloadMir for {}", .{err_union}),	2231	else => return self.fail("TODO implement genUnwrapErrorUnionPayloadMir for {}", .{err_union}),
2214	}	2232	}
...	@@ -2848,7 +2866,7 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2848,7 +2866,7 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
2848	else	2866	else
2849	0;	2867	0;
2850	const result = try self.copyToRegisterWithInstTracking(inst, union_ty, operand);	2868	const result = try self.copyToRegisterWithInstTracking(inst, union_ty, operand);
2851	try self.genShiftBinOpMir(.shr, Type.usize, result.register, .{ .immediate = shift });	2869	try self.genShiftBinOpMir(.shr, Type.usize, result, .{ .immediate = shift });
2852	break :blk MCValue{	2870	break :blk MCValue{
2853	.register = registerAlias(result.register, @intCast(u32, layout.tag_size)),	2871	.register = registerAlias(result.register, @intCast(u32, layout.tag_size)),
2854	};	2872	};
...	@@ -3769,12 +3787,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3769,12 +3787,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
3769	defer if (dst_mcv_lock) \|lock\| self.register_manager.unlockReg(lock);	3787	defer if (dst_mcv_lock) \|lock\| self.register_manager.unlockReg(lock);
3770		3788
3771	// Shift by struct_field_offset.	3789	// Shift by struct_field_offset.
3772	try self.genShiftBinOpMir(	3790	try self.genShiftBinOpMir(.shr, Type.usize, dst_mcv, .{ .immediate = field_bit_offset });
3773	.shr,
3774	Type.usize,
3775	dst_mcv.register,
3776	.{ .immediate = field_bit_offset },
3777	);
3778		3791
3779	// Mask to field_bit_size bits	3792	// Mask to field_bit_size bits
3780	const field_bit_size = field_ty.bitSize(self.target.*);	3793	const field_bit_size = field_ty.bitSize(self.target.*);
...	@@ -3932,29 +3945,186 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue	...	@@ -3932,29 +3945,186 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue
3932	}	3945	}
3933		3946
3934	/// Clobbers .rcx for non-immediate shift value.	3947	/// Clobbers .rcx for non-immediate shift value.
3935	fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shift: MCValue) !void {	3948	fn genShiftBinOpMir(
3936	switch (tag) {	3949	self: *Self,
3937	.sal, .sar, .shl, .shr => {},	3950	tag: Mir.Inst.Tag,
3938	else => unreachable,	3951	ty: Type,
3939	}	3952	lhs_mcv: MCValue,
3940		3953	shift_mcv: MCValue,
3941	const abi_size = @intCast(u32, ty.abiSize(self.target.*));	3954	) !void {
3942	blk: {	3955	const rhs_mcv: MCValue = rhs: {
3943	switch (shift) {	3956	switch (shift_mcv) {
3944	.immediate => \|imm\| switch (imm) {	3957	.immediate => \|imm\| switch (imm) {
3945	0 => return,	3958	0 => return,
3946	else => return self.asmRegisterImmediate(tag, registerAlias(reg, abi_size), Immediate.u(imm)),	3959	else => break :rhs shift_mcv,
3947	},
3948	.register => \|shift_reg\| {
3949	if (shift_reg == .rcx) break :blk;
3950	},	3960	},
		3961	.register => \|shift_reg\| if (shift_reg == .rcx) break :rhs shift_mcv,
3951	else => {},	3962	else => {},
3952	}	3963	}
3953	self.register_manager.getRegAssumeFree(.rcx, null);	3964	self.register_manager.getRegAssumeFree(.rcx, null);
3954	try self.genSetReg(Type.u8, .rcx, shift);	3965	try self.genSetReg(Type.u8, .rcx, shift_mcv);
3955	}	3966	break :rhs .{ .register = .rcx };
		3967	};
3956		3968
3957	try self.asmRegisterRegister(tag, registerAlias(reg, abi_size), .cl);	3969	const abi_size = @intCast(u32, ty.abiSize(self.target.*));
		3970	if (abi_size <= 8) {
		3971	switch (lhs_mcv) {
		3972	.register => \|lhs_reg\| switch (rhs_mcv) {
		3973	.immediate => \|rhs_imm\| try self.asmRegisterImmediate(
		3974	tag,
		3975	registerAlias(lhs_reg, abi_size),
		3976	Immediate.u(rhs_imm),
		3977	),
		3978	.register => \|rhs_reg\| try self.asmRegisterRegister(
		3979	tag,
		3980	registerAlias(lhs_reg, abi_size),
		3981	registerAlias(rhs_reg, 1),
		3982	),
		3983	else => return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
		3984	@tagName(lhs_mcv),
		3985	@tagName(rhs_mcv),
		3986	}),
		3987	},
		3988	.stack_offset => \|lhs_off\| switch (rhs_mcv) {
		3989	.immediate => \|rhs_imm\| try self.asmMemoryImmediate(
		3990	tag,
		3991	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -lhs_off }),
		3992	Immediate.u(rhs_imm),
		3993	),
		3994	.register => \|rhs_reg\| try self.asmMemoryRegister(
		3995	tag,
		3996	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -lhs_off }),
		3997	registerAlias(rhs_reg, 1),
		3998	),
		3999	else => return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
		4000	@tagName(lhs_mcv),
		4001	@tagName(rhs_mcv),
		4002	}),
		4003	},
		4004	else => return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
		4005	@tagName(lhs_mcv),
		4006	@tagName(rhs_mcv),
		4007	}),
		4008	}
		4009	} else if (abi_size <= 16) {
		4010	const tmp_reg = try self.register_manager.allocReg(null, gp);
		4011	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
		4012	defer self.register_manager.unlockReg(tmp_lock);
		4013
		4014	const info: struct { offsets: [2]i32, double_tag: Mir.Inst.Tag } = switch (tag) {
		4015	.shl, .sal => .{ .offsets = .{ 0, 8 }, .double_tag = .shld },
		4016	.shr, .sar => .{ .offsets = .{ 8, 0 }, .double_tag = .shrd },
		4017	else => unreachable,
		4018	};
		4019	switch (lhs_mcv) {
		4020	.stack_offset => \|dst_off\| switch (rhs_mcv) {
		4021	.immediate => \|rhs_imm\| if (rhs_imm == 0) {} else if (rhs_imm < 64) {
		4022	try self.asmRegisterMemory(
		4023	.mov,
		4024	tmp_reg,
		4025	Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),
		4026	);
		4027	try self.asmMemoryRegisterImmediate(
		4028	info.double_tag,
		4029	Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[1] - dst_off }),
		4030	tmp_reg,
		4031	Immediate.u(rhs_imm),
		4032	);
		4033	try self.asmMemoryImmediate(
		4034	tag,
		4035	Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),
		4036	Immediate.u(rhs_imm),
		4037	);
		4038	} else {
		4039	assert(rhs_imm < 128);
		4040	try self.asmRegisterMemory(
		4041	.mov,
		4042	tmp_reg,
		4043	Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),
		4044	);
		4045	if (rhs_imm > 64) {
		4046	try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(rhs_imm - 64));
		4047	}
		4048	try self.asmMemoryRegister(
		4049	.mov,
		4050	Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[1] - dst_off }),
		4051	tmp_reg,
		4052	);
		4053	switch (tag) {
		4054	.shl, .sal, .shr => {
		4055	try self.asmRegisterRegister(.xor, tmp_reg.to32(), tmp_reg.to32());
		4056	try self.asmMemoryRegister(
		4057	.mov,
		4058	Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),
		4059	tmp_reg,
		4060	);
		4061	},
		4062	.sar => try self.asmMemoryImmediate(
		4063	tag,
		4064	Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),
		4065	Immediate.u(63),
		4066	),
		4067	else => unreachable,
		4068	}
		4069	},
		4070	else => {
		4071	const first_reg = try self.register_manager.allocReg(null, gp);
		4072	const first_lock = self.register_manager.lockRegAssumeUnused(first_reg);
		4073	defer self.register_manager.unlockReg(first_lock);
		4074
		4075	const second_reg = try self.register_manager.allocReg(null, gp);
		4076	const second_lock = self.register_manager.lockRegAssumeUnused(second_reg);
		4077	defer self.register_manager.unlockReg(second_lock);
		4078
		4079	try self.genSetReg(Type.u8, .cl, rhs_mcv);
		4080	try self.asmRegisterMemory(
		4081	.mov,
		4082	first_reg,
		4083	Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),
		4084	);
		4085	try self.asmRegisterMemory(
		4086	.mov,
		4087	second_reg,
		4088	Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[1] - dst_off }),
		4089	);
		4090	switch (tag) {
		4091	.shl, .sal, .shr => try self.asmRegisterRegister(
		4092	.xor,
		4093	tmp_reg.to32(),
		4094	tmp_reg.to32(),
		4095	),
		4096	.sar => {
		4097	try self.asmRegisterRegister(.mov, tmp_reg, first_reg);
		4098	try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(63));
		4099	},
		4100	else => unreachable,
		4101	}
		4102	try self.asmRegisterRegisterRegister(info.double_tag, second_reg, first_reg, .cl);
		4103	try self.asmRegisterRegister(tag, first_reg, .cl);
		4104	try self.asmRegisterImmediate(.cmp, .cl, Immediate.u(64));
		4105	try self.asmCmovccRegisterRegister(second_reg, first_reg, .ae);
		4106	try self.asmCmovccRegisterRegister(first_reg, tmp_reg, .ae);
		4107	try self.asmMemoryRegister(
		4108	.mov,
		4109	Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[1] - dst_off }),
		4110	second_reg,
		4111	);
		4112	try self.asmMemoryRegister(
		4113	.mov,
		4114	Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),
		4115	first_reg,
		4116	);
		4117	},
		4118	},
		4119	else => return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
		4120	@tagName(lhs_mcv),
		4121	@tagName(rhs_mcv),
		4122	}),
		4123	}
		4124	} else return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
		4125	@tagName(lhs_mcv),
		4126	@tagName(rhs_mcv),
		4127	});
3958	}	4128	}
3959		4129
3960	/// Result is always a register.	4130	/// Result is always a register.
...	@@ -3964,68 +4134,61 @@ fn genShiftBinOp(	...	@@ -3964,68 +4134,61 @@ fn genShiftBinOp(
3964	self: *Self,	4134	self: *Self,
3965	tag: Air.Inst.Tag,	4135	tag: Air.Inst.Tag,
3966	maybe_inst: ?Air.Inst.Index,	4136	maybe_inst: ?Air.Inst.Index,
3967	lhs: MCValue,	4137	lhs_mcv: MCValue,
3968	rhs: MCValue,	4138	rhs_mcv: MCValue,
3969	lhs_ty: Type,	4139	lhs_ty: Type,
3970	rhs_ty: Type,	4140	rhs_ty: Type,
3971	) !MCValue {	4141	) !MCValue {
3972	if (lhs_ty.zigTypeTag() == .Vector or lhs_ty.zigTypeTag() == .Float) {	4142	if (lhs_ty.zigTypeTag() == .Vector) {
3973	return self.fail("TODO implement genShiftBinOp for {}", .{lhs_ty.fmtDebug()});	4143	return self.fail("TODO implement genShiftBinOp for {}", .{lhs_ty.fmtDebug()});
3974	}	4144	}
3975	if (lhs_ty.abiSize(self.target.*) > 8) {	4145
		4146	assert(rhs_ty.abiSize(self.target.*) == 1);
		4147
		4148	const lhs_abi_size = lhs_ty.abiSize(self.target.*);
		4149	if (lhs_abi_size > 16) {
3976	return self.fail("TODO implement genShiftBinOp for {}", .{lhs_ty.fmtDebug()});	4150	return self.fail("TODO implement genShiftBinOp for {}", .{lhs_ty.fmtDebug()});
3977	}	4151	}
3978		4152
3979	assert(rhs_ty.abiSize(self.target.*) == 1);	4153	self.register_manager.getRegAssumeFree(.rcx, null);
		4154	const rcx_lock = self.register_manager.lockRegAssumeUnused(.rcx);
		4155	defer self.register_manager.unlockReg(rcx_lock);
3980		4156
3981	const lhs_lock: ?RegisterLock = switch (lhs) {	4157	const lhs_lock = switch (lhs_mcv) {
3982	.register => \|reg\| self.register_manager.lockReg(reg),	4158	.register => \|reg\| self.register_manager.lockReg(reg),
3983	else => null,	4159	else => null,
3984	};	4160	};
3985	defer if (lhs_lock) \|lock\| self.register_manager.unlockReg(lock);	4161	defer if (lhs_lock) \|lock\| self.register_manager.unlockReg(lock);
3986		4162
3987	const rhs_lock: ?RegisterLock = switch (rhs) {	4163	const rhs_lock = switch (rhs_mcv) {
3988	.register => \|reg\| self.register_manager.lockReg(reg),	4164	.register => \|reg\| self.register_manager.lockReg(reg),
3989	else => null,	4165	else => null,
3990	};	4166	};
3991	defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);	4167	defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
3992		4168
3993	self.register_manager.getRegAssumeFree(.rcx, null);	4169	const dst_mcv: MCValue = dst: {
3994	const rcx_lock = self.register_manager.lockRegAssumeUnused(.rcx);
3995	defer self.register_manager.unlockReg(rcx_lock);
3996
3997	const dst: MCValue = blk: {
3998	if (maybe_inst) \|inst\| {	4170	if (maybe_inst) \|inst\| {
3999	const bin_op = self.air.instructions.items(.data)[inst].bin_op;	4171	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
4000	// TODO dst can also be a memory location	4172	if (self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv)) break :dst lhs_mcv;
4001	if (self.reuseOperand(inst, bin_op.lhs, 0, lhs) and lhs.isRegister()) {
4002	break :blk lhs;
4003	}
4004	break :blk try self.copyToRegisterWithInstTracking(inst, lhs_ty, lhs);
4005	}	4173	}
4006	break :blk MCValue{ .register = try self.copyToTmpRegister(lhs_ty, lhs) };	4174	const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, maybe_inst, true);
		4175	try self.setRegOrMem(lhs_ty, dst_mcv, lhs_mcv);
		4176	break :dst dst_mcv;
4007	};	4177	};
4008		4178
4009	const signedness = lhs_ty.intInfo(self.target.*).signedness;	4179	const signedness = lhs_ty.intInfo(self.target.*).signedness;
4010	switch (tag) {	4180	try self.genShiftBinOpMir(switch (tag) {
4011	.shl => try self.genShiftBinOpMir(switch (signedness) {	4181	.shl, .shl_exact => switch (signedness) {
4012	.signed => .sal,	4182	.signed => .sal,
4013	.unsigned => .shl,	4183	.unsigned => .shl,
4014	}, lhs_ty, dst.register, rhs),	4184	},
4015		4185	.shr, .shr_exact => switch (signedness) {
4016	.shl_exact => try self.genShiftBinOpMir(.shl, lhs_ty, dst.register, rhs),
4017
4018	.shr,
4019	.shr_exact,
4020	=> try self.genShiftBinOpMir(switch (signedness) {
4021	.signed => .sar,	4186	.signed => .sar,
4022	.unsigned => .shr,	4187	.unsigned => .shr,
4023	}, lhs_ty, dst.register, rhs),	4188	},
4024
4025	else => unreachable,	4189	else => unreachable,
4026	}	4190	}, lhs_ty, dst_mcv, rhs_mcv);
4027		4191	return dst_mcv;
4028	return dst;
4029	}	4192	}
4030		4193
4031	/// Result is always a register.	4194	/// Result is always a register.
...	@@ -5552,7 +5715,7 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) !	...	@@ -5552,7 +5715,7 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) !
5552	const tmp_reg = try self.copyToTmpRegister(ty, operand);	5715	const tmp_reg = try self.copyToTmpRegister(ty, operand);
5553	if (err_off > 0) {	5716	if (err_off > 0) {
5554	const shift = @intCast(u6, err_off * 8);	5717	const shift = @intCast(u6, err_off * 8);
5555	try self.genShiftBinOpMir(.shr, ty, tmp_reg, .{ .immediate = shift });	5718	try self.genShiftBinOpMir(.shr, ty, .{ .register = tmp_reg }, .{ .immediate = shift });
5556	} else {	5719	} else {
5557	try self.truncateRegister(Type.anyerror, tmp_reg);	5720	try self.truncateRegister(Type.anyerror, tmp_reg);
5558	}	5721	}
...	@@ -6506,7 +6669,7 @@ fn genInlineMemcpyRegisterRegister(	...	@@ -6506,7 +6669,7 @@ fn genInlineMemcpyRegisterRegister(
6506	}), registerAlias(tmp_reg, nearest_power_of_two));	6669	}), registerAlias(tmp_reg, nearest_power_of_two));
6507		6670
6508	if (nearest_power_of_two > 1) {	6671	if (nearest_power_of_two > 1) {
6509	try self.genShiftBinOpMir(.shr, ty, tmp_reg, .{	6672	try self.genShiftBinOpMir(.shr, ty, .{ .register = tmp_reg }, .{
6510	.immediate = nearest_power_of_two * 8,	6673	.immediate = nearest_power_of_two * 8,
6511	});	6674	});
6512	}	6675	}
...	@@ -7032,7 +7195,7 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -7032,7 +7195,7 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
7032		7195
7033	try self.spillEflagsIfOccupied();	7196	try self.spillEflagsIfOccupied();
7034	_ = try self.addInst(.{ .tag = .cmpxchg, .ops = .lock_mr_sib, .data = .{ .rx = .{	7197	_ = try self.addInst(.{ .tag = .cmpxchg, .ops = .lock_mr_sib, .data = .{ .rx = .{
7035	.r1 = new_reg,	7198	.r = new_reg,
7036	.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),	7199	.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
7037	} } });	7200	} } });
7038		7201
...	@@ -7110,7 +7273,7 @@ fn atomicOp(	...	@@ -7110,7 +7273,7 @@ fn atomicOp(
7110	.xadd, .add, .sub, .@"and", .@"or", .xor => .lock_mr_sib,	7273	.xadd, .add, .sub, .@"and", .@"or", .xor => .lock_mr_sib,
7111	else => unreachable,	7274	else => unreachable,
7112	}, .data = .{ .rx = .{	7275	}, .data = .{ .rx = .{
7113	.r1 = registerAlias(dst_reg, val_abi_size),	7276	.r = registerAlias(dst_reg, val_abi_size),
7114	.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),	7277	.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
7115	} } });	7278	} } });
7116	return;	7279	return;
...	@@ -7702,8 +7865,8 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {	...	@@ -7702,8 +7865,8 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
7702	switch (int_info.signedness) {	7865	switch (int_info.signedness) {
7703	.signed => {	7866	.signed => {
7704	const shift = @intCast(u6, max_reg_bit_width - int_info.bits);	7867	const shift = @intCast(u6, max_reg_bit_width - int_info.bits);
7705	try self.genShiftBinOpMir(.sal, Type.isize, reg, .{ .immediate = shift });	7868	try self.genShiftBinOpMir(.sal, Type.isize, .{ .register = reg }, .{ .immediate = shift });
7706	try self.genShiftBinOpMir(.sar, Type.isize, reg, .{ .immediate = shift });	7869	try self.genShiftBinOpMir(.sar, Type.isize, .{ .register = reg }, .{ .immediate = shift });
7707	},	7870	},
7708	.unsigned => {	7871	.unsigned => {
7709	const shift = @intCast(u6, max_reg_bit_width - int_info.bits);	7872	const shift = @intCast(u6, max_reg_bit_width - int_info.bits);

src/arch/x86_64/Emit.zig+68-27

 @@ -121,7 +121,9 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
             .sbb,
             .sfence,
             .shl,
+            .shld,
             .shr,
+            .shrd,
             .sub,
             .syscall,
             .@"test",
 @@ -231,10 +233,10 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
     const prefix: Instruction.Prefix = switch (ops) {
         .lock_m_sib,
         .lock_m_rip,
-        .lock_mi_u_sib,
+        .lock_mi_sib_u,
-        .lock_mi_u_rip,
+        .lock_mi_rip_u,
-        .lock_mi_s_sib,
+        .lock_mi_sib_s,
-        .lock_mi_s_rip,
+        .lock_mi_rip_s,
         .lock_mr_sib,
         .lock_mr_rip,
         .lock_moffs_rax,
 @@ -249,31 +251,36 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
     switch (ops) {
         .none => {},
-        .imm_s => op1 = .{ .imm = Immediate.s(@bitCast(i32, data.imm)) },
+        .i_s => op1 = .{ .imm = Immediate.s(@bitCast(i32, data.i)) },
-        .imm_u => op1 = .{ .imm = Immediate.u(data.imm) },
+        .i_u => op1 = .{ .imm = Immediate.u(data.i) },
         .r => op1 = .{ .reg = data.r },
         .rr => {
             op1 = .{ .reg = data.rr.r1 };
             op2 = .{ .reg = data.rr.r2 };
         },
+        .rrr => {
+            op1 = .{ .reg = data.rrr.r1 };
+            op2 = .{ .reg = data.rrr.r2 };
+            op3 = .{ .reg = data.rrr.r3 };
+        },
         .ri_s, .ri_u => {
             const imm = switch (ops) {
-                .ri_s => Immediate.s(@bitCast(i32, data.ri.imm)),
+                .ri_s => Immediate.s(@bitCast(i32, data.ri.i)),
-                .ri_u => Immediate.u(data.ri.imm),
+                .ri_u => Immediate.u(data.ri.i),
                 else => unreachable,
             };
-            op1 = .{ .reg = data.ri.r1 };
+            op1 = .{ .reg = data.ri.r };
             op2 = .{ .imm = imm };
         },
         .ri64 => {
             const imm64 = emit.mir.extraData(Mir.Imm64, data.rx.payload).data;
-            op1 = .{ .reg = data.rx.r1 };
+            op1 = .{ .reg = data.rx.r };
             op2 = .{ .imm = Immediate.u(Mir.Imm64.decode(imm64)) };
         },
         .rri_s, .rri_u => {
             const imm = switch (ops) {
-                .rri_s => Immediate.s(@bitCast(i32, data.rri.imm)),
+                .rri_s => Immediate.s(@bitCast(i32, data.rri.i)),
-                .rri_u => Immediate.u(data.rri.imm),
+                .rri_u => Immediate.u(data.rri.i),
                 else => unreachable,
             };
             op1 = .{ .reg = data.rri.r1 };
 @@ -288,21 +295,21 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
             const mrip = emit.mir.extraData(Mir.MemoryRip, data.payload).data;
             op1 = .{ .mem = Mir.MemoryRip.decode(mrip) };
         },
-        .mi_s_sib, .mi_u_sib, .lock_mi_s_sib, .lock_mi_u_sib => {
+        .mi_sib_s, .mi_sib_u, .lock_mi_sib_s, .lock_mi_sib_u => {
-            const msib = emit.mir.extraData(Mir.MemorySib, data.xi.payload).data;
+            const msib = emit.mir.extraData(Mir.MemorySib, data.ix.payload).data;
             const imm = switch (ops) {
-                .mi_s_sib, .lock_mi_s_sib => Immediate.s(@bitCast(i32, data.xi.imm)),
+                .mi_sib_s, .lock_mi_sib_s => Immediate.s(@bitCast(i32, data.ix.i)),
-                .mi_u_sib, .lock_mi_u_sib => Immediate.u(data.xi.imm),
+                .mi_sib_u, .lock_mi_sib_u => Immediate.u(data.ix.i),
                 else => unreachable,
             };
             op1 = .{ .mem = Mir.MemorySib.decode(msib) };
             op2 = .{ .imm = imm };
         },
-        .mi_u_rip, .mi_s_rip, .lock_mi_u_rip, .lock_mi_s_rip => {
+        .mi_rip_u, .mi_rip_s, .lock_mi_rip_u, .lock_mi_rip_s => {
-            const mrip = emit.mir.extraData(Mir.MemoryRip, data.xi.payload).data;
+            const mrip = emit.mir.extraData(Mir.MemoryRip, data.ix.payload).data;
             const imm = switch (ops) {
-                .mi_s_rip, .lock_mi_s_rip => Immediate.s(@bitCast(i32, data.xi.imm)),
+                .mi_rip_s, .lock_mi_rip_s => Immediate.s(@bitCast(i32, data.ix.i)),
-                .mi_u_rip, .lock_mi_u_rip => Immediate.u(data.xi.imm),
+                .mi_rip_u, .lock_mi_rip_u => Immediate.u(data.ix.i),
                 else => unreachable,
             };
             op1 = .{ .mem = Mir.MemoryRip.decode(mrip) };
 @@ -310,7 +317,7 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
         },
         .rm_sib, .mr_sib, .lock_mr_sib => {
             const msib = emit.mir.extraData(Mir.MemorySib, data.rx.payload).data;
-            const op_r = .{ .reg = data.rx.r1 };
+            const op_r = .{ .reg = data.rx.r };
             const op_m = .{ .mem = Mir.MemorySib.decode(msib) };
             switch (ops) {
                 .rm_sib => {
 @@ -326,7 +333,7 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
         },
         .rm_rip, .mr_rip, .lock_mr_rip => {
             const mrip = emit.mir.extraData(Mir.MemoryRip, data.rx.payload).data;
-            const op_r = .{ .reg = data.rx.r1 };
+            const op_r = .{ .reg = data.rx.r };
             const op_m = .{ .mem = Mir.MemoryRip.decode(mrip) };
             switch (ops) {
                 .rm_rip => {
 @@ -340,6 +347,40 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
                 else => unreachable,
             }
         },
+        .mrr_sib => {
+            const msib = emit.mir.extraData(Mir.MemorySib, data.rrx.payload).data;
+            op1 = .{ .mem = Mir.MemorySib.decode(msib) };
+            op2 = .{ .reg = data.rrx.r1 };
+            op2 = .{ .reg = data.rrx.r2 };
+        },
+        .mrr_rip => {
+            const mrip = emit.mir.extraData(Mir.MemoryRip, data.rrx.payload).data;
+            op1 = .{ .mem = Mir.MemoryRip.decode(mrip) };
+            op2 = .{ .reg = data.rrx.r1 };
+            op2 = .{ .reg = data.rrx.r2 };
+        },
+        .mri_sib_u, .mri_sib_s => {
+            const msib = emit.mir.extraData(Mir.MemorySib, data.rix.payload).data;
+            const imm = switch (ops) {
+                .mri_sib_s => Immediate.s(@bitCast(i32, data.rix.i)),
+                .mri_sib_u, .lock_mi_rip_u => Immediate.u(data.rix.i),
+                else => unreachable,
+            };
+            op1 = .{ .mem = Mir.MemorySib.decode(msib) };
+            op2 = .{ .reg = data.rix.r };
+            op3 = .{ .imm = imm };
+        },
+        .mri_rip_u, .mri_rip_s => {
+            const mrip = emit.mir.extraData(Mir.MemoryRip, data.rix.payload).data;
+            const imm = switch (ops) {
+                .mri_rip_s => Immediate.s(@bitCast(i32, data.rix.i)),
+                .mri_rip_u, .lock_mi_rip_u => Immediate.u(data.rix.i),
+                else => unreachable,
+            };
+            op1 = .{ .mem = Mir.MemoryRip.decode(mrip) };
+            op2 = .{ .reg = data.rix.r };
+            op3 = .{ .imm = imm };
+        },
         else => return emit.fail("TODO handle generic encoding: {s}, {s}", .{
             @tagName(mnemonic),
             @tagName(ops),
 @@ -451,12 +492,12 @@ fn mirMovsx(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
         },
         .rm_sib => {
             const msib = emit.mir.extraData(Mir.MemorySib, data.rx.payload).data;
-            op1 = .{ .reg = data.rx.r1 };
+            op1 = .{ .reg = data.rx.r };
             op2 = .{ .mem = Mir.MemorySib.decode(msib) };
         },
         .rm_rip => {
             const mrip = emit.mir.extraData(Mir.MemoryRip, data.rx.payload).data;
-            op1 = .{ .reg = data.rx.r1 };
+            op1 = .{ .reg = data.rx.r };
             op2 = .{ .mem = Mir.MemoryRip.decode(mrip) };
         },
         else => unreachable, // TODO
 @@ -495,7 +536,7 @@ fn mirCmovcc(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
             const extra = emit.mir.extraData(Mir.MemorySib, data.payload).data;
             const mnemonic = mnemonicFromConditionCode("cmov", data.cc);
             return emit.encode(mnemonic, .{
-                .op1 = .{ .reg = data.r1 },
+                .op1 = .{ .reg = data.r },
                 .op2 = .{ .mem = Mir.MemorySib.decode(extra) },
             });
         },
 @@ -504,7 +545,7 @@ fn mirCmovcc(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
             const extra = emit.mir.extraData(Mir.MemoryRip, data.payload).data;
             const mnemonic = mnemonicFromConditionCode("cmov", data.cc);
             return emit.encode(mnemonic, .{
-                .op1 = .{ .reg = data.r1 },
+                .op1 = .{ .reg = data.r },
                 .op2 = .{ .mem = Mir.MemoryRip.decode(extra) },
             });
         },
 @@ -519,7 +560,7 @@ fn mirSetcc(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
             const data = emit.mir.instructions.items(.data)[inst].r_cc;
             const mnemonic = mnemonicFromConditionCode("set", data.cc);
             return emit.encode(mnemonic, .{
-                .op1 = .{ .reg = data.r1 },
+                .op1 = .{ .reg = data.r },
             });
         },
         .m_sib_cc => {

src/arch/x86_64/Encoding.zig+7-6

 @@ -262,15 +262,15 @@ pub fn format(
             try writer.print("+{s} ", .{tag});
         },
         .m, .mi, .m1, .mc => try writer.print("/{d} ", .{encoding.modRmExt()}),
-        .mr, .rm, .rmi => try writer.writeAll("/r "),
+        .mr, .rm, .rmi, .mri, .mrc => try writer.writeAll("/r "),
     }
     switch (encoding.op_en) {
-        .i, .d, .zi, .oi, .mi, .rmi => {
+        .i, .d, .zi, .oi, .mi, .rmi, .mri => {
             const op = switch (encoding.op_en) {
                 .i, .d => encoding.op1,
                 .zi, .oi, .mi => encoding.op2,
-                .rmi => encoding.op3,
+                .rmi, .mri => encoding.op3,
                 else => unreachable,
             };
             const tag = switch (op) {
 @@ -285,7 +285,7 @@ pub fn format(
             };
             try writer.print("{s} ", .{tag});
         },
-        .np, .fd, .td, .o, .m, .m1, .mc, .mr, .rm => {},
+        .np, .fd, .td, .o, .m, .m1, .mc, .mr, .rm, .mrc => {},
     }
     try writer.print("{s} ", .{@tagName(encoding.mnemonic)});
 @@ -334,7 +334,7 @@ pub const Mnemonic = enum {
     rcl, rcr, ret, rol, ror,
     sal, sar, sbb,
     scas, scasb, scasd, scasq, scasw,
-    shl, shr, sub, syscall,
+    shl, shld, shr, shrd, sub, syscall,
     seta, setae, setb, setbe, setc, sete, setg, setge, setl, setle, setna, setnae,
     setnb, setnbe, setnc, setne, setng, setnge, setnl, setnle, setno, setnp, setns,
     setnz, seto, setp, setpe, setpo, sets, setz,
 @@ -374,7 +374,8 @@ pub const OpEn = enum {
     i, zi,
     d, m,
     fd, td,
-    m1, mc, mi, mr, rm, rmi,
+    m1, mc, mi, mr, rm,
+    rmi, mri, mrc,
     // zig fmt: on
 };

src/arch/x86_64/Mir.zig+59-25

 @@ -138,8 +138,12 @@ pub const Inst = struct {
         sfence,
         /// Logical shift left
         shl,
+        /// Double precision shift left
+        shld,
         /// Logical shift right
         shr,
+        /// Double precision shift right
+        shrd,
         /// Subtract
         sub,
         /// Syscall
 @@ -284,10 +288,10 @@ pub const Inst = struct {
         ri64,
         /// Immediate (sign-extended) operand.
         /// Uses `imm` payload.
-        imm_s,
+        i_s,
         /// Immediate (unsigned) operand.
         /// Uses `imm` payload.
-        imm_u,
+        i_u,
         /// Relative displacement operand.
         /// Uses `imm` payload.
         rel,
 @@ -316,23 +320,41 @@ pub const Inst = struct {
         /// Uses `x_cc` with extra data of type `MemoryRip`.
         m_rip_cc,
         /// Memory (SIB), immediate (unsigned) operands.
-        /// Uses `xi` payload with extra data of type `MemorySib`.
+        /// Uses `ix` payload with extra data of type `MemorySib`.
-        mi_u_sib,
+        mi_sib_u,
         /// Memory (RIP), immediate (unsigned) operands.
-        /// Uses `xi` payload with extra data of type `MemoryRip`.
+        /// Uses `ix` payload with extra data of type `MemoryRip`.
-        mi_u_rip,
+        mi_rip_u,
         /// Memory (SIB), immediate (sign-extend) operands.
-        /// Uses `xi` payload with extra data of type `MemorySib`.
+        /// Uses `ix` payload with extra data of type `MemorySib`.
-        mi_s_sib,
+        mi_sib_s,
         /// Memory (RIP), immediate (sign-extend) operands.
-        /// Uses `xi` payload with extra data of type `MemoryRip`.
+        /// Uses `ix` payload with extra data of type `MemoryRip`.
-        mi_s_rip,
+        mi_rip_s,
         /// Memory (SIB), register operands.
         /// Uses `rx` payload with extra data of type `MemorySib`.
         mr_sib,
         /// Memory (RIP), register operands.
         /// Uses `rx` payload with extra data of type `MemoryRip`.
         mr_rip,
+        /// Memory (SIB), register, register operands.
+        /// Uses `rrx` payload with extra data of type `MemorySib`.
+        mrr_sib,
+        /// Memory (RIP), register, register operands.
+        /// Uses `rrx` payload with extra data of type `MemoryRip`.
+        mrr_rip,
+        /// Memory (SIB), register, immediate (unsigned) operands.
+        /// Uses `rix` payload with extra data of type `MemorySib`.
+        mri_sib_u,
+        /// Memory (RIP), register, immediate (unsigned) operands.
+        /// Uses `rix` payload with extra data of type `MemoryRip`.
+        mri_rip_u,
+        /// Memory (SIB), register, immediate (signed) operands.
+        /// Uses `rix` payload with extra data of type `MemorySib`.
+        mri_sib_s,
+        /// Memory (RIP), register, immediate (signed) operands.
+        /// Uses `rix` payload with extra data of type `MemoryRip`.
+        mri_rip_s,
         /// Rax, Memory moffs.
         /// Uses `payload` with extra data of type `MemoryMoffs`.
         rax_moffs,
 @@ -347,16 +369,16 @@ pub const Inst = struct {
         lock_m_rip,
         /// Memory (SIB), immediate (unsigned) operands with lock prefix.
         /// Uses `xi` payload with extra data of type `MemorySib`.
-        lock_mi_u_sib,
+        lock_mi_sib_u,
         /// Memory (RIP), immediate (unsigned) operands with lock prefix.
         /// Uses `xi` payload with extra data of type `MemoryRip`.
-        lock_mi_u_rip,
+        lock_mi_rip_u,
         /// Memory (SIB), immediate (sign-extend) operands with lock prefix.
         /// Uses `xi` payload with extra data of type `MemorySib`.
-        lock_mi_s_sib,
+        lock_mi_sib_s,
         /// Memory (RIP), immediate (sign-extend) operands with lock prefix.
         /// Uses `xi` payload with extra data of type `MemoryRip`.
-        lock_mi_s_rip,
+        lock_mi_rip_s,
         /// Memory (SIB), register operands with lock prefix.
         /// Uses `rx` payload with extra data of type `MemorySib`.
         lock_mr_sib,
 @@ -400,7 +422,7 @@ pub const Inst = struct {
             cc: bits.Condition,
         },
         /// A 32-bit immediate value.
-        imm: u32,
+        i: u32,
         r: Register,
         rr: struct {
             r1: Register,
 @@ -414,16 +436,16 @@ pub const Inst = struct {
         rri: struct {
             r1: Register,
             r2: Register,
-            imm: u32,
+            i: u32,
         },
         /// Condition code (CC), followed by custom payload found in extra.
         x_cc: struct {
-            payload: u32,
             cc: bits.Condition,
+            payload: u32,
         },
         /// Register with condition code (CC).
         r_cc: struct {
-            r1: Register,
+            r: Register,
             cc: bits.Condition,
         },
         /// Register, register with condition code (CC).
 @@ -434,24 +456,36 @@ pub const Inst = struct {
         },
         /// Register, immediate.
         ri: struct {
-            r1: Register,
+            r: Register,
-            imm: u32,
+            i: u32,
         },
         /// Register, followed by custom payload found in extra.
         rx: struct {
-            r1: Register,
+            r: Register,
             payload: u32,
         },
         /// Register with condition code (CC), followed by custom payload found in extra.
         rx_cc: struct {
-            r1: Register,
+            r: Register,
             cc: bits.Condition,
             payload: u32,
         },
-        /// Custom payload followed by an immediate.
+        /// Immediate, followed by Custom payload found in extra.
-        xi: struct {
+        ix: struct {
+            i: u32,
+            payload: u32,
+        },
+        /// Register, register, followed by Custom payload found in extra.
+        rrx: struct {
+            r1: Register,
+            r2: Register,
+            payload: u32,
+        },
+        /// Register, immediate, followed by Custom payload found in extra.
+        rix: struct {
+            r: Register,
+            i: u32,
             payload: u32,
-            imm: u32,
         },
         /// String instruction prefix and width.
         string: struct {

src/arch/x86_64/encoder.zig+9-11

...	@@ -174,7 +174,7 @@ pub const Instruction = struct {	...	@@ -174,7 +174,7 @@ pub const Instruction = struct {
174	.td => try encoder.imm64(inst.op1.mem.moffs.offset),	174	.td => try encoder.imm64(inst.op1.mem.moffs.offset),
175	else => {	175	else => {
176	const mem_op = switch (encoding.op_en) {	176	const mem_op = switch (encoding.op_en) {
177	.m, .mi, .m1, .mc, .mr => inst.op1,	177	.m, .mi, .m1, .mc, .mr, .mri, .mrc => inst.op1,
178	.rm, .rmi => inst.op2,	178	.rm, .rmi => inst.op2,
179	else => unreachable,	179	else => unreachable,
180	};	180	};
...	@@ -182,7 +182,7 @@ pub const Instruction = struct {	...	@@ -182,7 +182,7 @@ pub const Instruction = struct {
182	.reg => \|reg\| {	182	.reg => \|reg\| {
183	const rm = switch (encoding.op_en) {	183	const rm = switch (encoding.op_en) {
184	.m, .mi, .m1, .mc => encoding.modRmExt(),	184	.m, .mi, .m1, .mc => encoding.modRmExt(),
185	.mr => inst.op2.reg.lowEnc(),	185	.mr, .mri, .mrc => inst.op2.reg.lowEnc(),
186	.rm, .rmi => inst.op1.reg.lowEnc(),	186	.rm, .rmi => inst.op1.reg.lowEnc(),
187	else => unreachable,	187	else => unreachable,
188	};	188	};
...	@@ -191,7 +191,7 @@ pub const Instruction = struct {	...	@@ -191,7 +191,7 @@ pub const Instruction = struct {
191	.mem => \|mem\| {	191	.mem => \|mem\| {
192	const op = switch (encoding.op_en) {	192	const op = switch (encoding.op_en) {
193	.m, .mi, .m1, .mc => .none,	193	.m, .mi, .m1, .mc => .none,
194	.mr => inst.op2,	194	.mr, .mri, .mrc => inst.op2,
195	.rm, .rmi => inst.op1,	195	.rm, .rmi => inst.op1,
196	else => unreachable,	196	else => unreachable,
197	};	197	};
...	@@ -202,7 +202,7 @@ pub const Instruction = struct {	...	@@ -202,7 +202,7 @@ pub const Instruction = struct {
202		202
203	switch (encoding.op_en) {	203	switch (encoding.op_en) {
204	.mi => try encodeImm(inst.op2.imm, encoding.op2, encoder),	204	.mi => try encodeImm(inst.op2.imm, encoding.op2, encoder),
205	.rmi => try encodeImm(inst.op3.imm, encoding.op3, encoder),	205	.rmi, .mri => try encodeImm(inst.op3.imm, encoding.op3, encoder),
206	else => {},	206	else => {},
207	}	207	}
208	},	208	},
...	@@ -251,7 +251,7 @@ pub const Instruction = struct {	...	@@ -251,7 +251,7 @@ pub const Instruction = struct {
251	else => unreachable,	251	else => unreachable,
252	};	252	};
253	} else null,	253	} else null,
254	.m, .mi, .m1, .mc, .mr => if (inst.op1.isSegmentRegister()) blk: {	254	.m, .mi, .m1, .mc, .mr, .mri, .mrc => if (inst.op1.isSegmentRegister()) blk: {
255	break :blk switch (inst.op1) {	255	break :blk switch (inst.op1) {
256	.reg => \|r\| r,	256	.reg => \|r\| r,
257	.mem => \|m\| m.base().?,	257	.mem => \|m\| m.base().?,
...	@@ -275,13 +275,11 @@ pub const Instruction = struct {	...	@@ -275,13 +275,11 @@ pub const Instruction = struct {
275		275
276	switch (op_en) {	276	switch (op_en) {
277	.np, .i, .zi, .fd, .td, .d => {},	277	.np, .i, .zi, .fd, .td, .d => {},
278	.o, .oi => {	278	.o, .oi => rex.b = inst.op1.reg.isExtended(),
279	rex.b = inst.op1.reg.isExtended();	279	.m, .mi, .m1, .mc, .mr, .rm, .rmi, .mri, .mrc => {
280	},
281	.m, .mi, .m1, .mc, .mr, .rm, .rmi => {
282	const r_op = switch (op_en) {	280	const r_op = switch (op_en) {
283	.rm, .rmi => inst.op1,	281	.rm, .rmi => inst.op1,
284	.mr => inst.op2,	282	.mr, .mri, .mrc => inst.op2,
285	else => null,	283	else => null,
286	};	284	};
287	if (r_op) \|op\| {	285	if (r_op) \|op\| {
...	@@ -290,7 +288,7 @@ pub const Instruction = struct {	...	@@ -290,7 +288,7 @@ pub const Instruction = struct {
290		288
291	const b_x_op = switch (op_en) {	289	const b_x_op = switch (op_en) {
292	.rm, .rmi => inst.op2,	290	.rm, .rmi => inst.op2,
293	.m, .mi, .m1, .mc, .mr => inst.op1,	291	.m, .mi, .m1, .mc, .mr, .mri, .mrc => inst.op1,
294	else => unreachable,	292	else => unreachable,
295	};	293	};
296	switch (b_x_op) {	294	switch (b_x_op) {

src/arch/x86_64/encodings.zig+14

...	@@ -693,6 +693,13 @@ pub const table = &[_]Entry{	...	@@ -693,6 +693,13 @@ pub const table = &[_]Entry{
693	.{ .shl, .mi, .rm32, .imm8, .none, .none, &.{ 0xc1 }, 4, .none },	693	.{ .shl, .mi, .rm32, .imm8, .none, .none, &.{ 0xc1 }, 4, .none },
694	.{ .shl, .mi, .rm64, .imm8, .none, .none, &.{ 0xc1 }, 4, .long },	694	.{ .shl, .mi, .rm64, .imm8, .none, .none, &.{ 0xc1 }, 4, .long },
695		695
		696	.{ .shld, .mri, .rm16, .r16, .imm8, .none, &.{ 0x0f, 0xa4 }, 0, .none },
		697	.{ .shld, .mrc, .rm16, .r16, .cl, .none, &.{ 0x0f, 0xa5 }, 0, .none },
		698	.{ .shld, .mri, .rm32, .r32, .imm8, .none, &.{ 0x0f, 0xa4 }, 0, .none },
		699	.{ .shld, .mri, .rm64, .r64, .imm8, .none, &.{ 0x0f, 0xa4 }, 0, .long },
		700	.{ .shld, .mrc, .rm32, .r32, .cl, .none, &.{ 0x0f, 0xa5 }, 0, .none },
		701	.{ .shld, .mrc, .rm64, .r64, .cl, .none, &.{ 0x0f, 0xa5 }, 0, .long },
		702
696	.{ .shr, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 5, .none },	703	.{ .shr, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 5, .none },
697	.{ .shr, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 5, .rex },	704	.{ .shr, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 5, .rex },
698	.{ .shr, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 5, .none },	705	.{ .shr, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 5, .none },
...	@@ -709,6 +716,13 @@ pub const table = &[_]Entry{	...	@@ -709,6 +716,13 @@ pub const table = &[_]Entry{
709	.{ .shr, .mi, .rm32, .imm8, .none, .none, &.{ 0xc1 }, 5, .none },	716	.{ .shr, .mi, .rm32, .imm8, .none, .none, &.{ 0xc1 }, 5, .none },
710	.{ .shr, .mi, .rm64, .imm8, .none, .none, &.{ 0xc1 }, 5, .long },	717	.{ .shr, .mi, .rm64, .imm8, .none, .none, &.{ 0xc1 }, 5, .long },
711		718
		719	.{ .shrd, .mri, .rm16, .r16, .imm8, .none, &.{ 0x0f, 0xac }, 0, .none },
		720	.{ .shrd, .mrc, .rm16, .r16, .cl, .none, &.{ 0x0f, 0xad }, 0, .none },
		721	.{ .shrd, .mri, .rm32, .r32, .imm8, .none, &.{ 0x0f, 0xac }, 0, .none },
		722	.{ .shrd, .mri, .rm64, .r64, .imm8, .none, &.{ 0x0f, 0xac }, 0, .long },
		723	.{ .shrd, .mrc, .rm32, .r32, .cl, .none, &.{ 0x0f, 0xad }, 0, .none },
		724	.{ .shrd, .mrc, .rm64, .r64, .cl, .none, &.{ 0x0f, 0xad }, 0, .long },
		725
712	.{ .stos, .np, .m8, .none, .none, .none, &.{ 0xaa }, 0, .none },	726	.{ .stos, .np, .m8, .none, .none, .none, &.{ 0xaa }, 0, .none },
713	.{ .stos, .np, .m16, .none, .none, .none, &.{ 0xab }, 0, .none },	727	.{ .stos, .np, .m16, .none, .none, .none, &.{ 0xab }, 0, .none },
714	.{ .stos, .np, .m32, .none, .none, .none, &.{ 0xab }, 0, .none },	728	.{ .stos, .np, .m32, .none, .none, .none, &.{ 0xab }, 0, .none },