Commit d064cf639f0f05f0f5dda84c228783c37db010b8 | zig

author	Jacob Young <jacobly@ziglang.org> 2023-03-25 08:58:00-04:00
committer	Jacob Young <jacobly@ziglang.org> 2023-03-25 16:23:55-04:00
log	d064cf639f0f05f0f5dda84c228783c37db010b8
tree	c52cdd96e3198623a07dec5709bb9deeb16a7b1e
parent	77300c02d65af81da6019995e4f1a156ef364df4

x86_64: implement 128-bit shifts

6 files changed, 533 insertions(+), 282 deletions(-)

src
- arch
  - x86_64
    CodeGen.zig
    Emit.zig
    Encoding.zig
    Mir.zig
    encoder.zig
    encodings.zig

src/arch/x86_64/CodeGen.zig+376-213

...	...	@@ -409,10 +409,7 @@ fn asmSetccRegister(self: *Self, reg: Register, cc: bits.Condition) !void {
409	409	_ = try self.addInst(.{
410	410	.tag = .setcc,
411	411	.ops = .r_cc,
412		.data = .{ .r_cc = .{
413		.r1 = reg,
414		.cc = cc,
415		} },
	412	.data = .{ .r_cc = .{ .r = reg, .cc = cc } },
416	413	});
417	414	}
418	415
...	...	@@ -424,14 +421,11 @@ fn asmSetccMemory(self: *Self, m: Memory, cc: bits.Condition) !void {
424	421	.rip => .m_rip_cc,
425	422	else => unreachable,
426	423	},
427		.data = .{ .x_cc = .{
428		.payload = switch (m) {
429		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
430		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
431		else => unreachable,
432		},
433		.cc = cc,
434		} },
	424	.data = .{ .x_cc = .{ .cc = cc, .payload = switch (m) {
	425	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	426	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	427	else => unreachable,
	428	} } },
435	429	});
436	430	}
437	431
...	...	@@ -439,11 +433,7 @@ fn asmCmovccRegisterRegister(self: *Self, reg1: Register, reg2: Register, cc: bi
439	433	_ = try self.addInst(.{
440	434	.tag = .cmovcc,
441	435	.ops = .rr_cc,
442		.data = .{ .rr_cc = .{
443		.r1 = reg1,
444		.r2 = reg2,
445		.cc = cc,
446		} },
	436	.data = .{ .rr_cc = .{ .r1 = reg1, .r2 = reg2, .cc = cc } },
447	437	});
448	438	}
449	439
...	...	@@ -455,15 +445,11 @@ fn asmCmovccRegisterMemory(self: *Self, reg: Register, m: Memory, cc: bits.Condi
455	445	.rip => .rm_rip_cc,
456	446	else => unreachable,
457	447	},
458		.data = .{ .rx_cc = .{
459		.r1 = reg,
460		.cc = cc,
461		.payload = switch (m) {
462		.sib => try self.addExtra(Mir.MemorySib.encode(m)),
463		.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
464		else => unreachable,
465		},
466		} },
	448	.data = .{ .rx_cc = .{ .r = reg, .cc = cc, .payload = switch (m) {
	449	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	450	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	451	else => unreachable,
	452	} } },
467	453	});
468	454	}
469	455
...	...	@@ -479,10 +465,7 @@ fn asmJccReloc(self: *Self, target: Mir.Inst.Index, cc: bits.Condition) !Mir.Ins
479	465	return self.addInst(.{
480	466	.tag = .jcc,
481	467	.ops = .inst_cc,
482		.data = .{ .inst_cc = .{
483		.inst = target,
484		.cc = cc,
485		} },
	468	.data = .{ .inst_cc = .{ .inst = target, .cc = cc } },
486	469	});
487	470	}
488	471
...	...	@@ -503,13 +486,15 @@ fn asmRegister(self: *Self, tag: Mir.Inst.Tag, reg: Register) !void {
503	486	}
504	487
505	488	fn asmImmediate(self: *Self, tag: Mir.Inst.Tag, imm: Immediate) !void {
506		const ops: Mir.Inst.Ops = if (imm == .signed) .imm_s else .imm_u;
507	489	_ = try self.addInst(.{
508	490	.tag = tag,
509		.ops = ops,
510		.data = .{ .imm = switch (imm) {
511		.signed => \|x\| @bitCast(u32, x),
512		.unsigned => \|x\| @intCast(u32, x),
	491	.ops = switch (imm) {
	492	.signed => .i_s,
	493	.unsigned => .i_u,
	494	},
	495	.data = .{ .i = switch (imm) {
	496	.signed => \|s\| @bitCast(u32, s),
	497	.unsigned => \|u\| @intCast(u32, u),
513	498	} },
514	499	});
515	500	}
...	...	@@ -518,37 +503,43 @@ fn asmRegisterRegister(self: *Self, tag: Mir.Inst.Tag, reg1: Register, reg2: Reg
518	503	_ = try self.addInst(.{
519	504	.tag = tag,
520	505	.ops = .rr,
521		.data = .{ .rr = .{
522		.r1 = reg1,
523		.r2 = reg2,
524		} },
	506	.data = .{ .rr = .{ .r1 = reg1, .r2 = reg2 } },
525	507	});
526	508	}
527	509
528	510	fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.Tag, reg: Register, imm: Immediate) !void {
529	511	const ops: Mir.Inst.Ops = switch (imm) {
530	512	.signed => .ri_s,
531		.unsigned => \|x\| if (x <= math.maxInt(u32)) .ri_u else .ri64,
532		};
533		const data: Mir.Inst.Data = switch (ops) {
534		.ri_s => .{ .ri = .{
535		.r1 = reg,
536		.imm = @bitCast(u32, imm.signed),
537		} },
538		.ri_u => .{ .ri = .{
539		.r1 = reg,
540		.imm = @intCast(u32, imm.unsigned),
541		} },
542		.ri64 => .{ .rx = .{
543		.r1 = reg,
544		.payload = try self.addExtra(Mir.Imm64.encode(imm.unsigned)),
545		} },
546		else => unreachable,
	513	.unsigned => \|u\| if (math.cast(u32, u)) \|_\| .ri_u else .ri64,
547	514	};
548	515	_ = try self.addInst(.{
549	516	.tag = tag,
550	517	.ops = ops,
551		.data = data,
	518	.data = switch (ops) {
	519	.ri_s, .ri_u => .{ .ri = .{ .r = reg, .i = switch (imm) {
	520	.signed => \|s\| @bitCast(u32, s),
	521	.unsigned => \|u\| @intCast(u32, u),
	522	} } },
	523	.ri64 => .{ .rx = .{
	524	.r = reg,
	525	.payload = try self.addExtra(Mir.Imm64.encode(imm.unsigned)),
	526	} },
	527	else => unreachable,
	528	},
	529	});
	530	}
	531
	532	fn asmRegisterRegisterRegister(
	533	self: *Self,
	534	tag: Mir.Inst.Tag,
	535	reg1: Register,
	536	reg2: Register,
	537	reg3: Register,
	538	) !void {
	539	_ = try self.addInst(.{
	540	.tag = tag,
	541	.ops = .rrr,
	542	.data = .{ .rrr = .{ .r1 = reg1, .r2 = reg2, .r3 = reg3 } },
552	543	});
553	544	}
554	545
...	...	@@ -559,109 +550,142 @@ fn asmRegisterRegisterImmediate(
559	550	reg2: Register,
560	551	imm: Immediate,
561	552	) !void {
562		const ops: Mir.Inst.Ops = switch (imm) {
563		.signed => .rri_s,
564		.unsigned => .rri_u,
565		};
566		const data: Mir.Inst.Data = switch (ops) {
567		.rri_s => .{ .rri = .{
568		.r1 = reg1,
569		.r2 = reg2,
570		.imm = @bitCast(u32, imm.signed),
571		} },
572		.rri_u => .{ .rri = .{
573		.r1 = reg1,
574		.r2 = reg2,
575		.imm = @intCast(u32, imm.unsigned),
576		} },
577		else => unreachable,
578		};
579	553	_ = try self.addInst(.{
580	554	.tag = tag,
581		.ops = ops,
582		.data = data,
	555	.ops = switch (imm) {
	556	.signed => .rri_s,
	557	.unsigned => .rri_u,
	558	},
	559	.data = .{ .rri = .{ .r1 = reg1, .r2 = reg2, .i = switch (imm) {
	560	.signed => \|s\| @bitCast(u32, s),
	561	.unsigned => \|u\| @intCast(u32, u),
	562	} } },
583	563	});
584	564	}
585	565
586	566	fn asmMemory(self: *Self, tag: Mir.Inst.Tag, m: Memory) !void {
587		const ops: Mir.Inst.Ops = switch (m) {
588		.sib => .m_sib,
589		.rip => .m_rip,
590		else => unreachable,
591		};
592		const data: Mir.Inst.Data = .{ .payload = switch (ops) {
593		.m_sib => try self.addExtra(Mir.MemorySib.encode(m)),
594		.m_rip => try self.addExtra(Mir.MemoryRip.encode(m)),
595		else => unreachable,
596		} };
597	567	_ = try self.addInst(.{
598	568	.tag = tag,
599		.ops = ops,
600		.data = data,
	569	.ops = switch (m) {
	570	.sib => .m_sib,
	571	.rip => .m_rip,
	572	else => unreachable,
	573	},
	574	.data = .{ .payload = switch (m) {
	575	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	576	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	577	else => unreachable,
	578	} },
601	579	});
602	580	}
603	581
604		fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.Tag, m: Memory, imm: Immediate) !void {
605		const ops: Mir.Inst.Ops = switch (m) {
606		.sib => if (imm == .signed) .mi_s_sib else .mi_u_sib,
607		.rip => if (imm == .signed) .mi_s_rip else .mi_u_rip,
608		else => unreachable,
609		};
610		const payload: u32 = switch (ops) {
611		.mi_s_sib, .mi_u_sib => try self.addExtra(Mir.MemorySib.encode(m)),
612		.mi_s_rip, .mi_u_rip => try self.addExtra(Mir.MemoryRip.encode(m)),
613		else => unreachable,
614		};
615		const data: Mir.Inst.Data = .{
616		.xi = .{ .payload = payload, .imm = switch (imm) {
617		.signed => \|x\| @bitCast(u32, x),
618		.unsigned => \|x\| @intCast(u32, x),
619		} },
620		};
	582	fn asmRegisterMemory(self: *Self, tag: Mir.Inst.Tag, reg: Register, m: Memory) !void {
621	583	_ = try self.addInst(.{
622	584	.tag = tag,
623		.ops = ops,
624		.data = data,
	585	.ops = switch (m) {
	586	.sib => .rm_sib,
	587	.rip => .rm_rip,
	588	else => unreachable,
	589	},
	590	.data = .{ .rx = .{ .r = reg, .payload = switch (m) {
	591	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	592	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	593	else => unreachable,
	594	} } },
625	595	});
626	596	}
627	597
628		fn asmRegisterMemory(self: *Self, tag: Mir.Inst.Tag, reg: Register, m: Memory) !void {
629		const ops: Mir.Inst.Ops = switch (m) {
630		.sib => .rm_sib,
631		.rip => .rm_rip,
632		else => unreachable,
633		};
634		const data: Mir.Inst.Data = .{
635		.rx = .{ .r1 = reg, .payload = switch (ops) {
636		.rm_sib => try self.addExtra(Mir.MemorySib.encode(m)),
637		.rm_rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	598	fn asmMemoryRegister(self: *Self, tag: Mir.Inst.Tag, m: Memory, reg: Register) !void {
	599	_ = try self.addInst(.{
	600	.tag = tag,
	601	.ops = switch (m) {
	602	.sib => .mr_sib,
	603	.rip => .mr_rip,
638	604	else => unreachable,
639		} },
640		};
	605	},
	606	.data = .{ .rx = .{ .r = reg, .payload = switch (m) {
	607	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	608	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	609	else => unreachable,
	610	} } },
	611	});
	612	}
	613
	614	fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.Tag, m: Memory, imm: Immediate) !void {
641	615	_ = try self.addInst(.{
642	616	.tag = tag,
643		.ops = ops,
644		.data = data,
	617	.ops = switch (m) {
	618	.sib => switch (imm) {
	619	.signed => .mi_sib_s,
	620	.unsigned => .mi_sib_u,
	621	},
	622	.rip => switch (imm) {
	623	.signed => .mi_rip_s,
	624	.unsigned => .mi_rip_u,
	625	},
	626	else => unreachable,
	627	},
	628	.data = .{ .ix = .{ .i = switch (imm) {
	629	.signed => \|s\| @bitCast(u32, s),
	630	.unsigned => \|u\| @intCast(u32, u),
	631	}, .payload = switch (m) {
	632	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	633	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	634	else => unreachable,
	635	} } },
645	636	});
646	637	}
647	638
648		fn asmMemoryRegister(self: *Self, tag: Mir.Inst.Tag, m: Memory, reg: Register) !void {
649		const ops: Mir.Inst.Ops = switch (m) {
650		.sib => .mr_sib,
651		.rip => .mr_rip,
652		else => unreachable,
653		};
654		const data: Mir.Inst.Data = .{
655		.rx = .{ .r1 = reg, .payload = switch (ops) {
656		.mr_sib => try self.addExtra(Mir.MemorySib.encode(m)),
657		.mr_rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	639	fn asmMemoryRegisterRegister(
	640	self: *Self,
	641	tag: Mir.Inst.Tag,
	642	m: Memory,
	643	reg1: Register,
	644	reg2: Register,
	645	) !void {
	646	_ = try self.addInst(.{
	647	.tag = tag,
	648	.ops = switch (m) {
	649	.sib => .mrr_sib,
	650	.rip => .mrr_rip,
658	651	else => unreachable,
659		} },
660		};
	652	},
	653	.data = .{ .rrx = .{ .r1 = reg1, .r2 = reg2, .payload = switch (m) {
	654	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	655	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	656	else => unreachable,
	657	} } },
	658	});
	659	}
	660
	661	fn asmMemoryRegisterImmediate(
	662	self: *Self,
	663	tag: Mir.Inst.Tag,
	664	m: Memory,
	665	reg: Register,
	666	imm: Immediate,
	667	) !void {
661	668	_ = try self.addInst(.{
662	669	.tag = tag,
663		.ops = ops,
664		.data = data,
	670	.ops = switch (m) {
	671	.sib => switch (imm) {
	672	.signed => .mri_sib_s,
	673	.unsigned => .mri_sib_u,
	674	},
	675	.rip => switch (imm) {
	676	.signed => .mri_rip_s,
	677	.unsigned => .mri_sib_u,
	678	},
	679	else => unreachable,
	680	},
	681	.data = .{ .rix = .{ .r = reg, .i = switch (imm) {
	682	.signed => \|s\| @bitCast(u32, s),
	683	.unsigned => \|u\| @intCast(u32, u),
	684	}, .payload = switch (m) {
	685	.sib => try self.addExtra(Mir.MemorySib.encode(m)),
	686	.rip => try self.addExtra(Mir.MemoryRip.encode(m)),
	687	else => unreachable,
	688	} } },
665	689	});
666	690	}
667	691
...	...	@@ -768,18 +792,12 @@ fn gen(self: *Self) InnerError!void {
768	792	self.mir_instructions.set(backpatch_stack_sub, .{
769	793	.tag = .sub,
770	794	.ops = .ri_u,
771		.data = .{ .ri = .{
772		.r1 = .rsp,
773		.imm = aligned_stack_end,
774		} },
	795	.data = .{ .ri = .{ .r = .rsp, .i = aligned_stack_end } },
775	796	});
776	797	self.mir_instructions.set(backpatch_stack_add, .{
777	798	.tag = .add,
778	799	.ops = .ri_u,
779		.data = .{ .ri = .{
780		.r1 = .rsp,
781		.imm = aligned_stack_end,
782		} },
	800	.data = .{ .ri = .{ .r = .rsp, .i = aligned_stack_end } },
783	801	});
784	802
785	803	const save_reg_list = try self.addExtra(Mir.SaveRegisterList{
...	...	@@ -1732,6 +1750,7 @@ fn airAddSubShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1732	1750	.add_with_overflow => try self.genBinOp(null, .add, bin_op.lhs, bin_op.rhs),
1733	1751	.sub_with_overflow => try self.genBinOp(null, .sub, bin_op.lhs, bin_op.rhs),
1734	1752	.shl_with_overflow => blk: {
	1753	try self.register_manager.getReg(.rcx, null);
1735	1754	const lhs = try self.resolveInst(bin_op.lhs);
1736	1755	const rhs = try self.resolveInst(bin_op.rhs);
1737	1756	const shift_ty = self.air.typeOf(bin_op.rhs);
...	...	@@ -2022,6 +2041,7 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
2022	2041	try self.spillRegisters(&.{.rcx});
2023	2042
2024	2043	const tag = self.air.instructions.items(.tag)[inst];
	2044	try self.register_manager.getReg(.rcx, null);
2025	2045	const lhs = try self.resolveInst(bin_op.lhs);
2026	2046	const rhs = try self.resolveInst(bin_op.rhs);
2027	2047	const lhs_ty = self.air.typeOf(bin_op.lhs);
...	...	@@ -2151,7 +2171,7 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
2151	2171	const result = try self.copyToRegisterWithInstTracking(inst, err_union_ty, operand);
2152	2172	if (err_off > 0) {
2153	2173	const shift = @intCast(u6, err_off * 8);
2154		try self.genShiftBinOpMir(.shr, err_union_ty, result.register, .{ .immediate = shift });
	2174	try self.genShiftBinOpMir(.shr, err_union_ty, result, .{ .immediate = shift });
2155	2175	} else {
2156	2176	try self.truncateRegister(Type.anyerror, result.register);
2157	2177	}
...	...	@@ -2183,9 +2203,7 @@ fn genUnwrapErrorUnionPayloadMir(
2183	2203	const payload_ty = err_union_ty.errorUnionPayload();
2184	2204
2185	2205	const result: MCValue = result: {
2186		if (!payload_ty.hasRuntimeBitsIgnoreComptime()) {
2187		break :result MCValue.none;
2188		}
	2206	if (!payload_ty.hasRuntimeBitsIgnoreComptime()) break :result .none;
2189	2207
2190	2208	const payload_off = errUnionPayloadOffset(payload_ty, self.target.*);
2191	2209	switch (err_union) {
...	...	@@ -2198,17 +2216,17 @@ fn genUnwrapErrorUnionPayloadMir(
2198	2216	const eu_lock = self.register_manager.lockReg(reg);
2199	2217	defer if (eu_lock) \|lock\| self.register_manager.unlockReg(lock);
2200	2218
2201		const result_reg: Register = if (maybe_inst) \|inst\|
2202		(try self.copyToRegisterWithInstTracking(inst, err_union_ty, err_union)).register
	2219	const result_mcv: MCValue = if (maybe_inst) \|inst\|
	2220	try self.copyToRegisterWithInstTracking(inst, err_union_ty, err_union)
2203	2221	else
2204		try self.copyToTmpRegister(err_union_ty, err_union);
	2222	.{ .register = try self.copyToTmpRegister(err_union_ty, err_union) };
2205	2223	if (payload_off > 0) {
2206	2224	const shift = @intCast(u6, payload_off * 8);
2207		try self.genShiftBinOpMir(.shr, err_union_ty, result_reg, .{ .immediate = shift });
	2225	try self.genShiftBinOpMir(.shr, err_union_ty, result_mcv, .{ .immediate = shift });
2208	2226	} else {
2209		try self.truncateRegister(payload_ty, result_reg);
	2227	try self.truncateRegister(payload_ty, result_mcv.register);
2210	2228	}
2211		break :result MCValue{ .register = result_reg };
	2229	break :result result_mcv;
2212	2230	},
2213	2231	else => return self.fail("TODO implement genUnwrapErrorUnionPayloadMir for {}", .{err_union}),
2214	2232	}
...	...	@@ -2848,7 +2866,7 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
2848	2866	else
2849	2867	0;
2850	2868	const result = try self.copyToRegisterWithInstTracking(inst, union_ty, operand);
2851		try self.genShiftBinOpMir(.shr, Type.usize, result.register, .{ .immediate = shift });
	2869	try self.genShiftBinOpMir(.shr, Type.usize, result, .{ .immediate = shift });
2852	2870	break :blk MCValue{
2853	2871	.register = registerAlias(result.register, @intCast(u32, layout.tag_size)),
2854	2872	};
...	...	@@ -3769,12 +3787,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
3769	3787	defer if (dst_mcv_lock) \|lock\| self.register_manager.unlockReg(lock);
3770	3788
3771	3789	// Shift by struct_field_offset.
3772		try self.genShiftBinOpMir(
3773		.shr,
3774		Type.usize,
3775		dst_mcv.register,
3776		.{ .immediate = field_bit_offset },
3777		);
	3790	try self.genShiftBinOpMir(.shr, Type.usize, dst_mcv, .{ .immediate = field_bit_offset });
3778	3791
3779	3792	// Mask to field_bit_size bits
3780	3793	const field_bit_size = field_ty.bitSize(self.target.*);
...	...	@@ -3932,29 +3945,186 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue
3932	3945	}
3933	3946
3934	3947	/// Clobbers .rcx for non-immediate shift value.
3935		fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shift: MCValue) !void {
3936		switch (tag) {
3937		.sal, .sar, .shl, .shr => {},
3938		else => unreachable,
3939		}
3940
3941		const abi_size = @intCast(u32, ty.abiSize(self.target.*));
3942		blk: {
3943		switch (shift) {
	3948	fn genShiftBinOpMir(
	3949	self: *Self,
	3950	tag: Mir.Inst.Tag,
	3951	ty: Type,
	3952	lhs_mcv: MCValue,
	3953	shift_mcv: MCValue,
	3954	) !void {
	3955	const rhs_mcv: MCValue = rhs: {
	3956	switch (shift_mcv) {
3944	3957	.immediate => \|imm\| switch (imm) {
3945	3958	0 => return,
3946		else => return self.asmRegisterImmediate(tag, registerAlias(reg, abi_size), Immediate.u(imm)),
3947		},
3948		.register => \|shift_reg\| {
3949		if (shift_reg == .rcx) break :blk;
	3959	else => break :rhs shift_mcv,
3950	3960	},
	3961	.register => \|shift_reg\| if (shift_reg == .rcx) break :rhs shift_mcv,
3951	3962	else => {},
3952	3963	}
3953	3964	self.register_manager.getRegAssumeFree(.rcx, null);
3954		try self.genSetReg(Type.u8, .rcx, shift);
3955		}
	3965	try self.genSetReg(Type.u8, .rcx, shift_mcv);
	3966	break :rhs .{ .register = .rcx };
	3967	};
3956	3968
3957		try self.asmRegisterRegister(tag, registerAlias(reg, abi_size), .cl);
	3969	const abi_size = @intCast(u32, ty.abiSize(self.target.*));
	3970	if (abi_size <= 8) {
	3971	switch (lhs_mcv) {
	3972	.register => \|lhs_reg\| switch (rhs_mcv) {
	3973	.immediate => \|rhs_imm\| try self.asmRegisterImmediate(
	3974	tag,
	3975	registerAlias(lhs_reg, abi_size),
	3976	Immediate.u(rhs_imm),
	3977	),
	3978	.register => \|rhs_reg\| try self.asmRegisterRegister(
	3979	tag,
	3980	registerAlias(lhs_reg, abi_size),
	3981	registerAlias(rhs_reg, 1),
	3982	),
	3983	else => return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
	3984	@tagName(lhs_mcv),
	3985	@tagName(rhs_mcv),
	3986	}),
	3987	},
	3988	.stack_offset => \|lhs_off\| switch (rhs_mcv) {
	3989	.immediate => \|rhs_imm\| try self.asmMemoryImmediate(
	3990	tag,
	3991	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -lhs_off }),
	3992	Immediate.u(rhs_imm),
	3993	),
	3994	.register => \|rhs_reg\| try self.asmMemoryRegister(
	3995	tag,
	3996	Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -lhs_off }),
	3997	registerAlias(rhs_reg, 1),
	3998	),
	3999	else => return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
	4000	@tagName(lhs_mcv),
	4001	@tagName(rhs_mcv),
	4002	}),
	4003	},
	4004	else => return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
	4005	@tagName(lhs_mcv),
	4006	@tagName(rhs_mcv),
	4007	}),
	4008	}
	4009	} else if (abi_size <= 16) {
	4010	const tmp_reg = try self.register_manager.allocReg(null, gp);
	4011	const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
	4012	defer self.register_manager.unlockReg(tmp_lock);
	4013
	4014	const info: struct { offsets: [2]i32, double_tag: Mir.Inst.Tag } = switch (tag) {
	4015	.shl, .sal => .{ .offsets = .{ 0, 8 }, .double_tag = .shld },
	4016	.shr, .sar => .{ .offsets = .{ 8, 0 }, .double_tag = .shrd },
	4017	else => unreachable,
	4018	};
	4019	switch (lhs_mcv) {
	4020	.stack_offset => \|dst_off\| switch (rhs_mcv) {
	4021	.immediate => \|rhs_imm\| if (rhs_imm == 0) {} else if (rhs_imm < 64) {
	4022	try self.asmRegisterMemory(
	4023	.mov,
	4024	tmp_reg,
	4025	Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),
	4026	);
	4027	try self.asmMemoryRegisterImmediate(
	4028	info.double_tag,
	4029	Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[1] - dst_off }),
	4030	tmp_reg,
	4031	Immediate.u(rhs_imm),
	4032	);
	4033	try self.asmMemoryImmediate(
	4034	tag,
	4035	Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),
	4036	Immediate.u(rhs_imm),
	4037	);
	4038	} else {
	4039	assert(rhs_imm < 128);
	4040	try self.asmRegisterMemory(
	4041	.mov,
	4042	tmp_reg,
	4043	Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),
	4044	);
	4045	if (rhs_imm > 64) {
	4046	try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(rhs_imm - 64));
	4047	}
	4048	try self.asmMemoryRegister(
	4049	.mov,
	4050	Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[1] - dst_off }),
	4051	tmp_reg,
	4052	);
	4053	switch (tag) {
	4054	.shl, .sal, .shr => {
	4055	try self.asmRegisterRegister(.xor, tmp_reg.to32(), tmp_reg.to32());
	4056	try self.asmMemoryRegister(
	4057	.mov,
	4058	Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),
	4059	tmp_reg,
	4060	);
	4061	},
	4062	.sar => try self.asmMemoryImmediate(
	4063	tag,
	4064	Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),
	4065	Immediate.u(63),
	4066	),
	4067	else => unreachable,
	4068	}
	4069	},
	4070	else => {
	4071	const first_reg = try self.register_manager.allocReg(null, gp);
	4072	const first_lock = self.register_manager.lockRegAssumeUnused(first_reg);
	4073	defer self.register_manager.unlockReg(first_lock);
	4074
	4075	const second_reg = try self.register_manager.allocReg(null, gp);
	4076	const second_lock = self.register_manager.lockRegAssumeUnused(second_reg);
	4077	defer self.register_manager.unlockReg(second_lock);
	4078
	4079	try self.genSetReg(Type.u8, .cl, rhs_mcv);
	4080	try self.asmRegisterMemory(
	4081	.mov,
	4082	first_reg,
	4083	Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),
	4084	);
	4085	try self.asmRegisterMemory(
	4086	.mov,
	4087	second_reg,
	4088	Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[1] - dst_off }),
	4089	);
	4090	switch (tag) {
	4091	.shl, .sal, .shr => try self.asmRegisterRegister(
	4092	.xor,
	4093	tmp_reg.to32(),
	4094	tmp_reg.to32(),
	4095	),
	4096	.sar => {
	4097	try self.asmRegisterRegister(.mov, tmp_reg, first_reg);
	4098	try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(63));
	4099	},
	4100	else => unreachable,
	4101	}
	4102	try self.asmRegisterRegisterRegister(info.double_tag, second_reg, first_reg, .cl);
	4103	try self.asmRegisterRegister(tag, first_reg, .cl);
	4104	try self.asmRegisterImmediate(.cmp, .cl, Immediate.u(64));
	4105	try self.asmCmovccRegisterRegister(second_reg, first_reg, .ae);
	4106	try self.asmCmovccRegisterRegister(first_reg, tmp_reg, .ae);
	4107	try self.asmMemoryRegister(
	4108	.mov,
	4109	Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[1] - dst_off }),
	4110	second_reg,
	4111	);
	4112	try self.asmMemoryRegister(
	4113	.mov,
	4114	Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),
	4115	first_reg,
	4116	);
	4117	},
	4118	},
	4119	else => return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
	4120	@tagName(lhs_mcv),
	4121	@tagName(rhs_mcv),
	4122	}),
	4123	}
	4124	} else return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
	4125	@tagName(lhs_mcv),
	4126	@tagName(rhs_mcv),
	4127	});
3958	4128	}
3959	4129
3960	4130	/// Result is always a register.
...	...	@@ -3964,68 +4134,61 @@ fn genShiftBinOp(
3964	4134	self: *Self,
3965	4135	tag: Air.Inst.Tag,
3966	4136	maybe_inst: ?Air.Inst.Index,
3967		lhs: MCValue,
3968		rhs: MCValue,
	4137	lhs_mcv: MCValue,
	4138	rhs_mcv: MCValue,
3969	4139	lhs_ty: Type,
3970	4140	rhs_ty: Type,
3971	4141	) !MCValue {
3972		if (lhs_ty.zigTypeTag() == .Vector or lhs_ty.zigTypeTag() == .Float) {
	4142	if (lhs_ty.zigTypeTag() == .Vector) {
3973	4143	return self.fail("TODO implement genShiftBinOp for {}", .{lhs_ty.fmtDebug()});
3974	4144	}
3975		if (lhs_ty.abiSize(self.target.*) > 8) {
	4145
	4146	assert(rhs_ty.abiSize(self.target.*) == 1);
	4147
	4148	const lhs_abi_size = lhs_ty.abiSize(self.target.*);
	4149	if (lhs_abi_size > 16) {
3976	4150	return self.fail("TODO implement genShiftBinOp for {}", .{lhs_ty.fmtDebug()});
3977	4151	}
3978	4152
3979		assert(rhs_ty.abiSize(self.target.*) == 1);
	4153	self.register_manager.getRegAssumeFree(.rcx, null);
	4154	const rcx_lock = self.register_manager.lockRegAssumeUnused(.rcx);
	4155	defer self.register_manager.unlockReg(rcx_lock);
3980	4156
3981		const lhs_lock: ?RegisterLock = switch (lhs) {
	4157	const lhs_lock = switch (lhs_mcv) {
3982	4158	.register => \|reg\| self.register_manager.lockReg(reg),
3983	4159	else => null,
3984	4160	};
3985	4161	defer if (lhs_lock) \|lock\| self.register_manager.unlockReg(lock);
3986	4162
3987		const rhs_lock: ?RegisterLock = switch (rhs) {
	4163	const rhs_lock = switch (rhs_mcv) {
3988	4164	.register => \|reg\| self.register_manager.lockReg(reg),
3989	4165	else => null,
3990	4166	};
3991	4167	defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
3992	4168
3993		self.register_manager.getRegAssumeFree(.rcx, null);
3994		const rcx_lock = self.register_manager.lockRegAssumeUnused(.rcx);
3995		defer self.register_manager.unlockReg(rcx_lock);
3996
3997		const dst: MCValue = blk: {
	4169	const dst_mcv: MCValue = dst: {
3998	4170	if (maybe_inst) \|inst\| {
3999	4171	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
4000		// TODO dst can also be a memory location
4001		if (self.reuseOperand(inst, bin_op.lhs, 0, lhs) and lhs.isRegister()) {
4002		break :blk lhs;
4003		}
4004		break :blk try self.copyToRegisterWithInstTracking(inst, lhs_ty, lhs);
	4172	if (self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv)) break :dst lhs_mcv;
4005	4173	}
4006		break :blk MCValue{ .register = try self.copyToTmpRegister(lhs_ty, lhs) };
	4174	const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, maybe_inst, true);
	4175	try self.setRegOrMem(lhs_ty, dst_mcv, lhs_mcv);
	4176	break :dst dst_mcv;
4007	4177	};
4008	4178
4009	4179	const signedness = lhs_ty.intInfo(self.target.*).signedness;
4010		switch (tag) {
4011		.shl => try self.genShiftBinOpMir(switch (signedness) {
	4180	try self.genShiftBinOpMir(switch (tag) {
	4181	.shl, .shl_exact => switch (signedness) {
4012	4182	.signed => .sal,
4013	4183	.unsigned => .shl,
4014		}, lhs_ty, dst.register, rhs),
4015
4016		.shl_exact => try self.genShiftBinOpMir(.shl, lhs_ty, dst.register, rhs),
4017
4018		.shr,
4019		.shr_exact,
4020		=> try self.genShiftBinOpMir(switch (signedness) {
	4184	},
	4185	.shr, .shr_exact => switch (signedness) {
4021	4186	.signed => .sar,
4022	4187	.unsigned => .shr,
4023		}, lhs_ty, dst.register, rhs),
4024
	4188	},
4025	4189	else => unreachable,
4026		}
4027
4028		return dst;
	4190	}, lhs_ty, dst_mcv, rhs_mcv);
	4191	return dst_mcv;
4029	4192	}
4030	4193
4031	4194	/// Result is always a register.
...	...	@@ -5552,7 +5715,7 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) !
5552	5715	const tmp_reg = try self.copyToTmpRegister(ty, operand);
5553	5716	if (err_off > 0) {
5554	5717	const shift = @intCast(u6, err_off * 8);
5555		try self.genShiftBinOpMir(.shr, ty, tmp_reg, .{ .immediate = shift });
	5718	try self.genShiftBinOpMir(.shr, ty, .{ .register = tmp_reg }, .{ .immediate = shift });
5556	5719	} else {
5557	5720	try self.truncateRegister(Type.anyerror, tmp_reg);
5558	5721	}
...	...	@@ -6506,7 +6669,7 @@ fn genInlineMemcpyRegisterRegister(
6506	6669	}), registerAlias(tmp_reg, nearest_power_of_two));
6507	6670
6508	6671	if (nearest_power_of_two > 1) {
6509		try self.genShiftBinOpMir(.shr, ty, tmp_reg, .{
	6672	try self.genShiftBinOpMir(.shr, ty, .{ .register = tmp_reg }, .{
6510	6673	.immediate = nearest_power_of_two * 8,
6511	6674	});
6512	6675	}
...	...	@@ -7032,7 +7195,7 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
7032	7195
7033	7196	try self.spillEflagsIfOccupied();
7034	7197	_ = try self.addInst(.{ .tag = .cmpxchg, .ops = .lock_mr_sib, .data = .{ .rx = .{
7035		.r1 = new_reg,
	7198	.r = new_reg,
7036	7199	.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
7037	7200	} } });
7038	7201
...	...	@@ -7110,7 +7273,7 @@ fn atomicOp(
7110	7273	.xadd, .add, .sub, .@"and", .@"or", .xor => .lock_mr_sib,
7111	7274	else => unreachable,
7112	7275	}, .data = .{ .rx = .{
7113		.r1 = registerAlias(dst_reg, val_abi_size),
	7276	.r = registerAlias(dst_reg, val_abi_size),
7114	7277	.payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
7115	7278	} } });
7116	7279	return;
...	...	@@ -7702,8 +7865,8 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
7702	7865	switch (int_info.signedness) {
7703	7866	.signed => {
7704	7867	const shift = @intCast(u6, max_reg_bit_width - int_info.bits);
7705		try self.genShiftBinOpMir(.sal, Type.isize, reg, .{ .immediate = shift });
7706		try self.genShiftBinOpMir(.sar, Type.isize, reg, .{ .immediate = shift });
	7868	try self.genShiftBinOpMir(.sal, Type.isize, .{ .register = reg }, .{ .immediate = shift });
	7869	try self.genShiftBinOpMir(.sar, Type.isize, .{ .register = reg }, .{ .immediate = shift });
7707	7870	},
7708	7871	.unsigned => {
7709	7872	const shift = @intCast(u6, max_reg_bit_width - int_info.bits);

src/arch/x86_64/Emit.zig+68-27

...	...	@@ -121,7 +121,9 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
121	121	.sbb,
122	122	.sfence,
123	123	.shl,
	124	.shld,
124	125	.shr,
	126	.shrd,
125	127	.sub,
126	128	.syscall,
127	129	.@"test",
...	...	@@ -231,10 +233,10 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
231	233	const prefix: Instruction.Prefix = switch (ops) {
232	234	.lock_m_sib,
233	235	.lock_m_rip,
234		.lock_mi_u_sib,
235		.lock_mi_u_rip,
236		.lock_mi_s_sib,
237		.lock_mi_s_rip,
	236	.lock_mi_sib_u,
	237	.lock_mi_rip_u,
	238	.lock_mi_sib_s,
	239	.lock_mi_rip_s,
238	240	.lock_mr_sib,
239	241	.lock_mr_rip,
240	242	.lock_moffs_rax,
...	...	@@ -249,31 +251,36 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
249	251
250	252	switch (ops) {
251	253	.none => {},
252		.imm_s => op1 = .{ .imm = Immediate.s(@bitCast(i32, data.imm)) },
253		.imm_u => op1 = .{ .imm = Immediate.u(data.imm) },
	254	.i_s => op1 = .{ .imm = Immediate.s(@bitCast(i32, data.i)) },
	255	.i_u => op1 = .{ .imm = Immediate.u(data.i) },
254	256	.r => op1 = .{ .reg = data.r },
255	257	.rr => {
256	258	op1 = .{ .reg = data.rr.r1 };
257	259	op2 = .{ .reg = data.rr.r2 };
258	260	},
	261	.rrr => {
	262	op1 = .{ .reg = data.rrr.r1 };
	263	op2 = .{ .reg = data.rrr.r2 };
	264	op3 = .{ .reg = data.rrr.r3 };
	265	},
259	266	.ri_s, .ri_u => {
260	267	const imm = switch (ops) {
261		.ri_s => Immediate.s(@bitCast(i32, data.ri.imm)),
262		.ri_u => Immediate.u(data.ri.imm),
	268	.ri_s => Immediate.s(@bitCast(i32, data.ri.i)),
	269	.ri_u => Immediate.u(data.ri.i),
263	270	else => unreachable,
264	271	};
265		op1 = .{ .reg = data.ri.r1 };
	272	op1 = .{ .reg = data.ri.r };
266	273	op2 = .{ .imm = imm };
267	274	},
268	275	.ri64 => {
269	276	const imm64 = emit.mir.extraData(Mir.Imm64, data.rx.payload).data;
270		op1 = .{ .reg = data.rx.r1 };
	277	op1 = .{ .reg = data.rx.r };
271	278	op2 = .{ .imm = Immediate.u(Mir.Imm64.decode(imm64)) };
272	279	},
273	280	.rri_s, .rri_u => {
274	281	const imm = switch (ops) {
275		.rri_s => Immediate.s(@bitCast(i32, data.rri.imm)),
276		.rri_u => Immediate.u(data.rri.imm),
	282	.rri_s => Immediate.s(@bitCast(i32, data.rri.i)),
	283	.rri_u => Immediate.u(data.rri.i),
277	284	else => unreachable,
278	285	};
279	286	op1 = .{ .reg = data.rri.r1 };
...	...	@@ -288,21 +295,21 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
288	295	const mrip = emit.mir.extraData(Mir.MemoryRip, data.payload).data;
289	296	op1 = .{ .mem = Mir.MemoryRip.decode(mrip) };
290	297	},
291		.mi_s_sib, .mi_u_sib, .lock_mi_s_sib, .lock_mi_u_sib => {
292		const msib = emit.mir.extraData(Mir.MemorySib, data.xi.payload).data;
	298	.mi_sib_s, .mi_sib_u, .lock_mi_sib_s, .lock_mi_sib_u => {
	299	const msib = emit.mir.extraData(Mir.MemorySib, data.ix.payload).data;
293	300	const imm = switch (ops) {
294		.mi_s_sib, .lock_mi_s_sib => Immediate.s(@bitCast(i32, data.xi.imm)),
295		.mi_u_sib, .lock_mi_u_sib => Immediate.u(data.xi.imm),
	301	.mi_sib_s, .lock_mi_sib_s => Immediate.s(@bitCast(i32, data.ix.i)),
	302	.mi_sib_u, .lock_mi_sib_u => Immediate.u(data.ix.i),
296	303	else => unreachable,
297	304	};
298	305	op1 = .{ .mem = Mir.MemorySib.decode(msib) };
299	306	op2 = .{ .imm = imm };
300	307	},
301		.mi_u_rip, .mi_s_rip, .lock_mi_u_rip, .lock_mi_s_rip => {
302		const mrip = emit.mir.extraData(Mir.MemoryRip, data.xi.payload).data;
	308	.mi_rip_u, .mi_rip_s, .lock_mi_rip_u, .lock_mi_rip_s => {
	309	const mrip = emit.mir.extraData(Mir.MemoryRip, data.ix.payload).data;
303	310	const imm = switch (ops) {
304		.mi_s_rip, .lock_mi_s_rip => Immediate.s(@bitCast(i32, data.xi.imm)),
305		.mi_u_rip, .lock_mi_u_rip => Immediate.u(data.xi.imm),
	311	.mi_rip_s, .lock_mi_rip_s => Immediate.s(@bitCast(i32, data.ix.i)),
	312	.mi_rip_u, .lock_mi_rip_u => Immediate.u(data.ix.i),
306	313	else => unreachable,
307	314	};
308	315	op1 = .{ .mem = Mir.MemoryRip.decode(mrip) };
...	...	@@ -310,7 +317,7 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
310	317	},
311	318	.rm_sib, .mr_sib, .lock_mr_sib => {
312	319	const msib = emit.mir.extraData(Mir.MemorySib, data.rx.payload).data;
313		const op_r = .{ .reg = data.rx.r1 };
	320	const op_r = .{ .reg = data.rx.r };
314	321	const op_m = .{ .mem = Mir.MemorySib.decode(msib) };
315	322	switch (ops) {
316	323	.rm_sib => {
...	...	@@ -326,7 +333,7 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
326	333	},
327	334	.rm_rip, .mr_rip, .lock_mr_rip => {
328	335	const mrip = emit.mir.extraData(Mir.MemoryRip, data.rx.payload).data;
329		const op_r = .{ .reg = data.rx.r1 };
	336	const op_r = .{ .reg = data.rx.r };
330	337	const op_m = .{ .mem = Mir.MemoryRip.decode(mrip) };
331	338	switch (ops) {
332	339	.rm_rip => {
...	...	@@ -340,6 +347,40 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
340	347	else => unreachable,
341	348	}
342	349	},
	350	.mrr_sib => {
	351	const msib = emit.mir.extraData(Mir.MemorySib, data.rrx.payload).data;
	352	op1 = .{ .mem = Mir.MemorySib.decode(msib) };
	353	op2 = .{ .reg = data.rrx.r1 };
	354	op2 = .{ .reg = data.rrx.r2 };
	355	},
	356	.mrr_rip => {
	357	const mrip = emit.mir.extraData(Mir.MemoryRip, data.rrx.payload).data;
	358	op1 = .{ .mem = Mir.MemoryRip.decode(mrip) };
	359	op2 = .{ .reg = data.rrx.r1 };
	360	op2 = .{ .reg = data.rrx.r2 };
	361	},
	362	.mri_sib_u, .mri_sib_s => {
	363	const msib = emit.mir.extraData(Mir.MemorySib, data.rix.payload).data;
	364	const imm = switch (ops) {
	365	.mri_sib_s => Immediate.s(@bitCast(i32, data.rix.i)),
	366	.mri_sib_u, .lock_mi_rip_u => Immediate.u(data.rix.i),
	367	else => unreachable,
	368	};
	369	op1 = .{ .mem = Mir.MemorySib.decode(msib) };
	370	op2 = .{ .reg = data.rix.r };
	371	op3 = .{ .imm = imm };
	372	},
	373	.mri_rip_u, .mri_rip_s => {
	374	const mrip = emit.mir.extraData(Mir.MemoryRip, data.rix.payload).data;
	375	const imm = switch (ops) {
	376	.mri_rip_s => Immediate.s(@bitCast(i32, data.rix.i)),
	377	.mri_rip_u, .lock_mi_rip_u => Immediate.u(data.rix.i),
	378	else => unreachable,
	379	};
	380	op1 = .{ .mem = Mir.MemoryRip.decode(mrip) };
	381	op2 = .{ .reg = data.rix.r };
	382	op3 = .{ .imm = imm };
	383	},
343	384	else => return emit.fail("TODO handle generic encoding: {s}, {s}", .{
344	385	@tagName(mnemonic),
345	386	@tagName(ops),
...	...	@@ -451,12 +492,12 @@ fn mirMovsx(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
451	492	},
452	493	.rm_sib => {
453	494	const msib = emit.mir.extraData(Mir.MemorySib, data.rx.payload).data;
454		op1 = .{ .reg = data.rx.r1 };
	495	op1 = .{ .reg = data.rx.r };
455	496	op2 = .{ .mem = Mir.MemorySib.decode(msib) };
456	497	},
457	498	.rm_rip => {
458	499	const mrip = emit.mir.extraData(Mir.MemoryRip, data.rx.payload).data;
459		op1 = .{ .reg = data.rx.r1 };
	500	op1 = .{ .reg = data.rx.r };
460	501	op2 = .{ .mem = Mir.MemoryRip.decode(mrip) };
461	502	},
462	503	else => unreachable, // TODO
...	...	@@ -495,7 +536,7 @@ fn mirCmovcc(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
495	536	const extra = emit.mir.extraData(Mir.MemorySib, data.payload).data;
496	537	const mnemonic = mnemonicFromConditionCode("cmov", data.cc);
497	538	return emit.encode(mnemonic, .{
498		.op1 = .{ .reg = data.r1 },
	539	.op1 = .{ .reg = data.r },
499	540	.op2 = .{ .mem = Mir.MemorySib.decode(extra) },
500	541	});
501	542	},
...	...	@@ -504,7 +545,7 @@ fn mirCmovcc(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
504	545	const extra = emit.mir.extraData(Mir.MemoryRip, data.payload).data;
505	546	const mnemonic = mnemonicFromConditionCode("cmov", data.cc);
506	547	return emit.encode(mnemonic, .{
507		.op1 = .{ .reg = data.r1 },
	548	.op1 = .{ .reg = data.r },
508	549	.op2 = .{ .mem = Mir.MemoryRip.decode(extra) },
509	550	});
510	551	},
...	...	@@ -519,7 +560,7 @@ fn mirSetcc(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
519	560	const data = emit.mir.instructions.items(.data)[inst].r_cc;
520	561	const mnemonic = mnemonicFromConditionCode("set", data.cc);
521	562	return emit.encode(mnemonic, .{
522		.op1 = .{ .reg = data.r1 },
	563	.op1 = .{ .reg = data.r },
523	564	});
524	565	},
525	566	.m_sib_cc => {

src/arch/x86_64/Encoding.zig+7-6

...	...	@@ -262,15 +262,15 @@ pub fn format(
262	262	try writer.print("+{s} ", .{tag});
263	263	},
264	264	.m, .mi, .m1, .mc => try writer.print("/{d} ", .{encoding.modRmExt()}),
265		.mr, .rm, .rmi => try writer.writeAll("/r "),
	265	.mr, .rm, .rmi, .mri, .mrc => try writer.writeAll("/r "),
266	266	}
267	267
268	268	switch (encoding.op_en) {
269		.i, .d, .zi, .oi, .mi, .rmi => {
	269	.i, .d, .zi, .oi, .mi, .rmi, .mri => {
270	270	const op = switch (encoding.op_en) {
271	271	.i, .d => encoding.op1,
272	272	.zi, .oi, .mi => encoding.op2,
273		.rmi => encoding.op3,
	273	.rmi, .mri => encoding.op3,
274	274	else => unreachable,
275	275	};
276	276	const tag = switch (op) {
...	...	@@ -285,7 +285,7 @@ pub fn format(
285	285	};
286	286	try writer.print("{s} ", .{tag});
287	287	},
288		.np, .fd, .td, .o, .m, .m1, .mc, .mr, .rm => {},
	288	.np, .fd, .td, .o, .m, .m1, .mc, .mr, .rm, .mrc => {},
289	289	}
290	290
291	291	try writer.print("{s} ", .{@tagName(encoding.mnemonic)});
...	...	@@ -334,7 +334,7 @@ pub const Mnemonic = enum {
334	334	rcl, rcr, ret, rol, ror,
335	335	sal, sar, sbb,
336	336	scas, scasb, scasd, scasq, scasw,
337		shl, shr, sub, syscall,
	337	shl, shld, shr, shrd, sub, syscall,
338	338	seta, setae, setb, setbe, setc, sete, setg, setge, setl, setle, setna, setnae,
339	339	setnb, setnbe, setnc, setne, setng, setnge, setnl, setnle, setno, setnp, setns,
340	340	setnz, seto, setp, setpe, setpo, sets, setz,
...	...	@@ -374,7 +374,8 @@ pub const OpEn = enum {
374	374	i, zi,
375	375	d, m,
376	376	fd, td,
377		m1, mc, mi, mr, rm, rmi,
	377	m1, mc, mi, mr, rm,
	378	rmi, mri, mrc,
378	379	// zig fmt: on
379	380	};
380	381

src/arch/x86_64/Mir.zig+59-25

...	...	@@ -138,8 +138,12 @@ pub const Inst = struct {
138	138	sfence,
139	139	/// Logical shift left
140	140	shl,
	141	/// Double precision shift left
	142	shld,
141	143	/// Logical shift right
142	144	shr,
	145	/// Double precision shift right
	146	shrd,
143	147	/// Subtract
144	148	sub,
145	149	/// Syscall
...	...	@@ -284,10 +288,10 @@ pub const Inst = struct {
284	288	ri64,
285	289	/// Immediate (sign-extended) operand.
286	290	/// Uses `imm` payload.
287		imm_s,
	291	i_s,
288	292	/// Immediate (unsigned) operand.
289	293	/// Uses `imm` payload.
290		imm_u,
	294	i_u,
291	295	/// Relative displacement operand.
292	296	/// Uses `imm` payload.
293	297	rel,
...	...	@@ -316,23 +320,41 @@ pub const Inst = struct {
316	320	/// Uses `x_cc` with extra data of type `MemoryRip`.
317	321	m_rip_cc,
318	322	/// Memory (SIB), immediate (unsigned) operands.
319		/// Uses `xi` payload with extra data of type `MemorySib`.
320		mi_u_sib,
	323	/// Uses `ix` payload with extra data of type `MemorySib`.
	324	mi_sib_u,
321	325	/// Memory (RIP), immediate (unsigned) operands.
322		/// Uses `xi` payload with extra data of type `MemoryRip`.
323		mi_u_rip,
	326	/// Uses `ix` payload with extra data of type `MemoryRip`.
	327	mi_rip_u,
324	328	/// Memory (SIB), immediate (sign-extend) operands.
325		/// Uses `xi` payload with extra data of type `MemorySib`.
326		mi_s_sib,
	329	/// Uses `ix` payload with extra data of type `MemorySib`.
	330	mi_sib_s,
327	331	/// Memory (RIP), immediate (sign-extend) operands.
328		/// Uses `xi` payload with extra data of type `MemoryRip`.
329		mi_s_rip,
	332	/// Uses `ix` payload with extra data of type `MemoryRip`.
	333	mi_rip_s,
330	334	/// Memory (SIB), register operands.
331	335	/// Uses `rx` payload with extra data of type `MemorySib`.
332	336	mr_sib,
333	337	/// Memory (RIP), register operands.
334	338	/// Uses `rx` payload with extra data of type `MemoryRip`.
335	339	mr_rip,
	340	/// Memory (SIB), register, register operands.
	341	/// Uses `rrx` payload with extra data of type `MemorySib`.
	342	mrr_sib,
	343	/// Memory (RIP), register, register operands.
	344	/// Uses `rrx` payload with extra data of type `MemoryRip`.
	345	mrr_rip,
	346	/// Memory (SIB), register, immediate (unsigned) operands.
	347	/// Uses `rix` payload with extra data of type `MemorySib`.
	348	mri_sib_u,
	349	/// Memory (RIP), register, immediate (unsigned) operands.
	350	/// Uses `rix` payload with extra data of type `MemoryRip`.
	351	mri_rip_u,
	352	/// Memory (SIB), register, immediate (signed) operands.
	353	/// Uses `rix` payload with extra data of type `MemorySib`.
	354	mri_sib_s,
	355	/// Memory (RIP), register, immediate (signed) operands.
	356	/// Uses `rix` payload with extra data of type `MemoryRip`.
	357	mri_rip_s,
336	358	/// Rax, Memory moffs.
337	359	/// Uses `payload` with extra data of type `MemoryMoffs`.
338	360	rax_moffs,
...	...	@@ -347,16 +369,16 @@ pub const Inst = struct {
347	369	lock_m_rip,
348	370	/// Memory (SIB), immediate (unsigned) operands with lock prefix.
349	371	/// Uses `xi` payload with extra data of type `MemorySib`.
350		lock_mi_u_sib,
	372	lock_mi_sib_u,
351	373	/// Memory (RIP), immediate (unsigned) operands with lock prefix.
352	374	/// Uses `xi` payload with extra data of type `MemoryRip`.
353		lock_mi_u_rip,
	375	lock_mi_rip_u,
354	376	/// Memory (SIB), immediate (sign-extend) operands with lock prefix.
355	377	/// Uses `xi` payload with extra data of type `MemorySib`.
356		lock_mi_s_sib,
	378	lock_mi_sib_s,
357	379	/// Memory (RIP), immediate (sign-extend) operands with lock prefix.
358	380	/// Uses `xi` payload with extra data of type `MemoryRip`.
359		lock_mi_s_rip,
	381	lock_mi_rip_s,
360	382	/// Memory (SIB), register operands with lock prefix.
361	383	/// Uses `rx` payload with extra data of type `MemorySib`.
362	384	lock_mr_sib,
...	...	@@ -400,7 +422,7 @@ pub const Inst = struct {
400	422	cc: bits.Condition,
401	423	},
402	424	/// A 32-bit immediate value.
403		imm: u32,
	425	i: u32,
404	426	r: Register,
405	427	rr: struct {
406	428	r1: Register,
...	...	@@ -414,16 +436,16 @@ pub const Inst = struct {
414	436	rri: struct {
415	437	r1: Register,
416	438	r2: Register,
417		imm: u32,
	439	i: u32,
418	440	},
419	441	/// Condition code (CC), followed by custom payload found in extra.
420	442	x_cc: struct {
421		payload: u32,
422	443	cc: bits.Condition,
	444	payload: u32,
423	445	},
424	446	/// Register with condition code (CC).
425	447	r_cc: struct {
426		r1: Register,
	448	r: Register,
427	449	cc: bits.Condition,
428	450	},
429	451	/// Register, register with condition code (CC).
...	...	@@ -434,24 +456,36 @@ pub const Inst = struct {
434	456	},
435	457	/// Register, immediate.
436	458	ri: struct {
437		r1: Register,
438		imm: u32,
	459	r: Register,
	460	i: u32,
439	461	},
440	462	/// Register, followed by custom payload found in extra.
441	463	rx: struct {
442		r1: Register,
	464	r: Register,
443	465	payload: u32,
444	466	},
445	467	/// Register with condition code (CC), followed by custom payload found in extra.
446	468	rx_cc: struct {
447		r1: Register,
	469	r: Register,
448	470	cc: bits.Condition,
449	471	payload: u32,
450	472	},
451		/// Custom payload followed by an immediate.
452		xi: struct {
	473	/// Immediate, followed by Custom payload found in extra.
	474	ix: struct {
	475	i: u32,
	476	payload: u32,
	477	},
	478	/// Register, register, followed by Custom payload found in extra.
	479	rrx: struct {
	480	r1: Register,
	481	r2: Register,
	482	payload: u32,
	483	},
	484	/// Register, immediate, followed by Custom payload found in extra.
	485	rix: struct {
	486	r: Register,
	487	i: u32,
453	488	payload: u32,
454		imm: u32,
455	489	},
456	490	/// String instruction prefix and width.
457	491	string: struct {

src/arch/x86_64/encoder.zig+9-11

...	...	@@ -174,7 +174,7 @@ pub const Instruction = struct {
174	174	.td => try encoder.imm64(inst.op1.mem.moffs.offset),
175	175	else => {
176	176	const mem_op = switch (encoding.op_en) {
177		.m, .mi, .m1, .mc, .mr => inst.op1,
	177	.m, .mi, .m1, .mc, .mr, .mri, .mrc => inst.op1,
178	178	.rm, .rmi => inst.op2,
179	179	else => unreachable,
180	180	};
...	...	@@ -182,7 +182,7 @@ pub const Instruction = struct {
182	182	.reg => \|reg\| {
183	183	const rm = switch (encoding.op_en) {
184	184	.m, .mi, .m1, .mc => encoding.modRmExt(),
185		.mr => inst.op2.reg.lowEnc(),
	185	.mr, .mri, .mrc => inst.op2.reg.lowEnc(),
186	186	.rm, .rmi => inst.op1.reg.lowEnc(),
187	187	else => unreachable,
188	188	};
...	...	@@ -191,7 +191,7 @@ pub const Instruction = struct {
191	191	.mem => \|mem\| {
192	192	const op = switch (encoding.op_en) {
193	193	.m, .mi, .m1, .mc => .none,
194		.mr => inst.op2,
	194	.mr, .mri, .mrc => inst.op2,
195	195	.rm, .rmi => inst.op1,
196	196	else => unreachable,
197	197	};
...	...	@@ -202,7 +202,7 @@ pub const Instruction = struct {
202	202
203	203	switch (encoding.op_en) {
204	204	.mi => try encodeImm(inst.op2.imm, encoding.op2, encoder),
205		.rmi => try encodeImm(inst.op3.imm, encoding.op3, encoder),
	205	.rmi, .mri => try encodeImm(inst.op3.imm, encoding.op3, encoder),
206	206	else => {},
207	207	}
208	208	},
...	...	@@ -251,7 +251,7 @@ pub const Instruction = struct {
251	251	else => unreachable,
252	252	};
253	253	} else null,
254		.m, .mi, .m1, .mc, .mr => if (inst.op1.isSegmentRegister()) blk: {
	254	.m, .mi, .m1, .mc, .mr, .mri, .mrc => if (inst.op1.isSegmentRegister()) blk: {
255	255	break :blk switch (inst.op1) {
256	256	.reg => \|r\| r,
257	257	.mem => \|m\| m.base().?,
...	...	@@ -275,13 +275,11 @@ pub const Instruction = struct {
275	275
276	276	switch (op_en) {
277	277	.np, .i, .zi, .fd, .td, .d => {},
278		.o, .oi => {
279		rex.b = inst.op1.reg.isExtended();
280		},
281		.m, .mi, .m1, .mc, .mr, .rm, .rmi => {
	278	.o, .oi => rex.b = inst.op1.reg.isExtended(),
	279	.m, .mi, .m1, .mc, .mr, .rm, .rmi, .mri, .mrc => {
282	280	const r_op = switch (op_en) {
283	281	.rm, .rmi => inst.op1,
284		.mr => inst.op2,
	282	.mr, .mri, .mrc => inst.op2,
285	283	else => null,
286	284	};
287	285	if (r_op) \|op\| {
...	...	@@ -290,7 +288,7 @@ pub const Instruction = struct {
290	288
291	289	const b_x_op = switch (op_en) {
292	290	.rm, .rmi => inst.op2,
293		.m, .mi, .m1, .mc, .mr => inst.op1,
	291	.m, .mi, .m1, .mc, .mr, .mri, .mrc => inst.op1,
294	292	else => unreachable,
295	293	};
296	294	switch (b_x_op) {

src/arch/x86_64/encodings.zig+14

...	...	@@ -693,6 +693,13 @@ pub const table = &[_]Entry{
693	693	.{ .shl, .mi, .rm32, .imm8, .none, .none, &.{ 0xc1 }, 4, .none },
694	694	.{ .shl, .mi, .rm64, .imm8, .none, .none, &.{ 0xc1 }, 4, .long },
695	695
	696	.{ .shld, .mri, .rm16, .r16, .imm8, .none, &.{ 0x0f, 0xa4 }, 0, .none },
	697	.{ .shld, .mrc, .rm16, .r16, .cl, .none, &.{ 0x0f, 0xa5 }, 0, .none },
	698	.{ .shld, .mri, .rm32, .r32, .imm8, .none, &.{ 0x0f, 0xa4 }, 0, .none },
	699	.{ .shld, .mri, .rm64, .r64, .imm8, .none, &.{ 0x0f, 0xa4 }, 0, .long },
	700	.{ .shld, .mrc, .rm32, .r32, .cl, .none, &.{ 0x0f, 0xa5 }, 0, .none },
	701	.{ .shld, .mrc, .rm64, .r64, .cl, .none, &.{ 0x0f, 0xa5 }, 0, .long },
	702
696	703	.{ .shr, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 5, .none },
697	704	.{ .shr, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 5, .rex },
698	705	.{ .shr, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 5, .none },
...	...	@@ -709,6 +716,13 @@ pub const table = &[_]Entry{
709	716	.{ .shr, .mi, .rm32, .imm8, .none, .none, &.{ 0xc1 }, 5, .none },
710	717	.{ .shr, .mi, .rm64, .imm8, .none, .none, &.{ 0xc1 }, 5, .long },
711	718
	719	.{ .shrd, .mri, .rm16, .r16, .imm8, .none, &.{ 0x0f, 0xac }, 0, .none },
	720	.{ .shrd, .mrc, .rm16, .r16, .cl, .none, &.{ 0x0f, 0xad }, 0, .none },
	721	.{ .shrd, .mri, .rm32, .r32, .imm8, .none, &.{ 0x0f, 0xac }, 0, .none },
	722	.{ .shrd, .mri, .rm64, .r64, .imm8, .none, &.{ 0x0f, 0xac }, 0, .long },
	723	.{ .shrd, .mrc, .rm32, .r32, .cl, .none, &.{ 0x0f, 0xad }, 0, .none },
	724	.{ .shrd, .mrc, .rm64, .r64, .cl, .none, &.{ 0x0f, 0xad }, 0, .long },
	725
712	726	.{ .stos, .np, .m8, .none, .none, .none, &.{ 0xaa }, 0, .none },
713	727	.{ .stos, .np, .m16, .none, .none, .none, &.{ 0xab }, 0, .none },
714	728	.{ .stos, .np, .m32, .none, .none, .none, &.{ 0xab }, 0, .none },