authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-03-18 05:49:20-04:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2023-03-21 08:49:54+01:00
logedd63f9abaa7dae1786a4cd91cc3031264bd3ef0
tree2259ac328fe3d045d0a5ce318b55b611cff7da81
parentc865c8fb2a910e73e38192a8bd484935bae8f6fc

x86_64: reimplement inline memcpy and memset

7 files changed, 270 insertions(+), 215 deletions(-)

src/arch/x86_64/CodeGen.zig+88-137
......@@ -1286,7 +1286,7 @@ pub fn spillEflagsIfOccupied(self: *Self) !void {
12861286 }
12871287}
12881288
1289pub fn spillRegisters(self: *Self, comptime count: comptime_int, registers: [count]Register) !void {
1289pub fn spillRegisters(self: *Self, registers: []const Register) !void {
12901290 for (registers) |reg| {
12911291 try self.register_manager.getReg(reg, null);
12921292 }
......@@ -1540,7 +1540,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
15401540 break :result try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs);
15411541 }
15421542
1543 try self.spillRegisters(2, .{ .rax, .rdx });
1543 try self.spillRegisters(&.{ .rax, .rdx });
15441544
15451545 const lhs = try self.resolveInst(bin_op.lhs);
15461546 const rhs = try self.resolveInst(bin_op.rhs);
......@@ -1594,7 +1594,7 @@ fn airAddSubShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
15941594 try self.spillEflagsIfOccupied();
15951595
15961596 if (tag == .shl_with_overflow) {
1597 try self.spillRegisters(1, .{.rcx});
1597 try self.spillRegisters(&.{.rcx});
15981598 }
15991599
16001600 const partial: MCValue = switch (tag) {
......@@ -1721,7 +1721,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
17211721 try self.spillEflagsIfOccupied();
17221722 self.eflags_inst = inst;
17231723
1724 try self.spillRegisters(2, .{ .rax, .rdx });
1724 try self.spillRegisters(&.{ .rax, .rdx });
17251725
17261726 const lhs = try self.resolveInst(bin_op.lhs);
17271727 const rhs = try self.resolveInst(bin_op.rhs);
......@@ -1774,7 +1774,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
17741774 break :dst_reg dst_reg;
17751775 },
17761776 .unsigned => {
1777 try self.spillRegisters(2, .{ .rax, .rdx });
1777 try self.spillRegisters(&.{ .rax, .rdx });
17781778
17791779 const lhs = try self.resolveInst(bin_op.lhs);
17801780 const rhs = try self.resolveInst(bin_op.rhs);
......@@ -1888,7 +1888,7 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
18881888 return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
18891889 }
18901890
1891 try self.spillRegisters(1, .{.rcx});
1891 try self.spillRegisters(&.{.rcx});
18921892
18931893 const tag = self.air.instructions.items(.tag)[inst];
18941894 const lhs = try self.resolveInst(bin_op.lhs);
......@@ -2832,6 +2832,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
28322832 .unreach => unreachable,
28332833 .eflags => unreachable,
28342834 .undef => {
2835 if (!self.wantSafety()) return; // The already existing value will do just fine.
28352836 switch (abi_size) {
28362837 1 => try self.store(ptr, .{ .immediate = 0xaa }, ptr_ty, value_ty),
28372838 2 => try self.store(ptr, .{ .immediate = 0xaaaa }, ptr_ty, value_ty),
......@@ -4035,11 +4036,40 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
40354036 defer info.deinit(self);
40364037
40374038 try self.spillEflagsIfOccupied();
4039 try self.spillRegisters(abi.getCallerPreservedRegs(self.target.*));
40384040
4039 for (abi.getCallerPreservedRegs(self.target.*)) |reg| {
4040 try self.register_manager.getReg(reg, null);
4041 // set stack arguments first because this can clobber registers
4042 // also clobber spill arguments as we go
4043 if (info.return_value == .stack_offset) {
4044 try self.spillRegisters(&.{abi.getCAbiIntParamRegs(self.target.*)[0]});
4045 }
4046 for (args, info.args) |arg, mc_arg| {
4047 const arg_ty = self.air.typeOf(arg);
4048 const arg_mcv = try self.resolveInst(arg);
4049 // Here we do not use setRegOrMem even though the logic is similar, because
4050 // the function call will move the stack pointer, so the offsets are different.
4051 switch (mc_arg) {
4052 .none => {},
4053 .register => |reg| try self.spillRegisters(&.{reg}),
4054 .stack_offset => |off| {
4055 // TODO rewrite using `genSetStack`
4056 try self.genSetStackArg(arg_ty, off, arg_mcv);
4057 },
4058 .ptr_stack_offset => {
4059 return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{});
4060 },
4061 .undef => unreachable,
4062 .immediate => unreachable,
4063 .unreach => unreachable,
4064 .dead => unreachable,
4065 .memory => unreachable,
4066 .linker_load => unreachable,
4067 .eflags => unreachable,
4068 .register_overflow => unreachable,
4069 }
40414070 }
40424071
4072 // now we are free to set register arguments
40434073 const ret_reg_lock: ?RegisterLock = blk: {
40444074 if (info.return_value == .stack_offset) {
40454075 const ret_ty = fn_ty.fnReturnType();
......@@ -4049,7 +4079,6 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
40494079 log.debug("airCall: return value on stack at offset {}", .{stack_offset});
40504080
40514081 const ret_reg = abi.getCAbiIntParamRegs(self.target.*)[0];
4052 try self.register_manager.getReg(ret_reg, null);
40534082 try self.genSetReg(Type.usize, ret_reg, .{ .ptr_stack_offset = stack_offset });
40544083 const ret_reg_lock = self.register_manager.lockRegAssumeUnused(ret_reg);
40554084
......@@ -4061,25 +4090,12 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
40614090 };
40624091 defer if (ret_reg_lock) |lock| self.register_manager.unlockReg(lock);
40634092
4064 for (args, info.args) |arg, info_arg| {
4065 const mc_arg = info_arg;
4093 for (args, info.args) |arg, mc_arg| {
40664094 const arg_ty = self.air.typeOf(arg);
40674095 const arg_mcv = try self.resolveInst(arg);
4068 // Here we do not use setRegOrMem even though the logic is similar, because
4069 // the function call will move the stack pointer, so the offsets are different.
40704096 switch (mc_arg) {
4071 .none => continue,
4072 .register => |reg| {
4073 try self.register_manager.getReg(reg, null);
4074 try self.genSetReg(arg_ty, reg, arg_mcv);
4075 },
4076 .stack_offset => |off| {
4077 // TODO rewrite using `genSetStack`
4078 try self.genSetStackArg(arg_ty, off, arg_mcv);
4079 },
4080 .ptr_stack_offset => {
4081 return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{});
4082 },
4097 .none, .stack_offset, .ptr_stack_offset => {},
4098 .register => |reg| try self.genSetReg(arg_ty, reg, arg_mcv),
40834099 .undef => unreachable,
40844100 .immediate => unreachable,
40854101 .unreach => unreachable,
......@@ -5277,6 +5293,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
52775293 .dead => unreachable,
52785294 .unreach, .none => return,
52795295 .undef => {
5296 if (!self.wantSafety()) return; // The already existing value will do just fine.
52805297 if (abi_size <= 8) {
52815298 const reg = try self.copyToTmpRegister(ty, mcv);
52825299 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
......@@ -5384,8 +5401,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
53845401 .dead => unreachable,
53855402 .unreach, .none => return, // Nothing to do.
53865403 .undef => {
5387 if (!self.wantSafety())
5388 return; // The already existing value will do just fine.
5404 if (!self.wantSafety()) return; // The already existing value will do just fine.
53895405 // TODO Upgrade this to a memset call when we have that available.
53905406 switch (abi_size) {
53915407 1, 2, 4 => {
......@@ -5607,19 +5623,14 @@ fn genInlineMemcpy(
56075623 null;
56085624 defer if (dsbase_lock) |lock| self.register_manager.unlockReg(lock);
56095625
5610 const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null }, gp);
5611 const dst_addr_reg = regs[0];
5612 const src_addr_reg = regs[1];
5613 const index_reg = regs[2].to64();
5614 const count_reg = regs[3].to64();
5615 const tmp_reg = regs[4].to8();
5626 try self.spillRegisters(&.{ .rdi, .rsi, .rcx });
56165627
56175628 switch (dst_ptr) {
56185629 .memory, .linker_load => {
5619 try self.loadMemPtrIntoRegister(dst_addr_reg, Type.usize, dst_ptr);
5630 try self.loadMemPtrIntoRegister(.rdi, Type.usize, dst_ptr);
56205631 },
56215632 .ptr_stack_offset, .stack_offset => |off| {
5622 try self.asmRegisterMemory(.lea, dst_addr_reg.to64(), Memory.sib(.qword, .{
5633 try self.asmRegisterMemory(.lea, .rdi, Memory.sib(.qword, .{
56235634 .base = opts.dest_stack_base orelse .rbp,
56245635 .disp = -off,
56255636 }));
......@@ -5627,7 +5638,7 @@ fn genInlineMemcpy(
56275638 .register => |reg| {
56285639 try self.asmRegisterRegister(
56295640 .mov,
5630 registerAlias(dst_addr_reg, @intCast(u32, @divExact(reg.bitSize(), 8))),
5641 registerAlias(.rdi, @intCast(u32, @divExact(reg.bitSize(), 8))),
56315642 reg,
56325643 );
56335644 },
......@@ -5638,10 +5649,10 @@ fn genInlineMemcpy(
56385649
56395650 switch (src_ptr) {
56405651 .memory, .linker_load => {
5641 try self.loadMemPtrIntoRegister(src_addr_reg, Type.usize, src_ptr);
5652 try self.loadMemPtrIntoRegister(.rsi, Type.usize, src_ptr);
56425653 },
56435654 .ptr_stack_offset, .stack_offset => |off| {
5644 try self.asmRegisterMemory(.lea, src_addr_reg.to64(), Memory.sib(.qword, .{
5655 try self.asmRegisterMemory(.lea, .rsi, Memory.sib(.qword, .{
56455656 .base = opts.source_stack_base orelse .rbp,
56465657 .disp = -off,
56475658 }));
......@@ -5649,7 +5660,7 @@ fn genInlineMemcpy(
56495660 .register => |reg| {
56505661 try self.asmRegisterRegister(
56515662 .mov,
5652 registerAlias(src_addr_reg, @intCast(u32, @divExact(reg.bitSize(), 8))),
5663 registerAlias(.rsi, @intCast(u32, @divExact(reg.bitSize(), 8))),
56535664 reg,
56545665 );
56555666 },
......@@ -5658,37 +5669,12 @@ fn genInlineMemcpy(
56585669 },
56595670 }
56605671
5661 try self.genSetReg(Type.usize, count_reg, len);
5662 try self.asmRegisterImmediate(.mov, index_reg, Immediate.u(0));
5663 const loop_start = try self.addInst(.{
5664 .tag = .cmp,
5665 .ops = .ri_u,
5666 .data = .{ .ri = .{
5667 .r1 = count_reg,
5668 .imm = 0,
5669 } },
5672 try self.genSetReg(Type.usize, .rcx, len);
5673 _ = try self.addInst(.{
5674 .tag = .movs,
5675 .ops = .string,
5676 .data = .{ .string = .{ .repeat = .rep, .width = .b } },
56705677 });
5671 const loop_reloc = try self.asmJccReloc(undefined, .e);
5672 try self.asmRegisterMemory(.mov, tmp_reg.to8(), Memory.sib(.byte, .{
5673 .base = src_addr_reg,
5674 .scale_index = .{
5675 .scale = 1,
5676 .index = index_reg,
5677 },
5678 .disp = 0,
5679 }));
5680 try self.asmMemoryRegister(.mov, Memory.sib(.byte, .{
5681 .base = dst_addr_reg,
5682 .scale_index = .{
5683 .scale = 1,
5684 .index = index_reg,
5685 },
5686 .disp = 0,
5687 }), tmp_reg.to8());
5688 try self.asmRegisterImmediate(.add, index_reg, Immediate.u(1));
5689 try self.asmRegisterImmediate(.sub, count_reg, Immediate.u(1));
5690 _ = try self.asmJmpReloc(loop_start);
5691 try self.performReloc(loop_reloc);
56925678}
56935679
56945680fn genInlineMemset(
......@@ -5698,28 +5684,20 @@ fn genInlineMemset(
56985684 len: MCValue,
56995685 opts: InlineMemcpyOpts,
57005686) InnerError!void {
5701 const ssbase_lock: ?RegisterLock = if (opts.source_stack_base) |reg|
5702 self.register_manager.lockReg(reg)
5703 else
5704 null;
5705 defer if (ssbase_lock) |reg| self.register_manager.unlockReg(reg);
5706
57075687 const dsbase_lock: ?RegisterLock = if (opts.dest_stack_base) |reg|
57085688 self.register_manager.lockReg(reg)
57095689 else
57105690 null;
57115691 defer if (dsbase_lock) |lock| self.register_manager.unlockReg(lock);
57125692
5713 const regs = try self.register_manager.allocRegs(2, .{ null, null }, gp);
5714 const addr_reg = regs[0];
5715 const index_reg = regs[1].to64();
5693 try self.spillRegisters(&.{ .rdi, .al, .rcx });
57165694
57175695 switch (dst_ptr) {
57185696 .memory, .linker_load => {
5719 try self.loadMemPtrIntoRegister(addr_reg, Type.usize, dst_ptr);
5697 try self.loadMemPtrIntoRegister(.rdi, Type.usize, dst_ptr);
57205698 },
57215699 .ptr_stack_offset, .stack_offset => |off| {
5722 try self.asmRegisterMemory(.lea, addr_reg.to64(), Memory.sib(.qword, .{
5700 try self.asmRegisterMemory(.lea, .rdi, Memory.sib(.qword, .{
57235701 .base = opts.dest_stack_base orelse .rbp,
57245702 .disp = -off,
57255703 }));
......@@ -5727,48 +5705,22 @@ fn genInlineMemset(
57275705 .register => |reg| {
57285706 try self.asmRegisterRegister(
57295707 .mov,
5730 registerAlias(addr_reg, @intCast(u32, @divExact(reg.bitSize(), 8))),
5708 registerAlias(.rdi, @intCast(u32, @divExact(reg.bitSize(), 8))),
57315709 reg,
57325710 );
57335711 },
57345712 else => {
5735 return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr});
5713 return self.fail("TODO implement memset for setting stack when dest is {}", .{dst_ptr});
57365714 },
57375715 }
57385716
5739 try self.genSetReg(Type.usize, index_reg, len);
5740 try self.genBinOpMir(.sub, Type.usize, .{ .register = index_reg }, .{ .immediate = 1 });
5741
5742 const loop_start = try self.addInst(.{
5743 .tag = .cmp,
5744 .ops = .ri_s,
5745 .data = .{ .ri = .{
5746 .r1 = index_reg,
5747 .imm = @bitCast(u32, @as(i32, -1)),
5748 } },
5717 try self.genSetReg(Type.u8, .al, value);
5718 try self.genSetReg(Type.usize, .rcx, len);
5719 _ = try self.addInst(.{
5720 .tag = .stos,
5721 .ops = .string,
5722 .data = .{ .string = .{ .repeat = .rep, .width = .b } },
57495723 });
5750 const loop_reloc = try self.asmJccReloc(undefined, .e);
5751
5752 switch (value) {
5753 .immediate => |x| {
5754 if (x > math.maxInt(i32)) {
5755 return self.fail("TODO inline memset for value immediate larger than 32bits", .{});
5756 }
5757 try self.asmMemoryImmediate(.mov, Memory.sib(.byte, .{
5758 .base = addr_reg,
5759 .scale_index = .{
5760 .scale = 1,
5761 .index = index_reg,
5762 },
5763 .disp = 0,
5764 }), Immediate.u(@intCast(u8, x)));
5765 },
5766 else => return self.fail("TODO inline memset for value of type {}", .{value}),
5767 }
5768
5769 try self.asmRegisterImmediate(.sub, index_reg, Immediate.u(1));
5770 _ = try self.asmJmpReloc(loop_start);
5771 try self.performReloc(loop_reloc);
57725724}
57735725
57745726fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void {
......@@ -5788,8 +5740,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
57885740 },
57895741 .unreach, .none => return, // Nothing to do.
57905742 .undef => {
5791 if (!self.wantSafety())
5792 return; // The already existing value will do just fine.
5743 if (!self.wantSafety()) return; // The already existing value will do just fine.
57935744 // Write the debug undefined value.
57945745 switch (registerAlias(reg, abi_size).bitSize()) {
57955746 8 => return self.genSetReg(ty, reg, .{ .immediate = 0xaa }),
......@@ -5802,27 +5753,27 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
58025753 .eflags => |cc| {
58035754 return self.asmSetccRegister(reg.to8(), cc);
58045755 },
5805 .immediate => |x| {
5806 if (x == 0) {
5756 .immediate => |imm| {
5757 if (imm == 0) {
58075758 // 32-bit moves zero-extend to 64-bit, so xoring the 32-bit
58085759 // register is the fastest way to zero a register.
5809 return self.asmRegisterRegister(.xor, reg.to32(), reg.to32());
5810 }
5811 if (ty.isSignedInt()) {
5812 const signed_x = @bitCast(i64, x);
5813 if (math.minInt(i32) <= signed_x and signed_x <= math.maxInt(i32)) {
5814 return self.asmRegisterImmediate(
5815 .mov,
5816 registerAlias(reg, abi_size),
5817 Immediate.s(@intCast(i32, signed_x)),
5818 );
5819 }
5760 try self.asmRegisterRegister(.xor, reg.to32(), reg.to32());
5761 } else if (abi_size > 4 and math.cast(u32, imm) != null) {
5762 // 32-bit moves zero-extend to 64-bit.
5763 try self.asmRegisterImmediate(.mov, reg.to32(), Immediate.u(imm));
5764 } else if (abi_size <= 4 and @bitCast(i64, imm) < 0) {
5765 try self.asmRegisterImmediate(
5766 .mov,
5767 registerAlias(reg, abi_size),
5768 Immediate.s(@intCast(i32, @bitCast(i64, imm))),
5769 );
5770 } else {
5771 try self.asmRegisterImmediate(
5772 .mov,
5773 registerAlias(reg, abi_size),
5774 Immediate.u(imm),
5775 );
58205776 }
5821 return self.asmRegisterImmediate(
5822 .mov,
5823 registerAlias(reg, abi_size),
5824 Immediate.u(x),
5825 );
58265777 },
58275778 .register => |src_reg| {
58285779 // If the registers are the same, nothing to do.
......@@ -6136,7 +6087,7 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
61366087
61376088fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void {
61386089 _ = inst;
6139 return self.fail("TODO implement x86 airAtomicRaw", .{});
6090 return self.fail("TODO implement x86 airAtomicRmw", .{});
61406091}
61416092
61426093fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {
......@@ -6177,7 +6128,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index) !void {
61776128
61786129 try self.genInlineMemset(dst_ptr, src_val, len, .{});
61796130
6180 return self.finishAir(inst, .none, .{ pl_op.operand, .none, .none });
6131 return self.finishAir(inst, .none, .{ pl_op.operand, extra.lhs, extra.rhs });
61816132}
61826133
61836134fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
......@@ -6229,7 +6180,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
62296180
62306181 try self.genInlineMemcpy(dst_ptr, src, len, .{});
62316182
6232 return self.finishAir(inst, .none, .{ pl_op.operand, .none, .none });
6183 return self.finishAir(inst, .none, .{ pl_op.operand, extra.lhs, extra.rhs });
62336184}
62346185
62356186fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
src/arch/x86_64/Emit.zig+34-16
......@@ -130,6 +130,13 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
130130 .ucomisd,
131131 => try emit.mirEncodeGeneric(tag, inst),
132132
133 .cmps,
134 .lods,
135 .movs,
136 .scas,
137 .stos,
138 => try emit.mirString(tag, inst),
139
133140 .jmp_reloc => try emit.mirJmpReloc(inst),
134141
135142 .call_extern => try emit.mirCallExtern(inst),
......@@ -183,18 +190,8 @@ fn fixupRelocs(emit: *Emit) InnerError!void {
183190 }
184191}
185192
186fn encode(emit: *Emit, mnemonic: Instruction.Mnemonic, ops: struct {
187 op1: Instruction.Operand = .none,
188 op2: Instruction.Operand = .none,
189 op3: Instruction.Operand = .none,
190 op4: Instruction.Operand = .none,
191}) InnerError!void {
192 const inst = try Instruction.new(mnemonic, .{
193 .op1 = ops.op1,
194 .op2 = ops.op2,
195 .op3 = ops.op3,
196 .op4 = ops.op4,
197 });
193fn encode(emit: *Emit, mnemonic: Instruction.Mnemonic, ops: Instruction.Init) InnerError!void {
194 const inst = try Instruction.new(mnemonic, ops);
198195 return inst.encode(emit.code.writer());
199196}
200197
......@@ -318,6 +315,28 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE
318315 });
319316}
320317
318fn mirString(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
319 const ops = emit.mir.instructions.items(.ops)[inst];
320 switch (ops) {
321 .string => {
322 const data = emit.mir.instructions.items(.data)[inst].string;
323 const mnemonic = switch (tag) {
324 inline .cmps, .lods, .movs, .scas, .stos => |comptime_tag| switch (data.width) {
325 inline else => |comptime_width| @field(
326 Instruction.Mnemonic,
327 @tagName(comptime_tag) ++ @tagName(comptime_width),
328 ),
329 },
330 else => unreachable,
331 };
332 return emit.encode(mnemonic, .{ .prefix = switch (data.repeat) {
333 inline else => |comptime_repeat| @field(Instruction.Prefix, @tagName(comptime_repeat)),
334 } });
335 },
336 else => unreachable,
337 }
338}
339
321340fn mirMovMoffs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
322341 const ops = emit.mir.instructions.items(.ops)[inst];
323342 const payload = emit.mir.instructions.items(.data)[inst].payload;
......@@ -377,10 +396,9 @@ fn mirMovsx(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
377396}
378397
379398fn mnemonicFromConditionCode(comptime basename: []const u8, cc: bits.Condition) Instruction.Mnemonic {
380 inline for (@typeInfo(bits.Condition).Enum.fields) |field| {
381 if (mem.eql(u8, field.name, @tagName(cc)))
382 return @field(Instruction.Mnemonic, basename ++ field.name);
383 } else unreachable;
399 return switch (cc) {
400 inline else => |comptime_cc| @field(Instruction.Mnemonic, basename ++ @tagName(comptime_cc)),
401 };
384402}
385403
386404fn mirCmovcc(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
src/arch/x86_64/Encoding.zig+24-24
......@@ -24,12 +24,7 @@ opc: [7]u8,
2424modrm_ext: u3,
2525mode: Mode,
2626
27pub fn findByMnemonic(mnemonic: Mnemonic, args: struct {
28 op1: Instruction.Operand,
29 op2: Instruction.Operand,
30 op3: Instruction.Operand,
31 op4: Instruction.Operand,
32}) !?Encoding {
27pub fn findByMnemonic(mnemonic: Mnemonic, args: Instruction.Init) !?Encoding {
3328 const input_op1 = Op.fromOperand(args.op1);
3429 const input_op2 = Op.fromOperand(args.op2);
3530 const input_op3 = Op.fromOperand(args.op3);
......@@ -109,17 +104,13 @@ pub fn findByMnemonic(mnemonic: Mnemonic, args: struct {
109104 if (count == 1) return candidates[0];
110105
111106 const EncodingLength = struct {
112 fn estimate(encoding: Encoding, params: struct {
113 op1: Instruction.Operand,
114 op2: Instruction.Operand,
115 op3: Instruction.Operand,
116 op4: Instruction.Operand,
117 }) usize {
107 fn estimate(encoding: Encoding, params: Instruction.Init) usize {
118108 var inst = Instruction{
119109 .op1 = params.op1,
120110 .op2 = params.op2,
121111 .op3 = params.op3,
122112 .op4 = params.op4,
113 .prefix = params.prefix,
123114 .encoding = encoding,
124115 };
125116 var cwriter = std.io.countingWriter(std.io.null_writer);
......@@ -140,12 +131,7 @@ pub fn findByMnemonic(mnemonic: Mnemonic, args: struct {
140131 else => {},
141132 }
142133
143 const len = EncodingLength.estimate(candidate, .{
144 .op1 = args.op1,
145 .op2 = args.op2,
146 .op3 = args.op3,
147 .op4 = args.op4,
148 });
134 const len = EncodingLength.estimate(candidate, args);
149135 const current = shortest_encoding orelse {
150136 shortest_encoding = .{ .index = i, .len = len };
151137 continue;
......@@ -228,7 +214,11 @@ pub fn modRmExt(encoding: Encoding) u3 {
228214}
229215
230216pub fn operandBitSize(encoding: Encoding) u64 {
231 if (encoding.mode == .long) return 64;
217 switch (encoding.mode) {
218 .short => return 16,
219 .long => return 64,
220 else => {},
221 }
232222 const bit_size: u64 = switch (encoding.op_en) {
233223 .np => switch (encoding.op1) {
234224 .o16 => 16,
......@@ -317,10 +307,13 @@ pub const Mnemonic = enum {
317307 // zig fmt: off
318308 // General-purpose
319309 adc, add, @"and",
320 call, cbw, cwde, cdqe, cwd, cdq, cqo, cmp,
310 call, cbw, cdq, cdqe,
321311 cmova, cmovae, cmovb, cmovbe, cmovc, cmove, cmovg, cmovge, cmovl, cmovle, cmovna,
322312 cmovnae, cmovnb, cmovnbe, cmovnc, cmovne, cmovng, cmovnge, cmovnl, cmovnle, cmovno,
323313 cmovnp, cmovns, cmovnz, cmovo, cmovp, cmovpe, cmovpo, cmovs, cmovz,
314 cmp,
315 cmps, cmpsb, cmpsd, cmpsq, cmpsw,
316 cqo, cwd, cwde,
324317 div,
325318 fisttp, fld,
326319 idiv, imul, int3,
......@@ -328,15 +321,21 @@ pub const Mnemonic = enum {
328321 jnc, jne, jng, jnge, jnl, jnle, jno, jnp, jns, jnz, jo, jp, jpe, jpo, js, jz,
329322 jmp,
330323 lea,
331 mov, movsx, movsxd, movzx, mul,
324 lods, lodsb, lodsd, lodsq, lodsw,
325 mov,
326 movs, movsb, movsd, movsq, movsw,
327 movsx, movsxd, movzx, mul,
332328 nop,
333329 @"or",
334330 pop, push,
335331 ret,
336 sal, sar, sbb, shl, shr, sub, syscall,
332 sal, sar, sbb,
333 scas, scasb, scasd, scasq, scasw,
334 shl, shr, sub, syscall,
337335 seta, setae, setb, setbe, setc, sete, setg, setge, setl, setle, setna, setnae,
338336 setnb, setnbe, setnc, setne, setng, setnge, setnl, setnle, setno, setnp, setns,
339337 setnz, seto, setp, setpe, setpo, sets, setz,
338 stos, stosb, stosd, stosq, stosw,
340339 @"test",
341340 ud2,
342341 xor,
......@@ -351,10 +350,10 @@ pub const Mnemonic = enum {
351350 ucomiss,
352351 // SSE2
353352 addsd,
354 cmpsd,
353 //cmpsd,
355354 divsd,
356355 maxsd, minsd,
357 movq, movsd,
356 movq, //movsd,
358357 mulsd,
359358 subsd,
360359 ucomisd,
......@@ -591,6 +590,7 @@ pub const Op = enum {
591590
592591pub const Mode = enum {
593592 none,
593 short,
594594 fpu,
595595 rex,
596596 long,
src/arch/x86_64/Mir.zig+43
......@@ -150,6 +150,17 @@ pub const Inst = struct {
150150 /// Unordered compare scalar double-precision floating-point values
151151 ucomisd,
152152
153 /// Compare string operands
154 cmps,
155 /// Load string
156 lods,
157 /// Move data from string to string
158 movs,
159 /// Scan string
160 scas,
161 /// Store string
162 stos,
163
153164 /// Conditional move
154165 cmovcc,
155166 /// Conditional jump
......@@ -268,6 +279,30 @@ pub const Inst = struct {
268279 /// Memory (RIP), register operands.
269280 /// Uses `rx` payload with extra data of type `MemoryRip`.
270281 mr_rip,
282 /// Single memory (SIB) operand with lock prefix.
283 /// Uses `payload` with extra data of type `MemorySib`.
284 lock_m_sib,
285 /// Single memory (RIP) operand with lock prefix.
286 /// Uses `payload` with extra data of type `MemoryRip`.
287 lock_m_rip,
288 /// Memory (SIB), immediate (unsigned) operands with lock prefix.
289 /// Uses `xi` payload with extra data of type `MemorySib`.
290 lock_mi_u_sib,
291 /// Memory (RIP), immediate (unsigned) operands with lock prefix.
292 /// Uses `xi` payload with extra data of type `MemoryRip`.
293 lock_mi_u_rip,
294 /// Memory (SIB), immediate (sign-extend) operands with lock prefix.
295 /// Uses `xi` payload with extra data of type `MemorySib`.
296 lock_mi_s_sib,
297 /// Memory (RIP), immediate (sign-extend) operands with lock prefix.
298 /// Uses `xi` payload with extra data of type `MemoryRip`.
299 lock_mi_s_rip,
300 /// Memory (SIB), register operands with lock prefix.
301 /// Uses `rx` payload with extra data of type `MemorySib`.
302 lock_mr_sib,
303 /// Memory (RIP), register operands with lock prefix.
304 /// Uses `rx` payload with extra data of type `MemoryRip`.
305 lock_mr_rip,
271306 /// Rax, Memory moffs.
272307 /// Uses `payload` with extra data of type `MemoryMoffs`.
273308 rax_moffs,
......@@ -280,6 +315,9 @@ pub const Inst = struct {
280315 /// References another Mir instruction directly with condition code (CC).
281316 /// Uses `inst_cc` payload.
282317 inst_cc,
318 /// String repeat and width
319 /// Uses `string` payload.
320 string,
283321 /// Uses `reloc` payload.
284322 reloc,
285323 /// Linker relocation - GOT indirection.
......@@ -353,6 +391,11 @@ pub const Inst = struct {
353391 payload: u32,
354392 imm: u32,
355393 },
394 /// String instruction prefix and width.
395 string: struct {
396 repeat: bits.StringRepeat,
397 width: bits.StringWidth,
398 },
356399 /// Relocation for the linker where:
357400 /// * `atom_index` is the index of the source
358401 /// * `sym_index` is the index of the target
src/arch/x86_64/bits.zig+3
......@@ -6,6 +6,9 @@ const Allocator = std.mem.Allocator;
66const ArrayList = std.ArrayList;
77const DW = std.dwarf;
88
9pub const StringRepeat = enum(u3) { none, rep, repe, repz, repne, repnz };
10pub const StringWidth = enum(u2) { b, w, d, q };
11
912/// EFLAGS condition codes
1013pub const Condition = enum(u5) {
1114 /// above
src/arch/x86_64/encoder.zig+33-38
......@@ -15,10 +15,21 @@ pub const Instruction = struct {
1515 op2: Operand = .none,
1616 op3: Operand = .none,
1717 op4: Operand = .none,
18 prefix: Prefix = .none,
1819 encoding: Encoding,
1920
2021 pub const Mnemonic = Encoding.Mnemonic;
2122
23 pub const Prefix = enum(u3) {
24 none,
25 lock,
26 rep,
27 repe,
28 repz,
29 repne,
30 repnz,
31 };
32
2233 pub const Operand = union(enum) {
2334 none,
2435 reg: Register,
......@@ -96,18 +107,16 @@ pub const Instruction = struct {
96107 }
97108 };
98109
99 pub fn new(mnemonic: Mnemonic, args: struct {
110 pub const Init = struct {
111 prefix: Prefix = .none,
100112 op1: Operand = .none,
101113 op2: Operand = .none,
102114 op3: Operand = .none,
103115 op4: Operand = .none,
104 }) !Instruction {
105 const encoding = (try Encoding.findByMnemonic(mnemonic, .{
106 .op1 = args.op1,
107 .op2 = args.op2,
108 .op3 = args.op3,
109 .op4 = args.op4,
110 })) orelse {
116 };
117
118 pub fn new(mnemonic: Mnemonic, args: Init) !Instruction {
119 const encoding = (try Encoding.findByMnemonic(mnemonic, args)) orelse {
111120 log.debug("no encoding found for: {s} {s} {s} {s} {s}", .{
112121 @tagName(mnemonic),
113122 @tagName(Encoding.Op.fromOperand(args.op1)),
......@@ -119,6 +128,7 @@ pub const Instruction = struct {
119128 };
120129 log.debug("selected encoding: {}", .{encoding});
121130 return .{
131 .prefix = args.prefix,
122132 .op1 = args.op1,
123133 .op2 = args.op2,
124134 .op3 = args.op3,
......@@ -128,6 +138,7 @@ pub const Instruction = struct {
128138 }
129139
130140 pub fn fmtPrint(inst: Instruction, writer: anytype) !void {
141 if (inst.prefix != .none) try writer.print("{s} ", .{@tagName(inst.prefix)});
131142 try writer.print("{s}", .{@tagName(inst.encoding.mnemonic)});
132143 const ops = [_]struct { Operand, Encoding.Op }{
133144 .{ inst.op1, inst.encoding.op1 },
......@@ -215,6 +226,14 @@ pub const Instruction = struct {
215226 const op_en = enc.op_en;
216227
217228 var legacy = LegacyPrefixes{};
229
230 switch (inst.prefix) {
231 .none => {},
232 .lock => legacy.prefix_f0 = true,
233 .repne, .repnz => legacy.prefix_f2 = true,
234 .rep, .repe, .repz => legacy.prefix_f3 = true,
235 }
236
218237 if (enc.mode == .none) {
219238 const bit_size = enc.operandBitSize();
220239 if (bit_size == 16) {
......@@ -811,15 +830,11 @@ const TestEncode = struct {
811830 buffer: [32]u8 = undefined,
812831 index: usize = 0,
813832
814 fn encode(enc: *TestEncode, mnemonic: Instruction.Mnemonic, args: struct {
815 op1: Instruction.Operand = .none,
816 op2: Instruction.Operand = .none,
817 op3: Instruction.Operand = .none,
818 op4: Instruction.Operand = .none,
819 }) !void {
833 fn encode(enc: *TestEncode, mnemonic: Instruction.Mnemonic, args: Instruction.Init) !void {
820834 var stream = std.io.fixedBufferStream(&enc.buffer);
821835 var count_writer = std.io.countingWriter(stream.writer());
822836 const inst = try Instruction.new(mnemonic, .{
837 .prefix = args.prefix,
823838 .op1 = args.op1,
824839 .op2 = args.op2,
825840 .op3 = args.op3,
......@@ -1447,18 +1462,8 @@ test "lower NP encoding" {
14471462 try expectEqualHexStrings("\x0f\x05", enc.code(), "syscall");
14481463}
14491464
1450fn invalidInstruction(mnemonic: Instruction.Mnemonic, args: struct {
1451 op1: Instruction.Operand = .none,
1452 op2: Instruction.Operand = .none,
1453 op3: Instruction.Operand = .none,
1454 op4: Instruction.Operand = .none,
1455}) !void {
1456 const err = Instruction.new(mnemonic, .{
1457 .op1 = args.op1,
1458 .op2 = args.op2,
1459 .op3 = args.op3,
1460 .op4 = args.op4,
1461 });
1465fn invalidInstruction(mnemonic: Instruction.Mnemonic, args: Instruction.Init) !void {
1466 const err = Instruction.new(mnemonic, args);
14621467 try testing.expectError(error.InvalidInstruction, err);
14631468}
14641469
......@@ -1479,18 +1484,8 @@ test "invalid instruction" {
14791484 try invalidInstruction(.push, .{ .op1 = .{ .imm = Immediate.u(0x1000000000000000) } });
14801485}
14811486
1482fn cannotEncode(mnemonic: Instruction.Mnemonic, args: struct {
1483 op1: Instruction.Operand = .none,
1484 op2: Instruction.Operand = .none,
1485 op3: Instruction.Operand = .none,
1486 op4: Instruction.Operand = .none,
1487}) !void {
1488 try testing.expectError(error.CannotEncode, Instruction.new(mnemonic, .{
1489 .op1 = args.op1,
1490 .op2 = args.op2,
1491 .op3 = args.op3,
1492 .op4 = args.op4,
1493 }));
1487fn cannotEncode(mnemonic: Instruction.Mnemonic, args: Instruction.Init) !void {
1488 try testing.expectError(error.CannotEncode, Instruction.new(mnemonic, args));
14941489}
14951490
14961491test "cannot encode" {
src/arch/x86_64/encodings.zig+45
......@@ -207,6 +207,15 @@ pub const table = &[_]Entry{
207207 .{ .cmp, .rm, .r32, .rm32, .none, .none, &.{ 0x3b }, 0, .none },
208208 .{ .cmp, .rm, .r64, .rm64, .none, .none, &.{ 0x3b }, 0, .long },
209209
210 .{ .cmps, .np, .m8, .m8, .none, .none, &.{ 0xa6 }, 0, .none },
211 .{ .cmps, .np, .m16, .m16, .none, .none, &.{ 0xa7 }, 0, .none },
212 .{ .cmps, .np, .m32, .m32, .none, .none, &.{ 0xa7 }, 0, .none },
213 .{ .cmps, .np, .m64, .m64, .none, .none, &.{ 0xa7 }, 0, .long },
214 .{ .cmpsb, .np, .none, .none, .none, .none, &.{ 0xa6 }, 0, .none },
215 .{ .cmpsw, .np, .none, .none, .none, .none, &.{ 0xa7 }, 0, .short },
216 .{ .cmpsd, .np, .none, .none, .none, .none, &.{ 0xa7 }, 0, .none },
217 .{ .cmpsq, .np, .none, .none, .none, .none, &.{ 0xa7 }, 0, .long },
218
210219 .{ .div, .m, .rm8, .none, .none, .none, &.{ 0xf6 }, 6, .none },
211220 .{ .div, .m, .rm8, .none, .none, .none, &.{ 0xf6 }, 6, .rex },
212221 .{ .div, .m, .rm16, .none, .none, .none, &.{ 0xf7 }, 6, .none },
......@@ -283,6 +292,15 @@ pub const table = &[_]Entry{
283292 .{ .lea, .rm, .r32, .m, .none, .none, &.{ 0x8d }, 0, .none },
284293 .{ .lea, .rm, .r64, .m, .none, .none, &.{ 0x8d }, 0, .long },
285294
295 .{ .lods, .np, .m8, .none, .none, .none, &.{ 0xac }, 0, .none },
296 .{ .lods, .np, .m16, .none, .none, .none, &.{ 0xad }, 0, .none },
297 .{ .lods, .np, .m32, .none, .none, .none, &.{ 0xad }, 0, .none },
298 .{ .lods, .np, .m64, .none, .none, .none, &.{ 0xad }, 0, .long },
299 .{ .lodsb, .np, .none, .none, .none, .none, &.{ 0xac }, 0, .none },
300 .{ .lodsw, .np, .none, .none, .none, .none, &.{ 0xad }, 0, .short },
301 .{ .lodsd, .np, .none, .none, .none, .none, &.{ 0xad }, 0, .none },
302 .{ .lodsq, .np, .none, .none, .none, .none, &.{ 0xad }, 0, .long },
303
286304 .{ .mov, .mr, .rm8, .r8, .none, .none, &.{ 0x88 }, 0, .none },
287305 .{ .mov, .mr, .rm8, .r8, .none, .none, &.{ 0x88 }, 0, .rex },
288306 .{ .mov, .mr, .rm16, .r16, .none, .none, &.{ 0x89 }, 0, .none },
......@@ -316,6 +334,15 @@ pub const table = &[_]Entry{
316334 .{ .mov, .mi, .rm32, .imm32, .none, .none, &.{ 0xc7 }, 0, .none },
317335 .{ .mov, .mi, .rm64, .imm32s, .none, .none, &.{ 0xc7 }, 0, .long },
318336
337 .{ .movs, .np, .m8, .m8, .none, .none, &.{ 0xa4 }, 0, .none },
338 .{ .movs, .np, .m16, .m16, .none, .none, &.{ 0xa5 }, 0, .none },
339 .{ .movs, .np, .m32, .m32, .none, .none, &.{ 0xa5 }, 0, .none },
340 .{ .movs, .np, .m64, .m64, .none, .none, &.{ 0xa5 }, 0, .long },
341 .{ .movsb, .np, .none, .none, .none, .none, &.{ 0xa4 }, 0, .none },
342 .{ .movsw, .np, .none, .none, .none, .none, &.{ 0xa5 }, 0, .short },
343 .{ .movsd, .np, .none, .none, .none, .none, &.{ 0xa5 }, 0, .none },
344 .{ .movsq, .np, .none, .none, .none, .none, &.{ 0xa5 }, 0, .long },
345
319346 .{ .movsx, .rm, .r16, .rm8, .none, .none, &.{ 0x0f, 0xbe }, 0, .none },
320347 .{ .movsx, .rm, .r16, .rm8, .none, .none, &.{ 0x0f, 0xbe }, 0, .rex },
321348 .{ .movsx, .rm, .r32, .rm8, .none, .none, &.{ 0x0f, 0xbe }, 0, .none },
......@@ -435,6 +462,15 @@ pub const table = &[_]Entry{
435462 .{ .sbb, .rm, .r32, .rm32, .none, .none, &.{ 0x1b }, 0, .none },
436463 .{ .sbb, .rm, .r64, .rm64, .none, .none, &.{ 0x1b }, 0, .long },
437464
465 .{ .scas, .np, .m8, .none, .none, .none, &.{ 0xae }, 0, .none },
466 .{ .scas, .np, .m16, .none, .none, .none, &.{ 0xaf }, 0, .none },
467 .{ .scas, .np, .m32, .none, .none, .none, &.{ 0xaf }, 0, .none },
468 .{ .scas, .np, .m64, .none, .none, .none, &.{ 0xaf }, 0, .long },
469 .{ .scasb, .np, .none, .none, .none, .none, &.{ 0xae }, 0, .none },
470 .{ .scasw, .np, .none, .none, .none, .none, &.{ 0xaf }, 0, .short },
471 .{ .scasd, .np, .none, .none, .none, .none, &.{ 0xaf }, 0, .none },
472 .{ .scasq, .np, .none, .none, .none, .none, &.{ 0xaf }, 0, .long },
473
438474 .{ .seta, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x97 }, 0, .none },
439475 .{ .seta, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x97 }, 0, .rex },
440476 .{ .setae, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x93 }, 0, .none },
......@@ -528,6 +564,15 @@ pub const table = &[_]Entry{
528564 .{ .shr, .mi, .rm32, .imm8, .none, .none, &.{ 0xc1 }, 5, .none },
529565 .{ .shr, .mi, .rm64, .imm8, .none, .none, &.{ 0xc1 }, 5, .long },
530566
567 .{ .stos, .np, .m8, .none, .none, .none, &.{ 0xaa }, 0, .none },
568 .{ .stos, .np, .m16, .none, .none, .none, &.{ 0xab }, 0, .none },
569 .{ .stos, .np, .m32, .none, .none, .none, &.{ 0xab }, 0, .none },
570 .{ .stos, .np, .m64, .none, .none, .none, &.{ 0xab }, 0, .long },
571 .{ .stosb, .np, .none, .none, .none, .none, &.{ 0xaa }, 0, .none },
572 .{ .stosw, .np, .none, .none, .none, .none, &.{ 0xab }, 0, .short },
573 .{ .stosd, .np, .none, .none, .none, .none, &.{ 0xab }, 0, .none },
574 .{ .stosq, .np, .none, .none, .none, .none, &.{ 0xab }, 0, .long },
575
531576 .{ .sub, .zi, .al, .imm8, .none, .none, &.{ 0x2c }, 0, .none },
532577 .{ .sub, .zi, .ax, .imm16, .none, .none, &.{ 0x2d }, 0, .none },
533578 .{ .sub, .zi, .eax, .imm32, .none, .none, &.{ 0x2d }, 0, .none },