authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-01-15 22:27:56+01:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-01-15 22:27:56+01:00
log3901b6fb000b909e90347f6910ea6315ee03e220
tree2ff3fac79d429e6442c755717905a7a79a38f910
parente23f7c01ee91321bd7845397d45f189a3e161806
parent3af4e28dda7e4537255516fd1d575c2a34d904e7
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #10598 from ziglang/stage2-x86_64-imm-sizes

stage2: rename Isel back to Emit, and fix immediate casts in Emit for x86 64

6 files changed, 2191 insertions(+), 2141 deletions(-)

CMakeLists.txt+1-1
...@@ -578,7 +578,7 @@ set(ZIG_STAGE2_SOURCES...@@ -578,7 +578,7 @@ set(ZIG_STAGE2_SOURCES
578 "${CMAKE_SOURCE_DIR}/src/arch/wasm/Emit.zig"578 "${CMAKE_SOURCE_DIR}/src/arch/wasm/Emit.zig"
579 "${CMAKE_SOURCE_DIR}/src/arch/wasm/Mir.zig"579 "${CMAKE_SOURCE_DIR}/src/arch/wasm/Mir.zig"
580 "${CMAKE_SOURCE_DIR}/src/arch/x86_64/CodeGen.zig"580 "${CMAKE_SOURCE_DIR}/src/arch/x86_64/CodeGen.zig"
581 "${CMAKE_SOURCE_DIR}/src/arch/x86_64/Isel.zig"581 "${CMAKE_SOURCE_DIR}/src/arch/x86_64/Emit.zig"
582 "${CMAKE_SOURCE_DIR}/src/arch/x86_64/Mir.zig"582 "${CMAKE_SOURCE_DIR}/src/arch/x86_64/Mir.zig"
583 "${CMAKE_SOURCE_DIR}/src/arch/x86_64/bits.zig"583 "${CMAKE_SOURCE_DIR}/src/arch/x86_64/bits.zig"
584 "${CMAKE_SOURCE_DIR}/src/clang.zig"584 "${CMAKE_SOURCE_DIR}/src/clang.zig"
src/arch/x86_64/CodeGen.zig+53-49
...@@ -17,7 +17,7 @@ const DW = std.dwarf;...@@ -17,7 +17,7 @@ const DW = std.dwarf;
17const ErrorMsg = Module.ErrorMsg;17const ErrorMsg = Module.ErrorMsg;
18const FnResult = @import("../../codegen.zig").FnResult;18const FnResult = @import("../../codegen.zig").FnResult;
19const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError;19const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError;
20const Isel = @import("Isel.zig");20const Emit = @import("Emit.zig");
21const Liveness = @import("../../Liveness.zig");21const Liveness = @import("../../Liveness.zig");
22const Mir = @import("Mir.zig");22const Mir = @import("Mir.zig");
23const Module = @import("../../Module.zig");23const Module = @import("../../Module.zig");
...@@ -309,7 +309,7 @@ pub fn generate(...@@ -309,7 +309,7 @@ pub fn generate(
309 };309 };
310 defer mir.deinit(bin_file.allocator);310 defer mir.deinit(bin_file.allocator);
311311
312 var isel = Isel{312 var emit = Emit{
313 .mir = mir,313 .mir = mir,
314 .bin_file = bin_file,314 .bin_file = bin_file,
315 .debug_output = debug_output,315 .debug_output = debug_output,
...@@ -320,9 +320,9 @@ pub fn generate(...@@ -320,9 +320,9 @@ pub fn generate(
320 .prev_di_line = module_fn.lbrace_line,320 .prev_di_line = module_fn.lbrace_line,
321 .prev_di_column = module_fn.lbrace_column,321 .prev_di_column = module_fn.lbrace_column,
322 };322 };
323 defer isel.deinit();323 defer emit.deinit();
324 isel.lowerMir() catch |err| switch (err) {324 emit.lowerMir() catch |err| switch (err) {
325 error.IselFail => return FnResult{ .fail = isel.err_msg.? },325 error.EmitFail => return FnResult{ .fail = emit.err_msg.? },
326 else => |e| return e,326 else => |e| return e,
327 };327 };
328328
...@@ -497,14 +497,14 @@ fn gen(self: *Self) InnerError!void {...@@ -497,14 +497,14 @@ fn gen(self: *Self) InnerError!void {
497 .ops = (Mir.Ops{497 .ops = (Mir.Ops{
498 .reg1 = .rsp,498 .reg1 = .rsp,
499 }).encode(),499 }).encode(),
500 .data = .{ .imm = @intCast(i32, aligned_stack_end) + stack_adjustment },500 .data = .{ .imm = @bitCast(u32, @intCast(i32, aligned_stack_end) + stack_adjustment) },
501 });501 });
502 self.mir_instructions.set(backpatch_stack_add, .{502 self.mir_instructions.set(backpatch_stack_add, .{
503 .tag = .add,503 .tag = .add,
504 .ops = (Mir.Ops{504 .ops = (Mir.Ops{
505 .reg1 = .rsp,505 .reg1 = .rsp,
506 }).encode(),506 }).encode(),
507 .data = .{ .imm = @intCast(i32, aligned_stack_end) + stack_adjustment },507 .data = .{ .imm = @bitCast(u32, @intCast(i32, aligned_stack_end) + stack_adjustment) },
508 });508 });
509 }509 }
510 } else {510 } else {
...@@ -1347,7 +1347,7 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -1347,7 +1347,7 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
1347 .reg2 = .rbp,1347 .reg2 = .rbp,
1348 .flags = 0b01,1348 .flags = 0b01,
1349 }).encode(),1349 }).encode(),
1350 .data = .{ .imm = -@intCast(i32, off + 16) },1350 .data = .{ .imm = @bitCast(u32, -@intCast(i32, off + 16)) },
1351 });1351 });
1352 // add addr, offset1352 // add addr, offset
1353 _ = try self.addInst(.{1353 _ = try self.addInst(.{
...@@ -1555,7 +1555,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo...@@ -1555,7 +1555,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
1555 try self.genSetReg(Type.initTag(.u32), count_reg, .{ .immediate = @intCast(u32, abi_size) });1555 try self.genSetReg(Type.initTag(.u32), count_reg, .{ .immediate = @intCast(u32, abi_size) });
15561556
1557 return self.genInlineMemcpy(1557 return self.genInlineMemcpy(
1558 -@intCast(i32, off + abi_size),1558 @bitCast(u32, -@intCast(i32, off + abi_size)),
1559 registerAlias(addr_reg, @divExact(reg.size(), 8)),1559 registerAlias(addr_reg, @divExact(reg.size(), 8)),
1560 count_reg.to64(),1560 count_reg.to64(),
1561 tmp_reg.to8(),1561 tmp_reg.to8(),
...@@ -1637,7 +1637,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type...@@ -1637,7 +1637,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
1637 // introduce new MIR tag specifically for mov [reg + 0], imm1637 // introduce new MIR tag specifically for mov [reg + 0], imm
1638 const payload = try self.addExtra(Mir.ImmPair{1638 const payload = try self.addExtra(Mir.ImmPair{
1639 .dest_off = 0,1639 .dest_off = 0,
1640 .operand = @bitCast(i32, @intCast(u32, imm)),1640 .operand = @truncate(u32, imm),
1641 });1641 });
1642 _ = try self.addInst(.{1642 _ = try self.addInst(.{
1643 .tag = .mov_mem_imm,1643 .tag = .mov_mem_imm,
...@@ -1821,11 +1821,11 @@ fn genBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs:...@@ -1821,11 +1821,11 @@ fn genBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs:
1821 const dst_ty = self.air.typeOfIndex(inst);1821 const dst_ty = self.air.typeOfIndex(inst);
1822 const air_tags = self.air.instructions.items(.tag);1822 const air_tags = self.air.instructions.items(.tag);
1823 switch (air_tags[inst]) {1823 switch (air_tags[inst]) {
1824 .add, .addwrap => try self.genBinMathOpMir(.add, dst_ty, dst_mcv, src_mcv),1824 .add, .addwrap => try self.genBinMathOpMir(.add, dst_ty, .unsigned, dst_mcv, src_mcv),
1825 .bool_or, .bit_or => try self.genBinMathOpMir(.@"or", dst_ty, dst_mcv, src_mcv),1825 .bool_or, .bit_or => try self.genBinMathOpMir(.@"or", dst_ty, .unsigned, dst_mcv, src_mcv),
1826 .bool_and, .bit_and => try self.genBinMathOpMir(.@"and", dst_ty, dst_mcv, src_mcv),1826 .bool_and, .bit_and => try self.genBinMathOpMir(.@"and", dst_ty, .unsigned, dst_mcv, src_mcv),
1827 .sub, .subwrap => try self.genBinMathOpMir(.sub, dst_ty, dst_mcv, src_mcv),1827 .sub, .subwrap => try self.genBinMathOpMir(.sub, dst_ty, .unsigned, dst_mcv, src_mcv),
1828 .xor, .not => try self.genBinMathOpMir(.xor, dst_ty, dst_mcv, src_mcv),1828 .xor, .not => try self.genBinMathOpMir(.xor, dst_ty, .unsigned, dst_mcv, src_mcv),
1829 .mul, .mulwrap => try self.genIMulOpMir(dst_ty, dst_mcv, src_mcv),1829 .mul, .mulwrap => try self.genIMulOpMir(dst_ty, dst_mcv, src_mcv),
1830 else => unreachable,1830 else => unreachable,
1831 }1831 }
...@@ -1837,6 +1837,7 @@ fn genBinMathOpMir(...@@ -1837,6 +1837,7 @@ fn genBinMathOpMir(
1837 self: *Self,1837 self: *Self,
1838 mir_tag: Mir.Inst.Tag,1838 mir_tag: Mir.Inst.Tag,
1839 dst_ty: Type,1839 dst_ty: Type,
1840 signedness: std.builtin.Signedness,
1840 dst_mcv: MCValue,1841 dst_mcv: MCValue,
1841 src_mcv: MCValue,1842 src_mcv: MCValue,
1842) !void {1843) !void {
...@@ -1856,11 +1857,16 @@ fn genBinMathOpMir(...@@ -1856,11 +1857,16 @@ fn genBinMathOpMir(
1856 .ptr_stack_offset => unreachable,1857 .ptr_stack_offset => unreachable,
1857 .ptr_embedded_in_code => unreachable,1858 .ptr_embedded_in_code => unreachable,
1858 .register => |src_reg| {1859 .register => |src_reg| {
1860 // TODO think more carefully about this: is this actually correct?
1861 const reg_size = if (mir_tag == .cmp and signedness == .signed)
1862 @divExact(dst_reg.size(), 8)
1863 else
1864 @divExact(src_reg.size(), 8);
1859 _ = try self.addInst(.{1865 _ = try self.addInst(.{
1860 .tag = mir_tag,1866 .tag = mir_tag,
1861 .ops = (Mir.Ops{1867 .ops = (Mir.Ops{
1862 .reg1 = registerAlias(dst_reg, @divExact(src_reg.size(), 8)),1868 .reg1 = registerAlias(dst_reg, reg_size),
1863 .reg2 = src_reg,1869 .reg2 = registerAlias(src_reg, reg_size),
1864 }).encode(),1870 }).encode(),
1865 .data = undefined,1871 .data = undefined,
1866 });1872 });
...@@ -1872,7 +1878,7 @@ fn genBinMathOpMir(...@@ -1872,7 +1878,7 @@ fn genBinMathOpMir(
1872 .ops = (Mir.Ops{1878 .ops = (Mir.Ops{
1873 .reg1 = registerAlias(dst_reg, @intCast(u32, abi_size)),1879 .reg1 = registerAlias(dst_reg, @intCast(u32, abi_size)),
1874 }).encode(),1880 }).encode(),
1875 .data = .{ .imm = @intCast(i32, imm) },1881 .data = .{ .imm = @truncate(u32, imm) },
1876 });1882 });
1877 },1883 },
1878 .embedded_in_code, .memory => {1884 .embedded_in_code, .memory => {
...@@ -1891,7 +1897,7 @@ fn genBinMathOpMir(...@@ -1891,7 +1897,7 @@ fn genBinMathOpMir(
1891 .reg2 = .rbp,1897 .reg2 = .rbp,
1892 .flags = 0b01,1898 .flags = 0b01,
1893 }).encode(),1899 }).encode(),
1894 .data = .{ .imm = -@intCast(i32, adj_off) },1900 .data = .{ .imm = @bitCast(u32, -@intCast(i32, adj_off)) },
1895 });1901 });
1896 },1902 },
1897 .compare_flags_unsigned => {1903 .compare_flags_unsigned => {
...@@ -1926,7 +1932,7 @@ fn genBinMathOpMir(...@@ -1926,7 +1932,7 @@ fn genBinMathOpMir(
1926 .reg2 = registerAlias(src_reg, @intCast(u32, abi_size)),1932 .reg2 = registerAlias(src_reg, @intCast(u32, abi_size)),
1927 .flags = 0b10,1933 .flags = 0b10,
1928 }).encode(),1934 }).encode(),
1929 .data = .{ .imm = -@intCast(i32, adj_off) },1935 .data = .{ .imm = @bitCast(u32, -@intCast(i32, adj_off)) },
1930 });1936 });
1931 },1937 },
1932 .immediate => |imm| {1938 .immediate => |imm| {
...@@ -1947,8 +1953,8 @@ fn genBinMathOpMir(...@@ -1947,8 +1953,8 @@ fn genBinMathOpMir(
1947 else => unreachable,1953 else => unreachable,
1948 };1954 };
1949 const payload = try self.addExtra(Mir.ImmPair{1955 const payload = try self.addExtra(Mir.ImmPair{
1950 .dest_off = -@intCast(i32, adj_off),1956 .dest_off = @bitCast(u32, -@intCast(i32, adj_off)),
1951 .operand = @bitCast(i32, @intCast(u32, imm)),1957 .operand = @truncate(u32, imm),
1952 });1958 });
1953 _ = try self.addInst(.{1959 _ = try self.addInst(.{
1954 .tag = tag,1960 .tag = tag,
...@@ -2015,7 +2021,7 @@ fn genIMulOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !...@@ -2015,7 +2021,7 @@ fn genIMulOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !
2015 .reg2 = dst_reg.to32(),2021 .reg2 = dst_reg.to32(),
2016 .flags = 0b10,2022 .flags = 0b10,
2017 }).encode(),2023 }).encode(),
2018 .data = .{ .imm = @intCast(i32, imm) },2024 .data = .{ .imm = @truncate(u32, imm) },
2019 });2025 });
2020 } else {2026 } else {
2021 // TODO verify we don't spill and assign to the same register as dst_mcv2027 // TODO verify we don't spill and assign to the same register as dst_mcv
...@@ -2088,7 +2094,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {...@@ -2088,7 +2094,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
2088 const mcv = self.args[arg_index];2094 const mcv = self.args[arg_index];
2089 const payload = try self.addExtra(Mir.ArgDbgInfo{2095 const payload = try self.addExtra(Mir.ArgDbgInfo{
2090 .air_inst = inst,2096 .air_inst = inst,
2091 .arg_index = @intCast(u32, arg_index), // TODO can arg_index: u32?2097 .arg_index = @truncate(u32, arg_index), // TODO can arg_index: u32?
2092 });2098 });
2093 _ = try self.addInst(.{2099 _ = try self.addInst(.{
2094 .tag = .arg_dbg_info,2100 .tag = .arg_dbg_info,
...@@ -2196,7 +2202,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index) !void {...@@ -2196,7 +2202,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index) !void {
2196 .ops = (Mir.Ops{2202 .ops = (Mir.Ops{
2197 .flags = 0b01,2203 .flags = 0b01,
2198 }).encode(),2204 }).encode(),
2199 .data = .{ .imm = @bitCast(i32, got_addr) },2205 .data = .{ .imm = @truncate(u32, got_addr) },
2200 });2206 });
2201 } else if (func_value.castTag(.extern_fn)) |_| {2207 } else if (func_value.castTag(.extern_fn)) |_| {
2202 return self.fail("TODO implement calling extern functions", .{});2208 return self.fail("TODO implement calling extern functions", .{});
...@@ -2446,7 +2452,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {...@@ -2446,7 +2452,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
2446 // This instruction supports only signed 32-bit immediates at most.2452 // This instruction supports only signed 32-bit immediates at most.
2447 const src_mcv = try self.limitImmediateType(bin_op.rhs, i32);2453 const src_mcv = try self.limitImmediateType(bin_op.rhs, i32);
24482454
2449 try self.genBinMathOpMir(.cmp, ty, dst_mcv, src_mcv);2455 try self.genBinMathOpMir(.cmp, ty, signedness, dst_mcv, src_mcv);
2450 break :result switch (signedness) {2456 break :result switch (signedness) {
2451 .signed => MCValue{ .compare_flags_signed = op },2457 .signed => MCValue{ .compare_flags_signed = op },
2452 .unsigned => MCValue{ .compare_flags_unsigned = op },2458 .unsigned => MCValue{ .compare_flags_unsigned = op },
...@@ -2669,7 +2675,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {...@@ -2669,7 +2675,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
2669}2675}
26702676
2671fn isNull(self: *Self, ty: Type, operand: MCValue) !MCValue {2677fn isNull(self: *Self, ty: Type, operand: MCValue) !MCValue {
2672 try self.genBinMathOpMir(.cmp, ty, operand, MCValue{ .immediate = 0 });2678 try self.genBinMathOpMir(.cmp, ty, .unsigned, operand, MCValue{ .immediate = 0 });
2673 return MCValue{ .compare_flags_unsigned = .eq };2679 return MCValue{ .compare_flags_unsigned = .eq };
2674}2680}
26752681
...@@ -2686,7 +2692,7 @@ fn isErr(self: *Self, ty: Type, operand: MCValue) !MCValue {...@@ -2686,7 +2692,7 @@ fn isErr(self: *Self, ty: Type, operand: MCValue) !MCValue {
2686 return MCValue{ .immediate = 0 }; // always false2692 return MCValue{ .immediate = 0 }; // always false
2687 } else if (!payload_type.hasCodeGenBits()) {2693 } else if (!payload_type.hasCodeGenBits()) {
2688 if (err_type.abiSize(self.target.*) <= 8) {2694 if (err_type.abiSize(self.target.*) <= 8) {
2689 try self.genBinMathOpMir(.cmp, err_type, operand, MCValue{ .immediate = 0 });2695 try self.genBinMathOpMir(.cmp, err_type, .unsigned, operand, MCValue{ .immediate = 0 });
2690 return MCValue{ .compare_flags_unsigned = .gt };2696 return MCValue{ .compare_flags_unsigned = .gt };
2691 } else {2697 } else {
2692 return self.fail("TODO isErr for errors with size larger than register size", .{});2698 return self.fail("TODO isErr for errors with size larger than register size", .{});
...@@ -3121,8 +3127,8 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro...@@ -3121,8 +3127,8 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
3121 // offset from rbp, which is at the top of the stack frame.3127 // offset from rbp, which is at the top of the stack frame.
3122 // mov [rbp+offset], immediate3128 // mov [rbp+offset], immediate
3123 const payload = try self.addExtra(Mir.ImmPair{3129 const payload = try self.addExtra(Mir.ImmPair{
3124 .dest_off = -@intCast(i32, adj_off),3130 .dest_off = @bitCast(u32, -@intCast(i32, adj_off)),
3125 .operand = @bitCast(i32, @intCast(u32, x_big)),3131 .operand = @truncate(u32, x_big),
3126 });3132 });
3127 _ = try self.addInst(.{3133 _ = try self.addInst(.{
3128 .tag = .mov_mem_imm,3134 .tag = .mov_mem_imm,
...@@ -3147,8 +3153,8 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro...@@ -3147,8 +3153,8 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
3147 // insted just use two 32 bit writes to avoid register allocation3153 // insted just use two 32 bit writes to avoid register allocation
3148 {3154 {
3149 const payload = try self.addExtra(Mir.ImmPair{3155 const payload = try self.addExtra(Mir.ImmPair{
3150 .dest_off = negative_offset + 4,3156 .dest_off = @bitCast(u32, negative_offset + 4),
3151 .operand = @bitCast(i32, @truncate(u32, x_big >> 32)),3157 .operand = @truncate(u32, x_big >> 32),
3152 });3158 });
3153 _ = try self.addInst(.{3159 _ = try self.addInst(.{
3154 .tag = .mov_mem_imm,3160 .tag = .mov_mem_imm,
...@@ -3161,8 +3167,8 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro...@@ -3161,8 +3167,8 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
3161 }3167 }
3162 {3168 {
3163 const payload = try self.addExtra(Mir.ImmPair{3169 const payload = try self.addExtra(Mir.ImmPair{
3164 .dest_off = negative_offset,3170 .dest_off = @bitCast(u32, negative_offset),
3165 .operand = @bitCast(i32, @truncate(u32, x_big)),3171 .operand = @truncate(u32, x_big),
3166 });3172 });
3167 _ = try self.addInst(.{3173 _ = try self.addInst(.{
3168 .tag = .mov_mem_imm,3174 .tag = .mov_mem_imm,
...@@ -3192,7 +3198,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro...@@ -3192,7 +3198,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
3192 .reg2 = registerAlias(reg, @intCast(u32, abi_size)),3198 .reg2 = registerAlias(reg, @intCast(u32, abi_size)),
3193 .flags = 0b10,3199 .flags = 0b10,
3194 }).encode(),3200 }).encode(),
3195 .data = .{ .imm = -@intCast(i32, adj_off) },3201 .data = .{ .imm = @bitCast(u32, -@intCast(i32, adj_off)) },
3196 });3202 });
3197 },3203 },
3198 .memory, .embedded_in_code => {3204 .memory, .embedded_in_code => {
...@@ -3228,14 +3234,14 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro...@@ -3228,14 +3234,14 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
3228 .reg1 = addr_reg.to64(),3234 .reg1 = addr_reg.to64(),
3229 .reg2 = .rbp,3235 .reg2 = .rbp,
3230 }).encode(),3236 }).encode(),
3231 .data = .{ .imm = -@intCast(i32, off + abi_size) },3237 .data = .{ .imm = @bitCast(u32, -@intCast(i32, off + abi_size)) },
3232 });3238 });
32333239
3234 // TODO allow for abi_size to be u643240 // TODO allow for abi_size to be u64
3235 try self.genSetReg(Type.initTag(.u32), count_reg, .{ .immediate = @intCast(u32, abi_size) });3241 try self.genSetReg(Type.initTag(.u32), count_reg, .{ .immediate = @intCast(u32, abi_size) });
32363242
3237 return self.genInlineMemcpy(3243 return self.genInlineMemcpy(
3238 -@intCast(i32, stack_offset + abi_size),3244 @bitCast(u32, -@intCast(i32, stack_offset + abi_size)),
3239 addr_reg.to64(),3245 addr_reg.to64(),
3240 count_reg.to64(),3246 count_reg.to64(),
3241 tmp_reg.to8(),3247 tmp_reg.to8(),
...@@ -3246,7 +3252,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro...@@ -3246,7 +3252,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
32463252
3247fn genInlineMemcpy(3253fn genInlineMemcpy(
3248 self: *Self,3254 self: *Self,
3249 stack_offset: i32,3255 stack_offset: u32,
3250 addr_reg: Register,3256 addr_reg: Register,
3251 count_reg: Register,3257 count_reg: Register,
3252 tmp_reg: Register,3258 tmp_reg: Register,
...@@ -3361,7 +3367,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void...@@ -3361,7 +3367,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
3361 .reg1 = registerAlias(reg, @intCast(u32, ptr_abi_size)),3367 .reg1 = registerAlias(reg, @intCast(u32, ptr_abi_size)),
3362 .reg2 = .rbp,3368 .reg2 = .rbp,
3363 }).encode(),3369 }).encode(),
3364 .data = .{ .imm = -@intCast(i32, off) },3370 .data = .{ .imm = @bitCast(u32, -@intCast(i32, off)) },
3365 });3371 });
3366 },3372 },
3367 .ptr_embedded_in_code => unreachable,3373 .ptr_embedded_in_code => unreachable,
...@@ -3378,7 +3384,9 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void...@@ -3378,7 +3384,9 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
3378 else => unreachable,3384 else => unreachable,
3379 }3385 }
3380 },3386 },
3381 .compare_flags_unsigned => |op| {3387 .compare_flags_unsigned,
3388 .compare_flags_signed,
3389 => |op| {
3382 const tag: Mir.Inst.Tag = switch (op) {3390 const tag: Mir.Inst.Tag = switch (op) {
3383 .gte, .gt, .lt, .lte => .cond_set_byte_above_below,3391 .gte, .gt, .lt, .lte => .cond_set_byte_above_below,
3384 .eq, .neq => .cond_set_byte_eq_ne,3392 .eq, .neq => .cond_set_byte_eq_ne,
...@@ -3400,10 +3408,6 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void...@@ -3400,10 +3408,6 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
3400 .data = undefined,3408 .data = undefined,
3401 });3409 });
3402 },3410 },
3403 .compare_flags_signed => |op| {
3404 _ = op;
3405 return self.fail("TODO set register with compare flags value (signed)", .{});
3406 },
3407 .immediate => |x| {3411 .immediate => |x| {
3408 // 32-bit moves zero-extend to 64-bit, so xoring the 32-bit3412 // 32-bit moves zero-extend to 64-bit, so xoring the 32-bit
3409 // register is the fastest way to zero a register.3413 // register is the fastest way to zero a register.
...@@ -3426,7 +3430,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void...@@ -3426,7 +3430,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
3426 .ops = (Mir.Ops{3430 .ops = (Mir.Ops{
3427 .reg1 = registerAlias(reg, @intCast(u32, abi_size)),3431 .reg1 = registerAlias(reg, @intCast(u32, abi_size)),
3428 }).encode(),3432 }).encode(),
3429 .data = .{ .imm = @intCast(i32, x) },3433 .data = .{ .imm = @truncate(u32, x) },
3430 });3434 });
3431 return;3435 return;
3432 }3436 }
...@@ -3441,7 +3445,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void...@@ -3441,7 +3445,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
3441 _ = try self.addInst(.{3445 _ = try self.addInst(.{
3442 .tag = .movabs,3446 .tag = .movabs,
3443 .ops = (Mir.Ops{3447 .ops = (Mir.Ops{
3444 .reg1 = reg,3448 .reg1 = reg.to64(),
3445 }).encode(),3449 }).encode(),
3446 .data = .{ .payload = payload },3450 .data = .{ .payload = payload },
3447 });3451 });
...@@ -3482,7 +3486,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void...@@ -3482,7 +3486,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
3482 .reg1 = reg,3486 .reg1 = reg,
3483 .flags = 0b10,3487 .flags = 0b10,
3484 }).encode(),3488 }).encode(),
3485 .data = .{ .got_entry = @intCast(u32, x) },3489 .data = .{ .got_entry = @truncate(u32, x) },
3486 });3490 });
3487 // MOV reg, [reg]3491 // MOV reg, [reg]
3488 _ = try self.addInst(.{3492 _ = try self.addInst(.{
...@@ -3502,7 +3506,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void...@@ -3502,7 +3506,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
3502 .reg1 = reg,3506 .reg1 = reg,
3503 .flags = 0b01,3507 .flags = 0b01,
3504 }).encode(),3508 }).encode(),
3505 .data = .{ .imm = @intCast(i32, x) },3509 .data = .{ .imm = @truncate(u32, x) },
3506 });3510 });
3507 } else {3511 } else {
3508 // If this is RAX, we can use a direct load.3512 // If this is RAX, we can use a direct load.
...@@ -3561,7 +3565,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void...@@ -3561,7 +3565,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
3561 .reg2 = .rbp,3565 .reg2 = .rbp,
3562 .flags = 0b01,3566 .flags = 0b01,
3563 }).encode(),3567 }).encode(),
3564 .data = .{ .imm = -@intCast(i32, off) },3568 .data = .{ .imm = @bitCast(u32, -@intCast(i32, off)) },
3565 });3569 });
3566 },3570 },
3567 }3571 }
src/arch/x86_64/Emit.zig created+2070
...@@ -0,0 +1,2070 @@
1//! This file contains the functionality for lowering x86_64 MIR into
2//! machine code
3
4const Emit = @This();
5
6const std = @import("std");
7const assert = std.debug.assert;
8const bits = @import("bits.zig");
9const leb128 = std.leb;
10const link = @import("../../link.zig");
11const log = std.log.scoped(.codegen);
12const math = std.math;
13const mem = std.mem;
14const testing = std.testing;
15
16const Air = @import("../../Air.zig");
17const Allocator = mem.Allocator;
18const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;
19const DW = std.dwarf;
20const Encoder = bits.Encoder;
21const ErrorMsg = Module.ErrorMsg;
22const MCValue = @import("CodeGen.zig").MCValue;
23const Mir = @import("Mir.zig");
24const Module = @import("../../Module.zig");
25const Instruction = bits.Instruction;
26const Register = bits.Register;
27const Type = @import("../../type.zig").Type;
28
29mir: Mir,
30bin_file: *link.File,
31debug_output: DebugInfoOutput,
32target: *const std.Target,
33err_msg: ?*ErrorMsg = null,
34src_loc: Module.SrcLoc,
35code: *std.ArrayList(u8),
36
37prev_di_line: u32,
38prev_di_column: u32,
39/// Relative to the beginning of `code`.
40prev_di_pc: usize,
41
42code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .{},
43relocs: std.ArrayListUnmanaged(Reloc) = .{},
44
45const InnerError = error{
46 OutOfMemory,
47 EmitFail,
48};
49
50const Reloc = struct {
51 /// Offset of the instruction.
52 source: usize,
53 /// Target of the relocation.
54 target: Mir.Inst.Index,
55 /// Offset of the relocation within the instruction.
56 offset: usize,
57 /// Length of the instruction.
58 length: u5,
59};
60
61pub fn lowerMir(emit: *Emit) InnerError!void {
62 const mir_tags = emit.mir.instructions.items(.tag);
63
64 for (mir_tags) |tag, index| {
65 const inst = @intCast(u32, index);
66 try emit.code_offset_mapping.putNoClobber(emit.bin_file.allocator, inst, emit.code.items.len);
67 switch (tag) {
68 .adc => try emit.mirArith(.adc, inst),
69 .add => try emit.mirArith(.add, inst),
70 .sub => try emit.mirArith(.sub, inst),
71 .xor => try emit.mirArith(.xor, inst),
72 .@"and" => try emit.mirArith(.@"and", inst),
73 .@"or" => try emit.mirArith(.@"or", inst),
74 .sbb => try emit.mirArith(.sbb, inst),
75 .cmp => try emit.mirArith(.cmp, inst),
76 .mov => try emit.mirArith(.mov, inst),
77
78 .adc_mem_imm => try emit.mirArithMemImm(.adc, inst),
79 .add_mem_imm => try emit.mirArithMemImm(.add, inst),
80 .sub_mem_imm => try emit.mirArithMemImm(.sub, inst),
81 .xor_mem_imm => try emit.mirArithMemImm(.xor, inst),
82 .and_mem_imm => try emit.mirArithMemImm(.@"and", inst),
83 .or_mem_imm => try emit.mirArithMemImm(.@"or", inst),
84 .sbb_mem_imm => try emit.mirArithMemImm(.sbb, inst),
85 .cmp_mem_imm => try emit.mirArithMemImm(.cmp, inst),
86 .mov_mem_imm => try emit.mirArithMemImm(.mov, inst),
87
88 .adc_scale_src => try emit.mirArithScaleSrc(.adc, inst),
89 .add_scale_src => try emit.mirArithScaleSrc(.add, inst),
90 .sub_scale_src => try emit.mirArithScaleSrc(.sub, inst),
91 .xor_scale_src => try emit.mirArithScaleSrc(.xor, inst),
92 .and_scale_src => try emit.mirArithScaleSrc(.@"and", inst),
93 .or_scale_src => try emit.mirArithScaleSrc(.@"or", inst),
94 .sbb_scale_src => try emit.mirArithScaleSrc(.sbb, inst),
95 .cmp_scale_src => try emit.mirArithScaleSrc(.cmp, inst),
96 .mov_scale_src => try emit.mirArithScaleSrc(.mov, inst),
97
98 .adc_scale_dst => try emit.mirArithScaleDst(.adc, inst),
99 .add_scale_dst => try emit.mirArithScaleDst(.add, inst),
100 .sub_scale_dst => try emit.mirArithScaleDst(.sub, inst),
101 .xor_scale_dst => try emit.mirArithScaleDst(.xor, inst),
102 .and_scale_dst => try emit.mirArithScaleDst(.@"and", inst),
103 .or_scale_dst => try emit.mirArithScaleDst(.@"or", inst),
104 .sbb_scale_dst => try emit.mirArithScaleDst(.sbb, inst),
105 .cmp_scale_dst => try emit.mirArithScaleDst(.cmp, inst),
106 .mov_scale_dst => try emit.mirArithScaleDst(.mov, inst),
107
108 .adc_scale_imm => try emit.mirArithScaleImm(.adc, inst),
109 .add_scale_imm => try emit.mirArithScaleImm(.add, inst),
110 .sub_scale_imm => try emit.mirArithScaleImm(.sub, inst),
111 .xor_scale_imm => try emit.mirArithScaleImm(.xor, inst),
112 .and_scale_imm => try emit.mirArithScaleImm(.@"and", inst),
113 .or_scale_imm => try emit.mirArithScaleImm(.@"or", inst),
114 .sbb_scale_imm => try emit.mirArithScaleImm(.sbb, inst),
115 .cmp_scale_imm => try emit.mirArithScaleImm(.cmp, inst),
116 .mov_scale_imm => try emit.mirArithScaleImm(.mov, inst),
117
118 .movabs => try emit.mirMovabs(inst),
119
120 .lea => try emit.mirLea(inst),
121
122 .imul_complex => try emit.mirIMulComplex(inst),
123
124 .push => try emit.mirPushPop(.push, inst),
125 .pop => try emit.mirPushPop(.pop, inst),
126
127 .jmp => try emit.mirJmpCall(.jmp_near, inst),
128 .call => try emit.mirJmpCall(.call_near, inst),
129
130 .cond_jmp_greater_less,
131 .cond_jmp_above_below,
132 .cond_jmp_eq_ne,
133 => try emit.mirCondJmp(tag, inst),
134
135 .cond_set_byte_greater_less,
136 .cond_set_byte_above_below,
137 .cond_set_byte_eq_ne,
138 => try emit.mirCondSetByte(tag, inst),
139
140 .ret => try emit.mirRet(inst),
141
142 .syscall => try emit.mirSyscall(),
143
144 .@"test" => try emit.mirTest(inst),
145
146 .brk => try emit.mirBrk(),
147 .nop => try emit.mirNop(),
148
149 .call_extern => try emit.mirCallExtern(inst),
150
151 .dbg_line => try emit.mirDbgLine(inst),
152 .dbg_prologue_end => try emit.mirDbgPrologueEnd(inst),
153 .dbg_epilogue_begin => try emit.mirDbgEpilogueBegin(inst),
154 .arg_dbg_info => try emit.mirArgDbgInfo(inst),
155
156 .push_regs_from_callee_preserved_regs => try emit.mirPushPopRegsFromCalleePreservedRegs(.push, inst),
157 .pop_regs_from_callee_preserved_regs => try emit.mirPushPopRegsFromCalleePreservedRegs(.pop, inst),
158
159 else => {
160 return emit.fail("Implement MIR->Emit lowering for x86_64 for pseudo-inst: {s}", .{tag});
161 },
162 }
163 }
164
165 try emit.fixupRelocs();
166}
167
168pub fn deinit(emit: *Emit) void {
169 emit.relocs.deinit(emit.bin_file.allocator);
170 emit.code_offset_mapping.deinit(emit.bin_file.allocator);
171 emit.* = undefined;
172}
173
174fn fail(emit: *Emit, comptime format: []const u8, args: anytype) InnerError {
175 @setCold(true);
176 assert(emit.err_msg == null);
177 emit.err_msg = try ErrorMsg.create(emit.bin_file.allocator, emit.src_loc, format, args);
178 return error.EmitFail;
179}
180
181fn failWithLoweringError(emit: *Emit, err: LoweringError) InnerError {
182 return switch (err) {
183 error.RaxOperandExpected => emit.fail("Register.rax expected as destination operand", .{}),
184 error.OperandSizeMismatch => emit.fail("operand size mismatch", .{}),
185 else => |e| e,
186 };
187}
188
189fn fixupRelocs(emit: *Emit) InnerError!void {
190 // TODO this function currently assumes all relocs via JMP/CALL instructions are 32bit in size.
191 // This should be reversed like it is done in aarch64 MIR emit code: start with the smallest
192 // possible resolution, i.e., 8bit, and iteratively converge on the minimum required resolution
193 // until the entire decl is correctly emitted with all JMP/CALL instructions within range.
194 for (emit.relocs.items) |reloc| {
195 const target = emit.code_offset_mapping.get(reloc.target) orelse
196 return emit.fail("JMP/CALL relocation target not found!", .{});
197 const disp = @intCast(i32, @intCast(i64, target) - @intCast(i64, reloc.source + reloc.length));
198 mem.writeIntLittle(i32, emit.code.items[reloc.offset..][0..4], disp);
199 }
200}
201
202fn mirBrk(emit: *Emit) InnerError!void {
203 return lowerToZoEnc(.brk, emit.code) catch |err| emit.failWithLoweringError(err);
204}
205
206fn mirNop(emit: *Emit) InnerError!void {
207 return lowerToZoEnc(.nop, emit.code) catch |err| emit.failWithLoweringError(err);
208}
209
210fn mirSyscall(emit: *Emit) InnerError!void {
211 return lowerToZoEnc(.syscall, emit.code) catch |err| emit.failWithLoweringError(err);
212}
213
214fn mirPushPop(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
215 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
216 switch (ops.flags) {
217 0b00 => {
218 // PUSH/POP reg
219 return lowerToOEnc(tag, ops.reg1, emit.code) catch |err| emit.failWithLoweringError(err);
220 },
221 0b01 => {
222 // PUSH/POP r/m64
223 const imm = emit.mir.instructions.items(.data)[inst].imm;
224 const ptr_size: Memory.PtrSize = switch (immOpSize(imm)) {
225 16 => .word_ptr,
226 else => .qword_ptr,
227 };
228 return lowerToMEnc(tag, RegisterOrMemory.mem(ptr_size, .{
229 .disp = imm,
230 .base = ops.reg1,
231 }), emit.code) catch |err| emit.failWithLoweringError(err);
232 },
233 0b10 => {
234 // PUSH imm32
235 assert(tag == .push);
236 const imm = emit.mir.instructions.items(.data)[inst].imm;
237 return lowerToIEnc(.push, imm, emit.code) catch |err|
238 emit.failWithLoweringError(err);
239 },
240 0b11 => unreachable,
241 }
242}
243fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
244 const callee_preserved_regs = bits.callee_preserved_regs;
245 const regs = emit.mir.instructions.items(.data)[inst].regs_to_push_or_pop;
246 if (tag == .push) {
247 for (callee_preserved_regs) |reg, i| {
248 if ((regs >> @intCast(u5, i)) & 1 == 0) continue;
249 lowerToOEnc(.push, reg, emit.code) catch |err|
250 return emit.failWithLoweringError(err);
251 }
252 } else {
253 // pop in the reverse direction
254 var i = callee_preserved_regs.len;
255 while (i > 0) : (i -= 1) {
256 const reg = callee_preserved_regs[i - 1];
257 if ((regs >> @intCast(u5, i - 1)) & 1 == 0) continue;
258 lowerToOEnc(.pop, reg, emit.code) catch |err|
259 return emit.failWithLoweringError(err);
260 }
261 }
262}
263
264fn mirJmpCall(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
265 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
266 switch (ops.flags) {
267 0b00 => {
268 const target = emit.mir.instructions.items(.data)[inst].inst;
269 const source = emit.code.items.len;
270 lowerToDEnc(tag, 0, emit.code) catch |err|
271 return emit.failWithLoweringError(err);
272 try emit.relocs.append(emit.bin_file.allocator, .{
273 .source = source,
274 .target = target,
275 .offset = emit.code.items.len - 4,
276 .length = 5,
277 });
278 },
279 0b01 => {
280 if (ops.reg1 == .none) {
281 // JMP/CALL [imm]
282 const imm = emit.mir.instructions.items(.data)[inst].imm;
283 const ptr_size: Memory.PtrSize = switch (immOpSize(imm)) {
284 16 => .word_ptr,
285 else => .qword_ptr,
286 };
287 return lowerToMEnc(tag, RegisterOrMemory.mem(ptr_size, .{ .disp = imm }), emit.code) catch |err|
288 emit.failWithLoweringError(err);
289 }
290 // JMP/CALL reg
291 return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code) catch |err| emit.failWithLoweringError(err);
292 },
293 0b10 => {
294 // JMP/CALL r/m64
295 const imm = emit.mir.instructions.items(.data)[inst].imm;
296 return lowerToMEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
297 .disp = imm,
298 .base = ops.reg1,
299 }), emit.code) catch |err| emit.failWithLoweringError(err);
300 },
301 0b11 => return emit.fail("TODO unused JMP/CALL variant 0b11", .{}),
302 }
303}
304
305fn mirCondJmp(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
306 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
307 const target = emit.mir.instructions.items(.data)[inst].inst;
308 const tag = switch (mir_tag) {
309 .cond_jmp_greater_less => switch (ops.flags) {
310 0b00 => Tag.jge,
311 0b01 => Tag.jg,
312 0b10 => Tag.jl,
313 0b11 => Tag.jle,
314 },
315 .cond_jmp_above_below => switch (ops.flags) {
316 0b00 => Tag.jae,
317 0b01 => Tag.ja,
318 0b10 => Tag.jb,
319 0b11 => Tag.jbe,
320 },
321 .cond_jmp_eq_ne => switch (@truncate(u1, ops.flags)) {
322 0b0 => Tag.jne,
323 0b1 => Tag.je,
324 },
325 else => unreachable,
326 };
327 const source = emit.code.items.len;
328 lowerToDEnc(tag, 0, emit.code) catch |err|
329 return emit.failWithLoweringError(err);
330 try emit.relocs.append(emit.bin_file.allocator, .{
331 .source = source,
332 .target = target,
333 .offset = emit.code.items.len - 4,
334 .length = 6,
335 });
336}
337
338fn mirCondSetByte(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
339 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
340 const tag = switch (mir_tag) {
341 .cond_set_byte_greater_less => switch (ops.flags) {
342 0b00 => Tag.setge,
343 0b01 => Tag.setg,
344 0b10 => Tag.setl,
345 0b11 => Tag.setle,
346 },
347 .cond_set_byte_above_below => switch (ops.flags) {
348 0b00 => Tag.setae,
349 0b01 => Tag.seta,
350 0b10 => Tag.setb,
351 0b11 => Tag.setbe,
352 },
353 .cond_set_byte_eq_ne => switch (@truncate(u1, ops.flags)) {
354 0b0 => Tag.setne,
355 0b1 => Tag.sete,
356 },
357 else => unreachable,
358 };
359 return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1.to8()), emit.code) catch |err|
360 emit.failWithLoweringError(err);
361}
362
363fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
364 const tag = emit.mir.instructions.items(.tag)[inst];
365 assert(tag == .@"test");
366 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
367 switch (ops.flags) {
368 0b00 => {
369 if (ops.reg2 == .none) {
370 // TEST r/m64, imm32
371 // MI
372 const imm = emit.mir.instructions.items(.data)[inst].imm;
373 if (ops.reg1.to64() == .rax) {
374 // TEST rax, imm32
375 // I
376 return lowerToIEnc(.@"test", imm, emit.code) catch |err|
377 emit.failWithLoweringError(err);
378 }
379 return lowerToMiEnc(.@"test", RegisterOrMemory.reg(ops.reg1), imm, emit.code) catch |err|
380 emit.failWithLoweringError(err);
381 }
382 // TEST r/m64, r64
383 return emit.fail("TODO TEST r/m64, r64", .{});
384 },
385 else => return emit.fail("TODO more TEST alternatives", .{}),
386 }
387}
388
389fn mirRet(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
390 const tag = emit.mir.instructions.items(.tag)[inst];
391 assert(tag == .ret);
392 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
393 switch (ops.flags) {
394 0b00 => {
395 // RETF imm16
396 // I
397 const imm = emit.mir.instructions.items(.data)[inst].imm;
398 return lowerToIEnc(.ret_far, imm, emit.code) catch |err| emit.failWithLoweringError(err);
399 },
400 0b01 => {
401 return lowerToZoEnc(.ret_far, emit.code) catch |err| emit.failWithLoweringError(err);
402 },
403 0b10 => {
404 // RET imm16
405 // I
406 const imm = emit.mir.instructions.items(.data)[inst].imm;
407 return lowerToIEnc(.ret_near, imm, emit.code) catch |err| emit.failWithLoweringError(err);
408 },
409 0b11 => {
410 return lowerToZoEnc(.ret_near, emit.code) catch |err| emit.failWithLoweringError(err);
411 },
412 }
413}
414
415fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
416 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
417 switch (ops.flags) {
418 0b00 => {
419 if (ops.reg2 == .none) {
420 // mov reg1, imm32
421 // MI
422 const imm = emit.mir.instructions.items(.data)[inst].imm;
423 return lowerToMiEnc(tag, RegisterOrMemory.reg(ops.reg1), imm, emit.code) catch |err|
424 emit.failWithLoweringError(err);
425 }
426 // mov reg1, reg2
427 // RM
428 return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code) catch |err|
429 emit.failWithLoweringError(err);
430 },
431 0b01 => {
432 // mov reg1, [reg2 + imm32]
433 // RM
434 const imm = emit.mir.instructions.items(.data)[inst].imm;
435 const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
436 return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
437 .disp = imm,
438 .base = src_reg,
439 }), emit.code) catch |err| emit.failWithLoweringError(err);
440 },
441 0b10 => {
442 if (ops.reg2 == .none) {
443 return emit.fail("TODO unused variant: mov reg1, none, 0b10", .{});
444 }
445 // mov [reg1 + imm32], reg2
446 // MR
447 const imm = emit.mir.instructions.items(.data)[inst].imm;
448 return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg2.size()), .{
449 .disp = imm,
450 .base = ops.reg1,
451 }), ops.reg2, emit.code) catch |err| emit.failWithLoweringError(err);
452 },
453 0b11 => {
454 return emit.fail("TODO unused variant: mov reg1, reg2, 0b11", .{});
455 },
456 }
457}
458
459fn mirArithMemImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
460 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
461 assert(ops.reg2 == .none);
462 const payload = emit.mir.instructions.items(.data)[inst].payload;
463 const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
464 const ptr_size: Memory.PtrSize = switch (ops.flags) {
465 0b00 => .byte_ptr,
466 0b01 => .word_ptr,
467 0b10 => .dword_ptr,
468 0b11 => .qword_ptr,
469 };
470 return lowerToMiEnc(tag, RegisterOrMemory.mem(ptr_size, .{
471 .disp = imm_pair.dest_off,
472 .base = ops.reg1,
473 }), imm_pair.operand, emit.code) catch |err| emit.failWithLoweringError(err);
474}
475
476inline fn setRexWRegister(reg: Register) bool {
477 if (reg.size() == 64) return true;
478 return switch (reg) {
479 .ah, .bh, .ch, .dh => true,
480 else => false,
481 };
482}
483
484inline fn immOpSize(u_imm: u32) u8 {
485 const imm = @bitCast(i32, u_imm);
486 if (math.minInt(i8) <= imm and imm <= math.maxInt(i8)) {
487 return 8;
488 }
489 if (math.minInt(i16) <= imm and imm <= math.maxInt(i16)) {
490 return 16;
491 }
492 return 32;
493}
494
495fn mirArithScaleSrc(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
496 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
497 const scale = ops.flags;
498 const imm = emit.mir.instructions.items(.data)[inst].imm;
499 // OP reg1, [reg2 + scale*rcx + imm32]
500 const scale_index = ScaleIndex{
501 .scale = scale,
502 .index = .rcx,
503 };
504 return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
505 .disp = imm,
506 .base = ops.reg2,
507 .scale_index = scale_index,
508 }), emit.code) catch |err| emit.failWithLoweringError(err);
509}
510
511fn mirArithScaleDst(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
512 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
513 const scale = ops.flags;
514 const imm = emit.mir.instructions.items(.data)[inst].imm;
515 const scale_index = ScaleIndex{
516 .scale = scale,
517 .index = .rax,
518 };
519 if (ops.reg2 == .none) {
520 // OP qword ptr [reg1 + scale*rax + 0], imm32
521 return lowerToMiEnc(tag, RegisterOrMemory.mem(.qword_ptr, .{
522 .disp = 0,
523 .base = ops.reg1,
524 .scale_index = scale_index,
525 }), imm, emit.code) catch |err| emit.failWithLoweringError(err);
526 }
527 // OP [reg1 + scale*rax + imm32], reg2
528 return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg2.size()), .{
529 .disp = imm,
530 .base = ops.reg1,
531 .scale_index = scale_index,
532 }), ops.reg2, emit.code) catch |err| emit.failWithLoweringError(err);
533}
534
535fn mirArithScaleImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
536 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
537 const scale = ops.flags;
538 const payload = emit.mir.instructions.items(.data)[inst].payload;
539 const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data;
540 const scale_index = ScaleIndex{
541 .scale = scale,
542 .index = .rax,
543 };
544 // OP qword ptr [reg1 + scale*rax + imm32], imm32
545 return lowerToMiEnc(tag, RegisterOrMemory.mem(.qword_ptr, .{
546 .disp = imm_pair.dest_off,
547 .base = ops.reg1,
548 .scale_index = scale_index,
549 }), imm_pair.operand, emit.code) catch |err| emit.failWithLoweringError(err);
550}
551
552fn mirMovabs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
553 const tag = emit.mir.instructions.items(.tag)[inst];
554 assert(tag == .movabs);
555 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
556 const imm: u64 = if (ops.reg1.size() == 64) blk: {
557 const payload = emit.mir.instructions.items(.data)[inst].payload;
558 const imm = emit.mir.extraData(Mir.Imm64, payload).data;
559 break :blk imm.decode();
560 } else emit.mir.instructions.items(.data)[inst].imm;
561 if (ops.flags == 0b00) {
562 // movabs reg, imm64
563 // OI
564 return lowerToOiEnc(.mov, ops.reg1, imm, emit.code) catch |err| emit.failWithLoweringError(err);
565 }
566 if (ops.reg1 == .none) {
567 // movabs moffs64, rax
568 // TD
569 return lowerToTdEnc(.mov, imm, ops.reg2, emit.code) catch |err| emit.failWithLoweringError(err);
570 }
571 // movabs rax, moffs64
572 // FD
573 return lowerToFdEnc(.mov, ops.reg1, imm, emit.code) catch |err| emit.failWithLoweringError(err);
574}
575
576fn mirIMulComplex(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
577 const tag = emit.mir.instructions.items(.tag)[inst];
578 assert(tag == .imul_complex);
579 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
580 switch (ops.flags) {
581 0b00 => {
582 return lowerToRmEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code) catch |err|
583 emit.failWithLoweringError(err);
584 },
585 0b10 => {
586 const imm = emit.mir.instructions.items(.data)[inst].imm;
587 return lowerToRmiEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), imm, emit.code) catch |err|
588 emit.failWithLoweringError(err);
589 },
590 else => return emit.fail("TODO implement imul", .{}),
591 }
592}
593
594fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
595 const tag = emit.mir.instructions.items(.tag)[inst];
596 assert(tag == .lea);
597 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
598 switch (ops.flags) {
599 0b00 => {
600 // lea reg1, [reg2 + imm32]
601 // RM
602 const imm = emit.mir.instructions.items(.data)[inst].imm;
603 const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
604 return lowerToRmEnc(
605 .lea,
606 ops.reg1,
607 RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
608 .disp = imm,
609 .base = src_reg,
610 }),
611 emit.code,
612 ) catch |err| emit.failWithLoweringError(err);
613 },
614 0b01 => {
615 // lea reg1, [rip + imm32]
616 // RM
617 const start_offset = emit.code.items.len;
618 lowerToRmEnc(
619 .lea,
620 ops.reg1,
621 RegisterOrMemory.rip(Memory.PtrSize.fromBits(ops.reg1.size()), 0),
622 emit.code,
623 ) catch |err| return emit.failWithLoweringError(err);
624 const end_offset = emit.code.items.len;
625 // Backpatch the displacement
626 const payload = emit.mir.instructions.items(.data)[inst].payload;
627 const imm = emit.mir.extraData(Mir.Imm64, payload).data.decode();
628 const disp = @intCast(i32, @intCast(i64, imm) - @intCast(i64, end_offset - start_offset));
629 mem.writeIntLittle(i32, emit.code.items[end_offset - 4 ..][0..4], disp);
630 },
631 0b10 => {
632 // lea reg1, [rip + reloc]
633 // RM
634 lowerToRmEnc(
635 .lea,
636 ops.reg1,
637 RegisterOrMemory.rip(Memory.PtrSize.fromBits(ops.reg1.size()), 0),
638 emit.code,
639 ) catch |err| return emit.failWithLoweringError(err);
640 const end_offset = emit.code.items.len;
641 const got_entry = emit.mir.instructions.items(.data)[inst].got_entry;
642 if (emit.bin_file.cast(link.File.MachO)) |macho_file| {
643 // TODO I think the reloc might be in the wrong place.
644 const decl = macho_file.active_decl.?;
645 try decl.link.macho.relocs.append(emit.bin_file.allocator, .{
646 .offset = @intCast(u32, end_offset - 4),
647 .target = .{ .local = got_entry },
648 .addend = 0,
649 .subtractor = null,
650 .pcrel = true,
651 .length = 2,
652 .@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_GOT),
653 });
654 } else {
655 return emit.fail(
656 "TODO implement lea reg, [rip + reloc] for linking backends different than MachO",
657 .{},
658 );
659 }
660 },
661 0b11 => {
662 // lea reg, [rbp + rcx + imm32]
663 const imm = emit.mir.instructions.items(.data)[inst].imm;
664 const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
665 const scale_index = ScaleIndex{
666 .scale = 0,
667 .index = .rcx,
668 };
669 return lowerToRmEnc(
670 .lea,
671 ops.reg1,
672 RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
673 .disp = imm,
674 .base = src_reg,
675 .scale_index = scale_index,
676 }),
677 emit.code,
678 ) catch |err| emit.failWithLoweringError(err);
679 },
680 }
681}
682
683fn mirCallExtern(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
684 const tag = emit.mir.instructions.items(.tag)[inst];
685 assert(tag == .call_extern);
686 const n_strx = emit.mir.instructions.items(.data)[inst].extern_fn;
687 const offset = blk: {
688 // callq
689 lowerToDEnc(.call_near, 0, emit.code) catch |err|
690 return emit.failWithLoweringError(err);
691 break :blk @intCast(u32, emit.code.items.len) - 4;
692 };
693 if (emit.bin_file.cast(link.File.MachO)) |macho_file| {
694 // Add relocation to the decl.
695 try macho_file.active_decl.?.link.macho.relocs.append(emit.bin_file.allocator, .{
696 .offset = offset,
697 .target = .{ .global = n_strx },
698 .addend = 0,
699 .subtractor = null,
700 .pcrel = true,
701 .length = 2,
702 .@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_BRANCH),
703 });
704 } else {
705 return emit.fail("TODO implement call_extern for linking backends different than MachO", .{});
706 }
707}
708
709fn mirDbgLine(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
710 const tag = emit.mir.instructions.items(.tag)[inst];
711 assert(tag == .dbg_line);
712 const payload = emit.mir.instructions.items(.data)[inst].payload;
713 const dbg_line_column = emit.mir.extraData(Mir.DbgLineColumn, payload).data;
714 try emit.dbgAdvancePCAndLine(dbg_line_column.line, dbg_line_column.column);
715}
716
717fn dbgAdvancePCAndLine(emit: *Emit, line: u32, column: u32) InnerError!void {
718 const delta_line = @intCast(i32, line) - @intCast(i32, emit.prev_di_line);
719 const delta_pc: usize = emit.code.items.len - emit.prev_di_pc;
720 switch (emit.debug_output) {
721 .dwarf => |dbg_out| {
722 // TODO Look into using the DWARF special opcodes to compress this data.
723 // It lets you emit single-byte opcodes that add different numbers to
724 // both the PC and the line number at the same time.
725 try dbg_out.dbg_line.ensureUnusedCapacity(11);
726 dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_pc);
727 leb128.writeULEB128(dbg_out.dbg_line.writer(), delta_pc) catch unreachable;
728 if (delta_line != 0) {
729 dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_line);
730 leb128.writeILEB128(dbg_out.dbg_line.writer(), delta_line) catch unreachable;
731 }
732 dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.copy);
733 emit.prev_di_pc = emit.code.items.len;
734 emit.prev_di_line = line;
735 emit.prev_di_column = column;
736 emit.prev_di_pc = emit.code.items.len;
737 },
738 .plan9 => |dbg_out| {
739 if (delta_pc <= 0) return; // only do this when the pc changes
740 // we have already checked the target in the linker to make sure it is compatable
741 const quant = @import("../../link/Plan9/aout.zig").getPCQuant(emit.target.cpu.arch) catch unreachable;
742
743 // increasing the line number
744 try @import("../../link/Plan9.zig").changeLine(dbg_out.dbg_line, delta_line);
745 // increasing the pc
746 const d_pc_p9 = @intCast(i64, delta_pc) - quant;
747 if (d_pc_p9 > 0) {
748 // minus one because if its the last one, we want to leave space to change the line which is one quanta
749 var diff = @divExact(d_pc_p9, quant) - quant;
750 while (diff > 0) {
751 if (diff < 64) {
752 try dbg_out.dbg_line.append(@intCast(u8, diff + 128));
753 diff = 0;
754 } else {
755 try dbg_out.dbg_line.append(@intCast(u8, 64 + 128));
756 diff -= 64;
757 }
758 }
759 if (dbg_out.pcop_change_index.*) |pci|
760 dbg_out.dbg_line.items[pci] += 1;
761 dbg_out.pcop_change_index.* = @intCast(u32, dbg_out.dbg_line.items.len - 1);
762 } else if (d_pc_p9 == 0) {
763 // we don't need to do anything, because adding the quant does it for us
764 } else unreachable;
765 if (dbg_out.start_line.* == null)
766 dbg_out.start_line.* = emit.prev_di_line;
767 dbg_out.end_line.* = line;
768 // only do this if the pc changed
769 emit.prev_di_line = line;
770 emit.prev_di_column = column;
771 emit.prev_di_pc = emit.code.items.len;
772 },
773 .none => {},
774 }
775}
776
777fn mirDbgPrologueEnd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
778 const tag = emit.mir.instructions.items(.tag)[inst];
779 assert(tag == .dbg_prologue_end);
780 switch (emit.debug_output) {
781 .dwarf => |dbg_out| {
782 try dbg_out.dbg_line.append(DW.LNS.set_prologue_end);
783 try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
784 },
785 .plan9 => {},
786 .none => {},
787 }
788}
789
790fn mirDbgEpilogueBegin(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
791 const tag = emit.mir.instructions.items(.tag)[inst];
792 assert(tag == .dbg_epilogue_begin);
793 switch (emit.debug_output) {
794 .dwarf => |dbg_out| {
795 try dbg_out.dbg_line.append(DW.LNS.set_epilogue_begin);
796 try emit.dbgAdvancePCAndLine(emit.prev_di_line, emit.prev_di_column);
797 },
798 .plan9 => {},
799 .none => {},
800 }
801}
802
803fn mirArgDbgInfo(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
804 const tag = emit.mir.instructions.items(.tag)[inst];
805 assert(tag == .arg_dbg_info);
806 const payload = emit.mir.instructions.items(.data)[inst].payload;
807 const arg_dbg_info = emit.mir.extraData(Mir.ArgDbgInfo, payload).data;
808 const mcv = emit.mir.function.args[arg_dbg_info.arg_index];
809 try emit.genArgDbgInfo(arg_dbg_info.air_inst, mcv);
810}
811
812fn genArgDbgInfo(emit: *Emit, inst: Air.Inst.Index, mcv: MCValue) !void {
813 const ty_str = emit.mir.function.air.instructions.items(.data)[inst].ty_str;
814 const zir = &emit.mir.function.mod_fn.owner_decl.getFileScope().zir;
815 const name = zir.nullTerminatedString(ty_str.str);
816 const name_with_null = name.ptr[0 .. name.len + 1];
817 const ty = emit.mir.function.air.getRefType(ty_str.ty);
818
819 switch (mcv) {
820 .register => |reg| {
821 switch (emit.debug_output) {
822 .dwarf => |dbg_out| {
823 try dbg_out.dbg_info.ensureUnusedCapacity(3);
824 dbg_out.dbg_info.appendAssumeCapacity(link.File.Elf.abbrev_parameter);
825 dbg_out.dbg_info.appendSliceAssumeCapacity(&[2]u8{ // DW.AT.location, DW.FORM.exprloc
826 1, // ULEB128 dwarf expression length
827 reg.dwarfLocOp(),
828 });
829 try dbg_out.dbg_info.ensureUnusedCapacity(5 + name_with_null.len);
830 try emit.addDbgInfoTypeReloc(ty); // DW.AT.type, DW.FORM.ref4
831 dbg_out.dbg_info.appendSliceAssumeCapacity(name_with_null); // DW.AT.name, DW.FORM.string
832 },
833 .plan9 => {},
834 .none => {},
835 }
836 },
837 .stack_offset => {
838 switch (emit.debug_output) {
839 .dwarf => {},
840 .plan9 => {},
841 .none => {},
842 }
843 },
844 else => {},
845 }
846}
847
848/// Adds a Type to the .debug_info at the current position. The bytes will be populated later,
849/// after codegen for this symbol is done.
850fn addDbgInfoTypeReloc(emit: *Emit, ty: Type) !void {
851 switch (emit.debug_output) {
852 .dwarf => |dbg_out| {
853 assert(ty.hasCodeGenBits());
854 const index = dbg_out.dbg_info.items.len;
855 try dbg_out.dbg_info.resize(index + 4); // DW.AT.type, DW.FORM.ref4
856
857 const gop = try dbg_out.dbg_info_type_relocs.getOrPut(emit.bin_file.allocator, ty);
858 if (!gop.found_existing) {
859 gop.value_ptr.* = .{
860 .off = undefined,
861 .relocs = .{},
862 };
863 }
864 try gop.value_ptr.relocs.append(emit.bin_file.allocator, @intCast(u32, index));
865 },
866 .plan9 => {},
867 .none => {},
868 }
869}
870
871const Tag = enum {
872 adc,
873 add,
874 sub,
875 xor,
876 @"and",
877 @"or",
878 sbb,
879 cmp,
880 mov,
881 lea,
882 jmp_near,
883 call_near,
884 push,
885 pop,
886 @"test",
887 brk,
888 nop,
889 imul,
890 syscall,
891 ret_near,
892 ret_far,
893 jo,
894 jno,
895 jb,
896 jbe,
897 jc,
898 jnae,
899 jnc,
900 jae,
901 je,
902 jz,
903 jne,
904 jnz,
905 jna,
906 jnb,
907 jnbe,
908 ja,
909 js,
910 jns,
911 jpe,
912 jp,
913 jpo,
914 jnp,
915 jnge,
916 jl,
917 jge,
918 jnl,
919 jle,
920 jng,
921 jg,
922 jnle,
923 seto,
924 setno,
925 setb,
926 setc,
927 setnae,
928 setnb,
929 setnc,
930 setae,
931 sete,
932 setz,
933 setne,
934 setnz,
935 setbe,
936 setna,
937 seta,
938 setnbe,
939 sets,
940 setns,
941 setp,
942 setpe,
943 setnp,
944 setop,
945 setl,
946 setnge,
947 setnl,
948 setge,
949 setle,
950 setng,
951 setnle,
952 setg,
953
954 fn isSetCC(tag: Tag) bool {
955 return switch (tag) {
956 .seto,
957 .setno,
958 .setb,
959 .setc,
960 .setnae,
961 .setnb,
962 .setnc,
963 .setae,
964 .sete,
965 .setz,
966 .setne,
967 .setnz,
968 .setbe,
969 .setna,
970 .seta,
971 .setnbe,
972 .sets,
973 .setns,
974 .setp,
975 .setpe,
976 .setnp,
977 .setop,
978 .setl,
979 .setnge,
980 .setnl,
981 .setge,
982 .setle,
983 .setng,
984 .setnle,
985 .setg,
986 => true,
987 else => false,
988 };
989 }
990};
991
992const Encoding = enum {
993 /// OP
994 zo,
995
996 /// OP rel32
997 d,
998
999 /// OP r/m64
1000 m,
1001
1002 /// OP r64
1003 o,
1004
1005 /// OP imm32
1006 i,
1007
1008 /// OP r/m64, imm32
1009 mi,
1010
1011 /// OP r/m64, r64
1012 mr,
1013
1014 /// OP r64, r/m64
1015 rm,
1016
1017 /// OP r64, imm64
1018 oi,
1019
1020 /// OP al/ax/eax/rax, moffs
1021 fd,
1022
1023 /// OP moffs, al/ax/eax/rax
1024 td,
1025
1026 /// OP r64, r/m64, imm32
1027 rmi,
1028};
1029
1030const OpCode = union(enum) {
1031 one_byte: u8,
1032 two_byte: struct { _1: u8, _2: u8 },
1033
1034 fn oneByte(opc: u8) OpCode {
1035 return .{ .one_byte = opc };
1036 }
1037
1038 fn twoByte(opc1: u8, opc2: u8) OpCode {
1039 return .{ .two_byte = .{ ._1 = opc1, ._2 = opc2 } };
1040 }
1041
1042 fn encode(opc: OpCode, encoder: Encoder) void {
1043 switch (opc) {
1044 .one_byte => |v| encoder.opcode_1byte(v),
1045 .two_byte => |v| encoder.opcode_2byte(v._1, v._2),
1046 }
1047 }
1048
1049 fn encodeWithReg(opc: OpCode, encoder: Encoder, reg: Register) void {
1050 assert(opc == .one_byte);
1051 encoder.opcode_withReg(opc.one_byte, reg.lowId());
1052 }
1053};
1054
1055inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode {
1056 switch (enc) {
1057 .zo => return switch (tag) {
1058 .ret_near => OpCode.oneByte(0xc3),
1059 .ret_far => OpCode.oneByte(0xcb),
1060 .brk => OpCode.oneByte(0xcc),
1061 .nop => OpCode.oneByte(0x90),
1062 .syscall => OpCode.twoByte(0x0f, 0x05),
1063 else => null,
1064 },
1065 .d => return switch (tag) {
1066 .jmp_near => OpCode.oneByte(0xe9),
1067 .call_near => OpCode.oneByte(0xe8),
1068 .jo => if (is_one_byte) OpCode.oneByte(0x70) else OpCode.twoByte(0x0f, 0x80),
1069 .jno => if (is_one_byte) OpCode.oneByte(0x71) else OpCode.twoByte(0x0f, 0x81),
1070 .jb, .jc, .jnae => if (is_one_byte) OpCode.oneByte(0x72) else OpCode.twoByte(0x0f, 0x82),
1071 .jnb, .jnc, .jae => if (is_one_byte) OpCode.oneByte(0x73) else OpCode.twoByte(0x0f, 0x83),
1072 .je, .jz => if (is_one_byte) OpCode.oneByte(0x74) else OpCode.twoByte(0x0f, 0x84),
1073 .jne, .jnz => if (is_one_byte) OpCode.oneByte(0x75) else OpCode.twoByte(0x0f, 0x85),
1074 .jna, .jbe => if (is_one_byte) OpCode.oneByte(0x76) else OpCode.twoByte(0x0f, 0x86),
1075 .jnbe, .ja => if (is_one_byte) OpCode.oneByte(0x77) else OpCode.twoByte(0x0f, 0x87),
1076 .js => if (is_one_byte) OpCode.oneByte(0x78) else OpCode.twoByte(0x0f, 0x88),
1077 .jns => if (is_one_byte) OpCode.oneByte(0x79) else OpCode.twoByte(0x0f, 0x89),
1078 .jpe, .jp => if (is_one_byte) OpCode.oneByte(0x7a) else OpCode.twoByte(0x0f, 0x8a),
1079 .jpo, .jnp => if (is_one_byte) OpCode.oneByte(0x7b) else OpCode.twoByte(0x0f, 0x8b),
1080 .jnge, .jl => if (is_one_byte) OpCode.oneByte(0x7c) else OpCode.twoByte(0x0f, 0x8c),
1081 .jge, .jnl => if (is_one_byte) OpCode.oneByte(0x7d) else OpCode.twoByte(0x0f, 0x8d),
1082 .jle, .jng => if (is_one_byte) OpCode.oneByte(0x7e) else OpCode.twoByte(0x0f, 0x8e),
1083 .jg, .jnle => if (is_one_byte) OpCode.oneByte(0x7f) else OpCode.twoByte(0x0f, 0x8f),
1084 else => null,
1085 },
1086 .m => return switch (tag) {
1087 .jmp_near, .call_near, .push => OpCode.oneByte(0xff),
1088 .pop => OpCode.oneByte(0x8f),
1089 .seto => OpCode.twoByte(0x0f, 0x90),
1090 .setno => OpCode.twoByte(0x0f, 0x91),
1091 .setb, .setc, .setnae => OpCode.twoByte(0x0f, 0x92),
1092 .setnb, .setnc, .setae => OpCode.twoByte(0x0f, 0x93),
1093 .sete, .setz => OpCode.twoByte(0x0f, 0x94),
1094 .setne, .setnz => OpCode.twoByte(0x0f, 0x95),
1095 .setbe, .setna => OpCode.twoByte(0x0f, 0x96),
1096 .seta, .setnbe => OpCode.twoByte(0x0f, 0x97),
1097 .sets => OpCode.twoByte(0x0f, 0x98),
1098 .setns => OpCode.twoByte(0x0f, 0x99),
1099 .setp, .setpe => OpCode.twoByte(0x0f, 0x9a),
1100 .setnp, .setop => OpCode.twoByte(0x0f, 0x9b),
1101 .setl, .setnge => OpCode.twoByte(0x0f, 0x9c),
1102 .setnl, .setge => OpCode.twoByte(0x0f, 0x9d),
1103 .setle, .setng => OpCode.twoByte(0x0f, 0x9e),
1104 .setnle, .setg => OpCode.twoByte(0x0f, 0x9f),
1105 else => null,
1106 },
1107 .o => return switch (tag) {
1108 .push => OpCode.oneByte(0x50),
1109 .pop => OpCode.oneByte(0x58),
1110 else => null,
1111 },
1112 .i => return switch (tag) {
1113 .push => OpCode.oneByte(if (is_one_byte) 0x6a else 0x68),
1114 .@"test" => OpCode.oneByte(if (is_one_byte) 0xa8 else 0xa9),
1115 .ret_near => OpCode.oneByte(0xc2),
1116 .ret_far => OpCode.oneByte(0xca),
1117 else => null,
1118 },
1119 .mi => return switch (tag) {
1120 .adc, .add, .sub, .xor, .@"and", .@"or", .sbb, .cmp => OpCode.oneByte(if (is_one_byte) 0x80 else 0x81),
1121 .mov => OpCode.oneByte(if (is_one_byte) 0xc6 else 0xc7),
1122 .@"test" => OpCode.oneByte(if (is_one_byte) 0xf6 else 0xf7),
1123 else => null,
1124 },
1125 .mr => return switch (tag) {
1126 .adc => OpCode.oneByte(if (is_one_byte) 0x10 else 0x11),
1127 .add => OpCode.oneByte(if (is_one_byte) 0x00 else 0x01),
1128 .sub => OpCode.oneByte(if (is_one_byte) 0x28 else 0x29),
1129 .xor => OpCode.oneByte(if (is_one_byte) 0x30 else 0x31),
1130 .@"and" => OpCode.oneByte(if (is_one_byte) 0x20 else 0x21),
1131 .@"or" => OpCode.oneByte(if (is_one_byte) 0x08 else 0x09),
1132 .sbb => OpCode.oneByte(if (is_one_byte) 0x18 else 0x19),
1133 .cmp => OpCode.oneByte(if (is_one_byte) 0x38 else 0x39),
1134 .mov => OpCode.oneByte(if (is_one_byte) 0x88 else 0x89),
1135 else => null,
1136 },
1137 .rm => return switch (tag) {
1138 .adc => OpCode.oneByte(if (is_one_byte) 0x12 else 0x13),
1139 .add => OpCode.oneByte(if (is_one_byte) 0x02 else 0x03),
1140 .sub => OpCode.oneByte(if (is_one_byte) 0x2a else 0x2b),
1141 .xor => OpCode.oneByte(if (is_one_byte) 0x32 else 0x33),
1142 .@"and" => OpCode.oneByte(if (is_one_byte) 0x22 else 0x23),
1143 .@"or" => OpCode.oneByte(if (is_one_byte) 0x0b else 0x0b),
1144 .sbb => OpCode.oneByte(if (is_one_byte) 0x1a else 0x1b),
1145 .cmp => OpCode.oneByte(if (is_one_byte) 0x3a else 0x3b),
1146 .mov => OpCode.oneByte(if (is_one_byte) 0x8a else 0x8b),
1147 .lea => OpCode.oneByte(if (is_one_byte) 0x8c else 0x8d),
1148 .imul => OpCode.twoByte(0x0f, 0xaf),
1149 else => null,
1150 },
1151 .oi => return switch (tag) {
1152 .mov => OpCode.oneByte(if (is_one_byte) 0xb0 else 0xb8),
1153 else => null,
1154 },
1155 .fd => return switch (tag) {
1156 .mov => OpCode.oneByte(if (is_one_byte) 0xa0 else 0xa1),
1157 else => null,
1158 },
1159 .td => return switch (tag) {
1160 .mov => OpCode.oneByte(if (is_one_byte) 0xa2 else 0xa3),
1161 else => null,
1162 },
1163 .rmi => return switch (tag) {
1164 .imul => OpCode.oneByte(if (is_one_byte) 0x6b else 0x69),
1165 else => null,
1166 },
1167 }
1168}
1169
1170inline fn getModRmExt(tag: Tag) ?u3 {
1171 return switch (tag) {
1172 .adc => 0x2,
1173 .add => 0x0,
1174 .sub => 0x5,
1175 .xor => 0x6,
1176 .@"and" => 0x4,
1177 .@"or" => 0x1,
1178 .sbb => 0x3,
1179 .cmp => 0x7,
1180 .mov => 0x0,
1181 .jmp_near => 0x4,
1182 .call_near => 0x2,
1183 .push => 0x6,
1184 .pop => 0x0,
1185 .@"test" => 0x0,
1186 .seto,
1187 .setno,
1188 .setb,
1189 .setc,
1190 .setnae,
1191 .setnb,
1192 .setnc,
1193 .setae,
1194 .sete,
1195 .setz,
1196 .setne,
1197 .setnz,
1198 .setbe,
1199 .setna,
1200 .seta,
1201 .setnbe,
1202 .sets,
1203 .setns,
1204 .setp,
1205 .setpe,
1206 .setnp,
1207 .setop,
1208 .setl,
1209 .setnge,
1210 .setnl,
1211 .setge,
1212 .setle,
1213 .setng,
1214 .setnle,
1215 .setg,
1216 => 0x0,
1217 else => null,
1218 };
1219}
1220
1221const ScaleIndex = struct {
1222 scale: u2,
1223 index: Register,
1224};
1225
1226const Memory = struct {
1227 base: ?Register,
1228 rip: bool = false,
1229 disp: u32,
1230 ptr_size: PtrSize,
1231 scale_index: ?ScaleIndex = null,
1232
1233 const PtrSize = enum {
1234 byte_ptr,
1235 word_ptr,
1236 dword_ptr,
1237 qword_ptr,
1238
1239 fn fromBits(in_bits: u64) PtrSize {
1240 return switch (in_bits) {
1241 8 => .byte_ptr,
1242 16 => .word_ptr,
1243 32 => .dword_ptr,
1244 64 => .qword_ptr,
1245 else => unreachable,
1246 };
1247 }
1248
1249 /// Returns size in bits.
1250 fn size(ptr_size: PtrSize) u64 {
1251 return switch (ptr_size) {
1252 .byte_ptr => 8,
1253 .word_ptr => 16,
1254 .dword_ptr => 32,
1255 .qword_ptr => 64,
1256 };
1257 }
1258 };
1259
1260 fn encode(mem_op: Memory, encoder: Encoder, operand: u3) void {
1261 if (mem_op.base) |base| {
1262 const dst = base.lowId();
1263 const src = operand;
1264 if (dst == 4 or mem_op.scale_index != null) {
1265 if (mem_op.disp == 0 and dst != 5) {
1266 encoder.modRm_SIBDisp0(src);
1267 if (mem_op.scale_index) |si| {
1268 encoder.sib_scaleIndexBase(si.scale, si.index.lowId(), dst);
1269 } else {
1270 encoder.sib_base(dst);
1271 }
1272 } else if (immOpSize(mem_op.disp) == 8) {
1273 encoder.modRm_SIBDisp8(src);
1274 if (mem_op.scale_index) |si| {
1275 encoder.sib_scaleIndexBaseDisp8(si.scale, si.index.lowId(), dst);
1276 } else {
1277 encoder.sib_baseDisp8(dst);
1278 }
1279 encoder.disp8(@bitCast(i8, @truncate(u8, mem_op.disp)));
1280 } else {
1281 encoder.modRm_SIBDisp32(src);
1282 if (mem_op.scale_index) |si| {
1283 encoder.sib_scaleIndexBaseDisp32(si.scale, si.index.lowId(), dst);
1284 } else {
1285 encoder.sib_baseDisp32(dst);
1286 }
1287 encoder.disp32(@bitCast(i32, mem_op.disp));
1288 }
1289 } else {
1290 if (mem_op.disp == 0) {
1291 encoder.modRm_indirectDisp0(src, dst);
1292 } else if (immOpSize(mem_op.disp) == 8) {
1293 encoder.modRm_indirectDisp8(src, dst);
1294 encoder.disp8(@bitCast(i8, @truncate(u8, mem_op.disp)));
1295 } else {
1296 encoder.modRm_indirectDisp32(src, dst);
1297 encoder.disp32(@bitCast(i32, mem_op.disp));
1298 }
1299 }
1300 } else {
1301 if (mem_op.rip) {
1302 encoder.modRm_RIPDisp32(operand);
1303 } else {
1304 encoder.modRm_SIBDisp0(operand);
1305 if (mem_op.scale_index) |si| {
1306 encoder.sib_scaleIndexDisp32(si.scale, si.index.lowId());
1307 } else {
1308 encoder.sib_disp32();
1309 }
1310 }
1311 encoder.disp32(@bitCast(i32, mem_op.disp));
1312 }
1313 }
1314};
1315
1316fn encodeImm(encoder: Encoder, imm: u32, size: u64) void {
1317 switch (size) {
1318 8 => encoder.imm8(@bitCast(i8, @truncate(u8, imm))),
1319 16 => encoder.imm16(@bitCast(i16, @truncate(u16, imm))),
1320 32, 64 => encoder.imm32(@bitCast(i32, imm)),
1321 else => unreachable,
1322 }
1323}
1324
1325const RegisterOrMemory = union(enum) {
1326 register: Register,
1327 memory: Memory,
1328
1329 fn reg(register: Register) RegisterOrMemory {
1330 return .{ .register = register };
1331 }
1332
1333 fn mem(ptr_size: Memory.PtrSize, args: struct {
1334 disp: u32,
1335 base: ?Register = null,
1336 scale_index: ?ScaleIndex = null,
1337 }) RegisterOrMemory {
1338 return .{
1339 .memory = .{
1340 .base = args.base,
1341 .disp = args.disp,
1342 .ptr_size = ptr_size,
1343 .scale_index = args.scale_index,
1344 },
1345 };
1346 }
1347
1348 fn rip(ptr_size: Memory.PtrSize, disp: u32) RegisterOrMemory {
1349 return .{
1350 .memory = .{
1351 .base = null,
1352 .rip = true,
1353 .disp = disp,
1354 .ptr_size = ptr_size,
1355 },
1356 };
1357 }
1358};
1359
1360const LoweringError = error{
1361 OutOfMemory,
1362 OperandSizeMismatch,
1363 RaxOperandExpected,
1364};
1365
1366fn lowerToZoEnc(tag: Tag, code: *std.ArrayList(u8)) LoweringError!void {
1367 const opc = getOpCode(tag, .zo, false).?;
1368 const encoder = try Encoder.init(code, 1);
1369 opc.encode(encoder);
1370}
1371
1372fn lowerToIEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) LoweringError!void {
1373 if (tag == .ret_far or tag == .ret_near) {
1374 const encoder = try Encoder.init(code, 3);
1375 const opc = getOpCode(tag, .i, false).?;
1376 opc.encode(encoder);
1377 encoder.imm16(@bitCast(i16, @truncate(u16, imm)));
1378 return;
1379 }
1380 const opc = getOpCode(tag, .i, immOpSize(imm) == 8).?;
1381 const encoder = try Encoder.init(code, 5);
1382 if (immOpSize(imm) == 16) {
1383 encoder.prefix16BitMode();
1384 }
1385 opc.encode(encoder);
1386 encodeImm(encoder, imm, immOpSize(imm));
1387}
1388
1389fn lowerToOEnc(tag: Tag, reg: Register, code: *std.ArrayList(u8)) LoweringError!void {
1390 if (reg.size() != 16 and reg.size() != 64) {
1391 return error.OperandSizeMismatch; // TODO correct for push/pop, but is it universal?
1392 }
1393 const opc = getOpCode(tag, .o, false).?;
1394 const encoder = try Encoder.init(code, 3);
1395 if (reg.size() == 16) {
1396 encoder.prefix16BitMode();
1397 }
1398 encoder.rex(.{
1399 .w = false,
1400 .b = reg.isExtended(),
1401 });
1402 opc.encodeWithReg(encoder, reg);
1403}
1404
1405fn lowerToDEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) LoweringError!void {
1406 const opc = getOpCode(tag, .d, false).?;
1407 const encoder = try Encoder.init(code, 6);
1408 opc.encode(encoder);
1409 encoder.imm32(@bitCast(i32, imm));
1410}
1411
1412fn lowerToMEnc(tag: Tag, reg_or_mem: RegisterOrMemory, code: *std.ArrayList(u8)) LoweringError!void {
1413 const opc = getOpCode(tag, .m, false).?;
1414 const modrm_ext = getModRmExt(tag).?;
1415 switch (reg_or_mem) {
1416 .register => |reg| {
1417 const op_size_mismatch = blk: {
1418 if (tag.isSetCC() and reg.size() == 8)
1419 break :blk false;
1420 break :blk reg.size() != 64 and reg.size() != 16;
1421 };
1422 if (op_size_mismatch) {
1423 return error.OperandSizeMismatch;
1424 }
1425 const encoder = try Encoder.init(code, 4);
1426 if (reg.size() == 16) {
1427 encoder.prefix16BitMode();
1428 }
1429 encoder.rex(.{
1430 .w = switch (reg) {
1431 .ah, .bh, .ch, .dh => true,
1432 else => false,
1433 },
1434 .b = reg.isExtended(),
1435 });
1436 opc.encode(encoder);
1437 encoder.modRm_direct(modrm_ext, reg.lowId());
1438 },
1439 .memory => |mem_op| {
1440 if (mem_op.ptr_size != .qword_ptr and mem_op.ptr_size != .word_ptr) {
1441 return error.OperandSizeMismatch;
1442 }
1443 const encoder = try Encoder.init(code, 8);
1444 if (mem_op.ptr_size == .word_ptr) {
1445 encoder.prefix16BitMode();
1446 }
1447 if (mem_op.base) |base| {
1448 if (base.size() != 64) {
1449 return error.OperandSizeMismatch;
1450 }
1451 encoder.rex(.{
1452 .w = false,
1453 .b = base.isExtended(),
1454 });
1455 }
1456 opc.encode(encoder);
1457 mem_op.encode(encoder, modrm_ext);
1458 },
1459 }
1460}
1461
1462fn lowerToTdEnc(tag: Tag, moffs: u64, reg: Register, code: *std.ArrayList(u8)) LoweringError!void {
1463 return lowerToTdFdEnc(tag, reg, moffs, code, true);
1464}
1465
1466fn lowerToFdEnc(tag: Tag, reg: Register, moffs: u64, code: *std.ArrayList(u8)) LoweringError!void {
1467 return lowerToTdFdEnc(tag, reg, moffs, code, false);
1468}
1469
1470fn lowerToTdFdEnc(tag: Tag, reg: Register, moffs: u64, code: *std.ArrayList(u8), td: bool) LoweringError!void {
1471 if (reg.lowId() != Register.rax.lowId()) {
1472 return error.RaxOperandExpected;
1473 }
1474 const opc = if (td)
1475 getOpCode(tag, .td, reg.size() == 8).?
1476 else
1477 getOpCode(tag, .fd, reg.size() == 8).?;
1478 const encoder = try Encoder.init(code, 10);
1479 if (reg.size() == 16) {
1480 encoder.prefix16BitMode();
1481 }
1482 encoder.rex(.{
1483 .w = setRexWRegister(reg),
1484 });
1485 opc.encode(encoder);
1486 switch (reg.size()) {
1487 8 => encoder.imm8(@bitCast(i8, @truncate(u8, moffs))),
1488 16 => encoder.imm16(@bitCast(i16, @truncate(u16, moffs))),
1489 32 => encoder.imm32(@bitCast(i32, @truncate(u32, moffs))),
1490 64 => encoder.imm64(moffs),
1491 else => unreachable,
1492 }
1493}
1494
1495fn lowerToOiEnc(tag: Tag, reg: Register, imm: u64, code: *std.ArrayList(u8)) LoweringError!void {
1496 const opc = getOpCode(tag, .oi, reg.size() == 8).?;
1497 const encoder = try Encoder.init(code, 10);
1498 if (reg.size() == 16) {
1499 encoder.prefix16BitMode();
1500 }
1501 encoder.rex(.{
1502 .w = setRexWRegister(reg),
1503 .b = reg.isExtended(),
1504 });
1505 opc.encodeWithReg(encoder, reg);
1506 switch (reg.size()) {
1507 8 => encoder.imm8(@bitCast(i8, @truncate(u8, imm))),
1508 16 => encoder.imm16(@bitCast(i16, @truncate(u16, imm))),
1509 32 => encoder.imm32(@bitCast(i32, @truncate(u32, imm))),
1510 64 => encoder.imm64(imm),
1511 else => unreachable,
1512 }
1513}
1514
1515fn lowerToMiEnc(tag: Tag, reg_or_mem: RegisterOrMemory, imm: u32, code: *std.ArrayList(u8)) LoweringError!void {
1516 const modrm_ext = getModRmExt(tag).?;
1517 switch (reg_or_mem) {
1518 .register => |dst_reg| {
1519 const opc = getOpCode(tag, .mi, dst_reg.size() == 8).?;
1520 const encoder = try Encoder.init(code, 7);
1521 if (dst_reg.size() == 16) {
1522 // 0x66 prefix switches to the non-default size; here we assume a switch from
1523 // the default 32bits to 16bits operand-size.
1524 // More info: https://www.cs.uni-potsdam.de/desn/lehre/ss15/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf#page=32&zoom=auto,-159,773
1525 encoder.prefix16BitMode();
1526 }
1527 encoder.rex(.{
1528 .w = setRexWRegister(dst_reg),
1529 .b = dst_reg.isExtended(),
1530 });
1531 opc.encode(encoder);
1532 encoder.modRm_direct(modrm_ext, dst_reg.lowId());
1533 encodeImm(encoder, imm, dst_reg.size());
1534 },
1535 .memory => |dst_mem| {
1536 const opc = getOpCode(tag, .mi, dst_mem.ptr_size == .byte_ptr).?;
1537 const encoder = try Encoder.init(code, 12);
1538 if (dst_mem.ptr_size == .word_ptr) {
1539 encoder.prefix16BitMode();
1540 }
1541 if (dst_mem.base) |base| {
1542 if (base.size() != 64) {
1543 return error.OperandSizeMismatch;
1544 }
1545 encoder.rex(.{
1546 .w = dst_mem.ptr_size == .qword_ptr,
1547 .b = base.isExtended(),
1548 });
1549 } else {
1550 encoder.rex(.{
1551 .w = dst_mem.ptr_size == .qword_ptr,
1552 });
1553 }
1554 opc.encode(encoder);
1555 dst_mem.encode(encoder, modrm_ext);
1556 encodeImm(encoder, imm, dst_mem.ptr_size.size());
1557 },
1558 }
1559}
1560
1561fn lowerToRmEnc(
1562 tag: Tag,
1563 reg: Register,
1564 reg_or_mem: RegisterOrMemory,
1565 code: *std.ArrayList(u8),
1566) LoweringError!void {
1567 const opc = getOpCode(tag, .rm, reg.size() == 8).?;
1568 switch (reg_or_mem) {
1569 .register => |src_reg| {
1570 if (reg.size() != src_reg.size()) {
1571 return error.OperandSizeMismatch;
1572 }
1573 const encoder = try Encoder.init(code, 3);
1574 encoder.rex(.{
1575 .w = setRexWRegister(reg) or setRexWRegister(src_reg),
1576 .r = reg.isExtended(),
1577 .b = src_reg.isExtended(),
1578 });
1579 opc.encode(encoder);
1580 encoder.modRm_direct(reg.lowId(), src_reg.lowId());
1581 },
1582 .memory => |src_mem| {
1583 if (reg.size() != src_mem.ptr_size.size()) {
1584 return error.OperandSizeMismatch;
1585 }
1586 const encoder = try Encoder.init(code, 9);
1587 if (reg.size() == 16) {
1588 encoder.prefix16BitMode();
1589 }
1590 if (src_mem.base) |base| {
1591 // TODO handle 32-bit base register - requires prefix 0x67
1592 // Intel Manual, Vol 1, chapter 3.6 and 3.6.1
1593 if (base.size() != 64) {
1594 return error.OperandSizeMismatch;
1595 }
1596 encoder.rex(.{
1597 .w = setRexWRegister(reg),
1598 .r = reg.isExtended(),
1599 .b = base.isExtended(),
1600 });
1601 } else {
1602 encoder.rex(.{
1603 .w = setRexWRegister(reg),
1604 .r = reg.isExtended(),
1605 });
1606 }
1607 opc.encode(encoder);
1608 src_mem.encode(encoder, reg.lowId());
1609 },
1610 }
1611}
1612
1613fn lowerToMrEnc(
1614 tag: Tag,
1615 reg_or_mem: RegisterOrMemory,
1616 reg: Register,
1617 code: *std.ArrayList(u8),
1618) LoweringError!void {
1619 const opc = getOpCode(tag, .mr, reg.size() == 8).?;
1620 switch (reg_or_mem) {
1621 .register => |dst_reg| {
1622 if (dst_reg.size() != reg.size()) {
1623 return error.OperandSizeMismatch;
1624 }
1625 const encoder = try Encoder.init(code, 3);
1626 encoder.rex(.{
1627 .w = setRexWRegister(dst_reg) or setRexWRegister(reg),
1628 .r = reg.isExtended(),
1629 .b = dst_reg.isExtended(),
1630 });
1631 opc.encode(encoder);
1632 encoder.modRm_direct(reg.lowId(), dst_reg.lowId());
1633 },
1634 .memory => |dst_mem| {
1635 if (dst_mem.ptr_size.size() != reg.size()) {
1636 return error.OperandSizeMismatch;
1637 }
1638 const encoder = try Encoder.init(code, 9);
1639 if (reg.size() == 16) {
1640 encoder.prefix16BitMode();
1641 }
1642 if (dst_mem.base) |base| {
1643 if (base.size() != 64) {
1644 return error.OperandSizeMismatch;
1645 }
1646 encoder.rex(.{
1647 .w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg),
1648 .r = reg.isExtended(),
1649 .b = base.isExtended(),
1650 });
1651 } else {
1652 encoder.rex(.{
1653 .w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg),
1654 .r = reg.isExtended(),
1655 });
1656 }
1657 opc.encode(encoder);
1658 dst_mem.encode(encoder, reg.lowId());
1659 },
1660 }
1661}
1662
1663fn lowerToRmiEnc(
1664 tag: Tag,
1665 reg: Register,
1666 reg_or_mem: RegisterOrMemory,
1667 imm: u32,
1668 code: *std.ArrayList(u8),
1669) LoweringError!void {
1670 if (reg.size() == 8) {
1671 return error.OperandSizeMismatch;
1672 }
1673 const opc = getOpCode(tag, .rmi, false).?;
1674 const encoder = try Encoder.init(code, 13);
1675 if (reg.size() == 16) {
1676 encoder.prefix16BitMode();
1677 }
1678 switch (reg_or_mem) {
1679 .register => |src_reg| {
1680 if (reg.size() != src_reg.size()) {
1681 return error.OperandSizeMismatch;
1682 }
1683 encoder.rex(.{
1684 .w = setRexWRegister(reg) or setRexWRegister(src_reg),
1685 .r = reg.isExtended(),
1686 .b = src_reg.isExtended(),
1687 });
1688 opc.encode(encoder);
1689 encoder.modRm_direct(reg.lowId(), src_reg.lowId());
1690 },
1691 .memory => |src_mem| {
1692 if (src_mem.base) |base| {
1693 // TODO handle 32-bit base register - requires prefix 0x67
1694 // Intel Manual, Vol 1, chapter 3.6 and 3.6.1
1695 if (base.size() != 64) {
1696 return error.OperandSizeMismatch;
1697 }
1698 if (src_mem.ptr_size == .byte_ptr) {
1699 return error.OperandSizeMismatch;
1700 }
1701 encoder.rex(.{
1702 .w = setRexWRegister(reg),
1703 .r = reg.isExtended(),
1704 .b = base.isExtended(),
1705 });
1706 } else {
1707 encoder.rex(.{
1708 .w = setRexWRegister(reg),
1709 .r = reg.isExtended(),
1710 });
1711 }
1712 opc.encode(encoder);
1713 src_mem.encode(encoder, reg.lowId());
1714 },
1715 }
1716 encodeImm(encoder, imm, reg.size());
1717}
1718
1719fn expectEqualHexStrings(expected: []const u8, given: []const u8, assembly: []const u8) !void {
1720 assert(expected.len > 0);
1721 if (mem.eql(u8, expected, given)) return;
1722 const expected_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{std.fmt.fmtSliceHexLower(expected)});
1723 defer testing.allocator.free(expected_fmt);
1724 const given_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{std.fmt.fmtSliceHexLower(given)});
1725 defer testing.allocator.free(given_fmt);
1726 const idx = mem.indexOfDiff(u8, expected_fmt, given_fmt).?;
1727 var padding = try testing.allocator.alloc(u8, idx + 5);
1728 defer testing.allocator.free(padding);
1729 mem.set(u8, padding, ' ');
1730 std.debug.print("\nASM: {s}\nEXP: {s}\nGIV: {s}\n{s}^ -- first differing byte\n", .{
1731 assembly,
1732 expected_fmt,
1733 given_fmt,
1734 padding,
1735 });
1736 return error.TestFailed;
1737}
1738
1739const TestEmit = struct {
1740 code_buffer: std.ArrayList(u8),
1741 next: usize = 0,
1742
1743 fn init() TestEmit {
1744 return .{
1745 .code_buffer = std.ArrayList(u8).init(testing.allocator),
1746 };
1747 }
1748
1749 fn deinit(emit: *TestEmit) void {
1750 emit.code_buffer.deinit();
1751 emit.next = undefined;
1752 }
1753
1754 fn code(emit: *TestEmit) *std.ArrayList(u8) {
1755 emit.next = emit.code_buffer.items.len;
1756 return &emit.code_buffer;
1757 }
1758
1759 fn lowered(emit: TestEmit) []const u8 {
1760 return emit.code_buffer.items[emit.next..];
1761 }
1762};
1763
1764test "lower MI encoding" {
1765 var emit = TestEmit.init();
1766 defer emit.deinit();
1767 try lowerToMiEnc(.mov, RegisterOrMemory.reg(.rax), 0x10, emit.code());
1768 try expectEqualHexStrings("\x48\xc7\xc0\x10\x00\x00\x00", emit.lowered(), "mov rax, 0x10");
1769 try lowerToMiEnc(.mov, RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0, .base = .r11 }), 0x10, emit.code());
1770 try expectEqualHexStrings("\x41\xc7\x03\x10\x00\x00\x00", emit.lowered(), "mov dword ptr [r11 + 0], 0x10");
1771 try lowerToMiEnc(.add, RegisterOrMemory.mem(.dword_ptr, .{
1772 .disp = @bitCast(u32, @as(i32, -8)),
1773 .base = .rdx,
1774 }), 0x10, emit.code());
1775 try expectEqualHexStrings("\x81\x42\xF8\x10\x00\x00\x00", emit.lowered(), "add dword ptr [rdx - 8], 0x10");
1776 try lowerToMiEnc(.sub, RegisterOrMemory.mem(.dword_ptr, .{
1777 .disp = 0x10000000,
1778 .base = .r11,
1779 }), 0x10, emit.code());
1780 try expectEqualHexStrings(
1781 "\x41\x81\xab\x00\x00\x00\x10\x10\x00\x00\x00",
1782 emit.lowered(),
1783 "sub dword ptr [r11 + 0x10000000], 0x10",
1784 );
1785 try lowerToMiEnc(.@"and", RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0x10000000 }), 0x10, emit.code());
1786 try expectEqualHexStrings(
1787 "\x81\x24\x25\x00\x00\x00\x10\x10\x00\x00\x00",
1788 emit.lowered(),
1789 "and dword ptr [ds:0x10000000], 0x10",
1790 );
1791 try lowerToMiEnc(.@"and", RegisterOrMemory.mem(.dword_ptr, .{
1792 .disp = 0x10000000,
1793 .base = .r12,
1794 }), 0x10, emit.code());
1795 try expectEqualHexStrings(
1796 "\x41\x81\xA4\x24\x00\x00\x00\x10\x10\x00\x00\x00",
1797 emit.lowered(),
1798 "and dword ptr [r12 + 0x10000000], 0x10",
1799 );
1800 try lowerToMiEnc(.mov, RegisterOrMemory.rip(.qword_ptr, 0x10), 0x10, emit.code());
1801 try expectEqualHexStrings(
1802 "\x48\xC7\x05\x10\x00\x00\x00\x10\x00\x00\x00",
1803 emit.lowered(),
1804 "mov qword ptr [rip + 0x10], 0x10",
1805 );
1806 try lowerToMiEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
1807 .disp = @bitCast(u32, @as(i32, -8)),
1808 .base = .rbp,
1809 }), 0x10, emit.code());
1810 try expectEqualHexStrings(
1811 "\x48\xc7\x45\xf8\x10\x00\x00\x00",
1812 emit.lowered(),
1813 "mov qword ptr [rbp - 8], 0x10",
1814 );
1815 try lowerToMiEnc(.mov, RegisterOrMemory.mem(.word_ptr, .{
1816 .disp = @bitCast(u32, @as(i32, -2)),
1817 .base = .rbp,
1818 }), 0x10, emit.code());
1819 try expectEqualHexStrings("\x66\xC7\x45\xFE\x10\x00", emit.lowered(), "mov word ptr [rbp - 2], 0x10");
1820 try lowerToMiEnc(.mov, RegisterOrMemory.mem(.byte_ptr, .{
1821 .disp = @bitCast(u32, @as(i32, -1)),
1822 .base = .rbp,
1823 }), 0x10, emit.code());
1824 try expectEqualHexStrings("\xC6\x45\xFF\x10", emit.lowered(), "mov byte ptr [rbp - 1], 0x10");
1825 try lowerToMiEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
1826 .disp = 0x10000000,
1827 .scale_index = .{
1828 .scale = 1,
1829 .index = .rcx,
1830 },
1831 }), 0x10, emit.code());
1832 try expectEqualHexStrings(
1833 "\x48\xC7\x04\x4D\x00\x00\x00\x10\x10\x00\x00\x00",
1834 emit.lowered(),
1835 "mov qword ptr [rcx*2 + 0x10000000], 0x10",
1836 );
1837}
1838
1839test "lower RM encoding" {
1840 var emit = TestEmit.init();
1841 defer emit.deinit();
1842 try lowerToRmEnc(.mov, .rax, RegisterOrMemory.reg(.rbx), emit.code());
1843 try expectEqualHexStrings("\x48\x8b\xc3", emit.lowered(), "mov rax, rbx");
1844 try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0, .base = .r11 }), emit.code());
1845 try expectEqualHexStrings("\x49\x8b\x03", emit.lowered(), "mov rax, qword ptr [r11 + 0]");
1846 try lowerToRmEnc(.add, .r11, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10000000 }), emit.code());
1847 try expectEqualHexStrings(
1848 "\x4C\x03\x1C\x25\x00\x00\x00\x10",
1849 emit.lowered(),
1850 "add r11, qword ptr [ds:0x10000000]",
1851 );
1852 try lowerToRmEnc(.add, .r12b, RegisterOrMemory.mem(.byte_ptr, .{ .disp = 0x10000000 }), emit.code());
1853 try expectEqualHexStrings(
1854 "\x44\x02\x24\x25\x00\x00\x00\x10",
1855 emit.lowered(),
1856 "add r11b, byte ptr [ds:0x10000000]",
1857 );
1858 try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.qword_ptr, .{
1859 .disp = 0x10000000,
1860 .base = .r13,
1861 }), emit.code());
1862 try expectEqualHexStrings(
1863 "\x4D\x2B\x9D\x00\x00\x00\x10",
1864 emit.lowered(),
1865 "sub r11, qword ptr [r13 + 0x10000000]",
1866 );
1867 try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.qword_ptr, .{
1868 .disp = 0x10000000,
1869 .base = .r12,
1870 }), emit.code());
1871 try expectEqualHexStrings(
1872 "\x4D\x2B\x9C\x24\x00\x00\x00\x10",
1873 emit.lowered(),
1874 "sub r11, qword ptr [r12 + 0x10000000]",
1875 );
1876 try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{
1877 .disp = @bitCast(u32, @as(i32, -4)),
1878 .base = .rbp,
1879 }), emit.code());
1880 try expectEqualHexStrings("\x48\x8B\x45\xFC", emit.lowered(), "mov rax, qword ptr [rbp - 4]");
1881 try lowerToRmEnc(.lea, .rax, RegisterOrMemory.rip(.qword_ptr, 0x10), emit.code());
1882 try expectEqualHexStrings("\x48\x8D\x05\x10\x00\x00\x00", emit.lowered(), "lea rax, [rip + 0x10]");
1883 try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{
1884 .disp = @bitCast(u32, @as(i32, -8)),
1885 .base = .rbp,
1886 .scale_index = .{
1887 .scale = 0,
1888 .index = .rcx,
1889 },
1890 }), emit.code());
1891 try expectEqualHexStrings("\x48\x8B\x44\x0D\xF8", emit.lowered(), "mov rax, qword ptr [rbp + rcx*1 - 8]");
1892 try lowerToRmEnc(.mov, .eax, RegisterOrMemory.mem(.dword_ptr, .{
1893 .disp = @bitCast(u32, @as(i32, -4)),
1894 .base = .rbp,
1895 .scale_index = .{
1896 .scale = 2,
1897 .index = .rdx,
1898 },
1899 }), emit.code());
1900 try expectEqualHexStrings("\x8B\x44\x95\xFC", emit.lowered(), "mov eax, dword ptr [rbp + rdx*4 - 4]");
1901 try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{
1902 .disp = @bitCast(u32, @as(i32, -8)),
1903 .base = .rbp,
1904 .scale_index = .{
1905 .scale = 3,
1906 .index = .rcx,
1907 },
1908 }), emit.code());
1909 try expectEqualHexStrings("\x48\x8B\x44\xCD\xF8", emit.lowered(), "mov rax, qword ptr [rbp + rcx*8 - 8]");
1910 try lowerToRmEnc(.mov, .r8b, RegisterOrMemory.mem(.byte_ptr, .{
1911 .disp = @bitCast(u32, @as(i32, -24)),
1912 .base = .rsi,
1913 .scale_index = .{
1914 .scale = 0,
1915 .index = .rcx,
1916 },
1917 }), emit.code());
1918 try expectEqualHexStrings("\x44\x8A\x44\x0E\xE8", emit.lowered(), "mov r8b, byte ptr [rsi + rcx*1 - 24]");
1919 try lowerToRmEnc(.lea, .rsi, RegisterOrMemory.mem(.qword_ptr, .{
1920 .disp = 0,
1921 .base = .rbp,
1922 .scale_index = .{
1923 .scale = 0,
1924 .index = .rcx,
1925 },
1926 }), emit.code());
1927 try expectEqualHexStrings("\x48\x8D\x74\x0D\x00", emit.lowered(), "lea rsi, qword ptr [rbp + rcx*1 + 0]");
1928}
1929
1930test "lower MR encoding" {
1931 var emit = TestEmit.init();
1932 defer emit.deinit();
1933 try lowerToMrEnc(.mov, RegisterOrMemory.reg(.rax), .rbx, emit.code());
1934 try expectEqualHexStrings("\x48\x89\xd8", emit.lowered(), "mov rax, rbx");
1935 try lowerToMrEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
1936 .disp = @bitCast(u32, @as(i32, -4)),
1937 .base = .rbp,
1938 }), .r11, emit.code());
1939 try expectEqualHexStrings("\x4c\x89\x5d\xfc", emit.lowered(), "mov qword ptr [rbp - 4], r11");
1940 try lowerToMrEnc(.add, RegisterOrMemory.mem(.byte_ptr, .{ .disp = 0x10000000 }), .r12b, emit.code());
1941 try expectEqualHexStrings(
1942 "\x44\x00\x24\x25\x00\x00\x00\x10",
1943 emit.lowered(),
1944 "add byte ptr [ds:0x10000000], r12b",
1945 );
1946 try lowerToMrEnc(.add, RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0x10000000 }), .r12d, emit.code());
1947 try expectEqualHexStrings(
1948 "\x44\x01\x24\x25\x00\x00\x00\x10",
1949 emit.lowered(),
1950 "add dword ptr [ds:0x10000000], r12d",
1951 );
1952 try lowerToMrEnc(.sub, RegisterOrMemory.mem(.qword_ptr, .{
1953 .disp = 0x10000000,
1954 .base = .r11,
1955 }), .r12, emit.code());
1956 try expectEqualHexStrings(
1957 "\x4D\x29\xA3\x00\x00\x00\x10",
1958 emit.lowered(),
1959 "sub qword ptr [r11 + 0x10000000], r12",
1960 );
1961 try lowerToMrEnc(.mov, RegisterOrMemory.rip(.qword_ptr, 0x10), .r12, emit.code());
1962 try expectEqualHexStrings("\x4C\x89\x25\x10\x00\x00\x00", emit.lowered(), "mov qword ptr [rip + 0x10], r12");
1963}
1964
1965test "lower OI encoding" {
1966 var emit = TestEmit.init();
1967 defer emit.deinit();
1968 try lowerToOiEnc(.mov, .rax, 0x1000000000000000, emit.code());
1969 try expectEqualHexStrings(
1970 "\x48\xB8\x00\x00\x00\x00\x00\x00\x00\x10",
1971 emit.lowered(),
1972 "movabs rax, 0x1000000000000000",
1973 );
1974 try lowerToOiEnc(.mov, .r11, 0x1000000000000000, emit.code());
1975 try expectEqualHexStrings(
1976 "\x49\xBB\x00\x00\x00\x00\x00\x00\x00\x10",
1977 emit.lowered(),
1978 "movabs r11, 0x1000000000000000",
1979 );
1980 try lowerToOiEnc(.mov, .r11d, 0x10000000, emit.code());
1981 try expectEqualHexStrings("\x41\xBB\x00\x00\x00\x10", emit.lowered(), "mov r11d, 0x10000000");
1982 try lowerToOiEnc(.mov, .r11w, 0x1000, emit.code());
1983 try expectEqualHexStrings("\x66\x41\xBB\x00\x10", emit.lowered(), "mov r11w, 0x1000");
1984 try lowerToOiEnc(.mov, .r11b, 0x10, emit.code());
1985 try expectEqualHexStrings("\x41\xB3\x10", emit.lowered(), "mov r11b, 0x10");
1986}
1987
1988test "lower FD/TD encoding" {
1989 var emit = TestEmit.init();
1990 defer emit.deinit();
1991 try lowerToFdEnc(.mov, .rax, 0x1000000000000000, emit.code());
1992 try expectEqualHexStrings(
1993 "\x48\xa1\x00\x00\x00\x00\x00\x00\x00\x10",
1994 emit.lowered(),
1995 "mov rax, ds:0x1000000000000000",
1996 );
1997 try lowerToFdEnc(.mov, .eax, 0x10000000, emit.code());
1998 try expectEqualHexStrings("\xa1\x00\x00\x00\x10", emit.lowered(), "mov eax, ds:0x10000000");
1999 try lowerToFdEnc(.mov, .ax, 0x1000, emit.code());
2000 try expectEqualHexStrings("\x66\xa1\x00\x10", emit.lowered(), "mov ax, ds:0x1000");
2001 try lowerToFdEnc(.mov, .al, 0x10, emit.code());
2002 try expectEqualHexStrings("\xa0\x10", emit.lowered(), "mov al, ds:0x10");
2003}
2004
2005test "lower M encoding" {
2006 var emit = TestEmit.init();
2007 defer emit.deinit();
2008 try lowerToMEnc(.jmp_near, RegisterOrMemory.reg(.r12), emit.code());
2009 try expectEqualHexStrings("\x41\xFF\xE4", emit.lowered(), "jmp r12");
2010 try lowerToMEnc(.jmp_near, RegisterOrMemory.reg(.r12w), emit.code());
2011 try expectEqualHexStrings("\x66\x41\xFF\xE4", emit.lowered(), "jmp r12w");
2012 try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0, .base = .r12 }), emit.code());
2013 try expectEqualHexStrings("\x41\xFF\x24\x24", emit.lowered(), "jmp qword ptr [r12]");
2014 try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.word_ptr, .{ .disp = 0, .base = .r12 }), emit.code());
2015 try expectEqualHexStrings("\x66\x41\xFF\x24\x24", emit.lowered(), "jmp word ptr [r12]");
2016 try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10, .base = .r12 }), emit.code());
2017 try expectEqualHexStrings("\x41\xFF\x64\x24\x10", emit.lowered(), "jmp qword ptr [r12 + 0x10]");
2018 try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{
2019 .disp = 0x1000,
2020 .base = .r12,
2021 }), emit.code());
2022 try expectEqualHexStrings(
2023 "\x41\xFF\xA4\x24\x00\x10\x00\x00",
2024 emit.lowered(),
2025 "jmp qword ptr [r12 + 0x1000]",
2026 );
2027 try lowerToMEnc(.jmp_near, RegisterOrMemory.rip(.qword_ptr, 0x10), emit.code());
2028 try expectEqualHexStrings("\xFF\x25\x10\x00\x00\x00", emit.lowered(), "jmp qword ptr [rip + 0x10]");
2029 try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10 }), emit.code());
2030 try expectEqualHexStrings("\xFF\x24\x25\x10\x00\x00\x00", emit.lowered(), "jmp qword ptr [ds:0x10]");
2031 try lowerToMEnc(.seta, RegisterOrMemory.reg(.r11b), emit.code());
2032 try expectEqualHexStrings("\x41\x0F\x97\xC3", emit.lowered(), "seta r11b");
2033}
2034
2035test "lower O encoding" {
2036 var emit = TestEmit.init();
2037 defer emit.deinit();
2038 try lowerToOEnc(.pop, .r12, emit.code());
2039 try expectEqualHexStrings("\x41\x5c", emit.lowered(), "pop r12");
2040 try lowerToOEnc(.push, .r12w, emit.code());
2041 try expectEqualHexStrings("\x66\x41\x54", emit.lowered(), "push r12w");
2042}
2043
2044test "lower RMI encoding" {
2045 var emit = TestEmit.init();
2046 defer emit.deinit();
2047 try lowerToRmiEnc(.imul, .rax, RegisterOrMemory.mem(.qword_ptr, .{
2048 .disp = @bitCast(u32, @as(i32, -8)),
2049 .base = .rbp,
2050 }), 0x10, emit.code());
2051 try expectEqualHexStrings(
2052 "\x48\x69\x45\xF8\x10\x00\x00\x00",
2053 emit.lowered(),
2054 "imul rax, qword ptr [rbp - 8], 0x10",
2055 );
2056 try lowerToRmiEnc(.imul, .eax, RegisterOrMemory.mem(.dword_ptr, .{
2057 .disp = @bitCast(u32, @as(i32, -4)),
2058 .base = .rbp,
2059 }), 0x10, emit.code());
2060 try expectEqualHexStrings("\x69\x45\xFC\x10\x00\x00\x00", emit.lowered(), "imul eax, dword ptr [rbp - 4], 0x10");
2061 try lowerToRmiEnc(.imul, .ax, RegisterOrMemory.mem(.word_ptr, .{
2062 .disp = @bitCast(u32, @as(i32, -2)),
2063 .base = .rbp,
2064 }), 0x10, emit.code());
2065 try expectEqualHexStrings("\x66\x69\x45\xFE\x10\x00", emit.lowered(), "imul ax, word ptr [rbp - 2], 0x10");
2066 try lowerToRmiEnc(.imul, .r12, RegisterOrMemory.reg(.r12), 0x10, emit.code());
2067 try expectEqualHexStrings("\x4D\x69\xE4\x10\x00\x00\x00", emit.lowered(), "imul r12, r12, 0x10");
2068 try lowerToRmiEnc(.imul, .r12w, RegisterOrMemory.reg(.r12w), 0x10, emit.code());
2069 try expectEqualHexStrings("\x66\x45\x69\xE4\x10\x00", emit.lowered(), "imul r12w, r12w, 0x10");
2070}
src/arch/x86_64/Isel.zig deleted-2088
...@@ -1,2088 +0,0 @@
1//! This file contains the functionality for lowering x86_64 MIR into
2//! machine code
3
4const Isel = @This();
5
6const std = @import("std");
7const assert = std.debug.assert;
8const bits = @import("bits.zig");
9const leb128 = std.leb;
10const link = @import("../../link.zig");
11const log = std.log.scoped(.codegen);
12const math = std.math;
13const mem = std.mem;
14const testing = std.testing;
15
16const Air = @import("../../Air.zig");
17const Allocator = mem.Allocator;
18const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput;
19const DW = std.dwarf;
20const Encoder = bits.Encoder;
21const ErrorMsg = Module.ErrorMsg;
22const MCValue = @import("CodeGen.zig").MCValue;
23const Mir = @import("Mir.zig");
24const Module = @import("../../Module.zig");
25const Instruction = bits.Instruction;
26const Register = bits.Register;
27const Type = @import("../../type.zig").Type;
28
29mir: Mir,
30bin_file: *link.File,
31debug_output: DebugInfoOutput,
32target: *const std.Target,
33err_msg: ?*ErrorMsg = null,
34src_loc: Module.SrcLoc,
35code: *std.ArrayList(u8),
36
37prev_di_line: u32,
38prev_di_column: u32,
39/// Relative to the beginning of `code`.
40prev_di_pc: usize,
41
42code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .{},
43relocs: std.ArrayListUnmanaged(Reloc) = .{},
44
45const InnerError = error{
46 OutOfMemory,
47 Overflow,
48 IselFail,
49};
50
51const Reloc = struct {
52 /// Offset of the instruction.
53 source: u64,
54 /// Target of the relocation.
55 target: Mir.Inst.Index,
56 /// Offset of the relocation within the instruction.
57 offset: u64,
58 /// Length of the instruction.
59 length: u5,
60};
61
62pub fn lowerMir(isel: *Isel) InnerError!void {
63 const mir_tags = isel.mir.instructions.items(.tag);
64
65 for (mir_tags) |tag, index| {
66 const inst = @intCast(u32, index);
67 try isel.code_offset_mapping.putNoClobber(isel.bin_file.allocator, inst, isel.code.items.len);
68 switch (tag) {
69 .adc => try isel.mirArith(.adc, inst),
70 .add => try isel.mirArith(.add, inst),
71 .sub => try isel.mirArith(.sub, inst),
72 .xor => try isel.mirArith(.xor, inst),
73 .@"and" => try isel.mirArith(.@"and", inst),
74 .@"or" => try isel.mirArith(.@"or", inst),
75 .sbb => try isel.mirArith(.sbb, inst),
76 .cmp => try isel.mirArith(.cmp, inst),
77 .mov => try isel.mirArith(.mov, inst),
78
79 .adc_mem_imm => try isel.mirArithMemImm(.adc, inst),
80 .add_mem_imm => try isel.mirArithMemImm(.add, inst),
81 .sub_mem_imm => try isel.mirArithMemImm(.sub, inst),
82 .xor_mem_imm => try isel.mirArithMemImm(.xor, inst),
83 .and_mem_imm => try isel.mirArithMemImm(.@"and", inst),
84 .or_mem_imm => try isel.mirArithMemImm(.@"or", inst),
85 .sbb_mem_imm => try isel.mirArithMemImm(.sbb, inst),
86 .cmp_mem_imm => try isel.mirArithMemImm(.cmp, inst),
87 .mov_mem_imm => try isel.mirArithMemImm(.mov, inst),
88
89 .adc_scale_src => try isel.mirArithScaleSrc(.adc, inst),
90 .add_scale_src => try isel.mirArithScaleSrc(.add, inst),
91 .sub_scale_src => try isel.mirArithScaleSrc(.sub, inst),
92 .xor_scale_src => try isel.mirArithScaleSrc(.xor, inst),
93 .and_scale_src => try isel.mirArithScaleSrc(.@"and", inst),
94 .or_scale_src => try isel.mirArithScaleSrc(.@"or", inst),
95 .sbb_scale_src => try isel.mirArithScaleSrc(.sbb, inst),
96 .cmp_scale_src => try isel.mirArithScaleSrc(.cmp, inst),
97 .mov_scale_src => try isel.mirArithScaleSrc(.mov, inst),
98
99 .adc_scale_dst => try isel.mirArithScaleDst(.adc, inst),
100 .add_scale_dst => try isel.mirArithScaleDst(.add, inst),
101 .sub_scale_dst => try isel.mirArithScaleDst(.sub, inst),
102 .xor_scale_dst => try isel.mirArithScaleDst(.xor, inst),
103 .and_scale_dst => try isel.mirArithScaleDst(.@"and", inst),
104 .or_scale_dst => try isel.mirArithScaleDst(.@"or", inst),
105 .sbb_scale_dst => try isel.mirArithScaleDst(.sbb, inst),
106 .cmp_scale_dst => try isel.mirArithScaleDst(.cmp, inst),
107 .mov_scale_dst => try isel.mirArithScaleDst(.mov, inst),
108
109 .adc_scale_imm => try isel.mirArithScaleImm(.adc, inst),
110 .add_scale_imm => try isel.mirArithScaleImm(.add, inst),
111 .sub_scale_imm => try isel.mirArithScaleImm(.sub, inst),
112 .xor_scale_imm => try isel.mirArithScaleImm(.xor, inst),
113 .and_scale_imm => try isel.mirArithScaleImm(.@"and", inst),
114 .or_scale_imm => try isel.mirArithScaleImm(.@"or", inst),
115 .sbb_scale_imm => try isel.mirArithScaleImm(.sbb, inst),
116 .cmp_scale_imm => try isel.mirArithScaleImm(.cmp, inst),
117 .mov_scale_imm => try isel.mirArithScaleImm(.mov, inst),
118
119 .movabs => try isel.mirMovabs(inst),
120
121 .lea => try isel.mirLea(inst),
122
123 .imul_complex => try isel.mirIMulComplex(inst),
124
125 .push => try isel.mirPushPop(.push, inst),
126 .pop => try isel.mirPushPop(.pop, inst),
127
128 .jmp => try isel.mirJmpCall(.jmp_near, inst),
129 .call => try isel.mirJmpCall(.call_near, inst),
130
131 .cond_jmp_greater_less,
132 .cond_jmp_above_below,
133 .cond_jmp_eq_ne,
134 => try isel.mirCondJmp(tag, inst),
135
136 .cond_set_byte_greater_less,
137 .cond_set_byte_above_below,
138 .cond_set_byte_eq_ne,
139 => try isel.mirCondSetByte(tag, inst),
140
141 .ret => try isel.mirRet(inst),
142
143 .syscall => try isel.mirSyscall(),
144
145 .@"test" => try isel.mirTest(inst),
146
147 .brk => try isel.mirBrk(),
148 .nop => try isel.mirNop(),
149
150 .call_extern => try isel.mirCallExtern(inst),
151
152 .dbg_line => try isel.mirDbgLine(inst),
153 .dbg_prologue_end => try isel.mirDbgPrologueEnd(inst),
154 .dbg_epilogue_begin => try isel.mirDbgEpilogueBegin(inst),
155 .arg_dbg_info => try isel.mirArgDbgInfo(inst),
156
157 .push_regs_from_callee_preserved_regs => try isel.mirPushPopRegsFromCalleePreservedRegs(.push, inst),
158 .pop_regs_from_callee_preserved_regs => try isel.mirPushPopRegsFromCalleePreservedRegs(.pop, inst),
159
160 else => {
161 return isel.fail("Implement MIR->Isel lowering for x86_64 for pseudo-inst: {s}", .{tag});
162 },
163 }
164 }
165
166 try isel.fixupRelocs();
167}
168
169pub fn deinit(isel: *Isel) void {
170 isel.relocs.deinit(isel.bin_file.allocator);
171 isel.code_offset_mapping.deinit(isel.bin_file.allocator);
172 isel.* = undefined;
173}
174
175fn fail(isel: *Isel, comptime format: []const u8, args: anytype) InnerError {
176 @setCold(true);
177 assert(isel.err_msg == null);
178 isel.err_msg = try ErrorMsg.create(isel.bin_file.allocator, isel.src_loc, format, args);
179 return error.IselFail;
180}
181
182fn failWithLoweringError(isel: *Isel, err: LoweringError) InnerError {
183 return switch (err) {
184 error.RaxOperandExpected => isel.fail("Register.rax expected as destination operand", .{}),
185 error.OperandSizeMismatch => isel.fail("operand size mismatch", .{}),
186 else => |e| e,
187 };
188}
189
190fn fixupRelocs(isel: *Isel) InnerError!void {
191 // TODO this function currently assumes all relocs via JMP/CALL instructions are 32bit in size.
192 // This should be reversed like it is done in aarch64 MIR emit code: start with the smallest
193 // possible resolution, i.e., 8bit, and iteratively converge on the minimum required resolution
194 // until the entire decl is correctly emitted with all JMP/CALL instructions within range.
195 for (isel.relocs.items) |reloc| {
196 const offset = try math.cast(usize, reloc.offset);
197 const target = isel.code_offset_mapping.get(reloc.target) orelse
198 return isel.fail("JMP/CALL relocation target not found!", .{});
199 const disp = @intCast(i32, @intCast(i64, target) - @intCast(i64, reloc.source + reloc.length));
200 mem.writeIntLittle(i32, isel.code.items[offset..][0..4], disp);
201 }
202}
203
204fn mirBrk(isel: *Isel) InnerError!void {
205 return lowerToZoEnc(.brk, isel.code) catch |err| isel.failWithLoweringError(err);
206}
207
208fn mirNop(isel: *Isel) InnerError!void {
209 return lowerToZoEnc(.nop, isel.code) catch |err| isel.failWithLoweringError(err);
210}
211
212fn mirSyscall(isel: *Isel) InnerError!void {
213 return lowerToZoEnc(.syscall, isel.code) catch |err| isel.failWithLoweringError(err);
214}
215
216fn mirPushPop(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
217 const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
218 switch (ops.flags) {
219 0b00 => {
220 // PUSH/POP reg
221 return lowerToOEnc(tag, ops.reg1, isel.code) catch |err| isel.failWithLoweringError(err);
222 },
223 0b01 => {
224 // PUSH/POP r/m64
225 const imm = isel.mir.instructions.items(.data)[inst].imm;
226 const ptr_size: Memory.PtrSize = switch (immOpSize(imm)) {
227 16 => .word_ptr,
228 else => .qword_ptr,
229 };
230 return lowerToMEnc(tag, RegisterOrMemory.mem(ptr_size, .{
231 .disp = imm,
232 .base = ops.reg1,
233 }), isel.code) catch |err| isel.failWithLoweringError(err);
234 },
235 0b10 => {
236 // PUSH imm32
237 assert(tag == .push);
238 const imm = isel.mir.instructions.items(.data)[inst].imm;
239 return lowerToIEnc(.push, imm, isel.code) catch |err|
240 isel.failWithLoweringError(err);
241 },
242 0b11 => unreachable,
243 }
244}
245fn mirPushPopRegsFromCalleePreservedRegs(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
246 const callee_preserved_regs = bits.callee_preserved_regs;
247 const regs = isel.mir.instructions.items(.data)[inst].regs_to_push_or_pop;
248 if (tag == .push) {
249 for (callee_preserved_regs) |reg, i| {
250 if ((regs >> @intCast(u5, i)) & 1 == 0) continue;
251 lowerToOEnc(.push, reg, isel.code) catch |err|
252 return isel.failWithLoweringError(err);
253 }
254 } else {
255 // pop in the reverse direction
256 var i = callee_preserved_regs.len;
257 while (i > 0) : (i -= 1) {
258 const reg = callee_preserved_regs[i - 1];
259 if ((regs >> @intCast(u5, i - 1)) & 1 == 0) continue;
260 lowerToOEnc(.pop, reg, isel.code) catch |err|
261 return isel.failWithLoweringError(err);
262 }
263 }
264}
265
266fn mirJmpCall(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
267 const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
268 switch (ops.flags) {
269 0b00 => {
270 const target = isel.mir.instructions.items(.data)[inst].inst;
271 const source = isel.code.items.len;
272 lowerToDEnc(tag, 0, isel.code) catch |err|
273 return isel.failWithLoweringError(err);
274 try isel.relocs.append(isel.bin_file.allocator, .{
275 .source = source,
276 .target = target,
277 .offset = isel.code.items.len - 4,
278 .length = 5,
279 });
280 },
281 0b01 => {
282 if (ops.reg1 == .none) {
283 // JMP/CALL [imm]
284 const imm = isel.mir.instructions.items(.data)[inst].imm;
285 const ptr_size: Memory.PtrSize = switch (immOpSize(imm)) {
286 16 => .word_ptr,
287 else => .qword_ptr,
288 };
289 return lowerToMEnc(tag, RegisterOrMemory.mem(ptr_size, .{ .disp = imm }), isel.code) catch |err|
290 isel.failWithLoweringError(err);
291 }
292 // JMP/CALL reg
293 return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), isel.code) catch |err| isel.failWithLoweringError(err);
294 },
295 0b10 => {
296 // JMP/CALL r/m64
297 const imm = isel.mir.instructions.items(.data)[inst].imm;
298 return lowerToMEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
299 .disp = imm,
300 .base = ops.reg1,
301 }), isel.code) catch |err| isel.failWithLoweringError(err);
302 },
303 0b11 => return isel.fail("TODO unused JMP/CALL variant 0b11", .{}),
304 }
305}
306
307fn mirCondJmp(isel: *Isel, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
308 const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
309 const target = isel.mir.instructions.items(.data)[inst].inst;
310 const tag = switch (mir_tag) {
311 .cond_jmp_greater_less => switch (ops.flags) {
312 0b00 => Tag.jge,
313 0b01 => Tag.jg,
314 0b10 => Tag.jl,
315 0b11 => Tag.jle,
316 },
317 .cond_jmp_above_below => switch (ops.flags) {
318 0b00 => Tag.jae,
319 0b01 => Tag.ja,
320 0b10 => Tag.jb,
321 0b11 => Tag.jbe,
322 },
323 .cond_jmp_eq_ne => switch (@truncate(u1, ops.flags)) {
324 0b0 => Tag.jne,
325 0b1 => Tag.je,
326 },
327 else => unreachable,
328 };
329 const source = isel.code.items.len;
330 lowerToDEnc(tag, 0, isel.code) catch |err|
331 return isel.failWithLoweringError(err);
332 try isel.relocs.append(isel.bin_file.allocator, .{
333 .source = source,
334 .target = target,
335 .offset = isel.code.items.len - 4,
336 .length = 6,
337 });
338}
339
340fn mirCondSetByte(isel: *Isel, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void {
341 const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
342 const tag = switch (mir_tag) {
343 .cond_set_byte_greater_less => switch (ops.flags) {
344 0b00 => Tag.setge,
345 0b01 => Tag.setg,
346 0b10 => Tag.setl,
347 0b11 => Tag.setle,
348 },
349 .cond_set_byte_above_below => switch (ops.flags) {
350 0b00 => Tag.setae,
351 0b01 => Tag.seta,
352 0b10 => Tag.setb,
353 0b11 => Tag.setbe,
354 },
355 .cond_set_byte_eq_ne => switch (@truncate(u1, ops.flags)) {
356 0b0 => Tag.setne,
357 0b1 => Tag.sete,
358 },
359 else => unreachable,
360 };
361 return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1.to8()), isel.code) catch |err|
362 isel.failWithLoweringError(err);
363}
364
365fn mirTest(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
366 const tag = isel.mir.instructions.items(.tag)[inst];
367 assert(tag == .@"test");
368 const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
369 switch (ops.flags) {
370 0b00 => {
371 if (ops.reg2 == .none) {
372 // TEST r/m64, imm32
373 // MI
374 const imm = isel.mir.instructions.items(.data)[inst].imm;
375 if (ops.reg1.to64() == .rax) {
376 // TEST rax, imm32
377 // I
378 return lowerToIEnc(.@"test", imm, isel.code) catch |err|
379 isel.failWithLoweringError(err);
380 }
381 return lowerToMiEnc(.@"test", RegisterOrMemory.reg(ops.reg1), imm, isel.code) catch |err|
382 isel.failWithLoweringError(err);
383 }
384 // TEST r/m64, r64
385 return isel.fail("TODO TEST r/m64, r64", .{});
386 },
387 else => return isel.fail("TODO more TEST alternatives", .{}),
388 }
389}
390
391fn mirRet(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
392 const tag = isel.mir.instructions.items(.tag)[inst];
393 assert(tag == .ret);
394 const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
395 switch (ops.flags) {
396 0b00 => {
397 // RETF imm16
398 // I
399 const imm = isel.mir.instructions.items(.data)[inst].imm;
400 return lowerToIEnc(.ret_far, imm, isel.code) catch |err| isel.failWithLoweringError(err);
401 },
402 0b01 => {
403 return lowerToZoEnc(.ret_far, isel.code) catch |err| isel.failWithLoweringError(err);
404 },
405 0b10 => {
406 // RET imm16
407 // I
408 const imm = isel.mir.instructions.items(.data)[inst].imm;
409 return lowerToIEnc(.ret_near, imm, isel.code) catch |err| isel.failWithLoweringError(err);
410 },
411 0b11 => {
412 return lowerToZoEnc(.ret_near, isel.code) catch |err| isel.failWithLoweringError(err);
413 },
414 }
415}
416
417fn mirArith(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
418 const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
419 switch (ops.flags) {
420 0b00 => {
421 if (ops.reg2 == .none) {
422 // mov reg1, imm32
423 // MI
424 const imm = isel.mir.instructions.items(.data)[inst].imm;
425 return lowerToMiEnc(tag, RegisterOrMemory.reg(ops.reg1), imm, isel.code) catch |err|
426 isel.failWithLoweringError(err);
427 }
428 // mov reg1, reg2
429 // RM
430 return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), isel.code) catch |err|
431 isel.failWithLoweringError(err);
432 },
433 0b01 => {
434 // mov reg1, [reg2 + imm32]
435 // RM
436 const imm = isel.mir.instructions.items(.data)[inst].imm;
437 const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
438 return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
439 .disp = imm,
440 .base = src_reg,
441 }), isel.code) catch |err| isel.failWithLoweringError(err);
442 },
443 0b10 => {
444 if (ops.reg2 == .none) {
445 return isel.fail("TODO unused variant: mov reg1, none, 0b10", .{});
446 }
447 // mov [reg1 + imm32], reg2
448 // MR
449 const imm = isel.mir.instructions.items(.data)[inst].imm;
450 return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg2.size()), .{
451 .disp = imm,
452 .base = ops.reg1,
453 }), ops.reg2, isel.code) catch |err| isel.failWithLoweringError(err);
454 },
455 0b11 => {
456 return isel.fail("TODO unused variant: mov reg1, reg2, 0b11", .{});
457 },
458 }
459}
460
461fn mirArithMemImm(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
462 const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
463 assert(ops.reg2 == .none);
464 const payload = isel.mir.instructions.items(.data)[inst].payload;
465 const imm_pair = isel.mir.extraData(Mir.ImmPair, payload).data;
466 const ptr_size: Memory.PtrSize = switch (ops.flags) {
467 0b00 => .byte_ptr,
468 0b01 => .word_ptr,
469 0b10 => .dword_ptr,
470 0b11 => .qword_ptr,
471 };
472 return lowerToMiEnc(tag, RegisterOrMemory.mem(ptr_size, .{
473 .disp = imm_pair.dest_off,
474 .base = ops.reg1,
475 }), imm_pair.operand, isel.code) catch |err| isel.failWithLoweringError(err);
476}
477
478inline fn setRexWRegister(reg: Register) bool {
479 if (reg.size() == 64) return true;
480 return switch (reg) {
481 .ah, .bh, .ch, .dh => true,
482 else => false,
483 };
484}
485
486inline fn immOpSize(imm: i64) u8 {
487 blk: {
488 _ = math.cast(i8, imm) catch break :blk;
489 return 8;
490 }
491 blk: {
492 _ = math.cast(i16, imm) catch break :blk;
493 return 16;
494 }
495 blk: {
496 _ = math.cast(i32, imm) catch break :blk;
497 return 32;
498 }
499 return 64;
500}
501
502fn mirArithScaleSrc(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
503 const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
504 const scale = ops.flags;
505 const imm = isel.mir.instructions.items(.data)[inst].imm;
506 // OP reg1, [reg2 + scale*rcx + imm32]
507 const scale_index = ScaleIndex{
508 .scale = scale,
509 .index = .rcx,
510 };
511 return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
512 .disp = imm,
513 .base = ops.reg2,
514 .scale_index = scale_index,
515 }), isel.code) catch |err| isel.failWithLoweringError(err);
516}
517
518fn mirArithScaleDst(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
519 const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
520 const scale = ops.flags;
521 const imm = isel.mir.instructions.items(.data)[inst].imm;
522 const scale_index = ScaleIndex{
523 .scale = scale,
524 .index = .rax,
525 };
526 if (ops.reg2 == .none) {
527 // OP qword ptr [reg1 + scale*rax + 0], imm32
528 return lowerToMiEnc(tag, RegisterOrMemory.mem(.qword_ptr, .{
529 .disp = 0,
530 .base = ops.reg1,
531 .scale_index = scale_index,
532 }), imm, isel.code) catch |err| isel.failWithLoweringError(err);
533 }
534 // OP [reg1 + scale*rax + imm32], reg2
535 return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg2.size()), .{
536 .disp = imm,
537 .base = ops.reg1,
538 .scale_index = scale_index,
539 }), ops.reg2, isel.code) catch |err| isel.failWithLoweringError(err);
540}
541
542fn mirArithScaleImm(isel: *Isel, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
543 const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
544 const scale = ops.flags;
545 const payload = isel.mir.instructions.items(.data)[inst].payload;
546 const imm_pair = isel.mir.extraData(Mir.ImmPair, payload).data;
547 const scale_index = ScaleIndex{
548 .scale = scale,
549 .index = .rax,
550 };
551 // OP qword ptr [reg1 + scale*rax + imm32], imm32
552 return lowerToMiEnc(tag, RegisterOrMemory.mem(.qword_ptr, .{
553 .disp = imm_pair.dest_off,
554 .base = ops.reg1,
555 .scale_index = scale_index,
556 }), imm_pair.operand, isel.code) catch |err| isel.failWithLoweringError(err);
557}
558
559fn mirMovabs(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
560 const tag = isel.mir.instructions.items(.tag)[inst];
561 assert(tag == .movabs);
562 const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
563 const imm: i64 = if (ops.reg1.size() == 64) blk: {
564 const payload = isel.mir.instructions.items(.data)[inst].payload;
565 const imm = isel.mir.extraData(Mir.Imm64, payload).data;
566 break :blk @bitCast(i64, imm.decode());
567 } else isel.mir.instructions.items(.data)[inst].imm;
568 if (ops.flags == 0b00) {
569 // movabs reg, imm64
570 // OI
571 return lowerToOiEnc(.mov, ops.reg1, imm, isel.code) catch |err| isel.failWithLoweringError(err);
572 }
573 if (ops.reg1 == .none) {
574 // movabs moffs64, rax
575 // TD
576 return lowerToTdEnc(.mov, imm, ops.reg2, isel.code) catch |err| isel.failWithLoweringError(err);
577 }
578 // movabs rax, moffs64
579 // FD
580 return lowerToFdEnc(.mov, ops.reg1, imm, isel.code) catch |err| isel.failWithLoweringError(err);
581}
582
583fn mirIMulComplex(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
584 const tag = isel.mir.instructions.items(.tag)[inst];
585 assert(tag == .imul_complex);
586 const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
587 switch (ops.flags) {
588 0b00 => {
589 return lowerToRmEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), isel.code) catch |err|
590 isel.failWithLoweringError(err);
591 },
592 0b10 => {
593 const imm = isel.mir.instructions.items(.data)[inst].imm;
594 return lowerToRmiEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), imm, isel.code) catch |err|
595 isel.failWithLoweringError(err);
596 },
597 else => return isel.fail("TODO implement imul", .{}),
598 }
599}
600
601fn mirLea(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
602 const tag = isel.mir.instructions.items(.tag)[inst];
603 assert(tag == .lea);
604 const ops = Mir.Ops.decode(isel.mir.instructions.items(.ops)[inst]);
605 switch (ops.flags) {
606 0b00 => {
607 // lea reg1, [reg2 + imm32]
608 // RM
609 const imm = isel.mir.instructions.items(.data)[inst].imm;
610 const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
611 return lowerToRmEnc(
612 .lea,
613 ops.reg1,
614 RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
615 .disp = imm,
616 .base = src_reg,
617 }),
618 isel.code,
619 ) catch |err| isel.failWithLoweringError(err);
620 },
621 0b01 => {
622 // lea reg1, [rip + imm32]
623 // RM
624 const start_offset = isel.code.items.len;
625 lowerToRmEnc(
626 .lea,
627 ops.reg1,
628 RegisterOrMemory.rip(Memory.PtrSize.fromBits(ops.reg1.size()), 0),
629 isel.code,
630 ) catch |err| return isel.failWithLoweringError(err);
631 const end_offset = isel.code.items.len;
632 // Backpatch the displacement
633 const payload = isel.mir.instructions.items(.data)[inst].payload;
634 const imm = isel.mir.extraData(Mir.Imm64, payload).data.decode();
635 const disp = @intCast(i32, @intCast(i64, imm) - @intCast(i64, end_offset - start_offset));
636 mem.writeIntLittle(i32, isel.code.items[end_offset - 4 ..][0..4], disp);
637 },
638 0b10 => {
639 // lea reg1, [rip + reloc]
640 // RM
641 lowerToRmEnc(
642 .lea,
643 ops.reg1,
644 RegisterOrMemory.rip(Memory.PtrSize.fromBits(ops.reg1.size()), 0),
645 isel.code,
646 ) catch |err| return isel.failWithLoweringError(err);
647 const end_offset = isel.code.items.len;
648 const got_entry = isel.mir.instructions.items(.data)[inst].got_entry;
649 if (isel.bin_file.cast(link.File.MachO)) |macho_file| {
650 // TODO I think the reloc might be in the wrong place.
651 const decl = macho_file.active_decl.?;
652 try decl.link.macho.relocs.append(isel.bin_file.allocator, .{
653 .offset = @intCast(u32, end_offset - 4),
654 .target = .{ .local = got_entry },
655 .addend = 0,
656 .subtractor = null,
657 .pcrel = true,
658 .length = 2,
659 .@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_GOT),
660 });
661 } else {
662 return isel.fail(
663 "TODO implement lea reg, [rip + reloc] for linking backends different than MachO",
664 .{},
665 );
666 }
667 },
668 0b11 => {
669 // lea reg, [rbp + rcx + imm32]
670 const imm = isel.mir.instructions.items(.data)[inst].imm;
671 const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2;
672 const scale_index = ScaleIndex{
673 .scale = 0,
674 .index = .rcx,
675 };
676 return lowerToRmEnc(
677 .lea,
678 ops.reg1,
679 RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{
680 .disp = imm,
681 .base = src_reg,
682 .scale_index = scale_index,
683 }),
684 isel.code,
685 ) catch |err| isel.failWithLoweringError(err);
686 },
687 }
688}
689
690fn mirCallExtern(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
691 const tag = isel.mir.instructions.items(.tag)[inst];
692 assert(tag == .call_extern);
693 const n_strx = isel.mir.instructions.items(.data)[inst].extern_fn;
694 const offset = blk: {
695 // callq
696 lowerToDEnc(.call_near, 0, isel.code) catch |err|
697 return isel.failWithLoweringError(err);
698 break :blk @intCast(u32, isel.code.items.len) - 4;
699 };
700 if (isel.bin_file.cast(link.File.MachO)) |macho_file| {
701 // Add relocation to the decl.
702 try macho_file.active_decl.?.link.macho.relocs.append(isel.bin_file.allocator, .{
703 .offset = offset,
704 .target = .{ .global = n_strx },
705 .addend = 0,
706 .subtractor = null,
707 .pcrel = true,
708 .length = 2,
709 .@"type" = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_BRANCH),
710 });
711 } else {
712 return isel.fail("TODO implement call_extern for linking backends different than MachO", .{});
713 }
714}
715
716fn mirDbgLine(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
717 const tag = isel.mir.instructions.items(.tag)[inst];
718 assert(tag == .dbg_line);
719 const payload = isel.mir.instructions.items(.data)[inst].payload;
720 const dbg_line_column = isel.mir.extraData(Mir.DbgLineColumn, payload).data;
721 try isel.dbgAdvancePCAndLine(dbg_line_column.line, dbg_line_column.column);
722}
723
724fn dbgAdvancePCAndLine(isel: *Isel, line: u32, column: u32) InnerError!void {
725 const delta_line = @intCast(i32, line) - @intCast(i32, isel.prev_di_line);
726 const delta_pc: usize = isel.code.items.len - isel.prev_di_pc;
727 switch (isel.debug_output) {
728 .dwarf => |dbg_out| {
729 // TODO Look into using the DWARF special opcodes to compress this data.
730 // It lets you emit single-byte opcodes that add different numbers to
731 // both the PC and the line number at the same time.
732 try dbg_out.dbg_line.ensureUnusedCapacity(11);
733 dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_pc);
734 leb128.writeULEB128(dbg_out.dbg_line.writer(), delta_pc) catch unreachable;
735 if (delta_line != 0) {
736 dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.advance_line);
737 leb128.writeILEB128(dbg_out.dbg_line.writer(), delta_line) catch unreachable;
738 }
739 dbg_out.dbg_line.appendAssumeCapacity(DW.LNS.copy);
740 isel.prev_di_pc = isel.code.items.len;
741 isel.prev_di_line = line;
742 isel.prev_di_column = column;
743 isel.prev_di_pc = isel.code.items.len;
744 },
745 .plan9 => |dbg_out| {
746 if (delta_pc <= 0) return; // only do this when the pc changes
747 // we have already checked the target in the linker to make sure it is compatable
748 const quant = @import("../../link/Plan9/aout.zig").getPCQuant(isel.target.cpu.arch) catch unreachable;
749
750 // increasing the line number
751 try @import("../../link/Plan9.zig").changeLine(dbg_out.dbg_line, delta_line);
752 // increasing the pc
753 const d_pc_p9 = @intCast(i64, delta_pc) - quant;
754 if (d_pc_p9 > 0) {
755 // minus one because if its the last one, we want to leave space to change the line which is one quanta
756 var diff = @divExact(d_pc_p9, quant) - quant;
757 while (diff > 0) {
758 if (diff < 64) {
759 try dbg_out.dbg_line.append(@intCast(u8, diff + 128));
760 diff = 0;
761 } else {
762 try dbg_out.dbg_line.append(@intCast(u8, 64 + 128));
763 diff -= 64;
764 }
765 }
766 if (dbg_out.pcop_change_index.*) |pci|
767 dbg_out.dbg_line.items[pci] += 1;
768 dbg_out.pcop_change_index.* = @intCast(u32, dbg_out.dbg_line.items.len - 1);
769 } else if (d_pc_p9 == 0) {
770 // we don't need to do anything, because adding the quant does it for us
771 } else unreachable;
772 if (dbg_out.start_line.* == null)
773 dbg_out.start_line.* = isel.prev_di_line;
774 dbg_out.end_line.* = line;
775 // only do this if the pc changed
776 isel.prev_di_line = line;
777 isel.prev_di_column = column;
778 isel.prev_di_pc = isel.code.items.len;
779 },
780 .none => {},
781 }
782}
783
784fn mirDbgPrologueEnd(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
785 const tag = isel.mir.instructions.items(.tag)[inst];
786 assert(tag == .dbg_prologue_end);
787 switch (isel.debug_output) {
788 .dwarf => |dbg_out| {
789 try dbg_out.dbg_line.append(DW.LNS.set_prologue_end);
790 try isel.dbgAdvancePCAndLine(isel.prev_di_line, isel.prev_di_column);
791 },
792 .plan9 => {},
793 .none => {},
794 }
795}
796
797fn mirDbgEpilogueBegin(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
798 const tag = isel.mir.instructions.items(.tag)[inst];
799 assert(tag == .dbg_epilogue_begin);
800 switch (isel.debug_output) {
801 .dwarf => |dbg_out| {
802 try dbg_out.dbg_line.append(DW.LNS.set_epilogue_begin);
803 try isel.dbgAdvancePCAndLine(isel.prev_di_line, isel.prev_di_column);
804 },
805 .plan9 => {},
806 .none => {},
807 }
808}
809
810fn mirArgDbgInfo(isel: *Isel, inst: Mir.Inst.Index) InnerError!void {
811 const tag = isel.mir.instructions.items(.tag)[inst];
812 assert(tag == .arg_dbg_info);
813 const payload = isel.mir.instructions.items(.data)[inst].payload;
814 const arg_dbg_info = isel.mir.extraData(Mir.ArgDbgInfo, payload).data;
815 const mcv = isel.mir.function.args[arg_dbg_info.arg_index];
816 try isel.genArgDbgInfo(arg_dbg_info.air_inst, mcv);
817}
818
819fn genArgDbgInfo(isel: *Isel, inst: Air.Inst.Index, mcv: MCValue) !void {
820 const ty_str = isel.mir.function.air.instructions.items(.data)[inst].ty_str;
821 const zir = &isel.mir.function.mod_fn.owner_decl.getFileScope().zir;
822 const name = zir.nullTerminatedString(ty_str.str);
823 const name_with_null = name.ptr[0 .. name.len + 1];
824 const ty = isel.mir.function.air.getRefType(ty_str.ty);
825
826 switch (mcv) {
827 .register => |reg| {
828 switch (isel.debug_output) {
829 .dwarf => |dbg_out| {
830 try dbg_out.dbg_info.ensureUnusedCapacity(3);
831 dbg_out.dbg_info.appendAssumeCapacity(link.File.Elf.abbrev_parameter);
832 dbg_out.dbg_info.appendSliceAssumeCapacity(&[2]u8{ // DW.AT.location, DW.FORM.exprloc
833 1, // ULEB128 dwarf expression length
834 reg.dwarfLocOp(),
835 });
836 try dbg_out.dbg_info.ensureUnusedCapacity(5 + name_with_null.len);
837 try isel.addDbgInfoTypeReloc(ty); // DW.AT.type, DW.FORM.ref4
838 dbg_out.dbg_info.appendSliceAssumeCapacity(name_with_null); // DW.AT.name, DW.FORM.string
839 },
840 .plan9 => {},
841 .none => {},
842 }
843 },
844 .stack_offset => {
845 switch (isel.debug_output) {
846 .dwarf => {},
847 .plan9 => {},
848 .none => {},
849 }
850 },
851 else => {},
852 }
853}
854
855/// Adds a Type to the .debug_info at the current position. The bytes will be populated later,
856/// after codegen for this symbol is done.
857fn addDbgInfoTypeReloc(isel: *Isel, ty: Type) !void {
858 switch (isel.debug_output) {
859 .dwarf => |dbg_out| {
860 assert(ty.hasCodeGenBits());
861 const index = dbg_out.dbg_info.items.len;
862 try dbg_out.dbg_info.resize(index + 4); // DW.AT.type, DW.FORM.ref4
863
864 const gop = try dbg_out.dbg_info_type_relocs.getOrPut(isel.bin_file.allocator, ty);
865 if (!gop.found_existing) {
866 gop.value_ptr.* = .{
867 .off = undefined,
868 .relocs = .{},
869 };
870 }
871 try gop.value_ptr.relocs.append(isel.bin_file.allocator, @intCast(u32, index));
872 },
873 .plan9 => {},
874 .none => {},
875 }
876}
877
878const Tag = enum {
879 adc,
880 add,
881 sub,
882 xor,
883 @"and",
884 @"or",
885 sbb,
886 cmp,
887 mov,
888 lea,
889 jmp_near,
890 call_near,
891 push,
892 pop,
893 @"test",
894 brk,
895 nop,
896 imul,
897 syscall,
898 ret_near,
899 ret_far,
900 jo,
901 jno,
902 jb,
903 jbe,
904 jc,
905 jnae,
906 jnc,
907 jae,
908 je,
909 jz,
910 jne,
911 jnz,
912 jna,
913 jnb,
914 jnbe,
915 ja,
916 js,
917 jns,
918 jpe,
919 jp,
920 jpo,
921 jnp,
922 jnge,
923 jl,
924 jge,
925 jnl,
926 jle,
927 jng,
928 jg,
929 jnle,
930 seto,
931 setno,
932 setb,
933 setc,
934 setnae,
935 setnb,
936 setnc,
937 setae,
938 sete,
939 setz,
940 setne,
941 setnz,
942 setbe,
943 setna,
944 seta,
945 setnbe,
946 sets,
947 setns,
948 setp,
949 setpe,
950 setnp,
951 setop,
952 setl,
953 setnge,
954 setnl,
955 setge,
956 setle,
957 setng,
958 setnle,
959 setg,
960
961 fn isSetCC(tag: Tag) bool {
962 return switch (tag) {
963 .seto,
964 .setno,
965 .setb,
966 .setc,
967 .setnae,
968 .setnb,
969 .setnc,
970 .setae,
971 .sete,
972 .setz,
973 .setne,
974 .setnz,
975 .setbe,
976 .setna,
977 .seta,
978 .setnbe,
979 .sets,
980 .setns,
981 .setp,
982 .setpe,
983 .setnp,
984 .setop,
985 .setl,
986 .setnge,
987 .setnl,
988 .setge,
989 .setle,
990 .setng,
991 .setnle,
992 .setg,
993 => true,
994 else => false,
995 };
996 }
997};
998
999const Encoding = enum {
1000 /// OP
1001 zo,
1002
1003 /// OP rel32
1004 d,
1005
1006 /// OP r/m64
1007 m,
1008
1009 /// OP r64
1010 o,
1011
1012 /// OP imm32
1013 i,
1014
1015 /// OP r/m64, imm32
1016 mi,
1017
1018 /// OP r/m64, r64
1019 mr,
1020
1021 /// OP r64, r/m64
1022 rm,
1023
1024 /// OP r64, imm64
1025 oi,
1026
1027 /// OP al/ax/eax/rax, moffs
1028 fd,
1029
1030 /// OP moffs, al/ax/eax/rax
1031 td,
1032
1033 /// OP r64, r/m64, imm32
1034 rmi,
1035};
1036
1037const OpCode = union(enum) {
1038 one_byte: u8,
1039 two_byte: struct { _1: u8, _2: u8 },
1040
1041 fn oneByte(opc: u8) OpCode {
1042 return .{ .one_byte = opc };
1043 }
1044
1045 fn twoByte(opc1: u8, opc2: u8) OpCode {
1046 return .{ .two_byte = .{ ._1 = opc1, ._2 = opc2 } };
1047 }
1048
1049 fn encode(opc: OpCode, encoder: Encoder) void {
1050 switch (opc) {
1051 .one_byte => |v| encoder.opcode_1byte(v),
1052 .two_byte => |v| encoder.opcode_2byte(v._1, v._2),
1053 }
1054 }
1055
1056 fn encodeWithReg(opc: OpCode, encoder: Encoder, reg: Register) void {
1057 assert(opc == .one_byte);
1058 encoder.opcode_withReg(opc.one_byte, reg.lowId());
1059 }
1060};
1061
1062inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode {
1063 switch (enc) {
1064 .zo => return switch (tag) {
1065 .ret_near => OpCode.oneByte(0xc3),
1066 .ret_far => OpCode.oneByte(0xcb),
1067 .brk => OpCode.oneByte(0xcc),
1068 .nop => OpCode.oneByte(0x90),
1069 .syscall => OpCode.twoByte(0x0f, 0x05),
1070 else => null,
1071 },
1072 .d => return switch (tag) {
1073 .jmp_near => OpCode.oneByte(0xe9),
1074 .call_near => OpCode.oneByte(0xe8),
1075 .jo => if (is_one_byte) OpCode.oneByte(0x70) else OpCode.twoByte(0x0f, 0x80),
1076 .jno => if (is_one_byte) OpCode.oneByte(0x71) else OpCode.twoByte(0x0f, 0x81),
1077 .jb, .jc, .jnae => if (is_one_byte) OpCode.oneByte(0x72) else OpCode.twoByte(0x0f, 0x82),
1078 .jnb, .jnc, .jae => if (is_one_byte) OpCode.oneByte(0x73) else OpCode.twoByte(0x0f, 0x83),
1079 .je, .jz => if (is_one_byte) OpCode.oneByte(0x74) else OpCode.twoByte(0x0f, 0x84),
1080 .jne, .jnz => if (is_one_byte) OpCode.oneByte(0x75) else OpCode.twoByte(0x0f, 0x85),
1081 .jna, .jbe => if (is_one_byte) OpCode.oneByte(0x76) else OpCode.twoByte(0x0f, 0x86),
1082 .jnbe, .ja => if (is_one_byte) OpCode.oneByte(0x77) else OpCode.twoByte(0x0f, 0x87),
1083 .js => if (is_one_byte) OpCode.oneByte(0x78) else OpCode.twoByte(0x0f, 0x88),
1084 .jns => if (is_one_byte) OpCode.oneByte(0x79) else OpCode.twoByte(0x0f, 0x89),
1085 .jpe, .jp => if (is_one_byte) OpCode.oneByte(0x7a) else OpCode.twoByte(0x0f, 0x8a),
1086 .jpo, .jnp => if (is_one_byte) OpCode.oneByte(0x7b) else OpCode.twoByte(0x0f, 0x8b),
1087 .jnge, .jl => if (is_one_byte) OpCode.oneByte(0x7c) else OpCode.twoByte(0x0f, 0x8c),
1088 .jge, .jnl => if (is_one_byte) OpCode.oneByte(0x7d) else OpCode.twoByte(0x0f, 0x8d),
1089 .jle, .jng => if (is_one_byte) OpCode.oneByte(0x7e) else OpCode.twoByte(0x0f, 0x8e),
1090 .jg, .jnle => if (is_one_byte) OpCode.oneByte(0x7f) else OpCode.twoByte(0x0f, 0x8f),
1091 else => null,
1092 },
1093 .m => return switch (tag) {
1094 .jmp_near, .call_near, .push => OpCode.oneByte(0xff),
1095 .pop => OpCode.oneByte(0x8f),
1096 .seto => OpCode.twoByte(0x0f, 0x90),
1097 .setno => OpCode.twoByte(0x0f, 0x91),
1098 .setb, .setc, .setnae => OpCode.twoByte(0x0f, 0x92),
1099 .setnb, .setnc, .setae => OpCode.twoByte(0x0f, 0x93),
1100 .sete, .setz => OpCode.twoByte(0x0f, 0x94),
1101 .setne, .setnz => OpCode.twoByte(0x0f, 0x95),
1102 .setbe, .setna => OpCode.twoByte(0x0f, 0x96),
1103 .seta, .setnbe => OpCode.twoByte(0x0f, 0x97),
1104 .sets => OpCode.twoByte(0x0f, 0x98),
1105 .setns => OpCode.twoByte(0x0f, 0x99),
1106 .setp, .setpe => OpCode.twoByte(0x0f, 0x9a),
1107 .setnp, .setop => OpCode.twoByte(0x0f, 0x9b),
1108 .setl, .setnge => OpCode.twoByte(0x0f, 0x9c),
1109 .setnl, .setge => OpCode.twoByte(0x0f, 0x9d),
1110 .setle, .setng => OpCode.twoByte(0x0f, 0x9e),
1111 .setnle, .setg => OpCode.twoByte(0x0f, 0x9f),
1112 else => null,
1113 },
1114 .o => return switch (tag) {
1115 .push => OpCode.oneByte(0x50),
1116 .pop => OpCode.oneByte(0x58),
1117 else => null,
1118 },
1119 .i => return switch (tag) {
1120 .push => OpCode.oneByte(if (is_one_byte) 0x6a else 0x68),
1121 .@"test" => OpCode.oneByte(if (is_one_byte) 0xa8 else 0xa9),
1122 .ret_near => OpCode.oneByte(0xc2),
1123 .ret_far => OpCode.oneByte(0xca),
1124 else => null,
1125 },
1126 .mi => return switch (tag) {
1127 .adc, .add, .sub, .xor, .@"and", .@"or", .sbb, .cmp => OpCode.oneByte(if (is_one_byte) 0x80 else 0x81),
1128 .mov => OpCode.oneByte(if (is_one_byte) 0xc6 else 0xc7),
1129 .@"test" => OpCode.oneByte(if (is_one_byte) 0xf6 else 0xf7),
1130 else => null,
1131 },
1132 .mr => return switch (tag) {
1133 .adc => OpCode.oneByte(if (is_one_byte) 0x10 else 0x11),
1134 .add => OpCode.oneByte(if (is_one_byte) 0x00 else 0x01),
1135 .sub => OpCode.oneByte(if (is_one_byte) 0x28 else 0x29),
1136 .xor => OpCode.oneByte(if (is_one_byte) 0x30 else 0x31),
1137 .@"and" => OpCode.oneByte(if (is_one_byte) 0x20 else 0x21),
1138 .@"or" => OpCode.oneByte(if (is_one_byte) 0x08 else 0x09),
1139 .sbb => OpCode.oneByte(if (is_one_byte) 0x18 else 0x19),
1140 .cmp => OpCode.oneByte(if (is_one_byte) 0x38 else 0x39),
1141 .mov => OpCode.oneByte(if (is_one_byte) 0x88 else 0x89),
1142 else => null,
1143 },
1144 .rm => return switch (tag) {
1145 .adc => OpCode.oneByte(if (is_one_byte) 0x12 else 0x13),
1146 .add => OpCode.oneByte(if (is_one_byte) 0x02 else 0x03),
1147 .sub => OpCode.oneByte(if (is_one_byte) 0x2a else 0x2b),
1148 .xor => OpCode.oneByte(if (is_one_byte) 0x32 else 0x33),
1149 .@"and" => OpCode.oneByte(if (is_one_byte) 0x22 else 0x23),
1150 .@"or" => OpCode.oneByte(if (is_one_byte) 0x0b else 0x0b),
1151 .sbb => OpCode.oneByte(if (is_one_byte) 0x1a else 0x1b),
1152 .cmp => OpCode.oneByte(if (is_one_byte) 0x3a else 0x3b),
1153 .mov => OpCode.oneByte(if (is_one_byte) 0x8a else 0x8b),
1154 .lea => OpCode.oneByte(if (is_one_byte) 0x8c else 0x8d),
1155 .imul => OpCode.twoByte(0x0f, 0xaf),
1156 else => null,
1157 },
1158 .oi => return switch (tag) {
1159 .mov => OpCode.oneByte(if (is_one_byte) 0xb0 else 0xb8),
1160 else => null,
1161 },
1162 .fd => return switch (tag) {
1163 .mov => OpCode.oneByte(if (is_one_byte) 0xa0 else 0xa1),
1164 else => null,
1165 },
1166 .td => return switch (tag) {
1167 .mov => OpCode.oneByte(if (is_one_byte) 0xa2 else 0xa3),
1168 else => null,
1169 },
1170 .rmi => return switch (tag) {
1171 .imul => OpCode.oneByte(if (is_one_byte) 0x6b else 0x69),
1172 else => null,
1173 },
1174 }
1175}
1176
1177inline fn getModRmExt(tag: Tag) ?u3 {
1178 return switch (tag) {
1179 .adc => 0x2,
1180 .add => 0x0,
1181 .sub => 0x5,
1182 .xor => 0x6,
1183 .@"and" => 0x4,
1184 .@"or" => 0x1,
1185 .sbb => 0x3,
1186 .cmp => 0x7,
1187 .mov => 0x0,
1188 .jmp_near => 0x4,
1189 .call_near => 0x2,
1190 .push => 0x6,
1191 .pop => 0x0,
1192 .@"test" => 0x0,
1193 .seto,
1194 .setno,
1195 .setb,
1196 .setc,
1197 .setnae,
1198 .setnb,
1199 .setnc,
1200 .setae,
1201 .sete,
1202 .setz,
1203 .setne,
1204 .setnz,
1205 .setbe,
1206 .setna,
1207 .seta,
1208 .setnbe,
1209 .sets,
1210 .setns,
1211 .setp,
1212 .setpe,
1213 .setnp,
1214 .setop,
1215 .setl,
1216 .setnge,
1217 .setnl,
1218 .setge,
1219 .setle,
1220 .setng,
1221 .setnle,
1222 .setg,
1223 => 0x0,
1224 else => null,
1225 };
1226}
1227
1228const ScaleIndex = struct {
1229 scale: u2,
1230 index: Register,
1231};
1232
1233const Memory = struct {
1234 base: ?Register,
1235 rip: bool = false,
1236 disp: i32,
1237 ptr_size: PtrSize,
1238 scale_index: ?ScaleIndex = null,
1239
1240 const PtrSize = enum {
1241 byte_ptr,
1242 word_ptr,
1243 dword_ptr,
1244 qword_ptr,
1245
1246 fn fromBits(in_bits: u64) PtrSize {
1247 return switch (in_bits) {
1248 8 => .byte_ptr,
1249 16 => .word_ptr,
1250 32 => .dword_ptr,
1251 64 => .qword_ptr,
1252 else => unreachable,
1253 };
1254 }
1255
1256 /// Returns size in bits.
1257 fn size(ptr_size: PtrSize) u64 {
1258 return switch (ptr_size) {
1259 .byte_ptr => 8,
1260 .word_ptr => 16,
1261 .dword_ptr => 32,
1262 .qword_ptr => 64,
1263 };
1264 }
1265 };
1266
1267 fn encode(mem_op: Memory, encoder: Encoder, operand: u3) void {
1268 if (mem_op.base) |base| {
1269 const dst = base.lowId();
1270 const src = operand;
1271 if (dst == 4 or mem_op.scale_index != null) {
1272 if (mem_op.disp == 0 and dst != 5) {
1273 encoder.modRm_SIBDisp0(src);
1274 if (mem_op.scale_index) |si| {
1275 encoder.sib_scaleIndexBase(si.scale, si.index.lowId(), dst);
1276 } else {
1277 encoder.sib_base(dst);
1278 }
1279 } else if (immOpSize(mem_op.disp) == 8) {
1280 encoder.modRm_SIBDisp8(src);
1281 if (mem_op.scale_index) |si| {
1282 encoder.sib_scaleIndexBaseDisp8(si.scale, si.index.lowId(), dst);
1283 } else {
1284 encoder.sib_baseDisp8(dst);
1285 }
1286 encoder.disp8(@intCast(i8, mem_op.disp));
1287 } else {
1288 encoder.modRm_SIBDisp32(src);
1289 if (mem_op.scale_index) |si| {
1290 encoder.sib_scaleIndexBaseDisp32(si.scale, si.index.lowId(), dst);
1291 } else {
1292 encoder.sib_baseDisp32(dst);
1293 }
1294 encoder.disp32(mem_op.disp);
1295 }
1296 } else {
1297 if (mem_op.disp == 0) {
1298 encoder.modRm_indirectDisp0(src, dst);
1299 } else if (immOpSize(mem_op.disp) == 8) {
1300 encoder.modRm_indirectDisp8(src, dst);
1301 encoder.disp8(@intCast(i8, mem_op.disp));
1302 } else {
1303 encoder.modRm_indirectDisp32(src, dst);
1304 encoder.disp32(mem_op.disp);
1305 }
1306 }
1307 } else {
1308 if (mem_op.rip) {
1309 encoder.modRm_RIPDisp32(operand);
1310 } else {
1311 encoder.modRm_SIBDisp0(operand);
1312 if (mem_op.scale_index) |si| {
1313 encoder.sib_scaleIndexDisp32(si.scale, si.index.lowId());
1314 } else {
1315 encoder.sib_disp32();
1316 }
1317 }
1318 encoder.disp32(mem_op.disp);
1319 }
1320 }
1321};
1322
1323fn encodeImm(encoder: Encoder, imm: i32, size: u64) void {
1324 switch (size) {
1325 8 => encoder.imm8(@intCast(i8, imm)),
1326 16 => encoder.imm16(@intCast(i16, imm)),
1327 32, 64 => encoder.imm32(imm),
1328 else => unreachable,
1329 }
1330}
1331
1332const RegisterOrMemory = union(enum) {
1333 register: Register,
1334 memory: Memory,
1335
1336 fn reg(register: Register) RegisterOrMemory {
1337 return .{ .register = register };
1338 }
1339
1340 fn mem(ptr_size: Memory.PtrSize, args: struct {
1341 disp: i32,
1342 base: ?Register = null,
1343 scale_index: ?ScaleIndex = null,
1344 }) RegisterOrMemory {
1345 return .{
1346 .memory = .{
1347 .base = args.base,
1348 .disp = args.disp,
1349 .ptr_size = ptr_size,
1350 .scale_index = args.scale_index,
1351 },
1352 };
1353 }
1354
1355 fn rip(ptr_size: Memory.PtrSize, disp: i32) RegisterOrMemory {
1356 return .{
1357 .memory = .{
1358 .base = null,
1359 .rip = true,
1360 .disp = disp,
1361 .ptr_size = ptr_size,
1362 },
1363 };
1364 }
1365};
1366
1367const LoweringError = error{
1368 OutOfMemory,
1369 Overflow,
1370 OperandSizeMismatch,
1371 RaxOperandExpected,
1372};
1373
1374fn lowerToZoEnc(tag: Tag, code: *std.ArrayList(u8)) LoweringError!void {
1375 const opc = getOpCode(tag, .zo, false).?;
1376 const encoder = try Encoder.init(code, 1);
1377 opc.encode(encoder);
1378}
1379
1380fn lowerToIEnc(tag: Tag, imm: i32, code: *std.ArrayList(u8)) LoweringError!void {
1381 if (tag == .ret_far or tag == .ret_near) {
1382 const encoder = try Encoder.init(code, 3);
1383 const opc = getOpCode(tag, .i, false).?;
1384 opc.encode(encoder);
1385 encoder.imm16(@intCast(i16, imm));
1386 return;
1387 }
1388 const opc = getOpCode(tag, .i, immOpSize(imm) == 8).?;
1389 const encoder = try Encoder.init(code, 5);
1390 if (immOpSize(imm) == 16) {
1391 encoder.prefix16BitMode();
1392 }
1393 opc.encode(encoder);
1394 encodeImm(encoder, imm, immOpSize(imm));
1395}
1396
1397fn lowerToOEnc(tag: Tag, reg: Register, code: *std.ArrayList(u8)) LoweringError!void {
1398 if (reg.size() != 16 and reg.size() != 64) {
1399 return error.OperandSizeMismatch; // TODO correct for push/pop, but is it universal?
1400 }
1401 const opc = getOpCode(tag, .o, false).?;
1402 const encoder = try Encoder.init(code, 3);
1403 if (reg.size() == 16) {
1404 encoder.prefix16BitMode();
1405 }
1406 encoder.rex(.{
1407 .w = false,
1408 .b = reg.isExtended(),
1409 });
1410 opc.encodeWithReg(encoder, reg);
1411}
1412
1413fn lowerToDEnc(tag: Tag, imm: i32, code: *std.ArrayList(u8)) LoweringError!void {
1414 const opc = getOpCode(tag, .d, false).?;
1415 const encoder = try Encoder.init(code, 6);
1416 opc.encode(encoder);
1417 encoder.imm32(imm);
1418}
1419
1420fn lowerToMEnc(tag: Tag, reg_or_mem: RegisterOrMemory, code: *std.ArrayList(u8)) LoweringError!void {
1421 const opc = getOpCode(tag, .m, false).?;
1422 const modrm_ext = getModRmExt(tag).?;
1423 switch (reg_or_mem) {
1424 .register => |reg| {
1425 const op_size_mismatch = blk: {
1426 if (tag.isSetCC() and reg.size() == 8)
1427 break :blk false;
1428 break :blk reg.size() != 64 and reg.size() != 16;
1429 };
1430 if (op_size_mismatch) {
1431 return error.OperandSizeMismatch;
1432 }
1433 const encoder = try Encoder.init(code, 4);
1434 if (reg.size() == 16) {
1435 encoder.prefix16BitMode();
1436 }
1437 encoder.rex(.{
1438 .w = switch (reg) {
1439 .ah, .bh, .ch, .dh => true,
1440 else => false,
1441 },
1442 .b = reg.isExtended(),
1443 });
1444 opc.encode(encoder);
1445 encoder.modRm_direct(modrm_ext, reg.lowId());
1446 },
1447 .memory => |mem_op| {
1448 if (mem_op.ptr_size != .qword_ptr and mem_op.ptr_size != .word_ptr) {
1449 return error.OperandSizeMismatch;
1450 }
1451 const encoder = try Encoder.init(code, 8);
1452 if (mem_op.ptr_size == .word_ptr) {
1453 encoder.prefix16BitMode();
1454 }
1455 if (mem_op.base) |base| {
1456 if (base.size() != 64) {
1457 return error.OperandSizeMismatch;
1458 }
1459 encoder.rex(.{
1460 .w = false,
1461 .b = base.isExtended(),
1462 });
1463 }
1464 opc.encode(encoder);
1465 mem_op.encode(encoder, modrm_ext);
1466 },
1467 }
1468}
1469
1470fn lowerToTdEnc(tag: Tag, moffs: i64, reg: Register, code: *std.ArrayList(u8)) LoweringError!void {
1471 return lowerToTdFdEnc(tag, reg, moffs, code, true);
1472}
1473
1474fn lowerToFdEnc(tag: Tag, reg: Register, moffs: i64, code: *std.ArrayList(u8)) LoweringError!void {
1475 return lowerToTdFdEnc(tag, reg, moffs, code, false);
1476}
1477
1478fn lowerToTdFdEnc(tag: Tag, reg: Register, moffs: i64, code: *std.ArrayList(u8), td: bool) LoweringError!void {
1479 if (reg.lowId() != Register.rax.lowId()) {
1480 return error.RaxOperandExpected;
1481 }
1482 if (reg.size() != immOpSize(moffs)) {
1483 return error.OperandSizeMismatch;
1484 }
1485 const opc = if (td)
1486 getOpCode(tag, .td, reg.size() == 8).?
1487 else
1488 getOpCode(tag, .fd, reg.size() == 8).?;
1489 const encoder = try Encoder.init(code, 10);
1490 if (reg.size() == 16) {
1491 encoder.prefix16BitMode();
1492 }
1493 encoder.rex(.{
1494 .w = setRexWRegister(reg),
1495 });
1496 opc.encode(encoder);
1497 switch (reg.size()) {
1498 8 => {
1499 const moffs8 = try math.cast(i8, moffs);
1500 encoder.imm8(moffs8);
1501 },
1502 16 => {
1503 const moffs16 = try math.cast(i16, moffs);
1504 encoder.imm16(moffs16);
1505 },
1506 32 => {
1507 const moffs32 = try math.cast(i32, moffs);
1508 encoder.imm32(moffs32);
1509 },
1510 64 => {
1511 encoder.imm64(@bitCast(u64, moffs));
1512 },
1513 else => unreachable,
1514 }
1515}
1516
1517fn lowerToOiEnc(tag: Tag, reg: Register, imm: i64, code: *std.ArrayList(u8)) LoweringError!void {
1518 if (reg.size() != immOpSize(imm)) {
1519 return error.OperandSizeMismatch;
1520 }
1521 const opc = getOpCode(tag, .oi, reg.size() == 8).?;
1522 const encoder = try Encoder.init(code, 10);
1523 if (reg.size() == 16) {
1524 encoder.prefix16BitMode();
1525 }
1526 encoder.rex(.{
1527 .w = setRexWRegister(reg),
1528 .b = reg.isExtended(),
1529 });
1530 opc.encodeWithReg(encoder, reg);
1531 switch (reg.size()) {
1532 8 => {
1533 const imm8 = try math.cast(i8, imm);
1534 encoder.imm8(imm8);
1535 },
1536 16 => {
1537 const imm16 = try math.cast(i16, imm);
1538 encoder.imm16(imm16);
1539 },
1540 32 => {
1541 const imm32 = try math.cast(i32, imm);
1542 encoder.imm32(imm32);
1543 },
1544 64 => {
1545 encoder.imm64(@bitCast(u64, imm));
1546 },
1547 else => unreachable,
1548 }
1549}
1550
1551fn lowerToMiEnc(tag: Tag, reg_or_mem: RegisterOrMemory, imm: i32, code: *std.ArrayList(u8)) LoweringError!void {
1552 const modrm_ext = getModRmExt(tag).?;
1553 switch (reg_or_mem) {
1554 .register => |dst_reg| {
1555 const opc = getOpCode(tag, .mi, dst_reg.size() == 8).?;
1556 const encoder = try Encoder.init(code, 7);
1557 if (dst_reg.size() == 16) {
1558 // 0x66 prefix switches to the non-default size; here we assume a switch from
1559 // the default 32bits to 16bits operand-size.
1560 // More info: https://www.cs.uni-potsdam.de/desn/lehre/ss15/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf#page=32&zoom=auto,-159,773
1561 encoder.prefix16BitMode();
1562 }
1563 encoder.rex(.{
1564 .w = setRexWRegister(dst_reg),
1565 .b = dst_reg.isExtended(),
1566 });
1567 opc.encode(encoder);
1568 encoder.modRm_direct(modrm_ext, dst_reg.lowId());
1569 encodeImm(encoder, imm, dst_reg.size());
1570 },
1571 .memory => |dst_mem| {
1572 const opc = getOpCode(tag, .mi, dst_mem.ptr_size == .byte_ptr).?;
1573 const encoder = try Encoder.init(code, 12);
1574 if (dst_mem.ptr_size == .word_ptr) {
1575 encoder.prefix16BitMode();
1576 }
1577 if (dst_mem.base) |base| {
1578 if (base.size() != 64) {
1579 return error.OperandSizeMismatch;
1580 }
1581 encoder.rex(.{
1582 .w = dst_mem.ptr_size == .qword_ptr,
1583 .b = base.isExtended(),
1584 });
1585 } else {
1586 encoder.rex(.{
1587 .w = dst_mem.ptr_size == .qword_ptr,
1588 });
1589 }
1590 opc.encode(encoder);
1591 dst_mem.encode(encoder, modrm_ext);
1592 encodeImm(encoder, imm, dst_mem.ptr_size.size());
1593 },
1594 }
1595}
1596
1597fn lowerToRmEnc(
1598 tag: Tag,
1599 reg: Register,
1600 reg_or_mem: RegisterOrMemory,
1601 code: *std.ArrayList(u8),
1602) LoweringError!void {
1603 const opc = getOpCode(tag, .rm, reg.size() == 8).?;
1604 switch (reg_or_mem) {
1605 .register => |src_reg| {
1606 if (reg.size() != src_reg.size()) {
1607 return error.OperandSizeMismatch;
1608 }
1609 const encoder = try Encoder.init(code, 3);
1610 encoder.rex(.{
1611 .w = setRexWRegister(reg) or setRexWRegister(src_reg),
1612 .r = reg.isExtended(),
1613 .b = src_reg.isExtended(),
1614 });
1615 opc.encode(encoder);
1616 encoder.modRm_direct(reg.lowId(), src_reg.lowId());
1617 },
1618 .memory => |src_mem| {
1619 if (reg.size() != src_mem.ptr_size.size()) {
1620 return error.OperandSizeMismatch;
1621 }
1622 const encoder = try Encoder.init(code, 9);
1623 if (reg.size() == 16) {
1624 encoder.prefix16BitMode();
1625 }
1626 if (src_mem.base) |base| {
1627 // TODO handle 32-bit base register - requires prefix 0x67
1628 // Intel Manual, Vol 1, chapter 3.6 and 3.6.1
1629 if (base.size() != 64) {
1630 return error.OperandSizeMismatch;
1631 }
1632 encoder.rex(.{
1633 .w = setRexWRegister(reg),
1634 .r = reg.isExtended(),
1635 .b = base.isExtended(),
1636 });
1637 } else {
1638 encoder.rex(.{
1639 .w = setRexWRegister(reg),
1640 .r = reg.isExtended(),
1641 });
1642 }
1643 opc.encode(encoder);
1644 src_mem.encode(encoder, reg.lowId());
1645 },
1646 }
1647}
1648
1649fn lowerToMrEnc(
1650 tag: Tag,
1651 reg_or_mem: RegisterOrMemory,
1652 reg: Register,
1653 code: *std.ArrayList(u8),
1654) LoweringError!void {
1655 const opc = getOpCode(tag, .mr, reg.size() == 8).?;
1656 switch (reg_or_mem) {
1657 .register => |dst_reg| {
1658 if (dst_reg.size() != reg.size()) {
1659 return error.OperandSizeMismatch;
1660 }
1661 const encoder = try Encoder.init(code, 3);
1662 encoder.rex(.{
1663 .w = setRexWRegister(dst_reg) or setRexWRegister(reg),
1664 .r = reg.isExtended(),
1665 .b = dst_reg.isExtended(),
1666 });
1667 opc.encode(encoder);
1668 encoder.modRm_direct(reg.lowId(), dst_reg.lowId());
1669 },
1670 .memory => |dst_mem| {
1671 if (dst_mem.ptr_size.size() != reg.size()) {
1672 return error.OperandSizeMismatch;
1673 }
1674 const encoder = try Encoder.init(code, 9);
1675 if (reg.size() == 16) {
1676 encoder.prefix16BitMode();
1677 }
1678 if (dst_mem.base) |base| {
1679 if (base.size() != 64) {
1680 return error.OperandSizeMismatch;
1681 }
1682 encoder.rex(.{
1683 .w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg),
1684 .r = reg.isExtended(),
1685 .b = base.isExtended(),
1686 });
1687 } else {
1688 encoder.rex(.{
1689 .w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg),
1690 .r = reg.isExtended(),
1691 });
1692 }
1693 opc.encode(encoder);
1694 dst_mem.encode(encoder, reg.lowId());
1695 },
1696 }
1697}
1698
1699fn lowerToRmiEnc(
1700 tag: Tag,
1701 reg: Register,
1702 reg_or_mem: RegisterOrMemory,
1703 imm: i32,
1704 code: *std.ArrayList(u8),
1705) LoweringError!void {
1706 if (reg.size() == 8) {
1707 return error.OperandSizeMismatch;
1708 }
1709 const opc = getOpCode(tag, .rmi, false).?;
1710 const encoder = try Encoder.init(code, 13);
1711 if (reg.size() == 16) {
1712 encoder.prefix16BitMode();
1713 }
1714 switch (reg_or_mem) {
1715 .register => |src_reg| {
1716 if (reg.size() != src_reg.size()) {
1717 return error.OperandSizeMismatch;
1718 }
1719 encoder.rex(.{
1720 .w = setRexWRegister(reg) or setRexWRegister(src_reg),
1721 .r = reg.isExtended(),
1722 .b = src_reg.isExtended(),
1723 });
1724 opc.encode(encoder);
1725 encoder.modRm_direct(reg.lowId(), src_reg.lowId());
1726 },
1727 .memory => |src_mem| {
1728 if (src_mem.base) |base| {
1729 // TODO handle 32-bit base register - requires prefix 0x67
1730 // Intel Manual, Vol 1, chapter 3.6 and 3.6.1
1731 if (base.size() != 64) {
1732 return error.OperandSizeMismatch;
1733 }
1734 if (src_mem.ptr_size == .byte_ptr) {
1735 return error.OperandSizeMismatch;
1736 }
1737 encoder.rex(.{
1738 .w = setRexWRegister(reg),
1739 .r = reg.isExtended(),
1740 .b = base.isExtended(),
1741 });
1742 } else {
1743 encoder.rex(.{
1744 .w = setRexWRegister(reg),
1745 .r = reg.isExtended(),
1746 });
1747 }
1748 opc.encode(encoder);
1749 src_mem.encode(encoder, reg.lowId());
1750 },
1751 }
1752 encodeImm(encoder, imm, reg.size());
1753}
1754
1755fn expectEqualHexStrings(expected: []const u8, given: []const u8, assembly: []const u8) !void {
1756 assert(expected.len > 0);
1757 if (mem.eql(u8, expected, given)) return;
1758 const expected_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{std.fmt.fmtSliceHexLower(expected)});
1759 defer testing.allocator.free(expected_fmt);
1760 const given_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{std.fmt.fmtSliceHexLower(given)});
1761 defer testing.allocator.free(given_fmt);
1762 const idx = mem.indexOfDiff(u8, expected_fmt, given_fmt).?;
1763 var padding = try testing.allocator.alloc(u8, idx + 5);
1764 defer testing.allocator.free(padding);
1765 mem.set(u8, padding, ' ');
1766 std.debug.print("\nASM: {s}\nEXP: {s}\nGIV: {s}\n{s}^ -- first differing byte\n", .{
1767 assembly,
1768 expected_fmt,
1769 given_fmt,
1770 padding,
1771 });
1772 return error.TestFailed;
1773}
1774
1775const TestIsel = struct {
1776 code_buffer: std.ArrayList(u8),
1777 next: usize = 0,
1778
1779 fn init() TestIsel {
1780 return .{
1781 .code_buffer = std.ArrayList(u8).init(testing.allocator),
1782 };
1783 }
1784
1785 fn deinit(isel: *TestIsel) void {
1786 isel.code_buffer.deinit();
1787 isel.next = undefined;
1788 }
1789
1790 fn code(isel: *TestIsel) *std.ArrayList(u8) {
1791 isel.next = isel.code_buffer.items.len;
1792 return &isel.code_buffer;
1793 }
1794
1795 fn lowered(isel: TestIsel) []const u8 {
1796 return isel.code_buffer.items[isel.next..];
1797 }
1798};
1799
1800test "lower MI encoding" {
1801 var isel = TestIsel.init();
1802 defer isel.deinit();
1803 try lowerToMiEnc(.mov, RegisterOrMemory.reg(.rax), 0x10, isel.code());
1804 try expectEqualHexStrings("\x48\xc7\xc0\x10\x00\x00\x00", isel.lowered(), "mov rax, 0x10");
1805 try lowerToMiEnc(.mov, RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0, .base = .r11 }), 0x10, isel.code());
1806 try expectEqualHexStrings("\x41\xc7\x03\x10\x00\x00\x00", isel.lowered(), "mov dword ptr [r11 + 0], 0x10");
1807 try lowerToMiEnc(.add, RegisterOrMemory.mem(.dword_ptr, .{ .disp = -8, .base = .rdx }), 0x10, isel.code());
1808 try expectEqualHexStrings("\x81\x42\xF8\x10\x00\x00\x00", isel.lowered(), "add dword ptr [rdx - 8], 0x10");
1809 try lowerToMiEnc(.sub, RegisterOrMemory.mem(.dword_ptr, .{
1810 .disp = 0x10000000,
1811 .base = .r11,
1812 }), 0x10, isel.code());
1813 try expectEqualHexStrings(
1814 "\x41\x81\xab\x00\x00\x00\x10\x10\x00\x00\x00",
1815 isel.lowered(),
1816 "sub dword ptr [r11 + 0x10000000], 0x10",
1817 );
1818 try lowerToMiEnc(.@"and", RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0x10000000 }), 0x10, isel.code());
1819 try expectEqualHexStrings(
1820 "\x81\x24\x25\x00\x00\x00\x10\x10\x00\x00\x00",
1821 isel.lowered(),
1822 "and dword ptr [ds:0x10000000], 0x10",
1823 );
1824 try lowerToMiEnc(.@"and", RegisterOrMemory.mem(.dword_ptr, .{
1825 .disp = 0x10000000,
1826 .base = .r12,
1827 }), 0x10, isel.code());
1828 try expectEqualHexStrings(
1829 "\x41\x81\xA4\x24\x00\x00\x00\x10\x10\x00\x00\x00",
1830 isel.lowered(),
1831 "and dword ptr [r12 + 0x10000000], 0x10",
1832 );
1833 try lowerToMiEnc(.mov, RegisterOrMemory.rip(.qword_ptr, 0x10), 0x10, isel.code());
1834 try expectEqualHexStrings(
1835 "\x48\xC7\x05\x10\x00\x00\x00\x10\x00\x00\x00",
1836 isel.lowered(),
1837 "mov qword ptr [rip + 0x10], 0x10",
1838 );
1839 try lowerToMiEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{ .disp = -8, .base = .rbp }), 0x10, isel.code());
1840 try expectEqualHexStrings(
1841 "\x48\xc7\x45\xf8\x10\x00\x00\x00",
1842 isel.lowered(),
1843 "mov qword ptr [rbp - 8], 0x10",
1844 );
1845 try lowerToMiEnc(.mov, RegisterOrMemory.mem(.word_ptr, .{ .disp = -2, .base = .rbp }), 0x10, isel.code());
1846 try expectEqualHexStrings("\x66\xC7\x45\xFE\x10\x00", isel.lowered(), "mov word ptr [rbp - 2], 0x10");
1847 try lowerToMiEnc(.mov, RegisterOrMemory.mem(.byte_ptr, .{ .disp = -1, .base = .rbp }), 0x10, isel.code());
1848 try expectEqualHexStrings("\xC6\x45\xFF\x10", isel.lowered(), "mov byte ptr [rbp - 1], 0x10");
1849 try lowerToMiEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
1850 .disp = 0x10000000,
1851 .scale_index = .{
1852 .scale = 1,
1853 .index = .rcx,
1854 },
1855 }), 0x10, isel.code());
1856 try expectEqualHexStrings(
1857 "\x48\xC7\x04\x4D\x00\x00\x00\x10\x10\x00\x00\x00",
1858 isel.lowered(),
1859 "mov qword ptr [rcx*2 + 0x10000000], 0x10",
1860 );
1861}
1862
1863test "lower RM encoding" {
1864 var isel = TestIsel.init();
1865 defer isel.deinit();
1866 try lowerToRmEnc(.mov, .rax, RegisterOrMemory.reg(.rbx), isel.code());
1867 try expectEqualHexStrings("\x48\x8b\xc3", isel.lowered(), "mov rax, rbx");
1868 try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0, .base = .r11 }), isel.code());
1869 try expectEqualHexStrings("\x49\x8b\x03", isel.lowered(), "mov rax, qword ptr [r11 + 0]");
1870 try lowerToRmEnc(.add, .r11, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10000000 }), isel.code());
1871 try expectEqualHexStrings(
1872 "\x4C\x03\x1C\x25\x00\x00\x00\x10",
1873 isel.lowered(),
1874 "add r11, qword ptr [ds:0x10000000]",
1875 );
1876 try lowerToRmEnc(.add, .r12b, RegisterOrMemory.mem(.byte_ptr, .{ .disp = 0x10000000 }), isel.code());
1877 try expectEqualHexStrings(
1878 "\x44\x02\x24\x25\x00\x00\x00\x10",
1879 isel.lowered(),
1880 "add r11b, byte ptr [ds:0x10000000]",
1881 );
1882 try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.qword_ptr, .{
1883 .disp = 0x10000000,
1884 .base = .r13,
1885 }), isel.code());
1886 try expectEqualHexStrings(
1887 "\x4D\x2B\x9D\x00\x00\x00\x10",
1888 isel.lowered(),
1889 "sub r11, qword ptr [r13 + 0x10000000]",
1890 );
1891 try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.qword_ptr, .{
1892 .disp = 0x10000000,
1893 .base = .r12,
1894 }), isel.code());
1895 try expectEqualHexStrings(
1896 "\x4D\x2B\x9C\x24\x00\x00\x00\x10",
1897 isel.lowered(),
1898 "sub r11, qword ptr [r12 + 0x10000000]",
1899 );
1900 try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{ .disp = -4, .base = .rbp }), isel.code());
1901 try expectEqualHexStrings("\x48\x8B\x45\xFC", isel.lowered(), "mov rax, qword ptr [rbp - 4]");
1902 try lowerToRmEnc(.lea, .rax, RegisterOrMemory.rip(.qword_ptr, 0x10), isel.code());
1903 try expectEqualHexStrings("\x48\x8D\x05\x10\x00\x00\x00", isel.lowered(), "lea rax, [rip + 0x10]");
1904 try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{
1905 .disp = -8,
1906 .base = .rbp,
1907 .scale_index = .{
1908 .scale = 0,
1909 .index = .rcx,
1910 },
1911 }), isel.code());
1912 try expectEqualHexStrings("\x48\x8B\x44\x0D\xF8", isel.lowered(), "mov rax, qword ptr [rbp + rcx*1 - 8]");
1913 try lowerToRmEnc(.mov, .eax, RegisterOrMemory.mem(.dword_ptr, .{
1914 .disp = -4,
1915 .base = .rbp,
1916 .scale_index = .{
1917 .scale = 2,
1918 .index = .rdx,
1919 },
1920 }), isel.code());
1921 try expectEqualHexStrings("\x8B\x44\x95\xFC", isel.lowered(), "mov eax, dword ptr [rbp + rdx*4 - 4]");
1922 try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{
1923 .disp = -8,
1924 .base = .rbp,
1925 .scale_index = .{
1926 .scale = 3,
1927 .index = .rcx,
1928 },
1929 }), isel.code());
1930 try expectEqualHexStrings("\x48\x8B\x44\xCD\xF8", isel.lowered(), "mov rax, qword ptr [rbp + rcx*8 - 8]");
1931 try lowerToRmEnc(.mov, .r8b, RegisterOrMemory.mem(.byte_ptr, .{
1932 .disp = -24,
1933 .base = .rsi,
1934 .scale_index = .{
1935 .scale = 0,
1936 .index = .rcx,
1937 },
1938 }), isel.code());
1939 try expectEqualHexStrings("\x44\x8A\x44\x0E\xE8", isel.lowered(), "mov r8b, byte ptr [rsi + rcx*1 - 24]");
1940 try lowerToRmEnc(.lea, .rsi, RegisterOrMemory.mem(.qword_ptr, .{
1941 .disp = 0,
1942 .base = .rbp,
1943 .scale_index = .{
1944 .scale = 0,
1945 .index = .rcx,
1946 },
1947 }), isel.code());
1948 try expectEqualHexStrings("\x48\x8D\x74\x0D\x00", isel.lowered(), "lea rsi, qword ptr [rbp + rcx*1 + 0]");
1949}
1950
1951test "lower MR encoding" {
1952 var isel = TestIsel.init();
1953 defer isel.deinit();
1954 try lowerToMrEnc(.mov, RegisterOrMemory.reg(.rax), .rbx, isel.code());
1955 try expectEqualHexStrings("\x48\x89\xd8", isel.lowered(), "mov rax, rbx");
1956 try lowerToMrEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{ .disp = -4, .base = .rbp }), .r11, isel.code());
1957 try expectEqualHexStrings("\x4c\x89\x5d\xfc", isel.lowered(), "mov qword ptr [rbp - 4], r11");
1958 try lowerToMrEnc(.add, RegisterOrMemory.mem(.byte_ptr, .{ .disp = 0x10000000 }), .r12b, isel.code());
1959 try expectEqualHexStrings(
1960 "\x44\x00\x24\x25\x00\x00\x00\x10",
1961 isel.lowered(),
1962 "add byte ptr [ds:0x10000000], r12b",
1963 );
1964 try lowerToMrEnc(.add, RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0x10000000 }), .r12d, isel.code());
1965 try expectEqualHexStrings(
1966 "\x44\x01\x24\x25\x00\x00\x00\x10",
1967 isel.lowered(),
1968 "add dword ptr [ds:0x10000000], r12d",
1969 );
1970 try lowerToMrEnc(.sub, RegisterOrMemory.mem(.qword_ptr, .{
1971 .disp = 0x10000000,
1972 .base = .r11,
1973 }), .r12, isel.code());
1974 try expectEqualHexStrings(
1975 "\x4D\x29\xA3\x00\x00\x00\x10",
1976 isel.lowered(),
1977 "sub qword ptr [r11 + 0x10000000], r12",
1978 );
1979 try lowerToMrEnc(.mov, RegisterOrMemory.rip(.qword_ptr, 0x10), .r12, isel.code());
1980 try expectEqualHexStrings("\x4C\x89\x25\x10\x00\x00\x00", isel.lowered(), "mov qword ptr [rip + 0x10], r12");
1981}
1982
1983test "lower OI encoding" {
1984 var isel = TestIsel.init();
1985 defer isel.deinit();
1986 try lowerToOiEnc(.mov, .rax, 0x1000000000000000, isel.code());
1987 try expectEqualHexStrings(
1988 "\x48\xB8\x00\x00\x00\x00\x00\x00\x00\x10",
1989 isel.lowered(),
1990 "movabs rax, 0x1000000000000000",
1991 );
1992 try lowerToOiEnc(.mov, .r11, 0x1000000000000000, isel.code());
1993 try expectEqualHexStrings(
1994 "\x49\xBB\x00\x00\x00\x00\x00\x00\x00\x10",
1995 isel.lowered(),
1996 "movabs r11, 0x1000000000000000",
1997 );
1998 try lowerToOiEnc(.mov, .r11d, 0x10000000, isel.code());
1999 try expectEqualHexStrings("\x41\xBB\x00\x00\x00\x10", isel.lowered(), "mov r11d, 0x10000000");
2000 try lowerToOiEnc(.mov, .r11w, 0x1000, isel.code());
2001 try expectEqualHexStrings("\x66\x41\xBB\x00\x10", isel.lowered(), "mov r11w, 0x1000");
2002 try lowerToOiEnc(.mov, .r11b, 0x10, isel.code());
2003 try expectEqualHexStrings("\x41\xB3\x10", isel.lowered(), "mov r11b, 0x10");
2004}
2005
2006test "lower FD/TD encoding" {
2007 var isel = TestIsel.init();
2008 defer isel.deinit();
2009 try lowerToFdEnc(.mov, .rax, 0x1000000000000000, isel.code());
2010 try expectEqualHexStrings(
2011 "\x48\xa1\x00\x00\x00\x00\x00\x00\x00\x10",
2012 isel.lowered(),
2013 "mov rax, ds:0x1000000000000000",
2014 );
2015 try lowerToFdEnc(.mov, .eax, 0x10000000, isel.code());
2016 try expectEqualHexStrings("\xa1\x00\x00\x00\x10", isel.lowered(), "mov eax, ds:0x10000000");
2017 try lowerToFdEnc(.mov, .ax, 0x1000, isel.code());
2018 try expectEqualHexStrings("\x66\xa1\x00\x10", isel.lowered(), "mov ax, ds:0x1000");
2019 try lowerToFdEnc(.mov, .al, 0x10, isel.code());
2020 try expectEqualHexStrings("\xa0\x10", isel.lowered(), "mov al, ds:0x10");
2021}
2022
2023test "lower M encoding" {
2024 var isel = TestIsel.init();
2025 defer isel.deinit();
2026 try lowerToMEnc(.jmp_near, RegisterOrMemory.reg(.r12), isel.code());
2027 try expectEqualHexStrings("\x41\xFF\xE4", isel.lowered(), "jmp r12");
2028 try lowerToMEnc(.jmp_near, RegisterOrMemory.reg(.r12w), isel.code());
2029 try expectEqualHexStrings("\x66\x41\xFF\xE4", isel.lowered(), "jmp r12w");
2030 try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0, .base = .r12 }), isel.code());
2031 try expectEqualHexStrings("\x41\xFF\x24\x24", isel.lowered(), "jmp qword ptr [r12]");
2032 try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.word_ptr, .{ .disp = 0, .base = .r12 }), isel.code());
2033 try expectEqualHexStrings("\x66\x41\xFF\x24\x24", isel.lowered(), "jmp word ptr [r12]");
2034 try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10, .base = .r12 }), isel.code());
2035 try expectEqualHexStrings("\x41\xFF\x64\x24\x10", isel.lowered(), "jmp qword ptr [r12 + 0x10]");
2036 try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{
2037 .disp = 0x1000,
2038 .base = .r12,
2039 }), isel.code());
2040 try expectEqualHexStrings(
2041 "\x41\xFF\xA4\x24\x00\x10\x00\x00",
2042 isel.lowered(),
2043 "jmp qword ptr [r12 + 0x1000]",
2044 );
2045 try lowerToMEnc(.jmp_near, RegisterOrMemory.rip(.qword_ptr, 0x10), isel.code());
2046 try expectEqualHexStrings("\xFF\x25\x10\x00\x00\x00", isel.lowered(), "jmp qword ptr [rip + 0x10]");
2047 try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10 }), isel.code());
2048 try expectEqualHexStrings("\xFF\x24\x25\x10\x00\x00\x00", isel.lowered(), "jmp qword ptr [ds:0x10]");
2049 try lowerToMEnc(.seta, RegisterOrMemory.reg(.r11b), isel.code());
2050 try expectEqualHexStrings("\x41\x0F\x97\xC3", isel.lowered(), "seta r11b");
2051}
2052
2053test "lower O encoding" {
2054 var isel = TestIsel.init();
2055 defer isel.deinit();
2056 try lowerToOEnc(.pop, .r12, isel.code());
2057 try expectEqualHexStrings("\x41\x5c", isel.lowered(), "pop r12");
2058 try lowerToOEnc(.push, .r12w, isel.code());
2059 try expectEqualHexStrings("\x66\x41\x54", isel.lowered(), "push r12w");
2060}
2061
2062test "lower RMI encoding" {
2063 var isel = TestIsel.init();
2064 defer isel.deinit();
2065 try lowerToRmiEnc(.imul, .rax, RegisterOrMemory.mem(.qword_ptr, .{
2066 .disp = -8,
2067 .base = .rbp,
2068 }), 0x10, isel.code());
2069 try expectEqualHexStrings(
2070 "\x48\x69\x45\xF8\x10\x00\x00\x00",
2071 isel.lowered(),
2072 "imul rax, qword ptr [rbp - 8], 0x10",
2073 );
2074 try lowerToRmiEnc(.imul, .eax, RegisterOrMemory.mem(.dword_ptr, .{
2075 .disp = -4,
2076 .base = .rbp,
2077 }), 0x10, isel.code());
2078 try expectEqualHexStrings("\x69\x45\xFC\x10\x00\x00\x00", isel.lowered(), "imul eax, dword ptr [rbp - 4], 0x10");
2079 try lowerToRmiEnc(.imul, .ax, RegisterOrMemory.mem(.word_ptr, .{
2080 .disp = -2,
2081 .base = .rbp,
2082 }), 0x10, isel.code());
2083 try expectEqualHexStrings("\x66\x69\x45\xFE\x10\x00", isel.lowered(), "imul ax, word ptr [rbp - 2], 0x10");
2084 try lowerToRmiEnc(.imul, .r12, RegisterOrMemory.reg(.r12), 0x10, isel.code());
2085 try expectEqualHexStrings("\x4D\x69\xE4\x10\x00\x00\x00", isel.lowered(), "imul r12, r12, 0x10");
2086 try lowerToRmiEnc(.imul, .r12w, RegisterOrMemory.reg(.r12w), 0x10, isel.code());
2087 try expectEqualHexStrings("\x66\x45\x69\xE4\x10\x00", isel.lowered(), "imul r12w, r12w, 0x10");
2088}
src/arch/x86_64/Mir.zig+3-3
...@@ -302,7 +302,7 @@ pub const Inst = struct {...@@ -302,7 +302,7 @@ pub const Inst = struct {
302 /// Another instruction.302 /// Another instruction.
303 inst: Index,303 inst: Index,
304 /// A 32-bit immediate value.304 /// A 32-bit immediate value.
305 imm: i32,305 imm: u32,
306 /// An extern function.306 /// An extern function.
307 /// Index into the linker's string table.307 /// Index into the linker's string table.
308 extern_fn: u32,308 extern_fn: u32,
...@@ -324,8 +324,8 @@ pub const Inst = struct {...@@ -324,8 +324,8 @@ pub const Inst = struct {
324};324};
325325
326pub const ImmPair = struct {326pub const ImmPair = struct {
327 dest_off: i32,327 dest_off: u32,
328 operand: i32,328 operand: u32,
329};329};
330330
331pub const Imm64 = struct {331pub const Imm64 = struct {
test/stage2/x86_64.zig+64
...@@ -1700,6 +1700,36 @@ pub fn addCases(ctx: *TestContext) !void {...@@ -1700,6 +1700,36 @@ pub fn addCases(ctx: *TestContext) !void {
1700 \\ if (!ok) unreachable;1700 \\ if (!ok) unreachable;
1701 \\}1701 \\}
1702 , "");1702 , "");
1703 case.addCompareOutput(
1704 \\pub fn main() void {
1705 \\ var x: u16 = undefined;
1706 \\ set(&x);
1707 \\ assert(x == 123);
1708 \\}
1709 \\
1710 \\fn set(x: *u16) void {
1711 \\ x.* = 123;
1712 \\}
1713 \\
1714 \\fn assert(ok: bool) void {
1715 \\ if (!ok) unreachable;
1716 \\}
1717 , "");
1718 case.addCompareOutput(
1719 \\pub fn main() void {
1720 \\ var x: u8 = undefined;
1721 \\ set(&x);
1722 \\ assert(x == 123);
1723 \\}
1724 \\
1725 \\fn set(x: *u8) void {
1726 \\ x.* = 123;
1727 \\}
1728 \\
1729 \\fn assert(ok: bool) void {
1730 \\ if (!ok) unreachable;
1731 \\}
1732 , "");
1703 }1733 }
17041734
1705 {1735 {
...@@ -1737,6 +1767,40 @@ pub fn addCases(ctx: *TestContext) !void {...@@ -1737,6 +1767,40 @@ pub fn addCases(ctx: *TestContext) !void {
1737 \\ if (!ok) unreachable;1767 \\ if (!ok) unreachable;
1738 \\}1768 \\}
1739 , "");1769 , "");
1770 case.addCompareOutput(
1771 \\pub fn main() void {
1772 \\ var x: u8 = undefined;
1773 \\ const maybe_x = byPtr(&x);
1774 \\ assert(maybe_x != null);
1775 \\ maybe_x.?.* = 255;
1776 \\ assert(x == 255);
1777 \\}
1778 \\
1779 \\fn byPtr(x: *u8) ?*u8 {
1780 \\ return x;
1781 \\}
1782 \\
1783 \\fn assert(ok: bool) void {
1784 \\ if (!ok) unreachable;
1785 \\}
1786 , "");
1787 case.addCompareOutput(
1788 \\pub fn main() void {
1789 \\ var x: i8 = undefined;
1790 \\ const maybe_x = byPtr(&x);
1791 \\ assert(maybe_x != null);
1792 \\ maybe_x.?.* = -1;
1793 \\ assert(x == -1);
1794 \\}
1795 \\
1796 \\fn byPtr(x: *i8) ?*i8 {
1797 \\ return x;
1798 \\}
1799 \\
1800 \\fn assert(ok: bool) void {
1801 \\ if (!ok) unreachable;
1802 \\}
1803 , "");
1740 }1804 }
17411805
1742 {1806 {