authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-01-23 00:01:12+01:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-01-23 00:01:12+01:00
log081ce09575426086bca1164ae733e476b99a0253
treefbc6b204723691dfad10851137c7df67858d679b
parentc9ae24503dc8da2e59f46619695bf4eb863fb3ac
parent406c85f9ba056e10899feed18dae91e20942dc55
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #10664 from ziglang/stage2-x86_64-refactor-air-call

stage2: refactor how we preserve callee regs and how we pass args on the stack in x86_64 backend

6 files changed, 194 insertions(+), 245 deletions(-)

src/arch/x86_64/CodeGen.zig+147-205
......@@ -43,7 +43,7 @@ err_msg: ?*ErrorMsg,
4343args: []MCValue,
4444ret_mcv: MCValue,
4545fn_type: Type,
46arg_index: usize,
46arg_index: u32,
4747src_loc: Module.SrcLoc,
4848stack_align: u32,
4949
......@@ -61,8 +61,6 @@ end_di_column: u32,
6161/// which is a relative jump, based on the address following the reloc.
6262exitlude_jump_relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .{},
6363
64stack_args_relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .{},
65
6664/// Whenever there is a runtime branch, we push a Branch onto this stack,
6765/// and pop it off when the runtime branch joins. This provides an "overlay"
6866/// of the table of mappings from instructions to `MCValue` from within the branch.
......@@ -119,9 +117,9 @@ pub const MCValue = union(enum) {
119117 memory: u64,
120118 /// The value is one of the stack variables.
121119 /// If the type is a pointer, it means the pointer address is in the stack at this offset.
122 stack_offset: u32,
120 stack_offset: i32,
123121 /// The value is a pointer to one of the stack variables (payload is stack offset).
124 ptr_stack_offset: u32,
122 ptr_stack_offset: i32,
125123 /// The value is in the compare flags assuming an unsigned operation,
126124 /// with this operator applied on top of it.
127125 compare_flags_unsigned: math.CompareOperator,
......@@ -286,7 +284,6 @@ pub fn generate(
286284 defer function.exitlude_jump_relocs.deinit(bin_file.allocator);
287285 defer function.mir_instructions.deinit(bin_file.allocator);
288286 defer function.mir_extra.deinit(bin_file.allocator);
289 defer function.stack_args_relocs.deinit(bin_file.allocator);
290287 defer if (builtin.mode == .Debug) function.mir_to_air_map.deinit();
291288
292289 var call_info = function.resolveCallingConventionValues(fn_type) catch |err| switch (err) {
......@@ -378,13 +375,6 @@ pub fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {
378375fn gen(self: *Self) InnerError!void {
379376 const cc = self.fn_type.fnCallingConvention();
380377 if (cc != .Naked) {
381 // push the callee_preserved_regs that were used
382 const backpatch_push_callee_preserved_regs_i = try self.addInst(.{
383 .tag = .push_regs_from_callee_preserved_regs,
384 .ops = undefined,
385 .data = .{ .regs_to_push_or_pop = undefined }, // to be backpatched
386 });
387
388378 _ = try self.addInst(.{
389379 .tag = .push,
390380 .ops = (Mir.Ops{
......@@ -416,6 +406,15 @@ fn gen(self: *Self) InnerError!void {
416406 .data = undefined,
417407 });
418408
409 // push the callee_preserved_regs that were used
410 const backpatch_push_callee_preserved_regs_i = try self.addInst(.{
411 .tag = .push_regs_from_callee_preserved_regs,
412 .ops = (Mir.Ops{
413 .reg1 = .rbp,
414 }).encode(),
415 .data = .{ .payload = undefined }, // to be backpatched
416 });
417
419418 try self.genBody(self.air.getMainBody());
420419
421420 // TODO can single exitlude jump reloc be elided? What if it is not at the end of the code?
......@@ -429,6 +428,33 @@ fn gen(self: *Self) InnerError!void {
429428 self.mir_instructions.items(.data)[jmp_reloc].inst = @intCast(u32, self.mir_instructions.len);
430429 }
431430
431 // calculate the data for callee_preserved_regs to be pushed and popped
432 const callee_preserved_regs_payload = blk: {
433 var data = Mir.RegsToPushOrPop{
434 .regs = 0,
435 .disp = mem.alignForwardGeneric(u32, self.next_stack_offset, 8),
436 };
437 inline for (callee_preserved_regs) |reg, i| {
438 if (self.register_manager.isRegAllocated(reg)) {
439 data.regs |= 1 << @intCast(u5, i);
440 self.max_end_stack += 8;
441 }
442 }
443 break :blk try self.addExtra(data);
444 };
445
446 const data = self.mir_instructions.items(.data);
447 // backpatch the push instruction
448 data[backpatch_push_callee_preserved_regs_i].payload = callee_preserved_regs_payload;
449 // pop the callee_preserved_regs
450 _ = try self.addInst(.{
451 .tag = .pop_regs_from_callee_preserved_regs,
452 .ops = (Mir.Ops{
453 .reg1 = .rbp,
454 }).encode(),
455 .data = .{ .payload = callee_preserved_regs_payload },
456 });
457
432458 _ = try self.addInst(.{
433459 .tag = .dbg_epilogue_begin,
434460 .ops = undefined,
......@@ -450,34 +476,6 @@ fn gen(self: *Self) InnerError!void {
450476 .data = undefined,
451477 });
452478
453 // calculate the data for callee_preserved_regs to be pushed and popped
454 var callee_preserved_regs_push_data: u32 = 0x0;
455 // TODO this is required on macOS since macOS actively checks for stack alignment
456 // at every extern call site. As far as I can tell, macOS accounts for the typical
457 // function prologue first 2 instructions of:
458 // ...
459 // push rbp
460 // mov rsp, rbp
461 // ...
462 // Thus we don't need to adjust the stack for the first push instruction. However,
463 // any subsequent push of values on the stack such as when preserving registers,
464 // needs to be taken into account here.
465 var stack_adjustment: u32 = 0;
466 inline for (callee_preserved_regs) |reg, i| {
467 if (self.register_manager.isRegAllocated(reg)) {
468 callee_preserved_regs_push_data |= 1 << @intCast(u5, i);
469 stack_adjustment += @divExact(reg.size(), 8);
470 }
471 }
472 const data = self.mir_instructions.items(.data);
473 // backpatch the push instruction
474 data[backpatch_push_callee_preserved_regs_i].regs_to_push_or_pop = callee_preserved_regs_push_data;
475 // pop the callee_preserved_regs
476 _ = try self.addInst(.{
477 .tag = .pop_regs_from_callee_preserved_regs,
478 .ops = undefined,
479 .data = .{ .regs_to_push_or_pop = callee_preserved_regs_push_data },
480 });
481479 _ = try self.addInst(.{
482480 .tag = .ret,
483481 .ops = (Mir.Ops{
......@@ -487,37 +485,28 @@ fn gen(self: *Self) InnerError!void {
487485 });
488486
489487 // Adjust the stack
490 const stack_end = self.max_end_stack;
491 if (stack_end > math.maxInt(i32) - stack_adjustment) {
488 if (self.max_end_stack > math.maxInt(i32)) {
492489 return self.failSymbol("too much stack used in call parameters", .{});
493490 }
494491 // TODO we should reuse this mechanism to align the stack when calling any function even if
495492 // we do not pass any args on the stack BUT we still push regs to stack with `push` inst.
496 const aligned_stack_end = @intCast(u32, mem.alignForward(stack_end, self.stack_align));
497 if (aligned_stack_end > 0 or (stack_adjustment > 0 and self.target.isDarwin())) {
498 const imm = if (self.target.isDarwin()) aligned_stack_end + stack_adjustment else aligned_stack_end;
493 const aligned_stack_end = @intCast(u32, mem.alignForward(self.max_end_stack, self.stack_align));
494 if (aligned_stack_end > 0) {
499495 self.mir_instructions.set(backpatch_stack_sub, .{
500496 .tag = .sub,
501497 .ops = (Mir.Ops{
502498 .reg1 = .rsp,
503499 }).encode(),
504 .data = .{ .imm = imm },
500 .data = .{ .imm = aligned_stack_end },
505501 });
506502 self.mir_instructions.set(backpatch_stack_add, .{
507503 .tag = .add,
508504 .ops = (Mir.Ops{
509505 .reg1 = .rsp,
510506 }).encode(),
511 .data = .{ .imm = imm },
507 .data = .{ .imm = aligned_stack_end },
512508 });
513509 }
514 while (self.stack_args_relocs.popOrNull()) |index| {
515 // TODO like above, gotta figure out the alignment shenanigans for macOS, etc.
516 const adjustment = if (self.target.isDarwin()) 2 * stack_adjustment else stack_adjustment;
517 // +16 bytes to account for saved return address of the `call` instruction and
518 // `push rbp`.
519 self.mir_instructions.items(.data)[index].imm += adjustment + aligned_stack_end + 16;
520 }
521510 } else {
522511 _ = try self.addInst(.{
523512 .tag = .dbg_prologue_end,
......@@ -808,7 +797,7 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
808797 }
809798 }
810799 const stack_offset = try self.allocMem(inst, abi_size, abi_align);
811 return MCValue{ .stack_offset = stack_offset };
800 return MCValue{ .stack_offset = @intCast(i32, stack_offset) };
812801}
813802
814803pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {
......@@ -854,12 +843,12 @@ fn copyToNewRegisterWithExceptions(
854843
855844fn airAlloc(self: *Self, inst: Air.Inst.Index) !void {
856845 const stack_offset = try self.allocMemPtr(inst);
857 return self.finishAir(inst, .{ .ptr_stack_offset = stack_offset }, .{ .none, .none, .none });
846 return self.finishAir(inst, .{ .ptr_stack_offset = @intCast(i32, stack_offset) }, .{ .none, .none, .none });
858847}
859848
860849fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {
861850 const stack_offset = try self.allocMemPtr(inst);
862 return self.finishAir(inst, .{ .ptr_stack_offset = stack_offset }, .{ .none, .none, .none });
851 return self.finishAir(inst, .{ .ptr_stack_offset = @intCast(i32, stack_offset) }, .{ .none, .none, .none });
863852}
864853
865854fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
......@@ -1419,7 +1408,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
14191408 .reg1 = addr_reg.to64(),
14201409 .reg2 = .rbp,
14211410 }).encode(),
1422 .data = .{ .imm = @bitCast(u32, -@intCast(i32, off + array_abi_size)) },
1411 .data = .{ .imm = @bitCast(u32, -(off + @intCast(i32, array_abi_size))) },
14231412 });
14241413 },
14251414 else => return self.fail("TODO implement array_elem_val when array is {}", .{array}),
......@@ -1623,7 +1612,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
16231612 try self.genSetReg(Type.initTag(.u32), count_reg, .{ .immediate = @intCast(u32, abi_size) });
16241613
16251614 return self.genInlineMemcpy(
1626 @bitCast(u32, -@intCast(i32, off + abi_size)),
1615 -(off + @intCast(i32, abi_size)),
16271616 .rbp,
16281617 registerAlias(addr_reg, @divExact(reg.size(), 8)),
16291618 count_reg.to64(),
......@@ -1780,10 +1769,10 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde
17801769 return if (self.liveness.isUnused(inst)) .dead else result: {
17811770 const mcv = try self.resolveInst(operand);
17821771 const struct_ty = self.air.typeOf(operand).childType();
1783 const struct_size = @intCast(u32, struct_ty.abiSize(self.target.*));
1784 const struct_field_offset = @intCast(u32, struct_ty.structFieldOffset(index, self.target.*));
1772 const struct_size = @intCast(i32, struct_ty.abiSize(self.target.*));
1773 const struct_field_offset = @intCast(i32, struct_ty.structFieldOffset(index, self.target.*));
17851774 const struct_field_ty = struct_ty.structFieldType(index);
1786 const struct_field_size = @intCast(u32, struct_field_ty.abiSize(self.target.*));
1775 const struct_field_size = @intCast(i32, struct_field_ty.abiSize(self.target.*));
17871776
17881777 switch (mcv) {
17891778 .ptr_stack_offset => |off| {
......@@ -1803,10 +1792,10 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
18031792 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
18041793 const mcv = try self.resolveInst(operand);
18051794 const struct_ty = self.air.typeOf(operand);
1806 const struct_size = @intCast(u32, struct_ty.abiSize(self.target.*));
1807 const struct_field_offset = @intCast(u32, struct_ty.structFieldOffset(index, self.target.*));
1795 const struct_size = @intCast(i32, struct_ty.abiSize(self.target.*));
1796 const struct_field_offset = @intCast(i32, struct_ty.structFieldOffset(index, self.target.*));
18081797 const struct_field_ty = struct_ty.structFieldType(index);
1809 const struct_field_size = @intCast(u32, struct_field_ty.abiSize(self.target.*));
1798 const struct_field_size = @intCast(i32, struct_field_ty.abiSize(self.target.*));
18101799
18111800 switch (mcv) {
18121801 .stack_offset => |off| {
......@@ -1970,7 +1959,7 @@ fn genBinMathOpMir(
19701959 return self.fail("stack offset too large", .{});
19711960 }
19721961 const abi_size = dst_ty.abiSize(self.target.*);
1973 const adj_off = off + abi_size;
1962 const adj_off = off + @intCast(i32, abi_size);
19741963 _ = try self.addInst(.{
19751964 .tag = mir_tag,
19761965 .ops = (Mir.Ops{
......@@ -1978,7 +1967,7 @@ fn genBinMathOpMir(
19781967 .reg2 = .rbp,
19791968 .flags = 0b01,
19801969 }).encode(),
1981 .data = .{ .imm = @bitCast(u32, -@intCast(i32, adj_off)) },
1970 .data = .{ .imm = @bitCast(u32, -adj_off) },
19821971 });
19831972 },
19841973 .compare_flags_unsigned => {
......@@ -1997,7 +1986,7 @@ fn genBinMathOpMir(
19971986 if (abi_size > 8) {
19981987 return self.fail("TODO implement ADD/SUB/CMP for stack dst with large ABI", .{});
19991988 }
2000 const adj_off = off + abi_size;
1989 const adj_off = off + @intCast(i32, abi_size);
20011990
20021991 switch (src_mcv) {
20031992 .none => unreachable,
......@@ -2013,7 +2002,7 @@ fn genBinMathOpMir(
20132002 .reg2 = registerAlias(src_reg, @intCast(u32, abi_size)),
20142003 .flags = 0b10,
20152004 }).encode(),
2016 .data = .{ .imm = @bitCast(u32, -@intCast(i32, adj_off)) },
2005 .data = .{ .imm = @bitCast(u32, -adj_off) },
20172006 });
20182007 },
20192008 .immediate => |imm| {
......@@ -2034,7 +2023,7 @@ fn genBinMathOpMir(
20342023 else => unreachable,
20352024 };
20362025 const payload = try self.addExtra(Mir.ImmPair{
2037 .dest_off = @bitCast(u32, -@intCast(i32, adj_off)),
2026 .dest_off = @bitCast(u32, -adj_off),
20382027 .operand = @truncate(u32, imm),
20392028 });
20402029 _ = try self.addInst(.{
......@@ -2172,7 +2161,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
21722161 const mcv = self.args[arg_index];
21732162 const payload = try self.addExtra(Mir.ArgDbgInfo{
21742163 .air_inst = inst,
2175 .arg_index = @truncate(u32, arg_index), // TODO can arg_index: u32?
2164 .arg_index = arg_index,
21762165 });
21772166 _ = try self.addInst(.{
21782167 .tag = .arg_dbg_info,
......@@ -2188,58 +2177,13 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
21882177 self.register_manager.getRegAssumeFree(reg.to64(), inst);
21892178 break :blk mcv;
21902179 },
2191 .stack_offset => |off| {
2180 .stack_offset => {
21922181 const ty = self.air.typeOfIndex(inst);
21932182 const abi_size = ty.abiSize(self.target.*);
2194
2195 if (abi_size <= 8) {
2196 const reg = try self.register_manager.allocReg(inst, &.{});
2197 const reloc = try self.addInst(.{
2198 .tag = .mov,
2199 .ops = (Mir.Ops{
2200 .reg1 = registerAlias(reg, @intCast(u32, abi_size)),
2201 .reg2 = .rsp,
2202 .flags = 0b01,
2203 }).encode(),
2204 .data = .{ .imm = off },
2205 });
2206 try self.stack_args_relocs.append(self.bin_file.allocator, reloc);
2207 break :blk .{ .register = reg };
2208 }
2209
2210 // TODO copy ellision
2211 const dst_mcv = try self.allocRegOrMem(inst, false);
2212 const regs = try self.register_manager.allocRegs(3, .{ null, null, null }, &.{ .rax, .rcx });
2213 const addr_reg = regs[0];
2214 const count_reg = regs[1];
2215 const tmp_reg = regs[2];
2216
2217 try self.register_manager.getReg(.rax, null);
2218 try self.register_manager.getReg(.rcx, null);
2219
2220 const reloc = try self.addInst(.{
2221 .tag = .lea,
2222 .ops = (Mir.Ops{
2223 .reg1 = addr_reg.to64(),
2224 .reg2 = .rsp,
2225 }).encode(),
2226 .data = .{ .imm = off },
2227 });
2228 try self.stack_args_relocs.append(self.bin_file.allocator, reloc);
2229
2230 // TODO allow for abi_size to be u64
2231 try self.genSetReg(Type.initTag(.u32), count_reg, .{ .immediate = @intCast(u32, abi_size) });
2232 try self.genInlineMemcpy(
2233 @bitCast(u32, -@intCast(i32, dst_mcv.stack_offset + abi_size)),
2234 .rbp,
2235 addr_reg.to64(),
2236 count_reg.to64(),
2237 tmp_reg.to8(),
2238 );
2239
2240 break :blk dst_mcv;
2183 const off = @intCast(i32, (arg_index + 1) * abi_size) + 16;
2184 break :blk MCValue{ .stack_offset = -off };
22412185 },
2242 else => unreachable,
2186 else => return self.fail("TODO implement arg for {}", .{mcv}),
22432187 }
22442188 };
22452189
......@@ -2264,64 +2208,6 @@ fn airFence(self: *Self) !void {
22642208 //return self.finishAirBookkeeping();
22652209}
22662210
2267fn genSetStackArg(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerError!void {
2268 const abi_size = ty.abiSize(self.target.*);
2269 switch (mcv) {
2270 .dead => unreachable,
2271 .ptr_embedded_in_code => unreachable,
2272 .unreach, .none => return,
2273 .register => |reg| {
2274 _ = try self.addInst(.{
2275 .tag = .mov,
2276 .ops = (Mir.Ops{
2277 .reg1 = .rsp,
2278 .reg2 = registerAlias(reg, @intCast(u32, abi_size)),
2279 .flags = 0b10,
2280 }).encode(),
2281 .data = .{ .imm = @bitCast(u32, -@intCast(i32, stack_offset + abi_size)) },
2282 });
2283 },
2284 .ptr_stack_offset => {
2285 const reg = try self.copyToTmpRegister(ty, mcv);
2286 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
2287 },
2288 .stack_offset => |unadjusted_off| {
2289 if (abi_size <= 8) {
2290 const reg = try self.copyToTmpRegister(ty, mcv);
2291 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
2292 }
2293
2294 const regs = try self.register_manager.allocRegs(3, .{ null, null, null }, &.{ .rax, .rcx });
2295 const addr_reg = regs[0];
2296 const count_reg = regs[1];
2297 const tmp_reg = regs[2];
2298
2299 try self.register_manager.getReg(.rax, null);
2300 try self.register_manager.getReg(.rcx, null);
2301
2302 _ = try self.addInst(.{
2303 .tag = .lea,
2304 .ops = (Mir.Ops{
2305 .reg1 = addr_reg.to64(),
2306 .reg2 = .rbp,
2307 }).encode(),
2308 .data = .{ .imm = @bitCast(u32, -@intCast(i32, unadjusted_off + abi_size)) },
2309 });
2310
2311 // TODO allow for abi_size to be u64
2312 try self.genSetReg(Type.initTag(.u32), count_reg, .{ .immediate = @intCast(u32, abi_size) });
2313 try self.genInlineMemcpy(
2314 @bitCast(u32, -@intCast(i32, stack_offset + abi_size)),
2315 .rsp,
2316 addr_reg.to64(),
2317 count_reg.to64(),
2318 tmp_reg.to8(),
2319 );
2320 },
2321 else => return self.fail("TODO implement args on stack for {}", .{mcv}),
2322 }
2323}
2324
23252211fn airCall(self: *Self, inst: Air.Inst.Index) !void {
23262212 const pl_op = self.air.instructions.items(.data)[inst].pl_op;
23272213 const callee = pl_op.operand;
......@@ -2338,12 +2224,9 @@ fn airCall(self: *Self, inst: Air.Inst.Index) !void {
23382224 var info = try self.resolveCallingConventionValues(fn_ty);
23392225 defer info.deinit(self);
23402226
2341 var count: usize = info.args.len;
23422227 var stack_adjustment: u32 = 0;
2343 while (count > 0) : (count -= 1) {
2344 const arg_i = count - 1;
2228 for (args) |arg, arg_i| {
23452229 const mc_arg = info.args[arg_i];
2346 const arg = args[arg_i];
23472230 const arg_ty = self.air.typeOf(arg);
23482231 const arg_mcv = try self.resolveInst(args[arg_i]);
23492232 // Here we do not use setRegOrMem even though the logic is similar, because
......@@ -2355,9 +2238,9 @@ fn airCall(self: *Self, inst: Air.Inst.Index) !void {
23552238 try self.genSetReg(arg_ty, reg, arg_mcv);
23562239 },
23572240 .stack_offset => |off| {
2358 const abi_size = arg_ty.abiSize(self.target.*);
2241 const abi_size = @intCast(u32, arg_ty.abiSize(self.target.*));
23592242 try self.genSetStackArg(arg_ty, off, arg_mcv);
2360 stack_adjustment += @intCast(u32, abi_size);
2243 stack_adjustment += abi_size;
23612244 },
23622245 .ptr_stack_offset => {
23632246 return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{});
......@@ -3269,7 +3152,65 @@ fn setRegOrMem(self: *Self, ty: Type, loc: MCValue, val: MCValue) !void {
32693152 }
32703153}
32713154
3272fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerError!void {
3155fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerError!void {
3156 const abi_size = ty.abiSize(self.target.*);
3157 switch (mcv) {
3158 .dead => unreachable,
3159 .ptr_embedded_in_code => unreachable,
3160 .unreach, .none => return,
3161 .register => |reg| {
3162 _ = try self.addInst(.{
3163 .tag = .mov,
3164 .ops = (Mir.Ops{
3165 .reg1 = .rsp,
3166 .reg2 = registerAlias(reg, @intCast(u32, abi_size)),
3167 .flags = 0b10,
3168 }).encode(),
3169 .data = .{ .imm = @bitCast(u32, -(stack_offset + @intCast(i32, abi_size))) },
3170 });
3171 },
3172 .ptr_stack_offset => {
3173 const reg = try self.copyToTmpRegister(ty, mcv);
3174 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
3175 },
3176 .stack_offset => |unadjusted_off| {
3177 if (abi_size <= 8) {
3178 const reg = try self.copyToTmpRegister(ty, mcv);
3179 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
3180 }
3181
3182 const regs = try self.register_manager.allocRegs(3, .{ null, null, null }, &.{ .rax, .rcx });
3183 const addr_reg = regs[0];
3184 const count_reg = regs[1];
3185 const tmp_reg = regs[2];
3186
3187 try self.register_manager.getReg(.rax, null);
3188 try self.register_manager.getReg(.rcx, null);
3189
3190 _ = try self.addInst(.{
3191 .tag = .lea,
3192 .ops = (Mir.Ops{
3193 .reg1 = addr_reg.to64(),
3194 .reg2 = .rbp,
3195 }).encode(),
3196 .data = .{ .imm = @bitCast(u32, -(unadjusted_off + @intCast(i32, abi_size))) },
3197 });
3198
3199 // TODO allow for abi_size to be u64
3200 try self.genSetReg(Type.initTag(.u32), count_reg, .{ .immediate = @intCast(u32, abi_size) });
3201 try self.genInlineMemcpy(
3202 -(stack_offset + @intCast(i32, abi_size)),
3203 .rsp,
3204 addr_reg.to64(),
3205 count_reg.to64(),
3206 tmp_reg.to8(),
3207 );
3208 },
3209 else => return self.fail("TODO implement args on stack for {}", .{mcv}),
3210 }
3211}
3212
3213fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerError!void {
32733214 switch (mcv) {
32743215 .dead => unreachable,
32753216 .ptr_embedded_in_code => unreachable,
......@@ -3296,7 +3237,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
32963237 },
32973238 .immediate => |x_big| {
32983239 const abi_size = ty.abiSize(self.target.*);
3299 const adj_off = stack_offset + abi_size;
3240 const adj_off = stack_offset + @intCast(i32, abi_size);
33003241 if (adj_off > 128) {
33013242 return self.fail("TODO implement set stack variable with large stack offset", .{});
33023243 }
......@@ -3306,7 +3247,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
33063247 // offset from rbp, which is at the top of the stack frame.
33073248 // mov [rbp+offset], immediate
33083249 const payload = try self.addExtra(Mir.ImmPair{
3309 .dest_off = @bitCast(u32, -@intCast(i32, adj_off)),
3250 .dest_off = @bitCast(u32, -adj_off),
33103251 .operand = @truncate(u32, x_big),
33113252 });
33123253 _ = try self.addInst(.{
......@@ -3326,7 +3267,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
33263267 8 => {
33273268 // We have a positive stack offset value but we want a twos complement negative
33283269 // offset from rbp, which is at the top of the stack frame.
3329 const negative_offset = -@intCast(i32, adj_off);
3270 const negative_offset = -adj_off;
33303271
33313272 // 64 bit write to memory would take two mov's anyways so we
33323273 // insted just use two 32 bit writes to avoid register allocation
......@@ -3369,7 +3310,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
33693310 return self.fail("stack offset too large", .{});
33703311 }
33713312 const abi_size = ty.abiSize(self.target.*);
3372 const adj_off = stack_offset + abi_size;
3313 const adj_off = stack_offset + @intCast(i32, abi_size);
33733314 _ = try self.addInst(.{
33743315 .tag = .mov,
33753316 .ops = (Mir.Ops{
......@@ -3377,7 +3318,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
33773318 .reg2 = registerAlias(reg, @intCast(u32, abi_size)),
33783319 .flags = 0b10,
33793320 }).encode(),
3380 .data = .{ .imm = @bitCast(u32, -@intCast(i32, adj_off)) },
3321 .data = .{ .imm = @bitCast(u32, -adj_off) },
33813322 });
33823323 },
33833324 .memory, .embedded_in_code => {
......@@ -3403,7 +3344,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
34033344 return self.genSetStack(ty, stack_offset, MCValue{ .register = reg });
34043345 }
34053346
3406 const regs = try self.register_manager.allocRegs(3, .{ null, null, null }, &.{ .rax, .rcx });
3347 const regs = try self.register_manager.allocRegs(3, .{ null, null, null }, &.{ .rax, .rcx, .rbp });
34073348 const addr_reg = regs[0];
34083349 const count_reg = regs[1];
34093350 const tmp_reg = regs[2];
......@@ -3417,14 +3358,14 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
34173358 .reg1 = addr_reg.to64(),
34183359 .reg2 = .rbp,
34193360 }).encode(),
3420 .data = .{ .imm = @bitCast(u32, -@intCast(i32, off + abi_size)) },
3361 .data = .{ .imm = @bitCast(u32, -(off + @intCast(i32, abi_size))) },
34213362 });
34223363
34233364 // TODO allow for abi_size to be u64
34243365 try self.genSetReg(Type.initTag(.u32), count_reg, .{ .immediate = @intCast(u32, abi_size) });
34253366
34263367 return self.genInlineMemcpy(
3427 @bitCast(u32, -@intCast(i32, stack_offset + abi_size)),
3368 -(stack_offset + @intCast(i32, abi_size)),
34283369 .rbp,
34293370 addr_reg.to64(),
34303371 count_reg.to64(),
......@@ -3436,7 +3377,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro
34363377
34373378fn genInlineMemcpy(
34383379 self: *Self,
3439 stack_offset: u32,
3380 stack_offset: i32,
34403381 stack_reg: Register,
34413382 addr_reg: Register,
34423383 count_reg: Register,
......@@ -3494,7 +3435,7 @@ fn genInlineMemcpy(
34943435 .reg1 = stack_reg,
34953436 .reg2 = tmp_reg.to8(),
34963437 }).encode(),
3497 .data = .{ .imm = stack_offset },
3438 .data = .{ .imm = @bitCast(u32, stack_offset) },
34983439 });
34993440
35003441 // add rcx, 1
......@@ -3535,14 +3476,14 @@ fn genInlineMemcpy(
35353476 try self.performReloc(loop_reloc);
35363477}
35373478
3538fn genInlineMemset(self: *Self, ty: Type, stack_offset: u32, value: MCValue) InnerError!void {
3479fn genInlineMemset(self: *Self, ty: Type, stack_offset: i32, value: MCValue) InnerError!void {
35393480 try self.register_manager.getReg(.rax, null);
35403481 const abi_size = ty.abiSize(self.target.*);
3541 const adj_off = stack_offset + abi_size;
3482 const adj_off = stack_offset + @intCast(i32, abi_size);
35423483 if (adj_off > 128) {
35433484 return self.fail("TODO inline memset with large stack offset", .{});
35443485 }
3545 const negative_offset = @bitCast(u32, -@intCast(i32, adj_off));
3486 const negative_offset = @bitCast(u32, -adj_off);
35463487
35473488 // We are actually counting `abi_size` bytes; however, we reuse the index register
35483489 // as both the counter and offset scaler, hence we need to subtract one from `abi_size`
......@@ -3633,7 +3574,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
36333574 const ptr_abi_size = ty.abiSize(self.target.*);
36343575 const elem_ty = ty.childType();
36353576 const elem_abi_size = elem_ty.abiSize(self.target.*);
3636 const off = unadjusted_off + elem_abi_size;
3577 const off = unadjusted_off + @intCast(i32, elem_abi_size);
36373578 if (off < std.math.minInt(i32) or off > std.math.maxInt(i32)) {
36383579 return self.fail("stack offset too large", .{});
36393580 }
......@@ -3643,7 +3584,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
36433584 .reg1 = registerAlias(reg, @intCast(u32, ptr_abi_size)),
36443585 .reg2 = .rbp,
36453586 }).encode(),
3646 .data = .{ .imm = @bitCast(u32, -@intCast(i32, off)) },
3587 .data = .{ .imm = @bitCast(u32, -off) },
36473588 });
36483589 },
36493590 .ptr_embedded_in_code => unreachable,
......@@ -3830,7 +3771,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
38303771 },
38313772 .stack_offset => |unadjusted_off| {
38323773 const abi_size = ty.abiSize(self.target.*);
3833 const off = unadjusted_off + abi_size;
3774 const off = unadjusted_off + @intCast(i32, abi_size);
38343775 if (off < std.math.minInt(i32) or off > std.math.maxInt(i32)) {
38353776 return self.fail("stack offset too large", .{});
38363777 }
......@@ -3841,7 +3782,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
38413782 .reg2 = .rbp,
38423783 .flags = 0b01,
38433784 }).encode(),
3844 .data = .{ .imm = @bitCast(u32, -@intCast(i32, off)) },
3785 .data = .{ .imm = @bitCast(u32, -off) },
38453786 });
38463787 },
38473788 }
......@@ -3866,7 +3807,7 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
38663807 const array_ty = ptr_ty.childType();
38673808 const array_len = array_ty.arrayLenIncludingSentinel();
38683809 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else blk: {
3869 const stack_offset = try self.allocMem(inst, 16, 16);
3810 const stack_offset = @intCast(i32, try self.allocMem(inst, 16, 16));
38703811 try self.genSetStack(ptr_ty, stack_offset + 8, ptr);
38713812 try self.genSetStack(Type.initTag(.u64), stack_offset, .{ .immediate = array_len });
38723813 break :blk .{ .stack_offset = stack_offset };
......@@ -4247,6 +4188,7 @@ fn resolveCallingConventionValues(self: *Self, fn_ty: Type) !CallMCValues {
42474188 var next_stack_offset: u32 = 0;
42484189 var count: usize = param_types.len;
42494190 while (count > 0) : (count -= 1) {
4191 // for (param_types) |ty, i| {
42504192 const i = count - 1;
42514193 const ty = param_types[i];
42524194 if (!ty.hasCodeGenBits()) {
......@@ -4265,7 +4207,7 @@ fn resolveCallingConventionValues(self: *Self, fn_ty: Type) !CallMCValues {
42654207 // such as ptr and len of slices as separate registers.
42664208 // TODO: also we need to honor the C ABI for relevant types rather than passing on
42674209 // the stack here.
4268 result.args[i] = .{ .stack_offset = next_stack_offset };
4210 result.args[i] = .{ .stack_offset = @intCast(i32, next_stack_offset) };
42694211 next_stack_offset += param_size;
42704212 }
42714213 }
src/arch/x86_64/Emit.zig+19-17
......@@ -251,23 +251,25 @@ fn mirPushPop(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
251251 }
252252}
253253fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
254 const callee_preserved_regs = bits.callee_preserved_regs;
255 const regs = emit.mir.instructions.items(.data)[inst].regs_to_push_or_pop;
256 if (tag == .push) {
257 for (callee_preserved_regs) |reg, i| {
258 if ((regs >> @intCast(u5, i)) & 1 == 0) continue;
259 lowerToOEnc(.push, reg, emit.code) catch |err|
260 return emit.failWithLoweringError(err);
261 }
262 } else {
263 // pop in the reverse direction
264 var i = callee_preserved_regs.len;
265 while (i > 0) : (i -= 1) {
266 const reg = callee_preserved_regs[i - 1];
267 if ((regs >> @intCast(u5, i - 1)) & 1 == 0) continue;
268 lowerToOEnc(.pop, reg, emit.code) catch |err|
269 return emit.failWithLoweringError(err);
254 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
255 const payload = emit.mir.instructions.items(.data)[inst].payload;
256 const data = emit.mir.extraData(Mir.RegsToPushOrPop, payload).data;
257 const regs = data.regs;
258 var disp: u32 = data.disp + 8;
259 for (bits.callee_preserved_regs) |reg, i| {
260 if ((regs >> @intCast(u5, i)) & 1 == 0) continue;
261 if (tag == .push) {
262 lowerToMrEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{
263 .disp = @bitCast(u32, -@intCast(i32, disp)),
264 .base = ops.reg1,
265 }), reg.to64(), emit.code) catch |err| return emit.failWithLoweringError(err);
266 } else {
267 lowerToRmEnc(.mov, reg.to64(), RegisterOrMemory.mem(.qword_ptr, .{
268 .disp = @bitCast(u32, -@intCast(i32, disp)),
269 .base = ops.reg1,
270 }), emit.code) catch |err| return emit.failWithLoweringError(err);
270271 }
272 disp += 8;
271273 }
272274}
273275
......@@ -1603,7 +1605,7 @@ fn lowerToRmEnc(
16031605 if (reg.size() != src_reg.size()) {
16041606 return error.OperandSizeMismatch;
16051607 }
1606 const encoder = try Encoder.init(code, 3);
1608 const encoder = try Encoder.init(code, 4);
16071609 encoder.rex(.{
16081610 .w = setRexWRegister(reg) or setRexWRegister(src_reg),
16091611 .r = reg.isExtended(),
src/arch/x86_64/Mir.zig+5-2
......@@ -333,8 +333,6 @@ pub const Inst = struct {
333333 got_entry: u32,
334334 /// Index into `extra`. Meaning of what can be found there is context-dependent.
335335 payload: u32,
336 /// A bitfield of which callee_preserved_regs to push
337 regs_to_push_or_pop: u32,
338336 };
339337
340338 // Make sure we don't accidentally make instructions bigger than expected.
......@@ -346,6 +344,11 @@ pub const Inst = struct {
346344 }
347345};
348346
347pub const RegsToPushOrPop = struct {
348 regs: u32,
349 disp: u32,
350};
351
349352pub const ImmPair = struct {
350353 dest_off: u32,
351354 operand: u32,
src/arch/x86_64/PrintMir.zig+21-19
......@@ -180,26 +180,28 @@ fn mirPushPop(print: *const Print, tag: Mir.Inst.Tag, inst: Mir.Inst.Index, w: a
180180 try w.writeByte('\n');
181181}
182182fn mirPushPopRegsFromCalleePreservedRegs(print: *const Print, tag: Mir.Inst.Tag, inst: Mir.Inst.Index, w: anytype) !void {
183 const callee_preserved_regs = bits.callee_preserved_regs;
184 // PUSH/POP reg
185
186 const regs = print.mir.instructions.items(.data)[inst].regs_to_push_or_pop;
187 if (regs == 0) return w.writeAll("push/pop no regs from callee_preserved_regs\n");
188 if (tag == .push) {
189 try w.writeAll("push ");
190 for (callee_preserved_regs) |reg, i| {
191 if ((regs >> @intCast(u5, i)) & 1 == 0) continue;
192 try w.print("{s}, ", .{@tagName(reg)});
193 }
194 } else {
195 // pop in the reverse direction
196 var i = callee_preserved_regs.len;
197 try w.writeAll("pop ");
198 while (i > 0) : (i -= 1) {
199 if ((regs >> @intCast(u5, i - 1)) & 1 == 0) continue;
200 const reg = callee_preserved_regs[i - 1];
201 try w.print("{s}, ", .{@tagName(reg)});
183 const ops = Mir.Ops.decode(print.mir.instructions.items(.ops)[inst]);
184 const payload = print.mir.instructions.items(.data)[inst].payload;
185 const data = print.mir.extraData(Mir.RegsToPushOrPop, payload).data;
186 const regs = data.regs;
187 var disp: u32 = data.disp + 8;
188 if (regs == 0) return w.writeAll("no regs from callee_preserved_regs\n");
189 for (bits.callee_preserved_regs) |reg, i| {
190 if ((regs >> @intCast(u5, i)) & 1 == 0) continue;
191 if (tag == .push) {
192 try w.print("mov qword ptr [{s} + {d}], {s}", .{
193 @tagName(ops.reg1),
194 @bitCast(u32, -@intCast(i32, disp)),
195 @tagName(reg.to64()),
196 });
197 } else {
198 try w.print("mov {s}, qword ptr [{s} + {d}]", .{
199 @tagName(reg.to64()),
200 @tagName(ops.reg1),
201 @bitCast(u32, -@intCast(i32, disp)),
202 });
202203 }
204 disp += 8;
203205 }
204206 try w.writeByte('\n');
205207}
src/link/Elf.zig+1-1
......@@ -2118,7 +2118,7 @@ fn allocateTextBlock(self: *Elf, block_list: *TextBlockList, text_block: *TextBl
21182118 const sym = self.local_symbols.items[big_block.local_sym_index];
21192119 const capacity = big_block.capacity(self.*);
21202120 const ideal_capacity = padToIdeal(capacity);
2121 const ideal_capacity_end_vaddr = sym.st_value + ideal_capacity;
2121 const ideal_capacity_end_vaddr = std.math.add(u64, sym.st_value, ideal_capacity) catch ideal_capacity;
21222122 const capacity_end_vaddr = sym.st_value + capacity;
21232123 const new_start_vaddr_unaligned = capacity_end_vaddr - new_block_ideal_capacity;
21242124 const new_start_vaddr = mem.alignBackwardGeneric(u64, new_start_vaddr_unaligned, alignment);
src/link/MachO.zig+1-1
......@@ -5064,7 +5064,7 @@ fn allocateAtom(self: *MachO, atom: *Atom, new_atom_size: u64, alignment: u64, m
50645064 const sym = self.locals.items[big_atom.local_sym_index];
50655065 const capacity = big_atom.capacity(self.*);
50665066 const ideal_capacity = if (needs_padding) padToIdeal(capacity) else capacity;
5067 const ideal_capacity_end_vaddr = sym.n_value + ideal_capacity;
5067 const ideal_capacity_end_vaddr = math.add(u64, sym.n_value, ideal_capacity) catch ideal_capacity;
50685068 const capacity_end_vaddr = sym.n_value + capacity;
50695069 const new_start_vaddr_unaligned = capacity_end_vaddr - new_atom_ideal_capacity;
50705070 const new_start_vaddr = mem.alignBackwardGeneric(u64, new_start_vaddr_unaligned, alignment);