authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-04-23 22:25:53-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-04-26 19:05:17-04:00
log26d4f9b69e087f3a9ef818f878f12affc94317f9
tree6f9029aca24e97d961fcfbb10c09191b568eae83
parentf618398b24acdc3317e6fd81f486d49176ffcef9

x86_64: add frame indices

This allows abstract references to the stack frame which enables: * deferred frame layout until after the function has been processed * frame compaction (sorting by alignment) * being able to overalign the stack * references that change based on an overaligned stack * moving callee saved spills to the usual part (smaller code size) * shared call frame (avoids stack adjustments before and after calls)

7 files changed, 2503 insertions(+), 2542 deletions(-)

src/arch/x86_64/CodeGen.zig+2234-2362
...@@ -6,6 +6,7 @@ const codegen = @import("../../codegen.zig");...@@ -6,6 +6,7 @@ const codegen = @import("../../codegen.zig");
6const leb128 = std.leb;6const leb128 = std.leb;
7const link = @import("../../link.zig");7const link = @import("../../link.zig");
8const log = std.log.scoped(.codegen);8const log = std.log.scoped(.codegen);
9const tracking_log = std.log.scoped(.tracking);
9const math = std.math;10const math = std.math;
10const mem = std.mem;11const mem = std.mem;
11const trace = @import("../../tracy.zig").trace;12const trace = @import("../../tracy.zig").trace;
...@@ -40,6 +41,7 @@ const Memory = bits.Memory;...@@ -40,6 +41,7 @@ const Memory = bits.Memory;
40const Register = bits.Register;41const Register = bits.Register;
41const RegisterManager = abi.RegisterManager;42const RegisterManager = abi.RegisterManager;
42const RegisterLock = RegisterManager.RegisterLock;43const RegisterLock = RegisterManager.RegisterLock;
44const FrameIndex = bits.FrameIndex;
4345
44const gp = abi.RegisterClass.gp;46const gp = abi.RegisterClass.gp;
45const sse = abi.RegisterClass.sse;47const sse = abi.RegisterClass.sse;
...@@ -47,6 +49,7 @@ const sse = abi.RegisterClass.sse;...@@ -47,6 +49,7 @@ const sse = abi.RegisterClass.sse;
47const InnerError = CodeGenError || error{OutOfRegisters};49const InnerError = CodeGenError || error{OutOfRegisters};
4850
49const debug_wip_mir = false;51const debug_wip_mir = false;
52const debug_tracking = false;
5053
51gpa: Allocator,54gpa: Allocator,
52air: Air,55air: Air,
...@@ -57,11 +60,10 @@ target: *const std.Target,...@@ -57,11 +60,10 @@ target: *const std.Target,
57mod_fn: *const Module.Fn,60mod_fn: *const Module.Fn,
58err_msg: ?*ErrorMsg,61err_msg: ?*ErrorMsg,
59args: []MCValue,62args: []MCValue,
60ret_mcv: MCValue,63ret_mcv: InstTracking,
61fn_type: Type,64fn_type: Type,
62arg_index: u32,65arg_index: u32,
63src_loc: Module.SrcLoc,66src_loc: Module.SrcLoc,
64stack_align: u32,
6567
66eflags_inst: ?Air.Inst.Index = null,68eflags_inst: ?Air.Inst.Index = null,
6769
...@@ -86,17 +88,13 @@ inst_tracking: InstTrackingMap = .{},...@@ -86,17 +88,13 @@ inst_tracking: InstTrackingMap = .{},
86blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .{},88blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .{},
8789
88register_manager: RegisterManager = .{},90register_manager: RegisterManager = .{},
89/// Maps offset to what is stored there.
90stack: std.AutoHashMapUnmanaged(u32, StackAllocation) = .{},
9191
92/// Generation of the current scope, increments by 1 for every entered scope.92/// Generation of the current scope, increments by 1 for every entered scope.
93scope_generation: u32 = 0,93scope_generation: u32 = 0,
9494
95/// Offset from the stack base, representing the end of the stack frame.95frame_allocs: std.MultiArrayList(FrameAlloc) = .{},
96max_end_stack: u32 = 0,96free_frame_indices: std.AutoArrayHashMapUnmanaged(FrameIndex, void) = .{},
97/// Represents the current end stack offset. If there is no existing slot97frame_locs: std.MultiArrayList(Mir.FrameLoc) = .{},
98/// to place a new stack allocation, it goes here, and then bumps `max_end_stack`.
99next_stack_offset: u32 = 0,
10098
101/// Debug field, used to find bugs in the compiler.99/// Debug field, used to find bugs in the compiler.
102air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init,100air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init,
...@@ -108,6 +106,9 @@ const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {}...@@ -108,6 +106,9 @@ const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {}
108106
109const mir_to_air_map_init = if (builtin.mode == .Debug) std.AutoHashMapUnmanaged(Mir.Inst.Index, Air.Inst.Index){} else {};107const mir_to_air_map_init = if (builtin.mode == .Debug) std.AutoHashMapUnmanaged(Mir.Inst.Index, Air.Inst.Index){} else {};
110108
109const FrameAddr = struct { index: FrameIndex, off: i32 = 0 };
110const RegisterOffset = struct { reg: Register, off: i32 = 0 };
111
111pub const MCValue = union(enum) {112pub const MCValue = union(enum) {
112 /// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc.113 /// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc.
113 /// TODO Look into deleting this tag and using `dead` instead, since every use114 /// TODO Look into deleting this tag and using `dead` instead, since every use
...@@ -123,46 +124,59 @@ pub const MCValue = union(enum) {...@@ -123,46 +124,59 @@ pub const MCValue = union(enum) {
123 /// A pointer-sized integer that fits in a register.124 /// A pointer-sized integer that fits in a register.
124 /// If the type is a pointer, this is the pointer address in virtual address space.125 /// If the type is a pointer, this is the pointer address in virtual address space.
125 immediate: u64,126 immediate: u64,
126 /// The value is in a GP register.127 /// The value resides in the EFLAGS register.
128 eflags: Condition,
129 /// The value is in a register.
127 register: Register,130 register: Register,
131 /// The value is a constant offset from the value in a register.
132 register_offset: RegisterOffset,
128 /// The value is a tuple { wrapped, overflow } where wrapped value is stored in the GP register.133 /// The value is a tuple { wrapped, overflow } where wrapped value is stored in the GP register.
129 register_overflow: struct { reg: Register, eflags: Condition },134 register_overflow: struct { reg: Register, eflags: Condition },
130 /// The value is in memory at a hard-coded address.135 /// The value is in memory at a hard-coded address.
131 /// If the type is a pointer, it means the pointer address is at this memory location.136 /// If the type is a pointer, it means the pointer address is at this memory location.
132 memory: u64,137 memory: u64,
138 /// The value is in memory at a constant offset from the address in a register.
139 indirect: RegisterOffset,
133 /// The value is in memory.140 /// The value is in memory.
134 /// Payload is a symbol index.141 /// Payload is a symbol index.
135 load_direct: u32,142 load_direct: u32,
136 /// The value is a pointer to value in memory.143 /// The value is a pointer to a value in memory.
137 /// Payload is a symbol index.144 /// Payload is a symbol index.
138 lea_direct: u32,145 lea_direct: u32,
139 /// The value is in memory referenced indirectly via GOT.146 /// The value is in memory referenced indirectly via GOT.
140 /// Payload is a symbol index.147 /// Payload is a symbol index.
141 load_got: u32,148 load_got: u32,
149 /// The value is a pointer to a value referenced indirectly via GOT.
150 /// Payload is a symbol index.
151 lea_got: u32,
142 /// The value is a threadlocal variable.152 /// The value is a threadlocal variable.
143 /// Payload is a symbol index.153 /// Payload is a symbol index.
144 load_tlv: u32,154 load_tlv: u32,
145 /// The value is a pointer to threadlocal variable.155 /// The value is a pointer to a threadlocal variable.
146 /// Payload is a symbol index.156 /// Payload is a symbol index.
147 lea_tlv: u32,157 lea_tlv: u32,
148 /// The value is one of the stack variables.158 /// The value stored at an offset from a frame index
149 /// If the type is a pointer, it means the pointer address is in the stack at this offset.159 /// Payload is a frame address.
150 stack_offset: i32,160 load_frame: FrameAddr,
151 /// The value is a pointer to one of the stack variables (payload is stack offset).161 /// The address of an offset from a frame index
152 ptr_stack_offset: i32,162 /// Payload is a frame address.
153 /// The value resides in the EFLAGS register.163 lea_frame: FrameAddr,
154 eflags: Condition,164 /// This indicates that we have already allocated a frame index for this instruction,
165 /// but it has not been spilled there yet in the current control flow.
166 /// Payload is a frame index.
167 reserved_frame: FrameIndex,
155168
156 fn isMemory(mcv: MCValue) bool {169 fn isMemory(mcv: MCValue) bool {
157 return switch (mcv) {170 return switch (mcv) {
158 .memory,171 .memory,
159 .stack_offset,
160 .ptr_stack_offset,
161 .load_direct,172 .load_direct,
162 .lea_direct,173 .lea_direct,
163 .load_got,174 .load_got,
175 .lea_got,
164 .load_tlv,176 .load_tlv,
165 .lea_tlv,177 .lea_tlv,
178 .load_frame,
179 .lea_frame,
166 => true,180 => true,
167 else => false,181 else => false,
168 };182 };
...@@ -181,6 +195,134 @@ pub const MCValue = union(enum) {...@@ -181,6 +195,134 @@ pub const MCValue = union(enum) {
181 else => false,195 else => false,
182 };196 };
183 }197 }
198
199 fn getReg(mcv: MCValue) ?Register {
200 return switch (mcv) {
201 .register => |reg| reg,
202 .register_offset, .indirect => |ro| ro.reg,
203 .register_overflow => |ro| ro.reg,
204 else => null,
205 };
206 }
207
208 fn getCondition(mcv: MCValue) ?Condition {
209 return switch (mcv) {
210 .eflags => |cc| cc,
211 .register_overflow => |reg_ov| reg_ov.eflags,
212 else => null,
213 };
214 }
215
216 fn address(mcv: MCValue) MCValue {
217 return switch (mcv) {
218 .none,
219 .unreach,
220 .dead,
221 .undef,
222 .immediate,
223 .register,
224 .register_offset,
225 .eflags,
226 .register_overflow,
227 .lea_direct,
228 .lea_got,
229 .lea_tlv,
230 .lea_frame,
231 .reserved_frame,
232 => unreachable, // not in memory
233 .memory => |addr| .{ .immediate = addr },
234 .indirect => |reg_off| switch (reg_off.off) {
235 0 => .{ .register = reg_off.reg },
236 else => .{ .register_offset = reg_off },
237 },
238 .load_direct => |sym_index| .{ .lea_direct = sym_index },
239 .load_got => |sym_index| .{ .lea_got = sym_index },
240 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
241 .load_frame => |frame_addr| .{ .lea_frame = frame_addr },
242 };
243 }
244
245 fn deref(mcv: MCValue) MCValue {
246 return switch (mcv) {
247 .none,
248 .unreach,
249 .dead,
250 .undef,
251 .eflags,
252 .register_overflow,
253 .memory,
254 .indirect,
255 .load_direct,
256 .load_got,
257 .load_tlv,
258 .load_frame,
259 .reserved_frame,
260 => unreachable, // not a dereferenceable
261 .immediate => |addr| .{ .memory = addr },
262 .register => |reg| .{ .indirect = .{ .reg = reg } },
263 .register_offset => |reg_off| .{ .indirect = reg_off },
264 .lea_direct => |sym_index| .{ .load_direct = sym_index },
265 .lea_got => |sym_index| .{ .load_got = sym_index },
266 .lea_tlv => |sym_index| .{ .load_tlv = sym_index },
267 .lea_frame => |frame_addr| .{ .load_frame = frame_addr },
268 };
269 }
270
271 fn offset(mcv: MCValue, off: i32) MCValue {
272 return switch (mcv) {
273 .none,
274 .unreach,
275 .dead,
276 .undef,
277 .eflags,
278 .register_overflow,
279 .memory,
280 .indirect,
281 .load_direct,
282 .lea_direct,
283 .load_got,
284 .lea_got,
285 .load_tlv,
286 .lea_tlv,
287 .load_frame,
288 .reserved_frame,
289 => unreachable, // not offsettable
290 .immediate => |imm| .{ .immediate = @bitCast(u64, @bitCast(i64, imm) +% off) },
291 .register => |reg| .{ .register_offset = .{ .reg = reg, .off = off } },
292 .register_offset => |reg_off| .{
293 .register_offset = .{ .reg = reg_off.reg, .off = reg_off.off + off },
294 },
295 .lea_frame => |frame_addr| .{
296 .lea_frame = .{ .index = frame_addr.index, .off = frame_addr.off + off },
297 },
298 };
299 }
300
301 pub fn format(
302 mcv: MCValue,
303 comptime _: []const u8,
304 _: std.fmt.FormatOptions,
305 writer: anytype,
306 ) @TypeOf(writer).Error!void {
307 switch (mcv) {
308 .none, .unreach, .dead, .undef => try writer.print("({s})", .{@tagName(mcv)}),
309 .immediate => |pl| try writer.print("0x{x}", .{pl}),
310 .memory => |pl| try writer.print("[ds:0x{x}]", .{pl}),
311 inline .eflags, .register => |pl| try writer.print("{s}", .{@tagName(pl)}),
312 .register_offset => |pl| try writer.print("{s} + 0x{x}", .{ @tagName(pl.reg), pl.off }),
313 .register_overflow => |pl| try writer.print("{s}:{s}", .{ @tagName(pl.eflags), @tagName(pl.reg) }),
314 .indirect => |pl| try writer.print("[{s} + 0x{x}]", .{ @tagName(pl.reg), pl.off }),
315 .load_direct => |pl| try writer.print("[direct:{d}]", .{pl}),
316 .lea_direct => |pl| try writer.print("direct:{d}", .{pl}),
317 .load_got => |pl| try writer.print("[got:{d}]", .{pl}),
318 .lea_got => |pl| try writer.print("got:{d}", .{pl}),
319 .load_tlv => |pl| try writer.print("[tlv:{d}]", .{pl}),
320 .lea_tlv => |pl| try writer.print("tlv:{d}", .{pl}),
321 .load_frame => |pl| try writer.print("[{} + 0x{x}]", .{ pl.index, pl.off }),
322 .lea_frame => |pl| try writer.print("{} + 0x{x}", .{ pl.index, pl.off }),
323 .reserved_frame => |pl| try writer.print("(dead:{})", .{pl}),
324 }
325 }
184};326};
185327
186const InstTrackingMap = std.AutoArrayHashMapUnmanaged(Air.Inst.Index, InstTracking);328const InstTrackingMap = std.AutoArrayHashMapUnmanaged(Air.Inst.Index, InstTracking);
...@@ -189,89 +331,211 @@ const InstTracking = struct {...@@ -189,89 +331,211 @@ const InstTracking = struct {
189 short: MCValue,331 short: MCValue,
190332
191 fn init(result: MCValue) InstTracking {333 fn init(result: MCValue) InstTracking {
192 return .{ .long = result, .short = result };334 return .{ .long = switch (result) {
335 .none,
336 .unreach,
337 .undef,
338 .immediate,
339 .memory,
340 .load_direct,
341 .lea_direct,
342 .load_got,
343 .lea_got,
344 .load_tlv,
345 .lea_tlv,
346 .load_frame,
347 .lea_frame,
348 => result,
349 .dead,
350 .reserved_frame,
351 => unreachable,
352 .eflags,
353 .register,
354 .register_offset,
355 .register_overflow,
356 .indirect,
357 => .none,
358 }, .short = result };
193 }359 }
194360
195 fn getReg(self: InstTracking) ?Register {361 fn getReg(self: InstTracking) ?Register {
196 return switch (self.short) {362 return self.short.getReg();
197 .register => |reg| reg,
198 .register_overflow => |ro| ro.reg,
199 else => null,
200 };
201 }363 }
202364
203 fn getCondition(self: InstTracking) ?Condition {365 fn getCondition(self: InstTracking) ?Condition {
204 return switch (self.short) {366 return self.short.getCondition();
205 .eflags => |eflags| eflags,
206 .register_overflow => |ro| ro.eflags,
207 else => null,
208 };
209 }367 }
210368
211 fn spill(self: *InstTracking, function: *Self, inst: Air.Inst.Index) !void {369 fn spill(self: *InstTracking, function: *Self, inst: Air.Inst.Index) !void {
370 if (std.meta.eql(self.long, self.short)) return; // Already spilled
371 // Allocate or reuse frame index
372 switch (self.long) {
373 .none => self.long = try function.allocRegOrMem(inst, false),
374 .load_frame => {},
375 .reserved_frame => |index| self.long = .{ .load_frame = .{ .index = index } },
376 else => unreachable,
377 }
378 tracking_log.debug("spill %{d} from {} to {}", .{ inst, self.short, self.long });
379 try function.genCopy(function.air.typeOfIndex(inst), self.long, self.short);
380 }
381
382 fn reuseFrame(self: *InstTracking) void {
212 switch (self.long) {383 switch (self.long) {
384 .reserved_frame => |index| self.long = .{ .load_frame = .{ .index = index } },
385 else => {},
386 }
387 self.short = switch (self.long) {
213 .none,388 .none,
214 .dead,
215 .unreach,389 .unreach,
216 => unreachable,390 .undef,
391 .immediate,
392 .memory,
393 .load_direct,
394 .lea_direct,
395 .load_got,
396 .lea_got,
397 .load_tlv,
398 .lea_tlv,
399 .load_frame,
400 .lea_frame,
401 => self.long,
402 .dead,
403 .eflags,
217 .register,404 .register,
405 .register_offset,
218 .register_overflow,406 .register_overflow,
219 .eflags,407 .indirect,
220 => self.long = try function.allocRegOrMem(inst, self.short == .eflags),408 .reserved_frame,
221 .stack_offset => {},409 => unreachable,
410 };
411 }
412
413 fn trackSpill(self: *InstTracking, function: *Self, inst: Air.Inst.Index) void {
414 function.freeValue(self.short);
415 self.reuseFrame();
416 tracking_log.debug("%{d} => {} (spilled)", .{ inst, self.* });
417 }
418
419 fn verifyMaterialize(self: *InstTracking, target: InstTracking) void {
420 switch (self.long) {
421 .none,
422 .unreach,
222 .undef,423 .undef,
223 .immediate,424 .immediate,
224 .memory,425 .memory,
225 .load_direct,426 .load_direct,
226 .lea_direct,427 .lea_direct,
227 .load_got,428 .load_got,
429 .lea_got,
228 .load_tlv,430 .load_tlv,
229 .lea_tlv,431 .lea_tlv,
230 .ptr_stack_offset,432 .lea_frame,
231 => return, // these can be rematerialized without using a stack slot433 => assert(std.meta.eql(self.long, target.long)),
434 .load_frame,
435 .reserved_frame,
436 => switch (target.long) {
437 .none,
438 .load_frame,
439 .reserved_frame,
440 => {},
441 else => unreachable,
442 },
443 .dead,
444 .eflags,
445 .register,
446 .register_offset,
447 .register_overflow,
448 .indirect,
449 => unreachable,
232 }450 }
233 log.debug("spilling %{d} from {} to {}", .{ inst, self.short, self.long });
234 const ty = function.air.typeOfIndex(inst);
235 try function.setRegOrMem(ty, self.long, self.short);
236 }451 }
237452
238 fn trackSpill(self: *InstTracking, function: *Self) void {453 fn materialize(
239 if (self.getReg()) |reg| function.register_manager.freeReg(reg);454 self: *InstTracking,
240 switch (self.short) {455 function: *Self,
241 .none, .dead, .unreach => unreachable,456 inst: Air.Inst.Index,
242 else => {},457 target: InstTracking,
243 }458 ) !void {
244 self.short = self.long;459 self.verifyMaterialize(target);
460 try self.materializeUnsafe(function, inst, target);
245 }461 }
246462
247 fn materialize(self: *InstTracking, function: *Self, inst: Air.Inst.Index, reg: Register) !void {463 fn materializeUnsafe(
464 self: *InstTracking,
465 function: *Self,
466 inst: Air.Inst.Index,
467 target: InstTracking,
468 ) !void {
248 const ty = function.air.typeOfIndex(inst);469 const ty = function.air.typeOfIndex(inst);
249 try function.genSetReg(ty, reg, self.long);470 if ((self.long == .none or self.long == .reserved_frame) and target.long == .load_frame)
471 try function.genCopy(ty, target.long, self.short);
472 try function.genCopy(ty, target.short, self.short);
250 }473 }
251474
252 fn trackMaterialize(self: *InstTracking, function: *Self, inst: Air.Inst.Index, reg: Register) void {475 fn trackMaterialize(self: *InstTracking, inst: Air.Inst.Index, target: InstTracking) void {
253 assert(inst == function.register_manager.registers[476 self.verifyMaterialize(target);
254 RegisterManager.indexOfRegIntoTracked(reg).?477 // Don't clobber reserved frame indices
255 ]);478 self.long = if (target.long == .none) switch (self.long) {
256 self.short = .{ .register = reg };479 .load_frame => |addr| .{ .reserved_frame = addr.index },
480 .reserved_frame => self.long,
481 else => target.long,
482 } else target.long;
483 self.short = target.short;
484 tracking_log.debug("%{d} => {} (materialize)", .{ inst, self.* });
257 }485 }
258486
259 fn resurrect(self: *InstTracking, scope_generation: u32) void {487 fn resurrect(self: *InstTracking, inst: Air.Inst.Index, scope_generation: u32) void {
260 switch (self.short) {488 switch (self.short) {
261 .dead => |die_generation| if (die_generation >= scope_generation) {489 .dead => |die_generation| if (die_generation >= scope_generation) {
262 self.short = self.long;490 self.reuseFrame();
491 tracking_log.debug("%{d} => {} (resurrect)", .{ inst, self.* });
263 },492 },
264 else => {},493 else => {},
265 }494 }
266 }495 }
267496
268 fn die(self: *InstTracking, function: *Self) void {497 fn die(self: *InstTracking, function: *Self, inst: Air.Inst.Index) void {
269 function.freeValue(self.short);498 function.freeValue(self.short);
270 self.reuse(function);499 self.short = .{ .dead = function.scope_generation };
500 tracking_log.debug("%{d} => {} (death)", .{ inst, self.* });
271 }501 }
272502
273 fn reuse(self: *InstTracking, function: *Self) void {503 fn reuse(
504 self: *InstTracking,
505 function: *Self,
506 new_inst: Air.Inst.Index,
507 old_inst: Air.Inst.Index,
508 ) void {
274 self.short = .{ .dead = function.scope_generation };509 self.short = .{ .dead = function.scope_generation };
510 tracking_log.debug("%{d} => {} (reuse %{d})", .{ new_inst, self.*, old_inst });
511 }
512
513 pub fn format(
514 self: InstTracking,
515 comptime _: []const u8,
516 _: std.fmt.FormatOptions,
517 writer: anytype,
518 ) @TypeOf(writer).Error!void {
519 if (!std.meta.eql(self.long, self.short)) try writer.print("|{}| ", .{self.long});
520 try writer.print("{}", .{self.short});
521 }
522};
523
524const FrameAlloc = struct {
525 abi_size: u31,
526 abi_align: u5,
527 ref_count: u16,
528
529 fn init(alloc_abi: struct { size: u64, alignment: u32 }) FrameAlloc {
530 assert(math.isPowerOfTwo(alloc_abi.alignment));
531 return .{
532 .abi_size = @intCast(u31, alloc_abi.size),
533 .abi_align = math.log2_int(u32, alloc_abi.alignment),
534 .ref_count = 0,
535 };
536 }
537 fn initType(ty: Type, target: Target) FrameAlloc {
538 return init(.{ .size = ty.abiSize(target), .alignment = ty.abiAlignment(target) });
275 }539 }
276};540};
277541
...@@ -332,12 +596,13 @@ pub fn generate(...@@ -332,12 +596,13 @@ pub fn generate(
332 .fn_type = fn_type,596 .fn_type = fn_type,
333 .arg_index = 0,597 .arg_index = 0,
334 .src_loc = src_loc,598 .src_loc = src_loc,
335 .stack_align = undefined,
336 .end_di_line = module_fn.rbrace_line,599 .end_di_line = module_fn.rbrace_line,
337 .end_di_column = module_fn.rbrace_column,600 .end_di_column = module_fn.rbrace_column,
338 };601 };
339 defer {602 defer {
340 function.stack.deinit(gpa);603 function.frame_allocs.deinit(gpa);
604 function.free_frame_indices.deinit(gpa);
605 function.frame_locs.deinit(gpa);
341 var block_it = function.blocks.valueIterator();606 var block_it = function.blocks.valueIterator();
342 while (block_it.next()) |block| block.deinit(gpa);607 while (block_it.next()) |block| block.deinit(gpa);
343 function.blocks.deinit(gpa);608 function.blocks.deinit(gpa);
...@@ -349,7 +614,23 @@ pub fn generate(...@@ -349,7 +614,23 @@ pub fn generate(
349 if (builtin.mode == .Debug) function.mir_to_air_map.deinit(gpa);614 if (builtin.mode == .Debug) function.mir_to_air_map.deinit(gpa);
350 }615 }
351616
352 var call_info = function.resolveCallingConventionValues(fn_type, &.{}) catch |err| switch (err) {617 try function.frame_allocs.resize(gpa, FrameIndex.named_count);
618 function.frame_allocs.set(
619 @enumToInt(FrameIndex.stack_frame),
620 FrameAlloc.init(.{
621 .size = 0,
622 .alignment = if (mod.align_stack_fns.get(module_fn)) |set_align_stack|
623 set_align_stack.alignment
624 else
625 1,
626 }),
627 );
628 function.frame_allocs.set(
629 @enumToInt(FrameIndex.call_frame),
630 FrameAlloc.init(.{ .size = 0, .alignment = 1 }),
631 );
632
633 var call_info = function.resolveCallingConventionValues(fn_type, &.{}, .args_frame) catch |err| switch (err) {
353 error.CodegenFail => return Result{ .fail = function.err_msg.? },634 error.CodegenFail => return Result{ .fail = function.err_msg.? },
354 error.OutOfRegisters => return Result{635 error.OutOfRegisters => return Result{
355 .fail = try ErrorMsg.create(636 .fail = try ErrorMsg.create(
...@@ -365,8 +646,18 @@ pub fn generate(...@@ -365,8 +646,18 @@ pub fn generate(
365646
366 function.args = call_info.args;647 function.args = call_info.args;
367 function.ret_mcv = call_info.return_value;648 function.ret_mcv = call_info.return_value;
368 function.stack_align = call_info.stack_align;649 function.frame_allocs.set(@enumToInt(FrameIndex.ret_addr), FrameAlloc.init(.{
369 function.max_end_stack = call_info.stack_byte_count;650 .size = Type.usize.abiSize(function.target.*),
651 .alignment = @min(Type.usize.abiAlignment(function.target.*), call_info.stack_align),
652 }));
653 function.frame_allocs.set(@enumToInt(FrameIndex.base_ptr), FrameAlloc.init(.{
654 .size = Type.usize.abiSize(function.target.*),
655 .alignment = @min(Type.usize.abiAlignment(function.target.*) * 2, call_info.stack_align),
656 }));
657 function.frame_allocs.set(
658 @enumToInt(FrameIndex.args_frame),
659 FrameAlloc.init(.{ .size = call_info.stack_byte_count, .alignment = call_info.stack_align }),
660 );
370661
371 function.gen() catch |err| switch (err) {662 function.gen() catch |err| switch (err) {
372 error.CodegenFail => return Result{ .fail = function.err_msg.? },663 error.CodegenFail => return Result{ .fail = function.err_msg.? },
...@@ -379,6 +670,7 @@ pub fn generate(...@@ -379,6 +670,7 @@ pub fn generate(
379 var mir = Mir{670 var mir = Mir{
380 .instructions = function.mir_instructions.toOwnedSlice(),671 .instructions = function.mir_instructions.toOwnedSlice(),
381 .extra = try function.mir_extra.toOwnedSlice(bin_file.allocator),672 .extra = try function.mir_extra.toOwnedSlice(bin_file.allocator),
673 .frame_locs = function.frame_locs.toOwnedSlice(),
382 };674 };
383 defer mir.deinit(bin_file.allocator);675 defer mir.deinit(bin_file.allocator);
384676
...@@ -432,6 +724,7 @@ fn dumpWipMir(self: *Self, inst: Mir.Inst) !void {...@@ -432,6 +724,7 @@ fn dumpWipMir(self: *Self, inst: Mir.Inst) !void {
432 .mir = .{724 .mir = .{
433 .instructions = self.mir_instructions.slice(),725 .instructions = self.mir_instructions.slice(),
434 .extra = self.mir_extra.items,726 .extra = self.mir_extra.items,
727 .frame_locs = (std.MultiArrayList(Mir.FrameLoc){}).slice(),
435 },728 },
436 .target = self.target,729 .target = self.target,
437 .src_loc = self.src_loc,730 .src_loc = self.src_loc,
...@@ -458,6 +751,14 @@ fn dumpWipMir(self: *Self, inst: Mir.Inst) !void {...@@ -458,6 +751,14 @@ fn dumpWipMir(self: *Self, inst: Mir.Inst) !void {
458 }751 }
459}752}
460753
754fn dumpTracking(self: *Self) !void {
755 if (!debug_tracking) return;
756 const stderr = std.io.getStdErr().writer();
757
758 var it = self.inst_tracking.iterator();
759 while (it.next()) |entry| try stderr.print("%{d} = {}\n", .{ entry.key_ptr.*, entry.value_ptr.* });
760}
761
461fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {762fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {
462 const gpa = self.gpa;763 const gpa = self.gpa;
463 try self.mir_instructions.ensureUnusedCapacity(gpa, 1);764 try self.mir_instructions.ensureUnusedCapacity(gpa, 1);
...@@ -550,6 +851,14 @@ fn asmJccReloc(self: *Self, target: Mir.Inst.Index, cc: bits.Condition) !Mir.Ins...@@ -550,6 +851,14 @@ fn asmJccReloc(self: *Self, target: Mir.Inst.Index, cc: bits.Condition) !Mir.Ins
550 });851 });
551}852}
552853
854fn asmPlaceholder(self: *Self) !Mir.Inst.Index {
855 return self.addInst(.{
856 .tag = .dead,
857 .ops = undefined,
858 .data = undefined,
859 });
860}
861
553fn asmOpOnly(self: *Self, tag: Mir.Inst.Tag) !void {862fn asmOpOnly(self: *Self, tag: Mir.Inst.Tag) !void {
554 _ = try self.addInst(.{863 _ = try self.addInst(.{
555 .tag = tag,864 .tag = tag,
...@@ -765,44 +1074,32 @@ fn gen(self: *Self) InnerError!void {...@@ -765,44 +1074,32 @@ fn gen(self: *Self) InnerError!void {
765 const cc = self.fn_type.fnCallingConvention();1074 const cc = self.fn_type.fnCallingConvention();
766 if (cc != .Naked) {1075 if (cc != .Naked) {
767 try self.asmRegister(.push, .rbp);1076 try self.asmRegister(.push, .rbp);
1077 const backpatch_push_callee_preserved_regs = try self.asmPlaceholder();
768 try self.asmRegisterRegister(.mov, .rbp, .rsp);1078 try self.asmRegisterRegister(.mov, .rbp, .rsp);
7691079 const backpatch_frame_align = try self.asmPlaceholder();
770 // We want to subtract the aligned stack frame size from rsp here, but we don't1080 const backpatch_stack_alloc = try self.asmPlaceholder();
771 // yet know how big it will be, so we leave room for a 4-byte stack size.1081
772 // TODO During semantic analysis, check if there are no function calls. If there1082 switch (self.ret_mcv.long) {
773 // are none, here we can omit the part where we subtract and then add rsp.1083 .none, .unreach => {},
774 const backpatch_stack_sub = try self.addInst(.{1084 .indirect => {
775 .tag = .dead,1085 // The address where to store the return value for the caller is in a
776 .ops = undefined,1086 // register which the callee is free to clobber. Therefore, we purposely
777 .data = undefined,1087 // spill it to stack immediately.
778 });1088 const frame_index =
7791089 try self.allocFrameIndex(FrameAlloc.initType(Type.usize, self.target.*));
780 if (self.ret_mcv == .stack_offset) {1090 try self.genSetMem(
781 // The address where to store the return value for the caller is in a1091 .{ .frame = frame_index },
782 // register which the callee is free to clobber. Therefore, we purposely1092 0,
783 // spill it to stack immediately.1093 Type.usize,
784 const stack_offset = mem.alignForwardGeneric(u32, self.next_stack_offset + 8, 8);1094 self.ret_mcv.long.address().offset(-self.ret_mcv.short.indirect.off),
785 self.next_stack_offset = stack_offset;1095 );
786 self.max_end_stack = @max(self.max_end_stack, self.next_stack_offset);1096 self.ret_mcv.long = .{ .load_frame = .{ .index = frame_index } };
7871097 tracking_log.debug("spill {} to {}", .{ self.ret_mcv.long, frame_index });
788 const ret_reg = abi.getCAbiIntParamRegs(self.target.*)[0];1098 },
789 try self.genSetStack(Type.usize, @intCast(i32, stack_offset), MCValue{ .register = ret_reg }, .{});1099 else => unreachable,
790 self.ret_mcv = MCValue{ .stack_offset = @intCast(i32, stack_offset) };
791 log.debug("gen: spilling {s} to stack at offset {}", .{ @tagName(ret_reg), stack_offset });
792 }1100 }
7931101
794 _ = try self.addInst(.{1102 try self.asmOpOnly(.dbg_prologue_end);
795 .tag = .dbg_prologue_end,
796 .ops = undefined,
797 .data = undefined,
798 });
799
800 // Push callee-preserved regs that were used actually in use.
801 const backpatch_push_callee_preserved_regs = try self.addInst(.{
802 .tag = .dead,
803 .ops = undefined,
804 .data = undefined,
805 });
8061103
807 try self.genBody(self.air.getMainBody());1104 try self.genBody(self.air.getMainBody());
8081105
...@@ -814,69 +1111,43 @@ fn gen(self: *Self) InnerError!void {...@@ -814,69 +1111,43 @@ fn gen(self: *Self) InnerError!void {
814 // }1111 // }
815 // Eliding the reloc will cause a miscompilation in this case.1112 // Eliding the reloc will cause a miscompilation in this case.
816 for (self.exitlude_jump_relocs.items) |jmp_reloc| {1113 for (self.exitlude_jump_relocs.items) |jmp_reloc| {
817 self.mir_instructions.items(.data)[jmp_reloc].inst = @intCast(u32, self.mir_instructions.len);1114 self.mir_instructions.items(.data)[jmp_reloc].inst =
1115 @intCast(u32, self.mir_instructions.len);
818 }1116 }
8191117
820 // Create list of registers to save in the prologue.1118 try self.asmOpOnly(.dbg_epilogue_begin);
821 // TODO handle register classes1119 const backpatch_stack_dealloc = try self.asmPlaceholder();
822 var reg_list = Mir.RegisterList{};1120 const backpatch_pop_callee_preserved_regs = try self.asmPlaceholder();
823 const callee_preserved_regs = abi.getCalleePreservedRegs(self.target.*);
824 for (callee_preserved_regs) |reg| {
825 if (self.register_manager.isRegAllocated(reg)) {
826 reg_list.push(callee_preserved_regs, reg);
827 }
828 }
829 const saved_regs_stack_space: u32 = reg_list.count() * 8;
830
831 // Pop saved callee-preserved regs.
832 const backpatch_pop_callee_preserved_regs = try self.addInst(.{
833 .tag = .dead,
834 .ops = undefined,
835 .data = undefined,
836 });
837
838 _ = try self.addInst(.{
839 .tag = .dbg_epilogue_begin,
840 .ops = undefined,
841 .data = undefined,
842 });
843
844 // Maybe add rsp, x if required. This is backpatched later.
845 const backpatch_stack_add = try self.addInst(.{
846 .tag = .dead,
847 .ops = undefined,
848 .data = undefined,
849 });
850
851 try self.asmRegister(.pop, .rbp);1121 try self.asmRegister(.pop, .rbp);
852 try self.asmOpOnly(.ret);1122 try self.asmOpOnly(.ret);
8531123
854 // Adjust the stack1124 const frame_layout = try self.computeFrameLayout();
855 if (self.max_end_stack > math.maxInt(i32)) {1125 const need_frame_align = frame_layout.stack_mask != math.maxInt(u32);
856 return self.failSymbol("too much stack used in call parameters", .{});1126 const need_stack_adjust = frame_layout.stack_adjust > 0;
1127 const need_save_reg = frame_layout.save_reg_list.count() > 0;
1128 if (need_frame_align) {
1129 self.mir_instructions.set(backpatch_frame_align, .{
1130 .tag = .@"and",
1131 .ops = .ri_s,
1132 .data = .{ .ri = .{ .r = .rsp, .i = frame_layout.stack_mask } },
1133 });
857 }1134 }
8581135 if (need_stack_adjust) {
859 const aligned_stack_end = @intCast(1136 self.mir_instructions.set(backpatch_stack_alloc, .{
860 u32,
861 mem.alignForward(self.max_end_stack + saved_regs_stack_space, self.stack_align),
862 );
863 if (aligned_stack_end > 0) {
864 self.mir_instructions.set(backpatch_stack_sub, .{
865 .tag = .sub,1137 .tag = .sub,
866 .ops = .ri_u,1138 .ops = .ri_s,
867 .data = .{ .ri = .{ .r = .rsp, .i = aligned_stack_end } },1139 .data = .{ .ri = .{ .r = .rsp, .i = frame_layout.stack_adjust } },
868 });1140 });
869 self.mir_instructions.set(backpatch_stack_add, .{1141 }
870 .tag = .add,1142 if (need_frame_align or need_stack_adjust) {
871 .ops = .ri_u,1143 self.mir_instructions.set(backpatch_stack_dealloc, .{
872 .data = .{ .ri = .{ .r = .rsp, .i = aligned_stack_end } },1144 .tag = .mov,
873 });1145 .ops = .rr,
8741146 .data = .{ .rr = .{ .r1 = .rsp, .r2 = .rbp } },
875 const save_reg_list = try self.addExtra(Mir.SaveRegisterList{
876 .base_reg = @enumToInt(Register.rbp),
877 .register_list = reg_list.asInt(),
878 .stack_end = aligned_stack_end,
879 });1147 });
1148 }
1149 if (need_save_reg) {
1150 const save_reg_list = frame_layout.save_reg_list.asInt();
880 self.mir_instructions.set(backpatch_push_callee_preserved_regs, .{1151 self.mir_instructions.set(backpatch_push_callee_preserved_regs, .{
881 .tag = .push_regs,1152 .tag = .push_regs,
882 .ops = undefined,1153 .ops = undefined,
...@@ -889,30 +1160,19 @@ fn gen(self: *Self) InnerError!void {...@@ -889,30 +1160,19 @@ fn gen(self: *Self) InnerError!void {
889 });1160 });
890 }1161 }
891 } else {1162 } else {
892 _ = try self.addInst(.{1163 try self.asmOpOnly(.dbg_prologue_end);
893 .tag = .dbg_prologue_end,
894 .ops = undefined,
895 .data = undefined,
896 });
897
898 try self.genBody(self.air.getMainBody());1164 try self.genBody(self.air.getMainBody());
8991165 try self.asmOpOnly(.dbg_epilogue_begin);
900 _ = try self.addInst(.{
901 .tag = .dbg_epilogue_begin,
902 .ops = undefined,
903 .data = undefined,
904 });
905 }1166 }
9061167
907 // Drop them off at the rbrace.1168 // Drop them off at the rbrace.
908 const payload = try self.addExtra(Mir.DbgLineColumn{
909 .line = self.end_di_line,
910 .column = self.end_di_column,
911 });
912 _ = try self.addInst(.{1169 _ = try self.addInst(.{
913 .tag = .dbg_line,1170 .tag = .dbg_line,
914 .ops = undefined,1171 .ops = undefined,
915 .data = .{ .payload = payload },1172 .data = .{ .line_column = .{
1173 .line = self.end_di_line,
1174 .column = self.end_di_column,
1175 } },
916 });1176 });
917}1177}
9181178
...@@ -932,6 +1192,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -932,6 +1192,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
932 self.air,1192 self.air,
933 self.liveness,1193 self.liveness,
934 );1194 );
1195 self.dumpTracking() catch {};
9351196
936 const old_air_bookkeeping = self.air_bookkeeping;1197 const old_air_bookkeeping = self.air_bookkeeping;
937 try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1);1198 try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1);
...@@ -1187,23 +1448,16 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -1187,23 +1448,16 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
1187 while (it.next()) |index| {1448 while (it.next()) |index| {
1188 const tracked_inst = self.register_manager.registers[index];1449 const tracked_inst = self.register_manager.registers[index];
1189 const tracking = self.getResolvedInstValue(tracked_inst);1450 const tracking = self.getResolvedInstValue(tracked_inst);
1190 assert(RegisterManager.indexOfRegIntoTracked(switch (tracking.short) {1451 assert(RegisterManager.indexOfRegIntoTracked(tracking.getReg().?).? == index);
1191 .register => |reg| reg,
1192 .register_overflow => |ro| ro.reg,
1193 else => unreachable,
1194 }).? == index);
1195 }1452 }
1196 }1453 }
1197 }1454 }
1198 }1455 }
1456 self.dumpTracking() catch {};
1199}1457}
12001458
1201fn getValue(self: *Self, value: MCValue, inst: ?Air.Inst.Index) void {1459fn getValue(self: *Self, value: MCValue, inst: ?Air.Inst.Index) void {
1202 const reg = switch (value) {1460 const reg = value.getReg() orelse return;
1203 .register => |reg| reg,
1204 .register_overflow => |ro| ro.reg,
1205 else => return,
1206 };
1207 if (self.register_manager.isRegFree(reg)) {1461 if (self.register_manager.isRegFree(reg)) {
1208 self.register_manager.getRegAssumeFree(reg, inst);1462 self.register_manager.getRegAssumeFree(reg, inst);
1209 }1463 }
...@@ -1214,8 +1468,11 @@ fn freeValue(self: *Self, value: MCValue) void {...@@ -1214,8 +1468,11 @@ fn freeValue(self: *Self, value: MCValue) void {
1214 .register => |reg| {1468 .register => |reg| {
1215 self.register_manager.freeReg(reg);1469 self.register_manager.freeReg(reg);
1216 },1470 },
1217 .register_overflow => |ro| {1471 .register_offset => |reg_off| {
1218 self.register_manager.freeReg(ro.reg);1472 self.register_manager.freeReg(reg_off.reg);
1473 },
1474 .register_overflow => |reg_ov| {
1475 self.register_manager.freeReg(reg_ov.reg);
1219 self.eflags_inst = null;1476 self.eflags_inst = null;
1220 },1477 },
1221 .eflags => {1478 .eflags => {
...@@ -1231,10 +1488,10 @@ fn feed(self: *Self, bt: *Liveness.BigTomb, operand: Air.Inst.Ref) void {...@@ -1231,10 +1488,10 @@ fn feed(self: *Self, bt: *Liveness.BigTomb, operand: Air.Inst.Ref) void {
12311488
1232/// Asserts there is already capacity to insert into top branch inst_table.1489/// Asserts there is already capacity to insert into top branch inst_table.
1233fn processDeath(self: *Self, inst: Air.Inst.Index) void {1490fn processDeath(self: *Self, inst: Air.Inst.Index) void {
1234 const air_tags = self.air.instructions.items(.tag);1491 switch (self.air.instructions.items(.tag)[inst]) {
1235 if (air_tags[inst] == .constant) return;1492 .constant, .const_ty => unreachable,
1236 log.debug("%{d} => {}", .{ inst, MCValue.dead });1493 else => self.inst_tracking.getPtr(inst).?.die(self, inst),
1237 self.inst_tracking.getPtr(inst).?.die(self);1494 }
1238}1495}
12391496
1240/// Called when there are no operands, and the instruction is always unreferenced.1497/// Called when there are no operands, and the instruction is always unreferenced.
...@@ -1249,7 +1506,7 @@ fn finishAirResult(self: *Self, inst: Air.Inst.Index, result: MCValue) void {...@@ -1249,7 +1506,7 @@ fn finishAirResult(self: *Self, inst: Air.Inst.Index, result: MCValue) void {
1249 .none, .dead, .unreach => {},1506 .none, .dead, .unreach => {},
1250 else => unreachable, // Why didn't the result die?1507 else => unreachable, // Why didn't the result die?
1251 } else {1508 } else {
1252 log.debug("%{d} => {}", .{ inst, result });1509 tracking_log.debug("%{d} => {} (birth)", .{ inst, result });
1253 self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(result));1510 self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(result));
1254 // In some cases, an operand may be reused as the result.1511 // In some cases, an operand may be reused as the result.
1255 // If that operand died and was a register, it was freed by1512 // If that operand died and was a register, it was freed by
...@@ -1273,36 +1530,123 @@ fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Live...@@ -1273,36 +1530,123 @@ fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Live
1273 self.finishAirResult(inst, result);1530 self.finishAirResult(inst, result);
1274}1531}
12751532
1276fn allocMem(self: *Self, inst: ?Air.Inst.Index, abi_size: u32, abi_align: u32) !u32 {1533const FrameLayout = struct {
1277 if (abi_align > self.stack_align)1534 stack_mask: u32,
1278 self.stack_align = abi_align;1535 stack_adjust: u32,
1279 // TODO find a free slot instead of always appending1536 save_reg_list: Mir.RegisterList,
1280 const offset = mem.alignForwardGeneric(u32, self.next_stack_offset + abi_size, abi_align);1537};
1281 self.next_stack_offset = offset;1538
1282 self.max_end_stack = @max(self.max_end_stack, self.next_stack_offset);1539fn setFrameLoc(
1283 try self.stack.putNoClobber(self.gpa, offset, .{1540 self: *Self,
1284 .inst = inst,1541 frame_index: FrameIndex,
1285 .size = abi_size,1542 base: Register,
1286 });1543 offset: *i32,
1287 return offset;1544 comptime aligned: bool,
1545) void {
1546 const frame_i = @enumToInt(frame_index);
1547 if (aligned) {
1548 const alignment = @as(i32, 1) << self.frame_allocs.items(.abi_align)[frame_i];
1549 offset.* = mem.alignForwardGeneric(i32, offset.*, alignment);
1550 }
1551 self.frame_locs.set(frame_i, .{ .base = base, .disp = offset.* });
1552 offset.* += self.frame_allocs.items(.abi_size)[frame_i];
1553}
1554
1555fn computeFrameLayout(self: *Self) !FrameLayout {
1556 const frame_allocs_len = self.frame_allocs.len;
1557 try self.frame_locs.resize(self.gpa, frame_allocs_len);
1558 const stack_frame_order = try self.gpa.alloc(FrameIndex, frame_allocs_len - FrameIndex.named_count);
1559 defer self.gpa.free(stack_frame_order);
1560
1561 const frame_size = self.frame_allocs.items(.abi_size);
1562 const frame_align = self.frame_allocs.items(.abi_align);
1563 const frame_offset = self.frame_locs.items(.disp);
1564
1565 for (stack_frame_order, FrameIndex.named_count..) |*frame_order, frame_index|
1566 frame_order.* = @intToEnum(FrameIndex, frame_index);
1567 {
1568 const SortContext = struct {
1569 frame_align: @TypeOf(frame_align),
1570 pub fn lessThan(context: @This(), lhs: FrameIndex, rhs: FrameIndex) bool {
1571 return context.frame_align[@enumToInt(lhs)] > context.frame_align[@enumToInt(rhs)];
1572 }
1573 };
1574 const sort_context = SortContext{ .frame_align = frame_align };
1575 std.sort.sort(FrameIndex, stack_frame_order, sort_context, SortContext.lessThan);
1576 }
1577
1578 const call_frame_align = frame_align[@enumToInt(FrameIndex.call_frame)];
1579 const stack_frame_align = frame_align[@enumToInt(FrameIndex.stack_frame)];
1580 const args_frame_align = frame_align[@enumToInt(FrameIndex.args_frame)];
1581 const needed_align = @max(call_frame_align, stack_frame_align);
1582 const need_align_stack = needed_align > args_frame_align;
1583
1584 // Create list of registers to save in the prologue.
1585 // TODO handle register classes
1586 var save_reg_list = Mir.RegisterList{};
1587 const callee_preserved_regs = abi.getCalleePreservedRegs(self.target.*);
1588 for (callee_preserved_regs) |reg| {
1589 if (self.register_manager.isRegAllocated(reg)) {
1590 save_reg_list.push(callee_preserved_regs, reg);
1591 }
1592 }
1593
1594 var rbp_offset = @intCast(i32, save_reg_list.count() * 8);
1595 self.setFrameLoc(.base_ptr, .rbp, &rbp_offset, false);
1596 self.setFrameLoc(.ret_addr, .rbp, &rbp_offset, false);
1597 self.setFrameLoc(.args_frame, .rbp, &rbp_offset, false);
1598 const stack_frame_align_offset =
1599 if (need_align_stack) 0 else frame_offset[@enumToInt(FrameIndex.args_frame)];
1600
1601 var rsp_offset: i32 = 0;
1602 self.setFrameLoc(.call_frame, .rsp, &rsp_offset, true);
1603 self.setFrameLoc(.stack_frame, .rsp, &rsp_offset, true);
1604 for (stack_frame_order) |frame_index| self.setFrameLoc(frame_index, .rsp, &rsp_offset, true);
1605 rsp_offset += stack_frame_align_offset;
1606 rsp_offset = mem.alignForwardGeneric(i32, rsp_offset, @as(i32, 1) << needed_align);
1607 rsp_offset -= stack_frame_align_offset;
1608 frame_size[@enumToInt(FrameIndex.call_frame)] =
1609 @intCast(u31, rsp_offset - frame_offset[@enumToInt(FrameIndex.stack_frame)]);
1610
1611 return .{
1612 .stack_mask = @as(u32, math.maxInt(u32)) << (if (need_align_stack) needed_align else 0),
1613 .stack_adjust = @intCast(u32, rsp_offset - frame_offset[@enumToInt(FrameIndex.call_frame)]),
1614 .save_reg_list = save_reg_list,
1615 };
1288}1616}
12891617
1290/// Use a pointer instruction as the basis for allocating stack memory.1618fn allocFrameIndex(self: *Self, alloc: FrameAlloc) !FrameIndex {
1291fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !u32 {1619 const frame_allocs_slice = self.frame_allocs.slice();
1292 const ptr_ty = self.air.typeOfIndex(inst);1620 const frame_size = frame_allocs_slice.items(.abi_size);
1293 const elem_ty = ptr_ty.elemType();1621 const frame_align = frame_allocs_slice.items(.abi_align);
1622 for (self.free_frame_indices.keys(), 0..) |frame_index, free_i| {
1623 const abi_size = frame_size[@enumToInt(frame_index)];
1624 if (abi_size != alloc.abi_size) continue;
1625 const abi_align = &frame_align[@enumToInt(frame_index)];
1626 abi_align.* = @max(abi_align.*, alloc.abi_align);
12941627
1295 if (!elem_ty.hasRuntimeBitsIgnoreComptime()) {1628 const stack_frame_align = &frame_align[@enumToInt(FrameIndex.stack_frame)];
1296 return self.allocMem(inst, @sizeOf(usize), @alignOf(usize));1629 stack_frame_align.* = @max(stack_frame_align.*, alloc.abi_align);
1630
1631 _ = self.free_frame_indices.swapRemoveAt(free_i);
1632 return frame_index;
1297 }1633 }
1634 const frame_index = @intToEnum(FrameIndex, self.frame_allocs.len);
1635 try self.frame_allocs.append(self.gpa, alloc);
1636 return frame_index;
1637}
12981638
1299 const abi_size = math.cast(u32, elem_ty.abiSize(self.target.*)) orelse {1639/// Use a pointer instruction as the basis for allocating stack memory.
1300 const mod = self.bin_file.options.module.?;1640fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !FrameIndex {
1301 return self.fail("type '{}' too big to fit into stack frame", .{elem_ty.fmt(mod)});1641 const ptr_ty = self.air.typeOfIndex(inst);
1302 };1642 const val_ty = ptr_ty.childType();
1303 // TODO swap this for inst.ty.ptrAlign1643 return self.allocFrameIndex(FrameAlloc.init(.{
1304 const abi_align = ptr_ty.ptrAlignment(self.target.*);1644 .size = math.cast(u32, val_ty.abiSize(self.target.*)) orelse {
1305 return self.allocMem(inst, abi_size, abi_align);1645 const mod = self.bin_file.options.module.?;
1646 return self.fail("type '{}' too big to fit into stack frame", .{val_ty.fmt(mod)});
1647 },
1648 .alignment = @max(ptr_ty.ptrAlignment(self.target.*), 1),
1649 }));
1306}1650}
13071651
1308fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {1652fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
...@@ -1320,42 +1664,35 @@ fn allocRegOrMemAdvanced(self: *Self, elem_ty: Type, inst: ?Air.Inst.Index, reg_...@@ -1320,42 +1664,35 @@ fn allocRegOrMemAdvanced(self: *Self, elem_ty: Type, inst: ?Air.Inst.Index, reg_
1320 };1664 };
13211665
1322 if (reg_ok) {1666 if (reg_ok) {
1323 switch (elem_ty.zigTypeTag()) {1667 // Make sure the type can fit in a register before we try to allocate one.
1324 .Vector => return self.fail("TODO allocRegOrMem for Vector type", .{}),1668 const ptr_bits = self.target.cpu.arch.ptrBitWidth();
1325 .Float => {1669 const ptr_bytes: u64 = @divExact(ptr_bits, 8);
1326 if (intrinsicsAllowed(self.target.*, elem_ty)) {1670 if (abi_size <= ptr_bytes) {
1327 const ptr_bytes: u64 = 32;1671 if (self.register_manager.tryAllocReg(inst, try self.regClassForType(elem_ty))) |reg| {
1328 if (abi_size <= ptr_bytes) {1672 return MCValue{ .register = registerAlias(reg, abi_size) };
1329 if (self.register_manager.tryAllocReg(inst, sse)) |reg| {1673 }
1330 return MCValue{ .register = registerAlias(reg, abi_size) };
1331 }
1332 }
1333 }
1334
1335 return self.fail("TODO allocRegOrMem for Float type without SSE/AVX support", .{});
1336 },
1337 else => {
1338 // Make sure the type can fit in a register before we try to allocate one.
1339 const ptr_bits = self.target.cpu.arch.ptrBitWidth();
1340 const ptr_bytes: u64 = @divExact(ptr_bits, 8);
1341 if (abi_size <= ptr_bytes) {
1342 if (self.register_manager.tryAllocReg(inst, gp)) |reg| {
1343 return MCValue{ .register = registerAlias(reg, abi_size) };
1344 }
1345 }
1346 },
1347 }1674 }
1348 }1675 }
13491676
1350 const abi_align = elem_ty.abiAlignment(self.target.*);1677 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(elem_ty, self.target.*));
1351 if (abi_align > self.stack_align)1678 return .{ .load_frame = .{ .index = frame_index } };
1352 self.stack_align = abi_align;1679}
1353 const stack_offset = try self.allocMem(inst, abi_size, abi_align);1680
1354 return MCValue{ .stack_offset = @intCast(i32, stack_offset) };1681fn regClassForType(self: *Self, ty: Type) !RegisterManager.RegisterBitSet {
1682 return switch (ty.zigTypeTag()) {
1683 .Vector => self.fail("TODO regClassForType for {}", .{ty.fmt(self.bin_file.options.module.?)}),
1684 .Float => switch (ty.floatBits(self.target.*)) {
1685 32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse)) sse else gp,
1686 64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2)) sse else gp,
1687 else => gp,
1688 },
1689 else => gp,
1690 };
1355}1691}
13561692
1357const State = struct {1693const State = struct {
1358 registers: RegisterManager.TrackedRegisters,1694 registers: RegisterManager.TrackedRegisters,
1695 reg_tracking: [RegisterManager.RegisterBitSet.bit_length]InstTracking,
1359 free_registers: RegisterManager.RegisterBitSet,1696 free_registers: RegisterManager.RegisterBitSet,
1360 inst_tracking_len: u32,1697 inst_tracking_len: u32,
1361 scope_generation: u32,1698 scope_generation: u32,
...@@ -1370,8 +1707,14 @@ fn initRetroactiveState(self: *Self) State {...@@ -1370,8 +1707,14 @@ fn initRetroactiveState(self: *Self) State {
13701707
1371fn saveRetroactiveState(self: *Self, state: *State) !void {1708fn saveRetroactiveState(self: *Self, state: *State) !void {
1372 try self.spillEflagsIfOccupied();1709 try self.spillEflagsIfOccupied();
1373 state.registers = self.register_manager.registers;1710 const free_registers = self.register_manager.free_registers;
1374 state.free_registers = self.register_manager.free_registers;1711 var it = free_registers.iterator(.{ .kind = .unset });
1712 while (it.next()) |index| {
1713 const tracked_inst = self.register_manager.registers[index];
1714 state.registers[index] = tracked_inst;
1715 state.reg_tracking[index] = self.inst_tracking.get(tracked_inst).?;
1716 }
1717 state.free_registers = free_registers;
1375}1718}
13761719
1377fn saveState(self: *Self) !State {1720fn saveState(self: *Self) !State {
...@@ -1387,18 +1730,18 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt...@@ -1387,18 +1730,18 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
1387 close_scope: bool,1730 close_scope: bool,
1388}) !void {1731}) !void {
1389 if (opts.close_scope) {1732 if (opts.close_scope) {
1390 for (self.inst_tracking.values()[state.inst_tracking_len..]) |*tracking| tracking.die(self);1733 for (
1734 self.inst_tracking.keys()[state.inst_tracking_len..],
1735 self.inst_tracking.values()[state.inst_tracking_len..],
1736 ) |inst, *tracking| tracking.die(self, inst);
1391 self.inst_tracking.shrinkRetainingCapacity(state.inst_tracking_len);1737 self.inst_tracking.shrinkRetainingCapacity(state.inst_tracking_len);
1392 }1738 }
13931739
1394 if (opts.resurrect) for (self.inst_tracking.values()[0..state.inst_tracking_len]) |*tracking|1740 if (opts.resurrect) for (
1395 tracking.resurrect(state.scope_generation);1741 self.inst_tracking.keys()[0..state.inst_tracking_len],
1396 const air_tags = self.air.instructions.items(.tag);1742 self.inst_tracking.values()[0..state.inst_tracking_len],
1397 for (deaths) |death| switch (air_tags[death]) {1743 ) |inst, *tracking| tracking.resurrect(inst, state.scope_generation);
1398 .constant => {},1744 for (deaths) |death| self.processDeath(death);
1399 .const_ty => unreachable,
1400 else => self.inst_tracking.getPtr(death).?.die(self),
1401 };
14021745
1403 for (0..state.registers.len) |index| {1746 for (0..state.registers.len) |index| {
1404 const current_maybe_inst = if (self.register_manager.free_registers.isSet(index))1747 const current_maybe_inst = if (self.register_manager.free_registers.isSet(index))
...@@ -1411,26 +1754,34 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt...@@ -1411,26 +1754,34 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
1411 state.registers[index];1754 state.registers[index];
1412 if (std.debug.runtime_safety) if (target_maybe_inst) |target_inst|1755 if (std.debug.runtime_safety) if (target_maybe_inst) |target_inst|
1413 assert(self.inst_tracking.getIndex(target_inst).? < state.inst_tracking_len);1756 assert(self.inst_tracking.getIndex(target_inst).? < state.inst_tracking_len);
1414 if (current_maybe_inst == target_maybe_inst) continue;
1415 const reg = RegisterManager.regAtTrackedIndex(
1416 @intCast(RegisterManager.RegisterBitSet.ShiftInt, index),
1417 );
1418 if (opts.emit_instructions) {1757 if (opts.emit_instructions) {
1419 if (current_maybe_inst) |current_inst| {1758 if (current_maybe_inst) |current_inst| {
1420 try self.inst_tracking.getPtr(current_inst).?.spill(self, current_inst);1759 try self.inst_tracking.getPtr(current_inst).?.spill(self, current_inst);
1421 }1760 }
1422 if (target_maybe_inst) |target_inst| {1761 if (target_maybe_inst) |target_inst| {
1423 try self.inst_tracking.getPtr(target_inst).?.materialize(self, target_inst, reg);1762 try self.inst_tracking.getPtr(target_inst).?.materialize(
1763 self,
1764 target_inst,
1765 state.reg_tracking[index],
1766 );
1424 }1767 }
1425 }1768 }
1426 if (opts.update_tracking) {1769 if (opts.update_tracking) {
1427 if (current_maybe_inst) |current_inst| {1770 if (current_maybe_inst) |current_inst| {
1428 self.inst_tracking.getPtr(current_inst).?.trackSpill(self);1771 self.inst_tracking.getPtr(current_inst).?.trackSpill(self, current_inst);
1772 }
1773 {
1774 const reg = RegisterManager.regAtTrackedIndex(
1775 @intCast(RegisterManager.RegisterBitSet.ShiftInt, index),
1776 );
1777 self.register_manager.freeReg(reg);
1778 self.register_manager.getRegAssumeFree(reg, target_maybe_inst);
1429 }1779 }
1430 self.register_manager.freeReg(reg);
1431 self.register_manager.getRegAssumeFree(reg, target_maybe_inst);
1432 if (target_maybe_inst) |target_inst| {1780 if (target_maybe_inst) |target_inst| {
1433 self.inst_tracking.getPtr(target_inst).?.trackMaterialize(self, target_inst, reg);1781 self.inst_tracking.getPtr(target_inst).?.trackMaterialize(
1782 target_inst,
1783 state.reg_tracking[index],
1784 );
1434 }1785 }
1435 }1786 }
1436 }1787 }
...@@ -1438,7 +1789,7 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt...@@ -1438,7 +1789,7 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
1438 try self.inst_tracking.getPtr(inst).?.spill(self, inst);1789 try self.inst_tracking.getPtr(inst).?.spill(self, inst);
1439 if (opts.update_tracking) if (self.eflags_inst) |inst| {1790 if (opts.update_tracking) if (self.eflags_inst) |inst| {
1440 self.eflags_inst = null;1791 self.eflags_inst = null;
1441 self.inst_tracking.getPtr(inst).?.trackSpill(self);1792 self.inst_tracking.getPtr(inst).?.trackSpill(self, inst);
1442 };1793 };
14431794
1444 if (opts.update_tracking and std.debug.runtime_safety) {1795 if (opts.update_tracking and std.debug.runtime_safety) {
...@@ -1452,9 +1803,9 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt...@@ -1452,9 +1803,9 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
14521803
1453pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {1804pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {
1454 const tracking = self.inst_tracking.getPtr(inst).?;1805 const tracking = self.inst_tracking.getPtr(inst).?;
1455 assert(tracking.getReg().?.to64() == reg.to64());1806 assert(tracking.getReg().?.id() == reg.id());
1456 try tracking.spill(self, inst);1807 try tracking.spill(self, inst);
1457 tracking.trackSpill(self);1808 tracking.trackSpill(self, inst);
1458}1809}
14591810
1460pub fn spillEflagsIfOccupied(self: *Self) !void {1811pub fn spillEflagsIfOccupied(self: *Self) !void {
...@@ -1463,7 +1814,7 @@ pub fn spillEflagsIfOccupied(self: *Self) !void {...@@ -1463,7 +1814,7 @@ pub fn spillEflagsIfOccupied(self: *Self) !void {
1463 const tracking = self.inst_tracking.getPtr(inst).?;1814 const tracking = self.inst_tracking.getPtr(inst).?;
1464 assert(tracking.getCondition() != null);1815 assert(tracking.getCondition() != null);
1465 try tracking.spill(self, inst);1816 try tracking.spill(self, inst);
1466 tracking.trackSpill(self);1817 tracking.trackSpill(self, inst);
1467 }1818 }
1468}1819}
14691820
...@@ -1477,15 +1828,8 @@ pub fn spillRegisters(self: *Self, registers: []const Register) !void {...@@ -1477,15 +1828,8 @@ pub fn spillRegisters(self: *Self, registers: []const Register) !void {
1477/// allocated. A second call to `copyToTmpRegister` may return the same register.1828/// allocated. A second call to `copyToTmpRegister` may return the same register.
1478/// This can have a side effect of spilling instructions to the stack to free up a register.1829/// This can have a side effect of spilling instructions to the stack to free up a register.
1479fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {1830fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
1480 const reg_class: RegisterManager.RegisterBitSet = switch (ty.zigTypeTag()) {1831 const reg: Register = try self.register_manager.allocReg(null, try self.regClassForType(ty));
1481 .Float => blk: {1832 try self.genSetReg(reg, ty, mcv);
1482 if (intrinsicsAllowed(self.target.*, ty)) break :blk sse;
1483 return self.fail("TODO copy {} to register", .{ty.fmtDebug()});
1484 },
1485 else => gp,
1486 };
1487 const reg: Register = try self.register_manager.allocReg(null, reg_class);
1488 try self.genSetReg(ty, reg, mcv);
1489 return reg;1833 return reg;
1490}1834}
14911835
...@@ -1493,28 +1837,35 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {...@@ -1493,28 +1837,35 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
1493/// `reg_owner` is the instruction that gets associated with the register in the register table.1837/// `reg_owner` is the instruction that gets associated with the register in the register table.
1494/// This can have a side effect of spilling instructions to the stack to free up a register.1838/// This can have a side effect of spilling instructions to the stack to free up a register.
1495/// WARNING make sure that the allocated register matches the returned MCValue from an instruction!1839/// WARNING make sure that the allocated register matches the returned MCValue from an instruction!
1496fn copyToRegisterWithInstTracking(self: *Self, reg_owner: Air.Inst.Index, ty: Type, mcv: MCValue) !MCValue {1840fn copyToRegisterWithInstTracking(
1497 const reg_class: RegisterManager.RegisterBitSet = switch (ty.zigTypeTag()) {1841 self: *Self,
1498 .Float => blk: {1842 reg_owner: Air.Inst.Index,
1499 if (intrinsicsAllowed(self.target.*, ty)) break :blk sse;1843 ty: Type,
1500 return self.fail("TODO copy {} to register", .{ty.fmtDebug()});1844 mcv: MCValue,
1501 },1845) !MCValue {
1502 else => gp,1846 const reg: Register = try self.register_manager.allocReg(reg_owner, try self.regClassForType(ty));
1503 };1847 try self.genSetReg(reg, ty, mcv);
1504 const reg: Register = try self.register_manager.allocReg(reg_owner, reg_class);
1505 try self.genSetReg(ty, reg, mcv);
1506 return MCValue{ .register = reg };1848 return MCValue{ .register = reg };
1507}1849}
15081850
1509fn airAlloc(self: *Self, inst: Air.Inst.Index) !void {1851fn airAlloc(self: *Self, inst: Air.Inst.Index) !void {
1510 const stack_offset = try self.allocMemPtr(inst);1852 const result = MCValue{ .lea_frame = .{ .index = try self.allocMemPtr(inst) } };
1511 const result = MCValue{ .ptr_stack_offset = @intCast(i32, stack_offset) };
1512 return self.finishAir(inst, result, .{ .none, .none, .none });1853 return self.finishAir(inst, result, .{ .none, .none, .none });
1513}1854}
15141855
1515fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {1856fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {
1516 const stack_offset = try self.allocMemPtr(inst);1857 const result: MCValue = switch (self.ret_mcv.long) {
1517 const result = MCValue{ .ptr_stack_offset = @intCast(i32, stack_offset) };1858 else => unreachable,
1859 .none => .{ .lea_frame = .{ .index = try self.allocMemPtr(inst) } },
1860 .load_frame => .{ .register_offset = .{
1861 .reg = (try self.copyToRegisterWithInstTracking(
1862 inst,
1863 self.air.typeOfIndex(inst),
1864 self.ret_mcv.long,
1865 )).register,
1866 .off = self.ret_mcv.short.indirect.off,
1867 } },
1868 };
1518 return self.finishAir(inst, result, .{ .none, .none, .none });1869 return self.finishAir(inst, result, .{ .none, .none, .none });
1519}1870}
15201871
...@@ -1577,12 +1928,14 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {...@@ -1577,12 +1928,14 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
1577 registerAlias(src_reg, min_abi_size),1928 registerAlias(src_reg, min_abi_size),
1578 );1929 );
1579 },1930 },
1580 .stack_offset => |src_off| {1931 .load_frame => |frame_addr| try self.asmRegisterMemory(
1581 try self.asmRegisterMemory(tag, dst_alias, Memory.sib(1932 tag,
1582 Memory.PtrSize.fromSize(min_abi_size),1933 dst_alias,
1583 .{ .base = .rbp, .disp = -src_off },1934 Memory.sib(Memory.PtrSize.fromSize(min_abi_size), .{
1584 ));1935 .base = .{ .frame = frame_addr.index },
1585 },1936 .disp = frame_addr.off,
1937 }),
1938 ),
1586 else => return self.fail("TODO airIntCast from {s} to {s}", .{1939 else => return self.fail("TODO airIntCast from {s} to {s}", .{
1587 @tagName(src_mcv),1940 @tagName(src_mcv),
1588 @tagName(dst_mcv),1941 @tagName(dst_mcv),
...@@ -1591,7 +1944,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {...@@ -1591,7 +1944,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
1591 if (self.regExtraBits(min_ty) > 0) try self.truncateRegister(min_ty, dst_reg);1944 if (self.regExtraBits(min_ty) > 0) try self.truncateRegister(min_ty, dst_reg);
1592 },1945 },
1593 else => {1946 else => {
1594 try self.setRegOrMem(min_ty, dst_mcv, src_mcv);1947 try self.genCopy(min_ty, dst_mcv, src_mcv);
1595 const extra = dst_abi_size * 8 - dst_int_info.bits;1948 const extra = dst_abi_size * 8 - dst_int_info.bits;
1596 if (extra > 0) {1949 if (extra > 0) {
1597 try self.genShiftBinOpMir(switch (signedness) {1950 try self.genShiftBinOpMir(switch (signedness) {
...@@ -1653,16 +2006,22 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void {...@@ -1653,16 +2006,22 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
1653 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;2006 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
1654 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;2007 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
16552008
2009 const slice_ty = self.air.typeOfIndex(inst);
1656 const ptr = try self.resolveInst(bin_op.lhs);2010 const ptr = try self.resolveInst(bin_op.lhs);
1657 const ptr_ty = self.air.typeOf(bin_op.lhs);2011 const ptr_ty = self.air.typeOf(bin_op.lhs);
1658 const len = try self.resolveInst(bin_op.rhs);2012 const len = try self.resolveInst(bin_op.rhs);
1659 const len_ty = self.air.typeOf(bin_op.rhs);2013 const len_ty = self.air.typeOf(bin_op.rhs);
16602014
1661 const stack_offset = @intCast(i32, try self.allocMem(inst, 16, 16));2015 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(slice_ty, self.target.*));
1662 try self.genSetStack(ptr_ty, stack_offset, ptr, .{});2016 try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr);
1663 try self.genSetStack(len_ty, stack_offset - 8, len, .{});2017 try self.genSetMem(
1664 const result = MCValue{ .stack_offset = stack_offset };2018 .{ .frame = frame_index },
2019 @intCast(i32, ptr_ty.abiSize(self.target.*)),
2020 len_ty,
2021 len,
2022 );
16652023
2024 const result = MCValue{ .load_frame = .{ .index = frame_index } };
1666 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });2025 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1667}2026}
16682027
...@@ -1778,14 +2137,14 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -1778,14 +2137,14 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
17782137
1779 const reg_bits = self.regBitSize(ty);2138 const reg_bits = self.regBitSize(ty);
1780 const cc: Condition = if (ty.isSignedInt()) cc: {2139 const cc: Condition = if (ty.isSignedInt()) cc: {
1781 try self.genSetReg(ty, limit_reg, dst_mcv);2140 try self.genSetReg(limit_reg, ty, dst_mcv);
1782 try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });2141 try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
1783 try self.genBinOpMir(.xor, ty, limit_mcv, .{2142 try self.genBinOpMir(.xor, ty, limit_mcv, .{
1784 .immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,2143 .immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
1785 });2144 });
1786 break :cc .o;2145 break :cc .o;
1787 } else cc: {2146 } else cc: {
1788 try self.genSetReg(ty, limit_reg, .{2147 try self.genSetReg(limit_reg, ty, .{
1789 .immediate = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - reg_bits),2148 .immediate = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - reg_bits),
1790 });2149 });
1791 break :cc .c;2150 break :cc .c;
...@@ -1829,14 +2188,14 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -1829,14 +2188,14 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
18292188
1830 const reg_bits = self.regBitSize(ty);2189 const reg_bits = self.regBitSize(ty);
1831 const cc: Condition = if (ty.isSignedInt()) cc: {2190 const cc: Condition = if (ty.isSignedInt()) cc: {
1832 try self.genSetReg(ty, limit_reg, dst_mcv);2191 try self.genSetReg(limit_reg, ty, dst_mcv);
1833 try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });2192 try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
1834 try self.genBinOpMir(.xor, ty, limit_mcv, .{2193 try self.genBinOpMir(.xor, ty, limit_mcv, .{
1835 .immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,2194 .immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
1836 });2195 });
1837 break :cc .o;2196 break :cc .o;
1838 } else cc: {2197 } else cc: {
1839 try self.genSetReg(ty, limit_reg, .{ .immediate = 0 });2198 try self.genSetReg(limit_reg, ty, .{ .immediate = 0 });
1840 break :cc .c;2199 break :cc .c;
1841 };2200 };
1842 try self.genBinOpMir(.sub, ty, dst_mcv, rhs_mcv);2201 try self.genBinOpMir(.sub, ty, dst_mcv, rhs_mcv);
...@@ -1880,7 +2239,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -1880,7 +2239,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
18802239
1881 const reg_bits = self.regBitSize(ty);2240 const reg_bits = self.regBitSize(ty);
1882 const cc: Condition = if (ty.isSignedInt()) cc: {2241 const cc: Condition = if (ty.isSignedInt()) cc: {
1883 try self.genSetReg(ty, limit_reg, lhs_mcv);2242 try self.genSetReg(limit_reg, ty, lhs_mcv);
1884 try self.genBinOpMir(.xor, ty, limit_mcv, rhs_mcv);2243 try self.genBinOpMir(.xor, ty, limit_mcv, rhs_mcv);
1885 try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });2244 try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
1886 try self.genBinOpMir(.xor, ty, limit_mcv, .{2245 try self.genBinOpMir(.xor, ty, limit_mcv, .{
...@@ -1888,7 +2247,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -1888,7 +2247,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
1888 });2247 });
1889 break :cc .o;2248 break :cc .o;
1890 } else cc: {2249 } else cc: {
1891 try self.genSetReg(ty, limit_reg, .{2250 try self.genSetReg(limit_reg, ty, .{
1892 .immediate = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - reg_bits),2251 .immediate = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - reg_bits),
1893 });2252 });
1894 break :cc .c;2253 break :cc .c;
...@@ -1916,17 +2275,18 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -1916,17 +2275,18 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1916 .Int => {2275 .Int => {
1917 try self.spillEflagsIfOccupied();2276 try self.spillEflagsIfOccupied();
19182277
1919 const partial_mcv = switch (tag) {2278 const partial_mcv = try self.genBinOp(null, switch (tag) {
1920 .add_with_overflow => try self.genBinOp(null, .add, bin_op.lhs, bin_op.rhs),2279 .add_with_overflow => .add,
1921 .sub_with_overflow => try self.genBinOp(null, .sub, bin_op.lhs, bin_op.rhs),2280 .sub_with_overflow => .sub,
1922 else => unreachable,2281 else => unreachable,
1923 };2282 }, bin_op.lhs, bin_op.rhs);
1924 const int_info = ty.intInfo(self.target.*);2283 const int_info = ty.intInfo(self.target.*);
1925 const cc: Condition = switch (int_info.signedness) {2284 const cc: Condition = switch (int_info.signedness) {
1926 .unsigned => .c,2285 .unsigned => .c,
1927 .signed => .o,2286 .signed => .o,
1928 };2287 };
19292288
2289 const tuple_ty = self.air.typeOfIndex(inst);
1930 if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) {2290 if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) {
1931 switch (partial_mcv) {2291 switch (partial_mcv) {
1932 .register => |reg| {2292 .register => |reg| {
...@@ -1936,27 +2296,32 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -1936,27 +2296,32 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1936 else => {},2296 else => {},
1937 }2297 }
19382298
1939 const abi_size = @intCast(i32, ty.abiSize(self.target.*));2299 const frame_index =
1940 const dst_mcv = try self.allocRegOrMem(inst, false);2300 try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
1941 try self.genSetStack(2301 try self.genSetMem(
2302 .{ .frame = frame_index },
2303 @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)),
1942 Type.u1,2304 Type.u1,
1943 dst_mcv.stack_offset - abi_size,
1944 .{ .eflags = cc },2305 .{ .eflags = cc },
1945 .{},
1946 );2306 );
1947 try self.genSetStack(ty, dst_mcv.stack_offset, partial_mcv, .{});2307 try self.genSetMem(
1948 break :result dst_mcv;2308 .{ .frame = frame_index },
2309 @intCast(i32, tuple_ty.structFieldOffset(0, self.target.*)),
2310 ty,
2311 partial_mcv,
2312 );
2313 break :result .{ .load_frame = .{ .index = frame_index } };
1949 }2314 }
19502315
1951 const tuple_ty = self.air.typeOfIndex(inst);2316 const frame_index =
1952 const tuple_size = @intCast(u32, tuple_ty.abiSize(self.target.*));2317 try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
1953 const tuple_align = tuple_ty.abiAlignment(self.target.*);2318 try self.genSetFrameTruncatedOverflowCompare(
1954 const overflow_bit_offset = @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*));2319 tuple_ty,
1955 const stack_offset = @intCast(i32, try self.allocMem(inst, tuple_size, tuple_align));2320 frame_index,
19562321 partial_mcv.register,
1957 try self.genSetStackTruncatedOverflowCompare(ty, stack_offset, overflow_bit_offset, partial_mcv.register, cc);2322 cc,
19582323 );
1959 break :result .{ .stack_offset = stack_offset };2324 break :result .{ .load_frame = .{ .index = frame_index } };
1960 },2325 },
1961 else => unreachable,2326 else => unreachable,
1962 }2327 }
...@@ -1998,6 +2363,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -1998,6 +2363,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1998 try self.genBinOpMir(.cmp, lhs_ty, tmp_mcv, lhs);2363 try self.genBinOpMir(.cmp, lhs_ty, tmp_mcv, lhs);
1999 const cc = Condition.ne;2364 const cc = Condition.ne;
20002365
2366 const tuple_ty = self.air.typeOfIndex(inst);
2001 if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) {2367 if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) {
2002 switch (partial_mcv) {2368 switch (partial_mcv) {
2003 .register => |reg| {2369 .register => |reg| {
...@@ -2007,27 +2373,32 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -2007,27 +2373,32 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2007 else => {},2373 else => {},
2008 }2374 }
20092375
2010 const abi_size = @intCast(i32, lhs_ty.abiSize(self.target.*));2376 const frame_index =
2011 const dst_mcv = try self.allocRegOrMem(inst, false);2377 try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
2012 try self.genSetStack(2378 try self.genSetMem(
2013 Type.u1,2379 .{ .frame = frame_index },
2014 dst_mcv.stack_offset - abi_size,2380 @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)),
2381 tuple_ty.structFieldType(1),
2015 .{ .eflags = cc },2382 .{ .eflags = cc },
2016 .{},
2017 );2383 );
2018 try self.genSetStack(lhs_ty, dst_mcv.stack_offset, partial_mcv, .{});2384 try self.genSetMem(
2019 break :result dst_mcv;2385 .{ .frame = frame_index },
2386 @intCast(i32, tuple_ty.structFieldOffset(0, self.target.*)),
2387 tuple_ty.structFieldType(0),
2388 partial_mcv,
2389 );
2390 break :result .{ .load_frame = .{ .index = frame_index } };
2020 }2391 }
20212392
2022 const tuple_ty = self.air.typeOfIndex(inst);2393 const frame_index =
2023 const tuple_size = @intCast(u32, tuple_ty.abiSize(self.target.*));2394 try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
2024 const tuple_align = tuple_ty.abiAlignment(self.target.*);2395 try self.genSetFrameTruncatedOverflowCompare(
2025 const overflow_bit_offset = @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*));2396 tuple_ty,
2026 const stack_offset = @intCast(i32, try self.allocMem(inst, tuple_size, tuple_align));2397 frame_index,
20272398 partial_mcv.register,
2028 try self.genSetStackTruncatedOverflowCompare(lhs_ty, stack_offset, overflow_bit_offset, partial_mcv.register, cc);2399 cc,
20292400 );
2030 break :result .{ .stack_offset = stack_offset };2401 break :result .{ .load_frame = .{ .index = frame_index } };
2031 },2402 },
2032 else => unreachable,2403 else => unreachable,
2033 }2404 }
...@@ -2035,17 +2406,17 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -2035,17 +2406,17 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2035 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });2406 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2036}2407}
20372408
2038fn genSetStackTruncatedOverflowCompare(2409fn genSetFrameTruncatedOverflowCompare(
2039 self: *Self,2410 self: *Self,
2040 ty: Type,2411 tuple_ty: Type,
2041 stack_offset: i32,2412 frame_index: FrameIndex,
2042 overflow_bit_offset: i32,
2043 reg: Register,2413 reg: Register,
2044 cc: Condition,2414 cc: Condition,
2045) !void {2415) !void {
2046 const reg_lock = self.register_manager.lockReg(reg);2416 const reg_lock = self.register_manager.lockReg(reg);
2047 defer if (reg_lock) |lock| self.register_manager.unlockReg(lock);2417 defer if (reg_lock) |lock| self.register_manager.unlockReg(lock);
20482418
2419 const ty = tuple_ty.structFieldType(0);
2049 const int_info = ty.intInfo(self.target.*);2420 const int_info = ty.intInfo(self.target.*);
2050 const extended_ty = switch (int_info.signedness) {2421 const extended_ty = switch (int_info.signedness) {
2051 .signed => Type.isize,2422 .signed => Type.isize,
...@@ -2062,7 +2433,7 @@ fn genSetStackTruncatedOverflowCompare(...@@ -2062,7 +2433,7 @@ fn genSetStackTruncatedOverflowCompare(
2062 try self.asmSetccRegister(overflow_reg.to8(), cc);2433 try self.asmSetccRegister(overflow_reg.to8(), cc);
20632434
2064 const scratch_reg = temp_regs[1];2435 const scratch_reg = temp_regs[1];
2065 try self.genSetReg(extended_ty, scratch_reg, .{ .register = reg });2436 try self.genSetReg(scratch_reg, extended_ty, .{ .register = reg });
2066 try self.truncateRegister(ty, scratch_reg);2437 try self.truncateRegister(ty, scratch_reg);
2067 try self.genBinOpMir(2438 try self.genBinOpMir(
2068 .cmp,2439 .cmp,
...@@ -2080,10 +2451,18 @@ fn genSetStackTruncatedOverflowCompare(...@@ -2080,10 +2451,18 @@ fn genSetStackTruncatedOverflowCompare(
2080 .{ .register = eq_reg },2451 .{ .register = eq_reg },
2081 );2452 );
20822453
2083 try self.genSetStack(ty, stack_offset, .{ .register = scratch_reg }, .{});2454 try self.genSetMem(
2084 try self.genSetStack(Type.u1, stack_offset - overflow_bit_offset, .{2455 .{ .frame = frame_index },
2085 .register = overflow_reg.to8(),2456 @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)),
2086 }, .{});2457 tuple_ty.structFieldType(1),
2458 .{ .register = overflow_reg.to8() },
2459 );
2460 try self.genSetMem(
2461 .{ .frame = frame_index },
2462 @intCast(i32, tuple_ty.structFieldOffset(0, self.target.*)),
2463 ty,
2464 .{ .register = scratch_reg },
2465 );
2087}2466}
20882467
2089fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {2468fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
...@@ -2101,6 +2480,8 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -2101,6 +2480,8 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2101 .unsigned => .c,2480 .unsigned => .c,
2102 .signed => .o,2481 .signed => .o,
2103 };2482 };
2483
2484 const tuple_ty = self.air.typeOfIndex(inst);
2104 if (dst_info.bits >= 8 and math.isPowerOfTwo(dst_info.bits)) {2485 if (dst_info.bits >= 8 and math.isPowerOfTwo(dst_info.bits)) {
2105 var src_pl = Type.Payload.Bits{ .base = .{ .tag = switch (dst_info.signedness) {2486 var src_pl = Type.Payload.Bits{ .base = .{ .tag = switch (dst_info.signedness) {
2106 .signed => .int_signed,2487 .signed => .int_signed,
...@@ -2127,16 +2508,21 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -2127,16 +2508,21 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
21272508
2128 // For now, this is the only supported multiply that doesn't fit in a register.2509 // For now, this is the only supported multiply that doesn't fit in a register.
2129 assert(dst_info.bits == 128 and src_pl.data == 64);2510 assert(dst_info.bits == 128 and src_pl.data == 64);
2130 const dst_abi_size = @intCast(i32, dst_ty.abiSize(self.target.*));2511 const frame_index =
2131 const dst_mcv = try self.allocRegOrMem(inst, false);2512 try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
2132 try self.genSetStack(2513 try self.genSetMem(
2133 Type.u1,2514 .{ .frame = frame_index },
2134 dst_mcv.stack_offset - dst_abi_size,2515 @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)),
2135 .{ .immediate = 0 }, // 64x64 -> 128 never overflows2516 tuple_ty.structFieldType(1),
2136 .{},2517 .{ .immediate = 0 }, // overflow is impossible for 64-bit*64-bit -> 128-bit
2137 );2518 );
2138 try self.genSetStack(dst_ty, dst_mcv.stack_offset, partial_mcv, .{});2519 try self.genSetMem(
2139 break :result dst_mcv;2520 .{ .frame = frame_index },
2521 @intCast(i32, tuple_ty.structFieldOffset(0, self.target.*)),
2522 tuple_ty.structFieldType(0),
2523 partial_mcv,
2524 );
2525 break :result .{ .load_frame = .{ .index = frame_index } };
2140 }2526 }
21412527
2142 const dst_reg: Register = dst_reg: {2528 const dst_reg: Register = dst_reg: {
...@@ -2184,15 +2570,10 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -2184,15 +2570,10 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
2184 }2570 }
2185 };2571 };
21862572
2187 const tuple_ty = self.air.typeOfIndex(inst);2573 const frame_index =
2188 const tuple_size = @intCast(u32, tuple_ty.abiSize(self.target.*));2574 try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
2189 const tuple_align = tuple_ty.abiAlignment(self.target.*);2575 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, dst_reg, cc);
2190 const overflow_bit_offset = @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*));2576 break :result .{ .load_frame = .{ .index = frame_index } };
2191 const stack_offset = @intCast(i32, try self.allocMem(inst, tuple_size, tuple_align));
2192
2193 try self.genSetStackTruncatedOverflowCompare(dst_ty, stack_offset, overflow_bit_offset, dst_reg, cc);
2194
2195 break :result .{ .stack_offset = stack_offset };
2196 },2577 },
2197 else => unreachable,2578 else => unreachable,
2198 }2579 }
...@@ -2220,7 +2601,7 @@ fn genIntMulDivOpMir(...@@ -2220,7 +2601,7 @@ fn genIntMulDivOpMir(
2220 .register => |reg| if (reg.to64() == .rax) break :lhs,2601 .register => |reg| if (reg.to64() == .rax) break :lhs,
2221 else => {},2602 else => {},
2222 }2603 }
2223 try self.genSetReg(ty, .rax, lhs);2604 try self.genSetReg(.rax, ty, lhs);
2224 }2605 }
22252606
2226 switch (tag) {2607 switch (tag) {
...@@ -2230,16 +2611,26 @@ fn genIntMulDivOpMir(...@@ -2230,16 +2611,26 @@ fn genIntMulDivOpMir(
2230 .idiv => try self.asmOpOnly(.cqo),2611 .idiv => try self.asmOpOnly(.cqo),
2231 }2612 }
22322613
2233 const factor: MCValue = switch (rhs) {2614 const mat_rhs: MCValue = switch (rhs) {
2234 .register, .stack_offset => rhs,2615 .register, .indirect, .load_frame => rhs,
2235 else => .{ .register = try self.copyToTmpRegister(ty, rhs) },2616 else => .{ .register = try self.copyToTmpRegister(ty, rhs) },
2236 };2617 };
2237 switch (factor) {2618 switch (mat_rhs) {
2238 .register => |reg| try self.asmRegister(tag, reg),2619 .register => |reg| try self.asmRegister(tag, reg),
2239 .stack_offset => |off| try self.asmMemory(tag, Memory.sib(2620 .indirect, .load_frame => try self.asmMemory(
2240 Memory.PtrSize.fromSize(abi_size),2621 tag,
2241 .{ .base = .rbp, .disp = -off },2622 Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (mat_rhs) {
2242 )),2623 .indirect => |reg_off| .{
2624 .base = .{ .reg = reg_off.reg },
2625 .disp = reg_off.off,
2626 },
2627 .load_frame => |frame_addr| .{
2628 .base = .{ .frame = frame_addr.index },
2629 .disp = frame_addr.off,
2630 },
2631 else => unreachable,
2632 }),
2633 ),
2243 else => unreachable,2634 else => unreachable,
2244 }2635 }
2245}2636}
...@@ -2315,7 +2706,7 @@ fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {...@@ -2315,7 +2706,7 @@ fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {
2315 }2706 }
23162707
2317 const pl_mcv = try self.allocRegOrMem(inst, true);2708 const pl_mcv = try self.allocRegOrMem(inst, true);
2318 try self.setRegOrMem(pl_ty, pl_mcv, switch (opt_mcv) {2709 try self.genCopy(pl_ty, pl_mcv, switch (opt_mcv) {
2319 else => opt_mcv,2710 else => opt_mcv,
2320 .register_overflow => |ro| .{ .register = ro.reg },2711 .register_overflow => |ro| .{ .register = ro.reg },
2321 });2712 });
...@@ -2361,11 +2752,7 @@ fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {...@@ -2361,11 +2752,7 @@ fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
23612752
2362 const pl_ty = dst_ty.childType();2753 const pl_ty = dst_ty.childType();
2363 const pl_abi_size = @intCast(i32, pl_ty.abiSize(self.target.*));2754 const pl_abi_size = @intCast(i32, pl_ty.abiSize(self.target.*));
2364 try self.asmMemoryImmediate(2755 try self.genSetMem(.{ .reg = dst_mcv.register }, pl_abi_size, Type.bool, .{ .immediate = 1 });
2365 .mov,
2366 Memory.sib(.byte, .{ .base = dst_mcv.register, .disp = pl_abi_size }),
2367 Immediate.u(1),
2368 );
2369 break :result if (self.liveness.isUnused(inst)) .unreach else dst_mcv;2756 break :result if (self.liveness.isUnused(inst)) .unreach else dst_mcv;
2370 };2757 };
2371 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });2758 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
...@@ -2389,10 +2776,6 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {...@@ -2389,10 +2776,6 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
23892776
2390 const err_off = errUnionErrorOffset(payload_ty, self.target.*);2777 const err_off = errUnionErrorOffset(payload_ty, self.target.*);
2391 switch (operand) {2778 switch (operand) {
2392 .stack_offset => |off| {
2393 const offset = off - @intCast(i32, err_off);
2394 break :result MCValue{ .stack_offset = offset };
2395 },
2396 .register => |reg| {2779 .register => |reg| {
2397 // TODO reuse operand2780 // TODO reuse operand
2398 const eu_lock = self.register_manager.lockReg(reg);2781 const eu_lock = self.register_manager.lockReg(reg);
...@@ -2407,6 +2790,10 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {...@@ -2407,6 +2790,10 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
2407 }2790 }
2408 break :result result;2791 break :result result;
2409 },2792 },
2793 .load_frame => |frame_addr| break :result .{ .load_frame = .{
2794 .index = frame_addr.index,
2795 .off = frame_addr.off + @intCast(i32, err_off),
2796 } },
2410 else => return self.fail("TODO implement unwrap_err_err for {}", .{operand}),2797 else => return self.fail("TODO implement unwrap_err_err for {}", .{operand}),
2411 }2798 }
2412 };2799 };
...@@ -2434,10 +2821,10 @@ fn genUnwrapErrorUnionPayloadMir(...@@ -2434,10 +2821,10 @@ fn genUnwrapErrorUnionPayloadMir(
24342821
2435 const payload_off = errUnionPayloadOffset(payload_ty, self.target.*);2822 const payload_off = errUnionPayloadOffset(payload_ty, self.target.*);
2436 switch (err_union) {2823 switch (err_union) {
2437 .stack_offset => |off| {2824 .load_frame => |frame_addr| break :result .{ .load_frame = .{
2438 const offset = off - @intCast(i32, payload_off);2825 .index = frame_addr.index,
2439 break :result MCValue{ .stack_offset = offset };2826 .off = frame_addr.off + @intCast(i32, payload_off),
2440 },2827 } },
2441 .register => |reg| {2828 .register => |reg| {
2442 // TODO reuse operand2829 // TODO reuse operand
2443 const eu_lock = self.register_manager.lockReg(reg);2830 const eu_lock = self.register_manager.lockReg(reg);
...@@ -2488,7 +2875,10 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -2488,7 +2875,10 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
2488 try self.asmRegisterMemory(2875 try self.asmRegisterMemory(
2489 .mov,2876 .mov,
2490 registerAlias(dst_reg, err_abi_size),2877 registerAlias(dst_reg, err_abi_size),
2491 Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{ .base = src_reg, .disp = err_off }),2878 Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{
2879 .base = .{ .reg = src_reg },
2880 .disp = err_off,
2881 }),
2492 );2882 );
24932883
2494 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });2884 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
...@@ -2523,7 +2913,7 @@ fn airUnwrapErrUnionPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -2523,7 +2913,7 @@ fn airUnwrapErrUnionPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
2523 try self.asmRegisterMemory(2913 try self.asmRegisterMemory(
2524 .lea,2914 .lea,
2525 registerAlias(dst_reg, dst_abi_size),2915 registerAlias(dst_reg, dst_abi_size),
2526 Memory.sib(.qword, .{ .base = src_reg, .disp = pl_off }),2916 Memory.sib(.qword, .{ .base = .{ .reg = src_reg }, .disp = pl_off }),
2527 );2917 );
25282918
2529 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });2919 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
...@@ -2548,7 +2938,10 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {...@@ -2548,7 +2938,10 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
2548 const err_abi_size = @intCast(u32, err_ty.abiSize(self.target.*));2938 const err_abi_size = @intCast(u32, err_ty.abiSize(self.target.*));
2549 try self.asmMemoryImmediate(2939 try self.asmMemoryImmediate(
2550 .mov,2940 .mov,
2551 Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{ .base = src_reg, .disp = err_off }),2941 Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{
2942 .base = .{ .reg = src_reg },
2943 .disp = err_off,
2944 }),
2552 Immediate.u(0),2945 Immediate.u(0),
2553 );2946 );
25542947
...@@ -2567,7 +2960,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {...@@ -2567,7 +2960,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
2567 try self.asmRegisterMemory(2960 try self.asmRegisterMemory(
2568 .lea,2961 .lea,
2569 registerAlias(dst_reg, dst_abi_size),2962 registerAlias(dst_reg, dst_abi_size),
2570 Memory.sib(.qword, .{ .base = src_reg, .disp = pl_off }),2963 Memory.sib(.qword, .{ .base = .{ .reg = src_reg }, .disp = pl_off }),
2571 );2964 );
2572 break :result .{ .register = dst_reg };2965 break :result .{ .register = dst_reg };
2573 };2966 };
...@@ -2608,7 +3001,7 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {...@@ -2608,7 +3001,7 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
2608 defer if (pl_lock) |lock| self.register_manager.unlockReg(lock);3001 defer if (pl_lock) |lock| self.register_manager.unlockReg(lock);
26093002
2610 const opt_mcv = try self.allocRegOrMem(inst, true);3003 const opt_mcv = try self.allocRegOrMem(inst, true);
2611 try self.setRegOrMem(pl_ty, opt_mcv, pl_mcv);3004 try self.genCopy(pl_ty, opt_mcv, pl_mcv);
26123005
2613 if (!same_repr) {3006 if (!same_repr) {
2614 const pl_abi_size = @intCast(i32, pl_ty.abiSize(self.target.*));3007 const pl_abi_size = @intCast(i32, pl_ty.abiSize(self.target.*));
...@@ -2621,9 +3014,12 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {...@@ -2621,9 +3014,12 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
2621 Immediate.u(@intCast(u6, pl_abi_size * 8)),3014 Immediate.u(@intCast(u6, pl_abi_size * 8)),
2622 ),3015 ),
26233016
2624 .stack_offset => |off| try self.asmMemoryImmediate(3017 .load_frame => |frame_addr| try self.asmMemoryImmediate(
2625 .mov,3018 .mov,
2626 Memory.sib(.byte, .{ .base = .rbp, .disp = pl_abi_size - off }),3019 Memory.sib(.byte, .{
3020 .base = .{ .frame = frame_addr.index },
3021 .disp = frame_addr.off + pl_abi_size,
3022 }),
2627 Immediate.u(1),3023 Immediate.u(1),
2628 ),3024 ),
2629 }3025 }
...@@ -2637,26 +3033,21 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {...@@ -2637,26 +3033,21 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
2637fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {3033fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
2638 const ty_op = self.air.instructions.items(.data)[inst].ty_op;3034 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
26393035
2640 const error_union_ty = self.air.getRefType(ty_op.ty);3036 const eu_ty = self.air.getRefType(ty_op.ty);
2641 const payload_ty = error_union_ty.errorUnionPayload();3037 const pl_ty = eu_ty.errorUnionPayload();
3038 const err_ty = eu_ty.errorUnionSet();
2642 const operand = try self.resolveInst(ty_op.operand);3039 const operand = try self.resolveInst(ty_op.operand);
26433040
2644 const result: MCValue = result: {3041 const result: MCValue = result: {
2645 if (!payload_ty.hasRuntimeBitsIgnoreComptime()) {3042 if (!pl_ty.hasRuntimeBitsIgnoreComptime()) break :result .{ .immediate = 0 };
2646 break :result operand;
2647 }
2648
2649 const abi_size = @intCast(u32, error_union_ty.abiSize(self.target.*));
2650 const abi_align = error_union_ty.abiAlignment(self.target.*);
2651 const stack_offset = @intCast(i32, try self.allocMem(inst, abi_size, abi_align));
2652 const payload_off = errUnionPayloadOffset(payload_ty, self.target.*);
2653 const err_off = errUnionErrorOffset(payload_ty, self.target.*);
2654 try self.genSetStack(payload_ty, stack_offset - @intCast(i32, payload_off), operand, .{});
2655 try self.genSetStack(Type.anyerror, stack_offset - @intCast(i32, err_off), .{ .immediate = 0 }, .{});
26563043
2657 break :result MCValue{ .stack_offset = stack_offset };3044 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(eu_ty, self.target.*));
3045 const pl_off = @intCast(i32, errUnionPayloadOffset(pl_ty, self.target.*));
3046 const err_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*));
3047 try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, operand);
3048 try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, .{ .immediate = 0 });
3049 break :result .{ .load_frame = .{ .index = frame_index } };
2658 };3050 };
2659
2660 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });3051 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2661}3052}
26623053
...@@ -2664,26 +3055,21 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {...@@ -2664,26 +3055,21 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
2664fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {3055fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
2665 const ty_op = self.air.instructions.items(.data)[inst].ty_op;3056 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
26663057
2667 const error_union_ty = self.air.getRefType(ty_op.ty);3058 const eu_ty = self.air.getRefType(ty_op.ty);
2668 const payload_ty = error_union_ty.errorUnionPayload();3059 const pl_ty = eu_ty.errorUnionPayload();
2669 const operand = try self.resolveInst(ty_op.operand);3060 const err_ty = eu_ty.errorUnionSet();
26703061
2671 const result: MCValue = result: {3062 const result: MCValue = result: {
2672 if (!payload_ty.hasRuntimeBitsIgnoreComptime()) {3063 if (!pl_ty.hasRuntimeBitsIgnoreComptime()) break :result try self.resolveInst(ty_op.operand);
2673 break :result operand;
2674 }
2675
2676 const abi_size = @intCast(u32, error_union_ty.abiSize(self.target.*));
2677 const abi_align = error_union_ty.abiAlignment(self.target.*);
2678 const stack_offset = @intCast(i32, try self.allocMem(inst, abi_size, abi_align));
2679 const payload_off = errUnionPayloadOffset(payload_ty, self.target.*);
2680 const err_off = errUnionErrorOffset(payload_ty, self.target.*);
2681 try self.genSetStack(Type.anyerror, stack_offset - @intCast(i32, err_off), operand, .{});
2682 try self.genSetStack(payload_ty, stack_offset - @intCast(i32, payload_off), .undef, .{});
26833064
2684 break :result MCValue{ .stack_offset = stack_offset };3065 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(eu_ty, self.target.*));
3066 const pl_off = @intCast(i32, errUnionPayloadOffset(pl_ty, self.target.*));
3067 const err_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*));
3068 try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .undef);
3069 const operand = try self.resolveInst(ty_op.operand);
3070 try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, operand);
3071 break :result .{ .load_frame = .{ .index = frame_index } };
2685 };3072 };
2686
2687 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });3073 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2688}3074}
26893075
...@@ -2695,7 +3081,7 @@ fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -2695,7 +3081,7 @@ fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {
26953081
2696 const dst_mcv = try self.allocRegOrMem(inst, true);3082 const dst_mcv = try self.allocRegOrMem(inst, true);
2697 const dst_ty = self.air.typeOfIndex(inst);3083 const dst_ty = self.air.typeOfIndex(inst);
2698 try self.setRegOrMem(dst_ty, dst_mcv, src_mcv);3084 try self.genCopy(dst_ty, dst_mcv, src_mcv);
2699 break :result dst_mcv;3085 break :result dst_mcv;
2700 };3086 };
2701 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });3087 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
...@@ -2707,9 +3093,10 @@ fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {...@@ -2707,9 +3093,10 @@ fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {
2707 const operand = try self.resolveInst(ty_op.operand);3093 const operand = try self.resolveInst(ty_op.operand);
2708 const dst_mcv: MCValue = blk: {3094 const dst_mcv: MCValue = blk: {
2709 switch (operand) {3095 switch (operand) {
2710 .stack_offset => |off| {3096 .load_frame => |frame_addr| break :blk .{ .load_frame = .{
2711 break :blk MCValue{ .stack_offset = off - 8 };3097 .index = frame_addr.index,
2712 },3098 .off = frame_addr.off + 8,
3099 } },
2713 else => return self.fail("TODO implement slice_len for {}", .{operand}),3100 else => return self.fail("TODO implement slice_len for {}", .{operand}),
2714 }3101 }
2715 };3102 };
...@@ -2743,7 +3130,7 @@ fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -2743,7 +3130,7 @@ fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
2743 .lea,3130 .lea,
2744 registerAlias(dst_reg, dst_abi_size),3131 registerAlias(dst_reg, dst_abi_size),
2745 Memory.sib(.qword, .{3132 Memory.sib(.qword, .{
2746 .base = src_reg,3133 .base = .{ .reg = src_reg },
2747 .disp = @divExact(self.target.cpu.arch.ptrBitWidth(), 8),3134 .disp = @divExact(self.target.cpu.arch.ptrBitWidth(), 8),
2748 }),3135 }),
2749 );3136 );
...@@ -2771,7 +3158,7 @@ fn elemOffset(self: *Self, index_ty: Type, index: MCValue, elem_size: u64) !Regi...@@ -2771,7 +3158,7 @@ fn elemOffset(self: *Self, index_ty: Type, index: MCValue, elem_size: u64) !Regi
2771 // Optimisation: if index MCValue is an immediate, we can multiply in `comptime`3158 // Optimisation: if index MCValue is an immediate, we can multiply in `comptime`
2772 // and set the register directly to the scaled offset as an immediate.3159 // and set the register directly to the scaled offset as an immediate.
2773 const reg = try self.register_manager.allocReg(null, gp);3160 const reg = try self.register_manager.allocReg(null, gp);
2774 try self.genSetReg(index_ty, reg, .{ .immediate = imm * elem_size });3161 try self.genSetReg(reg, index_ty, .{ .immediate = imm * elem_size });
2775 break :blk reg;3162 break :blk reg;
2776 },3163 },
2777 else => {3164 else => {
...@@ -2812,10 +3199,14 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {...@@ -2812,10 +3199,14 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
28123199
2813 const addr_reg = try self.register_manager.allocReg(null, gp);3200 const addr_reg = try self.register_manager.allocReg(null, gp);
2814 switch (slice_mcv) {3201 switch (slice_mcv) {
2815 .stack_offset => |off| try self.asmRegisterMemory(.mov, addr_reg.to64(), Memory.sib(.qword, .{3202 .load_frame => |frame_addr| try self.asmRegisterMemory(
2816 .base = .rbp,3203 .mov,
2817 .disp = -off,3204 addr_reg.to64(),
2818 })),3205 Memory.sib(.qword, .{
3206 .base = .{ .frame = frame_addr.index },
3207 .disp = frame_addr.off,
3208 }),
3209 ),
2819 else => return self.fail("TODO implement slice_elem_ptr when slice is {}", .{slice_mcv}),3210 else => return self.fail("TODO implement slice_elem_ptr when slice is {}", .{slice_mcv}),
2820 }3211 }
2821 // TODO we could allocate register here, but need to expect addr register and potentially3212 // TODO we could allocate register here, but need to expect addr register and potentially
...@@ -2834,7 +3225,7 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -2834,7 +3225,7 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
2834 const slice_ptr_field_type = slice_ty.slicePtrFieldType(&buf);3225 const slice_ptr_field_type = slice_ty.slicePtrFieldType(&buf);
2835 const elem_ptr = try self.genSliceElemPtr(bin_op.lhs, bin_op.rhs);3226 const elem_ptr = try self.genSliceElemPtr(bin_op.lhs, bin_op.rhs);
2836 const dst_mcv = try self.allocRegOrMem(inst, false);3227 const dst_mcv = try self.allocRegOrMem(inst, false);
2837 try self.load(dst_mcv, elem_ptr, slice_ptr_field_type);3228 try self.load(dst_mcv, slice_ptr_field_type, elem_ptr);
28383229
2839 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });3230 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
2840}3231}
...@@ -2872,33 +3263,27 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -2872,33 +3263,27 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
2872 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);3263 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
2873 defer self.register_manager.unlockReg(offset_reg_lock);3264 defer self.register_manager.unlockReg(offset_reg_lock);
28743265
2875 const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();3266 const addr_reg = try self.register_manager.allocReg(null, gp);
2876 switch (array) {3267 switch (array) {
2877 .register => {3268 .register => {
2878 const off = @intCast(i32, try self.allocMem(3269 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, self.target.*));
2879 inst,3270 try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array);
2880 @intCast(u32, array_ty.abiSize(self.target.*)),3271 try self.asmRegisterMemory(
2881 array_ty.abiAlignment(self.target.*),3272 .lea,
2882 ));3273 addr_reg,
2883 try self.genSetStack(array_ty, off, array, .{});3274 Memory.sib(.qword, .{ .base = .{ .frame = frame_index } }),
2884 try self.asmRegisterMemory(.lea, addr_reg, Memory.sib(.qword, .{3275 );
2885 .base = .rbp,
2886 .disp = -off,
2887 }));
2888 },
2889 .stack_offset => |off| {
2890 try self.asmRegisterMemory(.lea, addr_reg, Memory.sib(.qword, .{
2891 .base = .rbp,
2892 .disp = -off,
2893 }));
2894 },3276 },
2895 .load_got => try self.genSetReg(array_ty, addr_reg, array),3277 .load_frame => |frame_addr| try self.asmRegisterMemory(
2896 .memory, .load_direct, .load_tlv => try self.genSetReg(Type.usize, addr_reg, switch (array) {3278 .lea,
2897 .memory => |addr| .{ .immediate = addr },3279 addr_reg,
2898 .load_direct => |sym_index| .{ .lea_direct = sym_index },3280 Memory.sib(.qword, .{ .base = .{ .frame = frame_addr.index }, .disp = frame_addr.off }),
2899 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },3281 ),
2900 else => unreachable,3282 .memory,
2901 }),3283 .load_direct,
3284 .load_got,
3285 .load_tlv,
3286 => try self.genSetReg(addr_reg, Type.usize, array.address()),
2902 .lea_direct, .lea_tlv => unreachable,3287 .lea_direct, .lea_tlv => unreachable,
2903 else => return self.fail("TODO implement array_elem_val when array is {}", .{array}),3288 else => return self.fail("TODO implement array_elem_val when array is {}", .{array}),
2904 }3289 }
...@@ -2907,7 +3292,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -2907,7 +3292,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
2907 // offset register.3292 // offset register.
2908 const dst_mcv = try self.allocRegOrMem(inst, false);3293 const dst_mcv = try self.allocRegOrMem(inst, false);
2909 try self.genBinOpMir(.add, Type.usize, .{ .register = addr_reg }, .{ .register = offset_reg });3294 try self.genBinOpMir(.add, Type.usize, .{ .register = addr_reg }, .{ .register = offset_reg });
2910 try self.load(dst_mcv, .{ .register = addr_reg.to64() }, array_ty);3295 try self.genCopy(elem_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } });
29113296
2912 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });3297 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
2913}3298}
...@@ -2948,7 +3333,7 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -2948,7 +3333,7 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
2948 else => null,3333 else => null,
2949 };3334 };
2950 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);3335 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
2951 try self.load(dst_mcv, .{ .register = elem_ptr_reg }, ptr_ty);3336 try self.load(dst_mcv, ptr_ty, .{ .register = elem_ptr_reg });
29523337
2953 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });3338 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
2954}3339}
...@@ -2987,8 +3372,8 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -2987,8 +3372,8 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
29873372
2988fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {3373fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
2989 const bin_op = self.air.instructions.items(.data)[inst].bin_op;3374 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
2990 const ptr_ty = self.air.typeOf(bin_op.lhs);3375 const ptr_union_ty = self.air.typeOf(bin_op.lhs);
2991 const union_ty = ptr_ty.childType();3376 const union_ty = ptr_union_ty.childType();
2992 const tag_ty = self.air.typeOf(bin_op.rhs);3377 const tag_ty = self.air.typeOf(bin_op.rhs);
2993 const layout = union_ty.unionGetLayout(self.target.*);3378 const layout = union_ty.unionGetLayout(self.target.*);
29943379
...@@ -3012,12 +3397,15 @@ fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {...@@ -3012,12 +3397,15 @@ fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
30123397
3013 const adjusted_ptr: MCValue = if (layout.payload_size > 0 and layout.tag_align < layout.payload_align) blk: {3398 const adjusted_ptr: MCValue = if (layout.payload_size > 0 and layout.tag_align < layout.payload_align) blk: {
3014 // TODO reusing the operand3399 // TODO reusing the operand
3015 const reg = try self.copyToTmpRegister(ptr_ty, ptr);3400 const reg = try self.copyToTmpRegister(ptr_union_ty, ptr);
3016 try self.genBinOpMir(.add, ptr_ty, .{ .register = reg }, .{ .immediate = layout.payload_size });3401 try self.genBinOpMir(.add, ptr_union_ty, .{ .register = reg }, .{ .immediate = layout.payload_size });
3017 break :blk MCValue{ .register = reg };3402 break :blk MCValue{ .register = reg };
3018 } else ptr;3403 } else ptr;
30193404
3020 try self.store(adjusted_ptr, tag, ptr_ty, tag_ty);3405 var ptr_tag_pl = ptr_union_ty.ptrInfo();
3406 ptr_tag_pl.data.pointee_type = tag_ty;
3407 const ptr_tag_ty = Type.initPayload(&ptr_tag_pl.base);
3408 try self.store(ptr_tag_ty, adjusted_ptr, tag);
30213409
3022 return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });3410 return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });
3023}3411}
...@@ -3044,11 +3432,14 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {...@@ -3044,11 +3432,14 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
3044 const tag_abi_size = tag_ty.abiSize(self.target.*);3432 const tag_abi_size = tag_ty.abiSize(self.target.*);
3045 const dst_mcv: MCValue = blk: {3433 const dst_mcv: MCValue = blk: {
3046 switch (operand) {3434 switch (operand) {
3047 .stack_offset => |off| {3435 .load_frame => |frame_addr| {
3048 if (tag_abi_size <= 8) {3436 if (tag_abi_size <= 8) {
3049 const offset: i32 = if (layout.tag_align < layout.payload_align) @intCast(i32, layout.payload_size) else 0;3437 const off: i32 = if (layout.tag_align < layout.payload_align)
3438 @intCast(i32, layout.payload_size)
3439 else
3440 0;
3050 break :blk try self.copyToRegisterWithInstTracking(inst, tag_ty, .{3441 break :blk try self.copyToRegisterWithInstTracking(inst, tag_ty, .{
3051 .stack_offset = off - offset,3442 .load_frame = .{ .index = frame_addr.index, .off = frame_addr.off + off },
3052 });3443 });
3053 }3444 }
30543445
...@@ -3131,7 +3522,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {...@@ -3131,7 +3522,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
3131 .nz,3522 .nz,
3132 );3523 );
31333524
3134 try self.genSetReg(dst_ty, dst_reg, .{ .immediate = src_bits - 1 });3525 try self.genSetReg(dst_reg, dst_ty, .{ .immediate = src_bits - 1 });
3135 try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .register = imm_reg });3526 try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .register = imm_reg });
3136 }3527 }
3137 break :result dst_mcv;3528 break :result dst_mcv;
...@@ -3171,7 +3562,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {...@@ -3171,7 +3562,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
3171 };3562 };
3172 const tmp_mcv = tmp: {3563 const tmp_mcv = tmp: {
3173 if (src_mcv.isImmediate() or self.liveness.operandDies(inst, 0)) break :tmp src_mcv;3564 if (src_mcv.isImmediate() or self.liveness.operandDies(inst, 0)) break :tmp src_mcv;
3174 try self.genSetReg(src_ty, dst_reg, src_mcv);3565 try self.genSetReg(dst_reg, src_ty, src_mcv);
3175 break :tmp dst_mcv;3566 break :tmp dst_mcv;
3176 };3567 };
3177 try self.genBinOpMir(.@"or", src_ty, tmp_mcv, mask_mcv);3568 try self.genBinOpMir(.@"or", src_ty, tmp_mcv, mask_mcv);
...@@ -3345,7 +3736,7 @@ fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, m...@@ -3345,7 +3736,7 @@ fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, m
3345 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_mcv.register);3736 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_mcv.register);
3346 defer self.register_manager.unlockReg(dst_lock);3737 defer self.register_manager.unlockReg(dst_lock);
33473738
3348 try self.genSetReg(src_ty, dst_mcv.register, src_mcv);3739 try self.genSetReg(dst_mcv.register, src_ty, src_mcv);
3349 switch (src_bits) {3740 switch (src_bits) {
3350 else => unreachable,3741 else => unreachable,
3351 16 => try self.genBinOpMir(.rol, src_ty, dst_mcv, .{ .immediate = 8 }),3742 16 => try self.genBinOpMir(.rol, src_ty, dst_mcv, .{ .immediate = 8 }),
...@@ -3450,7 +3841,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {...@@ -3450,7 +3841,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
3450 .lea,3841 .lea,
3451 if (src_abi_size > 4) tmp.to64() else tmp.to32(),3842 if (src_abi_size > 4) tmp.to64() else tmp.to32(),
3452 Memory.sib(.qword, .{3843 Memory.sib(.qword, .{
3453 .base = dst.to64(),3844 .base = .{ .reg = dst.to64() },
3454 .scale_index = .{ .index = tmp.to64(), .scale = 1 << 2 },3845 .scale_index = .{ .index = tmp.to64(), .scale = 1 << 2 },
3455 }),3846 }),
3456 );3847 );
...@@ -3473,7 +3864,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {...@@ -3473,7 +3864,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
3473 .lea,3864 .lea,
3474 if (src_abi_size > 4) dst.to64() else dst.to32(),3865 if (src_abi_size > 4) dst.to64() else dst.to32(),
3475 Memory.sib(.qword, .{3866 Memory.sib(.qword, .{
3476 .base = tmp.to64(),3867 .base = .{ .reg = tmp.to64() },
3477 .scale_index = .{ .index = dst.to64(), .scale = 1 << 1 },3868 .scale_index = .{ .index = dst.to64(), .scale = 1 << 1 },
3478 }),3869 }),
3479 );3870 );
...@@ -3506,6 +3897,17 @@ fn reuseOperand(...@@ -3506,6 +3897,17 @@ fn reuseOperand(
3506 operand: Air.Inst.Ref,3897 operand: Air.Inst.Ref,
3507 op_index: Liveness.OperandInt,3898 op_index: Liveness.OperandInt,
3508 mcv: MCValue,3899 mcv: MCValue,
3900) bool {
3901 return self.reuseOperandAdvanced(inst, operand, op_index, mcv, inst);
3902}
3903
3904fn reuseOperandAdvanced(
3905 self: *Self,
3906 inst: Air.Inst.Index,
3907 operand: Air.Inst.Ref,
3908 op_index: Liveness.OperandInt,
3909 mcv: MCValue,
3910 tracked_inst: Air.Inst.Index,
3509) bool {3911) bool {
3510 if (!self.liveness.operandDies(inst, op_index))3912 if (!self.liveness.operandDies(inst, op_index))
3511 return false;3913 return false;
...@@ -3516,25 +3918,23 @@ fn reuseOperand(...@@ -3516,25 +3918,23 @@ fn reuseOperand(
3516 // new instruction.3918 // new instruction.
3517 if (!self.register_manager.isRegFree(reg)) {3919 if (!self.register_manager.isRegFree(reg)) {
3518 if (RegisterManager.indexOfRegIntoTracked(reg)) |index| {3920 if (RegisterManager.indexOfRegIntoTracked(reg)) |index| {
3519 self.register_manager.registers[index] = inst;3921 self.register_manager.registers[index] = tracked_inst;
3520 }3922 }
3521 }3923 }
3522 log.debug("%{d} => {} (reused)", .{ inst, reg });
3523 },
3524 .stack_offset => |off| {
3525 log.debug("%{d} => stack offset {d} (reused)", .{ inst, off });
3526 },3924 },
3925 .load_frame => |frame_addr| if (frame_addr.index.isNamed()) return false,
3527 else => return false,3926 else => return false,
3528 }3927 }
35293928
3530 // Prevent the operand deaths processing code from deallocating it.3929 // Prevent the operand deaths processing code from deallocating it.
3531 self.liveness.clearOperandDeath(inst, op_index);3930 self.liveness.clearOperandDeath(inst, op_index);
3532 self.getResolvedInstValue(Air.refToIndex(operand).?).reuse(self);3931 const op_inst = Air.refToIndex(operand).?;
3932 self.getResolvedInstValue(op_inst).reuse(self, tracked_inst, op_inst);
35333933
3534 return true;3934 return true;
3535}3935}
35363936
3537fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_mcv: MCValue, ptr_ty: Type) InnerError!void {3937fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void {
3538 const ptr_info = ptr_ty.ptrInfo().data;3938 const ptr_info = ptr_ty.ptrInfo().data;
35393939
3540 const val_ty = ptr_info.pointee_type;3940 const val_ty = ptr_info.pointee_type;
...@@ -3564,10 +3964,14 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_mcv: MCValue, ptr_ty: Type) Inn...@@ -3564,10 +3964,14 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_mcv: MCValue, ptr_ty: Type) Inn
3564 if (val_bit_off < val_extra_bits) val_abi_size else val_abi_size * 2;3964 if (val_bit_off < val_extra_bits) val_abi_size else val_abi_size * 2;
3565 if (load_abi_size <= 8) {3965 if (load_abi_size <= 8) {
3566 const load_reg = registerAlias(dst_reg, load_abi_size);3966 const load_reg = registerAlias(dst_reg, load_abi_size);
3567 try self.asmRegisterMemory(.mov, load_reg, Memory.sib(3967 try self.asmRegisterMemory(
3568 Memory.PtrSize.fromSize(load_abi_size),3968 .mov,
3569 .{ .base = ptr_reg, .disp = val_byte_off },3969 load_reg,
3570 ));3970 Memory.sib(Memory.PtrSize.fromSize(load_abi_size), .{
3971 .base = .{ .reg = ptr_reg },
3972 .disp = val_byte_off,
3973 }),
3974 );
3571 try self.asmRegisterImmediate(.shr, load_reg, Immediate.u(val_bit_off));3975 try self.asmRegisterImmediate(.shr, load_reg, Immediate.u(val_bit_off));
3572 } else {3976 } else {
3573 const tmp_reg = registerAlias(try self.register_manager.allocReg(null, gp), val_abi_size);3977 const tmp_reg = registerAlias(try self.register_manager.allocReg(null, gp), val_abi_size);
...@@ -3575,85 +3979,60 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_mcv: MCValue, ptr_ty: Type) Inn...@@ -3575,85 +3979,60 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_mcv: MCValue, ptr_ty: Type) Inn
3575 defer self.register_manager.unlockReg(tmp_lock);3979 defer self.register_manager.unlockReg(tmp_lock);
35763980
3577 const dst_alias = registerAlias(dst_reg, val_abi_size);3981 const dst_alias = registerAlias(dst_reg, val_abi_size);
3578 try self.asmRegisterMemory(.mov, dst_alias, Memory.sib(3982 try self.asmRegisterMemory(
3579 Memory.PtrSize.fromSize(val_abi_size),3983 .mov,
3580 .{ .base = ptr_reg, .disp = val_byte_off },3984 dst_alias,
3581 ));3985 Memory.sib(Memory.PtrSize.fromSize(val_abi_size), .{
3582 try self.asmRegisterMemory(.mov, tmp_reg, Memory.sib(3986 .base = .{ .reg = ptr_reg },
3583 Memory.PtrSize.fromSize(val_abi_size),3987 .disp = val_byte_off,
3584 .{ .base = ptr_reg, .disp = val_byte_off + 1 },3988 }),
3585 ));3989 );
3990 try self.asmRegisterMemory(
3991 .mov,
3992 tmp_reg,
3993 Memory.sib(Memory.PtrSize.fromSize(val_abi_size), .{
3994 .base = .{ .reg = ptr_reg },
3995 .disp = val_byte_off + 1,
3996 }),
3997 );
3586 try self.asmRegisterRegisterImmediate(.shrd, dst_alias, tmp_reg, Immediate.u(val_bit_off));3998 try self.asmRegisterRegisterImmediate(.shrd, dst_alias, tmp_reg, Immediate.u(val_bit_off));
3587 }3999 }
35884000
3589 if (val_extra_bits > 0) try self.truncateRegister(val_ty, dst_reg);4001 if (val_extra_bits > 0) try self.truncateRegister(val_ty, dst_reg);
3590 try self.setRegOrMem(val_ty, dst_mcv, .{ .register = dst_reg });4002 try self.genCopy(val_ty, dst_mcv, .{ .register = dst_reg });
3591}4003}
3592
3593fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!void {
3594 const elem_ty = ptr_ty.elemType();
3595 const abi_size = @intCast(u32, elem_ty.abiSize(self.target.*));
3596 switch (ptr) {
3597 .none => unreachable,
3598 .undef => unreachable,
3599 .unreach => unreachable,
3600 .dead => unreachable,
3601 .eflags => unreachable,
3602 .register_overflow => unreachable,
3603 .immediate => |imm| {
3604 try self.setRegOrMem(elem_ty, dst_mcv, .{ .memory = imm });
3605 },
3606 .stack_offset => {
3607 const reg = try self.copyToTmpRegister(ptr_ty, ptr);
3608 try self.load(dst_mcv, .{ .register = reg }, ptr_ty);
3609 },
3610 .ptr_stack_offset => |off| {
3611 try self.setRegOrMem(elem_ty, dst_mcv, .{ .stack_offset = off });
3612 },
3613 .register => |reg| {
3614 const reg_lock = self.register_manager.lockReg(reg);
3615 defer if (reg_lock) |lock| self.register_manager.unlockReg(lock);
36164004
3617 switch (dst_mcv) {4005fn load(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void {
3618 .dead => unreachable,4006 const dst_ty = ptr_ty.childType();
3619 .undef => unreachable,4007 switch (ptr_mcv) {
3620 .eflags => unreachable,4008 .none,
3621 .register => |dst_reg| {4009 .unreach,
3622 try self.asmRegisterMemory(4010 .dead,
3623 .mov,4011 .undef,
3624 registerAlias(dst_reg, abi_size),4012 .eflags,
3625 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg }),4013 .register_overflow,
3626 );4014 .reserved_frame,
3627 },4015 => unreachable, // not a valid pointer
3628 .stack_offset => |off| {4016 .immediate,
3629 if (abi_size <= 8) {4017 .register,
3630 const tmp_reg = try self.register_manager.allocReg(null, gp);4018 .register_offset,
3631 try self.load(.{ .register = tmp_reg }, ptr, ptr_ty);4019 .lea_direct,
3632 return self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg }, .{});4020 .lea_got,
3633 }4021 .lea_tlv,
4022 .lea_frame,
4023 => try self.genCopy(dst_ty, dst_mcv, ptr_mcv.deref()),
4024 .memory,
4025 .indirect,
4026 .load_direct,
4027 .load_got,
4028 .load_tlv,
4029 .load_frame,
4030 => {
4031 const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv);
4032 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
4033 defer self.register_manager.unlockReg(addr_lock);
36344034
3635 try self.genInlineMemcpy(4035 try self.genCopy(dst_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } });
3636 .{ .ptr_stack_offset = off },
3637 ptr,
3638 .{ .immediate = abi_size },
3639 .{},
3640 );
3641 },
3642 else => return self.fail("TODO implement loading from register into {}", .{dst_mcv}),
3643 }
3644 },
3645 .load_direct => |sym_index| {
3646 const addr_reg = try self.copyToTmpRegister(Type.usize, .{ .lea_direct = sym_index });
3647 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
3648 defer self.register_manager.unlockReg(addr_reg_lock);
3649 // Load the pointer, which is stored in memory
3650 try self.asmRegisterMemory(.mov, addr_reg, Memory.sib(.qword, .{ .base = addr_reg }));
3651 try self.load(dst_mcv, .{ .register = addr_reg }, ptr_ty);
3652 },
3653 .load_tlv => |sym_index| try self.load(dst_mcv, .{ .lea_tlv = sym_index }, ptr_ty),
3654 .memory, .load_got, .lea_direct, .lea_tlv => {
3655 const reg = try self.copyToTmpRegister(ptr_ty, ptr);
3656 try self.load(dst_mcv, .{ .register = reg }, ptr_ty);
3657 },4036 },
3658 }4037 }
3659}4038}
...@@ -3669,53 +4048,48 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {...@@ -3669,53 +4048,48 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
3669 const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx });4048 const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx });
3670 defer for (reg_locks) |lock| self.register_manager.unlockReg(lock);4049 defer for (reg_locks) |lock| self.register_manager.unlockReg(lock);
36714050
3672 const ptr = try self.resolveInst(ty_op.operand);4051 const ptr_mcv = try self.resolveInst(ty_op.operand);
3673 const dst_mcv: MCValue = if (elem_size <= 8 and self.reuseOperand(inst, ty_op.operand, 0, ptr))4052 const dst_mcv = if (elem_size <= 8 and self.reuseOperand(inst, ty_op.operand, 0, ptr_mcv))
3674 // The MCValue that holds the pointer can be re-used as the value.4053 // The MCValue that holds the pointer can be re-used as the value.
3675 ptr4054 ptr_mcv
3676 else4055 else
3677 try self.allocRegOrMem(inst, true);4056 try self.allocRegOrMem(inst, true);
36784057
3679 const ptr_ty = self.air.typeOf(ty_op.operand);4058 const ptr_ty = self.air.typeOf(ty_op.operand);
3680 if (ptr_ty.ptrInfo().data.host_size > 0) {4059 if (ptr_ty.ptrInfo().data.host_size > 0) {
3681 try self.packedLoad(dst_mcv, ptr, ptr_ty);4060 try self.packedLoad(dst_mcv, ptr_ty, ptr_mcv);
3682 } else {4061 } else {
3683 try self.load(dst_mcv, ptr, ptr_ty);4062 try self.load(dst_mcv, ptr_ty, ptr_mcv);
3684 }4063 }
3685 break :result dst_mcv;4064 break :result dst_mcv;
3686 };4065 };
3687 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });4066 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3688}4067}
36894068
3690fn packedStore(4069fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerError!void {
3691 self: *Self,
3692 ptr_mcv: MCValue,
3693 val_mcv: MCValue,
3694 ptr_ty: Type,
3695 val_ty: Type,
3696) InnerError!void {
3697 const ptr_info = ptr_ty.ptrInfo().data;4070 const ptr_info = ptr_ty.ptrInfo().data;
4071 const src_ty = ptr_ty.childType();
36984072
3699 const limb_abi_size = @min(ptr_info.host_size, 8);4073 const limb_abi_size = @min(ptr_info.host_size, 8);
3700 const limb_abi_bits = limb_abi_size * 8;4074 const limb_abi_bits = limb_abi_size * 8;
37014075
3702 const val_bit_size = val_ty.bitSize(self.target.*);4076 const src_bit_size = src_ty.bitSize(self.target.*);
3703 const val_byte_off = @intCast(i32, ptr_info.bit_offset / limb_abi_bits * limb_abi_size);4077 const src_byte_off = @intCast(i32, ptr_info.bit_offset / limb_abi_bits * limb_abi_size);
3704 const val_bit_off = ptr_info.bit_offset % limb_abi_bits;4078 const src_bit_off = ptr_info.bit_offset % limb_abi_bits;
37054079
3706 const ptr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv);4080 const ptr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv);
3707 const ptr_lock = self.register_manager.lockRegAssumeUnused(ptr_reg);4081 const ptr_lock = self.register_manager.lockRegAssumeUnused(ptr_reg);
3708 defer self.register_manager.unlockReg(ptr_lock);4082 defer self.register_manager.unlockReg(ptr_lock);
37094083
3710 var limb_i: u16 = 0;4084 var limb_i: u16 = 0;
3711 while (limb_i * limb_abi_bits < val_bit_off + val_bit_size) : (limb_i += 1) {4085 while (limb_i * limb_abi_bits < src_bit_off + src_bit_size) : (limb_i += 1) {
3712 const part_bit_off = if (limb_i == 0) val_bit_off else 0;4086 const part_bit_off = if (limb_i == 0) src_bit_off else 0;
3713 const part_bit_size =4087 const part_bit_size =
3714 @min(val_bit_off + val_bit_size - limb_i * limb_abi_bits, limb_abi_bits) - part_bit_off;4088 @min(src_bit_off + src_bit_size - limb_i * limb_abi_bits, limb_abi_bits) - part_bit_off;
3715 const limb_mem = Memory.sib(4089 const limb_mem = Memory.sib(Memory.PtrSize.fromSize(limb_abi_size), .{
3716 Memory.PtrSize.fromSize(limb_abi_size),4090 .base = .{ .reg = ptr_reg },
3717 .{ .base = ptr_reg, .disp = val_byte_off + limb_i * limb_abi_bits },4091 .disp = src_byte_off + limb_i * limb_abi_bits,
3718 );4092 });
37194093
3720 const part_mask = (@as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - part_bit_size)) <<4094 const part_mask = (@as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - part_bit_size)) <<
3721 @intCast(u6, part_bit_off);4095 @intCast(u6, part_bit_off);
...@@ -3731,211 +4105,62 @@ fn packedStore(...@@ -3731,211 +4105,62 @@ fn packedStore(
3731 try self.asmMemoryRegister(.@"and", limb_mem, part_mask_reg);4105 try self.asmMemoryRegister(.@"and", limb_mem, part_mask_reg);
3732 }4106 }
37334107
3734 if (val_bit_size <= 64) {4108 if (src_bit_size <= 64) {
3735 const tmp_reg = try self.register_manager.allocReg(null, gp);4109 const tmp_reg = try self.register_manager.allocReg(null, gp);
3736 const tmp_mcv = MCValue{ .register = tmp_reg };4110 const tmp_mcv = MCValue{ .register = tmp_reg };
3737 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);4111 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3738 defer self.register_manager.unlockReg(tmp_lock);4112 defer self.register_manager.unlockReg(tmp_lock);
37394113
3740 try self.genSetReg(val_ty, tmp_reg, val_mcv);4114 try self.genSetReg(tmp_reg, src_ty, src_mcv);
3741 switch (limb_i) {4115 switch (limb_i) {
3742 0 => try self.genShiftBinOpMir(.shl, val_ty, tmp_mcv, .{ .immediate = val_bit_off }),4116 0 => try self.genShiftBinOpMir(.shl, src_ty, tmp_mcv, .{ .immediate = src_bit_off }),
3743 1 => try self.genShiftBinOpMir(.shr, val_ty, tmp_mcv, .{4117 1 => try self.genShiftBinOpMir(.shr, src_ty, tmp_mcv, .{
3744 .immediate = limb_abi_bits - val_bit_off,4118 .immediate = limb_abi_bits - src_bit_off,
3745 }),4119 }),
3746 else => unreachable,4120 else => unreachable,
3747 }4121 }
3748 try self.genBinOpMir(.@"and", val_ty, tmp_mcv, .{ .immediate = part_mask });4122 try self.genBinOpMir(.@"and", src_ty, tmp_mcv, .{ .immediate = part_mask });
3749 try self.asmMemoryRegister(.@"or", limb_mem, registerAlias(tmp_reg, limb_abi_size));4123 try self.asmMemoryRegister(.@"or", limb_mem, registerAlias(tmp_reg, limb_abi_size));
3750 } else return self.fail("TODO: implement packed store of {}", .{4124 } else return self.fail("TODO: implement packed store of {}", .{
3751 val_ty.fmt(self.bin_file.options.module.?),4125 src_ty.fmt(self.bin_file.options.module.?),
3752 });4126 });
3753 }4127 }
3754}4128}
37554129
3756fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type) InnerError!void {4130fn store(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerError!void {
3757 const abi_size = @intCast(u32, value_ty.abiSize(self.target.*));4131 const src_ty = ptr_ty.childType();
3758 switch (ptr) {4132 switch (ptr_mcv) {
3759 .none => unreachable,4133 .none,
3760 .undef => unreachable,4134 .unreach,
3761 .unreach => unreachable,4135 .dead,
3762 .dead => unreachable,4136 .undef,
3763 .eflags => unreachable,4137 .eflags,
3764 .register_overflow => unreachable,4138 .register_overflow,
3765 .immediate, .stack_offset => {4139 .reserved_frame,
3766 const reg = try self.copyToTmpRegister(ptr_ty, ptr);4140 => unreachable, // not a valid pointer
3767 try self.store(.{ .register = reg }, value, ptr_ty, value_ty);4141 .immediate,
3768 },4142 .register,
3769 .ptr_stack_offset => |off| {4143 .register_offset,
3770 try self.genSetStack(value_ty, off, value, .{});4144 .lea_direct,
3771 },4145 .lea_got,
3772 .register => |reg| {4146 .lea_tlv,
3773 const reg_lock = self.register_manager.lockReg(reg);4147 .lea_frame,
3774 defer if (reg_lock) |lock| self.register_manager.unlockReg(lock);4148 => try self.genCopy(src_ty, ptr_mcv.deref(), src_mcv),
37754149 .memory,
3776 switch (value) {4150 .indirect,
3777 .none => unreachable,4151 .load_direct,
3778 .dead => unreachable,4152 .load_got,
3779 .unreach => unreachable,4153 .load_tlv,
3780 .eflags => |cc| try self.asmSetccMemory(4154 .load_frame,
3781 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),4155 => {
3782 cc,4156 const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv);
3783 ),4157 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
3784 .undef => if (self.wantSafety()) switch (abi_size) {4158 defer self.register_manager.unlockReg(addr_lock);
3785 1 => try self.store(ptr, .{ .immediate = 0xaa }, ptr_ty, value_ty),
3786 2 => try self.store(ptr, .{ .immediate = 0xaaaa }, ptr_ty, value_ty),
3787 4 => try self.store(ptr, .{ .immediate = 0xaaaaaaaa }, ptr_ty, value_ty),
3788 8 => try self.store(ptr, .{ .immediate = 0xaaaaaaaaaaaaaaaa }, ptr_ty, value_ty),
3789 else => try self.genInlineMemset(
3790 ptr,
3791 .{ .immediate = 0xaa },
3792 .{ .immediate = abi_size },
3793 .{},
3794 ),
3795 },
3796 .immediate => |imm| switch (self.regBitSize(value_ty)) {
3797 8 => try self.asmMemoryImmediate(
3798 .mov,
3799 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
3800 if (math.cast(i8, @bitCast(i64, imm))) |small|
3801 Immediate.s(small)
3802 else
3803 Immediate.u(@intCast(u8, imm)),
3804 ),
3805 16 => try self.asmMemoryImmediate(
3806 .mov,
3807 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
3808 if (math.cast(i16, @bitCast(i64, imm))) |small|
3809 Immediate.s(small)
3810 else
3811 Immediate.u(@intCast(u16, imm)),
3812 ),
3813 32 => try self.asmMemoryImmediate(
3814 .mov,
3815 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
3816 if (math.cast(i32, @bitCast(i64, imm))) |small|
3817 Immediate.s(small)
3818 else
3819 Immediate.u(@intCast(u32, imm)),
3820 ),
3821 64 => if (math.cast(i32, @bitCast(i64, imm))) |small|
3822 try self.asmMemoryImmediate(
3823 .mov,
3824 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
3825 Immediate.s(small),
3826 )
3827 else
3828 try self.asmMemoryRegister(
3829 .mov,
3830 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
3831 registerAlias(try self.copyToTmpRegister(value_ty, value), abi_size),
3832 ),
3833 else => unreachable,
3834 },
3835 .register => |src_reg| try self.genInlineMemcpyRegisterRegister(
3836 value_ty,
3837 reg,
3838 src_reg,
3839 0,
3840 ),
3841 .register_overflow => |ro| {
3842 const ro_reg_lock = self.register_manager.lockReg(ro.reg);
3843 defer if (ro_reg_lock) |lock| self.register_manager.unlockReg(lock);
3844
3845 const wrapped_ty = value_ty.structFieldType(0);
3846 try self.genInlineMemcpyRegisterRegister(wrapped_ty, reg, ro.reg, 0);
3847
3848 const overflow_bit_ty = value_ty.structFieldType(1);
3849 const overflow_bit_offset = value_ty.structFieldOffset(1, self.target.*);
3850 const tmp_reg = try self.register_manager.allocReg(null, gp);
3851 try self.asmSetccRegister(tmp_reg.to8(), ro.eflags);
3852 try self.genInlineMemcpyRegisterRegister(
3853 overflow_bit_ty,
3854 reg,
3855 tmp_reg,
3856 -@intCast(i32, overflow_bit_offset),
3857 );
3858 },
3859 .memory, .load_tlv, .load_direct => if (abi_size <= 8) {
3860 const tmp_reg = try self.copyToTmpRegister(value_ty, value);
3861 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3862 defer self.register_manager.unlockReg(tmp_lock);
3863
3864 try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
3865 } else {
3866 const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
3867 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
3868 defer self.register_manager.unlockReg(addr_lock);
3869
3870 try self.genSetReg(Type.usize, addr_reg, switch (value) {
3871 .memory => |addr| .{ .immediate = addr },
3872 .load_direct => |sym_index| .{ .lea_direct = sym_index },
3873 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
3874 else => unreachable,
3875 });
3876 try self.genInlineMemcpy(
3877 ptr,
3878 .{ .register = addr_reg },
3879 .{ .immediate = abi_size },
3880 .{},
3881 );
3882 },
3883 .stack_offset => |off| if (abi_size <= 8) {
3884 const tmp_reg = try self.copyToTmpRegister(value_ty, value);
3885 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3886 defer self.register_manager.unlockReg(tmp_lock);
3887
3888 try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
3889 } else try self.genInlineMemcpy(
3890 ptr,
3891 .{ .ptr_stack_offset = off },
3892 .{ .immediate = abi_size },
3893 .{},
3894 ),
3895 .ptr_stack_offset, .load_got, .lea_direct, .lea_tlv => {
3896 const tmp_reg = try self.copyToTmpRegister(value_ty, value);
3897 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3898 defer self.register_manager.unlockReg(tmp_lock);
38994159
3900 try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);4160 try self.genCopy(src_ty, .{ .indirect = .{ .reg = addr_reg } }, src_mcv);
3901 },
3902 }
3903 },4161 },
3904 .memory, .load_direct, .load_tlv => {4162 }
3905 const value_lock: ?RegisterLock = switch (value) {4163}
3906 .register => |reg| self.register_manager.lockReg(reg),
3907 else => null,
3908 };
3909 defer if (value_lock) |lock| self.register_manager.unlockReg(lock);
3910
3911 const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
3912 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
3913 defer self.register_manager.unlockReg(addr_reg_lock);
3914
3915 switch (ptr) {
3916 .memory => |addr| try self.genSetReg(ptr_ty, addr_reg, .{ .immediate = addr }),
3917 .load_direct => |sym_index| try self.genSetReg(ptr_ty, addr_reg, .{ .lea_direct = sym_index }),
3918 .load_tlv => |sym_index| try self.genSetReg(ptr_ty, addr_reg, .{ .lea_tlv = sym_index }),
3919 else => unreachable,
3920 }
3921 if (ptr != .load_tlv) {
3922 // Load the pointer, which is stored in memory
3923 try self.asmRegisterMemory(.mov, addr_reg, Memory.sib(.qword, .{ .base = addr_reg }));
3924 }
3925
3926 const new_ptr = MCValue{ .register = addr_reg };
3927 try self.store(new_ptr, value, ptr_ty, value_ty);
3928 },
3929 .load_got, .lea_direct, .lea_tlv => {
3930 const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr);
3931 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
3932 defer self.register_manager.unlockReg(addr_reg_lock);
3933
3934 const new_ptr = MCValue{ .register = addr_reg };
3935 try self.store(new_ptr, value, ptr_ty, value_ty);
3936 },
3937 }
3938}
39394164
3940fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {4165fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
3941 if (safety) {4166 if (safety) {
...@@ -3944,15 +4169,13 @@ fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {...@@ -3944,15 +4169,13 @@ fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
3944 // TODO if the value is undef, don't lower this instruction4169 // TODO if the value is undef, don't lower this instruction
3945 }4170 }
3946 const bin_op = self.air.instructions.items(.data)[inst].bin_op;4171 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
3947 const ptr = try self.resolveInst(bin_op.lhs);4172 const ptr_mcv = try self.resolveInst(bin_op.lhs);
3948 const ptr_ty = self.air.typeOf(bin_op.lhs);4173 const ptr_ty = self.air.typeOf(bin_op.lhs);
3949 const value = try self.resolveInst(bin_op.rhs);4174 const src_mcv = try self.resolveInst(bin_op.rhs);
3950 const value_ty = self.air.typeOf(bin_op.rhs);
3951 log.debug("airStore(%{d}): {} <- {}", .{ inst, ptr, value });
3952 if (ptr_ty.ptrInfo().data.host_size > 0) {4175 if (ptr_ty.ptrInfo().data.host_size > 0) {
3953 try self.packedStore(ptr, value, ptr_ty, value_ty);4176 try self.packedStore(ptr_ty, ptr_mcv, src_mcv);
3954 } else {4177 } else {
3955 try self.store(ptr, value, ptr_ty, value_ty);4178 try self.store(ptr_ty, ptr_mcv, src_mcv);
3956 }4179 }
3957 return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });4180 return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });
3958}4181}
...@@ -3971,67 +4194,65 @@ fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, index: u8) !void {...@@ -3971,67 +4194,65 @@ fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, index: u8) !void {
3971}4194}
39724195
3973fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32) !MCValue {4196fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32) !MCValue {
4197 const ptr_field_ty = self.air.typeOfIndex(inst);
3974 const mcv = try self.resolveInst(operand);4198 const mcv = try self.resolveInst(operand);
3975 const ptr_ty = self.air.typeOf(operand);4199 const ptr_container_ty = self.air.typeOf(operand);
3976 const container_ty = ptr_ty.childType();4200 const container_ty = ptr_container_ty.childType();
3977 const field_offset = switch (container_ty.containerLayout()) {4201 const field_offset = switch (container_ty.containerLayout()) {
3978 .Auto, .Extern => @intCast(u32, container_ty.structFieldOffset(index, self.target.*)),4202 .Auto, .Extern => @intCast(u32, container_ty.structFieldOffset(index, self.target.*)),
3979 .Packed => if (container_ty.zigTypeTag() == .Struct and ptr_ty.ptrInfo().data.host_size == 0)4203 .Packed => if (container_ty.zigTypeTag() == .Struct and
4204 ptr_field_ty.ptrInfo().data.host_size == 0)
3980 container_ty.packedStructFieldByteOffset(index, self.target.*)4205 container_ty.packedStructFieldByteOffset(index, self.target.*)
3981 else4206 else
3982 0,4207 0,
3983 };4208 };
39844209
3985 const dst_mcv: MCValue = result: {4210 const result: MCValue = result: {
3986 switch (mcv) {4211 switch (mcv) {
3987 .stack_offset, .lea_tlv, .load_tlv => {4212 .load_frame, .lea_tlv, .load_tlv => {
3988 const offset_reg = try self.copyToTmpRegister(ptr_ty, .{4213 const offset_reg = try self.copyToTmpRegister(Type.usize, .{
3989 .immediate = field_offset,4214 .immediate = field_offset,
3990 });4215 });
3991 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);4216 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
3992 defer self.register_manager.unlockReg(offset_reg_lock);4217 defer self.register_manager.unlockReg(offset_reg_lock);
39934218
3994 const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, switch (mcv) {4219 const dst_mcv = try self.copyToRegisterWithInstTracking(inst, Type.usize, switch (mcv) {
3995 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },4220 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
3996 else => mcv,4221 else => mcv,
3997 });4222 });
3998 try self.genBinOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg });4223 try self.genBinOpMir(.add, Type.usize, dst_mcv, .{ .register = offset_reg });
3999 break :result dst_mcv;4224 break :result dst_mcv;
4000 },4225 },
4001 .ptr_stack_offset => |off| {4226 .indirect => |reg_off| break :result .{ .indirect = .{
4002 const ptr_stack_offset = off - @intCast(i32, field_offset);4227 .reg = reg_off.reg,
4003 break :result MCValue{ .ptr_stack_offset = ptr_stack_offset };4228 .off = reg_off.off + @intCast(i32, field_offset),
4004 },4229 } },
4005 .register => |reg| {4230 .lea_frame => |frame_addr| break :result .{ .lea_frame = .{
4006 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);4231 .index = frame_addr.index,
4007 defer self.register_manager.unlockReg(reg_lock);4232 .off = frame_addr.off + @intCast(i32, field_offset),
40084233 } },
4009 const offset_reg = try self.copyToTmpRegister(ptr_ty, .{4234 .register, .register_offset => {
4010 .immediate = field_offset,4235 const src_reg = mcv.getReg().?;
4011 });4236 const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
4012 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);4237 defer self.register_manager.unlockReg(src_lock);
4013 defer self.register_manager.unlockReg(offset_reg_lock);
4014
4015 const can_reuse_operand = self.reuseOperand(inst, operand, 0, mcv);
4016 const result_reg: Register = blk: {
4017 if (can_reuse_operand) {
4018 break :blk reg;
4019 } else {
4020 const result_reg = try self.register_manager.allocReg(inst, gp);
4021 try self.genSetReg(ptr_ty, result_reg, mcv);
4022 break :blk result_reg;
4023 }
4024 };
4025 const result_reg_lock = self.register_manager.lockReg(result_reg);
4026 defer if (result_reg_lock) |lock| self.register_manager.unlockReg(lock);
40274238
4028 try self.genBinOpMir(.add, ptr_ty, .{ .register = result_reg }, .{ .register = offset_reg });4239 const dst_mcv: MCValue = if (self.reuseOperand(inst, operand, 0, mcv))
4029 break :result MCValue{ .register = result_reg };4240 mcv
4241 else
4242 .{ .register = try self.copyToTmpRegister(ptr_field_ty, mcv) };
4243 break :result .{ .register_offset = .{
4244 .reg = dst_mcv.getReg().?,
4245 .off = switch (dst_mcv) {
4246 .register => 0,
4247 .register_offset => |reg_off| reg_off.off,
4248 else => unreachable,
4249 } + @intCast(i32, field_offset),
4250 } };
4030 },4251 },
4031 else => return self.fail("TODO implement fieldPtr for {}", .{mcv}),4252 else => return self.fail("TODO implement fieldPtr for {}", .{mcv}),
4032 }4253 }
4033 };4254 };
4034 return dst_mcv;4255 return result;
4035}4256}
40364257
4037fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {4258fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
...@@ -4055,7 +4276,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -4055,7 +4276,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
4055 };4276 };
40564277
4057 switch (src_mcv) {4278 switch (src_mcv) {
4058 .stack_offset => |src_off| {4279 .load_frame => |frame_addr| {
4059 const field_abi_size = @intCast(u32, field_ty.abiSize(self.target.*));4280 const field_abi_size = @intCast(u32, field_ty.abiSize(self.target.*));
4060 const limb_abi_size = @min(field_abi_size, 8);4281 const limb_abi_size = @min(field_abi_size, 8);
4061 const limb_abi_bits = limb_abi_size * 8;4282 const limb_abi_bits = limb_abi_size * 8;
...@@ -4063,11 +4284,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -4063,11 +4284,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
4063 const field_bit_off = field_off % limb_abi_bits;4284 const field_bit_off = field_off % limb_abi_bits;
40644285
4065 if (field_bit_off == 0) {4286 if (field_bit_off == 0) {
4066 const off_mcv = MCValue{ .stack_offset = src_off - field_byte_off };4287 const off_mcv = MCValue{ .load_frame = .{
4288 .index = frame_addr.index,
4289 .off = frame_addr.off + field_byte_off,
4290 } };
4067 if (self.reuseOperand(inst, operand, 0, src_mcv)) break :result off_mcv;4291 if (self.reuseOperand(inst, operand, 0, src_mcv)) break :result off_mcv;
40684292
4069 const dst_mcv = try self.allocRegOrMem(inst, true);4293 const dst_mcv = try self.allocRegOrMem(inst, true);
4070 try self.setRegOrMem(field_ty, dst_mcv, off_mcv);4294 try self.genCopy(field_ty, dst_mcv, off_mcv);
4071 break :result dst_mcv;4295 break :result dst_mcv;
4072 }4296 }
40734297
...@@ -4081,10 +4305,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -4081,10 +4305,14 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
4081 if (field_bit_off < field_extra_bits) field_abi_size else field_abi_size * 2;4305 if (field_bit_off < field_extra_bits) field_abi_size else field_abi_size * 2;
4082 if (load_abi_size <= 8) {4306 if (load_abi_size <= 8) {
4083 const load_reg = registerAlias(dst_reg, load_abi_size);4307 const load_reg = registerAlias(dst_reg, load_abi_size);
4084 try self.asmRegisterMemory(.mov, load_reg, Memory.sib(4308 try self.asmRegisterMemory(
4085 Memory.PtrSize.fromSize(load_abi_size),4309 .mov,
4086 .{ .base = .rbp, .disp = field_byte_off - src_off },4310 load_reg,
4087 ));4311 Memory.sib(Memory.PtrSize.fromSize(load_abi_size), .{
4312 .base = .{ .frame = frame_addr.index },
4313 .disp = frame_addr.off + field_byte_off,
4314 }),
4315 );
4088 try self.asmRegisterImmediate(.shr, load_reg, Immediate.u(field_bit_off));4316 try self.asmRegisterImmediate(.shr, load_reg, Immediate.u(field_bit_off));
4089 } else {4317 } else {
4090 const tmp_reg = registerAlias(4318 const tmp_reg = registerAlias(
...@@ -4095,14 +4323,22 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -4095,14 +4323,22 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
4095 defer self.register_manager.unlockReg(tmp_lock);4323 defer self.register_manager.unlockReg(tmp_lock);
40964324
4097 const dst_alias = registerAlias(dst_reg, field_abi_size);4325 const dst_alias = registerAlias(dst_reg, field_abi_size);
4098 try self.asmRegisterMemory(.mov, dst_alias, Memory.sib(4326 try self.asmRegisterMemory(
4099 Memory.PtrSize.fromSize(field_abi_size),4327 .mov,
4100 .{ .base = .rbp, .disp = field_byte_off - src_off },4328 dst_alias,
4101 ));4329 Memory.sib(Memory.PtrSize.fromSize(field_abi_size), .{
4102 try self.asmRegisterMemory(.mov, tmp_reg, Memory.sib(4330 .base = .{ .frame = frame_addr.index },
4103 Memory.PtrSize.fromSize(field_abi_size),4331 .disp = frame_addr.off + field_byte_off,
4104 .{ .base = .rbp, .disp = field_byte_off + 1 - src_off },4332 }),
4105 ));4333 );
4334 try self.asmRegisterMemory(
4335 .mov,
4336 tmp_reg,
4337 Memory.sib(Memory.PtrSize.fromSize(field_abi_size), .{
4338 .base = .{ .frame = frame_addr.index },
4339 .disp = frame_addr.off + field_byte_off + @intCast(i32, limb_abi_size),
4340 }),
4341 );
4106 try self.asmRegisterRegisterImmediate(4342 try self.asmRegisterRegisterImmediate(
4107 .shrd,4343 .shrd,
4108 dst_alias,4344 dst_alias,
...@@ -4249,41 +4485,51 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:...@@ -4249,41 +4485,51 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
42494485
4250fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue) !void {4486fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue) !void {
4251 const abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));4487 const abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
4488 if (abi_size > 8) return self.fail("TODO implement {} for {}", .{
4489 mir_tag,
4490 dst_ty.fmt(self.bin_file.options.module.?),
4491 });
4252 switch (dst_mcv) {4492 switch (dst_mcv) {
4253 .none => unreachable,4493 .none,
4254 .undef => unreachable,4494 .unreach,
4255 .dead, .unreach, .immediate => unreachable,4495 .dead,
4256 .eflags => unreachable,4496 .undef,
4257 .register_overflow => unreachable,4497 .immediate,
4498 .register_offset,
4499 .eflags,
4500 .register_overflow,
4501 .lea_direct,
4502 .lea_got,
4503 .lea_tlv,
4504 .lea_frame,
4505 .reserved_frame,
4506 => unreachable, // unmodifiable destination
4258 .register => |dst_reg| try self.asmRegister(mir_tag, registerAlias(dst_reg, abi_size)),4507 .register => |dst_reg| try self.asmRegister(mir_tag, registerAlias(dst_reg, abi_size)),
4259 .stack_offset => |off| {4508 .memory, .load_got, .load_direct, .load_tlv => {
4260 if (abi_size > 8) {4509 const addr_reg = try self.register_manager.allocReg(null, gp);
4261 return self.fail("TODO implement {} for stack dst with large ABI", .{mir_tag});
4262 }
4263
4264 try self.asmMemory(mir_tag, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
4265 .base = .rbp,
4266 .disp = -off,
4267 }));
4268 },
4269 .ptr_stack_offset => unreachable,
4270 .load_got, .lea_direct, .lea_tlv => unreachable,
4271 .memory, .load_direct, .load_tlv => {
4272 const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
4273 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);4510 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
4274 defer self.register_manager.unlockReg(addr_reg_lock);4511 defer self.register_manager.unlockReg(addr_reg_lock);
42754512
4276 try self.genSetReg(Type.usize, addr_reg, switch (dst_mcv) {4513 try self.genSetReg(addr_reg, Type.usize, dst_mcv.address());
4277 .memory => |addr| .{ .immediate = addr },
4278 .load_direct => |sym_index| .{ .lea_direct = sym_index },
4279 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
4280 else => unreachable,
4281 });
4282 try self.asmMemory(4514 try self.asmMemory(
4283 mir_tag,4515 mir_tag,
4284 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = addr_reg }),4516 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .{ .reg = addr_reg } }),
4285 );4517 );
4286 },4518 },
4519 .indirect, .load_frame => try self.asmMemory(
4520 mir_tag,
4521 Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (dst_mcv) {
4522 .indirect => |reg_off| .{
4523 .base = .{ .reg = reg_off.reg },
4524 .disp = reg_off.off,
4525 },
4526 .load_frame => |frame_addr| .{
4527 .base = .{ .frame = frame_addr.index },
4528 .disp = frame_addr.off,
4529 },
4530 else => unreachable,
4531 }),
4532 ),
4287 }4533 }
4288}4534}
42894535
...@@ -4305,7 +4551,7 @@ fn genShiftBinOpMir(...@@ -4305,7 +4551,7 @@ fn genShiftBinOpMir(
4305 else => {},4551 else => {},
4306 }4552 }
4307 self.register_manager.getRegAssumeFree(.rcx, null);4553 self.register_manager.getRegAssumeFree(.rcx, null);
4308 try self.genSetReg(Type.u8, .rcx, shift_mcv);4554 try self.genSetReg(.cl, Type.u8, shift_mcv);
4309 break :rhs .{ .register = .rcx };4555 break :rhs .{ .register = .rcx };
4310 };4556 };
43114557
...@@ -4328,21 +4574,42 @@ fn genShiftBinOpMir(...@@ -4328,21 +4574,42 @@ fn genShiftBinOpMir(
4328 @tagName(rhs_mcv),4574 @tagName(rhs_mcv),
4329 }),4575 }),
4330 },4576 },
4331 .stack_offset => |lhs_off| switch (rhs_mcv) {4577 .memory, .indirect, .load_frame => {
4332 .immediate => |rhs_imm| try self.asmMemoryImmediate(4578 const lhs_mem = Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (lhs_mcv) {
4333 tag,4579 .memory => |addr| .{
4334 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -lhs_off }),4580 .base = .{ .reg = .ds },
4335 Immediate.u(rhs_imm),4581 .disp = math.cast(i32, @bitCast(i64, addr)) orelse
4336 ),4582 return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
4337 .register => |rhs_reg| try self.asmMemoryRegister(4583 @tagName(lhs_mcv),
4338 tag,4584 @tagName(rhs_mcv),
4339 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -lhs_off }),4585 }),
4340 registerAlias(rhs_reg, 1),4586 },
4341 ),4587 .indirect => |reg_off| .{
4342 else => return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{4588 .base = .{ .reg = reg_off.reg },
4343 @tagName(lhs_mcv),4589 .disp = reg_off.off,
4344 @tagName(rhs_mcv),4590 },
4345 }),4591 .load_frame => |frame_addr| .{
4592 .base = .{ .frame = frame_addr.index },
4593 .disp = frame_addr.off,
4594 },
4595 else => unreachable,
4596 });
4597 switch (rhs_mcv) {
4598 .immediate => |rhs_imm| try self.asmMemoryImmediate(
4599 tag,
4600 lhs_mem,
4601 Immediate.u(rhs_imm),
4602 ),
4603 .register => |rhs_reg| try self.asmMemoryRegister(
4604 tag,
4605 lhs_mem,
4606 registerAlias(rhs_reg, 1),
4607 ),
4608 else => return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
4609 @tagName(lhs_mcv),
4610 @tagName(rhs_mcv),
4611 }),
4612 }
4346 },4613 },
4347 else => return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{4614 else => return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{
4348 @tagName(lhs_mcv),4615 @tagName(lhs_mcv),
...@@ -4360,22 +4627,31 @@ fn genShiftBinOpMir(...@@ -4360,22 +4627,31 @@ fn genShiftBinOpMir(
4360 else => unreachable,4627 else => unreachable,
4361 };4628 };
4362 switch (lhs_mcv) {4629 switch (lhs_mcv) {
4363 .stack_offset => |dst_off| switch (rhs_mcv) {4630 .load_frame => |dst_frame_addr| switch (rhs_mcv) {
4364 .immediate => |rhs_imm| if (rhs_imm == 0) {} else if (rhs_imm < 64) {4631 .immediate => |rhs_imm| if (rhs_imm == 0) {} else if (rhs_imm < 64) {
4365 try self.asmRegisterMemory(4632 try self.asmRegisterMemory(
4366 .mov,4633 .mov,
4367 tmp_reg,4634 tmp_reg,
4368 Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),4635 Memory.sib(.qword, .{
4636 .base = .{ .frame = dst_frame_addr.index },
4637 .disp = dst_frame_addr.off + info.offsets[0],
4638 }),
4369 );4639 );
4370 try self.asmMemoryRegisterImmediate(4640 try self.asmMemoryRegisterImmediate(
4371 info.double_tag,4641 info.double_tag,
4372 Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[1] - dst_off }),4642 Memory.sib(.qword, .{
4643 .base = .{ .frame = dst_frame_addr.index },
4644 .disp = dst_frame_addr.off + info.offsets[1],
4645 }),
4373 tmp_reg,4646 tmp_reg,
4374 Immediate.u(rhs_imm),4647 Immediate.u(rhs_imm),
4375 );4648 );
4376 try self.asmMemoryImmediate(4649 try self.asmMemoryImmediate(
4377 tag,4650 tag,
4378 Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),4651 Memory.sib(.qword, .{
4652 .base = .{ .frame = dst_frame_addr.index },
4653 .disp = dst_frame_addr.off + info.offsets[0],
4654 }),
4379 Immediate.u(rhs_imm),4655 Immediate.u(rhs_imm),
4380 );4656 );
4381 } else {4657 } else {
...@@ -4383,14 +4659,20 @@ fn genShiftBinOpMir(...@@ -4383,14 +4659,20 @@ fn genShiftBinOpMir(
4383 try self.asmRegisterMemory(4659 try self.asmRegisterMemory(
4384 .mov,4660 .mov,
4385 tmp_reg,4661 tmp_reg,
4386 Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),4662 Memory.sib(.qword, .{
4663 .base = .{ .frame = dst_frame_addr.index },
4664 .disp = dst_frame_addr.off + info.offsets[0],
4665 }),
4387 );4666 );
4388 if (rhs_imm > 64) {4667 if (rhs_imm > 64) {
4389 try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(rhs_imm - 64));4668 try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(rhs_imm - 64));
4390 }4669 }
4391 try self.asmMemoryRegister(4670 try self.asmMemoryRegister(
4392 .mov,4671 .mov,
4393 Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[1] - dst_off }),4672 Memory.sib(.qword, .{
4673 .base = .{ .frame = dst_frame_addr.index },
4674 .disp = dst_frame_addr.off + info.offsets[1],
4675 }),
4394 tmp_reg,4676 tmp_reg,
4395 );4677 );
4396 switch (tag) {4678 switch (tag) {
...@@ -4398,13 +4680,19 @@ fn genShiftBinOpMir(...@@ -4398,13 +4680,19 @@ fn genShiftBinOpMir(
4398 try self.asmRegisterRegister(.xor, tmp_reg.to32(), tmp_reg.to32());4680 try self.asmRegisterRegister(.xor, tmp_reg.to32(), tmp_reg.to32());
4399 try self.asmMemoryRegister(4681 try self.asmMemoryRegister(
4400 .mov,4682 .mov,
4401 Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),4683 Memory.sib(.qword, .{
4684 .base = .{ .frame = dst_frame_addr.index },
4685 .disp = dst_frame_addr.off + info.offsets[0],
4686 }),
4402 tmp_reg,4687 tmp_reg,
4403 );4688 );
4404 },4689 },
4405 .sar => try self.asmMemoryImmediate(4690 .sar => try self.asmMemoryImmediate(
4406 tag,4691 tag,
4407 Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),4692 Memory.sib(.qword, .{
4693 .base = .{ .frame = dst_frame_addr.index },
4694 .disp = dst_frame_addr.off + info.offsets[0],
4695 }),
4408 Immediate.u(63),4696 Immediate.u(63),
4409 ),4697 ),
4410 else => unreachable,4698 else => unreachable,
...@@ -4419,16 +4707,22 @@ fn genShiftBinOpMir(...@@ -4419,16 +4707,22 @@ fn genShiftBinOpMir(
4419 const second_lock = self.register_manager.lockRegAssumeUnused(second_reg);4707 const second_lock = self.register_manager.lockRegAssumeUnused(second_reg);
4420 defer self.register_manager.unlockReg(second_lock);4708 defer self.register_manager.unlockReg(second_lock);
44214709
4422 try self.genSetReg(Type.u8, .cl, rhs_mcv);4710 try self.genSetReg(.cl, Type.u8, rhs_mcv);
4423 try self.asmRegisterMemory(4711 try self.asmRegisterMemory(
4424 .mov,4712 .mov,
4425 first_reg,4713 first_reg,
4426 Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),4714 Memory.sib(.qword, .{
4715 .base = .{ .frame = dst_frame_addr.index },
4716 .disp = dst_frame_addr.off + info.offsets[0],
4717 }),
4427 );4718 );
4428 try self.asmRegisterMemory(4719 try self.asmRegisterMemory(
4429 .mov,4720 .mov,
4430 second_reg,4721 second_reg,
4431 Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[1] - dst_off }),4722 Memory.sib(.qword, .{
4723 .base = .{ .frame = dst_frame_addr.index },
4724 .disp = dst_frame_addr.off + info.offsets[1],
4725 }),
4432 );4726 );
4433 switch (tag) {4727 switch (tag) {
4434 .shl, .sal, .shr => try self.asmRegisterRegister(4728 .shl, .sal, .shr => try self.asmRegisterRegister(
...@@ -4449,12 +4743,18 @@ fn genShiftBinOpMir(...@@ -4449,12 +4743,18 @@ fn genShiftBinOpMir(
4449 try self.asmCmovccRegisterRegister(first_reg, tmp_reg, .ae);4743 try self.asmCmovccRegisterRegister(first_reg, tmp_reg, .ae);
4450 try self.asmMemoryRegister(4744 try self.asmMemoryRegister(
4451 .mov,4745 .mov,
4452 Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[1] - dst_off }),4746 Memory.sib(.qword, .{
4747 .base = .{ .frame = dst_frame_addr.index },
4748 .disp = dst_frame_addr.off + info.offsets[1],
4749 }),
4453 second_reg,4750 second_reg,
4454 );4751 );
4455 try self.asmMemoryRegister(4752 try self.asmMemoryRegister(
4456 .mov,4753 .mov,
4457 Memory.sib(.qword, .{ .base = .rbp, .disp = info.offsets[0] - dst_off }),4754 Memory.sib(.qword, .{
4755 .base = .{ .frame = dst_frame_addr.index },
4756 .disp = dst_frame_addr.off + info.offsets[0],
4757 }),
4458 first_reg,4758 first_reg,
4459 );4759 );
4460 },4760 },
...@@ -4515,7 +4815,7 @@ fn genShiftBinOp(...@@ -4515,7 +4815,7 @@ fn genShiftBinOp(
4515 if (self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv)) break :dst lhs_mcv;4815 if (self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv)) break :dst lhs_mcv;
4516 }4816 }
4517 const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, maybe_inst, true);4817 const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, maybe_inst, true);
4518 try self.setRegOrMem(lhs_ty, dst_mcv, lhs_mcv);4818 try self.genCopy(lhs_ty, dst_mcv, lhs_mcv);
4519 break :dst dst_mcv;4819 break :dst dst_mcv;
4520 };4820 };
45214821
...@@ -4610,14 +4910,22 @@ fn genMulDivBinOp(...@@ -4610,14 +4910,22 @@ fn genMulDivBinOp(
4610 }, dst_abi_size) };4910 }, dst_abi_size) };
46114911
4612 const dst_mcv = try self.allocRegOrMemAdvanced(dst_ty, maybe_inst, false);4912 const dst_mcv = try self.allocRegOrMemAdvanced(dst_ty, maybe_inst, false);
4613 try self.asmMemoryRegister(.mov, Memory.sib(.qword, .{4913 try self.asmMemoryRegister(
4614 .base = .rbp,4914 .mov,
4615 .disp = 0 - dst_mcv.stack_offset,4915 Memory.sib(.qword, .{
4616 }), .rax);4916 .base = .{ .frame = dst_mcv.load_frame.index },
4617 try self.asmMemoryRegister(.mov, Memory.sib(.qword, .{4917 .disp = dst_mcv.load_frame.off,
4618 .base = .rbp,4918 }),
4619 .disp = 8 - dst_mcv.stack_offset,4919 .rax,
4620 }), .rdx);4920 );
4921 try self.asmMemoryRegister(
4922 .mov,
4923 Memory.sib(.qword, .{
4924 .base = .{ .frame = dst_mcv.load_frame.index },
4925 .disp = dst_mcv.load_frame.off + 8,
4926 }),
4927 .rdx,
4928 );
4621 return dst_mcv;4929 return dst_mcv;
4622 },4930 },
46234931
...@@ -4775,11 +5083,7 @@ fn genBinOp(...@@ -4775,11 +5083,7 @@ fn genBinOp(
4775 }5083 }
4776 }5084 }
4777 const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, maybe_inst, true);5085 const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, maybe_inst, true);
4778 try self.setRegOrMem(lhs_ty, dst_mcv, switch (lhs) {5086 try self.genCopy(lhs_ty, dst_mcv, lhs);
4779 .load_direct => |sym_index| .{ .lea_direct = sym_index },
4780 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
4781 else => lhs,
4782 });
4783 break :dst dst_mcv;5087 break :dst dst_mcv;
4784 };5088 };
4785 const dst_mcv_lock: ?RegisterLock = switch (dst_mcv) {5089 const dst_mcv_lock: ?RegisterLock = switch (dst_mcv) {
...@@ -4878,10 +5182,21 @@ fn genBinOp(...@@ -4878,10 +5182,21 @@ fn genBinOp(
4878 .max,5182 .max,
4879 => switch (lhs_ty.zigTypeTag()) {5183 => switch (lhs_ty.zigTypeTag()) {
4880 .Int => {5184 .Int => {
4881 const mat_src_mcv = switch (src_mcv) {5185 const mat_src_mcv: MCValue = if (switch (src_mcv) {
4882 .immediate => MCValue{ .register = try self.copyToTmpRegister(rhs_ty, src_mcv) },5186 .immediate,
4883 else => src_mcv,5187 .eflags,
4884 };5188 .register_offset,
5189 .load_direct,
5190 .lea_direct,
5191 .load_got,
5192 .lea_got,
5193 .load_tlv,
5194 .lea_tlv,
5195 .lea_frame,
5196 => true,
5197 .memory => |addr| math.cast(i32, @bitCast(i64, addr)) == null,
5198 else => false,
5199 }) .{ .register = try self.copyToTmpRegister(rhs_ty, src_mcv) } else src_mcv;
4885 const mat_mcv_lock = switch (mat_src_mcv) {5200 const mat_mcv_lock = switch (mat_src_mcv) {
4886 .register => |reg| self.register_manager.lockReg(reg),5201 .register => |reg| self.register_manager.lockReg(reg),
4887 else => null,5202 else => null,
...@@ -4913,57 +5228,56 @@ fn genBinOp(...@@ -4913,57 +5228,56 @@ fn genBinOp(
4913 defer if (tmp_lock) |lock| self.register_manager.unlockReg(lock);5228 defer if (tmp_lock) |lock| self.register_manager.unlockReg(lock);
4914 switch (mat_src_mcv) {5229 switch (mat_src_mcv) {
4915 .none,5230 .none,
4916 .undef,
4917 .dead,
4918 .unreach,5231 .unreach,
5232 .dead,
5233 .undef,
4919 .immediate,5234 .immediate,
4920 .eflags,5235 .eflags,
5236 .register_offset,
4921 .register_overflow,5237 .register_overflow,
4922 .ptr_stack_offset,5238 .load_direct,
4923 .load_got,
4924 .lea_direct,5239 .lea_direct,
5240 .load_got,
5241 .lea_got,
5242 .load_tlv,
4925 .lea_tlv,5243 .lea_tlv,
5244 .lea_frame,
5245 .reserved_frame,
4926 => unreachable,5246 => unreachable,
4927 .register => |src_reg| try self.asmCmovccRegisterRegister(5247 .register => |src_reg| try self.asmCmovccRegisterRegister(
4928 registerAlias(tmp_reg, cmov_abi_size),5248 registerAlias(tmp_reg, cmov_abi_size),
4929 registerAlias(src_reg, cmov_abi_size),5249 registerAlias(src_reg, cmov_abi_size),
4930 cc,5250 cc,
4931 ),5251 ),
4932 .stack_offset => |off| try self.asmCmovccRegisterMemory(5252 .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
4933 registerAlias(tmp_reg, cmov_abi_size),5253 registerAlias(tmp_reg, cmov_abi_size),
4934 Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), .{5254 Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), switch (mat_src_mcv) {
4935 .base = .rbp,5255 .memory => |addr| .{
4936 .disp = -off,5256 .base = .{ .reg = .ds },
5257 .disp = @intCast(i32, @bitCast(i64, addr)),
5258 },
5259 .indirect => |reg_off| .{
5260 .base = .{ .reg = reg_off.reg },
5261 .disp = reg_off.off,
5262 },
5263 .load_frame => |frame_addr| .{
5264 .base = .{ .frame = frame_addr.index },
5265 .disp = frame_addr.off,
5266 },
5267 else => unreachable,
4937 }),5268 }),
4938 cc,5269 cc,
4939 ),5270 ),
4940 .memory, .load_direct, .load_tlv => {
4941 const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
4942 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
4943 defer self.register_manager.unlockReg(addr_reg_lock);
4944
4945 try self.genSetReg(Type.usize, addr_reg, switch (mat_src_mcv) {
4946 .memory => |addr| .{ .immediate = addr },
4947 .load_direct => |sym_index| .{ .lea_direct = sym_index },
4948 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
4949 else => unreachable,
4950 });
4951 try self.asmCmovccRegisterMemory(
4952 registerAlias(tmp_reg, cmov_abi_size),
4953 Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), .{ .base = addr_reg }),
4954 cc,
4955 );
4956 },
4957 }5271 }
4958 try self.setRegOrMem(lhs_ty, dst_mcv, .{ .register = tmp_reg });5272 try self.genCopy(lhs_ty, dst_mcv, .{ .register = tmp_reg });
4959 },5273 },
4960 .Float => try self.genBinOpMir(switch (lhs_ty.tag()) {5274 .Float => try self.genBinOpMir(switch (lhs_ty.floatBits(self.target.*)) {
4961 .f32 => switch (tag) {5275 32 => switch (tag) {
4962 .min => .minss,5276 .min => .minss,
4963 .max => .maxss,5277 .max => .maxss,
4964 else => unreachable,5278 else => unreachable,
4965 },5279 },
4966 .f64 => switch (tag) {5280 64 => switch (tag) {
4967 .min => .minsd,5281 .min => .minsd,
4968 .max => .maxsd,5282 .max => .maxsd,
4969 else => unreachable,5283 else => unreachable,
...@@ -4981,25 +5295,38 @@ fn genBinOp(...@@ -4981,25 +5295,38 @@ fn genBinOp(
4981fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void {5295fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void {
4982 const abi_size = @intCast(u32, ty.abiSize(self.target.*));5296 const abi_size = @intCast(u32, ty.abiSize(self.target.*));
4983 switch (dst_mcv) {5297 switch (dst_mcv) {
4984 .none => unreachable,5298 .none,
4985 .undef => unreachable,5299 .unreach,
4986 .dead, .unreach, .immediate => unreachable,5300 .dead,
4987 .eflags => unreachable,5301 .undef,
4988 .register_overflow => unreachable,5302 .immediate,
5303 .register_offset,
5304 .eflags,
5305 .register_overflow,
5306 .lea_direct,
5307 .lea_got,
5308 .lea_tlv,
5309 .lea_frame,
5310 .reserved_frame,
5311 => unreachable, // unmodifiable destination
4989 .register => |dst_reg| {5312 .register => |dst_reg| {
4990 const dst_alias = registerAlias(dst_reg, abi_size);5313 const dst_alias = registerAlias(dst_reg, abi_size);
4991 switch (src_mcv) {5314 switch (src_mcv) {
4992 .none => unreachable,5315 .none,
4993 .undef => unreachable,5316 .unreach,
4994 .dead, .unreach => unreachable,5317 .dead,
4995 .register_overflow => unreachable,5318 .undef,
5319 .register_overflow,
5320 .reserved_frame,
5321 => unreachable,
4996 .register => |src_reg| switch (ty.zigTypeTag()) {5322 .register => |src_reg| switch (ty.zigTypeTag()) {
4997 .Float => {5323 .Float => {
4998 if (intrinsicsAllowed(self.target.*, ty)) {5324 if (!Target.x86.featureSetHas(self.target.cpu.features, .sse))
4999 return self.asmRegisterRegister(mir_tag, dst_reg.to128(), src_reg.to128());5325 return self.fail("TODO genBinOpMir for {s} {} without sse", .{
5000 }5326 @tagName(mir_tag),
50015327 ty.fmt(self.bin_file.options.module.?),
5002 return self.fail("TODO genBinOpMir for float register-register and no intrinsics", .{});5328 });
5329 return self.asmRegisterRegister(mir_tag, dst_reg.to128(), src_reg.to128());
5003 },5330 },
5004 else => try self.asmRegisterRegister(5331 else => try self.asmRegisterRegister(
5005 mir_tag,5332 mir_tag,
...@@ -5041,14 +5368,16 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s...@@ -5041,14 +5368,16 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
5041 )),5368 )),
5042 else => unreachable,5369 else => unreachable,
5043 },5370 },
5044 .ptr_stack_offset,5371 .register_offset,
5372 .eflags,
5045 .memory,5373 .memory,
5046 .load_got,
5047 .lea_direct,
5048 .load_direct,5374 .load_direct,
5049 .lea_tlv,5375 .lea_direct,
5376 .load_got,
5377 .lea_got,
5050 .load_tlv,5378 .load_tlv,
5051 .eflags,5379 .lea_tlv,
5380 .lea_frame,
5052 => {5381 => {
5053 assert(abi_size <= 8);5382 assert(abi_size <= 8);
5054 const dst_reg_lock = self.register_manager.lockReg(dst_reg);5383 const dst_reg_lock = self.register_manager.lockReg(dst_reg);
...@@ -5057,75 +5386,84 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s...@@ -5057,75 +5386,84 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
5057 const reg = try self.copyToTmpRegister(ty, src_mcv);5386 const reg = try self.copyToTmpRegister(ty, src_mcv);
5058 return self.genBinOpMir(mir_tag, ty, dst_mcv, .{ .register = reg });5387 return self.genBinOpMir(mir_tag, ty, dst_mcv, .{ .register = reg });
5059 },5388 },
5060 .stack_offset => |off| try self.asmRegisterMemory(5389 .indirect, .load_frame => try self.asmRegisterMemory(
5061 mir_tag,5390 mir_tag,
5062 registerAlias(dst_reg, abi_size),5391 registerAlias(dst_reg, abi_size),
5063 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -off }),5392 Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (src_mcv) {
5393 .indirect => |reg_off| .{
5394 .base = .{ .reg = reg_off.reg },
5395 .disp = reg_off.off,
5396 },
5397 .load_frame => |frame_addr| .{
5398 .base = .{ .frame = frame_addr.index },
5399 .disp = frame_addr.off,
5400 },
5401 else => unreachable,
5402 }),
5064 ),5403 ),
5065 }5404 }
5066 },5405 },
5067 .memory, .load_got, .load_direct, .load_tlv, .stack_offset => {5406 .memory, .indirect, .load_got, .load_direct, .load_tlv, .load_frame => {
5068 const dst: ?struct {5407 const OpInfo = ?struct { addr_reg: Register, addr_lock: RegisterLock };
5069 addr_reg: Register,5408 const limb_abi_size = @min(abi_size, 8);
5070 addr_lock: RegisterLock,5409
5071 } = switch (dst_mcv) {5410 const dst_info: OpInfo = switch (dst_mcv) {
5072 else => unreachable,5411 else => unreachable,
5073 .memory, .load_got, .load_direct, .load_tlv => dst: {5412 .memory, .load_got, .load_direct, .load_tlv => dst: {
5074 const dst_addr_reg = (try self.register_manager.allocReg(null, gp)).to64();5413 const dst_addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
5075 const dst_addr_lock = self.register_manager.lockRegAssumeUnused(dst_addr_reg);5414 const dst_addr_lock = self.register_manager.lockRegAssumeUnused(dst_addr_reg);
5076 errdefer self.register_manager.unlockReg(dst_addr_lock);5415 errdefer self.register_manager.unlockReg(dst_addr_lock);
50775416
5078 try self.genSetReg(Type.usize, dst_addr_reg, switch (dst_mcv) {5417 try self.genSetReg(dst_addr_reg, Type.usize, dst_mcv.address());
5079 .memory => |addr| .{ .immediate = addr },
5080 .load_direct => |sym_index| .{ .lea_direct = sym_index },
5081 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
5082 else => dst_mcv,
5083 });
5084
5085 break :dst .{5418 break :dst .{
5086 .addr_reg = dst_addr_reg,5419 .addr_reg = dst_addr_reg,
5087 .addr_lock = dst_addr_lock,5420 .addr_lock = dst_addr_lock,
5088 };5421 };
5089 },5422 },
5090 .stack_offset => null,5423 .load_frame => null,
5091 };5424 };
5092 defer if (dst) |lock| self.register_manager.unlockReg(lock.addr_lock);5425 defer if (dst_info) |info| self.register_manager.unlockReg(info.addr_lock);
50935426
5094 const src: ?struct {5427 const src_info: OpInfo = switch (src_mcv) {
5095 limb_reg: Register,5428 .none,
5096 limb_lock: RegisterLock,5429 .unreach,
5097 addr_reg: Register,5430 .dead,
5098 addr_lock: RegisterLock,5431 .undef,
5099 } = switch (src_mcv) {5432 .register_overflow,
5100 else => null,5433 .reserved_frame,
5434 => unreachable,
5435 .immediate,
5436 .register,
5437 .register_offset,
5438 .eflags,
5439 .indirect,
5440 .lea_direct,
5441 .lea_got,
5442 .lea_tlv,
5443 .load_frame,
5444 .lea_frame,
5445 => null,
5101 .memory, .load_got, .load_direct, .load_tlv => src: {5446 .memory, .load_got, .load_direct, .load_tlv => src: {
5102 const src_limb_reg = try self.register_manager.allocReg(null, gp);5447 switch (src_mcv) {
5103 const src_limb_lock = self.register_manager.lockRegAssumeUnused(src_limb_reg);5448 .memory => |addr| if (math.cast(i32, @bitCast(i64, addr)) != null and
5104 errdefer self.register_manager.unlockReg(src_limb_lock);5449 math.cast(i32, @bitCast(i64, addr) + abi_size - limb_abi_size) != null)
5450 break :src null,
5451 .load_got, .load_direct, .load_tlv => {},
5452 else => unreachable,
5453 }
51055454
5106 const src_addr_reg = (try self.register_manager.allocReg(null, gp)).to64();5455 const src_addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
5107 const src_addr_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg);5456 const src_addr_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg);
5108 errdefer self.register_manager.unlockReg(src_addr_lock);5457 errdefer self.register_manager.unlockReg(src_addr_lock);
51095458
5110 try self.genSetReg(Type.usize, src_addr_reg, switch (src_mcv) {5459 try self.genSetReg(src_addr_reg, Type.usize, src_mcv.address());
5111 .memory => |addr| .{ .immediate = addr },
5112 .load_direct => |sym_index| .{ .lea_direct = sym_index },
5113 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
5114 else => src_mcv,
5115 });
5116
5117 break :src .{5460 break :src .{
5118 .addr_reg = src_addr_reg,5461 .addr_reg = src_addr_reg,
5119 .addr_lock = src_addr_lock,5462 .addr_lock = src_addr_lock,
5120 .limb_reg = src_limb_reg,
5121 .limb_lock = src_limb_lock,
5122 };5463 };
5123 },5464 },
5124 };5465 };
5125 defer if (src) |locks| {5466 defer if (src_info) |info| self.register_manager.unlockReg(info.addr_lock);
5126 self.register_manager.unlockReg(locks.limb_lock);
5127 self.register_manager.unlockReg(locks.addr_lock);
5128 };
51295467
5130 const ty_signedness =5468 const ty_signedness =
5131 if (ty.isAbiInt()) ty.intInfo(self.target.*).signedness else .unsigned;5469 if (ty.isAbiInt()) ty.intInfo(self.target.*).signedness else .unsigned;
...@@ -5133,7 +5471,6 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s...@@ -5133,7 +5471,6 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
5133 .signed => Type.usize,5471 .signed => Type.usize,
5134 .unsigned => Type.isize,5472 .unsigned => Type.isize,
5135 };5473 };
5136 const limb_abi_size = @min(abi_size, 8);
5137 var off: i32 = 0;5474 var off: i32 = 0;
5138 while (off < abi_size) : (off += 8) {5475 while (off < abi_size) : (off += 8) {
5139 const mir_limb_tag = switch (off) {5476 const mir_limb_tag = switch (off) {
...@@ -5150,35 +5487,45 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s...@@ -5150,35 +5487,45 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
5150 const dst_limb_mem = Memory.sib(5487 const dst_limb_mem = Memory.sib(
5151 Memory.PtrSize.fromSize(limb_abi_size),5488 Memory.PtrSize.fromSize(limb_abi_size),
5152 switch (dst_mcv) {5489 switch (dst_mcv) {
5153 else => unreachable,
5154 .stack_offset => |dst_off| .{
5155 .base = .rbp,
5156 .disp = off - dst_off,
5157 },
5158 .memory,5490 .memory,
5159 .load_got,5491 .load_got,
5160 .load_direct,5492 .load_direct,
5161 .load_tlv,5493 .load_tlv,
5162 => .{ .base = dst.?.addr_reg, .disp = off },5494 => .{ .base = .{ .reg = dst_info.?.addr_reg }, .disp = off },
5495 .indirect => |reg_off| .{
5496 .base = .{ .reg = reg_off.reg },
5497 .disp = reg_off.off + off,
5498 },
5499 .load_frame => |frame_addr| .{
5500 .base = .{ .frame = frame_addr.index },
5501 .disp = frame_addr.off + off,
5502 },
5503 else => unreachable,
5163 },5504 },
5164 );5505 );
5165 switch (src_mcv) {5506 switch (src_mcv) {
5166 .none => unreachable,5507 .none,
5167 .undef => unreachable,5508 .unreach,
5168 .dead, .unreach => unreachable,5509 .dead,
5169 .register_overflow => unreachable,5510 .undef,
5170 .register => |src_reg| {5511 .register_overflow,
5171 assert(off == 0);5512 .reserved_frame,
5172 try self.asmMemoryRegister(5513 => unreachable,
5514 .register => |src_reg| switch (off) {
5515 0 => try self.asmMemoryRegister(
5173 mir_limb_tag,5516 mir_limb_tag,
5174 dst_limb_mem,5517 dst_limb_mem,
5175 registerAlias(src_reg, limb_abi_size),5518 registerAlias(src_reg, limb_abi_size),
5176 );5519 ),
5520 else => unreachable,
5177 },5521 },
5178 .immediate => |src_imm| {5522 .immediate => |src_imm| {
5179 const imm = if (off == 0) src_imm else switch (ty_signedness) {5523 const imm = switch (off) {
5180 .signed => @bitCast(u64, @bitCast(i64, src_imm) >> 63),5524 0 => src_imm,
5181 .unsigned => 0,5525 else => switch (ty_signedness) {
5526 .signed => @bitCast(u64, @bitCast(i64, src_imm) >> 63),
5527 .unsigned => 0,
5528 },
5182 };5529 };
5183 switch (self.regBitSize(limb_ty)) {5530 switch (self.regBitSize(limb_ty)) {
5184 8 => try self.asmMemoryImmediate(5531 8 => try self.asmMemoryImmediate(
...@@ -5223,47 +5570,43 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s...@@ -5223,47 +5570,43 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
5223 else => unreachable,5570 else => unreachable,
5224 }5571 }
5225 },5572 },
5573 .register_offset,
5574 .eflags,
5226 .memory,5575 .memory,
5227 .load_got,5576 .indirect,
5228 .load_direct,5577 .load_direct,
5229 .load_tlv,
5230 => {
5231 try self.asmRegisterMemory(
5232 .mov,
5233 registerAlias(src.?.limb_reg, limb_abi_size),
5234 Memory.sib(
5235 Memory.PtrSize.fromSize(limb_abi_size),
5236 .{ .base = src.?.addr_reg, .disp = off },
5237 ),
5238 );
5239 try self.asmMemoryRegister(
5240 mir_limb_tag,
5241 dst_limb_mem,
5242 registerAlias(src.?.limb_reg, limb_abi_size),
5243 );
5244 },
5245 .stack_offset,
5246 .ptr_stack_offset,
5247 .eflags,
5248 .lea_direct,5578 .lea_direct,
5579 .load_got,
5580 .lea_got,
5581 .load_tlv,
5249 .lea_tlv,5582 .lea_tlv,
5583 .load_frame,
5584 .lea_frame,
5250 => {5585 => {
5251 const src_limb_reg = try self.copyToTmpRegister(limb_ty, switch (src_mcv) {5586 const src_limb_reg = try self.copyToTmpRegister(limb_ty, if (src_info) |info| .{
5252 .stack_offset => |src_off| .{ .stack_offset = src_off - off },5587 .indirect = .{ .reg = info.addr_reg, .off = off },
5253 .ptr_stack_offset,5588 } else switch (src_mcv) {
5254 .eflags,5589 .eflags,
5255 .load_got,5590 .register_offset,
5256 .lea_direct,5591 .lea_direct,
5592 .lea_got,
5257 .lea_tlv,5593 .lea_tlv,
5258 => off: {5594 .lea_frame,
5259 assert(off == 0);5595 => switch (off) {
5260 break :off src_mcv;5596 0 => src_mcv,
5597 else => .{ .immediate = 0 },
5261 },5598 },
5599 .memory => |addr| .{ .memory = @bitCast(u64, @bitCast(i64, addr) + off) },
5600 .indirect => |reg_off| .{ .indirect = .{
5601 .reg = reg_off.reg,
5602 .off = reg_off.off + off,
5603 } },
5604 .load_frame => |frame_addr| .{ .load_frame = .{
5605 .index = frame_addr.index,
5606 .off = frame_addr.off + off,
5607 } },
5262 else => unreachable,5608 else => unreachable,
5263 });5609 });
5264 const src_limb_lock = self.register_manager.lockReg(src_limb_reg);
5265 defer if (src_limb_lock) |lock| self.register_manager.unlockReg(lock);
5266
5267 try self.asmMemoryRegister(5610 try self.asmMemoryRegister(
5268 mir_limb_tag,5611 mir_limb_tag,
5269 dst_limb_mem,5612 dst_limb_mem,
...@@ -5273,9 +5616,6 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s...@@ -5273,9 +5616,6 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
5273 }5616 }
5274 }5617 }
5275 },5618 },
5276 .ptr_stack_offset => unreachable,
5277 .lea_tlv => unreachable,
5278 .lea_direct => unreachable,
5279 }5619 }
5280}5620}
52815621
...@@ -5284,27 +5624,33 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s...@@ -5284,27 +5624,33 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s
5284fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void {5624fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void {
5285 const abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));5625 const abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
5286 switch (dst_mcv) {5626 switch (dst_mcv) {
5287 .none => unreachable,5627 .none,
5288 .undef => unreachable,5628 .unreach,
5289 .dead, .unreach, .immediate => unreachable,5629 .dead,
5290 .eflags => unreachable,5630 .undef,
5291 .ptr_stack_offset => unreachable,5631 .immediate,
5292 .lea_direct => unreachable,5632 .register_offset,
5293 .lea_tlv => unreachable,5633 .eflags,
5294 .register_overflow => unreachable,5634 .register_overflow,
5635 .lea_direct,
5636 .lea_got,
5637 .lea_tlv,
5638 .lea_frame,
5639 .reserved_frame,
5640 => unreachable, // unmodifiable destination
5295 .register => |dst_reg| {5641 .register => |dst_reg| {
5296 const dst_alias = registerAlias(dst_reg, abi_size);5642 const dst_alias = registerAlias(dst_reg, abi_size);
5297 const dst_lock = self.register_manager.lockReg(dst_reg);5643 const dst_lock = self.register_manager.lockReg(dst_reg);
5298 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);5644 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
52995645
5300 switch (src_mcv) {5646 switch (src_mcv) {
5301 .none => unreachable,5647 .none,
5302 .undef => try self.genSetReg(dst_ty, dst_reg, .undef),5648 .unreach,
5303 .dead, .unreach => unreachable,5649 .dead,
5304 .ptr_stack_offset => unreachable,5650 .undef,
5305 .lea_direct => unreachable,5651 .register_overflow,
5306 .lea_tlv => unreachable,5652 .reserved_frame,
5307 .register_overflow => unreachable,5653 => unreachable,
5308 .register => |src_reg| try self.asmRegisterRegister(5654 .register => |src_reg| try self.asmRegisterRegister(
5309 .imul,5655 .imul,
5310 dst_alias,5656 dst_alias,
...@@ -5323,74 +5669,54 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M...@@ -5323,74 +5669,54 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
5323 return self.genIntMulComplexOpMir(dst_ty, dst_mcv, MCValue{ .register = src_reg });5669 return self.genIntMulComplexOpMir(dst_ty, dst_mcv, MCValue{ .register = src_reg });
5324 }5670 }
5325 },5671 },
5326 .stack_offset => |off| {5672 .register_offset,
5327 try self.asmRegisterMemory(5673 .eflags,
5328 .imul,
5329 dst_alias,
5330 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -off }),
5331 );
5332 },
5333 .memory,
5334 .load_got,
5335 .load_direct,5674 .load_direct,
5675 .lea_direct,
5676 .load_got,
5677 .lea_got,
5336 .load_tlv,5678 .load_tlv,
5337 => {5679 .lea_tlv,
5338 return self.fail("TODO implement x86 multiply source memory", .{});5680 .lea_frame,
5339 },5681 => try self.asmRegisterRegister(
5340 .eflags => {5682 .imul,
5341 return self.fail("TODO implement x86 multiply source eflags", .{});5683 dst_alias,
5342 },5684 registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size),
5685 ),
5686 .memory, .indirect, .load_frame => try self.asmRegisterMemory(
5687 .imul,
5688 dst_alias,
5689 Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (src_mcv) {
5690 .memory => |addr| .{
5691 .base = .{ .reg = .ds },
5692 .disp = math.cast(i32, @bitCast(i64, addr)) orelse
5693 return self.asmRegisterRegister(
5694 .imul,
5695 dst_alias,
5696 registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size),
5697 ),
5698 },
5699 .indirect => |reg_off| .{
5700 .base = .{ .reg = reg_off.reg },
5701 .disp = reg_off.off,
5702 },
5703 .load_frame => |frame_addr| .{
5704 .base = .{ .frame = frame_addr.index },
5705 .disp = frame_addr.off,
5706 },
5707 else => unreachable,
5708 }),
5709 ),
5343 }5710 }
5344 },5711 },
5345 .stack_offset => |off| {5712 .memory, .indirect, .load_direct, .load_got, .load_tlv, .load_frame => {
5346 switch (src_mcv) {5713 const tmp_reg = try self.copyToTmpRegister(dst_ty, dst_mcv);
5347 .none => unreachable,5714 const tmp_mcv = MCValue{ .register = tmp_reg };
5348 .undef => return self.genSetStack(dst_ty, off, .undef, .{}),5715 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
5349 .dead, .unreach => unreachable,5716 defer self.register_manager.unlockReg(tmp_lock);
5350 .ptr_stack_offset => unreachable,
5351 .lea_direct => unreachable,
5352 .lea_tlv => unreachable,
5353 .register_overflow => unreachable,
5354 .register => |src_reg| {
5355 // copy dst to a register
5356 const dst_reg = try self.copyToTmpRegister(dst_ty, dst_mcv);
5357 try self.asmRegisterRegister(
5358 .imul,
5359 registerAlias(dst_reg, abi_size),
5360 registerAlias(src_reg, abi_size),
5361 );
5362 // copy dst_reg back out
5363 return self.genSetStack(dst_ty, off, .{ .register = dst_reg }, .{});
5364 },
5365 .immediate => {
5366 // copy dst to a register
5367 const dst_reg = try self.copyToTmpRegister(dst_ty, dst_mcv);
5368 const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
5369 defer self.register_manager.unlockReg(dst_reg_lock);
5370
5371 try self.genIntMulComplexOpMir(dst_ty, .{ .register = dst_reg }, src_mcv);
53725717
5373 return self.genSetStack(dst_ty, off, .{ .register = dst_reg }, .{});5718 try self.genIntMulComplexOpMir(dst_ty, tmp_mcv, src_mcv);
5374 },5719 try self.genCopy(dst_ty, dst_mcv, tmp_mcv);
5375 .memory,
5376 .load_got,
5377 .load_direct,
5378 .load_tlv,
5379 .stack_offset,
5380 => {
5381 return self.fail("TODO implement x86 multiply source memory", .{});
5382 },
5383 .eflags => {
5384 return self.fail("TODO implement x86 multiply source eflags", .{});
5385 },
5386 }
5387 },
5388 .memory,
5389 .load_got,
5390 .load_direct,
5391 .load_tlv,
5392 => {
5393 return self.fail("TODO implement x86 multiply destination memory", .{});
5394 },5720 },
5395 }5721 }
5396}5722}
...@@ -5401,24 +5727,19 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {...@@ -5401,24 +5727,19 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
5401 while (self.args[arg_index] == .none) arg_index += 1;5727 while (self.args[arg_index] == .none) arg_index += 1;
5402 self.arg_index = arg_index + 1;5728 self.arg_index = arg_index + 1;
54035729
5404 const ty = self.air.typeOfIndex(inst);
5405 const mcv = self.args[arg_index];
5406 const src_index = self.air.instructions.items(.data)[inst].arg.src_index;
5407 const name = self.mod_fn.getParamName(self.bin_file.options.module.?, src_index);
5408
5409 const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {5730 const result: MCValue = if (self.liveness.isUnused(inst)) .unreach else result: {
5410 const dst_mcv: MCValue = switch (mcv) {5731 const dst_mcv = self.args[arg_index];
5411 .register => |reg| blk: {5732 switch (dst_mcv) {
5412 self.register_manager.getRegAssumeFree(reg.to64(), inst);5733 .register => |reg| self.register_manager.getRegAssumeFree(reg, inst),
5413 break :blk MCValue{ .register = reg };5734 .load_frame => {},
5414 },5735 else => return self.fail("TODO implement arg for {}", .{dst_mcv}),
5415 .stack_offset => |off| blk: {5736 }
5416 const offset = @intCast(i32, self.max_end_stack) - off + 16;5737
5417 break :blk MCValue{ .stack_offset = -offset };5738 const ty = self.air.typeOfIndex(inst);
5418 },5739 const src_index = self.air.instructions.items(.data)[inst].arg.src_index;
5419 else => return self.fail("TODO implement arg for {}", .{mcv}),5740 const name = self.mod_fn.getParamName(self.bin_file.options.module.?, src_index);
5420 };
5421 try self.genArgDbgInfo(ty, name, dst_mcv);5741 try self.genArgDbgInfo(ty, name, dst_mcv);
5742
5422 break :result dst_mcv;5743 break :result dst_mcv;
5423 };5744 };
5424 return self.finishAir(inst, result, .{ .none, .none, .none });5745 return self.finishAir(inst, result, .{ .none, .none, .none });
...@@ -5429,13 +5750,15 @@ fn genArgDbgInfo(self: Self, ty: Type, name: [:0]const u8, mcv: MCValue) !void {...@@ -5429,13 +5750,15 @@ fn genArgDbgInfo(self: Self, ty: Type, name: [:0]const u8, mcv: MCValue) !void {
5429 .dwarf => |dw| {5750 .dwarf => |dw| {
5430 const loc: link.File.Dwarf.DeclState.DbgInfoLoc = switch (mcv) {5751 const loc: link.File.Dwarf.DeclState.DbgInfoLoc = switch (mcv) {
5431 .register => |reg| .{ .register = reg.dwarfLocOp() },5752 .register => |reg| .{ .register = reg.dwarfLocOp() },
5432 .stack_offset => |off| .{5753 // TODO use a frame index
5433 .stack = .{5754 .load_frame => return,
5434 // TODO handle -fomit-frame-pointer5755 //.stack_offset => |off| .{
5435 .fp_register = Register.rbp.dwarfLocOpDeref(),5756 // .stack = .{
5436 .offset = -off,5757 // // TODO handle -fomit-frame-pointer
5437 },5758 // .fp_register = Register.rbp.dwarfLocOpDeref(),
5438 },5759 // .offset = -off,
5760 // },
5761 //},
5439 else => unreachable, // not a valid function parameter5762 else => unreachable, // not a valid function parameter
5440 };5763 };
5441 try dw.genArgDbgInfo(name, ty, self.mod_fn.owner_decl, loc);5764 try dw.genArgDbgInfo(name, ty, self.mod_fn.owner_decl, loc);
...@@ -5462,12 +5785,12 @@ fn genVarDbgInfo(...@@ -5462,12 +5785,12 @@ fn genVarDbgInfo(
5462 .dwarf => |dw| {5785 .dwarf => |dw| {
5463 const loc: link.File.Dwarf.DeclState.DbgInfoLoc = switch (mcv) {5786 const loc: link.File.Dwarf.DeclState.DbgInfoLoc = switch (mcv) {
5464 .register => |reg| .{ .register = reg.dwarfLocOp() },5787 .register => |reg| .{ .register = reg.dwarfLocOp() },
5465 .ptr_stack_offset,5788 // TODO use a frame index
5466 .stack_offset,5789 .load_frame, .lea_frame => return,
5467 => |off| .{ .stack = .{5790 //=> |off| .{ .stack = .{
5468 .fp_register = Register.rbp.dwarfLocOpDeref(),5791 // .fp_register = Register.rbp.dwarfLocOpDeref(),
5469 .offset = -off,5792 // .offset = -off,
5470 } },5793 //} },
5471 .memory => |address| .{ .memory = address },5794 .memory => |address| .{ .memory = address },
5472 .load_got => |sym_index| .{ .linker_load = .{ .type = .got, .sym_index = sym_index } },5795 .load_got => |sym_index| .{ .linker_load = .{ .type = .got, .sym_index = sym_index } },
5473 .load_direct => |sym_index| .{ .linker_load = .{ .type = .direct, .sym_index = sym_index } },5796 .load_direct => |sym_index| .{ .linker_load = .{ .type = .direct, .sym_index = sym_index } },
...@@ -5498,15 +5821,13 @@ fn airBreakpoint(self: *Self) !void {...@@ -5498,15 +5821,13 @@ fn airBreakpoint(self: *Self) !void {
54985821
5499fn airRetAddr(self: *Self, inst: Air.Inst.Index) !void {5822fn airRetAddr(self: *Self, inst: Air.Inst.Index) !void {
5500 const dst_mcv = try self.allocRegOrMem(inst, true);5823 const dst_mcv = try self.allocRegOrMem(inst, true);
5501 try self.setRegOrMem(Type.usize, dst_mcv, .{5824 try self.genCopy(Type.usize, dst_mcv, .{ .load_frame = .{ .index = .ret_addr } });
5502 .stack_offset = -@as(i32, @divExact(self.target.cpu.arch.ptrBitWidth(), 8)),
5503 });
5504 return self.finishAir(inst, dst_mcv, .{ .none, .none, .none });5825 return self.finishAir(inst, dst_mcv, .{ .none, .none, .none });
5505}5826}
55065827
5507fn airFrameAddress(self: *Self, inst: Air.Inst.Index) !void {5828fn airFrameAddress(self: *Self, inst: Air.Inst.Index) !void {
5508 const dst_mcv = try self.allocRegOrMem(inst, true);5829 const dst_mcv = try self.allocRegOrMem(inst, true);
5509 try self.setRegOrMem(Type.usize, dst_mcv, .{ .register = .rbp });5830 try self.genCopy(Type.usize, dst_mcv, .{ .lea_frame = .{ .index = .base_ptr } });
5510 return self.finishAir(inst, dst_mcv, .{ .none, .none, .none });5831 return self.finishAir(inst, dst_mcv, .{ .none, .none, .none });
5511}5832}
55125833
...@@ -5534,94 +5855,69 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier...@@ -5534,94 +5855,69 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
5534 else => unreachable,5855 else => unreachable,
5535 };5856 };
55365857
5537 var info = try self.resolveCallingConventionValues(fn_ty, args[fn_ty.fnParamLen()..]);5858 var info = try self.resolveCallingConventionValues(fn_ty, args[fn_ty.fnParamLen()..], .call_frame);
5538 defer info.deinit(self);5859 defer info.deinit(self);
55395860
5861 // We need a properly aligned and sized call frame to be able to call this function.
5862 {
5863 const needed_call_frame =
5864 FrameAlloc.init(.{ .size = info.stack_byte_count, .alignment = info.stack_align });
5865 const frame_allocs_slice = self.frame_allocs.slice();
5866 const stack_frame_size =
5867 &frame_allocs_slice.items(.abi_size)[@enumToInt(FrameIndex.call_frame)];
5868 stack_frame_size.* = @max(stack_frame_size.*, needed_call_frame.abi_size);
5869 const stack_frame_align =
5870 &frame_allocs_slice.items(.abi_align)[@enumToInt(FrameIndex.call_frame)];
5871 stack_frame_align.* = @max(stack_frame_align.*, needed_call_frame.abi_align);
5872 }
5873
5540 try self.spillEflagsIfOccupied();5874 try self.spillEflagsIfOccupied();
5541 try self.spillRegisters(abi.getCallerPreservedRegs(self.target.*));5875 try self.spillRegisters(abi.getCallerPreservedRegs(self.target.*));
55425876
5543 // set stack arguments first because this can clobber registers5877 // set stack arguments first because this can clobber registers
5544 // also clobber spill arguments as we go5878 // also clobber spill arguments as we go
5545 if (info.return_value == .stack_offset) {5879 switch (info.return_value.long) {
5546 try self.spillRegisters(&.{abi.getCAbiIntParamRegs(self.target.*)[0]});5880 .none, .unreach => {},
5881 .indirect => |reg_off| try self.spillRegisters(&.{reg_off.reg}),
5882 else => unreachable,
5547 }5883 }
5548 for (args, info.args) |arg, mc_arg| {5884 for (args, info.args) |arg, mc_arg| {
5549 const arg_ty = self.air.typeOf(arg);5885 const arg_ty = self.air.typeOf(arg);
5550 const arg_mcv = try self.resolveInst(arg);5886 const arg_mcv = try self.resolveInst(arg);
5551 // Here we do not use setRegOrMem even though the logic is similar, because
5552 // the function call will move the stack pointer, so the offsets are different.
5553 switch (mc_arg) {5887 switch (mc_arg) {
5554 .none => {},5888 .none => {},
5555 .register => |reg| try self.spillRegisters(&.{reg}),5889 .register => |reg| try self.spillRegisters(&.{reg}),
5556 .stack_offset => |off| {5890 .load_frame => try self.genCopy(arg_ty, mc_arg, arg_mcv),
5557 // TODO rewrite using `genSetStack`5891 else => unreachable,
5558 try self.genSetStackArg(arg_ty, off, arg_mcv);
5559 },
5560 .ptr_stack_offset => {
5561 return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{});
5562 },
5563 .undef => unreachable,
5564 .immediate => unreachable,
5565 .unreach => unreachable,
5566 .dead => unreachable,
5567 .memory => unreachable,
5568 .eflags => unreachable,
5569 .register_overflow => unreachable,
5570 .load_got => unreachable,
5571 .lea_direct => unreachable,
5572 .load_direct => unreachable,
5573 .lea_tlv => unreachable,
5574 .load_tlv => unreachable,
5575 }5892 }
5576 }5893 }
55775894
5578 // now we are free to set register arguments5895 // now we are free to set register arguments
5579 const ret_reg_lock: ?RegisterLock = blk: {5896 const ret_lock = switch (info.return_value.long) {
5580 if (info.return_value == .stack_offset) {5897 .none, .unreach => null,
5898 .indirect => |reg_off| lock: {
5581 const ret_ty = fn_ty.fnReturnType();5899 const ret_ty = fn_ty.fnReturnType();
5582 const ret_abi_size = @intCast(u32, ret_ty.abiSize(self.target.*));5900 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(ret_ty, self.target.*));
5583 const ret_abi_align = @intCast(u32, ret_ty.abiAlignment(self.target.*));5901 try self.genSetReg(reg_off.reg, Type.usize, .{
5584 const stack_offset = @intCast(i32, try self.allocMem(inst, ret_abi_size, ret_abi_align));5902 .lea_frame = .{ .index = frame_index, .off = -reg_off.off },
5585 log.debug("airCall: return value on stack at offset {}", .{stack_offset});5903 });
55865904 info.return_value.short = .{ .load_frame = .{ .index = frame_index } };
5587 const ret_reg = abi.getCAbiIntParamRegs(self.target.*)[0];5905 break :lock self.register_manager.lockRegAssumeUnused(reg_off.reg);
5588 try self.genSetReg(Type.usize, ret_reg, .{ .ptr_stack_offset = stack_offset });5906 },
5589 const ret_reg_lock = self.register_manager.lockRegAssumeUnused(ret_reg);5907 else => unreachable,
5590
5591 info.return_value.stack_offset = stack_offset;
5592
5593 break :blk ret_reg_lock;
5594 }
5595 break :blk null;
5596 };5908 };
5597 defer if (ret_reg_lock) |lock| self.register_manager.unlockReg(lock);5909 defer if (ret_lock) |lock| self.register_manager.unlockReg(lock);
55985910
5599 for (args, info.args) |arg, mc_arg| {5911 for (args, info.args) |arg, mc_arg| {
5600 const arg_ty = self.air.typeOf(arg);5912 const arg_ty = self.air.typeOf(arg);
5601 const arg_mcv = try self.resolveInst(arg);5913 const arg_mcv = try self.resolveInst(arg);
5602 switch (mc_arg) {5914 switch (mc_arg) {
5603 .none, .stack_offset, .ptr_stack_offset => {},5915 .none, .load_frame => {},
5604 .register => |reg| try self.genSetReg(arg_ty, reg, arg_mcv),5916 .register => try self.genCopy(arg_ty, mc_arg, arg_mcv),
5605 .undef => unreachable,5917 else => unreachable,
5606 .immediate => unreachable,
5607 .unreach => unreachable,
5608 .dead => unreachable,
5609 .memory => unreachable,
5610 .eflags => unreachable,
5611 .register_overflow => unreachable,
5612 .load_got => unreachable,
5613 .lea_direct => unreachable,
5614 .load_direct => unreachable,
5615 .lea_tlv => unreachable,
5616 .load_tlv => unreachable,
5617 }5918 }
5618 }5919 }
56195920
5620 if (info.stack_byte_count > 0) {
5621 // Adjust the stack
5622 try self.asmRegisterImmediate(.sub, .rsp, Immediate.u(info.stack_byte_count));
5623 }
5624
5625 // Due to incremental compilation, how function calls are generated depends5921 // Due to incremental compilation, how function calls are generated depends
5626 // on linking.5922 // on linking.
5627 const mod = self.bin_file.options.module.?;5923 const mod = self.bin_file.options.module.?;
...@@ -5635,16 +5931,16 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier...@@ -5635,16 +5931,16 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
5635 _ = try atom.getOrCreateOffsetTableEntry(elf_file);5931 _ = try atom.getOrCreateOffsetTableEntry(elf_file);
5636 const got_addr = atom.getOffsetTableAddress(elf_file);5932 const got_addr = atom.getOffsetTableAddress(elf_file);
5637 try self.asmMemory(.call, Memory.sib(.qword, .{5933 try self.asmMemory(.call, Memory.sib(.qword, .{
5638 .base = .ds,5934 .base = .{ .reg = .ds },
5639 .disp = @intCast(i32, got_addr),5935 .disp = @intCast(i32, got_addr),
5640 }));5936 }));
5641 } else if (self.bin_file.cast(link.File.Coff)) |_| {5937 } else if (self.bin_file.cast(link.File.Coff)) |_| {
5642 const sym_index = try self.getSymbolIndexForDecl(func.owner_decl);5938 const sym_index = try self.getSymbolIndexForDecl(func.owner_decl);
5643 try self.genSetReg(Type.usize, .rax, .{ .load_got = sym_index });5939 try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });
5644 try self.asmRegister(.call, .rax);5940 try self.asmRegister(.call, .rax);
5645 } else if (self.bin_file.cast(link.File.MachO)) |_| {5941 } else if (self.bin_file.cast(link.File.MachO)) |_| {
5646 const sym_index = try self.getSymbolIndexForDecl(func.owner_decl);5942 const sym_index = try self.getSymbolIndexForDecl(func.owner_decl);
5647 try self.genSetReg(Type.usize, .rax, .{ .load_got = sym_index });5943 try self.genSetReg(.rax, Type.usize, .{ .lea_got = sym_index });
5648 try self.asmRegister(.call, .rax);5944 try self.asmRegister(.call, .rax);
5649 } else if (self.bin_file.cast(link.File.Plan9)) |p9| {5945 } else if (self.bin_file.cast(link.File.Plan9)) |p9| {
5650 const decl_block_index = try p9.seeDecl(func.owner_decl);5946 const decl_block_index = try p9.seeDecl(func.owner_decl);
...@@ -5655,7 +5951,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier...@@ -5655,7 +5951,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
5655 const got_index = decl_block.got_index.?;5951 const got_index = decl_block.got_index.?;
5656 const fn_got_addr = got_addr + got_index * ptr_bytes;5952 const fn_got_addr = got_addr + got_index * ptr_bytes;
5657 try self.asmMemory(.call, Memory.sib(.qword, .{5953 try self.asmMemory(.call, Memory.sib(.qword, .{
5658 .base = .ds,5954 .base = .{ .reg = .ds },
5659 .disp = @intCast(i32, fn_got_addr),5955 .disp = @intCast(i32, fn_got_addr),
5660 }));5956 }));
5661 } else unreachable;5957 } else unreachable;
...@@ -5696,41 +5992,15 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier...@@ -5696,41 +5992,15 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
5696 } else {5992 } else {
5697 assert(ty.zigTypeTag() == .Pointer);5993 assert(ty.zigTypeTag() == .Pointer);
5698 const mcv = try self.resolveInst(callee);5994 const mcv = try self.resolveInst(callee);
5699 try self.genSetReg(Type.usize, .rax, mcv);5995 try self.genSetReg(.rax, Type.usize, mcv);
5700 try self.asmRegister(.call, .rax);5996 try self.asmRegister(.call, .rax);
5701 }5997 }
57025998
5703 if (info.stack_byte_count > 0) {
5704 // Readjust the stack
5705 try self.asmRegisterImmediate(.add, .rsp, Immediate.u(info.stack_byte_count));
5706 }
5707
5708 const result: MCValue = result: {
5709 if (self.liveness.isUnused(inst)) break :result .unreach;
5710
5711 switch (info.return_value) {
5712 .register => {
5713 // Save function return value in a new register
5714 break :result try self.copyToRegisterWithInstTracking(
5715 inst,
5716 self.air.typeOfIndex(inst),
5717 info.return_value,
5718 );
5719 },
5720 else => {},
5721 }
5722 break :result info.return_value;
5723 };
5724
5725 if (args.len <= Liveness.bpi - 2) {
5726 var buf = [1]Air.Inst.Ref{.none} ** (Liveness.bpi - 1);
5727 buf[0] = callee;
5728 std.mem.copy(Air.Inst.Ref, buf[1..], args);
5729 return self.finishAir(inst, result, buf);
5730 }
5731 var bt = self.liveness.iterateBigTomb(inst);5999 var bt = self.liveness.iterateBigTomb(inst);
5732 self.feed(&bt, callee);6000 self.feed(&bt, callee);
5733 for (args) |arg| self.feed(&bt, arg);6001 for (args) |arg| self.feed(&bt, arg);
6002
6003 const result = if (self.liveness.isUnused(inst)) .unreach else info.return_value.short;
5734 return self.finishAirResult(inst, result);6004 return self.finishAirResult(inst, result);
5735}6005}
57366006
...@@ -5738,18 +6008,19 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {...@@ -5738,18 +6008,19 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {
5738 const un_op = self.air.instructions.items(.data)[inst].un_op;6008 const un_op = self.air.instructions.items(.data)[inst].un_op;
5739 const operand = try self.resolveInst(un_op);6009 const operand = try self.resolveInst(un_op);
5740 const ret_ty = self.fn_type.fnReturnType();6010 const ret_ty = self.fn_type.fnReturnType();
5741 switch (self.ret_mcv) {6011 switch (self.ret_mcv.short) {
5742 .immediate => {6012 .none => {},
5743 assert(ret_ty.isError());6013 .register => try self.genCopy(ret_ty, self.ret_mcv.short, operand),
5744 },6014 .indirect => |reg_off| {
5745 .stack_offset => {6015 try self.register_manager.getReg(reg_off.reg, null);
5746 try self.store(self.ret_mcv, operand, Type.usize, ret_ty);6016 const lock = self.register_manager.lockRegAssumeUnused(reg_off.reg);
5747 },6017 defer self.register_manager.unlockReg(lock);
5748 else => {6018
5749 try self.setRegOrMem(ret_ty, self.ret_mcv, operand);6019 try self.genSetReg(reg_off.reg, Type.usize, self.ret_mcv.long);
6020 try self.genSetMem(.{ .reg = reg_off.reg }, reg_off.off, ret_ty, operand);
5750 },6021 },
6022 else => unreachable,
5751 }6023 }
5752 // TODO when implementing defer, this will need to jump to the appropriate defer expression.
5753 // TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction6024 // TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
5754 // which is available if the jump is 127 bytes or less forward.6025 // which is available if the jump is 127 bytes or less forward.
5755 const jmp_reloc = try self.asmJmpReloc(undefined);6026 const jmp_reloc = try self.asmJmpReloc(undefined);
...@@ -5761,25 +6032,12 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {...@@ -5761,25 +6032,12 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
5761 const un_op = self.air.instructions.items(.data)[inst].un_op;6032 const un_op = self.air.instructions.items(.data)[inst].un_op;
5762 const ptr = try self.resolveInst(un_op);6033 const ptr = try self.resolveInst(un_op);
5763 const ptr_ty = self.air.typeOf(un_op);6034 const ptr_ty = self.air.typeOf(un_op);
5764 const elem_ty = ptr_ty.elemType();6035 switch (self.ret_mcv.short) {
5765 const abi_size = elem_ty.abiSize(self.target.*);6036 .none => {},
5766 switch (self.ret_mcv) {6037 .register => try self.load(self.ret_mcv.short, ptr_ty, ptr),
5767 .immediate => {6038 .indirect => |reg_off| try self.genSetReg(reg_off.reg, ptr_ty, ptr),
5768 assert(elem_ty.isError());6039 else => unreachable,
5769 },
5770 .stack_offset => {
5771 const reg = try self.copyToTmpRegister(Type.usize, self.ret_mcv);
5772 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
5773 defer self.register_manager.unlockReg(reg_lock);
5774
5775 try self.genInlineMemcpy(.{ .register = reg }, ptr, .{ .immediate = abi_size }, .{});
5776 },
5777 else => {
5778 try self.load(self.ret_mcv, ptr, ptr_ty);
5779 try self.setRegOrMem(elem_ty, self.ret_mcv, self.ret_mcv);
5780 },
5781 }6040 }
5782 // TODO when implementing defer, this will need to jump to the appropriate defer expression.
5783 // TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction6041 // TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
5784 // which is available if the jump is 127 bytes or less forward.6042 // which is available if the jump is 127 bytes or less forward.
5785 const jmp_reloc = try self.asmJmpReloc(undefined);6043 const jmp_reloc = try self.asmJmpReloc(undefined);
...@@ -5822,7 +6080,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {...@@ -5822,7 +6080,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
5822 break :dst rhs_mcv;6080 break :dst rhs_mcv;
5823 } else if (dst_mem_ok) dst: {6081 } else if (dst_mem_ok) dst: {
5824 const dst_mcv = try self.allocTempRegOrMem(ty, true);6082 const dst_mcv = try self.allocTempRegOrMem(ty, true);
5825 try self.setRegOrMem(ty, dst_mcv, lhs_mcv);6083 try self.genCopy(ty, dst_mcv, lhs_mcv);
5826 break :dst dst_mcv;6084 break :dst dst_mcv;
5827 } else .{ .register = try self.copyToTmpRegister(ty, lhs_mcv) };6085 } else .{ .register = try self.copyToTmpRegister(ty, lhs_mcv) };
5828 const dst_lock = switch (dst_mcv) {6086 const dst_lock = switch (dst_mcv) {
...@@ -5865,26 +6123,27 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {...@@ -5865,26 +6123,27 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
5865 const atom = elf_file.getAtom(atom_index);6123 const atom = elf_file.getAtom(atom_index);
5866 _ = try atom.getOrCreateOffsetTableEntry(elf_file);6124 _ = try atom.getOrCreateOffsetTableEntry(elf_file);
5867 const got_addr = atom.getOffsetTableAddress(elf_file);6125 const got_addr = atom.getOffsetTableAddress(elf_file);
5868 try self.asmRegisterMemory(.mov, addr_reg.to64(), Memory.sib(.qword, .{6126 try self.asmRegisterMemory(
5869 .base = .ds,6127 .mov,
5870 .disp = @intCast(i32, got_addr),6128 addr_reg.to64(),
5871 }));6129 Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) }),
6130 );
5872 } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {6131 } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {
5873 const atom_index = try coff_file.getOrCreateAtomForLazySymbol(6132 const atom_index = try coff_file.getOrCreateAtomForLazySymbol(
5874 .{ .kind = .const_data, .ty = Type.anyerror },6133 .{ .kind = .const_data, .ty = Type.anyerror },
5875 4, // dword alignment6134 4, // dword alignment
5876 );6135 );
5877 const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;6136 const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
5878 try self.genSetReg(Type.usize, addr_reg, .{ .load_got = sym_index });6137 try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
5879 } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {6138 } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
5880 const atom_index = try macho_file.getOrCreateAtomForLazySymbol(6139 const atom_index = try macho_file.getOrCreateAtomForLazySymbol(
5881 .{ .kind = .const_data, .ty = Type.anyerror },6140 .{ .kind = .const_data, .ty = Type.anyerror },
5882 4, // dword alignment6141 4, // dword alignment
5883 );6142 );
5884 const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;6143 const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
5885 try self.genSetReg(Type.usize, addr_reg, .{ .load_got = sym_index });6144 try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
5886 } else {6145 } else {
5887 return self.fail("TODO implement airErrorName for x86_64 {s}", .{@tagName(self.bin_file.tag)});6146 return self.fail("TODO implement airCmpLtErrorsLen for x86_64 {s}", .{@tagName(self.bin_file.tag)});
5888 }6147 }
58896148
5890 try self.spillEflagsIfOccupied();6149 try self.spillEflagsIfOccupied();
...@@ -5900,7 +6159,7 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {...@@ -5900,7 +6159,7 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
5900 try self.asmRegisterMemory(6159 try self.asmRegisterMemory(
5901 .cmp,6160 .cmp,
5902 registerAlias(dst_reg, op_abi_size),6161 registerAlias(dst_reg, op_abi_size),
5903 Memory.sib(Memory.PtrSize.fromSize(op_abi_size), .{ .base = addr_reg }),6162 Memory.sib(Memory.PtrSize.fromSize(op_abi_size), .{ .base = .{ .reg = addr_reg } }),
5904 );6163 );
5905 const result = MCValue{ .eflags = .b };6164 const result = MCValue{ .eflags = .b };
5906 return self.finishAir(inst, result, .{ un_op, .none, .none });6165 return self.finishAir(inst, result, .{ un_op, .none, .none });
...@@ -5971,14 +6230,13 @@ fn genTry(...@@ -5971,14 +6230,13 @@ fn genTry(
59716230
5972fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {6231fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
5973 const dbg_stmt = self.air.instructions.items(.data)[inst].dbg_stmt;6232 const dbg_stmt = self.air.instructions.items(.data)[inst].dbg_stmt;
5974 const payload = try self.addExtra(Mir.DbgLineColumn{
5975 .line = dbg_stmt.line,
5976 .column = dbg_stmt.column,
5977 });
5978 _ = try self.addInst(.{6233 _ = try self.addInst(.{
5979 .tag = .dbg_line,6234 .tag = .dbg_line,
5980 .ops = undefined,6235 .ops = undefined,
5981 .data = .{ .payload = payload },6236 .data = .{ .line_column = .{
6237 .line = dbg_stmt.line,
6238 .column = dbg_stmt.column,
6239 } },
5982 });6240 });
5983 return self.finishAirBookkeeping();6241 return self.finishAirBookkeeping();
5984}6242}
...@@ -6002,8 +6260,6 @@ fn airDbgVar(self: *Self, inst: Air.Inst.Index) !void {...@@ -6002,8 +6260,6 @@ fn airDbgVar(self: *Self, inst: Air.Inst.Index) !void {
6002 const ty = self.air.typeOf(operand);6260 const ty = self.air.typeOf(operand);
6003 const mcv = try self.resolveInst(operand);6261 const mcv = try self.resolveInst(operand);
60046262
6005 log.debug("airDbgVar: %{d}: {}, {}", .{ inst, ty.fmtDebug(), mcv });
6006
6007 const name = self.air.nullTerminatedString(pl_op.payload);6263 const name = self.air.nullTerminatedString(pl_op.payload);
60086264
6009 const tag = self.air.instructions.items(.tag)[inst];6265 const tag = self.air.instructions.items(.tag)[inst];
...@@ -6025,7 +6281,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {...@@ -6025,7 +6281,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 {
6025 return self.asmJccReloc(undefined, .e);6281 return self.asmJccReloc(undefined, .e);
6026 },6282 },
6027 .immediate,6283 .immediate,
6028 .stack_offset,6284 .load_frame,
6029 => {6285 => {
6030 try self.spillEflagsIfOccupied();6286 try self.spillEflagsIfOccupied();
6031 if (abi_size <= 8) {6287 if (abi_size <= 8) {
...@@ -6118,11 +6374,14 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC...@@ -6118,11 +6374,14 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
6118 .dead,6374 .dead,
6119 .undef,6375 .undef,
6120 .immediate,6376 .immediate,
6121 .register_overflow,
6122 .ptr_stack_offset,
6123 .eflags,6377 .eflags,
6378 .register_offset,
6379 .register_overflow,
6124 .lea_direct,6380 .lea_direct,
6381 .lea_got,
6125 .lea_tlv,6382 .lea_tlv,
6383 .lea_frame,
6384 .reserved_frame,
6126 => unreachable,6385 => unreachable,
61276386
6128 .register => |opt_reg| {6387 .register => |opt_reg| {
...@@ -6152,27 +6411,36 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC...@@ -6152,27 +6411,36 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
6152 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);6411 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
6153 defer self.register_manager.unlockReg(addr_reg_lock);6412 defer self.register_manager.unlockReg(addr_reg_lock);
61546413
6155 try self.genSetReg(Type.usize, addr_reg, switch (opt_mcv) {6414 try self.genSetReg(addr_reg, Type.usize, opt_mcv.address());
6156 .memory => |addr| .{ .immediate = addr },
6157 .load_direct => |sym_index| .{ .lea_direct = sym_index },
6158 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
6159 else => opt_mcv,
6160 });
6161
6162 const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));6415 const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
6163 try self.asmMemoryImmediate(.cmp, Memory.sib(6416 try self.asmMemoryImmediate(
6164 Memory.PtrSize.fromSize(some_abi_size),6417 .cmp,
6165 .{ .base = addr_reg, .disp = some_info.off },6418 Memory.sib(Memory.PtrSize.fromSize(some_abi_size), .{
6166 ), Immediate.u(0));6419 .base = .{ .reg = addr_reg },
6420 .disp = some_info.off,
6421 }),
6422 Immediate.u(0),
6423 );
6167 return .{ .eflags = .e };6424 return .{ .eflags = .e };
6168 },6425 },
61696426
6170 .stack_offset => |off| {6427 .indirect, .load_frame => {
6171 const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));6428 const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
6172 try self.asmMemoryImmediate(.cmp, Memory.sib(6429 try self.asmMemoryImmediate(
6173 Memory.PtrSize.fromSize(some_abi_size),6430 .cmp,
6174 .{ .base = .rbp, .disp = some_info.off - off },6431 Memory.sib(Memory.PtrSize.fromSize(some_abi_size), switch (opt_mcv) {
6175 ), Immediate.u(0));6432 .indirect => |reg_off| .{
6433 .base = .{ .reg = reg_off.reg },
6434 .disp = reg_off.off + some_info.off,
6435 },
6436 .load_frame => |frame_addr| .{
6437 .base = .{ .frame = frame_addr.index },
6438 .disp = frame_addr.off + some_info.off,
6439 },
6440 else => unreachable,
6441 }),
6442 Immediate.u(0),
6443 );
6176 return .{ .eflags = .e };6444 return .{ .eflags = .e };
6177 },6445 },
6178 }6446 }
...@@ -6200,10 +6468,14 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)...@@ -6200,10 +6468,14 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)
6200 defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock);6468 defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock);
62016469
6202 const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));6470 const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*));
6203 try self.asmMemoryImmediate(.cmp, Memory.sib(6471 try self.asmMemoryImmediate(
6204 Memory.PtrSize.fromSize(some_abi_size),6472 .cmp,
6205 .{ .base = ptr_reg, .disp = some_info.off },6473 Memory.sib(Memory.PtrSize.fromSize(some_abi_size), .{
6206 ), Immediate.u(0));6474 .base = .{ .reg = ptr_reg },
6475 .disp = some_info.off,
6476 }),
6477 Immediate.u(0),
6478 );
6207 return .{ .eflags = .e };6479 return .{ .eflags = .e };
6208}6480}
62096481
...@@ -6221,10 +6493,6 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) !...@@ -6221,10 +6493,6 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) !
62216493
6222 const err_off = errUnionErrorOffset(ty.errorUnionPayload(), self.target.*);6494 const err_off = errUnionErrorOffset(ty.errorUnionPayload(), self.target.*);
6223 switch (operand) {6495 switch (operand) {
6224 .stack_offset => |off| {
6225 const offset = off - @intCast(i32, err_off);
6226 try self.genBinOpMir(.cmp, Type.anyerror, .{ .stack_offset = offset }, .{ .immediate = 0 });
6227 },
6228 .register => |reg| {6496 .register => |reg| {
6229 const eu_lock = self.register_manager.lockReg(reg);6497 const eu_lock = self.register_manager.lockReg(reg);
6230 defer if (eu_lock) |lock| self.register_manager.unlockReg(lock);6498 defer if (eu_lock) |lock| self.register_manager.unlockReg(lock);
...@@ -6238,6 +6506,15 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) !...@@ -6238,6 +6506,15 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) !
6238 }6506 }
6239 try self.genBinOpMir(.cmp, Type.anyerror, .{ .register = tmp_reg }, .{ .immediate = 0 });6507 try self.genBinOpMir(.cmp, Type.anyerror, .{ .register = tmp_reg }, .{ .immediate = 0 });
6240 },6508 },
6509 .load_frame => |frame_addr| try self.genBinOpMir(
6510 .cmp,
6511 Type.anyerror,
6512 .{ .load_frame = .{
6513 .index = frame_addr.index,
6514 .off = frame_addr.off + @intCast(i32, err_off),
6515 } },
6516 .{ .immediate = 0 },
6517 ),
6241 else => return self.fail("TODO implement isErr for {}", .{operand}),6518 else => return self.fail("TODO implement isErr for {}", .{operand}),
6242 }6519 }
62436520
...@@ -6324,9 +6601,9 @@ fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -6324,9 +6601,9 @@ fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {
6324 }6601 }
6325 };6602 };
6326 const ptr_ty = self.air.typeOf(un_op);6603 const ptr_ty = self.air.typeOf(un_op);
6327 try self.load(operand, operand_ptr, ptr_ty);6604 try self.load(operand, ptr_ty, operand_ptr);
63286605
6329 const result = try self.isErr(inst, ptr_ty.elemType(), operand);6606 const result = try self.isErr(inst, ptr_ty.childType(), operand);
63306607
6331 return self.finishAir(inst, result, .{ un_op, .none, .none });6608 return self.finishAir(inst, result, .{ un_op, .none, .none });
6332}6609}
...@@ -6358,9 +6635,9 @@ fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -6358,9 +6635,9 @@ fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {
6358 }6635 }
6359 };6636 };
6360 const ptr_ty = self.air.typeOf(un_op);6637 const ptr_ty = self.air.typeOf(un_op);
6361 try self.load(operand, operand_ptr, ptr_ty);6638 try self.load(operand, ptr_ty, operand_ptr);
63626639
6363 const result = try self.isNonErr(inst, ptr_ty.elemType(), operand);6640 const result = try self.isNonErr(inst, ptr_ty.childType(), operand);
63646641
6365 return self.finishAir(inst, result, .{ un_op, .none, .none });6642 return self.finishAir(inst, result, .{ un_op, .none, .none });
6366}6643}
...@@ -6389,6 +6666,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {...@@ -6389,6 +6666,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
63896666
6390fn airBlock(self: *Self, inst: Air.Inst.Index) !void {6667fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
6391 // A block is a setup to be able to jump to the end.6668 // A block is a setup to be able to jump to the end.
6669 const inst_tracking_i = self.inst_tracking.count();
6392 self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(.unreach));6670 self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(.unreach));
63936671
6394 self.scope_generation += 1;6672 self.scope_generation += 1;
...@@ -6412,8 +6690,9 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {...@@ -6412,8 +6690,9 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
6412 for (block_data.value.relocs.items) |reloc| try self.performReloc(reloc);6690 for (block_data.value.relocs.items) |reloc| try self.performReloc(reloc);
6413 }6691 }
64146692
6415 const tracking = self.inst_tracking.getPtr(inst).?;6693 if (std.debug.runtime_safety) assert(self.inst_tracking.getIndex(inst).? == inst_tracking_i);
6416 if (self.liveness.isUnused(inst)) tracking.die(self);6694 const tracking = &self.inst_tracking.values()[inst_tracking_i];
6695 if (self.liveness.isUnused(inst)) tracking.die(self, inst);
6417 self.getValue(tracking.short, inst);6696 self.getValue(tracking.short, inst);
6418 self.finishAirBookkeeping();6697 self.finishAirBookkeeping();
6419}6698}
...@@ -6520,35 +6799,38 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {...@@ -6520,35 +6799,38 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
6520 !block_ty.hasRuntimeBitsIgnoreComptime() or self.liveness.isUnused(br.block_inst);6799 !block_ty.hasRuntimeBitsIgnoreComptime() or self.liveness.isUnused(br.block_inst);
6521 const block_tracking = self.inst_tracking.getPtr(br.block_inst).?;6800 const block_tracking = self.inst_tracking.getPtr(br.block_inst).?;
6522 const block_data = self.blocks.getPtr(br.block_inst).?;6801 const block_data = self.blocks.getPtr(br.block_inst).?;
6802 const first_br = block_data.relocs.items.len == 0;
6803 const block_result = result: {
6804 if (block_unused) break :result .none;
6805
6806 if (self.reuseOperandAdvanced(inst, br.operand, 0, src_mcv, br.block_inst)) {
6807 if (first_br) break :result src_mcv;
6808
6809 if (block_tracking.getReg()) |block_reg|
6810 try self.register_manager.getReg(block_reg, br.block_inst);
6811 // .long = .none to avoid merging operand and block result stack frames.
6812 var current_tracking = InstTracking{ .long = .none, .short = src_mcv };
6813 try current_tracking.materializeUnsafe(self, br.block_inst, block_tracking.*);
6814 if (src_mcv.getReg()) |src_reg| self.register_manager.freeReg(src_reg);
6815 break :result block_tracking.short;
6816 }
65236817
6524 if (block_data.relocs.items.len == 0) {6818 const dst_mcv = if (first_br) try self.allocRegOrMem(br.block_inst, true) else dst: {
6525 block_tracking.* = InstTracking.init(result: {6819 self.getValue(block_tracking.short, br.block_inst);
6526 if (block_unused) break :result .none;6820 break :dst block_tracking.short;
6527 if (self.reuseOperand(inst, br.operand, 0, src_mcv)) {6821 };
6528 // Fix instruction tracking6822 try self.genCopy(block_ty, dst_mcv, src_mcv);
6529 switch (src_mcv) {6823 break :result dst_mcv;
6530 .register => |reg| if (RegisterManager.indexOfRegIntoTracked(reg)) |index| {6824 };
6531 self.register_manager.registers[index] = br.block_inst;
6532 },
6533 else => {},
6534 }
6535 break :result src_mcv;
6536 }
6537
6538 const new_mcv = try self.allocRegOrMem(br.block_inst, true);
6539 try self.setRegOrMem(block_ty, new_mcv, src_mcv);
6540 break :result new_mcv;
6541 });
6542 } else if (!block_unused) try self.setRegOrMem(block_ty, block_tracking.short, src_mcv);
65436825
6544 // Process operand death so that it is properly accounted for in the State below.6826 // Process operand death so that it is properly accounted for in the State below.
6545 if (self.liveness.operandDies(inst, 0)) {6827 if (self.liveness.operandDies(inst, 0)) {
6546 if (Air.refToIndex(br.operand)) |op_inst| self.processDeath(op_inst);6828 if (Air.refToIndex(br.operand)) |op_inst| self.processDeath(op_inst);
6547 }6829 }
65486830
6549 if (block_data.relocs.items.len == 0) {6831 if (first_br) {
6832 block_tracking.* = InstTracking.init(block_result);
6550 try self.saveRetroactiveState(&block_data.state);6833 try self.saveRetroactiveState(&block_data.state);
6551 block_tracking.die(self);
6552 } else try self.restoreState(block_data.state, &.{}, .{6834 } else try self.restoreState(block_data.state, &.{}, .{
6553 .emit_instructions = true,6835 .emit_instructions = true,
6554 .update_tracking = false,6836 .update_tracking = false,
...@@ -6556,6 +6838,9 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {...@@ -6556,6 +6838,9 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
6556 .close_scope = false,6838 .close_scope = false,
6557 });6839 });
65586840
6841 // Stop tracking block result without forgetting tracking info
6842 self.freeValue(block_tracking.short);
6843
6559 // Emit a jump with a relocation. It will be patched up after the block ends.6844 // Emit a jump with a relocation. It will be patched up after the block ends.
6560 // Leave the jump offset undefined6845 // Leave the jump offset undefined
6561 const jmp_reloc = try self.asmJmpReloc(undefined);6846 const jmp_reloc = try self.asmJmpReloc(undefined);
...@@ -6634,7 +6919,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -6634,7 +6919,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
66346919
6635 const arg_mcv = try self.resolveInst(input);6920 const arg_mcv = try self.resolveInst(input);
6636 try self.register_manager.getReg(reg, null);6921 try self.register_manager.getReg(reg, null);
6637 try self.genSetReg(self.air.typeOf(input), reg, arg_mcv);6922 try self.genSetReg(reg, self.air.typeOf(input), arg_mcv);
6638 }6923 }
66396924
6640 {6925 {
...@@ -6686,7 +6971,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -6686,7 +6971,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
6686 return self.fail("Invalid displacement: '{s}'", .{op_str});6971 return self.fail("Invalid displacement: '{s}'", .{op_str});
6687 op.* = .{ .mem = Memory.sib(6972 op.* = .{ .mem = Memory.sib(
6688 mnem_size orelse return self.fail("Unknown size: '{s}'", .{op_str}),6973 mnem_size orelse return self.fail("Unknown size: '{s}'", .{op_str}),
6689 .{ .base = reg, .disp = disp },6974 .{ .base = .{ .reg = reg }, .disp = disp },
6690 ) };6975 ) };
6691 } else {6976 } else {
6692 if (mnem_size) |size| if (reg.bitSize() != size.bitSize())6977 if (mnem_size) |size| if (reg.bitSize() != size.bitSize())
...@@ -6805,335 +7090,317 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -6805,335 +7090,317 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
6805 return self.finishAirResult(inst, result);7090 return self.finishAirResult(inst, result);
6806}7091}
68077092
6808/// Sets the value without any modifications to register allocation metadata or stack allocation metadata.7093fn movMirTag(self: *Self, ty: Type) !Mir.Inst.Tag {
6809fn setRegOrMem(self: *Self, ty: Type, loc: MCValue, val: MCValue) !void {7094 return switch (ty.zigTypeTag()) {
6810 switch (loc) {7095 else => .mov,
6811 .none => return,7096 .Float => switch (ty.floatBits(self.target.*)) {
6812 .immediate => unreachable,7097 16 => .mov,
6813 .register => |reg| return self.genSetReg(ty, reg, val),7098 32 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse)) .movss else .mov,
6814 .stack_offset => |off| return self.genSetStack(ty, off, val, .{}),7099 64 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse2)) .movsd else .mov,
6815 .memory => {7100 else => return self.fail("TODO movMirTag for {}", .{
6816 return self.fail("TODO implement setRegOrMem for memory", .{});7101 ty.fmt(self.bin_file.options.module.?),
7102 }),
6817 },7103 },
6818 else => {7104 };
6819 return self.fail("TODO implement setRegOrMem for {}", .{loc});7105}
7106
7107fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void {
7108 const src_lock = switch (src_mcv) {
7109 .register => |reg| self.register_manager.lockReg(reg),
7110 .register_overflow => |ro| self.register_manager.lockReg(ro.reg),
7111 else => null,
7112 };
7113 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
7114
7115 switch (dst_mcv) {
7116 .none,
7117 .unreach,
7118 .dead,
7119 .undef,
7120 .immediate,
7121 .eflags,
7122 .register_overflow,
7123 .lea_direct,
7124 .lea_got,
7125 .lea_tlv,
7126 .lea_frame,
7127 .reserved_frame,
7128 => unreachable, // unmodifiable destination
7129 .register => |reg| try self.genSetReg(reg, ty, src_mcv),
7130 .register_offset => |dst_reg_off| try self.genSetReg(dst_reg_off.reg, ty, switch (src_mcv) {
7131 .none,
7132 .unreach,
7133 .dead,
7134 .undef,
7135 .register_overflow,
7136 .reserved_frame,
7137 => unreachable,
7138 .immediate,
7139 .register,
7140 .register_offset,
7141 .lea_frame,
7142 => src_mcv.offset(-dst_reg_off.off),
7143 else => .{ .register_offset = .{
7144 .reg = try self.copyToTmpRegister(ty, src_mcv),
7145 .off = -dst_reg_off.off,
7146 } },
7147 }),
7148 .indirect => |reg_off| try self.genSetMem(.{ .reg = reg_off.reg }, reg_off.off, ty, src_mcv),
7149 .memory, .load_direct, .load_got, .load_tlv => {
7150 switch (dst_mcv) {
7151 .memory => |addr| if (math.cast(i32, @bitCast(i64, addr))) |small_addr|
7152 return self.genSetMem(.{ .reg = .ds }, small_addr, ty, src_mcv),
7153 .load_direct, .load_got, .load_tlv => {},
7154 else => unreachable,
7155 }
7156
7157 const addr_reg = try self.copyToTmpRegister(Type.usize, dst_mcv.address());
7158 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
7159 defer self.register_manager.unlockReg(addr_lock);
7160
7161 try self.genSetMem(.{ .reg = addr_reg }, 0, ty, src_mcv);
6820 },7162 },
7163 .load_frame => |frame_addr| try self.genSetMem(
7164 .{ .frame = frame_addr.index },
7165 frame_addr.off,
7166 ty,
7167 src_mcv,
7168 ),
6821 }7169 }
6822}7170}
68237171
6824fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerError!void {7172fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerError!void {
6825 const abi_size = @intCast(u32, ty.abiSize(self.target.*));7173 const abi_size = @intCast(u32, ty.abiSize(self.target.*));
6826 switch (mcv) {7174 if (abi_size > 8) return self.fail("genSetReg called with a value larger than one register", .{});
6827 .dead => unreachable,7175 switch (src_mcv) {
6828 .unreach, .none => return,7176 .none,
6829 .undef => {7177 .unreach,
6830 if (!self.wantSafety()) return; // The already existing value will do just fine.7178 .dead,
6831 if (abi_size <= 8) {7179 .register_overflow,
6832 const reg = try self.copyToTmpRegister(ty, mcv);7180 .reserved_frame,
6833 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });7181 => unreachable,
7182 .undef => if (self.wantSafety())
7183 try self.genSetReg(dst_reg.to64(), Type.usize, .{ .immediate = 0xaaaaaaaaaaaaaaaa }),
7184 .eflags => |cc| try self.asmSetccRegister(dst_reg.to8(), cc),
7185 .immediate => |imm| {
7186 if (imm == 0) {
7187 // 32-bit moves zero-extend to 64-bit, so xoring the 32-bit
7188 // register is the fastest way to zero a register.
7189 try self.asmRegisterRegister(.xor, dst_reg.to32(), dst_reg.to32());
7190 } else if (abi_size > 4 and math.cast(u32, imm) != null) {
7191 // 32-bit moves zero-extend to 64-bit.
7192 try self.asmRegisterImmediate(.mov, dst_reg.to32(), Immediate.u(imm));
7193 } else if (abi_size <= 4 and @bitCast(i64, imm) < 0) {
7194 try self.asmRegisterImmediate(
7195 .mov,
7196 registerAlias(dst_reg, abi_size),
7197 Immediate.s(@intCast(i32, @bitCast(i64, imm))),
7198 );
7199 } else {
7200 try self.asmRegisterImmediate(
7201 .mov,
7202 registerAlias(dst_reg, abi_size),
7203 Immediate.u(imm),
7204 );
6834 }7205 }
6835 try self.genInlineMemset(
6836 .{ .ptr_stack_offset = stack_offset },
6837 .{ .immediate = 0xaa },
6838 .{ .immediate = abi_size },
6839 .{ .dest_stack_base = .rsp },
6840 );
6841 },
6842 .register_overflow => return self.fail("TODO genSetStackArg for register with overflow bit", .{}),
6843 .eflags => {
6844 const reg = try self.copyToTmpRegister(ty, mcv);
6845 return self.genSetStackArg(ty, stack_offset, .{ .register = reg });
6846 },7206 },
6847 .immediate => |imm| {7207 .register => |reg| if (dst_reg.id() != reg.id()) try self.asmRegisterRegister(
6848 switch (abi_size) {7208 try self.movMirTag(ty),
6849 1, 2, 4 => {7209 registerAlias(dst_reg, abi_size),
6850 // TODO7210 registerAlias(reg, abi_size),
6851 // We have a positive stack offset value but we want a twos complement negative7211 ),
6852 // offset from rbp, which is at the top of the stack frame.7212 .register_offset, .indirect, .load_frame, .lea_frame => try self.asmRegisterMemory(
6853 const immediate = if (ty.isSignedInt())7213 switch (src_mcv) {
6854 Immediate.s(@intCast(i32, @bitCast(i64, imm)))7214 .register_offset => |reg_off| switch (reg_off.off) {
6855 else7215 0 => return self.genSetReg(dst_reg, ty, .{ .register = reg_off.reg }),
6856 Immediate.u(@intCast(u32, imm));7216 else => .lea,
6857 try self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
6858 .base = .rsp,
6859 .disp = -stack_offset,
6860 }), immediate);
6861 },7217 },
6862 8 => {7218 .indirect, .load_frame => try self.movMirTag(ty),
6863 const reg = try self.copyToTmpRegister(ty, mcv);7219 .lea_frame => .lea,
6864 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });7220 else => unreachable,
7221 },
7222 registerAlias(dst_reg, abi_size),
7223 Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (src_mcv) {
7224 .register_offset, .indirect => |reg_off| .{
7225 .base = .{ .reg = reg_off.reg },
7226 .disp = reg_off.off,
6865 },7227 },
6866 else => return self.fail("TODO implement inputs on stack for {} with abi size > 8", .{mcv}),7228 .load_frame, .lea_frame => |frame_addr| .{
6867 }7229 .base = .{ .frame = frame_addr.index },
6868 },7230 .disp = frame_addr.off,
6869 .memory,7231 },
6870 .load_direct,7232 else => unreachable,
6871 .load_tlv,7233 }),
6872 => {7234 ),
6873 if (abi_size <= 8) {7235 .memory, .load_direct, .load_got, .load_tlv => {
6874 const reg = try self.copyToTmpRegister(ty, mcv);7236 switch (src_mcv) {
6875 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });7237 .memory => |addr| if (math.cast(i32, @bitCast(i64, addr))) |small_addr|
7238 return self.asmRegisterMemory(
7239 try self.movMirTag(ty),
7240 registerAlias(dst_reg, abi_size),
7241 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
7242 .base = .{ .reg = .ds },
7243 .disp = small_addr,
7244 }),
7245 ),
7246 .load_direct => |sym_index| if (try self.movMirTag(ty) == .mov) {
7247 const atom_index = try self.getSymbolIndexForDecl(self.mod_fn.owner_decl);
7248 _ = try self.addInst(.{
7249 .tag = .mov_linker,
7250 .ops = .direct_reloc,
7251 .data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
7252 .reg = @enumToInt(dst_reg.to64()),
7253 .atom_index = atom_index,
7254 .sym_index = sym_index,
7255 }) },
7256 });
7257 return;
7258 },
7259 .load_got, .load_tlv => {},
7260 else => unreachable,
6876 }7261 }
68777262
6878 const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();7263 const addr_reg = try self.copyToTmpRegister(Type.usize, src_mcv.address());
6879 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);7264 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
6880 defer self.register_manager.unlockReg(addr_lock);7265 defer self.register_manager.unlockReg(addr_lock);
68817266
6882 try self.genSetReg(Type.usize, addr_reg, switch (mcv) {7267 try self.asmRegisterMemory(
6883 .memory => |addr| .{ .immediate = addr },7268 try self.movMirTag(ty),
6884 .load_direct => |sym_index| .{ .lea_direct = sym_index },7269 registerAlias(dst_reg, abi_size),
6885 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },7270 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .{ .reg = addr_reg } }),
6886 else => unreachable,
6887 });
6888 try self.genInlineMemcpy(
6889 .{ .ptr_stack_offset = stack_offset },
6890 .{ .register = addr_reg },
6891 .{ .immediate = abi_size },
6892 .{ .dest_stack_base = .rsp },
6893 );7271 );
6894 },7272 },
6895 .register => |reg| {7273 .lea_direct, .lea_got => |sym_index| {
6896 switch (ty.zigTypeTag()) {7274 const atom_index = try self.getSymbolIndexForDecl(self.mod_fn.owner_decl);
6897 .Float => {7275 _ = try self.addInst(.{
6898 if (intrinsicsAllowed(self.target.*, ty)) {7276 .tag = switch (src_mcv) {
6899 const tag: Mir.Inst.Tag = switch (ty.tag()) {7277 .lea_direct => .lea_linker,
6900 .f32 => .movss,7278 .lea_got => .mov_linker,
6901 .f64 => .movsd,7279 else => unreachable,
6902 else => return self.fail(
6903 "TODO genSetStackArg for register for type {}",
6904 .{ty.fmtDebug()},
6905 ),
6906 };
6907 // TODO verify this
6908 const ptr_size: Memory.PtrSize = switch (ty.tag()) {
6909 .f32 => .dword,
6910 .f64 => .qword,
6911 else => unreachable,
6912 };
6913 return self.asmMemoryRegister(tag, Memory.sib(ptr_size, .{
6914 .base = .rsp,
6915 .disp = -stack_offset,
6916 }), reg.to128());
6917 }
6918
6919 return self.fail("TODO genSetStackArg for register with no intrinsics", .{});
6920 },7280 },
6921 else => {7281 .ops = switch (src_mcv) {
6922 try self.asmMemoryRegister(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{7282 .lea_direct => .direct_reloc,
6923 .base = .rsp,7283 .lea_got => .got_reloc,
6924 .disp = -stack_offset,7284 else => unreachable,
6925 }), registerAlias(reg, abi_size));
6926 },7285 },
6927 }7286 .data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
6928 },7287 .reg = @enumToInt(dst_reg.to64()),
6929 .ptr_stack_offset,7288 .atom_index = atom_index,
6930 .load_got,7289 .sym_index = sym_index,
6931 .lea_direct,7290 }) },
6932 .lea_tlv,7291 });
6933 => {
6934 const reg = try self.copyToTmpRegister(ty, mcv);
6935 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
6936 },
6937 .stack_offset => |mcv_off| {
6938 if (abi_size <= 8) {
6939 const reg = try self.copyToTmpRegister(ty, mcv);
6940 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
6941 }
6942
6943 try self.genInlineMemcpy(
6944 .{ .ptr_stack_offset = stack_offset },
6945 .{ .ptr_stack_offset = mcv_off },
6946 .{ .immediate = abi_size },
6947 .{ .dest_stack_base = .rsp },
6948 );
6949 },7292 },
6950 }7293 .lea_tlv => |sym_index| {
7294 const atom_index = try self.getSymbolIndexForDecl(self.mod_fn.owner_decl);
7295 if (self.bin_file.cast(link.File.MachO)) |_| {
7296 _ = try self.addInst(.{
7297 .tag = .mov_linker,
7298 .ops = .tlv_reloc,
7299 .data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
7300 .reg = @enumToInt(Register.rdi),
7301 .atom_index = atom_index,
7302 .sym_index = sym_index,
7303 }) },
7304 });
7305 // TODO: spill registers before calling
7306 try self.asmMemory(.call, Memory.sib(.qword, .{ .base = .{ .reg = .rdi } }));
7307 try self.genSetReg(dst_reg.to64(), Type.usize, .{ .register = .rax });
7308 } else return self.fail("TODO emit ptr to TLV sequence on {s}", .{
7309 @tagName(self.bin_file.tag),
7310 });
7311 },
7312 }
6951}7313}
69527314
6953fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: InlineMemcpyOpts) InnerError!void {7315fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCValue) InnerError!void {
6954 const base_reg = opts.dest_stack_base orelse .rbp;
6955 const abi_size = @intCast(u32, ty.abiSize(self.target.*));7316 const abi_size = @intCast(u32, ty.abiSize(self.target.*));
6956 switch (mcv) {7317 const dst_ptr_mcv: MCValue = switch (base) {
6957 .dead => unreachable,7318 .none => .{ .immediate = @bitCast(u64, @as(i64, disp)) },
6958 .unreach, .none => return, // Nothing to do.7319 .reg => |base_reg| .{ .register_offset = .{ .reg = base_reg, .off = disp } },
6959 .undef => {7320 .frame => |base_frame_index| .{ .lea_frame = .{ .index = base_frame_index, .off = disp } },
6960 if (!self.wantSafety()) return; // The already existing value will do just fine.7321 };
6961 // TODO Upgrade this to a memset call when we have that available.7322 switch (src_mcv) {
6962 switch (abi_size) {7323 .none, .unreach, .dead, .reserved_frame => unreachable,
6963 1, 2, 4 => {7324 .undef => if (self.wantSafety())
6964 const value: u64 = switch (abi_size) {7325 try self.genInlineMemset(dst_ptr_mcv, .{ .immediate = 0xaa }, .{ .immediate = abi_size }),
6965 1 => 0xaa,7326 .immediate => |imm| switch (abi_size) {
6966 2 => 0xaaaa,7327 1, 2, 4 => {
6967 4 => 0xaaaaaaaa,7328 const immediate = if (ty.isSignedInt())
6968 else => unreachable,7329 Immediate.s(@truncate(i32, @bitCast(i64, imm)))
6969 };7330 else
6970 return self.asmMemoryImmediate(.mov, Memory.sib(7331 Immediate.u(@intCast(u32, imm));
6971 Memory.PtrSize.fromSize(abi_size),7332 try self.asmMemoryImmediate(
6972 .{ .base = base_reg, .disp = -stack_offset },7333 .mov,
6973 ), Immediate.u(value));7334 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),
6974 },7335 immediate,
6975 8 => return self.genSetStack(7336 );
6976 ty,7337 },
6977 stack_offset,7338 3, 5...7 => unreachable,
6978 .{ .immediate = 0xaaaaaaaaaaaaaaaa },7339 else => if (math.cast(i32, @bitCast(i64, imm))) |small| {
6979 opts,7340 try self.asmMemoryImmediate(
6980 ),7341 .mov,
6981 else => |x| return self.genInlineMemset(7342 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),
6982 .{ .ptr_stack_offset = stack_offset },7343 Immediate.s(small),
6983 .{ .immediate = 0xaa },7344 );
6984 .{ .immediate = x },7345 } else {
6985 opts,7346 var offset: i32 = 0;
6986 ),7347 while (offset < abi_size) : (offset += 4) try self.asmMemoryImmediate(
6987 }7348 .mov,
7349 Memory.sib(.dword, .{ .base = base, .disp = disp + offset }),
7350 if (ty.isSignedInt())
7351 Immediate.s(@truncate(
7352 i32,
7353 @bitCast(i64, imm) >> (math.cast(u6, offset * 8) orelse 63),
7354 ))
7355 else
7356 Immediate.u(@truncate(
7357 u32,
7358 if (math.cast(u6, offset * 8)) |shift| imm >> shift else 0,
7359 )),
7360 );
7361 },
6988 },7362 },
7363 .eflags => |cc| try self.asmSetccMemory(Memory.sib(.byte, .{ .base = base, .disp = disp }), cc),
7364 .register => |reg| try self.asmMemoryRegister(
7365 try self.movMirTag(ty),
7366 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base, .disp = disp }),
7367 registerAlias(reg, abi_size),
7368 ),
6989 .register_overflow => |ro| {7369 .register_overflow => |ro| {
6990 const reg_lock = self.register_manager.lockReg(ro.reg);7370 try self.genSetMem(
6991 defer if (reg_lock) |lock| self.register_manager.unlockReg(lock);7371 base,
69927372 disp + @intCast(i32, ty.structFieldOffset(0, self.target.*)),
6993 const wrapped_ty = ty.structFieldType(0);7373 ty.structFieldType(0),
6994 try self.genSetStack(wrapped_ty, stack_offset, .{ .register = ro.reg }, .{});7374 .{ .register = ro.reg },
69957375 );
6996 const overflow_bit_ty = ty.structFieldType(1);7376 try self.genSetMem(
6997 const overflow_bit_offset = ty.structFieldOffset(1, self.target.*);7377 base,
6998 const tmp_reg = try self.register_manager.allocReg(null, gp);7378 disp + @intCast(i32, ty.structFieldOffset(1, self.target.*)),
6999 try self.asmSetccRegister(tmp_reg.to8(), ro.eflags);7379 ty.structFieldType(1),
70007380 .{ .eflags = ro.eflags },
7001 return self.genSetStack(
7002 overflow_bit_ty,
7003 stack_offset - @intCast(i32, overflow_bit_offset),
7004 .{ .register = tmp_reg.to8() },
7005 .{},
7006 );7381 );
7007 },7382 },
7008 .eflags => |cc| try self.asmSetccMemory(7383 .register_offset,
7009 Memory.sib(.byte, .{ .base = base_reg, .disp = -stack_offset }),
7010 cc,
7011 ),
7012 .immediate => |imm| {
7013 // TODO
7014 switch (abi_size) {
7015 0 => {
7016 assert(ty.isError());
7017 try self.asmMemoryImmediate(.mov, Memory.sib(.byte, .{
7018 .base = base_reg,
7019 .disp = -stack_offset,
7020 }), Immediate.u(@truncate(u8, imm)));
7021 },
7022 1, 2, 4 => {
7023 const immediate = if (ty.isSignedInt())
7024 Immediate.s(@truncate(i32, @bitCast(i64, imm)))
7025 else
7026 Immediate.u(@intCast(u32, imm));
7027 try self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
7028 .base = base_reg,
7029 .disp = -stack_offset,
7030 }), immediate);
7031 },
7032 3, 5...7 => unreachable,
7033 else => {
7034 // 64 bit write to memory would take two mov's anyways so we
7035 // insted just use two 32 bit writes to avoid register allocation
7036 if (math.cast(i32, @bitCast(i64, imm))) |small| {
7037 try self.asmMemoryImmediate(.mov, Memory.sib(
7038 Memory.PtrSize.fromSize(abi_size),
7039 .{ .base = base_reg, .disp = -stack_offset },
7040 ), Immediate.s(small));
7041 } else {
7042 var offset: i32 = 0;
7043 while (offset < abi_size) : (offset += 4) try self.asmMemoryImmediate(
7044 .mov,
7045 Memory.sib(.dword, .{ .base = base_reg, .disp = offset - stack_offset }),
7046 if (ty.isSignedInt())
7047 Immediate.s(@truncate(
7048 i32,
7049 @bitCast(i64, imm) >> (math.cast(u6, offset * 8) orelse 63),
7050 ))
7051 else
7052 Immediate.u(@truncate(
7053 u32,
7054 if (math.cast(u6, offset * 8)) |shift| imm >> shift else 0,
7055 )),
7056 );
7057 }
7058 },
7059 }
7060 },
7061 .register => |reg| {
7062 switch (ty.zigTypeTag()) {
7063 .Float => {
7064 if (intrinsicsAllowed(self.target.*, ty)) {
7065 const tag: Mir.Inst.Tag = switch (ty.tag()) {
7066 .f32 => .movss,
7067 .f64 => .movsd,
7068 else => return self.fail(
7069 "TODO genSetStack for register for type {}",
7070 .{ty.fmtDebug()},
7071 ),
7072 };
7073 const ptr_size: Memory.PtrSize = switch (ty.tag()) {
7074 .f32 => .dword,
7075 .f64 => .qword,
7076 else => unreachable,
7077 };
7078 return self.asmMemoryRegister(tag, Memory.sib(ptr_size, .{
7079 .base = base_reg.to64(),
7080 .disp = -stack_offset,
7081 }), reg.to128());
7082 }
7083
7084 return self.fail("TODO genSetStack for register for type float with no intrinsics", .{});
7085 },
7086 else => {
7087 try self.genInlineMemcpyRegisterRegister(ty, base_reg, reg, stack_offset);
7088 },
7089 }
7090 },
7091 .memory,7384 .memory,
7385 .indirect,
7092 .load_direct,7386 .load_direct,
7093 .load_tlv,
7094 .lea_direct,7387 .lea_direct,
7095 .lea_tlv,
7096 => if (abi_size <= 8) {
7097 const reg = try self.copyToTmpRegister(ty, mcv);
7098 return self.genSetStack(ty, stack_offset, MCValue{ .register = reg }, opts);
7099 } else {
7100 const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
7101 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
7102 defer self.register_manager.unlockReg(addr_lock);
7103
7104 try self.genSetReg(Type.usize, addr_reg, switch (mcv) {
7105 .memory => |addr| .{ .immediate = addr },
7106 .load_direct => |sym_index| .{ .lea_direct = sym_index },
7107 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
7108 else => mcv,
7109 });
7110 try self.genInlineMemcpy(
7111 .{ .ptr_stack_offset = stack_offset },
7112 .{ .register = addr_reg },
7113 .{ .immediate = abi_size },
7114 .{},
7115 );
7116 },
7117 .stack_offset => |off| if (abi_size <= 8) {
7118 const tmp_reg = try self.copyToTmpRegister(ty, mcv);
7119 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
7120 defer self.register_manager.unlockReg(tmp_lock);
7121
7122 try self.genSetStack(ty, stack_offset, .{ .register = tmp_reg }, opts);
7123 } else try self.genInlineMemcpy(
7124 .{ .ptr_stack_offset = stack_offset },
7125 .{ .ptr_stack_offset = off },
7126 .{ .immediate = abi_size },
7127 .{},
7128 ),
7129 .ptr_stack_offset,
7130 .load_got,7388 .load_got,
7131 => {7389 .lea_got,
7132 const tmp_reg = try self.copyToTmpRegister(ty, mcv);7390 .load_tlv,
7133 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);7391 .lea_tlv,
7134 defer self.register_manager.unlockReg(tmp_lock);7392 .load_frame,
71357393 .lea_frame,
7136 try self.genSetStack(ty, stack_offset, .{ .register = tmp_reg }, opts);7394 => switch (abi_size) {
7395 0 => {},
7396 1, 2, 4, 8 => {
7397 const src_reg = try self.copyToTmpRegister(ty, src_mcv);
7398 const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
7399 defer self.register_manager.unlockReg(src_lock);
7400
7401 try self.genSetMem(base, disp, ty, .{ .register = src_reg });
7402 },
7403 else => try self.genInlineMemcpy(dst_ptr_mcv, src_mcv.address(), .{ .immediate = abi_size }),
7137 },7404 },
7138 }7405 }
7139}7406}
...@@ -7165,10 +7432,14 @@ fn genInlineMemcpyRegisterRegister(...@@ -7165,10 +7432,14 @@ fn genInlineMemcpyRegisterRegister(
7165 var remainder = abi_size;7432 var remainder = abi_size;
7166 while (remainder > 0) {7433 while (remainder > 0) {
7167 const nearest_power_of_two = @as(u6, 1) << math.log2_int(u3, @intCast(u3, remainder));7434 const nearest_power_of_two = @as(u6, 1) << math.log2_int(u3, @intCast(u3, remainder));
7168 try self.asmMemoryRegister(.mov, Memory.sib(Memory.PtrSize.fromSize(nearest_power_of_two), .{7435 try self.asmMemoryRegister(
7169 .base = dst_reg,7436 .mov,
7170 .disp = -next_offset,7437 Memory.sib(Memory.PtrSize.fromSize(nearest_power_of_two), .{
7171 }), registerAlias(tmp_reg, nearest_power_of_two));7438 .base = dst_reg,
7439 .disp = -next_offset,
7440 }),
7441 registerAlias(tmp_reg, nearest_power_of_two),
7442 );
71727443
7173 if (nearest_power_of_two > 1) {7444 if (nearest_power_of_two > 1) {
7174 try self.genShiftBinOpMir(.shr, ty, .{ .register = tmp_reg }, .{7445 try self.genShiftBinOpMir(.shr, ty, .{ .register = tmp_reg }, .{
...@@ -7191,121 +7462,11 @@ fn genInlineMemcpyRegisterRegister(...@@ -7191,121 +7462,11 @@ fn genInlineMemcpyRegisterRegister(
7191 }7462 }
7192}7463}
71937464
7194const InlineMemcpyOpts = struct {7465fn genInlineMemcpy(self: *Self, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue) InnerError!void {
7195 source_stack_base: ?Register = null,
7196 dest_stack_base: ?Register = null,
7197};
7198
7199fn genInlineMemcpy(
7200 self: *Self,
7201 dst_ptr: MCValue,
7202 src_ptr: MCValue,
7203 len: MCValue,
7204 opts: InlineMemcpyOpts,
7205) InnerError!void {
7206 const ssbase_lock: ?RegisterLock = if (opts.source_stack_base) |reg|
7207 self.register_manager.lockReg(reg)
7208 else
7209 null;
7210 defer if (ssbase_lock) |reg| self.register_manager.unlockReg(reg);
7211
7212 const dsbase_lock: ?RegisterLock = if (opts.dest_stack_base) |reg|
7213 self.register_manager.lockReg(reg)
7214 else
7215 null;
7216 defer if (dsbase_lock) |lock| self.register_manager.unlockReg(lock);
7217
7218 try self.spillRegisters(&.{ .rdi, .rsi, .rcx });7466 try self.spillRegisters(&.{ .rdi, .rsi, .rcx });
72197467 try self.genSetReg(.rdi, Type.usize, dst_ptr);
7220 switch (dst_ptr) {7468 try self.genSetReg(.rsi, Type.usize, src_ptr);
7221 .lea_tlv,7469 try self.genSetReg(.rcx, Type.usize, len);
7222 .load_tlv,
7223 => {
7224 try self.genSetReg(Type.usize, .rdi, switch (dst_ptr) {
7225 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
7226 else => dst_ptr,
7227 });
7228 },
7229 .memory,
7230 .load_got,
7231 .lea_direct,
7232 .load_direct,
7233 => {
7234 try self.genSetReg(Type.usize, .rdi, switch (dst_ptr) {
7235 .memory => |addr| .{ .immediate = addr },
7236 .load_direct => |sym_index| .{ .lea_direct = sym_index },
7237 else => dst_ptr,
7238 });
7239 // Load the pointer, which is stored in memory
7240 try self.asmRegisterMemory(.mov, .rdi, Memory.sib(.qword, .{ .base = .rdi }));
7241 },
7242 .stack_offset, .ptr_stack_offset => |off| {
7243 try self.asmRegisterMemory(switch (dst_ptr) {
7244 .stack_offset => .mov,
7245 .ptr_stack_offset => .lea,
7246 else => unreachable,
7247 }, .rdi, Memory.sib(.qword, .{
7248 .base = opts.dest_stack_base orelse .rbp,
7249 .disp = -off,
7250 }));
7251 },
7252 .register => |reg| {
7253 try self.asmRegisterRegister(
7254 .mov,
7255 registerAlias(.rdi, @intCast(u32, @divExact(reg.bitSize(), 8))),
7256 reg,
7257 );
7258 },
7259 else => {
7260 return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr});
7261 },
7262 }
7263
7264 switch (src_ptr) {
7265 .lea_tlv,
7266 .load_tlv,
7267 => {
7268 try self.genSetReg(Type.usize, .rsi, switch (src_ptr) {
7269 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
7270 else => dst_ptr,
7271 });
7272 },
7273 .memory,
7274 .load_got,
7275 .lea_direct,
7276 .load_direct,
7277 => {
7278 try self.genSetReg(Type.usize, .rsi, switch (src_ptr) {
7279 .memory => |addr| .{ .immediate = addr },
7280 .load_direct => |sym_index| .{ .lea_direct = sym_index },
7281 else => src_ptr,
7282 });
7283 // Load the pointer, which is stored in memory
7284 try self.asmRegisterMemory(.mov, .rsi, Memory.sib(.qword, .{ .base = .rsi }));
7285 },
7286 .stack_offset, .ptr_stack_offset => |off| {
7287 try self.asmRegisterMemory(switch (src_ptr) {
7288 .stack_offset => .mov,
7289 .ptr_stack_offset => .lea,
7290 else => unreachable,
7291 }, .rsi, Memory.sib(.qword, .{
7292 .base = opts.source_stack_base orelse .rbp,
7293 .disp = -off,
7294 }));
7295 },
7296 .register => |reg| {
7297 try self.asmRegisterRegister(
7298 .mov,
7299 registerAlias(.rsi, @intCast(u32, @divExact(reg.bitSize(), 8))),
7300 reg,
7301 );
7302 },
7303 else => {
7304 return self.fail("TODO implement memcpy for setting stack when src is {}", .{src_ptr});
7305 },
7306 }
7307
7308 try self.genSetReg(Type.usize, .rcx, len);
7309 _ = try self.addInst(.{7470 _ = try self.addInst(.{
7310 .tag = .movs,7471 .tag = .movs,
7311 .ops = .string,7472 .ops = .string,
...@@ -7313,67 +7474,11 @@ fn genInlineMemcpy(...@@ -7313,67 +7474,11 @@ fn genInlineMemcpy(
7313 });7474 });
7314}7475}
73157476
7316fn genInlineMemset(7477fn genInlineMemset(self: *Self, dst_ptr: MCValue, value: MCValue, len: MCValue) InnerError!void {
7317 self: *Self,
7318 dst_ptr: MCValue,
7319 value: MCValue,
7320 len: MCValue,
7321 opts: InlineMemcpyOpts,
7322) InnerError!void {
7323 const dsbase_lock: ?RegisterLock = if (opts.dest_stack_base) |reg|
7324 self.register_manager.lockReg(reg)
7325 else
7326 null;
7327 defer if (dsbase_lock) |lock| self.register_manager.unlockReg(lock);
7328
7329 try self.spillRegisters(&.{ .rdi, .al, .rcx });7478 try self.spillRegisters(&.{ .rdi, .al, .rcx });
73307479 try self.genSetReg(.rdi, Type.usize, dst_ptr);
7331 switch (dst_ptr) {7480 try self.genSetReg(.al, Type.u8, value);
7332 .lea_tlv,7481 try self.genSetReg(.rcx, Type.usize, len);
7333 .load_tlv,
7334 => {
7335 try self.genSetReg(Type.usize, .rdi, switch (dst_ptr) {
7336 .load_tlv => |sym_index| .{ .lea_tlv = sym_index },
7337 else => dst_ptr,
7338 });
7339 },
7340 .load_got => try self.genSetReg(Type.usize, .rdi, dst_ptr),
7341 .memory,
7342 .lea_direct,
7343 .load_direct,
7344 => {
7345 try self.genSetReg(Type.usize, .rdi, switch (dst_ptr) {
7346 .memory => |addr| .{ .immediate = addr },
7347 .load_direct => |sym_index| .{ .lea_direct = sym_index },
7348 else => dst_ptr,
7349 });
7350 // Load the pointer, which is stored in memory
7351 try self.asmRegisterMemory(.mov, .rdi, Memory.sib(.qword, .{ .base = .rdi }));
7352 },
7353 .stack_offset, .ptr_stack_offset => |off| {
7354 try self.asmRegisterMemory(switch (dst_ptr) {
7355 .stack_offset => .mov,
7356 .ptr_stack_offset => .lea,
7357 else => unreachable,
7358 }, .rdi, Memory.sib(.qword, .{
7359 .base = opts.dest_stack_base orelse .rbp,
7360 .disp = -off,
7361 }));
7362 },
7363 .register => |reg| {
7364 try self.asmRegisterRegister(
7365 .mov,
7366 registerAlias(.rdi, @intCast(u32, @divExact(reg.bitSize(), 8))),
7367 reg,
7368 );
7369 },
7370 else => {
7371 return self.fail("TODO implement memset for setting stack when dest is {}", .{dst_ptr});
7372 },
7373 }
7374
7375 try self.genSetReg(Type.u8, .al, value);
7376 try self.genSetReg(Type.usize, .rcx, len);
7377 _ = try self.addInst(.{7482 _ = try self.addInst(.{
7378 .tag = .stos,7483 .tag = .stos,
7379 .ops = .string,7484 .ops = .string,
...@@ -7381,307 +7486,6 @@ fn genInlineMemset(...@@ -7381,307 +7486,6 @@ fn genInlineMemset(
7381 });7486 });
7382}7487}
73837488
7384fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void {
7385 const abi_size = @intCast(u32, ty.abiSize(self.target.*));
7386 if (abi_size > 8) return self.fail("genSetReg called with a value larger than one register", .{});
7387 switch (mcv) {
7388 .dead => unreachable,
7389 .register_overflow => unreachable,
7390 .ptr_stack_offset => |off| {
7391 try self.asmRegisterMemory(
7392 .lea,
7393 registerAlias(reg, abi_size),
7394 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -off }),
7395 );
7396 },
7397 .unreach, .none => return, // Nothing to do.
7398 .undef => {
7399 if (!self.wantSafety()) return; // The already existing value will do just fine.
7400 // Write the debug undefined value.
7401 switch (self.regBitSize(ty)) {
7402 8 => return self.genSetReg(ty, reg, .{ .immediate = 0xaa }),
7403 16 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaa }),
7404 32 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaa }),
7405 64 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaaaaaaaaaa }),
7406 else => unreachable,
7407 }
7408 },
7409 .eflags => |cc| {
7410 return self.asmSetccRegister(reg.to8(), cc);
7411 },
7412 .immediate => |imm| {
7413 if (imm == 0) {
7414 // 32-bit moves zero-extend to 64-bit, so xoring the 32-bit
7415 // register is the fastest way to zero a register.
7416 try self.asmRegisterRegister(.xor, reg.to32(), reg.to32());
7417 } else if (abi_size > 4 and math.cast(u32, imm) != null) {
7418 // 32-bit moves zero-extend to 64-bit.
7419 try self.asmRegisterImmediate(.mov, reg.to32(), Immediate.u(imm));
7420 } else if (abi_size <= 4 and @bitCast(i64, imm) < 0) {
7421 try self.asmRegisterImmediate(
7422 .mov,
7423 registerAlias(reg, abi_size),
7424 Immediate.s(@intCast(i32, @bitCast(i64, imm))),
7425 );
7426 } else {
7427 try self.asmRegisterImmediate(
7428 .mov,
7429 registerAlias(reg, abi_size),
7430 Immediate.u(imm),
7431 );
7432 }
7433 },
7434 .register => |src_reg| {
7435 // If the registers are the same, nothing to do.
7436 if (src_reg.id() == reg.id())
7437 return;
7438
7439 switch (ty.zigTypeTag()) {
7440 .Int => switch (ty.intInfo(self.target.*).signedness) {
7441 .signed => {
7442 if (abi_size <= 4) {
7443 return self.asmRegisterRegister(
7444 .movsx,
7445 reg.to64(),
7446 registerAlias(src_reg, abi_size),
7447 );
7448 }
7449 },
7450 .unsigned => {
7451 if (abi_size <= 2) {
7452 return self.asmRegisterRegister(
7453 .movzx,
7454 reg.to64(),
7455 registerAlias(src_reg, abi_size),
7456 );
7457 }
7458 },
7459 },
7460 .Float => {
7461 if (intrinsicsAllowed(self.target.*, ty)) {
7462 const tag: Mir.Inst.Tag = switch (ty.tag()) {
7463 .f32 => .movss,
7464 .f64 => .movsd,
7465 else => return self.fail("TODO genSetReg from register for {}", .{ty.fmtDebug()}),
7466 };
7467 return self.asmRegisterRegister(tag, reg.to128(), src_reg.to128());
7468 }
7469 return self.fail("TODO genSetReg from register for float with no intrinsics", .{});
7470 },
7471 else => {},
7472 }
7473
7474 try self.asmRegisterRegister(.mov, registerAlias(reg, abi_size), registerAlias(src_reg, abi_size));
7475 },
7476 .memory => |addr| switch (ty.zigTypeTag()) {
7477 .Float => {
7478 const base_reg = (try self.register_manager.allocReg(null, gp)).to64();
7479 try self.genSetReg(Type.usize, base_reg, .{ .immediate = addr });
7480
7481 if (intrinsicsAllowed(self.target.*, ty)) {
7482 return self.asmRegisterMemory(
7483 switch (ty.tag()) {
7484 .f32 => .movss,
7485 .f64 => .movsd,
7486 else => return self.fail("TODO genSetReg from memory for {}", .{
7487 ty.fmt(self.bin_file.options.module.?),
7488 }),
7489 },
7490 reg.to128(),
7491 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base_reg }),
7492 );
7493 }
7494
7495 return self.fail("TODO genSetReg from memory for float with no intrinsics", .{});
7496 },
7497 else => {
7498 if (addr <= math.maxInt(i32)) {
7499 try self.asmRegisterMemory(
7500 .mov,
7501 registerAlias(reg, abi_size),
7502 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
7503 .base = .ds,
7504 .disp = @intCast(i32, addr),
7505 }),
7506 );
7507 } else {
7508 if (reg.to64() == .rax) {
7509 // If this is RAX, we can use a direct load.
7510 // Otherwise, we need to load the address, then indirectly load the value.
7511 _ = try self.addInst(.{
7512 .tag = .mov_moffs,
7513 .ops = .rax_moffs,
7514 .data = .{ .payload = try self.addExtra(Mir.MemoryMoffs.encode(.ds, addr)) },
7515 });
7516 } else {
7517 // Rather than duplicate the logic used for the move, we just use a self-call with a new MCValue.
7518 try self.genSetReg(Type.usize, reg, MCValue{ .immediate = addr });
7519 try self.asmRegisterMemory(
7520 .mov,
7521 registerAlias(reg, abi_size),
7522 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }),
7523 );
7524 }
7525 }
7526 },
7527 },
7528 .load_got, .lea_direct => |sym_index| {
7529 const atom_index = try self.getSymbolIndexForDecl(self.mod_fn.owner_decl);
7530 _ = try self.addInst(.{
7531 .tag = switch (mcv) {
7532 .load_got => .mov_linker,
7533 .lea_direct => .lea_linker,
7534 else => unreachable,
7535 },
7536 .ops = switch (mcv) {
7537 .load_got => .got_reloc,
7538 .lea_direct => .direct_reloc,
7539 else => unreachable,
7540 },
7541 .data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
7542 .reg = @enumToInt(reg),
7543 .atom_index = atom_index,
7544 .sym_index = sym_index,
7545 }) },
7546 });
7547 },
7548 .load_direct => |sym_index| {
7549 switch (ty.zigTypeTag()) {
7550 .Float => {
7551 const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
7552 try self.genSetReg(Type.usize, addr_reg, .{ .lea_direct = sym_index });
7553
7554 if (intrinsicsAllowed(self.target.*, ty)) {
7555 return self.asmRegisterMemory(
7556 switch (ty.tag()) {
7557 .f32 => .movss,
7558 .f64 => .movsd,
7559 else => return self.fail("TODO genSetReg from memory for {}", .{
7560 ty.fmt(self.bin_file.options.module.?),
7561 }),
7562 },
7563 reg.to128(),
7564 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = addr_reg }),
7565 );
7566 }
7567
7568 return self.fail("TODO genSetReg from memory for float with no intrinsics", .{});
7569 },
7570 else => {
7571 const atom_index = try self.getSymbolIndexForDecl(self.mod_fn.owner_decl);
7572 _ = try self.addInst(.{
7573 .tag = .mov_linker,
7574 .ops = .direct_reloc,
7575 .data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
7576 .reg = @enumToInt(registerAlias(reg, abi_size)),
7577 .atom_index = atom_index,
7578 .sym_index = sym_index,
7579 }) },
7580 });
7581 },
7582 }
7583 },
7584 .lea_tlv => |sym_index| {
7585 const atom_index = try self.getSymbolIndexForDecl(self.mod_fn.owner_decl);
7586 if (self.bin_file.cast(link.File.MachO)) |_| {
7587 _ = try self.addInst(.{
7588 .tag = .lea_linker,
7589 .ops = .tlv_reloc,
7590 .data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{
7591 .reg = @enumToInt(Register.rdi),
7592 .atom_index = atom_index,
7593 .sym_index = sym_index,
7594 }) },
7595 });
7596 // TODO: spill registers before calling
7597 try self.asmMemory(.call, Memory.sib(.qword, .{ .base = .rdi }));
7598 try self.genSetReg(Type.usize, reg, .{ .register = .rax });
7599 } else return self.fail("TODO emit ptr to TLV sequence on {s}", .{@tagName(self.bin_file.tag)});
7600 },
7601 .load_tlv => |sym_index| {
7602 const base_reg = switch (ty.zigTypeTag()) {
7603 .Float => (try self.register_manager.allocReg(null, gp)).to64(),
7604 else => reg.to64(),
7605 };
7606 try self.genSetReg(Type.usize, base_reg, .{ .lea_tlv = sym_index });
7607 switch (ty.zigTypeTag()) {
7608 .Float => if (intrinsicsAllowed(self.target.*, ty)) {
7609 return self.asmRegisterMemory(
7610 switch (ty.tag()) {
7611 .f32 => .movss,
7612 .f64 => .movsd,
7613 else => return self.fail("TODO genSetReg from memory for {}", .{
7614 ty.fmt(self.bin_file.options.module.?),
7615 }),
7616 },
7617 reg.to128(),
7618 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base_reg }),
7619 );
7620 } else return self.fail("TODO genSetReg from memory for float with no intrinsics", .{}),
7621 else => try self.asmRegisterMemory(
7622 .mov,
7623 registerAlias(reg, abi_size),
7624 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base_reg }),
7625 ),
7626 }
7627 },
7628 .stack_offset => |off| {
7629 switch (ty.zigTypeTag()) {
7630 .Int => switch (ty.intInfo(self.target.*).signedness) {
7631 .signed => {
7632 if (abi_size <= 4) {
7633 return self.asmRegisterMemory(
7634 .movsx,
7635 reg.to64(),
7636 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
7637 .base = .rbp,
7638 .disp = -off,
7639 }),
7640 );
7641 }
7642 },
7643 .unsigned => {
7644 if (abi_size <= 2) {
7645 return self.asmRegisterMemory(
7646 .movzx,
7647 reg.to64(),
7648 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
7649 .base = .rbp,
7650 .disp = -off,
7651 }),
7652 );
7653 }
7654 },
7655 },
7656 .Float => {
7657 if (intrinsicsAllowed(self.target.*, ty)) {
7658 const tag: Mir.Inst.Tag = switch (ty.tag()) {
7659 .f32 => .movss,
7660 .f64 => .movsd,
7661 else => return self.fail(
7662 "TODO genSetReg from stack offset for {}",
7663 .{ty.fmtDebug()},
7664 ),
7665 };
7666 return self.asmRegisterMemory(tag, reg.to128(), Memory.sib(
7667 Memory.PtrSize.fromSize(abi_size),
7668 .{ .base = .rbp, .disp = -off },
7669 ));
7670 }
7671 return self.fail("TODO genSetReg from stack offset for float with no intrinsics", .{});
7672 },
7673 else => {},
7674 }
7675
7676 try self.asmRegisterMemory(
7677 .mov,
7678 registerAlias(reg, abi_size),
7679 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .rbp, .disp = -off }),
7680 );
7681 },
7682 }
7683}
7684
7685fn airPtrToInt(self: *Self, inst: Air.Inst.Index) !void {7489fn airPtrToInt(self: *Self, inst: Air.Inst.Index) !void {
7686 const un_op = self.air.instructions.items(.data)[inst].un_op;7490 const un_op = self.air.instructions.items(.data)[inst].un_op;
7687 const result = result: {7491 const result = result: {
...@@ -7691,7 +7495,7 @@ fn airPtrToInt(self: *Self, inst: Air.Inst.Index) !void {...@@ -7691,7 +7495,7 @@ fn airPtrToInt(self: *Self, inst: Air.Inst.Index) !void {
76917495
7692 const dst_mcv = try self.allocRegOrMem(inst, true);7496 const dst_mcv = try self.allocRegOrMem(inst, true);
7693 const dst_ty = self.air.typeOfIndex(inst);7497 const dst_ty = self.air.typeOfIndex(inst);
7694 try self.setRegOrMem(dst_ty, dst_mcv, src_mcv);7498 try self.genCopy(dst_ty, dst_mcv, src_mcv);
7695 break :result dst_mcv;7499 break :result dst_mcv;
7696 };7500 };
7697 return self.finishAir(inst, result, .{ un_op, .none, .none });7501 return self.finishAir(inst, result, .{ un_op, .none, .none });
...@@ -7711,7 +7515,7 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {...@@ -7711,7 +7515,7 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
7711 defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);7515 defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
77127516
7713 const dest = try self.allocRegOrMem(inst, true);7517 const dest = try self.allocRegOrMem(inst, true);
7714 try self.setRegOrMem(self.air.typeOfIndex(inst), dest, operand);7518 try self.genCopy(self.air.typeOfIndex(inst), dest, operand);
7715 break :result dest;7519 break :result dest;
7716 };7520 };
7717 log.debug("airBitCast(%{d}): {}", .{ inst, result });7521 log.debug("airBitCast(%{d}): {}", .{ inst, result });
...@@ -7720,16 +7524,23 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {...@@ -7720,16 +7524,23 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
77207524
7721fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {7525fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
7722 const ty_op = self.air.instructions.items(.data)[inst].ty_op;7526 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
7527
7528 const slice_ty = self.air.typeOfIndex(inst);
7723 const ptr_ty = self.air.typeOf(ty_op.operand);7529 const ptr_ty = self.air.typeOf(ty_op.operand);
7724 const ptr = try self.resolveInst(ty_op.operand);7530 const ptr = try self.resolveInst(ty_op.operand);
7725 const array_ty = ptr_ty.childType();7531 const array_ty = ptr_ty.childType();
7726 const array_len = array_ty.arrayLen();7532 const array_len = array_ty.arrayLen();
77277533
7728 const stack_offset = @intCast(i32, try self.allocMem(inst, 16, 16));7534 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(slice_ty, self.target.*));
7729 try self.genSetStack(ptr_ty, stack_offset, ptr, .{});7535 try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr);
7730 try self.genSetStack(Type.u64, stack_offset - 8, .{ .immediate = array_len }, .{});7536 try self.genSetMem(
7537 .{ .frame = frame_index },
7538 @intCast(i32, ptr_ty.abiSize(self.target.*)),
7539 Type.usize,
7540 .{ .immediate = array_len },
7541 );
77317542
7732 const result = MCValue{ .stack_offset = stack_offset };7543 const result = MCValue{ .load_frame = .{ .index = frame_index } };
7733 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });7544 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
7734}7545}
77357546
...@@ -7758,29 +7569,31 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {...@@ -7758,29 +7569,31 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
7758 }7569 }
77597570
7760 // move float src to ST(0)7571 // move float src to ST(0)
7761 const stack_offset = switch (operand) {7572 const frame_addr: FrameAddr = switch (operand) {
7762 .stack_offset, .ptr_stack_offset => |offset| offset,7573 .load_frame => |frame_addr| frame_addr,
7763 else => blk: {7574 else => frame_addr: {
7764 const offset = @intCast(i32, try self.allocMem(7575 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(src_ty, self.target.*));
7765 inst,7576 try self.genSetMem(.{ .frame = frame_index }, 0, src_ty, operand);
7766 src_abi_size,7577 break :frame_addr .{ .index = frame_index };
7767 src_ty.abiAlignment(self.target.*),
7768 ));
7769 try self.genSetStack(src_ty, offset, operand, .{});
7770 break :blk offset;
7771 },7578 },
7772 };7579 };
7773 try self.asmMemory(.fld, Memory.sib(Memory.PtrSize.fromSize(src_abi_size), .{7580 try self.asmMemory(
7774 .base = .rbp,7581 .fld,
7775 .disp = -stack_offset,7582 Memory.sib(Memory.PtrSize.fromSize(src_abi_size), .{
7776 }));7583 .base = .{ .frame = frame_addr.index },
7584 .disp = frame_addr.off,
7585 }),
7586 );
77777587
7778 // convert7588 // convert
7779 const stack_dst = try self.allocRegOrMem(inst, false);7589 const stack_dst = try self.allocRegOrMem(inst, false);
7780 try self.asmMemory(.fisttp, Memory.sib(Memory.PtrSize.fromSize(dst_abi_size), .{7590 try self.asmMemory(
7781 .base = .rbp,7591 .fisttp,
7782 .disp = -stack_dst.stack_offset,7592 Memory.sib(Memory.PtrSize.fromSize(dst_abi_size), .{
7783 }));7593 .base = .{ .frame = stack_dst.load_frame.index },
7594 .disp = stack_dst.load_frame.off,
7595 }),
7596 );
77847597
7785 return self.finishAir(inst, stack_dst, .{ ty_op.operand, .none, .none });7598 return self.finishAir(inst, stack_dst, .{ ty_op.operand, .none, .none });
7786}7599}
...@@ -7800,20 +7613,32 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {...@@ -7800,20 +7613,32 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
78007613
7801 const exp_mcv = try self.resolveInst(extra.expected_value);7614 const exp_mcv = try self.resolveInst(extra.expected_value);
7802 if (val_abi_size > 8) switch (exp_mcv) {7615 if (val_abi_size > 8) switch (exp_mcv) {
7803 .stack_offset => |exp_off| {7616 .load_frame => |frame_addr| {
7804 try self.genSetReg(Type.usize, .rax, .{ .stack_offset = exp_off - 0 });7617 try self.genSetReg(.rax, Type.usize, .{ .load_frame = .{
7805 try self.genSetReg(Type.usize, .rdx, .{ .stack_offset = exp_off - 8 });7618 .index = frame_addr.index,
7619 .off = frame_addr.off + 0,
7620 } });
7621 try self.genSetReg(.rdx, Type.usize, .{ .load_frame = .{
7622 .index = frame_addr.index,
7623 .off = frame_addr.off + 8,
7624 } });
7806 },7625 },
7807 else => return self.fail("TODO implement cmpxchg for {s}", .{@tagName(exp_mcv)}),7626 else => return self.fail("TODO implement cmpxchg for {s}", .{@tagName(exp_mcv)}),
7808 } else try self.genSetReg(val_ty, .rax, exp_mcv);7627 } else try self.genSetReg(.rax, val_ty, exp_mcv);
7809 const rax_lock = self.register_manager.lockRegAssumeUnused(.rax);7628 const rax_lock = self.register_manager.lockRegAssumeUnused(.rax);
7810 defer self.register_manager.unlockReg(rax_lock);7629 defer self.register_manager.unlockReg(rax_lock);
78117630
7812 const new_mcv = try self.resolveInst(extra.new_value);7631 const new_mcv = try self.resolveInst(extra.new_value);
7813 const new_reg: Register = if (val_abi_size > 8) switch (new_mcv) {7632 const new_reg: Register = if (val_abi_size > 8) switch (new_mcv) {
7814 .stack_offset => |new_off| new: {7633 .load_frame => |frame_addr| new: {
7815 try self.genSetReg(Type.usize, .rbx, .{ .stack_offset = new_off - 0 });7634 try self.genSetReg(.rbx, Type.usize, .{ .load_frame = .{
7816 try self.genSetReg(Type.usize, .rcx, .{ .stack_offset = new_off - 8 });7635 .index = frame_addr.index,
7636 .off = frame_addr.off + 0,
7637 } });
7638 try self.genSetReg(.rcx, Type.usize, .{ .load_frame = .{
7639 .index = frame_addr.index,
7640 .off = frame_addr.off + 8,
7641 } });
7817 break :new undefined;7642 break :new undefined;
7818 },7643 },
7819 else => return self.fail("TODO implement cmpxchg for {s}", .{@tagName(exp_mcv)}),7644 else => return self.fail("TODO implement cmpxchg for {s}", .{@tagName(exp_mcv)}),
...@@ -7823,11 +7648,19 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {...@@ -7823,11 +7648,19 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
78237648
7824 const ptr_size = Memory.PtrSize.fromSize(val_abi_size);7649 const ptr_size = Memory.PtrSize.fromSize(val_abi_size);
7825 const ptr_mem = switch (ptr_mcv) {7650 const ptr_mem = switch (ptr_mcv) {
7826 .register => |reg| Memory.sib(ptr_size, .{ .base = reg }),7651 .register => |reg| Memory.sib(ptr_size, .{ .base = .{ .reg = reg } }),
7827 .ptr_stack_offset => |off| Memory.sib(ptr_size, .{ .base = .rbp, .disp = -off }),7652 .lea_frame => |frame_addr| Memory.sib(ptr_size, .{
7828 else => Memory.sib(ptr_size, .{ .base = try self.copyToTmpRegister(ptr_ty, ptr_mcv) }),7653 .base = .{ .frame = frame_addr.index },
7654 .disp = frame_addr.off,
7655 }),
7656 else => Memory.sib(ptr_size, .{ .base = .{
7657 .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv),
7658 } }),
7659 };
7660 const mem_lock = switch (ptr_mem.base()) {
7661 .none, .frame => null,
7662 .reg => |reg| self.register_manager.lockReg(reg),
7829 };7663 };
7830 const mem_lock = if (ptr_mem.base()) |reg| self.register_manager.lockReg(reg) else null;
7831 defer if (mem_lock) |lock| self.register_manager.unlockReg(lock);7664 defer if (mem_lock) |lock| self.register_manager.unlockReg(lock);
78327665
7833 try self.spillEflagsIfOccupied();7666 try self.spillEflagsIfOccupied();
...@@ -7851,9 +7684,24 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {...@@ -7851,9 +7684,24 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
7851 }7684 }
78527685
7853 const dst_mcv = try self.allocRegOrMem(inst, false);7686 const dst_mcv = try self.allocRegOrMem(inst, false);
7854 try self.genSetStack(Type.bool, dst_mcv.stack_offset - 16, .{ .eflags = .ne }, .{});7687 try self.genSetMem(
7855 try self.genSetStack(Type.usize, dst_mcv.stack_offset - 8, .{ .register = .rdx }, .{});7688 .{ .frame = dst_mcv.load_frame.index },
7856 try self.genSetStack(Type.usize, dst_mcv.stack_offset - 0, .{ .register = .rax }, .{});7689 dst_mcv.load_frame.off + 16,
7690 Type.bool,
7691 .{ .eflags = .ne },
7692 );
7693 try self.genSetMem(
7694 .{ .frame = dst_mcv.load_frame.index },
7695 dst_mcv.load_frame.off + 8,
7696 Type.usize,
7697 .{ .register = .rdx },
7698 );
7699 try self.genSetMem(
7700 .{ .frame = dst_mcv.load_frame.index },
7701 dst_mcv.load_frame.off + 0,
7702 Type.usize,
7703 .{ .register = .rax },
7704 );
7857 break :result dst_mcv;7705 break :result dst_mcv;
7858 };7706 };
7859 return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });7707 return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });
...@@ -7884,11 +7732,19 @@ fn atomicOp(...@@ -7884,11 +7732,19 @@ fn atomicOp(
7884 const val_abi_size = @intCast(u32, val_ty.abiSize(self.target.*));7732 const val_abi_size = @intCast(u32, val_ty.abiSize(self.target.*));
7885 const ptr_size = Memory.PtrSize.fromSize(val_abi_size);7733 const ptr_size = Memory.PtrSize.fromSize(val_abi_size);
7886 const ptr_mem = switch (ptr_mcv) {7734 const ptr_mem = switch (ptr_mcv) {
7887 .register => |reg| Memory.sib(ptr_size, .{ .base = reg }),7735 .register => |reg| Memory.sib(ptr_size, .{ .base = .{ .reg = reg } }),
7888 .ptr_stack_offset => |off| Memory.sib(ptr_size, .{ .base = .rbp, .disp = -off }),7736 .lea_frame => |frame_addr| Memory.sib(ptr_size, .{
7889 else => Memory.sib(ptr_size, .{ .base = try self.copyToTmpRegister(ptr_ty, ptr_mcv) }),7737 .base = .{ .frame = frame_addr.index },
7738 .disp = frame_addr.off,
7739 }),
7740 else => Memory.sib(ptr_size, .{ .base = .{
7741 .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv),
7742 } }),
7743 };
7744 const mem_lock = switch (ptr_mem.base()) {
7745 .none, .frame => null,
7746 .reg => |reg| self.register_manager.lockReg(reg),
7890 };7747 };
7891 const mem_lock = if (ptr_mem.base()) |reg| self.register_manager.lockReg(reg) else null;
7892 defer if (mem_lock) |lock| self.register_manager.unlockReg(lock);7748 defer if (mem_lock) |lock| self.register_manager.unlockReg(lock);
78937749
7894 const method: enum { lock, loop, libcall } = if (val_ty.isRuntimeFloat())7750 const method: enum { lock, loop, libcall } = if (val_ty.isRuntimeFloat())
...@@ -7928,7 +7784,7 @@ fn atomicOp(...@@ -7928,7 +7784,7 @@ fn atomicOp(
7928 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);7784 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
7929 defer self.register_manager.unlockReg(dst_lock);7785 defer self.register_manager.unlockReg(dst_lock);
79307786
7931 try self.genSetReg(val_ty, dst_reg, val_mcv);7787 try self.genSetReg(dst_reg, val_ty, val_mcv);
7932 if (rmw_op == std.builtin.AtomicRmwOp.Sub and tag == .xadd) {7788 if (rmw_op == std.builtin.AtomicRmwOp.Sub and tag == .xadd) {
7933 try self.genUnOpMir(.neg, val_ty, dst_mcv);7789 try self.genUnOpMir(.neg, val_ty, dst_mcv);
7934 }7790 }
...@@ -7952,10 +7808,10 @@ fn atomicOp(...@@ -7952,10 +7808,10 @@ fn atomicOp(
7952 try self.asmRegisterMemory(.mov, registerAlias(.rax, val_abi_size), ptr_mem);7808 try self.asmRegisterMemory(.mov, registerAlias(.rax, val_abi_size), ptr_mem);
7953 const loop = @intCast(u32, self.mir_instructions.len);7809 const loop = @intCast(u32, self.mir_instructions.len);
7954 if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {7810 if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
7955 try self.genSetReg(val_ty, tmp_reg, .{ .register = .rax });7811 try self.genSetReg(tmp_reg, val_ty, .{ .register = .rax });
7956 }7812 }
7957 if (rmw_op) |op| switch (op) {7813 if (rmw_op) |op| switch (op) {
7958 .Xchg => try self.genSetReg(val_ty, tmp_reg, val_mcv),7814 .Xchg => try self.genSetReg(tmp_reg, val_ty, val_mcv),
7959 .Add => try self.genBinOpMir(.add, val_ty, tmp_mcv, val_mcv),7815 .Add => try self.genBinOpMir(.add, val_ty, tmp_mcv, val_mcv),
7960 .Sub => try self.genBinOpMir(.sub, val_ty, tmp_mcv, val_mcv),7816 .Sub => try self.genBinOpMir(.sub, val_ty, tmp_mcv, val_mcv),
7961 .And => try self.genBinOpMir(.@"and", val_ty, tmp_mcv, val_mcv),7817 .And => try self.genBinOpMir(.@"and", val_ty, tmp_mcv, val_mcv),
...@@ -7990,12 +7846,12 @@ fn atomicOp(...@@ -7990,12 +7846,12 @@ fn atomicOp(
7990 registerAlias(val_reg, cmov_abi_size),7846 registerAlias(val_reg, cmov_abi_size),
7991 cc,7847 cc,
7992 ),7848 ),
7993 .stack_offset => |val_off| try self.asmCmovccRegisterMemory(7849 .load_frame => |frame_addr| try self.asmCmovccRegisterMemory(
7994 registerAlias(tmp_reg, cmov_abi_size),7850 registerAlias(tmp_reg, cmov_abi_size),
7995 Memory.sib(7851 Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), .{
7996 Memory.PtrSize.fromSize(cmov_abi_size),7852 .base = .{ .frame = frame_addr.index },
7997 .{ .base = .rbp, .disp = -val_off },7853 .disp = frame_addr.off,
7998 ),7854 }),
7999 cc,7855 cc,
8000 ),7856 ),
8001 else => {7857 else => {
...@@ -8028,9 +7884,15 @@ fn atomicOp(...@@ -8028,9 +7884,15 @@ fn atomicOp(
8028 }));7884 }));
8029 const loop = @intCast(u32, self.mir_instructions.len);7885 const loop = @intCast(u32, self.mir_instructions.len);
8030 switch (val_mcv) {7886 switch (val_mcv) {
8031 .stack_offset => |val_off| {7887 .load_frame => |frame_addr| {
8032 const val_lo_mem = Memory.sib(.qword, .{ .base = .rbp, .disp = 0 - val_off });7888 const val_lo_mem = Memory.sib(.qword, .{
8033 const val_hi_mem = Memory.sib(.qword, .{ .base = .rbp, .disp = 8 - val_off });7889 .base = .{ .frame = frame_addr.index },
7890 .disp = frame_addr.off + 0,
7891 });
7892 const val_hi_mem = Memory.sib(.qword, .{
7893 .base = .{ .frame = frame_addr.index },
7894 .disp = frame_addr.off + 8,
7895 });
80347896
8035 if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {7897 if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
8036 try self.asmRegisterRegister(.mov, .rbx, .rax);7898 try self.asmRegisterRegister(.mov, .rbx, .rax);
...@@ -8087,12 +7949,18 @@ fn atomicOp(...@@ -8087,12 +7949,18 @@ fn atomicOp(
8087 const dst_mcv = try self.allocTempRegOrMem(val_ty, false);7949 const dst_mcv = try self.allocTempRegOrMem(val_ty, false);
8088 try self.asmMemoryRegister(7950 try self.asmMemoryRegister(
8089 .mov,7951 .mov,
8090 Memory.sib(.qword, .{ .base = .rbp, .disp = 0 - dst_mcv.stack_offset }),7952 Memory.sib(.qword, .{
7953 .base = .{ .frame = dst_mcv.load_frame.index },
7954 .disp = dst_mcv.load_frame.off + 0,
7955 }),
8091 .rax,7956 .rax,
8092 );7957 );
8093 try self.asmMemoryRegister(7958 try self.asmMemoryRegister(
8094 .mov,7959 .mov,
8095 Memory.sib(.qword, .{ .base = .rbp, .disp = 8 - dst_mcv.stack_offset }),7960 Memory.sib(.qword, .{
7961 .base = .{ .frame = dst_mcv.load_frame.index },
7962 .disp = dst_mcv.load_frame.off + 8,
7963 }),
8096 .rdx,7964 .rdx,
8097 );7965 );
8098 return dst_mcv;7966 return dst_mcv;
...@@ -8139,7 +8007,7 @@ fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {...@@ -8139,7 +8007,7 @@ fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {
8139 else8007 else
8140 try self.allocRegOrMem(inst, true);8008 try self.allocRegOrMem(inst, true);
81418009
8142 try self.load(dst_mcv, ptr_mcv, ptr_ty);8010 try self.load(dst_mcv, ptr_ty, ptr_mcv);
8143 return self.finishAir(inst, dst_mcv, .{ atomic_load.ptr, .none, .none });8011 return self.finishAir(inst, dst_mcv, .{ atomic_load.ptr, .none, .none });
8144}8012}
81458013
...@@ -8184,16 +8052,16 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {...@@ -8184,16 +8052,16 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
8184 const elem_abi_size = @intCast(u31, elem_ty.abiSize(self.target.*));8052 const elem_abi_size = @intCast(u31, elem_ty.abiSize(self.target.*));
81858053
8186 if (elem_abi_size == 1) {8054 if (elem_abi_size == 1) {
8187 const ptr = switch (dst_ptr_ty.ptrSize()) {8055 const ptr: MCValue = switch (dst_ptr_ty.ptrSize()) {
8188 // TODO: this only handles slices stored in the stack8056 // TODO: this only handles slices stored in the stack
8189 .Slice => @as(MCValue, .{ .stack_offset = dst_ptr.stack_offset - 0 }),8057 .Slice => dst_ptr,
8190 .One => dst_ptr,8058 .One => dst_ptr,
8191 .C, .Many => unreachable,8059 .C, .Many => unreachable,
8192 };8060 };
8193 const len = switch (dst_ptr_ty.ptrSize()) {8061 const len: MCValue = switch (dst_ptr_ty.ptrSize()) {
8194 // TODO: this only handles slices stored in the stack8062 // TODO: this only handles slices stored in the stack
8195 .Slice => @as(MCValue, .{ .stack_offset = dst_ptr.stack_offset - 8 }),8063 .Slice => dst_ptr.address().offset(8).deref(),
8196 .One => @as(MCValue, .{ .immediate = dst_ptr_ty.childType().arrayLen() }),8064 .One => .{ .immediate = dst_ptr_ty.childType().arrayLen() },
8197 .C, .Many => unreachable,8065 .C, .Many => unreachable,
8198 };8066 };
8199 const len_lock: ?RegisterLock = switch (len) {8067 const len_lock: ?RegisterLock = switch (len) {
...@@ -8202,7 +8070,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {...@@ -8202,7 +8070,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
8202 };8070 };
8203 defer if (len_lock) |lock| self.register_manager.unlockReg(lock);8071 defer if (len_lock) |lock| self.register_manager.unlockReg(lock);
82048072
8205 try self.genInlineMemset(ptr, src_val, len, .{});8073 try self.genInlineMemset(ptr, src_val, len);
8206 return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });8074 return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
8207 }8075 }
82088076
...@@ -8215,8 +8083,8 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {...@@ -8215,8 +8083,8 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
8215 const slice_ptr_ty = dst_ptr_ty.slicePtrFieldType(&buf);8083 const slice_ptr_ty = dst_ptr_ty.slicePtrFieldType(&buf);
82168084
8217 // TODO: this only handles slices stored in the stack8085 // TODO: this only handles slices stored in the stack
8218 const ptr = @as(MCValue, .{ .stack_offset = dst_ptr.stack_offset - 0 });8086 const ptr = dst_ptr;
8219 const len = @as(MCValue, .{ .stack_offset = dst_ptr.stack_offset - 8 });8087 const len = dst_ptr.address().offset(8).deref();
82208088
8221 // Used to store the number of elements for comparison.8089 // Used to store the number of elements for comparison.
8222 // After comparison, updated to store number of bytes needed to copy.8090 // After comparison, updated to store number of bytes needed to copy.
...@@ -8225,55 +8093,51 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {...@@ -8225,55 +8093,51 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
8225 const len_lock = self.register_manager.lockRegAssumeUnused(len_reg);8093 const len_lock = self.register_manager.lockRegAssumeUnused(len_reg);
8226 defer self.register_manager.unlockReg(len_lock);8094 defer self.register_manager.unlockReg(len_lock);
82278095
8228 try self.asmRegisterMemory(.mov, len_reg, Memory.sib(.qword, .{8096 try self.genSetReg(len_reg, Type.usize, len);
8229 .base = .rbp,
8230 .disp = -len.stack_offset,
8231 }));
82328097
8233 const skip_reloc = try self.asmJccReloc(undefined, .z);8098 const skip_reloc = try self.asmJccReloc(undefined, .z);
8234 try self.store(ptr, src_val, slice_ptr_ty, elem_ty);8099 try self.store(slice_ptr_ty, ptr, src_val);
82358100
8236 const second_elem_ptr_reg = try self.register_manager.allocReg(null, gp);8101 const second_elem_ptr_reg = try self.register_manager.allocReg(null, gp);
8237 const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg };8102 const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg };
8238 const second_elem_ptr_lock = self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg);8103 const second_elem_ptr_lock = self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg);
8239 defer self.register_manager.unlockReg(second_elem_ptr_lock);8104 defer self.register_manager.unlockReg(second_elem_ptr_lock);
82408105
8241 try self.asmRegisterMemory(8106 try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{
8242 .lea,8107 .reg = try self.copyToTmpRegister(Type.usize, ptr),
8243 second_elem_ptr_reg,8108 .off = elem_abi_size,
8244 Memory.sib(.qword, .{8109 } });
8245 .base = try self.copyToTmpRegister(Type.usize, ptr),
8246 .disp = elem_abi_size,
8247 }),
8248 );
82498110
8250 try self.genBinOpMir(.sub, Type.usize, len_mcv, .{ .immediate = 1 });8111 try self.genBinOpMir(.sub, Type.usize, len_mcv, .{ .immediate = 1 });
8251 try self.asmRegisterRegisterImmediate(.imul, len_reg, len_reg, Immediate.u(elem_abi_size));8112 try self.asmRegisterRegisterImmediate(.imul, len_reg, len_reg, Immediate.u(elem_abi_size));
8252 try self.genInlineMemcpy(second_elem_ptr_mcv, ptr, len_mcv, .{});8113 try self.genInlineMemcpy(second_elem_ptr_mcv, ptr, len_mcv);
82538114
8254 try self.performReloc(skip_reloc);8115 try self.performReloc(skip_reloc);
8255 },8116 },
8256 .One => {8117 .One => {
8118 var elem_ptr_pl = Type.Payload.ElemType{
8119 .base = .{ .tag = .single_mut_pointer },
8120 .data = elem_ty,
8121 };
8122 const elem_ptr_ty = Type.initPayload(&elem_ptr_pl.base);
8123
8257 const len = dst_ptr_ty.childType().arrayLen();8124 const len = dst_ptr_ty.childType().arrayLen();
8125
8258 assert(len != 0); // prevented by Sema8126 assert(len != 0); // prevented by Sema
8259 try self.store(dst_ptr, src_val, dst_ptr_ty, elem_ty);8127 try self.store(elem_ptr_ty, dst_ptr, src_val);
82608128
8261 const second_elem_ptr_reg = try self.register_manager.allocReg(null, gp);8129 const second_elem_ptr_reg = try self.register_manager.allocReg(null, gp);
8262 const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg };8130 const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg };
8263 const second_elem_ptr_lock = self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg);8131 const second_elem_ptr_lock = self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg);
8264 defer self.register_manager.unlockReg(second_elem_ptr_lock);8132 defer self.register_manager.unlockReg(second_elem_ptr_lock);
82658133
8266 try self.asmRegisterMemory(8134 try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{
8267 .lea,8135 .reg = try self.copyToTmpRegister(Type.usize, dst_ptr),
8268 second_elem_ptr_reg,8136 .off = elem_abi_size,
8269 Memory.sib(.qword, .{8137 } });
8270 .base = try self.copyToTmpRegister(Type.usize, dst_ptr),
8271 .disp = elem_abi_size,
8272 }),
8273 );
82748138
8275 const bytes_to_copy: MCValue = .{ .immediate = elem_abi_size * (len - 1) };8139 const bytes_to_copy: MCValue = .{ .immediate = elem_abi_size * (len - 1) };
8276 try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, bytes_to_copy, .{});8140 try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, bytes_to_copy);
8277 },8141 },
8278 .C, .Many => unreachable,8142 .C, .Many => unreachable,
8279 }8143 }
...@@ -8299,9 +8163,9 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {...@@ -8299,9 +8163,9 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
8299 };8163 };
8300 defer if (src_ptr_lock) |lock| self.register_manager.unlockReg(lock);8164 defer if (src_ptr_lock) |lock| self.register_manager.unlockReg(lock);
83018165
8302 const len = switch (dst_ptr_ty.ptrSize()) {8166 const len: MCValue = switch (dst_ptr_ty.ptrSize()) {
8303 .Slice => @as(MCValue, .{ .stack_offset = dst_ptr.stack_offset - 8 }),8167 .Slice => dst_ptr.address().offset(8).deref(),
8304 .One => @as(MCValue, .{ .immediate = dst_ptr_ty.childType().arrayLen() }),8168 .One => .{ .immediate = dst_ptr_ty.childType().arrayLen() },
8305 .C, .Many => unreachable,8169 .C, .Many => unreachable,
8306 };8170 };
8307 const len_lock: ?RegisterLock = switch (len) {8171 const len_lock: ?RegisterLock = switch (len) {
...@@ -8311,7 +8175,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {...@@ -8311,7 +8175,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
8311 defer if (len_lock) |lock| self.register_manager.unlockReg(lock);8175 defer if (len_lock) |lock| self.register_manager.unlockReg(lock);
83128176
8313 // TODO: dst_ptr and src_ptr could be slices rather than raw pointers8177 // TODO: dst_ptr and src_ptr could be slices rather than raw pointers
8314 try self.genInlineMemcpy(dst_ptr, src_ptr, len, .{});8178 try self.genInlineMemcpy(dst_ptr, src_ptr, len);
83158179
8316 return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });8180 return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
8317}8181}
...@@ -8345,24 +8209,25 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {...@@ -8345,24 +8209,25 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
8345 const atom = elf_file.getAtom(atom_index);8209 const atom = elf_file.getAtom(atom_index);
8346 _ = try atom.getOrCreateOffsetTableEntry(elf_file);8210 _ = try atom.getOrCreateOffsetTableEntry(elf_file);
8347 const got_addr = atom.getOffsetTableAddress(elf_file);8211 const got_addr = atom.getOffsetTableAddress(elf_file);
8348 try self.asmRegisterMemory(.mov, addr_reg.to64(), Memory.sib(.qword, .{8212 try self.asmRegisterMemory(
8349 .base = .ds,8213 .mov,
8350 .disp = @intCast(i32, got_addr),8214 addr_reg.to64(),
8351 }));8215 Memory.sib(.qword, .{ .base = .{ .reg = .ds }, .disp = @intCast(i32, got_addr) }),
8216 );
8352 } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {8217 } else if (self.bin_file.cast(link.File.Coff)) |coff_file| {
8353 const atom_index = try coff_file.getOrCreateAtomForLazySymbol(8218 const atom_index = try coff_file.getOrCreateAtomForLazySymbol(
8354 .{ .kind = .const_data, .ty = Type.anyerror },8219 .{ .kind = .const_data, .ty = Type.anyerror },
8355 4, // dword alignment8220 4, // dword alignment
8356 );8221 );
8357 const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;8222 const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
8358 try self.genSetReg(Type.usize, addr_reg, .{ .load_got = sym_index });8223 try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
8359 } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {8224 } else if (self.bin_file.cast(link.File.MachO)) |macho_file| {
8360 const atom_index = try macho_file.getOrCreateAtomForLazySymbol(8225 const atom_index = try macho_file.getOrCreateAtomForLazySymbol(
8361 .{ .kind = .const_data, .ty = Type.anyerror },8226 .{ .kind = .const_data, .ty = Type.anyerror },
8362 4, // dword alignment8227 4, // dword alignment
8363 );8228 );
8364 const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;8229 const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;
8365 try self.genSetReg(Type.usize, addr_reg, .{ .load_got = sym_index });8230 try self.genSetReg(addr_reg, Type.usize, .{ .lea_got = sym_index });
8366 } else {8231 } else {
8367 return self.fail("TODO implement airErrorName for x86_64 {s}", .{@tagName(self.bin_file.tag)});8232 return self.fail("TODO implement airErrorName for x86_64 {s}", .{@tagName(self.bin_file.tag)});
8368 }8233 }
...@@ -8377,36 +8242,60 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {...@@ -8377,36 +8242,60 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
83778242
8378 try self.truncateRegister(err_ty, err_reg.to32());8243 try self.truncateRegister(err_ty, err_reg.to32());
83798244
8380 try self.asmRegisterMemory(.mov, start_reg.to32(), Memory.sib(.dword, .{8245 try self.asmRegisterMemory(
8381 .base = addr_reg.to64(),8246 .mov,
8382 .scale_index = .{ .scale = 4, .index = err_reg.to64() },8247 start_reg.to32(),
8383 .disp = 4,8248 Memory.sib(.dword, .{
8384 }));8249 .base = .{ .reg = addr_reg.to64() },
8385 try self.asmRegisterMemory(.mov, end_reg.to32(), Memory.sib(.dword, .{8250 .scale_index = .{ .scale = 4, .index = err_reg.to64() },
8386 .base = addr_reg.to64(),8251 .disp = 4,
8387 .scale_index = .{ .scale = 4, .index = err_reg.to64() },8252 }),
8388 .disp = 8,8253 );
8389 }));8254 try self.asmRegisterMemory(
8255 .mov,
8256 end_reg.to32(),
8257 Memory.sib(.dword, .{
8258 .base = .{ .reg = addr_reg.to64() },
8259 .scale_index = .{ .scale = 4, .index = err_reg.to64() },
8260 .disp = 8,
8261 }),
8262 );
8390 try self.asmRegisterRegister(.sub, end_reg.to32(), start_reg.to32());8263 try self.asmRegisterRegister(.sub, end_reg.to32(), start_reg.to32());
8391 try self.asmRegisterMemory(.lea, start_reg.to64(), Memory.sib(.byte, .{8264 try self.asmRegisterMemory(
8392 .base = addr_reg.to64(),8265 .lea,
8393 .scale_index = .{ .scale = 1, .index = start_reg.to64() },8266 start_reg.to64(),
8394 .disp = 0,8267 Memory.sib(.byte, .{
8395 }));8268 .base = .{ .reg = addr_reg.to64() },
8396 try self.asmRegisterMemory(.lea, end_reg.to32(), Memory.sib(.byte, .{8269 .scale_index = .{ .scale = 1, .index = start_reg.to64() },
8397 .base = end_reg.to64(),8270 .disp = 0,
8398 .disp = -1,8271 }),
8399 }));8272 );
8273 try self.asmRegisterMemory(
8274 .lea,
8275 end_reg.to32(),
8276 Memory.sib(.byte, .{
8277 .base = .{ .reg = end_reg.to64() },
8278 .disp = -1,
8279 }),
8280 );
84008281
8401 const dst_mcv = try self.allocRegOrMem(inst, false);8282 const dst_mcv = try self.allocRegOrMem(inst, false);
8402 try self.asmMemoryRegister(.mov, Memory.sib(.qword, .{8283 try self.asmMemoryRegister(
8403 .base = .rbp,8284 .mov,
8404 .disp = 0 - dst_mcv.stack_offset,8285 Memory.sib(.qword, .{
8405 }), start_reg.to64());8286 .base = .{ .frame = dst_mcv.load_frame.index },
8406 try self.asmMemoryRegister(.mov, Memory.sib(.qword, .{8287 .disp = dst_mcv.load_frame.off,
8407 .base = .rbp,8288 }),
8408 .disp = 8 - dst_mcv.stack_offset,8289 start_reg.to64(),
8409 }), end_reg.to64());8290 );
8291 try self.asmMemoryRegister(
8292 .mov,
8293 Memory.sib(.qword, .{
8294 .base = .{ .frame = dst_mcv.load_frame.index },
8295 .disp = dst_mcv.load_frame.off + 8,
8296 }),
8297 end_reg.to64(),
8298 );
84108299
8411 return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });8300 return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
8412}8301}
...@@ -8445,19 +8334,17 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -8445,19 +8334,17 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
8445 const len = @intCast(usize, result_ty.arrayLen());8334 const len = @intCast(usize, result_ty.arrayLen());
8446 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;8335 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
8447 const elements = @ptrCast([]const Air.Inst.Ref, self.air.extra[ty_pl.payload..][0..len]);8336 const elements = @ptrCast([]const Air.Inst.Ref, self.air.extra[ty_pl.payload..][0..len]);
8448 const abi_size = @intCast(u32, result_ty.abiSize(self.target.*));
8449 const abi_align = result_ty.abiAlignment(self.target.*);
8450 const result: MCValue = result: {8337 const result: MCValue = result: {
8451 switch (result_ty.zigTypeTag()) {8338 switch (result_ty.zigTypeTag()) {
8452 .Struct => {8339 .Struct => {
8453 const stack_offset = @intCast(i32, try self.allocMem(inst, abi_size, abi_align));8340 const frame_index =
8341 try self.allocFrameIndex(FrameAlloc.initType(result_ty, self.target.*));
8454 if (result_ty.containerLayout() == .Packed) {8342 if (result_ty.containerLayout() == .Packed) {
8455 const struct_obj = result_ty.castTag(.@"struct").?.data;8343 const struct_obj = result_ty.castTag(.@"struct").?.data;
8456 try self.genInlineMemset(8344 try self.genInlineMemset(
8457 .{ .ptr_stack_offset = stack_offset },8345 .{ .lea_frame = .{ .index = frame_index } },
8458 .{ .immediate = 0 },8346 .{ .immediate = 0 },
8459 .{ .immediate = abi_size },8347 .{ .immediate = result_ty.abiSize(self.target.*) },
8460 .{},
8461 );8348 );
8462 for (elements, 0..) |elem, elem_i| {8349 for (elements, 0..) |elem, elem_i| {
8463 if (result_ty.structFieldValueComptime(elem_i) != null) continue;8350 if (result_ty.structFieldValueComptime(elem_i) != null) continue;
...@@ -8465,7 +8352,10 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -8465,7 +8352,10 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
8465 const elem_ty = result_ty.structFieldType(elem_i);8352 const elem_ty = result_ty.structFieldType(elem_i);
8466 const elem_bit_size = @intCast(u32, elem_ty.bitSize(self.target.*));8353 const elem_bit_size = @intCast(u32, elem_ty.bitSize(self.target.*));
8467 if (elem_bit_size > 64) {8354 if (elem_bit_size > 64) {
8468 return self.fail("TODO airAggregateInit implement packed structs with large fields", .{});8355 return self.fail(
8356 "TODO airAggregateInit implement packed structs with large fields",
8357 .{},
8358 );
8469 }8359 }
8470 const elem_abi_size = @intCast(u32, elem_ty.abiSize(self.target.*));8360 const elem_abi_size = @intCast(u32, elem_ty.abiSize(self.target.*));
8471 const elem_abi_bits = elem_abi_size * 8;8361 const elem_abi_bits = elem_abi_size * 8;
...@@ -8503,7 +8393,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -8503,7 +8393,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
8503 try self.genBinOpMir(8393 try self.genBinOpMir(
8504 .@"or",8394 .@"or",
8505 elem_ty,8395 elem_ty,
8506 .{ .stack_offset = stack_offset - elem_byte_off },8396 .{ .load_frame = .{ .index = frame_index, .off = elem_byte_off } },
8507 .{ .register = elem_reg },8397 .{ .register = elem_reg },
8508 );8398 );
8509 if (elem_bit_off > elem_extra_bits) {8399 if (elem_bit_off > elem_extra_bits) {
...@@ -8520,8 +8410,10 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -8520,8 +8410,10 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
8520 try self.genBinOpMir(8410 try self.genBinOpMir(
8521 .@"or",8411 .@"or",
8522 elem_ty,8412 elem_ty,
8523 .{ .stack_offset = stack_offset - elem_byte_off -8413 .{ .load_frame = .{
8524 @intCast(i32, elem_abi_size) },8414 .index = frame_index,
8415 .off = elem_byte_off + @intCast(i32, elem_abi_size),
8416 } },
8525 .{ .register = reg },8417 .{ .register = reg },
8526 );8418 );
8527 }8419 }
...@@ -8536,12 +8428,13 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -8536,12 +8428,13 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
8536 .load_tlv => |sym_index| MCValue{ .lea_tlv = sym_index },8428 .load_tlv => |sym_index| MCValue{ .lea_tlv = sym_index },
8537 else => elem_mcv,8429 else => elem_mcv,
8538 };8430 };
8539 try self.genSetStack(elem_ty, stack_offset - elem_off, mat_elem_mcv, .{});8431 try self.genSetMem(.{ .frame = frame_index }, elem_off, elem_ty, mat_elem_mcv);
8540 }8432 }
8541 break :result .{ .stack_offset = stack_offset };8433 break :result .{ .load_frame = .{ .index = frame_index } };
8542 },8434 },
8543 .Array => {8435 .Array => {
8544 const stack_offset = @intCast(i32, try self.allocMem(inst, abi_size, abi_align));8436 const frame_index =
8437 try self.allocFrameIndex(FrameAlloc.initType(result_ty, self.target.*));
8545 const elem_ty = result_ty.childType();8438 const elem_ty = result_ty.childType();
8546 const elem_size = @intCast(u32, elem_ty.abiSize(self.target.*));8439 const elem_size = @intCast(u32, elem_ty.abiSize(self.target.*));
85478440
...@@ -8552,9 +8445,9 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -8552,9 +8445,9 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
8552 else => elem_mcv,8445 else => elem_mcv,
8553 };8446 };
8554 const elem_off = @intCast(i32, elem_size * elem_i);8447 const elem_off = @intCast(i32, elem_size * elem_i);
8555 try self.genSetStack(elem_ty, stack_offset - elem_off, mat_elem_mcv, .{});8448 try self.genSetMem(.{ .frame = frame_index }, elem_off, elem_ty, mat_elem_mcv);
8556 }8449 }
8557 break :result MCValue{ .stack_offset = stack_offset };8450 break :result .{ .load_frame = .{ .index = frame_index } };
8558 },8451 },
8559 .Vector => return self.fail("TODO implement aggregate_init for vectors", .{}),8452 .Vector => return self.fail("TODO implement aggregate_init for vectors", .{}),
8560 else => unreachable,8453 else => unreachable,
...@@ -8596,7 +8489,7 @@ fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {...@@ -8596,7 +8489,7 @@ fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {
8596 const ty = self.air.typeOf(ref);8489 const ty = self.air.typeOf(ref);
85978490
8598 // If the type has no codegen bits, no need to store it.8491 // If the type has no codegen bits, no need to store it.
8599 if (!ty.hasRuntimeBitsIgnoreComptime() and !ty.isError()) return .none;8492 if (!ty.hasRuntimeBitsIgnoreComptime()) return .none;
86008493
8601 if (Air.refToIndex(ref)) |inst| {8494 if (Air.refToIndex(ref)) |inst| {
8602 const mcv = switch (self.air.instructions.items(.tag)[inst]) {8495 const mcv = switch (self.air.instructions.items(.tag)[inst]) {
...@@ -8666,7 +8559,7 @@ fn genTypedValue(self: *Self, arg_tv: TypedValue) InnerError!MCValue {...@@ -8666,7 +8559,7 @@ fn genTypedValue(self: *Self, arg_tv: TypedValue) InnerError!MCValue {
8666 .immediate => |imm| .{ .immediate = imm },8559 .immediate => |imm| .{ .immediate = imm },
8667 .memory => |addr| .{ .memory = addr },8560 .memory => |addr| .{ .memory = addr },
8668 .load_direct => |sym_index| .{ .load_direct = sym_index },8561 .load_direct => |sym_index| .{ .load_direct = sym_index },
8669 .load_got => |sym_index| .{ .load_got = sym_index },8562 .load_got => |sym_index| .{ .lea_got = sym_index },
8670 .load_tlv => |sym_index| .{ .load_tlv = sym_index },8563 .load_tlv => |sym_index| .{ .load_tlv = sym_index },
8671 },8564 },
8672 .fail => |msg| {8565 .fail => |msg| {
...@@ -8679,9 +8572,9 @@ fn genTypedValue(self: *Self, arg_tv: TypedValue) InnerError!MCValue {...@@ -8679,9 +8572,9 @@ fn genTypedValue(self: *Self, arg_tv: TypedValue) InnerError!MCValue {
86798572
8680const CallMCValues = struct {8573const CallMCValues = struct {
8681 args: []MCValue,8574 args: []MCValue,
8682 return_value: MCValue,8575 return_value: InstTracking,
8683 stack_byte_count: u32,8576 stack_byte_count: u31,
8684 stack_align: u32,8577 stack_align: u31,
86858578
8686 fn deinit(self: *CallMCValues, func: *Self) void {8579 fn deinit(self: *CallMCValues, func: *Self) void {
8687 func.gpa.free(self.args);8580 func.gpa.free(self.args);
...@@ -8694,6 +8587,7 @@ fn resolveCallingConventionValues(...@@ -8694,6 +8587,7 @@ fn resolveCallingConventionValues(
8694 self: *Self,8587 self: *Self,
8695 fn_ty: Type,8588 fn_ty: Type,
8696 var_args: []const Air.Inst.Ref,8589 var_args: []const Air.Inst.Ref,
8590 stack_frame_base: FrameIndex,
8697) !CallMCValues {8591) !CallMCValues {
8698 const cc = fn_ty.fnCallingConvention();8592 const cc = fn_ty.fnCallingConvention();
8699 const param_len = fn_ty.fnParamLen();8593 const param_len = fn_ty.fnParamLen();
...@@ -8706,7 +8600,7 @@ fn resolveCallingConventionValues(...@@ -8706,7 +8600,7 @@ fn resolveCallingConventionValues(
8706 .args = try self.gpa.alloc(MCValue, param_types.len),8600 .args = try self.gpa.alloc(MCValue, param_types.len),
8707 // These undefined values must be populated before returning from this function.8601 // These undefined values must be populated before returning from this function.
8708 .return_value = undefined,8602 .return_value = undefined,
8709 .stack_byte_count = undefined,8603 .stack_byte_count = 0,
8710 .stack_align = undefined,8604 .stack_align = undefined,
8711 };8605 };
8712 errdefer self.gpa.free(result.args);8606 errdefer self.gpa.free(result.args);
...@@ -8716,40 +8610,46 @@ fn resolveCallingConventionValues(...@@ -8716,40 +8610,46 @@ fn resolveCallingConventionValues(
8716 switch (cc) {8610 switch (cc) {
8717 .Naked => {8611 .Naked => {
8718 assert(result.args.len == 0);8612 assert(result.args.len == 0);
8719 result.return_value = .unreach;8613 result.return_value = InstTracking.init(.unreach);
8720 result.stack_byte_count = 0;8614 result.stack_align = 8;
8721 result.stack_align = 1;
8722 return result;
8723 },8615 },
8724 .C => {8616 .C => {
8617 var param_reg_i: usize = 0;
8618 result.stack_align = 16;
8619
8620 switch (self.target.os.tag) {
8621 .windows => {
8622 // Align the stack to 16bytes before allocating shadow stack space (if any).
8623 result.stack_byte_count += @intCast(u31, 4 * Type.usize.abiSize(self.target.*));
8624 },
8625 else => {},
8626 }
8627
8725 // Return values8628 // Return values
8726 if (ret_ty.zigTypeTag() == .NoReturn) {8629 if (ret_ty.zigTypeTag() == .NoReturn) {
8727 result.return_value = .unreach;8630 result.return_value = InstTracking.init(.unreach);
8728 } else if (!ret_ty.hasRuntimeBitsIgnoreComptime() and !ret_ty.isError()) {8631 } else if (!ret_ty.hasRuntimeBitsIgnoreComptime()) {
8729 // TODO: is this even possible for C calling convention?8632 // TODO: is this even possible for C calling convention?
8730 result.return_value = .none;8633 result.return_value = InstTracking.init(.none);
8731 } else {8634 } else {
8732 const ret_ty_size = @intCast(u32, ret_ty.abiSize(self.target.*));8635 const ret_reg = abi.getCAbiIntReturnRegs(self.target.*)[0];
8733 if (ret_ty_size == 0) {8636 const ret_ty_size = @intCast(u31, ret_ty.abiSize(self.target.*));
8734 assert(ret_ty.isError());8637 if (ret_ty_size <= 8) {
8735 result.return_value = .{ .immediate = 0 };8638 const aliased_reg = registerAlias(ret_reg, ret_ty_size);
8736 } else if (ret_ty_size <= 8) {8639 result.return_value = .{ .short = .{ .register = aliased_reg }, .long = .none };
8737 const aliased_reg = registerAlias(abi.getCAbiIntReturnRegs(self.target.*)[0], ret_ty_size);
8738 result.return_value = .{ .register = aliased_reg };
8739 } else {8640 } else {
8740 // TODO: return argument cell should go first8641 const ret_indirect_reg = abi.getCAbiIntParamRegs(self.target.*)[param_reg_i];
8741 result.return_value = .{ .stack_offset = 0 };8642 param_reg_i += 1;
8643 result.return_value = .{
8644 .short = .{ .indirect = .{ .reg = ret_reg } },
8645 .long = .{ .indirect = .{ .reg = ret_indirect_reg } },
8646 };
8742 }8647 }
8743 }8648 }
87448649
8745 // Input params8650 // Input params
8746 var next_stack_offset: u32 = switch (result.return_value) {8651 for (param_types, result.args) |ty, *arg| {
8747 .stack_offset => |off| @intCast(u32, off),8652 assert(ty.hasRuntimeBitsIgnoreComptime());
8748 else => 0,
8749 };
8750
8751 for (param_types, result.args, 0..) |ty, *arg, i| {
8752 assert(ty.hasRuntimeBits());
87538653
8754 const classes: []const abi.Class = switch (self.target.os.tag) {8654 const classes: []const abi.Class = switch (self.target.os.tag) {
8755 .windows => &[1]abi.Class{abi.classifyWindows(ty, self.target.*)},8655 .windows => &[1]abi.Class{abi.classifyWindows(ty, self.target.*)},
...@@ -8760,8 +8660,10 @@ fn resolveCallingConventionValues(...@@ -8760,8 +8660,10 @@ fn resolveCallingConventionValues(
8760 }8660 }
8761 switch (classes[0]) {8661 switch (classes[0]) {
8762 .integer => blk: {8662 .integer => blk: {
8763 if (i >= abi.getCAbiIntParamRegs(self.target.*).len) break :blk; // fallthrough8663 if (param_reg_i >= abi.getCAbiIntParamRegs(self.target.*).len) break :blk;
8764 arg.* = .{ .register = abi.getCAbiIntParamRegs(self.target.*)[i] };8664 const param_reg = abi.getCAbiIntParamRegs(self.target.*)[param_reg_i];
8665 param_reg_i += 1;
8666 arg.* = .{ .register = param_reg };
8765 continue;8667 continue;
8766 },8668 },
8767 .memory => {}, // fallthrough8669 .memory => {}, // fallthrough
...@@ -8770,82 +8672,61 @@ fn resolveCallingConventionValues(...@@ -8770,82 +8672,61 @@ fn resolveCallingConventionValues(
8770 }),8672 }),
8771 }8673 }
87728674
8773 const param_size = @intCast(u32, ty.abiSize(self.target.*));8675 const param_size = @intCast(u31, ty.abiSize(self.target.*));
8774 const param_align = @intCast(u32, ty.abiAlignment(self.target.*));8676 const param_align = @intCast(u31, ty.abiAlignment(self.target.*));
8775 const offset = mem.alignForwardGeneric(u32, next_stack_offset + param_size, param_align);8677 result.stack_byte_count =
8776 arg.* = .{ .stack_offset = @intCast(i32, offset) };8678 mem.alignForwardGeneric(u31, result.stack_byte_count, param_align);
8777 next_stack_offset = offset;8679 arg.* = .{ .load_frame = .{
8778 }8680 .index = stack_frame_base,
87798681 .off = result.stack_byte_count,
8780 // Align the stack to 16bytes before allocating shadow stack space (if any).8682 } };
8781 const aligned_next_stack_offset = mem.alignForwardGeneric(u32, next_stack_offset, 16);8683 result.stack_byte_count += param_size;
8782 const padding = aligned_next_stack_offset - next_stack_offset;
8783 if (padding > 0) {
8784 for (result.args) |*arg| {
8785 if (arg.isRegister()) continue;
8786 arg.stack_offset += @intCast(i32, padding);
8787 }
8788 }8684 }
8789
8790 const shadow_stack_space: u32 = switch (self.target.os.tag) {
8791 .windows => @intCast(u32, 4 * @sizeOf(u64)),
8792 else => 0,
8793 };
8794
8795 // alignment padding | args ... | shadow stack space (if any) | ret addr | $rbp |
8796 result.stack_byte_count = aligned_next_stack_offset + shadow_stack_space;
8797 result.stack_align = 16;
8798 },8685 },
8799 .Unspecified => {8686 .Unspecified => {
8687 result.stack_align = 16;
8688
8800 // Return values8689 // Return values
8801 if (ret_ty.zigTypeTag() == .NoReturn) {8690 if (ret_ty.zigTypeTag() == .NoReturn) {
8802 result.return_value = .unreach;8691 result.return_value = InstTracking.init(.unreach);
8803 } else if (!ret_ty.hasRuntimeBitsIgnoreComptime() and !ret_ty.isError()) {8692 } else if (!ret_ty.hasRuntimeBitsIgnoreComptime()) {
8804 result.return_value = .none;8693 result.return_value = InstTracking.init(.none);
8805 } else {8694 } else {
8806 const ret_ty_size = @intCast(u32, ret_ty.abiSize(self.target.*));8695 const ret_reg = abi.getCAbiIntReturnRegs(self.target.*)[0];
8807 if (ret_ty_size == 0) {8696 const ret_ty_size = @intCast(u31, ret_ty.abiSize(self.target.*));
8808 assert(ret_ty.isError());8697 if (ret_ty_size <= 8 and !ret_ty.isRuntimeFloat()) {
8809 result.return_value = .{ .immediate = 0 };8698 const aliased_reg = registerAlias(ret_reg, ret_ty_size);
8810 } else if (ret_ty_size <= 8 and !ret_ty.isRuntimeFloat()) {8699 result.return_value = .{ .short = .{ .register = aliased_reg }, .long = .none };
8811 const aliased_reg = registerAlias(abi.getCAbiIntReturnRegs(self.target.*)[0], ret_ty_size);
8812 result.return_value = .{ .register = aliased_reg };
8813 } else {8700 } else {
8814 // We simply make the return MCValue a stack offset. However, the actual value8701 const ret_indirect_reg = abi.getCAbiIntParamRegs(self.target.*)[0];
8815 // for the offset will be populated later. We will also push the stack offset8702 result.return_value = .{
8816 // value into an appropriate register when we resolve the offset.8703 .short = .{ .indirect = .{ .reg = ret_reg } },
8817 result.return_value = .{ .stack_offset = 0 };8704 .long = .{ .indirect = .{ .reg = ret_indirect_reg } },
8705 };
8818 }8706 }
8819 }8707 }
88208708
8821 // Input params8709 // Input params
8822 var next_stack_offset: u32 = switch (result.return_value) {
8823 .stack_offset => |off| @intCast(u32, off),
8824 else => 0,
8825 };
8826
8827 for (param_types, result.args) |ty, *arg| {8710 for (param_types, result.args) |ty, *arg| {
8828 if (!ty.hasRuntimeBits()) {8711 if (!ty.hasRuntimeBitsIgnoreComptime()) {
8829 arg.* = .none;8712 arg.* = .none;
8830 continue;8713 continue;
8831 }8714 }
8832 const param_size = @intCast(u32, ty.abiSize(self.target.*));8715 const param_size = @intCast(u31, ty.abiSize(self.target.*));
8833 const param_align = @intCast(u32, ty.abiAlignment(self.target.*));8716 const param_align = @intCast(u31, ty.abiAlignment(self.target.*));
8834 const offset = mem.alignForwardGeneric(u32, next_stack_offset + param_size, param_align);8717 result.stack_byte_count =
8835 arg.* = .{ .stack_offset = @intCast(i32, offset) };8718 mem.alignForwardGeneric(u31, result.stack_byte_count, param_align);
8836 next_stack_offset = offset;8719 arg.* = .{ .load_frame = .{
8720 .index = stack_frame_base,
8721 .off = result.stack_byte_count,
8722 } };
8723 result.stack_byte_count += param_size;
8837 }8724 }
8838
8839 result.stack_align = 16;
8840 // TODO fix this so that the 16byte alignment padding is at the current value of $rsp, and push
8841 // the args onto the stack so that there is no padding between the first argument and
8842 // the standard preamble.
8843 // alignment padding | args ... | ret addr | $rbp |
8844 result.stack_byte_count = mem.alignForwardGeneric(u32, next_stack_offset, result.stack_align);
8845 },8725 },
8846 else => return self.fail("TODO implement function parameters and return values for {} on x86_64", .{cc}),8726 else => return self.fail("TODO implement function parameters and return values for {} on x86_64", .{cc}),
8847 }8727 }
88488728
8729 result.stack_byte_count = mem.alignForwardGeneric(u31, result.stack_byte_count, result.stack_align);
8849 return result;8730 return result;
8850}8731}
88518732
...@@ -8953,15 +8834,6 @@ fn regExtraBits(self: *Self, ty: Type) u64 {...@@ -8953,15 +8834,6 @@ fn regExtraBits(self: *Self, ty: Type) u64 {
8953 return self.regBitSize(ty) - ty.bitSize(self.target.*);8834 return self.regBitSize(ty) - ty.bitSize(self.target.*);
8954}8835}
89558836
8956fn intrinsicsAllowed(target: Target, ty: Type) bool {
8957 return switch (ty.tag()) {
8958 .f32,
8959 .f64,
8960 => Target.x86.featureSetHasAny(target.cpu.features, .{ .sse2, .avx, .avx2 }),
8961 else => unreachable, // TODO finish this off
8962 };
8963}
8964
8965fn hasAvxSupport(target: Target) bool {8837fn hasAvxSupport(target: Target) bool {
8966 return Target.x86.featureSetHasAny(target.cpu.features, .{ .avx, .avx2 });8838 return Target.x86.featureSetHasAny(target.cpu.features, .{ .avx, .avx2 });
8967}8839}
src/arch/x86_64/Emit.zig+8-7
...@@ -13,7 +13,9 @@ prev_di_pc: usize,...@@ -13,7 +13,9 @@ prev_di_pc: usize,
13code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .{},13code_offset_mapping: std.AutoHashMapUnmanaged(Mir.Inst.Index, usize) = .{},
14relocs: std.ArrayListUnmanaged(Reloc) = .{},14relocs: std.ArrayListUnmanaged(Reloc) = .{},
1515
16pub const Error = Lower.Error || error{EmitFail};16pub const Error = Lower.Error || error{
17 EmitFail,
18};
1719
18pub fn emitMir(emit: *Emit) Error!void {20pub fn emitMir(emit: *Emit) Error!void {
19 for (0..emit.lower.mir.instructions.len) |i| {21 for (0..emit.lower.mir.instructions.len) |i| {
...@@ -22,7 +24,7 @@ pub fn emitMir(emit: *Emit) Error!void {...@@ -22,7 +24,7 @@ pub fn emitMir(emit: *Emit) Error!void {
2224
23 const start_offset = @intCast(u32, emit.code.items.len);25 const start_offset = @intCast(u32, emit.code.items.len);
24 try emit.code_offset_mapping.putNoClobber(emit.lower.allocator, index, start_offset);26 try emit.code_offset_mapping.putNoClobber(emit.lower.allocator, index, start_offset);
25 for (try emit.lower.lowerMir(inst)) |lower_inst| try lower_inst.encode(emit.code.writer());27 for (try emit.lower.lowerMir(inst)) |lower_inst| try lower_inst.encode(emit.code.writer(), .{});
26 const end_offset = @intCast(u32, emit.code.items.len);28 const end_offset = @intCast(u32, emit.code.items.len);
2729
28 switch (inst.tag) {30 switch (inst.tag) {
...@@ -120,11 +122,10 @@ pub fn emitMir(emit: *Emit) Error!void {...@@ -120,11 +122,10 @@ pub fn emitMir(emit: *Emit) Error!void {
120 .length = 6,122 .length = 6,
121 }),123 }),
122124
123 .dbg_line => {125 .dbg_line => try emit.dbgAdvancePCAndLine(
124 const dbg_line_column =126 inst.data.line_column.line,
125 emit.lower.mir.extraData(Mir.DbgLineColumn, inst.data.payload).data;127 inst.data.line_column.column,
126 try emit.dbgAdvancePCAndLine(dbg_line_column.line, dbg_line_column.column);128 ),
127 },
128129
129 .dbg_prologue_end => {130 .dbg_prologue_end => {
130 switch (emit.debug_output) {131 switch (emit.debug_output) {
src/arch/x86_64/Encoding.zig+3-12
...@@ -47,17 +47,8 @@ pub fn findByMnemonic(...@@ -47,17 +47,8 @@ pub fn findByMnemonic(
47 },47 },
48 else => {},48 else => {},
49 } else false;49 } else false;
50 const rex_extended = for (ops) |op| switch (op) {50 const rex_extended = for (ops) |op| {
51 .reg => |r| if (r.isExtended()) break true,51 if (op.isBaseExtended() or op.isIndexExtended()) break true;
52 .mem => |m| {
53 if (m.base()) |base| {
54 if (base.isExtended()) break true;
55 }
56 if (m.scaleIndex()) |si| {
57 if (si.index.isExtended()) break true;
58 }
59 },
60 else => {},
61 } else false;52 } else false;
6253
63 if ((rex_required or rex_extended) and rex_invalid) return error.CannotEncode;54 if ((rex_required or rex_extended) and rex_invalid) return error.CannotEncode;
...@@ -544,7 +535,7 @@ fn estimateInstructionLength(prefix: Prefix, encoding: Encoding, ops: []const Op...@@ -544,7 +535,7 @@ fn estimateInstructionLength(prefix: Prefix, encoding: Encoding, ops: []const Op
544 std.mem.copy(Operand, &inst.ops, ops);535 std.mem.copy(Operand, &inst.ops, ops);
545536
546 var cwriter = std.io.countingWriter(std.io.null_writer);537 var cwriter = std.io.countingWriter(std.io.null_writer);
547 inst.encode(cwriter.writer()) catch unreachable; // Not allowed to fail here unless OOM.538 inst.encode(cwriter.writer(), .{ .allow_frame_loc = true }) catch unreachable; // Not allowed to fail here unless OOM.
548 return @intCast(usize, cwriter.bytes_written);539 return @intCast(usize, cwriter.bytes_written);
549}540}
550541
src/arch/x86_64/Lower.zig+12-19
...@@ -136,7 +136,8 @@ pub fn lowerMir(lower: *Lower, inst: Mir.Inst) Error![]const Instruction {...@@ -136,7 +136,8 @@ pub fn lowerMir(lower: *Lower, inst: Mir.Inst) Error![]const Instruction {
136 .setcc => try lower.mirSetcc(inst),136 .setcc => try lower.mirSetcc(inst),
137 .jcc => try lower.emit(.none, mnem_cc(.j, inst.data.inst_cc.cc), &.{.{ .imm = Immediate.s(0) }}),137 .jcc => try lower.emit(.none, mnem_cc(.j, inst.data.inst_cc.cc), &.{.{ .imm = Immediate.s(0) }}),
138138
139 .push_regs, .pop_regs => try lower.mirPushPopRegisterList(inst),139 .push_regs => try lower.mirPushPopRegisterList(inst, .push),
140 .pop_regs => try lower.mirPushPopRegisterList(inst, .pop),
140141
141 .dbg_line,142 .dbg_line,
142 .dbg_prologue_end,143 .dbg_prologue_end,
...@@ -190,7 +191,7 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {...@@ -190,7 +191,7 @@ fn imm(lower: Lower, ops: Mir.Inst.Ops, i: u32) Immediate {
190}191}
191192
192fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {193fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
193 return switch (ops) {194 return lower.mir.resolveFrameLoc(switch (ops) {
194 .rm_sib,195 .rm_sib,
195 .rm_sib_cc,196 .rm_sib_cc,
196 .m_sib,197 .m_sib,
...@@ -227,7 +228,7 @@ fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {...@@ -227,7 +228,7 @@ fn mem(lower: Lower, ops: Mir.Inst.Ops, payload: u32) Memory {
227 => lower.mir.extraData(Mir.MemoryMoffs, payload).data.decode(),228 => lower.mir.extraData(Mir.MemoryMoffs, payload).data.decode(),
228229
229 else => unreachable,230 else => unreachable,
230 };231 });
231}232}
232233
233fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) Error!void {234fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) Error!void {
...@@ -417,23 +418,15 @@ fn mirSetcc(lower: *Lower, inst: Mir.Inst) Error!void {...@@ -417,23 +418,15 @@ fn mirSetcc(lower: *Lower, inst: Mir.Inst) Error!void {
417 }418 }
418}419}
419420
420fn mirPushPopRegisterList(lower: *Lower, inst: Mir.Inst) Error!void {421fn mirPushPopRegisterList(lower: *Lower, inst: Mir.Inst, comptime mnemonic: Mnemonic) Error!void {
421 const save_reg_list = lower.mir.extraData(Mir.SaveRegisterList, inst.data.payload).data;422 const reg_list = Mir.RegisterList.fromInt(inst.data.payload);
422 const base = @intToEnum(Register, save_reg_list.base_reg);
423 var disp: i32 = -@intCast(i32, save_reg_list.stack_end);
424 const reg_list = Mir.RegisterList.fromInt(save_reg_list.register_list);
425 const callee_preserved_regs = abi.getCalleePreservedRegs(lower.target.*);423 const callee_preserved_regs = abi.getCalleePreservedRegs(lower.target.*);
426 for (callee_preserved_regs) |callee_preserved_reg| {424 var it = reg_list.iterator(.{ .direction = switch (mnemonic) {
427 if (!reg_list.isSet(callee_preserved_regs, callee_preserved_reg)) continue;425 .push => .reverse,
428 const reg_op = Operand{ .reg = callee_preserved_reg };426 .pop => .forward,
429 const mem_op = Operand{ .mem = Memory.sib(.qword, .{ .base = base, .disp = disp }) };427 else => unreachable,
430 try lower.emit(.none, .mov, switch (inst.tag) {428 } });
431 .push_regs => &.{ mem_op, reg_op },429 while (it.next()) |i| try lower.emit(.none, mnemonic, &.{.{ .reg = callee_preserved_regs[i] }});
432 .pop_regs => &.{ reg_op, mem_op },
433 else => unreachable,
434 });
435 disp += 8;
436 }
437}430}
438431
439fn mirLeaLinker(lower: *Lower, inst: Mir.Inst) Error!void {432fn mirLeaLinker(lower: *Lower, inst: Mir.Inst) Error!void {
src/arch/x86_64/Mir.zig+65-39
...@@ -23,6 +23,7 @@ const Register = bits.Register;...@@ -23,6 +23,7 @@ const Register = bits.Register;
23instructions: std.MultiArrayList(Inst).Slice,23instructions: std.MultiArrayList(Inst).Slice,
24/// The meaning of this data is determined by `Inst.Tag` value.24/// The meaning of this data is determined by `Inst.Tag` value.
25extra: []const u32,25extra: []const u32,
26frame_locs: std.MultiArrayList(FrameLoc).Slice,
2627
27pub const Inst = struct {28pub const Inst = struct {
28 tag: Tag,29 tag: Tag,
...@@ -241,13 +242,13 @@ pub const Inst = struct {...@@ -241,13 +242,13 @@ pub const Inst = struct {
241 /// Start of epilogue242 /// Start of epilogue
242 dbg_epilogue_begin,243 dbg_epilogue_begin,
243 /// Update debug line244 /// Update debug line
244 /// Uses `payload` payload with data of type `DbgLineColumn`.245 /// Uses `line_column` payload containing the line and column.
245 dbg_line,246 dbg_line,
246 /// Push registers247 /// Push registers
247 /// Uses `payload` payload with data of type `SaveRegisterList`.248 /// Uses `payload` payload containing `RegisterList.asInt` directly.
248 push_regs,249 push_regs,
249 /// Pop registers250 /// Pop registers
250 /// Uses `payload` payload with data of type `SaveRegisterList`.251 /// Uses `payload` payload containing `RegisterList.asInt` directly.
251 pop_regs,252 pop_regs,
252253
253 /// Tombstone254 /// Tombstone
...@@ -500,6 +501,11 @@ pub const Inst = struct {...@@ -500,6 +501,11 @@ pub const Inst = struct {
500 /// Index into the linker's symbol table.501 /// Index into the linker's symbol table.
501 sym_index: u32,502 sym_index: u32,
502 },503 },
504 /// Debug line and column position
505 line_column: struct {
506 line: u32,
507 column: u32,
508 },
503 /// Index into `extra`. Meaning of what can be found there is context-dependent.509 /// Index into `extra`. Meaning of what can be found there is context-dependent.
504 payload: u32,510 payload: u32,
505 };511 };
...@@ -522,12 +528,11 @@ pub const LeaRegisterReloc = struct {...@@ -522,12 +528,11 @@ pub const LeaRegisterReloc = struct {
522 sym_index: u32,528 sym_index: u32,
523};529};
524530
525/// Used in conjunction with `SaveRegisterList` payload to transfer a list of used registers531/// Used in conjunction with payload to transfer a list of used registers in a compact manner.
526/// in a compact manner.
527pub const RegisterList = struct {532pub const RegisterList = struct {
528 bitset: BitSet = BitSet.initEmpty(),533 bitset: BitSet = BitSet.initEmpty(),
529534
530 const BitSet = IntegerBitSet(@ctz(@as(u32, 0)));535 const BitSet = IntegerBitSet(32);
531 const Self = @This();536 const Self = @This();
532537
533 fn getIndexForReg(registers: []const Register, reg: Register) BitSet.MaskInt {538 fn getIndexForReg(registers: []const Register, reg: Register) BitSet.MaskInt {
...@@ -547,6 +552,10 @@ pub const RegisterList = struct {...@@ -547,6 +552,10 @@ pub const RegisterList = struct {
547 return self.bitset.isSet(index);552 return self.bitset.isSet(index);
548 }553 }
549554
555 pub fn iterator(self: Self, comptime options: std.bit_set.IteratorOptions) BitSet.Iterator(options) {
556 return self.bitset.iterator(options);
557 }
558
550 pub fn asInt(self: Self) u32 {559 pub fn asInt(self: Self) u32 {
551 return self.bitset.mask;560 return self.bitset.mask;
552 }561 }
...@@ -562,14 +571,6 @@ pub const RegisterList = struct {...@@ -562,14 +571,6 @@ pub const RegisterList = struct {
562 }571 }
563};572};
564573
565pub const SaveRegisterList = struct {
566 /// Base register
567 base_reg: u32,
568 /// Use `RegisterList` to populate.
569 register_list: u32,
570 stack_end: u32,
571};
572
573pub const Imm64 = struct {574pub const Imm64 = struct {
574 msb: u32,575 msb: u32,
575 lsb: u32,576 lsb: u32,
...@@ -593,41 +594,51 @@ pub const Imm64 = struct {...@@ -593,41 +594,51 @@ pub const Imm64 = struct {
593pub const MemorySib = struct {594pub const MemorySib = struct {
594 /// Size of the pointer.595 /// Size of the pointer.
595 ptr_size: u32,596 ptr_size: u32,
596 /// Base register. -1 means null, or no base register.597 /// Base register tag of type Memory.Base.Tag
597 base: i32,598 base_tag: u32,
598 /// Scale for index register. -1 means null, or no scale.599 /// Base register of type Register or FrameIndex
599 /// This has to be in sync with `index` field.600 base: u32,
600 scale: i32,601 /// Scale starting at bit 0 and index register starting at bit 4.
601 /// Index register. -1 means null, or no index register.602 scale_index: u32,
602 /// This has to be in sync with `scale` field.
603 index: i32,
604 /// Displacement value.603 /// Displacement value.
605 disp: i32,604 disp: i32,
606605
607 pub fn encode(mem: Memory) MemorySib {606 pub fn encode(mem: Memory) MemorySib {
608 const sib = mem.sib;607 const sib = mem.sib;
608 assert(sib.scale_index.scale == 0 or std.math.isPowerOfTwo(sib.scale_index.scale));
609 return .{609 return .{
610 .ptr_size = @enumToInt(sib.ptr_size),610 .ptr_size = @enumToInt(sib.ptr_size),
611 .base = if (sib.base) |r| @enumToInt(r) else -1,611 .base_tag = @enumToInt(@as(Memory.Base.Tag, sib.base)),
612 .scale = if (sib.scale_index) |si| si.scale else -1,612 .base = switch (sib.base) {
613 .index = if (sib.scale_index) |si| @enumToInt(si.index) else -1,613 .none => undefined,
614 .reg => |r| @enumToInt(r),
615 .frame => |fi| @enumToInt(fi),
616 },
617 .scale_index = @as(u32, sib.scale_index.scale) << 0 |
618 @as(u32, if (sib.scale_index.scale > 0)
619 @enumToInt(sib.scale_index.index)
620 else
621 undefined) << 4,
614 .disp = sib.disp,622 .disp = sib.disp,
615 };623 };
616 }624 }
617625
618 pub fn decode(msib: MemorySib) Memory {626 pub fn decode(msib: MemorySib) Memory {
619 const base: ?Register = if (msib.base == -1) null else @intToEnum(Register, msib.base);627 const scale = @truncate(u4, msib.scale_index);
620 const scale_index: ?Memory.ScaleIndex = if (msib.index == -1) null else .{628 assert(scale == 0 or std.math.isPowerOfTwo(scale));
621 .scale = @intCast(u4, msib.scale),629 return .{ .sib = .{
622 .index = @intToEnum(Register, msib.index),
623 };
624 const mem: Memory = .{ .sib = .{
625 .ptr_size = @intToEnum(Memory.PtrSize, msib.ptr_size),630 .ptr_size = @intToEnum(Memory.PtrSize, msib.ptr_size),
626 .base = base,631 .base = switch (@intToEnum(Memory.Base.Tag, msib.base_tag)) {
627 .scale_index = scale_index,632 .none => .none,
633 .reg => .{ .reg = @intToEnum(Register, msib.base) },
634 .frame => .{ .frame = @intToEnum(bits.FrameIndex, msib.base) },
635 },
636 .scale_index = .{
637 .scale = scale,
638 .index = if (scale > 0) @intToEnum(Register, msib.scale_index >> 4) else undefined,
639 },
628 .disp = msib.disp,640 .disp = msib.disp,
629 } };641 } };
630 return mem;
631 }642 }
632};643};
633644
...@@ -676,14 +687,10 @@ pub const MemoryMoffs = struct {...@@ -676,14 +687,10 @@ pub const MemoryMoffs = struct {
676 }687 }
677};688};
678689
679pub const DbgLineColumn = struct {
680 line: u32,
681 column: u32,
682};
683
684pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void {690pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void {
685 mir.instructions.deinit(gpa);691 mir.instructions.deinit(gpa);
686 gpa.free(mir.extra);692 gpa.free(mir.extra);
693 mir.frame_locs.deinit(gpa);
687 mir.* = undefined;694 mir.* = undefined;
688}695}
689696
...@@ -704,3 +711,22 @@ pub fn extraData(mir: Mir, comptime T: type, index: u32) struct { data: T, end:...@@ -704,3 +711,22 @@ pub fn extraData(mir: Mir, comptime T: type, index: u32) struct { data: T, end:
704 .end = i,711 .end = i,
705 };712 };
706}713}
714
715pub const FrameLoc = struct {
716 base: Register,
717 disp: i32,
718};
719
720pub fn resolveFrameLoc(mir: Mir, mem: Memory) Memory {
721 return switch (mem) {
722 .sib => |sib| switch (sib.base) {
723 .none, .reg => mem,
724 .frame => |index| if (mir.frame_locs.len > 0) Memory.sib(sib.ptr_size, .{
725 .base = .{ .reg = mir.frame_locs.items(.base)[@enumToInt(index)] },
726 .disp = mir.frame_locs.items(.disp)[@enumToInt(index)] + sib.disp,
727 .scale_index = mem.scaleIndex(),
728 }) else mem,
729 },
730 .rip, .moffs => mem,
731 };
732}
src/arch/x86_64/bits.zig+68-10
...@@ -405,14 +405,69 @@ test "Register classes" {...@@ -405,14 +405,69 @@ test "Register classes" {
405 try expect(Register.fs.class() == .segment);405 try expect(Register.fs.class() == .segment);
406}406}
407407
408pub const FrameIndex = enum(u32) {
409 // This index refers to the start of the arguments passed to this function
410 args_frame,
411 // This index refers to the return address pushed by a `call` and popped by a `ret`.
412 ret_addr,
413 // This index refers to the base pointer pushed in the prologue and popped in the epilogue.
414 base_ptr,
415 // This index refers to the entire stack frame.
416 stack_frame,
417 // This index refers to the start of the call frame for arguments passed to called functions
418 call_frame,
419 // Other indices are used for local variable stack slots
420 _,
421
422 pub const named_count = @typeInfo(FrameIndex).Enum.fields.len;
423
424 pub fn isNamed(fi: FrameIndex) bool {
425 return @enumToInt(fi) < named_count;
426 }
427
428 pub fn format(
429 fi: FrameIndex,
430 comptime fmt: []const u8,
431 options: std.fmt.FormatOptions,
432 writer: anytype,
433 ) @TypeOf(writer).Error!void {
434 try writer.writeAll("FrameIndex");
435 if (fi.isNamed()) {
436 try writer.writeByte('.');
437 try writer.writeAll(@tagName(fi));
438 } else {
439 try writer.writeByte('(');
440 try std.fmt.formatType(@enumToInt(fi), fmt, options, writer, 0);
441 try writer.writeByte(')');
442 }
443 }
444};
445
408pub const Memory = union(enum) {446pub const Memory = union(enum) {
409 sib: Sib,447 sib: Sib,
410 rip: Rip,448 rip: Rip,
411 moffs: Moffs,449 moffs: Moffs,
412450
413 pub const ScaleIndex = packed struct {451 pub const Base = union(enum) {
452 none,
453 reg: Register,
454 frame: FrameIndex,
455
456 pub const Tag = @typeInfo(Base).Union.tag_type.?;
457
458 pub fn isExtended(self: Base) bool {
459 return switch (self) {
460 .none, .frame => false, // neither rsp nor rbp are extended
461 .reg => |reg| reg.isExtended(),
462 };
463 }
464 };
465
466 pub const ScaleIndex = struct {
414 scale: u4,467 scale: u4,
415 index: Register,468 index: Register,
469
470 const none = ScaleIndex{ .scale = 0, .index = undefined };
416 };471 };
417472
418 pub const PtrSize = enum {473 pub const PtrSize = enum {
...@@ -460,8 +515,8 @@ pub const Memory = union(enum) {...@@ -460,8 +515,8 @@ pub const Memory = union(enum) {
460515
461 pub const Sib = struct {516 pub const Sib = struct {
462 ptr_size: PtrSize,517 ptr_size: PtrSize,
463 base: ?Register,518 base: Base,
464 scale_index: ?ScaleIndex,519 scale_index: ScaleIndex,
465 disp: i32,520 disp: i32,
466 };521 };
467522
...@@ -482,7 +537,7 @@ pub const Memory = union(enum) {...@@ -482,7 +537,7 @@ pub const Memory = union(enum) {
482537
483 pub fn sib(ptr_size: PtrSize, args: struct {538 pub fn sib(ptr_size: PtrSize, args: struct {
484 disp: i32 = 0,539 disp: i32 = 0,
485 base: ?Register = null,540 base: Base = .none,
486 scale_index: ?ScaleIndex = null,541 scale_index: ?ScaleIndex = null,
487 }) Memory {542 }) Memory {
488 if (args.scale_index) |si| assert(std.math.isPowerOfTwo(si.scale));543 if (args.scale_index) |si| assert(std.math.isPowerOfTwo(si.scale));
...@@ -490,7 +545,7 @@ pub const Memory = union(enum) {...@@ -490,7 +545,7 @@ pub const Memory = union(enum) {
490 .base = args.base,545 .base = args.base,
491 .disp = args.disp,546 .disp = args.disp,
492 .ptr_size = ptr_size,547 .ptr_size = ptr_size,
493 .scale_index = args.scale_index,548 .scale_index = if (args.scale_index) |si| si else ScaleIndex.none,
494 } };549 } };
495 }550 }
496551
...@@ -502,22 +557,25 @@ pub const Memory = union(enum) {...@@ -502,22 +557,25 @@ pub const Memory = union(enum) {
502 return switch (mem) {557 return switch (mem) {
503 .moffs => true,558 .moffs => true,
504 .rip => false,559 .rip => false,
505 .sib => |s| if (s.base) |r| r.class() == .segment else false,560 .sib => |s| switch (s.base) {
561 .none, .frame => false,
562 .reg => |reg| reg.class() == .segment,
563 },
506 };564 };
507 }565 }
508566
509 pub fn base(mem: Memory) ?Register {567 pub fn base(mem: Memory) Base {
510 return switch (mem) {568 return switch (mem) {
511 .moffs => |m| m.seg,569 .moffs => |m| .{ .reg = m.seg },
512 .sib => |s| s.base,570 .sib => |s| s.base,
513 .rip => null,571 .rip => .none,
514 };572 };
515 }573 }
516574
517 pub fn scaleIndex(mem: Memory) ?ScaleIndex {575 pub fn scaleIndex(mem: Memory) ?ScaleIndex {
518 return switch (mem) {576 return switch (mem) {
519 .moffs, .rip => null,577 .moffs, .rip => null,
520 .sib => |s| s.scale_index,578 .sib => |s| if (s.scale_index.scale > 0) s.scale_index else null,
521 };579 };
522 }580 }
523581
src/arch/x86_64/encoder.zig+113-93
...@@ -54,6 +54,21 @@ pub const Instruction = struct {...@@ -54,6 +54,21 @@ pub const Instruction = struct {
54 };54 };
55 }55 }
5656
57 pub fn isBaseExtended(op: Operand) bool {
58 return switch (op) {
59 .none, .imm => false,
60 .reg => |reg| reg.isExtended(),
61 .mem => |mem| mem.base().isExtended(),
62 };
63 }
64
65 pub fn isIndexExtended(op: Operand) bool {
66 return switch (op) {
67 .none, .reg, .imm => false,
68 .mem => |mem| if (mem.scaleIndex()) |si| si.index.isExtended() else false,
69 };
70 }
71
57 fn format(72 fn format(
58 op: Operand,73 op: Operand,
59 comptime unused_format_string: []const u8,74 comptime unused_format_string: []const u8,
...@@ -98,17 +113,24 @@ pub const Instruction = struct {...@@ -98,17 +113,24 @@ pub const Instruction = struct {
98 try writer.print("{s} ptr ", .{@tagName(sib.ptr_size)});113 try writer.print("{s} ptr ", .{@tagName(sib.ptr_size)});
99114
100 if (mem.isSegmentRegister()) {115 if (mem.isSegmentRegister()) {
101 return writer.print("{s}:0x{x}", .{ @tagName(sib.base.?), sib.disp });116 return writer.print("{s}:0x{x}", .{ @tagName(sib.base.reg), sib.disp });
102 }117 }
103118
104 try writer.writeByte('[');119 try writer.writeByte('[');
105120
106 var any = false;121 var any = false;
107 if (sib.base) |base| {122 switch (sib.base) {
108 try writer.print("{s}", .{@tagName(base)});123 .none => {},
109 any = true;124 .reg => |reg| {
125 try writer.print("{s}", .{@tagName(reg)});
126 any = true;
127 },
128 .frame => |frame| {
129 try writer.print("{}", .{frame});
130 any = true;
131 },
110 }132 }
111 if (sib.scale_index) |si| {133 if (mem.scaleIndex()) |si| {
112 if (any) try writer.writeAll(" + ");134 if (any) try writer.writeAll(" + ");
113 try writer.print("{s} * {d}", .{ @tagName(si.index), si.scale });135 try writer.print("{s} * {d}", .{ @tagName(si.index), si.scale });
114 any = true;136 any = true;
...@@ -182,8 +204,8 @@ pub const Instruction = struct {...@@ -182,8 +204,8 @@ pub const Instruction = struct {
182 }204 }
183 }205 }
184206
185 pub fn encode(inst: Instruction, writer: anytype) !void {207 pub fn encode(inst: Instruction, writer: anytype, comptime opts: Options) !void {
186 const encoder = Encoder(@TypeOf(writer)){ .writer = writer };208 const encoder = Encoder(@TypeOf(writer), opts){ .writer = writer };
187 const enc = inst.encoding;209 const enc = inst.encoding;
188 const data = enc.data;210 const data = enc.data;
189211
...@@ -269,22 +291,24 @@ pub const Instruction = struct {...@@ -269,22 +291,24 @@ pub const Instruction = struct {
269291
270 const segment_override: ?Register = switch (op_en) {292 const segment_override: ?Register = switch (op_en) {
271 .i, .zi, .o, .oi, .d, .np => null,293 .i, .zi, .o, .oi, .d, .np => null,
272 .fd => inst.ops[1].mem.base().?,294 .fd => inst.ops[1].mem.base().reg,
273 .td => inst.ops[0].mem.base().?,295 .td => inst.ops[0].mem.base().reg,
274 .rm, .rmi => if (inst.ops[1].isSegmentRegister()) blk: {296 .rm, .rmi => if (inst.ops[1].isSegmentRegister())
275 break :blk switch (inst.ops[1]) {297 switch (inst.ops[1]) {
276 .reg => |r| r,298 .reg => |reg| reg,
277 .mem => |m| m.base().?,299 .mem => |mem| mem.base().reg,
278 else => unreachable,300 else => unreachable,
279 };301 }
280 } else null,302 else
281 .m, .mi, .m1, .mc, .mr, .mri, .mrc => if (inst.ops[0].isSegmentRegister()) blk: {303 null,
282 break :blk switch (inst.ops[0]) {304 .m, .mi, .m1, .mc, .mr, .mri, .mrc => if (inst.ops[0].isSegmentRegister())
283 .reg => |r| r,305 switch (inst.ops[0]) {
284 .mem => |m| m.base().?,306 .reg => |reg| reg,
307 .mem => |mem| mem.base().reg,
285 else => unreachable,308 else => unreachable,
286 };309 }
287 } else null,310 else
311 null,
288 };312 };
289 if (segment_override) |seg| {313 if (segment_override) |seg| {
290 legacy.setSegmentOverride(seg);314 legacy.setSegmentOverride(seg);
...@@ -307,27 +331,17 @@ pub const Instruction = struct {...@@ -307,27 +331,17 @@ pub const Instruction = struct {
307 const r_op = switch (op_en) {331 const r_op = switch (op_en) {
308 .rm, .rmi => inst.ops[0],332 .rm, .rmi => inst.ops[0],
309 .mr, .mri, .mrc => inst.ops[1],333 .mr, .mri, .mrc => inst.ops[1],
310 else => null,334 else => .none,
311 };335 };
312 if (r_op) |op| {336 rex.r = r_op.isBaseExtended();
313 rex.r = op.reg.isExtended();
314 }
315337
316 const b_x_op = switch (op_en) {338 const b_x_op = switch (op_en) {
317 .rm, .rmi => inst.ops[1],339 .rm, .rmi => inst.ops[1],
318 .m, .mi, .m1, .mc, .mr, .mri, .mrc => inst.ops[0],340 .m, .mi, .m1, .mc, .mr, .mri, .mrc => inst.ops[0],
319 else => unreachable,341 else => unreachable,
320 };342 };
321 switch (b_x_op) {343 rex.b = b_x_op.isBaseExtended();
322 .reg => |r| {344 rex.x = b_x_op.isIndexExtended();
323 rex.b = r.isExtended();
324 },
325 .mem => |mem| {
326 rex.b = if (mem.base()) |base| base.isExtended() else false;
327 rex.x = if (mem.scaleIndex()) |si| si.index.isExtended() else false;
328 },
329 else => unreachable,
330 }
331 },345 },
332 }346 }
333347
...@@ -348,72 +362,75 @@ pub const Instruction = struct {...@@ -348,72 +362,75 @@ pub const Instruction = struct {
348362
349 switch (mem) {363 switch (mem) {
350 .moffs => unreachable,364 .moffs => unreachable,
351 .sib => |sib| {365 .sib => |sib| switch (sib.base) {
352 if (sib.base) |base| {366 .none => {
353 if (base.class() == .segment) {367 try encoder.modRm_SIBDisp0(operand_enc);
354 // TODO audit this wrt SIB368 if (mem.scaleIndex()) |si| {
355 try encoder.modRm_SIBDisp0(operand_enc);369 const scale = math.log2_int(u4, si.scale);
356 if (sib.scale_index) |si| {370 try encoder.sib_scaleIndexDisp32(scale, si.index.lowEnc());
357 const scale = math.log2_int(u4, si.scale);
358 try encoder.sib_scaleIndexDisp32(scale, si.index.lowEnc());
359 } else {
360 try encoder.sib_disp32();
361 }
362 try encoder.disp32(sib.disp);
363 } else {371 } else {
364 assert(base.class() == .general_purpose);372 try encoder.sib_disp32();
365 const dst = base.lowEnc();
366 const src = operand_enc;
367 if (dst == 4 or sib.scale_index != null) {
368 if (sib.disp == 0 and dst != 5) {
369 try encoder.modRm_SIBDisp0(src);
370 if (sib.scale_index) |si| {
371 const scale = math.log2_int(u4, si.scale);
372 try encoder.sib_scaleIndexBase(scale, si.index.lowEnc(), dst);
373 } else {
374 try encoder.sib_base(dst);
375 }
376 } else if (math.cast(i8, sib.disp)) |_| {
377 try encoder.modRm_SIBDisp8(src);
378 if (sib.scale_index) |si| {
379 const scale = math.log2_int(u4, si.scale);
380 try encoder.sib_scaleIndexBaseDisp8(scale, si.index.lowEnc(), dst);
381 } else {
382 try encoder.sib_baseDisp8(dst);
383 }
384 try encoder.disp8(@truncate(i8, sib.disp));
385 } else {
386 try encoder.modRm_SIBDisp32(src);
387 if (sib.scale_index) |si| {
388 const scale = math.log2_int(u4, si.scale);
389 try encoder.sib_scaleIndexBaseDisp32(scale, si.index.lowEnc(), dst);
390 } else {
391 try encoder.sib_baseDisp32(dst);
392 }
393 try encoder.disp32(sib.disp);
394 }
395 } else {
396 if (sib.disp == 0 and dst != 5) {
397 try encoder.modRm_indirectDisp0(src, dst);
398 } else if (math.cast(i8, sib.disp)) |_| {
399 try encoder.modRm_indirectDisp8(src, dst);
400 try encoder.disp8(@truncate(i8, sib.disp));
401 } else {
402 try encoder.modRm_indirectDisp32(src, dst);
403 try encoder.disp32(sib.disp);
404 }
405 }
406 }373 }
407 } else {374 try encoder.disp32(sib.disp);
375 },
376 .reg => |base| if (base.class() == .segment) {
377 // TODO audit this wrt SIB
408 try encoder.modRm_SIBDisp0(operand_enc);378 try encoder.modRm_SIBDisp0(operand_enc);
409 if (sib.scale_index) |si| {379 if (mem.scaleIndex()) |si| {
410 const scale = math.log2_int(u4, si.scale);380 const scale = math.log2_int(u4, si.scale);
411 try encoder.sib_scaleIndexDisp32(scale, si.index.lowEnc());381 try encoder.sib_scaleIndexDisp32(scale, si.index.lowEnc());
412 } else {382 } else {
413 try encoder.sib_disp32();383 try encoder.sib_disp32();
414 }384 }
415 try encoder.disp32(sib.disp);385 try encoder.disp32(sib.disp);
416 }386 } else {
387 assert(base.class() == .general_purpose);
388 const dst = base.lowEnc();
389 const src = operand_enc;
390 if (dst == 4 or mem.scaleIndex() != null) {
391 if (sib.disp == 0 and dst != 5) {
392 try encoder.modRm_SIBDisp0(src);
393 if (mem.scaleIndex()) |si| {
394 const scale = math.log2_int(u4, si.scale);
395 try encoder.sib_scaleIndexBase(scale, si.index.lowEnc(), dst);
396 } else {
397 try encoder.sib_base(dst);
398 }
399 } else if (math.cast(i8, sib.disp)) |_| {
400 try encoder.modRm_SIBDisp8(src);
401 if (mem.scaleIndex()) |si| {
402 const scale = math.log2_int(u4, si.scale);
403 try encoder.sib_scaleIndexBaseDisp8(scale, si.index.lowEnc(), dst);
404 } else {
405 try encoder.sib_baseDisp8(dst);
406 }
407 try encoder.disp8(@truncate(i8, sib.disp));
408 } else {
409 try encoder.modRm_SIBDisp32(src);
410 if (mem.scaleIndex()) |si| {
411 const scale = math.log2_int(u4, si.scale);
412 try encoder.sib_scaleIndexBaseDisp32(scale, si.index.lowEnc(), dst);
413 } else {
414 try encoder.sib_baseDisp32(dst);
415 }
416 try encoder.disp32(sib.disp);
417 }
418 } else {
419 if (sib.disp == 0 and dst != 5) {
420 try encoder.modRm_indirectDisp0(src, dst);
421 } else if (math.cast(i8, sib.disp)) |_| {
422 try encoder.modRm_indirectDisp8(src, dst);
423 try encoder.disp8(@truncate(i8, sib.disp));
424 } else {
425 try encoder.modRm_indirectDisp32(src, dst);
426 try encoder.disp32(sib.disp);
427 }
428 }
429 },
430 .frame => if (@TypeOf(encoder).options.allow_frame_loc) {
431 try encoder.modRm_indirectDisp32(operand_enc, undefined);
432 try encoder.disp32(undefined);
433 } else return error.CannotEncode,
417 },434 },
418 .rip => |rip| {435 .rip => |rip| {
419 try encoder.modRm_RIPDisp32(operand_enc);436 try encoder.modRm_RIPDisp32(operand_enc);
...@@ -482,11 +499,14 @@ pub const LegacyPrefixes = packed struct {...@@ -482,11 +499,14 @@ pub const LegacyPrefixes = packed struct {
482 }499 }
483};500};
484501
485fn Encoder(comptime T: type) type {502pub const Options = struct { allow_frame_loc: bool = false };
503
504fn Encoder(comptime T: type, comptime opts: Options) type {
486 return struct {505 return struct {
487 writer: T,506 writer: T,
488507
489 const Self = @This();508 const Self = @This();
509 pub const options = opts;
490510
491 // --------511 // --------
492 // Prefixes512 // Prefixes