| ... | ... | @@ -211,10 +211,10 @@ pub const MCValue = union(enum) { |
| 211 | 211 | /// The value is in memory at a constant offset from the address in a register. |
| 212 | 212 | indirect: bits.RegisterOffset, |
| 213 | 213 | indirect_load_frame: bits.FrameAddr, |
| 214 | | /// The value stored at an offset from a frame index |
| 214 | /// The value stored at an offset from a frame index. |
| 215 | 215 | /// Payload is a frame address. |
| 216 | 216 | load_frame: bits.FrameAddr, |
| 217 | | /// The address of an offset from a frame index |
| 217 | /// The address of an offset from a frame index. |
| 218 | 218 | /// Payload is a frame address. |
| 219 | 219 | lea_frame: bits.FrameAddr, |
| 220 | 220 | load_nav: InternPool.Nav.Index, |
| ... | ... | @@ -225,15 +225,17 @@ pub const MCValue = union(enum) { |
| 225 | 225 | lea_lazy_sym: link.File.LazySymbol, |
| 226 | 226 | load_extern_func: Mir.NullTerminatedString, |
| 227 | 227 | lea_extern_func: Mir.NullTerminatedString, |
| 228 | | /// Supports `integer_per_element` abi |
| 228 | /// The value is duplicated in two different registers. |
| 229 | register_tee: [4]Register, |
| 230 | /// Supports `integer_per_element` abi. |
| 229 | 231 | elementwise_gpr: ArgsInfo, |
| 230 | | /// Supports `sse_per_element` abi |
| 232 | /// Supports `sse_per_element` abi. |
| 231 | 233 | elementwise_sse: ArgsInfo, |
| 232 | | /// Supports `sse_per_xword` abi |
| 234 | /// Supports `sse_per_xword` abi. |
| 233 | 235 | xwordwise_sse: ArgsInfo, |
| 234 | | /// Supports `sse_per_yword` abi |
| 236 | /// Supports `sse_per_yword` abi. |
| 235 | 237 | ywordwise_sse: ArgsInfo, |
| 236 | | /// Supports `sse_per_zword` abi |
| 238 | /// Supports `sse_per_zword` abi. |
| 237 | 239 | zwordwise_sse: ArgsInfo, |
| 238 | 240 | /// This indicates that we have already allocated a frame index for this instruction, |
| 239 | 241 | /// but it has not been spilled there yet in the current control flow. |
| ... | ... | @@ -261,6 +263,7 @@ pub const MCValue = union(enum) { |
| 261 | 263 | .lea_lazy_sym, |
| 262 | 264 | .lea_extern_func, |
| 263 | 265 | .load_extern_func, |
| 266 | .register_tee, |
| 264 | 267 | .elementwise_gpr, |
| 265 | 268 | .elementwise_sse, |
| 266 | 269 | .xwordwise_sse, |
| ... | ... | @@ -386,6 +389,7 @@ pub const MCValue = union(enum) { |
| 386 | 389 | .lea_uav, |
| 387 | 390 | .lea_lazy_sym, |
| 388 | 391 | .lea_extern_func, |
| 392 | .register_tee, |
| 389 | 393 | .elementwise_gpr, |
| 390 | 394 | .elementwise_sse, |
| 391 | 395 | .xwordwise_sse, |
| ... | ... | @@ -427,6 +431,7 @@ pub const MCValue = union(enum) { |
| 427 | 431 | .load_uav, |
| 428 | 432 | .load_lazy_sym, |
| 429 | 433 | .load_extern_func, |
| 434 | .register_tee, |
| 430 | 435 | .elementwise_gpr, |
| 431 | 436 | .elementwise_sse, |
| 432 | 437 | .xwordwise_sse, |
| ... | ... | @@ -453,6 +458,7 @@ pub const MCValue = union(enum) { |
| 453 | 458 | .unreach, |
| 454 | 459 | .dead, |
| 455 | 460 | .undef, |
| 461 | .register_tee, |
| 456 | 462 | .elementwise_gpr, |
| 457 | 463 | .elementwise_sse, |
| 458 | 464 | .xwordwise_sse, |
| ... | ... | @@ -511,6 +517,7 @@ pub const MCValue = union(enum) { |
| 511 | 517 | .register_mask, |
| 512 | 518 | .indirect_load_frame, |
| 513 | 519 | .lea_frame, |
| 520 | .register_tee, |
| 514 | 521 | .elementwise_gpr, |
| 515 | 522 | .elementwise_sse, |
| 516 | 523 | .xwordwise_sse, |
| ... | ... | @@ -593,6 +600,7 @@ pub const MCValue = union(enum) { |
| 593 | 600 | .lea_lazy_sym => |pl| try w.print("lazy:{s}:{d}", .{ @tagName(pl.kind), @intFromEnum(pl.ty) }), |
| 594 | 601 | .load_extern_func => |pl| try w.print("[extern:{d}]", .{@intFromEnum(pl)}), |
| 595 | 602 | .lea_extern_func => |pl| try w.print("extern:{d}", .{@intFromEnum(pl)}), |
| 603 | .register_tee => |pl| try w.print("tee:{s}:{s}", .{ @tagName(pl[1]), @tagName(pl[0]) }), |
| 596 | 604 | .elementwise_gpr => |pl| try w.print("elementwise:gpr{d}:[{f} + 0x{x}]", .{ |
| 597 | 605 | pl.info.reg_index, pl.frame_index, pl.info.frame_off, |
| 598 | 606 | }), |
| ... | ... | @@ -640,6 +648,7 @@ const InstTracking = struct { |
| 640 | 648 | .lea_extern_func, |
| 641 | 649 | => result, |
| 642 | 650 | .dead, |
| 651 | .register_tee, |
| 643 | 652 | .elementwise_gpr, |
| 644 | 653 | .elementwise_sse, |
| 645 | 654 | .xwordwise_sse, |
| ... | ... | @@ -756,6 +765,7 @@ const InstTracking = struct { |
| 756 | 765 | .register_overflow, |
| 757 | 766 | .register_mask, |
| 758 | 767 | .indirect, |
| 768 | .register_tee, |
| 759 | 769 | .elementwise_gpr, |
| 760 | 770 | .elementwise_sse, |
| 761 | 771 | .xwordwise_sse, |
| ... | ... | @@ -174266,15 +174276,16 @@ fn restoreState(self: *CodeGen, state: State, deaths: []const Air.Inst.Index, co |
| 174266 | 174276 | const ExpectedContents = [@typeInfo(RegisterManager.TrackedRegisters).array.len]RegisterLock; |
| 174267 | 174277 | const bfa_buf_len = if (opts.update_tracking) 0 else 1; |
| 174268 | 174278 | var bfa_buf: [bfa_buf_len]ExpectedContents = undefined; |
| 174269 | | var stack = if (opts.update_tracking) {} else std.heap.BufferFirstAllocator.init(@ptrCast(&bfa_buf), self.gpa); |
| 174279 | var stack = if (!opts.update_tracking) std.heap.BufferFirstAllocator.init(@ptrCast(&bfa_buf), self.gpa); |
| 174280 | const allocator = if (!opts.update_tracking) stack.allocator(); |
| 174270 | 174281 | |
| 174271 | | var reg_locks = if (opts.update_tracking) {} else try std.array_list.Managed(RegisterLock).initCapacity( |
| 174272 | | stack.allocator(), |
| 174282 | var reg_locks = if (!opts.update_tracking) try std.ArrayList(RegisterLock).initCapacity( |
| 174283 | allocator, |
| 174273 | 174284 | @typeInfo(ExpectedContents).array.len, |
| 174274 | 174285 | ); |
| 174275 | 174286 | defer if (!opts.update_tracking) { |
| 174276 | 174287 | for (reg_locks.items) |lock| self.register_manager.unlockReg(lock); |
| 174277 | | reg_locks.deinit(); |
| 174288 | reg_locks.deinit(allocator); |
| 174278 | 174289 | }; |
| 174279 | 174290 | |
| 174280 | 174291 | for ( |
| ... | ... | @@ -174304,7 +174315,7 @@ fn restoreState(self: *CodeGen, state: State, deaths: []const Air.Inst.Index, co |
| 174304 | 174315 | self.inst_tracking.getPtr(target_inst).?.trackMaterialize(target_inst, reg_tracking); |
| 174305 | 174316 | } |
| 174306 | 174317 | } else if (target_maybe_inst) |_| |
| 174307 | | try reg_locks.append(self.register_manager.lockRegIndexAssumeUnused(reg_index)); |
| 174318 | try reg_locks.append(allocator, self.register_manager.lockRegIndexAssumeUnused(reg_index)); |
| 174308 | 174319 | } |
| 174309 | 174320 | if (opts.emit_instructions) if (self.eflags_inst) |inst| |
| 174310 | 174321 | try self.inst_tracking.getPtr(inst).?.spill(self, inst); |
| ... | ... | @@ -174517,6 +174528,7 @@ fn load(self: *CodeGen, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerE |
| 174517 | 174528 | .register_overflow, |
| 174518 | 174529 | .register_mask, |
| 174519 | 174530 | .indirect_load_frame, |
| 174531 | .register_tee, |
| 174520 | 174532 | .elementwise_gpr, |
| 174521 | 174533 | .elementwise_sse, |
| 174522 | 174534 | .xwordwise_sse, |
| ... | ... | @@ -174574,6 +174586,7 @@ fn store( |
| 174574 | 174586 | .register_overflow, |
| 174575 | 174587 | .register_mask, |
| 174576 | 174588 | .indirect_load_frame, |
| 174589 | .register_tee, |
| 174577 | 174590 | .elementwise_gpr, |
| 174578 | 174591 | .elementwise_sse, |
| 174579 | 174592 | .xwordwise_sse, |
| ... | ... | @@ -174628,6 +174641,7 @@ fn genUnOpMir(self: *CodeGen, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: |
| 174628 | 174641 | .lea_uav, |
| 174629 | 174642 | .lea_lazy_sym, |
| 174630 | 174643 | .lea_extern_func, |
| 174644 | .register_tee, |
| 174631 | 174645 | .elementwise_gpr, |
| 174632 | 174646 | .elementwise_sse, |
| 174633 | 174647 | .xwordwise_sse, |
| ... | ... | @@ -175322,6 +175336,7 @@ fn genBinOpMir( |
| 175322 | 175336 | .lea_lazy_sym, |
| 175323 | 175337 | .lea_extern_func, |
| 175324 | 175338 | .indirect_load_frame, |
| 175339 | .register_tee, |
| 175325 | 175340 | .elementwise_gpr, |
| 175326 | 175341 | .elementwise_sse, |
| 175327 | 175342 | .xwordwise_sse, |
| ... | ... | @@ -175363,6 +175378,7 @@ fn genBinOpMir( |
| 175363 | 175378 | .register_overflow, |
| 175364 | 175379 | .register_mask, |
| 175365 | 175380 | .indirect_load_frame, |
| 175381 | .register_tee, |
| 175366 | 175382 | .elementwise_gpr, |
| 175367 | 175383 | .elementwise_sse, |
| 175368 | 175384 | .xwordwise_sse, |
| ... | ... | @@ -175539,6 +175555,7 @@ fn genBinOpMir( |
| 175539 | 175555 | .register_overflow, |
| 175540 | 175556 | .register_mask, |
| 175541 | 175557 | .indirect_load_frame, |
| 175558 | .register_tee, |
| 175542 | 175559 | .elementwise_gpr, |
| 175543 | 175560 | .elementwise_sse, |
| 175544 | 175561 | .xwordwise_sse, |
| ... | ... | @@ -175647,6 +175664,7 @@ fn genBinOpMir( |
| 175647 | 175664 | .register_overflow, |
| 175648 | 175665 | .register_mask, |
| 175649 | 175666 | .indirect_load_frame, |
| 175667 | .register_tee, |
| 175650 | 175668 | .elementwise_gpr, |
| 175651 | 175669 | .elementwise_sse, |
| 175652 | 175670 | .xwordwise_sse, |
| ... | ... | @@ -176002,6 +176020,7 @@ fn genLocalDebugInfo(cg: *CodeGen, air_tag: Air.Inst.Tag, ty: Type, mcv: MCValue |
| 176002 | 176020 | .none, |
| 176003 | 176021 | .unreach, |
| 176004 | 176022 | .dead, |
| 176023 | .register_tee, |
| 176005 | 176024 | .elementwise_gpr, |
| 176006 | 176025 | .elementwise_sse, |
| 176007 | 176026 | .xwordwise_sse, |
| ... | ... | @@ -176060,6 +176079,7 @@ fn genLocalDebugInfo(cg: *CodeGen, air_tag: Air.Inst.Tag, ty: Type, mcv: MCValue |
| 176060 | 176079 | .none, |
| 176061 | 176080 | .unreach, |
| 176062 | 176081 | .dead, |
| 176082 | .register_tee, |
| 176063 | 176083 | .elementwise_gpr, |
| 176064 | 176084 | .elementwise_sse, |
| 176065 | 176085 | .xwordwise_sse, |
| ... | ... | @@ -176197,14 +176217,14 @@ fn genCall(cg: *CodeGen, info: union(enum) { |
| 176197 | 176217 | |
| 176198 | 176218 | const var_args = try allocator.alloc(Type, args.len - fn_info.param_types.len); |
| 176199 | 176219 | defer allocator.free(var_args); |
| 176200 | | for (var_args, arg_types[fn_info.param_types.len..]) |*var_arg, arg_ty| var_arg.* = arg_ty; |
| 176220 | @memcpy(var_args, arg_types[fn_info.param_types.len..]); |
| 176201 | 176221 | |
| 176202 | 176222 | const frame_indices = try allocator.alloc(FrameIndex, args.len); |
| 176203 | 176223 | defer allocator.free(frame_indices); |
| 176204 | 176224 | |
| 176205 | | var reg_locks: std.array_list.Managed(?RegisterLock) = .init(allocator); |
| 176206 | | defer reg_locks.deinit(); |
| 176207 | | try reg_locks.ensureTotalCapacity(16); |
| 176225 | var reg_locks: std.ArrayList(?RegisterLock) = .empty; |
| 176226 | defer reg_locks.deinit(allocator); |
| 176227 | try reg_locks.ensureTotalCapacity(allocator, 16); |
| 176208 | 176228 | defer for (reg_locks.items) |reg_lock| if (reg_lock) |lock| cg.register_manager.unlockReg(lock); |
| 176209 | 176229 | |
| 176210 | 176230 | var call_info = try cg.resolveCallingConventionValues(fn_info, var_args, .call_frame); |
| ... | ... | @@ -176235,169 +176255,165 @@ fn genCall(cg: *CodeGen, info: union(enum) { |
| 176235 | 176255 | .indirect => |reg_off| try cg.register_manager.getReg(reg_off.reg, null), |
| 176236 | 176256 | else => unreachable, |
| 176237 | 176257 | } |
| 176238 | | for (call_info.args, arg_types, args, frame_indices, 0..) |dst_arg, arg_ty, src_arg, *frame_index, arg_i| |
| 176239 | | switch (dst_arg) { |
| 176240 | | .none => {}, |
| 176241 | | .register => |reg| { |
| 176258 | for (call_info.args, arg_types, args, frame_indices) |dst_arg, arg_ty, src_arg, *frame_index| switch (dst_arg) { |
| 176259 | .none => {}, |
| 176260 | .register => |reg| { |
| 176261 | try cg.register_manager.getReg(reg, null); |
| 176262 | try reg_locks.append(allocator, cg.register_manager.lockReg(reg)); |
| 176263 | }, |
| 176264 | inline .register_pair, .register_triple, .register_quadruple => |regs| { |
| 176265 | for (regs) |reg| try cg.register_manager.getReg(reg, null); |
| 176266 | try reg_locks.appendSlice(allocator, &cg.register_manager.lockRegs(regs.len, regs)); |
| 176267 | }, |
| 176268 | .indirect => |reg_off| { |
| 176269 | frame_index.* = try cg.allocFrameIndex(.initType(arg_ty, zcu)); |
| 176270 | try cg.genSetMem(.{ .frame = frame_index.* }, 0, arg_ty, src_arg, opts); |
| 176271 | try cg.register_manager.getReg(reg_off.reg, null); |
| 176272 | try reg_locks.append(allocator, cg.register_manager.lockReg(reg_off.reg)); |
| 176273 | }, |
| 176274 | .indirect_load_frame => |frame_addr| { |
| 176275 | frame_index.* = try cg.allocFrameIndex(.initType(arg_ty, zcu)); |
| 176276 | try cg.genSetMem(.{ .frame = frame_index.* }, 0, arg_ty, src_arg, opts); |
| 176277 | try cg.genSetMem( |
| 176278 | .{ .frame = frame_addr.index }, |
| 176279 | frame_addr.off, |
| 176280 | .usize, |
| 176281 | .{ .lea_frame = .{ .index = frame_index.* } }, |
| 176282 | opts, |
| 176283 | ); |
| 176284 | }, |
| 176285 | .load_frame => { |
| 176286 | try cg.genCopy(arg_ty, dst_arg, src_arg, opts); |
| 176287 | try cg.freeValue(src_arg, .{}); |
| 176288 | }, |
| 176289 | .register_tee => |regs| { |
| 176290 | try reg_locks.ensureUnusedCapacity(allocator, regs.len); |
| 176291 | for (regs) |reg| if (reg != .none) { |
| 176242 | 176292 | try cg.register_manager.getReg(reg, null); |
| 176243 | | try reg_locks.append(cg.register_manager.lockReg(reg)); |
| 176244 | | |
| 176245 | | if (fn_info.is_var_args and |
| 176246 | | fn_info.cc == .x86_64_win and |
| 176247 | | reg.class() == .sse and |
| 176248 | | arg_i < abi.Win64.c_abi_int_param_regs.len) |
| 176249 | | { |
| 176250 | | // Floating point arguments must be duplicated into the equivalent integer registers on this ABI |
| 176251 | | const int_reg = abi.Win64.c_abi_int_param_regs[arg_i]; |
| 176252 | | try reg_locks.append(cg.register_manager.lockReg(int_reg)); |
| 176253 | | } |
| 176254 | | }, |
| 176255 | | .register_pair => |regs| { |
| 176256 | | for (regs) |reg| try cg.register_manager.getReg(reg, null); |
| 176257 | | try reg_locks.appendSlice(&cg.register_manager.lockRegs(2, regs)); |
| 176258 | | }, |
| 176259 | | .indirect => |reg_off| { |
| 176260 | | frame_index.* = try cg.allocFrameIndex(.initType(arg_ty, zcu)); |
| 176261 | | try cg.genSetMem(.{ .frame = frame_index.* }, 0, arg_ty, src_arg, opts); |
| 176262 | | try cg.register_manager.getReg(reg_off.reg, null); |
| 176263 | | try reg_locks.append(cg.register_manager.lockReg(reg_off.reg)); |
| 176264 | | }, |
| 176265 | | .indirect_load_frame => |frame_addr| { |
| 176266 | | frame_index.* = try cg.allocFrameIndex(.initType(arg_ty, zcu)); |
| 176267 | | try cg.genSetMem(.{ .frame = frame_index.* }, 0, arg_ty, src_arg, opts); |
| 176268 | | try cg.genSetMem( |
| 176269 | | .{ .frame = frame_addr.index }, |
| 176270 | | frame_addr.off, |
| 176271 | | .usize, |
| 176272 | | .{ .lea_frame = .{ .index = frame_index.* } }, |
| 176273 | | opts, |
| 176274 | | ); |
| 176275 | | }, |
| 176276 | | .load_frame => { |
| 176277 | | try cg.genCopy(arg_ty, dst_arg, src_arg, opts); |
| 176278 | | try cg.freeValue(src_arg, .{}); |
| 176279 | | }, |
| 176280 | | .elementwise_gpr, .elementwise_sse => |regs_frame_addr| { |
| 176281 | | const src_mem: Memory = if (src_arg.isBase()) try src_arg.mem(cg, .{ .size = .dword }) else .{ |
| 176282 | | .base = .{ .reg = try cg.copyToTmpRegister(.usize, switch (src_arg) { |
| 176283 | | else => src_arg, |
| 176284 | | .air_ref => |src_ref| try cg.resolveInst(src_ref), |
| 176285 | | }.address()) }, |
| 176286 | | .mod = .{ .rm = .{ .size = .dword } }, |
| 176287 | | }; |
| 176288 | | const src_lock = switch (src_mem.base) { |
| 176289 | | .reg => |src_reg| cg.register_manager.lockReg(src_reg), |
| 176290 | | else => null, |
| 176291 | | }; |
| 176292 | | defer if (src_lock) |lock| cg.register_manager.unlockReg(lock); |
| 176293 | | |
| 176294 | | const elem_rc: Register.Class, const param_regs = switch (dst_arg) { |
| 176295 | | else => unreachable, |
| 176296 | | .elementwise_gpr => .{ .general_purpose, abi.getCAbiIntParamRegs(fn_info.cc) }, |
| 176297 | | .elementwise_sse => .{ .sse, abi.getCAbiSseParamRegs(fn_info.cc, cg.target) }, |
| 176298 | | }; |
| 176299 | | const len = arg_ty.vectorLen(zcu); |
| 176300 | | const param_reg_len: u31 = |
| 176301 | | @intCast(@min(param_regs.len - regs_frame_addr.info.reg_index, len)); |
| 176293 | reg_locks.appendAssumeCapacity(cg.register_manager.lockReg(reg)); |
| 176294 | }; |
| 176295 | }, |
| 176296 | .elementwise_gpr, .elementwise_sse => |regs_frame_addr| { |
| 176297 | const src_mem: Memory = if (src_arg.isBase()) try src_arg.mem(cg, .{ .size = .dword }) else .{ |
| 176298 | .base = .{ .reg = try cg.copyToTmpRegister(.usize, switch (src_arg) { |
| 176299 | else => src_arg, |
| 176300 | .air_ref => |src_ref| try cg.resolveInst(src_ref), |
| 176301 | }.address()) }, |
| 176302 | .mod = .{ .rm = .{ .size = .dword } }, |
| 176303 | }; |
| 176304 | const src_lock = switch (src_mem.base) { |
| 176305 | .reg => |src_reg| cg.register_manager.lockReg(src_reg), |
| 176306 | else => null, |
| 176307 | }; |
| 176308 | defer if (src_lock) |lock| cg.register_manager.unlockReg(lock); |
| 176302 | 176309 | |
| 176303 | | if (len - param_reg_len > 0) { |
| 176304 | | const index_reg = try cg.register_manager.allocReg(null, abi.RegisterClass.gp); |
| 176305 | | const index_lock = cg.register_manager.lockRegAssumeUnused(index_reg); |
| 176306 | | defer cg.register_manager.unlockReg(index_lock); |
| 176307 | | try cg.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), .u(param_reg_len)); |
| 176310 | const elem_rc: Register.Class, const param_regs = switch (dst_arg) { |
| 176311 | else => unreachable, |
| 176312 | .elementwise_gpr => .{ .general_purpose, abi.getCAbiIntParamRegs(fn_info.cc) }, |
| 176313 | .elementwise_sse => .{ .sse, abi.getCAbiSseParamRegs(fn_info.cc, cg.target) }, |
| 176314 | }; |
| 176315 | const len = arg_ty.vectorLen(zcu); |
| 176316 | const param_reg_len: u31 = |
| 176317 | @intCast(@min(param_regs.len - regs_frame_addr.info.reg_index, len)); |
| 176308 | 176318 | |
| 176309 | | const loop: Mir.Inst.Index = @intCast(cg.mir_instructions.len); |
| 176310 | | const elem_ty = arg_ty.childType(zcu); |
| 176311 | | if (elem_ty.toIntern() == .bool_type) { |
| 176312 | | try cg.asmMemoryRegister(.{ ._, .bt }, src_mem, index_reg.to32()); |
| 176313 | | try cg.asmSetccMemory(.c, .{ |
| 176314 | | .base = .{ .frame = regs_frame_addr.frame_index }, |
| 176315 | | .mod = .{ .rm = .{ |
| 176316 | | .size = .byte, |
| 176317 | | .index = index_reg.to64(), |
| 176318 | | .scale = .@"8", |
| 176319 | | .disp = @as(i32, regs_frame_addr.info.frame_off) - 8 * param_reg_len, |
| 176320 | | } }, |
| 176321 | | }); |
| 176322 | | } else { |
| 176323 | | const elem_reg = |
| 176324 | | try cg.register_manager.allocReg(null, regSetForRegClass(elem_rc)); |
| 176325 | | const elem_lock = cg.register_manager.lockRegAssumeUnused(elem_reg); |
| 176326 | | defer cg.register_manager.unlockReg(elem_lock); |
| 176327 | | const elem_size = cg.memSize(elem_ty, .general_purpose); |
| 176328 | | const elem_alias = elem_reg.toSize(elem_size, cg.target); |
| 176319 | if (len - param_reg_len > 0) { |
| 176320 | const index_reg = try cg.register_manager.allocReg(null, abi.RegisterClass.gp); |
| 176321 | const index_lock = cg.register_manager.lockRegAssumeUnused(index_reg); |
| 176322 | defer cg.register_manager.unlockReg(index_lock); |
| 176323 | try cg.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), .u(param_reg_len)); |
| 176329 | 176324 | |
| 176330 | | const strat = try cg.moveStrategy(elem_ty, elem_rc, true); |
| 176331 | | assert(src_mem.mod.rm.index == .none and src_mem.mod.rm.scale == .@"1"); |
| 176332 | | try strat.read(cg, elem_alias, .{ |
| 176333 | | .base = src_mem.base, |
| 176334 | | .mod = .{ .rm = .{ |
| 176335 | | .size = elem_size, |
| 176336 | | .index = index_reg.to64(), |
| 176337 | | .scale = .fromFactor(@intCast(@divExact(elem_size.bitSize(cg.target), 8))), |
| 176338 | | .disp = src_mem.mod.rm.disp, |
| 176339 | | } }, |
| 176340 | | }); |
| 176341 | | try strat.write(cg, .{ |
| 176342 | | .base = .{ .frame = regs_frame_addr.frame_index }, |
| 176343 | | .mod = .{ .rm = .{ |
| 176344 | | .size = elem_size, |
| 176345 | | .index = index_reg.to64(), |
| 176346 | | .scale = .@"8", |
| 176347 | | .disp = @as(i32, regs_frame_addr.info.frame_off) - 8 * param_reg_len, |
| 176348 | | } }, |
| 176349 | | }, elem_alias); |
| 176350 | | } |
| 176351 | | if (cg.hasFeature(.slow_incdec)) { |
| 176352 | | try cg.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), .u(1)); |
| 176353 | | } else { |
| 176354 | | try cg.asmRegister(.{ ._c, .in }, index_reg.to32()); |
| 176355 | | } |
| 176356 | | try cg.asmRegisterImmediate(.{ ._, .cmp }, index_reg.to32(), .u(len)); |
| 176357 | | _ = try cg.asmJccReloc(.b, loop); |
| 176325 | const loop: Mir.Inst.Index = @intCast(cg.mir_instructions.len); |
| 176326 | const elem_ty = arg_ty.childType(zcu); |
| 176327 | if (elem_ty.toIntern() == .bool_type) { |
| 176328 | try cg.asmMemoryRegister(.{ ._, .bt }, src_mem, index_reg.to32()); |
| 176329 | try cg.asmSetccMemory(.c, .{ |
| 176330 | .base = .{ .frame = regs_frame_addr.frame_index }, |
| 176331 | .mod = .{ .rm = .{ |
| 176332 | .size = .byte, |
| 176333 | .index = index_reg.to64(), |
| 176334 | .scale = .@"8", |
| 176335 | .disp = @as(i32, regs_frame_addr.info.frame_off) - 8 * param_reg_len, |
| 176336 | } }, |
| 176337 | }); |
| 176338 | } else { |
| 176339 | const elem_reg = |
| 176340 | try cg.register_manager.allocReg(null, regSetForRegClass(elem_rc)); |
| 176341 | const elem_lock = cg.register_manager.lockRegAssumeUnused(elem_reg); |
| 176342 | defer cg.register_manager.unlockReg(elem_lock); |
| 176343 | const elem_size = cg.memSize(elem_ty, .general_purpose); |
| 176344 | const elem_alias = elem_reg.toSize(elem_size, cg.target); |
| 176345 | |
| 176346 | const strat = try cg.moveStrategy(elem_ty, elem_rc, true); |
| 176347 | assert(src_mem.mod.rm.index == .none and src_mem.mod.rm.scale == .@"1"); |
| 176348 | try strat.read(cg, elem_alias, .{ |
| 176349 | .base = src_mem.base, |
| 176350 | .mod = .{ .rm = .{ |
| 176351 | .size = elem_size, |
| 176352 | .index = index_reg.to64(), |
| 176353 | .scale = .fromFactor(@intCast(@divExact(elem_size.bitSize(cg.target), 8))), |
| 176354 | .disp = src_mem.mod.rm.disp, |
| 176355 | } }, |
| 176356 | }); |
| 176357 | try strat.write(cg, .{ |
| 176358 | .base = .{ .frame = regs_frame_addr.frame_index }, |
| 176359 | .mod = .{ .rm = .{ |
| 176360 | .size = elem_size, |
| 176361 | .index = index_reg.to64(), |
| 176362 | .scale = .@"8", |
| 176363 | .disp = @as(i32, regs_frame_addr.info.frame_off) - 8 * param_reg_len, |
| 176364 | } }, |
| 176365 | }, elem_alias); |
| 176358 | 176366 | } |
| 176359 | | |
| 176360 | | for (param_regs[regs_frame_addr.info.reg_index..][0..param_reg_len]) |dst_reg| { |
| 176361 | | try cg.register_manager.getReg(dst_reg, null); |
| 176362 | | try reg_locks.append(cg.register_manager.lockReg(dst_reg)); |
| 176367 | if (cg.hasFeature(.slow_incdec)) { |
| 176368 | try cg.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), .u(1)); |
| 176369 | } else { |
| 176370 | try cg.asmRegister(.{ ._c, .in }, index_reg.to32()); |
| 176363 | 176371 | } |
| 176364 | | }, |
| 176365 | | .xwordwise_sse, .ywordwise_sse, .zwordwise_sse => |regs_frame_addr| { |
| 176366 | | const elem_size: u31 = switch (dst_arg) { |
| 176367 | | else => unreachable, |
| 176368 | | .xwordwise_sse => 16, |
| 176369 | | .ywordwise_sse => 32, |
| 176370 | | .zwordwise_sse => 64, |
| 176371 | | }; |
| 176372 | | const param_gpr_regs = abi.getCAbiIntParamRegs(fn_info.cc); |
| 176373 | | const arg_size: u31 = @intCast(arg_ty.abiSize(zcu)); |
| 176374 | | const len = @divExact(arg_size, elem_size); |
| 176375 | | const param_gpr_len: u31 = |
| 176376 | | @intCast(@min(param_gpr_regs.len - regs_frame_addr.info.reg_index, len)); |
| 176372 | try cg.asmRegisterImmediate(.{ ._, .cmp }, index_reg.to32(), .u(len)); |
| 176373 | _ = try cg.asmJccReloc(.b, loop); |
| 176374 | } |
| 176377 | 176375 | |
| 176378 | | frame_index.* = try cg.allocFrameIndex(.initType(arg_ty, zcu)); |
| 176379 | | try cg.genSetMem(.{ .frame = frame_index.* }, 0, arg_ty, src_arg, opts); |
| 176380 | | |
| 176381 | | var frame_offset: i32 = regs_frame_addr.info.frame_off; |
| 176382 | | var arg_offset = elem_size * param_gpr_len; |
| 176383 | | while (arg_size - arg_offset != 0) : ({ |
| 176384 | | frame_offset += 8; |
| 176385 | | arg_offset += elem_size; |
| 176386 | | }) try cg.genSetMem( |
| 176387 | | .{ .frame = regs_frame_addr.frame_index }, |
| 176388 | | frame_offset, |
| 176389 | | .usize, |
| 176390 | | .{ .lea_frame = .{ .index = frame_index.*, .off = arg_offset } }, |
| 176391 | | opts, |
| 176392 | | ); |
| 176376 | for (param_regs[regs_frame_addr.info.reg_index..][0..param_reg_len]) |dst_reg| { |
| 176377 | try cg.register_manager.getReg(dst_reg, null); |
| 176378 | try reg_locks.append(allocator, cg.register_manager.lockReg(dst_reg)); |
| 176379 | } |
| 176380 | }, |
| 176381 | .xwordwise_sse, .ywordwise_sse, .zwordwise_sse => |regs_frame_addr| { |
| 176382 | const elem_size: u31 = switch (dst_arg) { |
| 176383 | else => unreachable, |
| 176384 | .xwordwise_sse => 16, |
| 176385 | .ywordwise_sse => 32, |
| 176386 | .zwordwise_sse => 64, |
| 176387 | }; |
| 176388 | const param_gpr_regs = abi.getCAbiIntParamRegs(fn_info.cc); |
| 176389 | const arg_size: u31 = @intCast(arg_ty.abiSize(zcu)); |
| 176390 | const len = @divExact(arg_size, elem_size); |
| 176391 | const param_gpr_len: u31 = |
| 176392 | @intCast(@min(param_gpr_regs.len - regs_frame_addr.info.reg_index, len)); |
| 176393 | |
| 176394 | frame_index.* = try cg.allocFrameIndex(.initType(arg_ty, zcu)); |
| 176395 | try cg.genSetMem(.{ .frame = frame_index.* }, 0, arg_ty, src_arg, opts); |
| 176396 | |
| 176397 | var frame_offset: i32 = regs_frame_addr.info.frame_off; |
| 176398 | var arg_offset = elem_size * param_gpr_len; |
| 176399 | while (arg_size - arg_offset != 0) : ({ |
| 176400 | frame_offset += 8; |
| 176401 | arg_offset += elem_size; |
| 176402 | }) try cg.genSetMem( |
| 176403 | .{ .frame = regs_frame_addr.frame_index }, |
| 176404 | frame_offset, |
| 176405 | .usize, |
| 176406 | .{ .lea_frame = .{ .index = frame_index.*, .off = arg_offset } }, |
| 176407 | opts, |
| 176408 | ); |
| 176393 | 176409 | |
| 176394 | | for (param_gpr_regs[regs_frame_addr.info.reg_index..][0..param_gpr_len]) |dst_reg| { |
| 176395 | | try cg.register_manager.getReg(dst_reg, null); |
| 176396 | | try reg_locks.append(cg.register_manager.lockReg(dst_reg)); |
| 176397 | | } |
| 176398 | | }, |
| 176399 | | else => unreachable, |
| 176400 | | }; |
| 176410 | for (param_gpr_regs[regs_frame_addr.info.reg_index..][0..param_gpr_len]) |dst_reg| { |
| 176411 | try cg.register_manager.getReg(dst_reg, null); |
| 176412 | try reg_locks.append(allocator, cg.register_manager.lockReg(dst_reg)); |
| 176413 | } |
| 176414 | }, |
| 176415 | else => unreachable, |
| 176416 | }; |
| 176401 | 176417 | |
| 176402 | 176418 | if (call_info.err_ret_trace_reg != .none) { |
| 176403 | 176419 | if (cg.inst_tracking.getPtr(err_ret_trace_index)) |err_ret_trace| { |
| ... | ... | @@ -176406,7 +176422,7 @@ fn genCall(cg: *CodeGen, info: union(enum) { |
| 176406 | 176422 | else => true, |
| 176407 | 176423 | }) { |
| 176408 | 176424 | try cg.register_manager.getReg(call_info.err_ret_trace_reg, err_ret_trace_index); |
| 176409 | | try reg_locks.append(cg.register_manager.lockReg(call_info.err_ret_trace_reg)); |
| 176425 | try reg_locks.append(allocator, cg.register_manager.lockReg(call_info.err_ret_trace_reg)); |
| 176410 | 176426 | |
| 176411 | 176427 | try cg.genSetReg(call_info.err_ret_trace_reg, .usize, err_ret_trace.short, opts); |
| 176412 | 176428 | err_ret_trace.trackMaterialize(err_ret_trace_index, .{ |
| ... | ... | @@ -176427,114 +176443,102 @@ fn genCall(cg: *CodeGen, info: union(enum) { |
| 176427 | 176443 | .lea_frame = .{ .index = frame_index, .off = -reg_off.off }, |
| 176428 | 176444 | }, opts); |
| 176429 | 176445 | call_info.return_value.short = .{ .load_frame = .{ .index = frame_index } }; |
| 176430 | | try reg_locks.append(cg.register_manager.lockReg(reg_off.reg)); |
| 176446 | try reg_locks.append(allocator, cg.register_manager.lockReg(reg_off.reg)); |
| 176431 | 176447 | }, |
| 176432 | 176448 | else => unreachable, |
| 176433 | 176449 | } |
| 176434 | 176450 | |
| 176435 | | for (call_info.args, arg_types, args, frame_indices, 0..) |dst_arg, arg_ty, src_arg, frame_index, arg_i| |
| 176436 | | switch (dst_arg) { |
| 176437 | | .none, .load_frame, .indirect_load_frame => {}, |
| 176438 | | .register => |dst_reg| switch (fn_info.cc) { |
| 176439 | | else => try cg.genSetReg(registerAlias( |
| 176440 | | dst_reg, |
| 176441 | | @intCast(arg_ty.abiSize(zcu)), |
| 176442 | | ), arg_ty, src_arg, opts), |
| 176443 | | .x86_64_sysv, .x86_64_win => { |
| 176444 | | const promoted_ty = cg.promoteInt(arg_ty); |
| 176445 | | const promoted_unaligned_size: u32 = @intCast(cg.unalignedSize(promoted_ty)); |
| 176446 | | const dst_alias = registerAlias(dst_reg, promoted_unaligned_size); |
| 176447 | | try cg.genSetReg(dst_alias, promoted_ty, src_arg, opts); |
| 176448 | | if (promoted_ty.toIntern() != arg_ty.toIntern()) |
| 176449 | | try cg.truncateRegister(arg_ty, dst_alias); |
| 176450 | | |
| 176451 | | if (fn_info.is_var_args and |
| 176452 | | fn_info.cc == .x86_64_win and |
| 176453 | | dst_reg.class() == .sse and |
| 176454 | | arg_i < abi.Win64.c_abi_int_param_regs.len) |
| 176455 | | { |
| 176456 | | const int_dst_reg = abi.Win64.c_abi_int_param_regs[arg_i]; |
| 176457 | | const int_dst_alias = registerAlias(int_dst_reg, promoted_unaligned_size); |
| 176458 | | try cg.genSetReg(int_dst_alias, promoted_ty, .{ .register = dst_alias }, opts); |
| 176459 | | } |
| 176460 | | }, |
| 176461 | | }, |
| 176462 | | .register_pair => try cg.genCopy(arg_ty, dst_arg, src_arg, opts), |
| 176463 | | .indirect => |reg_off| try cg.genSetReg(reg_off.reg, .usize, .{ |
| 176464 | | .lea_frame = .{ .index = frame_index, .off = -reg_off.off }, |
| 176465 | | }, opts), |
| 176466 | | .elementwise_gpr, .elementwise_sse => |regs_frame_addr| { |
| 176467 | | const src_mem: Memory = if (src_arg.isBase()) try src_arg.mem(cg, .{ .size = .dword }) else .{ |
| 176468 | | .base = .{ .reg = try cg.copyToTmpRegister( |
| 176469 | | .usize, |
| 176470 | | switch (src_arg) { |
| 176471 | | else => src_arg, |
| 176472 | | .air_ref => |src_ref| try cg.resolveInst(src_ref), |
| 176473 | | }.address(), |
| 176474 | | ) }, |
| 176475 | | .mod = .{ .rm = .{ .size = .dword } }, |
| 176476 | | }; |
| 176477 | | const src_lock = switch (src_mem.base) { |
| 176478 | | .reg => |src_reg| cg.register_manager.lockReg(src_reg), |
| 176479 | | else => null, |
| 176480 | | }; |
| 176481 | | defer if (src_lock) |lock| cg.register_manager.unlockReg(lock); |
| 176451 | for (call_info.args, arg_types, args, frame_indices) |dst_arg, arg_ty, src_arg, frame_index| switch (dst_arg) { |
| 176452 | .none, .load_frame, .indirect_load_frame => {}, |
| 176453 | .register => |dst_reg| try cg.genSetReg(registerAlias( |
| 176454 | dst_reg, |
| 176455 | @intCast(cg.unalignedSize(arg_ty)), |
| 176456 | ), arg_ty, src_arg, opts), |
| 176457 | .register_pair, |
| 176458 | .register_triple, |
| 176459 | .register_quadruple, |
| 176460 | => try cg.genCopy(arg_ty, dst_arg, src_arg, opts), |
| 176461 | .indirect => |reg_off| try cg.genSetReg(reg_off.reg, .usize, .{ |
| 176462 | .lea_frame = .{ .index = frame_index, .off = -reg_off.off }, |
| 176463 | }, opts), |
| 176464 | .register_tee => |dst_regs| { |
| 176465 | try cg.genSetReg(dst_regs[0], arg_ty, src_arg, opts); |
| 176466 | for (dst_regs[1..]) |dst_reg| if (dst_reg != .none) try cg.genSetReg(dst_reg, arg_ty, .{ |
| 176467 | .register = dst_regs[0], |
| 176468 | }, opts); |
| 176469 | }, |
| 176470 | .elementwise_gpr, .elementwise_sse => |regs_frame_addr| { |
| 176471 | const src_mem: Memory = if (src_arg.isBase()) try src_arg.mem(cg, .{ .size = .dword }) else .{ |
| 176472 | .base = .{ .reg = try cg.copyToTmpRegister( |
| 176473 | .usize, |
| 176474 | switch (src_arg) { |
| 176475 | else => src_arg, |
| 176476 | .air_ref => |src_ref| try cg.resolveInst(src_ref), |
| 176477 | }.address(), |
| 176478 | ) }, |
| 176479 | .mod = .{ .rm = .{ .size = .dword } }, |
| 176480 | }; |
| 176481 | const src_lock = switch (src_mem.base) { |
| 176482 | .reg => |src_reg| cg.register_manager.lockReg(src_reg), |
| 176483 | else => null, |
| 176484 | }; |
| 176485 | defer if (src_lock) |lock| cg.register_manager.unlockReg(lock); |
| 176482 | 176486 | |
| 176483 | | const elem_rc: Register.Class, const param_regs = switch (dst_arg) { |
| 176484 | | else => unreachable, |
| 176485 | | .elementwise_gpr => .{ .general_purpose, abi.getCAbiIntParamRegs(fn_info.cc) }, |
| 176486 | | .elementwise_sse => .{ .sse, abi.getCAbiSseParamRegs(fn_info.cc, cg.target) }, |
| 176487 | | }; |
| 176488 | | const len = arg_ty.vectorLen(zcu); |
| 176489 | | const elem_ty = arg_ty.childType(zcu); |
| 176490 | | const elem_size = cg.memSize(elem_ty, .general_purpose); |
| 176491 | | const elem_abi_size = @divExact(elem_size.bitSize(cg.target), 8); |
| 176492 | | const param_reg_len: u31 = |
| 176493 | | @intCast(@min(param_regs.len - regs_frame_addr.info.reg_index, len)); |
| 176494 | | const strat = if (elem_ty.toIntern() == .bool_type) strat: { |
| 176495 | | try cg.spillEflagsIfOccupied(); |
| 176496 | | break :strat undefined; |
| 176497 | | } else try cg.moveStrategy(elem_ty, elem_rc, true); |
| 176498 | | for ( |
| 176499 | | param_regs[regs_frame_addr.info.reg_index..][0..param_reg_len], |
| 176500 | | 0.., |
| 176501 | | ) |dst_reg, elem_index| if (elem_ty.toIntern() == .bool_type) { |
| 176502 | | try cg.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()); |
| 176503 | | try cg.asmMemoryImmediate(.{ ._, .bt }, src_mem, .u(elem_index)); |
| 176504 | | try cg.asmSetccRegister(.c, dst_reg.to8()); |
| 176505 | | } else try strat.read(cg, dst_reg.toSize(elem_size, cg.target), .{ |
| 176506 | | .base = src_mem.base, |
| 176507 | | .mod = .{ .rm = .{ |
| 176508 | | .size = elem_size, |
| 176509 | | .disp = src_mem.mod.rm.disp + @as(u31, @intCast(elem_abi_size * elem_index)), |
| 176510 | | } }, |
| 176511 | | }); |
| 176512 | | }, |
| 176513 | | .xwordwise_sse, .ywordwise_sse, .zwordwise_sse => |regs_frame_addr| { |
| 176514 | | const elem_size: u31 = switch (dst_arg) { |
| 176515 | | else => unreachable, |
| 176516 | | .xwordwise_sse => 16, |
| 176517 | | .ywordwise_sse => 32, |
| 176518 | | .zwordwise_sse => 64, |
| 176519 | | }; |
| 176520 | | const param_gpr_regs = abi.getCAbiIntParamRegs(fn_info.cc); |
| 176521 | | const len = @divExact(arg_ty.abiSize(zcu), elem_size); |
| 176522 | | const param_gpr_len: u31 = |
| 176523 | | @intCast(@min(param_gpr_regs.len - regs_frame_addr.info.reg_index, len)); |
| 176487 | const elem_rc: Register.Class, const param_regs = switch (dst_arg) { |
| 176488 | else => unreachable, |
| 176489 | .elementwise_gpr => .{ .general_purpose, abi.getCAbiIntParamRegs(fn_info.cc) }, |
| 176490 | .elementwise_sse => .{ .sse, abi.getCAbiSseParamRegs(fn_info.cc, cg.target) }, |
| 176491 | }; |
| 176492 | const len = arg_ty.vectorLen(zcu); |
| 176493 | const elem_ty = arg_ty.childType(zcu); |
| 176494 | const elem_size = cg.memSize(elem_ty, .general_purpose); |
| 176495 | const elem_abi_size = @divExact(elem_size.bitSize(cg.target), 8); |
| 176496 | const param_reg_len: u31 = |
| 176497 | @intCast(@min(param_regs.len - regs_frame_addr.info.reg_index, len)); |
| 176498 | const strat = if (elem_ty.toIntern() == .bool_type) strat: { |
| 176499 | try cg.spillEflagsIfOccupied(); |
| 176500 | break :strat undefined; |
| 176501 | } else try cg.moveStrategy(elem_ty, elem_rc, true); |
| 176502 | for ( |
| 176503 | param_regs[regs_frame_addr.info.reg_index..][0..param_reg_len], |
| 176504 | 0.., |
| 176505 | ) |dst_reg, elem_index| if (elem_ty.toIntern() == .bool_type) { |
| 176506 | try cg.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()); |
| 176507 | try cg.asmMemoryImmediate(.{ ._, .bt }, src_mem, .u(elem_index)); |
| 176508 | try cg.asmSetccRegister(.c, dst_reg.to8()); |
| 176509 | } else try strat.read(cg, dst_reg.toSize(elem_size, cg.target), .{ |
| 176510 | .base = src_mem.base, |
| 176511 | .mod = .{ .rm = .{ |
| 176512 | .size = elem_size, |
| 176513 | .disp = src_mem.mod.rm.disp + @as(u31, @intCast(elem_abi_size * elem_index)), |
| 176514 | } }, |
| 176515 | }); |
| 176516 | }, |
| 176517 | .xwordwise_sse, .ywordwise_sse, .zwordwise_sse => |regs_frame_addr| { |
| 176518 | const elem_size: u31 = switch (dst_arg) { |
| 176519 | else => unreachable, |
| 176520 | .xwordwise_sse => 16, |
| 176521 | .ywordwise_sse => 32, |
| 176522 | .zwordwise_sse => 64, |
| 176523 | }; |
| 176524 | const param_gpr_regs = abi.getCAbiIntParamRegs(fn_info.cc); |
| 176525 | const len = @divExact(arg_ty.abiSize(zcu), elem_size); |
| 176526 | const param_gpr_len: u31 = |
| 176527 | @intCast(@min(param_gpr_regs.len - regs_frame_addr.info.reg_index, len)); |
| 176524 | 176528 | |
| 176525 | | var arg_offset: u31 = 0; |
| 176526 | | for (param_gpr_regs[regs_frame_addr.info.reg_index..][0..param_gpr_len]) |dst_reg| { |
| 176527 | | try cg.genSetReg( |
| 176528 | | dst_reg, |
| 176529 | | .usize, |
| 176530 | | .{ .lea_frame = .{ .index = frame_index, .off = arg_offset } }, |
| 176531 | | opts, |
| 176532 | | ); |
| 176533 | | arg_offset += elem_size; |
| 176534 | | } |
| 176535 | | }, |
| 176536 | | else => unreachable, |
| 176537 | | }; |
| 176529 | var arg_offset: u31 = 0; |
| 176530 | for (param_gpr_regs[regs_frame_addr.info.reg_index..][0..param_gpr_len]) |dst_reg| { |
| 176531 | try cg.genSetReg( |
| 176532 | dst_reg, |
| 176533 | .usize, |
| 176534 | .{ .lea_frame = .{ .index = frame_index, .off = arg_offset } }, |
| 176535 | opts, |
| 176536 | ); |
| 176537 | arg_offset += elem_size; |
| 176538 | } |
| 176539 | }, |
| 176540 | else => unreachable, |
| 176541 | }; |
| 176538 | 176542 | |
| 176539 | 176543 | if (fn_info.is_var_args and fn_info.cc == .x86_64_sysv) |
| 176540 | 176544 | try cg.asmRegisterImmediate(.{ ._, .mov }, .al, .u(call_info.fp_count)); |
| ... | ... | @@ -177474,13 +177478,13 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 177474 | 177478 | const inputs = unwrapped_asm.inputs; |
| 177475 | 177479 | |
| 177476 | 177480 | var result: MCValue = .none; |
| 177477 | | var args: std.array_list.Managed(MCValue) = .init(self.gpa); |
| 177478 | | try args.ensureTotalCapacity(outputs.len + inputs.len); |
| 177481 | var args = try self.gpa.alloc(MCValue, outputs.len + inputs.len); |
| 177482 | var args_len: usize = 0; |
| 177479 | 177483 | defer { |
| 177480 | | for (args.items) |arg| if (arg.getReg()) |reg| self.register_manager.unlockReg(.{ |
| 177484 | for (args[0..args_len]) |arg| if (arg.getReg()) |reg| self.register_manager.unlockReg(.{ |
| 177481 | 177485 | .tracked_index = RegisterManager.indexOfRegIntoTracked(reg) orelse continue, |
| 177482 | 177486 | }); |
| 177483 | | args.deinit(); |
| 177487 | self.gpa.free(args); |
| 177484 | 177488 | } |
| 177485 | 177489 | var arg_map: std.StringHashMap(u8) = .init(self.gpa); |
| 177486 | 177490 | try arg_map.ensureTotalCapacity(@intCast(outputs.len + inputs.len)); |
| ... | ... | @@ -177539,10 +177543,10 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 177539 | 177543 | return self.fail("invalid register constraint: '{s}'", .{out.constraint}) |
| 177540 | 177544 | else if (rest.len == 1 and std.ascii.isDigit(rest[0])) { |
| 177541 | 177545 | const index = std.fmt.charToDigit(rest[0], 10) catch unreachable; |
| 177542 | | if (index >= args.items.len) return self.fail("constraint out of bounds: '{s}'", .{ |
| 177546 | if (index >= args_len) return self.fail("constraint out of bounds: '{s}'", .{ |
| 177543 | 177547 | out.constraint, |
| 177544 | 177548 | }); |
| 177545 | | break :arg_mcv args.items[index]; |
| 177549 | break :arg_mcv args[index]; |
| 177546 | 177550 | } else return self.fail("invalid constraint: '{s}'", .{out.constraint}); |
| 177547 | 177551 | break :arg_mcv if (arg_maybe_reg) |reg| .{ .register = reg } else arg: { |
| 177548 | 177552 | const ptr_mcv = try self.resolveInst(out.operand); |
| ... | ... | @@ -177560,8 +177564,9 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 177560 | 177564 | _ = self.register_manager.lockRegIndexAssumeUnused(tracked_index); |
| 177561 | 177565 | }; |
| 177562 | 177566 | if (!std.mem.eql(u8, out.name, "_")) |
| 177563 | | arg_map.putAssumeCapacityNoClobber(out.name, @intCast(args.items.len)); |
| 177564 | | args.appendAssumeCapacity(arg_mcv); |
| 177567 | arg_map.putAssumeCapacityNoClobber(out.name, @intCast(args_len)); |
| 177568 | args[args_len] = arg_mcv; |
| 177569 | args_len += 1; |
| 177565 | 177570 | if (out.operand == .none) result = arg_mcv; |
| 177566 | 177571 | if (is_read) try self.load(arg_mcv, self.typeOf(out.operand), .{ .air_ref = out.operand }); |
| 177567 | 177572 | } |
| ... | ... | @@ -177644,17 +177649,19 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 177644 | 177649 | break :arg .{ .register = reg }; |
| 177645 | 177650 | } else if (in.constraint.len == 1 and std.ascii.isDigit(in.constraint[0])) arg: { |
| 177646 | 177651 | const index = std.fmt.charToDigit(in.constraint[0], 10) catch unreachable; |
| 177647 | | if (index >= args.items.len) return self.fail("constraint out of bounds: '{s}'", .{in.constraint}); |
| 177648 | | try self.genCopy(ty, args.items[index], input_mcv, .{}); |
| 177649 | | break :arg args.items[index]; |
| 177652 | if (index >= args_len) return self.fail("constraint out of bounds: '{s}'", .{in.constraint}); |
| 177653 | try self.genCopy(ty, args[index], input_mcv, .{}); |
| 177654 | break :arg args[index]; |
| 177650 | 177655 | } else return self.fail("invalid constraint: '{s}'", .{in.constraint}); |
| 177651 | 177656 | if (arg_mcv.getReg()) |reg| if (RegisterManager.indexOfRegIntoTracked(reg)) |_| { |
| 177652 | 177657 | _ = self.register_manager.lockReg(reg); |
| 177653 | 177658 | }; |
| 177654 | 177659 | if (!std.mem.eql(u8, in.name, "_")) |
| 177655 | | arg_map.putAssumeCapacityNoClobber(in.name, @intCast(args.items.len)); |
| 177656 | | args.appendAssumeCapacity(arg_mcv); |
| 177660 | arg_map.putAssumeCapacityNoClobber(in.name, @intCast(args_len)); |
| 177661 | args[args_len] = arg_mcv; |
| 177662 | args_len += 1; |
| 177657 | 177663 | } |
| 177664 | assert(args_len == args.len); |
| 177658 | 177665 | |
| 177659 | 177666 | const ip = &zcu.intern_pool; |
| 177660 | 177667 | const clobbers_val: Value = .fromInterned(unwrapped_asm.clobbers); |
| ... | ... | @@ -177887,7 +177894,7 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 177887 | 177894 | op_str[colon_pos + ":".len .. op_str.len - "]".len] |
| 177888 | 177895 | else |
| 177889 | 177896 | ""; |
| 177890 | | op.* = switch (args.items[ |
| 177897 | op.* = switch (args[ |
| 177891 | 177898 | arg_map.get(op_str["%[".len .. colon orelse op_str.len - "]".len]) orelse |
| 177892 | 177899 | return self.fail("no matching constraint: '{s}'", .{op_str}) |
| 177893 | 177900 | ]) { |
| ... | ... | @@ -178031,7 +178038,7 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 178031 | 178038 | op_str[colon_pos + ":".len .. open - "]".len] |
| 178032 | 178039 | else |
| 178033 | 178040 | ""; |
| 178034 | | break :disp switch (args.items[ |
| 178041 | break :disp switch (args[ |
| 178035 | 178042 | arg_map.get(op_str["%[".len .. colon orelse open - "]".len]) orelse |
| 178036 | 178043 | return self.fail("no matching constraint: '{s}'", .{op_str}) |
| 178037 | 178044 | ]) { |
| ... | ... | @@ -178191,7 +178198,7 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 178191 | 178198 | |
| 178192 | 178199 | it = unwrapped_asm.iterateOutputs(); |
| 178193 | 178200 | while (it.next()) |out| { |
| 178194 | | const arg_mcv = args.items[it.current - 1]; |
| 178201 | const arg_mcv = args[it.current - 1]; |
| 178195 | 178202 | if (out.operand == .none) continue; |
| 178196 | 178203 | if (arg_mcv != .register) continue; |
| 178197 | 178204 | if (out.constraint.len == 2 and std.ascii.isDigit(out.constraint[1])) continue; |
| ... | ... | @@ -178638,6 +178645,7 @@ fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: C |
| 178638 | 178645 | .lea_uav, |
| 178639 | 178646 | .lea_lazy_sym, |
| 178640 | 178647 | .lea_extern_func, |
| 178648 | .register_tee, |
| 178641 | 178649 | .elementwise_gpr, |
| 178642 | 178650 | .elementwise_sse, |
| 178643 | 178651 | .xwordwise_sse, |
| ... | ... | @@ -178653,6 +178661,7 @@ fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: C |
| 178653 | 178661 | .dead, |
| 178654 | 178662 | .undef, |
| 178655 | 178663 | .register_overflow, |
| 178664 | .register_tee, |
| 178656 | 178665 | .elementwise_gpr, |
| 178657 | 178666 | .elementwise_sse, |
| 178658 | 178667 | .xwordwise_sse, |
| ... | ... | @@ -178844,6 +178853,7 @@ fn genSetReg( |
| 178844 | 178853 | .unreach, |
| 178845 | 178854 | .dead, |
| 178846 | 178855 | .indirect_load_frame, |
| 178856 | .register_tee, |
| 178847 | 178857 | .elementwise_gpr, |
| 178848 | 178858 | .elementwise_sse, |
| 178849 | 178859 | .xwordwise_sse, |
| ... | ... | @@ -179422,6 +179432,7 @@ fn genSetMem( |
| 179422 | 179432 | .unreach, |
| 179423 | 179433 | .dead, |
| 179424 | 179434 | .indirect_load_frame, |
| 179435 | .register_tee, |
| 179425 | 179436 | .elementwise_gpr, |
| 179426 | 179437 | .elementwise_sse, |
| 179427 | 179438 | .xwordwise_sse, |
| ... | ... | @@ -181569,142 +181580,139 @@ fn resolveCallingConventionValues( |
| 181569 | 181580 | .x86_64_win => result.stack_byte_count += @intCast(win64_shadow_space), |
| 181570 | 181581 | } |
| 181571 | 181582 | |
| 181572 | | // Return values |
| 181573 | | if (ret_ty.isNoReturn(zcu)) { |
| 181574 | | result.return_value = .init(.unreach); |
| 181575 | | } else if (!ret_ty.hasRuntimeBits(zcu)) { |
| 181576 | | // TODO: is this even possible for C calling convention? |
| 181577 | | result.return_value = .init(.none); |
| 181578 | | } else { |
| 181579 | | var ret_tracking: [4]InstTracking = undefined; |
| 181580 | | var ret_tracking_len: u32 = 0; |
| 181581 | | var ret_gpr = abi.getCAbiIntReturnRegs(cc); |
| 181582 | | var ret_sse = abi.getCAbiSseReturnRegs(cc); |
| 181583 | | var ret_x87 = abi.getCAbiX87ReturnRegs(cc); |
| 181584 | | |
| 181585 | | var classes_buf: [8]abi.Class = undefined; |
| 181586 | | const classes = classes: switch (cc) { |
| 181587 | | else => unreachable, |
| 181588 | | .x86_64_sysv => { |
| 181589 | | classes_buf = abi.classifySystemV(ret_ty, zcu, cg.target, .ret); |
| 181590 | | break :classes std.mem.sliceTo(&classes_buf, .none); |
| 181591 | | }, |
| 181592 | | .x86_64_win => { |
| 181593 | | classes_buf[0] = abi.classifyWindows(ret_ty, zcu, cg.target, .ret); |
| 181594 | | break :classes classes_buf[0..1]; |
| 181595 | | }, |
| 181596 | | }; |
| 181597 | | for (classes) |class| switch (class) { |
| 181598 | | .integer => { |
| 181599 | | ret_tracking[ret_tracking_len] = .init(.{ .register = registerAlias( |
| 181600 | | ret_gpr[0], |
| 181601 | | @intCast(@min(ret_ty.abiSize(zcu), 8)), |
| 181602 | | ) }); |
| 181603 | | ret_tracking_len += 1; |
| 181604 | | ret_gpr = ret_gpr[1..]; |
| 181605 | | }, |
| 181606 | | .sse, .float, .float_combine, .win_i128 => { |
| 181607 | | ret_tracking[ret_tracking_len] = .init(.{ |
| 181608 | | .register = registerAlias(ret_sse[0], @intCast(ret_ty.abiSize(zcu))), |
| 181609 | | }); |
| 181610 | | ret_tracking_len += 1; |
| 181611 | | ret_sse = ret_sse[1..]; |
| 181612 | | }, |
| 181613 | | .sseup => assert(ret_tracking[ret_tracking_len - 1].short.register.isClass(.sse)), |
| 181614 | | .x87 => { |
| 181615 | | ret_tracking[ret_tracking_len] = .init(.{ .register = ret_x87[0] }); |
| 181616 | | ret_tracking_len += 1; |
| 181617 | | ret_x87 = ret_x87[1..]; |
| 181618 | | }, |
| 181619 | | .x87up => assert(ret_tracking[ret_tracking_len - 1].short.register.isClass(.x87)), |
| 181620 | | .none => unreachable, |
| 181621 | | .memory => { |
| 181622 | | ret_tracking[ret_tracking_len] = .{ |
| 181623 | | .short = .{ .indirect = .{ .reg = ret_gpr[0].to64() } }, |
| 181624 | | .long = .{ .indirect = .{ .reg = param_gpr[param_gpr_index].to64() } }, |
| 181625 | | }; |
| 181626 | | ret_tracking_len += 1; |
| 181627 | | ret_gpr = ret_gpr[1..]; |
| 181628 | | param_gpr_index += 1; |
| 181629 | | }, |
| 181630 | | .integer_per_element => { |
| 181631 | | const len: u32 = @intCast(ret_ty.vectorLen(zcu)); |
| 181632 | | const alias_size: u32 = @intCast(@min(ret_ty.childType(zcu).abiSize(zcu), 8)); |
| 181633 | | for (ret_tracking[ret_tracking_len..][0..len], ret_gpr[0..len]) |*tracking, gpr| |
| 181634 | | tracking.* = .init(.{ .register = registerAlias(gpr, alias_size) }); |
| 181635 | | ret_tracking_len += len; |
| 181636 | | ret_gpr = ret_gpr[len..]; |
| 181637 | | }, |
| 181638 | | .sse_per_element => { |
| 181639 | | const len: u32 = @intCast(ret_ty.vectorLen(zcu)); |
| 181640 | | const alias_size: u32 = @intCast(@min(ret_ty.childType(zcu).abiSize(zcu), 8)); |
| 181641 | | for (ret_tracking[ret_tracking_len..][0..len], ret_sse[0..len]) |*tracking, sse| |
| 181642 | | tracking.* = .init(.{ .register = registerAlias(sse, alias_size) }); |
| 181643 | | ret_tracking_len += len; |
| 181644 | | ret_sse = ret_sse[len..]; |
| 181645 | | }, |
| 181646 | | .sse_sse_x87_per_qword, .sse_per_xword, .sse_per_yword, .sse_per_zword => { |
| 181647 | | const reg_size: u32 = switch (class) { |
| 181648 | | else => unreachable, |
| 181649 | | .sse_sse_x87_per_qword => 8, |
| 181650 | | .sse_per_xword => 16, |
| 181651 | | .sse_per_yword => 32, |
| 181652 | | .sse_per_zword => 64, |
| 181653 | | }; |
| 181654 | | var byte_offset: u32 = 0; |
| 181655 | | const unaligned_size = cg.unalignedSize(ret_ty); |
| 181656 | | while (byte_offset < unaligned_size) : (byte_offset += reg_size) { |
| 181657 | | switch (@as(enum { sse, x87 }, switch (class) { |
| 181583 | result.return_value = switch (ret_ty.classify(zcu)) { |
| 181584 | .no_possible_value => .init(.unreach), |
| 181585 | .one_possible_value => .init(.none), |
| 181586 | .runtime, .partially_comptime => return_value: { |
| 181587 | var ret_tracking: [4]InstTracking = undefined; |
| 181588 | var ret_tracking_len: u32 = 0; |
| 181589 | var ret_gpr = abi.getCAbiIntReturnRegs(cc); |
| 181590 | var ret_sse = abi.getCAbiSseReturnRegs(cc); |
| 181591 | var ret_x87 = abi.getCAbiX87ReturnRegs(cc); |
| 181592 | |
| 181593 | var classes_buf: [8]abi.Class = undefined; |
| 181594 | const classes = classes: switch (cc) { |
| 181595 | else => unreachable, |
| 181596 | .x86_64_sysv => { |
| 181597 | classes_buf = abi.classifySystemV(ret_ty, zcu, cg.target, .ret); |
| 181598 | break :classes &classes_buf; |
| 181599 | }, |
| 181600 | .x86_64_win => { |
| 181601 | classes_buf[0] = abi.classifyWindows(ret_ty, zcu, cg.target, .ret); |
| 181602 | break :classes classes_buf[0..1]; |
| 181603 | }, |
| 181604 | }; |
| 181605 | for (classes) |class| switch (class) { |
| 181606 | .integer => { |
| 181607 | ret_tracking[ret_tracking_len] = .init(.{ .register = registerAlias( |
| 181608 | ret_gpr[0], |
| 181609 | @intCast(@min(ret_ty.abiSize(zcu), 8)), |
| 181610 | ) }); |
| 181611 | ret_tracking_len += 1; |
| 181612 | ret_gpr = ret_gpr[1..]; |
| 181613 | }, |
| 181614 | .sse, .float, .float_combine, .win_i128 => { |
| 181615 | ret_tracking[ret_tracking_len] = .init(.{ |
| 181616 | .register = registerAlias(ret_sse[0], @intCast(ret_ty.abiSize(zcu))), |
| 181617 | }); |
| 181618 | ret_tracking_len += 1; |
| 181619 | ret_sse = ret_sse[1..]; |
| 181620 | }, |
| 181621 | .sseup => assert(ret_tracking[ret_tracking_len - 1].short.register.isClass(.sse)), |
| 181622 | .x87 => { |
| 181623 | ret_tracking[ret_tracking_len] = .init(.{ .register = ret_x87[0] }); |
| 181624 | ret_tracking_len += 1; |
| 181625 | ret_x87 = ret_x87[1..]; |
| 181626 | }, |
| 181627 | .x87up => assert(ret_tracking[ret_tracking_len - 1].short.register.isClass(.x87)), |
| 181628 | .none => {}, |
| 181629 | .memory => { |
| 181630 | ret_tracking[ret_tracking_len] = .{ |
| 181631 | .short = .{ .indirect = .{ .reg = ret_gpr[0].to64() } }, |
| 181632 | .long = .{ .indirect = .{ .reg = param_gpr[param_gpr_index].to64() } }, |
| 181633 | }; |
| 181634 | ret_tracking_len += 1; |
| 181635 | ret_gpr = ret_gpr[1..]; |
| 181636 | param_gpr_index += 1; |
| 181637 | }, |
| 181638 | .integer_per_element => { |
| 181639 | const len: u32 = @intCast(ret_ty.vectorLen(zcu)); |
| 181640 | const alias_size: u32 = @intCast(@min(ret_ty.childType(zcu).abiSize(zcu), 8)); |
| 181641 | for (ret_tracking[ret_tracking_len..][0..len], ret_gpr[0..len]) |*tracking, gpr| |
| 181642 | tracking.* = .init(.{ .register = registerAlias(gpr, alias_size) }); |
| 181643 | ret_tracking_len += len; |
| 181644 | ret_gpr = ret_gpr[len..]; |
| 181645 | }, |
| 181646 | .sse_per_element => { |
| 181647 | const len: u32 = @intCast(ret_ty.vectorLen(zcu)); |
| 181648 | const alias_size: u32 = @intCast(@min(ret_ty.childType(zcu).abiSize(zcu), 8)); |
| 181649 | for (ret_tracking[ret_tracking_len..][0..len], ret_sse[0..len]) |*tracking, sse| |
| 181650 | tracking.* = .init(.{ .register = registerAlias(sse, alias_size) }); |
| 181651 | ret_tracking_len += len; |
| 181652 | ret_sse = ret_sse[len..]; |
| 181653 | }, |
| 181654 | .sse_sse_x87_per_qword, .sse_per_xword, .sse_per_yword, .sse_per_zword => { |
| 181655 | const reg_size: u32 = switch (class) { |
| 181658 | 181656 | else => unreachable, |
| 181659 | | .sse_sse_x87_per_qword => switch (byte_offset) { |
| 181660 | | 0 => .sse, // duck |
| 181661 | | 8 => .sse, // duck |
| 181662 | | else => .x87, // goose! |
| 181663 | | }, |
| 181664 | | .sse_per_xword, .sse_per_yword, .sse_per_zword => .sse, |
| 181665 | | })) { |
| 181666 | | .sse => { |
| 181667 | | ret_tracking[ret_tracking_len] = .init(.{ |
| 181668 | | .register = registerAlias(ret_sse[0], reg_size), |
| 181669 | | }); |
| 181670 | | ret_tracking_len += 1; |
| 181671 | | ret_sse = ret_sse[1..]; |
| 181672 | | }, |
| 181673 | | .x87 => { |
| 181674 | | ret_tracking[ret_tracking_len] = .init(.{ |
| 181675 | | .register = registerAlias(ret_x87[0], reg_size), |
| 181676 | | }); |
| 181677 | | ret_tracking_len += 1; |
| 181678 | | ret_x87 = ret_x87[1..]; |
| 181679 | | }, |
| 181657 | .sse_sse_x87_per_qword => 8, |
| 181658 | .sse_per_xword => 16, |
| 181659 | .sse_per_yword => 32, |
| 181660 | .sse_per_zword => 64, |
| 181661 | }; |
| 181662 | var byte_offset: u32 = 0; |
| 181663 | const unaligned_size = cg.unalignedSize(ret_ty); |
| 181664 | while (byte_offset < unaligned_size) : (byte_offset += reg_size) { |
| 181665 | switch (@as(enum { sse, x87 }, switch (class) { |
| 181666 | else => unreachable, |
| 181667 | .sse_sse_x87_per_qword => switch (byte_offset) { |
| 181668 | 0 => .sse, // duck |
| 181669 | 8 => .sse, // duck |
| 181670 | else => .x87, // goose! |
| 181671 | }, |
| 181672 | .sse_per_xword, .sse_per_yword, .sse_per_zword => .sse, |
| 181673 | })) { |
| 181674 | .sse => { |
| 181675 | ret_tracking[ret_tracking_len] = .init(.{ |
| 181676 | .register = registerAlias(ret_sse[0], reg_size), |
| 181677 | }); |
| 181678 | ret_tracking_len += 1; |
| 181679 | ret_sse = ret_sse[1..]; |
| 181680 | }, |
| 181681 | .x87 => { |
| 181682 | ret_tracking[ret_tracking_len] = .init(.{ |
| 181683 | .register = registerAlias(ret_x87[0], reg_size), |
| 181684 | }); |
| 181685 | ret_tracking_len += 1; |
| 181686 | ret_x87 = ret_x87[1..]; |
| 181687 | }, |
| 181688 | } |
| 181680 | 181689 | } |
| 181681 | | } |
| 181682 | | }, |
| 181683 | | }; |
| 181684 | | result.return_value = switch (ret_tracking_len) { |
| 181685 | | else => unreachable, |
| 181686 | | 1 => ret_tracking[0], |
| 181687 | | 2 => .init(.{ .register_pair = .{ |
| 181688 | | ret_tracking[0].short.register, |
| 181689 | | ret_tracking[1].short.register, |
| 181690 | | } }), |
| 181691 | | 3 => .init(.{ .register_triple = .{ |
| 181692 | | ret_tracking[0].short.register, |
| 181693 | | ret_tracking[1].short.register, |
| 181694 | | ret_tracking[2].short.register, |
| 181695 | | } }), |
| 181696 | | 4 => .init(.{ .register_quadruple = .{ |
| 181697 | | ret_tracking[0].short.register, |
| 181698 | | ret_tracking[1].short.register, |
| 181699 | | ret_tracking[2].short.register, |
| 181700 | | ret_tracking[3].short.register, |
| 181701 | | } }), |
| 181702 | | }; |
| 181703 | | } |
| 181690 | }, |
| 181691 | }; |
| 181692 | break :return_value switch (ret_tracking_len) { |
| 181693 | else => unreachable, |
| 181694 | 1 => ret_tracking[0], |
| 181695 | 2 => .init(.{ .register_pair = .{ |
| 181696 | ret_tracking[0].short.register, |
| 181697 | ret_tracking[1].short.register, |
| 181698 | } }), |
| 181699 | 3 => .init(.{ .register_triple = .{ |
| 181700 | ret_tracking[0].short.register, |
| 181701 | ret_tracking[1].short.register, |
| 181702 | ret_tracking[2].short.register, |
| 181703 | } }), |
| 181704 | 4 => .init(.{ .register_quadruple = .{ |
| 181705 | ret_tracking[0].short.register, |
| 181706 | ret_tracking[1].short.register, |
| 181707 | ret_tracking[2].short.register, |
| 181708 | ret_tracking[3].short.register, |
| 181709 | } }), |
| 181710 | }; |
| 181711 | }, |
| 181712 | .fully_comptime => unreachable, |
| 181713 | }; |
| 181704 | 181714 | |
| 181705 | | // Input params |
| 181706 | | params: for (param_types, result.args) |ty, *arg| { |
| 181707 | | assert(ty.hasRuntimeBits(zcu)); |
| 181715 | params: for (0.., param_types, result.args) |param_index, ty, *arg| { |
| 181708 | 181716 | result.air_arg_count += 1; |
| 181709 | 181717 | switch (cc) { |
| 181710 | 181718 | else => unreachable, |
| ... | ... | @@ -181726,7 +181734,7 @@ fn resolveCallingConventionValues( |
| 181726 | 181734 | else => unreachable, |
| 181727 | 181735 | .x86_64_sysv => { |
| 181728 | 181736 | classes_buf = abi.classifySystemV(ty, zcu, cg.target, .arg); |
| 181729 | | break :classes std.mem.sliceTo(&classes_buf, .none); |
| 181737 | break :classes &classes_buf; |
| 181730 | 181738 | }, |
| 181731 | 181739 | .x86_64_win => { |
| 181732 | 181740 | classes_buf[0] = abi.classifyWindows(ty, zcu, cg.target, .arg); |
| ... | ... | @@ -181749,12 +181757,28 @@ fn resolveCallingConventionValues( |
| 181749 | 181757 | var byte_offset: u32 = 0; |
| 181750 | 181758 | while (byte_offset < abi_size) : (byte_offset += reg_size) { |
| 181751 | 181759 | if (param_sse_index >= param_sse.len) break :classes; |
| 181752 | | |
| 181753 | | const param_sse_reg = registerAlias(param_sse[param_sse_index], reg_size); |
| 181754 | | param_sse_index += 1; |
| 181755 | | |
| 181756 | | arg_mcv[arg_mcv_len] = .{ .register = param_sse_reg }; |
| 181760 | arg_mcv[arg_mcv_len] = arg_mcv: { |
| 181761 | const param_sse_reg = |
| 181762 | registerAlias(param_sse[param_sse_index], reg_size); |
| 181763 | switch (cc) { |
| 181764 | else => unreachable, |
| 181765 | .x86_64_sysv => {}, |
| 181766 | .x86_64_win => if (param_index >= fn_info.param_types.len) { |
| 181767 | const param_gpr_reg = |
| 181768 | registerAlias(param_gpr[param_gpr_index], reg_size); |
| 181769 | param_gpr_index += 1; |
| 181770 | break :arg_mcv .{ .register_tee = .{ |
| 181771 | param_sse_reg, |
| 181772 | param_gpr_reg, |
| 181773 | .none, |
| 181774 | .none, |
| 181775 | } }; |
| 181776 | }, |
| 181777 | } |
| 181778 | break :arg_mcv .{ .register = param_sse_reg }; |
| 181779 | }; |
| 181757 | 181780 | arg_mcv_len += 1; |
| 181781 | param_sse_index += 1; |
| 181758 | 181782 | } |
| 181759 | 181783 | }, |
| 181760 | 181784 | .sseup => assert(arg_mcv[arg_mcv_len - 1].register.isClass(.sse)), |
| ... | ... | @@ -181764,13 +181788,16 @@ fn resolveCallingConventionValues( |
| 181764 | 181788 | .x87, .x87up, .memory => break, |
| 181765 | 181789 | }, |
| 181766 | 181790 | .x86_64_win => if (param_gpr_index < param_gpr.len) { |
| 181767 | | arg_mcv[arg_mcv_len] = .{ .indirect = .{ .reg = param_gpr[param_gpr_index].to64() } }; |
| 181791 | arg_mcv[arg_mcv_len] = .{ .indirect = .{ |
| 181792 | .reg = param_gpr[param_gpr_index].to64(), |
| 181793 | } }; |
| 181768 | 181794 | arg_mcv_len += 1; |
| 181769 | 181795 | param_gpr_index += 1; |
| 181770 | 181796 | } else { |
| 181771 | 181797 | assert(arg_mcv_len == 0); |
| 181772 | 181798 | const param_align = Type.usize.abiAlignment(zcu); |
| 181773 | | result.stack_byte_count = @intCast(param_align.forward(result.stack_byte_count)); |
| 181799 | result.stack_byte_count = |
| 181800 | @intCast(param_align.forward(result.stack_byte_count)); |
| 181774 | 181801 | result.stack_align = result.stack_align.max(param_align); |
| 181775 | 181802 | arg.* = .{ .indirect_load_frame = .{ |
| 181776 | 181803 | .index = stack_frame_base, |
| ... | ... | @@ -181781,7 +181808,7 @@ fn resolveCallingConventionValues( |
| 181781 | 181808 | }, |
| 181782 | 181809 | else => unreachable, |
| 181783 | 181810 | }, |
| 181784 | | .none => unreachable, |
| 181811 | .none => {}, |
| 181785 | 181812 | .integer_per_element, |
| 181786 | 181813 | .sse_per_element, |
| 181787 | 181814 | .sse_per_xword, |
| ... | ... | @@ -181850,6 +181877,7 @@ fn resolveCallingConventionValues( |
| 181850 | 181877 | } else { |
| 181851 | 181878 | arg.* = switch (arg_mcv_len) { |
| 181852 | 181879 | else => unreachable, |
| 181880 | 0 => .none, |
| 181853 | 181881 | 1 => arg_mcv[0], |
| 181854 | 181882 | 2 => .{ .register_pair = .{ |
| 181855 | 181883 | arg_mcv[0].register, |
| ... | ... | @@ -181899,82 +181927,82 @@ fn resolveCallingConventionValues( |
| 181899 | 181927 | param_gpr = param_gpr[0 .. param_gpr.len - 1]; |
| 181900 | 181928 | } |
| 181901 | 181929 | |
| 181902 | | // Return values |
| 181903 | | result.return_value = if (ret_ty.isNoReturn(zcu)) |
| 181904 | | .init(.unreach) |
| 181905 | | else if (!ret_ty.hasRuntimeBits(zcu)) |
| 181906 | | .init(.none) |
| 181907 | | else return_value: { |
| 181908 | | const ret_gpr = abi.getCAbiIntReturnRegs(cc); |
| 181909 | | const ret_size: u31 = @intCast(ret_ty.abiSize(zcu)); |
| 181910 | | if (abi.zigcc.return_in_regs) switch (cg.regClassForType(ret_ty)) { |
| 181911 | | .general_purpose, .gphi => if (ret_size <= @as(u4, switch (cg.target.cpu.arch) { |
| 181912 | | else => unreachable, |
| 181913 | | .x86 => 4, |
| 181914 | | .x86_64 => 8, |
| 181915 | | })) |
| 181916 | | break :return_value .init(.{ .register = registerAlias(ret_gpr[0], ret_size) }) |
| 181917 | | else if (ret_gpr.len >= 2 and ret_ty.isSliceAtRuntime(zcu)) |
| 181918 | | break :return_value .init(.{ .register_pair = ret_gpr[0..2].* }), |
| 181919 | | .segment, .mmx, .ip, .cr, .dr => unreachable, |
| 181920 | | .x87 => if (ret_size <= 16) break :return_value .init(.{ .register = .st0 }), |
| 181921 | | .sse => if (ret_size <= cg.vectorSize(.float)) break :return_value .init(.{ |
| 181922 | | .register = registerAlias(abi.getCAbiSseReturnRegs(cc)[0], @max(ret_size, 16)), |
| 181923 | | }), |
| 181924 | | }; |
| 181925 | | const ret_indirect_reg = param_gpr[0]; |
| 181926 | | param_gpr = param_gpr[1..]; |
| 181927 | | break :return_value .{ |
| 181928 | | .short = .{ .indirect = .{ .reg = ret_gpr[0] } }, |
| 181929 | | .long = .{ .indirect = .{ .reg = ret_indirect_reg } }, |
| 181930 | | }; |
| 181930 | result.return_value = switch (ret_ty.classify(zcu)) { |
| 181931 | .no_possible_value => .init(.unreach), |
| 181932 | .one_possible_value => .init(.none), |
| 181933 | .runtime, .partially_comptime => return_value: { |
| 181934 | const ret_gpr = abi.getCAbiIntReturnRegs(cc); |
| 181935 | const ret_size: u31 = @intCast(ret_ty.abiSize(zcu)); |
| 181936 | if (abi.zigcc.return_in_regs) switch (cg.regClassForType(ret_ty)) { |
| 181937 | .general_purpose, .gphi => if (ret_size <= @as(u4, switch (cg.target.cpu.arch) { |
| 181938 | else => unreachable, |
| 181939 | .x86 => 4, |
| 181940 | .x86_64 => 8, |
| 181941 | })) |
| 181942 | break :return_value .init(.{ .register = registerAlias(ret_gpr[0], ret_size) }) |
| 181943 | else if (ret_gpr.len >= 2 and ret_ty.isSliceAtRuntime(zcu)) |
| 181944 | break :return_value .init(.{ .register_pair = ret_gpr[0..2].* }), |
| 181945 | .segment, .mmx, .ip, .cr, .dr => unreachable, |
| 181946 | .x87 => if (ret_size <= 16) break :return_value .init(.{ .register = .st0 }), |
| 181947 | .sse => if (ret_size <= cg.vectorSize(.float)) break :return_value .init(.{ |
| 181948 | .register = registerAlias(abi.getCAbiSseReturnRegs(cc)[0], @max(ret_size, 16)), |
| 181949 | }), |
| 181950 | }; |
| 181951 | const ret_indirect_reg = param_gpr[0]; |
| 181952 | param_gpr = param_gpr[1..]; |
| 181953 | break :return_value .{ |
| 181954 | .short = .{ .indirect = .{ .reg = ret_gpr[0] } }, |
| 181955 | .long = .{ .indirect = .{ .reg = ret_indirect_reg } }, |
| 181956 | }; |
| 181957 | }, |
| 181958 | .fully_comptime => unreachable, |
| 181931 | 181959 | }; |
| 181932 | 181960 | |
| 181933 | | // Input params |
| 181934 | | for (param_types, result.args) |param_ty, *arg| { |
| 181935 | | if (!param_ty.hasRuntimeBits(zcu)) { |
| 181936 | | arg.* = .none; |
| 181937 | | continue; |
| 181938 | | } |
| 181939 | | result.air_arg_count += 1; |
| 181940 | | const param_size: u31 = @intCast(param_ty.abiSize(zcu)); |
| 181941 | | if (abi.zigcc.params_in_regs) switch (cg.regClassForType(param_ty)) { |
| 181942 | | .general_purpose, .gphi => if (param_gpr.len >= 1 and param_size <= @as(u4, switch (cg.target.cpu.arch) { |
| 181943 | | else => unreachable, |
| 181944 | | .x86 => 4, |
| 181945 | | .x86_64 => 8, |
| 181946 | | })) { |
| 181947 | | arg.* = .{ .register = registerAlias(param_gpr[0], param_size) }; |
| 181948 | | param_gpr = param_gpr[1..]; |
| 181949 | | continue; |
| 181950 | | } else if (param_gpr.len >= 2 and param_ty.isSliceAtRuntime(zcu)) { |
| 181951 | | arg.* = .{ .register_pair = param_gpr[0..2].* }; |
| 181952 | | param_gpr = param_gpr[2..]; |
| 181953 | | continue; |
| 181954 | | }, |
| 181955 | | .segment, .mmx, .ip, .cr, .dr => unreachable, |
| 181956 | | .x87 => if (param_x87.len >= 1 and param_size <= 16) { |
| 181957 | | arg.* = .{ .register = param_x87[0] }; |
| 181958 | | param_x87 = param_x87[1..]; |
| 181959 | | continue; |
| 181960 | | }, |
| 181961 | | .sse => if (param_sse.len >= 1 and param_size <= cg.vectorSize(.float)) { |
| 181962 | | arg.* = .{ |
| 181963 | | .register = registerAlias(param_sse[0], @max(param_size, 16)), |
| 181964 | | }; |
| 181965 | | param_sse = param_sse[1..]; |
| 181966 | | continue; |
| 181967 | | }, |
| 181968 | | }; |
| 181969 | | const param_align = param_ty.abiAlignment(zcu); |
| 181970 | | result.stack_byte_count = @intCast(param_align.forward(result.stack_byte_count)); |
| 181971 | | result.stack_align = result.stack_align.max(param_align); |
| 181972 | | arg.* = .{ .load_frame = .{ |
| 181973 | | .index = stack_frame_base, |
| 181974 | | .off = result.stack_byte_count, |
| 181975 | | } }; |
| 181976 | | result.stack_byte_count += param_size; |
| 181977 | | } |
| 181961 | for (param_types, result.args) |param_ty, *arg| switch (param_ty.classify(zcu)) { |
| 181962 | .no_possible_value => arg.* = .unreach, |
| 181963 | .one_possible_value => arg.* = .none, |
| 181964 | .runtime, .partially_comptime => { |
| 181965 | result.air_arg_count += 1; |
| 181966 | const param_size: u31 = @intCast(param_ty.abiSize(zcu)); |
| 181967 | if (abi.zigcc.params_in_regs) switch (cg.regClassForType(param_ty)) { |
| 181968 | .general_purpose, .gphi => if (param_gpr.len >= 1 and param_size <= @as(u4, switch (cg.target.cpu.arch) { |
| 181969 | else => unreachable, |
| 181970 | .x86 => 4, |
| 181971 | .x86_64 => 8, |
| 181972 | })) { |
| 181973 | arg.* = .{ .register = registerAlias(param_gpr[0], param_size) }; |
| 181974 | param_gpr = param_gpr[1..]; |
| 181975 | continue; |
| 181976 | } else if (param_gpr.len >= 2 and param_ty.isSliceAtRuntime(zcu)) { |
| 181977 | arg.* = .{ .register_pair = param_gpr[0..2].* }; |
| 181978 | param_gpr = param_gpr[2..]; |
| 181979 | continue; |
| 181980 | }, |
| 181981 | .segment, .mmx, .ip, .cr, .dr => unreachable, |
| 181982 | .x87 => if (param_x87.len >= 1 and param_size <= 16) { |
| 181983 | arg.* = .{ .register = param_x87[0] }; |
| 181984 | param_x87 = param_x87[1..]; |
| 181985 | continue; |
| 181986 | }, |
| 181987 | .sse => if (param_sse.len >= 1 and param_size <= cg.vectorSize(.float)) { |
| 181988 | arg.* = .{ |
| 181989 | .register = registerAlias(param_sse[0], @max(param_size, 16)), |
| 181990 | }; |
| 181991 | param_sse = param_sse[1..]; |
| 181992 | continue; |
| 181993 | }, |
| 181994 | }; |
| 181995 | const param_align = param_ty.abiAlignment(zcu); |
| 181996 | result.stack_byte_count = @intCast(param_align.forward(result.stack_byte_count)); |
| 181997 | result.stack_align = result.stack_align.max(param_align); |
| 181998 | arg.* = .{ .load_frame = .{ |
| 181999 | .index = stack_frame_base, |
| 182000 | .off = result.stack_byte_count, |
| 182001 | } }; |
| 182002 | result.stack_byte_count += param_size; |
| 182003 | }, |
| 182004 | .fully_comptime => unreachable, |
| 182005 | }; |
| 181978 | 182006 | }, |
| 181979 | 182007 | else => return cg.fail("TODO implement function parameters and return values for {} on x86_64", .{cc}), |
| 181980 | 182008 | } |
| ... | ... | @@ -182241,13 +182269,11 @@ fn typeOfIndex(self: *CodeGen, inst: Air.Inst.Index) Type { |
| 182241 | 182269 | return Temp.typeOf(.{ .index = inst }, self); |
| 182242 | 182270 | } |
| 182243 | 182271 | |
| 182244 | | fn promoteInt(self: *CodeGen, ty: Type) Type { |
| 182245 | | const pt = self.pt; |
| 182272 | fn promoteInt(cg: *CodeGen, ty: Type) Type { |
| 182273 | const pt = cg.pt; |
| 182246 | 182274 | const zcu = pt.zcu; |
| 182247 | | const int_info: InternPool.Key.IntType = switch (ty.toIntern()) { |
| 182248 | | .bool_type => .{ .signedness = .unsigned, .bits = 1 }, |
| 182249 | | else => if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else return ty, |
| 182250 | | }; |
| 182275 | const int_info = cg.intInfo(ty) orelse return ty; |
| 182276 | if (int_info.bits == 0) return .void; |
| 182251 | 182277 | for ([_]Type{ |
| 182252 | 182278 | .c_int, .c_uint, |
| 182253 | 182279 | .c_long, .c_ulong, |
| ... | ... | @@ -182424,6 +182450,7 @@ const Temp = struct { |
| 182424 | 182450 | .lea_lazy_sym, |
| 182425 | 182451 | .lea_extern_func, |
| 182426 | 182452 | .load_extern_func, |
| 182453 | .register_tee, |
| 182427 | 182454 | .elementwise_gpr, |
| 182428 | 182455 | .elementwise_sse, |
| 182429 | 182456 | .xwordwise_sse, |
| ... | ... | @@ -182891,6 +182918,7 @@ const Temp = struct { |
| 182891 | 182918 | .register_overflow, |
| 182892 | 182919 | .register_mask, |
| 182893 | 182920 | .indirect_load_frame, |
| 182921 | .register_tee, |
| 182894 | 182922 | .elementwise_gpr, |
| 182895 | 182923 | .elementwise_sse, |
| 182896 | 182924 | .xwordwise_sse, |
| ... | ... | @@ -188002,6 +188030,7 @@ const Temp = struct { |
| 188002 | 188030 | .unreach, |
| 188003 | 188031 | .dead, |
| 188004 | 188032 | .indirect_load_frame, |
| 188033 | .register_tee, |
| 188005 | 188034 | .elementwise_gpr, |
| 188006 | 188035 | .elementwise_sse, |
| 188007 | 188036 | .xwordwise_sse, |
| ... | ... | @@ -190673,6 +190702,7 @@ const Select = struct { |
| 190673 | 190702 | .lea_extern_func => |extern_func| .{ .imm = .{ .extern_func = extern_func } }, |
| 190674 | 190703 | else => |mcv| .{ .mem = try mcv.mem(s.cg, .{ .size = op.flags.base.size }) }, |
| 190675 | 190704 | .lea_frame, |
| 190705 | .register_tee, |
| 190676 | 190706 | .elementwise_gpr, |
| 190677 | 190707 | .elementwise_sse, |
| 190678 | 190708 | .xwordwise_sse, |
Jacob Young <jacobly@ziglang.org> 2026-06-19 19:31:25-04:00
src