authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-03-08 14:27:57-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2025-03-08 14:27:57-05:00
log61c588d726f85551ad36c32fd2917087d3a4763b
tree45043ae885c12aaa053b98f221e9c4910699f1da
parent801a95035c4562bea1c8b80ae5fc8e05b9b22a2d
parent5d115632d4e458e7e9154f14856fb29935315cb2
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #22998 from jacobly0/x86_64-rewrite

x86_64: rewrite aggregate init

4 files changed, 349 insertions(+), 238 deletions(-)

src/arch/x86_64/CodeGen.zig+341-230
......@@ -2437,7 +2437,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
24372437
24382438 try cg.airArg(inst);
24392439
2440 try cg.resetTemps();
2440 try cg.resetTemps(@enumFromInt(0));
24412441 cg.checkInvariantsAfterAirInst();
24422442 },
24432443 else => break,
......@@ -2477,7 +2477,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
24772477 .shuffle => try cg.airShuffle(inst),
24782478 .reduce => try cg.airReduce(inst),
24792479 .reduce_optimized => try cg.airReduce(inst),
2480 .aggregate_init => try cg.airAggregateInit(inst),
24812480 // zig fmt: on
24822481
24832482 .arg => if (cg.debug_output != .none) {
......@@ -80843,6 +80842,74 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8084380842 for (ops[1..]) |op| try op.die(cg);
8084480843 try res[0].finish(inst, &.{ty_op.operand}, ops[0..1], cg);
8084580844 },
80845 .aggregate_init => |air_tag| if (use_old) try cg.airAggregateInit(inst) else fallback: {
80846 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
80847 const agg_ty = ty_pl.ty.toType();
80848 if ((agg_ty.isVector(zcu) and agg_ty.childType(zcu).toIntern() == .bool_type) or
80849 (agg_ty.zigTypeTag(zcu) == .@"struct" and agg_ty.containerLayout(zcu) == .@"packed")) break :fallback try cg.airAggregateInit(inst);
80850 var res = try cg.tempAllocMem(agg_ty);
80851 const reset_index = cg.next_temp_index;
80852 var bt = cg.liveness.iterateBigTomb(inst);
80853 switch (ip.indexToKey(agg_ty.toIntern())) {
80854 inline .array_type, .vector_type => |sequence_type| {
80855 const elems: []const Air.Inst.Ref = @ptrCast(cg.air.extra[ty_pl.payload..][0..@intCast(sequence_type.len)]);
80856 const elem_size = Type.fromInterned(sequence_type.child).abiSize(zcu);
80857 var elem_disp: u31 = 0;
80858 for (elems) |elem_ref| {
80859 var elem = try cg.tempFromOperand(elem_ref, bt.feed());
80860 try res.write(&elem, .{ .disp = elem_disp }, cg);
80861 try elem.die(cg);
80862 try cg.resetTemps(reset_index);
80863 elem_disp += @intCast(elem_size);
80864 }
80865 if (@hasField(@TypeOf(sequence_type), "sentinel") and sequence_type.sentinel != .none) {
80866 var sentinel = try cg.tempFromValue(.fromInterned(sequence_type.sentinel));
80867 try res.write(&sentinel, .{ .disp = elem_disp }, cg);
80868 try sentinel.die(cg);
80869 }
80870 },
80871 .struct_type => {
80872 const loaded_struct = ip.loadStructType(agg_ty.toIntern());
80873 const elems: []const Air.Inst.Ref = @ptrCast(cg.air.extra[ty_pl.payload..][0..loaded_struct.field_types.len]);
80874 switch (loaded_struct.layout) {
80875 .auto, .@"extern" => {
80876 for (elems, 0..) |elem_ref, field_index| {
80877 const elem_dies = bt.feed();
80878 if (loaded_struct.fieldIsComptime(ip, field_index)) continue;
80879 var elem = try cg.tempFromOperand(elem_ref, elem_dies);
80880 try res.write(&elem, .{ .disp = @intCast(loaded_struct.offsets.get(ip)[field_index]) }, cg);
80881 try elem.die(cg);
80882 try cg.resetTemps(reset_index);
80883 }
80884 },
80885 .@"packed" => return cg.fail("failed to select {s} {}", .{
80886 @tagName(air_tag),
80887 agg_ty.fmt(pt),
80888 }),
80889 }
80890 },
80891 .tuple_type => |tuple_type| {
80892 const elems: []const Air.Inst.Ref = @ptrCast(cg.air.extra[ty_pl.payload..][0..tuple_type.types.len]);
80893 var elem_disp: u31 = 0;
80894 for (elems, 0..) |elem_ref, field_index| {
80895 const elem_dies = bt.feed();
80896 if (tuple_type.values.get(ip)[field_index] != .none) continue;
80897 const field_type = Type.fromInterned(tuple_type.types.get(ip)[field_index]);
80898 elem_disp = @intCast(field_type.abiAlignment(zcu).forward(elem_disp));
80899 var elem = try cg.tempFromOperand(elem_ref, elem_dies);
80900 try res.write(&elem, .{ .disp = elem_disp }, cg);
80901 try elem.die(cg);
80902 try cg.resetTemps(reset_index);
80903 elem_disp += @intCast(field_type.abiSize(zcu));
80904 }
80905 },
80906 else => return cg.fail("failed to select {s} {}", .{
80907 @tagName(air_tag),
80908 agg_ty.fmt(pt),
80909 }),
80910 }
80911 try res.finish(inst, &.{}, &.{}, cg);
80912 },
8084680913 .union_init => if (use_old) try cg.airUnionInit(inst) else {
8084780914 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
8084880915 const extra = cg.air.extraData(Air.UnionInit, ty_pl.payload).data;
......@@ -82199,14 +82266,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8219982266 .c_va_start => try cg.airVaStart(inst),
8220082267 .work_item_id, .work_group_size, .work_group_id => unreachable,
8220182268 }
82202 try cg.resetTemps();
82269 try cg.resetTemps(@enumFromInt(0));
8220382270 cg.checkInvariantsAfterAirInst();
8220482271 }
8220582272 verbose_tracking_log.debug("{}", .{cg.fmtTracking()});
8220682273}
8220782274
82208fn genLazy(self: *CodeGen, lazy_sym: link.File.LazySymbol) InnerError!void {
82209 const pt = self.pt;
82275fn genLazy(cg: *CodeGen, lazy_sym: link.File.LazySymbol) InnerError!void {
82276 const pt = cg.pt;
8221082277 const zcu = pt.zcu;
8221182278 const ip = &zcu.intern_pool;
8221282279 switch (ip.indexToKey(lazy_sym.ty)) {
......@@ -82215,97 +82282,98 @@ fn genLazy(self: *CodeGen, lazy_sym: link.File.LazySymbol) InnerError!void {
8221582282 wip_mir_log.debug("{}.@tagName:", .{enum_ty.fmt(pt)});
8221682283
8221782284 const param_regs = abi.getCAbiIntParamRegs(.auto);
82218 const param_locks = self.register_manager.lockRegsAssumeUnused(2, param_regs[0..2].*);
82219 defer for (param_locks) |lock| self.register_manager.unlockReg(lock);
82285 const param_locks = cg.register_manager.lockRegsAssumeUnused(2, param_regs[0..2].*);
82286 defer for (param_locks) |lock| cg.register_manager.unlockReg(lock);
8222082287
8222182288 const ret_mcv: MCValue = .{ .register_pair = param_regs[0..2].* };
82222 const enum_mcv: MCValue = .{ .register = param_regs[0] };
82289 var enum_temp = try cg.tempInit(enum_ty, .{ .register = param_regs[0] });
8222382290
82224 const data_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
82225 const data_lock = self.register_manager.lockRegAssumeUnused(data_reg);
82226 defer self.register_manager.unlockReg(data_lock);
82227 try self.genLazySymbolRef(.lea, data_reg, .{ .kind = .const_data, .ty = lazy_sym.ty });
82291 const data_reg = try cg.register_manager.allocReg(null, abi.RegisterClass.gp);
82292 const data_lock = cg.register_manager.lockRegAssumeUnused(data_reg);
82293 defer cg.register_manager.unlockReg(data_lock);
82294 try cg.genLazySymbolRef(.lea, data_reg, .{ .kind = .const_data, .ty = lazy_sym.ty });
8222882295
8222982296 var data_off: i32 = 0;
82297 const reset_index = cg.next_temp_index;
8223082298 const tag_names = ip.loadEnumType(lazy_sym.ty).names;
8223182299 for (0..tag_names.len) |tag_index| {
82232 var enum_temp = try self.tempInit(enum_ty, enum_mcv);
82233
8223482300 const tag_name_len = tag_names.get(ip)[tag_index].length(ip);
82235 var tag_temp = try self.tempFromValue(try pt.enumValueFieldIndex(enum_ty, @intCast(tag_index)));
82236 const cc_temp = enum_temp.cmpInts(.neq, &tag_temp, self) catch |err| switch (err) {
82301 var tag_temp = try cg.tempFromValue(try pt.enumValueFieldIndex(enum_ty, @intCast(tag_index)));
82302 const cc_temp = enum_temp.cmpInts(.neq, &tag_temp, cg) catch |err| switch (err) {
8223782303 error.SelectFailed => unreachable,
8223882304 else => |e| return e,
8223982305 };
82240 try enum_temp.die(self);
82241 try tag_temp.die(self);
82242 const skip_reloc = try self.asmJccReloc(cc_temp.tracking(self).short.eflags, undefined);
82243 try cc_temp.die(self);
82244 try self.resetTemps();
82306 try tag_temp.die(cg);
82307 const skip_reloc = try cg.asmJccReloc(cc_temp.tracking(cg).short.eflags, undefined);
82308 try cc_temp.die(cg);
82309 try cg.resetTemps(reset_index);
8224582310
82246 try self.genSetReg(
82311 try cg.genSetReg(
8224782312 ret_mcv.register_pair[0],
8224882313 .usize,
8224982314 .{ .register_offset = .{ .reg = data_reg, .off = data_off } },
8225082315 .{},
8225182316 );
82252 try self.genSetReg(ret_mcv.register_pair[1], .usize, .{ .immediate = tag_name_len }, .{});
82253 try self.asmOpOnly(.{ ._, .ret });
82317 try cg.genSetReg(ret_mcv.register_pair[1], .usize, .{ .immediate = tag_name_len }, .{});
82318 try cg.asmOpOnly(.{ ._, .ret });
8225482319
82255 self.performReloc(skip_reloc);
82320 cg.performReloc(skip_reloc);
8225682321
8225782322 data_off += @intCast(tag_name_len + 1);
8225882323 }
82324 try enum_temp.die(cg);
8225982325
82260 try self.genSetReg(ret_mcv.register_pair[0], .usize, .{ .immediate = 0 }, .{});
82261 try self.asmOpOnly(.{ ._, .ret });
82326 try cg.genSetReg(ret_mcv.register_pair[0], .usize, .{ .immediate = 0 }, .{});
82327 try cg.asmOpOnly(.{ ._, .ret });
8226282328 },
8226382329 .error_set_type => |error_set_type| {
8226482330 const err_ty: Type = .fromInterned(lazy_sym.ty);
8226582331 wip_mir_log.debug("{}.@errorCast:", .{err_ty.fmt(pt)});
8226682332
8226782333 const param_regs = abi.getCAbiIntParamRegs(.auto);
82268 const param_locks = self.register_manager.lockRegsAssumeUnused(2, param_regs[0..2].*);
82269 defer for (param_locks) |lock| self.register_manager.unlockReg(lock);
82334 const param_locks = cg.register_manager.lockRegsAssumeUnused(2, param_regs[0..2].*);
82335 defer for (param_locks) |lock| cg.register_manager.unlockReg(lock);
8227082336
8227182337 const ret_mcv: MCValue = .{ .register = param_regs[0] };
8227282338 const err_mcv: MCValue = .{ .register = param_regs[0] };
82339 var err_temp = try cg.tempInit(err_ty, err_mcv);
8227382340
8227482341 const ExpectedContents = [32]Mir.Inst.Index;
8227582342 var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
82276 std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
82343 std.heap.stackFallback(@sizeOf(ExpectedContents), cg.gpa);
8227782344 const allocator = stack.get();
8227882345
8227982346 const relocs = try allocator.alloc(Mir.Inst.Index, error_set_type.names.len);
8228082347 defer allocator.free(relocs);
8228182348
82349 const reset_index = cg.next_temp_index;
8228282350 for (0.., relocs) |tag_index, *reloc| {
82283 var err_temp = try self.tempInit(err_ty, err_mcv);
82284
82285 var tag_temp = try self.tempInit(.anyerror, .{
82351 var tag_temp = try cg.tempInit(.anyerror, .{
8228682352 .immediate = ip.getErrorValueIfExists(error_set_type.names.get(ip)[tag_index]).?,
8228782353 });
82288 const cc_temp = err_temp.cmpInts(.eq, &tag_temp, self) catch |err| switch (err) {
82354 const cc_temp = err_temp.cmpInts(.eq, &tag_temp, cg) catch |err| switch (err) {
8228982355 error.SelectFailed => unreachable,
8229082356 else => |e| return e,
8229182357 };
82292 try err_temp.die(self);
82293 try tag_temp.die(self);
82294 reloc.* = try self.asmJccReloc(cc_temp.tracking(self).short.eflags, undefined);
82295 try cc_temp.die(self);
82296 try self.resetTemps();
82358 try tag_temp.die(cg);
82359 reloc.* = try cg.asmJccReloc(cc_temp.tracking(cg).short.eflags, undefined);
82360 try cc_temp.die(cg);
82361 try cg.resetTemps(reset_index);
8229782362 }
82363 try err_temp.die(cg);
8229882364
82299 try self.genCopy(.usize, ret_mcv, .{ .immediate = 0 }, .{});
82300 for (relocs) |reloc| self.performReloc(reloc);
82365 try cg.genCopy(.usize, ret_mcv, .{ .immediate = 0 }, .{});
82366 for (relocs) |reloc| cg.performReloc(reloc);
8230182367 assert(ret_mcv.register == err_mcv.register);
82302 try self.asmOpOnly(.{ ._, .ret });
82368 try cg.asmOpOnly(.{ ._, .ret });
8230382369 },
82304 else => return self.fail(
82370 else => return cg.fail(
8230582371 "TODO implement {s} for {}",
8230682372 .{ @tagName(lazy_sym.kind), Type.fromInterned(lazy_sym.ty).fmt(pt) },
8230782373 ),
8230882374 }
82375 try cg.resetTemps(@enumFromInt(0));
82376 cg.checkInvariantsAfterAirInst();
8230982377}
8231082378
8231182379fn getValue(self: *CodeGen, value: MCValue, inst: ?Air.Inst.Index) !void {
......@@ -93621,17 +93689,17 @@ fn lowerBlock(self: *CodeGen, inst: Air.Inst.Index, body: []const Air.Inst.Index
9362193689}
9362293690
9362393691fn lowerSwitchBr(
93624 self: *CodeGen,
93692 cg: *CodeGen,
9362593693 inst: Air.Inst.Index,
9362693694 switch_br: Air.UnwrappedSwitch,
9362793695 condition: MCValue,
9362893696 condition_dies: bool,
9362993697 is_loop: bool,
9363093698) !void {
93631 const zcu = self.pt.zcu;
93632 const condition_ty = self.typeOf(switch_br.operand);
93633 const condition_int_info = self.intInfo(condition_ty).?;
93634 const condition_int_ty = try self.pt.intType(condition_int_info.signedness, condition_int_info.bits);
93699 const zcu = cg.pt.zcu;
93700 const condition_ty = cg.typeOf(switch_br.operand);
93701 const condition_int_info = cg.intInfo(condition_ty).?;
93702 const condition_int_ty = try cg.pt.intType(condition_int_info.signedness, condition_int_info.bits);
9363593703
9363693704 const ExpectedContents = extern struct {
9363793705 liveness_deaths: [1 << 8 | 1]Air.Inst.Index,
......@@ -93639,15 +93707,15 @@ fn lowerSwitchBr(
9363993707 relocs: [1 << 6]Mir.Inst.Index,
9364093708 };
9364193709 var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
93642 std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
93710 std.heap.stackFallback(@sizeOf(ExpectedContents), cg.gpa);
9364393711 const allocator = stack.get();
9364493712
93645 const state = try self.saveState();
93713 const state = try cg.saveState();
9364693714
93647 const liveness = try self.liveness.getSwitchBr(allocator, inst, switch_br.cases_len + 1);
93715 const liveness = try cg.liveness.getSwitchBr(allocator, inst, switch_br.cases_len + 1);
9364893716 defer allocator.free(liveness.deaths);
9364993717
93650 if (!self.mod.pic and self.target.ofmt == .elf) table: {
93718 if (!cg.mod.pic and cg.target.ofmt == .elf) table: {
9365193719 var prong_items: u32 = 0;
9365293720 var min: ?Value = null;
9365393721 var max: ?Value = null;
......@@ -93690,41 +93758,41 @@ fn lowerSwitchBr(
9369093758 if (prong_items < table_len >> 2) break :table; // no more than 75% waste
9369193759
9369293760 const condition_index = if (condition_dies and condition.isModifiable()) condition else condition_index: {
93693 const condition_index = try self.allocTempRegOrMem(condition_ty, true);
93694 try self.genCopy(condition_ty, condition_index, condition, .{});
93761 const condition_index = try cg.allocTempRegOrMem(condition_ty, true);
93762 try cg.genCopy(condition_ty, condition_index, condition, .{});
9369593763 break :condition_index condition_index;
9369693764 };
93697 try self.spillEflagsIfOccupied();
93698 if (min.?.orderAgainstZero(zcu).compare(.neq)) try self.genBinOpMir(
93765 try cg.spillEflagsIfOccupied();
93766 if (min.?.orderAgainstZero(zcu).compare(.neq)) try cg.genBinOpMir(
9369993767 .{ ._, .sub },
9370093768 condition_ty,
9370193769 condition_index,
9370293770 .{ .air_ref = Air.internedToRef(min.?.toIntern()) },
9370393771 );
9370493772 const else_reloc = if (switch_br.else_body_len > 0) else_reloc: {
93705 var cond_temp = try self.tempInit(condition_ty, condition_index);
93706 var table_max_temp = try self.tempFromValue(try self.pt.intValue(condition_int_ty, table_len - 1));
93707 const cc_temp = cond_temp.cmpInts(.gt, &table_max_temp, self) catch |err| switch (err) {
93773 var cond_temp = try cg.tempInit(condition_ty, condition_index);
93774 var table_max_temp = try cg.tempFromValue(try cg.pt.intValue(condition_int_ty, table_len - 1));
93775 const cc_temp = cond_temp.cmpInts(.gt, &table_max_temp, cg) catch |err| switch (err) {
9370893776 error.SelectFailed => unreachable,
9370993777 else => |e| return e,
9371093778 };
93711 try cond_temp.die(self);
93712 try table_max_temp.die(self);
93713 const else_reloc = try self.asmJccReloc(cc_temp.tracking(self).short.eflags, undefined);
93714 try cc_temp.die(self);
93779 try cond_temp.die(cg);
93780 try table_max_temp.die(cg);
93781 const else_reloc = try cg.asmJccReloc(cc_temp.tracking(cg).short.eflags, undefined);
93782 try cc_temp.die(cg);
9371593783 break :else_reloc else_reloc;
9371693784 } else undefined;
93717 const table_start: u31 = @intCast(self.mir_table.items.len);
93785 const table_start: u31 = @intCast(cg.mir_table.items.len);
9371893786 {
9371993787 const condition_index_reg = if (condition_index.isRegister())
9372093788 condition_index.getReg().?
9372193789 else
93722 try self.copyToTmpRegister(.usize, condition_index);
93723 const condition_index_lock = self.register_manager.lockReg(condition_index_reg);
93724 defer if (condition_index_lock) |lock| self.register_manager.unlockReg(lock);
93725 try self.truncateRegister(condition_ty, condition_index_reg);
93726 const ptr_size = @divExact(self.target.ptrBitWidth(), 8);
93727 try self.asmMemory(.{ ._mp, .j }, .{
93790 try cg.copyToTmpRegister(.usize, condition_index);
93791 const condition_index_lock = cg.register_manager.lockReg(condition_index_reg);
93792 defer if (condition_index_lock) |lock| cg.register_manager.unlockReg(lock);
93793 try cg.truncateRegister(condition_ty, condition_index_reg);
93794 const ptr_size = @divExact(cg.target.ptrBitWidth(), 8);
93795 try cg.asmMemory(.{ ._mp, .j }, .{
9372893796 .base = .table,
9372993797 .mod = .{ .rm = .{
9373093798 .size = .ptr,
......@@ -93735,32 +93803,32 @@ fn lowerSwitchBr(
9373593803 });
9373693804 }
9373793805 const else_reloc_marker: u32 = 0;
93738 assert(self.mir_instructions.len > else_reloc_marker);
93739 try self.mir_table.appendNTimes(self.gpa, else_reloc_marker, table_len);
93740 if (is_loop) try self.loop_switches.putNoClobber(self.gpa, inst, .{
93806 assert(cg.mir_instructions.len > else_reloc_marker);
93807 try cg.mir_table.appendNTimes(cg.gpa, else_reloc_marker, table_len);
93808 if (is_loop) try cg.loop_switches.putNoClobber(cg.gpa, inst, .{
9374193809 .start = table_start,
9374293810 .len = table_len,
9374393811 .min = min.?,
9374493812 .else_relocs = if (switch_br.else_body_len > 0) .{ .forward = .empty } else .@"unreachable",
9374593813 });
9374693814 defer if (is_loop) {
93747 var loop_switch_data = self.loop_switches.fetchRemove(inst).?.value;
93815 var loop_switch_data = cg.loop_switches.fetchRemove(inst).?.value;
9374893816 switch (loop_switch_data.else_relocs) {
9374993817 .@"unreachable", .backward => {},
93750 .forward => |*else_relocs| else_relocs.deinit(self.gpa),
93818 .forward => |*else_relocs| else_relocs.deinit(cg.gpa),
9375193819 }
9375293820 };
9375393821 var cases_it = switch_br.iterateCases();
9375493822 while (cases_it.next()) |case| {
9375593823 {
93756 const table = self.mir_table.items[table_start..][0..table_len];
93824 const table = cg.mir_table.items[table_start..][0..table_len];
9375793825 for (case.items) |item| {
9375893826 const val = Value.fromInterned(item.toInterned().?);
9375993827 var val_space: Value.BigIntSpace = undefined;
9376093828 const val_bigint = val.toBigInt(&val_space, zcu);
9376193829 var index_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined };
9376293830 index_bigint.sub(val_bigint, min_bigint);
93763 table[index_bigint.toConst().to(u10) catch unreachable] = @intCast(self.mir_instructions.len);
93831 table[index_bigint.toConst().to(u10) catch unreachable] = @intCast(cg.mir_instructions.len);
9376493832 }
9376593833 for (case.ranges) |range| {
9376693834 var low_space: Value.BigIntSpace = undefined;
......@@ -93772,14 +93840,14 @@ fn lowerSwitchBr(
9377293840 const start = index_bigint.toConst().to(u10) catch unreachable;
9377393841 index_bigint.sub(high_bigint, min_bigint);
9377493842 const end = @as(u11, index_bigint.toConst().to(u10) catch unreachable) + 1;
93775 @memset(table[start..end], @intCast(self.mir_instructions.len));
93843 @memset(table[start..end], @intCast(cg.mir_instructions.len));
9377693844 }
9377793845 }
9377893846
93779 for (liveness.deaths[case.idx]) |operand| try self.processDeath(operand);
93847 for (liveness.deaths[case.idx]) |operand| try cg.processDeath(operand);
9378093848
93781 try self.genBodyBlock(case.body);
93782 try self.restoreState(state, &.{}, .{
93849 try cg.genBodyBlock(case.body);
93850 try cg.restoreState(state, &.{}, .{
9378393851 .emit_instructions = false,
9378493852 .update_tracking = true,
9378593853 .resurrect = true,
......@@ -93790,21 +93858,21 @@ fn lowerSwitchBr(
9379093858 const else_body = cases_it.elseBody();
9379193859
9379293860 const else_deaths = liveness.deaths.len - 1;
93793 for (liveness.deaths[else_deaths]) |operand| try self.processDeath(operand);
93861 for (liveness.deaths[else_deaths]) |operand| try cg.processDeath(operand);
9379493862
93795 self.performReloc(else_reloc);
93863 cg.performReloc(else_reloc);
9379693864 if (is_loop) {
93797 const loop_switch_data = self.loop_switches.getPtr(inst).?;
93798 for (loop_switch_data.else_relocs.forward.items) |reloc| self.performReloc(reloc);
93799 loop_switch_data.else_relocs.forward.deinit(self.gpa);
93800 loop_switch_data.else_relocs = .{ .backward = @intCast(self.mir_instructions.len) };
93865 const loop_switch_data = cg.loop_switches.getPtr(inst).?;
93866 for (loop_switch_data.else_relocs.forward.items) |reloc| cg.performReloc(reloc);
93867 loop_switch_data.else_relocs.forward.deinit(cg.gpa);
93868 loop_switch_data.else_relocs = .{ .backward = @intCast(cg.mir_instructions.len) };
9380193869 }
93802 for (self.mir_table.items[table_start..][0..table_len]) |*entry| if (entry.* == else_reloc_marker) {
93803 entry.* = @intCast(self.mir_instructions.len);
93870 for (cg.mir_table.items[table_start..][0..table_len]) |*entry| if (entry.* == else_reloc_marker) {
93871 entry.* = @intCast(cg.mir_instructions.len);
9380493872 };
9380593873
93806 try self.genBodyBlock(else_body);
93807 try self.restoreState(state, &.{}, .{
93874 try cg.genBodyBlock(else_body);
93875 try cg.restoreState(state, &.{}, .{
9380893876 .emit_instructions = false,
9380993877 .update_tracking = true,
9381093878 .resurrect = true,
......@@ -93819,9 +93887,12 @@ fn lowerSwitchBr(
9381993887 const relocs = try allocator.alloc(Mir.Inst.Index, case.items.len + case.ranges.len);
9382093888 defer allocator.free(relocs);
9382193889
93822 try self.spillEflagsIfOccupied();
93890 var cond_temp = try cg.tempInit(condition_ty, condition);
93891 const reset_index = cg.next_temp_index;
93892
93893 try cg.spillEflagsIfOccupied();
9382393894 for (case.items, relocs[0..case.items.len]) |item, *reloc| {
93824 const item_mcv = try self.resolveInst(item);
93895 const item_mcv = try cg.resolveInst(item);
9382593896 const cc: Condition = switch (condition) {
9382693897 .eflags => |cc| switch (item_mcv.immediate) {
9382793898 0 => cc.negate(),
......@@ -93829,27 +93900,24 @@ fn lowerSwitchBr(
9382993900 else => unreachable,
9383093901 },
9383193902 else => cc: {
93832 var cond_temp = try self.tempInit(condition_ty, condition);
93833 var item_temp = try self.tempInit(condition_ty, item_mcv);
93834 const cc_temp = cond_temp.cmpInts(.eq, &item_temp, self) catch |err| switch (err) {
93903 var item_temp = try cg.tempInit(condition_ty, item_mcv);
93904 const cc_temp = cond_temp.cmpInts(.eq, &item_temp, cg) catch |err| switch (err) {
9383593905 error.SelectFailed => unreachable,
9383693906 else => |e| return e,
9383793907 };
93838 try cond_temp.die(self);
93839 try item_temp.die(self);
93840 const cc = cc_temp.tracking(self).short.eflags;
93841 try cc_temp.die(self);
93842 try self.resetTemps();
93908 try item_temp.die(cg);
93909 const cc = cc_temp.tracking(cg).short.eflags;
93910 try cc_temp.die(cg);
93911 try cg.resetTemps(reset_index);
9384393912 break :cc cc;
9384493913 },
9384593914 };
93846 reloc.* = try self.asmJccReloc(cc, undefined);
93915 reloc.* = try cg.asmJccReloc(cc, undefined);
9384793916 }
9384893917
9384993918 for (case.ranges, relocs[case.items.len..]) |range, *reloc| {
93850 var cond_temp = try self.tempInit(condition_ty, condition);
93851 const min_mcv = try self.resolveInst(range[0]);
93852 const max_mcv = try self.resolveInst(range[1]);
93919 const min_mcv = try cg.resolveInst(range[0]);
93920 const max_mcv = try cg.resolveInst(range[1]);
9385393921 // `null` means always false.
9385493922 const lt_min = cc: switch (condition) {
9385593923 .eflags => |cc| switch (min_mcv.immediate) {
......@@ -93858,19 +93926,19 @@ fn lowerSwitchBr(
9385893926 else => unreachable,
9385993927 },
9386093928 else => {
93861 var min_temp = try self.tempInit(condition_ty, min_mcv);
93862 const cc_temp = cond_temp.cmpInts(.lt, &min_temp, self) catch |err| switch (err) {
93929 var min_temp = try cg.tempInit(condition_ty, min_mcv);
93930 const cc_temp = cond_temp.cmpInts(.lt, &min_temp, cg) catch |err| switch (err) {
9386393931 error.SelectFailed => unreachable,
9386493932 else => |e| return e,
9386593933 };
93866 try min_temp.die(self);
93867 const cc = cc_temp.tracking(self).short.eflags;
93868 try cc_temp.die(self);
93934 try min_temp.die(cg);
93935 const cc = cc_temp.tracking(cg).short.eflags;
93936 try cc_temp.die(cg);
9386993937 break :cc cc;
9387093938 },
9387193939 };
9387293940 const lt_min_reloc = if (lt_min) |cc| r: {
93873 break :r try self.asmJccReloc(cc, undefined);
93941 break :r try cg.asmJccReloc(cc, undefined);
9387493942 } else null;
9387593943 // `null` means always true.
9387693944 const lte_max = switch (condition) {
......@@ -93880,38 +93948,41 @@ fn lowerSwitchBr(
9388093948 else => unreachable,
9388193949 },
9388293950 else => cc: {
93883 var max_temp = try self.tempInit(condition_ty, max_mcv);
93884 const cc_temp = cond_temp.cmpInts(.lte, &max_temp, self) catch |err| switch (err) {
93951 var max_temp = try cg.tempInit(condition_ty, max_mcv);
93952 const cc_temp = cond_temp.cmpInts(.lte, &max_temp, cg) catch |err| switch (err) {
9388593953 error.SelectFailed => unreachable,
9388693954 else => |e| return e,
9388793955 };
93888 try max_temp.die(self);
93889 const cc = cc_temp.tracking(self).short.eflags;
93890 try cc_temp.die(self);
93956 try max_temp.die(cg);
93957 const cc = cc_temp.tracking(cg).short.eflags;
93958 try cc_temp.die(cg);
9389193959 break :cc cc;
9389293960 },
9389393961 };
93894 try cond_temp.die(self);
93895 try self.resetTemps();
93962 try cg.resetTemps(reset_index);
9389693963 // "Success" case is in `reloc`....
9389793964 if (lte_max) |cc| {
93898 reloc.* = try self.asmJccReloc(cc, undefined);
93965 reloc.* = try cg.asmJccReloc(cc, undefined);
9389993966 } else {
93900 reloc.* = try self.asmJmpReloc(undefined);
93967 reloc.* = try cg.asmJmpReloc(undefined);
9390193968 }
9390293969 // ...and "fail" case falls through to next checks.
93903 if (lt_min_reloc) |r| self.performReloc(r);
93970 if (lt_min_reloc) |r| cg.performReloc(r);
9390493971 }
9390593972
93973 try cond_temp.die(cg);
93974 try cg.resetTemps(@enumFromInt(0));
93975 cg.checkInvariantsAfterAirInst();
93976
9390693977 // The jump to skip this case if the conditions all failed.
93907 const skip_case_reloc = try self.asmJmpReloc(undefined);
93978 const skip_case_reloc = try cg.asmJmpReloc(undefined);
9390893979
93909 for (liveness.deaths[case.idx]) |operand| try self.processDeath(operand);
93980 for (liveness.deaths[case.idx]) |operand| try cg.processDeath(operand);
9391093981
9391193982 // Relocate all success cases to the body we're about to generate.
93912 for (relocs) |reloc| self.performReloc(reloc);
93913 try self.genBodyBlock(case.body);
93914 try self.restoreState(state, &.{}, .{
93983 for (relocs) |reloc| cg.performReloc(reloc);
93984 try cg.genBodyBlock(case.body);
93985 try cg.restoreState(state, &.{}, .{
9391593986 .emit_instructions = false,
9391693987 .update_tracking = true,
9391793988 .resurrect = true,
......@@ -93919,16 +93990,16 @@ fn lowerSwitchBr(
9391993990 });
9392093991
9392193992 // Relocate the "skip" branch to fall through to the next case.
93922 self.performReloc(skip_case_reloc);
93993 cg.performReloc(skip_case_reloc);
9392393994 }
9392493995 if (switch_br.else_body_len > 0) {
9392593996 const else_body = cases_it.elseBody();
9392693997
9392793998 const else_deaths = liveness.deaths.len - 1;
93928 for (liveness.deaths[else_deaths]) |operand| try self.processDeath(operand);
93999 for (liveness.deaths[else_deaths]) |operand| try cg.processDeath(operand);
9392994000
93930 try self.genBodyBlock(else_body);
93931 try self.restoreState(state, &.{}, .{
94001 try cg.genBodyBlock(else_body);
94002 try cg.restoreState(state, &.{}, .{
9393294003 .emit_instructions = false,
9393394004 .update_tracking = true,
9393494005 .resurrect = true,
......@@ -95003,7 +95074,7 @@ fn moveStrategy(cg: *CodeGen, ty: Type, class: Register.Class, aligned: bool) !M
9500395074 .mmx => {},
9500495075 .sse => switch (ty.zigTypeTag(zcu)) {
9500595076 else => {
95006 const classes = std.mem.sliceTo(&abi.classifySystemV(ty, zcu, cg.target.*, .other), .none);
95077 const classes = std.mem.sliceTo(&abi.classifySystemV(ty, zcu, cg.target, .other), .none);
9500795078 assert(std.mem.indexOfNone(abi.Class, classes, &.{
9500895079 .integer, .sse, .sseup, .memory, .float, .float_combine,
9500995080 }) == null);
......@@ -99635,7 +99706,7 @@ fn airVaArg(self: *CodeGen, inst: Air.Inst.Index) !void {
9963599706 const overflow_arg_area: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 8 } };
9963699707 const reg_save_area: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 16 } };
9963799708
99638 const classes = std.mem.sliceTo(&abi.classifySystemV(promote_ty, zcu, self.target.*, .arg), .none);
99709 const classes = std.mem.sliceTo(&abi.classifySystemV(promote_ty, zcu, self.target, .arg), .none);
9963999710 switch (classes[0]) {
9964099711 .integer => {
9964199712 assert(classes.len == 1);
......@@ -99980,7 +100051,7 @@ fn resolveCallingConventionValues(
99980100051 var ret_tracking_i: usize = 0;
99981100052
99982100053 const classes = switch (cc) {
99983 .x86_64_sysv => std.mem.sliceTo(&abi.classifySystemV(ret_ty, zcu, self.target.*, .ret), .none),
100054 .x86_64_sysv => std.mem.sliceTo(&abi.classifySystemV(ret_ty, zcu, self.target, .ret), .none),
99984100055 .x86_64_win => &.{abi.classifyWindows(ret_ty, zcu)},
99985100056 else => unreachable,
99986100057 };
......@@ -100069,7 +100140,7 @@ fn resolveCallingConventionValues(
100069100140 var arg_mcv_i: usize = 0;
100070100141
100071100142 const classes = switch (cc) {
100072 .x86_64_sysv => std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .arg), .none),
100143 .x86_64_sysv => std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target, .arg), .none),
100073100144 .x86_64_win => &.{abi.classifyWindows(ty, zcu)},
100074100145 else => unreachable,
100075100146 };
......@@ -100373,7 +100444,7 @@ fn splitType(self: *CodeGen, comptime parts_len: usize, ty: Type) ![parts_len]Ty
100373100444 error.DivisionByZero => unreachable,
100374100445 error.UnexpectedRemainder => {},
100375100446 };
100376 const classes = std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .other), .none);
100447 const classes = std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target, .other), .none);
100377100448 if (classes.len == parts_len) for (&parts, classes, 0..) |*part, class, part_i| {
100378100449 part.* = switch (class) {
100379100450 .integer => if (part_i < parts_len - 1)
......@@ -101339,6 +101410,7 @@ const Temp = struct {
101339101410 const val_mcv = val.tracking(cg).short;
101340101411 switch (val_mcv) {
101341101412 else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
101413 .none => {},
101342101414 .undef => if (opts.safe) {
101343101415 var dst_ptr = try cg.tempInit(.usize, dst.tracking(cg).short.address().offset(opts.disp));
101344101416 var pat = try cg.tempInit(.u8, .{ .immediate = 0xaa });
......@@ -101371,19 +101443,19 @@ const Temp = struct {
101371101443 .disp = opts.disp,
101372101444 }),
101373101445 ),
101374 .register => |val_reg| try dst.writeRegs(opts.disp, val_ty, &.{registerAlias(
101446 .register => |val_reg| try dst.writeReg(opts.disp, val_ty, registerAlias(
101375101447 val_reg,
101376101448 @intCast(val_ty.abiSize(cg.pt.zcu)),
101377 )}, cg),
101449 ), cg),
101378101450 inline .register_pair,
101379101451 .register_triple,
101380101452 .register_quadruple,
101381101453 => |val_regs| try dst.writeRegs(opts.disp, val_ty, &val_regs, cg),
101382101454 .register_offset => |val_reg_off| switch (val_reg_off.off) {
101383 0 => try dst.writeRegs(opts.disp, val_ty, &.{registerAlias(
101455 0 => try dst.writeReg(opts.disp, val_ty, registerAlias(
101384101456 val_reg_off.reg,
101385101457 @intCast(val_ty.abiSize(cg.pt.zcu)),
101386 )}, cg),
101458 ), cg),
101387101459 else => continue :val_to_gpr,
101388101460 },
101389101461 .register_overflow => |val_reg_ov| {
......@@ -101401,7 +101473,7 @@ const Temp = struct {
101401101473 else => std.debug.panic("{s}: {}\n", .{ @src().fn_name, val_ty.fmt(cg.pt) }),
101402101474 });
101403101475 const first_size: u31 = @intCast(first_ty.abiSize(cg.pt.zcu));
101404 try dst.writeRegs(opts.disp, first_ty, &.{registerAlias(val_reg_ov.reg, first_size)}, cg);
101476 try dst.writeReg(opts.disp, first_ty, registerAlias(val_reg_ov.reg, first_size), cg);
101405101477 try cg.asmSetccMemory(
101406101478 val_reg_ov.eflags,
101407101479 try dst.tracking(cg).short.mem(cg, .{
......@@ -101492,42 +101564,76 @@ const Temp = struct {
101492101564 }));
101493101565 }
101494101566
101567 fn writeReg(dst: Temp, disp: i32, src_ty: Type, src_reg: Register, cg: *CodeGen) InnerError!void {
101568 const src_abi_size: u31 = @intCast(src_ty.abiSize(cg.pt.zcu));
101569 const src_rc = src_reg.class();
101570 if (src_rc == .x87 or std.math.isPowerOfTwo(src_abi_size)) {
101571 const strat = try cg.moveStrategy(src_ty, src_rc, false);
101572 try strat.write(cg, try dst.tracking(cg).short.mem(cg, .{
101573 .size = .fromBitSize(@min(8 * src_abi_size, src_reg.bitSize())),
101574 .disp = disp,
101575 }), registerAlias(src_reg, src_abi_size));
101576 } else {
101577 const frame_size = std.math.ceilPowerOfTwoAssert(u32, src_abi_size);
101578 const frame_index = try cg.allocFrameIndex(.init(.{
101579 .size = frame_size,
101580 .alignment = .fromNonzeroByteUnits(frame_size),
101581 }));
101582 const strat = try cg.moveStrategy(src_ty, src_rc, true);
101583 try strat.write(cg, .{
101584 .base = .{ .frame = frame_index },
101585 .mod = .{ .rm = .{ .size = .fromSize(frame_size) } },
101586 }, registerAlias(src_reg, frame_size));
101587 var dst_ptr = try cg.tempInit(.usize, dst.tracking(cg).short.address());
101588 try dst_ptr.toOffset(disp, cg);
101589 var src_ptr = try cg.tempInit(.usize, .{ .lea_frame = .{ .index = frame_index } });
101590 var len = try cg.tempInit(.usize, .{ .immediate = src_abi_size });
101591 try dst_ptr.memcpy(&src_ptr, &len, cg);
101592 try dst_ptr.die(cg);
101593 try src_ptr.die(cg);
101594 try len.die(cg);
101595 }
101596 }
101597
101495101598 fn writeRegs(dst: Temp, disp: i32, src_ty: Type, src_regs: []const Register, cg: *CodeGen) InnerError!void {
101599 const zcu = cg.pt.zcu;
101600 const classes = std.mem.sliceTo(&abi.classifySystemV(src_ty, zcu, cg.target, .other), .none);
101601 var next_class_index: u4 = 0;
101496101602 var part_disp = disp;
101497 var src_abi_size: u32 = @intCast(src_ty.abiSize(cg.pt.zcu));
101603 var remaining_abi_size = src_ty.abiSize(zcu);
101498101604 for (src_regs) |src_reg| {
101499 const src_rc = src_reg.class();
101500 const part_bit_size = @min(8 * src_abi_size, src_reg.bitSize());
101501 const part_size = @divExact(part_bit_size, 8);
101502 if (src_rc == .x87 or std.math.isPowerOfTwo(part_size)) {
101503 const strat = try cg.moveStrategy(src_ty, src_rc, false);
101504 try strat.write(cg, try dst.tracking(cg).short.mem(cg, .{
101505 .size = .fromBitSize(part_bit_size),
101506 .disp = part_disp,
101507 }), registerAlias(src_reg, part_size));
101508 } else {
101509 const frame_size = std.math.ceilPowerOfTwoAssert(u32, part_size);
101510 const frame_index = try cg.allocFrameIndex(.init(.{
101511 .size = frame_size,
101512 .alignment = .fromNonzeroByteUnits(frame_size),
101513 }));
101514 const strat = try cg.moveStrategy(src_ty, src_rc, true);
101515 try strat.write(cg, .{
101516 .base = .{ .frame = frame_index },
101517 .mod = .{ .rm = .{ .size = .fromSize(frame_size) } },
101518 }, registerAlias(src_reg, frame_size));
101519 var dst_ptr = try cg.tempInit(.usize, dst.tracking(cg).short.address());
101520 try dst_ptr.toOffset(part_disp, cg);
101521 var src_ptr = try cg.tempInit(.usize, .{ .lea_frame = .{ .index = frame_index } });
101522 var len = try cg.tempInit(.usize, .{ .immediate = src_abi_size });
101523 try dst_ptr.memcpy(&src_ptr, &len, cg);
101524 try dst_ptr.die(cg);
101525 try src_ptr.die(cg);
101526 try len.die(cg);
101527 }
101605 const class_index = next_class_index;
101606 const class = classes[class_index];
101607 next_class_index = @intCast(switch (class) {
101608 .integer, .memory, .float, .float_combine => class_index + 1,
101609 .sse => std.mem.indexOfNonePos(abi.Class, classes, class_index + 1, &.{.sseup}) orelse classes.len,
101610 .x87 => std.mem.indexOfNonePos(abi.Class, classes, class_index + 1, &.{.x87up}) orelse classes.len,
101611 .sseup, .x87up, .complex_x87, .none, .win_i128, .integer_per_element => unreachable,
101612 });
101613 const part_size = switch (class) {
101614 .integer, .sse, .memory => @min(8 * @as(u7, next_class_index - class_index), remaining_abi_size),
101615 .x87 => 16,
101616 .float => 4,
101617 .float_combine => 8,
101618 .sseup, .x87up, .complex_x87, .none, .win_i128, .integer_per_element => unreachable,
101619 };
101620 try dst.writeReg(part_disp, switch (class) {
101621 .integer => .u64,
101622 .sse => switch (part_size) {
101623 else => unreachable,
101624 8 => .f64,
101625 16 => .vector_2_f64,
101626 32 => .vector_4_f64,
101627 },
101628 .x87 => .f80,
101629 .float => .f32,
101630 .float_combine => .vector_2_f32,
101631 .sseup, .x87up, .complex_x87, .memory, .none, .win_i128, .integer_per_element => unreachable,
101632 }, src_reg, cg);
101528101633 part_disp += part_size;
101529 src_abi_size -= part_size;
101634 remaining_abi_size -= part_size;
101530101635 }
101636 assert(next_class_index == classes.len);
101531101637 }
101532101638
101533101639 fn memcpy(dst: *Temp, src: *Temp, len: *Temp, cg: *CodeGen) InnerError!void {
......@@ -105786,9 +105892,9 @@ const Temp = struct {
105786105892 };
105787105893};
105788105894
105789fn resetTemps(cg: *CodeGen) InnerError!void {
105895fn resetTemps(cg: *CodeGen, from_index: Temp.Index) InnerError!void {
105790105896 var any_valid = false;
105791 for (0..@intFromEnum(cg.next_temp_index)) |temp_index| {
105897 for (@intFromEnum(from_index)..@intFromEnum(cg.next_temp_index)) |temp_index| {
105792105898 const temp: Temp.Index = @enumFromInt(temp_index);
105793105899 if (temp.isValid(cg)) {
105794105900 any_valid = true;
......@@ -105800,7 +105906,7 @@ fn resetTemps(cg: *CodeGen) InnerError!void {
105800105906 cg.temp_type[temp_index] = undefined;
105801105907 }
105802105908 if (any_valid) return cg.fail("failed to kill all temps", .{});
105803 cg.next_temp_index = @enumFromInt(0);
105909 cg.next_temp_index = from_index;
105804105910}
105805105911
105806105912fn reuseTemp(
......@@ -105889,70 +105995,75 @@ fn tempMemFromValue(cg: *CodeGen, value: Value) InnerError!Temp {
105889105995 return cg.tempInit(value.typeOf(cg.pt.zcu), try cg.lowerUav(value));
105890105996}
105891105997
105892fn tempFromOperand(
105893 cg: *CodeGen,
105894 inst: Air.Inst.Index,
105895 op_index: Liveness.OperandInt,
105896 op_ref: Air.Inst.Ref,
105897 ignore_death: bool,
105898) InnerError!Temp {
105998fn tempFromOperand(cg: *CodeGen, op_ref: Air.Inst.Ref, op_dies: bool) InnerError!Temp {
105899105999 const zcu = cg.pt.zcu;
105900106000 const ip = &zcu.intern_pool;
105901106001
105902 if (ignore_death or !cg.liveness.operandDies(inst, op_index)) {
105903 if (op_ref.toIndex()) |op_inst| return .{ .index = op_inst };
105904 const val = op_ref.toInterned().?;
105905 const gop = try cg.const_tracking.getOrPut(cg.gpa, val);
105906 if (!gop.found_existing) gop.value_ptr.* = .init(init: {
105907 const const_mcv = try cg.genTypedValue(.fromInterned(val));
105908 switch (const_mcv) {
105909 .lea_tlv => |tlv_sym| switch (cg.bin_file.tag) {
105910 .elf, .macho => {
105911 if (cg.mod.pic) {
105912 try cg.spillRegisters(&.{ .rdi, .rax });
105913 } else {
105914 try cg.spillRegisters(&.{.rax});
105915 }
105916 const frame_index = try cg.allocFrameIndex(.init(.{
105917 .size = 8,
105918 .alignment = .@"8",
105919 }));
105920 try cg.genSetMem(
105921 .{ .frame = frame_index },
105922 0,
105923 .usize,
105924 .{ .lea_symbol = .{ .sym_index = tlv_sym } },
105925 .{},
105926 );
105927 break :init .{ .load_frame = .{ .index = frame_index } };
105928 },
105929 else => break :init const_mcv,
106002 if (op_dies) {
106003 const temp_index = cg.next_temp_index;
106004 const temp: Temp = .{ .index = temp_index.toIndex() };
106005 const op_inst = op_ref.toIndex().?;
106006 const tracking = cg.getResolvedInstValue(op_inst);
106007 temp_index.tracking(cg).* = tracking.*;
106008 if (!cg.reuseTemp(temp.index, op_inst, tracking)) return .{ .index = op_ref.toIndex().? };
106009 cg.temp_type[@intFromEnum(temp_index)] = cg.typeOf(op_ref);
106010 cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
106011 return temp;
106012 }
106013
106014 if (op_ref.toIndex()) |op_inst| return .{ .index = op_inst };
106015 const val = op_ref.toInterned().?;
106016 const gop = try cg.const_tracking.getOrPut(cg.gpa, val);
106017 if (!gop.found_existing) gop.value_ptr.* = .init(init: {
106018 const const_mcv = try cg.genTypedValue(.fromInterned(val));
106019 switch (const_mcv) {
106020 .lea_tlv => |tlv_sym| switch (cg.bin_file.tag) {
106021 .elf, .macho => {
106022 if (cg.mod.pic) {
106023 try cg.spillRegisters(&.{ .rdi, .rax });
106024 } else {
106025 try cg.spillRegisters(&.{.rax});
106026 }
106027 const frame_index = try cg.allocFrameIndex(.init(.{
106028 .size = 8,
106029 .alignment = .@"8",
106030 }));
106031 try cg.genSetMem(
106032 .{ .frame = frame_index },
106033 0,
106034 .usize,
106035 .{ .lea_symbol = .{ .sym_index = tlv_sym } },
106036 .{},
106037 );
106038 break :init .{ .load_frame = .{ .index = frame_index } };
105930106039 },
105931106040 else => break :init const_mcv,
105932 }
105933 });
105934 return cg.tempInit(.fromInterned(ip.typeOf(val)), gop.value_ptr.short);
105935 }
106041 },
106042 else => break :init const_mcv,
106043 }
106044 });
106045 return cg.tempInit(.fromInterned(ip.typeOf(val)), gop.value_ptr.short);
106046}
105936106047
105937 const temp_index = cg.next_temp_index;
105938 const temp: Temp = .{ .index = temp_index.toIndex() };
105939 const op_inst = op_ref.toIndex().?;
105940 const tracking = cg.getResolvedInstValue(op_inst);
105941 temp_index.tracking(cg).* = tracking.*;
105942 if (!cg.reuseTemp(temp.index, op_inst, tracking)) return .{ .index = op_ref.toIndex().? };
105943 cg.temp_type[@intFromEnum(temp_index)] = cg.typeOf(op_ref);
105944 cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
105945 return temp;
106048fn tempsFromOperandsInner(
106049 cg: *CodeGen,
106050 inst: Air.Inst.Index,
106051 op_temps: []Temp,
106052 op_refs: []const Air.Inst.Ref,
106053) InnerError!void {
106054 for (op_temps, 0.., op_refs) |*op_temp, op_index, op_ref| op_temp.* = try cg.tempFromOperand(op_ref, for (op_refs[0..op_index]) |prev_op_ref| {
106055 if (op_ref == prev_op_ref) break false;
106056 } else cg.liveness.operandDies(inst, @intCast(op_index)));
105946106057}
105947106058
105948inline fn tempsFromOperands(cg: *CodeGen, inst: Air.Inst.Index, op_refs: anytype) InnerError![op_refs.len]Temp {
105949 var temps: [op_refs.len]Temp = undefined;
105950 inline for (&temps, 0.., op_refs) |*temp, op_index, op_ref| {
105951 temp.* = try cg.tempFromOperand(inst, op_index, op_ref, inline for (0..op_index) |prev_op_index| {
105952 if (op_ref == op_refs[prev_op_index]) break true;
105953 } else false);
105954 }
105955 return temps;
106059inline fn tempsFromOperands(
106060 cg: *CodeGen,
106061 inst: Air.Inst.Index,
106062 op_refs: anytype,
106063) InnerError![op_refs.len]Temp {
106064 var op_temps: [op_refs.len]Temp = undefined;
106065 try cg.tempsFromOperandsInner(inst, &op_temps, &op_refs);
106066 return op_temps;
105956106067}
105957106068
105958106069const Operand = union(enum) {
src/arch/x86_64/abi.zig+4-4
......@@ -100,7 +100,7 @@ pub const Context = enum { ret, arg, field, other };
100100
101101/// There are a maximum of 8 possible return slots. Returned values are in
102102/// the beginning of the array; unused slots are filled with .none.
103pub fn classifySystemV(ty: Type, zcu: *Zcu, target: std.Target, ctx: Context) [8]Class {
103pub fn classifySystemV(ty: Type, zcu: *Zcu, target: *const std.Target, ctx: Context) [8]Class {
104104 const memory_class = [_]Class{
105105 .memory, .none, .none, .none,
106106 .none, .none, .none, .none,
......@@ -148,7 +148,7 @@ pub fn classifySystemV(ty: Type, zcu: *Zcu, target: std.Target, ctx: Context) [8
148148 result[0] = .integer;
149149 return result;
150150 },
151 .float => switch (ty.floatBits(target)) {
151 .float => switch (ty.floatBits(target.*)) {
152152 16 => {
153153 if (ctx == .field) {
154154 result[0] = .memory;
......@@ -330,7 +330,7 @@ fn classifySystemVStruct(
330330 starting_byte_offset: u64,
331331 loaded_struct: InternPool.LoadedStructType,
332332 zcu: *Zcu,
333 target: std.Target,
333 target: *const std.Target,
334334) u64 {
335335 const ip = &zcu.intern_pool;
336336 var byte_offset = starting_byte_offset;
......@@ -379,7 +379,7 @@ fn classifySystemVUnion(
379379 starting_byte_offset: u64,
380380 loaded_union: InternPool.LoadedUnionType,
381381 zcu: *Zcu,
382 target: std.Target,
382 target: *const std.Target,
383383) u64 {
384384 const ip = &zcu.intern_pool;
385385 for (0..loaded_union.field_types.len) |field_index| {
src/codegen/llvm.zig+3-3
......@@ -11757,7 +11757,7 @@ fn firstParamSRet(fn_info: InternPool.Key.FuncType, zcu: *Zcu, target: std.Targe
1175711757}
1175811758
1175911759fn firstParamSRetSystemV(ty: Type, zcu: *Zcu, target: std.Target) bool {
11760 const class = x86_64_abi.classifySystemV(ty, zcu, target, .ret);
11760 const class = x86_64_abi.classifySystemV(ty, zcu, &target, .ret);
1176111761 if (class[0] == .memory) return true;
1176211762 if (class[0] == .x87 and class[2] != .none) return true;
1176311763 return false;
......@@ -11867,7 +11867,7 @@ fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.E
1186711867 return o.lowerType(return_type);
1186811868 }
1186911869 const target = zcu.getTarget();
11870 const classes = x86_64_abi.classifySystemV(return_type, zcu, target, .ret);
11870 const classes = x86_64_abi.classifySystemV(return_type, zcu, &target, .ret);
1187111871 if (classes[0] == .memory) return .void;
1187211872 var types_index: u32 = 0;
1187311873 var types_buffer: [8]Builder.Type = undefined;
......@@ -12145,7 +12145,7 @@ const ParamTypeIterator = struct {
1214512145 const zcu = it.object.pt.zcu;
1214612146 const ip = &zcu.intern_pool;
1214712147 const target = zcu.getTarget();
12148 const classes = x86_64_abi.classifySystemV(ty, zcu, target, .arg);
12148 const classes = x86_64_abi.classifySystemV(ty, zcu, &target, .arg);
1214912149 if (classes[0] == .memory) {
1215012150 it.zig_index += 1;
1215112151 it.llvm_index += 1;
src/main.zig+1-1
......@@ -39,7 +39,7 @@ test {
3939 _ = Package;
4040}
4141
42const thread_stack_size = 32 << 20;
42const thread_stack_size = 50 << 20;
4343
4444pub const std_options: std.Options = .{
4545 .wasiCwd = wasi_cwd,