authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-10-05 20:23:36-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-10-07 00:29:17-04:00
log5aeb13c3500f2244efe916a88278a86692dfc400
treed7f04d76b1445e679b058efdfc0b2958e6178ed4
parent9f8b2ff9e223315682fbe9237239714b7d8883f6

x86_64: implement `f80` movement

9 files changed, 187 insertions(+), 184 deletions(-)

src/arch/x86_64/CodeGen.zig+143-157
......@@ -48,6 +48,7 @@ const RegisterLock = RegisterManager.RegisterLock;
4848const FrameIndex = bits.FrameIndex;
4949
5050const gp = abi.RegisterClass.gp;
51const x87 = abi.RegisterClass.x87;
5152const sse = abi.RegisterClass.sse;
5253
5354const InnerError = CodeGenError || error{OutOfRegisters};
......@@ -532,8 +533,8 @@ const InstTracking = struct {
532533 };
533534 }
534535
535 fn trackSpill(self: *InstTracking, function: *Self, inst: Air.Inst.Index) void {
536 function.freeValue(self.short);
536 fn trackSpill(self: *InstTracking, function: *Self, inst: Air.Inst.Index) !void {
537 try function.freeValue(self.short);
537538 self.reuseFrame();
538539 tracking_log.debug("%{d} => {} (spilled)", .{ inst, self.* });
539540 }
......@@ -618,8 +619,8 @@ const InstTracking = struct {
618619 }
619620 }
620621
621 fn die(self: *InstTracking, function: *Self, inst: Air.Inst.Index) void {
622 function.freeValue(self.short);
622 fn die(self: *InstTracking, function: *Self, inst: Air.Inst.Index) !void {
623 try function.freeValue(self.short);
623624 self.short = .{ .dead = function.scope_generation };
624625 tracking_log.debug("%{d} => {} (death)", .{ inst, self.* });
625626 }
......@@ -980,7 +981,6 @@ fn formatWipMir(
980981 _: std.fmt.FormatOptions,
981982 writer: anytype,
982983) @TypeOf(writer).Error!void {
983 const mod = data.self.bin_file.options.module.?;
984984 var lower = Lower{
985985 .allocator = data.self.gpa,
986986 .mir = .{
......@@ -988,7 +988,7 @@ fn formatWipMir(
988988 .extra = data.self.mir_extra.items,
989989 .frame_locs = (std.MultiArrayList(Mir.FrameLoc){}).slice(),
990990 },
991 .cc = mod.typeToFunc(data.self.fn_type).?.cc,
991 .cc = .Unspecified,
992992 .src_loc = data.self.src_loc,
993993 };
994994 for ((lower.lowerMir(data.inst) catch |err| switch (err) {
......@@ -2132,9 +2132,12 @@ fn getValue(self: *Self, value: MCValue, inst: ?Air.Inst.Index) void {
21322132 self.register_manager.getRegAssumeFree(reg, inst);
21332133}
21342134
2135fn freeValue(self: *Self, value: MCValue) void {
2135fn freeValue(self: *Self, value: MCValue) !void {
21362136 switch (value) {
2137 .register => |reg| self.register_manager.freeReg(reg),
2137 .register => |reg| {
2138 if (reg.class() == .x87) try self.asmRegister(.{ .f_, .free }, reg);
2139 self.register_manager.freeReg(reg);
2140 },
21382141 .register_pair => |regs| for (regs) |reg| self.register_manager.freeReg(reg),
21392142 .register_offset => |reg_off| self.register_manager.freeReg(reg_off.reg),
21402143 .register_overflow => |reg_ov| {
......@@ -2146,13 +2149,13 @@ fn freeValue(self: *Self, value: MCValue) void {
21462149 }
21472150}
21482151
2149fn feed(self: *Self, bt: *Liveness.BigTomb, operand: Air.Inst.Ref) void {
2150 if (bt.feed()) if (Air.refToIndex(operand)) |inst| self.processDeath(inst);
2152fn feed(self: *Self, bt: *Liveness.BigTomb, operand: Air.Inst.Ref) !void {
2153 if (bt.feed()) if (Air.refToIndex(operand)) |inst| try self.processDeath(inst);
21512154}
21522155
21532156/// Asserts there is already capacity to insert into top branch inst_table.
2154fn processDeath(self: *Self, inst: Air.Inst.Index) void {
2155 self.inst_tracking.getPtr(inst).?.die(self, inst);
2157fn processDeath(self: *Self, inst: Air.Inst.Index) !void {
2158 try self.inst_tracking.getPtr(inst).?.die(self, inst);
21562159}
21572160
21582161/// Called when there are no operands, and the instruction is always unreferenced.
......@@ -2177,13 +2180,18 @@ fn finishAirResult(self: *Self, inst: Air.Inst.Index, result: MCValue) void {
21772180 self.finishAirBookkeeping();
21782181}
21792182
2180fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref) void {
2183fn finishAir(
2184 self: *Self,
2185 inst: Air.Inst.Index,
2186 result: MCValue,
2187 operands: [Liveness.bpi - 1]Air.Inst.Ref,
2188) !void {
21812189 var tomb_bits = self.liveness.getTombBits(inst);
21822190 for (operands) |op| {
21832191 const dies = @as(u1, @truncate(tomb_bits)) != 0;
21842192 tomb_bits >>= 1;
21852193 if (!dies) continue;
2186 self.processDeath(Air.refToIndexAllowNone(op) orelse continue);
2194 try self.processDeath(Air.refToIndexAllowNone(op) orelse continue);
21872195 }
21882196 self.finishAirResult(inst, result);
21892197}
......@@ -2409,7 +2417,7 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
24092417 for (
24102418 self.inst_tracking.keys()[state.inst_tracking_len..],
24112419 self.inst_tracking.values()[state.inst_tracking_len..],
2412 ) |inst, *tracking| tracking.die(self, inst);
2420 ) |inst, *tracking| try tracking.die(self, inst);
24132421 self.inst_tracking.shrinkRetainingCapacity(state.inst_tracking_len);
24142422 }
24152423
......@@ -2417,7 +2425,7 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
24172425 self.inst_tracking.keys()[0..state.inst_tracking_len],
24182426 self.inst_tracking.values()[0..state.inst_tracking_len],
24192427 ) |inst, *tracking| tracking.resurrect(inst, state.scope_generation);
2420 for (deaths) |death| self.processDeath(death);
2428 for (deaths) |death| try self.processDeath(death);
24212429
24222430 for (0..state.registers.len) |index| {
24232431 const current_maybe_inst = if (self.register_manager.free_registers.isSet(index))
......@@ -2444,7 +2452,7 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
24442452 }
24452453 if (opts.update_tracking) {
24462454 if (current_maybe_inst) |current_inst| {
2447 self.inst_tracking.getPtr(current_inst).?.trackSpill(self, current_inst);
2455 try self.inst_tracking.getPtr(current_inst).?.trackSpill(self, current_inst);
24482456 }
24492457 {
24502458 const reg = RegisterManager.regAtTrackedIndex(@intCast(index));
......@@ -2463,7 +2471,7 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
24632471 try self.inst_tracking.getPtr(inst).?.spill(self, inst);
24642472 if (opts.update_tracking) if (self.eflags_inst) |inst| {
24652473 self.eflags_inst = null;
2466 self.inst_tracking.getPtr(inst).?.trackSpill(self, inst);
2474 try self.inst_tracking.getPtr(inst).?.trackSpill(self, inst);
24672475 };
24682476
24692477 if (opts.update_tracking and std.debug.runtime_safety) {
......@@ -2481,7 +2489,7 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void
24812489 if (tracked_reg.id() == reg.id()) break;
24822490 } else unreachable; // spilled reg not tracked with spilled instruciton
24832491 try tracking.spill(self, inst);
2484 tracking.trackSpill(self, inst);
2492 try tracking.trackSpill(self, inst);
24852493}
24862494
24872495pub fn spillEflagsIfOccupied(self: *Self) !void {
......@@ -2490,7 +2498,7 @@ pub fn spillEflagsIfOccupied(self: *Self) !void {
24902498 const tracking = self.inst_tracking.getPtr(inst).?;
24912499 assert(tracking.getCondition() != null);
24922500 try tracking.spill(self, inst);
2493 tracking.trackSpill(self, inst);
2501 try tracking.trackSpill(self, inst);
24942502 }
24952503}
24962504
......@@ -8428,8 +8436,8 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
84288436 const ret = try self.genCall(.{ .air = pl_op.operand }, arg_tys, arg_vals);
84298437
84308438 var bt = self.liveness.iterateBigTomb(inst);
8431 self.feed(&bt, pl_op.operand);
8432 for (arg_refs) |arg_ref| self.feed(&bt, arg_ref);
8439 try self.feed(&bt, pl_op.operand);
8440 for (arg_refs) |arg_ref| try self.feed(&bt, arg_ref);
84338441
84348442 const result = if (self.liveness.isUnused(inst)) .unreach else ret;
84358443 return self.finishAirResult(inst, result);
......@@ -9086,13 +9094,13 @@ fn genTry(
90869094 const reloc = try self.genCondBrMir(Type.anyerror, is_err_mcv);
90879095
90889096 if (self.liveness.operandDies(inst, 0)) {
9089 if (Air.refToIndex(err_union)) |err_union_inst| self.processDeath(err_union_inst);
9097 if (Air.refToIndex(err_union)) |err_union_inst| try self.processDeath(err_union_inst);
90909098 }
90919099
90929100 self.scope_generation += 1;
90939101 const state = try self.saveState();
90949102
9095 for (liveness_cond_br.else_deaths) |operand| self.processDeath(operand);
9103 for (liveness_cond_br.else_deaths) |operand| try self.processDeath(operand);
90969104 try self.genBody(body);
90979105 try self.restoreState(state, &.{}, .{
90989106 .emit_instructions = false,
......@@ -9103,7 +9111,7 @@ fn genTry(
91039111
91049112 try self.performReloc(reloc);
91059113
9106 for (liveness_cond_br.then_deaths) |operand| self.processDeath(operand);
9114 for (liveness_cond_br.then_deaths) |operand| try self.processDeath(operand);
91079115
91089116 const result = if (self.liveness.isUnused(inst))
91099117 .unreach
......@@ -9196,13 +9204,13 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
91969204 // that death now instead of later as this has an effect on
91979205 // whether it needs to be spilled in the branches
91989206 if (self.liveness.operandDies(inst, 0)) {
9199 if (Air.refToIndex(pl_op.operand)) |op_inst| self.processDeath(op_inst);
9207 if (Air.refToIndex(pl_op.operand)) |op_inst| try self.processDeath(op_inst);
92009208 }
92019209
92029210 self.scope_generation += 1;
92039211 const state = try self.saveState();
92049212
9205 for (liveness_cond_br.then_deaths) |operand| self.processDeath(operand);
9213 for (liveness_cond_br.then_deaths) |operand| try self.processDeath(operand);
92069214 try self.genBody(then_body);
92079215 try self.restoreState(state, &.{}, .{
92089216 .emit_instructions = false,
......@@ -9213,7 +9221,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
92139221
92149222 try self.performReloc(reloc);
92159223
9216 for (liveness_cond_br.else_deaths) |operand| self.processDeath(operand);
9224 for (liveness_cond_br.else_deaths) |operand| try self.processDeath(operand);
92179225 try self.genBody(else_body);
92189226 try self.restoreState(state, &.{}, .{
92199227 .emit_instructions = false,
......@@ -9580,7 +9588,7 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
95809588
95819589 if (std.debug.runtime_safety) assert(self.inst_tracking.getIndex(inst).? == inst_tracking_i);
95829590 const tracking = &self.inst_tracking.values()[inst_tracking_i];
9583 if (self.liveness.isUnused(inst)) tracking.die(self, inst);
9591 if (self.liveness.isUnused(inst)) try tracking.die(self, inst);
95849592 self.getValue(tracking.short, inst);
95859593 self.finishAirBookkeeping();
95869594}
......@@ -9599,7 +9607,7 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
95999607 // that death now instead of later as this has an effect on
96009608 // whether it needs to be spilled in the branches
96019609 if (self.liveness.operandDies(inst, 0)) {
9602 if (Air.refToIndex(pl_op.operand)) |op_inst| self.processDeath(op_inst);
9610 if (Air.refToIndex(pl_op.operand)) |op_inst| try self.processDeath(op_inst);
96039611 }
96049612
96059613 self.scope_generation += 1;
......@@ -9622,7 +9630,7 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
96229630 reloc.* = try self.asmJccReloc(undefined, if (i < relocs.len - 1) .e else .ne);
96239631 }
96249632
9625 for (liveness.deaths[case_i]) |operand| self.processDeath(operand);
9633 for (liveness.deaths[case_i]) |operand| try self.processDeath(operand);
96269634
96279635 for (relocs[0 .. relocs.len - 1]) |reloc| try self.performReloc(reloc);
96289636 try self.genBody(case_body);
......@@ -9640,7 +9648,7 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
96409648 const else_body = self.air.extra[extra_index..][0..switch_br.data.else_body_len];
96419649
96429650 const else_deaths = liveness.deaths.len - 1;
9643 for (liveness.deaths[else_deaths]) |operand| self.processDeath(operand);
9651 for (liveness.deaths[else_deaths]) |operand| try self.processDeath(operand);
96449652
96459653 try self.genBody(else_body);
96469654 try self.restoreState(state, &.{}, .{
......@@ -9704,7 +9712,7 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
97049712
97059713 // Process operand death so that it is properly accounted for in the State below.
97069714 if (self.liveness.operandDies(inst, 0)) {
9707 if (Air.refToIndex(br.operand)) |op_inst| self.processDeath(op_inst);
9715 if (Air.refToIndex(br.operand)) |op_inst| try self.processDeath(op_inst);
97089716 }
97099717
97109718 if (first_br) {
......@@ -9718,7 +9726,7 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
97189726 });
97199727
97209728 // Stop tracking block result without forgetting tracking info
9721 self.freeValue(block_tracking.short);
9729 try self.freeValue(block_tracking.short);
97229730
97239731 // Emit a jump with a relocation. It will be patched up after the block ends.
97249732 // Leave the jump offset undefined
......@@ -10077,13 +10085,14 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
1007710085 return self.finishAir(inst, result, buf);
1007810086 }
1007910087 var bt = self.liveness.iterateBigTomb(inst);
10080 for (outputs) |output| if (output != .none) self.feed(&bt, output);
10081 for (inputs) |input| self.feed(&bt, input);
10088 for (outputs) |output| if (output != .none) try self.feed(&bt, output);
10089 for (inputs) |input| try self.feed(&bt, input);
1008210090 return self.finishAirResult(inst, result);
1008310091}
1008410092
1008510093const MoveStrategy = union(enum) {
1008610094 move: Mir.Inst.FixedTag,
10095 x87_load_store,
1008710096 insert_extract: InsertExtract,
1008810097 vex_insert_extract: InsertExtract,
1008910098
......@@ -10091,6 +10100,44 @@ const MoveStrategy = union(enum) {
1009110100 insert: Mir.Inst.FixedTag,
1009210101 extract: Mir.Inst.FixedTag,
1009310102 };
10103
10104 pub fn read(strat: MoveStrategy, self: *Self, dst_reg: Register, src_mem: Memory) !void {
10105 switch (strat) {
10106 .move => |tag| try self.asmRegisterMemory(tag, dst_reg, src_mem),
10107 .x87_load_store => {
10108 try self.asmMemory(.{ .f_, .ld }, src_mem);
10109 try self.asmRegister(.{ .f_p, .st }, @enumFromInt(@intFromEnum(dst_reg) + 1));
10110 },
10111 .insert_extract => |ie| try self.asmRegisterMemoryImmediate(
10112 ie.insert,
10113 dst_reg,
10114 src_mem,
10115 Immediate.u(0),
10116 ),
10117 .vex_insert_extract => |ie| try self.asmRegisterRegisterMemoryImmediate(
10118 ie.insert,
10119 dst_reg,
10120 dst_reg,
10121 src_mem,
10122 Immediate.u(0),
10123 ),
10124 }
10125 }
10126 pub fn write(strat: MoveStrategy, self: *Self, dst_mem: Memory, src_reg: Register) !void {
10127 switch (strat) {
10128 .move => |tag| try self.asmMemoryRegister(tag, dst_mem, src_reg),
10129 .x87_load_store => {
10130 try self.asmRegister(.{ .f_, .ld }, src_reg);
10131 try self.asmMemory(.{ .f_p, .st }, dst_mem);
10132 },
10133 .insert_extract, .vex_insert_extract => |ie| try self.asmMemoryRegisterImmediate(
10134 ie.extract,
10135 dst_mem,
10136 src_reg,
10137 Immediate.u(0),
10138 ),
10139 }
10140 }
1009410141};
1009510142fn moveStrategy(self: *Self, ty: Type, aligned: bool) !MoveStrategy {
1009610143 const mod = self.bin_file.options.module.?;
......@@ -10106,6 +10153,7 @@ fn moveStrategy(self: *Self, ty: Type, aligned: bool) !MoveStrategy {
1010610153 } },
1010710154 32 => return .{ .move = if (self.hasFeature(.avx)) .{ .v_ss, .mov } else .{ ._ss, .mov } },
1010810155 64 => return .{ .move = if (self.hasFeature(.avx)) .{ .v_sd, .mov } else .{ ._sd, .mov } },
10156 80 => return .x87_load_store,
1010910157 128 => return .{ .move = if (self.hasFeature(.avx))
1011010158 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
1011110159 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
......@@ -10370,7 +10418,7 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError
1037010418fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerError!void {
1037110419 const mod = self.bin_file.options.module.?;
1037210420 const abi_size: u32 = @intCast(ty.abiSize(mod));
10373 if (abi_size * 8 > dst_reg.bitSize())
10421 if (ty.bitSize(mod) > dst_reg.bitSize())
1037410422 return self.fail("genSetReg called with a value larger than dst_reg", .{});
1037510423 switch (src_mcv) {
1037610424 .none,
......@@ -10486,9 +10534,21 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
1048610534 .indirect,
1048710535 .load_frame,
1048810536 .lea_frame,
10489 => {
10490 const dst_alias = registerAlias(dst_reg, abi_size);
10491 const src_mem = Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (src_mcv) {
10537 => try @as(MoveStrategy, switch (src_mcv) {
10538 .register_offset => |reg_off| switch (reg_off.off) {
10539 0 => return self.genSetReg(dst_reg, ty, .{ .register = reg_off.reg }),
10540 else => .{ .move = .{ ._, .lea } },
10541 },
10542 .indirect => try self.moveStrategy(ty, false),
10543 .load_frame => |frame_addr| try self.moveStrategy(
10544 ty,
10545 self.getFrameAddrAlignment(frame_addr).compare(.gte, ty.abiAlignment(mod)),
10546 ),
10547 .lea_frame => .{ .move = .{ ._, .lea } },
10548 else => unreachable,
10549 }).read(self, registerAlias(dst_reg, abi_size), Memory.sib(
10550 self.memPtrSize(ty),
10551 switch (src_mcv) {
1049210552 .register_offset, .indirect => |reg_off| .{
1049310553 .base = .{ .reg = reg_off.reg },
1049410554 .disp = reg_off.off,
......@@ -10498,64 +10558,18 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
1049810558 .disp = frame_addr.off,
1049910559 },
1050010560 else => unreachable,
10501 });
10502 switch (@as(MoveStrategy, switch (src_mcv) {
10503 .register_offset => |reg_off| switch (reg_off.off) {
10504 0 => return self.genSetReg(dst_reg, ty, .{ .register = reg_off.reg }),
10505 else => .{ .move = .{ ._, .lea } },
10506 },
10507 .indirect => try self.moveStrategy(ty, false),
10508 .load_frame => |frame_addr| try self.moveStrategy(
10509 ty,
10510 self.getFrameAddrAlignment(frame_addr).compare(.gte, ty.abiAlignment(mod)),
10511 ),
10512 .lea_frame => .{ .move = .{ ._, .lea } },
10513 else => unreachable,
10514 })) {
10515 .move => |tag| try self.asmRegisterMemory(tag, dst_alias, src_mem),
10516 .insert_extract => |ie| try self.asmRegisterMemoryImmediate(
10517 ie.insert,
10518 dst_alias,
10519 src_mem,
10520 Immediate.u(0),
10521 ),
10522 .vex_insert_extract => |ie| try self.asmRegisterRegisterMemoryImmediate(
10523 ie.insert,
10524 dst_alias,
10525 dst_alias,
10526 src_mem,
10527 Immediate.u(0),
10528 ),
10529 }
10530 },
10561 },
10562 )),
1053110563 .memory, .load_direct, .load_got, .load_tlv => {
1053210564 switch (src_mcv) {
10533 .memory => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| {
10534 const dst_alias = registerAlias(dst_reg, abi_size);
10535 const src_mem = Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
10536 .base = .{ .reg = .ds },
10537 .disp = small_addr,
10538 });
10539 switch (try self.moveStrategy(ty, ty.abiAlignment(mod).check(
10540 @as(u32, @bitCast(small_addr)),
10541 ))) {
10542 .move => |tag| try self.asmRegisterMemory(tag, dst_alias, src_mem),
10543 .insert_extract => |ie| try self.asmRegisterMemoryImmediate(
10544 ie.insert,
10545 dst_alias,
10546 src_mem,
10547 Immediate.u(0),
10548 ),
10549 .vex_insert_extract => |ie| try self.asmRegisterRegisterMemoryImmediate(
10550 ie.insert,
10551 dst_alias,
10552 dst_alias,
10553 src_mem,
10554 Immediate.u(0),
10555 ),
10556 }
10557 return;
10558 },
10565 .memory => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr|
10566 return (try self.moveStrategy(
10567 ty,
10568 ty.abiAlignment(mod).check(@as(u32, @bitCast(small_addr))),
10569 )).read(self, registerAlias(dst_reg, abi_size), Memory.sib(
10570 self.memPtrSize(ty),
10571 .{ .base = .{ .reg = .ds }, .disp = small_addr },
10572 )),
1055910573 .load_direct => |sym_index| switch (ty.zigTypeTag(mod)) {
1056010574 else => {
1056110575 const atom_index = try self.owner.getSymbolIndex(self);
......@@ -10582,26 +10596,11 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
1058210596 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
1058310597 defer self.register_manager.unlockReg(addr_lock);
1058410598
10585 const dst_alias = registerAlias(dst_reg, abi_size);
10586 const src_mem = Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
10587 .base = .{ .reg = addr_reg },
10588 });
10589 switch (try self.moveStrategy(ty, false)) {
10590 .move => |tag| try self.asmRegisterMemory(tag, dst_alias, src_mem),
10591 .insert_extract => |ie| try self.asmRegisterMemoryImmediate(
10592 ie.insert,
10593 dst_alias,
10594 src_mem,
10595 Immediate.u(0),
10596 ),
10597 .vex_insert_extract => |ie| try self.asmRegisterRegisterMemoryImmediate(
10598 ie.insert,
10599 dst_alias,
10600 dst_alias,
10601 src_mem,
10602 Immediate.u(0),
10603 ),
10604 }
10599 try (try self.moveStrategy(ty, false)).read(
10600 self,
10601 registerAlias(dst_reg, abi_size),
10602 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .{ .reg = addr_reg } }),
10603 );
1060510604 },
1060610605 .lea_direct, .lea_got => |sym_index| {
1060710606 const atom_index = try self.owner.getSymbolIndex(self);
......@@ -10698,39 +10697,23 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
1069810697 },
1069910698 },
1070010699 .eflags => |cc| try self.asmSetccMemory(Memory.sib(.byte, .{ .base = base, .disp = disp }), cc),
10701 .register => |src_reg| {
10702 const dst_mem = Memory.sib(
10703 Memory.PtrSize.fromSize(abi_size),
10704 .{ .base = base, .disp = disp },
10705 );
10706 const src_alias = registerAlias(src_reg, abi_size);
10707 switch (try self.moveStrategy(ty, switch (base) {
10708 .none => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
10709 .reg => |reg| switch (reg) {
10710 .es, .cs, .ss, .ds => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
10711 else => false,
10712 },
10713 .frame => |frame_index| self.getFrameAddrAlignment(
10714 .{ .index = frame_index, .off = disp },
10715 ).compare(.gte, ty.abiAlignment(mod)),
10716 })) {
10717 .move => |tag| try self.asmMemoryRegister(tag, dst_mem, src_alias),
10718 .insert_extract, .vex_insert_extract => |ie| try self.asmMemoryRegisterImmediate(
10719 ie.extract,
10720 dst_mem,
10721 src_alias,
10722 Immediate.u(0),
10723 ),
10724 }
10725 },
10700 .register => |src_reg| try (try self.moveStrategy(ty, switch (base) {
10701 .none => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
10702 .reg => |reg| switch (reg) {
10703 .es, .cs, .ss, .ds => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
10704 else => false,
10705 },
10706 .frame => |frame_index| self.getFrameAddrAlignment(
10707 .{ .index = frame_index, .off = disp },
10708 ).compare(.gte, ty.abiAlignment(mod)),
10709 })).write(
10710 self,
10711 Memory.sib(self.memPtrSize(ty), .{ .base = base, .disp = disp }),
10712 registerAlias(src_reg, abi_size),
10713 ),
1072610714 .register_pair => |src_regs| for (src_regs, 0..) |src_reg, src_reg_i| {
1072710715 const part_size = @min(abi_size - src_reg_i * 8, 8);
10728 const dst_mem = Memory.sib(
10729 Memory.PtrSize.fromSize(part_size),
10730 .{ .base = base, .disp = disp + @as(i32, @intCast(src_reg_i * 8)) },
10731 );
10732 const src_alias = registerAlias(src_reg, part_size);
10733 switch (try self.moveStrategy(ty, switch (base) {
10716 try (try self.moveStrategy(ty, switch (base) {
1073410717 .none => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
1073510718 .reg => |reg| switch (reg) {
1073610719 .es, .cs, .ss, .ds => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
......@@ -10739,15 +10722,10 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
1073910722 .frame => |frame_index| self.getFrameAddrAlignment(
1074010723 .{ .index = frame_index, .off = disp },
1074110724 ).compare(.gte, ty.abiAlignment(mod)),
10742 })) {
10743 .move => |tag| try self.asmMemoryRegister(tag, dst_mem, src_alias),
10744 .insert_extract, .vex_insert_extract => |ie| try self.asmMemoryRegisterImmediate(
10745 ie.extract,
10746 dst_mem,
10747 src_alias,
10748 Immediate.u(0),
10749 ),
10750 }
10725 })).write(self, Memory.sib(
10726 Memory.PtrSize.fromSize(part_size),
10727 .{ .base = base, .disp = disp + @as(i32, @intCast(src_reg_i * 8)) },
10728 ), registerAlias(src_reg, part_size));
1075110729 },
1075210730 .register_overflow => |ro| {
1075310731 try self.genSetMem(
......@@ -12229,7 +12207,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
1222912207 return self.finishAir(inst, result, buf);
1223012208 }
1223112209 var bt = self.liveness.iterateBigTomb(inst);
12232 for (elements) |elem| self.feed(&bt, elem);
12210 for (elements) |elem| try self.feed(&bt, elem);
1223312211 return self.finishAirResult(inst, result);
1223412212}
1223512213
......@@ -12821,6 +12799,14 @@ fn registerAlias(reg: Register, size_bytes: u32) Register {
1282112799 };
1282212800}
1282312801
12802fn memPtrSize(self: *Self, ty: Type) Memory.PtrSize {
12803 const mod = self.bin_file.options.module.?;
12804 return switch (ty.zigTypeTag(mod)) {
12805 .Float => Memory.PtrSize.fromBitSize(ty.floatBits(self.target.*)),
12806 else => Memory.PtrSize.fromSize(@intCast(ty.abiSize(mod))),
12807 };
12808}
12809
1282412810/// Truncates the value in the register in place.
1282512811/// Clobbers any remaining bits.
1282612812fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
src/arch/x86_64/Encoding.zig+1-1
......@@ -260,7 +260,7 @@ pub const Mnemonic = enum {
260260 ud2,
261261 xadd, xchg, xor,
262262 // X87
263 fisttp, fld,
263 ffree, fisttp, fld, fst, fstp,
264264 // MMX
265265 movd, movq,
266266 packssdw, packsswb, packuswb,
src/arch/x86_64/Mir.zig+9-4
......@@ -342,14 +342,10 @@ pub const Inst = struct {
342342 div,
343343 ///
344344 int3,
345 /// Store integer with truncation
346 istt,
347345 /// Conditional jump
348346 j,
349347 /// Jump
350348 jmp,
351 /// Load floating-point value
352 ld,
353349 /// Load effective address
354350 lea,
355351 /// Load string
......@@ -446,6 +442,15 @@ pub const Inst = struct {
446442 /// Bitwise logical xor of packed double-precision floating-point values
447443 xor,
448444
445 /// Free floating-point register
446 free,
447 /// Store integer with truncation
448 istt,
449 /// Load floating-point value
450 ld,
451 /// Store floating-point value
452 st,
453
449454 /// Pack with signed saturation
450455 ackssw,
451456 /// Pack with signed saturation
src/arch/x86_64/abi.zig+14-12
......@@ -444,7 +444,7 @@ pub const SysV = struct {
444444 /// These registers need to be preserved (saved on the stack) and restored by the caller before
445445 /// the caller relinquishes control to a subroutine via call instruction (or similar).
446446 /// In other words, these registers are free to use by the callee.
447 pub const caller_preserved_regs = [_]Register{ .rax, .rcx, .rdx, .rsi, .rdi, .r8, .r9, .r10, .r11 } ++ sse_avx_regs;
447 pub const caller_preserved_regs = [_]Register{ .rax, .rcx, .rdx, .rsi, .rdi, .r8, .r9, .r10, .r11 } ++ x87_regs ++ sse_avx_regs;
448448
449449 pub const c_abi_int_param_regs = [_]Register{ .rdi, .rsi, .rdx, .rcx, .r8, .r9 };
450450 pub const c_abi_sse_param_regs = sse_avx_regs[0..8].*;
......@@ -459,7 +459,7 @@ pub const Win64 = struct {
459459 /// These registers need to be preserved (saved on the stack) and restored by the caller before
460460 /// the caller relinquishes control to a subroutine via call instruction (or similar).
461461 /// In other words, these registers are free to use by the callee.
462 pub const caller_preserved_regs = [_]Register{ .rax, .rcx, .rdx, .r8, .r9, .r10, .r11 } ++ sse_avx_regs;
462 pub const caller_preserved_regs = [_]Register{ .rax, .rcx, .rdx, .r8, .r9, .r10, .r11 } ++ x87_regs ++ sse_avx_regs;
463463
464464 pub const c_abi_int_param_regs = [_]Register{ .rcx, .rdx, .r8, .r9 };
465465 pub const c_abi_sse_param_regs = sse_avx_regs[0..4].*;
......@@ -531,30 +531,32 @@ pub fn getCAbiSseReturnRegs(cc: std.builtin.CallingConvention) []const Register
531531const gp_regs = [_]Register{
532532 .rax, .rcx, .rdx, .rbx, .rsi, .rdi, .r8, .r9, .r10, .r11, .r12, .r13, .r14, .r15,
533533};
534const x87_regs = [_]Register{
535 .st0, .st1, .st2, .st3, .st4, .st5, .st6, .st7,
536};
534537const sse_avx_regs = [_]Register{
535538 .ymm0, .ymm1, .ymm2, .ymm3, .ymm4, .ymm5, .ymm6, .ymm7,
536539 .ymm8, .ymm9, .ymm10, .ymm11, .ymm12, .ymm13, .ymm14, .ymm15,
537540};
538const allocatable_regs = gp_regs ++ sse_avx_regs;
539pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, &allocatable_regs);
541const allocatable_regs = gp_regs ++ x87_regs[0 .. x87_regs.len - 1] ++ sse_avx_regs;
542pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, allocatable_regs);
540543
541544// Register classes
542545const RegisterBitSet = RegisterManager.RegisterBitSet;
543546pub const RegisterClass = struct {
544547 pub const gp: RegisterBitSet = blk: {
545548 var set = RegisterBitSet.initEmpty();
546 set.setRangeValue(.{
547 .start = 0,
548 .end = gp_regs.len,
549 }, true);
549 for (allocatable_regs, 0..) |reg, index| if (reg.class() == .general_purpose) set.set(index);
550 break :blk set;
551 };
552 pub const x87: RegisterBitSet = blk: {
553 var set = RegisterBitSet.initEmpty();
554 for (allocatable_regs, 0..) |reg, index| if (reg.class() == .x87) set.set(index);
550555 break :blk set;
551556 };
552557 pub const sse: RegisterBitSet = blk: {
553558 var set = RegisterBitSet.initEmpty();
554 set.setRangeValue(.{
555 .start = gp_regs.len,
556 .end = allocatable_regs.len,
557 }, true);
559 for (allocatable_regs, 0..) |reg, index| if (reg.class() == .sse) set.set(index);
558560 break :blk set;
559561 };
560562};
src/arch/x86_64/encodings.zig+14-3
......@@ -829,13 +829,24 @@ pub const table = [_]Entry{
829829 .{ .xor, .rm, &.{ .r64, .rm64 }, &.{ 0x33 }, 0, .long, .none },
830830
831831 // X87
832 .{ .ffree, .o, &.{ .st }, &.{ 0xdd, 0xc0 }, 0, .none, .x87 },
833
832834 .{ .fisttp, .m, &.{ .m16 }, &.{ 0xdf }, 1, .none, .x87 },
833835 .{ .fisttp, .m, &.{ .m32 }, &.{ 0xdb }, 1, .none, .x87 },
834836 .{ .fisttp, .m, &.{ .m64 }, &.{ 0xdd }, 1, .none, .x87 },
835837
836 .{ .fld, .m, &.{ .m32 }, &.{ 0xd9 }, 0, .none, .x87 },
837 .{ .fld, .m, &.{ .m64 }, &.{ 0xdd }, 0, .none, .x87 },
838 .{ .fld, .m, &.{ .m80 }, &.{ 0xdb }, 5, .none, .x87 },
838 .{ .fld, .m, &.{ .m32 }, &.{ 0xd9 }, 0, .none, .x87 },
839 .{ .fld, .m, &.{ .m64 }, &.{ 0xdd }, 0, .none, .x87 },
840 .{ .fld, .m, &.{ .m80 }, &.{ 0xdb }, 5, .none, .x87 },
841 .{ .fld, .o, &.{ .st }, &.{ 0xd9, 0xc0 }, 0, .none, .x87 },
842
843 .{ .fst, .m, &.{ .m32 }, &.{ 0xd9 }, 2, .none, .x87 },
844 .{ .fst, .m, &.{ .m64 }, &.{ 0xdd }, 2, .none, .x87 },
845 .{ .fst, .o, &.{ .st }, &.{ 0xdd, 0xd0 }, 0, .none, .x87 },
846 .{ .fstp, .m, &.{ .m32 }, &.{ 0xd9 }, 3, .none, .x87 },
847 .{ .fstp, .m, &.{ .m64 }, &.{ 0xdd }, 3, .none, .x87 },
848 .{ .fstp, .m, &.{ .m80 }, &.{ 0xdb }, 7, .none, .x87 },
849 .{ .fstp, .o, &.{ .st }, &.{ 0xdd, 0xd8 }, 0, .none, .x87 },
839850
840851 // SSE
841852 .{ .addps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x58 }, 0, .none, .sse },
test/behavior/cast.zig+2-2
......@@ -118,11 +118,11 @@ test "@floatFromInt" {
118118
119119test "@floatFromInt(f80)" {
120120 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
121 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
122121 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
123122 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
124123 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
125124 if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArmOrThumb()) return error.SkipZigTest;
125 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
126126
127127 const S = struct {
128128 fn doTheTest(comptime Int: type) !void {
......@@ -1476,9 +1476,9 @@ test "pointer to empty struct literal to mutable slice" {
14761476test "coerce between pointers of compatible differently-named floats" {
14771477 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
14781478 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1479 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
14801479 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
14811480 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
1481 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
14821482
14831483 if (builtin.os.tag == .windows) {
14841484 // https://github.com/ziglang/zig/issues/12396
test/behavior/floatop.zig+3-3
......@@ -28,7 +28,7 @@ test "add f32/f64" {
2828}
2929
3030test "add f80/f128/c_longdouble" {
31 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
31 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
3232
3333 try testAdd(f80);
3434 try comptime testAdd(f80);
......@@ -59,7 +59,7 @@ test "sub f32/f64" {
5959}
6060
6161test "sub f80/f128/c_longdouble" {
62 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
62 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
6363
6464 try testSub(f80);
6565 try comptime testSub(f80);
......@@ -90,7 +90,7 @@ test "mul f32/f64" {
9090}
9191
9292test "mul f80/f128/c_longdouble" {
93 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
93 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
9494
9595 try testMul(f80);
9696 try comptime testMul(f80);
test/behavior/math.zig-1
......@@ -1468,7 +1468,6 @@ test "@round f32/f64" {
14681468test "@round f80" {
14691469 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
14701470 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1471 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
14721471 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
14731472 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
14741473 if (builtin.zig_backend == .stage2_c and comptime builtin.cpu.arch.isArmOrThumb()) return error.SkipZigTest;
test/behavior/widening.zig+1-1
......@@ -40,11 +40,11 @@ test "implicit unsigned integer to signed integer" {
4040}
4141
4242test "float widening" {
43 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
4443 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
4544 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
4645 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
4746 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
47 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf) return error.SkipZigTest;
4848
4949 var a: f16 = 12.34;
5050 var b: f32 = a;