authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-05-10 09:27:14+02:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-05-10 16:43:44+02:00
logef9e3fb2b6cabb711b3421250b033f1696f961ae
tree046561227f88b591386258c95c0b4295b20a8321
parent85ca14e35a6be6591faaac42a4c95cf183ef4c3b

x64: migrate div to genMulDivBinOp

1 files changed, 192 insertions(+), 163 deletions(-)

src/arch/x86_64/CodeGen.zig+192-163
......@@ -592,11 +592,11 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
592592 .sub => try self.airBinOp(inst),
593593 .subwrap => try self.airBinOp(inst),
594594 .sub_sat => try self.airSubSat(inst),
595 .mul => try self.airBinOp(inst),
596 .mulwrap => try self.airBinOp(inst),
595 .mul => try self.airMulDivBinOp(inst),
596 .mulwrap => try self.airMulDivBinOp(inst),
597597 .mul_sat => try self.airMulSat(inst),
598 .rem => try self.airBinOp(inst),
599 .mod => try self.airBinOp(inst),
598 .rem => try self.airMulDivBinOp(inst),
599 .mod => try self.airMulDivBinOp(inst),
600600 .shl, .shl_exact => try self.airShl(inst),
601601 .shl_sat => try self.airShlSat(inst),
602602 .min => try self.airMin(inst),
......@@ -626,7 +626,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
626626 .mul_with_overflow => try self.airMulWithOverflow(inst),
627627 .shl_with_overflow => try self.airShlWithOverflow(inst),
628628
629 .div_float, .div_trunc, .div_floor, .div_exact => try self.airDiv(inst),
629 .div_float, .div_trunc, .div_floor, .div_exact => try self.airMulDivBinOp(inst),
630630
631631 .cmp_lt => try self.airCmp(inst, .lt),
632632 .cmp_lte => try self.airCmp(inst, .lte),
......@@ -959,6 +959,12 @@ pub fn spillCompareFlagsIfOccupied(self: *Self) !void {
959959 }
960960}
961961
962pub fn spillRegisters(self: *Self, comptime count: comptime_int, registers: [count]Register) !void {
963 for (registers) |reg| {
964 try self.register_manager.getReg(reg, null);
965 }
966}
967
962968/// Copies a value to a register without tracking the register. The register is not considered
963969/// allocated. A second call to `copyToTmpRegister` may return the same register.
964970/// This can have a side effect of spilling instructions to the stack to free up a register.
......@@ -1252,6 +1258,26 @@ fn airBinOp(self: *Self, inst: Air.Inst.Index) !void {
12521258 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
12531259}
12541260
1261fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
1262 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1263
1264 if (self.liveness.isUnused(inst)) {
1265 return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
1266 }
1267
1268 const tag = self.air.instructions.items(.tag)[inst];
1269 const ty = self.air.typeOfIndex(inst);
1270
1271 try self.spillRegisters(2, .{ .rax, .rdx });
1272
1273 const lhs = try self.resolveInst(bin_op.lhs);
1274 const rhs = try self.resolveInst(bin_op.rhs);
1275
1276 const result = try self.genMulDivBinOp(tag, inst, ty, lhs, rhs);
1277
1278 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1279}
1280
12551281fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
12561282 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
12571283 const result: MCValue = if (self.liveness.isUnused(inst))
......@@ -1339,25 +1365,31 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
13391365 return self.fail("TODO implement mul_with_overflow for Ints larger than 64bits", .{});
13401366 }
13411367
1342 try self.spillCompareFlagsIfOccupied();
1343
1344 const lhs = try self.resolveInst(bin_op.lhs);
1345 const rhs = try self.resolveInst(bin_op.rhs);
1346
13471368 if (math.isPowerOfTwo(int_info.bits)) {
1369 try self.spillCompareFlagsIfOccupied();
13481370 self.compare_flags_inst = inst;
1349 const partial = try self.genBinOp(.mul, null, lhs, rhs, ty, ty);
1371
1372 try self.spillRegisters(2, .{ .rax, .rdx });
1373
1374 const lhs = try self.resolveInst(bin_op.lhs);
1375 const rhs = try self.resolveInst(bin_op.rhs);
1376
1377 const partial = try self.genMulDivBinOp(.mul, null, ty, lhs, rhs);
13501378 break :result switch (int_info.signedness) {
13511379 .signed => MCValue{ .register_overflow_signed = partial.register },
13521380 .unsigned => MCValue{ .register_overflow_unsigned = partial.register },
13531381 };
13541382 }
13551383
1384 try self.spillCompareFlagsIfOccupied();
13561385 self.compare_flags_inst = null;
13571386
13581387 const dst_reg: Register = dst_reg: {
13591388 switch (int_info.signedness) {
13601389 .signed => {
1390 const lhs = try self.resolveInst(bin_op.lhs);
1391 const rhs = try self.resolveInst(bin_op.rhs);
1392
13611393 const rhs_lock: ?RegisterLock = switch (rhs) {
13621394 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
13631395 else => null,
......@@ -1386,7 +1418,12 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
13861418 break :dst_reg dst_reg;
13871419 },
13881420 .unsigned => {
1389 const dst_mcv = try self.genBinOp(.mul, null, lhs, rhs, ty, ty);
1421 try self.spillRegisters(2, .{ .rax, .rdx });
1422
1423 const lhs = try self.resolveInst(bin_op.lhs);
1424 const rhs = try self.resolveInst(bin_op.rhs);
1425
1426 const dst_mcv = try self.genMulDivBinOp(.mul, null, ty, lhs, rhs);
13901427 break :dst_reg dst_mcv.register;
13911428 },
13921429 }
......@@ -1487,7 +1524,13 @@ fn genIntMulDivOpMir(
14871524 return self.fail("TODO implement genIntMulDivOpMir for ABI size larger than 8", .{});
14881525 }
14891526
1490 try self.genSetReg(ty, .rax, lhs);
1527 lhs: {
1528 switch (lhs) {
1529 .register => |reg| if (reg.to64() == .rax) break :lhs,
1530 else => {},
1531 }
1532 try self.genSetReg(ty, .rax, lhs);
1533 }
14911534
14921535 switch (signedness) {
14931536 .signed => {
......@@ -1609,96 +1652,6 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
16091652 return MCValue{ .register = divisor };
16101653}
16111654
1612fn airDiv(self: *Self, inst: Air.Inst.Index) !void {
1613 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1614
1615 if (self.liveness.isUnused(inst)) {
1616 return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none });
1617 }
1618
1619 const tag = self.air.instructions.items(.tag)[inst];
1620 const ty = self.air.typeOfIndex(inst);
1621
1622 if (ty.zigTypeTag() != .Int) {
1623 return self.fail("TODO implement {} for operands of dst type {}", .{ tag, ty.zigTypeTag() });
1624 }
1625
1626 if (tag == .div_float) {
1627 return self.fail("TODO implement {}", .{tag});
1628 }
1629
1630 const signedness = ty.intInfo(self.target.*).signedness;
1631
1632 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
1633 const track_rax: ?Air.Inst.Index = blk: {
1634 if (signedness == .unsigned) break :blk inst;
1635 switch (tag) {
1636 .div_exact, .div_trunc => break :blk inst,
1637 else => break :blk null,
1638 }
1639 };
1640 try self.register_manager.getReg(.rax, track_rax);
1641 try self.register_manager.getReg(.rdx, null);
1642 const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx });
1643 defer for (reg_locks) |reg| {
1644 self.register_manager.unlockReg(reg);
1645 };
1646
1647 const lhs = try self.resolveInst(bin_op.lhs);
1648 const lhs_lock: ?RegisterLock = switch (lhs) {
1649 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1650 else => null,
1651 };
1652 defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
1653
1654 const rhs: MCValue = blk: {
1655 const rhs = try self.resolveInst(bin_op.rhs);
1656 if (signedness == .signed) {
1657 switch (tag) {
1658 .div_floor => {
1659 const rhs_lock: ?RegisterLock = switch (rhs) {
1660 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1661 else => null,
1662 };
1663 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
1664
1665 break :blk try self.copyToRegisterWithInstTracking(inst, ty, rhs);
1666 },
1667 else => {},
1668 }
1669 }
1670 break :blk rhs;
1671 };
1672 const rhs_lock: ?RegisterLock = switch (rhs) {
1673 .register => |reg| self.register_manager.lockReg(reg),
1674 else => null,
1675 };
1676 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
1677
1678 const result: MCValue = result: {
1679 if (signedness == .unsigned) {
1680 try self.genIntMulDivOpMir(.div, ty, signedness, lhs, rhs);
1681 break :result MCValue{ .register = .rax };
1682 }
1683
1684 switch (tag) {
1685 .div_exact, .div_trunc => {
1686 try self.genIntMulDivOpMir(switch (signedness) {
1687 .signed => .idiv,
1688 .unsigned => .div,
1689 }, ty, signedness, lhs, rhs);
1690 break :result MCValue{ .register = .rax };
1691 },
1692 .div_floor => {
1693 break :result try self.genInlineIntDivFloor(ty, lhs, rhs);
1694 },
1695 else => unreachable,
1696 }
1697 };
1698
1699 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1700}
1701
17021655fn airShl(self: *Self, inst: Air.Inst.Index) !void {
17031656 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
17041657 if (self.liveness.isUnused(inst)) {
......@@ -3144,110 +3097,84 @@ fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void {
31443097}
31453098
31463099/// Result is always a register.
3147fn genBinOp(
3100/// Clobbers .rax and .rdx therefore care needs to be taken to spill operands upfront.
3101/// Asserts .rax and .rdx are free.
3102fn genMulDivBinOp(
31483103 self: *Self,
31493104 tag: Air.Inst.Tag,
31503105 maybe_inst: ?Air.Inst.Index,
3106 ty: Type,
31513107 lhs: MCValue,
31523108 rhs: MCValue,
3153 lhs_ty: Type,
3154 rhs_ty: Type,
31553109) !MCValue {
3156 const is_commutative: bool = switch (tag) {
3157 .add,
3158 .addwrap,
3159 .bool_or,
3160 .bit_or,
3161 .bool_and,
3162 .bit_and,
3163 .xor,
3164 => true,
3165
3166 .sub,
3167 .subwrap,
3168 .mul,
3169 .mulwrap,
3170 .div_exact,
3171 .div_trunc,
3172 .rem,
3173 .mod,
3174 .shl,
3175 .shr,
3176 .ptr_add,
3177 .ptr_sub,
3178 => false,
3110 if (ty.zigTypeTag() == .Vector or ty.zigTypeTag() == .Float) {
3111 return self.fail("TODO implement genBinOp for {}", .{ty.fmtDebug()});
3112 }
3113 if (ty.abiSize(self.target.*) > 8) {
3114 return self.fail("TODO implement genBinOp for {}", .{ty.fmtDebug()});
3115 }
3116 if (tag == .div_float) {
3117 return self.fail("TODO implement genMulDivBinOp for div_float", .{});
3118 }
31793119
3180 .div_float => return self.fail("TODO implement genBinOp for {}", .{tag}),
3120 assert(self.register_manager.isRegFree(.rax));
3121 assert(self.register_manager.isRegFree(.rdx));
31813122
3182 else => unreachable,
3123 const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx });
3124 defer for (reg_locks) |reg| {
3125 self.register_manager.unlockReg(reg);
31833126 };
31843127
3185 if (lhs_ty.zigTypeTag() == .Vector or lhs_ty.zigTypeTag() == .Float) {
3186 return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmtDebug()});
3187 }
3188 if (lhs_ty.abiSize(self.target.*) > 8) {
3189 return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmtDebug()});
3190 }
3128 const int_info = ty.intInfo(self.target.*);
3129 const signedness = int_info.signedness;
31913130
31923131 switch (tag) {
31933132 .mul,
31943133 .mulwrap,
31953134 => {
3196 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
31973135 try self.register_manager.getReg(.rax, maybe_inst);
31983136 try self.register_manager.getReg(.rdx, null);
3199 const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx });
3200 defer for (reg_locks) |reg| {
3201 self.register_manager.unlockReg(reg);
3202 };
32033137
3204 const int_info = lhs_ty.intInfo(self.target.*);
3205 try self.genIntMulDivOpMir(switch (int_info.signedness) {
3138 try self.genIntMulDivOpMir(switch (signedness) {
32063139 .signed => .imul,
32073140 .unsigned => .mul,
3208 }, lhs_ty, int_info.signedness, lhs, rhs);
3141 }, ty, signedness, lhs, rhs);
32093142
3210 return switch (int_info.signedness) {
3143 return switch (signedness) {
32113144 .signed => MCValue{ .register = .rax },
3212 .unsigned => MCValue{ .register = registerAlias(.rax, @intCast(u32, lhs_ty.abiSize(self.target.*))) },
3145 .unsigned => MCValue{ .register = registerAlias(.rax, @intCast(u32, ty.abiSize(self.target.*))) },
32133146 };
32143147 },
32153148 .mod,
32163149 .rem,
32173150 => {
3218 const int_info = lhs_ty.intInfo(self.target.*);
32193151 const track_inst_rdx: ?Air.Inst.Index = switch (tag) {
3220 .mod => if (int_info.signedness == .unsigned) maybe_inst else null,
3152 .mod => if (signedness == .unsigned) maybe_inst else null,
32213153 .rem => maybe_inst,
32223154 else => unreachable,
32233155 };
32243156
3225 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
32263157 try self.register_manager.getReg(.rax, null);
32273158 try self.register_manager.getReg(.rdx, track_inst_rdx);
3228 const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx });
3229 defer for (reg_locks) |reg| {
3230 self.register_manager.unlockReg(reg);
3231 };
32323159
3233 switch (int_info.signedness) {
3160 switch (signedness) {
32343161 .signed => {
32353162 switch (tag) {
32363163 .rem => {
3237 try self.genIntMulDivOpMir(.idiv, lhs_ty, .signed, lhs, rhs);
3164 try self.genIntMulDivOpMir(.idiv, ty, .signed, lhs, rhs);
32383165 return MCValue{ .register = .rdx };
32393166 },
32403167 .mod => {
3241 const div_floor = try self.genInlineIntDivFloor(lhs_ty, lhs, rhs);
3242 try self.genIntMulComplexOpMir(lhs_ty, div_floor, rhs);
3168 const div_floor = try self.genInlineIntDivFloor(ty, lhs, rhs);
3169 try self.genIntMulComplexOpMir(ty, div_floor, rhs);
32433170 const div_floor_lock = self.register_manager.lockReg(div_floor.register);
32443171 defer if (div_floor_lock) |lock| self.register_manager.unlockReg(lock);
32453172
32463173 const result: MCValue = if (maybe_inst) |inst|
3247 try self.copyToRegisterWithInstTracking(inst, lhs_ty, lhs)
3174 try self.copyToRegisterWithInstTracking(inst, ty, lhs)
32483175 else
3249 MCValue{ .register = try self.copyToTmpRegister(lhs_ty, lhs) };
3250 try self.genBinOpMir(.sub, lhs_ty, result, div_floor);
3176 MCValue{ .register = try self.copyToTmpRegister(ty, lhs) };
3177 try self.genBinOpMir(.sub, ty, result, div_floor);
32513178
32523179 return result;
32533180 },
......@@ -3255,14 +3182,114 @@ fn genBinOp(
32553182 }
32563183 },
32573184 .unsigned => {
3258 try self.genIntMulDivOpMir(.div, lhs_ty, .unsigned, lhs, rhs);
3185 try self.genIntMulDivOpMir(.div, ty, .unsigned, lhs, rhs);
32593186 return MCValue{ .register = .rdx };
32603187 },
32613188 }
32623189 },
3263 else => {},
3190 .div_trunc,
3191 .div_floor,
3192 .div_exact,
3193 => {
3194 const track_inst_rax: ?Air.Inst.Index = blk: {
3195 if (signedness == .unsigned) break :blk maybe_inst;
3196 switch (tag) {
3197 .div_exact, .div_trunc => break :blk maybe_inst,
3198 else => break :blk null,
3199 }
3200 };
3201
3202 try self.register_manager.getReg(.rax, track_inst_rax);
3203 try self.register_manager.getReg(.rdx, null);
3204
3205 const lhs_lock: ?RegisterLock = switch (lhs) {
3206 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
3207 else => null,
3208 };
3209 defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
3210
3211 const actual_rhs: MCValue = blk: {
3212 if (signedness == .signed) {
3213 switch (tag) {
3214 .div_floor => {
3215 const rhs_lock: ?RegisterLock = switch (rhs) {
3216 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
3217 else => null,
3218 };
3219 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
3220
3221 if (maybe_inst) |inst| {
3222 break :blk try self.copyToRegisterWithInstTracking(inst, ty, rhs);
3223 }
3224 break :blk MCValue{ .register = try self.copyToTmpRegister(ty, rhs) };
3225 },
3226 else => {},
3227 }
3228 }
3229 break :blk rhs;
3230 };
3231 const rhs_lock: ?RegisterLock = switch (actual_rhs) {
3232 .register => |reg| self.register_manager.lockReg(reg),
3233 else => null,
3234 };
3235 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
3236
3237 const result: MCValue = result: {
3238 if (signedness == .unsigned) {
3239 try self.genIntMulDivOpMir(.div, ty, signedness, lhs, actual_rhs);
3240 break :result MCValue{ .register = .rax };
3241 }
3242
3243 switch (tag) {
3244 .div_exact, .div_trunc => {
3245 try self.genIntMulDivOpMir(switch (signedness) {
3246 .signed => .idiv,
3247 .unsigned => .div,
3248 }, ty, signedness, lhs, actual_rhs);
3249 break :result MCValue{ .register = .rax };
3250 },
3251 .div_floor => {
3252 break :result try self.genInlineIntDivFloor(ty, lhs, actual_rhs);
3253 },
3254 else => unreachable,
3255 }
3256 };
3257 return result;
3258 },
3259 else => unreachable,
3260 }
3261}
3262
3263/// Result is always a register.
3264fn genBinOp(
3265 self: *Self,
3266 tag: Air.Inst.Tag,
3267 maybe_inst: ?Air.Inst.Index,
3268 lhs: MCValue,
3269 rhs: MCValue,
3270 lhs_ty: Type,
3271 rhs_ty: Type,
3272) !MCValue {
3273 if (lhs_ty.zigTypeTag() == .Vector or lhs_ty.zigTypeTag() == .Float) {
3274 return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmtDebug()});
3275 }
3276 if (lhs_ty.abiSize(self.target.*) > 8) {
3277 return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmtDebug()});
32643278 }
32653279
3280 const is_commutative: bool = switch (tag) {
3281 .add,
3282 .addwrap,
3283 .bool_or,
3284 .bit_or,
3285 .bool_and,
3286 .bit_and,
3287 .xor,
3288 => true,
3289
3290 else => false,
3291 };
3292
32663293 const lhs_lock: ?RegisterLock = switch (lhs) {
32673294 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
32683295 else => null,
......@@ -5460,6 +5487,7 @@ fn genInlineMemcpy(
54605487 len: MCValue,
54615488 opts: InlineMemcpyOpts,
54625489) InnerError!void {
5490 // TODO preserve contents of .rax and .rcx if not free, and then restore
54635491 try self.register_manager.getReg(.rax, null);
54645492 try self.register_manager.getReg(.rcx, null);
54655493
......@@ -5657,6 +5685,7 @@ fn genInlineMemset(
56575685 len: MCValue,
56585686 opts: InlineMemcpyOpts,
56595687) InnerError!void {
5688 // TODO preserve contents of .rax and then restore
56605689 try self.register_manager.getReg(.rax, null);
56615690 const rax_lock = self.register_manager.lockRegAssumeUnused(.rax);
56625691 defer self.register_manager.unlockReg(rax_lock);