authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-02-18 18:23:50+01:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-02-19 14:24:03+01:00
logda86839af085bb5921d7f0112fda5ad9536dda10
tree90deb1620584a934b6af15de9450a1c49c42e26b
parentbd396d7e076c09d162f679c7d26ba883dbe5d7cf

x64: clean up implementation of divs, mod, rem for integers

3 files changed, 195 insertions(+), 69 deletions(-)

src/arch/x86_64/CodeGen.zig+163-68
...@@ -1252,52 +1252,38 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -1252,52 +1252,38 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
1252 return self.fail("TODO implement airShlWithOverflow for {}", .{self.target.cpu.arch});1252 return self.fail("TODO implement airShlWithOverflow for {}", .{self.target.cpu.arch});
1253}1253}
12541254
1255/// Perform signed and unsigned integer division.1255/// Generates signed or unsigned integer division.
1256/// TODO it might be wise to split some functionality into integer and floating-point1256/// Requires use of .rax and .rdx registers. Spills them if necessary.
1257/// specialised functions.1257/// Quotient is saved in .rax and remainder in .rdx.
1258/// Supports AIR tag:1258fn genIntDivOpMir(
1259/// .div_exact, .div_trunc, .div_floor, .mod, .rem1259 self: *Self,
1260fn genDivOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air.Inst.Ref) !MCValue {1260 ty: Type,
1261 const dst_ty = self.air.typeOfIndex(inst);1261 signedness: std.builtin.Signedness,
1262 const tag = self.air.instructions.items(.tag)[inst];1262 lhs: MCValue,
12631263 rhs: MCValue,
1264 switch (tag) {1264) !void {
1265 .div_exact, .div_trunc, .div_floor, .mod, .rem => {},1265 const abi_size = @intCast(u32, ty.abiSize(self.target.*));
1266 .div_float => return self.fail("TODO implement genDivOp for {}", .{tag}),1266 if (abi_size > 8) {
1267 else => unreachable,1267 return self.fail("TODO implement genIntDivOpMir for ABI size larger than 8", .{});
1268 }
1269
1270 if (dst_ty.zigTypeTag() != .Int) {
1271 return self.fail("TODO implement {} for operands of type {}", .{ tag, dst_ty.zigTypeTag() });
1272 }
1273 if (dst_ty.abiSize(self.target.*) > 8) {
1274 return self.fail("TODO implement {} for ABI size larger than 8", .{tag});
1275 }
1276
1277 const signedness = dst_ty.intInfo(self.target.*).signedness;
1278 const tmp_ty = switch (signedness) {
1279 .signed => Type.isize,
1280 .unsigned => dst_ty,
1281 };
1282 const abi_size = @intCast(u32, tmp_ty.abiSize(self.target.*));
1283
1284 const lhs = try self.resolveInst(op_lhs);
1285 blk: {
1286 switch (lhs) {
1287 .register => |reg| {
1288 if (reg.to64() == .rax) break :blk;
1289 },
1290 else => {},
1291 }
1292 try self.register_manager.getReg(.rax, inst); // track inst -> rax in register manager
1293 try self.genSetReg(tmp_ty, .rax, lhs);
1294 }1268 }
12951269
1270 try self.register_manager.getReg(.rax, null);
1296 try self.register_manager.getReg(.rdx, null);1271 try self.register_manager.getReg(.rdx, null);
1297 self.register_manager.freezeRegs(&.{ .rax, .rdx });1272 self.register_manager.freezeRegs(&.{ .rax, .rdx });
1298 defer self.register_manager.unfreezeRegs(&.{ .rax, .rdx });1273 defer self.register_manager.unfreezeRegs(&.{ .rax, .rdx });
12991274
1300 // Prep rdx for the op1275 const dividend = switch (lhs) {
1276 .register => lhs,
1277 else => blk: {
1278 const reg = try self.copyToTmpRegister(ty, lhs);
1279 break :blk MCValue{ .register = reg };
1280 },
1281 };
1282 try self.genSetReg(ty, .rax, dividend);
1283
1284 self.register_manager.freezeRegs(&.{dividend.register});
1285 defer self.register_manager.unfreezeRegs(&.{dividend.register});
1286
1301 switch (signedness) {1287 switch (signedness) {
1302 .signed => {1288 .signed => {
1303 _ = try self.addInst(.{1289 _ = try self.addInst(.{
...@@ -1320,15 +1306,12 @@ fn genDivOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air...@@ -1320,15 +1306,12 @@ fn genDivOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air
1320 },1306 },
1321 }1307 }
13221308
1323 const rhs = try self.resolveInst(op_rhs);1309 const divisor = switch (rhs) {
1324 const divisor = blk: {1310 .register => rhs,
1325 switch (rhs) {1311 else => blk: {
1326 .register, .stack_offset => break :blk rhs,1312 const reg = try self.copyToTmpRegister(ty, rhs);
1327 else => {1313 break :blk MCValue{ .register = reg };
1328 const reg = try self.copyToTmpRegister(tmp_ty, rhs);1314 },
1329 break :blk MCValue{ .register = reg };
1330 },
1331 }
1332 };1315 };
1333 const op_tag: Mir.Inst.Tag = switch (signedness) {1316 const op_tag: Mir.Inst.Tag = switch (signedness) {
1334 .signed => .idiv,1317 .signed => .idiv,
...@@ -1340,7 +1323,7 @@ fn genDivOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air...@@ -1340,7 +1323,7 @@ fn genDivOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air
1340 _ = try self.addInst(.{1323 _ = try self.addInst(.{
1341 .tag = op_tag,1324 .tag = op_tag,
1342 .ops = (Mir.Ops{1325 .ops = (Mir.Ops{
1343 .reg1 = registerAlias(reg, abi_size),1326 .reg1 = reg,
1344 }).encode(),1327 }).encode(),
1345 .data = undefined,1328 .data = undefined,
1346 });1329 });
...@@ -1363,38 +1346,150 @@ fn genDivOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air...@@ -1363,38 +1346,150 @@ fn genDivOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air
1363 },1346 },
1364 else => unreachable,1347 else => unreachable,
1365 }1348 }
1349}
13661350
1367 return switch (tag) {1351fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCValue {
1368 .mod, .div_exact, .div_trunc, .div_floor => MCValue{ .register = .rax },1352 const signedness = ty.intInfo(self.target.*).signedness;
1369 .rem => MCValue{ .register = .rdx },1353 const dividend = switch (lhs) {
1370 else => unreachable,1354 .register => |reg| reg,
1355 else => try self.copyToTmpRegister(ty, lhs),
1371 };1356 };
1357 self.register_manager.freezeRegs(&.{dividend});
1358
1359 const divisor = switch (rhs) {
1360 .register => |reg| reg,
1361 else => try self.copyToTmpRegister(ty, rhs),
1362 };
1363 self.register_manager.freezeRegs(&.{divisor});
1364 defer self.register_manager.unfreezeRegs(&.{ dividend, divisor });
1365
1366 try self.genIntDivOpMir(Type.isize, signedness, .{ .register = dividend }, .{ .register = divisor });
1367
1368 _ = try self.addInst(.{
1369 .tag = .xor,
1370 .ops = (Mir.Ops{
1371 .reg1 = divisor.to64(),
1372 .reg2 = dividend.to64(),
1373 }).encode(),
1374 .data = undefined,
1375 });
1376 _ = try self.addInst(.{
1377 .tag = .sar,
1378 .ops = (Mir.Ops{
1379 .reg1 = divisor.to64(),
1380 .flags = 0b10,
1381 }).encode(),
1382 .data = .{ .imm = 63 },
1383 });
1384 _ = try self.addInst(.{
1385 .tag = .@"test",
1386 .ops = (Mir.Ops{
1387 .reg1 = .rdx,
1388 .reg2 = .rdx,
1389 }).encode(),
1390 .data = undefined,
1391 });
1392 _ = try self.addInst(.{
1393 .tag = .cond_mov_eq,
1394 .ops = (Mir.Ops{
1395 .reg1 = divisor.to64(),
1396 .reg2 = .rdx,
1397 }).encode(),
1398 .data = undefined,
1399 });
1400 try self.genBinMathOpMir(.add, Type.isize, .{ .register = divisor.to64() }, .{ .register = .rax });
1401 return MCValue{ .register = divisor };
1372}1402}
13731403
1374fn airDiv(self: *Self, inst: Air.Inst.Index) !void {1404fn airDiv(self: *Self, inst: Air.Inst.Index) !void {
1375 const bin_op = self.air.instructions.items(.data)[inst].bin_op;1405 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1376 const result: MCValue = if (self.liveness.isUnused(inst))1406 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1377 .dead1407 const tag = self.air.instructions.items(.tag)[inst];
1378 else1408 const ty = self.air.typeOfIndex(inst);
1379 try self.genDivOp(inst, bin_op.lhs, bin_op.rhs);1409
1410 if (ty.zigTypeTag() != .Int) {
1411 return self.fail("TODO implement {} for operands of dst type {}", .{ tag, ty.zigTypeTag() });
1412 }
1413
1414 if (tag == .div_float) {
1415 return self.fail("TODO implement {}", .{tag});
1416 }
1417
1418 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
1419 try self.register_manager.getReg(.rax, null);
1420 try self.register_manager.getReg(.rdx, null);
1421
1422 const lhs = try self.resolveInst(bin_op.lhs);
1423 const rhs = try self.resolveInst(bin_op.rhs);
1424
1425 const signedness = ty.intInfo(self.target.*).signedness;
1426 if (signedness == .unsigned) {
1427 try self.genIntDivOpMir(ty, signedness, lhs, rhs);
1428 break :result MCValue{ .register = .rax };
1429 }
1430
1431 switch (tag) {
1432 .div_exact, .div_trunc => {
1433 try self.genIntDivOpMir(ty, signedness, lhs, rhs);
1434 break :result MCValue{ .register = .rax };
1435 },
1436 .div_floor => {
1437 break :result try self.genInlineIntDivFloor(ty, lhs, rhs);
1438 },
1439 else => unreachable,
1440 }
1441 };
1380 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });1442 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1381}1443}
13821444
1383fn airRem(self: *Self, inst: Air.Inst.Index) !void {1445fn airRem(self: *Self, inst: Air.Inst.Index) !void {
1384 const bin_op = self.air.instructions.items(.data)[inst].bin_op;1446 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1385 const result: MCValue = if (self.liveness.isUnused(inst))1447 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1386 .dead1448 const ty = self.air.typeOfIndex(inst);
1387 else1449 if (ty.zigTypeTag() != .Int) {
1388 try self.genDivOp(inst, bin_op.lhs, bin_op.rhs);1450 return self.fail("TODO implement .rem for operands of dst type {}", .{ty.zigTypeTag()});
1451 }
1452 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
1453 try self.register_manager.getReg(.rax, null);
1454 try self.register_manager.getReg(.rdx, null);
1455 const lhs = try self.resolveInst(bin_op.lhs);
1456 const rhs = try self.resolveInst(bin_op.rhs);
1457 const signedness = ty.intInfo(self.target.*).signedness;
1458 try self.genIntDivOpMir(ty, signedness, lhs, rhs);
1459 break :result MCValue{ .register = .rdx };
1460 };
1389 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });1461 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1390}1462}
13911463
1392fn airMod(self: *Self, inst: Air.Inst.Index) !void {1464fn airMod(self: *Self, inst: Air.Inst.Index) !void {
1393 const bin_op = self.air.instructions.items(.data)[inst].bin_op;1465 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1394 const result: MCValue = if (self.liveness.isUnused(inst))1466 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1395 .dead1467 const ty = self.air.typeOfIndex(inst);
1396 else1468 if (ty.zigTypeTag() != .Int) {
1397 try self.genDivOp(inst, bin_op.lhs, bin_op.rhs);1469 return self.fail("TODO implement .mod for operands of dst type {}", .{ty.zigTypeTag()});
1470 }
1471 // Spill .rax and .rdx upfront to ensure we don't spill the operands too late.
1472 try self.register_manager.getReg(.rax, null);
1473 try self.register_manager.getReg(.rdx, null);
1474 const lhs = try self.resolveInst(bin_op.lhs);
1475 const rhs = try self.resolveInst(bin_op.rhs);
1476 const signedness = ty.intInfo(self.target.*).signedness;
1477 switch (signedness) {
1478 .unsigned => {
1479 try self.genIntDivOpMir(ty, signedness, lhs, rhs);
1480 break :result MCValue{ .register = .rdx };
1481 },
1482 .signed => {
1483 const div_floor = try self.genInlineIntDivFloor(ty, lhs, rhs);
1484 try self.genIMulOpMir(ty, div_floor, rhs);
1485
1486 const reg = try self.copyToTmpRegister(ty, lhs);
1487 try self.genBinMathOpMir(.sub, ty, .{ .register = reg }, div_floor);
1488
1489 break :result MCValue{ .register = reg };
1490 },
1491 }
1492 };
1398 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });1493 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1399}1494}
14001495
...@@ -4368,7 +4463,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void...@@ -4368,7 +4463,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
4368 .tag = .mov_sign_extend,4463 .tag = .mov_sign_extend,
4369 .ops = (Mir.Ops{4464 .ops = (Mir.Ops{
4370 .reg1 = reg.to64(),4465 .reg1 = reg.to64(),
4371 .reg2 = src_reg,4466 .reg2 = registerAlias(src_reg, abi_size),
4372 }).encode(),4467 }).encode(),
4373 .data = undefined,4468 .data = undefined,
4374 });4469 });
...@@ -4379,7 +4474,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void...@@ -4379,7 +4474,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
4379 .tag = .mov_zero_extend,4474 .tag = .mov_zero_extend,
4380 .ops = (Mir.Ops{4475 .ops = (Mir.Ops{
4381 .reg1 = reg.to64(),4476 .reg1 = reg.to64(),
4382 .reg2 = src_reg,4477 .reg2 = registerAlias(src_reg, abi_size),
4383 }).encode(),4478 }).encode(),
4384 .data = undefined,4479 .data = undefined,
4385 });4480 });
src/arch/x86_64/Emit.zig+25-1
...@@ -161,6 +161,8 @@ pub fn lowerMir(emit: *Emit) InnerError!void {...@@ -161,6 +161,8 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
161 .cond_set_byte_eq_ne,161 .cond_set_byte_eq_ne,
162 => try emit.mirCondSetByte(tag, inst),162 => try emit.mirCondSetByte(tag, inst),
163163
164 .cond_mov_eq => try emit.mirCondMov(.cmove, inst),
165
164 .ret => try emit.mirRet(inst),166 .ret => try emit.mirRet(inst),
165167
166 .syscall => try emit.mirSyscall(),168 .syscall => try emit.mirSyscall(),
...@@ -373,6 +375,24 @@ fn mirCondSetByte(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) Inne...@@ -373,6 +375,24 @@ fn mirCondSetByte(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) Inne
373 return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1.to8()), emit.code);375 return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1.to8()), emit.code);
374}376}
375377
378fn mirCondMov(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void {
379 const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]);
380 if (ops.flags == 0b00) {
381 return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code);
382 }
383 const imm = emit.mir.instructions.items(.data)[inst].imm;
384 const ptr_size: Memory.PtrSize = switch (ops.flags) {
385 0b00 => unreachable,
386 0b01 => .word_ptr,
387 0b10 => .dword_ptr,
388 0b11 => .qword_ptr,
389 };
390 return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(ptr_size, .{
391 .disp = imm,
392 .base = ops.reg2,
393 }), emit.code);
394}
395
376fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {396fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
377 const tag = emit.mir.instructions.items(.tag)[inst];397 const tag = emit.mir.instructions.items(.tag)[inst];
378 assert(tag == .@"test");398 assert(tag == .@"test");
...@@ -391,7 +411,7 @@ fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {...@@ -391,7 +411,7 @@ fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void {
391 return lowerToMiEnc(.@"test", RegisterOrMemory.reg(ops.reg1), imm, emit.code);411 return lowerToMiEnc(.@"test", RegisterOrMemory.reg(ops.reg1), imm, emit.code);
392 }412 }
393 // TEST r/m64, r64413 // TEST r/m64, r64
394 return emit.fail("TODO TEST r/m64, r64", .{});414 return lowerToMrEnc(.@"test", RegisterOrMemory.reg(ops.reg1), ops.reg2, emit.code);
395 },415 },
396 else => return emit.fail("TODO more TEST alternatives", .{}),416 else => return emit.fail("TODO more TEST alternatives", .{}),
397 }417 }
...@@ -1158,6 +1178,8 @@ const Tag = enum {...@@ -1158,6 +1178,8 @@ const Tag = enum {
1158 cwd,1178 cwd,
1159 cdq,1179 cdq,
1160 cqo,1180 cqo,
1181 cmove,
1182 cmovz,
11611183
1162 fn isSetCC(tag: Tag) bool {1184 fn isSetCC(tag: Tag) bool {
1163 return switch (tag) {1185 return switch (tag) {
...@@ -1365,6 +1387,7 @@ inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode {...@@ -1365,6 +1387,7 @@ inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode {
1365 .sbb => OpCode.oneByte(if (is_one_byte) 0x18 else 0x19),1387 .sbb => OpCode.oneByte(if (is_one_byte) 0x18 else 0x19),
1366 .cmp => OpCode.oneByte(if (is_one_byte) 0x38 else 0x39),1388 .cmp => OpCode.oneByte(if (is_one_byte) 0x38 else 0x39),
1367 .mov => OpCode.oneByte(if (is_one_byte) 0x88 else 0x89),1389 .mov => OpCode.oneByte(if (is_one_byte) 0x88 else 0x89),
1390 .@"test" => OpCode.oneByte(if (is_one_byte) 0x84 else 0x85),
1368 else => null,1391 else => null,
1369 },1392 },
1370 .rm => return switch (tag) {1393 .rm => return switch (tag) {
...@@ -1382,6 +1405,7 @@ inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode {...@@ -1382,6 +1405,7 @@ inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode {
1382 .movzx => OpCode.twoByte(0x0f, if (is_one_byte) 0xb6 else 0xb7),1405 .movzx => OpCode.twoByte(0x0f, if (is_one_byte) 0xb6 else 0xb7),
1383 .lea => OpCode.oneByte(if (is_one_byte) 0x8c else 0x8d),1406 .lea => OpCode.oneByte(if (is_one_byte) 0x8c else 0x8d),
1384 .imul => OpCode.twoByte(0x0f, 0xaf),1407 .imul => OpCode.twoByte(0x0f, 0xaf),
1408 .cmove, .cmovz => OpCode.twoByte(0x0f, 0x44),
1385 else => null,1409 else => null,
1386 },1410 },
1387 .oi => return switch (tag) {1411 .oi => return switch (tag) {
src/arch/x86_64/Mir.zig+7
...@@ -286,6 +286,13 @@ pub const Inst = struct {...@@ -286,6 +286,13 @@ pub const Inst = struct {
286 cond_jmp_eq_ne,286 cond_jmp_eq_ne,
287 cond_set_byte_eq_ne,287 cond_set_byte_eq_ne,
288288
289 /// ops flags:
290 /// 0b00 reg1, reg2,
291 /// 0b01 reg1, word ptr [reg2 + imm]
292 /// 0b10 reg1, dword ptr [reg2 + imm]
293 /// 0b11 reg1, qword ptr [reg2 + imm]
294 cond_mov_eq,
295
289 /// ops flags: form:296 /// ops flags: form:
290 /// 0b00 reg1297 /// 0b00 reg1
291 /// 0b01 [reg1 + imm32]298 /// 0b01 [reg1 + imm32]