authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-03-18 08:02:17-04:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2023-03-21 08:49:54+01:00
log53ec2a955eb02dc829af8610f17af42717d512fb
tree68b2ef11487f4415186efbd6bb1fed8bf0bb8add
parentedd63f9abaa7dae1786a4cd91cc3031264bd3ef0

x86_64: implement clz, ctz, and popCount

5 files changed, 169 insertions(+), 17 deletions(-)

src/arch/x86_64/CodeGen.zig+128-13
...@@ -2597,28 +2597,143 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {...@@ -2597,28 +2597,143 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
25972597
2598fn airClz(self: *Self, inst: Air.Inst.Index) !void {2598fn airClz(self: *Self, inst: Air.Inst.Index) !void {
2599 const ty_op = self.air.instructions.items(.data)[inst].ty_op;2599 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2600 const result: MCValue = if (self.liveness.isUnused(inst))2600 const result = result: {
2601 .dead2601 if (self.liveness.isUnused(inst)) break :result .dead;
2602 else2602
2603 return self.fail("TODO implement airClz for {}", .{self.target.cpu.arch});2603 const dst_ty = self.air.typeOfIndex(inst);
2604 const src_ty = self.air.typeOf(ty_op.operand);
2605 const src_bits = src_ty.bitSize(self.target.*);
2606
2607 const src_mcv = try self.resolveInst(ty_op.operand);
2608 const mat_src_mcv = switch (src_mcv) {
2609 .immediate => MCValue{ .register = try self.copyToTmpRegister(src_ty, src_mcv) },
2610 else => src_mcv,
2611 };
2612 const mat_src_lock = switch (mat_src_mcv) {
2613 .register => |reg| self.register_manager.lockReg(reg),
2614 else => null,
2615 };
2616 defer if (mat_src_lock) |lock| self.register_manager.unlockReg(lock);
2617
2618 const dst_reg = try self.register_manager.allocReg(inst, gp);
2619 const dst_mcv = MCValue{ .register = dst_reg };
2620 const dst_lock = self.register_manager.lockReg(dst_reg);
2621 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
2622
2623 if (Target.x86.featureSetHas(self.target.cpu.features, .lzcnt)) {
2624 try self.genBinOpMir(.lzcnt, src_ty, dst_mcv, mat_src_mcv);
2625 const src_abi_size = @intCast(u32, src_ty.abiSize(self.target.*));
2626 const extra_bits = registerAlias(dst_reg, src_abi_size).bitSize() - src_bits;
2627 if (extra_bits > 0) {
2628 try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .immediate = extra_bits });
2629 }
2630 break :result dst_mcv;
2631 }
2632
2633 const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits });
2634 const width_mcv = MCValue{ .register = width_reg };
2635 try self.genBinOpMir(.bsr, src_ty, dst_mcv, mat_src_mcv);
2636
2637 const dst_abi_size = @intCast(u32, @max(dst_ty.abiSize(self.target.*), 2));
2638 try self.asmCmovccRegisterRegister(
2639 registerAlias(dst_reg, dst_abi_size),
2640 registerAlias(width_reg, dst_abi_size),
2641 .z,
2642 );
2643
2644 try self.genBinOpMir(.sub, dst_ty, width_mcv, dst_mcv);
2645 break :result width_mcv;
2646 };
2604 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });2647 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2605}2648}
26062649
2607fn airCtz(self: *Self, inst: Air.Inst.Index) !void {2650fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
2608 const ty_op = self.air.instructions.items(.data)[inst].ty_op;2651 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2609 const result: MCValue = if (self.liveness.isUnused(inst))2652 const result = result: {
2610 .dead2653 if (self.liveness.isUnused(inst)) break :result .dead;
2611 else2654
2612 return self.fail("TODO implement airCtz for {}", .{self.target.cpu.arch});2655 const dst_ty = self.air.typeOfIndex(inst);
2656 const src_ty = self.air.typeOf(ty_op.operand);
2657 const src_bits = src_ty.bitSize(self.target.*);
2658
2659 const src_mcv = try self.resolveInst(ty_op.operand);
2660 const mat_src_mcv = switch (src_mcv) {
2661 .immediate => MCValue{ .register = try self.copyToTmpRegister(src_ty, src_mcv) },
2662 else => src_mcv,
2663 };
2664 const mat_src_lock = switch (mat_src_mcv) {
2665 .register => |reg| self.register_manager.lockReg(reg),
2666 else => null,
2667 };
2668 defer if (mat_src_lock) |lock| self.register_manager.unlockReg(lock);
2669
2670 const dst_reg = try self.register_manager.allocReg(inst, gp);
2671 const dst_mcv = MCValue{ .register = dst_reg };
2672 const dst_lock = self.register_manager.lockReg(dst_reg);
2673 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
2674
2675 if (Target.x86.featureSetHas(self.target.cpu.features, .bmi)) {
2676 const src_abi_size = @intCast(u32, src_ty.abiSize(self.target.*));
2677 const extra_bits = registerAlias(dst_reg, src_abi_size).bitSize() - src_bits;
2678 const masked_mcv = if (extra_bits > 0) masked: {
2679 const mask_mcv = MCValue{
2680 .immediate = ((@as(u64, 1) << @intCast(u6, extra_bits)) - 1) << @intCast(u6, src_bits),
2681 };
2682 const tmp_mcv = tmp: {
2683 if (src_mcv.isImmediate() or self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) {
2684 break :tmp src_mcv;
2685 }
2686 try self.genSetReg(src_ty, dst_reg, src_mcv);
2687 break :tmp dst_mcv;
2688 };
2689 try self.genBinOpMir(.@"or", src_ty, tmp_mcv, mask_mcv);
2690 break :masked tmp_mcv;
2691 } else mat_src_mcv;
2692 try self.genBinOpMir(.tzcnt, src_ty, dst_mcv, masked_mcv);
2693 break :result dst_mcv;
2694 }
2695
2696 const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits });
2697 try self.genBinOpMir(.bsf, src_ty, dst_mcv, mat_src_mcv);
2698
2699 const abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
2700 try self.asmCmovccRegisterRegister(
2701 registerAlias(dst_reg, abi_size),
2702 registerAlias(width_reg, abi_size),
2703 .z,
2704 );
2705
2706 break :result dst_mcv;
2707 };
2613 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });2708 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2614}2709}
26152710
2616fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {2711fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
2617 const ty_op = self.air.instructions.items(.data)[inst].ty_op;2712 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2618 const result: MCValue = if (self.liveness.isUnused(inst))2713 const result = result: {
2619 .dead2714 if (self.liveness.isUnused(inst)) break :result .dead;
2620 else2715
2621 return self.fail("TODO implement airPopcount for {}", .{self.target.cpu.arch});2716 const op_ty = self.air.typeOf(ty_op.operand);
2717
2718 if (Target.x86.featureSetHas(self.target.cpu.features, .popcnt)) {
2719 const op_mcv = try self.resolveInst(ty_op.operand);
2720 const mat_op_mcv = switch (op_mcv) {
2721 .immediate => MCValue{ .register = try self.copyToTmpRegister(op_ty, op_mcv) },
2722 else => op_mcv,
2723 };
2724 const mat_op_lock = switch (mat_op_mcv) {
2725 .register => |reg| self.register_manager.lockReg(reg),
2726 else => null,
2727 };
2728 defer if (mat_op_lock) |lock| self.register_manager.unlockReg(lock);
2729
2730 const dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, gp) };
2731 try self.genBinOpMir(.popcnt, op_ty, dst_mcv, mat_op_mcv);
2732 break :result dst_mcv;
2733 }
2734
2735 return self.fail("TODO implement airPopcount for {}", .{op_ty.fmt(self.bin_file.options.module.?)});
2736 };
2622 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });2737 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2623}2738}
26242739
...@@ -3491,7 +3606,7 @@ fn genBinOp(...@@ -3491,7 +3606,7 @@ fn genBinOp(
3491 if (lhs.isRegister() and self.reuseOperand(inst, lhs_air, 0, lhs)) {3606 if (lhs.isRegister() and self.reuseOperand(inst, lhs_air, 0, lhs)) {
3492 break :blk lhs;3607 break :blk lhs;
3493 }3608 }
3494 if (rhs.isRegister() and is_commutative and self.reuseOperand(inst, rhs_air, 1, rhs)) {3609 if (is_commutative and rhs.isRegister() and self.reuseOperand(inst, rhs_air, 1, rhs)) {
3495 flipped = true;3610 flipped = true;
3496 break :blk rhs;3611 break :blk rhs;
3497 }3612 }
src/arch/x86_64/Emit.zig+5
...@@ -73,6 +73,8 @@ pub fn lowerMir(emit: *Emit) InnerError!void {...@@ -73,6 +73,8 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
73 .adc,73 .adc,
74 .add,74 .add,
75 .@"and",75 .@"and",
76 .bsf,
77 .bsr,
76 .call,78 .call,
77 .cbw,79 .cbw,
78 .cwde,80 .cwde,
...@@ -89,12 +91,14 @@ pub fn lowerMir(emit: *Emit) InnerError!void {...@@ -89,12 +91,14 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
89 .int3,91 .int3,
90 .jmp,92 .jmp,
91 .lea,93 .lea,
94 .lzcnt,
92 .mov,95 .mov,
93 .movzx,96 .movzx,
94 .mul,97 .mul,
95 .nop,98 .nop,
96 .@"or",99 .@"or",
97 .pop,100 .pop,
101 .popcnt,
98 .push,102 .push,
99 .ret,103 .ret,
100 .sal,104 .sal,
...@@ -105,6 +109,7 @@ pub fn lowerMir(emit: *Emit) InnerError!void {...@@ -105,6 +109,7 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
105 .sub,109 .sub,
106 .syscall,110 .syscall,
107 .@"test",111 .@"test",
112 .tzcnt,
108 .ud2,113 .ud2,
109 .xor,114 .xor,
110115
src/arch/x86_64/Encoding.zig+4-2
...@@ -307,6 +307,7 @@ pub const Mnemonic = enum {...@@ -307,6 +307,7 @@ pub const Mnemonic = enum {
307 // zig fmt: off307 // zig fmt: off
308 // General-purpose308 // General-purpose
309 adc, add, @"and",309 adc, add, @"and",
310 bsf, bsr,
310 call, cbw, cdq, cdqe,311 call, cbw, cdq, cdqe,
311 cmova, cmovae, cmovb, cmovbe, cmovc, cmove, cmovg, cmovge, cmovl, cmovle, cmovna,312 cmova, cmovae, cmovb, cmovbe, cmovc, cmove, cmovg, cmovge, cmovl, cmovle, cmovna,
312 cmovnae, cmovnb, cmovnbe, cmovnc, cmovne, cmovng, cmovnge, cmovnl, cmovnle, cmovno,313 cmovnae, cmovnb, cmovnbe, cmovnc, cmovne, cmovng, cmovnge, cmovnl, cmovnle, cmovno,
...@@ -322,12 +323,13 @@ pub const Mnemonic = enum {...@@ -322,12 +323,13 @@ pub const Mnemonic = enum {
322 jmp, 323 jmp,
323 lea,324 lea,
324 lods, lodsb, lodsd, lodsq, lodsw,325 lods, lodsb, lodsd, lodsq, lodsw,
326 lzcnt,
325 mov,327 mov,
326 movs, movsb, movsd, movsq, movsw,328 movs, movsb, movsd, movsq, movsw,
327 movsx, movsxd, movzx, mul,329 movsx, movsxd, movzx, mul,
328 nop,330 nop,
329 @"or",331 @"or",
330 pop, push,332 pop, popcnt, push,
331 ret,333 ret,
332 sal, sar, sbb,334 sal, sar, sbb,
333 scas, scasb, scasd, scasq, scasw,335 scas, scasb, scasd, scasq, scasw,
...@@ -336,7 +338,7 @@ pub const Mnemonic = enum {...@@ -336,7 +338,7 @@ pub const Mnemonic = enum {
336 setnb, setnbe, setnc, setne, setng, setnge, setnl, setnle, setno, setnp, setns,338 setnb, setnbe, setnc, setne, setng, setnge, setnl, setnle, setno, setnp, setns,
337 setnz, seto, setp, setpe, setpo, sets, setz,339 setnz, seto, setp, setpe, setpo, sets, setz,
338 stos, stosb, stosd, stosq, stosw,340 stos, stosb, stosd, stosq, stosw,
339 @"test",341 @"test", tzcnt,
340 ud2,342 ud2,
341 xor,343 xor,
342 // SSE344 // SSE
src/arch/x86_64/Mir.zig+10
...@@ -38,6 +38,10 @@ pub const Inst = struct {...@@ -38,6 +38,10 @@ pub const Inst = struct {
38 add,38 add,
39 /// Logical and39 /// Logical and
40 @"and",40 @"and",
41 /// Bit scan forward
42 bsf,
43 /// Bit scan reverse
44 bsr,
41 /// Call45 /// Call
42 call,46 call,
43 /// Convert byte to word47 /// Convert byte to word
...@@ -70,6 +74,8 @@ pub const Inst = struct {...@@ -70,6 +74,8 @@ pub const Inst = struct {
70 jmp,74 jmp,
71 /// Load effective address75 /// Load effective address
72 lea,76 lea,
77 /// Count the number of leading zero bits
78 lzcnt,
73 /// Move79 /// Move
74 mov,80 mov,
75 /// Move with sign extension81 /// Move with sign extension
...@@ -84,6 +90,8 @@ pub const Inst = struct {...@@ -84,6 +90,8 @@ pub const Inst = struct {
84 @"or",90 @"or",
85 /// Pop91 /// Pop
86 pop,92 pop,
93 /// Return the count of number of bits set to 1
94 popcnt,
87 /// Push95 /// Push
88 push,96 push,
89 /// Return97 /// Return
...@@ -104,6 +112,8 @@ pub const Inst = struct {...@@ -104,6 +112,8 @@ pub const Inst = struct {
104 syscall,112 syscall,
105 /// Test condition113 /// Test condition
106 @"test",114 @"test",
115 /// Count the number of trailing zero bits
116 tzcnt,
107 /// Undefined instruction117 /// Undefined instruction
108 ud2,118 ud2,
109 /// Logical exclusive-or119 /// Logical exclusive-or
src/arch/x86_64/encodings.zig+22-2
...@@ -81,6 +81,14 @@ pub const table = &[_]Entry{...@@ -81,6 +81,14 @@ pub const table = &[_]Entry{
81 .{ .@"and", .rm, .r32, .rm32, .none, .none, &.{ 0x23 }, 0, .none },81 .{ .@"and", .rm, .r32, .rm32, .none, .none, &.{ 0x23 }, 0, .none },
82 .{ .@"and", .rm, .r64, .rm64, .none, .none, &.{ 0x23 }, 0, .long },82 .{ .@"and", .rm, .r64, .rm64, .none, .none, &.{ 0x23 }, 0, .long },
8383
84 .{ .bsf, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0xbc }, 0, .none },
85 .{ .bsf, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0xbc }, 0, .none },
86 .{ .bsf, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0xbc }, 0, .long },
87
88 .{ .bsr, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0xbd }, 0, .none },
89 .{ .bsr, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0xbd }, 0, .none },
90 .{ .bsr, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0xbd }, 0, .long },
91
84 // This is M encoding according to Intel, but D makes more sense here.92 // This is M encoding according to Intel, but D makes more sense here.
85 .{ .call, .d, .rel32, .none, .none, .none, &.{ 0xe8 }, 0, .none },93 .{ .call, .d, .rel32, .none, .none, .none, &.{ 0xe8 }, 0, .none },
86 .{ .call, .m, .rm64, .none, .none, .none, &.{ 0xff }, 2, .none },94 .{ .call, .m, .rm64, .none, .none, .none, &.{ 0xff }, 2, .none },
...@@ -301,6 +309,10 @@ pub const table = &[_]Entry{...@@ -301,6 +309,10 @@ pub const table = &[_]Entry{
301 .{ .lodsd, .np, .none, .none, .none, .none, &.{ 0xad }, 0, .none },309 .{ .lodsd, .np, .none, .none, .none, .none, &.{ 0xad }, 0, .none },
302 .{ .lodsq, .np, .none, .none, .none, .none, &.{ 0xad }, 0, .long },310 .{ .lodsq, .np, .none, .none, .none, .none, &.{ 0xad }, 0, .long },
303311
312 .{ .lzcnt, .rm, .r16, .rm16, .none, .none, &.{ 0xf3, 0x0f, 0xbd }, 0, .none },
313 .{ .lzcnt, .rm, .r32, .rm32, .none, .none, &.{ 0xf3, 0x0f, 0xbd }, 0, .none },
314 .{ .lzcnt, .rm, .r64, .rm64, .none, .none, &.{ 0xf3, 0x0f, 0xbd }, 0, .long },
315
304 .{ .mov, .mr, .rm8, .r8, .none, .none, &.{ 0x88 }, 0, .none },316 .{ .mov, .mr, .rm8, .r8, .none, .none, &.{ 0x88 }, 0, .none },
305 .{ .mov, .mr, .rm8, .r8, .none, .none, &.{ 0x88 }, 0, .rex },317 .{ .mov, .mr, .rm8, .r8, .none, .none, &.{ 0x88 }, 0, .rex },
306 .{ .mov, .mr, .rm16, .r16, .none, .none, &.{ 0x89 }, 0, .none },318 .{ .mov, .mr, .rm16, .r16, .none, .none, &.{ 0x89 }, 0, .none },
...@@ -397,6 +409,10 @@ pub const table = &[_]Entry{...@@ -397,6 +409,10 @@ pub const table = &[_]Entry{
397 .{ .pop, .m, .rm16, .none, .none, .none, &.{ 0x8f }, 0, .none },409 .{ .pop, .m, .rm16, .none, .none, .none, &.{ 0x8f }, 0, .none },
398 .{ .pop, .m, .rm64, .none, .none, .none, &.{ 0x8f }, 0, .none },410 .{ .pop, .m, .rm64, .none, .none, .none, &.{ 0x8f }, 0, .none },
399411
412 .{ .popcnt, .rm, .r16, .rm16, .none, .none, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none },
413 .{ .popcnt, .rm, .r32, .rm32, .none, .none, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none },
414 .{ .popcnt, .rm, .r64, .rm64, .none, .none, &.{ 0xf3, 0x0f, 0xb8 }, 0, .long },
415
400 .{ .push, .o, .r16, .none, .none, .none, &.{ 0x50 }, 0, .none },416 .{ .push, .o, .r16, .none, .none, .none, &.{ 0x50 }, 0, .none },
401 .{ .push, .o, .r64, .none, .none, .none, &.{ 0x50 }, 0, .none },417 .{ .push, .o, .r64, .none, .none, .none, &.{ 0x50 }, 0, .none },
402 .{ .push, .m, .rm16, .none, .none, .none, &.{ 0xff }, 6, .none },418 .{ .push, .m, .rm16, .none, .none, .none, &.{ 0xff }, 6, .none },
...@@ -596,8 +612,8 @@ pub const table = &[_]Entry{...@@ -596,8 +612,8 @@ pub const table = &[_]Entry{
596 .{ .sub, .rm, .r32, .rm32, .none, .none, &.{ 0x2b }, 0, .none },612 .{ .sub, .rm, .r32, .rm32, .none, .none, &.{ 0x2b }, 0, .none },
597 .{ .sub, .rm, .r64, .rm64, .none, .none, &.{ 0x2b }, 0, .long },613 .{ .sub, .rm, .r64, .rm64, .none, .none, &.{ 0x2b }, 0, .long },
598614
599 .{ .syscall, .np, .none, .none, .none, .none, &.{ 0x0f, 0x05 }, 0, .none },615 .{ .syscall, .np, .none, .none, .none, .none, &.{ 0x0f, 0x05 }, 0, .none }
600616,
601 .{ .@"test", .zi, .al, .imm8, .none, .none, &.{ 0xa8 }, 0, .none },617 .{ .@"test", .zi, .al, .imm8, .none, .none, &.{ 0xa8 }, 0, .none },
602 .{ .@"test", .zi, .ax, .imm16, .none, .none, &.{ 0xa9 }, 0, .none },618 .{ .@"test", .zi, .ax, .imm16, .none, .none, &.{ 0xa9 }, 0, .none },
603 .{ .@"test", .zi, .eax, .imm32, .none, .none, &.{ 0xa9 }, 0, .none },619 .{ .@"test", .zi, .eax, .imm32, .none, .none, &.{ 0xa9 }, 0, .none },
...@@ -613,6 +629,10 @@ pub const table = &[_]Entry{...@@ -613,6 +629,10 @@ pub const table = &[_]Entry{
613 .{ .@"test", .mr, .rm32, .r32, .none, .none, &.{ 0x85 }, 0, .none },629 .{ .@"test", .mr, .rm32, .r32, .none, .none, &.{ 0x85 }, 0, .none },
614 .{ .@"test", .mr, .rm64, .r64, .none, .none, &.{ 0x85 }, 0, .long },630 .{ .@"test", .mr, .rm64, .r64, .none, .none, &.{ 0x85 }, 0, .long },
615631
632 .{ .tzcnt, .rm, .r16, .rm16, .none, .none, &.{ 0xf3, 0x0f, 0xbc }, 0, .none },
633 .{ .tzcnt, .rm, .r32, .rm32, .none, .none, &.{ 0xf3, 0x0f, 0xbc }, 0, .none },
634 .{ .tzcnt, .rm, .r64, .rm64, .none, .none, &.{ 0xf3, 0x0f, 0xbc }, 0, .long },
635
616 .{ .ud2, .np, .none, .none, .none, .none, &.{ 0x0f, 0x0b }, 0, .none },636 .{ .ud2, .np, .none, .none, .none, .none, &.{ 0x0f, 0x0b }, 0, .none },
617637
618 .{ .xor, .zi, .al, .imm8, .none, .none, &.{ 0x34 }, 0, .none },638 .{ .xor, .zi, .al, .imm8, .none, .none, &.{ 0x34 }, 0, .none },