authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-02-12 20:44:30-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-02-12 20:44:30-07:00
loga005ac9d3c884c6254074ec150fe536881fe31b5
tree6c7d2016362d441ad77d77abbf76f5116b329c73
parent16ec848d2ab5702ad3794d30ed5d776b5abb60ce

stage2: implement `@popCount` for SIMD vectors

6 files changed, 61 insertions(+), 81 deletions(-)

src/Sema.zig+4-32
...@@ -720,7 +720,6 @@ fn analyzeBodyInner(...@@ -720,7 +720,6 @@ fn analyzeBodyInner(
720 .align_cast => try sema.zirAlignCast(block, inst),720 .align_cast => try sema.zirAlignCast(block, inst),
721 .has_decl => try sema.zirHasDecl(block, inst),721 .has_decl => try sema.zirHasDecl(block, inst),
722 .has_field => try sema.zirHasField(block, inst),722 .has_field => try sema.zirHasField(block, inst),
723 .pop_count => try sema.zirPopCount(block, inst),
724 .byte_swap => try sema.zirByteSwap(block, inst),723 .byte_swap => try sema.zirByteSwap(block, inst),
725 .bit_reverse => try sema.zirBitReverse(block, inst),724 .bit_reverse => try sema.zirBitReverse(block, inst),
726 .bit_offset_of => try sema.zirBitOffsetOf(block, inst),725 .bit_offset_of => try sema.zirBitOffsetOf(block, inst),
...@@ -743,8 +742,9 @@ fn analyzeBodyInner(...@@ -743,8 +742,9 @@ fn analyzeBodyInner(
743 .await_nosuspend => try sema.zirAwait(block, inst, true),742 .await_nosuspend => try sema.zirAwait(block, inst, true),
744 .extended => try sema.zirExtended(block, inst),743 .extended => try sema.zirExtended(block, inst),
745744
746 .clz => try sema.zirClzCtz(block, inst, .clz, Value.clz),745 .clz => try sema.zirBitCount(block, inst, .clz, Value.clz),
747 .ctz => try sema.zirClzCtz(block, inst, .ctz, Value.ctz),746 .ctz => try sema.zirBitCount(block, inst, .ctz, Value.ctz),
747 .pop_count => try sema.zirBitCount(block, inst, .popcount, Value.popCount),
748748
749 .sqrt => try sema.zirUnaryMath(block, inst, .sqrt, Value.sqrt),749 .sqrt => try sema.zirUnaryMath(block, inst, .sqrt, Value.sqrt),
750 .sin => try sema.zirUnaryMath(block, inst, .sin, Value.sin),750 .sin => try sema.zirUnaryMath(block, inst, .sin, Value.sin),
...@@ -11487,7 +11487,7 @@ fn zirAlignCast(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A...@@ -11487,7 +11487,7 @@ fn zirAlignCast(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A
11487 return sema.coerceCompatiblePtrs(block, dest_ty, ptr, ptr_src);11487 return sema.coerceCompatiblePtrs(block, dest_ty, ptr, ptr_src);
11488}11488}
1148911489
11490fn zirClzCtz(11490fn zirBitCount(
11491 sema: *Sema,11491 sema: *Sema,
11492 block: *Block,11492 block: *Block,
11493 inst: Zir.Inst.Index,11493 inst: Zir.Inst.Index,
...@@ -11550,34 +11550,6 @@ fn zirClzCtz(...@@ -11550,34 +11550,6 @@ fn zirClzCtz(
11550 }11550 }
11551}11551}
1155211552
11553fn zirPopCount(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
11554 const inst_data = sema.code.instructions.items(.data)[inst].un_node;
11555 const ty_src: LazySrcLoc = .{ .node_offset_builtin_call_arg0 = inst_data.src_node };
11556 const operand_src: LazySrcLoc = .{ .node_offset_builtin_call_arg1 = inst_data.src_node };
11557 const operand = sema.resolveInst(inst_data.operand);
11558 const operand_ty = sema.typeOf(operand);
11559 // TODO implement support for vectors
11560 if (operand_ty.zigTypeTag() != .Int) {
11561 return sema.fail(block, ty_src, "expected integer type, found '{}'", .{
11562 operand_ty,
11563 });
11564 }
11565 const target = sema.mod.getTarget();
11566 const bits = operand_ty.intInfo(target).bits;
11567 if (bits == 0) return Air.Inst.Ref.zero;
11568
11569 const result_ty = try Type.smallestUnsignedInt(sema.arena, bits);
11570
11571 const runtime_src = if (try sema.resolveMaybeUndefVal(block, operand_src, operand)) |val| {
11572 if (val.isUndef()) return sema.addConstUndef(result_ty);
11573 const result_val = try val.popCount(operand_ty, target, sema.arena);
11574 return sema.addConstant(result_ty, result_val);
11575 } else operand_src;
11576
11577 try sema.requireRuntimeBlock(block, runtime_src);
11578 return block.addTyOp(.popcount, result_ty, operand);
11579}
11580
11581fn zirByteSwap(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {11553fn zirByteSwap(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
11582 const inst_data = sema.code.instructions.items(.data)[inst].un_node;11554 const inst_data = sema.code.instructions.items(.data)[inst].un_node;
11583 const src = inst_data.src();11555 const src = inst_data.src();
src/codegen/llvm.zig+10-5
...@@ -2205,7 +2205,7 @@ pub const FuncGen = struct {...@@ -2205,7 +2205,7 @@ pub const FuncGen = struct {
2205 .get_union_tag => try self.airGetUnionTag(inst),2205 .get_union_tag => try self.airGetUnionTag(inst),
2206 .clz => try self.airClzCtz(inst, "ctlz"),2206 .clz => try self.airClzCtz(inst, "ctlz"),
2207 .ctz => try self.airClzCtz(inst, "cttz"),2207 .ctz => try self.airClzCtz(inst, "cttz"),
2208 .popcount => try self.airPopCount(inst, "ctpop"),2208 .popcount => try self.airPopCount(inst),
2209 .tag_name => try self.airTagName(inst),2209 .tag_name => try self.airTagName(inst),
2210 .error_name => try self.airErrorName(inst),2210 .error_name => try self.airErrorName(inst),
2211 .splat => try self.airSplat(inst),2211 .splat => try self.airSplat(inst),
...@@ -4364,7 +4364,7 @@ pub const FuncGen = struct {...@@ -4364,7 +4364,7 @@ pub const FuncGen = struct {
4364 }4364 }
4365 }4365 }
43664366
4367 fn airPopCount(self: *FuncGen, inst: Air.Inst.Index, prefix: [*:0]const u8) !?*const llvm.Value {4367 fn airPopCount(self: *FuncGen, inst: Air.Inst.Index) !?*const llvm.Value {
4368 if (self.liveness.isUnused(inst)) return null;4368 if (self.liveness.isUnused(inst)) return null;
43694369
4370 const ty_op = self.air.instructions.items(.data)[inst].ty_op;4370 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
...@@ -4372,11 +4372,16 @@ pub const FuncGen = struct {...@@ -4372,11 +4372,16 @@ pub const FuncGen = struct {
4372 const operand = try self.resolveInst(ty_op.operand);4372 const operand = try self.resolveInst(ty_op.operand);
4373 const target = self.dg.module.getTarget();4373 const target = self.dg.module.getTarget();
4374 const bits = operand_ty.intInfo(target).bits;4374 const bits = operand_ty.intInfo(target).bits;
4375 const vec_len: ?u32 = switch (operand_ty.zigTypeTag()) {
4376 .Vector => operand_ty.vectorLen(),
4377 else => null,
4378 };
43754379
4376 var fn_name_buf: [100]u8 = undefined;4380 var fn_name_buf: [100]u8 = undefined;
4377 const llvm_fn_name = std.fmt.bufPrintZ(&fn_name_buf, "llvm.{s}.i{d}", .{4381 const llvm_fn_name = if (vec_len) |len|
4378 prefix, bits,4382 std.fmt.bufPrintZ(&fn_name_buf, "llvm.ctpop.v{d}i{d}", .{ len, bits }) catch unreachable
4379 }) catch unreachable;4383 else
4384 std.fmt.bufPrintZ(&fn_name_buf, "llvm.ctpop.i{d}", .{bits}) catch unreachable;
4380 const fn_val = self.dg.object.llvm_module.getNamedFunction(llvm_fn_name) orelse blk: {4385 const fn_val = self.dg.object.llvm_module.getNamedFunction(llvm_fn_name) orelse blk: {
4381 const operand_llvm_ty = try self.dg.llvmType(operand_ty);4386 const operand_llvm_ty = try self.dg.llvmType(operand_ty);
4382 const param_types = [_]*const llvm.Type{operand_llvm_ty};4387 const param_types = [_]*const llvm.Type{operand_llvm_ty};
src/value.zig+27-18
...@@ -1303,6 +1303,33 @@ pub const Value = extern union {...@@ -1303,6 +1303,33 @@ pub const Value = extern union {
1303 }1303 }
1304 }1304 }
13051305
1306 pub fn popCount(val: Value, ty: Type, target: Target) u64 {
1307 assert(!val.isUndef());
1308 switch (val.tag()) {
1309 .zero, .bool_false => return 0,
1310 .one, .bool_true => return 1,
1311
1312 .int_u64 => return @popCount(u64, val.castTag(.int_u64).?.data),
1313
1314 else => {
1315 const info = ty.intInfo(target);
1316
1317 var buffer: Value.BigIntSpace = undefined;
1318 const operand_bigint = val.toBigInt(&buffer);
1319
1320 var limbs_buffer: [4]std.math.big.Limb = undefined;
1321 var result_bigint = BigIntMutable{
1322 .limbs = &limbs_buffer,
1323 .positive = undefined,
1324 .len = undefined,
1325 };
1326 result_bigint.popCount(operand_bigint, info.bits);
1327
1328 return result_bigint.toConst().to(u64) catch unreachable;
1329 },
1330 }
1331 }
1332
1306 /// Asserts the value is an integer and not undefined.1333 /// Asserts the value is an integer and not undefined.
1307 /// Returns the number of bits the value requires to represent stored in twos complement form.1334 /// Returns the number of bits the value requires to represent stored in twos complement form.
1308 pub fn intBitCountTwosComp(self: Value, target: Target) usize {1335 pub fn intBitCountTwosComp(self: Value, target: Target) usize {
...@@ -1340,24 +1367,6 @@ pub const Value = extern union {...@@ -1340,24 +1367,6 @@ pub const Value = extern union {
1340 }1367 }
1341 }1368 }
13421369
1343 pub fn popCount(val: Value, ty: Type, target: Target, arena: Allocator) !Value {
1344 assert(!val.isUndef());
1345
1346 const info = ty.intInfo(target);
1347
1348 var buffer: Value.BigIntSpace = undefined;
1349 const operand_bigint = val.toBigInt(&buffer);
1350
1351 const limbs = try arena.alloc(
1352 std.math.big.Limb,
1353 std.math.big.int.calcTwosCompLimbCount(info.bits),
1354 );
1355 var result_bigint = BigIntMutable{ .limbs = limbs, .positive = undefined, .len = undefined };
1356 result_bigint.popCount(operand_bigint, info.bits);
1357
1358 return fromBigInt(arena, result_bigint.toConst());
1359 }
1360
1361 /// Asserts the value is an integer, and the destination type is ComptimeInt or Int.1370 /// Asserts the value is an integer, and the destination type is ComptimeInt or Int.
1362 pub fn intFitsInType(self: Value, ty: Type, target: Target) bool {1371 pub fn intFitsInType(self: Value, ty: Type, target: Target) bool {
1363 switch (self.tag()) {1372 switch (self.tag()) {
test/behavior.zig-1
...@@ -153,7 +153,6 @@ test {...@@ -153,7 +153,6 @@ test {
153 _ = @import("behavior/ir_block_deps.zig");153 _ = @import("behavior/ir_block_deps.zig");
154 _ = @import("behavior/misc.zig");154 _ = @import("behavior/misc.zig");
155 _ = @import("behavior/muladd.zig");155 _ = @import("behavior/muladd.zig");
156 _ = @import("behavior/popcount_stage1.zig");
157 _ = @import("behavior/reflection.zig");156 _ = @import("behavior/reflection.zig");
158 _ = @import("behavior/select.zig");157 _ = @import("behavior/select.zig");
159 _ = @import("behavior/shuffle.zig");158 _ = @import("behavior/shuffle.zig");
test/behavior/popcount.zig+20-1
...@@ -1,7 +1,6 @@...@@ -1,7 +1,6 @@
1const std = @import("std");1const std = @import("std");
2const expect = std.testing.expect;2const expect = std.testing.expect;
3const expectEqual = std.testing.expectEqual;3const expectEqual = std.testing.expectEqual;
4const Vector = std.meta.Vector;
54
6test "@popCount integers" {5test "@popCount integers" {
7 comptime try testPopCountIntegers();6 comptime try testPopCountIntegers();
...@@ -44,3 +43,23 @@ fn testPopCountIntegers() !void {...@@ -44,3 +43,23 @@ fn testPopCountIntegers() !void {
44 try expect(@popCount(i128, @as(i128, 0b11111111000110001100010000100001000011000011100101010001)) == 24);43 try expect(@popCount(i128, @as(i128, 0b11111111000110001100010000100001000011000011100101010001)) == 24);
45 }44 }
46}45}
46
47test "@popCount vectors" {
48 comptime try testPopCountVectors();
49 try testPopCountVectors();
50}
51
52fn testPopCountVectors() !void {
53 {
54 var x: @Vector(8, u32) = [1]u32{0xffffffff} ** 8;
55 const expected = [1]u6{32} ** 8;
56 const result: [8]u6 = @popCount(u32, x);
57 try expect(std.mem.eql(u6, &expected, &result));
58 }
59 {
60 var x: @Vector(8, i16) = [1]i16{-1} ** 8;
61 const expected = [1]u5{16} ** 8;
62 const result: [8]u5 = @popCount(i16, x);
63 try expect(std.mem.eql(u5, &expected, &result));
64 }
65}
test/behavior/popcount_stage1.zig deleted-24
...@@ -1,24 +0,0 @@
1const std = @import("std");
2const expect = std.testing.expect;
3const expectEqual = std.testing.expectEqual;
4const Vector = std.meta.Vector;
5
6test "@popCount vectors" {
7 comptime try testPopCountVectors();
8 try testPopCountVectors();
9}
10
11fn testPopCountVectors() !void {
12 {
13 var x: Vector(8, u32) = [1]u32{0xffffffff} ** 8;
14 const expected = [1]u6{32} ** 8;
15 const result: [8]u6 = @popCount(u32, x);
16 try expect(std.mem.eql(u6, &expected, &result));
17 }
18 {
19 var x: Vector(8, i16) = [1]i16{-1} ** 8;
20 const expected = [1]u5{16} ** 8;
21 const result: [8]u5 = @popCount(i16, x);
22 try expect(std.mem.eql(u5, &expected, &result));
23 }
24}