authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-02-12 20:44:30-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-02-12 20:44:30-07:00
loga005ac9d3c884c6254074ec150fe536881fe31b5
tree6c7d2016362d441ad77d77abbf76f5116b329c73
parent16ec848d2ab5702ad3794d30ed5d776b5abb60ce

stage2: implement `@popCount` for SIMD vectors

6 files changed, 61 insertions(+), 81 deletions(-)

src/Sema.zig+4-32
......@@ -720,7 +720,6 @@ fn analyzeBodyInner(
720720 .align_cast => try sema.zirAlignCast(block, inst),
721721 .has_decl => try sema.zirHasDecl(block, inst),
722722 .has_field => try sema.zirHasField(block, inst),
723 .pop_count => try sema.zirPopCount(block, inst),
724723 .byte_swap => try sema.zirByteSwap(block, inst),
725724 .bit_reverse => try sema.zirBitReverse(block, inst),
726725 .bit_offset_of => try sema.zirBitOffsetOf(block, inst),
......@@ -743,8 +742,9 @@ fn analyzeBodyInner(
743742 .await_nosuspend => try sema.zirAwait(block, inst, true),
744743 .extended => try sema.zirExtended(block, inst),
745744
746 .clz => try sema.zirClzCtz(block, inst, .clz, Value.clz),
747 .ctz => try sema.zirClzCtz(block, inst, .ctz, Value.ctz),
745 .clz => try sema.zirBitCount(block, inst, .clz, Value.clz),
746 .ctz => try sema.zirBitCount(block, inst, .ctz, Value.ctz),
747 .pop_count => try sema.zirBitCount(block, inst, .popcount, Value.popCount),
748748
749749 .sqrt => try sema.zirUnaryMath(block, inst, .sqrt, Value.sqrt),
750750 .sin => try sema.zirUnaryMath(block, inst, .sin, Value.sin),
......@@ -11487,7 +11487,7 @@ fn zirAlignCast(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A
1148711487 return sema.coerceCompatiblePtrs(block, dest_ty, ptr, ptr_src);
1148811488}
1148911489
11490fn zirClzCtz(
11490fn zirBitCount(
1149111491 sema: *Sema,
1149211492 block: *Block,
1149311493 inst: Zir.Inst.Index,
......@@ -11550,34 +11550,6 @@ fn zirClzCtz(
1155011550 }
1155111551}
1155211552
11553fn zirPopCount(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
11554 const inst_data = sema.code.instructions.items(.data)[inst].un_node;
11555 const ty_src: LazySrcLoc = .{ .node_offset_builtin_call_arg0 = inst_data.src_node };
11556 const operand_src: LazySrcLoc = .{ .node_offset_builtin_call_arg1 = inst_data.src_node };
11557 const operand = sema.resolveInst(inst_data.operand);
11558 const operand_ty = sema.typeOf(operand);
11559 // TODO implement support for vectors
11560 if (operand_ty.zigTypeTag() != .Int) {
11561 return sema.fail(block, ty_src, "expected integer type, found '{}'", .{
11562 operand_ty,
11563 });
11564 }
11565 const target = sema.mod.getTarget();
11566 const bits = operand_ty.intInfo(target).bits;
11567 if (bits == 0) return Air.Inst.Ref.zero;
11568
11569 const result_ty = try Type.smallestUnsignedInt(sema.arena, bits);
11570
11571 const runtime_src = if (try sema.resolveMaybeUndefVal(block, operand_src, operand)) |val| {
11572 if (val.isUndef()) return sema.addConstUndef(result_ty);
11573 const result_val = try val.popCount(operand_ty, target, sema.arena);
11574 return sema.addConstant(result_ty, result_val);
11575 } else operand_src;
11576
11577 try sema.requireRuntimeBlock(block, runtime_src);
11578 return block.addTyOp(.popcount, result_ty, operand);
11579}
11580
1158111553fn zirByteSwap(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
1158211554 const inst_data = sema.code.instructions.items(.data)[inst].un_node;
1158311555 const src = inst_data.src();
src/codegen/llvm.zig+10-5
......@@ -2205,7 +2205,7 @@ pub const FuncGen = struct {
22052205 .get_union_tag => try self.airGetUnionTag(inst),
22062206 .clz => try self.airClzCtz(inst, "ctlz"),
22072207 .ctz => try self.airClzCtz(inst, "cttz"),
2208 .popcount => try self.airPopCount(inst, "ctpop"),
2208 .popcount => try self.airPopCount(inst),
22092209 .tag_name => try self.airTagName(inst),
22102210 .error_name => try self.airErrorName(inst),
22112211 .splat => try self.airSplat(inst),
......@@ -4364,7 +4364,7 @@ pub const FuncGen = struct {
43644364 }
43654365 }
43664366
4367 fn airPopCount(self: *FuncGen, inst: Air.Inst.Index, prefix: [*:0]const u8) !?*const llvm.Value {
4367 fn airPopCount(self: *FuncGen, inst: Air.Inst.Index) !?*const llvm.Value {
43684368 if (self.liveness.isUnused(inst)) return null;
43694369
43704370 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
......@@ -4372,11 +4372,16 @@ pub const FuncGen = struct {
43724372 const operand = try self.resolveInst(ty_op.operand);
43734373 const target = self.dg.module.getTarget();
43744374 const bits = operand_ty.intInfo(target).bits;
4375 const vec_len: ?u32 = switch (operand_ty.zigTypeTag()) {
4376 .Vector => operand_ty.vectorLen(),
4377 else => null,
4378 };
43754379
43764380 var fn_name_buf: [100]u8 = undefined;
4377 const llvm_fn_name = std.fmt.bufPrintZ(&fn_name_buf, "llvm.{s}.i{d}", .{
4378 prefix, bits,
4379 }) catch unreachable;
4381 const llvm_fn_name = if (vec_len) |len|
4382 std.fmt.bufPrintZ(&fn_name_buf, "llvm.ctpop.v{d}i{d}", .{ len, bits }) catch unreachable
4383 else
4384 std.fmt.bufPrintZ(&fn_name_buf, "llvm.ctpop.i{d}", .{bits}) catch unreachable;
43804385 const fn_val = self.dg.object.llvm_module.getNamedFunction(llvm_fn_name) orelse blk: {
43814386 const operand_llvm_ty = try self.dg.llvmType(operand_ty);
43824387 const param_types = [_]*const llvm.Type{operand_llvm_ty};
src/value.zig+27-18
......@@ -1303,6 +1303,33 @@ pub const Value = extern union {
13031303 }
13041304 }
13051305
1306 pub fn popCount(val: Value, ty: Type, target: Target) u64 {
1307 assert(!val.isUndef());
1308 switch (val.tag()) {
1309 .zero, .bool_false => return 0,
1310 .one, .bool_true => return 1,
1311
1312 .int_u64 => return @popCount(u64, val.castTag(.int_u64).?.data),
1313
1314 else => {
1315 const info = ty.intInfo(target);
1316
1317 var buffer: Value.BigIntSpace = undefined;
1318 const operand_bigint = val.toBigInt(&buffer);
1319
1320 var limbs_buffer: [4]std.math.big.Limb = undefined;
1321 var result_bigint = BigIntMutable{
1322 .limbs = &limbs_buffer,
1323 .positive = undefined,
1324 .len = undefined,
1325 };
1326 result_bigint.popCount(operand_bigint, info.bits);
1327
1328 return result_bigint.toConst().to(u64) catch unreachable;
1329 },
1330 }
1331 }
1332
13061333 /// Asserts the value is an integer and not undefined.
13071334 /// Returns the number of bits the value requires to represent stored in twos complement form.
13081335 pub fn intBitCountTwosComp(self: Value, target: Target) usize {
......@@ -1340,24 +1367,6 @@ pub const Value = extern union {
13401367 }
13411368 }
13421369
1343 pub fn popCount(val: Value, ty: Type, target: Target, arena: Allocator) !Value {
1344 assert(!val.isUndef());
1345
1346 const info = ty.intInfo(target);
1347
1348 var buffer: Value.BigIntSpace = undefined;
1349 const operand_bigint = val.toBigInt(&buffer);
1350
1351 const limbs = try arena.alloc(
1352 std.math.big.Limb,
1353 std.math.big.int.calcTwosCompLimbCount(info.bits),
1354 );
1355 var result_bigint = BigIntMutable{ .limbs = limbs, .positive = undefined, .len = undefined };
1356 result_bigint.popCount(operand_bigint, info.bits);
1357
1358 return fromBigInt(arena, result_bigint.toConst());
1359 }
1360
13611370 /// Asserts the value is an integer, and the destination type is ComptimeInt or Int.
13621371 pub fn intFitsInType(self: Value, ty: Type, target: Target) bool {
13631372 switch (self.tag()) {
test/behavior.zig-1
......@@ -153,7 +153,6 @@ test {
153153 _ = @import("behavior/ir_block_deps.zig");
154154 _ = @import("behavior/misc.zig");
155155 _ = @import("behavior/muladd.zig");
156 _ = @import("behavior/popcount_stage1.zig");
157156 _ = @import("behavior/reflection.zig");
158157 _ = @import("behavior/select.zig");
159158 _ = @import("behavior/shuffle.zig");
test/behavior/popcount.zig+20-1
......@@ -1,7 +1,6 @@
11const std = @import("std");
22const expect = std.testing.expect;
33const expectEqual = std.testing.expectEqual;
4const Vector = std.meta.Vector;
54
65test "@popCount integers" {
76 comptime try testPopCountIntegers();
......@@ -44,3 +43,23 @@ fn testPopCountIntegers() !void {
4443 try expect(@popCount(i128, @as(i128, 0b11111111000110001100010000100001000011000011100101010001)) == 24);
4544 }
4645}
46
47test "@popCount vectors" {
48 comptime try testPopCountVectors();
49 try testPopCountVectors();
50}
51
52fn testPopCountVectors() !void {
53 {
54 var x: @Vector(8, u32) = [1]u32{0xffffffff} ** 8;
55 const expected = [1]u6{32} ** 8;
56 const result: [8]u6 = @popCount(u32, x);
57 try expect(std.mem.eql(u6, &expected, &result));
58 }
59 {
60 var x: @Vector(8, i16) = [1]i16{-1} ** 8;
61 const expected = [1]u5{16} ** 8;
62 const result: [8]u5 = @popCount(i16, x);
63 try expect(std.mem.eql(u5, &expected, &result));
64 }
65}
test/behavior/popcount_stage1.zig deleted-24
......@@ -1,24 +0,0 @@
1const std = @import("std");
2const expect = std.testing.expect;
3const expectEqual = std.testing.expectEqual;
4const Vector = std.meta.Vector;
5
6test "@popCount vectors" {
7 comptime try testPopCountVectors();
8 try testPopCountVectors();
9}
10
11fn testPopCountVectors() !void {
12 {
13 var x: Vector(8, u32) = [1]u32{0xffffffff} ** 8;
14 const expected = [1]u6{32} ** 8;
15 const result: [8]u6 = @popCount(u32, x);
16 try expect(std.mem.eql(u6, &expected, &result));
17 }
18 {
19 var x: Vector(8, i16) = [1]i16{-1} ** 8;
20 const expected = [1]u5{16} ** 8;
21 const result: [8]u5 = @popCount(i16, x);
22 try expect(std.mem.eql(u5, &expected, &result));
23 }
24}