authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-29 16:39:05-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-29 16:39:05-07:00
logd1cea16f5cd29eb143ff9b3302e7ec56731647ea
tree03d3c11e9997182ce3e7aa29fa325f17590b6b63
parentbb636cb3bffadf5a03e592b97a1968492f16f8cd

move std.BloomFilter to the standard library orphanage

2 files changed, 0 insertions(+), 266 deletions(-)

lib/std/bloom_filter.zig deleted-265
...@@ -1,265 +0,0 @@
1// SPDX-License-Identifier: MIT
2// Copyright (c) 2015-2020 Zig Contributors
3// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
4// The MIT license requires this copyright notice to be included in all copies
5// and substantial portions of the software.
6const builtin = @import("builtin");
7const std = @import("std.zig");
8const math = std.math;
9const debug = std.debug;
10const assert = std.debug.assert;
11const testing = std.testing;
12
13/// There is a trade off of how quickly to fill a bloom filter;
14/// the number of items is:
15/// n_items / K * ln(2)
16/// the rate of false positives is:
17/// (1-e^(-K*N/n_items))^K
18/// where N is the number of items
19pub fn BloomFilter(
20 /// Size of bloom filter in cells, must be a power of two.
21 comptime n_items: usize,
22 /// Number of cells to set per item
23 comptime K: usize,
24 /// Cell type, should be:
25 /// - `bool` for a standard bloom filter
26 /// - an unsigned integer type for a counting bloom filter
27 comptime Cell: type,
28 /// endianess of the Cell
29 comptime endian: builtin.Endian,
30 /// Hash function to use
31 comptime hash: fn (out: []u8, Ki: usize, in: []const u8) void,
32) type {
33 assert(n_items > 0);
34 assert(math.isPowerOfTwo(n_items));
35 assert(K > 0);
36 const cellEmpty = if (Cell == bool) false else @as(Cell, 0);
37 const cellMax = if (Cell == bool) true else math.maxInt(Cell);
38 const n_bytes = (n_items * comptime std.meta.bitCount(Cell)) / 8;
39 assert(n_bytes > 0);
40 const Io = std.packed_int_array.PackedIntIo(Cell, endian);
41
42 return struct {
43 const Self = @This();
44 pub const items = n_items;
45 pub const Index = math.IntFittingRange(0, n_items - 1);
46
47 data: [n_bytes]u8 = [_]u8{0} ** n_bytes,
48
49 pub fn reset(self: *Self) void {
50 std.mem.set(u8, self.data[0..], 0);
51 }
52
53 pub fn @"union"(x: Self, y: Self) Self {
54 var r = Self{ .data = undefined };
55 inline for (x.data) |v, i| {
56 r.data[i] = v | y.data[i];
57 }
58 return r;
59 }
60
61 pub fn intersection(x: Self, y: Self) Self {
62 var r = Self{ .data = undefined };
63 inline for (x.data) |v, i| {
64 r.data[i] = v & y.data[i];
65 }
66 return r;
67 }
68
69 pub fn getCell(self: Self, cell: Index) Cell {
70 return Io.get(&self.data, cell, 0);
71 }
72
73 pub fn incrementCell(self: *Self, cell: Index) void {
74 if (Cell == bool or Cell == u1) {
75 // skip the 'get' operation
76 Io.set(&self.data, cell, 0, cellMax);
77 } else {
78 const old = Io.get(&self.data, cell, 0);
79 if (old != cellMax) {
80 Io.set(&self.data, cell, 0, old + 1);
81 }
82 }
83 }
84
85 pub fn clearCell(self: *Self, cell: Index) void {
86 Io.set(&self.data, cell, 0, cellEmpty);
87 }
88
89 pub fn add(self: *Self, item: []const u8) void {
90 comptime var i = 0;
91 inline while (i < K) : (i += 1) {
92 var K_th_bit: packed struct {
93 x: Index,
94 } = undefined;
95 hash(std.mem.asBytes(&K_th_bit), i, item);
96 incrementCell(self, K_th_bit.x);
97 }
98 }
99
100 pub fn contains(self: Self, item: []const u8) bool {
101 comptime var i = 0;
102 inline while (i < K) : (i += 1) {
103 var K_th_bit: packed struct {
104 x: Index,
105 } = undefined;
106 hash(std.mem.asBytes(&K_th_bit), i, item);
107 if (getCell(self, K_th_bit.x) == cellEmpty)
108 return false;
109 }
110 return true;
111 }
112
113 pub fn resize(self: Self, comptime newsize: usize) BloomFilter(newsize, K, Cell, endian, hash) {
114 var r: BloomFilter(newsize, K, Cell, endian, hash) = undefined;
115 if (newsize < n_items) {
116 std.mem.copy(u8, r.data[0..], self.data[0..r.data.len]);
117 var copied: usize = r.data.len;
118 while (copied < self.data.len) : (copied += r.data.len) {
119 for (self.data[copied .. copied + r.data.len]) |s, i| {
120 r.data[i] |= s;
121 }
122 }
123 } else if (newsize == n_items) {
124 r = self;
125 } else if (newsize > n_items) {
126 var copied: usize = 0;
127 while (copied < r.data.len) : (copied += self.data.len) {
128 std.mem.copy(u8, r.data[copied .. copied + self.data.len], &self.data);
129 }
130 }
131 return r;
132 }
133
134 /// Returns number of non-zero cells
135 pub fn popCount(self: Self) Index {
136 var n: Index = 0;
137 if (Cell == bool or Cell == u1) {
138 for (self.data) |b, i| {
139 n += @popCount(u8, b);
140 }
141 } else {
142 var i: usize = 0;
143 while (i < n_items) : (i += 1) {
144 const cell = self.getCell(@intCast(Index, i));
145 n += if (if (Cell == bool) cell else cell > 0) @as(Index, 1) else @as(Index, 0);
146 }
147 }
148 return n;
149 }
150
151 pub fn estimateItems(self: Self) f64 {
152 const m = comptime @intToFloat(f64, n_items);
153 const k = comptime @intToFloat(f64, K);
154 const X = @intToFloat(f64, self.popCount());
155 return (comptime (-m / k)) * math.log1p(X * comptime (-1 / m));
156 }
157 };
158}
159
160fn hashFunc(out: []u8, Ki: usize, in: []const u8) void {
161 var st = std.crypto.hash.Gimli.init(.{});
162 st.update(std.mem.asBytes(&Ki));
163 st.update(in);
164 st.final(out);
165}
166
167test "std.BloomFilter" {
168 // https://github.com/ziglang/zig/issues/5127
169 if (std.Target.current.cpu.arch == .mips) return error.SkipZigTest;
170
171 inline for ([_]type{ bool, u1, u2, u3, u4 }) |Cell| {
172 const emptyCell = if (Cell == bool) false else @as(Cell, 0);
173 const BF = BloomFilter(128 * 8, 8, Cell, builtin.endian, hashFunc);
174 var bf = BF{};
175 var i: usize = undefined;
176 // confirm that it is initialised to the empty filter
177 i = 0;
178 while (i < BF.items) : (i += 1) {
179 testing.expectEqual(emptyCell, bf.getCell(@intCast(BF.Index, i)));
180 }
181 testing.expectEqual(@as(BF.Index, 0), bf.popCount());
182 testing.expectEqual(@as(f64, 0), bf.estimateItems());
183 // fill in a few items
184 bf.incrementCell(42);
185 bf.incrementCell(255);
186 bf.incrementCell(256);
187 bf.incrementCell(257);
188 // check that they were set
189 testing.expectEqual(true, bf.getCell(42) != emptyCell);
190 testing.expectEqual(true, bf.getCell(255) != emptyCell);
191 testing.expectEqual(true, bf.getCell(256) != emptyCell);
192 testing.expectEqual(true, bf.getCell(257) != emptyCell);
193 // clear just one of them; make sure the rest are still set
194 bf.clearCell(256);
195 testing.expectEqual(true, bf.getCell(42) != emptyCell);
196 testing.expectEqual(true, bf.getCell(255) != emptyCell);
197 testing.expectEqual(false, bf.getCell(256) != emptyCell);
198 testing.expectEqual(true, bf.getCell(257) != emptyCell);
199 // reset any of the ones we've set and confirm we're back to the empty filter
200 bf.clearCell(42);
201 bf.clearCell(255);
202 bf.clearCell(257);
203 i = 0;
204 while (i < BF.items) : (i += 1) {
205 testing.expectEqual(emptyCell, bf.getCell(@intCast(BF.Index, i)));
206 }
207 testing.expectEqual(@as(BF.Index, 0), bf.popCount());
208 testing.expectEqual(@as(f64, 0), bf.estimateItems());
209
210 // Lets add a string
211 bf.add("foo");
212 testing.expectEqual(true, bf.contains("foo"));
213 {
214 // try adding same string again. make sure popcount is the same
215 const old_popcount = bf.popCount();
216 testing.expect(old_popcount > 0);
217 bf.add("foo");
218 testing.expectEqual(true, bf.contains("foo"));
219 testing.expectEqual(old_popcount, bf.popCount());
220 }
221
222 // Get back to empty filter via .reset
223 bf.reset();
224 // Double check that .reset worked
225 i = 0;
226 while (i < BF.items) : (i += 1) {
227 testing.expectEqual(emptyCell, bf.getCell(@intCast(BF.Index, i)));
228 }
229 testing.expectEqual(@as(BF.Index, 0), bf.popCount());
230 testing.expectEqual(@as(f64, 0), bf.estimateItems());
231
232 comptime var teststrings = [_][]const u8{
233 "foo",
234 "bar",
235 "a longer string",
236 "some more",
237 "the quick brown fox",
238 "unique string",
239 };
240 inline for (teststrings) |str| {
241 bf.add(str);
242 }
243 inline for (teststrings) |str| {
244 testing.expectEqual(true, bf.contains(str));
245 }
246
247 { // estimate should be close for low packing
248 const est = bf.estimateItems();
249 testing.expect(est > @intToFloat(f64, teststrings.len) - 1);
250 testing.expect(est < @intToFloat(f64, teststrings.len) + 1);
251 }
252
253 const larger_bf = bf.resize(4096);
254 inline for (teststrings) |str| {
255 testing.expectEqual(true, larger_bf.contains(str));
256 }
257 testing.expectEqual(@as(u12, bf.popCount()) * (4096 / 1024), larger_bf.popCount());
258
259 const smaller_bf = bf.resize(64);
260 inline for (teststrings) |str| {
261 testing.expectEqual(true, smaller_bf.contains(str));
262 }
263 testing.expect(bf.popCount() <= @as(u10, smaller_bf.popCount()) * (1024 / 64));
264 }
265}
lib/std/std.zig-1
...@@ -14,7 +14,6 @@ pub const AutoArrayHashMap = array_hash_map.AutoArrayHashMap;...@@ -14,7 +14,6 @@ pub const AutoArrayHashMap = array_hash_map.AutoArrayHashMap;
14pub const AutoArrayHashMapUnmanaged = array_hash_map.AutoArrayHashMapUnmanaged;14pub const AutoArrayHashMapUnmanaged = array_hash_map.AutoArrayHashMapUnmanaged;
15pub const AutoHashMap = hash_map.AutoHashMap;15pub const AutoHashMap = hash_map.AutoHashMap;
16pub const AutoHashMapUnmanaged = hash_map.AutoHashMapUnmanaged;16pub const AutoHashMapUnmanaged = hash_map.AutoHashMapUnmanaged;
17pub const BloomFilter = @import("bloom_filter.zig").BloomFilter;
18pub const BufMap = @import("buf_map.zig").BufMap;17pub const BufMap = @import("buf_map.zig").BufMap;
19pub const BufSet = @import("buf_set.zig").BufSet;18pub const BufSet = @import("buf_set.zig").BufSet;
20pub const ChildProcess = @import("child_process.zig").ChildProcess;19pub const ChildProcess = @import("child_process.zig").ChildProcess;