From e40513e97ff57960165b30a6b9fecfaad95bd1aa Mon Sep 17 00:00:00 2001 From: Benjamin Feng Date: Wed, 31 Jul 2019 21:26:39 -0500 Subject: [PATCH 01/23] Add builder.findProgram test and fix references --- std/build.zig | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/std/build.zig b/std/build.zig index 3a0c34c8d1ae2ad9dde4f4b304623b8067631ce6..acd3a1e33bdcc502d990eb95d528ea972de749d4 100644 --- a/std/build.zig +++ b/std/build.zig @@ -805,7 +805,7 @@ pub const Builder = struct { return name; } const full_path = try fs.path.join(self.allocator, [_][]const u8{ search_prefix, "bin", self.fmt("{}{}", name, exe_extension) }); - if (fs.path.real(self.allocator, full_path)) |real_path| { + if (fs.realpathAlloc(self.allocator, full_path)) |real_path| { return real_path; } else |_| { continue; @@ -817,10 +817,10 @@ pub const Builder = struct { if (fs.path.isAbsolute(name)) { return name; } - var it = mem.tokenize(PATH, []u8{fs.path.delimiter}); + var it = mem.tokenize(PATH, [_]u8{fs.path.delimiter}); while (it.next()) |path| { const full_path = try fs.path.join(self.allocator, [_][]const u8{ path, self.fmt("{}{}", name, exe_extension) }); - if (fs.path.real(self.allocator, full_path)) |real_path| { + if (fs.realpathAlloc(self.allocator, full_path)) |real_path| { return real_path; } else |_| { continue; @@ -834,7 +834,7 @@ pub const Builder = struct { } for (paths) |path| { const full_path = try fs.path.join(self.allocator, [_][]const u8{ path, self.fmt("{}{}", name, exe_extension) }); - if (fs.path.real(self.allocator, full_path)) |real_path| { + if (fs.realpathAlloc(self.allocator, full_path)) |real_path| { return real_path; } else |_| { continue; @@ -904,6 +904,15 @@ pub const Builder = struct { } }; +test "builder.findProgram compiles" { + //allocator: *Allocator, + //zig_exe: []const u8, + //build_root: []const u8, + //cache_root: []const u8, + const builder = try Builder.create(std.heap.direct_allocator, "zig", "zig-cache", "zig-cache"); + _ = builder.findProgram([_][]const u8{}, [_][]const u8{}) catch null; +} + pub const Version = struct { major: u32, minor: u32, -- 2.54.0 From 327abdba0b5de1f1eeef32e19b6a16ec4c7ec323 Mon Sep 17 00:00:00 2001 From: Benjamin Feng Date: Wed, 31 Jul 2019 21:28:25 -0500 Subject: [PATCH 02/23] More current style for error handling --- std/build.zig | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/std/build.zig b/std/build.zig index acd3a1e33bdcc502d990eb95d528ea972de749d4..997e2ab9014c0b6183cf414a2b58133c38f16d32 100644 --- a/std/build.zig +++ b/std/build.zig @@ -805,11 +805,7 @@ pub const Builder = struct { return name; } const full_path = try fs.path.join(self.allocator, [_][]const u8{ search_prefix, "bin", self.fmt("{}{}", name, exe_extension) }); - if (fs.realpathAlloc(self.allocator, full_path)) |real_path| { - return real_path; - } else |_| { - continue; - } + return fs.realpathAlloc(self.allocator, full_path) catch continue; } } if (self.env_map.get("PATH")) |PATH| { @@ -820,11 +816,7 @@ pub const Builder = struct { var it = mem.tokenize(PATH, [_]u8{fs.path.delimiter}); while (it.next()) |path| { const full_path = try fs.path.join(self.allocator, [_][]const u8{ path, self.fmt("{}{}", name, exe_extension) }); - if (fs.realpathAlloc(self.allocator, full_path)) |real_path| { - return real_path; - } else |_| { - continue; - } + return fs.realpathAlloc(self.allocator, full_path) catch continue; } } } @@ -834,11 +826,7 @@ pub const Builder = struct { } for (paths) |path| { const full_path = try fs.path.join(self.allocator, [_][]const u8{ path, self.fmt("{}{}", name, exe_extension) }); - if (fs.realpathAlloc(self.allocator, full_path)) |real_path| { - return real_path; - } else |_| { - continue; - } + return fs.realpathAlloc(self.allocator, full_path) catch continue; } } return error.FileNotFound; -- 2.54.0 From 723aea8369375d17d19bc1e6d02dbf6eb4d9ba49 Mon Sep 17 00:00:00 2001 From: Benjamin Feng Date: Wed, 31 Jul 2019 22:07:17 -0500 Subject: [PATCH 03/23] Default wasm-lib prefix to empty --- std/build.zig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/std/build.zig b/std/build.zig index 997e2ab9014c0b6183cf414a2b58133c38f16d32..d63e1ae8998e644084b01a4af49de63f6c9582da 100644 --- a/std/build.zig +++ b/std/build.zig @@ -1119,6 +1119,9 @@ pub const Target = union(enum) { } pub fn libPrefix(self: Target) []const u8 { + if (self.isWasm()) { + return ""; + } switch (self.getAbi()) { .msvc => return "", else => return "lib", -- 2.54.0 From c0c228b758150e017185dc280ef020ec3b7efdcf Mon Sep 17 00:00:00 2001 From: Euan Torano Date: Fri, 2 Aug 2019 13:19:49 +0100 Subject: [PATCH 04/23] Check if /dev/urandom is a character device --- std/os.zig | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/std/os.zig b/std/os.zig index 9ff2e8f87f22a71d30d35ed834e70bc2e29e1a63..da1d71d6fcb204291d12dd99876a2fcd781e76e6 100644 --- a/std/os.zig +++ b/std/os.zig @@ -133,6 +133,11 @@ fn getRandomBytesDevURandom(buf: []u8) !void { const fd = try openC(c"/dev/urandom", O_RDONLY | O_CLOEXEC, 0); defer close(fd); + const st = try fstat(fd); + if (!S_ISCHR(st.mode)) { + return OpenError.Unexpected; + } + const stream = &std.fs.File.openHandle(fd).inStream().stream; stream.readNoEof(buf) catch return error.Unexpected; } -- 2.54.0 From 1583efda69622b9c7809419a4320f4e8ea6fd4e3 Mon Sep 17 00:00:00 2001 From: Euan Torano Date: Fri, 2 Aug 2019 15:44:58 +0100 Subject: [PATCH 05/23] Fix call to S_ISCHR and implement for Mac --- std/os.zig | 4 ++-- std/os/bits/darwin.zig | 50 ++++++++++++++++++++++++++++++++++++++++++ std/os/darwin.zig | 1 + 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/std/os.zig b/std/os.zig index da1d71d6fcb204291d12dd99876a2fcd781e76e6..ce240bb95646a3068dc6003461669a6f406d2865 100644 --- a/std/os.zig +++ b/std/os.zig @@ -134,8 +134,8 @@ fn getRandomBytesDevURandom(buf: []u8) !void { defer close(fd); const st = try fstat(fd); - if (!S_ISCHR(st.mode)) { - return OpenError.Unexpected; + if (!system.S_ISCHR(st.mode)) { + return error.Unexpected; } const stream = &std.fs.File.openHandle(fd).inStream().stream; diff --git a/std/os/bits/darwin.zig b/std/os/bits/darwin.zig index b8d229dbe95f17c0b37f86362de9001173844130..dd4d46287e61dedf704c91398178eedf9b08ebe4 100644 --- a/std/os/bits/darwin.zig +++ b/std/os/bits/darwin.zig @@ -1116,3 +1116,53 @@ pub const stack_t = extern struct { ss_size: isize, ss_flags: i32, }; + +pub const S_IFMT = 0o170000; + +pub const S_IFIFO = 0o010000; +pub const S_IFCHR = 0o020000; +pub const S_IFDIR = 0o040000; +pub const S_IFBLK = 0o060000; +pub const S_IFREG = 0o100000; +pub const S_IFLNK = 0o120000; +pub const S_IFSOCK = 0o140000; +pub const S_IFWHT = 0o160000; + +pub const S_ISUID = 0o4000; +pub const S_ISGID = 0o2000; +pub const S_ISVTX = 0o1000; +pub const S_IRUSR = 0o400; +pub const S_IWUSR = 0o200; +pub const S_IXUSR = 0o100; + +pub fn S_ISFIFO(m: u32) bool { + return m & S_IFMT == S_IFIFO; +} + +pub fn S_ISCHR(m: u32) bool { + return m & S_IFMT == S_IFCHR; +} + +pub fn S_ISDIR(m: u32) bool { + return m & S_IFMT == S_IFDIR; +} + +pub fn S_ISBLK(m: u32) bool { + return m & S_IFMT == S_IFBLK; +} + +pub fn S_ISREG(m: u32) bool { + return m & S_IFMT == S_IFREG; +} + +pub fn S_ISLNK(m: u32) bool { + return m & S_IFMT == S_IFLNK; +} + +pub fn S_ISSOCK(m: u32) bool { + return m & S_IFMT == S_IFSOCK; +} + +pub fn S_IWHT(m: u32) bool { + return m & S_IFMT == S_IFWHT; +} diff --git a/std/os/darwin.zig b/std/os/darwin.zig index 67ce9a06cff54ead18902b8f3e69611128130466..c2b6801e22fa6cc6dc2a1ccce3b97631f62b0031 100644 --- a/std/os/darwin.zig +++ b/std/os/darwin.zig @@ -5,3 +5,4 @@ pub const is_the_target = switch (builtin.os) { else => false, }; pub usingnamespace std.c; +pub usingnamespace @import("bits.zig"); \ No newline at end of file -- 2.54.0 From 24fbd1f1d58125f34ca2ae52a592028f39412aa9 Mon Sep 17 00:00:00 2001 From: Euan Torano Date: Fri, 2 Aug 2019 15:59:40 +0100 Subject: [PATCH 06/23] Add S_* values for freebsd. --- std/os/bits/darwin.zig | 9 +++++++ std/os/bits/freebsd.zig | 59 +++++++++++++++++++++++++++++++++++++++++ std/os/freebsd.zig | 1 + 3 files changed, 69 insertions(+) diff --git a/std/os/bits/darwin.zig b/std/os/bits/darwin.zig index dd4d46287e61dedf704c91398178eedf9b08ebe4..483d4cda908f157d9a585339ee07cfaa7b6ad9b0 100644 --- a/std/os/bits/darwin.zig +++ b/std/os/bits/darwin.zig @@ -1131,9 +1131,18 @@ pub const S_IFWHT = 0o160000; pub const S_ISUID = 0o4000; pub const S_ISGID = 0o2000; pub const S_ISVTX = 0o1000; +pub const S_IRWXU = 0o700; pub const S_IRUSR = 0o400; pub const S_IWUSR = 0o200; pub const S_IXUSR = 0o100; +pub const S_IRWXG = 0o070; +pub const S_IRGRP = 0o040; +pub const S_IWGRP = 0o020; +pub const S_IXGRP = 0o010; +pub const S_IRWXO = 0o007; +pub const S_IROTH = 0o004; +pub const S_IWOTH = 0o002; +pub const S_IXOTH = 0o001; pub fn S_ISFIFO(m: u32) bool { return m & S_IFMT == S_IFIFO; diff --git a/std/os/bits/freebsd.zig b/std/os/bits/freebsd.zig index 198857983e67a837657081c39edbad953a3f8d1c..45432a6c076d82ac2845651a70e0f10ce89d8408 100644 --- a/std/os/bits/freebsd.zig +++ b/std/os/bits/freebsd.zig @@ -876,3 +876,62 @@ pub const stack_t = extern struct { ss_size: isize, ss_flags: i32, }; + +pub const S_IFMT = 0o170000; + +pub const S_IFIFO = 0o010000; +pub const S_IFCHR = 0o020000; +pub const S_IFDIR = 0o040000; +pub const S_IFBLK = 0o060000; +pub const S_IFREG = 0o100000; +pub const S_IFLNK = 0o120000; +pub const S_IFSOCK = 0o140000; +pub const S_IFWHT = 0o160000; + +pub const S_ISUID = 0o4000; +pub const S_ISGID = 0o2000; +pub const S_ISVTX = 0o1000; +pub const S_IRWXU = 0o700; +pub const S_IRUSR = 0o400; +pub const S_IWUSR = 0o200; +pub const S_IXUSR = 0o100; +pub const S_IRWXG = 0o070; +pub const S_IRGRP = 0o040; +pub const S_IWGRP = 0o020; +pub const S_IXGRP = 0o010; +pub const S_IRWXO = 0o007; +pub const S_IROTH = 0o004; +pub const S_IWOTH = 0o002; +pub const S_IXOTH = 0o001; + +pub fn S_ISFIFO(m: u32) bool { + return m & S_IFMT == S_IFIFO; +} + +pub fn S_ISCHR(m: u32) bool { + return m & S_IFMT == S_IFCHR; +} + +pub fn S_ISDIR(m: u32) bool { + return m & S_IFMT == S_IFDIR; +} + +pub fn S_ISBLK(m: u32) bool { + return m & S_IFMT == S_IFBLK; +} + +pub fn S_ISREG(m: u32) bool { + return m & S_IFMT == S_IFREG; +} + +pub fn S_ISLNK(m: u32) bool { + return m & S_IFMT == S_IFLNK; +} + +pub fn S_ISSOCK(m: u32) bool { + return m & S_IFMT == S_IFSOCK; +} + +pub fn S_IWHT(m: u32) bool { + return m & S_IFMT == S_IFWHT; +} diff --git a/std/os/freebsd.zig b/std/os/freebsd.zig index d418ccd415b8bf97788e6128ddb56943048479ef..e9efe6492060895796bc897b9eb62f38ff335847 100644 --- a/std/os/freebsd.zig +++ b/std/os/freebsd.zig @@ -2,3 +2,4 @@ const std = @import("../std.zig"); const builtin = @import("builtin"); pub const is_the_target = builtin.os == .freebsd; pub usingnamespace std.c; +pub usingnamespace @import("bits.zig"); \ No newline at end of file -- 2.54.0 From e68fee39847712f9316628e6efe8daa8fc13a5a5 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 2 Aug 2019 18:53:56 -0400 Subject: [PATCH 07/23] docs: add atomicrmw operations list --- doc/langref.html.in | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/doc/langref.html.in b/doc/langref.html.in index b5fe464c35d7ad06062503eee9d63b2eaea600b9..ac381e00b28323619b42dd25ba11184d7628243c 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -6330,6 +6330,22 @@ comptime { TODO right now bool is not accepted. Also I think we could make non powers of 2 work fine, maybe we can remove this restriction

+

+ Supported operations: +

+
    +
  • {#syntax#}.Xchg{#endsyntax#} - stores the operand unmodified.
    • +
  • {#syntax#}.Add{#endsyntax#} - for integers, twos complement wraparound addition. + Also supports {#link|Floats#}.
    • +
  • {#syntax#}.Sub{#endsyntax#} - for integers, twos complement wraparound subtraction. + Also supports {#link|Floats#}.
    • +
  • {#syntax#}.And{#endsyntax#} - bitwise and
    • +
  • {#syntax#}.Nand{#endsyntax#} - bitwise nand
    • +
  • {#syntax#}.Or{#endsyntax#} - bitwise or
    • +
  • {#syntax#}.Xor{#endsyntax#} - bitwise xor
    • +
  • {#syntax#}.Max{#endsyntax#} - stores the operand if it is larger. Supports integers and floats.
    • +
  • {#syntax#}.Min{#endsyntax#} - stores the operand if it is smaller. Supports integers and floats.
    • +
{#header_close#} {#header_open|@bitCast#} 
{#syntax#}@bitCast(comptime DestType: type, value: var) DestType{#endsyntax#}
-- 2.54.0 From 57830e43ee79dd0ceafc96fdad6c52e73edf1420 Mon Sep 17 00:00:00 2001 From: Christoffer Rasmussen Date: Wed, 31 Jul 2019 15:17:09 +0200 Subject: [PATCH 08/23] Fix public function lookup Previously it did not work, as lookup did not pass a correct parent type to doLookup. Expected *?*Node, got ?*Node. --- std/rb.zig | 51 +++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/std/rb.zig b/std/rb.zig index b5935a2eacbf2ce116cba1e403c5026e534c5061..0b84950544fe4defb9ba2994487c377803447f79 100644 --- a/std/rb.zig +++ b/std/rb.zig @@ -234,10 +234,13 @@ pub const Tree = struct { return null; } + /// lookup searches for the value of key, using binary search. It will + /// return a pointer to the node if it is there, otherwise it will return null. + /// Complexity guaranteed O(log n), where n is the number of nodes book-kept + /// by tree. pub fn lookup(tree: *Tree, key: *Node) ?*Node { - var parent: *Node = undefined; + var parent: ?*Node = undefined; var is_left: bool = undefined; - return doLookup(key, tree, &parent, &is_left); } @@ -545,3 +548,47 @@ test "rb" { num = testGetNumber(num.node.next().?); } } + + +test "inserting and looking up" { + var tree: Tree = undefined; + tree.init(testCompare); + var number: testNumber = undefined; + number.value = 1000; + _ = tree.insert(&number.node); + var dup: testNumber = undefined; + //Assert that tuples with identical value fields finds the same pointer + dup.value = 1000; + assert(tree.lookup(&dup.node) == &number.node); + //Assert that tuples with identical values do not clobber when inserted. + _ = tree.insert(&dup.node); + assert(tree.lookup(&dup.node) == &number.node); + assert(tree.lookup(&number.node) != &dup.node); + assert(testGetNumber(tree.lookup(&dup.node).?).value == testGetNumber(&dup.node).value); + //Assert that if looking for a non-existing value, return null. + var non_existing_value: testNumber = undefined; + non_existing_value.value = 1234; + assert(tree.lookup(&non_existing_value.node) == null); +} + +test "multiple inserts, followed by calling first and last" { + var tree: Tree = undefined; + tree.init(testCompare); + var zeroth: testNumber = undefined; + zeroth.value = 0; + var first: testNumber = undefined; + first.value = 1; + var second: testNumber = undefined; + second.value = 2; + var third: testNumber = undefined; + third.value = 3; + _ = tree.insert(&zeroth.node); + _ = tree.insert(&first.node); + _ = tree.insert(&second.node); + _ = tree.insert(&third.node); + assert(testGetNumber(tree.first().?).value == 0); + assert(testGetNumber(tree.last().?).value == 3); + var lookupNode: testNumber = undefined; + lookupNode.value = 3; + assert(tree.lookup(&lookupNode.node) == &third.node); +} -- 2.54.0 From 521aaf350185b5f01816b7a9ec604335edb3ac16 Mon Sep 17 00:00:00 2001 From: daurnimator Date: Sat, 3 Aug 2019 15:56:25 +1000 Subject: [PATCH 09/23] std: return Elf object from constructors instead of filling in pointer --- std/debug.zig | 3 +-- std/elf.zig | 10 ++++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/std/debug.zig b/std/debug.zig index 32f96d3e1559cf4cf7f08a8e21b4789367191e6f..d1c17343efe3ff02711e862c70f5b5a0c2f1617d 100644 --- a/std/debug.zig +++ b/std/debug.zig @@ -1024,8 +1024,7 @@ pub fn openElfDebugInfo( elf_seekable_stream: *DwarfSeekableStream, elf_in_stream: *DwarfInStream, ) !DwarfInfo { - var efile: elf.Elf = undefined; - try efile.openStream(allocator, elf_seekable_stream, elf_in_stream); + var efile = try elf.Elf.openStream(allocator, elf_seekable_stream, elf_in_stream); errdefer efile.close(); var di = DwarfInfo{ diff --git a/std/elf.zig b/std/elf.zig index c605a177a5acb5f3b12d639785541ad82104a361..0b3ea203989080a3c0ef45956a6cd40df5f01586 100644 --- a/std/elf.zig +++ b/std/elf.zig @@ -371,21 +371,21 @@ pub const Elf = struct { prealloc_file: File, /// Call close when done. - pub fn openPath(elf: *Elf, allocator: *mem.Allocator, path: []const u8) !void { + pub fn openPath(allocator: *mem.Allocator, path: []const u8) !Elf { @compileError("TODO implement"); } /// Call close when done. - pub fn openFile(elf: *Elf, allocator: *mem.Allocator, file: File) !void { + pub fn openFile(allocator: *mem.Allocator, file: File) !Elf { @compileError("TODO implement"); } pub fn openStream( - elf: *Elf, allocator: *mem.Allocator, seekable_stream: *io.SeekableStream(anyerror, anyerror), in: *io.InStream(anyerror), - ) !void { + ) !Elf { + var elf: Elf = undefined; elf.auto_close_stream = false; elf.allocator = allocator; elf.seekable_stream = seekable_stream; @@ -523,6 +523,8 @@ pub const Elf = struct { // not a string table return error.InvalidFormat; } + + return elf; } pub fn close(elf: *Elf) void { -- 2.54.0 From 8e157ccb23885b2a9a8a5d66c3955fc560fd0074 Mon Sep 17 00:00:00 2001 From: Euan T Date: Sat, 3 Aug 2019 07:50:30 +0100 Subject: [PATCH 10/23] Update returned error return when not a character device. Co-Authored-By: Andrew Kelley --- std/os.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/std/os.zig b/std/os.zig index ce240bb95646a3068dc6003461669a6f406d2865..2caab634d66b4b495c9270cbcb3b5e38e234daee 100644 --- a/std/os.zig +++ b/std/os.zig @@ -135,7 +135,7 @@ fn getRandomBytesDevURandom(buf: []u8) !void { const st = try fstat(fd); if (!system.S_ISCHR(st.mode)) { - return error.Unexpected; + return error.NoDevice; } const stream = &std.fs.File.openHandle(fd).inStream().stream; -- 2.54.0 From 08251fbc544aa03c77d4c311e267689592432282 Mon Sep 17 00:00:00 2001 From: Euan T Date: Sat, 3 Aug 2019 07:51:36 +0100 Subject: [PATCH 11/23] Omit system namespace. Co-Authored-By: Andrew Kelley --- std/os.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/std/os.zig b/std/os.zig index 2caab634d66b4b495c9270cbcb3b5e38e234daee..c2010bf6a95b6df6dafcd3591d67d3f5ad9a32b9 100644 --- a/std/os.zig +++ b/std/os.zig @@ -134,7 +134,7 @@ fn getRandomBytesDevURandom(buf: []u8) !void { defer close(fd); const st = try fstat(fd); - if (!system.S_ISCHR(st.mode)) { + if (!S_ISCHR(st.mode)) { return error.NoDevice; } -- 2.54.0 From 887eac0219345763f1ae9c8d9efad6950f6bbfe6 Mon Sep 17 00:00:00 2001 From: daurnimator Date: Sun, 4 Aug 2019 16:27:36 +1000 Subject: [PATCH 12/23] std: remove elf.auto_close_stream and elf.prealloc_file --- std/elf.zig | 5 ----- 1 file changed, 5 deletions(-) diff --git a/std/elf.zig b/std/elf.zig index 0b3ea203989080a3c0ef45956a6cd40df5f01586..37635895fd226d588e70505094a47dbd6484617c 100644 --- a/std/elf.zig +++ b/std/elf.zig @@ -356,7 +356,6 @@ pub const SectionHeader = struct { pub const Elf = struct { seekable_stream: *io.SeekableStream(anyerror, anyerror), in_stream: *io.InStream(anyerror), - auto_close_stream: bool, is_64: bool, endian: builtin.Endian, file_type: FileType, @@ -368,7 +367,6 @@ pub const Elf = struct { string_section: *SectionHeader, section_headers: []SectionHeader, allocator: *mem.Allocator, - prealloc_file: File, /// Call close when done. pub fn openPath(allocator: *mem.Allocator, path: []const u8) !Elf { @@ -386,7 +384,6 @@ pub const Elf = struct { in: *io.InStream(anyerror), ) !Elf { var elf: Elf = undefined; - elf.auto_close_stream = false; elf.allocator = allocator; elf.seekable_stream = seekable_stream; elf.in_stream = in; @@ -529,8 +526,6 @@ pub const Elf = struct { pub fn close(elf: *Elf) void { elf.allocator.free(elf.section_headers); - - if (elf.auto_close_stream) elf.prealloc_file.close(); } pub fn findSection(elf: *Elf, name: []const u8) !?*SectionHeader { -- 2.54.0 From 6150da3df99b41f89ea01a72e6c1b76fe4c36f89 Mon Sep 17 00:00:00 2001 From: Sahnvour Date: Thu, 27 Jun 2019 23:21:35 +0200 Subject: [PATCH 13/23] direct port of wyhash v2 also inspired by https://github.com/ManDeJan/zig-wyhash --- std/hash.zig | 4 ++ std/hash/wyhash.zig | 99 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 std/hash/wyhash.zig diff --git a/std/hash.zig b/std/hash.zig index 148504aa3972df221202259aa24945b31fa2e185..723860da3bfd9fdc8c877eb52100840e07477642 100644 --- a/std/hash.zig +++ b/std/hash.zig @@ -16,6 +16,7 @@ pub const SipHash128 = siphash.SipHash128; pub const murmur = @import("hash/murmur.zig"); pub const Murmur2_32 = murmur.Murmur2_32; + pub const Murmur2_64 = murmur.Murmur2_64; pub const Murmur3_32 = murmur.Murmur3_32; @@ -23,6 +24,8 @@ pub const cityhash = @import("hash/cityhash.zig"); pub const CityHash32 = cityhash.CityHash32; pub const CityHash64 = cityhash.CityHash64; +pub const wyhash = @import("hash/wyhash.zig").hash; + test "hash" { _ = @import("hash/adler.zig"); _ = @import("hash/crc.zig"); @@ -30,4 +33,5 @@ test "hash" { _ = @import("hash/siphash.zig"); _ = @import("hash/murmur.zig"); _ = @import("hash/cityhash.zig"); + _ = @import("hash/wyhash.zig"); } diff --git a/std/hash/wyhash.zig b/std/hash/wyhash.zig new file mode 100644 index 0000000000000000000000000000000000000000..57efe8fd631eb69be71764da1a50f2ae914c9bcd --- /dev/null +++ b/std/hash/wyhash.zig @@ -0,0 +1,99 @@ +const std = @import("std"); +const mem = std.mem; + +const primes = [_]u64{ + 0xa0761d6478bd642f, + 0xe7037ed1a0b428db, + 0x8ebc6af09c88c6e3, + 0x589965cc75374cc3, + 0x1d8e4e27c47d124f, +}; + +fn read_bytes(comptime bytes: u8, data: []const u8) u64 { + return mem.readVarInt(u64, data[0..bytes], @import("builtin").endian); +} + +fn read_8bytes_swapped(data: []const u8) u64 { + return (read_bytes(4, data) << 32 | read_bytes(4, data[4..])); +} + +fn mum(a: u64, b: u64) u64 { + var r: u128 = @intCast(u128, a) * @intCast(u128, b); + r = (r >> 64) ^ r; + return @truncate(u64, r); +} + +fn mix0(a: u64, b: u64, seed: u64) u64 { + return mum(a ^ seed ^ primes[0], b ^ seed ^ primes[1]); +} + +fn mix1(a: u64, b: u64, seed: u64) u64 { + return mum(a ^ seed ^ primes[2], b ^ seed ^ primes[3]); +} + +pub fn hash(key: []const u8, initial_seed: u64) u64 { + var seed = initial_seed; + + var i: usize = 0; + while (i + 32 <= key.len) : (i += 32) { + seed = mix0( + read_bytes(8, key[i..]), + read_bytes(8, key[i + 8 ..]), + seed, + ) ^ mix1( + read_bytes(8, key[i + 16 ..]), + read_bytes(8, key[i + 24 ..]), + seed, + ); + } + + const rem_len = @truncate(u5, key.len); + const rem_key = key[i..]; + seed = switch (rem_len) { + 0 => seed, + 1 => mix0(read_bytes(1, rem_key), primes[4], seed), + 2 => mix0(read_bytes(2, rem_key), primes[4], seed), + 3 => mix0((read_bytes(2, rem_key) << 8) | read_bytes(1, rem_key[2..]), primes[4], seed), + 4 => mix0(read_bytes(4, rem_key), primes[4], seed), + 5 => mix0((read_bytes(4, rem_key) << 8) | read_bytes(1, rem_key[4..]), primes[4], seed), + 6 => mix0((read_bytes(4, rem_key) << 16) | read_bytes(2, rem_key[4..]), primes[4], seed), + 7 => mix0((read_bytes(4, rem_key) << 24) | (read_bytes(2, rem_key[4..]) << 8) | read_bytes(1, rem_key[6..]), primes[4], seed), + 8 => mix0(read_8bytes_swapped(rem_key), primes[4], seed), + 9 => mix0(read_8bytes_swapped(rem_key), read_bytes(1, rem_key[8..]), seed), + 10 => mix0(read_8bytes_swapped(rem_key), read_bytes(2, rem_key[8..]), seed), + 11 => mix0(read_8bytes_swapped(rem_key), (read_bytes(2, rem_key[8..]) << 8) | read_bytes(1, rem_key[10..]), seed), + 12 => mix0(read_8bytes_swapped(rem_key), read_bytes(4, rem_key[8..]), seed), + 13 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 8) | read_bytes(1, rem_key[12..]), seed), + 14 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 16) | read_bytes(2, rem_key[12..]), seed), + 15 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 24) | (read_bytes(2, rem_key[12..]) << 8) | read_bytes(1, rem_key[14..]), seed), + 16 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed), + 17 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(1, rem_key[16..]), primes[4], seed), + 18 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(2, rem_key[16..]), primes[4], seed), + 19 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(2, rem_key[16..]) << 8) | read_bytes(1, rem_key[18..]), primes[4], seed), + 20 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(4, rem_key[16..]), primes[4], seed), + 21 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 8) | read_bytes(1, rem_key[20..]), primes[4], seed), + 22 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 16) | read_bytes(2, rem_key[20..]), primes[4], seed), + 23 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 24) | (read_bytes(2, rem_key[20..]) << 8) | read_bytes(1, rem_key[22..]), primes[4], seed), + 24 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), primes[4], seed), + 25 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(1, rem_key[24..]), seed), + 26 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(2, rem_key[24..]), seed), + 27 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(2, rem_key[24..]) << 8) | read_bytes(1, rem_key[26..]), seed), + 28 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(4, rem_key[24..]), seed), + 29 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 8) | read_bytes(1, rem_key[28..]), seed), + 30 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 16) | read_bytes(2, rem_key[28..]), seed), + 31 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 24) | (read_bytes(2, rem_key[28..]) << 8) | read_bytes(1, rem_key[30..]), seed), + }; + + return mum(seed ^ key.len, primes[4]); +} + +test "test vectors" { + const expectEqual = std.testing.expectEqual; + expectEqual(hash("", 0), 0x0); + expectEqual(hash("a", 1), 0xbed235177f41d328); + expectEqual(hash("abc", 2), 0xbe348debe59b27c3); + expectEqual(hash("message digest", 3), 0x37320f657213a290); + expectEqual(hash("abcdefghijklmnopqrstuvwxyz", 4), 0xd0b270e1d8a7019c); + expectEqual(hash("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 5), 0x602a1894d3bbfe7f); + expectEqual(hash("12345678901234567890123456789012345678901234567890123456789012345678901234567890", 6), 0x829e9c148b75970e); +} -- 2.54.0 From 5bd407b27890fbed82891289e6a2bf2da93c2a41 Mon Sep 17 00:00:00 2001 From: Sahnvour Date: Sun, 30 Jun 2019 11:35:57 +0200 Subject: [PATCH 14/23] use wyhash in std's hashmap, and improve autoHash to handle more types and behave more correctly --- std/hash_map.zig | 268 +++++++++++++++++++++++++++++++---------------- 1 file changed, 175 insertions(+), 93 deletions(-) diff --git a/std/hash_map.zig b/std/hash_map.zig index c99d1d249038cbc2267b4b92162ba39a062d0a68..6a8679ccd0a4d6346bb99f51c35e3a7eb8eb0fe4 100644 --- a/std/hash_map.zig +++ b/std/hash_map.zig @@ -4,6 +4,8 @@ const assert = debug.assert; const testing = std.testing; const math = std.math; const mem = std.mem; +const meta = std.meta; +const wyhash = std.hash.wyhash; const Allocator = mem.Allocator; const builtin = @import("builtin"); @@ -448,15 +450,17 @@ test "iterator hash map" { try reset_map.putNoClobber(2, 22); try reset_map.putNoClobber(3, 33); + // TODO this test depends on the hashing algorithm, because it assumes the + // order of the elements in the hashmap. This should not be the case. var keys = [_]i32{ + 1, 3, 2, - 1, }; var values = [_]i32{ + 11, 33, 22, - 11, }; var it = reset_map.iterator(); @@ -518,8 +522,8 @@ pub fn getTrivialEqlFn(comptime K: type) (fn (K, K) bool) { pub fn getAutoHashFn(comptime K: type) (fn (K) u32) { return struct { fn hash(key: K) u32 { - comptime var rng = comptime std.rand.DefaultPrng.init(0); - return autoHash(key, &rng.random, u32); + const h = autoHash(key, 0); + return @truncate(u32, h); } }.hash; } @@ -527,114 +531,192 @@ pub fn getAutoHashFn(comptime K: type) (fn (K) u32) { pub fn getAutoEqlFn(comptime K: type) (fn (K, K) bool) { return struct { fn eql(a: K, b: K) bool { - return autoEql(a, b); + return meta.eql(a, b); } }.eql; } -// TODO improve these hash functions -pub fn autoHash(key: var, comptime rng: *std.rand.Random, comptime HashInt: type) HashInt { - switch (@typeInfo(@typeOf(key))) { +/// Provides generic hashing for any eligible type. +/// Only hashes `key` itself, pointers are not followed. +/// The underlying hashing algorithm is wyhash. +pub fn autoHash(key: var, seed: u64) u64 { + // We use the fact that wyhash takes an input seed to "chain" hasing when the + // key has multiple parts that are not necessarily contiguous in memory. + const Key = @typeOf(key); + switch (@typeInfo(Key)) { builtin.TypeId.NoReturn, builtin.TypeId.Opaque, builtin.TypeId.Undefined, builtin.TypeId.ArgTuple, + builtin.TypeId.Void, + builtin.TypeId.Null, + builtin.TypeId.BoundFn, + builtin.TypeId.ComptimeFloat, + builtin.TypeId.ComptimeInt, + builtin.TypeId.Type, + builtin.TypeId.EnumLiteral, => @compileError("cannot hash this type"), - builtin.TypeId.Void, - builtin.TypeId.Null, - => return 0, - - builtin.TypeId.Int => |info| { - const unsigned_x = @bitCast(@IntType(false, info.bits), key); - if (info.bits <= HashInt.bit_count) { - return HashInt(unsigned_x) ^ comptime rng.scalar(HashInt); - } else { - return @truncate(HashInt, unsigned_x ^ comptime rng.scalar(@typeOf(unsigned_x))); + builtin.TypeId.Int => return wyhash(std.mem.asBytes(&key), seed), + + builtin.TypeId.Float => |info| return autoHash(@bitCast(@IntType(false, info.bits), key), seed), + + builtin.TypeId.Bool => return autoHash(@boolToInt(key), seed), + builtin.TypeId.Enum => return autoHash(@enumToInt(key), seed), + builtin.TypeId.ErrorSet => return autoHash(@errorToInt(key), seed), + builtin.TypeId.Promise, builtin.TypeId.Fn => return autoHash(@ptrToInt(key), seed), + + builtin.TypeId.Pointer => |info| return switch (info.size) { + builtin.TypeInfo.Pointer.Size.One, + builtin.TypeInfo.Pointer.Size.Many, + builtin.TypeInfo.Pointer.Size.C, + => return autoHash(@ptrToInt(key), seed), + + builtin.TypeInfo.Pointer.Size.Slice => return autoHash(key.len, autoHash(key.ptr, seed)), + }, + + builtin.TypeId.Optional => return if (key) |k| autoHash(k, seed) else 0, + + builtin.TypeId.Array => { + // TODO detect via a trait when Key has no padding bits to + // hash it as an array of bytes. + // Otherwise, hash every element. + var s = seed; + for (key) |element| { + // We reuse the hash of the previous element as the seed for the + // next one so that they're dependant. + s = autoHash(element, s); + } + return s; + }, + + builtin.TypeId.Vector => |info| { + // If there's no unused bits in the child type, we can just hash + // this as an array of bytes. + if (info.child.bit_count % 8 == 0) { + return wyhash(mem.asBytes(&key), seed); + } + + // Otherwise, hash every element. + var s = seed; + // TODO remove the copy to an array once field access is done. + const array: [info.len]info.child = key; + comptime var i: u32 = 0; + inline while (i < info.len) : (i += 1) { + s = autoHash(array[i], s); } + return s; }, - builtin.TypeId.Float => |info| { - return autoHash(@bitCast(@IntType(false, info.bits), key), rng, HashInt); + builtin.TypeId.Struct => |info| { + // TODO detect via a trait when Key has no padding bits to + // hash it as an array of bytes. + // Otherwise, hash every field. + var s = seed; + inline for (info.fields) |field| { + // We reuse the hash of the previous field as the seed for the + // next one so that they're dependant. + s = autoHash(@field(key, field.name), s); + } + return s; }, - builtin.TypeId.Bool => return autoHash(@boolToInt(key), rng, HashInt), - builtin.TypeId.Enum => return autoHash(@enumToInt(key), rng, HashInt), - builtin.TypeId.ErrorSet => return autoHash(@errorToInt(key), rng, HashInt), - builtin.TypeId.Promise, builtin.TypeId.Fn => return autoHash(@ptrToInt(key), rng, HashInt), - - builtin.TypeId.BoundFn, - builtin.TypeId.ComptimeFloat, - builtin.TypeId.ComptimeInt, - builtin.TypeId.Type, - builtin.TypeId.EnumLiteral, - => return 0, - - builtin.TypeId.Pointer => |info| switch (info.size) { - builtin.TypeInfo.Pointer.Size.One => @compileError("TODO auto hash for single item pointers"), - builtin.TypeInfo.Pointer.Size.Many => @compileError("TODO auto hash for many item pointers"), - builtin.TypeInfo.Pointer.Size.C => @compileError("TODO auto hash C pointers"), - builtin.TypeInfo.Pointer.Size.Slice => { - const interval = std.math.max(1, key.len / 256); - var i: usize = 0; - var h = comptime rng.scalar(HashInt); - while (i < key.len) : (i += interval) { - h ^= autoHash(key[i], rng, HashInt); + + builtin.TypeId.Union => |info| { + if (info.tag_type) |tag_type| { + const tag = meta.activeTag(key); + const s = autoHash(tag, seed); + inline for (info.fields) |field| { + const enum_field = field.enum_field.?; + if (enum_field.value == @enumToInt(tag)) { + return autoHash(@field(key, enum_field.name), s); + } } - return h; - }, + unreachable; + } else @compileError("cannot hash untagged union type: " ++ @typeName(Key) ++ ", provide your own hash function"); }, - builtin.TypeId.Optional => @compileError("TODO auto hash for optionals"), - builtin.TypeId.Array => @compileError("TODO auto hash for arrays"), - builtin.TypeId.Vector => @compileError("TODO auto hash for vectors"), - builtin.TypeId.Struct => @compileError("TODO auto hash for structs"), - builtin.TypeId.Union => @compileError("TODO auto hash for unions"), - builtin.TypeId.ErrorUnion => @compileError("TODO auto hash for unions"), + builtin.TypeId.ErrorUnion => { + return autoHash(key catch |err| return autoHash(err, seed), seed); + }, } } -pub fn autoEql(a: var, b: @typeOf(a)) bool { - switch (@typeInfo(@typeOf(a))) { - builtin.TypeId.NoReturn, - builtin.TypeId.Opaque, - builtin.TypeId.Undefined, - builtin.TypeId.ArgTuple, - => @compileError("cannot test equality of this type"), - builtin.TypeId.Void, - builtin.TypeId.Null, - => return true, - builtin.TypeId.Bool, - builtin.TypeId.Int, - builtin.TypeId.Float, - builtin.TypeId.ComptimeFloat, - builtin.TypeId.ComptimeInt, - builtin.TypeId.EnumLiteral, - builtin.TypeId.Promise, - builtin.TypeId.Enum, - builtin.TypeId.BoundFn, - builtin.TypeId.Fn, - builtin.TypeId.ErrorSet, - builtin.TypeId.Type, - => return a == b, - - builtin.TypeId.Pointer => |info| switch (info.size) { - builtin.TypeInfo.Pointer.Size.One => @compileError("TODO auto eql for single item pointers"), - builtin.TypeInfo.Pointer.Size.Many => @compileError("TODO auto eql for many item pointers"), - builtin.TypeInfo.Pointer.Size.C => @compileError("TODO auto eql for C pointers"), - builtin.TypeInfo.Pointer.Size.Slice => { - if (a.len != b.len) return false; - for (a) |a_item, i| { - if (!autoEql(a_item, b[i])) return false; - } - return true; - }, - }, +test "autoHash slice" { + const array1 = try std.heap.direct_allocator.create([6]u32); + defer std.heap.direct_allocator.destroy(array1); + array1.* = [_]u32{ 1, 2, 3, 4, 5, 6 }; + const array2 = [_]u32{ 1, 2, 3, 4, 5, 6 }; + const a = array1[0..]; + const b = array2[0..]; + const c = array1[0..3]; + testing.expect(autoHash(a, 0) == autoHash(a, 0)); + testing.expect(autoHash(a, 0) != autoHash(array1, 0)); + testing.expect(autoHash(a, 0) != autoHash(b, 0)); + testing.expect(autoHash(a, 0) != autoHash(c, 0)); +} - builtin.TypeId.Optional => @compileError("TODO auto eql for optionals"), - builtin.TypeId.Array => @compileError("TODO auto eql for arrays"), - builtin.TypeId.Struct => @compileError("TODO auto eql for structs"), - builtin.TypeId.Union => @compileError("TODO auto eql for unions"), - builtin.TypeId.ErrorUnion => @compileError("TODO auto eql for unions"), - builtin.TypeId.Vector => @compileError("TODO auto eql for vectors"), - } +test "autoHash optional" { + const a: ?u32 = 123; + const b: ?u32 = null; + testing.expectEqual(autoHash(a, 0), autoHash(u32(123), 0)); + testing.expect(autoHash(a, 0) != autoHash(b, 0)); + testing.expectEqual(autoHash(b, 0), 0); +} + +test "autoHash array" { + const a = [_]u32{ 1, 2, 3 }; + const h = autoHash(a, 0); + testing.expectEqual(h, autoHash(u32(3), autoHash(u32(2), autoHash(u32(1), 0)))); +} + +test "autoHash struct" { + const Foo = struct { + a: u32 = 1, + b: u32 = 2, + c: u32 = 3, + }; + const f = Foo{}; + const h = autoHash(f, 0); + testing.expectEqual(h, autoHash(u32(3), autoHash(u32(2), autoHash(u32(1), 0)))); +} + +test "autoHash union" { + const Foo = union(enum) { + A: u32, + B: f32, + C: u32, + }; + + const a = Foo{ .A = 18 }; + var b = Foo{ .B = 12.34 }; + const c = Foo{ .C = 18 }; + testing.expect(autoHash(a, 0) == autoHash(a, 0)); + testing.expect(autoHash(a, 0) != autoHash(b, 0)); + testing.expect(autoHash(a, 0) != autoHash(c, 0)); + + b = Foo{ .A = 18 }; + testing.expect(autoHash(a, 0) == autoHash(b, 0)); +} + +test "autoHash vector" { + const a: @Vector(4, u32) = [_]u32{ 1, 2, 3, 4 }; + const b: @Vector(4, u32) = [_]u32{ 1, 2, 3, 5 }; + const c: @Vector(4, u31) = [_]u31{ 1, 2, 3, 4 }; + testing.expect(autoHash(a, 0) == autoHash(a, 0)); + testing.expect(autoHash(a, 0) != autoHash(b, 0)); + testing.expect(autoHash(a, 0) != autoHash(c, 0)); +} + +test "autoHash error union" { + const Errors = error{Test}; + const Foo = struct { + a: u32 = 1, + b: u32 = 2, + c: u32 = 3, + }; + const f = Foo{}; + const g: Errors!Foo = Errors.Test; + testing.expect(autoHash(f, 0) != autoHash(g, 0)); + testing.expect(autoHash(f, 0) == autoHash(Foo{}, 0)); + testing.expect(autoHash(g, 0) == autoHash(Errors.Test, 0)); } -- 2.54.0 From c9ce43f59fc777055612aeea58db0849390bc204 Mon Sep 17 00:00:00 2001 From: Sahnvour Date: Sun, 30 Jun 2019 20:46:43 +0200 Subject: [PATCH 15/23] fix hashmap using strings as keys --- std/http/headers.zig | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/std/http/headers.zig b/std/http/headers.zig index 69ed494f3a432b901b52fe6ab2ee0a4dc588a6f8..7eb7fcc2c25f9f0f7486ea3a5ae7e3c7dbb7850a 100644 --- a/std/http/headers.zig +++ b/std/http/headers.zig @@ -102,9 +102,19 @@ test "HeaderEntry" { testing.expectEqualSlices(u8, "x", e.value); } +fn stringEql(a: []const u8, b: []const u8) bool { + if (a.len != b.len) return false; + if (a.ptr == b.ptr) return true; + return mem.compare(u8, a, b) == .Equal; +} + +fn stringHash(s: []const u8) u32 { + return @truncate(u32, std.hash.wyhash(s, 0)); +} + const HeaderList = std.ArrayList(HeaderEntry); const HeaderIndexList = std.ArrayList(usize); -const HeaderIndex = std.AutoHashMap([]const u8, HeaderIndexList); +const HeaderIndex = std.HashMap([]const u8, HeaderIndexList, stringHash, stringEql); pub const Headers = struct { // the owned header field name is stored in the index as part of the key -- 2.54.0 From 83dffc70afe4956c56f570ce5c854b17cbd6f218 Mon Sep 17 00:00:00 2001 From: Marc Tiehuis Date: Mon, 1 Jul 2019 23:23:26 +1200 Subject: [PATCH 16/23] Add iterative wyhash api --- std/hash.zig | 4 +- std/hash/wyhash.zig | 166 ++++++++++++++++++++++++++++---------------- std/hash_map.zig | 6 +- 3 files changed, 113 insertions(+), 63 deletions(-) diff --git a/std/hash.zig b/std/hash.zig index 723860da3bfd9fdc8c877eb52100840e07477642..e246fd0ad3248936f6ba7f7f953e792bbaf04b0e 100644 --- a/std/hash.zig +++ b/std/hash.zig @@ -17,6 +17,7 @@ pub const SipHash128 = siphash.SipHash128; pub const murmur = @import("hash/murmur.zig"); pub const Murmur2_32 = murmur.Murmur2_32; + pub const Murmur2_64 = murmur.Murmur2_64; pub const Murmur3_32 = murmur.Murmur3_32; @@ -24,7 +25,8 @@ pub const cityhash = @import("hash/cityhash.zig"); pub const CityHash32 = cityhash.CityHash32; pub const CityHash64 = cityhash.CityHash64; -pub const wyhash = @import("hash/wyhash.zig").hash; +const wyhash = @import("hash/wyhash.zig"); +pub const Wyhash = wyhash.Wyhash; test "hash" { _ = @import("hash/adler.zig"); diff --git a/std/hash/wyhash.zig b/std/hash/wyhash.zig index 57efe8fd631eb69be71764da1a50f2ae914c9bcd..49119c5a9595a45d9b44ee94e8f6ef6b0c181ea5 100644 --- a/std/hash/wyhash.zig +++ b/std/hash/wyhash.zig @@ -10,7 +10,7 @@ const primes = [_]u64{ }; fn read_bytes(comptime bytes: u8, data: []const u8) u64 { - return mem.readVarInt(u64, data[0..bytes], @import("builtin").endian); + return mem.readVarInt(u64, data[0..bytes], .Little); } fn read_8bytes_swapped(data: []const u8) u64 { @@ -18,7 +18,7 @@ fn read_8bytes_swapped(data: []const u8) u64 { } fn mum(a: u64, b: u64) u64 { - var r: u128 = @intCast(u128, a) * @intCast(u128, b); + var r = std.math.mulWide(u64, a, b); r = (r >> 64) ^ r; return @truncate(u64, r); } @@ -31,69 +31,117 @@ fn mix1(a: u64, b: u64, seed: u64) u64 { return mum(a ^ seed ^ primes[2], b ^ seed ^ primes[3]); } -pub fn hash(key: []const u8, initial_seed: u64) u64 { - var seed = initial_seed; - - var i: usize = 0; - while (i + 32 <= key.len) : (i += 32) { - seed = mix0( - read_bytes(8, key[i..]), - read_bytes(8, key[i + 8 ..]), - seed, +pub const Wyhash = struct { + seed: u64, + + buf: [32]u8, + buf_len: usize, + msg_len: usize, + + pub fn init(seed: u64) Wyhash { + return Wyhash{ + .seed = seed, + .buf = undefined, + .buf_len = 0, + .msg_len = 0, + }; + } + + fn round(self: *Wyhash, b: []const u8) void { + std.debug.assert(b.len == 32); + + self.seed = mix0( + read_bytes(8, b[0..]), + read_bytes(8, b[8..]), + self.seed, ) ^ mix1( - read_bytes(8, key[i + 16 ..]), - read_bytes(8, key[i + 24 ..]), - seed, + read_bytes(8, b[16..]), + read_bytes(8, b[24..]), + self.seed, ); } - const rem_len = @truncate(u5, key.len); - const rem_key = key[i..]; - seed = switch (rem_len) { - 0 => seed, - 1 => mix0(read_bytes(1, rem_key), primes[4], seed), - 2 => mix0(read_bytes(2, rem_key), primes[4], seed), - 3 => mix0((read_bytes(2, rem_key) << 8) | read_bytes(1, rem_key[2..]), primes[4], seed), - 4 => mix0(read_bytes(4, rem_key), primes[4], seed), - 5 => mix0((read_bytes(4, rem_key) << 8) | read_bytes(1, rem_key[4..]), primes[4], seed), - 6 => mix0((read_bytes(4, rem_key) << 16) | read_bytes(2, rem_key[4..]), primes[4], seed), - 7 => mix0((read_bytes(4, rem_key) << 24) | (read_bytes(2, rem_key[4..]) << 8) | read_bytes(1, rem_key[6..]), primes[4], seed), - 8 => mix0(read_8bytes_swapped(rem_key), primes[4], seed), - 9 => mix0(read_8bytes_swapped(rem_key), read_bytes(1, rem_key[8..]), seed), - 10 => mix0(read_8bytes_swapped(rem_key), read_bytes(2, rem_key[8..]), seed), - 11 => mix0(read_8bytes_swapped(rem_key), (read_bytes(2, rem_key[8..]) << 8) | read_bytes(1, rem_key[10..]), seed), - 12 => mix0(read_8bytes_swapped(rem_key), read_bytes(4, rem_key[8..]), seed), - 13 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 8) | read_bytes(1, rem_key[12..]), seed), - 14 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 16) | read_bytes(2, rem_key[12..]), seed), - 15 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 24) | (read_bytes(2, rem_key[12..]) << 8) | read_bytes(1, rem_key[14..]), seed), - 16 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed), - 17 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(1, rem_key[16..]), primes[4], seed), - 18 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(2, rem_key[16..]), primes[4], seed), - 19 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(2, rem_key[16..]) << 8) | read_bytes(1, rem_key[18..]), primes[4], seed), - 20 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(4, rem_key[16..]), primes[4], seed), - 21 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 8) | read_bytes(1, rem_key[20..]), primes[4], seed), - 22 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 16) | read_bytes(2, rem_key[20..]), primes[4], seed), - 23 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 24) | (read_bytes(2, rem_key[20..]) << 8) | read_bytes(1, rem_key[22..]), primes[4], seed), - 24 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), primes[4], seed), - 25 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(1, rem_key[24..]), seed), - 26 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(2, rem_key[24..]), seed), - 27 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(2, rem_key[24..]) << 8) | read_bytes(1, rem_key[26..]), seed), - 28 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(4, rem_key[24..]), seed), - 29 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 8) | read_bytes(1, rem_key[28..]), seed), - 30 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 16) | read_bytes(2, rem_key[28..]), seed), - 31 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 24) | (read_bytes(2, rem_key[28..]) << 8) | read_bytes(1, rem_key[30..]), seed), - }; + pub fn update(self: *Wyhash, b: []const u8) void { + var off: usize = 0; - return mum(seed ^ key.len, primes[4]); -} + // Partial from previous. + if (self.buf_len != 0 and self.buf_len + b.len > 32) { + off += 32 - self.buf_len; + mem.copy(u8, self.buf[self.buf_len..], b[0..off]); + self.round(self.buf[0..]); + self.buf_len = 0; + } + + // Full middle blocks. + while (off + 32 <= b.len) : (off += 32) { + @inlineCall(self.round, b[off .. off + 32]); + } + + // Remainder for next pass. + mem.copy(u8, self.buf[self.buf_len..], b[off..]); + self.buf_len += @intCast(u8, b[off..].len); + self.msg_len += b.len; + } + + pub fn final(self: *Wyhash) u64 { + const seed = self.seed; + const rem_len = @intCast(u5, self.buf_len); + const rem_key = self.buf[0..self.buf_len]; + + self.seed = switch (rem_len) { + 0 => seed, + 1 => mix0(read_bytes(1, rem_key), primes[4], seed), + 2 => mix0(read_bytes(2, rem_key), primes[4], seed), + 3 => mix0((read_bytes(2, rem_key) << 8) | read_bytes(1, rem_key[2..]), primes[4], seed), + 4 => mix0(read_bytes(4, rem_key), primes[4], seed), + 5 => mix0((read_bytes(4, rem_key) << 8) | read_bytes(1, rem_key[4..]), primes[4], seed), + 6 => mix0((read_bytes(4, rem_key) << 16) | read_bytes(2, rem_key[4..]), primes[4], seed), + 7 => mix0((read_bytes(4, rem_key) << 24) | (read_bytes(2, rem_key[4..]) << 8) | read_bytes(1, rem_key[6..]), primes[4], seed), + 8 => mix0(read_8bytes_swapped(rem_key), primes[4], seed), + 9 => mix0(read_8bytes_swapped(rem_key), read_bytes(1, rem_key[8..]), seed), + 10 => mix0(read_8bytes_swapped(rem_key), read_bytes(2, rem_key[8..]), seed), + 11 => mix0(read_8bytes_swapped(rem_key), (read_bytes(2, rem_key[8..]) << 8) | read_bytes(1, rem_key[10..]), seed), + 12 => mix0(read_8bytes_swapped(rem_key), read_bytes(4, rem_key[8..]), seed), + 13 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 8) | read_bytes(1, rem_key[12..]), seed), + 14 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 16) | read_bytes(2, rem_key[12..]), seed), + 15 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 24) | (read_bytes(2, rem_key[12..]) << 8) | read_bytes(1, rem_key[14..]), seed), + 16 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed), + 17 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(1, rem_key[16..]), primes[4], seed), + 18 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(2, rem_key[16..]), primes[4], seed), + 19 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(2, rem_key[16..]) << 8) | read_bytes(1, rem_key[18..]), primes[4], seed), + 20 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(4, rem_key[16..]), primes[4], seed), + 21 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 8) | read_bytes(1, rem_key[20..]), primes[4], seed), + 22 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 16) | read_bytes(2, rem_key[20..]), primes[4], seed), + 23 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 24) | (read_bytes(2, rem_key[20..]) << 8) | read_bytes(1, rem_key[22..]), primes[4], seed), + 24 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), primes[4], seed), + 25 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(1, rem_key[24..]), seed), + 26 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(2, rem_key[24..]), seed), + 27 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(2, rem_key[24..]) << 8) | read_bytes(1, rem_key[26..]), seed), + 28 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(4, rem_key[24..]), seed), + 29 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 8) | read_bytes(1, rem_key[28..]), seed), + 30 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 16) | read_bytes(2, rem_key[28..]), seed), + 31 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 24) | (read_bytes(2, rem_key[28..]) << 8) | read_bytes(1, rem_key[30..]), seed), + }; + + return mum(self.seed ^ self.msg_len, primes[4]); + } + + pub fn hash(seed: u64, input: []const u8) u64 { + var c = Wyhash.init(seed); + c.update(input); + return c.final(); + } +}; test "test vectors" { const expectEqual = std.testing.expectEqual; - expectEqual(hash("", 0), 0x0); - expectEqual(hash("a", 1), 0xbed235177f41d328); - expectEqual(hash("abc", 2), 0xbe348debe59b27c3); - expectEqual(hash("message digest", 3), 0x37320f657213a290); - expectEqual(hash("abcdefghijklmnopqrstuvwxyz", 4), 0xd0b270e1d8a7019c); - expectEqual(hash("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 5), 0x602a1894d3bbfe7f); - expectEqual(hash("12345678901234567890123456789012345678901234567890123456789012345678901234567890", 6), 0x829e9c148b75970e); + const hash = Wyhash.hash; + + expectEqual(hash(0, ""), 0x0); + expectEqual(hash(1, "a"), 0xbed235177f41d328); + expectEqual(hash(2, "abc"), 0xbe348debe59b27c3); + expectEqual(hash(3, "message digest"), 0x37320f657213a290); + expectEqual(hash(4, "abcdefghijklmnopqrstuvwxyz"), 0xd0b270e1d8a7019c); + expectEqual(hash(5, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"), 0x602a1894d3bbfe7f); + expectEqual(hash(6, "12345678901234567890123456789012345678901234567890123456789012345678901234567890"), 0x829e9c148b75970e); } diff --git a/std/hash_map.zig b/std/hash_map.zig index 6a8679ccd0a4d6346bb99f51c35e3a7eb8eb0fe4..71cfecdd6d5606437a1f3659976baaf67be30626 100644 --- a/std/hash_map.zig +++ b/std/hash_map.zig @@ -5,7 +5,7 @@ const testing = std.testing; const math = std.math; const mem = std.mem; const meta = std.meta; -const wyhash = std.hash.wyhash; +const Wyhash = std.hash.Wyhash; const Allocator = mem.Allocator; const builtin = @import("builtin"); @@ -557,7 +557,7 @@ pub fn autoHash(key: var, seed: u64) u64 { builtin.TypeId.EnumLiteral, => @compileError("cannot hash this type"), - builtin.TypeId.Int => return wyhash(std.mem.asBytes(&key), seed), + builtin.TypeId.Int => return Wyhash.hash(seed, std.mem.asBytes(&key)), builtin.TypeId.Float => |info| return autoHash(@bitCast(@IntType(false, info.bits), key), seed), @@ -594,7 +594,7 @@ pub fn autoHash(key: var, seed: u64) u64 { // If there's no unused bits in the child type, we can just hash // this as an array of bytes. if (info.child.bit_count % 8 == 0) { - return wyhash(mem.asBytes(&key), seed); + return Wyhash.hash(seed, mem.asBytes(&key)); } // Otherwise, hash every element. -- 2.54.0 From 4c93ccab5ad48ce61e4136c646b3123a06150083 Mon Sep 17 00:00:00 2001 From: Marc Tiehuis Date: Mon, 1 Jul 2019 23:23:40 +1200 Subject: [PATCH 17/23] Add throughput test program for hash functions --- std/hash/throughput_test.zig | 148 +++++++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 std/hash/throughput_test.zig diff --git a/std/hash/throughput_test.zig b/std/hash/throughput_test.zig new file mode 100644 index 0000000000000000000000000000000000000000..4b7e8ef344affd4eb2237adf2d41f75a2ca84925 --- /dev/null +++ b/std/hash/throughput_test.zig @@ -0,0 +1,148 @@ +const builtin = @import("builtin"); +const std = @import("std"); +const time = std.time; +const Timer = time.Timer; +const hash = std.hash; + +const KiB = 1024; +const MiB = 1024 * KiB; +const GiB = 1024 * MiB; + +var prng = std.rand.DefaultPrng.init(0); + +const Hash = struct { + ty: type, + name: []const u8, + init_u8s: ?[]const u8 = null, + init_u64: ?u64 = null, +}; + +const siphash_key = "0123456789abcdef"; + +const hashes = [_]Hash{ + Hash{ .ty = hash.Wyhash, .name = "wyhash", .init_u64 = 0 }, + Hash{ .ty = hash.SipHash64(1, 3), .name = "siphash(1,3)", .init_u8s = siphash_key }, + Hash{ .ty = hash.SipHash64(2, 4), .name = "siphash(2,4)", .init_u8s = siphash_key }, + Hash{ .ty = hash.Fnv1a_64, .name = "fnv1a" }, + Hash{ .ty = hash.Crc32, .name = "crc32" }, +}; + +const Result = struct { + hash: u64, + throughput: u64, +}; + +pub fn benchmarkHash(comptime H: var, bytes: usize) !Result { + var h = blk: { + if (H.init_u8s) |init| { + break :blk H.ty.init(init); + } + if (H.init_u64) |init| { + break :blk H.ty.init(init); + } + break :blk H.ty.init(); + }; + + var block: [8192]u8 = undefined; + prng.random.bytes(block[0..]); + + var offset: usize = 0; + var timer = try Timer.start(); + const start = timer.lap(); + while (offset < bytes) : (offset += block.len) { + h.update(block[0..]); + } + const end = timer.read(); + + const elapsed_s = @intToFloat(f64, end - start) / time.ns_per_s; + const throughput = @floatToInt(u64, @intToFloat(f64, bytes) / elapsed_s); + + return Result{ + .hash = h.final(), + .throughput = throughput, + }; +} + +fn usage() void { + std.debug.warn( + \\throughput_test [options] + \\ + \\Options: + \\ --filter [test-name] + \\ --seed [int] + \\ --count [int] + \\ --help + \\ + ); +} + +fn mode(comptime x: comptime_int) comptime_int { + return if (builtin.mode == builtin.Mode.Debug) x / 64 else x; +} + +// TODO(#1358): Replace with builtin formatted padding when available. +fn printPad(stdout: var, s: []const u8) !void { + var i: usize = 0; + while (i < 12 - s.len) : (i += 1) { + try stdout.print(" "); + } + try stdout.print("{}", s); +} + +pub fn main() !void { + var stdout_file = try std.io.getStdOut(); + var stdout_out_stream = stdout_file.outStream(); + const stdout = &stdout_out_stream.stream; + + var buffer: [1024]u8 = undefined; + var fixed = std.heap.FixedBufferAllocator.init(buffer[0..]); + const args = try std.process.argsAlloc(&fixed.allocator); + + var filter: ?[]u8 = ""; + var count: usize = mode(128 * MiB); + + var i: usize = 1; + while (i < args.len) : (i += 1) { + if (std.mem.eql(u8, args[i], "--seed")) { + i += 1; + if (i == args.len) { + usage(); + std.os.exit(1); + } + + const seed = try std.fmt.parseUnsigned(u32, args[i], 10); + prng.seed(seed); + } else if (std.mem.eql(u8, args[i], "--filter")) { + i += 1; + if (i == args.len) { + usage(); + std.os.exit(1); + } + + filter = args[i]; + } else if (std.mem.eql(u8, args[i], "--count")) { + i += 1; + if (i == args.len) { + usage(); + std.os.exit(1); + } + + const c = try std.fmt.parseUnsigned(u32, args[i], 10); + count = c * MiB; + } else if (std.mem.eql(u8, args[i], "--help")) { + usage(); + return; + } else { + usage(); + std.os.exit(1); + } + } + + inline for (hashes) |H| { + if (filter == null or std.mem.indexOf(u8, H.name, filter.?) != null) { + const result = try benchmarkHash(H, count); + try printPad(stdout, H.name); + try stdout.print(": {:4} MiB/s [{:16}]\n", result.throughput / (1 * MiB), result.hash); + } + } +} -- 2.54.0 From 8805a7b50985fca23969beab8636fbfbecd857ee Mon Sep 17 00:00:00 2001 From: Sahnvour Date: Tue, 2 Jul 2019 18:38:46 +0200 Subject: [PATCH 18/23] adapt http/headers.zig to wyhash's new interface --- std/http/headers.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/std/http/headers.zig b/std/http/headers.zig index 7eb7fcc2c25f9f0f7486ea3a5ae7e3c7dbb7850a..c588f2d05507467d895820c203a96b4b424f498d 100644 --- a/std/http/headers.zig +++ b/std/http/headers.zig @@ -109,7 +109,7 @@ fn stringEql(a: []const u8, b: []const u8) bool { } fn stringHash(s: []const u8) u32 { - return @truncate(u32, std.hash.wyhash(s, 0)); + return @truncate(u32, std.hash.Wyhash.hash(0, s)); } const HeaderList = std.ArrayList(HeaderEntry); -- 2.54.0 From 5bf63bfbf113d3921101311f1e3040890b94e798 Mon Sep 17 00:00:00 2001 From: Sahnvour Date: Tue, 2 Jul 2019 18:40:01 +0200 Subject: [PATCH 19/23] make use of hashing streaming interface in autoHash --- std/hash_map.zig | 154 ++++++++++++++++++++++++++--------------------- 1 file changed, 85 insertions(+), 69 deletions(-) diff --git a/std/hash_map.zig b/std/hash_map.zig index 71cfecdd6d5606437a1f3659976baaf67be30626..d906b54618947b3af952bd0394c8eb0d0dd5ae02 100644 --- a/std/hash_map.zig +++ b/std/hash_map.zig @@ -522,8 +522,9 @@ pub fn getTrivialEqlFn(comptime K: type) (fn (K, K) bool) { pub fn getAutoHashFn(comptime K: type) (fn (K) u32) { return struct { fn hash(key: K) u32 { - const h = autoHash(key, 0); - return @truncate(u32, h); + var hasher = Wyhash.init(0); + autoHash(&hasher, key); + return @truncate(u32, hasher.final()); } }.hash; } @@ -538,10 +539,7 @@ pub fn getAutoEqlFn(comptime K: type) (fn (K, K) bool) { /// Provides generic hashing for any eligible type. /// Only hashes `key` itself, pointers are not followed. -/// The underlying hashing algorithm is wyhash. -pub fn autoHash(key: var, seed: u64) u64 { - // We use the fact that wyhash takes an input seed to "chain" hasing when the - // key has multiple parts that are not necessarily contiguous in memory. +pub fn autoHash(hasher: var, key: var) void { const Key = @typeOf(key); switch (@typeInfo(Key)) { builtin.TypeId.NoReturn, @@ -557,91 +555,101 @@ pub fn autoHash(key: var, seed: u64) u64 { builtin.TypeId.EnumLiteral, => @compileError("cannot hash this type"), - builtin.TypeId.Int => return Wyhash.hash(seed, std.mem.asBytes(&key)), + builtin.TypeId.Int => hasher.update(std.mem.asBytes(&key)), - builtin.TypeId.Float => |info| return autoHash(@bitCast(@IntType(false, info.bits), key), seed), + builtin.TypeId.Float => |info| autoHash(hasher, @bitCast(@IntType(false, info.bits), key)), - builtin.TypeId.Bool => return autoHash(@boolToInt(key), seed), - builtin.TypeId.Enum => return autoHash(@enumToInt(key), seed), - builtin.TypeId.ErrorSet => return autoHash(@errorToInt(key), seed), - builtin.TypeId.Promise, builtin.TypeId.Fn => return autoHash(@ptrToInt(key), seed), + builtin.TypeId.Bool => autoHash(hasher, @boolToInt(key)), + builtin.TypeId.Enum => autoHash(hasher, @enumToInt(key)), + builtin.TypeId.ErrorSet => autoHash(hasher, @errorToInt(key)), + builtin.TypeId.Promise, builtin.TypeId.Fn => autoHash(hasher, @ptrToInt(key)), - builtin.TypeId.Pointer => |info| return switch (info.size) { + builtin.TypeId.Pointer => |info| switch (info.size) { builtin.TypeInfo.Pointer.Size.One, builtin.TypeInfo.Pointer.Size.Many, builtin.TypeInfo.Pointer.Size.C, - => return autoHash(@ptrToInt(key), seed), + => autoHash(hasher, @ptrToInt(key)), - builtin.TypeInfo.Pointer.Size.Slice => return autoHash(key.len, autoHash(key.ptr, seed)), + builtin.TypeInfo.Pointer.Size.Slice => { + autoHash(hasher, key.ptr); + autoHash(hasher, key.len); + }, }, - builtin.TypeId.Optional => return if (key) |k| autoHash(k, seed) else 0, + builtin.TypeId.Optional => if (key) |k| autoHash(hasher, k), builtin.TypeId.Array => { // TODO detect via a trait when Key has no padding bits to // hash it as an array of bytes. // Otherwise, hash every element. - var s = seed; for (key) |element| { - // We reuse the hash of the previous element as the seed for the - // next one so that they're dependant. - s = autoHash(element, s); + autoHash(hasher, element); } - return s; }, builtin.TypeId.Vector => |info| { - // If there's no unused bits in the child type, we can just hash - // this as an array of bytes. if (info.child.bit_count % 8 == 0) { - return Wyhash.hash(seed, mem.asBytes(&key)); + // If there's no unused bits in the child type, we can just hash + // this as an array of bytes. + hasher.update(mem.asBytes(&key)); + } else { + // Otherwise, hash every element. + // TODO remove the copy to an array once field access is done. + const array: [info.len]info.child = key; + comptime var i: u32 = 0; + inline while (i < info.len) : (i += 1) { + autoHash(hasher, array[i]); + } } - - // Otherwise, hash every element. - var s = seed; - // TODO remove the copy to an array once field access is done. - const array: [info.len]info.child = key; - comptime var i: u32 = 0; - inline while (i < info.len) : (i += 1) { - s = autoHash(array[i], s); - } - return s; }, builtin.TypeId.Struct => |info| { // TODO detect via a trait when Key has no padding bits to // hash it as an array of bytes. // Otherwise, hash every field. - var s = seed; inline for (info.fields) |field| { // We reuse the hash of the previous field as the seed for the // next one so that they're dependant. - s = autoHash(@field(key, field.name), s); + autoHash(hasher, @field(key, field.name)); } - return s; }, - builtin.TypeId.Union => |info| { + builtin.TypeId.Union => |info| blk: { if (info.tag_type) |tag_type| { const tag = meta.activeTag(key); - const s = autoHash(tag, seed); + const s = autoHash(hasher, tag); inline for (info.fields) |field| { const enum_field = field.enum_field.?; if (enum_field.value == @enumToInt(tag)) { - return autoHash(@field(key, enum_field.name), s); + autoHash(hasher, @field(key, enum_field.name)); + // TODO use a labelled break when it does not crash the compiler. + // break :blk; + return; } } unreachable; } else @compileError("cannot hash untagged union type: " ++ @typeName(Key) ++ ", provide your own hash function"); }, - builtin.TypeId.ErrorUnion => { - return autoHash(key catch |err| return autoHash(err, seed), seed); + builtin.TypeId.ErrorUnion => blk: { + const payload = key catch |err| { + autoHash(hasher, err); + break :blk; + }; + autoHash(hasher, payload); }, } } +fn testAutoHash(key: var) u64 { + var hasher = Wyhash.init(0); + autoHash(&hasher, key); + return hasher.final(); +} + test "autoHash slice" { + // Allocate one array dynamically so that we're assured it is not merged + // with the other by the optimization passes. const array1 = try std.heap.direct_allocator.create([6]u32); defer std.heap.direct_allocator.destroy(array1); array1.* = [_]u32{ 1, 2, 3, 4, 5, 6 }; @@ -649,38 +657,46 @@ test "autoHash slice" { const a = array1[0..]; const b = array2[0..]; const c = array1[0..3]; - testing.expect(autoHash(a, 0) == autoHash(a, 0)); - testing.expect(autoHash(a, 0) != autoHash(array1, 0)); - testing.expect(autoHash(a, 0) != autoHash(b, 0)); - testing.expect(autoHash(a, 0) != autoHash(c, 0)); + testing.expect(testAutoHash(a) == testAutoHash(a)); + testing.expect(testAutoHash(a) != testAutoHash(array1)); + testing.expect(testAutoHash(a) != testAutoHash(b)); + testing.expect(testAutoHash(a) != testAutoHash(c)); } -test "autoHash optional" { +test "testAutoHash optional" { const a: ?u32 = 123; const b: ?u32 = null; - testing.expectEqual(autoHash(a, 0), autoHash(u32(123), 0)); - testing.expect(autoHash(a, 0) != autoHash(b, 0)); - testing.expectEqual(autoHash(b, 0), 0); + testing.expectEqual(testAutoHash(a), testAutoHash(u32(123))); + testing.expect(testAutoHash(a) != testAutoHash(b)); + testing.expectEqual(testAutoHash(b), 0); } -test "autoHash array" { +test "testAutoHash array" { const a = [_]u32{ 1, 2, 3 }; - const h = autoHash(a, 0); - testing.expectEqual(h, autoHash(u32(3), autoHash(u32(2), autoHash(u32(1), 0)))); + const h = testAutoHash(a); + var hasher = Wyhash.init(0); + autoHash(&hasher, u32(1)); + autoHash(&hasher, u32(2)); + autoHash(&hasher, u32(3)); + testing.expectEqual(h, hasher.final()); } -test "autoHash struct" { +test "testAutoHash struct" { const Foo = struct { a: u32 = 1, b: u32 = 2, c: u32 = 3, }; const f = Foo{}; - const h = autoHash(f, 0); - testing.expectEqual(h, autoHash(u32(3), autoHash(u32(2), autoHash(u32(1), 0)))); + const h = testAutoHash(f); + var hasher = Wyhash.init(0); + autoHash(&hasher, u32(1)); + autoHash(&hasher, u32(2)); + autoHash(&hasher, u32(3)); + testing.expectEqual(h, hasher.final()); } -test "autoHash union" { +test "testAutoHash union" { const Foo = union(enum) { A: u32, B: f32, @@ -690,24 +706,24 @@ test "autoHash union" { const a = Foo{ .A = 18 }; var b = Foo{ .B = 12.34 }; const c = Foo{ .C = 18 }; - testing.expect(autoHash(a, 0) == autoHash(a, 0)); - testing.expect(autoHash(a, 0) != autoHash(b, 0)); - testing.expect(autoHash(a, 0) != autoHash(c, 0)); + testing.expect(testAutoHash(a) == testAutoHash(a)); + testing.expect(testAutoHash(a) != testAutoHash(b)); + testing.expect(testAutoHash(a) != testAutoHash(c)); b = Foo{ .A = 18 }; - testing.expect(autoHash(a, 0) == autoHash(b, 0)); + testing.expect(testAutoHash(a) == testAutoHash(b)); } -test "autoHash vector" { +test "testAutoHash vector" { const a: @Vector(4, u32) = [_]u32{ 1, 2, 3, 4 }; const b: @Vector(4, u32) = [_]u32{ 1, 2, 3, 5 }; const c: @Vector(4, u31) = [_]u31{ 1, 2, 3, 4 }; - testing.expect(autoHash(a, 0) == autoHash(a, 0)); - testing.expect(autoHash(a, 0) != autoHash(b, 0)); - testing.expect(autoHash(a, 0) != autoHash(c, 0)); + testing.expect(testAutoHash(a) == testAutoHash(a)); + testing.expect(testAutoHash(a) != testAutoHash(b)); + testing.expect(testAutoHash(a) != testAutoHash(c)); } -test "autoHash error union" { +test "testAutoHash error union" { const Errors = error{Test}; const Foo = struct { a: u32 = 1, @@ -716,7 +732,7 @@ test "autoHash error union" { }; const f = Foo{}; const g: Errors!Foo = Errors.Test; - testing.expect(autoHash(f, 0) != autoHash(g, 0)); - testing.expect(autoHash(f, 0) == autoHash(Foo{}, 0)); - testing.expect(autoHash(g, 0) == autoHash(Errors.Test, 0)); + testing.expect(testAutoHash(f) != testAutoHash(g)); + testing.expect(testAutoHash(f) == testAutoHash(Foo{})); + testing.expect(testAutoHash(g) == testAutoHash(Errors.Test)); } -- 2.54.0 From 4b5172d2879742b98e3e34b90b05ac28da9f39fe Mon Sep 17 00:00:00 2001 From: Sahnvour Date: Tue, 2 Jul 2019 19:46:51 +0200 Subject: [PATCH 20/23] move autoHash into its own module since it can be used with any hash function implementing a streaming interface --- std/hash.zig | 4 + std/hash/auto_hash.zig | 208 +++++++++++++++++++++++++++++++++++++++++ std/hash_map.zig | 201 +-------------------------------------- 3 files changed, 213 insertions(+), 200 deletions(-) create mode 100644 std/hash/auto_hash.zig diff --git a/std/hash.zig b/std/hash.zig index e246fd0ad3248936f6ba7f7f953e792bbaf04b0e..648f34b11da7168008349c570be17c4a7fd251f5 100644 --- a/std/hash.zig +++ b/std/hash.zig @@ -1,6 +1,9 @@ const adler = @import("hash/adler.zig"); pub const Adler32 = adler.Adler32; +const auto_hash = @import("hash/auto_hash.zig"); +pub const autoHash = auto_hash.autoHash; + // pub for polynomials + generic crc32 construction pub const crc = @import("hash/crc.zig"); pub const Crc32 = crc.Crc32; @@ -30,6 +33,7 @@ pub const Wyhash = wyhash.Wyhash; test "hash" { _ = @import("hash/adler.zig"); + _ = @import("hash/auto_hash.zig"); _ = @import("hash/crc.zig"); _ = @import("hash/fnv.zig"); _ = @import("hash/siphash.zig"); diff --git a/std/hash/auto_hash.zig b/std/hash/auto_hash.zig new file mode 100644 index 0000000000000000000000000000000000000000..b21af0a1d8be73582bcc02e90fb034737f9a21b8 --- /dev/null +++ b/std/hash/auto_hash.zig @@ -0,0 +1,208 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const mem = std.mem; +const meta = std.meta; + +/// Provides generic hashing for any eligible type. +/// Only hashes `key` itself, pointers are not followed. +pub fn autoHash(hasher: var, key: var) void { + const Key = @typeOf(key); + switch (@typeInfo(Key)) { + builtin.TypeId.NoReturn, + builtin.TypeId.Opaque, + builtin.TypeId.Undefined, + builtin.TypeId.ArgTuple, + builtin.TypeId.Void, + builtin.TypeId.Null, + builtin.TypeId.BoundFn, + builtin.TypeId.ComptimeFloat, + builtin.TypeId.ComptimeInt, + builtin.TypeId.Type, + builtin.TypeId.EnumLiteral, + => @compileError("cannot hash this type"), + + builtin.TypeId.Int => hasher.update(std.mem.asBytes(&key)), + + builtin.TypeId.Float => |info| autoHash(hasher, @bitCast(@IntType(false, info.bits), key)), + + builtin.TypeId.Bool => autoHash(hasher, @boolToInt(key)), + builtin.TypeId.Enum => autoHash(hasher, @enumToInt(key)), + builtin.TypeId.ErrorSet => autoHash(hasher, @errorToInt(key)), + builtin.TypeId.Promise, builtin.TypeId.Fn => autoHash(hasher, @ptrToInt(key)), + + builtin.TypeId.Pointer => |info| switch (info.size) { + builtin.TypeInfo.Pointer.Size.One, + builtin.TypeInfo.Pointer.Size.Many, + builtin.TypeInfo.Pointer.Size.C, + => autoHash(hasher, @ptrToInt(key)), + + builtin.TypeInfo.Pointer.Size.Slice => { + autoHash(hasher, key.ptr); + autoHash(hasher, key.len); + }, + }, + + builtin.TypeId.Optional => if (key) |k| autoHash(hasher, k), + + builtin.TypeId.Array => { + // TODO detect via a trait when Key has no padding bits to + // hash it as an array of bytes. + // Otherwise, hash every element. + for (key) |element| { + autoHash(hasher, element); + } + }, + + builtin.TypeId.Vector => |info| { + if (info.child.bit_count % 8 == 0) { + // If there's no unused bits in the child type, we can just hash + // this as an array of bytes. + hasher.update(mem.asBytes(&key)); + } else { + // Otherwise, hash every element. + // TODO remove the copy to an array once field access is done. + const array: [info.len]info.child = key; + comptime var i: u32 = 0; + inline while (i < info.len) : (i += 1) { + autoHash(hasher, array[i]); + } + } + }, + + builtin.TypeId.Struct => |info| { + // TODO detect via a trait when Key has no padding bits to + // hash it as an array of bytes. + // Otherwise, hash every field. + inline for (info.fields) |field| { + // We reuse the hash of the previous field as the seed for the + // next one so that they're dependant. + autoHash(hasher, @field(key, field.name)); + } + }, + + builtin.TypeId.Union => |info| blk: { + if (info.tag_type) |tag_type| { + const tag = meta.activeTag(key); + const s = autoHash(hasher, tag); + inline for (info.fields) |field| { + const enum_field = field.enum_field.?; + if (enum_field.value == @enumToInt(tag)) { + autoHash(hasher, @field(key, enum_field.name)); + // TODO use a labelled break when it does not crash the compiler. + // break :blk; + return; + } + } + unreachable; + } else @compileError("cannot hash untagged union type: " ++ @typeName(Key) ++ ", provide your own hash function"); + }, + + builtin.TypeId.ErrorUnion => blk: { + const payload = key catch |err| { + autoHash(hasher, err); + break :blk; + }; + autoHash(hasher, payload); + }, + } +} + +const testing = std.testing; +const Wyhash = std.hash.Wyhash; + +fn testAutoHash(key: var) u64 { + // Any hash could be used here, for testing autoHash. + var hasher = Wyhash.init(0); + autoHash(&hasher, key); + return hasher.final(); +} + +test "autoHash slice" { + // Allocate one array dynamically so that we're assured it is not merged + // with the other by the optimization passes. + const array1 = try std.heap.direct_allocator.create([6]u32); + defer std.heap.direct_allocator.destroy(array1); + array1.* = [_]u32{ 1, 2, 3, 4, 5, 6 }; + const array2 = [_]u32{ 1, 2, 3, 4, 5, 6 }; + const a = array1[0..]; + const b = array2[0..]; + const c = array1[0..3]; + testing.expect(testAutoHash(a) == testAutoHash(a)); + testing.expect(testAutoHash(a) != testAutoHash(array1)); + testing.expect(testAutoHash(a) != testAutoHash(b)); + testing.expect(testAutoHash(a) != testAutoHash(c)); +} + +test "testAutoHash optional" { + const a: ?u32 = 123; + const b: ?u32 = null; + testing.expectEqual(testAutoHash(a), testAutoHash(u32(123))); + testing.expect(testAutoHash(a) != testAutoHash(b)); + testing.expectEqual(testAutoHash(b), 0); +} + +test "testAutoHash array" { + const a = [_]u32{ 1, 2, 3 }; + const h = testAutoHash(a); + var hasher = Wyhash.init(0); + autoHash(&hasher, u32(1)); + autoHash(&hasher, u32(2)); + autoHash(&hasher, u32(3)); + testing.expectEqual(h, hasher.final()); +} + +test "testAutoHash struct" { + const Foo = struct { + a: u32 = 1, + b: u32 = 2, + c: u32 = 3, + }; + const f = Foo{}; + const h = testAutoHash(f); + var hasher = Wyhash.init(0); + autoHash(&hasher, u32(1)); + autoHash(&hasher, u32(2)); + autoHash(&hasher, u32(3)); + testing.expectEqual(h, hasher.final()); +} + +test "testAutoHash union" { + const Foo = union(enum) { + A: u32, + B: f32, + C: u32, + }; + + const a = Foo{ .A = 18 }; + var b = Foo{ .B = 12.34 }; + const c = Foo{ .C = 18 }; + testing.expect(testAutoHash(a) == testAutoHash(a)); + testing.expect(testAutoHash(a) != testAutoHash(b)); + testing.expect(testAutoHash(a) != testAutoHash(c)); + + b = Foo{ .A = 18 }; + testing.expect(testAutoHash(a) == testAutoHash(b)); +} + +test "testAutoHash vector" { + const a: @Vector(4, u32) = [_]u32{ 1, 2, 3, 4 }; + const b: @Vector(4, u32) = [_]u32{ 1, 2, 3, 5 }; + const c: @Vector(4, u31) = [_]u31{ 1, 2, 3, 4 }; + testing.expect(testAutoHash(a) == testAutoHash(a)); + testing.expect(testAutoHash(a) != testAutoHash(b)); + testing.expect(testAutoHash(a) != testAutoHash(c)); +} + +test "testAutoHash error union" { + const Errors = error{Test}; + const Foo = struct { + a: u32 = 1, + b: u32 = 2, + c: u32 = 3, + }; + const f = Foo{}; + const g: Errors!Foo = Errors.Test; + testing.expect(testAutoHash(f) != testAutoHash(g)); + testing.expect(testAutoHash(f) == testAutoHash(Foo{})); + testing.expect(testAutoHash(g) == testAutoHash(Errors.Test)); +} diff --git a/std/hash_map.zig b/std/hash_map.zig index d906b54618947b3af952bd0394c8eb0d0dd5ae02..ab3c4c248dcce188b0422f92335b2d7d17a43dad 100644 --- a/std/hash_map.zig +++ b/std/hash_map.zig @@ -5,6 +5,7 @@ const testing = std.testing; const math = std.math; const mem = std.mem; const meta = std.meta; +const autoHash = std.hash.autoHash; const Wyhash = std.hash.Wyhash; const Allocator = mem.Allocator; const builtin = @import("builtin"); @@ -536,203 +537,3 @@ pub fn getAutoEqlFn(comptime K: type) (fn (K, K) bool) { } }.eql; } - -/// Provides generic hashing for any eligible type. -/// Only hashes `key` itself, pointers are not followed. -pub fn autoHash(hasher: var, key: var) void { - const Key = @typeOf(key); - switch (@typeInfo(Key)) { - builtin.TypeId.NoReturn, - builtin.TypeId.Opaque, - builtin.TypeId.Undefined, - builtin.TypeId.ArgTuple, - builtin.TypeId.Void, - builtin.TypeId.Null, - builtin.TypeId.BoundFn, - builtin.TypeId.ComptimeFloat, - builtin.TypeId.ComptimeInt, - builtin.TypeId.Type, - builtin.TypeId.EnumLiteral, - => @compileError("cannot hash this type"), - - builtin.TypeId.Int => hasher.update(std.mem.asBytes(&key)), - - builtin.TypeId.Float => |info| autoHash(hasher, @bitCast(@IntType(false, info.bits), key)), - - builtin.TypeId.Bool => autoHash(hasher, @boolToInt(key)), - builtin.TypeId.Enum => autoHash(hasher, @enumToInt(key)), - builtin.TypeId.ErrorSet => autoHash(hasher, @errorToInt(key)), - builtin.TypeId.Promise, builtin.TypeId.Fn => autoHash(hasher, @ptrToInt(key)), - - builtin.TypeId.Pointer => |info| switch (info.size) { - builtin.TypeInfo.Pointer.Size.One, - builtin.TypeInfo.Pointer.Size.Many, - builtin.TypeInfo.Pointer.Size.C, - => autoHash(hasher, @ptrToInt(key)), - - builtin.TypeInfo.Pointer.Size.Slice => { - autoHash(hasher, key.ptr); - autoHash(hasher, key.len); - }, - }, - - builtin.TypeId.Optional => if (key) |k| autoHash(hasher, k), - - builtin.TypeId.Array => { - // TODO detect via a trait when Key has no padding bits to - // hash it as an array of bytes. - // Otherwise, hash every element. - for (key) |element| { - autoHash(hasher, element); - } - }, - - builtin.TypeId.Vector => |info| { - if (info.child.bit_count % 8 == 0) { - // If there's no unused bits in the child type, we can just hash - // this as an array of bytes. - hasher.update(mem.asBytes(&key)); - } else { - // Otherwise, hash every element. - // TODO remove the copy to an array once field access is done. - const array: [info.len]info.child = key; - comptime var i: u32 = 0; - inline while (i < info.len) : (i += 1) { - autoHash(hasher, array[i]); - } - } - }, - - builtin.TypeId.Struct => |info| { - // TODO detect via a trait when Key has no padding bits to - // hash it as an array of bytes. - // Otherwise, hash every field. - inline for (info.fields) |field| { - // We reuse the hash of the previous field as the seed for the - // next one so that they're dependant. - autoHash(hasher, @field(key, field.name)); - } - }, - - builtin.TypeId.Union => |info| blk: { - if (info.tag_type) |tag_type| { - const tag = meta.activeTag(key); - const s = autoHash(hasher, tag); - inline for (info.fields) |field| { - const enum_field = field.enum_field.?; - if (enum_field.value == @enumToInt(tag)) { - autoHash(hasher, @field(key, enum_field.name)); - // TODO use a labelled break when it does not crash the compiler. - // break :blk; - return; - } - } - unreachable; - } else @compileError("cannot hash untagged union type: " ++ @typeName(Key) ++ ", provide your own hash function"); - }, - - builtin.TypeId.ErrorUnion => blk: { - const payload = key catch |err| { - autoHash(hasher, err); - break :blk; - }; - autoHash(hasher, payload); - }, - } -} - -fn testAutoHash(key: var) u64 { - var hasher = Wyhash.init(0); - autoHash(&hasher, key); - return hasher.final(); -} - -test "autoHash slice" { - // Allocate one array dynamically so that we're assured it is not merged - // with the other by the optimization passes. - const array1 = try std.heap.direct_allocator.create([6]u32); - defer std.heap.direct_allocator.destroy(array1); - array1.* = [_]u32{ 1, 2, 3, 4, 5, 6 }; - const array2 = [_]u32{ 1, 2, 3, 4, 5, 6 }; - const a = array1[0..]; - const b = array2[0..]; - const c = array1[0..3]; - testing.expect(testAutoHash(a) == testAutoHash(a)); - testing.expect(testAutoHash(a) != testAutoHash(array1)); - testing.expect(testAutoHash(a) != testAutoHash(b)); - testing.expect(testAutoHash(a) != testAutoHash(c)); -} - -test "testAutoHash optional" { - const a: ?u32 = 123; - const b: ?u32 = null; - testing.expectEqual(testAutoHash(a), testAutoHash(u32(123))); - testing.expect(testAutoHash(a) != testAutoHash(b)); - testing.expectEqual(testAutoHash(b), 0); -} - -test "testAutoHash array" { - const a = [_]u32{ 1, 2, 3 }; - const h = testAutoHash(a); - var hasher = Wyhash.init(0); - autoHash(&hasher, u32(1)); - autoHash(&hasher, u32(2)); - autoHash(&hasher, u32(3)); - testing.expectEqual(h, hasher.final()); -} - -test "testAutoHash struct" { - const Foo = struct { - a: u32 = 1, - b: u32 = 2, - c: u32 = 3, - }; - const f = Foo{}; - const h = testAutoHash(f); - var hasher = Wyhash.init(0); - autoHash(&hasher, u32(1)); - autoHash(&hasher, u32(2)); - autoHash(&hasher, u32(3)); - testing.expectEqual(h, hasher.final()); -} - -test "testAutoHash union" { - const Foo = union(enum) { - A: u32, - B: f32, - C: u32, - }; - - const a = Foo{ .A = 18 }; - var b = Foo{ .B = 12.34 }; - const c = Foo{ .C = 18 }; - testing.expect(testAutoHash(a) == testAutoHash(a)); - testing.expect(testAutoHash(a) != testAutoHash(b)); - testing.expect(testAutoHash(a) != testAutoHash(c)); - - b = Foo{ .A = 18 }; - testing.expect(testAutoHash(a) == testAutoHash(b)); -} - -test "testAutoHash vector" { - const a: @Vector(4, u32) = [_]u32{ 1, 2, 3, 4 }; - const b: @Vector(4, u32) = [_]u32{ 1, 2, 3, 5 }; - const c: @Vector(4, u31) = [_]u31{ 1, 2, 3, 4 }; - testing.expect(testAutoHash(a) == testAutoHash(a)); - testing.expect(testAutoHash(a) != testAutoHash(b)); - testing.expect(testAutoHash(a) != testAutoHash(c)); -} - -test "testAutoHash error union" { - const Errors = error{Test}; - const Foo = struct { - a: u32 = 1, - b: u32 = 2, - c: u32 = 3, - }; - const f = Foo{}; - const g: Errors!Foo = Errors.Test; - testing.expect(testAutoHash(f) != testAutoHash(g)); - testing.expect(testAutoHash(f) == testAutoHash(Foo{})); - testing.expect(testAutoHash(g) == testAutoHash(Errors.Test)); -} -- 2.54.0 From 3faf5d38576616d033c343130607189eb9fe613c Mon Sep 17 00:00:00 2001 From: Sahnvour Date: Tue, 16 Jul 2019 20:31:02 +0200 Subject: [PATCH 21/23] wyhash: stateless is faster for both iterative hashing and small keys. --- std/hash/wyhash.zig | 52 +++++++++++++++++---------------------------- 1 file changed, 20 insertions(+), 32 deletions(-) diff --git a/std/hash/wyhash.zig b/std/hash/wyhash.zig index 49119c5a9595a45d9b44ee94e8f6ef6b0c181ea5..dfa5156cadc385bd33bbb9b3d7e65bb1b1796074 100644 --- a/std/hash/wyhash.zig +++ b/std/hash/wyhash.zig @@ -33,16 +33,11 @@ fn mix1(a: u64, b: u64, seed: u64) u64 { pub const Wyhash = struct { seed: u64, - - buf: [32]u8, - buf_len: usize, msg_len: usize, pub fn init(seed: u64) Wyhash { return Wyhash{ .seed = seed, - .buf = undefined, - .buf_len = 0, .msg_len = 0, }; } @@ -61,34 +56,12 @@ pub const Wyhash = struct { ); } - pub fn update(self: *Wyhash, b: []const u8) void { - var off: usize = 0; + fn partial(self: *Wyhash, b: []const u8) void { + const rem_key = b; + const rem_len = b.len; - // Partial from previous. - if (self.buf_len != 0 and self.buf_len + b.len > 32) { - off += 32 - self.buf_len; - mem.copy(u8, self.buf[self.buf_len..], b[0..off]); - self.round(self.buf[0..]); - self.buf_len = 0; - } - - // Full middle blocks. - while (off + 32 <= b.len) : (off += 32) { - @inlineCall(self.round, b[off .. off + 32]); - } - - // Remainder for next pass. - mem.copy(u8, self.buf[self.buf_len..], b[off..]); - self.buf_len += @intCast(u8, b[off..].len); - self.msg_len += b.len; - } - - pub fn final(self: *Wyhash) u64 { - const seed = self.seed; - const rem_len = @intCast(u5, self.buf_len); - const rem_key = self.buf[0..self.buf_len]; - - self.seed = switch (rem_len) { + var seed = self.seed; + seed = switch (@intCast(u5, rem_len)) { 0 => seed, 1 => mix0(read_bytes(1, rem_key), primes[4], seed), 2 => mix0(read_bytes(2, rem_key), primes[4], seed), @@ -122,7 +95,22 @@ pub const Wyhash = struct { 30 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 16) | read_bytes(2, rem_key[28..]), seed), 31 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 24) | (read_bytes(2, rem_key[28..]) << 8) | read_bytes(1, rem_key[30..]), seed), }; + self.seed = seed; + } + pub fn update(self: *Wyhash, b: []const u8) void { + var off: usize = 0; + + // Full middle blocks. + while (off + 32 <= b.len) : (off += 32) { + @inlineCall(self.round, b[off .. off + 32]); + } + + self.partial(b[off..]); + self.msg_len += b.len; + } + + pub fn final(self: *Wyhash) u64 { return mum(self.seed ^ self.msg_len, primes[4]); } -- 2.54.0 From 54255ee32e1e6c83b04c3e5f2f1dd7e8aa5e0dd7 Mon Sep 17 00:00:00 2001 From: Sahnvour Date: Tue, 16 Jul 2019 22:32:10 +0200 Subject: [PATCH 22/23] autohash: force inlining of integer hashing so that the optimizer can see the fast path based on key's size which is known at comptime otherwise it will always outline the call to hasher.update, resulting in much worse performance --- std/hash/auto_hash.zig | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/std/hash/auto_hash.zig b/std/hash/auto_hash.zig index b21af0a1d8be73582bcc02e90fb034737f9a21b8..2da9691ffdd34ceee124f9b434f6e444238a9a63 100644 --- a/std/hash/auto_hash.zig +++ b/std/hash/auto_hash.zig @@ -21,7 +21,9 @@ pub fn autoHash(hasher: var, key: var) void { builtin.TypeId.EnumLiteral, => @compileError("cannot hash this type"), - builtin.TypeId.Int => hasher.update(std.mem.asBytes(&key)), + // Help the optimizer see that hashing an int is easy by inlining! + // TODO Check if the situation is better after #561 is resolved. + builtin.TypeId.Int => @inlineCall(hasher.update, std.mem.asBytes(&key)), builtin.TypeId.Float => |info| autoHash(hasher, @bitCast(@IntType(false, info.bits), key)), -- 2.54.0 From 30466bccefedd5e795b72b422f98b8a58e786289 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 4 Aug 2019 15:15:11 -0400 Subject: [PATCH 23/23] update CONTRIBUTING.md --- CONTRIBUTING.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c19ef4882966d299faa73a8984ed90a3c9369d5f..2ee0e85ccfaeec5806d881108f1324d86060ac20 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -25,6 +25,7 @@ Here are some examples: * [Iterative Replacement of C with Zig](http://tiehuis.github.io/blog/zig1.html) * [The Right Tool for the Right Job: Redis Modules & Zig](https://www.youtube.com/watch?v=eCHM8-_poZY) + * [Writing a small ray tracer in Rust and Zig](https://nelari.us/post/raytracer_with_rust_and_zig/) Zig is a brand new language, with no advertising budget. Word of mouth is the only way people find out about the project, and the more people hear about it, @@ -45,8 +46,8 @@ The most highly regarded argument in such a discussion is a real world use case. The issue label [Contributor Friendly](https://github.com/ziglang/zig/issues?q=is%3Aissue+is%3Aopen+label%3A%22contributor+friendly%22) -exists to help contributors find issues that are "limited in scope and/or -knowledge of Zig internals." +exists to help you find issues that are **limited in scope and/or +knowledge of Zig internals.** ### Editing Source Code @@ -61,8 +62,7 @@ To test changes, do the following from the build directory: 1. Run `make install` (on POSIX) or `msbuild -p:Configuration=Release INSTALL.vcxproj` (on Windows). -2. `bin/zig build --build-file ../build.zig test` (on POSIX) or - `bin\zig.exe build --build-file ..\build.zig test` (on Windows). +2. `bin/zig build test` (on POSIX) or `bin\zig.exe build test` (on Windows). That runs the whole test suite, which does a lot of extra testing that you likely won't always need, and can take upwards of 2 hours. This is what the @@ -79,8 +79,8 @@ Another example is choosing a different set of things to test. For example, not the other ones. Combining this suggestion with the previous one, you could do this: -`bin/zig build --build-file ../build.zig test-std -Dskip-release` (on POSIX) or -`bin\zig.exe build --build-file ..\build.zig test-std -Dskip-release` (on Windows). +`bin/zig build test-std -Dskip-release` (on POSIX) or +`bin\zig.exe build test-std -Dskip-release` (on Windows). This will run only the standard library tests, in debug mode only, for all targets (it will cross-compile the tests for non-native targets but not run -- 2.54.0