From 23fb19fe4159b0ed41256b31ebac58934efcbb36 Mon Sep 17 00:00:00 2001 From: Asherah Connor Date: Wed, 13 Jan 2021 17:32:48 +1100 Subject: [PATCH 01/35] std.elf: expose parsing decoupled from std.fs.File --- lib/std/elf.zig | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/std/elf.zig b/lib/std/elf.zig index 6dfa3734141d3327f942d16f62d680d934d11567..09409ffba8c6970bef904415c8d3ccbf7c4debfc 100644 --- a/lib/std/elf.zig +++ b/lib/std/elf.zig @@ -365,8 +365,12 @@ const Header = struct { pub fn readHeader(file: File) !Header { var hdr_buf: [@sizeOf(Elf64_Ehdr)]u8 align(@alignOf(Elf64_Ehdr)) = undefined; try preadNoEof(file, &hdr_buf, 0); - const hdr32 = @ptrCast(*Elf32_Ehdr, &hdr_buf); - const hdr64 = @ptrCast(*Elf64_Ehdr, &hdr_buf); + return parseHeader(hdr_buf); +} + +pub fn parseHeader(hdr_buf: *align(@alignOf(Elf64_Ehdr)) const [@sizeOf(Elf64_Ehdr)]u8) !Header { + const hdr32 = @ptrCast(*const Elf32_Ehdr, hdr_buf); + const hdr64 = @ptrCast(*const Elf64_Ehdr, hdr_buf); if (!mem.eql(u8, hdr32.e_ident[0..4], "\x7fELF")) return error.InvalidElfMagic; if (hdr32.e_ident[EI_VERSION] != 1) return error.InvalidElfVersion; -- 2.54.0 From e6cdf9cebf68d77aa1292433f6455e4d4ff8587b Mon Sep 17 00:00:00 2001 From: Asherah Connor Date: Wed, 13 Jan 2021 17:39:36 +1100 Subject: [PATCH 02/35] std.elf: make Header pub --- lib/std/elf.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/elf.zig b/lib/std/elf.zig index 09409ffba8c6970bef904415c8d3ccbf7c4debfc..db30ca86134dec30c478c5bddad31672e6a6c992 100644 --- a/lib/std/elf.zig +++ b/lib/std/elf.zig @@ -335,7 +335,7 @@ pub const ET = extern enum(u16) { }; /// All integers are native endian. -const Header = struct { +pub const Header = struct { endian: builtin.Endian, is_64: bool, entry: u64, -- 2.54.0 From 1cea88917cc9fc3f5237c75702aed815c206e64d Mon Sep 17 00:00:00 2001 From: Asherah Connor Date: Wed, 13 Jan 2021 20:41:18 +1100 Subject: [PATCH 03/35] std.elf: call it Header.parse --- lib/std/elf.zig | 68 ++++++++++++++++++++++++------------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/lib/std/elf.zig b/lib/std/elf.zig index db30ca86134dec30c478c5bddad31672e6a6c992..c6238e7cc7ed2791c841ce0d1fe9f57bd352cc03 100644 --- a/lib/std/elf.zig +++ b/lib/std/elf.zig @@ -360,45 +360,45 @@ pub const Header = struct { .file = file, }; } + + pub fn parse(hdr_buf: *align(@alignOf(Elf64_Ehdr)) const [@sizeOf(Elf64_Ehdr)]u8) !Header { + const hdr32 = @ptrCast(*const Elf32_Ehdr, hdr_buf); + const hdr64 = @ptrCast(*const Elf64_Ehdr, hdr_buf); + if (!mem.eql(u8, hdr32.e_ident[0..4], "\x7fELF")) return error.InvalidElfMagic; + if (hdr32.e_ident[EI_VERSION] != 1) return error.InvalidElfVersion; + + const endian: std.builtin.Endian = switch (hdr32.e_ident[EI_DATA]) { + ELFDATA2LSB => .Little, + ELFDATA2MSB => .Big, + else => return error.InvalidElfEndian, + }; + const need_bswap = endian != std.builtin.endian; + + const is_64 = switch (hdr32.e_ident[EI_CLASS]) { + ELFCLASS32 => false, + ELFCLASS64 => true, + else => return error.InvalidElfClass, + }; + + return @as(Header, .{ + .endian = endian, + .is_64 = is_64, + .entry = int(is_64, need_bswap, hdr32.e_entry, hdr64.e_entry), + .phoff = int(is_64, need_bswap, hdr32.e_phoff, hdr64.e_phoff), + .shoff = int(is_64, need_bswap, hdr32.e_shoff, hdr64.e_shoff), + .phentsize = int(is_64, need_bswap, hdr32.e_phentsize, hdr64.e_phentsize), + .phnum = int(is_64, need_bswap, hdr32.e_phnum, hdr64.e_phnum), + .shentsize = int(is_64, need_bswap, hdr32.e_shentsize, hdr64.e_shentsize), + .shnum = int(is_64, need_bswap, hdr32.e_shnum, hdr64.e_shnum), + .shstrndx = int(is_64, need_bswap, hdr32.e_shstrndx, hdr64.e_shstrndx), + }); + } }; pub fn readHeader(file: File) !Header { var hdr_buf: [@sizeOf(Elf64_Ehdr)]u8 align(@alignOf(Elf64_Ehdr)) = undefined; try preadNoEof(file, &hdr_buf, 0); - return parseHeader(hdr_buf); -} - -pub fn parseHeader(hdr_buf: *align(@alignOf(Elf64_Ehdr)) const [@sizeOf(Elf64_Ehdr)]u8) !Header { - const hdr32 = @ptrCast(*const Elf32_Ehdr, hdr_buf); - const hdr64 = @ptrCast(*const Elf64_Ehdr, hdr_buf); - if (!mem.eql(u8, hdr32.e_ident[0..4], "\x7fELF")) return error.InvalidElfMagic; - if (hdr32.e_ident[EI_VERSION] != 1) return error.InvalidElfVersion; - - const endian: std.builtin.Endian = switch (hdr32.e_ident[EI_DATA]) { - ELFDATA2LSB => .Little, - ELFDATA2MSB => .Big, - else => return error.InvalidElfEndian, - }; - const need_bswap = endian != std.builtin.endian; - - const is_64 = switch (hdr32.e_ident[EI_CLASS]) { - ELFCLASS32 => false, - ELFCLASS64 => true, - else => return error.InvalidElfClass, - }; - - return @as(Header, .{ - .endian = endian, - .is_64 = is_64, - .entry = int(is_64, need_bswap, hdr32.e_entry, hdr64.e_entry), - .phoff = int(is_64, need_bswap, hdr32.e_phoff, hdr64.e_phoff), - .shoff = int(is_64, need_bswap, hdr32.e_shoff, hdr64.e_shoff), - .phentsize = int(is_64, need_bswap, hdr32.e_phentsize, hdr64.e_phentsize), - .phnum = int(is_64, need_bswap, hdr32.e_phnum, hdr64.e_phnum), - .shentsize = int(is_64, need_bswap, hdr32.e_shentsize, hdr64.e_shentsize), - .shnum = int(is_64, need_bswap, hdr32.e_shnum, hdr64.e_shnum), - .shstrndx = int(is_64, need_bswap, hdr32.e_shstrndx, hdr64.e_shstrndx), - }); + return Header.parse(hdr_buf); } pub const ProgramHeaderIterator = struct { -- 2.54.0 From 49ab6bb429dfb3c2ed67b6daf3b5b55033a44fd0 Mon Sep 17 00:00:00 2001 From: Asherah Connor Date: Wed, 13 Jan 2021 21:01:24 +1100 Subject: [PATCH 04/35] std.elf: actually pass the pointer --- lib/std/elf.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/elf.zig b/lib/std/elf.zig index c6238e7cc7ed2791c841ce0d1fe9f57bd352cc03..0be16cafe6ac68a14ad713c25a5833faf71403fd 100644 --- a/lib/std/elf.zig +++ b/lib/std/elf.zig @@ -398,7 +398,7 @@ pub const Header = struct { pub fn readHeader(file: File) !Header { var hdr_buf: [@sizeOf(Elf64_Ehdr)]u8 align(@alignOf(Elf64_Ehdr)) = undefined; try preadNoEof(file, &hdr_buf, 0); - return Header.parse(hdr_buf); + return Header.parse(&hdr_buf); } pub const ProgramHeaderIterator = struct { -- 2.54.0 From 32d69d70cf5c915e2b2b0664c269f7e029d3c2f4 Mon Sep 17 00:00:00 2001 From: Asherah Connor Date: Thu, 14 Jan 2021 15:30:28 +1100 Subject: [PATCH 05/35] expose phdr, shdr parsing --- lib/std/build/emit_raw.zig | 7 +- lib/std/elf.zig | 192 ++++++++++++++++++++----------------- src/test.zig | 5 +- 3 files changed, 110 insertions(+), 94 deletions(-) diff --git a/lib/std/build/emit_raw.zig b/lib/std/build/emit_raw.zig index 3d2c6124c1663d38bfc387257bfc596ec762012b..68572f143bf7f3c388f84cdb6e450cc011415f4b 100644 --- a/lib/std/build/emit_raw.zig +++ b/lib/std/build/emit_raw.zig @@ -51,9 +51,10 @@ const BinaryElfOutput = struct { .segments = ArrayList(*BinaryElfSegment).init(allocator), .sections = ArrayList(*BinaryElfSection).init(allocator), }; - const elf_hdr = try std.elf.readHeader(elf_file); + const elf_source = std.elf.FileParseSource{ .file = elf_file }; + const elf_hdr = try std.elf.Header.read(elf_source); - var section_headers = elf_hdr.section_header_iterator(elf_file); + var section_headers = elf_hdr.section_header_iterator(elf_source); while (try section_headers.next()) |section| { if (sectionValidForOutput(section)) { const newSection = try allocator.create(BinaryElfSection); @@ -67,7 +68,7 @@ const BinaryElfOutput = struct { } } - var program_headers = elf_hdr.program_header_iterator(elf_file); + var program_headers = elf_hdr.program_header_iterator(elf_source); while (try program_headers.next()) |phdr| { if (phdr.p_type == elf.PT_LOAD) { const newSegment = try allocator.create(BinaryElfSegment); diff --git a/lib/std/elf.zig b/lib/std/elf.zig index 0be16cafe6ac68a14ad713c25a5833faf71403fd..7bd5d1901784174f6e57bcf9480030e9fa625c1c 100644 --- a/lib/std/elf.zig +++ b/lib/std/elf.zig @@ -334,6 +334,40 @@ pub const ET = extern enum(u16) { pub const HIPROC = 0xffff; }; +pub const FileParseSource = struct { + file: std.fs.File, + + fn readNoEof(self: FileParseSource, buf: []u8, offset: u64) !void { + var i: usize = 0; + while (i < buf.len) { + const len = self.file.pread(buf[i .. buf.len - i], offset + i) catch |err| switch (err) { + error.SystemResources => return error.SystemResources, + error.IsDir => return error.UnableToReadElfFile, + error.OperationAborted => return error.UnableToReadElfFile, + error.BrokenPipe => return error.UnableToReadElfFile, + error.Unseekable => return error.UnableToReadElfFile, + error.ConnectionResetByPeer => return error.UnableToReadElfFile, + error.ConnectionTimedOut => return error.UnableToReadElfFile, + error.InputOutput => return error.FileSystem, + error.Unexpected => return error.Unexpected, + error.WouldBlock => return error.Unexpected, + error.NotOpenForReading => return error.Unexpected, + error.AccessDenied => return error.Unexpected, + }; + if (len == 0) return error.UnexpectedEndOfFile; + i += len; + } + } +}; + +pub const BufferParseSource = struct { + buffer: []const u8, + + fn readNoEof(self: BufferParseSource, buf: []u8, offset: u64) !void { + std.mem.copy(u8, buf, self.buffer[offset .. offset + buf.len]); + } +}; + /// All integers are native endian. pub const Header = struct { endian: builtin.Endian, @@ -347,20 +381,26 @@ pub const Header = struct { shnum: u16, shstrndx: u16, - pub fn program_header_iterator(self: Header, file: File) ProgramHeaderIterator { - return .{ + pub fn program_header_iterator(self: Header, parse_source: anytype) ProgramHeaderIterator(@TypeOf(parse_source)) { + return ProgramHeaderIterator(@TypeOf(parse_source)){ .elf_header = self, - .file = file, + .parse_source = parse_source, }; } - pub fn section_header_iterator(self: Header, file: File) SectionHeaderIterator { + pub fn section_header_iterator(self: Header, parse_source: anytype) SectionHeaderIterator { return .{ .elf_header = self, - .file = file, + .parse_source = parse_source, }; } + pub fn read(parse_source: anytype) !Header { + var hdr_buf: [@sizeOf(Elf64_Ehdr)]u8 align(@alignOf(Elf64_Ehdr)) = undefined; + try parse_source.readNoEof(&hdr_buf, 0); + return Header.parse(&hdr_buf); + } + pub fn parse(hdr_buf: *align(@alignOf(Elf64_Ehdr)) const [@sizeOf(Elf64_Ehdr)]u8) !Header { const hdr32 = @ptrCast(*const Elf32_Ehdr, hdr_buf); const hdr64 = @ptrCast(*const Elf64_Ehdr, hdr_buf); @@ -395,78 +435,74 @@ pub const Header = struct { } }; -pub fn readHeader(file: File) !Header { - var hdr_buf: [@sizeOf(Elf64_Ehdr)]u8 align(@alignOf(Elf64_Ehdr)) = undefined; - try preadNoEof(file, &hdr_buf, 0); - return Header.parse(&hdr_buf); -} +pub fn ProgramHeaderIterator(ParseSource: anytype) type { + return struct { + elf_header: Header, + parse_source: ParseSource, + index: usize = 0, -pub const ProgramHeaderIterator = struct { - elf_header: Header, - file: File, - index: usize = 0, + pub fn next(self: *@This()) !?Elf64_Phdr { + if (self.index >= self.elf_header.phnum) return null; + defer self.index += 1; - pub fn next(self: *ProgramHeaderIterator) !?Elf64_Phdr { - if (self.index >= self.elf_header.phnum) return null; - defer self.index += 1; + if (self.elf_header.is_64) { + var phdr: Elf64_Phdr = undefined; + const offset = self.elf_header.phoff + @sizeOf(@TypeOf(phdr)) * self.index; + try self.parse_source.readNoEof(mem.asBytes(&phdr), offset); - if (self.elf_header.is_64) { - var phdr: Elf64_Phdr = undefined; + // ELF endianness matches native endianness. + if (self.elf_header.endian == std.builtin.endian) return phdr; + + // Convert fields to native endianness. + return Elf64_Phdr{ + .p_type = @byteSwap(@TypeOf(phdr.p_type), phdr.p_type), + .p_offset = @byteSwap(@TypeOf(phdr.p_offset), phdr.p_offset), + .p_vaddr = @byteSwap(@TypeOf(phdr.p_vaddr), phdr.p_vaddr), + .p_paddr = @byteSwap(@TypeOf(phdr.p_paddr), phdr.p_paddr), + .p_filesz = @byteSwap(@TypeOf(phdr.p_filesz), phdr.p_filesz), + .p_memsz = @byteSwap(@TypeOf(phdr.p_memsz), phdr.p_memsz), + .p_flags = @byteSwap(@TypeOf(phdr.p_flags), phdr.p_flags), + .p_align = @byteSwap(@TypeOf(phdr.p_align), phdr.p_align), + }; + } + + var phdr: Elf32_Phdr = undefined; const offset = self.elf_header.phoff + @sizeOf(@TypeOf(phdr)) * self.index; - try preadNoEof(self.file, mem.asBytes(&phdr), offset); + try self.parse_source.readNoEof(mem.asBytes(&phdr), offset); - // ELF endianness matches native endianness. - if (self.elf_header.endian == std.builtin.endian) return phdr; + // ELF endianness does NOT match native endianness. + if (self.elf_header.endian != std.builtin.endian) { + // Convert fields to native endianness. + phdr = .{ + .p_type = @byteSwap(@TypeOf(phdr.p_type), phdr.p_type), + .p_offset = @byteSwap(@TypeOf(phdr.p_offset), phdr.p_offset), + .p_vaddr = @byteSwap(@TypeOf(phdr.p_vaddr), phdr.p_vaddr), + .p_paddr = @byteSwap(@TypeOf(phdr.p_paddr), phdr.p_paddr), + .p_filesz = @byteSwap(@TypeOf(phdr.p_filesz), phdr.p_filesz), + .p_memsz = @byteSwap(@TypeOf(phdr.p_memsz), phdr.p_memsz), + .p_flags = @byteSwap(@TypeOf(phdr.p_flags), phdr.p_flags), + .p_align = @byteSwap(@TypeOf(phdr.p_align), phdr.p_align), + }; + } - // Convert fields to native endianness. + // Convert 32-bit header to 64-bit. return Elf64_Phdr{ - .p_type = @byteSwap(@TypeOf(phdr.p_type), phdr.p_type), - .p_offset = @byteSwap(@TypeOf(phdr.p_offset), phdr.p_offset), - .p_vaddr = @byteSwap(@TypeOf(phdr.p_vaddr), phdr.p_vaddr), - .p_paddr = @byteSwap(@TypeOf(phdr.p_paddr), phdr.p_paddr), - .p_filesz = @byteSwap(@TypeOf(phdr.p_filesz), phdr.p_filesz), - .p_memsz = @byteSwap(@TypeOf(phdr.p_memsz), phdr.p_memsz), - .p_flags = @byteSwap(@TypeOf(phdr.p_flags), phdr.p_flags), - .p_align = @byteSwap(@TypeOf(phdr.p_align), phdr.p_align), + .p_type = phdr.p_type, + .p_offset = phdr.p_offset, + .p_vaddr = phdr.p_vaddr, + .p_paddr = phdr.p_paddr, + .p_filesz = phdr.p_filesz, + .p_memsz = phdr.p_memsz, + .p_flags = phdr.p_flags, + .p_align = phdr.p_align, }; } - - var phdr: Elf32_Phdr = undefined; - const offset = self.elf_header.phoff + @sizeOf(@TypeOf(phdr)) * self.index; - try preadNoEof(self.file, mem.asBytes(&phdr), offset); - - // ELF endianness does NOT match native endianness. - if (self.elf_header.endian != std.builtin.endian) { - // Convert fields to native endianness. - phdr = .{ - .p_type = @byteSwap(@TypeOf(phdr.p_type), phdr.p_type), - .p_offset = @byteSwap(@TypeOf(phdr.p_offset), phdr.p_offset), - .p_vaddr = @byteSwap(@TypeOf(phdr.p_vaddr), phdr.p_vaddr), - .p_paddr = @byteSwap(@TypeOf(phdr.p_paddr), phdr.p_paddr), - .p_filesz = @byteSwap(@TypeOf(phdr.p_filesz), phdr.p_filesz), - .p_memsz = @byteSwap(@TypeOf(phdr.p_memsz), phdr.p_memsz), - .p_flags = @byteSwap(@TypeOf(phdr.p_flags), phdr.p_flags), - .p_align = @byteSwap(@TypeOf(phdr.p_align), phdr.p_align), - }; - } - - // Convert 32-bit header to 64-bit. - return Elf64_Phdr{ - .p_type = phdr.p_type, - .p_offset = phdr.p_offset, - .p_vaddr = phdr.p_vaddr, - .p_paddr = phdr.p_paddr, - .p_filesz = phdr.p_filesz, - .p_memsz = phdr.p_memsz, - .p_flags = phdr.p_flags, - .p_align = phdr.p_align, - }; - } -}; + }; +} pub const SectionHeaderIterator = struct { elf_header: Header, - file: File, + parse_source: ParseSource, index: usize = 0, pub fn next(self: *SectionHeaderIterator) !?Elf64_Shdr { @@ -476,7 +512,7 @@ pub const SectionHeaderIterator = struct { if (self.elf_header.is_64) { var shdr: Elf64_Shdr = undefined; const offset = self.elf_header.shoff + @sizeOf(@TypeOf(shdr)) * self.index; - try preadNoEof(self.file, mem.asBytes(&shdr), offset); + try self.parse_source.readNoEof(mem.asBytes(&shdr), offset); // ELF endianness matches native endianness. if (self.elf_header.endian == std.builtin.endian) return shdr; @@ -498,7 +534,7 @@ pub const SectionHeaderIterator = struct { var shdr: Elf32_Shdr = undefined; const offset = self.elf_header.shoff + @sizeOf(@TypeOf(shdr)) * self.index; - try preadNoEof(self.file, mem.asBytes(&shdr), offset); + try self.parse_source.readNoEof(mem.asBytes(&shdr), offset); // ELF endianness does NOT match native endianness. if (self.elf_header.endian != std.builtin.endian) { @@ -553,28 +589,6 @@ pub fn int32(need_bswap: bool, int_32: anytype, comptime Int64: anytype) Int64 { } } -fn preadNoEof(file: std.fs.File, buf: []u8, offset: u64) !void { - var i: usize = 0; - while (i < buf.len) { - const len = file.pread(buf[i .. buf.len - i], offset + i) catch |err| switch (err) { - error.SystemResources => return error.SystemResources, - error.IsDir => return error.UnableToReadElfFile, - error.OperationAborted => return error.UnableToReadElfFile, - error.BrokenPipe => return error.UnableToReadElfFile, - error.Unseekable => return error.UnableToReadElfFile, - error.ConnectionResetByPeer => return error.UnableToReadElfFile, - error.ConnectionTimedOut => return error.UnableToReadElfFile, - error.InputOutput => return error.FileSystem, - error.Unexpected => return error.Unexpected, - error.WouldBlock => return error.Unexpected, - error.NotOpenForReading => return error.Unexpected, - error.AccessDenied => return error.Unexpected, - }; - if (len == 0) return error.UnexpectedEndOfFile; - i += len; - } -} - pub const EI_NIDENT = 16; pub const EI_CLASS = 4; diff --git a/src/test.zig b/src/test.zig index 59927525df5e91ab3d642ce0aff8237b2d9f6051..3e2a3a787c628b015be0ae4c25cc4fe828b02801 100644 --- a/src/test.zig +++ b/src/test.zig @@ -989,8 +989,9 @@ pub const TestContext = struct { var file = try tmp_dir.openFile(bin_name, .{ .read = true }); defer file.close(); - const header = try std.elf.readHeader(file); - var iterator = header.program_header_iterator(file); + const elf_source = std.elf.FileParseSource{ .file = file }; + const header = try std.elf.Header.read(elf_source); + var iterator = header.program_header_iterator(elf_source); var none_loaded = true; -- 2.54.0 From 9168b217b23c5270328c3846fec61930ec7d2533 Mon Sep 17 00:00:00 2001 From: Asherah Connor Date: Thu, 14 Jan 2021 17:52:06 +1100 Subject: [PATCH 06/35] fix SectionHeaderIterator impl --- lib/std/elf.zig | 124 ++++++++++++++++++++++++------------------------ 1 file changed, 63 insertions(+), 61 deletions(-) diff --git a/lib/std/elf.zig b/lib/std/elf.zig index 7bd5d1901784174f6e57bcf9480030e9fa625c1c..a9495a223832410f331e7a2d4fcaa914d1c8b991 100644 --- a/lib/std/elf.zig +++ b/lib/std/elf.zig @@ -388,8 +388,8 @@ pub const Header = struct { }; } - pub fn section_header_iterator(self: Header, parse_source: anytype) SectionHeaderIterator { - return .{ + pub fn section_header_iterator(self: Header, parse_source: anytype) SectionHeaderIterator(@TypeOf(parse_source)) { + return SectionHeaderIterator(@TypeOf(parse_source)){ .elf_header = self, .parse_source = parse_source, }; @@ -500,74 +500,76 @@ pub fn ProgramHeaderIterator(ParseSource: anytype) type { }; } -pub const SectionHeaderIterator = struct { - elf_header: Header, - parse_source: ParseSource, - index: usize = 0, +pub fn SectionHeaderIterator(ParseSource: anytype) type { + return struct { + elf_header: Header, + parse_source: ParseSource, + index: usize = 0, - pub fn next(self: *SectionHeaderIterator) !?Elf64_Shdr { - if (self.index >= self.elf_header.shnum) return null; - defer self.index += 1; + pub fn next(self: *@This()) !?Elf64_Shdr { + if (self.index >= self.elf_header.shnum) return null; + defer self.index += 1; - if (self.elf_header.is_64) { - var shdr: Elf64_Shdr = undefined; + if (self.elf_header.is_64) { + var shdr: Elf64_Shdr = undefined; + const offset = self.elf_header.shoff + @sizeOf(@TypeOf(shdr)) * self.index; + try self.parse_source.readNoEof(mem.asBytes(&shdr), offset); + + // ELF endianness matches native endianness. + if (self.elf_header.endian == std.builtin.endian) return shdr; + + // Convert fields to native endianness. + return Elf64_Shdr{ + .sh_name = @byteSwap(@TypeOf(shdr.sh_name), shdr.sh_name), + .sh_type = @byteSwap(@TypeOf(shdr.sh_type), shdr.sh_type), + .sh_flags = @byteSwap(@TypeOf(shdr.sh_flags), shdr.sh_flags), + .sh_addr = @byteSwap(@TypeOf(shdr.sh_addr), shdr.sh_addr), + .sh_offset = @byteSwap(@TypeOf(shdr.sh_offset), shdr.sh_offset), + .sh_size = @byteSwap(@TypeOf(shdr.sh_size), shdr.sh_size), + .sh_link = @byteSwap(@TypeOf(shdr.sh_link), shdr.sh_link), + .sh_info = @byteSwap(@TypeOf(shdr.sh_info), shdr.sh_info), + .sh_addralign = @byteSwap(@TypeOf(shdr.sh_addralign), shdr.sh_addralign), + .sh_entsize = @byteSwap(@TypeOf(shdr.sh_entsize), shdr.sh_entsize), + }; + } + + var shdr: Elf32_Shdr = undefined; const offset = self.elf_header.shoff + @sizeOf(@TypeOf(shdr)) * self.index; try self.parse_source.readNoEof(mem.asBytes(&shdr), offset); - // ELF endianness matches native endianness. - if (self.elf_header.endian == std.builtin.endian) return shdr; + // ELF endianness does NOT match native endianness. + if (self.elf_header.endian != std.builtin.endian) { + // Convert fields to native endianness. + shdr = .{ + .sh_name = @byteSwap(@TypeOf(shdr.sh_name), shdr.sh_name), + .sh_type = @byteSwap(@TypeOf(shdr.sh_type), shdr.sh_type), + .sh_flags = @byteSwap(@TypeOf(shdr.sh_flags), shdr.sh_flags), + .sh_addr = @byteSwap(@TypeOf(shdr.sh_addr), shdr.sh_addr), + .sh_offset = @byteSwap(@TypeOf(shdr.sh_offset), shdr.sh_offset), + .sh_size = @byteSwap(@TypeOf(shdr.sh_size), shdr.sh_size), + .sh_link = @byteSwap(@TypeOf(shdr.sh_link), shdr.sh_link), + .sh_info = @byteSwap(@TypeOf(shdr.sh_info), shdr.sh_info), + .sh_addralign = @byteSwap(@TypeOf(shdr.sh_addralign), shdr.sh_addralign), + .sh_entsize = @byteSwap(@TypeOf(shdr.sh_entsize), shdr.sh_entsize), + }; + } - // Convert fields to native endianness. + // Convert 32-bit header to 64-bit. return Elf64_Shdr{ - .sh_name = @byteSwap(@TypeOf(shdr.sh_name), shdr.sh_name), - .sh_type = @byteSwap(@TypeOf(shdr.sh_type), shdr.sh_type), - .sh_flags = @byteSwap(@TypeOf(shdr.sh_flags), shdr.sh_flags), - .sh_addr = @byteSwap(@TypeOf(shdr.sh_addr), shdr.sh_addr), - .sh_offset = @byteSwap(@TypeOf(shdr.sh_offset), shdr.sh_offset), - .sh_size = @byteSwap(@TypeOf(shdr.sh_size), shdr.sh_size), - .sh_link = @byteSwap(@TypeOf(shdr.sh_link), shdr.sh_link), - .sh_info = @byteSwap(@TypeOf(shdr.sh_info), shdr.sh_info), - .sh_addralign = @byteSwap(@TypeOf(shdr.sh_addralign), shdr.sh_addralign), - .sh_entsize = @byteSwap(@TypeOf(shdr.sh_entsize), shdr.sh_entsize), + .sh_name = shdr.sh_name, + .sh_type = shdr.sh_type, + .sh_flags = shdr.sh_flags, + .sh_addr = shdr.sh_addr, + .sh_offset = shdr.sh_offset, + .sh_size = shdr.sh_size, + .sh_link = shdr.sh_link, + .sh_info = shdr.sh_info, + .sh_addralign = shdr.sh_addralign, + .sh_entsize = shdr.sh_entsize, }; } - - var shdr: Elf32_Shdr = undefined; - const offset = self.elf_header.shoff + @sizeOf(@TypeOf(shdr)) * self.index; - try self.parse_source.readNoEof(mem.asBytes(&shdr), offset); - - // ELF endianness does NOT match native endianness. - if (self.elf_header.endian != std.builtin.endian) { - // Convert fields to native endianness. - shdr = .{ - .sh_name = @byteSwap(@TypeOf(shdr.sh_name), shdr.sh_name), - .sh_type = @byteSwap(@TypeOf(shdr.sh_type), shdr.sh_type), - .sh_flags = @byteSwap(@TypeOf(shdr.sh_flags), shdr.sh_flags), - .sh_addr = @byteSwap(@TypeOf(shdr.sh_addr), shdr.sh_addr), - .sh_offset = @byteSwap(@TypeOf(shdr.sh_offset), shdr.sh_offset), - .sh_size = @byteSwap(@TypeOf(shdr.sh_size), shdr.sh_size), - .sh_link = @byteSwap(@TypeOf(shdr.sh_link), shdr.sh_link), - .sh_info = @byteSwap(@TypeOf(shdr.sh_info), shdr.sh_info), - .sh_addralign = @byteSwap(@TypeOf(shdr.sh_addralign), shdr.sh_addralign), - .sh_entsize = @byteSwap(@TypeOf(shdr.sh_entsize), shdr.sh_entsize), - }; - } - - // Convert 32-bit header to 64-bit. - return Elf64_Shdr{ - .sh_name = shdr.sh_name, - .sh_type = shdr.sh_type, - .sh_flags = shdr.sh_flags, - .sh_addr = shdr.sh_addr, - .sh_offset = shdr.sh_offset, - .sh_size = shdr.sh_size, - .sh_link = shdr.sh_link, - .sh_info = shdr.sh_info, - .sh_addralign = shdr.sh_addralign, - .sh_entsize = shdr.sh_entsize, - }; - } -}; + }; +} pub fn int(is_64: bool, need_bswap: bool, int_32: anytype, int_64: anytype) @TypeOf(int_64) { if (is_64) { -- 2.54.0 From affb57aad97240bf183015e6acba7eab4a9276bb Mon Sep 17 00:00:00 2001 From: Asherah Connor Date: Fri, 15 Jan 2021 10:34:53 +1100 Subject: [PATCH 07/35] use interfaces --- lib/std/build/emit_raw.zig | 7 +++--- lib/std/elf.zig | 49 ++++++++------------------------------ src/test.zig | 5 ++-- 3 files changed, 15 insertions(+), 46 deletions(-) diff --git a/lib/std/build/emit_raw.zig b/lib/std/build/emit_raw.zig index 68572f143bf7f3c388f84cdb6e450cc011415f4b..e6351f509530d2634bdb13e6e8e0a6cc5ffcef73 100644 --- a/lib/std/build/emit_raw.zig +++ b/lib/std/build/emit_raw.zig @@ -51,10 +51,9 @@ const BinaryElfOutput = struct { .segments = ArrayList(*BinaryElfSegment).init(allocator), .sections = ArrayList(*BinaryElfSection).init(allocator), }; - const elf_source = std.elf.FileParseSource{ .file = elf_file }; - const elf_hdr = try std.elf.Header.read(elf_source); + const elf_hdr = try std.elf.Header.read(&elf_file); - var section_headers = elf_hdr.section_header_iterator(elf_source); + var section_headers = elf_hdr.section_header_iterator(&elf_file); while (try section_headers.next()) |section| { if (sectionValidForOutput(section)) { const newSection = try allocator.create(BinaryElfSection); @@ -68,7 +67,7 @@ const BinaryElfOutput = struct { } } - var program_headers = elf_hdr.program_header_iterator(elf_source); + var program_headers = elf_hdr.program_header_iterator(&elf_file); while (try program_headers.next()) |phdr| { if (phdr.p_type == elf.PT_LOAD) { const newSegment = try allocator.create(BinaryElfSegment); diff --git a/lib/std/elf.zig b/lib/std/elf.zig index a9495a223832410f331e7a2d4fcaa914d1c8b991..b124dcedd43bdaa9f024198641509d3bb7f5637c 100644 --- a/lib/std/elf.zig +++ b/lib/std/elf.zig @@ -334,40 +334,6 @@ pub const ET = extern enum(u16) { pub const HIPROC = 0xffff; }; -pub const FileParseSource = struct { - file: std.fs.File, - - fn readNoEof(self: FileParseSource, buf: []u8, offset: u64) !void { - var i: usize = 0; - while (i < buf.len) { - const len = self.file.pread(buf[i .. buf.len - i], offset + i) catch |err| switch (err) { - error.SystemResources => return error.SystemResources, - error.IsDir => return error.UnableToReadElfFile, - error.OperationAborted => return error.UnableToReadElfFile, - error.BrokenPipe => return error.UnableToReadElfFile, - error.Unseekable => return error.UnableToReadElfFile, - error.ConnectionResetByPeer => return error.UnableToReadElfFile, - error.ConnectionTimedOut => return error.UnableToReadElfFile, - error.InputOutput => return error.FileSystem, - error.Unexpected => return error.Unexpected, - error.WouldBlock => return error.Unexpected, - error.NotOpenForReading => return error.Unexpected, - error.AccessDenied => return error.Unexpected, - }; - if (len == 0) return error.UnexpectedEndOfFile; - i += len; - } - } -}; - -pub const BufferParseSource = struct { - buffer: []const u8, - - fn readNoEof(self: BufferParseSource, buf: []u8, offset: u64) !void { - std.mem.copy(u8, buf, self.buffer[offset .. offset + buf.len]); - } -}; - /// All integers are native endian. pub const Header = struct { endian: builtin.Endian, @@ -397,7 +363,8 @@ pub const Header = struct { pub fn read(parse_source: anytype) !Header { var hdr_buf: [@sizeOf(Elf64_Ehdr)]u8 align(@alignOf(Elf64_Ehdr)) = undefined; - try parse_source.readNoEof(&hdr_buf, 0); + try parse_source.seekableStream().seekTo(0); + try parse_source.reader().readNoEof(&hdr_buf); return Header.parse(&hdr_buf); } @@ -448,7 +415,8 @@ pub fn ProgramHeaderIterator(ParseSource: anytype) type { if (self.elf_header.is_64) { var phdr: Elf64_Phdr = undefined; const offset = self.elf_header.phoff + @sizeOf(@TypeOf(phdr)) * self.index; - try self.parse_source.readNoEof(mem.asBytes(&phdr), offset); + try self.parse_source.seekableStream().seekTo(offset); + try self.parse_source.reader().readNoEof(mem.asBytes(&phdr)); // ELF endianness matches native endianness. if (self.elf_header.endian == std.builtin.endian) return phdr; @@ -468,7 +436,8 @@ pub fn ProgramHeaderIterator(ParseSource: anytype) type { var phdr: Elf32_Phdr = undefined; const offset = self.elf_header.phoff + @sizeOf(@TypeOf(phdr)) * self.index; - try self.parse_source.readNoEof(mem.asBytes(&phdr), offset); + try self.parse_source.seekableStream().seekTo(offset); + try self.parse_source.reader().readNoEof(mem.asBytes(&phdr)); // ELF endianness does NOT match native endianness. if (self.elf_header.endian != std.builtin.endian) { @@ -513,7 +482,8 @@ pub fn SectionHeaderIterator(ParseSource: anytype) type { if (self.elf_header.is_64) { var shdr: Elf64_Shdr = undefined; const offset = self.elf_header.shoff + @sizeOf(@TypeOf(shdr)) * self.index; - try self.parse_source.readNoEof(mem.asBytes(&shdr), offset); + try self.parse_source.seekableStream().seekTo(offset); + try self.parse_source.reader().readNoEof(mem.asBytes(&shdr)); // ELF endianness matches native endianness. if (self.elf_header.endian == std.builtin.endian) return shdr; @@ -535,7 +505,8 @@ pub fn SectionHeaderIterator(ParseSource: anytype) type { var shdr: Elf32_Shdr = undefined; const offset = self.elf_header.shoff + @sizeOf(@TypeOf(shdr)) * self.index; - try self.parse_source.readNoEof(mem.asBytes(&shdr), offset); + try self.parse_source.seekableStream().seekTo(offset); + try self.parse_source.reader().readNoEof(mem.asBytes(&shdr)); // ELF endianness does NOT match native endianness. if (self.elf_header.endian != std.builtin.endian) { diff --git a/src/test.zig b/src/test.zig index 3e2a3a787c628b015be0ae4c25cc4fe828b02801..e6164d61d73fde91e514a7c126ec036f50e16b87 100644 --- a/src/test.zig +++ b/src/test.zig @@ -989,9 +989,8 @@ pub const TestContext = struct { var file = try tmp_dir.openFile(bin_name, .{ .read = true }); defer file.close(); - const elf_source = std.elf.FileParseSource{ .file = file }; - const header = try std.elf.Header.read(elf_source); - var iterator = header.program_header_iterator(elf_source); + const header = try std.elf.Header.read(&file); + var iterator = header.program_header_iterator(&file); var none_loaded = true; -- 2.54.0 From 1eb2e4801448aca29471bf6f7582135ddafb15fe Mon Sep 17 00:00:00 2001 From: Koakuma Date: Thu, 4 Feb 2021 20:51:41 +0700 Subject: [PATCH 08/35] std.debug.StackIterator: account for SPARC %fp quirk On SPARC, previous %fp is saved with a 14 slots offset from current %fp+bias. Also account for the bias constant at the new_fp calculation. --- lib/std/debug.zig | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/std/debug.zig b/lib/std/debug.zig index a434fe0e8b55a9a6c5c917109395435b08900c4a..b887899c7a3e0cd08e5960b2edeb5d19bc592050 100644 --- a/lib/std/debug.zig +++ b/lib/std/debug.zig @@ -366,8 +366,18 @@ pub const StackIterator = struct { // area, on pretty much every other architecture it points to the stack // slot where the previous frame pointer is saved. 2 * @sizeOf(usize) + else if (builtin.arch.isSPARC()) + // On SPARC the previous frame pointer is stored at 14 slots past %fp+BIAS. + 14 * @sizeOf(usize) else 0; + + const fp_bias = if (builtin.arch.isSPARC()) + // On SPARC frame pointers are biased by a constant. + 2047 + else + 0; + // Positive offset of the saved PC wrt the frame pointer. const pc_offset = if (builtin.arch == .powerpc64le) 2 * @sizeOf(usize) @@ -394,7 +404,7 @@ pub const StackIterator = struct { if (fp == 0 or !mem.isAligned(fp, @alignOf(usize))) return null; - const new_fp = @intToPtr(*const usize, fp).*; + const new_fp = math.add(usize, @intToPtr(*const usize, fp).*, fp_bias) catch return null; // Sanity check: the stack grows down thus all the parent frames must be // be at addresses that are greater (or equal) than the previous one. -- 2.54.0 From e3e4af727103d90cb191f130159928d237263e1f Mon Sep 17 00:00:00 2001 From: Koakuma Date: Thu, 4 Feb 2021 20:51:53 +0700 Subject: [PATCH 09/35] stage1: set gen_frame_size alignment to work around requirement mismatch Explicitly set the alignment requirements to 1 (i.e, mark the load as unaligned) since there are some architectures (e.g SPARCv9) which has different alignment requirements between a function pointer and usize pointer. On those architectures, not explicitly setting it will lead into @frameSize generating usize-aligned load instruction that could crash if the function pointer happens to be not usize-aligned. --- src/stage1/codegen.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/stage1/codegen.cpp b/src/stage1/codegen.cpp index 6aa134c3b0415e23f3682d740493c116234bb8cb..5236bb26ae6a63bdd0ecb42905e04aabc8231359 100644 --- a/src/stage1/codegen.cpp +++ b/src/stage1/codegen.cpp @@ -4160,7 +4160,9 @@ static LLVMValueRef gen_frame_size(CodeGen *g, LLVMValueRef fn_val) { LLVMValueRef casted_fn_val = LLVMBuildBitCast(g->builder, fn_val, ptr_usize_llvm_type, ""); LLVMValueRef negative_one = LLVMConstInt(LLVMInt32Type(), -1, true); LLVMValueRef prefix_ptr = LLVMBuildInBoundsGEP(g->builder, casted_fn_val, &negative_one, 1, ""); - return LLVMBuildLoad(g->builder, prefix_ptr, ""); + LLVMValueRef load_inst = LLVMBuildLoad(g->builder, prefix_ptr, ""); + LLVMSetAlignment(load_inst, 1); + return load_inst; } static void gen_init_stack_trace(CodeGen *g, LLVMValueRef trace_field_ptr, LLVMValueRef addrs_field_ptr) { -- 2.54.0 From d23dfdeab9358f0f026990e4d0f1da0ddaa6b177 Mon Sep 17 00:00:00 2001 From: Koakuma Date: Fri, 5 Feb 2021 00:25:01 +0700 Subject: [PATCH 10/35] Add comment explaining the alignment setting --- src/stage1/codegen.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/stage1/codegen.cpp b/src/stage1/codegen.cpp index 5236bb26ae6a63bdd0ecb42905e04aabc8231359..71a22ce79bd56484790ce682fc8034f0cbf2785e 100644 --- a/src/stage1/codegen.cpp +++ b/src/stage1/codegen.cpp @@ -4161,6 +4161,12 @@ static LLVMValueRef gen_frame_size(CodeGen *g, LLVMValueRef fn_val) { LLVMValueRef negative_one = LLVMConstInt(LLVMInt32Type(), -1, true); LLVMValueRef prefix_ptr = LLVMBuildInBoundsGEP(g->builder, casted_fn_val, &negative_one, 1, ""); LLVMValueRef load_inst = LLVMBuildLoad(g->builder, prefix_ptr, ""); + + // Some architectures (e.g SPARCv9) has different alignment requirements between a + // function/usize pointer and also require all loads to be aligned. + // On those architectures, not explicitly setting the alignment will lead into @frameSize + // generating usize-aligned load instruction that could crash if the function pointer + // happens to be not usize-aligned. LLVMSetAlignment(load_inst, 1); return load_inst; } -- 2.54.0 From 448a28325c7517e6cd62ba7932a4b6c836250757 Mon Sep 17 00:00:00 2001 From: Koakuma Date: Fri, 5 Feb 2021 00:28:07 +0700 Subject: [PATCH 11/35] Fix previous %fp calculation --- lib/std/debug.zig | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/std/debug.zig b/lib/std/debug.zig index b887899c7a3e0cd08e5960b2edeb5d19bc592050..9d90dd5c4b2f2bdbb5db6d27af47f01285dbbc8e 100644 --- a/lib/std/debug.zig +++ b/lib/std/debug.zig @@ -360,7 +360,7 @@ pub const StackIterator = struct { }; } - // Negative offset of the saved BP wrt the frame pointer. + // Offset of the saved BP wrt the frame pointer. const fp_offset = if (builtin.arch.isRISCV()) // On RISC-V the frame pointer points to the top of the saved register // area, on pretty much every other architecture it points to the stack @@ -398,7 +398,11 @@ pub const StackIterator = struct { } fn next_internal(self: *StackIterator) ?usize { - const fp = math.sub(usize, self.fp, fp_offset) catch return null; + const fp = if (builtin.arch.isSPARC()) + // On SPARC the offset is positive. (!) + math.add(usize, self.fp, fp_offset) catch return null + else + math.sub(usize, self.fp, fp_offset) catch return null; // Sanity check. if (fp == 0 or !mem.isAligned(fp, @alignOf(usize))) -- 2.54.0 From 345ac53836bce5eb39a04723b30ac82dd7d5098f Mon Sep 17 00:00:00 2001 From: joachimschmidt557 Date: Mon, 1 Mar 2021 22:43:42 +0100 Subject: [PATCH 12/35] stage2 ARM: Implement basic integer multiplication --- src/codegen.zig | 43 +++++++++++++++++++++++++++++++++++++++++++ src/ir.zig | 2 ++ src/zir.zig | 2 ++ src/zir_sema.zig | 1 + 4 files changed, 48 insertions(+) diff --git a/src/codegen.zig b/src/codegen.zig index 57fd732b426f7af1eb158d3ef4320360fff9b867..c3cd64cf73c8e4db9853a34a06bb03dcb1957c8b 100644 --- a/src/codegen.zig +++ b/src/codegen.zig @@ -899,6 +899,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { .load => return self.genLoad(inst.castTag(.load).?), .loop => return self.genLoop(inst.castTag(.loop).?), .not => return self.genNot(inst.castTag(.not).?), + .mul => return self.genMul(inst.castTag(.mul).?), .ptrtoint => return self.genPtrToInt(inst.castTag(.ptrtoint).?), .ref => return self.genRef(inst.castTag(.ref).?), .ret => return self.genRet(inst.castTag(.ret).?), @@ -1128,6 +1129,16 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { } } + fn genMul(self: *Self, inst: *ir.Inst.BinOp) !MCValue { + // No side effects, so if it's unreferenced, do nothing. + if (inst.base.isUnused()) + return MCValue.dead; + switch (arch) { + .arm, .armeb => return try self.genArmMul(&inst.base, inst.lhs, inst.rhs), + else => return self.fail(inst.base.src, "TODO implement mul for {}", .{self.target.cpu.arch}), + } + } + fn genBitAnd(self: *Self, inst: *ir.Inst.BinOp) !MCValue { // No side effects, so if it's unreferenced, do nothing. if (inst.base.isUnused()) @@ -1478,6 +1489,38 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { } } + fn genArmMul(self: *Self, inst: *ir.Inst, op_lhs: *ir.Inst, op_rhs: *ir.Inst) !MCValue { + const lhs = try self.resolveInst(op_lhs); + const rhs = try self.resolveInst(op_rhs); + + // Destination must be a register + // LHS must be a register + // RHS must be a register + var dst_mcv: MCValue = undefined; + var lhs_mcv: MCValue = undefined; + var rhs_mcv: MCValue = undefined; + if (self.reuseOperand(inst, 0, lhs)) { + // LHS is the destination + lhs_mcv = if (lhs != .register) try self.copyToNewRegister(inst, lhs) else lhs; + rhs_mcv = if (rhs != .register) try self.copyToNewRegister(inst, rhs) else rhs; + dst_mcv = lhs_mcv; + } else if (self.reuseOperand(inst, 1, rhs)) { + // RHS is the destination + lhs_mcv = if (lhs != .register) try self.copyToNewRegister(inst, lhs) else lhs; + rhs_mcv = if (rhs != .register) try self.copyToNewRegister(inst, rhs) else rhs; + dst_mcv = rhs_mcv; + } else { + // TODO save 1 copy instruction by directly allocating the destination register + // LHS is the destination + lhs_mcv = try self.copyToNewRegister(inst, lhs); + rhs_mcv = if (rhs != .register) try self.copyToNewRegister(inst, rhs) else rhs; + dst_mcv = lhs_mcv; + } + + writeInt(u32, try self.code.addManyAsArray(4), Instruction.mul(.al, dst_mcv.register, lhs_mcv.register, rhs_mcv.register).toU32()); + return dst_mcv; + } + /// ADD, SUB, XOR, OR, AND fn genX8664BinMath(self: *Self, inst: *ir.Inst, op_lhs: *ir.Inst, op_rhs: *ir.Inst, opx: u8, mr: u8) !MCValue { try self.code.ensureCapacity(self.code.items.len + 8); diff --git a/src/ir.zig b/src/ir.zig index eddc885d14dd5634f3da8448557228f4db208628..996f3b9782f31b2ed584a5065ed29f674a78f4bf 100644 --- a/src/ir.zig +++ b/src/ir.zig @@ -106,6 +106,7 @@ pub const Inst = struct { store, sub, unreach, + mul, not, floatcast, intcast, @@ -165,6 +166,7 @@ pub const Inst = struct { .add, .sub, + .mul, .cmp_lt, .cmp_lte, .cmp_eq, diff --git a/src/zir.zig b/src/zir.zig index d013f25ee5b3547470325c251cd0640965d4434a..7a3a1e2684f32582b133bb86f27c1a4c4c3eca61 100644 --- a/src/zir.zig +++ b/src/zir.zig @@ -1649,6 +1649,7 @@ const DumpTzir = struct { .add, .sub, + .mul, .cmp_lt, .cmp_lte, .cmp_eq, @@ -1771,6 +1772,7 @@ const DumpTzir = struct { .add, .sub, + .mul, .cmp_lt, .cmp_lte, .cmp_eq, diff --git a/src/zir_sema.zig b/src/zir_sema.zig index 864f766f5487a80404c8dcb6ebfcb144e35348ee..1cf550852fa74f97f5ba68bb612390cb438a5608 100644 --- a/src/zir_sema.zig +++ b/src/zir_sema.zig @@ -2075,6 +2075,7 @@ fn zirArithmetic(mod: *Module, scope: *Scope, inst: *zir.Inst.BinOp) InnerError! const ir_tag = switch (inst.base.tag) { .add => Inst.Tag.add, .sub => Inst.Tag.sub, + .mul => Inst.Tag.mul, else => return mod.fail(scope, inst.base.src, "TODO implement arithmetic for operand '{s}''", .{@tagName(inst.base.tag)}), }; -- 2.54.0 From cd7c870bd81391dd97c5c75eb3910382ba7280a1 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Fri, 12 Feb 2021 20:44:19 +0100 Subject: [PATCH 13/35] std: Deprecate 'x'/'X'/'e'/'E' special cases for u8 slices Let's follow the road paved by the removal of 'z'/'Z', the Formatter pattern is nice enough to let us remove the remaining four special cases and declare u8 slices free from any special casing! --- lib/std/crypto/25519/curve25519.zig | 4 +- lib/std/crypto/25519/ed25519.zig | 6 +- lib/std/crypto/25519/edwards25519.zig | 2 +- lib/std/crypto/25519/ristretto255.zig | 8 +- lib/std/crypto/25519/scalar.zig | 6 +- lib/std/crypto/chacha20.zig | 4 +- lib/std/fmt.zig | 130 ++++++++++++++++++++------ src/Cache.zig | 24 ++++- src/translate_c.zig | 2 +- 9 files changed, 140 insertions(+), 46 deletions(-) diff --git a/lib/std/crypto/25519/curve25519.zig b/lib/std/crypto/25519/curve25519.zig index 765ffa1629c119425823f27deda07fb0e2ae476d..e01b02436010313469dc02aa0bee748b63ac0673 100644 --- a/lib/std/crypto/25519/curve25519.zig +++ b/lib/std/crypto/25519/curve25519.zig @@ -115,9 +115,9 @@ test "curve25519" { const p = try Curve25519.basePoint.clampedMul(s); try p.rejectIdentity(); var buf: [128]u8 = undefined; - std.testing.expectEqualStrings(try std.fmt.bufPrint(&buf, "{X}", .{p.toBytes()}), "E6F2A4D1C28EE5C7AD0329268255A468AD407D2672824C0C0EB30EA6EF450145"); + std.testing.expectEqualStrings(try std.fmt.bufPrint(&buf, "{s}", .{std.fmt.fmtSliceHexUpper(&p.toBytes())}), "E6F2A4D1C28EE5C7AD0329268255A468AD407D2672824C0C0EB30EA6EF450145"); const q = try p.clampedMul(s); - std.testing.expectEqualStrings(try std.fmt.bufPrint(&buf, "{X}", .{q.toBytes()}), "3614E119FFE55EC55B87D6B19971A9F4CBC78EFE80BEC55B96392BABCC712537"); + std.testing.expectEqualStrings(try std.fmt.bufPrint(&buf, "{s}", .{std.fmt.fmtSliceHexUpper(&q.toBytes())}), "3614E119FFE55EC55B87D6B19971A9F4CBC78EFE80BEC55B96392BABCC712537"); try Curve25519.rejectNonCanonical(s); s[31] |= 0x80; diff --git a/lib/std/crypto/25519/ed25519.zig b/lib/std/crypto/25519/ed25519.zig index 5c7ec0cdac3024365783e729f751fda6003709ab..06a4826f58d24b75fec62a92146ca356d08286ad 100644 --- a/lib/std/crypto/25519/ed25519.zig +++ b/lib/std/crypto/25519/ed25519.zig @@ -210,8 +210,8 @@ test "ed25519 key pair creation" { _ = try fmt.hexToBytes(seed[0..], "8052030376d47112be7f73ed7a019293dd12ad910b654455798b4667d73de166"); const key_pair = try Ed25519.KeyPair.create(seed); var buf: [256]u8 = undefined; - std.testing.expectEqualStrings(try std.fmt.bufPrint(&buf, "{X}", .{key_pair.secret_key}), "8052030376D47112BE7F73ED7A019293DD12AD910B654455798B4667D73DE1662D6F7455D97B4A3A10D7293909D1A4F2058CB9A370E43FA8154BB280DB839083"); - std.testing.expectEqualStrings(try std.fmt.bufPrint(&buf, "{X}", .{key_pair.public_key}), "2D6F7455D97B4A3A10D7293909D1A4F2058CB9A370E43FA8154BB280DB839083"); + std.testing.expectEqualStrings(try std.fmt.bufPrint(&buf, "{s}", .{std.fmt.fmtSliceHexUpper(&key_pair.secret_key)}), "8052030376D47112BE7F73ED7A019293DD12AD910B654455798B4667D73DE1662D6F7455D97B4A3A10D7293909D1A4F2058CB9A370E43FA8154BB280DB839083"); + std.testing.expectEqualStrings(try std.fmt.bufPrint(&buf, "{s}", .{std.fmt.fmtSliceHexUpper(&key_pair.public_key)}), "2D6F7455D97B4A3A10D7293909D1A4F2058CB9A370E43FA8154BB280DB839083"); } test "ed25519 signature" { @@ -221,7 +221,7 @@ test "ed25519 signature" { const sig = try Ed25519.sign("test", key_pair, null); var buf: [128]u8 = undefined; - std.testing.expectEqualStrings(try std.fmt.bufPrint(&buf, "{X}", .{sig}), "10A442B4A80CC4225B154F43BEF28D2472CA80221951262EB8E0DF9091575E2687CC486E77263C3418C757522D54F84B0359236ABBBD4ACD20DC297FDCA66808"); + std.testing.expectEqualStrings(try std.fmt.bufPrint(&buf, "{s}", .{std.fmt.fmtSliceHexUpper(&sig)}), "10A442B4A80CC4225B154F43BEF28D2472CA80221951262EB8E0DF9091575E2687CC486E77263C3418C757522D54F84B0359236ABBBD4ACD20DC297FDCA66808"); try Ed25519.verify(sig, "test", key_pair.public_key); std.testing.expectError(error.InvalidSignature, Ed25519.verify(sig, "TEST", key_pair.public_key)); } diff --git a/lib/std/crypto/25519/edwards25519.zig b/lib/std/crypto/25519/edwards25519.zig index d4238f87bbf4d1c24d80b2c669e87b0d6293ee81..8d9922d80c1430b84885de9762adf05bb470388e 100644 --- a/lib/std/crypto/25519/edwards25519.zig +++ b/lib/std/crypto/25519/edwards25519.zig @@ -450,7 +450,7 @@ test "edwards25519 packing/unpacking" { var b = Edwards25519.basePoint; const pk = try b.mul(s); var buf: [128]u8 = undefined; - std.testing.expectEqualStrings(try std.fmt.bufPrint(&buf, "{X}", .{pk.toBytes()}), "074BC7E0FCBD587FDBC0969444245FADC562809C8F6E97E949AF62484B5B81A6"); + std.testing.expectEqualStrings(try std.fmt.bufPrint(&buf, "{s}", .{std.fmt.fmtSliceHexUpper(&pk.toBytes())}), "074BC7E0FCBD587FDBC0969444245FADC562809C8F6E97E949AF62484B5B81A6"); const small_order_ss: [7][32]u8 = .{ .{ diff --git a/lib/std/crypto/25519/ristretto255.zig b/lib/std/crypto/25519/ristretto255.zig index df85422f65ff6743729b4dd54fd5cee2467aa2db..46bb9697e21d45bdb462bccc8e54727c8e4259a4 100644 --- a/lib/std/crypto/25519/ristretto255.zig +++ b/lib/std/crypto/25519/ristretto255.zig @@ -170,21 +170,21 @@ pub const Ristretto255 = struct { test "ristretto255" { const p = Ristretto255.basePoint; var buf: [256]u8 = undefined; - std.testing.expectEqualStrings(try std.fmt.bufPrint(&buf, "{X}", .{p.toBytes()}), "E2F2AE0A6ABC4E71A884A961C500515F58E30B6AA582DD8DB6A65945E08D2D76"); + std.testing.expectEqualStrings(try std.fmt.bufPrint(&buf, "{s}", .{std.fmt.fmtSliceHexUpper(&p.toBytes())}), "E2F2AE0A6ABC4E71A884A961C500515F58E30B6AA582DD8DB6A65945E08D2D76"); var r: [Ristretto255.encoded_length]u8 = undefined; _ = try fmt.hexToBytes(r[0..], "6a493210f7499cd17fecb510ae0cea23a110e8d5b901f8acadd3095c73a3b919"); var q = try Ristretto255.fromBytes(r); q = q.dbl().add(p); - std.testing.expectEqualStrings(try std.fmt.bufPrint(&buf, "{X}", .{q.toBytes()}), "E882B131016B52C1D3337080187CF768423EFCCBB517BB495AB812C4160FF44E"); + std.testing.expectEqualStrings(try std.fmt.bufPrint(&buf, "{s}", .{std.fmt.fmtSliceHexUpper(&q.toBytes())}), "E882B131016B52C1D3337080187CF768423EFCCBB517BB495AB812C4160FF44E"); const s = [_]u8{15} ++ [_]u8{0} ** 31; const w = try p.mul(s); - std.testing.expectEqualStrings(try std.fmt.bufPrint(&buf, "{X}", .{w.toBytes()}), "E0C418F7C8D9C4CDD7395B93EA124F3AD99021BB681DFC3302A9D99A2E53E64E"); + std.testing.expectEqualStrings(try std.fmt.bufPrint(&buf, "{s}", .{std.fmt.fmtSliceHexUpper(&w.toBytes())}), "E0C418F7C8D9C4CDD7395B93EA124F3AD99021BB681DFC3302A9D99A2E53E64E"); std.testing.expect(p.dbl().dbl().dbl().dbl().equivalent(w.add(p))); const h = [_]u8{69} ** 32 ++ [_]u8{42} ** 32; const ph = Ristretto255.fromUniform(h); - std.testing.expectEqualStrings(try std.fmt.bufPrint(&buf, "{X}", .{ph.toBytes()}), "DCCA54E037A4311EFBEEF413ACD21D35276518970B7A61DC88F8587B493D5E19"); + std.testing.expectEqualStrings(try std.fmt.bufPrint(&buf, "{s}", .{std.fmt.fmtSliceHexUpper(&ph.toBytes())}), "DCCA54E037A4311EFBEEF413ACD21D35276518970B7A61DC88F8587B493D5E19"); } diff --git a/lib/std/crypto/25519/scalar.zig b/lib/std/crypto/25519/scalar.zig index ceff153bff9b7da77a3a72b2bf19217867711e59..e4fb2778075f6f18b3d09b0310737daad9ec9f6f 100644 --- a/lib/std/crypto/25519/scalar.zig +++ b/lib/std/crypto/25519/scalar.zig @@ -771,10 +771,10 @@ test "scalar25519" { var y = x.toBytes(); try rejectNonCanonical(y); var buf: [128]u8 = undefined; - std.testing.expectEqualStrings(try std.fmt.bufPrint(&buf, "{X}", .{y}), "1E979B917937F3DE71D18077F961F6CEFF01030405060708010203040506070F"); + std.testing.expectEqualStrings(try std.fmt.bufPrint(&buf, "{s}", .{std.fmt.fmtSliceHexUpper(&y)}), "1E979B917937F3DE71D18077F961F6CEFF01030405060708010203040506070F"); const reduced = reduce(field_size); - std.testing.expectEqualStrings(try std.fmt.bufPrint(&buf, "{X}", .{reduced}), "0000000000000000000000000000000000000000000000000000000000000000"); + std.testing.expectEqualStrings(try std.fmt.bufPrint(&buf, "{s}", .{std.fmt.fmtSliceHexUpper(&reduced)}), "0000000000000000000000000000000000000000000000000000000000000000"); } test "non-canonical scalar25519" { @@ -788,5 +788,5 @@ test "mulAdd overflow check" { const c: [32]u8 = [_]u8{0xff} ** 32; const x = mulAdd(a, b, c); var buf: [128]u8 = undefined; - std.testing.expectEqualStrings(try std.fmt.bufPrint(&buf, "{X}", .{x}), "D14DF91389432C25AD60FF9791B9FD1D67BEF517D273ECCE3D9A307C1B419903"); + std.testing.expectEqualStrings(try std.fmt.bufPrint(&buf, "{s}", .{std.fmt.fmtSliceHexUpper(&x)}), "D14DF91389432C25AD60FF9791B9FD1D67BEF517D273ECCE3D9A307C1B419903"); } diff --git a/lib/std/crypto/chacha20.zig b/lib/std/crypto/chacha20.zig index 0f797072793a34f221889080833beb0865133039..e01888e793d78b1730d17e716571386e98c69749 100644 --- a/lib/std/crypto/chacha20.zig +++ b/lib/std/crypto/chacha20.zig @@ -876,7 +876,7 @@ test "crypto.xchacha20" { var ciphertext: [input.len]u8 = undefined; XChaCha20IETF.xor(ciphertext[0..], input[0..], 0, key, nonce); var buf: [2 * ciphertext.len]u8 = undefined; - testing.expectEqualStrings(try std.fmt.bufPrint(&buf, "{X}", .{ciphertext}), "E0A1BCF939654AFDBDC1746EC49832647C19D891F0D1A81FC0C1703B4514BDEA584B512F6908C2C5E9DD18D5CBC1805DE5803FE3B9CA5F193FB8359E91FAB0C3BB40309A292EB1CF49685C65C4A3ADF4F11DB0CD2B6B67FBC174BC2E860E8F769FD3565BBFAD1C845E05A0FED9BE167C240D"); + testing.expectEqualStrings(try std.fmt.bufPrint(&buf, "{s}", .{std.fmt.fmtSliceHexUpper(&ciphertext)}), "E0A1BCF939654AFDBDC1746EC49832647C19D891F0D1A81FC0C1703B4514BDEA584B512F6908C2C5E9DD18D5CBC1805DE5803FE3B9CA5F193FB8359E91FAB0C3BB40309A292EB1CF49685C65C4A3ADF4F11DB0CD2B6B67FBC174BC2E860E8F769FD3565BBFAD1C845E05A0FED9BE167C240D"); } { const data = "Additional data"; @@ -885,7 +885,7 @@ test "crypto.xchacha20" { var out: [input.len]u8 = undefined; try xchacha20poly1305Open(out[0..], ciphertext[0..], data, key, nonce); var buf: [2 * ciphertext.len]u8 = undefined; - testing.expectEqualStrings(try std.fmt.bufPrint(&buf, "{X}", .{ciphertext}), "994D2DD32333F48E53650C02C7A2ABB8E018B0836D7175AEC779F52E961780768F815C58F1AA52D211498DB89B9216763F569C9433A6BBFCEFB4D4A49387A4C5207FBB3B5A92B5941294DF30588C6740D39DC16FA1F0E634F7246CF7CDCB978E44347D89381B7A74EB7084F754B90BDE9AAF5A94B8F2A85EFD0B50692AE2D425E234"); + testing.expectEqualStrings(try std.fmt.bufPrint(&buf, "{s}", .{std.fmt.fmtSliceHexUpper(&ciphertext)}), "994D2DD32333F48E53650C02C7A2ABB8E018B0836D7175AEC779F52E961780768F815C58F1AA52D211498DB89B9216763F569C9433A6BBFCEFB4D4A49387A4C5207FBB3B5A92B5941294DF30588C6740D39DC16FA1F0E634F7246CF7CDCB978E44347D89381B7A74EB7084F754B90BDE9AAF5A94B8F2A85EFD0B50692AE2D425E234"); testing.expectEqualSlices(u8, out[0..], input); ciphertext[0] += 1; testing.expectError(error.AuthenticationFailed, xchacha20poly1305Open(out[0..], ciphertext[0..], data, key, nonce)); diff --git a/lib/std/fmt.zig b/lib/std/fmt.zig index fca21000cfb5564e773178550fe25fe1a9f8832a..1f924bf00cb8b72ac949cf32819461a9b2e5cf4f 100644 --- a/lib/std/fmt.zig +++ b/lib/std/fmt.zig @@ -709,6 +709,87 @@ fn formatFloatValue( return formatBuf(buf_stream.getWritten(), options, writer); } +fn formatSliceHexImpl(comptime uppercase: bool) type { + const charset = "0123456789" ++ if (uppercase) "ABCDEF" else "abcdef"; + + return struct { + pub fn f( + bytes: []const u8, + comptime fmt: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) !void { + var buf: [2]u8 = undefined; + + for (bytes) |c| { + buf[0] = charset[c >> 4]; + buf[1] = charset[c & 15]; + try writer.writeAll(&buf); + } + } + }; +} + +const formatSliceHexLower = formatSliceHexImpl(false).f; +const formatSliceHexUpper = formatSliceHexImpl(true).f; + +/// Return a Formatter for a []const u8 where every byte is formatted as a pair +/// of lowercase hexadecimal digits. +pub fn fmtSliceHexLower(bytes: []const u8) std.fmt.Formatter(formatSliceHexLower) { + return .{ .data = bytes }; +} + +/// Return a Formatter for a []const u8 where every byte is formatted as a pair +/// of uppercase hexadecimal digits. +pub fn fmtSliceHexUpper(bytes: []const u8) std.fmt.Formatter(formatSliceHexUpper) { + return .{ .data = bytes }; +} + +fn formatSliceEscapeImpl(comptime uppercase: bool) type { + const charset = "0123456789" ++ if (uppercase) "ABCDEF" else "abcdef"; + + return struct { + pub fn f( + bytes: []const u8, + comptime fmt: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) !void { + var buf: [4]u8 = undefined; + + buf[0] = '\\'; + buf[1] = 'x'; + + for (bytes) |c| { + if (std.ascii.isPrint(c)) { + try writer.writeByte(c); + } else { + buf[2] = charset[c >> 4]; + buf[3] = charset[c & 15]; + try writer.writeAll(&buf); + } + } + } + }; +} + +const formatSliceEscapeLower = formatSliceEscapeImpl(false).f; +const formatSliceEscapeUpper = formatSliceEscapeImpl(true).f; + +/// Return a Formatter for a []const u8 where every non-printable ASCII +/// character is escaped as \xNN, where NN is the character in lowercase +/// hexadecimal notation. +pub fn fmtSliceEscapeLower(bytes: []const u8) std.fmt.Formatter(formatSliceEscapeLower) { + return .{ .data = bytes }; +} + +/// Return a Formatter for a []const u8 where every non-printable ASCII +/// character is escaped as \xNN, where NN is the character in uppercase +/// hexadecimal notation. +pub fn fmtSliceEscapeUpper(bytes: []const u8) std.fmt.Formatter(formatSliceEscapeUpper) { + return .{ .data = bytes }; +} + pub fn formatText( bytes: []const u8, comptime fmt: []const u8, @@ -717,21 +798,18 @@ pub fn formatText( ) !void { if (comptime std.mem.eql(u8, fmt, "s")) { return formatBuf(bytes, options, writer); - } else if (comptime (std.mem.eql(u8, fmt, "x") or std.mem.eql(u8, fmt, "X"))) { - for (bytes) |c| { - try formatInt(c, 16, fmt[0] == 'X', FormatOptions{ .width = 2, .fill = '0' }, writer); - } - return; - } else if (comptime (std.mem.eql(u8, fmt, "e") or std.mem.eql(u8, fmt, "E"))) { - for (bytes) |c| { - if (std.ascii.isPrint(c)) { - try writer.writeByte(c); - } else { - try writer.writeAll("\\x"); - try formatInt(c, 16, fmt[0] == 'E', FormatOptions{ .width = 2, .fill = '0' }, writer); - } - } - return; + } else if (comptime (std.mem.eql(u8, fmt, "x"))) { + @compileError("specifier 'x' has been deprecated, wrap your argument in std.fmt.fmtSliceHexLower instead"); + } else if (comptime (std.mem.eql(u8, fmt, "X"))) { + @compileError("specifier 'X' has been deprecated, wrap your argument in std.fmt.fmtSliceHexUpper instead"); + } else if (comptime (std.mem.eql(u8, fmt, "e"))) { + @compileError("specifier 'e' has been deprecated, wrap your argument in std.fmt.fmtSliceEscapeLower instead"); + } else if (comptime (std.mem.eql(u8, fmt, "E"))) { + @compileError("specifier 'X' has been deprecated, wrap your argument in std.fmt.fmtSliceEscapeUpper instead"); + } else if (comptime std.mem.eql(u8, fmt, "z")) { + @compileError("specifier 'z' has been deprecated, wrap your argument in std.zig.fmtId instead"); + } else if (comptime std.mem.eql(u8, fmt, "Z")) { + @compileError("specifier 'Z' has been deprecated, wrap your argument in std.zig.fmtEscapes instead"); } else { @compileError("Unsupported format string '" ++ fmt ++ "' for type '" ++ @typeName(@TypeOf(value)) ++ "'"); } @@ -1693,9 +1771,9 @@ test "slice" { } test "escape non-printable" { - try expectFmt("abc", "{e}", .{"abc"}); - try expectFmt("ab\\xffc", "{e}", .{"ab\xffc"}); - try expectFmt("ab\\xFFc", "{E}", .{"ab\xffc"}); + try expectFmt("abc", "{s}", .{fmtSliceEscapeLower("abc")}); + try expectFmt("ab\\xffc", "{s}", .{fmtSliceEscapeLower("ab\xffc")}); + try expectFmt("ab\\xFFc", "{s}", .{fmtSliceEscapeUpper("ab\xffc")}); } test "pointer" { @@ -1968,13 +2046,13 @@ test "struct.zero-size" { test "bytes.hex" { const some_bytes = "\xCA\xFE\xBA\xBE"; - try expectFmt("lowercase: cafebabe\n", "lowercase: {x}\n", .{some_bytes}); - try expectFmt("uppercase: CAFEBABE\n", "uppercase: {X}\n", .{some_bytes}); + try expectFmt("lowercase: cafebabe\n", "lowercase: {x}\n", .{fmtSliceHexLower(some_bytes)}); + try expectFmt("uppercase: CAFEBABE\n", "uppercase: {X}\n", .{fmtSliceHexUpper(some_bytes)}); //Test Slices - try expectFmt("uppercase: CAFE\n", "uppercase: {X}\n", .{some_bytes[0..2]}); - try expectFmt("lowercase: babe\n", "lowercase: {x}\n", .{some_bytes[2..]}); + try expectFmt("uppercase: CAFE\n", "uppercase: {X}\n", .{fmtSliceHexUpper(some_bytes[0..2])}); + try expectFmt("lowercase: babe\n", "lowercase: {x}\n", .{fmtSliceHexLower(some_bytes[2..])}); const bytes_with_zeros = "\x00\x0E\xBA\xBE"; - try expectFmt("lowercase: 000ebabe\n", "lowercase: {x}\n", .{bytes_with_zeros}); + try expectFmt("lowercase: 000ebabe\n", "lowercase: {x}\n", .{fmtSliceHexLower(bytes_with_zeros)}); } pub const trim = @compileError("deprecated; use std.mem.trim with std.ascii.spaces instead"); @@ -2002,9 +2080,9 @@ pub fn hexToBytes(out: []u8, input: []const u8) ![]u8 { test "hexToBytes" { var buf: [32]u8 = undefined; - try expectFmt("90" ** 32, "{X}", .{try hexToBytes(&buf, "90" ** 32)}); - try expectFmt("ABCD", "{X}", .{try hexToBytes(&buf, "ABCD")}); - try expectFmt("", "{X}", .{try hexToBytes(&buf, "")}); + try expectFmt("90" ** 32, "{s}", .{fmtSliceHexUpper(try hexToBytes(&buf, "90" ** 32))}); + try expectFmt("ABCD", "{s}", .{fmtSliceHexUpper(try hexToBytes(&buf, "ABCD"))}); + try expectFmt("", "{s}", .{fmtSliceHexUpper(try hexToBytes(&buf, ""))}); std.testing.expectError(error.InvalidCharacter, hexToBytes(&buf, "012Z")); std.testing.expectError(error.InvalidLength, hexToBytes(&buf, "AAA")); std.testing.expectError(error.NoSpaceLeft, hexToBytes(buf[0..1], "ABAB")); diff --git a/src/Cache.zig b/src/Cache.zig index 57ff9227faa06cb3c097c39aaa805125b5f343d0..f2fdafff9b36445155ec7205515770738b12da6d 100644 --- a/src/Cache.zig +++ b/src/Cache.zig @@ -153,7 +153,11 @@ pub const HashHelper = struct { hh.hasher.final(&bin_digest); var out_digest: [hex_digest_len]u8 = undefined; - _ = std.fmt.bufPrint(&out_digest, "{x}", .{bin_digest}) catch unreachable; + _ = std.fmt.bufPrint( + &out_digest, + "{s}", + .{std.fmt.fmtSliceHexLower(&bin_digest)}, + ) catch unreachable; return out_digest; } }; @@ -250,7 +254,11 @@ pub const Manifest = struct { var bin_digest: BinDigest = undefined; self.hash.hasher.final(&bin_digest); - _ = std.fmt.bufPrint(&self.hex_digest, "{x}", .{bin_digest}) catch unreachable; + _ = std.fmt.bufPrint( + &self.hex_digest, + "{s}", + .{std.fmt.fmtSliceHexLower(&bin_digest)}, + ) catch unreachable; self.hash.hasher = hasher_init; self.hash.hasher.update(&bin_digest); @@ -549,7 +557,11 @@ pub const Manifest = struct { self.hash.hasher.final(&bin_digest); var out_digest: [hex_digest_len]u8 = undefined; - _ = std.fmt.bufPrint(&out_digest, "{x}", .{bin_digest}) catch unreachable; + _ = std.fmt.bufPrint( + &out_digest, + "{s}", + .{std.fmt.fmtSliceHexLower(&bin_digest)}, + ) catch unreachable; return out_digest; } @@ -565,7 +577,11 @@ pub const Manifest = struct { var encoded_digest: [hex_digest_len]u8 = undefined; for (self.files.items) |file| { - _ = std.fmt.bufPrint(&encoded_digest, "{x}", .{file.bin_digest}) catch unreachable; + _ = std.fmt.bufPrint( + &encoded_digest, + "{s}", + .{std.fmt.fmtSliceHexLower(&file.bin_digest)}, + ) catch unreachable; try writer.print("{d} {d} {d} {s} {s}\n", .{ file.stat.size, file.stat.inode, diff --git a/src/translate_c.zig b/src/translate_c.zig index 6dac4bf0efc1e5cdfd0092039a6abb5b3f9398dc..ed80a13058aa2413220d4ff767d327ebc33a063d 100644 --- a/src/translate_c.zig +++ b/src/translate_c.zig @@ -4608,7 +4608,7 @@ fn parseCPrimaryExprInner(c: *Context, m: *MacroCtx, scope: *Scope) ParseError!N if (slice[0] != '\'' or slice[1] == '\\' or slice.len == 3) { return Tag.char_literal.create(c.arena, try zigifyEscapeSequences(c, m)); } else { - const str = try std.fmt.allocPrint(c.arena, "0x{x}", .{slice[1 .. slice.len - 1]}); + const str = try std.fmt.allocPrint(c.arena, "0x{s}", .{std.fmt.fmtSliceHexLower(slice[1 .. slice.len - 1])}); return Tag.integer_literal.create(c.arena, str); } }, -- 2.54.0 From 278bd60732df558cd43b67166cde8e0410e45428 Mon Sep 17 00:00:00 2001 From: joachimschmidt557 Date: Tue, 2 Mar 2021 00:15:03 +0100 Subject: [PATCH 14/35] stage2 ARM: Add tests for basic integer multiplication --- test/stage2/arm.zig | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/test/stage2/arm.zig b/test/stage2/arm.zig index 06e359ccffce3ddd052452019e9ff4f9ddf3a680..1bc3f23058aa90fb140d45d4f1be5a002c84a420 100644 --- a/test/stage2/arm.zig +++ b/test/stage2/arm.zig @@ -344,4 +344,38 @@ pub fn addCases(ctx: *TestContext) !void { "", ); } + + { + var case = ctx.exe("integer multiplication", linux_arm); + // Simple u32 integer multiplication + case.addCompareOutput( + \\export fn _start() noreturn { + \\ assert(mul(1, 1) == 1); + \\ assert(mul(42, 1) == 42); + \\ assert(mul(1, 42) == 42); + \\ assert(mul(123, 42) == 5166); + \\ exit(); + \\} + \\ + \\fn mul(x: u32, y: u32) u32 { + \\ return x * y; + \\} + \\ + \\fn assert(ok: bool) void { + \\ if (!ok) unreachable; + \\} + \\ + \\fn exit() noreturn { + \\ asm volatile ("svc #0" + \\ : + \\ : [number] "{r7}" (1), + \\ [arg1] "{r0}" (0) + \\ : "memory" + \\ ); + \\ unreachable; + \\} + , + "", + ); + } } -- 2.54.0 From 7b5b7bda87dc4121e2942042d19fd375c73f90e5 Mon Sep 17 00:00:00 2001 From: Isaac Freund Date: Mon, 1 Mar 2021 21:07:07 +0100 Subject: [PATCH 15/35] parser: fix infinite loop on missing comma in param list --- lib/std/zig/parse.zig | 32 ++++++++++++-------------------- lib/std/zig/parser_test.zig | 12 ++++++++++++ 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/lib/std/zig/parse.zig b/lib/std/zig/parse.zig index 7a6404fbb2ca45110da7f78d10183020d00da1b7..9b755c203375d9197df22a093d6957f21ff122de 100644 --- a/lib/std/zig/parse.zig +++ b/lib/std/zig/parse.zig @@ -3714,7 +3714,6 @@ const Parser = struct { if (p.eatToken(.r_paren)) |_| { return SmallSpan{ .zero_or_one = 0 }; } - continue; }, .r_paren => return SmallSpan{ .zero_or_one = 0 }, else => { @@ -3728,14 +3727,7 @@ const Parser = struct { const param_two = while (true) { switch (p.token_tags[p.nextToken()]) { - .comma => { - if (p.eatToken(.r_paren)) |_| { - return SmallSpan{ .zero_or_one = param_one }; - } - const param = try p.expectParamDecl(); - if (param != 0) break param; - continue; - }, + .comma => {}, .r_paren => return SmallSpan{ .zero_or_one = param_one }, .colon, .r_brace, .r_bracket => { p.tok_i -= 1; @@ -3748,6 +3740,11 @@ const Parser = struct { try p.warnExpected(.comma); }, } + if (p.eatToken(.r_paren)) |_| { + return SmallSpan{ .zero_or_one = param_one }; + } + const param = try p.expectParamDecl(); + if (param != 0) break param; } else unreachable; var list = std.ArrayList(Node.Index).init(p.gpa); @@ -3757,17 +3754,7 @@ const Parser = struct { while (true) { switch (p.token_tags[p.nextToken()]) { - .comma => { - if (p.token_tags[p.tok_i] == .r_paren) { - p.tok_i += 1; - return SmallSpan{ .multi = list.toOwnedSlice() }; - } - const param = try p.expectParamDecl(); - if (param != 0) { - try list.append(param); - } - continue; - }, + .comma => {}, .r_paren => return SmallSpan{ .multi = list.toOwnedSlice() }, .colon, .r_brace, .r_bracket => { p.tok_i -= 1; @@ -3780,6 +3767,11 @@ const Parser = struct { try p.warnExpected(.comma); }, } + if (p.eatToken(.r_paren)) |_| { + return SmallSpan{ .multi = list.toOwnedSlice() }; + } + const param = try p.expectParamDecl(); + if (param != 0) try list.append(param); } } diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index 2b9e3fb03cb3842ddda45625c6fb14335cfaaec3..375a041a0ab169f005dfe77e0176ea274c295a9f 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -4549,6 +4549,18 @@ test "recovery: missing for payload" { }); } +test "recovery: missing comma in params" { + try testError( + \\fn foo(comptime bool what what) void { } + \\fn bar(a: i32, b: i32 c) void { } + \\ + , &[_]Error{ + .expected_token, + .expected_token, + .expected_token, + }); +} + const std = @import("std"); const mem = std.mem; const warn = std.debug.warn; -- 2.54.0 From 1f861ecc95a827fc979f3371901f1ae93dd2c283 Mon Sep 17 00:00:00 2001 From: daurnimator Date: Sat, 2 Jan 2021 02:46:02 +1100 Subject: [PATCH 16/35] Bring back ZIG_SKIP_INSTALL_LIB_FILES --- CMakeLists.txt | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6e89d87ca9bbb76ec1c4e6204425775fd34b11ea..1c0218e305a797cc0f71d30e2c3d9dc31cebff7a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -801,31 +801,32 @@ endif() install(TARGETS zig DESTINATION bin) -set(ZIG_INSTALL_ARGS "build" - --override-lib-dir "${CMAKE_SOURCE_DIR}/lib" - "-Dlib-files-only" - --prefix "${CMAKE_INSTALL_PREFIX}" - "-Dconfig_h=${ZIG_CONFIG_H_OUT}" - install -) +set(ZIG_SKIP_INSTALL_LIB_FILES off CACHE BOOL + "Disable copying lib/ files to install prefix during the build phase") -# CODE has no effect with Visual Studio build system generator, therefore -# when using Visual Studio build system generator we resort to running -# `zig build install` during the build phase. -if(MSVC) - set(ZIG_SKIP_INSTALL_LIB_FILES off CACHE BOOL - "Windows-only: Disable copying lib/ files to install prefix during the build phase") - if(NOT ZIG_SKIP_INSTALL_LIB_FILES) +if(NOT ZIG_SKIP_INSTALL_LIB_FILES) + set(ZIG_INSTALL_ARGS "build" + --override-lib-dir "${CMAKE_SOURCE_DIR}/lib" + "-Dlib-files-only" + --prefix "${CMAKE_INSTALL_PREFIX}" + "-Dconfig_h=${ZIG_CONFIG_H_OUT}" + install + ) + + # CODE has no effect with Visual Studio build system generator, therefore + # when using Visual Studio build system generator we resort to running + # `zig build install` during the build phase. + if(MSVC) add_custom_target(zig_install_lib_files ALL COMMAND zig ${ZIG_INSTALL_ARGS} DEPENDS zig WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" ) + else() + get_target_property(zig_BINARY_DIR zig BINARY_DIR) + install(CODE "set(zig_EXE \"${ZIG_EXECUTABLE}\")") + install(CODE "set(ZIG_INSTALL_ARGS \"${ZIG_INSTALL_ARGS}\")") + install(CODE "set(CMAKE_SOURCE_DIR \"${CMAKE_SOURCE_DIR}\")") + install(SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/cmake/install.cmake) endif() -else() - get_target_property(zig_BINARY_DIR zig BINARY_DIR) - install(CODE "set(zig_EXE \"${ZIG_EXECUTABLE}\")") - install(CODE "set(ZIG_INSTALL_ARGS \"${ZIG_INSTALL_ARGS}\")") - install(CODE "set(CMAKE_SOURCE_DIR \"${CMAKE_SOURCE_DIR}\")") - install(SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/cmake/install.cmake) endif() -- 2.54.0 From 7613e51a57d2e2d0ae7d1101d059002f12b96c43 Mon Sep 17 00:00:00 2001 From: Martin Wickham Date: Thu, 4 Feb 2021 23:04:49 -0600 Subject: [PATCH 17/35] Add some bit set variants --- lib/std/bit_set.zig | 1254 +++++++++++++++++++++++++++++++++++++++++++ lib/std/math.zig | 56 ++ lib/std/mem.zig | 8 + lib/std/std.zig | 4 + 4 files changed, 1322 insertions(+) create mode 100644 lib/std/bit_set.zig diff --git a/lib/std/bit_set.zig b/lib/std/bit_set.zig new file mode 100644 index 0000000000000000000000000000000000000000..c3737ee20fb32f6dab70afaa0d52736226eedce9 --- /dev/null +++ b/lib/std/bit_set.zig @@ -0,0 +1,1254 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2021 Zig Contributors +// This file is part of [zig](https://ziglang.org/), which is MIT licensed. +// The MIT license requires this copyright notice to be included in all copies +// and substantial portions of the software. + +const std = @import("std"); +const assert = std.debug.assert; +const Allocator = std.mem.Allocator; + +//! This file defines several variants of bit sets. A bit set +//! is a densely stored set of integers with a known maximum, +//! in which each integer gets a single bit. Bit sets have very +//! fast presence checks, update operations, and union and intersection +//! operations. However, if the number of possible items is very +//! large and the number of actual items in a given set is usually +//! small, they may be less memory efficient than an array set. +//! +//! There are five variants defined here: +//! +//! IntegerBitSet: +//! A bit set with static size, which is backed by a single integer. +//! This set is good for sets with a small size, but may generate +//! inefficient code for larger sets, especially in debug mode. +//! +//! ArrayBitSet: +//! A bit set with static size, which is backed by an array of usize. +//! This set is good for sets with a larger size, but may use +//! more bytes than necessary if your set is small. +//! +//! StaticBitSet: +//! Picks either IntegerBitSet or ArrayBitSet depending on the requested +//! size. The interfaces of these two types match exactly, except for fields. +//! +//! DynamicBitSet: +//! A bit set with runtime known size, backed by an allocated slice +//! of usize. +//! +//! DynamicBitSetUnmanaged: +//! A variant of DynamicBitSet which does not store a pointer to its +//! allocator, in order to save space. + +/// Returns the optimal static bit set type for the specified number +/// of elements. The returned type will perform no allocations, +/// can be copied by value, and does not require deinitialization. +/// Both possible implementations fulfill the same interface. +pub fn StaticBitSet(comptime size: usize) type { + if (size <= @bitSizeOf(usize)) { + return IntegerBitSet(size); + } else { + return ArrayBitSet(usize, size); + } +} + +/// A bit set with static size, which is backed by a single integer. +/// This set is good for sets with a small size, but may generate +/// inefficient code for larger sets, especially in debug mode. +pub fn IntegerBitSet(comptime size: u16) type { + return struct { + const Self = @This(); + + // TODO: Make this a comptime field once those are fixed + /// The number of items in this bit set + pub const bit_length: usize = size; + + /// The integer type used to represent a mask in this bit set + pub const MaskInt = std.meta.Int(.unsigned, size); + + /// The integer type used to shift a mask in this bit set + pub const ShiftInt = std.math.Log2Int(MaskInt); + + /// The bit mask, as a single integer + mask: MaskInt, + + /// Creates a bit set with no elements present. + pub fn initEmpty() Self { + return .{ .mask = 0 }; + } + + /// Creates a bit set with all elements present. + pub fn initFull() Self { + return .{ .mask = ~@as(MaskInt, 0) }; + } + + /// Returns the number of bits in this bit set + pub inline fn capacity(self: Self) usize { + return bit_length; + } + + /// Returns true if the bit at the specified index + /// is present in the set, false otherwise. + pub fn isSet(self: Self, index: usize) bool { + assert(index < bit_length); + return (self.mask & maskBit(index)) != 0; + } + + /// Returns the total number of set bits in this bit set. + pub fn count(self: Self) usize { + return @popCount(MaskInt, self.mask); + } + + /// Changes the value of the specified bit of the bit + /// set to match the passed boolean. + pub fn setValue(self: *Self, index: usize, value: bool) void { + assert(index < bit_length); + if (MaskInt == u0) return; + const bit = maskBit(index); + const new_bit = bit & std.math.boolMask(MaskInt, value); + self.mask = (self.mask & ~bit) | new_bit; + } + + /// Adds a specific bit to the bit set + pub fn set(self: *Self, index: usize) void { + assert(index < bit_length); + self.mask |= maskBit(index); + } + + /// Removes a specific bit from the bit set + pub fn unset(self: *Self, index: usize) void { + assert(index < bit_length); + // Workaround for #7953 + if (MaskInt == u0) return; + self.mask &= ~maskBit(index); + } + + /// Flips a specific bit in the bit set + pub fn toggle(self: *Self, index: usize) void { + assert(index < bit_length); + self.mask ^= maskBit(index); + } + + /// Flips all bits in this bit set which are present + /// in the toggles bit set. + pub fn toggleSet(self: *Self, toggles: Self) void { + self.mask ^= toggles.mask; + } + + /// Flips every bit in the bit set. + pub fn toggleAll(self: *Self) void { + self.mask = ~self.mask; + } + + /// Performs a union of two bit sets, and stores the + /// result in the first one. Bits in the result are + /// set if the corresponding bits were set in either input. + pub fn setUnion(self: *Self, other: Self) void { + self.mask |= other.mask; + } + + /// Performs an intersection of two bit sets, and stores + /// the result in the first one. Bits in the result are + /// set if the corresponding bits were set in both inputs. + pub fn setIntersection(self: *Self, other: Self) void { + self.mask &= other.mask; + } + + /// Finds the index of the first set bit. + /// If no bits are set, returns null. + pub fn findFirstSet(self: Self) ?usize { + const mask = self.mask; + if (mask == 0) return null; + return @ctz(MaskInt, mask); + } + + /// Finds the index of the first set bit, and unsets it. + /// If no bits are set, returns null. + pub fn toggleFirstSet(self: *Self) ?usize { + const mask = self.mask; + if (mask == 0) return null; + const index = @ctz(MaskInt, mask); + self.mask = mask & (mask-1); + return index; + } + + /// Iterates through the items in the set, according to the options. + /// The default options (.{}) will iterate indices of set bits in + /// ascending order. Modifications to the underlying bit set may + /// or may not be observed by the iterator. + pub fn iterator(self: *const Self, comptime options: IteratorOptions) Iterator(options.direction) { + return .{ + .bits_remain = switch (options.kind) { + .set => self.mask, + .unset => ~self.mask, + }, + }; + } + + fn Iterator(comptime direction: IteratorOptions.Direction) type { + return struct { + const IterSelf = @This(); + // all bits which have not yet been iterated over + bits_remain: MaskInt, + + /// Returns the index of the next unvisited set bit + /// in the bit set, in ascending order. + pub fn next(self: *IterSelf) ?usize { + if (self.bits_remain == 0) return null; + + switch (direction) { + .forward => { + const next_index = @ctz(MaskInt, self.bits_remain); + self.bits_remain &= self.bits_remain - 1; + return next_index; + }, + .reverse => { + const leading_zeroes = @clz(MaskInt, self.bits_remain); + const top_bit = (@bitSizeOf(MaskInt) - 1) - leading_zeroes; + self.bits_remain &= (@as(MaskInt, 1) << @intCast(ShiftInt, top_bit)) - 1; + return top_bit; + }, + } + } + }; + } + + fn maskBit(index: usize) MaskInt { + if (MaskInt == u0) return 0; + return @as(MaskInt, 1) << @intCast(ShiftInt, index); + } + fn boolMaskBit(index: usize, value: bool) MaskInt { + if (MaskInt == u0) return 0; + return @as(MaskInt, @boolToInt(value)) << @intCast(ShiftInt, index); + } + }; +} + +/// A bit set with static size, which is backed by an array of usize. +/// This set is good for sets with a larger size, but may use +/// more bytes than necessary if your set is small. +pub fn ArrayBitSet(comptime MaskIntType: type, comptime size: usize) type { + const mask_info: std.builtin.TypeInfo = @typeInfo(MaskIntType); + + // Make sure the mask int is indeed an int + if (mask_info != .Int) @compileError("ArrayBitSet can only operate on integer masks, but was passed " ++ @typeName(MaskIntType)); + + // It must also be unsigned. + if (mask_info.Int.signedness != .unsigned) @compileError("ArrayBitSet requires an unsigned integer mask type, but was passed " ++ @typeName(MaskIntType)); + + // And it must not be empty. + if (MaskIntType == u0) + @compileError("ArrayBitSet requires a sized integer for its mask int. u0 does not work."); + + const byte_size = std.mem.byte_size_in_bits; + + // We use shift and truncate to decompose indices into mask indices and bit indices. + // This operation requires that the mask has an exact power of two number of bits. + if (!std.math.isPowerOfTwo(@bitSizeOf(MaskIntType))) { + var desired_bits = std.math.ceilPowerOfTwoAssert(usize, @bitSizeOf(MaskIntType)); + if (desired_bits < byte_size) desired_bits = byte_size; + const FixedMaskType = std.meta.Int(.unsigned, desired_bits); + @compileError("ArrayBitSet was passed integer type " ++ @typeName(MaskIntType) ++ + ", which is not a power of two. Please round this up to a power of two integer size (i.e. " ++ @typeName(FixedMaskType) ++ ")."); + } + + // Make sure the integer has no padding bits. + // Those would be wasteful here and are probably a mistake by the user. + // This case may be hit with small powers of two, like u4. + if (@bitSizeOf(MaskIntType) != @sizeOf(MaskIntType) * byte_size) { + var desired_bits = @sizeOf(MaskIntType) * byte_size; + desired_bits = std.math.ceilPowerOfTwoAssert(usize, desired_bits); + const FixedMaskType = std.meta.Int(.unsigned, desired_bits); + @compileError("ArrayBitSet was passed integer type " ++ @typeName(MaskIntType) ++ + ", which contains padding bits. Please round this up to an unpadded integer size (i.e. " ++ @typeName(FixedMaskType) ++ ")."); + } + + return struct { + const Self = @This(); + + // TODO: Make this a comptime field once those are fixed + /// The number of items in this bit set + pub const bit_length: usize = size; + + /// The integer type used to represent a mask in this bit set + pub const MaskInt = MaskIntType; + + /// The integer type used to shift a mask in this bit set + pub const ShiftInt = std.math.Log2Int(MaskInt); + + // bits in one mask + const mask_len = @bitSizeOf(MaskInt); + // total number of masks + const num_masks = (size + mask_len - 1) / mask_len; + // padding bits in the last mask (may be 0) + const last_pad_bits = mask_len * num_masks - size; + // Mask of valid bits in the last mask. + // All functions will ensure that the invalid + // bits in the last mask are zero. + pub const last_item_mask = ~@as(MaskInt, 0) >> last_pad_bits; + + /// The bit masks, ordered with lower indices first. + /// Padding bits at the end are undefined. + masks: [num_masks]MaskInt, + + /// Creates a bit set with no elements present. + pub fn initEmpty() Self { + return .{ .masks = [_]MaskInt{0} ** num_masks }; + } + + /// Creates a bit set with all elements present. + pub fn initFull() Self { + if (num_masks == 0) { + return .{ .masks = .{} }; + } else { + return .{ .masks = [_]MaskInt{~@as(MaskInt, 0)} ** (num_masks - 1) ++ [_]MaskInt{last_item_mask} }; + } + } + + /// Returns the number of bits in this bit set + pub inline fn capacity(self: Self) usize { + return bit_length; + } + + /// Returns true if the bit at the specified index + /// is present in the set, false otherwise. + pub fn isSet(self: Self, index: usize) bool { + assert(index < bit_length); + if (num_masks == 0) return false; // doesn't compile in this case + return (self.masks[maskIndex(index)] & maskBit(index)) != 0; + } + + /// Returns the total number of set bits in this bit set. + pub fn count(self: Self) usize { + var total: usize = 0; + for (self.masks) |mask| { + total += @popCount(MaskInt, mask); + } + return total; + } + + /// Changes the value of the specified bit of the bit + /// set to match the passed boolean. + pub fn setValue(self: *Self, index: usize, value: bool) void { + assert(index < bit_length); + if (num_masks == 0) return; // doesn't compile in this case + const bit = maskBit(index); + const mask_index = maskIndex(index); + const new_bit = bit & std.math.boolMask(MaskInt, value); + self.masks[mask_index] = (self.masks[mask_index] & ~bit) | new_bit; + } + + /// Adds a specific bit to the bit set + pub fn set(self: *Self, index: usize) void { + assert(index < bit_length); + if (num_masks == 0) return; // doesn't compile in this case + self.masks[maskIndex(index)] |= maskBit(index); + } + + /// Removes a specific bit from the bit set + pub fn unset(self: *Self, index: usize) void { + assert(index < bit_length); + if (num_masks == 0) return; // doesn't compile in this case + self.masks[maskIndex(index)] &= ~maskBit(index); + } + + /// Flips a specific bit in the bit set + pub fn toggle(self: *Self, index: usize) void { + assert(index < bit_length); + if (num_masks == 0) return; // doesn't compile in this case + self.masks[maskIndex(index)] ^= maskBit(index); + } + + /// Flips all bits in this bit set which are present + /// in the toggles bit set. + pub fn toggleSet(self: *Self, toggles: Self) void { + for (self.masks) |*mask, i| { + mask.* ^= toggles.masks[i]; + } + } + + /// Flips every bit in the bit set. + pub fn toggleAll(self: *Self) void { + for (self.masks) |*mask, i| { + mask.* = ~mask.*; + } + + // Zero the padding bits + if (num_masks > 0) { + self.masks[num_masks - 1] &= last_item_mask; + } + } + + /// Performs a union of two bit sets, and stores the + /// result in the first one. Bits in the result are + /// set if the corresponding bits were set in either input. + pub fn setUnion(self: *Self, other: Self) void { + for (self.masks) |*mask, i| { + mask.* |= other.masks[i]; + } + } + + /// Performs an intersection of two bit sets, and stores + /// the result in the first one. Bits in the result are + /// set if the corresponding bits were set in both inputs. + pub fn setIntersection(self: *Self, other: Self) void { + for (self.masks) |*mask, i| { + mask.* &= other.masks[i]; + } + } + + /// Finds the index of the first set bit. + /// If no bits are set, returns null. + pub fn findFirstSet(self: Self) ?usize { + var offset: usize = 0; + const mask = for (self.masks) |mask| { + if (mask != 0) break mask; + offset += @bitSizeOf(MaskInt); + } else return null; + return offset + @ctz(MaskInt, mask); + } + + /// Finds the index of the first set bit, and unsets it. + /// If no bits are set, returns null. + pub fn toggleFirstSet(self: *Self) ?usize { + var offset: usize = 0; + const mask = for (self.masks) |*mask| { + if (mask.* != 0) break mask; + offset += @bitSizeOf(MaskInt); + } else return null; + const index = @ctz(MaskInt, mask.*); + mask.* &= (mask.* - 1); + return offset + index; + } + + /// Iterates through the items in the set, according to the options. + /// The default options (.{}) will iterate indices of set bits in + /// ascending order. Modifications to the underlying bit set may + /// or may not be observed by the iterator. + pub fn iterator(self: *const Self, comptime options: IteratorOptions) BitSetIterator(MaskInt, options) { + return BitSetIterator(MaskInt, options).init(&self.masks, last_item_mask); + } + + fn maskBit(index: usize) MaskInt { + return @as(MaskInt, 1) << @truncate(ShiftInt, index); + } + fn maskIndex(index: usize) usize { + return index >> @bitSizeOf(ShiftInt); + } + fn boolMaskBit(index: usize, value: bool) MaskInt { + return @as(MaskInt, @boolToInt(value)) << @intCast(ShiftInt, index); + } + }; +} + +/// A bit set with runtime known size, backed by an allocated slice +/// of usize. The allocator must be tracked externally by the user. +pub const DynamicBitSetUnmanaged = struct { + const Self = @This(); + + /// The integer type used to represent a mask in this bit set + pub const MaskInt = usize; + + /// The integer type used to shift a mask in this bit set + pub const ShiftInt = std.math.Log2Int(MaskInt); + + /// The number of valid items in this bit set + bit_length: usize = 0, + + /// The bit masks, ordered with lower indices first. + /// Padding bits at the end must be zeroed. + masks: [*]MaskInt = empty_masks_ptr, + // This pointer is one usize after the actual allocation. + // That slot holds the size of the true allocation, which + // is needed by Zig's allocator interface in case a shrink + // fails. + + // Don't modify this value. Ideally it would go in const data so + // modifications would cause a bus error, but the only way + // to discard a const qualifier is through ptrToInt, which + // cannot currently round trip at comptime. + var empty_masks_data = [_]MaskInt{ 0, undefined }; + const empty_masks_ptr = empty_masks_data[1..2]; + + /// Creates a bit set with no elements present. + /// If bit_length is not zero, deinit must eventually be called. + pub fn initEmpty(bit_length: usize, allocator: *Allocator) !Self { + var self = Self{}; + try self.resize(bit_length, false, allocator); + return self; + } + + /// Creates a bit set with all elements present. + /// If bit_length is not zero, deinit must eventually be called. + pub fn initFull(bit_length: usize, allocator: *Allocator) !Self { + var self = Self{}; + try self.resize(bit_length, true, allocator); + return self; + } + + /// Resizes to a new bit_length. If the new length is larger + /// than the old length, fills any added bits with `fill`. + /// If new_len is not zero, deinit must eventually be called. + pub fn resize(self: *@This(), new_len: usize, fill: bool, allocator: *Allocator) !void { + const old_len = self.bit_length; + + const old_masks = numMasks(old_len); + const new_masks = numMasks(new_len); + + const old_allocation = (self.masks - 1)[0..(self.masks - 1)[0]]; + + if (new_masks == 0) { + assert(new_len == 0); + allocator.free(old_allocation); + self.masks = empty_masks_ptr; + self.bit_length = 0; + return; + } + + if (old_allocation.len != new_masks + 1) realloc: { + // If realloc fails, it may mean one of two things. + // If we are growing, it means we are out of memory. + // If we are shrinking, it means the allocator doesn't + // want to move the allocation. This means we need to + // hold on to the extra 8 bytes required to be able to free + // this allocation properly. + const new_allocation = allocator.realloc(old_allocation, new_masks + 1) catch |err| { + if (new_masks + 1 > old_allocation.len) return err; + break :realloc; + }; + + new_allocation[0] = new_allocation.len; + self.masks = new_allocation.ptr + 1; + } + + // If we increased in size, we need to set any new bits + // to the fill value. + if (new_len > old_len) { + // set the padding bits in the old last item to 1 + if (fill and old_masks > 0) { + const old_padding_bits = old_masks * @bitSizeOf(MaskInt) - old_len; + const old_mask = (~@as(MaskInt, 0)) >> @intCast(ShiftInt, old_padding_bits); + self.masks[old_masks - 1] |= ~old_mask; + } + + // fill in any new masks + if (new_masks > old_masks) { + const fill_value = std.math.boolMask(MaskInt, fill); + std.mem.set(MaskInt, self.masks[old_masks..new_masks], fill_value); + } + } + + // Zero out the padding bits + if (new_len > 0) { + const padding_bits = new_masks * @bitSizeOf(MaskInt) - new_len; + const last_item_mask = (~@as(MaskInt, 0)) >> @intCast(ShiftInt, padding_bits); + self.masks[new_masks - 1] &= last_item_mask; + } + + // And finally, save the new length. + self.bit_length = new_len; + } + + /// deinitializes the array and releases its memory. + /// The passed allocator must be the same one used for + /// init* or resize in the past. + pub fn deinit(self: *Self, allocator: *Allocator) void { + self.resize(0, false, allocator) catch unreachable; + } + + /// Creates a duplicate of this bit set, using the new allocator. + pub fn clone(self: *const Self, new_allocator: *Allocator) !Self { + const num_masks = numMasks(self.bit_length); + var copy = Self{}; + try copy.resize(self.bit_length, false, new_allocator); + std.mem.copy(MaskInt, copy.masks[0..num_masks], self.masks[0..num_masks]); + return copy; + } + + /// Returns the number of bits in this bit set + pub inline fn capacity(self: Self) usize { + return self.bit_length; + } + + /// Returns true if the bit at the specified index + /// is present in the set, false otherwise. + pub fn isSet(self: Self, index: usize) bool { + assert(index < self.bit_length); + return (self.masks[maskIndex(index)] & maskBit(index)) != 0; + } + + /// Returns the total number of set bits in this bit set. + pub fn count(self: Self) usize { + const num_masks = (self.bit_length + (@bitSizeOf(MaskInt) - 1)) / @bitSizeOf(MaskInt); + var total: usize = 0; + for (self.masks[0..num_masks]) |mask| { + // Note: This is where we depend on padding bits being zero + total += @popCount(MaskInt, mask); + } + return total; + } + + /// Changes the value of the specified bit of the bit + /// set to match the passed boolean. + pub fn setValue(self: *Self, index: usize, value: bool) void { + assert(index < self.bit_length); + const bit = maskBit(index); + const mask_index = maskIndex(index); + const new_bit = bit & std.math.boolMask(MaskInt, value); + self.masks[mask_index] = (self.masks[mask_index] & ~bit) | new_bit; + } + + /// Adds a specific bit to the bit set + pub fn set(self: *Self, index: usize) void { + assert(index < self.bit_length); + self.masks[maskIndex(index)] |= maskBit(index); + } + + /// Removes a specific bit from the bit set + pub fn unset(self: *Self, index: usize) void { + assert(index < self.bit_length); + self.masks[maskIndex(index)] &= ~maskBit(index); + } + + /// Flips a specific bit in the bit set + pub fn toggle(self: *Self, index: usize) void { + assert(index < self.bit_length); + self.masks[maskIndex(index)] ^= maskBit(index); + } + + /// Flips all bits in this bit set which are present + /// in the toggles bit set. Both sets must have the + /// same bit_length. + pub fn toggleSet(self: *Self, toggles: Self) void { + assert(toggles.bit_length == self.bit_length); + const num_masks = numMasks(self.bit_length); + for (self.masks[0..num_masks]) |*mask, i| { + mask.* ^= toggles.masks[i]; + } + } + + /// Flips every bit in the bit set. + pub fn toggleAll(self: *Self) void { + const bit_length = self.bit_length; + // avoid underflow if bit_length is zero + if (bit_length == 0) return; + + const num_masks = numMasks(self.bit_length); + for (self.masks[0..num_masks]) |*mask, i| { + mask.* = ~mask.*; + } + + const padding_bits = num_masks * @bitSizeOf(MaskInt) - bit_length; + const last_item_mask = (~@as(MaskInt, 0)) >> @intCast(ShiftInt, padding_bits); + self.masks[num_masks - 1] &= last_item_mask; + } + + /// Performs a union of two bit sets, and stores the + /// result in the first one. Bits in the result are + /// set if the corresponding bits were set in either input. + /// The two sets must both be the same bit_length. + pub fn setUnion(self: *Self, other: Self) void { + assert(other.bit_length == self.bit_length); + const num_masks = numMasks(self.bit_length); + for (self.masks[0..num_masks]) |*mask, i| { + mask.* |= other.masks[i]; + } + } + + /// Performs an intersection of two bit sets, and stores + /// the result in the first one. Bits in the result are + /// set if the corresponding bits were set in both inputs. + /// The two sets must both be the same bit_length. + pub fn setIntersection(self: *Self, other: Self) void { + assert(other.bit_length == self.bit_length); + const num_masks = numMasks(self.bit_length); + for (self.masks[0..num_masks]) |*mask, i| { + mask.* &= other.masks[i]; + } + } + + /// Finds the index of the first set bit. + /// If no bits are set, returns null. + pub fn findFirstSet(self: Self) ?usize { + var offset: usize = 0; + var mask = self.masks; + while (offset < self.bit_length) { + if (mask[0] != 0) break; + mask += 1; + offset += @bitSizeOf(MaskInt); + } else return null; + return offset + @ctz(MaskInt, mask[0]); + } + + /// Finds the index of the first set bit, and unsets it. + /// If no bits are set, returns null. + pub fn toggleFirstSet(self: *Self) ?usize { + var offset: usize = 0; + var mask = self.masks; + while (offset < self.bit_length) { + if (mask[0] != 0) break; + mask += 1; + offset += @bitSizeOf(MaskInt); + } else return null; + const index = @ctz(MaskInt, mask[0]); + mask[0] &= (mask[0]-1); + return offset + index; + } + + /// Iterates through the items in the set, according to the options. + /// The default options (.{}) will iterate indices of set bits in + /// ascending order. Modifications to the underlying bit set may + /// or may not be observed by the iterator. Resizing the underlying + /// bit set invalidates the iterator. + pub fn iterator(self: *const Self, comptime options: IteratorOptions) BitSetIterator(MaskInt, options) { + const num_masks = numMasks(self.bit_length); + const padding_bits = num_masks * @bitSizeOf(MaskInt) - self.bit_length; + const last_item_mask = (~@as(MaskInt, 0)) >> @intCast(ShiftInt, padding_bits); + return BitSetIterator(MaskInt, options).init(self.masks[0..num_masks], last_item_mask); + } + + fn maskBit(index: usize) MaskInt { + return @as(MaskInt, 1) << @truncate(ShiftInt, index); + } + fn maskIndex(index: usize) usize { + return index >> @bitSizeOf(ShiftInt); + } + fn boolMaskBit(index: usize, value: bool) MaskInt { + return @as(MaskInt, @boolToInt(value)) << @intCast(ShiftInt, index); + } + fn numMasks(bit_length: usize) usize { + return (bit_length + (@bitSizeOf(MaskInt) - 1)) / @bitSizeOf(MaskInt); + } +}; + +/// A bit set with runtime known size, backed by an allocated slice +/// of usize. Thin wrapper around DynamicBitSetUnmanaged which keeps +/// track of the allocator instance. +pub const DynamicBitSet = struct { + const Self = @This(); + + /// The integer type used to represent a mask in this bit set + pub const MaskInt = usize; + + /// The integer type used to shift a mask in this bit set + pub const ShiftInt = std.math.Log2Int(MaskInt); + + /// The allocator used by this bit set + allocator: *Allocator, + + /// The number of valid items in this bit set + unmanaged: DynamicBitSetUnmanaged = .{}, + + /// Creates a bit set with no elements present. + pub fn initEmpty(bit_length: usize, allocator: *Allocator) !Self { + return Self{ + .unmanaged = try DynamicBitSetUnmanaged.initEmpty(bit_length, allocator), + .allocator = allocator, + }; + } + + /// Creates a bit set with all elements present. + pub fn initFull(bit_length: usize, allocator: *Allocator) !Self { + return Self{ + .unmanaged = try DynamicBitSetUnmanaged.initFull(bit_length, allocator), + .allocator = allocator, + }; + } + + /// Resizes to a new length. If the new length is larger + /// than the old length, fills any added bits with `fill`. + pub fn resize(self: *@This(), new_len: usize, fill: bool) !void { + try self.unmanaged.resize(new_len, fill, self.allocator); + } + + /// deinitializes the array and releases its memory. + /// The passed allocator must be the same one used for + /// init* or resize in the past. + pub fn deinit(self: *Self) void { + self.unmanaged.deinit(self.allocator); + } + + /// Creates a duplicate of this bit set, using the new allocator. + pub fn clone(self: *const Self, new_allocator: *Allocator) !Self { + return Self{ + .unmanaged = try self.unmanaged.clone(new_allocator), + .allocator = new_allocator, + }; + } + + /// Returns the number of bits in this bit set + pub inline fn capacity(self: Self) usize { + return self.unmanaged.capacity(); + } + + /// Returns true if the bit at the specified index + /// is present in the set, false otherwise. + pub fn isSet(self: Self, index: usize) bool { + return self.unmanaged.isSet(index); + } + + /// Returns the total number of set bits in this bit set. + pub fn count(self: Self) usize { + return self.unmanaged.count(); + } + + /// Changes the value of the specified bit of the bit + /// set to match the passed boolean. + pub fn setValue(self: *Self, index: usize, value: bool) void { + self.unmanaged.setValue(index, value); + } + + /// Adds a specific bit to the bit set + pub fn set(self: *Self, index: usize) void { + self.unmanaged.set(index); + } + + /// Removes a specific bit from the bit set + pub fn unset(self: *Self, index: usize) void { + self.unmanaged.unset(index); + } + + /// Flips a specific bit in the bit set + pub fn toggle(self: *Self, index: usize) void { + self.unmanaged.toggle(index); + } + + /// Flips all bits in this bit set which are present + /// in the toggles bit set. Both sets must have the + /// same bit_length. + pub fn toggleSet(self: *Self, toggles: Self) void { + self.unmanaged.toggleSet(toggles.unmanaged); + } + + /// Flips every bit in the bit set. + pub fn toggleAll(self: *Self) void { + self.unmanaged.toggleAll(); + } + + /// Performs a union of two bit sets, and stores the + /// result in the first one. Bits in the result are + /// set if the corresponding bits were set in either input. + /// The two sets must both be the same bit_length. + pub fn setUnion(self: *Self, other: Self) void { + self.unmanaged.setUnion(other.unmanaged); + } + + /// Performs an intersection of two bit sets, and stores + /// the result in the first one. Bits in the result are + /// set if the corresponding bits were set in both inputs. + /// The two sets must both be the same bit_length. + pub fn setIntersection(self: *Self, other: Self) void { + self.unmanaged.setIntersection(other.unmanaged); + } + + /// Finds the index of the first set bit. + /// If no bits are set, returns null. + pub fn findFirstSet(self: Self) ?usize { + return self.unmanaged.findFirstSet(); + } + + /// Finds the index of the first set bit, and unsets it. + /// If no bits are set, returns null. + pub fn toggleFirstSet(self: *Self) ?usize { + return self.unmanaged.toggleFirstSet(); + } + + /// Iterates through the items in the set, according to the options. + /// The default options (.{}) will iterate indices of set bits in + /// ascending order. Modifications to the underlying bit set may + /// or may not be observed by the iterator. Resizing the underlying + /// bit set invalidates the iterator. + pub fn iterator(self: *Self, comptime options: IteratorOptions) BitSetIterator(MaskInt, options) { + return self.unmanaged.iterator(options); + } +}; + +/// Options for configuring an iterator over a bit set +pub const IteratorOptions = struct { + /// determines which bits should be visited + kind: Type = .set, + /// determines the order in which bit indices should be visited + direction: Direction = .forward, + + pub const Type = enum { + /// visit indexes of set bits + set, + /// visit indexes of unset bits + unset, + }; + + pub const Direction = enum { + /// visit indices in ascending order + forward, + /// visit indices in descending order. + /// Note that this may be slightly more expensive than forward iteration. + reverse, + }; +}; + +// The iterator is reusable between several bit set types +fn BitSetIterator(comptime MaskInt: type, comptime options: IteratorOptions) type { + const ShiftInt = std.math.Log2Int(MaskInt); + const kind = options.kind; + const direction = options.direction; + return struct { + const Self = @This(); + + // all bits which have not yet been iterated over + bits_remain: MaskInt, + // all words which have not yet been iterated over + words_remain: []const MaskInt, + // the offset of the current word + bit_offset: usize, + // the mask of the last word + last_word_mask: MaskInt, + + fn init(masks: []const MaskInt, last_word_mask: MaskInt) Self { + if (masks.len == 0) { + return Self{ + .bits_remain = 0, + .words_remain = &[_]MaskInt{}, + .last_word_mask = last_word_mask, + .bit_offset = 0, + }; + } else { + var result = Self{ + .bits_remain = 0, + .words_remain = masks, + .last_word_mask = last_word_mask, + .bit_offset = if (direction == .forward) 0 else (masks.len - 1) * @bitSizeOf(MaskInt), + }; + result.nextWord(true); + return result; + } + } + + /// Returns the index of the next unvisited set bit + /// in the bit set, in ascending order. + pub fn next(self: *Self) ?usize { + while (self.bits_remain == 0) { + if (self.words_remain.len == 0) return null; + self.nextWord(false); + switch (direction) { + .forward => self.bit_offset += @bitSizeOf(MaskInt), + .reverse => self.bit_offset -= @bitSizeOf(MaskInt), + } + } + + switch (direction) { + .forward => { + const next_index = @ctz(MaskInt, self.bits_remain) + self.bit_offset; + self.bits_remain &= self.bits_remain - 1; + return next_index; + }, + .reverse => { + const leading_zeroes = @clz(MaskInt, self.bits_remain); + const top_bit = (@bitSizeOf(MaskInt) - 1) - leading_zeroes; + const no_top_bit_mask = (@as(MaskInt, 1) << @intCast(ShiftInt, top_bit)) - 1; + self.bits_remain &= no_top_bit_mask; + return top_bit + self.bit_offset; + }, + } + } + + // Load the next word. Don't call this if there + // isn't a next word. If the next word is the + // last word, mask off the padding bits so we + // don't visit them. + inline fn nextWord(self: *Self, comptime is_first_word: bool) void { + var word = switch (direction) { + .forward => self.words_remain[0], + .reverse => self.words_remain[self.words_remain.len - 1], + }; + switch (kind) { + .set => {}, + .unset => { + word = ~word; + if ((direction == .reverse and is_first_word) or + (direction == .forward and self.words_remain.len == 1)) + { + word &= self.last_word_mask; + } + }, + } + switch (direction) { + .forward => self.words_remain = self.words_remain[1..], + .reverse => self.words_remain.len -= 1, + } + self.bits_remain = word; + } + }; +} + +// ---------------- Tests ----------------- + +const testing = std.testing; + +fn testBitSet(a: anytype, b: anytype, len: usize) void { + testing.expectEqual(len, a.capacity()); + testing.expectEqual(len, b.capacity()); + + { + var i: usize = 0; + while (i < len) : (i += 1) { + a.setValue(i, i & 1 == 0); + b.setValue(i, i & 2 == 0); + } + } + + testing.expectEqual((len + 1) / 2, a.count()); + testing.expectEqual((len + 3) / 4 + (len + 2) / 4, b.count()); + + { + var iter = a.iterator(.{}); + var i: usize = 0; + while (i < len) : (i += 2) { + testing.expectEqual(@as(?usize, i), iter.next()); + } + testing.expectEqual(@as(?usize, null), iter.next()); + testing.expectEqual(@as(?usize, null), iter.next()); + testing.expectEqual(@as(?usize, null), iter.next()); + } + a.toggleAll(); + { + var iter = a.iterator(.{}); + var i: usize = 1; + while (i < len) : (i += 2) { + testing.expectEqual(@as(?usize, i), iter.next()); + } + testing.expectEqual(@as(?usize, null), iter.next()); + testing.expectEqual(@as(?usize, null), iter.next()); + testing.expectEqual(@as(?usize, null), iter.next()); + } + + { + var iter = b.iterator(.{ .kind = .unset }); + var i: usize = 2; + while (i < len) : (i += 4) { + testing.expectEqual(@as(?usize, i), iter.next()); + if (i + 1 < len) { + testing.expectEqual(@as(?usize, i + 1), iter.next()); + } + } + testing.expectEqual(@as(?usize, null), iter.next()); + testing.expectEqual(@as(?usize, null), iter.next()); + testing.expectEqual(@as(?usize, null), iter.next()); + } + + { + var i: usize = 0; + while (i < len) : (i += 1) { + testing.expectEqual(i & 1 != 0, a.isSet(i)); + testing.expectEqual(i & 2 == 0, b.isSet(i)); + } + } + + a.setUnion(b.*); + { + var i: usize = 0; + while (i < len) : (i += 1) { + testing.expectEqual(i & 1 != 0 or i & 2 == 0, a.isSet(i)); + testing.expectEqual(i & 2 == 0, b.isSet(i)); + } + + i = len; + var set = a.iterator(.{ .direction = .reverse }); + var unset = a.iterator(.{ .kind = .unset, .direction = .reverse }); + while (i > 0) { + i -= 1; + if (i & 1 != 0 or i & 2 == 0) { + testing.expectEqual(@as(?usize, i), set.next()); + } else { + testing.expectEqual(@as(?usize, i), unset.next()); + } + } + testing.expectEqual(@as(?usize, null), set.next()); + testing.expectEqual(@as(?usize, null), set.next()); + testing.expectEqual(@as(?usize, null), set.next()); + testing.expectEqual(@as(?usize, null), unset.next()); + testing.expectEqual(@as(?usize, null), unset.next()); + testing.expectEqual(@as(?usize, null), unset.next()); + } + + a.toggleSet(b.*); + { + testing.expectEqual(len / 4, a.count()); + + var i: usize = 0; + while (i < len) : (i += 1) { + testing.expectEqual(i & 1 != 0 and i & 2 != 0, a.isSet(i)); + testing.expectEqual(i & 2 == 0, b.isSet(i)); + if (i & 1 == 0) { + a.set(i); + } else { + a.unset(i); + } + } + } + + a.setIntersection(b.*); + { + testing.expectEqual((len + 3) / 4, a.count()); + + var i: usize = 0; + while (i < len) : (i += 1) { + testing.expectEqual(i & 1 == 0 and i & 2 == 0, a.isSet(i)); + testing.expectEqual(i & 2 == 0, b.isSet(i)); + } + } + + a.toggleSet(a.*); + { + var iter = a.iterator(.{}); + testing.expectEqual(@as(?usize, null), iter.next()); + testing.expectEqual(@as(?usize, null), iter.next()); + testing.expectEqual(@as(?usize, null), iter.next()); + testing.expectEqual(@as(usize, 0), a.count()); + } + { + var iter = a.iterator(.{ .direction = .reverse }); + testing.expectEqual(@as(?usize, null), iter.next()); + testing.expectEqual(@as(?usize, null), iter.next()); + testing.expectEqual(@as(?usize, null), iter.next()); + testing.expectEqual(@as(usize, 0), a.count()); + } + + const test_bits = [_]usize{ + 0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 22, 31, 32, 63, 64, + 66, 95, 127, 160, 192, 1000 }; + for (test_bits) |i| { + if (i < a.capacity()) { + a.set(i); + } + } + + for (test_bits) |i| { + if (i < a.capacity()) { + testing.expectEqual(@as(?usize, i), a.findFirstSet()); + testing.expectEqual(@as(?usize, i), a.toggleFirstSet()); + } + } + testing.expectEqual(@as(?usize, null), a.findFirstSet()); + testing.expectEqual(@as(?usize, null), a.toggleFirstSet()); + testing.expectEqual(@as(?usize, null), a.findFirstSet()); + testing.expectEqual(@as(?usize, null), a.toggleFirstSet()); + testing.expectEqual(@as(usize, 0), a.count()); +} + +fn testStaticBitSet(comptime Set: type) void { + var a = Set.initEmpty(); + var b = Set.initFull(); + testing.expectEqual(@as(usize, 0), a.count()); + testing.expectEqual(@as(usize, Set.bit_length), b.count()); + + testBitSet(&a, &b, Set.bit_length); +} + +test "IntegerBitSet" { + testStaticBitSet(IntegerBitSet(0)); + testStaticBitSet(IntegerBitSet(1)); + testStaticBitSet(IntegerBitSet(2)); + testStaticBitSet(IntegerBitSet(5)); + testStaticBitSet(IntegerBitSet(8)); + testStaticBitSet(IntegerBitSet(32)); + testStaticBitSet(IntegerBitSet(64)); + testStaticBitSet(IntegerBitSet(127)); +} + +test "ArrayBitSet" { + inline for (.{ 0, 1, 2, 31, 32, 33, 63, 64, 65, 254, 500, 3000 }) |size| { + testStaticBitSet(ArrayBitSet(u8, size)); + testStaticBitSet(ArrayBitSet(u16, size)); + testStaticBitSet(ArrayBitSet(u32, size)); + testStaticBitSet(ArrayBitSet(u64, size)); + testStaticBitSet(ArrayBitSet(u128, size)); + } +} + +test "DynamicBitSetUnmanaged" { + const allocator = std.testing.allocator; + var a = try DynamicBitSetUnmanaged.initEmpty(300, allocator); + testing.expectEqual(@as(usize, 0), a.count()); + a.deinit(allocator); + + a = try DynamicBitSetUnmanaged.initEmpty(0, allocator); + defer a.deinit(allocator); + for ([_]usize{ 1, 2, 31, 32, 33, 0, 65, 64, 63, 500, 254, 3000 }) |size| { + const old_len = a.capacity(); + + var tmp = try a.clone(allocator); + defer tmp.deinit(allocator); + testing.expectEqual(old_len, tmp.capacity()); + var i: usize = 0; + while (i < old_len) : (i += 1) { + testing.expectEqual(a.isSet(i), tmp.isSet(i)); + } + + a.toggleSet(a); // zero a + tmp.toggleSet(tmp); + + try a.resize(size, true, allocator); + try tmp.resize(size, false, allocator); + + if (size > old_len) { + testing.expectEqual(size - old_len, a.count()); + } else { + testing.expectEqual(@as(usize, 0), a.count()); + } + testing.expectEqual(@as(usize, 0), tmp.count()); + + var b = try DynamicBitSetUnmanaged.initFull(size, allocator); + defer b.deinit(allocator); + testing.expectEqual(@as(usize, size), b.count()); + + testBitSet(&a, &b, size); + } +} + +test "DynamicBitSet" { + const allocator = std.testing.allocator; + var a = try DynamicBitSet.initEmpty(300, allocator); + testing.expectEqual(@as(usize, 0), a.count()); + a.deinit(); + + a = try DynamicBitSet.initEmpty(0, allocator); + defer a.deinit(); + for ([_]usize{ 1, 2, 31, 32, 33, 0, 65, 64, 63, 500, 254, 3000 }) |size| { + const old_len = a.capacity(); + + var tmp = try a.clone(allocator); + defer tmp.deinit(); + testing.expectEqual(old_len, tmp.capacity()); + var i: usize = 0; + while (i < old_len) : (i += 1) { + testing.expectEqual(a.isSet(i), tmp.isSet(i)); + } + + a.toggleSet(a); // zero a + tmp.toggleSet(tmp); // zero tmp + + try a.resize(size, true); + try tmp.resize(size, false); + + if (size > old_len) { + testing.expectEqual(size - old_len, a.count()); + } else { + testing.expectEqual(@as(usize, 0), a.count()); + } + testing.expectEqual(@as(usize, 0), tmp.count()); + + var b = try DynamicBitSet.initFull(size, allocator); + defer b.deinit(); + testing.expectEqual(@as(usize, size), b.count()); + + testBitSet(&a, &b, size); + } +} + +test "StaticBitSet" { + testing.expectEqual(IntegerBitSet(0), StaticBitSet(0)); + testing.expectEqual(IntegerBitSet(5), StaticBitSet(5)); + testing.expectEqual(IntegerBitSet(@bitSizeOf(usize)), StaticBitSet(@bitSizeOf(usize))); + testing.expectEqual(ArrayBitSet(usize, @bitSizeOf(usize) + 1), StaticBitSet(@bitSizeOf(usize) + 1)); + testing.expectEqual(ArrayBitSet(usize, 500), StaticBitSet(500)); +} diff --git a/lib/std/math.zig b/lib/std/math.zig index 6e7c5c09155d2466ca2472c90d352e49595d04f1..940284b763249fd8ffea7b8ced7bdd4457214442 100644 --- a/lib/std/math.zig +++ b/lib/std/math.zig @@ -1330,3 +1330,59 @@ test "math.comptime" { comptime const v = sin(@as(f32, 1)) + ln(@as(f32, 5)); testing.expect(v == sin(@as(f32, 1)) + ln(@as(f32, 5))); } + +/// Returns a mask of all ones if value is true, +/// and a mask of all zeroes if value is false. +/// Compiles to one instruction for register sized integers. +pub inline fn boolMask(comptime MaskInt: type, value: bool) MaskInt { + if (@typeInfo(MaskInt) != .Int) + @compileError("boolMask requires an integer mask type."); + + if (MaskInt == u0 or MaskInt == i0) + @compileError("boolMask cannot convert to u0 or i0, they are too small."); + + // The u1 and i1 cases tend to overflow, + // so we special case them here. + if (MaskInt == u1) return @boolToInt(value); + if (MaskInt == i1) { + // The @as here is a workaround for #7950 + return @bitCast(i1, @as(u1, @boolToInt(value))); + } + + // At comptime, -% is disallowed on unsigned values. + // So we need to jump through some hoops in that case. + // This is a workaround for #7951 + if (@typeInfo(@TypeOf(.{value})).Struct.fields[0].is_comptime) { + // Since it's comptime, we don't need this to generate nice code. + // We can just do a branch here. + return if (value) ~@as(MaskInt, 0) else 0; + } + + return -%@intCast(MaskInt, @boolToInt(value)); +} + +test "boolMask" { + const runTest = struct { + fn runTest() void { + testing.expectEqual(@as(u1, 0), boolMask(u1, false)); + testing.expectEqual(@as(u1, 1), boolMask(u1, true)); + + testing.expectEqual(@as(i1, 0), boolMask(i1, false)); + testing.expectEqual(@as(i1, -1), boolMask(i1, true)); + + testing.expectEqual(@as(u13, 0), boolMask(u13, false)); + testing.expectEqual(@as(u13, 0x1FFF), boolMask(u13, true)); + + testing.expectEqual(@as(i13, 0), boolMask(i13, false)); + testing.expectEqual(@as(i13, -1), boolMask(i13, true)); + + testing.expectEqual(@as(u32, 0), boolMask(u32, false)); + testing.expectEqual(@as(u32, 0xFFFF_FFFF), boolMask(u32, true)); + + testing.expectEqual(@as(i32, 0), boolMask(i32, false)); + testing.expectEqual(@as(i32, -1), boolMask(i32, true)); + } + }.runTest; + runTest(); + comptime runTest(); +} diff --git a/lib/std/mem.zig b/lib/std/mem.zig index 5f23a1040129f5f28850cbf7e76eeca4f6facd89..581fb16e6c284645da6b22e5f443155199dd369f 100644 --- a/lib/std/mem.zig +++ b/lib/std/mem.zig @@ -25,6 +25,14 @@ pub const page_size = switch (builtin.arch) { else => 4 * 1024, }; +/// The standard library currently thoroughly depends on byte size +/// being 8 bits. (see the use of u8 throughout allocation code as +/// the "byte" type.) Code which depends on this can reference this +/// declaration. If we ever try to port the standard library to a +/// non-8-bit-byte platform, this will allow us to search for things +/// which need to be updated. +pub const byte_size_in_bits = 8; + pub const Allocator = @import("mem/Allocator.zig"); /// Detects and asserts if the std.mem.Allocator interface is violated by the caller diff --git a/lib/std/std.zig b/lib/std/std.zig index c0d97a9d9c2ec677523197537cc10b8fb2401f44..a7e5bcb682782aed33111a2df677fe08500ad9bd 100644 --- a/lib/std/std.zig +++ b/lib/std/std.zig @@ -18,6 +18,8 @@ pub const BufSet = @import("buf_set.zig").BufSet; pub const ChildProcess = @import("child_process.zig").ChildProcess; pub const ComptimeStringMap = @import("comptime_string_map.zig").ComptimeStringMap; pub const DynLib = @import("dynamic_library.zig").DynLib; +pub const DynamicBitSet = bit_set.DynamicBitSet; +pub const DynamicBitSetUnmanaged = bit_set.DynamicBitSetUnmanaged; pub const HashMap = hash_map.HashMap; pub const HashMapUnmanaged = hash_map.HashMapUnmanaged; pub const MultiArrayList = @import("multi_array_list.zig").MultiArrayList; @@ -29,6 +31,7 @@ pub const PriorityQueue = @import("priority_queue.zig").PriorityQueue; pub const Progress = @import("Progress.zig"); pub const SemanticVersion = @import("SemanticVersion.zig"); pub const SinglyLinkedList = @import("linked_list.zig").SinglyLinkedList; +pub const StaticBitSet = bit_set.StaticBitSet; pub const StringHashMap = hash_map.StringHashMap; pub const StringHashMapUnmanaged = hash_map.StringHashMapUnmanaged; pub const StringArrayHashMap = array_hash_map.StringArrayHashMap; @@ -40,6 +43,7 @@ pub const Thread = @import("Thread.zig"); pub const array_hash_map = @import("array_hash_map.zig"); pub const atomic = @import("atomic.zig"); pub const base64 = @import("base64.zig"); +pub const bit_set = @import("bit_set.zig"); pub const build = @import("build.zig"); pub const builtin = @import("builtin.zig"); pub const c = @import("c.zig"); -- 2.54.0 From a20169a610b693794756bbc0139f0955c1294b48 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 1 Mar 2021 20:04:28 -0700 Subject: [PATCH 18/35] zig fmt the std lib --- lib/std/bit_set.zig | 27 ++++++++++++++------------- lib/std/fs/get_app_data_dir.zig | 2 +- lib/std/math.zig | 2 +- lib/std/os/bits/haiku.zig | 13 ++++++------- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/lib/std/bit_set.zig b/lib/std/bit_set.zig index c3737ee20fb32f6dab70afaa0d52736226eedce9..29ad0d7963c9f01c91ea0b036c23846d1802e086 100644 --- a/lib/std/bit_set.zig +++ b/lib/std/bit_set.zig @@ -4,10 +4,6 @@ // The MIT license requires this copyright notice to be included in all copies // and substantial portions of the software. -const std = @import("std"); -const assert = std.debug.assert; -const Allocator = std.mem.Allocator; - //! This file defines several variants of bit sets. A bit set //! is a densely stored set of integers with a known maximum, //! in which each integer gets a single bit. Bit sets have very @@ -40,6 +36,10 @@ const Allocator = std.mem.Allocator; //! A variant of DynamicBitSet which does not store a pointer to its //! allocator, in order to save space. +const std = @import("std"); +const assert = std.debug.assert; +const Allocator = std.mem.Allocator; + /// Returns the optimal static bit set type for the specified number /// of elements. The returned type will perform no allocations, /// can be copied by value, and does not require deinitialization. @@ -83,7 +83,7 @@ pub fn IntegerBitSet(comptime size: u16) type { } /// Returns the number of bits in this bit set - pub inline fn capacity(self: Self) usize { + pub fn capacity(self: Self) callconv(.Inline) usize { return bit_length; } @@ -168,7 +168,7 @@ pub fn IntegerBitSet(comptime size: u16) type { const mask = self.mask; if (mask == 0) return null; const index = @ctz(MaskInt, mask); - self.mask = mask & (mask-1); + self.mask = mask & (mask - 1); return index; } @@ -306,7 +306,7 @@ pub fn ArrayBitSet(comptime MaskIntType: type, comptime size: usize) type { } /// Returns the number of bits in this bit set - pub inline fn capacity(self: Self) usize { + pub fn capacity(self: Self) callconv(.Inline) usize { return bit_length; } @@ -566,7 +566,7 @@ pub const DynamicBitSetUnmanaged = struct { } /// Returns the number of bits in this bit set - pub inline fn capacity(self: Self) usize { + pub fn capacity(self: Self) callconv(.Inline) usize { return self.bit_length; } @@ -691,7 +691,7 @@ pub const DynamicBitSetUnmanaged = struct { offset += @bitSizeOf(MaskInt); } else return null; const index = @ctz(MaskInt, mask[0]); - mask[0] &= (mask[0]-1); + mask[0] &= (mask[0] - 1); return offset + index; } @@ -777,7 +777,7 @@ pub const DynamicBitSet = struct { } /// Returns the number of bits in this bit set - pub inline fn capacity(self: Self) usize { + pub fn capacity(self: Self) callconv(.Inline) usize { return self.unmanaged.capacity(); } @@ -955,7 +955,7 @@ fn BitSetIterator(comptime MaskInt: type, comptime options: IteratorOptions) typ // isn't a next word. If the next word is the // last word, mask off the padding bits so we // don't visit them. - inline fn nextWord(self: *Self, comptime is_first_word: bool) void { + fn nextWord(self: *Self, comptime is_first_word: bool) callconv(.Inline) void { var word = switch (direction) { .forward => self.words_remain[0], .reverse => self.words_remain[self.words_remain.len - 1], @@ -1114,8 +1114,9 @@ fn testBitSet(a: anytype, b: anytype, len: usize) void { } const test_bits = [_]usize{ - 0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 22, 31, 32, 63, 64, - 66, 95, 127, 160, 192, 1000 }; + 0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 22, 31, 32, 63, 64, + 66, 95, 127, 160, 192, 1000, + }; for (test_bits) |i| { if (i < a.capacity()) { a.set(i); diff --git a/lib/std/fs/get_app_data_dir.zig b/lib/std/fs/get_app_data_dir.zig index 18f8458eb24271073e9500961c5dad06854cda74..02c36f736af78a3506e255e058ee3a67751d4b1f 100644 --- a/lib/std/fs/get_app_data_dir.zig +++ b/lib/std/fs/get_app_data_dir.zig @@ -60,7 +60,7 @@ pub fn getAppDataDir(allocator: *mem.Allocator, appname: []const u8) GetAppDataD var dir_path_ptr: [*:0]u8 = undefined; // TODO look into directory_which const be_user_settings = 0xbbe; - const rc = os.system.find_directory(be_user_settings, -1, true, dir_path_ptr, 1) ; + const rc = os.system.find_directory(be_user_settings, -1, true, dir_path_ptr, 1); const settings_dir = try allocator.dupeZ(u8, mem.spanZ(dir_path_ptr)); defer allocator.free(settings_dir); switch (rc) { diff --git a/lib/std/math.zig b/lib/std/math.zig index 940284b763249fd8ffea7b8ced7bdd4457214442..d71cafe5efc9c9122dca20e908b3f6019c8f33ac 100644 --- a/lib/std/math.zig +++ b/lib/std/math.zig @@ -1334,7 +1334,7 @@ test "math.comptime" { /// Returns a mask of all ones if value is true, /// and a mask of all zeroes if value is false. /// Compiles to one instruction for register sized integers. -pub inline fn boolMask(comptime MaskInt: type, value: bool) MaskInt { +pub fn boolMask(comptime MaskInt: type, value: bool) callconv(.Inline) MaskInt { if (@typeInfo(MaskInt) != .Int) @compileError("boolMask requires an integer mask type."); diff --git a/lib/std/os/bits/haiku.zig b/lib/std/os/bits/haiku.zig index 59631fd40ec4de6786d4ae72eb6d4e857ffa696c..32093570d754754d95a5142516c5d18901f1fa90 100644 --- a/lib/std/os/bits/haiku.zig +++ b/lib/std/os/bits/haiku.zig @@ -180,8 +180,8 @@ pub const dirent = extern struct { }; pub const image_info = extern struct { - id: u32, //image_id - type: u32, // image_type + id: u32, + type: u32, sequence: i32, init_order: i32, init_routine: *c_void, @@ -806,17 +806,16 @@ pub const Sigaction = extern struct { pub const _SIG_WORDS = 4; pub const _SIG_MAXSIG = 128; - -pub inline fn _SIG_IDX(sig: usize) usize { +pub fn _SIG_IDX(sig: usize) callconv(.Inline) usize { return sig - 1; } -pub inline fn _SIG_WORD(sig: usize) usize { +pub fn _SIG_WORD(sig: usize) callconv(.Inline) usize { return_SIG_IDX(sig) >> 5; } -pub inline fn _SIG_BIT(sig: usize) usize { +pub fn _SIG_BIT(sig: usize) callconv(.Inline) usize { return 1 << (_SIG_IDX(sig) & 31); } -pub inline fn _SIG_VALID(sig: usize) usize { +pub fn _SIG_VALID(sig: usize) callconv(.Inline) usize { return sig <= _SIG_MAXSIG and sig > 0; } -- 2.54.0 From 3dd8396a5542af26636e8bf1e16a2f7f245121ac Mon Sep 17 00:00:00 2001 From: Vincent Rischmann Date: Sun, 28 Feb 2021 18:43:02 +0100 Subject: [PATCH 19/35] os/linux: fix IO_Uring.timeout According to the io_uring PDF (https://kernel.dk/io_uring.pdf) the timeout struct must be 64 bits on both 32 and 64 bit architectures. --- lib/std/os/bits/linux.zig | 5 +++++ lib/std/os/linux/io_uring.zig | 10 +++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/std/os/bits/linux.zig b/lib/std/os/bits/linux.zig index 8d3d5c49a3cc0b536d5afda8386438ba8958c9e1..21fa058aeff1bc3a1bb11b7861bb708b1175c781 100644 --- a/lib/std/os/bits/linux.zig +++ b/lib/std/os/bits/linux.zig @@ -2244,3 +2244,8 @@ pub const MADV_COLD = 20; pub const MADV_PAGEOUT = 21; pub const MADV_HWPOISON = 100; pub const MADV_SOFT_OFFLINE = 101; + +pub const __kernel_timespec = extern struct { + tv_sec: i64, + tv_nsec: i64, +}; diff --git a/lib/std/os/linux/io_uring.zig b/lib/std/os/linux/io_uring.zig index b47d4c7b3238bd7a6199874d03eb4d4b23967ff1..340020cf9b8c44c1a29cfaf11c4f9b745e851b3a 100644 --- a/lib/std/os/linux/io_uring.zig +++ b/lib/std/os/linux/io_uring.zig @@ -526,7 +526,7 @@ pub const IO_Uring = struct { pub fn timeout( self: *IO_Uring, user_data: u64, - ts: *const os.timespec, + ts: *const os.__kernel_timespec, count: u32, flags: u32, ) !*io_uring_sqe { @@ -884,7 +884,7 @@ pub fn io_uring_prep_close(sqe: *io_uring_sqe, fd: os.fd_t) void { pub fn io_uring_prep_timeout( sqe: *io_uring_sqe, - ts: *const os.timespec, + ts: *const os.__kernel_timespec, count: u32, flags: u32, ) void { @@ -1339,7 +1339,7 @@ test "timeout (after a relative time)" { const ms = 10; const margin = 5; - const ts = os.timespec{ .tv_sec = 0, .tv_nsec = ms * 1000000 }; + const ts = os.__kernel_timespec{ .tv_sec = 0, .tv_nsec = ms * 1000000 }; const started = std.time.milliTimestamp(); const sqe = try ring.timeout(0x55555555, &ts, 0, 0); @@ -1366,7 +1366,7 @@ test "timeout (after a number of completions)" { }; defer ring.deinit(); - const ts = os.timespec{ .tv_sec = 3, .tv_nsec = 0 }; + const ts = os.__kernel_timespec{ .tv_sec = 3, .tv_nsec = 0 }; const count_completions: u64 = 1; const sqe_timeout = try ring.timeout(0x66666666, &ts, count_completions, 0); testing.expectEqual(linux.IORING_OP.TIMEOUT, sqe_timeout.opcode); @@ -1399,7 +1399,7 @@ test "timeout_remove" { }; defer ring.deinit(); - const ts = os.timespec{ .tv_sec = 3, .tv_nsec = 0 }; + const ts = os.__kernel_timespec{ .tv_sec = 3, .tv_nsec = 0 }; const sqe_timeout = try ring.timeout(0x88888888, &ts, 0, 0); testing.expectEqual(linux.IORING_OP.TIMEOUT, sqe_timeout.opcode); testing.expectEqual(@as(u64, 0x88888888), sqe_timeout.user_data); -- 2.54.0 From 84f3b3dff2cc8f380a60e920cb1ca76e779571c6 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 2 Mar 2021 16:32:52 -0700 Subject: [PATCH 20/35] re-enable behavior tests: translate-c macros These were temporarily disabled in dd973fb365dbbe11ce5beac8b4889bfab3fddc4d and we forgot re-enable them until now. --- test/stage1/behavior.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/stage1/behavior.zig b/test/stage1/behavior.zig index f82631cd9e19b08db1ad374c937a4221a1b106a3..db6e5962912d9b294848bd2a1902588a283a141f 100644 --- a/test/stage1/behavior.zig +++ b/test/stage1/behavior.zig @@ -141,5 +141,5 @@ comptime { _ = @import("behavior/while.zig"); _ = @import("behavior/widening.zig"); _ = @import("behavior/src.zig"); - // _ = @import("behavior/translate_c_macros.zig"); + _ = @import("behavior/translate_c_macros.zig"); } -- 2.54.0 From 3ad9cb8b473820ec5ea11d85aa72e8ddc83cfa03 Mon Sep 17 00:00:00 2001 From: Isaac Freund Date: Tue, 2 Mar 2021 21:36:25 +0100 Subject: [PATCH 21/35] zig fmt: allow and trim whitespace around zig fmt: (off|on) Currently `// zig fmt: off` does not work as there are two spaces after the `//` instead of one. This can cause confusion, so allow arbitrary whitespace before the `zig fmt: (off|on)` in the comment but trim this whitespace to the canonical single space in the output. --- lib/std/zig/parser_test.zig | 19 +++++++++++++++++++ lib/std/zig/render.zig | 26 ++++++++++++++++---------- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index 375a041a0ab169f005dfe77e0176ea274c295a9f..c083d23932b377e341702f0844884936acb5c1cf 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -1108,6 +1108,25 @@ test "zig fmt: comment to disable/enable zig fmt first" { ); } +test "zig fmt: 'zig fmt: (off|on)' can be surrounded by arbitrary whitespace" { + try testTransform( + \\// Test trailing comma syntax + \\// zig fmt: off + \\ + \\const struct_trailing_comma = struct { x: i32, y: i32, }; + \\ + \\// zig fmt: on + , + \\// Test trailing comma syntax + \\// zig fmt: off + \\ + \\const struct_trailing_comma = struct { x: i32, y: i32, }; + \\ + \\// zig fmt: on + \\ + ); +} + test "zig fmt: comment to disable/enable zig fmt" { try testTransform( \\const a = b; diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index e12f7bc733b9822145a7063a523d32a3799eafe0..069b62af79b2f326db9b966487b4f08d300f8c90 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -2352,18 +2352,24 @@ fn renderComments(ais: *Ais, tree: ast.Tree, start: usize, end: usize) Error!boo } } - try ais.writer().print("{s}\n", .{trimmed_comment}); - index = 1 + (newline orelse return true); + index = 1 + (newline orelse end - 1); - if (ais.disabled_offset) |disabled_offset| { - if (mem.eql(u8, trimmed_comment, "// zig fmt: on")) { - // write the source for which formatting was disabled directly - // to the underlying writer, fixing up invaild whitespace - try writeFixingWhitespace(ais.underlying_writer, tree.source[disabled_offset..index]); - ais.disabled_offset = null; - } - } else if (mem.eql(u8, trimmed_comment, "// zig fmt: off")) { + const comment_content = mem.trimLeft(u8, trimmed_comment["//".len..], &std.ascii.spaces); + if (ais.disabled_offset != null and mem.eql(u8, comment_content, "zig fmt: on")) { + // Write the source for which formatting was disabled directly + // to the underlying writer, fixing up invaild whitespace. + const disabled_source = tree.source[ais.disabled_offset.?..comment_start]; + try writeFixingWhitespace(ais.underlying_writer, disabled_source); + ais.disabled_offset = null; + // Write with the canonical single space. + try ais.writer().writeAll("// zig fmt: on\n"); + } else if (ais.disabled_offset == null and mem.eql(u8, comment_content, "zig fmt: off")) { + // Write with the canonical single space. + try ais.writer().writeAll("// zig fmt: off\n"); ais.disabled_offset = index; + } else { + // Write the comment minus trailing whitespace. + try ais.writer().print("{s}\n", .{trimmed_comment}); } } -- 2.54.0 From d4ec0279d3ec59cf33b1b2fbf74f9ae82af753cc Mon Sep 17 00:00:00 2001 From: Timon Kruiper Date: Sat, 9 Jan 2021 16:22:43 +0100 Subject: [PATCH 22/35] stage2: add support for optionals in the LLVM backend We can now codegen optionals! This includes the following instructions: - is_null - is_null_ptr - is_non_null - is_non_null_ptr - optional_payload - optional_payload_ptr - br_void Also includes a test for optionals. --- src/astgen.zig | 29 ++++++--- src/codegen/llvm.zig | 113 +++++++++++++++++++++++++++++++--- src/codegen/llvm/bindings.zig | 9 +++ test/stage2/llvm.zig | 40 ++++++++++++ 4 files changed, 176 insertions(+), 15 deletions(-) diff --git a/src/astgen.zig b/src/astgen.zig index 63184e641b65cc6ec4d9faa90e6ba58903cf757a..2c2f41c3f690bc9c966ce064bfc072229657512d 100644 --- a/src/astgen.zig +++ b/src/astgen.zig @@ -453,13 +453,23 @@ pub fn expr(mod: *Module, scope: *Scope, rl: ResultLoc, node: ast.Node.Index) In return rvalue(mod, scope, rl, result); }, .unwrap_optional => { - const operand = try expr(mod, scope, rl, node_datas[node].lhs); - const op: zir.Inst.Tag = switch (rl) { - .ref => .optional_payload_safe_ptr, - else => .optional_payload_safe, - }; const src = token_starts[main_tokens[node]]; - return addZIRUnOp(mod, scope, src, op, operand); + switch (rl) { + .ref => return addZIRUnOp( + mod, + scope, + src, + .optional_payload_safe_ptr, + try expr(mod, scope, .ref, node_datas[node].lhs), + ), + else => return rvalue(mod, scope, rl, try addZIRUnOp( + mod, + scope, + src, + .optional_payload_safe, + try expr(mod, scope, .none, node_datas[node].lhs), + )), + } }, .block_two, .block_two_semicolon => { const statements = [2]ast.Node.Index{ node_datas[node].lhs, node_datas[node].rhs }; @@ -1701,7 +1711,12 @@ fn orelseCatchExpr( // This could be a pointer or value depending on the `rl` parameter. block_scope.break_count += 1; - const operand = try expr(mod, &block_scope.base, block_scope.break_result_loc, lhs); + const operand = try expr( + mod, + &block_scope.base, + if (block_scope.break_result_loc == .ref) .ref else .none, + lhs, + ); const cond = try addZIRUnOp(mod, &block_scope.base, src, cond_op, operand); const condbr = try addZIRInstSpecial(mod, &block_scope.base, src, zir.Inst.CondBr, .{ diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index df6a58b1e23a71a1e27f3afc8b056ad9d9bb1e86..f087957f1d157ba1ebbe2b9e91347106ba047341 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -397,6 +397,7 @@ pub const LLVMIRModule = struct { .block => try self.genBlock(inst.castTag(.block).?), .br => try self.genBr(inst.castTag(.br).?), .breakpoint => try self.genBreakpoint(inst.castTag(.breakpoint).?), + .br_void => try self.genBrVoid(inst.castTag(.br_void).?), .call => try self.genCall(inst.castTag(.call).?), .cmp_eq => try self.genCmp(inst.castTag(.cmp_eq).?, .eq), .cmp_gt => try self.genCmp(inst.castTag(.cmp_gt).?, .gt), @@ -406,6 +407,10 @@ pub const LLVMIRModule = struct { .cmp_neq => try self.genCmp(inst.castTag(.cmp_neq).?, .neq), .condbr => try self.genCondBr(inst.castTag(.condbr).?), .intcast => try self.genIntCast(inst.castTag(.intcast).?), + .is_non_null => try self.genIsNonNull(inst.castTag(.is_non_null).?, false), + .is_non_null_ptr => try self.genIsNonNull(inst.castTag(.is_non_null_ptr).?, true), + .is_null => try self.genIsNull(inst.castTag(.is_null).?, false), + .is_null_ptr => try self.genIsNull(inst.castTag(.is_null_ptr).?, true), .load => try self.genLoad(inst.castTag(.load).?), .loop => try self.genLoop(inst.castTag(.loop).?), .not => try self.genNot(inst.castTag(.not).?), @@ -414,6 +419,8 @@ pub const LLVMIRModule = struct { .store => try self.genStore(inst.castTag(.store).?), .sub => try self.genSub(inst.castTag(.sub).?), .unreach => self.genUnreach(inst.castTag(.unreach).?), + .optional_payload => try self.genOptionalPayload(inst.castTag(.optional_payload).?, false), + .optional_payload_ptr => try self.genOptionalPayload(inst.castTag(.optional_payload_ptr).?, true), .dbg_stmt => blk: { // TODO: implement debug info break :blk null; @@ -534,21 +541,29 @@ pub const LLVMIRModule = struct { } fn genBr(self: *LLVMIRModule, inst: *Inst.Br) !?*const llvm.Value { - // Get the block that we want to break to. var block = self.blocks.get(inst.block).?; - _ = self.builder.buildBr(block.parent_bb); // If the break doesn't break a value, then we don't have to add // the values to the lists. - if (!inst.operand.ty.hasCodeGenBits()) return null; + if (!inst.operand.ty.hasCodeGenBits()) { + // TODO: in astgen these instructions should turn into `br_void` instructions. + _ = self.builder.buildBr(block.parent_bb); + } else { + const val = try self.resolveInst(inst.operand); - // For the phi node, we need the basic blocks and the values of the - // break instructions. - try block.break_bbs.append(self.gpa, self.builder.getInsertBlock()); + // For the phi node, we need the basic blocks and the values of the + // break instructions. + try block.break_bbs.append(self.gpa, self.builder.getInsertBlock()); + try block.break_vals.append(self.gpa, val); - const val = try self.resolveInst(inst.operand); - try block.break_vals.append(self.gpa, val); + _ = self.builder.buildBr(block.parent_bb); + } + return null; + } + fn genBrVoid(self: *LLVMIRModule, inst: *Inst.BrVoid) !?*const llvm.Value { + var block = self.blocks.get(inst.block).?; + _ = self.builder.buildBr(block.parent_bb); return null; } @@ -591,6 +606,44 @@ pub const LLVMIRModule = struct { return null; } + fn genIsNonNull(self: *LLVMIRModule, inst: *Inst.UnOp, operand_is_ptr: bool) !?*const llvm.Value { + const operand = try self.resolveInst(inst.operand); + + if (operand_is_ptr) { + const index_type = self.context.intType(32); + + var indices: [2]*const llvm.Value = .{ + index_type.constNull(), + index_type.constInt(1, false), + }; + + return self.builder.buildLoad(self.builder.buildInBoundsGEP(operand, &indices, 2, ""), ""); + } else { + return self.builder.buildExtractValue(operand, 1, ""); + } + } + + fn genIsNull(self: *LLVMIRModule, inst: *Inst.UnOp, operand_is_ptr: bool) !?*const llvm.Value { + return self.builder.buildNot((try self.genIsNonNull(inst, operand_is_ptr)).?, ""); + } + + fn genOptionalPayload(self: *LLVMIRModule, inst: *Inst.UnOp, operand_is_ptr: bool) !?*const llvm.Value { + const operand = try self.resolveInst(inst.operand); + + if (operand_is_ptr) { + const index_type = self.context.intType(32); + + var indices: [2]*const llvm.Value = .{ + index_type.constNull(), + index_type.constNull(), + }; + + return self.builder.buildInBoundsGEP(operand, &indices, 2, ""); + } else { + return self.builder.buildExtractValue(operand, 0, ""); + } + } + fn genAdd(self: *LLVMIRModule, inst: *Inst.BinOp) !?*const llvm.Value { const lhs = try self.resolveInst(inst.lhs); const rhs = try self.resolveInst(inst.rhs); @@ -751,6 +804,13 @@ pub const LLVMIRModule = struct { // TODO: consider using buildInBoundsGEP2 for opaque pointers return self.builder.buildInBoundsGEP(val, &indices, 2, ""); }, + .ref_val => { + const elem_value = tv.val.castTag(.ref_val).?.data; + const elem_type = tv.ty.castPointer().?.data; + const alloca = self.buildAlloca(try self.getLLVMType(elem_type, src)); + _ = self.builder.buildStore(try self.genTypedValue(src, .{ .ty = elem_type, .val = elem_value }), alloca); + return alloca; + }, else => return self.fail(src, "TODO implement const of pointer type '{}'", .{tv.ty}), }, .Array => { @@ -765,6 +825,29 @@ pub const LLVMIRModule = struct { return self.fail(src, "TODO handle more array values", .{}); } }, + .Optional => { + if (!tv.ty.isPtrLikeOptional()) { + var buf: Type.Payload.ElemType = undefined; + const child_type = tv.ty.optionalChild(&buf); + const llvm_child_type = try self.getLLVMType(child_type, src); + + if (tv.val.tag() == .null_value) { + var optional_values: [2]*const llvm.Value = .{ + llvm_child_type.constNull(), + self.context.intType(1).constNull(), + }; + return self.context.constStruct(&optional_values, 2, false); + } else { + var optional_values: [2]*const llvm.Value = .{ + try self.genTypedValue(src, .{ .ty = child_type, .val = tv.val }), + self.context.intType(1).constAllOnes(), + }; + return self.context.constStruct(&optional_values, 2, false); + } + } else { + return self.fail(src, "TODO implement const of optional pointer", .{}); + } + }, else => return self.fail(src, "TODO implement const of type '{}'", .{tv.ty}), } } @@ -790,6 +873,20 @@ pub const LLVMIRModule = struct { const elem_type = try self.getLLVMType(t.elemType(), src); return elem_type.arrayType(@intCast(c_uint, t.abiSize(self.module.getTarget()))); }, + .Optional => { + if (!t.isPtrLikeOptional()) { + var buf: Type.Payload.ElemType = undefined; + const child_type = t.optionalChild(&buf); + + var optional_types: [2]*const llvm.Type = .{ + try self.getLLVMType(child_type, src), + self.context.intType(1), + }; + return self.context.structType(&optional_types, 2, false); + } else { + return self.fail(src, "TODO implement optional pointers as actual pointers", .{}); + } + }, else => return self.fail(src, "TODO implement getLLVMType for type '{}'", .{t}), } } diff --git a/src/codegen/llvm/bindings.zig b/src/codegen/llvm/bindings.zig index 6474957b1ce7369f9c4e00e52bb40f4f74166fd9..ccba3d9973745a80e5b33fbee3e38a32d6ce16de 100644 --- a/src/codegen/llvm/bindings.zig +++ b/src/codegen/llvm/bindings.zig @@ -21,9 +21,15 @@ pub const Context = opaque { pub const voidType = LLVMVoidTypeInContext; extern fn LLVMVoidTypeInContext(C: *const Context) *const Type; + pub const structType = LLVMStructTypeInContext; + extern fn LLVMStructTypeInContext(C: *const Context, ElementTypes: [*]*const Type, ElementCount: c_uint, Packed: LLVMBool) *const Type; + pub const constString = LLVMConstStringInContext; extern fn LLVMConstStringInContext(C: *const Context, Str: [*]const u8, Length: c_uint, DontNullTerminate: LLVMBool) *const Value; + pub const constStruct = LLVMConstStructInContext; + extern fn LLVMConstStructInContext(C: *const Context, ConstantVals: [*]*const Value, Count: c_uint, Packed: LLVMBool) *const Value; + pub const createBasicBlock = LLVMCreateBasicBlockInContext; extern fn LLVMCreateBasicBlockInContext(C: *const Context, Name: [*:0]const u8) *const BasicBlock; @@ -204,6 +210,9 @@ pub const Builder = opaque { pub const buildPhi = LLVMBuildPhi; extern fn LLVMBuildPhi(*const Builder, Ty: *const Type, Name: [*:0]const u8) *const Value; + + pub const buildExtractValue = LLVMBuildExtractValue; + extern fn LLVMBuildExtractValue(*const Builder, AggVal: *const Value, Index: c_uint, Name: [*:0]const u8) *const Value; }; pub const IntPredicate = extern enum { diff --git a/test/stage2/llvm.zig b/test/stage2/llvm.zig index f52ccecb68de8daac1717e92a87bad72502f52a2..0888903bebb92f121e8fd3a14c3edf4bf3049a6e 100644 --- a/test/stage2/llvm.zig +++ b/test/stage2/llvm.zig @@ -132,4 +132,44 @@ pub fn addCases(ctx: *TestContext) !void { \\} , ""); } + + { + var case = ctx.exeUsingLlvmBackend("optionals", linux_x64); + + case.addCompareOutput( + \\fn assert(ok: bool) void { + \\ if (!ok) unreachable; + \\} + \\ + \\export fn main() c_int { + \\ var opt_val: ?i32 = 10; + \\ var null_val: ?i32 = null; + \\ + \\ var val1: i32 = opt_val.?; + \\ const val1_1: i32 = opt_val.?; + \\ var ptr_val1 = &(opt_val.?); + \\ const ptr_val1_1 = &(opt_val.?); + \\ + \\ var val2: i32 = null_val orelse 20; + \\ const val2_2: i32 = null_val orelse 20; + \\ + \\ var value: i32 = 20; + \\ var ptr_val2 = &(null_val orelse value); + \\ + \\ const val3 = opt_val orelse 30; + \\ + \\ assert(val1 == 10); + \\ assert(val1_1 == 10); + \\ assert(ptr_val1.* == 10); + \\ assert(ptr_val1_1.* == 10); + \\ + \\ assert(val2 == 20); + \\ assert(val2_2 == 20); + \\ assert(ptr_val2.* == 20); + \\ + \\ assert(val3 == 10); + \\ return 0; + \\} + , ""); + } } -- 2.54.0 From ed6757ece6228f8bf43f9d8e0481f4160b9a884b Mon Sep 17 00:00:00 2001 From: Timon Kruiper Date: Tue, 19 Jan 2021 10:27:16 +0100 Subject: [PATCH 23/35] stage2: add a test for `for` loops in LLVM backend --- test/stage2/llvm.zig | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/stage2/llvm.zig b/test/stage2/llvm.zig index 0888903bebb92f121e8fd3a14c3edf4bf3049a6e..7e672d8a209799e0d5ec61660d6f9c1a0d2c5fd0 100644 --- a/test/stage2/llvm.zig +++ b/test/stage2/llvm.zig @@ -172,4 +172,23 @@ pub fn addCases(ctx: *TestContext) !void { \\} , ""); } + + { + var case = ctx.exeUsingLlvmBackend("for loop", linux_x64); + + case.addCompareOutput( + \\fn assert(ok: bool) void { + \\ if (!ok) unreachable; + \\} + \\ + \\export fn main() c_int { + \\ var x: u32 = 0; + \\ for ("hello") |_| { + \\ x += 1; + \\ } + \\ assert("hello".len == x); + \\ return 0; + \\} + , ""); + } } -- 2.54.0 From 6aa1ea9c59340a1f5b7560ecd97e45928bd4cf56 Mon Sep 17 00:00:00 2001 From: Timon Kruiper Date: Tue, 2 Mar 2021 11:44:33 +0100 Subject: [PATCH 24/35] stage2: fixup some formatting errors ({x} -> {s}) These were missed in cd7c870bd81391dd97c5c75eb3910382ba7280a1 --- src/link.zig | 4 ++-- src/link/Coff.zig | 6 +++--- src/link/Elf.zig | 6 +++--- src/link/MachO.zig | 6 +++--- src/link/Wasm.zig | 6 +++--- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/link.zig b/src/link.zig index 0a4cde0284a329befb24a2ccb38751f28d23337a..db3e973f84f92d2bb38d3215a6734e59294b22a0 100644 --- a/src/link.zig +++ b/src/link.zig @@ -550,11 +550,11 @@ pub const File = struct { id_symlink_basename, &prev_digest_buf, ) catch |err| b: { - log.debug("archive new_digest={x} readFile error: {s}", .{ digest, @errorName(err) }); + log.debug("archive new_digest={s} readFile error: {s}", .{ std.fmt.fmtSliceHexLower(&digest), @errorName(err) }); break :b prev_digest_buf[0..0]; }; if (mem.eql(u8, prev_digest, &digest)) { - log.debug("archive digest={x} match - skipping invocation", .{digest}); + log.debug("archive digest={s} match - skipping invocation", .{std.fmt.fmtSliceHexLower(&digest)}); base.lock = man.toOwnedLock(); return; } diff --git a/src/link/Coff.zig b/src/link/Coff.zig index 2eee19b4f6d40f617e1430549f22af1a18c7f22c..a73b8aaf9c0ad4cee510648e56d579305ca063c4 100644 --- a/src/link/Coff.zig +++ b/src/link/Coff.zig @@ -892,17 +892,17 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void { id_symlink_basename, &prev_digest_buf, ) catch |err| blk: { - log.debug("COFF LLD new_digest={x} error: {s}", .{ digest, @errorName(err) }); + log.debug("COFF LLD new_digest={s} error: {s}", .{ std.fmt.fmtSliceHexLower(&digest), @errorName(err) }); // Handle this as a cache miss. break :blk prev_digest_buf[0..0]; }; if (mem.eql(u8, prev_digest, &digest)) { - log.debug("COFF LLD digest={x} match - skipping invocation", .{digest}); + log.debug("COFF LLD digest={s} match - skipping invocation", .{std.fmt.fmtSliceHexLower(&digest)}); // Hot diggity dog! The output binary is already there. self.base.lock = man.toOwnedLock(); return; } - log.debug("COFF LLD prev_digest={x} new_digest={x}", .{ prev_digest, digest }); + log.debug("COFF LLD prev_digest={s} new_digest={s}", .{ std.fmt.fmtSliceHexLower(prev_digest), std.fmt.fmtSliceHexLower(&digest) }); // We are about to change the output file to be different, so we invalidate the build hash now. directory.handle.deleteFile(id_symlink_basename) catch |err| switch (err) { diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 1b6fbb0f0f11467bf8456916a1515b75d8cc5056..bfef2cd12c080e9f2dec16943d5c474e712c0159 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -1365,17 +1365,17 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { id_symlink_basename, &prev_digest_buf, ) catch |err| blk: { - log.debug("ELF LLD new_digest={x} error: {s}", .{ digest, @errorName(err) }); + log.debug("ELF LLD new_digest={s} error: {s}", .{ std.fmt.fmtSliceHexLower(&digest), @errorName(err) }); // Handle this as a cache miss. break :blk prev_digest_buf[0..0]; }; if (mem.eql(u8, prev_digest, &digest)) { - log.debug("ELF LLD digest={x} match - skipping invocation", .{digest}); + log.debug("ELF LLD digest={s} match - skipping invocation", .{std.fmt.fmtSliceHexLower(&digest)}); // Hot diggity dog! The output binary is already there. self.base.lock = man.toOwnedLock(); return; } - log.debug("ELF LLD prev_digest={x} new_digest={x}", .{ prev_digest, digest }); + log.debug("ELF LLD prev_digest={s} new_digest={s}", .{ std.fmt.fmtSliceHexLower(prev_digest), std.fmt.fmtSliceHexLower(&digest) }); // We are about to change the output file to be different, so we invalidate the build hash now. directory.handle.deleteFile(id_symlink_basename) catch |err| switch (err) { diff --git a/src/link/MachO.zig b/src/link/MachO.zig index 139a9b894007d0aabf8020c1d43ecec7a69806de..0f76925618198eb2c84e222aedb5527d4f4d04cf 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -556,17 +556,17 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void { id_symlink_basename, &prev_digest_buf, ) catch |err| blk: { - log.debug("MachO LLD new_digest={x} error: {s}", .{ digest, @errorName(err) }); + log.debug("MachO LLD new_digest={s} error: {s}", .{ std.fmt.fmtSliceHexLower(&digest), @errorName(err) }); // Handle this as a cache miss. break :blk prev_digest_buf[0..0]; }; if (mem.eql(u8, prev_digest, &digest)) { - log.debug("MachO LLD digest={x} match - skipping invocation", .{digest}); + log.debug("MachO LLD digest={s} match - skipping invocation", .{std.fmt.fmtSliceHexLower(&digest)}); // Hot diggity dog! The output binary is already there. self.base.lock = man.toOwnedLock(); return; } - log.debug("MachO LLD prev_digest={x} new_digest={x}", .{ prev_digest, digest }); + log.debug("MachO LLD prev_digest={s} new_digest={s}", .{ std.fmt.fmtSliceHexLower(prev_digest), std.fmt.fmtSliceHexLower(&digest) }); // We are about to change the output file to be different, so we invalidate the build hash now. directory.handle.deleteFile(id_symlink_basename) catch |err| switch (err) { diff --git a/src/link/Wasm.zig b/src/link/Wasm.zig index e0e10ad88dc014184c5d6cbee35c65fe4aea1926..71cb171d983f490b8cbbdaf6735da78883f6d100 100644 --- a/src/link/Wasm.zig +++ b/src/link/Wasm.zig @@ -391,17 +391,17 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void { id_symlink_basename, &prev_digest_buf, ) catch |err| blk: { - log.debug("WASM LLD new_digest={x} error: {s}", .{ digest, @errorName(err) }); + log.debug("WASM LLD new_digest={s} error: {s}", .{ std.fmt.fmtSliceHexLower(&digest), @errorName(err) }); // Handle this as a cache miss. break :blk prev_digest_buf[0..0]; }; if (mem.eql(u8, prev_digest, &digest)) { - log.debug("WASM LLD digest={x} match - skipping invocation", .{digest}); + log.debug("WASM LLD digest={s} match - skipping invocation", .{std.fmt.fmtSliceHexLower(&digest)}); // Hot diggity dog! The output binary is already there. self.base.lock = man.toOwnedLock(); return; } - log.debug("WASM LLD prev_digest={x} new_digest={x}", .{ prev_digest, digest }); + log.debug("WASM LLD prev_digest={s} new_digest={s}", .{ std.fmt.fmtSliceHexLower(prev_digest), std.fmt.fmtSliceHexLower(&digest) }); // We are about to change the output file to be different, so we invalidate the build hash now. directory.handle.deleteFile(id_symlink_basename) catch |err| switch (err) { -- 2.54.0 From 713f1138222dc40355c34c70d83b0a0805bd46c6 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 2 Mar 2021 21:59:23 -0700 Subject: [PATCH 25/35] stage2: improve orelse implementation * Now it supports being an lvalue (see additional lines in the test case). * Properly handles a pointer result location (see additional lines in the test case that assign the result of the orelse to a variable rather than a const). * Properly sets the result location type when possible, so that type inference of an `orelse` operand expression knows its result type. --- src/astgen.zig | 38 ++++++++++++++++++++++++++++++-------- src/zir.zig | 5 +++++ src/zir_sema.zig | 11 +++++++++++ test/stage2/llvm.zig | 9 +++++++++ 4 files changed, 55 insertions(+), 8 deletions(-) diff --git a/src/astgen.zig b/src/astgen.zig index 2c2f41c3f690bc9c966ce064bfc072229657512d..9f112531ee99d332b92a62edf6f45284957e2745 100644 --- a/src/astgen.zig +++ b/src/astgen.zig @@ -1709,14 +1709,13 @@ fn orelseCatchExpr( setBlockResultLoc(&block_scope, rl); defer block_scope.instructions.deinit(mod.gpa); - // This could be a pointer or value depending on the `rl` parameter. + // This could be a pointer or value depending on the `operand_rl` parameter. + // We cannot use `block_scope.break_result_loc` because that has the bare + // type, whereas this expression has the optional type. Later we make + // up for this fact by calling rvalue on the else branch. block_scope.break_count += 1; - const operand = try expr( - mod, - &block_scope.base, - if (block_scope.break_result_loc == .ref) .ref else .none, - lhs, - ); + const operand_rl = try makeOptionalTypeResultLoc(mod, &block_scope.base, src, block_scope.break_result_loc); + const operand = try expr(mod, &block_scope.base, operand_rl, lhs); const cond = try addZIRUnOp(mod, &block_scope.base, src, cond_op, operand); const condbr = try addZIRInstSpecial(mod, &block_scope.base, src, zir.Inst.CondBr, .{ @@ -1768,6 +1767,10 @@ fn orelseCatchExpr( // This could be a pointer or value depending on `unwrap_op`. const unwrapped_payload = try addZIRUnOp(mod, &else_scope.base, src, unwrap_op, operand); + const else_result = switch (rl) { + .ref => unwrapped_payload, + else => try rvalue(mod, &else_scope.base, block_scope.break_result_loc, unwrapped_payload), + }; return finishThenElseBlock( mod, @@ -1781,7 +1784,7 @@ fn orelseCatchExpr( src, src, then_result, - unwrapped_payload, + else_result, block, block, ); @@ -3970,6 +3973,25 @@ fn rlStrategy(rl: ResultLoc, block_scope: *Scope.GenZIR) ResultLoc.Strategy { } } +/// If the input ResultLoc is ref, returns ResultLoc.ref. Otherwise: +/// Returns ResultLoc.ty, where the type is determined by the input +/// ResultLoc type, wrapped in an optional type. If the input ResultLoc +/// has no type, .none is returned. +fn makeOptionalTypeResultLoc(mod: *Module, scope: *Scope, src: usize, rl: ResultLoc) !ResultLoc { + switch (rl) { + .ref => return ResultLoc.ref, + .discard, .none, .block_ptr, .inferred_ptr, .bitcasted_ptr => return ResultLoc.none, + .ty => |elem_ty| { + const wrapped_ty = try addZIRUnOp(mod, scope, src, .optional_type, elem_ty); + return ResultLoc{ .ty = wrapped_ty }; + }, + .ptr => |ptr_ty| { + const wrapped_ty = try addZIRUnOp(mod, scope, src, .optional_type_from_ptr_elem, ptr_ty); + return ResultLoc{ .ty = wrapped_ty }; + }, + } +} + fn setBlockResultLoc(block_scope: *Scope.GenZIR, parent_rl: ResultLoc) void { // Depending on whether the result location is a pointer or value, different // ZIR needs to be generated. In the former case we rely on storing to the diff --git a/src/zir.zig b/src/zir.zig index 7a3a1e2684f32582b133bb86f27c1a4c4c3eca61..1331f26dc719fceb4c842ee16db4f365c8e813fb 100644 --- a/src/zir.zig +++ b/src/zir.zig @@ -299,6 +299,9 @@ pub const Inst = struct { xor, /// Create an optional type '?T' optional_type, + /// Create an optional type '?T'. The operand is a pointer value. The optional type will + /// be the type of the pointer element, wrapped in an optional. + optional_type_from_ptr_elem, /// Create a union type. union_type, /// ?T => T with safety. @@ -397,6 +400,7 @@ pub const Inst = struct { .mut_slice_type, .const_slice_type, .optional_type, + .optional_type_from_ptr_elem, .optional_payload_safe, .optional_payload_unsafe, .optional_payload_safe_ptr, @@ -597,6 +601,7 @@ pub const Inst = struct { .typeof, .xor, .optional_type, + .optional_type_from_ptr_elem, .optional_payload_safe, .optional_payload_unsafe, .optional_payload_safe_ptr, diff --git a/src/zir_sema.zig b/src/zir_sema.zig index 1cf550852fa74f97f5ba68bb612390cb438a5608..cfc1ea328009fde3d75c1d6c134bc49e6599c0b6 100644 --- a/src/zir_sema.zig +++ b/src/zir_sema.zig @@ -131,6 +131,7 @@ pub fn analyzeInst(mod: *Module, scope: *Scope, old_inst: *zir.Inst) InnerError! .typeof => return zirTypeof(mod, scope, old_inst.castTag(.typeof).?), .typeof_peer => return zirTypeofPeer(mod, scope, old_inst.castTag(.typeof_peer).?), .optional_type => return zirOptionalType(mod, scope, old_inst.castTag(.optional_type).?), + .optional_type_from_ptr_elem => return zirOptionalTypeFromPtrElem(mod, scope, old_inst.castTag(.optional_type_from_ptr_elem).?), .optional_payload_safe => return zirOptionalPayload(mod, scope, old_inst.castTag(.optional_payload_safe).?, true), .optional_payload_unsafe => return zirOptionalPayload(mod, scope, old_inst.castTag(.optional_payload_unsafe).?, false), .optional_payload_safe_ptr => return zirOptionalPayloadPtr(mod, scope, old_inst.castTag(.optional_payload_safe_ptr).?, true), @@ -1093,6 +1094,16 @@ fn zirOptionalType(mod: *Module, scope: *Scope, optional: *zir.Inst.UnOp) InnerE return mod.constType(scope, optional.base.src, try mod.optionalType(scope, child_type)); } +fn zirOptionalTypeFromPtrElem(mod: *Module, scope: *Scope, inst: *zir.Inst.UnOp) InnerError!*Inst { + const tracy = trace(@src()); + defer tracy.end(); + + const ptr = try resolveInst(mod, scope, inst.positionals.operand); + const elem_ty = ptr.ty.elemType(); + + return mod.constType(scope, inst.base.src, try mod.optionalType(scope, elem_ty)); +} + fn zirArrayType(mod: *Module, scope: *Scope, array: *zir.Inst.BinOp) InnerError!*Inst { const tracy = trace(@src()); defer tracy.end(); diff --git a/test/stage2/llvm.zig b/test/stage2/llvm.zig index 7e672d8a209799e0d5ec61660d6f9c1a0d2c5fd0..4b00ed124ca01e0a0b56aed6bf41e65b33694c87 100644 --- a/test/stage2/llvm.zig +++ b/test/stage2/llvm.zig @@ -157,6 +157,7 @@ pub fn addCases(ctx: *TestContext) !void { \\ var ptr_val2 = &(null_val orelse value); \\ \\ const val3 = opt_val orelse 30; + \\ var val3_var = opt_val orelse 30; \\ \\ assert(val1 == 10); \\ assert(val1_1 == 10); @@ -168,6 +169,14 @@ pub fn addCases(ctx: *TestContext) !void { \\ assert(ptr_val2.* == 20); \\ \\ assert(val3 == 10); + \\ assert(val3_var == 10); + \\ + \\ (null_val orelse val2) = 1234; + \\ assert(val2 == 1234); + \\ + \\ (opt_val orelse val2) = 5678; + \\ assert(opt_val.? == 5678); + \\ \\ return 0; \\} , ""); -- 2.54.0 From 8b100792ebaa4fa17dd75fadc7a8bdb47328dfad Mon Sep 17 00:00:00 2001 From: g-w1 Date: Thu, 14 Jan 2021 10:14:29 -0500 Subject: [PATCH 26/35] stage2: error set merging with tests I had to come up with creative tests because we don't have error set type equality yet. --- src/zir_sema.zig | 76 +++++++++++++++++++++++++++++++++++++++++++- test/stage2/test.zig | 35 +++++++++++++++++++- 2 files changed, 109 insertions(+), 2 deletions(-) diff --git a/src/zir_sema.zig b/src/zir_sema.zig index cfc1ea328009fde3d75c1d6c134bc49e6599c0b6..acbe21466b4fdf36cd16068c59c7556b0746db1e 100644 --- a/src/zir_sema.zig +++ b/src/zir_sema.zig @@ -1196,7 +1196,81 @@ fn zirErrorValue(mod: *Module, scope: *Scope, inst: *zir.Inst.ErrorValue) InnerE fn zirMergeErrorSets(mod: *Module, scope: *Scope, inst: *zir.Inst.BinOp) InnerError!*Inst { const tracy = trace(@src()); defer tracy.end(); - return mod.fail(scope, inst.base.src, "TODO implement merge_error_sets", .{}); + + const rhs_ty = try resolveType(mod, scope, inst.positionals.rhs); + const lhs_ty = try resolveType(mod, scope, inst.positionals.lhs); + if (rhs_ty.zigTypeTag() != .ErrorSet) + return mod.fail(scope, inst.positionals.rhs.src, "expected error set type, found {}", .{rhs_ty}); + if (lhs_ty.zigTypeTag() != .ErrorSet) + return mod.fail(scope, inst.positionals.lhs.src, "expected error set type, found {}", .{lhs_ty}); + + // anything merged with anyerror is anyerror + if (lhs_ty.tag() == .anyerror or rhs_ty.tag() == .anyerror) + return mod.constInst(scope, inst.base.src, .{ + .ty = Type.initTag(.type), + .val = Value.initTag(.anyerror_type), + }); + // The declarations arena will store the hashmap. + var new_decl_arena = std.heap.ArenaAllocator.init(mod.gpa); + errdefer new_decl_arena.deinit(); + + const payload = try new_decl_arena.allocator.create(Value.Payload.ErrorSet); + payload.* = .{ + .base = .{ .tag = .error_set }, + .data = .{ + .fields = .{}, + .decl = undefined, // populated below + }, + }; + try payload.data.fields.ensureCapacity(&new_decl_arena.allocator, @intCast(u32, switch (rhs_ty.tag()) { + .error_set_single => 1, + .error_set => rhs_ty.castTag(.error_set).?.data.typed_value.most_recent.typed_value.val.castTag(.error_set).?.data.fields.size, + else => unreachable, + } + switch (lhs_ty.tag()) { + .error_set_single => 1, + .error_set => lhs_ty.castTag(.error_set).?.data.typed_value.most_recent.typed_value.val.castTag(.error_set).?.data.fields.size, + else => unreachable, + })); + + switch (lhs_ty.tag()) { + .error_set_single => { + const name = lhs_ty.castTag(.error_set_single).?.data; + const num = mod.global_error_set.get(name).?; + payload.data.fields.putAssumeCapacity(name, num); + }, + .error_set => { + var multiple = lhs_ty.castTag(.error_set).?.data.typed_value.most_recent.typed_value.val.castTag(.error_set).?.data.fields; + var it = multiple.iterator(); + while (it.next()) |entry| { + payload.data.fields.putAssumeCapacity(entry.key, entry.value); + } + }, + else => unreachable, + } + + switch (rhs_ty.tag()) { + .error_set_single => { + const name = rhs_ty.castTag(.error_set_single).?.data; + const num = mod.global_error_set.get(name).?; + payload.data.fields.putAssumeCapacity(name, num); + }, + .error_set => { + var multiple = rhs_ty.castTag(.error_set).?.data.typed_value.most_recent.typed_value.val.castTag(.error_set).?.data.fields; + var it = multiple.iterator(); + while (it.next()) |name| { + const entry = try mod.getErrorValue(name.key); + payload.data.fields.putAssumeCapacity(entry.key, entry.value); + } + }, + else => unreachable, + } + const new_decl = try mod.createAnonymousDecl(scope, &new_decl_arena, .{ + .ty = Type.initTag(.type), + .val = Value.initPayload(&payload.base), + }); + payload.data.decl = new_decl; + + return mod.analyzeDeclVal(scope, inst.base.src, new_decl); } fn zirEnumLiteral(mod: *Module, scope: *Scope, inst: *zir.Inst.EnumLiteral) InnerError!*Inst { diff --git a/test/stage2/test.zig b/test/stage2/test.zig index 9bd5655d22ad9f4f5a59b41789549b114eb953be..d475f5dff021ddbdd159877e0cdf7018ed834c4f 100644 --- a/test/stage2/test.zig +++ b/test/stage2/test.zig @@ -985,7 +985,7 @@ pub fn addCases(ctx: *TestContext) !void { "Hello, World!\n", ); try case.files.append(.{ - .src = + .src = \\pub fn print() void { \\ asm volatile ("syscall" \\ : @@ -1525,4 +1525,37 @@ pub fn addCases(ctx: *TestContext) !void { \\} , ""); } + { + var case = ctx.exe("merge error sets", linux_x64); + + case.addCompareOutput( + \\export fn _start() noreturn { + \\ const E = error{ A, B, D } || error { A, B, C }; + \\ const a = E.A; + \\ const b = E.B; + \\ const c = E.C; + \\ const d = E.D; + \\ const E2 = error { X, Y } || @TypeOf(error.Z); + \\ const x = E2.X; + \\ const y = E2.Y; + \\ const z = E2.Z; + \\ assert(anyerror || error { Z } == anyerror); + \\ exit(); + \\} + \\fn assert(b: bool) void { + \\ if (!b) unreachable; + \\} + \\fn exit() noreturn { + \\ asm volatile ("syscall" + \\ : + \\ : [number] "{rax}" (231), + \\ [arg1] "{rdi}" (0) + \\ : "rcx", "r11", "memory" + \\ ); + \\ unreachable; + \\} + , + "", + ); + } } -- 2.54.0 From ad3f7e0cf0d4176bfc60b15b47562bd1a8adb666 Mon Sep 17 00:00:00 2001 From: Veikka Tuominen Date: Wed, 3 Feb 2021 18:51:56 +0200 Subject: [PATCH 27/35] fix small inconsistency --- src/zir_sema.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/zir_sema.zig b/src/zir_sema.zig index acbe21466b4fdf36cd16068c59c7556b0746db1e..c2e01bcaed304f38e9ed06bfa4bc7f480640bedb 100644 --- a/src/zir_sema.zig +++ b/src/zir_sema.zig @@ -1257,13 +1257,13 @@ fn zirMergeErrorSets(mod: *Module, scope: *Scope, inst: *zir.Inst.BinOp) InnerEr .error_set => { var multiple = rhs_ty.castTag(.error_set).?.data.typed_value.most_recent.typed_value.val.castTag(.error_set).?.data.fields; var it = multiple.iterator(); - while (it.next()) |name| { - const entry = try mod.getErrorValue(name.key); + while (it.next()) |entry| { payload.data.fields.putAssumeCapacity(entry.key, entry.value); } }, else => unreachable, } + // TODO create name in format "error:line:column" const new_decl = try mod.createAnonymousDecl(scope, &new_decl_arena, .{ .ty = Type.initTag(.type), .val = Value.initPayload(&payload.base), -- 2.54.0 From 39957832071cd999d6abc16c91e12c4820a2a738 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 3 Mar 2021 00:17:36 -0700 Subject: [PATCH 28/35] astgen: fix crash looking for wrong token in error sets Fixes a regression from #7920. --- src/astgen.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/astgen.zig b/src/astgen.zig index 9f112531ee99d332b92a62edf6f45284957e2745..aaf38ed1ea771d9ff25604ac77e726ac9ea73be9 100644 --- a/src/astgen.zig +++ b/src/astgen.zig @@ -1655,7 +1655,7 @@ fn errorSetDecl( switch (token_tags[tok_i]) { .doc_comment, .comma => {}, .identifier => count += 1, - .r_paren => break :count count, + .r_brace => break :count count, else => unreachable, } } else unreachable; // TODO should not need else unreachable here @@ -1672,7 +1672,7 @@ fn errorSetDecl( fields[field_i] = try mod.identifierTokenString(scope, tok_i); field_i += 1; }, - .r_paren => break, + .r_brace => break, else => unreachable, } } -- 2.54.0 From 7fbe9e7d60fa17e49177f52df922bbf295ecc619 Mon Sep 17 00:00:00 2001 From: Josh Wolfe Date: Wed, 3 Mar 2021 14:30:46 -0500 Subject: [PATCH 29/35] update docs and grammar to allow CRLF line endings (#8063) --- doc/langref.html.in | 74 +++++++++++++++++++++++++++++++-------------- 1 file changed, 51 insertions(+), 23 deletions(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index a7163360153545a11f909c8393fc1d5825931abb..645c03dcbbca3eec0c9a32ac142f739153ebfc55 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -10447,13 +10447,40 @@ fn readU32Be() u32 {} {#header_close#} {#header_open|Source Encoding#}

Zig source code is encoded in UTF-8. An invalid UTF-8 byte sequence results in a compile error.

-

Throughout all zig source code (including in comments), some codepoints are never allowed:

+

Throughout all zig source code (including in comments), some code points are never allowed:

    -
  • Ascii control characters, except for U+000a (LF): U+0000 - U+0009, U+000b - U+0001f, U+007f. (Note that Windows line endings (CRLF) are not allowed, and hard tabs are not allowed.)
    • +
  • Ascii control characters, except for U+000a (LF), U+000d (CR), and U+0009 (HT): U+0000 - U+0008, U+000b - U+000c, U+000e - U+0001f, U+007f.
  • Non-Ascii Unicode line endings: U+0085 (NEL), U+2028 (LS), U+2029 (PS).
-

The codepoint U+000a (LF) (which is encoded as the single-byte value 0x0a) is the line terminator character. This character always terminates a line of zig source code (except possibly the last line of the file).

-

For some discussion on the rationale behind these design decisions, see issue #663

+

+ LF (byte value 0x0a, code point U+000a, {#syntax#}'\n'{#endsyntax#}) is the line terminator in Zig source code. + This byte value terminates every line of zig source code except the last line of the file. + It is recommended that non-empty source files end with an empty line, which means the last byte would be 0x0a (LF). +

+

+ Each LF may be immediately preceded by a single CR (byte value 0x0d, code point U+000d, {#syntax#}'\r'{#endsyntax#}) + to form a Windows style line ending, but this is discouraged. + A CR in any other context is not allowed. +

+

+ HT hard tabs (byte value 0x09, code point U+0009, {#syntax#}'\t'{#endsyntax#}) are interchangeable with + SP spaces (byte value 0x20, code point U+0020, {#syntax#}' '{#endsyntax#}) as a token separator, + but use of hard tabs is discouraged. See {#link|Grammar#}. +

+

+ Note that running zig fmt on a source file will implement all recommendations mentioned here. + Note also that the stage1 compiler does not yet support CR or HT control characters. +

+

+ Note that a tool reading Zig source code can make assumptions if the source code is assumed to be correct Zig code. + For example, when identifying the ends of lines, a tool can use a naive search such as /\n/, + or an advanced + search such as /\r\n?|[\n\u0085\u2028\u2029]/, and in either case line endings will be correctly identified. + For another example, when identifying the whitespace before the first token on a line, + a tool can either use a naive search such as /[ \t]/, + or an advanced search such as /\s/, + and in either case whitespace will be correctly identified. +

{#header_close#} {#header_open|Keyword Reference#} @@ -11373,6 +11400,7 @@ ExprList <- (Expr COMMA)* Expr? # *** Tokens *** eof <- !. +eol <- ('\r'? '\n') | eof hex <- [0-9a-fA-F] hex_ <- ('_'/hex) dec <- [0-9] @@ -11382,39 +11410,39 @@ dec_int <- dec (dec_* dec)? hex_int <- hex (hex_* dec)? char_escape - <- "\\x" hex hex - / "\\u{" hex+ "}" - / "\\" [nr\\t'"] + <- '\\x' hex hex + / '\\u{' hex+ '}' + / '\\' [nr\\t'"] char_char <- char_escape - / [^\\'\n] + / [^\\'\r\n] string_char <- char_escape - / [^\\"\n] + / [^\\"\r\n] -line_comment <- '//'[^\n]* -line_string <- ("\\\\" [^\n]* [ \n]*)+ -skip <- ([ \n] / line_comment)* +line_comment <- '//'[^\r\n]* eol +line_string <- ('\\\\' [^\r\n]* eol skip)+ +skip <- ([ \t] / eol / line_comment)* CHAR_LITERAL <- "'" char_char "'" skip FLOAT - <- "0x" hex_* hex "." hex_int ([pP] [-+]? hex_int)? skip - / dec_int "." dec_int ([eE] [-+]? dec_int)? skip - / "0x" hex_* hex "."? [pP] [-+]? hex_int skip - / dec_int "."? [eE] [-+]? dec_int skip + <- '0x' hex_* hex '.' hex_int ([pP] [-+]? hex_int)? skip + / dec_int '.' dec_int ([eE] [-+]? dec_int)? skip + / '0x' hex_* hex '.'? [pP] [-+]? hex_int skip + / dec_int '.'? [eE] [-+]? dec_int skip INTEGER - <- "0b" [_01]* [01] skip - / "0o" [_0-7]* [0-7] skip - / "0x" hex_* hex skip + <- '0b' [_01]* [01] skip + / '0o' [_0-7]* [0-7] skip + / '0x' hex_* hex skip / dec_int skip -STRINGLITERALSINGLE <- "\"" string_char* "\"" skip +STRINGLITERALSINGLE <- '"' string_char* '"' skip STRINGLITERAL <- STRINGLITERALSINGLE - / line_string skip + / line_string skip IDENTIFIER <- !keyword [A-Za-z_] [A-Za-z0-9_]* skip - / "@\"" string_char* "\"" skip -BUILTINIDENTIFIER <- "@"[A-Za-z_][A-Za-z0-9_]* skip + / '@"' string_char* '"' skip +BUILTINIDENTIFIER <- '@'[A-Za-z_][A-Za-z0-9_]* skip AMPERSAND <- '&' ![=] skip -- 2.54.0 From 904f774563c69feae9f73a8ee9637e3b760ecd3a Mon Sep 17 00:00:00 2001 From: Veikka Tuominen Date: Wed, 3 Mar 2021 11:58:09 +0200 Subject: [PATCH 30/35] translate-c: fix c tokenizer giving invalid tokens --- lib/std/c/tokenizer.zig | 32 ++++++++++++++++++++++++++++++-- lib/std/multi_array_list.zig | 2 +- test/run_translated_c.zig | 2 ++ 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/lib/std/c/tokenizer.zig b/lib/std/c/tokenizer.zig index 2e1969e269911fe69a8862a0981c257f3ff3b5a8..4399d3dc6c6df796d1af5cd1f67383c01f4fe822 100644 --- a/lib/std/c/tokenizer.zig +++ b/lib/std/c/tokenizer.zig @@ -401,7 +401,9 @@ pub const Tokenizer = struct { Zero, IntegerLiteralOct, IntegerLiteralBinary, + IntegerLiteralBinaryFirst, IntegerLiteralHex, + IntegerLiteralHexFirst, IntegerLiteral, IntegerSuffix, IntegerSuffixU, @@ -1046,10 +1048,10 @@ pub const Tokenizer = struct { state = .IntegerLiteralOct; }, 'b', 'B' => { - state = .IntegerLiteralBinary; + state = .IntegerLiteralBinaryFirst; }, 'x', 'X' => { - state = .IntegerLiteralHex; + state = .IntegerLiteralHexFirst; }, '.' => { state = .FloatFraction; @@ -1066,6 +1068,13 @@ pub const Tokenizer = struct { self.index -= 1; }, }, + .IntegerLiteralBinaryFirst => switch (c) { + '0'...'7' => state = .IntegerLiteralBinary, + else => { + result.id = .Invalid; + break; + }, + }, .IntegerLiteralBinary => switch (c) { '0', '1' => {}, else => { @@ -1073,6 +1082,19 @@ pub const Tokenizer = struct { self.index -= 1; }, }, + .IntegerLiteralHexFirst => switch (c) { + '0'...'9', 'a'...'f', 'A'...'F' => state = .IntegerLiteralHex, + '.' => { + state = .FloatFractionHex; + }, + 'p', 'P' => { + state = .FloatExponent; + }, + else => { + result.id = .Invalid; + break; + }, + }, .IntegerLiteralHex => switch (c) { '0'...'9', 'a'...'f', 'A'...'F' => {}, '.' => { @@ -1238,6 +1260,8 @@ pub const Tokenizer = struct { .MultiLineCommentAsterisk, .FloatExponent, .MacroString, + .IntegerLiteralBinaryFirst, + .IntegerLiteralHexFirst, => result.id = .Invalid, .FloatExponentDigits => result.id = if (counter == 0) .Invalid else .{ .FloatLiteral = .none }, @@ -1523,6 +1547,7 @@ test "num suffixes" { \\ 1.0f 1.0L 1.0 .0 1. \\ 0l 0lu 0ll 0llu 0 \\ 1u 1ul 1ull 1 + \\ 0x 0b \\ , &[_]Token.Id{ .{ .FloatLiteral = .f }, @@ -1542,6 +1567,9 @@ test "num suffixes" { .{ .IntegerLiteral = .llu }, .{ .IntegerLiteral = .none }, .Nl, + .Invalid, + .Invalid, + .Nl, }); } diff --git a/lib/std/multi_array_list.zig b/lib/std/multi_array_list.zig index 3306fd3ef0c2bc22a8507da688b639a7a2473bdd..99a9fff7f018316643239cc3e1e2f7d3b5470e01 100644 --- a/lib/std/multi_array_list.zig +++ b/lib/std/multi_array_list.zig @@ -136,7 +136,7 @@ pub fn MultiArrayList(comptime S: type) type { const slices = self.slice(); var result: S = undefined; inline for (fields) |field_info, i| { - @field(elem, field_info.name) = slices.items(@intToEnum(Field, i))[index]; + @field(result, field_info.name) = slices.items(@intToEnum(Field, i))[index]; } return result; } diff --git a/test/run_translated_c.zig b/test/run_translated_c.zig index 4eb0f1e50a74b0cdd207a3cae858270588ea34e2..07e449733f83a2459d415cce60ca0a88cd861642 100644 --- a/test/run_translated_c.zig +++ b/test/run_translated_c.zig @@ -27,6 +27,8 @@ pub fn addCases(cases: *tests.RunTranslatedCContext) void { \\#define FOO = \\#define PtrToPtr64(p) ((void *POINTER_64) p) \\#define STRUC_ALIGNED_STACK_COPY(t,s) ((CONST t *)(s)) + \\#define bar = 0x + \\#define baz = 0b \\int main(void) {} , ""); -- 2.54.0 From 2ebeb0dbf30751cec6fd7cc8578a7aac4cb068ce Mon Sep 17 00:00:00 2001 From: jacob gw Date: Mon, 1 Mar 2021 09:21:51 -0500 Subject: [PATCH 31/35] stage2: remove error number from error set map This saves memory since it is already stored in module as well as allowing for better threading. Part 2 of what is outlined in #8079. --- src/Module.zig | 12 ++++++------ src/value.zig | 2 +- src/zir_sema.zig | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Module.zig b/src/Module.zig index 3a72f1272beb905879413f4e812427a17c94f399..e6d509ace5e150e180c30756ebcf21828a9ce225 100644 --- a/src/Module.zig +++ b/src/Module.zig @@ -4083,15 +4083,15 @@ pub fn namedFieldPtr( const child_type = try val.toType(scope.arena()); switch (child_type.zigTypeTag()) { .ErrorSet => { + var name: []const u8 = undefined; // TODO resolve inferred error sets - const entry = if (val.castTag(.error_set)) |payload| - (payload.data.fields.getEntry(field_name) orelse - return mod.fail(scope, src, "no error named '{s}' in '{}'", .{ field_name, child_type })).* + if (val.castTag(.error_set)) |payload| + name = (payload.data.fields.getEntry(field_name) orelse return mod.fail(scope, src, "no error named '{s}' in '{}'", .{ field_name, child_type })).key else - try mod.getErrorValue(field_name); + name = (try mod.getErrorValue(field_name)).key; const result_type = if (child_type.tag() == .anyerror) - try Type.Tag.error_set_single.create(scope.arena(), entry.key) + try Type.Tag.error_set_single.create(scope.arena(), name) else child_type; @@ -4100,7 +4100,7 @@ pub fn namedFieldPtr( .val = try Value.Tag.ref_val.create( scope.arena(), try Value.Tag.@"error".create(scope.arena(), .{ - .name = entry.key, + .name = name, }), ), }); diff --git a/src/value.zig b/src/value.zig index ae94e4b424dacce767c818e1d8117263e47104c2..3fd2889fc86ac6a96bc461f73e8d56ed7fb338a2 100644 --- a/src/value.zig +++ b/src/value.zig @@ -2144,7 +2144,7 @@ pub const Value = extern union { base: Payload = .{ .tag = base_tag }, data: struct { /// TODO revisit this when we have the concept of the error tag type - fields: std.StringHashMapUnmanaged(u16), + fields: std.StringHashMapUnmanaged(void), decl: *Module.Decl, }, }; diff --git a/src/zir_sema.zig b/src/zir_sema.zig index c2e01bcaed304f38e9ed06bfa4bc7f480640bedb..6219f10f89b118291795e770b0016b4ccdb675ce 100644 --- a/src/zir_sema.zig +++ b/src/zir_sema.zig @@ -1165,7 +1165,7 @@ fn zirErrorSet(mod: *Module, scope: *Scope, inst: *zir.Inst.ErrorSet) InnerError for (inst.positionals.fields) |field_name| { const entry = try mod.getErrorValue(field_name); - if (payload.data.fields.fetchPutAssumeCapacity(entry.key, entry.value)) |prev| { + if (payload.data.fields.fetchPutAssumeCapacity(entry.key, {})) |_| { return mod.fail(scope, inst.base.src, "duplicate error: '{s}'", .{field_name}); } } -- 2.54.0 From fcd25065ef41ac93e10031ebea0ed2bc0effce17 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 3 Mar 2021 13:18:13 -0700 Subject: [PATCH 32/35] stage2: fix merge conflict with previous commit The compiler failed to build from source; this fixes it. --- src/zir_sema.zig | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/zir_sema.zig b/src/zir_sema.zig index 6219f10f89b118291795e770b0016b4ccdb675ce..27e31c619789ec22b204be83cf328b2c3c4b0f9e 100644 --- a/src/zir_sema.zig +++ b/src/zir_sema.zig @@ -1235,8 +1235,7 @@ fn zirMergeErrorSets(mod: *Module, scope: *Scope, inst: *zir.Inst.BinOp) InnerEr switch (lhs_ty.tag()) { .error_set_single => { const name = lhs_ty.castTag(.error_set_single).?.data; - const num = mod.global_error_set.get(name).?; - payload.data.fields.putAssumeCapacity(name, num); + payload.data.fields.putAssumeCapacity(name, {}); }, .error_set => { var multiple = lhs_ty.castTag(.error_set).?.data.typed_value.most_recent.typed_value.val.castTag(.error_set).?.data.fields; @@ -1251,8 +1250,7 @@ fn zirMergeErrorSets(mod: *Module, scope: *Scope, inst: *zir.Inst.BinOp) InnerEr switch (rhs_ty.tag()) { .error_set_single => { const name = rhs_ty.castTag(.error_set_single).?.data; - const num = mod.global_error_set.get(name).?; - payload.data.fields.putAssumeCapacity(name, num); + payload.data.fields.putAssumeCapacity(name, {}); }, .error_set => { var multiple = rhs_ty.castTag(.error_set).?.data.typed_value.most_recent.typed_value.val.castTag(.error_set).?.data.fields; -- 2.54.0 From 34ca6b7b449f065fd0f65f8cfad82140cafe4c7d Mon Sep 17 00:00:00 2001 From: daurnimator Date: Mon, 1 Mar 2021 21:47:11 +1100 Subject: [PATCH 33/35] std: add io.Writer.writeStruct We have readStruct, add writeStruct for symmetry --- lib/std/io/writer.zig | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/std/io/writer.zig b/lib/std/io/writer.zig index 0a9edb425aa59a0a56d7782a7b497ac24b4b1283..6f9386b8de137daa77a273cad7d7bf1a0ca38e47 100644 --- a/lib/std/io/writer.zig +++ b/lib/std/io/writer.zig @@ -4,6 +4,7 @@ // The MIT license requires this copyright notice to be included in all copies // and substantial portions of the software. const std = @import("../std.zig"); +const assert = std.debug.assert; const builtin = std.builtin; const mem = std.mem; @@ -86,5 +87,11 @@ pub fn Writer( mem.writeInt(T, &bytes, value, endian); return self.writeAll(&bytes); } + + pub fn writeStruct(self: Self, value: anytype) Error!void { + // Only extern and packed structs have defined in-memory layout. + comptime assert(@typeInfo(@TypeOf(value)).Struct.layout != builtin.TypeInfo.ContainerLayout.Auto); + return self.writeAll(mem.asBytes(&value)); + } }; } -- 2.54.0 From 98b24aa47f98d9c70d1e152228ac89dd41e0f637 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 4 Mar 2021 17:37:53 -0700 Subject: [PATCH 34/35] stage2: support environment variables for verbose options The presence of ZIG_VERBOSE_LINK now enables --verbose-link. The presence of ZIG_VERBOSE_CC now enables --verbose-cc. These are useful when debugging usage of `zig cc` which does not have CLI flags for these options, since they are not valid C compiler flags. --- src/main.zig | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/main.zig b/src/main.zig index bfac976c5cc607479cd84ef7890bb82736a4f5ba..3b71379cb0807e37c615c0b17ed85f2189de6af3 100644 --- a/src/main.zig +++ b/src/main.zig @@ -448,6 +448,15 @@ const Emit = union(enum) { } }; +fn optionalBoolEnvVar(arena: *Allocator, name: []const u8) !bool { + if (std.process.getEnvVarOwned(arena, name)) |value| { + return true; + } else |err| switch (err) { + error.EnvironmentVariableNotFound => return false, + else => |e| return e, + } +} + fn optionalStringEnvVar(arena: *Allocator, name: []const u8) !?[]const u8 { if (std.process.getEnvVarOwned(arena, name)) |value| { return value; @@ -482,8 +491,8 @@ fn buildOutputType( var single_threaded = false; var function_sections = false; var watch = false; - var verbose_link = false; - var verbose_cc = false; + var verbose_link = try optionalBoolEnvVar(arena, "ZIG_VERBOSE_LINK"); + var verbose_cc = try optionalBoolEnvVar(arena, "ZIG_VERBOSE_CC"); var verbose_tokenize = false; var verbose_ast = false; var verbose_ir = false; -- 2.54.0 From 041212a41cfaf029dc3eb9740467b721c76f406c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 4 Mar 2021 18:14:00 -0700 Subject: [PATCH 35/35] zig cc: fix handling of -MM flag Clang docs say: > Like -MMD, but also implies -E and writes to stdout by default. Previously, Zig handled this option by forwarding it directly to Clang, and disabling depfiles. However this did not adhere to Clang's documented behavior of these flags. Now, in addition to being forwarded directly to Clang, `-MM` also sets c_out_mode = .preprocessor, just like `-E`. Another issue I noticed is that Zig did not recognize the aliases for -MG, -MM, or -MMD. The aliases are now recognized. --- src/clang_options_data.zig | 10 +++++----- src/main.zig | 7 +++++++ tools/update_clang_options.zig | 18 +++++++++++++++++- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/clang_options_data.zig b/src/clang_options_data.zig index 0293a5327e96f28f90b182fcedf1ced187c8b10b..b1cb6c258c9b7aa97ceba92c7cbd610008970cca 100644 --- a/src/clang_options_data.zig +++ b/src/clang_options_data.zig @@ -46,7 +46,7 @@ flagpd1("M"), .{ .name = "MM", .syntax = .flag, - .zig_equivalent = .dep_file, + .zig_equivalent = .dep_file_mm, .pd1 = true, .pd2 = false, .psl = false, @@ -1878,7 +1878,7 @@ flagpsl("MT"), .{ .name = "print-missing-file-dependencies", .syntax = .flag, - .zig_equivalent = .other, + .zig_equivalent = .dep_file, .pd1 = false, .pd2 = true, .psl = false, @@ -1998,7 +1998,7 @@ flagpsl("MT"), .{ .name = "user-dependencies", .syntax = .flag, - .zig_equivalent = .other, + .zig_equivalent = .dep_file_mm, .pd1 = false, .pd2 = true, .psl = false, @@ -2022,7 +2022,7 @@ flagpsl("MT"), .{ .name = "write-dependencies", .syntax = .flag, - .zig_equivalent = .other, + .zig_equivalent = .dep_file, .pd1 = false, .pd2 = true, .psl = false, @@ -2030,7 +2030,7 @@ flagpsl("MT"), .{ .name = "write-user-dependencies", .syntax = .flag, - .zig_equivalent = .other, + .zig_equivalent = .dep_file, .pd1 = false, .pd2 = true, .psl = false, diff --git a/src/main.zig b/src/main.zig index 3b71379cb0807e37c615c0b17ed85f2189de6af3..011b9edf760e7b907ed358981b33ab70717c6148 100644 --- a/src/main.zig +++ b/src/main.zig @@ -1192,6 +1192,12 @@ fn buildOutputType( disable_c_depfile = true; try clang_argv.appendSlice(it.other_args); }, + .dep_file_mm => { // -MM + // "Like -MMD, but also implies -E and writes to stdout by default" + c_out_mode = .preprocessor; + disable_c_depfile = true; + try clang_argv.appendSlice(it.other_args); + }, .framework_dir => try framework_dirs.append(it.only_arg), .framework => try frameworks.append(it.only_arg), .nostdlibinc => want_native_include_dirs = false, @@ -3055,6 +3061,7 @@ pub const ClangArgIterator = struct { lib_dir, mcpu, dep_file, + dep_file_mm, framework_dir, framework, nostdlibinc, diff --git a/tools/update_clang_options.zig b/tools/update_clang_options.zig index 1cb134a7317b8e3bb7fa483e841397acb596b3bf..f0dfe1639d207817dd964b493ababad03f113ee9 100644 --- a/tools/update_clang_options.zig +++ b/tools/update_clang_options.zig @@ -268,6 +268,10 @@ const known_options = [_]KnownOpt{ .name = "MD", .ident = "dep_file", }, + .{ + .name = "write-dependencies", + .ident = "dep_file", + }, .{ .name = "MV", .ident = "dep_file", @@ -284,18 +288,30 @@ const known_options = [_]KnownOpt{ .name = "MG", .ident = "dep_file", }, + .{ + .name = "print-missing-file-dependencies", + .ident = "dep_file", + }, .{ .name = "MJ", .ident = "dep_file", }, .{ .name = "MM", - .ident = "dep_file", + .ident = "dep_file_mm", + }, + .{ + .name = "user-dependencies", + .ident = "dep_file_mm", }, .{ .name = "MMD", .ident = "dep_file", }, + .{ + .name = "write-user-dependencies", + .ident = "dep_file", + }, .{ .name = "MP", .ident = "dep_file", -- 2.54.0