authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-10-04 05:05:31-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-10-05 00:25:21-04:00
log906ce2ad7cb3960eb579ab46b79dff3b9eae1ef5
tree3c6ad18765f6b986e6105f3a91691c6f54f74943
parent606c7bcc89dd42be52661fe0a7626803e992308b

InternPool: use sequential string indices instead of byte offsets

This allows more bytes to be referenced by a smaller index range. Closes #22867 Closes #25297 Closes #25339

5 files changed, 102 insertions(+), 63 deletions(-)

src/Compilation.zig+7-2
...@@ -3684,6 +3684,7 @@ const Header = extern struct {...@@ -3684,6 +3684,7 @@ const Header = extern struct {
3684 items_len: u32,3684 items_len: u32,
3685 extra_len: u32,3685 extra_len: u32,
3686 limbs_len: u32,3686 limbs_len: u32,
3687 strings_len: u32,
3687 string_bytes_len: u32,3688 string_bytes_len: u32,
3688 tracked_insts_len: u32,3689 tracked_insts_len: u32,
3689 files_len: u32,3690 files_len: u32,
...@@ -3732,7 +3733,8 @@ pub fn saveState(comp: *Compilation) !void {...@@ -3732,7 +3733,8 @@ pub fn saveState(comp: *Compilation) !void {
3732 .items_len = @intCast(local.mutate.items.len),3733 .items_len = @intCast(local.mutate.items.len),
3733 .extra_len = @intCast(local.mutate.extra.len),3734 .extra_len = @intCast(local.mutate.extra.len),
3734 .limbs_len = @intCast(local.mutate.limbs.len),3735 .limbs_len = @intCast(local.mutate.limbs.len),
3735 .string_bytes_len = @intCast(local.mutate.strings.len),3736 .strings_len = @intCast(local.mutate.strings.len),
3737 .string_bytes_len = @intCast(local.mutate.string_bytes.len),
3736 .tracked_insts_len = @intCast(local.mutate.tracked_insts.len),3738 .tracked_insts_len = @intCast(local.mutate.tracked_insts.len),
3737 .files_len = @intCast(local.mutate.files.len),3739 .files_len = @intCast(local.mutate.files.len),
3738 },3740 },
...@@ -3775,8 +3777,11 @@ pub fn saveState(comp: *Compilation) !void {...@@ -3775,8 +3777,11 @@ pub fn saveState(comp: *Compilation) !void {
3775 addBuf(&bufs, @ptrCast(local.shared.items.view().items(.data)[0..pt_header.intern_pool.items_len]));3777 addBuf(&bufs, @ptrCast(local.shared.items.view().items(.data)[0..pt_header.intern_pool.items_len]));
3776 addBuf(&bufs, @ptrCast(local.shared.items.view().items(.tag)[0..pt_header.intern_pool.items_len]));3778 addBuf(&bufs, @ptrCast(local.shared.items.view().items(.tag)[0..pt_header.intern_pool.items_len]));
3777 }3779 }
3780 if (pt_header.intern_pool.strings_len > 0) {
3781 addBuf(&bufs, @ptrCast(local.shared.strings.view().items(.@"0")[0..pt_header.intern_pool.strings_len]));
3782 }
3778 if (pt_header.intern_pool.string_bytes_len > 0) {3783 if (pt_header.intern_pool.string_bytes_len > 0) {
3779 addBuf(&bufs, local.shared.strings.view().items(.@"0")[0..pt_header.intern_pool.string_bytes_len]);3784 addBuf(&bufs, local.shared.string_bytes.view().items(.@"0")[0..pt_header.intern_pool.string_bytes_len]);
3780 }3785 }
3781 if (pt_header.intern_pool.tracked_insts_len > 0) {3786 if (pt_header.intern_pool.tracked_insts_len > 0) {
3782 addBuf(&bufs, @ptrCast(local.shared.tracked_insts.view().items(.@"0")[0..pt_header.intern_pool.tracked_insts_len]));3787 addBuf(&bufs, @ptrCast(local.shared.tracked_insts.view().items(.@"0")[0..pt_header.intern_pool.tracked_insts_len]));
src/InternPool.zig+82-48
...@@ -196,7 +196,7 @@ pub const TrackedInst = extern struct {...@@ -196,7 +196,7 @@ pub const TrackedInst = extern struct {
196 pub fn wrap(unwrapped: Unwrapped, ip: *const InternPool) TrackedInst.Index {196 pub fn wrap(unwrapped: Unwrapped, ip: *const InternPool) TrackedInst.Index {
197 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());197 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());
198 assert(unwrapped.index <= ip.getIndexMask(u32));198 assert(unwrapped.index <= ip.getIndexMask(u32));
199 return @enumFromInt(@as(u32, @intFromEnum(unwrapped.tid)) << ip.tid_shift_32 |199 return @enumFromInt(@shlExact(@as(u32, @intFromEnum(unwrapped.tid)), ip.tid_shift_32) |
200 unwrapped.index);200 unwrapped.index);
201 }201 }
202 };202 };
...@@ -494,7 +494,7 @@ pub const ComptimeUnit = extern struct {...@@ -494,7 +494,7 @@ pub const ComptimeUnit = extern struct {
494 fn wrap(unwrapped: Unwrapped, ip: *const InternPool) ComptimeUnit.Id {494 fn wrap(unwrapped: Unwrapped, ip: *const InternPool) ComptimeUnit.Id {
495 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());495 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());
496 assert(unwrapped.index <= ip.getIndexMask(u32));496 assert(unwrapped.index <= ip.getIndexMask(u32));
497 return @enumFromInt(@as(u32, @intFromEnum(unwrapped.tid)) << ip.tid_shift_32 |497 return @enumFromInt(@shlExact(@as(u32, @intFromEnum(unwrapped.tid)), ip.tid_shift_32) |
498 unwrapped.index);498 unwrapped.index);
499 }499 }
500 };500 };
...@@ -713,7 +713,7 @@ pub const Nav = struct {...@@ -713,7 +713,7 @@ pub const Nav = struct {
713 fn wrap(unwrapped: Unwrapped, ip: *const InternPool) Nav.Index {713 fn wrap(unwrapped: Unwrapped, ip: *const InternPool) Nav.Index {
714 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());714 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());
715 assert(unwrapped.index <= ip.getIndexMask(u32));715 assert(unwrapped.index <= ip.getIndexMask(u32));
716 return @enumFromInt(@as(u32, @intFromEnum(unwrapped.tid)) << ip.tid_shift_32 |716 return @enumFromInt(@shlExact(@as(u32, @intFromEnum(unwrapped.tid)), ip.tid_shift_32) |
717 unwrapped.index);717 unwrapped.index);
718 }718 }
719 };719 };
...@@ -1061,6 +1061,7 @@ const Local = struct {...@@ -1061,6 +1061,7 @@ const Local = struct {
1061 extra: ListMutate,1061 extra: ListMutate,
1062 limbs: ListMutate,1062 limbs: ListMutate,
1063 strings: ListMutate,1063 strings: ListMutate,
1064 string_bytes: ListMutate,
1064 tracked_insts: ListMutate,1065 tracked_insts: ListMutate,
1065 files: ListMutate,1066 files: ListMutate,
1066 maps: ListMutate,1067 maps: ListMutate,
...@@ -1075,6 +1076,7 @@ const Local = struct {...@@ -1075,6 +1076,7 @@ const Local = struct {
1075 extra: Extra,1076 extra: Extra,
1076 limbs: Limbs,1077 limbs: Limbs,
1077 strings: Strings,1078 strings: Strings,
1079 string_bytes: StringBytes,
1078 tracked_insts: TrackedInsts,1080 tracked_insts: TrackedInsts,
1079 files: List(File),1081 files: List(File),
1080 maps: Maps,1082 maps: Maps,
...@@ -1098,7 +1100,8 @@ const Local = struct {...@@ -1098,7 +1100,8 @@ const Local = struct {
1098 @sizeOf(u64) => List(struct { u64 }),1100 @sizeOf(u64) => List(struct { u64 }),
1099 else => @compileError("unsupported host"),1101 else => @compileError("unsupported host"),
1100 };1102 };
1101 const Strings = List(struct { u8 });1103 const Strings = List(struct { u32 });
1104 const StringBytes = List(struct { u8 });
1102 const TrackedInsts = List(struct { TrackedInst.MaybeLost });1105 const TrackedInsts = List(struct { TrackedInst.MaybeLost });
1103 const Maps = List(struct { FieldMap });1106 const Maps = List(struct { FieldMap });
1104 const Navs = List(Nav.Repr);1107 const Navs = List(Nav.Repr);
...@@ -1428,17 +1431,27 @@ const Local = struct {...@@ -1428,17 +1431,27 @@ const Local = struct {
1428 };1431 };
1429 }1432 }
14301433
1434 /// A list of offsets into `string_bytes` for each string.
1435 pub fn getMutableStrings(local: *Local, gpa: Allocator) Strings.Mutable {
1436 return .{
1437 .gpa = gpa,
1438 .arena = &local.mutate.arena,
1439 .mutate = &local.mutate.strings,
1440 .list = &local.shared.strings,
1441 };
1442 }
1443
1431 /// In order to store references to strings in fewer bytes, we copy all1444 /// In order to store references to strings in fewer bytes, we copy all
1432 /// string bytes into here. String bytes can be null. It is up to whomever1445 /// string bytes into here. String bytes can be null. It is up to whomever
1433 /// is referencing the data here whether they want to store both index and length,1446 /// is referencing the data here whether they want to store both index and length,
1434 /// thus allowing null bytes, or store only index, and use null-termination. The1447 /// thus allowing null bytes, or store only index, and use null-termination. The
1435 /// `strings` array is agnostic to either usage.1448 /// `strings_bytes` array is agnostic to either usage.
1436 pub fn getMutableStrings(local: *Local, gpa: Allocator) Strings.Mutable {1449 pub fn getMutableStringBytes(local: *Local, gpa: Allocator) StringBytes.Mutable {
1437 return .{1450 return .{
1438 .gpa = gpa,1451 .gpa = gpa,
1439 .arena = &local.mutate.arena,1452 .arena = &local.mutate.arena,
1440 .mutate = &local.mutate.strings,1453 .mutate = &local.mutate.string_bytes,
1441 .list = &local.shared.strings,1454 .list = &local.shared.string_bytes,
1442 };1455 };
1443 }1456 }
14441457
...@@ -1611,7 +1624,7 @@ const Shard = struct {...@@ -1611,7 +1624,7 @@ const Shard = struct {
1611};1624};
16121625
1613fn getTidMask(ip: *const InternPool) u32 {1626fn getTidMask(ip: *const InternPool) u32 {
1614 return (@as(u32, 1) << ip.tid_width) - 1;1627 return @shlExact(@as(u32, 1), ip.tid_width) - 1;
1615}1628}
16161629
1617fn getIndexMask(ip: *const InternPool, comptime BackingInt: type) u32 {1630fn getIndexMask(ip: *const InternPool, comptime BackingInt: type) u32 {
...@@ -1652,7 +1665,7 @@ pub const MapIndex = enum(u32) {...@@ -1652,7 +1665,7 @@ pub const MapIndex = enum(u32) {
1652 fn wrap(unwrapped: Unwrapped, ip: *const InternPool) MapIndex {1665 fn wrap(unwrapped: Unwrapped, ip: *const InternPool) MapIndex {
1653 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());1666 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());
1654 assert(unwrapped.index <= ip.getIndexMask(u32));1667 assert(unwrapped.index <= ip.getIndexMask(u32));
1655 return @enumFromInt(@as(u32, @intFromEnum(unwrapped.tid)) << ip.tid_shift_32 |1668 return @enumFromInt(@shlExact(@as(u32, @intFromEnum(unwrapped.tid)), ip.tid_shift_32) |
1656 unwrapped.index);1669 unwrapped.index);
1657 }1670 }
1658 };1671 };
...@@ -1678,7 +1691,7 @@ pub const NamespaceIndex = enum(u32) {...@@ -1678,7 +1691,7 @@ pub const NamespaceIndex = enum(u32) {
1678 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());1691 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());
1679 assert(unwrapped.bucket_index <= ip.getIndexMask(u32) >> Local.namespaces_bucket_width);1692 assert(unwrapped.bucket_index <= ip.getIndexMask(u32) >> Local.namespaces_bucket_width);
1680 assert(unwrapped.index <= Local.namespaces_bucket_mask);1693 assert(unwrapped.index <= Local.namespaces_bucket_mask);
1681 return @enumFromInt(@as(u32, @intFromEnum(unwrapped.tid)) << ip.tid_shift_32 |1694 return @enumFromInt(@shlExact(@as(u32, @intFromEnum(unwrapped.tid)), ip.tid_shift_32) |
1682 unwrapped.bucket_index << Local.namespaces_bucket_width |1695 unwrapped.bucket_index << Local.namespaces_bucket_width |
1683 unwrapped.index);1696 unwrapped.index);
1684 }1697 }
...@@ -1721,7 +1734,7 @@ pub const FileIndex = enum(u32) {...@@ -1721,7 +1734,7 @@ pub const FileIndex = enum(u32) {
1721 fn wrap(unwrapped: Unwrapped, ip: *const InternPool) FileIndex {1734 fn wrap(unwrapped: Unwrapped, ip: *const InternPool) FileIndex {
1722 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());1735 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());
1723 assert(unwrapped.index <= ip.getIndexMask(u32));1736 assert(unwrapped.index <= ip.getIndexMask(u32));
1724 return @enumFromInt(@as(u32, @intFromEnum(unwrapped.tid)) << ip.tid_shift_32 |1737 return @enumFromInt(@shlExact(@as(u32, @intFromEnum(unwrapped.tid)), ip.tid_shift_32) |
1725 unwrapped.index);1738 unwrapped.index);
1726 }1739 }
1727 };1740 };
...@@ -1780,7 +1793,8 @@ pub const String = enum(u32) {...@@ -1780,7 +1793,8 @@ pub const String = enum(u32) {
1780 fn wrap(unwrapped: Unwrapped, ip: *const InternPool) String {1793 fn wrap(unwrapped: Unwrapped, ip: *const InternPool) String {
1781 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());1794 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());
1782 assert(unwrapped.index <= ip.getIndexMask(u32));1795 assert(unwrapped.index <= ip.getIndexMask(u32));
1783 return @enumFromInt(@as(u32, @intFromEnum(unwrapped.tid)) << ip.tid_shift_32 | unwrapped.index);1796 return @enumFromInt(@shlExact(@as(u32, @intFromEnum(unwrapped.tid)), ip.tid_shift_32) |
1797 unwrapped.index);
1784 }1798 }
1785 };1799 };
1786 fn unwrap(string: String, ip: *const InternPool) Unwrapped {1800 fn unwrap(string: String, ip: *const InternPool) Unwrapped {
...@@ -1791,9 +1805,11 @@ pub const String = enum(u32) {...@@ -1791,9 +1805,11 @@ pub const String = enum(u32) {
1791 }1805 }
17921806
1793 fn toOverlongSlice(string: String, ip: *const InternPool) []const u8 {1807 fn toOverlongSlice(string: String, ip: *const InternPool) []const u8 {
1794 const unwrapped_string = string.unwrap(ip);1808 const unwrapped = string.unwrap(ip);
1795 const strings = ip.getLocalShared(unwrapped_string.tid).strings.acquire();1809 const local_shared = ip.getLocalShared(unwrapped.tid);
1796 return strings.view().items(.@"0")[unwrapped_string.index..];1810 const strings = local_shared.strings.acquire().view().items(.@"0");
1811 const string_bytes = local_shared.string_bytes.acquire().view().items(.@"0");
1812 return string_bytes[strings[unwrapped.index]..];
1797 }1813 }
17981814
1799 const debug_state = InternPool.debug_state;1815 const debug_state = InternPool.debug_state;
...@@ -1848,12 +1864,18 @@ pub const NullTerminatedString = enum(u32) {...@@ -1848,12 +1864,18 @@ pub const NullTerminatedString = enum(u32) {
1848 }1864 }
18491865
1850 pub fn toSlice(string: NullTerminatedString, ip: *const InternPool) [:0]const u8 {1866 pub fn toSlice(string: NullTerminatedString, ip: *const InternPool) [:0]const u8 {
1851 const overlong_slice = string.toString().toOverlongSlice(ip);1867 const unwrapped = string.toString().unwrap(ip);
1852 return overlong_slice[0..std.mem.indexOfScalar(u8, overlong_slice, 0).? :0];1868 const local_shared = ip.getLocalShared(unwrapped.tid);
1869 const strings = local_shared.strings.acquire().view().items(.@"0");
1870 const string_bytes = local_shared.string_bytes.acquire().view().items(.@"0");
1871 return string_bytes[strings[unwrapped.index] .. strings[unwrapped.index + 1] - 1 :0];
1853 }1872 }
18541873
1855 pub fn length(string: NullTerminatedString, ip: *const InternPool) u32 {1874 pub fn length(string: NullTerminatedString, ip: *const InternPool) u32 {
1856 return @intCast(string.toSlice(ip).len);1875 const unwrapped = string.toString().unwrap(ip);
1876 const local_shared = ip.getLocalShared(unwrapped.tid);
1877 const strings = local_shared.strings.acquire().view().items(.@"0");
1878 return strings[unwrapped.index + 1] - 1 - strings[unwrapped.index];
1857 }1879 }
18581880
1859 pub fn eqlSlice(string: NullTerminatedString, slice: []const u8, ip: *const InternPool) bool {1881 pub fn eqlSlice(string: NullTerminatedString, slice: []const u8, ip: *const InternPool) bool {
...@@ -4767,7 +4789,8 @@ pub const Index = enum(u32) {...@@ -4767,7 +4789,8 @@ pub const Index = enum(u32) {
4767 fn wrap(unwrapped: Unwrapped, ip: *const InternPool) Index {4789 fn wrap(unwrapped: Unwrapped, ip: *const InternPool) Index {
4768 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());4790 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());
4769 assert(unwrapped.index <= ip.getIndexMask(u30));4791 assert(unwrapped.index <= ip.getIndexMask(u30));
4770 return @enumFromInt(@as(u32, @intFromEnum(unwrapped.tid)) << ip.tid_shift_30 | unwrapped.index);4792 return @enumFromInt(@shlExact(@as(u32, @intFromEnum(unwrapped.tid)), ip.tid_shift_30) |
4793 unwrapped.index);
4771 }4794 }
47724795
4773 pub fn getExtra(unwrapped: Unwrapped, ip: *const InternPool) Local.Extra {4796 pub fn getExtra(unwrapped: Unwrapped, ip: *const InternPool) Local.Extra {
...@@ -6795,6 +6818,7 @@ pub fn init(ip: *InternPool, gpa: Allocator, available_threads: usize) !void {...@@ -6795,6 +6818,7 @@ pub fn init(ip: *InternPool, gpa: Allocator, available_threads: usize) !void {
6795 .extra = .empty,6818 .extra = .empty,
6796 .limbs = .empty,6819 .limbs = .empty,
6797 .strings = .empty,6820 .strings = .empty,
6821 .string_bytes = .empty,
6798 .tracked_insts = .empty,6822 .tracked_insts = .empty,
6799 .files = .empty,6823 .files = .empty,
6800 .maps = .empty,6824 .maps = .empty,
...@@ -6810,6 +6834,7 @@ pub fn init(ip: *InternPool, gpa: Allocator, available_threads: usize) !void {...@@ -6810,6 +6834,7 @@ pub fn init(ip: *InternPool, gpa: Allocator, available_threads: usize) !void {
6810 .extra = .empty,6834 .extra = .empty,
6811 .limbs = .empty,6835 .limbs = .empty,
6812 .strings = .empty,6836 .strings = .empty,
6837 .string_bytes = .empty,
6813 .tracked_insts = .empty,6838 .tracked_insts = .empty,
6814 .files = .empty,6839 .files = .empty,
6815 .maps = .empty,6840 .maps = .empty,
...@@ -6819,6 +6844,7 @@ pub fn init(ip: *InternPool, gpa: Allocator, available_threads: usize) !void {...@@ -6819,6 +6844,7 @@ pub fn init(ip: *InternPool, gpa: Allocator, available_threads: usize) !void {
6819 .namespaces = .empty,6844 .namespaces = .empty,
6820 },6845 },
6821 });6846 });
6847 for (ip.locals) |*local| try local.getMutableStrings(gpa).append(.{0});
68226848
6823 ip.tid_width = @intCast(std.math.log2_int_ceil(usize, used_threads));6849 ip.tid_width = @intCast(std.math.log2_int_ceil(usize, used_threads));
6824 ip.tid_shift_30 = if (single_threaded) 0 else 30 - ip.tid_width;6850 ip.tid_shift_30 = if (single_threaded) 0 else 30 - ip.tid_width;
...@@ -8523,30 +8549,30 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All...@@ -8523,30 +8549,30 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All
8523 }8549 }
85248550
8525 if (child == .u8_type) bytes: {8551 if (child == .u8_type) bytes: {
8526 const strings = ip.getLocal(tid).getMutableStrings(gpa);8552 const string_bytes = ip.getLocal(tid).getMutableStringBytes(gpa);
8527 const start = strings.mutate.len;8553 const start = string_bytes.mutate.len;
8528 try strings.ensureUnusedCapacity(@intCast(len_including_sentinel + 1));8554 try string_bytes.ensureUnusedCapacity(@intCast(len_including_sentinel + 1));
8529 try extra.ensureUnusedCapacity(@typeInfo(Bytes).@"struct".fields.len);8555 try extra.ensureUnusedCapacity(@typeInfo(Bytes).@"struct".fields.len);
8530 switch (aggregate.storage) {8556 switch (aggregate.storage) {
8531 .bytes => |bytes| strings.appendSliceAssumeCapacity(.{bytes.toSlice(len, ip)}),8557 .bytes => |bytes| string_bytes.appendSliceAssumeCapacity(.{bytes.toSlice(len, ip)}),
8532 .elems => |elems| for (elems[0..@intCast(len)]) |elem| switch (ip.indexToKey(elem)) {8558 .elems => |elems| for (elems[0..@intCast(len)]) |elem| switch (ip.indexToKey(elem)) {
8533 .undef => {8559 .undef => {
8534 strings.shrinkRetainingCapacity(start);8560 string_bytes.shrinkRetainingCapacity(start);
8535 break :bytes;8561 break :bytes;
8536 },8562 },
8537 .int => |int| strings.appendAssumeCapacity(.{@intCast(int.storage.u64)}),8563 .int => |int| string_bytes.appendAssumeCapacity(.{@intCast(int.storage.u64)}),
8538 else => unreachable,8564 else => unreachable,
8539 },8565 },
8540 .repeated_elem => |elem| switch (ip.indexToKey(elem)) {8566 .repeated_elem => |elem| switch (ip.indexToKey(elem)) {
8541 .undef => break :bytes,8567 .undef => break :bytes,
8542 .int => |int| @memset(8568 .int => |int| @memset(
8543 strings.addManyAsSliceAssumeCapacity(@intCast(len))[0],8569 string_bytes.addManyAsSliceAssumeCapacity(@intCast(len))[0],
8544 @intCast(int.storage.u64),8570 @intCast(int.storage.u64),
8545 ),8571 ),
8546 else => unreachable,8572 else => unreachable,
8547 },8573 },
8548 }8574 }
8549 if (sentinel != .none) strings.appendAssumeCapacity(.{8575 if (sentinel != .none) string_bytes.appendAssumeCapacity(.{
8550 @intCast(ip.indexToKey(sentinel).int.storage.u64),8576 @intCast(ip.indexToKey(sentinel).int.storage.u64),
8551 });8577 });
8552 const string = try ip.getOrPutTrailingString(8578 const string = try ip.getOrPutTrailingString(
...@@ -11762,10 +11788,10 @@ pub fn getOrPutString(...@@ -11762,10 +11788,10 @@ pub fn getOrPutString(
11762 slice: []const u8,11788 slice: []const u8,
11763 comptime embedded_nulls: EmbeddedNulls,11789 comptime embedded_nulls: EmbeddedNulls,
11764) Allocator.Error!embedded_nulls.StringType() {11790) Allocator.Error!embedded_nulls.StringType() {
11765 const strings = ip.getLocal(tid).getMutableStrings(gpa);11791 const string_bytes = ip.getLocal(tid).getMutableStringBytes(gpa);
11766 try strings.ensureUnusedCapacity(slice.len + 1);11792 try string_bytes.ensureUnusedCapacity(slice.len + 1);
11767 strings.appendSliceAssumeCapacity(.{slice});11793 string_bytes.appendSliceAssumeCapacity(.{slice});
11768 strings.appendAssumeCapacity(.{0});11794 string_bytes.appendAssumeCapacity(.{0});
11769 return ip.getOrPutTrailingString(gpa, tid, @intCast(slice.len + 1), embedded_nulls);11795 return ip.getOrPutTrailingString(gpa, tid, @intCast(slice.len + 1), embedded_nulls);
11770}11796}
1177111797
...@@ -11780,8 +11806,8 @@ pub fn getOrPutStringFmt(...@@ -11780,8 +11806,8 @@ pub fn getOrPutStringFmt(
11780 // ensure that references to strings in args do not get invalidated11806 // ensure that references to strings in args do not get invalidated
11781 const format_z = format ++ .{0};11807 const format_z = format ++ .{0};
11782 const len: u32 = @intCast(std.fmt.count(format_z, args));11808 const len: u32 = @intCast(std.fmt.count(format_z, args));
11783 const strings = ip.getLocal(tid).getMutableStrings(gpa);11809 const string_bytes = ip.getLocal(tid).getMutableStringBytes(gpa);
11784 const slice = try strings.addManyAsSlice(len);11810 const slice = try string_bytes.addManyAsSlice(len);
11785 assert((std.fmt.bufPrint(slice[0], format_z, args) catch unreachable).len == len);11811 assert((std.fmt.bufPrint(slice[0], format_z, args) catch unreachable).len == len);
11786 return ip.getOrPutTrailingString(gpa, tid, len, embedded_nulls);11812 return ip.getOrPutTrailingString(gpa, tid, len, embedded_nulls);
11787}11813}
...@@ -11805,21 +11831,27 @@ pub fn getOrPutTrailingString(...@@ -11805,21 +11831,27 @@ pub fn getOrPutTrailingString(
11805 len: u32,11831 len: u32,
11806 comptime embedded_nulls: EmbeddedNulls,11832 comptime embedded_nulls: EmbeddedNulls,
11807) Allocator.Error!embedded_nulls.StringType() {11833) Allocator.Error!embedded_nulls.StringType() {
11808 const strings = ip.getLocal(tid).getMutableStrings(gpa);11834 const local = ip.getLocal(tid);
11809 const start: u32 = @intCast(strings.mutate.len - len);11835 const strings = local.getMutableStrings(gpa);
11810 if (len > 0 and strings.view().items(.@"0")[strings.mutate.len - 1] == 0) {11836 try strings.ensureUnusedCapacity(1);
11811 strings.mutate.len -= 1;11837 const string_bytes = local.getMutableStringBytes(gpa);
11838 const start: u32 = @intCast(string_bytes.mutate.len - len);
11839 if (len > 0 and string_bytes.view().items(.@"0")[string_bytes.mutate.len - 1] == 0) {
11840 string_bytes.mutate.len -= 1;
11812 } else {11841 } else {
11813 try strings.ensureUnusedCapacity(1);11842 try string_bytes.ensureUnusedCapacity(1);
11814 }11843 }
11815 const key: []const u8 = strings.view().items(.@"0")[start..];11844 const key: []const u8 = string_bytes.view().items(.@"0")[start..];
11816 const value: embedded_nulls.StringType() =11845 const value: embedded_nulls.StringType() = @enumFromInt(@intFromEnum((String.Unwrapped{
11817 @enumFromInt(@intFromEnum((String.Unwrapped{ .tid = tid, .index = start }).wrap(ip)));11846 .tid = tid,
11847 .index = strings.mutate.len - 1,
11848 }).wrap(ip)));
11818 const has_embedded_null = std.mem.indexOfScalar(u8, key, 0) != null;11849 const has_embedded_null = std.mem.indexOfScalar(u8, key, 0) != null;
11819 switch (embedded_nulls) {11850 switch (embedded_nulls) {
11820 .no_embedded_nulls => assert(!has_embedded_null),11851 .no_embedded_nulls => assert(!has_embedded_null),
11821 .maybe_embedded_nulls => if (has_embedded_null) {11852 .maybe_embedded_nulls => if (has_embedded_null) {
11822 strings.appendAssumeCapacity(.{0});11853 string_bytes.appendAssumeCapacity(.{0});
11854 strings.appendAssumeCapacity(.{string_bytes.mutate.len});
11823 return value;11855 return value;
11824 },11856 },
11825 }11857 }
...@@ -11837,7 +11869,7 @@ pub fn getOrPutTrailingString(...@@ -11837,7 +11869,7 @@ pub fn getOrPutTrailingString(
11837 const index = entry.acquire().unwrap() orelse break;11869 const index = entry.acquire().unwrap() orelse break;
11838 if (entry.hash != hash) continue;11870 if (entry.hash != hash) continue;
11839 if (!index.eqlSlice(key, ip)) continue;11871 if (!index.eqlSlice(key, ip)) continue;
11840 strings.shrinkRetainingCapacity(start);11872 string_bytes.shrinkRetainingCapacity(start);
11841 return @enumFromInt(@intFromEnum(index));11873 return @enumFromInt(@intFromEnum(index));
11842 }11874 }
11843 shard.mutate.string_map.mutex.lock();11875 shard.mutate.string_map.mutex.lock();
...@@ -11853,19 +11885,20 @@ pub fn getOrPutTrailingString(...@@ -11853,19 +11885,20 @@ pub fn getOrPutTrailingString(
11853 const index = entry.acquire().unwrap() orelse break;11885 const index = entry.acquire().unwrap() orelse break;
11854 if (entry.hash != hash) continue;11886 if (entry.hash != hash) continue;
11855 if (!index.eqlSlice(key, ip)) continue;11887 if (!index.eqlSlice(key, ip)) continue;
11856 strings.shrinkRetainingCapacity(start);11888 string_bytes.shrinkRetainingCapacity(start);
11857 return @enumFromInt(@intFromEnum(index));11889 return @enumFromInt(@intFromEnum(index));
11858 }11890 }
11859 defer shard.mutate.string_map.len += 1;11891 defer shard.mutate.string_map.len += 1;
11860 const map_header = map.header().*;11892 const map_header = map.header().*;
11861 if (shard.mutate.string_map.len < map_header.capacity * 3 / 5) {11893 if (shard.mutate.string_map.len < map_header.capacity * 3 / 5) {
11862 strings.appendAssumeCapacity(.{0});11894 string_bytes.appendAssumeCapacity(.{0});
11895 strings.appendAssumeCapacity(.{string_bytes.mutate.len});
11863 const entry = &map.entries[map_index];11896 const entry = &map.entries[map_index];
11864 entry.hash = hash;11897 entry.hash = hash;
11865 entry.release(@enumFromInt(@intFromEnum(value)));11898 entry.release(@enumFromInt(@intFromEnum(value)));
11866 return value;11899 return value;
11867 }11900 }
11868 const arena_state = &ip.getLocal(tid).mutate.arena;11901 const arena_state = &local.mutate.arena;
11869 var arena = arena_state.promote(gpa);11902 var arena = arena_state.promote(gpa);
11870 defer arena_state.* = arena.state;11903 defer arena_state.* = arena.state;
11871 const new_map_capacity = map_header.capacity * 2;11904 const new_map_capacity = map_header.capacity * 2;
...@@ -11901,7 +11934,8 @@ pub fn getOrPutTrailingString(...@@ -11901,7 +11934,8 @@ pub fn getOrPutTrailingString(
11901 map_index &= new_map_mask;11934 map_index &= new_map_mask;
11902 if (map.entries[map_index].value == .none) break;11935 if (map.entries[map_index].value == .none) break;
11903 }11936 }
11904 strings.appendAssumeCapacity(.{0});11937 string_bytes.appendAssumeCapacity(.{0});
11938 strings.appendAssumeCapacity(.{string_bytes.mutate.len});
11905 map.entries[map_index] = .{11939 map.entries[map_index] = .{
11906 .value = @enumFromInt(@intFromEnum(value)),11940 .value = @enumFromInt(@intFromEnum(value)),
11907 .hash = hash,11941 .hash = hash,
src/Value.zig+7-7
...@@ -66,8 +66,8 @@ pub fn toIpString(val: Value, ty: Type, pt: Zcu.PerThread) !InternPool.NullTermi...@@ -66,8 +66,8 @@ pub fn toIpString(val: Value, ty: Type, pt: Zcu.PerThread) !InternPool.NullTermi
66 .repeated_elem => |elem| {66 .repeated_elem => |elem| {
67 const byte: u8 = @intCast(Value.fromInterned(elem).toUnsignedInt(zcu));67 const byte: u8 = @intCast(Value.fromInterned(elem).toUnsignedInt(zcu));
68 const len: u32 = @intCast(ty.arrayLen(zcu));68 const len: u32 = @intCast(ty.arrayLen(zcu));
69 const strings = ip.getLocal(pt.tid).getMutableStrings(zcu.gpa);69 const string_bytes = ip.getLocal(pt.tid).getMutableStringBytes(zcu.gpa);
70 try strings.appendNTimes(.{byte}, len);70 try string_bytes.appendNTimes(.{byte}, len);
71 return ip.getOrPutTrailingString(zcu.gpa, pt.tid, len, .no_embedded_nulls);71 return ip.getOrPutTrailingString(zcu.gpa, pt.tid, len, .no_embedded_nulls);
72 },72 },
73 }73 }
...@@ -109,16 +109,16 @@ fn arrayToIpString(val: Value, len_u64: u64, pt: Zcu.PerThread) !InternPool.Null...@@ -109,16 +109,16 @@ fn arrayToIpString(val: Value, len_u64: u64, pt: Zcu.PerThread) !InternPool.Null
109 const gpa = zcu.gpa;109 const gpa = zcu.gpa;
110 const ip = &zcu.intern_pool;110 const ip = &zcu.intern_pool;
111 const len: u32 = @intCast(len_u64);111 const len: u32 = @intCast(len_u64);
112 const strings = ip.getLocal(pt.tid).getMutableStrings(gpa);112 const string_bytes = ip.getLocal(pt.tid).getMutableStringBytes(gpa);
113 try strings.ensureUnusedCapacity(len);113 try string_bytes.ensureUnusedCapacity(len);
114 for (0..len) |i| {114 for (0..len) |i| {
115 // I don't think elemValue has the possibility to affect ip.string_bytes. Let's115 // I don't think elemValue has the possibility to affect ip.string_bytes. Let's
116 // assert just to be sure.116 // assert just to be sure.
117 const prev_len = strings.mutate.len;117 const prev_len = string_bytes.mutate.len;
118 const elem_val = try val.elemValue(pt, i);118 const elem_val = try val.elemValue(pt, i);
119 assert(strings.mutate.len == prev_len);119 assert(string_bytes.mutate.len == prev_len);
120 const byte: u8 = @intCast(elem_val.toUnsignedInt(zcu));120 const byte: u8 = @intCast(elem_val.toUnsignedInt(zcu));
121 strings.appendAssumeCapacity(.{byte});121 string_bytes.appendAssumeCapacity(.{byte});
122 }122 }
123 return ip.getOrPutTrailingString(gpa, pt.tid, len, .no_embedded_nulls);123 return ip.getOrPutTrailingString(gpa, pt.tid, len, .no_embedded_nulls);
124}124}
src/Zcu.zig+2-2
...@@ -1115,8 +1115,8 @@ pub const File = struct {...@@ -1115,8 +1115,8 @@ pub const File = struct {
1115 pub fn internFullyQualifiedName(file: File, pt: Zcu.PerThread) !InternPool.NullTerminatedString {1115 pub fn internFullyQualifiedName(file: File, pt: Zcu.PerThread) !InternPool.NullTerminatedString {
1116 const gpa = pt.zcu.gpa;1116 const gpa = pt.zcu.gpa;
1117 const ip = &pt.zcu.intern_pool;1117 const ip = &pt.zcu.intern_pool;
1118 const strings = ip.getLocal(pt.tid).getMutableStrings(gpa);1118 const string_bytes = ip.getLocal(pt.tid).getMutableStringBytes(gpa);
1119 var w: Writer = .fixed((try strings.addManyAsSlice(file.fullyQualifiedNameLen()))[0]);1119 var w: Writer = .fixed((try string_bytes.addManyAsSlice(file.fullyQualifiedNameLen()))[0]);
1120 file.renderFullyQualifiedName(&w) catch unreachable;1120 file.renderFullyQualifiedName(&w) catch unreachable;
1121 assert(w.end == w.buffer.len);1121 assert(w.end == w.buffer.len);
1122 return ip.getOrPutTrailingString(gpa, pt.tid, @intCast(w.end), .no_embedded_nulls);1122 return ip.getOrPutTrailingString(gpa, pt.tid, @intCast(w.end), .no_embedded_nulls);
src/Zcu/PerThread.zig+4-4
...@@ -2460,10 +2460,10 @@ fn updateEmbedFileInner(...@@ -2460,10 +2460,10 @@ fn updateEmbedFileInner(
24602460
2461 // The loaded bytes of the file, including a sentinel 0 byte.2461 // The loaded bytes of the file, including a sentinel 0 byte.
2462 const ip_str: InternPool.String = str: {2462 const ip_str: InternPool.String = str: {
2463 const strings = ip.getLocal(tid).getMutableStrings(gpa);2463 const string_bytes = ip.getLocal(tid).getMutableStringBytes(gpa);
2464 const old_len = strings.mutate.len;2464 const old_len = string_bytes.mutate.len;
2465 errdefer strings.shrinkRetainingCapacity(old_len);2465 errdefer string_bytes.shrinkRetainingCapacity(old_len);
2466 const bytes = (try strings.addManyAsSlice(size_plus_one))[0];2466 const bytes = (try string_bytes.addManyAsSlice(size_plus_one))[0];
2467 var fr = file.reader(&.{});2467 var fr = file.reader(&.{});
2468 fr.size = stat.size;2468 fr.size = stat.size;
2469 fr.interface.readSliceAll(bytes[0..size]) catch |err| switch (err) {2469 fr.interface.readSliceAll(bytes[0..size]) catch |err| switch (err) {