| ... | @@ -196,7 +196,7 @@ pub const TrackedInst = extern struct { | ... | @@ -196,7 +196,7 @@ pub const TrackedInst = extern struct { |
| 196 | pub fn wrap(unwrapped: Unwrapped, ip: *const InternPool) TrackedInst.Index { | 196 | pub fn wrap(unwrapped: Unwrapped, ip: *const InternPool) TrackedInst.Index { |
| 197 | assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask()); | 197 | assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask()); |
| 198 | assert(unwrapped.index <= ip.getIndexMask(u32)); | 198 | assert(unwrapped.index <= ip.getIndexMask(u32)); |
| 199 | return @enumFromInt(@as(u32, @intFromEnum(unwrapped.tid)) << ip.tid_shift_32 | | 199 | return @enumFromInt(@shlExact(@as(u32, @intFromEnum(unwrapped.tid)), ip.tid_shift_32) | |
| 200 | unwrapped.index); | 200 | unwrapped.index); |
| 201 | } | 201 | } |
| 202 | }; | 202 | }; |
| ... | @@ -494,7 +494,7 @@ pub const ComptimeUnit = extern struct { | ... | @@ -494,7 +494,7 @@ pub const ComptimeUnit = extern struct { |
| 494 | fn wrap(unwrapped: Unwrapped, ip: *const InternPool) ComptimeUnit.Id { | 494 | fn wrap(unwrapped: Unwrapped, ip: *const InternPool) ComptimeUnit.Id { |
| 495 | assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask()); | 495 | assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask()); |
| 496 | assert(unwrapped.index <= ip.getIndexMask(u32)); | 496 | assert(unwrapped.index <= ip.getIndexMask(u32)); |
| 497 | return @enumFromInt(@as(u32, @intFromEnum(unwrapped.tid)) << ip.tid_shift_32 | | 497 | return @enumFromInt(@shlExact(@as(u32, @intFromEnum(unwrapped.tid)), ip.tid_shift_32) | |
| 498 | unwrapped.index); | 498 | unwrapped.index); |
| 499 | } | 499 | } |
| 500 | }; | 500 | }; |
| ... | @@ -713,7 +713,7 @@ pub const Nav = struct { | ... | @@ -713,7 +713,7 @@ pub const Nav = struct { |
| 713 | fn wrap(unwrapped: Unwrapped, ip: *const InternPool) Nav.Index { | 713 | fn wrap(unwrapped: Unwrapped, ip: *const InternPool) Nav.Index { |
| 714 | assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask()); | 714 | assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask()); |
| 715 | assert(unwrapped.index <= ip.getIndexMask(u32)); | 715 | assert(unwrapped.index <= ip.getIndexMask(u32)); |
| 716 | return @enumFromInt(@as(u32, @intFromEnum(unwrapped.tid)) << ip.tid_shift_32 | | 716 | return @enumFromInt(@shlExact(@as(u32, @intFromEnum(unwrapped.tid)), ip.tid_shift_32) | |
| 717 | unwrapped.index); | 717 | unwrapped.index); |
| 718 | } | 718 | } |
| 719 | }; | 719 | }; |
| ... | @@ -1061,6 +1061,7 @@ const Local = struct { | ... | @@ -1061,6 +1061,7 @@ const Local = struct { |
| 1061 | extra: ListMutate, | 1061 | extra: ListMutate, |
| 1062 | limbs: ListMutate, | 1062 | limbs: ListMutate, |
| 1063 | strings: ListMutate, | 1063 | strings: ListMutate, |
| | 1064 | string_bytes: ListMutate, |
| 1064 | tracked_insts: ListMutate, | 1065 | tracked_insts: ListMutate, |
| 1065 | files: ListMutate, | 1066 | files: ListMutate, |
| 1066 | maps: ListMutate, | 1067 | maps: ListMutate, |
| ... | @@ -1075,6 +1076,7 @@ const Local = struct { | ... | @@ -1075,6 +1076,7 @@ const Local = struct { |
| 1075 | extra: Extra, | 1076 | extra: Extra, |
| 1076 | limbs: Limbs, | 1077 | limbs: Limbs, |
| 1077 | strings: Strings, | 1078 | strings: Strings, |
| | 1079 | string_bytes: StringBytes, |
| 1078 | tracked_insts: TrackedInsts, | 1080 | tracked_insts: TrackedInsts, |
| 1079 | files: List(File), | 1081 | files: List(File), |
| 1080 | maps: Maps, | 1082 | maps: Maps, |
| ... | @@ -1098,7 +1100,8 @@ const Local = struct { | ... | @@ -1098,7 +1100,8 @@ const Local = struct { |
| 1098 | @sizeOf(u64) => List(struct { u64 }), | 1100 | @sizeOf(u64) => List(struct { u64 }), |
| 1099 | else => @compileError("unsupported host"), | 1101 | else => @compileError("unsupported host"), |
| 1100 | }; | 1102 | }; |
| 1101 | const Strings = List(struct { u8 }); | 1103 | const Strings = List(struct { u32 }); |
| | 1104 | const StringBytes = List(struct { u8 }); |
| 1102 | const TrackedInsts = List(struct { TrackedInst.MaybeLost }); | 1105 | const TrackedInsts = List(struct { TrackedInst.MaybeLost }); |
| 1103 | const Maps = List(struct { FieldMap }); | 1106 | const Maps = List(struct { FieldMap }); |
| 1104 | const Navs = List(Nav.Repr); | 1107 | const Navs = List(Nav.Repr); |
| ... | @@ -1428,17 +1431,27 @@ const Local = struct { | ... | @@ -1428,17 +1431,27 @@ const Local = struct { |
| 1428 | }; | 1431 | }; |
| 1429 | } | 1432 | } |
| 1430 | | 1433 | |
| | 1434 | /// A list of offsets into `string_bytes` for each string. |
| | 1435 | pub fn getMutableStrings(local: *Local, gpa: Allocator) Strings.Mutable { |
| | 1436 | return .{ |
| | 1437 | .gpa = gpa, |
| | 1438 | .arena = &local.mutate.arena, |
| | 1439 | .mutate = &local.mutate.strings, |
| | 1440 | .list = &local.shared.strings, |
| | 1441 | }; |
| | 1442 | } |
| | 1443 | |
| 1431 | /// In order to store references to strings in fewer bytes, we copy all | 1444 | /// In order to store references to strings in fewer bytes, we copy all |
| 1432 | /// string bytes into here. String bytes can be null. It is up to whomever | 1445 | /// string bytes into here. String bytes can be null. It is up to whomever |
| 1433 | /// is referencing the data here whether they want to store both index and length, | 1446 | /// is referencing the data here whether they want to store both index and length, |
| 1434 | /// thus allowing null bytes, or store only index, and use null-termination. The | 1447 | /// thus allowing null bytes, or store only index, and use null-termination. The |
| 1435 | /// `strings` array is agnostic to either usage. | 1448 | /// `strings_bytes` array is agnostic to either usage. |
| 1436 | pub fn getMutableStrings(local: *Local, gpa: Allocator) Strings.Mutable { | 1449 | pub fn getMutableStringBytes(local: *Local, gpa: Allocator) StringBytes.Mutable { |
| 1437 | return .{ | 1450 | return .{ |
| 1438 | .gpa = gpa, | 1451 | .gpa = gpa, |
| 1439 | .arena = &local.mutate.arena, | 1452 | .arena = &local.mutate.arena, |
| 1440 | .mutate = &local.mutate.strings, | 1453 | .mutate = &local.mutate.string_bytes, |
| 1441 | .list = &local.shared.strings, | 1454 | .list = &local.shared.string_bytes, |
| 1442 | }; | 1455 | }; |
| 1443 | } | 1456 | } |
| 1444 | | 1457 | |
| ... | @@ -1611,7 +1624,7 @@ const Shard = struct { | ... | @@ -1611,7 +1624,7 @@ const Shard = struct { |
| 1611 | }; | 1624 | }; |
| 1612 | | 1625 | |
| 1613 | fn getTidMask(ip: *const InternPool) u32 { | 1626 | fn getTidMask(ip: *const InternPool) u32 { |
| 1614 | return (@as(u32, 1) << ip.tid_width) - 1; | 1627 | return @shlExact(@as(u32, 1), ip.tid_width) - 1; |
| 1615 | } | 1628 | } |
| 1616 | | 1629 | |
| 1617 | fn getIndexMask(ip: *const InternPool, comptime BackingInt: type) u32 { | 1630 | fn getIndexMask(ip: *const InternPool, comptime BackingInt: type) u32 { |
| ... | @@ -1652,7 +1665,7 @@ pub const MapIndex = enum(u32) { | ... | @@ -1652,7 +1665,7 @@ pub const MapIndex = enum(u32) { |
| 1652 | fn wrap(unwrapped: Unwrapped, ip: *const InternPool) MapIndex { | 1665 | fn wrap(unwrapped: Unwrapped, ip: *const InternPool) MapIndex { |
| 1653 | assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask()); | 1666 | assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask()); |
| 1654 | assert(unwrapped.index <= ip.getIndexMask(u32)); | 1667 | assert(unwrapped.index <= ip.getIndexMask(u32)); |
| 1655 | return @enumFromInt(@as(u32, @intFromEnum(unwrapped.tid)) << ip.tid_shift_32 | | 1668 | return @enumFromInt(@shlExact(@as(u32, @intFromEnum(unwrapped.tid)), ip.tid_shift_32) | |
| 1656 | unwrapped.index); | 1669 | unwrapped.index); |
| 1657 | } | 1670 | } |
| 1658 | }; | 1671 | }; |
| ... | @@ -1678,7 +1691,7 @@ pub const NamespaceIndex = enum(u32) { | ... | @@ -1678,7 +1691,7 @@ pub const NamespaceIndex = enum(u32) { |
| 1678 | assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask()); | 1691 | assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask()); |
| 1679 | assert(unwrapped.bucket_index <= ip.getIndexMask(u32) >> Local.namespaces_bucket_width); | 1692 | assert(unwrapped.bucket_index <= ip.getIndexMask(u32) >> Local.namespaces_bucket_width); |
| 1680 | assert(unwrapped.index <= Local.namespaces_bucket_mask); | 1693 | assert(unwrapped.index <= Local.namespaces_bucket_mask); |
| 1681 | return @enumFromInt(@as(u32, @intFromEnum(unwrapped.tid)) << ip.tid_shift_32 | | 1694 | return @enumFromInt(@shlExact(@as(u32, @intFromEnum(unwrapped.tid)), ip.tid_shift_32) | |
| 1682 | unwrapped.bucket_index << Local.namespaces_bucket_width | | 1695 | unwrapped.bucket_index << Local.namespaces_bucket_width | |
| 1683 | unwrapped.index); | 1696 | unwrapped.index); |
| 1684 | } | 1697 | } |
| ... | @@ -1721,7 +1734,7 @@ pub const FileIndex = enum(u32) { | ... | @@ -1721,7 +1734,7 @@ pub const FileIndex = enum(u32) { |
| 1721 | fn wrap(unwrapped: Unwrapped, ip: *const InternPool) FileIndex { | 1734 | fn wrap(unwrapped: Unwrapped, ip: *const InternPool) FileIndex { |
| 1722 | assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask()); | 1735 | assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask()); |
| 1723 | assert(unwrapped.index <= ip.getIndexMask(u32)); | 1736 | assert(unwrapped.index <= ip.getIndexMask(u32)); |
| 1724 | return @enumFromInt(@as(u32, @intFromEnum(unwrapped.tid)) << ip.tid_shift_32 | | 1737 | return @enumFromInt(@shlExact(@as(u32, @intFromEnum(unwrapped.tid)), ip.tid_shift_32) | |
| 1725 | unwrapped.index); | 1738 | unwrapped.index); |
| 1726 | } | 1739 | } |
| 1727 | }; | 1740 | }; |
| ... | @@ -1780,7 +1793,8 @@ pub const String = enum(u32) { | ... | @@ -1780,7 +1793,8 @@ pub const String = enum(u32) { |
| 1780 | fn wrap(unwrapped: Unwrapped, ip: *const InternPool) String { | 1793 | fn wrap(unwrapped: Unwrapped, ip: *const InternPool) String { |
| 1781 | assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask()); | 1794 | assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask()); |
| 1782 | assert(unwrapped.index <= ip.getIndexMask(u32)); | 1795 | assert(unwrapped.index <= ip.getIndexMask(u32)); |
| 1783 | return @enumFromInt(@as(u32, @intFromEnum(unwrapped.tid)) << ip.tid_shift_32 | unwrapped.index); | 1796 | return @enumFromInt(@shlExact(@as(u32, @intFromEnum(unwrapped.tid)), ip.tid_shift_32) | |
| | 1797 | unwrapped.index); |
| 1784 | } | 1798 | } |
| 1785 | }; | 1799 | }; |
| 1786 | fn unwrap(string: String, ip: *const InternPool) Unwrapped { | 1800 | fn unwrap(string: String, ip: *const InternPool) Unwrapped { |
| ... | @@ -1791,9 +1805,11 @@ pub const String = enum(u32) { | ... | @@ -1791,9 +1805,11 @@ pub const String = enum(u32) { |
| 1791 | } | 1805 | } |
| 1792 | | 1806 | |
| 1793 | fn toOverlongSlice(string: String, ip: *const InternPool) []const u8 { | 1807 | fn toOverlongSlice(string: String, ip: *const InternPool) []const u8 { |
| 1794 | const unwrapped_string = string.unwrap(ip); | 1808 | const unwrapped = string.unwrap(ip); |
| 1795 | const strings = ip.getLocalShared(unwrapped_string.tid).strings.acquire(); | 1809 | const local_shared = ip.getLocalShared(unwrapped.tid); |
| 1796 | return strings.view().items(.@"0")[unwrapped_string.index..]; | 1810 | const strings = local_shared.strings.acquire().view().items(.@"0"); |
| | 1811 | const string_bytes = local_shared.string_bytes.acquire().view().items(.@"0"); |
| | 1812 | return string_bytes[strings[unwrapped.index]..]; |
| 1797 | } | 1813 | } |
| 1798 | | 1814 | |
| 1799 | const debug_state = InternPool.debug_state; | 1815 | const debug_state = InternPool.debug_state; |
| ... | @@ -1848,12 +1864,18 @@ pub const NullTerminatedString = enum(u32) { | ... | @@ -1848,12 +1864,18 @@ pub const NullTerminatedString = enum(u32) { |
| 1848 | } | 1864 | } |
| 1849 | | 1865 | |
| 1850 | pub fn toSlice(string: NullTerminatedString, ip: *const InternPool) [:0]const u8 { | 1866 | pub fn toSlice(string: NullTerminatedString, ip: *const InternPool) [:0]const u8 { |
| 1851 | const overlong_slice = string.toString().toOverlongSlice(ip); | 1867 | const unwrapped = string.toString().unwrap(ip); |
| 1852 | return overlong_slice[0..std.mem.indexOfScalar(u8, overlong_slice, 0).? :0]; | 1868 | const local_shared = ip.getLocalShared(unwrapped.tid); |
| | 1869 | const strings = local_shared.strings.acquire().view().items(.@"0"); |
| | 1870 | const string_bytes = local_shared.string_bytes.acquire().view().items(.@"0"); |
| | 1871 | return string_bytes[strings[unwrapped.index] .. strings[unwrapped.index + 1] - 1 :0]; |
| 1853 | } | 1872 | } |
| 1854 | | 1873 | |
| 1855 | pub fn length(string: NullTerminatedString, ip: *const InternPool) u32 { | 1874 | pub fn length(string: NullTerminatedString, ip: *const InternPool) u32 { |
| 1856 | return @intCast(string.toSlice(ip).len); | 1875 | const unwrapped = string.toString().unwrap(ip); |
| | 1876 | const local_shared = ip.getLocalShared(unwrapped.tid); |
| | 1877 | const strings = local_shared.strings.acquire().view().items(.@"0"); |
| | 1878 | return strings[unwrapped.index + 1] - 1 - strings[unwrapped.index]; |
| 1857 | } | 1879 | } |
| 1858 | | 1880 | |
| 1859 | pub fn eqlSlice(string: NullTerminatedString, slice: []const u8, ip: *const InternPool) bool { | 1881 | pub fn eqlSlice(string: NullTerminatedString, slice: []const u8, ip: *const InternPool) bool { |
| ... | @@ -4767,7 +4789,8 @@ pub const Index = enum(u32) { | ... | @@ -4767,7 +4789,8 @@ pub const Index = enum(u32) { |
| 4767 | fn wrap(unwrapped: Unwrapped, ip: *const InternPool) Index { | 4789 | fn wrap(unwrapped: Unwrapped, ip: *const InternPool) Index { |
| 4768 | assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask()); | 4790 | assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask()); |
| 4769 | assert(unwrapped.index <= ip.getIndexMask(u30)); | 4791 | assert(unwrapped.index <= ip.getIndexMask(u30)); |
| 4770 | return @enumFromInt(@as(u32, @intFromEnum(unwrapped.tid)) << ip.tid_shift_30 | unwrapped.index); | 4792 | return @enumFromInt(@shlExact(@as(u32, @intFromEnum(unwrapped.tid)), ip.tid_shift_30) | |
| | 4793 | unwrapped.index); |
| 4771 | } | 4794 | } |
| 4772 | | 4795 | |
| 4773 | pub fn getExtra(unwrapped: Unwrapped, ip: *const InternPool) Local.Extra { | 4796 | pub fn getExtra(unwrapped: Unwrapped, ip: *const InternPool) Local.Extra { |
| ... | @@ -6795,6 +6818,7 @@ pub fn init(ip: *InternPool, gpa: Allocator, available_threads: usize) !void { | ... | @@ -6795,6 +6818,7 @@ pub fn init(ip: *InternPool, gpa: Allocator, available_threads: usize) !void { |
| 6795 | .extra = .empty, | 6818 | .extra = .empty, |
| 6796 | .limbs = .empty, | 6819 | .limbs = .empty, |
| 6797 | .strings = .empty, | 6820 | .strings = .empty, |
| | 6821 | .string_bytes = .empty, |
| 6798 | .tracked_insts = .empty, | 6822 | .tracked_insts = .empty, |
| 6799 | .files = .empty, | 6823 | .files = .empty, |
| 6800 | .maps = .empty, | 6824 | .maps = .empty, |
| ... | @@ -6810,6 +6834,7 @@ pub fn init(ip: *InternPool, gpa: Allocator, available_threads: usize) !void { | ... | @@ -6810,6 +6834,7 @@ pub fn init(ip: *InternPool, gpa: Allocator, available_threads: usize) !void { |
| 6810 | .extra = .empty, | 6834 | .extra = .empty, |
| 6811 | .limbs = .empty, | 6835 | .limbs = .empty, |
| 6812 | .strings = .empty, | 6836 | .strings = .empty, |
| | 6837 | .string_bytes = .empty, |
| 6813 | .tracked_insts = .empty, | 6838 | .tracked_insts = .empty, |
| 6814 | .files = .empty, | 6839 | .files = .empty, |
| 6815 | .maps = .empty, | 6840 | .maps = .empty, |
| ... | @@ -6819,6 +6844,7 @@ pub fn init(ip: *InternPool, gpa: Allocator, available_threads: usize) !void { | ... | @@ -6819,6 +6844,7 @@ pub fn init(ip: *InternPool, gpa: Allocator, available_threads: usize) !void { |
| 6819 | .namespaces = .empty, | 6844 | .namespaces = .empty, |
| 6820 | }, | 6845 | }, |
| 6821 | }); | 6846 | }); |
| | 6847 | for (ip.locals) |*local| try local.getMutableStrings(gpa).append(.{0}); |
| 6822 | | 6848 | |
| 6823 | ip.tid_width = @intCast(std.math.log2_int_ceil(usize, used_threads)); | 6849 | ip.tid_width = @intCast(std.math.log2_int_ceil(usize, used_threads)); |
| 6824 | ip.tid_shift_30 = if (single_threaded) 0 else 30 - ip.tid_width; | 6850 | ip.tid_shift_30 = if (single_threaded) 0 else 30 - ip.tid_width; |
| ... | @@ -8523,30 +8549,30 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All | ... | @@ -8523,30 +8549,30 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All |
| 8523 | } | 8549 | } |
| 8524 | | 8550 | |
| 8525 | if (child == .u8_type) bytes: { | 8551 | if (child == .u8_type) bytes: { |
| 8526 | const strings = ip.getLocal(tid).getMutableStrings(gpa); | 8552 | const string_bytes = ip.getLocal(tid).getMutableStringBytes(gpa); |
| 8527 | const start = strings.mutate.len; | 8553 | const start = string_bytes.mutate.len; |
| 8528 | try strings.ensureUnusedCapacity(@intCast(len_including_sentinel + 1)); | 8554 | try string_bytes.ensureUnusedCapacity(@intCast(len_including_sentinel + 1)); |
| 8529 | try extra.ensureUnusedCapacity(@typeInfo(Bytes).@"struct".fields.len); | 8555 | try extra.ensureUnusedCapacity(@typeInfo(Bytes).@"struct".fields.len); |
| 8530 | switch (aggregate.storage) { | 8556 | switch (aggregate.storage) { |
| 8531 | .bytes => |bytes| strings.appendSliceAssumeCapacity(.{bytes.toSlice(len, ip)}), | 8557 | .bytes => |bytes| string_bytes.appendSliceAssumeCapacity(.{bytes.toSlice(len, ip)}), |
| 8532 | .elems => |elems| for (elems[0..@intCast(len)]) |elem| switch (ip.indexToKey(elem)) { | 8558 | .elems => |elems| for (elems[0..@intCast(len)]) |elem| switch (ip.indexToKey(elem)) { |
| 8533 | .undef => { | 8559 | .undef => { |
| 8534 | strings.shrinkRetainingCapacity(start); | 8560 | string_bytes.shrinkRetainingCapacity(start); |
| 8535 | break :bytes; | 8561 | break :bytes; |
| 8536 | }, | 8562 | }, |
| 8537 | .int => |int| strings.appendAssumeCapacity(.{@intCast(int.storage.u64)}), | 8563 | .int => |int| string_bytes.appendAssumeCapacity(.{@intCast(int.storage.u64)}), |
| 8538 | else => unreachable, | 8564 | else => unreachable, |
| 8539 | }, | 8565 | }, |
| 8540 | .repeated_elem => |elem| switch (ip.indexToKey(elem)) { | 8566 | .repeated_elem => |elem| switch (ip.indexToKey(elem)) { |
| 8541 | .undef => break :bytes, | 8567 | .undef => break :bytes, |
| 8542 | .int => |int| @memset( | 8568 | .int => |int| @memset( |
| 8543 | strings.addManyAsSliceAssumeCapacity(@intCast(len))[0], | 8569 | string_bytes.addManyAsSliceAssumeCapacity(@intCast(len))[0], |
| 8544 | @intCast(int.storage.u64), | 8570 | @intCast(int.storage.u64), |
| 8545 | ), | 8571 | ), |
| 8546 | else => unreachable, | 8572 | else => unreachable, |
| 8547 | }, | 8573 | }, |
| 8548 | } | 8574 | } |
| 8549 | if (sentinel != .none) strings.appendAssumeCapacity(.{ | 8575 | if (sentinel != .none) string_bytes.appendAssumeCapacity(.{ |
| 8550 | @intCast(ip.indexToKey(sentinel).int.storage.u64), | 8576 | @intCast(ip.indexToKey(sentinel).int.storage.u64), |
| 8551 | }); | 8577 | }); |
| 8552 | const string = try ip.getOrPutTrailingString( | 8578 | const string = try ip.getOrPutTrailingString( |
| ... | @@ -11762,10 +11788,10 @@ pub fn getOrPutString( | ... | @@ -11762,10 +11788,10 @@ pub fn getOrPutString( |
| 11762 | slice: []const u8, | 11788 | slice: []const u8, |
| 11763 | comptime embedded_nulls: EmbeddedNulls, | 11789 | comptime embedded_nulls: EmbeddedNulls, |
| 11764 | ) Allocator.Error!embedded_nulls.StringType() { | 11790 | ) Allocator.Error!embedded_nulls.StringType() { |
| 11765 | const strings = ip.getLocal(tid).getMutableStrings(gpa); | 11791 | const string_bytes = ip.getLocal(tid).getMutableStringBytes(gpa); |
| 11766 | try strings.ensureUnusedCapacity(slice.len + 1); | 11792 | try string_bytes.ensureUnusedCapacity(slice.len + 1); |
| 11767 | strings.appendSliceAssumeCapacity(.{slice}); | 11793 | string_bytes.appendSliceAssumeCapacity(.{slice}); |
| 11768 | strings.appendAssumeCapacity(.{0}); | 11794 | string_bytes.appendAssumeCapacity(.{0}); |
| 11769 | return ip.getOrPutTrailingString(gpa, tid, @intCast(slice.len + 1), embedded_nulls); | 11795 | return ip.getOrPutTrailingString(gpa, tid, @intCast(slice.len + 1), embedded_nulls); |
| 11770 | } | 11796 | } |
| 11771 | | 11797 | |
| ... | @@ -11780,8 +11806,8 @@ pub fn getOrPutStringFmt( | ... | @@ -11780,8 +11806,8 @@ pub fn getOrPutStringFmt( |
| 11780 | // ensure that references to strings in args do not get invalidated | 11806 | // ensure that references to strings in args do not get invalidated |
| 11781 | const format_z = format ++ .{0}; | 11807 | const format_z = format ++ .{0}; |
| 11782 | const len: u32 = @intCast(std.fmt.count(format_z, args)); | 11808 | const len: u32 = @intCast(std.fmt.count(format_z, args)); |
| 11783 | const strings = ip.getLocal(tid).getMutableStrings(gpa); | 11809 | const string_bytes = ip.getLocal(tid).getMutableStringBytes(gpa); |
| 11784 | const slice = try strings.addManyAsSlice(len); | 11810 | const slice = try string_bytes.addManyAsSlice(len); |
| 11785 | assert((std.fmt.bufPrint(slice[0], format_z, args) catch unreachable).len == len); | 11811 | assert((std.fmt.bufPrint(slice[0], format_z, args) catch unreachable).len == len); |
| 11786 | return ip.getOrPutTrailingString(gpa, tid, len, embedded_nulls); | 11812 | return ip.getOrPutTrailingString(gpa, tid, len, embedded_nulls); |
| 11787 | } | 11813 | } |
| ... | @@ -11805,21 +11831,27 @@ pub fn getOrPutTrailingString( | ... | @@ -11805,21 +11831,27 @@ pub fn getOrPutTrailingString( |
| 11805 | len: u32, | 11831 | len: u32, |
| 11806 | comptime embedded_nulls: EmbeddedNulls, | 11832 | comptime embedded_nulls: EmbeddedNulls, |
| 11807 | ) Allocator.Error!embedded_nulls.StringType() { | 11833 | ) Allocator.Error!embedded_nulls.StringType() { |
| 11808 | const strings = ip.getLocal(tid).getMutableStrings(gpa); | 11834 | const local = ip.getLocal(tid); |
| 11809 | const start: u32 = @intCast(strings.mutate.len - len); | 11835 | const strings = local.getMutableStrings(gpa); |
| 11810 | if (len > 0 and strings.view().items(.@"0")[strings.mutate.len - 1] == 0) { | 11836 | try strings.ensureUnusedCapacity(1); |
| 11811 | strings.mutate.len -= 1; | 11837 | const string_bytes = local.getMutableStringBytes(gpa); |
| | 11838 | const start: u32 = @intCast(string_bytes.mutate.len - len); |
| | 11839 | if (len > 0 and string_bytes.view().items(.@"0")[string_bytes.mutate.len - 1] == 0) { |
| | 11840 | string_bytes.mutate.len -= 1; |
| 11812 | } else { | 11841 | } else { |
| 11813 | try strings.ensureUnusedCapacity(1); | 11842 | try string_bytes.ensureUnusedCapacity(1); |
| 11814 | } | 11843 | } |
| 11815 | const key: []const u8 = strings.view().items(.@"0")[start..]; | 11844 | const key: []const u8 = string_bytes.view().items(.@"0")[start..]; |
| 11816 | const value: embedded_nulls.StringType() = | 11845 | const value: embedded_nulls.StringType() = @enumFromInt(@intFromEnum((String.Unwrapped{ |
| 11817 | @enumFromInt(@intFromEnum((String.Unwrapped{ .tid = tid, .index = start }).wrap(ip))); | 11846 | .tid = tid, |
| | 11847 | .index = strings.mutate.len - 1, |
| | 11848 | }).wrap(ip))); |
| 11818 | const has_embedded_null = std.mem.indexOfScalar(u8, key, 0) != null; | 11849 | const has_embedded_null = std.mem.indexOfScalar(u8, key, 0) != null; |
| 11819 | switch (embedded_nulls) { | 11850 | switch (embedded_nulls) { |
| 11820 | .no_embedded_nulls => assert(!has_embedded_null), | 11851 | .no_embedded_nulls => assert(!has_embedded_null), |
| 11821 | .maybe_embedded_nulls => if (has_embedded_null) { | 11852 | .maybe_embedded_nulls => if (has_embedded_null) { |
| 11822 | strings.appendAssumeCapacity(.{0}); | 11853 | string_bytes.appendAssumeCapacity(.{0}); |
| | 11854 | strings.appendAssumeCapacity(.{string_bytes.mutate.len}); |
| 11823 | return value; | 11855 | return value; |
| 11824 | }, | 11856 | }, |
| 11825 | } | 11857 | } |
| ... | @@ -11837,7 +11869,7 @@ pub fn getOrPutTrailingString( | ... | @@ -11837,7 +11869,7 @@ pub fn getOrPutTrailingString( |
| 11837 | const index = entry.acquire().unwrap() orelse break; | 11869 | const index = entry.acquire().unwrap() orelse break; |
| 11838 | if (entry.hash != hash) continue; | 11870 | if (entry.hash != hash) continue; |
| 11839 | if (!index.eqlSlice(key, ip)) continue; | 11871 | if (!index.eqlSlice(key, ip)) continue; |
| 11840 | strings.shrinkRetainingCapacity(start); | 11872 | string_bytes.shrinkRetainingCapacity(start); |
| 11841 | return @enumFromInt(@intFromEnum(index)); | 11873 | return @enumFromInt(@intFromEnum(index)); |
| 11842 | } | 11874 | } |
| 11843 | shard.mutate.string_map.mutex.lock(); | 11875 | shard.mutate.string_map.mutex.lock(); |
| ... | @@ -11853,19 +11885,20 @@ pub fn getOrPutTrailingString( | ... | @@ -11853,19 +11885,20 @@ pub fn getOrPutTrailingString( |
| 11853 | const index = entry.acquire().unwrap() orelse break; | 11885 | const index = entry.acquire().unwrap() orelse break; |
| 11854 | if (entry.hash != hash) continue; | 11886 | if (entry.hash != hash) continue; |
| 11855 | if (!index.eqlSlice(key, ip)) continue; | 11887 | if (!index.eqlSlice(key, ip)) continue; |
| 11856 | strings.shrinkRetainingCapacity(start); | 11888 | string_bytes.shrinkRetainingCapacity(start); |
| 11857 | return @enumFromInt(@intFromEnum(index)); | 11889 | return @enumFromInt(@intFromEnum(index)); |
| 11858 | } | 11890 | } |
| 11859 | defer shard.mutate.string_map.len += 1; | 11891 | defer shard.mutate.string_map.len += 1; |
| 11860 | const map_header = map.header().*; | 11892 | const map_header = map.header().*; |
| 11861 | if (shard.mutate.string_map.len < map_header.capacity * 3 / 5) { | 11893 | if (shard.mutate.string_map.len < map_header.capacity * 3 / 5) { |
| 11862 | strings.appendAssumeCapacity(.{0}); | 11894 | string_bytes.appendAssumeCapacity(.{0}); |
| | 11895 | strings.appendAssumeCapacity(.{string_bytes.mutate.len}); |
| 11863 | const entry = &map.entries[map_index]; | 11896 | const entry = &map.entries[map_index]; |
| 11864 | entry.hash = hash; | 11897 | entry.hash = hash; |
| 11865 | entry.release(@enumFromInt(@intFromEnum(value))); | 11898 | entry.release(@enumFromInt(@intFromEnum(value))); |
| 11866 | return value; | 11899 | return value; |
| 11867 | } | 11900 | } |
| 11868 | const arena_state = &ip.getLocal(tid).mutate.arena; | 11901 | const arena_state = &local.mutate.arena; |
| 11869 | var arena = arena_state.promote(gpa); | 11902 | var arena = arena_state.promote(gpa); |
| 11870 | defer arena_state.* = arena.state; | 11903 | defer arena_state.* = arena.state; |
| 11871 | const new_map_capacity = map_header.capacity * 2; | 11904 | const new_map_capacity = map_header.capacity * 2; |
| ... | @@ -11901,7 +11934,8 @@ pub fn getOrPutTrailingString( | ... | @@ -11901,7 +11934,8 @@ pub fn getOrPutTrailingString( |
| 11901 | map_index &= new_map_mask; | 11934 | map_index &= new_map_mask; |
| 11902 | if (map.entries[map_index].value == .none) break; | 11935 | if (map.entries[map_index].value == .none) break; |
| 11903 | } | 11936 | } |
| 11904 | strings.appendAssumeCapacity(.{0}); | 11937 | string_bytes.appendAssumeCapacity(.{0}); |
| | 11938 | strings.appendAssumeCapacity(.{string_bytes.mutate.len}); |
| 11905 | map.entries[map_index] = .{ | 11939 | map.entries[map_index] = .{ |
| 11906 | .value = @enumFromInt(@intFromEnum(value)), | 11940 | .value = @enumFromInt(@intFromEnum(value)), |
| 11907 | .hash = hash, | 11941 | .hash = hash, |