authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-06-15 16:18:41-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-07-07 22:59:52-04:00
logcda716ecc43929fd1c2c9679335b8b22f1b67d1a
tree43161efad1af25784f1ac0933a3a4a94352ef6d6
parentca02266157ee72e41068672c8ca6f928fcbf6fdf

InternPool: implement thread-safe hash map

4 files changed, 452 insertions(+), 195 deletions(-)

lib/std/Thread/Pool.zig+4
......@@ -291,3 +291,7 @@ pub fn waitAndWork(pool: *Pool, wait_group: *WaitGroup) void {
291291 return;
292292 }
293293}
294
295pub fn getIdCount(pool: *Pool) usize {
296 return 1 + pool.threads.len;
297}
src/Compilation.zig+2-2
......@@ -1397,7 +1397,7 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil
13971397 .error_limit = error_limit,
13981398 .llvm_object = null,
13991399 };
1400 try zcu.init();
1400 try zcu.init(options.thread_pool.getIdCount());
14011401 break :blk zcu;
14021402 } else blk: {
14031403 if (options.emit_h != null) return error.NoZigModuleForCHeader;
......@@ -2156,7 +2156,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
21562156 if (build_options.enable_debug_extensions and comp.verbose_generic_instances) {
21572157 std.debug.print("generic instances for '{s}:0x{x}':\n", .{
21582158 comp.root_name,
2159 @as(usize, @intFromPtr(zcu)),
2159 @intFromPtr(zcu),
21602160 });
21612161 zcu.intern_pool.dumpGenericInstances(gpa);
21622162 }
src/InternPool.zig+444-191
......@@ -2,9 +2,10 @@
22//! This data structure is self-contained, with the following exceptions:
33//! * Module.Namespace has a pointer to Module.File
44
5/// Maps `Key` to `Index`. `Key` objects are not stored anywhere; they are
6/// constructed lazily.
7map: std.AutoArrayHashMapUnmanaged(void, void) = .{},
5local: []Local = &.{},
6shard_shift: std.math.Log2Int(usize) = 0,
7shards: []Shard = &.{},
8
89items: std.MultiArrayList(Item) = .{},
910extra: std.ArrayListUnmanaged(u32) = .{},
1011/// On 32-bit systems, this array is ignored and extra is used for everything.
......@@ -351,6 +352,115 @@ pub const DepEntry = extern struct {
351352 };
352353};
353354
355const Local = struct {
356 aligned: void align(std.atomic.cache_line) = {},
357
358 /// node: Garbage.Node,
359 /// header: List.Header,
360 /// data: [capacity]u32,
361 /// tag: [capacity]Tag,
362 items: List,
363
364 /// node: Garbage.Node,
365 /// header: List.Header,
366 /// extra: [capacity]u32,
367 extra: List,
368
369 garbage: Garbage,
370
371 const List = struct {
372 entries: [*]u32,
373
374 const empty: List = .{
375 .entries = @constCast(&[_]u32{ 0, 0 })[Header.fields_len..].ptr,
376 };
377
378 fn acquire(list: *const List) List {
379 return .{ .entries = @atomicLoad([*]u32, &list.entries, .acquire) };
380 }
381 fn release(list: *List, new_list: List) void {
382 @atomicStore([*]u32, &list.entries, new_list.entries, .release);
383 }
384
385 const Header = extern struct {
386 len: u32,
387 capacity: u32,
388
389 const fields_len = @typeInfo(Header).Struct.fields.len;
390 };
391 fn header(list: List) *Header {
392 return @ptrCast(list.entries - Header.fields_len);
393 }
394 };
395
396 const Garbage = std.SinglyLinkedList(struct { buf_len: usize });
397 const garbage_align = @max(@alignOf(Garbage.Node), @alignOf(u32));
398
399 fn freeGarbage(garbage: *const Garbage.Node, gpa: Allocator) void {
400 gpa.free(@as([*]align(Local.garbage_align) const u8, @ptrCast(garbage))[0..garbage.data.buf_len]);
401 }
402};
403
404const Shard = struct {
405 aligned: void align(std.atomic.cache_line) = {},
406
407 mutate_mutex: std.Thread.Mutex.Recursive,
408
409 /// node: Local.Garbage.Node,
410 /// header: Map.Header,
411 /// entries: [capacity]Map.Entry,
412 map: Map,
413
414 const Map = struct {
415 entries: [*]u32,
416
417 const empty: Map = .{
418 .entries = @constCast(&[_]u32{ 0, 1, @intFromEnum(Index.none), 0 })[Header.fields_len..].ptr,
419 };
420
421 fn acquire(map: *const Map) Map {
422 return .{ .entries = @atomicLoad([*]u32, &map.entries, .acquire) };
423 }
424 fn release(map: *Map, new_map: Map) void {
425 @atomicStore([*]u32, &map.entries, new_map.entries, .release);
426 }
427
428 const Header = extern struct {
429 len: u32,
430 capacity: u32,
431
432 const fields_len: u32 = @typeInfo(Header).Struct.fields.len;
433
434 fn mask(head: *const Header) u32 {
435 assert(std.math.isPowerOfTwo(head.capacity));
436 assert(std.math.isPowerOfTwo(Entry.fields_len));
437 return (head.capacity - 1) * Entry.fields_len;
438 }
439 };
440 fn header(map: Map) *Header {
441 return @ptrCast(map.entries - Header.fields_len);
442 }
443
444 const Entry = extern struct {
445 index: Index,
446 hash: u32,
447
448 const fields_len: u32 = @typeInfo(Entry).Struct.fields.len;
449
450 fn acquire(entry: *const Entry) Index {
451 return @atomicLoad(Index, &entry.index, .acquire);
452 }
453 fn release(entry: *Entry, index: Index) void {
454 @atomicStore(Index, &entry.index, index, .release);
455 }
456 };
457 fn at(map: Map, index: usize) *Entry {
458 assert(index % Entry.fields_len == 0);
459 return @ptrCast(&map.entries[index]);
460 }
461 };
462};
463
354464const FieldMap = std.ArrayHashMapUnmanaged(void, void, std.array_hash_map.AutoContext(void), false);
355465
356466const builtin = @import("builtin");
......@@ -369,20 +479,6 @@ const Zcu = @import("Zcu.zig");
369479const Module = Zcu;
370480const Zir = std.zig.Zir;
371481
372const KeyAdapter = struct {
373 intern_pool: *const InternPool,
374
375 pub fn eql(ctx: @This(), a: Key, b_void: void, b_map_index: usize) bool {
376 _ = b_void;
377 if (ctx.intern_pool.items.items(.tag)[b_map_index] == .removed) return false;
378 return ctx.intern_pool.indexToKey(@enumFromInt(b_map_index)).eql(a, ctx.intern_pool);
379 }
380
381 pub fn hash(ctx: @This(), a: Key) u32 {
382 return a.hash32(ctx.intern_pool);
383 }
384};
385
386482/// An index into `maps` which might be `none`.
387483pub const OptionalMapIndex = enum(u32) {
388484 none = std.math.maxInt(u32),
......@@ -4535,17 +4631,27 @@ pub const MemoizedCall = struct {
45354631 result: Index,
45364632};
45374633
4538pub fn init(ip: *InternPool, gpa: Allocator) !void {
4634pub fn init(ip: *InternPool, gpa: Allocator, total_threads: usize) !void {
4635 errdefer ip.deinit(gpa);
45394636 assert(ip.items.len == 0);
45404637
4638 ip.local = try gpa.alloc(Local, total_threads);
4639 @memset(ip.local, .{
4640 .items = Local.List.empty,
4641 .extra = Local.List.empty,
4642 .garbage = .{},
4643 });
4644
4645 ip.shard_shift = @intCast(std.math.log2_int_ceil(usize, total_threads));
4646 ip.shards = try gpa.alloc(Shard, @as(usize, 1) << ip.shard_shift);
4647 @memset(ip.shards, .{
4648 .mutate_mutex = std.Thread.Mutex.Recursive.init,
4649 .map = Shard.Map.empty,
4650 });
4651
45414652 // Reserve string index 0 for an empty string.
45424653 assert((try ip.getOrPutString(gpa, .main, "", .no_embedded_nulls)) == .empty);
45434654
4544 // So that we can use `catch unreachable` below.
4545 try ip.items.ensureUnusedCapacity(gpa, static_keys.len);
4546 try ip.map.ensureUnusedCapacity(gpa, static_keys.len);
4547 try ip.extra.ensureUnusedCapacity(gpa, static_keys.len);
4548
45494655 // This inserts all the statically-known values into the intern pool in the
45504656 // order expected.
45514657 for (&static_keys, 0..) |key, key_index| switch (@as(Index, @enumFromInt(key_index))) {
......@@ -4574,12 +4680,9 @@ pub fn init(ip: *InternPool, gpa: Allocator) !void {
45744680 assert(ip.indexToKey(ip.typeOf(cc_inline)).int_type.bits ==
45754681 @typeInfo(@typeInfo(std.builtin.CallingConvention).Enum.tag_type).Int.bits);
45764682 }
4577
4578 assert(ip.items.len == static_keys.len);
45794683}
45804684
45814685pub fn deinit(ip: *InternPool, gpa: Allocator) void {
4582 ip.map.deinit(gpa);
45834686 ip.items.deinit(gpa);
45844687 ip.extra.deinit(gpa);
45854688 ip.limbs.deinit(gpa);
......@@ -4611,6 +4714,16 @@ pub fn deinit(ip: *InternPool, gpa: Allocator) void {
46114714
46124715 ip.files.deinit(gpa);
46134716
4717 gpa.free(ip.shards);
4718 for (ip.local) |*local| {
4719 var next = local.garbage.first;
4720 while (next) |cur| {
4721 next = cur.next;
4722 Local.freeGarbage(cur, gpa);
4723 }
4724 }
4725 gpa.free(ip.local);
4726
46144727 ip.* = undefined;
46154728}
46164729
......@@ -5239,10 +5352,133 @@ fn indexToKeyBigInt(ip: *const InternPool, limb_index: u32, positive: bool) Key
52395352 } };
52405353}
52415354
5355const GetOrPutKey = union(enum) {
5356 existing: Index,
5357 new: struct {
5358 shard: *Shard,
5359 map_index: u32,
5360 },
5361
5362 fn set(gop: *GetOrPutKey, index: Index) Index {
5363 switch (gop.*) {
5364 .existing => unreachable,
5365 .new => |info| {
5366 info.shard.map.at(info.map_index).release(index);
5367 info.shard.map.header().len += 1;
5368 info.shard.mutate_mutex.unlock();
5369 },
5370 }
5371 gop.* = .{ .existing = index };
5372 return index;
5373 }
5374
5375 fn assign(gop: *GetOrPutKey, new_gop: GetOrPutKey) void {
5376 gop.deinit();
5377 gop.* = new_gop;
5378 }
5379
5380 fn deinit(gop: *GetOrPutKey) void {
5381 switch (gop.*) {
5382 .existing => {},
5383 .new => |info| info.shard.mutate_mutex.unlock(),
5384 }
5385 gop.* = undefined;
5386 }
5387};
5388fn getOrPutKey(
5389 ip: *InternPool,
5390 gpa: Allocator,
5391 tid: Zcu.PerThread.Id,
5392 key: Key,
5393) Allocator.Error!GetOrPutKey {
5394 const full_hash = key.hash64(ip);
5395 const hash: u32 = @truncate(full_hash >> 32);
5396 const shard = &ip.shards[@intCast(full_hash & (ip.shards.len - 1))];
5397 var map = shard.map.acquire();
5398 var map_mask = map.header().mask();
5399 var map_index = hash;
5400 while (true) : (map_index += Shard.Map.Entry.fields_len) {
5401 map_index &= map_mask;
5402 const entry = map.at(map_index);
5403 const index = entry.acquire();
5404 if (index == .none) break;
5405 if (entry.hash == hash and ip.indexToKey(index).eql(key, ip))
5406 return .{ .existing = index };
5407 }
5408 shard.mutate_mutex.lock();
5409 errdefer shard.mutate_mutex.unlock();
5410 if (map.entries != shard.map.entries) {
5411 map = shard.map;
5412 map_mask = map.header().mask();
5413 map_index = hash;
5414 }
5415 while (true) : (map_index += Shard.Map.Entry.fields_len) {
5416 map_index &= map_mask;
5417 const entry = map.at(map_index);
5418 const index = entry.index;
5419 if (index == .none) break;
5420 if (entry.hash == hash and ip.indexToKey(index).eql(key, ip)) {
5421 defer shard.mutate_mutex.unlock();
5422 return .{ .existing = index };
5423 }
5424 }
5425 const map_header = map.header().*;
5426 if (map_header.len >= map_header.capacity * 3 / 5) {
5427 const new_map_capacity = map_header.capacity * 2;
5428 const new_map_buf = try gpa.alignedAlloc(
5429 u8,
5430 Local.garbage_align,
5431 @sizeOf(Local.Garbage.Node) + (Shard.Map.Header.fields_len +
5432 new_map_capacity * Shard.Map.Entry.fields_len) * @sizeOf(u32),
5433 );
5434 const new_node: *Local.Garbage.Node = @ptrCast(new_map_buf.ptr);
5435 new_node.* = .{ .data = .{ .buf_len = new_map_buf.len } };
5436 ip.local[@intFromEnum(tid)].garbage.prepend(new_node);
5437 const new_map_entries = std.mem.bytesAsSlice(
5438 u32,
5439 new_map_buf[@sizeOf(Local.Garbage.Node)..],
5440 )[Shard.Map.Header.fields_len..];
5441 const new_map: Shard.Map = .{ .entries = new_map_entries.ptr };
5442 new_map.header().* = .{
5443 .len = map_header.len,
5444 .capacity = new_map_capacity,
5445 };
5446 @memset(new_map_entries, @intFromEnum(Index.none));
5447 const new_map_mask = new_map.header().mask();
5448 map_index = 0;
5449 while (map_index < map_header.capacity * 2) : (map_index += Shard.Map.Entry.fields_len) {
5450 const entry = map.at(map_index);
5451 const index = entry.index;
5452 if (index == .none) continue;
5453 const item_hash = entry.hash;
5454 var new_map_index = item_hash;
5455 while (true) : (new_map_index += Shard.Map.Entry.fields_len) {
5456 new_map_index &= new_map_mask;
5457 const new_entry = new_map.at(new_map_index);
5458 if (new_entry.index != .none) continue;
5459 new_entry.* = .{
5460 .index = index,
5461 .hash = item_hash,
5462 };
5463 break;
5464 }
5465 }
5466 map = new_map;
5467 map_index = hash;
5468 while (true) : (map_index += Shard.Map.Entry.fields_len) {
5469 map_index &= new_map_mask;
5470 if (map.at(map_index).index == .none) break;
5471 }
5472 shard.map.release(new_map);
5473 }
5474 map.at(map_index).hash = hash;
5475 return .{ .new = .{ .shard = shard, .map_index = map_index } };
5476}
5477
52425478pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) Allocator.Error!Index {
5243 const adapter: KeyAdapter = .{ .intern_pool = ip };
5244 const gop = try ip.map.getOrPutAdapted(gpa, key, adapter);
5245 if (gop.found_existing) return @enumFromInt(gop.index);
5479 var gop = try ip.getOrPutKey(gpa, tid, key);
5480 defer gop.deinit();
5481 if (gop == .existing) return gop.existing;
52465482 try ip.items.ensureUnusedCapacity(gpa, 1);
52475483 switch (key) {
52485484 .int_type => |int_type| {
......@@ -5260,18 +5496,17 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All
52605496 assert(ptr_type.sentinel == .none or ip.typeOf(ptr_type.sentinel) == ptr_type.child);
52615497
52625498 if (ptr_type.flags.size == .Slice) {
5263 _ = ip.map.pop();
52645499 var new_key = key;
52655500 new_key.ptr_type.flags.size = .Many;
52665501 const ptr_type_index = try ip.get(gpa, tid, new_key);
5267 assert(!(try ip.map.getOrPutAdapted(gpa, key, adapter)).found_existing);
5502 gop.assign(try ip.getOrPutKey(gpa, tid, key));
52685503
52695504 try ip.items.ensureUnusedCapacity(gpa, 1);
52705505 ip.items.appendAssumeCapacity(.{
52715506 .tag = .type_slice,
52725507 .data = @intFromEnum(ptr_type_index),
52735508 });
5274 return @enumFromInt(ip.items.len - 1);
5509 return gop.set(@enumFromInt(ip.items.len - 1));
52755510 }
52765511
52775512 var ptr_type_adjusted = ptr_type;
......@@ -5295,7 +5530,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All
52955530 .child = array_type.child,
52965531 }),
52975532 });
5298 return @enumFromInt(ip.items.len - 1);
5533 return gop.set(@enumFromInt(ip.items.len - 1));
52995534 }
53005535 }
53015536
......@@ -5442,11 +5677,10 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All
54425677 },
54435678 .anon_decl => |anon_decl| if (ptrsHaveSameAlignment(ip, ptr.ty, ptr_type, anon_decl.orig_ty)) item: {
54445679 if (ptr.ty != anon_decl.orig_ty) {
5445 _ = ip.map.pop();
54465680 var new_key = key;
54475681 new_key.ptr.base_addr.anon_decl.orig_ty = ptr.ty;
5448 const new_gop = try ip.map.getOrPutAdapted(gpa, new_key, adapter);
5449 if (new_gop.found_existing) return @enumFromInt(new_gop.index);
5682 gop.assign(try ip.getOrPutKey(gpa, tid, new_key));
5683 if (gop == .existing) return gop.existing;
54505684 }
54515685 break :item .{
54525686 .tag = .ptr_anon_decl,
......@@ -5486,7 +5720,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All
54865720 .tag = .ptr_int,
54875721 .data = try ip.addExtra(gpa, PtrInt.init(ptr.ty, ptr.byte_offset)),
54885722 },
5489 .arr_elem, .field => |base_index| item: {
5723 .arr_elem, .field => |base_index| {
54905724 const base_ptr_type = ip.indexToKey(ip.typeOf(base_index.base)).ptr_type;
54915725 switch (ptr.base_addr) {
54925726 .arr_elem => assert(base_ptr_type.flags.size == .Many),
......@@ -5516,21 +5750,21 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All
55165750 },
55175751 else => unreachable,
55185752 }
5519 _ = ip.map.pop();
55205753 const index_index = try ip.get(gpa, tid, .{ .int = .{
55215754 .ty = .usize_type,
55225755 .storage = .{ .u64 = base_index.index },
55235756 } });
5524 assert(!(try ip.map.getOrPutAdapted(gpa, key, adapter)).found_existing);
5757 gop.assign(try ip.getOrPutKey(gpa, tid, key));
55255758 try ip.items.ensureUnusedCapacity(gpa, 1);
5526 break :item .{
5759 ip.items.appendAssumeCapacity(.{
55275760 .tag = switch (ptr.base_addr) {
55285761 .arr_elem => .ptr_elem,
55295762 .field => .ptr_field,
55305763 else => unreachable,
55315764 },
55325765 .data = try ip.addExtra(gpa, PtrBaseIndex.init(ptr.ty, base_index.base, index_index, ptr.byte_offset)),
5533 };
5766 });
5767 return gop.set(@enumFromInt(ip.items.len - 1));
55345768 },
55355769 });
55365770 },
......@@ -5566,7 +5800,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All
55665800 .lazy_ty = lazy_ty,
55675801 }),
55685802 });
5569 return @enumFromInt(ip.items.len - 1);
5803 return gop.set(@enumFromInt(ip.items.len - 1));
55705804 },
55715805 }
55725806 switch (int.ty) {
......@@ -5707,7 +5941,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All
57075941 .value = casted,
57085942 }),
57095943 });
5710 return @enumFromInt(ip.items.len - 1);
5944 return gop.set(@enumFromInt(ip.items.len - 1));
57115945 } else |_| {}
57125946
57135947 const tag: Tag = if (big_int.positive) .int_positive else .int_negative;
......@@ -5722,7 +5956,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All
57225956 .value = casted,
57235957 }),
57245958 });
5725 return @enumFromInt(ip.items.len - 1);
5959 return gop.set(@enumFromInt(ip.items.len - 1));
57265960 }
57275961
57285962 var buf: [2]Limb = undefined;
......@@ -5881,7 +6115,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All
58816115 .tag = .only_possible_value,
58826116 .data = @intFromEnum(aggregate.ty),
58836117 });
5884 return @enumFromInt(ip.items.len - 1);
6118 return gop.set(@enumFromInt(ip.items.len - 1));
58856119 }
58866120
58876121 switch (ty_key) {
......@@ -5914,7 +6148,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All
59146148 .tag = .only_possible_value,
59156149 .data = @intFromEnum(aggregate.ty),
59166150 });
5917 return @enumFromInt(ip.items.len - 1);
6151 return gop.set(@enumFromInt(ip.items.len - 1));
59186152 },
59196153 else => {},
59206154 }
......@@ -5929,12 +6163,11 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All
59296163 }
59306164 const elem = switch (aggregate.storage) {
59316165 .bytes => |bytes| elem: {
5932 _ = ip.map.pop();
59336166 const elem = try ip.get(gpa, tid, .{ .int = .{
59346167 .ty = .u8_type,
59356168 .storage = .{ .u64 = bytes.at(0, ip) },
59366169 } });
5937 assert(!(try ip.map.getOrPutAdapted(gpa, key, adapter)).found_existing);
6170 gop.assign(try ip.getOrPutKey(gpa, tid, key));
59386171 try ip.items.ensureUnusedCapacity(gpa, 1);
59396172 break :elem elem;
59406173 },
......@@ -5953,7 +6186,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All
59536186 .elem_val = elem,
59546187 }),
59556188 });
5956 return @enumFromInt(ip.items.len - 1);
6189 return gop.set(@enumFromInt(ip.items.len - 1));
59576190 }
59586191
59596192 if (child == .u8_type) bytes: {
......@@ -5997,7 +6230,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All
59976230 .bytes = string,
59986231 }),
59996232 });
6000 return @enumFromInt(ip.items.len - 1);
6233 return gop.set(@enumFromInt(ip.items.len - 1));
60016234 }
60026235
60036236 try ip.extra.ensureUnusedCapacity(
......@@ -6038,7 +6271,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All
60386271 ip.extra.appendSliceAssumeCapacity(@ptrCast(memoized_call.arg_values));
60396272 },
60406273 }
6041 return @enumFromInt(ip.items.len - 1);
6274 return gop.set(@enumFromInt(ip.items.len - 1));
60426275}
60436276
60446277pub const UnionTypeInit = struct {
......@@ -6076,11 +6309,10 @@ pub const UnionTypeInit = struct {
60766309pub fn getUnionType(
60776310 ip: *InternPool,
60786311 gpa: Allocator,
6079 _: Zcu.PerThread.Id,
6312 tid: Zcu.PerThread.Id,
60806313 ini: UnionTypeInit,
60816314) Allocator.Error!WipNamespaceType.Result {
6082 const adapter: KeyAdapter = .{ .intern_pool = ip };
6083 const gop = try ip.map.getOrPutAdapted(gpa, Key{ .union_type = switch (ini.key) {
6315 var gop = try ip.getOrPutKey(gpa, tid, .{ .union_type = switch (ini.key) {
60846316 .declared => |d| .{ .declared = .{
60856317 .zir_index = d.zir_index,
60866318 .captures = .{ .external = d.captures },
......@@ -6089,9 +6321,9 @@ pub fn getUnionType(
60896321 .zir_index = r.zir_index,
60906322 .type_hash = r.type_hash,
60916323 } },
6092 } }, adapter);
6093 if (gop.found_existing) return .{ .existing = @enumFromInt(gop.index) };
6094 errdefer _ = ip.map.pop();
6324 } });
6325 defer gop.deinit();
6326 if (gop == .existing) return .{ .existing = gop.existing };
60956327
60966328 const align_elements_len = if (ini.flags.any_aligned_fields) (ini.fields_len + 3) / 4 else 0;
60976329 const align_element: u32 = @bitCast([1]u8{@intFromEnum(Alignment.none)} ** 4);
......@@ -6167,7 +6399,7 @@ pub fn getUnionType(
61676399 }
61686400
61696401 return .{ .wip = .{
6170 .index = @enumFromInt(ip.items.len - 1),
6402 .index = gop.set(@enumFromInt(ip.items.len - 1)),
61716403 .decl_extra_index = extra_index + std.meta.fieldIndex(Tag.TypeUnion, "decl").?,
61726404 .namespace_extra_index = if (ini.has_namespace)
61736405 extra_index + std.meta.fieldIndex(Tag.TypeUnion, "namespace").?
......@@ -6225,11 +6457,10 @@ pub const StructTypeInit = struct {
62256457pub fn getStructType(
62266458 ip: *InternPool,
62276459 gpa: Allocator,
6228 _: Zcu.PerThread.Id,
6460 tid: Zcu.PerThread.Id,
62296461 ini: StructTypeInit,
62306462) Allocator.Error!WipNamespaceType.Result {
6231 const adapter: KeyAdapter = .{ .intern_pool = ip };
6232 const key: Key = .{ .struct_type = switch (ini.key) {
6463 var gop = try ip.getOrPutKey(gpa, tid, .{ .struct_type = switch (ini.key) {
62336464 .declared => |d| .{ .declared = .{
62346465 .zir_index = d.zir_index,
62356466 .captures = .{ .external = d.captures },
......@@ -6238,10 +6469,9 @@ pub fn getStructType(
62386469 .zir_index = r.zir_index,
62396470 .type_hash = r.type_hash,
62406471 } },
6241 } };
6242 const gop = try ip.map.getOrPutAdapted(gpa, key, adapter);
6243 if (gop.found_existing) return .{ .existing = @enumFromInt(gop.index) };
6244 errdefer _ = ip.map.pop();
6472 } });
6473 defer gop.deinit();
6474 if (gop == .existing) return .{ .existing = gop.existing };
62456475
62466476 const names_map = try ip.addMap(gpa, ini.fields_len);
62476477 errdefer _ = ip.maps.pop();
......@@ -6298,7 +6528,7 @@ pub fn getStructType(
62986528 ip.extra.appendNTimesAssumeCapacity(@intFromEnum(Index.none), ini.fields_len);
62996529 }
63006530 return .{ .wip = .{
6301 .index = @enumFromInt(ip.items.len - 1),
6531 .index = gop.set(@enumFromInt(ip.items.len - 1)),
63026532 .decl_extra_index = extra_index + std.meta.fieldIndex(Tag.TypeStructPacked, "decl").?,
63036533 .namespace_extra_index = if (ini.has_namespace)
63046534 extra_index + std.meta.fieldIndex(Tag.TypeStructPacked, "namespace").?
......@@ -6387,7 +6617,7 @@ pub fn getStructType(
63876617 }
63886618 ip.extra.appendNTimesAssumeCapacity(std.math.maxInt(u32), ini.fields_len);
63896619 return .{ .wip = .{
6390 .index = @enumFromInt(ip.items.len - 1),
6620 .index = gop.set(@enumFromInt(ip.items.len - 1)),
63916621 .decl_extra_index = extra_index + std.meta.fieldIndex(Tag.TypeStruct, "decl").?,
63926622 .namespace_extra_index = namespace_extra_index,
63936623 } };
......@@ -6404,7 +6634,7 @@ pub const AnonStructTypeInit = struct {
64046634pub fn getAnonStructType(
64056635 ip: *InternPool,
64066636 gpa: Allocator,
6407 _: Zcu.PerThread.Id,
6637 tid: Zcu.PerThread.Id,
64086638 ini: AnonStructTypeInit,
64096639) Allocator.Error!Index {
64106640 assert(ini.types.len == ini.values.len);
......@@ -6424,25 +6654,26 @@ pub fn getAnonStructType(
64246654 });
64256655 ip.extra.appendSliceAssumeCapacity(@ptrCast(ini.types));
64266656 ip.extra.appendSliceAssumeCapacity(@ptrCast(ini.values));
6657 errdefer ip.extra.items.len = prev_extra_len;
64276658
6428 const adapter: KeyAdapter = .{ .intern_pool = ip };
6429 const key: Key = .{
6659 var gop = try ip.getOrPutKey(gpa, tid, .{
64306660 .anon_struct_type = if (ini.names.len == 0) extraTypeTupleAnon(ip, extra_index) else k: {
64316661 assert(ini.names.len == ini.types.len);
64326662 ip.extra.appendSliceAssumeCapacity(@ptrCast(ini.names));
64336663 break :k extraTypeStructAnon(ip, extra_index);
64346664 },
6435 };
6436 const gop = try ip.map.getOrPutAdapted(gpa, key, adapter);
6437 if (gop.found_existing) {
6665 });
6666 defer gop.deinit();
6667 if (gop == .existing) {
64386668 ip.extra.items.len = prev_extra_len;
6439 return @enumFromInt(gop.index);
6669 return gop.existing;
64406670 }
6671
64416672 ip.items.appendAssumeCapacity(.{
64426673 .tag = if (ini.names.len == 0) .type_tuple_anon else .type_struct_anon,
64436674 .data = extra_index,
64446675 });
6445 return @enumFromInt(ip.items.len - 1);
6676 return gop.set(@enumFromInt(ip.items.len - 1));
64466677}
64476678
64486679/// This is equivalent to `Key.FuncType` but adjusted to have a slice for `param_types`.
......@@ -6463,7 +6694,7 @@ pub const GetFuncTypeKey = struct {
64636694pub fn getFuncType(
64646695 ip: *InternPool,
64656696 gpa: Allocator,
6466 _: Zcu.PerThread.Id,
6697 tid: Zcu.PerThread.Id,
64676698 key: GetFuncTypeKey,
64686699) Allocator.Error!Index {
64696700 // Validate input parameters.
......@@ -6501,33 +6732,33 @@ pub fn getFuncType(
65016732 if (key.comptime_bits != 0) ip.extra.appendAssumeCapacity(key.comptime_bits);
65026733 if (key.noalias_bits != 0) ip.extra.appendAssumeCapacity(key.noalias_bits);
65036734 ip.extra.appendSliceAssumeCapacity(@ptrCast(key.param_types));
6735 errdefer ip.extra.items.len = prev_extra_len;
65046736
6505 const adapter: KeyAdapter = .{ .intern_pool = ip };
6506 const gop = try ip.map.getOrPutAdapted(gpa, Key{
6737 var gop = try ip.getOrPutKey(gpa, tid, .{
65076738 .func_type = extraFuncType(ip, func_type_extra_index),
6508 }, adapter);
6509 if (gop.found_existing) {
6739 });
6740 defer gop.deinit();
6741 if (gop == .existing) {
65106742 ip.extra.items.len = prev_extra_len;
6511 return @enumFromInt(gop.index);
6743 return gop.existing;
65126744 }
65136745
65146746 ip.items.appendAssumeCapacity(.{
65156747 .tag = .type_function,
65166748 .data = func_type_extra_index,
65176749 });
6518 return @enumFromInt(ip.items.len - 1);
6750 return gop.set(@enumFromInt(ip.items.len - 1));
65196751}
65206752
65216753pub fn getExternFunc(
65226754 ip: *InternPool,
65236755 gpa: Allocator,
6524 _: Zcu.PerThread.Id,
6756 tid: Zcu.PerThread.Id,
65256757 key: Key.ExternFunc,
65266758) Allocator.Error!Index {
6527 const adapter: KeyAdapter = .{ .intern_pool = ip };
6528 const gop = try ip.map.getOrPutAdapted(gpa, Key{ .extern_func = key }, adapter);
6529 if (gop.found_existing) return @enumFromInt(gop.index);
6530 errdefer _ = ip.map.pop();
6759 var gop = try ip.getOrPutKey(gpa, tid, .{ .extern_func = key });
6760 defer gop.deinit();
6761 if (gop == .existing) return gop.existing;
65316762 const prev_extra_len = ip.extra.items.len;
65326763 const extra_index = try ip.addExtra(gpa, @as(Tag.ExternFunc, key));
65336764 errdefer ip.extra.items.len = prev_extra_len;
......@@ -6536,7 +6767,7 @@ pub fn getExternFunc(
65366767 .data = extra_index,
65376768 });
65386769 errdefer ip.items.len -= 1;
6539 return @enumFromInt(ip.items.len - 1);
6770 return gop.set(@enumFromInt(ip.items.len - 1));
65406771}
65416772
65426773pub const GetFuncDeclKey = struct {
......@@ -6554,7 +6785,7 @@ pub const GetFuncDeclKey = struct {
65546785pub fn getFuncDecl(
65556786 ip: *InternPool,
65566787 gpa: Allocator,
6557 _: Zcu.PerThread.Id,
6788 tid: Zcu.PerThread.Id,
65586789 key: GetFuncDeclKey,
65596790) Allocator.Error!Index {
65606791 // The strategy here is to add the function type unconditionally, then to
......@@ -6564,7 +6795,6 @@ pub fn getFuncDecl(
65646795
65656796 try ip.extra.ensureUnusedCapacity(gpa, @typeInfo(Tag.FuncDecl).Struct.fields.len);
65666797 try ip.items.ensureUnusedCapacity(gpa, 1);
6567 try ip.map.ensureUnusedCapacity(gpa, 1);
65686798
65696799 const func_decl_extra_index = ip.addExtraAssumeCapacity(Tag.FuncDecl{
65706800 .analysis = .{
......@@ -6583,22 +6813,22 @@ pub fn getFuncDecl(
65836813 .lbrace_column = key.lbrace_column,
65846814 .rbrace_column = key.rbrace_column,
65856815 });
6816 errdefer ip.extra.items.len = prev_extra_len;
65866817
6587 const adapter: KeyAdapter = .{ .intern_pool = ip };
6588 const gop = ip.map.getOrPutAssumeCapacityAdapted(Key{
6818 var gop = try ip.getOrPutKey(gpa, tid, .{
65896819 .func = extraFuncDecl(ip, func_decl_extra_index),
6590 }, adapter);
6591
6592 if (gop.found_existing) {
6820 });
6821 defer gop.deinit();
6822 if (gop == .existing) {
65936823 ip.extra.items.len = prev_extra_len;
6594 return @enumFromInt(gop.index);
6824 return gop.existing;
65956825 }
65966826
65976827 ip.items.appendAssumeCapacity(.{
65986828 .tag = .func_decl,
65996829 .data = func_decl_extra_index,
66006830 });
6601 return @enumFromInt(ip.items.len - 1);
6831 return gop.set(@enumFromInt(ip.items.len - 1));
66026832}
66036833
66046834pub const GetFuncDeclIesKey = struct {
......@@ -6626,7 +6856,7 @@ pub const GetFuncDeclIesKey = struct {
66266856pub fn getFuncDeclIes(
66276857 ip: *InternPool,
66286858 gpa: Allocator,
6629 _: Zcu.PerThread.Id,
6859 tid: Zcu.PerThread.Id,
66306860 key: GetFuncDeclIesKey,
66316861) Allocator.Error!Index {
66326862 // Validate input parameters.
......@@ -6639,7 +6869,6 @@ pub fn getFuncDeclIes(
66396869 const prev_extra_len = ip.extra.items.len;
66406870 const params_len: u32 = @intCast(key.param_types.len);
66416871
6642 try ip.map.ensureUnusedCapacity(gpa, 4);
66436872 try ip.extra.ensureUnusedCapacity(gpa, @typeInfo(Tag.FuncDecl).Struct.fields.len +
66446873 1 + // inferred_error_set
66456874 @typeInfo(Tag.ErrorUnionType).Struct.fields.len +
......@@ -6704,40 +6933,51 @@ pub fn getFuncDeclIes(
67046933 if (key.comptime_bits != 0) ip.extra.appendAssumeCapacity(key.comptime_bits);
67056934 if (key.noalias_bits != 0) ip.extra.appendAssumeCapacity(key.noalias_bits);
67066935 ip.extra.appendSliceAssumeCapacity(@ptrCast(key.param_types));
6936 errdefer {
6937 ip.items.len -= 4;
6938 ip.extra.items.len = prev_extra_len;
6939 }
67076940
67086941 ip.items.appendAssumeCapacity(.{
67096942 .tag = .type_function,
67106943 .data = func_type_extra_index,
67116944 });
67126945
6713 const adapter: KeyAdapter = .{ .intern_pool = ip };
6714 const gop = ip.map.getOrPutAssumeCapacityAdapted(Key{
6946 var gop = try ip.getOrPutKey(gpa, tid, .{
67156947 .func = extraFuncDecl(ip, func_decl_extra_index),
6716 }, adapter);
6717 if (!gop.found_existing) {
6718 assert(!ip.map.getOrPutAssumeCapacityAdapted(Key{ .error_union_type = .{
6719 .error_set_type = @enumFromInt(ip.items.len - 2),
6720 .payload_type = key.bare_return_type,
6721 } }, adapter).found_existing);
6722 assert(!ip.map.getOrPutAssumeCapacityAdapted(Key{
6723 .inferred_error_set_type = @enumFromInt(ip.items.len - 4),
6724 }, adapter).found_existing);
6725 assert(!ip.map.getOrPutAssumeCapacityAdapted(Key{
6726 .func_type = extraFuncType(ip, func_type_extra_index),
6727 }, adapter).found_existing);
6728 return @enumFromInt(ip.items.len - 4);
6729 }
6730
6731 // An existing function type was found; undo the additions to our two arrays.
6732 ip.items.len -= 4;
6733 ip.extra.items.len = prev_extra_len;
6734 return @enumFromInt(gop.index);
6948 });
6949 defer gop.deinit();
6950 if (gop == .existing) {
6951 // An existing function type was found; undo the additions to our two arrays.
6952 ip.items.len -= 4;
6953 ip.extra.items.len = prev_extra_len;
6954 return gop.existing;
6955 }
6956
6957 var eu_gop = try ip.getOrPutKey(gpa, tid, .{ .error_union_type = .{
6958 .error_set_type = @enumFromInt(ip.items.len - 2),
6959 .payload_type = key.bare_return_type,
6960 } });
6961 defer eu_gop.deinit();
6962 var ies_gop = try ip.getOrPutKey(gpa, tid, .{
6963 .inferred_error_set_type = @enumFromInt(ip.items.len - 4),
6964 });
6965 defer ies_gop.deinit();
6966 var ty_gop = try ip.getOrPutKey(gpa, tid, .{
6967 .func_type = extraFuncType(ip, func_type_extra_index),
6968 });
6969 defer ty_gop.deinit();
6970 const index = gop.set(@enumFromInt(ip.items.len - 4));
6971 _ = eu_gop.set(@enumFromInt(@intFromEnum(index) + 1));
6972 _ = ies_gop.set(@enumFromInt(@intFromEnum(index) + 2));
6973 _ = ty_gop.set(@enumFromInt(@intFromEnum(index) + 3));
6974 return index;
67356975}
67366976
67376977pub fn getErrorSetType(
67386978 ip: *InternPool,
67396979 gpa: Allocator,
6740 _: Zcu.PerThread.Id,
6980 tid: Zcu.PerThread.Id,
67416981 names: []const NullTerminatedString,
67426982) Allocator.Error!Index {
67436983 assert(std.sort.isSorted(NullTerminatedString, names, {}, NullTerminatedString.indexLessThan));
......@@ -6757,16 +6997,15 @@ pub fn getErrorSetType(
67576997 .names_map = predicted_names_map,
67586998 });
67596999 ip.extra.appendSliceAssumeCapacity(@ptrCast(names));
7000 errdefer ip.extra.items.len = prev_extra_len;
67607001
6761 const adapter: KeyAdapter = .{ .intern_pool = ip };
6762 const gop = try ip.map.getOrPutAdapted(gpa, Key{
7002 var gop = try ip.getOrPutKey(gpa, tid, .{
67637003 .error_set_type = extraErrorSet(ip, error_set_extra_index),
6764 }, adapter);
6765 errdefer _ = ip.map.pop();
6766
6767 if (gop.found_existing) {
7004 });
7005 defer gop.deinit();
7006 if (gop == .existing) {
67687007 ip.extra.items.len = prev_extra_len;
6769 return @enumFromInt(gop.index);
7008 return gop.existing;
67707009 }
67717010
67727011 try ip.items.append(gpa, .{
......@@ -6781,7 +7020,7 @@ pub fn getErrorSetType(
67817020
67827021 addStringsToMap(ip, names_map, names);
67837022
6784 return @enumFromInt(ip.items.len - 1);
7023 return gop.set(@enumFromInt(ip.items.len - 1));
67857024}
67867025
67877026pub const GetFuncInstanceKey = struct {
......@@ -6845,14 +7084,13 @@ pub fn getFuncInstance(
68457084 });
68467085 ip.extra.appendSliceAssumeCapacity(@ptrCast(arg.comptime_args));
68477086
6848 const gop = try ip.map.getOrPutAdapted(gpa, Key{
7087 var gop = try ip.getOrPutKey(gpa, tid, .{
68497088 .func = extraFuncInstance(ip, func_extra_index),
6850 }, KeyAdapter{ .intern_pool = ip });
6851 errdefer _ = ip.map.pop();
6852
6853 if (gop.found_existing) {
7089 });
7090 defer gop.deinit();
7091 if (gop == .existing) {
68547092 ip.extra.items.len = prev_extra_len;
6855 return @enumFromInt(gop.index);
7093 return gop.existing;
68567094 }
68577095
68587096 const func_index: Index = @enumFromInt(ip.items.len);
......@@ -6863,7 +7101,7 @@ pub fn getFuncInstance(
68637101 });
68647102 errdefer ip.items.len -= 1;
68657103
6866 return finishFuncInstance(
7104 return gop.set(try finishFuncInstance(
68677105 ip,
68687106 gpa,
68697107 tid,
......@@ -6872,7 +7110,7 @@ pub fn getFuncInstance(
68727110 func_extra_index,
68737111 arg.alignment,
68747112 arg.section,
6875 );
7113 ));
68767114}
68777115
68787116/// This function exists separately than `getFuncInstance` because it needs to
......@@ -6897,7 +7135,6 @@ pub fn getFuncInstanceIes(
68977135 const prev_extra_len = ip.extra.items.len;
68987136 const params_len: u32 = @intCast(arg.param_types.len);
68997137
6900 try ip.map.ensureUnusedCapacity(gpa, 4);
69017138 try ip.extra.ensureUnusedCapacity(gpa, @typeInfo(Tag.FuncInstance).Struct.fields.len +
69027139 1 + // inferred_error_set
69037140 arg.comptime_args.len +
......@@ -6970,30 +7207,37 @@ pub fn getFuncInstanceIes(
69707207 .tag = .type_function,
69717208 .data = func_type_extra_index,
69727209 });
7210 errdefer {
7211 ip.items.len -= 4;
7212 ip.extra.items.len = prev_extra_len;
7213 }
69737214
6974 const adapter: KeyAdapter = .{ .intern_pool = ip };
6975 const gop = ip.map.getOrPutAssumeCapacityAdapted(Key{
7215 var gop = try ip.getOrPutKey(gpa, tid, .{
69767216 .func = extraFuncInstance(ip, func_extra_index),
6977 }, adapter);
6978 if (gop.found_existing) {
7217 });
7218 defer gop.deinit();
7219 if (gop == .existing) {
69797220 // Hot path: undo the additions to our two arrays.
69807221 ip.items.len -= 4;
69817222 ip.extra.items.len = prev_extra_len;
6982 return @enumFromInt(gop.index);
7223 return gop.existing;
69837224 }
69847225
69857226 // Synchronize the map with items.
6986 assert(!ip.map.getOrPutAssumeCapacityAdapted(Key{ .error_union_type = .{
7227 var eu_gop = try ip.getOrPutKey(gpa, tid, .{ .error_union_type = .{
69877228 .error_set_type = error_set_type,
69887229 .payload_type = arg.bare_return_type,
6989 } }, adapter).found_existing);
6990 assert(!ip.map.getOrPutAssumeCapacityAdapted(Key{
7230 } });
7231 defer eu_gop.deinit();
7232 var ies_gop = try ip.getOrPutKey(gpa, tid, .{
69917233 .inferred_error_set_type = func_index,
6992 }, adapter).found_existing);
6993 assert(!ip.map.getOrPutAssumeCapacityAdapted(Key{
7234 });
7235 defer ies_gop.deinit();
7236 var ty_gop = try ip.getOrPutKey(gpa, tid, .{
69947237 .func_type = extraFuncType(ip, func_type_extra_index),
6995 }, adapter).found_existing);
6996 return finishFuncInstance(
7238 });
7239 defer ty_gop.deinit();
7240 const index = gop.set(try finishFuncInstance(
69977241 ip,
69987242 gpa,
69997243 tid,
......@@ -7002,7 +7246,11 @@ pub fn getFuncInstanceIes(
70027246 func_extra_index,
70037247 arg.alignment,
70047248 arg.section,
7005 );
7249 ));
7250 _ = eu_gop.set(@enumFromInt(@intFromEnum(index) + 1));
7251 _ = ies_gop.set(@enumFromInt(@intFromEnum(index) + 2));
7252 _ = ty_gop.set(@enumFromInt(@intFromEnum(index) + 3));
7253 return index;
70067254}
70077255
70087256fn finishFuncInstance(
......@@ -7135,11 +7383,10 @@ pub const WipEnumType = struct {
71357383pub fn getEnumType(
71367384 ip: *InternPool,
71377385 gpa: Allocator,
7138 _: Zcu.PerThread.Id,
7386 tid: Zcu.PerThread.Id,
71397387 ini: EnumTypeInit,
71407388) Allocator.Error!WipEnumType.Result {
7141 const adapter: KeyAdapter = .{ .intern_pool = ip };
7142 const gop = try ip.map.getOrPutAdapted(gpa, Key{ .enum_type = switch (ini.key) {
7389 var gop = try ip.getOrPutKey(gpa, tid, .{ .enum_type = switch (ini.key) {
71437390 .declared => |d| .{ .declared = .{
71447391 .zir_index = d.zir_index,
71457392 .captures = .{ .external = d.captures },
......@@ -7148,10 +7395,9 @@ pub fn getEnumType(
71487395 .zir_index = r.zir_index,
71497396 .type_hash = r.type_hash,
71507397 } },
7151 } }, adapter);
7152 if (gop.found_existing) return .{ .existing = @enumFromInt(gop.index) };
7153 assert(gop.index == ip.items.len);
7154 errdefer _ = ip.map.pop();
7398 } });
7399 defer gop.deinit();
7400 if (gop == .existing) return .{ .existing = gop.existing };
71557401
71567402 try ip.items.ensureUnusedCapacity(gpa, 1);
71577403
......@@ -7196,7 +7442,7 @@ pub fn getEnumType(
71967442 const names_start = ip.extra.items.len;
71977443 ip.extra.appendNTimesAssumeCapacity(undefined, ini.fields_len);
71987444 return .{ .wip = .{
7199 .index = @enumFromInt(gop.index),
7445 .index = gop.set(@enumFromInt(ip.items.len - 1)),
72007446 .tag_ty_index = extra_index + std.meta.fieldIndex(EnumAuto, "int_tag_type").?,
72017447 .decl_index = extra_index + std.meta.fieldIndex(EnumAuto, "decl").?,
72027448 .namespace_index = if (ini.has_namespace) extra_index + std.meta.fieldIndex(EnumAuto, "namespace").? else null,
......@@ -7260,7 +7506,7 @@ pub fn getEnumType(
72607506 ip.extra.appendNTimesAssumeCapacity(undefined, ini.fields_len);
72617507 }
72627508 return .{ .wip = .{
7263 .index = @enumFromInt(gop.index),
7509 .index = gop.set(@enumFromInt(ip.items.len - 1)),
72647510 .tag_ty_index = extra_index + std.meta.fieldIndex(EnumAuto, "int_tag_type").?,
72657511 .decl_index = extra_index + std.meta.fieldIndex(EnumAuto, "decl").?,
72667512 .namespace_index = if (ini.has_namespace) extra_index + std.meta.fieldIndex(EnumAuto, "namespace").? else null,
......@@ -7288,14 +7534,13 @@ const GeneratedTagEnumTypeInit = struct {
72887534pub fn getGeneratedTagEnumType(
72897535 ip: *InternPool,
72907536 gpa: Allocator,
7291 _: Zcu.PerThread.Id,
7537 tid: Zcu.PerThread.Id,
72927538 ini: GeneratedTagEnumTypeInit,
72937539) Allocator.Error!Index {
72947540 assert(ip.isUnion(ini.owner_union_ty));
72957541 assert(ip.isIntegerType(ini.tag_ty));
72967542 for (ini.values) |val| assert(ip.typeOf(val) == ini.tag_ty);
72977543
7298 try ip.map.ensureUnusedCapacity(gpa, 1);
72997544 try ip.items.ensureUnusedCapacity(gpa, 1);
73007545
73017546 const names_map = try ip.addMap(gpa, ini.names.len);
......@@ -7304,6 +7549,7 @@ pub fn getGeneratedTagEnumType(
73047549
73057550 const fields_len: u32 = @intCast(ini.names.len);
73067551
7552 const prev_extra_len = ip.extra.items.len;
73077553 switch (ini.tag_mode) {
73087554 .auto => {
73097555 try ip.extra.ensureUnusedCapacity(gpa, @typeInfo(EnumAuto).Struct.fields.len +
......@@ -7360,17 +7606,17 @@ pub fn getGeneratedTagEnumType(
73607606 ip.extra.appendSliceAssumeCapacity(@ptrCast(ini.values));
73617607 },
73627608 }
7363 // Same as above
7364 errdefer @compileError("error path leaks values_map and extra data");
7609 errdefer ip.extra.items.len = prev_extra_len;
7610 errdefer switch (ini.tag_mode) {
7611 .auto => {},
7612 .explicit, .nonexhaustive => _ = if (ini.values.len != 0) ip.maps.pop(),
7613 };
73657614
7366 // Capacity for this was ensured earlier
7367 const adapter: KeyAdapter = .{ .intern_pool = ip };
7368 const gop = ip.map.getOrPutAssumeCapacityAdapted(Key{ .enum_type = .{
7615 var gop = try ip.getOrPutKey(gpa, tid, .{ .enum_type = .{
73697616 .generated_tag = .{ .union_type = ini.owner_union_ty },
7370 } }, adapter);
7371 assert(!gop.found_existing);
7372 assert(gop.index == ip.items.len - 1);
7373 return @enumFromInt(gop.index);
7617 } });
7618 defer gop.deinit();
7619 return gop.set(@enumFromInt(ip.items.len - 1));
73747620}
73757621
73767622pub const OpaqueTypeInit = struct {
......@@ -7390,11 +7636,10 @@ pub const OpaqueTypeInit = struct {
73907636pub fn getOpaqueType(
73917637 ip: *InternPool,
73927638 gpa: Allocator,
7393 _: Zcu.PerThread.Id,
7639 tid: Zcu.PerThread.Id,
73947640 ini: OpaqueTypeInit,
73957641) Allocator.Error!WipNamespaceType.Result {
7396 const adapter: KeyAdapter = .{ .intern_pool = ip };
7397 const gop = try ip.map.getOrPutAdapted(gpa, Key{ .opaque_type = switch (ini.key) {
7642 var gop = try ip.getOrPutKey(gpa, tid, .{ .opaque_type = switch (ini.key) {
73987643 .declared => |d| .{ .declared = .{
73997644 .zir_index = d.zir_index,
74007645 .captures = .{ .external = d.captures },
......@@ -7403,9 +7648,9 @@ pub fn getOpaqueType(
74037648 .zir_index = r.zir_index,
74047649 .type_hash = 0,
74057650 } },
7406 } }, adapter);
7407 if (gop.found_existing) return .{ .existing = @enumFromInt(gop.index) };
7408 errdefer _ = ip.map.pop();
7651 } });
7652 defer gop.deinit();
7653 if (gop == .existing) return .{ .existing = gop.existing };
74097654 try ip.items.ensureUnusedCapacity(gpa, 1);
74107655 try ip.extra.ensureUnusedCapacity(gpa, @typeInfo(Tag.TypeOpaque).Struct.fields.len + switch (ini.key) {
74117656 .declared => |d| d.captures.len,
......@@ -7431,7 +7676,7 @@ pub fn getOpaqueType(
74317676 .reified => {},
74327677 }
74337678 return .{ .wip = .{
7434 .index = @enumFromInt(gop.index),
7679 .index = gop.set(@enumFromInt(ip.items.len - 1)),
74357680 .decl_extra_index = extra_index + std.meta.fieldIndex(Tag.TypeOpaque, "decl").?,
74367681 .namespace_extra_index = if (ini.has_namespace)
74377682 extra_index + std.meta.fieldIndex(Tag.TypeOpaque, "namespace").?
......@@ -7441,9 +7686,19 @@ pub fn getOpaqueType(
74417686}
74427687
74437688pub fn getIfExists(ip: *const InternPool, key: Key) ?Index {
7444 const adapter: KeyAdapter = .{ .intern_pool = ip };
7445 const index = ip.map.getIndexAdapted(key, adapter) orelse return null;
7446 return @enumFromInt(index);
7689 const full_hash = key.hash64(ip);
7690 const hash: u32 = @truncate(full_hash >> 32);
7691 const shard = &ip.shards[@intCast(full_hash & (ip.shards.len - 1))];
7692 const map = shard.map.acquire();
7693 const map_mask = map.header().mask();
7694 var map_index = hash;
7695 while (true) : (map_index += Shard.Map.Entry.fields_len) {
7696 map_index &= map_mask;
7697 const entry = map.at(map_index);
7698 const index = entry.acquire();
7699 if (index == .none) return null;
7700 if (entry.hash == hash and ip.indexToKey(index).eql(key, ip)) return index;
7701 }
74477702}
74487703
74497704pub fn getAssumeExists(ip: *const InternPool, key: Key) Index {
......@@ -7506,7 +7761,6 @@ pub fn remove(ip: *InternPool, index: Index) void {
75067761 if (@intFromEnum(index) == ip.items.len - 1) {
75077762 // Happy case - we can just drop the item without affecting any other indices.
75087763 ip.items.len -= 1;
7509 _ = ip.map.pop();
75107764 } else {
75117765 // We must preserve the item so that indices following it remain valid.
75127766 // Thus, we will rewrite the tag to `removed`, leaking the item until
......@@ -8133,35 +8387,34 @@ fn getCoercedFuncInstance(
81338387fn getCoercedFunc(
81348388 ip: *InternPool,
81358389 gpa: Allocator,
8136 _: Zcu.PerThread.Id,
8390 tid: Zcu.PerThread.Id,
81378391 func: Index,
81388392 ty: Index,
81398393) Allocator.Error!Index {
81408394 const prev_extra_len = ip.extra.items.len;
81418395 try ip.extra.ensureUnusedCapacity(gpa, @typeInfo(Tag.FuncCoerced).Struct.fields.len);
81428396 try ip.items.ensureUnusedCapacity(gpa, 1);
8143 try ip.map.ensureUnusedCapacity(gpa, 1);
81448397
81458398 const extra_index = ip.addExtraAssumeCapacity(Tag.FuncCoerced{
81468399 .ty = ty,
81478400 .func = func,
81488401 });
8402 errdefer ip.extra.items.len = prev_extra_len;
81498403
8150 const adapter: KeyAdapter = .{ .intern_pool = ip };
8151 const gop = ip.map.getOrPutAssumeCapacityAdapted(Key{
8404 var gop = try ip.getOrPutKey(gpa, tid, .{
81528405 .func = extraFuncCoerced(ip, extra_index),
8153 }, adapter);
8154
8155 if (gop.found_existing) {
8406 });
8407 defer gop.deinit();
8408 if (gop == .existing) {
81568409 ip.extra.items.len = prev_extra_len;
8157 return @enumFromInt(gop.index);
8410 return gop.existing;
81588411 }
81598412
81608413 ip.items.appendAssumeCapacity(.{
81618414 .tag = .func_coerced,
81628415 .data = extra_index,
81638416 });
8164 return @enumFromInt(ip.items.len - 1);
8417 return gop.set(@enumFromInt(ip.items.len - 1));
81658418}
81668419
81678420/// Asserts `val` has an integer type.
src/Zcu.zig+2-2
......@@ -2394,9 +2394,9 @@ pub const CompileError = error{
23942394 ComptimeBreak,
23952395};
23962396
2397pub fn init(mod: *Module) !void {
2397pub fn init(mod: *Module, thread_count: usize) !void {
23982398 const gpa = mod.gpa;
2399 try mod.intern_pool.init(gpa);
2399 try mod.intern_pool.init(gpa, thread_count);
24002400 try mod.global_error_set.put(gpa, .empty, {});
24012401}
24022402