| ... | @@ -2,9 +2,10 @@ | ... | @@ -2,9 +2,10 @@ |
| 2 | //! This data structure is self-contained, with the following exceptions: | 2 | //! This data structure is self-contained, with the following exceptions: |
| 3 | //! * Module.Namespace has a pointer to Module.File | 3 | //! * Module.Namespace has a pointer to Module.File |
| 4 | | 4 | |
| 5 | /// Maps `Key` to `Index`. `Key` objects are not stored anywhere; they are | 5 | local: []Local = &.{}, |
| 6 | /// constructed lazily. | 6 | shard_shift: std.math.Log2Int(usize) = 0, |
| 7 | map: std.AutoArrayHashMapUnmanaged(void, void) = .{}, | 7 | shards: []Shard = &.{}, |
| | 8 | |
| 8 | items: std.MultiArrayList(Item) = .{}, | 9 | items: std.MultiArrayList(Item) = .{}, |
| 9 | extra: std.ArrayListUnmanaged(u32) = .{}, | 10 | extra: std.ArrayListUnmanaged(u32) = .{}, |
| 10 | /// On 32-bit systems, this array is ignored and extra is used for everything. | 11 | /// On 32-bit systems, this array is ignored and extra is used for everything. |
| ... | @@ -351,6 +352,115 @@ pub const DepEntry = extern struct { | ... | @@ -351,6 +352,115 @@ pub const DepEntry = extern struct { |
| 351 | }; | 352 | }; |
| 352 | }; | 353 | }; |
| 353 | | 354 | |
| | 355 | const Local = struct { |
| | 356 | aligned: void align(std.atomic.cache_line) = {}, |
| | 357 | |
| | 358 | /// node: Garbage.Node, |
| | 359 | /// header: List.Header, |
| | 360 | /// data: [capacity]u32, |
| | 361 | /// tag: [capacity]Tag, |
| | 362 | items: List, |
| | 363 | |
| | 364 | /// node: Garbage.Node, |
| | 365 | /// header: List.Header, |
| | 366 | /// extra: [capacity]u32, |
| | 367 | extra: List, |
| | 368 | |
| | 369 | garbage: Garbage, |
| | 370 | |
| | 371 | const List = struct { |
| | 372 | entries: [*]u32, |
| | 373 | |
| | 374 | const empty: List = .{ |
| | 375 | .entries = @constCast(&[_]u32{ 0, 0 })[Header.fields_len..].ptr, |
| | 376 | }; |
| | 377 | |
| | 378 | fn acquire(list: *const List) List { |
| | 379 | return .{ .entries = @atomicLoad([*]u32, &list.entries, .acquire) }; |
| | 380 | } |
| | 381 | fn release(list: *List, new_list: List) void { |
| | 382 | @atomicStore([*]u32, &list.entries, new_list.entries, .release); |
| | 383 | } |
| | 384 | |
| | 385 | const Header = extern struct { |
| | 386 | len: u32, |
| | 387 | capacity: u32, |
| | 388 | |
| | 389 | const fields_len = @typeInfo(Header).Struct.fields.len; |
| | 390 | }; |
| | 391 | fn header(list: List) *Header { |
| | 392 | return @ptrCast(list.entries - Header.fields_len); |
| | 393 | } |
| | 394 | }; |
| | 395 | |
| | 396 | const Garbage = std.SinglyLinkedList(struct { buf_len: usize }); |
| | 397 | const garbage_align = @max(@alignOf(Garbage.Node), @alignOf(u32)); |
| | 398 | |
| | 399 | fn freeGarbage(garbage: *const Garbage.Node, gpa: Allocator) void { |
| | 400 | gpa.free(@as([*]align(Local.garbage_align) const u8, @ptrCast(garbage))[0..garbage.data.buf_len]); |
| | 401 | } |
| | 402 | }; |
| | 403 | |
| | 404 | const Shard = struct { |
| | 405 | aligned: void align(std.atomic.cache_line) = {}, |
| | 406 | |
| | 407 | mutate_mutex: std.Thread.Mutex.Recursive, |
| | 408 | |
| | 409 | /// node: Local.Garbage.Node, |
| | 410 | /// header: Map.Header, |
| | 411 | /// entries: [capacity]Map.Entry, |
| | 412 | map: Map, |
| | 413 | |
| | 414 | const Map = struct { |
| | 415 | entries: [*]u32, |
| | 416 | |
| | 417 | const empty: Map = .{ |
| | 418 | .entries = @constCast(&[_]u32{ 0, 1, @intFromEnum(Index.none), 0 })[Header.fields_len..].ptr, |
| | 419 | }; |
| | 420 | |
| | 421 | fn acquire(map: *const Map) Map { |
| | 422 | return .{ .entries = @atomicLoad([*]u32, &map.entries, .acquire) }; |
| | 423 | } |
| | 424 | fn release(map: *Map, new_map: Map) void { |
| | 425 | @atomicStore([*]u32, &map.entries, new_map.entries, .release); |
| | 426 | } |
| | 427 | |
| | 428 | const Header = extern struct { |
| | 429 | len: u32, |
| | 430 | capacity: u32, |
| | 431 | |
| | 432 | const fields_len: u32 = @typeInfo(Header).Struct.fields.len; |
| | 433 | |
| | 434 | fn mask(head: *const Header) u32 { |
| | 435 | assert(std.math.isPowerOfTwo(head.capacity)); |
| | 436 | assert(std.math.isPowerOfTwo(Entry.fields_len)); |
| | 437 | return (head.capacity - 1) * Entry.fields_len; |
| | 438 | } |
| | 439 | }; |
| | 440 | fn header(map: Map) *Header { |
| | 441 | return @ptrCast(map.entries - Header.fields_len); |
| | 442 | } |
| | 443 | |
| | 444 | const Entry = extern struct { |
| | 445 | index: Index, |
| | 446 | hash: u32, |
| | 447 | |
| | 448 | const fields_len: u32 = @typeInfo(Entry).Struct.fields.len; |
| | 449 | |
| | 450 | fn acquire(entry: *const Entry) Index { |
| | 451 | return @atomicLoad(Index, &entry.index, .acquire); |
| | 452 | } |
| | 453 | fn release(entry: *Entry, index: Index) void { |
| | 454 | @atomicStore(Index, &entry.index, index, .release); |
| | 455 | } |
| | 456 | }; |
| | 457 | fn at(map: Map, index: usize) *Entry { |
| | 458 | assert(index % Entry.fields_len == 0); |
| | 459 | return @ptrCast(&map.entries[index]); |
| | 460 | } |
| | 461 | }; |
| | 462 | }; |
| | 463 | |
| 354 | const FieldMap = std.ArrayHashMapUnmanaged(void, void, std.array_hash_map.AutoContext(void), false); | 464 | const FieldMap = std.ArrayHashMapUnmanaged(void, void, std.array_hash_map.AutoContext(void), false); |
| 355 | | 465 | |
| 356 | const builtin = @import("builtin"); | 466 | const builtin = @import("builtin"); |
| ... | @@ -369,20 +479,6 @@ const Zcu = @import("Zcu.zig"); | ... | @@ -369,20 +479,6 @@ const Zcu = @import("Zcu.zig"); |
| 369 | const Module = Zcu; | 479 | const Module = Zcu; |
| 370 | const Zir = std.zig.Zir; | 480 | const Zir = std.zig.Zir; |
| 371 | | 481 | |
| 372 | const KeyAdapter = struct { | | |
| 373 | intern_pool: *const InternPool, | | |
| 374 | | | |
| 375 | pub fn eql(ctx: @This(), a: Key, b_void: void, b_map_index: usize) bool { | | |
| 376 | _ = b_void; | | |
| 377 | if (ctx.intern_pool.items.items(.tag)[b_map_index] == .removed) return false; | | |
| 378 | return ctx.intern_pool.indexToKey(@enumFromInt(b_map_index)).eql(a, ctx.intern_pool); | | |
| 379 | } | | |
| 380 | | | |
| 381 | pub fn hash(ctx: @This(), a: Key) u32 { | | |
| 382 | return a.hash32(ctx.intern_pool); | | |
| 383 | } | | |
| 384 | }; | | |
| 385 | | | |
| 386 | /// An index into `maps` which might be `none`. | 482 | /// An index into `maps` which might be `none`. |
| 387 | pub const OptionalMapIndex = enum(u32) { | 483 | pub const OptionalMapIndex = enum(u32) { |
| 388 | none = std.math.maxInt(u32), | 484 | none = std.math.maxInt(u32), |
| ... | @@ -4535,17 +4631,27 @@ pub const MemoizedCall = struct { | ... | @@ -4535,17 +4631,27 @@ pub const MemoizedCall = struct { |
| 4535 | result: Index, | 4631 | result: Index, |
| 4536 | }; | 4632 | }; |
| 4537 | | 4633 | |
| 4538 | pub fn init(ip: *InternPool, gpa: Allocator) !void { | 4634 | pub fn init(ip: *InternPool, gpa: Allocator, total_threads: usize) !void { |
| | 4635 | errdefer ip.deinit(gpa); |
| 4539 | assert(ip.items.len == 0); | 4636 | assert(ip.items.len == 0); |
| 4540 | | 4637 | |
| | 4638 | ip.local = try gpa.alloc(Local, total_threads); |
| | 4639 | @memset(ip.local, .{ |
| | 4640 | .items = Local.List.empty, |
| | 4641 | .extra = Local.List.empty, |
| | 4642 | .garbage = .{}, |
| | 4643 | }); |
| | 4644 | |
| | 4645 | ip.shard_shift = @intCast(std.math.log2_int_ceil(usize, total_threads)); |
| | 4646 | ip.shards = try gpa.alloc(Shard, @as(usize, 1) << ip.shard_shift); |
| | 4647 | @memset(ip.shards, .{ |
| | 4648 | .mutate_mutex = std.Thread.Mutex.Recursive.init, |
| | 4649 | .map = Shard.Map.empty, |
| | 4650 | }); |
| | 4651 | |
| 4541 | // Reserve string index 0 for an empty string. | 4652 | // Reserve string index 0 for an empty string. |
| 4542 | assert((try ip.getOrPutString(gpa, .main, "", .no_embedded_nulls)) == .empty); | 4653 | assert((try ip.getOrPutString(gpa, .main, "", .no_embedded_nulls)) == .empty); |
| 4543 | | 4654 | |
| 4544 | // So that we can use `catch unreachable` below. | | |
| 4545 | try ip.items.ensureUnusedCapacity(gpa, static_keys.len); | | |
| 4546 | try ip.map.ensureUnusedCapacity(gpa, static_keys.len); | | |
| 4547 | try ip.extra.ensureUnusedCapacity(gpa, static_keys.len); | | |
| 4548 | | | |
| 4549 | // This inserts all the statically-known values into the intern pool in the | 4655 | // This inserts all the statically-known values into the intern pool in the |
| 4550 | // order expected. | 4656 | // order expected. |
| 4551 | for (&static_keys, 0..) |key, key_index| switch (@as(Index, @enumFromInt(key_index))) { | 4657 | for (&static_keys, 0..) |key, key_index| switch (@as(Index, @enumFromInt(key_index))) { |
| ... | @@ -4574,12 +4680,9 @@ pub fn init(ip: *InternPool, gpa: Allocator) !void { | ... | @@ -4574,12 +4680,9 @@ pub fn init(ip: *InternPool, gpa: Allocator) !void { |
| 4574 | assert(ip.indexToKey(ip.typeOf(cc_inline)).int_type.bits == | 4680 | assert(ip.indexToKey(ip.typeOf(cc_inline)).int_type.bits == |
| 4575 | @typeInfo(@typeInfo(std.builtin.CallingConvention).Enum.tag_type).Int.bits); | 4681 | @typeInfo(@typeInfo(std.builtin.CallingConvention).Enum.tag_type).Int.bits); |
| 4576 | } | 4682 | } |
| 4577 | | | |
| 4578 | assert(ip.items.len == static_keys.len); | | |
| 4579 | } | 4683 | } |
| 4580 | | 4684 | |
| 4581 | pub fn deinit(ip: *InternPool, gpa: Allocator) void { | 4685 | pub fn deinit(ip: *InternPool, gpa: Allocator) void { |
| 4582 | ip.map.deinit(gpa); | | |
| 4583 | ip.items.deinit(gpa); | 4686 | ip.items.deinit(gpa); |
| 4584 | ip.extra.deinit(gpa); | 4687 | ip.extra.deinit(gpa); |
| 4585 | ip.limbs.deinit(gpa); | 4688 | ip.limbs.deinit(gpa); |
| ... | @@ -4611,6 +4714,16 @@ pub fn deinit(ip: *InternPool, gpa: Allocator) void { | ... | @@ -4611,6 +4714,16 @@ pub fn deinit(ip: *InternPool, gpa: Allocator) void { |
| 4611 | | 4714 | |
| 4612 | ip.files.deinit(gpa); | 4715 | ip.files.deinit(gpa); |
| 4613 | | 4716 | |
| | 4717 | gpa.free(ip.shards); |
| | 4718 | for (ip.local) |*local| { |
| | 4719 | var next = local.garbage.first; |
| | 4720 | while (next) |cur| { |
| | 4721 | next = cur.next; |
| | 4722 | Local.freeGarbage(cur, gpa); |
| | 4723 | } |
| | 4724 | } |
| | 4725 | gpa.free(ip.local); |
| | 4726 | |
| 4614 | ip.* = undefined; | 4727 | ip.* = undefined; |
| 4615 | } | 4728 | } |
| 4616 | | 4729 | |
| ... | @@ -5239,10 +5352,133 @@ fn indexToKeyBigInt(ip: *const InternPool, limb_index: u32, positive: bool) Key | ... | @@ -5239,10 +5352,133 @@ fn indexToKeyBigInt(ip: *const InternPool, limb_index: u32, positive: bool) Key |
| 5239 | } }; | 5352 | } }; |
| 5240 | } | 5353 | } |
| 5241 | | 5354 | |
| | 5355 | const GetOrPutKey = union(enum) { |
| | 5356 | existing: Index, |
| | 5357 | new: struct { |
| | 5358 | shard: *Shard, |
| | 5359 | map_index: u32, |
| | 5360 | }, |
| | 5361 | |
| | 5362 | fn set(gop: *GetOrPutKey, index: Index) Index { |
| | 5363 | switch (gop.*) { |
| | 5364 | .existing => unreachable, |
| | 5365 | .new => |info| { |
| | 5366 | info.shard.map.at(info.map_index).release(index); |
| | 5367 | info.shard.map.header().len += 1; |
| | 5368 | info.shard.mutate_mutex.unlock(); |
| | 5369 | }, |
| | 5370 | } |
| | 5371 | gop.* = .{ .existing = index }; |
| | 5372 | return index; |
| | 5373 | } |
| | 5374 | |
| | 5375 | fn assign(gop: *GetOrPutKey, new_gop: GetOrPutKey) void { |
| | 5376 | gop.deinit(); |
| | 5377 | gop.* = new_gop; |
| | 5378 | } |
| | 5379 | |
| | 5380 | fn deinit(gop: *GetOrPutKey) void { |
| | 5381 | switch (gop.*) { |
| | 5382 | .existing => {}, |
| | 5383 | .new => |info| info.shard.mutate_mutex.unlock(), |
| | 5384 | } |
| | 5385 | gop.* = undefined; |
| | 5386 | } |
| | 5387 | }; |
| | 5388 | fn getOrPutKey( |
| | 5389 | ip: *InternPool, |
| | 5390 | gpa: Allocator, |
| | 5391 | tid: Zcu.PerThread.Id, |
| | 5392 | key: Key, |
| | 5393 | ) Allocator.Error!GetOrPutKey { |
| | 5394 | const full_hash = key.hash64(ip); |
| | 5395 | const hash: u32 = @truncate(full_hash >> 32); |
| | 5396 | const shard = &ip.shards[@intCast(full_hash & (ip.shards.len - 1))]; |
| | 5397 | var map = shard.map.acquire(); |
| | 5398 | var map_mask = map.header().mask(); |
| | 5399 | var map_index = hash; |
| | 5400 | while (true) : (map_index += Shard.Map.Entry.fields_len) { |
| | 5401 | map_index &= map_mask; |
| | 5402 | const entry = map.at(map_index); |
| | 5403 | const index = entry.acquire(); |
| | 5404 | if (index == .none) break; |
| | 5405 | if (entry.hash == hash and ip.indexToKey(index).eql(key, ip)) |
| | 5406 | return .{ .existing = index }; |
| | 5407 | } |
| | 5408 | shard.mutate_mutex.lock(); |
| | 5409 | errdefer shard.mutate_mutex.unlock(); |
| | 5410 | if (map.entries != shard.map.entries) { |
| | 5411 | map = shard.map; |
| | 5412 | map_mask = map.header().mask(); |
| | 5413 | map_index = hash; |
| | 5414 | } |
| | 5415 | while (true) : (map_index += Shard.Map.Entry.fields_len) { |
| | 5416 | map_index &= map_mask; |
| | 5417 | const entry = map.at(map_index); |
| | 5418 | const index = entry.index; |
| | 5419 | if (index == .none) break; |
| | 5420 | if (entry.hash == hash and ip.indexToKey(index).eql(key, ip)) { |
| | 5421 | defer shard.mutate_mutex.unlock(); |
| | 5422 | return .{ .existing = index }; |
| | 5423 | } |
| | 5424 | } |
| | 5425 | const map_header = map.header().*; |
| | 5426 | if (map_header.len >= map_header.capacity * 3 / 5) { |
| | 5427 | const new_map_capacity = map_header.capacity * 2; |
| | 5428 | const new_map_buf = try gpa.alignedAlloc( |
| | 5429 | u8, |
| | 5430 | Local.garbage_align, |
| | 5431 | @sizeOf(Local.Garbage.Node) + (Shard.Map.Header.fields_len + |
| | 5432 | new_map_capacity * Shard.Map.Entry.fields_len) * @sizeOf(u32), |
| | 5433 | ); |
| | 5434 | const new_node: *Local.Garbage.Node = @ptrCast(new_map_buf.ptr); |
| | 5435 | new_node.* = .{ .data = .{ .buf_len = new_map_buf.len } }; |
| | 5436 | ip.local[@intFromEnum(tid)].garbage.prepend(new_node); |
| | 5437 | const new_map_entries = std.mem.bytesAsSlice( |
| | 5438 | u32, |
| | 5439 | new_map_buf[@sizeOf(Local.Garbage.Node)..], |
| | 5440 | )[Shard.Map.Header.fields_len..]; |
| | 5441 | const new_map: Shard.Map = .{ .entries = new_map_entries.ptr }; |
| | 5442 | new_map.header().* = .{ |
| | 5443 | .len = map_header.len, |
| | 5444 | .capacity = new_map_capacity, |
| | 5445 | }; |
| | 5446 | @memset(new_map_entries, @intFromEnum(Index.none)); |
| | 5447 | const new_map_mask = new_map.header().mask(); |
| | 5448 | map_index = 0; |
| | 5449 | while (map_index < map_header.capacity * 2) : (map_index += Shard.Map.Entry.fields_len) { |
| | 5450 | const entry = map.at(map_index); |
| | 5451 | const index = entry.index; |
| | 5452 | if (index == .none) continue; |
| | 5453 | const item_hash = entry.hash; |
| | 5454 | var new_map_index = item_hash; |
| | 5455 | while (true) : (new_map_index += Shard.Map.Entry.fields_len) { |
| | 5456 | new_map_index &= new_map_mask; |
| | 5457 | const new_entry = new_map.at(new_map_index); |
| | 5458 | if (new_entry.index != .none) continue; |
| | 5459 | new_entry.* = .{ |
| | 5460 | .index = index, |
| | 5461 | .hash = item_hash, |
| | 5462 | }; |
| | 5463 | break; |
| | 5464 | } |
| | 5465 | } |
| | 5466 | map = new_map; |
| | 5467 | map_index = hash; |
| | 5468 | while (true) : (map_index += Shard.Map.Entry.fields_len) { |
| | 5469 | map_index &= new_map_mask; |
| | 5470 | if (map.at(map_index).index == .none) break; |
| | 5471 | } |
| | 5472 | shard.map.release(new_map); |
| | 5473 | } |
| | 5474 | map.at(map_index).hash = hash; |
| | 5475 | return .{ .new = .{ .shard = shard, .map_index = map_index } }; |
| | 5476 | } |
| | 5477 | |
| 5242 | pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) Allocator.Error!Index { | 5478 | pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) Allocator.Error!Index { |
| 5243 | const adapter: KeyAdapter = .{ .intern_pool = ip }; | 5479 | var gop = try ip.getOrPutKey(gpa, tid, key); |
| 5244 | const gop = try ip.map.getOrPutAdapted(gpa, key, adapter); | 5480 | defer gop.deinit(); |
| 5245 | if (gop.found_existing) return @enumFromInt(gop.index); | 5481 | if (gop == .existing) return gop.existing; |
| 5246 | try ip.items.ensureUnusedCapacity(gpa, 1); | 5482 | try ip.items.ensureUnusedCapacity(gpa, 1); |
| 5247 | switch (key) { | 5483 | switch (key) { |
| 5248 | .int_type => |int_type| { | 5484 | .int_type => |int_type| { |
| ... | @@ -5260,18 +5496,17 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All | ... | @@ -5260,18 +5496,17 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All |
| 5260 | assert(ptr_type.sentinel == .none or ip.typeOf(ptr_type.sentinel) == ptr_type.child); | 5496 | assert(ptr_type.sentinel == .none or ip.typeOf(ptr_type.sentinel) == ptr_type.child); |
| 5261 | | 5497 | |
| 5262 | if (ptr_type.flags.size == .Slice) { | 5498 | if (ptr_type.flags.size == .Slice) { |
| 5263 | _ = ip.map.pop(); | | |
| 5264 | var new_key = key; | 5499 | var new_key = key; |
| 5265 | new_key.ptr_type.flags.size = .Many; | 5500 | new_key.ptr_type.flags.size = .Many; |
| 5266 | const ptr_type_index = try ip.get(gpa, tid, new_key); | 5501 | const ptr_type_index = try ip.get(gpa, tid, new_key); |
| 5267 | assert(!(try ip.map.getOrPutAdapted(gpa, key, adapter)).found_existing); | 5502 | gop.assign(try ip.getOrPutKey(gpa, tid, key)); |
| 5268 | | 5503 | |
| 5269 | try ip.items.ensureUnusedCapacity(gpa, 1); | 5504 | try ip.items.ensureUnusedCapacity(gpa, 1); |
| 5270 | ip.items.appendAssumeCapacity(.{ | 5505 | ip.items.appendAssumeCapacity(.{ |
| 5271 | .tag = .type_slice, | 5506 | .tag = .type_slice, |
| 5272 | .data = @intFromEnum(ptr_type_index), | 5507 | .data = @intFromEnum(ptr_type_index), |
| 5273 | }); | 5508 | }); |
| 5274 | return @enumFromInt(ip.items.len - 1); | 5509 | return gop.set(@enumFromInt(ip.items.len - 1)); |
| 5275 | } | 5510 | } |
| 5276 | | 5511 | |
| 5277 | var ptr_type_adjusted = ptr_type; | 5512 | var ptr_type_adjusted = ptr_type; |
| ... | @@ -5295,7 +5530,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All | ... | @@ -5295,7 +5530,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All |
| 5295 | .child = array_type.child, | 5530 | .child = array_type.child, |
| 5296 | }), | 5531 | }), |
| 5297 | }); | 5532 | }); |
| 5298 | return @enumFromInt(ip.items.len - 1); | 5533 | return gop.set(@enumFromInt(ip.items.len - 1)); |
| 5299 | } | 5534 | } |
| 5300 | } | 5535 | } |
| 5301 | | 5536 | |
| ... | @@ -5442,11 +5677,10 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All | ... | @@ -5442,11 +5677,10 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All |
| 5442 | }, | 5677 | }, |
| 5443 | .anon_decl => |anon_decl| if (ptrsHaveSameAlignment(ip, ptr.ty, ptr_type, anon_decl.orig_ty)) item: { | 5678 | .anon_decl => |anon_decl| if (ptrsHaveSameAlignment(ip, ptr.ty, ptr_type, anon_decl.orig_ty)) item: { |
| 5444 | if (ptr.ty != anon_decl.orig_ty) { | 5679 | if (ptr.ty != anon_decl.orig_ty) { |
| 5445 | _ = ip.map.pop(); | | |
| 5446 | var new_key = key; | 5680 | var new_key = key; |
| 5447 | new_key.ptr.base_addr.anon_decl.orig_ty = ptr.ty; | 5681 | new_key.ptr.base_addr.anon_decl.orig_ty = ptr.ty; |
| 5448 | const new_gop = try ip.map.getOrPutAdapted(gpa, new_key, adapter); | 5682 | gop.assign(try ip.getOrPutKey(gpa, tid, new_key)); |
| 5449 | if (new_gop.found_existing) return @enumFromInt(new_gop.index); | 5683 | if (gop == .existing) return gop.existing; |
| 5450 | } | 5684 | } |
| 5451 | break :item .{ | 5685 | break :item .{ |
| 5452 | .tag = .ptr_anon_decl, | 5686 | .tag = .ptr_anon_decl, |
| ... | @@ -5486,7 +5720,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All | ... | @@ -5486,7 +5720,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All |
| 5486 | .tag = .ptr_int, | 5720 | .tag = .ptr_int, |
| 5487 | .data = try ip.addExtra(gpa, PtrInt.init(ptr.ty, ptr.byte_offset)), | 5721 | .data = try ip.addExtra(gpa, PtrInt.init(ptr.ty, ptr.byte_offset)), |
| 5488 | }, | 5722 | }, |
| 5489 | .arr_elem, .field => |base_index| item: { | 5723 | .arr_elem, .field => |base_index| { |
| 5490 | const base_ptr_type = ip.indexToKey(ip.typeOf(base_index.base)).ptr_type; | 5724 | const base_ptr_type = ip.indexToKey(ip.typeOf(base_index.base)).ptr_type; |
| 5491 | switch (ptr.base_addr) { | 5725 | switch (ptr.base_addr) { |
| 5492 | .arr_elem => assert(base_ptr_type.flags.size == .Many), | 5726 | .arr_elem => assert(base_ptr_type.flags.size == .Many), |
| ... | @@ -5516,21 +5750,21 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All | ... | @@ -5516,21 +5750,21 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All |
| 5516 | }, | 5750 | }, |
| 5517 | else => unreachable, | 5751 | else => unreachable, |
| 5518 | } | 5752 | } |
| 5519 | _ = ip.map.pop(); | | |
| 5520 | const index_index = try ip.get(gpa, tid, .{ .int = .{ | 5753 | const index_index = try ip.get(gpa, tid, .{ .int = .{ |
| 5521 | .ty = .usize_type, | 5754 | .ty = .usize_type, |
| 5522 | .storage = .{ .u64 = base_index.index }, | 5755 | .storage = .{ .u64 = base_index.index }, |
| 5523 | } }); | 5756 | } }); |
| 5524 | assert(!(try ip.map.getOrPutAdapted(gpa, key, adapter)).found_existing); | 5757 | gop.assign(try ip.getOrPutKey(gpa, tid, key)); |
| 5525 | try ip.items.ensureUnusedCapacity(gpa, 1); | 5758 | try ip.items.ensureUnusedCapacity(gpa, 1); |
| 5526 | break :item .{ | 5759 | ip.items.appendAssumeCapacity(.{ |
| 5527 | .tag = switch (ptr.base_addr) { | 5760 | .tag = switch (ptr.base_addr) { |
| 5528 | .arr_elem => .ptr_elem, | 5761 | .arr_elem => .ptr_elem, |
| 5529 | .field => .ptr_field, | 5762 | .field => .ptr_field, |
| 5530 | else => unreachable, | 5763 | else => unreachable, |
| 5531 | }, | 5764 | }, |
| 5532 | .data = try ip.addExtra(gpa, PtrBaseIndex.init(ptr.ty, base_index.base, index_index, ptr.byte_offset)), | 5765 | .data = try ip.addExtra(gpa, PtrBaseIndex.init(ptr.ty, base_index.base, index_index, ptr.byte_offset)), |
| 5533 | }; | 5766 | }); |
| | 5767 | return gop.set(@enumFromInt(ip.items.len - 1)); |
| 5534 | }, | 5768 | }, |
| 5535 | }); | 5769 | }); |
| 5536 | }, | 5770 | }, |
| ... | @@ -5566,7 +5800,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All | ... | @@ -5566,7 +5800,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All |
| 5566 | .lazy_ty = lazy_ty, | 5800 | .lazy_ty = lazy_ty, |
| 5567 | }), | 5801 | }), |
| 5568 | }); | 5802 | }); |
| 5569 | return @enumFromInt(ip.items.len - 1); | 5803 | return gop.set(@enumFromInt(ip.items.len - 1)); |
| 5570 | }, | 5804 | }, |
| 5571 | } | 5805 | } |
| 5572 | switch (int.ty) { | 5806 | switch (int.ty) { |
| ... | @@ -5707,7 +5941,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All | ... | @@ -5707,7 +5941,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All |
| 5707 | .value = casted, | 5941 | .value = casted, |
| 5708 | }), | 5942 | }), |
| 5709 | }); | 5943 | }); |
| 5710 | return @enumFromInt(ip.items.len - 1); | 5944 | return gop.set(@enumFromInt(ip.items.len - 1)); |
| 5711 | } else |_| {} | 5945 | } else |_| {} |
| 5712 | | 5946 | |
| 5713 | const tag: Tag = if (big_int.positive) .int_positive else .int_negative; | 5947 | const tag: Tag = if (big_int.positive) .int_positive else .int_negative; |
| ... | @@ -5722,7 +5956,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All | ... | @@ -5722,7 +5956,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All |
| 5722 | .value = casted, | 5956 | .value = casted, |
| 5723 | }), | 5957 | }), |
| 5724 | }); | 5958 | }); |
| 5725 | return @enumFromInt(ip.items.len - 1); | 5959 | return gop.set(@enumFromInt(ip.items.len - 1)); |
| 5726 | } | 5960 | } |
| 5727 | | 5961 | |
| 5728 | var buf: [2]Limb = undefined; | 5962 | var buf: [2]Limb = undefined; |
| ... | @@ -5881,7 +6115,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All | ... | @@ -5881,7 +6115,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All |
| 5881 | .tag = .only_possible_value, | 6115 | .tag = .only_possible_value, |
| 5882 | .data = @intFromEnum(aggregate.ty), | 6116 | .data = @intFromEnum(aggregate.ty), |
| 5883 | }); | 6117 | }); |
| 5884 | return @enumFromInt(ip.items.len - 1); | 6118 | return gop.set(@enumFromInt(ip.items.len - 1)); |
| 5885 | } | 6119 | } |
| 5886 | | 6120 | |
| 5887 | switch (ty_key) { | 6121 | switch (ty_key) { |
| ... | @@ -5914,7 +6148,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All | ... | @@ -5914,7 +6148,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All |
| 5914 | .tag = .only_possible_value, | 6148 | .tag = .only_possible_value, |
| 5915 | .data = @intFromEnum(aggregate.ty), | 6149 | .data = @intFromEnum(aggregate.ty), |
| 5916 | }); | 6150 | }); |
| 5917 | return @enumFromInt(ip.items.len - 1); | 6151 | return gop.set(@enumFromInt(ip.items.len - 1)); |
| 5918 | }, | 6152 | }, |
| 5919 | else => {}, | 6153 | else => {}, |
| 5920 | } | 6154 | } |
| ... | @@ -5929,12 +6163,11 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All | ... | @@ -5929,12 +6163,11 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All |
| 5929 | } | 6163 | } |
| 5930 | const elem = switch (aggregate.storage) { | 6164 | const elem = switch (aggregate.storage) { |
| 5931 | .bytes => |bytes| elem: { | 6165 | .bytes => |bytes| elem: { |
| 5932 | _ = ip.map.pop(); | | |
| 5933 | const elem = try ip.get(gpa, tid, .{ .int = .{ | 6166 | const elem = try ip.get(gpa, tid, .{ .int = .{ |
| 5934 | .ty = .u8_type, | 6167 | .ty = .u8_type, |
| 5935 | .storage = .{ .u64 = bytes.at(0, ip) }, | 6168 | .storage = .{ .u64 = bytes.at(0, ip) }, |
| 5936 | } }); | 6169 | } }); |
| 5937 | assert(!(try ip.map.getOrPutAdapted(gpa, key, adapter)).found_existing); | 6170 | gop.assign(try ip.getOrPutKey(gpa, tid, key)); |
| 5938 | try ip.items.ensureUnusedCapacity(gpa, 1); | 6171 | try ip.items.ensureUnusedCapacity(gpa, 1); |
| 5939 | break :elem elem; | 6172 | break :elem elem; |
| 5940 | }, | 6173 | }, |
| ... | @@ -5953,7 +6186,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All | ... | @@ -5953,7 +6186,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All |
| 5953 | .elem_val = elem, | 6186 | .elem_val = elem, |
| 5954 | }), | 6187 | }), |
| 5955 | }); | 6188 | }); |
| 5956 | return @enumFromInt(ip.items.len - 1); | 6189 | return gop.set(@enumFromInt(ip.items.len - 1)); |
| 5957 | } | 6190 | } |
| 5958 | | 6191 | |
| 5959 | if (child == .u8_type) bytes: { | 6192 | if (child == .u8_type) bytes: { |
| ... | @@ -5997,7 +6230,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All | ... | @@ -5997,7 +6230,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All |
| 5997 | .bytes = string, | 6230 | .bytes = string, |
| 5998 | }), | 6231 | }), |
| 5999 | }); | 6232 | }); |
| 6000 | return @enumFromInt(ip.items.len - 1); | 6233 | return gop.set(@enumFromInt(ip.items.len - 1)); |
| 6001 | } | 6234 | } |
| 6002 | | 6235 | |
| 6003 | try ip.extra.ensureUnusedCapacity( | 6236 | try ip.extra.ensureUnusedCapacity( |
| ... | @@ -6038,7 +6271,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All | ... | @@ -6038,7 +6271,7 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All |
| 6038 | ip.extra.appendSliceAssumeCapacity(@ptrCast(memoized_call.arg_values)); | 6271 | ip.extra.appendSliceAssumeCapacity(@ptrCast(memoized_call.arg_values)); |
| 6039 | }, | 6272 | }, |
| 6040 | } | 6273 | } |
| 6041 | return @enumFromInt(ip.items.len - 1); | 6274 | return gop.set(@enumFromInt(ip.items.len - 1)); |
| 6042 | } | 6275 | } |
| 6043 | | 6276 | |
| 6044 | pub const UnionTypeInit = struct { | 6277 | pub const UnionTypeInit = struct { |
| ... | @@ -6076,11 +6309,10 @@ pub const UnionTypeInit = struct { | ... | @@ -6076,11 +6309,10 @@ pub const UnionTypeInit = struct { |
| 6076 | pub fn getUnionType( | 6309 | pub fn getUnionType( |
| 6077 | ip: *InternPool, | 6310 | ip: *InternPool, |
| 6078 | gpa: Allocator, | 6311 | gpa: Allocator, |
| 6079 | _: Zcu.PerThread.Id, | 6312 | tid: Zcu.PerThread.Id, |
| 6080 | ini: UnionTypeInit, | 6313 | ini: UnionTypeInit, |
| 6081 | ) Allocator.Error!WipNamespaceType.Result { | 6314 | ) Allocator.Error!WipNamespaceType.Result { |
| 6082 | const adapter: KeyAdapter = .{ .intern_pool = ip }; | 6315 | var gop = try ip.getOrPutKey(gpa, tid, .{ .union_type = switch (ini.key) { |
| 6083 | const gop = try ip.map.getOrPutAdapted(gpa, Key{ .union_type = switch (ini.key) { | | |
| 6084 | .declared => |d| .{ .declared = .{ | 6316 | .declared => |d| .{ .declared = .{ |
| 6085 | .zir_index = d.zir_index, | 6317 | .zir_index = d.zir_index, |
| 6086 | .captures = .{ .external = d.captures }, | 6318 | .captures = .{ .external = d.captures }, |
| ... | @@ -6089,9 +6321,9 @@ pub fn getUnionType( | ... | @@ -6089,9 +6321,9 @@ pub fn getUnionType( |
| 6089 | .zir_index = r.zir_index, | 6321 | .zir_index = r.zir_index, |
| 6090 | .type_hash = r.type_hash, | 6322 | .type_hash = r.type_hash, |
| 6091 | } }, | 6323 | } }, |
| 6092 | } }, adapter); | 6324 | } }); |
| 6093 | if (gop.found_existing) return .{ .existing = @enumFromInt(gop.index) }; | 6325 | defer gop.deinit(); |
| 6094 | errdefer _ = ip.map.pop(); | 6326 | if (gop == .existing) return .{ .existing = gop.existing }; |
| 6095 | | 6327 | |
| 6096 | const align_elements_len = if (ini.flags.any_aligned_fields) (ini.fields_len + 3) / 4 else 0; | 6328 | const align_elements_len = if (ini.flags.any_aligned_fields) (ini.fields_len + 3) / 4 else 0; |
| 6097 | const align_element: u32 = @bitCast([1]u8{@intFromEnum(Alignment.none)} ** 4); | 6329 | const align_element: u32 = @bitCast([1]u8{@intFromEnum(Alignment.none)} ** 4); |
| ... | @@ -6167,7 +6399,7 @@ pub fn getUnionType( | ... | @@ -6167,7 +6399,7 @@ pub fn getUnionType( |
| 6167 | } | 6399 | } |
| 6168 | | 6400 | |
| 6169 | return .{ .wip = .{ | 6401 | return .{ .wip = .{ |
| 6170 | .index = @enumFromInt(ip.items.len - 1), | 6402 | .index = gop.set(@enumFromInt(ip.items.len - 1)), |
| 6171 | .decl_extra_index = extra_index + std.meta.fieldIndex(Tag.TypeUnion, "decl").?, | 6403 | .decl_extra_index = extra_index + std.meta.fieldIndex(Tag.TypeUnion, "decl").?, |
| 6172 | .namespace_extra_index = if (ini.has_namespace) | 6404 | .namespace_extra_index = if (ini.has_namespace) |
| 6173 | extra_index + std.meta.fieldIndex(Tag.TypeUnion, "namespace").? | 6405 | extra_index + std.meta.fieldIndex(Tag.TypeUnion, "namespace").? |
| ... | @@ -6225,11 +6457,10 @@ pub const StructTypeInit = struct { | ... | @@ -6225,11 +6457,10 @@ pub const StructTypeInit = struct { |
| 6225 | pub fn getStructType( | 6457 | pub fn getStructType( |
| 6226 | ip: *InternPool, | 6458 | ip: *InternPool, |
| 6227 | gpa: Allocator, | 6459 | gpa: Allocator, |
| 6228 | _: Zcu.PerThread.Id, | 6460 | tid: Zcu.PerThread.Id, |
| 6229 | ini: StructTypeInit, | 6461 | ini: StructTypeInit, |
| 6230 | ) Allocator.Error!WipNamespaceType.Result { | 6462 | ) Allocator.Error!WipNamespaceType.Result { |
| 6231 | const adapter: KeyAdapter = .{ .intern_pool = ip }; | 6463 | var gop = try ip.getOrPutKey(gpa, tid, .{ .struct_type = switch (ini.key) { |
| 6232 | const key: Key = .{ .struct_type = switch (ini.key) { | | |
| 6233 | .declared => |d| .{ .declared = .{ | 6464 | .declared => |d| .{ .declared = .{ |
| 6234 | .zir_index = d.zir_index, | 6465 | .zir_index = d.zir_index, |
| 6235 | .captures = .{ .external = d.captures }, | 6466 | .captures = .{ .external = d.captures }, |
| ... | @@ -6238,10 +6469,9 @@ pub fn getStructType( | ... | @@ -6238,10 +6469,9 @@ pub fn getStructType( |
| 6238 | .zir_index = r.zir_index, | 6469 | .zir_index = r.zir_index, |
| 6239 | .type_hash = r.type_hash, | 6470 | .type_hash = r.type_hash, |
| 6240 | } }, | 6471 | } }, |
| 6241 | } }; | 6472 | } }); |
| 6242 | const gop = try ip.map.getOrPutAdapted(gpa, key, adapter); | 6473 | defer gop.deinit(); |
| 6243 | if (gop.found_existing) return .{ .existing = @enumFromInt(gop.index) }; | 6474 | if (gop == .existing) return .{ .existing = gop.existing }; |
| 6244 | errdefer _ = ip.map.pop(); | | |
| 6245 | | 6475 | |
| 6246 | const names_map = try ip.addMap(gpa, ini.fields_len); | 6476 | const names_map = try ip.addMap(gpa, ini.fields_len); |
| 6247 | errdefer _ = ip.maps.pop(); | 6477 | errdefer _ = ip.maps.pop(); |
| ... | @@ -6298,7 +6528,7 @@ pub fn getStructType( | ... | @@ -6298,7 +6528,7 @@ pub fn getStructType( |
| 6298 | ip.extra.appendNTimesAssumeCapacity(@intFromEnum(Index.none), ini.fields_len); | 6528 | ip.extra.appendNTimesAssumeCapacity(@intFromEnum(Index.none), ini.fields_len); |
| 6299 | } | 6529 | } |
| 6300 | return .{ .wip = .{ | 6530 | return .{ .wip = .{ |
| 6301 | .index = @enumFromInt(ip.items.len - 1), | 6531 | .index = gop.set(@enumFromInt(ip.items.len - 1)), |
| 6302 | .decl_extra_index = extra_index + std.meta.fieldIndex(Tag.TypeStructPacked, "decl").?, | 6532 | .decl_extra_index = extra_index + std.meta.fieldIndex(Tag.TypeStructPacked, "decl").?, |
| 6303 | .namespace_extra_index = if (ini.has_namespace) | 6533 | .namespace_extra_index = if (ini.has_namespace) |
| 6304 | extra_index + std.meta.fieldIndex(Tag.TypeStructPacked, "namespace").? | 6534 | extra_index + std.meta.fieldIndex(Tag.TypeStructPacked, "namespace").? |
| ... | @@ -6387,7 +6617,7 @@ pub fn getStructType( | ... | @@ -6387,7 +6617,7 @@ pub fn getStructType( |
| 6387 | } | 6617 | } |
| 6388 | ip.extra.appendNTimesAssumeCapacity(std.math.maxInt(u32), ini.fields_len); | 6618 | ip.extra.appendNTimesAssumeCapacity(std.math.maxInt(u32), ini.fields_len); |
| 6389 | return .{ .wip = .{ | 6619 | return .{ .wip = .{ |
| 6390 | .index = @enumFromInt(ip.items.len - 1), | 6620 | .index = gop.set(@enumFromInt(ip.items.len - 1)), |
| 6391 | .decl_extra_index = extra_index + std.meta.fieldIndex(Tag.TypeStruct, "decl").?, | 6621 | .decl_extra_index = extra_index + std.meta.fieldIndex(Tag.TypeStruct, "decl").?, |
| 6392 | .namespace_extra_index = namespace_extra_index, | 6622 | .namespace_extra_index = namespace_extra_index, |
| 6393 | } }; | 6623 | } }; |
| ... | @@ -6404,7 +6634,7 @@ pub const AnonStructTypeInit = struct { | ... | @@ -6404,7 +6634,7 @@ pub const AnonStructTypeInit = struct { |
| 6404 | pub fn getAnonStructType( | 6634 | pub fn getAnonStructType( |
| 6405 | ip: *InternPool, | 6635 | ip: *InternPool, |
| 6406 | gpa: Allocator, | 6636 | gpa: Allocator, |
| 6407 | _: Zcu.PerThread.Id, | 6637 | tid: Zcu.PerThread.Id, |
| 6408 | ini: AnonStructTypeInit, | 6638 | ini: AnonStructTypeInit, |
| 6409 | ) Allocator.Error!Index { | 6639 | ) Allocator.Error!Index { |
| 6410 | assert(ini.types.len == ini.values.len); | 6640 | assert(ini.types.len == ini.values.len); |
| ... | @@ -6424,25 +6654,26 @@ pub fn getAnonStructType( | ... | @@ -6424,25 +6654,26 @@ pub fn getAnonStructType( |
| 6424 | }); | 6654 | }); |
| 6425 | ip.extra.appendSliceAssumeCapacity(@ptrCast(ini.types)); | 6655 | ip.extra.appendSliceAssumeCapacity(@ptrCast(ini.types)); |
| 6426 | ip.extra.appendSliceAssumeCapacity(@ptrCast(ini.values)); | 6656 | ip.extra.appendSliceAssumeCapacity(@ptrCast(ini.values)); |
| | 6657 | errdefer ip.extra.items.len = prev_extra_len; |
| 6427 | | 6658 | |
| 6428 | const adapter: KeyAdapter = .{ .intern_pool = ip }; | 6659 | var gop = try ip.getOrPutKey(gpa, tid, .{ |
| 6429 | const key: Key = .{ | | |
| 6430 | .anon_struct_type = if (ini.names.len == 0) extraTypeTupleAnon(ip, extra_index) else k: { | 6660 | .anon_struct_type = if (ini.names.len == 0) extraTypeTupleAnon(ip, extra_index) else k: { |
| 6431 | assert(ini.names.len == ini.types.len); | 6661 | assert(ini.names.len == ini.types.len); |
| 6432 | ip.extra.appendSliceAssumeCapacity(@ptrCast(ini.names)); | 6662 | ip.extra.appendSliceAssumeCapacity(@ptrCast(ini.names)); |
| 6433 | break :k extraTypeStructAnon(ip, extra_index); | 6663 | break :k extraTypeStructAnon(ip, extra_index); |
| 6434 | }, | 6664 | }, |
| 6435 | }; | 6665 | }); |
| 6436 | const gop = try ip.map.getOrPutAdapted(gpa, key, adapter); | 6666 | defer gop.deinit(); |
| 6437 | if (gop.found_existing) { | 6667 | if (gop == .existing) { |
| 6438 | ip.extra.items.len = prev_extra_len; | 6668 | ip.extra.items.len = prev_extra_len; |
| 6439 | return @enumFromInt(gop.index); | 6669 | return gop.existing; |
| 6440 | } | 6670 | } |
| | 6671 | |
| 6441 | ip.items.appendAssumeCapacity(.{ | 6672 | ip.items.appendAssumeCapacity(.{ |
| 6442 | .tag = if (ini.names.len == 0) .type_tuple_anon else .type_struct_anon, | 6673 | .tag = if (ini.names.len == 0) .type_tuple_anon else .type_struct_anon, |
| 6443 | .data = extra_index, | 6674 | .data = extra_index, |
| 6444 | }); | 6675 | }); |
| 6445 | return @enumFromInt(ip.items.len - 1); | 6676 | return gop.set(@enumFromInt(ip.items.len - 1)); |
| 6446 | } | 6677 | } |
| 6447 | | 6678 | |
| 6448 | /// This is equivalent to `Key.FuncType` but adjusted to have a slice for `param_types`. | 6679 | /// This is equivalent to `Key.FuncType` but adjusted to have a slice for `param_types`. |
| ... | @@ -6463,7 +6694,7 @@ pub const GetFuncTypeKey = struct { | ... | @@ -6463,7 +6694,7 @@ pub const GetFuncTypeKey = struct { |
| 6463 | pub fn getFuncType( | 6694 | pub fn getFuncType( |
| 6464 | ip: *InternPool, | 6695 | ip: *InternPool, |
| 6465 | gpa: Allocator, | 6696 | gpa: Allocator, |
| 6466 | _: Zcu.PerThread.Id, | 6697 | tid: Zcu.PerThread.Id, |
| 6467 | key: GetFuncTypeKey, | 6698 | key: GetFuncTypeKey, |
| 6468 | ) Allocator.Error!Index { | 6699 | ) Allocator.Error!Index { |
| 6469 | // Validate input parameters. | 6700 | // Validate input parameters. |
| ... | @@ -6501,33 +6732,33 @@ pub fn getFuncType( | ... | @@ -6501,33 +6732,33 @@ pub fn getFuncType( |
| 6501 | if (key.comptime_bits != 0) ip.extra.appendAssumeCapacity(key.comptime_bits); | 6732 | if (key.comptime_bits != 0) ip.extra.appendAssumeCapacity(key.comptime_bits); |
| 6502 | if (key.noalias_bits != 0) ip.extra.appendAssumeCapacity(key.noalias_bits); | 6733 | if (key.noalias_bits != 0) ip.extra.appendAssumeCapacity(key.noalias_bits); |
| 6503 | ip.extra.appendSliceAssumeCapacity(@ptrCast(key.param_types)); | 6734 | ip.extra.appendSliceAssumeCapacity(@ptrCast(key.param_types)); |
| | 6735 | errdefer ip.extra.items.len = prev_extra_len; |
| 6504 | | 6736 | |
| 6505 | const adapter: KeyAdapter = .{ .intern_pool = ip }; | 6737 | var gop = try ip.getOrPutKey(gpa, tid, .{ |
| 6506 | const gop = try ip.map.getOrPutAdapted(gpa, Key{ | | |
| 6507 | .func_type = extraFuncType(ip, func_type_extra_index), | 6738 | .func_type = extraFuncType(ip, func_type_extra_index), |
| 6508 | }, adapter); | 6739 | }); |
| 6509 | if (gop.found_existing) { | 6740 | defer gop.deinit(); |
| | 6741 | if (gop == .existing) { |
| 6510 | ip.extra.items.len = prev_extra_len; | 6742 | ip.extra.items.len = prev_extra_len; |
| 6511 | return @enumFromInt(gop.index); | 6743 | return gop.existing; |
| 6512 | } | 6744 | } |
| 6513 | | 6745 | |
| 6514 | ip.items.appendAssumeCapacity(.{ | 6746 | ip.items.appendAssumeCapacity(.{ |
| 6515 | .tag = .type_function, | 6747 | .tag = .type_function, |
| 6516 | .data = func_type_extra_index, | 6748 | .data = func_type_extra_index, |
| 6517 | }); | 6749 | }); |
| 6518 | return @enumFromInt(ip.items.len - 1); | 6750 | return gop.set(@enumFromInt(ip.items.len - 1)); |
| 6519 | } | 6751 | } |
| 6520 | | 6752 | |
| 6521 | pub fn getExternFunc( | 6753 | pub fn getExternFunc( |
| 6522 | ip: *InternPool, | 6754 | ip: *InternPool, |
| 6523 | gpa: Allocator, | 6755 | gpa: Allocator, |
| 6524 | _: Zcu.PerThread.Id, | 6756 | tid: Zcu.PerThread.Id, |
| 6525 | key: Key.ExternFunc, | 6757 | key: Key.ExternFunc, |
| 6526 | ) Allocator.Error!Index { | 6758 | ) Allocator.Error!Index { |
| 6527 | const adapter: KeyAdapter = .{ .intern_pool = ip }; | 6759 | var gop = try ip.getOrPutKey(gpa, tid, .{ .extern_func = key }); |
| 6528 | const gop = try ip.map.getOrPutAdapted(gpa, Key{ .extern_func = key }, adapter); | 6760 | defer gop.deinit(); |
| 6529 | if (gop.found_existing) return @enumFromInt(gop.index); | 6761 | if (gop == .existing) return gop.existing; |
| 6530 | errdefer _ = ip.map.pop(); | | |
| 6531 | const prev_extra_len = ip.extra.items.len; | 6762 | const prev_extra_len = ip.extra.items.len; |
| 6532 | const extra_index = try ip.addExtra(gpa, @as(Tag.ExternFunc, key)); | 6763 | const extra_index = try ip.addExtra(gpa, @as(Tag.ExternFunc, key)); |
| 6533 | errdefer ip.extra.items.len = prev_extra_len; | 6764 | errdefer ip.extra.items.len = prev_extra_len; |
| ... | @@ -6536,7 +6767,7 @@ pub fn getExternFunc( | ... | @@ -6536,7 +6767,7 @@ pub fn getExternFunc( |
| 6536 | .data = extra_index, | 6767 | .data = extra_index, |
| 6537 | }); | 6768 | }); |
| 6538 | errdefer ip.items.len -= 1; | 6769 | errdefer ip.items.len -= 1; |
| 6539 | return @enumFromInt(ip.items.len - 1); | 6770 | return gop.set(@enumFromInt(ip.items.len - 1)); |
| 6540 | } | 6771 | } |
| 6541 | | 6772 | |
| 6542 | pub const GetFuncDeclKey = struct { | 6773 | pub const GetFuncDeclKey = struct { |
| ... | @@ -6554,7 +6785,7 @@ pub const GetFuncDeclKey = struct { | ... | @@ -6554,7 +6785,7 @@ pub const GetFuncDeclKey = struct { |
| 6554 | pub fn getFuncDecl( | 6785 | pub fn getFuncDecl( |
| 6555 | ip: *InternPool, | 6786 | ip: *InternPool, |
| 6556 | gpa: Allocator, | 6787 | gpa: Allocator, |
| 6557 | _: Zcu.PerThread.Id, | 6788 | tid: Zcu.PerThread.Id, |
| 6558 | key: GetFuncDeclKey, | 6789 | key: GetFuncDeclKey, |
| 6559 | ) Allocator.Error!Index { | 6790 | ) Allocator.Error!Index { |
| 6560 | // The strategy here is to add the function type unconditionally, then to | 6791 | // The strategy here is to add the function type unconditionally, then to |
| ... | @@ -6564,7 +6795,6 @@ pub fn getFuncDecl( | ... | @@ -6564,7 +6795,6 @@ pub fn getFuncDecl( |
| 6564 | | 6795 | |
| 6565 | try ip.extra.ensureUnusedCapacity(gpa, @typeInfo(Tag.FuncDecl).Struct.fields.len); | 6796 | try ip.extra.ensureUnusedCapacity(gpa, @typeInfo(Tag.FuncDecl).Struct.fields.len); |
| 6566 | try ip.items.ensureUnusedCapacity(gpa, 1); | 6797 | try ip.items.ensureUnusedCapacity(gpa, 1); |
| 6567 | try ip.map.ensureUnusedCapacity(gpa, 1); | | |
| 6568 | | 6798 | |
| 6569 | const func_decl_extra_index = ip.addExtraAssumeCapacity(Tag.FuncDecl{ | 6799 | const func_decl_extra_index = ip.addExtraAssumeCapacity(Tag.FuncDecl{ |
| 6570 | .analysis = .{ | 6800 | .analysis = .{ |
| ... | @@ -6583,22 +6813,22 @@ pub fn getFuncDecl( | ... | @@ -6583,22 +6813,22 @@ pub fn getFuncDecl( |
| 6583 | .lbrace_column = key.lbrace_column, | 6813 | .lbrace_column = key.lbrace_column, |
| 6584 | .rbrace_column = key.rbrace_column, | 6814 | .rbrace_column = key.rbrace_column, |
| 6585 | }); | 6815 | }); |
| | 6816 | errdefer ip.extra.items.len = prev_extra_len; |
| 6586 | | 6817 | |
| 6587 | const adapter: KeyAdapter = .{ .intern_pool = ip }; | 6818 | var gop = try ip.getOrPutKey(gpa, tid, .{ |
| 6588 | const gop = ip.map.getOrPutAssumeCapacityAdapted(Key{ | | |
| 6589 | .func = extraFuncDecl(ip, func_decl_extra_index), | 6819 | .func = extraFuncDecl(ip, func_decl_extra_index), |
| 6590 | }, adapter); | 6820 | }); |
| 6591 | | 6821 | defer gop.deinit(); |
| 6592 | if (gop.found_existing) { | 6822 | if (gop == .existing) { |
| 6593 | ip.extra.items.len = prev_extra_len; | 6823 | ip.extra.items.len = prev_extra_len; |
| 6594 | return @enumFromInt(gop.index); | 6824 | return gop.existing; |
| 6595 | } | 6825 | } |
| 6596 | | 6826 | |
| 6597 | ip.items.appendAssumeCapacity(.{ | 6827 | ip.items.appendAssumeCapacity(.{ |
| 6598 | .tag = .func_decl, | 6828 | .tag = .func_decl, |
| 6599 | .data = func_decl_extra_index, | 6829 | .data = func_decl_extra_index, |
| 6600 | }); | 6830 | }); |
| 6601 | return @enumFromInt(ip.items.len - 1); | 6831 | return gop.set(@enumFromInt(ip.items.len - 1)); |
| 6602 | } | 6832 | } |
| 6603 | | 6833 | |
| 6604 | pub const GetFuncDeclIesKey = struct { | 6834 | pub const GetFuncDeclIesKey = struct { |
| ... | @@ -6626,7 +6856,7 @@ pub const GetFuncDeclIesKey = struct { | ... | @@ -6626,7 +6856,7 @@ pub const GetFuncDeclIesKey = struct { |
| 6626 | pub fn getFuncDeclIes( | 6856 | pub fn getFuncDeclIes( |
| 6627 | ip: *InternPool, | 6857 | ip: *InternPool, |
| 6628 | gpa: Allocator, | 6858 | gpa: Allocator, |
| 6629 | _: Zcu.PerThread.Id, | 6859 | tid: Zcu.PerThread.Id, |
| 6630 | key: GetFuncDeclIesKey, | 6860 | key: GetFuncDeclIesKey, |
| 6631 | ) Allocator.Error!Index { | 6861 | ) Allocator.Error!Index { |
| 6632 | // Validate input parameters. | 6862 | // Validate input parameters. |
| ... | @@ -6639,7 +6869,6 @@ pub fn getFuncDeclIes( | ... | @@ -6639,7 +6869,6 @@ pub fn getFuncDeclIes( |
| 6639 | const prev_extra_len = ip.extra.items.len; | 6869 | const prev_extra_len = ip.extra.items.len; |
| 6640 | const params_len: u32 = @intCast(key.param_types.len); | 6870 | const params_len: u32 = @intCast(key.param_types.len); |
| 6641 | | 6871 | |
| 6642 | try ip.map.ensureUnusedCapacity(gpa, 4); | | |
| 6643 | try ip.extra.ensureUnusedCapacity(gpa, @typeInfo(Tag.FuncDecl).Struct.fields.len + | 6872 | try ip.extra.ensureUnusedCapacity(gpa, @typeInfo(Tag.FuncDecl).Struct.fields.len + |
| 6644 | 1 + // inferred_error_set | 6873 | 1 + // inferred_error_set |
| 6645 | @typeInfo(Tag.ErrorUnionType).Struct.fields.len + | 6874 | @typeInfo(Tag.ErrorUnionType).Struct.fields.len + |
| ... | @@ -6704,40 +6933,51 @@ pub fn getFuncDeclIes( | ... | @@ -6704,40 +6933,51 @@ pub fn getFuncDeclIes( |
| 6704 | if (key.comptime_bits != 0) ip.extra.appendAssumeCapacity(key.comptime_bits); | 6933 | if (key.comptime_bits != 0) ip.extra.appendAssumeCapacity(key.comptime_bits); |
| 6705 | if (key.noalias_bits != 0) ip.extra.appendAssumeCapacity(key.noalias_bits); | 6934 | if (key.noalias_bits != 0) ip.extra.appendAssumeCapacity(key.noalias_bits); |
| 6706 | ip.extra.appendSliceAssumeCapacity(@ptrCast(key.param_types)); | 6935 | ip.extra.appendSliceAssumeCapacity(@ptrCast(key.param_types)); |
| | 6936 | errdefer { |
| | 6937 | ip.items.len -= 4; |
| | 6938 | ip.extra.items.len = prev_extra_len; |
| | 6939 | } |
| 6707 | | 6940 | |
| 6708 | ip.items.appendAssumeCapacity(.{ | 6941 | ip.items.appendAssumeCapacity(.{ |
| 6709 | .tag = .type_function, | 6942 | .tag = .type_function, |
| 6710 | .data = func_type_extra_index, | 6943 | .data = func_type_extra_index, |
| 6711 | }); | 6944 | }); |
| 6712 | | 6945 | |
| 6713 | const adapter: KeyAdapter = .{ .intern_pool = ip }; | 6946 | var gop = try ip.getOrPutKey(gpa, tid, .{ |
| 6714 | const gop = ip.map.getOrPutAssumeCapacityAdapted(Key{ | | |
| 6715 | .func = extraFuncDecl(ip, func_decl_extra_index), | 6947 | .func = extraFuncDecl(ip, func_decl_extra_index), |
| 6716 | }, adapter); | 6948 | }); |
| 6717 | if (!gop.found_existing) { | 6949 | defer gop.deinit(); |
| 6718 | assert(!ip.map.getOrPutAssumeCapacityAdapted(Key{ .error_union_type = .{ | 6950 | if (gop == .existing) { |
| 6719 | .error_set_type = @enumFromInt(ip.items.len - 2), | 6951 | // An existing function type was found; undo the additions to our two arrays. |
| 6720 | .payload_type = key.bare_return_type, | 6952 | ip.items.len -= 4; |
| 6721 | } }, adapter).found_existing); | 6953 | ip.extra.items.len = prev_extra_len; |
| 6722 | assert(!ip.map.getOrPutAssumeCapacityAdapted(Key{ | 6954 | return gop.existing; |
| 6723 | .inferred_error_set_type = @enumFromInt(ip.items.len - 4), | 6955 | } |
| 6724 | }, adapter).found_existing); | 6956 | |
| 6725 | assert(!ip.map.getOrPutAssumeCapacityAdapted(Key{ | 6957 | var eu_gop = try ip.getOrPutKey(gpa, tid, .{ .error_union_type = .{ |
| 6726 | .func_type = extraFuncType(ip, func_type_extra_index), | 6958 | .error_set_type = @enumFromInt(ip.items.len - 2), |
| 6727 | }, adapter).found_existing); | 6959 | .payload_type = key.bare_return_type, |
| 6728 | return @enumFromInt(ip.items.len - 4); | 6960 | } }); |
| 6729 | } | 6961 | defer eu_gop.deinit(); |
| 6730 | | 6962 | var ies_gop = try ip.getOrPutKey(gpa, tid, .{ |
| 6731 | // An existing function type was found; undo the additions to our two arrays. | 6963 | .inferred_error_set_type = @enumFromInt(ip.items.len - 4), |
| 6732 | ip.items.len -= 4; | 6964 | }); |
| 6733 | ip.extra.items.len = prev_extra_len; | 6965 | defer ies_gop.deinit(); |
| 6734 | return @enumFromInt(gop.index); | 6966 | var ty_gop = try ip.getOrPutKey(gpa, tid, .{ |
| | 6967 | .func_type = extraFuncType(ip, func_type_extra_index), |
| | 6968 | }); |
| | 6969 | defer ty_gop.deinit(); |
| | 6970 | const index = gop.set(@enumFromInt(ip.items.len - 4)); |
| | 6971 | _ = eu_gop.set(@enumFromInt(@intFromEnum(index) + 1)); |
| | 6972 | _ = ies_gop.set(@enumFromInt(@intFromEnum(index) + 2)); |
| | 6973 | _ = ty_gop.set(@enumFromInt(@intFromEnum(index) + 3)); |
| | 6974 | return index; |
| 6735 | } | 6975 | } |
| 6736 | | 6976 | |
| 6737 | pub fn getErrorSetType( | 6977 | pub fn getErrorSetType( |
| 6738 | ip: *InternPool, | 6978 | ip: *InternPool, |
| 6739 | gpa: Allocator, | 6979 | gpa: Allocator, |
| 6740 | _: Zcu.PerThread.Id, | 6980 | tid: Zcu.PerThread.Id, |
| 6741 | names: []const NullTerminatedString, | 6981 | names: []const NullTerminatedString, |
| 6742 | ) Allocator.Error!Index { | 6982 | ) Allocator.Error!Index { |
| 6743 | assert(std.sort.isSorted(NullTerminatedString, names, {}, NullTerminatedString.indexLessThan)); | 6983 | assert(std.sort.isSorted(NullTerminatedString, names, {}, NullTerminatedString.indexLessThan)); |
| ... | @@ -6757,16 +6997,15 @@ pub fn getErrorSetType( | ... | @@ -6757,16 +6997,15 @@ pub fn getErrorSetType( |
| 6757 | .names_map = predicted_names_map, | 6997 | .names_map = predicted_names_map, |
| 6758 | }); | 6998 | }); |
| 6759 | ip.extra.appendSliceAssumeCapacity(@ptrCast(names)); | 6999 | ip.extra.appendSliceAssumeCapacity(@ptrCast(names)); |
| | 7000 | errdefer ip.extra.items.len = prev_extra_len; |
| 6760 | | 7001 | |
| 6761 | const adapter: KeyAdapter = .{ .intern_pool = ip }; | 7002 | var gop = try ip.getOrPutKey(gpa, tid, .{ |
| 6762 | const gop = try ip.map.getOrPutAdapted(gpa, Key{ | | |
| 6763 | .error_set_type = extraErrorSet(ip, error_set_extra_index), | 7003 | .error_set_type = extraErrorSet(ip, error_set_extra_index), |
| 6764 | }, adapter); | 7004 | }); |
| 6765 | errdefer _ = ip.map.pop(); | 7005 | defer gop.deinit(); |
| 6766 | | 7006 | if (gop == .existing) { |
| 6767 | if (gop.found_existing) { | | |
| 6768 | ip.extra.items.len = prev_extra_len; | 7007 | ip.extra.items.len = prev_extra_len; |
| 6769 | return @enumFromInt(gop.index); | 7008 | return gop.existing; |
| 6770 | } | 7009 | } |
| 6771 | | 7010 | |
| 6772 | try ip.items.append(gpa, .{ | 7011 | try ip.items.append(gpa, .{ |
| ... | @@ -6781,7 +7020,7 @@ pub fn getErrorSetType( | ... | @@ -6781,7 +7020,7 @@ pub fn getErrorSetType( |
| 6781 | | 7020 | |
| 6782 | addStringsToMap(ip, names_map, names); | 7021 | addStringsToMap(ip, names_map, names); |
| 6783 | | 7022 | |
| 6784 | return @enumFromInt(ip.items.len - 1); | 7023 | return gop.set(@enumFromInt(ip.items.len - 1)); |
| 6785 | } | 7024 | } |
| 6786 | | 7025 | |
| 6787 | pub const GetFuncInstanceKey = struct { | 7026 | pub const GetFuncInstanceKey = struct { |
| ... | @@ -6845,14 +7084,13 @@ pub fn getFuncInstance( | ... | @@ -6845,14 +7084,13 @@ pub fn getFuncInstance( |
| 6845 | }); | 7084 | }); |
| 6846 | ip.extra.appendSliceAssumeCapacity(@ptrCast(arg.comptime_args)); | 7085 | ip.extra.appendSliceAssumeCapacity(@ptrCast(arg.comptime_args)); |
| 6847 | | 7086 | |
| 6848 | const gop = try ip.map.getOrPutAdapted(gpa, Key{ | 7087 | var gop = try ip.getOrPutKey(gpa, tid, .{ |
| 6849 | .func = extraFuncInstance(ip, func_extra_index), | 7088 | .func = extraFuncInstance(ip, func_extra_index), |
| 6850 | }, KeyAdapter{ .intern_pool = ip }); | 7089 | }); |
| 6851 | errdefer _ = ip.map.pop(); | 7090 | defer gop.deinit(); |
| 6852 | | 7091 | if (gop == .existing) { |
| 6853 | if (gop.found_existing) { | | |
| 6854 | ip.extra.items.len = prev_extra_len; | 7092 | ip.extra.items.len = prev_extra_len; |
| 6855 | return @enumFromInt(gop.index); | 7093 | return gop.existing; |
| 6856 | } | 7094 | } |
| 6857 | | 7095 | |
| 6858 | const func_index: Index = @enumFromInt(ip.items.len); | 7096 | const func_index: Index = @enumFromInt(ip.items.len); |
| ... | @@ -6863,7 +7101,7 @@ pub fn getFuncInstance( | ... | @@ -6863,7 +7101,7 @@ pub fn getFuncInstance( |
| 6863 | }); | 7101 | }); |
| 6864 | errdefer ip.items.len -= 1; | 7102 | errdefer ip.items.len -= 1; |
| 6865 | | 7103 | |
| 6866 | return finishFuncInstance( | 7104 | return gop.set(try finishFuncInstance( |
| 6867 | ip, | 7105 | ip, |
| 6868 | gpa, | 7106 | gpa, |
| 6869 | tid, | 7107 | tid, |
| ... | @@ -6872,7 +7110,7 @@ pub fn getFuncInstance( | ... | @@ -6872,7 +7110,7 @@ pub fn getFuncInstance( |
| 6872 | func_extra_index, | 7110 | func_extra_index, |
| 6873 | arg.alignment, | 7111 | arg.alignment, |
| 6874 | arg.section, | 7112 | arg.section, |
| 6875 | ); | 7113 | )); |
| 6876 | } | 7114 | } |
| 6877 | | 7115 | |
| 6878 | /// This function exists separately than `getFuncInstance` because it needs to | 7116 | /// This function exists separately than `getFuncInstance` because it needs to |
| ... | @@ -6897,7 +7135,6 @@ pub fn getFuncInstanceIes( | ... | @@ -6897,7 +7135,6 @@ pub fn getFuncInstanceIes( |
| 6897 | const prev_extra_len = ip.extra.items.len; | 7135 | const prev_extra_len = ip.extra.items.len; |
| 6898 | const params_len: u32 = @intCast(arg.param_types.len); | 7136 | const params_len: u32 = @intCast(arg.param_types.len); |
| 6899 | | 7137 | |
| 6900 | try ip.map.ensureUnusedCapacity(gpa, 4); | | |
| 6901 | try ip.extra.ensureUnusedCapacity(gpa, @typeInfo(Tag.FuncInstance).Struct.fields.len + | 7138 | try ip.extra.ensureUnusedCapacity(gpa, @typeInfo(Tag.FuncInstance).Struct.fields.len + |
| 6902 | 1 + // inferred_error_set | 7139 | 1 + // inferred_error_set |
| 6903 | arg.comptime_args.len + | 7140 | arg.comptime_args.len + |
| ... | @@ -6970,30 +7207,37 @@ pub fn getFuncInstanceIes( | ... | @@ -6970,30 +7207,37 @@ pub fn getFuncInstanceIes( |
| 6970 | .tag = .type_function, | 7207 | .tag = .type_function, |
| 6971 | .data = func_type_extra_index, | 7208 | .data = func_type_extra_index, |
| 6972 | }); | 7209 | }); |
| | 7210 | errdefer { |
| | 7211 | ip.items.len -= 4; |
| | 7212 | ip.extra.items.len = prev_extra_len; |
| | 7213 | } |
| 6973 | | 7214 | |
| 6974 | const adapter: KeyAdapter = .{ .intern_pool = ip }; | 7215 | var gop = try ip.getOrPutKey(gpa, tid, .{ |
| 6975 | const gop = ip.map.getOrPutAssumeCapacityAdapted(Key{ | | |
| 6976 | .func = extraFuncInstance(ip, func_extra_index), | 7216 | .func = extraFuncInstance(ip, func_extra_index), |
| 6977 | }, adapter); | 7217 | }); |
| 6978 | if (gop.found_existing) { | 7218 | defer gop.deinit(); |
| | 7219 | if (gop == .existing) { |
| 6979 | // Hot path: undo the additions to our two arrays. | 7220 | // Hot path: undo the additions to our two arrays. |
| 6980 | ip.items.len -= 4; | 7221 | ip.items.len -= 4; |
| 6981 | ip.extra.items.len = prev_extra_len; | 7222 | ip.extra.items.len = prev_extra_len; |
| 6982 | return @enumFromInt(gop.index); | 7223 | return gop.existing; |
| 6983 | } | 7224 | } |
| 6984 | | 7225 | |
| 6985 | // Synchronize the map with items. | 7226 | // Synchronize the map with items. |
| 6986 | assert(!ip.map.getOrPutAssumeCapacityAdapted(Key{ .error_union_type = .{ | 7227 | var eu_gop = try ip.getOrPutKey(gpa, tid, .{ .error_union_type = .{ |
| 6987 | .error_set_type = error_set_type, | 7228 | .error_set_type = error_set_type, |
| 6988 | .payload_type = arg.bare_return_type, | 7229 | .payload_type = arg.bare_return_type, |
| 6989 | } }, adapter).found_existing); | 7230 | } }); |
| 6990 | assert(!ip.map.getOrPutAssumeCapacityAdapted(Key{ | 7231 | defer eu_gop.deinit(); |
| | 7232 | var ies_gop = try ip.getOrPutKey(gpa, tid, .{ |
| 6991 | .inferred_error_set_type = func_index, | 7233 | .inferred_error_set_type = func_index, |
| 6992 | }, adapter).found_existing); | 7234 | }); |
| 6993 | assert(!ip.map.getOrPutAssumeCapacityAdapted(Key{ | 7235 | defer ies_gop.deinit(); |
| | 7236 | var ty_gop = try ip.getOrPutKey(gpa, tid, .{ |
| 6994 | .func_type = extraFuncType(ip, func_type_extra_index), | 7237 | .func_type = extraFuncType(ip, func_type_extra_index), |
| 6995 | }, adapter).found_existing); | 7238 | }); |
| 6996 | return finishFuncInstance( | 7239 | defer ty_gop.deinit(); |
| | 7240 | const index = gop.set(try finishFuncInstance( |
| 6997 | ip, | 7241 | ip, |
| 6998 | gpa, | 7242 | gpa, |
| 6999 | tid, | 7243 | tid, |
| ... | @@ -7002,7 +7246,11 @@ pub fn getFuncInstanceIes( | ... | @@ -7002,7 +7246,11 @@ pub fn getFuncInstanceIes( |
| 7002 | func_extra_index, | 7246 | func_extra_index, |
| 7003 | arg.alignment, | 7247 | arg.alignment, |
| 7004 | arg.section, | 7248 | arg.section, |
| 7005 | ); | 7249 | )); |
| | 7250 | _ = eu_gop.set(@enumFromInt(@intFromEnum(index) + 1)); |
| | 7251 | _ = ies_gop.set(@enumFromInt(@intFromEnum(index) + 2)); |
| | 7252 | _ = ty_gop.set(@enumFromInt(@intFromEnum(index) + 3)); |
| | 7253 | return index; |
| 7006 | } | 7254 | } |
| 7007 | | 7255 | |
| 7008 | fn finishFuncInstance( | 7256 | fn finishFuncInstance( |
| ... | @@ -7135,11 +7383,10 @@ pub const WipEnumType = struct { | ... | @@ -7135,11 +7383,10 @@ pub const WipEnumType = struct { |
| 7135 | pub fn getEnumType( | 7383 | pub fn getEnumType( |
| 7136 | ip: *InternPool, | 7384 | ip: *InternPool, |
| 7137 | gpa: Allocator, | 7385 | gpa: Allocator, |
| 7138 | _: Zcu.PerThread.Id, | 7386 | tid: Zcu.PerThread.Id, |
| 7139 | ini: EnumTypeInit, | 7387 | ini: EnumTypeInit, |
| 7140 | ) Allocator.Error!WipEnumType.Result { | 7388 | ) Allocator.Error!WipEnumType.Result { |
| 7141 | const adapter: KeyAdapter = .{ .intern_pool = ip }; | 7389 | var gop = try ip.getOrPutKey(gpa, tid, .{ .enum_type = switch (ini.key) { |
| 7142 | const gop = try ip.map.getOrPutAdapted(gpa, Key{ .enum_type = switch (ini.key) { | | |
| 7143 | .declared => |d| .{ .declared = .{ | 7390 | .declared => |d| .{ .declared = .{ |
| 7144 | .zir_index = d.zir_index, | 7391 | .zir_index = d.zir_index, |
| 7145 | .captures = .{ .external = d.captures }, | 7392 | .captures = .{ .external = d.captures }, |
| ... | @@ -7148,10 +7395,9 @@ pub fn getEnumType( | ... | @@ -7148,10 +7395,9 @@ pub fn getEnumType( |
| 7148 | .zir_index = r.zir_index, | 7395 | .zir_index = r.zir_index, |
| 7149 | .type_hash = r.type_hash, | 7396 | .type_hash = r.type_hash, |
| 7150 | } }, | 7397 | } }, |
| 7151 | } }, adapter); | 7398 | } }); |
| 7152 | if (gop.found_existing) return .{ .existing = @enumFromInt(gop.index) }; | 7399 | defer gop.deinit(); |
| 7153 | assert(gop.index == ip.items.len); | 7400 | if (gop == .existing) return .{ .existing = gop.existing }; |
| 7154 | errdefer _ = ip.map.pop(); | | |
| 7155 | | 7401 | |
| 7156 | try ip.items.ensureUnusedCapacity(gpa, 1); | 7402 | try ip.items.ensureUnusedCapacity(gpa, 1); |
| 7157 | | 7403 | |
| ... | @@ -7196,7 +7442,7 @@ pub fn getEnumType( | ... | @@ -7196,7 +7442,7 @@ pub fn getEnumType( |
| 7196 | const names_start = ip.extra.items.len; | 7442 | const names_start = ip.extra.items.len; |
| 7197 | ip.extra.appendNTimesAssumeCapacity(undefined, ini.fields_len); | 7443 | ip.extra.appendNTimesAssumeCapacity(undefined, ini.fields_len); |
| 7198 | return .{ .wip = .{ | 7444 | return .{ .wip = .{ |
| 7199 | .index = @enumFromInt(gop.index), | 7445 | .index = gop.set(@enumFromInt(ip.items.len - 1)), |
| 7200 | .tag_ty_index = extra_index + std.meta.fieldIndex(EnumAuto, "int_tag_type").?, | 7446 | .tag_ty_index = extra_index + std.meta.fieldIndex(EnumAuto, "int_tag_type").?, |
| 7201 | .decl_index = extra_index + std.meta.fieldIndex(EnumAuto, "decl").?, | 7447 | .decl_index = extra_index + std.meta.fieldIndex(EnumAuto, "decl").?, |
| 7202 | .namespace_index = if (ini.has_namespace) extra_index + std.meta.fieldIndex(EnumAuto, "namespace").? else null, | 7448 | .namespace_index = if (ini.has_namespace) extra_index + std.meta.fieldIndex(EnumAuto, "namespace").? else null, |
| ... | @@ -7260,7 +7506,7 @@ pub fn getEnumType( | ... | @@ -7260,7 +7506,7 @@ pub fn getEnumType( |
| 7260 | ip.extra.appendNTimesAssumeCapacity(undefined, ini.fields_len); | 7506 | ip.extra.appendNTimesAssumeCapacity(undefined, ini.fields_len); |
| 7261 | } | 7507 | } |
| 7262 | return .{ .wip = .{ | 7508 | return .{ .wip = .{ |
| 7263 | .index = @enumFromInt(gop.index), | 7509 | .index = gop.set(@enumFromInt(ip.items.len - 1)), |
| 7264 | .tag_ty_index = extra_index + std.meta.fieldIndex(EnumAuto, "int_tag_type").?, | 7510 | .tag_ty_index = extra_index + std.meta.fieldIndex(EnumAuto, "int_tag_type").?, |
| 7265 | .decl_index = extra_index + std.meta.fieldIndex(EnumAuto, "decl").?, | 7511 | .decl_index = extra_index + std.meta.fieldIndex(EnumAuto, "decl").?, |
| 7266 | .namespace_index = if (ini.has_namespace) extra_index + std.meta.fieldIndex(EnumAuto, "namespace").? else null, | 7512 | .namespace_index = if (ini.has_namespace) extra_index + std.meta.fieldIndex(EnumAuto, "namespace").? else null, |
| ... | @@ -7288,14 +7534,13 @@ const GeneratedTagEnumTypeInit = struct { | ... | @@ -7288,14 +7534,13 @@ const GeneratedTagEnumTypeInit = struct { |
| 7288 | pub fn getGeneratedTagEnumType( | 7534 | pub fn getGeneratedTagEnumType( |
| 7289 | ip: *InternPool, | 7535 | ip: *InternPool, |
| 7290 | gpa: Allocator, | 7536 | gpa: Allocator, |
| 7291 | _: Zcu.PerThread.Id, | 7537 | tid: Zcu.PerThread.Id, |
| 7292 | ini: GeneratedTagEnumTypeInit, | 7538 | ini: GeneratedTagEnumTypeInit, |
| 7293 | ) Allocator.Error!Index { | 7539 | ) Allocator.Error!Index { |
| 7294 | assert(ip.isUnion(ini.owner_union_ty)); | 7540 | assert(ip.isUnion(ini.owner_union_ty)); |
| 7295 | assert(ip.isIntegerType(ini.tag_ty)); | 7541 | assert(ip.isIntegerType(ini.tag_ty)); |
| 7296 | for (ini.values) |val| assert(ip.typeOf(val) == ini.tag_ty); | 7542 | for (ini.values) |val| assert(ip.typeOf(val) == ini.tag_ty); |
| 7297 | | 7543 | |
| 7298 | try ip.map.ensureUnusedCapacity(gpa, 1); | | |
| 7299 | try ip.items.ensureUnusedCapacity(gpa, 1); | 7544 | try ip.items.ensureUnusedCapacity(gpa, 1); |
| 7300 | | 7545 | |
| 7301 | const names_map = try ip.addMap(gpa, ini.names.len); | 7546 | const names_map = try ip.addMap(gpa, ini.names.len); |
| ... | @@ -7304,6 +7549,7 @@ pub fn getGeneratedTagEnumType( | ... | @@ -7304,6 +7549,7 @@ pub fn getGeneratedTagEnumType( |
| 7304 | | 7549 | |
| 7305 | const fields_len: u32 = @intCast(ini.names.len); | 7550 | const fields_len: u32 = @intCast(ini.names.len); |
| 7306 | | 7551 | |
| | 7552 | const prev_extra_len = ip.extra.items.len; |
| 7307 | switch (ini.tag_mode) { | 7553 | switch (ini.tag_mode) { |
| 7308 | .auto => { | 7554 | .auto => { |
| 7309 | try ip.extra.ensureUnusedCapacity(gpa, @typeInfo(EnumAuto).Struct.fields.len + | 7555 | try ip.extra.ensureUnusedCapacity(gpa, @typeInfo(EnumAuto).Struct.fields.len + |
| ... | @@ -7360,17 +7606,17 @@ pub fn getGeneratedTagEnumType( | ... | @@ -7360,17 +7606,17 @@ pub fn getGeneratedTagEnumType( |
| 7360 | ip.extra.appendSliceAssumeCapacity(@ptrCast(ini.values)); | 7606 | ip.extra.appendSliceAssumeCapacity(@ptrCast(ini.values)); |
| 7361 | }, | 7607 | }, |
| 7362 | } | 7608 | } |
| 7363 | // Same as above | 7609 | errdefer ip.extra.items.len = prev_extra_len; |
| 7364 | errdefer @compileError("error path leaks values_map and extra data"); | 7610 | errdefer switch (ini.tag_mode) { |
| | 7611 | .auto => {}, |
| | 7612 | .explicit, .nonexhaustive => _ = if (ini.values.len != 0) ip.maps.pop(), |
| | 7613 | }; |
| 7365 | | 7614 | |
| 7366 | // Capacity for this was ensured earlier | 7615 | var gop = try ip.getOrPutKey(gpa, tid, .{ .enum_type = .{ |
| 7367 | const adapter: KeyAdapter = .{ .intern_pool = ip }; | | |
| 7368 | const gop = ip.map.getOrPutAssumeCapacityAdapted(Key{ .enum_type = .{ | | |
| 7369 | .generated_tag = .{ .union_type = ini.owner_union_ty }, | 7616 | .generated_tag = .{ .union_type = ini.owner_union_ty }, |
| 7370 | } }, adapter); | 7617 | } }); |
| 7371 | assert(!gop.found_existing); | 7618 | defer gop.deinit(); |
| 7372 | assert(gop.index == ip.items.len - 1); | 7619 | return gop.set(@enumFromInt(ip.items.len - 1)); |
| 7373 | return @enumFromInt(gop.index); | | |
| 7374 | } | 7620 | } |
| 7375 | | 7621 | |
| 7376 | pub const OpaqueTypeInit = struct { | 7622 | pub const OpaqueTypeInit = struct { |
| ... | @@ -7390,11 +7636,10 @@ pub const OpaqueTypeInit = struct { | ... | @@ -7390,11 +7636,10 @@ pub const OpaqueTypeInit = struct { |
| 7390 | pub fn getOpaqueType( | 7636 | pub fn getOpaqueType( |
| 7391 | ip: *InternPool, | 7637 | ip: *InternPool, |
| 7392 | gpa: Allocator, | 7638 | gpa: Allocator, |
| 7393 | _: Zcu.PerThread.Id, | 7639 | tid: Zcu.PerThread.Id, |
| 7394 | ini: OpaqueTypeInit, | 7640 | ini: OpaqueTypeInit, |
| 7395 | ) Allocator.Error!WipNamespaceType.Result { | 7641 | ) Allocator.Error!WipNamespaceType.Result { |
| 7396 | const adapter: KeyAdapter = .{ .intern_pool = ip }; | 7642 | var gop = try ip.getOrPutKey(gpa, tid, .{ .opaque_type = switch (ini.key) { |
| 7397 | const gop = try ip.map.getOrPutAdapted(gpa, Key{ .opaque_type = switch (ini.key) { | | |
| 7398 | .declared => |d| .{ .declared = .{ | 7643 | .declared => |d| .{ .declared = .{ |
| 7399 | .zir_index = d.zir_index, | 7644 | .zir_index = d.zir_index, |
| 7400 | .captures = .{ .external = d.captures }, | 7645 | .captures = .{ .external = d.captures }, |
| ... | @@ -7403,9 +7648,9 @@ pub fn getOpaqueType( | ... | @@ -7403,9 +7648,9 @@ pub fn getOpaqueType( |
| 7403 | .zir_index = r.zir_index, | 7648 | .zir_index = r.zir_index, |
| 7404 | .type_hash = 0, | 7649 | .type_hash = 0, |
| 7405 | } }, | 7650 | } }, |
| 7406 | } }, adapter); | 7651 | } }); |
| 7407 | if (gop.found_existing) return .{ .existing = @enumFromInt(gop.index) }; | 7652 | defer gop.deinit(); |
| 7408 | errdefer _ = ip.map.pop(); | 7653 | if (gop == .existing) return .{ .existing = gop.existing }; |
| 7409 | try ip.items.ensureUnusedCapacity(gpa, 1); | 7654 | try ip.items.ensureUnusedCapacity(gpa, 1); |
| 7410 | try ip.extra.ensureUnusedCapacity(gpa, @typeInfo(Tag.TypeOpaque).Struct.fields.len + switch (ini.key) { | 7655 | try ip.extra.ensureUnusedCapacity(gpa, @typeInfo(Tag.TypeOpaque).Struct.fields.len + switch (ini.key) { |
| 7411 | .declared => |d| d.captures.len, | 7656 | .declared => |d| d.captures.len, |
| ... | @@ -7431,7 +7676,7 @@ pub fn getOpaqueType( | ... | @@ -7431,7 +7676,7 @@ pub fn getOpaqueType( |
| 7431 | .reified => {}, | 7676 | .reified => {}, |
| 7432 | } | 7677 | } |
| 7433 | return .{ .wip = .{ | 7678 | return .{ .wip = .{ |
| 7434 | .index = @enumFromInt(gop.index), | 7679 | .index = gop.set(@enumFromInt(ip.items.len - 1)), |
| 7435 | .decl_extra_index = extra_index + std.meta.fieldIndex(Tag.TypeOpaque, "decl").?, | 7680 | .decl_extra_index = extra_index + std.meta.fieldIndex(Tag.TypeOpaque, "decl").?, |
| 7436 | .namespace_extra_index = if (ini.has_namespace) | 7681 | .namespace_extra_index = if (ini.has_namespace) |
| 7437 | extra_index + std.meta.fieldIndex(Tag.TypeOpaque, "namespace").? | 7682 | extra_index + std.meta.fieldIndex(Tag.TypeOpaque, "namespace").? |
| ... | @@ -7441,9 +7686,19 @@ pub fn getOpaqueType( | ... | @@ -7441,9 +7686,19 @@ pub fn getOpaqueType( |
| 7441 | } | 7686 | } |
| 7442 | | 7687 | |
| 7443 | pub fn getIfExists(ip: *const InternPool, key: Key) ?Index { | 7688 | pub fn getIfExists(ip: *const InternPool, key: Key) ?Index { |
| 7444 | const adapter: KeyAdapter = .{ .intern_pool = ip }; | 7689 | const full_hash = key.hash64(ip); |
| 7445 | const index = ip.map.getIndexAdapted(key, adapter) orelse return null; | 7690 | const hash: u32 = @truncate(full_hash >> 32); |
| 7446 | return @enumFromInt(index); | 7691 | const shard = &ip.shards[@intCast(full_hash & (ip.shards.len - 1))]; |
| | 7692 | const map = shard.map.acquire(); |
| | 7693 | const map_mask = map.header().mask(); |
| | 7694 | var map_index = hash; |
| | 7695 | while (true) : (map_index += Shard.Map.Entry.fields_len) { |
| | 7696 | map_index &= map_mask; |
| | 7697 | const entry = map.at(map_index); |
| | 7698 | const index = entry.acquire(); |
| | 7699 | if (index == .none) return null; |
| | 7700 | if (entry.hash == hash and ip.indexToKey(index).eql(key, ip)) return index; |
| | 7701 | } |
| 7447 | } | 7702 | } |
| 7448 | | 7703 | |
| 7449 | pub fn getAssumeExists(ip: *const InternPool, key: Key) Index { | 7704 | pub fn getAssumeExists(ip: *const InternPool, key: Key) Index { |
| ... | @@ -7506,7 +7761,6 @@ pub fn remove(ip: *InternPool, index: Index) void { | ... | @@ -7506,7 +7761,6 @@ pub fn remove(ip: *InternPool, index: Index) void { |
| 7506 | if (@intFromEnum(index) == ip.items.len - 1) { | 7761 | if (@intFromEnum(index) == ip.items.len - 1) { |
| 7507 | // Happy case - we can just drop the item without affecting any other indices. | 7762 | // Happy case - we can just drop the item without affecting any other indices. |
| 7508 | ip.items.len -= 1; | 7763 | ip.items.len -= 1; |
| 7509 | _ = ip.map.pop(); | | |
| 7510 | } else { | 7764 | } else { |
| 7511 | // We must preserve the item so that indices following it remain valid. | 7765 | // We must preserve the item so that indices following it remain valid. |
| 7512 | // Thus, we will rewrite the tag to `removed`, leaking the item until | 7766 | // Thus, we will rewrite the tag to `removed`, leaking the item until |
| ... | @@ -8133,35 +8387,34 @@ fn getCoercedFuncInstance( | ... | @@ -8133,35 +8387,34 @@ fn getCoercedFuncInstance( |
| 8133 | fn getCoercedFunc( | 8387 | fn getCoercedFunc( |
| 8134 | ip: *InternPool, | 8388 | ip: *InternPool, |
| 8135 | gpa: Allocator, | 8389 | gpa: Allocator, |
| 8136 | _: Zcu.PerThread.Id, | 8390 | tid: Zcu.PerThread.Id, |
| 8137 | func: Index, | 8391 | func: Index, |
| 8138 | ty: Index, | 8392 | ty: Index, |
| 8139 | ) Allocator.Error!Index { | 8393 | ) Allocator.Error!Index { |
| 8140 | const prev_extra_len = ip.extra.items.len; | 8394 | const prev_extra_len = ip.extra.items.len; |
| 8141 | try ip.extra.ensureUnusedCapacity(gpa, @typeInfo(Tag.FuncCoerced).Struct.fields.len); | 8395 | try ip.extra.ensureUnusedCapacity(gpa, @typeInfo(Tag.FuncCoerced).Struct.fields.len); |
| 8142 | try ip.items.ensureUnusedCapacity(gpa, 1); | 8396 | try ip.items.ensureUnusedCapacity(gpa, 1); |
| 8143 | try ip.map.ensureUnusedCapacity(gpa, 1); | | |
| 8144 | | 8397 | |
| 8145 | const extra_index = ip.addExtraAssumeCapacity(Tag.FuncCoerced{ | 8398 | const extra_index = ip.addExtraAssumeCapacity(Tag.FuncCoerced{ |
| 8146 | .ty = ty, | 8399 | .ty = ty, |
| 8147 | .func = func, | 8400 | .func = func, |
| 8148 | }); | 8401 | }); |
| | 8402 | errdefer ip.extra.items.len = prev_extra_len; |
| 8149 | | 8403 | |
| 8150 | const adapter: KeyAdapter = .{ .intern_pool = ip }; | 8404 | var gop = try ip.getOrPutKey(gpa, tid, .{ |
| 8151 | const gop = ip.map.getOrPutAssumeCapacityAdapted(Key{ | | |
| 8152 | .func = extraFuncCoerced(ip, extra_index), | 8405 | .func = extraFuncCoerced(ip, extra_index), |
| 8153 | }, adapter); | 8406 | }); |
| 8154 | | 8407 | defer gop.deinit(); |
| 8155 | if (gop.found_existing) { | 8408 | if (gop == .existing) { |
| 8156 | ip.extra.items.len = prev_extra_len; | 8409 | ip.extra.items.len = prev_extra_len; |
| 8157 | return @enumFromInt(gop.index); | 8410 | return gop.existing; |
| 8158 | } | 8411 | } |
| 8159 | | 8412 | |
| 8160 | ip.items.appendAssumeCapacity(.{ | 8413 | ip.items.appendAssumeCapacity(.{ |
| 8161 | .tag = .func_coerced, | 8414 | .tag = .func_coerced, |
| 8162 | .data = extra_index, | 8415 | .data = extra_index, |
| 8163 | }); | 8416 | }); |
| 8164 | return @enumFromInt(ip.items.len - 1); | 8417 | return gop.set(@enumFromInt(ip.items.len - 1)); |
| 8165 | } | 8418 | } |
| 8166 | | 8419 | |
| 8167 | /// Asserts `val` has an integer type. | 8420 | /// Asserts `val` has an integer type. |
Jacob Young <jacobly@ziglang.org> 2024-06-15 16:18:41-04:00
lib