| ... | @@ -46,14 +46,6 @@ namespaces_free_list: std.ArrayListUnmanaged(NamespaceIndex) = .{}, | ... | @@ -46,14 +46,6 @@ namespaces_free_list: std.ArrayListUnmanaged(NamespaceIndex) = .{}, |
| 46 | /// These are not serialized; it is computed upon deserialization. | 46 | /// These are not serialized; it is computed upon deserialization. |
| 47 | maps: std.ArrayListUnmanaged(FieldMap) = .{}, | 47 | maps: std.ArrayListUnmanaged(FieldMap) = .{}, |
| 48 | | 48 | |
| 49 | /// Used for finding the index inside `string_bytes`. | | |
| 50 | string_table: std.HashMapUnmanaged( | | |
| 51 | u32, | | |
| 52 | void, | | |
| 53 | std.hash_map.StringIndexContext, | | |
| 54 | std.hash_map.default_max_load_percentage, | | |
| 55 | ) = .{}, | | |
| 56 | | | |
| 57 | /// An index into `tracked_insts` gives a reference to a single ZIR instruction which | 49 | /// An index into `tracked_insts` gives a reference to a single ZIR instruction which |
| 58 | /// persists across incremental updates. | 50 | /// persists across incremental updates. |
| 59 | tracked_insts: std.AutoArrayHashMapUnmanaged(TrackedInst, void) = .{}, | 51 | tracked_insts: std.AutoArrayHashMapUnmanaged(TrackedInst, void) = .{}, |
| ... | @@ -358,22 +350,31 @@ const Local = struct { | ... | @@ -358,22 +350,31 @@ const Local = struct { |
| 358 | /// node: Garbage.Node, | 350 | /// node: Garbage.Node, |
| 359 | /// header: List.Header, | 351 | /// header: List.Header, |
| 360 | /// data: [capacity]u32, | 352 | /// data: [capacity]u32, |
| 361 | /// tag: [capacity]Tag, | 353 | /// tag: [header.capacity]Tag, |
| 362 | items: List, | 354 | items: List, |
| 363 | | 355 | |
| 364 | /// node: Garbage.Node, | 356 | /// node: Garbage.Node, |
| 365 | /// header: List.Header, | 357 | /// header: List.Header, |
| 366 | /// extra: [capacity]u32, | 358 | /// extra: [header.capacity]u32, |
| 367 | extra: List, | 359 | extra: List, |
| 368 | | 360 | |
| | 361 | /// node: Garbage.Node, |
| | 362 | /// header: List.Header, |
| | 363 | /// bytes: [header.capacity]u8, |
| | 364 | strings: List, |
| | 365 | |
| 369 | garbage: Garbage, | 366 | garbage: Garbage, |
| 370 | | 367 | |
| 371 | const List = struct { | 368 | const List = struct { |
| 372 | entries: [*]u32, | 369 | entries: [*]u32, |
| 373 | | 370 | |
| 374 | const empty: List = .{ | 371 | const empty: List = .{ .entries = @constCast(&(extern struct { |
| 375 | .entries = @constCast(&[_]u32{ 0, 0 })[Header.fields_len..].ptr, | 372 | header: Header, |
| 376 | }; | 373 | entries: [0]u32, |
| | 374 | }{ |
| | 375 | .header = .{ .len = 0, .capacity = 0 }, |
| | 376 | .entries = .{}, |
| | 377 | }).entries) }; |
| 377 | | 378 | |
| 378 | fn acquire(list: *const List) List { | 379 | fn acquire(list: *const List) List { |
| 379 | return .{ .entries = @atomicLoad([*]u32, &list.entries, .acquire) }; | 380 | return .{ .entries = @atomicLoad([*]u32, &list.entries, .acquire) }; |
| ... | @@ -402,63 +403,75 @@ const Local = struct { | ... | @@ -402,63 +403,75 @@ const Local = struct { |
| 402 | }; | 403 | }; |
| 403 | | 404 | |
| 404 | const Shard = struct { | 405 | const Shard = struct { |
| 405 | aligned: void align(std.atomic.cache_line) = {}, | 406 | shared: struct { |
| 406 | | 407 | map: Map(Index), |
| 407 | mutate_mutex: std.Thread.Mutex.Recursive, | 408 | string_map: Map(OptionalNullTerminatedString), |
| 408 | | 409 | } align(std.atomic.cache_line), |
| 409 | /// node: Local.Garbage.Node, | 410 | mutate: struct { |
| 410 | /// header: Map.Header, | 411 | // TODO: measure cost of sharing unrelated mutate state |
| 411 | /// entries: [capacity]Map.Entry, | 412 | map: Mutate align(std.atomic.cache_line), |
| 412 | map: Map, | 413 | string_map: Mutate align(std.atomic.cache_line), |
| | 414 | }, |
| 413 | | 415 | |
| 414 | const Map = struct { | 416 | const Mutate = struct { |
| 415 | entries: [*]u32, | 417 | mutex: std.Thread.Mutex.Recursive, |
| | 418 | len: u32, |
| 416 | | 419 | |
| 417 | const empty: Map = .{ | 420 | const empty: Mutate = .{ |
| 418 | .entries = @constCast(&[_]u32{ 0, 1, @intFromEnum(Index.none), 0 })[Header.fields_len..].ptr, | 421 | .mutex = std.Thread.Mutex.Recursive.init, |
| | 422 | .len = 0, |
| 419 | }; | 423 | }; |
| | 424 | }; |
| 420 | | 425 | |
| 421 | fn acquire(map: *const Map) Map { | 426 | fn Map(comptime Value: type) type { |
| 422 | return .{ .entries = @atomicLoad([*]u32, &map.entries, .acquire) }; | 427 | comptime assert(@typeInfo(Value).Enum.tag_type == u32); |
| 423 | } | 428 | _ = @as(Value, .none); // expected .none key |
| 424 | fn release(map: *Map, new_map: Map) void { | 429 | return struct { |
| 425 | @atomicStore([*]u32, &map.entries, new_map.entries, .release); | 430 | /// node: Local.Garbage.Node, |
| 426 | } | 431 | /// header: Header, |
| 427 | | 432 | /// entries: [header.capacity]Entry, |
| 428 | const Header = extern struct { | 433 | entries: [*]Entry, |
| 429 | len: u32, | 434 | |
| 430 | capacity: u32, | 435 | const empty: @This() = .{ .entries = @constCast(&(extern struct { |
| | 436 | header: Header, |
| | 437 | entries: [1]Entry, |
| | 438 | }{ |
| | 439 | .header = .{ .capacity = 1 }, |
| | 440 | .entries = .{.{ .value = .none, .hash = undefined }}, |
| | 441 | }).entries) }; |
| | 442 | |
| | 443 | fn acquire(map: *const @This()) @This() { |
| | 444 | return .{ .entries = @atomicLoad([*]Entry, &map.entries, .acquire) }; |
| | 445 | } |
| | 446 | fn release(map: *@This(), new_map: @This()) void { |
| | 447 | @atomicStore([*]Entry, &map.entries, new_map.entries, .release); |
| | 448 | } |
| 431 | | 449 | |
| 432 | const fields_len: u32 = @typeInfo(Header).Struct.fields.len; | 450 | const Header = extern struct { |
| | 451 | capacity: u32, |
| 433 | | 452 | |
| 434 | fn mask(head: *const Header) u32 { | 453 | fn mask(head: *const Header) u32 { |
| 435 | assert(std.math.isPowerOfTwo(head.capacity)); | 454 | assert(std.math.isPowerOfTwo(head.capacity)); |
| 436 | assert(std.math.isPowerOfTwo(Entry.fields_len)); | 455 | return head.capacity - 1; |
| 437 | return (head.capacity - 1) * Entry.fields_len; | 456 | } |
| | 457 | }; |
| | 458 | fn header(map: @This()) *Header { |
| | 459 | return &(@as([*]Header, @ptrCast(map.entries)) - 1)[0]; |
| 438 | } | 460 | } |
| 439 | }; | | |
| 440 | fn header(map: Map) *Header { | | |
| 441 | return @ptrCast(map.entries - Header.fields_len); | | |
| 442 | } | | |
| 443 | | 461 | |
| 444 | const Entry = extern struct { | 462 | const Entry = extern struct { |
| 445 | index: Index, | 463 | value: Value, |
| 446 | hash: u32, | 464 | hash: u32, |
| 447 | | 465 | |
| 448 | const fields_len: u32 = @typeInfo(Entry).Struct.fields.len; | 466 | fn acquire(entry: *const Entry) Value { |
| 449 | | 467 | return @atomicLoad(Value, &entry.value, .acquire); |
| 450 | fn acquire(entry: *const Entry) Index { | 468 | } |
| 451 | return @atomicLoad(Index, &entry.index, .acquire); | 469 | fn release(entry: *Entry, value: Value) void { |
| 452 | } | 470 | @atomicStore(Value, &entry.value, value, .release); |
| 453 | fn release(entry: *Entry, index: Index) void { | 471 | } |
| 454 | @atomicStore(Index, &entry.index, index, .release); | 472 | }; |
| 455 | } | | |
| 456 | }; | 473 | }; |
| 457 | fn at(map: Map, index: usize) *Entry { | 474 | } |
| 458 | assert(index % Entry.fields_len == 0); | | |
| 459 | return @ptrCast(&map.entries[index]); | | |
| 460 | } | | |
| 461 | }; | | |
| 462 | }; | 475 | }; |
| 463 | | 476 | |
| 464 | const FieldMap = std.ArrayHashMapUnmanaged(void, void, std.array_hash_map.AutoContext(void), false); | 477 | const FieldMap = std.ArrayHashMapUnmanaged(void, void, std.array_hash_map.AutoContext(void), false); |
| ... | @@ -618,9 +631,13 @@ pub const NullTerminatedString = enum(u32) { | ... | @@ -618,9 +631,13 @@ pub const NullTerminatedString = enum(u32) { |
| 618 | return @enumFromInt(@intFromEnum(self)); | 631 | return @enumFromInt(@intFromEnum(self)); |
| 619 | } | 632 | } |
| 620 | | 633 | |
| | 634 | fn toOverlongSlice(string: NullTerminatedString, ip: *const InternPool) []const u8 { |
| | 635 | return ip.string_bytes.items[@intFromEnum(string)..]; |
| | 636 | } |
| | 637 | |
| 621 | pub fn toSlice(string: NullTerminatedString, ip: *const InternPool) [:0]const u8 { | 638 | pub fn toSlice(string: NullTerminatedString, ip: *const InternPool) [:0]const u8 { |
| 622 | const slice = ip.string_bytes.items[@intFromEnum(string)..]; | 639 | const overlong_slice = string.toOverlongSlice(ip); |
| 623 | return slice[0..std.mem.indexOfScalar(u8, slice, 0).? :0]; | 640 | return overlong_slice[0..std.mem.indexOfScalar(u8, overlong_slice, 0).? :0]; |
| 624 | } | 641 | } |
| 625 | | 642 | |
| 626 | pub fn length(string: NullTerminatedString, ip: *const InternPool) u32 { | 643 | pub fn length(string: NullTerminatedString, ip: *const InternPool) u32 { |
| ... | @@ -628,7 +645,10 @@ pub const NullTerminatedString = enum(u32) { | ... | @@ -628,7 +645,10 @@ pub const NullTerminatedString = enum(u32) { |
| 628 | } | 645 | } |
| 629 | | 646 | |
| 630 | pub fn eqlSlice(string: NullTerminatedString, slice: []const u8, ip: *const InternPool) bool { | 647 | pub fn eqlSlice(string: NullTerminatedString, slice: []const u8, ip: *const InternPool) bool { |
| 631 | return std.mem.eql(u8, string.toSlice(ip), slice); | 648 | const overlong_slice = string.toOverlongSlice(ip); |
| | 649 | return overlong_slice.len > slice.len and |
| | 650 | std.mem.eql(u8, overlong_slice[0..slice.len], slice) and |
| | 651 | overlong_slice[slice.len] == 0; |
| 632 | } | 652 | } |
| 633 | | 653 | |
| 634 | const Adapter = struct { | 654 | const Adapter = struct { |
| ... | @@ -4639,14 +4659,21 @@ pub fn init(ip: *InternPool, gpa: Allocator, total_threads: usize) !void { | ... | @@ -4639,14 +4659,21 @@ pub fn init(ip: *InternPool, gpa: Allocator, total_threads: usize) !void { |
| 4639 | @memset(ip.local, .{ | 4659 | @memset(ip.local, .{ |
| 4640 | .items = Local.List.empty, | 4660 | .items = Local.List.empty, |
| 4641 | .extra = Local.List.empty, | 4661 | .extra = Local.List.empty, |
| | 4662 | .strings = Local.List.empty, |
| 4642 | .garbage = .{}, | 4663 | .garbage = .{}, |
| 4643 | }); | 4664 | }); |
| 4644 | | 4665 | |
| 4645 | ip.shard_shift = @intCast(std.math.log2_int_ceil(usize, total_threads)); | 4666 | ip.shard_shift = @intCast(std.math.log2_int_ceil(usize, total_threads)); |
| 4646 | ip.shards = try gpa.alloc(Shard, @as(usize, 1) << ip.shard_shift); | 4667 | ip.shards = try gpa.alloc(Shard, @as(usize, 1) << ip.shard_shift); |
| 4647 | @memset(ip.shards, .{ | 4668 | @memset(ip.shards, .{ |
| 4648 | .mutate_mutex = std.Thread.Mutex.Recursive.init, | 4669 | .shared = .{ |
| 4649 | .map = Shard.Map.empty, | 4670 | .map = Shard.Map(Index).empty, |
| | 4671 | .string_map = Shard.Map(OptionalNullTerminatedString).empty, |
| | 4672 | }, |
| | 4673 | .mutate = .{ |
| | 4674 | .map = Shard.Mutate.empty, |
| | 4675 | .string_map = Shard.Mutate.empty, |
| | 4676 | }, |
| 4650 | }); | 4677 | }); |
| 4651 | | 4678 | |
| 4652 | // Reserve string index 0 for an empty string. | 4679 | // Reserve string index 0 for an empty string. |
| ... | @@ -4697,8 +4724,6 @@ pub fn deinit(ip: *InternPool, gpa: Allocator) void { | ... | @@ -4697,8 +4724,6 @@ pub fn deinit(ip: *InternPool, gpa: Allocator) void { |
| 4697 | for (ip.maps.items) |*map| map.deinit(gpa); | 4724 | for (ip.maps.items) |*map| map.deinit(gpa); |
| 4698 | ip.maps.deinit(gpa); | 4725 | ip.maps.deinit(gpa); |
| 4699 | | 4726 | |
| 4700 | ip.string_table.deinit(gpa); | | |
| 4701 | | | |
| 4702 | ip.tracked_insts.deinit(gpa); | 4727 | ip.tracked_insts.deinit(gpa); |
| 4703 | | 4728 | |
| 4704 | ip.src_hash_deps.deinit(gpa); | 4729 | ip.src_hash_deps.deinit(gpa); |
| ... | @@ -5363,9 +5388,9 @@ const GetOrPutKey = union(enum) { | ... | @@ -5363,9 +5388,9 @@ const GetOrPutKey = union(enum) { |
| 5363 | switch (gop.*) { | 5388 | switch (gop.*) { |
| 5364 | .existing => unreachable, | 5389 | .existing => unreachable, |
| 5365 | .new => |info| { | 5390 | .new => |info| { |
| 5366 | info.shard.map.at(info.map_index).release(index); | 5391 | info.shard.shared.map.entries[info.map_index].release(index); |
| 5367 | info.shard.map.header().len += 1; | 5392 | info.shard.mutate.map.len += 1; |
| 5368 | info.shard.mutate_mutex.unlock(); | 5393 | info.shard.mutate.map.mutex.unlock(); |
| 5369 | }, | 5394 | }, |
| 5370 | } | 5395 | } |
| 5371 | gop.* = .{ .existing = index }; | 5396 | gop.* = .{ .existing = index }; |
| ... | @@ -5380,7 +5405,7 @@ const GetOrPutKey = union(enum) { | ... | @@ -5380,7 +5405,7 @@ const GetOrPutKey = union(enum) { |
| 5380 | fn deinit(gop: *GetOrPutKey) void { | 5405 | fn deinit(gop: *GetOrPutKey) void { |
| 5381 | switch (gop.*) { | 5406 | switch (gop.*) { |
| 5382 | .existing => {}, | 5407 | .existing => {}, |
| 5383 | .new => |info| info.shard.mutate_mutex.unlock(), | 5408 | .new => |info| info.shard.mutate.map.mutex.unlock(), |
| 5384 | } | 5409 | } |
| 5385 | gop.* = undefined; | 5410 | gop.* = undefined; |
| 5386 | } | 5411 | } |
| ... | @@ -5394,70 +5419,69 @@ fn getOrPutKey( | ... | @@ -5394,70 +5419,69 @@ fn getOrPutKey( |
| 5394 | const full_hash = key.hash64(ip); | 5419 | const full_hash = key.hash64(ip); |
| 5395 | const hash: u32 = @truncate(full_hash >> 32); | 5420 | const hash: u32 = @truncate(full_hash >> 32); |
| 5396 | const shard = &ip.shards[@intCast(full_hash & (ip.shards.len - 1))]; | 5421 | const shard = &ip.shards[@intCast(full_hash & (ip.shards.len - 1))]; |
| 5397 | var map = shard.map.acquire(); | 5422 | var map = shard.shared.map.acquire(); |
| | 5423 | const Map = @TypeOf(map); |
| 5398 | var map_mask = map.header().mask(); | 5424 | var map_mask = map.header().mask(); |
| 5399 | var map_index = hash; | 5425 | var map_index = hash; |
| 5400 | while (true) : (map_index += Shard.Map.Entry.fields_len) { | 5426 | while (true) : (map_index += 1) { |
| 5401 | map_index &= map_mask; | 5427 | map_index &= map_mask; |
| 5402 | const entry = map.at(map_index); | 5428 | const entry = &map.entries[map_index]; |
| 5403 | const index = entry.acquire(); | 5429 | const index = entry.acquire(); |
| 5404 | if (index == .none) break; | 5430 | if (index == .none) break; |
| 5405 | if (entry.hash == hash and ip.indexToKey(index).eql(key, ip)) | 5431 | if (entry.hash != hash) continue; |
| 5406 | return .{ .existing = index }; | 5432 | if (ip.indexToKey(index).eql(key, ip)) return .{ .existing = index }; |
| 5407 | } | 5433 | } |
| 5408 | shard.mutate_mutex.lock(); | 5434 | shard.mutate.map.mutex.lock(); |
| 5409 | errdefer shard.mutate_mutex.unlock(); | 5435 | errdefer shard.mutate.map.mutex.unlock(); |
| 5410 | if (map.entries != shard.map.entries) { | 5436 | if (map.entries != shard.shared.map.entries) { |
| 5411 | map = shard.map; | 5437 | map = shard.shared.map; |
| 5412 | map_mask = map.header().mask(); | 5438 | map_mask = map.header().mask(); |
| 5413 | map_index = hash; | 5439 | map_index = hash; |
| 5414 | } | 5440 | } |
| 5415 | while (true) : (map_index += Shard.Map.Entry.fields_len) { | 5441 | while (true) : (map_index += 1) { |
| 5416 | map_index &= map_mask; | 5442 | map_index &= map_mask; |
| 5417 | const entry = map.at(map_index); | 5443 | const entry = &map.entries[map_index]; |
| 5418 | const index = entry.index; | 5444 | const index = entry.value; |
| 5419 | if (index == .none) break; | 5445 | if (index == .none) break; |
| 5420 | if (entry.hash == hash and ip.indexToKey(index).eql(key, ip)) { | 5446 | if (entry.hash != hash) continue; |
| 5421 | defer shard.mutate_mutex.unlock(); | 5447 | if (ip.indexToKey(index).eql(key, ip)) { |
| | 5448 | defer shard.mutate.map.mutex.unlock(); |
| 5422 | return .{ .existing = index }; | 5449 | return .{ .existing = index }; |
| 5423 | } | 5450 | } |
| 5424 | } | 5451 | } |
| 5425 | const map_header = map.header().*; | 5452 | const map_header = map.header().*; |
| 5426 | if (map_header.len >= map_header.capacity * 3 / 5) { | 5453 | if (shard.mutate.map.len >= map_header.capacity * 3 / 5) { |
| 5427 | const new_map_capacity = map_header.capacity * 2; | 5454 | const new_map_capacity = map_header.capacity * 2; |
| 5428 | const new_map_buf = try gpa.alignedAlloc( | 5455 | const new_map_buf = try gpa.alignedAlloc( |
| 5429 | u8, | 5456 | u8, |
| 5430 | Local.garbage_align, | 5457 | Local.garbage_align, |
| 5431 | @sizeOf(Local.Garbage.Node) + (Shard.Map.Header.fields_len + | 5458 | @sizeOf(Local.Garbage.Node) + @sizeOf(Map.Header) + |
| 5432 | new_map_capacity * Shard.Map.Entry.fields_len) * @sizeOf(u32), | 5459 | new_map_capacity * @sizeOf(Map.Entry), |
| 5433 | ); | 5460 | ); |
| 5434 | const new_node: *Local.Garbage.Node = @ptrCast(new_map_buf.ptr); | 5461 | const new_node: *Local.Garbage.Node = @ptrCast(new_map_buf.ptr); |
| 5435 | new_node.* = .{ .data = .{ .buf_len = new_map_buf.len } }; | 5462 | new_node.* = .{ .data = .{ .buf_len = new_map_buf.len } }; |
| 5436 | ip.local[@intFromEnum(tid)].garbage.prepend(new_node); | 5463 | ip.local[@intFromEnum(tid)].garbage.prepend(new_node); |
| 5437 | const new_map_entries = std.mem.bytesAsSlice( | 5464 | const new_map_entries = std.mem.bytesAsSlice( |
| 5438 | u32, | 5465 | Map.Entry, |
| 5439 | new_map_buf[@sizeOf(Local.Garbage.Node)..], | 5466 | new_map_buf[@sizeOf(Local.Garbage.Node) + @sizeOf(Map.Header) ..], |
| 5440 | )[Shard.Map.Header.fields_len..]; | 5467 | ); |
| 5441 | const new_map: Shard.Map = .{ .entries = new_map_entries.ptr }; | 5468 | const new_map: Map = .{ .entries = new_map_entries.ptr }; |
| 5442 | new_map.header().* = .{ | 5469 | new_map.header().* = .{ .capacity = new_map_capacity }; |
| 5443 | .len = map_header.len, | 5470 | @memset(new_map_entries, .{ .value = .none, .hash = undefined }); |
| 5444 | .capacity = new_map_capacity, | | |
| 5445 | }; | | |
| 5446 | @memset(new_map_entries, @intFromEnum(Index.none)); | | |
| 5447 | const new_map_mask = new_map.header().mask(); | 5471 | const new_map_mask = new_map.header().mask(); |
| 5448 | map_index = 0; | 5472 | map_index = 0; |
| 5449 | while (map_index < map_header.capacity * 2) : (map_index += Shard.Map.Entry.fields_len) { | 5473 | while (map_index < map_header.capacity) : (map_index += 1) { |
| 5450 | const entry = map.at(map_index); | 5474 | const entry = &map.entries[map_index]; |
| 5451 | const index = entry.index; | 5475 | const index = entry.value; |
| 5452 | if (index == .none) continue; | 5476 | if (index == .none) continue; |
| 5453 | const item_hash = entry.hash; | 5477 | const item_hash = entry.hash; |
| 5454 | var new_map_index = item_hash; | 5478 | var new_map_index = item_hash; |
| 5455 | while (true) : (new_map_index += Shard.Map.Entry.fields_len) { | 5479 | while (true) : (new_map_index += 1) { |
| 5456 | new_map_index &= new_map_mask; | 5480 | new_map_index &= new_map_mask; |
| 5457 | const new_entry = new_map.at(new_map_index); | 5481 | const new_entry = &new_map.entries[new_map_index]; |
| 5458 | if (new_entry.index != .none) continue; | 5482 | if (new_entry.value != .none) continue; |
| 5459 | new_entry.* = .{ | 5483 | new_entry.* = .{ |
| 5460 | .index = index, | 5484 | .value = index, |
| 5461 | .hash = item_hash, | 5485 | .hash = item_hash, |
| 5462 | }; | 5486 | }; |
| 5463 | break; | 5487 | break; |
| ... | @@ -5465,13 +5489,13 @@ fn getOrPutKey( | ... | @@ -5465,13 +5489,13 @@ fn getOrPutKey( |
| 5465 | } | 5489 | } |
| 5466 | map = new_map; | 5490 | map = new_map; |
| 5467 | map_index = hash; | 5491 | map_index = hash; |
| 5468 | while (true) : (map_index += Shard.Map.Entry.fields_len) { | 5492 | while (true) : (map_index += 1) { |
| 5469 | map_index &= new_map_mask; | 5493 | map_index &= new_map_mask; |
| 5470 | if (map.at(map_index).index == .none) break; | 5494 | if (map.entries[map_index].value == .none) break; |
| 5471 | } | 5495 | } |
| 5472 | shard.map.release(new_map); | 5496 | shard.shared.map.release(new_map); |
| 5473 | } | 5497 | } |
| 5474 | map.at(map_index).hash = hash; | 5498 | map.entries[map_index].hash = hash; |
| 5475 | return .{ .new = .{ .shard = shard, .map_index = map_index } }; | 5499 | return .{ .new = .{ .shard = shard, .map_index = map_index } }; |
| 5476 | } | 5500 | } |
| 5477 | | 5501 | |
| ... | @@ -7689,22 +7713,19 @@ pub fn getIfExists(ip: *const InternPool, key: Key) ?Index { | ... | @@ -7689,22 +7713,19 @@ pub fn getIfExists(ip: *const InternPool, key: Key) ?Index { |
| 7689 | const full_hash = key.hash64(ip); | 7713 | const full_hash = key.hash64(ip); |
| 7690 | const hash: u32 = @truncate(full_hash >> 32); | 7714 | const hash: u32 = @truncate(full_hash >> 32); |
| 7691 | const shard = &ip.shards[@intCast(full_hash & (ip.shards.len - 1))]; | 7715 | const shard = &ip.shards[@intCast(full_hash & (ip.shards.len - 1))]; |
| 7692 | const map = shard.map.acquire(); | 7716 | const map = shard.shared.map.acquire(); |
| 7693 | const map_mask = map.header().mask(); | 7717 | const map_mask = map.header().mask(); |
| 7694 | var map_index = hash; | 7718 | var map_index = hash; |
| 7695 | while (true) : (map_index += Shard.Map.Entry.fields_len) { | 7719 | while (true) : (map_index += 1) { |
| 7696 | map_index &= map_mask; | 7720 | map_index &= map_mask; |
| 7697 | const entry = map.at(map_index); | 7721 | const entry = &map.entries[map_index]; |
| 7698 | const index = entry.acquire(); | 7722 | const index = entry.acquire(); |
| 7699 | if (index == .none) return null; | 7723 | if (index == .none) return null; |
| 7700 | if (entry.hash == hash and ip.indexToKey(index).eql(key, ip)) return index; | 7724 | if (entry.hash != hash) continue; |
| | 7725 | if (ip.indexToKey(index).eql(key, ip)) return index; |
| 7701 | } | 7726 | } |
| 7702 | } | 7727 | } |
| 7703 | | 7728 | |
| 7704 | pub fn getAssumeExists(ip: *const InternPool, key: Key) Index { | | |
| 7705 | return ip.getIfExists(key).?; | | |
| 7706 | } | | |
| 7707 | | | |
| 7708 | fn addStringsToMap( | 7729 | fn addStringsToMap( |
| 7709 | ip: *InternPool, | 7730 | ip: *InternPool, |
| 7710 | map_index: MapIndex, | 7731 | map_index: MapIndex, |
| ... | @@ -8618,7 +8639,13 @@ fn dumpStatsFallible(ip: *const InternPool, arena: Allocator) anyerror!void { | ... | @@ -8618,7 +8639,13 @@ fn dumpStatsFallible(ip: *const InternPool, arena: Allocator) anyerror!void { |
| 8618 | .type_inferred_error_set => 0, | 8639 | .type_inferred_error_set => 0, |
| 8619 | .type_enum_explicit, .type_enum_nonexhaustive => b: { | 8640 | .type_enum_explicit, .type_enum_nonexhaustive => b: { |
| 8620 | const info = ip.extraData(EnumExplicit, data); | 8641 | const info = ip.extraData(EnumExplicit, data); |
| 8621 | var ints = @typeInfo(EnumExplicit).Struct.fields.len + info.captures_len + info.fields_len; | 8642 | var ints = @typeInfo(EnumExplicit).Struct.fields.len; |
| | 8643 | if (info.zir_index == .none) ints += 1; |
| | 8644 | ints += if (info.captures_len != std.math.maxInt(u32)) |
| | 8645 | info.captures_len |
| | 8646 | else |
| | 8647 | @typeInfo(PackedU64).Struct.fields.len; |
| | 8648 | ints += info.fields_len; |
| 8622 | if (info.values_map != .none) ints += info.fields_len; | 8649 | if (info.values_map != .none) ints += info.fields_len; |
| 8623 | break :b @sizeOf(u32) * ints; | 8650 | break :b @sizeOf(u32) * ints; |
| 8624 | }, | 8651 | }, |
| ... | @@ -9084,7 +9111,6 @@ pub fn getOrPutTrailingString( | ... | @@ -9084,7 +9111,6 @@ pub fn getOrPutTrailingString( |
| 9084 | len: usize, | 9111 | len: usize, |
| 9085 | comptime embedded_nulls: EmbeddedNulls, | 9112 | comptime embedded_nulls: EmbeddedNulls, |
| 9086 | ) Allocator.Error!embedded_nulls.StringType() { | 9113 | ) Allocator.Error!embedded_nulls.StringType() { |
| 9087 | _ = tid; | | |
| 9088 | const string_bytes = &ip.string_bytes; | 9114 | const string_bytes = &ip.string_bytes; |
| 9089 | const str_index: u32 = @intCast(string_bytes.items.len - len); | 9115 | const str_index: u32 = @intCast(string_bytes.items.len - len); |
| 9090 | if (len > 0 and string_bytes.getLast() == 0) { | 9116 | if (len > 0 and string_bytes.getLast() == 0) { |
| ... | @@ -9101,25 +9127,123 @@ pub fn getOrPutTrailingString( | ... | @@ -9101,25 +9127,123 @@ pub fn getOrPutTrailingString( |
| 9101 | return @enumFromInt(str_index); | 9127 | return @enumFromInt(str_index); |
| 9102 | }, | 9128 | }, |
| 9103 | } | 9129 | } |
| 9104 | const gop = try ip.string_table.getOrPutContextAdapted(gpa, key, std.hash_map.StringIndexAdapter{ | 9130 | const maybe_existing_index = try ip.getOrPutStringValue(gpa, tid, key, @enumFromInt(str_index)); |
| 9105 | .bytes = string_bytes, | 9131 | if (maybe_existing_index.unwrap()) |existing_index| { |
| 9106 | }, std.hash_map.StringIndexContext{ | | |
| 9107 | .bytes = string_bytes, | | |
| 9108 | }); | | |
| 9109 | if (gop.found_existing) { | | |
| 9110 | string_bytes.shrinkRetainingCapacity(str_index); | 9132 | string_bytes.shrinkRetainingCapacity(str_index); |
| 9111 | return @enumFromInt(gop.key_ptr.*); | 9133 | return @enumFromInt(@intFromEnum(existing_index)); |
| 9112 | } else { | 9134 | } else { |
| 9113 | gop.key_ptr.* = str_index; | | |
| 9114 | string_bytes.appendAssumeCapacity(0); | 9135 | string_bytes.appendAssumeCapacity(0); |
| 9115 | return @enumFromInt(str_index); | 9136 | return @enumFromInt(str_index); |
| 9116 | } | 9137 | } |
| 9117 | } | 9138 | } |
| 9118 | | 9139 | |
| 9119 | pub fn getString(ip: *InternPool, s: []const u8) OptionalNullTerminatedString { | 9140 | fn getOrPutStringValue( |
| 9120 | return if (ip.string_table.getKeyAdapted(s, std.hash_map.StringIndexAdapter{ | 9141 | ip: *InternPool, |
| 9121 | .bytes = &ip.string_bytes, | 9142 | gpa: Allocator, |
| 9122 | })) |index| @enumFromInt(index) else .none; | 9143 | tid: Zcu.PerThread.Id, |
| | 9144 | key: []const u8, |
| | 9145 | value: NullTerminatedString, |
| | 9146 | ) Allocator.Error!OptionalNullTerminatedString { |
| | 9147 | const full_hash = Hash.hash(0, key); |
| | 9148 | const hash: u32 = @truncate(full_hash >> 32); |
| | 9149 | const shard = &ip.shards[@intCast(full_hash & (ip.shards.len - 1))]; |
| | 9150 | var map = shard.shared.string_map.acquire(); |
| | 9151 | const Map = @TypeOf(map); |
| | 9152 | var map_mask = map.header().mask(); |
| | 9153 | var map_index = hash; |
| | 9154 | while (true) : (map_index += 1) { |
| | 9155 | map_index &= map_mask; |
| | 9156 | const entry = &map.entries[map_index]; |
| | 9157 | const index = entry.acquire().unwrap() orelse break; |
| | 9158 | if (entry.hash != hash) continue; |
| | 9159 | if (index.eqlSlice(key, ip)) return index.toOptional(); |
| | 9160 | } |
| | 9161 | shard.mutate.string_map.mutex.lock(); |
| | 9162 | defer shard.mutate.string_map.mutex.unlock(); |
| | 9163 | if (map.entries != shard.shared.string_map.entries) { |
| | 9164 | shard.mutate.string_map.len += 1; |
| | 9165 | map = shard.shared.string_map; |
| | 9166 | map_mask = map.header().mask(); |
| | 9167 | map_index = hash; |
| | 9168 | } |
| | 9169 | while (true) : (map_index += 1) { |
| | 9170 | map_index &= map_mask; |
| | 9171 | const entry = &map.entries[map_index]; |
| | 9172 | const index = entry.acquire().unwrap() orelse break; |
| | 9173 | if (entry.hash != hash) continue; |
| | 9174 | if (index.eqlSlice(key, ip)) return index.toOptional(); |
| | 9175 | } |
| | 9176 | defer shard.mutate.string_map.len += 1; |
| | 9177 | const map_header = map.header().*; |
| | 9178 | if (shard.mutate.string_map.len < map_header.capacity * 3 / 5) { |
| | 9179 | const entry = &map.entries[map_index]; |
| | 9180 | entry.hash = hash; |
| | 9181 | entry.release(value.toOptional()); |
| | 9182 | return .none; |
| | 9183 | } |
| | 9184 | const new_map_capacity = map_header.capacity * 2; |
| | 9185 | const new_map_buf = try gpa.alignedAlloc( |
| | 9186 | u8, |
| | 9187 | Local.garbage_align, |
| | 9188 | @sizeOf(Local.Garbage.Node) + @sizeOf(Map.Header) + |
| | 9189 | new_map_capacity * @sizeOf(Map.Entry), |
| | 9190 | ); |
| | 9191 | const new_node: *Local.Garbage.Node = @ptrCast(new_map_buf.ptr); |
| | 9192 | new_node.* = .{ .data = .{ .buf_len = new_map_buf.len } }; |
| | 9193 | ip.local[@intFromEnum(tid)].garbage.prepend(new_node); |
| | 9194 | const new_map_entries = std.mem.bytesAsSlice( |
| | 9195 | Map.Entry, |
| | 9196 | new_map_buf[@sizeOf(Local.Garbage.Node) + @sizeOf(Map.Header) ..], |
| | 9197 | ); |
| | 9198 | const new_map: Map = .{ .entries = new_map_entries.ptr }; |
| | 9199 | new_map.header().* = .{ .capacity = new_map_capacity }; |
| | 9200 | @memset(new_map_entries, .{ .value = .none, .hash = undefined }); |
| | 9201 | const new_map_mask = new_map.header().mask(); |
| | 9202 | map_index = 0; |
| | 9203 | while (map_index < map_header.capacity) : (map_index += 1) { |
| | 9204 | const entry = &map.entries[map_index]; |
| | 9205 | const index = entry.value.unwrap() orelse continue; |
| | 9206 | const item_hash = entry.hash; |
| | 9207 | var new_map_index = item_hash; |
| | 9208 | while (true) : (new_map_index += 1) { |
| | 9209 | new_map_index &= new_map_mask; |
| | 9210 | const new_entry = &new_map.entries[new_map_index]; |
| | 9211 | if (new_entry.value != .none) continue; |
| | 9212 | new_entry.* = .{ |
| | 9213 | .value = index.toOptional(), |
| | 9214 | .hash = item_hash, |
| | 9215 | }; |
| | 9216 | break; |
| | 9217 | } |
| | 9218 | } |
| | 9219 | map = new_map; |
| | 9220 | map_index = hash; |
| | 9221 | while (true) : (map_index += 1) { |
| | 9222 | map_index &= new_map_mask; |
| | 9223 | if (map.entries[map_index].value == .none) break; |
| | 9224 | } |
| | 9225 | map.entries[map_index] = .{ |
| | 9226 | .value = value.toOptional(), |
| | 9227 | .hash = hash, |
| | 9228 | }; |
| | 9229 | shard.shared.string_map.release(new_map); |
| | 9230 | return .none; |
| | 9231 | } |
| | 9232 | |
| | 9233 | pub fn getString(ip: *InternPool, key: []const u8) OptionalNullTerminatedString { |
| | 9234 | const full_hash = Hash.hash(0, key); |
| | 9235 | const hash: u32 = @truncate(full_hash >> 32); |
| | 9236 | const shard = &ip.shards[@intCast(full_hash & (ip.shards.len - 1))]; |
| | 9237 | const map = shard.shared.string_map.acquire(); |
| | 9238 | const map_mask = map.header().mask(); |
| | 9239 | var map_index = hash; |
| | 9240 | while (true) : (map_index += 1) { |
| | 9241 | map_index &= map_mask; |
| | 9242 | const entry = map.at(map_index); |
| | 9243 | const index = entry.acquire().unwrap() orelse return null; |
| | 9244 | if (entry.hash != hash) continue; |
| | 9245 | if (index.eqlSlice(key, ip)) return index; |
| | 9246 | } |
| 9123 | } | 9247 | } |
| 9124 | | 9248 | |
| 9125 | pub fn typeOf(ip: *const InternPool, index: Index) Index { | 9249 | pub fn typeOf(ip: *const InternPool, index: Index) Index { |
Jacob Young <jacobly@ziglang.org> 2024-06-15 19:58:29-04:00