authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-07-16 14:49:49-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2024-07-16 14:49:49-04:00
log88bb0fd288acb6a20abed57cdf459cc4fc788b89
tree4a488eb0c9ef76f63d13d12de1649ddab9b8caac
parentd8f81372f148ad2ee5aab12cc7ea55562764b3e7
parent00fdbf05f39931d1f6c5808e8da4afca85357214
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #20632 from jacobly0/codegen-thread

InternPool: enable separate codegen/linking thread

6 files changed, 114 insertions(+), 63 deletions(-)

lib/std/Progress.zig+1-7
......@@ -669,14 +669,8 @@ fn appendTreeSymbol(symbol: TreeSymbol, buf: []u8, start_i: usize) usize {
669669fn clearWrittenWithEscapeCodes() anyerror!void {
670670 if (!global_progress.need_clear) return;
671671
672 var i: usize = 0;
673 const buf = global_progress.draw_buffer;
674
675 buf[i..][0..clear.len].* = clear.*;
676 i += clear.len;
677
678672 global_progress.need_clear = false;
679 try write(buf[0..i]);
673 try write(clear);
680674}
681675
682676/// U+25BA or ►
lib/std/mem.zig+6-5
......@@ -1050,15 +1050,16 @@ pub fn indexOfSentinel(comptime T: type, comptime sentinel: T, p: [*:sentinel]co
10501050 // as we don't read into a new page. This should be the case for most architectures
10511051 // which use paged memory, however should be confirmed before adding a new arch below.
10521052 .aarch64, .x86, .x86_64 => if (std.simd.suggestVectorLength(T)) |block_len| {
1053 const block_size = @sizeOf(T) * block_len;
10531054 const Block = @Vector(block_len, T);
10541055 const mask: Block = @splat(sentinel);
10551056
1056 comptime std.debug.assert(std.mem.page_size % @sizeOf(Block) == 0);
1057 comptime std.debug.assert(std.mem.page_size % block_size == 0);
10571058
10581059 // First block may be unaligned
10591060 const start_addr = @intFromPtr(&p[i]);
10601061 const offset_in_page = start_addr & (std.mem.page_size - 1);
1061 if (offset_in_page <= std.mem.page_size - @sizeOf(Block)) {
1062 if (offset_in_page <= std.mem.page_size - block_size) {
10621063 // Will not read past the end of a page, full block.
10631064 const block: Block = p[i..][0..block_len].*;
10641065 const matches = block == mask;
......@@ -1066,19 +1067,19 @@ pub fn indexOfSentinel(comptime T: type, comptime sentinel: T, p: [*:sentinel]co
10661067 return i + std.simd.firstTrue(matches).?;
10671068 }
10681069
1069 i += (std.mem.alignForward(usize, start_addr, @alignOf(Block)) - start_addr) / @sizeOf(T);
1070 i += @divExact(std.mem.alignForward(usize, start_addr, block_size) - start_addr, @sizeOf(T));
10701071 } else {
10711072 // Would read over a page boundary. Per-byte at a time until aligned or found.
10721073 // 0.39% chance this branch is taken for 4K pages at 16b block length.
10731074 //
10741075 // An alternate strategy is to do read a full block (the last in the page) and
10751076 // mask the entries before the pointer.
1076 while ((@intFromPtr(&p[i]) & (@alignOf(Block) - 1)) != 0) : (i += 1) {
1077 while ((@intFromPtr(&p[i]) & (block_size - 1)) != 0) : (i += 1) {
10771078 if (p[i] == sentinel) return i;
10781079 }
10791080 }
10801081
1081 std.debug.assert(std.mem.isAligned(@intFromPtr(&p[i]), @alignOf(Block)));
1082 std.debug.assert(std.mem.isAligned(@intFromPtr(&p[i]), block_size));
10821083 while (true) {
10831084 const block: *const Block = @ptrCast(@alignCast(p[i..][0..block_len]));
10841085 const matches = block.* == mask;
src/InternPool.zig+100-46
......@@ -10,6 +10,8 @@ shards: []Shard = &.{},
1010global_error_set: GlobalErrorSet = GlobalErrorSet.empty,
1111/// Cached number of active bits in a `tid`.
1212tid_width: if (single_threaded) u0 else std.math.Log2Int(u32) = 0,
13/// Cached shift amount to put a `tid` in the top bits of a 30-bit value.
14tid_shift_30: if (single_threaded) u0 else std.math.Log2Int(u32) = if (single_threaded) 0 else 31,
1315/// Cached shift amount to put a `tid` in the top bits of a 31-bit value.
1416tid_shift_31: if (single_threaded) u0 else std.math.Log2Int(u32) = if (single_threaded) 0 else 31,
1517/// Cached shift amount to put a `tid` in the top bits of a 32-bit value.
......@@ -53,7 +55,7 @@ free_dep_entries: std.ArrayListUnmanaged(DepEntry.Index) = .{},
5355/// Whether a multi-threaded intern pool is useful.
5456/// Currently `false` until the intern pool is actually accessed
5557/// from multiple threads to reduce the cost of this data structure.
56const want_multi_threaded = false;
58const want_multi_threaded = true;
5759
5860/// Whether a single-threaded intern pool impl is in use.
5961pub const single_threaded = builtin.single_threaded or !want_multi_threaded;
......@@ -941,8 +943,12 @@ const Shard = struct {
941943 return @atomicLoad(Value, &entry.value, .acquire);
942944 }
943945 fn release(entry: *Entry, value: Value) void {
946 assert(value != .none);
944947 @atomicStore(Value, &entry.value, value, .release);
945948 }
949 fn resetUnordered(entry: *Entry) void {
950 @atomicStore(Value, &entry.value, .none, .unordered);
951 }
946952 };
947953 };
948954 }
......@@ -4089,8 +4095,8 @@ pub const Index = enum(u32) {
40894095
40904096 fn wrap(unwrapped: Unwrapped, ip: *const InternPool) Index {
40914097 assert(@intFromEnum(unwrapped.tid) <= ip.getTidMask());
4092 assert(unwrapped.index <= ip.getIndexMask(u31));
4093 return @enumFromInt(@as(u32, @intFromEnum(unwrapped.tid)) << ip.tid_shift_31 | unwrapped.index);
4098 assert(unwrapped.index <= ip.getIndexMask(u30));
4099 return @enumFromInt(@as(u32, @intFromEnum(unwrapped.tid)) << ip.tid_shift_30 | unwrapped.index);
40944100 }
40954101
40964102 pub fn getExtra(unwrapped: Unwrapped, ip: *const InternPool) Local.Extra {
......@@ -4129,8 +4135,8 @@ pub const Index = enum(u32) {
41294135 .tid = .main,
41304136 .index = @intFromEnum(index),
41314137 } else .{
4132 .tid = @enumFromInt(@intFromEnum(index) >> ip.tid_shift_31 & ip.getTidMask()),
4133 .index = @intFromEnum(index) & ip.getIndexMask(u31),
4138 .tid = @enumFromInt(@intFromEnum(index) >> ip.tid_shift_30 & ip.getTidMask()),
4139 .index = @intFromEnum(index) & ip.getIndexMask(u30),
41344140 };
41354141 }
41364142
......@@ -5820,6 +5826,7 @@ pub fn init(ip: *InternPool, gpa: Allocator, available_threads: usize) !void {
58205826 });
58215827
58225828 ip.tid_width = @intCast(std.math.log2_int_ceil(usize, used_threads));
5829 ip.tid_shift_30 = if (single_threaded) 0 else 30 - ip.tid_width;
58235830 ip.tid_shift_31 = if (single_threaded) 0 else 31 - ip.tid_width;
58245831 ip.tid_shift_32 = if (single_threaded) 0 else ip.tid_shift_31 +| 1;
58255832 ip.shards = try gpa.alloc(Shard, @as(usize, 1) << ip.tid_width);
......@@ -6585,35 +6592,55 @@ const GetOrPutKey = union(enum) {
65856592 },
65866593
65876594 fn put(gop: *GetOrPutKey) Index {
6588 return gop.putAt(0);
6589 }
6590 fn putAt(gop: *GetOrPutKey, offset: u32) Index {
65916595 switch (gop.*) {
65926596 .existing => unreachable,
6593 .new => |info| {
6597 .new => |*info| {
65946598 const index = Index.Unwrapped.wrap(.{
65956599 .tid = info.tid,
6596 .index = info.ip.getLocal(info.tid).mutate.items.len - 1 - offset,
6600 .index = info.ip.getLocal(info.tid).mutate.items.len - 1,
65976601 }, info.ip);
6598 info.shard.shared.map.entries[info.map_index].release(index);
6602 gop.putTentative(index);
6603 gop.putFinal(index);
6604 return index;
6605 },
6606 }
6607 }
6608
6609 fn putTentative(gop: *GetOrPutKey, index: Index) void {
6610 assert(index != .none);
6611 switch (gop.*) {
6612 .existing => unreachable,
6613 .new => |*info| gop.new.shard.shared.map.entries[info.map_index].release(index),
6614 }
6615 }
6616
6617 fn putFinal(gop: *GetOrPutKey, index: Index) void {
6618 assert(index != .none);
6619 switch (gop.*) {
6620 .existing => unreachable,
6621 .new => |info| {
6622 assert(info.shard.shared.map.entries[info.map_index].value == index);
65996623 info.shard.mutate.map.len += 1;
66006624 info.shard.mutate.map.mutex.unlock();
66016625 gop.* = .{ .existing = index };
6602 return index;
66036626 },
66046627 }
66056628 }
66066629
6607 fn assign(gop: *GetOrPutKey, new_gop: GetOrPutKey) void {
6608 gop.deinit();
6609 gop.* = new_gop;
6630 fn cancel(gop: *GetOrPutKey) void {
6631 switch (gop.*) {
6632 .existing => {},
6633 .new => |info| info.shard.mutate.map.mutex.unlock(),
6634 }
6635 gop.* = .{ .existing = undefined };
66106636 }
66116637
66126638 fn deinit(gop: *GetOrPutKey) void {
66136639 switch (gop.*) {
66146640 .existing => {},
6615 .new => |info| info.shard.mutate.map.mutex.unlock(),
6641 .new => |info| info.shard.shared.map.entries[info.map_index].resetUnordered(),
66166642 }
6643 gop.cancel();
66176644 gop.* = undefined;
66186645 }
66196646};
......@@ -6622,6 +6649,15 @@ fn getOrPutKey(
66226649 gpa: Allocator,
66236650 tid: Zcu.PerThread.Id,
66246651 key: Key,
6652) Allocator.Error!GetOrPutKey {
6653 return ip.getOrPutKeyEnsuringAdditionalCapacity(gpa, tid, key, 0);
6654}
6655fn getOrPutKeyEnsuringAdditionalCapacity(
6656 ip: *InternPool,
6657 gpa: Allocator,
6658 tid: Zcu.PerThread.Id,
6659 key: Key,
6660 additional_capacity: u32,
66256661) Allocator.Error!GetOrPutKey {
66266662 const full_hash = key.hash64(ip);
66276663 const hash: u32 = @truncate(full_hash >> 32);
......@@ -6657,11 +6693,16 @@ fn getOrPutKey(
66576693 }
66586694 }
66596695 const map_header = map.header().*;
6660 if (shard.mutate.map.len >= map_header.capacity * 3 / 5) {
6696 const required = shard.mutate.map.len + additional_capacity;
6697 if (required >= map_header.capacity * 3 / 5) {
66616698 const arena_state = &ip.getLocal(tid).mutate.arena;
66626699 var arena = arena_state.promote(gpa);
66636700 defer arena_state.* = arena.state;
6664 const new_map_capacity = map_header.capacity * 2;
6701 var new_map_capacity = map_header.capacity;
6702 while (true) {
6703 new_map_capacity *= 2;
6704 if (required < new_map_capacity * 3 / 5) break;
6705 }
66656706 const new_map_buf = try arena.allocator().alignedAlloc(
66666707 u8,
66676708 Map.alignment,
......@@ -6730,10 +6771,11 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All
67306771 assert(ptr_type.sentinel == .none or ip.typeOf(ptr_type.sentinel) == ptr_type.child);
67316772
67326773 if (ptr_type.flags.size == .Slice) {
6774 gop.cancel();
67336775 var new_key = key;
67346776 new_key.ptr_type.flags.size = .Many;
67356777 const ptr_type_index = try ip.get(gpa, tid, new_key);
6736 gop.assign(try ip.getOrPutKey(gpa, tid, key));
6778 gop = try ip.getOrPutKey(gpa, tid, key);
67376779
67386780 try items.ensureUnusedCapacity(1);
67396781 items.appendAssumeCapacity(.{
......@@ -6913,9 +6955,10 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All
69136955 },
69146956 .anon_decl => |anon_decl| if (ptrsHaveSameAlignment(ip, ptr.ty, ptr_type, anon_decl.orig_ty)) item: {
69156957 if (ptr.ty != anon_decl.orig_ty) {
6958 gop.cancel();
69166959 var new_key = key;
69176960 new_key.ptr.base_addr.anon_decl.orig_ty = ptr.ty;
6918 gop.assign(try ip.getOrPutKey(gpa, tid, new_key));
6961 gop = try ip.getOrPutKey(gpa, tid, new_key);
69196962 if (gop == .existing) return gop.existing;
69206963 }
69216964 break :item .{
......@@ -6986,11 +7029,12 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All
69867029 },
69877030 else => unreachable,
69887031 }
7032 gop.cancel();
69897033 const index_index = try ip.get(gpa, tid, .{ .int = .{
69907034 .ty = .usize_type,
69917035 .storage = .{ .u64 = base_index.index },
69927036 } });
6993 gop.assign(try ip.getOrPutKey(gpa, tid, key));
7037 gop = try ip.getOrPutKey(gpa, tid, key);
69947038 try items.ensureUnusedCapacity(1);
69957039 items.appendAssumeCapacity(.{
69967040 .tag = switch (ptr.base_addr) {
......@@ -7399,11 +7443,12 @@ pub fn get(ip: *InternPool, gpa: Allocator, tid: Zcu.PerThread.Id, key: Key) All
73997443 }
74007444 const elem = switch (aggregate.storage) {
74017445 .bytes => |bytes| elem: {
7446 gop.cancel();
74027447 const elem = try ip.get(gpa, tid, .{ .int = .{
74037448 .ty = .u8_type,
74047449 .storage = .{ .u64 = bytes.at(0, ip) },
74057450 } });
7406 gop.assign(try ip.getOrPutKey(gpa, tid, key));
7451 gop = try ip.getOrPutKey(gpa, tid, key);
74077452 try items.ensureUnusedCapacity(1);
74087453 break :elem elem;
74097454 },
......@@ -8221,9 +8266,9 @@ pub fn getFuncDeclIes(
82218266 extra.mutate.len = prev_extra_len;
82228267 }
82238268
8224 var func_gop = try ip.getOrPutKey(gpa, tid, .{
8269 var func_gop = try ip.getOrPutKeyEnsuringAdditionalCapacity(gpa, tid, .{
82258270 .func = extraFuncDecl(tid, extra.list.*, func_decl_extra_index),
8226 });
8271 }, 3);
82278272 defer func_gop.deinit();
82288273 if (func_gop == .existing) {
82298274 // An existing function type was found; undo the additions to our two arrays.
......@@ -8231,23 +8276,28 @@ pub fn getFuncDeclIes(
82318276 extra.mutate.len = prev_extra_len;
82328277 return func_gop.existing;
82338278 }
8234 var error_union_type_gop = try ip.getOrPutKey(gpa, tid, .{ .error_union_type = .{
8279 func_gop.putTentative(func_index);
8280 var error_union_type_gop = try ip.getOrPutKeyEnsuringAdditionalCapacity(gpa, tid, .{ .error_union_type = .{
82358281 .error_set_type = error_set_type,
82368282 .payload_type = key.bare_return_type,
8237 } });
8283 } }, 2);
82388284 defer error_union_type_gop.deinit();
8239 var error_set_type_gop = try ip.getOrPutKey(gpa, tid, .{
8285 error_union_type_gop.putTentative(error_union_type);
8286 var error_set_type_gop = try ip.getOrPutKeyEnsuringAdditionalCapacity(gpa, tid, .{
82408287 .inferred_error_set_type = func_index,
8241 });
8288 }, 1);
82428289 defer error_set_type_gop.deinit();
8290 error_set_type_gop.putTentative(error_set_type);
82438291 var func_ty_gop = try ip.getOrPutKey(gpa, tid, .{
82448292 .func_type = extraFuncType(tid, extra.list.*, func_type_extra_index),
82458293 });
82468294 defer func_ty_gop.deinit();
8247 assert(func_gop.putAt(3) == func_index);
8248 assert(error_union_type_gop.putAt(2) == error_union_type);
8249 assert(error_set_type_gop.putAt(1) == error_set_type);
8250 assert(func_ty_gop.putAt(0) == func_ty);
8295 func_ty_gop.putTentative(func_ty);
8296
8297 func_gop.putFinal(func_index);
8298 error_union_type_gop.putFinal(error_union_type);
8299 error_set_type_gop.putFinal(error_set_type);
8300 func_ty_gop.putFinal(func_ty);
82518301 return func_index;
82528302}
82538303
......@@ -8506,9 +8556,9 @@ pub fn getFuncInstanceIes(
85068556 extra.mutate.len = prev_extra_len;
85078557 }
85088558
8509 var func_gop = try ip.getOrPutKey(gpa, tid, .{
8559 var func_gop = try ip.getOrPutKeyEnsuringAdditionalCapacity(gpa, tid, .{
85108560 .func = ip.extraFuncInstance(tid, extra.list.*, func_extra_index),
8511 });
8561 }, 3);
85128562 defer func_gop.deinit();
85138563 if (func_gop == .existing) {
85148564 // Hot path: undo the additions to our two arrays.
......@@ -8516,19 +8566,23 @@ pub fn getFuncInstanceIes(
85168566 extra.mutate.len = prev_extra_len;
85178567 return func_gop.existing;
85188568 }
8519 var error_union_type_gop = try ip.getOrPutKey(gpa, tid, .{ .error_union_type = .{
8569 func_gop.putTentative(func_index);
8570 var error_union_type_gop = try ip.getOrPutKeyEnsuringAdditionalCapacity(gpa, tid, .{ .error_union_type = .{
85208571 .error_set_type = error_set_type,
85218572 .payload_type = arg.bare_return_type,
8522 } });
8573 } }, 2);
85238574 defer error_union_type_gop.deinit();
8524 var error_set_type_gop = try ip.getOrPutKey(gpa, tid, .{
8575 error_union_type_gop.putTentative(error_union_type);
8576 var error_set_type_gop = try ip.getOrPutKeyEnsuringAdditionalCapacity(gpa, tid, .{
85258577 .inferred_error_set_type = func_index,
8526 });
8578 }, 1);
85278579 defer error_set_type_gop.deinit();
8580 error_set_type_gop.putTentative(error_set_type);
85288581 var func_ty_gop = try ip.getOrPutKey(gpa, tid, .{
85298582 .func_type = extraFuncType(tid, extra.list.*, func_type_extra_index),
85308583 });
85318584 defer func_ty_gop.deinit();
8585 func_ty_gop.putTentative(func_ty);
85328586 try finishFuncInstance(
85338587 ip,
85348588 gpa,
......@@ -8540,10 +8594,11 @@ pub fn getFuncInstanceIes(
85408594 arg.alignment,
85418595 arg.section,
85428596 );
8543 assert(func_gop.putAt(3) == func_index);
8544 assert(error_union_type_gop.putAt(2) == error_union_type);
8545 assert(error_set_type_gop.putAt(1) == error_set_type);
8546 assert(func_ty_gop.putAt(0) == func_ty);
8597
8598 func_gop.putFinal(func_index);
8599 error_union_type_gop.putFinal(error_union_type);
8600 error_set_type_gop.putFinal(error_set_type);
8601 func_ty_gop.putFinal(func_ty);
85478602 return func_index;
85488603}
85498604
......@@ -10839,19 +10894,18 @@ pub fn getBackingDecl(ip: *const InternPool, val: Index) OptionalDeclIndex {
1083910894 while (true) {
1084010895 const unwrapped_base = base.unwrap(ip);
1084110896 const base_item = unwrapped_base.getItem(ip);
10842 const base_extra_items = unwrapped_base.getExtra(ip).view().items(.@"0");
1084310897 switch (base_item.tag) {
10844 .ptr_decl => return @enumFromInt(base_extra_items[
10898 .ptr_decl => return @enumFromInt(unwrapped_base.getExtra(ip).view().items(.@"0")[
1084510899 base_item.data + std.meta.fieldIndex(PtrDecl, "decl").?
1084610900 ]),
1084710901 inline .ptr_eu_payload,
1084810902 .ptr_opt_payload,
1084910903 .ptr_elem,
1085010904 .ptr_field,
10851 => |tag| base = @enumFromInt(base_extra_items[
10905 => |tag| base = @enumFromInt(unwrapped_base.getExtra(ip).view().items(.@"0")[
1085210906 base_item.data + std.meta.fieldIndex(tag.Payload(), "base").?
1085310907 ]),
10854 .ptr_slice => base = @enumFromInt(base_extra_items[
10908 .ptr_slice => base = @enumFromInt(unwrapped_base.getExtra(ip).view().items(.@"0")[
1085510909 base_item.data + std.meta.fieldIndex(PtrSlice, "ptr").?
1085610910 ]),
1085710911 else => return .none,
src/Zcu/PerThread.zig+2-1
......@@ -3,7 +3,8 @@ zcu: *Zcu,
33/// Dense, per-thread unique index.
44tid: Id,
55
6pub const Id = if (InternPool.single_threaded) enum { main } else enum(u8) { main, _ };
6pub const IdBacking = u7;
7pub const Id = if (InternPool.single_threaded) enum { main } else enum(IdBacking) { main, _ };
78
89pub fn destroyDecl(pt: Zcu.PerThread, decl_index: Zcu.Decl.Index) void {
910 const zcu = pt.zcu;
src/main.zig+3-3
......@@ -3110,7 +3110,7 @@ fn buildOutputType(
31103110 var thread_pool: ThreadPool = undefined;
31113111 try thread_pool.init(.{
31123112 .allocator = gpa,
3113 .n_jobs = @min(@max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1), std.math.maxInt(u8)),
3113 .n_jobs = @min(@max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1), std.math.maxInt(Zcu.PerThread.IdBacking)),
31143114 .track_ids = true,
31153115 });
31163116 defer thread_pool.deinit();
......@@ -4964,7 +4964,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
49644964 var thread_pool: ThreadPool = undefined;
49654965 try thread_pool.init(.{
49664966 .allocator = gpa,
4967 .n_jobs = @min(@max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1), std.math.maxInt(u8)),
4967 .n_jobs = @min(@max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1), std.math.maxInt(Zcu.PerThread.IdBacking)),
49684968 .track_ids = true,
49694969 });
49704970 defer thread_pool.deinit();
......@@ -5402,7 +5402,7 @@ fn jitCmd(
54025402 var thread_pool: ThreadPool = undefined;
54035403 try thread_pool.init(.{
54045404 .allocator = gpa,
5405 .n_jobs = @min(@max(std.Thread.getCpuCount() catch 1, 1), std.math.maxInt(u8)),
5405 .n_jobs = @min(@max(std.Thread.getCpuCount() catch 1, 1), std.math.maxInt(Zcu.PerThread.IdBacking)),
54065406 .track_ids = true,
54075407 });
54085408 defer thread_pool.deinit();
src/target.zig+2-1
......@@ -572,7 +572,8 @@ pub inline fn backendSupportsFeature(backend: std.builtin.CompilerBackend, compt
572572 else => false,
573573 },
574574 .separate_thread => switch (backend) {
575 else => false,
575 .stage2_llvm => false,
576 else => true,
576577 },
577578 };
578579}