authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-03-02 19:46:22-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-05-25 18:54:34-07:00
log8aec13b6ab1f0a8586f3e5cf5600d70697ac209d
treea22b8daa2c6eefa3299b0be35d5495b04a640402
parent8331c59ee2ec3e57ffcc4c0ae57792a995afa865

Configuration: serialize remaining Module information

also handle properly Module circular references and introduce a general deduplication mechanism.

4 files changed, 172 insertions(+), 109 deletions(-)

lib/compiler/Maker/ScannedConfig.zig+2
......@@ -87,6 +87,7 @@ fn printValue(sc: *const ScannedConfig, s: *Serializer, comptime Field: type, fi
8787 .union_list => comptime unreachable,
8888 .length_prefixed_list => comptime unreachable,
8989 .flag_union => comptime unreachable,
90 .multi_list => comptime unreachable,
9091 } else if (std.enums.tagName(Field, field_value)) |name| {
9192 try s.ident(name);
9293 } else {
......@@ -111,6 +112,7 @@ fn printValue(sc: *const ScannedConfig, s: *Serializer, comptime Field: type, fi
111112 .extended => @compileError("TODO"),
112113 .union_list => @compileError("TODO"),
113114 .flag_union => try printValue(sc, s, Field.Union, field_value.u),
115 .multi_list => @compileError("TODO"),
114116 },
115117 else => @compileError("not implemented: " ++ @typeName(Field)),
116118 },
lib/compiler/configurer.zig+55-48
......@@ -294,9 +294,8 @@ const Serialize = struct {
294294 }
295295
296296 fn addSystemLib(s: *Serialize, sl: *const std.Build.Module.SystemLib) !Configuration.SystemLib.Index {
297 log.err("TODO deduplicate addSystemLib", .{});
298297 const wc = s.wc;
299 return @enumFromInt(try wc.addExtra(@as(Configuration.SystemLib, .{
298 return @enumFromInt(try wc.addDeduped(@as(Configuration.SystemLib, .{
300299 .flags = .{
301300 .needed = sl.needed,
302301 .weak = sl.weak,
......@@ -362,7 +361,6 @@ const Serialize = struct {
362361
363362 const wc = s.wc;
364363 const arena = s.arena;
365 const gpa = wc.gpa;
366364
367365 const include_dirs = try arena.alloc(Configuration.Module.IncludeDir, m.include_dirs.items.len);
368366 for (include_dirs, m.include_dirs.items) |*dest, src| dest.* = switch (src) {
......@@ -393,66 +391,55 @@ const Serialize = struct {
393391 .win32_resource_file => |wrf| .{ .win32_resource_file = try addRcSourceFile(s, wrf) },
394392 };
395393
394 const frameworks = try arena.alloc(Configuration.Module.Framework, m.frameworks.entries.len);
395 for (frameworks, m.frameworks.keys(), m.frameworks.values()) |*dest, name, options| dest.* = .{
396 .flags = .{
397 .needed = options.needed,
398 .weak = options.weak,
399 },
400 .name = try wc.addString(name),
401 };
402
396403 const lib_paths = try arena.alloc(Configuration.LazyPath, m.lib_paths.items.len);
397404 for (lib_paths, m.lib_paths.items) |*dest, src| dest.* = try addLazyPath(s, src);
398405
399406 const c_macros = try initStringList(s, m.c_macros.items);
400407 const export_symbol_names = try initStringList(s, m.export_symbol_names);
401408
402 const import_table: Configuration.ImportTable = @enumFromInt(wc.extra.items.len);
403 const import_table_extra_len = 1 + 2 * m.import_table.entries.len;
404 try wc.extra.ensureUnusedCapacity(gpa, import_table_extra_len);
405 wc.extra.items.len += import_table_extra_len;
406 wc.extra.appendAssumeCapacity(@intCast(m.import_table.entries.len));
407 wc.extra.items[@intFromEnum(import_table)] = @intCast(m.import_table.entries.len);
408 for (
409 m.import_table.keys(),
410 @intFromEnum(import_table) + 1..,
411 ) |mod_name, extra_index| {
412 wc.extra.items[extra_index] = @intFromEnum(try wc.addString(mod_name));
413 }
414 for (
415 m.import_table.values(),
416 @intFromEnum(import_table) + 1 + m.import_table.entries.len..,
417 ) |dep, extra_index| {
418 log.err("TODO module dependencies can be cyclic", .{});
419 wc.extra.items[extra_index] = @intFromEnum(try addModule(s, dep));
420 }
421
422409 const module_index: Configuration.Module.Index = @enumFromInt(try wc.addExtra(@as(Configuration.Module, .{
423410 .flags = .{
424411 .optimize = .init(m.optimize),
425412 .strip = .init(m.strip),
426413 .unwind_tables = .init(m.unwind_tables),
427414 .dwarf_format = .init(m.dwarf_format),
428 .single_threaded = .init(m.strip),
429 .stack_protector = .init(m.strip),
430 .stack_check = .init(m.strip),
415 .single_threaded = .init(m.single_threaded),
416 .stack_protector = .init(m.stack_protector),
417 .stack_check = .init(m.stack_check),
431418 .sanitize_c = .init(m.sanitize_c),
432 .sanitize_thread = .init(m.strip),
433 .fuzz = .init(m.strip),
419 .sanitize_thread = .init(m.sanitize_thread),
420 .fuzz = .init(m.fuzz),
434421 .code_model = m.code_model,
435422 .c_macros = c_macros.len != 0,
436423 .include_dirs = include_dirs.len != 0,
437424 .lib_paths = lib_paths.len != 0,
438425 .rpaths = rpaths.len != 0,
439 .frameworks = m.frameworks.entries.len != 0,
426 .frameworks = frameworks.len != 0,
440427 .link_objects = link_objects.len != 0,
441428 .export_symbol_names = export_symbol_names.len != 0,
442429 },
443430 .flags2 = .{
444 .valgrind = .init(m.strip),
445 .pic = .init(m.strip),
446 .red_zone = .init(m.strip),
447 .omit_frame_pointer = .init(m.strip),
448 .error_tracing = .init(m.strip),
449 .link_libc = .init(m.strip),
450 .link_libcpp = .init(m.strip),
451 .no_builtin = .init(m.strip),
431 .valgrind = .init(m.valgrind),
432 .pic = .init(m.pic),
433 .red_zone = .init(m.red_zone),
434 .omit_frame_pointer = .init(m.omit_frame_pointer),
435 .error_tracing = .init(m.error_tracing),
436 .link_libc = .init(m.link_libc),
437 .link_libcpp = .init(m.link_libcpp),
438 .no_builtin = .init(m.no_builtin),
452439 },
453440 .owner = try s.builderToPackage(m.owner),
454441 .root_source_file = try s.addOptionalLazyPathEnum(m.root_source_file),
455 .import_table = import_table,
442 .import_table = .invalid,
456443 .resolved_target = try addOptionalResolvedTarget(wc, m.resolved_target),
457444 .c_macros = .{ .slice = c_macros },
458445 .lib_paths = .{ .slice = lib_paths },
......@@ -460,12 +447,33 @@ const Serialize = struct {
460447 .include_dirs = .init(include_dirs),
461448 .rpaths = .init(rpaths),
462449 .link_objects = .init(link_objects),
450 .frameworks = .{ .slice = frameworks },
463451 })));
464452
465 log.err("TODO serialize the trailing Module data", .{});
466
453 // The import table is the only place that modules can form dependency
454 // loops. Therefore, we populate the module indexes only after adding
455 // the module to module_map.
467456 try s.module_map.putNoClobber(arena, m, module_index);
468457
458 var imports = try std.MultiArrayList(Configuration.ImportTable.Import).initCapacity(arena, m.import_table.entries.len);
459 imports.len = m.import_table.entries.len;
460 for (
461 imports.items(.name),
462 imports.items(.module),
463 m.import_table.keys(),
464 m.import_table.values(),
465 ) |*dest_name, *dest_module, src_name, src_module| {
466 dest_name.* = try wc.addString(src_name);
467 dest_module.* = try addModule(s, src_module);
468 }
469
470 comptime assert(std.mem.eql(u8, @typeInfo(Configuration.Module).@"struct".fields[2].name, "import_table"));
471 comptime assert(@typeInfo(Configuration.Module).@"struct".fields[2].type == Configuration.ImportTable.Index);
472 assert(wc.extra.items[@intFromEnum(module_index) + 2] == @intFromEnum(Configuration.ImportTable.Index.invalid));
473 wc.extra.items[@intFromEnum(module_index) + 2] = try wc.addDeduped(@as(Configuration.ImportTable, .{
474 .imports = .{ .mal = imports },
475 }));
476
469477 return module_index;
470478 }
471479
......@@ -502,12 +510,12 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void {
502510 }
503511
504512 // Add and then de-duplicate dependencies.
505 const deps = d: {
506 const deps: Configuration.Deps = @enumFromInt(wc.extra.items.len);
507 for (try wc.reserveLengthPrefixed(step.dependencies.items.len), step.dependencies.items) |*dep, dep_step|
508 dep.* = @intCast(s.step_map.getIndex(dep_step).?);
509 break :d try wc.dedupeDeps(deps);
510 };
513 const dep_steps = try arena.alloc(Configuration.Step.Index, step.dependencies.items.len);
514 for (dep_steps, step.dependencies.items) |*dest, src|
515 dest.* = @enumFromInt(s.step_map.getIndex(src).?);
516 const deps: Configuration.Deps.Index = @enumFromInt(try wc.addDeduped(@as(Configuration.Deps, .{
517 .steps = .{ .slice = dep_steps },
518 })));
511519
512520 try wc.steps.ensureTotalCapacity(gpa, s.step_map.entries.capacity);
513521 wc.steps.appendAssumeCapacity(.{
......@@ -791,8 +799,7 @@ fn addOptionalResolvedTarget(
791799 optional_resolved_target: ?std.Build.ResolvedTarget,
792800) !Configuration.ResolvedTarget.OptionalIndex {
793801 const resolved_target = optional_resolved_target orelse return .none;
794 log.debug("TODO deduplicate resolved targets", .{});
795 return @enumFromInt(try wc.addExtra(@as(Configuration.ResolvedTarget, .{
802 return @enumFromInt(try wc.addDeduped(@as(Configuration.ResolvedTarget, .{
796803 .query = try wc.addTargetQuery(resolved_target.query),
797804 .result = try wc.addTarget(resolved_target.result),
798805 })));
lib/std/Build/Step/Compile.zig+1
......@@ -43,6 +43,7 @@ export_memory: bool = false,
4343/// For WebAssembly targets, this will allow for undefined symbols to
4444/// be imported from the host environment.
4545import_symbols: bool = false,
46/// (WebAssembly) import function table from the host environment
4647import_table: bool = false,
4748export_table: bool = false,
4849initial_memory: ?u64 = null,
lib/std/zig/Configuration.zig+114-61
......@@ -33,9 +33,8 @@ pub const Header = extern struct {
3333pub const Wip = struct {
3434 gpa: Allocator,
3535 string_table: StringTable = .empty,
36 /// De-duplicates an array inside `extra` that has first element length
37 /// followed by length elements.
38 length_prefixed_table: LengthPrefixedTable = .empty,
36 /// De-duplicates an array inside `extra`.
37 dedupe_table: DedupeTable = .empty,
3938 targets_table: TargetsTable = .empty,
4039
4140 string_bytes: std.ArrayList(u8) = .empty,
......@@ -46,25 +45,27 @@ pub const Wip = struct {
4645 path_deps: std.MultiArrayList(Path) = .empty,
4746 extra: std.ArrayList(u32) = .empty,
4847
49 const LengthPrefixedTable = std.HashMapUnmanaged(u32, void, LengthPrefixedContext, std.hash_map.default_max_load_percentage);
48 const DedupeTable = std.HashMapUnmanaged(ExtraSlice, void, ExtraSlice.Context, std.hash_map.default_max_load_percentage);
5049 const TargetsTable = std.HashMapUnmanaged(TargetQuery.Index, void, TargetsTableContext, std.hash_map.default_max_load_percentage);
5150
52 const LengthPrefixedContext = struct {
53 extra: []const u32,
51 const ExtraSlice = struct {
52 index: u32,
53 len: u32,
5454
55 pub fn eql(ctx: @This(), a: u32, b: u32) bool {
56 const len_a = ctx.extra[a];
57 const len_b = ctx.extra[b];
58 const slice_a = ctx.extra[a + 1 ..][0..len_a];
59 const slice_b = ctx.extra[b + 1 ..][0..len_b];
60 return std.mem.eql(u32, slice_a, slice_b);
61 }
55 const Context = struct {
56 extra: []const u32,
6257
63 pub fn hash(ctx: @This(), key: u32) u64 {
64 const len = ctx.extra[key];
65 const slice = ctx.extra[key + 1 ..][0..len];
66 return std.hash_map.hashString(@ptrCast(slice));
67 }
58 pub fn eql(ctx: @This(), a: ExtraSlice, b: ExtraSlice) bool {
59 const slice_a = ctx.extra[a.index..][0..a.len];
60 const slice_b = ctx.extra[b.index..][0..b.len];
61 return std.mem.eql(u32, slice_a, slice_b);
62 }
63
64 pub fn hash(ctx: @This(), key: ExtraSlice) u64 {
65 const slice = ctx.extra[key.index..][0..key.len];
66 return std.hash_map.hashString(@ptrCast(slice));
67 }
68 };
6869 };
6970
7071 const TargetsTableContext = struct {
......@@ -323,34 +324,33 @@ pub const Wip = struct {
323324 }
324325 }
325326
326 pub fn reserveLengthPrefixed(wip: *Wip, n: usize) Allocator.Error![]u32 {
327 const slice = try wip.extra.addManyAsSlice(wip.gpa, n + 1);
328 slice[0] = @intCast(n);
329 return slice[1..];
327 pub fn addExtra(wip: *Wip, extra: anytype) Allocator.Error!u32 {
328 const extra_len = Storage.extraLen(extra);
329 try wip.extra.ensureUnusedCapacity(wip.gpa, extra_len);
330 return addExtraAssumeCapacity(wip, extra);
330331 }
331332
332 pub fn dedupeLengthPrefixed(wip: *Wip, index: u32) Allocator.Error!u32 {
333 assert(wip.extra.items.len == index + wip.extra.items[index] + 1);
333 /// Same as `addExtra` but uses a hash map to possibly return an already
334 /// existing index instead of appending to `extra`.
335 pub fn addDeduped(wip: *Wip, extra: anytype) Allocator.Error!u32 {
334336 const gpa = wip.gpa;
335 const gop = try wip.length_prefixed_table.getOrPutContext(gpa, index, @as(LengthPrefixedContext, .{
336 .extra = wip.extra.items,
337 }));
337 const revert_index = wip.extra.items.len;
338 const extra_len = Storage.extraLen(extra);
339 try wip.extra.ensureUnusedCapacity(gpa, extra_len);
340 const new_index = addExtraAssumeCapacity(wip, extra);
341 const len: u32 = @intCast(wip.extra.items.len - new_index);
342
343 const gop = try wip.dedupe_table.getOrPutContext(gpa, .{
344 .index = new_index,
345 .len = len,
346 }, @as(ExtraSlice.Context, .{ .extra = wip.extra.items }));
347
338348 if (gop.found_existing) {
339 wip.extra.items.len = index;
340 return gop.key_ptr.*;
341 } else {
342 return index;
349 wip.extra.items.len = revert_index;
350 return gop.key_ptr.index;
343351 }
344 }
345352
346 pub fn dedupeDeps(wip: *Wip, deps: Deps) Allocator.Error!Deps {
347 return @enumFromInt(try dedupeLengthPrefixed(wip, @intFromEnum(deps)));
348 }
349
350 pub fn addExtra(wip: *Wip, extra: anytype) Allocator.Error!u32 {
351 const extra_len = Storage.extraLen(extra);
352 try wip.extra.ensureUnusedCapacity(wip.gpa, extra_len);
353 return addExtraAssumeCapacity(wip, extra);
353 return new_index;
354354 }
355355
356356 pub fn addExtraAssumeCapacity(wip: *Wip, extra: anytype) u32 {
......@@ -399,7 +399,7 @@ pub const AvailableOption = extern struct {
399399pub const Step = extern struct {
400400 name: String,
401401 owner: Package.Index,
402 deps: Deps,
402 deps: Deps.Index,
403403 max_rss: MaxRss,
404404 extended: Storage.Extended(Flags, union(Tag) {
405405 check_file: CheckFile,
......@@ -1074,14 +1074,12 @@ pub const Package = struct {
10741074 };
10751075};
10761076
1077/// Trailing:
1078/// * frameworks: FlagsPrefixedList(FrameworkFlags), // if flag is set
10791077pub const Module = struct {
10801078 flags: Flags,
10811079 flags2: Flags2,
1080 import_table: ImportTable.Index,
10821081 owner: Package.Index,
10831082 root_source_file: OptionalLazyPath,
1084 import_table: ImportTable,
10851083 resolved_target: ResolvedTarget.OptionalIndex,
10861084 c_macros: Storage.FlagLengthPrefixedList(.flags, .c_macros, String),
10871085 lib_paths: Storage.FlagLengthPrefixedList(.flags, .lib_paths, LazyPath),
......@@ -1089,6 +1087,7 @@ pub const Module = struct {
10891087 include_dirs: Storage.UnionList(.flags, .include_dirs, IncludeDir),
10901088 rpaths: Storage.UnionList(.flags, .rpaths, RPath),
10911089 link_objects: Storage.UnionList(.flags, .link_objects, LinkObject),
1090 frameworks: Storage.FlagLengthPrefixedList(.flags, .frameworks, Framework),
10921091
10931092 pub const Optimize = enum(u3) {
10941093 debug,
......@@ -1220,28 +1219,47 @@ pub const Module = struct {
12201219 win32_resource_file: RcSourceFile.Index,
12211220 };
12221221
1223 pub const FrameworkFlags = packed struct(u2) {
1224 needed: bool,
1225 weak: bool,
1222 pub const Framework = struct {
1223 flags: @This().Flags,
1224 name: String,
1225
1226 pub const Flags = packed struct(u32) {
1227 needed: bool,
1228 weak: bool,
1229 _: u30 = 0,
1230 };
12261231 };
12271232};
12281233
1229/// Points into `extra`, first element is len, then:
1230/// * import_name: String, // for each len
1231/// * Module.Index, // for each len
1232pub const ImportTable = enum(u32) {
1233 _,
1234pub const ImportTable = struct {
1235 imports: Storage.MultiList(Import),
1236
1237 pub const Import = struct {
1238 name: String,
1239 module: Module.Index,
1240 };
1241
1242 /// Points into `extra`.
1243 pub const Index = enum(u32) {
1244 invalid = maxInt(u32),
1245 _,
1246 };
12341247};
12351248
1236/// Points into `extra`, where the first element is count of deps, following
1237/// elements is `Step.Index` per count.
1238pub const Deps = enum(u32) {
1239 _,
1249pub const Deps = struct {
1250 steps: Storage.LengthPrefixedList(Step.Index),
12401251
1241 pub fn slice(deps: Deps, c: *const Configuration) []Step.Index {
1242 const len = c.extra[@intFromEnum(deps)];
1243 return @ptrCast(c.extra[@intFromEnum(deps) + 1 ..][0..len]);
1244 }
1252 pub const Index = enum(u32) {
1253 _,
1254
1255 pub fn get(this: @This(), c: *const Configuration) Deps {
1256 return extraData(c, Deps, @intFromEnum(this));
1257 }
1258
1259 pub fn slice(this: @This(), c: *const Configuration) []const Step.Index {
1260 return get(this, c).steps.slice;
1261 }
1262 };
12451263};
12461264
12471265/// Points into `extra`, where the first element is count of strings, following
......@@ -1760,6 +1778,7 @@ pub const Storage = enum {
17601778 flag_length_prefixed_list,
17611779 union_list,
17621780 flag_union,
1781 multi_list,
17631782
17641783 /// The presence of the field is determined by a boolean within a packed
17651784 /// struct.
......@@ -1853,7 +1872,8 @@ pub const Storage = enum {
18531872 };
18541873 }
18551874
1856 /// The field contains a u32 length followed by that many items.
1875 /// The field contains a u32 length followed by that many items, each
1876 /// element bitcastable to u32.
18571877 pub fn LengthPrefixedList(comptime ElemArg: type) type {
18581878 return struct {
18591879 slice: []const Elem,
......@@ -1867,6 +1887,17 @@ pub const Storage = enum {
18671887 };
18681888 }
18691889
1890 /// The field contains a u32 length followed by that many items for the
1891 /// first field, that many items for the second field, etc.
1892 pub fn MultiList(comptime ElemArg: type) type {
1893 return struct {
1894 mal: std.MultiArrayList(Elem),
1895
1896 pub const storage: Storage = .multi_list;
1897 pub const Elem = ElemArg;
1898 };
1899 }
1900
18701901 /// `UnionArg` is a tagged union with a small integer for the enum tag.
18711902 ///
18721903 /// A field in flags determines whether the metadata is present.
......@@ -2028,6 +2059,16 @@ pub const Storage = enum {
20282059 defer i.* = data_start + len;
20292060 return .{ .slice = @ptrCast(buffer[data_start..][0..len]) };
20302061 },
2062 .multi_list => {
2063 const data_start = i.* + 1;
2064 const len = buffer[data_start - 1];
2065 defer i.* = data_start + len * @typeInfo(Field.Elem).@"struct".fields.len;
2066 return .{ .mal = .{
2067 .bytes = @ptrCast(buffer[data_start..][0..len]),
2068 .len = len,
2069 .capacity = len,
2070 } };
2071 },
20312072 .union_list => {
20322073 const flags = @field(container, @tagName(Field.flags));
20332074 const flag = @field(flags, @tagName(Field.flag));
......@@ -2082,6 +2123,7 @@ pub const Storage = enum {
20822123 .auto => switch (Field.storage) {
20832124 .flag_optional, .enum_optional, .extended => 1,
20842125 .length_prefixed_list, .flag_length_prefixed_list => field.slice.len + 1,
2126 .multi_list => 1 + field.mal.len * @typeInfo(Field.Elem).@"struct".fields.len,
20852127 .union_list => Field.extraLen(field.len),
20862128 .flag_union => switch (field.u) {
20872129 inline else => |v| extraFieldLen(v),
......@@ -2153,6 +2195,17 @@ pub const Storage = enum {
21532195 @memcpy(buffer[i + 1 ..][0..len], @as([]const u32, @ptrCast(value.slice)));
21542196 return len + 1;
21552197 },
2198 .multi_list => {
2199 const len: u32 = @intCast(value.mal.len);
2200 if (len == 0) return 0;
2201 buffer[i] = len;
2202 const fields = @typeInfo(Field.Elem).@"struct".fields;
2203 inline for (0..fields.len) |field_i| @memcpy(
2204 buffer[i + 1 + field_i * len ..][0..len],
2205 @as([]const u32, @ptrCast(value.mal.items(@enumFromInt(field_i)))),
2206 );
2207 return 1 + fields.len * len;
2208 },
21562209 .union_list => {
21572210 if (value.len == 0) return 0;
21582211 const Tag = @typeInfo(Field.Union).@"union".tag_type.?;