authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-03-19 02:24:33-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-05-25 18:54:34-07:00
log81ee4ab32c8617af3ce9690d562131a6a0924f1c
treebaaac2ab23998b5a8244fc6555f0e47e8f5b1f01
parent8bc09132121d751760b332a30a82c3dfa343a48a

configurer: serialize all data from run steps

3 files changed, 355 insertions(+), 54 deletions(-)

BRANCH_TODO+2
......@@ -1,3 +1,5 @@
1* make more stuff use IndexType
2* make addExtra return Index using reflection
13* remove Cache from configurer
24* implement the build options
35* don't forget to add -listen arg back
lib/compiler/configurer.zig+226-18
......@@ -351,6 +351,169 @@ const Serialize = struct {
351351 })));
352352 }
353353
354 fn addEnvironMap(s: *Serialize, opt_map: ?*std.process.Environ.Map) !?Configuration.EnvironMap.Index {
355 const wc = s.wc;
356 const map = opt_map orelse return null;
357 return @enumFromInt(try wc.addDeduped(@as(Configuration.EnvironMap, .{
358 .keys = try wc.addStringList(map.array_hash_map.keys()),
359 .values = try wc.addStringList(map.array_hash_map.values()),
360 })));
361 }
362
363 fn initArgsList(s: *Serialize, args: []const Step.Run.Arg) ![]const Configuration.Step.Run.Arg.Index {
364 const wc = s.wc;
365 const result = try s.arena.alloc(Configuration.Step.Run.Arg.Index, args.len);
366 for (result, args) |*dest, src| {
367 dest.* = @enumFromInt(try wc.addExtra(@as(Configuration.Step.Run.Arg, switch (src) {
368 .artifact => |a| .{
369 .flags = .{
370 .tag = .artifact,
371 .prefix = a.prefix.len != 0,
372 .suffix = false,
373 .basename = false,
374 .path = false,
375 .producer = true,
376 .generated = false,
377 .dep_file = false,
378 },
379 .prefix = .{ .value = try s.addOptionalString(a.prefix) },
380 .suffix = .{ .value = null },
381 .basename = .{ .value = null },
382 .path = .{ .value = null },
383 .producer = .{ .value = stepIndex(s, &a.artifact.step) },
384 .generated = .{ .value = null },
385 },
386 .lazy_path => |a| .{
387 .flags = .{
388 .tag = .path_file,
389 .prefix = a.prefix.len != 0,
390 .suffix = false,
391 .basename = false,
392 .path = true,
393 .producer = false,
394 .generated = false,
395 .dep_file = false,
396 },
397 .prefix = .{ .value = try s.addOptionalString(a.prefix) },
398 .suffix = .{ .value = null },
399 .basename = .{ .value = null },
400 .path = .{ .value = try addLazyPath(s, a.lazy_path) },
401 .producer = .{ .value = null },
402 .generated = .{ .value = null },
403 },
404 .decorated_directory => |a| .{
405 .flags = .{
406 .tag = .path_directory,
407 .prefix = a.prefix.len != 0,
408 .suffix = a.suffix.len != 0,
409 .basename = false,
410 .path = true,
411 .producer = false,
412 .generated = false,
413 .dep_file = false,
414 },
415 .prefix = .{ .value = try addOptionalString(s, a.prefix) },
416 .suffix = .{ .value = try addOptionalString(s, a.suffix) },
417 .basename = .{ .value = null },
418 .path = .{ .value = try addLazyPath(s, a.lazy_path) },
419 .producer = .{ .value = null },
420 .generated = .{ .value = null },
421 },
422 .file_content => |a| .{
423 .flags = .{
424 .tag = .file_content,
425 .prefix = a.prefix.len != 0,
426 .suffix = false,
427 .basename = false,
428 .path = true,
429 .producer = false,
430 .generated = false,
431 .dep_file = false,
432 },
433 .prefix = .{ .value = try addOptionalString(s, a.prefix) },
434 .suffix = .{ .value = null },
435 .basename = .{ .value = null },
436 .path = .{ .value = try addLazyPath(s, a.lazy_path) },
437 .producer = .{ .value = null },
438 .generated = .{ .value = null },
439 },
440 .bytes => |a| .{
441 .flags = .{
442 .tag = .string,
443 .prefix = true,
444 .suffix = false,
445 .basename = false,
446 .path = false,
447 .producer = false,
448 .generated = false,
449 .dep_file = false,
450 },
451 .prefix = .{ .value = try addOptionalString(s, a) },
452 .suffix = .{ .value = null },
453 .basename = .{ .value = null },
454 .path = .{ .value = null },
455 .producer = .{ .value = null },
456 .generated = .{ .value = null },
457 },
458 .output_file => |a| .{
459 .flags = .{
460 .tag = .output_file,
461 .prefix = a.prefix.len != 0,
462 .suffix = false,
463 .basename = a.basename.len != 0,
464 .path = false,
465 .producer = false,
466 .generated = true,
467 .dep_file = false,
468 },
469 .prefix = .{ .value = try addOptionalString(s, a.prefix) },
470 .suffix = .{ .value = null },
471 .basename = .{ .value = try addOptionalString(s, a.basename) },
472 .path = .{ .value = null },
473 .producer = .{ .value = null },
474 .generated = .{ .value = a.generated_file },
475 },
476 .output_directory => |a| .{
477 .flags = .{
478 .tag = .output_directory,
479 .prefix = a.prefix.len != 0,
480 .suffix = false,
481 .basename = a.basename.len != 0,
482 .path = false,
483 .producer = false,
484 .generated = true,
485 .dep_file = false,
486 },
487 .prefix = .{ .value = try addOptionalString(s, a.prefix) },
488 .suffix = .{ .value = null },
489 .basename = .{ .value = try addOptionalString(s, a.basename) },
490 .path = .{ .value = null },
491 .producer = .{ .value = null },
492 .generated = .{ .value = a.generated_file },
493 },
494 .cli_rest_positionals => .{
495 .flags = .{
496 .tag = .cli_rest_positionals,
497 .prefix = false,
498 .suffix = false,
499 .basename = false,
500 .path = false,
501 .producer = false,
502 .generated = false,
503 .dep_file = false,
504 },
505 .prefix = .{ .value = null },
506 .suffix = .{ .value = null },
507 .basename = .{ .value = null },
508 .path = .{ .value = null },
509 .producer = .{ .value = null },
510 .generated = .{ .value = null },
511 },
512 })));
513 }
514 return result;
515 }
516
354517 fn initLazyPathList(s: *Serialize, list: []const std.Build.LazyPath) ![]const Configuration.LazyPath.Index {
355518 const result = try s.arena.alloc(Configuration.LazyPath.Index, list.len);
356519 for (result, list) |*dest, src| dest.* = try addLazyPath(s, src);
......@@ -768,16 +931,33 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void {
768931 .update_source_files => @panic("TODO"),
769932 .run => e: {
770933 const run: *Step.Run = @fieldParentPtr("step", step);
771
772 const captured_stdout: Configuration.OptionalString = if (run.captured_stdout) |cs|
773 .init(try wc.addString(cs.output.basename))
774 else
775 .none;
776
777 const captured_stderr: Configuration.OptionalString = if (run.captured_stderr) |cs|
778 .init(try wc.addString(cs.output.basename))
779 else
780 .none;
934 var expect_stderr_exact: ?Configuration.Bytes = null;
935 var expect_stdout_exact: ?Configuration.Bytes = null;
936 var expect_stderr_match: std.ArrayList(Configuration.Bytes) = .empty;
937 var expect_stdout_match: std.ArrayList(Configuration.Bytes) = .empty;
938 var expect_term: ?struct {
939 status: Configuration.Step.Run.ExpectTermStatus,
940 value: u32,
941 } = null;
942 switch (run.stdio) {
943 .check => |checks| for (checks.items) |check| switch (check) {
944 .expect_stderr_exact => |bytes| expect_stderr_exact = try wc.addBytes(bytes),
945 .expect_stdout_exact => |bytes| expect_stdout_exact = try wc.addBytes(bytes),
946 .expect_stderr_match => |bytes| {
947 try expect_stderr_match.append(arena, try wc.addBytes(bytes));
948 },
949 .expect_stdout_match => |bytes| {
950 try expect_stdout_match.append(arena, try wc.addBytes(bytes));
951 },
952 .expect_term => |t| expect_term = switch (t) {
953 .exited => |x| .{ .status = .exited, .value = x },
954 .signal => |x| .{ .status = .signal, .value = @intFromEnum(x) },
955 .stopped => |x| .{ .status = .stopped, .value = x },
956 .unknown => |x| .{ .status = .unknown, .value = x },
957 },
958 },
959 else => {},
960 }
781961
782962 const extra_index = try wc.addExtra(@as(Configuration.Step.Run, .{
783963 .flags = .{
......@@ -802,16 +982,44 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void {
802982 .stderr_trim_whitespace = if (run.captured_stderr) |cs| cs.trim_whitespace else .none,
803983 .stdio_limit = run.stdio_limit != .unlimited,
804984 .producer = run.producer != null,
985 .cwd = run.cwd != null,
986 .captured_stdout = run.captured_stdout != null,
987 .captured_stderr = run.captured_stderr != null,
988 .environ_map = run.environ_map != null,
805989 },
806 .file_inputs_len = @intCast(run.file_inputs.items.len),
807 .args_len = @intCast(run.argv.items.len),
808 .cwd = try s.addOptionalLazyPathEnum(run.cwd),
809 .captured_stdout = captured_stdout,
810 .captured_stderr = captured_stderr,
990 .flags2 = .{
991 .expect_stderr_exact = expect_stderr_exact != null,
992 .expect_stdout_exact = expect_stdout_exact != null,
993 .expect_stderr_match = expect_stderr_match.items.len != 0,
994 .expect_stdout_match = expect_stdout_match.items.len != 0,
995 .expect_term = expect_term != null,
996 .expect_term_status = if (expect_term) |t| t.status else .exited,
997 },
998 .file_inputs = .{ .slice = try s.initLazyPathList(run.file_inputs.items) },
999 .args = .{ .slice = try s.initArgsList(run.argv.items) },
1000 .cwd = .{ .value = try s.addOptionalLazyPath(run.cwd) },
1001 .captured_stdout = .{ .value = if (run.captured_stdout) |cs| .{
1002 .basename = try wc.addString(cs.output.basename),
1003 .generated_file = cs.output.generated_file,
1004 } else null },
1005 .captured_stderr = .{ .value = if (run.captured_stderr) |cs| .{
1006 .basename = try wc.addString(cs.output.basename),
1007 .generated_file = cs.output.generated_file,
1008 } else null },
1009 .environ_map = .{ .value = try s.addEnvironMap(run.environ_map) },
1010 .expect_term_value = .{ .value = if (expect_term) |t| t.value else null },
1011 .stdio_limit = .{ .value = run.stdio_limit.toInt() },
1012 .producer = .{ .value = if (run.producer) |cs| s.stepIndex(&cs.step) else null },
1013 .expect_stderr_exact = .{ .value = if (expect_stderr_exact) |bytes| bytes else null },
1014 .expect_stdout_exact = .{ .value = if (expect_stdout_exact) |bytes| bytes else null },
1015 .expect_stderr_match = .{ .slice = expect_stderr_match.items },
1016 .expect_stdout_match = .{ .slice = expect_stdout_match.items },
1017 .stdin = .{ .u = switch (run.stdin) {
1018 .none => .none,
1019 .bytes => |bytes| .{ .bytes = try wc.addBytes(bytes) },
1020 .lazy_path => |lp| .{ .lazy_path = try s.addLazyPath(lp) },
1021 } },
8111022 }));
812
813 log.err("TODO serialize the trailing Run step data", .{});
814
8151023 break :e @enumFromInt(extra_index);
8161024 },
8171025 .check_file => @panic("TODO"),
lib/std/Build/Configuration.zig+127-36
......@@ -188,6 +188,18 @@ pub const Wip = struct {
188188 return .init(try addString(wip, bytes orelse return .none));
189189 }
190190
191 pub fn addStringList(wip: *Wip, list: []const []const u8) Allocator.Error!StringList {
192 _ = wip;
193 _ = list;
194 @panic("TODO");
195 }
196
197 pub fn addBytes(wip: *Wip, bytes: []const u8) Allocator.Error!Bytes {
198 _ = wip;
199 _ = bytes;
200 @panic("TODO");
201 }
202
191203 pub fn addSemVer(wip: *Wip, sv: std.SemanticVersion) Allocator.Error!String {
192204 var buffer: [256]u8 = undefined;
193205 var writer: std.Io.Writer = .fixed(&buffer);
......@@ -500,52 +512,65 @@ pub const Step = extern struct {
500512 };
501513 };
502514
503 /// Trailing:
504 /// * LazyPath.Index for each file_inputs_len
505 /// * Arg for each args_len
506 /// * environ_map if corresponding flag is set
507 /// * stdin: Bytes, // if StdIn.bytes is chosen
508 /// * stdin: LazyPath.Index, // if StdIn.lazy_path is chosen
509 /// * checks: Checks, // if StdIo.check is chosen
510 /// * stdio_limit: u64, // if stdio_limit is set
511 /// * producer: Step.Index, // if producer is set. always compile step
512515 pub const Run = struct {
513516 flags: @This().Flags,
514 file_inputs_len: u32,
515 args_len: u32,
516 cwd: LazyPath.OptionalIndex,
517 captured_stdout: OptionalString, // basename
518 captured_stderr: OptionalString, // basename
519
520 /// Trailing:
521 /// * String if prefix set
522 /// * String if suffix set
523 /// * String if basename set
524 /// * Step.Index which is always a compile step if tag is artifact
525 /// * LazyPath.Index if tag is path_file, path_directory, or file_content
517 flags2: Flags2,
518 args: Storage.LengthPrefixedList(Arg.Index),
519 cwd: Storage.FlagOptional(.flags, .cwd, LazyPath.Index),
520 captured_stdout: Storage.FlagOptional(.flags, .captured_stdout, CapturedStream),
521 captured_stderr: Storage.FlagOptional(.flags, .captured_stderr, CapturedStream),
522 file_inputs: Storage.LengthPrefixedList(LazyPath.Index),
523 stdio_limit: Storage.FlagOptional(.flags, .stdio_limit, u64),
524 /// Always a compile step.
525 producer: Storage.FlagOptional(.flags, .producer, Step.Index),
526 /// First half is keys, second half is values.
527 environ_map: Storage.FlagOptional(.flags, .environ_map, EnvironMap.Index),
528 stdin: Storage.FlagUnion(.flags, .stdin, StdIn),
529 expect_stderr_exact: Storage.FlagOptional(.flags2, .expect_stderr_exact, Bytes),
530 expect_stdout_exact: Storage.FlagOptional(.flags2, .expect_stdout_exact, Bytes),
531 expect_stderr_match: Storage.FlagLengthPrefixedList(.flags2, .expect_stderr_match, Bytes),
532 expect_stdout_match: Storage.FlagLengthPrefixedList(.flags2, .expect_stdout_match, Bytes),
533 expect_term_value: Storage.FlagOptional(.flags2, .expect_term, u32),
534
535 pub const CapturedStream = extern struct {
536 generated_file: GeneratedFileIndex,
537 basename: String,
538 };
539
526540 pub const Arg = struct {
527 flags: Arg.Flags,
541 flags: @This().Flags,
542 prefix: Storage.FlagOptional(.flags, .prefix, String),
543 suffix: Storage.FlagOptional(.flags, .suffix, String),
544 basename: Storage.FlagOptional(.flags, .basename, String),
545 path: Storage.FlagOptional(.flags, .path, LazyPath.Index),
546 /// Always a compile step.
547 producer: Storage.FlagOptional(.flags, .producer, Step.Index),
548 generated: Storage.FlagOptional(.flags, .generated, GeneratedFileIndex),
528549
529550 pub const Flags = packed struct(u32) {
530551 tag: Arg.Tag,
531552 prefix: bool,
532553 suffix: bool,
533554 basename: bool,
534 /// Implies Tag is output_file
555 path: bool,
556 producer: bool,
557 generated: bool,
535558 dep_file: bool,
536 _: u20 = 0,
559 _: u22 = 0,
537560 };
538561
539 pub const Tag = enum(u8) {
562 pub const Tag = enum(u3) {
540563 artifact,
541564 path_file,
542565 path_directory,
566 string,
543567 file_content,
544 bytes,
545568 output_file,
546569 output_directory,
547570 cli_rest_positionals,
548571 };
572
573 pub const Index = IndexType(@This());
549574 };
550575
551576 pub const Color = enum(u4) {
......@@ -562,26 +587,47 @@ pub const Step = extern struct {
562587 manual,
563588 };
564589
565 pub const StdIn = enum(u2) { none, bytes, lazy_path };
590 pub const StdIn = union(@This().Tag) {
591 none: void,
592 bytes: Bytes,
593 lazy_path: LazyPath.Index,
594
595 pub const Tag = enum(u2) { none, bytes, lazy_path };
596 };
566597 pub const TrimWhitespace = enum(u2) { none, all, leading, trailing };
567598 pub const StdIo = enum(u2) { infer_from_args, inherit, check, zig_test };
568599
600 pub const ExpectTermStatus = enum(u2) { exited, signal, stopped, unknown };
601
569602 pub const Flags = packed struct(u32) {
570603 tag: Tag = .run,
571
572604 disable_zig_progress: bool,
573605 skip_foreign_checks: bool,
574606 failing_to_execute_foreign_is_an_error: bool,
575607 has_side_effects: bool,
576608 test_runner_mode: bool,
577609 color: Color,
578 stdin: StdIn,
610 stdin: StdIn.Tag,
579611 stdio: StdIo,
580612 stdout_trim_whitespace: TrimWhitespace,
581613 stderr_trim_whitespace: TrimWhitespace,
582614 stdio_limit: bool,
583615 producer: bool,
584 _: u8 = 0,
616 cwd: bool,
617 captured_stdout: bool,
618 captured_stderr: bool,
619 environ_map: bool,
620 _: u4 = 0,
621 };
622
623 pub const Flags2 = packed struct(u32) {
624 expect_stderr_exact: bool,
625 expect_stdout_exact: bool,
626 expect_stderr_match: bool,
627 expect_stdout_match: bool,
628 expect_term: bool,
629 expect_term_status: ExpectTermStatus,
630 _: u25 = 0,
585631 };
586632 };
587633
......@@ -1395,17 +1441,37 @@ pub const Deps = struct {
13951441 };
13961442};
13971443
1444pub const EnvironMap = struct {
1445 keys: StringList,
1446 values: StringList,
1447
1448 pub const Index = IndexType(@This());
1449};
1450
13981451/// Points into `extra`, where the first element is count of strings, following
13991452/// elements is `String` per count.
14001453///
14011454/// Stored identically to `Deps`.
1455pub const StringList = enum(u32) {
1456 _,
1457
1458 pub fn slice(this: @This(), c: *const Configuration) []const String {
1459 const len = c.extra[@intFromEnum(this)];
1460 return @ptrCast(c.extra[@intFromEnum(this) + 1 ..][0..len]);
1461 }
1462};
1463
14021464pub const OptionalStringList = enum(u32) {
14031465 none = max_u32,
14041466 _,
14051467
1406 pub fn slice(osl: OptionalStringList, c: *const Configuration) ?[]const String {
1407 const len = c.extra[@intFromEnum(osl)];
1408 return @ptrCast(c.extra[@intFromEnum(osl) + 1 ..][0..len]);
1468 pub fn unwrap(this: @This()) ?StringList {
1469 if (this == .none) return null;
1470 return @enumFromInt(@intFromEnum(this));
1471 }
1472
1473 pub fn slice(this: @This(), c: *const Configuration) ?[]const String {
1474 return (unwrap(this) orelse return null).slice(c);
14091475 }
14101476};
14111477
......@@ -1499,6 +1565,13 @@ pub const String = enum(u32) {
14991565 }
15001566};
15011567
1568/// Arbitrary sequence of bytes that may contain null bytes.
1569pub const Bytes = extern struct {
1570 /// Points into `string_bytes`.
1571 index: u32,
1572 len: u32,
1573};
1574
15021575pub const DefaultingBool = enum(u2) {
15031576 false,
15041577 true,
......@@ -2359,7 +2432,11 @@ pub const Storage = enum {
23592432 },
23602433 },
23612434 },
2362 .@"extern" => comptime unreachable,
2435 .@"extern" => {
2436 const n = @divExact(@sizeOf(Field), @sizeOf(u32));
2437 defer i.* += n;
2438 return @bitCast(buffer[i.*..][0..n].*);
2439 },
23632440 },
23642441 else => comptime unreachable,
23652442 }
......@@ -2404,7 +2481,7 @@ pub const Storage = enum {
24042481 inline else => |v| extraFieldLen(v),
24052482 },
24062483 },
2407 .@"extern" => comptime unreachable,
2484 .@"extern" => @divExact(@sizeOf(Field), @sizeOf(u32)),
24082485 },
24092486 else => @compileError("bad type: " ++ @typeName(Field)),
24102487 };
......@@ -2520,13 +2597,27 @@ pub const Storage = enum {
25202597 },
25212598 },
25222599 },
2523 .@"extern" => comptime unreachable,
2600 .@"extern" => {
2601 const n = @divExact(@sizeOf(Field), @sizeOf(u32));
2602 buffer[i..][0..n].* = @bitCast(value);
2603 return n;
2604 },
25242605 },
25252606 else => @compileError("bad field type: " ++ @typeName(Field)),
25262607 }
25272608 }
25282609};
25292610
2611fn IndexType(comptime T: type) type {
2612 return enum(u32) {
2613 _,
2614
2615 pub fn get(this: @This(), c: *const Configuration) T {
2616 return extraData(c, T, @intFromEnum(this));
2617 }
2618 };
2619}
2620
25302621pub fn extraData(c: *const Configuration, comptime T: type, index: usize) T {
25312622 var i: usize = index;
25322623 return Storage.data(c.extra, &i, T);