authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-02-28 16:52:35-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-03-15 10:48:13-07:00
log986a30e373f6b2f0da2de64570013c83cacc17b6
tree5033b9860e08795a5719ab6619c122060ad69165
parentc583d140135fe5a57d055d9c0b8bdf59698f29e1

integrate the build runner and the compiler server

The compiler now provides a server protocol for an interactive session with another process. The build runner uses this protocol to communicate compilation errors semantically from zig compiler subprocesses to the build runner. The protocol is exposed via stdin/stdout, or on a network socket, depending on whether the CLI flag `--listen=-` or e.g. `--listen=127.0.0.1:1337` is used. Additionally: * add the zig version string to the build runner cache prefix * remove --prominent-compile-errors CLI flag because it no longer does anything. Compilation errors are now unconditionally displayed at the bottom of the build summary output when using the terminal-based build runner. * Remove the color field from std.Build. The build steps are no longer supposed to interact with stderr directly. Instead they communicate semantically back to the build runner, which has its own logic about TTY configuration. * Use the cleanExit() pattern in the build runner. * Build steps can now use error.MakeFailed when they have already properly reported an error, or they can fail with any other error code in which case the build runner will create a simple message based on this error code.

9 files changed, 524 insertions(+), 221 deletions(-)

lib/build_runner.zig+81-32
......@@ -1,6 +1,7 @@
11const root = @import("@build");
22const std = @import("std");
33const builtin = @import("builtin");
4const assert = std.debug.assert;
45const io = std.io;
56const fmt = std.fmt;
67const mem = std.mem;
......@@ -71,8 +72,7 @@ pub fn main() !void {
7172 cache.addPrefix(build_root_directory);
7273 cache.addPrefix(local_cache_directory);
7374 cache.addPrefix(global_cache_directory);
74
75 //cache.hash.addBytes(builtin.zig_version);
75 cache.hash.addBytes(builtin.zig_version_string);
7676
7777 const builder = try std.Build.create(
7878 allocator,
......@@ -95,10 +95,8 @@ pub fn main() !void {
9595 var install_prefix: ?[]const u8 = null;
9696 var dir_list = std.Build.DirList{};
9797
98 // before arg parsing, check for the NO_COLOR environment variable
99 // if it exists, default the color setting to .off
100 // explicit --color arguments will still override this setting.
101 builder.color = if (process.hasEnvVarConstant("NO_COLOR")) .off else .auto;
98 const Color = enum { auto, off, on };
99 var color: Color = .auto;
102100
103101 while (nextArg(args, &arg_idx)) |arg| {
104102 if (mem.startsWith(u8, arg, "-D")) {
......@@ -166,7 +164,7 @@ pub fn main() !void {
166164 std.debug.print("expected [auto|on|off] after --color", .{});
167165 usageAndErr(builder, false, stderr_stream);
168166 };
169 builder.color = std.meta.stringToEnum(@TypeOf(builder.color), next_arg) orelse {
167 color = std.meta.stringToEnum(Color, next_arg) orelse {
170168 std.debug.print("expected [auto|on|off] after --color, found '{s}'", .{next_arg});
171169 usageAndErr(builder, false, stderr_stream);
172170 };
......@@ -200,8 +198,6 @@ pub fn main() !void {
200198 builder.verbose_cc = true;
201199 } else if (mem.eql(u8, arg, "--verbose-llvm-cpu-features")) {
202200 builder.verbose_llvm_cpu_features = true;
203 } else if (mem.eql(u8, arg, "--prominent-compile-errors")) {
204 builder.prominent_compile_errors = true;
205201 } else if (mem.eql(u8, arg, "-fwine")) {
206202 builder.enable_wine = true;
207203 } else if (mem.eql(u8, arg, "-fno-wine")) {
......@@ -257,6 +253,12 @@ pub fn main() !void {
257253 }
258254 }
259255
256 const ttyconf: std.debug.TTY.Config = switch (color) {
257 .auto => std.debug.detectTTYConfig(std.io.getStdErr()),
258 .on => .escape_codes,
259 .off => .no_color,
260 };
261
260262 var progress: std.Progress = .{};
261263 const main_progress_node = progress.start("", 0);
262264 defer main_progress_node.end();
......@@ -272,11 +274,15 @@ pub fn main() !void {
272274 if (builder.validateUserInputDidItFail())
273275 usageAndErr(builder, true, stderr_stream);
274276
275 runStepNames(builder, targets.items, main_progress_node, thread_pool_options) catch |err| {
276 switch (err) {
277 error.UncleanExit => process.exit(1),
278 else => return err,
279 }
277 runStepNames(
278 builder,
279 targets.items,
280 main_progress_node,
281 thread_pool_options,
282 ttyconf,
283 ) catch |err| switch (err) {
284 error.UncleanExit => process.exit(1),
285 else => return err,
280286 };
281287}
282288
......@@ -285,6 +291,7 @@ fn runStepNames(
285291 step_names: []const []const u8,
286292 parent_prog_node: *std.Progress.Node,
287293 thread_pool_options: std.Thread.Pool.Options,
294 ttyconf: std.debug.TTY.Config,
288295) !void {
289296 var step_stack = ArrayList(*Step).init(b.allocator);
290297 defer step_stack.deinit();
......@@ -332,12 +339,14 @@ fn runStepNames(
332339
333340 wait_group.start();
334341 thread_pool.spawn(workerMakeOneStep, .{
335 &wait_group, &thread_pool, b, step, &step_prog,
342 &wait_group, &thread_pool, b, step, &step_prog, ttyconf,
336343 }) catch @panic("OOM");
337344 }
338345 }
339346
340 var any_failed = false;
347 var success_count: usize = 0;
348 var failure_count: usize = 0;
349 var pending_count: usize = 0;
341350
342351 for (step_stack.items) |s| {
343352 switch (s.state) {
......@@ -349,20 +358,42 @@ fn runStepNames(
349358 // A -> B -> C (failure)
350359 // B will be marked as dependency_failure, while A may never be queued, and thus
351360 // remain in the initial state of precheck_done.
352 .dependency_failure, .precheck_done => continue,
353 .success => continue,
354 .failure => {
355 any_failed = true;
356 std.debug.print("{s}: {s}\n", .{
357 s.name, @errorName(s.result.err_code),
358 });
359 },
361 .dependency_failure, .precheck_done => pending_count += 1,
362 .success => success_count += 1,
363 .failure => failure_count += 1,
360364 }
361365 }
362366
363 if (any_failed) {
364 process.exit(1);
365 }
367 const stderr = std.io.getStdErr();
368
369 const total_count = success_count + failure_count + pending_count;
370 stderr.writer().print("build summary: {d}/{d} steps succeeded; {d} failed\n", .{
371 success_count, total_count, failure_count,
372 }) catch {};
373 if (failure_count == 0) return cleanExit();
374
375 for (step_stack.items) |s| switch (s.state) {
376 .failure => {
377 // TODO print the dep prefix too
378 ttyconf.setColor(stderr, .Bold) catch break;
379 stderr.writeAll(s.name) catch break;
380 ttyconf.setColor(stderr, .Reset) catch break;
381
382 if (s.result_error_bundle.errorMessageCount() > 0) {
383 stderr.writer().print(": {d} compilation errors:\n", .{
384 s.result_error_bundle.errorMessageCount(),
385 }) catch break;
386 s.result_error_bundle.renderToStdErr(ttyconf);
387 } else {
388 stderr.writer().print(": {d} error messages (printed above)\n", .{
389 s.result_error_msgs.items.len,
390 }) catch break;
391 }
392 },
393 else => continue,
394 };
395
396 process.exit(1);
366397}
367398
368399fn checkForDependencyLoop(
......@@ -407,6 +438,7 @@ fn workerMakeOneStep(
407438 b: *std.Build,
408439 s: *Step,
409440 prog_node: *std.Progress.Node,
441 ttyconf: std.debug.TTY.Config,
410442) void {
411443 defer wg.finish();
412444
......@@ -446,17 +478,26 @@ fn workerMakeOneStep(
446478 const make_result = s.make();
447479
448480 // No matter the result, we want to display error/warning messages.
449 if (s.result.error_msgs.items.len > 0) {
481 if (s.result_error_msgs.items.len > 0) {
450482 sub_prog_node.context.lock_stderr();
451483 defer sub_prog_node.context.unlock_stderr();
452484
453 for (s.result.error_msgs.items) |msg| {
454 std.io.getStdErr().writeAll(msg) catch break;
485 const stderr = std.io.getStdErr();
486
487 for (s.result_error_msgs.items) |msg| {
488 // TODO print the dep prefix too
489 ttyconf.setColor(stderr, .Bold) catch break;
490 stderr.writeAll(s.name) catch break;
491 stderr.writeAll(": ") catch break;
492 ttyconf.setColor(stderr, .Red) catch break;
493 stderr.writeAll("error: ") catch break;
494 ttyconf.setColor(stderr, .Reset) catch break;
495 stderr.writeAll(msg) catch break;
455496 }
456497 }
457498
458499 make_result catch |err| {
459 s.result.err_code = err;
500 assert(err == error.MakeFailed);
460501 @atomicStore(Step.State, &s.state, .failure, .SeqCst);
461502 return;
462503 };
......@@ -467,7 +508,7 @@ fn workerMakeOneStep(
467508 for (s.dependants.items) |dep| {
468509 wg.start();
469510 thread_pool.spawn(workerMakeOneStep, .{
470 wg, thread_pool, b, dep, prog_node,
511 wg, thread_pool, b, dep, prog_node, ttyconf,
471512 }) catch @panic("OOM");
472513 }
473514}
......@@ -601,3 +642,11 @@ fn argsRest(args: [][]const u8, idx: usize) ?[][]const u8 {
601642 if (idx >= args.len) return null;
602643 return args[idx..];
603644}
645
646fn cleanExit() void {
647 if (builtin.mode == .Debug) {
648 return;
649 } else {
650 process.exit(0);
651 }
652}
lib/std/Build.zig+118-29
......@@ -59,9 +59,6 @@ verbose_air: bool,
5959verbose_llvm_ir: bool,
6060verbose_cimport: bool,
6161verbose_llvm_cpu_features: bool,
62/// The purpose of executing the command is for a human to read compile errors from the terminal
63prominent_compile_errors: bool,
64color: enum { auto, on, off } = .auto,
6562reference_trace: ?u32 = null,
6663invalid_user_input: bool,
6764zig_exe: []const u8,
......@@ -211,7 +208,6 @@ pub fn create(
211208 .verbose_llvm_ir = false,
212209 .verbose_cimport = false,
213210 .verbose_llvm_cpu_features = false,
214 .prominent_compile_errors = false,
215211 .invalid_user_input = false,
216212 .allocator = allocator,
217213 .user_input_options = UserInputOptionsMap.init(allocator),
......@@ -295,8 +291,6 @@ fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: Cache.Direc
295291 .verbose_llvm_ir = parent.verbose_llvm_ir,
296292 .verbose_cimport = parent.verbose_cimport,
297293 .verbose_llvm_cpu_features = parent.verbose_llvm_cpu_features,
298 .prominent_compile_errors = parent.prominent_compile_errors,
299 .color = parent.color,
300294 .reference_trace = parent.reference_trace,
301295 .invalid_user_input = false,
302296 .zig_exe = parent.zig_exe,
......@@ -1409,54 +1403,149 @@ pub fn execAllowFail(
14091403 }
14101404}
14111405
1412pub fn execFromStep(b: *Build, argv: []const []const u8, s: *Step) ![]u8 {
1406/// This function is used exclusively for spawning and communicating with the zig compiler.
1407/// TODO: move to build_runner.zig
1408pub fn execFromStep(b: *Build, argv: []const []const u8, s: *Step) ![]const u8 {
14131409 assert(argv.len != 0);
14141410
14151411 if (b.verbose) {
1416 printCmd(b.allocator, null, argv);
1412 const text = try allocPrintCmd(b.allocator, null, argv);
1413 try s.result_error_msgs.append(b.allocator, text);
14171414 }
14181415
14191416 if (!process.can_spawn) {
1420 try s.result.error_msgs.append(b.allocator, b.fmt("Unable to spawn the following command: cannot spawn child processes\n{s}", .{
1417 try s.result_error_msgs.append(b.allocator, b.fmt("Unable to spawn the following command: cannot spawn child processes\n{s}", .{
14211418 try allocPrintCmd(b.allocator, null, argv),
14221419 }));
1423 return error.CannotSpawnProcesses;
1420 return error.MakeFailed;
14241421 }
14251422
1426 const result = std.ChildProcess.exec(.{
1427 .allocator = b.allocator,
1428 .argv = argv,
1429 .env_map = b.env_map,
1430 .max_output_bytes = 10 * 1024 * 1024,
1431 }) catch |err| {
1432 try s.result.error_msgs.append(b.allocator, b.fmt("unable to spawn the following command: {s}\n{s}", .{
1433 @errorName(err), try allocPrintCmd(b.allocator, null, argv),
1434 }));
1435 return error.ExecFailed;
1436 };
1423 var child = std.ChildProcess.init(argv, b.allocator);
1424 child.env_map = b.env_map;
1425 child.stdin_behavior = .Pipe;
1426 child.stdout_behavior = .Pipe;
1427 child.stderr_behavior = .Pipe;
1428
1429 try child.spawn();
14371430
1438 if (result.stderr.len != 0) {
1439 try s.result.error_msgs.append(b.allocator, result.stderr);
1431 var poller = std.io.poll(b.allocator, enum { stdout, stderr }, .{
1432 .stdout = child.stdout.?,
1433 .stderr = child.stderr.?,
1434 });
1435 defer poller.deinit();
1436
1437 try sendMessage(child.stdin.?, .update);
1438 try sendMessage(child.stdin.?, .exit);
1439
1440 const Header = std.zig.Server.Message.Header;
1441 var result: ?[]const u8 = null;
1442
1443 while (try poller.poll()) {
1444 const stdout = poller.fifo(.stdout);
1445 const buf = stdout.readableSlice(0);
1446 assert(stdout.readableLength() == buf.len);
1447 if (buf.len >= @sizeOf(Header)) {
1448 const header = @ptrCast(*align(1) const Header, buf[0..@sizeOf(Header)]);
1449 const header_and_msg_len = header.bytes_len + @sizeOf(Header);
1450 if (buf.len >= header_and_msg_len) {
1451 const body = buf[@sizeOf(Header)..];
1452 switch (header.tag) {
1453 .zig_version => {
1454 if (!mem.eql(u8, builtin.zig_version_string, body)) {
1455 try s.result_error_msgs.append(
1456 b.allocator,
1457 b.fmt("zig version mismatch build runner vs compiler: '{s}' vs '{s}'", .{
1458 builtin.zig_version_string, body,
1459 }),
1460 );
1461 return error.MakeFailed;
1462 }
1463 },
1464 .error_bundle => {
1465 const EbHdr = std.zig.Server.Message.ErrorBundle;
1466 const eb_hdr = @ptrCast(*align(1) const EbHdr, body);
1467 const extra_bytes =
1468 body[@sizeOf(EbHdr)..][0 .. @sizeOf(u32) * eb_hdr.extra_len];
1469 const string_bytes =
1470 body[@sizeOf(EbHdr) + extra_bytes.len ..][0..eb_hdr.string_bytes_len];
1471 // TODO: use @ptrCast when the compiler supports it
1472 const unaligned_extra = mem.bytesAsSlice(u32, extra_bytes);
1473 const extra_array = try b.allocator.alloc(u32, unaligned_extra.len);
1474 // TODO: use @memcpy when it supports slices
1475 for (extra_array, unaligned_extra) |*dst, src| dst.* = src;
1476 s.result_error_bundle = .{
1477 .string_bytes = try b.allocator.dupe(u8, string_bytes),
1478 .extra = extra_array,
1479 };
1480 },
1481 .progress => {
1482 @panic("TODO handle progress message");
1483 },
1484 .emit_bin_path => {
1485 @panic("TODO handle emit_bin_path message");
1486 },
1487 _ => {
1488 // Unrecognized message.
1489 },
1490 }
1491 stdout.discard(header_and_msg_len);
1492 }
1493 }
1494 }
1495
1496 const stderr = poller.fifo(.stderr);
1497 if (stderr.readableLength() > 0) {
1498 try s.result_error_msgs.append(b.allocator, try stderr.toOwnedSlice());
14401499 }
14411500
1442 switch (result.term) {
1501 // Send EOF to stdin.
1502 child.stdin.?.close();
1503 child.stdin = null;
1504
1505 const term = try child.wait();
1506 switch (term) {
14431507 .Exited => |code| {
14441508 if (code != 0) {
1445 try s.result.error_msgs.append(b.allocator, b.fmt("the following command exited with error code {d}:\n{s}", .{
1509 try s.result_error_msgs.append(b.allocator, b.fmt("the following command exited with error code {d}:\n{s}", .{
14461510 code, try allocPrintCmd(b.allocator, null, argv),
14471511 }));
1448 return error.ExitCodeFailure;
1512 return error.MakeFailed;
14491513 }
1450 return result.stdout;
14511514 },
14521515 .Signal, .Stopped, .Unknown => |code| {
14531516 _ = code;
1454 try s.result.error_msgs.append(b.allocator, b.fmt("the following command terminated unexpectedly:\n{s}", .{
1517 try s.result_error_msgs.append(b.allocator, b.fmt("the following command terminated unexpectedly:\n{s}", .{
14551518 try allocPrintCmd(b.allocator, null, argv),
14561519 }));
1457 return error.ProcessTerminated;
1520 return error.MakeFailed;
14581521 },
14591522 }
1523
1524 if (s.result_error_bundle.errorMessageCount() > 0) {
1525 try s.result_error_msgs.append(
1526 b.allocator,
1527 b.fmt("the following command failed with {d} compilation errors:\n{s}", .{
1528 s.result_error_bundle.errorMessageCount(),
1529 try allocPrintCmd(b.allocator, null, argv),
1530 }),
1531 );
1532 return error.MakeFailed;
1533 }
1534
1535 return result orelse {
1536 try s.result_error_msgs.append(b.allocator, b.fmt("the following command failed to communicate the compilation result:\n{s}", .{
1537 try allocPrintCmd(b.allocator, null, argv),
1538 }));
1539 return error.MakeFailed;
1540 };
1541}
1542
1543fn sendMessage(file: fs.File, tag: std.zig.Client.Message.Tag) !void {
1544 const header: std.zig.Client.Message.Header = .{
1545 .tag = tag,
1546 .bytes_len = 0,
1547 };
1548 try file.writeAll(std.mem.asBytes(&header));
14601549}
14611550
14621551/// This is a helper function to be called from build.zig scripts, *not* from
lib/std/Build/CompileStep.zig+1-5
......@@ -1177,11 +1177,6 @@ fn make(step: *Step) !void {
11771177 };
11781178 try zig_args.append(cmd);
11791179
1180 if (builder.color != .auto) {
1181 try zig_args.append("--color");
1182 try zig_args.append(@tagName(builder.color));
1183 }
1184
11851180 if (builder.reference_trace) |some| {
11861181 try zig_args.append(try std.fmt.allocPrint(builder.allocator, "-freference-trace={d}", .{some}));
11871182 }
......@@ -1834,6 +1829,7 @@ fn make(step: *Step) !void {
18341829 }
18351830
18361831 try zig_args.append("--enable-cache");
1832 try zig_args.append("--listen=-");
18371833
18381834 // Windows has an argument length limit of 32,766 characters, macOS 262,144 and Linux
18391835 // 2,097,152. If our args exceed 30 KiB, we instead write them to a "response file" and
lib/std/Build/RunStep.zig+2-6
......@@ -419,12 +419,8 @@ pub fn runCommand(
419419 };
420420
421421 if (!termMatches(expected_term, term)) {
422 if (builder.prominent_compile_errors) {
423 std.debug.print("Run step {} (expected {})\n", .{ fmtTerm(term), fmtTerm(expected_term) });
424 } else {
425 std.debug.print("The following command {} (expected {}):\n", .{ fmtTerm(term), fmtTerm(expected_term) });
426 printCmd(cwd, argv);
427 }
422 std.debug.print("The following command {} (expected {}):\n", .{ fmtTerm(term), fmtTerm(expected_term) });
423 printCmd(cwd, argv);
428424 return error.UnexpectedExit;
429425 }
430426
lib/std/Build/Step.zig+18-11
......@@ -6,15 +6,13 @@ dependencies: std.ArrayList(*Step),
66/// then populated during dependency loop checking in the build runner.
77dependants: std.ArrayListUnmanaged(*Step),
88state: State,
9/// Populated only if state is success.
10result: struct {
11 err_code: anyerror,
12 error_msgs: std.ArrayListUnmanaged([]const u8),
13},
149/// The return addresss associated with creation of this step that can be useful
1510/// to print along with debugging messages.
1611debug_stack_trace: [n_debug_stack_frames]usize,
1712
13result_error_msgs: std.ArrayListUnmanaged([]const u8),
14result_error_bundle: std.zig.ErrorBundle,
15
1816const n_debug_stack_frames = 4;
1917
2018pub const State = enum {
......@@ -94,16 +92,25 @@ pub fn init(allocator: Allocator, options: Options) Step {
9492 .dependencies = std.ArrayList(*Step).init(allocator),
9593 .dependants = .{},
9694 .state = .precheck_unstarted,
97 .result = .{
98 .err_code = undefined,
99 .error_msgs = .{},
100 },
10195 .debug_stack_trace = addresses,
96 .result_error_msgs = .{},
97 .result_error_bundle = std.zig.ErrorBundle.empty,
10298 };
10399}
104100
105pub fn make(self: *Step) !void {
106 try self.makeFn(self);
101/// If the Step's `make` function reports `error.MakeFailed`, it indicates they
102/// have already reported the error. Otherwise, we add a simple error report
103/// here.
104pub fn make(s: *Step) error{MakeFailed}!void {
105 return s.makeFn(s) catch |err| {
106 if (err != error.MakeFailed) {
107 const gpa = s.dependencies.allocator;
108 s.result_error_msgs.append(gpa, std.fmt.allocPrint(gpa, "{s} failed: {s}", .{
109 s.name, @errorName(err),
110 }) catch @panic("OOM")) catch @panic("OOM");
111 }
112 return error.MakeFailed;
113 };
107114}
108115
109116pub fn dependOn(self: *Step, other: *Step) void {
lib/std/zig.zig+2
......@@ -4,6 +4,8 @@ const fmt = @import("zig/fmt.zig");
44const assert = std.debug.assert;
55
66pub const ErrorBundle = @import("zig/ErrorBundle.zig");
7pub const Server = @import("zig/Server.zig");
8pub const Client = @import("zig/Client.zig");
79pub const Token = tokenizer.Token;
810pub const Tokenizer = tokenizer.Tokenizer;
911pub const fmtId = fmt.fmtId;
lib/std/zig/Client.zig created+32
......@@ -0,0 +1,32 @@
1pub const Message = struct {
2 pub const Header = extern struct {
3 tag: Tag,
4 /// Size of the body only; does not include this Header.
5 bytes_len: u32,
6 };
7
8 pub const Tag = enum(u32) {
9 /// Tells the compiler to shut down cleanly.
10 /// No body.
11 exit,
12 /// Tells the compiler to detect changes in source files and update the
13 /// affected output compilation artifacts.
14 /// If one of the compilation artifacts is an executable that is
15 /// running as a child process, the compiler will wait for it to exit
16 /// before performing the update.
17 /// No body.
18 update,
19 /// Tells the compiler to execute the executable as a child process.
20 /// No body.
21 run,
22 /// Tells the compiler to detect changes in source files and update the
23 /// affected output compilation artifacts.
24 /// If one of the compilation artifacts is an executable that is
25 /// running as a child process, the compiler will perform a hot code
26 /// swap.
27 /// No body.
28 hot_update,
29
30 _,
31 };
32};
lib/std/zig/Server.zig created+28
......@@ -0,0 +1,28 @@
1pub const Message = struct {
2 pub const Header = extern struct {
3 tag: Tag,
4 /// Size of the body only; does not include this Header.
5 bytes_len: u32,
6 };
7
8 pub const Tag = enum(u32) {
9 /// Body is a UTF-8 string.
10 zig_version,
11 /// Body is an ErrorBundle.
12 error_bundle,
13 /// Body is a UTF-8 string.
14 progress,
15 /// Body is a UTF-8 string.
16 emit_bin_path,
17 _,
18 };
19
20 /// Trailing:
21 /// * extra: [extra_len]u32,
22 /// * string_bytes: [string_bytes_len]u8,
23 /// See `std.zig.ErrorBundle`.
24 pub const ErrorBundle = extern struct {
25 extra_len: u32,
26 string_bytes_len: u32,
27 };
28};
src/main.zig+242-138
......@@ -668,6 +668,12 @@ const ArgMode = union(enum) {
668668 run,
669669};
670670
671const Listen = union(enum) {
672 none,
673 ip4: std.net.Ip4Address,
674 stdio,
675};
676
671677fn buildOutputType(
672678 gpa: Allocator,
673679 arena: Allocator,
......@@ -689,7 +695,7 @@ fn buildOutputType(
689695 var function_sections = false;
690696 var no_builtin = false;
691697 var watch = false;
692 var listen_addr: ?std.net.Ip4Address = null;
698 var listen: Listen = .none;
693699 var debug_compile_errors = false;
694700 var verbose_link = (builtin.os.tag != .wasi or builtin.link_libc) and std.process.hasEnvVarConstant("ZIG_VERBOSE_LINK");
695701 var verbose_cc = (builtin.os.tag != .wasi or builtin.link_libc) and std.process.hasEnvVarConstant("ZIG_VERBOSE_CC");
......@@ -1149,14 +1155,22 @@ fn buildOutputType(
11491155 }
11501156 } else if (mem.eql(u8, arg, "--listen")) {
11511157 const next_arg = args_iter.nextOrFatal();
1152 // example: --listen 127.0.0.1:9000
1153 var it = std.mem.split(u8, next_arg, ":");
1154 const host = it.next().?;
1155 const port_text = it.next() orelse "14735";
1156 const port = std.fmt.parseInt(u16, port_text, 10) catch |err|
1157 fatal("invalid port number: '{s}': {s}", .{ port_text, @errorName(err) });
1158 listen_addr = std.net.Ip4Address.parse(host, port) catch |err|
1159 fatal("invalid host: '{s}': {s}", .{ host, @errorName(err) });
1158 if (mem.eql(u8, next_arg, "-")) {
1159 listen = .stdio;
1160 watch = true;
1161 } else {
1162 // example: --listen 127.0.0.1:9000
1163 var it = std.mem.split(u8, next_arg, ":");
1164 const host = it.next().?;
1165 const port_text = it.next() orelse "14735";
1166 const port = std.fmt.parseInt(u16, port_text, 10) catch |err|
1167 fatal("invalid port number: '{s}': {s}", .{ port_text, @errorName(err) });
1168 listen = .{ .ip4 = std.net.Ip4Address.parse(host, port) catch |err|
1169 fatal("invalid host: '{s}': {s}", .{ host, @errorName(err) }) };
1170 watch = true;
1171 }
1172 } else if (mem.eql(u8, arg, "--listen=-")) {
1173 listen = .stdio;
11601174 watch = true;
11611175 } else if (mem.eql(u8, arg, "--debug-link-snapshot")) {
11621176 if (!build_options.enable_link_snapshots) {
......@@ -3277,6 +3291,47 @@ fn buildOutputType(
32773291 return cmdTranslateC(comp, arena, have_enable_cache);
32783292 }
32793293
3294 switch (listen) {
3295 .none => {},
3296 .stdio => {
3297 try serve(
3298 comp,
3299 std.io.getStdIn(),
3300 std.io.getStdOut(),
3301 test_exec_args.items,
3302 self_exe_path,
3303 arg_mode,
3304 all_args,
3305 runtime_args_start,
3306 );
3307 return cleanExit();
3308 },
3309 .ip4 => |ip4_addr| {
3310 var server = std.net.StreamServer.init(.{
3311 .reuse_address = true,
3312 });
3313 defer server.deinit();
3314
3315 try server.listen(.{ .in = ip4_addr });
3316
3317 while (true) {
3318 const conn = try server.accept();
3319 defer conn.stream.close();
3320
3321 try serve(
3322 comp,
3323 .{ .handle = conn.stream.handle },
3324 .{ .handle = conn.stream.handle },
3325 test_exec_args.items,
3326 self_exe_path,
3327 arg_mode,
3328 all_args,
3329 runtime_args_start,
3330 );
3331 }
3332 },
3333 }
3334
32803335 const hook: AfterUpdateHook = blk: {
32813336 if (!have_enable_cache)
32823337 break :blk .none;
......@@ -3354,6 +3409,12 @@ fn buildOutputType(
33543409 );
33553410 }
33563411
3412 // TODO move this REPL implementation to the standard library / build
3413 // system and have it be a CLI abstraction layer on top of the real, actual
3414 // binary protocol of the compiler. Make it actually interface through the
3415 // server protocol. This way the REPL does not have any special powers that
3416 // an IDE couldn't also have.
3417
33573418 const stdin = std.io.getStdIn().reader();
33583419 const stderr = std.io.getStdErr().writer();
33593420 var repl_buf: [1024]u8 = undefined;
......@@ -3367,123 +3428,6 @@ fn buildOutputType(
33673428
33683429 var last_cmd: ReplCmd = .help;
33693430
3370 if (listen_addr) |ip4_addr| {
3371 var server = std.net.StreamServer.init(.{
3372 .reuse_address = true,
3373 });
3374 defer server.deinit();
3375
3376 try server.listen(.{ .in = ip4_addr });
3377
3378 while (true) {
3379 const conn = try server.accept();
3380 defer conn.stream.close();
3381
3382 var buf: [100]u8 = undefined;
3383 var child_pid: ?i32 = null;
3384
3385 while (true) {
3386 try comp.makeBinFileExecutable();
3387
3388 const amt = try conn.stream.read(&buf);
3389 const line = buf[0..amt];
3390 const actual_line = mem.trimRight(u8, line, "\r\n ");
3391
3392 const cmd: ReplCmd = blk: {
3393 if (mem.eql(u8, actual_line, "update")) {
3394 break :blk .update;
3395 } else if (mem.eql(u8, actual_line, "exit")) {
3396 break;
3397 } else if (mem.eql(u8, actual_line, "help")) {
3398 break :blk .help;
3399 } else if (mem.eql(u8, actual_line, "run")) {
3400 break :blk .run;
3401 } else if (mem.eql(u8, actual_line, "update-and-run")) {
3402 break :blk .update_and_run;
3403 } else if (actual_line.len == 0) {
3404 break :blk last_cmd;
3405 } else {
3406 try stderr.print("unknown command: {s}\n", .{actual_line});
3407 continue;
3408 }
3409 };
3410 last_cmd = cmd;
3411 switch (cmd) {
3412 .update => {
3413 tracy.frameMark();
3414 if (output_mode == .Exe) {
3415 try comp.makeBinFileWritable();
3416 }
3417 updateModule(gpa, comp, hook) catch |err| switch (err) {
3418 error.SemanticAnalyzeFail => continue,
3419 else => |e| return e,
3420 };
3421 },
3422 .help => {
3423 try stderr.writeAll(repl_help);
3424 },
3425 .run => {
3426 tracy.frameMark();
3427 try runOrTest(
3428 comp,
3429 gpa,
3430 arena,
3431 test_exec_args.items,
3432 self_exe_path.?,
3433 arg_mode,
3434 target_info,
3435 watch,
3436 &comp_destroyed,
3437 all_args,
3438 runtime_args_start,
3439 link_libc,
3440 );
3441 },
3442 .update_and_run => {
3443 tracy.frameMark();
3444 if (child_pid) |pid| {
3445 try conn.stream.writer().print("hot code swap requested for pid {d}", .{pid});
3446 try comp.hotCodeSwap(pid);
3447
3448 var errors = try comp.getAllErrorsAlloc();
3449 defer errors.deinit(comp.gpa);
3450
3451 if (errors.errorMessageCount() > 0) {
3452 const ttyconf: std.debug.TTY.Config = switch (comp.color) {
3453 .auto => std.debug.detectTTYConfig(std.io.getStdErr()),
3454 .on => .escape_codes,
3455 .off => .no_color,
3456 };
3457 try errors.renderToWriter(ttyconf, conn.stream.writer());
3458 continue;
3459 }
3460 } else {
3461 if (output_mode == .Exe) {
3462 try comp.makeBinFileWritable();
3463 }
3464 updateModule(gpa, comp, hook) catch |err| switch (err) {
3465 error.SemanticAnalyzeFail => continue,
3466 else => |e| return e,
3467 };
3468 try comp.makeBinFileExecutable();
3469
3470 child_pid = try runOrTestHotSwap(
3471 comp,
3472 gpa,
3473 arena,
3474 test_exec_args.items,
3475 self_exe_path.?,
3476 arg_mode,
3477 all_args,
3478 runtime_args_start,
3479 );
3480 }
3481 },
3482 }
3483 }
3484 }
3485 }
3486
34873431 while (watch) {
34883432 try stderr.print("(zig) ", .{});
34893433 try comp.makeBinFileExecutable();
......@@ -3576,6 +3520,173 @@ fn buildOutputType(
35763520 return cleanExit();
35773521}
35783522
3523fn serve(
3524 comp: *Compilation,
3525 in: fs.File,
3526 out: fs.File,
3527 test_exec_args: []const ?[]const u8,
3528 self_exe_path: ?[]const u8,
3529 arg_mode: ArgMode,
3530 all_args: []const []const u8,
3531 runtime_args_start: ?usize,
3532) !void {
3533 const gpa = comp.gpa;
3534
3535 try serveMessage(out, .{
3536 .tag = .zig_version,
3537 .bytes_len = build_options.version.len,
3538 }, &.{
3539 build_options.version,
3540 });
3541
3542 var child_pid: ?i32 = null;
3543 var receive_fifo = std.fifo.LinearFifo(u8, .Dynamic).init(gpa);
3544 defer receive_fifo.deinit();
3545
3546 while (true) {
3547 const hdr = try receiveMessage(in, &receive_fifo);
3548
3549 switch (hdr.tag) {
3550 .exit => {
3551 return cleanExit();
3552 },
3553 .update => {
3554 tracy.frameMark();
3555 if (comp.bin_file.options.output_mode == .Exe) {
3556 try comp.makeBinFileWritable();
3557 }
3558 try comp.update();
3559 try comp.makeBinFileExecutable();
3560 try serveUpdateResults(out, comp);
3561 },
3562 .run => {
3563 if (child_pid != null) {
3564 @panic("TODO block until the child exits");
3565 }
3566 @panic("TODO call runOrTest");
3567 //try runOrTest(
3568 // comp,
3569 // gpa,
3570 // arena,
3571 // test_exec_args,
3572 // self_exe_path.?,
3573 // arg_mode,
3574 // target_info,
3575 // true,
3576 // &comp_destroyed,
3577 // all_args,
3578 // runtime_args_start,
3579 // link_libc,
3580 //);
3581 },
3582 .hot_update => {
3583 tracy.frameMark();
3584 if (child_pid) |pid| {
3585 try comp.hotCodeSwap(pid);
3586 try serveUpdateResults(out, comp);
3587 } else {
3588 if (comp.bin_file.options.output_mode == .Exe) {
3589 try comp.makeBinFileWritable();
3590 }
3591 try comp.update();
3592 try comp.makeBinFileExecutable();
3593 try serveUpdateResults(out, comp);
3594
3595 child_pid = try runOrTestHotSwap(
3596 comp,
3597 gpa,
3598 test_exec_args,
3599 self_exe_path.?,
3600 arg_mode,
3601 all_args,
3602 runtime_args_start,
3603 );
3604 }
3605 },
3606 _ => {
3607 @panic("TODO unrecognized message from client");
3608 },
3609 }
3610 }
3611}
3612
3613fn serveMessage(
3614 out: fs.File,
3615 header: std.zig.Server.Message.Header,
3616 bufs: []const []const u8,
3617) !void {
3618 var iovecs: [10]std.os.iovec_const = undefined;
3619 iovecs[0] = .{
3620 .iov_base = @ptrCast([*]const u8, &header),
3621 .iov_len = @sizeOf(std.zig.Server.Message.Header),
3622 };
3623 for (bufs, iovecs[1 .. bufs.len + 1]) |buf, *iovec| {
3624 iovec.* = .{
3625 .iov_base = buf.ptr,
3626 .iov_len = buf.len,
3627 };
3628 }
3629 try out.writevAll(iovecs[0 .. bufs.len + 1]);
3630}
3631
3632fn serveErrorBundle(out: fs.File, error_bundle: std.zig.ErrorBundle) !void {
3633 const eb_hdr: std.zig.Server.Message.ErrorBundle = .{
3634 .extra_len = @intCast(u32, error_bundle.extra.len),
3635 .string_bytes_len = @intCast(u32, error_bundle.string_bytes.len),
3636 };
3637 const bytes_len = @sizeOf(std.zig.Server.Message.ErrorBundle) +
3638 4 * error_bundle.extra.len + error_bundle.string_bytes.len;
3639 try serveMessage(out, .{
3640 .tag = .error_bundle,
3641 .bytes_len = @intCast(u32, bytes_len),
3642 }, &.{
3643 std.mem.asBytes(&eb_hdr),
3644 // TODO: implement @ptrCast between slices changing the length
3645 std.mem.sliceAsBytes(error_bundle.extra),
3646 error_bundle.string_bytes,
3647 });
3648}
3649
3650fn serveUpdateResults(out: fs.File, comp: *Compilation) !void {
3651 const gpa = comp.gpa;
3652 var error_bundle = try comp.getAllErrorsAlloc();
3653 defer error_bundle.deinit(gpa);
3654 if (error_bundle.errorMessageCount() > 0) {
3655 try serveErrorBundle(out, error_bundle);
3656 } else if (comp.bin_file.options.emit) |emit| {
3657 const full_path = try emit.directory.join(gpa, &.{emit.sub_path});
3658 defer gpa.free(full_path);
3659
3660 try serveMessage(out, .{
3661 .tag = .emit_bin_path,
3662 .bytes_len = @intCast(u32, full_path.len),
3663 }, &.{
3664 full_path,
3665 });
3666 }
3667}
3668
3669fn receiveMessage(in: fs.File, fifo: *std.fifo.LinearFifo(u8, .Dynamic)) !std.zig.Client.Message.Header {
3670 const Header = std.zig.Client.Message.Header;
3671
3672 while (true) {
3673 const buf = fifo.readableSlice(0);
3674 assert(fifo.readableLength() == buf.len);
3675 if (buf.len >= @sizeOf(Header)) {
3676 const header = @ptrCast(*align(1) const Header, buf[0..@sizeOf(Header)]);
3677 if (header.bytes_len != 0)
3678 return error.InvalidClientMessage;
3679 const result = header.*;
3680 fifo.discard(@sizeOf(Header));
3681 return result;
3682 }
3683
3684 const write_buffer = try fifo.writableWithSize(256);
3685 const amt = try in.read(write_buffer);
3686 fifo.update(amt);
3687 }
3688}
3689
35793690const ModuleDepIterator = struct {
35803691 split: mem.SplitIterator(u8),
35813692
......@@ -3765,7 +3876,6 @@ fn runOrTest(
37653876fn runOrTestHotSwap(
37663877 comp: *Compilation,
37673878 gpa: Allocator,
3768 arena: Allocator,
37693879 test_exec_args: []const ?[]const u8,
37703880 self_exe_path: []const u8,
37713881 arg_mode: ArgMode,
......@@ -3775,9 +3885,10 @@ fn runOrTestHotSwap(
37753885 const exe_emit = comp.bin_file.options.emit.?;
37763886 // A naive `directory.join` here will indeed get the correct path to the binary,
37773887 // however, in the case of cwd, we actually want `./foo` so that the path can be executed.
3778 const exe_path = try fs.path.join(arena, &[_][]const u8{
3888 const exe_path = try fs.path.join(gpa, &[_][]const u8{
37793889 exe_emit.directory.path orelse ".", exe_emit.sub_path,
37803890 });
3891 defer gpa.free(exe_path);
37813892
37823893 var argv = std.ArrayList([]const u8).init(gpa);
37833894 defer argv.deinit();
......@@ -3807,7 +3918,7 @@ fn runOrTestHotSwap(
38073918 if (runtime_args_start) |i| {
38083919 try argv.appendSlice(all_args[i..]);
38093920 }
3810 var child = std.ChildProcess.init(argv.items, arena);
3921 var child = std.ChildProcess.init(argv.items, gpa);
38113922
38123923 child.stdin_behavior = .Inherit;
38133924 child.stdout_behavior = .Inherit;
......@@ -4206,7 +4317,6 @@ pub const usage_build =
42064317
42074318pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
42084319 var color: Color = .auto;
4209 var prominent_compile_errors: bool = false;
42104320
42114321 // We want to release all the locks before executing the child process, so we make a nice
42124322 // big block here to ensure the cleanup gets run when we extract out our argv.
......@@ -4267,8 +4377,6 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
42674377 i += 1;
42684378 override_global_cache_dir = args[i];
42694379 continue;
4270 } else if (mem.eql(u8, arg, "--prominent-compile-errors")) {
4271 prominent_compile_errors = true;
42724380 } else if (mem.eql(u8, arg, "-freference-trace")) {
42734381 try child_argv.append(arg);
42744382 reference_trace = 256;
......@@ -4535,12 +4643,8 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
45354643 .Exited => |code| {
45364644 if (code == 0) return cleanExit();
45374645
4538 if (prominent_compile_errors) {
4539 fatal("the build command failed with exit code {d}", .{code});
4540 } else {
4541 const cmd = try std.mem.join(arena, " ", child_argv);
4542 fatal("the following build command failed with exit code {d}:\n{s}", .{ code, cmd });
4543 }
4646 const cmd = try std.mem.join(arena, " ", child_argv);
4647 fatal("the following build command failed with exit code {d}:\n{s}", .{ code, cmd });
45444648 },
45454649 else => {
45464650 const cmd = try std.mem.join(arena, " ", child_argv);