authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-03-09 21:17:56-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-03-15 10:48:14-07:00
loga333bb91ffcf63fbe377ddc82fd7f1633e269430
tree95ed82003498b27d49e5e865b1e75d1a0fc0c3ff
parent59f5df3af9c31ffef33862c15bcbce6b4227ff1f

zig objcopy: support the compiler protocol

This commit extracts out server code into src/Server.zig and uses it both in the main CLI as well as `zig objcopy`. std.Build.ObjCopyStep now adds `--listen=-` to the CLI for `zig objcopy` and observes the protocol for progress and other kinds of integrations. This fixes the last two test failures of this branch when I run `zig build test` locally.

5 files changed, 177 insertions(+), 97 deletions(-)

CMakeLists.txt+1
...@@ -623,6 +623,7 @@ set(ZIG_STAGE2_SOURCES...@@ -623,6 +623,7 @@ set(ZIG_STAGE2_SOURCES
623 "${CMAKE_SOURCE_DIR}/src/print_targets.zig"623 "${CMAKE_SOURCE_DIR}/src/print_targets.zig"
624 "${CMAKE_SOURCE_DIR}/src/print_zir.zig"624 "${CMAKE_SOURCE_DIR}/src/print_zir.zig"
625 "${CMAKE_SOURCE_DIR}/src/register_manager.zig"625 "${CMAKE_SOURCE_DIR}/src/register_manager.zig"
626 "${CMAKE_SOURCE_DIR}/src/Server.zig"
626 "${CMAKE_SOURCE_DIR}/src/target.zig"627 "${CMAKE_SOURCE_DIR}/src/target.zig"
627 "${CMAKE_SOURCE_DIR}/src/tracy.zig"628 "${CMAKE_SOURCE_DIR}/src/tracy.zig"
628 "${CMAKE_SOURCE_DIR}/src/translate_c.zig"629 "${CMAKE_SOURCE_DIR}/src/translate_c.zig"
lib/std/Build/ObjCopyStep.zig+2
...@@ -113,6 +113,8 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {...@@ -113,6 +113,8 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {
113 };113 };
114114
115 try argv.appendSlice(&.{ full_src_path, full_dest_path });115 try argv.appendSlice(&.{ full_src_path, full_dest_path });
116
117 try argv.append("--listen=-");
116 _ = try step.evalZigProcess(argv.items, prog_node);118 _ = try step.evalZigProcess(argv.items, prog_node);
117119
118 self.output_file.path = full_dest_path;120 self.output_file.path = full_dest_path;
src/Server.zig created+113
...@@ -0,0 +1,113 @@
1in: std.fs.File,
2out: std.fs.File,
3receive_fifo: std.fifo.LinearFifo(u8, .Dynamic),
4
5pub const Options = struct {
6 gpa: Allocator,
7 in: std.fs.File,
8 out: std.fs.File,
9};
10
11pub fn init(options: Options) !Server {
12 var s: Server = .{
13 .in = options.in,
14 .out = options.out,
15 .receive_fifo = std.fifo.LinearFifo(u8, .Dynamic).init(options.gpa),
16 };
17 try s.serveStringMessage(.zig_version, build_options.version);
18 return s;
19}
20
21pub fn deinit(s: *Server) void {
22 s.receive_fifo.deinit();
23 s.* = undefined;
24}
25
26pub fn receiveMessage(s: *Server) !InMessage.Header {
27 const Header = InMessage.Header;
28 const fifo = &s.receive_fifo;
29
30 while (true) {
31 const buf = fifo.readableSlice(0);
32 assert(fifo.readableLength() == buf.len);
33 if (buf.len >= @sizeOf(Header)) {
34 const header = @ptrCast(*align(1) const Header, buf[0..@sizeOf(Header)]);
35 if (header.bytes_len != 0)
36 return error.InvalidClientMessage;
37 const result = header.*;
38 fifo.discard(@sizeOf(Header));
39 return result;
40 }
41
42 const write_buffer = try fifo.writableWithSize(256);
43 const amt = try s.in.read(write_buffer);
44 fifo.update(amt);
45 }
46}
47
48pub fn serveStringMessage(s: *Server, tag: OutMessage.Tag, msg: []const u8) !void {
49 return s.serveMessage(.{
50 .tag = tag,
51 .bytes_len = @intCast(u32, msg.len),
52 }, &.{msg});
53}
54
55pub fn serveMessage(
56 s: *const Server,
57 header: OutMessage.Header,
58 bufs: []const []const u8,
59) !void {
60 var iovecs: [10]std.os.iovec_const = undefined;
61 iovecs[0] = .{
62 .iov_base = @ptrCast([*]const u8, &header),
63 .iov_len = @sizeOf(OutMessage.Header),
64 };
65 for (bufs, iovecs[1 .. bufs.len + 1]) |buf, *iovec| {
66 iovec.* = .{
67 .iov_base = buf.ptr,
68 .iov_len = buf.len,
69 };
70 }
71 try s.out.writevAll(iovecs[0 .. bufs.len + 1]);
72}
73
74pub fn serveEmitBinPath(
75 s: *Server,
76 fs_path: []const u8,
77 header: std.zig.Server.Message.EmitBinPath,
78) !void {
79 try s.serveMessage(.{
80 .tag = .emit_bin_path,
81 .bytes_len = @intCast(u32, fs_path.len + @sizeOf(std.zig.Server.Message.EmitBinPath)),
82 }, &.{
83 std.mem.asBytes(&header),
84 fs_path,
85 });
86}
87
88pub fn serveErrorBundle(s: *Server, error_bundle: std.zig.ErrorBundle) !void {
89 const eb_hdr: std.zig.Server.Message.ErrorBundle = .{
90 .extra_len = @intCast(u32, error_bundle.extra.len),
91 .string_bytes_len = @intCast(u32, error_bundle.string_bytes.len),
92 };
93 const bytes_len = @sizeOf(std.zig.Server.Message.ErrorBundle) +
94 4 * error_bundle.extra.len + error_bundle.string_bytes.len;
95 try s.serveMessage(.{
96 .tag = .error_bundle,
97 .bytes_len = @intCast(u32, bytes_len),
98 }, &.{
99 std.mem.asBytes(&eb_hdr),
100 // TODO: implement @ptrCast between slices changing the length
101 std.mem.sliceAsBytes(error_bundle.extra),
102 error_bundle.string_bytes,
103 });
104}
105
106const OutMessage = std.zig.Server.Message;
107const InMessage = std.zig.Client.Message;
108
109const Server = @This();
110const std = @import("std");
111const build_options = @import("build_options");
112const Allocator = std.mem.Allocator;
113const assert = std.debug.assert;
src/main.zig+18-93
...@@ -26,6 +26,7 @@ const target_util = @import("target.zig");...@@ -26,6 +26,7 @@ const target_util = @import("target.zig");
26const crash_report = @import("crash_report.zig");26const crash_report = @import("crash_report.zig");
27const Module = @import("Module.zig");27const Module = @import("Module.zig");
28const AstGen = @import("AstGen.zig");28const AstGen = @import("AstGen.zig");
29const Server = @import("Server.zig");
2930
30pub const std_options = struct {31pub const std_options = struct {
31 pub const wasiCwd = wasi_cwd;32 pub const wasiCwd = wasi_cwd;
...@@ -3540,11 +3541,14 @@ fn serve(...@@ -3540,11 +3541,14 @@ fn serve(
3540) !void {3541) !void {
3541 const gpa = comp.gpa;3542 const gpa = comp.gpa;
35423543
3543 try serveStringMessage(out, .zig_version, build_options.version);3544 var server = try Server.init(.{
3545 .gpa = gpa,
3546 .in = in,
3547 .out = out,
3548 });
3549 defer server.deinit();
35443550
3545 var child_pid: ?std.ChildProcess.Id = null;3551 var child_pid: ?std.ChildProcess.Id = null;
3546 var receive_fifo = std.fifo.LinearFifo(u8, .Dynamic).init(gpa);
3547 defer receive_fifo.deinit();
35483552
3549 var progress: std.Progress = .{3553 var progress: std.Progress = .{
3550 .terminal = null,3554 .terminal = null,
...@@ -3564,7 +3568,7 @@ fn serve(...@@ -3564,7 +3568,7 @@ fn serve(
3564 main_progress_node.context = &progress;3568 main_progress_node.context = &progress;
35653569
3566 while (true) {3570 while (true) {
3567 const hdr = try receiveMessage(in, &receive_fifo);3571 const hdr = try server.receiveMessage();
35683572
3569 switch (hdr.tag) {3573 switch (hdr.tag) {
3570 .exit => {3574 .exit => {
...@@ -3580,7 +3584,7 @@ fn serve(...@@ -3580,7 +3584,7 @@ fn serve(
3580 const arena = arena_instance.allocator();3584 const arena = arena_instance.allocator();
3581 var output: TranslateCOutput = undefined;3585 var output: TranslateCOutput = undefined;
3582 try cmdTranslateC(comp, arena, &output);3586 try cmdTranslateC(comp, arena, &output);
3583 try serveEmitBinPath(out, output.path, .{3587 try server.serveEmitBinPath(output.path, .{
3584 .flags = .{ .cache_hit = output.cache_hit },3588 .flags = .{ .cache_hit = output.cache_hit },
3585 });3589 });
3586 continue;3590 continue;
...@@ -3594,7 +3598,7 @@ fn serve(...@@ -3594,7 +3598,7 @@ fn serve(
3594 var reset: std.Thread.ResetEvent = .{};3598 var reset: std.Thread.ResetEvent = .{};
35953599
3596 var progress_thread = try std.Thread.spawn(.{}, progressThread, .{3600 var progress_thread = try std.Thread.spawn(.{}, progressThread, .{
3597 &progress, out, &reset,3601 &progress, &server, &reset,
3598 });3602 });
3599 defer {3603 defer {
3600 reset.set();3604 reset.set();
...@@ -3605,7 +3609,7 @@ fn serve(...@@ -3605,7 +3609,7 @@ fn serve(
3605 }3609 }
36063610
3607 try comp.makeBinFileExecutable();3611 try comp.makeBinFileExecutable();
3608 try serveUpdateResults(out, comp);3612 try serveUpdateResults(&server, comp);
3609 },3613 },
3610 .run => {3614 .run => {
3611 if (child_pid != null) {3615 if (child_pid != null) {
...@@ -3632,14 +3636,14 @@ fn serve(...@@ -3632,14 +3636,14 @@ fn serve(
3632 assert(main_progress_node.recently_updated_child == null);3636 assert(main_progress_node.recently_updated_child == null);
3633 if (child_pid) |pid| {3637 if (child_pid) |pid| {
3634 try comp.hotCodeSwap(main_progress_node, pid);3638 try comp.hotCodeSwap(main_progress_node, pid);
3635 try serveUpdateResults(out, comp);3639 try serveUpdateResults(&server, comp);
3636 } else {3640 } else {
3637 if (comp.bin_file.options.output_mode == .Exe) {3641 if (comp.bin_file.options.output_mode == .Exe) {
3638 try comp.makeBinFileWritable();3642 try comp.makeBinFileWritable();
3639 }3643 }
3640 try comp.update(main_progress_node);3644 try comp.update(main_progress_node);
3641 try comp.makeBinFileExecutable();3645 try comp.makeBinFileExecutable();
3642 try serveUpdateResults(out, comp);3646 try serveUpdateResults(&server, comp);
36433647
3644 child_pid = try runOrTestHotSwap(3648 child_pid = try runOrTestHotSwap(
3645 comp,3649 comp,
...@@ -3659,7 +3663,7 @@ fn serve(...@@ -3659,7 +3663,7 @@ fn serve(
3659 }3663 }
3660}3664}
36613665
3662fn progressThread(progress: *std.Progress, out: fs.File, reset: *std.Thread.ResetEvent) void {3666fn progressThread(progress: *std.Progress, server: *const Server, reset: *std.Thread.ResetEvent) void {
3663 while (true) {3667 while (true) {
3664 if (reset.timedWait(500 * std.time.ns_per_ms)) |_| {3668 if (reset.timedWait(500 * std.time.ns_per_ms)) |_| {
3665 // The Compilation update has completed.3669 // The Compilation update has completed.
...@@ -3705,7 +3709,7 @@ fn progressThread(progress: *std.Progress, out: fs.File, reset: *std.Thread.Rese...@@ -3705,7 +3709,7 @@ fn progressThread(progress: *std.Progress, out: fs.File, reset: *std.Thread.Rese
37053709
3706 const progress_string = buf.slice();3710 const progress_string = buf.slice();
37073711
3708 serveMessage(out, .{3712 server.serveMessage(.{
3709 .tag = .progress,3713 .tag = .progress,
3710 .bytes_len = @intCast(u32, progress_string.len),3714 .bytes_len = @intCast(u32, progress_string.len),
3711 }, &.{3715 }, &.{
...@@ -3716,100 +3720,21 @@ fn progressThread(progress: *std.Progress, out: fs.File, reset: *std.Thread.Rese...@@ -3716,100 +3720,21 @@ fn progressThread(progress: *std.Progress, out: fs.File, reset: *std.Thread.Rese
3716 }3720 }
3717}3721}
37183722
3719fn serveMessage(3723fn serveUpdateResults(s: *Server, comp: *Compilation) !void {
3720 out: fs.File,
3721 header: std.zig.Server.Message.Header,
3722 bufs: []const []const u8,
3723) !void {
3724 var iovecs: [10]std.os.iovec_const = undefined;
3725 iovecs[0] = .{
3726 .iov_base = @ptrCast([*]const u8, &header),
3727 .iov_len = @sizeOf(std.zig.Server.Message.Header),
3728 };
3729 for (bufs, iovecs[1 .. bufs.len + 1]) |buf, *iovec| {
3730 iovec.* = .{
3731 .iov_base = buf.ptr,
3732 .iov_len = buf.len,
3733 };
3734 }
3735 try out.writevAll(iovecs[0 .. bufs.len + 1]);
3736}
3737
3738fn serveErrorBundle(out: fs.File, error_bundle: std.zig.ErrorBundle) !void {
3739 const eb_hdr: std.zig.Server.Message.ErrorBundle = .{
3740 .extra_len = @intCast(u32, error_bundle.extra.len),
3741 .string_bytes_len = @intCast(u32, error_bundle.string_bytes.len),
3742 };
3743 const bytes_len = @sizeOf(std.zig.Server.Message.ErrorBundle) +
3744 4 * error_bundle.extra.len + error_bundle.string_bytes.len;
3745 try serveMessage(out, .{
3746 .tag = .error_bundle,
3747 .bytes_len = @intCast(u32, bytes_len),
3748 }, &.{
3749 std.mem.asBytes(&eb_hdr),
3750 // TODO: implement @ptrCast between slices changing the length
3751 std.mem.sliceAsBytes(error_bundle.extra),
3752 error_bundle.string_bytes,
3753 });
3754}
3755
3756fn serveUpdateResults(out: fs.File, comp: *Compilation) !void {
3757 const gpa = comp.gpa;3724 const gpa = comp.gpa;
3758 var error_bundle = try comp.getAllErrorsAlloc();3725 var error_bundle = try comp.getAllErrorsAlloc();
3759 defer error_bundle.deinit(gpa);3726 defer error_bundle.deinit(gpa);
3760 if (error_bundle.errorMessageCount() > 0) {3727 if (error_bundle.errorMessageCount() > 0) {
3761 try serveErrorBundle(out, error_bundle);3728 try s.serveErrorBundle(error_bundle);
3762 } else if (comp.bin_file.options.emit) |emit| {3729 } else if (comp.bin_file.options.emit) |emit| {
3763 const full_path = try emit.directory.join(gpa, &.{emit.sub_path});3730 const full_path = try emit.directory.join(gpa, &.{emit.sub_path});
3764 defer gpa.free(full_path);3731 defer gpa.free(full_path);
3765 try serveEmitBinPath(out, full_path, .{3732 try s.serveEmitBinPath(full_path, .{
3766 .flags = .{ .cache_hit = comp.last_update_was_cache_hit },3733 .flags = .{ .cache_hit = comp.last_update_was_cache_hit },
3767 });3734 });
3768 }3735 }
3769}3736}
37703737
3771fn serveEmitBinPath(
3772 out: fs.File,
3773 fs_path: []const u8,
3774 header: std.zig.Server.Message.EmitBinPath,
3775) !void {
3776 try serveMessage(out, .{
3777 .tag = .emit_bin_path,
3778 .bytes_len = @intCast(u32, fs_path.len + @sizeOf(std.zig.Server.Message.EmitBinPath)),
3779 }, &.{
3780 std.mem.asBytes(&header),
3781 fs_path,
3782 });
3783}
3784
3785fn serveStringMessage(out: fs.File, tag: std.zig.Server.Message.Tag, s: []const u8) !void {
3786 try serveMessage(out, .{
3787 .tag = tag,
3788 .bytes_len = @intCast(u32, s.len),
3789 }, &.{s});
3790}
3791
3792fn receiveMessage(in: fs.File, fifo: *std.fifo.LinearFifo(u8, .Dynamic)) !std.zig.Client.Message.Header {
3793 const Header = std.zig.Client.Message.Header;
3794
3795 while (true) {
3796 const buf = fifo.readableSlice(0);
3797 assert(fifo.readableLength() == buf.len);
3798 if (buf.len >= @sizeOf(Header)) {
3799 const header = @ptrCast(*align(1) const Header, buf[0..@sizeOf(Header)]);
3800 if (header.bytes_len != 0)
3801 return error.InvalidClientMessage;
3802 const result = header.*;
3803 fifo.discard(@sizeOf(Header));
3804 return result;
3805 }
3806
3807 const write_buffer = try fifo.writableWithSize(256);
3808 const amt = try in.read(write_buffer);
3809 fifo.update(amt);
3810 }
3811}
3812
3813const ModuleDepIterator = struct {3738const ModuleDepIterator = struct {
3814 split: mem.SplitIterator(u8),3739 split: mem.SplitIterator(u8),
38153740
src/objcopy.zig+43-4
...@@ -4,22 +4,25 @@ const fs = std.fs;...@@ -4,22 +4,25 @@ const fs = std.fs;
4const elf = std.elf;4const elf = std.elf;
5const Allocator = std.mem.Allocator;5const Allocator = std.mem.Allocator;
6const File = std.fs.File;6const File = std.fs.File;
7const assert = std.debug.assert;
8
7const main = @import("main.zig");9const main = @import("main.zig");
8const fatal = main.fatal;10const fatal = main.fatal;
9const cleanExit = main.cleanExit;11const cleanExit = main.cleanExit;
12const Server = @import("Server.zig");
1013
11pub fn cmdObjCopy(14pub fn cmdObjCopy(
12 gpa: Allocator,15 gpa: Allocator,
13 arena: Allocator,16 arena: Allocator,
14 args: []const []const u8,17 args: []const []const u8,
15) !void {18) !void {
16 _ = gpa;
17 var i: usize = 0;19 var i: usize = 0;
18 var opt_out_fmt: ?std.Target.ObjectFormat = null;20 var opt_out_fmt: ?std.Target.ObjectFormat = null;
19 var opt_input: ?[]const u8 = null;21 var opt_input: ?[]const u8 = null;
20 var opt_output: ?[]const u8 = null;22 var opt_output: ?[]const u8 = null;
21 var only_section: ?[]const u8 = null;23 var only_section: ?[]const u8 = null;
22 var pad_to: ?u64 = null;24 var pad_to: ?u64 = null;
25 var listen = false;
23 while (i < args.len) : (i += 1) {26 while (i < args.len) : (i += 1) {
24 const arg = args[i];27 const arg = args[i];
25 if (!mem.startsWith(u8, arg, "-")) {28 if (!mem.startsWith(u8, arg, "-")) {
...@@ -54,6 +57,8 @@ pub fn cmdObjCopy(...@@ -54,6 +57,8 @@ pub fn cmdObjCopy(
54 i += 1;57 i += 1;
55 if (i >= args.len) fatal("expected another argument after '{s}'", .{arg});58 if (i >= args.len) fatal("expected another argument after '{s}'", .{arg});
56 only_section = args[i];59 only_section = args[i];
60 } else if (mem.eql(u8, arg, "--listen=-")) {
61 listen = true;
57 } else if (mem.startsWith(u8, arg, "--only-section=")) {62 } else if (mem.startsWith(u8, arg, "--only-section=")) {
58 only_section = arg["--output-target=".len..];63 only_section = arg["--output-target=".len..];
59 } else if (mem.eql(u8, arg, "--pad-to")) {64 } else if (mem.eql(u8, arg, "--pad-to")) {
...@@ -102,10 +107,44 @@ pub fn cmdObjCopy(...@@ -102,10 +107,44 @@ pub fn cmdObjCopy(
102 .only_section = only_section,107 .only_section = only_section,
103 .pad_to = pad_to,108 .pad_to = pad_to,
104 });109 });
105 return cleanExit();
106 },110 },
107 else => fatal("unsupported output object format: {s}", .{@tagName(out_fmt)}),111 else => fatal("unsupported output object format: {s}", .{@tagName(out_fmt)}),
108 }112 }
113
114 if (listen) {
115 var server = try Server.init(.{
116 .gpa = gpa,
117 .in = std.io.getStdIn(),
118 .out = std.io.getStdOut(),
119 });
120 defer server.deinit();
121
122 var seen_update = false;
123 while (true) {
124 const hdr = try server.receiveMessage();
125 switch (hdr.tag) {
126 .exit => {
127 return cleanExit();
128 },
129 .update => {
130 if (seen_update) {
131 std.debug.print("zig objcopy only supports 1 update for now\n", .{});
132 std.process.exit(1);
133 }
134 seen_update = true;
135
136 try server.serveEmitBinPath(output, .{
137 .flags = .{ .cache_hit = false },
138 });
139 },
140 else => {
141 std.debug.print("unsupported message: {s}", .{@tagName(hdr.tag)});
142 std.process.exit(1);
143 },
144 }
145 }
146 }
147 return cleanExit();
109}148}
110149
111const usage =150const usage =
...@@ -417,7 +456,7 @@ const HexWriter = struct {...@@ -417,7 +456,7 @@ const HexWriter = struct {
417 }456 }
418457
419 fn Address(address: u32) Record {458 fn Address(address: u32) Record {
420 std.debug.assert(address > 0xFFFF);459 assert(address > 0xFFFF);
421 const segment = @intCast(u16, address / 0x10000);460 const segment = @intCast(u16, address / 0x10000);
422 if (address > 0xFFFFF) {461 if (address > 0xFFFFF) {
423 return Record{462 return Record{
...@@ -460,7 +499,7 @@ const HexWriter = struct {...@@ -460,7 +499,7 @@ const HexWriter = struct {
460 const BUFSIZE = 1 + (1 + 2 + 1 + MAX_PAYLOAD_LEN + 1) * 2 + linesep.len;499 const BUFSIZE = 1 + (1 + 2 + 1 + MAX_PAYLOAD_LEN + 1) * 2 + linesep.len;
461 var outbuf: [BUFSIZE]u8 = undefined;500 var outbuf: [BUFSIZE]u8 = undefined;
462 const payload_bytes = self.getPayloadBytes();501 const payload_bytes = self.getPayloadBytes();
463 std.debug.assert(payload_bytes.len <= MAX_PAYLOAD_LEN);502 assert(payload_bytes.len <= MAX_PAYLOAD_LEN);
464503
465 const line = try std.fmt.bufPrint(&outbuf, ":{0X:0>2}{1X:0>4}{2X:0>2}{3s}{4X:0>2}" ++ linesep, .{504 const line = try std.fmt.bufPrint(&outbuf, ":{0X:0>2}{1X:0>4}{2X:0>2}{3s}{4X:0>2}" ++ linesep, .{
466 @intCast(u8, payload_bytes.len),505 @intCast(u8, payload_bytes.len),