authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-03-09 21:17:56-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-03-15 10:48:14-07:00
loga333bb91ffcf63fbe377ddc82fd7f1633e269430
tree95ed82003498b27d49e5e865b1e75d1a0fc0c3ff
parent59f5df3af9c31ffef33862c15bcbce6b4227ff1f

zig objcopy: support the compiler protocol

This commit extracts out server code into src/Server.zig and uses it both in the main CLI as well as `zig objcopy`. std.Build.ObjCopyStep now adds `--listen=-` to the CLI for `zig objcopy` and observes the protocol for progress and other kinds of integrations. This fixes the last two test failures of this branch when I run `zig build test` locally.

5 files changed, 177 insertions(+), 97 deletions(-)

CMakeLists.txt+1
......@@ -623,6 +623,7 @@ set(ZIG_STAGE2_SOURCES
623623 "${CMAKE_SOURCE_DIR}/src/print_targets.zig"
624624 "${CMAKE_SOURCE_DIR}/src/print_zir.zig"
625625 "${CMAKE_SOURCE_DIR}/src/register_manager.zig"
626 "${CMAKE_SOURCE_DIR}/src/Server.zig"
626627 "${CMAKE_SOURCE_DIR}/src/target.zig"
627628 "${CMAKE_SOURCE_DIR}/src/tracy.zig"
628629 "${CMAKE_SOURCE_DIR}/src/translate_c.zig"
lib/std/Build/ObjCopyStep.zig+2
......@@ -113,6 +113,8 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {
113113 };
114114
115115 try argv.appendSlice(&.{ full_src_path, full_dest_path });
116
117 try argv.append("--listen=-");
116118 _ = try step.evalZigProcess(argv.items, prog_node);
117119
118120 self.output_file.path = full_dest_path;
src/Server.zig created+113
......@@ -0,0 +1,113 @@
1in: std.fs.File,
2out: std.fs.File,
3receive_fifo: std.fifo.LinearFifo(u8, .Dynamic),
4
5pub const Options = struct {
6 gpa: Allocator,
7 in: std.fs.File,
8 out: std.fs.File,
9};
10
11pub fn init(options: Options) !Server {
12 var s: Server = .{
13 .in = options.in,
14 .out = options.out,
15 .receive_fifo = std.fifo.LinearFifo(u8, .Dynamic).init(options.gpa),
16 };
17 try s.serveStringMessage(.zig_version, build_options.version);
18 return s;
19}
20
21pub fn deinit(s: *Server) void {
22 s.receive_fifo.deinit();
23 s.* = undefined;
24}
25
26pub fn receiveMessage(s: *Server) !InMessage.Header {
27 const Header = InMessage.Header;
28 const fifo = &s.receive_fifo;
29
30 while (true) {
31 const buf = fifo.readableSlice(0);
32 assert(fifo.readableLength() == buf.len);
33 if (buf.len >= @sizeOf(Header)) {
34 const header = @ptrCast(*align(1) const Header, buf[0..@sizeOf(Header)]);
35 if (header.bytes_len != 0)
36 return error.InvalidClientMessage;
37 const result = header.*;
38 fifo.discard(@sizeOf(Header));
39 return result;
40 }
41
42 const write_buffer = try fifo.writableWithSize(256);
43 const amt = try s.in.read(write_buffer);
44 fifo.update(amt);
45 }
46}
47
48pub fn serveStringMessage(s: *Server, tag: OutMessage.Tag, msg: []const u8) !void {
49 return s.serveMessage(.{
50 .tag = tag,
51 .bytes_len = @intCast(u32, msg.len),
52 }, &.{msg});
53}
54
55pub fn serveMessage(
56 s: *const Server,
57 header: OutMessage.Header,
58 bufs: []const []const u8,
59) !void {
60 var iovecs: [10]std.os.iovec_const = undefined;
61 iovecs[0] = .{
62 .iov_base = @ptrCast([*]const u8, &header),
63 .iov_len = @sizeOf(OutMessage.Header),
64 };
65 for (bufs, iovecs[1 .. bufs.len + 1]) |buf, *iovec| {
66 iovec.* = .{
67 .iov_base = buf.ptr,
68 .iov_len = buf.len,
69 };
70 }
71 try s.out.writevAll(iovecs[0 .. bufs.len + 1]);
72}
73
74pub fn serveEmitBinPath(
75 s: *Server,
76 fs_path: []const u8,
77 header: std.zig.Server.Message.EmitBinPath,
78) !void {
79 try s.serveMessage(.{
80 .tag = .emit_bin_path,
81 .bytes_len = @intCast(u32, fs_path.len + @sizeOf(std.zig.Server.Message.EmitBinPath)),
82 }, &.{
83 std.mem.asBytes(&header),
84 fs_path,
85 });
86}
87
88pub fn serveErrorBundle(s: *Server, error_bundle: std.zig.ErrorBundle) !void {
89 const eb_hdr: std.zig.Server.Message.ErrorBundle = .{
90 .extra_len = @intCast(u32, error_bundle.extra.len),
91 .string_bytes_len = @intCast(u32, error_bundle.string_bytes.len),
92 };
93 const bytes_len = @sizeOf(std.zig.Server.Message.ErrorBundle) +
94 4 * error_bundle.extra.len + error_bundle.string_bytes.len;
95 try s.serveMessage(.{
96 .tag = .error_bundle,
97 .bytes_len = @intCast(u32, bytes_len),
98 }, &.{
99 std.mem.asBytes(&eb_hdr),
100 // TODO: implement @ptrCast between slices changing the length
101 std.mem.sliceAsBytes(error_bundle.extra),
102 error_bundle.string_bytes,
103 });
104}
105
106const OutMessage = std.zig.Server.Message;
107const InMessage = std.zig.Client.Message;
108
109const Server = @This();
110const std = @import("std");
111const build_options = @import("build_options");
112const Allocator = std.mem.Allocator;
113const assert = std.debug.assert;
src/main.zig+18-93
......@@ -26,6 +26,7 @@ const target_util = @import("target.zig");
2626const crash_report = @import("crash_report.zig");
2727const Module = @import("Module.zig");
2828const AstGen = @import("AstGen.zig");
29const Server = @import("Server.zig");
2930
3031pub const std_options = struct {
3132 pub const wasiCwd = wasi_cwd;
......@@ -3540,11 +3541,14 @@ fn serve(
35403541) !void {
35413542 const gpa = comp.gpa;
35423543
3543 try serveStringMessage(out, .zig_version, build_options.version);
3544 var server = try Server.init(.{
3545 .gpa = gpa,
3546 .in = in,
3547 .out = out,
3548 });
3549 defer server.deinit();
35443550
35453551 var child_pid: ?std.ChildProcess.Id = null;
3546 var receive_fifo = std.fifo.LinearFifo(u8, .Dynamic).init(gpa);
3547 defer receive_fifo.deinit();
35483552
35493553 var progress: std.Progress = .{
35503554 .terminal = null,
......@@ -3564,7 +3568,7 @@ fn serve(
35643568 main_progress_node.context = &progress;
35653569
35663570 while (true) {
3567 const hdr = try receiveMessage(in, &receive_fifo);
3571 const hdr = try server.receiveMessage();
35683572
35693573 switch (hdr.tag) {
35703574 .exit => {
......@@ -3580,7 +3584,7 @@ fn serve(
35803584 const arena = arena_instance.allocator();
35813585 var output: TranslateCOutput = undefined;
35823586 try cmdTranslateC(comp, arena, &output);
3583 try serveEmitBinPath(out, output.path, .{
3587 try server.serveEmitBinPath(output.path, .{
35843588 .flags = .{ .cache_hit = output.cache_hit },
35853589 });
35863590 continue;
......@@ -3594,7 +3598,7 @@ fn serve(
35943598 var reset: std.Thread.ResetEvent = .{};
35953599
35963600 var progress_thread = try std.Thread.spawn(.{}, progressThread, .{
3597 &progress, out, &reset,
3601 &progress, &server, &reset,
35983602 });
35993603 defer {
36003604 reset.set();
......@@ -3605,7 +3609,7 @@ fn serve(
36053609 }
36063610
36073611 try comp.makeBinFileExecutable();
3608 try serveUpdateResults(out, comp);
3612 try serveUpdateResults(&server, comp);
36093613 },
36103614 .run => {
36113615 if (child_pid != null) {
......@@ -3632,14 +3636,14 @@ fn serve(
36323636 assert(main_progress_node.recently_updated_child == null);
36333637 if (child_pid) |pid| {
36343638 try comp.hotCodeSwap(main_progress_node, pid);
3635 try serveUpdateResults(out, comp);
3639 try serveUpdateResults(&server, comp);
36363640 } else {
36373641 if (comp.bin_file.options.output_mode == .Exe) {
36383642 try comp.makeBinFileWritable();
36393643 }
36403644 try comp.update(main_progress_node);
36413645 try comp.makeBinFileExecutable();
3642 try serveUpdateResults(out, comp);
3646 try serveUpdateResults(&server, comp);
36433647
36443648 child_pid = try runOrTestHotSwap(
36453649 comp,
......@@ -3659,7 +3663,7 @@ fn serve(
36593663 }
36603664}
36613665
3662fn progressThread(progress: *std.Progress, out: fs.File, reset: *std.Thread.ResetEvent) void {
3666fn progressThread(progress: *std.Progress, server: *const Server, reset: *std.Thread.ResetEvent) void {
36633667 while (true) {
36643668 if (reset.timedWait(500 * std.time.ns_per_ms)) |_| {
36653669 // The Compilation update has completed.
......@@ -3705,7 +3709,7 @@ fn progressThread(progress: *std.Progress, out: fs.File, reset: *std.Thread.Rese
37053709
37063710 const progress_string = buf.slice();
37073711
3708 serveMessage(out, .{
3712 server.serveMessage(.{
37093713 .tag = .progress,
37103714 .bytes_len = @intCast(u32, progress_string.len),
37113715 }, &.{
......@@ -3716,100 +3720,21 @@ fn progressThread(progress: *std.Progress, out: fs.File, reset: *std.Thread.Rese
37163720 }
37173721}
37183722
3719fn serveMessage(
3720 out: fs.File,
3721 header: std.zig.Server.Message.Header,
3722 bufs: []const []const u8,
3723) !void {
3724 var iovecs: [10]std.os.iovec_const = undefined;
3725 iovecs[0] = .{
3726 .iov_base = @ptrCast([*]const u8, &header),
3727 .iov_len = @sizeOf(std.zig.Server.Message.Header),
3728 };
3729 for (bufs, iovecs[1 .. bufs.len + 1]) |buf, *iovec| {
3730 iovec.* = .{
3731 .iov_base = buf.ptr,
3732 .iov_len = buf.len,
3733 };
3734 }
3735 try out.writevAll(iovecs[0 .. bufs.len + 1]);
3736}
3737
3738fn serveErrorBundle(out: fs.File, error_bundle: std.zig.ErrorBundle) !void {
3739 const eb_hdr: std.zig.Server.Message.ErrorBundle = .{
3740 .extra_len = @intCast(u32, error_bundle.extra.len),
3741 .string_bytes_len = @intCast(u32, error_bundle.string_bytes.len),
3742 };
3743 const bytes_len = @sizeOf(std.zig.Server.Message.ErrorBundle) +
3744 4 * error_bundle.extra.len + error_bundle.string_bytes.len;
3745 try serveMessage(out, .{
3746 .tag = .error_bundle,
3747 .bytes_len = @intCast(u32, bytes_len),
3748 }, &.{
3749 std.mem.asBytes(&eb_hdr),
3750 // TODO: implement @ptrCast between slices changing the length
3751 std.mem.sliceAsBytes(error_bundle.extra),
3752 error_bundle.string_bytes,
3753 });
3754}
3755
3756fn serveUpdateResults(out: fs.File, comp: *Compilation) !void {
3723fn serveUpdateResults(s: *Server, comp: *Compilation) !void {
37573724 const gpa = comp.gpa;
37583725 var error_bundle = try comp.getAllErrorsAlloc();
37593726 defer error_bundle.deinit(gpa);
37603727 if (error_bundle.errorMessageCount() > 0) {
3761 try serveErrorBundle(out, error_bundle);
3728 try s.serveErrorBundle(error_bundle);
37623729 } else if (comp.bin_file.options.emit) |emit| {
37633730 const full_path = try emit.directory.join(gpa, &.{emit.sub_path});
37643731 defer gpa.free(full_path);
3765 try serveEmitBinPath(out, full_path, .{
3732 try s.serveEmitBinPath(full_path, .{
37663733 .flags = .{ .cache_hit = comp.last_update_was_cache_hit },
37673734 });
37683735 }
37693736}
37703737
3771fn serveEmitBinPath(
3772 out: fs.File,
3773 fs_path: []const u8,
3774 header: std.zig.Server.Message.EmitBinPath,
3775) !void {
3776 try serveMessage(out, .{
3777 .tag = .emit_bin_path,
3778 .bytes_len = @intCast(u32, fs_path.len + @sizeOf(std.zig.Server.Message.EmitBinPath)),
3779 }, &.{
3780 std.mem.asBytes(&header),
3781 fs_path,
3782 });
3783}
3784
3785fn serveStringMessage(out: fs.File, tag: std.zig.Server.Message.Tag, s: []const u8) !void {
3786 try serveMessage(out, .{
3787 .tag = tag,
3788 .bytes_len = @intCast(u32, s.len),
3789 }, &.{s});
3790}
3791
3792fn receiveMessage(in: fs.File, fifo: *std.fifo.LinearFifo(u8, .Dynamic)) !std.zig.Client.Message.Header {
3793 const Header = std.zig.Client.Message.Header;
3794
3795 while (true) {
3796 const buf = fifo.readableSlice(0);
3797 assert(fifo.readableLength() == buf.len);
3798 if (buf.len >= @sizeOf(Header)) {
3799 const header = @ptrCast(*align(1) const Header, buf[0..@sizeOf(Header)]);
3800 if (header.bytes_len != 0)
3801 return error.InvalidClientMessage;
3802 const result = header.*;
3803 fifo.discard(@sizeOf(Header));
3804 return result;
3805 }
3806
3807 const write_buffer = try fifo.writableWithSize(256);
3808 const amt = try in.read(write_buffer);
3809 fifo.update(amt);
3810 }
3811}
3812
38133738const ModuleDepIterator = struct {
38143739 split: mem.SplitIterator(u8),
38153740
src/objcopy.zig+43-4
......@@ -4,22 +4,25 @@ const fs = std.fs;
44const elf = std.elf;
55const Allocator = std.mem.Allocator;
66const File = std.fs.File;
7const assert = std.debug.assert;
8
79const main = @import("main.zig");
810const fatal = main.fatal;
911const cleanExit = main.cleanExit;
12const Server = @import("Server.zig");
1013
1114pub fn cmdObjCopy(
1215 gpa: Allocator,
1316 arena: Allocator,
1417 args: []const []const u8,
1518) !void {
16 _ = gpa;
1719 var i: usize = 0;
1820 var opt_out_fmt: ?std.Target.ObjectFormat = null;
1921 var opt_input: ?[]const u8 = null;
2022 var opt_output: ?[]const u8 = null;
2123 var only_section: ?[]const u8 = null;
2224 var pad_to: ?u64 = null;
25 var listen = false;
2326 while (i < args.len) : (i += 1) {
2427 const arg = args[i];
2528 if (!mem.startsWith(u8, arg, "-")) {
......@@ -54,6 +57,8 @@ pub fn cmdObjCopy(
5457 i += 1;
5558 if (i >= args.len) fatal("expected another argument after '{s}'", .{arg});
5659 only_section = args[i];
60 } else if (mem.eql(u8, arg, "--listen=-")) {
61 listen = true;
5762 } else if (mem.startsWith(u8, arg, "--only-section=")) {
5863 only_section = arg["--output-target=".len..];
5964 } else if (mem.eql(u8, arg, "--pad-to")) {
......@@ -102,10 +107,44 @@ pub fn cmdObjCopy(
102107 .only_section = only_section,
103108 .pad_to = pad_to,
104109 });
105 return cleanExit();
106110 },
107111 else => fatal("unsupported output object format: {s}", .{@tagName(out_fmt)}),
108112 }
113
114 if (listen) {
115 var server = try Server.init(.{
116 .gpa = gpa,
117 .in = std.io.getStdIn(),
118 .out = std.io.getStdOut(),
119 });
120 defer server.deinit();
121
122 var seen_update = false;
123 while (true) {
124 const hdr = try server.receiveMessage();
125 switch (hdr.tag) {
126 .exit => {
127 return cleanExit();
128 },
129 .update => {
130 if (seen_update) {
131 std.debug.print("zig objcopy only supports 1 update for now\n", .{});
132 std.process.exit(1);
133 }
134 seen_update = true;
135
136 try server.serveEmitBinPath(output, .{
137 .flags = .{ .cache_hit = false },
138 });
139 },
140 else => {
141 std.debug.print("unsupported message: {s}", .{@tagName(hdr.tag)});
142 std.process.exit(1);
143 },
144 }
145 }
146 }
147 return cleanExit();
109148}
110149
111150const usage =
......@@ -417,7 +456,7 @@ const HexWriter = struct {
417456 }
418457
419458 fn Address(address: u32) Record {
420 std.debug.assert(address > 0xFFFF);
459 assert(address > 0xFFFF);
421460 const segment = @intCast(u16, address / 0x10000);
422461 if (address > 0xFFFFF) {
423462 return Record{
......@@ -460,7 +499,7 @@ const HexWriter = struct {
460499 const BUFSIZE = 1 + (1 + 2 + 1 + MAX_PAYLOAD_LEN + 1) * 2 + linesep.len;
461500 var outbuf: [BUFSIZE]u8 = undefined;
462501 const payload_bytes = self.getPayloadBytes();
463 std.debug.assert(payload_bytes.len <= MAX_PAYLOAD_LEN);
502 assert(payload_bytes.len <= MAX_PAYLOAD_LEN);
464503
465504 const line = try std.fmt.bufPrint(&outbuf, ":{0X:0>2}{1X:0>4}{2X:0>2}{3s}{4X:0>2}" ++ linesep, .{
466505 @intCast(u8, payload_bytes.len),