authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-08-22 08:38:41+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-08-22 08:38:41+02:00
log6c020cdb767192757b6c4b43e2f14c5394760431
tree130b47a125baeff98040a4eb819d90d4075086cf
parent9cf667a21bfef3660183ed67baacb5b32c0f06e2
parentd4c56804dfd3f53de1f0950fac9478b096a2a0b4
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #12557 from Luukdegram/wasm-archive

wasm-linker: Improve archive linking

8 files changed, 139 insertions(+), 63 deletions(-)

lib/std/build.zig+2-3
...@@ -1898,11 +1898,10 @@ pub const LibExeObjStep = struct {...@@ -1898,11 +1898,10 @@ pub const LibExeObjStep = struct {
1898 pub fn runEmulatable(exe: *LibExeObjStep) *EmulatableRunStep {1898 pub fn runEmulatable(exe: *LibExeObjStep) *EmulatableRunStep {
1899 assert(exe.kind == .exe or exe.kind == .test_exe);1899 assert(exe.kind == .exe or exe.kind == .test_exe);
19001900
1901 const run_step = EmulatableRunStep.create(exe.builder.fmt("run {s}", .{exe.step.name}), exe);1901 const run_step = EmulatableRunStep.create(exe.builder, exe.builder.fmt("run {s}", .{exe.step.name}), exe);
1902 if (exe.vcpkg_bin_path) |path| {1902 if (exe.vcpkg_bin_path) |path| {
1903 run_step.addPathDir(path);1903 RunStep.addPathDirInternal(&run_step.step, exe.builder, path);
1904 }1904 }
1905
1906 return run_step;1905 return run_step;
1907 }1906 }
19081907
lib/std/build/RunStep.zig+1-1
...@@ -101,7 +101,7 @@ pub fn addPathDir(self: *RunStep, search_path: []const u8) void {...@@ -101,7 +101,7 @@ pub fn addPathDir(self: *RunStep, search_path: []const u8) void {
101}101}
102102
103/// For internal use only, users of `RunStep` should use `addPathDir` directly.103/// For internal use only, users of `RunStep` should use `addPathDir` directly.
104fn addPathDirInternal(step: *Step, builder: *Builder, search_path: []const u8) void {104pub fn addPathDirInternal(step: *Step, builder: *Builder, search_path: []const u8) void {
105 const env_map = getEnvMapInternal(step, builder.allocator);105 const env_map = getEnvMapInternal(step, builder.allocator);
106106
107 const key = "PATH";107 const key = "PATH";
src/link/Wasm.zig+8-5
...@@ -378,7 +378,7 @@ fn parseObjectFile(self: *Wasm, path: []const u8) !bool {...@@ -378,7 +378,7 @@ fn parseObjectFile(self: *Wasm, path: []const u8) !bool {
378 const file = try fs.cwd().openFile(path, .{});378 const file = try fs.cwd().openFile(path, .{});
379 errdefer file.close();379 errdefer file.close();
380380
381 var object = Object.create(self.base.allocator, file, path) catch |err| switch (err) {381 var object = Object.create(self.base.allocator, file, path, null) catch |err| switch (err) {
382 error.InvalidMagicByte, error.NotObjectFile => return false,382 error.InvalidMagicByte, error.NotObjectFile => return false,
383 else => |e| return e,383 else => |e| return e,
384 };384 };
...@@ -595,8 +595,8 @@ fn resolveSymbolsInArchives(self: *Wasm) !void {...@@ -595,8 +595,8 @@ fn resolveSymbolsInArchives(self: *Wasm) !void {
595 // Parse object and and resolve symbols again before we check remaining595 // Parse object and and resolve symbols again before we check remaining
596 // undefined symbols.596 // undefined symbols.
597 const object_file_index = @intCast(u16, self.objects.items.len);597 const object_file_index = @intCast(u16, self.objects.items.len);
598 const object = try self.objects.addOne(self.base.allocator);598 var object = try archive.parseObject(self.base.allocator, offset.items[0]);
599 object.* = try archive.parseObject(self.base.allocator, offset.items[0]);599 try self.objects.append(self.base.allocator, object);
600 try self.resolveSymbolsInObject(object_file_index);600 try self.resolveSymbolsInObject(object_file_index);
601601
602 // continue loop for any remaining undefined symbols that still exist602 // continue loop for any remaining undefined symbols that still exist
...@@ -860,7 +860,8 @@ fn getGlobalType(self: *const Wasm, loc: SymbolLoc) wasm.GlobalType {...@@ -860,7 +860,8 @@ fn getGlobalType(self: *const Wasm, loc: SymbolLoc) wasm.GlobalType {
860 if (is_undefined) {860 if (is_undefined) {
861 return obj.findImport(.global, symbol.index).kind.global;861 return obj.findImport(.global, symbol.index).kind.global;
862 }862 }
863 return obj.globals[symbol.index].global_type;863 const import_global_count = obj.importedCountByKind(.global);
864 return obj.globals[symbol.index - import_global_count].global_type;
864 }865 }
865 if (is_undefined) {866 if (is_undefined) {
866 return self.imports.get(loc).?.kind.global;867 return self.imports.get(loc).?.kind.global;
...@@ -880,7 +881,9 @@ fn getFunctionSignature(self: *const Wasm, loc: SymbolLoc) wasm.Type {...@@ -880,7 +881,9 @@ fn getFunctionSignature(self: *const Wasm, loc: SymbolLoc) wasm.Type {
880 const ty_index = obj.findImport(.function, symbol.index).kind.function;881 const ty_index = obj.findImport(.function, symbol.index).kind.function;
881 return obj.func_types[ty_index];882 return obj.func_types[ty_index];
882 }883 }
883 return obj.func_types[obj.functions[symbol.index].type_index];884 const import_function_count = obj.importedCountByKind(.function);
885 const type_index = obj.functions[symbol.index - import_function_count].type_index;
886 return obj.func_types[type_index];
884 }887 }
885 if (is_undefined) {888 if (is_undefined) {
886 const ty_index = self.imports.get(loc).?.kind.function;889 const ty_index = self.imports.get(loc).?.kind.function;
src/link/Wasm/Archive.zig+69-52
...@@ -15,6 +15,12 @@ name: []const u8,...@@ -15,6 +15,12 @@ name: []const u8,
1515
16header: ar_hdr = undefined,16header: ar_hdr = undefined,
1717
18/// A list of long file names, delimited by a LF character (0x0a).
19/// This is stored as a single slice of bytes, as the header-names
20/// point to the character index of a file name, rather than the index
21/// in the list.
22long_file_names: []const u8 = undefined,
23
18/// Parsed table of contents.24/// Parsed table of contents.
19/// Each symbol name points to a list of all definition25/// Each symbol name points to a list of all definition
20/// sites within the current static archive.26/// sites within the current static archive.
...@@ -53,32 +59,33 @@ const ar_hdr = extern struct {...@@ -53,32 +59,33 @@ const ar_hdr = extern struct {
53 /// Always contains ARFMAG.59 /// Always contains ARFMAG.
54 ar_fmag: [2]u8,60 ar_fmag: [2]u8,
5561
56 const NameOrLength = union(enum) {62 const NameOrIndex = union(enum) {
57 Name: []const u8,63 name: []const u8,
58 Length: u32,64 index: u32,
59 };65 };
60 fn nameOrLength(self: ar_hdr) !NameOrLength {66
61 const value = getValue(&self.ar_name);67 fn nameOrIndex(archive: ar_hdr) !NameOrIndex {
68 const value = getValue(&archive.ar_name);
62 const slash_index = mem.indexOfScalar(u8, value, '/') orelse return error.MalformedArchive;69 const slash_index = mem.indexOfScalar(u8, value, '/') orelse return error.MalformedArchive;
63 const len = value.len;70 const len = value.len;
64 if (slash_index == len - 1) {71 if (slash_index == len - 1) {
65 // Name stored directly72 // Name stored directly
66 return NameOrLength{ .Name = value };73 return NameOrIndex{ .name = value };
67 } else {74 } else {
68 // Name follows the header directly and its length is encoded in75 // Name follows the header directly and its length is encoded in
69 // the name field.76 // the name field.
70 const length = try std.fmt.parseInt(u32, value[slash_index + 1 ..], 10);77 const index = try std.fmt.parseInt(u32, value[slash_index + 1 ..], 10);
71 return NameOrLength{ .Length = length };78 return NameOrIndex{ .index = index };
72 }79 }
73 }80 }
7481
75 fn date(self: ar_hdr) !u64 {82 fn date(archive: ar_hdr) !u64 {
76 const value = getValue(&self.ar_date);83 const value = getValue(&archive.ar_date);
77 return std.fmt.parseInt(u64, value, 10);84 return std.fmt.parseInt(u64, value, 10);
78 }85 }
7986
80 fn size(self: ar_hdr) !u32 {87 fn size(archive: ar_hdr) !u32 {
81 const value = getValue(&self.ar_size);88 const value = getValue(&archive.ar_size);
82 return std.fmt.parseInt(u32, value, 10);89 return std.fmt.parseInt(u32, value, 10);
83 }90 }
8491
...@@ -87,18 +94,19 @@ const ar_hdr = extern struct {...@@ -87,18 +94,19 @@ const ar_hdr = extern struct {
87 }94 }
88};95};
8996
90pub fn deinit(self: *Archive, allocator: Allocator) void {97pub fn deinit(archive: *Archive, allocator: Allocator) void {
91 for (self.toc.keys()) |*key| {98 for (archive.toc.keys()) |*key| {
92 allocator.free(key.*);99 allocator.free(key.*);
93 }100 }
94 for (self.toc.values()) |*value| {101 for (archive.toc.values()) |*value| {
95 value.deinit(allocator);102 value.deinit(allocator);
96 }103 }
97 self.toc.deinit(allocator);104 archive.toc.deinit(allocator);
105 allocator.free(archive.long_file_names);
98}106}
99107
100pub fn parse(self: *Archive, allocator: Allocator) !void {108pub fn parse(archive: *Archive, allocator: Allocator) !void {
101 const reader = self.file.reader();109 const reader = archive.file.reader();
102110
103 const magic = try reader.readBytesNoEof(SARMAG);111 const magic = try reader.readBytesNoEof(SARMAG);
104 if (!mem.eql(u8, &magic, ARMAG)) {112 if (!mem.eql(u8, &magic, ARMAG)) {
...@@ -106,38 +114,31 @@ pub fn parse(self: *Archive, allocator: Allocator) !void {...@@ -106,38 +114,31 @@ pub fn parse(self: *Archive, allocator: Allocator) !void {
106 return error.NotArchive;114 return error.NotArchive;
107 }115 }
108116
109 self.header = try reader.readStruct(ar_hdr);117 archive.header = try reader.readStruct(ar_hdr);
110 if (!mem.eql(u8, &self.header.ar_fmag, ARFMAG)) {118 if (!mem.eql(u8, &archive.header.ar_fmag, ARFMAG)) {
111 log.debug("invalid header delimiter: expected '{s}', found '{s}'", .{ ARFMAG, self.header.ar_fmag });119 log.debug("invalid header delimiter: expected '{s}', found '{s}'", .{ ARFMAG, archive.header.ar_fmag });
112 return error.NotArchive;120 return error.NotArchive;
113 }121 }
114122
115 try self.parseTableOfContents(allocator, reader);123 try archive.parseTableOfContents(allocator, reader);
124 try archive.parseNameTable(allocator, reader);
116}125}
117126
118fn parseName(allocator: Allocator, header: ar_hdr, reader: anytype) ![]u8 {127fn parseName(archive: *const Archive, header: ar_hdr) ![]const u8 {
119 const name_or_length = try header.nameOrLength();128 const name_or_index = try header.nameOrIndex();
120 var name: []u8 = undefined;129 switch (name_or_index) {
121 switch (name_or_length) {130 .name => |name| return name,
122 .Name => |n| {131 .index => |index| {
123 name = try allocator.dupe(u8, n);132 const name = mem.sliceTo(archive.long_file_names[index..], 0x0a);
124 },133 return mem.trimRight(u8, name, "/");
125 .Length => |len| {
126 var n = try allocator.alloc(u8, len);
127 defer allocator.free(n);
128 try reader.readNoEof(n);
129 const actual_len = mem.indexOfScalar(u8, n, @as(u8, 0)) orelse n.len;
130 name = try allocator.dupe(u8, n[0..actual_len]);
131 },134 },
132 }135 }
133 return name;
134}136}
135137
136fn parseTableOfContents(self: *Archive, allocator: Allocator, reader: anytype) !void {138fn parseTableOfContents(archive: *Archive, allocator: Allocator, reader: anytype) !void {
137 log.debug("parsing table of contents for archive file '{s}'", .{self.name});
138 // size field can have extra spaces padded in front as well as the end,139 // size field can have extra spaces padded in front as well as the end,
139 // so we trim those first before parsing the ASCII value.140 // so we trim those first before parsing the ASCII value.
140 const size_trimmed = std.mem.trim(u8, &self.header.ar_size, " ");141 const size_trimmed = mem.trim(u8, &archive.header.ar_size, " ");
141 const sym_tab_size = try std.fmt.parseInt(u32, size_trimmed, 10);142 const sym_tab_size = try std.fmt.parseInt(u32, size_trimmed, 10);
142143
143 const num_symbols = try reader.readIntBig(u32);144 const num_symbols = try reader.readIntBig(u32);
...@@ -157,7 +158,7 @@ fn parseTableOfContents(self: *Archive, allocator: Allocator, reader: anytype) !...@@ -157,7 +158,7 @@ fn parseTableOfContents(self: *Archive, allocator: Allocator, reader: anytype) !
157158
158 var i: usize = 0;159 var i: usize = 0;
159 while (i < sym_tab.len) {160 while (i < sym_tab.len) {
160 const string = std.mem.sliceTo(sym_tab[i..], 0);161 const string = mem.sliceTo(sym_tab[i..], 0);
161 if (string.len == 0) {162 if (string.len == 0) {
162 i += 1;163 i += 1;
163 continue;164 continue;
...@@ -165,7 +166,7 @@ fn parseTableOfContents(self: *Archive, allocator: Allocator, reader: anytype) !...@@ -165,7 +166,7 @@ fn parseTableOfContents(self: *Archive, allocator: Allocator, reader: anytype) !
165 i += string.len;166 i += string.len;
166 const name = try allocator.dupe(u8, string);167 const name = try allocator.dupe(u8, string);
167 errdefer allocator.free(name);168 errdefer allocator.free(name);
168 const gop = try self.toc.getOrPut(allocator, name);169 const gop = try archive.toc.getOrPut(allocator, name);
169 if (gop.found_existing) {170 if (gop.found_existing) {
170 allocator.free(name);171 allocator.free(name);
171 } else {172 } else {
...@@ -175,33 +176,49 @@ fn parseTableOfContents(self: *Archive, allocator: Allocator, reader: anytype) !...@@ -175,33 +176,49 @@ fn parseTableOfContents(self: *Archive, allocator: Allocator, reader: anytype) !
175 }176 }
176}177}
177178
179fn parseNameTable(archive: *Archive, allocator: Allocator, reader: anytype) !void {
180 const header: ar_hdr = try reader.readStruct(ar_hdr);
181 if (!mem.eql(u8, &header.ar_fmag, ARFMAG)) {
182 log.err("invalid header delimiter: expected '{s}', found '{s}'", .{ ARFMAG, header.ar_fmag });
183 return error.MalformedArchive;
184 }
185 if (!mem.eql(u8, header.ar_name[0..2], "//")) {
186 log.err("invalid archive. Long name table missing", .{});
187 return error.MalformedArchive;
188 }
189 const table_size = try header.size();
190 const long_file_names = try allocator.alloc(u8, table_size);
191 errdefer allocator.free(long_file_names);
192 try reader.readNoEof(long_file_names);
193 archive.long_file_names = long_file_names;
194}
195
178/// From a given file offset, starts reading for a file header.196/// From a given file offset, starts reading for a file header.
179/// When found, parses the object file into an `Object` and returns it.197/// When found, parses the object file into an `Object` and returns it.
180pub fn parseObject(self: Archive, allocator: Allocator, file_offset: u32) !Object {198pub fn parseObject(archive: Archive, allocator: Allocator, file_offset: u32) !Object {
181 try self.file.seekTo(file_offset);199 try archive.file.seekTo(file_offset);
182 const reader = self.file.reader();200 const reader = archive.file.reader();
183 const header = try reader.readStruct(ar_hdr);201 const header = try reader.readStruct(ar_hdr);
184 const current_offset = try self.file.getPos();202 const current_offset = try archive.file.getPos();
185 try self.file.seekTo(0);203 try archive.file.seekTo(0);
186204
187 if (!mem.eql(u8, &header.ar_fmag, ARFMAG)) {205 if (!mem.eql(u8, &header.ar_fmag, ARFMAG)) {
188 log.err("invalid header delimiter: expected '{s}', found '{s}'", .{ ARFMAG, header.ar_fmag });206 log.err("invalid header delimiter: expected '{s}', found '{s}'", .{ ARFMAG, header.ar_fmag });
189 return error.MalformedArchive;207 return error.MalformedArchive;
190 }208 }
191209
192 const object_name = try parseName(allocator, header, reader);210 const object_name = try archive.parseName(header);
193 defer allocator.free(object_name);
194
195 const name = name: {211 const name = name: {
196 var buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined;212 var buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined;
197 const path = try std.os.realpath(self.name, &buffer);213 const path = try std.os.realpath(archive.name, &buffer);
198 break :name try std.fmt.allocPrint(allocator, "{s}({s})", .{ path, object_name });214 break :name try std.fmt.allocPrint(allocator, "{s}({s})", .{ path, object_name });
199 };215 };
200 defer allocator.free(name);216 defer allocator.free(name);
201217
202 const object_file = try std.fs.cwd().openFile(self.name, .{});218 const object_file = try std.fs.cwd().openFile(archive.name, .{});
203 errdefer object_file.close();219 errdefer object_file.close();
204220
221 const object_file_size = try header.size();
205 try object_file.seekTo(current_offset);222 try object_file.seekTo(current_offset);
206 return Object.create(allocator, object_file, name);223 return Object.create(allocator, object_file, name, object_file_size);
207}224}
src/link/Wasm/Object.zig+21-2
...@@ -105,14 +105,33 @@ pub const InitError = error{NotObjectFile} || ParseError || std.fs.File.ReadErro...@@ -105,14 +105,33 @@ pub const InitError = error{NotObjectFile} || ParseError || std.fs.File.ReadErro
105105
106/// Initializes a new `Object` from a wasm object file.106/// Initializes a new `Object` from a wasm object file.
107/// This also parses and verifies the object file.107/// This also parses and verifies the object file.
108pub fn create(gpa: Allocator, file: std.fs.File, name: []const u8) InitError!Object {108/// When a max size is given, will only parse up to the given size,
109/// else will read until the end of the file.
110pub fn create(gpa: Allocator, file: std.fs.File, name: []const u8, maybe_max_size: ?usize) InitError!Object {
109 var object: Object = .{111 var object: Object = .{
110 .file = file,112 .file = file,
111 .name = try gpa.dupe(u8, name),113 .name = try gpa.dupe(u8, name),
112 };114 };
113115
114 var is_object_file: bool = false;116 var is_object_file: bool = false;
115 try object.parse(gpa, file.reader(), &is_object_file);117 const size = maybe_max_size orelse size: {
118 errdefer gpa.free(object.name);
119 const stat = try file.stat();
120 break :size @intCast(usize, stat.size);
121 };
122
123 const file_contents = try gpa.alloc(u8, size);
124 defer gpa.free(file_contents);
125 var file_reader = file.reader();
126 var read: usize = 0;
127 while (read < size) {
128 const n = try file_reader.read(file_contents[read..]);
129 std.debug.assert(n != 0);
130 read += n;
131 }
132 var fbs = std.io.fixedBufferStream(file_contents);
133
134 try object.parse(gpa, fbs.reader(), &is_object_file);
116 errdefer object.deinit(gpa);135 errdefer object.deinit(gpa);
117 if (!is_object_file) return error.NotObjectFile;136 if (!is_object_file) return error.NotObjectFile;
118137
test/link.zig+5
...@@ -47,6 +47,11 @@ fn addWasmCases(cases: *tests.StandaloneContext) void {...@@ -47,6 +47,11 @@ fn addWasmCases(cases: *tests.StandaloneContext) void {
47 .build_modes = true,47 .build_modes = true,
48 .requires_stage2 = true,48 .requires_stage2 = true,
49 });49 });
50
51 cases.addBuildFile("test/link/wasm/archive/build.zig", .{
52 .build_modes = true,
53 .requires_stage2 = true,
54 });
50}55}
5156
52fn addMachOCases(cases: *tests.StandaloneContext) void {57fn addMachOCases(cases: *tests.StandaloneContext) void {
test/link/wasm/archive/build.zig created+27
...@@ -0,0 +1,27 @@
1const std = @import("std");
2const Builder = std.build.Builder;
3
4pub fn build(b: *Builder) void {
5 const mode = b.standardReleaseOptions();
6
7 const test_step = b.step("test", "Test");
8 test_step.dependOn(b.getInstallStep());
9
10 // The code in question will pull-in compiler-rt,
11 // and therefore link with its archive file.
12 const lib = b.addSharedLibrary("main", "main.zig", .unversioned);
13 lib.setBuildMode(mode);
14 lib.setTarget(.{ .cpu_arch = .wasm32, .os_tag = .freestanding });
15 lib.use_llvm = false;
16 lib.use_stage1 = false;
17 lib.use_lld = false;
18
19 const check = lib.checkObject(.wasm);
20 check.checkStart("Section import");
21 check.checkNext("entries 1"); // __truncsfhf2 should have been resolved, so only 1 import (compiler-rt's memcpy).
22
23 check.checkStart("Section custom");
24 check.checkNext("name __truncsfhf2"); // Ensure it was imported and resolved
25
26 test_step.dependOn(&check.step);
27}
test/link/wasm/archive/main.zig created+6
...@@ -0,0 +1,6 @@
1export fn foo() void {
2 var a: f16 = 2.2;
3 // this will pull-in compiler-rt
4 var b = @trunc(a);
5 _ = b;
6}