authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2020-10-07 19:36:50+02:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2020-10-07 20:34:34+02:00
logbdab4f53c1fa614fcd89468f305184fa36520039
tree34e93942e73d9ff114a7ac93fa4ea280c2a0c796
parentb13b36a71d63b6dfe4beda940fa6f9488fb3690a
signaturelock-open Commit is signed but in an unrecognized format.

Move trie structure into its own file-module

Signed-off-by: Jakub Konka <kubkon@jakubkonka.com>

2 files changed, 268 insertions(+), 200 deletions(-)

src/link/MachO.zig+9-200
...@@ -20,6 +20,8 @@ const File = link.File;...@@ -20,6 +20,8 @@ const File = link.File;
20const Cache = @import("../Cache.zig");20const Cache = @import("../Cache.zig");
21const target_util = @import("../target.zig");21const target_util = @import("../target.zig");
2222
23const Trie = @import("MachO/Trie.zig");
24
23pub const base_tag: File.Tag = File.Tag.macho;25pub const base_tag: File.Tag = File.Tag.macho;
2426
25const LoadCommand = union(enum) {27const LoadCommand = union(enum) {
...@@ -64,156 +66,6 @@ const LoadCommand = union(enum) {...@@ -64,156 +66,6 @@ const LoadCommand = union(enum) {
64 }66 }
65};67};
6668
67/// Represents export trie used in MachO executables and dynamic libraries.
68/// The purpose of an export trie is to encode as compactly as possible all
69/// export symbols for the loader `dyld`.
70/// The export trie encodes offset and other information using ULEB128
71/// encoding, and is part of the __LINKEDIT segment.
72const Trie = struct {
73 const Node = struct {
74 const Edge = struct {
75 from: *Node,
76 to: *Node,
77 label: []const u8,
78
79 pub fn deinit(self: *Edge, alloc: *Allocator) void {
80 self.to.deinit(alloc);
81 alloc.destroy(self.to);
82 self.from = undefined;
83 self.to = undefined;
84 }
85 };
86
87 export_flags: ?u64 = null,
88 offset: ?u64 = null,
89 edges: std.ArrayListUnmanaged(Edge) = .{},
90
91 pub fn deinit(self: *Node, alloc: *Allocator) void {
92 for (self.edges.items) |*edge| {
93 edge.deinit(alloc);
94 }
95 self.edges.deinit(alloc);
96 }
97
98 pub fn put(self: *Node, alloc: *Allocator, fromEdge: ?*Edge, prefix: usize, label: []const u8) !*Node {
99 // Traverse all edges.
100 for (self.edges.items) |*edge| {
101 const match = mem.indexOfDiff(u8, edge.label, label) orelse return self; // Got a full match, don't do anything.
102 if (match - prefix > 0) {
103 // If we match, we advance further down the trie.
104 return edge.to.put(alloc, edge, match, label);
105 }
106 }
107
108 if (fromEdge) |from| {
109 if (mem.eql(u8, from.label, label[0..prefix])) {
110 if (prefix == label.len) return self;
111 } else {
112 // Fixup nodes. We need to insert an intermediate node between
113 // from.to and self.
114 const mid = try alloc.create(Node);
115 mid.* = .{};
116 const to_label = from.label;
117 from.to = mid;
118 from.label = label[0..prefix];
119
120 try mid.edges.append(alloc, .{
121 .from = mid,
122 .to = self,
123 .label = to_label,
124 });
125
126 if (prefix == label.len) return self; // We're done.
127
128 const new_node = try alloc.create(Node);
129 new_node.* = .{};
130
131 try mid.edges.append(alloc, .{
132 .from = mid,
133 .to = new_node,
134 .label = label,
135 });
136
137 return new_node;
138 }
139 }
140
141 // Add a new edge.
142 const node = try alloc.create(Node);
143 node.* = .{};
144
145 try self.edges.append(alloc, .{
146 .from = self,
147 .to = node,
148 .label = label,
149 });
150
151 return node;
152 }
153
154 pub fn write(self: Node, alloc: *Allocator, buffer: *std.ArrayListUnmanaged(u8)) Trie.WriteError!void {
155 if (self.offset) |off| {
156 // Terminal node info: encode export flags and vmaddr offset of this symbol.
157 var info_buf_len: usize = 0;
158 var info_buf: [@sizeOf(u64) * 2]u8 = undefined;
159 info_buf_len += try std.debug.leb.writeULEB128Mem(info_buf[0..], self.export_flags.?);
160 info_buf_len += try std.debug.leb.writeULEB128Mem(info_buf[info_buf_len..], off);
161
162 // Encode the size of the terminal node info.
163 var size_buf: [@sizeOf(u64)]u8 = undefined;
164 const size_buf_len = try std.debug.leb.writeULEB128Mem(size_buf[0..], info_buf_len);
165
166 // Now, write them to the output buffer.
167 try buffer.ensureCapacity(alloc, buffer.items.len + info_buf_len + size_buf_len);
168 buffer.appendSliceAssumeCapacity(size_buf[0..size_buf_len]);
169 buffer.appendSliceAssumeCapacity(info_buf[0..info_buf_len]);
170 } else {
171 // Non-terminal node is delimited by 0 byte.
172 try buffer.append(alloc, 0);
173 }
174 // Write number of edges (max legal number of edges is 256).
175 try buffer.append(alloc, @intCast(u8, self.edges.items.len));
176
177 var node_offset_info: [@sizeOf(u8)]u64 = undefined;
178 for (self.edges.items) |edge, i| {
179 // Write edges labels leaving out space in-between to later populate
180 // with offsets to each node.
181 try buffer.ensureCapacity(alloc, buffer.items.len + edge.label.len + 1 + @sizeOf(u64)); // +1 to account for null-byte
182 buffer.appendSliceAssumeCapacity(edge.label);
183 buffer.appendAssumeCapacity(0);
184 node_offset_info[i] = buffer.items.len;
185 const padding = [_]u8{0} ** @sizeOf(u64);
186 buffer.appendSliceAssumeCapacity(padding[0..]);
187 }
188
189 for (self.edges.items) |edge, i| {
190 const offset = buffer.items.len;
191 try edge.to.write(alloc, buffer);
192 // We can now populate the offset to the node pointed by this edge.
193 var offset_buf: [@sizeOf(u64)]u8 = undefined;
194 const offset_buf_len = try std.debug.leb.writeULEB128Mem(offset_buf[0..], offset);
195 mem.copy(u8, buffer.items[node_offset_info[i]..], offset_buf[0..offset_buf_len]);
196 }
197 }
198 };
199
200 root: Node,
201
202 pub fn put(self: *Trie, alloc: *Allocator, word: []const u8) !*Node {
203 return self.root.put(alloc, null, 0, word);
204 }
205
206 pub const WriteError = error{ OutOfMemory, NoSpaceLeft };
207
208 pub fn write(self: Trie, alloc: *Allocator, buffer: *std.ArrayListUnmanaged(u8)) WriteError!void {
209 return self.root.write(alloc, buffer);
210 }
211
212 pub fn deinit(self: *Trie, alloc: *Allocator) void {
213 self.root.deinit(alloc);
214 }
215};
216
217base: File,69base: File,
21870
219/// Table of all load commands71/// Table of all load commands
...@@ -1541,19 +1393,21 @@ fn writeExportTrie(self: *MachO) !void {...@@ -1541,19 +1393,21 @@ fn writeExportTrie(self: *MachO) !void {
1541 defer trie.deinit(self.base.allocator);1393 defer trie.deinit(self.base.allocator);
15421394
1543 const text_segment = self.load_commands.items[self.text_segment_cmd_index.?].Segment;1395 const text_segment = self.load_commands.items[self.text_segment_cmd_index.?].Segment;
1544
1545 for (self.global_symbols.items) |symbol| {1396 for (self.global_symbols.items) |symbol| {
1546 // TODO figure out if we should put all global symbols into the export trie1397 // TODO figure out if we should put all global symbols into the export trie
1547 const name = self.getString(symbol.n_strx);1398 const name = self.getString(symbol.n_strx);
1548 const node = try trie.put(self.base.allocator, name);1399 assert(symbol.n_value >= text_segment.vmaddr);
1549 node.offset = symbol.n_value - text_segment.vmaddr;1400 try trie.put(self.base.allocator, .{
1550 node.export_flags = 0; // TODO workout creation of export flags1401 .name = name,
1402 .offset = symbol.n_value - text_segment.vmaddr,
1403 .export_flags = 0, // TODO workout creation of export flags
1404 });
1551 }1405 }
15521406
1553 var buffer: std.ArrayListUnmanaged(u8) = .{};1407 var buffer: std.ArrayListUnmanaged(u8) = .{};
1554 defer buffer.deinit(self.base.allocator);1408 defer buffer.deinit(self.base.allocator);
15551409
1556 try trie.write(self.base.allocator, &buffer);1410 try trie.writeULEB128Mem(self.base.allocator, &buffer);
15571411
1558 const dyld_info = &self.load_commands.items[self.dyld_info_cmd_index.?].DyldInfo;1412 const dyld_info = &self.load_commands.items[self.dyld_info_cmd_index.?].DyldInfo;
1559 try self.base.file.?.pwriteAll(buffer.items, dyld_info.export_off);1413 try self.base.file.?.pwriteAll(buffer.items, dyld_info.export_off);
...@@ -1688,48 +1542,3 @@ fn satMul(a: anytype, b: anytype) @TypeOf(a, b) {...@@ -1688,48 +1542,3 @@ fn satMul(a: anytype, b: anytype) @TypeOf(a, b) {
1688 const T = @TypeOf(a, b);1542 const T = @TypeOf(a, b);
1689 return std.math.mul(T, a, b) catch std.math.maxInt(T);1543 return std.math.mul(T, a, b) catch std.math.maxInt(T);
1690}1544}
1691
1692test "Trie basic" {
1693 const testing = @import("std").testing;
1694 var gpa = testing.allocator;
1695
1696 var trie: Trie = .{
1697 .root = .{},
1698 };
1699 defer trie.deinit(gpa);
1700
1701 // root
1702 testing.expect(trie.root.edges.items.len == 0);
1703
1704 // root --- _st ---> node
1705 try trie.put(gpa, "_st");
1706 testing.expect(trie.root.edges.items.len == 1);
1707 testing.expect(mem.eql(u8, trie.root.edges.items[0].label, "_st"));
1708
1709 {
1710 // root --- _st ---> node --- _start ---> node
1711 try trie.put(gpa, "_start");
1712 testing.expect(trie.root.edges.items.len == 1);
1713
1714 const nextEdge = &trie.root.edges.items[0];
1715 testing.expect(mem.eql(u8, nextEdge.label, "_st"));
1716 testing.expect(nextEdge.to.edges.items.len == 1);
1717 testing.expect(mem.eql(u8, nextEdge.to.edges.items[0].label, "_start"));
1718 }
1719 {
1720 // root --- _ ---> node --- _st ---> node --- _start ---> node
1721 // |
1722 // | --- _main ---> node
1723 try trie.put(gpa, "_main");
1724 testing.expect(trie.root.edges.items.len == 1);
1725
1726 const nextEdge = &trie.root.edges.items[0];
1727 testing.expect(mem.eql(u8, nextEdge.label, "_"));
1728 testing.expect(nextEdge.to.edges.items.len == 2);
1729 testing.expect(mem.eql(u8, nextEdge.to.edges.items[0].label, "_st"));
1730 testing.expect(mem.eql(u8, nextEdge.to.edges.items[1].label, "_main"));
1731
1732 const nextNextEdge = &nextEdge.to.edges.items[0];
1733 testing.expect(mem.eql(u8, nextNextEdge.to.edges.items[0].label, "_start"));
1734 }
1735}
src/link/MachO/Trie.zig created+259
...@@ -0,0 +1,259 @@
1/// Represents export trie used in MachO executables and dynamic libraries.
2/// The purpose of an export trie is to encode as compactly as possible all
3/// export symbols for the loader `dyld`.
4/// The export trie encodes offset and other information using ULEB128
5/// encoding, and is part of the __LINKEDIT segment.
6///
7/// Description from loader.h:
8///
9/// The symbols exported by a dylib are encoded in a trie. This is a compact
10/// representation that factors out common prefixes. It also reduces LINKEDIT pages
11/// in RAM because it encodes all information (name, address, flags) in one small,
12/// contiguous range. The export area is a stream of nodes. The first node sequentially
13/// is the start node for the trie.
14///
15/// Nodes for a symbol start with a uleb128 that is the length of the exported symbol
16/// information for the string so far. If there is no exported symbol, the node starts
17/// with a zero byte. If there is exported info, it follows the length.
18///
19/// First is a uleb128 containing flags. Normally, it is followed by a uleb128 encoded
20/// offset which is location of the content named by the symbol from the mach_header
21/// for the image. If the flags is EXPORT_SYMBOL_FLAGS_REEXPORT, then following the flags
22/// is a uleb128 encoded library ordinal, then a zero terminated UTF8 string. If the string
23/// is zero length, then the symbol is re-export from the specified dylib with the same name.
24/// If the flags is EXPORT_SYMBOL_FLAGS_STUB_AND_RESOLVER, then following the flags is two
25/// uleb128s: the stub offset and the resolver offset. The stub is used by non-lazy pointers.
26/// The resolver is used by lazy pointers and must be called to get the actual address to use.
27///
28/// After the optional exported symbol information is a byte of how many edges (0-255) that
29/// this node has leaving it, followed by each edge. Each edge is a zero terminated UTF8 of
30/// the addition chars in the symbol, followed by a uleb128 offset for the node that edge points to.
31const Trie = @This();
32
33const std = @import("std");
34const mem = std.mem;
35const leb = std.debug.leb;
36const log = std.log.scoped(.link);
37const Allocator = mem.Allocator;
38
39pub const Symbol = struct {
40 name: []const u8,
41 offset: u64,
42 export_flags: u64,
43};
44
45const Edge = struct {
46 from: *Node,
47 to: *Node,
48 label: []const u8,
49
50 fn deinit(self: *Edge, alloc: *Allocator) void {
51 self.to.deinit(alloc);
52 alloc.destroy(self.to);
53 self.from = undefined;
54 self.to = undefined;
55 }
56};
57
58const Node = struct {
59 export_flags: ?u64 = null,
60 offset: ?u64 = null,
61 edges: std.ArrayListUnmanaged(Edge) = .{},
62
63 fn deinit(self: *Node, alloc: *Allocator) void {
64 for (self.edges.items) |*edge| {
65 edge.deinit(alloc);
66 }
67 self.edges.deinit(alloc);
68 }
69
70 fn put(self: *Node, alloc: *Allocator, fromEdge: ?*Edge, prefix: usize, label: []const u8) !*Node {
71 // Traverse all edges.
72 for (self.edges.items) |*edge| {
73 const match = mem.indexOfDiff(u8, edge.label, label) orelse return self; // Got a full match, don't do anything.
74 if (match - prefix > 0) {
75 // If we match, we advance further down the trie.
76 return edge.to.put(alloc, edge, match, label);
77 }
78 }
79
80 if (fromEdge) |from| {
81 if (mem.eql(u8, from.label, label[0..prefix])) {
82 if (prefix == label.len) return self;
83 } else {
84 // Fixup nodes. We need to insert an intermediate node between
85 // from.to and self.
86 // Is: A -> B
87 // Should be: A -> C -> B
88 const mid = try alloc.create(Node);
89 mid.* = .{};
90 const to_label = from.label;
91 from.to = mid;
92 from.label = label[0..prefix];
93
94 try mid.edges.append(alloc, .{
95 .from = mid,
96 .to = self,
97 .label = to_label,
98 });
99
100 if (prefix == label.len) return self; // We're done.
101
102 const new_node = try alloc.create(Node);
103 new_node.* = .{};
104
105 try mid.edges.append(alloc, .{
106 .from = mid,
107 .to = new_node,
108 .label = label,
109 });
110
111 return new_node;
112 }
113 }
114
115 // Add a new edge.
116 const node = try alloc.create(Node);
117 node.* = .{};
118
119 try self.edges.append(alloc, .{
120 .from = self,
121 .to = node,
122 .label = label,
123 });
124
125 return node;
126 }
127
128 fn writeULEB128Mem(self: Node, alloc: *Allocator, buffer: *std.ArrayListUnmanaged(u8)) Trie.WriteError!void {
129 if (self.offset) |offset| {
130 // Terminal node info: encode export flags and vmaddr offset of this symbol.
131 var info_buf_len: usize = 0;
132 var info_buf: [@sizeOf(u64) * 2]u8 = undefined;
133 info_buf_len += try leb.writeULEB128Mem(info_buf[0..], self.export_flags.?);
134 info_buf_len += try leb.writeULEB128Mem(info_buf[info_buf_len..], offset);
135
136 // Encode the size of the terminal node info.
137 var size_buf: [@sizeOf(u64)]u8 = undefined;
138 const size_buf_len = try leb.writeULEB128Mem(size_buf[0..], info_buf_len);
139
140 // Now, write them to the output buffer.
141 try buffer.ensureCapacity(alloc, buffer.items.len + info_buf_len + size_buf_len);
142 buffer.appendSliceAssumeCapacity(size_buf[0..size_buf_len]);
143 buffer.appendSliceAssumeCapacity(info_buf[0..info_buf_len]);
144 } else {
145 // Non-terminal node is delimited by 0 byte.
146 try buffer.append(alloc, 0);
147 }
148 // Write number of edges (max legal number of edges is 256).
149 try buffer.append(alloc, @intCast(u8, self.edges.items.len));
150
151 var node_offset_info: [@sizeOf(u8)]u64 = undefined;
152 for (self.edges.items) |edge, i| {
153 // Write edges labels leaving out space in-between to later populate
154 // with offsets to each node.
155 try buffer.ensureCapacity(alloc, buffer.items.len + edge.label.len + 1 + @sizeOf(u64)); // +1 to account for null-byte
156 buffer.appendSliceAssumeCapacity(edge.label);
157 buffer.appendAssumeCapacity(0);
158 node_offset_info[i] = buffer.items.len;
159 const padding = [_]u8{0} ** @sizeOf(u64);
160 buffer.appendSliceAssumeCapacity(padding[0..]);
161 }
162
163 for (self.edges.items) |edge, i| {
164 const offset = buffer.items.len;
165 try edge.to.writeULEB128Mem(alloc, buffer);
166 // We can now populate the offset to the node pointed by this edge.
167 // TODO this is not the approach taken by `ld64` which does several iterations
168 // to close the gap between the space encoding the offset to the node pointed
169 // by this edge. However, it seems that as long as we are contiguous, the padding
170 // introduced here should not influence the performance of `dyld`. I'm leaving
171 // this TODO here though as a reminder to re-investigate in the future and especially
172 // when we start working on dylibs in case `dyld` refuses to cooperate and/or the
173 // performance is noticably sufferring.
174 // Link to official impl: https://opensource.apple.com/source/ld64/ld64-123.2.1/src/abstraction/MachOTrie.hpp
175 var offset_buf: [@sizeOf(u64)]u8 = undefined;
176 const offset_buf_len = try leb.writeULEB128Mem(offset_buf[0..], offset);
177 mem.copy(u8, buffer.items[node_offset_info[i]..], offset_buf[0..offset_buf_len]);
178 }
179 }
180};
181
182root: Node,
183
184/// Insert a symbol into the trie, updating the prefixes in the process.
185/// This operation may change the layout of the trie by splicing edges in
186/// certain circumstances.
187pub fn put(self: *Trie, alloc: *Allocator, symbol: Symbol) !void {
188 const node = try self.root.put(alloc, null, 0, symbol.name);
189 node.offset = symbol.offset;
190 node.export_flags = symbol.export_flags;
191}
192
193pub const WriteError = error{ OutOfMemory, NoSpaceLeft };
194
195/// Write the trie to a buffer ULEB128 encoded.
196pub fn writeULEB128Mem(self: Trie, alloc: *Allocator, buffer: *std.ArrayListUnmanaged(u8)) WriteError!void {
197 return self.root.writeULEB128Mem(alloc, buffer);
198}
199
200pub fn deinit(self: *Trie, alloc: *Allocator) void {
201 self.root.deinit(alloc);
202}
203
204test "Trie basic" {
205 const testing = @import("std").testing;
206 var gpa = testing.allocator;
207
208 var trie: Trie = .{
209 .root = .{},
210 };
211 defer trie.deinit(gpa);
212
213 // root
214 testing.expect(trie.root.edges.items.len == 0);
215
216 // root --- _st ---> node
217 try trie.put(gpa, .{
218 .name = "_st",
219 .offset = 0,
220 .export_flags = 0,
221 });
222 testing.expect(trie.root.edges.items.len == 1);
223 testing.expect(mem.eql(u8, trie.root.edges.items[0].label, "_st"));
224
225 {
226 // root --- _st ---> node --- _start ---> node
227 try trie.put(gpa, .{
228 .name = "_start",
229 .offset = 0,
230 .export_flags = 0,
231 });
232 testing.expect(trie.root.edges.items.len == 1);
233
234 const nextEdge = &trie.root.edges.items[0];
235 testing.expect(mem.eql(u8, nextEdge.label, "_st"));
236 testing.expect(nextEdge.to.edges.items.len == 1);
237 testing.expect(mem.eql(u8, nextEdge.to.edges.items[0].label, "_start"));
238 }
239 {
240 // root --- _ ---> node --- _st ---> node --- _start ---> node
241 // |
242 // | --- _main ---> node
243 try trie.put(gpa, .{
244 .name = "_main",
245 .offset = 0,
246 .export_flags = 0,
247 });
248 testing.expect(trie.root.edges.items.len == 1);
249
250 const nextEdge = &trie.root.edges.items[0];
251 testing.expect(mem.eql(u8, nextEdge.label, "_"));
252 testing.expect(nextEdge.to.edges.items.len == 2);
253 testing.expect(mem.eql(u8, nextEdge.to.edges.items[0].label, "_st"));
254 testing.expect(mem.eql(u8, nextEdge.to.edges.items[1].label, "_main"));
255
256 const nextNextEdge = &nextEdge.to.edges.items[0];
257 testing.expect(mem.eql(u8, nextNextEdge.to.edges.items[0].label, "_start"));
258 }
259}