authorgravatar for alex@alexrp.comAlex Rønne Petersen <alex@alexrp.com> 2024-12-23 00:09:52+01:00
committergravatar for alex@alexrp.comAlex Rønne Petersen <alex@alexrp.com> 2024-12-23 00:09:52+01:00
log1cb798256ff6625d4c64900d07a67de05617b1a2
treebed595a760474ed369f90cb30c22649ec8a32bd1
parent1f5315e774684fb0f4a945f779a89d021588bba7
signaturebadge-check Signed by SSH key SHA256:7B/LJ7bpR1eX8aCXSr4mtd5M45VMPKcx9zY8e95b5QM

fetch_them_macos_headers: Simplify, remove unused code.

1 files changed, 33 insertions(+), 477 deletions(-)

tools/fetch_them_macos_headers.zig+33-477
......@@ -7,22 +7,19 @@ const assert = std.debug.assert;
77const tmpDir = std.testing.tmpDir;
88
99const Allocator = mem.Allocator;
10const Blake3 = std.crypto.hash.Blake3;
1110const OsTag = std.Target.Os.Tag;
1211
1312var general_purpose_allocator = std.heap.GeneralPurposeAllocator(.{}){};
1413const gpa = general_purpose_allocator.allocator();
1514
1615const Arch = enum {
17 any,
1816 aarch64,
1917 x86_64,
2018};
2119
22const Abi = enum { any, none };
20const Abi = enum { none };
2321
2422const OsVer = enum(u32) {
25 any = 0,
2623 catalina = 10,
2724 big_sur = 11,
2825 monterey = 12,
......@@ -37,22 +34,6 @@ const Target = struct {
3734 os_ver: OsVer,
3835 abi: Abi = .none,
3936
40 fn hash(a: Target) u32 {
41 var hasher = std.hash.Wyhash.init(0);
42 std.hash.autoHash(&hasher, a.arch);
43 std.hash.autoHash(&hasher, a.os);
44 std.hash.autoHash(&hasher, a.os_ver);
45 std.hash.autoHash(&hasher, a.abi);
46 return @as(u32, @truncate(hasher.final()));
47 }
48
49 fn eql(a: Target, b: Target) bool {
50 return a.arch == b.arch and
51 a.os == b.os and
52 a.os_ver == b.os_ver and
53 a.abi == b.abi;
54 }
55
5637 fn name(self: Target, allocator: Allocator) ![]const u8 {
5738 return std.fmt.allocPrint(allocator, "{s}-{s}-{s}", .{
5839 @tagName(self.arch),
......@@ -62,7 +43,6 @@ const Target = struct {
6243 }
6344
6445 fn fullName(self: Target, allocator: Allocator) ![]const u8 {
65 if (self.os_ver == .any) return self.name(allocator);
6646 return std.fmt.allocPrint(allocator, "{s}-{s}.{d}-{s}", .{
6747 @tagName(self.arch),
6848 @tagName(self.os),
......@@ -72,123 +52,13 @@ const Target = struct {
7252 }
7353};
7454
75const targets = [_]Target{
76 Target{
77 .arch = .any,
78 .abi = .any,
79 .os_ver = .any,
80 },
81 Target{
82 .arch = .aarch64,
83 .os_ver = .any,
84 },
85 Target{
86 .arch = .x86_64,
87 .os_ver = .any,
88 },
89 Target{
90 .arch = .x86_64,
91 .os_ver = .catalina,
92 },
93 Target{
94 .arch = .x86_64,
95 .os_ver = .big_sur,
96 },
97 Target{
98 .arch = .x86_64,
99 .os_ver = .monterey,
100 },
101 Target{
102 .arch = .x86_64,
103 .os_ver = .ventura,
104 },
105 Target{
106 .arch = .x86_64,
107 .os_ver = .sonoma,
108 },
109 Target{
110 .arch = .x86_64,
111 .os_ver = .sequoia,
112 },
113 Target{
114 .arch = .aarch64,
115 .os_ver = .big_sur,
116 },
117 Target{
118 .arch = .aarch64,
119 .os_ver = .monterey,
120 },
121 Target{
122 .arch = .aarch64,
123 .os_ver = .ventura,
124 },
125 Target{
126 .arch = .aarch64,
127 .os_ver = .sonoma,
128 },
129 Target{
130 .arch = .aarch64,
131 .os_ver = .sequoia,
132 },
133};
134
13555const headers_source_prefix: []const u8 = "headers";
13656
137const Contents = struct {
138 bytes: []const u8,
139 hit_count: usize,
140 hash: []const u8,
141 is_generic: bool,
142
143 fn hitCountLessThan(context: void, lhs: *const Contents, rhs: *const Contents) bool {
144 _ = context;
145 return lhs.hit_count < rhs.hit_count;
146 }
147};
148
149const TargetToHashContext = struct {
150 pub fn hash(self: @This(), target: Target) u32 {
151 _ = self;
152 return target.hash();
153 }
154 pub fn eql(self: @This(), a: Target, b: Target, b_index: usize) bool {
155 _ = self;
156 _ = b_index;
157 return a.eql(b);
158 }
159};
160const TargetToHash = std.ArrayHashMap(Target, []const u8, TargetToHashContext, true);
161
162const HashToContents = std.StringHashMap(Contents);
163const PathTable = std.StringHashMap(*TargetToHash);
164
165/// The don't-dedup-list contains file paths with known problematic headers
166/// which while contain the same contents between architectures, should not be
167/// deduped since they contain includes, etc. which are relative and thus cannot be separated
168/// into a shared include dir such as `any-macos-any`.
169const dont_dedup_list = &[_][]const u8{
170 "libkern/OSAtomic.h",
171 "libkern/OSAtomicDeprecated.h",
172 "libkern/OSSpinLockDeprecated.h",
173 "libkern/OSAtomicQueue.h",
174};
175
176fn generateDontDedupMap(arena: Allocator) !std.StringHashMap(void) {
177 var map = std.StringHashMap(void).init(arena);
178 try map.ensureTotalCapacity(dont_dedup_list.len);
179 for (dont_dedup_list) |path| {
180 map.putAssumeCapacityNoClobber(path, {});
181 }
182 return map;
183}
184
18557const usage =
186 \\fetch_them_macos_headers fetch
187 \\fetch_them_macos_headers dedup
58 \\fetch_them_macos_headers [options] [cc args]
18859 \\
189 \\Commands:
190 \\ fetch Fetch libc headers into headers/<arch>-macos.<os_ver> dir
191 \\ dedup Generate deduplicated dirs into a given <destination> path
60 \\Options:
61 \\ --sysroot Path to macOS SDK
19262 \\
19363 \\General Options:
19464 \\-h, --help Print this help and exit
......@@ -197,70 +67,17 @@ const usage =
19767pub fn main() anyerror!void {
19868 var arena = std.heap.ArenaAllocator.init(gpa);
19969 defer arena.deinit();
70 const allocator = arena.allocator();
20071
201 const all_args = try std.process.argsAlloc(arena.allocator());
202 const args = all_args[1..];
203 if (args.len == 0) fatal("no command or option specified", .{});
204
205 const cmd = args[0];
206 if (mem.eql(u8, cmd, "--help") or mem.eql(u8, cmd, "-h")) {
207 return info(usage, .{});
208 } else if (mem.eql(u8, cmd, "dedup")) {
209 return dedup(arena.allocator(), args[1..]);
210 } else if (mem.eql(u8, cmd, "fetch")) {
211 return fetch(arena.allocator(), args[1..]);
212 } else fatal("unknown command or option: {s}", .{cmd});
213}
214
215const ArgsIterator = struct {
216 args: []const []const u8,
217 i: usize = 0,
218
219 fn next(it: *@This()) ?[]const u8 {
220 if (it.i >= it.args.len) {
221 return null;
222 }
223 defer it.i += 1;
224 return it.args[it.i];
225 }
226
227 fn nextOrFatal(it: *@This()) []const u8 {
228 const arg = it.next() orelse fatal("expected parameter after '{s}'", .{it.args[it.i - 1]});
229 return arg;
230 }
231};
232
233fn info(comptime format: []const u8, args: anytype) void {
234 const msg = std.fmt.allocPrint(gpa, "info: " ++ format ++ "\n", args) catch return;
235 std.io.getStdOut().writeAll(msg) catch {};
236}
237
238fn fatal(comptime format: []const u8, args: anytype) noreturn {
239 ret: {
240 const msg = std.fmt.allocPrint(gpa, "fatal: " ++ format ++ "\n", args) catch break :ret;
241 std.io.getStdErr().writeAll(msg) catch {};
242 }
243 std.process.exit(1);
244}
245
246const fetch_usage =
247 \\fetch_them_macos_headers fetch
248 \\
249 \\Options:
250 \\ --sysroot Path to macOS SDK
251 \\
252 \\General Options:
253 \\-h, --help Print this help and exit
254;
72 const args = try std.process.argsAlloc(allocator);
25573
256fn fetch(arena: Allocator, args: []const []const u8) !void {
257 var argv = std.ArrayList([]const u8).init(arena);
74 var argv = std.ArrayList([]const u8).init(allocator);
25875 var sysroot: ?[]const u8 = null;
25976
260 var args_iter = ArgsIterator{ .args = args };
77 var args_iter = ArgsIterator{ .args = args[1..] };
26178 while (args_iter.next()) |arg| {
26279 if (mem.eql(u8, arg, "--help") or mem.eql(u8, arg, "-h")) {
263 return info(fetch_usage, .{});
80 return info(usage, .{});
26481 } else if (mem.eql(u8, arg, "--sysroot")) {
26582 sysroot = args_iter.nextOrFatal();
26683 } else try argv.append(arg);
......@@ -268,17 +85,17 @@ fn fetch(arena: Allocator, args: []const []const u8) !void {
26885
26986 const sysroot_path = sysroot orelse blk: {
27087 const target = try std.zig.system.resolveTargetQuery(.{});
271 break :blk std.zig.system.darwin.getSdk(arena, target) orelse
88 break :blk std.zig.system.darwin.getSdk(allocator, target) orelse
27289 fatal("no SDK found; you can provide one explicitly with '--sysroot' flag", .{});
27390 };
27491
27592 var sdk_dir = try std.fs.cwd().openDir(sysroot_path, .{});
27693 defer sdk_dir.close();
277 const sdk_info = try sdk_dir.readFileAlloc(arena, "SDKSettings.json", std.math.maxInt(u32));
94 const sdk_info = try sdk_dir.readFileAlloc(allocator, "SDKSettings.json", std.math.maxInt(u32));
27895
27996 const parsed_json = try std.json.parseFromSlice(struct {
28097 DefaultProperties: struct { MACOSX_DEPLOYMENT_TARGET: []const u8 },
281 }, arena, sdk_info, .{ .ignore_unknown_fields = true });
98 }, allocator, sdk_info, .{ .ignore_unknown_fields = true });
28299
283100 const version = Version.parse(parsed_json.value.DefaultProperties.MACOSX_DEPLOYMENT_TARGET) orelse
284101 fatal("don't know how to parse SDK version: {s}", .{
......@@ -303,7 +120,7 @@ fn fetch(arena: Allocator, args: []const []const u8) !void {
303120 .arch = arch,
304121 .os_ver = os_ver,
305122 };
306 try fetchTarget(arena, argv.items, sysroot_path, target, version, tmp);
123 try fetchTarget(allocator, argv.items, sysroot_path, target, version, tmp);
307124 }
308125}
309126
......@@ -333,7 +150,6 @@ fn fetchTarget(
333150 switch (target.arch) {
334151 .x86_64 => "x86_64",
335152 .aarch64 => "arm64",
336 else => unreachable,
337153 },
338154 macos_version,
339155 "-isysroot",
......@@ -359,7 +175,7 @@ fn fetchTarget(
359175 std.log.err("{s}", .{res.stderr});
360176 }
361177
362 // Read in the contents of `upgrade.o.d`
178 // Read in the contents of `macos-headers.o.d`
363179 const headers_list_file = try tmp.dir.openFile(headers_list_filename, .{});
364180 defer headers_list_file.close();
365181
......@@ -411,295 +227,35 @@ fn fetchTarget(
411227 }
412228}
413229
414const dedup_usage =
415 \\fetch_them_macos_headers dedup [path]
416 \\
417 \\General Options:
418 \\-h, --help Print this help and exit
419;
420
421/// Dedups libs headers assuming the following layered structure:
422/// layer 1: x86_64-macos.10 x86_64-macos.11 x86_64-macos.12 aarch64-macos.11 aarch64-macos.12
423/// layer 2: any-macos.10 any-macos.11 any-macos.12
424/// layer 3: any-macos
425///
426/// The first layer consists of headers specific to a CPU architecture AND macOS version. The second
427/// layer consists of headers common to a macOS version across CPU architectures, and the final
428/// layer consists of headers common to all libc headers.
429fn dedup(arena: Allocator, args: []const []const u8) !void {
430 var path: ?[]const u8 = null;
431 var args_iter = ArgsIterator{ .args = args };
432 while (args_iter.next()) |arg| {
433 if (mem.eql(u8, arg, "--help") or mem.eql(u8, arg, "-h")) {
434 return info(dedup_usage, .{});
435 } else {
436 if (path != null) fatal("too many arguments", .{});
437 path = arg;
438 }
439 }
440
441 const dest_path = path orelse fatal("no destination path specified", .{});
442 var dest_dir = fs.cwd().makeOpenPath(dest_path, .{}) catch |err| switch (err) {
443 error.NotDir => fatal("path '{s}' not a directory", .{dest_path}),
444 else => return err,
445 };
446 defer dest_dir.close();
447
448 var dont_dedup_map = try generateDontDedupMap(arena);
449 var layer_2_targets = std.ArrayList(TargetWithPrefix).init(arena);
450
451 for (&[_]OsVer{ .catalina, .big_sur, .monterey, .ventura, .sonoma, .sequoia }) |os_ver| {
452 var layer_1_targets = std.ArrayList(TargetWithPrefix).init(arena);
453
454 for (targets) |target| {
455 if (target.os_ver != os_ver) continue;
456 try layer_1_targets.append(.{
457 .prefix = headers_source_prefix,
458 .target = target,
459 });
460 }
461
462 if (layer_1_targets.items.len < 2) {
463 try layer_2_targets.appendSlice(layer_1_targets.items);
464 continue;
465 }
466
467 const layer_2_target = try dedupDirs(arena, .{
468 .os_ver = os_ver,
469 .dest_path = dest_path,
470 .dest_dir = dest_dir,
471 .targets = layer_1_targets.items,
472 .dont_dedup_map = &dont_dedup_map,
473 });
474 try layer_2_targets.append(layer_2_target);
475 }
476
477 const layer_3_target = try dedupDirs(arena, .{
478 .os_ver = .any,
479 .dest_path = dest_path,
480 .dest_dir = dest_dir,
481 .targets = layer_2_targets.items,
482 .dont_dedup_map = &dont_dedup_map,
483 });
484 assert(layer_3_target.target.eql(targets[0]));
485}
486
487const TargetWithPrefix = struct {
488 prefix: []const u8,
489 target: Target,
490};
491
492const DedupDirsArgs = struct {
493 os_ver: OsVer,
494 dest_path: []const u8,
495 dest_dir: fs.Dir,
496 targets: []const TargetWithPrefix,
497 dont_dedup_map: *const std.StringHashMap(void),
498};
499
500fn dedupDirs(arena: Allocator, args: DedupDirsArgs) !TargetWithPrefix {
501 var tmp = tmpDir(.{ .iterate = true });
502 defer tmp.cleanup();
503
504 var path_table = PathTable.init(arena);
505 var hash_to_contents = HashToContents.init(arena);
506
507 var savings = FindResult{};
508 for (args.targets) |target| {
509 const res = try findDuplicates(target.target, arena, target.prefix, &path_table, &hash_to_contents);
510 savings.max_bytes_saved += res.max_bytes_saved;
511 savings.total_bytes += res.total_bytes;
512 }
513
514 info("summary: {} could be reduced to {}", .{
515 std.fmt.fmtIntSizeBin(savings.total_bytes),
516 std.fmt.fmtIntSizeBin(savings.total_bytes - savings.max_bytes_saved),
517 });
230const ArgsIterator = struct {
231 args: []const []const u8,
232 i: usize = 0,
518233
519 const output_target = Target{
520 .arch = .any,
521 .abi = .any,
522 .os_ver = args.os_ver,
523 };
524 const common_name = try output_target.fullName(arena);
525
526 var missed_opportunity_bytes: usize = 0;
527 // Iterate path_table. For each path, put all the hashes into a list. Sort by hit_count.
528 // The hash with the highest hit_count gets to be the "generic" one. Everybody else
529 // gets their header in a separate arch directory.
530 var path_it = path_table.iterator();
531 while (path_it.next()) |path_kv| {
532 if (!args.dont_dedup_map.contains(path_kv.key_ptr.*)) {
533 var contents_list = std.ArrayList(*Contents).init(arena);
534 {
535 var hash_it = path_kv.value_ptr.*.iterator();
536 while (hash_it.next()) |hash_kv| {
537 const contents = &hash_to_contents.getEntry(hash_kv.value_ptr.*).?.value_ptr.*;
538 try contents_list.append(contents);
539 }
540 }
541 std.mem.sort(*Contents, contents_list.items, {}, Contents.hitCountLessThan);
542 const best_contents = contents_list.popOrNull().?;
543 if (best_contents.hit_count > 1) {
544 // Put it in `any-macos-none`.
545 const full_path = try fs.path.join(arena, &[_][]const u8{ common_name, path_kv.key_ptr.* });
546 try tmp.dir.makePath(fs.path.dirname(full_path).?);
547 try tmp.dir.writeFile(.{ .sub_path = full_path, .data = best_contents.bytes });
548 best_contents.is_generic = true;
549 while (contents_list.popOrNull()) |contender| {
550 if (contender.hit_count > 1) {
551 const this_missed_bytes = contender.hit_count * contender.bytes.len;
552 missed_opportunity_bytes += this_missed_bytes;
553 info("Missed opportunity ({}): {s}", .{
554 std.fmt.fmtIntSizeBin(this_missed_bytes),
555 path_kv.key_ptr.*,
556 });
557 } else break;
558 }
559 }
560 }
561 var hash_it = path_kv.value_ptr.*.iterator();
562 while (hash_it.next()) |hash_kv| {
563 const contents = &hash_to_contents.getEntry(hash_kv.value_ptr.*).?.value_ptr.*;
564 if (contents.is_generic) continue;
565
566 const target = hash_kv.key_ptr.*;
567 const target_name = try target.fullName(arena);
568 const full_path = try fs.path.join(arena, &[_][]const u8{ target_name, path_kv.key_ptr.* });
569 try tmp.dir.makePath(fs.path.dirname(full_path).?);
570 try tmp.dir.writeFile(.{ .sub_path = full_path, .data = contents.bytes });
234 fn next(it: *@This()) ?[]const u8 {
235 if (it.i >= it.args.len) {
236 return null;
571237 }
238 defer it.i += 1;
239 return it.args[it.i];
572240 }
573241
574 for (args.targets) |target| {
575 const target_name = try target.target.fullName(arena);
576 try args.dest_dir.deleteTree(target_name);
577 }
578 try args.dest_dir.deleteTree(common_name);
579
580 var tmp_it = tmp.dir.iterate();
581 while (try tmp_it.next()) |entry| {
582 switch (entry.kind) {
583 .directory => {
584 const sub_dir = try tmp.dir.openDir(entry.name, .{ .iterate = true });
585 const dest_sub_dir = try args.dest_dir.makeOpenPath(entry.name, .{});
586 try copyDirAll(sub_dir, dest_sub_dir);
587 },
588 else => info("unexpected file format: not a directory: '{s}'", .{entry.name}),
589 }
242 fn nextOrFatal(it: *@This()) []const u8 {
243 const arg = it.next() orelse fatal("expected parameter after '{s}'", .{it.args[it.i - 1]});
244 return arg;
590245 }
591
592 return TargetWithPrefix{
593 .prefix = args.dest_path,
594 .target = output_target,
595 };
596}
597
598const FindResult = struct {
599 max_bytes_saved: usize = 0,
600 total_bytes: usize = 0,
601246};
602247
603fn findDuplicates(
604 target: Target,
605 arena: Allocator,
606 dest_path: []const u8,
607 path_table: *PathTable,
608 hash_to_contents: *HashToContents,
609) !FindResult {
610 var result = FindResult{};
611
612 const target_name = try target.fullName(arena);
613 const target_include_dir = try fs.path.join(arena, &[_][]const u8{ dest_path, target_name });
614 var dir_stack = std.ArrayList([]const u8).init(arena);
615 try dir_stack.append(target_include_dir);
616
617 while (dir_stack.popOrNull()) |full_dir_name| {
618 var dir = fs.cwd().openDir(full_dir_name, .{ .iterate = true }) catch |err| switch (err) {
619 error.FileNotFound => break,
620 error.AccessDenied => break,
621 else => return err,
622 };
623 defer dir.close();
624
625 var dir_it = dir.iterate();
626
627 while (try dir_it.next()) |entry| {
628 const full_path = try fs.path.join(arena, &[_][]const u8{ full_dir_name, entry.name });
629 switch (entry.kind) {
630 .directory => try dir_stack.append(full_path),
631 .file => {
632 const rel_path = try fs.path.relative(arena, target_include_dir, full_path);
633 const max_size = 2 * 1024 * 1024 * 1024;
634 const raw_bytes = try fs.cwd().readFileAlloc(arena, full_path, max_size);
635 const trimmed = mem.trim(u8, raw_bytes, " \r\n\t");
636 result.total_bytes += raw_bytes.len;
637 const hash = try arena.alloc(u8, 32);
638 var hasher = Blake3.init(.{});
639 hasher.update(rel_path);
640 hasher.update(trimmed);
641 hasher.final(hash);
642 const gop = try hash_to_contents.getOrPut(hash);
643 if (gop.found_existing) {
644 result.max_bytes_saved += raw_bytes.len;
645 gop.value_ptr.hit_count += 1;
646 info("duplicate: {s} {s} ({})", .{
647 target_name,
648 rel_path,
649 std.fmt.fmtIntSizeBin(raw_bytes.len),
650 });
651 } else {
652 gop.value_ptr.* = Contents{
653 .bytes = trimmed,
654 .hit_count = 1,
655 .hash = hash,
656 .is_generic = false,
657 };
658 }
659 const path_gop = try path_table.getOrPut(rel_path);
660 const target_to_hash = if (path_gop.found_existing) path_gop.value_ptr.* else blk: {
661 const ptr = try arena.create(TargetToHash);
662 ptr.* = TargetToHash.init(arena);
663 path_gop.value_ptr.* = ptr;
664 break :blk ptr;
665 };
666 try target_to_hash.putNoClobber(target, hash);
667 },
668 else => info("unexpected file: {s}", .{full_path}),
669 }
670 }
671 }
672
673 return result;
248fn info(comptime format: []const u8, args: anytype) void {
249 const msg = std.fmt.allocPrint(gpa, "info: " ++ format ++ "\n", args) catch return;
250 std.io.getStdOut().writeAll(msg) catch {};
674251}
675252
676fn copyDirAll(source: fs.Dir, dest: fs.Dir) anyerror!void {
677 var it = source.iterate();
678 while (try it.next()) |next| {
679 switch (next.kind) {
680 .directory => {
681 var sub_dir = try dest.makeOpenPath(next.name, .{});
682 var sub_source = try source.openDir(next.name, .{ .iterate = true });
683 defer {
684 sub_dir.close();
685 sub_source.close();
686 }
687 try copyDirAll(sub_source, sub_dir);
688 },
689 .file => {
690 var source_file = try source.openFile(next.name, .{});
691 var dest_file = try dest.createFile(next.name, .{});
692 defer {
693 source_file.close();
694 dest_file.close();
695 }
696 const stat = try source_file.stat();
697 const ncopied = try source_file.copyRangeAll(0, dest_file, 0, stat.size);
698 assert(ncopied == stat.size);
699 },
700 else => |kind| info("unexpected file kind '{s}' will be ignored", .{@tagName(kind)}),
701 }
253fn fatal(comptime format: []const u8, args: anytype) noreturn {
254 ret: {
255 const msg = std.fmt.allocPrint(gpa, "fatal: " ++ format ++ "\n", args) catch break :ret;
256 std.io.getStdErr().writeAll(msg) catch {};
702257 }
258 std.process.exit(1);
703259}
704260
705261const Version = struct {