1//! To get started, run this tool with no args and read the help message.
2//!
3//! The build system of Linux requires specifying a single target
4//! architecture. Meanwhile, Zig supports out-of-the-box cross compilation for
5//! every target. So the process to create libc headers that Zig ships is to use
6//! this tool.
7//!
8//! First, use the Linux build systems to create installations of all the
9//! targets in the `linux_targets` variable.
10//!
11//! Next, run this tool to create a new directory which puts .h files into
12//! <arch> subdirectories, with `any-linux-any` being files that apply to
13//! all architectures.
14//!
15//! You'll then have to manually update Zig source repo with these new files.
16
17const std = @import("std");
18const Io = std.Io;
19const Dir = std.Io.Dir;
20const Arch = std.Target.Cpu.Arch;
21const Abi = std.Target.Abi;
22const assert = std.debug.assert;
23const Blake3 = std.crypto.hash.Blake3;
24
25const LibCTarget = struct {
26 name: []const u8,
27 arch: MultiArch,
28};
29
30const MultiArch = union(enum) {
31 arm,
32 arm64,
33 loongarch,
34 mips,
35 powerpc,
36 riscv,
37 sparc,
38 x86,
39 specific: Arch,
40
41 fn eql(a: MultiArch, b: MultiArch) bool {
42 if (@backingInt(a) != @backingInt(b))
43 return false;
44 if (a != .specific)
45 return true;
46 return a.specific == b.specific;
47 }
48};
49
50const linux_targets = [_]LibCTarget{
51 LibCTarget{
52 .name = "arc",
53 .arch = MultiArch{ .specific = Arch.arc },
54 },
55 LibCTarget{
56 .name = "arm",
57 .arch = .arm,
58 },
59 LibCTarget{
60 .name = "arm64",
61 .arch = .{ .specific = .aarch64 },
62 },
63 LibCTarget{
64 .name = "csky",
65 .arch = .{ .specific = .csky },
66 },
67 LibCTarget{
68 .name = "hexagon",
69 .arch = .{ .specific = .hexagon },
70 },
71 LibCTarget{
72 .name = "m68k",
73 .arch = .{ .specific = .m68k },
74 },
75 LibCTarget{
76 .name = "loongarch",
77 .arch = .loongarch,
78 },
79 LibCTarget{
80 .name = "mips",
81 .arch = .mips,
82 },
83 LibCTarget{
84 .name = "powerpc",
85 .arch = .powerpc,
86 },
87 LibCTarget{
88 .name = "riscv",
89 .arch = .riscv,
90 },
91 LibCTarget{
92 .name = "s390",
93 .arch = .{ .specific = .s390x },
94 },
95 LibCTarget{
96 .name = "sparc",
97 .arch = .{ .specific = .sparc },
98 },
99 LibCTarget{
100 .name = "x86",
101 .arch = .x86,
102 },
103 LibCTarget{
104 .name = "xtensa",
105 .arch = .{ .specific = .xtensa },
106 },
107};
108
109const DestTarget = struct {
110 arch: MultiArch,
111
112 const HashContext = struct {
113 pub fn hash(self: @This(), a: DestTarget) u32 {
114 _ = self;
115 var hasher = std.hash.Wyhash.init(0);
116 std.hash.autoHash(&hasher, a.arch);
117 return @as(u32, @truncate(hasher.final()));
118 }
119
120 pub fn eql(self: @This(), a: DestTarget, b: DestTarget, b_index: usize) bool {
121 _ = self;
122 _ = b_index;
123 return a.arch.eql(b.arch);
124 }
125 };
126};
127
128const Contents = struct {
129 bytes: []const u8,
130 hit_count: usize,
131 hash: []const u8,
132 is_generic: bool,
133
134 fn hitCountLessThan(context: void, lhs: *const Contents, rhs: *const Contents) bool {
135 _ = context;
136 return lhs.hit_count < rhs.hit_count;
137 }
138};
139
140const HashToContents = std.StringHashMap(Contents);
141const TargetToHash = std.array_hash_map.Custom(DestTarget, []const u8, DestTarget.HashContext, true);
142const PathTable = std.StringHashMap(*TargetToHash);
143
144pub fn main(init: std.process.Init) !void {
145 const arena = init.arena.allocator();
146 const io = init.io;
147 const args = try init.minimal.args.toSlice(arena);
148 const environ_map = init.environ_map;
149 const cwd = try std.process.currentPathAlloc(io, arena);
150
151 var search_paths = std.array_list.Managed([]const u8).init(arena);
152 var opt_out_dir: ?[]const u8 = null;
153
154 var arg_i: usize = 1;
155 while (arg_i < args.len) : (arg_i += 1) {
156 if (std.mem.eql(u8, args[arg_i], "--help"))
157 usageAndExit(args[0]);
158 if (arg_i + 1 >= args.len) {
159 std.debug.print("expected argument after '{s}'\n", .{args[arg_i]});
160 usageAndExit(args[0]);
161 }
162
163 if (std.mem.eql(u8, args[arg_i], "--search-path")) {
164 try search_paths.append(args[arg_i + 1]);
165 } else if (std.mem.eql(u8, args[arg_i], "--out")) {
166 assert(opt_out_dir == null);
167 opt_out_dir = args[arg_i + 1];
168 } else {
169 std.debug.print("unrecognized argument: {s}\n", .{args[arg_i]});
170 usageAndExit(args[0]);
171 }
172
173 arg_i += 1;
174 }
175
176 const out_dir = opt_out_dir orelse usageAndExit(args[0]);
177 const generic_name = "any-linux-any";
178
179 var path_table = PathTable.init(arena);
180 var hash_to_contents = HashToContents.init(arena);
181 var max_bytes_saved: usize = 0;
182 var total_bytes: usize = 0;
183
184 var hasher = Blake3.init(.{});
185
186 for (linux_targets) |linux_target| {
187 const dest_target = DestTarget{
188 .arch = linux_target.arch,
189 };
190 search: for (search_paths.items) |search_path| {
191 const target_include_dir = try Dir.path.join(arena, &.{
192 search_path, linux_target.name, "include",
193 });
194 var dir_stack = std.array_list.Managed([]const u8).init(arena);
195 try dir_stack.append(target_include_dir);
196
197 while (dir_stack.pop()) |full_dir_name| {
198 var dir = Dir.cwd().openDir(io, full_dir_name, .{ .iterate = true }) catch |err| switch (err) {
199 error.FileNotFound => continue :search,
200 error.AccessDenied => continue :search,
201 else => return err,
202 };
203 defer dir.close(io);
204
205 var dir_it = dir.iterate();
206
207 while (try dir_it.next(io)) |entry| {
208 const full_path = try Dir.path.join(arena, &[_][]const u8{ full_dir_name, entry.name });
209 switch (entry.kind) {
210 .directory => try dir_stack.append(full_path),
211 .file => {
212 const rel_path = try Dir.path.relative(arena, cwd, environ_map, target_include_dir, full_path);
213 const max_size = 2 * 1024 * 1024 * 1024;
214 const raw_bytes = try Dir.cwd().readFileAlloc(io, full_path, arena, .limited(max_size));
215 const trimmed = std.mem.trim(u8, raw_bytes, " \r\n\t");
216 total_bytes += raw_bytes.len;
217 const hash = try arena.alloc(u8, 32);
218 hasher = Blake3.init(.{});
219 hasher.update(rel_path);
220 hasher.update(trimmed);
221 hasher.final(hash);
222 const gop = try hash_to_contents.getOrPut(hash);
223 if (gop.found_existing) {
224 max_bytes_saved += raw_bytes.len;
225 gop.value_ptr.hit_count += 1;
226 std.debug.print("duplicate: {s} {s} ({B})\n", .{
227 linux_target.name,
228 rel_path,
229 raw_bytes.len,
230 });
231 } else {
232 gop.value_ptr.* = Contents{
233 .bytes = trimmed,
234 .hit_count = 1,
235 .hash = hash,
236 .is_generic = false,
237 };
238 }
239 const path_gop = try path_table.getOrPut(rel_path);
240 const target_to_hash = if (path_gop.found_existing) path_gop.value_ptr.* else blk: {
241 const ptr = try arena.create(TargetToHash);
242 ptr.* = .empty;
243 path_gop.value_ptr.* = ptr;
244 break :blk ptr;
245 };
246 try target_to_hash.putNoClobber(arena, dest_target, hash);
247 },
248 else => std.debug.print("warning: weird file: {s}\n", .{full_path}),
249 }
250 }
251 }
252 break;
253 } else {
254 std.debug.print("warning: libc target not found: {s}\n", .{linux_target.name});
255 }
256 }
257 std.debug.print("summary: {B} could be reduced to {B}\n", .{
258 total_bytes,
259 total_bytes - max_bytes_saved,
260 });
261 try Dir.cwd().createDirPath(io, out_dir);
262
263 var missed_opportunity_bytes: usize = 0;
264 // iterate path_table. for each path, put all the hashes into a list. sort by hit_count.
265 // the hash with the highest hit_count gets to be the "generic" one. everybody else
266 // gets their header in a separate arch directory.
267 var path_it = path_table.iterator();
268 while (path_it.next()) |path_kv| {
269 var contents_list = std.array_list.Managed(*Contents).init(arena);
270 {
271 var hash_it = path_kv.value_ptr.*.iterator();
272 while (hash_it.next()) |hash_kv| {
273 const contents = hash_to_contents.getPtr(hash_kv.value_ptr.*).?;
274 try contents_list.append(contents);
275 }
276 }
277 std.mem.sort(*Contents, contents_list.items, {}, Contents.hitCountLessThan);
278 const best_contents = contents_list.pop().?;
279 if (best_contents.hit_count > 1) {
280 // worth it to make it generic
281 const full_path = try Dir.path.join(arena, &[_][]const u8{ out_dir, generic_name, path_kv.key_ptr.* });
282 try Dir.cwd().createDirPath(io, Dir.path.dirname(full_path).?);
283 try Dir.cwd().writeFile(io, .{ .sub_path = full_path, .data = best_contents.bytes });
284 best_contents.is_generic = true;
285 while (contents_list.pop()) |contender| {
286 if (contender.hit_count > 1) {
287 const this_missed_bytes = contender.hit_count * contender.bytes.len;
288 missed_opportunity_bytes += this_missed_bytes;
289 std.debug.print("Missed opportunity ({B}): {s}\n", .{
290 this_missed_bytes,
291 path_kv.key_ptr.*,
292 });
293 } else break;
294 }
295 }
296 var hash_it = path_kv.value_ptr.*.iterator();
297 while (hash_it.next()) |hash_kv| {
298 const contents = hash_to_contents.get(hash_kv.value_ptr.*).?;
299 if (contents.is_generic) continue;
300
301 const dest_target = hash_kv.key_ptr.*;
302 const arch_name = switch (dest_target.arch) {
303 .specific => |a| @tagName(a),
304 else => @tagName(dest_target.arch),
305 };
306 const out_subpath = try std.fmt.allocPrint(arena, "{s}-linux-any", .{arch_name});
307 const full_path = try Dir.path.join(arena, &[_][]const u8{ out_dir, out_subpath, path_kv.key_ptr.* });
308 try Dir.cwd().createDirPath(io, Dir.path.dirname(full_path).?);
309 try Dir.cwd().writeFile(io, .{ .sub_path = full_path, .data = contents.bytes });
310 }
311 }
312
313 const bad_files = [_][]const u8{
314 "any-linux-any/linux/netfilter/xt_CONNMARK.h",
315 "any-linux-any/linux/netfilter/xt_DSCP.h",
316 "any-linux-any/linux/netfilter/xt_MARK.h",
317 "any-linux-any/linux/netfilter/xt_RATEEST.h",
318 "any-linux-any/linux/netfilter/xt_TCPMSS.h",
319 "any-linux-any/linux/netfilter_ipv4/ipt_ECN.h",
320 "any-linux-any/linux/netfilter_ipv4/ipt_TTL.h",
321 "any-linux-any/linux/netfilter_ipv6/ip6t_HL.h",
322 };
323 for (bad_files) |bad_file| {
324 const full_path = try Dir.path.join(arena, &[_][]const u8{ out_dir, bad_file });
325 try Dir.cwd().deleteFile(io, full_path);
326 }
327}
328
329fn usageAndExit(arg0: []const u8) noreturn {
330 std.debug.print("Usage: {s} [--search-path <dir>] --out <dir> --abi <name>\n", .{arg0});
331 std.debug.print("--search-path can be used any number of times.\n", .{});
332 std.debug.print(" subdirectories of search paths look like, e.g. x86_64-linux-gnu\n", .{});
333 std.debug.print("--out is a dir that will be created, and populated with the results\n", .{});
334 std.process.exit(1);
335}