authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-05-03 18:10:33-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-05-03 21:03:29-07:00
log16d368d0d28b1654dae5e405b9e6e067f833e1cf
treea9a2377f4789ec3a80fab9778c32cc923baae002
parenta72292513e378159c542a785ca69f8111bacbdcf

introduce std.process.raiseFileDescriptorLimit

2 files changed, 48 insertions(+), 51 deletions(-)

lib/std/process.zig+46
...@@ -1742,3 +1742,49 @@ pub fn cleanExit() void {...@@ -1742,3 +1742,49 @@ pub fn cleanExit() void {
1742 exit(0);1742 exit(0);
1743 }1743 }
1744}1744}
1745
1746/// Raise the open file descriptor limit.
1747///
1748/// On some systems, this raises the limit before seeing ProcessFdQuotaExceeded
1749/// errors. On other systems, this does nothing.
1750pub fn raiseFileDescriptorLimit() void {
1751 const have_rlimit = switch (native_os) {
1752 .windows, .wasi => false,
1753 else => true,
1754 };
1755 if (!have_rlimit) return;
1756
1757 var lim = posix.getrlimit(.NOFILE) catch return; // Oh well; we tried.
1758 if (native_os.isDarwin()) {
1759 // On Darwin, `NOFILE` is bounded by a hardcoded value `OPEN_MAX`.
1760 // According to the man pages for setrlimit():
1761 // setrlimit() now returns with errno set to EINVAL in places that historically succeeded.
1762 // It no longer accepts "rlim_cur = RLIM.INFINITY" for RLIM.NOFILE.
1763 // Use "rlim_cur = min(OPEN_MAX, rlim_max)".
1764 lim.max = @min(std.c.OPEN_MAX, lim.max);
1765 }
1766 if (lim.cur == lim.max) return;
1767
1768 // Do a binary search for the limit.
1769 var min: posix.rlim_t = lim.cur;
1770 var max: posix.rlim_t = 1 << 20;
1771 // But if there's a defined upper bound, don't search, just set it.
1772 if (lim.max != posix.RLIM.INFINITY) {
1773 min = lim.max;
1774 max = lim.max;
1775 }
1776
1777 while (true) {
1778 lim.cur = min + @divTrunc(max - min, 2); // on freebsd rlim_t is signed
1779 if (posix.setrlimit(.NOFILE, lim)) |_| {
1780 min = lim.cur;
1781 } else |_| {
1782 max = lim.cur;
1783 }
1784 if (min + 1 >= max) break;
1785 }
1786}
1787
1788test raiseFileDescriptorLimit {
1789 raiseFileDescriptorLimit();
1790}
src/main.zig+2-51
...@@ -3197,7 +3197,7 @@ fn buildOutputType(...@@ -3197,7 +3197,7 @@ fn buildOutputType(
3197 break :b .incremental;3197 break :b .incremental;
3198 };3198 };
31993199
3200 gimmeMoreOfThoseSweetSweetFileDescriptors();3200 process.raiseFileDescriptorLimit();
32013201
3202 const comp = Compilation.create(gpa, arena, .{3202 const comp = Compilation.create(gpa, arena, .{
3203 .zig_lib_directory = zig_lib_directory,3203 .zig_lib_directory = zig_lib_directory,
...@@ -4908,7 +4908,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {...@@ -4908,7 +4908,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
4908 .basename = exe_basename,4908 .basename = exe_basename,
4909 };4909 };
49104910
4911 gimmeMoreOfThoseSweetSweetFileDescriptors();4911 process.raiseFileDescriptorLimit();
49124912
4913 var zig_lib_directory: Compilation.Directory = if (override_lib_dir) |lib_dir| .{4913 var zig_lib_directory: Compilation.Directory = if (override_lib_dir) |lib_dir| .{
4914 .path = lib_dir,4914 .path = lib_dir,
...@@ -5973,55 +5973,6 @@ fn parseCodeModel(arg: []const u8) std.builtin.CodeModel {...@@ -5973,55 +5973,6 @@ fn parseCodeModel(arg: []const u8) std.builtin.CodeModel {
5973 fatal("unsupported machine code model: '{s}'", .{arg});5973 fatal("unsupported machine code model: '{s}'", .{arg});
5974}5974}
59755975
5976/// Raise the open file descriptor limit. Ask and ye shall receive.
5977/// For one example of why this is handy, consider the case of building musl libc.
5978/// We keep a lock open for each of the object files in the form of a file descriptor
5979/// until they are finally put into an archive file. This is to allow a zig-cache
5980/// garbage collector to run concurrently to zig processes, and to allow multiple
5981/// zig processes to run concurrently with each other, without clobbering each other.
5982fn gimmeMoreOfThoseSweetSweetFileDescriptors() void {
5983 const have_rlimit = switch (native_os) {
5984 .windows, .wasi => false,
5985 else => true,
5986 };
5987 if (!have_rlimit) return;
5988 const posix = std.posix;
5989
5990 var lim = posix.getrlimit(.NOFILE) catch return; // Oh well; we tried.
5991 if (native_os.isDarwin()) {
5992 // On Darwin, `NOFILE` is bounded by a hardcoded value `OPEN_MAX`.
5993 // According to the man pages for setrlimit():
5994 // setrlimit() now returns with errno set to EINVAL in places that historically succeeded.
5995 // It no longer accepts "rlim_cur = RLIM.INFINITY" for RLIM.NOFILE.
5996 // Use "rlim_cur = min(OPEN_MAX, rlim_max)".
5997 lim.max = @min(std.c.OPEN_MAX, lim.max);
5998 }
5999 if (lim.cur == lim.max) return;
6000
6001 // Do a binary search for the limit.
6002 var min: posix.rlim_t = lim.cur;
6003 var max: posix.rlim_t = 1 << 20;
6004 // But if there's a defined upper bound, don't search, just set it.
6005 if (lim.max != posix.RLIM.INFINITY) {
6006 min = lim.max;
6007 max = lim.max;
6008 }
6009
6010 while (true) {
6011 lim.cur = min + @divTrunc(max - min, 2); // on freebsd rlim_t is signed
6012 if (posix.setrlimit(.NOFILE, lim)) |_| {
6013 min = lim.cur;
6014 } else |_| {
6015 max = lim.cur;
6016 }
6017 if (min + 1 >= max) break;
6018 }
6019}
6020
6021test "fds" {
6022 gimmeMoreOfThoseSweetSweetFileDescriptors();
6023}
6024
6025const usage_ast_check =5976const usage_ast_check =
6026 \\Usage: zig ast-check [file]5977 \\Usage: zig ast-check [file]
6027 \\5978 \\