authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-06-29 17:29:59-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2021-06-29 17:29:59-04:00
log37fbf5b0d3b0be131903e4895ee3703393b32d8f
treed646900dc75b5863b776d117dd43e6c8005130dc
parent2c385e58f96ea080bd6c732de422f84c6c38c2a2
parente32530b6a31432e43cc4d4d793796007423f2edb
signature Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #9258 from ziglang/shared-cache-locking

Shared Cache Locking

7 files changed, 487 insertions(+), 157 deletions(-)

doc/langref.html.in+8
......@@ -10713,6 +10713,14 @@ fn readU32Be() u32 {}
1071310713 See the Zig Standard Library for more examples.
1071410714 </p>
1071510715 {#header_close#}
10716 {#header_open|Doc Comment Guidance#}
10717 <ul>
10718 <li>Omit any information that is redundant based on the name of the thing being documented.</li>
10719 <li>Duplicating information onto multiple similar functions is encouraged because it helps IDEs and other tools provide better help text.</li>
10720 <li>Use the word <strong>assume</strong> to indicate invariants that cause {#link|Undefined Behavior#} when violated.</li>
10721 <li>Use the word <strong>assert</strong> to indicate invariants that cause <em>safety-checked</em> {#link|Undefined Behavior#} when violated.</li>
10722 </ul>
10723 {#header_close#}
1071610724 {#header_close#}
1071710725 {#header_open|Source Encoding#}
1071810726 <p>Zig source code is encoded in UTF-8. An invalid UTF-8 byte sequence results in a compile error.</p>
lib/std/fs.zig+47-17
......@@ -883,24 +883,39 @@ pub const Dir = struct {
883883 /// [WTF-16](https://simonsapin.github.io/wtf-8/#potentially-ill-formed-utf-16) encoded.
884884 pub fn openFileW(self: Dir, sub_path_w: []const u16, flags: File.OpenFlags) File.OpenError!File {
885885 const w = os.windows;
886 return @as(File, .{
887 .handle = try os.windows.OpenFile(sub_path_w, .{
886 const file: File = .{
887 .handle = try w.OpenFile(sub_path_w, .{
888888 .dir = self.fd,
889889 .access_mask = w.SYNCHRONIZE |
890890 (if (flags.read) @as(u32, w.GENERIC_READ) else 0) |
891891 (if (flags.write) @as(u32, w.GENERIC_WRITE) else 0),
892 .share_access = switch (flags.lock) {
893 .None => w.FILE_SHARE_WRITE | w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
894 .Shared => w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
895 .Exclusive => w.FILE_SHARE_DELETE,
896 },
897 .share_access_nonblocking = flags.lock_nonblocking,
898892 .creation = w.FILE_OPEN,
899893 .io_mode = flags.intended_io_mode,
900894 }),
901895 .capable_io_mode = std.io.default_mode,
902896 .intended_io_mode = flags.intended_io_mode,
903 });
897 };
898 var io: w.IO_STATUS_BLOCK = undefined;
899 const range_off: w.LARGE_INTEGER = 0;
900 const range_len: w.LARGE_INTEGER = 1;
901 const exclusive = switch (flags.lock) {
902 .None => return file,
903 .Shared => false,
904 .Exclusive => true,
905 };
906 try w.LockFile(
907 file.handle,
908 null,
909 null,
910 null,
911 &io,
912 &range_off,
913 &range_len,
914 null,
915 @boolToInt(flags.lock_nonblocking),
916 @boolToInt(exclusive),
917 );
918 return file;
904919 }
905920
906921 /// Creates, opens, or overwrites a file with write access.
......@@ -1019,16 +1034,10 @@ pub const Dir = struct {
10191034 pub fn createFileW(self: Dir, sub_path_w: []const u16, flags: File.CreateFlags) File.OpenError!File {
10201035 const w = os.windows;
10211036 const read_flag = if (flags.read) @as(u32, w.GENERIC_READ) else 0;
1022 return @as(File, .{
1037 const file: File = .{
10231038 .handle = try os.windows.OpenFile(sub_path_w, .{
10241039 .dir = self.fd,
10251040 .access_mask = w.SYNCHRONIZE | w.GENERIC_WRITE | read_flag,
1026 .share_access = switch (flags.lock) {
1027 .None => w.FILE_SHARE_WRITE | w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
1028 .Shared => w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
1029 .Exclusive => w.FILE_SHARE_DELETE,
1030 },
1031 .share_access_nonblocking = flags.lock_nonblocking,
10321041 .creation = if (flags.exclusive)
10331042 @as(u32, w.FILE_CREATE)
10341043 else if (flags.truncate)
......@@ -1039,7 +1048,28 @@ pub const Dir = struct {
10391048 }),
10401049 .capable_io_mode = std.io.default_mode,
10411050 .intended_io_mode = flags.intended_io_mode,
1042 });
1051 };
1052 var io: w.IO_STATUS_BLOCK = undefined;
1053 const range_off: w.LARGE_INTEGER = 0;
1054 const range_len: w.LARGE_INTEGER = 1;
1055 const exclusive = switch (flags.lock) {
1056 .None => return file,
1057 .Shared => false,
1058 .Exclusive => true,
1059 };
1060 try w.LockFile(
1061 file.handle,
1062 null,
1063 null,
1064 null,
1065 &io,
1066 &range_off,
1067 &range_len,
1068 null,
1069 @boolToInt(flags.lock_nonblocking),
1070 @boolToInt(exclusive),
1071 );
1072 return file;
10431073 }
10441074
10451075 pub const openRead = @compileError("deprecated in favor of openFile");
lib/std/fs/file.zig+197-14
......@@ -74,17 +74,28 @@ pub const File = struct {
7474 read: bool = true,
7575 write: bool = false,
7676
77 /// Open the file with a lock to prevent other processes from accessing it at the
78 /// same time. An exclusive lock will prevent other processes from acquiring a lock.
79 /// A shared lock will prevent other processes from acquiring a exclusive lock, but
80 /// doesn't prevent other process from getting their own shared locks.
77 /// Open the file with an advisory lock to coordinate with other processes
78 /// accessing it at the same time. An exclusive lock will prevent other
79 /// processes from acquiring a lock. A shared lock will prevent other
80 /// processes from acquiring a exclusive lock, but does not prevent
81 /// other process from getting their own shared locks.
8182 ///
82 /// Note that the lock is only advisory on Linux, except in very specific cirsumstances[1].
83 /// The lock is advisory, except on Linux in very specific cirsumstances[1].
8384 /// This means that a process that does not respect the locking API can still get access
8485 /// to the file, despite the lock.
8586 ///
86 /// Windows' file locks are mandatory, and any process attempting to access the file will
87 /// receive an error.
87 /// On these operating systems, the lock is acquired atomically with
88 /// opening the file:
89 /// * Darwin
90 /// * DragonFlyBSD
91 /// * FreeBSD
92 /// * Haiku
93 /// * NetBSD
94 /// * OpenBSD
95 /// On these operating systems, the lock is acquired via a separate syscall
96 /// after opening the file:
97 /// * Linux
98 /// * Windows
8899 ///
89100 /// [1]: https://www.kernel.org/doc/Documentation/filesystems/mandatory-locking.txt
90101 lock: Lock = .None,
......@@ -120,17 +131,28 @@ pub const File = struct {
120131 /// `error.PathAlreadyExists` to be returned.
121132 exclusive: bool = false,
122133
123 /// Open the file with a lock to prevent other processes from accessing it at the
124 /// same time. An exclusive lock will prevent other processes from acquiring a lock.
125 /// A shared lock will prevent other processes from acquiring a exclusive lock, but
126 /// doesn't prevent other process from getting their own shared locks.
134 /// Open the file with an advisory lock to coordinate with other processes
135 /// accessing it at the same time. An exclusive lock will prevent other
136 /// processes from acquiring a lock. A shared lock will prevent other
137 /// processes from acquiring a exclusive lock, but does not prevent
138 /// other process from getting their own shared locks.
127139 ///
128 /// Note that the lock is only advisory on Linux, except in very specific cirsumstances[1].
140 /// The lock is advisory, except on Linux in very specific cirsumstances[1].
129141 /// This means that a process that does not respect the locking API can still get access
130142 /// to the file, despite the lock.
131143 ///
132 /// Windows's file locks are mandatory, and any process attempting to access the file will
133 /// receive an error.
144 /// On these operating systems, the lock is acquired atomically with
145 /// opening the file:
146 /// * Darwin
147 /// * DragonFlyBSD
148 /// * FreeBSD
149 /// * Haiku
150 /// * NetBSD
151 /// * OpenBSD
152 /// On these operating systems, the lock is acquired via a separate syscall
153 /// after opening the file:
154 /// * Linux
155 /// * Windows
134156 ///
135157 /// [1]: https://www.kernel.org/doc/Documentation/filesystems/mandatory-locking.txt
136158 lock: Lock = .None,
......@@ -829,4 +851,165 @@ pub const File = struct {
829851 pub fn seekableStream(file: File) SeekableStream {
830852 return .{ .context = file };
831853 }
854
855 const range_off: windows.LARGE_INTEGER = 0;
856 const range_len: windows.LARGE_INTEGER = 1;
857
858 pub const LockError = error{
859 SystemResources,
860 } || os.UnexpectedError;
861
862 /// Blocks when an incompatible lock is held by another process.
863 /// A process may hold only one type of lock (shared or exclusive) on
864 /// a file. When a process terminates in any way, the lock is released.
865 ///
866 /// Assumes the file is unlocked.
867 ///
868 /// TODO: integrate with async I/O
869 pub fn lock(file: File, l: Lock) LockError!void {
870 if (is_windows) {
871 var io_status_block: windows.IO_STATUS_BLOCK = undefined;
872 const exclusive = switch (l) {
873 .None => return,
874 .Shared => false,
875 .Exclusive => true,
876 };
877 return windows.LockFile(
878 file.handle,
879 null,
880 null,
881 null,
882 &io_status_block,
883 &range_off,
884 &range_len,
885 null,
886 windows.FALSE, // non-blocking=false
887 @boolToInt(exclusive),
888 ) catch |err| switch (err) {
889 error.WouldBlock => unreachable, // non-blocking=false
890 else => |e| return e,
891 };
892 } else {
893 return os.flock(file.handle, switch (l) {
894 .None => os.LOCK_UN,
895 .Shared => os.LOCK_SH,
896 .Exclusive => os.LOCK_EX,
897 }) catch |err| switch (err) {
898 error.WouldBlock => unreachable, // non-blocking=false
899 else => |e| return e,
900 };
901 }
902 }
903
904 /// Assumes the file is locked.
905 pub fn unlock(file: File) void {
906 if (is_windows) {
907 var io_status_block: windows.IO_STATUS_BLOCK = undefined;
908 return windows.UnlockFile(
909 file.handle,
910 &io_status_block,
911 &range_off,
912 &range_len,
913 null,
914 ) catch |err| switch (err) {
915 error.RangeNotLocked => unreachable, // Function assumes unlocked.
916 error.Unexpected => unreachable, // Resource deallocation must succeed.
917 };
918 } else {
919 return os.flock(file.handle, os.LOCK_UN) catch |err| switch (err) {
920 error.WouldBlock => unreachable, // unlocking can't block
921 error.SystemResources => unreachable, // We are deallocating resources.
922 error.Unexpected => unreachable, // Resource deallocation must succeed.
923 };
924 }
925 }
926
927 /// Attempts to obtain a lock, returning `true` if the lock is
928 /// obtained, and `false` if there was an existing incompatible lock held.
929 /// A process may hold only one type of lock (shared or exclusive) on
930 /// a file. When a process terminates in any way, the lock is released.
931 ///
932 /// Assumes the file is unlocked.
933 ///
934 /// TODO: integrate with async I/O
935 pub fn tryLock(file: File, l: Lock) LockError!bool {
936 if (is_windows) {
937 var io_status_block: windows.IO_STATUS_BLOCK = undefined;
938 const exclusive = switch (l) {
939 .None => return,
940 .Shared => false,
941 .Exclusive => true,
942 };
943 windows.LockFile(
944 file.handle,
945 null,
946 null,
947 null,
948 &io_status_block,
949 &range_off,
950 &range_len,
951 null,
952 windows.TRUE, // non-blocking=true
953 @boolToInt(exclusive),
954 ) catch |err| switch (err) {
955 error.WouldBlock => return false,
956 else => |e| return e,
957 };
958 } else {
959 os.flock(file.handle, switch (l) {
960 .None => os.LOCK_UN,
961 .Shared => os.LOCK_SH | os.LOCK_NB,
962 .Exclusive => os.LOCK_EX | os.LOCK_NB,
963 }) catch |err| switch (err) {
964 error.WouldBlock => return false,
965 else => |e| return e,
966 };
967 }
968 return true;
969 }
970
971 /// Assumes the file is already locked in exclusive mode.
972 /// Atomically modifies the lock to be in shared mode, without releasing it.
973 ///
974 /// TODO: integrate with async I/O
975 pub fn downgradeLock(file: File) LockError!void {
976 if (is_windows) {
977 // On Windows it works like a semaphore + exclusivity flag. To implement this
978 // function, we first obtain another lock in shared mode. This changes the
979 // exclusivity flag, but increments the semaphore to 2. So we follow up with
980 // an NtUnlockFile which decrements the semaphore but does not modify the
981 // exclusivity flag.
982 var io_status_block: windows.IO_STATUS_BLOCK = undefined;
983 windows.LockFile(
984 file.handle,
985 null,
986 null,
987 null,
988 &io_status_block,
989 &range_off,
990 &range_len,
991 null,
992 windows.TRUE, // non-blocking=true
993 windows.FALSE, // exclusive=false
994 ) catch |err| switch (err) {
995 error.WouldBlock => unreachable, // File was not locked in exclusive mode.
996 else => |e| return e,
997 };
998 return windows.UnlockFile(
999 file.handle,
1000 &io_status_block,
1001 &range_off,
1002 &range_len,
1003 null,
1004 ) catch |err| switch (err) {
1005 error.RangeNotLocked => unreachable, // File was not locked.
1006 error.Unexpected => unreachable, // Resource deallocation must succeed.
1007 };
1008 } else {
1009 return os.flock(file.handle, os.LOCK_SH | os.LOCK_NB) catch |err| switch (err) {
1010 error.WouldBlock => unreachable, // File was not locked in exclusive mode.
1011 else => |e| return e,
1012 };
1013 }
1014 }
8321015};
lib/std/os/windows.zig+91-50
......@@ -49,7 +49,6 @@ pub const OpenFileOptions = struct {
4949 dir: ?HANDLE = null,
5050 sa: ?*SECURITY_ATTRIBUTES = null,
5151 share_access: ULONG = FILE_SHARE_WRITE | FILE_SHARE_READ | FILE_SHARE_DELETE,
52 share_access_nonblocking: bool = false,
5352 creation: ULONG,
5453 io_mode: std.io.ModeOverride,
5554 /// If true, tries to open path as a directory.
......@@ -60,8 +59,6 @@ pub const OpenFileOptions = struct {
6059 follow_symlinks: bool = true,
6160};
6261
63/// TODO when share_access_nonblocking is false, this implementation uses
64/// untinterruptible sleep() to block. This is not the final iteration of the API.
6562pub fn OpenFile(sub_path_w: []const u16, options: OpenFileOptions) OpenError!HANDLE {
6663 if (mem.eql(u16, sub_path_w, &[_]u16{'.'}) and !options.open_dir) {
6764 return error.IsDir;
......@@ -94,53 +91,39 @@ pub fn OpenFile(sub_path_w: []const u16, options: OpenFileOptions) OpenError!HAN
9491 // If we're not following symlinks, we need to ensure we don't pass in any synchronization flags such as FILE_SYNCHRONOUS_IO_NONALERT.
9592 const flags: ULONG = if (options.follow_symlinks) file_or_dir_flag | blocking_flag else file_or_dir_flag | FILE_OPEN_REPARSE_POINT;
9693
97 var delay: usize = 1;
98 while (true) {
99 const rc = ntdll.NtCreateFile(
100 &result,
101 options.access_mask,
102 &attr,
103 &io,
104 null,
105 FILE_ATTRIBUTE_NORMAL,
106 options.share_access,
107 options.creation,
108 flags,
109 null,
110 0,
111 );
112 switch (rc) {
113 .SUCCESS => {
114 if (std.io.is_async and options.io_mode == .evented) {
115 _ = CreateIoCompletionPort(result, std.event.Loop.instance.?.os_data.io_port, undefined, undefined) catch undefined;
116 }
117 return result;
118 },
119 .OBJECT_NAME_INVALID => unreachable,
120 .OBJECT_NAME_NOT_FOUND => return error.FileNotFound,
121 .OBJECT_PATH_NOT_FOUND => return error.FileNotFound,
122 .NO_MEDIA_IN_DEVICE => return error.NoDevice,
123 .INVALID_PARAMETER => unreachable,
124 .SHARING_VIOLATION => {
125 if (options.share_access_nonblocking) {
126 return error.WouldBlock;
127 }
128 // TODO sleep in a way that is interruptable
129 // TODO integrate with async I/O
130 std.time.sleep(delay);
131 if (delay < 1 * std.time.ns_per_s) {
132 delay *= 2;
133 }
134 continue;
135 },
136 .ACCESS_DENIED => return error.AccessDenied,
137 .PIPE_BUSY => return error.PipeBusy,
138 .OBJECT_PATH_SYNTAX_BAD => unreachable,
139 .OBJECT_NAME_COLLISION => return error.PathAlreadyExists,
140 .FILE_IS_A_DIRECTORY => return error.IsDir,
141 .NOT_A_DIRECTORY => return error.NotDir,
142 else => return unexpectedStatus(rc),
143 }
94 const rc = ntdll.NtCreateFile(
95 &result,
96 options.access_mask,
97 &attr,
98 &io,
99 null,
100 FILE_ATTRIBUTE_NORMAL,
101 options.share_access,
102 options.creation,
103 flags,
104 null,
105 0,
106 );
107 switch (rc) {
108 .SUCCESS => {
109 if (std.io.is_async and options.io_mode == .evented) {
110 _ = CreateIoCompletionPort(result, std.event.Loop.instance.?.os_data.io_port, undefined, undefined) catch undefined;
111 }
112 return result;
113 },
114 .OBJECT_NAME_INVALID => unreachable,
115 .OBJECT_NAME_NOT_FOUND => return error.FileNotFound,
116 .OBJECT_PATH_NOT_FOUND => return error.FileNotFound,
117 .NO_MEDIA_IN_DEVICE => return error.NoDevice,
118 .INVALID_PARAMETER => unreachable,
119 .SHARING_VIOLATION => return error.AccessDenied,
120 .ACCESS_DENIED => return error.AccessDenied,
121 .PIPE_BUSY => return error.PipeBusy,
122 .OBJECT_PATH_SYNTAX_BAD => unreachable,
123 .OBJECT_NAME_COLLISION => return error.PathAlreadyExists,
124 .FILE_IS_A_DIRECTORY => return error.IsDir,
125 .NOT_A_DIRECTORY => return error.NotDir,
126 else => return unexpectedStatus(rc),
144127 }
145128}
146129
......@@ -1679,6 +1662,64 @@ pub fn SetFileTime(
16791662 }
16801663}
16811664
1665pub const LockFileError = error{
1666 SystemResources,
1667 WouldBlock,
1668} || std.os.UnexpectedError;
1669
1670pub fn LockFile(
1671 FileHandle: HANDLE,
1672 Event: ?HANDLE,
1673 ApcRoutine: ?*IO_APC_ROUTINE,
1674 ApcContext: ?*c_void,
1675 IoStatusBlock: *IO_STATUS_BLOCK,
1676 ByteOffset: *const LARGE_INTEGER,
1677 Length: *const LARGE_INTEGER,
1678 Key: ?*ULONG,
1679 FailImmediately: BOOLEAN,
1680 ExclusiveLock: BOOLEAN,
1681) !void {
1682 const rc = ntdll.NtLockFile(
1683 FileHandle,
1684 Event,
1685 ApcRoutine,
1686 ApcContext,
1687 IoStatusBlock,
1688 ByteOffset,
1689 Length,
1690 Key,
1691 FailImmediately,
1692 ExclusiveLock,
1693 );
1694 switch (rc) {
1695 .SUCCESS => return,
1696 .INSUFFICIENT_RESOURCES => return error.SystemResources,
1697 .LOCK_NOT_GRANTED => return error.WouldBlock,
1698 .ACCESS_VIOLATION => unreachable, // bad io_status_block pointer
1699 else => return unexpectedStatus(rc),
1700 }
1701}
1702
1703pub const UnlockFileError = error{
1704 RangeNotLocked,
1705} || std.os.UnexpectedError;
1706
1707pub fn UnlockFile(
1708 FileHandle: HANDLE,
1709 IoStatusBlock: *IO_STATUS_BLOCK,
1710 ByteOffset: *const LARGE_INTEGER,
1711 Length: *const LARGE_INTEGER,
1712 Key: ?*ULONG,
1713) !void {
1714 const rc = ntdll.NtUnlockFile(FileHandle, IoStatusBlock, ByteOffset, Length, Key);
1715 switch (rc) {
1716 .SUCCESS => return,
1717 .RANGE_NOT_LOCKED => return error.RangeNotLocked,
1718 .ACCESS_VIOLATION => unreachable, // bad io_status_block pointer
1719 else => return unexpectedStatus(rc),
1720 }
1721}
1722
16821723pub fn teb() *TEB {
16831724 return switch (builtin.target.cpu.arch) {
16841725 .i386 => asm volatile (
lib/std/os/windows/ntdll.zig+21
......@@ -139,3 +139,24 @@ pub extern "NtDll" fn RtlWaitOnAddress(
139139 AddressSize: SIZE_T,
140140 Timeout: ?*const LARGE_INTEGER,
141141) callconv(WINAPI) NTSTATUS;
142
143pub extern "NtDll" fn NtLockFile(
144 FileHandle: HANDLE,
145 Event: ?HANDLE,
146 ApcRoutine: ?*IO_APC_ROUTINE,
147 ApcContext: ?*c_void,
148 IoStatusBlock: *IO_STATUS_BLOCK,
149 ByteOffset: *const LARGE_INTEGER,
150 Length: *const LARGE_INTEGER,
151 Key: ?*ULONG,
152 FailImmediately: BOOLEAN,
153 ExclusiveLock: BOOLEAN,
154) callconv(WINAPI) NTSTATUS;
155
156pub extern "NtDll" fn NtUnlockFile(
157 FileHandle: HANDLE,
158 IoStatusBlock: *IO_STATUS_BLOCK,
159 ByteOffset: *const LARGE_INTEGER,
160 Length: *const LARGE_INTEGER,
161 Key: ?*ULONG,
162) callconv(WINAPI) NTSTATUS;
src/Cache.zig+123-54
......@@ -181,6 +181,12 @@ pub const Manifest = struct {
181181 hash: HashHelper,
182182 manifest_file: ?fs.File,
183183 manifest_dirty: bool,
184 /// Set this flag to true before calling hit() in order to indicate that
185 /// upon a cache hit, the code using the cache will not modify the files
186 /// within the cache directory. This allows multiple processes to utilize
187 /// the same cache directory at the same time.
188 want_shared_lock: bool = true,
189 have_exclusive_lock: bool = false,
184190 files: std.ArrayListUnmanaged(File) = .{},
185191 hex_digest: [hex_digest_len]u8,
186192 /// Populated when hit() returns an error because of one
......@@ -257,7 +263,9 @@ pub const Manifest = struct {
257263 ///
258264 /// This function will also acquire an exclusive lock to the manifest file. This means
259265 /// that a process holding a Manifest will block any other process attempting to
260 /// acquire the lock.
266 /// acquire the lock. If `want_shared_lock` is `true`, a cache hit guarantees the
267 /// manifest file to be locked in shared mode, and a cache miss guarantees the manifest
268 /// file to be locked in exclusive mode.
261269 ///
262270 /// The lock on the manifest file is released when `deinit` is called. As another
263271 /// option, one may call `toOwnedLock` to obtain a smaller object which can represent
......@@ -285,31 +293,62 @@ pub const Manifest = struct {
285293 mem.copy(u8, &manifest_file_path, &self.hex_digest);
286294 manifest_file_path[self.hex_digest.len..][0..ext.len].* = ext.*;
287295
288 if (self.files.items.len != 0) {
289 self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{
290 .read = true,
291 .truncate = false,
292 .lock = .Exclusive,
293 });
294 } else {
296 if (self.files.items.len == 0) {
295297 // If there are no file inputs, we check if the manifest file exists instead of
296298 // comparing the hashes on the files used for the cached item
297 self.manifest_file = self.cache.manifest_dir.openFile(&manifest_file_path, .{
299 while (true) {
300 if (self.cache.manifest_dir.openFile(&manifest_file_path, .{
301 .read = true,
302 .write = true,
303 .lock = .Exclusive,
304 .lock_nonblocking = self.want_shared_lock,
305 })) |manifest_file| {
306 self.manifest_file = manifest_file;
307 self.have_exclusive_lock = true;
308 break;
309 } else |open_err| switch (open_err) {
310 error.WouldBlock => {
311 self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{
312 .lock = .Shared,
313 });
314 break;
315 },
316 error.FileNotFound => {
317 if (self.cache.manifest_dir.createFile(&manifest_file_path, .{
318 .read = true,
319 .truncate = false,
320 .lock = .Exclusive,
321 .lock_nonblocking = self.want_shared_lock,
322 })) |manifest_file| {
323 self.manifest_file = manifest_file;
324 self.manifest_dirty = true;
325 self.have_exclusive_lock = true;
326 return false; // cache miss; exclusive lock already held
327 } else |err| switch (err) {
328 error.WouldBlock => continue,
329 else => |e| return e,
330 }
331 },
332 else => |e| return e,
333 }
334 }
335 } else {
336 if (self.cache.manifest_dir.createFile(&manifest_file_path, .{
298337 .read = true,
299 .write = true,
338 .truncate = false,
300339 .lock = .Exclusive,
301 }) catch |err| switch (err) {
302 error.FileNotFound => {
303 self.manifest_dirty = true;
304 self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{
305 .read = true,
306 .truncate = false,
307 .lock = .Exclusive,
340 .lock_nonblocking = self.want_shared_lock,
341 })) |manifest_file| {
342 self.manifest_file = manifest_file;
343 self.have_exclusive_lock = true;
344 } else |err| switch (err) {
345 error.WouldBlock => {
346 self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{
347 .lock = .Shared,
308348 });
309 return false;
310349 },
311350 else => |e| return e,
312 };
351 }
313352 }
314353
315354 const file_contents = try self.manifest_file.?.reader().readAllAlloc(self.cache.gpa, manifest_file_size_max);
......@@ -360,7 +399,10 @@ pub const Manifest = struct {
360399 }
361400
362401 const this_file = fs.cwd().openFile(cache_hash_file.path.?, .{ .read = true }) catch |err| switch (err) {
363 error.FileNotFound => return false,
402 error.FileNotFound => {
403 try self.upgradeToExclusiveLock();
404 return false;
405 },
364406 else => return error.CacheUnavailable,
365407 };
366408 defer this_file.close();
......@@ -405,6 +447,7 @@ pub const Manifest = struct {
405447 // cache miss
406448 // keep the manifest file open
407449 self.unhit(bin_digest, input_file_count);
450 try self.upgradeToExclusiveLock();
408451 return false;
409452 }
410453
......@@ -417,9 +460,11 @@ pub const Manifest = struct {
417460 return err;
418461 };
419462 }
463 try self.upgradeToExclusiveLock();
420464 return false;
421465 }
422466
467 try self.downgradeToSharedLock();
423468 return true;
424469 }
425470
......@@ -585,34 +630,58 @@ pub const Manifest = struct {
585630 return out_digest;
586631 }
587632
633 /// If `want_shared_lock` is true, this function automatically downgrades the
634 /// lock from exclusive to shared.
588635 pub fn writeManifest(self: *Manifest) !void {
589636 const manifest_file = self.manifest_file.?;
590 if (!self.manifest_dirty) return;
591
592 var contents = std.ArrayList(u8).init(self.cache.gpa);
593 defer contents.deinit();
637 if (self.manifest_dirty) {
638 self.manifest_dirty = false;
639
640 var contents = std.ArrayList(u8).init(self.cache.gpa);
641 defer contents.deinit();
642
643 const writer = contents.writer();
644 var encoded_digest: [hex_digest_len]u8 = undefined;
645
646 for (self.files.items) |file| {
647 _ = std.fmt.bufPrint(
648 &encoded_digest,
649 "{s}",
650 .{std.fmt.fmtSliceHexLower(&file.bin_digest)},
651 ) catch unreachable;
652 try writer.print("{d} {d} {d} {s} {s}\n", .{
653 file.stat.size,
654 file.stat.inode,
655 file.stat.mtime,
656 &encoded_digest,
657 file.path,
658 });
659 }
594660
595 const writer = contents.writer();
596 var encoded_digest: [hex_digest_len]u8 = undefined;
661 try manifest_file.setEndPos(contents.items.len);
662 try manifest_file.pwriteAll(contents.items, 0);
663 }
597664
598 for (self.files.items) |file| {
599 _ = std.fmt.bufPrint(
600 &encoded_digest,
601 "{s}",
602 .{std.fmt.fmtSliceHexLower(&file.bin_digest)},
603 ) catch unreachable;
604 try writer.print("{d} {d} {d} {s} {s}\n", .{
605 file.stat.size,
606 file.stat.inode,
607 file.stat.mtime,
608 &encoded_digest,
609 file.path,
610 });
665 if (self.want_shared_lock) {
666 try self.downgradeToSharedLock();
611667 }
668 }
669
670 fn downgradeToSharedLock(self: *Manifest) !void {
671 if (!self.have_exclusive_lock) return;
672 const manifest_file = self.manifest_file.?;
673 try manifest_file.downgradeLock();
674 self.have_exclusive_lock = false;
675 }
612676
613 try manifest_file.setEndPos(contents.items.len);
614 try manifest_file.pwriteAll(contents.items, 0);
615 self.manifest_dirty = false;
677 fn upgradeToExclusiveLock(self: *Manifest) !void {
678 if (self.have_exclusive_lock) return;
679 const manifest_file = self.manifest_file.?;
680 // Here we intentionally have a period where the lock is released, in case there are
681 // other processes holding a shared lock.
682 manifest_file.unlock();
683 try manifest_file.lock(.Exclusive);
684 self.have_exclusive_lock = true;
616685 }
617686
618687 /// Obtain only the data needed to maintain a lock on the manifest file.
......@@ -881,27 +950,27 @@ test "no file inputs" {
881950 defer cache.manifest_dir.close();
882951
883952 {
884 var ch = cache.obtain();
885 defer ch.deinit();
953 var man = cache.obtain();
954 defer man.deinit();
886955
887 ch.hash.addBytes("1234");
956 man.hash.addBytes("1234");
888957
889958 // There should be nothing in the cache
890 try testing.expectEqual(false, try ch.hit());
959 try testing.expectEqual(false, try man.hit());
891960
892 digest1 = ch.final();
961 digest1 = man.final();
893962
894 try ch.writeManifest();
963 try man.writeManifest();
895964 }
896965 {
897 var ch = cache.obtain();
898 defer ch.deinit();
966 var man = cache.obtain();
967 defer man.deinit();
899968
900 ch.hash.addBytes("1234");
969 man.hash.addBytes("1234");
901970
902 try testing.expect(try ch.hit());
903 digest2 = ch.final();
904 try ch.writeManifest();
971 try testing.expect(try man.hit());
972 digest2 = man.final();
973 try man.writeManifest();
905974 }
906975
907976 try testing.expectEqual(digest1, digest2);
src/Compilation.zig-22
......@@ -39,7 +39,6 @@ gpa: *Allocator,
3939arena_state: std.heap.ArenaAllocator.State,
4040bin_file: *link.File,
4141c_object_table: std.AutoArrayHashMapUnmanaged(*CObject, void) = .{},
42c_object_cache_digest_set: std.AutoHashMapUnmanaged(Cache.BinDigest, void) = .{},
4342stage1_lock: ?Cache.Lock = null,
4443stage1_cache_manifest: *Cache.Manifest = undefined,
4544
......@@ -1590,7 +1589,6 @@ pub fn destroy(self: *Compilation) void {
15901589 key.destroy(gpa);
15911590 }
15921591 self.c_object_table.deinit(gpa);
1593 self.c_object_cache_digest_set.deinit(gpa);
15941592
15951593 for (self.failed_c_objects.values()) |value| {
15961594 value.destroy(gpa);
......@@ -1627,7 +1625,6 @@ pub fn update(self: *Compilation) !void {
16271625 defer tracy.end();
16281626
16291627 self.clearMiscFailures();
1630 self.c_object_cache_digest_set.clearRetainingCapacity();
16311628
16321629 // For compiling C objects, we rely on the cache hash system to avoid duplicating work.
16331630 // Add a Job for each C object.
......@@ -2615,25 +2612,6 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: *std.P
26152612
26162613 try man.hashCSource(c_object.src);
26172614
2618 {
2619 const is_collision = blk: {
2620 const bin_digest = man.hash.peekBin();
2621
2622 const lock = comp.mutex.acquire();
2623 defer lock.release();
2624
2625 const gop = try comp.c_object_cache_digest_set.getOrPut(comp.gpa, bin_digest);
2626 break :blk gop.found_existing;
2627 };
2628 if (is_collision) {
2629 return comp.failCObj(
2630 c_object,
2631 "the same source file was already added to the same compilation with the same flags",
2632 .{},
2633 );
2634 }
2635 }
2636
26372615 var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa);
26382616 defer arena_allocator.deinit();
26392617 const arena = &arena_allocator.allocator;