authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-10-01 13:43:25-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-10-01 13:50:55-04:00
logaf229c1fdc89f69273f69627ab5a0304dab11572
treed49bd02b57017ce751c999cd8e6e970e15ad6ee4
parentd1ec8377d1fcc9874c40e6603f64087f0b310677
signature Commit is signed but in an unrecognized format.

std lib (breaking): posixRead can return less than buffer size

closes #1414 std.io.InStream.read now can return less than buffer size introduce std.io.InStream.readFull for previous behavior add std.os.File.openWriteNoClobberC rename std.os.deleteFileWindows to std.os.deleteFileW remove std.os.deleteFilePosix add std.os.deleteFileC std.os.copyFile no longer takes an allocator std.os.copyFileMode no longer takes an allocator std.os.AtomicFile no longer takes an allocator add std.os.renameW add windows support for std.os.renameC add a test for std.os.AtomicFile

7 files changed, 201 insertions(+), 139 deletions(-)

std/build.zig+1-1
...@@ -634,7 +634,7 @@ pub const Builder = struct {...@@ -634,7 +634,7 @@ pub const Builder = struct {
634 warn("Unable to create path {}: {}\n", dirname, @errorName(err));634 warn("Unable to create path {}: {}\n", dirname, @errorName(err));
635 return err;635 return err;
636 };636 };
637 os.copyFileMode(self.allocator, abs_source_path, dest_path, mode) catch |err| {637 os.copyFileMode(abs_source_path, dest_path, mode) catch |err| {
638 warn("Unable to copy {} to {}: {}\n", abs_source_path, dest_path, @errorName(err));638 warn("Unable to copy {} to {}: {}\n", abs_source_path, dest_path, @errorName(err));
639 return err;639 return err;
640 };640 };
std/event/io.zig+20-12
...@@ -21,6 +21,24 @@ pub fn InStream(comptime ReadError: type) type {...@@ -21,6 +21,24 @@ pub fn InStream(comptime ReadError: type) type {
21 return await (async self.readFn(self, buffer) catch unreachable);21 return await (async self.readFn(self, buffer) catch unreachable);
22 }22 }
2323
24 /// Return the number of bytes read. If it is less than buffer.len
25 /// it means end of stream.
26 pub async fn readFull(self: *Self, buffer: []u8) !usize {
27 var index: usize = 0;
28 while (index != buf.len) {
29 const amt_read = try await (async self.read(buf[index..]) catch unreachable);
30 if (amt_read == 0) return index;
31 index += amt_read;
32 }
33 return index;
34 }
35
36 /// Same as `readFull` but end of stream returns `error.EndOfStream`.
37 pub async fn readNoEof(self: *Self, buf: []u8) !void {
38 const amt_read = try await (async self.readFull(buf[index..]) catch unreachable);
39 if (amt_read < buf.len) return error.EndOfStream;
40 }
41
24 pub async fn readIntLe(self: *Self, comptime T: type) !T {42 pub async fn readIntLe(self: *Self, comptime T: type) !T {
25 return await (async self.readInt(builtin.Endian.Little, T) catch unreachable);43 return await (async self.readInt(builtin.Endian.Little, T) catch unreachable);
26 }44 }
...@@ -31,24 +49,14 @@ pub fn InStream(comptime ReadError: type) type {...@@ -31,24 +49,14 @@ pub fn InStream(comptime ReadError: type) type {
3149
32 pub async fn readInt(self: *Self, endian: builtin.Endian, comptime T: type) !T {50 pub async fn readInt(self: *Self, endian: builtin.Endian, comptime T: type) !T {
33 var bytes: [@sizeOf(T)]u8 = undefined;51 var bytes: [@sizeOf(T)]u8 = undefined;
34 try await (async self.readFull(bytes[0..]) catch unreachable);52 try await (async self.readNoEof(bytes[0..]) catch unreachable);
35 return mem.readInt(bytes, T, endian);53 return mem.readInt(bytes, T, endian);
36 }54 }
3755
38 /// Same as `read` but end of stream returns `error.EndOfStream`.
39 pub async fn readFull(self: *Self, buf: []u8) !void {
40 var index: usize = 0;
41 while (index != buf.len) {
42 const amt_read = try await (async self.read(buf[index..]) catch unreachable);
43 if (amt_read == 0) return error.EndOfStream;
44 index += amt_read;
45 }
46 }
47
48 pub async fn readStruct(self: *Self, comptime T: type, ptr: *T) !void {56 pub async fn readStruct(self: *Self, comptime T: type, ptr: *T) !void {
49 // Only extern and packed structs have defined in-memory layout.57 // Only extern and packed structs have defined in-memory layout.
50 comptime assert(@typeInfo(T).Struct.layout != builtin.TypeInfo.ContainerLayout.Auto);58 comptime assert(@typeInfo(T).Struct.layout != builtin.TypeInfo.ContainerLayout.Auto);
51 return await (async self.readFull(@sliceToBytes((*[1]T)(ptr)[0..])) catch unreachable);59 return await (async self.readNoEof(@sliceToBytes((*[1]T)(ptr)[0..])) catch unreachable);
52 }60 }
53 };61 };
54}62}
std/io.zig+31-7
...@@ -51,7 +51,7 @@ pub fn InStream(comptime ReadError: type) type {...@@ -51,7 +51,7 @@ pub fn InStream(comptime ReadError: type) type {
51 var actual_buf_len: usize = 0;51 var actual_buf_len: usize = 0;
52 while (true) {52 while (true) {
53 const dest_slice = buffer.toSlice()[actual_buf_len..];53 const dest_slice = buffer.toSlice()[actual_buf_len..];
54 const bytes_read = try self.readFn(self, dest_slice);54 const bytes_read = try self.readFull(dest_slice);
55 actual_buf_len += bytes_read;55 actual_buf_len += bytes_read;
5656
57 if (bytes_read != dest_slice.len) {57 if (bytes_read != dest_slice.len) {
...@@ -111,14 +111,27 @@ pub fn InStream(comptime ReadError: type) type {...@@ -111,14 +111,27 @@ pub fn InStream(comptime ReadError: type) type {
111 return buf.toOwnedSlice();111 return buf.toOwnedSlice();
112 }112 }
113113
114 /// Returns the number of bytes read. It may be less than buffer.len.
115 /// If the number of bytes read is 0, it means end of stream.
116 /// End of stream is not an error condition.
117 pub fn read(self: *Self, buffer: []u8) !usize {
118 return self.readFn(self, buffer);
119 }
120
114 /// Returns the number of bytes read. If the number read is smaller than buf.len, it121 /// Returns the number of bytes read. If the number read is smaller than buf.len, it
115 /// means the stream reached the end. Reaching the end of a stream is not an error122 /// means the stream reached the end. Reaching the end of a stream is not an error
116 /// condition.123 /// condition.
117 pub fn read(self: *Self, buffer: []u8) !usize {124 pub fn readFull(self: *Self, buffer: []u8) !usize {
118 return self.readFn(self, buffer);125 var index: usize = 0;
126 while (index != buffer.len) {
127 const amt = try self.read(buffer[index..]);
128 if (amt == 0) return index;
129 index += amt;
130 }
131 return index;
119 }132 }
120133
121 /// Same as `read` but end of stream returns `error.EndOfStream`.134 /// Same as `readFull` but end of stream returns `error.EndOfStream`.
122 pub fn readNoEof(self: *Self, buf: []u8) !void {135 pub fn readNoEof(self: *Self, buf: []u8) !void {
123 const amt_read = try self.read(buf);136 const amt_read = try self.read(buf);
124 if (amt_read < buf.len) return error.EndOfStream;137 if (amt_read < buf.len) return error.EndOfStream;
...@@ -136,6 +149,11 @@ pub fn InStream(comptime ReadError: type) type {...@@ -136,6 +149,11 @@ pub fn InStream(comptime ReadError: type) type {
136 return @bitCast(i8, try self.readByte());149 return @bitCast(i8, try self.readByte());
137 }150 }
138151
152 /// Reads a native-endian integer
153 pub fn readIntNe(self: *Self, comptime T: type) !T {
154 return self.readInt(builtin.endian, T);
155 }
156
139 pub fn readIntLe(self: *Self, comptime T: type) !T {157 pub fn readIntLe(self: *Self, comptime T: type) !T {
140 return self.readInt(builtin.Endian.Little, T);158 return self.readInt(builtin.Endian.Little, T);
141 }159 }
...@@ -202,6 +220,11 @@ pub fn OutStream(comptime WriteError: type) type {...@@ -202,6 +220,11 @@ pub fn OutStream(comptime WriteError: type) type {
202 }220 }
203 }221 }
204222
223 /// Write a native-endian integer.
224 pub fn writeIntNe(self: *Self, comptime T: type, value: T) !void {
225 return self.writeInt(builtin.endian, T, value);
226 }
227
205 pub fn writeIntLe(self: *Self, comptime T: type, value: T) !void {228 pub fn writeIntLe(self: *Self, comptime T: type, value: T) !void {
206 return self.writeInt(builtin.Endian.Little, T, value);229 return self.writeInt(builtin.Endian.Little, T, value);
207 }230 }
...@@ -537,6 +560,7 @@ pub const BufferedAtomicFile = struct {...@@ -537,6 +560,7 @@ pub const BufferedAtomicFile = struct {
537 atomic_file: os.AtomicFile,560 atomic_file: os.AtomicFile,
538 file_stream: os.File.OutStream,561 file_stream: os.File.OutStream,
539 buffered_stream: BufferedOutStream(os.File.WriteError),562 buffered_stream: BufferedOutStream(os.File.WriteError),
563 allocator: *mem.Allocator,
540564
541 pub fn create(allocator: *mem.Allocator, dest_path: []const u8) !*BufferedAtomicFile {565 pub fn create(allocator: *mem.Allocator, dest_path: []const u8) !*BufferedAtomicFile {
542 // TODO with well defined copy elision we don't need this allocation566 // TODO with well defined copy elision we don't need this allocation
...@@ -544,10 +568,11 @@ pub const BufferedAtomicFile = struct {...@@ -544,10 +568,11 @@ pub const BufferedAtomicFile = struct {
544 .atomic_file = undefined,568 .atomic_file = undefined,
545 .file_stream = undefined,569 .file_stream = undefined,
546 .buffered_stream = undefined,570 .buffered_stream = undefined,
571 .allocator = allocator,
547 });572 });
548 errdefer allocator.destroy(self);573 errdefer allocator.destroy(self);
549574
550 self.atomic_file = try os.AtomicFile.init(allocator, dest_path, os.File.default_mode);575 self.atomic_file = try os.AtomicFile.init(dest_path, os.File.default_mode);
551 errdefer self.atomic_file.deinit();576 errdefer self.atomic_file.deinit();
552577
553 self.file_stream = self.atomic_file.file.outStream();578 self.file_stream = self.atomic_file.file.outStream();
...@@ -557,9 +582,8 @@ pub const BufferedAtomicFile = struct {...@@ -557,9 +582,8 @@ pub const BufferedAtomicFile = struct {
557582
558 /// always call destroy, even after successful finish()583 /// always call destroy, even after successful finish()
559 pub fn destroy(self: *BufferedAtomicFile) void {584 pub fn destroy(self: *BufferedAtomicFile) void {
560 const allocator = self.atomic_file.allocator;
561 self.atomic_file.deinit();585 self.atomic_file.deinit();
562 allocator.destroy(self);586 self.allocator.destroy(self);
563 }587 }
564588
565 pub fn finish(self: *BufferedAtomicFile) !void {589 pub fn finish(self: *BufferedAtomicFile) !void {
std/os/child_process.zig+4-6
...@@ -792,13 +792,11 @@ fn forkChildErrReport(fd: i32, err: ChildProcess.SpawnError) noreturn {...@@ -792,13 +792,11 @@ fn forkChildErrReport(fd: i32, err: ChildProcess.SpawnError) noreturn {
792const ErrInt = @IntType(false, @sizeOf(error) * 8);792const ErrInt = @IntType(false, @sizeOf(error) * 8);
793793
794fn writeIntFd(fd: i32, value: ErrInt) !void {794fn writeIntFd(fd: i32, value: ErrInt) !void {
795 var bytes: [@sizeOf(ErrInt)]u8 = undefined;795 const stream = &os.File.openHandle(fd).outStream().stream;
796 mem.writeInt(bytes[0..], value, builtin.endian);796 stream.writeIntNe(ErrInt, value) catch return error.SystemResources;
797 os.posixWrite(fd, bytes[0..]) catch return error.SystemResources;
798}797}
799798
800fn readIntFd(fd: i32) !ErrInt {799fn readIntFd(fd: i32) !ErrInt {
801 var bytes: [@sizeOf(ErrInt)]u8 = undefined;800 const stream = &os.File.openHandle(fd).inStream().stream;
802 os.posixRead(fd, bytes[0..]) catch return error.SystemResources;801 return stream.readIntNe(ErrInt) catch return error.SystemResources;
803 return mem.readInt(bytes[0..], ErrInt, builtin.endian);
804}802}
std/os/file.zig+16-25
...@@ -102,12 +102,24 @@ pub const File = struct {...@@ -102,12 +102,24 @@ pub const File = struct {
102 /// If a file already exists in the destination this returns OpenError.PathAlreadyExists102 /// If a file already exists in the destination this returns OpenError.PathAlreadyExists
103 /// Call close to clean up.103 /// Call close to clean up.
104 pub fn openWriteNoClobber(path: []const u8, file_mode: Mode) OpenError!File {104 pub fn openWriteNoClobber(path: []const u8, file_mode: Mode) OpenError!File {
105 if (is_posix) {
106 const path_c = try os.toPosixPath(path);
107 return openWriteNoClobberC(path_c, file_mode);
108 } else if (is_windows) {
109 const path_w = try windows_util.sliceToPrefixedFileW(path);
110 return openWriteNoClobberW(&path_w, file_mode);
111 } else {
112 @compileError("TODO implement openWriteMode for this OS");
113 }
114 }
115
116 pub fn openWriteNoClobberC(path: [*]const u8, file_mode: Mode) OpenError!File {
105 if (is_posix) {117 if (is_posix) {
106 const flags = posix.O_LARGEFILE | posix.O_WRONLY | posix.O_CREAT | posix.O_CLOEXEC | posix.O_EXCL;118 const flags = posix.O_LARGEFILE | posix.O_WRONLY | posix.O_CREAT | posix.O_CLOEXEC | posix.O_EXCL;
107 const fd = try os.posixOpen(path, flags, file_mode);119 const fd = try os.posixOpenC(path, flags, file_mode);
108 return openHandle(fd);120 return openHandle(fd);
109 } else if (is_windows) {121 } else if (is_windows) {
110 const path_w = try windows_util.sliceToPrefixedFileW(path);122 const path_w = try windows_util.cStrToPrefixedFileW(path);
111 return openWriteNoClobberW(&path_w, file_mode);123 return openWriteNoClobberW(&path_w, file_mode);
112 } else {124 } else {
113 @compileError("TODO implement openWriteMode for this OS");125 @compileError("TODO implement openWriteMode for this OS");
...@@ -369,28 +381,7 @@ pub const File = struct {...@@ -369,28 +381,7 @@ pub const File = struct {
369381
370 pub fn read(self: File, buffer: []u8) ReadError!usize {382 pub fn read(self: File, buffer: []u8) ReadError!usize {
371 if (is_posix) {383 if (is_posix) {
372 var index: usize = 0;384 return os.posixRead(self.handle, buffer);
373 while (index < buffer.len) {
374 const amt_read = posix.read(self.handle, buffer.ptr + index, buffer.len - index);
375 const read_err = posix.getErrno(amt_read);
376 if (read_err > 0) {
377 switch (read_err) {
378 posix.EINTR => continue,
379 posix.EINVAL => unreachable,
380 posix.EFAULT => unreachable,
381 posix.EAGAIN => unreachable,
382 posix.EBADF => unreachable, // always a race condition
383 posix.EIO => return error.InputOutput,
384 posix.EISDIR => return error.IsDir,
385 posix.ENOBUFS => return error.SystemResources,
386 posix.ENOMEM => return error.SystemResources,
387 else => return os.unexpectedErrorPosix(read_err),
388 }
389 }
390 if (amt_read == 0) return index;
391 index += amt_read;
392 }
393 return index;
394 } else if (is_windows) {385 } else if (is_windows) {
395 var index: usize = 0;386 var index: usize = 0;
396 while (index < buffer.len) {387 while (index < buffer.len) {
...@@ -409,7 +400,7 @@ pub const File = struct {...@@ -409,7 +400,7 @@ pub const File = struct {
409 }400 }
410 return index;401 return index;
411 } else {402 } else {
412 unreachable;403 @compileError("Unsupported OS");
413 }404 }
414 }405 }
415406
std/os/index.zig+108-88
...@@ -104,30 +104,17 @@ pub fn getRandomBytes(buf: []u8) !void {...@@ -104,30 +104,17 @@ pub fn getRandomBytes(buf: []u8) !void {
104 Os.linux => while (true) {104 Os.linux => while (true) {
105 // TODO check libc version and potentially call c.getrandom.105 // TODO check libc version and potentially call c.getrandom.
106 // See #397106 // See #397
107 const err = posix.getErrno(posix.getrandom(buf.ptr, buf.len, 0));107 const errno = posix.getErrno(posix.getrandom(buf.ptr, buf.len, 0));
108 if (err > 0) {108 switch (errno) {
109 switch (err) {109 0 => return,
110 posix.EINVAL => unreachable,110 posix.EINVAL => unreachable,
111 posix.EFAULT => unreachable,111 posix.EFAULT => unreachable,
112 posix.EINTR => continue,112 posix.EINTR => continue,
113 posix.ENOSYS => {113 posix.ENOSYS => return getRandomBytesDevURandom(buf),
114 const fd = try posixOpenC(c"/dev/urandom", posix.O_RDONLY | posix.O_CLOEXEC, 0);114 else => return unexpectedErrorPosix(errno),
115 defer close(fd);
116
117 try posixRead(fd, buf);
118 return;
119 },
120 else => return unexpectedErrorPosix(err),
121 }
122 }115 }
123 return;
124 },
125 Os.macosx, Os.ios => {
126 const fd = try posixOpenC(c"/dev/urandom", posix.O_RDONLY | posix.O_CLOEXEC, 0);
127 defer close(fd);
128
129 try posixRead(fd, buf);
130 },116 },
117 Os.macosx, Os.ios => return getRandomBytesDevURandom(buf),
131 Os.windows => {118 Os.windows => {
132 // Call RtlGenRandom() instead of CryptGetRandom() on Windows119 // Call RtlGenRandom() instead of CryptGetRandom() on Windows
133 // https://github.com/rust-lang-nursery/rand/issues/111120 // https://github.com/rust-lang-nursery/rand/issues/111
...@@ -151,6 +138,22 @@ pub fn getRandomBytes(buf: []u8) !void {...@@ -151,6 +138,22 @@ pub fn getRandomBytes(buf: []u8) !void {
151 }138 }
152}139}
153140
141fn getRandomBytesDevURandom(buf: []u8) !void {
142 const fd = try posixOpenC(c"/dev/urandom", posix.O_RDONLY | posix.O_CLOEXEC, 0);
143 defer close(fd);
144
145 const stream = &File.openHandle(fd).inStream().stream;
146 stream.readNoEof(buf) catch |err| switch (err) {
147 error.EndOfStream => unreachable,
148 error.OperationAborted => unreachable,
149 error.BrokenPipe => unreachable,
150 error.Unexpected => return error.Unexpected,
151 error.InputOutput => return error.Unexpected,
152 error.SystemResources => return error.Unexpected,
153 error.IsDir => unreachable,
154 };
155}
156
154test "os.getRandomBytes" {157test "os.getRandomBytes" {
155 var buf_a: [50]u8 = undefined;158 var buf_a: [50]u8 = undefined;
156 var buf_b: [50]u8 = undefined;159 var buf_b: [50]u8 = undefined;
...@@ -235,8 +238,9 @@ pub const PosixReadError = error{...@@ -235,8 +238,9 @@ pub const PosixReadError = error{
235 Unexpected,238 Unexpected,
236};239};
237240
238/// Calls POSIX read, and keeps trying if it gets interrupted.241/// Returns the number of bytes that were read, which can be less than
239pub fn posixRead(fd: i32, buf: []u8) !void {242/// buf.len. If 0 bytes were read, that means EOF.
243pub fn posixRead(fd: i32, buf: []u8) PosixReadError!usize {
240 // Linux can return EINVAL when read amount is > 0x7ffff000244 // Linux can return EINVAL when read amount is > 0x7ffff000
241 // See https://github.com/ziglang/zig/pull/743#issuecomment-363158274245 // See https://github.com/ziglang/zig/pull/743#issuecomment-363158274
242 const max_buf_len = 0x7ffff000;246 const max_buf_len = 0x7ffff000;
...@@ -249,7 +253,9 @@ pub fn posixRead(fd: i32, buf: []u8) !void {...@@ -249,7 +253,9 @@ pub fn posixRead(fd: i32, buf: []u8) !void {
249 switch (err) {253 switch (err) {
250 0 => {254 0 => {
251 index += rc;255 index += rc;
252 continue;256 if (rc == want_to_read) continue;
257 // Read returned less than buf.len.
258 return index;
253 },259 },
254 posix.EINTR => continue,260 posix.EINTR => continue,
255 posix.EINVAL => unreachable,261 posix.EINVAL => unreachable,
...@@ -263,6 +269,7 @@ pub fn posixRead(fd: i32, buf: []u8) !void {...@@ -263,6 +269,7 @@ pub fn posixRead(fd: i32, buf: []u8) !void {
263 else => return unexpectedErrorPosix(err),269 else => return unexpectedErrorPosix(err),
264 }270 }
265 }271 }
272 return index;
266}273}
267274
268/// Number of bytes read is returned. Upon reading end-of-file, zero is returned.275/// Number of bytes read is returned. Upon reading end-of-file, zero is returned.
...@@ -962,16 +969,16 @@ pub const DeleteFileError = error{...@@ -962,16 +969,16 @@ pub const DeleteFileError = error{
962969
963pub fn deleteFile(file_path: []const u8) DeleteFileError!void {970pub fn deleteFile(file_path: []const u8) DeleteFileError!void {
964 if (builtin.os == Os.windows) {971 if (builtin.os == Os.windows) {
965 return deleteFileWindows(file_path);972 const file_path_w = try windows_util.sliceToPrefixedFileW(file_path);
973 return deleteFileW(&file_path_w);
966 } else {974 } else {
967 return deleteFilePosix(file_path);975 const file_path_c = try toPosixPath(file_path);
976 return deleteFileC(&file_path_c);
968 }977 }
969}978}
970979
971pub fn deleteFileWindows(file_path: []const u8) !void {980pub fn deleteFileW(file_path: [*]const u16) DeleteFileError!void {
972 const file_path_w = try windows_util.sliceToPrefixedFileW(file_path);981 if (windows.DeleteFileW(file_path) == 0) {
973
974 if (windows.DeleteFileW(&file_path_w) == 0) {
975 const err = windows.GetLastError();982 const err = windows.GetLastError();
976 switch (err) {983 switch (err) {
977 windows.ERROR.FILE_NOT_FOUND => return error.FileNotFound,984 windows.ERROR.FILE_NOT_FOUND => return error.FileNotFound,
...@@ -983,50 +990,49 @@ pub fn deleteFileWindows(file_path: []const u8) !void {...@@ -983,50 +990,49 @@ pub fn deleteFileWindows(file_path: []const u8) !void {
983 }990 }
984}991}
985992
986pub fn deleteFilePosixC(file_path: [*]const u8) !void {993pub fn deleteFileC(file_path: [*]const u8) DeleteFileError!void {
987 const err = posix.getErrno(posix.unlink(file_path));994 if (is_windows) {
988 switch (err) {995 const file_path_w = try windows_util.cStrToPrefixedFileW(file_path);
989 0 => return,996 return deleteFileW(&file_path_w);
990 posix.EACCES => return error.AccessDenied,997 } else {
991 posix.EPERM => return error.AccessDenied,998 const err = posix.getErrno(posix.unlink(file_path));
992 posix.EBUSY => return error.FileBusy,999 switch (err) {
993 posix.EFAULT => unreachable,1000 0 => return,
994 posix.EINVAL => unreachable,1001 posix.EACCES => return error.AccessDenied,
995 posix.EIO => return error.FileSystem,1002 posix.EPERM => return error.AccessDenied,
996 posix.EISDIR => return error.IsDir,1003 posix.EBUSY => return error.FileBusy,
997 posix.ELOOP => return error.SymLinkLoop,1004 posix.EFAULT => unreachable,
998 posix.ENAMETOOLONG => return error.NameTooLong,1005 posix.EINVAL => unreachable,
999 posix.ENOENT => return error.FileNotFound,1006 posix.EIO => return error.FileSystem,
1000 posix.ENOTDIR => return error.NotDir,1007 posix.EISDIR => return error.IsDir,
1001 posix.ENOMEM => return error.SystemResources,1008 posix.ELOOP => return error.SymLinkLoop,
1002 posix.EROFS => return error.ReadOnlyFileSystem,1009 posix.ENAMETOOLONG => return error.NameTooLong,
1003 else => return unexpectedErrorPosix(err),1010 posix.ENOENT => return error.FileNotFound,
1011 posix.ENOTDIR => return error.NotDir,
1012 posix.ENOMEM => return error.SystemResources,
1013 posix.EROFS => return error.ReadOnlyFileSystem,
1014 else => return unexpectedErrorPosix(err),
1015 }
1004 }1016 }
1005}1017}
10061018
1007pub fn deleteFilePosix(file_path: []const u8) !void {
1008 const file_path_c = try toPosixPath(file_path);
1009 return deleteFilePosixC(&file_path_c);
1010}
1011
1012/// Guaranteed to be atomic. However until https://patchwork.kernel.org/patch/9636735/ is1019/// Guaranteed to be atomic. However until https://patchwork.kernel.org/patch/9636735/ is
1013/// merged and readily available,1020/// merged and readily available,
1014/// there is a possibility of power loss or application termination leaving temporary files present1021/// there is a possibility of power loss or application termination leaving temporary files present
1015/// in the same directory as dest_path.1022/// in the same directory as dest_path.
1016/// Destination file will have the same mode as the source file.1023/// Destination file will have the same mode as the source file.
1017/// TODO investigate if this can work with no allocator1024pub fn copyFile(source_path: []const u8, dest_path: []const u8) !void {
1018pub fn copyFile(allocator: *Allocator, source_path: []const u8, dest_path: []const u8) !void {
1019 var in_file = try os.File.openRead(source_path);1025 var in_file = try os.File.openRead(source_path);
1020 defer in_file.close();1026 defer in_file.close();
10211027
1022 const mode = try in_file.mode();1028 const mode = try in_file.mode();
10231029
1024 var atomic_file = try AtomicFile.init(allocator, dest_path, mode);1030 var atomic_file = try AtomicFile.init(dest_path, mode);
1025 defer atomic_file.deinit();1031 defer atomic_file.deinit();
10261032
1027 var buf: [page_size]u8 = undefined;1033 var buf: [page_size]u8 = undefined;
1028 while (true) {1034 while (true) {
1029 const amt = try in_file.read(buf[0..]);1035 const amt = try in_file.readFull(buf[0..]);
1030 try atomic_file.file.write(buf[0..amt]);1036 try atomic_file.file.write(buf[0..amt]);
1031 if (amt != buf.len) {1037 if (amt != buf.len) {
1032 return atomic_file.finish();1038 return atomic_file.finish();
...@@ -1037,12 +1043,11 @@ pub fn copyFile(allocator: *Allocator, source_path: []const u8, dest_path: []con...@@ -1037,12 +1043,11 @@ pub fn copyFile(allocator: *Allocator, source_path: []const u8, dest_path: []con
1037/// Guaranteed to be atomic. However until https://patchwork.kernel.org/patch/9636735/ is1043/// Guaranteed to be atomic. However until https://patchwork.kernel.org/patch/9636735/ is
1038/// merged and readily available,1044/// merged and readily available,
1039/// there is a possibility of power loss or application termination leaving temporary files present1045/// there is a possibility of power loss or application termination leaving temporary files present
1040/// TODO investigate if this can work with no allocator1046pub fn copyFileMode(source_path: []const u8, dest_path: []const u8, mode: File.Mode) !void {
1041pub fn copyFileMode(allocator: *Allocator, source_path: []const u8, dest_path: []const u8, mode: File.Mode) !void {
1042 var in_file = try os.File.openRead(source_path);1047 var in_file = try os.File.openRead(source_path);
1043 defer in_file.close();1048 defer in_file.close();
10441049
1045 var atomic_file = try AtomicFile.init(allocator, dest_path, mode);1050 var atomic_file = try AtomicFile.init(dest_path, mode);
1046 defer atomic_file.deinit();1051 defer atomic_file.deinit();
10471052
1048 var buf: [page_size]u8 = undefined;1053 var buf: [page_size]u8 = undefined;
...@@ -1056,35 +1061,38 @@ pub fn copyFileMode(allocator: *Allocator, source_path: []const u8, dest_path: [...@@ -1056,35 +1061,38 @@ pub fn copyFileMode(allocator: *Allocator, source_path: []const u8, dest_path: [
1056}1061}
10571062
1058pub const AtomicFile = struct {1063pub const AtomicFile = struct {
1059 /// TODO investigate if we can make this work with no allocator
1060 allocator: *Allocator,
1061 file: os.File,1064 file: os.File,
1062 tmp_path: []u8,1065 tmp_path_buf: [MAX_PATH_BYTES]u8,
1063 dest_path: []const u8,1066 dest_path: []const u8,
1064 finished: bool,1067 finished: bool,
10651068
1069 const InitError = os.File.OpenError;
1070
1066 /// dest_path must remain valid for the lifetime of AtomicFile1071 /// dest_path must remain valid for the lifetime of AtomicFile
1067 /// call finish to atomically replace dest_path with contents1072 /// call finish to atomically replace dest_path with contents
1068 pub fn init(allocator: *Allocator, dest_path: []const u8, mode: File.Mode) !AtomicFile {1073 /// TODO once we have null terminated pointers, use the
1074 /// openWriteNoClobberN function
1075 pub fn init(dest_path: []const u8, mode: File.Mode) InitError!AtomicFile {
1069 const dirname = os.path.dirname(dest_path);1076 const dirname = os.path.dirname(dest_path);
1070
1071 var rand_buf: [12]u8 = undefined;1077 var rand_buf: [12]u8 = undefined;
1072
1073 const dirname_component_len = if (dirname) |d| d.len + 1 else 0;1078 const dirname_component_len = if (dirname) |d| d.len + 1 else 0;
1074 const tmp_path = try allocator.alloc(u8, dirname_component_len +1079 const encoded_rand_len = comptime base64.Base64Encoder.calcSize(rand_buf.len);
1075 base64.Base64Encoder.calcSize(rand_buf.len));1080 const tmp_path_len = dirname_component_len + encoded_rand_len;
1076 errdefer allocator.free(tmp_path);1081 var tmp_path_buf: [MAX_PATH_BYTES]u8 = undefined;
1082 if (tmp_path_len >= tmp_path_buf.len) return error.NameTooLong;
10771083
1078 if (dirname) |dir| {1084 if (dirname) |dir| {
1079 mem.copy(u8, tmp_path[0..], dir);1085 mem.copy(u8, tmp_path_buf[0..], dir);
1080 tmp_path[dir.len] = os.path.sep;1086 tmp_path_buf[dir.len] = os.path.sep;
1081 }1087 }
10821088
1089 tmp_path_buf[tmp_path_len] = 0;
1090
1083 while (true) {1091 while (true) {
1084 try getRandomBytes(rand_buf[0..]);1092 try getRandomBytes(rand_buf[0..]);
1085 b64_fs_encoder.encode(tmp_path[dirname_component_len..], rand_buf);1093 b64_fs_encoder.encode(tmp_path_buf[dirname_component_len..tmp_path_len], rand_buf);
10861094
1087 const file = os.File.openWriteNoClobber(tmp_path, mode) catch |err| switch (err) {1095 const file = os.File.openWriteNoClobberC(&tmp_path_buf, mode) catch |err| switch (err) {
1088 error.PathAlreadyExists => continue,1096 error.PathAlreadyExists => continue,
1089 // TODO zig should figure out that this error set does not include PathAlreadyExists since1097 // TODO zig should figure out that this error set does not include PathAlreadyExists since
1090 // it is handled in the above switch1098 // it is handled in the above switch
...@@ -1092,9 +1100,8 @@ pub const AtomicFile = struct {...@@ -1092,9 +1100,8 @@ pub const AtomicFile = struct {
1092 };1100 };
10931101
1094 return AtomicFile{1102 return AtomicFile{
1095 .allocator = allocator,
1096 .file = file,1103 .file = file,
1097 .tmp_path = tmp_path,1104 .tmp_path_buf = tmp_path_buf,
1098 .dest_path = dest_path,1105 .dest_path = dest_path,
1099 .finished = false,1106 .finished = false,
1100 };1107 };
...@@ -1105,8 +1112,7 @@ pub const AtomicFile = struct {...@@ -1105,8 +1112,7 @@ pub const AtomicFile = struct {
1105 pub fn deinit(self: *AtomicFile) void {1112 pub fn deinit(self: *AtomicFile) void {
1106 if (!self.finished) {1113 if (!self.finished) {
1107 self.file.close();1114 self.file.close();
1108 deleteFile(self.tmp_path) catch {};1115 deleteFileC(&self.tmp_path_buf) catch {};
1109 self.allocator.free(self.tmp_path);
1110 self.finished = true;1116 self.finished = true;
1111 }1117 }
1112 }1118 }
...@@ -1114,15 +1120,25 @@ pub const AtomicFile = struct {...@@ -1114,15 +1120,25 @@ pub const AtomicFile = struct {
1114 pub fn finish(self: *AtomicFile) !void {1120 pub fn finish(self: *AtomicFile) !void {
1115 assert(!self.finished);1121 assert(!self.finished);
1116 self.file.close();1122 self.file.close();
1117 try rename(self.tmp_path, self.dest_path);
1118 self.allocator.free(self.tmp_path);
1119 self.finished = true;1123 self.finished = true;
1124 if (is_posix) {
1125 const dest_path_c = try toPosixPath(self.dest_path);
1126 return renameC(&self.tmp_path_buf, &dest_path_c);
1127 } else if (is_windows) {
1128 const dest_path_w = try windows_util.sliceToPrefixedFileW(self.dest_path);
1129 const tmp_path_w = try windows_util.cStrToPrefixedFileW(&self.tmp_path_buf);
1130 return renameW(&tmp_path_w, &dest_path_w);
1131 } else {
1132 @compileError("Unsupported OS");
1133 }
1120 }1134 }
1121};1135};
11221136
1123pub fn renameC(old_path: [*]const u8, new_path: [*]const u8) !void {1137pub fn renameC(old_path: [*]const u8, new_path: [*]const u8) !void {
1124 if (is_windows) {1138 if (is_windows) {
1125 @compileError("TODO implement for windows");1139 const old_path_w = try windows_util.cStrToPrefixedFileW(old_path);
1140 const new_path_w = try windows_util.cStrToPrefixedFileW(new_path);
1141 return renameW(&old_path_w, &new_path_w);
1126 } else {1142 } else {
1127 const err = posix.getErrno(posix.rename(old_path, new_path));1143 const err = posix.getErrno(posix.rename(old_path, new_path));
1128 switch (err) {1144 switch (err) {
...@@ -1150,17 +1166,21 @@ pub fn renameC(old_path: [*]const u8, new_path: [*]const u8) !void {...@@ -1150,17 +1166,21 @@ pub fn renameC(old_path: [*]const u8, new_path: [*]const u8) !void {
1150 }1166 }
1151}1167}
11521168
1169pub fn renameW(old_path: [*]const u16, new_path: [*]const u16) !void {
1170 const flags = windows.MOVEFILE_REPLACE_EXISTING | windows.MOVEFILE_WRITE_THROUGH;
1171 if (windows.MoveFileExW(old_path, new_path, flags) == 0) {
1172 const err = windows.GetLastError();
1173 switch (err) {
1174 else => return unexpectedErrorWindows(err),
1175 }
1176 }
1177}
1178
1153pub fn rename(old_path: []const u8, new_path: []const u8) !void {1179pub fn rename(old_path: []const u8, new_path: []const u8) !void {
1154 if (is_windows) {1180 if (is_windows) {
1155 const flags = windows.MOVEFILE_REPLACE_EXISTING | windows.MOVEFILE_WRITE_THROUGH;
1156 const old_path_w = try windows_util.sliceToPrefixedFileW(old_path);1181 const old_path_w = try windows_util.sliceToPrefixedFileW(old_path);
1157 const new_path_w = try windows_util.sliceToPrefixedFileW(new_path);1182 const new_path_w = try windows_util.sliceToPrefixedFileW(new_path);
1158 if (windows.MoveFileExW(&old_path_w, &new_path_w, flags) == 0) {1183 return renameW(&old_path_w, &new_path_w);
1159 const err = windows.GetLastError();
1160 switch (err) {
1161 else => return unexpectedErrorWindows(err),
1162 }
1163 }
1164 } else {1184 } else {
1165 const old_path_c = try toPosixPath(old_path);1185 const old_path_c = try toPosixPath(old_path);
1166 const new_path_c = try toPosixPath(new_path);1186 const new_path_c = try toPosixPath(new_path);
std/os/test.zig+21
...@@ -2,6 +2,7 @@ const std = @import("../index.zig");...@@ -2,6 +2,7 @@ const std = @import("../index.zig");
2const os = std.os;2const os = std.os;
3const assert = std.debug.assert;3const assert = std.debug.assert;
4const io = std.io;4const io = std.io;
5const mem = std.mem;
56
6const a = std.debug.global_allocator;7const a = std.debug.global_allocator;
78
...@@ -80,3 +81,23 @@ test "cpu count" {...@@ -80,3 +81,23 @@ test "cpu count" {
80 const cpu_count = try std.os.cpuCount(a);81 const cpu_count = try std.os.cpuCount(a);
81 assert(cpu_count >= 1);82 assert(cpu_count >= 1);
82}83}
84
85test "AtomicFile" {
86 var buffer: [1024]u8 = undefined;
87 const allocator = &std.heap.FixedBufferAllocator.init(buffer[0..]).allocator;
88 const test_out_file = "tmp_atomic_file_test_dest.txt";
89 const test_content =
90 \\ hello!
91 \\ this is a test file
92 ;
93 {
94 var af = try os.AtomicFile.init(test_out_file, os.File.default_mode);
95 defer af.deinit();
96 try af.file.write(test_content);
97 try af.finish();
98 }
99 const content = try io.readFileAlloc(allocator, test_out_file);
100 assert(mem.eql(u8, content, test_content));
101
102 try os.deleteFile(test_out_file);
103}