authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-06-20 18:27:37-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-06-20 18:27:37-04:00
logd87cd06296a759ce398b50a437b8a1444413c6be
treea644b31ed378efd2549c6e05506ac30095de860b
parentbc0ca73887d12ff71e12ad473bab22631a29b1aa

rework zig fmt to use less syscalls and open fds

* `std.fs.Dir.Entry.Kind` is moved to `std.fs.File.Kind` * `std.fs.File.Stat` gains the `kind` field, so performing a stat() on a File now tells what kind of file it is. On Windows this only will distinguish between directories and files. * rework zig fmt logic so that in the case of opening a file and discovering it to be a directory, it closes the file descriptor before re-opening it with O_DIRECTORY, using fewer simultaneous open file descriptors when walking a directory tree. * rework zig fmt logic so that it pays attention to the kind of directory entries, and when it sees a sub-directory it attempts to open it as a directory rather than a file, reducing the number of open() syscalls when walking a directory tree.

3 files changed, 79 insertions(+), 46 deletions(-)

lib/std/fs.zig+1-11
...@@ -261,17 +261,7 @@ pub const Dir = struct {...@@ -261,17 +261,7 @@ pub const Dir = struct {
261 name: []const u8,261 name: []const u8,
262 kind: Kind,262 kind: Kind,
263263
264 pub const Kind = enum {264 pub const Kind = File.Kind;
265 BlockDevice,
266 CharacterDevice,
267 Directory,
268 NamedPipe,
269 SymLink,
270 File,
271 UnixDomainSocket,
272 Whiteout,
273 Unknown,
274 };
275 };265 };
276266
277 const IteratorError = error{AccessDenied} || os.UnexpectedError;267 const IteratorError = error{AccessDenied} || os.UnexpectedError;
lib/std/fs/file.zig+25-1
...@@ -29,6 +29,18 @@ pub const File = struct {...@@ -29,6 +29,18 @@ pub const File = struct {
29 pub const Mode = os.mode_t;29 pub const Mode = os.mode_t;
30 pub const INode = os.ino_t;30 pub const INode = os.ino_t;
3131
32 pub const Kind = enum {
33 BlockDevice,
34 CharacterDevice,
35 Directory,
36 NamedPipe,
37 SymLink,
38 File,
39 UnixDomainSocket,
40 Whiteout,
41 Unknown,
42 };
43
32 pub const default_mode = switch (builtin.os.tag) {44 pub const default_mode = switch (builtin.os.tag) {
33 .windows => 0,45 .windows => 0,
34 .wasi => 0,46 .wasi => 0,
...@@ -219,13 +231,14 @@ pub const File = struct {...@@ -219,13 +231,14 @@ pub const File = struct {
219 /// unique across time, as some file systems may reuse an inode after its file has been deleted.231 /// unique across time, as some file systems may reuse an inode after its file has been deleted.
220 /// Some systems may change the inode of a file over time.232 /// Some systems may change the inode of a file over time.
221 ///233 ///
222 /// On Linux, the inode _is_ structure that stores the metadata, and the inode _number_ is what234 /// On Linux, the inode is a structure that stores the metadata, and the inode _number_ is what
223 /// you see here: the index number of the inode.235 /// you see here: the index number of the inode.
224 ///236 ///
225 /// The FileIndex on Windows is similar. It is a number for a file that is unique to each filesystem.237 /// The FileIndex on Windows is similar. It is a number for a file that is unique to each filesystem.
226 inode: INode,238 inode: INode,
227 size: u64,239 size: u64,
228 mode: Mode,240 mode: Mode,
241 kind: Kind,
229242
230 /// Access time in nanoseconds, relative to UTC 1970-01-01.243 /// Access time in nanoseconds, relative to UTC 1970-01-01.
231 atime: i128,244 atime: i128,
...@@ -254,6 +267,7 @@ pub const File = struct {...@@ -254,6 +267,7 @@ pub const File = struct {
254 .inode = info.InternalInformation.IndexNumber,267 .inode = info.InternalInformation.IndexNumber,
255 .size = @bitCast(u64, info.StandardInformation.EndOfFile),268 .size = @bitCast(u64, info.StandardInformation.EndOfFile),
256 .mode = 0,269 .mode = 0,
270 .kind = if (info.StandardInformation.Directory == 0) .File else .Directory,
257 .atime = windows.fromSysTime(info.BasicInformation.LastAccessTime),271 .atime = windows.fromSysTime(info.BasicInformation.LastAccessTime),
258 .mtime = windows.fromSysTime(info.BasicInformation.LastWriteTime),272 .mtime = windows.fromSysTime(info.BasicInformation.LastWriteTime),
259 .ctime = windows.fromSysTime(info.BasicInformation.CreationTime),273 .ctime = windows.fromSysTime(info.BasicInformation.CreationTime),
...@@ -268,6 +282,16 @@ pub const File = struct {...@@ -268,6 +282,16 @@ pub const File = struct {
268 .inode = st.ino,282 .inode = st.ino,
269 .size = @bitCast(u64, st.size),283 .size = @bitCast(u64, st.size),
270 .mode = st.mode,284 .mode = st.mode,
285 .kind = switch (st.mode & os.S_IFMT) {
286 os.S_IFBLK => .BlockDevice,
287 os.S_IFCHR => .CharacterDevice,
288 os.S_IFDIR => .Directory,
289 os.S_IFIFO => .NamedPipe,
290 os.S_IFLNK => .SymLink,
291 os.S_IFREG => .File,
292 os.S_IFSOCK => .UnixDomainSocket,
293 else => .Unknown,
294 },
271 .atime = @as(i128, atime.tv_sec) * std.time.ns_per_s + atime.tv_nsec,295 .atime = @as(i128, atime.tv_sec) * std.time.ns_per_s + atime.tv_nsec,
272 .mtime = @as(i128, mtime.tv_sec) * std.time.ns_per_s + mtime.tv_nsec,296 .mtime = @as(i128, mtime.tv_sec) * std.time.ns_per_s + mtime.tv_nsec,
273 .ctime = @as(i128, ctime.tv_sec) * std.time.ns_per_s + ctime.tv_nsec,297 .ctime = @as(i128, ctime.tv_sec) * std.time.ns_per_s + ctime.tv_nsec,
src-self-hosted/main.zig+53-34
...@@ -670,11 +670,12 @@ const FmtError = error{...@@ -670,11 +670,12 @@ const FmtError = error{
670 ReadOnlyFileSystem,670 ReadOnlyFileSystem,
671 LinkQuotaExceeded,671 LinkQuotaExceeded,
672 FileBusy,672 FileBusy,
673 EndOfStream,
673} || fs.File.OpenError;674} || fs.File.OpenError;
674675
675fn fmtPath(fmt: *Fmt, file_path: []const u8, check_mode: bool) FmtError!void {676fn fmtPath(fmt: *Fmt, file_path: []const u8, check_mode: bool) FmtError!void {
676 // get the real path here to avoid Windows failing on relative file paths with . or .. in them677 // get the real path here to avoid Windows failing on relative file paths with . or .. in them
677 var real_path = fs.realpathAlloc(fmt.gpa, file_path) catch |err| {678 const real_path = fs.realpathAlloc(fmt.gpa, file_path) catch |err| {
678 std.debug.warn("unable to open '{}': {}\n", .{ file_path, err });679 std.debug.warn("unable to open '{}': {}\n", .{ file_path, err });
679 fmt.any_error = true;680 fmt.any_error = true;
680 return;681 return;
...@@ -684,47 +685,65 @@ fn fmtPath(fmt: *Fmt, file_path: []const u8, check_mode: bool) FmtError!void {...@@ -684,47 +685,65 @@ fn fmtPath(fmt: *Fmt, file_path: []const u8, check_mode: bool) FmtError!void {
684 if (fmt.seen.exists(real_path)) return;685 if (fmt.seen.exists(real_path)) return;
685 try fmt.seen.put(real_path);686 try fmt.seen.put(real_path);
686687
687 const source_file = fs.cwd().openFile(real_path, .{}) catch |err| {688 fmtPathFile(fmt, file_path, check_mode, real_path) catch |err| switch (err) {
688 std.debug.warn("unable to open '{}': {}\n", .{ file_path, err });689 error.IsDir, error.AccessDenied => return fmtPathDir(fmt, file_path, check_mode, real_path),
689 fmt.any_error = true;690 else => {
690 return;691 std.debug.warn("unable to format '{}': {}\n", .{ file_path, err });
692 fmt.any_error = true;
693 return;
694 },
691 };695 };
692 defer source_file.close();696}
693697
694 const stat = source_file.stat() catch |err| {698fn fmtPathDir(fmt: *Fmt, file_path: []const u8, check_mode: bool, parent_real_path: []const u8) FmtError!void {
695 std.debug.warn("unable to stat '{}': {}\n", .{ file_path, err });699 var dir = try fs.cwd().openDir(parent_real_path, .{ .iterate = true });
696 fmt.any_error = true;700 defer dir.close();
697 return;701
698 };702 var dir_it = dir.iterate();
703 while (try dir_it.next()) |entry| {
704 const is_dir = entry.kind == .Directory;
705 if (is_dir or mem.endsWith(u8, entry.name, ".zig")) {
706 const full_path = try fs.path.join(fmt.gpa, &[_][]const u8{ file_path, entry.name });
707 const sub_real_path = fs.realpathAlloc(fmt.gpa, full_path) catch |err| {
708 std.debug.warn("unable to open '{}': {}\n", .{ file_path, err });
709 fmt.any_error = true;
710 return;
711 };
712 defer fmt.gpa.free(sub_real_path);
713
714 if (fmt.seen.exists(sub_real_path)) return;
715 try fmt.seen.put(sub_real_path);
716
717 if (is_dir) {
718 try fmtPathDir(fmt, full_path, check_mode, sub_real_path);
719 } else {
720 fmtPathFile(fmt, full_path, check_mode, sub_real_path) catch |err| {
721 std.debug.warn("unable to format '{}': {}\n", .{ full_path, err });
722 fmt.any_error = true;
723 return;
724 };
725 }
726 }
727 }
728}
699729
700 const source_code = source_file.readAllAlloc(fmt.gpa, stat.size, max_src_size) catch |err| switch (err) {730fn fmtPathFile(fmt: *Fmt, file_path: []const u8, check_mode: bool, real_path: []const u8) FmtError!void {
701 error.IsDir => {731 const source_file = try fs.cwd().openFile(real_path, .{});
702 var dir = try fs.cwd().openDir(file_path, .{ .iterate = true });732 defer source_file.close();
703 defer dir.close();
704733
705 var dir_it = dir.iterate();734 const stat = try source_file.stat();
706735
707 while (try dir_it.next()) |entry| {736 if (stat.kind == .Directory)
708 if (entry.kind == .Directory or mem.endsWith(u8, entry.name, ".zig")) {737 return error.IsDir;
709 const full_path = try fs.path.join(fmt.gpa, &[_][]const u8{ file_path, entry.name });738
710 try fmtPath(fmt, full_path, check_mode);739 const source_code = source_file.readAllAlloc(fmt.gpa, stat.size, max_src_size) catch |err| switch (err) {
711 }740 error.ConnectionResetByPeer => unreachable,
712 }741 error.ConnectionTimedOut => unreachable,
713 return;742 else => |e| return e,
714 },
715 else => {
716 std.debug.warn("unable to read '{}': {}\n", .{ file_path, err });
717 fmt.any_error = true;
718 return;
719 },
720 };743 };
721 defer fmt.gpa.free(source_code);744 defer fmt.gpa.free(source_code);
722745
723 const tree = std.zig.parse(fmt.gpa, source_code) catch |err| {746 const tree = try std.zig.parse(fmt.gpa, source_code);
724 std.debug.warn("error parsing file '{}': {}\n", .{ file_path, err });
725 fmt.any_error = true;
726 return;
727 };
728 defer tree.deinit();747 defer tree.deinit();
729748
730 for (tree.errors) |parse_error| {749 for (tree.errors) |parse_error| {