authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-06-20 18:27:37-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-06-20 18:27:37-04:00
logd87cd06296a759ce398b50a437b8a1444413c6be
treea644b31ed378efd2549c6e05506ac30095de860b
parentbc0ca73887d12ff71e12ad473bab22631a29b1aa

rework zig fmt to use less syscalls and open fds

* `std.fs.Dir.Entry.Kind` is moved to `std.fs.File.Kind` * `std.fs.File.Stat` gains the `kind` field, so performing a stat() on a File now tells what kind of file it is. On Windows this only will distinguish between directories and files. * rework zig fmt logic so that in the case of opening a file and discovering it to be a directory, it closes the file descriptor before re-opening it with O_DIRECTORY, using fewer simultaneous open file descriptors when walking a directory tree. * rework zig fmt logic so that it pays attention to the kind of directory entries, and when it sees a sub-directory it attempts to open it as a directory rather than a file, reducing the number of open() syscalls when walking a directory tree.

3 files changed, 79 insertions(+), 46 deletions(-)

lib/std/fs.zig+1-11
......@@ -261,17 +261,7 @@ pub const Dir = struct {
261261 name: []const u8,
262262 kind: Kind,
263263
264 pub const Kind = enum {
265 BlockDevice,
266 CharacterDevice,
267 Directory,
268 NamedPipe,
269 SymLink,
270 File,
271 UnixDomainSocket,
272 Whiteout,
273 Unknown,
274 };
264 pub const Kind = File.Kind;
275265 };
276266
277267 const IteratorError = error{AccessDenied} || os.UnexpectedError;
lib/std/fs/file.zig+25-1
......@@ -29,6 +29,18 @@ pub const File = struct {
2929 pub const Mode = os.mode_t;
3030 pub const INode = os.ino_t;
3131
32 pub const Kind = enum {
33 BlockDevice,
34 CharacterDevice,
35 Directory,
36 NamedPipe,
37 SymLink,
38 File,
39 UnixDomainSocket,
40 Whiteout,
41 Unknown,
42 };
43
3244 pub const default_mode = switch (builtin.os.tag) {
3345 .windows => 0,
3446 .wasi => 0,
......@@ -219,13 +231,14 @@ pub const File = struct {
219231 /// unique across time, as some file systems may reuse an inode after its file has been deleted.
220232 /// Some systems may change the inode of a file over time.
221233 ///
222 /// On Linux, the inode _is_ structure that stores the metadata, and the inode _number_ is what
234 /// On Linux, the inode is a structure that stores the metadata, and the inode _number_ is what
223235 /// you see here: the index number of the inode.
224236 ///
225237 /// The FileIndex on Windows is similar. It is a number for a file that is unique to each filesystem.
226238 inode: INode,
227239 size: u64,
228240 mode: Mode,
241 kind: Kind,
229242
230243 /// Access time in nanoseconds, relative to UTC 1970-01-01.
231244 atime: i128,
......@@ -254,6 +267,7 @@ pub const File = struct {
254267 .inode = info.InternalInformation.IndexNumber,
255268 .size = @bitCast(u64, info.StandardInformation.EndOfFile),
256269 .mode = 0,
270 .kind = if (info.StandardInformation.Directory == 0) .File else .Directory,
257271 .atime = windows.fromSysTime(info.BasicInformation.LastAccessTime),
258272 .mtime = windows.fromSysTime(info.BasicInformation.LastWriteTime),
259273 .ctime = windows.fromSysTime(info.BasicInformation.CreationTime),
......@@ -268,6 +282,16 @@ pub const File = struct {
268282 .inode = st.ino,
269283 .size = @bitCast(u64, st.size),
270284 .mode = st.mode,
285 .kind = switch (st.mode & os.S_IFMT) {
286 os.S_IFBLK => .BlockDevice,
287 os.S_IFCHR => .CharacterDevice,
288 os.S_IFDIR => .Directory,
289 os.S_IFIFO => .NamedPipe,
290 os.S_IFLNK => .SymLink,
291 os.S_IFREG => .File,
292 os.S_IFSOCK => .UnixDomainSocket,
293 else => .Unknown,
294 },
271295 .atime = @as(i128, atime.tv_sec) * std.time.ns_per_s + atime.tv_nsec,
272296 .mtime = @as(i128, mtime.tv_sec) * std.time.ns_per_s + mtime.tv_nsec,
273297 .ctime = @as(i128, ctime.tv_sec) * std.time.ns_per_s + ctime.tv_nsec,
src-self-hosted/main.zig+53-34
......@@ -670,11 +670,12 @@ const FmtError = error{
670670 ReadOnlyFileSystem,
671671 LinkQuotaExceeded,
672672 FileBusy,
673 EndOfStream,
673674} || fs.File.OpenError;
674675
675676fn fmtPath(fmt: *Fmt, file_path: []const u8, check_mode: bool) FmtError!void {
676677 // get the real path here to avoid Windows failing on relative file paths with . or .. in them
677 var real_path = fs.realpathAlloc(fmt.gpa, file_path) catch |err| {
678 const real_path = fs.realpathAlloc(fmt.gpa, file_path) catch |err| {
678679 std.debug.warn("unable to open '{}': {}\n", .{ file_path, err });
679680 fmt.any_error = true;
680681 return;
......@@ -684,47 +685,65 @@ fn fmtPath(fmt: *Fmt, file_path: []const u8, check_mode: bool) FmtError!void {
684685 if (fmt.seen.exists(real_path)) return;
685686 try fmt.seen.put(real_path);
686687
687 const source_file = fs.cwd().openFile(real_path, .{}) catch |err| {
688 std.debug.warn("unable to open '{}': {}\n", .{ file_path, err });
689 fmt.any_error = true;
690 return;
688 fmtPathFile(fmt, file_path, check_mode, real_path) catch |err| switch (err) {
689 error.IsDir, error.AccessDenied => return fmtPathDir(fmt, file_path, check_mode, real_path),
690 else => {
691 std.debug.warn("unable to format '{}': {}\n", .{ file_path, err });
692 fmt.any_error = true;
693 return;
694 },
691695 };
692 defer source_file.close();
696}
693697
694 const stat = source_file.stat() catch |err| {
695 std.debug.warn("unable to stat '{}': {}\n", .{ file_path, err });
696 fmt.any_error = true;
697 return;
698 };
698fn fmtPathDir(fmt: *Fmt, file_path: []const u8, check_mode: bool, parent_real_path: []const u8) FmtError!void {
699 var dir = try fs.cwd().openDir(parent_real_path, .{ .iterate = true });
700 defer dir.close();
701
702 var dir_it = dir.iterate();
703 while (try dir_it.next()) |entry| {
704 const is_dir = entry.kind == .Directory;
705 if (is_dir or mem.endsWith(u8, entry.name, ".zig")) {
706 const full_path = try fs.path.join(fmt.gpa, &[_][]const u8{ file_path, entry.name });
707 const sub_real_path = fs.realpathAlloc(fmt.gpa, full_path) catch |err| {
708 std.debug.warn("unable to open '{}': {}\n", .{ file_path, err });
709 fmt.any_error = true;
710 return;
711 };
712 defer fmt.gpa.free(sub_real_path);
713
714 if (fmt.seen.exists(sub_real_path)) return;
715 try fmt.seen.put(sub_real_path);
716
717 if (is_dir) {
718 try fmtPathDir(fmt, full_path, check_mode, sub_real_path);
719 } else {
720 fmtPathFile(fmt, full_path, check_mode, sub_real_path) catch |err| {
721 std.debug.warn("unable to format '{}': {}\n", .{ full_path, err });
722 fmt.any_error = true;
723 return;
724 };
725 }
726 }
727 }
728}
699729
700 const source_code = source_file.readAllAlloc(fmt.gpa, stat.size, max_src_size) catch |err| switch (err) {
701 error.IsDir => {
702 var dir = try fs.cwd().openDir(file_path, .{ .iterate = true });
703 defer dir.close();
730fn fmtPathFile(fmt: *Fmt, file_path: []const u8, check_mode: bool, real_path: []const u8) FmtError!void {
731 const source_file = try fs.cwd().openFile(real_path, .{});
732 defer source_file.close();
704733
705 var dir_it = dir.iterate();
734 const stat = try source_file.stat();
706735
707 while (try dir_it.next()) |entry| {
708 if (entry.kind == .Directory or mem.endsWith(u8, entry.name, ".zig")) {
709 const full_path = try fs.path.join(fmt.gpa, &[_][]const u8{ file_path, entry.name });
710 try fmtPath(fmt, full_path, check_mode);
711 }
712 }
713 return;
714 },
715 else => {
716 std.debug.warn("unable to read '{}': {}\n", .{ file_path, err });
717 fmt.any_error = true;
718 return;
719 },
736 if (stat.kind == .Directory)
737 return error.IsDir;
738
739 const source_code = source_file.readAllAlloc(fmt.gpa, stat.size, max_src_size) catch |err| switch (err) {
740 error.ConnectionResetByPeer => unreachable,
741 error.ConnectionTimedOut => unreachable,
742 else => |e| return e,
720743 };
721744 defer fmt.gpa.free(source_code);
722745
723 const tree = std.zig.parse(fmt.gpa, source_code) catch |err| {
724 std.debug.warn("error parsing file '{}': {}\n", .{ file_path, err });
725 fmt.any_error = true;
726 return;
727 };
746 const tree = try std.zig.parse(fmt.gpa, source_code);
728747 defer tree.deinit();
729748
730749 for (tree.errors) |parse_error| {