authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-12-10 23:09:08-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-12-23 22:15:09-08:00
log94ef56ee26558daea3c7a1468f898c94735d1658
tree86e22c258d78ca40fb09504493bc2daaff029d6e
parent7bc0166b7c34ac0120f50aceb7132ffaa4aeec83

std.Io.Dir: fix walking

* Make Io.Dir.Reader lower level, accepting a buffer. * Make Io.Dir.Iterator higher level, requesting only one Entry with every call to `next`.

2 files changed, 44 insertions(+), 43 deletions(-)

lib/std/Io/Dir.zig+40-28
...@@ -95,8 +95,13 @@ pub const Reader = struct {...@@ -95,8 +95,13 @@ pub const Reader = struct {
95 dir: Dir,95 dir: Dir,
96 state: State,96 state: State,
97 /// Stores I/O implementation specific data.97 /// Stores I/O implementation specific data.
98 buffer: [2048]u8 align(@alignOf(usize)),98 buffer: []u8 align(@alignOf(usize)),
99 /// Index of next entry in `buffer`.
99 index: usize,100 index: usize,
101 /// Fill position of `buffer`.
102 end: usize,
103
104 pub const min_buffer_len = 32;
100105
101 pub const State = enum {106 pub const State = enum {
102 /// Indicates the next call to `read` should rewind and start over the107 /// Indicates the next call to `read` should rewind and start over the
...@@ -112,27 +117,45 @@ pub const Reader = struct {...@@ -112,27 +117,45 @@ pub const Reader = struct {
112 SystemResources,117 SystemResources,
113 } || Io.UnexpectedError || Io.Cancelable;118 } || Io.UnexpectedError || Io.Cancelable;
114119
115 pub fn init(dir: Dir) Reader {120 pub fn init(dir: Dir, buffer: []align(@alignOf(usize)) u8) Reader {
121 assert(buffer.len >= min_buffer_len);
116 return .{122 return .{
117 .dir = dir,123 .dir = dir,
118 .state = .reset,124 .state = .reset,
119 .index = 0,125 .index = 0,
120 .buffer = undefined,126 .end = 0,
127 .buffer = buffer,
121 };128 };
122 }129 }
123130
131 /// All `Entry.name` are invalidated with the next call to `read` or
132 /// `next`.
124 pub fn read(r: *Reader, io: Io, buffer: []Entry) Error!usize {133 pub fn read(r: *Reader, io: Io, buffer: []Entry) Error!usize {
125 return io.vtable.dirRead(io.userdata, r, buffer);134 return io.vtable.dirRead(io.userdata, r, buffer);
126 }135 }
136
137 /// `Entry.name` is invalidated with the next call to `read` or `next`.
138 pub fn next(r: *Reader, io: Io) Error!?Entry {
139 var buffer: [1]Entry = undefined;
140 while (true) {
141 const n = try read(r, io, &buffer);
142 if (n == 1) return buffer[0];
143 if (r.state == .finished) return null;
144 }
145 }
127};146};
128147
148/// This API is designed for convenience rather than performance:
149/// * It chooses a buffer size rather than allowing the user to provide one.
150/// * It is movable by only requesting one `Entry` at a time from the `Io`
151/// implementation rather than doing batch operations.
152///
153/// Still, it will do a decent job of minimizing syscall overhead. For a
154/// lower level abstraction, see `Reader`. For a higher level abstraction,
155/// see `Walker`.
129pub const Iterator = struct {156pub const Iterator = struct {
130 reader: Reader,157 reader: Reader,
131 buffer: [32]Entry,158 reader_buffer: [2048]u8 align(@alignOf(usize)),
132 /// Index of next entry in `buffer`.
133 index: usize,
134 /// Fill position of `buffer`.
135 end: usize,
136159
137 pub const Error = Reader.Error;160 pub const Error = Reader.Error;
138161
...@@ -142,27 +165,16 @@ pub const Iterator = struct {...@@ -142,27 +165,16 @@ pub const Iterator = struct {
142 .dir = dir,165 .dir = dir,
143 .state = reader_state,166 .state = reader_state,
144 .index = 0,167 .index = 0,
168 .end = 0,
145 .buffer = undefined,169 .buffer = undefined,
146 },170 },
147 .buffer = undefined,171 .reader_buffer = undefined,
148 .index = 0,
149 .end = 0,
150 };172 };
151 }173 }
152174
153 pub fn next(it: *Iterator, io: Io) Error!?Entry {175 pub fn next(it: *Iterator, io: Io) Error!?Entry {
154 if (it.end - it.index == 0) {176 it.reader.buffer = &it.reader_buffer;
155 if (it.reader.state == .finished) return null;177 return it.reader.next(io);
156 it.end = try it.reader.read(io, &it.buffer);
157 it.index = 0;
158 if (it.end - it.index == 0) {
159 assert(it.reader.state == .finished);
160 return null;
161 }
162 }
163 const index = it.index;
164 it.index = index + 1;
165 return it.buffer[index];
166 }178 }
167};179};
168180
...@@ -178,14 +190,14 @@ pub fn iterateAssumeFirstIteration(dir: Dir) Iterator {...@@ -178,14 +190,14 @@ pub fn iterateAssumeFirstIteration(dir: Dir) Iterator {
178}190}
179191
180pub const SelectiveWalker = struct {192pub const SelectiveWalker = struct {
181 stack: std.ArrayList(Walker.StackItem),193 stack: std.ArrayList(StackItem),
182 name_buffer: std.ArrayList(u8),194 name_buffer: std.ArrayList(u8),
183 allocator: Allocator,195 allocator: Allocator,
184196
185 pub const Error = Io.Dir.Iterator.Error || Allocator.Error;197 pub const Error = Iterator.Error || Allocator.Error;
186198
187 const StackItem = struct {199 const StackItem = struct {
188 iter: Dir.Iterator,200 iter: Iterator,
189 dirname_len: usize,201 dirname_len: usize,
190 };202 };
191203
...@@ -942,7 +954,7 @@ pub fn renameAbsolute(io: Io, old_path: []const u8, new_path: []const u8) Rename...@@ -942,7 +954,7 @@ pub fn renameAbsolute(io: Io, old_path: []const u8, new_path: []const u8) Rename
942 return io.vtable.dirRename(io.userdata, my_cwd, old_path, my_cwd, new_path);954 return io.vtable.dirRename(io.userdata, my_cwd, old_path, my_cwd, new_path);
943}955}
944956
945/// Use with `Dir.symLink`, `Dir.symLinkAtomic`, and `symLinkAbsolute` to957/// Use with `symLink`, `symLinkAtomic`, and `symLinkAbsolute` to
946/// specify whether the symlink will point to a file or a directory. This value958/// specify whether the symlink will point to a file or a directory. This value
947/// is ignored on all hosts except Windows where creating symlinks to different959/// is ignored on all hosts except Windows where creating symlinks to different
948/// resource types, requires different flags. By default, `symLinkAbsolute` is960/// resource types, requires different flags. By default, `symLinkAbsolute` is
...@@ -1182,7 +1194,7 @@ pub fn deleteTree(dir: Dir, io: Io, sub_path: []const u8) DeleteTreeError!void {...@@ -1182,7 +1194,7 @@ pub fn deleteTree(dir: Dir, io: Io, sub_path: []const u8) DeleteTreeError!void {
1182 const StackItem = struct {1194 const StackItem = struct {
1183 name: []const u8,1195 name: []const u8,
1184 parent_dir: Dir,1196 parent_dir: Dir,
1185 iter: Dir.Iterator,1197 iter: Iterator,
11861198
1187 fn closeAll(inner_io: Io, items: []@This()) void {1199 fn closeAll(inner_io: Io, items: []@This()) void {
1188 for (items) |*item| item.iter.reader.dir.close(inner_io);1200 for (items) |*item| item.iter.reader.dir.close(inner_io);
lib/std/Io/Threaded.zig+4-15
...@@ -3280,19 +3280,9 @@ fn dirReadLinux(userdata: ?*anyopaque, dr: *Dir.Reader, buffer: []Dir.Entry) Dir...@@ -3280,19 +3280,9 @@ fn dirReadLinux(userdata: ?*anyopaque, dr: *Dir.Reader, buffer: []Dir.Entry) Dir
3280 const linux = std.os.linux;3280 const linux = std.os.linux;
3281 const t: *Threaded = @ptrCast(@alignCast(userdata));3281 const t: *Threaded = @ptrCast(@alignCast(userdata));
3282 const current_thread = Thread.getCurrent(t);3282 const current_thread = Thread.getCurrent(t);
3283 const Header = extern struct {
3284 fill_end: usize,
3285 };
3286 const header: *Header = @ptrCast(&dr.buffer);
3287 const header_end: usize = @sizeOf(Header);
3288 if (dr.index < header_end) {
3289 // Initialize header.
3290 dr.index = header_end;
3291 header.* = .{ .fill_end = header_end };
3292 }
3293 var buffer_index: usize = 0;3283 var buffer_index: usize = 0;
3294 while (buffer.len - buffer_index != 0) {3284 while (buffer.len - buffer_index != 0) {
3295 if (header.fill_end - dr.index == 0) {3285 if (dr.end - dr.index == 0) {
3296 // Refill the buffer, unless we've already created references to3286 // Refill the buffer, unless we've already created references to
3297 // buffered data.3287 // buffered data.
3298 if (buffer_index != 0) break;3288 if (buffer_index != 0) break;
...@@ -3303,10 +3293,9 @@ fn dirReadLinux(userdata: ?*anyopaque, dr: *Dir.Reader, buffer: []Dir.Entry) Dir...@@ -3303,10 +3293,9 @@ fn dirReadLinux(userdata: ?*anyopaque, dr: *Dir.Reader, buffer: []Dir.Entry) Dir
3303 };3293 };
3304 dr.state = .reading;3294 dr.state = .reading;
3305 }3295 }
3306 const dents_buffer = dr.buffer[header_end..];
3307 try current_thread.beginSyscall();3296 try current_thread.beginSyscall();
3308 const n = while (true) {3297 const n = while (true) {
3309 const rc = linux.getdents64(dr.dir.handle, dents_buffer.ptr, dents_buffer.len);3298 const rc = linux.getdents64(dr.dir.handle, dr.buffer.ptr, dr.buffer.len);
3310 switch (linux.errno(rc)) {3299 switch (linux.errno(rc)) {
3311 .SUCCESS => {3300 .SUCCESS => {
3312 current_thread.endSyscall();3301 current_thread.endSyscall();
...@@ -3342,8 +3331,8 @@ fn dirReadLinux(userdata: ?*anyopaque, dr: *Dir.Reader, buffer: []Dir.Entry) Dir...@@ -3342,8 +3331,8 @@ fn dirReadLinux(userdata: ?*anyopaque, dr: *Dir.Reader, buffer: []Dir.Entry) Dir
3342 dr.state = .finished;3331 dr.state = .finished;
3343 return 0;3332 return 0;
3344 }3333 }
3345 dr.index = header_end;3334 dr.index = 0;
3346 header.fill_end = header_end + n;3335 dr.end = n;
3347 }3336 }
3348 const linux_entry: *align(1) linux.dirent64 = @ptrCast(&dr.buffer[dr.index]);3337 const linux_entry: *align(1) linux.dirent64 = @ptrCast(&dr.buffer[dr.index]);
3349 const next_index = dr.index + linux_entry.reclen;3338 const next_index = dr.index + linux_entry.reclen;