authorgravatar for mail@isaacfreund.comIsaac Freund <mail@isaacfreund.com> 2025-09-05 16:28:08+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2025-09-05 16:28:08+02:00
logac42eaaadd0650ffc281f9a1ed1a642fde8984b7
tree9c22e0f8b598097ac264313075450156469b8e2a
parent9fa2394f8c00d060931d69fb6f342f7f2e3d826e
parentd26b53264744842fe15227142679d481fa68e491
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #24905 from gooncreeper/file-reader-buffered

Fix Io.Writer sendFile bugs with buffered reader contents

3 files changed, 102 insertions(+), 30 deletions(-)

lib/std/Io/Writer.zig+51-13
......@@ -921,7 +921,8 @@ pub fn sendFileHeader(
921921/// Asserts nonzero buffer capacity.
922922pub fn sendFileReading(w: *Writer, file_reader: *File.Reader, limit: Limit) FileReadingError!usize {
923923 const dest = limit.slice(try w.writableSliceGreedy(1));
924 const n = try file_reader.read(dest);
924 const n = try file_reader.interface.readSliceShort(dest);
925 if (n == 0) return error.EndOfStream;
925926 w.advance(n);
926927 return n;
927928}
......@@ -934,17 +935,24 @@ pub fn sendFileReading(w: *Writer, file_reader: *File.Reader, limit: Limit) File
934935///
935936/// Asserts nonzero buffer capacity.
936937pub fn sendFileAll(w: *Writer, file_reader: *File.Reader, limit: Limit) FileAllError!usize {
937 // The fallback sendFileReadingAll() path asserts non-zero buffer capacity.
938 // Explicitly assert it here as well to ensure the assert is hit even if
939 // the fallback path is not taken.
938 // The fallback case uses `stream`. For `File.Reader`, this requires a minumum buffer size of
939 // one since it uses `writableSliceGreedy(1)`. Asserting this here ensures that this will be
940 // hit even when the fallback is not needed.
940941 assert(w.buffer.len > 0);
942
941943 var remaining = @intFromEnum(limit);
942944 while (remaining > 0) {
943945 const n = sendFile(w, file_reader, .limited(remaining)) catch |err| switch (err) {
944946 error.EndOfStream => break,
945947 error.Unimplemented => {
946948 file_reader.mode = file_reader.mode.toReading();
947 remaining -= try w.sendFileReadingAll(file_reader, .limited(remaining));
949 while (remaining > 0) {
950 remaining -= file_reader.interface.stream(w, .limited(remaining)) catch |e| switch (e) {
951 error.EndOfStream => break,
952 error.ReadFailed => return error.ReadFailed,
953 error.WriteFailed => return error.WriteFailed,
954 };
955 }
948956 break;
949957 },
950958 else => |e| return e,
......@@ -2276,6 +2284,12 @@ pub const Discarding = struct {
22762284 const d: *Discarding = @alignCast(@fieldParentPtr("writer", w));
22772285 d.count += w.end;
22782286 w.end = 0;
2287 const buffered_n = limit.minInt64(file_reader.interface.bufferedLen());
2288 if (buffered_n != 0) {
2289 file_reader.interface.toss(buffered_n);
2290 d.count += buffered_n;
2291 return buffered_n;
2292 }
22792293 if (limit == .nothing) return 0;
22802294 if (file_reader.getSize()) |size| {
22812295 const n = limit.minInt64(size - file_reader.pos);
......@@ -2767,7 +2781,9 @@ pub const Allocating = struct {
27672781 if (additional == 0) return error.EndOfStream;
27682782 a.ensureUnusedCapacity(limit.minInt64(additional)) catch return error.WriteFailed;
27692783 const dest = limit.slice(a.writer.buffer[a.writer.end..]);
2770 const n = try file_reader.read(dest);
2784 const n = try file_reader.interface.readSliceShort(dest);
2785 // If it was a short read, then EOF has been reached and `file_reader.size`
2786 // has been set and the EOF case will be hit on subsequent calls.
27712787 a.writer.end += n;
27722788 return n;
27732789 }
......@@ -2818,18 +2834,18 @@ test "discarding sendFile" {
28182834
28192835 const file = try tmp_dir.dir.createFile("input.txt", .{ .read = true });
28202836 defer file.close();
2821 var r_buffer: [256]u8 = undefined;
2837 var r_buffer: [2]u8 = undefined;
28222838 var file_writer: std.fs.File.Writer = .init(file, &r_buffer);
2823 try file_writer.interface.writeByte('h');
2839 try file_writer.interface.writeAll("abcd");
28242840 try file_writer.interface.flush();
28252841
28262842 var file_reader = file_writer.moveToReader();
28272843 try file_reader.seekTo(0);
2844 try file_reader.interface.fill(2);
28282845
28292846 var w_buffer: [256]u8 = undefined;
28302847 var discarding: Writer.Discarding = .init(&w_buffer);
2831
2832 _ = try file_reader.interface.streamRemaining(&discarding.writer);
2848 try testing.expectEqual(4, discarding.writer.sendFileAll(&file_reader, .unlimited));
28332849}
28342850
28352851test "allocating sendFile" {
......@@ -2838,18 +2854,40 @@ test "allocating sendFile" {
28382854
28392855 const file = try tmp_dir.dir.createFile("input.txt", .{ .read = true });
28402856 defer file.close();
2841 var r_buffer: [256]u8 = undefined;
2857 var r_buffer: [2]u8 = undefined;
28422858 var file_writer: std.fs.File.Writer = .init(file, &r_buffer);
2843 try file_writer.interface.writeByte('h');
2859 try file_writer.interface.writeAll("abcd");
28442860 try file_writer.interface.flush();
28452861
28462862 var file_reader = file_writer.moveToReader();
28472863 try file_reader.seekTo(0);
2864 try file_reader.interface.fill(2);
28482865
28492866 var allocating: Writer.Allocating = .init(testing.allocator);
28502867 defer allocating.deinit();
2868 try allocating.ensureUnusedCapacity(1);
2869 try testing.expectEqual(4, allocating.writer.sendFileAll(&file_reader, .unlimited));
2870 try testing.expectEqualStrings("abcd", allocating.writer.buffered());
2871}
28512872
2852 _ = try file_reader.interface.streamRemaining(&allocating.writer);
2873test sendFileReading {
2874 var tmp_dir = testing.tmpDir(.{});
2875 defer tmp_dir.cleanup();
2876
2877 const file = try tmp_dir.dir.createFile("input.txt", .{ .read = true });
2878 defer file.close();
2879 var r_buffer: [2]u8 = undefined;
2880 var file_writer: std.fs.File.Writer = .init(file, &r_buffer);
2881 try file_writer.interface.writeAll("abcd");
2882 try file_writer.interface.flush();
2883
2884 var file_reader = file_writer.moveToReader();
2885 try file_reader.seekTo(0);
2886 try file_reader.interface.fill(2);
2887
2888 var w_buffer: [1]u8 = undefined;
2889 var discarding: Writer.Discarding = .init(&w_buffer);
2890 try testing.expectEqual(4, discarding.writer.sendFileReadingAll(&file_reader, .unlimited));
28532891}
28542892
28552893test writeStruct {
lib/std/fs/File.zig+22-17
......@@ -1154,6 +1154,7 @@ pub const Reader = struct {
11541154 };
11551155 }
11561156
1157 /// If `error.EndOfStream` has been hit, this cannot fail.
11571158 pub fn getSize(r: *Reader) SizeError!u64 {
11581159 return r.size orelse {
11591160 if (r.size_err) |err| return err;
......@@ -1440,7 +1441,7 @@ pub const Reader = struct {
14401441 }
14411442 }
14421443
1443 pub fn readPositional(r: *Reader, dest: []u8) std.Io.Reader.Error!usize {
1444 fn readPositional(r: *Reader, dest: []u8) std.Io.Reader.Error!usize {
14441445 const n = r.file.pread(dest, r.pos) catch |err| switch (err) {
14451446 error.Unseekable => {
14461447 r.mode = r.mode.toStreaming();
......@@ -1467,7 +1468,7 @@ pub const Reader = struct {
14671468 return n;
14681469 }
14691470
1470 pub fn readStreaming(r: *Reader, dest: []u8) std.Io.Reader.Error!usize {
1471 fn readStreaming(r: *Reader, dest: []u8) std.Io.Reader.Error!usize {
14711472 const n = r.file.read(dest) catch |err| {
14721473 r.err = err;
14731474 return error.ReadFailed;
......@@ -1480,14 +1481,6 @@ pub const Reader = struct {
14801481 return n;
14811482 }
14821483
1483 pub fn read(r: *Reader, dest: []u8) std.Io.Reader.Error!usize {
1484 switch (r.mode) {
1485 .positional, .positional_reading => return readPositional(r, dest),
1486 .streaming, .streaming_reading => return readStreaming(r, dest),
1487 .failure => return error.ReadFailed,
1488 }
1489 }
1490
14911484 pub fn atEnd(r: *Reader) bool {
14921485 // Even if stat fails, size is set when end is encountered.
14931486 const size = r.size orelse return false;
......@@ -1803,9 +1796,15 @@ pub const Writer = struct {
18031796 file_reader.size = file_reader.pos;
18041797 return error.EndOfStream;
18051798 }
1806 const consumed = io_w.consume(@intCast(sbytes));
1807 file_reader.seekTo(file_reader.pos + consumed) catch return error.ReadFailed;
1808 return consumed;
1799 const n = io_w.consume(@intCast(sbytes));
1800 if (n <= file_reader.interface.bufferedLen()) {
1801 file_reader.interface.toss(n);
1802 } else {
1803 const direct_n = n - file_reader.interface.bufferedLen();
1804 file_reader.interface.tossBuffered();
1805 file_reader.seekBy(@intCast(direct_n)) catch return error.ReadFailed;
1806 }
1807 return n;
18091808 }
18101809
18111810 if (native_os.isDarwin() and w.mode == .streaming) sf: {
......@@ -1864,9 +1863,15 @@ pub const Writer = struct {
18641863 file_reader.size = file_reader.pos;
18651864 return error.EndOfStream;
18661865 }
1867 const consumed = io_w.consume(@bitCast(len));
1868 file_reader.seekTo(file_reader.pos + consumed) catch return error.ReadFailed;
1869 return consumed;
1866 const n = io_w.consume(@bitCast(len));
1867 if (n <= file_reader.interface.bufferedLen()) {
1868 file_reader.interface.toss(n);
1869 } else {
1870 const direct_n = n - file_reader.interface.bufferedLen();
1871 file_reader.interface.tossBuffered();
1872 file_reader.seekBy(@intCast(direct_n)) catch return error.ReadFailed;
1873 }
1874 return n;
18701875 }
18711876
18721877 if (native_os == .linux and w.mode == .streaming) sf: {
......@@ -1998,7 +2003,7 @@ pub const Writer = struct {
19982003 reader_buffered: []const u8,
19992004 ) std.Io.Writer.FileError!usize {
20002005 const n = try drain(io_w, &.{reader_buffered}, 1);
2001 file_reader.seekTo(file_reader.pos + n) catch return error.ReadFailed;
2006 file_reader.interface.toss(n);
20022007 return n;
20032008 }
20042009
lib/std/fs/test.zig+29
......@@ -2180,3 +2180,32 @@ test "seekTo flushes buffered data" {
21802180 try file_reader.interface.readSliceAll(&buf);
21812181 try std.testing.expectEqualStrings(contents, &buf);
21822182}
2183
2184test "File.Writer sendfile with buffered contents" {
2185 var tmp_dir = testing.tmpDir(.{});
2186 defer tmp_dir.cleanup();
2187
2188 try tmp_dir.dir.writeFile(.{ .sub_path = "a", .data = "bcd" });
2189 const in = try tmp_dir.dir.openFile("a", .{});
2190 defer in.close();
2191 const out = try tmp_dir.dir.createFile("b", .{});
2192 defer out.close();
2193
2194 var in_buf: [2]u8 = undefined;
2195 var in_r = in.reader(&in_buf);
2196 _ = try in_r.getSize(); // Catch seeks past end by populating size
2197 try in_r.interface.fill(2);
2198
2199 var out_buf: [1]u8 = undefined;
2200 var out_w = out.writerStreaming(&out_buf);
2201 try out_w.interface.writeByte('a');
2202 try testing.expectEqual(3, try out_w.interface.sendFileAll(&in_r, .unlimited));
2203 try out_w.interface.flush();
2204
2205 var check = try tmp_dir.dir.openFile("b", .{});
2206 defer check.close();
2207 var check_buf: [4]u8 = undefined;
2208 var check_r = check.reader(&check_buf);
2209 try testing.expectEqualStrings("abcd", try check_r.interface.take(4));
2210 try testing.expectError(error.EndOfStream, check_r.interface.takeByte());
2211}