authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-09-06 10:23:41+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-10-09 12:08:03-07:00
logf661ab6c36c0366331451986a29ee14ac05fd8eb
treea7ea8365f91c0dff9026558bbacd4e034f7fe72b
parent9ea4d9aa3bc184a1be1199f3b519796d417c7765

std.Io.Reader: fix delimiter bugs

Fix `takeDelimiter` and `takeDelimiterExclusive` tossing too many bytes (#25132) Also add/improve test coverage for all delimiter and sentinel methods, update usages of `takeDelimiterExclusive` to not rely on the fixed bug, tweak a handful of doc comments, and slightly simplify some logic. I have not fixed #24950 in this commit because I am a little less certain about the appropriate solution there. Resolves: #25132 Co-authored-by: Andrew Kelley <andrew@ziglang.org>

3 files changed, 70 insertions(+), 15 deletions(-)

lib/std/Io/Reader.zig+66-9
......@@ -449,7 +449,6 @@ pub fn readVecAll(r: *Reader, data: [][]u8) Error!void {
449449/// is returned instead.
450450///
451451/// See also:
452/// * `peek`
453452/// * `toss`
454453pub fn peek(r: *Reader, n: usize) Error![]u8 {
455454 try r.fill(n);
......@@ -700,7 +699,7 @@ pub const DelimiterError = error{
700699};
701700
702701/// Returns a slice of the next bytes of buffered data from the stream until
703/// `sentinel` is found, advancing the seek position.
702/// `sentinel` is found, advancing the seek position past the sentinel.
704703///
705704/// Returned slice has a sentinel.
706705///
......@@ -733,7 +732,7 @@ pub fn peekSentinel(r: *Reader, comptime sentinel: u8) DelimiterError![:sentinel
733732}
734733
735734/// Returns a slice of the next bytes of buffered data from the stream until
736/// `delimiter` is found, advancing the seek position.
735/// `delimiter` is found, advancing the seek position past the delimiter.
737736///
738737/// Returned slice includes the delimiter as the last byte.
739738///
......@@ -786,7 +785,8 @@ pub fn peekDelimiterInclusive(r: *Reader, delimiter: u8) DelimiterError![]u8 {
786785}
787786
788787/// Returns a slice of the next bytes of buffered data from the stream until
789/// `delimiter` is found, advancing the seek position.
788/// `delimiter` is found, advancing the seek position up to (but not past)
789/// the delimiter.
790790///
791791/// Returned slice excludes the delimiter. End-of-stream is treated equivalent
792792/// to a delimiter, unless it would result in a length 0 return value, in which
......@@ -800,20 +800,44 @@ pub fn peekDelimiterInclusive(r: *Reader, delimiter: u8) DelimiterError![]u8 {
800800/// Invalidates previously returned values from `peek`.
801801///
802802/// See also:
803/// * `takeDelimiter`
803804/// * `takeDelimiterInclusive`
804805/// * `peekDelimiterExclusive`
805806pub fn takeDelimiterExclusive(r: *Reader, delimiter: u8) DelimiterError![]u8 {
806 const result = r.peekDelimiterInclusive(delimiter) catch |err| switch (err) {
807 const result = try r.peekDelimiterExclusive(delimiter);
808 r.toss(result.len);
809 return result;
810}
811
812/// Returns a slice of the next bytes of buffered data from the stream until
813/// `delimiter` is found, advancing the seek position past the delimiter.
814///
815/// Returned slice excludes the delimiter. End-of-stream is treated equivalent
816/// to a delimiter, unless it would result in a length 0 return value, in which
817/// case `null` is returned instead.
818///
819/// If the delimiter is not found within a number of bytes matching the
820/// capacity of this `Reader`, `error.StreamTooLong` is returned. In
821/// such case, the stream state is unmodified as if this function was never
822/// called.
823///
824/// Invalidates previously returned values from `peek`.
825///
826/// See also:
827/// * `takeDelimiterInclusive`
828/// * `takeDelimiterExclusive`
829pub fn takeDelimiter(r: *Reader, delimiter: u8) error{ ReadFailed, StreamTooLong }!?[]u8 {
830 const inclusive = r.peekDelimiterInclusive(delimiter) catch |err| switch (err) {
807831 error.EndOfStream => {
808832 const remaining = r.buffer[r.seek..r.end];
809 if (remaining.len == 0) return error.EndOfStream;
833 if (remaining.len == 0) return null;
810834 r.toss(remaining.len);
811835 return remaining;
812836 },
813837 else => |e| return e,
814838 };
815 r.toss(result.len);
816 return result[0 .. result.len - 1];
839 r.toss(inclusive.len);
840 return inclusive[0 .. inclusive.len - 1];
817841}
818842
819843/// Returns a slice of the next bytes of buffered data from the stream until
......@@ -1336,6 +1360,9 @@ test peekSentinel {
13361360 var r: Reader = .fixed("ab\nc");
13371361 try testing.expectEqualStrings("ab", try r.peekSentinel('\n'));
13381362 try testing.expectEqualStrings("ab", try r.peekSentinel('\n'));
1363 r.toss(3);
1364 try testing.expectError(error.EndOfStream, r.peekSentinel('\n'));
1365 try testing.expectEqualStrings("c", try r.peek(1));
13391366}
13401367
13411368test takeDelimiterInclusive {
......@@ -1350,22 +1377,52 @@ test peekDelimiterInclusive {
13501377 try testing.expectEqualStrings("ab\n", try r.peekDelimiterInclusive('\n'));
13511378 r.toss(3);
13521379 try testing.expectError(error.EndOfStream, r.peekDelimiterInclusive('\n'));
1380 try testing.expectEqualStrings("c", try r.peek(1));
13531381}
13541382
13551383test takeDelimiterExclusive {
13561384 var r: Reader = .fixed("ab\nc");
1385
13571386 try testing.expectEqualStrings("ab", try r.takeDelimiterExclusive('\n'));
1387 try testing.expectEqualStrings("", try r.takeDelimiterExclusive('\n'));
1388 try testing.expectEqualStrings("", try r.takeDelimiterExclusive('\n'));
1389 try testing.expectEqualStrings("\n", try r.take(1));
1390
13581391 try testing.expectEqualStrings("c", try r.takeDelimiterExclusive('\n'));
13591392 try testing.expectError(error.EndOfStream, r.takeDelimiterExclusive('\n'));
13601393}
13611394
13621395test peekDelimiterExclusive {
13631396 var r: Reader = .fixed("ab\nc");
1397
13641398 try testing.expectEqualStrings("ab", try r.peekDelimiterExclusive('\n'));
13651399 try testing.expectEqualStrings("ab", try r.peekDelimiterExclusive('\n'));
1366 r.toss(3);
1400 r.toss(2);
1401 try testing.expectEqualStrings("", try r.peekDelimiterExclusive('\n'));
1402 try testing.expectEqualStrings("\n", try r.take(1));
1403
13671404 try testing.expectEqualStrings("c", try r.peekDelimiterExclusive('\n'));
13681405 try testing.expectEqualStrings("c", try r.peekDelimiterExclusive('\n'));
1406 r.toss(1);
1407 try testing.expectError(error.EndOfStream, r.peekDelimiterExclusive('\n'));
1408}
1409
1410test takeDelimiter {
1411 var r: Reader = .fixed("ab\nc\n\nd");
1412 try testing.expectEqualStrings("ab", (try r.takeDelimiter('\n')).?);
1413 try testing.expectEqualStrings("c", (try r.takeDelimiter('\n')).?);
1414 try testing.expectEqualStrings("", (try r.takeDelimiter('\n')).?);
1415 try testing.expectEqualStrings("d", (try r.takeDelimiter('\n')).?);
1416 try testing.expectEqual(null, try r.takeDelimiter('\n'));
1417 try testing.expectEqual(null, try r.takeDelimiter('\n'));
1418
1419 r = .fixed("ab\nc\n\nd\n"); // one trailing newline does not affect behavior
1420 try testing.expectEqualStrings("ab", (try r.takeDelimiter('\n')).?);
1421 try testing.expectEqualStrings("c", (try r.takeDelimiter('\n')).?);
1422 try testing.expectEqualStrings("", (try r.takeDelimiter('\n')).?);
1423 try testing.expectEqualStrings("d", (try r.takeDelimiter('\n')).?);
1424 try testing.expectEqual(null, try r.takeDelimiter('\n'));
1425 try testing.expectEqual(null, try r.takeDelimiter('\n'));
13691426}
13701427
13711428test streamDelimiter {
lib/std/net.zig+3-2
......@@ -1393,7 +1393,7 @@ fn parseHosts(
13931393 br: *Io.Reader,
13941394) error{ OutOfMemory, ReadFailed }!void {
13951395 while (true) {
1396 const line = br.takeDelimiterExclusive('\n') catch |err| switch (err) {
1396 const line = br.takeDelimiter('\n') catch |err| switch (err) {
13971397 error.StreamTooLong => {
13981398 // Skip lines that are too long.
13991399 _ = br.discardDelimiterInclusive('\n') catch |e| switch (e) {
......@@ -1403,7 +1403,8 @@ fn parseHosts(
14031403 continue;
14041404 },
14051405 error.ReadFailed => return error.ReadFailed,
1406 error.EndOfStream => break,
1406 } orelse {
1407 break; // end of stream
14071408 };
14081409 var split_it = mem.splitScalar(u8, line, '#');
14091410 const no_comment_line = split_it.first();
lib/std/zig/system/linux.zig+1-4
......@@ -358,14 +358,11 @@ fn CpuinfoParser(comptime impl: anytype) type {
358358 return struct {
359359 fn parse(arch: Target.Cpu.Arch, reader: *std.Io.Reader) !?Target.Cpu {
360360 var obj: impl = .{};
361 while (reader.takeDelimiterExclusive('\n')) |line| {
361 while (try reader.takeDelimiter('\n')) |line| {
362362 const colon_pos = mem.indexOfScalar(u8, line, ':') orelse continue;
363363 const key = mem.trimEnd(u8, line[0..colon_pos], " \t");
364364 const value = mem.trimStart(u8, line[colon_pos + 1 ..], " \t");
365365 if (!try obj.line_hook(key, value)) break;
366 } else |err| switch (err) {
367 error.EndOfStream => {},
368 else => |e| return e,
369366 }
370367 return obj.finalize(arch);
371368 }