| author | |
| committer | |
| log | 529aa9f270863cfb832bbc65d67c54527126a382 |
| tree | 0a3b1fa91cf1d528b0c6cb080a4cc7ef76851559 |
| parent | 0bdd1b5274ceb6ca4145fc880a3d78276801c3fe |
| parent | d83d79c032cf72d26a20068c9f5172b8dba56df8 |
| signature |
std.Io: Writer and Reader bug fixes related to sendFile, delimiters, Limited, and seeking10 files changed, 503 insertions(+), 307 deletions(-)
build.zig+7-2| ... | ... | @@ -81,12 +81,13 @@ pub fn build(b: *std.Build) !void { |
| 81 | 81 | docs_step.dependOn(langref_step); |
| 82 | 82 | docs_step.dependOn(std_docs_step); |
| 83 | 83 | |
| 84 | const test_default_only = b.option(bool, "test-default-only", "Limit test matrix to exactly one target configuration") orelse false; | |
| 84 | 85 | const skip_debug = b.option(bool, "skip-debug", "Main test suite skips debug builds") orelse false; |
| 85 | const skip_release = b.option(bool, "skip-release", "Main test suite skips release builds") orelse false; | |
| 86 | const skip_release = b.option(bool, "skip-release", "Main test suite skips release builds") orelse test_default_only; | |
| 86 | 87 | const skip_release_small = b.option(bool, "skip-release-small", "Main test suite skips release-small builds") orelse skip_release; |
| 87 | 88 | const skip_release_fast = b.option(bool, "skip-release-fast", "Main test suite skips release-fast builds") orelse skip_release; |
| 88 | 89 | const skip_release_safe = b.option(bool, "skip-release-safe", "Main test suite skips release-safe builds") orelse skip_release; |
| 89 | const skip_non_native = b.option(bool, "skip-non-native", "Main test suite skips non-native builds") orelse false; | |
| 90 | const skip_non_native = b.option(bool, "skip-non-native", "Main test suite skips non-native builds") orelse test_default_only; | |
| 90 | 91 | const skip_libc = b.option(bool, "skip-libc", "Main test suite skips tests that link libc") orelse false; |
| 91 | 92 | const skip_single_threaded = b.option(bool, "skip-single-threaded", "Main test suite skips tests that are single-threaded") orelse false; |
| 92 | 93 | const skip_compile_errors = b.option(bool, "skip-compile-errors", "Main test suite skips compile error tests") orelse false; |
| ... | ... | @@ -449,6 +450,7 @@ pub fn build(b: *std.Build) !void { |
| 449 | 450 | .include_paths = &.{}, |
| 450 | 451 | .skip_single_threaded = skip_single_threaded, |
| 451 | 452 | .skip_non_native = skip_non_native, |
| 453 | .test_default_only = test_default_only, | |
| 452 | 454 | .skip_freebsd = skip_freebsd, |
| 453 | 455 | .skip_netbsd = skip_netbsd, |
| 454 | 456 | .skip_windows = skip_windows, |
| ... | ... | @@ -471,6 +473,7 @@ pub fn build(b: *std.Build) !void { |
| 471 | 473 | .include_paths = &.{}, |
| 472 | 474 | .skip_single_threaded = true, |
| 473 | 475 | .skip_non_native = skip_non_native, |
| 476 | .test_default_only = test_default_only, | |
| 474 | 477 | .skip_freebsd = skip_freebsd, |
| 475 | 478 | .skip_netbsd = skip_netbsd, |
| 476 | 479 | .skip_windows = skip_windows, |
| ... | ... | @@ -492,6 +495,7 @@ pub fn build(b: *std.Build) !void { |
| 492 | 495 | .include_paths = &.{}, |
| 493 | 496 | .skip_single_threaded = true, |
| 494 | 497 | .skip_non_native = skip_non_native, |
| 498 | .test_default_only = test_default_only, | |
| 495 | 499 | .skip_freebsd = skip_freebsd, |
| 496 | 500 | .skip_netbsd = skip_netbsd, |
| 497 | 501 | .skip_windows = skip_windows, |
| ... | ... | @@ -513,6 +517,7 @@ pub fn build(b: *std.Build) !void { |
| 513 | 517 | .include_paths = &.{}, |
| 514 | 518 | .skip_single_threaded = skip_single_threaded, |
| 515 | 519 | .skip_non_native = skip_non_native, |
| 520 | .test_default_only = test_default_only, | |
| 516 | 521 | .skip_freebsd = skip_freebsd, |
| 517 | 522 | .skip_netbsd = skip_netbsd, |
| 518 | 523 | .skip_windows = skip_windows, |
lib/std/Io/Reader.zig+72-37| ... | ... | @@ -481,7 +481,6 @@ pub fn readVecAll(r: *Reader, data: [][]u8) Error!void { |
| 481 | 481 | /// is returned instead. |
| 482 | 482 | /// |
| 483 | 483 | /// See also: |
| 484 | /// * `peek` | |
| 485 | 484 | /// * `toss` |
| 486 | 485 | pub fn peek(r: *Reader, n: usize) Error![]u8 { |
| 487 | 486 | try r.fill(n); |
| ... | ... | @@ -732,7 +731,7 @@ pub const DelimiterError = error{ |
| 732 | 731 | }; |
| 733 | 732 | |
| 734 | 733 | /// Returns a slice of the next bytes of buffered data from the stream until |
| 735 | /// `sentinel` is found, advancing the seek position. | |
| 734 | /// `sentinel` is found, advancing the seek position past the sentinel. | |
| 736 | 735 | /// |
| 737 | 736 | /// Returned slice has a sentinel. |
| 738 | 737 | /// |
| ... | ... | @@ -765,7 +764,7 @@ pub fn peekSentinel(r: *Reader, comptime sentinel: u8) DelimiterError![:sentinel |
| 765 | 764 | } |
| 766 | 765 | |
| 767 | 766 | /// Returns a slice of the next bytes of buffered data from the stream until |
| 768 | /// `delimiter` is found, advancing the seek position. | |
| 767 | /// `delimiter` is found, advancing the seek position past the delimiter. | |
| 769 | 768 | /// |
| 770 | 769 | /// Returned slice includes the delimiter as the last byte. |
| 771 | 770 | /// |
| ... | ... | @@ -793,32 +792,42 @@ pub fn takeDelimiterInclusive(r: *Reader, delimiter: u8) DelimiterError![]u8 { |
| 793 | 792 | /// * `peekDelimiterExclusive` |
| 794 | 793 | /// * `takeDelimiterInclusive` |
| 795 | 794 | pub fn peekDelimiterInclusive(r: *Reader, delimiter: u8) DelimiterError![]u8 { |
| 796 | const buffer = r.buffer[0..r.end]; | |
| 797 | const seek = r.seek; | |
| 798 | if (std.mem.indexOfScalarPos(u8, buffer, seek, delimiter)) |delimiter_index| { | |
| 799 | @branchHint(.likely); | |
| 800 | return buffer[seek .. delimiter_index + 1]; | |
| 795 | { | |
| 796 | const contents = r.buffer[0..r.end]; | |
| 797 | const seek = r.seek; | |
| 798 | if (std.mem.findScalarPos(u8, contents, seek, delimiter)) |end| { | |
| 799 | @branchHint(.likely); | |
| 800 | return contents[seek .. end + 1]; | |
| 801 | } | |
| 801 | 802 | } |
| 802 | // TODO take a parameter for max search length rather than relying on buffer capacity | |
| 803 | try rebase(r, r.buffer.len); | |
| 804 | while (r.buffer.len - r.end != 0) { | |
| 805 | const existing_buffered_len = r.end - r.seek; | |
| 806 | const end_cap = r.buffer[r.end..]; | |
| 807 | var writer: Writer = .fixed(end_cap); | |
| 808 | const n = r.vtable.stream(r, &writer, .limited(end_cap.len)) catch |err| switch (err) { | |
| 809 | error.WriteFailed => unreachable, | |
| 810 | else => |e| return e, | |
| 811 | }; | |
| 812 | r.end += n; | |
| 813 | if (std.mem.indexOfScalarPos(u8, r.buffer[0..r.end], r.seek + existing_buffered_len, delimiter)) |delimiter_index| { | |
| 814 | return r.buffer[r.seek .. delimiter_index + 1]; | |
| 803 | while (true) { | |
| 804 | const content_len = r.end - r.seek; | |
| 805 | if (r.buffer.len - content_len == 0) break; | |
| 806 | try fillMore(r); | |
| 807 | const seek = r.seek; | |
| 808 | const contents = r.buffer[0..r.end]; | |
| 809 | if (std.mem.findScalarPos(u8, contents, seek + content_len, delimiter)) |end| { | |
| 810 | return contents[seek .. end + 1]; | |
| 815 | 811 | } |
| 816 | 812 | } |
| 817 | return error.StreamTooLong; | |
| 813 | // It might or might not be end of stream. There is no more buffer space | |
| 814 | // left to disambiguate. If `StreamTooLong` was added to `RebaseError` then | |
| 815 | // this logic could be replaced by removing the exit condition from the | |
| 816 | // above while loop. That error code would represent when `buffer` capacity | |
| 817 | // is too small for an operation, replacing the current use of asserts. | |
| 818 | var failing_writer = Writer.failing; | |
| 819 | while (r.vtable.stream(r, &failing_writer, .limited(1))) |n| { | |
| 820 | assert(n == 0); | |
| 821 | } else |err| switch (err) { | |
| 822 | error.WriteFailed => return error.StreamTooLong, | |
| 823 | error.ReadFailed => |e| return e, | |
| 824 | error.EndOfStream => |e| return e, | |
| 825 | } | |
| 818 | 826 | } |
| 819 | 827 | |
| 820 | 828 | /// Returns a slice of the next bytes of buffered data from the stream until |
| 821 | /// `delimiter` is found, advancing the seek position up to the delimiter. | |
| 829 | /// `delimiter` is found, advancing the seek position up to (but not past) | |
| 830 | /// the delimiter. | |
| 822 | 831 | /// |
| 823 | 832 | /// Returned slice excludes the delimiter. End-of-stream is treated equivalent |
| 824 | 833 | /// to a delimiter, unless it would result in a length 0 return value, in which |
| ... | ... | @@ -832,20 +841,13 @@ pub fn peekDelimiterInclusive(r: *Reader, delimiter: u8) DelimiterError![]u8 { |
| 832 | 841 | /// Invalidates previously returned values from `peek`. |
| 833 | 842 | /// |
| 834 | 843 | /// See also: |
| 844 | /// * `takeDelimiter` | |
| 835 | 845 | /// * `takeDelimiterInclusive` |
| 836 | 846 | /// * `peekDelimiterExclusive` |
| 837 | 847 | pub fn takeDelimiterExclusive(r: *Reader, delimiter: u8) DelimiterError![]u8 { |
| 838 | const result = r.peekDelimiterInclusive(delimiter) catch |err| switch (err) { | |
| 839 | error.EndOfStream => { | |
| 840 | const remaining = r.buffer[r.seek..r.end]; | |
| 841 | if (remaining.len == 0) return error.EndOfStream; | |
| 842 | r.toss(remaining.len); | |
| 843 | return remaining; | |
| 844 | }, | |
| 845 | else => |e| return e, | |
| 846 | }; | |
| 848 | const result = try r.peekDelimiterExclusive(delimiter); | |
| 847 | 849 | r.toss(result.len); |
| 848 | return result[0 .. result.len - 1]; | |
| 850 | return result; | |
| 849 | 851 | } |
| 850 | 852 | |
| 851 | 853 | /// Returns a slice of the next bytes of buffered data from the stream until |
| ... | ... | @@ -866,7 +868,7 @@ pub fn takeDelimiterExclusive(r: *Reader, delimiter: u8) DelimiterError![]u8 { |
| 866 | 868 | /// * `takeDelimiterInclusive` |
| 867 | 869 | /// * `takeDelimiterExclusive` |
| 868 | 870 | pub fn takeDelimiter(r: *Reader, delimiter: u8) error{ ReadFailed, StreamTooLong }!?[]u8 { |
| 869 | const result = r.peekDelimiterInclusive(delimiter) catch |err| switch (err) { | |
| 871 | const inclusive = r.peekDelimiterInclusive(delimiter) catch |err| switch (err) { | |
| 870 | 872 | error.EndOfStream => { |
| 871 | 873 | const remaining = r.buffer[r.seek..r.end]; |
| 872 | 874 | if (remaining.len == 0) return null; |
| ... | ... | @@ -875,8 +877,8 @@ pub fn takeDelimiter(r: *Reader, delimiter: u8) error{ ReadFailed, StreamTooLong |
| 875 | 877 | }, |
| 876 | 878 | else => |e| return e, |
| 877 | 879 | }; |
| 878 | r.toss(result.len + 1); | |
| 879 | return result[0 .. result.len - 1]; | |
| 880 | r.toss(inclusive.len); | |
| 881 | return inclusive[0 .. inclusive.len - 1]; | |
| 880 | 882 | } |
| 881 | 883 | |
| 882 | 884 | /// Returns a slice of the next bytes of buffered data from the stream until |
| ... | ... | @@ -1403,6 +1405,9 @@ test peekSentinel { |
| 1403 | 1405 | var r: Reader = .fixed("ab\nc"); |
| 1404 | 1406 | try testing.expectEqualStrings("ab", try r.peekSentinel('\n')); |
| 1405 | 1407 | try testing.expectEqualStrings("ab", try r.peekSentinel('\n')); |
| 1408 | r.toss(3); | |
| 1409 | try testing.expectError(error.EndOfStream, r.peekSentinel('\n')); | |
| 1410 | try testing.expectEqualStrings("c", try r.peek(1)); | |
| 1406 | 1411 | } |
| 1407 | 1412 | |
| 1408 | 1413 | test takeDelimiterInclusive { |
| ... | ... | @@ -1417,22 +1422,52 @@ test peekDelimiterInclusive { |
| 1417 | 1422 | try testing.expectEqualStrings("ab\n", try r.peekDelimiterInclusive('\n')); |
| 1418 | 1423 | r.toss(3); |
| 1419 | 1424 | try testing.expectError(error.EndOfStream, r.peekDelimiterInclusive('\n')); |
| 1425 | try testing.expectEqualStrings("c", try r.peek(1)); | |
| 1420 | 1426 | } |
| 1421 | 1427 | |
| 1422 | 1428 | test takeDelimiterExclusive { |
| 1423 | 1429 | var r: Reader = .fixed("ab\nc"); |
| 1430 | ||
| 1424 | 1431 | try testing.expectEqualStrings("ab", try r.takeDelimiterExclusive('\n')); |
| 1432 | try testing.expectEqualStrings("", try r.takeDelimiterExclusive('\n')); | |
| 1433 | try testing.expectEqualStrings("", try r.takeDelimiterExclusive('\n')); | |
| 1434 | try testing.expectEqualStrings("\n", try r.take(1)); | |
| 1435 | ||
| 1425 | 1436 | try testing.expectEqualStrings("c", try r.takeDelimiterExclusive('\n')); |
| 1426 | 1437 | try testing.expectError(error.EndOfStream, r.takeDelimiterExclusive('\n')); |
| 1427 | 1438 | } |
| 1428 | 1439 | |
| 1429 | 1440 | test peekDelimiterExclusive { |
| 1430 | 1441 | var r: Reader = .fixed("ab\nc"); |
| 1442 | ||
| 1431 | 1443 | try testing.expectEqualStrings("ab", try r.peekDelimiterExclusive('\n')); |
| 1432 | 1444 | try testing.expectEqualStrings("ab", try r.peekDelimiterExclusive('\n')); |
| 1433 | r.toss(3); | |
| 1445 | r.toss(2); | |
| 1446 | try testing.expectEqualStrings("", try r.peekDelimiterExclusive('\n')); | |
| 1447 | try testing.expectEqualStrings("\n", try r.take(1)); | |
| 1448 | ||
| 1434 | 1449 | try testing.expectEqualStrings("c", try r.peekDelimiterExclusive('\n')); |
| 1435 | 1450 | try testing.expectEqualStrings("c", try r.peekDelimiterExclusive('\n')); |
| 1451 | r.toss(1); | |
| 1452 | try testing.expectError(error.EndOfStream, r.peekDelimiterExclusive('\n')); | |
| 1453 | } | |
| 1454 | ||
| 1455 | test takeDelimiter { | |
| 1456 | var r: Reader = .fixed("ab\nc\n\nd"); | |
| 1457 | try testing.expectEqualStrings("ab", (try r.takeDelimiter('\n')).?); | |
| 1458 | try testing.expectEqualStrings("c", (try r.takeDelimiter('\n')).?); | |
| 1459 | try testing.expectEqualStrings("", (try r.takeDelimiter('\n')).?); | |
| 1460 | try testing.expectEqualStrings("d", (try r.takeDelimiter('\n')).?); | |
| 1461 | try testing.expectEqual(null, try r.takeDelimiter('\n')); | |
| 1462 | try testing.expectEqual(null, try r.takeDelimiter('\n')); | |
| 1463 | ||
| 1464 | r = .fixed("ab\nc\n\nd\n"); // one trailing newline does not affect behavior | |
| 1465 | try testing.expectEqualStrings("ab", (try r.takeDelimiter('\n')).?); | |
| 1466 | try testing.expectEqualStrings("c", (try r.takeDelimiter('\n')).?); | |
| 1467 | try testing.expectEqualStrings("", (try r.takeDelimiter('\n')).?); | |
| 1468 | try testing.expectEqualStrings("d", (try r.takeDelimiter('\n')).?); | |
| 1469 | try testing.expectEqual(null, try r.takeDelimiter('\n')); | |
| 1470 | try testing.expectEqual(null, try r.takeDelimiter('\n')); | |
| 1436 | 1471 | } |
| 1437 | 1472 | |
| 1438 | 1473 | test streamDelimiter { |
lib/std/Io/Reader/Limited.zig+44| ... | ... | @@ -27,6 +27,7 @@ pub fn init(reader: *Reader, limit: Limit, buffer: []u8) Limited { |
| 27 | 27 | |
| 28 | 28 | fn stream(r: *Reader, w: *Writer, limit: Limit) Reader.StreamError!usize { |
| 29 | 29 | const l: *Limited = @fieldParentPtr("interface", r); |
| 30 | if (l.remaining == .nothing) return error.EndOfStream; | |
| 30 | 31 | const combined_limit = limit.min(l.remaining); |
| 31 | 32 | const n = try l.unlimited.stream(w, combined_limit); |
| 32 | 33 | l.remaining = l.remaining.subtract(n).?; |
| ... | ... | @@ -51,8 +52,51 @@ test stream { |
| 51 | 52 | |
| 52 | 53 | fn discard(r: *Reader, limit: Limit) Reader.Error!usize { |
| 53 | 54 | const l: *Limited = @fieldParentPtr("interface", r); |
| 55 | if (l.remaining == .nothing) return error.EndOfStream; | |
| 54 | 56 | const combined_limit = limit.min(l.remaining); |
| 55 | 57 | const n = try l.unlimited.discard(combined_limit); |
| 56 | 58 | l.remaining = l.remaining.subtract(n).?; |
| 57 | 59 | return n; |
| 58 | 60 | } |
| 61 | ||
| 62 | test "end of stream, read, hit limit exactly" { | |
| 63 | var f: Reader = .fixed("i'm dying"); | |
| 64 | var l = f.limited(.limited(4), &.{}); | |
| 65 | const r = &l.interface; | |
| 66 | ||
| 67 | var buf: [2]u8 = undefined; | |
| 68 | try r.readSliceAll(&buf); | |
| 69 | try r.readSliceAll(&buf); | |
| 70 | try std.testing.expectError(error.EndOfStream, l.interface.readSliceAll(&buf)); | |
| 71 | } | |
| 72 | ||
| 73 | test "end of stream, read, hit limit after partial read" { | |
| 74 | var f: Reader = .fixed("i'm dying"); | |
| 75 | var l = f.limited(.limited(5), &.{}); | |
| 76 | const r = &l.interface; | |
| 77 | ||
| 78 | var buf: [2]u8 = undefined; | |
| 79 | try r.readSliceAll(&buf); | |
| 80 | try r.readSliceAll(&buf); | |
| 81 | try std.testing.expectError(error.EndOfStream, l.interface.readSliceAll(&buf)); | |
| 82 | } | |
| 83 | ||
| 84 | test "end of stream, discard, hit limit exactly" { | |
| 85 | var f: Reader = .fixed("i'm dying"); | |
| 86 | var l = f.limited(.limited(4), &.{}); | |
| 87 | const r = &l.interface; | |
| 88 | ||
| 89 | try r.discardAll(2); | |
| 90 | try r.discardAll(2); | |
| 91 | try std.testing.expectError(error.EndOfStream, l.interface.discardAll(2)); | |
| 92 | } | |
| 93 | ||
| 94 | test "end of stream, discard, hit limit after partial read" { | |
| 95 | var f: Reader = .fixed("i'm dying"); | |
| 96 | var l = f.limited(.limited(5), &.{}); | |
| 97 | const r = &l.interface; | |
| 98 | ||
| 99 | try r.discardAll(2); | |
| 100 | try r.discardAll(2); | |
| 101 | try std.testing.expectError(error.EndOfStream, l.interface.discardAll(2)); | |
| 102 | } |
lib/std/Io/Writer.zig+31-6| ... | ... | @@ -923,10 +923,12 @@ pub fn sendFileHeader( |
| 923 | 923 | return n; |
| 924 | 924 | } |
| 925 | 925 | |
| 926 | /// Asserts nonzero buffer capacity. | |
| 926 | /// Asserts nonzero buffer capacity and nonzero `limit`. | |
| 927 | 927 | pub fn sendFileReading(w: *Writer, file_reader: *File.Reader, limit: Limit) FileReadingError!usize { |
| 928 | assert(limit != .nothing); | |
| 928 | 929 | const dest = limit.slice(try w.writableSliceGreedy(1)); |
| 929 | const n = try file_reader.read(dest); | |
| 930 | const n = try file_reader.interface.readSliceShort(dest); | |
| 931 | if (n == 0) return error.EndOfStream; | |
| 930 | 932 | w.advance(n); |
| 931 | 933 | return n; |
| 932 | 934 | } |
| ... | ... | @@ -2778,7 +2780,8 @@ pub const Allocating = struct { |
| 2778 | 2780 | if (additional == 0) return error.EndOfStream; |
| 2779 | 2781 | a.ensureUnusedCapacity(limit.minInt64(additional)) catch return error.WriteFailed; |
| 2780 | 2782 | const dest = limit.slice(a.writer.buffer[a.writer.end..]); |
| 2781 | const n = try file_reader.read(dest); | |
| 2783 | const n = try file_reader.interface.readSliceShort(dest); | |
| 2784 | if (n == 0) return error.EndOfStream; | |
| 2782 | 2785 | a.writer.end += n; |
| 2783 | 2786 | return n; |
| 2784 | 2787 | } |
| ... | ... | @@ -2849,18 +2852,40 @@ test "allocating sendFile" { |
| 2849 | 2852 | |
| 2850 | 2853 | const file = try tmp_dir.dir.createFile("input.txt", .{ .read = true }); |
| 2851 | 2854 | defer file.close(); |
| 2852 | var r_buffer: [256]u8 = undefined; | |
| 2855 | var r_buffer: [2]u8 = undefined; | |
| 2853 | 2856 | var file_writer: std.fs.File.Writer = .init(file, &r_buffer); |
| 2854 | try file_writer.interface.writeByte('h'); | |
| 2857 | try file_writer.interface.writeAll("abcd"); | |
| 2855 | 2858 | try file_writer.interface.flush(); |
| 2856 | 2859 | |
| 2857 | 2860 | var file_reader = file_writer.moveToReader(); |
| 2858 | 2861 | try file_reader.seekTo(0); |
| 2862 | try file_reader.interface.fill(2); | |
| 2859 | 2863 | |
| 2860 | 2864 | var allocating: Writer.Allocating = .init(testing.allocator); |
| 2861 | 2865 | defer allocating.deinit(); |
| 2866 | try allocating.ensureUnusedCapacity(1); | |
| 2867 | try testing.expectEqual(4, allocating.writer.sendFileAll(&file_reader, .unlimited)); | |
| 2868 | try testing.expectEqualStrings("abcd", allocating.writer.buffered()); | |
| 2869 | } | |
| 2870 | ||
| 2871 | test sendFileReading { | |
| 2872 | var tmp_dir = testing.tmpDir(.{}); | |
| 2873 | defer tmp_dir.cleanup(); | |
| 2874 | ||
| 2875 | const file = try tmp_dir.dir.createFile("input.txt", .{ .read = true }); | |
| 2876 | defer file.close(); | |
| 2877 | var r_buffer: [2]u8 = undefined; | |
| 2878 | var file_writer: std.fs.File.Writer = .init(file, &r_buffer); | |
| 2879 | try file_writer.interface.writeAll("abcd"); | |
| 2880 | try file_writer.interface.flush(); | |
| 2862 | 2881 | |
| 2863 | _ = try file_reader.interface.streamRemaining(&allocating.writer); | |
| 2882 | var file_reader = file_writer.moveToReader(); | |
| 2883 | try file_reader.seekTo(0); | |
| 2884 | try file_reader.interface.fill(2); | |
| 2885 | ||
| 2886 | var w_buffer: [1]u8 = undefined; | |
| 2887 | var discarding: Writer.Discarding = .init(&w_buffer); | |
| 2888 | try testing.expectEqual(4, discarding.writer.sendFileReadingAll(&file_reader, .unlimited)); | |
| 2864 | 2889 | } |
| 2865 | 2890 | |
| 2866 | 2891 | test writeStruct { |
lib/std/fs/File.zig+103-103| ... | ... | @@ -1189,7 +1189,7 @@ pub const Reader = struct { |
| 1189 | 1189 | pub fn seekBy(r: *Reader, offset: i64) Reader.SeekError!void { |
| 1190 | 1190 | switch (r.mode) { |
| 1191 | 1191 | .positional, .positional_reading => { |
| 1192 | setPosAdjustingBuffer(r, @intCast(@as(i64, @intCast(r.pos)) + offset)); | |
| 1192 | setLogicalPos(r, @intCast(@as(i64, @intCast(logicalPos(r))) + offset)); | |
| 1193 | 1193 | }, |
| 1194 | 1194 | .streaming, .streaming_reading => { |
| 1195 | 1195 | if (posix.SEEK == void) { |
| ... | ... | @@ -1198,7 +1198,7 @@ pub const Reader = struct { |
| 1198 | 1198 | } |
| 1199 | 1199 | const seek_err = r.seek_err orelse e: { |
| 1200 | 1200 | if (posix.lseek_CUR(r.file.handle, offset)) |_| { |
| 1201 | setPosAdjustingBuffer(r, @intCast(@as(i64, @intCast(r.pos)) + offset)); | |
| 1201 | setLogicalPos(r, @intCast(@as(i64, @intCast(logicalPos(r))) + offset)); | |
| 1202 | 1202 | return; |
| 1203 | 1203 | } else |err| { |
| 1204 | 1204 | r.seek_err = err; |
| ... | ... | @@ -1222,16 +1222,16 @@ pub const Reader = struct { |
| 1222 | 1222 | pub fn seekTo(r: *Reader, offset: u64) Reader.SeekError!void { |
| 1223 | 1223 | switch (r.mode) { |
| 1224 | 1224 | .positional, .positional_reading => { |
| 1225 | setPosAdjustingBuffer(r, offset); | |
| 1225 | setLogicalPos(r, offset); | |
| 1226 | 1226 | }, |
| 1227 | 1227 | .streaming, .streaming_reading => { |
| 1228 | if (offset >= r.pos) return Reader.seekBy(r, @intCast(offset - r.pos)); | |
| 1228 | if (offset >= r.pos) return Reader.seekBy(r, @intCast(offset - logicalPos(r))); | |
| 1229 | 1229 | if (r.seek_err) |err| return err; |
| 1230 | 1230 | posix.lseek_SET(r.file.handle, offset) catch |err| { |
| 1231 | 1231 | r.seek_err = err; |
| 1232 | 1232 | return err; |
| 1233 | 1233 | }; |
| 1234 | setPosAdjustingBuffer(r, offset); | |
| 1234 | setLogicalPos(r, offset); | |
| 1235 | 1235 | }, |
| 1236 | 1236 | .failure => return r.seek_err.?, |
| 1237 | 1237 | } |
| ... | ... | @@ -1241,7 +1241,7 @@ pub const Reader = struct { |
| 1241 | 1241 | return r.pos - r.interface.bufferedLen(); |
| 1242 | 1242 | } |
| 1243 | 1243 | |
| 1244 | fn setPosAdjustingBuffer(r: *Reader, offset: u64) void { | |
| 1244 | fn setLogicalPos(r: *Reader, offset: u64) void { | |
| 1245 | 1245 | const logical_pos = logicalPos(r); |
| 1246 | 1246 | if (offset < logical_pos or offset >= r.pos) { |
| 1247 | 1247 | r.interface.seek = 0; |
| ... | ... | @@ -1269,13 +1269,15 @@ pub const Reader = struct { |
| 1269 | 1269 | }, |
| 1270 | 1270 | .positional_reading => { |
| 1271 | 1271 | const dest = limit.slice(try w.writableSliceGreedy(1)); |
| 1272 | const n = try readPositional(r, dest); | |
| 1272 | var data: [1][]u8 = .{dest}; | |
| 1273 | const n = try readVecPositional(r, &data); | |
| 1273 | 1274 | w.advance(n); |
| 1274 | 1275 | return n; |
| 1275 | 1276 | }, |
| 1276 | 1277 | .streaming_reading => { |
| 1277 | 1278 | const dest = limit.slice(try w.writableSliceGreedy(1)); |
| 1278 | const n = try readStreaming(r, dest); | |
| 1279 | var data: [1][]u8 = .{dest}; | |
| 1280 | const n = try readVecStreaming(r, &data); | |
| 1279 | 1281 | w.advance(n); |
| 1280 | 1282 | return n; |
| 1281 | 1283 | }, |
| ... | ... | @@ -1286,92 +1288,98 @@ pub const Reader = struct { |
| 1286 | 1288 | fn readVec(io_reader: *std.Io.Reader, data: [][]u8) std.Io.Reader.Error!usize { |
| 1287 | 1289 | const r: *Reader = @alignCast(@fieldParentPtr("interface", io_reader)); |
| 1288 | 1290 | switch (r.mode) { |
| 1289 | .positional, .positional_reading => { | |
| 1290 | if (is_windows) { | |
| 1291 | // Unfortunately, `ReadFileScatter` cannot be used since it | |
| 1292 | // requires page alignment. | |
| 1293 | if (io_reader.seek == io_reader.end) { | |
| 1294 | io_reader.seek = 0; | |
| 1295 | io_reader.end = 0; | |
| 1296 | } | |
| 1297 | const first = data[0]; | |
| 1298 | if (first.len >= io_reader.buffer.len - io_reader.end) { | |
| 1299 | return readPositional(r, first); | |
| 1300 | } else { | |
| 1301 | io_reader.end += try readPositional(r, io_reader.buffer[io_reader.end..]); | |
| 1302 | return 0; | |
| 1303 | } | |
| 1304 | } | |
| 1305 | var iovecs_buffer: [max_buffers_len]posix.iovec = undefined; | |
| 1306 | const dest_n, const data_size = try io_reader.writableVectorPosix(&iovecs_buffer, data); | |
| 1307 | const dest = iovecs_buffer[0..dest_n]; | |
| 1308 | assert(dest[0].len > 0); | |
| 1309 | const n = posix.preadv(r.file.handle, dest, r.pos) catch |err| switch (err) { | |
| 1310 | error.Unseekable => { | |
| 1311 | r.mode = r.mode.toStreaming(); | |
| 1312 | const pos = r.pos; | |
| 1313 | if (pos != 0) { | |
| 1314 | r.pos = 0; | |
| 1315 | r.seekBy(@intCast(pos)) catch { | |
| 1316 | r.mode = .failure; | |
| 1317 | return error.ReadFailed; | |
| 1318 | }; | |
| 1319 | } | |
| 1320 | return 0; | |
| 1321 | }, | |
| 1322 | else => |e| { | |
| 1323 | r.err = e; | |
| 1291 | .positional, .positional_reading => return readVecPositional(r, data), | |
| 1292 | .streaming, .streaming_reading => return readVecStreaming(r, data), | |
| 1293 | .failure => return error.ReadFailed, | |
| 1294 | } | |
| 1295 | } | |
| 1296 | ||
| 1297 | fn readVecPositional(r: *Reader, data: [][]u8) std.Io.Reader.Error!usize { | |
| 1298 | const io_reader = &r.interface; | |
| 1299 | if (is_windows) { | |
| 1300 | // Unfortunately, `ReadFileScatter` cannot be used since it | |
| 1301 | // requires page alignment. | |
| 1302 | if (io_reader.seek == io_reader.end) { | |
| 1303 | io_reader.seek = 0; | |
| 1304 | io_reader.end = 0; | |
| 1305 | } | |
| 1306 | const first = data[0]; | |
| 1307 | if (first.len >= io_reader.buffer.len - io_reader.end) { | |
| 1308 | return readPositional(r, first); | |
| 1309 | } else { | |
| 1310 | io_reader.end += try readPositional(r, io_reader.buffer[io_reader.end..]); | |
| 1311 | return 0; | |
| 1312 | } | |
| 1313 | } | |
| 1314 | var iovecs_buffer: [max_buffers_len]posix.iovec = undefined; | |
| 1315 | const dest_n, const data_size = try io_reader.writableVectorPosix(&iovecs_buffer, data); | |
| 1316 | const dest = iovecs_buffer[0..dest_n]; | |
| 1317 | assert(dest[0].len > 0); | |
| 1318 | const n = posix.preadv(r.file.handle, dest, r.pos) catch |err| switch (err) { | |
| 1319 | error.Unseekable => { | |
| 1320 | r.mode = r.mode.toStreaming(); | |
| 1321 | const pos = r.pos; | |
| 1322 | if (pos != 0) { | |
| 1323 | r.pos = 0; | |
| 1324 | r.seekBy(@intCast(pos)) catch { | |
| 1325 | r.mode = .failure; | |
| 1324 | 1326 | return error.ReadFailed; |
| 1325 | }, | |
| 1326 | }; | |
| 1327 | if (n == 0) { | |
| 1328 | r.size = r.pos; | |
| 1329 | return error.EndOfStream; | |
| 1330 | } | |
| 1331 | r.pos += n; | |
| 1332 | if (n > data_size) { | |
| 1333 | io_reader.end += n - data_size; | |
| 1334 | return data_size; | |
| 1327 | }; | |
| 1335 | 1328 | } |
| 1336 | return n; | |
| 1329 | return 0; | |
| 1337 | 1330 | }, |
| 1338 | .streaming, .streaming_reading => { | |
| 1339 | if (is_windows) { | |
| 1340 | // Unfortunately, `ReadFileScatter` cannot be used since it | |
| 1341 | // requires page alignment. | |
| 1342 | if (io_reader.seek == io_reader.end) { | |
| 1343 | io_reader.seek = 0; | |
| 1344 | io_reader.end = 0; | |
| 1345 | } | |
| 1346 | const first = data[0]; | |
| 1347 | if (first.len >= io_reader.buffer.len - io_reader.end) { | |
| 1348 | return readStreaming(r, first); | |
| 1349 | } else { | |
| 1350 | io_reader.end += try readStreaming(r, io_reader.buffer[io_reader.end..]); | |
| 1351 | return 0; | |
| 1352 | } | |
| 1353 | } | |
| 1354 | var iovecs_buffer: [max_buffers_len]posix.iovec = undefined; | |
| 1355 | const dest_n, const data_size = try io_reader.writableVectorPosix(&iovecs_buffer, data); | |
| 1356 | const dest = iovecs_buffer[0..dest_n]; | |
| 1357 | assert(dest[0].len > 0); | |
| 1358 | const n = posix.readv(r.file.handle, dest) catch |err| { | |
| 1359 | r.err = err; | |
| 1360 | return error.ReadFailed; | |
| 1361 | }; | |
| 1362 | if (n == 0) { | |
| 1363 | r.size = r.pos; | |
| 1364 | return error.EndOfStream; | |
| 1365 | } | |
| 1366 | r.pos += n; | |
| 1367 | if (n > data_size) { | |
| 1368 | io_reader.end += n - data_size; | |
| 1369 | return data_size; | |
| 1370 | } | |
| 1371 | return n; | |
| 1331 | else => |e| { | |
| 1332 | r.err = e; | |
| 1333 | return error.ReadFailed; | |
| 1372 | 1334 | }, |
| 1373 | .failure => return error.ReadFailed, | |
| 1335 | }; | |
| 1336 | if (n == 0) { | |
| 1337 | r.size = r.pos; | |
| 1338 | return error.EndOfStream; | |
| 1339 | } | |
| 1340 | r.pos += n; | |
| 1341 | if (n > data_size) { | |
| 1342 | io_reader.end += n - data_size; | |
| 1343 | return data_size; | |
| 1374 | 1344 | } |
| 1345 | return n; | |
| 1346 | } | |
| 1347 | ||
| 1348 | fn readVecStreaming(r: *Reader, data: [][]u8) std.Io.Reader.Error!usize { | |
| 1349 | const io_reader = &r.interface; | |
| 1350 | if (is_windows) { | |
| 1351 | // Unfortunately, `ReadFileScatter` cannot be used since it | |
| 1352 | // requires page alignment. | |
| 1353 | if (io_reader.seek == io_reader.end) { | |
| 1354 | io_reader.seek = 0; | |
| 1355 | io_reader.end = 0; | |
| 1356 | } | |
| 1357 | const first = data[0]; | |
| 1358 | if (first.len >= io_reader.buffer.len - io_reader.end) { | |
| 1359 | return readStreaming(r, first); | |
| 1360 | } else { | |
| 1361 | io_reader.end += try readStreaming(r, io_reader.buffer[io_reader.end..]); | |
| 1362 | return 0; | |
| 1363 | } | |
| 1364 | } | |
| 1365 | var iovecs_buffer: [max_buffers_len]posix.iovec = undefined; | |
| 1366 | const dest_n, const data_size = try io_reader.writableVectorPosix(&iovecs_buffer, data); | |
| 1367 | const dest = iovecs_buffer[0..dest_n]; | |
| 1368 | assert(dest[0].len > 0); | |
| 1369 | const n = posix.readv(r.file.handle, dest) catch |err| { | |
| 1370 | r.err = err; | |
| 1371 | return error.ReadFailed; | |
| 1372 | }; | |
| 1373 | if (n == 0) { | |
| 1374 | r.size = r.pos; | |
| 1375 | return error.EndOfStream; | |
| 1376 | } | |
| 1377 | r.pos += n; | |
| 1378 | if (n > data_size) { | |
| 1379 | io_reader.end += n - data_size; | |
| 1380 | return data_size; | |
| 1381 | } | |
| 1382 | return n; | |
| 1375 | 1383 | } |
| 1376 | 1384 | |
| 1377 | 1385 | fn discard(io_reader: *std.Io.Reader, limit: std.Io.Limit) std.Io.Reader.Error!usize { |
| ... | ... | @@ -1440,7 +1448,7 @@ pub const Reader = struct { |
| 1440 | 1448 | } |
| 1441 | 1449 | } |
| 1442 | 1450 | |
| 1443 | pub fn readPositional(r: *Reader, dest: []u8) std.Io.Reader.Error!usize { | |
| 1451 | fn readPositional(r: *Reader, dest: []u8) std.Io.Reader.Error!usize { | |
| 1444 | 1452 | const n = r.file.pread(dest, r.pos) catch |err| switch (err) { |
| 1445 | 1453 | error.Unseekable => { |
| 1446 | 1454 | r.mode = r.mode.toStreaming(); |
| ... | ... | @@ -1467,7 +1475,7 @@ pub const Reader = struct { |
| 1467 | 1475 | return n; |
| 1468 | 1476 | } |
| 1469 | 1477 | |
| 1470 | pub fn readStreaming(r: *Reader, dest: []u8) std.Io.Reader.Error!usize { | |
| 1478 | fn readStreaming(r: *Reader, dest: []u8) std.Io.Reader.Error!usize { | |
| 1471 | 1479 | const n = r.file.read(dest) catch |err| { |
| 1472 | 1480 | r.err = err; |
| 1473 | 1481 | return error.ReadFailed; |
| ... | ... | @@ -1480,14 +1488,6 @@ pub const Reader = struct { |
| 1480 | 1488 | return n; |
| 1481 | 1489 | } |
| 1482 | 1490 | |
| 1483 | pub fn read(r: *Reader, dest: []u8) std.Io.Reader.Error!usize { | |
| 1484 | switch (r.mode) { | |
| 1485 | .positional, .positional_reading => return readPositional(r, dest), | |
| 1486 | .streaming, .streaming_reading => return readStreaming(r, dest), | |
| 1487 | .failure => return error.ReadFailed, | |
| 1488 | } | |
| 1489 | } | |
| 1490 | ||
| 1491 | 1491 | pub fn atEnd(r: *Reader) bool { |
| 1492 | 1492 | // Even if stat fails, size is set when end is encountered. |
| 1493 | 1493 | const size = r.size orelse return false; |
| ... | ... | @@ -1732,7 +1732,7 @@ pub const Writer = struct { |
| 1732 | 1732 | ) std.Io.Writer.FileError!usize { |
| 1733 | 1733 | const reader_buffered = file_reader.interface.buffered(); |
| 1734 | 1734 | if (reader_buffered.len >= @intFromEnum(limit)) |
| 1735 | return sendFileBuffered(io_w, file_reader, reader_buffered); | |
| 1735 | return sendFileBuffered(io_w, file_reader, limit.slice(reader_buffered)); | |
| 1736 | 1736 | const writer_buffered = io_w.buffered(); |
| 1737 | 1737 | const file_limit = @intFromEnum(limit) - reader_buffered.len; |
| 1738 | 1738 | const w: *Writer = @alignCast(@fieldParentPtr("interface", io_w)); |
| ... | ... | @@ -1804,7 +1804,7 @@ pub const Writer = struct { |
| 1804 | 1804 | return error.EndOfStream; |
| 1805 | 1805 | } |
| 1806 | 1806 | const consumed = io_w.consume(@intCast(sbytes)); |
| 1807 | file_reader.seekTo(file_reader.pos + consumed) catch return error.ReadFailed; | |
| 1807 | file_reader.seekBy(@intCast(consumed)) catch return error.ReadFailed; | |
| 1808 | 1808 | return consumed; |
| 1809 | 1809 | } |
| 1810 | 1810 | |
| ... | ... | @@ -1865,7 +1865,7 @@ pub const Writer = struct { |
| 1865 | 1865 | return error.EndOfStream; |
| 1866 | 1866 | } |
| 1867 | 1867 | const consumed = io_w.consume(@bitCast(len)); |
| 1868 | file_reader.seekTo(file_reader.pos + consumed) catch return error.ReadFailed; | |
| 1868 | file_reader.seekBy(@intCast(consumed)) catch return error.ReadFailed; | |
| 1869 | 1869 | return consumed; |
| 1870 | 1870 | } |
| 1871 | 1871 | |
| ... | ... | @@ -1998,7 +1998,7 @@ pub const Writer = struct { |
| 1998 | 1998 | reader_buffered: []const u8, |
| 1999 | 1999 | ) std.Io.Writer.FileError!usize { |
| 2000 | 2000 | const n = try drain(io_w, &.{reader_buffered}, 1); |
| 2001 | file_reader.seekTo(file_reader.pos + n) catch return error.ReadFailed; | |
| 2001 | file_reader.seekBy(@intCast(n)) catch return error.ReadFailed; | |
| 2002 | 2002 | return n; |
| 2003 | 2003 | } |
| 2004 | 2004 |
lib/std/fs/test.zig+66| ... | ... | @@ -1525,6 +1525,41 @@ test "sendfile" { |
| 1525 | 1525 | try testing.expectEqualStrings("header1\nsecond header\nine1\nsecontrailer1\nsecond trailer\n", written_buf[0..amt]); |
| 1526 | 1526 | } |
| 1527 | 1527 | |
| 1528 | test "sendfile with buffered data" { | |
| 1529 | var tmp = tmpDir(.{}); | |
| 1530 | defer tmp.cleanup(); | |
| 1531 | ||
| 1532 | try tmp.dir.makePath("os_test_tmp"); | |
| 1533 | ||
| 1534 | var dir = try tmp.dir.openDir("os_test_tmp", .{}); | |
| 1535 | defer dir.close(); | |
| 1536 | ||
| 1537 | var src_file = try dir.createFile("sendfile1.txt", .{ .read = true }); | |
| 1538 | defer src_file.close(); | |
| 1539 | ||
| 1540 | try src_file.writeAll("AAAABBBB"); | |
| 1541 | ||
| 1542 | var dest_file = try dir.createFile("sendfile2.txt", .{ .read = true }); | |
| 1543 | defer dest_file.close(); | |
| 1544 | ||
| 1545 | var src_buffer: [32]u8 = undefined; | |
| 1546 | var file_reader = src_file.reader(&src_buffer); | |
| 1547 | ||
| 1548 | try file_reader.seekTo(0); | |
| 1549 | try file_reader.interface.fill(8); | |
| 1550 | ||
| 1551 | var fallback_buffer: [32]u8 = undefined; | |
| 1552 | var file_writer = dest_file.writer(&fallback_buffer); | |
| 1553 | ||
| 1554 | try std.testing.expectEqual(4, try file_writer.interface.sendFileAll(&file_reader, .limited(4))); | |
| 1555 | ||
| 1556 | var written_buf: [8]u8 = undefined; | |
| 1557 | const amt = try dest_file.preadAll(&written_buf, 0); | |
| 1558 | ||
| 1559 | try std.testing.expectEqual(4, amt); | |
| 1560 | try std.testing.expectEqualSlices(u8, "AAAA", written_buf[0..amt]); | |
| 1561 | } | |
| 1562 | ||
| 1528 | 1563 | test "copyRangeAll" { |
| 1529 | 1564 | var tmp = tmpDir(.{}); |
| 1530 | 1565 | defer tmp.cleanup(); |
| ... | ... | @@ -2250,3 +2285,34 @@ test "seekTo flushes buffered data" { |
| 2250 | 2285 | try file_reader.interface.readSliceAll(&buf); |
| 2251 | 2286 | try std.testing.expectEqualStrings(contents, &buf); |
| 2252 | 2287 | } |
| 2288 | ||
| 2289 | test "File.Writer sendfile with buffered contents" { | |
| 2290 | var tmp_dir = testing.tmpDir(.{}); | |
| 2291 | defer tmp_dir.cleanup(); | |
| 2292 | ||
| 2293 | { | |
| 2294 | try tmp_dir.dir.writeFile(.{ .sub_path = "a", .data = "bcd" }); | |
| 2295 | const in = try tmp_dir.dir.openFile("a", .{}); | |
| 2296 | defer in.close(); | |
| 2297 | const out = try tmp_dir.dir.createFile("b", .{}); | |
| 2298 | defer out.close(); | |
| 2299 | ||
| 2300 | var in_buf: [2]u8 = undefined; | |
| 2301 | var in_r = in.reader(&in_buf); | |
| 2302 | _ = try in_r.getSize(); // Catch seeks past end by populating size | |
| 2303 | try in_r.interface.fill(2); | |
| 2304 | ||
| 2305 | var out_buf: [1]u8 = undefined; | |
| 2306 | var out_w = out.writerStreaming(&out_buf); | |
| 2307 | try out_w.interface.writeByte('a'); | |
| 2308 | try testing.expectEqual(3, try out_w.interface.sendFileAll(&in_r, .unlimited)); | |
| 2309 | try out_w.interface.flush(); | |
| 2310 | } | |
| 2311 | ||
| 2312 | var check = try tmp_dir.dir.openFile("b", .{}); | |
| 2313 | defer check.close(); | |
| 2314 | var check_buf: [4]u8 = undefined; | |
| 2315 | var check_r = check.reader(&check_buf); | |
| 2316 | try testing.expectEqualStrings("abcd", try check_r.interface.take(4)); | |
| 2317 | try testing.expectError(error.EndOfStream, check_r.interface.takeByte()); | |
| 2318 | } |
lib/std/net.zig+3-2| ... | ... | @@ -1396,7 +1396,7 @@ fn parseHosts( |
| 1396 | 1396 | br: *Io.Reader, |
| 1397 | 1397 | ) error{ OutOfMemory, ReadFailed }!void { |
| 1398 | 1398 | while (true) { |
| 1399 | const line = br.takeDelimiterExclusive('\n') catch |err| switch (err) { | |
| 1399 | const line = br.takeDelimiter('\n') catch |err| switch (err) { | |
| 1400 | 1400 | error.StreamTooLong => { |
| 1401 | 1401 | // Skip lines that are too long. |
| 1402 | 1402 | _ = br.discardDelimiterInclusive('\n') catch |e| switch (e) { |
| ... | ... | @@ -1406,7 +1406,8 @@ fn parseHosts( |
| 1406 | 1406 | continue; |
| 1407 | 1407 | }, |
| 1408 | 1408 | error.ReadFailed => return error.ReadFailed, |
| 1409 | error.EndOfStream => break, | |
| 1409 | } orelse { | |
| 1410 | break; // end of stream | |
| 1410 | 1411 | }; |
| 1411 | 1412 | var split_it = mem.splitScalar(u8, line, '#'); |
| 1412 | 1413 | const no_comment_line = split_it.first(); |
lib/std/zig/system/linux.zig+1-4| ... | ... | @@ -359,14 +359,11 @@ fn CpuinfoParser(comptime impl: anytype) type { |
| 359 | 359 | return struct { |
| 360 | 360 | fn parse(arch: Target.Cpu.Arch, reader: *std.Io.Reader) !?Target.Cpu { |
| 361 | 361 | var obj: impl = .{}; |
| 362 | while (reader.takeDelimiterExclusive('\n')) |line| { | |
| 362 | while (try reader.takeDelimiter('\n')) |line| { | |
| 363 | 363 | const colon_pos = mem.indexOfScalar(u8, line, ':') orelse continue; |
| 364 | 364 | const key = mem.trimEnd(u8, line[0..colon_pos], " \t"); |
| 365 | 365 | const value = mem.trimStart(u8, line[colon_pos + 1 ..], " \t"); |
| 366 | 366 | if (!try obj.line_hook(key, value)) break; |
| 367 | } else |err| switch (err) { | |
| 368 | error.EndOfStream => {}, | |
| 369 | else => |e| return e, | |
| 370 | 367 | } |
| 371 | 368 | return obj.finalize(arch); |
| 372 | 369 | } |
src/link/MappedFile.zig+2-1| ... | ... | @@ -411,7 +411,8 @@ pub const Node = extern struct { |
| 411 | 411 | .failure, |
| 412 | 412 | => { |
| 413 | 413 | const dest = limit.slice(interface.unusedCapacitySlice()); |
| 414 | const n = try file_reader.read(dest); | |
| 414 | const n = try file_reader.interface.readSliceShort(dest); | |
| 415 | if (n == 0) return error.EndOfStream; | |
| 415 | 416 | interface.end += n; |
| 416 | 417 | return n; |
| 417 | 418 | }, |
test/tests.zig+174-152| ... | ... | @@ -44,7 +44,7 @@ const test_targets = blk: { |
| 44 | 44 | break :blk [_]TestTarget{ |
| 45 | 45 | // Native Targets |
| 46 | 46 | |
| 47 | .{}, | |
| 47 | .{}, // 0 index must be all defaults | |
| 48 | 48 | .{ |
| 49 | 49 | .link_libc = true, |
| 50 | 50 | }, |
| ... | ... | @@ -2224,6 +2224,7 @@ const ModuleTestOptions = struct { |
| 2224 | 2224 | desc: []const u8, |
| 2225 | 2225 | optimize_modes: []const OptimizeMode, |
| 2226 | 2226 | include_paths: []const []const u8, |
| 2227 | test_default_only: bool, | |
| 2227 | 2228 | skip_single_threaded: bool, |
| 2228 | 2229 | skip_non_native: bool, |
| 2229 | 2230 | skip_freebsd: bool, |
| ... | ... | @@ -2235,13 +2236,21 @@ const ModuleTestOptions = struct { |
| 2235 | 2236 | skip_libc: bool, |
| 2236 | 2237 | max_rss: usize = 0, |
| 2237 | 2238 | no_builtin: bool = false, |
| 2238 | build_options: ?*std.Build.Step.Options = null, | |
| 2239 | build_options: ?*Step.Options = null, | |
| 2239 | 2240 | }; |
| 2240 | 2241 | |
| 2241 | 2242 | pub fn addModuleTests(b: *std.Build, options: ModuleTestOptions) *Step { |
| 2242 | 2243 | const step = b.step(b.fmt("test-{s}", .{options.name}), options.desc); |
| 2243 | 2244 | |
| 2244 | for_targets: for (test_targets) |test_target| { | |
| 2245 | if (options.test_default_only) { | |
| 2246 | const test_target = &test_targets[0]; | |
| 2247 | const resolved_target = b.resolveTargetQuery(test_target.target); | |
| 2248 | const triple_txt = resolved_target.query.zigTriple(b.allocator) catch @panic("OOM"); | |
| 2249 | addOneModuleTest(b, step, test_target, &resolved_target, triple_txt, options); | |
| 2250 | return step; | |
| 2251 | } | |
| 2252 | ||
| 2253 | for_targets: for (&test_targets) |*test_target| { | |
| 2245 | 2254 | if (test_target.skip_modules.len > 0) { |
| 2246 | 2255 | for (test_target.skip_modules) |skip_mod| { |
| 2247 | 2256 | if (std.mem.eql(u8, options.name, skip_mod)) continue :for_targets; |
| ... | ... | @@ -2306,167 +2315,180 @@ pub fn addModuleTests(b: *std.Build, options: ModuleTestOptions) *Step { |
| 2306 | 2315 | } else false; |
| 2307 | 2316 | if (!want_this_mode) continue; |
| 2308 | 2317 | |
| 2309 | const libc_suffix = if (test_target.link_libc == true) "-libc" else ""; | |
| 2310 | const model_txt = target.cpu.model.name; | |
| 2318 | addOneModuleTest(b, step, test_target, &resolved_target, triple_txt, options); | |
| 2319 | } | |
| 2320 | return step; | |
| 2321 | } | |
| 2311 | 2322 | |
| 2312 | // wasm32-wasi builds need more RAM, idk why | |
| 2313 | const max_rss = if (target.os.tag == .wasi) | |
| 2314 | options.max_rss * 2 | |
| 2315 | else | |
| 2316 | options.max_rss; | |
| 2323 | fn addOneModuleTest( | |
| 2324 | b: *std.Build, | |
| 2325 | step: *Step, | |
| 2326 | test_target: *const TestTarget, | |
| 2327 | resolved_target: *const std.Build.ResolvedTarget, | |
| 2328 | triple_txt: []const u8, | |
| 2329 | options: ModuleTestOptions, | |
| 2330 | ) void { | |
| 2331 | const target = &resolved_target.result; | |
| 2332 | const libc_suffix = if (test_target.link_libc == true) "-libc" else ""; | |
| 2333 | const model_txt = target.cpu.model.name; | |
| 2334 | ||
| 2335 | // wasm32-wasi builds need more RAM, idk why | |
| 2336 | const max_rss = if (target.os.tag == .wasi) | |
| 2337 | options.max_rss * 2 | |
| 2338 | else | |
| 2339 | options.max_rss; | |
| 2340 | ||
| 2341 | const these_tests = b.addTest(.{ | |
| 2342 | .root_module = b.createModule(.{ | |
| 2343 | .root_source_file = b.path(options.root_src), | |
| 2344 | .optimize = test_target.optimize_mode, | |
| 2345 | .target = resolved_target.*, | |
| 2346 | .link_libc = test_target.link_libc, | |
| 2347 | .pic = test_target.pic, | |
| 2348 | .strip = test_target.strip, | |
| 2349 | .single_threaded = test_target.single_threaded, | |
| 2350 | }), | |
| 2351 | .max_rss = max_rss, | |
| 2352 | .filters = options.test_filters, | |
| 2353 | .use_llvm = test_target.use_llvm, | |
| 2354 | .use_lld = test_target.use_lld, | |
| 2355 | .zig_lib_dir = b.path("lib"), | |
| 2356 | }); | |
| 2357 | these_tests.linkage = test_target.linkage; | |
| 2358 | if (options.no_builtin) these_tests.root_module.no_builtin = false; | |
| 2359 | if (options.build_options) |build_options| { | |
| 2360 | these_tests.root_module.addOptions("build_options", build_options); | |
| 2361 | } | |
| 2362 | const single_threaded_suffix = if (test_target.single_threaded == true) "-single" else ""; | |
| 2363 | const backend_suffix = if (test_target.use_llvm == true) | |
| 2364 | "-llvm" | |
| 2365 | else if (target.ofmt == std.Target.ObjectFormat.c) | |
| 2366 | "-cbe" | |
| 2367 | else if (test_target.use_llvm == false) | |
| 2368 | "-selfhosted" | |
| 2369 | else | |
| 2370 | ""; | |
| 2371 | const use_lld = if (test_target.use_lld == false) "-no-lld" else ""; | |
| 2372 | const linkage_name = if (test_target.linkage) |linkage| switch (linkage) { | |
| 2373 | inline else => |t| "-" ++ @tagName(t), | |
| 2374 | } else ""; | |
| 2375 | const use_pic = if (test_target.pic == true) "-pic" else ""; | |
| 2376 | ||
| 2377 | for (options.include_paths) |include_path| these_tests.root_module.addIncludePath(b.path(include_path)); | |
| 2378 | ||
| 2379 | const qualified_name = b.fmt("{s}-{s}-{s}-{t}{s}{s}{s}{s}{s}{s}", .{ | |
| 2380 | options.name, | |
| 2381 | triple_txt, | |
| 2382 | model_txt, | |
| 2383 | test_target.optimize_mode, | |
| 2384 | libc_suffix, | |
| 2385 | single_threaded_suffix, | |
| 2386 | backend_suffix, | |
| 2387 | use_lld, | |
| 2388 | linkage_name, | |
| 2389 | use_pic, | |
| 2390 | }); | |
| 2317 | 2391 | |
| 2318 | const these_tests = b.addTest(.{ | |
| 2319 | .root_module = b.createModule(.{ | |
| 2320 | .root_source_file = b.path(options.root_src), | |
| 2321 | .optimize = test_target.optimize_mode, | |
| 2322 | .target = resolved_target, | |
| 2323 | .link_libc = test_target.link_libc, | |
| 2324 | .pic = test_target.pic, | |
| 2325 | .strip = test_target.strip, | |
| 2326 | .single_threaded = test_target.single_threaded, | |
| 2327 | }), | |
| 2328 | .max_rss = max_rss, | |
| 2329 | .filters = options.test_filters, | |
| 2330 | .use_llvm = test_target.use_llvm, | |
| 2331 | .use_lld = test_target.use_lld, | |
| 2332 | .zig_lib_dir = b.path("lib"), | |
| 2392 | if (target.ofmt == std.Target.ObjectFormat.c) { | |
| 2393 | var altered_query = test_target.target; | |
| 2394 | altered_query.ofmt = null; | |
| 2395 | ||
| 2396 | const compile_c = b.createModule(.{ | |
| 2397 | .root_source_file = null, | |
| 2398 | .link_libc = test_target.link_libc, | |
| 2399 | .target = b.resolveTargetQuery(altered_query), | |
| 2333 | 2400 | }); |
| 2334 | these_tests.linkage = test_target.linkage; | |
| 2335 | if (options.no_builtin) these_tests.root_module.no_builtin = false; | |
| 2336 | if (options.build_options) |build_options| { | |
| 2337 | these_tests.root_module.addOptions("build_options", build_options); | |
| 2338 | } | |
| 2339 | const single_threaded_suffix = if (test_target.single_threaded == true) "-single" else ""; | |
| 2340 | const backend_suffix = if (test_target.use_llvm == true) | |
| 2341 | "-llvm" | |
| 2342 | else if (target.ofmt == std.Target.ObjectFormat.c) | |
| 2343 | "-cbe" | |
| 2344 | else if (test_target.use_llvm == false) | |
| 2345 | "-selfhosted" | |
| 2346 | else | |
| 2347 | ""; | |
| 2348 | const use_lld = if (test_target.use_lld == false) "-no-lld" else ""; | |
| 2349 | const linkage_name = if (test_target.linkage) |linkage| switch (linkage) { | |
| 2350 | inline else => |t| "-" ++ @tagName(t), | |
| 2351 | } else ""; | |
| 2352 | const use_pic = if (test_target.pic == true) "-pic" else ""; | |
| 2353 | ||
| 2354 | for (options.include_paths) |include_path| these_tests.root_module.addIncludePath(b.path(include_path)); | |
| 2355 | ||
| 2356 | const qualified_name = b.fmt("{s}-{s}-{s}-{s}{s}{s}{s}{s}{s}{s}", .{ | |
| 2357 | options.name, | |
| 2358 | triple_txt, | |
| 2359 | model_txt, | |
| 2360 | @tagName(test_target.optimize_mode), | |
| 2361 | libc_suffix, | |
| 2362 | single_threaded_suffix, | |
| 2363 | backend_suffix, | |
| 2364 | use_lld, | |
| 2365 | linkage_name, | |
| 2366 | use_pic, | |
| 2401 | const compile_c_exe = b.addExecutable(.{ | |
| 2402 | .name = qualified_name, | |
| 2403 | .root_module = compile_c, | |
| 2404 | .zig_lib_dir = b.path("lib"), | |
| 2367 | 2405 | }); |
| 2368 | 2406 | |
| 2369 | if (target.ofmt == std.Target.ObjectFormat.c) { | |
| 2370 | var altered_query = test_target.target; | |
| 2371 | altered_query.ofmt = null; | |
| 2372 | ||
| 2373 | const compile_c = b.createModule(.{ | |
| 2374 | .root_source_file = null, | |
| 2375 | .link_libc = test_target.link_libc, | |
| 2376 | .target = b.resolveTargetQuery(altered_query), | |
| 2377 | }); | |
| 2378 | const compile_c_exe = b.addExecutable(.{ | |
| 2379 | .name = qualified_name, | |
| 2380 | .root_module = compile_c, | |
| 2381 | .zig_lib_dir = b.path("lib"), | |
| 2382 | }); | |
| 2383 | ||
| 2384 | compile_c.addCSourceFile(.{ | |
| 2385 | .file = these_tests.getEmittedBin(), | |
| 2386 | .flags = &.{ | |
| 2387 | // Tracking issue for making the C backend generate C89 compatible code: | |
| 2388 | // https://github.com/ziglang/zig/issues/19468 | |
| 2389 | "-std=c99", | |
| 2390 | "-Werror", | |
| 2391 | ||
| 2392 | "-Wall", | |
| 2393 | "-Wembedded-directive", | |
| 2394 | "-Wempty-translation-unit", | |
| 2395 | "-Wextra", | |
| 2396 | "-Wgnu", | |
| 2397 | "-Winvalid-utf8", | |
| 2398 | "-Wkeyword-macro", | |
| 2399 | "-Woverlength-strings", | |
| 2400 | ||
| 2401 | // Tracking issue for making the C backend generate code | |
| 2402 | // that does not trigger warnings: | |
| 2403 | // https://github.com/ziglang/zig/issues/19467 | |
| 2404 | ||
| 2405 | // spotted everywhere | |
| 2406 | "-Wno-builtin-requires-header", | |
| 2407 | ||
| 2408 | // spotted on linux | |
| 2409 | "-Wno-braced-scalar-init", | |
| 2410 | "-Wno-excess-initializers", | |
| 2411 | "-Wno-incompatible-pointer-types-discards-qualifiers", | |
| 2412 | "-Wno-unused", | |
| 2413 | "-Wno-unused-parameter", | |
| 2414 | ||
| 2415 | // spotted on darwin | |
| 2416 | "-Wno-incompatible-pointer-types", | |
| 2417 | ||
| 2418 | // https://github.com/llvm/llvm-project/issues/153314 | |
| 2419 | "-Wno-unterminated-string-initialization", | |
| 2420 | ||
| 2421 | // In both Zig and C it is legal to return a pointer to a | |
| 2422 | // local. The C backend lowers such thing directly, so the | |
| 2423 | // corresponding warning in C must be disabled. | |
| 2424 | "-Wno-return-stack-address", | |
| 2425 | }, | |
| 2426 | }); | |
| 2427 | compile_c.addIncludePath(b.path("lib")); // for zig.h | |
| 2428 | if (target.os.tag == .windows) { | |
| 2429 | if (true) { | |
| 2430 | // Unfortunately this requires about 8G of RAM for clang to compile | |
| 2431 | // and our Windows CI runners do not have this much. | |
| 2432 | step.dependOn(&these_tests.step); | |
| 2433 | continue; | |
| 2434 | } | |
| 2407 | compile_c.addCSourceFile(.{ | |
| 2408 | .file = these_tests.getEmittedBin(), | |
| 2409 | .flags = &.{ | |
| 2410 | // Tracking issue for making the C backend generate C89 compatible code: | |
| 2411 | // https://github.com/ziglang/zig/issues/19468 | |
| 2412 | "-std=c99", | |
| 2413 | "-Werror", | |
| 2414 | ||
| 2415 | "-Wall", | |
| 2416 | "-Wembedded-directive", | |
| 2417 | "-Wempty-translation-unit", | |
| 2418 | "-Wextra", | |
| 2419 | "-Wgnu", | |
| 2420 | "-Winvalid-utf8", | |
| 2421 | "-Wkeyword-macro", | |
| 2422 | "-Woverlength-strings", | |
| 2423 | ||
| 2424 | // Tracking issue for making the C backend generate code | |
| 2425 | // that does not trigger warnings: | |
| 2426 | // https://github.com/ziglang/zig/issues/19467 | |
| 2427 | ||
| 2428 | // spotted everywhere | |
| 2429 | "-Wno-builtin-requires-header", | |
| 2430 | ||
| 2431 | // spotted on linux | |
| 2432 | "-Wno-braced-scalar-init", | |
| 2433 | "-Wno-excess-initializers", | |
| 2434 | "-Wno-incompatible-pointer-types-discards-qualifiers", | |
| 2435 | "-Wno-unused", | |
| 2436 | "-Wno-unused-parameter", | |
| 2437 | ||
| 2438 | // spotted on darwin | |
| 2439 | "-Wno-incompatible-pointer-types", | |
| 2440 | ||
| 2441 | // https://github.com/llvm/llvm-project/issues/153314 | |
| 2442 | "-Wno-unterminated-string-initialization", | |
| 2443 | ||
| 2444 | // In both Zig and C it is legal to return a pointer to a | |
| 2445 | // local. The C backend lowers such thing directly, so the | |
| 2446 | // corresponding warning in C must be disabled. | |
| 2447 | "-Wno-return-stack-address", | |
| 2448 | }, | |
| 2449 | }); | |
| 2450 | compile_c.addIncludePath(b.path("lib")); // for zig.h | |
| 2451 | if (target.os.tag == .windows) { | |
| 2452 | if (true) { | |
| 2453 | // Unfortunately this requires about 8G of RAM for clang to compile | |
| 2454 | // and our Windows CI runners do not have this much. | |
| 2455 | // TODO This is not an appropriate way to work around this problem. | |
| 2456 | step.dependOn(&these_tests.step); | |
| 2457 | return; | |
| 2458 | } | |
| 2459 | if (test_target.link_libc == false) { | |
| 2460 | compile_c_exe.subsystem = .Console; | |
| 2461 | compile_c.linkSystemLibrary("kernel32", .{}); | |
| 2462 | compile_c.linkSystemLibrary("ntdll", .{}); | |
| 2463 | } | |
| 2464 | if (mem.eql(u8, options.name, "std")) { | |
| 2435 | 2465 | if (test_target.link_libc == false) { |
| 2436 | compile_c_exe.subsystem = .Console; | |
| 2437 | compile_c.linkSystemLibrary("kernel32", .{}); | |
| 2438 | compile_c.linkSystemLibrary("ntdll", .{}); | |
| 2439 | } | |
| 2440 | if (mem.eql(u8, options.name, "std")) { | |
| 2441 | if (test_target.link_libc == false) { | |
| 2442 | compile_c.linkSystemLibrary("shell32", .{}); | |
| 2443 | compile_c.linkSystemLibrary("advapi32", .{}); | |
| 2444 | } | |
| 2445 | compile_c.linkSystemLibrary("crypt32", .{}); | |
| 2446 | compile_c.linkSystemLibrary("ws2_32", .{}); | |
| 2447 | compile_c.linkSystemLibrary("ole32", .{}); | |
| 2466 | compile_c.linkSystemLibrary("shell32", .{}); | |
| 2467 | compile_c.linkSystemLibrary("advapi32", .{}); | |
| 2448 | 2468 | } |
| 2469 | compile_c.linkSystemLibrary("crypt32", .{}); | |
| 2470 | compile_c.linkSystemLibrary("ws2_32", .{}); | |
| 2471 | compile_c.linkSystemLibrary("ole32", .{}); | |
| 2449 | 2472 | } |
| 2473 | } | |
| 2450 | 2474 | |
| 2451 | const run = b.addRunArtifact(compile_c_exe); | |
| 2452 | run.skip_foreign_checks = true; | |
| 2453 | run.enableTestRunnerMode(); | |
| 2454 | run.setName(b.fmt("run test {s}", .{qualified_name})); | |
| 2475 | const run = b.addRunArtifact(compile_c_exe); | |
| 2476 | run.skip_foreign_checks = true; | |
| 2477 | run.enableTestRunnerMode(); | |
| 2478 | run.setName(b.fmt("run test {s}", .{qualified_name})); | |
| 2455 | 2479 | |
| 2456 | step.dependOn(&run.step); | |
| 2457 | } else if (target.cpu.arch.isSpirV()) { | |
| 2458 | // Don't run spirv binaries | |
| 2459 | _ = these_tests.getEmittedBin(); | |
| 2460 | step.dependOn(&these_tests.step); | |
| 2461 | } else { | |
| 2462 | const run = b.addRunArtifact(these_tests); | |
| 2463 | run.skip_foreign_checks = true; | |
| 2464 | run.setName(b.fmt("run test {s}", .{qualified_name})); | |
| 2480 | step.dependOn(&run.step); | |
| 2481 | } else if (target.cpu.arch.isSpirV()) { | |
| 2482 | // Don't run spirv binaries | |
| 2483 | _ = these_tests.getEmittedBin(); | |
| 2484 | step.dependOn(&these_tests.step); | |
| 2485 | } else { | |
| 2486 | const run = b.addRunArtifact(these_tests); | |
| 2487 | run.skip_foreign_checks = true; | |
| 2488 | run.setName(b.fmt("run test {s}", .{qualified_name})); | |
| 2465 | 2489 | |
| 2466 | step.dependOn(&run.step); | |
| 2467 | } | |
| 2490 | step.dependOn(&run.step); | |
| 2468 | 2491 | } |
| 2469 | return step; | |
| 2470 | 2492 | } |
| 2471 | 2493 | |
| 2472 | 2494 | pub fn wouldUseLlvm(use_llvm: ?bool, query: std.Target.Query, optimize_mode: OptimizeMode) bool { |