authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-05-04 20:01:31-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-01 16:35:28-07:00
log2abbcb6d2a199c9d535a950db6098b8224d510aa
treeadf4002d01fa728cc9b53f5a48d455f9ceff9d96
parent6fe9c8f03626651b7cafe01ffedac17b33588b29

std.http: fix chunked transfer flushing and end-of-stream handling

4 files changed, 119 insertions(+), 53 deletions(-)

lib/std/http.zig+81-39
......@@ -331,9 +331,12 @@ pub const Reader = struct {
331331 /// making invalid API usage cause assertion failures rather than HTTP
332332 /// protocol violations.
333333 state: State,
334 /// Number of bytes of HTTP trailers. These are at the end of a
335 /// transfer-encoding: chunked message.
336 trailers_len: usize = 0,
334 /// HTTP trailer bytes. These are at the end of a transfer-encoding:
335 /// chunked message. This data is available only after calling one of the
336 /// "end" functions and points to data inside the buffer of `in`, and is
337 /// therefore invalidated on the next call to `receiveHead`, or any other
338 /// read from `in`.
339 trailers: []const u8 = &.{},
337340 body_err: ?BodyError = null,
338341 /// Stolen from `in`.
339342 head_buffer: []u8 = &.{},
......@@ -344,7 +347,6 @@ pub const Reader = struct {
344347 head = 0,
345348 n = 1,
346349 rn = 2,
347 done = std.math.maxInt(u64),
348350 _,
349351
350352 pub fn init(integer: u64) RemainingChunkLen {
......@@ -371,6 +373,7 @@ pub const Reader = struct {
371373
372374 pub const BodyError = error{
373375 HttpChunkInvalid,
376 HttpChunkTruncated,
374377 HttpHeadersOversize,
375378 };
376379
......@@ -393,6 +396,7 @@ pub const Reader = struct {
393396 /// Buffers the entire head into `head_buffer`, invalidating the previous
394397 /// `head_buffer`, if any.
395398 pub fn receiveHead(reader: *Reader) HeadError!void {
399 reader.trailers = &.{};
396400 const in = reader.in;
397401 in.restitute(reader.head_buffer.len);
398402 in.rebase();
......@@ -544,7 +548,11 @@ pub const Reader = struct {
544548 limit: std.io.Reader.Limit,
545549 ) std.io.Reader.RwError!usize {
546550 const reader: *Reader = @alignCast(@ptrCast(ctx));
547 const chunk_len_ptr = &reader.state.body_remaining_chunk_len;
551 const chunk_len_ptr = switch (reader.state) {
552 .ready => return error.EndOfStream,
553 .body_remaining_chunk_len => |*x| x,
554 else => unreachable,
555 };
548556 const in = reader.in;
549557 len: switch (chunk_len_ptr.*) {
550558 .head => {
......@@ -557,7 +565,7 @@ pub const Reader = struct {
557565 in.toss(i);
558566 },
559567 else => {
560 try in.fill(max_chunk_header_len);
568 try endless(reader, in.fill(max_chunk_header_len));
561569 const next_i = cp.feed(in.bufferContents()[i..]);
562570 if (cp.state != .data) return reader.failBody(error.HttpChunkInvalid);
563571 const header_len = i + next_i;
......@@ -566,7 +574,7 @@ pub const Reader = struct {
566574 },
567575 }
568576 if (cp.chunk_len == 0) return parseTrailers(reader, 0);
569 const n = try in.read(bw, limit.min(.limited(cp.chunk_len)));
577 const n = try endless(reader, in.read(bw, limit.min(.limited(cp.chunk_len))));
570578 chunk_len_ptr.* = .init(cp.chunk_len + 2 - n);
571579 return n;
572580 },
......@@ -576,27 +584,31 @@ pub const Reader = struct {
576584 continue :len .head;
577585 },
578586 .rn => {
579 const rn = try in.peekArray(2);
587 const rn = try endless(reader, in.peekArray(2));
580588 if (rn[0] != '\r' or rn[1] != '\n') return reader.failBody(error.HttpChunkInvalid);
581589 in.toss(2);
582590 continue :len .head;
583591 },
584592 else => |remaining_chunk_len| {
585 const n = try in.read(bw, limit.min(.limited(@intFromEnum(remaining_chunk_len) - 2)));
593 const n = try endless(reader, in.read(bw, limit.min(.limited(@intFromEnum(remaining_chunk_len) - 2))));
586594 chunk_len_ptr.* = .init(@intFromEnum(remaining_chunk_len) - n);
587595 return n;
588596 },
589 .done => return error.EndOfStream,
590597 }
591598 }
592599
593600 fn chunkedReadVec(ctx: ?*anyopaque, data: []const []u8) std.io.Reader.Error!usize {
594601 const reader: *Reader = @alignCast(@ptrCast(ctx));
595 const chunk_len_ptr = &reader.state.body_remaining_chunk_len;
602 const chunk_len_ptr = switch (reader.state) {
603 .ready => return error.EndOfStream,
604 .body_remaining_chunk_len => |*x| x,
605 else => unreachable,
606 };
596607 const in = reader.in;
597608 var already_requested_more = false;
598609 var amt_read: usize = 0;
599610 data: for (data) |d| {
611 var d_i: usize = 0;
600612 len: switch (chunk_len_ptr.*) {
601613 .head => {
602614 var cp: ChunkParser = .init;
......@@ -609,7 +621,7 @@ pub const Reader = struct {
609621 return amt_read;
610622 }
611623 already_requested_more = true;
612 try in.fill(max_chunk_header_len);
624 try endless(reader, in.fill(max_chunk_header_len));
613625 const next_i = cp.feed(in.bufferContents()[i..]);
614626 if (cp.state != .data) return reader.failBody(error.HttpChunkInvalid);
615627 const header_len = i + next_i;
......@@ -624,23 +636,24 @@ pub const Reader = struct {
624636 },
625637 .n => {
626638 if (in.bufferContents().len < 1) already_requested_more = true;
627 if ((try in.takeByte()) != '\n') return reader.failBody(error.HttpChunkInvalid);
639 if ((try endless(reader, in.takeByte())) != '\n') return reader.failBody(error.HttpChunkInvalid);
628640 continue :len .head;
629641 },
630642 .rn => {
631643 if (in.bufferContents().len < 2) already_requested_more = true;
632 const rn = try in.takeArray(2);
644 const rn = try endless(reader, in.takeArray(2));
633645 if (rn[0] != '\r' or rn[1] != '\n') return reader.failBody(error.HttpChunkInvalid);
634646 continue :len .head;
635647 },
636648 else => |remaining_chunk_len| {
637649 const available_buffer = in.bufferContents();
638 const copy_len = @min(available_buffer.len, d.len, remaining_chunk_len.int() - 2);
639 @memcpy(d[0..copy_len], available_buffer[0..copy_len]);
650 const copy_len = @min(available_buffer.len, d.len - d_i, remaining_chunk_len.int() - 2);
651 @memcpy(d[d_i..][0..copy_len], available_buffer[0..copy_len]);
652 d_i += copy_len;
640653 amt_read += copy_len;
641654 in.toss(copy_len);
642655 const next_chunk_len: RemainingChunkLen = .init(remaining_chunk_len.int() - copy_len);
643 if (copy_len == d.len) {
656 if (d.len - d_i == 0) {
644657 chunk_len_ptr.* = next_chunk_len;
645658 continue :data;
646659 }
......@@ -649,10 +662,9 @@ pub const Reader = struct {
649662 return amt_read;
650663 }
651664 already_requested_more = true;
652 try in.fill(3);
665 try endless(reader, in.fillMore());
653666 continue :len next_chunk_len;
654667 },
655 .done => return error.EndOfStream,
656668 }
657669 }
658670 return amt_read;
......@@ -660,7 +672,11 @@ pub const Reader = struct {
660672
661673 fn chunkedDiscard(ctx: ?*anyopaque, limit: std.io.Reader.Limit) std.io.Reader.Error!usize {
662674 const reader: *Reader = @alignCast(@ptrCast(ctx));
663 const chunk_len_ptr = &reader.state.body_remaining_chunk_len;
675 const chunk_len_ptr = switch (reader.state) {
676 .ready => return error.EndOfStream,
677 .body_remaining_chunk_len => |*x| x,
678 else => unreachable,
679 };
664680 const in = reader.in;
665681 len: switch (chunk_len_ptr.*) {
666682 .head => {
......@@ -673,7 +689,7 @@ pub const Reader = struct {
673689 in.toss(i);
674690 },
675691 else => {
676 try in.fill(max_chunk_header_len);
692 try endless(reader, in.fill(max_chunk_header_len));
677693 const next_i = cp.feed(in.bufferContents()[i..]);
678694 if (cp.state != .data) return reader.failBody(error.HttpChunkInvalid);
679695 const header_len = i + next_i;
......@@ -682,27 +698,26 @@ pub const Reader = struct {
682698 },
683699 }
684700 if (cp.chunk_len == 0) return parseTrailers(reader, 0);
685 const n = try in.discard(limit.min(.limited(cp.chunk_len)));
701 const n = try endless(reader, in.discard(limit.min(.limited(cp.chunk_len))));
686702 chunk_len_ptr.* = .init(cp.chunk_len + 2 - n);
687703 return n;
688704 },
689705 .n => {
690 if ((try in.peekByte()) != '\n') return reader.failBody(error.HttpChunkInvalid);
706 if ((try endless(reader, in.peekByte())) != '\n') return reader.failBody(error.HttpChunkInvalid);
691707 in.toss(1);
692708 continue :len .head;
693709 },
694710 .rn => {
695 const rn = try in.peekArray(2);
711 const rn = try endless(reader, in.peekArray(2));
696712 if (rn[0] != '\r' or rn[1] != '\n') return reader.failBody(error.HttpChunkInvalid);
697713 in.toss(2);
698714 continue :len .head;
699715 },
700716 else => |remaining_chunk_len| {
701 const n = try in.discard(limit.min(.limited(remaining_chunk_len.int() - 2)));
717 const n = try endless(reader, in.discard(limit.min(.limited(remaining_chunk_len.int() - 2))));
702718 chunk_len_ptr.* = .init(remaining_chunk_len.int() - n);
703719 return n;
704720 },
705 .done => return error.EndOfStream,
706721 }
707722 }
708723
......@@ -717,9 +732,8 @@ pub const Reader = struct {
717732 try in.fill(trailers_len + 1);
718733 trailers_len += hp.feed(in.bufferContents()[trailers_len..]);
719734 if (hp.state == .finished) {
720 reader.state.body_remaining_chunk_len = .done;
721735 reader.state = .ready;
722 reader.trailers_len = trailers_len;
736 reader.trailers = in.bufferContents()[0..trailers_len];
723737 return amt_read;
724738 }
725739 }
......@@ -729,6 +743,13 @@ pub const Reader = struct {
729743 r.body_err = err;
730744 return error.ReadFailed;
731745 }
746
747 fn endless(r: *Reader, x: anytype) @TypeOf(x) {
748 return x catch |err| switch (err) {
749 error.EndOfStream => return failBody(r, error.HttpChunkTruncated),
750 else => return err,
751 };
752 }
732753};
733754
734755pub const Decompressor = struct {
......@@ -823,18 +844,23 @@ pub const BodyWriter = struct {
823844 };
824845
825846 /// Sends all buffered data across `BodyWriter.http_protocol_output`.
826 ///
827 /// Some buffered data will remain if transfer-encoding is chunked and the
828 /// BodyWriter is mid-chunk.
829847 pub fn flush(w: *BodyWriter) WriteError!void {
848 const out = w.http_protocol_output;
830849 switch (w.state) {
831 .end, .none, .content_length => return w.http_protocol_output.flush(),
850 .end, .none, .content_length => return out.flush(),
832851 .chunked => |*chunked| switch (chunked.*) {
833 .offset => |*offset| {
834 try w.http_protocol_output.flushLimit(.limited(offset.*));
835 offset.* = 0;
852 .offset => |offset| {
853 const chunk_len = out.end - offset - chunk_header_template.len;
854 if (chunk_len > 0) {
855 writeHex(out.buffer[offset..][0..chunk_len_digits], chunk_len);
856 chunked.* = .{ .chunk_len = 2 };
857 } else {
858 out.end = offset;
859 chunked.* = .{ .chunk_len = 0 };
860 }
861 try out.flush();
836862 },
837 .chunk_len => return w.http_protocol_output.flush(),
863 .chunk_len => return out.flush(),
838864 },
839865 }
840866 }
......@@ -875,7 +901,7 @@ pub const BodyWriter = struct {
875901 w.state = .end;
876902 },
877903 .none => {},
878 .chunked => return endChunked(w, .{}),
904 .chunked => return endChunkedUnflushed(w, .{}),
879905 }
880906 }
881907
......@@ -883,6 +909,21 @@ pub const BodyWriter = struct {
883909 trailers: []const Header = &.{},
884910 };
885911
912 /// Writes the end-of-stream message and any optional trailers, flushing
913 /// the underlying stream.
914 ///
915 /// Asserts that the BodyWriter is using transfer-encoding: chunked.
916 ///
917 /// Respects the value of `elide` to omit all data after the headers.
918 ///
919 /// See also:
920 /// * `endChunkedUnflushed`
921 /// * `end`
922 pub fn endChunked(w: *BodyWriter, options: EndChunkedOptions) WriteError!void {
923 try endChunkedUnflushed(w, options);
924 try w.http_protocol_output.flush();
925 }
926
886927 /// Writes the end-of-stream message and any optional trailers.
887928 ///
888929 /// Does not flush.
......@@ -892,9 +933,10 @@ pub const BodyWriter = struct {
892933 /// Respects the value of `elide` to omit all data after the headers.
893934 ///
894935 /// See also:
895 /// * `end`
936 /// * `endChunked`
896937 /// * `endUnflushed`
897 pub fn endChunked(w: *BodyWriter, options: EndChunkedOptions) WriteError!void {
938 /// * `end`
939 pub fn endChunkedUnflushed(w: *BodyWriter, options: EndChunkedOptions) WriteError!void {
898940 const chunked = &w.state.chunked;
899941 if (w.elide) {
900942 w.state = .end;
lib/std/http/Client.zig+11
......@@ -703,6 +703,16 @@ pub const Response = struct {
703703 pub fn bodyErr(response: *const Response) ?http.Reader.BodyError {
704704 return response.request.reader.body_err;
705705 }
706
707 pub fn iterateTrailers(response: *const Response) http.HeaderIterator {
708 const r = &response.request.reader;
709 assert(r.state == .ready);
710 return .{
711 .bytes = r.trailers,
712 .index = 0,
713 .is_trailer = true,
714 };
715 }
706716};
707717
708718pub const Request = struct {
......@@ -951,6 +961,7 @@ pub const Request = struct {
951961 HttpRedirectLocationInvalid,
952962 HttpContentEncodingUnsupported,
953963 HttpChunkInvalid,
964 HttpChunkTruncated,
954965 HttpHeadersOversize,
955966 UnsupportedUriScheme,
956967
lib/std/http/test.zig+4-2
......@@ -72,20 +72,22 @@ test "trailers" {
7272
7373 try expectEqualStrings("Hello, World!\n", body);
7474
75 var it = response.head.iterateHeaders();
7675 {
76 var it = response.head.iterateHeaders();
7777 const header = it.next().?;
7878 try expect(!it.is_trailer);
7979 try expectEqualStrings("transfer-encoding", header.name);
8080 try expectEqualStrings("chunked", header.value);
81 try expectEqual(null, it.next());
8182 }
8283 {
84 var it = response.iterateTrailers();
8385 const header = it.next().?;
8486 try expect(it.is_trailer);
8587 try expectEqualStrings("X-Checksum", header.name);
8688 try expectEqualStrings("aaaa", header.value);
89 try expectEqual(null, it.next());
8790 }
88 try expectEqual(null, it.next());
8991 }
9092
9193 // connection has been kept alive
lib/std/io/BufferedReader.zig+23-12
......@@ -357,37 +357,48 @@ pub fn discardRemaining(br: *BufferedReader) Reader.ShortError!usize {
357357///
358358/// See also:
359359/// * `peek`
360/// * `readSliceShort`
360361pub fn readSlice(br: *BufferedReader, buffer: []u8) Reader.Error!void {
362 const n = try readSliceShort(br, buffer);
363 if (n != buffer.len) return error.EndOfStream;
364}
365
366/// Fill `buffer` with the next `buffer.len` bytes from the stream, advancing
367/// the seek position.
368///
369/// Invalidates previously returned values from `peek`.
370///
371/// Returns the number of bytes read, which is less than `buffer.len` if and
372/// only if the stream reached the end.
373///
374/// See also:
375/// * `readSlice`
376pub fn readSliceShort(br: *BufferedReader, buffer: []u8) Reader.ShortError!usize {
361377 const in_buffer = br.buffer[br.seek..br.end];
362378 const copy_len = @min(buffer.len, in_buffer.len);
363379 @memcpy(buffer[0..copy_len], in_buffer[0..copy_len]);
364 if (copy_len == buffer.len) {
380 if (buffer.len - copy_len == 0) {
365381 br.seek += copy_len;
366 return;
382 return buffer.len;
367383 }
368384 var i: usize = copy_len;
369385 br.end = 0;
370386 br.seek = 0;
371387 while (true) {
372388 const remaining = buffer[i..];
373 const n = try br.unbuffered_reader.readVec(&.{ remaining, br.buffer });
389 const n = br.unbuffered_reader.readVec(&.{ remaining, br.buffer }) catch |err| switch (err) {
390 error.EndOfStream => return i,
391 error.ReadFailed => return error.ReadFailed,
392 };
374393 if (n < remaining.len) {
375394 i += n;
376395 continue;
377396 }
378397 br.end = n - remaining.len;
379 return;
398 return buffer.len;
380399 }
381400}
382401
383/// Returns the number of bytes read, which is less than `buffer.len` if and
384/// only if the stream reached the end.
385pub fn readSliceShort(br: *BufferedReader, buffer: []u8) Reader.ShortError!usize {
386 _ = br;
387 _ = buffer;
388 @panic("TODO");
389}
390
391402pub const ReadAllocError = Reader.Error || Allocator.Error;
392403
393404/// The function is inline to avoid the dead code in case `endian` is