authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-08-07 16:11:59-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-08-07 19:54:39-07:00
log5ce8e9325b7aa15cbcc77221fc7075b6c46619cc
treea29cd6b99bc833f566d0f086202d99c78d80bfaa
parent5998a8cebe3973d70c258b2a1440c5c3252d3539

std.http.Client: fix fetching by adding a buffer

3 files changed, 27 insertions(+), 30 deletions(-)

lib/std/http.zig+5-4
......@@ -412,7 +412,7 @@ pub const Reader = struct {
412412 /// * `interfaceDecompressing`
413413 pub fn bodyReader(
414414 reader: *Reader,
415 buffer: []u8,
415 transfer_buffer: []u8,
416416 transfer_encoding: TransferEncoding,
417417 content_length: ?u64,
418418 ) *std.Io.Reader {
......@@ -421,7 +421,7 @@ pub const Reader = struct {
421421 .chunked => {
422422 reader.state = .{ .body_remaining_chunk_len = .head };
423423 reader.interface = .{
424 .buffer = buffer,
424 .buffer = transfer_buffer,
425425 .seek = 0,
426426 .end = 0,
427427 .vtable = &.{
......@@ -435,7 +435,7 @@ pub const Reader = struct {
435435 if (content_length) |len| {
436436 reader.state = .{ .body_remaining_content_length = len };
437437 reader.interface = .{
438 .buffer = buffer,
438 .buffer = transfer_buffer,
439439 .seek = 0,
440440 .end = 0,
441441 .vtable = &.{
......@@ -460,6 +460,7 @@ pub const Reader = struct {
460460 /// * `interface`
461461 pub fn bodyReaderDecompressing(
462462 reader: *Reader,
463 transfer_buffer: []u8,
463464 transfer_encoding: TransferEncoding,
464465 content_length: ?u64,
465466 content_encoding: ContentEncoding,
......@@ -488,7 +489,7 @@ pub const Reader = struct {
488489 .compress => unreachable,
489490 }
490491 }
491 const transfer_reader = bodyReader(reader, &.{}, transfer_encoding, content_length);
492 const transfer_reader = bodyReader(reader, transfer_buffer, transfer_encoding, content_length);
492493 return decompressor.init(transfer_reader, decompression_buffer, content_encoding);
493494 }
494495
lib/std/http/Client.zig+17-22
......@@ -13,8 +13,8 @@ const net = std.net;
1313const Uri = std.Uri;
1414const Allocator = mem.Allocator;
1515const assert = std.debug.assert;
16const Writer = std.io.Writer;
17const Reader = std.io.Reader;
16const Writer = std.Io.Writer;
17const Reader = std.Io.Reader;
1818
1919const Client = @This();
2020
......@@ -704,12 +704,12 @@ pub const Response = struct {
704704 ///
705705 /// See also:
706706 /// * `readerDecompressing`
707 pub fn reader(response: *Response, buffer: []u8) *Reader {
707 pub fn reader(response: *Response, transfer_buffer: []u8) *Reader {
708708 response.head.invalidateStrings();
709709 const req = response.request;
710710 if (!req.method.responseHasBody()) return .ending;
711711 const head = &response.head;
712 return req.reader.bodyReader(buffer, head.transfer_encoding, head.content_length);
712 return req.reader.bodyReader(transfer_buffer, head.transfer_encoding, head.content_length);
713713 }
714714
715715 /// If compressed body has been negotiated this will return decompressed bytes.
......@@ -723,12 +723,14 @@ pub const Response = struct {
723723 /// * `reader`
724724 pub fn readerDecompressing(
725725 response: *Response,
726 transfer_buffer: []u8,
726727 decompressor: *http.Decompressor,
727728 decompression_buffer: []u8,
728729 ) *Reader {
729730 response.head.invalidateStrings();
730731 const head = &response.head;
731732 return response.request.reader.bodyReaderDecompressing(
733 transfer_buffer,
732734 head.transfer_encoding,
733735 head.content_length,
734736 head.content_encoding,
......@@ -1322,7 +1324,7 @@ pub const basic_authorization = struct {
13221324 const user: Uri.Component = uri.user orelse .empty;
13231325 const password: Uri.Component = uri.password orelse .empty;
13241326
1325 var dw: std.io.Writer.Discarding = .init(&.{});
1327 var dw: Writer.Discarding = .init(&.{});
13261328 user.formatUser(&dw.writer) catch unreachable; // discarding
13271329 const user_len = dw.count + dw.writer.end;
13281330
......@@ -1696,8 +1698,8 @@ pub const FetchOptions = struct {
16961698 /// `null` means it will be heap-allocated.
16971699 decompress_buffer: ?[]u8 = null,
16981700 redirect_behavior: ?Request.RedirectBehavior = null,
1699 /// If the server sends a body, it will be stored here.
1700 response_storage: ?ResponseStorage = null,
1701 /// If the server sends a body, it will be written here.
1702 response_writer: ?*Writer = null,
17011703
17021704 location: Location,
17031705 method: ?http.Method = null,
......@@ -1725,7 +1727,7 @@ pub const FetchOptions = struct {
17251727 list: *std.ArrayListUnmanaged(u8),
17261728 /// If null then only the existing capacity will be used.
17271729 allocator: ?Allocator = null,
1728 append_limit: std.io.Limit = .unlimited,
1730 append_limit: std.Io.Limit = .unlimited,
17291731 };
17301732};
17311733
......@@ -1778,7 +1780,7 @@ pub fn fetch(client: *Client, options: FetchOptions) FetchError!FetchResult {
17781780
17791781 var response = try req.receiveHead(redirect_buffer);
17801782
1781 const storage = options.response_storage orelse {
1783 const response_writer = options.response_writer orelse {
17821784 const reader = response.reader(&.{});
17831785 _ = reader.discardRemaining() catch |err| switch (err) {
17841786 error.ReadFailed => return response.bodyErr().?,
......@@ -1794,21 +1796,14 @@ pub fn fetch(client: *Client, options: FetchOptions) FetchError!FetchResult {
17941796 };
17951797 defer if (options.decompress_buffer == null) client.allocator.free(decompress_buffer);
17961798
1799 var transfer_buffer: [64]u8 = undefined;
17971800 var decompressor: http.Decompressor = undefined;
1798 const reader = response.readerDecompressing(&decompressor, decompress_buffer);
1799 const list = storage.list;
1801 const reader = response.readerDecompressing(&transfer_buffer, &decompressor, decompress_buffer);
18001802
1801 if (storage.allocator) |allocator| {
1802 reader.appendRemaining(allocator, null, list, storage.append_limit) catch |err| switch (err) {
1803 error.ReadFailed => return response.bodyErr().?,
1804 else => |e| return e,
1805 };
1806 } else {
1807 const buf = storage.append_limit.slice(list.unusedCapacitySlice());
1808 list.items.len += reader.readSliceShort(buf) catch |err| switch (err) {
1809 error.ReadFailed => return response.bodyErr().?,
1810 };
1811 }
1803 _ = reader.streamRemaining(response_writer) catch |err| switch (err) {
1804 error.ReadFailed => return response.bodyErr().?,
1805 else => |e| return e,
1806 };
18121807
18131808 return .{ .status = response.head.status };
18141809}
lib/std/http/test.zig+5-4
......@@ -1006,8 +1006,9 @@ fn echoTests(client: *http.Client, port: u16) !void {
10061006 const location = try std.fmt.allocPrint(gpa, "http://127.0.0.1:{d}/echo-content#fetch", .{port});
10071007 defer gpa.free(location);
10081008
1009 var body: std.ArrayListUnmanaged(u8) = .empty;
1010 defer body.deinit(gpa);
1009 var body: std.Io.Writer.Allocating = .init(gpa);
1010 defer body.deinit();
1011 try body.ensureUnusedCapacity(64);
10111012
10121013 const res = try client.fetch(.{
10131014 .location = .{ .url = location },
......@@ -1016,10 +1017,10 @@ fn echoTests(client: *http.Client, port: u16) !void {
10161017 .extra_headers = &.{
10171018 .{ .name = "content-type", .value = "text/plain" },
10181019 },
1019 .response_storage = .{ .allocator = gpa, .list = &body },
1020 .response_writer = &body.writer,
10201021 });
10211022 try expectEqual(.ok, res.status);
1022 try expectEqualStrings("Hello, World!\n", body.items);
1023 try expectEqualStrings("Hello, World!\n", body.getWritten());
10231024 }
10241025
10251026 { // expect: 100-continue