authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-16 21:02:05-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-23 02:37:11-07:00
log743a0c966de933e3d0a271f942c2db525df5dbe8
tree4b5e037f933ca37963a68f68b4dd05a517d8dbca
parent0ddcb8341822cf9e3fd555c29a9e011b9a91d9fc

std.http.Client: remove bad decisions from fetch()

* "storage" is a better name than "strategy". * The most flexible memory-based storage API is appending to an ArrayList. * HTTP method should default to POST if there is a payload. * Avoid storing unnecessary data in the FetchResult * Avoid the need for a deinit() method in the FetchResult The decisions that this logic made about how to handle files is beyond repair: - fail to use sendfile() on a plain connection - redundant stat - does not handle arbitrary streams So, file-based response storage is no longer supported. Users should use the lower-level open() API which allows avoiding these pitfalls.

2 files changed, 50 insertions(+), 78 deletions(-)

lib/std/http/Client.zig+42-73
...@@ -700,7 +700,7 @@ pub const Request = struct {...@@ -700,7 +700,7 @@ pub const Request = struct {
700 pub const SendError = Connection.WriteError || error{ InvalidContentLength, UnsupportedTransferEncoding };700 pub const SendError = Connection.WriteError || error{ InvalidContentLength, UnsupportedTransferEncoding };
701701
702 pub const SendOptions = struct {702 pub const SendOptions = struct {
703 /// Specifies that the uri should be used as is. You guarantee that the uri is already escaped.703 /// Specifies that the uri is already escaped.
704 raw_uri: bool = false,704 raw_uri: bool = false,
705 };705 };
706706
...@@ -1562,12 +1562,16 @@ pub fn open(...@@ -1562,12 +1562,16 @@ pub fn open(
15621562
1563pub const FetchOptions = struct {1563pub const FetchOptions = struct {
1564 server_header_buffer: ?[]u8 = null,1564 server_header_buffer: ?[]u8 = null,
1565 response_strategy: ResponseStrategy = .{ .storage = .{ .dynamic = 16 * 1024 * 1024 } },
1566 redirect_behavior: ?Request.RedirectBehavior = null,1565 redirect_behavior: ?Request.RedirectBehavior = null,
15671566
1567 /// If the server sends a body, it will be appended to this ArrayList.
1568 /// `max_append_size` provides an upper limit for how much they can grow.
1569 response_storage: ResponseStorage = .ignore,
1570 max_append_size: ?usize = null,
1571
1568 location: Location,1572 location: Location,
1569 method: http.Method = .GET,1573 method: ?http.Method = null,
1570 payload: Payload = .none,1574 payload: ?[]const u8 = null,
1571 raw_uri: bool = false,1575 raw_uri: bool = false,
15721576
1573 /// Standard headers that have default, but overridable, behavior.1577 /// Standard headers that have default, but overridable, behavior.
...@@ -1586,111 +1590,76 @@ pub const FetchOptions = struct {...@@ -1586,111 +1590,76 @@ pub const FetchOptions = struct {
1586 uri: Uri,1590 uri: Uri,
1587 };1591 };
15881592
1589 pub const Payload = union(enum) {1593 pub const ResponseStorage = union(enum) {
1590 string: []const u8,1594 ignore,
1591 file: std.fs.File,1595 /// Only the existing capacity will be used.
1592 none,1596 static: *std.ArrayListUnmanaged(u8),
1593 };1597 dynamic: *std.ArrayList(u8),
1594
1595 pub const ResponseStrategy = union(enum) {
1596 storage: StorageStrategy,
1597 file: std.fs.File,
1598 none,
1599 };
1600
1601 pub const StorageStrategy = union(enum) {
1602 /// In this case, the client's Allocator will be used to store the
1603 /// entire HTTP header. This value is the maximum total size of
1604 /// HTTP headers allowed, otherwise
1605 /// error.HttpHeadersExceededSizeLimit is returned from read().
1606 dynamic: usize,
1607 /// This is used to store the entire HTTP header. If the HTTP
1608 /// header is too big to fit, `error.HttpHeadersExceededSizeLimit`
1609 /// is returned from read(). When this is used, `error.OutOfMemory`
1610 /// cannot be returned from `read()`.
1611 static: []u8,
1612 };1598 };
1613};1599};
16141600
1615pub const FetchResult = struct {1601pub const FetchResult = struct {
1616 status: http.Status,1602 status: http.Status,
1617 body: ?[]const u8 = null,
1618
1619 allocator: Allocator,
1620 options: FetchOptions,
1621
1622 pub fn deinit(res: *FetchResult) void {
1623 if (res.options.response_strategy == .storage and
1624 res.options.response_strategy.storage == .dynamic)
1625 {
1626 if (res.body) |body| res.allocator.free(body);
1627 }
1628 }
1629};1603};
16301604
1631/// Perform a one-shot HTTP request with the provided options.1605/// Perform a one-shot HTTP request with the provided options.
1632///1606///
1633/// This function is threadsafe.1607/// This function is threadsafe.
1634pub fn fetch(client: *Client, allocator: Allocator, options: FetchOptions) !FetchResult {1608pub fn fetch(client: *Client, options: FetchOptions) !FetchResult {
1635 const uri = switch (options.location) {1609 const uri = switch (options.location) {
1636 .url => |u| try Uri.parse(u),1610 .url => |u| try Uri.parse(u),
1637 .uri => |u| u,1611 .uri => |u| u,
1638 };1612 };
1639 var server_header_buffer: [16 * 1024]u8 = undefined;1613 var server_header_buffer: [16 * 1024]u8 = undefined;
16401614
1641 var req = try open(client, options.method, uri, .{1615 const method: http.Method = options.method orelse
1616 if (options.payload != null) .POST else .GET;
1617
1618 var req = try open(client, method, uri, .{
1642 .server_header_buffer = options.server_header_buffer orelse &server_header_buffer,1619 .server_header_buffer = options.server_header_buffer orelse &server_header_buffer,
1643 .redirect_behavior = options.redirect_behavior orelse1620 .redirect_behavior = options.redirect_behavior orelse
1644 if (options.payload == .none) @enumFromInt(3) else .unhandled,1621 if (options.payload == null) @enumFromInt(3) else .unhandled,
1645 .headers = options.headers,1622 .headers = options.headers,
1646 .extra_headers = options.extra_headers,1623 .extra_headers = options.extra_headers,
1647 .privileged_headers = options.privileged_headers,1624 .privileged_headers = options.privileged_headers,
1648 });1625 });
1649 defer req.deinit();1626 defer req.deinit();
16501627
1651 switch (options.payload) {1628 if (options.payload) |payload| req.transfer_encoding = .{ .content_length = payload.len };
1652 .string => |str| req.transfer_encoding = .{ .content_length = str.len },
1653 .file => |file| req.transfer_encoding = .{ .content_length = (try file.stat()).size },
1654 .none => {},
1655 }
16561629
1657 try req.send(.{ .raw_uri = options.raw_uri });1630 try req.send(.{ .raw_uri = options.raw_uri });
16581631
1659 switch (options.payload) {1632 if (options.payload) |payload| try req.writeAll(payload);
1660 .string => |str| try req.writeAll(str),
1661 .file => |file| {
1662 var fifo = std.fifo.LinearFifo(u8, .{ .Static = 8192 }).init();
1663 try fifo.pump(file.reader(), req.writer());
1664 },
1665 .none => {},
1666 }
16671633
1668 try req.finish();1634 try req.finish();
1669 try req.wait();1635 try req.wait();
16701636
1671 var res: FetchResult = .{1637 switch (options.response_storage) {
1672 .status = req.response.status,1638 .ignore => {
1673 .allocator = allocator,1639 // Take advantage of request internals to discard the response body
1674 .options = options,1640 // and make the connection available for another request.
1675 };1641 req.response.skip = true;
16761642 assert(try req.transferRead(&.{}) == 0); // No buffer is necessary when skipping.
1677 switch (options.response_strategy) {
1678 .storage => |storage| switch (storage) {
1679 .dynamic => |max| res.body = try req.reader().readAllAlloc(allocator, max),
1680 .static => |buf| res.body = buf[0..try req.reader().readAll(buf)],
1681 },1643 },
1682 .file => |file| {1644 .dynamic => |list| {
1683 var fifo = std.fifo.LinearFifo(u8, .{ .Static = 8192 }).init();1645 const max_append_size = options.max_append_size orelse 2 * 1024 * 1024;
1684 try fifo.pump(req.reader(), file.writer());1646 try req.reader().readAllArrayList(list, max_append_size);
1685 },1647 },
1686 .none => { // Take advantage of request internals to discard the response body and make the connection available for another request.1648 .static => |list| {
1687 req.response.skip = true;1649 const buf = b: {
16881650 const buf = list.unusedCapacitySlice();
1689 assert(try req.transferRead(&.{}) == 0); // we're skipping, no buffer is necessary1651 if (options.max_append_size) |len| {
1652 if (len < buf.len) break :b buf[0..len];
1653 }
1654 break :b buf;
1655 };
1656 list.items.len += try req.reader().readAll(buf);
1690 },1657 },
1691 }1658 }
16921659
1693 return res;1660 return .{
1661 .status = req.response.status,
1662 };
1694}1663}
16951664
1696test {1665test {
test/standalone/http.zig+8-5
...@@ -586,17 +586,20 @@ pub fn main() !void {...@@ -586,17 +586,20 @@ pub fn main() !void {
586 defer calloc.free(location);586 defer calloc.free(location);
587587
588 log.info("{s}", .{location});588 log.info("{s}", .{location});
589 var res = try client.fetch(calloc, .{589 var body = std.ArrayList(u8).init(calloc);
590 defer body.deinit();
591
592 const res = try client.fetch(.{
590 .location = .{ .url = location },593 .location = .{ .url = location },
591 .method = .POST,594 .method = .POST,
592 .payload = .{ .string = "Hello, World!\n" },595 .payload = "Hello, World!\n",
593 .extra_headers = &.{596 .extra_headers = &.{
594 .{ .name = "content-type", .value = "text/plain" },597 .{ .name = "content-type", .value = "text/plain" },
595 },598 },
599 .response_storage = .{ .dynamic = &body },
596 });600 });
597 defer res.deinit();601 try testing.expectEqual(.ok, res.status);
598602 try testing.expectEqualStrings("Hello, World!\n", body.items);
599 try testing.expectEqualStrings("Hello, World!\n", res.body.?);
600 }603 }
601604
602 { // expect: 100-continue605 { // expect: 100-continue