Commit 90bd4f226e2ba03634d31c73df06bf0a90fa0231 | zig

author	Andrew Kelley <andrew@ziglang.org> 2024-02-11 17:17:09-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2024-02-23 02:37:10-07:00
log	90bd4f226e2ba03634d31c73df06bf0a90fa0231
tree	0a5cc52fa31ac0e0d9f8f40bce9534c67c4bd89c
parent	f1cf300c8fa9842ec9c812310bdc9f3aeeb75359

std.http: remove the ability to heap-allocate headers

The buffer for HTTP headers is now always provided via a static buffer.
As a consequence, OutOfMemory is no longer a member of the read() error
set, and the API and implementation of Client and Server are simplified.

error.HttpHeadersExceededSizeLimit is renamed to
error.HttpHeadersOversize.

5 files changed, 210 insertions(+), 179 deletions(-)

lib
- std
  - http
    Client.zig
    Server.zig
    protocol.zig
src
- Package
  - Fetch.zig
  - Fetch
    git.zig

lib/std/http/Client.zig+78-54

...	...	@@ -20,9 +20,7 @@ const proto = @import("protocol.zig");
20	20
21	21	pub const disable_tls = std.options.http_disable_tls;
22	22
23		/// Allocator used for all allocations made by the client.
24		///
25		/// This allocator must be thread-safe.
	23	/// Used for all client allocations. Must be thread-safe.
26	24	allocator: Allocator,
27	25
28	26	ca_bundle: if (disable_tls) void else std.crypto.Certificate.Bundle = if (disable_tls) {} else .{},
...	...	@@ -35,10 +33,12 @@ next_https_rescan_certs: bool = true,
35	33	/// The pool of connections that can be reused (and currently in use).
36	34	connection_pool: ConnectionPool = .{},
37	35
38		/// This is the proxy that will handle http:// connections. It must not be modified when the client has any active connections.
	36	/// This is the proxy that will handle http:// connections. It must not be
	37	/// modified when the client has any active connections.
39	38	http_proxy: ?Proxy = null,
40	39
41		/// This is the proxy that will handle https:// connections. It must not be modified when the client has any active connections.
	40	/// This is the proxy that will handle https:// connections. It must not be
	41	/// modified when the client has any active connections.
42	42	https_proxy: ?Proxy = null,
43	43
44	44	/// A set of linked lists of connections that can be reused.
...	...	@@ -609,10 +609,6 @@ pub const Request = struct {
609	609	req.headers.deinit();
610	610	req.response.headers.deinit();
611	611
612		if (req.response.parser.header_bytes_owned) {
613		req.response.parser.header_bytes.deinit(req.client.allocator);
614		}
615
616	612	if (req.connection) \|connection\| {
617	613	if (!req.response.parser.done) {
618	614	// If the response wasn't fully read, then we need to close the connection.
...	...	@@ -810,27 +806,38 @@ pub const Request = struct {
810	806	return index;
811	807	}
812	808
813		pub const WaitError = RequestError \|\| SendError \|\| TransferReadError \|\| proto.HeadersParser.CheckCompleteHeadError \|\| Response.ParseError \|\| Uri.ParseError \|\| error{ TooManyHttpRedirects, RedirectRequiresResend, HttpRedirectMissingLocation, CompressionInitializationFailed, CompressionNotSupported };
	809	pub const WaitError = RequestError \|\| SendError \|\| TransferReadError \|\|
	810	proto.HeadersParser.CheckCompleteHeadError \|\| Response.ParseError \|\| Uri.ParseError \|\|
	811	error{ // TODO: file zig fmt issue for this bad indentation
	812	TooManyHttpRedirects,
	813	RedirectRequiresResend,
	814	HttpRedirectMissingLocation,
	815	CompressionInitializationFailed,
	816	CompressionNotSupported,
	817	};
814	818
815	819	/// Waits for a response from the server and parses any headers that are sent.
816	820	/// This function will block until the final response is received.
817	821	///
818		/// If `handle_redirects` is true and the request has no payload, then this function will automatically follow
819		/// redirects. If a request payload is present, then this function will error with error.RedirectRequiresResend.
	822	/// If `handle_redirects` is true and the request has no payload, then this
	823	/// function will automatically follow redirects. If a request payload is
	824	/// present, then this function will error with
	825	/// error.RedirectRequiresResend.
820	826	///
821		/// Must be called after `send` and, if any data was written to the request body, then also after `finish`.
	827	/// Must be called after `send` and, if any data was written to the request
	828	/// body, then also after `finish`.
822	829	pub fn wait(req: *Request) WaitError!void {
823	830	while (true) { // handle redirects
824	831	while (true) { // read headers
825	832	try req.connection.?.fill();
826	833
827		const nchecked = try req.response.parser.checkCompleteHead(req.client.allocator, req.connection.?.peek());
	834	const nchecked = try req.response.parser.checkCompleteHead(req.connection.?.peek());
828	835	req.connection.?.drop(@intCast(nchecked));
829	836
830	837	if (req.response.parser.state.isContent()) break;
831	838	}
832	839
833		try req.response.parse(req.response.parser.header_bytes.items, false);
	840	try req.response.parse(req.response.parser.get(), false);
834	841
835	842	if (req.response.status == .@"continue") {
836	843	req.response.parser.done = true; // we're done parsing the continue response, reset to prepare for the real response
...	...	@@ -891,7 +898,8 @@ pub const Request = struct {
891	898	if (req.response.status.class() == .redirect and req.handle_redirects) {
892	899	req.response.skip = true;
893	900
894		// skip the body of the redirect response, this will at least leave the connection in a known good state.
	901	// skip the body of the redirect response, this will at least
	902	// leave the connection in a known good state.
895	903	const empty = @as([*]u8, undefined)[0..0];
896	904	assert(try req.transferRead(empty) == 0); // we're skipping, no buffer is necessary
897	905
...	...	@@ -908,7 +916,10 @@ pub const Request = struct {
908	916	const resolved_url = try req.uri.resolve(new_url, false, arena);
909	917
910	918	// is the redirect location on the same domain, or a subdomain of the original request?
911		const is_same_domain_or_subdomain = std.ascii.endsWithIgnoreCase(resolved_url.host.?, req.uri.host.?) and (resolved_url.host.?.len == req.uri.host.?.len or resolved_url.host.?[resolved_url.host.?.len - req.uri.host.?.len - 1] == '.');
	919	const is_same_domain_or_subdomain =
	920	std.ascii.endsWithIgnoreCase(resolved_url.host.?, req.uri.host.?) and
	921	(resolved_url.host.?.len == req.uri.host.?.len or
	922	resolved_url.host.?[resolved_url.host.?.len - req.uri.host.?.len - 1] == '.');
912	923
913	924	if (resolved_url.host == null or !is_same_domain_or_subdomain or !std.ascii.eqlIgnoreCase(resolved_url.scheme, req.uri.scheme)) {
914	925	// we're redirecting to a different domain, strip privileged headers like cookies
...	...	@@ -957,7 +968,8 @@ pub const Request = struct {
957	968	}
958	969	}
959	970
960		pub const ReadError = TransferReadError \|\| proto.HeadersParser.CheckCompleteHeadError \|\| error{ DecompressionFailure, InvalidTrailers };
	971	pub const ReadError = TransferReadError \|\| proto.HeadersParser.CheckCompleteHeadError \|\|
	972	error{ DecompressionFailure, InvalidTrailers };
961	973
962	974	pub const Reader = std.io.Reader(*Request, ReadError, read);
963	975
...	...	@@ -980,14 +992,16 @@ pub const Request = struct {
980	992	while (!req.response.parser.state.isContent()) { // read trailing headers
981	993	try req.connection.?.fill();
982	994
983		const nchecked = try req.response.parser.checkCompleteHead(req.client.allocator, req.connection.?.peek());
	995	const nchecked = try req.response.parser.checkCompleteHead(req.connection.?.peek());
984	996	req.connection.?.drop(@intCast(nchecked));
985	997	}
986	998
987	999	if (has_trail) {
988		// The response headers before the trailers are already guaranteed to be valid, so they will always be parsed again and cannot return an error.
	1000	// The response headers before the trailers are already
	1001	// guaranteed to be valid, so they will always be parsed again
	1002	// and cannot return an error.
989	1003	// This will only fail for a malformed trailer.
990		req.response.parse(req.response.parser.header_bytes.items, true) catch return error.InvalidTrailers;
	1004	req.response.parse(req.response.parser.get(), true) catch return error.InvalidTrailers;
991	1005	}
992	1006	}
993	1007
...	...	@@ -1362,13 +1376,11 @@ pub fn connectTunnel(
1362	1376	.fragment = null,
1363	1377	};
1364	1378
1365		// we can use a small buffer here because a CONNECT response should be very small
1366	1379	var buffer: [8096]u8 = undefined;
1367
1368	1380	var req = client.open(.CONNECT, uri, proxy.headers, .{
1369	1381	.handle_redirects = false,
1370	1382	.connection = conn,
1371		.header_strategy = .{ .static = &buffer },
	1383	.server_header_buffer = &buffer,
1372	1384	}) catch \|err\| {
1373	1385	std.log.debug("err {}", .{err});
1374	1386	break :tunnel err;
...	...	@@ -1445,7 +1457,9 @@ pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connectio
1445	1457	return client.connectTcp(host, port, protocol);
1446	1458	}
1447	1459
1448		pub const RequestError = ConnectTcpError \|\| ConnectErrorPartial \|\| Request.SendError \|\| std.fmt.ParseIntError \|\| Connection.WriteError \|\| error{
	1460	pub const RequestError = ConnectTcpError \|\| ConnectErrorPartial \|\| Request.SendError \|\|
	1461	std.fmt.ParseIntError \|\| Connection.WriteError \|\|
	1462	error{ // TODO: file a zig fmt issue for this bad indentation
1449	1463	UnsupportedUrlScheme,
1450	1464	UriMissingHost,
1451	1465
...	...	@@ -1456,36 +1470,29 @@ pub const RequestError = ConnectTcpError \|\| ConnectErrorPartial \|\| Request.SendE
1456	1470	pub const RequestOptions = struct {
1457	1471	version: http.Version = .@"HTTP/1.1",
1458	1472
1459		/// Automatically ignore 100 Continue responses. This assumes you don't care, and will have sent the body before you
1460		/// wait for the response.
	1473	/// Automatically ignore 100 Continue responses. This assumes you don't
	1474	/// care, and will have sent the body before you wait for the response.
1461	1475	///
1462		/// If this is not the case AND you know the server will send a 100 Continue, set this to false and wait for a
1463		/// response before sending the body. If you wait AND the server does not send a 100 Continue before you finish the
1464		/// request, then the request will deadlock.
	1476	/// If this is not the case AND you know the server will send a 100
	1477	/// Continue, set this to false and wait for a response before sending the
	1478	/// body. If you wait AND the server does not send a 100 Continue before
	1479	/// you finish the request, then the request will deadlock.
1465	1480	handle_continue: bool = true,
1466	1481
1467		/// Automatically follow redirects. This will only follow redirects for repeatable requests (ie. with no payload or the server has acknowledged the payload)
	1482	/// Automatically follow redirects. This will only follow redirects for
	1483	/// repeatable requests (ie. with no payload or the server has acknowledged
	1484	/// the payload).
1468	1485	handle_redirects: bool = true,
1469	1486
1470	1487	/// How many redirects to follow before returning an error.
1471	1488	max_redirects: u32 = 3,
1472		header_strategy: StorageStrategy = .{ .dynamic = 16 * 1024 },
	1489	/// Externally-owned memory used to store the server's entire HTTP header.
	1490	/// `error.HttpHeadersOversize` is returned from read() when a
	1491	/// client sends too many bytes of HTTP headers.
	1492	server_header_buffer: []u8,
1473	1493
1474	1494	/// Must be an already acquired connection.
1475	1495	connection: ?*Connection = null,
1476
1477		pub const StorageStrategy = union(enum) {
1478		/// In this case, the client's Allocator will be used to store the
1479		/// entire HTTP header. This value is the maximum total size of
1480		/// HTTP headers allowed, otherwise
1481		/// error.HttpHeadersExceededSizeLimit is returned from read().
1482		dynamic: usize,
1483		/// This is used to store the entire HTTP header. If the HTTP
1484		/// header is too big to fit, `error.HttpHeadersExceededSizeLimit`
1485		/// is returned from read(). When this is used, `error.OutOfMemory`
1486		/// cannot be returned from `read()`.
1487		static: []u8,
1488		};
1489	1496	};
1490	1497
1491	1498	pub const protocol_map = std.ComptimeStringMap(Connection.Protocol, .{
...	...	@@ -1502,7 +1509,13 @@ pub const protocol_map = std.ComptimeStringMap(Connection.Protocol, .{
1502	1509	///
1503	1510	/// The caller is responsible for calling `deinit()` on the `Request`.
1504	1511	/// This function is threadsafe.
1505		pub fn open(client: *Client, method: http.Method, uri: Uri, headers: http.Headers, options: RequestOptions) RequestError!Request {
	1512	pub fn open(
	1513	client: *Client,
	1514	method: http.Method,
	1515	uri: Uri,
	1516	headers: http.Headers,
	1517	options: RequestOptions,
	1518	) RequestError!Request {
1506	1519	const protocol = protocol_map.get(uri.scheme) orelse return error.UnsupportedUrlScheme;
1507	1520
1508	1521	const port: u16 = uri.port orelse switch (protocol) {
...	...	@@ -1541,10 +1554,7 @@ pub fn open(client: *Client, method: http.Method, uri: Uri, headers: http.Header
1541	1554	.reason = undefined,
1542	1555	.version = undefined,
1543	1556	.headers = http.Headers{ .allocator = client.allocator, .owned = false },
1544		.parser = switch (options.header_strategy) {
1545		.dynamic => \|max\| proto.HeadersParser.initDynamic(max),
1546		.static => \|buf\| proto.HeadersParser.initStatic(buf),
1547		},
	1557	.parser = proto.HeadersParser.init(options.server_header_buffer),
1548	1558	},
1549	1559	.arena = undefined,
1550	1560	};
...	...	@@ -1568,17 +1578,30 @@ pub const FetchOptions = struct {
1568	1578	};
1569	1579
1570	1580	pub const ResponseStrategy = union(enum) {
1571		storage: RequestOptions.StorageStrategy,
	1581	storage: StorageStrategy,
1572	1582	file: std.fs.File,
1573	1583	none,
1574	1584	};
1575	1585
1576		header_strategy: RequestOptions.StorageStrategy = .{ .dynamic = 16 * 1024 },
	1586	pub const StorageStrategy = union(enum) {
	1587	/// In this case, the client's Allocator will be used to store the
	1588	/// entire HTTP header. This value is the maximum total size of
	1589	/// HTTP headers allowed, otherwise
	1590	/// error.HttpHeadersExceededSizeLimit is returned from read().
	1591	dynamic: usize,
	1592	/// This is used to store the entire HTTP header. If the HTTP
	1593	/// header is too big to fit, `error.HttpHeadersExceededSizeLimit`
	1594	/// is returned from read(). When this is used, `error.OutOfMemory`
	1595	/// cannot be returned from `read()`.
	1596	static: []u8,
	1597	};
	1598
	1599	server_header_buffer: ?[]u8 = null,
1577	1600	response_strategy: ResponseStrategy = .{ .storage = .{ .dynamic = 16 * 1024 * 1024 } },
1578	1601
1579	1602	location: Location,
1580	1603	method: http.Method = .GET,
1581		headers: http.Headers = http.Headers{ .allocator = std.heap.page_allocator, .owned = false },
	1604	headers: http.Headers = .{ .allocator = std.heap.page_allocator, .owned = false },
1582	1605	payload: Payload = .none,
1583	1606	raw_uri: bool = false,
1584	1607	};
...	...	@@ -1613,9 +1636,10 @@ pub fn fetch(client: *Client, allocator: Allocator, options: FetchOptions) !Fetc
1613	1636	.url => \|u\| try Uri.parse(u),
1614	1637	.uri => \|u\| u,
1615	1638	};
	1639	var server_header_buffer: [16 * 1024]u8 = undefined;
1616	1640
1617	1641	var req = try open(client, options.method, uri, options.headers, .{
1618		.header_strategy = options.header_strategy,
	1642	.server_header_buffer = options.server_header_buffer orelse &server_header_buffer,
1619	1643	.handle_redirects = options.payload == .none,
1620	1644	});
1621	1645	defer req.deinit();

lib/std/http/Server.zig+35-42

...	...	@@ -1,6 +1,7 @@
1	1	//! HTTP Server implementation.
2	2	//!
3		//! This server assumes all clients are well behaved and standard compliant; it can and will deadlock if a client holds a connection open without sending a request.
	3	//! This server assumes clients are well behaved and standard compliant; it
	4	//! deadlocks if a client holds a connection open without sending a request.
4	5	//!
5	6	//! Example usage:
6	7	//!
...	...	@@ -17,7 +18,7 @@
17	18	//! while (res.reset() != .closing) {
18	19	//! res.wait() catch \|err\| switch (err) {
19	20	//! error.HttpHeadersInvalid => break,
20		//! error.HttpHeadersExceededSizeLimit => {
	21	//! error.HttpHeadersOversize => {
21	22	//! res.status = .request_header_fields_too_large;
22	23	//! res.send() catch break;
23	24	//! break;
...	...	@@ -39,6 +40,7 @@
39	40	//! }
40	41	//! ```
41	42
	43	const builtin = @import("builtin");
42	44	const std = @import("../std.zig");
43	45	const testing = std.testing;
44	46	const http = std.http;
...	...	@@ -86,7 +88,7 @@ pub const Connection = struct {
86	88	const nread = try conn.rawReadAtLeast(conn.read_buf[0..], 1);
87	89	if (nread == 0) return error.EndOfStream;
88	90	conn.read_start = 0;
89		conn.read_end = @as(u16, @intCast(nread));
	91	conn.read_end = @intCast(nread);
90	92	}
91	93
92	94	pub fn peek(conn: *Connection) []const u8 {
...	...	@@ -382,10 +384,6 @@ pub const Response = struct {
382	384
383	385	res.headers.deinit();
384	386	res.request.headers.deinit();
385
386		if (res.request.parser.header_bytes_owned) {
387		res.request.parser.header_bytes.deinit(res.allocator);
388		}
389	387	}
390	388
391	389	pub const ResetState = enum { reset, closing };
...	...	@@ -548,17 +546,24 @@ pub const Response = struct {
548	546	return index;
549	547	}
550	548
551		pub const WaitError = Connection.ReadError \|\| proto.HeadersParser.CheckCompleteHeadError \|\| Request.ParseError \|\| error{ CompressionInitializationFailed, CompressionNotSupported };
	549	pub const WaitError = Connection.ReadError \|\|
	550	proto.HeadersParser.CheckCompleteHeadError \|\| Request.ParseError \|\|
	551	error{ CompressionInitializationFailed, CompressionNotSupported };
552	552
553	553	/// Wait for the client to send a complete request head.
554	554	///
555	555	/// For correct behavior, the following rules must be followed:
556	556	///
557		/// * If this returns any error in `Connection.ReadError`, you MUST immediately close the connection by calling `deinit`.
558		/// * If this returns `error.HttpHeadersInvalid`, you MAY immediately close the connection by calling `deinit`.
559		/// * If this returns `error.HttpHeadersExceededSizeLimit`, you MUST respond with a 431 status code and then call `deinit`.
560		/// * If this returns any error in `Request.ParseError`, you MUST respond with a 400 status code and then call `deinit`.
561		/// * If this returns any other error, you MUST respond with a 400 status code and then call `deinit`.
	557	/// * If this returns any error in `Connection.ReadError`, you MUST
	558	/// immediately close the connection by calling `deinit`.
	559	/// * If this returns `error.HttpHeadersInvalid`, you MAY immediately close
	560	/// the connection by calling `deinit`.
	561	/// * If this returns `error.HttpHeadersOversize`, you MUST
	562	/// respond with a 431 status code and then call `deinit`.
	563	/// * If this returns any error in `Request.ParseError`, you MUST respond
	564	/// with a 400 status code and then call `deinit`.
	565	/// * If this returns any other error, you MUST respond with a 400 status
	566	/// code and then call `deinit`.
562	567	/// * If the request has an Expect header containing 100-continue, you MUST either:
563	568	/// * Respond with a 100 status code, then call `wait` again.
564	569	/// * Respond with a 417 status code.
...	...	@@ -571,14 +576,14 @@ pub const Response = struct {
571	576	while (true) {
572	577	try res.connection.fill();
573	578
574		const nchecked = try res.request.parser.checkCompleteHead(res.allocator, res.connection.peek());
575		res.connection.drop(@as(u16, @intCast(nchecked)));
	579	const nchecked = try res.request.parser.checkCompleteHead(res.connection.peek());
	580	res.connection.drop(@intCast(nchecked));
576	581
577	582	if (res.request.parser.state.isContent()) break;
578	583	}
579	584
580	585	res.request.headers = .{ .allocator = res.allocator, .owned = true };
581		try res.request.parse(res.request.parser.header_bytes.items);
	586	try res.request.parse(res.request.parser.get());
582	587
583	588	if (res.request.transfer_encoding != .none) {
584	589	switch (res.request.transfer_encoding) {
...	...	@@ -641,16 +646,18 @@ pub const Response = struct {
641	646	while (!res.request.parser.state.isContent()) { // read trailing headers
642	647	try res.connection.fill();
643	648
644		const nchecked = try res.request.parser.checkCompleteHead(res.allocator, res.connection.peek());
645		res.connection.drop(@as(u16, @intCast(nchecked)));
	649	const nchecked = try res.request.parser.checkCompleteHead(res.connection.peek());
	650	res.connection.drop(@intCast(nchecked));
646	651	}
647	652
648	653	if (has_trail) {
649	654	res.request.headers = http.Headers{ .allocator = res.allocator, .owned = false };
650	655
651		// The response headers before the trailers are already guaranteed to be valid, so they will always be parsed again and cannot return an error.
	656	// The response headers before the trailers are already
	657	// guaranteed to be valid, so they will always be parsed again
	658	// and cannot return an error.
652	659	// This will only fail for a malformed trailer.
653		res.request.parse(res.request.parser.header_bytes.items) catch return error.InvalidTrailers;
	660	res.request.parse(res.request.parser.get()) catch return error.InvalidTrailers;
654	661	}
655	662	}
656	663
...	...	@@ -751,29 +758,19 @@ pub fn listen(server: *Server, address: net.Address) ListenError!void {
751	758
752	759	pub const AcceptError = net.StreamServer.AcceptError \|\| Allocator.Error;
753	760
754		pub const HeaderStrategy = union(enum) {
755		/// In this case, the client's Allocator will be used to store the
756		/// entire HTTP header. This value is the maximum total size of
757		/// HTTP headers allowed, otherwise
758		/// error.HttpHeadersExceededSizeLimit is returned from read().
759		dynamic: usize,
760		/// This is used to store the entire HTTP header. If the HTTP
761		/// header is too big to fit, `error.HttpHeadersExceededSizeLimit`
762		/// is returned from read(). When this is used, `error.OutOfMemory`
763		/// cannot be returned from `read()`.
764		static: []u8,
765		};
766
767	761	pub const AcceptOptions = struct {
768	762	allocator: Allocator,
769		header_strategy: HeaderStrategy = .{ .dynamic = 8192 },
	763	/// Externally-owned memory used to store the client's entire HTTP header.
	764	/// `error.HttpHeadersOversize` is returned from read() when a
	765	/// client sends too many bytes of HTTP headers.
	766	client_header_buffer: []u8,
770	767	};
771	768
772	769	/// Accept a new connection.
773	770	pub fn accept(server: *Server, options: AcceptOptions) AcceptError!Response {
774	771	const in = try server.socket.accept();
775	772
776		return Response{
	773	return .{
777	774	.allocator = options.allocator,
778	775	.address = in.address,
779	776	.connection = .{
...	...	@@ -786,17 +783,12 @@ pub fn accept(server: *Server, options: AcceptOptions) AcceptError!Response {
786	783	.method = undefined,
787	784	.target = undefined,
788	785	.headers = .{ .allocator = options.allocator, .owned = false },
789		.parser = switch (options.header_strategy) {
790		.dynamic => \|max\| proto.HeadersParser.initDynamic(max),
791		.static => \|buf\| proto.HeadersParser.initStatic(buf),
792		},
	786	.parser = proto.HeadersParser.init(options.client_header_buffer),
793	787	},
794	788	};
795	789	}
796	790
797	791	test "HTTP server handles a chunked transfer coding request" {
798		const builtin = @import("builtin");
799
800	792	// This test requires spawning threads.
801	793	if (builtin.single_threaded) {
802	794	return error.SkipZigTest;
...	...	@@ -823,9 +815,10 @@ test "HTTP server handles a chunked transfer coding request" {
823	815
824	816	const server_thread = try std.Thread.spawn(.{}, (struct {
825	817	fn apply(s: *std.http.Server) !void {
	818	var header_buffer: [max_header_size]u8 = undefined;
826	819	var res = try s.accept(.{
827	820	.allocator = allocator,
828		.header_strategy = .{ .dynamic = max_header_size },
	821	.client_header_buffer = &header_buffer,
829	822	});
830	823	defer res.deinit();
831	824	defer _ = res.reset();

lib/std/http/protocol.zig+70-74

...	...	@@ -34,54 +34,49 @@ pub const State = enum {
34	34
35	35	pub const HeadersParser = struct {
36	36	state: State = .start,
37		/// Whether or not `header_bytes` is allocated or was provided as a fixed buffer.
38		header_bytes_owned: bool,
39		/// Either a fixed buffer of len `max_header_bytes` or a dynamic buffer that can grow up to `max_header_bytes`.
	37	/// A fixed buffer of len `max_header_bytes`.
40	38	/// Pointers into this buffer are not stable until after a message is complete.
41		header_bytes: std.ArrayListUnmanaged(u8),
42		/// The maximum allowed size of `header_bytes`.
43		max_header_bytes: usize,
44		next_chunk_length: u64 = 0,
	39	header_bytes_buffer: []u8,
	40	header_bytes_len: u32,
	41	next_chunk_length: u64,
45	42	/// Whether this parser is done parsing a complete message.
46	43	/// A message is only done when the entire payload has been read.
47		done: bool = false,
	44	done: bool,
48	45
49		/// Initializes the parser with a dynamically growing header buffer of up to `max` bytes.
50		pub fn initDynamic(max: usize) HeadersParser {
	46	/// Initializes the parser with a provided buffer `buf`.
	47	pub fn init(buf: []u8) HeadersParser {
51	48	return .{
52		.header_bytes = .{},
53		.max_header_bytes = max,
54		.header_bytes_owned = true,
	49	.header_bytes_buffer = buf,
	50	.header_bytes_len = 0,
	51	.done = false,
	52	.next_chunk_length = 0,
55	53	};
56	54	}
57	55
58		/// Initializes the parser with a provided buffer `buf`.
59		pub fn initStatic(buf: []u8) HeadersParser {
60		return .{
61		.header_bytes = .{ .items = buf[0..0], .capacity = buf.len },
62		.max_header_bytes = buf.len,
63		.header_bytes_owned = false,
	56	/// Reinitialize the parser.
	57	/// Asserts the parser is in the "done" state.
	58	pub fn reset(hp: *HeadersParser) void {
	59	assert(hp.done);
	60	hp.* = .{
	61	.state = .start,
	62	.header_bytes_buffer = hp.header_bytes_buffer,
	63	.header_bytes_len = 0,
	64	.done = false,
	65	.next_chunk_length = 0,
64	66	};
65	67	}
66	68
67		/// Completely resets the parser to it's initial state.
68		/// This must be called after a message is complete.
69		pub fn reset(r: *HeadersParser) void {
70		assert(r.done); // The message must be completely read before reset, otherwise the parser is in an invalid state.
71
72		r.header_bytes.clearRetainingCapacity();
73
74		r.* = .{
75		.header_bytes = r.header_bytes,
76		.max_header_bytes = r.max_header_bytes,
77		.header_bytes_owned = r.header_bytes_owned,
78		};
	69	pub fn get(hp: HeadersParser) []u8 {
	70	return hp.header_bytes_buffer[0..hp.header_bytes_len];
79	71	}
80	72
81		/// Returns the number of bytes consumed by headers. This is always less than or equal to `bytes.len`.
82		/// You should check `r.state.isContent()` after this to check if the headers are done.
	73	/// Returns the number of bytes consumed by headers. This is always less
	74	/// than or equal to `bytes.len`.
	75	/// You should check `r.state.isContent()` after this to check if the
	76	/// headers are done.
83	77	///
84		/// If the amount returned is less than `bytes.len`, you may assume that the parser is in a content state and the
	78	/// If the amount returned is less than `bytes.len`, you may assume that
	79	/// the parser is in a content state and the
85	80	/// first byte of content is located at `bytes[result]`.
86	81	pub fn findHeadersEnd(r: *HeadersParser, bytes: []const u8) u32 {
87	82	const vector_len: comptime_int = @max(std.simd.suggestVectorLength(u8) orelse 1, 8);
...	...	@@ -410,11 +405,14 @@ pub const HeadersParser = struct {
410	405	}
411	406	}
412	407
413		/// Returns the number of bytes consumed by the chunk size. This is always less than or equal to `bytes.len`.
414		/// You should check `r.state == .chunk_data` after this to check if the chunk size has been fully parsed.
	408	/// Returns the number of bytes consumed by the chunk size. This is always
	409	/// less than or equal to `bytes.len`.
	410	/// You should check `r.state == .chunk_data` after this to check if the
	411	/// chunk size has been fully parsed.
415	412	///
416		/// If the amount returned is less than `bytes.len`, you may assume that the parser is in the `chunk_data` state
417		/// and that the first byte of the chunk is at `bytes[result]`.
	413	/// If the amount returned is less than `bytes.len`, you may assume that
	414	/// the parser is in the `chunk_data` state and that the first byte of the
	415	/// chunk is at `bytes[result]`.
418	416	pub fn findChunkedLen(r: *HeadersParser, bytes: []const u8) u32 {
419	417	const len = @as(u32, @intCast(bytes.len));
420	418
...	...	@@ -488,30 +486,27 @@ pub const HeadersParser = struct {
488	486	return len;
489	487	}
490	488
491		/// Returns whether or not the parser has finished parsing a complete message. A message is only complete after the
492		/// entire body has been read and any trailing headers have been parsed.
	489	/// Returns whether or not the parser has finished parsing a complete
	490	/// message. A message is only complete after the entire body has been read
	491	/// and any trailing headers have been parsed.
493	492	pub fn isComplete(r: *HeadersParser) bool {
494	493	return r.done and r.state == .finished;
495	494	}
496	495
497		pub const CheckCompleteHeadError = mem.Allocator.Error \|\| error{HttpHeadersExceededSizeLimit};
	496	pub const CheckCompleteHeadError = error{HttpHeadersOversize};
498	497
499		/// Pushes `in` into the parser. Returns the number of bytes consumed by the header. Any header bytes are appended
500		/// to the `header_bytes` buffer.
501		///
502		/// This function only uses `allocator` if `r.header_bytes_owned` is true, and may be undefined otherwise.
503		pub fn checkCompleteHead(r: *HeadersParser, allocator: std.mem.Allocator, in: []const u8) CheckCompleteHeadError!u32 {
504		if (r.state.isContent()) return 0;
	498	/// Pushes `in` into the parser. Returns the number of bytes consumed by
	499	/// the header. Any header bytes are appended to `header_bytes_buffer`.
	500	pub fn checkCompleteHead(hp: *HeadersParser, in: []const u8) CheckCompleteHeadError!u32 {
	501	if (hp.state.isContent()) return 0;
505	502
506		const i = r.findHeadersEnd(in);
	503	const i = hp.findHeadersEnd(in);
507	504	const data = in[0..i];
508		if (r.header_bytes.items.len + data.len > r.max_header_bytes) {
509		return error.HttpHeadersExceededSizeLimit;
510		} else {
511		if (r.header_bytes_owned) try r.header_bytes.ensureUnusedCapacity(allocator, data.len);
	505	if (hp.header_bytes_len + data.len > hp.header_bytes_buffer.len)
	506	return error.HttpHeadersOversize;
512	507
513		r.header_bytes.appendSliceAssumeCapacity(data);
514		}
	508	@memcpy(hp.header_bytes_buffer[hp.header_bytes_len..][0..data.len], data);
	509	hp.header_bytes_len += @intCast(data.len);
515	510
516	511	return i;
517	512	}
...	...	@@ -520,7 +515,8 @@ pub const HeadersParser = struct {
520	515	HttpChunkInvalid,
521	516	};
522	517
523		/// Reads the body of the message into `buffer`. Returns the number of bytes placed in the buffer.
	518	/// Reads the body of the message into `buffer`. Returns the number of
	519	/// bytes placed in the buffer.
524	520	///
525	521	/// If `skip` is true, the buffer will be unused and the body will be skipped.
526	522	///
...	...	@@ -718,7 +714,7 @@ test "HeadersParser.findHeadersEnd" {
718	714	const data = "GET / HTTP/1.1\r\nHost: localhost\r\n\r\nHello";
719	715
720	716	for (0..36) \|i\| {
721		r = HeadersParser.initDynamic(0);
	717	r = HeadersParser.init(&.{});
722	718	try std.testing.expectEqual(@as(u32, @intCast(i)), r.findHeadersEnd(data[0..i]));
723	719	try std.testing.expectEqual(@as(u32, @intCast(35 - i)), r.findHeadersEnd(data[i..]));
724	720	}
...	...	@@ -728,7 +724,7 @@ test "HeadersParser.findChunkedLen" {
728	724	var r: HeadersParser = undefined;
729	725	const data = "Ff\r\nf0f000 ; ext\n0\r\nffffffffffffffffffffffffffffffffffffffff\r\n";
730	726
731		r = HeadersParser.initDynamic(0);
	727	r = HeadersParser.init(&.{});
732	728	r.state = .chunk_head_size;
733	729	r.next_chunk_length = 0;
734	730
...	...	@@ -761,9 +757,9 @@ test "HeadersParser.findChunkedLen" {
761	757
762	758	test "HeadersParser.read length" {
763	759	// mock BufferedConnection for read
	760	var headers_buf: [256]u8 = undefined;
764	761
765		var r = HeadersParser.initDynamic(256);
766		defer r.header_bytes.deinit(std.testing.allocator);
	762	var r = HeadersParser.init(&headers_buf);
767	763	const data = "GET / HTTP/1.1\r\nHost: localhost\r\nContent-Length: 5\r\n\r\nHello";
768	764
769	765	var conn: MockBufferedConnection = .{
...	...	@@ -773,8 +769,8 @@ test "HeadersParser.read length" {
773	769	while (true) { // read headers
774	770	try conn.fill();
775	771
776		const nchecked = try r.checkCompleteHead(std.testing.allocator, conn.peek());
777		conn.drop(@as(u16, @intCast(nchecked)));
	772	const nchecked = try r.checkCompleteHead(conn.peek());
	773	conn.drop(@intCast(nchecked));
778	774
779	775	if (r.state.isContent()) break;
780	776	}
...	...	@@ -786,14 +782,14 @@ test "HeadersParser.read length" {
786	782	try std.testing.expectEqual(@as(usize, 5), len);
787	783	try std.testing.expectEqualStrings("Hello", buf[0..len]);
788	784
789		try std.testing.expectEqualStrings("GET / HTTP/1.1\r\nHost: localhost\r\nContent-Length: 5\r\n\r\n", r.header_bytes.items);
	785	try std.testing.expectEqualStrings("GET / HTTP/1.1\r\nHost: localhost\r\nContent-Length: 5\r\n\r\n", r.get());
790	786	}
791	787
792	788	test "HeadersParser.read chunked" {
793	789	// mock BufferedConnection for read
794	790
795		var r = HeadersParser.initDynamic(256);
796		defer r.header_bytes.deinit(std.testing.allocator);
	791	var headers_buf: [256]u8 = undefined;
	792	var r = HeadersParser.init(&headers_buf);
797	793	const data = "GET / HTTP/1.1\r\nHost: localhost\r\n\r\n2\r\nHe\r\n2\r\nll\r\n1\r\no\r\n0\r\n\r\n";
798	794
799	795	var conn: MockBufferedConnection = .{
...	...	@@ -803,8 +799,8 @@ test "HeadersParser.read chunked" {
803	799	while (true) { // read headers
804	800	try conn.fill();
805	801
806		const nchecked = try r.checkCompleteHead(std.testing.allocator, conn.peek());
807		conn.drop(@as(u16, @intCast(nchecked)));
	802	const nchecked = try r.checkCompleteHead(conn.peek());
	803	conn.drop(@intCast(nchecked));
808	804
809	805	if (r.state.isContent()) break;
810	806	}
...	...	@@ -815,14 +811,14 @@ test "HeadersParser.read chunked" {
815	811	try std.testing.expectEqual(@as(usize, 5), len);
816	812	try std.testing.expectEqualStrings("Hello", buf[0..len]);
817	813
818		try std.testing.expectEqualStrings("GET / HTTP/1.1\r\nHost: localhost\r\n\r\n", r.header_bytes.items);
	814	try std.testing.expectEqualStrings("GET / HTTP/1.1\r\nHost: localhost\r\n\r\n", r.get());
819	815	}
820	816
821	817	test "HeadersParser.read chunked trailer" {
822	818	// mock BufferedConnection for read
823	819
824		var r = HeadersParser.initDynamic(256);
825		defer r.header_bytes.deinit(std.testing.allocator);
	820	var headers_buf: [256]u8 = undefined;
	821	var r = HeadersParser.init(&headers_buf);
826	822	const data = "GET / HTTP/1.1\r\nHost: localhost\r\n\r\n2\r\nHe\r\n2\r\nll\r\n1\r\no\r\n0\r\nContent-Type: text/plain\r\n\r\n";
827	823
828	824	var conn: MockBufferedConnection = .{
...	...	@@ -832,8 +828,8 @@ test "HeadersParser.read chunked trailer" {
832	828	while (true) { // read headers
833	829	try conn.fill();
834	830
835		const nchecked = try r.checkCompleteHead(std.testing.allocator, conn.peek());
836		conn.drop(@as(u16, @intCast(nchecked)));
	831	const nchecked = try r.checkCompleteHead(conn.peek());
	832	conn.drop(@intCast(nchecked));
837	833
838	834	if (r.state.isContent()) break;
839	835	}
...	...	@@ -847,11 +843,11 @@ test "HeadersParser.read chunked trailer" {
847	843	while (true) { // read headers
848	844	try conn.fill();
849	845
850		const nchecked = try r.checkCompleteHead(std.testing.allocator, conn.peek());
851		conn.drop(@as(u16, @intCast(nchecked)));
	846	const nchecked = try r.checkCompleteHead(conn.peek());
	847	conn.drop(@intCast(nchecked));
852	848
853	849	if (r.state.isContent()) break;
854	850	}
855	851
856		try std.testing.expectEqualStrings("GET / HTTP/1.1\r\nHost: localhost\r\n\r\nContent-Type: text/plain\r\n\r\n", r.header_bytes.items);
	852	try std.testing.expectEqualStrings("GET / HTTP/1.1\r\nHost: localhost\r\n\r\nContent-Type: text/plain\r\n\r\n", r.get());
857	853	}

src/Package/Fetch.zig+14-7

...	...	@@ -354,7 +354,8 @@ pub fn run(f: *Fetch) RunError!void {
354	354	.{ path_or_url, @errorName(file_err), @errorName(uri_err) },
355	355	));
356	356	};
357		var resource = try f.initResource(uri);
	357	var server_header_buffer: [header_buffer_size]u8 = undefined;
	358	var resource = try f.initResource(uri, &server_header_buffer);
358	359	return runResource(f, uri.path, &resource, null);
359	360	}
360	361	},
...	...	@@ -415,7 +416,8 @@ pub fn run(f: *Fetch) RunError!void {
415	416	f.location_tok,
416	417	try eb.printString("invalid URI: {s}", .{@errorName(err)}),
417	418	);
418		var resource = try f.initResource(uri);
	419	var server_header_buffer: [header_buffer_size]u8 = undefined;
	420	var resource = try f.initResource(uri, &server_header_buffer);
419	421	return runResource(f, uri.path, &resource, remote.hash);
420	422	}
421	423
...	...	@@ -876,7 +878,9 @@ const FileType = enum {
876	878	}
877	879	};
878	880
879		fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {
	881	const header_buffer_size = 16 * 1024;
	882
	883	fn initResource(f: *Fetch, uri: std.Uri, server_header_buffer: []u8) RunError!Resource {
880	884	const gpa = f.arena.child_allocator;
881	885	const arena = f.arena.allocator();
882	886	const eb = &f.error_bundle;
...	...	@@ -894,10 +898,12 @@ fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {
894	898	if (ascii.eqlIgnoreCase(uri.scheme, "http") or
895	899	ascii.eqlIgnoreCase(uri.scheme, "https"))
896	900	{
897		var h = std.http.Headers{ .allocator = gpa };
	901	var h: std.http.Headers = .{ .allocator = gpa };
898	902	defer h.deinit();
899	903
900		var req = http_client.open(.GET, uri, h, .{}) catch \|err\| {
	904	var req = http_client.open(.GET, uri, h, .{
	905	.server_header_buffer = server_header_buffer,
	906	}) catch \|err\| {
901	907	return f.fail(f.location_tok, try eb.printString(
902	908	"unable to connect to server: {s}",
903	909	.{@errorName(err)},
...	...	@@ -935,7 +941,7 @@ fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {
935	941	transport_uri.scheme = uri.scheme["git+".len..];
936	942	var redirect_uri: []u8 = undefined;
937	943	var session: git.Session = .{ .transport = http_client, .uri = transport_uri };
938		session.discoverCapabilities(gpa, &redirect_uri) catch \|err\| switch (err) {
	944	session.discoverCapabilities(gpa, &redirect_uri, server_header_buffer) catch \|err\| switch (err) {
939	945	error.Redirected => {
940	946	defer gpa.free(redirect_uri);
941	947	return f.fail(f.location_tok, try eb.printString(
...	...	@@ -961,6 +967,7 @@ fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {
961	967	var ref_iterator = session.listRefs(gpa, .{
962	968	.ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag },
963	969	.include_peeled = true,
	970	.server_header_buffer = server_header_buffer,
964	971	}) catch \|err\| {
965	972	return f.fail(f.location_tok, try eb.printString(
966	973	"unable to list refs: {s}",
...	...	@@ -1003,7 +1010,7 @@ fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {
1003	1010	_ = std.fmt.bufPrint(&want_oid_buf, "{}", .{
1004	1011	std.fmt.fmtSliceHexLower(&want_oid),
1005	1012	}) catch unreachable;
1006		var fetch_stream = session.fetch(gpa, &.{&want_oid_buf}) catch \|err\| {
	1013	var fetch_stream = session.fetch(gpa, &.{&want_oid_buf}, server_header_buffer) catch \|err\| {
1007	1014	return f.fail(f.location_tok, try eb.printString(
1008	1015	"unable to create fetch stream: {s}",
1009	1016	.{@errorName(err)},

src/Package/Fetch/git.zig+13-2

...	...	@@ -494,8 +494,9 @@ pub const Session = struct {
494	494	session: *Session,
495	495	allocator: Allocator,
496	496	redirect_uri: *[]u8,
	497	http_headers_buffer: []u8,
497	498	) !void {
498		var capability_iterator = try session.getCapabilities(allocator, redirect_uri);
	499	var capability_iterator = try session.getCapabilities(allocator, redirect_uri, http_headers_buffer);
499	500	defer capability_iterator.deinit();
500	501	while (try capability_iterator.next()) \|capability\| {
501	502	if (mem.eql(u8, capability.key, "agent")) {
...	...	@@ -521,6 +522,7 @@ pub const Session = struct {
521	522	session: Session,
522	523	allocator: Allocator,
523	524	redirect_uri: *[]u8,
	525	http_headers_buffer: []u8,
524	526	) !CapabilityIterator {
525	527	var info_refs_uri = session.uri;
526	528	info_refs_uri.path = try std.fs.path.resolvePosix(allocator, &.{ "/", session.uri.path, "info/refs" });
...	...	@@ -534,6 +536,7 @@ pub const Session = struct {
534	536
535	537	var request = try session.transport.open(.GET, info_refs_uri, headers, .{
536	538	.max_redirects = 3,
	539	.server_header_buffer = http_headers_buffer,
537	540	});
538	541	errdefer request.deinit();
539	542	try request.send(.{});
...	...	@@ -620,6 +623,7 @@ pub const Session = struct {
620	623	include_symrefs: bool = false,
621	624	/// Whether to include the peeled object ID for returned tag refs.
622	625	include_peeled: bool = false,
	626	server_header_buffer: []u8,
623	627	};
624	628
625	629	/// Returns an iterator over refs known to the server.
...	...	@@ -658,6 +662,7 @@ pub const Session = struct {
658	662
659	663	var request = try session.transport.open(.POST, upload_pack_uri, headers, .{
660	664	.handle_redirects = false,
	665	.server_header_buffer = options.server_header_buffer,
661	666	});
662	667	errdefer request.deinit();
663	668	request.transfer_encoding = .{ .content_length = body.items.len };
...	...	@@ -721,7 +726,12 @@ pub const Session = struct {
721	726
722	727	/// Fetches the given refs from the server. A shallow fetch (depth 1) is
723	728	/// performed if the server supports it.
724		pub fn fetch(session: Session, allocator: Allocator, wants: []const []const u8) !FetchStream {
	729	pub fn fetch(
	730	session: Session,
	731	allocator: Allocator,
	732	wants: []const []const u8,
	733	http_headers_buffer: []u8,
	734	) !FetchStream {
725	735	var upload_pack_uri = session.uri;
726	736	upload_pack_uri.path = try std.fs.path.resolvePosix(allocator, &.{ "/", session.uri.path, "git-upload-pack" });
727	737	defer allocator.free(upload_pack_uri.path);
...	...	@@ -758,6 +768,7 @@ pub const Session = struct {
758	768
759	769	var request = try session.transport.open(.POST, upload_pack_uri, headers, .{
760	770	.handle_redirects = false,
	771	.server_header_buffer = http_headers_buffer,
761	772	});
762	773	errdefer request.deinit();
763	774	request.transfer_encoding = .{ .content_length = body.items.len };