Commit f40f81cbfb0a867b32d4406e198550730de268cd | zig

author	Andrew Kelley <andrew@ziglang.org> 2023-09-17 00:41:30-07:00
committer	GitHub <noreply@github.com> 2023-09-17 00:41:30-07:00
log	f40f81cbfb0a867b32d4406e198550730de268cd
tree	5e8b04ffd75a73a927f5f32eeb0c79d119c585f9
parent	5389af2c1c8c50942da20dc5b0cc29cdca45e0e9
parent	4689d93cb204a4143770105200eb65dcdca5d7a0
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #16929 from truemedian/more-http

std.http: handle Expect: 100-continue, improve redirect logic, add Client.fetch for simple requests

6 files changed, 425 insertions(+), 79 deletions(-)

lib
- std
  - http.zig
  - http
    Client.zig
    Headers.zig
    Server.zig
    protocol.zig
test
- standalone
  - http.zig

lib/std/http.zig+37-12

...	...	@@ -1,3 +1,5 @@
	1	const std = @import("std.zig");
	2
1	3	pub const Client = @import("http/Client.zig");
2	4	pub const Server = @import("http/Server.zig");
3	5	pub const protocol = @import("http/protocol.zig");
...	...	@@ -14,16 +16,36 @@ pub const Version = enum {
14	16	/// https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods
15	17	/// https://datatracker.ietf.org/doc/html/rfc7231#section-4 Initial definition
16	18	/// https://datatracker.ietf.org/doc/html/rfc5789#section-2 PATCH
17		pub const Method = enum {
18		GET,
19		HEAD,
20		POST,
21		PUT,
22		DELETE,
23		CONNECT,
24		OPTIONS,
25		TRACE,
26		PATCH,
	19	pub const Method = enum(u64) { // TODO: should be u192 or u256, but neither is supported by the C backend, and therefore cannot pass CI
	20	GET = parse("GET"),
	21	HEAD = parse("HEAD"),
	22	POST = parse("POST"),
	23	PUT = parse("PUT"),
	24	DELETE = parse("DELETE"),
	25	CONNECT = parse("CONNECT"),
	26	OPTIONS = parse("OPTIONS"),
	27	TRACE = parse("TRACE"),
	28	PATCH = parse("PATCH"),
	29
	30	_,
	31
	32	/// Converts `s` into a type that may be used as a `Method` field.
	33	/// Asserts that `s` is 24 or fewer bytes.
	34	pub fn parse(s: []const u8) u64 {
	35	var x: u64 = 0;
	36	@memcpy(std.mem.asBytes(&x)[0..s.len], s);
	37	return x;
	38	}
	39
	40	pub fn write(self: Method, w: anytype) !void {
	41	const bytes = std.mem.asBytes(&@intFromEnum(self));
	42	const str = std.mem.sliceTo(bytes, 0);
	43	try w.writeAll(str);
	44	}
	45
	46	pub fn format(value: Method, comptime _: []const u8, _: std.fmt.FormatOptions, writer: anytype) @TypeOf(writer).Error!void {
	47	return try value.write(writer);
	48	}
27	49
28	50	/// Returns true if a request of this method is allowed to have a body
29	51	/// Actual behavior from servers may vary and should still be checked
...	...	@@ -31,6 +53,7 @@ pub const Method = enum {
31	53	return switch (self) {
32	54	.POST, .PUT, .PATCH => true,
33	55	.GET, .HEAD, .DELETE, .CONNECT, .OPTIONS, .TRACE => false,
	56	else => true,
34	57	};
35	58	}
36	59
...	...	@@ -40,6 +63,7 @@ pub const Method = enum {
40	63	return switch (self) {
41	64	.GET, .POST, .DELETE, .CONNECT, .OPTIONS, .PATCH => true,
42	65	.HEAD, .PUT, .TRACE => false,
	66	else => true,
43	67	};
44	68	}
45	69
...	...	@@ -50,6 +74,7 @@ pub const Method = enum {
50	74	return switch (self) {
51	75	.GET, .HEAD, .OPTIONS, .TRACE => true,
52	76	.POST, .PUT, .DELETE, .CONNECT, .PATCH => false,
	77	else => false,
53	78	};
54	79	}
55	80
...	...	@@ -60,6 +85,7 @@ pub const Method = enum {
60	85	return switch (self) {
61	86	.GET, .HEAD, .PUT, .DELETE, .OPTIONS, .TRACE => true,
62	87	.CONNECT, .POST, .PATCH => false,
	88	else => false,
63	89	};
64	90	}
65	91
...	...	@@ -70,6 +96,7 @@ pub const Method = enum {
70	96	return switch (self) {
71	97	.GET, .HEAD => true,
72	98	.POST, .PUT, .DELETE, .CONNECT, .OPTIONS, .TRACE, .PATCH => false,
	99	else => false,
73	100	};
74	101	}
75	102	};
...	...	@@ -269,8 +296,6 @@ pub const Connection = enum {
269	296	close,
270	297	};
271	298
272		const std = @import("std.zig");
273
274	299	test {
275	300	_ = Client;
276	301	_ = Method;

lib/std/http/Client.zig+216-24

...	...	@@ -365,8 +365,11 @@ pub const Response = struct {
365	365	if (trailing) continue;
366	366
367	367	if (std.ascii.eqlIgnoreCase(header_name, "content-length")) {
368		if (res.content_length != null) return error.HttpHeadersInvalid;
369		res.content_length = std.fmt.parseInt(u64, header_value, 10) catch return error.InvalidContentLength;
	368	const content_length = std.fmt.parseInt(u64, header_value, 10) catch return error.InvalidContentLength;
	369
	370	if (res.content_length != null and res.content_length != content_length) return error.HttpHeadersInvalid;
	371
	372	res.content_length = content_length;
370	373	} else if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) {
371	374	// Transfer-Encoding: second, first
372	375	// Transfer-Encoding: deflate, chunked
...	...	@@ -475,6 +478,7 @@ pub const Request = struct {
475	478	.zstd => \|*zstd\| zstd.deinit(),
476	479	}
477	480
	481	req.headers.deinit();
478	482	req.response.headers.deinit();
479	483
480	484	if (req.response.parser.header_bytes_owned) {
...	...	@@ -536,10 +540,12 @@ pub const Request = struct {
536	540
537	541	/// Send the request to the server.
538	542	pub fn start(req: *Request) StartError!void {
	543	if (!req.method.requestHasBody() and req.transfer_encoding != .none) return error.UnsupportedTransferEncoding;
	544
539	545	var buffered = std.io.bufferedWriter(req.connection.?.data.writer());
540	546	const w = buffered.writer();
541	547
542		try w.writeAll(@tagName(req.method));
	548	try req.method.write(w);
543	549	try w.writeByte(' ');
544	550
545	551	if (req.method == .CONNECT) {
...	...	@@ -607,22 +613,29 @@ pub const Request = struct {
607	613	}
608	614	}
609	615
610		try w.print("{}", .{req.headers});
	616	for (req.headers.list.items) \|entry\| {
	617	if (entry.value.len == 0) continue;
	618
	619	try w.writeAll(entry.name);
	620	try w.writeAll(": ");
	621	try w.writeAll(entry.value);
	622	try w.writeAll("\r\n");
	623	}
611	624
612	625	try w.writeAll("\r\n");
613	626
614	627	try buffered.flush();
615	628	}
616	629
617		pub const TransferReadError = Connection.ReadError \|\| proto.HeadersParser.ReadError;
	630	const TransferReadError = Connection.ReadError \|\| proto.HeadersParser.ReadError;
618	631
619		pub const TransferReader = std.io.Reader(*Request, TransferReadError, transferRead);
	632	const TransferReader = std.io.Reader(*Request, TransferReadError, transferRead);
620	633
621		pub fn transferReader(req: *Request) TransferReader {
	634	fn transferReader(req: *Request) TransferReader {
622	635	return .{ .context = req };
623	636	}
624	637
625		pub fn transferRead(req: *Request, buf: []u8) TransferReadError!usize {
	638	fn transferRead(req: *Request, buf: []u8) TransferReadError!usize {
626	639	if (req.response.parser.done) return 0;
627	640
628	641	var index: usize = 0;
...	...	@@ -635,13 +648,13 @@ pub const Request = struct {
635	648	return index;
636	649	}
637	650
638		pub const WaitError = RequestError \|\| StartError \|\| TransferReadError \|\| proto.HeadersParser.CheckCompleteHeadError \|\| Response.ParseError \|\| Uri.ParseError \|\| error{ TooManyHttpRedirects, CannotRedirect, HttpRedirectMissingLocation, CompressionInitializationFailed, CompressionNotSupported };
	651	pub const WaitError = RequestError \|\| StartError \|\| TransferReadError \|\| proto.HeadersParser.CheckCompleteHeadError \|\| Response.ParseError \|\| Uri.ParseError \|\| error{ TooManyHttpRedirects, RedirectRequiresResend, HttpRedirectMissingLocation, CompressionInitializationFailed, CompressionNotSupported };
639	652
640	653	/// Waits for a response from the server and parses any headers that are sent.
641	654	/// This function will block until the final response is received.
642	655	///
643	656	/// If `handle_redirects` is true and the request has no payload, then this function will automatically follow
644		/// redirects. If a request payload is present, then this function will error with error.CannotRedirect.
	657	/// redirects. If a request payload is present, then this function will error with error.RedirectRequiresResend.
645	658	pub fn wait(req: *Request) WaitError!void {
646	659	while (true) { // handle redirects
647	660	while (true) { // read headers
...	...	@@ -655,17 +668,19 @@ pub const Request = struct {
655	668
656	669	try req.response.parse(req.response.parser.header_bytes.items, false);
657	670
658		if (req.response.status == .switching_protocols) {
659		req.connection.?.data.closing = false;
660		req.response.parser.done = true;
	671	if (req.response.status == .@"continue") {
	672	req.response.parser.done = true; // we're done parsing the continue response, reset to prepare for the real response
	673	req.response.parser.reset();
	674	break;
661	675	}
662	676
663		if (req.method == .CONNECT and req.response.status == .ok) {
	677	// we're switching protocols, so this connection is no longer doing http
	678	if (req.response.status == .switching_protocols or (req.method == .CONNECT and req.response.status == .ok)) {
664	679	req.connection.?.data.closing = false;
665	680	req.response.parser.done = true;
666	681	}
667	682
668		// we default to using keep-alive if not provided
	683	// we default to using keep-alive if not provided in the client if the server asks for it
669	684	const req_connection = req.headers.getFirstValue("connection");
670	685	const req_keepalive = req_connection != null and !std.ascii.eqlIgnoreCase("close", req_connection.?);
671	686
...	...	@@ -697,9 +712,10 @@ pub const Request = struct {
697	712	req.response.parser.done = true;
698	713	}
699	714
700		if (req.transfer_encoding == .none and req.response.status.class() == .redirect and req.handle_redirects) {
	715	if (req.response.status.class() == .redirect and req.handle_redirects) {
701	716	req.response.skip = true;
702	717
	718	// skip the body of the redirect response, this will at least leave the connection in a known good state.
703	719	const empty = @as([*]u8, undefined)[0..0];
704	720	assert(try req.transferRead(empty) == 0); // we're skipping, no buffer is necessary
705	721
...	...	@@ -715,6 +731,30 @@ pub const Request = struct {
715	731	const new_url = Uri.parse(location_duped) catch try Uri.parseWithoutScheme(location_duped);
716	732	const resolved_url = try req.uri.resolve(new_url, false, arena);
717	733
	734	// is the redirect location on the same domain, or a subdomain of the original request?
	735	const is_same_domain_or_subdomain = std.ascii.endsWithIgnoreCase(resolved_url.host.?, req.uri.host.?) and (resolved_url.host.?.len == req.uri.host.?.len or resolved_url.host.?[resolved_url.host.?.len - req.uri.host.?.len - 1] == '.');
	736
	737	if (resolved_url.host == null or !is_same_domain_or_subdomain or !std.ascii.eqlIgnoreCase(resolved_url.scheme, req.uri.scheme)) {
	738	// we're redirecting to a different domain, strip privileged headers like cookies
	739	_ = req.headers.delete("authorization");
	740	_ = req.headers.delete("www-authenticate");
	741	_ = req.headers.delete("cookie");
	742	_ = req.headers.delete("cookie2");
	743	}
	744
	745	if (req.response.status == .see_other or ((req.response.status == .moved_permanently or req.response.status == .found) and req.method == .POST)) {
	746	// we're redirecting to a GET, so we need to change the method and remove the body
	747	req.method = .GET;
	748	req.transfer_encoding = .none;
	749	_ = req.headers.delete("transfer-encoding");
	750	_ = req.headers.delete("content-length");
	751	_ = req.headers.delete("content-type");
	752	}
	753
	754	if (req.transfer_encoding != .none) {
	755	return error.RedirectRequiresResend; // The request body has already been sent. The request is still in a valid state, but the redirect must be handled manually.
	756	}
	757
718	758	try req.redirect(resolved_url);
719	759
720	760	try req.start();
...	...	@@ -735,9 +775,6 @@ pub const Request = struct {
735	775	};
736	776	}
737	777
738		if (req.response.status.class() == .redirect and req.handle_redirects and req.transfer_encoding != .none)
739		return error.CannotRedirect; // The request body has already been sent. The request is still in a valid state, but the redirect must be handled manually.
740
741	778	break;
742	779	}
743	780	}
...	...	@@ -921,6 +958,40 @@ pub fn connectUnproxied(client: *Client, host: []const u8, port: u16, protocol:
921	958	return conn;
922	959	}
923	960
	961	pub const ConnectUnixError = Allocator.Error \|\| std.os.SocketError \|\| error{ NameTooLong, Unsupported } \|\| std.os.ConnectError;
	962
	963	pub fn connectUnix(client: Client, path: []const u8) ConnectUnixError!ConnectionPool.Node {
	964	if (!net.has_unix_sockets) return error.Unsupported;
	965
	966	if (client.connection_pool.findConnection(.{
	967	.host = path,
	968	.port = 0,
	969	.is_tls = false,
	970	})) \|node\|
	971	return node;
	972
	973	const conn = try client.allocator.create(ConnectionPool.Node);
	974	errdefer client.allocator.destroy(conn);
	975	conn.* = .{ .data = undefined };
	976
	977	const stream = try std.net.connectUnixSocket(path);
	978	errdefer stream.close();
	979
	980	conn.data = .{
	981	.stream = stream,
	982	.tls_client = undefined,
	983	.protocol = .plain,
	984
	985	.host = try client.allocator.dupe(u8, path),
	986	.port = 0,
	987	};
	988	errdefer client.allocator.free(conn.data.host);
	989
	990	client.connection_pool.addUsed(conn);
	991
	992	return conn;
	993	}
	994
924	995	// Prevents a dependency loop in request()
925	996	const ConnectErrorPartial = ConnectUnproxiedError \|\| error{ UnsupportedUrlScheme, ConnectionRefused };
926	997	pub const ConnectError = ConnectErrorPartial \|\| RequestError;
...	...	@@ -956,17 +1027,17 @@ pub const RequestError = ConnectUnproxiedError \|\| ConnectErrorPartial \|\| Request
956	1027	UnsupportedTransferEncoding,
957	1028	};
958	1029
959		pub const Options = struct {
	1030	pub const RequestOptions = struct {
960	1031	version: http.Version = .@"HTTP/1.1",
961	1032
962	1033	handle_redirects: bool = true,
963	1034	max_redirects: u32 = 3,
964		header_strategy: HeaderStrategy = .{ .dynamic = 16 * 1024 },
	1035	header_strategy: StorageStrategy = .{ .dynamic = 16 * 1024 },
965	1036
966	1037	/// Must be an already acquired connection.
967	1038	connection: ?*ConnectionPool.Node = null,
968	1039
969		pub const HeaderStrategy = union(enum) {
	1040	pub const StorageStrategy = union(enum) {
970	1041	/// In this case, the client's Allocator will be used to store the
971	1042	/// entire HTTP header. This value is the maximum total size of
972	1043	/// HTTP headers allowed, otherwise
...	...	@@ -988,8 +1059,12 @@ pub const protocol_map = std.ComptimeStringMap(Connection.Protocol, .{
988	1059	});
989	1060
990	1061	/// Form and send a http request to a server.
	1062	///
	1063	/// `uri` must remain alive during the entire request.
	1064	/// `headers` is cloned and may be freed after this function returns.
	1065	///
991	1066	/// This function is threadsafe.
992		pub fn request(client: *Client, method: http.Method, uri: Uri, headers: http.Headers, options: Options) RequestError!Request {
	1067	pub fn request(client: *Client, method: http.Method, uri: Uri, headers: http.Headers, options: RequestOptions) RequestError!Request {
993	1068	const protocol = protocol_map.get(uri.scheme) orelse return error.UnsupportedUrlScheme;
994	1069
995	1070	const port: u16 = uri.port orelse switch (protocol) {
...	...	@@ -1015,7 +1090,7 @@ pub fn request(client: *Client, method: http.Method, uri: Uri, headers: http.Hea
1015	1090	.uri = uri,
1016	1091	.client = client,
1017	1092	.connection = conn,
1018		.headers = headers,
	1093	.headers = try headers.clone(client.allocator), // Headers must be cloned to properly handle header transformations in redirects.
1019	1094	.method = method,
1020	1095	.version = options.version,
1021	1096	.redirects_left = options.max_redirects,
...	...	@@ -1039,6 +1114,123 @@ pub fn request(client: *Client, method: http.Method, uri: Uri, headers: http.Hea
1039	1114	return req;
1040	1115	}
1041	1116
	1117	pub const FetchOptions = struct {
	1118	pub const Location = union(enum) {
	1119	url: []const u8,
	1120	uri: Uri,
	1121	};
	1122
	1123	pub const Payload = union(enum) {
	1124	string: []const u8,
	1125	file: std.fs.File,
	1126	none,
	1127	};
	1128
	1129	pub const ResponseStrategy = union(enum) {
	1130	storage: RequestOptions.StorageStrategy,
	1131	file: std.fs.File,
	1132	none,
	1133	};
	1134
	1135	header_strategy: RequestOptions.StorageStrategy = .{ .dynamic = 16 * 1024 },
	1136	response_strategy: ResponseStrategy = .{ .storage = .{ .dynamic = 16 * 1024 * 1024 } },
	1137
	1138	location: Location,
	1139	method: http.Method = .GET,
	1140	headers: http.Headers = http.Headers{ .allocator = std.heap.page_allocator, .owned = false },
	1141	payload: Payload = .none,
	1142	};
	1143
	1144	pub const FetchResult = struct {
	1145	status: http.Status,
	1146	body: ?[]const u8 = null,
	1147	headers: http.Headers,
	1148
	1149	allocator: Allocator,
	1150	options: FetchOptions,
	1151
	1152	pub fn deinit(res: *FetchResult) void {
	1153	if (res.options.response_strategy == .storage and res.options.response_strategy.storage == .dynamic) {
	1154	if (res.body) \|body\| res.allocator.free(body);
	1155	}
	1156
	1157	res.headers.deinit();
	1158	}
	1159	};
	1160
	1161	pub fn fetch(client: *Client, allocator: Allocator, options: FetchOptions) !FetchResult {
	1162	const has_transfer_encoding = options.headers.contains("transfer-encoding");
	1163	const has_content_length = options.headers.contains("content-length");
	1164
	1165	if (has_content_length or has_transfer_encoding) return error.UnsupportedHeader;
	1166
	1167	const uri = switch (options.location) {
	1168	.url => \|u\| try Uri.parse(u),
	1169	.uri => \|u\| u,
	1170	};
	1171
	1172	var req = try request(client, options.method, uri, options.headers, .{
	1173	.header_strategy = options.header_strategy,
	1174	.handle_redirects = options.payload == .none,
	1175	});
	1176	defer req.deinit();
	1177
	1178	{ // Block to maintain lock of file to attempt to prevent a race condition where another process modifies the file while we are reading it.
	1179	// This relies on other processes actually obeying the advisory lock, which is not guaranteed.
	1180	if (options.payload == .file) try options.payload.file.lock(.shared);
	1181	defer if (options.payload == .file) options.payload.file.unlock();
	1182
	1183	switch (options.payload) {
	1184	.string => \|str\| req.transfer_encoding = .{ .content_length = str.len },
	1185	.file => \|file\| req.transfer_encoding = .{ .content_length = (try file.stat()).size },
	1186	.none => {},
	1187	}
	1188
	1189	try req.start();
	1190
	1191	switch (options.payload) {
	1192	.string => \|str\| try req.writeAll(str),
	1193	.file => \|file\| {
	1194	try file.seekTo(0);
	1195	var fifo = std.fifo.LinearFifo(u8, .{ .Static = 8192 }).init();
	1196	try fifo.pump(file.reader(), req.writer());
	1197	},
	1198	.none => {},
	1199	}
	1200
	1201	try req.finish();
	1202	}
	1203
	1204	try req.wait();
	1205
	1206	var res = FetchResult{
	1207	.status = req.response.status,
	1208	.headers = try req.response.headers.clone(allocator),
	1209
	1210	.allocator = allocator,
	1211	.options = options,
	1212	};
	1213
	1214	switch (options.response_strategy) {
	1215	.storage => \|storage\| switch (storage) {
	1216	.dynamic => \|max\| res.body = try req.reader().readAllAlloc(allocator, max),
	1217	.static => \|buf\| res.body = buf[0..try req.reader().readAll(buf)],
	1218	},
	1219	.file => \|file\| {
	1220	var fifo = std.fifo.LinearFifo(u8, .{ .Static = 8192 }).init();
	1221	try fifo.pump(req.reader(), file.writer());
	1222	},
	1223	.none => { // Take advantage of request internals to discard the response body and make the connection available for another request.
	1224	req.response.skip = true;
	1225
	1226	const empty = @as([*]u8, undefined)[0..0];
	1227	assert(try req.transferRead(empty) == 0); // we're skipping, no buffer is necessary
	1228	},
	1229	}
	1230
	1231	return res;
	1232	}
	1233
1042	1234	test {
1043	1235	const builtin = @import("builtin");
1044	1236	const native_endian = comptime builtin.cpu.arch.endian();

lib/std/http/Headers.zig+26-1

...	...	@@ -57,6 +57,18 @@ pub const Headers = struct {
57	57	return .{ .allocator = allocator };
58	58	}
59	59
	60	pub fn initList(allocator: Allocator, list: []const Field) Headers {
	61	var new = Headers.init(allocator);
	62
	63	try new.list.ensureTotalCapacity(allocator, list.len);
	64	try new.index.ensureTotalCapacity(allocator, list.len);
	65	for (list) \|field\| {
	66	try new.append(field.name, field.value);
	67	}
	68
	69	return new;
	70	}
	71
60	72	pub fn deinit(headers: *Headers) void {
61	73	headers.deallocateIndexListsAndFields();
62	74	headers.index.deinit(headers.allocator);
...	...	@@ -78,7 +90,7 @@ pub const Headers = struct {
78	90	entry.name = kv.key_ptr.*;
79	91	try kv.value_ptr.append(headers.allocator, n);
80	92	} else {
81		const name_duped = if (headers.owned) try headers.allocator.dupe(u8, name) else name;
	93	const name_duped = if (headers.owned) try std.ascii.allocLowerString(headers.allocator, name) else name;
82	94	errdefer if (headers.owned) headers.allocator.free(name_duped);
83	95
84	96	entry.name = name_duped;
...	...	@@ -97,6 +109,7 @@ pub const Headers = struct {
97	109	return headers.index.contains(name);
98	110	}
99	111
	112	/// Removes all headers with the given name.
100	113	pub fn delete(headers: *Headers, name: []const u8) bool {
101	114	if (headers.index.fetchRemove(name)) \|kv\| {
102	115	var index = kv.value;
...	...	@@ -268,6 +281,18 @@ pub const Headers = struct {
268	281	headers.index.clearRetainingCapacity();
269	282	headers.list.clearRetainingCapacity();
270	283	}
	284
	285	pub fn clone(headers: Headers, allocator: Allocator) !Headers {
	286	var new = Headers.init(allocator);
	287
	288	try new.list.ensureTotalCapacity(allocator, headers.list.capacity);
	289	try new.index.ensureTotalCapacity(allocator, headers.index.capacity());
	290	for (headers.list.items) \|field\| {
	291	try new.append(field.name, field.value);
	292	}
	293
	294	return new;
	295	}
271	296	};
272	297
273	298	test "Headers.append" {

lib/std/http/Server.zig+46-36

...	...	@@ -185,8 +185,10 @@ pub const Request = struct {
185	185	return error.HttpHeadersInvalid;
186	186
187	187	const method_end = mem.indexOfScalar(u8, first_line, ' ') orelse return error.HttpHeadersInvalid;
	188	if (method_end > 24) return error.HttpHeadersInvalid;
	189
188	190	const method_str = first_line[0..method_end];
189		const method = std.meta.stringToEnum(http.Method, method_str) orelse return error.UnknownHttpMethod;
	191	const method: http.Method = @enumFromInt(http.Method.parse(method_str));
190	192
191	193	const version_start = mem.lastIndexOfScalar(u8, first_line, ' ') orelse return error.HttpHeadersInvalid;
192	194	if (version_start == method_end) return error.HttpHeadersInvalid;
...	...	@@ -411,59 +413,67 @@ pub const Response = struct {
411	413	}
412	414	try w.writeAll("\r\n");
413	415
414		if (!res.headers.contains("server")) {
415		try w.writeAll("Server: zig (std.http)\r\n");
416		}
	416	if (res.status == .@"continue") {
	417	res.state = .waited; // we still need to send another request after this
	418	} else {
	419	if (!res.headers.contains("server")) {
	420	try w.writeAll("Server: zig (std.http)\r\n");
	421	}
417	422
418		if (!res.headers.contains("connection")) {
419		const req_connection = res.request.headers.getFirstValue("connection");
420		const req_keepalive = req_connection != null and !std.ascii.eqlIgnoreCase("close", req_connection.?);
	423	if (!res.headers.contains("connection")) {
	424	const req_connection = res.request.headers.getFirstValue("connection");
	425	const req_keepalive = req_connection != null and !std.ascii.eqlIgnoreCase("close", req_connection.?);
421	426
422		if (req_keepalive) {
423		try w.writeAll("Connection: keep-alive\r\n");
424		} else {
425		try w.writeAll("Connection: close\r\n");
	427	if (req_keepalive) {
	428	try w.writeAll("Connection: keep-alive\r\n");
	429	} else {
	430	try w.writeAll("Connection: close\r\n");
	431	}
426	432	}
427		}
428	433
429		const has_transfer_encoding = res.headers.contains("transfer-encoding");
430		const has_content_length = res.headers.contains("content-length");
	434	const has_transfer_encoding = res.headers.contains("transfer-encoding");
	435	const has_content_length = res.headers.contains("content-length");
431	436
432		if (!has_transfer_encoding and !has_content_length) {
433		switch (res.transfer_encoding) {
434		.chunked => try w.writeAll("Transfer-Encoding: chunked\r\n"),
435		.content_length => \|content_length\| try w.print("Content-Length: {d}\r\n", .{content_length}),
436		.none => {},
437		}
438		} else {
439		if (has_content_length) {
440		const content_length = std.fmt.parseInt(u64, res.headers.getFirstValue("content-length").?, 10) catch return error.InvalidContentLength;
441
442		res.transfer_encoding = .{ .content_length = content_length };
443		} else if (has_transfer_encoding) {
444		const transfer_encoding = res.headers.getFirstValue("transfer-encoding").?;
445		if (std.mem.eql(u8, transfer_encoding, "chunked")) {
446		res.transfer_encoding = .chunked;
447		} else {
448		return error.UnsupportedTransferEncoding;
	437	if (!has_transfer_encoding and !has_content_length) {
	438	switch (res.transfer_encoding) {
	439	.chunked => try w.writeAll("Transfer-Encoding: chunked\r\n"),
	440	.content_length => \|content_length\| try w.print("Content-Length: {d}\r\n", .{content_length}),
	441	.none => {},
449	442	}
450	443	} else {
451		res.transfer_encoding = .none;
	444	if (has_content_length) {
	445	const content_length = std.fmt.parseInt(u64, res.headers.getFirstValue("content-length").?, 10) catch return error.InvalidContentLength;
	446
	447	res.transfer_encoding = .{ .content_length = content_length };
	448	} else if (has_transfer_encoding) {
	449	const transfer_encoding = res.headers.getFirstValue("transfer-encoding").?;
	450	if (std.mem.eql(u8, transfer_encoding, "chunked")) {
	451	res.transfer_encoding = .chunked;
	452	} else {
	453	return error.UnsupportedTransferEncoding;
	454	}
	455	} else {
	456	res.transfer_encoding = .none;
	457	}
452	458	}
	459
	460	try w.print("{}", .{res.headers});
453	461	}
454	462
455		try w.print("{}", .{res.headers});
	463	if (res.request.method == .HEAD) {
	464	res.transfer_encoding = .none;
	465	}
456	466
457	467	try w.writeAll("\r\n");
458	468
459	469	try buffered.flush();
460	470	}
461	471
462		pub const TransferReadError = Connection.ReadError \|\| proto.HeadersParser.ReadError;
	472	const TransferReadError = Connection.ReadError \|\| proto.HeadersParser.ReadError;
463	473
464		pub const TransferReader = std.io.Reader(*Response, TransferReadError, transferRead);
	474	const TransferReader = std.io.Reader(*Response, TransferReadError, transferRead);
465	475
466		pub fn transferReader(res: *Response) TransferReader {
	476	fn transferReader(res: *Response) TransferReader {
467	477	return .{ .context = res };
468	478	}
469	479

lib/std/http/protocol.zig+6-3

...	...	@@ -534,9 +534,9 @@ pub const HeadersParser = struct {
534	534
535	535	if (r.next_chunk_length == 0) r.done = true;
536	536
537		return 0;
538		} else {
539		const out_avail = buffer.len;
	537	return out_index;
	538	} else if (out_index < buffer.len) {
	539	const out_avail = buffer.len - out_index;
540	540
541	541	const can_read = @as(usize, @intCast(@min(data_avail, out_avail)));
542	542	const nread = try conn.read(buffer[0..can_read]);
...	...	@@ -545,6 +545,8 @@ pub const HeadersParser = struct {
545	545	if (r.next_chunk_length == 0) r.done = true;
546	546
547	547	return nread;
	548	} else {
	549	return out_index;
548	550	}
549	551	},
550	552	.chunk_data_suffix, .chunk_data_suffix_r, .chunk_head_size, .chunk_head_ext, .chunk_head_r => {
...	...	@@ -558,6 +560,7 @@ pub const HeadersParser = struct {
558	560	.chunk_data => if (r.next_chunk_length == 0) {
559	561	if (std.mem.eql(u8, conn.peek(), "\r\n")) {
560	562	r.state = .finished;
	563	r.done = true;
561	564	} else {
562	565	// The trailer section is formatted identically to the header section.
563	566	r.state = .seen_rn;

test/standalone/http.zig+94-3

...	...	@@ -20,7 +20,19 @@ var server: Server = undefined;
20	20	fn handleRequest(res: *Server.Response) !void {
21	21	const log = std.log.scoped(.server);
22	22
23		log.info("{s} {s} {s}", .{ @tagName(res.request.method), @tagName(res.request.version), res.request.target });
	23	log.info("{} {s} {s}", .{ res.request.method, @tagName(res.request.version), res.request.target });
	24
	25	if (res.request.headers.contains("expect")) {
	26	if (mem.eql(u8, res.request.headers.getFirstValue("expect").?, "100-continue")) {
	27	res.status = .@"continue";
	28	try res.do();
	29	res.status = .ok;
	30	} else {
	31	res.status = .expectation_failed;
	32	try res.do();
	33	return;
	34	}
	35	}
24	36
25	37	const body = try res.reader().readAllAlloc(salloc, 8192);
26	38	defer salloc.free(body);
...	...	@@ -43,6 +55,8 @@ fn handleRequest(res: *Server.Response) !void {
43	55	try res.writeAll("Hello, ");
44	56	try res.writeAll("World!\n");
45	57	try res.finish();
	58	} else {
	59	try testing.expectEqual(res.writeAll("errors"), error.NotWriteable);
46	60	}
47	61	} else if (mem.startsWith(u8, res.request.target, "/large")) {
48	62	res.transfer_encoding = .{ .content_length = 14 * 1024 + 14 * 10 };
...	...	@@ -62,7 +76,7 @@ fn handleRequest(res: *Server.Response) !void {
62	76	}
63	77
64	78	try res.finish();
65		} else if (mem.eql(u8, res.request.target, "/echo-content")) {
	79	} else if (mem.startsWith(u8, res.request.target, "/echo-content")) {
66	80	try testing.expectEqualStrings("Hello, World!\n", body);
67	81	try testing.expectEqualStrings("text/plain", res.request.headers.getFirstValue("content-type").?);
68	82
...	...	@@ -571,7 +585,84 @@ pub fn main() !void {
571	585	// connection has been kept alive
572	586	try testing.expect(client.connection_pool.free_len == 1);
573	587
574		{ // issue 16282
	588	{ // Client.fetch()
	589	var h = http.Headers{ .allocator = calloc };
	590	defer h.deinit();
	591
	592	try h.append("content-type", "text/plain");
	593
	594	const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/echo-content#fetch", .{port});
	595	defer calloc.free(location);
	596
	597	log.info("{s}", .{location});
	598	var res = try client.fetch(calloc, .{
	599	.location = .{ .url = location },
	600	.method = .POST,
	601	.headers = h,
	602	.payload = .{ .string = "Hello, World!\n" },
	603	});
	604	defer res.deinit();
	605
	606	try testing.expectEqualStrings("Hello, World!\n", res.body.?);
	607	}
	608
	609	{ // expect: 100-continue
	610	var h = http.Headers{ .allocator = calloc };
	611	defer h.deinit();
	612
	613	try h.append("expect", "100-continue");
	614	try h.append("content-type", "text/plain");
	615
	616	const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/echo-content#expect-100", .{port});
	617	defer calloc.free(location);
	618	const uri = try std.Uri.parse(location);
	619
	620	log.info("{s}", .{location});
	621	var req = try client.request(.POST, uri, h, .{});
	622	defer req.deinit();
	623
	624	req.transfer_encoding = .chunked;
	625
	626	try req.start();
	627	try req.wait();
	628	try testing.expectEqual(http.Status.@"continue", req.response.status);
	629
	630	try req.writeAll("Hello, ");
	631	try req.writeAll("World!\n");
	632	try req.finish();
	633
	634	try req.wait();
	635	try testing.expectEqual(http.Status.ok, req.response.status);
	636
	637	const body = try req.reader().readAllAlloc(calloc, 8192);
	638	defer calloc.free(body);
	639
	640	try testing.expectEqualStrings("Hello, World!\n", body);
	641	}
	642
	643	{ // expect: garbage
	644	var h = http.Headers{ .allocator = calloc };
	645	defer h.deinit();
	646
	647	try h.append("content-type", "text/plain");
	648	try h.append("expect", "garbage");
	649
	650	const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/echo-content#expect-garbage", .{port});
	651	defer calloc.free(location);
	652	const uri = try std.Uri.parse(location);
	653
	654	log.info("{s}", .{location});
	655	var req = try client.request(.POST, uri, h, .{});
	656	defer req.deinit();
	657
	658	req.transfer_encoding = .chunked;
	659
	660	try req.start();
	661	try req.wait();
	662	try testing.expectEqual(http.Status.expectation_failed, req.response.status);
	663	}
	664
	665	{ // issue 16282 * This test leaves the client in an invalid state, it must be last *
575	666	const location = try std.fmt.allocPrint(calloc, "http://127.0.0.1:{d}/get", .{port});
576	667	defer calloc.free(location);
577	668	const uri = try std.Uri.parse(location);