authorgravatar for truemedian@gmail.comNameless <truemedian@gmail.com> 2023-10-02 19:57:43-05:00
committergravatar for truemedian@gmail.comNameless <truemedian@gmail.com> 2023-10-21 20:52:58-05:00
log1afeada2d95e50efe651bd6227719ca4003dad96
tree0799938787b302420653064771942bb589546b36
parent7d50634e0ad4355e339bc243a2e2842693e133f9
signature Commit is signed but in an unrecognized format.

std.http.Client: enhance proxy support

adds connectTunnel to form a HTTP CONNECT tunnel to the desired host. Primarily implemented for proxies, but like connectUnix may be called by any user. adds loadDefaultProxies to load proxy information from common environment variables (http_proxy, HTTP_PROXY, https_proxy, HTTPS_PROXY, all_proxy, ALL_PROXY). - no_proxy and NO_PROXY are currently unsupported. splits proxy into http_proxy and https_proxy, adds headers field for arbitrary headers to each proxy.

3 files changed, 357 insertions(+), 115 deletions(-)

lib/std/Uri.zig+79-33
...@@ -208,24 +208,45 @@ pub fn parseWithoutScheme(text: []const u8) ParseError!Uri {...@@ -208,24 +208,45 @@ pub fn parseWithoutScheme(text: []const u8) ParseError!Uri {
208 return uri;208 return uri;
209}209}
210210
211pub fn format(211pub const WriteToStreamOptions = struct {
212 /// When true, include the scheme part of the URI.
213 scheme: bool = false,
214
215 /// When true, include the user and password part of the URI. Ignored if `authority` is false.
216 authentication: bool = false,
217
218 /// When true, include the authority part of the URI.
219 authority: bool = false,
220
221 /// When true, include the path part of the URI.
222 path: bool = false,
223
224 /// When true, include the query part of the URI. Ignored when `path` is false.
225 query: bool = false,
226
227 /// When true, include the fragment part of the URI. Ignored when `path` is false.
228 fragment: bool = false,
229
230 /// When true, do not escape any part of the URI.
231 raw: bool = false,
232};
233
234pub fn writeToStream(
212 uri: Uri,235 uri: Uri,
213 comptime fmt: []const u8,236 options: WriteToStreamOptions,
214 options: std.fmt.FormatOptions,
215 writer: anytype,237 writer: anytype,
216) @TypeOf(writer).Error!void {238) @TypeOf(writer).Error!void {
217 _ = options;239 if (options.scheme) {
218
219 const needs_absolute = comptime std.mem.indexOf(u8, fmt, "+") != null;
220 const needs_path = comptime std.mem.indexOf(u8, fmt, "/") != null or fmt.len == 0;
221 const raw_uri = comptime std.mem.indexOf(u8, fmt, "r") != null;
222 const needs_fragment = comptime std.mem.indexOf(u8, fmt, "#") != null;
223
224 if (needs_absolute) {
225 try writer.writeAll(uri.scheme);240 try writer.writeAll(uri.scheme);
226 try writer.writeAll(":");241 try writer.writeAll(":");
227 if (uri.host) |host| {242
243 if (options.authority and uri.host != null) {
228 try writer.writeAll("//");244 try writer.writeAll("//");
245 }
246 }
247
248 if (options.authority) {
249 if (options.authentication and uri.host != null) {
229 if (uri.user) |user| {250 if (uri.user) |user| {
230 try writer.writeAll(user);251 try writer.writeAll(user);
231 if (uri.password) |password| {252 if (uri.password) |password| {
...@@ -234,7 +255,9 @@ pub fn format(...@@ -234,7 +255,9 @@ pub fn format(
234 }255 }
235 try writer.writeAll("@");256 try writer.writeAll("@");
236 }257 }
258 }
237259
260 if (uri.host) |host| {
238 try writer.writeAll(host);261 try writer.writeAll(host);
239262
240 if (uri.port) |port| {263 if (uri.port) |port| {
...@@ -244,39 +267,62 @@ pub fn format(...@@ -244,39 +267,62 @@ pub fn format(
244 }267 }
245 }268 }
246269
247 if (needs_path) {270 if (options.path) {
248 if (uri.path.len == 0) {271 if (uri.path.len == 0) {
249 try writer.writeAll("/");272 try writer.writeAll("/");
273 } else if (options.raw) {
274 try writer.writeAll(uri.path);
250 } else {275 } else {
251 if (raw_uri) {276 try writeEscapedPath(writer, uri.path);
252 try writer.writeAll(uri.path);
253 } else {
254 try Uri.writeEscapedPath(writer, uri.path);
255 }
256 }277 }
257278
258 if (uri.query) |q| {279 if (options.query) if (uri.query) |q| {
259 try writer.writeAll("?");280 try writer.writeAll("?");
260 if (raw_uri) {281 if (options.raw) {
261 try writer.writeAll(q);282 try writer.writeAll(q);
262 } else {283 } else {
263 try Uri.writeEscapedQuery(writer, q);284 try writeEscapedQuery(writer, q);
264 }285 }
265 }286 };
266287
267 if (needs_fragment) {288 if (options.fragment) if (uri.fragment) |f| {
268 if (uri.fragment) |f| {289 try writer.writeAll("#");
269 try writer.writeAll("#");290 if (options.raw) {
270 if (raw_uri) {291 try writer.writeAll(f);
271 try writer.writeAll(f);292 } else {
272 } else {293 try writeEscapedQuery(writer, f);
273 try Uri.writeEscapedQuery(writer, f);
274 }
275 }294 }
276 }295 };
277 }296 }
278}297}
279298
299pub fn format(
300 uri: Uri,
301 comptime fmt: []const u8,
302 options: std.fmt.FormatOptions,
303 writer: anytype,
304) @TypeOf(writer).Error!void {
305 _ = options;
306
307 const scheme = comptime std.mem.indexOf(u8, fmt, ":") != null or fmt.len == 0;
308 const authentication = comptime std.mem.indexOf(u8, fmt, "@") != null or fmt.len == 0;
309 const authority = comptime std.mem.indexOf(u8, fmt, "+") != null or fmt.len == 0;
310 const path = comptime std.mem.indexOf(u8, fmt, "/") != null or fmt.len == 0;
311 const query = comptime std.mem.indexOf(u8, fmt, "?") != null or fmt.len == 0;
312 const fragment = comptime std.mem.indexOf(u8, fmt, "#") != null or fmt.len == 0;
313 const raw = comptime std.mem.indexOf(u8, fmt, "r") != null or fmt.len == 0;
314
315 return writeToStream(uri, .{
316 .scheme = scheme,
317 .authentication = authentication,
318 .authority = authority,
319 .path = path,
320 .query = query,
321 .fragment = fragment,
322 .raw = raw,
323 }, writer);
324}
325
280/// Parses the URI or returns an error.326/// Parses the URI or returns an error.
281/// The return value will contain unescaped strings pointing into the327/// The return value will contain unescaped strings pointing into the
282/// original `text`. Each component that is provided, will be non-`null`.328/// original `text`. Each component that is provided, will be non-`null`.
...@@ -709,7 +755,7 @@ test "URI query escaping" {...@@ -709,7 +755,7 @@ test "URI query escaping" {
709 const parsed = try Uri.parse(address);755 const parsed = try Uri.parse(address);
710756
711 // format the URI to escape it757 // format the URI to escape it
712 const formatted_uri = try std.fmt.allocPrint(std.testing.allocator, "{}", .{parsed});758 const formatted_uri = try std.fmt.allocPrint(std.testing.allocator, "{/?}", .{parsed});
713 defer std.testing.allocator.free(formatted_uri);759 defer std.testing.allocator.free(formatted_uri);
714 try std.testing.expectEqualStrings("/?response-content-type=application%2Foctet-stream", formatted_uri);760 try std.testing.expectEqualStrings("/?response-content-type=application%2Foctet-stream", formatted_uri);
715}761}
...@@ -727,6 +773,6 @@ test "format" {...@@ -727,6 +773,6 @@ test "format" {
727 };773 };
728 var buf = std.ArrayList(u8).init(std.testing.allocator);774 var buf = std.ArrayList(u8).init(std.testing.allocator);
729 defer buf.deinit();775 defer buf.deinit();
730 try uri.format("+/", .{}, buf.writer());776 try uri.format(":/?#", .{}, buf.writer());
731 try std.testing.expectEqualSlices(u8, "file:/foo/bar/baz", buf.items);777 try std.testing.expectEqualSlices(u8, "file:/foo/bar/baz", buf.items);
732}778}
lib/std/http/Client.zig+258-68
...@@ -18,6 +18,7 @@ pub const connection_pool_size = std.options.http_connection_pool_size;...@@ -18,6 +18,7 @@ pub const connection_pool_size = std.options.http_connection_pool_size;
18allocator: Allocator,18allocator: Allocator,
19ca_bundle: std.crypto.Certificate.Bundle = .{},19ca_bundle: std.crypto.Certificate.Bundle = .{},
20ca_bundle_mutex: std.Thread.Mutex = .{},20ca_bundle_mutex: std.Thread.Mutex = .{},
21
21/// When this is `true`, the next time this client performs an HTTPS request,22/// When this is `true`, the next time this client performs an HTTPS request,
22/// it will first rescan the system for root certificates.23/// it will first rescan the system for root certificates.
23next_https_rescan_certs: bool = true,24next_https_rescan_certs: bool = true,
...@@ -25,7 +26,11 @@ next_https_rescan_certs: bool = true,...@@ -25,7 +26,11 @@ next_https_rescan_certs: bool = true,
25/// The pool of connections that can be reused (and currently in use).26/// The pool of connections that can be reused (and currently in use).
26connection_pool: ConnectionPool = .{},27connection_pool: ConnectionPool = .{},
2728
28proxy: ?HttpProxy = null,29/// This is the proxy that will handle http:// connections. It *must not* be modified when the client has any active connections.
30http_proxy: ?ProxyInformation = null,
31
32/// This is the proxy that will handle https:// connections. It *must not* be modified when the client has any active connections.
33https_proxy: ?ProxyInformation = null,
2934
30/// A set of linked lists of connections that can be reused.35/// A set of linked lists of connections that can be reused.
31pub const ConnectionPool = struct {36pub const ConnectionPool = struct {
...@@ -33,7 +38,7 @@ pub const ConnectionPool = struct {...@@ -33,7 +38,7 @@ pub const ConnectionPool = struct {
33 pub const Criteria = struct {38 pub const Criteria = struct {
34 host: []const u8,39 host: []const u8,
35 port: u16,40 port: u16,
36 is_tls: bool,41 protocol: Connection.Protocol,
37 };42 };
3843
39 const Queue = std.DoublyLinkedList(Connection);44 const Queue = std.DoublyLinkedList(Connection);
...@@ -55,9 +60,9 @@ pub const ConnectionPool = struct {...@@ -55,9 +60,9 @@ pub const ConnectionPool = struct {
5560
56 var next = pool.free.last;61 var next = pool.free.last;
57 while (next) |node| : (next = node.prev) {62 while (next) |node| : (next = node.prev) {
58 if ((node.data.protocol == .tls) != criteria.is_tls) continue;63 if (node.data.protocol != criteria.protocol) continue;
59 if (node.data.port != criteria.port) continue;64 if (node.data.port != criteria.port) continue;
60 if (!mem.eql(u8, node.data.host, criteria.host)) continue;65 if (!std.ascii.eqlIgnoreCase(node.data.host, criteria.host)) continue;
6166
62 pool.acquireUnsafe(node);67 pool.acquireUnsafe(node);
63 return node;68 return node;
...@@ -84,23 +89,23 @@ pub const ConnectionPool = struct {...@@ -84,23 +89,23 @@ pub const ConnectionPool = struct {
8489
85 /// Tries to release a connection back to the connection pool. This function is threadsafe.90 /// Tries to release a connection back to the connection pool. This function is threadsafe.
86 /// If the connection is marked as closing, it will be closed instead.91 /// If the connection is marked as closing, it will be closed instead.
87 pub fn release(pool: *ConnectionPool, client: *Client, node: *Node) void {92 pub fn release(pool: *ConnectionPool, allocator: Allocator, node: *Node) void {
88 pool.mutex.lock();93 pool.mutex.lock();
89 defer pool.mutex.unlock();94 defer pool.mutex.unlock();
9095
91 pool.used.remove(node);96 pool.used.remove(node);
9297
93 if (node.data.closing) {98 if (node.data.closing or pool.free_size == 0) {
94 node.data.deinit(client);99 node.data.close(allocator);
95 return client.allocator.destroy(node);100 return allocator.destroy(node);
96 }101 }
97102
98 if (pool.free_len >= pool.free_size) {103 if (pool.free_len >= pool.free_size) {
99 const popped = pool.free.popFirst() orelse unreachable;104 const popped = pool.free.popFirst() orelse unreachable;
100 pool.free_len -= 1;105 pool.free_len -= 1;
101106
102 popped.data.deinit(client);107 popped.data.close(allocator);
103 client.allocator.destroy(popped);108 allocator.destroy(popped);
104 }109 }
105110
106 if (node.data.proxied) {111 if (node.data.proxied) {
...@@ -128,7 +133,7 @@ pub const ConnectionPool = struct {...@@ -128,7 +133,7 @@ pub const ConnectionPool = struct {
128 defer client.allocator.destroy(node);133 defer client.allocator.destroy(node);
129 next = node.next;134 next = node.next;
130135
131 node.data.deinit(client);136 node.data.close(client.allocator);
132 }137 }
133138
134 next = pool.used.first;139 next = pool.used.first;
...@@ -136,7 +141,7 @@ pub const ConnectionPool = struct {...@@ -136,7 +141,7 @@ pub const ConnectionPool = struct {
136 defer client.allocator.destroy(node);141 defer client.allocator.destroy(node);
137 next = node.next;142 next = node.next;
138143
139 node.data.deinit(client);144 node.data.close(client.allocator);
140 }145 }
141146
142 pool.* = undefined;147 pool.* = undefined;
...@@ -283,19 +288,15 @@ pub const Connection = struct {...@@ -283,19 +288,15 @@ pub const Connection = struct {
283 return Writer{ .context = conn };288 return Writer{ .context = conn };
284 }289 }
285290
286 pub fn close(conn: *Connection, client: *const Client) void {291 pub fn close(conn: *Connection, allocator: Allocator) void {
287 if (conn.protocol == .tls) {292 if (conn.protocol == .tls) {
288 // try to cleanly close the TLS connection, for any server that cares.293 // try to cleanly close the TLS connection, for any server that cares.
289 _ = conn.tls_client.writeEnd(conn.stream, "", true) catch {};294 _ = conn.tls_client.writeEnd(conn.stream, "", true) catch {};
290 client.allocator.destroy(conn.tls_client);295 allocator.destroy(conn.tls_client);
291 }296 }
292297
293 conn.stream.close();298 conn.stream.close();
294 }299 allocator.free(conn.host);
295
296 pub fn deinit(conn: *Connection, client: *const Client) void {
297 conn.close(client);
298 client.allocator.free(conn.host);
299 }300 }
300};301};
301302
...@@ -490,7 +491,7 @@ pub const Request = struct {...@@ -490,7 +491,7 @@ pub const Request = struct {
490 // If the response wasn't fully read, then we need to close the connection.491 // If the response wasn't fully read, then we need to close the connection.
491 connection.data.closing = true;492 connection.data.closing = true;
492 }493 }
493 req.client.connection_pool.release(req.client, connection);494 req.client.connection_pool.release(req.client.allocator, connection);
494 }495 }
495496
496 req.arena.deinit();497 req.arena.deinit();
...@@ -509,7 +510,7 @@ pub const Request = struct {...@@ -509,7 +510,7 @@ pub const Request = struct {
509 .zstd => |*zstd| zstd.deinit(),510 .zstd => |*zstd| zstd.deinit(),
510 }511 }
511512
512 req.client.connection_pool.release(req.client, req.connection.?);513 req.client.connection_pool.release(req.client.allocator, req.connection.?);
513 req.connection = null;514 req.connection = null;
514515
515 const protocol = protocol_map.get(uri.scheme) orelse return error.UnsupportedUrlScheme;516 const protocol = protocol_map.get(uri.scheme) orelse return error.UnsupportedUrlScheme;
...@@ -554,24 +555,16 @@ pub const Request = struct {...@@ -554,24 +555,16 @@ pub const Request = struct {
554 try w.writeByte(' ');555 try w.writeByte(' ');
555556
556 if (req.method == .CONNECT) {557 if (req.method == .CONNECT) {
557 try w.writeAll(req.uri.host.?);558 try req.uri.writeToStream(.{ .authority = true }, w);
558 try w.writeByte(':');
559 try w.print("{}", .{req.uri.port.?});
560 } else {559 } else {
561 if (req.connection.?.data.proxied) {560 try req.uri.writeToStream(.{
562 // proxied connections require the full uri561 .scheme = req.connection.?.data.proxied,
563 if (options.raw_uri) {562 .authentication = req.connection.?.data.proxied,
564 try w.print("{+/r}", .{req.uri});563 .authority = req.connection.?.data.proxied,
565 } else {564 .path = true,
566 try w.print("{+/}", .{req.uri});565 .query = true,
567 }566 .raw = options.raw_uri,
568 } else {567 }, w);
569 if (options.raw_uri) {
570 try w.print("{/r}", .{req.uri});
571 } else {
572 try w.print("{/}", .{req.uri});
573 }
574 }
575 }568 }
576 try w.writeByte(' ');569 try w.writeByte(' ');
577 try w.writeAll(@tagName(req.version));570 try w.writeAll(@tagName(req.version));
...@@ -579,7 +572,7 @@ pub const Request = struct {...@@ -579,7 +572,7 @@ pub const Request = struct {
579572
580 if (!req.headers.contains("host")) {573 if (!req.headers.contains("host")) {
581 try w.writeAll("Host: ");574 try w.writeAll("Host: ");
582 try w.writeAll(req.uri.host.?);575 try req.uri.writeToStream(.{ .authority = true }, w);
583 try w.writeAll("\r\n");576 try w.writeAll("\r\n");
584 }577 }
585578
...@@ -636,6 +629,24 @@ pub const Request = struct {...@@ -636,6 +629,24 @@ pub const Request = struct {
636 try w.writeAll("\r\n");629 try w.writeAll("\r\n");
637 }630 }
638631
632 if (req.connection.?.data.proxied) {
633 const proxy_headers: ?http.Headers = switch (req.connection.?.data.protocol) {
634 .plain => if (req.client.http_proxy) |proxy| proxy.headers else null,
635 .tls => if (req.client.https_proxy) |proxy| proxy.headers else null,
636 };
637
638 if (proxy_headers) |headers| {
639 for (headers.list.items) |entry| {
640 if (entry.value.len == 0) continue;
641
642 try w.writeAll(entry.name);
643 try w.writeAll(": ");
644 try w.writeAll(entry.value);
645 try w.writeAll("\r\n");
646 }
647 }
648 }
649
639 try w.writeAll("\r\n");650 try w.writeAll("\r\n");
640651
641 try buffered.flush();652 try buffered.flush();
...@@ -893,18 +904,15 @@ pub const Request = struct {...@@ -893,18 +904,15 @@ pub const Request = struct {
893 }904 }
894};905};
895906
896pub const HttpProxy = struct {907pub const ProxyInformation = struct {
897 pub const ProxyAuthentication = union(enum) {908 allocator: Allocator,
898 basic: []const u8,909 headers: http.Headers,
899 custom: []const u8,
900 };
901910
902 protocol: Connection.Protocol,911 protocol: Connection.Protocol,
903 host: []const u8,912 host: []const u8,
904 port: ?u16 = null,913 port: u16,
905914
906 /// The value for the Proxy-Authorization header.915 supports_connect: bool = true,
907 auth: ?ProxyAuthentication = null,
908};916};
909917
910/// Release all associated resources with the client.918/// Release all associated resources with the client.
...@@ -912,19 +920,115 @@ pub const HttpProxy = struct {...@@ -912,19 +920,115 @@ pub const HttpProxy = struct {
912pub fn deinit(client: *Client) void {920pub fn deinit(client: *Client) void {
913 client.connection_pool.deinit(client);921 client.connection_pool.deinit(client);
914922
923 if (client.http_proxy) |*proxy| {
924 proxy.allocator.free(proxy.host);
925 proxy.headers.deinit();
926 }
927
928 if (client.https_proxy) |*proxy| {
929 proxy.allocator.free(proxy.host);
930 proxy.headers.deinit();
931 }
932
915 client.ca_bundle.deinit(client.allocator);933 client.ca_bundle.deinit(client.allocator);
916 client.* = undefined;934 client.* = undefined;
917}935}
918936
919pub const ConnectUnproxiedError = Allocator.Error || error{ ConnectionRefused, NetworkUnreachable, ConnectionTimedOut, ConnectionResetByPeer, TemporaryNameServerFailure, NameServerFailure, UnknownHostName, HostLacksNetworkAddresses, UnexpectedConnectFailure, TlsInitializationFailed };937/// Uses the *_proxy environment variable to set any unset proxies for the client.
938/// This function *must not* be called when the client has any active connections.
939pub fn loadDefaultProxies(client: *Client) !void {
940 if (client.http_proxy == null) http: {
941 const content: []const u8 = if (std.process.hasEnvVarConstant("http_proxy"))
942 try std.process.getEnvVarOwned(client.allocator, "http_proxy")
943 else if (std.process.hasEnvVarConstant("HTTP_PROXY"))
944 try std.process.getEnvVarOwned(client.allocator, "HTTP_PROXY")
945 else if (std.process.hasEnvVarConstant("all_proxy"))
946 try std.process.getEnvVarOwned(client.allocator, "all_proxy")
947 else if (std.process.hasEnvVarConstant("ALL_PROXY"))
948 try std.process.getEnvVarOwned(client.allocator, "ALL_PROXY")
949 else
950 break :http;
951 defer client.allocator.free(content);
952
953 const uri = try Uri.parse(content);
954
955 const protocol = protocol_map.get(uri.scheme) orelse return error.UnsupportedUrlScheme;
956 client.http_proxy = .{
957 .allocator = client.allocator,
958 .headers = .{ .allocator = client.allocator },
959
960 .protocol = protocol,
961 .host = if (uri.host) |host| try client.allocator.dupe(u8, host) else return error.UriMissingHost,
962 .port = uri.port orelse switch (protocol) {
963 .plain => 80,
964 .tls => 443,
965 },
966 };
967
968 if (uri.user != null and uri.password != null) {
969 const unencoded = try std.fmt.allocPrint(client.allocator, "{s}:{s}", .{ uri.user.?, uri.password.? });
970 defer client.allocator.free(unencoded);
971
972 const buffer = try client.allocator.alloc(u8, std.base64.standard.Encoder.calcSize(unencoded.len));
973 defer client.allocator.free(buffer);
974
975 const result = std.base64.standard.Encoder.encode(buffer, unencoded);
976
977 try client.http_proxy.?.headers.append("proxy-authorization", result);
978 }
979 }
980
981 if (client.https_proxy == null) https: {
982 const content: []const u8 = if (std.process.hasEnvVarConstant("https_proxy"))
983 try std.process.getEnvVarOwned(client.allocator, "https_proxy")
984 else if (std.process.hasEnvVarConstant("HTTPS_PROXY"))
985 try std.process.getEnvVarOwned(client.allocator, "HTTPS_PROXY")
986 else if (std.process.hasEnvVarConstant("all_proxy"))
987 try std.process.getEnvVarOwned(client.allocator, "all_proxy")
988 else if (std.process.hasEnvVarConstant("ALL_PROXY"))
989 try std.process.getEnvVarOwned(client.allocator, "ALL_PROXY")
990 else
991 break :https;
992 defer client.allocator.free(content);
993
994 const uri = try Uri.parse(content);
995
996 const protocol = protocol_map.get(uri.scheme) orelse return error.UnsupportedUrlScheme;
997 client.http_proxy = .{
998 .allocator = client.allocator,
999 .headers = .{ .allocator = client.allocator },
1000
1001 .protocol = protocol,
1002 .host = if (uri.host) |host| try client.allocator.dupe(u8, host) else return error.UriMissingHost,
1003 .port = uri.port orelse switch (protocol) {
1004 .plain => 80,
1005 .tls => 443,
1006 },
1007 };
1008
1009 if (uri.user != null and uri.password != null) {
1010 const unencoded = try std.fmt.allocPrint(client.allocator, "{s}:{s}", .{ uri.user.?, uri.password.? });
1011 defer client.allocator.free(unencoded);
1012
1013 const buffer = try client.allocator.alloc(u8, std.base64.standard.Encoder.calcSize(unencoded.len));
1014 defer client.allocator.free(buffer);
1015
1016 const result = std.base64.standard.Encoder.encode(buffer, unencoded);
1017
1018 try client.https_proxy.?.headers.append("proxy-authorization", result);
1019 }
1020 }
1021}
1022
1023pub const ConnectTcpError = Allocator.Error || error{ ConnectionRefused, NetworkUnreachable, ConnectionTimedOut, ConnectionResetByPeer, TemporaryNameServerFailure, NameServerFailure, UnknownHostName, HostLacksNetworkAddresses, UnexpectedConnectFailure, TlsInitializationFailed };
9201024
921/// Connect to `host:port` using the specified protocol. This will reuse a connection if one is already open.1025/// Connect to `host:port` using the specified protocol. This will reuse a connection if one is already open.
922/// This function is threadsafe.1026/// This function is threadsafe.
923pub fn connectUnproxied(client: *Client, host: []const u8, port: u16, protocol: Connection.Protocol) ConnectUnproxiedError!*ConnectionPool.Node {1027pub fn connectTcp(client: *Client, host: []const u8, port: u16, protocol: Connection.Protocol) ConnectTcpError!*ConnectionPool.Node {
924 if (client.connection_pool.findConnection(.{1028 if (client.connection_pool.findConnection(.{
925 .host = host,1029 .host = host,
926 .port = port,1030 .port = port,
927 .is_tls = protocol == .tls,1031 .protocol = protocol,
928 })) |node|1032 })) |node|
929 return node;1033 return node;
9301034
...@@ -948,8 +1052,8 @@ pub fn connectUnproxied(client: *Client, host: []const u8, port: u16, protocol:...@@ -948,8 +1052,8 @@ pub fn connectUnproxied(client: *Client, host: []const u8, port: u16, protocol:
948 conn.data = .{1052 conn.data = .{
949 .stream = stream,1053 .stream = stream,
950 .tls_client = undefined,1054 .tls_client = undefined,
951 .protocol = protocol,
9521055
1056 .protocol = protocol,
953 .host = try client.allocator.dupe(u8, host),1057 .host = try client.allocator.dupe(u8, host),
954 .port = port,1058 .port = port,
955 };1059 };
...@@ -981,7 +1085,7 @@ pub fn connectUnix(client: *Client, path: []const u8) ConnectUnixError!*Connecti...@@ -981,7 +1085,7 @@ pub fn connectUnix(client: *Client, path: []const u8) ConnectUnixError!*Connecti
981 if (client.connection_pool.findConnection(.{1085 if (client.connection_pool.findConnection(.{
982 .host = path,1086 .host = path,
983 .port = 0,1087 .port = 0,
984 .is_tls = false,1088 .protocol = .plain,
985 })) |node|1089 })) |node|
986 return node;1090 return node;
9871091
...@@ -1007,34 +1111,120 @@ pub fn connectUnix(client: *Client, path: []const u8) ConnectUnixError!*Connecti...@@ -1007,34 +1111,120 @@ pub fn connectUnix(client: *Client, path: []const u8) ConnectUnixError!*Connecti
1007 return conn;1111 return conn;
1008}1112}
10091113
1010// Prevents a dependency loop in request()1114pub fn connectTunnel(
1011const ConnectErrorPartial = ConnectUnproxiedError || error{ UnsupportedUrlScheme, ConnectionRefused };1115 client: *Client,
1012pub const ConnectError = ConnectErrorPartial || RequestError;1116 proxy: *ProxyInformation,
1117 tunnel_host: []const u8,
1118 tunnel_port: u16,
1119) !*ConnectionPool.Node {
1120 if (!proxy.supports_connect) return error.TunnelNotSupported;
10131121
1014pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connection.Protocol) ConnectError!*ConnectionPool.Node {
1015 if (client.connection_pool.findConnection(.{1122 if (client.connection_pool.findConnection(.{
1016 .host = host,1123 .host = tunnel_host,
1017 .port = port,1124 .port = tunnel_port,
1018 .is_tls = protocol == .tls,1125 .protocol = proxy.protocol,
1019 })) |node|1126 })) |node|
1020 return node;1127 return node;
10211128
1022 if (client.proxy) |proxy| {1129 var maybe_valid = false;
1023 const proxy_port: u16 = proxy.port orelse switch (proxy.protocol) {1130 _ = tunnel: {
1024 .plain => 80,1131 const conn = try client.connectTcp(proxy.host, proxy.port, proxy.protocol);
1025 .tls => 443,1132 errdefer {
1133 conn.data.closing = true;
1134 client.connection_pool.release(client.allocator, conn);
1135 }
1136
1137 const uri = Uri{
1138 .scheme = "http",
1139 .user = null,
1140 .password = null,
1141 .host = tunnel_host,
1142 .port = tunnel_port,
1143 .path = "",
1144 .query = null,
1145 .fragment = null,
1026 };1146 };
10271147
1028 const conn = try client.connectUnproxied(proxy.host, proxy_port, proxy.protocol);1148 // we can use a small buffer here because a CONNECT response should be very small
1029 conn.data.proxied = true;1149 var buffer: [8096]u8 = undefined;
1150
1151 var req = client.request(.CONNECT, uri, proxy.headers, .{
1152 .handle_redirects = false,
1153 .connection = conn,
1154 .header_strategy = .{ .static = buffer[0..] },
1155 }) catch |err| {
1156 std.log.debug("err {}", .{err});
1157 break :tunnel err;
1158 };
1159 defer req.deinit();
1160
1161 req.start(.{ .raw_uri = true }) catch |err| break :tunnel err;
1162 req.wait() catch |err| break :tunnel err;
1163
1164 if (req.response.status.class() == .server_error) {
1165 maybe_valid = true;
1166 break :tunnel error.ServerError;
1167 }
1168
1169 if (req.response.status != .ok) break :tunnel error.ConnectionRefused;
10301170
1171 // this connection is now a tunnel, so we can't use it for anything else, it will only be released when the client is de-initialized.
1172 req.connection = null;
1173
1174 client.allocator.free(conn.data.host);
1175 conn.data.host = try client.allocator.dupe(u8, tunnel_host);
1176 errdefer client.allocator.free(conn.data.host);
1177
1178 conn.data.port = tunnel_port;
1179 conn.data.closing = false;
1180
1181 return conn;
1182 } catch {
1183 // something went wrong with the tunnel
1184 proxy.supports_connect = maybe_valid;
1185 return error.TunnelNotSupported;
1186 };
1187}
1188
1189// Prevents a dependency loop in request()
1190const ConnectErrorPartial = ConnectTcpError || error{ UnsupportedUrlScheme, ConnectionRefused };
1191pub const ConnectError = ConnectErrorPartial || RequestError;
1192
1193pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connection.Protocol) ConnectError!*ConnectionPool.Node {
1194 // pointer required so that `supports_connect` can be updated if a CONNECT fails
1195 const potential_proxy: ?*ProxyInformation = switch (protocol) {
1196 .plain => if (client.http_proxy) |*proxy_info| proxy_info else null,
1197 .tls => if (client.https_proxy) |*proxy_info| proxy_info else null,
1198 };
1199
1200 if (potential_proxy) |proxy| {
1201 // don't attempt to proxy the proxy thru itself.
1202 if (std.mem.eql(u8, proxy.host, host) and proxy.port == port and proxy.protocol == protocol) {
1203 return client.connectTcp(host, port, protocol);
1204 }
1205
1206 _ = if (proxy.supports_connect) tunnel: {
1207 return connectTunnel(client, proxy, host, port) catch |err| switch (err) {
1208 error.TunnelNotSupported => break :tunnel,
1209 else => |e| return e,
1210 };
1211 };
1212
1213 // fall back to using the proxy as a normal http proxy
1214 const conn = try client.connectTcp(proxy.host, proxy.port, proxy.protocol);
1215 errdefer {
1216 conn.data.closing = true;
1217 client.connection_pool.release(conn);
1218 }
1219
1220 conn.data.proxied = true;
1031 return conn;1221 return conn;
1032 } else {
1033 return client.connectUnproxied(host, port, protocol);
1034 }1222 }
1223
1224 return client.connectTcp(host, port, protocol);
1035}1225}
10361226
1037pub const RequestError = ConnectUnproxiedError || ConnectErrorPartial || Request.StartError || std.fmt.ParseIntError || Connection.WriteError || error{1227pub const RequestError = ConnectTcpError || ConnectErrorPartial || Request.StartError || std.fmt.ParseIntError || Connection.WriteError || error{
1038 UnsupportedUrlScheme,1228 UnsupportedUrlScheme,
1039 UriMissingHost,1229 UriMissingHost,
10401230
test/standalone/http.zig+20-14
...@@ -226,8 +226,11 @@ pub fn main() !void {...@@ -226,8 +226,11 @@ pub fn main() !void {
226 const server_thread = try std.Thread.spawn(.{}, serverThread, .{&server});226 const server_thread = try std.Thread.spawn(.{}, serverThread, .{&server});
227227
228 var client = Client{ .allocator = calloc };228 var client = Client{ .allocator = calloc };
229 errdefer client.deinit();
229 // defer client.deinit(); handled below230 // defer client.deinit(); handled below
230231
232 try client.loadDefaultProxies();
233
231 { // read content-length response234 { // read content-length response
232 var h = http.Headers{ .allocator = calloc };235 var h = http.Headers{ .allocator = calloc };
233 defer h.deinit();236 defer h.deinit();
...@@ -251,7 +254,7 @@ pub fn main() !void {...@@ -251,7 +254,7 @@ pub fn main() !void {
251 }254 }
252255
253 // connection has been kept alive256 // connection has been kept alive
254 try testing.expect(client.connection_pool.free_len == 1);257 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
255258
256 { // read large content-length response259 { // read large content-length response
257 var h = http.Headers{ .allocator = calloc };260 var h = http.Headers{ .allocator = calloc };
...@@ -275,7 +278,7 @@ pub fn main() !void {...@@ -275,7 +278,7 @@ pub fn main() !void {
275 }278 }
276279
277 // connection has been kept alive280 // connection has been kept alive
278 try testing.expect(client.connection_pool.free_len == 1);281 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
279282
280 { // send head request and not read chunked283 { // send head request and not read chunked
281 var h = http.Headers{ .allocator = calloc };284 var h = http.Headers{ .allocator = calloc };
...@@ -301,7 +304,7 @@ pub fn main() !void {...@@ -301,7 +304,7 @@ pub fn main() !void {
301 }304 }
302305
303 // connection has been kept alive306 // connection has been kept alive
304 try testing.expect(client.connection_pool.free_len == 1);307 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
305308
306 { // read chunked response309 { // read chunked response
307 var h = http.Headers{ .allocator = calloc };310 var h = http.Headers{ .allocator = calloc };
...@@ -326,7 +329,7 @@ pub fn main() !void {...@@ -326,7 +329,7 @@ pub fn main() !void {
326 }329 }
327330
328 // connection has been kept alive331 // connection has been kept alive
329 try testing.expect(client.connection_pool.free_len == 1);332 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
330333
331 { // send head request and not read chunked334 { // send head request and not read chunked
332 var h = http.Headers{ .allocator = calloc };335 var h = http.Headers{ .allocator = calloc };
...@@ -352,7 +355,7 @@ pub fn main() !void {...@@ -352,7 +355,7 @@ pub fn main() !void {
352 }355 }
353356
354 // connection has been kept alive357 // connection has been kept alive
355 try testing.expect(client.connection_pool.free_len == 1);358 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
356359
357 { // check trailing headers360 { // check trailing headers
358 var h = http.Headers{ .allocator = calloc };361 var h = http.Headers{ .allocator = calloc };
...@@ -377,7 +380,7 @@ pub fn main() !void {...@@ -377,7 +380,7 @@ pub fn main() !void {
377 }380 }
378381
379 // connection has been kept alive382 // connection has been kept alive
380 try testing.expect(client.connection_pool.free_len == 1);383 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
381384
382 { // send content-length request385 { // send content-length request
383 var h = http.Headers{ .allocator = calloc };386 var h = http.Headers{ .allocator = calloc };
...@@ -409,7 +412,7 @@ pub fn main() !void {...@@ -409,7 +412,7 @@ pub fn main() !void {
409 }412 }
410413
411 // connection has been kept alive414 // connection has been kept alive
412 try testing.expect(client.connection_pool.free_len == 1);415 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
413416
414 { // read content-length response with connection close417 { // read content-length response with connection close
415 var h = http.Headers{ .allocator = calloc };418 var h = http.Headers{ .allocator = calloc };
...@@ -468,7 +471,7 @@ pub fn main() !void {...@@ -468,7 +471,7 @@ pub fn main() !void {
468 }471 }
469472
470 // connection has been kept alive473 // connection has been kept alive
471 try testing.expect(client.connection_pool.free_len == 1);474 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
472475
473 { // relative redirect476 { // relative redirect
474 var h = http.Headers{ .allocator = calloc };477 var h = http.Headers{ .allocator = calloc };
...@@ -492,7 +495,7 @@ pub fn main() !void {...@@ -492,7 +495,7 @@ pub fn main() !void {
492 }495 }
493496
494 // connection has been kept alive497 // connection has been kept alive
495 try testing.expect(client.connection_pool.free_len == 1);498 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
496499
497 { // redirect from root500 { // redirect from root
498 var h = http.Headers{ .allocator = calloc };501 var h = http.Headers{ .allocator = calloc };
...@@ -516,7 +519,7 @@ pub fn main() !void {...@@ -516,7 +519,7 @@ pub fn main() !void {
516 }519 }
517520
518 // connection has been kept alive521 // connection has been kept alive
519 try testing.expect(client.connection_pool.free_len == 1);522 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
520523
521 { // absolute redirect524 { // absolute redirect
522 var h = http.Headers{ .allocator = calloc };525 var h = http.Headers{ .allocator = calloc };
...@@ -540,7 +543,7 @@ pub fn main() !void {...@@ -540,7 +543,7 @@ pub fn main() !void {
540 }543 }
541544
542 // connection has been kept alive545 // connection has been kept alive
543 try testing.expect(client.connection_pool.free_len == 1);546 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
544547
545 { // too many redirects548 { // too many redirects
546 var h = http.Headers{ .allocator = calloc };549 var h = http.Headers{ .allocator = calloc };
...@@ -562,7 +565,7 @@ pub fn main() !void {...@@ -562,7 +565,7 @@ pub fn main() !void {
562 }565 }
563566
564 // connection has been kept alive567 // connection has been kept alive
565 try testing.expect(client.connection_pool.free_len == 1);568 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
566569
567 { // check client without segfault by connection error after redirection570 { // check client without segfault by connection error after redirection
568 var h = http.Headers{ .allocator = calloc };571 var h = http.Headers{ .allocator = calloc };
...@@ -579,11 +582,14 @@ pub fn main() !void {...@@ -579,11 +582,14 @@ pub fn main() !void {
579 try req.start(.{});582 try req.start(.{});
580 const result = req.wait();583 const result = req.wait();
581584
582 try testing.expectError(error.ConnectionRefused, result); // expects not segfault but the regular error585 // a proxy without an upstream is likely to return a 5xx status.
586 if (client.http_proxy == null) {
587 try testing.expectError(error.ConnectionRefused, result); // expects not segfault but the regular error
588 }
583 }589 }
584590
585 // connection has been kept alive591 // connection has been kept alive
586 try testing.expect(client.connection_pool.free_len == 1);592 try testing.expect(client.http_proxy != null or client.connection_pool.free_len == 1);
587593
588 { // Client.fetch()594 { // Client.fetch()
589 var h = http.Headers{ .allocator = calloc };595 var h = http.Headers{ .allocator = calloc };