authorgravatar for truemedian@gmail.comNameless <truemedian@gmail.com> 2023-04-14 12:38:13-05:00
committergravatar for truemedian@gmail.comNameless <truemedian@gmail.com> 2023-04-17 19:14:48-05:00
log96533b1289f210b415e12d4cf5bbac466279c2e5
tree1f566443a986b4a8b0d05edfb585023e9616b70b
parent2c492064fbc882fa31256209d201ade1bb20cb92
signature Commit is signed but in an unrecognized format.

std.http: very basic http client proxy

4 files changed, 155 insertions(+), 50 deletions(-)

lib/std/Uri.zig+76
...@@ -27,6 +27,18 @@ pub fn escapeQuery(allocator: std.mem.Allocator, input: []const u8) error{OutOfM...@@ -27,6 +27,18 @@ pub fn escapeQuery(allocator: std.mem.Allocator, input: []const u8) error{OutOfM
27 return escapeStringWithFn(allocator, input, isQueryChar);27 return escapeStringWithFn(allocator, input, isQueryChar);
28}28}
2929
30pub fn writeEscapedString(writer: anytype, input: []const u8) !void {
31 return writeEscapedStringWithFn(writer, input, isUnreserved);
32}
33
34pub fn writeEscapedPath(writer: anytype, input: []const u8) !void {
35 return writeEscapedStringWithFn(writer, input, isPathChar);
36}
37
38pub fn writeEscapedQuery(writer: anytype, input: []const u8) !void {
39 return writeEscapedStringWithFn(writer, input, isQueryChar);
40}
41
30pub fn escapeStringWithFn(allocator: std.mem.Allocator, input: []const u8, comptime keepUnescaped: fn (c: u8) bool) std.mem.Allocator.Error![]const u8 {42pub fn escapeStringWithFn(allocator: std.mem.Allocator, input: []const u8, comptime keepUnescaped: fn (c: u8) bool) std.mem.Allocator.Error![]const u8 {
31 var outsize: usize = 0;43 var outsize: usize = 0;
32 for (input) |c| {44 for (input) |c| {
...@@ -52,6 +64,16 @@ pub fn escapeStringWithFn(allocator: std.mem.Allocator, input: []const u8, compt...@@ -52,6 +64,16 @@ pub fn escapeStringWithFn(allocator: std.mem.Allocator, input: []const u8, compt
52 return output;64 return output;
53}65}
5466
67pub fn writeEscapedStringWithFn(writer: anytype, input: []const u8, comptime keepUnescaped: fn (c: u8) bool) @TypeOf(writer).Error!void {
68 for (input) |c| {
69 if (keepUnescaped(c)) {
70 try writer.writeByte(c);
71 } else {
72 try writer.print("%{X:0>2}", .{c});
73 }
74 }
75}
76
55/// Parses a URI string and unescapes all %XX where XX is a valid hex number. Otherwise, verbatim copies77/// Parses a URI string and unescapes all %XX where XX is a valid hex number. Otherwise, verbatim copies
56/// them to the output.78/// them to the output.
57pub fn unescapeString(allocator: std.mem.Allocator, input: []const u8) error{OutOfMemory}![]const u8 {79pub fn unescapeString(allocator: std.mem.Allocator, input: []const u8) error{OutOfMemory}![]const u8 {
...@@ -184,6 +206,60 @@ pub fn parseWithoutScheme(text: []const u8) ParseError!Uri {...@@ -184,6 +206,60 @@ pub fn parseWithoutScheme(text: []const u8) ParseError!Uri {
184 return uri;206 return uri;
185}207}
186208
209pub fn format(
210 uri: Uri,
211 comptime fmt: []const u8,
212 options: std.fmt.FormatOptions,
213 writer: anytype,
214) @TypeOf(writer).Error!void {
215 _ = options;
216
217 const needs_absolute = comptime std.mem.indexOf(u8, fmt, "+") != null;
218 const needs_path = comptime std.mem.indexOf(u8, fmt, "/") != null or fmt.len == 0;
219
220 if (needs_absolute) {
221 try writer.writeAll(uri.scheme);
222 try writer.writeAll(":");
223 if (uri.host) |host| {
224 try writer.writeAll("//");
225
226 if (uri.user) |user| {
227 try writer.writeAll(user);
228 if (uri.password) |password| {
229 try writer.writeAll(":");
230 try writer.writeAll(password);
231 }
232 try writer.writeAll("@");
233 }
234
235 try writer.writeAll(host);
236
237 if (uri.port) |port| {
238 try writer.writeAll(":");
239 try std.fmt.formatInt(port, 10, .lower, .{}, writer);
240 }
241 }
242 }
243
244 if (needs_path) {
245 if (uri.path.len == 0) {
246 try writer.writeAll("/");
247 } else {
248 try Uri.writeEscapedPath(writer, uri.path);
249 }
250
251 if (uri.query) |q| {
252 try writer.writeAll("?");
253 try Uri.writeEscapedQuery(writer, q);
254 }
255
256 if (uri.fragment) |f| {
257 try writer.writeAll("#");
258 try Uri.writeEscapedQuery(writer, f);
259 }
260 }
261}
262
187/// Parses the URI or returns an error.263/// Parses the URI or returns an error.
188/// The return value will contain unescaped strings pointing into the264/// The return value will contain unescaped strings pointing into the
189/// original `text`. Each component that is provided, will be non-`null`.265/// original `text`. Each component that is provided, will be non-`null`.
lib/std/http.zig+1-1
...@@ -265,7 +265,7 @@ pub const Connection = enum {...@@ -265,7 +265,7 @@ pub const Connection = enum {
265 close,265 close,
266};266};
267267
268pub const CustomHeader = struct {268pub const Header = struct {
269 name: []const u8,269 name: []const u8,
270 value: []const u8,270 value: []const u8,
271};271};
lib/std/http/Client.zig+77-48
...@@ -25,27 +25,7 @@ next_https_rescan_certs: bool = true,...@@ -25,27 +25,7 @@ next_https_rescan_certs: bool = true,
25/// The pool of connections that can be reused (and currently in use).25/// The pool of connections that can be reused (and currently in use).
26connection_pool: ConnectionPool = .{},26connection_pool: ConnectionPool = .{},
2727
28pub const ExtraError = union(enum) {28proxy: ?HttpProxy = null,
29 pub const TcpConnectError = std.net.TcpConnectToHostError;
30 pub const TlsError = std.crypto.tls.Client.InitError(net.Stream);
31 pub const WriteError = BufferedConnection.WriteError;
32 pub const ReadError = BufferedConnection.ReadError || error{HttpChunkInvalid};
33 pub const CaBundleError = std.crypto.Certificate.Bundle.RescanError;
34
35 pub const ZlibInitError = error{ BadHeader, InvalidCompression, InvalidWindowSize, Unsupported, EndOfStream, OutOfMemory } || Request.TransferReadError;
36 pub const GzipInitError = error{ BadHeader, InvalidCompression, OutOfMemory, WrongChecksum, EndOfStream, StreamTooLong } || Request.TransferReadError;
37 // pub const DecompressError = Compression.DeflateDecompressor.Error || Compression.GzipDecompressor.Error || Compression.ZstdDecompressor.Error;
38 pub const DecompressError = anyerror; // FIXME: the above line causes a false positive dependency loop
39
40 zlib_init: ZlibInitError, // error.CompressionInitializationFailed
41 gzip_init: GzipInitError, // error.CompressionInitializationFailed
42 connect: TcpConnectError, // error.ConnectionFailed
43 ca_bundle: CaBundleError, // error.CertificateAuthorityBundleFailed
44 tls: TlsError, // error.TlsInitializationFailed
45 write: WriteError, // error.WriteFailed
46 read: ReadError, // error.ReadFailed
47 decompress: DecompressError, // error.ReadFailed
48};
4929
50/// A set of linked lists of connections that can be reused.30/// A set of linked lists of connections that can be reused.
51pub const ConnectionPool = struct {31pub const ConnectionPool = struct {
...@@ -61,6 +41,7 @@ pub const ConnectionPool = struct {...@@ -61,6 +41,7 @@ pub const ConnectionPool = struct {
61 host: []u8,41 host: []u8,
62 port: u16,42 port: u16,
6343
44 proxied: bool = false,
64 closing: bool = false,45 closing: bool = false,
6546
66 pub fn deinit(self: *StoredConnection, client: *Client) void {47 pub fn deinit(self: *StoredConnection, client: *Client) void {
...@@ -137,7 +118,12 @@ pub const ConnectionPool = struct {...@@ -137,7 +118,12 @@ pub const ConnectionPool = struct {
137 return client.allocator.destroy(popped);118 return client.allocator.destroy(popped);
138 }119 }
139120
140 pool.free.append(node);121 if (node.data.proxied) {
122 pool.free.prepend(node); // proxied connections go to the end of the queue, always try direct connections first
123 } else {
124 pool.free.append(node);
125 }
126
141 pool.free_len += 1;127 pool.free_len += 1;
142 }128 }
143129
...@@ -546,9 +532,10 @@ pub const Request = struct {...@@ -546,9 +532,10 @@ pub const Request = struct {
546 if (!req.response.parser.done) {532 if (!req.response.parser.done) {
547 // If the response wasn't fully read, then we need to close the connection.533 // If the response wasn't fully read, then we need to close the connection.
548 req.connection.data.closing = true;534 req.connection.data.closing = true;
549 req.client.connection_pool.release(req.client, req.connection);
550 }535 }
551536
537 req.client.connection_pool.release(req.client, req.connection);
538
552 req.arena.deinit();539 req.arena.deinit();
553 req.* = undefined;540 req.* = undefined;
554 }541 }
...@@ -557,30 +544,20 @@ pub const Request = struct {...@@ -557,30 +544,20 @@ pub const Request = struct {
557 var buffered = std.io.bufferedWriter(req.connection.data.buffered.writer());544 var buffered = std.io.bufferedWriter(req.connection.data.buffered.writer());
558 const w = buffered.writer();545 const w = buffered.writer();
559546
560 const escaped_path = try Uri.escapePath(req.client.allocator, uri.path);
561 defer req.client.allocator.free(escaped_path);
562
563 const escaped_query = if (uri.query) |q| try Uri.escapeQuery(req.client.allocator, q) else null;
564 defer if (escaped_query) |q| req.client.allocator.free(q);
565
566 const escaped_fragment = if (uri.fragment) |f| try Uri.escapeQuery(req.client.allocator, f) else null;
567 defer if (escaped_fragment) |f| req.client.allocator.free(f);
568
569 try w.writeAll(@tagName(headers.method));547 try w.writeAll(@tagName(headers.method));
570 try w.writeByte(' ');548 try w.writeByte(' ');
571 if (escaped_path.len == 0) {549
572 try w.writeByte('/');550 if (req.headers.method == .CONNECT) {
551 try w.writeAll(uri.host.?);
552 try w.writeByte(':');
553 try w.print("{}", .{uri.port.?});
554 } else if (req.connection.data.proxied) {
555 // proxied connections require the full uri
556 try w.print("{+/}", .{uri});
573 } else {557 } else {
574 try w.writeAll(escaped_path);558 try w.print("{/}", .{uri});
575 }
576 if (escaped_query) |q| {
577 try w.writeByte('?');
578 try w.writeAll(q);
579 }
580 if (escaped_fragment) |f| {
581 try w.writeByte('#');
582 try w.writeAll(f);
583 }559 }
560
584 try w.writeByte(' ');561 try w.writeByte(' ');
585 try w.writeAll(@tagName(headers.version));562 try w.writeAll(@tagName(headers.version));
586 try w.writeAll("\r\nHost: ");563 try w.writeAll("\r\nHost: ");
...@@ -659,6 +636,12 @@ pub const Request = struct {...@@ -659,6 +636,12 @@ pub const Request = struct {
659 req.response.parser.done = true;636 req.response.parser.done = true;
660 }637 }
661638
639 if (req.headers.method == .CONNECT and req.response.headers.status == .ok) {
640 req.connection.data.closing = false;
641 req.connection.data.proxied = true;
642 req.response.parser.done = true;
643 }
644
662 if (req.headers.connection == .keep_alive and req.response.headers.connection == .keep_alive) {645 if (req.headers.connection == .keep_alive and req.response.headers.connection == .keep_alive) {
663 req.connection.data.closing = false;646 req.connection.data.closing = false;
664 } else {647 } else {
...@@ -802,7 +785,7 @@ pub const Request = struct {...@@ -802,7 +785,7 @@ pub const Request = struct {
802 }785 }
803 }786 }
804787
805 pub const FinishError = WriteError || error{ MessageNotCompleted };788 pub const FinishError = WriteError || error{MessageNotCompleted};
806789
807 /// Finish the body of a request. This notifies the server that you have no more data to send.790 /// Finish the body of a request. This notifies the server that you have no more data to send.
808 pub fn finish(req: *Request) FinishError!void {791 pub fn finish(req: *Request) FinishError!void {
...@@ -817,6 +800,20 @@ pub const Request = struct {...@@ -817,6 +800,20 @@ pub const Request = struct {
817 }800 }
818};801};
819802
803pub const HttpProxy = struct {
804 pub const ProxyAuthentication = union(enum) {
805 basic: []const u8,
806 custom: []const u8,
807 };
808
809 protocol: Connection.Protocol,
810 host: []const u8,
811 port: ?u16 = null,
812
813 /// The value for the Proxy-Authorization header.
814 auth: ?ProxyAuthentication = null,
815};
816
820/// Release all associated resources with the client.817/// Release all associated resources with the client.
821/// TODO: currently leaks all request allocated data818/// TODO: currently leaks all request allocated data
822pub fn deinit(client: *Client) void {819pub fn deinit(client: *Client) void {
...@@ -826,11 +823,11 @@ pub fn deinit(client: *Client) void {...@@ -826,11 +823,11 @@ pub fn deinit(client: *Client) void {
826 client.* = undefined;823 client.* = undefined;
827}824}
828825
829pub const ConnectError = Allocator.Error || error{ ConnectionRefused, NetworkUnreachable, ConnectionTimedOut, ConnectionResetByPeer, TemporaryNameServerFailure, NameServerFailure, UnknownHostName, HostLacksNetworkAddresses, UnexpectedConnectFailure, TlsInitializationFailed };826pub const ConnectUnproxiedError = Allocator.Error || error{ ConnectionRefused, NetworkUnreachable, ConnectionTimedOut, ConnectionResetByPeer, TemporaryNameServerFailure, NameServerFailure, UnknownHostName, HostLacksNetworkAddresses, UnexpectedConnectFailure, TlsInitializationFailed };
830827
831/// Connect to `host:port` using the specified protocol. This will reuse a connection if one is already open.828/// Connect to `host:port` using the specified protocol. This will reuse a connection if one is already open.
832/// This function is threadsafe.829/// This function is threadsafe.
833pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connection.Protocol) ConnectError!*ConnectionPool.Node {830pub fn connectUnproxied(client: *Client, host: []const u8, port: u16, protocol: Connection.Protocol) ConnectUnproxiedError!*ConnectionPool.Node {
834 if (client.connection_pool.findConnection(.{831 if (client.connection_pool.findConnection(.{
835 .host = host,832 .host = host,
836 .port = port,833 .port = port,
...@@ -884,7 +881,34 @@ pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connectio...@@ -884,7 +881,34 @@ pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connectio
884 return conn;881 return conn;
885}882}
886883
887pub const RequestError = ConnectError || BufferedConnection.WriteError || error{884// Prevents a dependency loop in request()
885const ConnectErrorPartial = ConnectUnproxiedError || error{ UnsupportedUrlScheme, ConnectionRefused };
886pub const ConnectError = ConnectErrorPartial || RequestError;
887
888pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connection.Protocol) ConnectError!*ConnectionPool.Node {
889 if (client.connection_pool.findConnection(.{
890 .host = host,
891 .port = port,
892 .is_tls = protocol == .tls,
893 })) |node|
894 return node;
895
896 if (client.proxy) |proxy| {
897 const proxy_port: u16 = proxy.port orelse switch (proxy.protocol) {
898 .plain => 80,
899 .tls => 443,
900 };
901
902 const conn = try client.connectUnproxied(proxy.host, proxy_port, proxy.protocol);
903 conn.data.proxied = true;
904
905 return conn;
906 } else {
907 return client.connectUnproxied(host, port, protocol);
908 }
909}
910
911pub const RequestError = ConnectUnproxiedError || ConnectErrorPartial || BufferedConnection.WriteError || error{
888 UnsupportedUrlScheme,912 UnsupportedUrlScheme,
889 UriMissingHost,913 UriMissingHost,
890914
...@@ -896,6 +920,9 @@ pub const Options = struct {...@@ -896,6 +920,9 @@ pub const Options = struct {
896 max_redirects: u32 = 3,920 max_redirects: u32 = 3,
897 header_strategy: HeaderStrategy = .{ .dynamic = 16 * 1024 },921 header_strategy: HeaderStrategy = .{ .dynamic = 16 * 1024 },
898922
923 /// Must be an already acquired connection.
924 connection: ?*ConnectionPool.Node = null,
925
899 pub const HeaderStrategy = union(enum) {926 pub const HeaderStrategy = union(enum) {
900 /// In this case, the client's Allocator will be used to store the927 /// In this case, the client's Allocator will be used to store the
901 /// entire HTTP header. This value is the maximum total size of928 /// entire HTTP header. This value is the maximum total size of
...@@ -939,10 +966,12 @@ pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Opt...@@ -939,10 +966,12 @@ pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Opt
939 }966 }
940 }967 }
941968
969 const conn = options.connection orelse try client.connect(host, port, protocol);
970
942 var req: Request = .{971 var req: Request = .{
943 .uri = uri,972 .uri = uri,
944 .client = client,973 .client = client,
945 .connection = try client.connect(host, port, protocol),974 .connection = conn,
946 .headers = headers,975 .headers = headers,
947 .redirects_left = options.max_redirects,976 .redirects_left = options.max_redirects,
948 .handle_redirects = options.handle_redirects,977 .handle_redirects = options.handle_redirects,
lib/std/http/protocol.zig+1-1
...@@ -1,4 +1,4 @@...@@ -1,4 +1,4 @@
1const std = @import("std");1const std = @import("../std.zig");
2const testing = std.testing;2const testing = std.testing;
3const mem = std.mem;3const mem = std.mem;
44