authorgravatar for truemedian@gmail.comNameless <truemedian@gmail.com> 2023-04-14 12:38:13-05:00
committergravatar for truemedian@gmail.comNameless <truemedian@gmail.com> 2023-04-17 19:14:48-05:00
log96533b1289f210b415e12d4cf5bbac466279c2e5
tree1f566443a986b4a8b0d05edfb585023e9616b70b
parent2c492064fbc882fa31256209d201ade1bb20cb92
signature Commit is signed but in an unrecognized format.

std.http: very basic http client proxy

4 files changed, 155 insertions(+), 50 deletions(-)

lib/std/Uri.zig+76
......@@ -27,6 +27,18 @@ pub fn escapeQuery(allocator: std.mem.Allocator, input: []const u8) error{OutOfM
2727 return escapeStringWithFn(allocator, input, isQueryChar);
2828}
2929
30pub fn writeEscapedString(writer: anytype, input: []const u8) !void {
31 return writeEscapedStringWithFn(writer, input, isUnreserved);
32}
33
34pub fn writeEscapedPath(writer: anytype, input: []const u8) !void {
35 return writeEscapedStringWithFn(writer, input, isPathChar);
36}
37
38pub fn writeEscapedQuery(writer: anytype, input: []const u8) !void {
39 return writeEscapedStringWithFn(writer, input, isQueryChar);
40}
41
3042pub fn escapeStringWithFn(allocator: std.mem.Allocator, input: []const u8, comptime keepUnescaped: fn (c: u8) bool) std.mem.Allocator.Error![]const u8 {
3143 var outsize: usize = 0;
3244 for (input) |c| {
......@@ -52,6 +64,16 @@ pub fn escapeStringWithFn(allocator: std.mem.Allocator, input: []const u8, compt
5264 return output;
5365}
5466
67pub fn writeEscapedStringWithFn(writer: anytype, input: []const u8, comptime keepUnescaped: fn (c: u8) bool) @TypeOf(writer).Error!void {
68 for (input) |c| {
69 if (keepUnescaped(c)) {
70 try writer.writeByte(c);
71 } else {
72 try writer.print("%{X:0>2}", .{c});
73 }
74 }
75}
76
5577/// Parses a URI string and unescapes all %XX where XX is a valid hex number. Otherwise, verbatim copies
5678/// them to the output.
5779pub fn unescapeString(allocator: std.mem.Allocator, input: []const u8) error{OutOfMemory}![]const u8 {
......@@ -184,6 +206,60 @@ pub fn parseWithoutScheme(text: []const u8) ParseError!Uri {
184206 return uri;
185207}
186208
209pub fn format(
210 uri: Uri,
211 comptime fmt: []const u8,
212 options: std.fmt.FormatOptions,
213 writer: anytype,
214) @TypeOf(writer).Error!void {
215 _ = options;
216
217 const needs_absolute = comptime std.mem.indexOf(u8, fmt, "+") != null;
218 const needs_path = comptime std.mem.indexOf(u8, fmt, "/") != null or fmt.len == 0;
219
220 if (needs_absolute) {
221 try writer.writeAll(uri.scheme);
222 try writer.writeAll(":");
223 if (uri.host) |host| {
224 try writer.writeAll("//");
225
226 if (uri.user) |user| {
227 try writer.writeAll(user);
228 if (uri.password) |password| {
229 try writer.writeAll(":");
230 try writer.writeAll(password);
231 }
232 try writer.writeAll("@");
233 }
234
235 try writer.writeAll(host);
236
237 if (uri.port) |port| {
238 try writer.writeAll(":");
239 try std.fmt.formatInt(port, 10, .lower, .{}, writer);
240 }
241 }
242 }
243
244 if (needs_path) {
245 if (uri.path.len == 0) {
246 try writer.writeAll("/");
247 } else {
248 try Uri.writeEscapedPath(writer, uri.path);
249 }
250
251 if (uri.query) |q| {
252 try writer.writeAll("?");
253 try Uri.writeEscapedQuery(writer, q);
254 }
255
256 if (uri.fragment) |f| {
257 try writer.writeAll("#");
258 try Uri.writeEscapedQuery(writer, f);
259 }
260 }
261}
262
187263/// Parses the URI or returns an error.
188264/// The return value will contain unescaped strings pointing into the
189265/// original `text`. Each component that is provided, will be non-`null`.
lib/std/http.zig+1-1
......@@ -265,7 +265,7 @@ pub const Connection = enum {
265265 close,
266266};
267267
268pub const CustomHeader = struct {
268pub const Header = struct {
269269 name: []const u8,
270270 value: []const u8,
271271};
lib/std/http/Client.zig+77-48
......@@ -25,27 +25,7 @@ next_https_rescan_certs: bool = true,
2525/// The pool of connections that can be reused (and currently in use).
2626connection_pool: ConnectionPool = .{},
2727
28pub const ExtraError = union(enum) {
29 pub const TcpConnectError = std.net.TcpConnectToHostError;
30 pub const TlsError = std.crypto.tls.Client.InitError(net.Stream);
31 pub const WriteError = BufferedConnection.WriteError;
32 pub const ReadError = BufferedConnection.ReadError || error{HttpChunkInvalid};
33 pub const CaBundleError = std.crypto.Certificate.Bundle.RescanError;
34
35 pub const ZlibInitError = error{ BadHeader, InvalidCompression, InvalidWindowSize, Unsupported, EndOfStream, OutOfMemory } || Request.TransferReadError;
36 pub const GzipInitError = error{ BadHeader, InvalidCompression, OutOfMemory, WrongChecksum, EndOfStream, StreamTooLong } || Request.TransferReadError;
37 // pub const DecompressError = Compression.DeflateDecompressor.Error || Compression.GzipDecompressor.Error || Compression.ZstdDecompressor.Error;
38 pub const DecompressError = anyerror; // FIXME: the above line causes a false positive dependency loop
39
40 zlib_init: ZlibInitError, // error.CompressionInitializationFailed
41 gzip_init: GzipInitError, // error.CompressionInitializationFailed
42 connect: TcpConnectError, // error.ConnectionFailed
43 ca_bundle: CaBundleError, // error.CertificateAuthorityBundleFailed
44 tls: TlsError, // error.TlsInitializationFailed
45 write: WriteError, // error.WriteFailed
46 read: ReadError, // error.ReadFailed
47 decompress: DecompressError, // error.ReadFailed
48};
28proxy: ?HttpProxy = null,
4929
5030/// A set of linked lists of connections that can be reused.
5131pub const ConnectionPool = struct {
......@@ -61,6 +41,7 @@ pub const ConnectionPool = struct {
6141 host: []u8,
6242 port: u16,
6343
44 proxied: bool = false,
6445 closing: bool = false,
6546
6647 pub fn deinit(self: *StoredConnection, client: *Client) void {
......@@ -137,7 +118,12 @@ pub const ConnectionPool = struct {
137118 return client.allocator.destroy(popped);
138119 }
139120
140 pool.free.append(node);
121 if (node.data.proxied) {
122 pool.free.prepend(node); // proxied connections go to the end of the queue, always try direct connections first
123 } else {
124 pool.free.append(node);
125 }
126
141127 pool.free_len += 1;
142128 }
143129
......@@ -546,9 +532,10 @@ pub const Request = struct {
546532 if (!req.response.parser.done) {
547533 // If the response wasn't fully read, then we need to close the connection.
548534 req.connection.data.closing = true;
549 req.client.connection_pool.release(req.client, req.connection);
550535 }
551536
537 req.client.connection_pool.release(req.client, req.connection);
538
552539 req.arena.deinit();
553540 req.* = undefined;
554541 }
......@@ -557,30 +544,20 @@ pub const Request = struct {
557544 var buffered = std.io.bufferedWriter(req.connection.data.buffered.writer());
558545 const w = buffered.writer();
559546
560 const escaped_path = try Uri.escapePath(req.client.allocator, uri.path);
561 defer req.client.allocator.free(escaped_path);
562
563 const escaped_query = if (uri.query) |q| try Uri.escapeQuery(req.client.allocator, q) else null;
564 defer if (escaped_query) |q| req.client.allocator.free(q);
565
566 const escaped_fragment = if (uri.fragment) |f| try Uri.escapeQuery(req.client.allocator, f) else null;
567 defer if (escaped_fragment) |f| req.client.allocator.free(f);
568
569547 try w.writeAll(@tagName(headers.method));
570548 try w.writeByte(' ');
571 if (escaped_path.len == 0) {
572 try w.writeByte('/');
549
550 if (req.headers.method == .CONNECT) {
551 try w.writeAll(uri.host.?);
552 try w.writeByte(':');
553 try w.print("{}", .{uri.port.?});
554 } else if (req.connection.data.proxied) {
555 // proxied connections require the full uri
556 try w.print("{+/}", .{uri});
573557 } else {
574 try w.writeAll(escaped_path);
575 }
576 if (escaped_query) |q| {
577 try w.writeByte('?');
578 try w.writeAll(q);
579 }
580 if (escaped_fragment) |f| {
581 try w.writeByte('#');
582 try w.writeAll(f);
558 try w.print("{/}", .{uri});
583559 }
560
584561 try w.writeByte(' ');
585562 try w.writeAll(@tagName(headers.version));
586563 try w.writeAll("\r\nHost: ");
......@@ -659,6 +636,12 @@ pub const Request = struct {
659636 req.response.parser.done = true;
660637 }
661638
639 if (req.headers.method == .CONNECT and req.response.headers.status == .ok) {
640 req.connection.data.closing = false;
641 req.connection.data.proxied = true;
642 req.response.parser.done = true;
643 }
644
662645 if (req.headers.connection == .keep_alive and req.response.headers.connection == .keep_alive) {
663646 req.connection.data.closing = false;
664647 } else {
......@@ -802,7 +785,7 @@ pub const Request = struct {
802785 }
803786 }
804787
805 pub const FinishError = WriteError || error{ MessageNotCompleted };
788 pub const FinishError = WriteError || error{MessageNotCompleted};
806789
807790 /// Finish the body of a request. This notifies the server that you have no more data to send.
808791 pub fn finish(req: *Request) FinishError!void {
......@@ -817,6 +800,20 @@ pub const Request = struct {
817800 }
818801};
819802
803pub const HttpProxy = struct {
804 pub const ProxyAuthentication = union(enum) {
805 basic: []const u8,
806 custom: []const u8,
807 };
808
809 protocol: Connection.Protocol,
810 host: []const u8,
811 port: ?u16 = null,
812
813 /// The value for the Proxy-Authorization header.
814 auth: ?ProxyAuthentication = null,
815};
816
820817/// Release all associated resources with the client.
821818/// TODO: currently leaks all request allocated data
822819pub fn deinit(client: *Client) void {
......@@ -826,11 +823,11 @@ pub fn deinit(client: *Client) void {
826823 client.* = undefined;
827824}
828825
829pub const ConnectError = Allocator.Error || error{ ConnectionRefused, NetworkUnreachable, ConnectionTimedOut, ConnectionResetByPeer, TemporaryNameServerFailure, NameServerFailure, UnknownHostName, HostLacksNetworkAddresses, UnexpectedConnectFailure, TlsInitializationFailed };
826pub const ConnectUnproxiedError = Allocator.Error || error{ ConnectionRefused, NetworkUnreachable, ConnectionTimedOut, ConnectionResetByPeer, TemporaryNameServerFailure, NameServerFailure, UnknownHostName, HostLacksNetworkAddresses, UnexpectedConnectFailure, TlsInitializationFailed };
830827
831828/// Connect to `host:port` using the specified protocol. This will reuse a connection if one is already open.
832829/// This function is threadsafe.
833pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connection.Protocol) ConnectError!*ConnectionPool.Node {
830pub fn connectUnproxied(client: *Client, host: []const u8, port: u16, protocol: Connection.Protocol) ConnectUnproxiedError!*ConnectionPool.Node {
834831 if (client.connection_pool.findConnection(.{
835832 .host = host,
836833 .port = port,
......@@ -884,7 +881,34 @@ pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connectio
884881 return conn;
885882}
886883
887pub const RequestError = ConnectError || BufferedConnection.WriteError || error{
884// Prevents a dependency loop in request()
885const ConnectErrorPartial = ConnectUnproxiedError || error{ UnsupportedUrlScheme, ConnectionRefused };
886pub const ConnectError = ConnectErrorPartial || RequestError;
887
888pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connection.Protocol) ConnectError!*ConnectionPool.Node {
889 if (client.connection_pool.findConnection(.{
890 .host = host,
891 .port = port,
892 .is_tls = protocol == .tls,
893 })) |node|
894 return node;
895
896 if (client.proxy) |proxy| {
897 const proxy_port: u16 = proxy.port orelse switch (proxy.protocol) {
898 .plain => 80,
899 .tls => 443,
900 };
901
902 const conn = try client.connectUnproxied(proxy.host, proxy_port, proxy.protocol);
903 conn.data.proxied = true;
904
905 return conn;
906 } else {
907 return client.connectUnproxied(host, port, protocol);
908 }
909}
910
911pub const RequestError = ConnectUnproxiedError || ConnectErrorPartial || BufferedConnection.WriteError || error{
888912 UnsupportedUrlScheme,
889913 UriMissingHost,
890914
......@@ -896,6 +920,9 @@ pub const Options = struct {
896920 max_redirects: u32 = 3,
897921 header_strategy: HeaderStrategy = .{ .dynamic = 16 * 1024 },
898922
923 /// Must be an already acquired connection.
924 connection: ?*ConnectionPool.Node = null,
925
899926 pub const HeaderStrategy = union(enum) {
900927 /// In this case, the client's Allocator will be used to store the
901928 /// entire HTTP header. This value is the maximum total size of
......@@ -939,10 +966,12 @@ pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Opt
939966 }
940967 }
941968
969 const conn = options.connection orelse try client.connect(host, port, protocol);
970
942971 var req: Request = .{
943972 .uri = uri,
944973 .client = client,
945 .connection = try client.connect(host, port, protocol),
974 .connection = conn,
946975 .headers = headers,
947976 .redirects_left = options.max_redirects,
948977 .handle_redirects = options.handle_redirects,
lib/std/http/protocol.zig+1-1
......@@ -1,4 +1,4 @@
1const std = @import("std");
1const std = @import("../std.zig");
22const testing = std.testing;
33const mem = std.mem;
44