authorgravatar for truemedian@gmail.comNameless <truemedian@gmail.com> 2023-03-06 23:35:35-06:00
committergravatar for truemedian@gmail.comNameless <truemedian@gmail.com> 2023-03-09 14:55:13-06:00
log0a4130f364c2714b206257d0cf589103da823407
tree6c790af4ce7c8c98cca5e1114602a103d8a06ca1
parentfd2f906d1ede2b65ba21eec59137b2d4b676eedc
signature Commit is signed but in an unrecognized format.

std.http: handle relative redirects

4 files changed, 196 insertions(+), 61 deletions(-)

lib/std/Uri.zig+96-13
......@@ -16,15 +16,27 @@ fragment: ?[]const u8,
1616
1717/// Applies URI encoding and replaces all reserved characters with their respective %XX code.
1818pub fn escapeString(allocator: std.mem.Allocator, input: []const u8) error{OutOfMemory}![]const u8 {
19 return escapeStringWithFn(allocator, input, isUnreserved);
20}
21
22pub fn escapePath(allocator: std.mem.Allocator, input: []const u8) error{OutOfMemory}![]const u8 {
23 return escapeStringWithFn(allocator, input, isPathChar);
24}
25
26pub fn escapeQuery(allocator: std.mem.Allocator, input: []const u8) error{OutOfMemory}![]const u8 {
27 return escapeStringWithFn(allocator, input, isQueryChar);
28}
29
30pub fn escapeStringWithFn(allocator: std.mem.Allocator, input: []const u8, comptime keepUnescaped: fn (c: u8) bool) std.mem.Allocator.Error![]const u8 {
1931 var outsize: usize = 0;
2032 for (input) |c| {
21 outsize += if (isUnreserved(c)) @as(usize, 1) else 3;
33 outsize += if (keepUnescaped(c)) @as(usize, 1) else 3;
2234 }
2335 var output = try allocator.alloc(u8, outsize);
2436 var outptr: usize = 0;
2537
2638 for (input) |c| {
27 if (isUnreserved(c)) {
39 if (keepUnescaped(c)) {
2840 output[outptr] = c;
2941 outptr += 1;
3042 } else {
......@@ -94,13 +106,14 @@ pub fn unescapeString(allocator: std.mem.Allocator, input: []const u8) error{Out
94106
95107pub const ParseError = error{ UnexpectedCharacter, InvalidFormat, InvalidPort };
96108
97/// Parses the URI or returns an error.
109/// Parses the URI or returns an error. This function is not compliant, but is required to parse
110/// some forms of URIs in the wild. Such as HTTP Location headers.
98111/// The return value will contain unescaped strings pointing into the
99112/// original `text`. Each component that is provided, will be non-`null`.
100pub fn parse(text: []const u8) ParseError!Uri {
113pub fn parseWithoutScheme(text: []const u8) ParseError!Uri {
101114 var reader = SliceReader{ .slice = text };
102115 var uri = Uri{
103 .scheme = reader.readWhile(isSchemeChar),
116 .scheme = "",
104117 .user = null,
105118 .password = null,
106119 .host = null,
......@@ -110,14 +123,6 @@ pub fn parse(text: []const u8) ParseError!Uri {
110123 .fragment = null,
111124 };
112125
113 // after the scheme, a ':' must appear
114 if (reader.get()) |c| {
115 if (c != ':')
116 return error.UnexpectedCharacter;
117 } else {
118 return error.InvalidFormat;
119 }
120
121126 if (reader.peekPrefix("//")) { // authority part
122127 std.debug.assert(reader.get().? == '/');
123128 std.debug.assert(reader.get().? == '/');
......@@ -179,6 +184,76 @@ pub fn parse(text: []const u8) ParseError!Uri {
179184 return uri;
180185}
181186
187/// Parses the URI or returns an error.
188/// The return value will contain unescaped strings pointing into the
189/// original `text`. Each component that is provided, will be non-`null`.
190pub fn parse(text: []const u8) ParseError!Uri {
191 var reader = SliceReader{ .slice = text };
192 const scheme = reader.readWhile(isSchemeChar);
193
194 // after the scheme, a ':' must appear
195 if (reader.get()) |c| {
196 if (c != ':')
197 return error.UnexpectedCharacter;
198 } else {
199 return error.InvalidFormat;
200 }
201
202 var uri = try parseWithoutScheme(reader.readUntilEof());
203 uri.scheme = scheme;
204
205 return uri;
206}
207
208/// Resolves a URI against a base URI, conforming to RFC 3986, Section 5.
209/// arena owns any memory allocated by this function.
210pub fn resolve(Base: Uri, R: Uri, strict: bool, arena: std.mem.Allocator) !Uri {
211 var T: Uri = undefined;
212
213 if (R.scheme.len > 0 and !((!strict) and (std.mem.eql(u8, R.scheme, Base.scheme)))) {
214 T.scheme = R.scheme;
215 T.user = R.user;
216 T.host = R.host;
217 T.port = R.port;
218 T.path = try std.fs.path.resolvePosix(arena, &.{ "/", R.path });
219 T.query = R.query;
220 } else {
221 if (R.host) |host| {
222 T.user = R.user;
223 T.host = host;
224 T.port = R.port;
225 T.path = R.path;
226 T.path = try std.fs.path.resolvePosix(arena, &.{ "/", R.path });
227 T.query = R.query;
228 } else {
229 if (R.path.len == 0) {
230 T.path = Base.path;
231 if (R.query) |query| {
232 T.query = query;
233 } else {
234 T.query = Base.query;
235 }
236 } else {
237 if (R.path[0] == '/') {
238 T.path = try std.fs.path.resolvePosix(arena, &.{ "/", R.path });
239 } else {
240 T.path = try std.fs.path.resolvePosix(arena, &.{ "/", Base.path, R.path });
241 }
242 T.query = R.query;
243 }
244
245 T.user = Base.user;
246 T.host = Base.host;
247 T.port = Base.port;
248 }
249 T.scheme = Base.scheme;
250 }
251
252 T.fragment = R.fragment;
253
254 return T;
255}
256
182257const SliceReader = struct {
183258 const Self = @This();
184259
......@@ -284,6 +359,14 @@ fn isPathSeparator(c: u8) bool {
284359 };
285360}
286361
362fn isPathChar(c: u8) bool {
363 return isUnreserved(c) or isSubLimit(c) or c == '/' or c == ':' or c == '@';
364}
365
366fn isQueryChar(c: u8) bool {
367 return isPathChar(c) or c == '?';
368}
369
287370fn isQuerySeparator(c: u8) bool {
288371 return switch (c) {
289372 '#' => true,
lib/std/crypto/tls/Client.zig+1-1
......@@ -89,7 +89,7 @@ pub const StreamInterface = struct {
8989};
9090
9191pub fn InitError(comptime Stream: type) type {
92 return std.mem.Allocator.Error || Stream.WriteError || Stream.ReadError || error {
92 return std.mem.Allocator.Error || Stream.WriteError || Stream.ReadError || error{
9393 InsufficientEntropy,
9494 DiskQuota,
9595 LockViolation,
lib/std/http/Client.zig+96-44
......@@ -29,9 +29,10 @@ const ConnectionPool = std.TailQueue(Connection);
2929const ConnectionNode = ConnectionPool.Node;
3030
3131/// Acquires an existing connection from the connection pool. This function is threadsafe.
32pub fn acquire(client: *Client, node: *ConnectionNode) void {
33 client.connection_mutex.lock();
34 defer client.connection_mutex.unlock();
32/// If the caller already holds the connection mutex, it should pass `true` for `held`.
33pub fn acquire(client: *Client, node: *ConnectionNode, held: bool) void {
34 if (!held) client.connection_mutex.lock();
35 defer if (!held) client.connection_mutex.unlock();
3536
3637 client.connection_pool.remove(node);
3738 client.connection_used.append(node);
......@@ -40,16 +41,17 @@ pub fn acquire(client: *Client, node: *ConnectionNode) void {
4041/// Tries to release a connection back to the connection pool. This function is threadsafe.
4142/// If the connection is marked as closing, it will be closed instead.
4243pub fn release(client: *Client, node: *ConnectionNode) void {
44 client.connection_mutex.lock();
45 defer client.connection_mutex.unlock();
46
47 client.connection_used.remove(node);
48
4349 if (node.data.closing) {
4450 node.data.close(client);
4551
4652 return client.allocator.destroy(node);
4753 }
4854
49 client.connection_mutex.lock();
50 defer client.connection_mutex.unlock();
51
52 client.connection_used.remove(node);
5355 client.connection_pool.append(node);
5456}
5557
......@@ -83,7 +85,7 @@ pub const Connection = struct {
8385 }
8486 }
8587
86 pub const ReadError = std.net.Stream.ReadError || error{
88 pub const ReadError = net.Stream.ReadError || error{
8789 TlsConnectionTruncated,
8890 TlsRecordOverflow,
8991 TlsDecodeError,
......@@ -115,7 +117,7 @@ pub const Connection = struct {
115117 }
116118 }
117119
118 pub const WriteError = std.net.Stream.WriteError || error{};
120 pub const WriteError = net.Stream.WriteError || error{};
119121 pub const Writer = std.io.Writer(*Connection, WriteError, write);
120122
121123 pub fn writer(conn: *Connection) Writer {
......@@ -139,14 +141,21 @@ pub const Request = struct {
139141 const read_buffer_size = 8192;
140142 const ReadBufferIndex = std.math.IntFittingRange(0, read_buffer_size);
141143
144 uri: Uri,
142145 client: *Client,
143146 connection: *ConnectionNode,
144 redirects_left: u32,
145147 response: Response,
146148 /// These are stored in Request so that they are available when following
147149 /// redirects.
148150 headers: Headers,
149151
152 redirects_left: u32,
153 handle_redirects: bool,
154 compression_init: bool,
155
156 /// Used as a allocator for resolving redirects locations.
157 arena: std.heap.ArenaAllocator,
158
150159 /// Read buffer for the connection. This is used to pull in large amounts of data from the connection even if the user asks for a small amount. This can probably be removed with careful planning.
151160 read_buffer: [read_buffer_size]u8 = undefined,
152161 read_buffer_start: ReadBufferIndex = 0,
......@@ -661,6 +670,7 @@ pub const Request = struct {
661670 pub const Headers = struct {
662671 version: http.Version = .@"HTTP/1.1",
663672 method: http.Method = .GET,
673 user_agent: []const u8 = "Zig (std.http)",
664674 connection: http.Connection = .keep_alive,
665675 transfer_encoding: RequestTransfer = .none,
666676
......@@ -668,6 +678,7 @@ pub const Request = struct {
668678 };
669679
670680 pub const Options = struct {
681 handle_redirects: bool = true,
671682 max_redirects: u32 = 3,
672683 header_strategy: HeaderStrategy = .{ .dynamic = 16 * 1024 },
673684
......@@ -703,10 +714,11 @@ pub const Request = struct {
703714 req.client.release(req.connection);
704715 }
705716
717 req.arena.deinit();
706718 req.* = undefined;
707719 }
708720
709 const ReadRawError = Connection.ReadError || std.Uri.ParseError || RequestError || error{
721 const ReadRawError = Connection.ReadError || Uri.ParseError || RequestError || error{
710722 UnexpectedEndOfStream,
711723 TooManyHttpRedirects,
712724 HttpRedirectMissingLocation,
......@@ -723,9 +735,7 @@ pub const Request = struct {
723735 var index: usize = 0;
724736 while (index == 0) {
725737 const amt = try req.readRawAdvanced(buffer[index..]);
726 const zero_means_end = req.response.done and req.response.headers.status.class() != .redirect;
727
728 if (amt == 0 and zero_means_end) break;
738 if (amt == 0 and req.response.done) break;
729739 index += amt;
730740 }
731741
......@@ -769,6 +779,8 @@ pub const Request = struct {
769779 }
770780 } else if (req.response.headers.content_length) |content_length| {
771781 req.response.next_chunk_length = content_length;
782
783 if (content_length == 0) req.response.done = true;
772784 } else {
773785 req.response.done = true;
774786 }
......@@ -779,7 +791,7 @@ pub const Request = struct {
779791 return 0;
780792 }
781793
782 pub const WaitForCompleteHeadError = ReadRawError || error {
794 pub const WaitForCompleteHeadError = ReadRawError || error{
783795 UnexpectedEndOfStream,
784796
785797 HttpHeadersExceededSizeLimit,
......@@ -810,27 +822,8 @@ pub const Request = struct {
810822
811823 /// This one can return 0 without meaning EOF.
812824 fn readRawAdvanced(req: *Request, buffer: []u8) !usize {
813 if (req.response.done) {
814 if (req.response.headers.status.class() == .redirect) {
815 if (req.redirects_left == 0) return error.TooManyHttpRedirects;
816
817 const location = req.response.headers.location orelse
818 return error.HttpRedirectMissingLocation;
819 const new_url = try std.Uri.parse(location);
820 const new_req = try req.client.request(new_url, req.headers, .{
821 .max_redirects = req.redirects_left - 1,
822 .header_strategy = if (req.response.header_bytes_owned) .{
823 .dynamic = req.response.max_header_bytes,
824 } else .{
825 .static = req.response.header_bytes.unusedCapacitySlice(),
826 },
827 });
828 req.deinit();
829 req.* = new_req;
830 } else {
831 return 0;
832 }
833 }
825 assert(req.response.state.isContent());
826 if (req.response.done) return 0;
834827
835828 // var in: []const u8 = undefined;
836829 if (req.read_buffer_start == req.read_buffer_len) {
......@@ -851,7 +844,7 @@ pub const Request = struct {
851844 const data_avail = req.response.next_chunk_length;
852845 const out_avail = buffer.len;
853846
854 if (req.response.state.isContent() and req.response.headers.status.class() == .redirect) {
847 if (req.handle_redirects and req.response.headers.status.class() == .redirect) {
855848 const can_read = @intCast(usize, @min(buf_avail, data_avail));
856849 req.response.next_chunk_length -= can_read;
857850
......@@ -859,7 +852,6 @@ pub const Request = struct {
859852 req.client.release(req.connection);
860853 req.connection = undefined;
861854 req.response.done = true;
862 continue;
863855 }
864856
865857 return 0; // skip over as much data as possible
......@@ -943,7 +935,7 @@ pub const Request = struct {
943935 const data_avail = req.response.next_chunk_length;
944936 const out_avail = buffer.len - out_index;
945937
946 if (req.response.state.isContent() and req.response.headers.status.class() == .redirect) {
938 if (req.handle_redirects and req.response.headers.status.class() == .redirect) {
947939 const can_read = @intCast(usize, @min(buf_avail, data_avail));
948940 req.response.next_chunk_length -= can_read;
949941
......@@ -990,9 +982,41 @@ pub const Request = struct {
990982 }
991983
992984 pub fn read(req: *Request, buffer: []u8) ReadError!usize {
993 if (!req.response.state.isContent()) try req.waitForCompleteHead();
985 while (true) {
986 if (!req.response.state.isContent()) try req.waitForCompleteHead();
987
988 if (req.handle_redirects and req.response.headers.status.class() == .redirect) {
989 assert(try req.readRaw(buffer) == 0);
990
991 if (req.redirects_left == 0) return error.TooManyHttpRedirects;
992
993 const location = req.response.headers.location orelse
994 return error.HttpRedirectMissingLocation;
995 const new_url = Uri.parse(location) catch try Uri.parseWithoutScheme(location);
996
997 var new_arena = std.heap.ArenaAllocator.init(req.client.allocator);
998 const resolved_url = try req.uri.resolve(new_url, false, new_arena.allocator());
999 errdefer new_arena.deinit();
1000
1001 req.arena.deinit();
1002 req.arena = new_arena;
1003
1004 const new_req = try req.client.request(resolved_url, req.headers, .{
1005 .max_redirects = req.redirects_left - 1,
1006 .header_strategy = if (req.response.header_bytes_owned) .{
1007 .dynamic = req.response.max_header_bytes,
1008 } else .{
1009 .static = req.response.header_bytes.unusedCapacitySlice(),
1010 },
1011 });
1012 req.deinit();
1013 req.* = new_req;
1014 } else {
1015 break;
1016 }
1017 }
9941018
995 if (req.response.compression == .none and req.response.state.isContent()) {
1019 if (req.response.compression == .none) {
9961020 if (req.response.headers.transfer_compression) |compression| {
9971021 switch (compression) {
9981022 .compress => unreachable,
......@@ -1084,6 +1108,8 @@ pub const Request = struct {
10841108};
10851109
10861110pub fn deinit(client: *Client) void {
1111 client.connection_mutex.lock();
1112
10871113 var next = client.connection_pool.first;
10881114 while (next) |node| {
10891115 next = node.next;
......@@ -1106,7 +1132,7 @@ pub fn deinit(client: *Client) void {
11061132 client.* = undefined;
11071133}
11081134
1109pub const ConnectError = std.mem.Allocator.Error || std.net.TcpConnectToHostError || std.crypto.tls.Client.InitError(std.net.Stream);
1135pub const ConnectError = std.mem.Allocator.Error || net.TcpConnectToHostError || std.crypto.tls.Client.InitError(net.Stream);
11101136
11111137pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connection.Protocol) ConnectError!*ConnectionNode {
11121138 { // Search through the connection pool for a potential connection.
......@@ -1120,7 +1146,7 @@ pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connectio
11201146 const same_protocol = node.data.protocol == protocol;
11211147
11221148 if (same_host and same_port and same_protocol) {
1123 client.acquire(node);
1149 client.acquire(node, true);
11241150 return node;
11251151 }
11261152
......@@ -1168,6 +1194,7 @@ pub const RequestError = ConnectError || Connection.WriteError || error{
11681194 InvalidPadding,
11691195 MissingEndCertificateMarker,
11701196 Unseekable,
1197 EndOfStream,
11711198};
11721199
11731200pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Request.Options) RequestError!Request {
......@@ -1196,27 +1223,52 @@ pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Req
11961223 }
11971224
11981225 var req: Request = .{
1226 .uri = uri,
11991227 .client = client,
12001228 .headers = headers,
12011229 .connection = try client.connect(host, port, protocol),
12021230 .redirects_left = options.max_redirects,
1231 .handle_redirects = options.handle_redirects,
1232 .compression_init = false,
12031233 .response = switch (options.header_strategy) {
12041234 .dynamic => |max| Request.Response.initDynamic(max),
12051235 .static => |buf| Request.Response.initStatic(buf),
12061236 },
1237 .arena = undefined,
12071238 };
12081239
1240 req.arena = std.heap.ArenaAllocator.init(client.allocator);
1241
12091242 {
12101243 var buffered = std.io.bufferedWriter(req.connection.data.writer());
12111244 const writer = buffered.writer();
12121245
1246 const escaped_path = try Uri.escapePath(client.allocator, uri.path);
1247 defer client.allocator.free(escaped_path);
1248
1249 const escaped_query = if (uri.query) |q| try Uri.escapeQuery(client.allocator, q) else null;
1250 defer if (escaped_query) |q| client.allocator.free(q);
1251
1252 const escaped_fragment = if (uri.fragment) |f| try Uri.escapeQuery(client.allocator, f) else null;
1253 defer if (escaped_fragment) |f| client.allocator.free(f);
1254
12131255 try writer.writeAll(@tagName(headers.method));
12141256 try writer.writeByte(' ');
1215 try writer.writeAll(uri.path);
1257 try writer.writeAll(escaped_path);
1258 if (escaped_query) |q| {
1259 try writer.writeByte('?');
1260 try writer.writeAll(q);
1261 }
1262 if (escaped_fragment) |f| {
1263 try writer.writeByte('#');
1264 try writer.writeAll(f);
1265 }
12161266 try writer.writeByte(' ');
12171267 try writer.writeAll(@tagName(headers.version));
12181268 try writer.writeAll("\r\nHost: ");
12191269 try writer.writeAll(host);
1270 try writer.writeAll("\r\nUser-Agent: ");
1271 try writer.writeAll(headers.user_agent);
12201272 if (headers.connection == .close) {
12211273 try writer.writeAll("\r\nConnection: close");
12221274 } else {
lib/std/net.zig+3-3
......@@ -741,9 +741,9 @@ pub fn tcpConnectToAddress(address: Address) TcpConnectToAddressError!Stream {
741741 return Stream{ .handle = sockfd };
742742}
743743
744const GetAddressListError = std.mem.Allocator.Error || std.fs.File.OpenError || std.fs.File.ReadError || std.os.SocketError || std.os.BindError || error {
744const GetAddressListError = std.mem.Allocator.Error || std.fs.File.OpenError || std.fs.File.ReadError || std.os.SocketError || std.os.BindError || error{
745745 // TODO: break this up into error sets from the various underlying functions
746
746
747747 TemporaryNameServerFailure,
748748 NameServerFailure,
749749 AddressFamilyNotSupported,
......@@ -760,7 +760,7 @@ const GetAddressListError = std.mem.Allocator.Error || std.fs.File.OpenError ||
760760 Incomplete,
761761 InvalidIpv4Mapping,
762762 InvalidIPAddressFormat,
763
763
764764 InterfaceNotFound,
765765 FileSystem,
766766};