authorgravatar for truemedian@gmail.comNameless <truemedian@gmail.com> 2023-03-06 20:13:15-06:00
committergravatar for truemedian@gmail.comNameless <truemedian@gmail.com> 2023-03-09 14:54:26-06:00
logfd2f906d1ede2b65ba21eec59137b2d4b676eedc
tree11273d17254e2be90d947f8810e9e5d3e523eb6e
parent8d86194b6e31788263d2cbdd03e2a8cde4134c37
signature Commit is signed but in an unrecognized format.

std.http: handle compressed payloads

2 files changed, 496 insertions(+), 267 deletions(-)

lib/std/http.zig+10
...@@ -253,6 +253,16 @@ pub const TransferEncoding = enum {...@@ -253,6 +253,16 @@ pub const TransferEncoding = enum {
253 gzip,253 gzip,
254};254};
255255
256pub const Connection = enum {
257 keep_alive,
258 close,
259};
260
261pub const CustomHeader = struct {
262 name: []const u8,
263 value: []const u8,
264};
265
256const std = @import("std.zig");266const std = @import("std.zig");
257267
258test {268test {
lib/std/http/Client.zig+486-267
...@@ -21,27 +21,51 @@ ca_bundle: std.crypto.Certificate.Bundle = .{},...@@ -21,27 +21,51 @@ ca_bundle: std.crypto.Certificate.Bundle = .{},
21/// it will first rescan the system for root certificates.21/// it will first rescan the system for root certificates.
22next_https_rescan_certs: bool = true,22next_https_rescan_certs: bool = true,
2323
24connection_pool: std.TailQueue(Connection) = .{},24connection_mutex: std.Thread.Mutex = .{},
25connection_pool: ConnectionPool = .{},
26connection_used: ConnectionPool = .{},
2527
26const ConnectionPool = std.TailQueue(Connection);28const ConnectionPool = std.TailQueue(Connection);
27const ConnectionNode = ConnectionPool.Node;29const ConnectionNode = ConnectionPool.Node;
2830
31/// Acquires an existing connection from the connection pool. This function is threadsafe.
32pub fn acquire(client: *Client, node: *ConnectionNode) void {
33 client.connection_mutex.lock();
34 defer client.connection_mutex.unlock();
35
36 client.connection_pool.remove(node);
37 client.connection_used.append(node);
38}
39
40/// Tries to release a connection back to the connection pool. This function is threadsafe.
41/// If the connection is marked as closing, it will be closed instead.
29pub fn release(client: *Client, node: *ConnectionNode) void {42pub fn release(client: *Client, node: *ConnectionNode) void {
30 if (node.data.unusable) return node.data.close(client);43 if (node.data.closing) {
44 node.data.close(client);
45
46 return client.allocator.destroy(node);
47 }
48
49 client.connection_mutex.lock();
50 defer client.connection_mutex.unlock();
3151
52 client.connection_used.remove(node);
32 client.connection_pool.append(node);53 client.connection_pool.append(node);
33}54}
3455
56const DeflateDecompressor = std.compress.zlib.ZlibStream(Request.ReaderRaw);
57const GzipDecompressor = std.compress.gzip.Decompress(Request.ReaderRaw);
58
35pub const Connection = struct {59pub const Connection = struct {
36 stream: net.Stream,60 stream: net.Stream,
37 /// undefined unless protocol is tls.61 /// undefined unless protocol is tls.
38 tls_client: std.crypto.tls.Client, // TODO: allocate this, it's currently 16 KB.62 tls_client: *std.crypto.tls.Client, // TODO: allocate this, it's currently 16 KB.
39 protocol: Protocol,63 protocol: Protocol,
40 host: []u8,64 host: []u8,
41 port: u16,65 port: u16,
4266
43 // This connection has been part of a non keepalive request and cannot be added to the pool.67 // This connection has been part of a non keepalive request and cannot be added to the pool.
44 unusable: bool = false,68 closing: bool = false,
4569
46 pub const Protocol = enum { plain, tls };70 pub const Protocol = enum { plain, tls };
4771
...@@ -59,6 +83,24 @@ pub const Connection = struct {...@@ -59,6 +83,24 @@ pub const Connection = struct {
59 }83 }
60 }84 }
6185
86 pub const ReadError = std.net.Stream.ReadError || error{
87 TlsConnectionTruncated,
88 TlsRecordOverflow,
89 TlsDecodeError,
90 TlsAlert,
91 TlsBadRecordMac,
92 Overflow,
93 TlsBadLength,
94 TlsIllegalParameter,
95 TlsUnexpectedMessage,
96 };
97
98 pub const Reader = std.io.Reader(*Connection, ReadError, read);
99
100 pub fn reader(conn: *Connection) Reader {
101 return Reader{ .context = conn };
102 }
103
62 pub fn writeAll(conn: *Connection, buffer: []const u8) !void {104 pub fn writeAll(conn: *Connection, buffer: []const u8) !void {
63 switch (conn.protocol) {105 switch (conn.protocol) {
64 .plain => return conn.stream.writeAll(buffer),106 .plain => return conn.stream.writeAll(buffer),
...@@ -73,10 +115,18 @@ pub const Connection = struct {...@@ -73,10 +115,18 @@ pub const Connection = struct {
73 }115 }
74 }116 }
75117
118 pub const WriteError = std.net.Stream.WriteError || error{};
119 pub const Writer = std.io.Writer(*Connection, WriteError, write);
120
121 pub fn writer(conn: *Connection) Writer {
122 return Writer{ .context = conn };
123 }
124
76 pub fn close(conn: *Connection, client: *const Client) void {125 pub fn close(conn: *Connection, client: *const Client) void {
77 if (conn.protocol == .tls) {126 if (conn.protocol == .tls) {
78 // try to cleanly close the TLS connection, for any server that cares.127 // try to cleanly close the TLS connection, for any server that cares.
79 _ = conn.tls_client.writeEnd(conn.stream, "", true) catch {};128 _ = conn.tls_client.writeEnd(conn.stream, "", true) catch {};
129 client.allocator.destroy(conn.tls_client);
80 }130 }
81131
82 conn.stream.close();132 conn.stream.close();
...@@ -85,10 +135,10 @@ pub const Connection = struct {...@@ -85,10 +135,10 @@ pub const Connection = struct {
85 }135 }
86};136};
87137
88/// TODO: emit error.UnexpectedEndOfStream or something like that when the read
89/// data does not match the content length. This is necessary since HTTPS disables
90/// close_notify protection on underlying TLS streams.
91pub const Request = struct {138pub const Request = struct {
139 const read_buffer_size = 8192;
140 const ReadBufferIndex = std.math.IntFittingRange(0, read_buffer_size);
141
92 client: *Client,142 client: *Client,
93 connection: *ConnectionNode,143 connection: *ConnectionNode,
94 redirects_left: u32,144 redirects_left: u32,
...@@ -97,6 +147,11 @@ pub const Request = struct {...@@ -97,6 +147,11 @@ pub const Request = struct {
97 /// redirects.147 /// redirects.
98 headers: Headers,148 headers: Headers,
99149
150 /// Read buffer for the connection. This is used to pull in large amounts of data from the connection even if the user asks for a small amount. This can probably be removed with careful planning.
151 read_buffer: [read_buffer_size]u8 = undefined,
152 read_buffer_start: ReadBufferIndex = 0,
153 read_buffer_len: ReadBufferIndex = 0,
154
100 pub const Response = struct {155 pub const Response = struct {
101 headers: Response.Headers,156 headers: Response.Headers,
102 state: State,157 state: State,
...@@ -106,15 +161,24 @@ pub const Request = struct {...@@ -106,15 +161,24 @@ pub const Request = struct {
106 header_bytes: std.ArrayListUnmanaged(u8),161 header_bytes: std.ArrayListUnmanaged(u8),
107 max_header_bytes: usize,162 max_header_bytes: usize,
108 next_chunk_length: u64,163 next_chunk_length: u64,
109 done: bool,164 done: bool = false,
165
166 compression: union(enum) {
167 deflate: DeflateDecompressor,
168 gzip: GzipDecompressor,
169 none: void,
170 } = .none,
110171
111 pub const Headers = struct {172 pub const Headers = struct {
112 status: http.Status,173 status: http.Status,
113 version: http.Version,174 version: http.Version,
114 location: ?[]const u8 = null,175 location: ?[]const u8 = null,
115 content_length: ?u64 = null,176 content_length: ?u64 = null,
116 transfer_encoding: ?http.TransferEncoding = null,177 transfer_encoding: ?http.TransferEncoding = null, // This should only ever be chunked, compression is handled separately.
117 connection_close: bool = true,178 transfer_compression: ?http.TransferEncoding = null,
179 connection: http.Connection = .close,
180
181 number_of_headers: usize = 0,
118182
119 pub fn parse(bytes: []const u8) !Response.Headers {183 pub fn parse(bytes: []const u8) !Response.Headers {
120 var it = mem.split(u8, bytes[0 .. bytes.len - 4], "\r\n");184 var it = mem.split(u8, bytes[0 .. bytes.len - 4], "\r\n");
...@@ -137,6 +201,8 @@ pub const Request = struct {...@@ -137,6 +201,8 @@ pub const Request = struct {
137 };201 };
138202
139 while (it.next()) |line| {203 while (it.next()) |line| {
204 headers.number_of_headers += 1;
205
140 if (line.len == 0) return error.HttpHeadersInvalid;206 if (line.len == 0) return error.HttpHeadersInvalid;
141 switch (line[0]) {207 switch (line[0]) {
142 ' ', '\t' => return error.HttpHeaderContinuationsUnsupported,208 ' ', '\t' => return error.HttpHeaderContinuationsUnsupported,
...@@ -152,14 +218,65 @@ pub const Request = struct {...@@ -152,14 +218,65 @@ pub const Request = struct {
152 if (headers.content_length != null) return error.HttpHeadersInvalid;218 if (headers.content_length != null) return error.HttpHeadersInvalid;
153 headers.content_length = try std.fmt.parseInt(u64, header_value, 10);219 headers.content_length = try std.fmt.parseInt(u64, header_value, 10);
154 } else if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) {220 } else if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) {
155 if (headers.transfer_encoding != null) return error.HttpHeadersInvalid;221 if (headers.transfer_encoding != null or headers.transfer_compression != null) return error.HttpHeadersInvalid;
156 headers.transfer_encoding = std.meta.stringToEnum(http.TransferEncoding, header_value) orelse222
223 // Transfer-Encoding: second, first
224 // Transfer-Encoding: deflate, chunked
225 var iter = std.mem.splitBackwards(u8, header_value, ",");
226
227 if (iter.next()) |first| {
228 const kind = std.meta.stringToEnum(
229 http.TransferEncoding,
230 std.mem.trim(u8, first, " "),
231 ) orelse
232 return error.HttpTransferEncodingUnsupported;
233
234 switch (kind) {
235 .chunked => headers.transfer_encoding = .chunked,
236 .compress => headers.transfer_compression = .compress,
237 .deflate => headers.transfer_compression = .deflate,
238 .gzip => headers.transfer_compression = .gzip,
239 }
240 }
241
242 if (iter.next()) |second| {
243 if (headers.transfer_compression != null) return error.HttpTransferEncodingUnsupported;
244
245 const kind = std.meta.stringToEnum(
246 http.TransferEncoding,
247 std.mem.trim(u8, second, " "),
248 ) orelse
249 return error.HttpTransferEncodingUnsupported;
250
251 switch (kind) {
252 .chunked => return error.HttpHeadersInvalid, // chunked must come last
253 .compress => return error.HttpTransferEncodingUnsupported, // compress not supported
254 .deflate => headers.transfer_compression = .deflate,
255 .gzip => headers.transfer_compression = .gzip,
256 }
257 }
258
259 if (iter.next()) |_| return error.HttpTransferEncodingUnsupported;
260 } else if (std.ascii.eqlIgnoreCase(header_name, "content-encoding")) {
261 if (headers.transfer_compression != null) return error.HttpHeadersInvalid;
262
263 const kind = std.meta.stringToEnum(
264 http.TransferEncoding,
265 std.mem.trim(u8, header_value, " "),
266 ) orelse
157 return error.HttpTransferEncodingUnsupported;267 return error.HttpTransferEncodingUnsupported;
268
269 switch (kind) {
270 .chunked => return error.HttpHeadersInvalid, // not transfer encoding
271 .compress => return error.HttpTransferEncodingUnsupported, // compress not supported
272 .deflate => headers.transfer_compression = .deflate,
273 .gzip => headers.transfer_compression = .gzip,
274 }
158 } else if (std.ascii.eqlIgnoreCase(header_name, "connection")) {275 } else if (std.ascii.eqlIgnoreCase(header_name, "connection")) {
159 if (std.ascii.eqlIgnoreCase(header_value, "keep-alive")) {276 if (std.ascii.eqlIgnoreCase(header_value, "keep-alive")) {
160 headers.connection_close = false;277 headers.connection = .keep_alive;
161 } else if (std.ascii.eqlIgnoreCase(header_value, "close")) {278 } else if (std.ascii.eqlIgnoreCase(header_value, "close")) {
162 headers.connection_close = true;279 headers.connection = .close;
163 } else {280 } else {
164 return error.HttpConnectionHeaderUnsupported;281 return error.HttpConnectionHeaderUnsupported;
165 }282 }
...@@ -238,7 +355,6 @@ pub const Request = struct {...@@ -238,7 +355,6 @@ pub const Request = struct {
238 .max_header_bytes = max,355 .max_header_bytes = max,
239 .header_bytes_owned = true,356 .header_bytes_owned = true,
240 .next_chunk_length = undefined,357 .next_chunk_length = undefined,
241 .done = false,
242 };358 };
243 }359 }
244360
...@@ -250,7 +366,6 @@ pub const Request = struct {...@@ -250,7 +366,6 @@ pub const Request = struct {
250 .max_header_bytes = buf.len,366 .max_header_bytes = buf.len,
251 .header_bytes_owned = false,367 .header_bytes_owned = false,
252 .next_chunk_length = undefined,368 .next_chunk_length = undefined,
253 .done = false,
254 };369 };
255 }370 }
256371
...@@ -537,10 +652,19 @@ pub const Request = struct {...@@ -537,10 +652,19 @@ pub const Request = struct {
537 }652 }
538 };653 };
539654
655 pub const RequestTransfer = union(enum) {
656 content_length: u64,
657 chunked: void,
658 none: void,
659 };
660
540 pub const Headers = struct {661 pub const Headers = struct {
541 version: http.Version = .@"HTTP/1.1",662 version: http.Version = .@"HTTP/1.1",
542 method: http.Method = .GET,663 method: http.Method = .GET,
543 connection_close: bool = false,664 connection: http.Connection = .keep_alive,
665 transfer_encoding: RequestTransfer = .none,
666
667 custom: []const http.CustomHeader = &[_]http.CustomHeader{},
544 };668 };
545669
546 pub const Options = struct {670 pub const Options = struct {
...@@ -561,167 +685,131 @@ pub const Request = struct {...@@ -561,167 +685,131 @@ pub const Request = struct {
561 };685 };
562 };686 };
563687
564 /// May be skipped if header strategy is buffer.688 /// Frees all resources associated with the request.
565 pub fn deinit(req: *Request) void {689 pub fn deinit(req: *Request) void {
690 switch (req.response.compression) {
691 .none => {},
692 .deflate => |*deflate| deflate.deinit(),
693 .gzip => |*gzip| gzip.deinit(),
694 }
695
566 if (req.response.header_bytes_owned) {696 if (req.response.header_bytes_owned) {
567 req.response.header_bytes.deinit(req.client.allocator);697 req.response.header_bytes.deinit(req.client.allocator);
568 }698 }
699
700 if (!req.response.done) {
701 // If the response wasn't fully read, then we need to close the connection.
702 req.connection.data.closing = true;
703 req.client.release(req.connection);
704 }
705
569 req.* = undefined;706 req.* = undefined;
570 }707 }
571708
572 pub const Reader = std.io.Reader(*Request, ReadError, read);709 const ReadRawError = Connection.ReadError || std.Uri.ParseError || RequestError || error{
710 UnexpectedEndOfStream,
711 TooManyHttpRedirects,
712 HttpRedirectMissingLocation,
713 HttpHeadersInvalid,
714 };
573715
574 pub fn reader(req: *Request) Reader {716 const ReaderRaw = std.io.Reader(*Request, ReadRawError, readRaw);
575 return .{ .context = req };717
718 /// Read from the underlying stream, without decompressing or parsing the headers. Must be called
719 /// after waitForCompleteHead() has returned successfully.
720 pub fn readRaw(req: *Request, buffer: []u8) ReadRawError!usize {
721 assert(req.response.state.isContent());
722
723 var index: usize = 0;
724 while (index == 0) {
725 const amt = try req.readRawAdvanced(buffer[index..]);
726 const zero_means_end = req.response.done and req.response.headers.status.class() != .redirect;
727
728 if (amt == 0 and zero_means_end) break;
729 index += amt;
730 }
731
732 return index;
576 }733 }
577734
578 pub fn readAll(req: *Request, buffer: []u8) !usize {735 fn checkForCompleteHead(req: *Request, buffer: []u8) !usize {
579 return readAtLeast(req, buffer, buffer.len);736 switch (req.response.state) {
737 .invalid => unreachable,
738 .start, .seen_r, .seen_rn, .seen_rnr => {},
739 else => return 0, // No more headers to read.
740 }
741
742 const i = req.response.findHeadersEnd(buffer[0..]);
743 if (req.response.state == .invalid) return error.HttpHeadersInvalid;
744
745 const headers_data = buffer[0..i];
746 if (req.response.header_bytes.items.len + headers_data.len > req.response.max_header_bytes) {
747 return error.HttpHeadersExceededSizeLimit;
748 }
749 try req.response.header_bytes.appendSlice(req.client.allocator, headers_data);
750
751 if (req.response.state == .finished) {
752 req.response.headers = try Response.Headers.parse(req.response.header_bytes.items);
753
754 if (req.response.headers.connection == .keep_alive) {
755 req.connection.data.closing = false;
756 } else {
757 req.connection.data.closing = true;
758 }
759
760 if (req.response.headers.transfer_encoding) |transfer_encoding| {
761 switch (transfer_encoding) {
762 .chunked => {
763 req.response.next_chunk_length = 0;
764 req.response.state = .chunk_size;
765 },
766 .compress => unreachable,
767 .deflate => unreachable,
768 .gzip => unreachable,
769 }
770 } else if (req.response.headers.content_length) |content_length| {
771 req.response.next_chunk_length = content_length;
772 } else {
773 req.response.done = true;
774 }
775
776 return i;
777 }
778
779 return 0;
580 }780 }
581781
582 pub const ReadError = net.Stream.ReadError || error{782 pub const WaitForCompleteHeadError = ReadRawError || error {
583 // From HTTP protocol783 UnexpectedEndOfStream,
584 HttpHeadersInvalid,784
585 HttpHeadersExceededSizeLimit,785 HttpHeadersExceededSizeLimit,
586 HttpRedirectMissingLocation,
587 HttpTransferEncodingUnsupported,
588 HttpConnectionHeaderUnsupported,
589 HttpContentLengthUnknown,
590 TooManyHttpRedirects,
591 ShortHttpStatusLine,786 ShortHttpStatusLine,
592 BadHttpVersion,787 BadHttpVersion,
593 HttpHeaderContinuationsUnsupported,788 HttpHeaderContinuationsUnsupported,
594 UnsupportedUrlScheme,789 HttpTransferEncodingUnsupported,
595 UriMissingHost,790 HttpConnectionHeaderUnsupported,
596 UnknownHostName,
597
598 // Network problems
599 NetworkUnreachable,
600 HostLacksNetworkAddresses,
601 TemporaryNameServerFailure,
602 NameServerFailure,
603 ProtocolFamilyNotAvailable,
604 ProtocolNotSupported,
605
606 // System resource problems
607 ProcessFdQuotaExceeded,
608 SystemFdQuotaExceeded,
609 OutOfMemory,
610
611 // TLS problems
612 InsufficientEntropy,
613 TlsConnectionTruncated,
614 TlsRecordOverflow,
615 TlsDecodeError,
616 TlsAlert,
617 TlsBadRecordMac,
618 TlsBadLength,
619 TlsIllegalParameter,
620 TlsUnexpectedMessage,
621 TlsDecryptFailure,
622 CertificateFieldHasInvalidLength,
623 CertificateHostMismatch,
624 CertificatePublicKeyInvalid,
625 CertificateExpired,
626 CertificateFieldHasWrongDataType,
627 CertificateIssuerMismatch,
628 CertificateNotYetValid,
629 CertificateSignatureAlgorithmMismatch,
630 CertificateSignatureAlgorithmUnsupported,
631 CertificateSignatureInvalid,
632 CertificateSignatureInvalidLength,
633 CertificateSignatureNamedCurveUnsupported,
634 CertificateSignatureUnsupportedBitCount,
635 TlsCertificateNotVerified,
636 TlsBadSignatureScheme,
637 TlsBadRsaSignatureBitCount,
638 TlsDecryptError,
639 UnsupportedCertificateVersion,
640 CertificateTimeInvalid,
641 CertificateHasUnrecognizedObjectId,
642 CertificateHasInvalidBitString,
643 CertificateAuthorityBundleTooBig,
644
645 // TODO: convert to higher level errors
646 InvalidFormat,
647 InvalidPort,
648 UnexpectedCharacter,
649 Overflow,
650 InvalidCharacter,
651 AddressFamilyNotSupported,
652 AddressInUse,
653 AddressNotAvailable,
654 ConnectionPending,
655 ConnectionRefused,
656 FileNotFound,
657 PermissionDenied,
658 ServiceUnavailable,
659 SocketTypeNotSupported,
660 FileTooBig,
661 LockViolation,
662 NoSpaceLeft,
663 NotOpenForWriting,
664 InvalidEncoding,
665 IdentityElement,
666 NonCanonical,
667 SignatureVerificationFailed,
668 MessageTooLong,
669 NegativeIntoUnsigned,
670 TargetTooSmall,
671 BufferTooSmall,
672 InvalidSignature,
673 NotSquare,
674 DiskQuota,
675 InvalidEnd,
676 Incomplete,
677 InvalidIpv4Mapping,
678 InvalidIPAddressFormat,
679 BadPathName,
680 DeviceBusy,
681 FileBusy,
682 FileLocksNotSupported,
683 InvalidHandle,
684 InvalidUtf8,
685 NameTooLong,
686 NoDevice,
687 PathAlreadyExists,
688 PipeBusy,
689 SharingViolation,
690 SymLinkLoop,
691 FileSystem,
692 InterfaceNotFound,
693 AlreadyBound,
694 FileDescriptorNotASocket,
695 NetworkSubsystemFailed,
696 NotDir,
697 ReadOnlyFileSystem,
698 Unseekable,
699 MissingEndCertificateMarker,
700 InvalidPadding,
701 EndOfStream,
702 InvalidArgument,
703 };791 };
704792
705 pub fn read(req: *Request, buffer: []u8) ReadError!usize {793 /// Reads a complete response head. Any leftover data is stored in the request. This function is idempotent.
706 return readAtLeast(req, buffer, 1);794 pub fn waitForCompleteHead(req: *Request) WaitForCompleteHeadError!void {
707 }795 if (req.response.state.isContent()) return;
708
709 pub fn readAtLeast(req: *Request, buffer: []u8, len: usize) !usize {
710 assert(len <= buffer.len);
711 var index: usize = 0;
712 while (index < len) {
713 const amt = try readAdvanced(req, buffer[index..]);
714 const zero_means_end = req.response.done and req.response.headers.status.class() != .redirect;
715796
716 if (amt == 0 and zero_means_end) break;797 while (true) {
717 index += amt;798 const nread = try req.connection.data.read(req.read_buffer[0..]);
799 const amt = try checkForCompleteHead(req, req.read_buffer[0..nread]);
800
801 if (amt != 0) {
802 req.read_buffer_start = @intCast(ReadBufferIndex, amt);
803 req.read_buffer_len = @intCast(ReadBufferIndex, nread);
804 return;
805 } else if (nread == 0) {
806 return error.UnexpectedEndOfStream;
807 }
718 }808 }
719 return index;
720 }809 }
721810
722 /// This one can return 0 without meaning EOF.811 /// This one can return 0 without meaning EOF.
723 /// TODO change to readvAdvanced812 fn readRawAdvanced(req: *Request, buffer: []u8) !usize {
724 pub fn readAdvanced(req: *Request, buffer: []u8) !usize {
725 if (req.response.done) {813 if (req.response.done) {
726 if (req.response.headers.status.class() == .redirect) {814 if (req.response.headers.status.class() == .redirect) {
727 if (req.redirects_left == 0) return error.TooManyHttpRedirects;815 if (req.redirects_left == 0) return error.TooManyHttpRedirects;
...@@ -744,82 +832,56 @@ pub const Request = struct {...@@ -744,82 +832,56 @@ pub const Request = struct {
744 }832 }
745 }833 }
746834
747 var in = buffer[0..try req.connection.data.read(buffer)];835 // var in: []const u8 = undefined;
836 if (req.read_buffer_start == req.read_buffer_len) {
837 const nread = try req.connection.data.read(req.read_buffer[0..]);
838 if (nread == 0) return error.UnexpectedEndOfStream;
839
840 req.read_buffer_start = 0;
841 req.read_buffer_len = @intCast(ReadBufferIndex, nread);
842 }
843
748 var out_index: usize = 0;844 var out_index: usize = 0;
749 while (true) {845 while (true) {
750 switch (req.response.state) {846 switch (req.response.state) {
751 .invalid => unreachable,847 .invalid, .start, .seen_r, .seen_rn, .seen_rnr => unreachable,
752 .start, .seen_r, .seen_rn, .seen_rnr => {848 .finished => {
753 const i = req.response.findHeadersEnd(in);849 // TODO https://github.com/ziglang/zig/issues/14039
754 if (req.response.state == .invalid) return error.HttpHeadersInvalid;850 const buf_avail = req.read_buffer_len - req.read_buffer_start;
755851 const data_avail = req.response.next_chunk_length;
756 const headers_data = in[0..i];852 const out_avail = buffer.len;
757 if (req.response.header_bytes.items.len + headers_data.len > req.response.max_header_bytes) {853
758 return error.HttpHeadersExceededSizeLimit;854 if (req.response.state.isContent() and req.response.headers.status.class() == .redirect) {
759 }855 const can_read = @intCast(usize, @min(buf_avail, data_avail));
760 try req.response.header_bytes.appendSlice(req.client.allocator, headers_data);856 req.response.next_chunk_length -= can_read;
761857
762 if (req.response.state == .finished) {858 if (req.response.next_chunk_length == 0) {
763 req.response.headers = try Response.Headers.parse(req.response.header_bytes.items);859 req.client.release(req.connection);
764860 req.connection = undefined;
765 if (req.response.headers.connection_close == true) {861 req.response.done = true;
766 req.connection.data.unusable = true;862 continue;
767 } else {
768 req.connection.data.unusable = false;
769 }
770
771 if (req.response.headers.transfer_encoding) |transfer_encoding| {
772 switch (transfer_encoding) {
773 .chunked => {
774 req.response.next_chunk_length = 0;
775 req.response.state = .chunk_size;
776 },
777 .compress => return error.HttpTransferEncodingUnsupported,
778 .deflate => return error.HttpTransferEncodingUnsupported,
779 .gzip => return error.HttpTransferEncodingUnsupported,
780 }
781 } else if (req.response.headers.content_length) |content_length| {
782 req.response.next_chunk_length = content_length;
783 } else {
784 return error.HttpContentLengthUnknown;
785 }863 }
786864
787 in = in[i..];865 return 0; // skip over as much data as possible
788 continue;
789 }866 }
790867
791 assert(out_index == 0);868 const can_read = @intCast(usize, @min(@min(buf_avail, data_avail), out_avail));
792 return 0;869 req.response.next_chunk_length -= can_read;
793 },870
794 .finished => {871 mem.copy(u8, buffer[0..], req.read_buffer[req.read_buffer_start..][0..can_read]);
795 const sub_amt = @intCast(usize, @min(req.response.next_chunk_length, in.len));872 req.read_buffer_start += @intCast(ReadBufferIndex, can_read);
796 req.response.next_chunk_length -= sub_amt;
797873
798 if (req.response.next_chunk_length == 0) {874 if (req.response.next_chunk_length == 0) {
799 req.client.release(req.connection);875 req.client.release(req.connection);
800 req.connection = undefined;876 req.connection = undefined;
801
802 req.response.done = true;877 req.response.done = true;
803 assert(in.len == sub_amt); // TODO: figure out how to not read more than necessary.
804
805 if (req.response.state.isContent() and req.response.headers.status.class() == .redirect) return 0;
806
807 mem.copy(u8, buffer[out_index..], in[0..sub_amt]);
808 return out_index + sub_amt;
809 }878 }
810879
811 if (req.response.state.isContent() and req.response.headers.status.class() == .redirect) return 0;880 return can_read;
812
813 if (in.ptr == buffer.ptr) {
814 return sub_amt;
815 } else {
816 mem.copy(u8, buffer[out_index..], in[0..sub_amt]);
817 return out_index + sub_amt;
818 }
819 },881 },
820 .chunk_size_prefix_r => switch (in.len) {882 .chunk_size_prefix_r => switch (req.read_buffer_len - req.read_buffer_start) {
821 0 => return out_index,883 0 => return out_index,
822 1 => switch (in[0]) {884 1 => switch (req.read_buffer[req.read_buffer_start]) {
823 '\r' => {885 '\r' => {
824 req.response.state = .chunk_size_prefix_n;886 req.response.state = .chunk_size_prefix_n;
825 return out_index;887 return out_index;
...@@ -829,9 +891,9 @@ pub const Request = struct {...@@ -829,9 +891,9 @@ pub const Request = struct {
829 return error.HttpHeadersInvalid;891 return error.HttpHeadersInvalid;
830 },892 },
831 },893 },
832 else => switch (int16(in[0..2])) {894 else => switch (int16(req.read_buffer[req.read_buffer_start..][0..2])) {
833 int16("\r\n") => {895 int16("\r\n") => {
834 in = in[2..];896 req.read_buffer_start += 2;
835 req.response.state = .chunk_size;897 req.response.state = .chunk_size;
836 continue;898 continue;
837 },899 },
...@@ -841,11 +903,11 @@ pub const Request = struct {...@@ -841,11 +903,11 @@ pub const Request = struct {
841 },903 },
842 },904 },
843 },905 },
844 .chunk_size_prefix_n => switch (in.len) {906 .chunk_size_prefix_n => switch (req.read_buffer_len - req.read_buffer_start) {
845 0 => return out_index,907 0 => return out_index,
846 else => switch (in[0]) {908 else => switch (req.read_buffer[req.read_buffer_start]) {
847 '\n' => {909 '\n' => {
848 in = in[1..];910 req.read_buffer_start += 1;
849 req.response.state = .chunk_size;911 req.response.state = .chunk_size;
850 continue;912 continue;
851 },913 },
...@@ -856,7 +918,7 @@ pub const Request = struct {...@@ -856,7 +918,7 @@ pub const Request = struct {
856 },918 },
857 },919 },
858 .chunk_size, .chunk_r => {920 .chunk_size, .chunk_r => {
859 const i = req.response.findChunkedLen(in);921 const i = req.response.findChunkedLen(req.read_buffer[req.read_buffer_start..req.read_buffer_len]);
860 switch (req.response.state) {922 switch (req.response.state) {
861 .invalid => return error.HttpHeadersInvalid,923 .invalid => return error.HttpHeadersInvalid,
862 .chunk_data => {924 .chunk_data => {
...@@ -867,7 +929,8 @@ pub const Request = struct {...@@ -867,7 +929,8 @@ pub const Request = struct {
867929
868 return out_index;930 return out_index;
869 }931 }
870 in = in[i..];932
933 req.read_buffer_start += @intCast(ReadBufferIndex, i);
871 continue;934 continue;
872 },935 },
873 .chunk_size => return out_index,936 .chunk_size => return out_index,
...@@ -876,34 +939,129 @@ pub const Request = struct {...@@ -876,34 +939,129 @@ pub const Request = struct {
876 },939 },
877 .chunk_data => {940 .chunk_data => {
878 // TODO https://github.com/ziglang/zig/issues/14039941 // TODO https://github.com/ziglang/zig/issues/14039
879 const sub_amt = @intCast(usize, @min(req.response.next_chunk_length, in.len));942 const buf_avail = req.read_buffer_len - req.read_buffer_start;
880 req.response.next_chunk_length -= sub_amt;943 const data_avail = req.response.next_chunk_length;
944 const out_avail = buffer.len - out_index;
945
946 if (req.response.state.isContent() and req.response.headers.status.class() == .redirect) {
947 const can_read = @intCast(usize, @min(buf_avail, data_avail));
948 req.response.next_chunk_length -= can_read;
949
950 if (req.response.next_chunk_length == 0) {
951 req.client.release(req.connection);
952 req.connection = undefined;
953 req.response.done = true;
954 continue;
955 }
956
957 return 0; // skip over as much data as possible
958 }
959
960 const can_read = @intCast(usize, @min(@min(buf_avail, data_avail), out_avail));
961 req.response.next_chunk_length -= can_read;
962
963 mem.copy(u8, buffer[out_index..], req.read_buffer[req.read_buffer_start..][0..can_read]);
964 req.read_buffer_start += @intCast(ReadBufferIndex, can_read);
965 out_index += can_read;
881966
882 if (req.response.next_chunk_length == 0) {967 if (req.response.next_chunk_length == 0) {
883 req.response.state = .chunk_size_prefix_r;968 req.response.state = .chunk_size_prefix_r;
884 in = in[sub_amt..];
885
886 if (req.response.headers.status.class() == .redirect) continue;
887969
888 mem.copy(u8, buffer[out_index..], in[0..sub_amt]);
889 out_index += sub_amt;
890 continue;970 continue;
891 }971 }
892972
893 if (req.response.headers.status.class() == .redirect) return 0;973 return out_index;
894
895 if (in.ptr == buffer.ptr) {
896 return sub_amt;
897 } else {
898 mem.copy(u8, buffer[out_index..], in[0..sub_amt]);
899 out_index += sub_amt;
900 return out_index;
901 }
902 },974 },
903 }975 }
904 }976 }
905 }977 }
906978
979 pub const ReadError = DeflateDecompressor.Error || GzipDecompressor.Error || WaitForCompleteHeadError || error{
980 BadHeader,
981 InvalidCompression,
982 StreamTooLong,
983 InvalidWindowSize,
984 };
985
986 pub const Reader = std.io.Reader(*Request, ReadError, read);
987
988 pub fn reader(req: *Request) Reader {
989 return .{ .context = req };
990 }
991
992 pub fn read(req: *Request, buffer: []u8) ReadError!usize {
993 if (!req.response.state.isContent()) try req.waitForCompleteHead();
994
995 if (req.response.compression == .none and req.response.state.isContent()) {
996 if (req.response.headers.transfer_compression) |compression| {
997 switch (compression) {
998 .compress => unreachable,
999 .deflate => req.response.compression = .{
1000 .deflate = try std.compress.zlib.zlibStream(req.client.allocator, ReaderRaw{ .context = req }),
1001 },
1002 .gzip => req.response.compression = .{
1003 .gzip = try std.compress.gzip.decompress(req.client.allocator, ReaderRaw{ .context = req }),
1004 },
1005 .chunked => unreachable,
1006 }
1007 }
1008 }
1009
1010 return switch (req.response.compression) {
1011 .deflate => |*deflate| try deflate.read(buffer),
1012 .gzip => |*gzip| try gzip.read(buffer),
1013 else => try req.readRaw(buffer),
1014 };
1015 }
1016
1017 pub fn readAll(req: *Request, buffer: []u8) !usize {
1018 var index: usize = 0;
1019 while (index < buffer.len) {
1020 const amt = try read(req, buffer[index..]);
1021 if (amt == 0) break;
1022 index += amt;
1023 }
1024 return index;
1025 }
1026
1027 pub const WriteError = Connection.WriteError || error{MessageTooLong};
1028
1029 pub const Writer = std.io.Writer(*Request, WriteError, write);
1030
1031 pub fn writer(req: *Request) Writer {
1032 return .{ .context = req };
1033 }
1034
1035 /// Write `bytes` to the server. The `transfer_encoding` request header determines how data will be sent.
1036 pub fn write(req: *Request, bytes: []const u8) !usize {
1037 switch (req.headers.transfer_encoding) {
1038 .chunked => {
1039 try req.connection.data.writer().print("{x}\r\n", .{bytes.len});
1040 try req.connection.data.writeAll(bytes);
1041 try req.connection.data.writeAll("\r\n");
1042
1043 return bytes.len;
1044 },
1045 .content_length => |*len| {
1046 if (len.* < bytes.len) return error.MessageTooLong;
1047
1048 const amt = try req.connection.data.write(bytes);
1049 len.* -= amt;
1050 return amt;
1051 },
1052 .none => return error.NotWriteable,
1053 }
1054 }
1055
1056 /// Finish the body of a request. This notifies the server that you have no more data to send.
1057 pub fn finish(req: *Request) !void {
1058 switch (req.headers.transfer_encoding) {
1059 .chunked => try req.connection.data.writeAll("0\r\n"),
1060 .content_length => |len| if (len != 0) return error.MessageNotCompleted,
1061 .none => {},
1062 }
1063 }
1064
907 inline fn int16(array: *const [2]u8) u16 {1065 inline fn int16(array: *const [2]u8) u16 {
908 return @bitCast(u16, array.*);1066 return @bitCast(u16, array.*);
909 }1067 }
...@@ -917,6 +1075,10 @@ pub const Request = struct {...@@ -917,6 +1075,10 @@ pub const Request = struct {
917 }1075 }
9181076
919 test {1077 test {
1078 const builtin = @import("builtin");
1079
1080 if (builtin.os.tag == .wasi) return error.SkipZigTest;
1081
920 _ = Response;1082 _ = Response;
921 }1083 }
922};1084};
...@@ -931,23 +1093,39 @@ pub fn deinit(client: *Client) void {...@@ -931,23 +1093,39 @@ pub fn deinit(client: *Client) void {
931 client.allocator.destroy(node);1093 client.allocator.destroy(node);
932 }1094 }
9331095
1096 next = client.connection_used.first;
1097 while (next) |node| {
1098 next = node.next;
1099
1100 node.data.close(client);
1101
1102 client.allocator.destroy(node);
1103 }
1104
934 client.ca_bundle.deinit(client.allocator);1105 client.ca_bundle.deinit(client.allocator);
935 client.* = undefined;1106 client.* = undefined;
936}1107}
9371108
938pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connection.Protocol) !*ConnectionNode {1109pub const ConnectError = std.mem.Allocator.Error || std.net.TcpConnectToHostError || std.crypto.tls.Client.InitError(std.net.Stream);
939 var potential = client.connection_pool.last;
940 while (potential) |node| {
941 const same_host = mem.eql(u8, node.data.host, host);
942 const same_port = node.data.port == port;
943 const same_protocol = node.data.protocol == protocol;
9441110
945 if (same_host and same_port and same_protocol) {1111pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connection.Protocol) ConnectError!*ConnectionNode {
946 client.connection_pool.remove(node);1112 { // Search through the connection pool for a potential connection.
947 return node;1113 client.connection_mutex.lock();
948 }1114 defer client.connection_mutex.unlock();
9491115
950 potential = node.prev;1116 var potential = client.connection_pool.last;
1117 while (potential) |node| {
1118 const same_host = mem.eql(u8, node.data.host, host);
1119 const same_port = node.data.port == port;
1120 const same_protocol = node.data.protocol == protocol;
1121
1122 if (same_host and same_port and same_protocol) {
1123 client.acquire(node);
1124 return node;
1125 }
1126
1127 potential = node.prev;
1128 }
951 }1129 }
9521130
953 const conn = try client.allocator.create(ConnectionNode);1131 const conn = try client.allocator.create(ConnectionNode);
...@@ -964,17 +1142,35 @@ pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connectio...@@ -964,17 +1142,35 @@ pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connectio
964 switch (protocol) {1142 switch (protocol) {
965 .plain => {},1143 .plain => {},
966 .tls => {1144 .tls => {
967 conn.data.tls_client = try std.crypto.tls.Client.init(conn.data.stream, client.ca_bundle, host);1145 conn.data.tls_client = try client.allocator.create(std.crypto.tls.Client);
1146 conn.data.tls_client.* = try std.crypto.tls.Client.init(conn.data.stream, client.ca_bundle, host);
968 // This is appropriate for HTTPS because the HTTP headers contain1147 // This is appropriate for HTTPS because the HTTP headers contain
969 // the content length which is used to detect truncation attacks.1148 // the content length which is used to detect truncation attacks.
970 conn.data.tls_client.allow_truncation_attacks = true;1149 conn.data.tls_client.allow_truncation_attacks = true;
971 },1150 },
972 }1151 }
9731152
1153 {
1154 client.connection_mutex.lock();
1155 defer client.connection_mutex.unlock();
1156
1157 client.connection_used.append(conn);
1158 }
1159
974 return conn;1160 return conn;
975}1161}
9761162
977pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Request.Options) !Request {1163pub const RequestError = ConnectError || Connection.WriteError || error{
1164 UnsupportedUrlScheme,
1165 UriMissingHost,
1166
1167 CertificateAuthorityBundleTooBig,
1168 InvalidPadding,
1169 MissingEndCertificateMarker,
1170 Unseekable,
1171};
1172
1173pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Request.Options) RequestError!Request {
978 const protocol: Connection.Protocol = if (mem.eql(u8, uri.scheme, "http"))1174 const protocol: Connection.Protocol = if (mem.eql(u8, uri.scheme, "http"))
979 .plain1175 .plain
980 else if (mem.eql(u8, uri.scheme, "https"))1176 else if (mem.eql(u8, uri.scheme, "https"))
...@@ -990,8 +1186,13 @@ pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Req...@@ -990,8 +1186,13 @@ pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Req
990 const host = uri.host orelse return error.UriMissingHost;1186 const host = uri.host orelse return error.UriMissingHost;
9911187
992 if (client.next_https_rescan_certs and protocol == .tls) {1188 if (client.next_https_rescan_certs and protocol == .tls) {
993 try client.ca_bundle.rescan(client.allocator);1189 client.connection_mutex.lock(); // TODO: this could be so much better than reusing the connection pool mutex.
994 client.next_https_rescan_certs = false;1190 defer client.connection_mutex.unlock();
1191
1192 if (client.next_https_rescan_certs) {
1193 try client.ca_bundle.rescan(client.allocator);
1194 client.next_https_rescan_certs = false;
1195 }
995 }1196 }
9961197
997 var req: Request = .{1198 var req: Request = .{
...@@ -1006,23 +1207,39 @@ pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Req...@@ -1006,23 +1207,39 @@ pub fn request(client: *Client, uri: Uri, headers: Request.Headers, options: Req
1006 };1207 };
10071208
1008 {1209 {
1009 var h = try std.BoundedArray(u8, 1000).init(0);1210 var buffered = std.io.bufferedWriter(req.connection.data.writer());
1010 try h.appendSlice(@tagName(headers.method));1211 const writer = buffered.writer();
1011 try h.appendSlice(" ");1212
1012 try h.appendSlice(uri.path);1213 try writer.writeAll(@tagName(headers.method));
1013 try h.appendSlice(" ");1214 try writer.writeByte(' ');
1014 try h.appendSlice(@tagName(headers.version));1215 try writer.writeAll(uri.path);
1015 try h.appendSlice("\r\nHost: ");1216 try writer.writeByte(' ');
1016 try h.appendSlice(host);1217 try writer.writeAll(@tagName(headers.version));
1017 if (headers.connection_close) {1218 try writer.writeAll("\r\nHost: ");
1018 try h.appendSlice("\r\nConnection: close");1219 try writer.writeAll(host);
1220 if (headers.connection == .close) {
1221 try writer.writeAll("\r\nConnection: close");
1019 } else {1222 } else {
1020 try h.appendSlice("\r\nConnection: keep-alive");1223 try writer.writeAll("\r\nConnection: keep-alive");
1021 }1224 }
1022 try h.appendSlice("\r\n\r\n");1225 try writer.writeAll("\r\nAccept-Encoding: gzip, deflate");
10231226
1024 const header_bytes = h.slice();1227 switch (headers.transfer_encoding) {
1025 try req.connection.data.writeAll(header_bytes);1228 .chunked => try writer.writeAll("\r\nTransfer-Encoding: chunked"),
1229 .content_length => |content_length| try writer.print("\r\nContent-Length: {d}", .{content_length}),
1230 .none => {},
1231 }
1232
1233 for (headers.custom) |header| {
1234 try writer.writeAll("\r\n");
1235 try writer.writeAll(header.name);
1236 try writer.writeAll(": ");
1237 try writer.writeAll(header.value);
1238 }
1239
1240 try writer.writeAll("\r\n\r\n");
1241
1242 try buffered.flush();
1026 }1243 }
10271244
1028 return req;1245 return req;
...@@ -1036,5 +1253,7 @@ test {...@@ -1036,5 +1253,7 @@ test {
1036 return error.SkipZigTest;1253 return error.SkipZigTest;
1037 }1254 }
10381255
1256 if (builtin.os.tag == .wasi) return error.SkipZigTest;
1257
1039 _ = Request;1258 _ = Request;
1040}1259}