authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-12-28 16:37:22-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-01-02 16:57:16-07:00
log940d368e7ea95d2bb8185e71af3d1ec0328917dc
treea2338ec0d47f74c0badfe0a22d2e77e5044e3082
parent21ab99174eabc9ae8efa2b19890d9cab51773b35

std.crypto.tls.Client: fix the read function

The read function has been renamed to readAdvanced since it has slightly different semantics than typical read functions, specifically regarding the end-of-file. A higher level read function is implemented on top. Now, API users may pass small buffers to the read function and everything will work fine. This is done by re-decrypting the same ciphertext record with each call to read() until the record is finished being transmitted. If the buffer supplied to read() is large enough, then any given ciphertext record will only be decrypted once, since it decrypts directly to the read() buffer and therefore does not need any memcpy. On the other hand, if the buffer supplied to read() is small, then the ciphertext is decrypted into a stack buffer, a subset is copied to the read() buffer, and then the entire ciphertext record is saved for the next call to read().

3 files changed, 136 insertions(+), 46 deletions(-)

lib/std/crypto/tls/Client.zig+129-34
......@@ -18,14 +18,20 @@ const array = tls.array;
1818const enum_array = tls.enum_array;
1919const Certificate = crypto.Certificate;
2020
21application_cipher: ApplicationCipher,
2221read_seq: u64,
2322write_seq: u64,
24/// The size is enough to contain exactly one TLSCiphertext record.
25partially_read_buffer: [tls.max_ciphertext_record_len]u8,
2623/// The number of partially read bytes inside `partially_read_buffer`.
2724partially_read_len: u15,
25/// The number of cleartext bytes from decoding `partially_read_buffer` which
26/// have already been transferred via read() calls. This implementation will
27/// re-decrypt bytes from `partially_read_buffer` when the buffer supplied by
28/// the read() API user is not large enough.
29partial_cleartext_index: u15,
30application_cipher: ApplicationCipher,
2831eof: bool,
32/// The size is enough to contain exactly one TLSCiphertext record.
33/// Contains encrypted bytes.
34partially_read_buffer: [tls.max_ciphertext_record_len]u8,
2935
3036/// `host` is only borrowed during this function call.
3137pub fn init(stream: net.Stream, ca_bundle: Certificate.Bundle, host: []const u8) !Client {
......@@ -596,6 +602,7 @@ pub fn init(stream: net.Stream, ca_bundle: Certificate.Bundle, host: []const u8)
596602 .application_cipher = app_cipher,
597603 .read_seq = 0,
598604 .write_seq = 0,
605 .partial_cleartext_index = 0,
599606 .partially_read_buffer = undefined,
600607 .partially_read_len = @intCast(u15, len - end),
601608 .eof = false,
......@@ -722,27 +729,85 @@ pub fn writeAll(c: *Client, stream: net.Stream, bytes: []const u8) !void {
722729 }
723730}
724731
725/// Returns number of bytes that have been read, which are now populated inside
726/// `buffer`. A return value of zero bytes does not necessarily mean end of
727/// stream. Instead, the `eof` flag is set upon end of stream. The `eof` flag
728/// may be set after any call to `read`, including when greater than zero bytes
729/// are returned, and this function asserts that `eof` is `false`.
730pub fn read(c: *Client, stream: net.Stream, buffer: []u8) !usize {
732/// Returns the number of bytes read, calling the underlying read function the
733/// minimal number of times until the buffer has at least `len` bytes filled.
734/// If the number read is less than `len` it means the stream reached the end.
735/// Reaching the end of the stream is not an error condition.
736pub fn readAtLeast(c: *Client, stream: anytype, buffer: []u8, len: usize) !usize {
737 assert(len <= buffer.len);
738 if (c.eof) return 0;
739 var index: usize = 0;
740 while (index < len) {
741 index += try c.readAdvanced(stream, buffer[index..]);
742 if (c.eof) break;
743 }
744 return index;
745}
746
747pub fn read(c: *Client, stream: anytype, buffer: []u8) !usize {
748 return readAtLeast(c, stream, buffer, 1);
749}
750
751/// Returns the number of bytes read. If the number read is smaller than
752/// `buffer.len`, it means the stream reached the end. Reaching the end of the
753/// stream is not an error condition.
754pub fn readAll(c: *Client, stream: anytype, buffer: []u8) !usize {
755 return readAtLeast(c, stream, buffer, buffer.len);
756}
757
758/// Returns number of bytes that have been read, populated inside `buffer`. A
759/// return value of zero bytes does not mean end of stream. Instead, the `eof`
760/// flag is set upon end of stream. The `eof` flag may be set after any call to
761/// `read`, including when greater than zero bytes are returned, and this
762/// function asserts that `eof` is `false`.
763/// See `read` for a higher level function that has the same, familiar API
764/// as other read functions, such as `std.fs.File.read`.
765/// It is recommended to use a buffer size with length at least
766/// `tls.max_ciphertext_len` bytes to avoid redundantly decrypting the same
767/// encoded data.
768pub fn readAdvanced(c: *Client, stream: net.Stream, buffer: []u8) !usize {
731769 assert(!c.eof);
732770 const prev_len = c.partially_read_len;
733 var in_buf: [max_ciphertext_len * 4]u8 = undefined;
734 mem.copy(u8, &in_buf, c.partially_read_buffer[0..prev_len]);
771 // Ideally, this buffer would never be used. It is needed when `buffer` is too small
772 // to fit the cleartext, which may be as large as `max_ciphertext_len`.
773 var cleartext_stack_buffer: [max_ciphertext_len]u8 = undefined;
774 // This buffer is typically used, except, as an optimization when a very large
775 // `buffer` is provided, we use half of it for buffering ciphertext and the
776 // other half for outputting cleartext.
777 var in_stack_buffer: [max_ciphertext_len * 4]u8 = undefined;
778 const half_buffer_len = buffer.len / 2;
779 const out_in: struct { []u8, []u8 } = if (half_buffer_len >= in_stack_buffer.len) .{
780 buffer[0..half_buffer_len],
781 buffer[half_buffer_len..],
782 } else .{
783 buffer,
784 &in_stack_buffer,
785 };
786 const out_buf = out_in[0];
787 const in_buf = out_in[1];
788 mem.copy(u8, in_buf, c.partially_read_buffer[0..prev_len]);
735789
736790 // Capacity of output buffer, in records, rounded up.
737 const buf_cap = (buffer.len +| (max_ciphertext_len - 1)) / max_ciphertext_len;
791 const buf_cap = (out_buf.len +| (max_ciphertext_len - 1)) / max_ciphertext_len;
738792 const wanted_read_len = buf_cap * (max_ciphertext_len + tls.ciphertext_record_header_len);
739 const ask_slice = in_buf[prev_len..@min(wanted_read_len, in_buf.len)];
740 const actual_read_len = try stream.read(ask_slice);
741 const frag = in_buf[0 .. prev_len + actual_read_len];
742 if (frag.len == 0) {
743 // This is either a truncation attack, or a bug in the server.
744 return error.TlsConnectionTruncated;
745 }
793 const ask_len = @max(wanted_read_len, cleartext_stack_buffer.len);
794 const ask_slice = in_buf[prev_len..][0..@min(ask_len, in_buf.len - prev_len)];
795 assert(ask_slice.len > 0);
796 const frag = frag: {
797 if (prev_len >= 5) {
798 const record_size = mem.readIntBig(u16, in_buf[3..][0..2]);
799 if (prev_len >= 5 + record_size) {
800 // We can use our buffered data without calling read().
801 break :frag in_buf[0..prev_len];
802 }
803 }
804 const actual_read_len = try stream.read(ask_slice);
805 if (actual_read_len == 0) {
806 // This is either a truncation attack, or a bug in the server.
807 return error.TlsConnectionTruncated;
808 }
809 break :frag in_buf[0 .. prev_len + actual_read_len];
810 };
746811 var in: usize = 0;
747812 var out: usize = 0;
748813
......@@ -750,6 +815,7 @@ pub fn read(c: *Client, stream: net.Stream, buffer: []u8) !usize {
750815 if (in + tls.ciphertext_record_header_len > frag.len) {
751816 return finishRead(c, frag, in, out);
752817 }
818 const record_start = in;
753819 const ct = @intToEnum(ContentType, frag[in]);
754820 in += 1;
755821 const legacy_version = mem.readIntBig(u16, frag[in..][0..2]);
......@@ -767,7 +833,7 @@ pub fn read(c: *Client, stream: net.Stream, buffer: []u8) !usize {
767833 @panic("TODO handle an alert here");
768834 },
769835 .application_data => {
770 const cleartext_len = switch (c.application_cipher) {
836 const cleartext = switch (c.application_cipher) {
771837 inline else => |*p| c: {
772838 const P = @TypeOf(p.*);
773839 const V = @Vector(P.AEAD.nonce_length, u8);
......@@ -776,29 +842,29 @@ pub fn read(c: *Client, stream: net.Stream, buffer: []u8) !usize {
776842 const ciphertext = frag[in..][0..ciphertext_len];
777843 in += ciphertext_len;
778844 const auth_tag = frag[in..][0..P.AEAD.tag_length].*;
779 const cleartext = buffer[out..][0..ciphertext_len];
780845 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);
846 // Here we use read_seq and then intentionally don't
847 // increment it until later when it is certain the same
848 // ciphertext does not need to be decrypted again.
781849 const operand: V = pad ++ @bitCast([8]u8, big(c.read_seq));
782 c.read_seq += 1;
783850 const nonce: [P.AEAD.nonce_length]u8 = @as(V, p.server_iv) ^ operand;
784 //std.debug.print("seq: {d} nonce: {} server_key: {} server_iv: {}\n", .{
785 // c.read_seq - 1,
786 // std.fmt.fmtSliceHexLower(&nonce),
787 // std.fmt.fmtSliceHexLower(&p.server_key),
788 // std.fmt.fmtSliceHexLower(&p.server_iv),
789 //});
851 const cleartext_buf = if (c.partial_cleartext_index == 0 and out + ciphertext.len <= out_buf.len)
852 out_buf[out..]
853 else
854 &cleartext_stack_buffer;
855 const cleartext = cleartext_buf[0..ciphertext.len];
790856 P.AEAD.decrypt(cleartext, ciphertext, auth_tag, ad, nonce, p.server_key) catch
791857 return error.TlsBadRecordMac;
792 break :c cleartext.len;
858 break :c cleartext;
793859 },
794860 };
795861
796 const cleartext = buffer[out..][0..cleartext_len];
797862 const inner_ct = @intToEnum(ContentType, cleartext[cleartext.len - 1]);
798863 switch (inner_ct) {
799864 .alert => {
800 const level = @intToEnum(tls.AlertLevel, buffer[out]);
801 const desc = @intToEnum(tls.AlertDescription, buffer[out + 1]);
865 c.read_seq += 1;
866 const level = @intToEnum(tls.AlertLevel, out_buf[out]);
867 const desc = @intToEnum(tls.AlertDescription, out_buf[out + 1]);
802868 if (desc == .close_notify) {
803869 c.eof = true;
804870 return out;
......@@ -807,6 +873,7 @@ pub fn read(c: *Client, stream: net.Stream, buffer: []u8) !usize {
807873 return error.TlsAlert;
808874 },
809875 .handshake => {
876 c.read_seq += 1;
810877 var ct_i: usize = 0;
811878 while (true) {
812879 const handshake_type = @intToEnum(tls.HandshakeType, cleartext[ct_i]);
......@@ -819,7 +886,7 @@ pub fn read(c: *Client, stream: net.Stream, buffer: []u8) !usize {
819886 const handshake = cleartext[ct_i..next_handshake_i];
820887 switch (handshake_type) {
821888 .new_session_ticket => {
822 std.debug.print("server sent a new session ticket\n", .{});
889 // This client implementation ignores new session tickets.
823890 },
824891 .key_update => {
825892 switch (c.application_cipher) {
......@@ -859,7 +926,35 @@ pub fn read(c: *Client, stream: net.Stream, buffer: []u8) !usize {
859926 }
860927 },
861928 .application_data => {
862 out += cleartext_len - 1;
929 // Determine whether the output buffer or a stack
930 // buffer was used for storing the cleartext.
931 if (c.partial_cleartext_index == 0 and
932 out + cleartext.len <= out_buf.len)
933 {
934 // Output buffer was used directly which means no
935 // memory copying needs to occur, and we can move
936 // on to the next ciphertext record.
937 out += cleartext.len - 1;
938 c.read_seq += 1;
939 } else {
940 // Stack buffer was used, so we must copy to the output buffer.
941 const dest = out_buf[out..];
942 const rest = cleartext[c.partial_cleartext_index..];
943 const src = rest[0..@min(rest.len, dest.len)];
944 mem.copy(u8, dest, src);
945 out += src.len;
946 c.partial_cleartext_index = @intCast(
947 @TypeOf(c.partial_cleartext_index),
948 c.partial_cleartext_index + src.len,
949 );
950 if (c.partial_cleartext_index >= cleartext.len) {
951 c.partial_cleartext_index = 0;
952 c.read_seq += 1;
953 } else {
954 in = record_start;
955 return finishRead(c, frag, in, out);
956 }
957 }
863958 },
864959 else => {
865960 std.debug.print("inner content type: {d}\n", .{inner_ct});
lib/std/http/Client.zig+3-9
......@@ -63,16 +63,10 @@ pub const Request = struct {
6363 }
6464
6565 pub fn readAtLeast(req: *Request, buffer: []u8, len: usize) !usize {
66 var index: usize = 0;
67 while (index < len) {
68 const amt = try req.read(buffer[index..]);
69 index += amt;
70 switch (req.protocol) {
71 .http => if (amt == 0) break,
72 .https => if (req.tls_client.eof) break,
73 }
66 switch (req.protocol) {
67 .http => return req.stream.readAtLeast(buffer, len),
68 .https => return req.tls_client.readAtLeast(req.stream, buffer, len),
7469 }
75 return index;
7670 }
7771};
7872
lib/std/net.zig+4-3
......@@ -1680,11 +1680,12 @@ pub const Stream = struct {
16801680 }
16811681
16821682 /// Returns the number of bytes read, calling the underlying read function
1683 /// the minimal number of times until at least the buffer has at least
1684 /// `len` bytes filled. If the number read is less than `len` it means the
1685 /// stream reached the end. Reaching the end of the stream is not an error
1683 /// the minimal number of times until the buffer has at least `len` bytes
1684 /// filled. If the number read is less than `len` it means the stream
1685 /// reached the end. Reaching the end of the stream is not an error
16861686 /// condition.
16871687 pub fn readAtLeast(s: Stream, buffer: []u8, len: usize) ReadError!usize {
1688 assert(len <= buffer.len);
16881689 var index: usize = 0;
16891690 while (index < len) {
16901691 const amt = try s.read(buffer[index..]);