| ... | @@ -1,62 +1,447 @@ | ... | @@ -1,62 +1,447 @@ |
| 1 | //! This API is a barely-touched, barely-functional http client, just the | 1 | //! This API is a barely-touched, barely-functional http client, just the |
| 2 | //! absolute minimum thing I needed in order to test `std.crypto.tls`. Bear | 2 | //! absolute minimum thing I needed in order to test `std.crypto.tls`. Bear |
| 3 | //! with me and I promise the API will become useful and streamlined. | 3 | //! with me and I promise the API will become useful and streamlined. |
| | 4 | //! |
| | 5 | //! TODO: send connection: keep-alive and LRU cache a configurable number of |
| | 6 | //! open connections to skip DNS and TLS handshake for subsequent requests. |
| 4 | | 7 | |
| 5 | const std = @import("../std.zig"); | 8 | const std = @import("../std.zig"); |
| | 9 | const mem = std.mem; |
| 6 | const assert = std.debug.assert; | 10 | const assert = std.debug.assert; |
| 7 | const http = std.http; | 11 | const http = std.http; |
| 8 | const net = std.net; | 12 | const net = std.net; |
| 9 | const Client = @This(); | 13 | const Client = @This(); |
| 10 | const Url = std.Url; | 14 | const Url = std.Url; |
| | 15 | const Allocator = std.mem.Allocator; |
| | 16 | const testing = std.testing; |
| 11 | | 17 | |
| 12 | allocator: std.mem.Allocator, | 18 | /// Used for tcpConnectToHost and storing HTTP headers when an externally |
| 13 | headers: std.ArrayListUnmanaged(u8) = .{}, | 19 | /// managed buffer is not provided. |
| 14 | active_requests: usize = 0, | 20 | allocator: Allocator, |
| 15 | ca_bundle: std.crypto.Certificate.Bundle = .{}, | 21 | ca_bundle: std.crypto.Certificate.Bundle = .{}, |
| 16 | | 22 | |
| | 23 | pub const Connection = struct { |
| | 24 | stream: net.Stream, |
| | 25 | /// undefined unless protocol is tls. |
| | 26 | tls_client: std.crypto.tls.Client, |
| | 27 | protocol: Protocol, |
| | 28 | |
| | 29 | pub const Protocol = enum { plain, tls }; |
| | 30 | |
| | 31 | pub fn read(conn: *Connection, buffer: []u8) !usize { |
| | 32 | switch (conn.protocol) { |
| | 33 | .plain => return conn.stream.read(buffer), |
| | 34 | .tls => return conn.tls_client.read(conn.stream, buffer), |
| | 35 | } |
| | 36 | } |
| | 37 | |
| | 38 | pub fn readAtLeast(conn: *Connection, buffer: []u8, len: usize) !usize { |
| | 39 | switch (conn.protocol) { |
| | 40 | .plain => return conn.stream.readAtLeast(buffer, len), |
| | 41 | .tls => return conn.tls_client.readAtLeast(conn.stream, buffer, len), |
| | 42 | } |
| | 43 | } |
| | 44 | |
| | 45 | pub fn writeAll(conn: *Connection, buffer: []const u8) !void { |
| | 46 | switch (conn.protocol) { |
| | 47 | .plain => return conn.stream.writeAll(buffer), |
| | 48 | .tls => return conn.tls_client.writeAll(conn.stream, buffer), |
| | 49 | } |
| | 50 | } |
| | 51 | |
| | 52 | pub fn write(conn: *Connection, buffer: []const u8) !usize { |
| | 53 | switch (conn.protocol) { |
| | 54 | .plain => return conn.stream.write(buffer), |
| | 55 | .tls => return conn.tls_client.write(conn.stream, buffer), |
| | 56 | } |
| | 57 | } |
| | 58 | }; |
| | 59 | |
| 17 | /// TODO: emit error.UnexpectedEndOfStream or something like that when the read | 60 | /// TODO: emit error.UnexpectedEndOfStream or something like that when the read |
| 18 | /// data does not match the content length. This is necessary since HTTPS disables | 61 | /// data does not match the content length. This is necessary since HTTPS disables |
| 19 | /// close_notify protection on underlying TLS streams. | 62 | /// close_notify protection on underlying TLS streams. |
| 20 | pub const Request = struct { | 63 | pub const Request = struct { |
| 21 | client: *Client, | 64 | client: *Client, |
| 22 | stream: net.Stream, | 65 | connection: Connection, |
| 23 | headers: std.ArrayListUnmanaged(u8) = .{}, | 66 | redirects_left: u32, |
| 24 | tls_client: std.crypto.tls.Client, | 67 | response: Response, |
| 25 | protocol: Protocol, | 68 | /// These are stored in Request so that they are available when following |
| 26 | response_headers: http.Headers = .{}, | 69 | /// redirects. |
| | 70 | headers: Headers, |
| 27 | | 71 | |
| 28 | pub const Protocol = enum { http, https }; | 72 | pub const Response = struct { |
| | 73 | headers: Response.Headers, |
| | 74 | state: State, |
| | 75 | header_bytes_owned: bool, |
| | 76 | /// This could either be a fixed buffer provided by the API user or it |
| | 77 | /// could be our own array list. |
| | 78 | header_bytes: std.ArrayListUnmanaged(u8), |
| | 79 | max_header_bytes: usize, |
| 29 | | 80 | |
| 30 | pub const Options = struct { | 81 | pub const Headers = struct { |
| | 82 | location: ?[]const u8 = null, |
| | 83 | status: http.Status, |
| | 84 | version: http.Version, |
| | 85 | content_length: ?u64 = null, |
| | 86 | |
| | 87 | pub fn parse(bytes: []const u8) !Response.Headers { |
| | 88 | var it = mem.split(u8, bytes[0 .. bytes.len - 4], "\r\n"); |
| | 89 | |
| | 90 | const first_line = it.first(); |
| | 91 | if (first_line.len < 12) |
| | 92 | return error.ShortHttpStatusLine; |
| | 93 | |
| | 94 | const version: http.Version = switch (int64(first_line[0..8])) { |
| | 95 | int64("HTTP/1.0") => .@"HTTP/1.0", |
| | 96 | int64("HTTP/1.1") => .@"HTTP/1.1", |
| | 97 | else => return error.BadHttpVersion, |
| | 98 | }; |
| | 99 | if (first_line[8] != ' ') return error.HttpHeadersInvalid; |
| | 100 | const status = @intToEnum(http.Status, parseInt3(first_line[9..12].*)); |
| | 101 | |
| | 102 | var headers: Response.Headers = .{ |
| | 103 | .version = version, |
| | 104 | .status = status, |
| | 105 | }; |
| | 106 | |
| | 107 | while (it.next()) |line| { |
| | 108 | if (line.len == 0) return error.HttpHeadersInvalid; |
| | 109 | switch (line[0]) { |
| | 110 | ' ', '\t' => return error.HttpHeaderContinuationsUnsupported, |
| | 111 | else => {}, |
| | 112 | } |
| | 113 | var line_it = mem.split(u8, line, ": "); |
| | 114 | const header_name = line_it.first(); |
| | 115 | const header_value = line_it.rest(); |
| | 116 | if (std.ascii.eqlIgnoreCase(header_name, "location")) { |
| | 117 | if (headers.location != null) return error.HttpHeadersInvalid; |
| | 118 | headers.location = header_value; |
| | 119 | } else if (std.ascii.eqlIgnoreCase(header_name, "content-length")) { |
| | 120 | if (headers.content_length != null) return error.HttpHeadersInvalid; |
| | 121 | headers.content_length = try std.fmt.parseInt(u64, header_value, 10); |
| | 122 | } |
| | 123 | } |
| | 124 | |
| | 125 | return headers; |
| | 126 | } |
| | 127 | |
| | 128 | test "parse headers" { |
| | 129 | const example = |
| | 130 | "HTTP/1.1 301 Moved Permanently\r\n" ++ |
| | 131 | "Location: https://www.example.com/\r\n" ++ |
| | 132 | "Content-Type: text/html; charset=UTF-8\r\n" ++ |
| | 133 | "Content-Length: 220\r\n\r\n"; |
| | 134 | const parsed = try Response.Headers.parse(example); |
| | 135 | try testing.expectEqual(http.Version.@"HTTP/1.1", parsed.version); |
| | 136 | try testing.expectEqual(http.Status.moved_permanently, parsed.status); |
| | 137 | try testing.expectEqualStrings("https://www.example.com/", parsed.location orelse |
| | 138 | return error.TestFailed); |
| | 139 | try testing.expectEqual(@as(?u64, 220), parsed.content_length); |
| | 140 | } |
| | 141 | |
| | 142 | test "header continuation" { |
| | 143 | const example = |
| | 144 | "HTTP/1.0 200 OK\r\n" ++ |
| | 145 | "Content-Type: text/html;\r\n charset=UTF-8\r\n" ++ |
| | 146 | "Content-Length: 220\r\n\r\n"; |
| | 147 | try testing.expectError( |
| | 148 | error.HttpHeaderContinuationsUnsupported, |
| | 149 | Response.Headers.parse(example), |
| | 150 | ); |
| | 151 | } |
| | 152 | |
| | 153 | test "extra content length" { |
| | 154 | const example = |
| | 155 | "HTTP/1.0 200 OK\r\n" ++ |
| | 156 | "Content-Length: 220\r\n" ++ |
| | 157 | "Content-Type: text/html; charset=UTF-8\r\n" ++ |
| | 158 | "content-length: 220\r\n\r\n"; |
| | 159 | try testing.expectError( |
| | 160 | error.HttpHeadersInvalid, |
| | 161 | Response.Headers.parse(example), |
| | 162 | ); |
| | 163 | } |
| | 164 | }; |
| | 165 | |
| | 166 | pub const State = enum { |
| | 167 | invalid, |
| | 168 | finished, |
| | 169 | start, |
| | 170 | seen_r, |
| | 171 | seen_rn, |
| | 172 | seen_rnr, |
| | 173 | }; |
| | 174 | |
| | 175 | pub fn initDynamic(max: usize) Response { |
| | 176 | return .{ |
| | 177 | .state = .start, |
| | 178 | .headers = undefined, |
| | 179 | .header_bytes = .{}, |
| | 180 | .max_header_bytes = max, |
| | 181 | .header_bytes_owned = true, |
| | 182 | }; |
| | 183 | } |
| | 184 | |
| | 185 | pub fn initStatic(buf: []u8) Response { |
| | 186 | return .{ |
| | 187 | .state = .start, |
| | 188 | .headers = undefined, |
| | 189 | .header_bytes = .{ .items = buf[0..0], .capacity = buf.len }, |
| | 190 | .max_header_bytes = buf.len, |
| | 191 | .header_bytes_owned = false, |
| | 192 | }; |
| | 193 | } |
| | 194 | |
| | 195 | /// Returns how many bytes are part of HTTP headers. Always less than or |
| | 196 | /// equal to bytes.len. If the amount returned is less than bytes.len, it |
| | 197 | /// means the headers ended and the first byte after the double \r\n\r\n is |
| | 198 | /// located at `bytes[result]`. |
| | 199 | pub fn findHeadersEnd(r: *Response, bytes: []const u8) usize { |
| | 200 | var index: usize = 0; |
| | 201 | |
| | 202 | // TODO: https://github.com/ziglang/zig/issues/8220 |
| | 203 | state: while (true) { |
| | 204 | switch (r.state) { |
| | 205 | .invalid => unreachable, |
| | 206 | .finished => unreachable, |
| | 207 | .start => while (true) { |
| | 208 | switch (bytes.len - index) { |
| | 209 | 0 => return index, |
| | 210 | 1 => { |
| | 211 | if (bytes[index] == '\r') |
| | 212 | r.state = .seen_r; |
| | 213 | return index + 1; |
| | 214 | }, |
| | 215 | 2 => { |
| | 216 | if (int16(bytes[index..][0..2]) == int16("\r\n")) { |
| | 217 | r.state = .seen_rn; |
| | 218 | } else if (bytes[index + 1] == '\r') { |
| | 219 | r.state = .seen_r; |
| | 220 | } |
| | 221 | return index + 2; |
| | 222 | }, |
| | 223 | 3 => { |
| | 224 | if (int16(bytes[index..][0..2]) == int16("\r\n") and |
| | 225 | bytes[index + 2] == '\r') |
| | 226 | { |
| | 227 | r.state = .seen_rnr; |
| | 228 | } else if (int16(bytes[index + 1 ..][0..2]) == int16("\r\n")) { |
| | 229 | r.state = .seen_rn; |
| | 230 | } else if (bytes[index + 2] == '\r') { |
| | 231 | r.state = .seen_r; |
| | 232 | } |
| | 233 | return index + 3; |
| | 234 | }, |
| | 235 | 4...15 => { |
| | 236 | if (int32(bytes[index..][0..4]) == int32("\r\n\r\n")) { |
| | 237 | r.state = .finished; |
| | 238 | return index + 4; |
| | 239 | } else if (int16(bytes[index + 1 ..][0..2]) == int16("\r\n") and |
| | 240 | bytes[index + 3] == '\r') |
| | 241 | { |
| | 242 | r.state = .seen_rnr; |
| | 243 | index += 4; |
| | 244 | continue :state; |
| | 245 | } else if (int16(bytes[index + 2 ..][0..2]) == int16("\r\n")) { |
| | 246 | r.state = .seen_rn; |
| | 247 | index += 4; |
| | 248 | continue :state; |
| | 249 | } else if (bytes[index + 3] == '\r') { |
| | 250 | r.state = .seen_r; |
| | 251 | index += 4; |
| | 252 | continue :state; |
| | 253 | } |
| | 254 | index += 4; |
| | 255 | continue; |
| | 256 | }, |
| | 257 | else => { |
| | 258 | const chunk = bytes[index..][0..16]; |
| | 259 | const v: @Vector(16, u8) = chunk.*; |
| | 260 | const matches_r = v == @splat(16, @as(u8, '\r')); |
| | 261 | const iota = std.simd.iota(u8, 16); |
| | 262 | const default = @splat(16, @as(u8, 16)); |
| | 263 | const sub_index = @reduce(.Min, @select(u8, matches_r, iota, default)); |
| | 264 | switch (sub_index) { |
| | 265 | 0...12 => { |
| | 266 | index += sub_index + 4; |
| | 267 | if (int32(chunk[sub_index..][0..4]) == int32("\r\n\r\n")) { |
| | 268 | r.state = .finished; |
| | 269 | return index; |
| | 270 | } |
| | 271 | continue; |
| | 272 | }, |
| | 273 | 13 => { |
| | 274 | index += 16; |
| | 275 | if (int16(chunk[14..][0..2]) == int16("\n\r")) { |
| | 276 | r.state = .seen_rnr; |
| | 277 | continue :state; |
| | 278 | } |
| | 279 | continue; |
| | 280 | }, |
| | 281 | 14 => { |
| | 282 | index += 16; |
| | 283 | if (chunk[15] == '\n') { |
| | 284 | r.state = .seen_rn; |
| | 285 | continue :state; |
| | 286 | } |
| | 287 | continue; |
| | 288 | }, |
| | 289 | 15 => { |
| | 290 | r.state = .seen_r; |
| | 291 | index += 16; |
| | 292 | continue :state; |
| | 293 | }, |
| | 294 | 16 => { |
| | 295 | index += 16; |
| | 296 | continue; |
| | 297 | }, |
| | 298 | else => unreachable, |
| | 299 | } |
| | 300 | }, |
| | 301 | } |
| | 302 | }, |
| | 303 | |
| | 304 | .seen_r => switch (bytes.len - index) { |
| | 305 | 0 => return index, |
| | 306 | 1 => { |
| | 307 | switch (bytes[index]) { |
| | 308 | '\n' => r.state = .seen_rn, |
| | 309 | '\r' => r.state = .seen_r, |
| | 310 | else => r.state = .start, |
| | 311 | } |
| | 312 | return index + 1; |
| | 313 | }, |
| | 314 | 2 => { |
| | 315 | if (int16(bytes[index..][0..2]) == int16("\n\r")) { |
| | 316 | r.state = .seen_rnr; |
| | 317 | return index + 2; |
| | 318 | } |
| | 319 | r.state = .start; |
| | 320 | return index + 2; |
| | 321 | }, |
| | 322 | else => { |
| | 323 | if (int16(bytes[index..][0..2]) == int16("\n\r") and |
| | 324 | bytes[index + 2] == '\n') |
| | 325 | { |
| | 326 | r.state = .finished; |
| | 327 | return index + 3; |
| | 328 | } |
| | 329 | index += 3; |
| | 330 | r.state = .start; |
| | 331 | continue :state; |
| | 332 | }, |
| | 333 | }, |
| | 334 | .seen_rn => switch (bytes.len - index) { |
| | 335 | 0 => return index, |
| | 336 | 1 => { |
| | 337 | switch (bytes[index]) { |
| | 338 | '\r' => r.state = .seen_rnr, |
| | 339 | else => r.state = .start, |
| | 340 | } |
| | 341 | return index + 1; |
| | 342 | }, |
| | 343 | else => { |
| | 344 | if (int16(bytes[index..][0..2]) == int16("\r\n")) { |
| | 345 | r.state = .finished; |
| | 346 | return index + 2; |
| | 347 | } |
| | 348 | index += 2; |
| | 349 | r.state = .start; |
| | 350 | continue :state; |
| | 351 | }, |
| | 352 | }, |
| | 353 | .seen_rnr => switch (bytes.len - index) { |
| | 354 | 0 => return index, |
| | 355 | else => { |
| | 356 | if (bytes[index] == '\n') { |
| | 357 | r.state = .finished; |
| | 358 | return index + 1; |
| | 359 | } |
| | 360 | index += 1; |
| | 361 | r.state = .start; |
| | 362 | continue :state; |
| | 363 | }, |
| | 364 | }, |
| | 365 | } |
| | 366 | |
| | 367 | return index; |
| | 368 | } |
| | 369 | } |
| | 370 | |
| | 371 | fn parseInt3(nnn: @Vector(3, u8)) u10 { |
| | 372 | const zero: @Vector(3, u8) = .{ '0', '0', '0' }; |
| | 373 | const mmm: @Vector(3, u10) = .{ 100, 10, 1 }; |
| | 374 | return @reduce(.Add, @as(@Vector(3, u10), nnn -% zero) *% mmm); |
| | 375 | } |
| | 376 | |
| | 377 | test parseInt3 { |
| | 378 | const expectEqual = std.testing.expectEqual; |
| | 379 | try expectEqual(@as(u10, 0), parseInt3("000".*)); |
| | 380 | try expectEqual(@as(u10, 418), parseInt3("418".*)); |
| | 381 | try expectEqual(@as(u10, 999), parseInt3("999".*)); |
| | 382 | } |
| | 383 | |
| | 384 | inline fn int16(array: *const [2]u8) u16 { |
| | 385 | return @bitCast(u16, array.*); |
| | 386 | } |
| | 387 | |
| | 388 | inline fn int32(array: *const [4]u8) u32 { |
| | 389 | return @bitCast(u32, array.*); |
| | 390 | } |
| | 391 | |
| | 392 | inline fn int64(array: *const [8]u8) u64 { |
| | 393 | return @bitCast(u64, array.*); |
| | 394 | } |
| | 395 | |
| | 396 | test "find headers end basic" { |
| | 397 | var buffer: [1]u8 = undefined; |
| | 398 | var r = Response.initStatic(&buffer); |
| | 399 | try testing.expectEqual(@as(usize, 10), r.findHeadersEnd("HTTP/1.1 4")); |
| | 400 | try testing.expectEqual(@as(usize, 2), r.findHeadersEnd("18")); |
| | 401 | try testing.expectEqual(@as(usize, 8), r.findHeadersEnd(" lol\r\n\r\nblah blah")); |
| | 402 | } |
| | 403 | |
| | 404 | test "find headers end vectorized" { |
| | 405 | var buffer: [1]u8 = undefined; |
| | 406 | var r = Response.initStatic(&buffer); |
| | 407 | const example = |
| | 408 | "HTTP/1.1 301 Moved Permanently\r\n" ++ |
| | 409 | "Location: https://www.example.com/\r\n" ++ |
| | 410 | "Content-Type: text/html; charset=UTF-8\r\n" ++ |
| | 411 | "Content-Length: 220\r\n" ++ |
| | 412 | "\r\ncontent"; |
| | 413 | try testing.expectEqual(@as(usize, 131), r.findHeadersEnd(example)); |
| | 414 | } |
| | 415 | }; |
| | 416 | |
| | 417 | pub const Headers = struct { |
| 31 | method: http.Method = .GET, | 418 | method: http.Method = .GET, |
| 32 | }; | 419 | }; |
| 33 | | 420 | |
| 34 | pub fn deinit(req: *Request) void { | 421 | pub const Options = struct { |
| 35 | req.client.active_requests -= 1; | 422 | max_redirects: u32 = 3, |
| 36 | req.headers.deinit(req.client.allocator); | 423 | header_strategy: HeaderStrategy = .{ .dynamic = 16 * 1024 }, |
| 37 | req.* = undefined; | | |
| 38 | } | | |
| 39 | | 424 | |
| 40 | pub fn addHeader(req: *Request, name: []const u8, value: []const u8) !void { | 425 | pub const HeaderStrategy = union(enum) { |
| 41 | const gpa = req.client.allocator; | 426 | /// In this case, the client's Allocator will be used to store the |
| 42 | // Ensure an extra +2 for the \r\n in end() | 427 | /// entire HTTP header. This value is the maximum total size of |
| 43 | try req.headers.ensureUnusedCapacity(gpa, name.len + value.len + 6); | 428 | /// HTTP headers allowed, otherwise |
| 44 | req.headers.appendSliceAssumeCapacity(name); | 429 | /// error.HttpHeadersExceededSizeLimit is returned from read(). |
| 45 | req.headers.appendSliceAssumeCapacity(": "); | 430 | dynamic: usize, |
| 46 | req.headers.appendSliceAssumeCapacity(value); | 431 | /// This is used to store the entire HTTP header. If the HTTP |
| 47 | req.headers.appendSliceAssumeCapacity("\r\n"); | 432 | /// header is too big to fit, `error.HttpHeadersExceededSizeLimit` |
| 48 | } | 433 | /// is returned from read(). When this is used, `error.OutOfMemory` |
| | 434 | /// cannot be returned from `read()`. |
| | 435 | static: []u8, |
| | 436 | }; |
| | 437 | }; |
| 49 | | 438 | |
| 50 | pub fn end(req: *Request) !void { | 439 | /// May be skipped if header strategy is buffer. |
| 51 | req.headers.appendSliceAssumeCapacity("\r\n"); | 440 | pub fn deinit(req: *Request) void { |
| 52 | switch (req.protocol) { | 441 | if (req.response.header_bytes_owned) { |
| 53 | .http => { | 442 | req.response.header_bytes.deinit(req.client.allocator); |
| 54 | try req.stream.writeAll(req.headers.items); | | |
| 55 | }, | | |
| 56 | .https => { | | |
| 57 | try req.tls_client.writeAll(req.stream, req.headers.items); | | |
| 58 | }, | | |
| 59 | } | 443 | } |
| | 444 | req.* = undefined; |
| 60 | } | 445 | } |
| 61 | | 446 | |
| 62 | pub fn readAll(req: *Request, buffer: []u8) !usize { | 447 | pub fn readAll(req: *Request, buffer: []u8) !usize { |
| ... | @@ -71,7 +456,7 @@ pub const Request = struct { | ... | @@ -71,7 +456,7 @@ pub const Request = struct { |
| 71 | assert(len <= buffer.len); | 456 | assert(len <= buffer.len); |
| 72 | var index: usize = 0; | 457 | var index: usize = 0; |
| 73 | while (index < len) { | 458 | while (index < len) { |
| 74 | const headers_finished = req.response_headers.state == .finished; | 459 | const headers_finished = req.response.state == .finished; |
| 75 | const amt = try readAdvanced(req, buffer[index..]); | 460 | const amt = try readAdvanced(req, buffer[index..]); |
| 76 | if (amt == 0 and headers_finished) break; | 461 | if (amt == 0 and headers_finished) break; |
| 77 | index += amt; | 462 | index += amt; |
| ... | @@ -82,100 +467,126 @@ pub const Request = struct { | ... | @@ -82,100 +467,126 @@ pub const Request = struct { |
| 82 | /// This one can return 0 without meaning EOF. | 467 | /// This one can return 0 without meaning EOF. |
| 83 | /// TODO change to readvAdvanced | 468 | /// TODO change to readvAdvanced |
| 84 | pub fn readAdvanced(req: *Request, buffer: []u8) !usize { | 469 | pub fn readAdvanced(req: *Request, buffer: []u8) !usize { |
| 85 | if (req.response_headers.state == .finished) return readRaw(req, buffer); | 470 | if (req.response.state == .finished) return req.connection.read(buffer); |
| 86 | | 471 | |
| 87 | const amt = try readRaw(req, buffer); | 472 | const amt = try req.connection.read(buffer); |
| 88 | const data = buffer[0..amt]; | 473 | const data = buffer[0..amt]; |
| 89 | const i = req.response_headers.feed(data); | 474 | const i = req.response.findHeadersEnd(data); |
| 90 | if (req.response_headers.state == .invalid) return error.InvalidHttpHeaders; | 475 | if (req.response.state == .invalid) return error.HttpHeadersInvalid; |
| 91 | if (i < data.len) { | 476 | |
| 92 | const rest = data[i..]; | 477 | const headers_data = data[0..i]; |
| 93 | std.mem.copy(u8, buffer, rest); | 478 | if (req.response.header_bytes.items.len + headers_data.len > req.response.max_header_bytes) { |
| 94 | return rest.len; | 479 | return error.HttpHeadersExceededSizeLimit; |
| 95 | } | 480 | } |
| 96 | return 0; | 481 | try req.response.header_bytes.appendSlice(req.client.allocator, headers_data); |
| 97 | } | | |
| 98 | | 482 | |
| 99 | /// Only abstracts over http/https. | 483 | if (req.response.state == .finished) { |
| 100 | fn readRaw(req: *Request, buffer: []u8) !usize { | 484 | req.response.headers = try Response.Headers.parse(req.response.header_bytes.items); |
| 101 | switch (req.protocol) { | 485 | } |
| 102 | .http => return req.stream.read(buffer), | 486 | |
| 103 | .https => return req.tls_client.read(req.stream, buffer), | 487 | if (req.response.headers.status.class() == .redirect) { |
| | 488 | if (req.redirects_left == 0) return error.TooManyHttpRedirects; |
| | 489 | const location = req.response.headers.location orelse |
| | 490 | return error.HttpRedirectMissingLocation; |
| | 491 | const new_url = try std.Url.parse(location); |
| | 492 | const new_req = try req.client.request(new_url, req.headers, .{ |
| | 493 | .max_redirects = req.redirects_left - 1, |
| | 494 | .header_strategy = if (req.response.header_bytes_owned) .{ |
| | 495 | .dynamic = req.response.max_header_bytes, |
| | 496 | } else .{ |
| | 497 | .static = req.response.header_bytes.unusedCapacitySlice(), |
| | 498 | }, |
| | 499 | }); |
| | 500 | req.deinit(); |
| | 501 | req.* = new_req; |
| | 502 | return readAdvanced(req, buffer); |
| 104 | } | 503 | } |
| 105 | } | | |
| 106 | | 504 | |
| 107 | /// Only abstracts over http/https. | 505 | const body_data = data[i..]; |
| 108 | fn readAtLeastRaw(req: *Request, buffer: []u8, len: usize) !usize { | 506 | if (body_data.len > 0) { |
| 109 | switch (req.protocol) { | 507 | mem.copy(u8, buffer, body_data); |
| 110 | .http => return req.stream.readAtLeast(buffer, len), | 508 | return body_data.len; |
| 111 | .https => return req.tls_client.readAtLeast(req.stream, buffer, len), | | |
| 112 | } | 509 | } |
| | 510 | return 0; |
| | 511 | } |
| | 512 | |
| | 513 | test { |
| | 514 | _ = Response; |
| 113 | } | 515 | } |
| 114 | }; | 516 | }; |
| 115 | | 517 | |
| 116 | pub fn deinit(client: *Client) void { | 518 | pub fn deinit(client: *Client, gpa: Allocator) void { |
| 117 | assert(client.active_requests == 0); | 519 | client.ca_bundle.deinit(gpa); |
| 118 | client.headers.deinit(client.allocator); | | |
| 119 | client.* = undefined; | 520 | client.* = undefined; |
| 120 | } | 521 | } |
| 121 | | 522 | |
| 122 | pub fn request(client: *Client, url: Url, options: Request.Options) !Request { | 523 | pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connection.Protocol) !Connection { |
| 123 | const protocol = std.meta.stringToEnum(Request.Protocol, url.scheme) orelse | 524 | var conn: Connection = .{ |
| 124 | return error.UnsupportedUrlScheme; | 525 | .stream = try net.tcpConnectToHost(client.allocator, host, port), |
| 125 | const port: u16 = url.port orelse switch (protocol) { | | |
| 126 | .http => 80, | | |
| 127 | .https => 443, | | |
| 128 | }; | | |
| 129 | | | |
| 130 | var req: Request = .{ | | |
| 131 | .client = client, | | |
| 132 | .stream = try net.tcpConnectToHost(client.allocator, url.host, port), | | |
| 133 | .protocol = protocol, | | |
| 134 | .tls_client = undefined, | 526 | .tls_client = undefined, |
| | 527 | .protocol = protocol, |
| 135 | }; | 528 | }; |
| 136 | client.active_requests += 1; | | |
| 137 | errdefer req.deinit(); | | |
| 138 | | 529 | |
| 139 | switch (protocol) { | 530 | switch (protocol) { |
| 140 | .http => {}, | 531 | .plain => {}, |
| 141 | .https => { | 532 | .tls => { |
| 142 | req.tls_client = try std.crypto.tls.Client.init(req.stream, client.ca_bundle, url.host); | 533 | conn.tls_client = try std.crypto.tls.Client.init(conn.stream, client.ca_bundle, host); |
| 143 | // This is appropriate for HTTPS because the HTTP headers contain | 534 | // This is appropriate for HTTPS because the HTTP headers contain |
| 144 | // the content length which is used to detect truncation attacks. | 535 | // the content length which is used to detect truncation attacks. |
| 145 | req.tls_client.allow_truncation_attacks = true; | 536 | conn.tls_client.allow_truncation_attacks = true; |
| 146 | }, | 537 | }, |
| 147 | } | 538 | } |
| 148 | | 539 | |
| 149 | try req.headers.ensureUnusedCapacity( | 540 | return conn; |
| 150 | client.allocator, | 541 | } |
| 151 | @tagName(options.method).len + | 542 | |
| 152 | 1 + | 543 | pub fn request(client: *Client, url: Url, headers: Request.Headers, options: Request.Options) !Request { |
| 153 | url.path.len + | 544 | const protocol: Connection.Protocol = if (mem.eql(u8, url.scheme, "http")) |
| 154 | " HTTP/1.1\r\nHost: ".len + | 545 | .plain |
| 155 | url.host.len + | 546 | else if (mem.eql(u8, url.scheme, "https")) |
| 156 | "\r\nUpgrade-Insecure-Requests: 1\r\n".len + | 547 | .tls |
| 157 | client.headers.items.len + | 548 | else |
| 158 | 2, // for the \r\n at the end of headers | 549 | return error.UnsupportedUrlScheme; |
| 159 | ); | 550 | |
| 160 | req.headers.appendSliceAssumeCapacity(@tagName(options.method)); | 551 | const port: u16 = url.port orelse switch (protocol) { |
| 161 | req.headers.appendSliceAssumeCapacity(" "); | 552 | .plain => 80, |
| 162 | req.headers.appendSliceAssumeCapacity(url.path); | 553 | .tls => 443, |
| 163 | req.headers.appendSliceAssumeCapacity(" HTTP/1.1\r\nHost: "); | 554 | }; |
| 164 | req.headers.appendSliceAssumeCapacity(url.host); | 555 | |
| 165 | switch (protocol) { | 556 | var req: Request = .{ |
| 166 | .https => req.headers.appendSliceAssumeCapacity("\r\nUpgrade-Insecure-Requests: 1\r\n"), | 557 | .client = client, |
| 167 | .http => req.headers.appendSliceAssumeCapacity("\r\n"), | 558 | .headers = headers, |
| | 559 | .connection = try client.connect(url.host, port, protocol), |
| | 560 | .redirects_left = options.max_redirects, |
| | 561 | .response = switch (options.header_strategy) { |
| | 562 | .dynamic => |max| Request.Response.initDynamic(max), |
| | 563 | .static => |buf| Request.Response.initStatic(buf), |
| | 564 | }, |
| | 565 | }; |
| | 566 | |
| | 567 | { |
| | 568 | var h = try std.BoundedArray(u8, 1000).init(0); |
| | 569 | try h.appendSlice(@tagName(headers.method)); |
| | 570 | try h.appendSlice(" "); |
| | 571 | try h.appendSlice(url.path); |
| | 572 | try h.appendSlice(" HTTP/1.1\r\nHost: "); |
| | 573 | try h.appendSlice(url.host); |
| | 574 | try h.appendSlice("\r\nConnection: close\r\n\r\n"); |
| | 575 | |
| | 576 | const header_bytes = h.slice(); |
| | 577 | try req.connection.writeAll(header_bytes); |
| 168 | } | 578 | } |
| 169 | req.headers.appendSliceAssumeCapacity(client.headers.items); | | |
| 170 | | 579 | |
| 171 | return req; | 580 | return req; |
| 172 | } | 581 | } |
| 173 | | 582 | |
| 174 | pub fn addHeader(client: *Client, name: []const u8, value: []const u8) !void { | 583 | test { |
| 175 | const gpa = client.allocator; | 584 | const builtin = @import("builtin"); |
| 176 | try client.headers.ensureUnusedCapacity(gpa, name.len + value.len + 4); | 585 | const native_endian = comptime builtin.cpu.arch.endian(); |
| 177 | client.headers.appendSliceAssumeCapacity(name); | 586 | if (builtin.zig_backend == .stage2_llvm and native_endian == .Big) { |
| 178 | client.headers.appendSliceAssumeCapacity(": "); | 587 | // https://github.com/ziglang/zig/issues/13782 |
| 179 | client.headers.appendSliceAssumeCapacity(value); | 588 | return error.SkipZigTest; |
| 180 | client.headers.appendSliceAssumeCapacity("\r\n"); | 589 | } |
| | 590 | |
| | 591 | _ = Request; |
| 181 | } | 592 | } |
Andrew Kelley <andrew@ziglang.org> 2023-01-05 19:42:59-07:00
GitHub <noreply@github.com> 2023-01-05 19:42:59-07:00