authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-08-07 18:37:49-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2024-08-07 18:37:49-07:00
log7a7421c74970a0d52286fa33086e80af29fc8d0b
tree90093255b851acc7f917e6120c75f0f302140c02
parent4381bac79270249f954c864145d69b20e7dbafa4
parenta805454dea86c3dfc3e3cc9fabaad1b6c0e2b9aa
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #20973 from ziglang/fix-json-utf8

std.json: fix partial strings getting dropped when multi-byte codepoints span input buffers

2 files changed, 116 insertions(+), 20 deletions(-)

lib/std/json/scanner.zig+81-19
......@@ -897,7 +897,7 @@ pub const Scanner = struct {
897897 },
898898 .number_post_dot => {
899899 if (self.cursor >= self.input.len) return self.endOfBufferInNumber(false);
900 switch (try self.expectByte()) {
900 switch (self.input[self.cursor]) {
901901 '0'...'9' => {
902902 self.cursor += 1;
903903 self.state = .number_frac;
......@@ -1032,7 +1032,8 @@ pub const Scanner = struct {
10321032 return error.BufferUnderrun;
10331033 },
10341034 .string_backslash => {
1035 switch (try self.expectByte()) {
1035 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1036 switch (self.input[self.cursor]) {
10361037 '"', '\\', '/' => {
10371038 // Since these characters now represent themselves literally,
10381039 // we can simply begin the next plaintext slice here.
......@@ -1080,7 +1081,8 @@ pub const Scanner = struct {
10801081 }
10811082 },
10821083 .string_backslash_u => {
1083 const c = try self.expectByte();
1084 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1085 const c = self.input[self.cursor];
10841086 switch (c) {
10851087 '0'...'9' => {
10861088 self.utf16_code_units[0] = @as(u16, c - '0') << 12;
......@@ -1098,7 +1100,8 @@ pub const Scanner = struct {
10981100 continue :state_loop;
10991101 },
11001102 .string_backslash_u_1 => {
1101 const c = try self.expectByte();
1103 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1104 const c = self.input[self.cursor];
11021105 switch (c) {
11031106 '0'...'9' => {
11041107 self.utf16_code_units[0] |= @as(u16, c - '0') << 8;
......@@ -1116,7 +1119,8 @@ pub const Scanner = struct {
11161119 continue :state_loop;
11171120 },
11181121 .string_backslash_u_2 => {
1119 const c = try self.expectByte();
1122 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1123 const c = self.input[self.cursor];
11201124 switch (c) {
11211125 '0'...'9' => {
11221126 self.utf16_code_units[0] |= @as(u16, c - '0') << 4;
......@@ -1134,7 +1138,8 @@ pub const Scanner = struct {
11341138 continue :state_loop;
11351139 },
11361140 .string_backslash_u_3 => {
1137 const c = try self.expectByte();
1141 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1142 const c = self.input[self.cursor];
11381143 switch (c) {
11391144 '0'...'9' => {
11401145 self.utf16_code_units[0] |= c - '0';
......@@ -1160,7 +1165,8 @@ pub const Scanner = struct {
11601165 }
11611166 },
11621167 .string_surrogate_half => {
1163 switch (try self.expectByte()) {
1168 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1169 switch (self.input[self.cursor]) {
11641170 '\\' => {
11651171 self.cursor += 1;
11661172 self.state = .string_surrogate_half_backslash;
......@@ -1170,7 +1176,8 @@ pub const Scanner = struct {
11701176 }
11711177 },
11721178 .string_surrogate_half_backslash => {
1173 switch (try self.expectByte()) {
1179 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1180 switch (self.input[self.cursor]) {
11741181 'u' => {
11751182 self.cursor += 1;
11761183 self.state = .string_surrogate_half_backslash_u;
......@@ -1180,7 +1187,8 @@ pub const Scanner = struct {
11801187 }
11811188 },
11821189 .string_surrogate_half_backslash_u => {
1183 switch (try self.expectByte()) {
1190 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1191 switch (self.input[self.cursor]) {
11841192 'D', 'd' => {
11851193 self.cursor += 1;
11861194 self.utf16_code_units[1] = 0xD << 12;
......@@ -1191,7 +1199,8 @@ pub const Scanner = struct {
11911199 }
11921200 },
11931201 .string_surrogate_half_backslash_u_1 => {
1194 const c = try self.expectByte();
1202 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1203 const c = self.input[self.cursor];
11951204 switch (c) {
11961205 'C'...'F' => {
11971206 self.cursor += 1;
......@@ -1209,7 +1218,8 @@ pub const Scanner = struct {
12091218 }
12101219 },
12111220 .string_surrogate_half_backslash_u_2 => {
1212 const c = try self.expectByte();
1221 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1222 const c = self.input[self.cursor];
12131223 switch (c) {
12141224 '0'...'9' => {
12151225 self.cursor += 1;
......@@ -1233,7 +1243,8 @@ pub const Scanner = struct {
12331243 }
12341244 },
12351245 .string_surrogate_half_backslash_u_3 => {
1236 const c = try self.expectByte();
1246 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1247 const c = self.input[self.cursor];
12371248 switch (c) {
12381249 '0'...'9' => {
12391250 self.utf16_code_units[1] |= c - '0';
......@@ -1254,7 +1265,8 @@ pub const Scanner = struct {
12541265 },
12551266
12561267 .string_utf8_last_byte => {
1257 switch (try self.expectByte()) {
1268 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1269 switch (self.input[self.cursor]) {
12581270 0x80...0xBF => {
12591271 self.cursor += 1;
12601272 self.state = .string;
......@@ -1264,7 +1276,8 @@ pub const Scanner = struct {
12641276 }
12651277 },
12661278 .string_utf8_second_to_last_byte => {
1267 switch (try self.expectByte()) {
1279 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1280 switch (self.input[self.cursor]) {
12681281 0x80...0xBF => {
12691282 self.cursor += 1;
12701283 self.state = .string_utf8_last_byte;
......@@ -1274,7 +1287,8 @@ pub const Scanner = struct {
12741287 }
12751288 },
12761289 .string_utf8_second_to_last_byte_guard_against_overlong => {
1277 switch (try self.expectByte()) {
1290 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1291 switch (self.input[self.cursor]) {
12781292 0xA0...0xBF => {
12791293 self.cursor += 1;
12801294 self.state = .string_utf8_last_byte;
......@@ -1284,7 +1298,8 @@ pub const Scanner = struct {
12841298 }
12851299 },
12861300 .string_utf8_second_to_last_byte_guard_against_surrogate_half => {
1287 switch (try self.expectByte()) {
1301 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1302 switch (self.input[self.cursor]) {
12881303 0x80...0x9F => {
12891304 self.cursor += 1;
12901305 self.state = .string_utf8_last_byte;
......@@ -1294,7 +1309,8 @@ pub const Scanner = struct {
12941309 }
12951310 },
12961311 .string_utf8_third_to_last_byte => {
1297 switch (try self.expectByte()) {
1312 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1313 switch (self.input[self.cursor]) {
12981314 0x80...0xBF => {
12991315 self.cursor += 1;
13001316 self.state = .string_utf8_second_to_last_byte;
......@@ -1304,7 +1320,8 @@ pub const Scanner = struct {
13041320 }
13051321 },
13061322 .string_utf8_third_to_last_byte_guard_against_overlong => {
1307 switch (try self.expectByte()) {
1323 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1324 switch (self.input[self.cursor]) {
13081325 0x90...0xBF => {
13091326 self.cursor += 1;
13101327 self.state = .string_utf8_second_to_last_byte;
......@@ -1314,7 +1331,8 @@ pub const Scanner = struct {
13141331 }
13151332 },
13161333 .string_utf8_third_to_last_byte_guard_against_too_large => {
1317 switch (try self.expectByte()) {
1334 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1335 switch (self.input[self.cursor]) {
13181336 0x80...0x8F => {
13191337 self.cursor += 1;
13201338 self.state = .string_utf8_second_to_last_byte;
......@@ -1666,6 +1684,17 @@ pub const Scanner = struct {
16661684 self.value_start = self.cursor;
16671685 return slice;
16681686 }
1687 fn takeValueSliceMinusTrailingOffset(self: *@This(), trailing_negative_offset: usize) []const u8 {
1688 // Check if the escape sequence started before the current input buffer.
1689 // (The algebra here is awkward to avoid unsigned underflow,
1690 // but it's just making sure the slice on the next line isn't UB.)
1691 if (self.cursor <= self.value_start + trailing_negative_offset) return "";
1692 const slice = self.input[self.value_start .. self.cursor - trailing_negative_offset];
1693 // When trailing_negative_offset is non-zero, setting self.value_start doesn't matter,
1694 // because we always set it again while emitting the .partial_string_escaped_*.
1695 self.value_start = self.cursor;
1696 return slice;
1697 }
16691698
16701699 fn endOfBufferInNumber(self: *@This(), allow_end: bool) !Token {
16711700 const slice = self.takeValueSlice();
......@@ -1678,6 +1707,39 @@ pub const Scanner = struct {
16781707 return Token{ .partial_number = slice };
16791708 }
16801709
1710 fn endOfBufferInString(self: *@This()) !Token {
1711 if (self.is_end_of_input) return error.UnexpectedEndOfInput;
1712 const slice = self.takeValueSliceMinusTrailingOffset(switch (self.state) {
1713 // Don't include the escape sequence in the partial string.
1714 .string_backslash => 1,
1715 .string_backslash_u => 2,
1716 .string_backslash_u_1 => 3,
1717 .string_backslash_u_2 => 4,
1718 .string_backslash_u_3 => 5,
1719 .string_surrogate_half => 6,
1720 .string_surrogate_half_backslash => 7,
1721 .string_surrogate_half_backslash_u => 8,
1722 .string_surrogate_half_backslash_u_1 => 9,
1723 .string_surrogate_half_backslash_u_2 => 10,
1724 .string_surrogate_half_backslash_u_3 => 11,
1725
1726 // Include everything up to the cursor otherwise.
1727 .string,
1728 .string_utf8_last_byte,
1729 .string_utf8_second_to_last_byte,
1730 .string_utf8_second_to_last_byte_guard_against_overlong,
1731 .string_utf8_second_to_last_byte_guard_against_surrogate_half,
1732 .string_utf8_third_to_last_byte,
1733 .string_utf8_third_to_last_byte_guard_against_overlong,
1734 .string_utf8_third_to_last_byte_guard_against_too_large,
1735 => 0,
1736
1737 else => unreachable,
1738 });
1739 if (slice.len == 0) return error.BufferUnderrun;
1740 return Token{ .partial_string = slice };
1741 }
1742
16811743 fn partialStringCodepoint(code_point: u21) Token {
16821744 var buf: [4]u8 = undefined;
16831745 switch (std.unicode.utf8Encode(code_point, &buf) catch unreachable) {
lib/std/json/scanner_test.zig+35-1
......@@ -310,10 +310,44 @@ fn expectEqualTokens(expected_token: Token, actual_token: Token) !void {
310310 .number => |expected_value| {
311311 try std.testing.expectEqualStrings(expected_value, actual_token.number);
312312 },
313 .allocated_number => |expected_value| {
314 try std.testing.expectEqualStrings(expected_value, actual_token.allocated_number);
315 },
316 .partial_number => |expected_value| {
317 try std.testing.expectEqualStrings(expected_value, actual_token.partial_number);
318 },
319
313320 .string => |expected_value| {
314321 try std.testing.expectEqualStrings(expected_value, actual_token.string);
315322 },
316 else => {},
323 .allocated_string => |expected_value| {
324 try std.testing.expectEqualStrings(expected_value, actual_token.allocated_string);
325 },
326 .partial_string => |expected_value| {
327 try std.testing.expectEqualStrings(expected_value, actual_token.partial_string);
328 },
329 .partial_string_escaped_1 => |expected_value| {
330 try std.testing.expectEqualStrings(&expected_value, &actual_token.partial_string_escaped_1);
331 },
332 .partial_string_escaped_2 => |expected_value| {
333 try std.testing.expectEqualStrings(&expected_value, &actual_token.partial_string_escaped_2);
334 },
335 .partial_string_escaped_3 => |expected_value| {
336 try std.testing.expectEqualStrings(&expected_value, &actual_token.partial_string_escaped_3);
337 },
338 .partial_string_escaped_4 => |expected_value| {
339 try std.testing.expectEqualStrings(&expected_value, &actual_token.partial_string_escaped_4);
340 },
341
342 .object_begin,
343 .object_end,
344 .array_begin,
345 .array_end,
346 .true,
347 .false,
348 .null,
349 .end_of_document,
350 => {},
317351 }
318352}
319353