authorgravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2024-08-07 05:47:32-04:00
committergravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2024-08-07 06:29:34-04:00
loga805454dea86c3dfc3e3cc9fabaad1b6c0e2b9aa
treee9925073c43151d96ebe4eab50a7aee0ec2051b8
parent8bec737ca22d0f994f606c13cb4a23b6287dad0c

fix partial strings getting dropped when multi-byte codepoints span input buffers

1 files changed, 81 insertions(+), 19 deletions(-)

lib/std/json/scanner.zig+81-19
...@@ -897,7 +897,7 @@ pub const Scanner = struct {...@@ -897,7 +897,7 @@ pub const Scanner = struct {
897 },897 },
898 .number_post_dot => {898 .number_post_dot => {
899 if (self.cursor >= self.input.len) return self.endOfBufferInNumber(false);899 if (self.cursor >= self.input.len) return self.endOfBufferInNumber(false);
900 switch (try self.expectByte()) {900 switch (self.input[self.cursor]) {
901 '0'...'9' => {901 '0'...'9' => {
902 self.cursor += 1;902 self.cursor += 1;
903 self.state = .number_frac;903 self.state = .number_frac;
...@@ -1032,7 +1032,8 @@ pub const Scanner = struct {...@@ -1032,7 +1032,8 @@ pub const Scanner = struct {
1032 return error.BufferUnderrun;1032 return error.BufferUnderrun;
1033 },1033 },
1034 .string_backslash => {1034 .string_backslash => {
1035 switch (try self.expectByte()) {1035 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1036 switch (self.input[self.cursor]) {
1036 '"', '\\', '/' => {1037 '"', '\\', '/' => {
1037 // Since these characters now represent themselves literally,1038 // Since these characters now represent themselves literally,
1038 // we can simply begin the next plaintext slice here.1039 // we can simply begin the next plaintext slice here.
...@@ -1080,7 +1081,8 @@ pub const Scanner = struct {...@@ -1080,7 +1081,8 @@ pub const Scanner = struct {
1080 }1081 }
1081 },1082 },
1082 .string_backslash_u => {1083 .string_backslash_u => {
1083 const c = try self.expectByte();1084 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1085 const c = self.input[self.cursor];
1084 switch (c) {1086 switch (c) {
1085 '0'...'9' => {1087 '0'...'9' => {
1086 self.utf16_code_units[0] = @as(u16, c - '0') << 12;1088 self.utf16_code_units[0] = @as(u16, c - '0') << 12;
...@@ -1098,7 +1100,8 @@ pub const Scanner = struct {...@@ -1098,7 +1100,8 @@ pub const Scanner = struct {
1098 continue :state_loop;1100 continue :state_loop;
1099 },1101 },
1100 .string_backslash_u_1 => {1102 .string_backslash_u_1 => {
1101 const c = try self.expectByte();1103 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1104 const c = self.input[self.cursor];
1102 switch (c) {1105 switch (c) {
1103 '0'...'9' => {1106 '0'...'9' => {
1104 self.utf16_code_units[0] |= @as(u16, c - '0') << 8;1107 self.utf16_code_units[0] |= @as(u16, c - '0') << 8;
...@@ -1116,7 +1119,8 @@ pub const Scanner = struct {...@@ -1116,7 +1119,8 @@ pub const Scanner = struct {
1116 continue :state_loop;1119 continue :state_loop;
1117 },1120 },
1118 .string_backslash_u_2 => {1121 .string_backslash_u_2 => {
1119 const c = try self.expectByte();1122 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1123 const c = self.input[self.cursor];
1120 switch (c) {1124 switch (c) {
1121 '0'...'9' => {1125 '0'...'9' => {
1122 self.utf16_code_units[0] |= @as(u16, c - '0') << 4;1126 self.utf16_code_units[0] |= @as(u16, c - '0') << 4;
...@@ -1134,7 +1138,8 @@ pub const Scanner = struct {...@@ -1134,7 +1138,8 @@ pub const Scanner = struct {
1134 continue :state_loop;1138 continue :state_loop;
1135 },1139 },
1136 .string_backslash_u_3 => {1140 .string_backslash_u_3 => {
1137 const c = try self.expectByte();1141 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1142 const c = self.input[self.cursor];
1138 switch (c) {1143 switch (c) {
1139 '0'...'9' => {1144 '0'...'9' => {
1140 self.utf16_code_units[0] |= c - '0';1145 self.utf16_code_units[0] |= c - '0';
...@@ -1160,7 +1165,8 @@ pub const Scanner = struct {...@@ -1160,7 +1165,8 @@ pub const Scanner = struct {
1160 }1165 }
1161 },1166 },
1162 .string_surrogate_half => {1167 .string_surrogate_half => {
1163 switch (try self.expectByte()) {1168 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1169 switch (self.input[self.cursor]) {
1164 '\\' => {1170 '\\' => {
1165 self.cursor += 1;1171 self.cursor += 1;
1166 self.state = .string_surrogate_half_backslash;1172 self.state = .string_surrogate_half_backslash;
...@@ -1170,7 +1176,8 @@ pub const Scanner = struct {...@@ -1170,7 +1176,8 @@ pub const Scanner = struct {
1170 }1176 }
1171 },1177 },
1172 .string_surrogate_half_backslash => {1178 .string_surrogate_half_backslash => {
1173 switch (try self.expectByte()) {1179 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1180 switch (self.input[self.cursor]) {
1174 'u' => {1181 'u' => {
1175 self.cursor += 1;1182 self.cursor += 1;
1176 self.state = .string_surrogate_half_backslash_u;1183 self.state = .string_surrogate_half_backslash_u;
...@@ -1180,7 +1187,8 @@ pub const Scanner = struct {...@@ -1180,7 +1187,8 @@ pub const Scanner = struct {
1180 }1187 }
1181 },1188 },
1182 .string_surrogate_half_backslash_u => {1189 .string_surrogate_half_backslash_u => {
1183 switch (try self.expectByte()) {1190 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1191 switch (self.input[self.cursor]) {
1184 'D', 'd' => {1192 'D', 'd' => {
1185 self.cursor += 1;1193 self.cursor += 1;
1186 self.utf16_code_units[1] = 0xD << 12;1194 self.utf16_code_units[1] = 0xD << 12;
...@@ -1191,7 +1199,8 @@ pub const Scanner = struct {...@@ -1191,7 +1199,8 @@ pub const Scanner = struct {
1191 }1199 }
1192 },1200 },
1193 .string_surrogate_half_backslash_u_1 => {1201 .string_surrogate_half_backslash_u_1 => {
1194 const c = try self.expectByte();1202 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1203 const c = self.input[self.cursor];
1195 switch (c) {1204 switch (c) {
1196 'C'...'F' => {1205 'C'...'F' => {
1197 self.cursor += 1;1206 self.cursor += 1;
...@@ -1209,7 +1218,8 @@ pub const Scanner = struct {...@@ -1209,7 +1218,8 @@ pub const Scanner = struct {
1209 }1218 }
1210 },1219 },
1211 .string_surrogate_half_backslash_u_2 => {1220 .string_surrogate_half_backslash_u_2 => {
1212 const c = try self.expectByte();1221 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1222 const c = self.input[self.cursor];
1213 switch (c) {1223 switch (c) {
1214 '0'...'9' => {1224 '0'...'9' => {
1215 self.cursor += 1;1225 self.cursor += 1;
...@@ -1233,7 +1243,8 @@ pub const Scanner = struct {...@@ -1233,7 +1243,8 @@ pub const Scanner = struct {
1233 }1243 }
1234 },1244 },
1235 .string_surrogate_half_backslash_u_3 => {1245 .string_surrogate_half_backslash_u_3 => {
1236 const c = try self.expectByte();1246 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1247 const c = self.input[self.cursor];
1237 switch (c) {1248 switch (c) {
1238 '0'...'9' => {1249 '0'...'9' => {
1239 self.utf16_code_units[1] |= c - '0';1250 self.utf16_code_units[1] |= c - '0';
...@@ -1254,7 +1265,8 @@ pub const Scanner = struct {...@@ -1254,7 +1265,8 @@ pub const Scanner = struct {
1254 },1265 },
12551266
1256 .string_utf8_last_byte => {1267 .string_utf8_last_byte => {
1257 switch (try self.expectByte()) {1268 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1269 switch (self.input[self.cursor]) {
1258 0x80...0xBF => {1270 0x80...0xBF => {
1259 self.cursor += 1;1271 self.cursor += 1;
1260 self.state = .string;1272 self.state = .string;
...@@ -1264,7 +1276,8 @@ pub const Scanner = struct {...@@ -1264,7 +1276,8 @@ pub const Scanner = struct {
1264 }1276 }
1265 },1277 },
1266 .string_utf8_second_to_last_byte => {1278 .string_utf8_second_to_last_byte => {
1267 switch (try self.expectByte()) {1279 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1280 switch (self.input[self.cursor]) {
1268 0x80...0xBF => {1281 0x80...0xBF => {
1269 self.cursor += 1;1282 self.cursor += 1;
1270 self.state = .string_utf8_last_byte;1283 self.state = .string_utf8_last_byte;
...@@ -1274,7 +1287,8 @@ pub const Scanner = struct {...@@ -1274,7 +1287,8 @@ pub const Scanner = struct {
1274 }1287 }
1275 },1288 },
1276 .string_utf8_second_to_last_byte_guard_against_overlong => {1289 .string_utf8_second_to_last_byte_guard_against_overlong => {
1277 switch (try self.expectByte()) {1290 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1291 switch (self.input[self.cursor]) {
1278 0xA0...0xBF => {1292 0xA0...0xBF => {
1279 self.cursor += 1;1293 self.cursor += 1;
1280 self.state = .string_utf8_last_byte;1294 self.state = .string_utf8_last_byte;
...@@ -1284,7 +1298,8 @@ pub const Scanner = struct {...@@ -1284,7 +1298,8 @@ pub const Scanner = struct {
1284 }1298 }
1285 },1299 },
1286 .string_utf8_second_to_last_byte_guard_against_surrogate_half => {1300 .string_utf8_second_to_last_byte_guard_against_surrogate_half => {
1287 switch (try self.expectByte()) {1301 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1302 switch (self.input[self.cursor]) {
1288 0x80...0x9F => {1303 0x80...0x9F => {
1289 self.cursor += 1;1304 self.cursor += 1;
1290 self.state = .string_utf8_last_byte;1305 self.state = .string_utf8_last_byte;
...@@ -1294,7 +1309,8 @@ pub const Scanner = struct {...@@ -1294,7 +1309,8 @@ pub const Scanner = struct {
1294 }1309 }
1295 },1310 },
1296 .string_utf8_third_to_last_byte => {1311 .string_utf8_third_to_last_byte => {
1297 switch (try self.expectByte()) {1312 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1313 switch (self.input[self.cursor]) {
1298 0x80...0xBF => {1314 0x80...0xBF => {
1299 self.cursor += 1;1315 self.cursor += 1;
1300 self.state = .string_utf8_second_to_last_byte;1316 self.state = .string_utf8_second_to_last_byte;
...@@ -1304,7 +1320,8 @@ pub const Scanner = struct {...@@ -1304,7 +1320,8 @@ pub const Scanner = struct {
1304 }1320 }
1305 },1321 },
1306 .string_utf8_third_to_last_byte_guard_against_overlong => {1322 .string_utf8_third_to_last_byte_guard_against_overlong => {
1307 switch (try self.expectByte()) {1323 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1324 switch (self.input[self.cursor]) {
1308 0x90...0xBF => {1325 0x90...0xBF => {
1309 self.cursor += 1;1326 self.cursor += 1;
1310 self.state = .string_utf8_second_to_last_byte;1327 self.state = .string_utf8_second_to_last_byte;
...@@ -1314,7 +1331,8 @@ pub const Scanner = struct {...@@ -1314,7 +1331,8 @@ pub const Scanner = struct {
1314 }1331 }
1315 },1332 },
1316 .string_utf8_third_to_last_byte_guard_against_too_large => {1333 .string_utf8_third_to_last_byte_guard_against_too_large => {
1317 switch (try self.expectByte()) {1334 if (self.cursor >= self.input.len) return self.endOfBufferInString();
1335 switch (self.input[self.cursor]) {
1318 0x80...0x8F => {1336 0x80...0x8F => {
1319 self.cursor += 1;1337 self.cursor += 1;
1320 self.state = .string_utf8_second_to_last_byte;1338 self.state = .string_utf8_second_to_last_byte;
...@@ -1666,6 +1684,17 @@ pub const Scanner = struct {...@@ -1666,6 +1684,17 @@ pub const Scanner = struct {
1666 self.value_start = self.cursor;1684 self.value_start = self.cursor;
1667 return slice;1685 return slice;
1668 }1686 }
1687 fn takeValueSliceMinusTrailingOffset(self: *@This(), trailing_negative_offset: usize) []const u8 {
1688 // Check if the escape sequence started before the current input buffer.
1689 // (The algebra here is awkward to avoid unsigned underflow,
1690 // but it's just making sure the slice on the next line isn't UB.)
1691 if (self.cursor <= self.value_start + trailing_negative_offset) return "";
1692 const slice = self.input[self.value_start .. self.cursor - trailing_negative_offset];
1693 // When trailing_negative_offset is non-zero, setting self.value_start doesn't matter,
1694 // because we always set it again while emitting the .partial_string_escaped_*.
1695 self.value_start = self.cursor;
1696 return slice;
1697 }
16691698
1670 fn endOfBufferInNumber(self: *@This(), allow_end: bool) !Token {1699 fn endOfBufferInNumber(self: *@This(), allow_end: bool) !Token {
1671 const slice = self.takeValueSlice();1700 const slice = self.takeValueSlice();
...@@ -1678,6 +1707,39 @@ pub const Scanner = struct {...@@ -1678,6 +1707,39 @@ pub const Scanner = struct {
1678 return Token{ .partial_number = slice };1707 return Token{ .partial_number = slice };
1679 }1708 }
16801709
1710 fn endOfBufferInString(self: *@This()) !Token {
1711 if (self.is_end_of_input) return error.UnexpectedEndOfInput;
1712 const slice = self.takeValueSliceMinusTrailingOffset(switch (self.state) {
1713 // Don't include the escape sequence in the partial string.
1714 .string_backslash => 1,
1715 .string_backslash_u => 2,
1716 .string_backslash_u_1 => 3,
1717 .string_backslash_u_2 => 4,
1718 .string_backslash_u_3 => 5,
1719 .string_surrogate_half => 6,
1720 .string_surrogate_half_backslash => 7,
1721 .string_surrogate_half_backslash_u => 8,
1722 .string_surrogate_half_backslash_u_1 => 9,
1723 .string_surrogate_half_backslash_u_2 => 10,
1724 .string_surrogate_half_backslash_u_3 => 11,
1725
1726 // Include everything up to the cursor otherwise.
1727 .string,
1728 .string_utf8_last_byte,
1729 .string_utf8_second_to_last_byte,
1730 .string_utf8_second_to_last_byte_guard_against_overlong,
1731 .string_utf8_second_to_last_byte_guard_against_surrogate_half,
1732 .string_utf8_third_to_last_byte,
1733 .string_utf8_third_to_last_byte_guard_against_overlong,
1734 .string_utf8_third_to_last_byte_guard_against_too_large,
1735 => 0,
1736
1737 else => unreachable,
1738 });
1739 if (slice.len == 0) return error.BufferUnderrun;
1740 return Token{ .partial_string = slice };
1741 }
1742
1681 fn partialStringCodepoint(code_point: u21) Token {1743 fn partialStringCodepoint(code_point: u21) Token {
1682 var buf: [4]u8 = undefined;1744 var buf: [4]u8 = undefined;
1683 switch (std.unicode.utf8Encode(code_point, &buf) catch unreachable) {1745 switch (std.unicode.utf8Encode(code_point, &buf) catch unreachable) {