authorgravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2023-08-15 06:11:59-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-08-15 09:11:59-04:00
loga155e35850def120af90993e1b4309b80b97eb85
treed886aa1293c40d8565fae63a168a5c59755d0b17
parentf7b82ed416e32a9164c67a35924b9b99e86707fa
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

std.json: Fix decoding of UTF-16 surrogate pairs (#16830)

* std.unicode: Add more UTF-16 decoding functions This mostly makes parts of Utf16LeIterator reusable * std.json: Fix decoding of UTF-16 surrogate pairs Before this commit, there were 524,288 codepoints that would get decoded improperly. After this commit, there are 0. Fixes #16828

3 files changed, 95 insertions(+), 46 deletions(-)

lib/std/json/scanner.zig+34-39
...@@ -414,7 +414,7 @@ pub const Scanner = struct {...@@ -414,7 +414,7 @@ pub const Scanner = struct {
414 string_is_object_key: bool = false,414 string_is_object_key: bool = false,
415 stack: BitStack,415 stack: BitStack,
416 value_start: usize = undefined,416 value_start: usize = undefined,
417 unicode_code_point: u21 = undefined,417 utf16_code_units: [2]u16 = undefined,
418418
419 input: []const u8 = "",419 input: []const u8 = "",
420 cursor: usize = 0,420 cursor: usize = 0,
...@@ -1083,13 +1083,13 @@ pub const Scanner = struct {...@@ -1083,13 +1083,13 @@ pub const Scanner = struct {
1083 const c = try self.expectByte();1083 const c = try self.expectByte();
1084 switch (c) {1084 switch (c) {
1085 '0'...'9' => {1085 '0'...'9' => {
1086 self.unicode_code_point = @as(u21, c - '0') << 12;1086 self.utf16_code_units[0] = @as(u16, c - '0') << 12;
1087 },1087 },
1088 'A'...'F' => {1088 'A'...'F' => {
1089 self.unicode_code_point = @as(u21, c - 'A' + 10) << 12;1089 self.utf16_code_units[0] = @as(u16, c - 'A' + 10) << 12;
1090 },1090 },
1091 'a'...'f' => {1091 'a'...'f' => {
1092 self.unicode_code_point = @as(u21, c - 'a' + 10) << 12;1092 self.utf16_code_units[0] = @as(u16, c - 'a' + 10) << 12;
1093 },1093 },
1094 else => return error.SyntaxError,1094 else => return error.SyntaxError,
1095 }1095 }
...@@ -1101,13 +1101,13 @@ pub const Scanner = struct {...@@ -1101,13 +1101,13 @@ pub const Scanner = struct {
1101 const c = try self.expectByte();1101 const c = try self.expectByte();
1102 switch (c) {1102 switch (c) {
1103 '0'...'9' => {1103 '0'...'9' => {
1104 self.unicode_code_point |= @as(u21, c - '0') << 8;1104 self.utf16_code_units[0] |= @as(u16, c - '0') << 8;
1105 },1105 },
1106 'A'...'F' => {1106 'A'...'F' => {
1107 self.unicode_code_point |= @as(u21, c - 'A' + 10) << 8;1107 self.utf16_code_units[0] |= @as(u16, c - 'A' + 10) << 8;
1108 },1108 },
1109 'a'...'f' => {1109 'a'...'f' => {
1110 self.unicode_code_point |= @as(u21, c - 'a' + 10) << 8;1110 self.utf16_code_units[0] |= @as(u16, c - 'a' + 10) << 8;
1111 },1111 },
1112 else => return error.SyntaxError,1112 else => return error.SyntaxError,
1113 }1113 }
...@@ -1119,13 +1119,13 @@ pub const Scanner = struct {...@@ -1119,13 +1119,13 @@ pub const Scanner = struct {
1119 const c = try self.expectByte();1119 const c = try self.expectByte();
1120 switch (c) {1120 switch (c) {
1121 '0'...'9' => {1121 '0'...'9' => {
1122 self.unicode_code_point |= @as(u21, c - '0') << 4;1122 self.utf16_code_units[0] |= @as(u16, c - '0') << 4;
1123 },1123 },
1124 'A'...'F' => {1124 'A'...'F' => {
1125 self.unicode_code_point |= @as(u21, c - 'A' + 10) << 4;1125 self.utf16_code_units[0] |= @as(u16, c - 'A' + 10) << 4;
1126 },1126 },
1127 'a'...'f' => {1127 'a'...'f' => {
1128 self.unicode_code_point |= @as(u21, c - 'a' + 10) << 4;1128 self.utf16_code_units[0] |= @as(u16, c - 'a' + 10) << 4;
1129 },1129 },
1130 else => return error.SyntaxError,1130 else => return error.SyntaxError,
1131 }1131 }
...@@ -1137,31 +1137,26 @@ pub const Scanner = struct {...@@ -1137,31 +1137,26 @@ pub const Scanner = struct {
1137 const c = try self.expectByte();1137 const c = try self.expectByte();
1138 switch (c) {1138 switch (c) {
1139 '0'...'9' => {1139 '0'...'9' => {
1140 self.unicode_code_point |= c - '0';1140 self.utf16_code_units[0] |= c - '0';
1141 },1141 },
1142 'A'...'F' => {1142 'A'...'F' => {
1143 self.unicode_code_point |= c - 'A' + 10;1143 self.utf16_code_units[0] |= c - 'A' + 10;
1144 },1144 },
1145 'a'...'f' => {1145 'a'...'f' => {
1146 self.unicode_code_point |= c - 'a' + 10;1146 self.utf16_code_units[0] |= c - 'a' + 10;
1147 },1147 },
1148 else => return error.SyntaxError,1148 else => return error.SyntaxError,
1149 }1149 }
1150 self.cursor += 1;1150 self.cursor += 1;
1151 switch (self.unicode_code_point) {1151 if (std.unicode.utf16IsHighSurrogate(self.utf16_code_units[0])) {
1152 0xD800...0xDBFF => {1152 self.state = .string_surrogate_half;
1153 // High surrogate half.1153 continue :state_loop;
1154 self.unicode_code_point = 0x10000 | (self.unicode_code_point << 10);1154 } else if (std.unicode.utf16IsLowSurrogate(self.utf16_code_units[0])) {
1155 self.state = .string_surrogate_half;1155 return error.SyntaxError; // Unexpected low surrogate half.
1156 continue :state_loop;1156 } else {
1157 },1157 self.value_start = self.cursor;
1158 0xDC00...0xDFFF => return error.SyntaxError, // Unexpected low surrogate half.1158 self.state = .string;
1159 else => {1159 return partialStringCodepoint(self.utf16_code_units[0]);
1160 // Code point from a single UTF-16 code unit.
1161 self.value_start = self.cursor;
1162 self.state = .string;
1163 return self.partialStringCodepoint();
1164 },
1165 }1160 }
1166 },1161 },
1167 .string_surrogate_half => {1162 .string_surrogate_half => {
...@@ -1188,6 +1183,7 @@ pub const Scanner = struct {...@@ -1188,6 +1183,7 @@ pub const Scanner = struct {
1188 switch (try self.expectByte()) {1183 switch (try self.expectByte()) {
1189 'D', 'd' => {1184 'D', 'd' => {
1190 self.cursor += 1;1185 self.cursor += 1;
1186 self.utf16_code_units[1] = 0xD << 12;
1191 self.state = .string_surrogate_half_backslash_u_1;1187 self.state = .string_surrogate_half_backslash_u_1;
1192 continue :state_loop;1188 continue :state_loop;
1193 },1189 },
...@@ -1199,13 +1195,13 @@ pub const Scanner = struct {...@@ -1199,13 +1195,13 @@ pub const Scanner = struct {
1199 switch (c) {1195 switch (c) {
1200 'C'...'F' => {1196 'C'...'F' => {
1201 self.cursor += 1;1197 self.cursor += 1;
1202 self.unicode_code_point |= @as(u21, c - 'C') << 8;1198 self.utf16_code_units[1] |= @as(u16, c - 'A' + 10) << 8;
1203 self.state = .string_surrogate_half_backslash_u_2;1199 self.state = .string_surrogate_half_backslash_u_2;
1204 continue :state_loop;1200 continue :state_loop;
1205 },1201 },
1206 'c'...'f' => {1202 'c'...'f' => {
1207 self.cursor += 1;1203 self.cursor += 1;
1208 self.unicode_code_point |= @as(u21, c - 'c') << 8;1204 self.utf16_code_units[1] |= @as(u16, c - 'a' + 10) << 8;
1209 self.state = .string_surrogate_half_backslash_u_2;1205 self.state = .string_surrogate_half_backslash_u_2;
1210 continue :state_loop;1206 continue :state_loop;
1211 },1207 },
...@@ -1217,19 +1213,19 @@ pub const Scanner = struct {...@@ -1217,19 +1213,19 @@ pub const Scanner = struct {
1217 switch (c) {1213 switch (c) {
1218 '0'...'9' => {1214 '0'...'9' => {
1219 self.cursor += 1;1215 self.cursor += 1;
1220 self.unicode_code_point |= @as(u21, c - '0') << 4;1216 self.utf16_code_units[1] |= @as(u16, c - '0') << 4;
1221 self.state = .string_surrogate_half_backslash_u_3;1217 self.state = .string_surrogate_half_backslash_u_3;
1222 continue :state_loop;1218 continue :state_loop;
1223 },1219 },
1224 'A'...'F' => {1220 'A'...'F' => {
1225 self.cursor += 1;1221 self.cursor += 1;
1226 self.unicode_code_point |= @as(u21, c - 'A' + 10) << 4;1222 self.utf16_code_units[1] |= @as(u16, c - 'A' + 10) << 4;
1227 self.state = .string_surrogate_half_backslash_u_3;1223 self.state = .string_surrogate_half_backslash_u_3;
1228 continue :state_loop;1224 continue :state_loop;
1229 },1225 },
1230 'a'...'f' => {1226 'a'...'f' => {
1231 self.cursor += 1;1227 self.cursor += 1;
1232 self.unicode_code_point |= @as(u21, c - 'a' + 10) << 4;1228 self.utf16_code_units[1] |= @as(u16, c - 'a' + 10) << 4;
1233 self.state = .string_surrogate_half_backslash_u_3;1229 self.state = .string_surrogate_half_backslash_u_3;
1234 continue :state_loop;1230 continue :state_loop;
1235 },1231 },
...@@ -1240,20 +1236,21 @@ pub const Scanner = struct {...@@ -1240,20 +1236,21 @@ pub const Scanner = struct {
1240 const c = try self.expectByte();1236 const c = try self.expectByte();
1241 switch (c) {1237 switch (c) {
1242 '0'...'9' => {1238 '0'...'9' => {
1243 self.unicode_code_point |= c - '0';1239 self.utf16_code_units[1] |= c - '0';
1244 },1240 },
1245 'A'...'F' => {1241 'A'...'F' => {
1246 self.unicode_code_point |= c - 'A' + 10;1242 self.utf16_code_units[1] |= c - 'A' + 10;
1247 },1243 },
1248 'a'...'f' => {1244 'a'...'f' => {
1249 self.unicode_code_point |= c - 'a' + 10;1245 self.utf16_code_units[1] |= c - 'a' + 10;
1250 },1246 },
1251 else => return error.SyntaxError,1247 else => return error.SyntaxError,
1252 }1248 }
1253 self.cursor += 1;1249 self.cursor += 1;
1254 self.value_start = self.cursor;1250 self.value_start = self.cursor;
1255 self.state = .string;1251 self.state = .string;
1256 return self.partialStringCodepoint();1252 const code_point = std.unicode.utf16DecodeSurrogatePair(&self.utf16_code_units) catch unreachable;
1253 return partialStringCodepoint(code_point);
1257 },1254 },
12581255
1259 .string_utf8_last_byte => {1256 .string_utf8_last_byte => {
...@@ -1681,9 +1678,7 @@ pub const Scanner = struct {...@@ -1681,9 +1678,7 @@ pub const Scanner = struct {
1681 return Token{ .partial_number = slice };1678 return Token{ .partial_number = slice };
1682 }1679 }
16831680
1684 fn partialStringCodepoint(self: *@This()) Token {1681 fn partialStringCodepoint(code_point: u21) Token {
1685 const code_point = self.unicode_code_point;
1686 self.unicode_code_point = undefined;
1687 var buf: [4]u8 = undefined;1682 var buf: [4]u8 = undefined;
1688 switch (std.unicode.utf8Encode(code_point, &buf) catch unreachable) {1683 switch (std.unicode.utf8Encode(code_point, &buf) catch unreachable) {
1689 1 => return Token{ .partial_string_escaped_1 = buf[0..1].* },1684 1 => return Token{ .partial_string_escaped_1 = buf[0..1].* },
lib/std/json/scanner_test.zig+1
...@@ -236,6 +236,7 @@ const string_test_cases = .{...@@ -236,6 +236,7 @@ const string_test_cases = .{
236 .{ "\\u000a", "\n" },236 .{ "\\u000a", "\n" },
237 .{ "𝄞", "\u{1D11E}" },237 .{ "𝄞", "\u{1D11E}" },
238 .{ "\\uD834\\uDD1E", "\u{1D11E}" },238 .{ "\\uD834\\uDD1E", "\u{1D11E}" },
239 .{ "\\uD87F\\uDFFE", "\u{2FFFE}" },
239 .{ "\\uff20", "＠" },240 .{ "\\uff20", "＠" },
240};241};
241242
lib/std/unicode.zig+60-7
...@@ -293,6 +293,58 @@ pub const Utf8Iterator = struct {...@@ -293,6 +293,58 @@ pub const Utf8Iterator = struct {
293 }293 }
294};294};
295295
296pub fn utf16IsHighSurrogate(c: u16) bool {
297 return c & ~@as(u16, 0x03ff) == 0xd800;
298}
299
300pub fn utf16IsLowSurrogate(c: u16) bool {
301 return c & ~@as(u16, 0x03ff) == 0xdc00;
302}
303
304/// Returns how many code units the UTF-16 representation would require
305/// for the given codepoint.
306pub fn utf16CodepointSequenceLength(c: u21) !u2 {
307 if (c <= 0xFFFF) return 1;
308 if (c <= 0x10FFFF) return 2;
309 return error.CodepointTooLarge;
310}
311
312test utf16CodepointSequenceLength {
313 try testing.expectEqual(@as(u2, 1), try utf16CodepointSequenceLength('a'));
314 try testing.expectEqual(@as(u2, 1), try utf16CodepointSequenceLength(0xFFFF));
315 try testing.expectEqual(@as(u2, 2), try utf16CodepointSequenceLength(0x10000));
316 try testing.expectEqual(@as(u2, 2), try utf16CodepointSequenceLength(0x10FFFF));
317 try testing.expectError(error.CodepointTooLarge, utf16CodepointSequenceLength(0x110000));
318}
319
320/// Given the first code unit of a UTF-16 codepoint, returns a number 1-2
321/// indicating the total length of the codepoint in UTF-16 code units.
322/// If this code unit does not match the form of a UTF-16 start code unit, returns Utf16InvalidStartCodeUnit.
323pub fn utf16CodeUnitSequenceLength(first_code_unit: u16) !u2 {
324 if (utf16IsHighSurrogate(first_code_unit)) return 2;
325 if (utf16IsLowSurrogate(first_code_unit)) return error.Utf16InvalidStartCodeUnit;
326 return 1;
327}
328
329test utf16CodeUnitSequenceLength {
330 try testing.expectEqual(@as(u2, 1), try utf16CodeUnitSequenceLength('a'));
331 try testing.expectEqual(@as(u2, 1), try utf16CodeUnitSequenceLength(0xFFFF));
332 try testing.expectEqual(@as(u2, 2), try utf16CodeUnitSequenceLength(0xDBFF));
333 try testing.expectError(error.Utf16InvalidStartCodeUnit, utf16CodeUnitSequenceLength(0xDFFF));
334}
335
336/// Decodes the codepoint encoded in the given pair of UTF-16 code units.
337/// Asserts that `surrogate_pair.len >= 2` and that the first code unit is a high surrogate.
338/// If the second code unit is not a low surrogate, error.ExpectedSecondSurrogateHalf is returned.
339pub fn utf16DecodeSurrogatePair(surrogate_pair: []const u16) !u21 {
340 assert(surrogate_pair.len >= 2);
341 assert(utf16IsHighSurrogate(surrogate_pair[0]));
342 const high_half: u21 = surrogate_pair[0];
343 const low_half = surrogate_pair[1];
344 if (!utf16IsLowSurrogate(low_half)) return error.ExpectedSecondSurrogateHalf;
345 return 0x10000 + ((high_half & 0x03ff) << 10) | (low_half & 0x03ff);
346}
347
296pub const Utf16LeIterator = struct {348pub const Utf16LeIterator = struct {
297 bytes: []const u8,349 bytes: []const u8,
298 i: usize,350 i: usize,
...@@ -307,19 +359,20 @@ pub const Utf16LeIterator = struct {...@@ -307,19 +359,20 @@ pub const Utf16LeIterator = struct {
307 pub fn nextCodepoint(it: *Utf16LeIterator) !?u21 {359 pub fn nextCodepoint(it: *Utf16LeIterator) !?u21 {
308 assert(it.i <= it.bytes.len);360 assert(it.i <= it.bytes.len);
309 if (it.i == it.bytes.len) return null;361 if (it.i == it.bytes.len) return null;
310 const c0: u21 = mem.readIntLittle(u16, it.bytes[it.i..][0..2]);362 var code_units: [2]u16 = undefined;
363 code_units[0] = mem.readIntLittle(u16, it.bytes[it.i..][0..2]);
311 it.i += 2;364 it.i += 2;
312 if (c0 & ~@as(u21, 0x03ff) == 0xd800) {365 if (utf16IsHighSurrogate(code_units[0])) {
313 // surrogate pair366 // surrogate pair
314 if (it.i >= it.bytes.len) return error.DanglingSurrogateHalf;367 if (it.i >= it.bytes.len) return error.DanglingSurrogateHalf;
315 const c1: u21 = mem.readIntLittle(u16, it.bytes[it.i..][0..2]);368 code_units[1] = mem.readIntLittle(u16, it.bytes[it.i..][0..2]);
316 if (c1 & ~@as(u21, 0x03ff) != 0xdc00) return error.ExpectedSecondSurrogateHalf;369 const codepoint = try utf16DecodeSurrogatePair(&code_units);
317 it.i += 2;370 it.i += 2;
318 return 0x10000 + (((c0 & 0x03ff) << 10) | (c1 & 0x03ff));371 return codepoint;
319 } else if (c0 & ~@as(u21, 0x03ff) == 0xdc00) {372 } else if (utf16IsLowSurrogate(code_units[0])) {
320 return error.UnexpectedSecondSurrogateHalf;373 return error.UnexpectedSecondSurrogateHalf;
321 } else {374 } else {
322 return c0;375 return code_units[0];
323 }376 }
324 }377 }
325};378};