authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-12-29 18:31:10-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2019-12-29 18:31:10-05:00
log54231e832bae780c5012fc5cd30932447f1e1d47
tree4475e1625e81f320e8ffe358f4ee8dca557569ba
parent6af39aa49afeb3498d6c5dfa0b60a0fdc15ca47c
parent6d3b95a708c87ed81bd4f270b956913d82a0972d
signature Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #3648 from xackus/json-unescape

breaking: JSON unescape

2 files changed, 196 insertions(+), 47 deletions(-)

lib/std/json.zig+142-27
...@@ -10,18 +10,18 @@ const maxInt = std.math.maxInt;...@@ -10,18 +10,18 @@ const maxInt = std.math.maxInt;
1010
11pub const WriteStream = @import("json/write_stream.zig").WriteStream;11pub const WriteStream = @import("json/write_stream.zig").WriteStream;
1212
13// A single token slice into the parent string.13/// A single token slice into the parent string.
14//14///
15// Use `token.slice()` on the input at the current position to get the current slice.15/// Use `token.slice()` on the input at the current position to get the current slice.
16pub const Token = struct {16pub const Token = struct {
17 id: Id,17 id: Id,
18 // How many bytes do we skip before counting18 /// How many bytes do we skip before counting
19 offset: u1,19 offset: u1,
20 // Whether string contains a \uXXXX sequence and cannot be zero-copied20 /// Whether string contains an escape sequence and cannot be zero-copied
21 string_has_escape: bool,21 string_has_escape: bool,
22 // Whether number is simple and can be represented by an integer (i.e. no `.` or `e`)22 /// Whether number is simple and can be represented by an integer (i.e. no `.` or `e`)
23 number_is_integer: bool,23 number_is_integer: bool,
24 // How many bytes from the current position behind the start of this token is.24 /// How many bytes from the current position behind the start of this token is.
25 count: usize,25 count: usize,
2626
27 pub const Id = enum {27 pub const Id = enum {
...@@ -66,7 +66,7 @@ pub const Token = struct {...@@ -66,7 +66,7 @@ pub const Token = struct {
66 };66 };
67 }67 }
6868
69 // A marker token is a zero-length69 /// A marker token is a zero-length
70 pub fn initMarker(id: Id) Token {70 pub fn initMarker(id: Id) Token {
71 return Token{71 return Token{
72 .id = id,72 .id = id,
...@@ -77,19 +77,19 @@ pub const Token = struct {...@@ -77,19 +77,19 @@ pub const Token = struct {
77 };77 };
78 }78 }
7979
80 // Slice into the underlying input string.80 /// Slice into the underlying input string.
81 pub fn slice(self: Token, input: []const u8, i: usize) []const u8 {81 pub fn slice(self: Token, input: []const u8, i: usize) []const u8 {
82 return input[i + self.offset - self.count .. i + self.offset];82 return input[i + self.offset - self.count .. i + self.offset];
83 }83 }
84};84};
8585
86// A small streaming JSON parser. This accepts input one byte at a time and returns tokens as86/// A small streaming JSON parser. This accepts input one byte at a time and returns tokens as
87// they are encountered. No copies or allocations are performed during parsing and the entire87/// they are encountered. No copies or allocations are performed during parsing and the entire
88// parsing state requires ~40-50 bytes of stack space.88/// parsing state requires ~40-50 bytes of stack space.
89//89///
90// Conforms strictly to RFC8529.90/// Conforms strictly to RFC8529.
91//91///
92// For a non-byte based wrapper, consider using TokenStream instead.92/// For a non-byte based wrapper, consider using TokenStream instead.
93pub const StreamingParser = struct {93pub const StreamingParser = struct {
94 // Current state94 // Current state
95 state: State,95 state: State,
...@@ -205,10 +205,10 @@ pub const StreamingParser = struct {...@@ -205,10 +205,10 @@ pub const StreamingParser = struct {
205 InvalidControlCharacter,205 InvalidControlCharacter,
206 };206 };
207207
208 // Give another byte to the parser and obtain any new tokens. This may (rarely) return two208 /// Give another byte to the parser and obtain any new tokens. This may (rarely) return two
209 // tokens. token2 is always null if token1 is null.209 /// tokens. token2 is always null if token1 is null.
210 //210 ///
211 // There is currently no error recovery on a bad stream.211 /// There is currently no error recovery on a bad stream.
212 pub fn feed(p: *StreamingParser, c: u8, token1: *?Token, token2: *?Token) Error!void {212 pub fn feed(p: *StreamingParser, c: u8, token1: *?Token, token2: *?Token) Error!void {
213 token1.* = null;213 token1.* = null;
214 token2.* = null;214 token2.* = null;
...@@ -866,7 +866,7 @@ pub const StreamingParser = struct {...@@ -866,7 +866,7 @@ pub const StreamingParser = struct {
866 }866 }
867};867};
868868
869// A small wrapper over a StreamingParser for full slices. Returns a stream of json Tokens.869/// A small wrapper over a StreamingParser for full slices. Returns a stream of json Tokens.
870pub const TokenStream = struct {870pub const TokenStream = struct {
871 i: usize,871 i: usize,
872 slice: []const u8,872 slice: []const u8,
...@@ -905,7 +905,13 @@ pub const TokenStream = struct {...@@ -905,7 +905,13 @@ pub const TokenStream = struct {
905 }905 }
906 }906 }
907907
908 if (self.parser.complete) {908 // Without this a bare number fails, becasue the streaming parser doesn't know it ended
909 try self.parser.feed(' ', &t1, &t2);
910 self.i += 1;
911
912 if (t1) |token| {
913 return token;
914 } else if (self.parser.complete) {
909 return null;915 return null;
910 } else {916 } else {
911 return error.UnexpectedEndOfJson;917 return error.UnexpectedEndOfJson;
...@@ -971,8 +977,8 @@ test "json.token" {...@@ -971,8 +977,8 @@ test "json.token" {
971 testing.expect((try p.next()) == null);977 testing.expect((try p.next()) == null);
972}978}
973979
974// Validate a JSON string. This does not limit number precision so a decoder may not necessarily980/// Validate a JSON string. This does not limit number precision so a decoder may not necessarily
975// be able to decode the string even if this returns true.981/// be able to decode the string even if this returns true.
976pub fn validate(s: []const u8) bool {982pub fn validate(s: []const u8) bool {
977 var p = StreamingParser.init();983 var p = StreamingParser.init();
978984
...@@ -1009,6 +1015,8 @@ pub const ValueTree = struct {...@@ -1009,6 +1015,8 @@ pub const ValueTree = struct {
1009pub const ObjectMap = StringHashMap(Value);1015pub const ObjectMap = StringHashMap(Value);
1010pub const Array = ArrayList(Value);1016pub const Array = ArrayList(Value);
10111017
1018/// Represents a JSON value
1019/// Currently only supports numbers that fit into i64 or f64.
1012pub const Value = union(enum) {1020pub const Value = union(enum) {
1013 Null,1021 Null,
1014 Bool: bool,1022 Bool: bool,
...@@ -1055,7 +1063,7 @@ pub const Value = union(enum) {...@@ -1055,7 +1063,7 @@ pub const Value = union(enum) {
1055 }1063 }
1056};1064};
10571065
1058// A non-stream JSON parser which constructs a tree of Value's.1066/// A non-stream JSON parser which constructs a tree of Value's.
1059pub const Parser = struct {1067pub const Parser = struct {
1060 allocator: *Allocator,1068 allocator: *Allocator,
1061 state: State,1069 state: State,
...@@ -1124,7 +1132,10 @@ pub const Parser = struct {...@@ -1124,7 +1132,10 @@ pub const Parser = struct {
1124 p.state = State.ObjectValue;1132 p.state = State.ObjectValue;
1125 },1133 },
1126 else => {1134 else => {
1127 unreachable;1135 // The streaming parser would return an error eventually.
1136 // To prevent invalid state we return an error now.
1137 // TODO make the streaming parser return an error as soon as it encounters an invalid object key
1138 return error.InvalidLiteral;
1128 },1139 },
1129 },1140 },
1130 State.ObjectValue => {1141 State.ObjectValue => {
...@@ -1266,7 +1277,7 @@ pub const Parser = struct {...@@ -1266,7 +1277,7 @@ pub const Parser = struct {
1266 // TODO: We don't strictly have to copy values which do not contain any escape1277 // TODO: We don't strictly have to copy values which do not contain any escape
1267 // characters if flagged with the option.1278 // characters if flagged with the option.
1268 const slice = token.slice(input, i);1279 const slice = token.slice(input, i);
1269 return Value{ .String = try mem.dupe(allocator, u8, slice) };1280 return Value{ .String = try unescapeStringAlloc(allocator, slice) };
1270 }1281 }
12711282
1272 fn parseNumber(p: *Parser, token: Token, input: []const u8, i: usize) !Value {1283 fn parseNumber(p: *Parser, token: Token, input: []const u8, i: usize) !Value {
...@@ -1277,6 +1288,77 @@ pub const Parser = struct {...@@ -1277,6 +1288,77 @@ pub const Parser = struct {
1277 }1288 }
1278};1289};
12791290
1291// Unescape a JSON string
1292// Only to be used on strings already validated by the parser
1293// (note the unreachable statements and lack of bounds checking)
1294// Optimized for arena allocators, uses Allocator.shrink
1295//
1296// Idea: count how many bytes we will need to allocate in the streaming parser and store it
1297// in the token to avoid allocating too much memory or iterating through the string again
1298// Downside: need to find how many bytes a unicode escape sequence will produce twice
1299fn unescapeStringAlloc(alloc: *Allocator, input: []const u8) ![]u8 {
1300 const output = try alloc.alloc(u8, input.len);
1301 errdefer alloc.free(output);
1302
1303 var inIndex: usize = 0;
1304 var outIndex: usize = 0;
1305
1306 while(inIndex < input.len) {
1307 if(input[inIndex] != '\\'){
1308 // not an escape sequence
1309 output[outIndex] = input[inIndex];
1310 inIndex += 1;
1311 outIndex += 1;
1312 } else if(input[inIndex + 1] != 'u'){
1313 // a simple escape sequence
1314 output[outIndex] = @as(u8,
1315 switch(input[inIndex + 1]){
1316 '\\' => '\\',
1317 '/' => '/',
1318 'n' => '\n',
1319 'r' => '\r',
1320 't' => '\t',
1321 'f' => 12,
1322 'b' => 8,
1323 '"' => '"',
1324 else => unreachable
1325 }
1326 );
1327 inIndex += 2;
1328 outIndex += 1;
1329 } else {
1330 // a unicode escape sequence
1331 const firstCodeUnit = std.fmt.parseInt(u16, input[inIndex+2 .. inIndex+6], 16) catch unreachable;
1332
1333 // guess optimistically that it's not a surrogate pair
1334 if(std.unicode.utf8Encode(firstCodeUnit, output[outIndex..])) |byteCount| {
1335 outIndex += byteCount;
1336 inIndex += 6;
1337 } else |err| {
1338 // it might be a surrogate pair
1339 if(err != error.Utf8CannotEncodeSurrogateHalf) {
1340 return error.InvalidUnicodeHexSymbol;
1341 }
1342 // check if a second code unit is present
1343 if(inIndex + 7 >= input.len or input[inIndex + 6] != '\\' or input[inIndex + 7] != 'u'){
1344 return error.InvalidUnicodeHexSymbol;
1345 }
1346
1347 const secondCodeUnit = std.fmt.parseInt(u16, input[inIndex+8 .. inIndex+12], 16) catch unreachable;
1348
1349 if(std.unicode.utf16leToUtf8(output[outIndex..], [2]u16{ firstCodeUnit, secondCodeUnit })) |byteCount| {
1350 outIndex += byteCount;
1351 inIndex += 12;
1352 } else |_| {
1353 return error.InvalidUnicodeHexSymbol;
1354 }
1355 }
1356 }
1357 }
1358
1359 return alloc.shrink(output, outIndex);
1360}
1361
1280test "json.parser.dynamic" {1362test "json.parser.dynamic" {
1281 var p = Parser.init(debug.global_allocator, false);1363 var p = Parser.init(debug.global_allocator, false);
1282 defer p.deinit();1364 defer p.deinit();
...@@ -1399,3 +1481,36 @@ test "integer after float has proper type" {...@@ -1399,3 +1481,36 @@ test "integer after float has proper type" {
1399 );1481 );
1400 std.testing.expect(json.Object.getValue("ints").?.Array.at(0) == .Integer);1482 std.testing.expect(json.Object.getValue("ints").?.Array.at(0) == .Integer);
1401}1483}
1484
1485test "escaped characters" {
1486 const input =
1487 \\{
1488 \\ "backslash": "\\",
1489 \\ "forwardslash": "\/",
1490 \\ "newline": "\n",
1491 \\ "carriagereturn": "\r",
1492 \\ "tab": "\t",
1493 \\ "formfeed": "\f",
1494 \\ "backspace": "\b",
1495 \\ "doublequote": "\"",
1496 \\ "unicode": "\u0105",
1497 \\ "surrogatepair": "\ud83d\ude02"
1498 \\}
1499 ;
1500
1501 var p = Parser.init(debug.global_allocator, false);
1502 const tree = try p.parse(input);
1503
1504 const obj = tree.root.Object;
1505
1506 testing.expectEqualSlices(u8, obj.get("backslash").?.value.String, "\\");
1507 testing.expectEqualSlices(u8, obj.get("forwardslash").?.value.String, "/");
1508 testing.expectEqualSlices(u8, obj.get("newline").?.value.String, "\n");
1509 testing.expectEqualSlices(u8, obj.get("carriagereturn").?.value.String, "\r");
1510 testing.expectEqualSlices(u8, obj.get("tab").?.value.String, "\t");
1511 testing.expectEqualSlices(u8, obj.get("formfeed").?.value.String, "\x0C");
1512 testing.expectEqualSlices(u8, obj.get("backspace").?.value.String, "\x08");
1513 testing.expectEqualSlices(u8, obj.get("doublequote").?.value.String, "\"");
1514 testing.expectEqualSlices(u8, obj.get("unicode").?.value.String, "ą");
1515 testing.expectEqualSlices(u8, obj.get("surrogatepair").?.value.String, "😂");
1516}
lib/std/json/test.zig+54-20
...@@ -7,14 +7,46 @@ const std = @import("../std.zig");...@@ -7,14 +7,46 @@ const std = @import("../std.zig");
77
8fn ok(comptime s: []const u8) void {8fn ok(comptime s: []const u8) void {
9 std.testing.expect(std.json.validate(s));9 std.testing.expect(std.json.validate(s));
10
11 var mem_buffer: [1024 * 20]u8 = undefined;
12 const allocator = &std.heap.FixedBufferAllocator.init(&mem_buffer).allocator;
13 var p = std.json.Parser.init(allocator, false);
14
15 _ = p.parse(s) catch unreachable;
10}16}
1117
12fn err(comptime s: []const u8) void {18fn err(comptime s: []const u8) void {
13 std.testing.expect(!std.json.validate(s));19 std.testing.expect(!std.json.validate(s));
20
21 var mem_buffer: [1024 * 20]u8 = undefined;
22 const allocator = &std.heap.FixedBufferAllocator.init(&mem_buffer).allocator;
23 var p = std.json.Parser.init(allocator, false);
24
25 if(p.parse(s)) |_| {
26 unreachable;
27 } else |_| {}
14}28}
1529
16fn any(comptime s: []const u8) void {30fn any(comptime s: []const u8) void {
17 std.testing.expect(true);31 _ = std.json.validate(s);
32
33 var mem_buffer: [1024 * 20]u8 = undefined;
34 const allocator = &std.heap.FixedBufferAllocator.init(&mem_buffer).allocator;
35 var p = std.json.Parser.init(allocator, false);
36
37 _ = p.parse(s) catch {};
38}
39
40fn anyStreamingErrNonStreaming(comptime s: []const u8) void {
41 _ = std.json.validate(s);
42
43 var mem_buffer: [1024 * 20]u8 = undefined;
44 const allocator = &std.heap.FixedBufferAllocator.init(&mem_buffer).allocator;
45 var p = std.json.Parser.init(allocator, false);
46
47 if(p.parse(s)) |_| {
48 unreachable;
49 } else |_| {}
18}50}
1951
20////////////////////////////////////////////////////////////////////////////////////////////////////52////////////////////////////////////////////////////////////////////////////////////////////////////
...@@ -611,9 +643,9 @@ test "n_array_colon_instead_of_comma" {...@@ -611,9 +643,9 @@ test "n_array_colon_instead_of_comma" {
611}643}
612644
613test "n_array_comma_after_close" {645test "n_array_comma_after_close" {
614 //err(646 err(
615 // \\[""],647 \\[""],
616 //);648 );
617}649}
618650
619test "n_array_comma_and_number" {651test "n_array_comma_and_number" {
...@@ -641,9 +673,9 @@ test "n_array_extra_close" {...@@ -641,9 +673,9 @@ test "n_array_extra_close" {
641}673}
642674
643test "n_array_extra_comma" {675test "n_array_extra_comma" {
644 //err(676 err(
645 // \\["",]677 \\["",]
646 //);678 );
647}679}
648680
649test "n_array_incomplete_invalid_value" {681test "n_array_incomplete_invalid_value" {
...@@ -1708,9 +1740,11 @@ test "i_number_double_huge_neg_exp" {...@@ -1708,9 +1740,11 @@ test "i_number_double_huge_neg_exp" {
1708}1740}
17091741
1710test "i_number_huge_exp" {1742test "i_number_huge_exp" {
1711 any(1743 return error.SkipZigTest;
1712 \\[0.4e00669999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999969999999006]1744 // FIXME Integer overflow in parseFloat
1713 );1745// any(
1746// \\[0.4e00669999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999969999999006]
1747// );
1714}1748}
17151749
1716test "i_number_neg_int_huge_exp" {1750test "i_number_neg_int_huge_exp" {
...@@ -1762,49 +1796,49 @@ test "i_number_very_big_negative_int" {...@@ -1762,49 +1796,49 @@ test "i_number_very_big_negative_int" {
1762}1796}
17631797
1764test "i_object_key_lone_2nd_surrogate" {1798test "i_object_key_lone_2nd_surrogate" {
1765 any(1799 anyStreamingErrNonStreaming(
1766 \\{"\uDFAA":0}1800 \\{"\uDFAA":0}
1767 );1801 );
1768}1802}
17691803
1770test "i_string_1st_surrogate_but_2nd_missing" {1804test "i_string_1st_surrogate_but_2nd_missing" {
1771 any(1805 anyStreamingErrNonStreaming(
1772 \\["\uDADA"]1806 \\["\uDADA"]
1773 );1807 );
1774}1808}
17751809
1776test "i_string_1st_valid_surrogate_2nd_invalid" {1810test "i_string_1st_valid_surrogate_2nd_invalid" {
1777 any(1811 anyStreamingErrNonStreaming(
1778 \\["\uD888\u1234"]1812 \\["\uD888\u1234"]
1779 );1813 );
1780}1814}
17811815
1782test "i_string_incomplete_surrogate_and_escape_valid" {1816test "i_string_incomplete_surrogate_and_escape_valid" {
1783 any(1817 anyStreamingErrNonStreaming(
1784 \\["\uD800\n"]1818 \\["\uD800\n"]
1785 );1819 );
1786}1820}
17871821
1788test "i_string_incomplete_surrogate_pair" {1822test "i_string_incomplete_surrogate_pair" {
1789 any(1823 anyStreamingErrNonStreaming(
1790 \\["\uDd1ea"]1824 \\["\uDd1ea"]
1791 );1825 );
1792}1826}
17931827
1794test "i_string_incomplete_surrogates_escape_valid" {1828test "i_string_incomplete_surrogates_escape_valid" {
1795 any(1829 anyStreamingErrNonStreaming(
1796 \\["\uD800\uD800\n"]1830 \\["\uD800\uD800\n"]
1797 );1831 );
1798}1832}
17991833
1800test "i_string_invalid_lonely_surrogate" {1834test "i_string_invalid_lonely_surrogate" {
1801 any(1835 anyStreamingErrNonStreaming(
1802 \\["\ud800"]1836 \\["\ud800"]
1803 );1837 );
1804}1838}
18051839
1806test "i_string_invalid_surrogate" {1840test "i_string_invalid_surrogate" {
1807 any(1841 anyStreamingErrNonStreaming(
1808 \\["\ud800abc"]1842 \\["\ud800abc"]
1809 );1843 );
1810}1844}
...@@ -1816,7 +1850,7 @@ test "i_string_invalid_utf-8" {...@@ -1816,7 +1850,7 @@ test "i_string_invalid_utf-8" {
1816}1850}
18171851
1818test "i_string_inverted_surrogates_U+1D11E" {1852test "i_string_inverted_surrogates_U+1D11E" {
1819 any(1853 anyStreamingErrNonStreaming(
1820 \\["\uDd1e\uD834"]1854 \\["\uDd1e\uD834"]
1821 );1855 );
1822}1856}
...@@ -1828,7 +1862,7 @@ test "i_string_iso_latin_1" {...@@ -1828,7 +1862,7 @@ test "i_string_iso_latin_1" {
1828}1862}
18291863
1830test "i_string_lone_second_surrogate" {1864test "i_string_lone_second_surrogate" {
1831 any(1865 anyStreamingErrNonStreaming(
1832 \\["\uDFAA"]1866 \\["\uDFAA"]
1833 );1867 );
1834}1868}