authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-05-30 14:59:02-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2021-05-30 14:59:02-04:00
log11ae6c42c1851fc44b286e5a76f73d55ea0bca5e
treefecbff35be00027c69a563334b9def3fd7c23578
parent0ad1c04dd99d130f4fec124940f05df5132ee5a2
parent32815914a487ab6d69a29ef1ff8d7e527ffc82ff
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #7918 from EthanGruffudd/json-ignore-fields

Add option to ignore unknown fields when parsing json

1 files changed, 117 insertions(+), 1 deletions(-)

lib/std/json.zig+117-1
...@@ -1105,6 +1105,10 @@ pub const TokenStream = struct {...@@ -1105,6 +1105,10 @@ pub const TokenStream = struct {
1105 };1105 };
1106 }1106 }
11071107
1108 fn stackUsed(self: *TokenStream) u8 {
1109 return self.parser.stack_used + if (self.token != null) @as(u8, 1) else 0;
1110 }
1111
1108 pub fn next(self: *TokenStream) Error!?Token {1112 pub fn next(self: *TokenStream) Error!?Token {
1109 if (self.token) |token| {1113 if (self.token) |token| {
1110 self.token = null;1114 self.token = null;
...@@ -1457,8 +1461,73 @@ pub const ParseOptions = struct {...@@ -1457,8 +1461,73 @@ pub const ParseOptions = struct {
1457 Error,1461 Error,
1458 UseLast,1462 UseLast,
1459 } = .Error,1463 } = .Error,
1464
1465 /// If false, finding an unknown field returns an error.
1466 ignore_unknown_fields: bool = false,
1460};1467};
14611468
1469fn skipValue(tokens: *TokenStream) !void {
1470 const original_depth = tokens.stackUsed();
1471
1472 // Return an error if no value is found
1473 _ = try tokens.next();
1474 if (tokens.stackUsed() < original_depth) return error.UnexpectedJsonDepth;
1475 if (tokens.stackUsed() == original_depth) return;
1476
1477 while (try tokens.next()) |_| {
1478 if (tokens.stackUsed() == original_depth) return;
1479 }
1480}
1481
1482test "skipValue" {
1483 try skipValue(&TokenStream.init("false"));
1484 try skipValue(&TokenStream.init("true"));
1485 try skipValue(&TokenStream.init("null"));
1486 try skipValue(&TokenStream.init("42"));
1487 try skipValue(&TokenStream.init("42.0"));
1488 try skipValue(&TokenStream.init("\"foo\""));
1489 try skipValue(&TokenStream.init("[101, 111, 121]"));
1490 try skipValue(&TokenStream.init("{}"));
1491 try skipValue(&TokenStream.init("{\"foo\": \"bar\"}"));
1492
1493 { // An absurd number of nestings
1494 const nestings = 256;
1495
1496 try testing.expectError(
1497 error.TooManyNestedItems,
1498 skipValue(&TokenStream.init("[" ** nestings ++ "]" ** nestings)),
1499 );
1500 }
1501
1502 { // Would a number token cause problems in a deeply-nested array?
1503 const nestings = 255;
1504 const deeply_nested_array = "[" ** nestings ++ "0.118, 999, 881.99, 911.9, 725, 3" ++ "]" ** nestings;
1505
1506 try skipValue(&TokenStream.init(deeply_nested_array));
1507
1508 try testing.expectError(
1509 error.TooManyNestedItems,
1510 skipValue(&TokenStream.init("[" ++ deeply_nested_array ++ "]")),
1511 );
1512 }
1513
1514 // Mismatched brace/square bracket
1515 try testing.expectError(
1516 error.UnexpectedClosingBrace,
1517 skipValue(&TokenStream.init("[102, 111, 111}")),
1518 );
1519
1520 { // should fail if no value found (e.g. immediate close of object)
1521 var empty_object = TokenStream.init("{}");
1522 assert(.ObjectBegin == (try empty_object.next()).?);
1523 try testing.expectError(error.UnexpectedJsonDepth, skipValue(&empty_object));
1524
1525 var empty_array = TokenStream.init("[]");
1526 assert(.ArrayBegin == (try empty_array.next()).?);
1527 try testing.expectError(error.UnexpectedJsonDepth, skipValue(&empty_array));
1528 }
1529}
1530
1462fn parseInternal(comptime T: type, token: Token, tokens: *TokenStream, options: ParseOptions) !T {1531fn parseInternal(comptime T: type, token: Token, tokens: *TokenStream, options: ParseOptions) !T {
1463 switch (@typeInfo(T)) {1532 switch (@typeInfo(T)) {
1464 .Bool => {1533 .Bool => {
...@@ -1598,7 +1667,14 @@ fn parseInternal(comptime T: type, token: Token, tokens: *TokenStream, options:...@@ -1598,7 +1667,14 @@ fn parseInternal(comptime T: type, token: Token, tokens: *TokenStream, options:
1598 break;1667 break;
1599 }1668 }
1600 }1669 }
1601 if (!found) return error.UnknownField;1670 if (!found) {
1671 if (options.ignore_unknown_fields) {
1672 try skipValue(tokens);
1673 continue;
1674 } else {
1675 return error.UnknownField;
1676 }
1677 }
1602 },1678 },
1603 else => return error.UnexpectedToken,1679 else => return error.UnexpectedToken,
1604 }1680 }
...@@ -2040,6 +2116,46 @@ test "parse into struct with duplicate field" {...@@ -2040,6 +2116,46 @@ test "parse into struct with duplicate field" {
2040 try testing.expectError(error.UnexpectedValue, parse(T3, &TokenStream.init(str), options_last));2116 try testing.expectError(error.UnexpectedValue, parse(T3, &TokenStream.init(str), options_last));
2041}2117}
20422118
2119test "parse into struct ignoring unknown fields" {
2120 const T = struct {
2121 int: i64,
2122 language: []const u8,
2123 };
2124
2125 const ops = ParseOptions{
2126 .allocator = testing.allocator,
2127 .ignore_unknown_fields = true,
2128 };
2129
2130 const r = try parse(T, &std.json.TokenStream.init(
2131 \\{
2132 \\ "int": 420,
2133 \\ "float": 3.14,
2134 \\ "with\\escape": true,
2135 \\ "with\u0105unicode\ud83d\ude02": false,
2136 \\ "optional": null,
2137 \\ "static_array": [66.6, 420.420, 69.69],
2138 \\ "dynamic_array": [66.6, 420.420, 69.69],
2139 \\ "complex": {
2140 \\ "nested": "zig"
2141 \\ },
2142 \\ "veryComplex": [
2143 \\ {
2144 \\ "foo": "zig"
2145 \\ }, {
2146 \\ "foo": "rocks"
2147 \\ }
2148 \\ ],
2149 \\ "a_union": 100000,
2150 \\ "language": "zig"
2151 \\}
2152 ), ops);
2153 defer parseFree(T, r, ops);
2154
2155 try testing.expectEqual(@as(i64, 420), r.int);
2156 try testing.expectEqualSlices(u8, "zig", r.language);
2157}
2158
2043/// A non-stream JSON parser which constructs a tree of Value's.2159/// A non-stream JSON parser which constructs a tree of Value's.
2044pub const Parser = struct {2160pub const Parser = struct {
2045 allocator: *Allocator,2161 allocator: *Allocator,