Commit 32cb9462ffa0a9df7a080d67eaf3a5762173f742 | zig

author	Josh Wolfe <thejoshwolfe@gmail.com> 2023-06-19 11:21:37-04:00
committer	GitHub <noreply@github.com> 2023-06-19 11:21:37-04:00
log	32cb9462ffa0a9df7a080d67eaf3a5762173f742
tree	b3d25551b9cfed2e0b21f12864bcc33cae9d6739
parent	423d7b848b1953173df99fde1f83166dc68c2a2c
signature	Signed by PGP key 4AEE18F83AFDEB23

std: Support user-provided jsonParse method. Unify json.Parser and json.parse* (#15705)

11 files changed, 471 insertions(+), 696 deletions(-)

lib
- std
  - json.zig
  - json
    dynamic.zig
    dynamic_test.zig
    static.zig
    static_test.zig
    test.zig
tools

lib/std/json.zig+66-7

...	...	@@ -1,19 +1,73 @@
1	1	//! JSON parsing and stringification conforming to RFC 8259. https://datatracker.ietf.org/doc/html/rfc8259
2	2	//!
3		//! The low-level `Scanner` API reads from an input slice or successive slices of inputs,
	3	//! The low-level `Scanner` API produces `Token`s from an input slice or successive slices of inputs,
4	4	//! The `Reader` API connects a `std.io.Reader` to a `Scanner`.
5	5	//!
6		//! The high-level `parseFromSlice` and `parseFromTokenSource` deserializes a JSON document into a Zig type.
7		//! The high-level `Parser` parses any JSON document into a dynamically typed `ValueTree` that has its own memory arena.
	6	//! The high-level `parseFromSlice` and `parseFromTokenSource` deserialize a JSON document into a Zig type.
	7	//! Parse into a dynamically-typed `Value` to load any JSON value for runtime inspection.
8	8	//!
9	9	//! The low-level `writeStream` emits syntax-conformant JSON tokens to a `std.io.Writer`.
10		//! The high-level `stringify` serializes a Zig type into JSON.
	10	//! The high-level `stringify` serializes a Zig or `Value` type into JSON.
	11
	12	const testing = @import("std").testing;
	13	const ArrayList = @import("std").ArrayList;
	14
	15	test Scanner {
	16	var scanner = Scanner.initCompleteInput(testing.allocator, "{\"foo\": 123}\n");
	17	defer scanner.deinit();
	18	try testing.expectEqual(Token.object_begin, try scanner.next());
	19	try testing.expectEqualSlices(u8, "foo", (try scanner.next()).string);
	20	try testing.expectEqualSlices(u8, "123", (try scanner.next()).number);
	21	try testing.expectEqual(Token.object_end, try scanner.next());
	22	try testing.expectEqual(Token.end_of_document, try scanner.next());
	23	}
	24
	25	test parseFromSlice {
	26	var parsed_str = try parseFromSlice([]const u8, testing.allocator, "\"a\\u0020b\"", .{});
	27	defer parsed_str.deinit();
	28	try testing.expectEqualSlices(u8, "a b", parsed_str.value);
	29
	30	const T = struct { a: i32 = -1, b: [2]u8 };
	31	var parsed_struct = try parseFromSlice(T, testing.allocator, "{\"b\":\"xy\"}", .{});
	32	defer parsed_struct.deinit();
	33	try testing.expectEqual(@as(i32, -1), parsed_struct.value.a); // default value
	34	try testing.expectEqualSlices(u8, "xy", parsed_struct.value.b[0..]);
	35	}
	36
	37	test Value {
	38	var parsed = try parseFromSlice(Value, testing.allocator, "{\"anything\": \"goes\"}", .{});
	39	defer parsed.deinit();
	40	try testing.expectEqualSlices(u8, "goes", parsed.value.object.get("anything").?.string);
	41	}
	42
	43	test writeStream {
	44	var out = ArrayList(u8).init(testing.allocator);
	45	defer out.deinit();
	46	var write_stream = writeStream(out.writer(), 99);
	47	try write_stream.beginObject();
	48	try write_stream.objectField("foo");
	49	try write_stream.emitNumber(123);
	50	try write_stream.endObject();
	51	const expected =
	52	\\{
	53	\\ "foo": 123
	54	\\}
	55	;
	56	try testing.expectEqualSlices(u8, expected, out.items);
	57	}
	58
	59	test stringify {
	60	var out = ArrayList(u8).init(testing.allocator);
	61	defer out.deinit();
	62
	63	const T = struct { a: i32, b: []const u8 };
	64	try stringify(T{ .a = 123, .b = "xy" }, .{}, out.writer());
	65	try testing.expectEqualSlices(u8, "{\"a\":123,\"b\":\"xy\"}", out.items);
	66	}
11	67
12		pub const ValueTree = @import("json/dynamic.zig").ValueTree;
13	68	pub const ObjectMap = @import("json/dynamic.zig").ObjectMap;
14	69	pub const Array = @import("json/dynamic.zig").Array;
15	70	pub const Value = @import("json/dynamic.zig").Value;
16		pub const Parser = @import("json/dynamic.zig").Parser;
17	71
18	72	pub const validate = @import("json/scanner.zig").validate;
19	73	pub const Error = @import("json/scanner.zig").Error;
...	...	@@ -30,9 +84,11 @@ pub const isNumberFormattedLikeAnInteger = @import("json/scanner.zig").isNumberF
30	84
31	85	pub const ParseOptions = @import("json/static.zig").ParseOptions;
32	86	pub const parseFromSlice = @import("json/static.zig").parseFromSlice;
	87	pub const parseFromSliceLeaky = @import("json/static.zig").parseFromSliceLeaky;
33	88	pub const parseFromTokenSource = @import("json/static.zig").parseFromTokenSource;
	89	pub const parseFromTokenSourceLeaky = @import("json/static.zig").parseFromTokenSourceLeaky;
34	90	pub const ParseError = @import("json/static.zig").ParseError;
35		pub const parseFree = @import("json/static.zig").parseFree;
	91	pub const Parsed = @import("json/static.zig").Parsed;
36	92
37	93	pub const StringifyOptions = @import("json/stringify.zig").StringifyOptions;
38	94	pub const encodeJsonString = @import("json/stringify.zig").encodeJsonString;
...	...	@@ -45,6 +101,9 @@ pub const writeStream = @import("json/write_stream.zig").writeStream;
45	101
46	102	// Deprecations
47	103	pub const parse = @compileError("Deprecated; use parseFromSlice() or parseFromTokenSource() instead.");
	104	pub const parseFree = @compileError("Deprecated; call Parsed(T).deinit() instead.");
	105	pub const Parser = @compileError("Deprecated; use parseFromSlice(Value) or parseFromTokenSource(Value) instead.");
	106	pub const ValueTree = @compileError("Deprecated; use Parsed(Value) instead.");
48	107	pub const StreamingParser = @compileError("Deprecated; use json.Scanner or json.Reader instead.");
49	108	pub const TokenStream = @compileError("Deprecated; use json.Scanner or json.Reader instead.");
50	109

lib/std/json/dynamic.zig+106-251

...	...	@@ -8,26 +8,20 @@ const Allocator = std.mem.Allocator;
8	8	const StringifyOptions = @import("./stringify.zig").StringifyOptions;
9	9	const stringify = @import("./stringify.zig").stringify;
10	10
	11	const ParseOptions = @import("./static.zig").ParseOptions;
	12	const ParseError = @import("./static.zig").ParseError;
	13
11	14	const JsonScanner = @import("./scanner.zig").Scanner;
12	15	const AllocWhen = @import("./scanner.zig").AllocWhen;
13	16	const Token = @import("./scanner.zig").Token;
14	17	const isNumberFormattedLikeAnInteger = @import("./scanner.zig").isNumberFormattedLikeAnInteger;
15	18
16		pub const ValueTree = struct {
17		arena: *ArenaAllocator,
18		root: Value,
19
20		pub fn deinit(self: *ValueTree) void {
21		self.arena.deinit();
22		self.arena.child_allocator.destroy(self.arena);
23		}
24		};
25
26	19	pub const ObjectMap = StringArrayHashMap(Value);
27	20	pub const Array = ArrayList(Value);
28	21
29		/// Represents a JSON value
30		/// Currently only supports numbers that fit into i64 or f64.
	22	/// Represents any JSON value, potentially containing other JSON values.
	23	/// A .float value may be an approximation of the original value.
	24	/// Arbitrary precision numbers can be represented by .number_string values.
31	25	pub const Value = union(enum) {
32	26	null,
33	27	bool: bool,
...	...	@@ -38,6 +32,33 @@ pub const Value = union(enum) {
38	32	array: Array,
39	33	object: ObjectMap,
40	34
	35	pub fn parseFromNumberSlice(s: []const u8) Value {
	36	if (!isNumberFormattedLikeAnInteger(s)) {
	37	const f = std.fmt.parseFloat(f64, s) catch unreachable;
	38	if (std.math.isFinite(f)) {
	39	return Value{ .float = f };
	40	} else {
	41	return Value{ .number_string = s };
	42	}
	43	}
	44	if (std.fmt.parseInt(i64, s, 10)) \|i\| {
	45	return Value{ .integer = i };
	46	} else \|e\| {
	47	switch (e) {
	48	error.Overflow => return Value{ .number_string = s },
	49	error.InvalidCharacter => unreachable,
	50	}
	51	}
	52	}
	53
	54	pub fn dump(self: Value) void {
	55	std.debug.getStderrMutex().lock();
	56	defer std.debug.getStderrMutex().unlock();
	57
	58	const stderr = std.io.getStdErr().writer();
	59	stringify(self, .{}, stderr) catch return;
	60	}
	61
41	62	pub fn jsonStringify(
42	63	value: @This(),
43	64	options: StringifyOptions,
...	...	@@ -80,264 +101,98 @@ pub const Value = union(enum) {
80	101	}
81	102	}
82	103
83		pub fn dump(self: Value) void {
84		std.debug.getStderrMutex().lock();
85		defer std.debug.getStderrMutex().unlock();
86
87		const stderr = std.io.getStdErr().writer();
88		stringify(self, .{}, stderr) catch return;
89		}
90		};
91
92		/// A non-stream JSON parser which constructs a tree of Value's.
93		pub const Parser = struct {
94		allocator: Allocator,
95		state: State,
96		alloc_when: AllocWhen,
97		// Stores parent nodes and un-combined Values.
98		stack: Array,
99
100		const State = enum {
101		object_key,
102		object_value,
103		array_value,
104		simple,
105		};
106
107		pub fn init(allocator: Allocator, alloc_when: AllocWhen) Parser {
108		return Parser{
109		.allocator = allocator,
110		.state = .simple,
111		.alloc_when = alloc_when,
112		.stack = Array.init(allocator),
113		};
114		}
115
116		pub fn deinit(p: *Parser) void {
117		p.stack.deinit();
118		}
119
120		pub fn reset(p: *Parser) void {
121		p.state = .simple;
122		p.stack.shrinkRetainingCapacity(0);
123		}
124
125		pub fn parse(p: *Parser, input: []const u8) !ValueTree {
126		var scanner = JsonScanner.initCompleteInput(p.allocator, input);
127		defer scanner.deinit();
128
129		var arena = try p.allocator.create(ArenaAllocator);
130		errdefer p.allocator.destroy(arena);
131
132		arena.* = ArenaAllocator.init(p.allocator);
133		errdefer arena.deinit();
134
135		const allocator = arena.allocator();
	104	pub fn jsonParse(allocator: Allocator, source: anytype, options: ParseOptions) ParseError(@TypeOf(source.*))!@This() {
	105	_ = options;
	106	// The grammar of the stack is:
	107	// (.array \| .object .string)*
	108	var stack = Array.init(allocator);
	109	defer stack.deinit();
136	110
137	111	while (true) {
138		const token = try scanner.nextAlloc(allocator, p.alloc_when);
139		if (token == .end_of_document) break;
140		try p.transition(allocator, token);
141		}
142
143		debug.assert(p.stack.items.len == 1);
144
145		return ValueTree{
146		.arena = arena,
147		.root = p.stack.items[0],
148		};
149		}
150
151		// Even though p.allocator exists, we take an explicit allocator so that allocation state
152		// can be cleaned up on error correctly during a `parse` on call.
153		fn transition(p: *Parser, allocator: Allocator, token: Token) !void {
154		switch (p.state) {
155		.object_key => switch (token) {
156		.object_end => {
157		if (p.stack.items.len == 1) {
158		return;
159		}
160
161		var value = p.stack.pop();
162		try p.pushToParent(&value);
163		},
164		.string => \|s\| {
165		try p.stack.append(Value{ .string = s });
166		p.state = .object_value;
	112	// Assert the stack grammar at the top of the stack.
	113	debug.assert(stack.items.len == 0 or
	114	stack.items[stack.items.len - 1] == .array or
	115	(stack.items[stack.items.len - 2] == .object and stack.items[stack.items.len - 1] == .string));
	116
	117	switch (try source.nextAlloc(allocator, .alloc_if_needed)) {
	118	inline .string, .allocated_string => \|s\| {
	119	return try handleCompleteValue(&stack, allocator, source, Value{ .string = s }) orelse continue;
167	120	},
168		.allocated_string => \|s\| {
169		try p.stack.append(Value{ .string = s });
170		p.state = .object_value;
	121	inline .number, .allocated_number => \|slice\| {
	122	return try handleCompleteValue(&stack, allocator, source, Value.parseFromNumberSlice(slice)) orelse continue;
171	123	},
172		else => unreachable,
173		},
174		.object_value => {
175		var object = &p.stack.items[p.stack.items.len - 2].object;
176		var key = p.stack.items[p.stack.items.len - 1].string;
177
178		switch (token) {
179		.object_begin => {
180		try p.stack.append(Value{ .object = ObjectMap.init(allocator) });
181		p.state = .object_key;
182		},
183		.array_begin => {
184		try p.stack.append(Value{ .array = Array.init(allocator) });
185		p.state = .array_value;
186		},
187		.string => \|s\| {
188		try object.put(key, Value{ .string = s });
189		_ = p.stack.pop();
190		p.state = .object_key;
191		},
192		.allocated_string => \|s\| {
193		try object.put(key, Value{ .string = s });
194		_ = p.stack.pop();
195		p.state = .object_key;
196		},
197		.number => \|slice\| {
198		try object.put(key, try p.parseNumber(slice));
199		_ = p.stack.pop();
200		p.state = .object_key;
201		},
202		.allocated_number => \|slice\| {
203		try object.put(key, try p.parseNumber(slice));
204		_ = p.stack.pop();
205		p.state = .object_key;
206		},
207		.true => {
208		try object.put(key, Value{ .bool = true });
209		_ = p.stack.pop();
210		p.state = .object_key;
211		},
212		.false => {
213		try object.put(key, Value{ .bool = false });
214		_ = p.stack.pop();
215		p.state = .object_key;
216		},
217		.null => {
218		try object.put(key, .null);
219		_ = p.stack.pop();
220		p.state = .object_key;
221		},
222		.object_end, .array_end, .end_of_document => unreachable,
223		.partial_number, .partial_string, .partial_string_escaped_1, .partial_string_escaped_2, .partial_string_escaped_3, .partial_string_escaped_4 => unreachable,
224		}
225		},
226		.array_value => {
227		var array = &p.stack.items[p.stack.items.len - 1].array;
228	124
229		switch (token) {
230		.array_end => {
231		if (p.stack.items.len == 1) {
232		return;
233		}
	125	.null => return try handleCompleteValue(&stack, allocator, source, .null) orelse continue,
	126	.true => return try handleCompleteValue(&stack, allocator, source, Value{ .bool = true }) orelse continue,
	127	.false => return try handleCompleteValue(&stack, allocator, source, Value{ .bool = false }) orelse continue,
234	128
235		var value = p.stack.pop();
236		try p.pushToParent(&value);
237		},
238		.object_begin => {
239		try p.stack.append(Value{ .object = ObjectMap.init(allocator) });
240		p.state = .object_key;
241		},
242		.array_begin => {
243		try p.stack.append(Value{ .array = Array.init(allocator) });
244		p.state = .array_value;
245		},
246		.string => \|s\| {
247		try array.append(Value{ .string = s });
248		},
249		.allocated_string => \|s\| {
250		try array.append(Value{ .string = s });
251		},
252		.number => \|slice\| {
253		try array.append(try p.parseNumber(slice));
254		},
255		.allocated_number => \|slice\| {
256		try array.append(try p.parseNumber(slice));
257		},
258		.true => {
259		try array.append(Value{ .bool = true });
260		},
261		.false => {
262		try array.append(Value{ .bool = false });
263		},
264		.null => {
265		try array.append(.null);
266		},
267		.object_end, .end_of_document => unreachable,
268		.partial_number, .partial_string, .partial_string_escaped_1, .partial_string_escaped_2, .partial_string_escaped_3, .partial_string_escaped_4 => unreachable,
269		}
270		},
271		.simple => switch (token) {
272	129	.object_begin => {
273		try p.stack.append(Value{ .object = ObjectMap.init(allocator) });
274		p.state = .object_key;
	130	switch (try source.nextAlloc(allocator, .alloc_if_needed)) {
	131	.object_end => return try handleCompleteValue(&stack, allocator, source, Value{ .object = ObjectMap.init(allocator) }) orelse continue,
	132	inline .string, .allocated_string => \|key\| {
	133	try stack.appendSlice(&[_]Value{
	134	Value{ .object = ObjectMap.init(allocator) },
	135	Value{ .string = key },
	136	});
	137	},
	138	else => unreachable,
	139	}
275	140	},
276	141	.array_begin => {
277		try p.stack.append(Value{ .array = Array.init(allocator) });
278		p.state = .array_value;
279		},
280		.string => \|s\| {
281		try p.stack.append(Value{ .string = s });
282		},
283		.allocated_string => \|s\| {
284		try p.stack.append(Value{ .string = s });
285		},
286		.number => \|slice\| {
287		try p.stack.append(try p.parseNumber(slice));
288		},
289		.allocated_number => \|slice\| {
290		try p.stack.append(try p.parseNumber(slice));
	142	try stack.append(Value{ .array = Array.init(allocator) });
291	143	},
292		.true => {
293		try p.stack.append(Value{ .bool = true });
294		},
295		.false => {
296		try p.stack.append(Value{ .bool = false });
297		},
298		.null => {
299		try p.stack.append(.null);
300		},
301		.object_end, .array_end, .end_of_document => unreachable,
302		.partial_number, .partial_string, .partial_string_escaped_1, .partial_string_escaped_2, .partial_string_escaped_3, .partial_string_escaped_4 => unreachable,
303		},
	144	.array_end => return try handleCompleteValue(&stack, allocator, source, stack.pop()) orelse continue,
	145
	146	else => unreachable,
	147	}
304	148	}
305	149	}
	150	};
306	151
307		fn pushToParent(p: Parser, value: const Value) !void {
308		switch (p.stack.items[p.stack.items.len - 1]) {
309		// Object Parent -> [ ..., object, <key>, value ]
	152	fn handleCompleteValue(stack: *Array, allocator: Allocator, source: anytype, value_: Value) !?Value {
	153	if (stack.items.len == 0) return value_;
	154	var value = value_;
	155	while (true) {
	156	// Assert the stack grammar at the top of the stack.
	157	debug.assert(stack.items[stack.items.len - 1] == .array or
	158	(stack.items[stack.items.len - 2] == .object and stack.items[stack.items.len - 1] == .string));
	159	switch (stack.items[stack.items.len - 1]) {
310	160	.string => \|key\| {
311		_ = p.stack.pop();
312
313		var object = &p.stack.items[p.stack.items.len - 1].object;
314		try object.put(key, value.*);
315		p.state = .object_key;
	161	// stack: [..., .object, .string]
	162	_ = stack.pop();
	163
	164	// stack: [..., .object]
	165	var object = &stack.items[stack.items.len - 1].object;
	166	try object.put(key, value);
	167
	168	// This is an invalid state to leave the stack in,
	169	// so we have to process the next token before we return.
	170	switch (try source.nextAlloc(allocator, .alloc_if_needed)) {
	171	.object_end => {
	172	// This object is complete.
	173	value = stack.pop();
	174	// Effectively recurse now that we have a complete value.
	175	if (stack.items.len == 0) return value;
	176	continue;
	177	},
	178	inline .string, .allocated_string => \|next_key\| {
	179	// We've got another key.
	180	try stack.append(Value{ .string = next_key });
	181	// stack: [..., .object, .string]
	182	return null;
	183	},
	184	else => unreachable,
	185	}
316	186	},
317		// Array Parent -> [ ..., <array>, value ]
318	187	.array => \|*array\| {
319		try array.append(value.*);
320		p.state = .array_value;
321		},
322		else => {
323		unreachable;
	188	// stack: [..., .array]
	189	try array.append(value);
	190	return null;
324	191	},
	192	else => unreachable,
325	193	}
326	194	}
327
328		fn parseNumber(p: *Parser, slice: []const u8) !Value {
329		_ = p;
330		return if (isNumberFormattedLikeAnInteger(slice))
331		Value{
332		.integer = std.fmt.parseInt(i64, slice, 10) catch \|e\| switch (e) {
333		error.Overflow => return Value{ .number_string = slice },
334		error.InvalidCharacter => \|err\| return err,
335		},
336		}
337		else
338		Value{ .float = try std.fmt.parseFloat(f64, slice) };
339		}
340		};
	195	}
341	196
342	197	test {
343	198	_ = @import("dynamic_test.zig");

lib/std/json/dynamic_test.zig+28-71

...	...	@@ -5,12 +5,14 @@ const testing = std.testing;
5	5	const ObjectMap = @import("dynamic.zig").ObjectMap;
6	6	const Array = @import("dynamic.zig").Array;
7	7	const Value = @import("dynamic.zig").Value;
8		const Parser = @import("dynamic.zig").Parser;
9	8
10		test "json.parser.dynamic" {
11		var p = Parser.init(testing.allocator, .alloc_if_needed);
12		defer p.deinit();
	9	const parseFromSlice = @import("static.zig").parseFromSlice;
	10	const parseFromSliceLeaky = @import("static.zig").parseFromSliceLeaky;
	11	const parseFromTokenSource = @import("static.zig").parseFromTokenSource;
	12
	13	const jsonReader = @import("scanner.zig").reader;
13	14
	15	test "json.parser.dynamic" {
14	16	const s =
15	17	\\{
16	18	\\ "Image": {
...	...	@@ -31,10 +33,10 @@ test "json.parser.dynamic" {
31	33	\\}
32	34	;
33	35
34		var tree = try p.parse(s);
35		defer tree.deinit();
	36	var parsed = try parseFromSlice(Value, testing.allocator, s, .{});
	37	defer parsed.deinit();
36	38
37		var root = tree.root;
	39	var root = parsed.value;
38	40
39	41	var image = root.object.get("Image").?;
40	42
...	...	@@ -98,22 +100,22 @@ test "write json then parse it" {
98	100
99	101	try jw.endObject();
100	102
101		var parser = Parser.init(testing.allocator, .alloc_if_needed);
102		defer parser.deinit();
103		var tree = try parser.parse(fixed_buffer_stream.getWritten());
104		defer tree.deinit();
105
106		try testing.expect(tree.root.object.get("f").?.bool == false);
107		try testing.expect(tree.root.object.get("t").?.bool == true);
108		try testing.expect(tree.root.object.get("int").?.integer == 1234);
109		try testing.expect(tree.root.object.get("array").?.array.items[0].null == {});
110		try testing.expect(tree.root.object.get("array").?.array.items[1].float == 12.34);
111		try testing.expect(mem.eql(u8, tree.root.object.get("str").?.string, "hello"));
	103	fixed_buffer_stream = std.io.fixedBufferStream(fixed_buffer_stream.getWritten());
	104	var json_reader = jsonReader(testing.allocator, fixed_buffer_stream.reader());
	105	defer json_reader.deinit();
	106	var parsed = try parseFromTokenSource(Value, testing.allocator, &json_reader, .{});
	107	defer parsed.deinit();
	108
	109	try testing.expect(parsed.value.object.get("f").?.bool == false);
	110	try testing.expect(parsed.value.object.get("t").?.bool == true);
	111	try testing.expect(parsed.value.object.get("int").?.integer == 1234);
	112	try testing.expect(parsed.value.object.get("array").?.array.items[0].null == {});
	113	try testing.expect(parsed.value.object.get("array").?.array.items[1].float == 12.34);
	114	try testing.expect(mem.eql(u8, parsed.value.object.get("str").?.string, "hello"));
112	115	}
113	116
114		fn testParse(arena_allocator: std.mem.Allocator, json_str: []const u8) !Value {
115		var p = Parser.init(arena_allocator, .alloc_if_needed);
116		return (try p.parse(json_str)).root;
	117	fn testParse(allocator: std.mem.Allocator, json_str: []const u8) !Value {
	118	return parseFromSliceLeaky(Value, allocator, json_str, .{});
117	119	}
118	120
119	121	test "parsing empty string gives appropriate error" {
...	...	@@ -122,22 +124,16 @@ test "parsing empty string gives appropriate error" {
122	124	try testing.expectError(error.UnexpectedEndOfInput, testParse(arena_allocator.allocator(), ""));
123	125	}
124	126
125		test "parse tree should not contain dangling pointers" {
126		var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
127		defer arena_allocator.deinit();
128
129		var p = Parser.init(arena_allocator.allocator(), .alloc_if_needed);
130		defer p.deinit();
131
132		var tree = try p.parse("[]");
133		defer tree.deinit();
	127	test "Value.array allocator should still be usable after parsing" {
	128	var parsed = try parseFromSlice(Value, std.testing.allocator, "[]", .{});
	129	defer parsed.deinit();
134	130
135	131	// Allocation should succeed
136	132	var i: usize = 0;
137	133	while (i < 100) : (i += 1) {
138		try tree.root.array.append(Value{ .integer = 100 });
	134	try parsed.value.array.append(Value{ .integer = 100 });
139	135	}
140		try testing.expectEqual(tree.root.array.items.len, 100);
	136	try testing.expectEqual(parsed.value.array.items.len, 100);
141	137	}
142	138
143	139	test "integer after float has proper type" {
...	...	@@ -184,45 +180,6 @@ test "escaped characters" {
184	180	try testing.expectEqualSlices(u8, obj.get("surrogatepair").?.string, "😂");
185	181	}
186	182
187		test "string copy option" {
188		const input =
189		\\{
190		\\ "noescape": "aą😂",
191		\\ "simple": "\\\/\n\r\t\f\b\"",
192		\\ "unicode": "\u0105",
193		\\ "surrogatepair": "\ud83d\ude02"
194		\\}
195		;
196
197		var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
198		defer arena_allocator.deinit();
199		const allocator = arena_allocator.allocator();
200
201		var parser = Parser.init(allocator, .alloc_if_needed);
202		const tree_nocopy = try parser.parse(input);
203		const obj_nocopy = tree_nocopy.root.object;
204
205		parser = Parser.init(allocator, .alloc_always);
206		const tree_copy = try parser.parse(input);
207		const obj_copy = tree_copy.root.object;
208
209		for ([_][]const u8{ "noescape", "simple", "unicode", "surrogatepair" }) \|field_name\| {
210		try testing.expectEqualSlices(u8, obj_nocopy.get(field_name).?.string, obj_copy.get(field_name).?.string);
211		}
212
213		const nocopy_addr = &obj_nocopy.get("noescape").?.string[0];
214		const copy_addr = &obj_copy.get("noescape").?.string[0];
215
216		var found_nocopy = false;
217		for (input, 0..) \|_, index\| {
218		try testing.expect(copy_addr != &input[index]);
219		if (nocopy_addr == &input[index]) {
220		found_nocopy = true;
221		}
222		}
223		try testing.expect(found_nocopy);
224		}
225
226	183	test "Value.jsonStringify" {
227	184	{
228	185	var buffer: [10]u8 = undefined;

lib/std/json/static.zig+115-238

...	...	@@ -1,6 +1,7 @@
1	1	const std = @import("std");
2	2	const assert = std.debug.assert;
3	3	const Allocator = std.mem.Allocator;
	4	const ArenaAllocator = std.heap.ArenaAllocator;
4	5	const ArrayList = std.ArrayList;
5	6
6	7	const Scanner = @import("./scanner.zig").Scanner;
...	...	@@ -27,27 +28,79 @@ pub const ParseOptions = struct {
27	28	max_value_len: ?usize = null,
28	29	};
29	30
30		/// Parses the json document from s and returns the result.
31		/// The provided allocator is used both for temporary allocations during parsing the document,
32		/// and also to allocate any pointer values in the return type.
33		/// If T contains any pointers, free the memory with `std.json.parseFree`.
	31	pub fn Parsed(comptime T: type) type {
	32	return struct {
	33	arena: *ArenaAllocator,
	34	value: T,
	35
	36	pub fn deinit(self: @This()) void {
	37	const allocator = self.arena.child_allocator;
	38	self.arena.deinit();
	39	allocator.destroy(self.arena);
	40	}
	41	};
	42	}
	43
	44	/// Parses the json document from `s` and returns the result packaged in a `std.json.Parsed`.
	45	/// You must call `deinit()` of the returned object to clean up allocated resources.
34	46	/// Note that `error.BufferUnderrun` is not actually possible to return from this function.
35		pub fn parseFromSlice(comptime T: type, allocator: Allocator, s: []const u8, options: ParseOptions) ParseError(T, Scanner)!T {
	47	pub fn parseFromSlice(
	48	comptime T: type,
	49	allocator: Allocator,
	50	s: []const u8,
	51	options: ParseOptions,
	52	) ParseError(Scanner)!Parsed(T) {
36	53	var scanner = Scanner.initCompleteInput(allocator, s);
37	54	defer scanner.deinit();
38	55
39	56	return parseFromTokenSource(T, allocator, &scanner, options);
40	57	}
41	58
	59	/// Parses the json document from `s` and returns the result.
	60	/// Allocations made during this operation are not carefully tracked and may not be possible to individually clean up.
	61	/// It is recommended to use a `std.heap.ArenaAllocator` or similar.
	62	pub fn parseFromSliceLeaky(
	63	comptime T: type,
	64	allocator: Allocator,
	65	s: []const u8,
	66	options: ParseOptions,
	67	) ParseError(Scanner)!T {
	68	var scanner = Scanner.initCompleteInput(allocator, s);
	69	defer scanner.deinit();
	70
	71	return parseFromTokenSourceLeaky(T, allocator, &scanner, options);
	72	}
	73
42	74	/// `scanner_or_reader` must be either a `std.json.Scanner` with complete input or a `std.json.Reader`.
43		/// allocator is used to allocate the data of T if necessary,
44		/// such as if T is `*u32` or `[]u32`.
45		/// If T contains any pointers, free the memory with `std.json.parseFree`.
46		/// If T contains no pointers, the allocator may sometimes be used for temporary allocations,
47		/// but no call to `std.json.parseFree` will be necessary;
48		/// all temporary allocations will be freed before this function returns.
49	75	/// Note that `error.BufferUnderrun` is not actually possible to return from this function.
50		pub fn parseFromTokenSource(comptime T: type, allocator: Allocator, scanner_or_reader: anytype, options: ParseOptions) ParseError(T, @TypeOf(scanner_or_reader.*))!T {
	76	pub fn parseFromTokenSource(
	77	comptime T: type,
	78	allocator: Allocator,
	79	scanner_or_reader: anytype,
	80	options: ParseOptions,
	81	) ParseError(@TypeOf(scanner_or_reader.*))!Parsed(T) {
	82	var parsed = Parsed(T){
	83	.arena = try allocator.create(ArenaAllocator),
	84	.value = undefined,
	85	};
	86	errdefer allocator.destroy(parsed.arena);
	87	parsed.arena.* = ArenaAllocator.init(allocator);
	88	errdefer parsed.arena.deinit();
	89
	90	parsed.value = try parseFromTokenSourceLeaky(T, parsed.arena.allocator(), scanner_or_reader, options);
	91
	92	return parsed;
	93	}
	94
	95	/// `scanner_or_reader` must be either a `std.json.Scanner` with complete input or a `std.json.Reader`.
	96	/// Allocations made during this operation are not carefully tracked and may not be possible to individually clean up.
	97	/// It is recommended to use a `std.heap.ArenaAllocator` or similar.
	98	pub fn parseFromTokenSourceLeaky(
	99	comptime T: type,
	100	allocator: Allocator,
	101	scanner_or_reader: anytype,
	102	options: ParseOptions,
	103	) ParseError(@TypeOf(scanner_or_reader.*))!T {
51	104	if (@TypeOf(scanner_or_reader.*) == Scanner) {
52	105	assert(scanner_or_reader.is_end_of_input);
53	106	}
...	...	@@ -61,80 +114,30 @@ pub fn parseFromTokenSource(comptime T: type, allocator: Allocator, scanner_or_r
61	114	}
62	115	}
63	116
64		const r = try parseInternal(T, allocator, scanner_or_reader, resolved_options);
65		errdefer parseFree(T, allocator, r);
	117	const value = try parseInternal(T, allocator, scanner_or_reader, resolved_options);
66	118
67	119	assert(.end_of_document == try scanner_or_reader.next());
68	120
69		return r;
	121	return value;
70	122	}
71	123
72		/// The error set that will be returned from parsing T from *Source.
73		/// Note that this may contain error.BufferUnderrun, but that error will never actually be returned.
74		pub fn ParseError(comptime T: type, comptime Source: type) type {
75		// `inferred_types` is used to avoid infinite recursion for recursive type definitions.
76		const inferred_types = [_]type{};
	124	/// The error set that will be returned when parsing from `*Source`.
	125	/// Note that this may contain `error.BufferUnderrun`, but that error will never actually be returned.
	126	pub fn ParseError(comptime Source: type) type {
77	127	// A few of these will either always be present or present enough of the time that
78	128	// omitting them is more confusing than always including them.
79		return error{UnexpectedToken} \|\| Source.NextError \|\| Source.PeekError \|\|
80		ParseInternalErrorImpl(T, Source, &inferred_types);
81		}
82
83		fn ParseInternalErrorImpl(comptime T: type, comptime Source: type, comptime inferred_types: []const type) type {
84		for (inferred_types) \|ty\| {
85		if (T == ty) return error{};
86		}
87
88		switch (@typeInfo(T)) {
89		.Bool => return error{},
90		.Float, .ComptimeFloat => return Source.AllocError \|\| std.fmt.ParseFloatError,
91		.Int, .ComptimeInt => {
92		return Source.AllocError \|\| error{ InvalidNumber, Overflow } \|\|
93		std.fmt.ParseIntError \|\| std.fmt.ParseFloatError;
94		},
95		.Optional => \|optional_info\| return ParseInternalErrorImpl(optional_info.child, Source, inferred_types ++ [_]type{T}),
96		.Enum => return Source.AllocError \|\| error{InvalidEnumTag},
97		.Union => \|unionInfo\| {
98		if (unionInfo.tag_type) \|_\| {
99		var errors = Source.AllocError \|\| error{UnknownField};
100		for (unionInfo.fields) \|u_field\| {
101		errors = errors \|\| ParseInternalErrorImpl(u_field.type, Source, inferred_types ++ [_]type{T});
102		}
103		return errors;
104		} else {
105		@compileError("Unable to parse into untagged union '" ++ @typeName(T) ++ "'");
106		}
107		},
108		.Struct => \|structInfo\| {
109		var errors = Scanner.AllocError \|\| error{
110		DuplicateField,
111		UnknownField,
112		MissingField,
113		};
114		for (structInfo.fields) \|field\| {
115		errors = errors \|\| ParseInternalErrorImpl(field.type, Source, inferred_types ++ [_]type{T});
116		}
117		return errors;
118		},
119		.Array => \|arrayInfo\| {
120		return error{LengthMismatch} \|\|
121		ParseInternalErrorImpl(arrayInfo.child, Source, inferred_types ++ [_]type{T});
122		},
123		.Vector => \|vecInfo\| {
124		return error{LengthMismatch} \|\|
125		ParseInternalErrorImpl(vecInfo.child, Source, inferred_types ++ [_]type{T});
126		},
127		.Pointer => \|ptrInfo\| {
128		switch (ptrInfo.size) {
129		.One, .Slice => {
130		return ParseInternalErrorImpl(ptrInfo.child, Source, inferred_types ++ [_]type{T});
131		},
132		else => @compileError("Unable to parse into type '" ++ @typeName(T) ++ "'"),
133		}
134		},
135		else => return error{},
136		}
137		unreachable;
	129	return error{
	130	UnexpectedToken,
	131	InvalidNumber,
	132	Overflow,
	133	InvalidEnumTag,
	134	DuplicateField,
	135	UnknownField,
	136	MissingField,
	137	LengthMismatch,
	138	} \|\|
	139	std.fmt.ParseIntError \|\| std.fmt.ParseFloatError \|\|
	140	Source.NextError \|\| Source.PeekError \|\| Source.AllocError;
138	141	}
139	142
140	143	fn parseInternal(
...	...	@@ -142,7 +145,7 @@ fn parseInternal(
142	145	allocator: Allocator,
143	146	source: anytype,
144	147	options: ParseOptions,
145		) ParseError(T, @TypeOf(source.*))!T {
	148	) ParseError(@TypeOf(source.*))!T {
146	149	switch (@typeInfo(T)) {
147	150	.Bool => {
148	151	return switch (try source.next()) {
...	...	@@ -155,8 +158,7 @@ fn parseInternal(
155	158	const token = try source.nextAllocMax(allocator, .alloc_if_needed, options.max_value_len.?);
156	159	defer freeAllocated(allocator, token);
157	160	const slice = switch (token) {
158		.number, .string => \|slice\| slice,
159		.allocated_number, .allocated_string => \|slice\| slice,
	161	inline .number, .allocated_number, .string, .allocated_string => \|slice\| slice,
160	162	else => return error.UnexpectedToken,
161	163	};
162	164	return try std.fmt.parseFloat(T, slice);
...	...	@@ -165,8 +167,7 @@ fn parseInternal(
165	167	const token = try source.nextAllocMax(allocator, .alloc_if_needed, options.max_value_len.?);
166	168	defer freeAllocated(allocator, token);
167	169	const slice = switch (token) {
168		.number, .string => \|slice\| slice,
169		.allocated_number, .allocated_string => \|slice\| slice,
	170	inline .number, .allocated_number, .string, .allocated_string => \|slice\| slice,
170	171	else => return error.UnexpectedToken,
171	172	};
172	173	if (isNumberFormattedLikeAnInteger(slice))
...	...	@@ -189,11 +190,14 @@ fn parseInternal(
189	190	}
190	191	},
191	192	.Enum => \|enumInfo\| {
	193	if (comptime std.meta.trait.hasFn("jsonParse")(T)) {
	194	return T.jsonParse(allocator, source, options);
	195	}
	196
192	197	const token = try source.nextAllocMax(allocator, .alloc_if_needed, options.max_value_len.?);
193	198	defer freeAllocated(allocator, token);
194	199	const slice = switch (token) {
195		.number, .string => \|slice\| slice,
196		.allocated_number, .allocated_string => \|slice\| slice,
	200	inline .number, .allocated_number, .string, .allocated_string => \|slice\| slice,
197	201	else => return error.UnexpectedToken,
198	202	};
199	203	// Check for a named value.
...	...	@@ -204,30 +208,18 @@ fn parseInternal(
204	208	return try std.meta.intToEnum(T, n);
205	209	},
206	210	.Union => \|unionInfo\| {
207		const UnionTagType = unionInfo.tag_type orelse @compileError("Unable to parse into untagged union '" ++ @typeName(T) ++ "'");
	211	if (comptime std.meta.trait.hasFn("jsonParse")(T)) {
	212	return T.jsonParse(allocator, source, options);
	213	}
	214
	215	if (unionInfo.tag_type == null) @compileError("Unable to parse into untagged union '" ++ @typeName(T) ++ "'");
208	216
209	217	if (.object_begin != try source.next()) return error.UnexpectedToken;
210	218
211	219	var result: ?T = null;
212		errdefer {
213		if (result) \|r\| {
214		inline for (unionInfo.fields) \|u_field\| {
215		if (r == @field(UnionTagType, u_field.name)) {
216		parseFree(u_field.type, allocator, @field(r, u_field.name));
217		}
218		}
219		}
220		}
221
222	220	var name_token: ?Token = try source.nextAllocMax(allocator, .alloc_if_needed, options.max_value_len.?);
223		errdefer {
224		if (name_token) \|t\| {
225		freeAllocated(allocator, t);
226		}
227		}
228	221	const field_name = switch (name_token.?) {
229		.string => \|slice\| slice,
230		.allocated_string => \|slice\| slice,
	222	inline .string, .allocated_string => \|slice\| slice,
231	223	else => return error.UnexpectedToken,
232	224	};
233	225
...	...	@@ -265,13 +257,6 @@ fn parseInternal(
265	257
266	258	var r: T = undefined;
267	259	var fields_seen: usize = 0;
268		errdefer {
269		inline for (0..structInfo.fields.len) \|i\| {
270		if (i < fields_seen) {
271		parseFree(structInfo.fields[i].type, allocator, r[i]);
272		}
273		}
274		}
275	260	inline for (0..structInfo.fields.len) \|i\| {
276	261	r[i] = try parseInternal(structInfo.fields[i].type, allocator, source, options);
277	262	fields_seen = i + 1;
...	...	@@ -282,29 +267,20 @@ fn parseInternal(
282	267	return r;
283	268	}
284	269
	270	if (comptime std.meta.trait.hasFn("jsonParse")(T)) {
	271	return T.jsonParse(allocator, source, options);
	272	}
	273
285	274	if (.object_begin != try source.next()) return error.UnexpectedToken;
286	275
287	276	var r: T = undefined;
288	277	var fields_seen = [_]bool{false} ** structInfo.fields.len;
289		errdefer {
290		inline for (structInfo.fields, 0..) \|field, i\| {
291		if (fields_seen[i]) {
292		parseFree(field.type, allocator, @field(r, field.name));
293		}
294		}
295		}
296	278
297	279	while (true) {
298	280	var name_token: ?Token = try source.nextAllocMax(allocator, .alloc_if_needed, options.max_value_len.?);
299		errdefer {
300		if (name_token) \|t\| {
301		freeAllocated(allocator, t);
302		}
303		}
304	281	const field_name = switch (name_token.?) {
305	282	.object_end => break, // No more fields.
306		.string => \|slice\| slice,
307		.allocated_string => \|slice\| slice,
	283	inline .string, .allocated_string => \|slice\| slice,
308	284	else => return error.UnexpectedToken,
309	285	};
310	286
...	...	@@ -319,18 +295,13 @@ fn parseInternal(
319	295	if (fields_seen[i]) {
320	296	switch (options.duplicate_field_behavior) {
321	297	.use_first => {
322		// Parse and then delete the redundant value.
	298	// Parse and ignore the redundant value.
323	299	// We don't want to skip the value, because we want type checking.
324		const ignored_value = try parseInternal(field.type, allocator, source, options);
325		parseFree(field.type, allocator, ignored_value);
	300	_ = try parseInternal(field.type, allocator, source, options);
326	301	break;
327	302	},
328	303	.@"error" => return error.DuplicateField,
329		.use_last => {
330		// Delete the stale value. We're about to get a new one.
331		parseFree(field.type, allocator, @field(r, field.name));
332		fields_seen[i] = false;
333		},
	304	.use_last => {},
334	305	}
335	306	}
336	307	@field(r, field.name) = try parseInternal(field.type, allocator, source, options);
...	...	@@ -428,7 +399,6 @@ fn parseInternal(
428	399	switch (ptrInfo.size) {
429	400	.One => {
430	401	const r: *ptrInfo.child = try allocator.create(ptrInfo.child);
431		errdefer allocator.destroy(r);
432	402	r.* = try parseInternal(ptrInfo.child, allocator, source, options);
433	403	return r;
434	404	},
...	...	@@ -439,13 +409,6 @@ fn parseInternal(
439	409
440	410	// Typical array.
441	411	var arraylist = ArrayList(ptrInfo.child).init(allocator);
442		errdefer {
443		while (arraylist.popOrNull()) \|v\| {
444		parseFree(ptrInfo.child, allocator, v);
445		}
446		arraylist.deinit();
447		}
448
449	412	while (true) {
450	413	switch (try source.peekNextTokenType()) {
451	414	.array_end => {
...	...	@@ -473,13 +436,20 @@ fn parseInternal(
473	436	if (ptrInfo.sentinel) \|sentinel_ptr\| {
474	437	// Use our own array list so we can append the sentinel.
475	438	var value_list = ArrayList(u8).init(allocator);
476		errdefer value_list.deinit();
477	439	_ = try source.allocNextIntoArrayList(&value_list, .alloc_always);
478	440	return try value_list.toOwnedSliceSentinel(@ptrCast(const u8, sentinel_ptr).);
479	441	}
480		switch (try source.nextAllocMax(allocator, .alloc_always, options.max_value_len.?)) {
481		.allocated_string => \|slice\| return slice,
482		else => unreachable,
	442	if (ptrInfo.is_const) {
	443	switch (try source.nextAllocMax(allocator, .alloc_if_needed, options.max_value_len.?)) {
	444	inline .string, .allocated_string => \|slice\| return slice,
	445	else => unreachable,
	446	}
	447	} else {
	448	// Have to allocate to get a mutable copy.
	449	switch (try source.nextAllocMax(allocator, .alloc_always, options.max_value_len.?)) {
	450	.allocated_string => \|slice\| return slice,
	451	else => unreachable,
	452	}
483	453	}
484	454	},
485	455	else => return error.UnexpectedToken,
...	...	@@ -505,13 +475,6 @@ fn parseInternalArray(
505	475
506	476	var r: T = undefined;
507	477	var i: usize = 0;
508		errdefer {
509		// Without the len check `r[i]` is not allowed
510		if (len > 0) while (true) : (i -= 1) {
511		parseFree(Child, allocator, r[i]);
512		if (i == 0) break;
513		};
514		}
515	478	while (i < len) : (i += 1) {
516	479	r[i] = try parseInternal(Child, allocator, source, options);
517	480	}
...	...	@@ -530,92 +493,6 @@ fn freeAllocated(allocator: Allocator, token: Token) void {
530	493	}
531	494	}
532	495
533		/// Releases resources created by parseFromSlice() or parseFromTokenSource().
534		pub fn parseFree(comptime T: type, allocator: Allocator, value: T) void {
535		switch (@typeInfo(T)) {
536		.Bool, .Float, .ComptimeFloat, .Int, .ComptimeInt, .Enum => {},
537		.Optional => {
538		if (value) \|v\| {
539		return parseFree(@TypeOf(v), allocator, v);
540		}
541		},
542		.Union => \|unionInfo\| {
543		if (unionInfo.tag_type) \|UnionTagType\| {
544		inline for (unionInfo.fields) \|u_field\| {
545		if (value == @field(UnionTagType, u_field.name)) {
546		parseFree(u_field.type, allocator, @field(value, u_field.name));
547		break;
548		}
549		}
550		} else {
551		unreachable;
552		}
553		},
554		.Struct => \|structInfo\| {
555		inline for (structInfo.fields) \|field\| {
556		var should_free = true;
557		if (field.default_value) \|default\| {
558		switch (@typeInfo(field.type)) {
559		// We must not attempt to free pointers to struct default values
560		.Pointer => \|fieldPtrInfo\| {
561		const field_value = @field(value, field.name);
562		const field_ptr = switch (fieldPtrInfo.size) {
563		.One => field_value,
564		.Slice => field_value.ptr,
565		else => unreachable, // Other pointer types are not parseable
566		};
567		const field_addr = @ptrToInt(field_ptr);
568
569		const casted_default = @ptrCast(const field.type, @alignCast(@alignOf(field.type), default)).;
570		const default_ptr = switch (fieldPtrInfo.size) {
571		.One => casted_default,
572		.Slice => casted_default.ptr,
573		else => unreachable, // Other pointer types are not parseable
574		};
575		const default_addr = @ptrToInt(default_ptr);
576
577		if (field_addr == default_addr) {
578		should_free = false;
579		}
580		},
581		else => {},
582		}
583		}
584		if (should_free) {
585		parseFree(field.type, allocator, @field(value, field.name));
586		}
587		}
588		},
589		.Array => \|arrayInfo\| {
590		for (value) \|v\| {
591		parseFree(arrayInfo.child, allocator, v);
592		}
593		},
594		.Vector => \|vecInfo\| {
595		var i: usize = 0;
596		while (i < vecInfo.len) : (i += 1) {
597		parseFree(vecInfo.child, allocator, value[i]);
598		}
599		},
600		.Pointer => \|ptrInfo\| {
601		switch (ptrInfo.size) {
602		.One => {
603		parseFree(ptrInfo.child, allocator, value.*);
604		allocator.destroy(value);
605		},
606		.Slice => {
607		for (value) \|v\| {
608		parseFree(ptrInfo.child, allocator, v);
609		}
610		allocator.free(value);
611		},
612		else => unreachable,
613		}
614		},
615		else => unreachable,
616		}
617		}
618
619	496	test {
620	497	_ = @import("./static_test.zig");
621	498	}

lib/std/json/static_test.zig+112-105

...	...	@@ -1,29 +1,31 @@
1	1	const std = @import("std");
2	2	const testing = std.testing;
	3	const ArenaAllocator = std.heap.ArenaAllocator;
3	4
4	5	const parseFromSlice = @import("./static.zig").parseFromSlice;
	6	const parseFromSliceLeaky = @import("./static.zig").parseFromSliceLeaky;
5	7	const parseFromTokenSource = @import("./static.zig").parseFromTokenSource;
6		const parseFree = @import("./static.zig").parseFree;
	8	const parseFromTokenSourceLeaky = @import("./static.zig").parseFromTokenSourceLeaky;
7	9	const ParseOptions = @import("./static.zig").ParseOptions;
8	10	const JsonScanner = @import("./scanner.zig").Scanner;
9	11	const jsonReader = @import("./scanner.zig").reader;
10	12
11	13	test "parse" {
12		try testing.expectEqual(false, try parseFromSlice(bool, testing.allocator, "false", .{}));
13		try testing.expectEqual(true, try parseFromSlice(bool, testing.allocator, "true", .{}));
14		try testing.expectEqual(@as(u1, 1), try parseFromSlice(u1, testing.allocator, "1", .{}));
15		try testing.expectError(error.Overflow, parseFromSlice(u1, testing.allocator, "50", .{}));
16		try testing.expectEqual(@as(u64, 42), try parseFromSlice(u64, testing.allocator, "42", .{}));
17		try testing.expectEqual(@as(f64, 42), try parseFromSlice(f64, testing.allocator, "42.0", .{}));
18		try testing.expectEqual(@as(?bool, null), try parseFromSlice(?bool, testing.allocator, "null", .{}));
19		try testing.expectEqual(@as(?bool, true), try parseFromSlice(?bool, testing.allocator, "true", .{}));
20
21		try testing.expectEqual(@as([3]u8, "foo".*), try parseFromSlice([3]u8, testing.allocator, "\"foo\"", .{}));
22		try testing.expectEqual(@as([3]u8, "foo".*), try parseFromSlice([3]u8, testing.allocator, "[102, 111, 111]", .{}));
23		try testing.expectEqual(@as([0]u8, undefined), try parseFromSlice([0]u8, testing.allocator, "[]", .{}));
24
25		try testing.expectEqual(@as(u64, 12345678901234567890), try parseFromSlice(u64, testing.allocator, "\"12345678901234567890\"", .{}));
26		try testing.expectEqual(@as(f64, 123.456), try parseFromSlice(f64, testing.allocator, "\"123.456\"", .{}));
	14	try testing.expectEqual(false, try parseFromSliceLeaky(bool, testing.allocator, "false", .{}));
	15	try testing.expectEqual(true, try parseFromSliceLeaky(bool, testing.allocator, "true", .{}));
	16	try testing.expectEqual(@as(u1, 1), try parseFromSliceLeaky(u1, testing.allocator, "1", .{}));
	17	try testing.expectError(error.Overflow, parseFromSliceLeaky(u1, testing.allocator, "50", .{}));
	18	try testing.expectEqual(@as(u64, 42), try parseFromSliceLeaky(u64, testing.allocator, "42", .{}));
	19	try testing.expectEqual(@as(f64, 42), try parseFromSliceLeaky(f64, testing.allocator, "42.0", .{}));
	20	try testing.expectEqual(@as(?bool, null), try parseFromSliceLeaky(?bool, testing.allocator, "null", .{}));
	21	try testing.expectEqual(@as(?bool, true), try parseFromSliceLeaky(?bool, testing.allocator, "true", .{}));
	22
	23	try testing.expectEqual(@as([3]u8, "foo".*), try parseFromSliceLeaky([3]u8, testing.allocator, "\"foo\"", .{}));
	24	try testing.expectEqual(@as([3]u8, "foo".*), try parseFromSliceLeaky([3]u8, testing.allocator, "[102, 111, 111]", .{}));
	25	try testing.expectEqual(@as([0]u8, undefined), try parseFromSliceLeaky([0]u8, testing.allocator, "[]", .{}));
	26
	27	try testing.expectEqual(@as(u64, 12345678901234567890), try parseFromSliceLeaky(u64, testing.allocator, "\"12345678901234567890\"", .{}));
	28	try testing.expectEqual(@as(f64, 123.456), try parseFromSliceLeaky(f64, testing.allocator, "\"123.456\"", .{}));
27	29	}
28	30
29	31	test "parse into enum" {
...	...	@@ -32,37 +34,37 @@ test "parse into enum" {
32	34	Bar,
33	35	@"with\\escape",
34	36	};
35		try testing.expectEqual(@as(T, .Foo), try parseFromSlice(T, testing.allocator, "\"Foo\"", .{}));
36		try testing.expectEqual(@as(T, .Foo), try parseFromSlice(T, testing.allocator, "42", .{}));
37		try testing.expectEqual(@as(T, .@"with\\escape"), try parseFromSlice(T, testing.allocator, "\"with\\\\escape\"", .{}));
38		try testing.expectError(error.InvalidEnumTag, parseFromSlice(T, testing.allocator, "5", .{}));
39		try testing.expectError(error.InvalidEnumTag, parseFromSlice(T, testing.allocator, "\"Qux\"", .{}));
	37	try testing.expectEqual(@as(T, .Foo), try parseFromSliceLeaky(T, testing.allocator, "\"Foo\"", .{}));
	38	try testing.expectEqual(@as(T, .Foo), try parseFromSliceLeaky(T, testing.allocator, "42", .{}));
	39	try testing.expectEqual(@as(T, .@"with\\escape"), try parseFromSliceLeaky(T, testing.allocator, "\"with\\\\escape\"", .{}));
	40	try testing.expectError(error.InvalidEnumTag, parseFromSliceLeaky(T, testing.allocator, "5", .{}));
	41	try testing.expectError(error.InvalidEnumTag, parseFromSliceLeaky(T, testing.allocator, "\"Qux\"", .{}));
40	42	}
41	43
42	44	test "parse into that allocates a slice" {
43	45	{
44	46	// string as string
45		const r = try parseFromSlice([]u8, testing.allocator, "\"foo\"", .{});
46		defer parseFree([]u8, testing.allocator, r);
47		try testing.expectEqualSlices(u8, "foo", r);
	47	const parsed = try parseFromSlice([]u8, testing.allocator, "\"foo\"", .{});
	48	defer parsed.deinit();
	49	try testing.expectEqualSlices(u8, "foo", parsed.value);
48	50	}
49	51	{
50	52	// string as array of u8 integers
51		const r = try parseFromSlice([]u8, testing.allocator, "[102, 111, 111]", .{});
52		defer parseFree([]u8, testing.allocator, r);
53		try testing.expectEqualSlices(u8, "foo", r);
	53	const parsed = try parseFromSlice([]u8, testing.allocator, "[102, 111, 111]", .{});
	54	defer parsed.deinit();
	55	try testing.expectEqualSlices(u8, "foo", parsed.value);
54	56	}
55	57	{
56		const r = try parseFromSlice([]u8, testing.allocator, "\"with\\\\escape\"", .{});
57		defer parseFree([]u8, testing.allocator, r);
58		try testing.expectEqualSlices(u8, "with\\escape", r);
	58	const parsed = try parseFromSlice([]u8, testing.allocator, "\"with\\\\escape\"", .{});
	59	defer parsed.deinit();
	60	try testing.expectEqualSlices(u8, "with\\escape", parsed.value);
59	61	}
60	62	}
61	63
62	64	test "parse into sentinel slice" {
63		const result = try parseFromSlice([:0]const u8, testing.allocator, "\"\\n\"", .{});
64		defer parseFree([:0]const u8, testing.allocator, result);
65		try testing.expect(std.mem.eql(u8, result, "\n"));
	65	const parsed = try parseFromSlice([:0]const u8, testing.allocator, "\"\\n\"", .{});
	66	defer parsed.deinit();
	67	try testing.expect(std.mem.eql(u8, parsed.value, "\n"));
66	68	}
67	69
68	70	test "parse into tagged union" {
...	...	@@ -72,9 +74,12 @@ test "parse into tagged union" {
72	74	float: f64,
73	75	string: []const u8,
74	76	};
75		try testing.expectEqual(T{ .float = 1.5 }, try parseFromSlice(T, testing.allocator, "{\"float\":1.5}", .{}));
76		try testing.expectEqual(T{ .int = 1 }, try parseFromSlice(T, testing.allocator, "{\"int\":1}", .{}));
77		try testing.expectEqual(T{ .nothing = {} }, try parseFromSlice(T, testing.allocator, "{\"nothing\":{}}", .{}));
	77	try testing.expectEqual(T{ .float = 1.5 }, try parseFromSliceLeaky(T, testing.allocator, "{\"float\":1.5}", .{}));
	78	try testing.expectEqual(T{ .int = 1 }, try parseFromSliceLeaky(T, testing.allocator, "{\"int\":1}", .{}));
	79	try testing.expectEqual(T{ .nothing = {} }, try parseFromSliceLeaky(T, testing.allocator, "{\"nothing\":{}}", .{}));
	80	const parsed = try parseFromSlice(T, testing.allocator, "{\"string\":\"foo\"}", .{});
	81	defer parsed.deinit();
	82	try testing.expectEqualSlices(u8, "foo", parsed.value.string);
78	83	}
79	84
80	85	test "parse into tagged union errors" {
...	...	@@ -84,42 +89,36 @@ test "parse into tagged union errors" {
84	89	float: f64,
85	90	string: []const u8,
86	91	};
87		try testing.expectError(error.UnexpectedToken, parseFromSlice(T, testing.allocator, "42", .{}));
88		try testing.expectError(error.UnexpectedToken, parseFromSlice(T, testing.allocator, "{}", .{}));
89		try testing.expectError(error.UnknownField, parseFromSlice(T, testing.allocator, "{\"bogus\":1}", .{}));
90		try testing.expectError(error.UnexpectedToken, parseFromSlice(T, testing.allocator, "{\"int\":1, \"int\":1", .{}));
91		try testing.expectError(error.UnexpectedToken, parseFromSlice(T, testing.allocator, "{\"int\":1, \"float\":1.0}", .{}));
92		try testing.expectError(error.UnexpectedToken, parseFromSlice(T, testing.allocator, "{\"nothing\":null}", .{}));
93		try testing.expectError(error.UnexpectedToken, parseFromSlice(T, testing.allocator, "{\"nothing\":{\"no\":0}}", .{}));
	92	var arena = ArenaAllocator.init(testing.allocator);
	93	defer arena.deinit();
	94	try testing.expectError(error.UnexpectedToken, parseFromSliceLeaky(T, arena.allocator(), "42", .{}));
	95	try testing.expectError(error.SyntaxError, parseFromSliceLeaky(T, arena.allocator(), "{\"int\":1} 42", .{}));
	96	try testing.expectError(error.UnexpectedToken, parseFromSliceLeaky(T, arena.allocator(), "{}", .{}));
	97	try testing.expectError(error.UnknownField, parseFromSliceLeaky(T, arena.allocator(), "{\"bogus\":1}", .{}));
	98	try testing.expectError(error.UnexpectedToken, parseFromSliceLeaky(T, arena.allocator(), "{\"int\":1, \"int\":1", .{}));
	99	try testing.expectError(error.UnexpectedToken, parseFromSliceLeaky(T, arena.allocator(), "{\"int\":1, \"float\":1.0}", .{}));
	100	try testing.expectError(error.UnexpectedToken, parseFromSliceLeaky(T, arena.allocator(), "{\"nothing\":null}", .{}));
	101	try testing.expectError(error.UnexpectedToken, parseFromSliceLeaky(T, arena.allocator(), "{\"nothing\":{\"no\":0}}", .{}));
94	102
95	103	// Allocator failure
96	104	var fail_alloc = testing.FailingAllocator.init(testing.allocator, 0);
97		const failing_allocator = fail_alloc.allocator();
98		try testing.expectError(error.OutOfMemory, parseFromSlice(T, failing_allocator, "{\"string\"\"foo\"}", .{}));
99		}
100
101		test "parseFree descends into tagged union" {
102		const T = union(enum) {
103		nothing,
104		int: i32,
105		float: f64,
106		string: []const u8,
107		};
108		const r = try parseFromSlice(T, testing.allocator, "{\"string\":\"foo\"}", .{});
109		try testing.expectEqualSlices(u8, "foo", r.string);
110		parseFree(T, testing.allocator, r);
	105	try testing.expectError(error.OutOfMemory, parseFromSlice(T, fail_alloc.allocator(), "{\"string\"\"foo\"}", .{}));
111	106	}
112	107
113	108	test "parse into struct with no fields" {
114	109	const T = struct {};
115		try testing.expectEqual(T{}, try parseFromSlice(T, testing.allocator, "{}", .{}));
	110	const parsed = try parseFromSlice(T, testing.allocator, "{}", .{});
	111	defer parsed.deinit();
	112	try testing.expectEqual(T{}, parsed.value);
116	113	}
117	114
118	115	const test_const_value: usize = 123;
119	116
120	117	test "parse into struct with default const pointer field" {
121	118	const T = struct { a: *const usize = &test_const_value };
122		try testing.expectEqual(T{}, try parseFromSlice(T, testing.allocator, "{}", .{}));
	119	const parsed = try parseFromSlice(T, testing.allocator, "{}", .{});
	120	defer parsed.deinit();
	121	try testing.expectEqual(T{}, parsed.value);
123	122	}
124	123
125	124	const test_default_usize: usize = 123;
...	...	@@ -138,10 +137,9 @@ test "freeing parsed structs with pointers to default values" {
138	137	str_slice: []const []const u8 = &test_default_str_slice,
139	138	};
140	139
141		const parsed = try parseFromSlice(T, testing.allocator, "{}", .{});
142		try testing.expectEqual(T{}, parsed);
143		// This will panic if it tries to free global constants:
144		parseFree(T, testing.allocator, parsed);
	140	var parsed = try parseFromSlice(T, testing.allocator, "{}", .{});
	141	try testing.expectEqual(T{}, parsed.value);
	142	defer parsed.deinit();
145	143	}
146	144
147	145	test "parse into struct where destination and source lengths mismatch" {
...	...	@@ -201,8 +199,9 @@ test "parse into struct with misc fields" {
201	199	\\ }
202	200	\\}
203	201	;
204		const r = try parseFromSlice(T, testing.allocator, document_str, .{});
205		defer parseFree(T, testing.allocator, r);
	202	const parsed = try parseFromSlice(T, testing.allocator, document_str, .{});
	203	defer parsed.deinit();
	204	const r = &parsed.value;
206	205	try testing.expectEqual(@as(i64, 420), r.int);
207	206	try testing.expectEqual(@as(f64, 3.14), r.float);
208	207	try testing.expectEqual(true, r.@"with\\escape");
...	...	@@ -238,24 +237,20 @@ test "parse into struct with strings and arrays with sentinels" {
238	237	\\ "simple_data": [4, 5, 6]
239	238	\\}
240	239	;
241		const r = try parseFromSlice(T, testing.allocator, document_str, .{});
242		defer parseFree(T, testing.allocator, r);
	240	const parsed = try parseFromSlice(T, testing.allocator, document_str, .{});
	241	defer parsed.deinit();
243	242
244		try testing.expectEqualSentinel(u8, 0, "zig", r.language);
	243	try testing.expectEqualSentinel(u8, 0, "zig", parsed.value.language);
245	244
246	245	const data = [_:99]i32{ 1, 2, 3 };
247		try testing.expectEqualSentinel(i32, 99, data[0..data.len], r.data);
	246	try testing.expectEqualSentinel(i32, 99, data[0..data.len], parsed.value.data);
248	247
249	248	// Make sure that arrays who aren't supposed to have a sentinel still parse without one.
250		try testing.expectEqual(@as(?i32, null), std.meta.sentinel(@TypeOf(r.simple_data)));
251		try testing.expectEqual(@as(?u8, null), std.meta.sentinel(@TypeOf(r.language_without_sentinel)));
	249	try testing.expectEqual(@as(?i32, null), std.meta.sentinel(@TypeOf(parsed.value.simple_data)));
	250	try testing.expectEqual(@as(?u8, null), std.meta.sentinel(@TypeOf(parsed.value.language_without_sentinel)));
252	251	}
253	252
254	253	test "parse into struct with duplicate field" {
255		// allow allocator to detect double frees by keeping bucket in use
256		const ballast = try testing.allocator.alloc(u64, 1);
257		defer testing.allocator.free(ballast);
258
259	254	const options_first = ParseOptions{ .duplicate_field_behavior = .use_first };
260	255	const options_last = ParseOptions{ .duplicate_field_behavior = .use_last };
261	256
...	...	@@ -266,9 +261,12 @@ test "parse into struct with duplicate field" {
266	261	try testing.expectError(error.InvalidNumber, parseFromSlice(T1, testing.allocator, str, options_first));
267	262	try testing.expectError(error.InvalidNumber, parseFromSlice(T1, testing.allocator, str, options_last));
268	263
	264	var arena = ArenaAllocator.init(testing.allocator);
	265	defer arena.deinit();
	266
269	267	const T2 = struct { a: f64 };
270		try testing.expectEqual(T2{ .a = 1.0 }, try parseFromSlice(T2, testing.allocator, str, options_first));
271		try testing.expectEqual(T2{ .a = 0.25 }, try parseFromSlice(T2, testing.allocator, str, options_last));
	268	try testing.expectEqual(T2{ .a = 1.0 }, try parseFromSliceLeaky(T2, arena.allocator(), str, options_first));
	269	try testing.expectEqual(T2{ .a = 0.25 }, try parseFromSliceLeaky(T2, arena.allocator(), str, options_last));
272	270	}
273	271
274	272	test "parse into struct ignoring unknown fields" {
...	...	@@ -302,11 +300,11 @@ test "parse into struct ignoring unknown fields" {
302	300	\\ "language": "zig"
303	301	\\}
304	302	;
305		const r = try parseFromSlice(T, testing.allocator, str, .{ .ignore_unknown_fields = true });
306		defer parseFree(T, testing.allocator, r);
	303	const parsed = try parseFromSlice(T, testing.allocator, str, .{ .ignore_unknown_fields = true });
	304	defer parsed.deinit();
307	305
308		try testing.expectEqual(@as(i64, 420), r.int);
309		try testing.expectEqualSlices(u8, "zig", r.language);
	306	try testing.expectEqual(@as(i64, 420), parsed.value.int);
	307	try testing.expectEqualSlices(u8, "zig", parsed.value.language);
310	308	}
311	309
312	310	test "parse into tuple" {
...	...	@@ -343,8 +341,9 @@ test "parse into tuple" {
343	341	\\ {"float": 12.34}
344	342	\\]
345	343	;
346		const r = try parseFromSlice(T, testing.allocator, str, .{});
347		defer parseFree(T, testing.allocator, r);
	344	const parsed = try parseFromSlice(T, testing.allocator, str, .{});
	345	defer parsed.deinit();
	346	const r = parsed.value;
348	347	try testing.expectEqual(@as(i64, 420), r[0]);
349	348	try testing.expectEqual(@as(f64, 3.14), r[1]);
350	349	try testing.expectEqual(true, r[2]);
...	...	@@ -368,10 +367,10 @@ test "parse into recursive union definition" {
368	367	values: ParseIntoRecursiveUnionDefinitionValue,
369	368	};
370	369
371		const r = try parseFromSlice(T, testing.allocator, "{\"values\":{\"array\":[{\"integer\":58}]}}", .{});
372		defer parseFree(T, testing.allocator, r);
	370	const parsed = try parseFromSlice(T, testing.allocator, "{\"values\":{\"array\":[{\"integer\":58}]}}", .{});
	371	defer parsed.deinit();
373	372
374		try testing.expectEqual(@as(i64, 58), r.values.array[0].integer);
	373	try testing.expectEqual(@as(i64, 58), parsed.value.values.array[0].integer);
375	374	}
376	375
377	376	const ParseIntoDoubleRecursiveUnionValueFirst = union(enum) {
...	...	@@ -389,29 +388,37 @@ test "parse into double recursive union definition" {
389	388	values: ParseIntoDoubleRecursiveUnionValueFirst,
390	389	};
391	390
392		const r = try parseFromSlice(T, testing.allocator, "{\"values\":{\"array\":[{\"array\":[{\"integer\":58}]}]}}", .{});
393		defer parseFree(T, testing.allocator, r);
	391	const parsed = try parseFromSlice(T, testing.allocator, "{\"values\":{\"array\":[{\"array\":[{\"integer\":58}]}]}}", .{});
	392	defer parsed.deinit();
394	393
395		try testing.expectEqual(@as(i64, 58), r.values.array[0].array[0].integer);
	394	try testing.expectEqual(@as(i64, 58), parsed.value.values.array[0].array[0].integer);
396	395	}
397	396
398	397	test "parse exponential into int" {
399	398	const T = struct { int: i64 };
400		const r = try parseFromSlice(T, testing.allocator, "{ \"int\": 4.2e2 }", .{});
	399	const r = try parseFromSliceLeaky(T, testing.allocator, "{ \"int\": 4.2e2 }", .{});
401	400	try testing.expectEqual(@as(i64, 420), r.int);
402		try testing.expectError(error.InvalidNumber, parseFromSlice(T, testing.allocator, "{ \"int\": 0.042e2 }", .{}));
403		try testing.expectError(error.Overflow, parseFromSlice(T, testing.allocator, "{ \"int\": 18446744073709551616.0 }", .{}));
	401	try testing.expectError(error.InvalidNumber, parseFromSliceLeaky(T, testing.allocator, "{ \"int\": 0.042e2 }", .{}));
	402	try testing.expectError(error.Overflow, parseFromSliceLeaky(T, testing.allocator, "{ \"int\": 18446744073709551616.0 }", .{}));
404	403	}
405	404
406	405	test "parseFromTokenSource" {
407		var scanner = JsonScanner.initCompleteInput(testing.allocator, "123");
408		defer scanner.deinit();
409		try testing.expectEqual(@as(u32, 123), try parseFromTokenSource(u32, testing.allocator, &scanner, .{}));
410
411		var stream = std.io.fixedBufferStream("123");
412		var json_reader = jsonReader(std.testing.allocator, stream.reader());
413		defer json_reader.deinit();
414		try testing.expectEqual(@as(u32, 123), try parseFromTokenSource(u32, testing.allocator, &json_reader, .{}));
	406	{
	407	var scanner = JsonScanner.initCompleteInput(testing.allocator, "123");
	408	defer scanner.deinit();
	409	var parsed = try parseFromTokenSource(u32, testing.allocator, &scanner, .{});
	410	defer parsed.deinit();
	411	try testing.expectEqual(@as(u32, 123), parsed.value);
	412	}
	413
	414	{
	415	var stream = std.io.fixedBufferStream("123");
	416	var json_reader = jsonReader(std.testing.allocator, stream.reader());
	417	defer json_reader.deinit();
	418	var parsed = try parseFromTokenSource(u32, testing.allocator, &json_reader, .{});
	419	defer parsed.deinit();
	420	try testing.expectEqual(@as(u32, 123), parsed.value);
	421	}
415	422	}
416	423
417	424	test "max_value_len" {
...	...	@@ -429,9 +436,9 @@ test "parse into vector" {
429	436	\\ "vec_i32": [4, 5, 6, 7]
430	437	\\}
431	438	;
432		const r = try parseFromSlice(T, testing.allocator, s, .{});
433		defer parseFree(T, testing.allocator, r);
434		try testing.expectApproxEqAbs(@as(f32, 1.5), r.vec_f32[0], 0.0000001);
435		try testing.expectApproxEqAbs(@as(f32, 2.5), r.vec_f32[1], 0.0000001);
436		try testing.expectEqual(@Vector(4, i32){ 4, 5, 6, 7 }, r.vec_i32);
	439	const parsed = try parseFromSlice(T, testing.allocator, s, .{});
	440	defer parsed.deinit();
	441	try testing.expectApproxEqAbs(@as(f32, 1.5), parsed.value.vec_f32[0], 0.0000001);
	442	try testing.expectApproxEqAbs(@as(f32, 2.5), parsed.value.vec_f32[1], 0.0000001);
	443	try testing.expectEqual(@Vector(4, i32){ 4, 5, 6, 7 }, parsed.value.vec_i32);
437	444	}

lib/std/json/test.zig+7-11

...	...	@@ -1,8 +1,9 @@
1	1	const std = @import("std");
2	2	const testing = std.testing;
3		const Parser = @import("./dynamic.zig").Parser;
	3	const parseFromSlice = @import("./static.zig").parseFromSlice;
4	4	const validate = @import("./scanner.zig").validate;
5	5	const JsonScanner = @import("./scanner.zig").Scanner;
	6	const Value = @import("./dynamic.zig").Value;
6	7
7	8	// Support for JSONTestSuite.zig
8	9	pub fn ok(s: []const u8) !void {
...	...	@@ -26,10 +27,8 @@ fn testLowLevelScanner(s: []const u8) !void {
26	27	}
27	28	}
28	29	fn testHighLevelDynamicParser(s: []const u8) !void {
29		var p = Parser.init(testing.allocator, .alloc_if_needed);
30		defer p.deinit();
31		var tree = try p.parse(s);
32		defer tree.deinit();
	30	var parsed = try parseFromSlice(Value, testing.allocator, s, .{});
	31	defer parsed.deinit();
33	32	}
34	33
35	34	// Additional tests not part of test JSONTestSuite.
...	...	@@ -47,15 +46,12 @@ test "n_object_closed_missing_value" {
47	46	fn roundTrip(s: []const u8) !void {
48	47	try testing.expect(try validate(testing.allocator, s));
49	48
50		var p = Parser.init(testing.allocator, .alloc_if_needed);
51		defer p.deinit();
52
53		var tree = try p.parse(s);
54		defer tree.deinit();
	49	var parsed = try parseFromSlice(Value, testing.allocator, s, .{});
	50	defer parsed.deinit();
55	51
56	52	var buf: [256]u8 = undefined;
57	53	var fbs = std.io.fixedBufferStream(&buf);
58		try tree.root.jsonStringify(.{}, fbs.writer());
	54	try parsed.value.jsonStringify(.{}, fbs.writer());
59	55
60	56	try testing.expectEqualStrings(s, fbs.getWritten());
61	57	}

tools/gen_spirv_spec.zig+7-6

...	...	@@ -20,15 +20,16 @@ pub fn main() !void {
20	20	// Required for json parsing.
21	21	@setEvalBranchQuota(10000);
22	22
23		var registry = try std.json.parseFromSlice(g.Registry, allocator, spec, .{});
24
25		const core_reg = switch (registry) {
26		.core => \|core_reg\| core_reg,
27		.extension => return error.TODOSpirVExtensionSpec,
	23	var scanner = std.json.Scanner.initCompleteInput(allocator, spec);
	24	var diagnostics = std.json.Diagnostics{};
	25	scanner.enableDiagnostics(&diagnostics);
	26	var parsed = std.json.parseFromTokenSource(g.CoreRegistry, allocator, &scanner, .{}) catch \|err\| {
	27	std.debug.print("line,col: {},{}\n", .{ diagnostics.getLine(), diagnostics.getColumn() });
	28	return err;
28	29	};
29	30
30	31	var bw = std.io.bufferedWriter(std.io.getStdOut().writer());
31		try render(bw.writer(), allocator, core_reg);
	32	try render(bw.writer(), allocator, parsed.value);
32	33	try bw.flush();
33	34	}
34	35

tools/spirv/grammar.zig+17

...	...	@@ -1,6 +1,9 @@
1	1	//! See https://www.khronos.org/registry/spir-v/specs/unified1/MachineReadableGrammar.html
2	2	//! and the files in https://github.com/KhronosGroup/SPIRV-Headers/blob/master/include/spirv/unified1/
3	3	//! Note: Non-canonical casing in these structs used to match SPIR-V spec json.
	4
	5	const std = @import("std");
	6
4	7	pub const Registry = union(enum) {
5	8	core: CoreRegistry,
6	9	extension: ExtensionRegistry,
...	...	@@ -79,6 +82,20 @@ pub const Enumerant = struct {
79	82	value: union(enum) {
80	83	bitflag: []const u8, // Hexadecimal representation of the value
81	84	int: u31,
	85
	86	pub fn jsonParse(
	87	allocator: std.mem.Allocator,
	88	source: anytype,
	89	options: std.json.ParseOptions,
	90	) std.json.ParseError(@TypeOf(source.*))!@This() {
	91	_ = options;
	92	switch (try source.nextAlloc(allocator, .alloc_if_needed)) {
	93	inline .string, .allocated_string => \|s\| return @This(){ .bitflag = s },
	94	inline .number, .allocated_number => \|s\| return @This(){ .int = try std.fmt.parseInt(u31, s, 10) },
	95	else => return error.UnexpectedToken,
	96	}
	97	}
	98	pub const jsonStringify = @compileError("not supported");
82	99	},
83	100	capabilities: [][]const u8 = &[_][]const u8{},
84	101	/// Valid for .ValueEnum and .BitEnum

tools/update_clang_options.zig+3-3

...	...	@@ -624,9 +624,9 @@ pub fn main() anyerror!void {
624	624	},
625	625	};
626	626
627		var parser = json.Parser.init(allocator, .alloc_if_needed);
628		const tree = try parser.parse(json_text);
629		const root_map = &tree.root.object;
	627	const parsed = try json.parseFromSlice(json.Value, allocator, json_text, .{});
	628	defer parsed.deinit();
	629	const root_map = &parsed.value.object;
630	630
631	631	var all_objects = std.ArrayList(*json.ObjectMap).init(allocator);
632	632	{

tools/update_cpu_features.zig+3-3

...	...	@@ -1054,14 +1054,14 @@ fn processOneTarget(job: Job) anyerror!void {
1054	1054	var json_parse_progress = progress_node.start("parse JSON", 0);
1055	1055	json_parse_progress.activate();
1056	1056
1057		var parser = json.Parser.init(arena, .alloc_if_needed);
1058		const tree = try parser.parse(json_text);
	1057	const parsed = try json.parseFromSlice(json.Value, arena, json_text, .{});
	1058	defer parsed.deinit();
	1059	const root_map = &parsed.value.object;
1059	1060	json_parse_progress.end();
1060	1061
1061	1062	var render_progress = progress_node.start("render zig code", 0);
1062	1063	render_progress.activate();
1063	1064
1064		const root_map = &tree.root.object;
1065	1065	var features_table = std.StringHashMap(Feature).init(arena);
1066	1066	var all_features = std.ArrayList(Feature).init(arena);
1067	1067	var all_cpus = std.ArrayList(Cpu).init(arena);

tools/update_spirv_features.zig+7-1

...	...	@@ -74,7 +74,13 @@ pub fn main() !void {
74	74
75	75	const registry_path = try fs.path.join(allocator, &.{ spirv_headers_root, "include", "spirv", "unified1", "spirv.core.grammar.json" });
76	76	const registry_json = try std.fs.cwd().readFileAlloc(allocator, registry_path, std.math.maxInt(usize));
77		const registry = try std.json.parseFromSlice(g.CoreRegistry, allocator, registry_json, .{});
	77	var scanner = std.json.Scanner.initCompleteInput(allocator, registry_json);
	78	var diagnostics = std.json.Diagnostics{};
	79	scanner.enableDiagnostics(&diagnostics);
	80	const registry = std.json.parseFromTokenSourceLeaky(g.CoreRegistry, allocator, &scanner, .{}) catch \|err\| {
	81	std.debug.print("line,col: {},{}\n", .{ diagnostics.getLine(), diagnostics.getColumn() });
	82	return err;
	83	};
78	84
79	85	const capabilities = for (registry.operand_kinds) \|opkind\| {
80	86	if (std.mem.eql(u8, opkind.kind, "Capability"))