Commit 018b743c7a83c2af5e5b6ba9aae1a4703e306f71 | zig

author	Josh Wolfe <thejoshwolfe@gmail.com> 2023-05-13 14:31:53-04:00
committer	GitHub <noreply@github.com> 2023-05-13 14:31:53-04:00
log	018b743c7a83c2af5e5b6ba9aae1a4703e306f71
tree	3e113cc28cc3dcaace4917980c2813b1a6de2654
parent	c7bf8bab38f8b89c1371eedb9229e00a29b5ca5b
signature	Signed by PGP key 4AEE18F83AFDEB23

std: Rewrite low-level json api to support streaming (#15602)

20 files changed, 5793 insertions(+), 5845 deletions(-)

lib/std/json.zig+50-2809

...	@@ -1,2818 +1,59 @@	...	@@ -1,2818 +1,59 @@
1	// JSON parser conforming to RFC8259.	1	//! JSON parsing and stringification conforming to RFC 8259. https://datatracker.ietf.org/doc/html/rfc8259
2	//	2	//!
3	// https://tools.ietf.org/html/rfc8259	3	//! The low-level `Scanner` API reads from an input slice or successive slices of inputs,
4		4	//! The `Reader` API connects a `std.io.Reader` to a `Scanner`.
5	const builtin = @import("builtin");	5	//!
6	const std = @import("std.zig");	6	//! The high-level `parseFromSlice` and `parseFromTokenSource` deserializes a JSON document into a Zig type.
7	const debug = std.debug;	7	//! The high-level `Parser` parses any JSON document into a dynamically typed `ValueTree` that has its own memory arena.
8	const assert = debug.assert;	8	//!
9	const testing = std.testing;	9	//! The low-level `writeStream` emits syntax-conformant JSON tokens to a `std.io.Writer`.
10	const mem = std.mem;	10	//! The high-level `stringify` serializes a Zig type into JSON.
11	const maxInt = std.math.maxInt;	11
		12	pub const ValueTree = @import("json/dynamic.zig").ValueTree;
		13	pub const ObjectMap = @import("json/dynamic.zig").ObjectMap;
		14	pub const Array = @import("json/dynamic.zig").Array;
		15	pub const Value = @import("json/dynamic.zig").Value;
		16	pub const Parser = @import("json/dynamic.zig").Parser;
		17
		18	pub const validate = @import("json/scanner.zig").validate;
		19	pub const Error = @import("json/scanner.zig").Error;
		20	pub const reader = @import("json/scanner.zig").reader;
		21	pub const default_buffer_size = @import("json/scanner.zig").default_buffer_size;
		22	pub const Token = @import("json/scanner.zig").Token;
		23	pub const TokenType = @import("json/scanner.zig").TokenType;
		24	pub const Diagnostics = @import("json/scanner.zig").Diagnostics;
		25	pub const AllocWhen = @import("json/scanner.zig").AllocWhen;
		26	pub const default_max_value_len = @import("json/scanner.zig").default_max_value_len;
		27	pub const Reader = @import("json/scanner.zig").Reader;
		28	pub const Scanner = @import("json/scanner.zig").Scanner;
		29	pub const isNumberFormattedLikeAnInteger = @import("json/scanner.zig").isNumberFormattedLikeAnInteger;
		30
		31	pub const ParseOptions = @import("json/static.zig").ParseOptions;
		32	pub const parseFromSlice = @import("json/static.zig").parseFromSlice;
		33	pub const parseFromTokenSource = @import("json/static.zig").parseFromTokenSource;
		34	pub const ParseError = @import("json/static.zig").ParseError;
		35	pub const parseFree = @import("json/static.zig").parseFree;
		36
		37	pub const StringifyOptions = @import("json/stringify.zig").StringifyOptions;
		38	pub const encodeJsonString = @import("json/stringify.zig").encodeJsonString;
		39	pub const encodeJsonStringChars = @import("json/stringify.zig").encodeJsonStringChars;
		40	pub const stringify = @import("json/stringify.zig").stringify;
		41	pub const stringifyAlloc = @import("json/stringify.zig").stringifyAlloc;
12		42
13	pub const WriteStream = @import("json/write_stream.zig").WriteStream;	43	pub const WriteStream = @import("json/write_stream.zig").WriteStream;
14	pub const writeStream = @import("json/write_stream.zig").writeStream;	44	pub const writeStream = @import("json/write_stream.zig").writeStream;
15		45
16	const StringEscapes = union(enum) {	46	// Deprecations
17	None,	47	pub const parse = @compileError("Deprecated; use parseFromSlice() or parseFromTokenSource() instead.");
18		48	pub const StreamingParser = @compileError("Deprecated; use json.Scanner or json.Reader instead.");
19	Some: struct {	49	pub const TokenStream = @compileError("Deprecated; use json.Scanner or json.Reader instead.");
20	size_diff: isize,
21	},
22	};
23
24	/// Checks to see if a string matches what it would be as a json-encoded string
25	/// Assumes that `encoded` is a well-formed json string
26	fn encodesTo(decoded: []const u8, encoded: []const u8) bool {
27	var i: usize = 0;
28	var j: usize = 0;
29	while (i < decoded.len) {
30	if (j >= encoded.len) return false;
31	if (encoded[j] != '\\') {
32	if (decoded[i] != encoded[j]) return false;
33	j += 1;
34	i += 1;
35	} else {
36	const escape_type = encoded[j + 1];
37	if (escape_type != 'u') {
38	const t: u8 = switch (escape_type) {
39	'\\' => '\\',
40	'/' => '/',
41	'n' => '\n',
42	'r' => '\r',
43	't' => '\t',
44	'f' => 12,
45	'b' => 8,
46	'"' => '"',
47	else => unreachable,
48	};
49	if (decoded[i] != t) return false;
50	j += 2;
51	i += 1;
52	} else {
53	var codepoint = std.fmt.parseInt(u21, encoded[j + 2 .. j + 6], 16) catch unreachable;
54	j += 6;
55	if (codepoint >= 0xD800 and codepoint < 0xDC00) {
56	// surrogate pair
57	assert(encoded[j] == '\\');
58	assert(encoded[j + 1] == 'u');
59	const low_surrogate = std.fmt.parseInt(u21, encoded[j + 2 .. j + 6], 16) catch unreachable;
60	codepoint = 0x10000 + (((codepoint & 0x03ff) << 10) \| (low_surrogate & 0x03ff));
61	j += 6;
62	}
63	var buf: [4]u8 = undefined;
64	const len = std.unicode.utf8Encode(codepoint, &buf) catch unreachable;
65	if (i + len > decoded.len) return false;
66	if (!mem.eql(u8, decoded[i..][0..len], buf[0..len])) return false;
67	i += len;
68	}
69	}
70	}
71	assert(i == decoded.len);
72	assert(j == encoded.len);
73	return true;
74	}
75
76	/// A single token slice into the parent string.
77	///
78	/// Use `token.slice()` on the input at the current position to get the current slice.
79	pub const Token = union(enum) {
80	ObjectBegin,
81	ObjectEnd,
82	ArrayBegin,
83	ArrayEnd,
84	String: struct {
85	/// How many bytes the token is.
86	count: usize,
87
88	/// Whether string contains an escape sequence and cannot be zero-copied
89	escapes: StringEscapes,
90
91	pub fn decodedLength(self: @This()) usize {
92	return self.count +% switch (self.escapes) {
93	.None => 0,
94	.Some => \|s\| @bitCast(usize, s.size_diff),
95	};
96	}
97
98	/// Slice into the underlying input string.
99	pub fn slice(self: @This(), input: []const u8, i: usize) []const u8 {
100	return input[i - self.count .. i];
101	}
102	},
103	Number: struct {
104	/// How many bytes the token is.
105	count: usize,
106
107	/// Whether number is simple and can be represented by an integer (i.e. no `.` or `e`)
108	is_integer: bool,
109
110	/// Slice into the underlying input string.
111	pub fn slice(self: @This(), input: []const u8, i: usize) []const u8 {
112	return input[i - self.count .. i];
113	}
114	},
115	True,
116	False,
117	Null,
118	};
119
120	const AggregateContainerType = enum(u1) { object, array };
121
122	// A LIFO bit-stack. Tracks which container-types have been entered during parse.
123	fn AggregateContainerStack(comptime n: usize) type {
124	return struct {
125	const Self = @This();
126
127	const element_bitcount = 8 * @sizeOf(usize);
128	const element_count = n / element_bitcount;
129	const ElementType = @Type(.{ .Int = .{ .signedness = .unsigned, .bits = element_bitcount } });
130	const ElementShiftAmountType = std.math.Log2Int(ElementType);
131
132	comptime {
133	std.debug.assert(n % element_bitcount == 0);
134	}
135
136	memory: [element_count]ElementType,
137	len: usize,
138
139	pub fn init(self: *Self) void {
140	self.memory = [_]ElementType{0} ** element_count;
141	self.len = 0;
142	}
143
144	pub fn push(self: *Self, ty: AggregateContainerType) ?void {
145	if (self.len >= n) {
146	return null;
147	}
148
149	const index = self.len / element_bitcount;
150	const sub_index = @intCast(ElementShiftAmountType, self.len % element_bitcount);
151	const clear_mask = ~(@as(ElementType, 1) << sub_index);
152	const set_bits = @as(ElementType, @enumToInt(ty)) << sub_index;
153
154	self.memory[index] &= clear_mask;
155	self.memory[index] \|= set_bits;
156	self.len += 1;
157	}
158
159	pub fn peek(self: *Self) ?AggregateContainerType {
160	if (self.len == 0) {
161	return null;
162	}
163
164	const bit_to_extract = self.len - 1;
165	const index = bit_to_extract / element_bitcount;
166	const sub_index = @intCast(ElementShiftAmountType, bit_to_extract % element_bitcount);
167	const bit = @intCast(u1, (self.memory[index] >> sub_index) & 1);
168	return @intToEnum(AggregateContainerType, bit);
169	}
170
171	pub fn pop(self: *Self) ?AggregateContainerType {
172	if (self.peek()) \|ty\| {
173	self.len -= 1;
174	return ty;
175	}
176
177	return null;
178	}
179	};
180	}
181
182	/// A small streaming JSON parser. This accepts input one byte at a time and returns tokens as
183	/// they are encountered. No copies or allocations are performed during parsing and the entire
184	/// parsing state requires ~40-50 bytes of stack space.
185	///
186	/// Conforms strictly to RFC8259.
187	///
188	/// For a non-byte based wrapper, consider using TokenStream instead.
189	pub const StreamingParser = struct {
190	const default_max_nestings = 256;
191
192	// Current state
193	state: State,
194	// How many bytes we have counted for the current token
195	count: usize,
196	// What state to follow after parsing a string (either property or value string)
197	after_string_state: State,
198	// What state to follow after parsing a value (either top-level or value end)
199	after_value_state: State,
200	// If we stopped now, would the complete parsed string to now be a valid json string
201	complete: bool,
202	// Current token flags to pass through to the next generated, see Token.
203	string_escapes: StringEscapes,
204	// When in .String states, was the previous character a high surrogate?
205	string_last_was_high_surrogate: bool,
206	// Used inside of StringEscapeHexUnicode* states
207	string_unicode_codepoint: u21,
208	// The first byte needs to be stored to validate 3- and 4-byte sequences.
209	sequence_first_byte: u8 = undefined,
210	// When in .Number states, is the number a (still) valid integer?
211	number_is_integer: bool,
212	// Bit-stack for nested object/map literals (max 256 nestings).
213	stack: AggregateContainerStack(default_max_nestings),
214
215	pub fn init() StreamingParser {
216	var p: StreamingParser = undefined;
217	p.reset();
218	return p;
219	}
220
221	pub fn reset(p: *StreamingParser) void {
222	p.state = .TopLevelBegin;
223	p.count = 0;
224	// Set before ever read in main transition function
225	p.after_string_state = undefined;
226	p.after_value_state = .ValueEnd; // handle end of values normally
227	p.stack.init();
228	p.complete = false;
229	p.string_escapes = undefined;
230	p.string_last_was_high_surrogate = undefined;
231	p.string_unicode_codepoint = undefined;
232	p.number_is_integer = undefined;
233	}
234
235	pub const State = enum(u8) {
236	// These must be first with these explicit values as we rely on them for indexing the
237	// bit-stack directly and avoiding a branch.
238	ObjectSeparator = 0,
239	ValueEnd = 1,
240
241	TopLevelBegin,
242	TopLevelEnd,
243
244	ValueBegin,
245	ValueBeginNoClosing,
246
247	String,
248	StringUtf8Byte2Of2,
249	StringUtf8Byte2Of3,
250	StringUtf8Byte3Of3,
251	StringUtf8Byte2Of4,
252	StringUtf8Byte3Of4,
253	StringUtf8Byte4Of4,
254	StringEscapeCharacter,
255	StringEscapeHexUnicode4,
256	StringEscapeHexUnicode3,
257	StringEscapeHexUnicode2,
258	StringEscapeHexUnicode1,
259
260	Number,
261	NumberMaybeDotOrExponent,
262	NumberMaybeDigitOrDotOrExponent,
263	NumberFractionalRequired,
264	NumberFractional,
265	NumberMaybeExponent,
266	NumberExponent,
267	NumberExponentDigitsRequired,
268	NumberExponentDigits,
269
270	TrueLiteral1,
271	TrueLiteral2,
272	TrueLiteral3,
273
274	FalseLiteral1,
275	FalseLiteral2,
276	FalseLiteral3,
277	FalseLiteral4,
278
279	NullLiteral1,
280	NullLiteral2,
281	NullLiteral3,
282
283	// Given an aggregate container type, return the state which should be entered after
284	// processing a complete value type.
285	pub fn fromAggregateContainerType(ty: AggregateContainerType) State {
286	comptime {
287	std.debug.assert(@enumToInt(AggregateContainerType.object) == @enumToInt(State.ObjectSeparator));
288	std.debug.assert(@enumToInt(AggregateContainerType.array) == @enumToInt(State.ValueEnd));
289	}
290
291	return @intToEnum(State, @enumToInt(ty));
292	}
293	};
294
295	pub const Error = error{
296	InvalidTopLevel,
297	TooManyNestedItems,
298	TooManyClosingItems,
299	InvalidValueBegin,
300	InvalidValueEnd,
301	UnbalancedBrackets,
302	UnbalancedBraces,
303	UnexpectedClosingBracket,
304	UnexpectedClosingBrace,
305	InvalidNumber,
306	InvalidSeparator,
307	InvalidLiteral,
308	InvalidEscapeCharacter,
309	InvalidUnicodeHexSymbol,
310	InvalidUtf8Byte,
311	InvalidTopLevelTrailing,
312	InvalidControlCharacter,
313	};
314
315	/// Give another byte to the parser and obtain any new tokens. This may (rarely) return two
316	/// tokens. token2 is always null if token1 is null.
317	///
318	/// There is currently no error recovery on a bad stream.
319	pub fn feed(p: StreamingParser, c: u8, token1: ?Token, token2: *?Token) Error!void {
320	token1.* = null;
321	token2.* = null;
322	p.count += 1;
323
324	// unlikely
325	if (try p.transition(c, token1)) {
326	_ = try p.transition(c, token2);
327	}
328	}
329
330	// Perform a single transition on the state machine and return any possible token.
331	fn transition(p: StreamingParser, c: u8, token: ?Token) Error!bool {
332	switch (p.state) {
333	.TopLevelBegin => switch (c) {
334	'{' => {
335	p.stack.push(.object) orelse return error.TooManyNestedItems;
336	p.state = .ValueBegin;
337	p.after_string_state = .ObjectSeparator;
338
339	token.* = Token.ObjectBegin;
340	},
341	'[' => {
342	p.stack.push(.array) orelse return error.TooManyNestedItems;
343	p.state = .ValueBegin;
344	p.after_string_state = .ValueEnd;
345
346	token.* = Token.ArrayBegin;
347	},
348	'-' => {
349	p.number_is_integer = true;
350	p.state = .Number;
351	p.after_value_state = .TopLevelEnd;
352	p.count = 0;
353	},
354	'0' => {
355	p.number_is_integer = true;
356	p.state = .NumberMaybeDotOrExponent;
357	p.after_value_state = .TopLevelEnd;
358	p.count = 0;
359	},
360	'1'...'9' => {
361	p.number_is_integer = true;
362	p.state = .NumberMaybeDigitOrDotOrExponent;
363	p.after_value_state = .TopLevelEnd;
364	p.count = 0;
365	},
366	'"' => {
367	p.state = .String;
368	p.after_value_state = .TopLevelEnd;
369	// We don't actually need the following since after_value_state should override.
370	p.after_string_state = .ValueEnd;
371	p.string_escapes = .None;
372	p.string_last_was_high_surrogate = false;
373	p.count = 0;
374	},
375	't' => {
376	p.state = .TrueLiteral1;
377	p.after_value_state = .TopLevelEnd;
378	p.count = 0;
379	},
380	'f' => {
381	p.state = .FalseLiteral1;
382	p.after_value_state = .TopLevelEnd;
383	p.count = 0;
384	},
385	'n' => {
386	p.state = .NullLiteral1;
387	p.after_value_state = .TopLevelEnd;
388	p.count = 0;
389	},
390	0x09, 0x0A, 0x0D, 0x20 => {
391	// whitespace
392	},
393	else => {
394	return error.InvalidTopLevel;
395	},
396	},
397
398	.TopLevelEnd => switch (c) {
399	0x09, 0x0A, 0x0D, 0x20 => {
400	// whitespace
401	},
402	else => {
403	return error.InvalidTopLevelTrailing;
404	},
405	},
406
407	.ValueBegin => switch (c) {
408	// NOTE: These are shared in ValueEnd as well, think we can reorder states to
409	// be a bit clearer and avoid this duplication.
410	'}' => {
411	const last_type = p.stack.peek() orelse return error.TooManyClosingItems;
412
413	if (last_type != .object) {
414	return error.UnexpectedClosingBrace;
415	}
416
417	_ = p.stack.pop();
418	p.state = .ValueBegin;
419	p.after_string_state = State.fromAggregateContainerType(last_type);
420
421	switch (p.stack.len) {
422	0 => {
423	p.complete = true;
424	p.state = .TopLevelEnd;
425	},
426	else => {
427	p.state = .ValueEnd;
428	},
429	}
430
431	token.* = Token.ObjectEnd;
432	},
433	']' => {
434	const last_type = p.stack.peek() orelse return error.TooManyClosingItems;
435
436	if (last_type != .array) {
437	return error.UnexpectedClosingBracket;
438	}
439
440	_ = p.stack.pop();
441	p.state = .ValueBegin;
442	p.after_string_state = State.fromAggregateContainerType(last_type);
443
444	switch (p.stack.len) {
445	0 => {
446	p.complete = true;
447	p.state = .TopLevelEnd;
448	},
449	else => {
450	p.state = .ValueEnd;
451	},
452	}
453
454	token.* = Token.ArrayEnd;
455	},
456	'{' => {
457	p.stack.push(.object) orelse return error.TooManyNestedItems;
458
459	p.state = .ValueBegin;
460	p.after_string_state = .ObjectSeparator;
461
462	token.* = Token.ObjectBegin;
463	},
464	'[' => {
465	p.stack.push(.array) orelse return error.TooManyNestedItems;
466
467	p.state = .ValueBegin;
468	p.after_string_state = .ValueEnd;
469
470	token.* = Token.ArrayBegin;
471	},
472	'-' => {
473	p.number_is_integer = true;
474	p.state = .Number;
475	p.count = 0;
476	},
477	'0' => {
478	p.number_is_integer = true;
479	p.state = .NumberMaybeDotOrExponent;
480	p.count = 0;
481	},
482	'1'...'9' => {
483	p.number_is_integer = true;
484	p.state = .NumberMaybeDigitOrDotOrExponent;
485	p.count = 0;
486	},
487	'"' => {
488	p.state = .String;
489	p.string_escapes = .None;
490	p.string_last_was_high_surrogate = false;
491	p.count = 0;
492	},
493	't' => {
494	p.state = .TrueLiteral1;
495	p.count = 0;
496	},
497	'f' => {
498	p.state = .FalseLiteral1;
499	p.count = 0;
500	},
501	'n' => {
502	p.state = .NullLiteral1;
503	p.count = 0;
504	},
505	0x09, 0x0A, 0x0D, 0x20 => {
506	// whitespace
507	},
508	else => {
509	return error.InvalidValueBegin;
510	},
511	},
512
513	// TODO: A bit of duplication here and in the following state, redo.
514	.ValueBeginNoClosing => switch (c) {
515	'{' => {
516	p.stack.push(.object) orelse return error.TooManyNestedItems;
517
518	p.state = .ValueBegin;
519	p.after_string_state = .ObjectSeparator;
520
521	token.* = Token.ObjectBegin;
522	},
523	'[' => {
524	p.stack.push(.array) orelse return error.TooManyNestedItems;
525
526	p.state = .ValueBegin;
527	p.after_string_state = .ValueEnd;
528
529	token.* = Token.ArrayBegin;
530	},
531	'-' => {
532	p.number_is_integer = true;
533	p.state = .Number;
534	p.count = 0;
535	},
536	'0' => {
537	p.number_is_integer = true;
538	p.state = .NumberMaybeDotOrExponent;
539	p.count = 0;
540	},
541	'1'...'9' => {
542	p.number_is_integer = true;
543	p.state = .NumberMaybeDigitOrDotOrExponent;
544	p.count = 0;
545	},
546	'"' => {
547	p.state = .String;
548	p.string_escapes = .None;
549	p.string_last_was_high_surrogate = false;
550	p.count = 0;
551	},
552	't' => {
553	p.state = .TrueLiteral1;
554	p.count = 0;
555	},
556	'f' => {
557	p.state = .FalseLiteral1;
558	p.count = 0;
559	},
560	'n' => {
561	p.state = .NullLiteral1;
562	p.count = 0;
563	},
564	0x09, 0x0A, 0x0D, 0x20 => {
565	// whitespace
566	},
567	else => {
568	return error.InvalidValueBegin;
569	},
570	},
571
572	.ValueEnd => switch (c) {
573	',' => {
574	const last_type = p.stack.peek() orelse unreachable;
575	p.after_string_state = State.fromAggregateContainerType(last_type);
576	p.state = .ValueBeginNoClosing;
577	},
578	']' => {
579	const last_type = p.stack.peek() orelse return error.TooManyClosingItems;
580
581	if (last_type != .array) {
582	return error.UnexpectedClosingBracket;
583	}
584
585	_ = p.stack.pop();
586	p.state = .ValueEnd;
587	p.after_string_state = State.fromAggregateContainerType(last_type);
588
589	if (p.stack.len == 0) {
590	p.complete = true;
591	p.state = .TopLevelEnd;
592	}
593
594	token.* = Token.ArrayEnd;
595	},
596	'}' => {
597	const last_type = p.stack.peek() orelse return error.TooManyClosingItems;
598
599	if (last_type != .object) {
600	return error.UnexpectedClosingBrace;
601	}
602
603	_ = p.stack.pop();
604	p.state = .ValueEnd;
605	p.after_string_state = State.fromAggregateContainerType(last_type);
606
607	if (p.stack.len == 0) {
608	p.complete = true;
609	p.state = .TopLevelEnd;
610	}
611
612	token.* = Token.ObjectEnd;
613	},
614	0x09, 0x0A, 0x0D, 0x20 => {
615	// whitespace
616	},
617	else => {
618	return error.InvalidValueEnd;
619	},
620	},
621
622	.ObjectSeparator => switch (c) {
623	':' => {
624	p.state = .ValueBeginNoClosing;
625	p.after_string_state = .ValueEnd;
626	},
627	0x09, 0x0A, 0x0D, 0x20 => {
628	// whitespace
629	},
630	else => {
631	return error.InvalidSeparator;
632	},
633	},
634
635	.String => switch (c) {
636	0x00...0x1F => {
637	return error.InvalidControlCharacter;
638	},
639	'"' => {
640	p.state = p.after_string_state;
641	if (p.after_value_state == .TopLevelEnd) {
642	p.state = .TopLevelEnd;
643	p.complete = true;
644	}
645
646	token.* = .{
647	.String = .{
648	.count = p.count - 1,
649	.escapes = p.string_escapes,
650	},
651	};
652	p.string_escapes = undefined;
653	p.string_last_was_high_surrogate = undefined;
654	},
655	'\\' => {
656	p.state = .StringEscapeCharacter;
657	switch (p.string_escapes) {
658	.None => {
659	p.string_escapes = .{ .Some = .{ .size_diff = 0 } };
660	},
661	.Some => {},
662	}
663	},
664	0x20, 0x21, 0x23...0x5B, 0x5D...0x7F => {
665	// non-control ascii
666	p.string_last_was_high_surrogate = false;
667	},
668	0xC2...0xDF => {
669	p.state = .StringUtf8Byte2Of2;
670	},
671	0xE0...0xEF => {
672	p.state = .StringUtf8Byte2Of3;
673	p.sequence_first_byte = c;
674	},
675	0xF0...0xF4 => {
676	p.state = .StringUtf8Byte2Of4;
677	p.sequence_first_byte = c;
678	},
679	else => {
680	return error.InvalidUtf8Byte;
681	},
682	},
683
684	.StringUtf8Byte2Of2 => switch (c >> 6) {
685	0b10 => p.state = .String,
686	else => return error.InvalidUtf8Byte,
687	},
688	.StringUtf8Byte2Of3 => {
689	switch (p.sequence_first_byte) {
690	0xE0 => switch (c) {
691	0xA0...0xBF => {},
692	else => return error.InvalidUtf8Byte,
693	},
694	0xE1...0xEF => switch (c) {
695	0x80...0xBF => {},
696	else => return error.InvalidUtf8Byte,
697	},
698	else => return error.InvalidUtf8Byte,
699	}
700	p.state = .StringUtf8Byte3Of3;
701	},
702	.StringUtf8Byte3Of3 => switch (c) {
703	0x80...0xBF => p.state = .String,
704	else => return error.InvalidUtf8Byte,
705	},
706	.StringUtf8Byte2Of4 => {
707	switch (p.sequence_first_byte) {
708	0xF0 => switch (c) {
709	0x90...0xBF => {},
710	else => return error.InvalidUtf8Byte,
711	},
712	0xF1...0xF3 => switch (c) {
713	0x80...0xBF => {},
714	else => return error.InvalidUtf8Byte,
715	},
716	0xF4 => switch (c) {
717	0x80...0x8F => {},
718	else => return error.InvalidUtf8Byte,
719	},
720	else => return error.InvalidUtf8Byte,
721	}
722	p.state = .StringUtf8Byte3Of4;
723	},
724	.StringUtf8Byte3Of4 => switch (c) {
725	0x80...0xBF => p.state = .StringUtf8Byte4Of4,
726	else => return error.InvalidUtf8Byte,
727	},
728	.StringUtf8Byte4Of4 => switch (c) {
729	0x80...0xBF => p.state = .String,
730	else => return error.InvalidUtf8Byte,
731	},
732
733	.StringEscapeCharacter => switch (c) {
734	// NOTE: '/' is allowed as an escaped character but it also is allowed
735	// as unescaped according to the RFC. There is a reported errata which suggests
736	// removing the non-escaped variant but it makes more sense to simply disallow
737	// it as an escape code here.
738	//
739	// The current JSONTestSuite tests rely on both of this behaviour being present
740	// however, so we default to the status quo where both are accepted until this
741	// is further clarified.
742	'"', '\\', '/', 'b', 'f', 'n', 'r', 't' => {
743	p.string_escapes.Some.size_diff -= 1;
744	p.state = .String;
745	p.string_last_was_high_surrogate = false;
746	},
747	'u' => {
748	p.state = .StringEscapeHexUnicode4;
749	},
750	else => {
751	return error.InvalidEscapeCharacter;
752	},
753	},
754
755	.StringEscapeHexUnicode4 => {
756	var codepoint: u21 = undefined;
757	switch (c) {
758	else => return error.InvalidUnicodeHexSymbol,
759	'0'...'9' => {
760	codepoint = c - '0';
761	},
762	'A'...'F' => {
763	codepoint = c - 'A' + 10;
764	},
765	'a'...'f' => {
766	codepoint = c - 'a' + 10;
767	},
768	}
769	p.state = .StringEscapeHexUnicode3;
770	p.string_unicode_codepoint = codepoint << 12;
771	},
772
773	.StringEscapeHexUnicode3 => {
774	var codepoint: u21 = undefined;
775	switch (c) {
776	else => return error.InvalidUnicodeHexSymbol,
777	'0'...'9' => {
778	codepoint = c - '0';
779	},
780	'A'...'F' => {
781	codepoint = c - 'A' + 10;
782	},
783	'a'...'f' => {
784	codepoint = c - 'a' + 10;
785	},
786	}
787	p.state = .StringEscapeHexUnicode2;
788	p.string_unicode_codepoint \|= codepoint << 8;
789	},
790
791	.StringEscapeHexUnicode2 => {
792	var codepoint: u21 = undefined;
793	switch (c) {
794	else => return error.InvalidUnicodeHexSymbol,
795	'0'...'9' => {
796	codepoint = c - '0';
797	},
798	'A'...'F' => {
799	codepoint = c - 'A' + 10;
800	},
801	'a'...'f' => {
802	codepoint = c - 'a' + 10;
803	},
804	}
805	p.state = .StringEscapeHexUnicode1;
806	p.string_unicode_codepoint \|= codepoint << 4;
807	},
808
809	.StringEscapeHexUnicode1 => {
810	var codepoint: u21 = undefined;
811	switch (c) {
812	else => return error.InvalidUnicodeHexSymbol,
813	'0'...'9' => {
814	codepoint = c - '0';
815	},
816	'A'...'F' => {
817	codepoint = c - 'A' + 10;
818	},
819	'a'...'f' => {
820	codepoint = c - 'a' + 10;
821	},
822	}
823	p.state = .String;
824	p.string_unicode_codepoint \|= codepoint;
825	if (p.string_unicode_codepoint < 0xD800 or p.string_unicode_codepoint >= 0xE000) {
826	// not part of surrogate pair
827	p.string_escapes.Some.size_diff -= @as(isize, 6 - (std.unicode.utf8CodepointSequenceLength(p.string_unicode_codepoint) catch unreachable));
828	p.string_last_was_high_surrogate = false;
829	} else if (p.string_unicode_codepoint < 0xDC00) {
830	// 'high' surrogate
831	// takes 3 bytes to encode a half surrogate pair into wtf8
832	p.string_escapes.Some.size_diff -= 6 - 3;
833	p.string_last_was_high_surrogate = true;
834	} else {
835	// 'low' surrogate
836	p.string_escapes.Some.size_diff -= 6;
837	if (p.string_last_was_high_surrogate) {
838	// takes 4 bytes to encode a full surrogate pair into utf8
839	// 3 bytes are already reserved by high surrogate
840	p.string_escapes.Some.size_diff -= -1;
841	} else {
842	// takes 3 bytes to encode a half surrogate pair into wtf8
843	p.string_escapes.Some.size_diff -= -3;
844	}
845	p.string_last_was_high_surrogate = false;
846	}
847	p.string_unicode_codepoint = undefined;
848	},
849
850	.Number => {
851	p.complete = p.after_value_state == .TopLevelEnd;
852	switch (c) {
853	'0' => {
854	p.state = .NumberMaybeDotOrExponent;
855	},
856	'1'...'9' => {
857	p.state = .NumberMaybeDigitOrDotOrExponent;
858	},
859	else => {
860	return error.InvalidNumber;
861	},
862	}
863	},
864
865	.NumberMaybeDotOrExponent => {
866	p.complete = p.after_value_state == .TopLevelEnd;
867	switch (c) {
868	'.' => {
869	p.number_is_integer = false;
870	p.state = .NumberFractionalRequired;
871	},
872	'e', 'E' => {
873	p.number_is_integer = false;
874	p.state = .NumberExponent;
875	},
876	else => {
877	p.state = p.after_value_state;
878	token.* = .{
879	.Number = .{
880	.count = p.count,
881	.is_integer = p.number_is_integer,
882	},
883	};
884	p.number_is_integer = undefined;
885	return true;
886	},
887	}
888	},
889
890	.NumberMaybeDigitOrDotOrExponent => {
891	p.complete = p.after_value_state == .TopLevelEnd;
892	switch (c) {
893	'.' => {
894	p.number_is_integer = false;
895	p.state = .NumberFractionalRequired;
896	},
897	'e', 'E' => {
898	p.number_is_integer = false;
899	p.state = .NumberExponent;
900	},
901	'0'...'9' => {
902	// another digit
903	},
904	else => {
905	p.state = p.after_value_state;
906	token.* = .{
907	.Number = .{
908	.count = p.count,
909	.is_integer = p.number_is_integer,
910	},
911	};
912	return true;
913	},
914	}
915	},
916
917	.NumberFractionalRequired => {
918	p.complete = p.after_value_state == .TopLevelEnd;
919	switch (c) {
920	'0'...'9' => {
921	p.state = .NumberFractional;
922	},
923	else => {
924	return error.InvalidNumber;
925	},
926	}
927	},
928
929	.NumberFractional => {
930	p.complete = p.after_value_state == .TopLevelEnd;
931	switch (c) {
932	'0'...'9' => {
933	// another digit
934	},
935	'e', 'E' => {
936	p.number_is_integer = false;
937	p.state = .NumberExponent;
938	},
939	else => {
940	p.state = p.after_value_state;
941	token.* = .{
942	.Number = .{
943	.count = p.count,
944	.is_integer = p.number_is_integer,
945	},
946	};
947	return true;
948	},
949	}
950	},
951
952	.NumberMaybeExponent => {
953	p.complete = p.after_value_state == .TopLevelEnd;
954	switch (c) {
955	'e', 'E' => {
956	p.number_is_integer = false;
957	p.state = .NumberExponent;
958	},
959	else => {
960	p.state = p.after_value_state;
961	token.* = .{
962	.Number = .{
963	.count = p.count,
964	.is_integer = p.number_is_integer,
965	},
966	};
967	return true;
968	},
969	}
970	},
971
972	.NumberExponent => switch (c) {
973	'-', '+' => {
974	p.complete = false;
975	p.state = .NumberExponentDigitsRequired;
976	},
977	'0'...'9' => {
978	p.complete = p.after_value_state == .TopLevelEnd;
979	p.state = .NumberExponentDigits;
980	},
981	else => {
982	return error.InvalidNumber;
983	},
984	},
985
986	.NumberExponentDigitsRequired => switch (c) {
987	'0'...'9' => {
988	p.complete = p.after_value_state == .TopLevelEnd;
989	p.state = .NumberExponentDigits;
990	},
991	else => {
992	return error.InvalidNumber;
993	},
994	},
995
996	.NumberExponentDigits => {
997	p.complete = p.after_value_state == .TopLevelEnd;
998	switch (c) {
999	'0'...'9' => {
1000	// another digit
1001	},
1002	else => {
1003	p.state = p.after_value_state;
1004	token.* = .{
1005	.Number = .{
1006	.count = p.count,
1007	.is_integer = p.number_is_integer,
1008	},
1009	};
1010	return true;
1011	},
1012	}
1013	},
1014
1015	.TrueLiteral1 => switch (c) {
1016	'r' => p.state = .TrueLiteral2,
1017	else => return error.InvalidLiteral,
1018	},
1019
1020	.TrueLiteral2 => switch (c) {
1021	'u' => p.state = .TrueLiteral3,
1022	else => return error.InvalidLiteral,
1023	},
1024
1025	.TrueLiteral3 => switch (c) {
1026	'e' => {
1027	p.state = p.after_value_state;
1028	p.complete = p.state == .TopLevelEnd;
1029	token.* = Token.True;
1030	},
1031	else => {
1032	return error.InvalidLiteral;
1033	},
1034	},
1035
1036	.FalseLiteral1 => switch (c) {
1037	'a' => p.state = .FalseLiteral2,
1038	else => return error.InvalidLiteral,
1039	},
1040
1041	.FalseLiteral2 => switch (c) {
1042	'l' => p.state = .FalseLiteral3,
1043	else => return error.InvalidLiteral,
1044	},
1045
1046	.FalseLiteral3 => switch (c) {
1047	's' => p.state = .FalseLiteral4,
1048	else => return error.InvalidLiteral,
1049	},
1050
1051	.FalseLiteral4 => switch (c) {
1052	'e' => {
1053	p.state = p.after_value_state;
1054	p.complete = p.state == .TopLevelEnd;
1055	token.* = Token.False;
1056	},
1057	else => {
1058	return error.InvalidLiteral;
1059	},
1060	},
1061
1062	.NullLiteral1 => switch (c) {
1063	'u' => p.state = .NullLiteral2,
1064	else => return error.InvalidLiteral,
1065	},
1066
1067	.NullLiteral2 => switch (c) {
1068	'l' => p.state = .NullLiteral3,
1069	else => return error.InvalidLiteral,
1070	},
1071
1072	.NullLiteral3 => switch (c) {
1073	'l' => {
1074	p.state = p.after_value_state;
1075	p.complete = p.state == .TopLevelEnd;
1076	token.* = Token.Null;
1077	},
1078	else => {
1079	return error.InvalidLiteral;
1080	},
1081	},
1082	}
1083
1084	return false;
1085	}
1086	};
1087
1088	/// A small wrapper over a StreamingParser for full slices. Returns a stream of json Tokens.
1089	pub const TokenStream = struct {
1090	i: usize,
1091	slice: []const u8,
1092	parser: StreamingParser,
1093	token: ?Token,
1094
1095	pub const Error = StreamingParser.Error \|\| error{UnexpectedEndOfJson};
1096
1097	pub fn init(slice: []const u8) TokenStream {
1098	return TokenStream{
1099	.i = 0,
1100	.slice = slice,
1101	.parser = StreamingParser.init(),
1102	.token = null,
1103	};
1104	}
1105
1106	fn stackUsed(self: *TokenStream) usize {
1107	return self.parser.stack.len + if (self.token != null) @as(usize, 1) else 0;
1108	}
1109
1110	pub fn next(self: *TokenStream) Error!?Token {
1111	if (self.token) \|token\| {
1112	self.token = null;
1113	return token;
1114	}
1115
1116	var t1: ?Token = undefined;
1117	var t2: ?Token = undefined;
1118
1119	while (self.i < self.slice.len) {
1120	try self.parser.feed(self.slice[self.i], &t1, &t2);
1121	self.i += 1;
1122
1123	if (t1) \|token\| {
1124	self.token = t2;
1125	return token;
1126	}
1127	}
1128
1129	// Without this a bare number fails, the streaming parser doesn't know the input ended
1130	try self.parser.feed(' ', &t1, &t2);
1131	self.i += 1;
1132
1133	if (t1) \|token\| {
1134	return token;
1135	} else if (self.parser.complete) {
1136	return null;
1137	} else {
1138	return error.UnexpectedEndOfJson;
1139	}
1140	}
1141	};
1142
1143	/// Validate a JSON string. This does not limit number precision so a decoder may not necessarily
1144	/// be able to decode the string even if this returns true.
1145	pub fn validate(s: []const u8) bool {
1146	var p = StreamingParser.init();
1147
1148	for (s) \|c\| {
1149	var token1: ?Token = undefined;
1150	var token2: ?Token = undefined;
1151
1152	p.feed(c, &token1, &token2) catch {
1153	return false;
1154	};
1155	}
1156
1157	return p.complete;
1158	}
1159
1160	const Allocator = std.mem.Allocator;
1161	const ArenaAllocator = std.heap.ArenaAllocator;
1162	const ArrayList = std.ArrayList;
1163	const StringArrayHashMap = std.StringArrayHashMap;
1164
1165	pub const ValueTree = struct {
1166	arena: *ArenaAllocator,
1167	root: Value,
1168
1169	pub fn deinit(self: *ValueTree) void {
1170	self.arena.deinit();
1171	self.arena.child_allocator.destroy(self.arena);
1172	}
1173	};
1174
1175	pub const ObjectMap = StringArrayHashMap(Value);
1176	pub const Array = ArrayList(Value);
1177
1178	/// Represents a JSON value
1179	/// Currently only supports numbers that fit into i64 or f64.
1180	pub const Value = union(enum) {
1181	Null,
1182	Bool: bool,
1183	Integer: i64,
1184	Float: f64,
1185	NumberString: []const u8,
1186	String: []const u8,
1187	Array: Array,
1188	Object: ObjectMap,
1189
1190	pub fn jsonStringify(
1191	value: @This(),
1192	options: StringifyOptions,
1193	out_stream: anytype,
1194	) @TypeOf(out_stream).Error!void {
1195	switch (value) {
1196	.Null => try stringify(null, options, out_stream),
1197	.Bool => \|inner\| try stringify(inner, options, out_stream),
1198	.Integer => \|inner\| try stringify(inner, options, out_stream),
1199	.Float => \|inner\| try stringify(inner, options, out_stream),
1200	.NumberString => \|inner\| try out_stream.writeAll(inner),
1201	.String => \|inner\| try stringify(inner, options, out_stream),
1202	.Array => \|inner\| try stringify(inner.items, options, out_stream),
1203	.Object => \|inner\| {
1204	try out_stream.writeByte('{');
1205	var field_output = false;
1206	var child_options = options;
1207	if (child_options.whitespace) \|*child_whitespace\| {
1208	child_whitespace.indent_level += 1;
1209	}
1210	var it = inner.iterator();
1211	while (it.next()) \|entry\| {
1212	if (!field_output) {
1213	field_output = true;
1214	} else {
1215	try out_stream.writeByte(',');
1216	}
1217	if (child_options.whitespace) \|child_whitespace\| {
1218	try child_whitespace.outputIndent(out_stream);
1219	}
1220
1221	try stringify(entry.key_ptr.*, options, out_stream);
1222	try out_stream.writeByte(':');
1223	if (child_options.whitespace) \|child_whitespace\| {
1224	if (child_whitespace.separator) {
1225	try out_stream.writeByte(' ');
1226	}
1227	}
1228	try stringify(entry.value_ptr.*, child_options, out_stream);
1229	}
1230	if (field_output) {
1231	if (options.whitespace) \|whitespace\| {
1232	try whitespace.outputIndent(out_stream);
1233	}
1234	}
1235	try out_stream.writeByte('}');
1236	},
1237	}
1238	}
1239
1240	pub fn dump(self: Value) void {
1241	std.debug.getStderrMutex().lock();
1242	defer std.debug.getStderrMutex().unlock();
1243
1244	const stderr = std.io.getStdErr().writer();
1245	std.json.stringify(self, std.json.StringifyOptions{ .whitespace = null }, stderr) catch return;
1246	}
1247	};
1248
1249	/// parse tokens from a stream, returning `false` if they do not decode to `value`
1250	fn parsesTo(comptime T: type, value: T, tokens: *TokenStream, options: ParseOptions) !bool {
1251	// TODO: should be able to write this function to not require an allocator
1252	const tmp = try parse(T, tokens, options);
1253	defer parseFree(T, tmp, options);
1254
1255	return parsedEqual(tmp, value);
1256	}
1257
1258	/// Returns if a value returned by `parse` is deep-equal to another value
1259	fn parsedEqual(a: anytype, b: @TypeOf(a)) bool {
1260	switch (@typeInfo(@TypeOf(a))) {
1261	.Optional => {
1262	if (a == null and b == null) return true;
1263	if (a == null or b == null) return false;
1264	return parsedEqual(a.?, b.?);
1265	},
1266	.Union => \|info\| {
1267	if (info.tag_type) \|UnionTag\| {
1268	const tag_a = std.meta.activeTag(a);
1269	const tag_b = std.meta.activeTag(b);
1270	if (tag_a != tag_b) return false;
1271
1272	inline for (info.fields) \|field_info\| {
1273	if (@field(UnionTag, field_info.name) == tag_a) {
1274	return parsedEqual(@field(a, field_info.name), @field(b, field_info.name));
1275	}
1276	}
1277	return false;
1278	} else {
1279	unreachable;
1280	}
1281	},
1282	.Array => {
1283	for (a, 0..) \|e, i\|
1284	if (!parsedEqual(e, b[i])) return false;
1285	return true;
1286	},
1287	.Struct => \|info\| {
1288	inline for (info.fields) \|field_info\| {
1289	if (!parsedEqual(@field(a, field_info.name), @field(b, field_info.name))) return false;
1290	}
1291	return true;
1292	},
1293	.Pointer => \|ptrInfo\| switch (ptrInfo.size) {
1294	.One => return parsedEqual(a., b.),
1295	.Slice => {
1296	if (a.len != b.len) return false;
1297	for (a, 0..) \|e, i\|
1298	if (!parsedEqual(e, b[i])) return false;
1299	return true;
1300	},
1301	.Many, .C => unreachable,
1302	},
1303	else => return a == b,
1304	}
1305	unreachable;
1306	}
1307
1308	pub const ParseOptions = struct {
1309	allocator: ?Allocator = null,
1310
1311	/// Behaviour when a duplicate field is encountered.
1312	duplicate_field_behavior: enum {
1313	UseFirst,
1314	Error,
1315	UseLast,
1316	} = .Error,
1317
1318	/// If false, finding an unknown field returns an error.
1319	ignore_unknown_fields: bool = false,
1320
1321	allow_trailing_data: bool = false,
1322	};
1323
1324	const SkipValueError = error{UnexpectedJsonDepth} \|\| TokenStream.Error;
1325
1326	fn skipValue(tokens: *TokenStream) SkipValueError!void {
1327	const original_depth = tokens.stackUsed();
1328
1329	// Return an error if no value is found
1330	_ = try tokens.next();
1331	if (tokens.stackUsed() < original_depth) return error.UnexpectedJsonDepth;
1332	if (tokens.stackUsed() == original_depth) return;
1333
1334	while (try tokens.next()) \|_\| {
1335	if (tokens.stackUsed() == original_depth) return;
1336	}
1337	}
1338
1339	fn ParseInternalError(comptime T: type) type {
1340	// `inferred_types` is used to avoid infinite recursion for recursive type definitions.
1341	const inferred_types = [_]type{};
1342	return ParseInternalErrorImpl(T, &inferred_types);
1343	}
1344
1345	fn ParseInternalErrorImpl(comptime T: type, comptime inferred_types: []const type) type {
1346	for (inferred_types) \|ty\| {
1347	if (T == ty) return error{};
1348	}
1349
1350	switch (@typeInfo(T)) {
1351	.Bool => return error{UnexpectedToken},
1352	.Float, .ComptimeFloat => return error{UnexpectedToken} \|\| std.fmt.ParseFloatError,
1353	.Int, .ComptimeInt => {
1354	return error{ UnexpectedToken, InvalidNumber, Overflow } \|\|
1355	std.fmt.ParseIntError \|\| std.fmt.ParseFloatError;
1356	},
1357	.Optional => \|optionalInfo\| {
1358	return ParseInternalErrorImpl(optionalInfo.child, inferred_types ++ [_]type{T});
1359	},
1360	.Enum => return error{ UnexpectedToken, InvalidEnumTag } \|\| std.fmt.ParseIntError \|\|
1361	std.meta.IntToEnumError \|\| std.meta.IntToEnumError,
1362	.Union => \|unionInfo\| {
1363	if (unionInfo.tag_type) \|_\| {
1364	var errors = error{NoUnionMembersMatched};
1365	for (unionInfo.fields) \|u_field\| {
1366	errors = errors \|\| ParseInternalErrorImpl(u_field.type, inferred_types ++ [_]type{T});
1367	}
1368	return errors;
1369	} else {
1370	@compileError("Unable to parse into untagged union '" ++ @typeName(T) ++ "'");
1371	}
1372	},
1373	.Struct => \|structInfo\| {
1374	var errors = error{
1375	DuplicateJSONField,
1376	UnexpectedEndOfJson,
1377	UnexpectedToken,
1378	UnexpectedValue,
1379	UnknownField,
1380	MissingField,
1381	} \|\| SkipValueError \|\| TokenStream.Error;
1382	for (structInfo.fields) \|field\| {
1383	errors = errors \|\| ParseInternalErrorImpl(field.type, inferred_types ++ [_]type{T});
1384	}
1385	return errors;
1386	},
1387	.Array => \|arrayInfo\| {
1388	return error{ UnexpectedEndOfJson, UnexpectedToken, LengthMismatch } \|\| TokenStream.Error \|\|
1389	UnescapeValidStringError \|\|
1390	ParseInternalErrorImpl(arrayInfo.child, inferred_types ++ [_]type{T});
1391	},
1392	.Vector => \|vecInfo\| {
1393	return error{ UnexpectedEndOfJson, UnexpectedToken, LengthMismatch } \|\| TokenStream.Error \|\|
1394	UnescapeValidStringError \|\|
1395	ParseInternalErrorImpl(vecInfo.child, inferred_types ++ [_]type{T});
1396	},
1397	.Pointer => \|ptrInfo\| {
1398	var errors = error{AllocatorRequired} \|\| std.mem.Allocator.Error;
1399	switch (ptrInfo.size) {
1400	.One => {
1401	return errors \|\| ParseInternalErrorImpl(ptrInfo.child, inferred_types ++ [_]type{T});
1402	},
1403	.Slice => {
1404	return errors \|\| error{ UnexpectedEndOfJson, UnexpectedToken } \|\|
1405	ParseInternalErrorImpl(ptrInfo.child, inferred_types ++ [_]type{T}) \|\|
1406	UnescapeValidStringError \|\| TokenStream.Error;
1407	},
1408	else => @compileError("Unable to parse into type '" ++ @typeName(T) ++ "'"),
1409	}
1410	},
1411	else => return error{},
1412	}
1413	unreachable;
1414	}
1415
1416	fn parseInternalArray(
1417	comptime T: type,
1418	comptime Elt: type,
1419	comptime arr_len: usize,
1420	tokens: *TokenStream,
1421	options: ParseOptions,
1422	) ParseInternalError(T)!T {
1423	var r: T = undefined;
1424	var i: usize = 0;
1425	var child_options = options;
1426	child_options.allow_trailing_data = true;
1427	errdefer {
1428	// Without the r.len check `r[i]` is not allowed
1429	if (arr_len > 0) while (true) : (i -= 1) {
1430	parseFree(Elt, r[i], options);
1431	if (i == 0) break;
1432	};
1433	}
1434	if (arr_len > 0) while (i < arr_len) : (i += 1) {
1435	r[i] = try parse(Elt, tokens, child_options);
1436	};
1437	const tok = (try tokens.next()) orelse return error.UnexpectedEndOfJson;
1438	switch (tok) {
1439	.ArrayEnd => {},
1440	else => return error.UnexpectedToken,
1441	}
1442	return r;
1443	}
1444
1445	fn parseInternal(
1446	comptime T: type,
1447	token: Token,
1448	tokens: *TokenStream,
1449	options: ParseOptions,
1450	) ParseInternalError(T)!T {
1451	switch (@typeInfo(T)) {
1452	.Bool => {
1453	return switch (token) {
1454	.True => true,
1455	.False => false,
1456	else => error.UnexpectedToken,
1457	};
1458	},
1459	.Float, .ComptimeFloat => {
1460	switch (token) {
1461	.Number => \|numberToken\| return try std.fmt.parseFloat(T, numberToken.slice(tokens.slice, tokens.i - 1)),
1462	.String => \|stringToken\| return try std.fmt.parseFloat(T, stringToken.slice(tokens.slice, tokens.i - 1)),
1463	else => return error.UnexpectedToken,
1464	}
1465	},
1466	.Int, .ComptimeInt => {
1467	switch (token) {
1468	.Number => \|numberToken\| {
1469	if (numberToken.is_integer)
1470	return try std.fmt.parseInt(T, numberToken.slice(tokens.slice, tokens.i - 1), 10);
1471	const float = try std.fmt.parseFloat(f128, numberToken.slice(tokens.slice, tokens.i - 1));
1472	if (@round(float) != float) return error.InvalidNumber;
1473	if (float > std.math.maxInt(T) or float < std.math.minInt(T)) return error.Overflow;
1474	return @floatToInt(T, float);
1475	},
1476	.String => \|stringToken\| {
1477	return std.fmt.parseInt(T, stringToken.slice(tokens.slice, tokens.i - 1), 10) catch \|err\| {
1478	switch (err) {
1479	error.Overflow => return err,
1480	error.InvalidCharacter => {
1481	const float = try std.fmt.parseFloat(f128, stringToken.slice(tokens.slice, tokens.i - 1));
1482	if (@round(float) != float) return error.InvalidNumber;
1483	if (float > std.math.maxInt(T) or float < std.math.minInt(T)) return error.Overflow;
1484	return @floatToInt(T, float);
1485	},
1486	}
1487	};
1488	},
1489	else => return error.UnexpectedToken,
1490	}
1491	},
1492	.Optional => \|optionalInfo\| {
1493	if (token == .Null) {
1494	return null;
1495	} else {
1496	return try parseInternal(optionalInfo.child, token, tokens, options);
1497	}
1498	},
1499	.Enum => \|enumInfo\| {
1500	switch (token) {
1501	.Number => \|numberToken\| {
1502	if (!numberToken.is_integer) return error.UnexpectedToken;
1503	const n = try std.fmt.parseInt(enumInfo.tag_type, numberToken.slice(tokens.slice, tokens.i - 1), 10);
1504	return try std.meta.intToEnum(T, n);
1505	},
1506	.String => \|stringToken\| {
1507	const source_slice = stringToken.slice(tokens.slice, tokens.i - 1);
1508	switch (stringToken.escapes) {
1509	.None => return std.meta.stringToEnum(T, source_slice) orelse return error.InvalidEnumTag,
1510	.Some => {
1511	inline for (enumInfo.fields) \|field\| {
1512	if (field.name.len == stringToken.decodedLength() and encodesTo(field.name, source_slice)) {
1513	return @field(T, field.name);
1514	}
1515	}
1516	return error.InvalidEnumTag;
1517	},
1518	}
1519	},
1520	else => return error.UnexpectedToken,
1521	}
1522	},
1523	.Union => \|unionInfo\| {
1524	if (unionInfo.tag_type) \|_\| {
1525	// try each of the union fields until we find one that matches
1526	inline for (unionInfo.fields) \|u_field\| {
1527	// take a copy of tokens so we can withhold mutations until success
1528	var tokens_copy = tokens.*;
1529	if (parseInternal(u_field.type, token, &tokens_copy, options)) \|value\| {
1530	tokens.* = tokens_copy;
1531	return @unionInit(T, u_field.name, value);
1532	} else \|err\| {
1533	// Bubble up error.OutOfMemory
1534	// Parsing some types won't have OutOfMemory in their
1535	// error-sets, for the condition to be valid, merge it in.
1536	if (@as(@TypeOf(err) \|\| error{OutOfMemory}, err) == error.OutOfMemory) return err;
1537	// Bubble up AllocatorRequired, as it indicates missing option
1538	if (@as(@TypeOf(err) \|\| error{AllocatorRequired}, err) == error.AllocatorRequired) return err;
1539	// otherwise continue through the `inline for`
1540	}
1541	}
1542	return error.NoUnionMembersMatched;
1543	} else {
1544	@compileError("Unable to parse into untagged union '" ++ @typeName(T) ++ "'");
1545	}
1546	},
1547	.Struct => \|structInfo\| {
1548	if (structInfo.is_tuple) {
1549	switch (token) {
1550	.ArrayBegin => {},
1551	else => return error.UnexpectedToken,
1552	}
1553	var r: T = undefined;
1554	var child_options = options;
1555	child_options.allow_trailing_data = true;
1556	var fields_seen: usize = 0;
1557	errdefer {
1558	inline for (0..structInfo.fields.len) \|i\| {
1559	if (i < fields_seen) {
1560	parseFree(structInfo.fields[i].type, r[i], options);
1561	}
1562	}
1563	}
1564	inline for (0..structInfo.fields.len) \|i\| {
1565	r[i] = try parse(structInfo.fields[i].type, tokens, child_options);
1566	fields_seen = i + 1;
1567	}
1568	const tok = (try tokens.next()) orelse return error.UnexpectedEndOfJson;
1569	switch (tok) {
1570	.ArrayEnd => {},
1571	else => return error.UnexpectedToken,
1572	}
1573	return r;
1574	}
1575
1576	switch (token) {
1577	.ObjectBegin => {},
1578	else => return error.UnexpectedToken,
1579	}
1580	var r: T = undefined;
1581	var fields_seen = [_]bool{false} ** structInfo.fields.len;
1582	errdefer {
1583	inline for (structInfo.fields, 0..) \|field, i\| {
1584	if (fields_seen[i] and !field.is_comptime) {
1585	parseFree(field.type, @field(r, field.name), options);
1586	}
1587	}
1588	}
1589
1590	while (true) {
1591	switch ((try tokens.next()) orelse return error.UnexpectedEndOfJson) {
1592	.ObjectEnd => break,
1593	.String => \|stringToken\| {
1594	const key_source_slice = stringToken.slice(tokens.slice, tokens.i - 1);
1595	var child_options = options;
1596	child_options.allow_trailing_data = true;
1597	var found = false;
1598	inline for (structInfo.fields, 0..) \|field, i\| {
1599	if (switch (stringToken.escapes) {
1600	.None => mem.eql(u8, field.name, key_source_slice),
1601	.Some => (field.name.len == stringToken.decodedLength() and encodesTo(field.name, key_source_slice)),
1602	}) {
1603	if (fields_seen[i]) {
1604	switch (options.duplicate_field_behavior) {
1605	.UseFirst => {
1606	// unconditionally ignore value. for comptime fields, this skips check against default_value
1607	parseFree(field.type, try parse(field.type, tokens, child_options), child_options);
1608	found = true;
1609	break;
1610	},
1611	.Error => return error.DuplicateJSONField,
1612	.UseLast => {
1613	if (!field.is_comptime) {
1614	parseFree(field.type, @field(r, field.name), child_options);
1615	}
1616	fields_seen[i] = false;
1617	},
1618	}
1619	}
1620	if (field.is_comptime) {
1621	if (!try parsesTo(field.type, @ptrCast(align(1) const field.type, field.default_value.?)., tokens, child_options)) {
1622	return error.UnexpectedValue;
1623	}
1624	} else {
1625	@field(r, field.name) = try parse(field.type, tokens, child_options);
1626	}
1627	fields_seen[i] = true;
1628	found = true;
1629	break;
1630	}
1631	}
1632	if (!found) {
1633	if (options.ignore_unknown_fields) {
1634	try skipValue(tokens);
1635	continue;
1636	} else {
1637	return error.UnknownField;
1638	}
1639	}
1640	},
1641	else => return error.UnexpectedToken,
1642	}
1643	}
1644	inline for (structInfo.fields, 0..) \|field, i\| {
1645	if (!fields_seen[i]) {
1646	if (field.default_value) \|default_ptr\| {
1647	if (!field.is_comptime) {
1648	const default = @ptrCast(align(1) const field.type, default_ptr).;
1649	@field(r, field.name) = default;
1650	}
1651	} else {
1652	return error.MissingField;
1653	}
1654	}
1655	}
1656	return r;
1657	},
1658	.Array => \|arrayInfo\| {
1659	switch (token) {
1660	.ArrayBegin => {
1661	const len = @typeInfo(T).Array.len;
1662	return parseInternalArray(T, arrayInfo.child, len, tokens, options);
1663	},
1664	.String => \|stringToken\| {
1665	if (arrayInfo.child != u8) return error.UnexpectedToken;
1666	var r: T = undefined;
1667	const source_slice = stringToken.slice(tokens.slice, tokens.i - 1);
1668	if (r.len != stringToken.decodedLength()) return error.LengthMismatch;
1669	switch (stringToken.escapes) {
1670	.None => @memcpy(r[0..source_slice.len], source_slice),
1671	.Some => try unescapeValidString(&r, source_slice),
1672	}
1673	return r;
1674	},
1675	else => return error.UnexpectedToken,
1676	}
1677	},
1678	.Vector => \|vecInfo\| {
1679	switch (token) {
1680	.ArrayBegin => {
1681	const len = @typeInfo(T).Vector.len;
1682	return parseInternalArray(T, vecInfo.child, len, tokens, options);
1683	},
1684	else => return error.UnexpectedToken,
1685	}
1686	},
1687	.Pointer => \|ptrInfo\| {
1688	const allocator = options.allocator orelse return error.AllocatorRequired;
1689	switch (ptrInfo.size) {
1690	.One => {
1691	const r: *ptrInfo.child = try allocator.create(ptrInfo.child);
1692	errdefer allocator.destroy(r);
1693	r.* = try parseInternal(ptrInfo.child, token, tokens, options);
1694	return r;
1695	},
1696	.Slice => {
1697	switch (token) {
1698	.ArrayBegin => {
1699	var arraylist = std.ArrayList(ptrInfo.child).init(allocator);
1700	errdefer {
1701	while (arraylist.popOrNull()) \|v\| {
1702	parseFree(ptrInfo.child, v, options);
1703	}
1704	arraylist.deinit();
1705	}
1706
1707	while (true) {
1708	const tok = (try tokens.next()) orelse return error.UnexpectedEndOfJson;
1709	switch (tok) {
1710	.ArrayEnd => break,
1711	else => {},
1712	}
1713
1714	try arraylist.ensureUnusedCapacity(1);
1715	const v = try parseInternal(ptrInfo.child, tok, tokens, options);
1716	arraylist.appendAssumeCapacity(v);
1717	}
1718
1719	if (ptrInfo.sentinel) \|some\| {
1720	const sentinel_value = @ptrCast(align(1) const ptrInfo.child, some).;
1721	return try arraylist.toOwnedSliceSentinel(sentinel_value);
1722	}
1723
1724	return try arraylist.toOwnedSlice();
1725	},
1726	.String => \|stringToken\| {
1727	if (ptrInfo.child != u8) return error.UnexpectedToken;
1728	const source_slice = stringToken.slice(tokens.slice, tokens.i - 1);
1729	const len = stringToken.decodedLength();
1730	const output = if (ptrInfo.sentinel) \|sentinel_ptr\|
1731	try allocator.allocSentinel(u8, len, @ptrCast(const u8, sentinel_ptr).)
1732	else
1733	try allocator.alloc(u8, len);
1734	errdefer allocator.free(output);
1735	switch (stringToken.escapes) {
1736	.None => @memcpy(output[0..source_slice.len], source_slice),
1737	.Some => try unescapeValidString(output, source_slice),
1738	}
1739
1740	return output;
1741	},
1742	else => return error.UnexpectedToken,
1743	}
1744	},
1745	else => @compileError("Unable to parse into type '" ++ @typeName(T) ++ "'"),
1746	}
1747	},
1748	else => @compileError("Unable to parse into type '" ++ @typeName(T) ++ "'"),
1749	}
1750	unreachable;
1751	}
1752
1753	pub fn ParseError(comptime T: type) type {
1754	return ParseInternalError(T) \|\| error{UnexpectedEndOfJson} \|\| TokenStream.Error;
1755	}
1756
1757	pub fn parse(comptime T: type, tokens: *TokenStream, options: ParseOptions) ParseError(T)!T {
1758	const token = (try tokens.next()) orelse return error.UnexpectedEndOfJson;
1759	const r = try parseInternal(T, token, tokens, options);
1760	errdefer parseFree(T, r, options);
1761	if (!options.allow_trailing_data) {
1762	if ((try tokens.next()) != null) unreachable;
1763	assert(tokens.i >= tokens.slice.len);
1764	}
1765	return r;
1766	}
1767
1768	/// Releases resources created by `parse`.
1769	/// Should be called with the same type and `ParseOptions` that were passed to `parse`
1770	pub fn parseFree(comptime T: type, value: T, options: ParseOptions) void {
1771	switch (@typeInfo(T)) {
1772	.Bool, .Float, .ComptimeFloat, .Int, .ComptimeInt, .Enum => {},
1773	.Optional => {
1774	if (value) \|v\| {
1775	return parseFree(@TypeOf(v), v, options);
1776	}
1777	},
1778	.Union => \|unionInfo\| {
1779	if (unionInfo.tag_type) \|UnionTagType\| {
1780	inline for (unionInfo.fields) \|u_field\| {
1781	if (value == @field(UnionTagType, u_field.name)) {
1782	parseFree(u_field.type, @field(value, u_field.name), options);
1783	break;
1784	}
1785	}
1786	} else {
1787	unreachable;
1788	}
1789	},
1790	.Struct => \|structInfo\| {
1791	inline for (structInfo.fields) \|field\| {
1792	if (!field.is_comptime) {
1793	var should_free = true;
1794	if (field.default_value) \|default\| {
1795	switch (@typeInfo(field.type)) {
1796	// We must not attempt to free pointers to struct default values
1797	.Pointer => \|fieldPtrInfo\| {
1798	const field_value = @field(value, field.name);
1799	const field_ptr = switch (fieldPtrInfo.size) {
1800	.One => field_value,
1801	.Slice => field_value.ptr,
1802	else => unreachable, // Other pointer types are not parseable
1803	};
1804	const field_addr = @ptrToInt(field_ptr);
1805
1806	const casted_default = @ptrCast(const field.type, @alignCast(@alignOf(field.type), default)).;
1807	const default_ptr = switch (fieldPtrInfo.size) {
1808	.One => casted_default,
1809	.Slice => casted_default.ptr,
1810	else => unreachable, // Other pointer types are not parseable
1811	};
1812	const default_addr = @ptrToInt(default_ptr);
1813
1814	if (field_addr == default_addr) {
1815	should_free = false;
1816	}
1817	},
1818	else => {},
1819	}
1820	}
1821	if (should_free) {
1822	parseFree(field.type, @field(value, field.name), options);
1823	}
1824	}
1825	}
1826	},
1827	.Array => \|arrayInfo\| {
1828	for (value) \|v\| {
1829	parseFree(arrayInfo.child, v, options);
1830	}
1831	},
1832	.Vector => \|vecInfo\| {
1833	var i: usize = 0;
1834	var v_len: usize = @typeInfo(@TypeOf(value)).Vector.len;
1835	while (i < v_len) : (i += 1) {
1836	parseFree(vecInfo.child, value[i], options);
1837	}
1838	},
1839	.Pointer => \|ptrInfo\| {
1840	const allocator = options.allocator orelse unreachable;
1841	switch (ptrInfo.size) {
1842	.One => {
1843	parseFree(ptrInfo.child, value.*, options);
1844	allocator.destroy(value);
1845	},
1846	.Slice => {
1847	for (value) \|v\| {
1848	parseFree(ptrInfo.child, v, options);
1849	}
1850	allocator.free(value);
1851	},
1852	else => unreachable,
1853	}
1854	},
1855	else => unreachable,
1856	}
1857	}
1858
1859	/// A non-stream JSON parser which constructs a tree of Value's.
1860	pub const Parser = struct {
1861	allocator: Allocator,
1862	state: State,
1863	copy_strings: bool,
1864	// Stores parent nodes and un-combined Values.
1865	stack: Array,
1866
1867	const State = enum {
1868	ObjectKey,
1869	ObjectValue,
1870	ArrayValue,
1871	Simple,
1872	};
1873
1874	pub fn init(allocator: Allocator, copy_strings: bool) Parser {
1875	return Parser{
1876	.allocator = allocator,
1877	.state = .Simple,
1878	.copy_strings = copy_strings,
1879	.stack = Array.init(allocator),
1880	};
1881	}
1882
1883	pub fn deinit(p: *Parser) void {
1884	p.stack.deinit();
1885	}
1886
1887	pub fn reset(p: *Parser) void {
1888	p.state = .Simple;
1889	p.stack.shrinkRetainingCapacity(0);
1890	}
1891
1892	pub fn parse(p: *Parser, input: []const u8) !ValueTree {
1893	var s = TokenStream.init(input);
1894
1895	var arena = try p.allocator.create(ArenaAllocator);
1896	errdefer p.allocator.destroy(arena);
1897
1898	arena.* = ArenaAllocator.init(p.allocator);
1899	errdefer arena.deinit();
1900
1901	const allocator = arena.allocator();
1902
1903	while (try s.next()) \|token\| {
1904	try p.transition(allocator, input, s.i - 1, token);
1905	}
1906
1907	debug.assert(p.stack.items.len == 1);
1908
1909	return ValueTree{
1910	.arena = arena,
1911	.root = p.stack.items[0],
1912	};
1913	}
1914
1915	// Even though p.allocator exists, we take an explicit allocator so that allocation state
1916	// can be cleaned up on error correctly during a `parse` on call.
1917	fn transition(p: *Parser, allocator: Allocator, input: []const u8, i: usize, token: Token) !void {
1918	switch (p.state) {
1919	.ObjectKey => switch (token) {
1920	.ObjectEnd => {
1921	if (p.stack.items.len == 1) {
1922	return;
1923	}
1924
1925	var value = p.stack.pop();
1926	try p.pushToParent(&value);
1927	},
1928	.String => \|s\| {
1929	try p.stack.append(try p.parseString(allocator, s, input, i));
1930	p.state = .ObjectValue;
1931	},
1932	else => {
1933	// The streaming parser would return an error eventually.
1934	// To prevent invalid state we return an error now.
1935	// TODO make the streaming parser return an error as soon as it encounters an invalid object key
1936	return error.InvalidLiteral;
1937	},
1938	},
1939	.ObjectValue => {
1940	var object = &p.stack.items[p.stack.items.len - 2].Object;
1941	var key = p.stack.items[p.stack.items.len - 1].String;
1942
1943	switch (token) {
1944	.ObjectBegin => {
1945	try p.stack.append(Value{ .Object = ObjectMap.init(allocator) });
1946	p.state = .ObjectKey;
1947	},
1948	.ArrayBegin => {
1949	try p.stack.append(Value{ .Array = Array.init(allocator) });
1950	p.state = .ArrayValue;
1951	},
1952	.String => \|s\| {
1953	try object.put(key, try p.parseString(allocator, s, input, i));
1954	_ = p.stack.pop();
1955	p.state = .ObjectKey;
1956	},
1957	.Number => \|n\| {
1958	try object.put(key, try p.parseNumber(n, input, i));
1959	_ = p.stack.pop();
1960	p.state = .ObjectKey;
1961	},
1962	.True => {
1963	try object.put(key, Value{ .Bool = true });
1964	_ = p.stack.pop();
1965	p.state = .ObjectKey;
1966	},
1967	.False => {
1968	try object.put(key, Value{ .Bool = false });
1969	_ = p.stack.pop();
1970	p.state = .ObjectKey;
1971	},
1972	.Null => {
1973	try object.put(key, Value.Null);
1974	_ = p.stack.pop();
1975	p.state = .ObjectKey;
1976	},
1977	.ObjectEnd, .ArrayEnd => {
1978	unreachable;
1979	},
1980	}
1981	},
1982	.ArrayValue => {
1983	var array = &p.stack.items[p.stack.items.len - 1].Array;
1984
1985	switch (token) {
1986	.ArrayEnd => {
1987	if (p.stack.items.len == 1) {
1988	return;
1989	}
1990
1991	var value = p.stack.pop();
1992	try p.pushToParent(&value);
1993	},
1994	.ObjectBegin => {
1995	try p.stack.append(Value{ .Object = ObjectMap.init(allocator) });
1996	p.state = .ObjectKey;
1997	},
1998	.ArrayBegin => {
1999	try p.stack.append(Value{ .Array = Array.init(allocator) });
2000	p.state = .ArrayValue;
2001	},
2002	.String => \|s\| {
2003	try array.append(try p.parseString(allocator, s, input, i));
2004	},
2005	.Number => \|n\| {
2006	try array.append(try p.parseNumber(n, input, i));
2007	},
2008	.True => {
2009	try array.append(Value{ .Bool = true });
2010	},
2011	.False => {
2012	try array.append(Value{ .Bool = false });
2013	},
2014	.Null => {
2015	try array.append(Value.Null);
2016	},
2017	.ObjectEnd => {
2018	unreachable;
2019	},
2020	}
2021	},
2022	.Simple => switch (token) {
2023	.ObjectBegin => {
2024	try p.stack.append(Value{ .Object = ObjectMap.init(allocator) });
2025	p.state = .ObjectKey;
2026	},
2027	.ArrayBegin => {
2028	try p.stack.append(Value{ .Array = Array.init(allocator) });
2029	p.state = .ArrayValue;
2030	},
2031	.String => \|s\| {
2032	try p.stack.append(try p.parseString(allocator, s, input, i));
2033	},
2034	.Number => \|n\| {
2035	try p.stack.append(try p.parseNumber(n, input, i));
2036	},
2037	.True => {
2038	try p.stack.append(Value{ .Bool = true });
2039	},
2040	.False => {
2041	try p.stack.append(Value{ .Bool = false });
2042	},
2043	.Null => {
2044	try p.stack.append(Value.Null);
2045	},
2046	.ObjectEnd, .ArrayEnd => {
2047	unreachable;
2048	},
2049	},
2050	}
2051	}
2052
2053	fn pushToParent(p: Parser, value: const Value) !void {
2054	switch (p.stack.items[p.stack.items.len - 1]) {
2055	// Object Parent -> [ ..., object, <key>, value ]
2056	Value.String => \|key\| {
2057	_ = p.stack.pop();
2058
2059	var object = &p.stack.items[p.stack.items.len - 1].Object;
2060	try object.put(key, value.*);
2061	p.state = .ObjectKey;
2062	},
2063	// Array Parent -> [ ..., <array>, value ]
2064	Value.Array => \|*array\| {
2065	try array.append(value.*);
2066	p.state = .ArrayValue;
2067	},
2068	else => {
2069	unreachable;
2070	},
2071	}
2072	}
2073
2074	fn parseString(p: *Parser, allocator: Allocator, s: std.meta.TagPayload(Token, Token.String), input: []const u8, i: usize) !Value {
2075	const slice = s.slice(input, i);
2076	switch (s.escapes) {
2077	.None => return Value{ .String = if (p.copy_strings) try allocator.dupe(u8, slice) else slice },
2078	.Some => {
2079	const output = try allocator.alloc(u8, s.decodedLength());
2080	errdefer allocator.free(output);
2081	try unescapeValidString(output, slice);
2082	return Value{ .String = output };
2083	},
2084	}
2085	}
2086
2087	fn parseNumber(p: *Parser, n: std.meta.TagPayload(Token, Token.Number), input: []const u8, i: usize) !Value {
2088	_ = p;
2089	return if (n.is_integer)
2090	Value{
2091	.Integer = std.fmt.parseInt(i64, n.slice(input, i), 10) catch \|e\| switch (e) {
2092	error.Overflow => return Value{ .NumberString = n.slice(input, i) },
2093	error.InvalidCharacter => \|err\| return err,
2094	},
2095	}
2096	else
2097	Value{ .Float = try std.fmt.parseFloat(f64, n.slice(input, i)) };
2098	}
2099	};
2100
2101	pub const UnescapeValidStringError = error{InvalidUnicodeHexSymbol};
2102
2103	/// Unescape a JSON string
2104	/// Only to be used on strings already validated by the parser
2105	/// (note the unreachable statements and lack of bounds checking)
2106	pub fn unescapeValidString(output: []u8, input: []const u8) UnescapeValidStringError!void {
2107	var inIndex: usize = 0;
2108	var outIndex: usize = 0;
2109
2110	while (inIndex < input.len) {
2111	if (input[inIndex] != '\\') {
2112	// not an escape sequence
2113	output[outIndex] = input[inIndex];
2114	inIndex += 1;
2115	outIndex += 1;
2116	} else if (input[inIndex + 1] != 'u') {
2117	// a simple escape sequence
2118	output[outIndex] = @as(u8, switch (input[inIndex + 1]) {
2119	'\\' => '\\',
2120	'/' => '/',
2121	'n' => '\n',
2122	'r' => '\r',
2123	't' => '\t',
2124	'f' => 12,
2125	'b' => 8,
2126	'"' => '"',
2127	else => unreachable,
2128	});
2129	inIndex += 2;
2130	outIndex += 1;
2131	} else {
2132	// a unicode escape sequence
2133	const firstCodeUnit = std.fmt.parseInt(u16, input[inIndex + 2 .. inIndex + 6], 16) catch unreachable;
2134
2135	// guess optimistically that it's not a surrogate pair
2136	if (std.unicode.utf8Encode(firstCodeUnit, output[outIndex..])) \|byteCount\| {
2137	outIndex += byteCount;
2138	inIndex += 6;
2139	} else \|err\| {
2140	// it might be a surrogate pair
2141	if (err != error.Utf8CannotEncodeSurrogateHalf) {
2142	return error.InvalidUnicodeHexSymbol;
2143	}
2144	// check if a second code unit is present
2145	if (inIndex + 7 >= input.len or input[inIndex + 6] != '\\' or input[inIndex + 7] != 'u') {
2146	return error.InvalidUnicodeHexSymbol;
2147	}
2148
2149	const secondCodeUnit = std.fmt.parseInt(u16, input[inIndex + 8 .. inIndex + 12], 16) catch unreachable;
2150
2151	const utf16le_seq = [2]u16{
2152	mem.nativeToLittle(u16, firstCodeUnit),
2153	mem.nativeToLittle(u16, secondCodeUnit),
2154	};
2155	if (std.unicode.utf16leToUtf8(output[outIndex..], &utf16le_seq)) \|byteCount\| {
2156	outIndex += byteCount;
2157	inIndex += 12;
2158	} else \|_\| {
2159	return error.InvalidUnicodeHexSymbol;
2160	}
2161	}
2162	}
2163	}
2164	assert(outIndex == output.len);
2165	}
2166
2167	pub const StringifyOptions = struct {
2168	pub const Whitespace = struct {
2169	/// How many indentation levels deep are we?
2170	indent_level: usize = 0,
2171
2172	/// What character(s) should be used for indentation?
2173	indent: union(enum) {
2174	Space: u8,
2175	Tab: void,
2176	None: void,
2177	} = .{ .Space = 4 },
2178
2179	/// After a colon, should whitespace be inserted?
2180	separator: bool = true,
2181
2182	pub fn outputIndent(
2183	whitespace: @This(),
2184	out_stream: anytype,
2185	) @TypeOf(out_stream).Error!void {
2186	var char: u8 = undefined;
2187	var n_chars: usize = undefined;
2188	switch (whitespace.indent) {
2189	.Space => \|n_spaces\| {
2190	char = ' ';
2191	n_chars = n_spaces;
2192	},
2193	.Tab => {
2194	char = '\t';
2195	n_chars = 1;
2196	},
2197	.None => return,
2198	}
2199	try out_stream.writeByte('\n');
2200	n_chars *= whitespace.indent_level;
2201	try out_stream.writeByteNTimes(char, n_chars);
2202	}
2203	};
2204
2205	/// Controls the whitespace emitted
2206	whitespace: ?Whitespace = null,
2207
2208	/// Should optional fields with null value be written?
2209	emit_null_optional_fields: bool = true,
2210
2211	string: StringOptions = StringOptions{ .String = .{} },
2212
2213	/// Should []u8 be serialised as a string? or an array?
2214	pub const StringOptions = union(enum) {
2215	Array,
2216	String: StringOutputOptions,
2217
2218	/// String output options
2219	const StringOutputOptions = struct {
2220	/// Should '/' be escaped in strings?
2221	escape_solidus: bool = false,
2222
2223	/// Should unicode characters be escaped in strings?
2224	escape_unicode: bool = false,
2225	};
2226	};
2227	};
2228
2229	fn outputUnicodeEscape(
2230	codepoint: u21,
2231	out_stream: anytype,
2232	) !void {
2233	if (codepoint <= 0xFFFF) {
2234	// If the character is in the Basic Multilingual Plane (U+0000 through U+FFFF),
2235	// then it may be represented as a six-character sequence: a reverse solidus, followed
2236	// by the lowercase letter u, followed by four hexadecimal digits that encode the character's code point.
2237	try out_stream.writeAll("\\u");
2238	try std.fmt.formatIntValue(codepoint, "x", std.fmt.FormatOptions{ .width = 4, .fill = '0' }, out_stream);
2239	} else {
2240	assert(codepoint <= 0x10FFFF);
2241	// To escape an extended character that is not in the Basic Multilingual Plane,
2242	// the character is represented as a 12-character sequence, encoding the UTF-16 surrogate pair.
2243	const high = @intCast(u16, (codepoint - 0x10000) >> 10) + 0xD800;
2244	const low = @intCast(u16, codepoint & 0x3FF) + 0xDC00;
2245	try out_stream.writeAll("\\u");
2246	try std.fmt.formatIntValue(high, "x", std.fmt.FormatOptions{ .width = 4, .fill = '0' }, out_stream);
2247	try out_stream.writeAll("\\u");
2248	try std.fmt.formatIntValue(low, "x", std.fmt.FormatOptions{ .width = 4, .fill = '0' }, out_stream);
2249	}
2250	}
2251
2252	/// Write `string` to `writer` as a JSON encoded string.
2253	pub fn encodeJsonString(string: []const u8, options: StringifyOptions, writer: anytype) !void {
2254	try writer.writeByte('\"');
2255	try encodeJsonStringChars(string, options, writer);
2256	try writer.writeByte('\"');
2257	}
2258
2259	/// Write `chars` to `writer` as JSON encoded string characters.
2260	pub fn encodeJsonStringChars(chars: []const u8, options: StringifyOptions, writer: anytype) !void {
2261	var i: usize = 0;
2262	while (i < chars.len) : (i += 1) {
2263	switch (chars[i]) {
2264	// normal ascii character
2265	0x20...0x21, 0x23...0x2E, 0x30...0x5B, 0x5D...0x7F => \|c\| try writer.writeByte(c),
2266	// only 2 characters that must be escaped
2267	'\\' => try writer.writeAll("\\\\"),
2268	'\"' => try writer.writeAll("\\\""),
2269	// solidus is optional to escape
2270	'/' => {
2271	if (options.string.String.escape_solidus) {
2272	try writer.writeAll("\\/");
2273	} else {
2274	try writer.writeByte('/');
2275	}
2276	},
2277	// control characters with short escapes
2278	// TODO: option to switch between unicode and 'short' forms?
2279	0x8 => try writer.writeAll("\\b"),
2280	0xC => try writer.writeAll("\\f"),
2281	'\n' => try writer.writeAll("\\n"),
2282	'\r' => try writer.writeAll("\\r"),
2283	'\t' => try writer.writeAll("\\t"),
2284	else => {
2285	const ulen = std.unicode.utf8ByteSequenceLength(chars[i]) catch unreachable;
2286	// control characters (only things left with 1 byte length) should always be printed as unicode escapes
2287	if (ulen == 1 or options.string.String.escape_unicode) {
2288	const codepoint = std.unicode.utf8Decode(chars[i..][0..ulen]) catch unreachable;
2289	try outputUnicodeEscape(codepoint, writer);
2290	} else {
2291	try writer.writeAll(chars[i..][0..ulen]);
2292	}
2293	i += ulen - 1;
2294	},
2295	}
2296	}
2297	}
2298
2299	pub fn stringify(
2300	value: anytype,
2301	options: StringifyOptions,
2302	out_stream: anytype,
2303	) !void {
2304	const T = @TypeOf(value);
2305	switch (@typeInfo(T)) {
2306	.Float, .ComptimeFloat => {
2307	return std.fmt.formatFloatScientific(value, std.fmt.FormatOptions{}, out_stream);
2308	},
2309	.Int, .ComptimeInt => {
2310	return std.fmt.formatIntValue(value, "", std.fmt.FormatOptions{}, out_stream);
2311	},
2312	.Bool => {
2313	return out_stream.writeAll(if (value) "true" else "false");
2314	},
2315	.Null => {
2316	return out_stream.writeAll("null");
2317	},
2318	.Optional => {
2319	if (value) \|payload\| {
2320	return try stringify(payload, options, out_stream);
2321	} else {
2322	return try stringify(null, options, out_stream);
2323	}
2324	},
2325	.Enum => {
2326	if (comptime std.meta.trait.hasFn("jsonStringify")(T)) {
2327	return value.jsonStringify(options, out_stream);
2328	}
2329
2330	@compileError("Unable to stringify enum '" ++ @typeName(T) ++ "'");
2331	},
2332	.Union => {
2333	if (comptime std.meta.trait.hasFn("jsonStringify")(T)) {
2334	return value.jsonStringify(options, out_stream);
2335	}
2336
2337	const info = @typeInfo(T).Union;
2338	if (info.tag_type) \|UnionTagType\| {
2339	inline for (info.fields) \|u_field\| {
2340	if (value == @field(UnionTagType, u_field.name)) {
2341	return try stringify(@field(value, u_field.name), options, out_stream);
2342	}
2343	}
2344	} else {
2345	@compileError("Unable to stringify untagged union '" ++ @typeName(T) ++ "'");
2346	}
2347	},
2348	.Struct => \|S\| {
2349	if (comptime std.meta.trait.hasFn("jsonStringify")(T)) {
2350	return value.jsonStringify(options, out_stream);
2351	}
2352
2353	try out_stream.writeByte(if (S.is_tuple) '[' else '{');
2354	var field_output = false;
2355	var child_options = options;
2356	if (child_options.whitespace) \|*child_whitespace\| {
2357	child_whitespace.indent_level += 1;
2358	}
2359	inline for (S.fields) \|Field\| {
2360	// don't include void fields
2361	if (Field.type == void) continue;
2362
2363	var emit_field = true;
2364
2365	// don't include optional fields that are null when emit_null_optional_fields is set to false
2366	if (@typeInfo(Field.type) == .Optional) {
2367	if (options.emit_null_optional_fields == false) {
2368	if (@field(value, Field.name) == null) {
2369	emit_field = false;
2370	}
2371	}
2372	}
2373
2374	if (emit_field) {
2375	if (!field_output) {
2376	field_output = true;
2377	} else {
2378	try out_stream.writeByte(',');
2379	}
2380	if (child_options.whitespace) \|child_whitespace\| {
2381	try child_whitespace.outputIndent(out_stream);
2382	}
2383	if (!S.is_tuple) {
2384	try encodeJsonString(Field.name, options, out_stream);
2385	try out_stream.writeByte(':');
2386	if (child_options.whitespace) \|child_whitespace\| {
2387	if (child_whitespace.separator) {
2388	try out_stream.writeByte(' ');
2389	}
2390	}
2391	}
2392	try stringify(@field(value, Field.name), child_options, out_stream);
2393	}
2394	}
2395	if (field_output) {
2396	if (options.whitespace) \|whitespace\| {
2397	try whitespace.outputIndent(out_stream);
2398	}
2399	}
2400	try out_stream.writeByte(if (S.is_tuple) ']' else '}');
2401	return;
2402	},
2403	.ErrorSet => return stringify(@as([]const u8, @errorName(value)), options, out_stream),
2404	.Pointer => \|ptr_info\| switch (ptr_info.size) {
2405	.One => switch (@typeInfo(ptr_info.child)) {
2406	.Array => {
2407	const Slice = []const std.meta.Elem(ptr_info.child);
2408	return stringify(@as(Slice, value), options, out_stream);
2409	},
2410	else => {
2411	// TODO: avoid loops?
2412	return stringify(value.*, options, out_stream);
2413	},
2414	},
2415	.Many, .Slice => {
2416	if (ptr_info.size == .Many and ptr_info.sentinel == null)
2417	@compileError("unable to stringify type '" ++ @typeName(T) ++ "' without sentinel");
2418	const slice = if (ptr_info.size == .Many) mem.span(value) else value;
2419
2420	if (ptr_info.child == u8 and options.string == .String and std.unicode.utf8ValidateSlice(slice)) {
2421	try encodeJsonString(slice, options, out_stream);
2422	return;
2423	}
2424
2425	try out_stream.writeByte('[');
2426	var child_options = options;
2427	if (child_options.whitespace) \|*whitespace\| {
2428	whitespace.indent_level += 1;
2429	}
2430	for (slice, 0..) \|x, i\| {
2431	if (i != 0) {
2432	try out_stream.writeByte(',');
2433	}
2434	if (child_options.whitespace) \|child_whitespace\| {
2435	try child_whitespace.outputIndent(out_stream);
2436	}
2437	try stringify(x, child_options, out_stream);
2438	}
2439	if (slice.len != 0) {
2440	if (options.whitespace) \|whitespace\| {
2441	try whitespace.outputIndent(out_stream);
2442	}
2443	}
2444	try out_stream.writeByte(']');
2445	return;
2446	},
2447	else => @compileError("Unable to stringify type '" ++ @typeName(T) ++ "'"),
2448	},
2449	.Array => return stringify(&value, options, out_stream),
2450	.Vector => \|info\| {
2451	const array: [info.len]info.child = value;
2452	return stringify(&array, options, out_stream);
2453	},
2454	else => @compileError("Unable to stringify type '" ++ @typeName(T) ++ "'"),
2455	}
2456	unreachable;
2457	}
2458
2459	// Same as `stringify` but accepts an Allocator and stores result in dynamically allocated memory instead of using a Writer.
2460	// Caller owns returned memory.
2461	pub fn stringifyAlloc(allocator: std.mem.Allocator, value: anytype, options: StringifyOptions) ![]const u8 {
2462	var list = std.ArrayList(u8).init(allocator);
2463	errdefer list.deinit();
2464	try stringify(value, options, list.writer());
2465	return list.toOwnedSlice();
2466	}
2467		50
2468	test {	51	test {
2469	_ = @import("json/test.zig");	52	_ = @import("json/test.zig");
		53	_ = @import("json/scanner.zig");
2470	_ = @import("json/write_stream.zig");	54	_ = @import("json/write_stream.zig");
2471	}	55	_ = @import("json/dynamic.zig");
2472		56	_ = @import("json/static.zig");
2473	test "stringify null optional fields" {	57	_ = @import("json/stringify.zig");
2474	const MyStruct = struct {	58	_ = @import("json/JSONTestSuite_test.zig");
2475	optional: ?[]const u8 = null,
2476	required: []const u8 = "something",
2477	another_optional: ?[]const u8 = null,
2478	another_required: []const u8 = "something else",
2479	};
2480	try teststringify(
2481	\\{"optional":null,"required":"something","another_optional":null,"another_required":"something else"}
2482	,
2483	MyStruct{},
2484	StringifyOptions{},
2485	);
2486	try teststringify(
2487	\\{"required":"something","another_required":"something else"}
2488	,
2489	MyStruct{},
2490	StringifyOptions{ .emit_null_optional_fields = false },
2491	);
2492
2493	var ts = TokenStream.init(
2494	\\{"required":"something","another_required":"something else"}
2495	);
2496	try std.testing.expect(try parsesTo(MyStruct, MyStruct{}, &ts, .{
2497	.allocator = std.testing.allocator,
2498	}));
2499	}
2500
2501	test "skipValue" {
2502	var ts = TokenStream.init("false");
2503	try skipValue(&ts);
2504	ts = TokenStream.init("true");
2505	try skipValue(&ts);
2506	ts = TokenStream.init("null");
2507	try skipValue(&ts);
2508	ts = TokenStream.init("42");
2509	try skipValue(&ts);
2510	ts = TokenStream.init("42.0");
2511	try skipValue(&ts);
2512	ts = TokenStream.init("\"foo\"");
2513	try skipValue(&ts);
2514	ts = TokenStream.init("[101, 111, 121]");
2515	try skipValue(&ts);
2516	ts = TokenStream.init("{}");
2517	try skipValue(&ts);
2518	ts = TokenStream.init("{\"foo\": \"bar\"}");
2519	try skipValue(&ts);
2520
2521	{ // An absurd number of nestings
2522	const nestings = StreamingParser.default_max_nestings + 1;
2523
2524	ts = TokenStream.init("[" nestings ++ "]" nestings);
2525	try testing.expectError(error.TooManyNestedItems, skipValue(&ts));
2526	}
2527
2528	{ // Would a number token cause problems in a deeply-nested array?
2529	const nestings = StreamingParser.default_max_nestings;
2530	const deeply_nested_array = "[" nestings ++ "0.118, 999, 881.99, 911.9, 725, 3" ++ "]" nestings;
2531
2532	ts = TokenStream.init(deeply_nested_array);
2533	try skipValue(&ts);
2534
2535	ts = TokenStream.init("[" ++ deeply_nested_array ++ "]");
2536	try testing.expectError(error.TooManyNestedItems, skipValue(&ts));
2537	}
2538
2539	// Mismatched brace/square bracket
2540	ts = TokenStream.init("[102, 111, 111}");
2541	try testing.expectError(error.UnexpectedClosingBrace, skipValue(&ts));
2542
2543	{ // should fail if no value found (e.g. immediate close of object)
2544	var empty_object = TokenStream.init("{}");
2545	assert(.ObjectBegin == (try empty_object.next()).?);
2546	try testing.expectError(error.UnexpectedJsonDepth, skipValue(&empty_object));
2547
2548	var empty_array = TokenStream.init("[]");
2549	assert(.ArrayBegin == (try empty_array.next()).?);
2550	try testing.expectError(error.UnexpectedJsonDepth, skipValue(&empty_array));
2551	}
2552	}
2553
2554	test "stringify basic types" {
2555	try teststringify("false", false, StringifyOptions{});
2556	try teststringify("true", true, StringifyOptions{});
2557	try teststringify("null", @as(?u8, null), StringifyOptions{});
2558	try teststringify("null", @as(?*u32, null), StringifyOptions{});
2559	try teststringify("42", 42, StringifyOptions{});
2560	try teststringify("4.2e+01", 42.0, StringifyOptions{});
2561	try teststringify("42", @as(u8, 42), StringifyOptions{});
2562	try teststringify("42", @as(u128, 42), StringifyOptions{});
2563	try teststringify("4.2e+01", @as(f32, 42), StringifyOptions{});
2564	try teststringify("4.2e+01", @as(f64, 42), StringifyOptions{});
2565	try teststringify("\"ItBroke\"", @as(anyerror, error.ItBroke), StringifyOptions{});
2566	}
2567
2568	test "stringify string" {
2569	try teststringify("\"hello\"", "hello", StringifyOptions{});
2570	try teststringify("\"with\\nescapes\\r\"", "with\nescapes\r", StringifyOptions{});
2571	try teststringify("\"with\\nescapes\\r\"", "with\nescapes\r", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
2572	try teststringify("\"with unicode\\u0001\"", "with unicode\u{1}", StringifyOptions{});
2573	try teststringify("\"with unicode\\u0001\"", "with unicode\u{1}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
2574	try teststringify("\"with unicode\u{80}\"", "with unicode\u{80}", StringifyOptions{});
2575	try teststringify("\"with unicode\\u0080\"", "with unicode\u{80}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
2576	try teststringify("\"with unicode\u{FF}\"", "with unicode\u{FF}", StringifyOptions{});
2577	try teststringify("\"with unicode\\u00ff\"", "with unicode\u{FF}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
2578	try teststringify("\"with unicode\u{100}\"", "with unicode\u{100}", StringifyOptions{});
2579	try teststringify("\"with unicode\\u0100\"", "with unicode\u{100}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
2580	try teststringify("\"with unicode\u{800}\"", "with unicode\u{800}", StringifyOptions{});
2581	try teststringify("\"with unicode\\u0800\"", "with unicode\u{800}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
2582	try teststringify("\"with unicode\u{8000}\"", "with unicode\u{8000}", StringifyOptions{});
2583	try teststringify("\"with unicode\\u8000\"", "with unicode\u{8000}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
2584	try teststringify("\"with unicode\u{D799}\"", "with unicode\u{D799}", StringifyOptions{});
2585	try teststringify("\"with unicode\\ud799\"", "with unicode\u{D799}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
2586	try teststringify("\"with unicode\u{10000}\"", "with unicode\u{10000}", StringifyOptions{});
2587	try teststringify("\"with unicode\\ud800\\udc00\"", "with unicode\u{10000}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
2588	try teststringify("\"with unicode\u{10FFFF}\"", "with unicode\u{10FFFF}", StringifyOptions{});
2589	try teststringify("\"with unicode\\udbff\\udfff\"", "with unicode\u{10FFFF}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
2590	try teststringify("\"/\"", "/", StringifyOptions{});
2591	try teststringify("\"\\/\"", "/", StringifyOptions{ .string = .{ .String = .{ .escape_solidus = true } } });
2592	}
2593
2594	test "stringify many-item sentinel-terminated string" {
2595	try teststringify("\"hello\"", @as([*:0]const u8, "hello"), StringifyOptions{});
2596	try teststringify("\"with\\nescapes\\r\"", @as([*:0]const u8, "with\nescapes\r"), StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
2597	try teststringify("\"with unicode\\u0001\"", @as([*:0]const u8, "with unicode\u{1}"), StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
2598	}
2599
2600	test "stringify tagged unions" {
2601	try teststringify("42", union(enum) {
2602	Foo: u32,
2603	Bar: bool,
2604	}{ .Foo = 42 }, StringifyOptions{});
2605	}
2606
2607	test "stringify struct" {
2608	try teststringify("{\"foo\":42}", struct {
2609	foo: u32,
2610	}{ .foo = 42 }, StringifyOptions{});
2611	}
2612
2613	test "stringify struct with string as array" {
2614	try teststringify("{\"foo\":\"bar\"}", .{ .foo = "bar" }, StringifyOptions{});
2615	try teststringify("{\"foo\":[98,97,114]}", .{ .foo = "bar" }, StringifyOptions{ .string = .Array });
2616	}
2617
2618	test "stringify struct with indentation" {
2619	try teststringify(
2620	\\{
2621	\\ "foo": 42,
2622	\\ "bar": [
2623	\\ 1,
2624	\\ 2,
2625	\\ 3
2626	\\ ]
2627	\\}
2628	,
2629	struct {
2630	foo: u32,
2631	bar: [3]u32,
2632	}{
2633	.foo = 42,
2634	.bar = .{ 1, 2, 3 },
2635	},
2636	StringifyOptions{
2637	.whitespace = .{},
2638	},
2639	);
2640	try teststringify(
2641	"{\n\t\"foo\":42,\n\t\"bar\":[\n\t\t1,\n\t\t2,\n\t\t3\n\t]\n}",
2642	struct {
2643	foo: u32,
2644	bar: [3]u32,
2645	}{
2646	.foo = 42,
2647	.bar = .{ 1, 2, 3 },
2648	},
2649	StringifyOptions{
2650	.whitespace = .{
2651	.indent = .Tab,
2652	.separator = false,
2653	},
2654	},
2655	);
2656	try teststringify(
2657	\\{"foo":42,"bar":[1,2,3]}
2658	,
2659	struct {
2660	foo: u32,
2661	bar: [3]u32,
2662	}{
2663	.foo = 42,
2664	.bar = .{ 1, 2, 3 },
2665	},
2666	StringifyOptions{
2667	.whitespace = .{
2668	.indent = .None,
2669	.separator = false,
2670	},
2671	},
2672	);
2673	}
2674
2675	test "stringify struct with void field" {
2676	try teststringify("{\"foo\":42}", struct {
2677	foo: u32,
2678	bar: void = {},
2679	}{ .foo = 42 }, StringifyOptions{});
2680	}
2681
2682	test "stringify array of structs" {
2683	const MyStruct = struct {
2684	foo: u32,
2685	};
2686	try teststringify("[{\"foo\":42},{\"foo\":100},{\"foo\":1000}]", [_]MyStruct{
2687	MyStruct{ .foo = 42 },
2688	MyStruct{ .foo = 100 },
2689	MyStruct{ .foo = 1000 },
2690	}, StringifyOptions{});
2691	}
2692
2693	test "stringify struct with custom stringifier" {
2694	try teststringify("[\"something special\",42]", struct {
2695	foo: u32,
2696	const Self = @This();
2697	pub fn jsonStringify(
2698	value: Self,
2699	options: StringifyOptions,
2700	out_stream: anytype,
2701	) !void {
2702	_ = value;
2703	try out_stream.writeAll("[\"something special\",");
2704	try stringify(42, options, out_stream);
2705	try out_stream.writeByte(']');
2706	}
2707	}{ .foo = 42 }, StringifyOptions{});
2708	}
2709
2710	test "stringify vector" {
2711	try teststringify("[1,1]", @splat(2, @as(u32, 1)), StringifyOptions{});
2712	}
2713
2714	test "stringify tuple" {
2715	try teststringify("[\"foo\",42]", std.meta.Tuple(&.{ []const u8, usize }){ "foo", 42 }, StringifyOptions{});
2716	}
2717
2718	fn teststringify(expected: []const u8, value: anytype, options: StringifyOptions) !void {
2719	const ValidationWriter = struct {
2720	const Self = @This();
2721	pub const Writer = std.io.Writer(*Self, Error, write);
2722	pub const Error = error{
2723	TooMuchData,
2724	DifferentData,
2725	};
2726
2727	expected_remaining: []const u8,
2728
2729	fn init(exp: []const u8) Self {
2730	return .{ .expected_remaining = exp };
2731	}
2732
2733	pub fn writer(self: *Self) Writer {
2734	return .{ .context = self };
2735	}
2736
2737	fn write(self: *Self, bytes: []const u8) Error!usize {
2738	if (self.expected_remaining.len < bytes.len) {
2739	std.debug.print(
2740	\\====== expected this output: =========
2741	\\{s}
2742	\\======== instead found this: =========
2743	\\{s}
2744	\\======================================
2745	, .{
2746	self.expected_remaining,
2747	bytes,
2748	});
2749	return error.TooMuchData;
2750	}
2751	if (!mem.eql(u8, self.expected_remaining[0..bytes.len], bytes)) {
2752	std.debug.print(
2753	\\====== expected this output: =========
2754	\\{s}
2755	\\======== instead found this: =========
2756	\\{s}
2757	\\======================================
2758	, .{
2759	self.expected_remaining[0..bytes.len],
2760	bytes,
2761	});
2762	return error.DifferentData;
2763	}
2764	self.expected_remaining = self.expected_remaining[bytes.len..];
2765	return bytes.len;
2766	}
2767	};
2768
2769	var vos = ValidationWriter.init(expected);
2770	try stringify(value, options, vos.writer());
2771	if (vos.expected_remaining.len > 0) return error.NotEnoughData;
2772	}
2773
2774	test "encodesTo" {
2775	// same
2776	try testing.expectEqual(true, encodesTo("false", "false"));
2777	// totally different
2778	try testing.expectEqual(false, encodesTo("false", "true"));
2779	// different lengths
2780	try testing.expectEqual(false, encodesTo("false", "other"));
2781	// with escape
2782	try testing.expectEqual(true, encodesTo("\\", "\\\\"));
2783	try testing.expectEqual(true, encodesTo("with\nescape", "with\\nescape"));
2784	// with unicode
2785	try testing.expectEqual(true, encodesTo("ą", "\\u0105"));
2786	try testing.expectEqual(true, encodesTo("😂", "\\ud83d\\ude02"));
2787	try testing.expectEqual(true, encodesTo("withąunicode😂", "with\\u0105unicode\\ud83d\\ude02"));
2788	}
2789
2790	test "deserializing string with escape sequence into sentinel slice" {
2791	const json = "\"\\n\"";
2792	var token_stream = std.json.TokenStream.init(json);
2793	const options = ParseOptions{ .allocator = std.testing.allocator };
2794
2795	// Pre-fix, this line would panic:
2796	const result = try std.json.parse([:0]const u8, &token_stream, options);
2797	defer std.json.parseFree([:0]const u8, result, options);
2798
2799	// Double-check that we're getting the right result
2800	try testing.expect(mem.eql(u8, result, "\n"));
2801	}
2802
2803	test "stringify struct with custom stringify that returns a custom error" {
2804	var ret = std.json.stringify(struct {
2805	field: Field = .{},
2806
2807	pub const Field = struct {
2808	field: ?[]*Field = null,
2809
2810	const Self = @This();
2811	pub fn jsonStringify(_: Self, _: StringifyOptions, _: anytype) error{CustomError}!void {
2812	return error.CustomError;
2813	}
2814	};
2815	}{}, StringifyOptions{}, std.io.null_writer);
2816
2817	try std.testing.expectError(error.CustomError, ret);
2818	}	59	}

lib/std/json/JSONTestSuite_test.zig created+960

...	@@ -0,0 +1,960 @@
1	// This file was generated by _generate_JSONTestSuite.zig
2	// These test cases are sourced from: https://github.com/nst/JSONTestSuite
3	const ok = @import("./test.zig").ok;
4	const err = @import("./test.zig").err;
5	const any = @import("./test.zig").any;
6
7	test "i_number_double_huge_neg_exp.json" {
8	try any("[123.456e-789]");
9	}
10	test "i_number_huge_exp.json" {
11	try any("[0.4e00669999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999969999999006]");
12	}
13	test "i_number_neg_int_huge_exp.json" {
14	try any("[-1e+9999]");
15	}
16	test "i_number_pos_double_huge_exp.json" {
17	try any("[1.5e+9999]");
18	}
19	test "i_number_real_neg_overflow.json" {
20	try any("[-123123e100000]");
21	}
22	test "i_number_real_pos_overflow.json" {
23	try any("[123123e100000]");
24	}
25	test "i_number_real_underflow.json" {
26	try any("[123e-10000000]");
27	}
28	test "i_number_too_big_neg_int.json" {
29	try any("[-123123123123123123123123123123]");
30	}
31	test "i_number_too_big_pos_int.json" {
32	try any("[100000000000000000000]");
33	}
34	test "i_number_very_big_negative_int.json" {
35	try any("[-237462374673276894279832749832423479823246327846]");
36	}
37	test "i_object_key_lone_2nd_surrogate.json" {
38	try any("{\"\\uDFAA\":0}");
39	}
40	test "i_string_1st_surrogate_but_2nd_missing.json" {
41	try any("[\"\\uDADA\"]");
42	}
43	test "i_string_1st_valid_surrogate_2nd_invalid.json" {
44	try any("[\"\\uD888\\u1234\"]");
45	}
46	test "i_string_UTF-16LE_with_BOM.json" {
47	try any("\xff\xfe[\x00\"\x00\xe9\x00\"\x00]\x00");
48	}
49	test "i_string_UTF-8_invalid_sequence.json" {
50	try any("[\"\xe6\x97\xa5\xd1\x88\xfa\"]");
51	}
52	test "i_string_UTF8_surrogate_U+D800.json" {
53	try any("[\"\xed\xa0\x80\"]");
54	}
55	test "i_string_incomplete_surrogate_and_escape_valid.json" {
56	try any("[\"\\uD800\\n\"]");
57	}
58	test "i_string_incomplete_surrogate_pair.json" {
59	try any("[\"\\uDd1ea\"]");
60	}
61	test "i_string_incomplete_surrogates_escape_valid.json" {
62	try any("[\"\\uD800\\uD800\\n\"]");
63	}
64	test "i_string_invalid_lonely_surrogate.json" {
65	try any("[\"\\ud800\"]");
66	}
67	test "i_string_invalid_surrogate.json" {
68	try any("[\"\\ud800abc\"]");
69	}
70	test "i_string_invalid_utf-8.json" {
71	try any("[\"\xff\"]");
72	}
73	test "i_string_inverted_surrogates_U+1D11E.json" {
74	try any("[\"\\uDd1e\\uD834\"]");
75	}
76	test "i_string_iso_latin_1.json" {
77	try any("[\"\xe9\"]");
78	}
79	test "i_string_lone_second_surrogate.json" {
80	try any("[\"\\uDFAA\"]");
81	}
82	test "i_string_lone_utf8_continuation_byte.json" {
83	try any("[\"\x81\"]");
84	}
85	test "i_string_not_in_unicode_range.json" {
86	try any("[\"\xf4\xbf\xbf\xbf\"]");
87	}
88	test "i_string_overlong_sequence_2_bytes.json" {
89	try any("[\"\xc0\xaf\"]");
90	}
91	test "i_string_overlong_sequence_6_bytes.json" {
92	try any("[\"\xfc\x83\xbf\xbf\xbf\xbf\"]");
93	}
94	test "i_string_overlong_sequence_6_bytes_null.json" {
95	try any("[\"\xfc\x80\x80\x80\x80\x80\"]");
96	}
97	test "i_string_truncated-utf-8.json" {
98	try any("[\"\xe0\xff\"]");
99	}
100	test "i_string_utf16BE_no_BOM.json" {
101	try any("\x00[\x00\"\x00\xe9\x00\"\x00]");
102	}
103	test "i_string_utf16LE_no_BOM.json" {
104	try any("[\x00\"\x00\xe9\x00\"\x00]\x00");
105	}
106	test "i_structure_500_nested_arrays.json" {
107	try any("[" 500 ++ "]" 500);
108	}
109	test "i_structure_UTF-8_BOM_empty_object.json" {
110	try any("\xef\xbb\xbf{}");
111	}
112	test "n_array_1_true_without_comma.json" {
113	try err("[1 true]");
114	}
115	test "n_array_a_invalid_utf8.json" {
116	try err("[a\xe5]");
117	}
118	test "n_array_colon_instead_of_comma.json" {
119	try err("[\"\": 1]");
120	}
121	test "n_array_comma_after_close.json" {
122	try err("[\"\"],");
123	}
124	test "n_array_comma_and_number.json" {
125	try err("[,1]");
126	}
127	test "n_array_double_comma.json" {
128	try err("[1,,2]");
129	}
130	test "n_array_double_extra_comma.json" {
131	try err("[\"x\",,]");
132	}
133	test "n_array_extra_close.json" {
134	try err("[\"x\"]]");
135	}
136	test "n_array_extra_comma.json" {
137	try err("[\"\",]");
138	}
139	test "n_array_incomplete.json" {
140	try err("[\"x\"");
141	}
142	test "n_array_incomplete_invalid_value.json" {
143	try err("[x");
144	}
145	test "n_array_inner_array_no_comma.json" {
146	try err("[3[4]]");
147	}
148	test "n_array_invalid_utf8.json" {
149	try err("[\xff]");
150	}
151	test "n_array_items_separated_by_semicolon.json" {
152	try err("[1:2]");
153	}
154	test "n_array_just_comma.json" {
155	try err("[,]");
156	}
157	test "n_array_just_minus.json" {
158	try err("[-]");
159	}
160	test "n_array_missing_value.json" {
161	try err("[ , \"\"]");
162	}
163	test "n_array_newlines_unclosed.json" {
164	try err("[\"a\",\n4\n,1,");
165	}
166	test "n_array_number_and_comma.json" {
167	try err("[1,]");
168	}
169	test "n_array_number_and_several_commas.json" {
170	try err("[1,,]");
171	}
172	test "n_array_spaces_vertical_tab_formfeed.json" {
173	try err("[\"\x0ba\"\\f]");
174	}
175	test "n_array_star_inside.json" {
176	try err("[*]");
177	}
178	test "n_array_unclosed.json" {
179	try err("[\"\"");
180	}
181	test "n_array_unclosed_trailing_comma.json" {
182	try err("[1,");
183	}
184	test "n_array_unclosed_with_new_lines.json" {
185	try err("[1,\n1\n,1");
186	}
187	test "n_array_unclosed_with_object_inside.json" {
188	try err("[{}");
189	}
190	test "n_incomplete_false.json" {
191	try err("[fals]");
192	}
193	test "n_incomplete_null.json" {
194	try err("[nul]");
195	}
196	test "n_incomplete_true.json" {
197	try err("[tru]");
198	}
199	test "n_multidigit_number_then_00.json" {
200	try err("123\x00");
201	}
202	test "n_number_++.json" {
203	try err("[++1234]");
204	}
205	test "n_number_+1.json" {
206	try err("[+1]");
207	}
208	test "n_number_+Inf.json" {
209	try err("[+Inf]");
210	}
211	test "n_number_-01.json" {
212	try err("[-01]");
213	}
214	test "n_number_-1.0..json" {
215	try err("[-1.0.]");
216	}
217	test "n_number_-2..json" {
218	try err("[-2.]");
219	}
220	test "n_number_-NaN.json" {
221	try err("[-NaN]");
222	}
223	test "n_number_.-1.json" {
224	try err("[.-1]");
225	}
226	test "n_number_.2e-3.json" {
227	try err("[.2e-3]");
228	}
229	test "n_number_0.1.2.json" {
230	try err("[0.1.2]");
231	}
232	test "n_number_0.3e+.json" {
233	try err("[0.3e+]");
234	}
235	test "n_number_0.3e.json" {
236	try err("[0.3e]");
237	}
238	test "n_number_0.e1.json" {
239	try err("[0.e1]");
240	}
241	test "n_number_0_capital_E+.json" {
242	try err("[0E+]");
243	}
244	test "n_number_0_capital_E.json" {
245	try err("[0E]");
246	}
247	test "n_number_0e+.json" {
248	try err("[0e+]");
249	}
250	test "n_number_0e.json" {
251	try err("[0e]");
252	}
253	test "n_number_1.0e+.json" {
254	try err("[1.0e+]");
255	}
256	test "n_number_1.0e-.json" {
257	try err("[1.0e-]");
258	}
259	test "n_number_1.0e.json" {
260	try err("[1.0e]");
261	}
262	test "n_number_1_000.json" {
263	try err("[1 000.0]");
264	}
265	test "n_number_1eE2.json" {
266	try err("[1eE2]");
267	}
268	test "n_number_2.e+3.json" {
269	try err("[2.e+3]");
270	}
271	test "n_number_2.e-3.json" {
272	try err("[2.e-3]");
273	}
274	test "n_number_2.e3.json" {
275	try err("[2.e3]");
276	}
277	test "n_number_9.e+.json" {
278	try err("[9.e+]");
279	}
280	test "n_number_Inf.json" {
281	try err("[Inf]");
282	}
283	test "n_number_NaN.json" {
284	try err("[NaN]");
285	}
286	test "n_number_U+FF11_fullwidth_digit_one.json" {
287	try err("[\xef\xbc\x91]");
288	}
289	test "n_number_expression.json" {
290	try err("[1+2]");
291	}
292	test "n_number_hex_1_digit.json" {
293	try err("[0x1]");
294	}
295	test "n_number_hex_2_digits.json" {
296	try err("[0x42]");
297	}
298	test "n_number_infinity.json" {
299	try err("[Infinity]");
300	}
301	test "n_number_invalid+-.json" {
302	try err("[0e+-1]");
303	}
304	test "n_number_invalid-negative-real.json" {
305	try err("[-123.123foo]");
306	}
307	test "n_number_invalid-utf-8-in-bigger-int.json" {
308	try err("[123\xe5]");
309	}
310	test "n_number_invalid-utf-8-in-exponent.json" {
311	try err("[1e1\xe5]");
312	}
313	test "n_number_invalid-utf-8-in-int.json" {
314	try err("[0\xe5]\n");
315	}
316	test "n_number_minus_infinity.json" {
317	try err("[-Infinity]");
318	}
319	test "n_number_minus_sign_with_trailing_garbage.json" {
320	try err("[-foo]");
321	}
322	test "n_number_minus_space_1.json" {
323	try err("[- 1]");
324	}
325	test "n_number_neg_int_starting_with_zero.json" {
326	try err("[-012]");
327	}
328	test "n_number_neg_real_without_int_part.json" {
329	try err("[-.123]");
330	}
331	test "n_number_neg_with_garbage_at_end.json" {
332	try err("[-1x]");
333	}
334	test "n_number_real_garbage_after_e.json" {
335	try err("[1ea]");
336	}
337	test "n_number_real_with_invalid_utf8_after_e.json" {
338	try err("[1e\xe5]");
339	}
340	test "n_number_real_without_fractional_part.json" {
341	try err("[1.]");
342	}
343	test "n_number_starting_with_dot.json" {
344	try err("[.123]");
345	}
346	test "n_number_with_alpha.json" {
347	try err("[1.2a-3]");
348	}
349	test "n_number_with_alpha_char.json" {
350	try err("[1.8011670033376514H-308]");
351	}
352	test "n_number_with_leading_zero.json" {
353	try err("[012]");
354	}
355	test "n_object_bad_value.json" {
356	try err("[\"x\", truth]");
357	}
358	test "n_object_bracket_key.json" {
359	try err("{[: \"x\"}\n");
360	}
361	test "n_object_comma_instead_of_colon.json" {
362	try err("{\"x\", null}");
363	}
364	test "n_object_double_colon.json" {
365	try err("{\"x\"::\"b\"}");
366	}
367	test "n_object_emoji.json" {
368	try err("{\xf0\x9f\x87\xa8\xf0\x9f\x87\xad}");
369	}
370	test "n_object_garbage_at_end.json" {
371	try err("{\"a\":\"a\" 123}");
372	}
373	test "n_object_key_with_single_quotes.json" {
374	try err("{key: 'value'}");
375	}
376	test "n_object_lone_continuation_byte_in_key_and_trailing_comma.json" {
377	try err("{\"\xb9\":\"0\",}");
378	}
379	test "n_object_missing_colon.json" {
380	try err("{\"a\" b}");
381	}
382	test "n_object_missing_key.json" {
383	try err("{:\"b\"}");
384	}
385	test "n_object_missing_semicolon.json" {
386	try err("{\"a\" \"b\"}");
387	}
388	test "n_object_missing_value.json" {
389	try err("{\"a\":");
390	}
391	test "n_object_no-colon.json" {
392	try err("{\"a\"");
393	}
394	test "n_object_non_string_key.json" {
395	try err("{1:1}");
396	}
397	test "n_object_non_string_key_but_huge_number_instead.json" {
398	try err("{9999E9999:1}");
399	}
400	test "n_object_repeated_null_null.json" {
401	try err("{null:null,null:null}");
402	}
403	test "n_object_several_trailing_commas.json" {
404	try err("{\"id\":0,,,,,}");
405	}
406	test "n_object_single_quote.json" {
407	try err("{'a':0}");
408	}
409	test "n_object_trailing_comma.json" {
410	try err("{\"id\":0,}");
411	}
412	test "n_object_trailing_comment.json" {
413	try err("{\"a\":\"b\"}/**/");
414	}
415	test "n_object_trailing_comment_open.json" {
416	try err("{\"a\":\"b\"}/**//");
417	}
418	test "n_object_trailing_comment_slash_open.json" {
419	try err("{\"a\":\"b\"}//");
420	}
421	test "n_object_trailing_comment_slash_open_incomplete.json" {
422	try err("{\"a\":\"b\"}/");
423	}
424	test "n_object_two_commas_in_a_row.json" {
425	try err("{\"a\":\"b\",,\"c\":\"d\"}");
426	}
427	test "n_object_unquoted_key.json" {
428	try err("{a: \"b\"}");
429	}
430	test "n_object_unterminated-value.json" {
431	try err("{\"a\":\"a");
432	}
433	test "n_object_with_single_string.json" {
434	try err("{ \"foo\" : \"bar\", \"a\" }");
435	}
436	test "n_object_with_trailing_garbage.json" {
437	try err("{\"a\":\"b\"}#");
438	}
439	test "n_single_space.json" {
440	try err(" ");
441	}
442	test "n_string_1_surrogate_then_escape.json" {
443	try err("[\"\\uD800\\\"]");
444	}
445	test "n_string_1_surrogate_then_escape_u.json" {
446	try err("[\"\\uD800\\u\"]");
447	}
448	test "n_string_1_surrogate_then_escape_u1.json" {
449	try err("[\"\\uD800\\u1\"]");
450	}
451	test "n_string_1_surrogate_then_escape_u1x.json" {
452	try err("[\"\\uD800\\u1x\"]");
453	}
454	test "n_string_accentuated_char_no_quotes.json" {
455	try err("[\xc3\xa9]");
456	}
457	test "n_string_backslash_00.json" {
458	try err("[\"\\\x00\"]");
459	}
460	test "n_string_escape_x.json" {
461	try err("[\"\\x00\"]");
462	}
463	test "n_string_escaped_backslash_bad.json" {
464	try err("[\"\\\\\\\"]");
465	}
466	test "n_string_escaped_ctrl_char_tab.json" {
467	try err("[\"\\\x09\"]");
468	}
469	test "n_string_escaped_emoji.json" {
470	try err("[\"\\\xf0\x9f\x8c\x80\"]");
471	}
472	test "n_string_incomplete_escape.json" {
473	try err("[\"\\\"]");
474	}
475	test "n_string_incomplete_escaped_character.json" {
476	try err("[\"\\u00A\"]");
477	}
478	test "n_string_incomplete_surrogate.json" {
479	try err("[\"\\uD834\\uDd\"]");
480	}
481	test "n_string_incomplete_surrogate_escape_invalid.json" {
482	try err("[\"\\uD800\\uD800\\x\"]");
483	}
484	test "n_string_invalid-utf-8-in-escape.json" {
485	try err("[\"\\u\xe5\"]");
486	}
487	test "n_string_invalid_backslash_esc.json" {
488	try err("[\"\\a\"]");
489	}
490	test "n_string_invalid_unicode_escape.json" {
491	try err("[\"\\uqqqq\"]");
492	}
493	test "n_string_invalid_utf8_after_escape.json" {
494	try err("[\"\\\xe5\"]");
495	}
496	test "n_string_leading_uescaped_thinspace.json" {
497	try err("[\\u0020\"asd\"]");
498	}
499	test "n_string_no_quotes_with_bad_escape.json" {
500	try err("[\\n]");
501	}
502	test "n_string_single_doublequote.json" {
503	try err("\"");
504	}
505	test "n_string_single_quote.json" {
506	try err("['single quote']");
507	}
508	test "n_string_single_string_no_double_quotes.json" {
509	try err("abc");
510	}
511	test "n_string_start_escape_unclosed.json" {
512	try err("[\"\\");
513	}
514	test "n_string_unescaped_ctrl_char.json" {
515	try err("[\"a\x00a\"]");
516	}
517	test "n_string_unescaped_newline.json" {
518	try err("[\"new\nline\"]");
519	}
520	test "n_string_unescaped_tab.json" {
521	try err("[\"\x09\"]");
522	}
523	test "n_string_unicode_CapitalU.json" {
524	try err("\"\\UA66D\"");
525	}
526	test "n_string_with_trailing_garbage.json" {
527	try err("\"\"x");
528	}
529	test "n_structure_100000_opening_arrays.json" {
530	try err("[" ** 100000);
531	}
532	test "n_structure_U+2060_word_joined.json" {
533	try err("[\xe2\x81\xa0]");
534	}
535	test "n_structure_UTF8_BOM_no_data.json" {
536	try err("\xef\xbb\xbf");
537	}
538	test "n_structure_angle_bracket_..json" {
539	try err("<.>");
540	}
541	test "n_structure_angle_bracket_null.json" {
542	try err("[<null>]");
543	}
544	test "n_structure_array_trailing_garbage.json" {
545	try err("[1]x");
546	}
547	test "n_structure_array_with_extra_array_close.json" {
548	try err("[1]]");
549	}
550	test "n_structure_array_with_unclosed_string.json" {
551	try err("[\"asd]");
552	}
553	test "n_structure_ascii-unicode-identifier.json" {
554	try err("a\xc3\xa5");
555	}
556	test "n_structure_capitalized_True.json" {
557	try err("[True]");
558	}
559	test "n_structure_close_unopened_array.json" {
560	try err("1]");
561	}
562	test "n_structure_comma_instead_of_closing_brace.json" {
563	try err("{\"x\": true,");
564	}
565	test "n_structure_double_array.json" {
566	try err("[][]");
567	}
568	test "n_structure_end_array.json" {
569	try err("]");
570	}
571	test "n_structure_incomplete_UTF8_BOM.json" {
572	try err("\xef\xbb{}");
573	}
574	test "n_structure_lone-invalid-utf-8.json" {
575	try err("\xe5");
576	}
577	test "n_structure_lone-open-bracket.json" {
578	try err("[");
579	}
580	test "n_structure_no_data.json" {
581	try err("");
582	}
583	test "n_structure_null-byte-outside-string.json" {
584	try err("[\x00]");
585	}
586	test "n_structure_number_with_trailing_garbage.json" {
587	try err("2@");
588	}
589	test "n_structure_object_followed_by_closing_object.json" {
590	try err("{}}");
591	}
592	test "n_structure_object_unclosed_no_value.json" {
593	try err("{\"\":");
594	}
595	test "n_structure_object_with_comment.json" {
596	try err("{\"a\":/comment/\"b\"}");
597	}
598	test "n_structure_object_with_trailing_garbage.json" {
599	try err("{\"a\": true} \"x\"");
600	}
601	test "n_structure_open_array_apostrophe.json" {
602	try err("['");
603	}
604	test "n_structure_open_array_comma.json" {
605	try err("[,");
606	}
607	test "n_structure_open_array_object.json" {
608	try err("[{\"\":" ** 50000 ++ "\n");
609	}
610	test "n_structure_open_array_open_object.json" {
611	try err("[{");
612	}
613	test "n_structure_open_array_open_string.json" {
614	try err("[\"a");
615	}
616	test "n_structure_open_array_string.json" {
617	try err("[\"a\"");
618	}
619	test "n_structure_open_object.json" {
620	try err("{");
621	}
622	test "n_structure_open_object_close_array.json" {
623	try err("{]");
624	}
625	test "n_structure_open_object_comma.json" {
626	try err("{,");
627	}
628	test "n_structure_open_object_open_array.json" {
629	try err("{[");
630	}
631	test "n_structure_open_object_open_string.json" {
632	try err("{\"a");
633	}
634	test "n_structure_open_object_string_with_apostrophes.json" {
635	try err("{'a'");
636	}
637	test "n_structure_open_open.json" {
638	try err("[\"\\{[\"\\{[\"\\{[\"\\{");
639	}
640	test "n_structure_single_eacute.json" {
641	try err("\xe9");
642	}
643	test "n_structure_single_star.json" {
644	try err("*");
645	}
646	test "n_structure_trailing_#.json" {
647	try err("{\"a\":\"b\"}#{}");
648	}
649	test "n_structure_uescaped_LF_before_string.json" {
650	try err("[\\u000A\"\"]");
651	}
652	test "n_structure_unclosed_array.json" {
653	try err("[1");
654	}
655	test "n_structure_unclosed_array_partial_null.json" {
656	try err("[ false, nul");
657	}
658	test "n_structure_unclosed_array_unfinished_false.json" {
659	try err("[ true, fals");
660	}
661	test "n_structure_unclosed_array_unfinished_true.json" {
662	try err("[ false, tru");
663	}
664	test "n_structure_unclosed_object.json" {
665	try err("{\"asd\":\"asd\"");
666	}
667	test "n_structure_unicode-identifier.json" {
668	try err("\xc3\xa5");
669	}
670	test "n_structure_whitespace_U+2060_word_joiner.json" {
671	try err("[\xe2\x81\xa0]");
672	}
673	test "n_structure_whitespace_formfeed.json" {
674	try err("[\x0c]");
675	}
676	test "y_array_arraysWithSpaces.json" {
677	try ok("[[] ]");
678	}
679	test "y_array_empty-string.json" {
680	try ok("[\"\"]");
681	}
682	test "y_array_empty.json" {
683	try ok("[]");
684	}
685	test "y_array_ending_with_newline.json" {
686	try ok("[\"a\"]");
687	}
688	test "y_array_false.json" {
689	try ok("[false]");
690	}
691	test "y_array_heterogeneous.json" {
692	try ok("[null, 1, \"1\", {}]");
693	}
694	test "y_array_null.json" {
695	try ok("[null]");
696	}
697	test "y_array_with_1_and_newline.json" {
698	try ok("[1\n]");
699	}
700	test "y_array_with_leading_space.json" {
701	try ok(" [1]");
702	}
703	test "y_array_with_several_null.json" {
704	try ok("[1,null,null,null,2]");
705	}
706	test "y_array_with_trailing_space.json" {
707	try ok("[2] ");
708	}
709	test "y_number.json" {
710	try ok("[123e65]");
711	}
712	test "y_number_0e+1.json" {
713	try ok("[0e+1]");
714	}
715	test "y_number_0e1.json" {
716	try ok("[0e1]");
717	}
718	test "y_number_after_space.json" {
719	try ok("[ 4]");
720	}
721	test "y_number_double_close_to_zero.json" {
722	try ok("[-0.000000000000000000000000000000000000000000000000000000000000000000000000000001]\n");
723	}
724	test "y_number_int_with_exp.json" {
725	try ok("[20e1]");
726	}
727	test "y_number_minus_zero.json" {
728	try ok("[-0]");
729	}
730	test "y_number_negative_int.json" {
731	try ok("[-123]");
732	}
733	test "y_number_negative_one.json" {
734	try ok("[-1]");
735	}
736	test "y_number_negative_zero.json" {
737	try ok("[-0]");
738	}
739	test "y_number_real_capital_e.json" {
740	try ok("[1E22]");
741	}
742	test "y_number_real_capital_e_neg_exp.json" {
743	try ok("[1E-2]");
744	}
745	test "y_number_real_capital_e_pos_exp.json" {
746	try ok("[1E+2]");
747	}
748	test "y_number_real_exponent.json" {
749	try ok("[123e45]");
750	}
751	test "y_number_real_fraction_exponent.json" {
752	try ok("[123.456e78]");
753	}
754	test "y_number_real_neg_exp.json" {
755	try ok("[1e-2]");
756	}
757	test "y_number_real_pos_exponent.json" {
758	try ok("[1e+2]");
759	}
760	test "y_number_simple_int.json" {
761	try ok("[123]");
762	}
763	test "y_number_simple_real.json" {
764	try ok("[123.456789]");
765	}
766	test "y_object.json" {
767	try ok("{\"asd\":\"sdf\", \"dfg\":\"fgh\"}");
768	}
769	test "y_object_basic.json" {
770	try ok("{\"asd\":\"sdf\"}");
771	}
772	test "y_object_duplicated_key.json" {
773	try ok("{\"a\":\"b\",\"a\":\"c\"}");
774	}
775	test "y_object_duplicated_key_and_value.json" {
776	try ok("{\"a\":\"b\",\"a\":\"b\"}");
777	}
778	test "y_object_empty.json" {
779	try ok("{}");
780	}
781	test "y_object_empty_key.json" {
782	try ok("{\"\":0}");
783	}
784	test "y_object_escaped_null_in_key.json" {
785	try ok("{\"foo\\u0000bar\": 42}");
786	}
787	test "y_object_extreme_numbers.json" {
788	try ok("{ \"min\": -1.0e+28, \"max\": 1.0e+28 }");
789	}
790	test "y_object_long_strings.json" {
791	try ok("{\"x\":[{\"id\": \"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"}], \"id\": \"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"}");
792	}
793	test "y_object_simple.json" {
794	try ok("{\"a\":[]}");
795	}
796	test "y_object_string_unicode.json" {
797	try ok("{\"title\":\"\\u041f\\u043e\\u043b\\u0442\\u043e\\u0440\\u0430 \\u0417\\u0435\\u043c\\u043b\\u0435\\u043a\\u043e\\u043f\\u0430\" }");
798	}
799	test "y_object_with_newlines.json" {
800	try ok("{\n\"a\": \"b\"\n}");
801	}
802	test "y_string_1_2_3_bytes_UTF-8_sequences.json" {
803	try ok("[\"\\u0060\\u012a\\u12AB\"]");
804	}
805	test "y_string_accepted_surrogate_pair.json" {
806	try ok("[\"\\uD801\\udc37\"]");
807	}
808	test "y_string_accepted_surrogate_pairs.json" {
809	try ok("[\"\\ud83d\\ude39\\ud83d\\udc8d\"]");
810	}
811	test "y_string_allowed_escapes.json" {
812	try ok("[\"\\\"\\\\\\/\\b\\f\\n\\r\\t\"]");
813	}
814	test "y_string_backslash_and_u_escaped_zero.json" {
815	try ok("[\"\\\\u0000\"]");
816	}
817	test "y_string_backslash_doublequotes.json" {
818	try ok("[\"\\\"\"]");
819	}
820	test "y_string_comments.json" {
821	try ok("[\"a/b/c/*d//e\"]");
822	}
823	test "y_string_double_escape_a.json" {
824	try ok("[\"\\\\a\"]");
825	}
826	test "y_string_double_escape_n.json" {
827	try ok("[\"\\\\n\"]");
828	}
829	test "y_string_escaped_control_character.json" {
830	try ok("[\"\\u0012\"]");
831	}
832	test "y_string_escaped_noncharacter.json" {
833	try ok("[\"\\uFFFF\"]");
834	}
835	test "y_string_in_array.json" {
836	try ok("[\"asd\"]");
837	}
838	test "y_string_in_array_with_leading_space.json" {
839	try ok("[ \"asd\"]");
840	}
841	test "y_string_last_surrogates_1_and_2.json" {
842	try ok("[\"\\uDBFF\\uDFFF\"]");
843	}
844	test "y_string_nbsp_uescaped.json" {
845	try ok("[\"new\\u00A0line\"]");
846	}
847	test "y_string_nonCharacterInUTF-8_U+10FFFF.json" {
848	try ok("[\"\xf4\x8f\xbf\xbf\"]");
849	}
850	test "y_string_nonCharacterInUTF-8_U+FFFF.json" {
851	try ok("[\"\xef\xbf\xbf\"]");
852	}
853	test "y_string_null_escape.json" {
854	try ok("[\"\\u0000\"]");
855	}
856	test "y_string_one-byte-utf-8.json" {
857	try ok("[\"\\u002c\"]");
858	}
859	test "y_string_pi.json" {
860	try ok("[\"\xcf\x80\"]");
861	}
862	test "y_string_reservedCharacterInUTF-8_U+1BFFF.json" {
863	try ok("[\"\xf0\x9b\xbf\xbf\"]");
864	}
865	test "y_string_simple_ascii.json" {
866	try ok("[\"asd \"]");
867	}
868	test "y_string_space.json" {
869	try ok("\" \"");
870	}
871	test "y_string_surrogates_U+1D11E_MUSICAL_SYMBOL_G_CLEF.json" {
872	try ok("[\"\\uD834\\uDd1e\"]");
873	}
874	test "y_string_three-byte-utf-8.json" {
875	try ok("[\"\\u0821\"]");
876	}
877	test "y_string_two-byte-utf-8.json" {
878	try ok("[\"\\u0123\"]");
879	}
880	test "y_string_u+2028_line_sep.json" {
881	try ok("[\"\xe2\x80\xa8\"]");
882	}
883	test "y_string_u+2029_par_sep.json" {
884	try ok("[\"\xe2\x80\xa9\"]");
885	}
886	test "y_string_uEscape.json" {
887	try ok("[\"\\u0061\\u30af\\u30EA\\u30b9\"]");
888	}
889	test "y_string_uescaped_newline.json" {
890	try ok("[\"new\\u000Aline\"]");
891	}
892	test "y_string_unescaped_char_delete.json" {
893	try ok("[\"\x7f\"]");
894	}
895	test "y_string_unicode.json" {
896	try ok("[\"\\uA66D\"]");
897	}
898	test "y_string_unicodeEscapedBackslash.json" {
899	try ok("[\"\\u005C\"]");
900	}
901	test "y_string_unicode_2.json" {
902	try ok("[\"\xe2\x8d\x82\xe3\x88\xb4\xe2\x8d\x82\"]");
903	}
904	test "y_string_unicode_U+10FFFE_nonchar.json" {
905	try ok("[\"\\uDBFF\\uDFFE\"]");
906	}
907	test "y_string_unicode_U+1FFFE_nonchar.json" {
908	try ok("[\"\\uD83F\\uDFFE\"]");
909	}
910	test "y_string_unicode_U+200B_ZERO_WIDTH_SPACE.json" {
911	try ok("[\"\\u200B\"]");
912	}
913	test "y_string_unicode_U+2064_invisible_plus.json" {
914	try ok("[\"\\u2064\"]");
915	}
916	test "y_string_unicode_U+FDD0_nonchar.json" {
917	try ok("[\"\\uFDD0\"]");
918	}
919	test "y_string_unicode_U+FFFE_nonchar.json" {
920	try ok("[\"\\uFFFE\"]");
921	}
922	test "y_string_unicode_escaped_double_quote.json" {
923	try ok("[\"\\u0022\"]");
924	}
925	test "y_string_utf8.json" {
926	try ok("[\"\xe2\x82\xac\xf0\x9d\x84\x9e\"]");
927	}
928	test "y_string_with_del_character.json" {
929	try ok("[\"a\x7fa\"]");
930	}
931	test "y_structure_lonely_false.json" {
932	try ok("false");
933	}
934	test "y_structure_lonely_int.json" {
935	try ok("42");
936	}
937	test "y_structure_lonely_negative_real.json" {
938	try ok("-0.1");
939	}
940	test "y_structure_lonely_null.json" {
941	try ok("null");
942	}
943	test "y_structure_lonely_string.json" {
944	try ok("\"asd\"");
945	}
946	test "y_structure_lonely_true.json" {
947	try ok("true");
948	}
949	test "y_structure_string_empty.json" {
950	try ok("\"\"");
951	}
952	test "y_structure_trailing_newline.json" {
953	try ok("[\"a\"]\n");
954	}
955	test "y_structure_true_in_array.json" {
956	try ok("[true]");
957	}
958	test "y_structure_whitespace_array.json" {
959	try ok(" [] ");
960	}

lib/std/json/dynamic.zig created+344

...	@@ -0,0 +1,344 @@
1	const std = @import("std");
2	const debug = std.debug;
3	const ArenaAllocator = std.heap.ArenaAllocator;
4	const ArrayList = std.ArrayList;
5	const StringArrayHashMap = std.StringArrayHashMap;
6	const Allocator = std.mem.Allocator;
7
8	const StringifyOptions = @import("./stringify.zig").StringifyOptions;
9	const stringify = @import("./stringify.zig").stringify;
10
11	const JsonScanner = @import("./scanner.zig").Scanner;
12	const AllocWhen = @import("./scanner.zig").AllocWhen;
13	const Token = @import("./scanner.zig").Token;
14	const isNumberFormattedLikeAnInteger = @import("./scanner.zig").isNumberFormattedLikeAnInteger;
15
16	pub const ValueTree = struct {
17	arena: *ArenaAllocator,
18	root: Value,
19
20	pub fn deinit(self: *ValueTree) void {
21	self.arena.deinit();
22	self.arena.child_allocator.destroy(self.arena);
23	}
24	};
25
26	pub const ObjectMap = StringArrayHashMap(Value);
27	pub const Array = ArrayList(Value);
28
29	/// Represents a JSON value
30	/// Currently only supports numbers that fit into i64 or f64.
31	pub const Value = union(enum) {
32	null,
33	bool: bool,
34	integer: i64,
35	float: f64,
36	number_string: []const u8,
37	string: []const u8,
38	array: Array,
39	object: ObjectMap,
40
41	pub fn jsonStringify(
42	value: @This(),
43	options: StringifyOptions,
44	out_stream: anytype,
45	) @TypeOf(out_stream).Error!void {
46	switch (value) {
47	.null => try stringify(null, options, out_stream),
48	.bool => \|inner\| try stringify(inner, options, out_stream),
49	.integer => \|inner\| try stringify(inner, options, out_stream),
50	.float => \|inner\| try stringify(inner, options, out_stream),
51	.number_string => \|inner\| try out_stream.writeAll(inner),
52	.string => \|inner\| try stringify(inner, options, out_stream),
53	.array => \|inner\| try stringify(inner.items, options, out_stream),
54	.object => \|inner\| {
55	try out_stream.writeByte('{');
56	var field_output = false;
57	var child_options = options;
58	child_options.whitespace.indent_level += 1;
59	var it = inner.iterator();
60	while (it.next()) \|entry\| {
61	if (!field_output) {
62	field_output = true;
63	} else {
64	try out_stream.writeByte(',');
65	}
66	try child_options.whitespace.outputIndent(out_stream);
67
68	try stringify(entry.key_ptr.*, options, out_stream);
69	try out_stream.writeByte(':');
70	if (child_options.whitespace.separator) {
71	try out_stream.writeByte(' ');
72	}
73	try stringify(entry.value_ptr.*, child_options, out_stream);
74	}
75	if (field_output) {
76	try options.whitespace.outputIndent(out_stream);
77	}
78	try out_stream.writeByte('}');
79	},
80	}
81	}
82
83	pub fn dump(self: Value) void {
84	std.debug.getStderrMutex().lock();
85	defer std.debug.getStderrMutex().unlock();
86
87	const stderr = std.io.getStdErr().writer();
88	stringify(self, .{}, stderr) catch return;
89	}
90	};
91
92	/// A non-stream JSON parser which constructs a tree of Value's.
93	pub const Parser = struct {
94	allocator: Allocator,
95	state: State,
96	alloc_when: AllocWhen,
97	// Stores parent nodes and un-combined Values.
98	stack: Array,
99
100	const State = enum {
101	object_key,
102	object_value,
103	array_value,
104	simple,
105	};
106
107	pub fn init(allocator: Allocator, alloc_when: AllocWhen) Parser {
108	return Parser{
109	.allocator = allocator,
110	.state = .simple,
111	.alloc_when = alloc_when,
112	.stack = Array.init(allocator),
113	};
114	}
115
116	pub fn deinit(p: *Parser) void {
117	p.stack.deinit();
118	}
119
120	pub fn reset(p: *Parser) void {
121	p.state = .simple;
122	p.stack.shrinkRetainingCapacity(0);
123	}
124
125	pub fn parse(p: *Parser, input: []const u8) !ValueTree {
126	var scanner = JsonScanner.initCompleteInput(p.allocator, input);
127	defer scanner.deinit();
128
129	var arena = try p.allocator.create(ArenaAllocator);
130	errdefer p.allocator.destroy(arena);
131
132	arena.* = ArenaAllocator.init(p.allocator);
133	errdefer arena.deinit();
134
135	const allocator = arena.allocator();
136
137	while (true) {
138	const token = try scanner.nextAlloc(allocator, p.alloc_when);
139	if (token == .end_of_document) break;
140	try p.transition(allocator, token);
141	}
142
143	debug.assert(p.stack.items.len == 1);
144
145	return ValueTree{
146	.arena = arena,
147	.root = p.stack.items[0],
148	};
149	}
150
151	// Even though p.allocator exists, we take an explicit allocator so that allocation state
152	// can be cleaned up on error correctly during a `parse` on call.
153	fn transition(p: *Parser, allocator: Allocator, token: Token) !void {
154	switch (p.state) {
155	.object_key => switch (token) {
156	.object_end => {
157	if (p.stack.items.len == 1) {
158	return;
159	}
160
161	var value = p.stack.pop();
162	try p.pushToParent(&value);
163	},
164	.string => \|s\| {
165	try p.stack.append(Value{ .string = s });
166	p.state = .object_value;
167	},
168	.allocated_string => \|s\| {
169	try p.stack.append(Value{ .string = s });
170	p.state = .object_value;
171	},
172	else => unreachable,
173	},
174	.object_value => {
175	var object = &p.stack.items[p.stack.items.len - 2].object;
176	var key = p.stack.items[p.stack.items.len - 1].string;
177
178	switch (token) {
179	.object_begin => {
180	try p.stack.append(Value{ .object = ObjectMap.init(allocator) });
181	p.state = .object_key;
182	},
183	.array_begin => {
184	try p.stack.append(Value{ .array = Array.init(allocator) });
185	p.state = .array_value;
186	},
187	.string => \|s\| {
188	try object.put(key, Value{ .string = s });
189	_ = p.stack.pop();
190	p.state = .object_key;
191	},
192	.allocated_string => \|s\| {
193	try object.put(key, Value{ .string = s });
194	_ = p.stack.pop();
195	p.state = .object_key;
196	},
197	.number => \|slice\| {
198	try object.put(key, try p.parseNumber(slice));
199	_ = p.stack.pop();
200	p.state = .object_key;
201	},
202	.allocated_number => \|slice\| {
203	try object.put(key, try p.parseNumber(slice));
204	_ = p.stack.pop();
205	p.state = .object_key;
206	},
207	.true => {
208	try object.put(key, Value{ .bool = true });
209	_ = p.stack.pop();
210	p.state = .object_key;
211	},
212	.false => {
213	try object.put(key, Value{ .bool = false });
214	_ = p.stack.pop();
215	p.state = .object_key;
216	},
217	.null => {
218	try object.put(key, .null);
219	_ = p.stack.pop();
220	p.state = .object_key;
221	},
222	.object_end, .array_end, .end_of_document => unreachable,
223	.partial_number, .partial_string, .partial_string_escaped_1, .partial_string_escaped_2, .partial_string_escaped_3, .partial_string_escaped_4 => unreachable,
224	}
225	},
226	.array_value => {
227	var array = &p.stack.items[p.stack.items.len - 1].array;
228
229	switch (token) {
230	.array_end => {
231	if (p.stack.items.len == 1) {
232	return;
233	}
234
235	var value = p.stack.pop();
236	try p.pushToParent(&value);
237	},
238	.object_begin => {
239	try p.stack.append(Value{ .object = ObjectMap.init(allocator) });
240	p.state = .object_key;
241	},
242	.array_begin => {
243	try p.stack.append(Value{ .array = Array.init(allocator) });
244	p.state = .array_value;
245	},
246	.string => \|s\| {
247	try array.append(Value{ .string = s });
248	},
249	.allocated_string => \|s\| {
250	try array.append(Value{ .string = s });
251	},
252	.number => \|slice\| {
253	try array.append(try p.parseNumber(slice));
254	},
255	.allocated_number => \|slice\| {
256	try array.append(try p.parseNumber(slice));
257	},
258	.true => {
259	try array.append(Value{ .bool = true });
260	},
261	.false => {
262	try array.append(Value{ .bool = false });
263	},
264	.null => {
265	try array.append(.null);
266	},
267	.object_end, .end_of_document => unreachable,
268	.partial_number, .partial_string, .partial_string_escaped_1, .partial_string_escaped_2, .partial_string_escaped_3, .partial_string_escaped_4 => unreachable,
269	}
270	},
271	.simple => switch (token) {
272	.object_begin => {
273	try p.stack.append(Value{ .object = ObjectMap.init(allocator) });
274	p.state = .object_key;
275	},
276	.array_begin => {
277	try p.stack.append(Value{ .array = Array.init(allocator) });
278	p.state = .array_value;
279	},
280	.string => \|s\| {
281	try p.stack.append(Value{ .string = s });
282	},
283	.allocated_string => \|s\| {
284	try p.stack.append(Value{ .string = s });
285	},
286	.number => \|slice\| {
287	try p.stack.append(try p.parseNumber(slice));
288	},
289	.allocated_number => \|slice\| {
290	try p.stack.append(try p.parseNumber(slice));
291	},
292	.true => {
293	try p.stack.append(Value{ .bool = true });
294	},
295	.false => {
296	try p.stack.append(Value{ .bool = false });
297	},
298	.null => {
299	try p.stack.append(.null);
300	},
301	.object_end, .array_end, .end_of_document => unreachable,
302	.partial_number, .partial_string, .partial_string_escaped_1, .partial_string_escaped_2, .partial_string_escaped_3, .partial_string_escaped_4 => unreachable,
303	},
304	}
305	}
306
307	fn pushToParent(p: Parser, value: const Value) !void {
308	switch (p.stack.items[p.stack.items.len - 1]) {
309	// Object Parent -> [ ..., object, <key>, value ]
310	.string => \|key\| {
311	_ = p.stack.pop();
312
313	var object = &p.stack.items[p.stack.items.len - 1].object;
314	try object.put(key, value.*);
315	p.state = .object_key;
316	},
317	// Array Parent -> [ ..., <array>, value ]
318	.array => \|*array\| {
319	try array.append(value.*);
320	p.state = .array_value;
321	},
322	else => {
323	unreachable;
324	},
325	}
326	}
327
328	fn parseNumber(p: *Parser, slice: []const u8) !Value {
329	_ = p;
330	return if (isNumberFormattedLikeAnInteger(slice))
331	Value{
332	.integer = std.fmt.parseInt(i64, slice, 10) catch \|e\| switch (e) {
333	error.Overflow => return Value{ .number_string = slice },
334	error.InvalidCharacter => \|err\| return err,
335	},
336	}
337	else
338	Value{ .float = try std.fmt.parseFloat(f64, slice) };
339	}
340	};
341
342	test {
343	_ = @import("dynamic_test.zig");
344	}

lib/std/json/dynamic_test.zig created+285

...	@@ -0,0 +1,285 @@
1	const std = @import("std");
2	const mem = std.mem;
3	const testing = std.testing;
4
5	const ObjectMap = @import("dynamic.zig").ObjectMap;
6	const Array = @import("dynamic.zig").Array;
7	const Value = @import("dynamic.zig").Value;
8	const Parser = @import("dynamic.zig").Parser;
9
10	test "json.parser.dynamic" {
11	var p = Parser.init(testing.allocator, .alloc_if_needed);
12	defer p.deinit();
13
14	const s =
15	\\{
16	\\ "Image": {
17	\\ "Width": 800,
18	\\ "Height": 600,
19	\\ "Title": "View from 15th Floor",
20	\\ "Thumbnail": {
21	\\ "Url": "http://www.example.com/image/481989943",
22	\\ "Height": 125,
23	\\ "Width": 100
24	\\ },
25	\\ "Animated" : false,
26	\\ "IDs": [116, 943, 234, 38793],
27	\\ "ArrayOfObject": [{"n": "m"}],
28	\\ "double": 1.3412,
29	\\ "LargeInt": 18446744073709551615
30	\\ }
31	\\}
32	;
33
34	var tree = try p.parse(s);
35	defer tree.deinit();
36
37	var root = tree.root;
38
39	var image = root.object.get("Image").?;
40
41	const width = image.object.get("Width").?;
42	try testing.expect(width.integer == 800);
43
44	const height = image.object.get("Height").?;
45	try testing.expect(height.integer == 600);
46
47	const title = image.object.get("Title").?;
48	try testing.expect(mem.eql(u8, title.string, "View from 15th Floor"));
49
50	const animated = image.object.get("Animated").?;
51	try testing.expect(animated.bool == false);
52
53	const array_of_object = image.object.get("ArrayOfObject").?;
54	try testing.expect(array_of_object.array.items.len == 1);
55
56	const obj0 = array_of_object.array.items[0].object.get("n").?;
57	try testing.expect(mem.eql(u8, obj0.string, "m"));
58
59	const double = image.object.get("double").?;
60	try testing.expect(double.float == 1.3412);
61
62	const large_int = image.object.get("LargeInt").?;
63	try testing.expect(mem.eql(u8, large_int.number_string, "18446744073709551615"));
64	}
65
66	const writeStream = @import("./write_stream.zig").writeStream;
67	test "write json then parse it" {
68	var out_buffer: [1000]u8 = undefined;
69
70	var fixed_buffer_stream = std.io.fixedBufferStream(&out_buffer);
71	const out_stream = fixed_buffer_stream.writer();
72	var jw = writeStream(out_stream, 4);
73
74	try jw.beginObject();
75
76	try jw.objectField("f");
77	try jw.emitBool(false);
78
79	try jw.objectField("t");
80	try jw.emitBool(true);
81
82	try jw.objectField("int");
83	try jw.emitNumber(1234);
84
85	try jw.objectField("array");
86	try jw.beginArray();
87
88	try jw.arrayElem();
89	try jw.emitNull();
90
91	try jw.arrayElem();
92	try jw.emitNumber(12.34);
93
94	try jw.endArray();
95
96	try jw.objectField("str");
97	try jw.emitString("hello");
98
99	try jw.endObject();
100
101	var parser = Parser.init(testing.allocator, .alloc_if_needed);
102	defer parser.deinit();
103	var tree = try parser.parse(fixed_buffer_stream.getWritten());
104	defer tree.deinit();
105
106	try testing.expect(tree.root.object.get("f").?.bool == false);
107	try testing.expect(tree.root.object.get("t").?.bool == true);
108	try testing.expect(tree.root.object.get("int").?.integer == 1234);
109	try testing.expect(tree.root.object.get("array").?.array.items[0].null == {});
110	try testing.expect(tree.root.object.get("array").?.array.items[1].float == 12.34);
111	try testing.expect(mem.eql(u8, tree.root.object.get("str").?.string, "hello"));
112	}
113
114	fn testParse(arena_allocator: std.mem.Allocator, json_str: []const u8) !Value {
115	var p = Parser.init(arena_allocator, .alloc_if_needed);
116	return (try p.parse(json_str)).root;
117	}
118
119	test "parsing empty string gives appropriate error" {
120	var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
121	defer arena_allocator.deinit();
122	try testing.expectError(error.UnexpectedEndOfInput, testParse(arena_allocator.allocator(), ""));
123	}
124
125	test "parse tree should not contain dangling pointers" {
126	var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
127	defer arena_allocator.deinit();
128
129	var p = Parser.init(arena_allocator.allocator(), .alloc_if_needed);
130	defer p.deinit();
131
132	var tree = try p.parse("[]");
133	defer tree.deinit();
134
135	// Allocation should succeed
136	var i: usize = 0;
137	while (i < 100) : (i += 1) {
138	try tree.root.array.append(Value{ .integer = 100 });
139	}
140	try testing.expectEqual(tree.root.array.items.len, 100);
141	}
142
143	test "integer after float has proper type" {
144	var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
145	defer arena_allocator.deinit();
146	const parsed = try testParse(arena_allocator.allocator(),
147	\\{
148	\\ "float": 3.14,
149	\\ "ints": [1, 2, 3]
150	\\}
151	);
152	try std.testing.expect(parsed.object.get("ints").?.array.items[0] == .integer);
153	}
154
155	test "escaped characters" {
156	var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
157	defer arena_allocator.deinit();
158	const input =
159	\\{
160	\\ "backslash": "\\",
161	\\ "forwardslash": "\/",
162	\\ "newline": "\n",
163	\\ "carriagereturn": "\r",
164	\\ "tab": "\t",
165	\\ "formfeed": "\f",
166	\\ "backspace": "\b",
167	\\ "doublequote": "\"",
168	\\ "unicode": "\u0105",
169	\\ "surrogatepair": "\ud83d\ude02"
170	\\}
171	;
172
173	const obj = (try testParse(arena_allocator.allocator(), input)).object;
174
175	try testing.expectEqualSlices(u8, obj.get("backslash").?.string, "\\");
176	try testing.expectEqualSlices(u8, obj.get("forwardslash").?.string, "/");
177	try testing.expectEqualSlices(u8, obj.get("newline").?.string, "\n");
178	try testing.expectEqualSlices(u8, obj.get("carriagereturn").?.string, "\r");
179	try testing.expectEqualSlices(u8, obj.get("tab").?.string, "\t");
180	try testing.expectEqualSlices(u8, obj.get("formfeed").?.string, "\x0C");
181	try testing.expectEqualSlices(u8, obj.get("backspace").?.string, "\x08");
182	try testing.expectEqualSlices(u8, obj.get("doublequote").?.string, "\"");
183	try testing.expectEqualSlices(u8, obj.get("unicode").?.string, "ą");
184	try testing.expectEqualSlices(u8, obj.get("surrogatepair").?.string, "😂");
185	}
186
187	test "string copy option" {
188	const input =
189	\\{
190	\\ "noescape": "aą😂",
191	\\ "simple": "\\\/\n\r\t\f\b\"",
192	\\ "unicode": "\u0105",
193	\\ "surrogatepair": "\ud83d\ude02"
194	\\}
195	;
196
197	var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
198	defer arena_allocator.deinit();
199	const allocator = arena_allocator.allocator();
200
201	var parser = Parser.init(allocator, .alloc_if_needed);
202	const tree_nocopy = try parser.parse(input);
203	const obj_nocopy = tree_nocopy.root.object;
204
205	parser = Parser.init(allocator, .alloc_always);
206	const tree_copy = try parser.parse(input);
207	const obj_copy = tree_copy.root.object;
208
209	for ([_][]const u8{ "noescape", "simple", "unicode", "surrogatepair" }) \|field_name\| {
210	try testing.expectEqualSlices(u8, obj_nocopy.get(field_name).?.string, obj_copy.get(field_name).?.string);
211	}
212
213	const nocopy_addr = &obj_nocopy.get("noescape").?.string[0];
214	const copy_addr = &obj_copy.get("noescape").?.string[0];
215
216	var found_nocopy = false;
217	for (input, 0..) \|_, index\| {
218	try testing.expect(copy_addr != &input[index]);
219	if (nocopy_addr == &input[index]) {
220	found_nocopy = true;
221	}
222	}
223	try testing.expect(found_nocopy);
224	}
225
226	test "Value.jsonStringify" {
227	{
228	var buffer: [10]u8 = undefined;
229	var fbs = std.io.fixedBufferStream(&buffer);
230	try @as(Value, .null).jsonStringify(.{}, fbs.writer());
231	try testing.expectEqualSlices(u8, fbs.getWritten(), "null");
232	}
233	{
234	var buffer: [10]u8 = undefined;
235	var fbs = std.io.fixedBufferStream(&buffer);
236	try (Value{ .bool = true }).jsonStringify(.{}, fbs.writer());
237	try testing.expectEqualSlices(u8, fbs.getWritten(), "true");
238	}
239	{
240	var buffer: [10]u8 = undefined;
241	var fbs = std.io.fixedBufferStream(&buffer);
242	try (Value{ .integer = 42 }).jsonStringify(.{}, fbs.writer());
243	try testing.expectEqualSlices(u8, fbs.getWritten(), "42");
244	}
245	{
246	var buffer: [10]u8 = undefined;
247	var fbs = std.io.fixedBufferStream(&buffer);
248	try (Value{ .number_string = "43" }).jsonStringify(.{}, fbs.writer());
249	try testing.expectEqualSlices(u8, fbs.getWritten(), "43");
250	}
251	{
252	var buffer: [10]u8 = undefined;
253	var fbs = std.io.fixedBufferStream(&buffer);
254	try (Value{ .float = 42 }).jsonStringify(.{}, fbs.writer());
255	try testing.expectEqualSlices(u8, fbs.getWritten(), "4.2e+01");
256	}
257	{
258	var buffer: [10]u8 = undefined;
259	var fbs = std.io.fixedBufferStream(&buffer);
260	try (Value{ .string = "weeee" }).jsonStringify(.{}, fbs.writer());
261	try testing.expectEqualSlices(u8, fbs.getWritten(), "\"weeee\"");
262	}
263	{
264	var buffer: [10]u8 = undefined;
265	var fbs = std.io.fixedBufferStream(&buffer);
266	var vals = [_]Value{
267	.{ .integer = 1 },
268	.{ .integer = 2 },
269	.{ .number_string = "3" },
270	};
271	try (Value{
272	.array = Array.fromOwnedSlice(undefined, &vals),
273	}).jsonStringify(.{}, fbs.writer());
274	try testing.expectEqualSlices(u8, fbs.getWritten(), "[1,2,3]");
275	}
276	{
277	var buffer: [10]u8 = undefined;
278	var fbs = std.io.fixedBufferStream(&buffer);
279	var obj = ObjectMap.init(testing.allocator);
280	defer obj.deinit();
281	try obj.putNoClobber("a", .{ .string = "b" });
282	try (Value{ .object = obj }).jsonStringify(.{}, fbs.writer());
283	try testing.expectEqualSlices(u8, fbs.getWritten(), "{\"a\":\"b\"}");
284	}
285	}

lib/std/json/scanner.zig created+1764

...	@@ -0,0 +1,1764 @@
1	// Notes on standards compliance: https://datatracker.ietf.org/doc/html/rfc8259
2	// * RFC 8259 requires JSON documents be valid UTF-8,
3	// but makes an allowance for systems that are "part of a closed ecosystem".
4	// I have no idea what that's supposed to mean in the context of a standard specification.
5	// This implementation requires inputs to be valid UTF-8.
6	// * RFC 8259 contradicts itself regarding whether lowercase is allowed in \u hex digits,
7	// but this is probably a bug in the spec, and it's clear that lowercase is meant to be allowed.
8	// (RFC 5234 defines HEXDIG to only allow uppercase.)
9	// * When RFC 8259 refers to a "character", I assume they really mean a "Unicode scalar value".
10	// See http://www.unicode.org/glossary/#unicode_scalar_value .
11	// * RFC 8259 doesn't explicitly disallow unpaired surrogate halves in \u escape sequences,
12	// but vaguely implies that \u escapes are for encoding Unicode "characters" (i.e. Unicode scalar values?),
13	// which would mean that unpaired surrogate halves are forbidden.
14	// By contrast ECMA-404 (a competing(/compatible?) JSON standard, which JavaScript's JSON.parse() conforms to)
15	// explicitly allows unpaired surrogate halves.
16	// This implementation forbids unpaired surrogate halves in \u sequences.
17	// If a high surrogate half appears in a \u sequence,
18	// then a low surrogate half must immediately follow in \u notation.
19	// * RFC 8259 allows implementations to "accept non-JSON forms or extensions".
20	// This implementation does not accept any of that.
21	// * RFC 8259 allows implementations to put limits on "the size of texts",
22	// "the maximum depth of nesting", "the range and precision of numbers",
23	// and "the length and character contents of strings".
24	// This low-level implementation does not limit these,
25	// except where noted above, and except that nesting depth requires memory allocation.
26	// Note that this low-level API does not interpret numbers numerically,
27	// but simply emits their source form for some higher level code to make sense of.
28	// * This low-level implementation allows duplicate object keys,
29	// and key/value pairs are emitted in the order they appear in the input.
30
31	const std = @import("std");
32
33	const Allocator = std.mem.Allocator;
34	const ArrayList = std.ArrayList;
35	const assert = std.debug.assert;
36
37	/// Scan the input and check for malformed JSON.
38	/// On `SyntaxError` or `UnexpectedEndOfInput`, returns `false`.
39	/// Returns any errors from the allocator as-is, which is unlikely,
40	/// but can be caused by extreme nesting depth in the input.
41	pub fn validate(allocator: Allocator, s: []const u8) Allocator.Error!bool {
42	var scanner = Scanner.initCompleteInput(allocator, s);
43	defer scanner.deinit();
44
45	while (true) {
46	const token = scanner.next() catch \|err\| switch (err) {
47	error.SyntaxError, error.UnexpectedEndOfInput => return false,
48	error.OutOfMemory => return error.OutOfMemory,
49	error.BufferUnderrun => unreachable,
50	};
51	if (token == .end_of_document) break;
52	}
53
54	return true;
55	}
56
57	/// The parsing errors are divided into two categories:
58	/// * `SyntaxError` is for clearly malformed JSON documents,
59	/// such as giving an input document that isn't JSON at all.
60	/// * `UnexpectedEndOfInput` is for signaling that everything's been
61	/// valid so far, but the input appears to be truncated for some reason.
62	/// Note that a completely empty (or whitespace-only) input will give `UnexpectedEndOfInput`.
63	pub const Error = error{ SyntaxError, UnexpectedEndOfInput };
64
65	/// Calls `std.json.Reader` with `std.json.default_buffer_size`.
66	pub fn reader(allocator: Allocator, io_reader: anytype) Reader(default_buffer_size, @TypeOf(io_reader)) {
67	return Reader(default_buffer_size, @TypeOf(io_reader)).init(allocator, io_reader);
68	}
69	/// Used by `json.reader`.
70	pub const default_buffer_size = 0x1000;
71
72	/// The tokens emitted by `std.json.Scanner` and `std.json.Reader` `.next*()` functions follow this grammar:
73	/// ```
74	/// <document> = <value> .end_of_document
75	/// <value> =
76	/// \| <object>
77	/// \| <array>
78	/// \| <number>
79	/// \| <string>
80	/// \| .true
81	/// \| .false
82	/// \| .null
83	/// <object> = .object_begin ( <string> <value> )* .object_end
84	/// <array> = .array_begin ( <value> )* .array_end
85	/// <number> = <It depends. See below.>
86	/// <string> = <It depends. See below.>
87	/// ```
88	///
89	/// What you get for `<number>` and `<string>` values depends on which `next*()` method you call:
90	///
91	/// ```
92	/// next():
93	/// <number> = ( .partial_number )* .number
94	/// <string> = ( <partial_string> )* .string
95	/// <partial_string> =
96	/// \| .partial_string
97	/// \| .partial_string_escaped_1
98	/// \| .partial_string_escaped_2
99	/// \| .partial_string_escaped_3
100	/// \| .partial_string_escaped_4
101	///
102	/// nextAlloc*(..., .alloc_always):
103	/// <number> = .allocated_number
104	/// <string> = .allocated_string
105	///
106	/// nextAlloc*(..., .alloc_if_needed):
107	/// <number> =
108	/// \| .number
109	/// \| .allocated_number
110	/// <string> =
111	/// \| .string
112	/// \| .allocated_string
113	/// ```
114	///
115	/// For all tokens with a `[]const u8`, `[]u8`, or `[n]u8` payload, the payload represents the content of the value.
116	/// For number values, this is the representation of the number exactly as it appears in the input.
117	/// For strings, this is the content of the string after resolving escape sequences.
118	///
119	/// For `.allocated_number` and `.allocated_string`, the `[]u8` payloads are allocations made with the given allocator.
120	/// You are responsible for managing that memory. `json.Reader.deinit()` does not free those allocations.
121	///
122	/// The `.partial_*` tokens indicate that a value spans multiple input buffers or that a string contains escape sequences.
123	/// To get a complete value in memory, you need to concatenate the values yourself.
124	/// Calling `nextAlloc()` does this for you, and returns an `.allocated_` token with the result.
125	///
126	/// For tokens with a `[]const u8` payload, the payload is a slice into the current input buffer.
127	/// The memory may become undefined during the next call to `json.Scanner.feedInput()`
128	/// or any `json.Reader` method whose return error set includes `json.Error`.
129	/// To keep the value persistently, it recommended to make a copy or to use `.alloc_always`,
130	/// which makes a copy for you.
131	///
132	/// Note that `.number` and `.string` tokens that follow `.partial_*` tokens may have `0` length to indicate that
133	/// the previously partial value is completed with no additional bytes.
134	/// (This can happen when the break between input buffers happens to land on the exact end of a value. E.g. `"[1234"`, `"]"`.)
135	/// `.partial_*` tokens never have `0` length.
136	///
137	/// The recommended strategy for using the different `next*()` methods is something like this:
138	///
139	/// When you're expecting an object key, use `.alloc_if_needed`.
140	/// You often don't need a copy of the key string to persist; you might just check which field it is.
141	/// In the case that the key happens to require an allocation, free it immediately after checking it.
142	///
143	/// When you're expecting a meaningful string value (such as on the right of a `:`),
144	/// use `.alloc_always` in order to keep the value valid throughout parsing the rest of the document.
145	///
146	/// When you're expecting a number value, use `.alloc_if_needed`.
147	/// You're probably going to be parsing the string representation of the number into a numeric representation,
148	/// so you need the complete string representation only temporarily.
149	///
150	/// When you're skipping an unrecognized value, use `skipValue()`.
151	pub const Token = union(enum) {
152	object_begin,
153	object_end,
154	array_begin,
155	array_end,
156
157	true,
158	false,
159	null,
160
161	number: []const u8,
162	partial_number: []const u8,
163	allocated_number: []u8,
164
165	string: []const u8,
166	partial_string: []const u8,
167	partial_string_escaped_1: [1]u8,
168	partial_string_escaped_2: [2]u8,
169	partial_string_escaped_3: [3]u8,
170	partial_string_escaped_4: [4]u8,
171	allocated_string: []u8,
172
173	end_of_document,
174	};
175
176	/// This is only used in `peekNextTokenType()` and gives a categorization based on the first byte of the next token that will be emitted from a `next*()` call.
177	pub const TokenType = enum {
178	object_begin,
179	object_end,
180	array_begin,
181	array_end,
182	true,
183	false,
184	null,
185	number,
186	string,
187	end_of_document,
188	};
189
190	/// To enable diagnostics, declare `var diagnostics = Diagnostics{};` then call `source.enableDiagnostics(&diagnostics);`
191	/// where `source` is either a `std.json.Reader` or a `std.json.Scanner` that has just been initialized.
192	/// At any time, notably just after an error, call `getLine()`, `getColumn()`, and/or `getByteOffset()`
193	/// to get meaningful information from this.
194	pub const Diagnostics = struct {
195	line_number: u64 = 1,
196	line_start_cursor: usize = @bitCast(usize, @as(isize, -1)), // Start just "before" the input buffer to get a 1-based column for line 1.
197	total_bytes_before_current_input: u64 = 0,
198	cursor_pointer: *const usize = undefined,
199
200	/// Starts at 1.
201	pub fn getLine(self: *const @This()) u64 {
202	return self.line_number;
203	}
204	/// Starts at 1.
205	pub fn getColumn(self: *const @This()) u64 {
206	return self.cursor_pointer.* -% self.line_start_cursor;
207	}
208	/// Starts at 0. Measures the byte offset since the start of the input.
209	pub fn getByteOffset(self: *const @This()) u64 {
210	return self.total_bytes_before_current_input + self.cursor_pointer.*;
211	}
212	};
213
214	/// See the documentation for `std.json.Token`.
215	pub const AllocWhen = enum { alloc_if_needed, alloc_always };
216
217	/// For security, the maximum size allocated to store a single string or number value is limited to 4MiB by default.
218	/// This limit can be specified by calling `nextAllocMax()` instead of `nextAlloc()`.
219	pub const default_max_value_len = 4 * 1024 * 1024;
220
221	/// Connects a `std.io.Reader` to a `std.json.Scanner`.
222	/// All `next*()` methods here handle `error.BufferUnderrun` from `std.json.Scanner`, and then read from the reader.
223	pub fn Reader(comptime buffer_size: usize, comptime ReaderType: type) type {
224	return struct {
225	scanner: Scanner,
226	reader: ReaderType,
227
228	buffer: [buffer_size]u8 = undefined,
229
230	/// The allocator is only used to track `[]` and `{}` nesting levels.
231	pub fn init(allocator: Allocator, io_reader: ReaderType) @This() {
232	return .{
233	.scanner = Scanner.initStreaming(allocator),
234	.reader = io_reader,
235	};
236	}
237	pub fn deinit(self: *@This()) void {
238	self.scanner.deinit();
239	self.* = undefined;
240	}
241
242	/// Calls `std.json.Scanner.enableDiagnostics`.
243	pub fn enableDiagnostics(self: @This(), diagnostics: Diagnostics) void {
244	self.scanner.enableDiagnostics(diagnostics);
245	}
246
247	pub const NextError = ReaderType.Error \|\| Error \|\| Allocator.Error;
248	pub const SkipError = NextError;
249	pub const AllocError = NextError \|\| error{ValueTooLong};
250	pub const PeekError = ReaderType.Error \|\| Error;
251
252	/// Equivalent to `nextAllocMax(allocator, when, default_max_value_len);`
253	/// See also `std.json.Token` for documentation of `nextAlloc*()` function behavior.
254	pub fn nextAlloc(self: *@This(), allocator: Allocator, when: AllocWhen) AllocError!Token {
255	return self.nextAllocMax(allocator, when, default_max_value_len);
256	}
257	/// See also `std.json.Token` for documentation of `nextAlloc*()` function behavior.
258	pub fn nextAllocMax(self: *@This(), allocator: Allocator, when: AllocWhen, max_value_len: usize) AllocError!Token {
259	const token_type = try self.peekNextTokenType();
260	switch (token_type) {
261	.number, .string => {
262	var value_list = ArrayList(u8).init(allocator);
263	errdefer {
264	value_list.deinit();
265	}
266	if (try self.allocNextIntoArrayListMax(&value_list, when, max_value_len)) \|slice\| {
267	return if (token_type == .number)
268	Token{ .number = slice }
269	else
270	Token{ .string = slice };
271	} else {
272	return if (token_type == .number)
273	Token{ .allocated_number = try value_list.toOwnedSlice() }
274	else
275	Token{ .allocated_string = try value_list.toOwnedSlice() };
276	}
277	},
278
279	// Simple tokens never alloc.
280	.object_begin,
281	.object_end,
282	.array_begin,
283	.array_end,
284	.true,
285	.false,
286	.null,
287	.end_of_document,
288	=> return try self.next(),
289	}
290	}
291
292	/// Equivalent to `allocNextIntoArrayListMax(value_list, when, default_max_value_len);`
293	pub fn allocNextIntoArrayList(self: @This(), value_list: ArrayList(u8), when: AllocWhen) AllocError!?[]const u8 {
294	return self.allocNextIntoArrayListMax(value_list, when, default_max_value_len);
295	}
296	/// Calls `std.json.Scanner.allocNextIntoArrayListMax` and handles `error.BufferUnderrun`.
297	pub fn allocNextIntoArrayListMax(self: @This(), value_list: ArrayList(u8), when: AllocWhen, max_value_len: usize) AllocError!?[]const u8 {
298	while (true) {
299	return self.scanner.allocNextIntoArrayListMax(value_list, when, max_value_len) catch \|err\| switch (err) {
300	error.BufferUnderrun => {
301	try self.refillBuffer();
302	continue;
303	},
304	else => \|other_err\| return other_err,
305	};
306	}
307	}
308
309	/// Like `std.json.Scanner.skipValue`, but handles `error.BufferUnderrun`.
310	pub fn skipValue(self: *@This()) SkipError!void {
311	switch (try self.peekNextTokenType()) {
312	.object_begin, .array_begin => {
313	try self.skipUntilStackHeight(self.stackHeight());
314	},
315	.number, .string => {
316	while (true) {
317	switch (try self.next()) {
318	.partial_number,
319	.partial_string,
320	.partial_string_escaped_1,
321	.partial_string_escaped_2,
322	.partial_string_escaped_3,
323	.partial_string_escaped_4,
324	=> continue,
325
326	.number, .string => break,
327
328	else => unreachable,
329	}
330	}
331	},
332	.true, .false, .null => {
333	_ = try self.next();
334	},
335
336	.object_end, .array_end, .end_of_document => unreachable, // Attempt to skip a non-value token.
337	}
338	}
339	/// Like `std.json.Scanner.skipUntilStackHeight()` but handles `error.BufferUnderrun`.
340	pub fn skipUntilStackHeight(self: *@This(), terminal_stack_height: u32) NextError!void {
341	while (true) {
342	return self.scanner.skipUntilStackHeight(terminal_stack_height) catch \|err\| switch (err) {
343	error.BufferUnderrun => {
344	try self.refillBuffer();
345	continue;
346	},
347	else => \|other_err\| return other_err,
348	};
349	}
350	}
351
352	/// Calls `std.json.Scanner.stackHeight`.
353	pub fn stackHeight(self: *const @This()) u32 {
354	return self.scanner.stackHeight();
355	}
356	/// Calls `std.json.Scanner.ensureTotalStackCapacity`.
357	pub fn ensureTotalStackCapacity(self: *@This(), height: u32) Allocator.Error!void {
358	try self.scanner.ensureTotalStackCapacity(height);
359	}
360
361	/// See `std.json.Token` for documentation of this function.
362	pub fn next(self: *@This()) NextError!Token {
363	while (true) {
364	return self.scanner.next() catch \|err\| switch (err) {
365	error.BufferUnderrun => {
366	try self.refillBuffer();
367	continue;
368	},
369	else => \|other_err\| return other_err,
370	};
371	}
372	}
373
374	/// See `std.json.Scanner.peekNextTokenType()`.
375	pub fn peekNextTokenType(self: *@This()) PeekError!TokenType {
376	while (true) {
377	return self.scanner.peekNextTokenType() catch \|err\| switch (err) {
378	error.BufferUnderrun => {
379	try self.refillBuffer();
380	continue;
381	},
382	else => \|other_err\| return other_err,
383	};
384	}
385	}
386
387	fn refillBuffer(self: *@This()) ReaderType.Error!void {
388	const input = self.buffer[0..try self.reader.read(self.buffer[0..])];
389	if (input.len > 0) {
390	self.scanner.feedInput(input);
391	} else {
392	self.scanner.endInput();
393	}
394	}
395	};
396	}
397
398	/// The lowest level parsing API in this package;
399	/// supports streaming input with a low memory footprint.
400	/// The memory requirement is `O(d)` where d is the nesting depth of `[]` or `{}` containers in the input.
401	/// Specifically `d/8` bytes are required for this purpose,
402	/// with some extra buffer according to the implementation of `std.ArrayList`.
403	///
404	/// This scanner can emit partial tokens; see `std.json.Token`.
405	/// The input to this class is a sequence of input buffers that you must supply one at a time.
406	/// Call `feedInput()` with the first buffer, then call `next()` repeatedly until `error.BufferUnderrun` is returned.
407	/// Then call `feedInput()` again and so forth.
408	/// Call `endInput()` when the last input buffer has been given to `feedInput()`, either immediately after calling `feedInput()`,
409	/// or when `error.BufferUnderrun` requests more data and there is no more.
410	/// Be sure to call `next()` after calling `endInput()` until `Token.end_of_document` has been returned.
411	pub const Scanner = struct {
412	state: State = .value,
413	string_is_object_key: bool = false,
414	stack: BitStack,
415	value_start: usize = undefined,
416	unicode_code_point: u21 = undefined,
417
418	input: []const u8 = "",
419	cursor: usize = 0,
420	is_end_of_input: bool = false,
421	diagnostics: ?*Diagnostics = null,
422
423	/// The allocator is only used to track `[]` and `{}` nesting levels.
424	pub fn initStreaming(allocator: Allocator) @This() {
425	return .{
426	.stack = BitStack.init(allocator),
427	};
428	}
429	/// Use this if your input is a single slice.
430	/// This is effectively equivalent to:
431	/// ```
432	/// initStreaming(allocator);
433	/// feedInput(complete_input);
434	/// endInput();
435	/// ```
436	pub fn initCompleteInput(allocator: Allocator, complete_input: []const u8) @This() {
437	return .{
438	.stack = BitStack.init(allocator),
439	.input = complete_input,
440	.is_end_of_input = true,
441	};
442	}
443	pub fn deinit(self: *@This()) void {
444	self.stack.deinit();
445	self.* = undefined;
446	}
447
448	pub fn enableDiagnostics(self: @This(), diagnostics: Diagnostics) void {
449	diagnostics.cursor_pointer = &self.cursor;
450	self.diagnostics = diagnostics;
451	}
452
453	/// Call this whenever you get `error.BufferUnderrun` from `next()`.
454	/// When there is no more input to provide, call `endInput()`.
455	pub fn feedInput(self: *@This(), input: []const u8) void {
456	assert(self.cursor == self.input.len); // Not done with the last input slice.
457	if (self.diagnostics) \|diag\| {
458	diag.total_bytes_before_current_input += self.input.len;
459	// This usually goes "negative" to measure how far before the beginning
460	// of the new buffer the current line started.
461	diag.line_start_cursor -%= self.cursor;
462	}
463	self.input = input;
464	self.cursor = 0;
465	self.value_start = 0;
466	}
467	/// Call this when you will no longer call `feedInput()` anymore.
468	/// This can be called either immediately after the last `feedInput()`,
469	/// or at any time afterward, such as when getting `error.BufferUnderrun` from `next()`.
470	/// Don't forget to call `next*()` after `endInput()` until you get `.end_of_document`.
471	pub fn endInput(self: *@This()) void {
472	self.is_end_of_input = true;
473	}
474
475	pub const NextError = Error \|\| Allocator.Error \|\| error{BufferUnderrun};
476	pub const AllocError = Error \|\| Allocator.Error \|\| error{ValueTooLong};
477	pub const PeekError = Error \|\| error{BufferUnderrun};
478	pub const SkipError = Error \|\| Allocator.Error;
479	pub const AllocIntoArrayListError = AllocError \|\| error{BufferUnderrun};
480
481	/// Equivalent to `nextAllocMax(allocator, when, default_max_value_len);`
482	/// This function is only available after `endInput()` (or `initCompleteInput()`) has been called.
483	/// See also `std.json.Token` for documentation of `nextAlloc*()` function behavior.
484	pub fn nextAlloc(self: *@This(), allocator: Allocator, when: AllocWhen) AllocError!Token {
485	return self.nextAllocMax(allocator, when, default_max_value_len);
486	}
487
488	/// This function is only available after `endInput()` (or `initCompleteInput()`) has been called.
489	/// See also `std.json.Token` for documentation of `nextAlloc*()` function behavior.
490	pub fn nextAllocMax(self: *@This(), allocator: Allocator, when: AllocWhen, max_value_len: usize) AllocError!Token {
491	assert(self.is_end_of_input); // This function is not available in streaming mode.
492	const token_type = self.peekNextTokenType() catch \|e\| switch (e) {
493	error.BufferUnderrun => unreachable,
494	else => \|err\| return err,
495	};
496	switch (token_type) {
497	.number, .string => {
498	var value_list = ArrayList(u8).init(allocator);
499	errdefer {
500	value_list.deinit();
501	}
502	if (self.allocNextIntoArrayListMax(&value_list, when, max_value_len) catch \|e\| switch (e) {
503	error.BufferUnderrun => unreachable,
504	else => \|err\| return err,
505	}) \|slice\| {
506	return if (token_type == .number)
507	Token{ .number = slice }
508	else
509	Token{ .string = slice };
510	} else {
511	return if (token_type == .number)
512	Token{ .allocated_number = try value_list.toOwnedSlice() }
513	else
514	Token{ .allocated_string = try value_list.toOwnedSlice() };
515	}
516	},
517
518	// Simple tokens never alloc.
519	.object_begin,
520	.object_end,
521	.array_begin,
522	.array_end,
523	.true,
524	.false,
525	.null,
526	.end_of_document,
527	=> return self.next() catch \|e\| switch (e) {
528	error.BufferUnderrun => unreachable,
529	else => \|err\| return err,
530	},
531	}
532	}
533
534	/// Equivalent to `allocNextIntoArrayListMax(value_list, when, default_max_value_len);`
535	pub fn allocNextIntoArrayList(self: @This(), value_list: ArrayList(u8), when: AllocWhen) AllocIntoArrayListError!?[]const u8 {
536	return self.allocNextIntoArrayListMax(value_list, when, default_max_value_len);
537	}
538	/// The next token type must be either `.number` or `.string`. See `peekNextTokenType()`.
539	/// When allocation is not necessary with `.alloc_if_needed`,
540	/// this method returns the content slice from the input buffer, and `value_list` is not touched.
541	/// When allocation is necessary or with `.alloc_always`, this method concatenates partial tokens into the given `value_list`,
542	/// and returns `null` once the final `.number` or `.string` token has been written into it.
543	/// In case of an `error.BufferUnderrun`, partial values will be left in the given value_list.
544	/// The given `value_list` is never reset by this method, so an `error.BufferUnderrun` situation
545	/// can be resumed by passing the same array list in again.
546	/// This method does not indicate whether the token content being returned is for a `.number` or `.string` token type;
547	/// the caller of this method is expected to know which type of token is being processed.
548	pub fn allocNextIntoArrayListMax(self: @This(), value_list: ArrayList(u8), when: AllocWhen, max_value_len: usize) AllocIntoArrayListError!?[]const u8 {
549	while (true) {
550	const token = try self.next();
551	switch (token) {
552	// Accumulate partial values.
553	.partial_number, .partial_string => \|slice\| {
554	try appendSlice(value_list, slice, max_value_len);
555	},
556	.partial_string_escaped_1 => \|buf\| {
557	try appendSlice(value_list, buf[0..], max_value_len);
558	},
559	.partial_string_escaped_2 => \|buf\| {
560	try appendSlice(value_list, buf[0..], max_value_len);
561	},
562	.partial_string_escaped_3 => \|buf\| {
563	try appendSlice(value_list, buf[0..], max_value_len);
564	},
565	.partial_string_escaped_4 => \|buf\| {
566	try appendSlice(value_list, buf[0..], max_value_len);
567	},
568
569	// Return complete values.
570	.number => \|slice\| {
571	if (when == .alloc_if_needed and value_list.items.len == 0) {
572	// No alloc necessary.
573	return slice;
574	}
575	try appendSlice(value_list, slice, max_value_len);
576	// The token is complete.
577	return null;
578	},
579	.string => \|slice\| {
580	if (when == .alloc_if_needed and value_list.items.len == 0) {
581	// No alloc necessary.
582	return slice;
583	}
584	try appendSlice(value_list, slice, max_value_len);
585	// The token is complete.
586	return null;
587	},
588
589	.object_begin,
590	.object_end,
591	.array_begin,
592	.array_end,
593	.true,
594	.false,
595	.null,
596	.end_of_document,
597	=> unreachable, // Only .number and .string token types are allowed here. Check peekNextTokenType() before calling this.
598
599	.allocated_number, .allocated_string => unreachable,
600	}
601	}
602	}
603
604	/// This function is only available after `endInput()` (or `initCompleteInput()`) has been called.
605	/// If the next token type is `.object_begin` or `.array_begin`,
606	/// this function calls `next()` repeatedly until the corresponding `.object_end` or `.array_end` is found.
607	/// If the next token type is `.number` or `.string`,
608	/// this function calls `next()` repeatedly until the (non `.partial_*`) `.number` or `.string` token is found.
609	/// If the next token type is `.true`, `.false`, or `.null`, this function calls `next()` once.
610	/// The next token type must not be `.object_end`, `.array_end`, or `.end_of_document`;
611	/// see `peekNextTokenType()`.
612	pub fn skipValue(self: *@This()) SkipError!void {
613	assert(self.is_end_of_input); // This function is not available in streaming mode.
614	switch (self.peekNextTokenType() catch \|e\| switch (e) {
615	error.BufferUnderrun => unreachable,
616	else => \|err\| return err,
617	}) {
618	.object_begin, .array_begin => {
619	self.skipUntilStackHeight(self.stackHeight()) catch \|e\| switch (e) {
620	error.BufferUnderrun => unreachable,
621	else => \|err\| return err,
622	};
623	},
624	.number, .string => {
625	while (true) {
626	switch (self.next() catch \|e\| switch (e) {
627	error.BufferUnderrun => unreachable,
628	else => \|err\| return err,
629	}) {
630	.partial_number,
631	.partial_string,
632	.partial_string_escaped_1,
633	.partial_string_escaped_2,
634	.partial_string_escaped_3,
635	.partial_string_escaped_4,
636	=> continue,
637
638	.number, .string => break,
639
640	else => unreachable,
641	}
642	}
643	},
644	.true, .false, .null => {
645	_ = self.next() catch \|e\| switch (e) {
646	error.BufferUnderrun => unreachable,
647	else => \|err\| return err,
648	};
649	},
650
651	.object_end, .array_end, .end_of_document => unreachable, // Attempt to skip a non-value token.
652	}
653	}
654
655	/// Skip tokens until an `.object_end` or `.array_end` token results in a `stackHeight()` equal the given stack height.
656	/// Unlike `skipValue()`, this function is available in streaming mode.
657	pub fn skipUntilStackHeight(self: *@This(), terminal_stack_height: u32) NextError!void {
658	while (true) {
659	switch (try self.next()) {
660	.object_end, .array_end => {
661	if (self.stackHeight() == terminal_stack_height) break;
662	},
663	.end_of_document => unreachable,
664	else => continue,
665	}
666	}
667	}
668
669	/// The depth of `{}` or `[]` nesting levels at the current position.
670	pub fn stackHeight(self: *const @This()) u32 {
671	return self.stack.bit_len;
672	}
673
674	/// Pre allocate memory to hold the given number of nesting levels.
675	/// `stackHeight()` up to the given number will not cause allocations.
676	pub fn ensureTotalStackCapacity(self: *@This(), height: u32) Allocator.Error!void {
677	try self.stack.ensureTotalCapacity(height);
678	}
679
680	/// See `std.json.Token` for documentation of this function.
681	pub fn next(self: *@This()) NextError!Token {
682	state_loop: while (true) {
683	switch (self.state) {
684	.value => {
685	switch (try self.skipWhitespaceExpectByte()) {
686	// Object, Array
687	'{' => {
688	try self.stack.push(OBJECT_MODE);
689	self.cursor += 1;
690	self.state = .object_start;
691	return .object_begin;
692	},
693	'[' => {
694	try self.stack.push(ARRAY_MODE);
695	self.cursor += 1;
696	self.state = .array_start;
697	return .array_begin;
698	},
699
700	// String
701	'"' => {
702	self.cursor += 1;
703	self.value_start = self.cursor;
704	self.state = .string;
705	continue :state_loop;
706	},
707
708	// Number
709	'1'...'9' => {
710	self.value_start = self.cursor;
711	self.cursor += 1;
712	self.state = .number_int;
713	continue :state_loop;
714	},
715	'0' => {
716	self.value_start = self.cursor;
717	self.cursor += 1;
718	self.state = .number_leading_zero;
719	continue :state_loop;
720	},
721	'-' => {
722	self.value_start = self.cursor;
723	self.cursor += 1;
724	self.state = .number_minus;
725	continue :state_loop;
726	},
727
728	// literal values
729	't' => {
730	self.cursor += 1;
731	self.state = .literal_t;
732	continue :state_loop;
733	},
734	'f' => {
735	self.cursor += 1;
736	self.state = .literal_f;
737	continue :state_loop;
738	},
739	'n' => {
740	self.cursor += 1;
741	self.state = .literal_n;
742	continue :state_loop;
743	},
744
745	else => return error.SyntaxError,
746	}
747	},
748
749	.post_value => {
750	if (try self.skipWhitespaceCheckEnd()) return .end_of_document;
751
752	const c = self.input[self.cursor];
753	if (self.string_is_object_key) {
754	self.string_is_object_key = false;
755	switch (c) {
756	':' => {
757	self.cursor += 1;
758	self.state = .value;
759	continue :state_loop;
760	},
761	else => return error.SyntaxError,
762	}
763	}
764
765	switch (c) {
766	'}' => {
767	if (self.stack.pop() != OBJECT_MODE) return error.SyntaxError;
768	self.cursor += 1;
769	// stay in .post_value state.
770	return .object_end;
771	},
772	']' => {
773	if (self.stack.pop() != ARRAY_MODE) return error.SyntaxError;
774	self.cursor += 1;
775	// stay in .post_value state.
776	return .array_end;
777	},
778	',' => {
779	switch (self.stack.peek()) {
780	OBJECT_MODE => {
781	self.state = .object_post_comma;
782	},
783	ARRAY_MODE => {
784	self.state = .value;
785	},
786	}
787	self.cursor += 1;
788	continue :state_loop;
789	},
790	else => return error.SyntaxError,
791	}
792	},
793
794	.object_start => {
795	switch (try self.skipWhitespaceExpectByte()) {
796	'"' => {
797	self.cursor += 1;
798	self.value_start = self.cursor;
799	self.state = .string;
800	self.string_is_object_key = true;
801	continue :state_loop;
802	},
803	'}' => {
804	self.cursor += 1;
805	_ = self.stack.pop();
806	self.state = .post_value;
807	return .object_end;
808	},
809	else => return error.SyntaxError,
810	}
811	},
812	.object_post_comma => {
813	switch (try self.skipWhitespaceExpectByte()) {
814	'"' => {
815	self.cursor += 1;
816	self.value_start = self.cursor;
817	self.state = .string;
818	self.string_is_object_key = true;
819	continue :state_loop;
820	},
821	else => return error.SyntaxError,
822	}
823	},
824
825	.array_start => {
826	switch (try self.skipWhitespaceExpectByte()) {
827	']' => {
828	self.cursor += 1;
829	_ = self.stack.pop();
830	self.state = .post_value;
831	return .array_end;
832	},
833	else => {
834	self.state = .value;
835	continue :state_loop;
836	},
837	}
838	},
839
840	.number_minus => {
841	if (self.cursor >= self.input.len) return self.endOfBufferInNumber(false);
842	switch (self.input[self.cursor]) {
843	'0' => {
844	self.cursor += 1;
845	self.state = .number_leading_zero;
846	continue :state_loop;
847	},
848	'1'...'9' => {
849	self.cursor += 1;
850	self.state = .number_int;
851	continue :state_loop;
852	},
853	else => return error.SyntaxError,
854	}
855	},
856	.number_leading_zero => {
857	if (self.cursor >= self.input.len) return self.endOfBufferInNumber(true);
858	switch (self.input[self.cursor]) {
859	'.' => {
860	self.cursor += 1;
861	self.state = .number_post_dot;
862	continue :state_loop;
863	},
864	'e', 'E' => {
865	self.cursor += 1;
866	self.state = .number_post_e;
867	continue :state_loop;
868	},
869	else => {
870	self.state = .post_value;
871	return Token{ .number = self.takeValueSlice() };
872	},
873	}
874	},
875	.number_int => {
876	while (self.cursor < self.input.len) : (self.cursor += 1) {
877	switch (self.input[self.cursor]) {
878	'0'...'9' => continue,
879	'.' => {
880	self.cursor += 1;
881	self.state = .number_post_dot;
882	continue :state_loop;
883	},
884	'e', 'E' => {
885	self.cursor += 1;
886	self.state = .number_post_e;
887	continue :state_loop;
888	},
889	else => {
890	self.state = .post_value;
891	return Token{ .number = self.takeValueSlice() };
892	},
893	}
894	}
895	return self.endOfBufferInNumber(true);
896	},
897	.number_post_dot => {
898	if (self.cursor >= self.input.len) return self.endOfBufferInNumber(false);
899	switch (try self.expectByte()) {
900	'0'...'9' => {
901	self.cursor += 1;
902	self.state = .number_frac;
903	continue :state_loop;
904	},
905	else => return error.SyntaxError,
906	}
907	},
908	.number_frac => {
909	while (self.cursor < self.input.len) : (self.cursor += 1) {
910	switch (self.input[self.cursor]) {
911	'0'...'9' => continue,
912	'e', 'E' => {
913	self.cursor += 1;
914	self.state = .number_post_e;
915	continue :state_loop;
916	},
917	else => {
918	self.state = .post_value;
919	return Token{ .number = self.takeValueSlice() };
920	},
921	}
922	}
923	return self.endOfBufferInNumber(true);
924	},
925	.number_post_e => {
926	if (self.cursor >= self.input.len) return self.endOfBufferInNumber(false);
927	switch (self.input[self.cursor]) {
928	'0'...'9' => {
929	self.cursor += 1;
930	self.state = .number_exp;
931	continue :state_loop;
932	},
933	'+', '-' => {
934	self.cursor += 1;
935	self.state = .number_post_e_sign;
936	continue :state_loop;
937	},
938	else => return error.SyntaxError,
939	}
940	},
941	.number_post_e_sign => {
942	if (self.cursor >= self.input.len) return self.endOfBufferInNumber(false);
943	switch (self.input[self.cursor]) {
944	'0'...'9' => {
945	self.cursor += 1;
946	self.state = .number_exp;
947	continue :state_loop;
948	},
949	else => return error.SyntaxError,
950	}
951	},
952	.number_exp => {
953	while (self.cursor < self.input.len) : (self.cursor += 1) {
954	switch (self.input[self.cursor]) {
955	'0'...'9' => continue,
956	else => {
957	self.state = .post_value;
958	return Token{ .number = self.takeValueSlice() };
959	},
960	}
961	}
962	return self.endOfBufferInNumber(true);
963	},
964
965	.string => {
966	while (self.cursor < self.input.len) : (self.cursor += 1) {
967	switch (self.input[self.cursor]) {
968	0...0x1f => return error.SyntaxError, // Bare ASCII control code in string.
969
970	// ASCII plain text.
971	0x20...('"' - 1), ('"' + 1)...('\\' - 1), ('\\' + 1)...0x7F => continue,
972
973	// Special characters.
974	'"' => {
975	const result = Token{ .string = self.takeValueSlice() };
976	self.cursor += 1;
977	self.state = .post_value;
978	return result;
979	},
980	'\\' => {
981	const slice = self.takeValueSlice();
982	self.cursor += 1;
983	self.state = .string_backslash;
984	if (slice.len > 0) return Token{ .partial_string = slice };
985	continue :state_loop;
986	},
987
988	// UTF-8 validation.
989	// See http://unicode.org/mail-arch/unicode-ml/y2003-m02/att-0467/01-The_Algorithm_to_Valide_an_UTF-8_String
990	0xC2...0xDF => {
991	self.cursor += 1;
992	self.state = .string_utf8_last_byte;
993	continue :state_loop;
994	},
995	0xE0 => {
996	self.cursor += 1;
997	self.state = .string_utf8_second_to_last_byte_guard_against_overlong;
998	continue :state_loop;
999	},
1000	0xE1...0xEC, 0xEE...0xEF => {
1001	self.cursor += 1;
1002	self.state = .string_utf8_second_to_last_byte;
1003	continue :state_loop;
1004	},
1005	0xED => {
1006	self.cursor += 1;
1007	self.state = .string_utf8_second_to_last_byte_guard_against_surrogate_half;
1008	continue :state_loop;
1009	},
1010	0xF0 => {
1011	self.cursor += 1;
1012	self.state = .string_utf8_third_to_last_byte_guard_against_overlong;
1013	continue :state_loop;
1014	},
1015	0xF1...0xF3 => {
1016	self.cursor += 1;
1017	self.state = .string_utf8_third_to_last_byte;
1018	continue :state_loop;
1019	},
1020	0xF4 => {
1021	self.cursor += 1;
1022	self.state = .string_utf8_third_to_last_byte_guard_against_too_large;
1023	continue :state_loop;
1024	},
1025	0x80...0xC1, 0xF5...0xFF => return error.SyntaxError, // Invalid UTF-8.
1026	}
1027	}
1028	if (self.is_end_of_input) return error.UnexpectedEndOfInput;
1029	const slice = self.takeValueSlice();
1030	if (slice.len > 0) return Token{ .partial_string = slice };
1031	return error.BufferUnderrun;
1032	},
1033	.string_backslash => {
1034	switch (try self.expectByte()) {
1035	'"', '\\', '/' => {
1036	// Since these characters now represent themselves literally,
1037	// we can simply begin the next plaintext slice here.
1038	self.value_start = self.cursor;
1039	self.cursor += 1;
1040	self.state = .string;
1041	continue :state_loop;
1042	},
1043	'b' => {
1044	self.cursor += 1;
1045	self.value_start = self.cursor;
1046	self.state = .string;
1047	return Token{ .partial_string_escaped_1 = [_]u8{0x08} };
1048	},
1049	'f' => {
1050	self.cursor += 1;
1051	self.value_start = self.cursor;
1052	self.state = .string;
1053	return Token{ .partial_string_escaped_1 = [_]u8{0x0c} };
1054	},
1055	'n' => {
1056	self.cursor += 1;
1057	self.value_start = self.cursor;
1058	self.state = .string;
1059	return Token{ .partial_string_escaped_1 = [_]u8{'\n'} };
1060	},
1061	'r' => {
1062	self.cursor += 1;
1063	self.value_start = self.cursor;
1064	self.state = .string;
1065	return Token{ .partial_string_escaped_1 = [_]u8{'\r'} };
1066	},
1067	't' => {
1068	self.cursor += 1;
1069	self.value_start = self.cursor;
1070	self.state = .string;
1071	return Token{ .partial_string_escaped_1 = [_]u8{'\t'} };
1072	},
1073	'u' => {
1074	self.cursor += 1;
1075	self.state = .string_backslash_u;
1076	continue :state_loop;
1077	},
1078	else => return error.SyntaxError,
1079	}
1080	},
1081	.string_backslash_u => {
1082	const c = try self.expectByte();
1083	switch (c) {
1084	'0'...'9' => {
1085	self.unicode_code_point = @as(u21, c - '0') << 12;
1086	},
1087	'A'...'F' => {
1088	self.unicode_code_point = @as(u21, c - 'A' + 10) << 12;
1089	},
1090	'a'...'f' => {
1091	self.unicode_code_point = @as(u21, c - 'a' + 10) << 12;
1092	},
1093	else => return error.SyntaxError,
1094	}
1095	self.cursor += 1;
1096	self.state = .string_backslash_u_1;
1097	continue :state_loop;
1098	},
1099	.string_backslash_u_1 => {
1100	const c = try self.expectByte();
1101	switch (c) {
1102	'0'...'9' => {
1103	self.unicode_code_point \|= @as(u21, c - '0') << 8;
1104	},
1105	'A'...'F' => {
1106	self.unicode_code_point \|= @as(u21, c - 'A' + 10) << 8;
1107	},
1108	'a'...'f' => {
1109	self.unicode_code_point \|= @as(u21, c - 'a' + 10) << 8;
1110	},
1111	else => return error.SyntaxError,
1112	}
1113	self.cursor += 1;
1114	self.state = .string_backslash_u_2;
1115	continue :state_loop;
1116	},
1117	.string_backslash_u_2 => {
1118	const c = try self.expectByte();
1119	switch (c) {
1120	'0'...'9' => {
1121	self.unicode_code_point \|= @as(u21, c - '0') << 4;
1122	},
1123	'A'...'F' => {
1124	self.unicode_code_point \|= @as(u21, c - 'A' + 10) << 4;
1125	},
1126	'a'...'f' => {
1127	self.unicode_code_point \|= @as(u21, c - 'a' + 10) << 4;
1128	},
1129	else => return error.SyntaxError,
1130	}
1131	self.cursor += 1;
1132	self.state = .string_backslash_u_3;
1133	continue :state_loop;
1134	},
1135	.string_backslash_u_3 => {
1136	const c = try self.expectByte();
1137	switch (c) {
1138	'0'...'9' => {
1139	self.unicode_code_point \|= c - '0';
1140	},
1141	'A'...'F' => {
1142	self.unicode_code_point \|= c - 'A' + 10;
1143	},
1144	'a'...'f' => {
1145	self.unicode_code_point \|= c - 'a' + 10;
1146	},
1147	else => return error.SyntaxError,
1148	}
1149	self.cursor += 1;
1150	switch (self.unicode_code_point) {
1151	0xD800...0xDBFF => {
1152	// High surrogate half.
1153	self.unicode_code_point = 0x10000 \| (self.unicode_code_point << 10);
1154	self.state = .string_surrogate_half;
1155	continue :state_loop;
1156	},
1157	0xDC00...0xDFFF => return error.SyntaxError, // Unexpected low surrogate half.
1158	else => {
1159	// Code point from a single UTF-16 code unit.
1160	self.value_start = self.cursor;
1161	self.state = .string;
1162	return self.partialStringCodepoint();
1163	},
1164	}
1165	},
1166	.string_surrogate_half => {
1167	switch (try self.expectByte()) {
1168	'\\' => {
1169	self.cursor += 1;
1170	self.state = .string_surrogate_half_backslash;
1171	continue :state_loop;
1172	},
1173	else => return error.SyntaxError, // Expected low surrogate half.
1174	}
1175	},
1176	.string_surrogate_half_backslash => {
1177	switch (try self.expectByte()) {
1178	'u' => {
1179	self.cursor += 1;
1180	self.state = .string_surrogate_half_backslash_u;
1181	continue :state_loop;
1182	},
1183	else => return error.SyntaxError, // Expected low surrogate half.
1184	}
1185	},
1186	.string_surrogate_half_backslash_u => {
1187	switch (try self.expectByte()) {
1188	'D', 'd' => {
1189	self.cursor += 1;
1190	self.state = .string_surrogate_half_backslash_u_1;
1191	continue :state_loop;
1192	},
1193	else => return error.SyntaxError, // Expected low surrogate half.
1194	}
1195	},
1196	.string_surrogate_half_backslash_u_1 => {
1197	const c = try self.expectByte();
1198	switch (c) {
1199	'C'...'F' => {
1200	self.cursor += 1;
1201	self.unicode_code_point \|= @as(u21, c - 'C') << 8;
1202	self.state = .string_surrogate_half_backslash_u_2;
1203	continue :state_loop;
1204	},
1205	'c'...'f' => {
1206	self.cursor += 1;
1207	self.unicode_code_point \|= @as(u21, c - 'c') << 8;
1208	self.state = .string_surrogate_half_backslash_u_2;
1209	continue :state_loop;
1210	},
1211	else => return error.SyntaxError, // Expected low surrogate half.
1212	}
1213	},
1214	.string_surrogate_half_backslash_u_2 => {
1215	const c = try self.expectByte();
1216	switch (c) {
1217	'0'...'9' => {
1218	self.cursor += 1;
1219	self.unicode_code_point \|= @as(u21, c - '0') << 4;
1220	self.state = .string_surrogate_half_backslash_u_3;
1221	continue :state_loop;
1222	},
1223	'A'...'F' => {
1224	self.cursor += 1;
1225	self.unicode_code_point \|= @as(u21, c - 'A' + 10) << 4;
1226	self.state = .string_surrogate_half_backslash_u_3;
1227	continue :state_loop;
1228	},
1229	'a'...'f' => {
1230	self.cursor += 1;
1231	self.unicode_code_point \|= @as(u21, c - 'a' + 10) << 4;
1232	self.state = .string_surrogate_half_backslash_u_3;
1233	continue :state_loop;
1234	},
1235	else => return error.SyntaxError,
1236	}
1237	},
1238	.string_surrogate_half_backslash_u_3 => {
1239	const c = try self.expectByte();
1240	switch (c) {
1241	'0'...'9' => {
1242	self.unicode_code_point \|= c - '0';
1243	},
1244	'A'...'F' => {
1245	self.unicode_code_point \|= c - 'A' + 10;
1246	},
1247	'a'...'f' => {
1248	self.unicode_code_point \|= c - 'a' + 10;
1249	},
1250	else => return error.SyntaxError,
1251	}
1252	self.cursor += 1;
1253	self.value_start = self.cursor;
1254	self.state = .string;
1255	return self.partialStringCodepoint();
1256	},
1257
1258	.string_utf8_last_byte => {
1259	switch (try self.expectByte()) {
1260	0x80...0xBF => {
1261	self.cursor += 1;
1262	self.state = .string;
1263	continue :state_loop;
1264	},
1265	else => return error.SyntaxError, // Invalid UTF-8.
1266	}
1267	},
1268	.string_utf8_second_to_last_byte => {
1269	switch (try self.expectByte()) {
1270	0x80...0xBF => {
1271	self.cursor += 1;
1272	self.state = .string_utf8_last_byte;
1273	continue :state_loop;
1274	},
1275	else => return error.SyntaxError, // Invalid UTF-8.
1276	}
1277	},
1278	.string_utf8_second_to_last_byte_guard_against_overlong => {
1279	switch (try self.expectByte()) {
1280	0xA0...0xBF => {
1281	self.cursor += 1;
1282	self.state = .string_utf8_last_byte;
1283	continue :state_loop;
1284	},
1285	else => return error.SyntaxError, // Invalid UTF-8.
1286	}
1287	},
1288	.string_utf8_second_to_last_byte_guard_against_surrogate_half => {
1289	switch (try self.expectByte()) {
1290	0x80...0x9F => {
1291	self.cursor += 1;
1292	self.state = .string_utf8_last_byte;
1293	continue :state_loop;
1294	},
1295	else => return error.SyntaxError, // Invalid UTF-8.
1296	}
1297	},
1298	.string_utf8_third_to_last_byte => {
1299	switch (try self.expectByte()) {
1300	0x80...0xBF => {
1301	self.cursor += 1;
1302	self.state = .string_utf8_second_to_last_byte;
1303	continue :state_loop;
1304	},
1305	else => return error.SyntaxError, // Invalid UTF-8.
1306	}
1307	},
1308	.string_utf8_third_to_last_byte_guard_against_overlong => {
1309	switch (try self.expectByte()) {
1310	0x90...0xBF => {
1311	self.cursor += 1;
1312	self.state = .string_utf8_second_to_last_byte;
1313	continue :state_loop;
1314	},
1315	else => return error.SyntaxError, // Invalid UTF-8.
1316	}
1317	},
1318	.string_utf8_third_to_last_byte_guard_against_too_large => {
1319	switch (try self.expectByte()) {
1320	0x80...0x8F => {
1321	self.cursor += 1;
1322	self.state = .string_utf8_second_to_last_byte;
1323	continue :state_loop;
1324	},
1325	else => return error.SyntaxError, // Invalid UTF-8.
1326	}
1327	},
1328
1329	.literal_t => {
1330	switch (try self.expectByte()) {
1331	'r' => {
1332	self.cursor += 1;
1333	self.state = .literal_tr;
1334	continue :state_loop;
1335	},
1336	else => return error.SyntaxError,
1337	}
1338	},
1339	.literal_tr => {
1340	switch (try self.expectByte()) {
1341	'u' => {
1342	self.cursor += 1;
1343	self.state = .literal_tru;
1344	continue :state_loop;
1345	},
1346	else => return error.SyntaxError,
1347	}
1348	},
1349	.literal_tru => {
1350	switch (try self.expectByte()) {
1351	'e' => {
1352	self.cursor += 1;
1353	self.state = .post_value;
1354	return .true;
1355	},
1356	else => return error.SyntaxError,
1357	}
1358	},
1359	.literal_f => {
1360	switch (try self.expectByte()) {
1361	'a' => {
1362	self.cursor += 1;
1363	self.state = .literal_fa;
1364	continue :state_loop;
1365	},
1366	else => return error.SyntaxError,
1367	}
1368	},
1369	.literal_fa => {
1370	switch (try self.expectByte()) {
1371	'l' => {
1372	self.cursor += 1;
1373	self.state = .literal_fal;
1374	continue :state_loop;
1375	},
1376	else => return error.SyntaxError,
1377	}
1378	},
1379	.literal_fal => {
1380	switch (try self.expectByte()) {
1381	's' => {
1382	self.cursor += 1;
1383	self.state = .literal_fals;
1384	continue :state_loop;
1385	},
1386	else => return error.SyntaxError,
1387	}
1388	},
1389	.literal_fals => {
1390	switch (try self.expectByte()) {
1391	'e' => {
1392	self.cursor += 1;
1393	self.state = .post_value;
1394	return .false;
1395	},
1396	else => return error.SyntaxError,
1397	}
1398	},
1399	.literal_n => {
1400	switch (try self.expectByte()) {
1401	'u' => {
1402	self.cursor += 1;
1403	self.state = .literal_nu;
1404	continue :state_loop;
1405	},
1406	else => return error.SyntaxError,
1407	}
1408	},
1409	.literal_nu => {
1410	switch (try self.expectByte()) {
1411	'l' => {
1412	self.cursor += 1;
1413	self.state = .literal_nul;
1414	continue :state_loop;
1415	},
1416	else => return error.SyntaxError,
1417	}
1418	},
1419	.literal_nul => {
1420	switch (try self.expectByte()) {
1421	'l' => {
1422	self.cursor += 1;
1423	self.state = .post_value;
1424	return .null;
1425	},
1426	else => return error.SyntaxError,
1427	}
1428	},
1429	}
1430	unreachable;
1431	}
1432	}
1433
1434	/// Seeks ahead in the input until the first byte of the next token (or the end of the input)
1435	/// determines which type of token will be returned from the next `next*()` call.
1436	/// This function is idempotent, only advancing past commas, colons, and inter-token whitespace.
1437	pub fn peekNextTokenType(self: *@This()) PeekError!TokenType {
1438	state_loop: while (true) {
1439	switch (self.state) {
1440	.value => {
1441	switch (try self.skipWhitespaceExpectByte()) {
1442	'{' => return .object_begin,
1443	'[' => return .array_begin,
1444	'"' => return .string,
1445	'-', '0'...'9' => return .number,
1446	't' => return .true,
1447	'f' => return .false,
1448	'n' => return .null,
1449	else => return error.SyntaxError,
1450	}
1451	},
1452
1453	.post_value => {
1454	if (try self.skipWhitespaceCheckEnd()) return .end_of_document;
1455
1456	const c = self.input[self.cursor];
1457	if (self.string_is_object_key) {
1458	self.string_is_object_key = false;
1459	switch (c) {
1460	':' => {
1461	self.cursor += 1;
1462	self.state = .value;
1463	continue :state_loop;
1464	},
1465	else => return error.SyntaxError,
1466	}
1467	}
1468
1469	switch (c) {
1470	'}' => return .object_end,
1471	']' => return .array_end,
1472	',' => {
1473	switch (self.stack.peek()) {
1474	OBJECT_MODE => {
1475	self.state = .object_post_comma;
1476	},
1477	ARRAY_MODE => {
1478	self.state = .value;
1479	},
1480	}
1481	self.cursor += 1;
1482	continue :state_loop;
1483	},
1484	else => return error.SyntaxError,
1485	}
1486	},
1487
1488	.object_start => {
1489	switch (try self.skipWhitespaceExpectByte()) {
1490	'"' => return .string,
1491	'}' => return .object_end,
1492	else => return error.SyntaxError,
1493	}
1494	},
1495	.object_post_comma => {
1496	switch (try self.skipWhitespaceExpectByte()) {
1497	'"' => return .string,
1498	else => return error.SyntaxError,
1499	}
1500	},
1501
1502	.array_start => {
1503	switch (try self.skipWhitespaceExpectByte()) {
1504	']' => return .array_end,
1505	else => {
1506	self.state = .value;
1507	continue :state_loop;
1508	},
1509	}
1510	},
1511
1512	.number_minus,
1513	.number_leading_zero,
1514	.number_int,
1515	.number_post_dot,
1516	.number_frac,
1517	.number_post_e,
1518	.number_post_e_sign,
1519	.number_exp,
1520	=> return .number,
1521
1522	.string,
1523	.string_backslash,
1524	.string_backslash_u,
1525	.string_backslash_u_1,
1526	.string_backslash_u_2,
1527	.string_backslash_u_3,
1528	.string_surrogate_half,
1529	.string_surrogate_half_backslash,
1530	.string_surrogate_half_backslash_u,
1531	.string_surrogate_half_backslash_u_1,
1532	.string_surrogate_half_backslash_u_2,
1533	.string_surrogate_half_backslash_u_3,
1534	=> return .string,
1535
1536	.string_utf8_last_byte,
1537	.string_utf8_second_to_last_byte,
1538	.string_utf8_second_to_last_byte_guard_against_overlong,
1539	.string_utf8_second_to_last_byte_guard_against_surrogate_half,
1540	.string_utf8_third_to_last_byte,
1541	.string_utf8_third_to_last_byte_guard_against_overlong,
1542	.string_utf8_third_to_last_byte_guard_against_too_large,
1543	=> return .string,
1544
1545	.literal_t,
1546	.literal_tr,
1547	.literal_tru,
1548	=> return .true,
1549	.literal_f,
1550	.literal_fa,
1551	.literal_fal,
1552	.literal_fals,
1553	=> return .false,
1554	.literal_n,
1555	.literal_nu,
1556	.literal_nul,
1557	=> return .null,
1558	}
1559	unreachable;
1560	}
1561	}
1562
1563	const State = enum {
1564	value,
1565	post_value,
1566
1567	object_start,
1568	object_post_comma,
1569
1570	array_start,
1571
1572	number_minus,
1573	number_leading_zero,
1574	number_int,
1575	number_post_dot,
1576	number_frac,
1577	number_post_e,
1578	number_post_e_sign,
1579	number_exp,
1580
1581	string,
1582	string_backslash,
1583	string_backslash_u,
1584	string_backslash_u_1,
1585	string_backslash_u_2,
1586	string_backslash_u_3,
1587	string_surrogate_half,
1588	string_surrogate_half_backslash,
1589	string_surrogate_half_backslash_u,
1590	string_surrogate_half_backslash_u_1,
1591	string_surrogate_half_backslash_u_2,
1592	string_surrogate_half_backslash_u_3,
1593
1594	// From http://unicode.org/mail-arch/unicode-ml/y2003-m02/att-0467/01-The_Algorithm_to_Valide_an_UTF-8_String
1595	string_utf8_last_byte, // State A
1596	string_utf8_second_to_last_byte, // State B
1597	string_utf8_second_to_last_byte_guard_against_overlong, // State C
1598	string_utf8_second_to_last_byte_guard_against_surrogate_half, // State D
1599	string_utf8_third_to_last_byte, // State E
1600	string_utf8_third_to_last_byte_guard_against_overlong, // State F
1601	string_utf8_third_to_last_byte_guard_against_too_large, // State G
1602
1603	literal_t,
1604	literal_tr,
1605	literal_tru,
1606	literal_f,
1607	literal_fa,
1608	literal_fal,
1609	literal_fals,
1610	literal_n,
1611	literal_nu,
1612	literal_nul,
1613	};
1614
1615	fn expectByte(self: *const @This()) !u8 {
1616	if (self.cursor < self.input.len) {
1617	return self.input[self.cursor];
1618	}
1619	// No byte.
1620	if (self.is_end_of_input) return error.UnexpectedEndOfInput;
1621	return error.BufferUnderrun;
1622	}
1623
1624	fn skipWhitespace(self: *@This()) void {
1625	while (self.cursor < self.input.len) : (self.cursor += 1) {
1626	switch (self.input[self.cursor]) {
1627	// Whitespace
1628	' ', '\t', '\r' => continue,
1629	'\n' => {
1630	if (self.diagnostics) \|diag\| {
1631	diag.line_number += 1;
1632	// This will count the newline itself,
1633	// which means a straight-forward subtraction will give a 1-based column number.
1634	diag.line_start_cursor = self.cursor;
1635	}
1636	continue;
1637	},
1638	else => return,
1639	}
1640	}
1641	}
1642
1643	fn skipWhitespaceExpectByte(self: *@This()) !u8 {
1644	self.skipWhitespace();
1645	return self.expectByte();
1646	}
1647
1648	fn skipWhitespaceCheckEnd(self: *@This()) !bool {
1649	self.skipWhitespace();
1650	if (self.cursor >= self.input.len) {
1651	// End of buffer.
1652	if (self.is_end_of_input) {
1653	// End of everything.
1654	if (self.stackHeight() == 0) {
1655	// We did it!
1656	return true;
1657	}
1658	return error.UnexpectedEndOfInput;
1659	}
1660	return error.BufferUnderrun;
1661	}
1662	if (self.stackHeight() == 0) return error.SyntaxError;
1663	return false;
1664	}
1665
1666	fn takeValueSlice(self: *@This()) []const u8 {
1667	const slice = self.input[self.value_start..self.cursor];
1668	self.value_start = self.cursor;
1669	return slice;
1670	}
1671
1672	fn endOfBufferInNumber(self: *@This(), allow_end: bool) !Token {
1673	const slice = self.takeValueSlice();
1674	if (self.is_end_of_input) {
1675	if (!allow_end) return error.UnexpectedEndOfInput;
1676	self.state = .post_value;
1677	return Token{ .number = slice };
1678	}
1679	if (slice.len == 0) return error.BufferUnderrun;
1680	return Token{ .partial_number = slice };
1681	}
1682
1683	fn partialStringCodepoint(self: *@This()) Token {
1684	const code_point = self.unicode_code_point;
1685	self.unicode_code_point = undefined;
1686	var buf: [4]u8 = undefined;
1687	switch (std.unicode.utf8Encode(code_point, &buf) catch unreachable) {
1688	1 => return Token{ .partial_string_escaped_1 = buf[0..1].* },
1689	2 => return Token{ .partial_string_escaped_2 = buf[0..2].* },
1690	3 => return Token{ .partial_string_escaped_3 = buf[0..3].* },
1691	4 => return Token{ .partial_string_escaped_4 = buf[0..4].* },
1692	else => unreachable,
1693	}
1694	}
1695	};
1696
1697	const OBJECT_MODE = 0;
1698	const ARRAY_MODE = 1;
1699
1700	const BitStack = struct {
1701	bytes: std.ArrayList(u8),
1702	bit_len: u32 = 0,
1703
1704	pub fn init(allocator: Allocator) @This() {
1705	return .{
1706	.bytes = std.ArrayList(u8).init(allocator),
1707	};
1708	}
1709
1710	pub fn deinit(self: *@This()) void {
1711	self.bytes.deinit();
1712	self.* = undefined;
1713	}
1714
1715	pub fn ensureTotalCapacity(self: *@This(), bit_capcity: u32) Allocator.Error!void {
1716	const byte_capacity = (bit_capcity + 7) >> 3;
1717	try self.bytes.ensureTotalCapacity(byte_capacity);
1718	}
1719
1720	pub fn push(self: *@This(), b: u1) Allocator.Error!void {
1721	const byte_index = self.bit_len >> 3;
1722	const bit_index = @intCast(u3, self.bit_len & 7);
1723
1724	if (self.bytes.items.len <= byte_index) {
1725	try self.bytes.append(0);
1726	}
1727
1728	self.bytes.items[byte_index] &= ~(@as(u8, 1) << bit_index);
1729	self.bytes.items[byte_index] \|= @as(u8, b) << bit_index;
1730
1731	self.bit_len += 1;
1732	}
1733
1734	pub fn peek(self: *const @This()) u1 {
1735	const byte_index = (self.bit_len - 1) >> 3;
1736	const bit_index = @intCast(u3, (self.bit_len - 1) & 7);
1737	return @intCast(u1, (self.bytes.items[byte_index] >> bit_index) & 1);
1738	}
1739
1740	pub fn pop(self: *@This()) u1 {
1741	const b = self.peek();
1742	self.bit_len -= 1;
1743	return b;
1744	}
1745	};
1746
1747	fn appendSlice(list: *std.ArrayList(u8), buf: []const u8, max_value_len: usize) !void {
1748	const new_len = std.math.add(usize, list.items.len, buf.len) catch return error.ValueTooLong;
1749	if (new_len > max_value_len) return error.ValueTooLong;
1750	try list.appendSlice(buf);
1751	}
1752
1753	/// For the slice you get from a `Token.number` or `Token.allocated_number`,
1754	/// this function returns true if the number doesn't contain any fraction or exponent components.
1755	/// Note, the numeric value encoded by the value may still be an integer, such as `1.0`.
1756	/// This function is meant to give a hint about whether integer parsing or float parsing should be used on the value.
1757	/// This function will not give meaningful results on non-numeric input.
1758	pub fn isNumberFormattedLikeAnInteger(value: []const u8) bool {
1759	return std.mem.indexOfAny(u8, value, ".eE") == null;
1760	}
1761
1762	test {
1763	_ = @import("./scanner_test.zig");
1764	}

lib/std/json/scanner_test.zig created+466

...	@@ -0,0 +1,466 @@
1	const std = @import("std");
2	const JsonScanner = @import("./scanner.zig").Scanner;
3	const jsonReader = @import("./scanner.zig").reader;
4	const JsonReader = @import("./scanner.zig").Reader;
5	const Token = @import("./scanner.zig").Token;
6	const TokenType = @import("./scanner.zig").TokenType;
7	const Diagnostics = @import("./scanner.zig").Diagnostics;
8	const Error = @import("./scanner.zig").Error;
9	const validate = @import("./scanner.zig").validate;
10
11	const example_document_str =
12	\\{
13	\\ "Image": {
14	\\ "Width": 800,
15	\\ "Height": 600,
16	\\ "Title": "View from 15th Floor",
17	\\ "Thumbnail": {
18	\\ "Url": "http://www.example.com/image/481989943",
19	\\ "Height": 125,
20	\\ "Width": 100
21	\\ },
22	\\ "Animated" : false,
23	\\ "IDs": [116, 943, 234, 38793]
24	\\ }
25	\\}
26	;
27
28	fn expectNext(scanner_or_reader: anytype, expected_token: Token) !void {
29	return expectEqualTokens(expected_token, try scanner_or_reader.next());
30	}
31
32	fn expectPeekNext(scanner_or_reader: anytype, expected_token_type: TokenType, expected_token: Token) !void {
33	try std.testing.expectEqual(expected_token_type, try scanner_or_reader.peekNextTokenType());
34	try expectEqualTokens(expected_token, try scanner_or_reader.next());
35	}
36
37	test "json.token" {
38	var scanner = JsonScanner.initCompleteInput(std.testing.allocator, example_document_str);
39	defer scanner.deinit();
40
41	try expectNext(&scanner, .object_begin);
42	try expectNext(&scanner, Token{ .string = "Image" });
43	try expectNext(&scanner, .object_begin);
44	try expectNext(&scanner, Token{ .string = "Width" });
45	try expectNext(&scanner, Token{ .number = "800" });
46	try expectNext(&scanner, Token{ .string = "Height" });
47	try expectNext(&scanner, Token{ .number = "600" });
48	try expectNext(&scanner, Token{ .string = "Title" });
49	try expectNext(&scanner, Token{ .string = "View from 15th Floor" });
50	try expectNext(&scanner, Token{ .string = "Thumbnail" });
51	try expectNext(&scanner, .object_begin);
52	try expectNext(&scanner, Token{ .string = "Url" });
53	try expectNext(&scanner, Token{ .string = "http://www.example.com/image/481989943" });
54	try expectNext(&scanner, Token{ .string = "Height" });
55	try expectNext(&scanner, Token{ .number = "125" });
56	try expectNext(&scanner, Token{ .string = "Width" });
57	try expectNext(&scanner, Token{ .number = "100" });
58	try expectNext(&scanner, .object_end);
59	try expectNext(&scanner, Token{ .string = "Animated" });
60	try expectNext(&scanner, .false);
61	try expectNext(&scanner, Token{ .string = "IDs" });
62	try expectNext(&scanner, .array_begin);
63	try expectNext(&scanner, Token{ .number = "116" });
64	try expectNext(&scanner, Token{ .number = "943" });
65	try expectNext(&scanner, Token{ .number = "234" });
66	try expectNext(&scanner, Token{ .number = "38793" });
67	try expectNext(&scanner, .array_end);
68	try expectNext(&scanner, .object_end);
69	try expectNext(&scanner, .object_end);
70	try expectNext(&scanner, .end_of_document);
71	}
72
73	const all_types_test_case =
74	\\[
75	\\ "", "a\nb",
76	\\ 0, 0.0, -1.1e-1,
77	\\ true, false, null,
78	\\ {"a": {}},
79	\\ []
80	\\]
81	;
82
83	fn testAllTypes(source: anytype, large_buffer: bool) !void {
84	try expectPeekNext(source, .array_begin, .array_begin);
85	try expectPeekNext(source, .string, Token{ .string = "" });
86	try expectPeekNext(source, .string, Token{ .partial_string = "a" });
87	try expectPeekNext(source, .string, Token{ .partial_string_escaped_1 = "\n".* });
88	if (large_buffer) {
89	try expectPeekNext(source, .string, Token{ .string = "b" });
90	} else {
91	try expectPeekNext(source, .string, Token{ .partial_string = "b" });
92	try expectPeekNext(source, .string, Token{ .string = "" });
93	}
94	if (large_buffer) {
95	try expectPeekNext(source, .number, Token{ .number = "0" });
96	} else {
97	try expectPeekNext(source, .number, Token{ .partial_number = "0" });
98	try expectPeekNext(source, .number, Token{ .number = "" });
99	}
100	if (large_buffer) {
101	try expectPeekNext(source, .number, Token{ .number = "0.0" });
102	} else {
103	try expectPeekNext(source, .number, Token{ .partial_number = "0" });
104	try expectPeekNext(source, .number, Token{ .partial_number = "." });
105	try expectPeekNext(source, .number, Token{ .partial_number = "0" });
106	try expectPeekNext(source, .number, Token{ .number = "" });
107	}
108	if (large_buffer) {
109	try expectPeekNext(source, .number, Token{ .number = "-1.1e-1" });
110	} else {
111	try expectPeekNext(source, .number, Token{ .partial_number = "-" });
112	try expectPeekNext(source, .number, Token{ .partial_number = "1" });
113	try expectPeekNext(source, .number, Token{ .partial_number = "." });
114	try expectPeekNext(source, .number, Token{ .partial_number = "1" });
115	try expectPeekNext(source, .number, Token{ .partial_number = "e" });
116	try expectPeekNext(source, .number, Token{ .partial_number = "-" });
117	try expectPeekNext(source, .number, Token{ .partial_number = "1" });
118	try expectPeekNext(source, .number, Token{ .number = "" });
119	}
120	try expectPeekNext(source, .true, .true);
121	try expectPeekNext(source, .false, .false);
122	try expectPeekNext(source, .null, .null);
123	try expectPeekNext(source, .object_begin, .object_begin);
124	if (large_buffer) {
125	try expectPeekNext(source, .string, Token{ .string = "a" });
126	} else {
127	try expectPeekNext(source, .string, Token{ .partial_string = "a" });
128	try expectPeekNext(source, .string, Token{ .string = "" });
129	}
130	try expectPeekNext(source, .object_begin, .object_begin);
131	try expectPeekNext(source, .object_end, .object_end);
132	try expectPeekNext(source, .object_end, .object_end);
133	try expectPeekNext(source, .array_begin, .array_begin);
134	try expectPeekNext(source, .array_end, .array_end);
135	try expectPeekNext(source, .array_end, .array_end);
136	try expectPeekNext(source, .end_of_document, .end_of_document);
137	}
138
139	test "peek all types" {
140	var scanner = JsonScanner.initCompleteInput(std.testing.allocator, all_types_test_case);
141	defer scanner.deinit();
142	try testAllTypes(&scanner, true);
143
144	var stream = std.io.fixedBufferStream(all_types_test_case);
145	var json_reader = jsonReader(std.testing.allocator, stream.reader());
146	defer json_reader.deinit();
147	try testAllTypes(&json_reader, true);
148
149	var tiny_stream = std.io.fixedBufferStream(all_types_test_case);
150	var tiny_json_reader = JsonReader(1, @TypeOf(tiny_stream.reader())).init(std.testing.allocator, tiny_stream.reader());
151	defer tiny_json_reader.deinit();
152	try testAllTypes(&tiny_json_reader, false);
153	}
154
155	test "json.token mismatched close" {
156	var scanner = JsonScanner.initCompleteInput(std.testing.allocator, "[102, 111, 111 }");
157	defer scanner.deinit();
158	try expectNext(&scanner, .array_begin);
159	try expectNext(&scanner, Token{ .number = "102" });
160	try expectNext(&scanner, Token{ .number = "111" });
161	try expectNext(&scanner, Token{ .number = "111" });
162	try std.testing.expectError(error.SyntaxError, scanner.next());
163	}
164
165	test "json.token premature object close" {
166	var scanner = JsonScanner.initCompleteInput(std.testing.allocator, "{ \"key\": }");
167	defer scanner.deinit();
168	try expectNext(&scanner, .object_begin);
169	try expectNext(&scanner, Token{ .string = "key" });
170	try std.testing.expectError(error.SyntaxError, scanner.next());
171	}
172
173	test "JsonScanner basic" {
174	var scanner = JsonScanner.initCompleteInput(std.testing.allocator, example_document_str);
175	defer scanner.deinit();
176
177	while (true) {
178	const token = try scanner.next();
179	if (token == .end_of_document) break;
180	}
181	}
182
183	test "JsonReader basic" {
184	var stream = std.io.fixedBufferStream(example_document_str);
185
186	var json_reader = jsonReader(std.testing.allocator, stream.reader());
187	defer json_reader.deinit();
188
189	while (true) {
190	const token = try json_reader.next();
191	if (token == .end_of_document) break;
192	}
193	}
194
195	const number_test_stems = .{
196	.{ "", "-" },
197	.{ "0", "1", "10", "9999999999999999999999999" },
198	.{ "", ".0", ".999999999999999999999999" },
199	.{ "", "e0", "E0", "e+0", "e-0", "e9999999999999999999999999999" },
200	};
201	const number_test_items = blk: {
202	comptime var ret: []const []const u8 = &[_][]const u8{};
203	for (number_test_stems[0]) \|s0\| {
204	for (number_test_stems[1]) \|s1\| {
205	for (number_test_stems[2]) \|s2\| {
206	for (number_test_stems[3]) \|s3\| {
207	ret = ret ++ &[_][]const u8{s0 ++ s1 ++ s2 ++ s3};
208	}
209	}
210	}
211	}
212	break :blk ret;
213	};
214
215	test "numbers" {
216	for (number_test_items) \|number_str\| {
217	var scanner = JsonScanner.initCompleteInput(std.testing.allocator, number_str);
218	defer scanner.deinit();
219
220	const token = try scanner.next();
221	const value = token.number; // assert this is a number
222	try std.testing.expectEqualStrings(number_str, value);
223
224	try std.testing.expectEqual(Token.end_of_document, try scanner.next());
225	}
226	}
227
228	const string_test_cases = .{
229	// The left is JSON without the "quotes".
230	// The right is the expected unescaped content.
231	.{ "", "" },
232	.{ "\\\\", "\\" },
233	.{ "a\\\\b", "a\\b" },
234	.{ "a\\\"b", "a\"b" },
235	.{ "\\n", "\n" },
236	.{ "\\u000a", "\n" },
237	.{ "𝄞", "\u{1D11E}" },
238	.{ "\\uD834\\uDD1E", "\u{1D11E}" },
239	.{ "\\uff20", "＠" },
240	};
241
242	test "strings" {
243	inline for (string_test_cases) \|tuple\| {
244	var stream = std.io.fixedBufferStream("\"" ++ tuple[0] ++ "\"");
245	var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
246	defer arena.deinit();
247	var json_reader = jsonReader(std.testing.allocator, stream.reader());
248	defer json_reader.deinit();
249
250	const token = try json_reader.nextAlloc(arena.allocator(), .alloc_if_needed);
251	const value = switch (token) {
252	.string => \|value\| value,
253	.allocated_string => \|value\| value,
254	else => return error.ExpectedString,
255	};
256	try std.testing.expectEqualStrings(tuple[1], value);
257
258	try std.testing.expectEqual(Token.end_of_document, try json_reader.next());
259	}
260	}
261
262	const nesting_test_cases = .{
263	.{ null, "[]" },
264	.{ null, "{}" },
265	.{ error.SyntaxError, "[}" },
266	.{ error.SyntaxError, "{]" },
267	.{ null, "[" 1000 ++ "]" 1000 },
268	.{ null, "{\"\":" 1000 ++ "0" ++ "}" 1000 },
269	.{ error.SyntaxError, "[" 1000 ++ "]" 999 ++ "}" },
270	.{ error.SyntaxError, "{\"\":" 1000 ++ "0" ++ "}" 999 ++ "]" },
271	.{ error.SyntaxError, "[" 1000 ++ "]" 1001 },
272	.{ error.SyntaxError, "{\"\":" 1000 ++ "0" ++ "}" 1001 },
273	.{ error.UnexpectedEndOfInput, "[" 1000 ++ "]" 999 },
274	.{ error.UnexpectedEndOfInput, "{\"\":" 1000 ++ "0" ++ "}" 999 },
275	};
276
277	test "nesting" {
278	inline for (nesting_test_cases) \|tuple\| {
279	const maybe_error = tuple[0];
280	const document_str = tuple[1];
281
282	expectMaybeError(document_str, maybe_error) catch \|err\| {
283	std.debug.print("in json document: {s}\n", .{document_str});
284	return err;
285	};
286	}
287	}
288
289	fn expectMaybeError(document_str: []const u8, maybe_error: ?Error) !void {
290	var scanner = JsonScanner.initCompleteInput(std.testing.allocator, document_str);
291	defer scanner.deinit();
292
293	while (true) {
294	const token = scanner.next() catch \|err\| {
295	if (maybe_error) \|expected_err\| {
296	if (err == expected_err) return;
297	}
298	return err;
299	};
300	if (token == .end_of_document) break;
301	}
302	if (maybe_error != null) return error.ExpectedError;
303	}
304
305	fn expectEqualTokens(expected_token: Token, actual_token: Token) !void {
306	try std.testing.expectEqual(std.meta.activeTag(expected_token), std.meta.activeTag(actual_token));
307	switch (expected_token) {
308	.number => \|expected_value\| {
309	try std.testing.expectEqualStrings(expected_value, actual_token.number);
310	},
311	.string => \|expected_value\| {
312	try std.testing.expectEqualStrings(expected_value, actual_token.string);
313	},
314	else => {},
315	}
316	}
317
318	fn testTinyBufferSize(document_str: []const u8) !void {
319	var tiny_stream = std.io.fixedBufferStream(document_str);
320	var normal_stream = std.io.fixedBufferStream(document_str);
321
322	var tiny_json_reader = JsonReader(1, @TypeOf(tiny_stream.reader())).init(std.testing.allocator, tiny_stream.reader());
323	defer tiny_json_reader.deinit();
324	var normal_json_reader = JsonReader(0x1000, @TypeOf(normal_stream.reader())).init(std.testing.allocator, normal_stream.reader());
325	defer normal_json_reader.deinit();
326
327	expectEqualStreamOfTokens(&normal_json_reader, &tiny_json_reader) catch \|err\| {
328	std.debug.print("in json document: {s}\n", .{document_str});
329	return err;
330	};
331	}
332	fn expectEqualStreamOfTokens(control_json_reader: anytype, test_json_reader: anytype) !void {
333	var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
334	defer arena.deinit();
335	while (true) {
336	const control_token = try control_json_reader.nextAlloc(arena.allocator(), .alloc_always);
337	const test_token = try test_json_reader.nextAlloc(arena.allocator(), .alloc_always);
338	try expectEqualTokens(control_token, test_token);
339	if (control_token == .end_of_document) break;
340	_ = arena.reset(.retain_capacity);
341	}
342	}
343
344	test "BufferUnderrun" {
345	try testTinyBufferSize(example_document_str);
346	for (number_test_items) \|number_str\| {
347	try testTinyBufferSize(number_str);
348	}
349	inline for (string_test_cases) \|tuple\| {
350	try testTinyBufferSize("\"" ++ tuple[0] ++ "\"");
351	}
352	}
353
354	test "json.validate" {
355	try std.testing.expectEqual(true, try validate(std.testing.allocator, "{}"));
356	try std.testing.expectEqual(true, try validate(std.testing.allocator, "[]"));
357	try std.testing.expectEqual(false, try validate(std.testing.allocator, "[{[[[[{}]]]]}]"));
358	try std.testing.expectEqual(false, try validate(std.testing.allocator, "{]"));
359	try std.testing.expectEqual(false, try validate(std.testing.allocator, "[}"));
360	try std.testing.expectEqual(false, try validate(std.testing.allocator, "{{{{[]}}}]"));
361	}
362
363	fn testSkipValue(s: []const u8) !void {
364	var scanner = JsonScanner.initCompleteInput(std.testing.allocator, s);
365	defer scanner.deinit();
366	try scanner.skipValue();
367	try expectEqualTokens(.end_of_document, try scanner.next());
368
369	var stream = std.io.fixedBufferStream(s);
370	var json_reader = jsonReader(std.testing.allocator, stream.reader());
371	defer json_reader.deinit();
372	try json_reader.skipValue();
373	try expectEqualTokens(.end_of_document, try json_reader.next());
374	}
375
376	test "skipValue" {
377	try testSkipValue("false");
378	try testSkipValue("true");
379	try testSkipValue("null");
380	try testSkipValue("42");
381	try testSkipValue("42.0");
382	try testSkipValue("\"foo\"");
383	try testSkipValue("[101, 111, 121]");
384	try testSkipValue("{}");
385	try testSkipValue("{\"foo\": \"bar\\nbaz\"}");
386
387	// An absurd number of nestings
388	const nestings = 1000;
389	try testSkipValue("[" nestings ++ "]" nestings);
390
391	// Would a number token cause problems in a deeply-nested array?
392	try testSkipValue("[" nestings ++ "0.118, 999, 881.99, 911.9, 725, 3" ++ "]" nestings);
393
394	// Mismatched brace/square bracket
395	try std.testing.expectError(error.SyntaxError, testSkipValue("[102, 111, 111}"));
396	}
397
398	fn testEnsureStackCapacity(do_ensure: bool) !void {
399	var fail_alloc = std.testing.FailingAllocator.init(std.testing.allocator, 1);
400	const failing_allocator = fail_alloc.allocator();
401
402	const nestings = 999; // intentionally not a power of 2.
403	var scanner = JsonScanner.initCompleteInput(failing_allocator, "[" nestings ++ "]" nestings);
404	defer scanner.deinit();
405
406	if (do_ensure) {
407	try scanner.ensureTotalStackCapacity(nestings);
408	}
409
410	try scanner.skipValue();
411	try std.testing.expectEqual(Token.end_of_document, try scanner.next());
412	}
413	test "ensureTotalStackCapacity" {
414	// Once to demonstrate failure.
415	try std.testing.expectError(error.OutOfMemory, testEnsureStackCapacity(false));
416	// Then to demonstrate it works.
417	try testEnsureStackCapacity(true);
418	}
419
420	fn testDiagnosticsFromSource(expected_error: ?anyerror, line: u64, col: u64, byte_offset: u64, source: anytype) !void {
421	var diagnostics = Diagnostics{};
422	source.enableDiagnostics(&diagnostics);
423
424	if (expected_error) \|expected_err\| {
425	try std.testing.expectError(expected_err, source.skipValue());
426	} else {
427	try source.skipValue();
428	try std.testing.expectEqual(Token.end_of_document, try source.next());
429	}
430	try std.testing.expectEqual(line, diagnostics.getLine());
431	try std.testing.expectEqual(col, diagnostics.getColumn());
432	try std.testing.expectEqual(byte_offset, diagnostics.getByteOffset());
433	}
434	fn testDiagnostics(expected_error: ?anyerror, line: u64, col: u64, byte_offset: u64, s: []const u8) !void {
435	var scanner = JsonScanner.initCompleteInput(std.testing.allocator, s);
436	defer scanner.deinit();
437	try testDiagnosticsFromSource(expected_error, line, col, byte_offset, &scanner);
438
439	var tiny_stream = std.io.fixedBufferStream(s);
440	var tiny_json_reader = JsonReader(1, @TypeOf(tiny_stream.reader())).init(std.testing.allocator, tiny_stream.reader());
441	defer tiny_json_reader.deinit();
442	try testDiagnosticsFromSource(expected_error, line, col, byte_offset, &tiny_json_reader);
443
444	var medium_stream = std.io.fixedBufferStream(s);
445	var medium_json_reader = JsonReader(5, @TypeOf(medium_stream.reader())).init(std.testing.allocator, medium_stream.reader());
446	defer medium_json_reader.deinit();
447	try testDiagnosticsFromSource(expected_error, line, col, byte_offset, &medium_json_reader);
448	}
449	test "enableDiagnostics" {
450	try testDiagnostics(error.UnexpectedEndOfInput, 1, 1, 0, "");
451	try testDiagnostics(null, 1, 3, 2, "[]");
452	try testDiagnostics(null, 2, 2, 3, "[\n]");
453	try testDiagnostics(null, 14, 2, example_document_str.len, example_document_str);
454
455	try testDiagnostics(error.SyntaxError, 3, 1, 25,
456	\\{
457	\\ "common": "mistake",
458	\\}
459	);
460
461	inline for ([_]comptime_int{ 5, 6, 7, 99 }) \|reps\| {
462	// The error happens 1 byte before the end.
463	const s = "[" ** reps ++ "}";
464	try testDiagnostics(error.SyntaxError, 1, s.len, s.len - 1, s);
465	}
466	}

lib/std/json/static.zig created+621

...	@@ -0,0 +1,621 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const Allocator = std.mem.Allocator;
4	const ArrayList = std.ArrayList;
5
6	const Scanner = @import("./scanner.zig").Scanner;
7	const Token = @import("./scanner.zig").Token;
8	const AllocWhen = @import("./scanner.zig").AllocWhen;
9	const default_max_value_len = @import("./scanner.zig").default_max_value_len;
10	const isNumberFormattedLikeAnInteger = @import("./scanner.zig").isNumberFormattedLikeAnInteger;
11
12	pub const ParseOptions = struct {
13	/// Behaviour when a duplicate field is encountered.
14	duplicate_field_behavior: enum {
15	use_first,
16	@"error",
17	use_last,
18	} = .@"error",
19
20	/// If false, finding an unknown field returns an error.
21	ignore_unknown_fields: bool = false,
22
23	/// Passed to json.Scanner.nextAllocMax() or json.Reader.nextAllocMax().
24	/// The default for parseFromSlice() or parseFromTokenSource() with a *json.Scanner input
25	/// is the length of the input slice, which means error.ValueTooLong will never be returned.
26	/// The default for parseFromTokenSource() with a *json.Reader is default_max_value_len.
27	max_value_len: ?usize = null,
28	};
29
30	/// Parses the json document from s and returns the result.
31	/// The provided allocator is used both for temporary allocations during parsing the document,
32	/// and also to allocate any pointer values in the return type.
33	/// If T contains any pointers, free the memory with `std.json.parseFree`.
34	/// Note that `error.BufferUnderrun` is not actually possible to return from this function.
35	pub fn parseFromSlice(comptime T: type, allocator: Allocator, s: []const u8, options: ParseOptions) ParseError(T, Scanner)!T {
36	var scanner = Scanner.initCompleteInput(allocator, s);
37	defer scanner.deinit();
38
39	return parseFromTokenSource(T, allocator, &scanner, options);
40	}
41
42	/// `scanner_or_reader` must be either a `std.json.Scanner` with complete input or a `std.json.Reader`.
43	/// allocator is used to allocate the data of T if necessary,
44	/// such as if T is `*u32` or `[]u32`.
45	/// If T contains any pointers, free the memory with `std.json.parseFree`.
46	/// If T contains no pointers, the allocator may sometimes be used for temporary allocations,
47	/// but no call to `std.json.parseFree` will be necessary;
48	/// all temporary allocations will be freed before this function returns.
49	/// Note that `error.BufferUnderrun` is not actually possible to return from this function.
50	pub fn parseFromTokenSource(comptime T: type, allocator: Allocator, scanner_or_reader: anytype, options: ParseOptions) ParseError(T, @TypeOf(scanner_or_reader.*))!T {
51	if (@TypeOf(scanner_or_reader.*) == Scanner) {
52	assert(scanner_or_reader.is_end_of_input);
53	}
54
55	var resolved_options = options;
56	if (resolved_options.max_value_len == null) {
57	if (@TypeOf(scanner_or_reader.*) == Scanner) {
58	resolved_options.max_value_len = scanner_or_reader.input.len;
59	} else {
60	resolved_options.max_value_len = default_max_value_len;
61	}
62	}
63
64	const r = try parseInternal(T, allocator, scanner_or_reader, resolved_options);
65	errdefer parseFree(T, allocator, r);
66
67	assert(.end_of_document == try scanner_or_reader.next());
68
69	return r;
70	}
71
72	/// The error set that will be returned from parsing T from *Source.
73	/// Note that this may contain error.BufferUnderrun, but that error will never actually be returned.
74	pub fn ParseError(comptime T: type, comptime Source: type) type {
75	// `inferred_types` is used to avoid infinite recursion for recursive type definitions.
76	const inferred_types = [_]type{};
77	// A few of these will either always be present or present enough of the time that
78	// omitting them is more confusing than always including them.
79	return error{UnexpectedToken} \|\| Source.NextError \|\| Source.PeekError \|\|
80	ParseInternalErrorImpl(T, Source, &inferred_types);
81	}
82
83	fn ParseInternalErrorImpl(comptime T: type, comptime Source: type, comptime inferred_types: []const type) type {
84	for (inferred_types) \|ty\| {
85	if (T == ty) return error{};
86	}
87
88	switch (@typeInfo(T)) {
89	.Bool => return error{},
90	.Float, .ComptimeFloat => return Source.AllocError \|\| std.fmt.ParseFloatError,
91	.Int, .ComptimeInt => {
92	return Source.AllocError \|\| error{ InvalidNumber, Overflow } \|\|
93	std.fmt.ParseIntError \|\| std.fmt.ParseFloatError;
94	},
95	.Optional => \|optional_info\| return ParseInternalErrorImpl(optional_info.child, Source, inferred_types ++ [_]type{T}),
96	.Enum => return Source.AllocError \|\| error{InvalidEnumTag},
97	.Union => \|unionInfo\| {
98	if (unionInfo.tag_type) \|_\| {
99	var errors = Source.AllocError \|\| error{UnknownField};
100	for (unionInfo.fields) \|u_field\| {
101	errors = errors \|\| ParseInternalErrorImpl(u_field.type, Source, inferred_types ++ [_]type{T});
102	}
103	return errors;
104	} else {
105	@compileError("Unable to parse into untagged union '" ++ @typeName(T) ++ "'");
106	}
107	},
108	.Struct => \|structInfo\| {
109	var errors = Scanner.AllocError \|\| error{
110	DuplicateField,
111	UnknownField,
112	MissingField,
113	};
114	for (structInfo.fields) \|field\| {
115	errors = errors \|\| ParseInternalErrorImpl(field.type, Source, inferred_types ++ [_]type{T});
116	}
117	return errors;
118	},
119	.Array => \|arrayInfo\| {
120	return error{LengthMismatch} \|\|
121	ParseInternalErrorImpl(arrayInfo.child, Source, inferred_types ++ [_]type{T});
122	},
123	.Vector => \|vecInfo\| {
124	return error{LengthMismatch} \|\|
125	ParseInternalErrorImpl(vecInfo.child, Source, inferred_types ++ [_]type{T});
126	},
127	.Pointer => \|ptrInfo\| {
128	switch (ptrInfo.size) {
129	.One, .Slice => {
130	return ParseInternalErrorImpl(ptrInfo.child, Source, inferred_types ++ [_]type{T});
131	},
132	else => @compileError("Unable to parse into type '" ++ @typeName(T) ++ "'"),
133	}
134	},
135	else => return error{},
136	}
137	unreachable;
138	}
139
140	fn parseInternal(
141	comptime T: type,
142	allocator: Allocator,
143	source: anytype,
144	options: ParseOptions,
145	) ParseError(T, @TypeOf(source.*))!T {
146	switch (@typeInfo(T)) {
147	.Bool => {
148	return switch (try source.next()) {
149	.true => true,
150	.false => false,
151	else => error.UnexpectedToken,
152	};
153	},
154	.Float, .ComptimeFloat => {
155	const token = try source.nextAllocMax(allocator, .alloc_if_needed, options.max_value_len.?);
156	defer freeAllocated(allocator, token);
157	const slice = switch (token) {
158	.number, .string => \|slice\| slice,
159	.allocated_number, .allocated_string => \|slice\| slice,
160	else => return error.UnexpectedToken,
161	};
162	return try std.fmt.parseFloat(T, slice);
163	},
164	.Int, .ComptimeInt => {
165	const token = try source.nextAllocMax(allocator, .alloc_if_needed, options.max_value_len.?);
166	defer freeAllocated(allocator, token);
167	const slice = switch (token) {
168	.number, .string => \|slice\| slice,
169	.allocated_number, .allocated_string => \|slice\| slice,
170	else => return error.UnexpectedToken,
171	};
172	if (isNumberFormattedLikeAnInteger(slice))
173	return std.fmt.parseInt(T, slice, 10);
174	// Try to coerce a float to an integer.
175	const float = try std.fmt.parseFloat(f128, slice);
176	if (@round(float) != float) return error.InvalidNumber;
177	if (float > std.math.maxInt(T) or float < std.math.minInt(T)) return error.Overflow;
178	return @floatToInt(T, float);
179	},
180	.Optional => \|optionalInfo\| {
181	switch (try source.peekNextTokenType()) {
182	.null => {
183	_ = try source.next();
184	return null;
185	},
186	else => {
187	return try parseInternal(optionalInfo.child, allocator, source, options);
188	},
189	}
190	},
191	.Enum => \|enumInfo\| {
192	const token = try source.nextAllocMax(allocator, .alloc_if_needed, options.max_value_len.?);
193	defer freeAllocated(allocator, token);
194	const slice = switch (token) {
195	.number, .string => \|slice\| slice,
196	.allocated_number, .allocated_string => \|slice\| slice,
197	else => return error.UnexpectedToken,
198	};
199	// Check for a named value.
200	if (std.meta.stringToEnum(T, slice)) \|value\| return value;
201	// Check for a numeric value.
202	if (!isNumberFormattedLikeAnInteger(slice)) return error.InvalidEnumTag;
203	const n = std.fmt.parseInt(enumInfo.tag_type, slice, 10) catch return error.InvalidEnumTag;
204	return try std.meta.intToEnum(T, n);
205	},
206	.Union => \|unionInfo\| {
207	const UnionTagType = unionInfo.tag_type orelse @compileError("Unable to parse into untagged union '" ++ @typeName(T) ++ "'");
208
209	if (.object_begin != try source.next()) return error.UnexpectedToken;
210
211	var result: ?T = null;
212	errdefer {
213	if (result) \|r\| {
214	inline for (unionInfo.fields) \|u_field\| {
215	if (r == @field(UnionTagType, u_field.name)) {
216	parseFree(u_field.type, allocator, @field(r, u_field.name));
217	}
218	}
219	}
220	}
221
222	var name_token: ?Token = try source.nextAllocMax(allocator, .alloc_if_needed, options.max_value_len.?);
223	errdefer {
224	if (name_token) \|t\| {
225	freeAllocated(allocator, t);
226	}
227	}
228	const field_name = switch (name_token.?) {
229	.string => \|slice\| slice,
230	.allocated_string => \|slice\| slice,
231	else => return error.UnexpectedToken,
232	};
233
234	inline for (unionInfo.fields) \|u_field\| {
235	if (std.mem.eql(u8, u_field.name, field_name)) {
236	// Free the name token now in case we're using an allocator that optimizes freeing the last allocated object.
237	// (Recursing into parseInternal() might trigger more allocations.)
238	freeAllocated(allocator, name_token.?);
239	name_token = null;
240
241	if (u_field.type == void) {
242	// void isn't really a json type, but we can support void payload union tags with {} as a value.
243	if (.object_begin != try source.next()) return error.UnexpectedToken;
244	if (.object_end != try source.next()) return error.UnexpectedToken;
245	result = @unionInit(T, u_field.name, {});
246	} else {
247	// Recurse.
248	result = @unionInit(T, u_field.name, try parseInternal(u_field.type, allocator, source, options));
249	}
250	break;
251	}
252	} else {
253	// Didn't match anything.
254	return error.UnknownField;
255	}
256
257	if (.object_end != try source.next()) return error.UnexpectedToken;
258
259	return result.?;
260	},
261
262	.Struct => \|structInfo\| {
263	if (structInfo.is_tuple) {
264	if (.array_begin != try source.next()) return error.UnexpectedToken;
265
266	var r: T = undefined;
267	var fields_seen: usize = 0;
268	errdefer {
269	inline for (0..structInfo.fields.len) \|i\| {
270	if (i < fields_seen) {
271	parseFree(structInfo.fields[i].type, allocator, r[i]);
272	}
273	}
274	}
275	inline for (0..structInfo.fields.len) \|i\| {
276	r[i] = try parseInternal(structInfo.fields[i].type, allocator, source, options);
277	fields_seen = i + 1;
278	}
279
280	if (.array_end != try source.next()) return error.UnexpectedToken;
281
282	return r;
283	}
284
285	if (.object_begin != try source.next()) return error.UnexpectedToken;
286
287	var r: T = undefined;
288	var fields_seen = [_]bool{false} ** structInfo.fields.len;
289	errdefer {
290	inline for (structInfo.fields, 0..) \|field, i\| {
291	if (fields_seen[i]) {
292	parseFree(field.type, allocator, @field(r, field.name));
293	}
294	}
295	}
296
297	while (true) {
298	var name_token: ?Token = try source.nextAllocMax(allocator, .alloc_if_needed, options.max_value_len.?);
299	errdefer {
300	if (name_token) \|t\| {
301	freeAllocated(allocator, t);
302	}
303	}
304	const field_name = switch (name_token.?) {
305	.object_end => break, // No more fields.
306	.string => \|slice\| slice,
307	.allocated_string => \|slice\| slice,
308	else => return error.UnexpectedToken,
309	};
310
311	inline for (structInfo.fields, 0..) \|field, i\| {
312	if (field.is_comptime) @compileError("comptime fields are not supported: " ++ @typeName(T) ++ "." ++ field.name);
313	if (std.mem.eql(u8, field.name, field_name)) {
314	// Free the name token now in case we're using an allocator that optimizes freeing the last allocated object.
315	// (Recursing into parseInternal() might trigger more allocations.)
316	freeAllocated(allocator, name_token.?);
317	name_token = null;
318
319	if (fields_seen[i]) {
320	switch (options.duplicate_field_behavior) {
321	.use_first => {
322	// Parse and then delete the redundant value.
323	// We don't want to skip the value, because we want type checking.
324	const ignored_value = try parseInternal(field.type, allocator, source, options);
325	parseFree(field.type, allocator, ignored_value);
326	break;
327	},
328	.@"error" => return error.DuplicateField,
329	.use_last => {
330	// Delete the stale value. We're about to get a new one.
331	parseFree(field.type, allocator, @field(r, field.name));
332	fields_seen[i] = false;
333	},
334	}
335	}
336	@field(r, field.name) = try parseInternal(field.type, allocator, source, options);
337	fields_seen[i] = true;
338	break;
339	}
340	} else {
341	// Didn't match anything.
342	freeAllocated(allocator, name_token.?);
343	if (options.ignore_unknown_fields) {
344	try source.skipValue();
345	} else {
346	return error.UnknownField;
347	}
348	}
349	}
350	inline for (structInfo.fields, 0..) \|field, i\| {
351	if (!fields_seen[i]) {
352	if (field.default_value) \|default_ptr\| {
353	const default = @ptrCast(align(1) const field.type, default_ptr).;
354	@field(r, field.name) = default;
355	} else {
356	return error.MissingField;
357	}
358	}
359	}
360	return r;
361	},
362
363	.Array => \|arrayInfo\| {
364	switch (try source.peekNextTokenType()) {
365	.array_begin => {
366	// Typical array.
367	return parseInternalArray(T, arrayInfo.child, arrayInfo.len, allocator, source, options);
368	},
369	.string => {
370	if (arrayInfo.child != u8) return error.UnexpectedToken;
371	// Fixed-length string.
372
373	var r: T = undefined;
374	var i: usize = 0;
375	while (true) {
376	switch (try source.next()) {
377	.string => \|slice\| {
378	if (i + slice.len != r.len) return error.LengthMismatch;
379	@memcpy(r[i..][0..slice.len], slice);
380	break;
381	},
382	.partial_string => \|slice\| {
383	if (i + slice.len > r.len) return error.LengthMismatch;
384	@memcpy(r[i..][0..slice.len], slice);
385	i += slice.len;
386	},
387	.partial_string_escaped_1 => \|arr\| {
388	if (i + arr.len > r.len) return error.LengthMismatch;
389	@memcpy(r[i..][0..arr.len], arr[0..]);
390	i += arr.len;
391	},
392	.partial_string_escaped_2 => \|arr\| {
393	if (i + arr.len > r.len) return error.LengthMismatch;
394	@memcpy(r[i..][0..arr.len], arr[0..]);
395	i += arr.len;
396	},
397	.partial_string_escaped_3 => \|arr\| {
398	if (i + arr.len > r.len) return error.LengthMismatch;
399	@memcpy(r[i..][0..arr.len], arr[0..]);
400	i += arr.len;
401	},
402	.partial_string_escaped_4 => \|arr\| {
403	if (i + arr.len > r.len) return error.LengthMismatch;
404	@memcpy(r[i..][0..arr.len], arr[0..]);
405	i += arr.len;
406	},
407	else => unreachable,
408	}
409	}
410
411	return r;
412	},
413
414	else => return error.UnexpectedToken,
415	}
416	},
417
418	.Vector => \|vecInfo\| {
419	switch (try source.peekNextTokenType()) {
420	.array_begin => {
421	return parseInternalArray(T, vecInfo.child, vecInfo.len, allocator, source, options);
422	},
423	else => return error.UnexpectedToken,
424	}
425	},
426
427	.Pointer => \|ptrInfo\| {
428	switch (ptrInfo.size) {
429	.One => {
430	const r: *ptrInfo.child = try allocator.create(ptrInfo.child);
431	errdefer allocator.destroy(r);
432	r.* = try parseInternal(ptrInfo.child, allocator, source, options);
433	return r;
434	},
435	.Slice => {
436	switch (try source.peekNextTokenType()) {
437	.array_begin => {
438	_ = try source.next();
439
440	// Typical array.
441	var arraylist = ArrayList(ptrInfo.child).init(allocator);
442	errdefer {
443	while (arraylist.popOrNull()) \|v\| {
444	parseFree(ptrInfo.child, allocator, v);
445	}
446	arraylist.deinit();
447	}
448
449	while (true) {
450	switch (try source.peekNextTokenType()) {
451	.array_end => {
452	_ = try source.next();
453	break;
454	},
455	else => {},
456	}
457
458	try arraylist.ensureUnusedCapacity(1);
459	arraylist.appendAssumeCapacity(try parseInternal(ptrInfo.child, allocator, source, options));
460	}
461
462	if (ptrInfo.sentinel) \|some\| {
463	const sentinel_value = @ptrCast(align(1) const ptrInfo.child, some).;
464	return try arraylist.toOwnedSliceSentinel(sentinel_value);
465	}
466
467	return try arraylist.toOwnedSlice();
468	},
469	.string => {
470	if (ptrInfo.child != u8) return error.UnexpectedToken;
471
472	// Dynamic length string.
473	if (ptrInfo.sentinel) \|sentinel_ptr\| {
474	// Use our own array list so we can append the sentinel.
475	var value_list = ArrayList(u8).init(allocator);
476	errdefer value_list.deinit();
477	_ = try source.allocNextIntoArrayList(&value_list, .alloc_always);
478	return try value_list.toOwnedSliceSentinel(@ptrCast(const u8, sentinel_ptr).);
479	}
480	switch (try source.nextAllocMax(allocator, .alloc_always, options.max_value_len.?)) {
481	.allocated_string => \|slice\| return slice,
482	else => unreachable,
483	}
484	},
485	else => return error.UnexpectedToken,
486	}
487	},
488	else => @compileError("Unable to parse into type '" ++ @typeName(T) ++ "'"),
489	}
490	},
491	else => @compileError("Unable to parse into type '" ++ @typeName(T) ++ "'"),
492	}
493	unreachable;
494	}
495
496	fn parseInternalArray(
497	comptime T: type,
498	comptime Child: type,
499	comptime len: comptime_int,
500	allocator: Allocator,
501	source: anytype,
502	options: ParseOptions,
503	) !T {
504	assert(.array_begin == try source.next());
505
506	var r: T = undefined;
507	var i: usize = 0;
508	errdefer {
509	// Without the len check `r[i]` is not allowed
510	if (len > 0) while (true) : (i -= 1) {
511	parseFree(Child, allocator, r[i]);
512	if (i == 0) break;
513	};
514	}
515	while (i < len) : (i += 1) {
516	r[i] = try parseInternal(Child, allocator, source, options);
517	}
518
519	if (.array_end != try source.next()) return error.UnexpectedToken;
520
521	return r;
522	}
523
524	fn freeAllocated(allocator: Allocator, token: Token) void {
525	switch (token) {
526	.allocated_number, .allocated_string => \|slice\| {
527	allocator.free(slice);
528	},
529	else => {},
530	}
531	}
532
533	/// Releases resources created by parseFromSlice() or parseFromTokenSource().
534	pub fn parseFree(comptime T: type, allocator: Allocator, value: T) void {
535	switch (@typeInfo(T)) {
536	.Bool, .Float, .ComptimeFloat, .Int, .ComptimeInt, .Enum => {},
537	.Optional => {
538	if (value) \|v\| {
539	return parseFree(@TypeOf(v), allocator, v);
540	}
541	},
542	.Union => \|unionInfo\| {
543	if (unionInfo.tag_type) \|UnionTagType\| {
544	inline for (unionInfo.fields) \|u_field\| {
545	if (value == @field(UnionTagType, u_field.name)) {
546	parseFree(u_field.type, allocator, @field(value, u_field.name));
547	break;
548	}
549	}
550	} else {
551	unreachable;
552	}
553	},
554	.Struct => \|structInfo\| {
555	inline for (structInfo.fields) \|field\| {
556	var should_free = true;
557	if (field.default_value) \|default\| {
558	switch (@typeInfo(field.type)) {
559	// We must not attempt to free pointers to struct default values
560	.Pointer => \|fieldPtrInfo\| {
561	const field_value = @field(value, field.name);
562	const field_ptr = switch (fieldPtrInfo.size) {
563	.One => field_value,
564	.Slice => field_value.ptr,
565	else => unreachable, // Other pointer types are not parseable
566	};
567	const field_addr = @ptrToInt(field_ptr);
568
569	const casted_default = @ptrCast(const field.type, @alignCast(@alignOf(field.type), default)).;
570	const default_ptr = switch (fieldPtrInfo.size) {
571	.One => casted_default,
572	.Slice => casted_default.ptr,
573	else => unreachable, // Other pointer types are not parseable
574	};
575	const default_addr = @ptrToInt(default_ptr);
576
577	if (field_addr == default_addr) {
578	should_free = false;
579	}
580	},
581	else => {},
582	}
583	}
584	if (should_free) {
585	parseFree(field.type, allocator, @field(value, field.name));
586	}
587	}
588	},
589	.Array => \|arrayInfo\| {
590	for (value) \|v\| {
591	parseFree(arrayInfo.child, allocator, v);
592	}
593	},
594	.Vector => \|vecInfo\| {
595	var i: usize = 0;
596	while (i < vecInfo.len) : (i += 1) {
597	parseFree(vecInfo.child, allocator, value[i]);
598	}
599	},
600	.Pointer => \|ptrInfo\| {
601	switch (ptrInfo.size) {
602	.One => {
603	parseFree(ptrInfo.child, allocator, value.*);
604	allocator.destroy(value);
605	},
606	.Slice => {
607	for (value) \|v\| {
608	parseFree(ptrInfo.child, allocator, v);
609	}
610	allocator.free(value);
611	},
612	else => unreachable,
613	}
614	},
615	else => unreachable,
616	}
617	}
618
619	test {
620	_ = @import("./static_test.zig");
621	}

lib/std/json/static_test.zig created+437

...	@@ -0,0 +1,437 @@
1	const std = @import("std");
2	const testing = std.testing;
3
4	const parseFromSlice = @import("./static.zig").parseFromSlice;
5	const parseFromTokenSource = @import("./static.zig").parseFromTokenSource;
6	const parseFree = @import("./static.zig").parseFree;
7	const ParseOptions = @import("./static.zig").ParseOptions;
8	const JsonScanner = @import("./scanner.zig").Scanner;
9	const jsonReader = @import("./scanner.zig").reader;
10
11	test "parse" {
12	try testing.expectEqual(false, try parseFromSlice(bool, testing.allocator, "false", .{}));
13	try testing.expectEqual(true, try parseFromSlice(bool, testing.allocator, "true", .{}));
14	try testing.expectEqual(@as(u1, 1), try parseFromSlice(u1, testing.allocator, "1", .{}));
15	try testing.expectError(error.Overflow, parseFromSlice(u1, testing.allocator, "50", .{}));
16	try testing.expectEqual(@as(u64, 42), try parseFromSlice(u64, testing.allocator, "42", .{}));
17	try testing.expectEqual(@as(f64, 42), try parseFromSlice(f64, testing.allocator, "42.0", .{}));
18	try testing.expectEqual(@as(?bool, null), try parseFromSlice(?bool, testing.allocator, "null", .{}));
19	try testing.expectEqual(@as(?bool, true), try parseFromSlice(?bool, testing.allocator, "true", .{}));
20
21	try testing.expectEqual(@as([3]u8, "foo".*), try parseFromSlice([3]u8, testing.allocator, "\"foo\"", .{}));
22	try testing.expectEqual(@as([3]u8, "foo".*), try parseFromSlice([3]u8, testing.allocator, "[102, 111, 111]", .{}));
23	try testing.expectEqual(@as([0]u8, undefined), try parseFromSlice([0]u8, testing.allocator, "[]", .{}));
24
25	try testing.expectEqual(@as(u64, 12345678901234567890), try parseFromSlice(u64, testing.allocator, "\"12345678901234567890\"", .{}));
26	try testing.expectEqual(@as(f64, 123.456), try parseFromSlice(f64, testing.allocator, "\"123.456\"", .{}));
27	}
28
29	test "parse into enum" {
30	const T = enum(u32) {
31	Foo = 42,
32	Bar,
33	@"with\\escape",
34	};
35	try testing.expectEqual(@as(T, .Foo), try parseFromSlice(T, testing.allocator, "\"Foo\"", .{}));
36	try testing.expectEqual(@as(T, .Foo), try parseFromSlice(T, testing.allocator, "42", .{}));
37	try testing.expectEqual(@as(T, .@"with\\escape"), try parseFromSlice(T, testing.allocator, "\"with\\\\escape\"", .{}));
38	try testing.expectError(error.InvalidEnumTag, parseFromSlice(T, testing.allocator, "5", .{}));
39	try testing.expectError(error.InvalidEnumTag, parseFromSlice(T, testing.allocator, "\"Qux\"", .{}));
40	}
41
42	test "parse into that allocates a slice" {
43	{
44	// string as string
45	const r = try parseFromSlice([]u8, testing.allocator, "\"foo\"", .{});
46	defer parseFree([]u8, testing.allocator, r);
47	try testing.expectEqualSlices(u8, "foo", r);
48	}
49	{
50	// string as array of u8 integers
51	const r = try parseFromSlice([]u8, testing.allocator, "[102, 111, 111]", .{});
52	defer parseFree([]u8, testing.allocator, r);
53	try testing.expectEqualSlices(u8, "foo", r);
54	}
55	{
56	const r = try parseFromSlice([]u8, testing.allocator, "\"with\\\\escape\"", .{});
57	defer parseFree([]u8, testing.allocator, r);
58	try testing.expectEqualSlices(u8, "with\\escape", r);
59	}
60	}
61
62	test "parse into sentinel slice" {
63	const result = try parseFromSlice([:0]const u8, testing.allocator, "\"\\n\"", .{});
64	defer parseFree([:0]const u8, testing.allocator, result);
65	try testing.expect(std.mem.eql(u8, result, "\n"));
66	}
67
68	test "parse into tagged union" {
69	const T = union(enum) {
70	nothing,
71	int: i32,
72	float: f64,
73	string: []const u8,
74	};
75	try testing.expectEqual(T{ .float = 1.5 }, try parseFromSlice(T, testing.allocator, "{\"float\":1.5}", .{}));
76	try testing.expectEqual(T{ .int = 1 }, try parseFromSlice(T, testing.allocator, "{\"int\":1}", .{}));
77	try testing.expectEqual(T{ .nothing = {} }, try parseFromSlice(T, testing.allocator, "{\"nothing\":{}}", .{}));
78	}
79
80	test "parse into tagged union errors" {
81	const T = union(enum) {
82	nothing,
83	int: i32,
84	float: f64,
85	string: []const u8,
86	};
87	try testing.expectError(error.UnexpectedToken, parseFromSlice(T, testing.allocator, "42", .{}));
88	try testing.expectError(error.UnexpectedToken, parseFromSlice(T, testing.allocator, "{}", .{}));
89	try testing.expectError(error.UnknownField, parseFromSlice(T, testing.allocator, "{\"bogus\":1}", .{}));
90	try testing.expectError(error.UnexpectedToken, parseFromSlice(T, testing.allocator, "{\"int\":1, \"int\":1", .{}));
91	try testing.expectError(error.UnexpectedToken, parseFromSlice(T, testing.allocator, "{\"int\":1, \"float\":1.0}", .{}));
92	try testing.expectError(error.UnexpectedToken, parseFromSlice(T, testing.allocator, "{\"nothing\":null}", .{}));
93	try testing.expectError(error.UnexpectedToken, parseFromSlice(T, testing.allocator, "{\"nothing\":{\"no\":0}}", .{}));
94
95	// Allocator failure
96	var fail_alloc = testing.FailingAllocator.init(testing.allocator, 0);
97	const failing_allocator = fail_alloc.allocator();
98	try testing.expectError(error.OutOfMemory, parseFromSlice(T, failing_allocator, "{\"string\"\"foo\"}", .{}));
99	}
100
101	test "parseFree descends into tagged union" {
102	const T = union(enum) {
103	nothing,
104	int: i32,
105	float: f64,
106	string: []const u8,
107	};
108	const r = try parseFromSlice(T, testing.allocator, "{\"string\":\"foo\"}", .{});
109	try testing.expectEqualSlices(u8, "foo", r.string);
110	parseFree(T, testing.allocator, r);
111	}
112
113	test "parse into struct with no fields" {
114	const T = struct {};
115	try testing.expectEqual(T{}, try parseFromSlice(T, testing.allocator, "{}", .{}));
116	}
117
118	const test_const_value: usize = 123;
119
120	test "parse into struct with default const pointer field" {
121	const T = struct { a: *const usize = &test_const_value };
122	try testing.expectEqual(T{}, try parseFromSlice(T, testing.allocator, "{}", .{}));
123	}
124
125	const test_default_usize: usize = 123;
126	const test_default_usize_ptr: *align(1) const usize = &test_default_usize;
127	const test_default_str: []const u8 = "test str";
128	const test_default_str_slice: [2][]const u8 = [_][]const u8{
129	"test1",
130	"test2",
131	};
132
133	test "freeing parsed structs with pointers to default values" {
134	const T = struct {
135	int: *const usize = &test_default_usize,
136	int_ptr: *allowzero align(1) const usize = test_default_usize_ptr,
137	str: []const u8 = test_default_str,
138	str_slice: []const []const u8 = &test_default_str_slice,
139	};
140
141	const parsed = try parseFromSlice(T, testing.allocator, "{}", .{});
142	try testing.expectEqual(T{}, parsed);
143	// This will panic if it tries to free global constants:
144	parseFree(T, testing.allocator, parsed);
145	}
146
147	test "parse into struct where destination and source lengths mismatch" {
148	const T = struct { a: [2]u8 };
149	try testing.expectError(error.LengthMismatch, parseFromSlice(T, testing.allocator, "{\"a\": \"bbb\"}", .{}));
150	}
151
152	test "parse into struct with misc fields" {
153	const T = struct {
154	int: i64,
155	float: f64,
156	@"with\\escape": bool,
157	@"withąunicode😂": bool,
158	language: []const u8,
159	optional: ?bool,
160	default_field: i32 = 42,
161	static_array: [3]f64,
162	dynamic_array: []f64,
163
164	complex: struct {
165	nested: []const u8,
166	},
167
168	veryComplex: []struct {
169	foo: []const u8,
170	},
171
172	a_union: Union,
173	const Union = union(enum) {
174	x: u8,
175	float: f64,
176	string: []const u8,
177	};
178	};
179	var document_str =
180	\\{
181	\\ "int": 420,
182	\\ "float": 3.14,
183	\\ "with\\escape": true,
184	\\ "with\u0105unicode\ud83d\ude02": false,
185	\\ "language": "zig",
186	\\ "optional": null,
187	\\ "static_array": [66.6, 420.420, 69.69],
188	\\ "dynamic_array": [66.6, 420.420, 69.69],
189	\\ "complex": {
190	\\ "nested": "zig"
191	\\ },
192	\\ "veryComplex": [
193	\\ {
194	\\ "foo": "zig"
195	\\ }, {
196	\\ "foo": "rocks"
197	\\ }
198	\\ ],
199	\\ "a_union": {
200	\\ "float": 100000
201	\\ }
202	\\}
203	;
204	const r = try parseFromSlice(T, testing.allocator, document_str, .{});
205	defer parseFree(T, testing.allocator, r);
206	try testing.expectEqual(@as(i64, 420), r.int);
207	try testing.expectEqual(@as(f64, 3.14), r.float);
208	try testing.expectEqual(true, r.@"with\\escape");
209	try testing.expectEqual(false, r.@"withąunicode😂");
210	try testing.expectEqualSlices(u8, "zig", r.language);
211	try testing.expectEqual(@as(?bool, null), r.optional);
212	try testing.expectEqual(@as(i32, 42), r.default_field);
213	try testing.expectEqual(@as(f64, 66.6), r.static_array[0]);
214	try testing.expectEqual(@as(f64, 420.420), r.static_array[1]);
215	try testing.expectEqual(@as(f64, 69.69), r.static_array[2]);
216	try testing.expectEqual(@as(usize, 3), r.dynamic_array.len);
217	try testing.expectEqual(@as(f64, 66.6), r.dynamic_array[0]);
218	try testing.expectEqual(@as(f64, 420.420), r.dynamic_array[1]);
219	try testing.expectEqual(@as(f64, 69.69), r.dynamic_array[2]);
220	try testing.expectEqualSlices(u8, r.complex.nested, "zig");
221	try testing.expectEqualSlices(u8, "zig", r.veryComplex[0].foo);
222	try testing.expectEqualSlices(u8, "rocks", r.veryComplex[1].foo);
223	try testing.expectEqual(T.Union{ .float = 100000 }, r.a_union);
224	}
225
226	test "parse into struct with strings and arrays with sentinels" {
227	const T = struct {
228	language: [:0]const u8,
229	language_without_sentinel: []const u8,
230	data: [:99]const i32,
231	simple_data: []const i32,
232	};
233	var document_str =
234	\\{
235	\\ "language": "zig",
236	\\ "language_without_sentinel": "zig again!",
237	\\ "data": [1, 2, 3],
238	\\ "simple_data": [4, 5, 6]
239	\\}
240	;
241	const r = try parseFromSlice(T, testing.allocator, document_str, .{});
242	defer parseFree(T, testing.allocator, r);
243
244	try testing.expectEqualSentinel(u8, 0, "zig", r.language);
245
246	const data = [_:99]i32{ 1, 2, 3 };
247	try testing.expectEqualSentinel(i32, 99, data[0..data.len], r.data);
248
249	// Make sure that arrays who aren't supposed to have a sentinel still parse without one.
250	try testing.expectEqual(@as(?i32, null), std.meta.sentinel(@TypeOf(r.simple_data)));
251	try testing.expectEqual(@as(?u8, null), std.meta.sentinel(@TypeOf(r.language_without_sentinel)));
252	}
253
254	test "parse into struct with duplicate field" {
255	// allow allocator to detect double frees by keeping bucket in use
256	const ballast = try testing.allocator.alloc(u64, 1);
257	defer testing.allocator.free(ballast);
258
259	const options_first = ParseOptions{ .duplicate_field_behavior = .use_first };
260	const options_last = ParseOptions{ .duplicate_field_behavior = .use_last };
261
262	const str = "{ \"a\": 1, \"a\": 0.25 }";
263
264	const T1 = struct { a: *u64 };
265	// both .use_first and .use_last should fail because second "a" value isn't a u64
266	try testing.expectError(error.InvalidNumber, parseFromSlice(T1, testing.allocator, str, options_first));
267	try testing.expectError(error.InvalidNumber, parseFromSlice(T1, testing.allocator, str, options_last));
268
269	const T2 = struct { a: f64 };
270	try testing.expectEqual(T2{ .a = 1.0 }, try parseFromSlice(T2, testing.allocator, str, options_first));
271	try testing.expectEqual(T2{ .a = 0.25 }, try parseFromSlice(T2, testing.allocator, str, options_last));
272	}
273
274	test "parse into struct ignoring unknown fields" {
275	const T = struct {
276	int: i64,
277	language: []const u8,
278	};
279
280	var str =
281	\\{
282	\\ "int": 420,
283	\\ "float": 3.14,
284	\\ "with\\escape": true,
285	\\ "with\u0105unicode\ud83d\ude02": false,
286	\\ "optional": null,
287	\\ "static_array": [66.6, 420.420, 69.69],
288	\\ "dynamic_array": [66.6, 420.420, 69.69],
289	\\ "complex": {
290	\\ "nested": "zig"
291	\\ },
292	\\ "veryComplex": [
293	\\ {
294	\\ "foo": "zig"
295	\\ }, {
296	\\ "foo": "rocks"
297	\\ }
298	\\ ],
299	\\ "a_union": {
300	\\ "float": 100000
301	\\ },
302	\\ "language": "zig"
303	\\}
304	;
305	const r = try parseFromSlice(T, testing.allocator, str, .{ .ignore_unknown_fields = true });
306	defer parseFree(T, testing.allocator, r);
307
308	try testing.expectEqual(@as(i64, 420), r.int);
309	try testing.expectEqualSlices(u8, "zig", r.language);
310	}
311
312	test "parse into tuple" {
313	const Union = union(enum) {
314	char: u8,
315	float: f64,
316	string: []const u8,
317	};
318	const T = std.meta.Tuple(&.{
319	i64,
320	f64,
321	bool,
322	[]const u8,
323	?bool,
324	struct {
325	foo: i32,
326	bar: []const u8,
327	},
328	std.meta.Tuple(&.{ u8, []const u8, u8 }),
329	Union,
330	});
331	var str =
332	\\[
333	\\ 420,
334	\\ 3.14,
335	\\ true,
336	\\ "zig",
337	\\ null,
338	\\ {
339	\\ "foo": 1,
340	\\ "bar": "zero"
341	\\ },
342	\\ [4, "två", 42],
343	\\ {"float": 12.34}
344	\\]
345	;
346	const r = try parseFromSlice(T, testing.allocator, str, .{});
347	defer parseFree(T, testing.allocator, r);
348	try testing.expectEqual(@as(i64, 420), r[0]);
349	try testing.expectEqual(@as(f64, 3.14), r[1]);
350	try testing.expectEqual(true, r[2]);
351	try testing.expectEqualSlices(u8, "zig", r[3]);
352	try testing.expectEqual(@as(?bool, null), r[4]);
353	try testing.expectEqual(@as(i32, 1), r[5].foo);
354	try testing.expectEqualSlices(u8, "zero", r[5].bar);
355	try testing.expectEqual(@as(u8, 4), r[6][0]);
356	try testing.expectEqualSlices(u8, "två", r[6][1]);
357	try testing.expectEqual(@as(u8, 42), r[6][2]);
358	try testing.expectEqual(Union{ .float = 12.34 }, r[7]);
359	}
360
361	const ParseIntoRecursiveUnionDefinitionValue = union(enum) {
362	integer: i64,
363	array: []const ParseIntoRecursiveUnionDefinitionValue,
364	};
365
366	test "parse into recursive union definition" {
367	const T = struct {
368	values: ParseIntoRecursiveUnionDefinitionValue,
369	};
370
371	const r = try parseFromSlice(T, testing.allocator, "{\"values\":{\"array\":[{\"integer\":58}]}}", .{});
372	defer parseFree(T, testing.allocator, r);
373
374	try testing.expectEqual(@as(i64, 58), r.values.array[0].integer);
375	}
376
377	const ParseIntoDoubleRecursiveUnionValueFirst = union(enum) {
378	integer: i64,
379	array: []const ParseIntoDoubleRecursiveUnionValueSecond,
380	};
381
382	const ParseIntoDoubleRecursiveUnionValueSecond = union(enum) {
383	boolean: bool,
384	array: []const ParseIntoDoubleRecursiveUnionValueFirst,
385	};
386
387	test "parse into double recursive union definition" {
388	const T = struct {
389	values: ParseIntoDoubleRecursiveUnionValueFirst,
390	};
391
392	const r = try parseFromSlice(T, testing.allocator, "{\"values\":{\"array\":[{\"array\":[{\"integer\":58}]}]}}", .{});
393	defer parseFree(T, testing.allocator, r);
394
395	try testing.expectEqual(@as(i64, 58), r.values.array[0].array[0].integer);
396	}
397
398	test "parse exponential into int" {
399	const T = struct { int: i64 };
400	const r = try parseFromSlice(T, testing.allocator, "{ \"int\": 4.2e2 }", .{});
401	try testing.expectEqual(@as(i64, 420), r.int);
402	try testing.expectError(error.InvalidNumber, parseFromSlice(T, testing.allocator, "{ \"int\": 0.042e2 }", .{}));
403	try testing.expectError(error.Overflow, parseFromSlice(T, testing.allocator, "{ \"int\": 18446744073709551616.0 }", .{}));
404	}
405
406	test "parseFromTokenSource" {
407	var scanner = JsonScanner.initCompleteInput(testing.allocator, "123");
408	defer scanner.deinit();
409	try testing.expectEqual(@as(u32, 123), try parseFromTokenSource(u32, testing.allocator, &scanner, .{}));
410
411	var stream = std.io.fixedBufferStream("123");
412	var json_reader = jsonReader(std.testing.allocator, stream.reader());
413	defer json_reader.deinit();
414	try testing.expectEqual(@as(u32, 123), try parseFromTokenSource(u32, testing.allocator, &json_reader, .{}));
415	}
416
417	test "max_value_len" {
418	try testing.expectError(error.ValueTooLong, parseFromSlice([]u8, testing.allocator, "\"0123456789\"", .{ .max_value_len = 5 }));
419	}
420
421	test "parse into vector" {
422	const T = struct {
423	vec_i32: @Vector(4, i32),
424	vec_f32: @Vector(2, f32),
425	};
426	var s =
427	\\{
428	\\ "vec_f32": [1.5, 2.5],
429	\\ "vec_i32": [4, 5, 6, 7]
430	\\}
431	;
432	const r = try parseFromSlice(T, testing.allocator, s, .{});
433	defer parseFree(T, testing.allocator, r);
434	try testing.expectApproxEqAbs(@as(f32, 1.5), r.vec_f32[0], 0.0000001);
435	try testing.expectApproxEqAbs(@as(f32, 2.5), r.vec_f32[1], 0.0000001);
436	try testing.expectEqual(@Vector(4, i32){ 4, 5, 6, 7 }, r.vec_i32);
437	}

lib/std/json/stringify.zig created+313

...	@@ -0,0 +1,313 @@
1	const std = @import("std");
2	const mem = std.mem;
3	const assert = std.debug.assert;
4
5	pub const StringifyOptions = struct {
6	pub const Whitespace = struct {
7	/// How many indentation levels deep are we?
8	indent_level: usize = 0,
9
10	/// What character(s) should be used for indentation?
11	indent: union(enum) {
12	space: u8,
13	tab: void,
14	none: void,
15	} = .{ .space = 4 },
16
17	/// After a colon, should whitespace be inserted?
18	separator: bool = true,
19
20	pub fn outputIndent(
21	whitespace: @This(),
22	out_stream: anytype,
23	) @TypeOf(out_stream).Error!void {
24	var char: u8 = undefined;
25	var n_chars: usize = undefined;
26	switch (whitespace.indent) {
27	.space => \|n_spaces\| {
28	char = ' ';
29	n_chars = n_spaces;
30	},
31	.tab => {
32	char = '\t';
33	n_chars = 1;
34	},
35	.none => return,
36	}
37	try out_stream.writeByte('\n');
38	n_chars *= whitespace.indent_level;
39	try out_stream.writeByteNTimes(char, n_chars);
40	}
41	};
42
43	/// Controls the whitespace emitted
44	whitespace: Whitespace = .{ .indent = .none, .separator = false },
45
46	/// Should optional fields with null value be written?
47	emit_null_optional_fields: bool = true,
48
49	string: StringOptions = StringOptions{ .String = .{} },
50
51	/// Should []u8 be serialised as a string? or an array?
52	pub const StringOptions = union(enum) {
53	Array,
54	String: StringOutputOptions,
55
56	/// String output options
57	const StringOutputOptions = struct {
58	/// Should '/' be escaped in strings?
59	escape_solidus: bool = false,
60
61	/// Should unicode characters be escaped in strings?
62	escape_unicode: bool = false,
63	};
64	};
65	};
66
67	fn outputUnicodeEscape(
68	codepoint: u21,
69	out_stream: anytype,
70	) !void {
71	if (codepoint <= 0xFFFF) {
72	// If the character is in the Basic Multilingual Plane (U+0000 through U+FFFF),
73	// then it may be represented as a six-character sequence: a reverse solidus, followed
74	// by the lowercase letter u, followed by four hexadecimal digits that encode the character's code point.
75	try out_stream.writeAll("\\u");
76	try std.fmt.formatIntValue(codepoint, "x", std.fmt.FormatOptions{ .width = 4, .fill = '0' }, out_stream);
77	} else {
78	assert(codepoint <= 0x10FFFF);
79	// To escape an extended character that is not in the Basic Multilingual Plane,
80	// the character is represented as a 12-character sequence, encoding the UTF-16 surrogate pair.
81	const high = @intCast(u16, (codepoint - 0x10000) >> 10) + 0xD800;
82	const low = @intCast(u16, codepoint & 0x3FF) + 0xDC00;
83	try out_stream.writeAll("\\u");
84	try std.fmt.formatIntValue(high, "x", std.fmt.FormatOptions{ .width = 4, .fill = '0' }, out_stream);
85	try out_stream.writeAll("\\u");
86	try std.fmt.formatIntValue(low, "x", std.fmt.FormatOptions{ .width = 4, .fill = '0' }, out_stream);
87	}
88	}
89
90	/// Write `string` to `writer` as a JSON encoded string.
91	pub fn encodeJsonString(string: []const u8, options: StringifyOptions, writer: anytype) !void {
92	try writer.writeByte('\"');
93	try encodeJsonStringChars(string, options, writer);
94	try writer.writeByte('\"');
95	}
96
97	/// Write `chars` to `writer` as JSON encoded string characters.
98	pub fn encodeJsonStringChars(chars: []const u8, options: StringifyOptions, writer: anytype) !void {
99	var i: usize = 0;
100	while (i < chars.len) : (i += 1) {
101	switch (chars[i]) {
102	// normal ascii character
103	0x20...0x21, 0x23...0x2E, 0x30...0x5B, 0x5D...0x7F => \|c\| try writer.writeByte(c),
104	// only 2 characters that must be escaped
105	'\\' => try writer.writeAll("\\\\"),
106	'\"' => try writer.writeAll("\\\""),
107	// solidus is optional to escape
108	'/' => {
109	if (options.string.String.escape_solidus) {
110	try writer.writeAll("\\/");
111	} else {
112	try writer.writeByte('/');
113	}
114	},
115	// control characters with short escapes
116	// TODO: option to switch between unicode and 'short' forms?
117	0x8 => try writer.writeAll("\\b"),
118	0xC => try writer.writeAll("\\f"),
119	'\n' => try writer.writeAll("\\n"),
120	'\r' => try writer.writeAll("\\r"),
121	'\t' => try writer.writeAll("\\t"),
122	else => {
123	const ulen = std.unicode.utf8ByteSequenceLength(chars[i]) catch unreachable;
124	// control characters (only things left with 1 byte length) should always be printed as unicode escapes
125	if (ulen == 1 or options.string.String.escape_unicode) {
126	const codepoint = std.unicode.utf8Decode(chars[i..][0..ulen]) catch unreachable;
127	try outputUnicodeEscape(codepoint, writer);
128	} else {
129	try writer.writeAll(chars[i..][0..ulen]);
130	}
131	i += ulen - 1;
132	},
133	}
134	}
135	}
136
137	pub fn stringify(
138	value: anytype,
139	options: StringifyOptions,
140	out_stream: anytype,
141	) !void {
142	const T = @TypeOf(value);
143	switch (@typeInfo(T)) {
144	.Float, .ComptimeFloat => {
145	return std.fmt.formatFloatScientific(value, std.fmt.FormatOptions{}, out_stream);
146	},
147	.Int, .ComptimeInt => {
148	return std.fmt.formatIntValue(value, "", std.fmt.FormatOptions{}, out_stream);
149	},
150	.Bool => {
151	return out_stream.writeAll(if (value) "true" else "false");
152	},
153	.Null => {
154	return out_stream.writeAll("null");
155	},
156	.Optional => {
157	if (value) \|payload\| {
158	return try stringify(payload, options, out_stream);
159	} else {
160	return try stringify(null, options, out_stream);
161	}
162	},
163	.Enum => {
164	if (comptime std.meta.trait.hasFn("jsonStringify")(T)) {
165	return value.jsonStringify(options, out_stream);
166	}
167
168	@compileError("Unable to stringify enum '" ++ @typeName(T) ++ "'");
169	},
170	.Union => {
171	if (comptime std.meta.trait.hasFn("jsonStringify")(T)) {
172	return value.jsonStringify(options, out_stream);
173	}
174
175	const info = @typeInfo(T).Union;
176	if (info.tag_type) \|UnionTagType\| {
177	try out_stream.writeByte('{');
178	var child_options = options;
179	child_options.whitespace.indent_level += 1;
180	inline for (info.fields) \|u_field\| {
181	if (value == @field(UnionTagType, u_field.name)) {
182	try child_options.whitespace.outputIndent(out_stream);
183	try encodeJsonString(u_field.name, options, out_stream);
184	try out_stream.writeByte(':');
185	if (child_options.whitespace.separator) {
186	try out_stream.writeByte(' ');
187	}
188	if (u_field.type == void) {
189	try out_stream.writeAll("{}");
190	} else {
191	try stringify(@field(value, u_field.name), child_options, out_stream);
192	}
193	break;
194	}
195	} else {
196	unreachable; // No active tag?
197	}
198	try options.whitespace.outputIndent(out_stream);
199	try out_stream.writeByte('}');
200	return;
201	} else {
202	@compileError("Unable to stringify untagged union '" ++ @typeName(T) ++ "'");
203	}
204	},
205	.Struct => \|S\| {
206	if (comptime std.meta.trait.hasFn("jsonStringify")(T)) {
207	return value.jsonStringify(options, out_stream);
208	}
209
210	try out_stream.writeByte(if (S.is_tuple) '[' else '{');
211	var field_output = false;
212	var child_options = options;
213	child_options.whitespace.indent_level += 1;
214	inline for (S.fields) \|Field\| {
215	// don't include void fields
216	if (Field.type == void) continue;
217
218	var emit_field = true;
219
220	// don't include optional fields that are null when emit_null_optional_fields is set to false
221	if (@typeInfo(Field.type) == .Optional) {
222	if (options.emit_null_optional_fields == false) {
223	if (@field(value, Field.name) == null) {
224	emit_field = false;
225	}
226	}
227	}
228
229	if (emit_field) {
230	if (!field_output) {
231	field_output = true;
232	} else {
233	try out_stream.writeByte(',');
234	}
235	try child_options.whitespace.outputIndent(out_stream);
236	if (!S.is_tuple) {
237	try encodeJsonString(Field.name, options, out_stream);
238	try out_stream.writeByte(':');
239	if (child_options.whitespace.separator) {
240	try out_stream.writeByte(' ');
241	}
242	}
243	try stringify(@field(value, Field.name), child_options, out_stream);
244	}
245	}
246	if (field_output) {
247	try options.whitespace.outputIndent(out_stream);
248	}
249	try out_stream.writeByte(if (S.is_tuple) ']' else '}');
250	return;
251	},
252	.ErrorSet => return stringify(@as([]const u8, @errorName(value)), options, out_stream),
253	.Pointer => \|ptr_info\| switch (ptr_info.size) {
254	.One => switch (@typeInfo(ptr_info.child)) {
255	.Array => {
256	const Slice = []const std.meta.Elem(ptr_info.child);
257	return stringify(@as(Slice, value), options, out_stream);
258	},
259	else => {
260	// TODO: avoid loops?
261	return stringify(value.*, options, out_stream);
262	},
263	},
264	.Many, .Slice => {
265	if (ptr_info.size == .Many and ptr_info.sentinel == null)
266	@compileError("unable to stringify type '" ++ @typeName(T) ++ "' without sentinel");
267	const slice = if (ptr_info.size == .Many) mem.span(value) else value;
268
269	if (ptr_info.child == u8 and options.string == .String and std.unicode.utf8ValidateSlice(slice)) {
270	try encodeJsonString(slice, options, out_stream);
271	return;
272	}
273
274	try out_stream.writeByte('[');
275	var child_options = options;
276	child_options.whitespace.indent_level += 1;
277	for (slice, 0..) \|x, i\| {
278	if (i != 0) {
279	try out_stream.writeByte(',');
280	}
281	try child_options.whitespace.outputIndent(out_stream);
282	try stringify(x, child_options, out_stream);
283	}
284	if (slice.len != 0) {
285	try options.whitespace.outputIndent(out_stream);
286	}
287	try out_stream.writeByte(']');
288	return;
289	},
290	else => @compileError("Unable to stringify type '" ++ @typeName(T) ++ "'"),
291	},
292	.Array => return stringify(&value, options, out_stream),
293	.Vector => \|info\| {
294	const array: [info.len]info.child = value;
295	return stringify(&array, options, out_stream);
296	},
297	else => @compileError("Unable to stringify type '" ++ @typeName(T) ++ "'"),
298	}
299	unreachable;
300	}
301
302	// Same as `stringify` but accepts an Allocator and stores result in dynamically allocated memory instead of using a Writer.
303	// Caller owns returned memory.
304	pub fn stringifyAlloc(allocator: std.mem.Allocator, value: anytype, options: StringifyOptions) ![]const u8 {
305	var list = std.ArrayList(u8).init(allocator);
306	errdefer list.deinit();
307	try stringify(value, options, list.writer());
308	return list.toOwnedSlice();
309	}
310
311	test {
312	_ = @import("./stringify_test.zig");
313	}

lib/std/json/stringify_test.zig created+280

...	@@ -0,0 +1,280 @@
1	const std = @import("std");
2	const mem = std.mem;
3	const testing = std.testing;
4
5	const StringifyOptions = @import("stringify.zig").StringifyOptions;
6	const stringify = @import("stringify.zig").stringify;
7	const stringifyAlloc = @import("stringify.zig").stringifyAlloc;
8
9	test "stringify null optional fields" {
10	const MyStruct = struct {
11	optional: ?[]const u8 = null,
12	required: []const u8 = "something",
13	another_optional: ?[]const u8 = null,
14	another_required: []const u8 = "something else",
15	};
16	try teststringify(
17	\\{"optional":null,"required":"something","another_optional":null,"another_required":"something else"}
18	,
19	MyStruct{},
20	StringifyOptions{},
21	);
22	try teststringify(
23	\\{"required":"something","another_required":"something else"}
24	,
25	MyStruct{},
26	StringifyOptions{ .emit_null_optional_fields = false },
27	);
28	}
29
30	test "stringify basic types" {
31	try teststringify("false", false, StringifyOptions{});
32	try teststringify("true", true, StringifyOptions{});
33	try teststringify("null", @as(?u8, null), StringifyOptions{});
34	try teststringify("null", @as(?*u32, null), StringifyOptions{});
35	try teststringify("42", 42, StringifyOptions{});
36	try teststringify("4.2e+01", 42.0, StringifyOptions{});
37	try teststringify("42", @as(u8, 42), StringifyOptions{});
38	try teststringify("42", @as(u128, 42), StringifyOptions{});
39	try teststringify("4.2e+01", @as(f32, 42), StringifyOptions{});
40	try teststringify("4.2e+01", @as(f64, 42), StringifyOptions{});
41	try teststringify("\"ItBroke\"", @as(anyerror, error.ItBroke), StringifyOptions{});
42	}
43
44	test "stringify string" {
45	try teststringify("\"hello\"", "hello", StringifyOptions{});
46	try teststringify("\"with\\nescapes\\r\"", "with\nescapes\r", StringifyOptions{});
47	try teststringify("\"with\\nescapes\\r\"", "with\nescapes\r", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
48	try teststringify("\"with unicode\\u0001\"", "with unicode\u{1}", StringifyOptions{});
49	try teststringify("\"with unicode\\u0001\"", "with unicode\u{1}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
50	try teststringify("\"with unicode\u{80}\"", "with unicode\u{80}", StringifyOptions{});
51	try teststringify("\"with unicode\\u0080\"", "with unicode\u{80}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
52	try teststringify("\"with unicode\u{FF}\"", "with unicode\u{FF}", StringifyOptions{});
53	try teststringify("\"with unicode\\u00ff\"", "with unicode\u{FF}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
54	try teststringify("\"with unicode\u{100}\"", "with unicode\u{100}", StringifyOptions{});
55	try teststringify("\"with unicode\\u0100\"", "with unicode\u{100}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
56	try teststringify("\"with unicode\u{800}\"", "with unicode\u{800}", StringifyOptions{});
57	try teststringify("\"with unicode\\u0800\"", "with unicode\u{800}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
58	try teststringify("\"with unicode\u{8000}\"", "with unicode\u{8000}", StringifyOptions{});
59	try teststringify("\"with unicode\\u8000\"", "with unicode\u{8000}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
60	try teststringify("\"with unicode\u{D799}\"", "with unicode\u{D799}", StringifyOptions{});
61	try teststringify("\"with unicode\\ud799\"", "with unicode\u{D799}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
62	try teststringify("\"with unicode\u{10000}\"", "with unicode\u{10000}", StringifyOptions{});
63	try teststringify("\"with unicode\\ud800\\udc00\"", "with unicode\u{10000}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
64	try teststringify("\"with unicode\u{10FFFF}\"", "with unicode\u{10FFFF}", StringifyOptions{});
65	try teststringify("\"with unicode\\udbff\\udfff\"", "with unicode\u{10FFFF}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
66	try teststringify("\"/\"", "/", StringifyOptions{});
67	try teststringify("\"\\/\"", "/", StringifyOptions{ .string = .{ .String = .{ .escape_solidus = true } } });
68	}
69
70	test "stringify many-item sentinel-terminated string" {
71	try teststringify("\"hello\"", @as([*:0]const u8, "hello"), StringifyOptions{});
72	try teststringify("\"with\\nescapes\\r\"", @as([*:0]const u8, "with\nescapes\r"), StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
73	try teststringify("\"with unicode\\u0001\"", @as([*:0]const u8, "with unicode\u{1}"), StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
74	}
75
76	test "stringify tagged unions" {
77	const T = union(enum) {
78	nothing,
79	foo: u32,
80	bar: bool,
81	};
82	try teststringify("{\"nothing\":{}}", T{ .nothing = {} }, StringifyOptions{});
83	try teststringify("{\"foo\":42}", T{ .foo = 42 }, StringifyOptions{});
84	try teststringify("{\"bar\":true}", T{ .bar = true }, StringifyOptions{});
85	}
86
87	test "stringify struct" {
88	try teststringify("{\"foo\":42}", struct {
89	foo: u32,
90	}{ .foo = 42 }, StringifyOptions{});
91	}
92
93	test "stringify struct with string as array" {
94	try teststringify("{\"foo\":\"bar\"}", .{ .foo = "bar" }, StringifyOptions{});
95	try teststringify("{\"foo\":[98,97,114]}", .{ .foo = "bar" }, StringifyOptions{ .string = .Array });
96	}
97
98	test "stringify struct with indentation" {
99	try teststringify(
100	\\{
101	\\ "foo": 42,
102	\\ "bar": [
103	\\ 1,
104	\\ 2,
105	\\ 3
106	\\ ]
107	\\}
108	,
109	struct {
110	foo: u32,
111	bar: [3]u32,
112	}{
113	.foo = 42,
114	.bar = .{ 1, 2, 3 },
115	},
116	StringifyOptions{
117	.whitespace = .{},
118	},
119	);
120	try teststringify(
121	"{\n\t\"foo\":42,\n\t\"bar\":[\n\t\t1,\n\t\t2,\n\t\t3\n\t]\n}",
122	struct {
123	foo: u32,
124	bar: [3]u32,
125	}{
126	.foo = 42,
127	.bar = .{ 1, 2, 3 },
128	},
129	StringifyOptions{
130	.whitespace = .{
131	.indent = .tab,
132	.separator = false,
133	},
134	},
135	);
136	try teststringify(
137	\\{"foo":42,"bar":[1,2,3]}
138	,
139	struct {
140	foo: u32,
141	bar: [3]u32,
142	}{
143	.foo = 42,
144	.bar = .{ 1, 2, 3 },
145	},
146	StringifyOptions{
147	.whitespace = .{
148	.indent = .none,
149	.separator = false,
150	},
151	},
152	);
153	}
154
155	test "stringify struct with void field" {
156	try teststringify("{\"foo\":42}", struct {
157	foo: u32,
158	bar: void = {},
159	}{ .foo = 42 }, StringifyOptions{});
160	}
161
162	test "stringify array of structs" {
163	const MyStruct = struct {
164	foo: u32,
165	};
166	try teststringify("[{\"foo\":42},{\"foo\":100},{\"foo\":1000}]", [_]MyStruct{
167	MyStruct{ .foo = 42 },
168	MyStruct{ .foo = 100 },
169	MyStruct{ .foo = 1000 },
170	}, StringifyOptions{});
171	}
172
173	test "stringify struct with custom stringifier" {
174	try teststringify("[\"something special\",42]", struct {
175	foo: u32,
176	const Self = @This();
177	pub fn jsonStringify(
178	value: Self,
179	options: StringifyOptions,
180	out_stream: anytype,
181	) !void {
182	_ = value;
183	try out_stream.writeAll("[\"something special\",");
184	try stringify(42, options, out_stream);
185	try out_stream.writeByte(']');
186	}
187	}{ .foo = 42 }, StringifyOptions{});
188	}
189
190	test "stringify vector" {
191	try teststringify("[1,1]", @splat(2, @as(u32, 1)), StringifyOptions{});
192	}
193
194	test "stringify tuple" {
195	try teststringify("[\"foo\",42]", std.meta.Tuple(&.{ []const u8, usize }){ "foo", 42 }, StringifyOptions{});
196	}
197
198	fn teststringify(expected: []const u8, value: anytype, options: StringifyOptions) !void {
199	const ValidationWriter = struct {
200	const Self = @This();
201	pub const Writer = std.io.Writer(*Self, Error, write);
202	pub const Error = error{
203	TooMuchData,
204	DifferentData,
205	};
206
207	expected_remaining: []const u8,
208
209	fn init(exp: []const u8) Self {
210	return .{ .expected_remaining = exp };
211	}
212
213	pub fn writer(self: *Self) Writer {
214	return .{ .context = self };
215	}
216
217	fn write(self: *Self, bytes: []const u8) Error!usize {
218	if (self.expected_remaining.len < bytes.len) {
219	std.debug.print(
220	\\====== expected this output: =========
221	\\{s}
222	\\======== instead found this: =========
223	\\{s}
224	\\======================================
225	, .{
226	self.expected_remaining,
227	bytes,
228	});
229	return error.TooMuchData;
230	}
231	if (!mem.eql(u8, self.expected_remaining[0..bytes.len], bytes)) {
232	std.debug.print(
233	\\====== expected this output: =========
234	\\{s}
235	\\======== instead found this: =========
236	\\{s}
237	\\======================================
238	, .{
239	self.expected_remaining[0..bytes.len],
240	bytes,
241	});
242	return error.DifferentData;
243	}
244	self.expected_remaining = self.expected_remaining[bytes.len..];
245	return bytes.len;
246	}
247	};
248
249	var vos = ValidationWriter.init(expected);
250	try stringify(value, options, vos.writer());
251	if (vos.expected_remaining.len > 0) return error.NotEnoughData;
252	}
253
254	test "stringify struct with custom stringify that returns a custom error" {
255	var ret = stringify(struct {
256	field: Field = .{},
257
258	pub const Field = struct {
259	field: ?[]*Field = null,
260
261	const Self = @This();
262	pub fn jsonStringify(_: Self, _: StringifyOptions, _: anytype) error{CustomError}!void {
263	return error.CustomError;
264	}
265	};
266	}{}, StringifyOptions{}, std.io.null_writer);
267
268	try std.testing.expectError(error.CustomError, ret);
269	}
270
271	test "stringify alloc" {
272	const allocator = std.testing.allocator;
273	const expected =
274	\\{"foo":"bar","answer":42,"my_friend":"sammy"}
275	;
276	const actual = try stringifyAlloc(allocator, .{ .foo = "bar", .answer = 42, .my_friend = "sammy" }, .{});
277	defer allocator.free(actual);
278
279	try std.testing.expectEqualStrings(expected, actual);
280	}

lib/std/json/test.zig+76-2923

...	@@ -1,2960 +1,113 @@	...	@@ -1,2960 +1,113 @@
1	// RFC 8529 conformance tests.	1	const std = @import("std");
2	//
3	// Tests are taken from https://github.com/nst/JSONTestSuite
4	// Read also http://seriot.ch/parsing_json.php for a good overview.
5
6	const std = @import("../std.zig");
7	const json = std.json;
8	const testing = std.testing;	2	const testing = std.testing;
9	const TokenStream = std.json.TokenStream;	3	const Parser = @import("./dynamic.zig").Parser;
10	const parse = std.json.parse;	4	const validate = @import("./scanner.zig").validate;
11	const ParseOptions = std.json.ParseOptions;	5	const JsonScanner = @import("./scanner.zig").Scanner;
12	const parseFree = std.json.parseFree;
13	const Parser = std.json.Parser;
14	const mem = std.mem;
15	const writeStream = std.json.writeStream;
16	const Value = std.json.Value;
17	const StringifyOptions = std.json.StringifyOptions;
18	const stringify = std.json.stringify;
19	const stringifyAlloc = std.json.stringifyAlloc;
20	const StreamingParser = std.json.StreamingParser;
21	const Token = std.json.Token;
22	const validate = std.json.validate;
23	const Array = std.json.Array;
24	const ObjectMap = std.json.ObjectMap;
25	const assert = std.debug.assert;
26
27	fn testNonStreaming(s: []const u8) !void {
28	var p = json.Parser.init(testing.allocator, false);
29	defer p.deinit();
30
31	var tree = try p.parse(s);
32	defer tree.deinit();
33	}
34
35	fn ok(s: []const u8) !void {
36	try testing.expect(json.validate(s));
37		6
38	try testNonStreaming(s);	7	// Support for JSONTestSuite.zig
		8	pub fn ok(s: []const u8) !void {
		9	try testLowLevelScanner(s);
		10	try testHighLevelDynamicParser(s);
39	}	11	}
40		12	pub fn err(s: []const u8) !void {
41	fn err(s: []const u8) !void {	13	try testing.expect(std.meta.isError(testLowLevelScanner(s)));
42	try testing.expect(!json.validate(s));	14	try testing.expect(std.meta.isError(testHighLevelDynamicParser(s)));
43
44	try testing.expect(std.meta.isError(testNonStreaming(s)));
45	}
46
47	fn utf8Error(s: []const u8) !void {
48	try testing.expect(!json.validate(s));
49
50	try testing.expectError(error.InvalidUtf8Byte, testNonStreaming(s));
51	}	15	}
52		16	pub fn any(s: []const u8) !void {
53	fn any(s: []const u8) !void {	17	testLowLevelScanner(s) catch {};
54	_ = json.validate(s);	18	testHighLevelDynamicParser(s) catch {};
55
56	testNonStreaming(s) catch {};
57	}	19	}
58		20	fn testLowLevelScanner(s: []const u8) !void {
59	fn anyStreamingErrNonStreaming(s: []const u8) !void {	21	var scanner = JsonScanner.initCompleteInput(testing.allocator, s);
60	_ = json.validate(s);	22	defer scanner.deinit();
61		23	while (true) {
62	try testing.expect(std.meta.isError(testNonStreaming(s)));	24	const token = try scanner.next();
		25	if (token == .end_of_document) break;
		26	}
63	}	27	}
64		28	fn testHighLevelDynamicParser(s: []const u8) !void {
65	fn roundTrip(s: []const u8) !void {	29	var p = Parser.init(testing.allocator, .alloc_if_needed);
66	try testing.expect(json.validate(s));
67
68	var p = json.Parser.init(testing.allocator, false);
69	defer p.deinit();	30	defer p.deinit();
70
71	var tree = try p.parse(s);	31	var tree = try p.parse(s);
72	defer tree.deinit();	32	defer tree.deinit();
73
74	var buf: [256]u8 = undefined;
75	var fbs = std.io.fixedBufferStream(&buf);
76	try tree.root.jsonStringify(.{}, fbs.writer());
77
78	try testing.expectEqualStrings(s, fbs.getWritten());
79	}	33	}
80		34
81	////////////////////////////////////////////////////////////////////////////////////////////////////
82	//
83	// Additional tests not part of test JSONTestSuite.	35	// Additional tests not part of test JSONTestSuite.
84
85	test "y_trailing_comma_after_empty" {	36	test "y_trailing_comma_after_empty" {
86	try roundTrip(	37	try roundTrip(
87	\\{"1":[],"2":{},"3":"4"}	38	\\{"1":[],"2":{},"3":"4"}
88	);	39	);
89	}	40	}
90
91	test "n_object_closed_missing_value" {	41	test "n_object_closed_missing_value" {
92	try err(	42	try err(
93	\\{"a":}	43	\\{"a":}
94	);	44	);
95	}	45	}
96		46
97	////////////////////////////////////////////////////////////////////////////////////////////////////	47	fn roundTrip(s: []const u8) !void {
98		48	try testing.expect(try validate(testing.allocator, s));
99	test "y_array_arraysWithSpaces" {
100	try ok(
101	\\[[] ]
102	);
103	}
104
105	test "y_array_empty" {
106	try roundTrip(
107	\\[]
108	);
109	}
110
111	test "y_array_empty-string" {
112	try roundTrip(
113	\\[""]
114	);
115	}
116
117	test "y_array_ending_with_newline" {
118	try roundTrip(
119	\\["a"]
120	);
121	}
122
123	test "y_array_false" {
124	try roundTrip(
125	\\[false]
126	);
127	}
128
129	test "y_array_heterogeneous" {
130	try ok(
131	\\[null, 1, "1", {}]
132	);
133	}
134
135	test "y_array_null" {
136	try roundTrip(
137	\\[null]
138	);
139	}
140
141	test "y_array_with_1_and_newline" {
142	try ok(
143	\\[1
144	\\]
145	);
146	}
147
148	test "y_array_with_leading_space" {
149	try ok(
150	\\ [1]
151	);
152	}
153
154	test "y_array_with_several_null" {
155	try roundTrip(
156	\\[1,null,null,null,2]
157	);
158	}
159
160	test "y_array_with_trailing_space" {
161	try ok("[2] ");
162	}
163
164	test "y_number_0e+1" {
165	try ok(
166	\\[0e+1]
167	);
168	}
169
170	test "y_number_0e1" {
171	try ok(
172	\\[0e1]
173	);
174	}
175
176	test "y_number_after_space" {
177	try ok(
178	\\[ 4]
179	);
180	}
181
182	test "y_number_double_close_to_zero" {
183	try ok(
184	\\[-0.000000000000000000000000000000000000000000000000000000000000000000000000000001]
185	);
186	}
187
188	test "y_number_int_with_exp" {
189	try ok(
190	\\[20e1]
191	);
192	}
193
194	test "y_number" {
195	try ok(
196	\\[123e65]
197	);
198	}
199
200	test "y_number_minus_zero" {
201	try ok(
202	\\[-0]
203	);
204	}
205
206	test "y_number_negative_int" {
207	try roundTrip(
208	\\[-123]
209	);
210	}
211
212	test "y_number_negative_one" {
213	try roundTrip(
214	\\[-1]
215	);
216	}
217
218	test "y_number_negative_zero" {
219	try ok(
220	\\[-0]
221	);
222	}
223
224	test "y_number_real_capital_e" {
225	try ok(
226	\\[1E22]
227	);
228	}
229
230	test "y_number_real_capital_e_neg_exp" {
231	try ok(
232	\\[1E-2]
233	);
234	}
235
236	test "y_number_real_capital_e_pos_exp" {
237	try ok(
238	\\[1E+2]
239	);
240	}
241
242	test "y_number_real_exponent" {
243	try ok(
244	\\[123e45]
245	);
246	}
247
248	test "y_number_real_fraction_exponent" {
249	try ok(
250	\\[123.456e78]
251	);
252	}
253
254	test "y_number_real_neg_exp" {
255	try ok(
256	\\[1e-2]
257	);
258	}
259
260	test "y_number_real_pos_exponent" {
261	try ok(
262	\\[1e+2]
263	);
264	}
265
266	test "y_number_simple_int" {
267	try roundTrip(
268	\\[123]
269	);
270	}
271
272	test "y_number_simple_real" {
273	try ok(
274	\\[123.456789]
275	);
276	}
277
278	test "y_object_basic" {
279	try roundTrip(
280	\\{"asd":"sdf"}
281	);
282	}
283
284	test "y_object_duplicated_key_and_value" {
285	try ok(
286	\\{"a":"b","a":"b"}
287	);
288	}
289
290	test "y_object_duplicated_key" {
291	try ok(
292	\\{"a":"b","a":"c"}
293	);
294	}
295
296	test "y_object_empty" {
297	try roundTrip(
298	\\{}
299	);
300	}
301
302	test "y_object_empty_key" {
303	try roundTrip(
304	\\{"":0}
305	);
306	}
307
308	test "y_object_escaped_null_in_key" {
309	try ok(
310	\\{"foo\u0000bar": 42}
311	);
312	}
313
314	test "y_object_extreme_numbers" {
315	try ok(
316	\\{ "min": -1.0e+28, "max": 1.0e+28 }
317	);
318	}
319
320	test "y_object" {
321	try ok(
322	\\{"asd":"sdf", "dfg":"fgh"}
323	);
324	}
325
326	test "y_object_long_strings" {
327	try ok(
328	\\{"x":[{"id": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"}], "id": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"}
329	);
330	}
331
332	test "y_object_simple" {
333	try roundTrip(
334	\\{"a":[]}
335	);
336	}
337
338	test "y_object_string_unicode" {
339	try ok(
340	\\{"title":"\u041f\u043e\u043b\u0442\u043e\u0440\u0430 \u0417\u0435\u043c\u043b\u0435\u043a\u043e\u043f\u0430" }
341	);
342	}
343
344	test "y_object_with_newlines" {
345	try ok(
346	\\{
347	\\"a": "b"
348	\\}
349	);
350	}
351
352	test "y_string_1_2_3_bytes_UTF-8_sequences" {
353	try ok(
354	\\["\u0060\u012a\u12AB"]
355	);
356	}
357
358	test "y_string_accepted_surrogate_pair" {
359	try ok(
360	\\["\uD801\udc37"]
361	);
362	}
363
364	test "y_string_accepted_surrogate_pairs" {
365	try ok(
366	\\["\ud83d\ude39\ud83d\udc8d"]
367	);
368	}
369
370	test "y_string_allowed_escapes" {
371	try ok(
372	\\["\"\\\/\b\f\n\r\t"]
373	);
374	}
375
376	test "y_string_backslash_and_u_escaped_zero" {
377	try ok(
378	\\["\\u0000"]
379	);
380	}
381
382	test "y_string_backslash_doublequotes" {
383	try roundTrip(
384	\\["\""]
385	);
386	}
387
388	test "y_string_comments" {
389	try ok(
390	\\["a/b/c/*d//e"]
391	);
392	}
393
394	test "y_string_double_escape_a" {
395	try ok(
396	\\["\\a"]
397	);
398	}
399
400	test "y_string_double_escape_n" {
401	try roundTrip(
402	\\["\\n"]
403	);
404	}
405
406	test "y_string_escaped_control_character" {
407	try ok(
408	\\["\u0012"]
409	);
410	}
411
412	test "y_string_escaped_noncharacter" {
413	try ok(
414	\\["\uFFFF"]
415	);
416	}
417
418	test "y_string_in_array" {
419	try ok(
420	\\["asd"]
421	);
422	}
423
424	test "y_string_in_array_with_leading_space" {
425	try ok(
426	\\[ "asd"]
427	);
428	}
429
430	test "y_string_last_surrogates_1_and_2" {
431	try ok(
432	\\["\uDBFF\uDFFF"]
433	);
434	}
435
436	test "y_string_nbsp_uescaped" {
437	try ok(
438	\\["new\u00A0line"]
439	);
440	}
441
442	test "y_string_nonCharacterInUTF-8_U+10FFFF" {
443	try ok(
444	\\["􏿿"]
445	);
446	}
447
448	test "y_string_nonCharacterInUTF-8_U+FFFF" {
449	try ok(
450	\\[""]
451	);
452	}
453
454	test "y_string_null_escape" {
455	try ok(
456	\\["\u0000"]
457	);
458	}
459
460	test "y_string_one-byte-utf-8" {
461	try ok(
462	\\["\u002c"]
463	);
464	}
465
466	test "y_string_pi" {
467	try ok(
468	\\["π"]
469	);
470	}
471
472	test "y_string_reservedCharacterInUTF-8_U+1BFFF" {
473	try ok(
474	\\["𛿿"]
475	);
476	}
477
478	test "y_string_simple_ascii" {
479	try ok(
480	\\["asd "]
481	);
482	}
483		49
484	test "y_string_space" {	50	var p = Parser.init(testing.allocator, .alloc_if_needed);
485	try roundTrip(	51	defer p.deinit();
486	\\" "
487	);
488	}
489		52
490	test "y_string_surrogates_U+1D11E_MUSICAL_SYMBOL_G_CLEF" {	53	var tree = try p.parse(s);
491	try ok(	54	defer tree.deinit();
492	\\["\uD834\uDd1e"]
493	);
494	}
495		55
496	test "y_string_three-byte-utf-8" {	56	var buf: [256]u8 = undefined;
497	try ok(	57	var fbs = std.io.fixedBufferStream(&buf);
498	\\["\u0821"]	58	try tree.root.jsonStringify(.{}, fbs.writer());
499	);
500	}
501		59
502	test "y_string_two-byte-utf-8" {	60	try testing.expectEqualStrings(s, fbs.getWritten());
503	try ok(
504	\\["\u0123"]
505	);
506	}	61	}
507		62
508	test "y_string_u+2028_line_sep" {	63	test "truncated UTF-8 sequence" {
509	try ok("[\"\xe2\x80\xa8\"]");	64	try err("\"\xc2\"");
		65	try err("\"\xdf\"");
		66	try err("\"\xed\xa0\"");
		67	try err("\"\xf0\x80\"");
		68	try err("\"\xf0\x80\x80\"");
510	}	69	}
511		70
512	test "y_string_u+2029_par_sep" {	71	test "invalid continuation byte" {
513	try ok("[\"\xe2\x80\xa9\"]");	72	try err("\"\xc2\x00\"");
		73	try err("\"\xc2\x7f\"");
		74	try err("\"\xc2\xc0\"");
		75	try err("\"\xc3\xc1\"");
		76	try err("\"\xc4\xf5\"");
		77	try err("\"\xc5\xff\"");
		78	try err("\"\xe4\x80\x00\"");
		79	try err("\"\xe5\x80\x10\"");
		80	try err("\"\xe6\x80\xc0\"");
		81	try err("\"\xe7\x80\xf5\"");
		82	try err("\"\xe8\x00\x80\"");
		83	try err("\"\xf2\x00\x80\x80\"");
		84	try err("\"\xf0\x80\x00\x80\"");
		85	try err("\"\xf1\x80\xc0\x80\"");
		86	try err("\"\xf2\x80\x80\x00\"");
		87	try err("\"\xf3\x80\x80\xc0\"");
		88	try err("\"\xf4\x80\x80\xf5\"");
514	}	89	}
515		90
516	test "y_string_uescaped_newline" {	91	test "disallowed overlong form" {
517	try ok(	92	try err("\"\xc0\x80\"");
518	\\["new\u000Aline"]	93	try err("\"\xc0\x90\"");
519	);	94	try err("\"\xc1\x80\"");
520	}	95	try err("\"\xc1\x90\"");
521		96	try err("\"\xe0\x80\x80\"");
522	test "y_string_uEscape" {	97	try err("\"\xf0\x80\x80\x80\"");
523	try ok(
524	\\["\u0061\u30af\u30EA\u30b9"]
525	);
526	}
527
528	test "y_string_unescaped_char_delete" {
529	try ok("[\"\x7f\"]");
530	}
531
532	test "y_string_unicode_2" {
533	try ok(
534	\\["⍂㈴⍂"]
535	);
536	}
537
538	test "y_string_unicodeEscapedBackslash" {
539	try ok(
540	\\["\u005C"]
541	);
542	}
543
544	test "y_string_unicode_escaped_double_quote" {
545	try ok(
546	\\["\u0022"]
547	);
548	}
549
550	test "y_string_unicode" {
551	try ok(
552	\\["\uA66D"]
553	);
554	}
555
556	test "y_string_unicode_U+10FFFE_nonchar" {
557	try ok(
558	\\["\uDBFF\uDFFE"]
559	);
560	}
561
562	test "y_string_unicode_U+1FFFE_nonchar" {
563	try ok(
564	\\["\uD83F\uDFFE"]
565	);
566	}
567
568	test "y_string_unicode_U+200B_ZERO_WIDTH_SPACE" {
569	try ok(
570	\\["\u200B"]
571	);
572	}
573
574	test "y_string_unicode_U+2064_invisible_plus" {
575	try ok(
576	\\["\u2064"]
577	);
578	}
579
580	test "y_string_unicode_U+FDD0_nonchar" {
581	try ok(
582	\\["\uFDD0"]
583	);
584	}
585
586	test "y_string_unicode_U+FFFE_nonchar" {
587	try ok(
588	\\["\uFFFE"]
589	);
590	}
591
592	test "y_string_utf8" {
593	try ok(
594	\\["€𝄞"]
595	);
596	}
597
598	test "y_string_with_del_character" {
599	try ok("[\"a\x7fa\"]");
600	}
601
602	test "y_structure_lonely_false" {
603	try roundTrip(
604	\\false
605	);
606	}
607
608	test "y_structure_lonely_int" {
609	try roundTrip(
610	\\42
611	);
612	}
613
614	test "y_structure_lonely_negative_real" {
615	try ok(
616	\\-0.1
617	);
618	}
619
620	test "y_structure_lonely_null" {
621	try roundTrip(
622	\\null
623	);
624	}
625
626	test "y_structure_lonely_string" {
627	try roundTrip(
628	\\"asd"
629	);
630	}
631
632	test "y_structure_lonely_true" {
633	try roundTrip(
634	\\true
635	);
636	}
637
638	test "y_structure_string_empty" {
639	try roundTrip(
640	\\""
641	);
642	}
643
644	test "y_structure_trailing_newline" {
645	try roundTrip(
646	\\["a"]
647	);
648	}
649
650	test "y_structure_true_in_array" {
651	try roundTrip(
652	\\[true]
653	);
654	}
655
656	test "y_structure_whitespace_array" {
657	try ok(" [] ");
658	}
659
660	////////////////////////////////////////////////////////////////////////////////////////////////////
661
662	test "n_array_1_true_without_comma" {
663	try err(
664	\\[1 true]
665	);
666	}
667
668	test "n_array_a_invalid_utf8" {
669	try err(
670	\\[aå]
671	);
672	}
673
674	test "n_array_colon_instead_of_comma" {
675	try err(
676	\\["": 1]
677	);
678	}
679
680	test "n_array_comma_after_close" {
681	try err(
682	\\[""],
683	);
684	}
685
686	test "n_array_comma_and_number" {
687	try err(
688	\\[,1]
689	);
690	}
691
692	test "n_array_double_comma" {
693	try err(
694	\\[1,,2]
695	);
696	}
697
698	test "n_array_double_extra_comma" {
699	try err(
700	\\["x",,]
701	);
702	}
703
704	test "n_array_extra_close" {
705	try err(
706	\\["x"]]
707	);
708	}
709
710	test "n_array_extra_comma" {
711	try err(
712	\\["",]
713	);
714	}
715
716	test "n_array_incomplete_invalid_value" {
717	try err(
718	\\[x
719	);
720	}
721
722	test "n_array_incomplete" {
723	try err(
724	\\["x"
725	);
726	}
727
728	test "n_array_inner_array_no_comma" {
729	try err(
730	\\[3[4]]
731	);
732	}
733
734	test "n_array_invalid_utf8" {
735	try err(
736	\\[ÿ]
737	);
738	}
739
740	test "n_array_items_separated_by_semicolon" {
741	try err(
742	\\[1:2]
743	);
744	}
745
746	test "n_array_just_comma" {
747	try err(
748	\\[,]
749	);
750	}
751
752	test "n_array_just_minus" {
753	try err(
754	\\[-]
755	);
756	}
757
758	test "n_array_missing_value" {
759	try err(
760	\\[ , ""]
761	);
762	}
763
764	test "n_array_newlines_unclosed" {
765	try err(
766	\\["a",
767	\\4
768	\\,1,
769	);
770	}
771
772	test "n_array_number_and_comma" {
773	try err(
774	\\[1,]
775	);
776	}
777
778	test "n_array_number_and_several_commas" {
779	try err(
780	\\[1,,]
781	);
782	}
783
784	test "n_array_spaces_vertical_tab_formfeed" {
785	try err("[\"\x0aa\"\\f]");
786	}
787
788	test "n_array_star_inside" {
789	try err(
790	\\[*]
791	);
792	}
793
794	test "n_array_unclosed" {
795	try err(
796	\\[""
797	);
798	}
799
800	test "n_array_unclosed_trailing_comma" {
801	try err(
802	\\[1,
803	);
804	}
805
806	test "n_array_unclosed_with_new_lines" {
807	try err(
808	\\[1,
809	\\1
810	\\,1
811	);
812	}
813
814	test "n_array_unclosed_with_object_inside" {
815	try err(
816	\\[{}
817	);
818	}
819
820	test "n_incomplete_false" {
821	try err(
822	\\[fals]
823	);
824	}
825
826	test "n_incomplete_null" {
827	try err(
828	\\[nul]
829	);
830	}
831
832	test "n_incomplete_true" {
833	try err(
834	\\[tru]
835	);
836	}
837
838	test "n_multidigit_number_then_00" {
839	try err("123\x00");
840	}
841
842	test "n_number_0.1.2" {
843	try err(
844	\\[0.1.2]
845	);
846	}
847
848	test "n_number_-01" {
849	try err(
850	\\[-01]
851	);
852	}
853
854	test "n_number_0.3e" {
855	try err(
856	\\[0.3e]
857	);
858	}
859
860	test "n_number_0.3e+" {
861	try err(
862	\\[0.3e+]
863	);
864	}
865
866	test "n_number_0_capital_E" {
867	try err(
868	\\[0E]
869	);
870	}
871
872	test "n_number_0_capital_E+" {
873	try err(
874	\\[0E+]
875	);
876	}
877
878	test "n_number_0.e1" {
879	try err(
880	\\[0.e1]
881	);
882	}
883
884	test "n_number_0e" {
885	try err(
886	\\[0e]
887	);
888	}
889
890	test "n_number_0e+" {
891	try err(
892	\\[0e+]
893	);
894	}
895
896	test "n_number_1_000" {
897	try err(
898	\\[1 000.0]
899	);
900	}
901
902	test "n_number_1.0e-" {
903	try err(
904	\\[1.0e-]
905	);
906	}
907
908	test "n_number_1.0e" {
909	try err(
910	\\[1.0e]
911	);
912	}
913
914	test "n_number_1.0e+" {
915	try err(
916	\\[1.0e+]
917	);
918	}
919
920	test "n_number_-1.0." {
921	try err(
922	\\[-1.0.]
923	);
924	}
925
926	test "n_number_1eE2" {
927	try err(
928	\\[1eE2]
929	);
930	}
931
932	test "n_number_.-1" {
933	try err(
934	\\[.-1]
935	);
936	}
937
938	test "n_number_+1" {
939	try err(
940	\\[+1]
941	);
942	}
943
944	test "n_number_.2e-3" {
945	try err(
946	\\[.2e-3]
947	);
948	}
949
950	test "n_number_2.e-3" {
951	try err(
952	\\[2.e-3]
953	);
954	}
955
956	test "n_number_2.e+3" {
957	try err(
958	\\[2.e+3]
959	);
960	}
961
962	test "n_number_2.e3" {
963	try err(
964	\\[2.e3]
965	);
966	}
967
968	test "n_number_-2." {
969	try err(
970	\\[-2.]
971	);
972	}
973
974	test "n_number_9.e+" {
975	try err(
976	\\[9.e+]
977	);
978	}
979
980	test "n_number_expression" {
981	try err(
982	\\[1+2]
983	);
984	}
985
986	test "n_number_hex_1_digit" {
987	try err(
988	\\[0x1]
989	);
990	}
991
992	test "n_number_hex_2_digits" {
993	try err(
994	\\[0x42]
995	);
996	}
997
998	test "n_number_infinity" {
999	try err(
1000	\\[Infinity]
1001	);
1002	}
1003
1004	test "n_number_+Inf" {
1005	try err(
1006	\\[+Inf]
1007	);
1008	}
1009
1010	test "n_number_Inf" {
1011	try err(
1012	\\[Inf]
1013	);
1014	}
1015
1016	test "n_number_invalid+-" {
1017	try err(
1018	\\[0e+-1]
1019	);
1020	}
1021
1022	test "n_number_invalid-negative-real" {
1023	try err(
1024	\\[-123.123foo]
1025	);
1026	}
1027
1028	test "n_number_invalid-utf-8-in-bigger-int" {
1029	try err(
1030	\\[123å]
1031	);
1032	}
1033
1034	test "n_number_invalid-utf-8-in-exponent" {
1035	try err(
1036	\\[1e1å]
1037	);
1038	}
1039
1040	test "n_number_invalid-utf-8-in-int" {
1041	try err(
1042	\\[0å]
1043	);
1044	}
1045
1046	test "n_number_++" {
1047	try err(
1048	\\[++1234]
1049	);
1050	}
1051
1052	test "n_number_minus_infinity" {
1053	try err(
1054	\\[-Infinity]
1055	);
1056	}
1057
1058	test "n_number_minus_sign_with_trailing_garbage" {
1059	try err(
1060	\\[-foo]
1061	);
1062	}
1063
1064	test "n_number_minus_space_1" {
1065	try err(
1066	\\[- 1]
1067	);
1068	}
1069
1070	test "n_number_-NaN" {
1071	try err(
1072	\\[-NaN]
1073	);
1074	}
1075
1076	test "n_number_NaN" {
1077	try err(
1078	\\[NaN]
1079	);
1080	}
1081
1082	test "n_number_neg_int_starting_with_zero" {
1083	try err(
1084	\\[-012]
1085	);
1086	}
1087
1088	test "n_number_neg_real_without_int_part" {
1089	try err(
1090	\\[-.123]
1091	);
1092	}
1093
1094	test "n_number_neg_with_garbage_at_end" {
1095	try err(
1096	\\[-1x]
1097	);
1098	}
1099
1100	test "n_number_real_garbage_after_e" {
1101	try err(
1102	\\[1ea]
1103	);
1104	}
1105
1106	test "n_number_real_with_invalid_utf8_after_e" {
1107	try err(
1108	\\[1eå]
1109	);
1110	}
1111
1112	test "n_number_real_without_fractional_part" {
1113	try err(
1114	\\[1.]
1115	);
1116	}
1117
1118	test "n_number_starting_with_dot" {
1119	try err(
1120	\\[.123]
1121	);
1122	}
1123
1124	test "n_number_U+FF11_fullwidth_digit_one" {
1125	try err(
1126	\\[ï¼]
1127	);
1128	}
1129
1130	test "n_number_with_alpha_char" {
1131	try err(
1132	\\[1.8011670033376514H-308]
1133	);
1134	}
1135
1136	test "n_number_with_alpha" {
1137	try err(
1138	\\[1.2a-3]
1139	);
1140	}
1141
1142	test "n_number_with_leading_zero" {
1143	try err(
1144	\\[012]
1145	);
1146	}
1147
1148	test "n_object_bad_value" {
1149	try err(
1150	\\["x", truth]
1151	);
1152	}
1153
1154	test "n_object_bracket_key" {
1155	try err(
1156	\\{[: "x"}
1157	);
1158	}
1159
1160	test "n_object_comma_instead_of_colon" {
1161	try err(
1162	\\{"x", null}
1163	);
1164	}
1165
1166	test "n_object_double_colon" {
1167	try err(
1168	\\{"x"::"b"}
1169	);
1170	}
1171
1172	test "n_object_emoji" {
1173	try err(
1174	\\{ð¨ð}
1175	);
1176	}
1177
1178	test "n_object_garbage_at_end" {
1179	try err(
1180	\\{"a":"a" 123}
1181	);
1182	}
1183
1184	test "n_object_key_with_single_quotes" {
1185	try err(
1186	\\{key: 'value'}
1187	);
1188	}
1189
1190	test "n_object_lone_continuation_byte_in_key_and_trailing_comma" {
1191	try err(
1192	\\{"¹":"0",}
1193	);
1194	}
1195
1196	test "n_object_missing_colon" {
1197	try err(
1198	\\{"a" b}
1199	);
1200	}
1201
1202	test "n_object_missing_key" {
1203	try err(
1204	\\{:"b"}
1205	);
1206	}
1207
1208	test "n_object_missing_semicolon" {
1209	try err(
1210	\\{"a" "b"}
1211	);
1212	}
1213
1214	test "n_object_missing_value" {
1215	try err(
1216	\\{"a":
1217	);
1218	}
1219
1220	test "n_object_no-colon" {
1221	try err(
1222	\\{"a"
1223	);
1224	}
1225
1226	test "n_object_non_string_key_but_huge_number_instead" {
1227	try err(
1228	\\{9999E9999:1}
1229	);
1230	}
1231
1232	test "n_object_non_string_key" {
1233	try err(
1234	\\{1:1}
1235	);
1236	}
1237
1238	test "n_object_repeated_null_null" {
1239	try err(
1240	\\{null:null,null:null}
1241	);
1242	}
1243
1244	test "n_object_several_trailing_commas" {
1245	try err(
1246	\\{"id":0,,,,,}
1247	);
1248	}
1249
1250	test "n_object_single_quote" {
1251	try err(
1252	\\{'a':0}
1253	);
1254	}
1255
1256	test "n_object_trailing_comma" {
1257	try err(
1258	\\{"id":0,}
1259	);
1260	}
1261
1262	test "n_object_trailing_comment" {
1263	try err(
1264	\\{"a":"b"}/**/
1265	);
1266	}
1267
1268	test "n_object_trailing_comment_open" {
1269	try err(
1270	\\{"a":"b"}/**//
1271	);
1272	}
1273
1274	test "n_object_trailing_comment_slash_open_incomplete" {
1275	try err(
1276	\\{"a":"b"}/
1277	);
1278	}
1279
1280	test "n_object_trailing_comment_slash_open" {
1281	try err(
1282	\\{"a":"b"}//
1283	);
1284	}
1285
1286	test "n_object_two_commas_in_a_row" {
1287	try err(
1288	\\{"a":"b",,"c":"d"}
1289	);
1290	}
1291
1292	test "n_object_unquoted_key" {
1293	try err(
1294	\\{a: "b"}
1295	);
1296	}
1297
1298	test "n_object_unterminated-value" {
1299	try err(
1300	\\{"a":"a
1301	);
1302	}
1303
1304	test "n_object_with_single_string" {
1305	try err(
1306	\\{ "foo" : "bar", "a" }
1307	);
1308	}
1309
1310	test "n_object_with_trailing_garbage" {
1311	try err(
1312	\\{"a":"b"}#
1313	);
1314	}
1315
1316	test "n_single_space" {
1317	try err(" ");
1318	}
1319
1320	test "n_string_1_surrogate_then_escape" {
1321	try err(
1322	\\["\uD800\"]
1323	);
1324	}
1325
1326	test "n_string_1_surrogate_then_escape_u1" {
1327	try err(
1328	\\["\uD800\u1"]
1329	);
1330	}
1331
1332	test "n_string_1_surrogate_then_escape_u1x" {
1333	try err(
1334	\\["\uD800\u1x"]
1335	);
1336	}
1337
1338	test "n_string_1_surrogate_then_escape_u" {
1339	try err(
1340	\\["\uD800\u"]
1341	);
1342	}
1343
1344	test "n_string_accentuated_char_no_quotes" {
1345	try err(
1346	\\[Ã©]
1347	);
1348	}
1349
1350	test "n_string_backslash_00" {
1351	try err("[\"\x00\"]");
1352	}
1353
1354	test "n_string_escaped_backslash_bad" {
1355	try err(
1356	\\["\\\"]
1357	);
1358	}
1359
1360	test "n_string_escaped_ctrl_char_tab" {
1361	try err("\x5b\x22\x5c\x09\x22\x5d");
1362	}
1363
1364	test "n_string_escaped_emoji" {
1365	try err("[\"\x5c\xc3\xb0\xc2\x9f\xc2\x8c\xc2\x80\"]");
1366	}
1367
1368	test "n_string_escape_x" {
1369	try err(
1370	\\["\x00"]
1371	);
1372	}
1373
1374	test "n_string_incomplete_escaped_character" {
1375	try err(
1376	\\["\u00A"]
1377	);
1378	}
1379
1380	test "n_string_incomplete_escape" {
1381	try err(
1382	\\["\"]
1383	);
1384	}
1385
1386	test "n_string_incomplete_surrogate_escape_invalid" {
1387	try err(
1388	\\["\uD800\uD800\x"]
1389	);
1390	}
1391
1392	test "n_string_incomplete_surrogate" {
1393	try err(
1394	\\["\uD834\uDd"]
1395	);
1396	}
1397
1398	test "n_string_invalid_backslash_esc" {
1399	try err(
1400	\\["\a"]
1401	);
1402	}
1403
1404	test "n_string_invalid_unicode_escape" {
1405	try err(
1406	\\["\uqqqq"]
1407	);
1408	}
1409
1410	test "n_string_invalid_utf8_after_escape" {
1411	try err("[\"\\\x75\xc3\xa5\"]");
1412	}
1413
1414	test "n_string_invalid-utf-8-in-escape" {
1415	try err(
1416	\\["\uå"]
1417	);
1418	}
1419
1420	test "n_string_leading_uescaped_thinspace" {
1421	try err(
1422	\\[\u0020"asd"]
1423	);
1424	}
1425
1426	test "n_string_no_quotes_with_bad_escape" {
1427	try err(
1428	\\[\n]
1429	);
1430	}
1431
1432	test "n_string_single_doublequote" {
1433	try err(
1434	\\"
1435	);
1436	}
1437
1438	test "n_string_single_quote" {
1439	try err(
1440	\\['single quote']
1441	);
1442	}
1443
1444	test "n_string_single_string_no_double_quotes" {
1445	try err(
1446	\\abc
1447	);
1448	}
1449
1450	test "n_string_start_escape_unclosed" {
1451	try err(
1452	\\["\
1453	);
1454	}
1455
1456	test "n_string_unescaped_crtl_char" {
1457	try err("[\"a\x00a\"]");
1458	}
1459
1460	test "n_string_unescaped_newline" {
1461	try err(
1462	\\["new
1463	\\line"]
1464	);
1465	}
1466
1467	test "n_string_unescaped_tab" {
1468	try err("[\"\t\"]");
1469	}
1470
1471	test "n_string_unicode_CapitalU" {
1472	try err(
1473	\\"\UA66D"
1474	);
1475	}
1476
1477	test "n_string_with_trailing_garbage" {
1478	try err(
1479	\\""x
1480	);
1481	}
1482
1483	test "n_structure_100000_opening_arrays" {
1484	try err("[" ** 100000);
1485	}
1486
1487	test "n_structure_angle_bracket_." {
1488	try err(
1489	\\<.>
1490	);
1491	}
1492
1493	test "n_structure_angle_bracket_null" {
1494	try err(
1495	\\[<null>]
1496	);
1497	}
1498
1499	test "n_structure_array_trailing_garbage" {
1500	try err(
1501	\\[1]x
1502	);
1503	}
1504
1505	test "n_structure_array_with_extra_array_close" {
1506	try err(
1507	\\[1]]
1508	);
1509	}
1510
1511	test "n_structure_array_with_unclosed_string" {
1512	try err(
1513	\\["asd]
1514	);
1515	}
1516
1517	test "n_structure_ascii-unicode-identifier" {
1518	try err(
1519	\\aÃ¥
1520	);
1521	}
1522
1523	test "n_structure_capitalized_True" {
1524	try err(
1525	\\[True]
1526	);
1527	}
1528
1529	test "n_structure_close_unopened_array" {
1530	try err(
1531	\\1]
1532	);
1533	}
1534
1535	test "n_structure_comma_instead_of_closing_brace" {
1536	try err(
1537	\\{"x": true,
1538	);
1539	}
1540
1541	test "n_structure_double_array" {
1542	try err(
1543	\\[][]
1544	);
1545	}
1546
1547	test "n_structure_end_array" {
1548	try err(
1549	\\]
1550	);
1551	}
1552
1553	test "n_structure_incomplete_UTF8_BOM" {
1554	try err(
1555	\\ï»{}
1556	);
1557	}
1558
1559	test "n_structure_lone-invalid-utf-8" {
1560	try err(
1561	\\å
1562	);
1563	}
1564
1565	test "n_structure_lone-open-bracket" {
1566	try err(
1567	\\[
1568	);
1569	}
1570
1571	test "n_structure_no_data" {
1572	try err(
1573	\\
1574	);
1575	}
1576
1577	test "n_structure_null-byte-outside-string" {
1578	try err("[\x00]");
1579	}
1580
1581	test "n_structure_number_with_trailing_garbage" {
1582	try err(
1583	\\2@
1584	);
1585	}
1586
1587	test "n_structure_object_followed_by_closing_object" {
1588	try err(
1589	\\{}}
1590	);
1591	}
1592
1593	test "n_structure_object_unclosed_no_value" {
1594	try err(
1595	\\{"":
1596	);
1597	}
1598
1599	test "n_structure_object_with_comment" {
1600	try err(
1601	\\{"a":/comment/"b"}
1602	);
1603	}
1604
1605	test "n_structure_object_with_trailing_garbage" {
1606	try err(
1607	\\{"a": true} "x"
1608	);
1609	}
1610
1611	test "n_structure_open_array_apostrophe" {
1612	try err(
1613	\\['
1614	);
1615	}
1616
1617	test "n_structure_open_array_comma" {
1618	try err(
1619	\\[,
1620	);
1621	}
1622
1623	test "n_structure_open_array_object" {
1624	try err("[{\"\":" ** 50000);
1625	}
1626
1627	test "n_structure_open_array_open_object" {
1628	try err(
1629	\\[{
1630	);
1631	}
1632
1633	test "n_structure_open_array_open_string" {
1634	try err(
1635	\\["a
1636	);
1637	}
1638
1639	test "n_structure_open_array_string" {
1640	try err(
1641	\\["a"
1642	);
1643	}
1644
1645	test "n_structure_open_object_close_array" {
1646	try err(
1647	\\{]
1648	);
1649	}
1650
1651	test "n_structure_open_object_comma" {
1652	try err(
1653	\\{,
1654	);
1655	}
1656
1657	test "n_structure_open_object" {
1658	try err(
1659	\\{
1660	);
1661	}
1662
1663	test "n_structure_open_object_open_array" {
1664	try err(
1665	\\{[
1666	);
1667	}
1668
1669	test "n_structure_open_object_open_string" {
1670	try err(
1671	\\{"a
1672	);
1673	}
1674
1675	test "n_structure_open_object_string_with_apostrophes" {
1676	try err(
1677	\\{'a'
1678	);
1679	}
1680
1681	test "n_structure_open_open" {
1682	try err(
1683	\\["\{["\{["\{["\{
1684	);
1685	}
1686
1687	test "n_structure_single_eacute" {
1688	try err(
1689	\\é
1690	);
1691	}
1692
1693	test "n_structure_single_star" {
1694	try err(
1695	\\*
1696	);
1697	}
1698
1699	test "n_structure_trailing_#" {
1700	try err(
1701	\\{"a":"b"}#{}
1702	);
1703	}
1704
1705	test "n_structure_U+2060_word_joined" {
1706	try err(
1707	\\[â ]
1708	);
1709	}
1710
1711	test "n_structure_uescaped_LF_before_string" {
1712	try err(
1713	\\[\u000A""]
1714	);
1715	}
1716
1717	test "n_structure_unclosed_array" {
1718	try err(
1719	\\[1
1720	);
1721	}
1722
1723	test "n_structure_unclosed_array_partial_null" {
1724	try err(
1725	\\[ false, nul
1726	);
1727	}
1728
1729	test "n_structure_unclosed_array_unfinished_false" {
1730	try err(
1731	\\[ true, fals
1732	);
1733	}
1734
1735	test "n_structure_unclosed_array_unfinished_true" {
1736	try err(
1737	\\[ false, tru
1738	);
1739	}
1740
1741	test "n_structure_unclosed_object" {
1742	try err(
1743	\\{"asd":"asd"
1744	);
1745	}
1746
1747	test "n_structure_unicode-identifier" {
1748	try err(
1749	\\Ã¥
1750	);
1751	}
1752
1753	test "n_structure_UTF8_BOM_no_data" {
1754	try err(
1755	\\ï»¿
1756	);
1757	}
1758
1759	test "n_structure_whitespace_formfeed" {
1760	try err("[\x0c]");
1761	}
1762
1763	test "n_structure_whitespace_U+2060_word_joiner" {
1764	try err(
1765	\\[â ]
1766	);
1767	}
1768
1769	////////////////////////////////////////////////////////////////////////////////////////////////////
1770
1771	test "i_number_double_huge_neg_exp" {
1772	try any(
1773	\\[123.456e-789]
1774	);
1775	}
1776
1777	test "i_number_huge_exp" {
1778	try any(
1779	\\[0.4e00669999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999969999999006]
1780	);
1781	}
1782
1783	test "i_number_neg_int_huge_exp" {
1784	try any(
1785	\\[-1e+9999]
1786	);
1787	}
1788
1789	test "i_number_pos_double_huge_exp" {
1790	try any(
1791	\\[1.5e+9999]
1792	);
1793	}
1794
1795	test "i_number_real_neg_overflow" {
1796	try any(
1797	\\[-123123e100000]
1798	);
1799	}
1800
1801	test "i_number_real_pos_overflow" {
1802	try any(
1803	\\[123123e100000]
1804	);
1805	}
1806
1807	test "i_number_real_underflow" {
1808	try any(
1809	\\[123e-10000000]
1810	);
1811	}
1812
1813	test "i_number_too_big_neg_int" {
1814	try any(
1815	\\[-123123123123123123123123123123]
1816	);
1817	}
1818
1819	test "i_number_too_big_pos_int" {
1820	try any(
1821	\\[100000000000000000000]
1822	);
1823	}
1824
1825	test "i_number_very_big_negative_int" {
1826	try any(
1827	\\[-237462374673276894279832749832423479823246327846]
1828	);
1829	}
1830
1831	test "i_object_key_lone_2nd_surrogate" {
1832	try anyStreamingErrNonStreaming(
1833	\\{"\uDFAA":0}
1834	);
1835	}
1836
1837	test "i_string_1st_surrogate_but_2nd_missing" {
1838	try anyStreamingErrNonStreaming(
1839	\\["\uDADA"]
1840	);
1841	}
1842
1843	test "i_string_1st_valid_surrogate_2nd_invalid" {
1844	try anyStreamingErrNonStreaming(
1845	\\["\uD888\u1234"]
1846	);
1847	}
1848
1849	test "i_string_incomplete_surrogate_and_escape_valid" {
1850	try anyStreamingErrNonStreaming(
1851	\\["\uD800\n"]
1852	);
1853	}
1854
1855	test "i_string_incomplete_surrogate_pair" {
1856	try anyStreamingErrNonStreaming(
1857	\\["\uDd1ea"]
1858	);
1859	}
1860
1861	test "i_string_incomplete_surrogates_escape_valid" {
1862	try anyStreamingErrNonStreaming(
1863	\\["\uD800\uD800\n"]
1864	);
1865	}
1866
1867	test "i_string_invalid_lonely_surrogate" {
1868	try anyStreamingErrNonStreaming(
1869	\\["\ud800"]
1870	);
1871	}
1872
1873	test "i_string_invalid_surrogate" {
1874	try anyStreamingErrNonStreaming(
1875	\\["\ud800abc"]
1876	);
1877	}
1878
1879	test "i_string_invalid_utf-8" {
1880	try any(
1881	\\["ÿ"]
1882	);
1883	}
1884
1885	test "i_string_inverted_surrogates_U+1D11E" {
1886	try anyStreamingErrNonStreaming(
1887	\\["\uDd1e\uD834"]
1888	);
1889	}
1890
1891	test "i_string_iso_latin_1" {
1892	try any(
1893	\\["é"]
1894	);
1895	}
1896
1897	test "i_string_lone_second_surrogate" {
1898	try anyStreamingErrNonStreaming(
1899	\\["\uDFAA"]
1900	);
1901	}
1902
1903	test "i_string_lone_utf8_continuation_byte" {
1904	try any(
1905	\\[""]
1906	);
1907	}
1908
1909	test "i_string_not_in_unicode_range" {
1910	try any(
1911	\\["ô¿¿¿"]
1912	);
1913	}
1914
1915	test "i_string_overlong_sequence_2_bytes" {
1916	try any(
1917	\\["À¯"]
1918	);
1919	}
1920
1921	test "i_string_overlong_sequence_6_bytes" {
1922	try any(
1923	\\["ü¿¿¿¿"]
1924	);
1925	}
1926
1927	test "i_string_overlong_sequence_6_bytes_null" {
1928	try any(
1929	\\["ü"]
1930	);
1931	}
1932
1933	test "i_string_truncated-utf-8" {
1934	try any(
1935	\\["àÿ"]
1936	);
1937	}
1938
1939	test "i_string_utf16BE_no_BOM" {
1940	try any("\x00\x5b\x00\x22\x00\xc3\xa9\x00\x22\x00\x5d");
1941	}
1942
1943	test "i_string_utf16LE_no_BOM" {
1944	try any("\x5b\x00\x22\x00\xc3\xa9\x00\x22\x00\x5d\x00");
1945	}
1946
1947	test "i_string_UTF-16LE_with_BOM" {
1948	try any("\xc3\xbf\xc3\xbe\x5b\x00\x22\x00\xc3\xa9\x00\x22\x00\x5d\x00");
1949	}
1950
1951	test "i_string_UTF-8_invalid_sequence" {
1952	try any(
1953	\\["æ¥Ñú"]
1954	);
1955	}
1956
1957	test "i_string_UTF8_surrogate_U+D800" {
1958	try any(
1959	\\["í "]
1960	);
1961	}
1962
1963	test "i_structure_500_nested_arrays" {
1964	try any(("[" 500) ++ ("]" 500));
1965	}
1966
1967	test "i_structure_UTF-8_BOM_empty_object" {
1968	try any(
1969	\\ï»¿{}
1970	);
1971	}
1972
1973	test "truncated UTF-8 sequence" {
1974	try utf8Error("\"\xc2\"");
1975	try utf8Error("\"\xdf\"");
1976	try utf8Error("\"\xed\xa0\"");
1977	try utf8Error("\"\xf0\x80\"");
1978	try utf8Error("\"\xf0\x80\x80\"");
1979	}
1980
1981	test "invalid continuation byte" {
1982	try utf8Error("\"\xc2\x00\"");
1983	try utf8Error("\"\xc2\x7f\"");
1984	try utf8Error("\"\xc2\xc0\"");
1985	try utf8Error("\"\xc3\xc1\"");
1986	try utf8Error("\"\xc4\xf5\"");
1987	try utf8Error("\"\xc5\xff\"");
1988	try utf8Error("\"\xe4\x80\x00\"");
1989	try utf8Error("\"\xe5\x80\x10\"");
1990	try utf8Error("\"\xe6\x80\xc0\"");
1991	try utf8Error("\"\xe7\x80\xf5\"");
1992	try utf8Error("\"\xe8\x00\x80\"");
1993	try utf8Error("\"\xf2\x00\x80\x80\"");
1994	try utf8Error("\"\xf0\x80\x00\x80\"");
1995	try utf8Error("\"\xf1\x80\xc0\x80\"");
1996	try utf8Error("\"\xf2\x80\x80\x00\"");
1997	try utf8Error("\"\xf3\x80\x80\xc0\"");
1998	try utf8Error("\"\xf4\x80\x80\xf5\"");
1999	}
2000
2001	test "disallowed overlong form" {
2002	try utf8Error("\"\xc0\x80\"");
2003	try utf8Error("\"\xc0\x90\"");
2004	try utf8Error("\"\xc1\x80\"");
2005	try utf8Error("\"\xc1\x90\"");
2006	try utf8Error("\"\xe0\x80\x80\"");
2007	try utf8Error("\"\xf0\x80\x80\x80\"");
2008	}	98	}
2009		99
2010	test "out of UTF-16 range" {	100	test "out of UTF-16 range" {
2011	try utf8Error("\"\xf4\x90\x80\x80\"");	101	try err("\"\xf4\x90\x80\x80\"");
2012	try utf8Error("\"\xf5\x80\x80\x80\"");	102	try err("\"\xf5\x80\x80\x80\"");
2013	try utf8Error("\"\xf6\x80\x80\x80\"");	103	try err("\"\xf6\x80\x80\x80\"");
2014	try utf8Error("\"\xf7\x80\x80\x80\"");	104	try err("\"\xf7\x80\x80\x80\"");
2015	try utf8Error("\"\xf8\x80\x80\x80\"");	105	try err("\"\xf8\x80\x80\x80\"");
2016	try utf8Error("\"\xf9\x80\x80\x80\"");	106	try err("\"\xf9\x80\x80\x80\"");
2017	try utf8Error("\"\xfa\x80\x80\x80\"");	107	try err("\"\xfa\x80\x80\x80\"");
2018	try utf8Error("\"\xfb\x80\x80\x80\"");	108	try err("\"\xfb\x80\x80\x80\"");
2019	try utf8Error("\"\xfc\x80\x80\x80\"");	109	try err("\"\xfc\x80\x80\x80\"");
2020	try utf8Error("\"\xfd\x80\x80\x80\"");	110	try err("\"\xfd\x80\x80\x80\"");
2021	try utf8Error("\"\xfe\x80\x80\x80\"");	111	try err("\"\xfe\x80\x80\x80\"");
2022	try utf8Error("\"\xff\x80\x80\x80\"");	112	try err("\"\xff\x80\x80\x80\"");
2023	}
2024
2025	test "parse" {
2026	var ts = TokenStream.init("false");
2027	try testing.expectEqual(false, try parse(bool, &ts, ParseOptions{}));
2028	ts = TokenStream.init("true");
2029	try testing.expectEqual(true, try parse(bool, &ts, ParseOptions{}));
2030	ts = TokenStream.init("1");
2031	try testing.expectEqual(@as(u1, 1), try parse(u1, &ts, ParseOptions{}));
2032	ts = TokenStream.init("50");
2033	try testing.expectError(error.Overflow, parse(u1, &ts, ParseOptions{}));
2034	ts = TokenStream.init("42");
2035	try testing.expectEqual(@as(u64, 42), try parse(u64, &ts, ParseOptions{}));
2036	ts = TokenStream.init("42.0");
2037	try testing.expectEqual(@as(f64, 42), try parse(f64, &ts, ParseOptions{}));
2038	ts = TokenStream.init("null");
2039	try testing.expectEqual(@as(?bool, null), try parse(?bool, &ts, ParseOptions{}));
2040	ts = TokenStream.init("true");
2041	try testing.expectEqual(@as(?bool, true), try parse(?bool, &ts, ParseOptions{}));
2042
2043	ts = TokenStream.init("\"foo\"");
2044	try testing.expectEqual(@as([3]u8, "foo".*), try parse([3]u8, &ts, ParseOptions{}));
2045	ts = TokenStream.init("[102, 111, 111]");
2046	try testing.expectEqual(@as([3]u8, "foo".*), try parse([3]u8, &ts, ParseOptions{}));
2047	ts = TokenStream.init("[]");
2048	try testing.expectEqual(@as([0]u8, undefined), try parse([0]u8, &ts, ParseOptions{}));
2049
2050	ts = TokenStream.init("\"12345678901234567890\"");
2051	try testing.expectEqual(@as(u64, 12345678901234567890), try parse(u64, &ts, ParseOptions{}));
2052	ts = TokenStream.init("\"123.456\"");
2053	try testing.expectEqual(@as(f64, 123.456), try parse(f64, &ts, ParseOptions{}));
2054	}
2055
2056	test "parse into enum" {
2057	const T = enum(u32) {
2058	Foo = 42,
2059	Bar,
2060	@"with\\escape",
2061	};
2062	var ts = TokenStream.init("\"Foo\"");
2063	try testing.expectEqual(@as(T, .Foo), try parse(T, &ts, ParseOptions{}));
2064	ts = TokenStream.init("42");
2065	try testing.expectEqual(@as(T, .Foo), try parse(T, &ts, ParseOptions{}));
2066	ts = TokenStream.init("\"with\\\\escape\"");
2067	try testing.expectEqual(@as(T, .@"with\\escape"), try parse(T, &ts, ParseOptions{}));
2068	ts = TokenStream.init("5");
2069	try testing.expectError(error.InvalidEnumTag, parse(T, &ts, ParseOptions{}));
2070	ts = TokenStream.init("\"Qux\"");
2071	try testing.expectError(error.InvalidEnumTag, parse(T, &ts, ParseOptions{}));
2072	}
2073
2074	test "parse with trailing data" {
2075	var ts = TokenStream.init("falsed");
2076	try testing.expectEqual(false, try parse(bool, &ts, ParseOptions{ .allow_trailing_data = true }));
2077	ts = TokenStream.init("falsed");
2078	try testing.expectError(error.InvalidTopLevelTrailing, parse(bool, &ts, ParseOptions{ .allow_trailing_data = false }));
2079	// trailing whitespace is okay
2080	ts = TokenStream.init("false \n");
2081	try testing.expectEqual(false, try parse(bool, &ts, ParseOptions{ .allow_trailing_data = false }));
2082	}
2083
2084	test "parse into that allocates a slice" {
2085	var ts = TokenStream.init("\"foo\"");
2086	try testing.expectError(error.AllocatorRequired, parse([]u8, &ts, ParseOptions{}));
2087
2088	const options = ParseOptions{ .allocator = testing.allocator };
2089	{
2090	ts = TokenStream.init("\"foo\"");
2091	const r = try parse([]u8, &ts, options);
2092	defer parseFree([]u8, r, options);
2093	try testing.expectEqualSlices(u8, "foo", r);
2094	}
2095	{
2096	ts = TokenStream.init("[102, 111, 111]");
2097	const r = try parse([]u8, &ts, options);
2098	defer parseFree([]u8, r, options);
2099	try testing.expectEqualSlices(u8, "foo", r);
2100	}
2101	{
2102	ts = TokenStream.init("\"with\\\\escape\"");
2103	const r = try parse([]u8, &ts, options);
2104	defer parseFree([]u8, r, options);
2105	try testing.expectEqualSlices(u8, "with\\escape", r);
2106	}
2107	}
2108
2109	test "parse into tagged union" {
2110	{
2111	const T = union(enum) {
2112	int: i32,
2113	float: f64,
2114	string: []const u8,
2115	};
2116	var ts = TokenStream.init("1.5");
2117	try testing.expectEqual(T{ .float = 1.5 }, try parse(T, &ts, ParseOptions{}));
2118	}
2119
2120	{ // failing allocations should be bubbled up instantly without trying next member
2121	var fail_alloc = testing.FailingAllocator.init(testing.allocator, 0);
2122	const options = ParseOptions{ .allocator = fail_alloc.allocator() };
2123	const T = union(enum) {
2124	// both fields here match the input
2125	string: []const u8,
2126	array: [3]u8,
2127	};
2128	var ts = TokenStream.init("[1,2,3]");
2129	try testing.expectError(error.OutOfMemory, parse(T, &ts, options));
2130	}
2131
2132	{
2133	// if multiple matches possible, takes first option
2134	const T = union(enum) {
2135	x: u8,
2136	y: u8,
2137	};
2138	var ts = TokenStream.init("42");
2139	try testing.expectEqual(T{ .x = 42 }, try parse(T, &ts, ParseOptions{}));
2140	}
2141
2142	{ // needs to back out when first union member doesn't match
2143	const T = union(enum) {
2144	A: struct { x: u32 },
2145	B: struct { y: u32 },
2146	};
2147	var ts = TokenStream.init("{\"y\":42}");
2148	try testing.expectEqual(T{ .B = .{ .y = 42 } }, try parse(T, &ts, ParseOptions{}));
2149	}
2150	}
2151
2152	test "parse union bubbles up AllocatorRequired" {
2153	{ // string member first in union (and not matching)
2154	const T = union(enum) {
2155	string: []const u8,
2156	int: i32,
2157	};
2158	var ts = TokenStream.init("42");
2159	try testing.expectError(error.AllocatorRequired, parse(T, &ts, ParseOptions{}));
2160	}
2161
2162	{ // string member not first in union (and matching)
2163	const T = union(enum) {
2164	int: i32,
2165	float: f64,
2166	string: []const u8,
2167	};
2168	var ts = TokenStream.init("\"foo\"");
2169	try testing.expectError(error.AllocatorRequired, parse(T, &ts, ParseOptions{}));
2170	}
2171	}
2172
2173	test "parseFree descends into tagged union" {
2174	var fail_alloc = testing.FailingAllocator.init(testing.allocator, 1);
2175	const options = ParseOptions{ .allocator = fail_alloc.allocator() };
2176	const T = union(enum) {
2177	int: i32,
2178	float: f64,
2179	string: []const u8,
2180	};
2181	// use a string with unicode escape so we know result can't be a reference to global constant
2182	var ts = TokenStream.init("\"with\\u0105unicode\"");
2183	const r = try parse(T, &ts, options);
2184	try testing.expectEqual(std.meta.Tag(T).string, @as(std.meta.Tag(T), r));
2185	try testing.expectEqualSlices(u8, "withąunicode", r.string);
2186	try testing.expectEqual(@as(usize, 0), fail_alloc.deallocations);
2187	parseFree(T, r, options);
2188	try testing.expectEqual(@as(usize, 1), fail_alloc.deallocations);
2189	}
2190
2191	test "parse with comptime field" {
2192	{
2193	const T = struct {
2194	comptime a: i32 = 0,
2195	b: bool,
2196	};
2197	var ts = TokenStream.init(
2198	\\{
2199	\\ "a": 0,
2200	\\ "b": true
2201	\\}
2202	);
2203	try testing.expectEqual(T{ .a = 0, .b = true }, try parse(T, &ts, ParseOptions{}));
2204	}
2205
2206	{ // string comptime values currently require an allocator
2207	const T = union(enum) {
2208	foo: struct {
2209	comptime kind: []const u8 = "boolean",
2210	b: bool,
2211	},
2212	bar: struct {
2213	comptime kind: []const u8 = "float",
2214	b: f64,
2215	},
2216	};
2217
2218	const options = ParseOptions{
2219	.allocator = std.testing.allocator,
2220	};
2221
2222	var ts = TokenStream.init(
2223	\\{
2224	\\ "kind": "float",
2225	\\ "b": 1.0
2226	\\}
2227	);
2228	const r = try parse(T, &ts, options);
2229
2230	// check that parseFree doesn't try to free comptime fields
2231	parseFree(T, r, options);
2232	}
2233	}
2234
2235	test "parse into struct with no fields" {
2236	const T = struct {};
2237	var ts = TokenStream.init("{}");
2238	try testing.expectEqual(T{}, try parse(T, &ts, ParseOptions{}));
2239	}
2240
2241	const test_const_value: usize = 123;
2242
2243	test "parse into struct with default const pointer field" {
2244	const T = struct { a: *const usize = &test_const_value };
2245	var ts = TokenStream.init("{}");
2246	try testing.expectEqual(T{}, try parse(T, &ts, .{}));
2247	}
2248
2249	const test_default_usize: usize = 123;
2250	const test_default_usize_ptr: *align(1) const usize = &test_default_usize;
2251	const test_default_str: []const u8 = "test str";
2252	const test_default_str_slice: [2][]const u8 = [_][]const u8{
2253	"test1",
2254	"test2",
2255	};
2256
2257	test "freeing parsed structs with pointers to default values" {
2258	const T = struct {
2259	int: *const usize = &test_default_usize,
2260	int_ptr: *allowzero align(1) const usize = test_default_usize_ptr,
2261	str: []const u8 = test_default_str,
2262	str_slice: []const []const u8 = &test_default_str_slice,
2263	};
2264
2265	var ts = json.TokenStream.init("{}");
2266	const options = .{ .allocator = std.heap.page_allocator };
2267	const parsed = try json.parse(T, &ts, options);
2268
2269	try testing.expectEqual(T{}, parsed);
2270
2271	json.parseFree(T, parsed, options);
2272	}
2273
2274	test "parse into struct where destination and source lengths mismatch" {
2275	const T = struct { a: [2]u8 };
2276	var ts = TokenStream.init("{\"a\": \"bbb\"}");
2277	try testing.expectError(error.LengthMismatch, parse(T, &ts, ParseOptions{}));
2278	}
2279
2280	test "parse into struct with misc fields" {
2281	@setEvalBranchQuota(10000);
2282	const options = ParseOptions{ .allocator = testing.allocator };
2283	const T = struct {
2284	int: i64,
2285	float: f64,
2286	@"with\\escape": bool,
2287	@"withąunicode😂": bool,
2288	language: []const u8,
2289	optional: ?bool,
2290	default_field: i32 = 42,
2291	static_array: [3]f64,
2292	dynamic_array: []f64,
2293
2294	complex: struct {
2295	nested: []const u8,
2296	},
2297
2298	veryComplex: []struct {
2299	foo: []const u8,
2300	},
2301
2302	a_union: Union,
2303	const Union = union(enum) {
2304	x: u8,
2305	float: f64,
2306	string: []const u8,
2307	};
2308	};
2309	var ts = TokenStream.init(
2310	\\{
2311	\\ "int": 420,
2312	\\ "float": 3.14,
2313	\\ "with\\escape": true,
2314	\\ "with\u0105unicode\ud83d\ude02": false,
2315	\\ "language": "zig",
2316	\\ "optional": null,
2317	\\ "static_array": [66.6, 420.420, 69.69],
2318	\\ "dynamic_array": [66.6, 420.420, 69.69],
2319	\\ "complex": {
2320	\\ "nested": "zig"
2321	\\ },
2322	\\ "veryComplex": [
2323	\\ {
2324	\\ "foo": "zig"
2325	\\ }, {
2326	\\ "foo": "rocks"
2327	\\ }
2328	\\ ],
2329	\\ "a_union": 100000
2330	\\}
2331	);
2332	const r = try parse(T, &ts, options);
2333	defer parseFree(T, r, options);
2334	try testing.expectEqual(@as(i64, 420), r.int);
2335	try testing.expectEqual(@as(f64, 3.14), r.float);
2336	try testing.expectEqual(true, r.@"with\\escape");
2337	try testing.expectEqual(false, r.@"withąunicode😂");
2338	try testing.expectEqualSlices(u8, "zig", r.language);
2339	try testing.expectEqual(@as(?bool, null), r.optional);
2340	try testing.expectEqual(@as(i32, 42), r.default_field);
2341	try testing.expectEqual(@as(f64, 66.6), r.static_array[0]);
2342	try testing.expectEqual(@as(f64, 420.420), r.static_array[1]);
2343	try testing.expectEqual(@as(f64, 69.69), r.static_array[2]);
2344	try testing.expectEqual(@as(usize, 3), r.dynamic_array.len);
2345	try testing.expectEqual(@as(f64, 66.6), r.dynamic_array[0]);
2346	try testing.expectEqual(@as(f64, 420.420), r.dynamic_array[1]);
2347	try testing.expectEqual(@as(f64, 69.69), r.dynamic_array[2]);
2348	try testing.expectEqualSlices(u8, r.complex.nested, "zig");
2349	try testing.expectEqualSlices(u8, "zig", r.veryComplex[0].foo);
2350	try testing.expectEqualSlices(u8, "rocks", r.veryComplex[1].foo);
2351	try testing.expectEqual(T.Union{ .float = 100000 }, r.a_union);
2352	}
2353
2354	test "parse into struct with strings and arrays with sentinels" {
2355	@setEvalBranchQuota(10000);
2356	const options = ParseOptions{ .allocator = testing.allocator };
2357	const T = struct {
2358	language: [:0]const u8,
2359	language_without_sentinel: []const u8,
2360	data: [:99]const i32,
2361	simple_data: []const i32,
2362	};
2363	var ts = TokenStream.init(
2364	\\{
2365	\\ "language": "zig",
2366	\\ "language_without_sentinel": "zig again!",
2367	\\ "data": [1, 2, 3],
2368	\\ "simple_data": [4, 5, 6]
2369	\\}
2370	);
2371	const r = try parse(T, &ts, options);
2372	defer parseFree(T, r, options);
2373
2374	try testing.expectEqualSentinel(u8, 0, "zig", r.language);
2375
2376	const data = [_:99]i32{ 1, 2, 3 };
2377	try testing.expectEqualSentinel(i32, 99, data[0..data.len], r.data);
2378
2379	// Make sure that arrays who aren't supposed to have a sentinel still parse without one.
2380	try testing.expectEqual(@as(?i32, null), std.meta.sentinel(@TypeOf(r.simple_data)));
2381	try testing.expectEqual(@as(?u8, null), std.meta.sentinel(@TypeOf(r.language_without_sentinel)));
2382	}
2383
2384	test "parse into struct with duplicate field" {
2385	// allow allocator to detect double frees by keeping bucket in use
2386	const ballast = try testing.allocator.alloc(u64, 1);
2387	defer testing.allocator.free(ballast);
2388
2389	const options_first = ParseOptions{ .allocator = testing.allocator, .duplicate_field_behavior = .UseFirst };
2390
2391	const options_last = ParseOptions{
2392	.allocator = testing.allocator,
2393	.duplicate_field_behavior = .UseLast,
2394	};
2395
2396	const str = "{ \"a\": 1, \"a\": 0.25 }";
2397
2398	const T1 = struct { a: *u64 };
2399	// both .UseFirst and .UseLast should fail because second "a" value isn't a u64
2400	var ts = TokenStream.init(str);
2401	try testing.expectError(error.InvalidNumber, parse(T1, &ts, options_first));
2402	ts = TokenStream.init(str);
2403	try testing.expectError(error.InvalidNumber, parse(T1, &ts, options_last));
2404
2405	const T2 = struct { a: f64 };
2406	ts = TokenStream.init(str);
2407	try testing.expectEqual(T2{ .a = 1.0 }, try parse(T2, &ts, options_first));
2408	ts = TokenStream.init(str);
2409	try testing.expectEqual(T2{ .a = 0.25 }, try parse(T2, &ts, options_last));
2410
2411	const T3 = struct { comptime a: f64 = 1.0 };
2412	// .UseFirst should succeed because second "a" value is unconditionally ignored (even though != 1.0)
2413	const t3 = T3{ .a = 1.0 };
2414	ts = TokenStream.init(str);
2415	try testing.expectEqual(t3, try parse(T3, &ts, options_first));
2416	// .UseLast should fail because second "a" value is 0.25 which is not equal to default value of 1.0
2417	ts = TokenStream.init(str);
2418	try testing.expectError(error.UnexpectedValue, parse(T3, &ts, options_last));
2419	}
2420
2421	test "parse into struct ignoring unknown fields" {
2422	const T = struct {
2423	int: i64,
2424	language: []const u8,
2425	};
2426
2427	const ops = ParseOptions{
2428	.allocator = testing.allocator,
2429	.ignore_unknown_fields = true,
2430	};
2431
2432	var ts = TokenStream.init(
2433	\\{
2434	\\ "int": 420,
2435	\\ "float": 3.14,
2436	\\ "with\\escape": true,
2437	\\ "with\u0105unicode\ud83d\ude02": false,
2438	\\ "optional": null,
2439	\\ "static_array": [66.6, 420.420, 69.69],
2440	\\ "dynamic_array": [66.6, 420.420, 69.69],
2441	\\ "complex": {
2442	\\ "nested": "zig"
2443	\\ },
2444	\\ "veryComplex": [
2445	\\ {
2446	\\ "foo": "zig"
2447	\\ }, {
2448	\\ "foo": "rocks"
2449	\\ }
2450	\\ ],
2451	\\ "a_union": 100000,
2452	\\ "language": "zig"
2453	\\}
2454	);
2455	const r = try parse(T, &ts, ops);
2456	defer parseFree(T, r, ops);
2457
2458	try testing.expectEqual(@as(i64, 420), r.int);
2459	try testing.expectEqualSlices(u8, "zig", r.language);
2460	}
2461
2462	test "parse into tuple" {
2463	const options = ParseOptions{ .allocator = testing.allocator };
2464	const Union = union(enum) {
2465	char: u8,
2466	float: f64,
2467	string: []const u8,
2468	};
2469	const T = std.meta.Tuple(&.{
2470	i64,
2471	f64,
2472	bool,
2473	[]const u8,
2474	?bool,
2475	struct {
2476	foo: i32,
2477	bar: []const u8,
2478	},
2479	std.meta.Tuple(&.{ u8, []const u8, u8 }),
2480	Union,
2481	});
2482	var ts = TokenStream.init(
2483	\\[
2484	\\ 420,
2485	\\ 3.14,
2486	\\ true,
2487	\\ "zig",
2488	\\ null,
2489	\\ {
2490	\\ "foo": 1,
2491	\\ "bar": "zero"
2492	\\ },
2493	\\ [4, "två", 42],
2494	\\ 12.34
2495	\\]
2496	);
2497	const r = try parse(T, &ts, options);
2498	defer parseFree(T, r, options);
2499	try testing.expectEqual(@as(i64, 420), r[0]);
2500	try testing.expectEqual(@as(f64, 3.14), r[1]);
2501	try testing.expectEqual(true, r[2]);
2502	try testing.expectEqualSlices(u8, "zig", r[3]);
2503	try testing.expectEqual(@as(?bool, null), r[4]);
2504	try testing.expectEqual(@as(i32, 1), r[5].foo);
2505	try testing.expectEqualSlices(u8, "zero", r[5].bar);
2506	try testing.expectEqual(@as(u8, 4), r[6][0]);
2507	try testing.expectEqualSlices(u8, "två", r[6][1]);
2508	try testing.expectEqual(@as(u8, 42), r[6][2]);
2509	try testing.expectEqual(Union{ .float = 12.34 }, r[7]);
2510	}
2511
2512	const ParseIntoRecursiveUnionDefinitionValue = union(enum) {
2513	integer: i64,
2514	array: []const ParseIntoRecursiveUnionDefinitionValue,
2515	};
2516
2517	test "parse into recursive union definition" {
2518	const T = struct {
2519	values: ParseIntoRecursiveUnionDefinitionValue,
2520	};
2521	const ops = ParseOptions{ .allocator = testing.allocator };
2522
2523	var ts = TokenStream.init("{\"values\":[58]}");
2524	const r = try parse(T, &ts, ops);
2525	defer parseFree(T, r, ops);
2526
2527	try testing.expectEqual(@as(i64, 58), r.values.array[0].integer);
2528	}
2529
2530	const ParseIntoDoubleRecursiveUnionValueFirst = union(enum) {
2531	integer: i64,
2532	array: []const ParseIntoDoubleRecursiveUnionValueSecond,
2533	};
2534
2535	const ParseIntoDoubleRecursiveUnionValueSecond = union(enum) {
2536	boolean: bool,
2537	array: []const ParseIntoDoubleRecursiveUnionValueFirst,
2538	};
2539
2540	test "parse into double recursive union definition" {
2541	const T = struct {
2542	values: ParseIntoDoubleRecursiveUnionValueFirst,
2543	};
2544	const ops = ParseOptions{ .allocator = testing.allocator };
2545
2546	var ts = TokenStream.init("{\"values\":[[58]]}");
2547	const r = try parse(T, &ts, ops);
2548	defer parseFree(T, r, ops);
2549
2550	try testing.expectEqual(@as(i64, 58), r.values.array[0].array[0].integer);
2551	}
2552
2553	test "parse into vector" {
2554	const options = ParseOptions{ .allocator = testing.allocator };
2555	const T = struct {
2556	vec_i32: @Vector(4, i32),
2557	vec_f32: @Vector(2, f32),
2558	};
2559	var ts = TokenStream.init(
2560	\\{
2561	\\ "vec_f32": [1.5, 2.5],
2562	\\ "vec_i32": [4, 5, 6, 7]
2563	\\}
2564	);
2565	const r = try parse(T, &ts, options);
2566	defer parseFree(T, r, options);
2567	try testing.expectApproxEqAbs(@as(f32, 1.5), r.vec_f32[0], 0.0000001);
2568	try testing.expectApproxEqAbs(@as(f32, 2.5), r.vec_f32[1], 0.0000001);
2569	try testing.expectEqual(@Vector(4, i32){ 4, 5, 6, 7 }, r.vec_i32);
2570	}
2571
2572	test "json.parser.dynamic" {
2573	var p = Parser.init(testing.allocator, false);
2574	defer p.deinit();
2575
2576	const s =
2577	\\{
2578	\\ "Image": {
2579	\\ "Width": 800,
2580	\\ "Height": 600,
2581	\\ "Title": "View from 15th Floor",
2582	\\ "Thumbnail": {
2583	\\ "Url": "http://www.example.com/image/481989943",
2584	\\ "Height": 125,
2585	\\ "Width": 100
2586	\\ },
2587	\\ "Animated" : false,
2588	\\ "IDs": [116, 943, 234, 38793],
2589	\\ "ArrayOfObject": [{"n": "m"}],
2590	\\ "double": 1.3412,
2591	\\ "LargeInt": 18446744073709551615
2592	\\ }
2593	\\}
2594	;
2595
2596	var tree = try p.parse(s);
2597	defer tree.deinit();
2598
2599	var root = tree.root;
2600
2601	var image = root.Object.get("Image").?;
2602
2603	const width = image.Object.get("Width").?;
2604	try testing.expect(width.Integer == 800);
2605
2606	const height = image.Object.get("Height").?;
2607	try testing.expect(height.Integer == 600);
2608
2609	const title = image.Object.get("Title").?;
2610	try testing.expect(mem.eql(u8, title.String, "View from 15th Floor"));
2611
2612	const animated = image.Object.get("Animated").?;
2613	try testing.expect(animated.Bool == false);
2614
2615	const array_of_object = image.Object.get("ArrayOfObject").?;
2616	try testing.expect(array_of_object.Array.items.len == 1);
2617
2618	const obj0 = array_of_object.Array.items[0].Object.get("n").?;
2619	try testing.expect(mem.eql(u8, obj0.String, "m"));
2620
2621	const double = image.Object.get("double").?;
2622	try testing.expect(double.Float == 1.3412);
2623
2624	const large_int = image.Object.get("LargeInt").?;
2625	try testing.expect(mem.eql(u8, large_int.NumberString, "18446744073709551615"));
2626	}
2627
2628	test "write json then parse it" {
2629	var out_buffer: [1000]u8 = undefined;
2630
2631	var fixed_buffer_stream = std.io.fixedBufferStream(&out_buffer);
2632	const out_stream = fixed_buffer_stream.writer();
2633	var jw = writeStream(out_stream, 4);
2634
2635	try jw.beginObject();
2636
2637	try jw.objectField("f");
2638	try jw.emitBool(false);
2639
2640	try jw.objectField("t");
2641	try jw.emitBool(true);
2642
2643	try jw.objectField("int");
2644	try jw.emitNumber(1234);
2645
2646	try jw.objectField("array");
2647	try jw.beginArray();
2648
2649	try jw.arrayElem();
2650	try jw.emitNull();
2651
2652	try jw.arrayElem();
2653	try jw.emitNumber(12.34);
2654
2655	try jw.endArray();
2656
2657	try jw.objectField("str");
2658	try jw.emitString("hello");
2659
2660	try jw.endObject();
2661
2662	var parser = Parser.init(testing.allocator, false);
2663	defer parser.deinit();
2664	var tree = try parser.parse(fixed_buffer_stream.getWritten());
2665	defer tree.deinit();
2666
2667	try testing.expect(tree.root.Object.get("f").?.Bool == false);
2668	try testing.expect(tree.root.Object.get("t").?.Bool == true);
2669	try testing.expect(tree.root.Object.get("int").?.Integer == 1234);
2670	try testing.expect(tree.root.Object.get("array").?.Array.items[0].Null == {});
2671	try testing.expect(tree.root.Object.get("array").?.Array.items[1].Float == 12.34);
2672	try testing.expect(mem.eql(u8, tree.root.Object.get("str").?.String, "hello"));
2673	}
2674
2675	fn testParse(arena_allocator: std.mem.Allocator, json_str: []const u8) !Value {
2676	var p = Parser.init(arena_allocator, false);
2677	return (try p.parse(json_str)).root;
2678	}
2679
2680	test "parsing empty string gives appropriate error" {
2681	var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
2682	defer arena_allocator.deinit();
2683	try testing.expectError(error.UnexpectedEndOfJson, testParse(arena_allocator.allocator(), ""));
2684	}
2685
2686	test "parse tree should not contain dangling pointers" {
2687	var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
2688	defer arena_allocator.deinit();
2689
2690	var p = json.Parser.init(arena_allocator.allocator(), false);
2691	defer p.deinit();
2692
2693	var tree = try p.parse("[]");
2694	defer tree.deinit();
2695
2696	// Allocation should succeed
2697	var i: usize = 0;
2698	while (i < 100) : (i += 1) {
2699	try tree.root.Array.append(std.json.Value{ .Integer = 100 });
2700	}
2701	try testing.expectEqual(tree.root.Array.items.len, 100);
2702	}
2703
2704	test "integer after float has proper type" {
2705	var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
2706	defer arena_allocator.deinit();
2707	const parsed = try testParse(arena_allocator.allocator(),
2708	\\{
2709	\\ "float": 3.14,
2710	\\ "ints": [1, 2, 3]
2711	\\}
2712	);
2713	try std.testing.expect(parsed.Object.get("ints").?.Array.items[0] == .Integer);
2714	}
2715
2716	test "parse exponential into int" {
2717	const T = struct { int: i64 };
2718	var ts = TokenStream.init("{ \"int\": 4.2e2 }");
2719	const r = try parse(T, &ts, ParseOptions{});
2720	try testing.expectEqual(@as(i64, 420), r.int);
2721	ts = TokenStream.init("{ \"int\": 0.042e2 }");
2722	try testing.expectError(error.InvalidNumber, parse(T, &ts, ParseOptions{}));
2723	ts = TokenStream.init("{ \"int\": 18446744073709551616.0 }");
2724	try testing.expectError(error.Overflow, parse(T, &ts, ParseOptions{}));
2725	}
2726
2727	test "escaped characters" {
2728	var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
2729	defer arena_allocator.deinit();
2730	const input =
2731	\\{
2732	\\ "backslash": "\\",
2733	\\ "forwardslash": "\/",
2734	\\ "newline": "\n",
2735	\\ "carriagereturn": "\r",
2736	\\ "tab": "\t",
2737	\\ "formfeed": "\f",
2738	\\ "backspace": "\b",
2739	\\ "doublequote": "\"",
2740	\\ "unicode": "\u0105",
2741	\\ "surrogatepair": "\ud83d\ude02"
2742	\\}
2743	;
2744
2745	const obj = (try testParse(arena_allocator.allocator(), input)).Object;
2746
2747	try testing.expectEqualSlices(u8, obj.get("backslash").?.String, "\\");
2748	try testing.expectEqualSlices(u8, obj.get("forwardslash").?.String, "/");
2749	try testing.expectEqualSlices(u8, obj.get("newline").?.String, "\n");
2750	try testing.expectEqualSlices(u8, obj.get("carriagereturn").?.String, "\r");
2751	try testing.expectEqualSlices(u8, obj.get("tab").?.String, "\t");
2752	try testing.expectEqualSlices(u8, obj.get("formfeed").?.String, "\x0C");
2753	try testing.expectEqualSlices(u8, obj.get("backspace").?.String, "\x08");
2754	try testing.expectEqualSlices(u8, obj.get("doublequote").?.String, "\"");
2755	try testing.expectEqualSlices(u8, obj.get("unicode").?.String, "ą");
2756	try testing.expectEqualSlices(u8, obj.get("surrogatepair").?.String, "😂");
2757	}
2758
2759	test "string copy option" {
2760	const input =
2761	\\{
2762	\\ "noescape": "aą😂",
2763	\\ "simple": "\\\/\n\r\t\f\b\"",
2764	\\ "unicode": "\u0105",
2765	\\ "surrogatepair": "\ud83d\ude02"
2766	\\}
2767	;
2768
2769	var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
2770	defer arena_allocator.deinit();
2771	const allocator = arena_allocator.allocator();
2772
2773	var parser = Parser.init(allocator, false);
2774	const tree_nocopy = try parser.parse(input);
2775	const obj_nocopy = tree_nocopy.root.Object;
2776
2777	parser = Parser.init(allocator, true);
2778	const tree_copy = try parser.parse(input);
2779	const obj_copy = tree_copy.root.Object;
2780
2781	for ([_][]const u8{ "noescape", "simple", "unicode", "surrogatepair" }) \|field_name\| {
2782	try testing.expectEqualSlices(u8, obj_nocopy.get(field_name).?.String, obj_copy.get(field_name).?.String);
2783	}
2784
2785	const nocopy_addr = &obj_nocopy.get("noescape").?.String[0];
2786	const copy_addr = &obj_copy.get("noescape").?.String[0];
2787
2788	var found_nocopy = false;
2789	for (input, 0..) \|_, index\| {
2790	try testing.expect(copy_addr != &input[index]);
2791	if (nocopy_addr == &input[index]) {
2792	found_nocopy = true;
2793	}
2794	}
2795	try testing.expect(found_nocopy);
2796	}
2797
2798	test "stringify alloc" {
2799	const allocator = std.testing.allocator;
2800	const expected =
2801	\\{"foo":"bar","answer":42,"my_friend":"sammy"}
2802	;
2803	const actual = try stringifyAlloc(allocator, .{ .foo = "bar", .answer = 42, .my_friend = "sammy" }, .{});
2804	defer allocator.free(actual);
2805
2806	try std.testing.expectEqualStrings(expected, actual);
2807	}
2808
2809	test "json.serialize issue #5959" {
2810	var parser: StreamingParser = undefined;
2811	// StreamingParser has multiple internal fields set to undefined. This causes issues when using
2812	// expectEqual so these are zeroed. We are testing for equality here only because this is a
2813	// known small test reproduction which hits the relevant LLVM issue.
2814	@memset(@ptrCast([*]u8, &parser)[0..@sizeOf(StreamingParser)], 0);
2815	try std.testing.expectEqual(parser, parser);
2816	}
2817
2818	fn checkNext(p: *TokenStream, id: std.meta.Tag(Token)) !void {
2819	const token = (p.next() catch unreachable).?;
2820	try testing.expect(std.meta.activeTag(token) == id);
2821	}
2822
2823	test "json.token" {
2824	const s =
2825	\\{
2826	\\ "Image": {
2827	\\ "Width": 800,
2828	\\ "Height": 600,
2829	\\ "Title": "View from 15th Floor",
2830	\\ "Thumbnail": {
2831	\\ "Url": "http://www.example.com/image/481989943",
2832	\\ "Height": 125,
2833	\\ "Width": 100
2834	\\ },
2835	\\ "Animated" : false,
2836	\\ "IDs": [116, 943, 234, 38793]
2837	\\ }
2838	\\}
2839	;
2840
2841	var p = TokenStream.init(s);
2842
2843	try checkNext(&p, .ObjectBegin);
2844	try checkNext(&p, .String); // Image
2845	try checkNext(&p, .ObjectBegin);
2846	try checkNext(&p, .String); // Width
2847	try checkNext(&p, .Number);
2848	try checkNext(&p, .String); // Height
2849	try checkNext(&p, .Number);
2850	try checkNext(&p, .String); // Title
2851	try checkNext(&p, .String);
2852	try checkNext(&p, .String); // Thumbnail
2853	try checkNext(&p, .ObjectBegin);
2854	try checkNext(&p, .String); // Url
2855	try checkNext(&p, .String);
2856	try checkNext(&p, .String); // Height
2857	try checkNext(&p, .Number);
2858	try checkNext(&p, .String); // Width
2859	try checkNext(&p, .Number);
2860	try checkNext(&p, .ObjectEnd);
2861	try checkNext(&p, .String); // Animated
2862	try checkNext(&p, .False);
2863	try checkNext(&p, .String); // IDs
2864	try checkNext(&p, .ArrayBegin);
2865	try checkNext(&p, .Number);
2866	try checkNext(&p, .Number);
2867	try checkNext(&p, .Number);
2868	try checkNext(&p, .Number);
2869	try checkNext(&p, .ArrayEnd);
2870	try checkNext(&p, .ObjectEnd);
2871	try checkNext(&p, .ObjectEnd);
2872
2873	try testing.expect((try p.next()) == null);
2874	}
2875
2876	test "json.token mismatched close" {
2877	var p = TokenStream.init("[102, 111, 111 }");
2878	try checkNext(&p, .ArrayBegin);
2879	try checkNext(&p, .Number);
2880	try checkNext(&p, .Number);
2881	try checkNext(&p, .Number);
2882	try testing.expectError(error.UnexpectedClosingBrace, p.next());
2883	}
2884
2885	test "json.token premature object close" {
2886	var p = TokenStream.init("{ \"key\": }");
2887	try checkNext(&p, .ObjectBegin);
2888	try checkNext(&p, .String);
2889	try testing.expectError(error.InvalidValueBegin, p.next());
2890	}
2891
2892	test "json.validate" {
2893	try testing.expectEqual(true, validate("{}"));
2894	try testing.expectEqual(true, validate("[]"));
2895	try testing.expectEqual(true, validate("[{[[[[{}]]]]}]"));
2896	try testing.expectEqual(false, validate("{]"));
2897	try testing.expectEqual(false, validate("[}"));
2898	try testing.expectEqual(false, validate("{{{{[]}}}]"));
2899	}
2900
2901	test "Value.jsonStringify" {
2902	{
2903	var buffer: [10]u8 = undefined;
2904	var fbs = std.io.fixedBufferStream(&buffer);
2905	try @as(Value, .Null).jsonStringify(.{}, fbs.writer());
2906	try testing.expectEqualSlices(u8, fbs.getWritten(), "null");
2907	}
2908	{
2909	var buffer: [10]u8 = undefined;
2910	var fbs = std.io.fixedBufferStream(&buffer);
2911	try (Value{ .Bool = true }).jsonStringify(.{}, fbs.writer());
2912	try testing.expectEqualSlices(u8, fbs.getWritten(), "true");
2913	}
2914	{
2915	var buffer: [10]u8 = undefined;
2916	var fbs = std.io.fixedBufferStream(&buffer);
2917	try (Value{ .Integer = 42 }).jsonStringify(.{}, fbs.writer());
2918	try testing.expectEqualSlices(u8, fbs.getWritten(), "42");
2919	}
2920	{
2921	var buffer: [10]u8 = undefined;
2922	var fbs = std.io.fixedBufferStream(&buffer);
2923	try (Value{ .NumberString = "43" }).jsonStringify(.{}, fbs.writer());
2924	try testing.expectEqualSlices(u8, fbs.getWritten(), "43");
2925	}
2926	{
2927	var buffer: [10]u8 = undefined;
2928	var fbs = std.io.fixedBufferStream(&buffer);
2929	try (Value{ .Float = 42 }).jsonStringify(.{}, fbs.writer());
2930	try testing.expectEqualSlices(u8, fbs.getWritten(), "4.2e+01");
2931	}
2932	{
2933	var buffer: [10]u8 = undefined;
2934	var fbs = std.io.fixedBufferStream(&buffer);
2935	try (Value{ .String = "weeee" }).jsonStringify(.{}, fbs.writer());
2936	try testing.expectEqualSlices(u8, fbs.getWritten(), "\"weeee\"");
2937	}
2938	{
2939	var buffer: [10]u8 = undefined;
2940	var fbs = std.io.fixedBufferStream(&buffer);
2941	var vals = [_]Value{
2942	.{ .Integer = 1 },
2943	.{ .Integer = 2 },
2944	.{ .NumberString = "3" },
2945	};
2946	try (Value{
2947	.Array = Array.fromOwnedSlice(undefined, &vals),
2948	}).jsonStringify(.{}, fbs.writer());
2949	try testing.expectEqualSlices(u8, fbs.getWritten(), "[1,2,3]");
2950	}
2951	{
2952	var buffer: [10]u8 = undefined;
2953	var fbs = std.io.fixedBufferStream(&buffer);
2954	var obj = ObjectMap.init(testing.allocator);
2955	defer obj.deinit();
2956	try obj.putNoClobber("a", .{ .String = "b" });
2957	try (Value{ .Object = obj }).jsonStringify(.{}, fbs.writer());
2958	try testing.expectEqualSlices(u8, fbs.getWritten(), "{\"a\":\"b\"}");
2959	}
2960	}	113	}

lib/std/json/write_stream.zig+63-56

 @@ -1,14 +1,19 @@
-const std = @import("../std.zig");
+const std = @import("std");
 const assert = std.debug.assert;
 const maxInt = std.math.maxInt;
+const StringifyOptions = @import("./stringify.zig").StringifyOptions;
+const jsonStringify = @import("./stringify.zig").stringify;
+const Value = @import("./dynamic.zig").Value;
 const State = enum {
-    Complete,
+    complete,
-    Value,
+    value,
-    ArrayStart,
+    array_start,
-    Array,
+    array,
-    ObjectStart,
+    object_start,
-    Object,
+    object,
 };
 /// Writes JSON ([RFC8259](https://tools.ietf.org/html/rfc8259)) formatted data
 @@ -21,9 +26,9 @@ pub fn WriteStream(comptime OutStream: type, comptime max_depth: usize) type {
         pub const Stream = OutStream;
-        whitespace: std.json.StringifyOptions.Whitespace = std.json.StringifyOptions.Whitespace{
+        whitespace: StringifyOptions.Whitespace = StringifyOptions.Whitespace{
             .indent_level = 0,
-            .indent = .{ .Space = 1 },
+            .indent = .{ .space = 1 },
         },
         stream: OutStream,
 @@ -36,38 +41,38 @@ pub fn WriteStream(comptime OutStream: type, comptime max_depth: usize) type {
                 .state_index = 1,
                 .state = undefined,
             };
-            self.state[0] = .Complete;
+            self.state[0] = .complete;
-            self.state[1] = .Value;
+            self.state[1] = .value;
             return self;
         }
         pub fn beginArray(self: *Self) !void {
-            assert(self.state[self.state_index] == State.Value); // need to call arrayElem or objectField
+            assert(self.state[self.state_index] == State.value); // need to call arrayElem or objectField
             try self.stream.writeByte('[');
-            self.state[self.state_index] = State.ArrayStart;
+            self.state[self.state_index] = State.array_start;
             self.whitespace.indent_level += 1;
         }
         pub fn beginObject(self: *Self) !void {
-            assert(self.state[self.state_index] == State.Value); // need to call arrayElem or objectField
+            assert(self.state[self.state_index] == State.value); // need to call arrayElem or objectField
             try self.stream.writeByte('{');
-            self.state[self.state_index] = State.ObjectStart;
+            self.state[self.state_index] = State.object_start;
             self.whitespace.indent_level += 1;
         }
         pub fn arrayElem(self: *Self) !void {
             const state = self.state[self.state_index];
             switch (state) {
-                .Complete => unreachable,
+                .complete => unreachable,
-                .Value => unreachable,
+                .value => unreachable,
-                .ObjectStart => unreachable,
+                .object_start => unreachable,
-                .Object => unreachable,
+                .object => unreachable,
-                .Array, .ArrayStart => {
+                .array, .array_start => {
-                    if (state == .Array) {
+                    if (state == .array) {
                         try self.stream.writeByte(',');
                     }
-                    self.state[self.state_index] = .Array;
+                    self.state[self.state_index] = .array;
-                    self.pushState(.Value);
+                    self.pushState(.value);
                     try self.indent();
                 },
             }
 @@ -76,16 +81,16 @@ pub fn WriteStream(comptime OutStream: type, comptime max_depth: usize) type {
         pub fn objectField(self: *Self, name: []const u8) !void {
             const state = self.state[self.state_index];
             switch (state) {
-                .Complete => unreachable,
+                .complete => unreachable,
-                .Value => unreachable,
+                .value => unreachable,
-                .ArrayStart => unreachable,
+                .array_start => unreachable,
-                .Array => unreachable,
+                .array => unreachable,
-                .Object, .ObjectStart => {
+                .object, .object_start => {
-                    if (state == .Object) {
+                    if (state == .object) {
                         try self.stream.writeByte(',');
                     }
-                    self.state[self.state_index] = .Object;
+                    self.state[self.state_index] = .object;
-                    self.pushState(.Value);
+                    self.pushState(.value);
                     try self.indent();
                     try self.writeEscapedString(name);
                     try self.stream.writeByte(':');
 @@ -98,16 +103,16 @@ pub fn WriteStream(comptime OutStream: type, comptime max_depth: usize) type {
         pub fn endArray(self: *Self) !void {
             switch (self.state[self.state_index]) {
-                .Complete => unreachable,
+                .complete => unreachable,
-                .Value => unreachable,
+                .value => unreachable,
-                .ObjectStart => unreachable,
+                .object_start => unreachable,
-                .Object => unreachable,
+                .object => unreachable,
-                .ArrayStart => {
+                .array_start => {
                     self.whitespace.indent_level -= 1;
                     try self.stream.writeByte(']');
                     self.popState();
                 },
-                .Array => {
+                .array => {
                     self.whitespace.indent_level -= 1;
                     try self.indent();
                     self.popState();
 @@ -118,16 +123,16 @@ pub fn WriteStream(comptime OutStream: type, comptime max_depth: usize) type {
         pub fn endObject(self: *Self) !void {
             switch (self.state[self.state_index]) {
-                .Complete => unreachable,
+                .complete => unreachable,
-                .Value => unreachable,
+                .value => unreachable,
-                .ArrayStart => unreachable,
+                .array_start => unreachable,
-                .Array => unreachable,
+                .array => unreachable,
-                .ObjectStart => {
+                .object_start => {
                     self.whitespace.indent_level -= 1;
                     try self.stream.writeByte('}');
                     self.popState();
                 },
-                .Object => {
+                .object => {
                     self.whitespace.indent_level -= 1;
                     try self.indent();
                     self.popState();
 @@ -137,13 +142,13 @@ pub fn WriteStream(comptime OutStream: type, comptime max_depth: usize) type {
         }
         pub fn emitNull(self: *Self) !void {
-            assert(self.state[self.state_index] == State.Value);
+            assert(self.state[self.state_index] == State.value);
             try self.stringify(null);
             self.popState();
         }
         pub fn emitBool(self: *Self, value: bool) !void {
-            assert(self.state[self.state_index] == State.Value);
+            assert(self.state[self.state_index] == State.value);
             try self.stringify(value);
             self.popState();
         }
 @@ -154,7 +159,7 @@ pub fn WriteStream(comptime OutStream: type, comptime max_depth: usize) type {
             /// in a IEEE 754 double float, otherwise emitted as a string to the full precision.
             value: anytype,
         ) !void {
-            assert(self.state[self.state_index] == State.Value);
+            assert(self.state[self.state_index] == State.value);
             switch (@typeInfo(@TypeOf(value))) {
                 .Int => |info| {
                     if (info.bits < 53) {
 @@ -183,7 +188,7 @@ pub fn WriteStream(comptime OutStream: type, comptime max_depth: usize) type {
         }
         pub fn emitString(self: *Self, string: []const u8) !void {
-            assert(self.state[self.state_index] == State.Value);
+            assert(self.state[self.state_index] == State.value);
             try self.writeEscapedString(string);
             self.popState();
         }
 @@ -194,9 +199,9 @@ pub fn WriteStream(comptime OutStream: type, comptime max_depth: usize) type {
         }
         /// Writes the complete json into the output stream
-        pub fn emitJson(self: *Self, json: std.json.Value) Stream.Error!void {
+        pub fn emitJson(self: *Self, value: Value) Stream.Error!void {
-            assert(self.state[self.state_index] == State.Value);
+            assert(self.state[self.state_index] == State.value);
-            try self.stringify(json);
+            try self.stringify(value);
             self.popState();
         }
 @@ -215,7 +220,7 @@ pub fn WriteStream(comptime OutStream: type, comptime max_depth: usize) type {
         }
         fn stringify(self: *Self, value: anytype) !void {
-            try std.json.stringify(value, std.json.StringifyOptions{
+            try jsonStringify(value, StringifyOptions{
                 .whitespace = self.whitespace,
             }, self.stream);
         }
 @@ -229,6 +234,8 @@ pub fn writeStream(
     return WriteStream(@TypeOf(out_stream), max_depth).init(out_stream);
 }
+const ObjectMap = @import("./dynamic.zig").ObjectMap;
 test "json write stream" {
     var out_buf: [1024]u8 = undefined;
     var slice_stream = std.io.fixedBufferStream(&out_buf);
 @@ -237,7 +244,7 @@ test "json write stream" {
     var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
     defer arena_allocator.deinit();
-    var w = std.json.writeStream(out, 10);
+    var w = writeStream(out, 10);
     try w.beginObject();
 @@ -285,9 +292,9 @@ test "json write stream" {
     try std.testing.expect(std.mem.eql(u8, expected, result));
 }
-fn getJsonObject(allocator: std.mem.Allocator) !std.json.Value {
+fn getJsonObject(allocator: std.mem.Allocator) !Value {
-    var value = std.json.Value{ .Object = std.json.ObjectMap.init(allocator) };
+    var value = Value{ .object = ObjectMap.init(allocator) };
-    try value.Object.put("one", std.json.Value{ .Integer = @intCast(i64, 1) });
+    try value.object.put("one", Value{ .integer = @intCast(i64, 1) });
-    try value.Object.put("two", std.json.Value{ .Float = 2.0 });
+    try value.object.put("two", Value{ .float = 2.0 });
     return value;
 }

src/Autodoc.zig+7-7

...	@@ -295,7 +295,7 @@ pub fn generateZirData(self: *Autodoc) !void {	...	@@ -295,7 +295,7 @@ pub fn generateZirData(self: *Autodoc) !void {
295	try std.json.stringify(	295	try std.json.stringify(
296	data,	296	data,
297	.{	297	.{
298	.whitespace = .{ .indent = .None, .separator = false },	298	.whitespace = .{ .indent = .none, .separator = false },
299	.emit_null_optional_fields = true,	299	.emit_null_optional_fields = true,
300	},	300	},
301	out,	301	out,
...	@@ -444,7 +444,7 @@ const DocData = struct {	...	@@ -444,7 +444,7 @@ const DocData = struct {
444	w: anytype,	444	w: anytype,
445	) !void {	445	) !void {
446	var jsw = std.json.writeStream(w, 15);	446	var jsw = std.json.writeStream(w, 15);
447	if (opts.whitespace) \|ws\| jsw.whitespace = ws;	447	jsw.whitespace = opts.whitespace;
448	try jsw.beginObject();	448	try jsw.beginObject();
449	inline for (comptime std.meta.tags(std.meta.FieldEnum(DocData))) \|f\| {	449	inline for (comptime std.meta.tags(std.meta.FieldEnum(DocData))) \|f\| {
450	const f_name = @tagName(f);	450	const f_name = @tagName(f);
...	@@ -495,7 +495,7 @@ const DocData = struct {	...	@@ -495,7 +495,7 @@ const DocData = struct {
495	w: anytype,	495	w: anytype,
496	) !void {	496	) !void {
497	var jsw = std.json.writeStream(w, 15);	497	var jsw = std.json.writeStream(w, 15);
498	if (opts.whitespace) \|ws\| jsw.whitespace = ws;	498	jsw.whitespace = opts.whitespace;
499		499
500	try jsw.beginObject();	500	try jsw.beginObject();
501	inline for (comptime std.meta.tags(std.meta.FieldEnum(DocModule))) \|f\| {	501	inline for (comptime std.meta.tags(std.meta.FieldEnum(DocModule))) \|f\| {
...	@@ -529,7 +529,7 @@ const DocData = struct {	...	@@ -529,7 +529,7 @@ const DocData = struct {
529	w: anytype,	529	w: anytype,
530	) !void {	530	) !void {
531	var jsw = std.json.writeStream(w, 15);	531	var jsw = std.json.writeStream(w, 15);
532	if (opts.whitespace) \|ws\| jsw.whitespace = ws;	532	jsw.whitespace = opts.whitespace;
533	try jsw.beginArray();	533	try jsw.beginArray();
534	inline for (comptime std.meta.fields(Decl)) \|f\| {	534	inline for (comptime std.meta.fields(Decl)) \|f\| {
535	try jsw.arrayElem();	535	try jsw.arrayElem();
...	@@ -556,7 +556,7 @@ const DocData = struct {	...	@@ -556,7 +556,7 @@ const DocData = struct {
556	w: anytype,	556	w: anytype,
557	) !void {	557	) !void {
558	var jsw = std.json.writeStream(w, 15);	558	var jsw = std.json.writeStream(w, 15);
559	if (opts.whitespace) \|ws\| jsw.whitespace = ws;	559	jsw.whitespace = opts.whitespace;
560	try jsw.beginArray();	560	try jsw.beginArray();
561	inline for (comptime std.meta.fields(AstNode)) \|f\| {	561	inline for (comptime std.meta.fields(AstNode)) \|f\| {
562	try jsw.arrayElem();	562	try jsw.arrayElem();
...	@@ -689,7 +689,7 @@ const DocData = struct {	...	@@ -689,7 +689,7 @@ const DocData = struct {
689	) !void {	689	) !void {
690	const active_tag = std.meta.activeTag(self);	690	const active_tag = std.meta.activeTag(self);
691	var jsw = std.json.writeStream(w, 15);	691	var jsw = std.json.writeStream(w, 15);
692	if (opts.whitespace) \|ws\| jsw.whitespace = ws;	692	jsw.whitespace = opts.whitespace;
693	try jsw.beginArray();	693	try jsw.beginArray();
694	try jsw.arrayElem();	694	try jsw.arrayElem();
695	try jsw.emitNumber(@enumToInt(active_tag));	695	try jsw.emitNumber(@enumToInt(active_tag));
...	@@ -831,7 +831,7 @@ const DocData = struct {	...	@@ -831,7 +831,7 @@ const DocData = struct {
831	) @TypeOf(w).Error!void {	831	) @TypeOf(w).Error!void {
832	const active_tag = std.meta.activeTag(self);	832	const active_tag = std.meta.activeTag(self);
833	var jsw = std.json.writeStream(w, 15);	833	var jsw = std.json.writeStream(w, 15);
834	if (opts.whitespace) \|ws\| jsw.whitespace = ws;	834	jsw.whitespace = opts.whitespace;
835	try jsw.beginObject();	835	try jsw.beginObject();
836	if (active_tag == .declIndex) {	836	if (active_tag == .declIndex) {
837	try jsw.objectField("declRef");	837	try jsw.objectField("declRef");

src/print_env.zig+1-1

...	@@ -28,7 +28,7 @@ pub fn cmdEnv(gpa: Allocator, args: []const []const u8, stdout: std.fs.File.Writ	...	@@ -28,7 +28,7 @@ pub fn cmdEnv(gpa: Allocator, args: []const []const u8, stdout: std.fs.File.Writ
28	var bw = std.io.bufferedWriter(stdout);	28	var bw = std.io.bufferedWriter(stdout);
29	const w = bw.writer();	29	const w = bw.writer();
30		30
31	var jws = std.json.WriteStream(@TypeOf(w), 6).init(w);	31	var jws = std.json.writeStream(w, 6);
32	try jws.beginObject();	32	try jws.beginObject();
33		33
34	try jws.objectField("zig_exe");	34	try jws.objectField("zig_exe");

src/print_targets.zig+1-1

...	@@ -40,7 +40,7 @@ pub fn cmdTargets(	...	@@ -40,7 +40,7 @@ pub fn cmdTargets(
40		40
41	var bw = io.bufferedWriter(stdout);	41	var bw = io.bufferedWriter(stdout);
42	const w = bw.writer();	42	const w = bw.writer();
43	var jws = std.json.WriteStream(@TypeOf(w), 6).init(w);	43	var jws = std.json.writeStream(w, 6);
44		44
45	try jws.beginObject();	45	try jws.beginObject();
46		46

tools/gen_spirv_spec.zig+1-2

...	@@ -20,8 +20,7 @@ pub fn main() !void {	...	@@ -20,8 +20,7 @@ pub fn main() !void {
20	// Required for json parsing.	20	// Required for json parsing.
21	@setEvalBranchQuota(10000);	21	@setEvalBranchQuota(10000);
22		22
23	var tokens = std.json.TokenStream.init(spec);	23	var registry = try std.json.parseFromSlice(g.Registry, allocator, spec, .{});
24	var registry = try std.json.parse(g.Registry, &tokens, .{ .allocator = allocator });
25		24
26	const core_reg = switch (registry) {	25	const core_reg = switch (registry) {
27	.core => \|core_reg\| core_reg,	26	.core => \|core_reg\| core_reg,

tools/generate_JSONTestSuite.zig created+79

...	@@ -0,0 +1,79 @@
1	// zig run this file inside the test_parsing/ directory of this repo: https://github.com/nst/JSONTestSuite
2
3	const std = @import("std");
4
5	pub fn main() !void {
6	var gpa = std.heap.GeneralPurposeAllocator(.{}){};
7	var allocator = gpa.allocator();
8
9	var output = std.io.getStdOut().writer();
10	try output.writeAll(
11	\\// This file was generated by _generate_JSONTestSuite.zig
12	\\// These test cases are sourced from: https://github.com/nst/JSONTestSuite
13	\\const ok = @import("./test.zig").ok;
14	\\const err = @import("./test.zig").err;
15	\\const any = @import("./test.zig").any;
16	\\
17	\\
18	);
19
20	var names = std.ArrayList([]const u8).init(allocator);
21	var cwd = try std.fs.cwd().openIterableDir(".", .{});
22	var it = cwd.iterate();
23	while (try it.next()) \|entry\| {
24	try names.append(try allocator.dupe(u8, entry.name));
25	}
26	std.sort.sort([]const u8, names.items, {}, (struct {
27	fn lessThan(_: void, a: []const u8, b: []const u8) bool {
28	return std.mem.lessThan(u8, a, b);
29	}
30	}).lessThan);
31
32	for (names.items) \|name\| {
33	const contents = try std.fs.cwd().readFileAlloc(allocator, name, 250001);
34	try output.writeAll("test ");
35	try writeString(output, name);
36	try output.writeAll(" {\n try ");
37	switch (name[0]) {
38	'y' => try output.writeAll("ok"),
39	'n' => try output.writeAll("err"),
40	'i' => try output.writeAll("any"),
41	else => unreachable,
42	}
43	try output.writeByte('(');
44	try writeString(output, contents);
45	try output.writeAll(");\n}\n");
46	}
47	}
48
49	const i_structure_500_nested_arrays = "[" 500 ++ "]" 500;
50	const n_structure_100000_opening_arrays = "[" ** 100000;
51	const n_structure_open_array_object = "[{\"\":" ** 50000 ++ "\n";
52
53	fn writeString(writer: anytype, s: []const u8) !void {
54	if (s.len > 200) {
55	// There are a few of these we can compress with Zig expressions.
56	if (std.mem.eql(u8, s, i_structure_500_nested_arrays)) {
57	return writer.writeAll("\"[\" 500 ++ \"]\" 500");
58	} else if (std.mem.eql(u8, s, n_structure_100000_opening_arrays)) {
59	return writer.writeAll("\"[\" ** 100000");
60	} else if (std.mem.eql(u8, s, n_structure_open_array_object)) {
61	return writer.writeAll("\"[{\\\"\\\":\" ** 50000 ++ \"\\n\"");
62	}
63	unreachable;
64	}
65	try writer.writeByte('"');
66	for (s) \|b\| {
67	switch (b) {
68	0...('\n' - 1),
69	('\n' + 1)...0x1f,
70	0x7f...0xff,
71	=> try writer.print("\\x{x:0>2}", .{b}),
72	'\n' => try writer.writeAll("\\n"),
73	'"' => try writer.writeAll("\\\""),
74	'\\' => try writer.writeAll("\\\\"),
75	else => try writer.writeByte(b),
76	}
77	}
78	try writer.writeByte('"');
79	}

tools/update_clang_options.zig+22-22

...	@@ -624,9 +624,9 @@ pub fn main() anyerror!void {	...	@@ -624,9 +624,9 @@ pub fn main() anyerror!void {
624	},	624	},
625	};	625	};
626		626
627	var parser = json.Parser.init(allocator, false);	627	var parser = json.Parser.init(allocator, .alloc_if_needed);
628	const tree = try parser.parse(json_text);	628	const tree = try parser.parse(json_text);
629	const root_map = &tree.root.Object;	629	const root_map = &tree.root.object;
630		630
631	var all_objects = std.ArrayList(*json.ObjectMap).init(allocator);	631	var all_objects = std.ArrayList(*json.ObjectMap).init(allocator);
632	{	632	{
...	@@ -634,14 +634,14 @@ pub fn main() anyerror!void {	...	@@ -634,14 +634,14 @@ pub fn main() anyerror!void {
634	it_map: while (it.next()) \|kv\| {	634	it_map: while (it.next()) \|kv\| {
635	if (kv.key_ptr.len == 0) continue;	635	if (kv.key_ptr.len == 0) continue;
636	if (kv.key_ptr.*[0] == '!') continue;	636	if (kv.key_ptr.*[0] == '!') continue;
637	if (kv.value_ptr.* != .Object) continue;	637	if (kv.value_ptr.* != .object) continue;
638	if (!kv.value_ptr.Object.contains("NumArgs")) continue;	638	if (!kv.value_ptr.object.contains("NumArgs")) continue;
639	if (!kv.value_ptr.Object.contains("Name")) continue;	639	if (!kv.value_ptr.object.contains("Name")) continue;
640	for (blacklisted_options) \|blacklisted_key\| {	640	for (blacklisted_options) \|blacklisted_key\| {
641	if (std.mem.eql(u8, blacklisted_key, kv.key_ptr.*)) continue :it_map;	641	if (std.mem.eql(u8, blacklisted_key, kv.key_ptr.*)) continue :it_map;
642	}	642	}
643	if (kv.value_ptr.Object.get("Name").?.String.len == 0) continue;	643	if (kv.value_ptr.object.get("Name").?.string.len == 0) continue;
644	try all_objects.append(&kv.value_ptr.Object);	644	try all_objects.append(&kv.value_ptr.object);
645	}	645	}
646	}	646	}
647	// Some options have multiple matches. As an example, "-Wl,foo" matches both	647	// Some options have multiple matches. As an example, "-Wl,foo" matches both
...	@@ -666,12 +666,12 @@ pub fn main() anyerror!void {	...	@@ -666,12 +666,12 @@ pub fn main() anyerror!void {
666	);	666	);
667		667
668	for (all_objects.items) \|obj\| {	668	for (all_objects.items) \|obj\| {
669	const name = obj.get("Name").?.String;	669	const name = obj.get("Name").?.string;
670	var pd1 = false;	670	var pd1 = false;
671	var pd2 = false;	671	var pd2 = false;
672	var pslash = false;	672	var pslash = false;
673	for (obj.get("Prefixes").?.Array.items) \|prefix_json\| {	673	for (obj.get("Prefixes").?.array.items) \|prefix_json\| {
674	const prefix = prefix_json.String;	674	const prefix = prefix_json.string;
675	if (std.mem.eql(u8, prefix, "-")) {	675	if (std.mem.eql(u8, prefix, "-")) {
676	pd1 = true;	676	pd1 = true;
677	} else if (std.mem.eql(u8, prefix, "--")) {	677	} else if (std.mem.eql(u8, prefix, "--")) {
...	@@ -790,9 +790,9 @@ const Syntax = union(enum) {	...	@@ -790,9 +790,9 @@ const Syntax = union(enum) {
790	};	790	};
791		791
792	fn objSyntax(obj: *json.ObjectMap) ?Syntax {	792	fn objSyntax(obj: *json.ObjectMap) ?Syntax {
793	const num_args = @intCast(u8, obj.get("NumArgs").?.Integer);	793	const num_args = @intCast(u8, obj.get("NumArgs").?.integer);
794	for (obj.get("!superclasses").?.Array.items) \|superclass_json\| {	794	for (obj.get("!superclasses").?.array.items) \|superclass_json\| {
795	const superclass = superclass_json.String;	795	const superclass = superclass_json.string;
796	if (std.mem.eql(u8, superclass, "Joined")) {	796	if (std.mem.eql(u8, superclass, "Joined")) {
797	return .joined;	797	return .joined;
798	} else if (std.mem.eql(u8, superclass, "CLJoined")) {	798	} else if (std.mem.eql(u8, superclass, "CLJoined")) {
...	@@ -831,20 +831,20 @@ fn objSyntax(obj: *json.ObjectMap) ?Syntax {	...	@@ -831,20 +831,20 @@ fn objSyntax(obj: *json.ObjectMap) ?Syntax {
831	return .{ .multi_arg = num_args };	831	return .{ .multi_arg = num_args };
832	}	832	}
833	}	833	}
834	const name = obj.get("Name").?.String;	834	const name = obj.get("Name").?.string;
835	if (std.mem.eql(u8, name, "<input>")) {	835	if (std.mem.eql(u8, name, "<input>")) {
836	return .flag;	836	return .flag;
837	} else if (std.mem.eql(u8, name, "<unknown>")) {	837	} else if (std.mem.eql(u8, name, "<unknown>")) {
838	return .flag;	838	return .flag;
839	}	839	}
840	const kind_def = obj.get("Kind").?.Object.get("def").?.String;	840	const kind_def = obj.get("Kind").?.object.get("def").?.string;
841	if (std.mem.eql(u8, kind_def, "KIND_FLAG")) {	841	if (std.mem.eql(u8, kind_def, "KIND_FLAG")) {
842	return .flag;	842	return .flag;
843	}	843	}
844	const key = obj.get("!name").?.String;	844	const key = obj.get("!name").?.string;
845	std.debug.print("{s} (key {s}) has unrecognized superclasses:\n", .{ name, key });	845	std.debug.print("{s} (key {s}) has unrecognized superclasses:\n", .{ name, key });
846	for (obj.get("!superclasses").?.Array.items) \|superclass_json\| {	846	for (obj.get("!superclasses").?.array.items) \|superclass_json\| {
847	std.debug.print(" {s}\n", .{superclass_json.String});	847	std.debug.print(" {s}\n", .{superclass_json.string});
848	}	848	}
849	//std.process.exit(1);	849	//std.process.exit(1);
850	return null;	850	return null;
...	@@ -883,15 +883,15 @@ fn objectLessThan(context: void, a: json.ObjectMap, b: json.ObjectMap) bool {	...	@@ -883,15 +883,15 @@ fn objectLessThan(context: void, a: json.ObjectMap, b: json.ObjectMap) bool {
883	}	883	}
884		884
885	if (!a_match_with_eql and !b_match_with_eql) {	885	if (!a_match_with_eql and !b_match_with_eql) {
886	const a_name = a.get("Name").?.String;	886	const a_name = a.get("Name").?.string;
887	const b_name = b.get("Name").?.String;	887	const b_name = b.get("Name").?.string;
888	if (a_name.len != b_name.len) {	888	if (a_name.len != b_name.len) {
889	return a_name.len > b_name.len;	889	return a_name.len > b_name.len;
890	}	890	}
891	}	891	}
892		892
893	const a_key = a.get("!name").?.String;	893	const a_key = a.get("!name").?.string;
894	const b_key = b.get("!name").?.String;	894	const b_key = b.get("!name").?.string;
895	return std.mem.lessThan(u8, a_key, b_key);	895	return std.mem.lessThan(u8, a_key, b_key);
896	}	896	}
897		897

tools/update_cpu_features.zig+22-22

...	@@ -1054,14 +1054,14 @@ fn processOneTarget(job: Job) anyerror!void {	...	@@ -1054,14 +1054,14 @@ fn processOneTarget(job: Job) anyerror!void {
1054	var json_parse_progress = progress_node.start("parse JSON", 0);	1054	var json_parse_progress = progress_node.start("parse JSON", 0);
1055	json_parse_progress.activate();	1055	json_parse_progress.activate();
1056		1056
1057	var parser = json.Parser.init(arena, false);	1057	var parser = json.Parser.init(arena, .alloc_if_needed);
1058	const tree = try parser.parse(json_text);	1058	const tree = try parser.parse(json_text);
1059	json_parse_progress.end();	1059	json_parse_progress.end();
1060		1060
1061	var render_progress = progress_node.start("render zig code", 0);	1061	var render_progress = progress_node.start("render zig code", 0);
1062	render_progress.activate();	1062	render_progress.activate();
1063		1063
1064	const root_map = &tree.root.Object;	1064	const root_map = &tree.root.object;
1065	var features_table = std.StringHashMap(Feature).init(arena);	1065	var features_table = std.StringHashMap(Feature).init(arena);
1066	var all_features = std.ArrayList(Feature).init(arena);	1066	var all_features = std.ArrayList(Feature).init(arena);
1067	var all_cpus = std.ArrayList(Cpu).init(arena);	1067	var all_cpus = std.ArrayList(Cpu).init(arena);
...	@@ -1070,21 +1070,21 @@ fn processOneTarget(job: Job) anyerror!void {	...	@@ -1070,21 +1070,21 @@ fn processOneTarget(job: Job) anyerror!void {
1070	root_it: while (it.next()) \|kv\| {	1070	root_it: while (it.next()) \|kv\| {
1071	if (kv.key_ptr.len == 0) continue;	1071	if (kv.key_ptr.len == 0) continue;
1072	if (kv.key_ptr.*[0] == '!') continue;	1072	if (kv.key_ptr.*[0] == '!') continue;
1073	if (kv.value_ptr.* != .Object) continue;	1073	if (kv.value_ptr.* != .object) continue;
1074	if (hasSuperclass(&kv.value_ptr.Object, "SubtargetFeature")) {	1074	if (hasSuperclass(&kv.value_ptr.object, "SubtargetFeature")) {
1075	const llvm_name = kv.value_ptr.Object.get("Name").?.String;	1075	const llvm_name = kv.value_ptr.object.get("Name").?.string;
1076	if (llvm_name.len == 0) continue;	1076	if (llvm_name.len == 0) continue;
1077		1077
1078	var zig_name = try llvmNameToZigName(arena, llvm_name);	1078	var zig_name = try llvmNameToZigName(arena, llvm_name);
1079	var desc = kv.value_ptr.Object.get("Desc").?.String;	1079	var desc = kv.value_ptr.object.get("Desc").?.string;
1080	var deps = std.ArrayList([]const u8).init(arena);	1080	var deps = std.ArrayList([]const u8).init(arena);
1081	var omit = false;	1081	var omit = false;
1082	var flatten = false;	1082	var flatten = false;
1083	const implies = kv.value_ptr.Object.get("Implies").?.Array;	1083	const implies = kv.value_ptr.object.get("Implies").?.array;
1084	for (implies.items) \|imply\| {	1084	for (implies.items) \|imply\| {
1085	const other_key = imply.Object.get("def").?.String;	1085	const other_key = imply.object.get("def").?.string;
1086	const other_obj = &root_map.getPtr(other_key).?.Object;	1086	const other_obj = &root_map.getPtr(other_key).?.object;
1087	const other_llvm_name = other_obj.get("Name").?.String;	1087	const other_llvm_name = other_obj.get("Name").?.string;
1088	const other_zig_name = (try llvmNameToZigNameOmit(	1088	const other_zig_name = (try llvmNameToZigNameOmit(
1089	arena,	1089	arena,
1090	llvm_target,	1090	llvm_target,
...	@@ -1126,17 +1126,17 @@ fn processOneTarget(job: Job) anyerror!void {	...	@@ -1126,17 +1126,17 @@ fn processOneTarget(job: Job) anyerror!void {
1126	try all_features.append(feature);	1126	try all_features.append(feature);
1127	}	1127	}
1128	}	1128	}
1129	if (hasSuperclass(&kv.value_ptr.Object, "Processor")) {	1129	if (hasSuperclass(&kv.value_ptr.object, "Processor")) {
1130	const llvm_name = kv.value_ptr.Object.get("Name").?.String;	1130	const llvm_name = kv.value_ptr.object.get("Name").?.string;
1131	if (llvm_name.len == 0) continue;	1131	if (llvm_name.len == 0) continue;
1132		1132
1133	var zig_name = try llvmNameToZigName(arena, llvm_name);	1133	var zig_name = try llvmNameToZigName(arena, llvm_name);
1134	var deps = std.ArrayList([]const u8).init(arena);	1134	var deps = std.ArrayList([]const u8).init(arena);
1135	const features = kv.value_ptr.Object.get("Features").?.Array;	1135	const features = kv.value_ptr.object.get("Features").?.array;
1136	for (features.items) \|feature\| {	1136	for (features.items) \|feature\| {
1137	const feature_key = feature.Object.get("def").?.String;	1137	const feature_key = feature.object.get("def").?.string;
1138	const feature_obj = &root_map.getPtr(feature_key).?.Object;	1138	const feature_obj = &root_map.getPtr(feature_key).?.object;
1139	const feature_llvm_name = feature_obj.get("Name").?.String;	1139	const feature_llvm_name = feature_obj.get("Name").?.string;
1140	if (feature_llvm_name.len == 0) continue;	1140	if (feature_llvm_name.len == 0) continue;
1141	const feature_zig_name = (try llvmNameToZigNameOmit(	1141	const feature_zig_name = (try llvmNameToZigNameOmit(
1142	arena,	1142	arena,
...	@@ -1145,11 +1145,11 @@ fn processOneTarget(job: Job) anyerror!void {	...	@@ -1145,11 +1145,11 @@ fn processOneTarget(job: Job) anyerror!void {
1145	)) orelse continue;	1145	)) orelse continue;
1146	try deps.append(feature_zig_name);	1146	try deps.append(feature_zig_name);
1147	}	1147	}
1148	const tune_features = kv.value_ptr.Object.get("TuneFeatures").?.Array;	1148	const tune_features = kv.value_ptr.object.get("TuneFeatures").?.array;
1149	for (tune_features.items) \|feature\| {	1149	for (tune_features.items) \|feature\| {
1150	const feature_key = feature.Object.get("def").?.String;	1150	const feature_key = feature.object.get("def").?.string;
1151	const feature_obj = &root_map.getPtr(feature_key).?.Object;	1151	const feature_obj = &root_map.getPtr(feature_key).?.object;
1152	const feature_llvm_name = feature_obj.get("Name").?.String;	1152	const feature_llvm_name = feature_obj.get("Name").?.string;
1153	if (feature_llvm_name.len == 0) continue;	1153	if (feature_llvm_name.len == 0) continue;
1154	const feature_zig_name = (try llvmNameToZigNameOmit(	1154	const feature_zig_name = (try llvmNameToZigNameOmit(
1155	arena,	1155	arena,
...	@@ -1431,8 +1431,8 @@ fn llvmNameToZigNameOmit(	...	@@ -1431,8 +1431,8 @@ fn llvmNameToZigNameOmit(
1431		1431
1432	fn hasSuperclass(obj: *json.ObjectMap, class_name: []const u8) bool {	1432	fn hasSuperclass(obj: *json.ObjectMap, class_name: []const u8) bool {
1433	const superclasses_json = obj.get("!superclasses") orelse return false;	1433	const superclasses_json = obj.get("!superclasses") orelse return false;
1434	for (superclasses_json.Array.items) \|superclass_json\| {	1434	for (superclasses_json.array.items) \|superclass_json\| {
1435	const superclass = superclass_json.String;	1435	const superclass = superclass_json.string;
1436	if (std.mem.eql(u8, superclass, class_name)) {	1436	if (std.mem.eql(u8, superclass, class_name)) {
1437	return true;	1437	return true;
1438	}	1438	}

tools/update_spirv_features.zig+1-2

...	@@ -74,8 +74,7 @@ pub fn main() !void {	...	@@ -74,8 +74,7 @@ pub fn main() !void {
74		74
75	const registry_path = try fs.path.join(allocator, &.{ spirv_headers_root, "include", "spirv", "unified1", "spirv.core.grammar.json" });	75	const registry_path = try fs.path.join(allocator, &.{ spirv_headers_root, "include", "spirv", "unified1", "spirv.core.grammar.json" });
76	const registry_json = try std.fs.cwd().readFileAlloc(allocator, registry_path, std.math.maxInt(usize));	76	const registry_json = try std.fs.cwd().readFileAlloc(allocator, registry_path, std.math.maxInt(usize));
77	var tokens = std.json.TokenStream.init(registry_json);	77	const registry = try std.json.parseFromSlice(g.CoreRegistry, allocator, registry_json, .{});
78	const registry = try std.json.parse(g.CoreRegistry, &tokens, .{ .allocator = allocator });
79		78
80	const capabilities = for (registry.operand_kinds) \|opkind\| {	79	const capabilities = for (registry.operand_kinds) \|opkind\| {
81	if (std.mem.eql(u8, opkind.kind, "Capability"))	80	if (std.mem.eql(u8, opkind.kind, "Capability"))