Commit 018b743c7a83c2af5e5b6ba9aae1a4703e306f71 | zig

author	Josh Wolfe <thejoshwolfe@gmail.com> 2023-05-13 14:31:53-04:00
committer	GitHub <noreply@github.com> 2023-05-13 14:31:53-04:00
log	018b743c7a83c2af5e5b6ba9aae1a4703e306f71
tree	3e113cc28cc3dcaace4917980c2813b1a6de2654
parent	c7bf8bab38f8b89c1371eedb9229e00a29b5ca5b
signature	Signed by PGP key 4AEE18F83AFDEB23

std: Rewrite low-level json api to support streaming (#15602)

20 files changed, 5793 insertions(+), 5845 deletions(-)

lib/std/json.zig+50-2809

...	...	@@ -1,2818 +1,59 @@
1		// JSON parser conforming to RFC8259.
2		//
3		// https://tools.ietf.org/html/rfc8259
4
5		const builtin = @import("builtin");
6		const std = @import("std.zig");
7		const debug = std.debug;
8		const assert = debug.assert;
9		const testing = std.testing;
10		const mem = std.mem;
11		const maxInt = std.math.maxInt;
	1	//! JSON parsing and stringification conforming to RFC 8259. https://datatracker.ietf.org/doc/html/rfc8259
	2	//!
	3	//! The low-level `Scanner` API reads from an input slice or successive slices of inputs,
	4	//! The `Reader` API connects a `std.io.Reader` to a `Scanner`.
	5	//!
	6	//! The high-level `parseFromSlice` and `parseFromTokenSource` deserializes a JSON document into a Zig type.
	7	//! The high-level `Parser` parses any JSON document into a dynamically typed `ValueTree` that has its own memory arena.
	8	//!
	9	//! The low-level `writeStream` emits syntax-conformant JSON tokens to a `std.io.Writer`.
	10	//! The high-level `stringify` serializes a Zig type into JSON.
	11
	12	pub const ValueTree = @import("json/dynamic.zig").ValueTree;
	13	pub const ObjectMap = @import("json/dynamic.zig").ObjectMap;
	14	pub const Array = @import("json/dynamic.zig").Array;
	15	pub const Value = @import("json/dynamic.zig").Value;
	16	pub const Parser = @import("json/dynamic.zig").Parser;
	17
	18	pub const validate = @import("json/scanner.zig").validate;
	19	pub const Error = @import("json/scanner.zig").Error;
	20	pub const reader = @import("json/scanner.zig").reader;
	21	pub const default_buffer_size = @import("json/scanner.zig").default_buffer_size;
	22	pub const Token = @import("json/scanner.zig").Token;
	23	pub const TokenType = @import("json/scanner.zig").TokenType;
	24	pub const Diagnostics = @import("json/scanner.zig").Diagnostics;
	25	pub const AllocWhen = @import("json/scanner.zig").AllocWhen;
	26	pub const default_max_value_len = @import("json/scanner.zig").default_max_value_len;
	27	pub const Reader = @import("json/scanner.zig").Reader;
	28	pub const Scanner = @import("json/scanner.zig").Scanner;
	29	pub const isNumberFormattedLikeAnInteger = @import("json/scanner.zig").isNumberFormattedLikeAnInteger;
	30
	31	pub const ParseOptions = @import("json/static.zig").ParseOptions;
	32	pub const parseFromSlice = @import("json/static.zig").parseFromSlice;
	33	pub const parseFromTokenSource = @import("json/static.zig").parseFromTokenSource;
	34	pub const ParseError = @import("json/static.zig").ParseError;
	35	pub const parseFree = @import("json/static.zig").parseFree;
	36
	37	pub const StringifyOptions = @import("json/stringify.zig").StringifyOptions;
	38	pub const encodeJsonString = @import("json/stringify.zig").encodeJsonString;
	39	pub const encodeJsonStringChars = @import("json/stringify.zig").encodeJsonStringChars;
	40	pub const stringify = @import("json/stringify.zig").stringify;
	41	pub const stringifyAlloc = @import("json/stringify.zig").stringifyAlloc;
12	42
13	43	pub const WriteStream = @import("json/write_stream.zig").WriteStream;
14	44	pub const writeStream = @import("json/write_stream.zig").writeStream;
15	45
16		const StringEscapes = union(enum) {
17		None,
18
19		Some: struct {
20		size_diff: isize,
21		},
22		};
23
24		/// Checks to see if a string matches what it would be as a json-encoded string
25		/// Assumes that `encoded` is a well-formed json string
26		fn encodesTo(decoded: []const u8, encoded: []const u8) bool {
27		var i: usize = 0;
28		var j: usize = 0;
29		while (i < decoded.len) {
30		if (j >= encoded.len) return false;
31		if (encoded[j] != '\\') {
32		if (decoded[i] != encoded[j]) return false;
33		j += 1;
34		i += 1;
35		} else {
36		const escape_type = encoded[j + 1];
37		if (escape_type != 'u') {
38		const t: u8 = switch (escape_type) {
39		'\\' => '\\',
40		'/' => '/',
41		'n' => '\n',
42		'r' => '\r',
43		't' => '\t',
44		'f' => 12,
45		'b' => 8,
46		'"' => '"',
47		else => unreachable,
48		};
49		if (decoded[i] != t) return false;
50		j += 2;
51		i += 1;
52		} else {
53		var codepoint = std.fmt.parseInt(u21, encoded[j + 2 .. j + 6], 16) catch unreachable;
54		j += 6;
55		if (codepoint >= 0xD800 and codepoint < 0xDC00) {
56		// surrogate pair
57		assert(encoded[j] == '\\');
58		assert(encoded[j + 1] == 'u');
59		const low_surrogate = std.fmt.parseInt(u21, encoded[j + 2 .. j + 6], 16) catch unreachable;
60		codepoint = 0x10000 + (((codepoint & 0x03ff) << 10) \| (low_surrogate & 0x03ff));
61		j += 6;
62		}
63		var buf: [4]u8 = undefined;
64		const len = std.unicode.utf8Encode(codepoint, &buf) catch unreachable;
65		if (i + len > decoded.len) return false;
66		if (!mem.eql(u8, decoded[i..][0..len], buf[0..len])) return false;
67		i += len;
68		}
69		}
70		}
71		assert(i == decoded.len);
72		assert(j == encoded.len);
73		return true;
74		}
75
76		/// A single token slice into the parent string.
77		///
78		/// Use `token.slice()` on the input at the current position to get the current slice.
79		pub const Token = union(enum) {
80		ObjectBegin,
81		ObjectEnd,
82		ArrayBegin,
83		ArrayEnd,
84		String: struct {
85		/// How many bytes the token is.
86		count: usize,
87
88		/// Whether string contains an escape sequence and cannot be zero-copied
89		escapes: StringEscapes,
90
91		pub fn decodedLength(self: @This()) usize {
92		return self.count +% switch (self.escapes) {
93		.None => 0,
94		.Some => \|s\| @bitCast(usize, s.size_diff),
95		};
96		}
97
98		/// Slice into the underlying input string.
99		pub fn slice(self: @This(), input: []const u8, i: usize) []const u8 {
100		return input[i - self.count .. i];
101		}
102		},
103		Number: struct {
104		/// How many bytes the token is.
105		count: usize,
106
107		/// Whether number is simple and can be represented by an integer (i.e. no `.` or `e`)
108		is_integer: bool,
109
110		/// Slice into the underlying input string.
111		pub fn slice(self: @This(), input: []const u8, i: usize) []const u8 {
112		return input[i - self.count .. i];
113		}
114		},
115		True,
116		False,
117		Null,
118		};
119
120		const AggregateContainerType = enum(u1) { object, array };
121
122		// A LIFO bit-stack. Tracks which container-types have been entered during parse.
123		fn AggregateContainerStack(comptime n: usize) type {
124		return struct {
125		const Self = @This();
126
127		const element_bitcount = 8 * @sizeOf(usize);
128		const element_count = n / element_bitcount;
129		const ElementType = @Type(.{ .Int = .{ .signedness = .unsigned, .bits = element_bitcount } });
130		const ElementShiftAmountType = std.math.Log2Int(ElementType);
131
132		comptime {
133		std.debug.assert(n % element_bitcount == 0);
134		}
135
136		memory: [element_count]ElementType,
137		len: usize,
138
139		pub fn init(self: *Self) void {
140		self.memory = [_]ElementType{0} ** element_count;
141		self.len = 0;
142		}
143
144		pub fn push(self: *Self, ty: AggregateContainerType) ?void {
145		if (self.len >= n) {
146		return null;
147		}
148
149		const index = self.len / element_bitcount;
150		const sub_index = @intCast(ElementShiftAmountType, self.len % element_bitcount);
151		const clear_mask = ~(@as(ElementType, 1) << sub_index);
152		const set_bits = @as(ElementType, @enumToInt(ty)) << sub_index;
153
154		self.memory[index] &= clear_mask;
155		self.memory[index] \|= set_bits;
156		self.len += 1;
157		}
158
159		pub fn peek(self: *Self) ?AggregateContainerType {
160		if (self.len == 0) {
161		return null;
162		}
163
164		const bit_to_extract = self.len - 1;
165		const index = bit_to_extract / element_bitcount;
166		const sub_index = @intCast(ElementShiftAmountType, bit_to_extract % element_bitcount);
167		const bit = @intCast(u1, (self.memory[index] >> sub_index) & 1);
168		return @intToEnum(AggregateContainerType, bit);
169		}
170
171		pub fn pop(self: *Self) ?AggregateContainerType {
172		if (self.peek()) \|ty\| {
173		self.len -= 1;
174		return ty;
175		}
176
177		return null;
178		}
179		};
180		}
181
182		/// A small streaming JSON parser. This accepts input one byte at a time and returns tokens as
183		/// they are encountered. No copies or allocations are performed during parsing and the entire
184		/// parsing state requires ~40-50 bytes of stack space.
185		///
186		/// Conforms strictly to RFC8259.
187		///
188		/// For a non-byte based wrapper, consider using TokenStream instead.
189		pub const StreamingParser = struct {
190		const default_max_nestings = 256;
191
192		// Current state
193		state: State,
194		// How many bytes we have counted for the current token
195		count: usize,
196		// What state to follow after parsing a string (either property or value string)
197		after_string_state: State,
198		// What state to follow after parsing a value (either top-level or value end)
199		after_value_state: State,
200		// If we stopped now, would the complete parsed string to now be a valid json string
201		complete: bool,
202		// Current token flags to pass through to the next generated, see Token.
203		string_escapes: StringEscapes,
204		// When in .String states, was the previous character a high surrogate?
205		string_last_was_high_surrogate: bool,
206		// Used inside of StringEscapeHexUnicode* states
207		string_unicode_codepoint: u21,
208		// The first byte needs to be stored to validate 3- and 4-byte sequences.
209		sequence_first_byte: u8 = undefined,
210		// When in .Number states, is the number a (still) valid integer?
211		number_is_integer: bool,
212		// Bit-stack for nested object/map literals (max 256 nestings).
213		stack: AggregateContainerStack(default_max_nestings),
214
215		pub fn init() StreamingParser {
216		var p: StreamingParser = undefined;
217		p.reset();
218		return p;
219		}
220
221		pub fn reset(p: *StreamingParser) void {
222		p.state = .TopLevelBegin;
223		p.count = 0;
224		// Set before ever read in main transition function
225		p.after_string_state = undefined;
226		p.after_value_state = .ValueEnd; // handle end of values normally
227		p.stack.init();
228		p.complete = false;
229		p.string_escapes = undefined;
230		p.string_last_was_high_surrogate = undefined;
231		p.string_unicode_codepoint = undefined;
232		p.number_is_integer = undefined;
233		}
234
235		pub const State = enum(u8) {
236		// These must be first with these explicit values as we rely on them for indexing the
237		// bit-stack directly and avoiding a branch.
238		ObjectSeparator = 0,
239		ValueEnd = 1,
240
241		TopLevelBegin,
242		TopLevelEnd,
243
244		ValueBegin,
245		ValueBeginNoClosing,
246
247		String,
248		StringUtf8Byte2Of2,
249		StringUtf8Byte2Of3,
250		StringUtf8Byte3Of3,
251		StringUtf8Byte2Of4,
252		StringUtf8Byte3Of4,
253		StringUtf8Byte4Of4,
254		StringEscapeCharacter,
255		StringEscapeHexUnicode4,
256		StringEscapeHexUnicode3,
257		StringEscapeHexUnicode2,
258		StringEscapeHexUnicode1,
259
260		Number,
261		NumberMaybeDotOrExponent,
262		NumberMaybeDigitOrDotOrExponent,
263		NumberFractionalRequired,
264		NumberFractional,
265		NumberMaybeExponent,
266		NumberExponent,
267		NumberExponentDigitsRequired,
268		NumberExponentDigits,
269
270		TrueLiteral1,
271		TrueLiteral2,
272		TrueLiteral3,
273
274		FalseLiteral1,
275		FalseLiteral2,
276		FalseLiteral3,
277		FalseLiteral4,
278
279		NullLiteral1,
280		NullLiteral2,
281		NullLiteral3,
282
283		// Given an aggregate container type, return the state which should be entered after
284		// processing a complete value type.
285		pub fn fromAggregateContainerType(ty: AggregateContainerType) State {
286		comptime {
287		std.debug.assert(@enumToInt(AggregateContainerType.object) == @enumToInt(State.ObjectSeparator));
288		std.debug.assert(@enumToInt(AggregateContainerType.array) == @enumToInt(State.ValueEnd));
289		}
290
291		return @intToEnum(State, @enumToInt(ty));
292		}
293		};
294
295		pub const Error = error{
296		InvalidTopLevel,
297		TooManyNestedItems,
298		TooManyClosingItems,
299		InvalidValueBegin,
300		InvalidValueEnd,
301		UnbalancedBrackets,
302		UnbalancedBraces,
303		UnexpectedClosingBracket,
304		UnexpectedClosingBrace,
305		InvalidNumber,
306		InvalidSeparator,
307		InvalidLiteral,
308		InvalidEscapeCharacter,
309		InvalidUnicodeHexSymbol,
310		InvalidUtf8Byte,
311		InvalidTopLevelTrailing,
312		InvalidControlCharacter,
313		};
314
315		/// Give another byte to the parser and obtain any new tokens. This may (rarely) return two
316		/// tokens. token2 is always null if token1 is null.
317		///
318		/// There is currently no error recovery on a bad stream.
319		pub fn feed(p: StreamingParser, c: u8, token1: ?Token, token2: *?Token) Error!void {
320		token1.* = null;
321		token2.* = null;
322		p.count += 1;
323
324		// unlikely
325		if (try p.transition(c, token1)) {
326		_ = try p.transition(c, token2);
327		}
328		}
329
330		// Perform a single transition on the state machine and return any possible token.
331		fn transition(p: StreamingParser, c: u8, token: ?Token) Error!bool {
332		switch (p.state) {
333		.TopLevelBegin => switch (c) {
334		'{' => {
335		p.stack.push(.object) orelse return error.TooManyNestedItems;
336		p.state = .ValueBegin;
337		p.after_string_state = .ObjectSeparator;
338
339		token.* = Token.ObjectBegin;
340		},
341		'[' => {
342		p.stack.push(.array) orelse return error.TooManyNestedItems;
343		p.state = .ValueBegin;
344		p.after_string_state = .ValueEnd;
345
346		token.* = Token.ArrayBegin;
347		},
348		'-' => {
349		p.number_is_integer = true;
350		p.state = .Number;
351		p.after_value_state = .TopLevelEnd;
352		p.count = 0;
353		},
354		'0' => {
355		p.number_is_integer = true;
356		p.state = .NumberMaybeDotOrExponent;
357		p.after_value_state = .TopLevelEnd;
358		p.count = 0;
359		},
360		'1'...'9' => {
361		p.number_is_integer = true;
362		p.state = .NumberMaybeDigitOrDotOrExponent;
363		p.after_value_state = .TopLevelEnd;
364		p.count = 0;
365		},
366		'"' => {
367		p.state = .String;
368		p.after_value_state = .TopLevelEnd;
369		// We don't actually need the following since after_value_state should override.
370		p.after_string_state = .ValueEnd;
371		p.string_escapes = .None;
372		p.string_last_was_high_surrogate = false;
373		p.count = 0;
374		},
375		't' => {
376		p.state = .TrueLiteral1;
377		p.after_value_state = .TopLevelEnd;
378		p.count = 0;
379		},
380		'f' => {
381		p.state = .FalseLiteral1;
382		p.after_value_state = .TopLevelEnd;
383		p.count = 0;
384		},
385		'n' => {
386		p.state = .NullLiteral1;
387		p.after_value_state = .TopLevelEnd;
388		p.count = 0;
389		},
390		0x09, 0x0A, 0x0D, 0x20 => {
391		// whitespace
392		},
393		else => {
394		return error.InvalidTopLevel;
395		},
396		},
397
398		.TopLevelEnd => switch (c) {
399		0x09, 0x0A, 0x0D, 0x20 => {
400		// whitespace
401		},
402		else => {
403		return error.InvalidTopLevelTrailing;
404		},
405		},
406
407		.ValueBegin => switch (c) {
408		// NOTE: These are shared in ValueEnd as well, think we can reorder states to
409		// be a bit clearer and avoid this duplication.
410		'}' => {
411		const last_type = p.stack.peek() orelse return error.TooManyClosingItems;
412
413		if (last_type != .object) {
414		return error.UnexpectedClosingBrace;
415		}
416
417		_ = p.stack.pop();
418		p.state = .ValueBegin;
419		p.after_string_state = State.fromAggregateContainerType(last_type);
420
421		switch (p.stack.len) {
422		0 => {
423		p.complete = true;
424		p.state = .TopLevelEnd;
425		},
426		else => {
427		p.state = .ValueEnd;
428		},
429		}
430
431		token.* = Token.ObjectEnd;
432		},
433		']' => {
434		const last_type = p.stack.peek() orelse return error.TooManyClosingItems;
435
436		if (last_type != .array) {
437		return error.UnexpectedClosingBracket;
438		}
439
440		_ = p.stack.pop();
441		p.state = .ValueBegin;
442		p.after_string_state = State.fromAggregateContainerType(last_type);
443
444		switch (p.stack.len) {
445		0 => {
446		p.complete = true;
447		p.state = .TopLevelEnd;
448		},
449		else => {
450		p.state = .ValueEnd;
451		},
452		}
453
454		token.* = Token.ArrayEnd;
455		},
456		'{' => {
457		p.stack.push(.object) orelse return error.TooManyNestedItems;
458
459		p.state = .ValueBegin;
460		p.after_string_state = .ObjectSeparator;
461
462		token.* = Token.ObjectBegin;
463		},
464		'[' => {
465		p.stack.push(.array) orelse return error.TooManyNestedItems;
466
467		p.state = .ValueBegin;
468		p.after_string_state = .ValueEnd;
469
470		token.* = Token.ArrayBegin;
471		},
472		'-' => {
473		p.number_is_integer = true;
474		p.state = .Number;
475		p.count = 0;
476		},
477		'0' => {
478		p.number_is_integer = true;
479		p.state = .NumberMaybeDotOrExponent;
480		p.count = 0;
481		},
482		'1'...'9' => {
483		p.number_is_integer = true;
484		p.state = .NumberMaybeDigitOrDotOrExponent;
485		p.count = 0;
486		},
487		'"' => {
488		p.state = .String;
489		p.string_escapes = .None;
490		p.string_last_was_high_surrogate = false;
491		p.count = 0;
492		},
493		't' => {
494		p.state = .TrueLiteral1;
495		p.count = 0;
496		},
497		'f' => {
498		p.state = .FalseLiteral1;
499		p.count = 0;
500		},
501		'n' => {
502		p.state = .NullLiteral1;
503		p.count = 0;
504		},
505		0x09, 0x0A, 0x0D, 0x20 => {
506		// whitespace
507		},
508		else => {
509		return error.InvalidValueBegin;
510		},
511		},
512
513		// TODO: A bit of duplication here and in the following state, redo.
514		.ValueBeginNoClosing => switch (c) {
515		'{' => {
516		p.stack.push(.object) orelse return error.TooManyNestedItems;
517
518		p.state = .ValueBegin;
519		p.after_string_state = .ObjectSeparator;
520
521		token.* = Token.ObjectBegin;
522		},
523		'[' => {
524		p.stack.push(.array) orelse return error.TooManyNestedItems;
525
526		p.state = .ValueBegin;
527		p.after_string_state = .ValueEnd;
528
529		token.* = Token.ArrayBegin;
530		},
531		'-' => {
532		p.number_is_integer = true;
533		p.state = .Number;
534		p.count = 0;
535		},
536		'0' => {
537		p.number_is_integer = true;
538		p.state = .NumberMaybeDotOrExponent;
539		p.count = 0;
540		},
541		'1'...'9' => {
542		p.number_is_integer = true;
543		p.state = .NumberMaybeDigitOrDotOrExponent;
544		p.count = 0;
545		},
546		'"' => {
547		p.state = .String;
548		p.string_escapes = .None;
549		p.string_last_was_high_surrogate = false;
550		p.count = 0;
551		},
552		't' => {
553		p.state = .TrueLiteral1;
554		p.count = 0;
555		},
556		'f' => {
557		p.state = .FalseLiteral1;
558		p.count = 0;
559		},
560		'n' => {
561		p.state = .NullLiteral1;
562		p.count = 0;
563		},
564		0x09, 0x0A, 0x0D, 0x20 => {
565		// whitespace
566		},
567		else => {
568		return error.InvalidValueBegin;
569		},
570		},
571
572		.ValueEnd => switch (c) {
573		',' => {
574		const last_type = p.stack.peek() orelse unreachable;
575		p.after_string_state = State.fromAggregateContainerType(last_type);
576		p.state = .ValueBeginNoClosing;
577		},
578		']' => {
579		const last_type = p.stack.peek() orelse return error.TooManyClosingItems;
580
581		if (last_type != .array) {
582		return error.UnexpectedClosingBracket;
583		}
584
585		_ = p.stack.pop();
586		p.state = .ValueEnd;
587		p.after_string_state = State.fromAggregateContainerType(last_type);
588
589		if (p.stack.len == 0) {
590		p.complete = true;
591		p.state = .TopLevelEnd;
592		}
593
594		token.* = Token.ArrayEnd;
595		},
596		'}' => {
597		const last_type = p.stack.peek() orelse return error.TooManyClosingItems;
598
599		if (last_type != .object) {
600		return error.UnexpectedClosingBrace;
601		}
602
603		_ = p.stack.pop();
604		p.state = .ValueEnd;
605		p.after_string_state = State.fromAggregateContainerType(last_type);
606
607		if (p.stack.len == 0) {
608		p.complete = true;
609		p.state = .TopLevelEnd;
610		}
611
612		token.* = Token.ObjectEnd;
613		},
614		0x09, 0x0A, 0x0D, 0x20 => {
615		// whitespace
616		},
617		else => {
618		return error.InvalidValueEnd;
619		},
620		},
621
622		.ObjectSeparator => switch (c) {
623		':' => {
624		p.state = .ValueBeginNoClosing;
625		p.after_string_state = .ValueEnd;
626		},
627		0x09, 0x0A, 0x0D, 0x20 => {
628		// whitespace
629		},
630		else => {
631		return error.InvalidSeparator;
632		},
633		},
634
635		.String => switch (c) {
636		0x00...0x1F => {
637		return error.InvalidControlCharacter;
638		},
639		'"' => {
640		p.state = p.after_string_state;
641		if (p.after_value_state == .TopLevelEnd) {
642		p.state = .TopLevelEnd;
643		p.complete = true;
644		}
645
646		token.* = .{
647		.String = .{
648		.count = p.count - 1,
649		.escapes = p.string_escapes,
650		},
651		};
652		p.string_escapes = undefined;
653		p.string_last_was_high_surrogate = undefined;
654		},
655		'\\' => {
656		p.state = .StringEscapeCharacter;
657		switch (p.string_escapes) {
658		.None => {
659		p.string_escapes = .{ .Some = .{ .size_diff = 0 } };
660		},
661		.Some => {},
662		}
663		},
664		0x20, 0x21, 0x23...0x5B, 0x5D...0x7F => {
665		// non-control ascii
666		p.string_last_was_high_surrogate = false;
667		},
668		0xC2...0xDF => {
669		p.state = .StringUtf8Byte2Of2;
670		},
671		0xE0...0xEF => {
672		p.state = .StringUtf8Byte2Of3;
673		p.sequence_first_byte = c;
674		},
675		0xF0...0xF4 => {
676		p.state = .StringUtf8Byte2Of4;
677		p.sequence_first_byte = c;
678		},
679		else => {
680		return error.InvalidUtf8Byte;
681		},
682		},
683
684		.StringUtf8Byte2Of2 => switch (c >> 6) {
685		0b10 => p.state = .String,
686		else => return error.InvalidUtf8Byte,
687		},
688		.StringUtf8Byte2Of3 => {
689		switch (p.sequence_first_byte) {
690		0xE0 => switch (c) {
691		0xA0...0xBF => {},
692		else => return error.InvalidUtf8Byte,
693		},
694		0xE1...0xEF => switch (c) {
695		0x80...0xBF => {},
696		else => return error.InvalidUtf8Byte,
697		},
698		else => return error.InvalidUtf8Byte,
699		}
700		p.state = .StringUtf8Byte3Of3;
701		},
702		.StringUtf8Byte3Of3 => switch (c) {
703		0x80...0xBF => p.state = .String,
704		else => return error.InvalidUtf8Byte,
705		},
706		.StringUtf8Byte2Of4 => {
707		switch (p.sequence_first_byte) {
708		0xF0 => switch (c) {
709		0x90...0xBF => {},
710		else => return error.InvalidUtf8Byte,
711		},
712		0xF1...0xF3 => switch (c) {
713		0x80...0xBF => {},
714		else => return error.InvalidUtf8Byte,
715		},
716		0xF4 => switch (c) {
717		0x80...0x8F => {},
718		else => return error.InvalidUtf8Byte,
719		},
720		else => return error.InvalidUtf8Byte,
721		}
722		p.state = .StringUtf8Byte3Of4;
723		},
724		.StringUtf8Byte3Of4 => switch (c) {
725		0x80...0xBF => p.state = .StringUtf8Byte4Of4,
726		else => return error.InvalidUtf8Byte,
727		},
728		.StringUtf8Byte4Of4 => switch (c) {
729		0x80...0xBF => p.state = .String,
730		else => return error.InvalidUtf8Byte,
731		},
732
733		.StringEscapeCharacter => switch (c) {
734		// NOTE: '/' is allowed as an escaped character but it also is allowed
735		// as unescaped according to the RFC. There is a reported errata which suggests
736		// removing the non-escaped variant but it makes more sense to simply disallow
737		// it as an escape code here.
738		//
739		// The current JSONTestSuite tests rely on both of this behaviour being present
740		// however, so we default to the status quo where both are accepted until this
741		// is further clarified.
742		'"', '\\', '/', 'b', 'f', 'n', 'r', 't' => {
743		p.string_escapes.Some.size_diff -= 1;
744		p.state = .String;
745		p.string_last_was_high_surrogate = false;
746		},
747		'u' => {
748		p.state = .StringEscapeHexUnicode4;
749		},
750		else => {
751		return error.InvalidEscapeCharacter;
752		},
753		},
754
755		.StringEscapeHexUnicode4 => {
756		var codepoint: u21 = undefined;
757		switch (c) {
758		else => return error.InvalidUnicodeHexSymbol,
759		'0'...'9' => {
760		codepoint = c - '0';
761		},
762		'A'...'F' => {
763		codepoint = c - 'A' + 10;
764		},
765		'a'...'f' => {
766		codepoint = c - 'a' + 10;
767		},
768		}
769		p.state = .StringEscapeHexUnicode3;
770		p.string_unicode_codepoint = codepoint << 12;
771		},
772
773		.StringEscapeHexUnicode3 => {
774		var codepoint: u21 = undefined;
775		switch (c) {
776		else => return error.InvalidUnicodeHexSymbol,
777		'0'...'9' => {
778		codepoint = c - '0';
779		},
780		'A'...'F' => {
781		codepoint = c - 'A' + 10;
782		},
783		'a'...'f' => {
784		codepoint = c - 'a' + 10;
785		},
786		}
787		p.state = .StringEscapeHexUnicode2;
788		p.string_unicode_codepoint \|= codepoint << 8;
789		},
790
791		.StringEscapeHexUnicode2 => {
792		var codepoint: u21 = undefined;
793		switch (c) {
794		else => return error.InvalidUnicodeHexSymbol,
795		'0'...'9' => {
796		codepoint = c - '0';
797		},
798		'A'...'F' => {
799		codepoint = c - 'A' + 10;
800		},
801		'a'...'f' => {
802		codepoint = c - 'a' + 10;
803		},
804		}
805		p.state = .StringEscapeHexUnicode1;
806		p.string_unicode_codepoint \|= codepoint << 4;
807		},
808
809		.StringEscapeHexUnicode1 => {
810		var codepoint: u21 = undefined;
811		switch (c) {
812		else => return error.InvalidUnicodeHexSymbol,
813		'0'...'9' => {
814		codepoint = c - '0';
815		},
816		'A'...'F' => {
817		codepoint = c - 'A' + 10;
818		},
819		'a'...'f' => {
820		codepoint = c - 'a' + 10;
821		},
822		}
823		p.state = .String;
824		p.string_unicode_codepoint \|= codepoint;
825		if (p.string_unicode_codepoint < 0xD800 or p.string_unicode_codepoint >= 0xE000) {
826		// not part of surrogate pair
827		p.string_escapes.Some.size_diff -= @as(isize, 6 - (std.unicode.utf8CodepointSequenceLength(p.string_unicode_codepoint) catch unreachable));
828		p.string_last_was_high_surrogate = false;
829		} else if (p.string_unicode_codepoint < 0xDC00) {
830		// 'high' surrogate
831		// takes 3 bytes to encode a half surrogate pair into wtf8
832		p.string_escapes.Some.size_diff -= 6 - 3;
833		p.string_last_was_high_surrogate = true;
834		} else {
835		// 'low' surrogate
836		p.string_escapes.Some.size_diff -= 6;
837		if (p.string_last_was_high_surrogate) {
838		// takes 4 bytes to encode a full surrogate pair into utf8
839		// 3 bytes are already reserved by high surrogate
840		p.string_escapes.Some.size_diff -= -1;
841		} else {
842		// takes 3 bytes to encode a half surrogate pair into wtf8
843		p.string_escapes.Some.size_diff -= -3;
844		}
845		p.string_last_was_high_surrogate = false;
846		}
847		p.string_unicode_codepoint = undefined;
848		},
849
850		.Number => {
851		p.complete = p.after_value_state == .TopLevelEnd;
852		switch (c) {
853		'0' => {
854		p.state = .NumberMaybeDotOrExponent;
855		},
856		'1'...'9' => {
857		p.state = .NumberMaybeDigitOrDotOrExponent;
858		},
859		else => {
860		return error.InvalidNumber;
861		},
862		}
863		},
864
865		.NumberMaybeDotOrExponent => {
866		p.complete = p.after_value_state == .TopLevelEnd;
867		switch (c) {
868		'.' => {
869		p.number_is_integer = false;
870		p.state = .NumberFractionalRequired;
871		},
872		'e', 'E' => {
873		p.number_is_integer = false;
874		p.state = .NumberExponent;
875		},
876		else => {
877		p.state = p.after_value_state;
878		token.* = .{
879		.Number = .{
880		.count = p.count,
881		.is_integer = p.number_is_integer,
882		},
883		};
884		p.number_is_integer = undefined;
885		return true;
886		},
887		}
888		},
889
890		.NumberMaybeDigitOrDotOrExponent => {
891		p.complete = p.after_value_state == .TopLevelEnd;
892		switch (c) {
893		'.' => {
894		p.number_is_integer = false;
895		p.state = .NumberFractionalRequired;
896		},
897		'e', 'E' => {
898		p.number_is_integer = false;
899		p.state = .NumberExponent;
900		},
901		'0'...'9' => {
902		// another digit
903		},
904		else => {
905		p.state = p.after_value_state;
906		token.* = .{
907		.Number = .{
908		.count = p.count,
909		.is_integer = p.number_is_integer,
910		},
911		};
912		return true;
913		},
914		}
915		},
916
917		.NumberFractionalRequired => {
918		p.complete = p.after_value_state == .TopLevelEnd;
919		switch (c) {
920		'0'...'9' => {
921		p.state = .NumberFractional;
922		},
923		else => {
924		return error.InvalidNumber;
925		},
926		}
927		},
928
929		.NumberFractional => {
930		p.complete = p.after_value_state == .TopLevelEnd;
931		switch (c) {
932		'0'...'9' => {
933		// another digit
934		},
935		'e', 'E' => {
936		p.number_is_integer = false;
937		p.state = .NumberExponent;
938		},
939		else => {
940		p.state = p.after_value_state;
941		token.* = .{
942		.Number = .{
943		.count = p.count,
944		.is_integer = p.number_is_integer,
945		},
946		};
947		return true;
948		},
949		}
950		},
951
952		.NumberMaybeExponent => {
953		p.complete = p.after_value_state == .TopLevelEnd;
954		switch (c) {
955		'e', 'E' => {
956		p.number_is_integer = false;
957		p.state = .NumberExponent;
958		},
959		else => {
960		p.state = p.after_value_state;
961		token.* = .{
962		.Number = .{
963		.count = p.count,
964		.is_integer = p.number_is_integer,
965		},
966		};
967		return true;
968		},
969		}
970		},
971
972		.NumberExponent => switch (c) {
973		'-', '+' => {
974		p.complete = false;
975		p.state = .NumberExponentDigitsRequired;
976		},
977		'0'...'9' => {
978		p.complete = p.after_value_state == .TopLevelEnd;
979		p.state = .NumberExponentDigits;
980		},
981		else => {
982		return error.InvalidNumber;
983		},
984		},
985
986		.NumberExponentDigitsRequired => switch (c) {
987		'0'...'9' => {
988		p.complete = p.after_value_state == .TopLevelEnd;
989		p.state = .NumberExponentDigits;
990		},
991		else => {
992		return error.InvalidNumber;
993		},
994		},
995
996		.NumberExponentDigits => {
997		p.complete = p.after_value_state == .TopLevelEnd;
998		switch (c) {
999		'0'...'9' => {
1000		// another digit
1001		},
1002		else => {
1003		p.state = p.after_value_state;
1004		token.* = .{
1005		.Number = .{
1006		.count = p.count,
1007		.is_integer = p.number_is_integer,
1008		},
1009		};
1010		return true;
1011		},
1012		}
1013		},
1014
1015		.TrueLiteral1 => switch (c) {
1016		'r' => p.state = .TrueLiteral2,
1017		else => return error.InvalidLiteral,
1018		},
1019
1020		.TrueLiteral2 => switch (c) {
1021		'u' => p.state = .TrueLiteral3,
1022		else => return error.InvalidLiteral,
1023		},
1024
1025		.TrueLiteral3 => switch (c) {
1026		'e' => {
1027		p.state = p.after_value_state;
1028		p.complete = p.state == .TopLevelEnd;
1029		token.* = Token.True;
1030		},
1031		else => {
1032		return error.InvalidLiteral;
1033		},
1034		},
1035
1036		.FalseLiteral1 => switch (c) {
1037		'a' => p.state = .FalseLiteral2,
1038		else => return error.InvalidLiteral,
1039		},
1040
1041		.FalseLiteral2 => switch (c) {
1042		'l' => p.state = .FalseLiteral3,
1043		else => return error.InvalidLiteral,
1044		},
1045
1046		.FalseLiteral3 => switch (c) {
1047		's' => p.state = .FalseLiteral4,
1048		else => return error.InvalidLiteral,
1049		},
1050
1051		.FalseLiteral4 => switch (c) {
1052		'e' => {
1053		p.state = p.after_value_state;
1054		p.complete = p.state == .TopLevelEnd;
1055		token.* = Token.False;
1056		},
1057		else => {
1058		return error.InvalidLiteral;
1059		},
1060		},
1061
1062		.NullLiteral1 => switch (c) {
1063		'u' => p.state = .NullLiteral2,
1064		else => return error.InvalidLiteral,
1065		},
1066
1067		.NullLiteral2 => switch (c) {
1068		'l' => p.state = .NullLiteral3,
1069		else => return error.InvalidLiteral,
1070		},
1071
1072		.NullLiteral3 => switch (c) {
1073		'l' => {
1074		p.state = p.after_value_state;
1075		p.complete = p.state == .TopLevelEnd;
1076		token.* = Token.Null;
1077		},
1078		else => {
1079		return error.InvalidLiteral;
1080		},
1081		},
1082		}
1083
1084		return false;
1085		}
1086		};
1087
1088		/// A small wrapper over a StreamingParser for full slices. Returns a stream of json Tokens.
1089		pub const TokenStream = struct {
1090		i: usize,
1091		slice: []const u8,
1092		parser: StreamingParser,
1093		token: ?Token,
1094
1095		pub const Error = StreamingParser.Error \|\| error{UnexpectedEndOfJson};
1096
1097		pub fn init(slice: []const u8) TokenStream {
1098		return TokenStream{
1099		.i = 0,
1100		.slice = slice,
1101		.parser = StreamingParser.init(),
1102		.token = null,
1103		};
1104		}
1105
1106		fn stackUsed(self: *TokenStream) usize {
1107		return self.parser.stack.len + if (self.token != null) @as(usize, 1) else 0;
1108		}
1109
1110		pub fn next(self: *TokenStream) Error!?Token {
1111		if (self.token) \|token\| {
1112		self.token = null;
1113		return token;
1114		}
1115
1116		var t1: ?Token = undefined;
1117		var t2: ?Token = undefined;
1118
1119		while (self.i < self.slice.len) {
1120		try self.parser.feed(self.slice[self.i], &t1, &t2);
1121		self.i += 1;
1122
1123		if (t1) \|token\| {
1124		self.token = t2;
1125		return token;
1126		}
1127		}
1128
1129		// Without this a bare number fails, the streaming parser doesn't know the input ended
1130		try self.parser.feed(' ', &t1, &t2);
1131		self.i += 1;
1132
1133		if (t1) \|token\| {
1134		return token;
1135		} else if (self.parser.complete) {
1136		return null;
1137		} else {
1138		return error.UnexpectedEndOfJson;
1139		}
1140		}
1141		};
1142
1143		/// Validate a JSON string. This does not limit number precision so a decoder may not necessarily
1144		/// be able to decode the string even if this returns true.
1145		pub fn validate(s: []const u8) bool {
1146		var p = StreamingParser.init();
1147
1148		for (s) \|c\| {
1149		var token1: ?Token = undefined;
1150		var token2: ?Token = undefined;
1151
1152		p.feed(c, &token1, &token2) catch {
1153		return false;
1154		};
1155		}
1156
1157		return p.complete;
1158		}
1159
1160		const Allocator = std.mem.Allocator;
1161		const ArenaAllocator = std.heap.ArenaAllocator;
1162		const ArrayList = std.ArrayList;
1163		const StringArrayHashMap = std.StringArrayHashMap;
1164
1165		pub const ValueTree = struct {
1166		arena: *ArenaAllocator,
1167		root: Value,
1168
1169		pub fn deinit(self: *ValueTree) void {
1170		self.arena.deinit();
1171		self.arena.child_allocator.destroy(self.arena);
1172		}
1173		};
1174
1175		pub const ObjectMap = StringArrayHashMap(Value);
1176		pub const Array = ArrayList(Value);
1177
1178		/// Represents a JSON value
1179		/// Currently only supports numbers that fit into i64 or f64.
1180		pub const Value = union(enum) {
1181		Null,
1182		Bool: bool,
1183		Integer: i64,
1184		Float: f64,
1185		NumberString: []const u8,
1186		String: []const u8,
1187		Array: Array,
1188		Object: ObjectMap,
1189
1190		pub fn jsonStringify(
1191		value: @This(),
1192		options: StringifyOptions,
1193		out_stream: anytype,
1194		) @TypeOf(out_stream).Error!void {
1195		switch (value) {
1196		.Null => try stringify(null, options, out_stream),
1197		.Bool => \|inner\| try stringify(inner, options, out_stream),
1198		.Integer => \|inner\| try stringify(inner, options, out_stream),
1199		.Float => \|inner\| try stringify(inner, options, out_stream),
1200		.NumberString => \|inner\| try out_stream.writeAll(inner),
1201		.String => \|inner\| try stringify(inner, options, out_stream),
1202		.Array => \|inner\| try stringify(inner.items, options, out_stream),
1203		.Object => \|inner\| {
1204		try out_stream.writeByte('{');
1205		var field_output = false;
1206		var child_options = options;
1207		if (child_options.whitespace) \|*child_whitespace\| {
1208		child_whitespace.indent_level += 1;
1209		}
1210		var it = inner.iterator();
1211		while (it.next()) \|entry\| {
1212		if (!field_output) {
1213		field_output = true;
1214		} else {
1215		try out_stream.writeByte(',');
1216		}
1217		if (child_options.whitespace) \|child_whitespace\| {
1218		try child_whitespace.outputIndent(out_stream);
1219		}
1220
1221		try stringify(entry.key_ptr.*, options, out_stream);
1222		try out_stream.writeByte(':');
1223		if (child_options.whitespace) \|child_whitespace\| {
1224		if (child_whitespace.separator) {
1225		try out_stream.writeByte(' ');
1226		}
1227		}
1228		try stringify(entry.value_ptr.*, child_options, out_stream);
1229		}
1230		if (field_output) {
1231		if (options.whitespace) \|whitespace\| {
1232		try whitespace.outputIndent(out_stream);
1233		}
1234		}
1235		try out_stream.writeByte('}');
1236		},
1237		}
1238		}
1239
1240		pub fn dump(self: Value) void {
1241		std.debug.getStderrMutex().lock();
1242		defer std.debug.getStderrMutex().unlock();
1243
1244		const stderr = std.io.getStdErr().writer();
1245		std.json.stringify(self, std.json.StringifyOptions{ .whitespace = null }, stderr) catch return;
1246		}
1247		};
1248
1249		/// parse tokens from a stream, returning `false` if they do not decode to `value`
1250		fn parsesTo(comptime T: type, value: T, tokens: *TokenStream, options: ParseOptions) !bool {
1251		// TODO: should be able to write this function to not require an allocator
1252		const tmp = try parse(T, tokens, options);
1253		defer parseFree(T, tmp, options);
1254
1255		return parsedEqual(tmp, value);
1256		}
1257
1258		/// Returns if a value returned by `parse` is deep-equal to another value
1259		fn parsedEqual(a: anytype, b: @TypeOf(a)) bool {
1260		switch (@typeInfo(@TypeOf(a))) {
1261		.Optional => {
1262		if (a == null and b == null) return true;
1263		if (a == null or b == null) return false;
1264		return parsedEqual(a.?, b.?);
1265		},
1266		.Union => \|info\| {
1267		if (info.tag_type) \|UnionTag\| {
1268		const tag_a = std.meta.activeTag(a);
1269		const tag_b = std.meta.activeTag(b);
1270		if (tag_a != tag_b) return false;
1271
1272		inline for (info.fields) \|field_info\| {
1273		if (@field(UnionTag, field_info.name) == tag_a) {
1274		return parsedEqual(@field(a, field_info.name), @field(b, field_info.name));
1275		}
1276		}
1277		return false;
1278		} else {
1279		unreachable;
1280		}
1281		},
1282		.Array => {
1283		for (a, 0..) \|e, i\|
1284		if (!parsedEqual(e, b[i])) return false;
1285		return true;
1286		},
1287		.Struct => \|info\| {
1288		inline for (info.fields) \|field_info\| {
1289		if (!parsedEqual(@field(a, field_info.name), @field(b, field_info.name))) return false;
1290		}
1291		return true;
1292		},
1293		.Pointer => \|ptrInfo\| switch (ptrInfo.size) {
1294		.One => return parsedEqual(a., b.),
1295		.Slice => {
1296		if (a.len != b.len) return false;
1297		for (a, 0..) \|e, i\|
1298		if (!parsedEqual(e, b[i])) return false;
1299		return true;
1300		},
1301		.Many, .C => unreachable,
1302		},
1303		else => return a == b,
1304		}
1305		unreachable;
1306		}
1307
1308		pub const ParseOptions = struct {
1309		allocator: ?Allocator = null,
1310
1311		/// Behaviour when a duplicate field is encountered.
1312		duplicate_field_behavior: enum {
1313		UseFirst,
1314		Error,
1315		UseLast,
1316		} = .Error,
1317
1318		/// If false, finding an unknown field returns an error.
1319		ignore_unknown_fields: bool = false,
1320
1321		allow_trailing_data: bool = false,
1322		};
1323
1324		const SkipValueError = error{UnexpectedJsonDepth} \|\| TokenStream.Error;
1325
1326		fn skipValue(tokens: *TokenStream) SkipValueError!void {
1327		const original_depth = tokens.stackUsed();
1328
1329		// Return an error if no value is found
1330		_ = try tokens.next();
1331		if (tokens.stackUsed() < original_depth) return error.UnexpectedJsonDepth;
1332		if (tokens.stackUsed() == original_depth) return;
1333
1334		while (try tokens.next()) \|_\| {
1335		if (tokens.stackUsed() == original_depth) return;
1336		}
1337		}
1338
1339		fn ParseInternalError(comptime T: type) type {
1340		// `inferred_types` is used to avoid infinite recursion for recursive type definitions.
1341		const inferred_types = [_]type{};
1342		return ParseInternalErrorImpl(T, &inferred_types);
1343		}
1344
1345		fn ParseInternalErrorImpl(comptime T: type, comptime inferred_types: []const type) type {
1346		for (inferred_types) \|ty\| {
1347		if (T == ty) return error{};
1348		}
1349
1350		switch (@typeInfo(T)) {
1351		.Bool => return error{UnexpectedToken},
1352		.Float, .ComptimeFloat => return error{UnexpectedToken} \|\| std.fmt.ParseFloatError,
1353		.Int, .ComptimeInt => {
1354		return error{ UnexpectedToken, InvalidNumber, Overflow } \|\|
1355		std.fmt.ParseIntError \|\| std.fmt.ParseFloatError;
1356		},
1357		.Optional => \|optionalInfo\| {
1358		return ParseInternalErrorImpl(optionalInfo.child, inferred_types ++ [_]type{T});
1359		},
1360		.Enum => return error{ UnexpectedToken, InvalidEnumTag } \|\| std.fmt.ParseIntError \|\|
1361		std.meta.IntToEnumError \|\| std.meta.IntToEnumError,
1362		.Union => \|unionInfo\| {
1363		if (unionInfo.tag_type) \|_\| {
1364		var errors = error{NoUnionMembersMatched};
1365		for (unionInfo.fields) \|u_field\| {
1366		errors = errors \|\| ParseInternalErrorImpl(u_field.type, inferred_types ++ [_]type{T});
1367		}
1368		return errors;
1369		} else {
1370		@compileError("Unable to parse into untagged union '" ++ @typeName(T) ++ "'");
1371		}
1372		},
1373		.Struct => \|structInfo\| {
1374		var errors = error{
1375		DuplicateJSONField,
1376		UnexpectedEndOfJson,
1377		UnexpectedToken,
1378		UnexpectedValue,
1379		UnknownField,
1380		MissingField,
1381		} \|\| SkipValueError \|\| TokenStream.Error;
1382		for (structInfo.fields) \|field\| {
1383		errors = errors \|\| ParseInternalErrorImpl(field.type, inferred_types ++ [_]type{T});
1384		}
1385		return errors;
1386		},
1387		.Array => \|arrayInfo\| {
1388		return error{ UnexpectedEndOfJson, UnexpectedToken, LengthMismatch } \|\| TokenStream.Error \|\|
1389		UnescapeValidStringError \|\|
1390		ParseInternalErrorImpl(arrayInfo.child, inferred_types ++ [_]type{T});
1391		},
1392		.Vector => \|vecInfo\| {
1393		return error{ UnexpectedEndOfJson, UnexpectedToken, LengthMismatch } \|\| TokenStream.Error \|\|
1394		UnescapeValidStringError \|\|
1395		ParseInternalErrorImpl(vecInfo.child, inferred_types ++ [_]type{T});
1396		},
1397		.Pointer => \|ptrInfo\| {
1398		var errors = error{AllocatorRequired} \|\| std.mem.Allocator.Error;
1399		switch (ptrInfo.size) {
1400		.One => {
1401		return errors \|\| ParseInternalErrorImpl(ptrInfo.child, inferred_types ++ [_]type{T});
1402		},
1403		.Slice => {
1404		return errors \|\| error{ UnexpectedEndOfJson, UnexpectedToken } \|\|
1405		ParseInternalErrorImpl(ptrInfo.child, inferred_types ++ [_]type{T}) \|\|
1406		UnescapeValidStringError \|\| TokenStream.Error;
1407		},
1408		else => @compileError("Unable to parse into type '" ++ @typeName(T) ++ "'"),
1409		}
1410		},
1411		else => return error{},
1412		}
1413		unreachable;
1414		}
1415
1416		fn parseInternalArray(
1417		comptime T: type,
1418		comptime Elt: type,
1419		comptime arr_len: usize,
1420		tokens: *TokenStream,
1421		options: ParseOptions,
1422		) ParseInternalError(T)!T {
1423		var r: T = undefined;
1424		var i: usize = 0;
1425		var child_options = options;
1426		child_options.allow_trailing_data = true;
1427		errdefer {
1428		// Without the r.len check `r[i]` is not allowed
1429		if (arr_len > 0) while (true) : (i -= 1) {
1430		parseFree(Elt, r[i], options);
1431		if (i == 0) break;
1432		};
1433		}
1434		if (arr_len > 0) while (i < arr_len) : (i += 1) {
1435		r[i] = try parse(Elt, tokens, child_options);
1436		};
1437		const tok = (try tokens.next()) orelse return error.UnexpectedEndOfJson;
1438		switch (tok) {
1439		.ArrayEnd => {},
1440		else => return error.UnexpectedToken,
1441		}
1442		return r;
1443		}
1444
1445		fn parseInternal(
1446		comptime T: type,
1447		token: Token,
1448		tokens: *TokenStream,
1449		options: ParseOptions,
1450		) ParseInternalError(T)!T {
1451		switch (@typeInfo(T)) {
1452		.Bool => {
1453		return switch (token) {
1454		.True => true,
1455		.False => false,
1456		else => error.UnexpectedToken,
1457		};
1458		},
1459		.Float, .ComptimeFloat => {
1460		switch (token) {
1461		.Number => \|numberToken\| return try std.fmt.parseFloat(T, numberToken.slice(tokens.slice, tokens.i - 1)),
1462		.String => \|stringToken\| return try std.fmt.parseFloat(T, stringToken.slice(tokens.slice, tokens.i - 1)),
1463		else => return error.UnexpectedToken,
1464		}
1465		},
1466		.Int, .ComptimeInt => {
1467		switch (token) {
1468		.Number => \|numberToken\| {
1469		if (numberToken.is_integer)
1470		return try std.fmt.parseInt(T, numberToken.slice(tokens.slice, tokens.i - 1), 10);
1471		const float = try std.fmt.parseFloat(f128, numberToken.slice(tokens.slice, tokens.i - 1));
1472		if (@round(float) != float) return error.InvalidNumber;
1473		if (float > std.math.maxInt(T) or float < std.math.minInt(T)) return error.Overflow;
1474		return @floatToInt(T, float);
1475		},
1476		.String => \|stringToken\| {
1477		return std.fmt.parseInt(T, stringToken.slice(tokens.slice, tokens.i - 1), 10) catch \|err\| {
1478		switch (err) {
1479		error.Overflow => return err,
1480		error.InvalidCharacter => {
1481		const float = try std.fmt.parseFloat(f128, stringToken.slice(tokens.slice, tokens.i - 1));
1482		if (@round(float) != float) return error.InvalidNumber;
1483		if (float > std.math.maxInt(T) or float < std.math.minInt(T)) return error.Overflow;
1484		return @floatToInt(T, float);
1485		},
1486		}
1487		};
1488		},
1489		else => return error.UnexpectedToken,
1490		}
1491		},
1492		.Optional => \|optionalInfo\| {
1493		if (token == .Null) {
1494		return null;
1495		} else {
1496		return try parseInternal(optionalInfo.child, token, tokens, options);
1497		}
1498		},
1499		.Enum => \|enumInfo\| {
1500		switch (token) {
1501		.Number => \|numberToken\| {
1502		if (!numberToken.is_integer) return error.UnexpectedToken;
1503		const n = try std.fmt.parseInt(enumInfo.tag_type, numberToken.slice(tokens.slice, tokens.i - 1), 10);
1504		return try std.meta.intToEnum(T, n);
1505		},
1506		.String => \|stringToken\| {
1507		const source_slice = stringToken.slice(tokens.slice, tokens.i - 1);
1508		switch (stringToken.escapes) {
1509		.None => return std.meta.stringToEnum(T, source_slice) orelse return error.InvalidEnumTag,
1510		.Some => {
1511		inline for (enumInfo.fields) \|field\| {
1512		if (field.name.len == stringToken.decodedLength() and encodesTo(field.name, source_slice)) {
1513		return @field(T, field.name);
1514		}
1515		}
1516		return error.InvalidEnumTag;
1517		},
1518		}
1519		},
1520		else => return error.UnexpectedToken,
1521		}
1522		},
1523		.Union => \|unionInfo\| {
1524		if (unionInfo.tag_type) \|_\| {
1525		// try each of the union fields until we find one that matches
1526		inline for (unionInfo.fields) \|u_field\| {
1527		// take a copy of tokens so we can withhold mutations until success
1528		var tokens_copy = tokens.*;
1529		if (parseInternal(u_field.type, token, &tokens_copy, options)) \|value\| {
1530		tokens.* = tokens_copy;
1531		return @unionInit(T, u_field.name, value);
1532		} else \|err\| {
1533		// Bubble up error.OutOfMemory
1534		// Parsing some types won't have OutOfMemory in their
1535		// error-sets, for the condition to be valid, merge it in.
1536		if (@as(@TypeOf(err) \|\| error{OutOfMemory}, err) == error.OutOfMemory) return err;
1537		// Bubble up AllocatorRequired, as it indicates missing option
1538		if (@as(@TypeOf(err) \|\| error{AllocatorRequired}, err) == error.AllocatorRequired) return err;
1539		// otherwise continue through the `inline for`
1540		}
1541		}
1542		return error.NoUnionMembersMatched;
1543		} else {
1544		@compileError("Unable to parse into untagged union '" ++ @typeName(T) ++ "'");
1545		}
1546		},
1547		.Struct => \|structInfo\| {
1548		if (structInfo.is_tuple) {
1549		switch (token) {
1550		.ArrayBegin => {},
1551		else => return error.UnexpectedToken,
1552		}
1553		var r: T = undefined;
1554		var child_options = options;
1555		child_options.allow_trailing_data = true;
1556		var fields_seen: usize = 0;
1557		errdefer {
1558		inline for (0..structInfo.fields.len) \|i\| {
1559		if (i < fields_seen) {
1560		parseFree(structInfo.fields[i].type, r[i], options);
1561		}
1562		}
1563		}
1564		inline for (0..structInfo.fields.len) \|i\| {
1565		r[i] = try parse(structInfo.fields[i].type, tokens, child_options);
1566		fields_seen = i + 1;
1567		}
1568		const tok = (try tokens.next()) orelse return error.UnexpectedEndOfJson;
1569		switch (tok) {
1570		.ArrayEnd => {},
1571		else => return error.UnexpectedToken,
1572		}
1573		return r;
1574		}
1575
1576		switch (token) {
1577		.ObjectBegin => {},
1578		else => return error.UnexpectedToken,
1579		}
1580		var r: T = undefined;
1581		var fields_seen = [_]bool{false} ** structInfo.fields.len;
1582		errdefer {
1583		inline for (structInfo.fields, 0..) \|field, i\| {
1584		if (fields_seen[i] and !field.is_comptime) {
1585		parseFree(field.type, @field(r, field.name), options);
1586		}
1587		}
1588		}
1589
1590		while (true) {
1591		switch ((try tokens.next()) orelse return error.UnexpectedEndOfJson) {
1592		.ObjectEnd => break,
1593		.String => \|stringToken\| {
1594		const key_source_slice = stringToken.slice(tokens.slice, tokens.i - 1);
1595		var child_options = options;
1596		child_options.allow_trailing_data = true;
1597		var found = false;
1598		inline for (structInfo.fields, 0..) \|field, i\| {
1599		if (switch (stringToken.escapes) {
1600		.None => mem.eql(u8, field.name, key_source_slice),
1601		.Some => (field.name.len == stringToken.decodedLength() and encodesTo(field.name, key_source_slice)),
1602		}) {
1603		if (fields_seen[i]) {
1604		switch (options.duplicate_field_behavior) {
1605		.UseFirst => {
1606		// unconditionally ignore value. for comptime fields, this skips check against default_value
1607		parseFree(field.type, try parse(field.type, tokens, child_options), child_options);
1608		found = true;
1609		break;
1610		},
1611		.Error => return error.DuplicateJSONField,
1612		.UseLast => {
1613		if (!field.is_comptime) {
1614		parseFree(field.type, @field(r, field.name), child_options);
1615		}
1616		fields_seen[i] = false;
1617		},
1618		}
1619		}
1620		if (field.is_comptime) {
1621		if (!try parsesTo(field.type, @ptrCast(align(1) const field.type, field.default_value.?)., tokens, child_options)) {
1622		return error.UnexpectedValue;
1623		}
1624		} else {
1625		@field(r, field.name) = try parse(field.type, tokens, child_options);
1626		}
1627		fields_seen[i] = true;
1628		found = true;
1629		break;
1630		}
1631		}
1632		if (!found) {
1633		if (options.ignore_unknown_fields) {
1634		try skipValue(tokens);
1635		continue;
1636		} else {
1637		return error.UnknownField;
1638		}
1639		}
1640		},
1641		else => return error.UnexpectedToken,
1642		}
1643		}
1644		inline for (structInfo.fields, 0..) \|field, i\| {
1645		if (!fields_seen[i]) {
1646		if (field.default_value) \|default_ptr\| {
1647		if (!field.is_comptime) {
1648		const default = @ptrCast(align(1) const field.type, default_ptr).;
1649		@field(r, field.name) = default;
1650		}
1651		} else {
1652		return error.MissingField;
1653		}
1654		}
1655		}
1656		return r;
1657		},
1658		.Array => \|arrayInfo\| {
1659		switch (token) {
1660		.ArrayBegin => {
1661		const len = @typeInfo(T).Array.len;
1662		return parseInternalArray(T, arrayInfo.child, len, tokens, options);
1663		},
1664		.String => \|stringToken\| {
1665		if (arrayInfo.child != u8) return error.UnexpectedToken;
1666		var r: T = undefined;
1667		const source_slice = stringToken.slice(tokens.slice, tokens.i - 1);
1668		if (r.len != stringToken.decodedLength()) return error.LengthMismatch;
1669		switch (stringToken.escapes) {
1670		.None => @memcpy(r[0..source_slice.len], source_slice),
1671		.Some => try unescapeValidString(&r, source_slice),
1672		}
1673		return r;
1674		},
1675		else => return error.UnexpectedToken,
1676		}
1677		},
1678		.Vector => \|vecInfo\| {
1679		switch (token) {
1680		.ArrayBegin => {
1681		const len = @typeInfo(T).Vector.len;
1682		return parseInternalArray(T, vecInfo.child, len, tokens, options);
1683		},
1684		else => return error.UnexpectedToken,
1685		}
1686		},
1687		.Pointer => \|ptrInfo\| {
1688		const allocator = options.allocator orelse return error.AllocatorRequired;
1689		switch (ptrInfo.size) {
1690		.One => {
1691		const r: *ptrInfo.child = try allocator.create(ptrInfo.child);
1692		errdefer allocator.destroy(r);
1693		r.* = try parseInternal(ptrInfo.child, token, tokens, options);
1694		return r;
1695		},
1696		.Slice => {
1697		switch (token) {
1698		.ArrayBegin => {
1699		var arraylist = std.ArrayList(ptrInfo.child).init(allocator);
1700		errdefer {
1701		while (arraylist.popOrNull()) \|v\| {
1702		parseFree(ptrInfo.child, v, options);
1703		}
1704		arraylist.deinit();
1705		}
1706
1707		while (true) {
1708		const tok = (try tokens.next()) orelse return error.UnexpectedEndOfJson;
1709		switch (tok) {
1710		.ArrayEnd => break,
1711		else => {},
1712		}
1713
1714		try arraylist.ensureUnusedCapacity(1);
1715		const v = try parseInternal(ptrInfo.child, tok, tokens, options);
1716		arraylist.appendAssumeCapacity(v);
1717		}
1718
1719		if (ptrInfo.sentinel) \|some\| {
1720		const sentinel_value = @ptrCast(align(1) const ptrInfo.child, some).;
1721		return try arraylist.toOwnedSliceSentinel(sentinel_value);
1722		}
1723
1724		return try arraylist.toOwnedSlice();
1725		},
1726		.String => \|stringToken\| {
1727		if (ptrInfo.child != u8) return error.UnexpectedToken;
1728		const source_slice = stringToken.slice(tokens.slice, tokens.i - 1);
1729		const len = stringToken.decodedLength();
1730		const output = if (ptrInfo.sentinel) \|sentinel_ptr\|
1731		try allocator.allocSentinel(u8, len, @ptrCast(const u8, sentinel_ptr).)
1732		else
1733		try allocator.alloc(u8, len);
1734		errdefer allocator.free(output);
1735		switch (stringToken.escapes) {
1736		.None => @memcpy(output[0..source_slice.len], source_slice),
1737		.Some => try unescapeValidString(output, source_slice),
1738		}
1739
1740		return output;
1741		},
1742		else => return error.UnexpectedToken,
1743		}
1744		},
1745		else => @compileError("Unable to parse into type '" ++ @typeName(T) ++ "'"),
1746		}
1747		},
1748		else => @compileError("Unable to parse into type '" ++ @typeName(T) ++ "'"),
1749		}
1750		unreachable;
1751		}
1752
1753		pub fn ParseError(comptime T: type) type {
1754		return ParseInternalError(T) \|\| error{UnexpectedEndOfJson} \|\| TokenStream.Error;
1755		}
1756
1757		pub fn parse(comptime T: type, tokens: *TokenStream, options: ParseOptions) ParseError(T)!T {
1758		const token = (try tokens.next()) orelse return error.UnexpectedEndOfJson;
1759		const r = try parseInternal(T, token, tokens, options);
1760		errdefer parseFree(T, r, options);
1761		if (!options.allow_trailing_data) {
1762		if ((try tokens.next()) != null) unreachable;
1763		assert(tokens.i >= tokens.slice.len);
1764		}
1765		return r;
1766		}
1767
1768		/// Releases resources created by `parse`.
1769		/// Should be called with the same type and `ParseOptions` that were passed to `parse`
1770		pub fn parseFree(comptime T: type, value: T, options: ParseOptions) void {
1771		switch (@typeInfo(T)) {
1772		.Bool, .Float, .ComptimeFloat, .Int, .ComptimeInt, .Enum => {},
1773		.Optional => {
1774		if (value) \|v\| {
1775		return parseFree(@TypeOf(v), v, options);
1776		}
1777		},
1778		.Union => \|unionInfo\| {
1779		if (unionInfo.tag_type) \|UnionTagType\| {
1780		inline for (unionInfo.fields) \|u_field\| {
1781		if (value == @field(UnionTagType, u_field.name)) {
1782		parseFree(u_field.type, @field(value, u_field.name), options);
1783		break;
1784		}
1785		}
1786		} else {
1787		unreachable;
1788		}
1789		},
1790		.Struct => \|structInfo\| {
1791		inline for (structInfo.fields) \|field\| {
1792		if (!field.is_comptime) {
1793		var should_free = true;
1794		if (field.default_value) \|default\| {
1795		switch (@typeInfo(field.type)) {
1796		// We must not attempt to free pointers to struct default values
1797		.Pointer => \|fieldPtrInfo\| {
1798		const field_value = @field(value, field.name);
1799		const field_ptr = switch (fieldPtrInfo.size) {
1800		.One => field_value,
1801		.Slice => field_value.ptr,
1802		else => unreachable, // Other pointer types are not parseable
1803		};
1804		const field_addr = @ptrToInt(field_ptr);
1805
1806		const casted_default = @ptrCast(const field.type, @alignCast(@alignOf(field.type), default)).;
1807		const default_ptr = switch (fieldPtrInfo.size) {
1808		.One => casted_default,
1809		.Slice => casted_default.ptr,
1810		else => unreachable, // Other pointer types are not parseable
1811		};
1812		const default_addr = @ptrToInt(default_ptr);
1813
1814		if (field_addr == default_addr) {
1815		should_free = false;
1816		}
1817		},
1818		else => {},
1819		}
1820		}
1821		if (should_free) {
1822		parseFree(field.type, @field(value, field.name), options);
1823		}
1824		}
1825		}
1826		},
1827		.Array => \|arrayInfo\| {
1828		for (value) \|v\| {
1829		parseFree(arrayInfo.child, v, options);
1830		}
1831		},
1832		.Vector => \|vecInfo\| {
1833		var i: usize = 0;
1834		var v_len: usize = @typeInfo(@TypeOf(value)).Vector.len;
1835		while (i < v_len) : (i += 1) {
1836		parseFree(vecInfo.child, value[i], options);
1837		}
1838		},
1839		.Pointer => \|ptrInfo\| {
1840		const allocator = options.allocator orelse unreachable;
1841		switch (ptrInfo.size) {
1842		.One => {
1843		parseFree(ptrInfo.child, value.*, options);
1844		allocator.destroy(value);
1845		},
1846		.Slice => {
1847		for (value) \|v\| {
1848		parseFree(ptrInfo.child, v, options);
1849		}
1850		allocator.free(value);
1851		},
1852		else => unreachable,
1853		}
1854		},
1855		else => unreachable,
1856		}
1857		}
1858
1859		/// A non-stream JSON parser which constructs a tree of Value's.
1860		pub const Parser = struct {
1861		allocator: Allocator,
1862		state: State,
1863		copy_strings: bool,
1864		// Stores parent nodes and un-combined Values.
1865		stack: Array,
1866
1867		const State = enum {
1868		ObjectKey,
1869		ObjectValue,
1870		ArrayValue,
1871		Simple,
1872		};
1873
1874		pub fn init(allocator: Allocator, copy_strings: bool) Parser {
1875		return Parser{
1876		.allocator = allocator,
1877		.state = .Simple,
1878		.copy_strings = copy_strings,
1879		.stack = Array.init(allocator),
1880		};
1881		}
1882
1883		pub fn deinit(p: *Parser) void {
1884		p.stack.deinit();
1885		}
1886
1887		pub fn reset(p: *Parser) void {
1888		p.state = .Simple;
1889		p.stack.shrinkRetainingCapacity(0);
1890		}
1891
1892		pub fn parse(p: *Parser, input: []const u8) !ValueTree {
1893		var s = TokenStream.init(input);
1894
1895		var arena = try p.allocator.create(ArenaAllocator);
1896		errdefer p.allocator.destroy(arena);
1897
1898		arena.* = ArenaAllocator.init(p.allocator);
1899		errdefer arena.deinit();
1900
1901		const allocator = arena.allocator();
1902
1903		while (try s.next()) \|token\| {
1904		try p.transition(allocator, input, s.i - 1, token);
1905		}
1906
1907		debug.assert(p.stack.items.len == 1);
1908
1909		return ValueTree{
1910		.arena = arena,
1911		.root = p.stack.items[0],
1912		};
1913		}
1914
1915		// Even though p.allocator exists, we take an explicit allocator so that allocation state
1916		// can be cleaned up on error correctly during a `parse` on call.
1917		fn transition(p: *Parser, allocator: Allocator, input: []const u8, i: usize, token: Token) !void {
1918		switch (p.state) {
1919		.ObjectKey => switch (token) {
1920		.ObjectEnd => {
1921		if (p.stack.items.len == 1) {
1922		return;
1923		}
1924
1925		var value = p.stack.pop();
1926		try p.pushToParent(&value);
1927		},
1928		.String => \|s\| {
1929		try p.stack.append(try p.parseString(allocator, s, input, i));
1930		p.state = .ObjectValue;
1931		},
1932		else => {
1933		// The streaming parser would return an error eventually.
1934		// To prevent invalid state we return an error now.
1935		// TODO make the streaming parser return an error as soon as it encounters an invalid object key
1936		return error.InvalidLiteral;
1937		},
1938		},
1939		.ObjectValue => {
1940		var object = &p.stack.items[p.stack.items.len - 2].Object;
1941		var key = p.stack.items[p.stack.items.len - 1].String;
1942
1943		switch (token) {
1944		.ObjectBegin => {
1945		try p.stack.append(Value{ .Object = ObjectMap.init(allocator) });
1946		p.state = .ObjectKey;
1947		},
1948		.ArrayBegin => {
1949		try p.stack.append(Value{ .Array = Array.init(allocator) });
1950		p.state = .ArrayValue;
1951		},
1952		.String => \|s\| {
1953		try object.put(key, try p.parseString(allocator, s, input, i));
1954		_ = p.stack.pop();
1955		p.state = .ObjectKey;
1956		},
1957		.Number => \|n\| {
1958		try object.put(key, try p.parseNumber(n, input, i));
1959		_ = p.stack.pop();
1960		p.state = .ObjectKey;
1961		},
1962		.True => {
1963		try object.put(key, Value{ .Bool = true });
1964		_ = p.stack.pop();
1965		p.state = .ObjectKey;
1966		},
1967		.False => {
1968		try object.put(key, Value{ .Bool = false });
1969		_ = p.stack.pop();
1970		p.state = .ObjectKey;
1971		},
1972		.Null => {
1973		try object.put(key, Value.Null);
1974		_ = p.stack.pop();
1975		p.state = .ObjectKey;
1976		},
1977		.ObjectEnd, .ArrayEnd => {
1978		unreachable;
1979		},
1980		}
1981		},
1982		.ArrayValue => {
1983		var array = &p.stack.items[p.stack.items.len - 1].Array;
1984
1985		switch (token) {
1986		.ArrayEnd => {
1987		if (p.stack.items.len == 1) {
1988		return;
1989		}
1990
1991		var value = p.stack.pop();
1992		try p.pushToParent(&value);
1993		},
1994		.ObjectBegin => {
1995		try p.stack.append(Value{ .Object = ObjectMap.init(allocator) });
1996		p.state = .ObjectKey;
1997		},
1998		.ArrayBegin => {
1999		try p.stack.append(Value{ .Array = Array.init(allocator) });
2000		p.state = .ArrayValue;
2001		},
2002		.String => \|s\| {
2003		try array.append(try p.parseString(allocator, s, input, i));
2004		},
2005		.Number => \|n\| {
2006		try array.append(try p.parseNumber(n, input, i));
2007		},
2008		.True => {
2009		try array.append(Value{ .Bool = true });
2010		},
2011		.False => {
2012		try array.append(Value{ .Bool = false });
2013		},
2014		.Null => {
2015		try array.append(Value.Null);
2016		},
2017		.ObjectEnd => {
2018		unreachable;
2019		},
2020		}
2021		},
2022		.Simple => switch (token) {
2023		.ObjectBegin => {
2024		try p.stack.append(Value{ .Object = ObjectMap.init(allocator) });
2025		p.state = .ObjectKey;
2026		},
2027		.ArrayBegin => {
2028		try p.stack.append(Value{ .Array = Array.init(allocator) });
2029		p.state = .ArrayValue;
2030		},
2031		.String => \|s\| {
2032		try p.stack.append(try p.parseString(allocator, s, input, i));
2033		},
2034		.Number => \|n\| {
2035		try p.stack.append(try p.parseNumber(n, input, i));
2036		},
2037		.True => {
2038		try p.stack.append(Value{ .Bool = true });
2039		},
2040		.False => {
2041		try p.stack.append(Value{ .Bool = false });
2042		},
2043		.Null => {
2044		try p.stack.append(Value.Null);
2045		},
2046		.ObjectEnd, .ArrayEnd => {
2047		unreachable;
2048		},
2049		},
2050		}
2051		}
2052
2053		fn pushToParent(p: Parser, value: const Value) !void {
2054		switch (p.stack.items[p.stack.items.len - 1]) {
2055		// Object Parent -> [ ..., object, <key>, value ]
2056		Value.String => \|key\| {
2057		_ = p.stack.pop();
2058
2059		var object = &p.stack.items[p.stack.items.len - 1].Object;
2060		try object.put(key, value.*);
2061		p.state = .ObjectKey;
2062		},
2063		// Array Parent -> [ ..., <array>, value ]
2064		Value.Array => \|*array\| {
2065		try array.append(value.*);
2066		p.state = .ArrayValue;
2067		},
2068		else => {
2069		unreachable;
2070		},
2071		}
2072		}
2073
2074		fn parseString(p: *Parser, allocator: Allocator, s: std.meta.TagPayload(Token, Token.String), input: []const u8, i: usize) !Value {
2075		const slice = s.slice(input, i);
2076		switch (s.escapes) {
2077		.None => return Value{ .String = if (p.copy_strings) try allocator.dupe(u8, slice) else slice },
2078		.Some => {
2079		const output = try allocator.alloc(u8, s.decodedLength());
2080		errdefer allocator.free(output);
2081		try unescapeValidString(output, slice);
2082		return Value{ .String = output };
2083		},
2084		}
2085		}
2086
2087		fn parseNumber(p: *Parser, n: std.meta.TagPayload(Token, Token.Number), input: []const u8, i: usize) !Value {
2088		_ = p;
2089		return if (n.is_integer)
2090		Value{
2091		.Integer = std.fmt.parseInt(i64, n.slice(input, i), 10) catch \|e\| switch (e) {
2092		error.Overflow => return Value{ .NumberString = n.slice(input, i) },
2093		error.InvalidCharacter => \|err\| return err,
2094		},
2095		}
2096		else
2097		Value{ .Float = try std.fmt.parseFloat(f64, n.slice(input, i)) };
2098		}
2099		};
2100
2101		pub const UnescapeValidStringError = error{InvalidUnicodeHexSymbol};
2102
2103		/// Unescape a JSON string
2104		/// Only to be used on strings already validated by the parser
2105		/// (note the unreachable statements and lack of bounds checking)
2106		pub fn unescapeValidString(output: []u8, input: []const u8) UnescapeValidStringError!void {
2107		var inIndex: usize = 0;
2108		var outIndex: usize = 0;
2109
2110		while (inIndex < input.len) {
2111		if (input[inIndex] != '\\') {
2112		// not an escape sequence
2113		output[outIndex] = input[inIndex];
2114		inIndex += 1;
2115		outIndex += 1;
2116		} else if (input[inIndex + 1] != 'u') {
2117		// a simple escape sequence
2118		output[outIndex] = @as(u8, switch (input[inIndex + 1]) {
2119		'\\' => '\\',
2120		'/' => '/',
2121		'n' => '\n',
2122		'r' => '\r',
2123		't' => '\t',
2124		'f' => 12,
2125		'b' => 8,
2126		'"' => '"',
2127		else => unreachable,
2128		});
2129		inIndex += 2;
2130		outIndex += 1;
2131		} else {
2132		// a unicode escape sequence
2133		const firstCodeUnit = std.fmt.parseInt(u16, input[inIndex + 2 .. inIndex + 6], 16) catch unreachable;
2134
2135		// guess optimistically that it's not a surrogate pair
2136		if (std.unicode.utf8Encode(firstCodeUnit, output[outIndex..])) \|byteCount\| {
2137		outIndex += byteCount;
2138		inIndex += 6;
2139		} else \|err\| {
2140		// it might be a surrogate pair
2141		if (err != error.Utf8CannotEncodeSurrogateHalf) {
2142		return error.InvalidUnicodeHexSymbol;
2143		}
2144		// check if a second code unit is present
2145		if (inIndex + 7 >= input.len or input[inIndex + 6] != '\\' or input[inIndex + 7] != 'u') {
2146		return error.InvalidUnicodeHexSymbol;
2147		}
2148
2149		const secondCodeUnit = std.fmt.parseInt(u16, input[inIndex + 8 .. inIndex + 12], 16) catch unreachable;
2150
2151		const utf16le_seq = [2]u16{
2152		mem.nativeToLittle(u16, firstCodeUnit),
2153		mem.nativeToLittle(u16, secondCodeUnit),
2154		};
2155		if (std.unicode.utf16leToUtf8(output[outIndex..], &utf16le_seq)) \|byteCount\| {
2156		outIndex += byteCount;
2157		inIndex += 12;
2158		} else \|_\| {
2159		return error.InvalidUnicodeHexSymbol;
2160		}
2161		}
2162		}
2163		}
2164		assert(outIndex == output.len);
2165		}
2166
2167		pub const StringifyOptions = struct {
2168		pub const Whitespace = struct {
2169		/// How many indentation levels deep are we?
2170		indent_level: usize = 0,
2171
2172		/// What character(s) should be used for indentation?
2173		indent: union(enum) {
2174		Space: u8,
2175		Tab: void,
2176		None: void,
2177		} = .{ .Space = 4 },
2178
2179		/// After a colon, should whitespace be inserted?
2180		separator: bool = true,
2181
2182		pub fn outputIndent(
2183		whitespace: @This(),
2184		out_stream: anytype,
2185		) @TypeOf(out_stream).Error!void {
2186		var char: u8 = undefined;
2187		var n_chars: usize = undefined;
2188		switch (whitespace.indent) {
2189		.Space => \|n_spaces\| {
2190		char = ' ';
2191		n_chars = n_spaces;
2192		},
2193		.Tab => {
2194		char = '\t';
2195		n_chars = 1;
2196		},
2197		.None => return,
2198		}
2199		try out_stream.writeByte('\n');
2200		n_chars *= whitespace.indent_level;
2201		try out_stream.writeByteNTimes(char, n_chars);
2202		}
2203		};
2204
2205		/// Controls the whitespace emitted
2206		whitespace: ?Whitespace = null,
2207
2208		/// Should optional fields with null value be written?
2209		emit_null_optional_fields: bool = true,
2210
2211		string: StringOptions = StringOptions{ .String = .{} },
2212
2213		/// Should []u8 be serialised as a string? or an array?
2214		pub const StringOptions = union(enum) {
2215		Array,
2216		String: StringOutputOptions,
2217
2218		/// String output options
2219		const StringOutputOptions = struct {
2220		/// Should '/' be escaped in strings?
2221		escape_solidus: bool = false,
2222
2223		/// Should unicode characters be escaped in strings?
2224		escape_unicode: bool = false,
2225		};
2226		};
2227		};
2228
2229		fn outputUnicodeEscape(
2230		codepoint: u21,
2231		out_stream: anytype,
2232		) !void {
2233		if (codepoint <= 0xFFFF) {
2234		// If the character is in the Basic Multilingual Plane (U+0000 through U+FFFF),
2235		// then it may be represented as a six-character sequence: a reverse solidus, followed
2236		// by the lowercase letter u, followed by four hexadecimal digits that encode the character's code point.
2237		try out_stream.writeAll("\\u");
2238		try std.fmt.formatIntValue(codepoint, "x", std.fmt.FormatOptions{ .width = 4, .fill = '0' }, out_stream);
2239		} else {
2240		assert(codepoint <= 0x10FFFF);
2241		// To escape an extended character that is not in the Basic Multilingual Plane,
2242		// the character is represented as a 12-character sequence, encoding the UTF-16 surrogate pair.
2243		const high = @intCast(u16, (codepoint - 0x10000) >> 10) + 0xD800;
2244		const low = @intCast(u16, codepoint & 0x3FF) + 0xDC00;
2245		try out_stream.writeAll("\\u");
2246		try std.fmt.formatIntValue(high, "x", std.fmt.FormatOptions{ .width = 4, .fill = '0' }, out_stream);
2247		try out_stream.writeAll("\\u");
2248		try std.fmt.formatIntValue(low, "x", std.fmt.FormatOptions{ .width = 4, .fill = '0' }, out_stream);
2249		}
2250		}
2251
2252		/// Write `string` to `writer` as a JSON encoded string.
2253		pub fn encodeJsonString(string: []const u8, options: StringifyOptions, writer: anytype) !void {
2254		try writer.writeByte('\"');
2255		try encodeJsonStringChars(string, options, writer);
2256		try writer.writeByte('\"');
2257		}
2258
2259		/// Write `chars` to `writer` as JSON encoded string characters.
2260		pub fn encodeJsonStringChars(chars: []const u8, options: StringifyOptions, writer: anytype) !void {
2261		var i: usize = 0;
2262		while (i < chars.len) : (i += 1) {
2263		switch (chars[i]) {
2264		// normal ascii character
2265		0x20...0x21, 0x23...0x2E, 0x30...0x5B, 0x5D...0x7F => \|c\| try writer.writeByte(c),
2266		// only 2 characters that must be escaped
2267		'\\' => try writer.writeAll("\\\\"),
2268		'\"' => try writer.writeAll("\\\""),
2269		// solidus is optional to escape
2270		'/' => {
2271		if (options.string.String.escape_solidus) {
2272		try writer.writeAll("\\/");
2273		} else {
2274		try writer.writeByte('/');
2275		}
2276		},
2277		// control characters with short escapes
2278		// TODO: option to switch between unicode and 'short' forms?
2279		0x8 => try writer.writeAll("\\b"),
2280		0xC => try writer.writeAll("\\f"),
2281		'\n' => try writer.writeAll("\\n"),
2282		'\r' => try writer.writeAll("\\r"),
2283		'\t' => try writer.writeAll("\\t"),
2284		else => {
2285		const ulen = std.unicode.utf8ByteSequenceLength(chars[i]) catch unreachable;
2286		// control characters (only things left with 1 byte length) should always be printed as unicode escapes
2287		if (ulen == 1 or options.string.String.escape_unicode) {
2288		const codepoint = std.unicode.utf8Decode(chars[i..][0..ulen]) catch unreachable;
2289		try outputUnicodeEscape(codepoint, writer);
2290		} else {
2291		try writer.writeAll(chars[i..][0..ulen]);
2292		}
2293		i += ulen - 1;
2294		},
2295		}
2296		}
2297		}
2298
2299		pub fn stringify(
2300		value: anytype,
2301		options: StringifyOptions,
2302		out_stream: anytype,
2303		) !void {
2304		const T = @TypeOf(value);
2305		switch (@typeInfo(T)) {
2306		.Float, .ComptimeFloat => {
2307		return std.fmt.formatFloatScientific(value, std.fmt.FormatOptions{}, out_stream);
2308		},
2309		.Int, .ComptimeInt => {
2310		return std.fmt.formatIntValue(value, "", std.fmt.FormatOptions{}, out_stream);
2311		},
2312		.Bool => {
2313		return out_stream.writeAll(if (value) "true" else "false");
2314		},
2315		.Null => {
2316		return out_stream.writeAll("null");
2317		},
2318		.Optional => {
2319		if (value) \|payload\| {
2320		return try stringify(payload, options, out_stream);
2321		} else {
2322		return try stringify(null, options, out_stream);
2323		}
2324		},
2325		.Enum => {
2326		if (comptime std.meta.trait.hasFn("jsonStringify")(T)) {
2327		return value.jsonStringify(options, out_stream);
2328		}
2329
2330		@compileError("Unable to stringify enum '" ++ @typeName(T) ++ "'");
2331		},
2332		.Union => {
2333		if (comptime std.meta.trait.hasFn("jsonStringify")(T)) {
2334		return value.jsonStringify(options, out_stream);
2335		}
2336
2337		const info = @typeInfo(T).Union;
2338		if (info.tag_type) \|UnionTagType\| {
2339		inline for (info.fields) \|u_field\| {
2340		if (value == @field(UnionTagType, u_field.name)) {
2341		return try stringify(@field(value, u_field.name), options, out_stream);
2342		}
2343		}
2344		} else {
2345		@compileError("Unable to stringify untagged union '" ++ @typeName(T) ++ "'");
2346		}
2347		},
2348		.Struct => \|S\| {
2349		if (comptime std.meta.trait.hasFn("jsonStringify")(T)) {
2350		return value.jsonStringify(options, out_stream);
2351		}
2352
2353		try out_stream.writeByte(if (S.is_tuple) '[' else '{');
2354		var field_output = false;
2355		var child_options = options;
2356		if (child_options.whitespace) \|*child_whitespace\| {
2357		child_whitespace.indent_level += 1;
2358		}
2359		inline for (S.fields) \|Field\| {
2360		// don't include void fields
2361		if (Field.type == void) continue;
2362
2363		var emit_field = true;
2364
2365		// don't include optional fields that are null when emit_null_optional_fields is set to false
2366		if (@typeInfo(Field.type) == .Optional) {
2367		if (options.emit_null_optional_fields == false) {
2368		if (@field(value, Field.name) == null) {
2369		emit_field = false;
2370		}
2371		}
2372		}
2373
2374		if (emit_field) {
2375		if (!field_output) {
2376		field_output = true;
2377		} else {
2378		try out_stream.writeByte(',');
2379		}
2380		if (child_options.whitespace) \|child_whitespace\| {
2381		try child_whitespace.outputIndent(out_stream);
2382		}
2383		if (!S.is_tuple) {
2384		try encodeJsonString(Field.name, options, out_stream);
2385		try out_stream.writeByte(':');
2386		if (child_options.whitespace) \|child_whitespace\| {
2387		if (child_whitespace.separator) {
2388		try out_stream.writeByte(' ');
2389		}
2390		}
2391		}
2392		try stringify(@field(value, Field.name), child_options, out_stream);
2393		}
2394		}
2395		if (field_output) {
2396		if (options.whitespace) \|whitespace\| {
2397		try whitespace.outputIndent(out_stream);
2398		}
2399		}
2400		try out_stream.writeByte(if (S.is_tuple) ']' else '}');
2401		return;
2402		},
2403		.ErrorSet => return stringify(@as([]const u8, @errorName(value)), options, out_stream),
2404		.Pointer => \|ptr_info\| switch (ptr_info.size) {
2405		.One => switch (@typeInfo(ptr_info.child)) {
2406		.Array => {
2407		const Slice = []const std.meta.Elem(ptr_info.child);
2408		return stringify(@as(Slice, value), options, out_stream);
2409		},
2410		else => {
2411		// TODO: avoid loops?
2412		return stringify(value.*, options, out_stream);
2413		},
2414		},
2415		.Many, .Slice => {
2416		if (ptr_info.size == .Many and ptr_info.sentinel == null)
2417		@compileError("unable to stringify type '" ++ @typeName(T) ++ "' without sentinel");
2418		const slice = if (ptr_info.size == .Many) mem.span(value) else value;
2419
2420		if (ptr_info.child == u8 and options.string == .String and std.unicode.utf8ValidateSlice(slice)) {
2421		try encodeJsonString(slice, options, out_stream);
2422		return;
2423		}
2424
2425		try out_stream.writeByte('[');
2426		var child_options = options;
2427		if (child_options.whitespace) \|*whitespace\| {
2428		whitespace.indent_level += 1;
2429		}
2430		for (slice, 0..) \|x, i\| {
2431		if (i != 0) {
2432		try out_stream.writeByte(',');
2433		}
2434		if (child_options.whitespace) \|child_whitespace\| {
2435		try child_whitespace.outputIndent(out_stream);
2436		}
2437		try stringify(x, child_options, out_stream);
2438		}
2439		if (slice.len != 0) {
2440		if (options.whitespace) \|whitespace\| {
2441		try whitespace.outputIndent(out_stream);
2442		}
2443		}
2444		try out_stream.writeByte(']');
2445		return;
2446		},
2447		else => @compileError("Unable to stringify type '" ++ @typeName(T) ++ "'"),
2448		},
2449		.Array => return stringify(&value, options, out_stream),
2450		.Vector => \|info\| {
2451		const array: [info.len]info.child = value;
2452		return stringify(&array, options, out_stream);
2453		},
2454		else => @compileError("Unable to stringify type '" ++ @typeName(T) ++ "'"),
2455		}
2456		unreachable;
2457		}
2458
2459		// Same as `stringify` but accepts an Allocator and stores result in dynamically allocated memory instead of using a Writer.
2460		// Caller owns returned memory.
2461		pub fn stringifyAlloc(allocator: std.mem.Allocator, value: anytype, options: StringifyOptions) ![]const u8 {
2462		var list = std.ArrayList(u8).init(allocator);
2463		errdefer list.deinit();
2464		try stringify(value, options, list.writer());
2465		return list.toOwnedSlice();
2466		}
	46	// Deprecations
	47	pub const parse = @compileError("Deprecated; use parseFromSlice() or parseFromTokenSource() instead.");
	48	pub const StreamingParser = @compileError("Deprecated; use json.Scanner or json.Reader instead.");
	49	pub const TokenStream = @compileError("Deprecated; use json.Scanner or json.Reader instead.");
2467	50
2468	51	test {
2469	52	_ = @import("json/test.zig");
	53	_ = @import("json/scanner.zig");
2470	54	_ = @import("json/write_stream.zig");
2471		}
2472
2473		test "stringify null optional fields" {
2474		const MyStruct = struct {
2475		optional: ?[]const u8 = null,
2476		required: []const u8 = "something",
2477		another_optional: ?[]const u8 = null,
2478		another_required: []const u8 = "something else",
2479		};
2480		try teststringify(
2481		\\{"optional":null,"required":"something","another_optional":null,"another_required":"something else"}
2482		,
2483		MyStruct{},
2484		StringifyOptions{},
2485		);
2486		try teststringify(
2487		\\{"required":"something","another_required":"something else"}
2488		,
2489		MyStruct{},
2490		StringifyOptions{ .emit_null_optional_fields = false },
2491		);
2492
2493		var ts = TokenStream.init(
2494		\\{"required":"something","another_required":"something else"}
2495		);
2496		try std.testing.expect(try parsesTo(MyStruct, MyStruct{}, &ts, .{
2497		.allocator = std.testing.allocator,
2498		}));
2499		}
2500
2501		test "skipValue" {
2502		var ts = TokenStream.init("false");
2503		try skipValue(&ts);
2504		ts = TokenStream.init("true");
2505		try skipValue(&ts);
2506		ts = TokenStream.init("null");
2507		try skipValue(&ts);
2508		ts = TokenStream.init("42");
2509		try skipValue(&ts);
2510		ts = TokenStream.init("42.0");
2511		try skipValue(&ts);
2512		ts = TokenStream.init("\"foo\"");
2513		try skipValue(&ts);
2514		ts = TokenStream.init("[101, 111, 121]");
2515		try skipValue(&ts);
2516		ts = TokenStream.init("{}");
2517		try skipValue(&ts);
2518		ts = TokenStream.init("{\"foo\": \"bar\"}");
2519		try skipValue(&ts);
2520
2521		{ // An absurd number of nestings
2522		const nestings = StreamingParser.default_max_nestings + 1;
2523
2524		ts = TokenStream.init("[" nestings ++ "]" nestings);
2525		try testing.expectError(error.TooManyNestedItems, skipValue(&ts));
2526		}
2527
2528		{ // Would a number token cause problems in a deeply-nested array?
2529		const nestings = StreamingParser.default_max_nestings;
2530		const deeply_nested_array = "[" nestings ++ "0.118, 999, 881.99, 911.9, 725, 3" ++ "]" nestings;
2531
2532		ts = TokenStream.init(deeply_nested_array);
2533		try skipValue(&ts);
2534
2535		ts = TokenStream.init("[" ++ deeply_nested_array ++ "]");
2536		try testing.expectError(error.TooManyNestedItems, skipValue(&ts));
2537		}
2538
2539		// Mismatched brace/square bracket
2540		ts = TokenStream.init("[102, 111, 111}");
2541		try testing.expectError(error.UnexpectedClosingBrace, skipValue(&ts));
2542
2543		{ // should fail if no value found (e.g. immediate close of object)
2544		var empty_object = TokenStream.init("{}");
2545		assert(.ObjectBegin == (try empty_object.next()).?);
2546		try testing.expectError(error.UnexpectedJsonDepth, skipValue(&empty_object));
2547
2548		var empty_array = TokenStream.init("[]");
2549		assert(.ArrayBegin == (try empty_array.next()).?);
2550		try testing.expectError(error.UnexpectedJsonDepth, skipValue(&empty_array));
2551		}
2552		}
2553
2554		test "stringify basic types" {
2555		try teststringify("false", false, StringifyOptions{});
2556		try teststringify("true", true, StringifyOptions{});
2557		try teststringify("null", @as(?u8, null), StringifyOptions{});
2558		try teststringify("null", @as(?*u32, null), StringifyOptions{});
2559		try teststringify("42", 42, StringifyOptions{});
2560		try teststringify("4.2e+01", 42.0, StringifyOptions{});
2561		try teststringify("42", @as(u8, 42), StringifyOptions{});
2562		try teststringify("42", @as(u128, 42), StringifyOptions{});
2563		try teststringify("4.2e+01", @as(f32, 42), StringifyOptions{});
2564		try teststringify("4.2e+01", @as(f64, 42), StringifyOptions{});
2565		try teststringify("\"ItBroke\"", @as(anyerror, error.ItBroke), StringifyOptions{});
2566		}
2567
2568		test "stringify string" {
2569		try teststringify("\"hello\"", "hello", StringifyOptions{});
2570		try teststringify("\"with\\nescapes\\r\"", "with\nescapes\r", StringifyOptions{});
2571		try teststringify("\"with\\nescapes\\r\"", "with\nescapes\r", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
2572		try teststringify("\"with unicode\\u0001\"", "with unicode\u{1}", StringifyOptions{});
2573		try teststringify("\"with unicode\\u0001\"", "with unicode\u{1}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
2574		try teststringify("\"with unicode\u{80}\"", "with unicode\u{80}", StringifyOptions{});
2575		try teststringify("\"with unicode\\u0080\"", "with unicode\u{80}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
2576		try teststringify("\"with unicode\u{FF}\"", "with unicode\u{FF}", StringifyOptions{});
2577		try teststringify("\"with unicode\\u00ff\"", "with unicode\u{FF}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
2578		try teststringify("\"with unicode\u{100}\"", "with unicode\u{100}", StringifyOptions{});
2579		try teststringify("\"with unicode\\u0100\"", "with unicode\u{100}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
2580		try teststringify("\"with unicode\u{800}\"", "with unicode\u{800}", StringifyOptions{});
2581		try teststringify("\"with unicode\\u0800\"", "with unicode\u{800}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
2582		try teststringify("\"with unicode\u{8000}\"", "with unicode\u{8000}", StringifyOptions{});
2583		try teststringify("\"with unicode\\u8000\"", "with unicode\u{8000}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
2584		try teststringify("\"with unicode\u{D799}\"", "with unicode\u{D799}", StringifyOptions{});
2585		try teststringify("\"with unicode\\ud799\"", "with unicode\u{D799}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
2586		try teststringify("\"with unicode\u{10000}\"", "with unicode\u{10000}", StringifyOptions{});
2587		try teststringify("\"with unicode\\ud800\\udc00\"", "with unicode\u{10000}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
2588		try teststringify("\"with unicode\u{10FFFF}\"", "with unicode\u{10FFFF}", StringifyOptions{});
2589		try teststringify("\"with unicode\\udbff\\udfff\"", "with unicode\u{10FFFF}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
2590		try teststringify("\"/\"", "/", StringifyOptions{});
2591		try teststringify("\"\\/\"", "/", StringifyOptions{ .string = .{ .String = .{ .escape_solidus = true } } });
2592		}
2593
2594		test "stringify many-item sentinel-terminated string" {
2595		try teststringify("\"hello\"", @as([*:0]const u8, "hello"), StringifyOptions{});
2596		try teststringify("\"with\\nescapes\\r\"", @as([*:0]const u8, "with\nescapes\r"), StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
2597		try teststringify("\"with unicode\\u0001\"", @as([*:0]const u8, "with unicode\u{1}"), StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
2598		}
2599
2600		test "stringify tagged unions" {
2601		try teststringify("42", union(enum) {
2602		Foo: u32,
2603		Bar: bool,
2604		}{ .Foo = 42 }, StringifyOptions{});
2605		}
2606
2607		test "stringify struct" {
2608		try teststringify("{\"foo\":42}", struct {
2609		foo: u32,
2610		}{ .foo = 42 }, StringifyOptions{});
2611		}
2612
2613		test "stringify struct with string as array" {
2614		try teststringify("{\"foo\":\"bar\"}", .{ .foo = "bar" }, StringifyOptions{});
2615		try teststringify("{\"foo\":[98,97,114]}", .{ .foo = "bar" }, StringifyOptions{ .string = .Array });
2616		}
2617
2618		test "stringify struct with indentation" {
2619		try teststringify(
2620		\\{
2621		\\ "foo": 42,
2622		\\ "bar": [
2623		\\ 1,
2624		\\ 2,
2625		\\ 3
2626		\\ ]
2627		\\}
2628		,
2629		struct {
2630		foo: u32,
2631		bar: [3]u32,
2632		}{
2633		.foo = 42,
2634		.bar = .{ 1, 2, 3 },
2635		},
2636		StringifyOptions{
2637		.whitespace = .{},
2638		},
2639		);
2640		try teststringify(
2641		"{\n\t\"foo\":42,\n\t\"bar\":[\n\t\t1,\n\t\t2,\n\t\t3\n\t]\n}",
2642		struct {
2643		foo: u32,
2644		bar: [3]u32,
2645		}{
2646		.foo = 42,
2647		.bar = .{ 1, 2, 3 },
2648		},
2649		StringifyOptions{
2650		.whitespace = .{
2651		.indent = .Tab,
2652		.separator = false,
2653		},
2654		},
2655		);
2656		try teststringify(
2657		\\{"foo":42,"bar":[1,2,3]}
2658		,
2659		struct {
2660		foo: u32,
2661		bar: [3]u32,
2662		}{
2663		.foo = 42,
2664		.bar = .{ 1, 2, 3 },
2665		},
2666		StringifyOptions{
2667		.whitespace = .{
2668		.indent = .None,
2669		.separator = false,
2670		},
2671		},
2672		);
2673		}
2674
2675		test "stringify struct with void field" {
2676		try teststringify("{\"foo\":42}", struct {
2677		foo: u32,
2678		bar: void = {},
2679		}{ .foo = 42 }, StringifyOptions{});
2680		}
2681
2682		test "stringify array of structs" {
2683		const MyStruct = struct {
2684		foo: u32,
2685		};
2686		try teststringify("[{\"foo\":42},{\"foo\":100},{\"foo\":1000}]", [_]MyStruct{
2687		MyStruct{ .foo = 42 },
2688		MyStruct{ .foo = 100 },
2689		MyStruct{ .foo = 1000 },
2690		}, StringifyOptions{});
2691		}
2692
2693		test "stringify struct with custom stringifier" {
2694		try teststringify("[\"something special\",42]", struct {
2695		foo: u32,
2696		const Self = @This();
2697		pub fn jsonStringify(
2698		value: Self,
2699		options: StringifyOptions,
2700		out_stream: anytype,
2701		) !void {
2702		_ = value;
2703		try out_stream.writeAll("[\"something special\",");
2704		try stringify(42, options, out_stream);
2705		try out_stream.writeByte(']');
2706		}
2707		}{ .foo = 42 }, StringifyOptions{});
2708		}
2709
2710		test "stringify vector" {
2711		try teststringify("[1,1]", @splat(2, @as(u32, 1)), StringifyOptions{});
2712		}
2713
2714		test "stringify tuple" {
2715		try teststringify("[\"foo\",42]", std.meta.Tuple(&.{ []const u8, usize }){ "foo", 42 }, StringifyOptions{});
2716		}
2717
2718		fn teststringify(expected: []const u8, value: anytype, options: StringifyOptions) !void {
2719		const ValidationWriter = struct {
2720		const Self = @This();
2721		pub const Writer = std.io.Writer(*Self, Error, write);
2722		pub const Error = error{
2723		TooMuchData,
2724		DifferentData,
2725		};
2726
2727		expected_remaining: []const u8,
2728
2729		fn init(exp: []const u8) Self {
2730		return .{ .expected_remaining = exp };
2731		}
2732
2733		pub fn writer(self: *Self) Writer {
2734		return .{ .context = self };
2735		}
2736
2737		fn write(self: *Self, bytes: []const u8) Error!usize {
2738		if (self.expected_remaining.len < bytes.len) {
2739		std.debug.print(
2740		\\====== expected this output: =========
2741		\\{s}
2742		\\======== instead found this: =========
2743		\\{s}
2744		\\======================================
2745		, .{
2746		self.expected_remaining,
2747		bytes,
2748		});
2749		return error.TooMuchData;
2750		}
2751		if (!mem.eql(u8, self.expected_remaining[0..bytes.len], bytes)) {
2752		std.debug.print(
2753		\\====== expected this output: =========
2754		\\{s}
2755		\\======== instead found this: =========
2756		\\{s}
2757		\\======================================
2758		, .{
2759		self.expected_remaining[0..bytes.len],
2760		bytes,
2761		});
2762		return error.DifferentData;
2763		}
2764		self.expected_remaining = self.expected_remaining[bytes.len..];
2765		return bytes.len;
2766		}
2767		};
2768
2769		var vos = ValidationWriter.init(expected);
2770		try stringify(value, options, vos.writer());
2771		if (vos.expected_remaining.len > 0) return error.NotEnoughData;
2772		}
2773
2774		test "encodesTo" {
2775		// same
2776		try testing.expectEqual(true, encodesTo("false", "false"));
2777		// totally different
2778		try testing.expectEqual(false, encodesTo("false", "true"));
2779		// different lengths
2780		try testing.expectEqual(false, encodesTo("false", "other"));
2781		// with escape
2782		try testing.expectEqual(true, encodesTo("\\", "\\\\"));
2783		try testing.expectEqual(true, encodesTo("with\nescape", "with\\nescape"));
2784		// with unicode
2785		try testing.expectEqual(true, encodesTo("ą", "\\u0105"));
2786		try testing.expectEqual(true, encodesTo("😂", "\\ud83d\\ude02"));
2787		try testing.expectEqual(true, encodesTo("withąunicode😂", "with\\u0105unicode\\ud83d\\ude02"));
2788		}
2789
2790		test "deserializing string with escape sequence into sentinel slice" {
2791		const json = "\"\\n\"";
2792		var token_stream = std.json.TokenStream.init(json);
2793		const options = ParseOptions{ .allocator = std.testing.allocator };
2794
2795		// Pre-fix, this line would panic:
2796		const result = try std.json.parse([:0]const u8, &token_stream, options);
2797		defer std.json.parseFree([:0]const u8, result, options);
2798
2799		// Double-check that we're getting the right result
2800		try testing.expect(mem.eql(u8, result, "\n"));
2801		}
2802
2803		test "stringify struct with custom stringify that returns a custom error" {
2804		var ret = std.json.stringify(struct {
2805		field: Field = .{},
2806
2807		pub const Field = struct {
2808		field: ?[]*Field = null,
2809
2810		const Self = @This();
2811		pub fn jsonStringify(_: Self, _: StringifyOptions, _: anytype) error{CustomError}!void {
2812		return error.CustomError;
2813		}
2814		};
2815		}{}, StringifyOptions{}, std.io.null_writer);
2816
2817		try std.testing.expectError(error.CustomError, ret);
	55	_ = @import("json/dynamic.zig");
	56	_ = @import("json/static.zig");
	57	_ = @import("json/stringify.zig");
	58	_ = @import("json/JSONTestSuite_test.zig");
2818	59	}

lib/std/json/JSONTestSuite_test.zig created+960

...	...	@@ -0,0 +1,960 @@
	1	// This file was generated by _generate_JSONTestSuite.zig
	2	// These test cases are sourced from: https://github.com/nst/JSONTestSuite
	3	const ok = @import("./test.zig").ok;
	4	const err = @import("./test.zig").err;
	5	const any = @import("./test.zig").any;
	6
	7	test "i_number_double_huge_neg_exp.json" {
	8	try any("[123.456e-789]");
	9	}
	10	test "i_number_huge_exp.json" {
	11	try any("[0.4e00669999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999969999999006]");
	12	}
	13	test "i_number_neg_int_huge_exp.json" {
	14	try any("[-1e+9999]");
	15	}
	16	test "i_number_pos_double_huge_exp.json" {
	17	try any("[1.5e+9999]");
	18	}
	19	test "i_number_real_neg_overflow.json" {
	20	try any("[-123123e100000]");
	21	}
	22	test "i_number_real_pos_overflow.json" {
	23	try any("[123123e100000]");
	24	}
	25	test "i_number_real_underflow.json" {
	26	try any("[123e-10000000]");
	27	}
	28	test "i_number_too_big_neg_int.json" {
	29	try any("[-123123123123123123123123123123]");
	30	}
	31	test "i_number_too_big_pos_int.json" {
	32	try any("[100000000000000000000]");
	33	}
	34	test "i_number_very_big_negative_int.json" {
	35	try any("[-237462374673276894279832749832423479823246327846]");
	36	}
	37	test "i_object_key_lone_2nd_surrogate.json" {
	38	try any("{\"\\uDFAA\":0}");
	39	}
	40	test "i_string_1st_surrogate_but_2nd_missing.json" {
	41	try any("[\"\\uDADA\"]");
	42	}
	43	test "i_string_1st_valid_surrogate_2nd_invalid.json" {
	44	try any("[\"\\uD888\\u1234\"]");
	45	}
	46	test "i_string_UTF-16LE_with_BOM.json" {
	47	try any("\xff\xfe[\x00\"\x00\xe9\x00\"\x00]\x00");
	48	}
	49	test "i_string_UTF-8_invalid_sequence.json" {
	50	try any("[\"\xe6\x97\xa5\xd1\x88\xfa\"]");
	51	}
	52	test "i_string_UTF8_surrogate_U+D800.json" {
	53	try any("[\"\xed\xa0\x80\"]");
	54	}
	55	test "i_string_incomplete_surrogate_and_escape_valid.json" {
	56	try any("[\"\\uD800\\n\"]");
	57	}
	58	test "i_string_incomplete_surrogate_pair.json" {
	59	try any("[\"\\uDd1ea\"]");
	60	}
	61	test "i_string_incomplete_surrogates_escape_valid.json" {
	62	try any("[\"\\uD800\\uD800\\n\"]");
	63	}
	64	test "i_string_invalid_lonely_surrogate.json" {
	65	try any("[\"\\ud800\"]");
	66	}
	67	test "i_string_invalid_surrogate.json" {
	68	try any("[\"\\ud800abc\"]");
	69	}
	70	test "i_string_invalid_utf-8.json" {
	71	try any("[\"\xff\"]");
	72	}
	73	test "i_string_inverted_surrogates_U+1D11E.json" {
	74	try any("[\"\\uDd1e\\uD834\"]");
	75	}
	76	test "i_string_iso_latin_1.json" {
	77	try any("[\"\xe9\"]");
	78	}
	79	test "i_string_lone_second_surrogate.json" {
	80	try any("[\"\\uDFAA\"]");
	81	}
	82	test "i_string_lone_utf8_continuation_byte.json" {
	83	try any("[\"\x81\"]");
	84	}
	85	test "i_string_not_in_unicode_range.json" {
	86	try any("[\"\xf4\xbf\xbf\xbf\"]");
	87	}
	88	test "i_string_overlong_sequence_2_bytes.json" {
	89	try any("[\"\xc0\xaf\"]");
	90	}
	91	test "i_string_overlong_sequence_6_bytes.json" {
	92	try any("[\"\xfc\x83\xbf\xbf\xbf\xbf\"]");
	93	}
	94	test "i_string_overlong_sequence_6_bytes_null.json" {
	95	try any("[\"\xfc\x80\x80\x80\x80\x80\"]");
	96	}
	97	test "i_string_truncated-utf-8.json" {
	98	try any("[\"\xe0\xff\"]");
	99	}
	100	test "i_string_utf16BE_no_BOM.json" {
	101	try any("\x00[\x00\"\x00\xe9\x00\"\x00]");
	102	}
	103	test "i_string_utf16LE_no_BOM.json" {
	104	try any("[\x00\"\x00\xe9\x00\"\x00]\x00");
	105	}
	106	test "i_structure_500_nested_arrays.json" {
	107	try any("[" 500 ++ "]" 500);
	108	}
	109	test "i_structure_UTF-8_BOM_empty_object.json" {
	110	try any("\xef\xbb\xbf{}");
	111	}
	112	test "n_array_1_true_without_comma.json" {
	113	try err("[1 true]");
	114	}
	115	test "n_array_a_invalid_utf8.json" {
	116	try err("[a\xe5]");
	117	}
	118	test "n_array_colon_instead_of_comma.json" {
	119	try err("[\"\": 1]");
	120	}
	121	test "n_array_comma_after_close.json" {
	122	try err("[\"\"],");
	123	}
	124	test "n_array_comma_and_number.json" {
	125	try err("[,1]");
	126	}
	127	test "n_array_double_comma.json" {
	128	try err("[1,,2]");
	129	}
	130	test "n_array_double_extra_comma.json" {
	131	try err("[\"x\",,]");
	132	}
	133	test "n_array_extra_close.json" {
	134	try err("[\"x\"]]");
	135	}
	136	test "n_array_extra_comma.json" {
	137	try err("[\"\",]");
	138	}
	139	test "n_array_incomplete.json" {
	140	try err("[\"x\"");
	141	}
	142	test "n_array_incomplete_invalid_value.json" {
	143	try err("[x");
	144	}
	145	test "n_array_inner_array_no_comma.json" {
	146	try err("[3[4]]");
	147	}
	148	test "n_array_invalid_utf8.json" {
	149	try err("[\xff]");
	150	}
	151	test "n_array_items_separated_by_semicolon.json" {
	152	try err("[1:2]");
	153	}
	154	test "n_array_just_comma.json" {
	155	try err("[,]");
	156	}
	157	test "n_array_just_minus.json" {
	158	try err("[-]");
	159	}
	160	test "n_array_missing_value.json" {
	161	try err("[ , \"\"]");
	162	}
	163	test "n_array_newlines_unclosed.json" {
	164	try err("[\"a\",\n4\n,1,");
	165	}
	166	test "n_array_number_and_comma.json" {
	167	try err("[1,]");
	168	}
	169	test "n_array_number_and_several_commas.json" {
	170	try err("[1,,]");
	171	}
	172	test "n_array_spaces_vertical_tab_formfeed.json" {
	173	try err("[\"\x0ba\"\\f]");
	174	}
	175	test "n_array_star_inside.json" {
	176	try err("[*]");
	177	}
	178	test "n_array_unclosed.json" {
	179	try err("[\"\"");
	180	}
	181	test "n_array_unclosed_trailing_comma.json" {
	182	try err("[1,");
	183	}
	184	test "n_array_unclosed_with_new_lines.json" {
	185	try err("[1,\n1\n,1");
	186	}
	187	test "n_array_unclosed_with_object_inside.json" {
	188	try err("[{}");
	189	}
	190	test "n_incomplete_false.json" {
	191	try err("[fals]");
	192	}
	193	test "n_incomplete_null.json" {
	194	try err("[nul]");
	195	}
	196	test "n_incomplete_true.json" {
	197	try err("[tru]");
	198	}
	199	test "n_multidigit_number_then_00.json" {
	200	try err("123\x00");
	201	}
	202	test "n_number_++.json" {
	203	try err("[++1234]");
	204	}
	205	test "n_number_+1.json" {
	206	try err("[+1]");
	207	}
	208	test "n_number_+Inf.json" {
	209	try err("[+Inf]");
	210	}
	211	test "n_number_-01.json" {
	212	try err("[-01]");
	213	}
	214	test "n_number_-1.0..json" {
	215	try err("[-1.0.]");
	216	}
	217	test "n_number_-2..json" {
	218	try err("[-2.]");
	219	}
	220	test "n_number_-NaN.json" {
	221	try err("[-NaN]");
	222	}
	223	test "n_number_.-1.json" {
	224	try err("[.-1]");
	225	}
	226	test "n_number_.2e-3.json" {
	227	try err("[.2e-3]");
	228	}
	229	test "n_number_0.1.2.json" {
	230	try err("[0.1.2]");
	231	}
	232	test "n_number_0.3e+.json" {
	233	try err("[0.3e+]");
	234	}
	235	test "n_number_0.3e.json" {
	236	try err("[0.3e]");
	237	}
	238	test "n_number_0.e1.json" {
	239	try err("[0.e1]");
	240	}
	241	test "n_number_0_capital_E+.json" {
	242	try err("[0E+]");
	243	}
	244	test "n_number_0_capital_E.json" {
	245	try err("[0E]");
	246	}
	247	test "n_number_0e+.json" {
	248	try err("[0e+]");
	249	}
	250	test "n_number_0e.json" {
	251	try err("[0e]");
	252	}
	253	test "n_number_1.0e+.json" {
	254	try err("[1.0e+]");
	255	}
	256	test "n_number_1.0e-.json" {
	257	try err("[1.0e-]");
	258	}
	259	test "n_number_1.0e.json" {
	260	try err("[1.0e]");
	261	}
	262	test "n_number_1_000.json" {
	263	try err("[1 000.0]");
	264	}
	265	test "n_number_1eE2.json" {
	266	try err("[1eE2]");
	267	}
	268	test "n_number_2.e+3.json" {
	269	try err("[2.e+3]");
	270	}
	271	test "n_number_2.e-3.json" {
	272	try err("[2.e-3]");
	273	}
	274	test "n_number_2.e3.json" {
	275	try err("[2.e3]");
	276	}
	277	test "n_number_9.e+.json" {
	278	try err("[9.e+]");
	279	}
	280	test "n_number_Inf.json" {
	281	try err("[Inf]");
	282	}
	283	test "n_number_NaN.json" {
	284	try err("[NaN]");
	285	}
	286	test "n_number_U+FF11_fullwidth_digit_one.json" {
	287	try err("[\xef\xbc\x91]");
	288	}
	289	test "n_number_expression.json" {
	290	try err("[1+2]");
	291	}
	292	test "n_number_hex_1_digit.json" {
	293	try err("[0x1]");
	294	}
	295	test "n_number_hex_2_digits.json" {
	296	try err("[0x42]");
	297	}
	298	test "n_number_infinity.json" {
	299	try err("[Infinity]");
	300	}
	301	test "n_number_invalid+-.json" {
	302	try err("[0e+-1]");
	303	}
	304	test "n_number_invalid-negative-real.json" {
	305	try err("[-123.123foo]");
	306	}
	307	test "n_number_invalid-utf-8-in-bigger-int.json" {
	308	try err("[123\xe5]");
	309	}
	310	test "n_number_invalid-utf-8-in-exponent.json" {
	311	try err("[1e1\xe5]");
	312	}
	313	test "n_number_invalid-utf-8-in-int.json" {
	314	try err("[0\xe5]\n");
	315	}
	316	test "n_number_minus_infinity.json" {
	317	try err("[-Infinity]");
	318	}
	319	test "n_number_minus_sign_with_trailing_garbage.json" {
	320	try err("[-foo]");
	321	}
	322	test "n_number_minus_space_1.json" {
	323	try err("[- 1]");
	324	}
	325	test "n_number_neg_int_starting_with_zero.json" {
	326	try err("[-012]");
	327	}
	328	test "n_number_neg_real_without_int_part.json" {
	329	try err("[-.123]");
	330	}
	331	test "n_number_neg_with_garbage_at_end.json" {
	332	try err("[-1x]");
	333	}
	334	test "n_number_real_garbage_after_e.json" {
	335	try err("[1ea]");
	336	}
	337	test "n_number_real_with_invalid_utf8_after_e.json" {
	338	try err("[1e\xe5]");
	339	}
	340	test "n_number_real_without_fractional_part.json" {
	341	try err("[1.]");
	342	}
	343	test "n_number_starting_with_dot.json" {
	344	try err("[.123]");
	345	}
	346	test "n_number_with_alpha.json" {
	347	try err("[1.2a-3]");
	348	}
	349	test "n_number_with_alpha_char.json" {
	350	try err("[1.8011670033376514H-308]");
	351	}
	352	test "n_number_with_leading_zero.json" {
	353	try err("[012]");
	354	}
	355	test "n_object_bad_value.json" {
	356	try err("[\"x\", truth]");
	357	}
	358	test "n_object_bracket_key.json" {
	359	try err("{[: \"x\"}\n");
	360	}
	361	test "n_object_comma_instead_of_colon.json" {
	362	try err("{\"x\", null}");
	363	}
	364	test "n_object_double_colon.json" {
	365	try err("{\"x\"::\"b\"}");
	366	}
	367	test "n_object_emoji.json" {
	368	try err("{\xf0\x9f\x87\xa8\xf0\x9f\x87\xad}");
	369	}
	370	test "n_object_garbage_at_end.json" {
	371	try err("{\"a\":\"a\" 123}");
	372	}
	373	test "n_object_key_with_single_quotes.json" {
	374	try err("{key: 'value'}");
	375	}
	376	test "n_object_lone_continuation_byte_in_key_and_trailing_comma.json" {
	377	try err("{\"\xb9\":\"0\",}");
	378	}
	379	test "n_object_missing_colon.json" {
	380	try err("{\"a\" b}");
	381	}
	382	test "n_object_missing_key.json" {
	383	try err("{:\"b\"}");
	384	}
	385	test "n_object_missing_semicolon.json" {
	386	try err("{\"a\" \"b\"}");
	387	}
	388	test "n_object_missing_value.json" {
	389	try err("{\"a\":");
	390	}
	391	test "n_object_no-colon.json" {
	392	try err("{\"a\"");
	393	}
	394	test "n_object_non_string_key.json" {
	395	try err("{1:1}");
	396	}
	397	test "n_object_non_string_key_but_huge_number_instead.json" {
	398	try err("{9999E9999:1}");
	399	}
	400	test "n_object_repeated_null_null.json" {
	401	try err("{null:null,null:null}");
	402	}
	403	test "n_object_several_trailing_commas.json" {
	404	try err("{\"id\":0,,,,,}");
	405	}
	406	test "n_object_single_quote.json" {
	407	try err("{'a':0}");
	408	}
	409	test "n_object_trailing_comma.json" {
	410	try err("{\"id\":0,}");
	411	}
	412	test "n_object_trailing_comment.json" {
	413	try err("{\"a\":\"b\"}/**/");
	414	}
	415	test "n_object_trailing_comment_open.json" {
	416	try err("{\"a\":\"b\"}/**//");
	417	}
	418	test "n_object_trailing_comment_slash_open.json" {
	419	try err("{\"a\":\"b\"}//");
	420	}
	421	test "n_object_trailing_comment_slash_open_incomplete.json" {
	422	try err("{\"a\":\"b\"}/");
	423	}
	424	test "n_object_two_commas_in_a_row.json" {
	425	try err("{\"a\":\"b\",,\"c\":\"d\"}");
	426	}
	427	test "n_object_unquoted_key.json" {
	428	try err("{a: \"b\"}");
	429	}
	430	test "n_object_unterminated-value.json" {
	431	try err("{\"a\":\"a");
	432	}
	433	test "n_object_with_single_string.json" {
	434	try err("{ \"foo\" : \"bar\", \"a\" }");
	435	}
	436	test "n_object_with_trailing_garbage.json" {
	437	try err("{\"a\":\"b\"}#");
	438	}
	439	test "n_single_space.json" {
	440	try err(" ");
	441	}
	442	test "n_string_1_surrogate_then_escape.json" {
	443	try err("[\"\\uD800\\\"]");
	444	}
	445	test "n_string_1_surrogate_then_escape_u.json" {
	446	try err("[\"\\uD800\\u\"]");
	447	}
	448	test "n_string_1_surrogate_then_escape_u1.json" {
	449	try err("[\"\\uD800\\u1\"]");
	450	}
	451	test "n_string_1_surrogate_then_escape_u1x.json" {
	452	try err("[\"\\uD800\\u1x\"]");
	453	}
	454	test "n_string_accentuated_char_no_quotes.json" {
	455	try err("[\xc3\xa9]");
	456	}
	457	test "n_string_backslash_00.json" {
	458	try err("[\"\\\x00\"]");
	459	}
	460	test "n_string_escape_x.json" {
	461	try err("[\"\\x00\"]");
	462	}
	463	test "n_string_escaped_backslash_bad.json" {
	464	try err("[\"\\\\\\\"]");
	465	}
	466	test "n_string_escaped_ctrl_char_tab.json" {
	467	try err("[\"\\\x09\"]");
	468	}
	469	test "n_string_escaped_emoji.json" {
	470	try err("[\"\\\xf0\x9f\x8c\x80\"]");
	471	}
	472	test "n_string_incomplete_escape.json" {
	473	try err("[\"\\\"]");
	474	}
	475	test "n_string_incomplete_escaped_character.json" {
	476	try err("[\"\\u00A\"]");
	477	}
	478	test "n_string_incomplete_surrogate.json" {
	479	try err("[\"\\uD834\\uDd\"]");
	480	}
	481	test "n_string_incomplete_surrogate_escape_invalid.json" {
	482	try err("[\"\\uD800\\uD800\\x\"]");
	483	}
	484	test "n_string_invalid-utf-8-in-escape.json" {
	485	try err("[\"\\u\xe5\"]");
	486	}
	487	test "n_string_invalid_backslash_esc.json" {
	488	try err("[\"\\a\"]");
	489	}
	490	test "n_string_invalid_unicode_escape.json" {
	491	try err("[\"\\uqqqq\"]");
	492	}
	493	test "n_string_invalid_utf8_after_escape.json" {
	494	try err("[\"\\\xe5\"]");
	495	}
	496	test "n_string_leading_uescaped_thinspace.json" {
	497	try err("[\\u0020\"asd\"]");
	498	}
	499	test "n_string_no_quotes_with_bad_escape.json" {
	500	try err("[\\n]");
	501	}
	502	test "n_string_single_doublequote.json" {
	503	try err("\"");
	504	}
	505	test "n_string_single_quote.json" {
	506	try err("['single quote']");
	507	}
	508	test "n_string_single_string_no_double_quotes.json" {
	509	try err("abc");
	510	}
	511	test "n_string_start_escape_unclosed.json" {
	512	try err("[\"\\");
	513	}
	514	test "n_string_unescaped_ctrl_char.json" {
	515	try err("[\"a\x00a\"]");
	516	}
	517	test "n_string_unescaped_newline.json" {
	518	try err("[\"new\nline\"]");
	519	}
	520	test "n_string_unescaped_tab.json" {
	521	try err("[\"\x09\"]");
	522	}
	523	test "n_string_unicode_CapitalU.json" {
	524	try err("\"\\UA66D\"");
	525	}
	526	test "n_string_with_trailing_garbage.json" {
	527	try err("\"\"x");
	528	}
	529	test "n_structure_100000_opening_arrays.json" {
	530	try err("[" ** 100000);
	531	}
	532	test "n_structure_U+2060_word_joined.json" {
	533	try err("[\xe2\x81\xa0]");
	534	}
	535	test "n_structure_UTF8_BOM_no_data.json" {
	536	try err("\xef\xbb\xbf");
	537	}
	538	test "n_structure_angle_bracket_..json" {
	539	try err("<.>");
	540	}
	541	test "n_structure_angle_bracket_null.json" {
	542	try err("[<null>]");
	543	}
	544	test "n_structure_array_trailing_garbage.json" {
	545	try err("[1]x");
	546	}
	547	test "n_structure_array_with_extra_array_close.json" {
	548	try err("[1]]");
	549	}
	550	test "n_structure_array_with_unclosed_string.json" {
	551	try err("[\"asd]");
	552	}
	553	test "n_structure_ascii-unicode-identifier.json" {
	554	try err("a\xc3\xa5");
	555	}
	556	test "n_structure_capitalized_True.json" {
	557	try err("[True]");
	558	}
	559	test "n_structure_close_unopened_array.json" {
	560	try err("1]");
	561	}
	562	test "n_structure_comma_instead_of_closing_brace.json" {
	563	try err("{\"x\": true,");
	564	}
	565	test "n_structure_double_array.json" {
	566	try err("[][]");
	567	}
	568	test "n_structure_end_array.json" {
	569	try err("]");
	570	}
	571	test "n_structure_incomplete_UTF8_BOM.json" {
	572	try err("\xef\xbb{}");
	573	}
	574	test "n_structure_lone-invalid-utf-8.json" {
	575	try err("\xe5");
	576	}
	577	test "n_structure_lone-open-bracket.json" {
	578	try err("[");
	579	}
	580	test "n_structure_no_data.json" {
	581	try err("");
	582	}
	583	test "n_structure_null-byte-outside-string.json" {
	584	try err("[\x00]");
	585	}
	586	test "n_structure_number_with_trailing_garbage.json" {
	587	try err("2@");
	588	}
	589	test "n_structure_object_followed_by_closing_object.json" {
	590	try err("{}}");
	591	}
	592	test "n_structure_object_unclosed_no_value.json" {
	593	try err("{\"\":");
	594	}
	595	test "n_structure_object_with_comment.json" {
	596	try err("{\"a\":/comment/\"b\"}");
	597	}
	598	test "n_structure_object_with_trailing_garbage.json" {
	599	try err("{\"a\": true} \"x\"");
	600	}
	601	test "n_structure_open_array_apostrophe.json" {
	602	try err("['");
	603	}
	604	test "n_structure_open_array_comma.json" {
	605	try err("[,");
	606	}
	607	test "n_structure_open_array_object.json" {
	608	try err("[{\"\":" ** 50000 ++ "\n");
	609	}
	610	test "n_structure_open_array_open_object.json" {
	611	try err("[{");
	612	}
	613	test "n_structure_open_array_open_string.json" {
	614	try err("[\"a");
	615	}
	616	test "n_structure_open_array_string.json" {
	617	try err("[\"a\"");
	618	}
	619	test "n_structure_open_object.json" {
	620	try err("{");
	621	}
	622	test "n_structure_open_object_close_array.json" {
	623	try err("{]");
	624	}
	625	test "n_structure_open_object_comma.json" {
	626	try err("{,");
	627	}
	628	test "n_structure_open_object_open_array.json" {
	629	try err("{[");
	630	}
	631	test "n_structure_open_object_open_string.json" {
	632	try err("{\"a");
	633	}
	634	test "n_structure_open_object_string_with_apostrophes.json" {
	635	try err("{'a'");
	636	}
	637	test "n_structure_open_open.json" {
	638	try err("[\"\\{[\"\\{[\"\\{[\"\\{");
	639	}
	640	test "n_structure_single_eacute.json" {
	641	try err("\xe9");
	642	}
	643	test "n_structure_single_star.json" {
	644	try err("*");
	645	}
	646	test "n_structure_trailing_#.json" {
	647	try err("{\"a\":\"b\"}#{}");
	648	}
	649	test "n_structure_uescaped_LF_before_string.json" {
	650	try err("[\\u000A\"\"]");
	651	}
	652	test "n_structure_unclosed_array.json" {
	653	try err("[1");
	654	}
	655	test "n_structure_unclosed_array_partial_null.json" {
	656	try err("[ false, nul");
	657	}
	658	test "n_structure_unclosed_array_unfinished_false.json" {
	659	try err("[ true, fals");
	660	}
	661	test "n_structure_unclosed_array_unfinished_true.json" {
	662	try err("[ false, tru");
	663	}
	664	test "n_structure_unclosed_object.json" {
	665	try err("{\"asd\":\"asd\"");
	666	}
	667	test "n_structure_unicode-identifier.json" {
	668	try err("\xc3\xa5");
	669	}
	670	test "n_structure_whitespace_U+2060_word_joiner.json" {
	671	try err("[\xe2\x81\xa0]");
	672	}
	673	test "n_structure_whitespace_formfeed.json" {
	674	try err("[\x0c]");
	675	}
	676	test "y_array_arraysWithSpaces.json" {
	677	try ok("[[] ]");
	678	}
	679	test "y_array_empty-string.json" {
	680	try ok("[\"\"]");
	681	}
	682	test "y_array_empty.json" {
	683	try ok("[]");
	684	}
	685	test "y_array_ending_with_newline.json" {
	686	try ok("[\"a\"]");
	687	}
	688	test "y_array_false.json" {
	689	try ok("[false]");
	690	}
	691	test "y_array_heterogeneous.json" {
	692	try ok("[null, 1, \"1\", {}]");
	693	}
	694	test "y_array_null.json" {
	695	try ok("[null]");
	696	}
	697	test "y_array_with_1_and_newline.json" {
	698	try ok("[1\n]");
	699	}
	700	test "y_array_with_leading_space.json" {
	701	try ok(" [1]");
	702	}
	703	test "y_array_with_several_null.json" {
	704	try ok("[1,null,null,null,2]");
	705	}
	706	test "y_array_with_trailing_space.json" {
	707	try ok("[2] ");
	708	}
	709	test "y_number.json" {
	710	try ok("[123e65]");
	711	}
	712	test "y_number_0e+1.json" {
	713	try ok("[0e+1]");
	714	}
	715	test "y_number_0e1.json" {
	716	try ok("[0e1]");
	717	}
	718	test "y_number_after_space.json" {
	719	try ok("[ 4]");
	720	}
	721	test "y_number_double_close_to_zero.json" {
	722	try ok("[-0.000000000000000000000000000000000000000000000000000000000000000000000000000001]\n");
	723	}
	724	test "y_number_int_with_exp.json" {
	725	try ok("[20e1]");
	726	}
	727	test "y_number_minus_zero.json" {
	728	try ok("[-0]");
	729	}
	730	test "y_number_negative_int.json" {
	731	try ok("[-123]");
	732	}
	733	test "y_number_negative_one.json" {
	734	try ok("[-1]");
	735	}
	736	test "y_number_negative_zero.json" {
	737	try ok("[-0]");
	738	}
	739	test "y_number_real_capital_e.json" {
	740	try ok("[1E22]");
	741	}
	742	test "y_number_real_capital_e_neg_exp.json" {
	743	try ok("[1E-2]");
	744	}
	745	test "y_number_real_capital_e_pos_exp.json" {
	746	try ok("[1E+2]");
	747	}
	748	test "y_number_real_exponent.json" {
	749	try ok("[123e45]");
	750	}
	751	test "y_number_real_fraction_exponent.json" {
	752	try ok("[123.456e78]");
	753	}
	754	test "y_number_real_neg_exp.json" {
	755	try ok("[1e-2]");
	756	}
	757	test "y_number_real_pos_exponent.json" {
	758	try ok("[1e+2]");
	759	}
	760	test "y_number_simple_int.json" {
	761	try ok("[123]");
	762	}
	763	test "y_number_simple_real.json" {
	764	try ok("[123.456789]");
	765	}
	766	test "y_object.json" {
	767	try ok("{\"asd\":\"sdf\", \"dfg\":\"fgh\"}");
	768	}
	769	test "y_object_basic.json" {
	770	try ok("{\"asd\":\"sdf\"}");
	771	}
	772	test "y_object_duplicated_key.json" {
	773	try ok("{\"a\":\"b\",\"a\":\"c\"}");
	774	}
	775	test "y_object_duplicated_key_and_value.json" {
	776	try ok("{\"a\":\"b\",\"a\":\"b\"}");
	777	}
	778	test "y_object_empty.json" {
	779	try ok("{}");
	780	}
	781	test "y_object_empty_key.json" {
	782	try ok("{\"\":0}");
	783	}
	784	test "y_object_escaped_null_in_key.json" {
	785	try ok("{\"foo\\u0000bar\": 42}");
	786	}
	787	test "y_object_extreme_numbers.json" {
	788	try ok("{ \"min\": -1.0e+28, \"max\": 1.0e+28 }");
	789	}
	790	test "y_object_long_strings.json" {
	791	try ok("{\"x\":[{\"id\": \"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"}], \"id\": \"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"}");
	792	}
	793	test "y_object_simple.json" {
	794	try ok("{\"a\":[]}");
	795	}
	796	test "y_object_string_unicode.json" {
	797	try ok("{\"title\":\"\\u041f\\u043e\\u043b\\u0442\\u043e\\u0440\\u0430 \\u0417\\u0435\\u043c\\u043b\\u0435\\u043a\\u043e\\u043f\\u0430\" }");
	798	}
	799	test "y_object_with_newlines.json" {
	800	try ok("{\n\"a\": \"b\"\n}");
	801	}
	802	test "y_string_1_2_3_bytes_UTF-8_sequences.json" {
	803	try ok("[\"\\u0060\\u012a\\u12AB\"]");
	804	}
	805	test "y_string_accepted_surrogate_pair.json" {
	806	try ok("[\"\\uD801\\udc37\"]");
	807	}
	808	test "y_string_accepted_surrogate_pairs.json" {
	809	try ok("[\"\\ud83d\\ude39\\ud83d\\udc8d\"]");
	810	}
	811	test "y_string_allowed_escapes.json" {
	812	try ok("[\"\\\"\\\\\\/\\b\\f\\n\\r\\t\"]");
	813	}
	814	test "y_string_backslash_and_u_escaped_zero.json" {
	815	try ok("[\"\\\\u0000\"]");
	816	}
	817	test "y_string_backslash_doublequotes.json" {
	818	try ok("[\"\\\"\"]");
	819	}
	820	test "y_string_comments.json" {
	821	try ok("[\"a/b/c/*d//e\"]");
	822	}
	823	test "y_string_double_escape_a.json" {
	824	try ok("[\"\\\\a\"]");
	825	}
	826	test "y_string_double_escape_n.json" {
	827	try ok("[\"\\\\n\"]");
	828	}
	829	test "y_string_escaped_control_character.json" {
	830	try ok("[\"\\u0012\"]");
	831	}
	832	test "y_string_escaped_noncharacter.json" {
	833	try ok("[\"\\uFFFF\"]");
	834	}
	835	test "y_string_in_array.json" {
	836	try ok("[\"asd\"]");
	837	}
	838	test "y_string_in_array_with_leading_space.json" {
	839	try ok("[ \"asd\"]");
	840	}
	841	test "y_string_last_surrogates_1_and_2.json" {
	842	try ok("[\"\\uDBFF\\uDFFF\"]");
	843	}
	844	test "y_string_nbsp_uescaped.json" {
	845	try ok("[\"new\\u00A0line\"]");
	846	}
	847	test "y_string_nonCharacterInUTF-8_U+10FFFF.json" {
	848	try ok("[\"\xf4\x8f\xbf\xbf\"]");
	849	}
	850	test "y_string_nonCharacterInUTF-8_U+FFFF.json" {
	851	try ok("[\"\xef\xbf\xbf\"]");
	852	}
	853	test "y_string_null_escape.json" {
	854	try ok("[\"\\u0000\"]");
	855	}
	856	test "y_string_one-byte-utf-8.json" {
	857	try ok("[\"\\u002c\"]");
	858	}
	859	test "y_string_pi.json" {
	860	try ok("[\"\xcf\x80\"]");
	861	}
	862	test "y_string_reservedCharacterInUTF-8_U+1BFFF.json" {
	863	try ok("[\"\xf0\x9b\xbf\xbf\"]");
	864	}
	865	test "y_string_simple_ascii.json" {
	866	try ok("[\"asd \"]");
	867	}
	868	test "y_string_space.json" {
	869	try ok("\" \"");
	870	}
	871	test "y_string_surrogates_U+1D11E_MUSICAL_SYMBOL_G_CLEF.json" {
	872	try ok("[\"\\uD834\\uDd1e\"]");
	873	}
	874	test "y_string_three-byte-utf-8.json" {
	875	try ok("[\"\\u0821\"]");
	876	}
	877	test "y_string_two-byte-utf-8.json" {
	878	try ok("[\"\\u0123\"]");
	879	}
	880	test "y_string_u+2028_line_sep.json" {
	881	try ok("[\"\xe2\x80\xa8\"]");
	882	}
	883	test "y_string_u+2029_par_sep.json" {
	884	try ok("[\"\xe2\x80\xa9\"]");
	885	}
	886	test "y_string_uEscape.json" {
	887	try ok("[\"\\u0061\\u30af\\u30EA\\u30b9\"]");
	888	}
	889	test "y_string_uescaped_newline.json" {
	890	try ok("[\"new\\u000Aline\"]");
	891	}
	892	test "y_string_unescaped_char_delete.json" {
	893	try ok("[\"\x7f\"]");
	894	}
	895	test "y_string_unicode.json" {
	896	try ok("[\"\\uA66D\"]");
	897	}
	898	test "y_string_unicodeEscapedBackslash.json" {
	899	try ok("[\"\\u005C\"]");
	900	}
	901	test "y_string_unicode_2.json" {
	902	try ok("[\"\xe2\x8d\x82\xe3\x88\xb4\xe2\x8d\x82\"]");
	903	}
	904	test "y_string_unicode_U+10FFFE_nonchar.json" {
	905	try ok("[\"\\uDBFF\\uDFFE\"]");
	906	}
	907	test "y_string_unicode_U+1FFFE_nonchar.json" {
	908	try ok("[\"\\uD83F\\uDFFE\"]");
	909	}
	910	test "y_string_unicode_U+200B_ZERO_WIDTH_SPACE.json" {
	911	try ok("[\"\\u200B\"]");
	912	}
	913	test "y_string_unicode_U+2064_invisible_plus.json" {
	914	try ok("[\"\\u2064\"]");
	915	}
	916	test "y_string_unicode_U+FDD0_nonchar.json" {
	917	try ok("[\"\\uFDD0\"]");
	918	}
	919	test "y_string_unicode_U+FFFE_nonchar.json" {
	920	try ok("[\"\\uFFFE\"]");
	921	}
	922	test "y_string_unicode_escaped_double_quote.json" {
	923	try ok("[\"\\u0022\"]");
	924	}
	925	test "y_string_utf8.json" {
	926	try ok("[\"\xe2\x82\xac\xf0\x9d\x84\x9e\"]");
	927	}
	928	test "y_string_with_del_character.json" {
	929	try ok("[\"a\x7fa\"]");
	930	}
	931	test "y_structure_lonely_false.json" {
	932	try ok("false");
	933	}
	934	test "y_structure_lonely_int.json" {
	935	try ok("42");
	936	}
	937	test "y_structure_lonely_negative_real.json" {
	938	try ok("-0.1");
	939	}
	940	test "y_structure_lonely_null.json" {
	941	try ok("null");
	942	}
	943	test "y_structure_lonely_string.json" {
	944	try ok("\"asd\"");
	945	}
	946	test "y_structure_lonely_true.json" {
	947	try ok("true");
	948	}
	949	test "y_structure_string_empty.json" {
	950	try ok("\"\"");
	951	}
	952	test "y_structure_trailing_newline.json" {
	953	try ok("[\"a\"]\n");
	954	}
	955	test "y_structure_true_in_array.json" {
	956	try ok("[true]");
	957	}
	958	test "y_structure_whitespace_array.json" {
	959	try ok(" [] ");
	960	}

lib/std/json/dynamic.zig created+344

...	...	@@ -0,0 +1,344 @@
	1	const std = @import("std");
	2	const debug = std.debug;
	3	const ArenaAllocator = std.heap.ArenaAllocator;
	4	const ArrayList = std.ArrayList;
	5	const StringArrayHashMap = std.StringArrayHashMap;
	6	const Allocator = std.mem.Allocator;
	7
	8	const StringifyOptions = @import("./stringify.zig").StringifyOptions;
	9	const stringify = @import("./stringify.zig").stringify;
	10
	11	const JsonScanner = @import("./scanner.zig").Scanner;
	12	const AllocWhen = @import("./scanner.zig").AllocWhen;
	13	const Token = @import("./scanner.zig").Token;
	14	const isNumberFormattedLikeAnInteger = @import("./scanner.zig").isNumberFormattedLikeAnInteger;
	15
	16	pub const ValueTree = struct {
	17	arena: *ArenaAllocator,
	18	root: Value,
	19
	20	pub fn deinit(self: *ValueTree) void {
	21	self.arena.deinit();
	22	self.arena.child_allocator.destroy(self.arena);
	23	}
	24	};
	25
	26	pub const ObjectMap = StringArrayHashMap(Value);
	27	pub const Array = ArrayList(Value);
	28
	29	/// Represents a JSON value
	30	/// Currently only supports numbers that fit into i64 or f64.
	31	pub const Value = union(enum) {
	32	null,
	33	bool: bool,
	34	integer: i64,
	35	float: f64,
	36	number_string: []const u8,
	37	string: []const u8,
	38	array: Array,
	39	object: ObjectMap,
	40
	41	pub fn jsonStringify(
	42	value: @This(),
	43	options: StringifyOptions,
	44	out_stream: anytype,
	45	) @TypeOf(out_stream).Error!void {
	46	switch (value) {
	47	.null => try stringify(null, options, out_stream),
	48	.bool => \|inner\| try stringify(inner, options, out_stream),
	49	.integer => \|inner\| try stringify(inner, options, out_stream),
	50	.float => \|inner\| try stringify(inner, options, out_stream),
	51	.number_string => \|inner\| try out_stream.writeAll(inner),
	52	.string => \|inner\| try stringify(inner, options, out_stream),
	53	.array => \|inner\| try stringify(inner.items, options, out_stream),
	54	.object => \|inner\| {
	55	try out_stream.writeByte('{');
	56	var field_output = false;
	57	var child_options = options;
	58	child_options.whitespace.indent_level += 1;
	59	var it = inner.iterator();
	60	while (it.next()) \|entry\| {
	61	if (!field_output) {
	62	field_output = true;
	63	} else {
	64	try out_stream.writeByte(',');
	65	}
	66	try child_options.whitespace.outputIndent(out_stream);
	67
	68	try stringify(entry.key_ptr.*, options, out_stream);
	69	try out_stream.writeByte(':');
	70	if (child_options.whitespace.separator) {
	71	try out_stream.writeByte(' ');
	72	}
	73	try stringify(entry.value_ptr.*, child_options, out_stream);
	74	}
	75	if (field_output) {
	76	try options.whitespace.outputIndent(out_stream);
	77	}
	78	try out_stream.writeByte('}');
	79	},
	80	}
	81	}
	82
	83	pub fn dump(self: Value) void {
	84	std.debug.getStderrMutex().lock();
	85	defer std.debug.getStderrMutex().unlock();
	86
	87	const stderr = std.io.getStdErr().writer();
	88	stringify(self, .{}, stderr) catch return;
	89	}
	90	};
	91
	92	/// A non-stream JSON parser which constructs a tree of Value's.
	93	pub const Parser = struct {
	94	allocator: Allocator,
	95	state: State,
	96	alloc_when: AllocWhen,
	97	// Stores parent nodes and un-combined Values.
	98	stack: Array,
	99
	100	const State = enum {
	101	object_key,
	102	object_value,
	103	array_value,
	104	simple,
	105	};
	106
	107	pub fn init(allocator: Allocator, alloc_when: AllocWhen) Parser {
	108	return Parser{
	109	.allocator = allocator,
	110	.state = .simple,
	111	.alloc_when = alloc_when,
	112	.stack = Array.init(allocator),
	113	};
	114	}
	115
	116	pub fn deinit(p: *Parser) void {
	117	p.stack.deinit();
	118	}
	119
	120	pub fn reset(p: *Parser) void {
	121	p.state = .simple;
	122	p.stack.shrinkRetainingCapacity(0);
	123	}
	124
	125	pub fn parse(p: *Parser, input: []const u8) !ValueTree {
	126	var scanner = JsonScanner.initCompleteInput(p.allocator, input);
	127	defer scanner.deinit();
	128
	129	var arena = try p.allocator.create(ArenaAllocator);
	130	errdefer p.allocator.destroy(arena);
	131
	132	arena.* = ArenaAllocator.init(p.allocator);
	133	errdefer arena.deinit();
	134
	135	const allocator = arena.allocator();
	136
	137	while (true) {
	138	const token = try scanner.nextAlloc(allocator, p.alloc_when);
	139	if (token == .end_of_document) break;
	140	try p.transition(allocator, token);
	141	}
	142
	143	debug.assert(p.stack.items.len == 1);
	144
	145	return ValueTree{
	146	.arena = arena,
	147	.root = p.stack.items[0],
	148	};
	149	}
	150
	151	// Even though p.allocator exists, we take an explicit allocator so that allocation state
	152	// can be cleaned up on error correctly during a `parse` on call.
	153	fn transition(p: *Parser, allocator: Allocator, token: Token) !void {
	154	switch (p.state) {
	155	.object_key => switch (token) {
	156	.object_end => {
	157	if (p.stack.items.len == 1) {
	158	return;
	159	}
	160
	161	var value = p.stack.pop();
	162	try p.pushToParent(&value);
	163	},
	164	.string => \|s\| {
	165	try p.stack.append(Value{ .string = s });
	166	p.state = .object_value;
	167	},
	168	.allocated_string => \|s\| {
	169	try p.stack.append(Value{ .string = s });
	170	p.state = .object_value;
	171	},
	172	else => unreachable,
	173	},
	174	.object_value => {
	175	var object = &p.stack.items[p.stack.items.len - 2].object;
	176	var key = p.stack.items[p.stack.items.len - 1].string;
	177
	178	switch (token) {
	179	.object_begin => {
	180	try p.stack.append(Value{ .object = ObjectMap.init(allocator) });
	181	p.state = .object_key;
	182	},
	183	.array_begin => {
	184	try p.stack.append(Value{ .array = Array.init(allocator) });
	185	p.state = .array_value;
	186	},
	187	.string => \|s\| {
	188	try object.put(key, Value{ .string = s });
	189	_ = p.stack.pop();
	190	p.state = .object_key;
	191	},
	192	.allocated_string => \|s\| {
	193	try object.put(key, Value{ .string = s });
	194	_ = p.stack.pop();
	195	p.state = .object_key;
	196	},
	197	.number => \|slice\| {
	198	try object.put(key, try p.parseNumber(slice));
	199	_ = p.stack.pop();
	200	p.state = .object_key;
	201	},
	202	.allocated_number => \|slice\| {
	203	try object.put(key, try p.parseNumber(slice));
	204	_ = p.stack.pop();
	205	p.state = .object_key;
	206	},
	207	.true => {
	208	try object.put(key, Value{ .bool = true });
	209	_ = p.stack.pop();
	210	p.state = .object_key;
	211	},
	212	.false => {
	213	try object.put(key, Value{ .bool = false });
	214	_ = p.stack.pop();
	215	p.state = .object_key;
	216	},
	217	.null => {
	218	try object.put(key, .null);
	219	_ = p.stack.pop();
	220	p.state = .object_key;
	221	},
	222	.object_end, .array_end, .end_of_document => unreachable,
	223	.partial_number, .partial_string, .partial_string_escaped_1, .partial_string_escaped_2, .partial_string_escaped_3, .partial_string_escaped_4 => unreachable,
	224	}
	225	},
	226	.array_value => {
	227	var array = &p.stack.items[p.stack.items.len - 1].array;
	228
	229	switch (token) {
	230	.array_end => {
	231	if (p.stack.items.len == 1) {
	232	return;
	233	}
	234
	235	var value = p.stack.pop();
	236	try p.pushToParent(&value);
	237	},
	238	.object_begin => {
	239	try p.stack.append(Value{ .object = ObjectMap.init(allocator) });
	240	p.state = .object_key;
	241	},
	242	.array_begin => {
	243	try p.stack.append(Value{ .array = Array.init(allocator) });
	244	p.state = .array_value;
	245	},
	246	.string => \|s\| {
	247	try array.append(Value{ .string = s });
	248	},
	249	.allocated_string => \|s\| {
	250	try array.append(Value{ .string = s });
	251	},
	252	.number => \|slice\| {
	253	try array.append(try p.parseNumber(slice));
	254	},
	255	.allocated_number => \|slice\| {
	256	try array.append(try p.parseNumber(slice));
	257	},
	258	.true => {
	259	try array.append(Value{ .bool = true });
	260	},
	261	.false => {
	262	try array.append(Value{ .bool = false });
	263	},
	264	.null => {
	265	try array.append(.null);
	266	},
	267	.object_end, .end_of_document => unreachable,
	268	.partial_number, .partial_string, .partial_string_escaped_1, .partial_string_escaped_2, .partial_string_escaped_3, .partial_string_escaped_4 => unreachable,
	269	}
	270	},
	271	.simple => switch (token) {
	272	.object_begin => {
	273	try p.stack.append(Value{ .object = ObjectMap.init(allocator) });
	274	p.state = .object_key;
	275	},
	276	.array_begin => {
	277	try p.stack.append(Value{ .array = Array.init(allocator) });
	278	p.state = .array_value;
	279	},
	280	.string => \|s\| {
	281	try p.stack.append(Value{ .string = s });
	282	},
	283	.allocated_string => \|s\| {
	284	try p.stack.append(Value{ .string = s });
	285	},
	286	.number => \|slice\| {
	287	try p.stack.append(try p.parseNumber(slice));
	288	},
	289	.allocated_number => \|slice\| {
	290	try p.stack.append(try p.parseNumber(slice));
	291	},
	292	.true => {
	293	try p.stack.append(Value{ .bool = true });
	294	},
	295	.false => {
	296	try p.stack.append(Value{ .bool = false });
	297	},
	298	.null => {
	299	try p.stack.append(.null);
	300	},
	301	.object_end, .array_end, .end_of_document => unreachable,
	302	.partial_number, .partial_string, .partial_string_escaped_1, .partial_string_escaped_2, .partial_string_escaped_3, .partial_string_escaped_4 => unreachable,
	303	},
	304	}
	305	}
	306
	307	fn pushToParent(p: Parser, value: const Value) !void {
	308	switch (p.stack.items[p.stack.items.len - 1]) {
	309	// Object Parent -> [ ..., object, <key>, value ]
	310	.string => \|key\| {
	311	_ = p.stack.pop();
	312
	313	var object = &p.stack.items[p.stack.items.len - 1].object;
	314	try object.put(key, value.*);
	315	p.state = .object_key;
	316	},
	317	// Array Parent -> [ ..., <array>, value ]
	318	.array => \|*array\| {
	319	try array.append(value.*);
	320	p.state = .array_value;
	321	},
	322	else => {
	323	unreachable;
	324	},
	325	}
	326	}
	327
	328	fn parseNumber(p: *Parser, slice: []const u8) !Value {
	329	_ = p;
	330	return if (isNumberFormattedLikeAnInteger(slice))
	331	Value{
	332	.integer = std.fmt.parseInt(i64, slice, 10) catch \|e\| switch (e) {
	333	error.Overflow => return Value{ .number_string = slice },
	334	error.InvalidCharacter => \|err\| return err,
	335	},
	336	}
	337	else
	338	Value{ .float = try std.fmt.parseFloat(f64, slice) };
	339	}
	340	};
	341
	342	test {
	343	_ = @import("dynamic_test.zig");
	344	}

lib/std/json/dynamic_test.zig created+285

...	...	@@ -0,0 +1,285 @@
	1	const std = @import("std");
	2	const mem = std.mem;
	3	const testing = std.testing;
	4
	5	const ObjectMap = @import("dynamic.zig").ObjectMap;
	6	const Array = @import("dynamic.zig").Array;
	7	const Value = @import("dynamic.zig").Value;
	8	const Parser = @import("dynamic.zig").Parser;
	9
	10	test "json.parser.dynamic" {
	11	var p = Parser.init(testing.allocator, .alloc_if_needed);
	12	defer p.deinit();
	13
	14	const s =
	15	\\{
	16	\\ "Image": {
	17	\\ "Width": 800,
	18	\\ "Height": 600,
	19	\\ "Title": "View from 15th Floor",
	20	\\ "Thumbnail": {
	21	\\ "Url": "http://www.example.com/image/481989943",
	22	\\ "Height": 125,
	23	\\ "Width": 100
	24	\\ },
	25	\\ "Animated" : false,
	26	\\ "IDs": [116, 943, 234, 38793],
	27	\\ "ArrayOfObject": [{"n": "m"}],
	28	\\ "double": 1.3412,
	29	\\ "LargeInt": 18446744073709551615
	30	\\ }
	31	\\}
	32	;
	33
	34	var tree = try p.parse(s);
	35	defer tree.deinit();
	36
	37	var root = tree.root;
	38
	39	var image = root.object.get("Image").?;
	40
	41	const width = image.object.get("Width").?;
	42	try testing.expect(width.integer == 800);
	43
	44	const height = image.object.get("Height").?;
	45	try testing.expect(height.integer == 600);
	46
	47	const title = image.object.get("Title").?;
	48	try testing.expect(mem.eql(u8, title.string, "View from 15th Floor"));
	49
	50	const animated = image.object.get("Animated").?;
	51	try testing.expect(animated.bool == false);
	52
	53	const array_of_object = image.object.get("ArrayOfObject").?;
	54	try testing.expect(array_of_object.array.items.len == 1);
	55
	56	const obj0 = array_of_object.array.items[0].object.get("n").?;
	57	try testing.expect(mem.eql(u8, obj0.string, "m"));
	58
	59	const double = image.object.get("double").?;
	60	try testing.expect(double.float == 1.3412);
	61
	62	const large_int = image.object.get("LargeInt").?;
	63	try testing.expect(mem.eql(u8, large_int.number_string, "18446744073709551615"));
	64	}
	65
	66	const writeStream = @import("./write_stream.zig").writeStream;
	67	test "write json then parse it" {
	68	var out_buffer: [1000]u8 = undefined;
	69
	70	var fixed_buffer_stream = std.io.fixedBufferStream(&out_buffer);
	71	const out_stream = fixed_buffer_stream.writer();
	72	var jw = writeStream(out_stream, 4);
	73
	74	try jw.beginObject();
	75
	76	try jw.objectField("f");
	77	try jw.emitBool(false);
	78
	79	try jw.objectField("t");
	80	try jw.emitBool(true);
	81
	82	try jw.objectField("int");
	83	try jw.emitNumber(1234);
	84
	85	try jw.objectField("array");
	86	try jw.beginArray();
	87
	88	try jw.arrayElem();
	89	try jw.emitNull();
	90
	91	try jw.arrayElem();
	92	try jw.emitNumber(12.34);
	93
	94	try jw.endArray();
	95
	96	try jw.objectField("str");
	97	try jw.emitString("hello");
	98
	99	try jw.endObject();
	100
	101	var parser = Parser.init(testing.allocator, .alloc_if_needed);
	102	defer parser.deinit();
	103	var tree = try parser.parse(fixed_buffer_stream.getWritten());
	104	defer tree.deinit();
	105
	106	try testing.expect(tree.root.object.get("f").?.bool == false);
	107	try testing.expect(tree.root.object.get("t").?.bool == true);
	108	try testing.expect(tree.root.object.get("int").?.integer == 1234);
	109	try testing.expect(tree.root.object.get("array").?.array.items[0].null == {});
	110	try testing.expect(tree.root.object.get("array").?.array.items[1].float == 12.34);
	111	try testing.expect(mem.eql(u8, tree.root.object.get("str").?.string, "hello"));
	112	}
	113
	114	fn testParse(arena_allocator: std.mem.Allocator, json_str: []const u8) !Value {
	115	var p = Parser.init(arena_allocator, .alloc_if_needed);
	116	return (try p.parse(json_str)).root;
	117	}
	118
	119	test "parsing empty string gives appropriate error" {
	120	var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
	121	defer arena_allocator.deinit();
	122	try testing.expectError(error.UnexpectedEndOfInput, testParse(arena_allocator.allocator(), ""));
	123	}
	124
	125	test "parse tree should not contain dangling pointers" {
	126	var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
	127	defer arena_allocator.deinit();
	128
	129	var p = Parser.init(arena_allocator.allocator(), .alloc_if_needed);
	130	defer p.deinit();
	131
	132	var tree = try p.parse("[]");
	133	defer tree.deinit();
	134
	135	// Allocation should succeed
	136	var i: usize = 0;
	137	while (i < 100) : (i += 1) {
	138	try tree.root.array.append(Value{ .integer = 100 });
	139	}
	140	try testing.expectEqual(tree.root.array.items.len, 100);
	141	}
	142
	143	test "integer after float has proper type" {
	144	var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
	145	defer arena_allocator.deinit();
	146	const parsed = try testParse(arena_allocator.allocator(),
	147	\\{
	148	\\ "float": 3.14,
	149	\\ "ints": [1, 2, 3]
	150	\\}
	151	);
	152	try std.testing.expect(parsed.object.get("ints").?.array.items[0] == .integer);
	153	}
	154
	155	test "escaped characters" {
	156	var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
	157	defer arena_allocator.deinit();
	158	const input =
	159	\\{
	160	\\ "backslash": "\\",
	161	\\ "forwardslash": "\/",
	162	\\ "newline": "\n",
	163	\\ "carriagereturn": "\r",
	164	\\ "tab": "\t",
	165	\\ "formfeed": "\f",
	166	\\ "backspace": "\b",
	167	\\ "doublequote": "\"",
	168	\\ "unicode": "\u0105",
	169	\\ "surrogatepair": "\ud83d\ude02"
	170	\\}
	171	;
	172
	173	const obj = (try testParse(arena_allocator.allocator(), input)).object;
	174
	175	try testing.expectEqualSlices(u8, obj.get("backslash").?.string, "\\");
	176	try testing.expectEqualSlices(u8, obj.get("forwardslash").?.string, "/");
	177	try testing.expectEqualSlices(u8, obj.get("newline").?.string, "\n");
	178	try testing.expectEqualSlices(u8, obj.get("carriagereturn").?.string, "\r");
	179	try testing.expectEqualSlices(u8, obj.get("tab").?.string, "\t");
	180	try testing.expectEqualSlices(u8, obj.get("formfeed").?.string, "\x0C");
	181	try testing.expectEqualSlices(u8, obj.get("backspace").?.string, "\x08");
	182	try testing.expectEqualSlices(u8, obj.get("doublequote").?.string, "\"");
	183	try testing.expectEqualSlices(u8, obj.get("unicode").?.string, "ą");
	184	try testing.expectEqualSlices(u8, obj.get("surrogatepair").?.string, "😂");
	185	}
	186
	187	test "string copy option" {
	188	const input =
	189	\\{
	190	\\ "noescape": "aą😂",
	191	\\ "simple": "\\\/\n\r\t\f\b\"",
	192	\\ "unicode": "\u0105",
	193	\\ "surrogatepair": "\ud83d\ude02"
	194	\\}
	195	;
	196
	197	var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
	198	defer arena_allocator.deinit();
	199	const allocator = arena_allocator.allocator();
	200
	201	var parser = Parser.init(allocator, .alloc_if_needed);
	202	const tree_nocopy = try parser.parse(input);
	203	const obj_nocopy = tree_nocopy.root.object;
	204
	205	parser = Parser.init(allocator, .alloc_always);
	206	const tree_copy = try parser.parse(input);
	207	const obj_copy = tree_copy.root.object;
	208
	209	for ([_][]const u8{ "noescape", "simple", "unicode", "surrogatepair" }) \|field_name\| {
	210	try testing.expectEqualSlices(u8, obj_nocopy.get(field_name).?.string, obj_copy.get(field_name).?.string);
	211	}
	212
	213	const nocopy_addr = &obj_nocopy.get("noescape").?.string[0];
	214	const copy_addr = &obj_copy.get("noescape").?.string[0];
	215
	216	var found_nocopy = false;
	217	for (input, 0..) \|_, index\| {
	218	try testing.expect(copy_addr != &input[index]);
	219	if (nocopy_addr == &input[index]) {
	220	found_nocopy = true;
	221	}
	222	}
	223	try testing.expect(found_nocopy);
	224	}
	225
	226	test "Value.jsonStringify" {
	227	{
	228	var buffer: [10]u8 = undefined;
	229	var fbs = std.io.fixedBufferStream(&buffer);
	230	try @as(Value, .null).jsonStringify(.{}, fbs.writer());
	231	try testing.expectEqualSlices(u8, fbs.getWritten(), "null");
	232	}
	233	{
	234	var buffer: [10]u8 = undefined;
	235	var fbs = std.io.fixedBufferStream(&buffer);
	236	try (Value{ .bool = true }).jsonStringify(.{}, fbs.writer());
	237	try testing.expectEqualSlices(u8, fbs.getWritten(), "true");
	238	}
	239	{
	240	var buffer: [10]u8 = undefined;
	241	var fbs = std.io.fixedBufferStream(&buffer);
	242	try (Value{ .integer = 42 }).jsonStringify(.{}, fbs.writer());
	243	try testing.expectEqualSlices(u8, fbs.getWritten(), "42");
	244	}
	245	{
	246	var buffer: [10]u8 = undefined;
	247	var fbs = std.io.fixedBufferStream(&buffer);
	248	try (Value{ .number_string = "43" }).jsonStringify(.{}, fbs.writer());
	249	try testing.expectEqualSlices(u8, fbs.getWritten(), "43");
	250	}
	251	{
	252	var buffer: [10]u8 = undefined;
	253	var fbs = std.io.fixedBufferStream(&buffer);
	254	try (Value{ .float = 42 }).jsonStringify(.{}, fbs.writer());
	255	try testing.expectEqualSlices(u8, fbs.getWritten(), "4.2e+01");
	256	}
	257	{
	258	var buffer: [10]u8 = undefined;
	259	var fbs = std.io.fixedBufferStream(&buffer);
	260	try (Value{ .string = "weeee" }).jsonStringify(.{}, fbs.writer());
	261	try testing.expectEqualSlices(u8, fbs.getWritten(), "\"weeee\"");
	262	}
	263	{
	264	var buffer: [10]u8 = undefined;
	265	var fbs = std.io.fixedBufferStream(&buffer);
	266	var vals = [_]Value{
	267	.{ .integer = 1 },
	268	.{ .integer = 2 },
	269	.{ .number_string = "3" },
	270	};
	271	try (Value{
	272	.array = Array.fromOwnedSlice(undefined, &vals),
	273	}).jsonStringify(.{}, fbs.writer());
	274	try testing.expectEqualSlices(u8, fbs.getWritten(), "[1,2,3]");
	275	}
	276	{
	277	var buffer: [10]u8 = undefined;
	278	var fbs = std.io.fixedBufferStream(&buffer);
	279	var obj = ObjectMap.init(testing.allocator);
	280	defer obj.deinit();
	281	try obj.putNoClobber("a", .{ .string = "b" });
	282	try (Value{ .object = obj }).jsonStringify(.{}, fbs.writer());
	283	try testing.expectEqualSlices(u8, fbs.getWritten(), "{\"a\":\"b\"}");
	284	}
	285	}

lib/std/json/scanner.zig created+1764

...	...	@@ -0,0 +1,1764 @@
	1	// Notes on standards compliance: https://datatracker.ietf.org/doc/html/rfc8259
	2	// * RFC 8259 requires JSON documents be valid UTF-8,
	3	// but makes an allowance for systems that are "part of a closed ecosystem".
	4	// I have no idea what that's supposed to mean in the context of a standard specification.
	5	// This implementation requires inputs to be valid UTF-8.
	6	// * RFC 8259 contradicts itself regarding whether lowercase is allowed in \u hex digits,
	7	// but this is probably a bug in the spec, and it's clear that lowercase is meant to be allowed.
	8	// (RFC 5234 defines HEXDIG to only allow uppercase.)
	9	// * When RFC 8259 refers to a "character", I assume they really mean a "Unicode scalar value".
	10	// See http://www.unicode.org/glossary/#unicode_scalar_value .
	11	// * RFC 8259 doesn't explicitly disallow unpaired surrogate halves in \u escape sequences,
	12	// but vaguely implies that \u escapes are for encoding Unicode "characters" (i.e. Unicode scalar values?),
	13	// which would mean that unpaired surrogate halves are forbidden.
	14	// By contrast ECMA-404 (a competing(/compatible?) JSON standard, which JavaScript's JSON.parse() conforms to)
	15	// explicitly allows unpaired surrogate halves.
	16	// This implementation forbids unpaired surrogate halves in \u sequences.
	17	// If a high surrogate half appears in a \u sequence,
	18	// then a low surrogate half must immediately follow in \u notation.
	19	// * RFC 8259 allows implementations to "accept non-JSON forms or extensions".
	20	// This implementation does not accept any of that.
	21	// * RFC 8259 allows implementations to put limits on "the size of texts",
	22	// "the maximum depth of nesting", "the range and precision of numbers",
	23	// and "the length and character contents of strings".
	24	// This low-level implementation does not limit these,
	25	// except where noted above, and except that nesting depth requires memory allocation.
	26	// Note that this low-level API does not interpret numbers numerically,
	27	// but simply emits their source form for some higher level code to make sense of.
	28	// * This low-level implementation allows duplicate object keys,
	29	// and key/value pairs are emitted in the order they appear in the input.
	30
	31	const std = @import("std");
	32
	33	const Allocator = std.mem.Allocator;
	34	const ArrayList = std.ArrayList;
	35	const assert = std.debug.assert;
	36
	37	/// Scan the input and check for malformed JSON.
	38	/// On `SyntaxError` or `UnexpectedEndOfInput`, returns `false`.
	39	/// Returns any errors from the allocator as-is, which is unlikely,
	40	/// but can be caused by extreme nesting depth in the input.
	41	pub fn validate(allocator: Allocator, s: []const u8) Allocator.Error!bool {
	42	var scanner = Scanner.initCompleteInput(allocator, s);
	43	defer scanner.deinit();
	44
	45	while (true) {
	46	const token = scanner.next() catch \|err\| switch (err) {
	47	error.SyntaxError, error.UnexpectedEndOfInput => return false,
	48	error.OutOfMemory => return error.OutOfMemory,
	49	error.BufferUnderrun => unreachable,
	50	};
	51	if (token == .end_of_document) break;
	52	}
	53
	54	return true;
	55	}
	56
	57	/// The parsing errors are divided into two categories:
	58	/// * `SyntaxError` is for clearly malformed JSON documents,
	59	/// such as giving an input document that isn't JSON at all.
	60	/// * `UnexpectedEndOfInput` is for signaling that everything's been
	61	/// valid so far, but the input appears to be truncated for some reason.
	62	/// Note that a completely empty (or whitespace-only) input will give `UnexpectedEndOfInput`.
	63	pub const Error = error{ SyntaxError, UnexpectedEndOfInput };
	64
	65	/// Calls `std.json.Reader` with `std.json.default_buffer_size`.
	66	pub fn reader(allocator: Allocator, io_reader: anytype) Reader(default_buffer_size, @TypeOf(io_reader)) {
	67	return Reader(default_buffer_size, @TypeOf(io_reader)).init(allocator, io_reader);
	68	}
	69	/// Used by `json.reader`.
	70	pub const default_buffer_size = 0x1000;
	71
	72	/// The tokens emitted by `std.json.Scanner` and `std.json.Reader` `.next*()` functions follow this grammar:
	73	/// ```
	74	/// <document> = <value> .end_of_document
	75	/// <value> =
	76	/// \| <object>
	77	/// \| <array>
	78	/// \| <number>
	79	/// \| <string>
	80	/// \| .true
	81	/// \| .false
	82	/// \| .null
	83	/// <object> = .object_begin ( <string> <value> )* .object_end
	84	/// <array> = .array_begin ( <value> )* .array_end
	85	/// <number> = <It depends. See below.>
	86	/// <string> = <It depends. See below.>
	87	/// ```
	88	///
	89	/// What you get for `<number>` and `<string>` values depends on which `next*()` method you call:
	90	///
	91	/// ```
	92	/// next():
	93	/// <number> = ( .partial_number )* .number
	94	/// <string> = ( <partial_string> )* .string
	95	/// <partial_string> =
	96	/// \| .partial_string
	97	/// \| .partial_string_escaped_1
	98	/// \| .partial_string_escaped_2
	99	/// \| .partial_string_escaped_3
	100	/// \| .partial_string_escaped_4
	101	///
	102	/// nextAlloc*(..., .alloc_always):
	103	/// <number> = .allocated_number
	104	/// <string> = .allocated_string
	105	///
	106	/// nextAlloc*(..., .alloc_if_needed):
	107	/// <number> =
	108	/// \| .number
	109	/// \| .allocated_number
	110	/// <string> =
	111	/// \| .string
	112	/// \| .allocated_string
	113	/// ```
	114	///
	115	/// For all tokens with a `[]const u8`, `[]u8`, or `[n]u8` payload, the payload represents the content of the value.
	116	/// For number values, this is the representation of the number exactly as it appears in the input.
	117	/// For strings, this is the content of the string after resolving escape sequences.
	118	///
	119	/// For `.allocated_number` and `.allocated_string`, the `[]u8` payloads are allocations made with the given allocator.
	120	/// You are responsible for managing that memory. `json.Reader.deinit()` does not free those allocations.
	121	///
	122	/// The `.partial_*` tokens indicate that a value spans multiple input buffers or that a string contains escape sequences.
	123	/// To get a complete value in memory, you need to concatenate the values yourself.
	124	/// Calling `nextAlloc()` does this for you, and returns an `.allocated_` token with the result.
	125	///
	126	/// For tokens with a `[]const u8` payload, the payload is a slice into the current input buffer.
	127	/// The memory may become undefined during the next call to `json.Scanner.feedInput()`
	128	/// or any `json.Reader` method whose return error set includes `json.Error`.
	129	/// To keep the value persistently, it recommended to make a copy or to use `.alloc_always`,
	130	/// which makes a copy for you.
	131	///
	132	/// Note that `.number` and `.string` tokens that follow `.partial_*` tokens may have `0` length to indicate that
	133	/// the previously partial value is completed with no additional bytes.
	134	/// (This can happen when the break between input buffers happens to land on the exact end of a value. E.g. `"[1234"`, `"]"`.)
	135	/// `.partial_*` tokens never have `0` length.
	136	///
	137	/// The recommended strategy for using the different `next*()` methods is something like this:
	138	///
	139	/// When you're expecting an object key, use `.alloc_if_needed`.
	140	/// You often don't need a copy of the key string to persist; you might just check which field it is.
	141	/// In the case that the key happens to require an allocation, free it immediately after checking it.
	142	///
	143	/// When you're expecting a meaningful string value (such as on the right of a `:`),
	144	/// use `.alloc_always` in order to keep the value valid throughout parsing the rest of the document.
	145	///
	146	/// When you're expecting a number value, use `.alloc_if_needed`.
	147	/// You're probably going to be parsing the string representation of the number into a numeric representation,
	148	/// so you need the complete string representation only temporarily.
	149	///
	150	/// When you're skipping an unrecognized value, use `skipValue()`.
	151	pub const Token = union(enum) {
	152	object_begin,
	153	object_end,
	154	array_begin,
	155	array_end,
	156
	157	true,
	158	false,
	159	null,
	160
	161	number: []const u8,
	162	partial_number: []const u8,
	163	allocated_number: []u8,
	164
	165	string: []const u8,
	166	partial_string: []const u8,
	167	partial_string_escaped_1: [1]u8,
	168	partial_string_escaped_2: [2]u8,
	169	partial_string_escaped_3: [3]u8,
	170	partial_string_escaped_4: [4]u8,
	171	allocated_string: []u8,
	172
	173	end_of_document,
	174	};
	175
	176	/// This is only used in `peekNextTokenType()` and gives a categorization based on the first byte of the next token that will be emitted from a `next*()` call.
	177	pub const TokenType = enum {
	178	object_begin,
	179	object_end,
	180	array_begin,
	181	array_end,
	182	true,
	183	false,
	184	null,
	185	number,
	186	string,
	187	end_of_document,
	188	};
	189
	190	/// To enable diagnostics, declare `var diagnostics = Diagnostics{};` then call `source.enableDiagnostics(&diagnostics);`
	191	/// where `source` is either a `std.json.Reader` or a `std.json.Scanner` that has just been initialized.
	192	/// At any time, notably just after an error, call `getLine()`, `getColumn()`, and/or `getByteOffset()`
	193	/// to get meaningful information from this.
	194	pub const Diagnostics = struct {
	195	line_number: u64 = 1,
	196	line_start_cursor: usize = @bitCast(usize, @as(isize, -1)), // Start just "before" the input buffer to get a 1-based column for line 1.
	197	total_bytes_before_current_input: u64 = 0,
	198	cursor_pointer: *const usize = undefined,
	199
	200	/// Starts at 1.
	201	pub fn getLine(self: *const @This()) u64 {
	202	return self.line_number;
	203	}
	204	/// Starts at 1.
	205	pub fn getColumn(self: *const @This()) u64 {
	206	return self.cursor_pointer.* -% self.line_start_cursor;
	207	}
	208	/// Starts at 0. Measures the byte offset since the start of the input.
	209	pub fn getByteOffset(self: *const @This()) u64 {
	210	return self.total_bytes_before_current_input + self.cursor_pointer.*;
	211	}
	212	};
	213
	214	/// See the documentation for `std.json.Token`.
	215	pub const AllocWhen = enum { alloc_if_needed, alloc_always };
	216
	217	/// For security, the maximum size allocated to store a single string or number value is limited to 4MiB by default.
	218	/// This limit can be specified by calling `nextAllocMax()` instead of `nextAlloc()`.
	219	pub const default_max_value_len = 4 * 1024 * 1024;
	220
	221	/// Connects a `std.io.Reader` to a `std.json.Scanner`.
	222	/// All `next*()` methods here handle `error.BufferUnderrun` from `std.json.Scanner`, and then read from the reader.
	223	pub fn Reader(comptime buffer_size: usize, comptime ReaderType: type) type {
	224	return struct {
	225	scanner: Scanner,
	226	reader: ReaderType,
	227
	228	buffer: [buffer_size]u8 = undefined,
	229
	230	/// The allocator is only used to track `[]` and `{}` nesting levels.
	231	pub fn init(allocator: Allocator, io_reader: ReaderType) @This() {
	232	return .{
	233	.scanner = Scanner.initStreaming(allocator),
	234	.reader = io_reader,
	235	};
	236	}
	237	pub fn deinit(self: *@This()) void {
	238	self.scanner.deinit();
	239	self.* = undefined;
	240	}
	241
	242	/// Calls `std.json.Scanner.enableDiagnostics`.
	243	pub fn enableDiagnostics(self: @This(), diagnostics: Diagnostics) void {
	244	self.scanner.enableDiagnostics(diagnostics);
	245	}
	246
	247	pub const NextError = ReaderType.Error \|\| Error \|\| Allocator.Error;
	248	pub const SkipError = NextError;
	249	pub const AllocError = NextError \|\| error{ValueTooLong};
	250	pub const PeekError = ReaderType.Error \|\| Error;
	251
	252	/// Equivalent to `nextAllocMax(allocator, when, default_max_value_len);`
	253	/// See also `std.json.Token` for documentation of `nextAlloc*()` function behavior.
	254	pub fn nextAlloc(self: *@This(), allocator: Allocator, when: AllocWhen) AllocError!Token {
	255	return self.nextAllocMax(allocator, when, default_max_value_len);
	256	}
	257	/// See also `std.json.Token` for documentation of `nextAlloc*()` function behavior.
	258	pub fn nextAllocMax(self: *@This(), allocator: Allocator, when: AllocWhen, max_value_len: usize) AllocError!Token {
	259	const token_type = try self.peekNextTokenType();
	260	switch (token_type) {
	261	.number, .string => {
	262	var value_list = ArrayList(u8).init(allocator);
	263	errdefer {
	264	value_list.deinit();
	265	}
	266	if (try self.allocNextIntoArrayListMax(&value_list, when, max_value_len)) \|slice\| {
	267	return if (token_type == .number)
	268	Token{ .number = slice }
	269	else
	270	Token{ .string = slice };
	271	} else {
	272	return if (token_type == .number)
	273	Token{ .allocated_number = try value_list.toOwnedSlice() }
	274	else
	275	Token{ .allocated_string = try value_list.toOwnedSlice() };
	276	}
	277	},
	278
	279	// Simple tokens never alloc.
	280	.object_begin,
	281	.object_end,
	282	.array_begin,
	283	.array_end,
	284	.true,
	285	.false,
	286	.null,
	287	.end_of_document,
	288	=> return try self.next(),
	289	}
	290	}
	291
	292	/// Equivalent to `allocNextIntoArrayListMax(value_list, when, default_max_value_len);`
	293	pub fn allocNextIntoArrayList(self: @This(), value_list: ArrayList(u8), when: AllocWhen) AllocError!?[]const u8 {
	294	return self.allocNextIntoArrayListMax(value_list, when, default_max_value_len);
	295	}
	296	/// Calls `std.json.Scanner.allocNextIntoArrayListMax` and handles `error.BufferUnderrun`.
	297	pub fn allocNextIntoArrayListMax(self: @This(), value_list: ArrayList(u8), when: AllocWhen, max_value_len: usize) AllocError!?[]const u8 {
	298	while (true) {
	299	return self.scanner.allocNextIntoArrayListMax(value_list, when, max_value_len) catch \|err\| switch (err) {
	300	error.BufferUnderrun => {
	301	try self.refillBuffer();
	302	continue;
	303	},
	304	else => \|other_err\| return other_err,
	305	};
	306	}
	307	}
	308
	309	/// Like `std.json.Scanner.skipValue`, but handles `error.BufferUnderrun`.
	310	pub fn skipValue(self: *@This()) SkipError!void {
	311	switch (try self.peekNextTokenType()) {
	312	.object_begin, .array_begin => {
	313	try self.skipUntilStackHeight(self.stackHeight());
	314	},
	315	.number, .string => {
	316	while (true) {
	317	switch (try self.next()) {
	318	.partial_number,
	319	.partial_string,
	320	.partial_string_escaped_1,
	321	.partial_string_escaped_2,
	322	.partial_string_escaped_3,
	323	.partial_string_escaped_4,
	324	=> continue,
	325
	326	.number, .string => break,
	327
	328	else => unreachable,
	329	}
	330	}
	331	},
	332	.true, .false, .null => {
	333	_ = try self.next();
	334	},
	335
	336	.object_end, .array_end, .end_of_document => unreachable, // Attempt to skip a non-value token.
	337	}
	338	}
	339	/// Like `std.json.Scanner.skipUntilStackHeight()` but handles `error.BufferUnderrun`.
	340	pub fn skipUntilStackHeight(self: *@This(), terminal_stack_height: u32) NextError!void {
	341	while (true) {
	342	return self.scanner.skipUntilStackHeight(terminal_stack_height) catch \|err\| switch (err) {
	343	error.BufferUnderrun => {
	344	try self.refillBuffer();
	345	continue;
	346	},
	347	else => \|other_err\| return other_err,
	348	};
	349	}
	350	}
	351
	352	/// Calls `std.json.Scanner.stackHeight`.
	353	pub fn stackHeight(self: *const @This()) u32 {
	354	return self.scanner.stackHeight();
	355	}
	356	/// Calls `std.json.Scanner.ensureTotalStackCapacity`.
	357	pub fn ensureTotalStackCapacity(self: *@This(), height: u32) Allocator.Error!void {
	358	try self.scanner.ensureTotalStackCapacity(height);
	359	}
	360
	361	/// See `std.json.Token` for documentation of this function.
	362	pub fn next(self: *@This()) NextError!Token {
	363	while (true) {
	364	return self.scanner.next() catch \|err\| switch (err) {
	365	error.BufferUnderrun => {
	366	try self.refillBuffer();
	367	continue;
	368	},
	369	else => \|other_err\| return other_err,
	370	};
	371	}
	372	}
	373
	374	/// See `std.json.Scanner.peekNextTokenType()`.
	375	pub fn peekNextTokenType(self: *@This()) PeekError!TokenType {
	376	while (true) {
	377	return self.scanner.peekNextTokenType() catch \|err\| switch (err) {
	378	error.BufferUnderrun => {
	379	try self.refillBuffer();
	380	continue;
	381	},
	382	else => \|other_err\| return other_err,
	383	};
	384	}
	385	}
	386
	387	fn refillBuffer(self: *@This()) ReaderType.Error!void {
	388	const input = self.buffer[0..try self.reader.read(self.buffer[0..])];
	389	if (input.len > 0) {
	390	self.scanner.feedInput(input);
	391	} else {
	392	self.scanner.endInput();
	393	}
	394	}
	395	};
	396	}
	397
	398	/// The lowest level parsing API in this package;
	399	/// supports streaming input with a low memory footprint.
	400	/// The memory requirement is `O(d)` where d is the nesting depth of `[]` or `{}` containers in the input.
	401	/// Specifically `d/8` bytes are required for this purpose,
	402	/// with some extra buffer according to the implementation of `std.ArrayList`.
	403	///
	404	/// This scanner can emit partial tokens; see `std.json.Token`.
	405	/// The input to this class is a sequence of input buffers that you must supply one at a time.
	406	/// Call `feedInput()` with the first buffer, then call `next()` repeatedly until `error.BufferUnderrun` is returned.
	407	/// Then call `feedInput()` again and so forth.
	408	/// Call `endInput()` when the last input buffer has been given to `feedInput()`, either immediately after calling `feedInput()`,
	409	/// or when `error.BufferUnderrun` requests more data and there is no more.
	410	/// Be sure to call `next()` after calling `endInput()` until `Token.end_of_document` has been returned.
	411	pub const Scanner = struct {
	412	state: State = .value,
	413	string_is_object_key: bool = false,
	414	stack: BitStack,
	415	value_start: usize = undefined,
	416	unicode_code_point: u21 = undefined,
	417
	418	input: []const u8 = "",
	419	cursor: usize = 0,
	420	is_end_of_input: bool = false,
	421	diagnostics: ?*Diagnostics = null,
	422
	423	/// The allocator is only used to track `[]` and `{}` nesting levels.
	424	pub fn initStreaming(allocator: Allocator) @This() {
	425	return .{
	426	.stack = BitStack.init(allocator),
	427	};
	428	}
	429	/// Use this if your input is a single slice.
	430	/// This is effectively equivalent to:
	431	/// ```
	432	/// initStreaming(allocator);
	433	/// feedInput(complete_input);
	434	/// endInput();
	435	/// ```
	436	pub fn initCompleteInput(allocator: Allocator, complete_input: []const u8) @This() {
	437	return .{
	438	.stack = BitStack.init(allocator),
	439	.input = complete_input,
	440	.is_end_of_input = true,
	441	};
	442	}
	443	pub fn deinit(self: *@This()) void {
	444	self.stack.deinit();
	445	self.* = undefined;
	446	}
	447
	448	pub fn enableDiagnostics(self: @This(), diagnostics: Diagnostics) void {
	449	diagnostics.cursor_pointer = &self.cursor;
	450	self.diagnostics = diagnostics;
	451	}
	452
	453	/// Call this whenever you get `error.BufferUnderrun` from `next()`.
	454	/// When there is no more input to provide, call `endInput()`.
	455	pub fn feedInput(self: *@This(), input: []const u8) void {
	456	assert(self.cursor == self.input.len); // Not done with the last input slice.
	457	if (self.diagnostics) \|diag\| {
	458	diag.total_bytes_before_current_input += self.input.len;
	459	// This usually goes "negative" to measure how far before the beginning
	460	// of the new buffer the current line started.
	461	diag.line_start_cursor -%= self.cursor;
	462	}
	463	self.input = input;
	464	self.cursor = 0;
	465	self.value_start = 0;
	466	}
	467	/// Call this when you will no longer call `feedInput()` anymore.
	468	/// This can be called either immediately after the last `feedInput()`,
	469	/// or at any time afterward, such as when getting `error.BufferUnderrun` from `next()`.
	470	/// Don't forget to call `next*()` after `endInput()` until you get `.end_of_document`.
	471	pub fn endInput(self: *@This()) void {
	472	self.is_end_of_input = true;
	473	}
	474
	475	pub const NextError = Error \|\| Allocator.Error \|\| error{BufferUnderrun};
	476	pub const AllocError = Error \|\| Allocator.Error \|\| error{ValueTooLong};
	477	pub const PeekError = Error \|\| error{BufferUnderrun};
	478	pub const SkipError = Error \|\| Allocator.Error;
	479	pub const AllocIntoArrayListError = AllocError \|\| error{BufferUnderrun};
	480
	481	/// Equivalent to `nextAllocMax(allocator, when, default_max_value_len);`
	482	/// This function is only available after `endInput()` (or `initCompleteInput()`) has been called.
	483	/// See also `std.json.Token` for documentation of `nextAlloc*()` function behavior.
	484	pub fn nextAlloc(self: *@This(), allocator: Allocator, when: AllocWhen) AllocError!Token {
	485	return self.nextAllocMax(allocator, when, default_max_value_len);
	486	}
	487
	488	/// This function is only available after `endInput()` (or `initCompleteInput()`) has been called.
	489	/// See also `std.json.Token` for documentation of `nextAlloc*()` function behavior.
	490	pub fn nextAllocMax(self: *@This(), allocator: Allocator, when: AllocWhen, max_value_len: usize) AllocError!Token {
	491	assert(self.is_end_of_input); // This function is not available in streaming mode.
	492	const token_type = self.peekNextTokenType() catch \|e\| switch (e) {
	493	error.BufferUnderrun => unreachable,
	494	else => \|err\| return err,
	495	};
	496	switch (token_type) {
	497	.number, .string => {
	498	var value_list = ArrayList(u8).init(allocator);
	499	errdefer {
	500	value_list.deinit();
	501	}
	502	if (self.allocNextIntoArrayListMax(&value_list, when, max_value_len) catch \|e\| switch (e) {
	503	error.BufferUnderrun => unreachable,
	504	else => \|err\| return err,
	505	}) \|slice\| {
	506	return if (token_type == .number)
	507	Token{ .number = slice }
	508	else
	509	Token{ .string = slice };
	510	} else {
	511	return if (token_type == .number)
	512	Token{ .allocated_number = try value_list.toOwnedSlice() }
	513	else
	514	Token{ .allocated_string = try value_list.toOwnedSlice() };
	515	}
	516	},
	517
	518	// Simple tokens never alloc.
	519	.object_begin,
	520	.object_end,
	521	.array_begin,
	522	.array_end,
	523	.true,
	524	.false,
	525	.null,
	526	.end_of_document,
	527	=> return self.next() catch \|e\| switch (e) {
	528	error.BufferUnderrun => unreachable,
	529	else => \|err\| return err,
	530	},
	531	}
	532	}
	533
	534	/// Equivalent to `allocNextIntoArrayListMax(value_list, when, default_max_value_len);`
	535	pub fn allocNextIntoArrayList(self: @This(), value_list: ArrayList(u8), when: AllocWhen) AllocIntoArrayListError!?[]const u8 {
	536	return self.allocNextIntoArrayListMax(value_list, when, default_max_value_len);
	537	}
	538	/// The next token type must be either `.number` or `.string`. See `peekNextTokenType()`.
	539	/// When allocation is not necessary with `.alloc_if_needed`,
	540	/// this method returns the content slice from the input buffer, and `value_list` is not touched.
	541	/// When allocation is necessary or with `.alloc_always`, this method concatenates partial tokens into the given `value_list`,
	542	/// and returns `null` once the final `.number` or `.string` token has been written into it.
	543	/// In case of an `error.BufferUnderrun`, partial values will be left in the given value_list.
	544	/// The given `value_list` is never reset by this method, so an `error.BufferUnderrun` situation
	545	/// can be resumed by passing the same array list in again.
	546	/// This method does not indicate whether the token content being returned is for a `.number` or `.string` token type;
	547	/// the caller of this method is expected to know which type of token is being processed.
	548	pub fn allocNextIntoArrayListMax(self: @This(), value_list: ArrayList(u8), when: AllocWhen, max_value_len: usize) AllocIntoArrayListError!?[]const u8 {
	549	while (true) {
	550	const token = try self.next();
	551	switch (token) {
	552	// Accumulate partial values.
	553	.partial_number, .partial_string => \|slice\| {
	554	try appendSlice(value_list, slice, max_value_len);
	555	},
	556	.partial_string_escaped_1 => \|buf\| {
	557	try appendSlice(value_list, buf[0..], max_value_len);
	558	},
	559	.partial_string_escaped_2 => \|buf\| {
	560	try appendSlice(value_list, buf[0..], max_value_len);
	561	},
	562	.partial_string_escaped_3 => \|buf\| {
	563	try appendSlice(value_list, buf[0..], max_value_len);
	564	},
	565	.partial_string_escaped_4 => \|buf\| {
	566	try appendSlice(value_list, buf[0..], max_value_len);
	567	},
	568
	569	// Return complete values.
	570	.number => \|slice\| {
	571	if (when == .alloc_if_needed and value_list.items.len == 0) {
	572	// No alloc necessary.
	573	return slice;
	574	}
	575	try appendSlice(value_list, slice, max_value_len);
	576	// The token is complete.
	577	return null;
	578	},
	579	.string => \|slice\| {
	580	if (when == .alloc_if_needed and value_list.items.len == 0) {
	581	// No alloc necessary.
	582	return slice;
	583	}
	584	try appendSlice(value_list, slice, max_value_len);
	585	// The token is complete.
	586	return null;
	587	},
	588
	589	.object_begin,
	590	.object_end,
	591	.array_begin,
	592	.array_end,
	593	.true,
	594	.false,
	595	.null,
	596	.end_of_document,
	597	=> unreachable, // Only .number and .string token types are allowed here. Check peekNextTokenType() before calling this.
	598
	599	.allocated_number, .allocated_string => unreachable,
	600	}
	601	}
	602	}
	603
	604	/// This function is only available after `endInput()` (or `initCompleteInput()`) has been called.
	605	/// If the next token type is `.object_begin` or `.array_begin`,
	606	/// this function calls `next()` repeatedly until the corresponding `.object_end` or `.array_end` is found.
	607	/// If the next token type is `.number` or `.string`,
	608	/// this function calls `next()` repeatedly until the (non `.partial_*`) `.number` or `.string` token is found.
	609	/// If the next token type is `.true`, `.false`, or `.null`, this function calls `next()` once.
	610	/// The next token type must not be `.object_end`, `.array_end`, or `.end_of_document`;
	611	/// see `peekNextTokenType()`.
	612	pub fn skipValue(self: *@This()) SkipError!void {
	613	assert(self.is_end_of_input); // This function is not available in streaming mode.
	614	switch (self.peekNextTokenType() catch \|e\| switch (e) {
	615	error.BufferUnderrun => unreachable,
	616	else => \|err\| return err,
	617	}) {
	618	.object_begin, .array_begin => {
	619	self.skipUntilStackHeight(self.stackHeight()) catch \|e\| switch (e) {
	620	error.BufferUnderrun => unreachable,
	621	else => \|err\| return err,
	622	};
	623	},
	624	.number, .string => {
	625	while (true) {
	626	switch (self.next() catch \|e\| switch (e) {
	627	error.BufferUnderrun => unreachable,
	628	else => \|err\| return err,
	629	}) {
	630	.partial_number,
	631	.partial_string,
	632	.partial_string_escaped_1,
	633	.partial_string_escaped_2,
	634	.partial_string_escaped_3,
	635	.partial_string_escaped_4,
	636	=> continue,
	637
	638	.number, .string => break,
	639
	640	else => unreachable,
	641	}
	642	}
	643	},
	644	.true, .false, .null => {
	645	_ = self.next() catch \|e\| switch (e) {
	646	error.BufferUnderrun => unreachable,
	647	else => \|err\| return err,
	648	};
	649	},
	650
	651	.object_end, .array_end, .end_of_document => unreachable, // Attempt to skip a non-value token.
	652	}
	653	}
	654
	655	/// Skip tokens until an `.object_end` or `.array_end` token results in a `stackHeight()` equal the given stack height.
	656	/// Unlike `skipValue()`, this function is available in streaming mode.
	657	pub fn skipUntilStackHeight(self: *@This(), terminal_stack_height: u32) NextError!void {
	658	while (true) {
	659	switch (try self.next()) {
	660	.object_end, .array_end => {
	661	if (self.stackHeight() == terminal_stack_height) break;
	662	},
	663	.end_of_document => unreachable,
	664	else => continue,
	665	}
	666	}
	667	}
	668
	669	/// The depth of `{}` or `[]` nesting levels at the current position.
	670	pub fn stackHeight(self: *const @This()) u32 {
	671	return self.stack.bit_len;
	672	}
	673
	674	/// Pre allocate memory to hold the given number of nesting levels.
	675	/// `stackHeight()` up to the given number will not cause allocations.
	676	pub fn ensureTotalStackCapacity(self: *@This(), height: u32) Allocator.Error!void {
	677	try self.stack.ensureTotalCapacity(height);
	678	}
	679
	680	/// See `std.json.Token` for documentation of this function.
	681	pub fn next(self: *@This()) NextError!Token {
	682	state_loop: while (true) {
	683	switch (self.state) {
	684	.value => {
	685	switch (try self.skipWhitespaceExpectByte()) {
	686	// Object, Array
	687	'{' => {
	688	try self.stack.push(OBJECT_MODE);
	689	self.cursor += 1;
	690	self.state = .object_start;
	691	return .object_begin;
	692	},
	693	'[' => {
	694	try self.stack.push(ARRAY_MODE);
	695	self.cursor += 1;
	696	self.state = .array_start;
	697	return .array_begin;
	698	},
	699
	700	// String
	701	'"' => {
	702	self.cursor += 1;
	703	self.value_start = self.cursor;
	704	self.state = .string;
	705	continue :state_loop;
	706	},
	707
	708	// Number
	709	'1'...'9' => {
	710	self.value_start = self.cursor;
	711	self.cursor += 1;
	712	self.state = .number_int;
	713	continue :state_loop;
	714	},
	715	'0' => {
	716	self.value_start = self.cursor;
	717	self.cursor += 1;
	718	self.state = .number_leading_zero;
	719	continue :state_loop;
	720	},
	721	'-' => {
	722	self.value_start = self.cursor;
	723	self.cursor += 1;
	724	self.state = .number_minus;
	725	continue :state_loop;
	726	},
	727
	728	// literal values
	729	't' => {
	730	self.cursor += 1;
	731	self.state = .literal_t;
	732	continue :state_loop;
	733	},
	734	'f' => {
	735	self.cursor += 1;
	736	self.state = .literal_f;
	737	continue :state_loop;
	738	},
	739	'n' => {
	740	self.cursor += 1;
	741	self.state = .literal_n;
	742	continue :state_loop;
	743	},
	744
	745	else => return error.SyntaxError,
	746	}
	747	},
	748
	749	.post_value => {
	750	if (try self.skipWhitespaceCheckEnd()) return .end_of_document;
	751
	752	const c = self.input[self.cursor];
	753	if (self.string_is_object_key) {
	754	self.string_is_object_key = false;
	755	switch (c) {
	756	':' => {
	757	self.cursor += 1;
	758	self.state = .value;
	759	continue :state_loop;
	760	},
	761	else => return error.SyntaxError,
	762	}
	763	}
	764
	765	switch (c) {
	766	'}' => {
	767	if (self.stack.pop() != OBJECT_MODE) return error.SyntaxError;
	768	self.cursor += 1;
	769	// stay in .post_value state.
	770	return .object_end;
	771	},
	772	']' => {
	773	if (self.stack.pop() != ARRAY_MODE) return error.SyntaxError;
	774	self.cursor += 1;
	775	// stay in .post_value state.
	776	return .array_end;
	777	},
	778	',' => {
	779	switch (self.stack.peek()) {
	780	OBJECT_MODE => {
	781	self.state = .object_post_comma;
	782	},
	783	ARRAY_MODE => {
	784	self.state = .value;
	785	},
	786	}
	787	self.cursor += 1;
	788	continue :state_loop;
	789	},
	790	else => return error.SyntaxError,
	791	}
	792	},
	793
	794	.object_start => {
	795	switch (try self.skipWhitespaceExpectByte()) {
	796	'"' => {
	797	self.cursor += 1;
	798	self.value_start = self.cursor;
	799	self.state = .string;
	800	self.string_is_object_key = true;
	801	continue :state_loop;
	802	},
	803	'}' => {
	804	self.cursor += 1;
	805	_ = self.stack.pop();
	806	self.state = .post_value;
	807	return .object_end;
	808	},
	809	else => return error.SyntaxError,
	810	}
	811	},
	812	.object_post_comma => {
	813	switch (try self.skipWhitespaceExpectByte()) {
	814	'"' => {
	815	self.cursor += 1;
	816	self.value_start = self.cursor;
	817	self.state = .string;
	818	self.string_is_object_key = true;
	819	continue :state_loop;
	820	},
	821	else => return error.SyntaxError,
	822	}
	823	},
	824
	825	.array_start => {
	826	switch (try self.skipWhitespaceExpectByte()) {
	827	']' => {
	828	self.cursor += 1;
	829	_ = self.stack.pop();
	830	self.state = .post_value;
	831	return .array_end;
	832	},
	833	else => {
	834	self.state = .value;
	835	continue :state_loop;
	836	},
	837	}
	838	},
	839
	840	.number_minus => {
	841	if (self.cursor >= self.input.len) return self.endOfBufferInNumber(false);
	842	switch (self.input[self.cursor]) {
	843	'0' => {
	844	self.cursor += 1;
	845	self.state = .number_leading_zero;
	846	continue :state_loop;
	847	},
	848	'1'...'9' => {
	849	self.cursor += 1;
	850	self.state = .number_int;
	851	continue :state_loop;
	852	},
	853	else => return error.SyntaxError,
	854	}
	855	},
	856	.number_leading_zero => {
	857	if (self.cursor >= self.input.len) return self.endOfBufferInNumber(true);
	858	switch (self.input[self.cursor]) {
	859	'.' => {
	860	self.cursor += 1;
	861	self.state = .number_post_dot;
	862	continue :state_loop;
	863	},
	864	'e', 'E' => {
	865	self.cursor += 1;
	866	self.state = .number_post_e;
	867	continue :state_loop;
	868	},
	869	else => {
	870	self.state = .post_value;
	871	return Token{ .number = self.takeValueSlice() };
	872	},
	873	}
	874	},
	875	.number_int => {
	876	while (self.cursor < self.input.len) : (self.cursor += 1) {
	877	switch (self.input[self.cursor]) {
	878	'0'...'9' => continue,
	879	'.' => {
	880	self.cursor += 1;
	881	self.state = .number_post_dot;
	882	continue :state_loop;
	883	},
	884	'e', 'E' => {
	885	self.cursor += 1;
	886	self.state = .number_post_e;
	887	continue :state_loop;
	888	},
	889	else => {
	890	self.state = .post_value;
	891	return Token{ .number = self.takeValueSlice() };
	892	},
	893	}
	894	}
	895	return self.endOfBufferInNumber(true);
	896	},
	897	.number_post_dot => {
	898	if (self.cursor >= self.input.len) return self.endOfBufferInNumber(false);
	899	switch (try self.expectByte()) {
	900	'0'...'9' => {
	901	self.cursor += 1;
	902	self.state = .number_frac;
	903	continue :state_loop;
	904	},
	905	else => return error.SyntaxError,
	906	}
	907	},
	908	.number_frac => {
	909	while (self.cursor < self.input.len) : (self.cursor += 1) {
	910	switch (self.input[self.cursor]) {
	911	'0'...'9' => continue,
	912	'e', 'E' => {
	913	self.cursor += 1;
	914	self.state = .number_post_e;
	915	continue :state_loop;
	916	},
	917	else => {
	918	self.state = .post_value;
	919	return Token{ .number = self.takeValueSlice() };
	920	},
	921	}
	922	}
	923	return self.endOfBufferInNumber(true);
	924	},
	925	.number_post_e => {
	926	if (self.cursor >= self.input.len) return self.endOfBufferInNumber(false);
	927	switch (self.input[self.cursor]) {
	928	'0'...'9' => {
	929	self.cursor += 1;
	930	self.state = .number_exp;
	931	continue :state_loop;
	932	},
	933	'+', '-' => {
	934	self.cursor += 1;
	935	self.state = .number_post_e_sign;
	936	continue :state_loop;
	937	},
	938	else => return error.SyntaxError,
	939	}
	940	},
	941	.number_post_e_sign => {
	942	if (self.cursor >= self.input.len) return self.endOfBufferInNumber(false);
	943	switch (self.input[self.cursor]) {
	944	'0'...'9' => {
	945	self.cursor += 1;
	946	self.state = .number_exp;
	947	continue :state_loop;
	948	},
	949	else => return error.SyntaxError,
	950	}
	951	},
	952	.number_exp => {
	953	while (self.cursor < self.input.len) : (self.cursor += 1) {
	954	switch (self.input[self.cursor]) {
	955	'0'...'9' => continue,
	956	else => {
	957	self.state = .post_value;
	958	return Token{ .number = self.takeValueSlice() };
	959	},
	960	}
	961	}
	962	return self.endOfBufferInNumber(true);
	963	},
	964
	965	.string => {
	966	while (self.cursor < self.input.len) : (self.cursor += 1) {
	967	switch (self.input[self.cursor]) {
	968	0...0x1f => return error.SyntaxError, // Bare ASCII control code in string.
	969
	970	// ASCII plain text.
	971	0x20...('"' - 1), ('"' + 1)...('\\' - 1), ('\\' + 1)...0x7F => continue,
	972
	973	// Special characters.
	974	'"' => {
	975	const result = Token{ .string = self.takeValueSlice() };
	976	self.cursor += 1;
	977	self.state = .post_value;
	978	return result;
	979	},
	980	'\\' => {
	981	const slice = self.takeValueSlice();
	982	self.cursor += 1;
	983	self.state = .string_backslash;
	984	if (slice.len > 0) return Token{ .partial_string = slice };
	985	continue :state_loop;
	986	},
	987
	988	// UTF-8 validation.
	989	// See http://unicode.org/mail-arch/unicode-ml/y2003-m02/att-0467/01-The_Algorithm_to_Valide_an_UTF-8_String
	990	0xC2...0xDF => {
	991	self.cursor += 1;
	992	self.state = .string_utf8_last_byte;
	993	continue :state_loop;
	994	},
	995	0xE0 => {
	996	self.cursor += 1;
	997	self.state = .string_utf8_second_to_last_byte_guard_against_overlong;
	998	continue :state_loop;
	999	},
	1000	0xE1...0xEC, 0xEE...0xEF => {
	1001	self.cursor += 1;
	1002	self.state = .string_utf8_second_to_last_byte;
	1003	continue :state_loop;
	1004	},
	1005	0xED => {
	1006	self.cursor += 1;
	1007	self.state = .string_utf8_second_to_last_byte_guard_against_surrogate_half;
	1008	continue :state_loop;
	1009	},
	1010	0xF0 => {
	1011	self.cursor += 1;
	1012	self.state = .string_utf8_third_to_last_byte_guard_against_overlong;
	1013	continue :state_loop;
	1014	},
	1015	0xF1...0xF3 => {
	1016	self.cursor += 1;
	1017	self.state = .string_utf8_third_to_last_byte;
	1018	continue :state_loop;
	1019	},
	1020	0xF4 => {
	1021	self.cursor += 1;
	1022	self.state = .string_utf8_third_to_last_byte_guard_against_too_large;
	1023	continue :state_loop;
	1024	},
	1025	0x80...0xC1, 0xF5...0xFF => return error.SyntaxError, // Invalid UTF-8.
	1026	}
	1027	}
	1028	if (self.is_end_of_input) return error.UnexpectedEndOfInput;
	1029	const slice = self.takeValueSlice();
	1030	if (slice.len > 0) return Token{ .partial_string = slice };
	1031	return error.BufferUnderrun;
	1032	},
	1033	.string_backslash => {
	1034	switch (try self.expectByte()) {
	1035	'"', '\\', '/' => {
	1036	// Since these characters now represent themselves literally,
	1037	// we can simply begin the next plaintext slice here.
	1038	self.value_start = self.cursor;
	1039	self.cursor += 1;
	1040	self.state = .string;
	1041	continue :state_loop;
	1042	},
	1043	'b' => {
	1044	self.cursor += 1;
	1045	self.value_start = self.cursor;
	1046	self.state = .string;
	1047	return Token{ .partial_string_escaped_1 = [_]u8{0x08} };
	1048	},
	1049	'f' => {
	1050	self.cursor += 1;
	1051	self.value_start = self.cursor;
	1052	self.state = .string;
	1053	return Token{ .partial_string_escaped_1 = [_]u8{0x0c} };
	1054	},
	1055	'n' => {
	1056	self.cursor += 1;
	1057	self.value_start = self.cursor;
	1058	self.state = .string;
	1059	return Token{ .partial_string_escaped_1 = [_]u8{'\n'} };
	1060	},
	1061	'r' => {
	1062	self.cursor += 1;
	1063	self.value_start = self.cursor;
	1064	self.state = .string;
	1065	return Token{ .partial_string_escaped_1 = [_]u8{'\r'} };
	1066	},
	1067	't' => {
	1068	self.cursor += 1;
	1069	self.value_start = self.cursor;
	1070	self.state = .string;
	1071	return Token{ .partial_string_escaped_1 = [_]u8{'\t'} };
	1072	},
	1073	'u' => {
	1074	self.cursor += 1;
	1075	self.state = .string_backslash_u;
	1076	continue :state_loop;
	1077	},
	1078	else => return error.SyntaxError,
	1079	}
	1080	},
	1081	.string_backslash_u => {
	1082	const c = try self.expectByte();
	1083	switch (c) {
	1084	'0'...'9' => {
	1085	self.unicode_code_point = @as(u21, c - '0') << 12;
	1086	},
	1087	'A'...'F' => {
	1088	self.unicode_code_point = @as(u21, c - 'A' + 10) << 12;
	1089	},
	1090	'a'...'f' => {
	1091	self.unicode_code_point = @as(u21, c - 'a' + 10) << 12;
	1092	},
	1093	else => return error.SyntaxError,
	1094	}
	1095	self.cursor += 1;
	1096	self.state = .string_backslash_u_1;
	1097	continue :state_loop;
	1098	},
	1099	.string_backslash_u_1 => {
	1100	const c = try self.expectByte();
	1101	switch (c) {
	1102	'0'...'9' => {
	1103	self.unicode_code_point \|= @as(u21, c - '0') << 8;
	1104	},
	1105	'A'...'F' => {
	1106	self.unicode_code_point \|= @as(u21, c - 'A' + 10) << 8;
	1107	},
	1108	'a'...'f' => {
	1109	self.unicode_code_point \|= @as(u21, c - 'a' + 10) << 8;
	1110	},
	1111	else => return error.SyntaxError,
	1112	}
	1113	self.cursor += 1;
	1114	self.state = .string_backslash_u_2;
	1115	continue :state_loop;
	1116	},
	1117	.string_backslash_u_2 => {
	1118	const c = try self.expectByte();
	1119	switch (c) {
	1120	'0'...'9' => {
	1121	self.unicode_code_point \|= @as(u21, c - '0') << 4;
	1122	},
	1123	'A'...'F' => {
	1124	self.unicode_code_point \|= @as(u21, c - 'A' + 10) << 4;
	1125	},
	1126	'a'...'f' => {
	1127	self.unicode_code_point \|= @as(u21, c - 'a' + 10) << 4;
	1128	},
	1129	else => return error.SyntaxError,
	1130	}
	1131	self.cursor += 1;
	1132	self.state = .string_backslash_u_3;
	1133	continue :state_loop;
	1134	},
	1135	.string_backslash_u_3 => {
	1136	const c = try self.expectByte();
	1137	switch (c) {
	1138	'0'...'9' => {
	1139	self.unicode_code_point \|= c - '0';
	1140	},
	1141	'A'...'F' => {
	1142	self.unicode_code_point \|= c - 'A' + 10;
	1143	},
	1144	'a'...'f' => {
	1145	self.unicode_code_point \|= c - 'a' + 10;
	1146	},
	1147	else => return error.SyntaxError,
	1148	}
	1149	self.cursor += 1;
	1150	switch (self.unicode_code_point) {
	1151	0xD800...0xDBFF => {
	1152	// High surrogate half.
	1153	self.unicode_code_point = 0x10000 \| (self.unicode_code_point << 10);
	1154	self.state = .string_surrogate_half;
	1155	continue :state_loop;
	1156	},
	1157	0xDC00...0xDFFF => return error.SyntaxError, // Unexpected low surrogate half.
	1158	else => {
	1159	// Code point from a single UTF-16 code unit.
	1160	self.value_start = self.cursor;
	1161	self.state = .string;
	1162	return self.partialStringCodepoint();
	1163	},
	1164	}
	1165	},
	1166	.string_surrogate_half => {
	1167	switch (try self.expectByte()) {
	1168	'\\' => {
	1169	self.cursor += 1;
	1170	self.state = .string_surrogate_half_backslash;
	1171	continue :state_loop;
	1172	},
	1173	else => return error.SyntaxError, // Expected low surrogate half.
	1174	}
	1175	},
	1176	.string_surrogate_half_backslash => {
	1177	switch (try self.expectByte()) {
	1178	'u' => {
	1179	self.cursor += 1;
	1180	self.state = .string_surrogate_half_backslash_u;
	1181	continue :state_loop;
	1182	},
	1183	else => return error.SyntaxError, // Expected low surrogate half.
	1184	}
	1185	},
	1186	.string_surrogate_half_backslash_u => {
	1187	switch (try self.expectByte()) {
	1188	'D', 'd' => {
	1189	self.cursor += 1;
	1190	self.state = .string_surrogate_half_backslash_u_1;
	1191	continue :state_loop;
	1192	},
	1193	else => return error.SyntaxError, // Expected low surrogate half.
	1194	}
	1195	},
	1196	.string_surrogate_half_backslash_u_1 => {
	1197	const c = try self.expectByte();
	1198	switch (c) {
	1199	'C'...'F' => {
	1200	self.cursor += 1;
	1201	self.unicode_code_point \|= @as(u21, c - 'C') << 8;
	1202	self.state = .string_surrogate_half_backslash_u_2;
	1203	continue :state_loop;
	1204	},
	1205	'c'...'f' => {
	1206	self.cursor += 1;
	1207	self.unicode_code_point \|= @as(u21, c - 'c') << 8;
	1208	self.state = .string_surrogate_half_backslash_u_2;
	1209	continue :state_loop;
	1210	},
	1211	else => return error.SyntaxError, // Expected low surrogate half.
	1212	}
	1213	},
	1214	.string_surrogate_half_backslash_u_2 => {
	1215	const c = try self.expectByte();
	1216	switch (c) {
	1217	'0'...'9' => {
	1218	self.cursor += 1;
	1219	self.unicode_code_point \|= @as(u21, c - '0') << 4;
	1220	self.state = .string_surrogate_half_backslash_u_3;
	1221	continue :state_loop;
	1222	},
	1223	'A'...'F' => {
	1224	self.cursor += 1;
	1225	self.unicode_code_point \|= @as(u21, c - 'A' + 10) << 4;
	1226	self.state = .string_surrogate_half_backslash_u_3;
	1227	continue :state_loop;
	1228	},
	1229	'a'...'f' => {
	1230	self.cursor += 1;
	1231	self.unicode_code_point \|= @as(u21, c - 'a' + 10) << 4;
	1232	self.state = .string_surrogate_half_backslash_u_3;
	1233	continue :state_loop;
	1234	},
	1235	else => return error.SyntaxError,
	1236	}
	1237	},
	1238	.string_surrogate_half_backslash_u_3 => {
	1239	const c = try self.expectByte();
	1240	switch (c) {
	1241	'0'...'9' => {
	1242	self.unicode_code_point \|= c - '0';
	1243	},
	1244	'A'...'F' => {
	1245	self.unicode_code_point \|= c - 'A' + 10;
	1246	},
	1247	'a'...'f' => {
	1248	self.unicode_code_point \|= c - 'a' + 10;
	1249	},
	1250	else => return error.SyntaxError,
	1251	}
	1252	self.cursor += 1;
	1253	self.value_start = self.cursor;
	1254	self.state = .string;
	1255	return self.partialStringCodepoint();
	1256	},
	1257
	1258	.string_utf8_last_byte => {
	1259	switch (try self.expectByte()) {
	1260	0x80...0xBF => {
	1261	self.cursor += 1;
	1262	self.state = .string;
	1263	continue :state_loop;
	1264	},
	1265	else => return error.SyntaxError, // Invalid UTF-8.
	1266	}
	1267	},
	1268	.string_utf8_second_to_last_byte => {
	1269	switch (try self.expectByte()) {
	1270	0x80...0xBF => {
	1271	self.cursor += 1;
	1272	self.state = .string_utf8_last_byte;
	1273	continue :state_loop;
	1274	},
	1275	else => return error.SyntaxError, // Invalid UTF-8.
	1276	}
	1277	},
	1278	.string_utf8_second_to_last_byte_guard_against_overlong => {
	1279	switch (try self.expectByte()) {
	1280	0xA0...0xBF => {
	1281	self.cursor += 1;
	1282	self.state = .string_utf8_last_byte;
	1283	continue :state_loop;
	1284	},
	1285	else => return error.SyntaxError, // Invalid UTF-8.
	1286	}
	1287	},
	1288	.string_utf8_second_to_last_byte_guard_against_surrogate_half => {
	1289	switch (try self.expectByte()) {
	1290	0x80...0x9F => {
	1291	self.cursor += 1;
	1292	self.state = .string_utf8_last_byte;
	1293	continue :state_loop;
	1294	},
	1295	else => return error.SyntaxError, // Invalid UTF-8.
	1296	}
	1297	},
	1298	.string_utf8_third_to_last_byte => {
	1299	switch (try self.expectByte()) {
	1300	0x80...0xBF => {
	1301	self.cursor += 1;
	1302	self.state = .string_utf8_second_to_last_byte;
	1303	continue :state_loop;
	1304	},
	1305	else => return error.SyntaxError, // Invalid UTF-8.
	1306	}
	1307	},
	1308	.string_utf8_third_to_last_byte_guard_against_overlong => {
	1309	switch (try self.expectByte()) {
	1310	0x90...0xBF => {
	1311	self.cursor += 1;
	1312	self.state = .string_utf8_second_to_last_byte;
	1313	continue :state_loop;
	1314	},
	1315	else => return error.SyntaxError, // Invalid UTF-8.
	1316	}
	1317	},
	1318	.string_utf8_third_to_last_byte_guard_against_too_large => {
	1319	switch (try self.expectByte()) {
	1320	0x80...0x8F => {
	1321	self.cursor += 1;
	1322	self.state = .string_utf8_second_to_last_byte;
	1323	continue :state_loop;
	1324	},
	1325	else => return error.SyntaxError, // Invalid UTF-8.
	1326	}
	1327	},
	1328
	1329	.literal_t => {
	1330	switch (try self.expectByte()) {
	1331	'r' => {
	1332	self.cursor += 1;
	1333	self.state = .literal_tr;
	1334	continue :state_loop;
	1335	},
	1336	else => return error.SyntaxError,
	1337	}
	1338	},
	1339	.literal_tr => {
	1340	switch (try self.expectByte()) {
	1341	'u' => {
	1342	self.cursor += 1;
	1343	self.state = .literal_tru;
	1344	continue :state_loop;
	1345	},
	1346	else => return error.SyntaxError,
	1347	}
	1348	},
	1349	.literal_tru => {
	1350	switch (try self.expectByte()) {
	1351	'e' => {
	1352	self.cursor += 1;
	1353	self.state = .post_value;
	1354	return .true;
	1355	},
	1356	else => return error.SyntaxError,
	1357	}
	1358	},
	1359	.literal_f => {
	1360	switch (try self.expectByte()) {
	1361	'a' => {
	1362	self.cursor += 1;
	1363	self.state = .literal_fa;
	1364	continue :state_loop;
	1365	},
	1366	else => return error.SyntaxError,
	1367	}
	1368	},
	1369	.literal_fa => {
	1370	switch (try self.expectByte()) {
	1371	'l' => {
	1372	self.cursor += 1;
	1373	self.state = .literal_fal;
	1374	continue :state_loop;
	1375	},
	1376	else => return error.SyntaxError,
	1377	}
	1378	},
	1379	.literal_fal => {
	1380	switch (try self.expectByte()) {
	1381	's' => {
	1382	self.cursor += 1;
	1383	self.state = .literal_fals;
	1384	continue :state_loop;
	1385	},
	1386	else => return error.SyntaxError,
	1387	}
	1388	},
	1389	.literal_fals => {
	1390	switch (try self.expectByte()) {
	1391	'e' => {
	1392	self.cursor += 1;
	1393	self.state = .post_value;
	1394	return .false;
	1395	},
	1396	else => return error.SyntaxError,
	1397	}
	1398	},
	1399	.literal_n => {
	1400	switch (try self.expectByte()) {
	1401	'u' => {
	1402	self.cursor += 1;
	1403	self.state = .literal_nu;
	1404	continue :state_loop;
	1405	},
	1406	else => return error.SyntaxError,
	1407	}
	1408	},
	1409	.literal_nu => {
	1410	switch (try self.expectByte()) {
	1411	'l' => {
	1412	self.cursor += 1;
	1413	self.state = .literal_nul;
	1414	continue :state_loop;
	1415	},
	1416	else => return error.SyntaxError,
	1417	}
	1418	},
	1419	.literal_nul => {
	1420	switch (try self.expectByte()) {
	1421	'l' => {
	1422	self.cursor += 1;
	1423	self.state = .post_value;
	1424	return .null;
	1425	},
	1426	else => return error.SyntaxError,
	1427	}
	1428	},
	1429	}
	1430	unreachable;
	1431	}
	1432	}
	1433
	1434	/// Seeks ahead in the input until the first byte of the next token (or the end of the input)
	1435	/// determines which type of token will be returned from the next `next*()` call.
	1436	/// This function is idempotent, only advancing past commas, colons, and inter-token whitespace.
	1437	pub fn peekNextTokenType(self: *@This()) PeekError!TokenType {
	1438	state_loop: while (true) {
	1439	switch (self.state) {
	1440	.value => {
	1441	switch (try self.skipWhitespaceExpectByte()) {
	1442	'{' => return .object_begin,
	1443	'[' => return .array_begin,
	1444	'"' => return .string,
	1445	'-', '0'...'9' => return .number,
	1446	't' => return .true,
	1447	'f' => return .false,
	1448	'n' => return .null,
	1449	else => return error.SyntaxError,
	1450	}
	1451	},
	1452
	1453	.post_value => {
	1454	if (try self.skipWhitespaceCheckEnd()) return .end_of_document;
	1455
	1456	const c = self.input[self.cursor];
	1457	if (self.string_is_object_key) {
	1458	self.string_is_object_key = false;
	1459	switch (c) {
	1460	':' => {
	1461	self.cursor += 1;
	1462	self.state = .value;
	1463	continue :state_loop;
	1464	},
	1465	else => return error.SyntaxError,
	1466	}
	1467	}
	1468
	1469	switch (c) {
	1470	'}' => return .object_end,
	1471	']' => return .array_end,
	1472	',' => {
	1473	switch (self.stack.peek()) {
	1474	OBJECT_MODE => {
	1475	self.state = .object_post_comma;
	1476	},
	1477	ARRAY_MODE => {
	1478	self.state = .value;
	1479	},
	1480	}
	1481	self.cursor += 1;
	1482	continue :state_loop;
	1483	},
	1484	else => return error.SyntaxError,
	1485	}
	1486	},
	1487
	1488	.object_start => {
	1489	switch (try self.skipWhitespaceExpectByte()) {
	1490	'"' => return .string,
	1491	'}' => return .object_end,
	1492	else => return error.SyntaxError,
	1493	}
	1494	},
	1495	.object_post_comma => {
	1496	switch (try self.skipWhitespaceExpectByte()) {
	1497	'"' => return .string,
	1498	else => return error.SyntaxError,
	1499	}
	1500	},
	1501
	1502	.array_start => {
	1503	switch (try self.skipWhitespaceExpectByte()) {
	1504	']' => return .array_end,
	1505	else => {
	1506	self.state = .value;
	1507	continue :state_loop;
	1508	},
	1509	}
	1510	},
	1511
	1512	.number_minus,
	1513	.number_leading_zero,
	1514	.number_int,
	1515	.number_post_dot,
	1516	.number_frac,
	1517	.number_post_e,
	1518	.number_post_e_sign,
	1519	.number_exp,
	1520	=> return .number,
	1521
	1522	.string,
	1523	.string_backslash,
	1524	.string_backslash_u,
	1525	.string_backslash_u_1,
	1526	.string_backslash_u_2,
	1527	.string_backslash_u_3,
	1528	.string_surrogate_half,
	1529	.string_surrogate_half_backslash,
	1530	.string_surrogate_half_backslash_u,
	1531	.string_surrogate_half_backslash_u_1,
	1532	.string_surrogate_half_backslash_u_2,
	1533	.string_surrogate_half_backslash_u_3,
	1534	=> return .string,
	1535
	1536	.string_utf8_last_byte,
	1537	.string_utf8_second_to_last_byte,
	1538	.string_utf8_second_to_last_byte_guard_against_overlong,
	1539	.string_utf8_second_to_last_byte_guard_against_surrogate_half,
	1540	.string_utf8_third_to_last_byte,
	1541	.string_utf8_third_to_last_byte_guard_against_overlong,
	1542	.string_utf8_third_to_last_byte_guard_against_too_large,
	1543	=> return .string,
	1544
	1545	.literal_t,
	1546	.literal_tr,
	1547	.literal_tru,
	1548	=> return .true,
	1549	.literal_f,
	1550	.literal_fa,
	1551	.literal_fal,
	1552	.literal_fals,
	1553	=> return .false,
	1554	.literal_n,
	1555	.literal_nu,
	1556	.literal_nul,
	1557	=> return .null,
	1558	}
	1559	unreachable;
	1560	}
	1561	}
	1562
	1563	const State = enum {
	1564	value,
	1565	post_value,
	1566
	1567	object_start,
	1568	object_post_comma,
	1569
	1570	array_start,
	1571
	1572	number_minus,
	1573	number_leading_zero,
	1574	number_int,
	1575	number_post_dot,
	1576	number_frac,
	1577	number_post_e,
	1578	number_post_e_sign,
	1579	number_exp,
	1580
	1581	string,
	1582	string_backslash,
	1583	string_backslash_u,
	1584	string_backslash_u_1,
	1585	string_backslash_u_2,
	1586	string_backslash_u_3,
	1587	string_surrogate_half,
	1588	string_surrogate_half_backslash,
	1589	string_surrogate_half_backslash_u,
	1590	string_surrogate_half_backslash_u_1,
	1591	string_surrogate_half_backslash_u_2,
	1592	string_surrogate_half_backslash_u_3,
	1593
	1594	// From http://unicode.org/mail-arch/unicode-ml/y2003-m02/att-0467/01-The_Algorithm_to_Valide_an_UTF-8_String
	1595	string_utf8_last_byte, // State A
	1596	string_utf8_second_to_last_byte, // State B
	1597	string_utf8_second_to_last_byte_guard_against_overlong, // State C
	1598	string_utf8_second_to_last_byte_guard_against_surrogate_half, // State D
	1599	string_utf8_third_to_last_byte, // State E
	1600	string_utf8_third_to_last_byte_guard_against_overlong, // State F
	1601	string_utf8_third_to_last_byte_guard_against_too_large, // State G
	1602
	1603	literal_t,
	1604	literal_tr,
	1605	literal_tru,
	1606	literal_f,
	1607	literal_fa,
	1608	literal_fal,
	1609	literal_fals,
	1610	literal_n,
	1611	literal_nu,
	1612	literal_nul,
	1613	};
	1614
	1615	fn expectByte(self: *const @This()) !u8 {
	1616	if (self.cursor < self.input.len) {
	1617	return self.input[self.cursor];
	1618	}
	1619	// No byte.
	1620	if (self.is_end_of_input) return error.UnexpectedEndOfInput;
	1621	return error.BufferUnderrun;
	1622	}
	1623
	1624	fn skipWhitespace(self: *@This()) void {
	1625	while (self.cursor < self.input.len) : (self.cursor += 1) {
	1626	switch (self.input[self.cursor]) {
	1627	// Whitespace
	1628	' ', '\t', '\r' => continue,
	1629	'\n' => {
	1630	if (self.diagnostics) \|diag\| {
	1631	diag.line_number += 1;
	1632	// This will count the newline itself,
	1633	// which means a straight-forward subtraction will give a 1-based column number.
	1634	diag.line_start_cursor = self.cursor;
	1635	}
	1636	continue;
	1637	},
	1638	else => return,
	1639	}
	1640	}
	1641	}
	1642
	1643	fn skipWhitespaceExpectByte(self: *@This()) !u8 {
	1644	self.skipWhitespace();
	1645	return self.expectByte();
	1646	}
	1647
	1648	fn skipWhitespaceCheckEnd(self: *@This()) !bool {
	1649	self.skipWhitespace();
	1650	if (self.cursor >= self.input.len) {
	1651	// End of buffer.
	1652	if (self.is_end_of_input) {
	1653	// End of everything.
	1654	if (self.stackHeight() == 0) {
	1655	// We did it!
	1656	return true;
	1657	}
	1658	return error.UnexpectedEndOfInput;
	1659	}
	1660	return error.BufferUnderrun;
	1661	}
	1662	if (self.stackHeight() == 0) return error.SyntaxError;
	1663	return false;
	1664	}
	1665
	1666	fn takeValueSlice(self: *@This()) []const u8 {
	1667	const slice = self.input[self.value_start..self.cursor];
	1668	self.value_start = self.cursor;
	1669	return slice;
	1670	}
	1671
	1672	fn endOfBufferInNumber(self: *@This(), allow_end: bool) !Token {
	1673	const slice = self.takeValueSlice();
	1674	if (self.is_end_of_input) {
	1675	if (!allow_end) return error.UnexpectedEndOfInput;
	1676	self.state = .post_value;
	1677	return Token{ .number = slice };
	1678	}
	1679	if (slice.len == 0) return error.BufferUnderrun;
	1680	return Token{ .partial_number = slice };
	1681	}
	1682
	1683	fn partialStringCodepoint(self: *@This()) Token {
	1684	const code_point = self.unicode_code_point;
	1685	self.unicode_code_point = undefined;
	1686	var buf: [4]u8 = undefined;
	1687	switch (std.unicode.utf8Encode(code_point, &buf) catch unreachable) {
	1688	1 => return Token{ .partial_string_escaped_1 = buf[0..1].* },
	1689	2 => return Token{ .partial_string_escaped_2 = buf[0..2].* },
	1690	3 => return Token{ .partial_string_escaped_3 = buf[0..3].* },
	1691	4 => return Token{ .partial_string_escaped_4 = buf[0..4].* },
	1692	else => unreachable,
	1693	}
	1694	}
	1695	};
	1696
	1697	const OBJECT_MODE = 0;
	1698	const ARRAY_MODE = 1;
	1699
	1700	const BitStack = struct {
	1701	bytes: std.ArrayList(u8),
	1702	bit_len: u32 = 0,
	1703
	1704	pub fn init(allocator: Allocator) @This() {
	1705	return .{
	1706	.bytes = std.ArrayList(u8).init(allocator),
	1707	};
	1708	}
	1709
	1710	pub fn deinit(self: *@This()) void {
	1711	self.bytes.deinit();
	1712	self.* = undefined;
	1713	}
	1714
	1715	pub fn ensureTotalCapacity(self: *@This(), bit_capcity: u32) Allocator.Error!void {
	1716	const byte_capacity = (bit_capcity + 7) >> 3;
	1717	try self.bytes.ensureTotalCapacity(byte_capacity);
	1718	}
	1719
	1720	pub fn push(self: *@This(), b: u1) Allocator.Error!void {
	1721	const byte_index = self.bit_len >> 3;
	1722	const bit_index = @intCast(u3, self.bit_len & 7);
	1723
	1724	if (self.bytes.items.len <= byte_index) {
	1725	try self.bytes.append(0);
	1726	}
	1727
	1728	self.bytes.items[byte_index] &= ~(@as(u8, 1) << bit_index);
	1729	self.bytes.items[byte_index] \|= @as(u8, b) << bit_index;
	1730
	1731	self.bit_len += 1;
	1732	}
	1733
	1734	pub fn peek(self: *const @This()) u1 {
	1735	const byte_index = (self.bit_len - 1) >> 3;
	1736	const bit_index = @intCast(u3, (self.bit_len - 1) & 7);
	1737	return @intCast(u1, (self.bytes.items[byte_index] >> bit_index) & 1);
	1738	}
	1739
	1740	pub fn pop(self: *@This()) u1 {
	1741	const b = self.peek();
	1742	self.bit_len -= 1;
	1743	return b;
	1744	}
	1745	};
	1746
	1747	fn appendSlice(list: *std.ArrayList(u8), buf: []const u8, max_value_len: usize) !void {
	1748	const new_len = std.math.add(usize, list.items.len, buf.len) catch return error.ValueTooLong;
	1749	if (new_len > max_value_len) return error.ValueTooLong;
	1750	try list.appendSlice(buf);
	1751	}
	1752
	1753	/// For the slice you get from a `Token.number` or `Token.allocated_number`,
	1754	/// this function returns true if the number doesn't contain any fraction or exponent components.
	1755	/// Note, the numeric value encoded by the value may still be an integer, such as `1.0`.
	1756	/// This function is meant to give a hint about whether integer parsing or float parsing should be used on the value.
	1757	/// This function will not give meaningful results on non-numeric input.
	1758	pub fn isNumberFormattedLikeAnInteger(value: []const u8) bool {
	1759	return std.mem.indexOfAny(u8, value, ".eE") == null;
	1760	}
	1761
	1762	test {
	1763	_ = @import("./scanner_test.zig");
	1764	}

lib/std/json/scanner_test.zig created+466

...	...	@@ -0,0 +1,466 @@
	1	const std = @import("std");
	2	const JsonScanner = @import("./scanner.zig").Scanner;
	3	const jsonReader = @import("./scanner.zig").reader;
	4	const JsonReader = @import("./scanner.zig").Reader;
	5	const Token = @import("./scanner.zig").Token;
	6	const TokenType = @import("./scanner.zig").TokenType;
	7	const Diagnostics = @import("./scanner.zig").Diagnostics;
	8	const Error = @import("./scanner.zig").Error;
	9	const validate = @import("./scanner.zig").validate;
	10
	11	const example_document_str =
	12	\\{
	13	\\ "Image": {
	14	\\ "Width": 800,
	15	\\ "Height": 600,
	16	\\ "Title": "View from 15th Floor",
	17	\\ "Thumbnail": {
	18	\\ "Url": "http://www.example.com/image/481989943",
	19	\\ "Height": 125,
	20	\\ "Width": 100
	21	\\ },
	22	\\ "Animated" : false,
	23	\\ "IDs": [116, 943, 234, 38793]
	24	\\ }
	25	\\}
	26	;
	27
	28	fn expectNext(scanner_or_reader: anytype, expected_token: Token) !void {
	29	return expectEqualTokens(expected_token, try scanner_or_reader.next());
	30	}
	31
	32	fn expectPeekNext(scanner_or_reader: anytype, expected_token_type: TokenType, expected_token: Token) !void {
	33	try std.testing.expectEqual(expected_token_type, try scanner_or_reader.peekNextTokenType());
	34	try expectEqualTokens(expected_token, try scanner_or_reader.next());
	35	}
	36
	37	test "json.token" {
	38	var scanner = JsonScanner.initCompleteInput(std.testing.allocator, example_document_str);
	39	defer scanner.deinit();
	40
	41	try expectNext(&scanner, .object_begin);
	42	try expectNext(&scanner, Token{ .string = "Image" });
	43	try expectNext(&scanner, .object_begin);
	44	try expectNext(&scanner, Token{ .string = "Width" });
	45	try expectNext(&scanner, Token{ .number = "800" });
	46	try expectNext(&scanner, Token{ .string = "Height" });
	47	try expectNext(&scanner, Token{ .number = "600" });
	48	try expectNext(&scanner, Token{ .string = "Title" });
	49	try expectNext(&scanner, Token{ .string = "View from 15th Floor" });
	50	try expectNext(&scanner, Token{ .string = "Thumbnail" });
	51	try expectNext(&scanner, .object_begin);
	52	try expectNext(&scanner, Token{ .string = "Url" });
	53	try expectNext(&scanner, Token{ .string = "http://www.example.com/image/481989943" });
	54	try expectNext(&scanner, Token{ .string = "Height" });
	55	try expectNext(&scanner, Token{ .number = "125" });
	56	try expectNext(&scanner, Token{ .string = "Width" });
	57	try expectNext(&scanner, Token{ .number = "100" });
	58	try expectNext(&scanner, .object_end);
	59	try expectNext(&scanner, Token{ .string = "Animated" });
	60	try expectNext(&scanner, .false);
	61	try expectNext(&scanner, Token{ .string = "IDs" });
	62	try expectNext(&scanner, .array_begin);
	63	try expectNext(&scanner, Token{ .number = "116" });
	64	try expectNext(&scanner, Token{ .number = "943" });
	65	try expectNext(&scanner, Token{ .number = "234" });
	66	try expectNext(&scanner, Token{ .number = "38793" });
	67	try expectNext(&scanner, .array_end);
	68	try expectNext(&scanner, .object_end);
	69	try expectNext(&scanner, .object_end);
	70	try expectNext(&scanner, .end_of_document);
	71	}
	72
	73	const all_types_test_case =
	74	\\[
	75	\\ "", "a\nb",
	76	\\ 0, 0.0, -1.1e-1,
	77	\\ true, false, null,
	78	\\ {"a": {}},
	79	\\ []
	80	\\]
	81	;
	82
	83	fn testAllTypes(source: anytype, large_buffer: bool) !void {
	84	try expectPeekNext(source, .array_begin, .array_begin);
	85	try expectPeekNext(source, .string, Token{ .string = "" });
	86	try expectPeekNext(source, .string, Token{ .partial_string = "a" });
	87	try expectPeekNext(source, .string, Token{ .partial_string_escaped_1 = "\n".* });
	88	if (large_buffer) {
	89	try expectPeekNext(source, .string, Token{ .string = "b" });
	90	} else {
	91	try expectPeekNext(source, .string, Token{ .partial_string = "b" });
	92	try expectPeekNext(source, .string, Token{ .string = "" });
	93	}
	94	if (large_buffer) {
	95	try expectPeekNext(source, .number, Token{ .number = "0" });
	96	} else {
	97	try expectPeekNext(source, .number, Token{ .partial_number = "0" });
	98	try expectPeekNext(source, .number, Token{ .number = "" });
	99	}
	100	if (large_buffer) {
	101	try expectPeekNext(source, .number, Token{ .number = "0.0" });
	102	} else {
	103	try expectPeekNext(source, .number, Token{ .partial_number = "0" });
	104	try expectPeekNext(source, .number, Token{ .partial_number = "." });
	105	try expectPeekNext(source, .number, Token{ .partial_number = "0" });
	106	try expectPeekNext(source, .number, Token{ .number = "" });
	107	}
	108	if (large_buffer) {
	109	try expectPeekNext(source, .number, Token{ .number = "-1.1e-1" });
	110	} else {
	111	try expectPeekNext(source, .number, Token{ .partial_number = "-" });
	112	try expectPeekNext(source, .number, Token{ .partial_number = "1" });
	113	try expectPeekNext(source, .number, Token{ .partial_number = "." });
	114	try expectPeekNext(source, .number, Token{ .partial_number = "1" });
	115	try expectPeekNext(source, .number, Token{ .partial_number = "e" });
	116	try expectPeekNext(source, .number, Token{ .partial_number = "-" });
	117	try expectPeekNext(source, .number, Token{ .partial_number = "1" });
	118	try expectPeekNext(source, .number, Token{ .number = "" });
	119	}
	120	try expectPeekNext(source, .true, .true);
	121	try expectPeekNext(source, .false, .false);
	122	try expectPeekNext(source, .null, .null);
	123	try expectPeekNext(source, .object_begin, .object_begin);
	124	if (large_buffer) {
	125	try expectPeekNext(source, .string, Token{ .string = "a" });
	126	} else {
	127	try expectPeekNext(source, .string, Token{ .partial_string = "a" });
	128	try expectPeekNext(source, .string, Token{ .string = "" });
	129	}
	130	try expectPeekNext(source, .object_begin, .object_begin);
	131	try expectPeekNext(source, .object_end, .object_end);
	132	try expectPeekNext(source, .object_end, .object_end);
	133	try expectPeekNext(source, .array_begin, .array_begin);
	134	try expectPeekNext(source, .array_end, .array_end);
	135	try expectPeekNext(source, .array_end, .array_end);
	136	try expectPeekNext(source, .end_of_document, .end_of_document);
	137	}
	138
	139	test "peek all types" {
	140	var scanner = JsonScanner.initCompleteInput(std.testing.allocator, all_types_test_case);
	141	defer scanner.deinit();
	142	try testAllTypes(&scanner, true);
	143
	144	var stream = std.io.fixedBufferStream(all_types_test_case);
	145	var json_reader = jsonReader(std.testing.allocator, stream.reader());
	146	defer json_reader.deinit();
	147	try testAllTypes(&json_reader, true);
	148
	149	var tiny_stream = std.io.fixedBufferStream(all_types_test_case);
	150	var tiny_json_reader = JsonReader(1, @TypeOf(tiny_stream.reader())).init(std.testing.allocator, tiny_stream.reader());
	151	defer tiny_json_reader.deinit();
	152	try testAllTypes(&tiny_json_reader, false);
	153	}
	154
	155	test "json.token mismatched close" {
	156	var scanner = JsonScanner.initCompleteInput(std.testing.allocator, "[102, 111, 111 }");
	157	defer scanner.deinit();
	158	try expectNext(&scanner, .array_begin);
	159	try expectNext(&scanner, Token{ .number = "102" });
	160	try expectNext(&scanner, Token{ .number = "111" });
	161	try expectNext(&scanner, Token{ .number = "111" });
	162	try std.testing.expectError(error.SyntaxError, scanner.next());
	163	}
	164
	165	test "json.token premature object close" {
	166	var scanner = JsonScanner.initCompleteInput(std.testing.allocator, "{ \"key\": }");
	167	defer scanner.deinit();
	168	try expectNext(&scanner, .object_begin);
	169	try expectNext(&scanner, Token{ .string = "key" });
	170	try std.testing.expectError(error.SyntaxError, scanner.next());
	171	}
	172
	173	test "JsonScanner basic" {
	174	var scanner = JsonScanner.initCompleteInput(std.testing.allocator, example_document_str);
	175	defer scanner.deinit();
	176
	177	while (true) {
	178	const token = try scanner.next();
	179	if (token == .end_of_document) break;
	180	}
	181	}
	182
	183	test "JsonReader basic" {
	184	var stream = std.io.fixedBufferStream(example_document_str);
	185
	186	var json_reader = jsonReader(std.testing.allocator, stream.reader());
	187	defer json_reader.deinit();
	188
	189	while (true) {
	190	const token = try json_reader.next();
	191	if (token == .end_of_document) break;
	192	}
	193	}
	194
	195	const number_test_stems = .{
	196	.{ "", "-" },
	197	.{ "0", "1", "10", "9999999999999999999999999" },
	198	.{ "", ".0", ".999999999999999999999999" },
	199	.{ "", "e0", "E0", "e+0", "e-0", "e9999999999999999999999999999" },
	200	};
	201	const number_test_items = blk: {
	202	comptime var ret: []const []const u8 = &[_][]const u8{};
	203	for (number_test_stems[0]) \|s0\| {
	204	for (number_test_stems[1]) \|s1\| {
	205	for (number_test_stems[2]) \|s2\| {
	206	for (number_test_stems[3]) \|s3\| {
	207	ret = ret ++ &[_][]const u8{s0 ++ s1 ++ s2 ++ s3};
	208	}
	209	}
	210	}
	211	}
	212	break :blk ret;
	213	};
	214
	215	test "numbers" {
	216	for (number_test_items) \|number_str\| {
	217	var scanner = JsonScanner.initCompleteInput(std.testing.allocator, number_str);
	218	defer scanner.deinit();
	219
	220	const token = try scanner.next();
	221	const value = token.number; // assert this is a number
	222	try std.testing.expectEqualStrings(number_str, value);
	223
	224	try std.testing.expectEqual(Token.end_of_document, try scanner.next());
	225	}
	226	}
	227
	228	const string_test_cases = .{
	229	// The left is JSON without the "quotes".
	230	// The right is the expected unescaped content.
	231	.{ "", "" },
	232	.{ "\\\\", "\\" },
	233	.{ "a\\\\b", "a\\b" },
	234	.{ "a\\\"b", "a\"b" },
	235	.{ "\\n", "\n" },
	236	.{ "\\u000a", "\n" },
	237	.{ "𝄞", "\u{1D11E}" },
	238	.{ "\\uD834\\uDD1E", "\u{1D11E}" },
	239	.{ "\\uff20", "＠" },
	240	};
	241
	242	test "strings" {
	243	inline for (string_test_cases) \|tuple\| {
	244	var stream = std.io.fixedBufferStream("\"" ++ tuple[0] ++ "\"");
	245	var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
	246	defer arena.deinit();
	247	var json_reader = jsonReader(std.testing.allocator, stream.reader());
	248	defer json_reader.deinit();
	249
	250	const token = try json_reader.nextAlloc(arena.allocator(), .alloc_if_needed);
	251	const value = switch (token) {
	252	.string => \|value\| value,
	253	.allocated_string => \|value\| value,
	254	else => return error.ExpectedString,
	255	};
	256	try std.testing.expectEqualStrings(tuple[1], value);
	257
	258	try std.testing.expectEqual(Token.end_of_document, try json_reader.next());
	259	}
	260	}
	261
	262	const nesting_test_cases = .{
	263	.{ null, "[]" },
	264	.{ null, "{}" },
	265	.{ error.SyntaxError, "[}" },
	266	.{ error.SyntaxError, "{]" },
	267	.{ null, "[" 1000 ++ "]" 1000 },
	268	.{ null, "{\"\":" 1000 ++ "0" ++ "}" 1000 },
	269	.{ error.SyntaxError, "[" 1000 ++ "]" 999 ++ "}" },
	270	.{ error.SyntaxError, "{\"\":" 1000 ++ "0" ++ "}" 999 ++ "]" },
	271	.{ error.SyntaxError, "[" 1000 ++ "]" 1001 },
	272	.{ error.SyntaxError, "{\"\":" 1000 ++ "0" ++ "}" 1001 },
	273	.{ error.UnexpectedEndOfInput, "[" 1000 ++ "]" 999 },
	274	.{ error.UnexpectedEndOfInput, "{\"\":" 1000 ++ "0" ++ "}" 999 },
	275	};
	276
	277	test "nesting" {
	278	inline for (nesting_test_cases) \|tuple\| {
	279	const maybe_error = tuple[0];
	280	const document_str = tuple[1];
	281
	282	expectMaybeError(document_str, maybe_error) catch \|err\| {
	283	std.debug.print("in json document: {s}\n", .{document_str});
	284	return err;
	285	};
	286	}
	287	}
	288
	289	fn expectMaybeError(document_str: []const u8, maybe_error: ?Error) !void {
	290	var scanner = JsonScanner.initCompleteInput(std.testing.allocator, document_str);
	291	defer scanner.deinit();
	292
	293	while (true) {
	294	const token = scanner.next() catch \|err\| {
	295	if (maybe_error) \|expected_err\| {
	296	if (err == expected_err) return;
	297	}
	298	return err;
	299	};
	300	if (token == .end_of_document) break;
	301	}
	302	if (maybe_error != null) return error.ExpectedError;
	303	}
	304
	305	fn expectEqualTokens(expected_token: Token, actual_token: Token) !void {
	306	try std.testing.expectEqual(std.meta.activeTag(expected_token), std.meta.activeTag(actual_token));
	307	switch (expected_token) {
	308	.number => \|expected_value\| {
	309	try std.testing.expectEqualStrings(expected_value, actual_token.number);
	310	},
	311	.string => \|expected_value\| {
	312	try std.testing.expectEqualStrings(expected_value, actual_token.string);
	313	},
	314	else => {},
	315	}
	316	}
	317
	318	fn testTinyBufferSize(document_str: []const u8) !void {
	319	var tiny_stream = std.io.fixedBufferStream(document_str);
	320	var normal_stream = std.io.fixedBufferStream(document_str);
	321
	322	var tiny_json_reader = JsonReader(1, @TypeOf(tiny_stream.reader())).init(std.testing.allocator, tiny_stream.reader());
	323	defer tiny_json_reader.deinit();
	324	var normal_json_reader = JsonReader(0x1000, @TypeOf(normal_stream.reader())).init(std.testing.allocator, normal_stream.reader());
	325	defer normal_json_reader.deinit();
	326
	327	expectEqualStreamOfTokens(&normal_json_reader, &tiny_json_reader) catch \|err\| {
	328	std.debug.print("in json document: {s}\n", .{document_str});
	329	return err;
	330	};
	331	}
	332	fn expectEqualStreamOfTokens(control_json_reader: anytype, test_json_reader: anytype) !void {
	333	var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
	334	defer arena.deinit();
	335	while (true) {
	336	const control_token = try control_json_reader.nextAlloc(arena.allocator(), .alloc_always);
	337	const test_token = try test_json_reader.nextAlloc(arena.allocator(), .alloc_always);
	338	try expectEqualTokens(control_token, test_token);
	339	if (control_token == .end_of_document) break;
	340	_ = arena.reset(.retain_capacity);
	341	}
	342	}
	343
	344	test "BufferUnderrun" {
	345	try testTinyBufferSize(example_document_str);
	346	for (number_test_items) \|number_str\| {
	347	try testTinyBufferSize(number_str);
	348	}
	349	inline for (string_test_cases) \|tuple\| {
	350	try testTinyBufferSize("\"" ++ tuple[0] ++ "\"");
	351	}
	352	}
	353
	354	test "json.validate" {
	355	try std.testing.expectEqual(true, try validate(std.testing.allocator, "{}"));
	356	try std.testing.expectEqual(true, try validate(std.testing.allocator, "[]"));
	357	try std.testing.expectEqual(false, try validate(std.testing.allocator, "[{[[[[{}]]]]}]"));
	358	try std.testing.expectEqual(false, try validate(std.testing.allocator, "{]"));
	359	try std.testing.expectEqual(false, try validate(std.testing.allocator, "[}"));
	360	try std.testing.expectEqual(false, try validate(std.testing.allocator, "{{{{[]}}}]"));
	361	}
	362
	363	fn testSkipValue(s: []const u8) !void {
	364	var scanner = JsonScanner.initCompleteInput(std.testing.allocator, s);
	365	defer scanner.deinit();
	366	try scanner.skipValue();
	367	try expectEqualTokens(.end_of_document, try scanner.next());
	368
	369	var stream = std.io.fixedBufferStream(s);
	370	var json_reader = jsonReader(std.testing.allocator, stream.reader());
	371	defer json_reader.deinit();
	372	try json_reader.skipValue();
	373	try expectEqualTokens(.end_of_document, try json_reader.next());
	374	}
	375
	376	test "skipValue" {
	377	try testSkipValue("false");
	378	try testSkipValue("true");
	379	try testSkipValue("null");
	380	try testSkipValue("42");
	381	try testSkipValue("42.0");
	382	try testSkipValue("\"foo\"");
	383	try testSkipValue("[101, 111, 121]");
	384	try testSkipValue("{}");
	385	try testSkipValue("{\"foo\": \"bar\\nbaz\"}");
	386
	387	// An absurd number of nestings
	388	const nestings = 1000;
	389	try testSkipValue("[" nestings ++ "]" nestings);
	390
	391	// Would a number token cause problems in a deeply-nested array?
	392	try testSkipValue("[" nestings ++ "0.118, 999, 881.99, 911.9, 725, 3" ++ "]" nestings);
	393
	394	// Mismatched brace/square bracket
	395	try std.testing.expectError(error.SyntaxError, testSkipValue("[102, 111, 111}"));
	396	}
	397
	398	fn testEnsureStackCapacity(do_ensure: bool) !void {
	399	var fail_alloc = std.testing.FailingAllocator.init(std.testing.allocator, 1);
	400	const failing_allocator = fail_alloc.allocator();
	401
	402	const nestings = 999; // intentionally not a power of 2.
	403	var scanner = JsonScanner.initCompleteInput(failing_allocator, "[" nestings ++ "]" nestings);
	404	defer scanner.deinit();
	405
	406	if (do_ensure) {
	407	try scanner.ensureTotalStackCapacity(nestings);
	408	}
	409
	410	try scanner.skipValue();
	411	try std.testing.expectEqual(Token.end_of_document, try scanner.next());
	412	}
	413	test "ensureTotalStackCapacity" {
	414	// Once to demonstrate failure.
	415	try std.testing.expectError(error.OutOfMemory, testEnsureStackCapacity(false));
	416	// Then to demonstrate it works.
	417	try testEnsureStackCapacity(true);
	418	}
	419
	420	fn testDiagnosticsFromSource(expected_error: ?anyerror, line: u64, col: u64, byte_offset: u64, source: anytype) !void {
	421	var diagnostics = Diagnostics{};
	422	source.enableDiagnostics(&diagnostics);
	423
	424	if (expected_error) \|expected_err\| {
	425	try std.testing.expectError(expected_err, source.skipValue());
	426	} else {
	427	try source.skipValue();
	428	try std.testing.expectEqual(Token.end_of_document, try source.next());
	429	}
	430	try std.testing.expectEqual(line, diagnostics.getLine());
	431	try std.testing.expectEqual(col, diagnostics.getColumn());
	432	try std.testing.expectEqual(byte_offset, diagnostics.getByteOffset());
	433	}
	434	fn testDiagnostics(expected_error: ?anyerror, line: u64, col: u64, byte_offset: u64, s: []const u8) !void {
	435	var scanner = JsonScanner.initCompleteInput(std.testing.allocator, s);
	436	defer scanner.deinit();
	437	try testDiagnosticsFromSource(expected_error, line, col, byte_offset, &scanner);
	438
	439	var tiny_stream = std.io.fixedBufferStream(s);
	440	var tiny_json_reader = JsonReader(1, @TypeOf(tiny_stream.reader())).init(std.testing.allocator, tiny_stream.reader());
	441	defer tiny_json_reader.deinit();
	442	try testDiagnosticsFromSource(expected_error, line, col, byte_offset, &tiny_json_reader);
	443
	444	var medium_stream = std.io.fixedBufferStream(s);
	445	var medium_json_reader = JsonReader(5, @TypeOf(medium_stream.reader())).init(std.testing.allocator, medium_stream.reader());
	446	defer medium_json_reader.deinit();
	447	try testDiagnosticsFromSource(expected_error, line, col, byte_offset, &medium_json_reader);
	448	}
	449	test "enableDiagnostics" {
	450	try testDiagnostics(error.UnexpectedEndOfInput, 1, 1, 0, "");
	451	try testDiagnostics(null, 1, 3, 2, "[]");
	452	try testDiagnostics(null, 2, 2, 3, "[\n]");
	453	try testDiagnostics(null, 14, 2, example_document_str.len, example_document_str);
	454
	455	try testDiagnostics(error.SyntaxError, 3, 1, 25,
	456	\\{
	457	\\ "common": "mistake",
	458	\\}
	459	);
	460
	461	inline for ([_]comptime_int{ 5, 6, 7, 99 }) \|reps\| {
	462	// The error happens 1 byte before the end.
	463	const s = "[" ** reps ++ "}";
	464	try testDiagnostics(error.SyntaxError, 1, s.len, s.len - 1, s);
	465	}
	466	}

lib/std/json/static.zig created+621

...	...	@@ -0,0 +1,621 @@
	1	const std = @import("std");
	2	const assert = std.debug.assert;
	3	const Allocator = std.mem.Allocator;
	4	const ArrayList = std.ArrayList;
	5
	6	const Scanner = @import("./scanner.zig").Scanner;
	7	const Token = @import("./scanner.zig").Token;
	8	const AllocWhen = @import("./scanner.zig").AllocWhen;
	9	const default_max_value_len = @import("./scanner.zig").default_max_value_len;
	10	const isNumberFormattedLikeAnInteger = @import("./scanner.zig").isNumberFormattedLikeAnInteger;
	11
	12	pub const ParseOptions = struct {
	13	/// Behaviour when a duplicate field is encountered.
	14	duplicate_field_behavior: enum {
	15	use_first,
	16	@"error",
	17	use_last,
	18	} = .@"error",
	19
	20	/// If false, finding an unknown field returns an error.
	21	ignore_unknown_fields: bool = false,
	22
	23	/// Passed to json.Scanner.nextAllocMax() or json.Reader.nextAllocMax().
	24	/// The default for parseFromSlice() or parseFromTokenSource() with a *json.Scanner input
	25	/// is the length of the input slice, which means error.ValueTooLong will never be returned.
	26	/// The default for parseFromTokenSource() with a *json.Reader is default_max_value_len.
	27	max_value_len: ?usize = null,
	28	};
	29
	30	/// Parses the json document from s and returns the result.
	31	/// The provided allocator is used both for temporary allocations during parsing the document,
	32	/// and also to allocate any pointer values in the return type.
	33	/// If T contains any pointers, free the memory with `std.json.parseFree`.
	34	/// Note that `error.BufferUnderrun` is not actually possible to return from this function.
	35	pub fn parseFromSlice(comptime T: type, allocator: Allocator, s: []const u8, options: ParseOptions) ParseError(T, Scanner)!T {
	36	var scanner = Scanner.initCompleteInput(allocator, s);
	37	defer scanner.deinit();
	38
	39	return parseFromTokenSource(T, allocator, &scanner, options);
	40	}
	41
	42	/// `scanner_or_reader` must be either a `std.json.Scanner` with complete input or a `std.json.Reader`.
	43	/// allocator is used to allocate the data of T if necessary,
	44	/// such as if T is `*u32` or `[]u32`.
	45	/// If T contains any pointers, free the memory with `std.json.parseFree`.
	46	/// If T contains no pointers, the allocator may sometimes be used for temporary allocations,
	47	/// but no call to `std.json.parseFree` will be necessary;
	48	/// all temporary allocations will be freed before this function returns.
	49	/// Note that `error.BufferUnderrun` is not actually possible to return from this function.
	50	pub fn parseFromTokenSource(comptime T: type, allocator: Allocator, scanner_or_reader: anytype, options: ParseOptions) ParseError(T, @TypeOf(scanner_or_reader.*))!T {
	51	if (@TypeOf(scanner_or_reader.*) == Scanner) {
	52	assert(scanner_or_reader.is_end_of_input);
	53	}
	54
	55	var resolved_options = options;
	56	if (resolved_options.max_value_len == null) {
	57	if (@TypeOf(scanner_or_reader.*) == Scanner) {
	58	resolved_options.max_value_len = scanner_or_reader.input.len;
	59	} else {
	60	resolved_options.max_value_len = default_max_value_len;
	61	}
	62	}
	63
	64	const r = try parseInternal(T, allocator, scanner_or_reader, resolved_options);
	65	errdefer parseFree(T, allocator, r);
	66
	67	assert(.end_of_document == try scanner_or_reader.next());
	68
	69	return r;
	70	}
	71
	72	/// The error set that will be returned from parsing T from *Source.
	73	/// Note that this may contain error.BufferUnderrun, but that error will never actually be returned.
	74	pub fn ParseError(comptime T: type, comptime Source: type) type {
	75	// `inferred_types` is used to avoid infinite recursion for recursive type definitions.
	76	const inferred_types = [_]type{};
	77	// A few of these will either always be present or present enough of the time that
	78	// omitting them is more confusing than always including them.
	79	return error{UnexpectedToken} \|\| Source.NextError \|\| Source.PeekError \|\|
	80	ParseInternalErrorImpl(T, Source, &inferred_types);
	81	}
	82
	83	fn ParseInternalErrorImpl(comptime T: type, comptime Source: type, comptime inferred_types: []const type) type {
	84	for (inferred_types) \|ty\| {
	85	if (T == ty) return error{};
	86	}
	87
	88	switch (@typeInfo(T)) {
	89	.Bool => return error{},
	90	.Float, .ComptimeFloat => return Source.AllocError \|\| std.fmt.ParseFloatError,
	91	.Int, .ComptimeInt => {
	92	return Source.AllocError \|\| error{ InvalidNumber, Overflow } \|\|
	93	std.fmt.ParseIntError \|\| std.fmt.ParseFloatError;
	94	},
	95	.Optional => \|optional_info\| return ParseInternalErrorImpl(optional_info.child, Source, inferred_types ++ [_]type{T}),
	96	.Enum => return Source.AllocError \|\| error{InvalidEnumTag},
	97	.Union => \|unionInfo\| {
	98	if (unionInfo.tag_type) \|_\| {
	99	var errors = Source.AllocError \|\| error{UnknownField};
	100	for (unionInfo.fields) \|u_field\| {
	101	errors = errors \|\| ParseInternalErrorImpl(u_field.type, Source, inferred_types ++ [_]type{T});
	102	}
	103	return errors;
	104	} else {
	105	@compileError("Unable to parse into untagged union '" ++ @typeName(T) ++ "'");
	106	}
	107	},
	108	.Struct => \|structInfo\| {
	109	var errors = Scanner.AllocError \|\| error{
	110	DuplicateField,
	111	UnknownField,
	112	MissingField,
	113	};
	114	for (structInfo.fields) \|field\| {
	115	errors = errors \|\| ParseInternalErrorImpl(field.type, Source, inferred_types ++ [_]type{T});
	116	}
	117	return errors;
	118	},
	119	.Array => \|arrayInfo\| {
	120	return error{LengthMismatch} \|\|
	121	ParseInternalErrorImpl(arrayInfo.child, Source, inferred_types ++ [_]type{T});
	122	},
	123	.Vector => \|vecInfo\| {
	124	return error{LengthMismatch} \|\|
	125	ParseInternalErrorImpl(vecInfo.child, Source, inferred_types ++ [_]type{T});
	126	},
	127	.Pointer => \|ptrInfo\| {
	128	switch (ptrInfo.size) {
	129	.One, .Slice => {
	130	return ParseInternalErrorImpl(ptrInfo.child, Source, inferred_types ++ [_]type{T});
	131	},
	132	else => @compileError("Unable to parse into type '" ++ @typeName(T) ++ "'"),
	133	}
	134	},
	135	else => return error{},
	136	}
	137	unreachable;
	138	}
	139
	140	fn parseInternal(
	141	comptime T: type,
	142	allocator: Allocator,
	143	source: anytype,
	144	options: ParseOptions,
	145	) ParseError(T, @TypeOf(source.*))!T {
	146	switch (@typeInfo(T)) {
	147	.Bool => {
	148	return switch (try source.next()) {
	149	.true => true,
	150	.false => false,
	151	else => error.UnexpectedToken,
	152	};
	153	},
	154	.Float, .ComptimeFloat => {
	155	const token = try source.nextAllocMax(allocator, .alloc_if_needed, options.max_value_len.?);
	156	defer freeAllocated(allocator, token);
	157	const slice = switch (token) {
	158	.number, .string => \|slice\| slice,
	159	.allocated_number, .allocated_string => \|slice\| slice,
	160	else => return error.UnexpectedToken,
	161	};
	162	return try std.fmt.parseFloat(T, slice);
	163	},
	164	.Int, .ComptimeInt => {
	165	const token = try source.nextAllocMax(allocator, .alloc_if_needed, options.max_value_len.?);
	166	defer freeAllocated(allocator, token);
	167	const slice = switch (token) {
	168	.number, .string => \|slice\| slice,
	169	.allocated_number, .allocated_string => \|slice\| slice,
	170	else => return error.UnexpectedToken,
	171	};
	172	if (isNumberFormattedLikeAnInteger(slice))
	173	return std.fmt.parseInt(T, slice, 10);
	174	// Try to coerce a float to an integer.
	175	const float = try std.fmt.parseFloat(f128, slice);
	176	if (@round(float) != float) return error.InvalidNumber;
	177	if (float > std.math.maxInt(T) or float < std.math.minInt(T)) return error.Overflow;
	178	return @floatToInt(T, float);
	179	},
	180	.Optional => \|optionalInfo\| {
	181	switch (try source.peekNextTokenType()) {
	182	.null => {
	183	_ = try source.next();
	184	return null;
	185	},
	186	else => {
	187	return try parseInternal(optionalInfo.child, allocator, source, options);
	188	},
	189	}
	190	},
	191	.Enum => \|enumInfo\| {
	192	const token = try source.nextAllocMax(allocator, .alloc_if_needed, options.max_value_len.?);
	193	defer freeAllocated(allocator, token);
	194	const slice = switch (token) {
	195	.number, .string => \|slice\| slice,
	196	.allocated_number, .allocated_string => \|slice\| slice,
	197	else => return error.UnexpectedToken,
	198	};
	199	// Check for a named value.
	200	if (std.meta.stringToEnum(T, slice)) \|value\| return value;
	201	// Check for a numeric value.
	202	if (!isNumberFormattedLikeAnInteger(slice)) return error.InvalidEnumTag;
	203	const n = std.fmt.parseInt(enumInfo.tag_type, slice, 10) catch return error.InvalidEnumTag;
	204	return try std.meta.intToEnum(T, n);
	205	},
	206	.Union => \|unionInfo\| {
	207	const UnionTagType = unionInfo.tag_type orelse @compileError("Unable to parse into untagged union '" ++ @typeName(T) ++ "'");
	208
	209	if (.object_begin != try source.next()) return error.UnexpectedToken;
	210
	211	var result: ?T = null;
	212	errdefer {
	213	if (result) \|r\| {
	214	inline for (unionInfo.fields) \|u_field\| {
	215	if (r == @field(UnionTagType, u_field.name)) {
	216	parseFree(u_field.type, allocator, @field(r, u_field.name));
	217	}
	218	}
	219	}
	220	}
	221
	222	var name_token: ?Token = try source.nextAllocMax(allocator, .alloc_if_needed, options.max_value_len.?);
	223	errdefer {
	224	if (name_token) \|t\| {
	225	freeAllocated(allocator, t);
	226	}
	227	}
	228	const field_name = switch (name_token.?) {
	229	.string => \|slice\| slice,
	230	.allocated_string => \|slice\| slice,
	231	else => return error.UnexpectedToken,
	232	};
	233
	234	inline for (unionInfo.fields) \|u_field\| {
	235	if (std.mem.eql(u8, u_field.name, field_name)) {
	236	// Free the name token now in case we're using an allocator that optimizes freeing the last allocated object.
	237	// (Recursing into parseInternal() might trigger more allocations.)
	238	freeAllocated(allocator, name_token.?);
	239	name_token = null;
	240
	241	if (u_field.type == void) {
	242	// void isn't really a json type, but we can support void payload union tags with {} as a value.
	243	if (.object_begin != try source.next()) return error.UnexpectedToken;
	244	if (.object_end != try source.next()) return error.UnexpectedToken;
	245	result = @unionInit(T, u_field.name, {});
	246	} else {
	247	// Recurse.
	248	result = @unionInit(T, u_field.name, try parseInternal(u_field.type, allocator, source, options));
	249	}
	250	break;
	251	}
	252	} else {
	253	// Didn't match anything.
	254	return error.UnknownField;
	255	}
	256
	257	if (.object_end != try source.next()) return error.UnexpectedToken;
	258
	259	return result.?;
	260	},
	261
	262	.Struct => \|structInfo\| {
	263	if (structInfo.is_tuple) {
	264	if (.array_begin != try source.next()) return error.UnexpectedToken;
	265
	266	var r: T = undefined;
	267	var fields_seen: usize = 0;
	268	errdefer {
	269	inline for (0..structInfo.fields.len) \|i\| {
	270	if (i < fields_seen) {
	271	parseFree(structInfo.fields[i].type, allocator, r[i]);
	272	}
	273	}
	274	}
	275	inline for (0..structInfo.fields.len) \|i\| {
	276	r[i] = try parseInternal(structInfo.fields[i].type, allocator, source, options);
	277	fields_seen = i + 1;
	278	}
	279
	280	if (.array_end != try source.next()) return error.UnexpectedToken;
	281
	282	return r;
	283	}
	284
	285	if (.object_begin != try source.next()) return error.UnexpectedToken;
	286
	287	var r: T = undefined;
	288	var fields_seen = [_]bool{false} ** structInfo.fields.len;
	289	errdefer {
	290	inline for (structInfo.fields, 0..) \|field, i\| {
	291	if (fields_seen[i]) {
	292	parseFree(field.type, allocator, @field(r, field.name));
	293	}
	294	}
	295	}
	296
	297	while (true) {
	298	var name_token: ?Token = try source.nextAllocMax(allocator, .alloc_if_needed, options.max_value_len.?);
	299	errdefer {
	300	if (name_token) \|t\| {
	301	freeAllocated(allocator, t);
	302	}
	303	}
	304	const field_name = switch (name_token.?) {
	305	.object_end => break, // No more fields.
	306	.string => \|slice\| slice,
	307	.allocated_string => \|slice\| slice,
	308	else => return error.UnexpectedToken,
	309	};
	310
	311	inline for (structInfo.fields, 0..) \|field, i\| {
	312	if (field.is_comptime) @compileError("comptime fields are not supported: " ++ @typeName(T) ++ "." ++ field.name);
	313	if (std.mem.eql(u8, field.name, field_name)) {
	314	// Free the name token now in case we're using an allocator that optimizes freeing the last allocated object.
	315	// (Recursing into parseInternal() might trigger more allocations.)
	316	freeAllocated(allocator, name_token.?);
	317	name_token = null;
	318
	319	if (fields_seen[i]) {
	320	switch (options.duplicate_field_behavior) {
	321	.use_first => {
	322	// Parse and then delete the redundant value.
	323	// We don't want to skip the value, because we want type checking.
	324	const ignored_value = try parseInternal(field.type, allocator, source, options);
	325	parseFree(field.type, allocator, ignored_value);
	326	break;
	327	},
	328	.@"error" => return error.DuplicateField,
	329	.use_last => {
	330	// Delete the stale value. We're about to get a new one.
	331	parseFree(field.type, allocator, @field(r, field.name));
	332	fields_seen[i] = false;
	333	},
	334	}
	335	}
	336	@field(r, field.name) = try parseInternal(field.type, allocator, source, options);
	337	fields_seen[i] = true;
	338	break;
	339	}
	340	} else {
	341	// Didn't match anything.
	342	freeAllocated(allocator, name_token.?);
	343	if (options.ignore_unknown_fields) {
	344	try source.skipValue();
	345	} else {
	346	return error.UnknownField;
	347	}
	348	}
	349	}
	350	inline for (structInfo.fields, 0..) \|field, i\| {
	351	if (!fields_seen[i]) {
	352	if (field.default_value) \|default_ptr\| {
	353	const default = @ptrCast(align(1) const field.type, default_ptr).;
	354	@field(r, field.name) = default;
	355	} else {
	356	return error.MissingField;
	357	}
	358	}
	359	}
	360	return r;
	361	},
	362
	363	.Array => \|arrayInfo\| {
	364	switch (try source.peekNextTokenType()) {
	365	.array_begin => {
	366	// Typical array.
	367	return parseInternalArray(T, arrayInfo.child, arrayInfo.len, allocator, source, options);
	368	},
	369	.string => {
	370	if (arrayInfo.child != u8) return error.UnexpectedToken;
	371	// Fixed-length string.
	372
	373	var r: T = undefined;
	374	var i: usize = 0;
	375	while (true) {
	376	switch (try source.next()) {
	377	.string => \|slice\| {
	378	if (i + slice.len != r.len) return error.LengthMismatch;
	379	@memcpy(r[i..][0..slice.len], slice);
	380	break;
	381	},
	382	.partial_string => \|slice\| {
	383	if (i + slice.len > r.len) return error.LengthMismatch;
	384	@memcpy(r[i..][0..slice.len], slice);
	385	i += slice.len;
	386	},
	387	.partial_string_escaped_1 => \|arr\| {
	388	if (i + arr.len > r.len) return error.LengthMismatch;
	389	@memcpy(r[i..][0..arr.len], arr[0..]);
	390	i += arr.len;
	391	},
	392	.partial_string_escaped_2 => \|arr\| {
	393	if (i + arr.len > r.len) return error.LengthMismatch;
	394	@memcpy(r[i..][0..arr.len], arr[0..]);
	395	i += arr.len;
	396	},
	397	.partial_string_escaped_3 => \|arr\| {
	398	if (i + arr.len > r.len) return error.LengthMismatch;
	399	@memcpy(r[i..][0..arr.len], arr[0..]);
	400	i += arr.len;
	401	},
	402	.partial_string_escaped_4 => \|arr\| {
	403	if (i + arr.len > r.len) return error.LengthMismatch;
	404	@memcpy(r[i..][0..arr.len], arr[0..]);
	405	i += arr.len;
	406	},
	407	else => unreachable,
	408	}
	409	}
	410
	411	return r;
	412	},
	413
	414	else => return error.UnexpectedToken,
	415	}
	416	},
	417
	418	.Vector => \|vecInfo\| {
	419	switch (try source.peekNextTokenType()) {
	420	.array_begin => {
	421	return parseInternalArray(T, vecInfo.child, vecInfo.len, allocator, source, options);
	422	},
	423	else => return error.UnexpectedToken,
	424	}
	425	},
	426
	427	.Pointer => \|ptrInfo\| {
	428	switch (ptrInfo.size) {
	429	.One => {
	430	const r: *ptrInfo.child = try allocator.create(ptrInfo.child);
	431	errdefer allocator.destroy(r);
	432	r.* = try parseInternal(ptrInfo.child, allocator, source, options);
	433	return r;
	434	},
	435	.Slice => {
	436	switch (try source.peekNextTokenType()) {
	437	.array_begin => {
	438	_ = try source.next();
	439
	440	// Typical array.
	441	var arraylist = ArrayList(ptrInfo.child).init(allocator);
	442	errdefer {
	443	while (arraylist.popOrNull()) \|v\| {
	444	parseFree(ptrInfo.child, allocator, v);
	445	}
	446	arraylist.deinit();
	447	}
	448
	449	while (true) {
	450	switch (try source.peekNextTokenType()) {
	451	.array_end => {
	452	_ = try source.next();
	453	break;
	454	},
	455	else => {},
	456	}
	457
	458	try arraylist.ensureUnusedCapacity(1);
	459	arraylist.appendAssumeCapacity(try parseInternal(ptrInfo.child, allocator, source, options));
	460	}
	461
	462	if (ptrInfo.sentinel) \|some\| {
	463	const sentinel_value = @ptrCast(align(1) const ptrInfo.child, some).;
	464	return try arraylist.toOwnedSliceSentinel(sentinel_value);
	465	}
	466
	467	return try arraylist.toOwnedSlice();
	468	},
	469	.string => {
	470	if (ptrInfo.child != u8) return error.UnexpectedToken;
	471
	472	// Dynamic length string.
	473	if (ptrInfo.sentinel) \|sentinel_ptr\| {
	474	// Use our own array list so we can append the sentinel.
	475	var value_list = ArrayList(u8).init(allocator);
	476	errdefer value_list.deinit();
	477	_ = try source.allocNextIntoArrayList(&value_list, .alloc_always);
	478	return try value_list.toOwnedSliceSentinel(@ptrCast(const u8, sentinel_ptr).);
	479	}
	480	switch (try source.nextAllocMax(allocator, .alloc_always, options.max_value_len.?)) {
	481	.allocated_string => \|slice\| return slice,
	482	else => unreachable,
	483	}
	484	},
	485	else => return error.UnexpectedToken,
	486	}
	487	},
	488	else => @compileError("Unable to parse into type '" ++ @typeName(T) ++ "'"),
	489	}
	490	},
	491	else => @compileError("Unable to parse into type '" ++ @typeName(T) ++ "'"),
	492	}
	493	unreachable;
	494	}
	495
	496	fn parseInternalArray(
	497	comptime T: type,
	498	comptime Child: type,
	499	comptime len: comptime_int,
	500	allocator: Allocator,
	501	source: anytype,
	502	options: ParseOptions,
	503	) !T {
	504	assert(.array_begin == try source.next());
	505
	506	var r: T = undefined;
	507	var i: usize = 0;
	508	errdefer {
	509	// Without the len check `r[i]` is not allowed
	510	if (len > 0) while (true) : (i -= 1) {
	511	parseFree(Child, allocator, r[i]);
	512	if (i == 0) break;
	513	};
	514	}
	515	while (i < len) : (i += 1) {
	516	r[i] = try parseInternal(Child, allocator, source, options);
	517	}
	518
	519	if (.array_end != try source.next()) return error.UnexpectedToken;
	520
	521	return r;
	522	}
	523
	524	fn freeAllocated(allocator: Allocator, token: Token) void {
	525	switch (token) {
	526	.allocated_number, .allocated_string => \|slice\| {
	527	allocator.free(slice);
	528	},
	529	else => {},
	530	}
	531	}
	532
	533	/// Releases resources created by parseFromSlice() or parseFromTokenSource().
	534	pub fn parseFree(comptime T: type, allocator: Allocator, value: T) void {
	535	switch (@typeInfo(T)) {
	536	.Bool, .Float, .ComptimeFloat, .Int, .ComptimeInt, .Enum => {},
	537	.Optional => {
	538	if (value) \|v\| {
	539	return parseFree(@TypeOf(v), allocator, v);
	540	}
	541	},
	542	.Union => \|unionInfo\| {
	543	if (unionInfo.tag_type) \|UnionTagType\| {
	544	inline for (unionInfo.fields) \|u_field\| {
	545	if (value == @field(UnionTagType, u_field.name)) {
	546	parseFree(u_field.type, allocator, @field(value, u_field.name));
	547	break;
	548	}
	549	}
	550	} else {
	551	unreachable;
	552	}
	553	},
	554	.Struct => \|structInfo\| {
	555	inline for (structInfo.fields) \|field\| {
	556	var should_free = true;
	557	if (field.default_value) \|default\| {
	558	switch (@typeInfo(field.type)) {
	559	// We must not attempt to free pointers to struct default values
	560	.Pointer => \|fieldPtrInfo\| {
	561	const field_value = @field(value, field.name);
	562	const field_ptr = switch (fieldPtrInfo.size) {
	563	.One => field_value,
	564	.Slice => field_value.ptr,
	565	else => unreachable, // Other pointer types are not parseable
	566	};
	567	const field_addr = @ptrToInt(field_ptr);
	568
	569	const casted_default = @ptrCast(const field.type, @alignCast(@alignOf(field.type), default)).;
	570	const default_ptr = switch (fieldPtrInfo.size) {
	571	.One => casted_default,
	572	.Slice => casted_default.ptr,
	573	else => unreachable, // Other pointer types are not parseable
	574	};
	575	const default_addr = @ptrToInt(default_ptr);
	576
	577	if (field_addr == default_addr) {
	578	should_free = false;
	579	}
	580	},
	581	else => {},
	582	}
	583	}
	584	if (should_free) {
	585	parseFree(field.type, allocator, @field(value, field.name));
	586	}
	587	}
	588	},
	589	.Array => \|arrayInfo\| {
	590	for (value) \|v\| {
	591	parseFree(arrayInfo.child, allocator, v);
	592	}
	593	},
	594	.Vector => \|vecInfo\| {
	595	var i: usize = 0;
	596	while (i < vecInfo.len) : (i += 1) {
	597	parseFree(vecInfo.child, allocator, value[i]);
	598	}
	599	},
	600	.Pointer => \|ptrInfo\| {
	601	switch (ptrInfo.size) {
	602	.One => {
	603	parseFree(ptrInfo.child, allocator, value.*);
	604	allocator.destroy(value);
	605	},
	606	.Slice => {
	607	for (value) \|v\| {
	608	parseFree(ptrInfo.child, allocator, v);
	609	}
	610	allocator.free(value);
	611	},
	612	else => unreachable,
	613	}
	614	},
	615	else => unreachable,
	616	}
	617	}
	618
	619	test {
	620	_ = @import("./static_test.zig");
	621	}

lib/std/json/static_test.zig created+437

...	...	@@ -0,0 +1,437 @@
	1	const std = @import("std");
	2	const testing = std.testing;
	3
	4	const parseFromSlice = @import("./static.zig").parseFromSlice;
	5	const parseFromTokenSource = @import("./static.zig").parseFromTokenSource;
	6	const parseFree = @import("./static.zig").parseFree;
	7	const ParseOptions = @import("./static.zig").ParseOptions;
	8	const JsonScanner = @import("./scanner.zig").Scanner;
	9	const jsonReader = @import("./scanner.zig").reader;
	10
	11	test "parse" {
	12	try testing.expectEqual(false, try parseFromSlice(bool, testing.allocator, "false", .{}));
	13	try testing.expectEqual(true, try parseFromSlice(bool, testing.allocator, "true", .{}));
	14	try testing.expectEqual(@as(u1, 1), try parseFromSlice(u1, testing.allocator, "1", .{}));
	15	try testing.expectError(error.Overflow, parseFromSlice(u1, testing.allocator, "50", .{}));
	16	try testing.expectEqual(@as(u64, 42), try parseFromSlice(u64, testing.allocator, "42", .{}));
	17	try testing.expectEqual(@as(f64, 42), try parseFromSlice(f64, testing.allocator, "42.0", .{}));
	18	try testing.expectEqual(@as(?bool, null), try parseFromSlice(?bool, testing.allocator, "null", .{}));
	19	try testing.expectEqual(@as(?bool, true), try parseFromSlice(?bool, testing.allocator, "true", .{}));
	20
	21	try testing.expectEqual(@as([3]u8, "foo".*), try parseFromSlice([3]u8, testing.allocator, "\"foo\"", .{}));
	22	try testing.expectEqual(@as([3]u8, "foo".*), try parseFromSlice([3]u8, testing.allocator, "[102, 111, 111]", .{}));
	23	try testing.expectEqual(@as([0]u8, undefined), try parseFromSlice([0]u8, testing.allocator, "[]", .{}));
	24
	25	try testing.expectEqual(@as(u64, 12345678901234567890), try parseFromSlice(u64, testing.allocator, "\"12345678901234567890\"", .{}));
	26	try testing.expectEqual(@as(f64, 123.456), try parseFromSlice(f64, testing.allocator, "\"123.456\"", .{}));
	27	}
	28
	29	test "parse into enum" {
	30	const T = enum(u32) {
	31	Foo = 42,
	32	Bar,
	33	@"with\\escape",
	34	};
	35	try testing.expectEqual(@as(T, .Foo), try parseFromSlice(T, testing.allocator, "\"Foo\"", .{}));
	36	try testing.expectEqual(@as(T, .Foo), try parseFromSlice(T, testing.allocator, "42", .{}));
	37	try testing.expectEqual(@as(T, .@"with\\escape"), try parseFromSlice(T, testing.allocator, "\"with\\\\escape\"", .{}));
	38	try testing.expectError(error.InvalidEnumTag, parseFromSlice(T, testing.allocator, "5", .{}));
	39	try testing.expectError(error.InvalidEnumTag, parseFromSlice(T, testing.allocator, "\"Qux\"", .{}));
	40	}
	41
	42	test "parse into that allocates a slice" {
	43	{
	44	// string as string
	45	const r = try parseFromSlice([]u8, testing.allocator, "\"foo\"", .{});
	46	defer parseFree([]u8, testing.allocator, r);
	47	try testing.expectEqualSlices(u8, "foo", r);
	48	}
	49	{
	50	// string as array of u8 integers
	51	const r = try parseFromSlice([]u8, testing.allocator, "[102, 111, 111]", .{});
	52	defer parseFree([]u8, testing.allocator, r);
	53	try testing.expectEqualSlices(u8, "foo", r);
	54	}
	55	{
	56	const r = try parseFromSlice([]u8, testing.allocator, "\"with\\\\escape\"", .{});
	57	defer parseFree([]u8, testing.allocator, r);
	58	try testing.expectEqualSlices(u8, "with\\escape", r);
	59	}
	60	}
	61
	62	test "parse into sentinel slice" {
	63	const result = try parseFromSlice([:0]const u8, testing.allocator, "\"\\n\"", .{});
	64	defer parseFree([:0]const u8, testing.allocator, result);
	65	try testing.expect(std.mem.eql(u8, result, "\n"));
	66	}
	67
	68	test "parse into tagged union" {
	69	const T = union(enum) {
	70	nothing,
	71	int: i32,
	72	float: f64,
	73	string: []const u8,
	74	};
	75	try testing.expectEqual(T{ .float = 1.5 }, try parseFromSlice(T, testing.allocator, "{\"float\":1.5}", .{}));
	76	try testing.expectEqual(T{ .int = 1 }, try parseFromSlice(T, testing.allocator, "{\"int\":1}", .{}));
	77	try testing.expectEqual(T{ .nothing = {} }, try parseFromSlice(T, testing.allocator, "{\"nothing\":{}}", .{}));
	78	}
	79
	80	test "parse into tagged union errors" {
	81	const T = union(enum) {
	82	nothing,
	83	int: i32,
	84	float: f64,
	85	string: []const u8,
	86	};
	87	try testing.expectError(error.UnexpectedToken, parseFromSlice(T, testing.allocator, "42", .{}));
	88	try testing.expectError(error.UnexpectedToken, parseFromSlice(T, testing.allocator, "{}", .{}));
	89	try testing.expectError(error.UnknownField, parseFromSlice(T, testing.allocator, "{\"bogus\":1}", .{}));
	90	try testing.expectError(error.UnexpectedToken, parseFromSlice(T, testing.allocator, "{\"int\":1, \"int\":1", .{}));
	91	try testing.expectError(error.UnexpectedToken, parseFromSlice(T, testing.allocator, "{\"int\":1, \"float\":1.0}", .{}));
	92	try testing.expectError(error.UnexpectedToken, parseFromSlice(T, testing.allocator, "{\"nothing\":null}", .{}));
	93	try testing.expectError(error.UnexpectedToken, parseFromSlice(T, testing.allocator, "{\"nothing\":{\"no\":0}}", .{}));
	94
	95	// Allocator failure
	96	var fail_alloc = testing.FailingAllocator.init(testing.allocator, 0);
	97	const failing_allocator = fail_alloc.allocator();
	98	try testing.expectError(error.OutOfMemory, parseFromSlice(T, failing_allocator, "{\"string\"\"foo\"}", .{}));
	99	}
	100
	101	test "parseFree descends into tagged union" {
	102	const T = union(enum) {
	103	nothing,
	104	int: i32,
	105	float: f64,
	106	string: []const u8,
	107	};
	108	const r = try parseFromSlice(T, testing.allocator, "{\"string\":\"foo\"}", .{});
	109	try testing.expectEqualSlices(u8, "foo", r.string);
	110	parseFree(T, testing.allocator, r);
	111	}
	112
	113	test "parse into struct with no fields" {
	114	const T = struct {};
	115	try testing.expectEqual(T{}, try parseFromSlice(T, testing.allocator, "{}", .{}));
	116	}
	117
	118	const test_const_value: usize = 123;
	119
	120	test "parse into struct with default const pointer field" {
	121	const T = struct { a: *const usize = &test_const_value };
	122	try testing.expectEqual(T{}, try parseFromSlice(T, testing.allocator, "{}", .{}));
	123	}
	124
	125	const test_default_usize: usize = 123;
	126	const test_default_usize_ptr: *align(1) const usize = &test_default_usize;
	127	const test_default_str: []const u8 = "test str";
	128	const test_default_str_slice: [2][]const u8 = [_][]const u8{
	129	"test1",
	130	"test2",
	131	};
	132
	133	test "freeing parsed structs with pointers to default values" {
	134	const T = struct {
	135	int: *const usize = &test_default_usize,
	136	int_ptr: *allowzero align(1) const usize = test_default_usize_ptr,
	137	str: []const u8 = test_default_str,
	138	str_slice: []const []const u8 = &test_default_str_slice,
	139	};
	140
	141	const parsed = try parseFromSlice(T, testing.allocator, "{}", .{});
	142	try testing.expectEqual(T{}, parsed);
	143	// This will panic if it tries to free global constants:
	144	parseFree(T, testing.allocator, parsed);
	145	}
	146
	147	test "parse into struct where destination and source lengths mismatch" {
	148	const T = struct { a: [2]u8 };
	149	try testing.expectError(error.LengthMismatch, parseFromSlice(T, testing.allocator, "{\"a\": \"bbb\"}", .{}));
	150	}
	151
	152	test "parse into struct with misc fields" {
	153	const T = struct {
	154	int: i64,
	155	float: f64,
	156	@"with\\escape": bool,
	157	@"withąunicode😂": bool,
	158	language: []const u8,
	159	optional: ?bool,
	160	default_field: i32 = 42,
	161	static_array: [3]f64,
	162	dynamic_array: []f64,
	163
	164	complex: struct {
	165	nested: []const u8,
	166	},
	167
	168	veryComplex: []struct {
	169	foo: []const u8,
	170	},
	171
	172	a_union: Union,
	173	const Union = union(enum) {
	174	x: u8,
	175	float: f64,
	176	string: []const u8,
	177	};
	178	};
	179	var document_str =
	180	\\{
	181	\\ "int": 420,
	182	\\ "float": 3.14,
	183	\\ "with\\escape": true,
	184	\\ "with\u0105unicode\ud83d\ude02": false,
	185	\\ "language": "zig",
	186	\\ "optional": null,
	187	\\ "static_array": [66.6, 420.420, 69.69],
	188	\\ "dynamic_array": [66.6, 420.420, 69.69],
	189	\\ "complex": {
	190	\\ "nested": "zig"
	191	\\ },
	192	\\ "veryComplex": [
	193	\\ {
	194	\\ "foo": "zig"
	195	\\ }, {
	196	\\ "foo": "rocks"
	197	\\ }
	198	\\ ],
	199	\\ "a_union": {
	200	\\ "float": 100000
	201	\\ }
	202	\\}
	203	;
	204	const r = try parseFromSlice(T, testing.allocator, document_str, .{});
	205	defer parseFree(T, testing.allocator, r);
	206	try testing.expectEqual(@as(i64, 420), r.int);
	207	try testing.expectEqual(@as(f64, 3.14), r.float);
	208	try testing.expectEqual(true, r.@"with\\escape");
	209	try testing.expectEqual(false, r.@"withąunicode😂");
	210	try testing.expectEqualSlices(u8, "zig", r.language);
	211	try testing.expectEqual(@as(?bool, null), r.optional);
	212	try testing.expectEqual(@as(i32, 42), r.default_field);
	213	try testing.expectEqual(@as(f64, 66.6), r.static_array[0]);
	214	try testing.expectEqual(@as(f64, 420.420), r.static_array[1]);
	215	try testing.expectEqual(@as(f64, 69.69), r.static_array[2]);
	216	try testing.expectEqual(@as(usize, 3), r.dynamic_array.len);
	217	try testing.expectEqual(@as(f64, 66.6), r.dynamic_array[0]);
	218	try testing.expectEqual(@as(f64, 420.420), r.dynamic_array[1]);
	219	try testing.expectEqual(@as(f64, 69.69), r.dynamic_array[2]);
	220	try testing.expectEqualSlices(u8, r.complex.nested, "zig");
	221	try testing.expectEqualSlices(u8, "zig", r.veryComplex[0].foo);
	222	try testing.expectEqualSlices(u8, "rocks", r.veryComplex[1].foo);
	223	try testing.expectEqual(T.Union{ .float = 100000 }, r.a_union);
	224	}
	225
	226	test "parse into struct with strings and arrays with sentinels" {
	227	const T = struct {
	228	language: [:0]const u8,
	229	language_without_sentinel: []const u8,
	230	data: [:99]const i32,
	231	simple_data: []const i32,
	232	};
	233	var document_str =
	234	\\{
	235	\\ "language": "zig",
	236	\\ "language_without_sentinel": "zig again!",
	237	\\ "data": [1, 2, 3],
	238	\\ "simple_data": [4, 5, 6]
	239	\\}
	240	;
	241	const r = try parseFromSlice(T, testing.allocator, document_str, .{});
	242	defer parseFree(T, testing.allocator, r);
	243
	244	try testing.expectEqualSentinel(u8, 0, "zig", r.language);
	245
	246	const data = [_:99]i32{ 1, 2, 3 };
	247	try testing.expectEqualSentinel(i32, 99, data[0..data.len], r.data);
	248
	249	// Make sure that arrays who aren't supposed to have a sentinel still parse without one.
	250	try testing.expectEqual(@as(?i32, null), std.meta.sentinel(@TypeOf(r.simple_data)));
	251	try testing.expectEqual(@as(?u8, null), std.meta.sentinel(@TypeOf(r.language_without_sentinel)));
	252	}
	253
	254	test "parse into struct with duplicate field" {
	255	// allow allocator to detect double frees by keeping bucket in use
	256	const ballast = try testing.allocator.alloc(u64, 1);
	257	defer testing.allocator.free(ballast);
	258
	259	const options_first = ParseOptions{ .duplicate_field_behavior = .use_first };
	260	const options_last = ParseOptions{ .duplicate_field_behavior = .use_last };
	261
	262	const str = "{ \"a\": 1, \"a\": 0.25 }";
	263
	264	const T1 = struct { a: *u64 };
	265	// both .use_first and .use_last should fail because second "a" value isn't a u64
	266	try testing.expectError(error.InvalidNumber, parseFromSlice(T1, testing.allocator, str, options_first));
	267	try testing.expectError(error.InvalidNumber, parseFromSlice(T1, testing.allocator, str, options_last));
	268
	269	const T2 = struct { a: f64 };
	270	try testing.expectEqual(T2{ .a = 1.0 }, try parseFromSlice(T2, testing.allocator, str, options_first));
	271	try testing.expectEqual(T2{ .a = 0.25 }, try parseFromSlice(T2, testing.allocator, str, options_last));
	272	}
	273
	274	test "parse into struct ignoring unknown fields" {
	275	const T = struct {
	276	int: i64,
	277	language: []const u8,
	278	};
	279
	280	var str =
	281	\\{
	282	\\ "int": 420,
	283	\\ "float": 3.14,
	284	\\ "with\\escape": true,
	285	\\ "with\u0105unicode\ud83d\ude02": false,
	286	\\ "optional": null,
	287	\\ "static_array": [66.6, 420.420, 69.69],
	288	\\ "dynamic_array": [66.6, 420.420, 69.69],
	289	\\ "complex": {
	290	\\ "nested": "zig"
	291	\\ },
	292	\\ "veryComplex": [
	293	\\ {
	294	\\ "foo": "zig"
	295	\\ }, {
	296	\\ "foo": "rocks"
	297	\\ }
	298	\\ ],
	299	\\ "a_union": {
	300	\\ "float": 100000
	301	\\ },
	302	\\ "language": "zig"
	303	\\}
	304	;
	305	const r = try parseFromSlice(T, testing.allocator, str, .{ .ignore_unknown_fields = true });
	306	defer parseFree(T, testing.allocator, r);
	307
	308	try testing.expectEqual(@as(i64, 420), r.int);
	309	try testing.expectEqualSlices(u8, "zig", r.language);
	310	}
	311
	312	test "parse into tuple" {
	313	const Union = union(enum) {
	314	char: u8,
	315	float: f64,
	316	string: []const u8,
	317	};
	318	const T = std.meta.Tuple(&.{
	319	i64,
	320	f64,
	321	bool,
	322	[]const u8,
	323	?bool,
	324	struct {
	325	foo: i32,
	326	bar: []const u8,
	327	},
	328	std.meta.Tuple(&.{ u8, []const u8, u8 }),
	329	Union,
	330	});
	331	var str =
	332	\\[
	333	\\ 420,
	334	\\ 3.14,
	335	\\ true,
	336	\\ "zig",
	337	\\ null,
	338	\\ {
	339	\\ "foo": 1,
	340	\\ "bar": "zero"
	341	\\ },
	342	\\ [4, "två", 42],
	343	\\ {"float": 12.34}
	344	\\]
	345	;
	346	const r = try parseFromSlice(T, testing.allocator, str, .{});
	347	defer parseFree(T, testing.allocator, r);
	348	try testing.expectEqual(@as(i64, 420), r[0]);
	349	try testing.expectEqual(@as(f64, 3.14), r[1]);
	350	try testing.expectEqual(true, r[2]);
	351	try testing.expectEqualSlices(u8, "zig", r[3]);
	352	try testing.expectEqual(@as(?bool, null), r[4]);
	353	try testing.expectEqual(@as(i32, 1), r[5].foo);
	354	try testing.expectEqualSlices(u8, "zero", r[5].bar);
	355	try testing.expectEqual(@as(u8, 4), r[6][0]);
	356	try testing.expectEqualSlices(u8, "två", r[6][1]);
	357	try testing.expectEqual(@as(u8, 42), r[6][2]);
	358	try testing.expectEqual(Union{ .float = 12.34 }, r[7]);
	359	}
	360
	361	const ParseIntoRecursiveUnionDefinitionValue = union(enum) {
	362	integer: i64,
	363	array: []const ParseIntoRecursiveUnionDefinitionValue,
	364	};
	365
	366	test "parse into recursive union definition" {
	367	const T = struct {
	368	values: ParseIntoRecursiveUnionDefinitionValue,
	369	};
	370
	371	const r = try parseFromSlice(T, testing.allocator, "{\"values\":{\"array\":[{\"integer\":58}]}}", .{});
	372	defer parseFree(T, testing.allocator, r);
	373
	374	try testing.expectEqual(@as(i64, 58), r.values.array[0].integer);
	375	}
	376
	377	const ParseIntoDoubleRecursiveUnionValueFirst = union(enum) {
	378	integer: i64,
	379	array: []const ParseIntoDoubleRecursiveUnionValueSecond,
	380	};
	381
	382	const ParseIntoDoubleRecursiveUnionValueSecond = union(enum) {
	383	boolean: bool,
	384	array: []const ParseIntoDoubleRecursiveUnionValueFirst,
	385	};
	386
	387	test "parse into double recursive union definition" {
	388	const T = struct {
	389	values: ParseIntoDoubleRecursiveUnionValueFirst,
	390	};
	391
	392	const r = try parseFromSlice(T, testing.allocator, "{\"values\":{\"array\":[{\"array\":[{\"integer\":58}]}]}}", .{});
	393	defer parseFree(T, testing.allocator, r);
	394
	395	try testing.expectEqual(@as(i64, 58), r.values.array[0].array[0].integer);
	396	}
	397
	398	test "parse exponential into int" {
	399	const T = struct { int: i64 };
	400	const r = try parseFromSlice(T, testing.allocator, "{ \"int\": 4.2e2 }", .{});
	401	try testing.expectEqual(@as(i64, 420), r.int);
	402	try testing.expectError(error.InvalidNumber, parseFromSlice(T, testing.allocator, "{ \"int\": 0.042e2 }", .{}));
	403	try testing.expectError(error.Overflow, parseFromSlice(T, testing.allocator, "{ \"int\": 18446744073709551616.0 }", .{}));
	404	}
	405
	406	test "parseFromTokenSource" {
	407	var scanner = JsonScanner.initCompleteInput(testing.allocator, "123");
	408	defer scanner.deinit();
	409	try testing.expectEqual(@as(u32, 123), try parseFromTokenSource(u32, testing.allocator, &scanner, .{}));
	410
	411	var stream = std.io.fixedBufferStream("123");
	412	var json_reader = jsonReader(std.testing.allocator, stream.reader());
	413	defer json_reader.deinit();
	414	try testing.expectEqual(@as(u32, 123), try parseFromTokenSource(u32, testing.allocator, &json_reader, .{}));
	415	}
	416
	417	test "max_value_len" {
	418	try testing.expectError(error.ValueTooLong, parseFromSlice([]u8, testing.allocator, "\"0123456789\"", .{ .max_value_len = 5 }));
	419	}
	420
	421	test "parse into vector" {
	422	const T = struct {
	423	vec_i32: @Vector(4, i32),
	424	vec_f32: @Vector(2, f32),
	425	};
	426	var s =
	427	\\{
	428	\\ "vec_f32": [1.5, 2.5],
	429	\\ "vec_i32": [4, 5, 6, 7]
	430	\\}
	431	;
	432	const r = try parseFromSlice(T, testing.allocator, s, .{});
	433	defer parseFree(T, testing.allocator, r);
	434	try testing.expectApproxEqAbs(@as(f32, 1.5), r.vec_f32[0], 0.0000001);
	435	try testing.expectApproxEqAbs(@as(f32, 2.5), r.vec_f32[1], 0.0000001);
	436	try testing.expectEqual(@Vector(4, i32){ 4, 5, 6, 7 }, r.vec_i32);
	437	}

lib/std/json/stringify.zig created+313

...	...	@@ -0,0 +1,313 @@
	1	const std = @import("std");
	2	const mem = std.mem;
	3	const assert = std.debug.assert;
	4
	5	pub const StringifyOptions = struct {
	6	pub const Whitespace = struct {
	7	/// How many indentation levels deep are we?
	8	indent_level: usize = 0,
	9
	10	/// What character(s) should be used for indentation?
	11	indent: union(enum) {
	12	space: u8,
	13	tab: void,
	14	none: void,
	15	} = .{ .space = 4 },
	16
	17	/// After a colon, should whitespace be inserted?
	18	separator: bool = true,
	19
	20	pub fn outputIndent(
	21	whitespace: @This(),
	22	out_stream: anytype,
	23	) @TypeOf(out_stream).Error!void {
	24	var char: u8 = undefined;
	25	var n_chars: usize = undefined;
	26	switch (whitespace.indent) {
	27	.space => \|n_spaces\| {
	28	char = ' ';
	29	n_chars = n_spaces;
	30	},
	31	.tab => {
	32	char = '\t';
	33	n_chars = 1;
	34	},
	35	.none => return,
	36	}
	37	try out_stream.writeByte('\n');
	38	n_chars *= whitespace.indent_level;
	39	try out_stream.writeByteNTimes(char, n_chars);
	40	}
	41	};
	42
	43	/// Controls the whitespace emitted
	44	whitespace: Whitespace = .{ .indent = .none, .separator = false },
	45
	46	/// Should optional fields with null value be written?
	47	emit_null_optional_fields: bool = true,
	48
	49	string: StringOptions = StringOptions{ .String = .{} },
	50
	51	/// Should []u8 be serialised as a string? or an array?
	52	pub const StringOptions = union(enum) {
	53	Array,
	54	String: StringOutputOptions,
	55
	56	/// String output options
	57	const StringOutputOptions = struct {
	58	/// Should '/' be escaped in strings?
	59	escape_solidus: bool = false,
	60
	61	/// Should unicode characters be escaped in strings?
	62	escape_unicode: bool = false,
	63	};
	64	};
	65	};
	66
	67	fn outputUnicodeEscape(
	68	codepoint: u21,
	69	out_stream: anytype,
	70	) !void {
	71	if (codepoint <= 0xFFFF) {
	72	// If the character is in the Basic Multilingual Plane (U+0000 through U+FFFF),
	73	// then it may be represented as a six-character sequence: a reverse solidus, followed
	74	// by the lowercase letter u, followed by four hexadecimal digits that encode the character's code point.
	75	try out_stream.writeAll("\\u");
	76	try std.fmt.formatIntValue(codepoint, "x", std.fmt.FormatOptions{ .width = 4, .fill = '0' }, out_stream);
	77	} else {
	78	assert(codepoint <= 0x10FFFF);
	79	// To escape an extended character that is not in the Basic Multilingual Plane,
	80	// the character is represented as a 12-character sequence, encoding the UTF-16 surrogate pair.
	81	const high = @intCast(u16, (codepoint - 0x10000) >> 10) + 0xD800;
	82	const low = @intCast(u16, codepoint & 0x3FF) + 0xDC00;
	83	try out_stream.writeAll("\\u");
	84	try std.fmt.formatIntValue(high, "x", std.fmt.FormatOptions{ .width = 4, .fill = '0' }, out_stream);
	85	try out_stream.writeAll("\\u");
	86	try std.fmt.formatIntValue(low, "x", std.fmt.FormatOptions{ .width = 4, .fill = '0' }, out_stream);
	87	}
	88	}
	89
	90	/// Write `string` to `writer` as a JSON encoded string.
	91	pub fn encodeJsonString(string: []const u8, options: StringifyOptions, writer: anytype) !void {
	92	try writer.writeByte('\"');
	93	try encodeJsonStringChars(string, options, writer);
	94	try writer.writeByte('\"');
	95	}
	96
	97	/// Write `chars` to `writer` as JSON encoded string characters.
	98	pub fn encodeJsonStringChars(chars: []const u8, options: StringifyOptions, writer: anytype) !void {
	99	var i: usize = 0;
	100	while (i < chars.len) : (i += 1) {
	101	switch (chars[i]) {
	102	// normal ascii character
	103	0x20...0x21, 0x23...0x2E, 0x30...0x5B, 0x5D...0x7F => \|c\| try writer.writeByte(c),
	104	// only 2 characters that must be escaped
	105	'\\' => try writer.writeAll("\\\\"),
	106	'\"' => try writer.writeAll("\\\""),
	107	// solidus is optional to escape
	108	'/' => {
	109	if (options.string.String.escape_solidus) {
	110	try writer.writeAll("\\/");
	111	} else {
	112	try writer.writeByte('/');
	113	}
	114	},
	115	// control characters with short escapes
	116	// TODO: option to switch between unicode and 'short' forms?
	117	0x8 => try writer.writeAll("\\b"),
	118	0xC => try writer.writeAll("\\f"),
	119	'\n' => try writer.writeAll("\\n"),
	120	'\r' => try writer.writeAll("\\r"),
	121	'\t' => try writer.writeAll("\\t"),
	122	else => {
	123	const ulen = std.unicode.utf8ByteSequenceLength(chars[i]) catch unreachable;
	124	// control characters (only things left with 1 byte length) should always be printed as unicode escapes
	125	if (ulen == 1 or options.string.String.escape_unicode) {
	126	const codepoint = std.unicode.utf8Decode(chars[i..][0..ulen]) catch unreachable;
	127	try outputUnicodeEscape(codepoint, writer);
	128	} else {
	129	try writer.writeAll(chars[i..][0..ulen]);
	130	}
	131	i += ulen - 1;
	132	},
	133	}
	134	}
	135	}
	136
	137	pub fn stringify(
	138	value: anytype,
	139	options: StringifyOptions,
	140	out_stream: anytype,
	141	) !void {
	142	const T = @TypeOf(value);
	143	switch (@typeInfo(T)) {
	144	.Float, .ComptimeFloat => {
	145	return std.fmt.formatFloatScientific(value, std.fmt.FormatOptions{}, out_stream);
	146	},
	147	.Int, .ComptimeInt => {
	148	return std.fmt.formatIntValue(value, "", std.fmt.FormatOptions{}, out_stream);
	149	},
	150	.Bool => {
	151	return out_stream.writeAll(if (value) "true" else "false");
	152	},
	153	.Null => {
	154	return out_stream.writeAll("null");
	155	},
	156	.Optional => {
	157	if (value) \|payload\| {
	158	return try stringify(payload, options, out_stream);
	159	} else {
	160	return try stringify(null, options, out_stream);
	161	}
	162	},
	163	.Enum => {
	164	if (comptime std.meta.trait.hasFn("jsonStringify")(T)) {
	165	return value.jsonStringify(options, out_stream);
	166	}
	167
	168	@compileError("Unable to stringify enum '" ++ @typeName(T) ++ "'");
	169	},
	170	.Union => {
	171	if (comptime std.meta.trait.hasFn("jsonStringify")(T)) {
	172	return value.jsonStringify(options, out_stream);
	173	}
	174
	175	const info = @typeInfo(T).Union;
	176	if (info.tag_type) \|UnionTagType\| {
	177	try out_stream.writeByte('{');
	178	var child_options = options;
	179	child_options.whitespace.indent_level += 1;
	180	inline for (info.fields) \|u_field\| {
	181	if (value == @field(UnionTagType, u_field.name)) {
	182	try child_options.whitespace.outputIndent(out_stream);
	183	try encodeJsonString(u_field.name, options, out_stream);
	184	try out_stream.writeByte(':');
	185	if (child_options.whitespace.separator) {
	186	try out_stream.writeByte(' ');
	187	}
	188	if (u_field.type == void) {
	189	try out_stream.writeAll("{}");
	190	} else {
	191	try stringify(@field(value, u_field.name), child_options, out_stream);
	192	}
	193	break;
	194	}
	195	} else {
	196	unreachable; // No active tag?
	197	}
	198	try options.whitespace.outputIndent(out_stream);
	199	try out_stream.writeByte('}');
	200	return;
	201	} else {
	202	@compileError("Unable to stringify untagged union '" ++ @typeName(T) ++ "'");
	203	}
	204	},
	205	.Struct => \|S\| {
	206	if (comptime std.meta.trait.hasFn("jsonStringify")(T)) {
	207	return value.jsonStringify(options, out_stream);
	208	}
	209
	210	try out_stream.writeByte(if (S.is_tuple) '[' else '{');
	211	var field_output = false;
	212	var child_options = options;
	213	child_options.whitespace.indent_level += 1;
	214	inline for (S.fields) \|Field\| {
	215	// don't include void fields
	216	if (Field.type == void) continue;
	217
	218	var emit_field = true;
	219
	220	// don't include optional fields that are null when emit_null_optional_fields is set to false
	221	if (@typeInfo(Field.type) == .Optional) {
	222	if (options.emit_null_optional_fields == false) {
	223	if (@field(value, Field.name) == null) {
	224	emit_field = false;
	225	}
	226	}
	227	}
	228
	229	if (emit_field) {
	230	if (!field_output) {
	231	field_output = true;
	232	} else {
	233	try out_stream.writeByte(',');
	234	}
	235	try child_options.whitespace.outputIndent(out_stream);
	236	if (!S.is_tuple) {
	237	try encodeJsonString(Field.name, options, out_stream);
	238	try out_stream.writeByte(':');
	239	if (child_options.whitespace.separator) {
	240	try out_stream.writeByte(' ');
	241	}
	242	}
	243	try stringify(@field(value, Field.name), child_options, out_stream);
	244	}
	245	}
	246	if (field_output) {
	247	try options.whitespace.outputIndent(out_stream);
	248	}
	249	try out_stream.writeByte(if (S.is_tuple) ']' else '}');
	250	return;
	251	},
	252	.ErrorSet => return stringify(@as([]const u8, @errorName(value)), options, out_stream),
	253	.Pointer => \|ptr_info\| switch (ptr_info.size) {
	254	.One => switch (@typeInfo(ptr_info.child)) {
	255	.Array => {
	256	const Slice = []const std.meta.Elem(ptr_info.child);
	257	return stringify(@as(Slice, value), options, out_stream);
	258	},
	259	else => {
	260	// TODO: avoid loops?
	261	return stringify(value.*, options, out_stream);
	262	},
	263	},
	264	.Many, .Slice => {
	265	if (ptr_info.size == .Many and ptr_info.sentinel == null)
	266	@compileError("unable to stringify type '" ++ @typeName(T) ++ "' without sentinel");
	267	const slice = if (ptr_info.size == .Many) mem.span(value) else value;
	268
	269	if (ptr_info.child == u8 and options.string == .String and std.unicode.utf8ValidateSlice(slice)) {
	270	try encodeJsonString(slice, options, out_stream);
	271	return;
	272	}
	273
	274	try out_stream.writeByte('[');
	275	var child_options = options;
	276	child_options.whitespace.indent_level += 1;
	277	for (slice, 0..) \|x, i\| {
	278	if (i != 0) {
	279	try out_stream.writeByte(',');
	280	}
	281	try child_options.whitespace.outputIndent(out_stream);
	282	try stringify(x, child_options, out_stream);
	283	}
	284	if (slice.len != 0) {
	285	try options.whitespace.outputIndent(out_stream);
	286	}
	287	try out_stream.writeByte(']');
	288	return;
	289	},
	290	else => @compileError("Unable to stringify type '" ++ @typeName(T) ++ "'"),
	291	},
	292	.Array => return stringify(&value, options, out_stream),
	293	.Vector => \|info\| {
	294	const array: [info.len]info.child = value;
	295	return stringify(&array, options, out_stream);
	296	},
	297	else => @compileError("Unable to stringify type '" ++ @typeName(T) ++ "'"),
	298	}
	299	unreachable;
	300	}
	301
	302	// Same as `stringify` but accepts an Allocator and stores result in dynamically allocated memory instead of using a Writer.
	303	// Caller owns returned memory.
	304	pub fn stringifyAlloc(allocator: std.mem.Allocator, value: anytype, options: StringifyOptions) ![]const u8 {
	305	var list = std.ArrayList(u8).init(allocator);
	306	errdefer list.deinit();
	307	try stringify(value, options, list.writer());
	308	return list.toOwnedSlice();
	309	}
	310
	311	test {
	312	_ = @import("./stringify_test.zig");
	313	}

lib/std/json/stringify_test.zig created+280

...	...	@@ -0,0 +1,280 @@
	1	const std = @import("std");
	2	const mem = std.mem;
	3	const testing = std.testing;
	4
	5	const StringifyOptions = @import("stringify.zig").StringifyOptions;
	6	const stringify = @import("stringify.zig").stringify;
	7	const stringifyAlloc = @import("stringify.zig").stringifyAlloc;
	8
	9	test "stringify null optional fields" {
	10	const MyStruct = struct {
	11	optional: ?[]const u8 = null,
	12	required: []const u8 = "something",
	13	another_optional: ?[]const u8 = null,
	14	another_required: []const u8 = "something else",
	15	};
	16	try teststringify(
	17	\\{"optional":null,"required":"something","another_optional":null,"another_required":"something else"}
	18	,
	19	MyStruct{},
	20	StringifyOptions{},
	21	);
	22	try teststringify(
	23	\\{"required":"something","another_required":"something else"}
	24	,
	25	MyStruct{},
	26	StringifyOptions{ .emit_null_optional_fields = false },
	27	);
	28	}
	29
	30	test "stringify basic types" {
	31	try teststringify("false", false, StringifyOptions{});
	32	try teststringify("true", true, StringifyOptions{});
	33	try teststringify("null", @as(?u8, null), StringifyOptions{});
	34	try teststringify("null", @as(?*u32, null), StringifyOptions{});
	35	try teststringify("42", 42, StringifyOptions{});
	36	try teststringify("4.2e+01", 42.0, StringifyOptions{});
	37	try teststringify("42", @as(u8, 42), StringifyOptions{});
	38	try teststringify("42", @as(u128, 42), StringifyOptions{});
	39	try teststringify("4.2e+01", @as(f32, 42), StringifyOptions{});
	40	try teststringify("4.2e+01", @as(f64, 42), StringifyOptions{});
	41	try teststringify("\"ItBroke\"", @as(anyerror, error.ItBroke), StringifyOptions{});
	42	}
	43
	44	test "stringify string" {
	45	try teststringify("\"hello\"", "hello", StringifyOptions{});
	46	try teststringify("\"with\\nescapes\\r\"", "with\nescapes\r", StringifyOptions{});
	47	try teststringify("\"with\\nescapes\\r\"", "with\nescapes\r", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
	48	try teststringify("\"with unicode\\u0001\"", "with unicode\u{1}", StringifyOptions{});
	49	try teststringify("\"with unicode\\u0001\"", "with unicode\u{1}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
	50	try teststringify("\"with unicode\u{80}\"", "with unicode\u{80}", StringifyOptions{});
	51	try teststringify("\"with unicode\\u0080\"", "with unicode\u{80}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
	52	try teststringify("\"with unicode\u{FF}\"", "with unicode\u{FF}", StringifyOptions{});
	53	try teststringify("\"with unicode\\u00ff\"", "with unicode\u{FF}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
	54	try teststringify("\"with unicode\u{100}\"", "with unicode\u{100}", StringifyOptions{});
	55	try teststringify("\"with unicode\\u0100\"", "with unicode\u{100}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
	56	try teststringify("\"with unicode\u{800}\"", "with unicode\u{800}", StringifyOptions{});
	57	try teststringify("\"with unicode\\u0800\"", "with unicode\u{800}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
	58	try teststringify("\"with unicode\u{8000}\"", "with unicode\u{8000}", StringifyOptions{});
	59	try teststringify("\"with unicode\\u8000\"", "with unicode\u{8000}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
	60	try teststringify("\"with unicode\u{D799}\"", "with unicode\u{D799}", StringifyOptions{});
	61	try teststringify("\"with unicode\\ud799\"", "with unicode\u{D799}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
	62	try teststringify("\"with unicode\u{10000}\"", "with unicode\u{10000}", StringifyOptions{});
	63	try teststringify("\"with unicode\\ud800\\udc00\"", "with unicode\u{10000}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
	64	try teststringify("\"with unicode\u{10FFFF}\"", "with unicode\u{10FFFF}", StringifyOptions{});
	65	try teststringify("\"with unicode\\udbff\\udfff\"", "with unicode\u{10FFFF}", StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
	66	try teststringify("\"/\"", "/", StringifyOptions{});
	67	try teststringify("\"\\/\"", "/", StringifyOptions{ .string = .{ .String = .{ .escape_solidus = true } } });
	68	}
	69
	70	test "stringify many-item sentinel-terminated string" {
	71	try teststringify("\"hello\"", @as([*:0]const u8, "hello"), StringifyOptions{});
	72	try teststringify("\"with\\nescapes\\r\"", @as([*:0]const u8, "with\nescapes\r"), StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
	73	try teststringify("\"with unicode\\u0001\"", @as([*:0]const u8, "with unicode\u{1}"), StringifyOptions{ .string = .{ .String = .{ .escape_unicode = true } } });
	74	}
	75
	76	test "stringify tagged unions" {
	77	const T = union(enum) {
	78	nothing,
	79	foo: u32,
	80	bar: bool,
	81	};
	82	try teststringify("{\"nothing\":{}}", T{ .nothing = {} }, StringifyOptions{});
	83	try teststringify("{\"foo\":42}", T{ .foo = 42 }, StringifyOptions{});
	84	try teststringify("{\"bar\":true}", T{ .bar = true }, StringifyOptions{});
	85	}
	86
	87	test "stringify struct" {
	88	try teststringify("{\"foo\":42}", struct {
	89	foo: u32,
	90	}{ .foo = 42 }, StringifyOptions{});
	91	}
	92
	93	test "stringify struct with string as array" {
	94	try teststringify("{\"foo\":\"bar\"}", .{ .foo = "bar" }, StringifyOptions{});
	95	try teststringify("{\"foo\":[98,97,114]}", .{ .foo = "bar" }, StringifyOptions{ .string = .Array });
	96	}
	97
	98	test "stringify struct with indentation" {
	99	try teststringify(
	100	\\{
	101	\\ "foo": 42,
	102	\\ "bar": [
	103	\\ 1,
	104	\\ 2,
	105	\\ 3
	106	\\ ]
	107	\\}
	108	,
	109	struct {
	110	foo: u32,
	111	bar: [3]u32,
	112	}{
	113	.foo = 42,
	114	.bar = .{ 1, 2, 3 },
	115	},
	116	StringifyOptions{
	117	.whitespace = .{},
	118	},
	119	);
	120	try teststringify(
	121	"{\n\t\"foo\":42,\n\t\"bar\":[\n\t\t1,\n\t\t2,\n\t\t3\n\t]\n}",
	122	struct {
	123	foo: u32,
	124	bar: [3]u32,
	125	}{
	126	.foo = 42,
	127	.bar = .{ 1, 2, 3 },
	128	},
	129	StringifyOptions{
	130	.whitespace = .{
	131	.indent = .tab,
	132	.separator = false,
	133	},
	134	},
	135	);
	136	try teststringify(
	137	\\{"foo":42,"bar":[1,2,3]}
	138	,
	139	struct {
	140	foo: u32,
	141	bar: [3]u32,
	142	}{
	143	.foo = 42,
	144	.bar = .{ 1, 2, 3 },
	145	},
	146	StringifyOptions{
	147	.whitespace = .{
	148	.indent = .none,
	149	.separator = false,
	150	},
	151	},
	152	);
	153	}
	154
	155	test "stringify struct with void field" {
	156	try teststringify("{\"foo\":42}", struct {
	157	foo: u32,
	158	bar: void = {},
	159	}{ .foo = 42 }, StringifyOptions{});
	160	}
	161
	162	test "stringify array of structs" {
	163	const MyStruct = struct {
	164	foo: u32,
	165	};
	166	try teststringify("[{\"foo\":42},{\"foo\":100},{\"foo\":1000}]", [_]MyStruct{
	167	MyStruct{ .foo = 42 },
	168	MyStruct{ .foo = 100 },
	169	MyStruct{ .foo = 1000 },
	170	}, StringifyOptions{});
	171	}
	172
	173	test "stringify struct with custom stringifier" {
	174	try teststringify("[\"something special\",42]", struct {
	175	foo: u32,
	176	const Self = @This();
	177	pub fn jsonStringify(
	178	value: Self,
	179	options: StringifyOptions,
	180	out_stream: anytype,
	181	) !void {
	182	_ = value;
	183	try out_stream.writeAll("[\"something special\",");
	184	try stringify(42, options, out_stream);
	185	try out_stream.writeByte(']');
	186	}
	187	}{ .foo = 42 }, StringifyOptions{});
	188	}
	189
	190	test "stringify vector" {
	191	try teststringify("[1,1]", @splat(2, @as(u32, 1)), StringifyOptions{});
	192	}
	193
	194	test "stringify tuple" {
	195	try teststringify("[\"foo\",42]", std.meta.Tuple(&.{ []const u8, usize }){ "foo", 42 }, StringifyOptions{});
	196	}
	197
	198	fn teststringify(expected: []const u8, value: anytype, options: StringifyOptions) !void {
	199	const ValidationWriter = struct {
	200	const Self = @This();
	201	pub const Writer = std.io.Writer(*Self, Error, write);
	202	pub const Error = error{
	203	TooMuchData,
	204	DifferentData,
	205	};
	206
	207	expected_remaining: []const u8,
	208
	209	fn init(exp: []const u8) Self {
	210	return .{ .expected_remaining = exp };
	211	}
	212
	213	pub fn writer(self: *Self) Writer {
	214	return .{ .context = self };
	215	}
	216
	217	fn write(self: *Self, bytes: []const u8) Error!usize {
	218	if (self.expected_remaining.len < bytes.len) {
	219	std.debug.print(
	220	\\====== expected this output: =========
	221	\\{s}
	222	\\======== instead found this: =========
	223	\\{s}
	224	\\======================================
	225	, .{
	226	self.expected_remaining,
	227	bytes,
	228	});
	229	return error.TooMuchData;
	230	}
	231	if (!mem.eql(u8, self.expected_remaining[0..bytes.len], bytes)) {
	232	std.debug.print(
	233	\\====== expected this output: =========
	234	\\{s}
	235	\\======== instead found this: =========
	236	\\{s}
	237	\\======================================
	238	, .{
	239	self.expected_remaining[0..bytes.len],
	240	bytes,
	241	});
	242	return error.DifferentData;
	243	}
	244	self.expected_remaining = self.expected_remaining[bytes.len..];
	245	return bytes.len;
	246	}
	247	};
	248
	249	var vos = ValidationWriter.init(expected);
	250	try stringify(value, options, vos.writer());
	251	if (vos.expected_remaining.len > 0) return error.NotEnoughData;
	252	}
	253
	254	test "stringify struct with custom stringify that returns a custom error" {
	255	var ret = stringify(struct {
	256	field: Field = .{},
	257
	258	pub const Field = struct {
	259	field: ?[]*Field = null,
	260
	261	const Self = @This();
	262	pub fn jsonStringify(_: Self, _: StringifyOptions, _: anytype) error{CustomError}!void {
	263	return error.CustomError;
	264	}
	265	};
	266	}{}, StringifyOptions{}, std.io.null_writer);
	267
	268	try std.testing.expectError(error.CustomError, ret);
	269	}
	270
	271	test "stringify alloc" {
	272	const allocator = std.testing.allocator;
	273	const expected =
	274	\\{"foo":"bar","answer":42,"my_friend":"sammy"}
	275	;
	276	const actual = try stringifyAlloc(allocator, .{ .foo = "bar", .answer = 42, .my_friend = "sammy" }, .{});
	277	defer allocator.free(actual);
	278
	279	try std.testing.expectEqualStrings(expected, actual);
	280	}

lib/std/json/test.zig+76-2923

...	...	@@ -1,2960 +1,113 @@
1		// RFC 8529 conformance tests.
2		//
3		// Tests are taken from https://github.com/nst/JSONTestSuite
4		// Read also http://seriot.ch/parsing_json.php for a good overview.
5
6		const std = @import("../std.zig");
7		const json = std.json;
	1	const std = @import("std");
8	2	const testing = std.testing;
9		const TokenStream = std.json.TokenStream;
10		const parse = std.json.parse;
11		const ParseOptions = std.json.ParseOptions;
12		const parseFree = std.json.parseFree;
13		const Parser = std.json.Parser;
14		const mem = std.mem;
15		const writeStream = std.json.writeStream;
16		const Value = std.json.Value;
17		const StringifyOptions = std.json.StringifyOptions;
18		const stringify = std.json.stringify;
19		const stringifyAlloc = std.json.stringifyAlloc;
20		const StreamingParser = std.json.StreamingParser;
21		const Token = std.json.Token;
22		const validate = std.json.validate;
23		const Array = std.json.Array;
24		const ObjectMap = std.json.ObjectMap;
25		const assert = std.debug.assert;
26
27		fn testNonStreaming(s: []const u8) !void {
28		var p = json.Parser.init(testing.allocator, false);
29		defer p.deinit();
30
31		var tree = try p.parse(s);
32		defer tree.deinit();
33		}
34
35		fn ok(s: []const u8) !void {
36		try testing.expect(json.validate(s));
	3	const Parser = @import("./dynamic.zig").Parser;
	4	const validate = @import("./scanner.zig").validate;
	5	const JsonScanner = @import("./scanner.zig").Scanner;
37	6
38		try testNonStreaming(s);
	7	// Support for JSONTestSuite.zig
	8	pub fn ok(s: []const u8) !void {
	9	try testLowLevelScanner(s);
	10	try testHighLevelDynamicParser(s);
39	11	}
40
41		fn err(s: []const u8) !void {
42		try testing.expect(!json.validate(s));
43
44		try testing.expect(std.meta.isError(testNonStreaming(s)));
45		}
46
47		fn utf8Error(s: []const u8) !void {
48		try testing.expect(!json.validate(s));
49
50		try testing.expectError(error.InvalidUtf8Byte, testNonStreaming(s));
	12	pub fn err(s: []const u8) !void {
	13	try testing.expect(std.meta.isError(testLowLevelScanner(s)));
	14	try testing.expect(std.meta.isError(testHighLevelDynamicParser(s)));
51	15	}
52
53		fn any(s: []const u8) !void {
54		_ = json.validate(s);
55
56		testNonStreaming(s) catch {};
	16	pub fn any(s: []const u8) !void {
	17	testLowLevelScanner(s) catch {};
	18	testHighLevelDynamicParser(s) catch {};
57	19	}
58
59		fn anyStreamingErrNonStreaming(s: []const u8) !void {
60		_ = json.validate(s);
61
62		try testing.expect(std.meta.isError(testNonStreaming(s)));
	20	fn testLowLevelScanner(s: []const u8) !void {
	21	var scanner = JsonScanner.initCompleteInput(testing.allocator, s);
	22	defer scanner.deinit();
	23	while (true) {
	24	const token = try scanner.next();
	25	if (token == .end_of_document) break;
	26	}
63	27	}
64
65		fn roundTrip(s: []const u8) !void {
66		try testing.expect(json.validate(s));
67
68		var p = json.Parser.init(testing.allocator, false);
	28	fn testHighLevelDynamicParser(s: []const u8) !void {
	29	var p = Parser.init(testing.allocator, .alloc_if_needed);
69	30	defer p.deinit();
70
71	31	var tree = try p.parse(s);
72	32	defer tree.deinit();
73
74		var buf: [256]u8 = undefined;
75		var fbs = std.io.fixedBufferStream(&buf);
76		try tree.root.jsonStringify(.{}, fbs.writer());
77
78		try testing.expectEqualStrings(s, fbs.getWritten());
79	33	}
80	34
81		////////////////////////////////////////////////////////////////////////////////////////////////////
82		//
83	35	// Additional tests not part of test JSONTestSuite.
84
85	36	test "y_trailing_comma_after_empty" {
86	37	try roundTrip(
87	38	\\{"1":[],"2":{},"3":"4"}
88	39	);
89	40	}
90
91	41	test "n_object_closed_missing_value" {
92	42	try err(
93	43	\\{"a":}
94	44	);
95	45	}
96	46
97		////////////////////////////////////////////////////////////////////////////////////////////////////
98
99		test "y_array_arraysWithSpaces" {
100		try ok(
101		\\[[] ]
102		);
103		}
104
105		test "y_array_empty" {
106		try roundTrip(
107		\\[]
108		);
109		}
110
111		test "y_array_empty-string" {
112		try roundTrip(
113		\\[""]
114		);
115		}
116
117		test "y_array_ending_with_newline" {
118		try roundTrip(
119		\\["a"]
120		);
121		}
122
123		test "y_array_false" {
124		try roundTrip(
125		\\[false]
126		);
127		}
128
129		test "y_array_heterogeneous" {
130		try ok(
131		\\[null, 1, "1", {}]
132		);
133		}
134
135		test "y_array_null" {
136		try roundTrip(
137		\\[null]
138		);
139		}
140
141		test "y_array_with_1_and_newline" {
142		try ok(
143		\\[1
144		\\]
145		);
146		}
147
148		test "y_array_with_leading_space" {
149		try ok(
150		\\ [1]
151		);
152		}
153
154		test "y_array_with_several_null" {
155		try roundTrip(
156		\\[1,null,null,null,2]
157		);
158		}
159
160		test "y_array_with_trailing_space" {
161		try ok("[2] ");
162		}
163
164		test "y_number_0e+1" {
165		try ok(
166		\\[0e+1]
167		);
168		}
169
170		test "y_number_0e1" {
171		try ok(
172		\\[0e1]
173		);
174		}
175
176		test "y_number_after_space" {
177		try ok(
178		\\[ 4]
179		);
180		}
181
182		test "y_number_double_close_to_zero" {
183		try ok(
184		\\[-0.000000000000000000000000000000000000000000000000000000000000000000000000000001]
185		);
186		}
187
188		test "y_number_int_with_exp" {
189		try ok(
190		\\[20e1]
191		);
192		}
193
194		test "y_number" {
195		try ok(
196		\\[123e65]
197		);
198		}
199
200		test "y_number_minus_zero" {
201		try ok(
202		\\[-0]
203		);
204		}
205
206		test "y_number_negative_int" {
207		try roundTrip(
208		\\[-123]
209		);
210		}
211
212		test "y_number_negative_one" {
213		try roundTrip(
214		\\[-1]
215		);
216		}
217
218		test "y_number_negative_zero" {
219		try ok(
220		\\[-0]
221		);
222		}
223
224		test "y_number_real_capital_e" {
225		try ok(
226		\\[1E22]
227		);
228		}
229
230		test "y_number_real_capital_e_neg_exp" {
231		try ok(
232		\\[1E-2]
233		);
234		}
235
236		test "y_number_real_capital_e_pos_exp" {
237		try ok(
238		\\[1E+2]
239		);
240		}
241
242		test "y_number_real_exponent" {
243		try ok(
244		\\[123e45]
245		);
246		}
247
248		test "y_number_real_fraction_exponent" {
249		try ok(
250		\\[123.456e78]
251		);
252		}
253
254		test "y_number_real_neg_exp" {
255		try ok(
256		\\[1e-2]
257		);
258		}
259
260		test "y_number_real_pos_exponent" {
261		try ok(
262		\\[1e+2]
263		);
264		}
265
266		test "y_number_simple_int" {
267		try roundTrip(
268		\\[123]
269		);
270		}
271
272		test "y_number_simple_real" {
273		try ok(
274		\\[123.456789]
275		);
276		}
277
278		test "y_object_basic" {
279		try roundTrip(
280		\\{"asd":"sdf"}
281		);
282		}
283
284		test "y_object_duplicated_key_and_value" {
285		try ok(
286		\\{"a":"b","a":"b"}
287		);
288		}
289
290		test "y_object_duplicated_key" {
291		try ok(
292		\\{"a":"b","a":"c"}
293		);
294		}
295
296		test "y_object_empty" {
297		try roundTrip(
298		\\{}
299		);
300		}
301
302		test "y_object_empty_key" {
303		try roundTrip(
304		\\{"":0}
305		);
306		}
307
308		test "y_object_escaped_null_in_key" {
309		try ok(
310		\\{"foo\u0000bar": 42}
311		);
312		}
313
314		test "y_object_extreme_numbers" {
315		try ok(
316		\\{ "min": -1.0e+28, "max": 1.0e+28 }
317		);
318		}
319
320		test "y_object" {
321		try ok(
322		\\{"asd":"sdf", "dfg":"fgh"}
323		);
324		}
325
326		test "y_object_long_strings" {
327		try ok(
328		\\{"x":[{"id": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"}], "id": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"}
329		);
330		}
331
332		test "y_object_simple" {
333		try roundTrip(
334		\\{"a":[]}
335		);
336		}
337
338		test "y_object_string_unicode" {
339		try ok(
340		\\{"title":"\u041f\u043e\u043b\u0442\u043e\u0440\u0430 \u0417\u0435\u043c\u043b\u0435\u043a\u043e\u043f\u0430" }
341		);
342		}
343
344		test "y_object_with_newlines" {
345		try ok(
346		\\{
347		\\"a": "b"
348		\\}
349		);
350		}
351
352		test "y_string_1_2_3_bytes_UTF-8_sequences" {
353		try ok(
354		\\["\u0060\u012a\u12AB"]
355		);
356		}
357
358		test "y_string_accepted_surrogate_pair" {
359		try ok(
360		\\["\uD801\udc37"]
361		);
362		}
363
364		test "y_string_accepted_surrogate_pairs" {
365		try ok(
366		\\["\ud83d\ude39\ud83d\udc8d"]
367		);
368		}
369
370		test "y_string_allowed_escapes" {
371		try ok(
372		\\["\"\\\/\b\f\n\r\t"]
373		);
374		}
375
376		test "y_string_backslash_and_u_escaped_zero" {
377		try ok(
378		\\["\\u0000"]
379		);
380		}
381
382		test "y_string_backslash_doublequotes" {
383		try roundTrip(
384		\\["\""]
385		);
386		}
387
388		test "y_string_comments" {
389		try ok(
390		\\["a/b/c/*d//e"]
391		);
392		}
393
394		test "y_string_double_escape_a" {
395		try ok(
396		\\["\\a"]
397		);
398		}
399
400		test "y_string_double_escape_n" {
401		try roundTrip(
402		\\["\\n"]
403		);
404		}
405
406		test "y_string_escaped_control_character" {
407		try ok(
408		\\["\u0012"]
409		);
410		}
411
412		test "y_string_escaped_noncharacter" {
413		try ok(
414		\\["\uFFFF"]
415		);
416		}
417
418		test "y_string_in_array" {
419		try ok(
420		\\["asd"]
421		);
422		}
423
424		test "y_string_in_array_with_leading_space" {
425		try ok(
426		\\[ "asd"]
427		);
428		}
429
430		test "y_string_last_surrogates_1_and_2" {
431		try ok(
432		\\["\uDBFF\uDFFF"]
433		);
434		}
435
436		test "y_string_nbsp_uescaped" {
437		try ok(
438		\\["new\u00A0line"]
439		);
440		}
441
442		test "y_string_nonCharacterInUTF-8_U+10FFFF" {
443		try ok(
444		\\["􏿿"]
445		);
446		}
447
448		test "y_string_nonCharacterInUTF-8_U+FFFF" {
449		try ok(
450		\\[""]
451		);
452		}
453
454		test "y_string_null_escape" {
455		try ok(
456		\\["\u0000"]
457		);
458		}
459
460		test "y_string_one-byte-utf-8" {
461		try ok(
462		\\["\u002c"]
463		);
464		}
465
466		test "y_string_pi" {
467		try ok(
468		\\["π"]
469		);
470		}
471
472		test "y_string_reservedCharacterInUTF-8_U+1BFFF" {
473		try ok(
474		\\["𛿿"]
475		);
476		}
477
478		test "y_string_simple_ascii" {
479		try ok(
480		\\["asd "]
481		);
482		}
	47	fn roundTrip(s: []const u8) !void {
	48	try testing.expect(try validate(testing.allocator, s));
483	49
484		test "y_string_space" {
485		try roundTrip(
486		\\" "
487		);
488		}
	50	var p = Parser.init(testing.allocator, .alloc_if_needed);
	51	defer p.deinit();
489	52
490		test "y_string_surrogates_U+1D11E_MUSICAL_SYMBOL_G_CLEF" {
491		try ok(
492		\\["\uD834\uDd1e"]
493		);
494		}
	53	var tree = try p.parse(s);
	54	defer tree.deinit();
495	55
496		test "y_string_three-byte-utf-8" {
497		try ok(
498		\\["\u0821"]
499		);
500		}
	56	var buf: [256]u8 = undefined;
	57	var fbs = std.io.fixedBufferStream(&buf);
	58	try tree.root.jsonStringify(.{}, fbs.writer());
501	59
502		test "y_string_two-byte-utf-8" {
503		try ok(
504		\\["\u0123"]
505		);
	60	try testing.expectEqualStrings(s, fbs.getWritten());
506	61	}
507	62
508		test "y_string_u+2028_line_sep" {
509		try ok("[\"\xe2\x80\xa8\"]");
	63	test "truncated UTF-8 sequence" {
	64	try err("\"\xc2\"");
	65	try err("\"\xdf\"");
	66	try err("\"\xed\xa0\"");
	67	try err("\"\xf0\x80\"");
	68	try err("\"\xf0\x80\x80\"");
510	69	}
511	70
512		test "y_string_u+2029_par_sep" {
513		try ok("[\"\xe2\x80\xa9\"]");
	71	test "invalid continuation byte" {
	72	try err("\"\xc2\x00\"");
	73	try err("\"\xc2\x7f\"");
	74	try err("\"\xc2\xc0\"");
	75	try err("\"\xc3\xc1\"");
	76	try err("\"\xc4\xf5\"");
	77	try err("\"\xc5\xff\"");
	78	try err("\"\xe4\x80\x00\"");
	79	try err("\"\xe5\x80\x10\"");
	80	try err("\"\xe6\x80\xc0\"");
	81	try err("\"\xe7\x80\xf5\"");
	82	try err("\"\xe8\x00\x80\"");
	83	try err("\"\xf2\x00\x80\x80\"");
	84	try err("\"\xf0\x80\x00\x80\"");
	85	try err("\"\xf1\x80\xc0\x80\"");
	86	try err("\"\xf2\x80\x80\x00\"");
	87	try err("\"\xf3\x80\x80\xc0\"");
	88	try err("\"\xf4\x80\x80\xf5\"");
514	89	}
515	90
516		test "y_string_uescaped_newline" {
517		try ok(
518		\\["new\u000Aline"]
519		);
520		}
521
522		test "y_string_uEscape" {
523		try ok(
524		\\["\u0061\u30af\u30EA\u30b9"]
525		);
526		}
527
528		test "y_string_unescaped_char_delete" {
529		try ok("[\"\x7f\"]");
530		}
531
532		test "y_string_unicode_2" {
533		try ok(
534		\\["⍂㈴⍂"]
535		);
536		}
537
538		test "y_string_unicodeEscapedBackslash" {
539		try ok(
540		\\["\u005C"]
541		);
542		}
543
544		test "y_string_unicode_escaped_double_quote" {
545		try ok(
546		\\["\u0022"]
547		);
548		}
549
550		test "y_string_unicode" {
551		try ok(
552		\\["\uA66D"]
553		);
554		}
555
556		test "y_string_unicode_U+10FFFE_nonchar" {
557		try ok(
558		\\["\uDBFF\uDFFE"]
559		);
560		}
561
562		test "y_string_unicode_U+1FFFE_nonchar" {
563		try ok(
564		\\["\uD83F\uDFFE"]
565		);
566		}
567
568		test "y_string_unicode_U+200B_ZERO_WIDTH_SPACE" {
569		try ok(
570		\\["\u200B"]
571		);
572		}
573
574		test "y_string_unicode_U+2064_invisible_plus" {
575		try ok(
576		\\["\u2064"]
577		);
578		}
579
580		test "y_string_unicode_U+FDD0_nonchar" {
581		try ok(
582		\\["\uFDD0"]
583		);
584		}
585
586		test "y_string_unicode_U+FFFE_nonchar" {
587		try ok(
588		\\["\uFFFE"]
589		);
590		}
591
592		test "y_string_utf8" {
593		try ok(
594		\\["€𝄞"]
595		);
596		}
597
598		test "y_string_with_del_character" {
599		try ok("[\"a\x7fa\"]");
600		}
601
602		test "y_structure_lonely_false" {
603		try roundTrip(
604		\\false
605		);
606		}
607
608		test "y_structure_lonely_int" {
609		try roundTrip(
610		\\42
611		);
612		}
613
614		test "y_structure_lonely_negative_real" {
615		try ok(
616		\\-0.1
617		);
618		}
619
620		test "y_structure_lonely_null" {
621		try roundTrip(
622		\\null
623		);
624		}
625
626		test "y_structure_lonely_string" {
627		try roundTrip(
628		\\"asd"
629		);
630		}
631
632		test "y_structure_lonely_true" {
633		try roundTrip(
634		\\true
635		);
636		}
637
638		test "y_structure_string_empty" {
639		try roundTrip(
640		\\""
641		);
642		}
643
644		test "y_structure_trailing_newline" {
645		try roundTrip(
646		\\["a"]
647		);
648		}
649
650		test "y_structure_true_in_array" {
651		try roundTrip(
652		\\[true]
653		);
654		}
655
656		test "y_structure_whitespace_array" {
657		try ok(" [] ");
658		}
659
660		////////////////////////////////////////////////////////////////////////////////////////////////////
661
662		test "n_array_1_true_without_comma" {
663		try err(
664		\\[1 true]
665		);
666		}
667
668		test "n_array_a_invalid_utf8" {
669		try err(
670		\\[aå]
671		);
672		}
673
674		test "n_array_colon_instead_of_comma" {
675		try err(
676		\\["": 1]
677		);
678		}
679
680		test "n_array_comma_after_close" {
681		try err(
682		\\[""],
683		);
684		}
685
686		test "n_array_comma_and_number" {
687		try err(
688		\\[,1]
689		);
690		}
691
692		test "n_array_double_comma" {
693		try err(
694		\\[1,,2]
695		);
696		}
697
698		test "n_array_double_extra_comma" {
699		try err(
700		\\["x",,]
701		);
702		}
703
704		test "n_array_extra_close" {
705		try err(
706		\\["x"]]
707		);
708		}
709
710		test "n_array_extra_comma" {
711		try err(
712		\\["",]
713		);
714		}
715
716		test "n_array_incomplete_invalid_value" {
717		try err(
718		\\[x
719		);
720		}
721
722		test "n_array_incomplete" {
723		try err(
724		\\["x"
725		);
726		}
727
728		test "n_array_inner_array_no_comma" {
729		try err(
730		\\[3[4]]
731		);
732		}
733
734		test "n_array_invalid_utf8" {
735		try err(
736		\\[ÿ]
737		);
738		}
739
740		test "n_array_items_separated_by_semicolon" {
741		try err(
742		\\[1:2]
743		);
744		}
745
746		test "n_array_just_comma" {
747		try err(
748		\\[,]
749		);
750		}
751
752		test "n_array_just_minus" {
753		try err(
754		\\[-]
755		);
756		}
757
758		test "n_array_missing_value" {
759		try err(
760		\\[ , ""]
761		);
762		}
763
764		test "n_array_newlines_unclosed" {
765		try err(
766		\\["a",
767		\\4
768		\\,1,
769		);
770		}
771
772		test "n_array_number_and_comma" {
773		try err(
774		\\[1,]
775		);
776		}
777
778		test "n_array_number_and_several_commas" {
779		try err(
780		\\[1,,]
781		);
782		}
783
784		test "n_array_spaces_vertical_tab_formfeed" {
785		try err("[\"\x0aa\"\\f]");
786		}
787
788		test "n_array_star_inside" {
789		try err(
790		\\[*]
791		);
792		}
793
794		test "n_array_unclosed" {
795		try err(
796		\\[""
797		);
798		}
799
800		test "n_array_unclosed_trailing_comma" {
801		try err(
802		\\[1,
803		);
804		}
805
806		test "n_array_unclosed_with_new_lines" {
807		try err(
808		\\[1,
809		\\1
810		\\,1
811		);
812		}
813
814		test "n_array_unclosed_with_object_inside" {
815		try err(
816		\\[{}
817		);
818		}
819
820		test "n_incomplete_false" {
821		try err(
822		\\[fals]
823		);
824		}
825
826		test "n_incomplete_null" {
827		try err(
828		\\[nul]
829		);
830		}
831
832		test "n_incomplete_true" {
833		try err(
834		\\[tru]
835		);
836		}
837
838		test "n_multidigit_number_then_00" {
839		try err("123\x00");
840		}
841
842		test "n_number_0.1.2" {
843		try err(
844		\\[0.1.2]
845		);
846		}
847
848		test "n_number_-01" {
849		try err(
850		\\[-01]
851		);
852		}
853
854		test "n_number_0.3e" {
855		try err(
856		\\[0.3e]
857		);
858		}
859
860		test "n_number_0.3e+" {
861		try err(
862		\\[0.3e+]
863		);
864		}
865
866		test "n_number_0_capital_E" {
867		try err(
868		\\[0E]
869		);
870		}
871
872		test "n_number_0_capital_E+" {
873		try err(
874		\\[0E+]
875		);
876		}
877
878		test "n_number_0.e1" {
879		try err(
880		\\[0.e1]
881		);
882		}
883
884		test "n_number_0e" {
885		try err(
886		\\[0e]
887		);
888		}
889
890		test "n_number_0e+" {
891		try err(
892		\\[0e+]
893		);
894		}
895
896		test "n_number_1_000" {
897		try err(
898		\\[1 000.0]
899		);
900		}
901
902		test "n_number_1.0e-" {
903		try err(
904		\\[1.0e-]
905		);
906		}
907
908		test "n_number_1.0e" {
909		try err(
910		\\[1.0e]
911		);
912		}
913
914		test "n_number_1.0e+" {
915		try err(
916		\\[1.0e+]
917		);
918		}
919
920		test "n_number_-1.0." {
921		try err(
922		\\[-1.0.]
923		);
924		}
925
926		test "n_number_1eE2" {
927		try err(
928		\\[1eE2]
929		);
930		}
931
932		test "n_number_.-1" {
933		try err(
934		\\[.-1]
935		);
936		}
937
938		test "n_number_+1" {
939		try err(
940		\\[+1]
941		);
942		}
943
944		test "n_number_.2e-3" {
945		try err(
946		\\[.2e-3]
947		);
948		}
949
950		test "n_number_2.e-3" {
951		try err(
952		\\[2.e-3]
953		);
954		}
955
956		test "n_number_2.e+3" {
957		try err(
958		\\[2.e+3]
959		);
960		}
961
962		test "n_number_2.e3" {
963		try err(
964		\\[2.e3]
965		);
966		}
967
968		test "n_number_-2." {
969		try err(
970		\\[-2.]
971		);
972		}
973
974		test "n_number_9.e+" {
975		try err(
976		\\[9.e+]
977		);
978		}
979
980		test "n_number_expression" {
981		try err(
982		\\[1+2]
983		);
984		}
985
986		test "n_number_hex_1_digit" {
987		try err(
988		\\[0x1]
989		);
990		}
991
992		test "n_number_hex_2_digits" {
993		try err(
994		\\[0x42]
995		);
996		}
997
998		test "n_number_infinity" {
999		try err(
1000		\\[Infinity]
1001		);
1002		}
1003
1004		test "n_number_+Inf" {
1005		try err(
1006		\\[+Inf]
1007		);
1008		}
1009
1010		test "n_number_Inf" {
1011		try err(
1012		\\[Inf]
1013		);
1014		}
1015
1016		test "n_number_invalid+-" {
1017		try err(
1018		\\[0e+-1]
1019		);
1020		}
1021
1022		test "n_number_invalid-negative-real" {
1023		try err(
1024		\\[-123.123foo]
1025		);
1026		}
1027
1028		test "n_number_invalid-utf-8-in-bigger-int" {
1029		try err(
1030		\\[123å]
1031		);
1032		}
1033
1034		test "n_number_invalid-utf-8-in-exponent" {
1035		try err(
1036		\\[1e1å]
1037		);
1038		}
1039
1040		test "n_number_invalid-utf-8-in-int" {
1041		try err(
1042		\\[0å]
1043		);
1044		}
1045
1046		test "n_number_++" {
1047		try err(
1048		\\[++1234]
1049		);
1050		}
1051
1052		test "n_number_minus_infinity" {
1053		try err(
1054		\\[-Infinity]
1055		);
1056		}
1057
1058		test "n_number_minus_sign_with_trailing_garbage" {
1059		try err(
1060		\\[-foo]
1061		);
1062		}
1063
1064		test "n_number_minus_space_1" {
1065		try err(
1066		\\[- 1]
1067		);
1068		}
1069
1070		test "n_number_-NaN" {
1071		try err(
1072		\\[-NaN]
1073		);
1074		}
1075
1076		test "n_number_NaN" {
1077		try err(
1078		\\[NaN]
1079		);
1080		}
1081
1082		test "n_number_neg_int_starting_with_zero" {
1083		try err(
1084		\\[-012]
1085		);
1086		}
1087
1088		test "n_number_neg_real_without_int_part" {
1089		try err(
1090		\\[-.123]
1091		);
1092		}
1093
1094		test "n_number_neg_with_garbage_at_end" {
1095		try err(
1096		\\[-1x]
1097		);
1098		}
1099
1100		test "n_number_real_garbage_after_e" {
1101		try err(
1102		\\[1ea]
1103		);
1104		}
1105
1106		test "n_number_real_with_invalid_utf8_after_e" {
1107		try err(
1108		\\[1eå]
1109		);
1110		}
1111
1112		test "n_number_real_without_fractional_part" {
1113		try err(
1114		\\[1.]
1115		);
1116		}
1117
1118		test "n_number_starting_with_dot" {
1119		try err(
1120		\\[.123]
1121		);
1122		}
1123
1124		test "n_number_U+FF11_fullwidth_digit_one" {
1125		try err(
1126		\\[ï¼]
1127		);
1128		}
1129
1130		test "n_number_with_alpha_char" {
1131		try err(
1132		\\[1.8011670033376514H-308]
1133		);
1134		}
1135
1136		test "n_number_with_alpha" {
1137		try err(
1138		\\[1.2a-3]
1139		);
1140		}
1141
1142		test "n_number_with_leading_zero" {
1143		try err(
1144		\\[012]
1145		);
1146		}
1147
1148		test "n_object_bad_value" {
1149		try err(
1150		\\["x", truth]
1151		);
1152		}
1153
1154		test "n_object_bracket_key" {
1155		try err(
1156		\\{[: "x"}
1157		);
1158		}
1159
1160		test "n_object_comma_instead_of_colon" {
1161		try err(
1162		\\{"x", null}
1163		);
1164		}
1165
1166		test "n_object_double_colon" {
1167		try err(
1168		\\{"x"::"b"}
1169		);
1170		}
1171
1172		test "n_object_emoji" {
1173		try err(
1174		\\{ð¨ð}
1175		);
1176		}
1177
1178		test "n_object_garbage_at_end" {
1179		try err(
1180		\\{"a":"a" 123}
1181		);
1182		}
1183
1184		test "n_object_key_with_single_quotes" {
1185		try err(
1186		\\{key: 'value'}
1187		);
1188		}
1189
1190		test "n_object_lone_continuation_byte_in_key_and_trailing_comma" {
1191		try err(
1192		\\{"¹":"0",}
1193		);
1194		}
1195
1196		test "n_object_missing_colon" {
1197		try err(
1198		\\{"a" b}
1199		);
1200		}
1201
1202		test "n_object_missing_key" {
1203		try err(
1204		\\{:"b"}
1205		);
1206		}
1207
1208		test "n_object_missing_semicolon" {
1209		try err(
1210		\\{"a" "b"}
1211		);
1212		}
1213
1214		test "n_object_missing_value" {
1215		try err(
1216		\\{"a":
1217		);
1218		}
1219
1220		test "n_object_no-colon" {
1221		try err(
1222		\\{"a"
1223		);
1224		}
1225
1226		test "n_object_non_string_key_but_huge_number_instead" {
1227		try err(
1228		\\{9999E9999:1}
1229		);
1230		}
1231
1232		test "n_object_non_string_key" {
1233		try err(
1234		\\{1:1}
1235		);
1236		}
1237
1238		test "n_object_repeated_null_null" {
1239		try err(
1240		\\{null:null,null:null}
1241		);
1242		}
1243
1244		test "n_object_several_trailing_commas" {
1245		try err(
1246		\\{"id":0,,,,,}
1247		);
1248		}
1249
1250		test "n_object_single_quote" {
1251		try err(
1252		\\{'a':0}
1253		);
1254		}
1255
1256		test "n_object_trailing_comma" {
1257		try err(
1258		\\{"id":0,}
1259		);
1260		}
1261
1262		test "n_object_trailing_comment" {
1263		try err(
1264		\\{"a":"b"}/**/
1265		);
1266		}
1267
1268		test "n_object_trailing_comment_open" {
1269		try err(
1270		\\{"a":"b"}/**//
1271		);
1272		}
1273
1274		test "n_object_trailing_comment_slash_open_incomplete" {
1275		try err(
1276		\\{"a":"b"}/
1277		);
1278		}
1279
1280		test "n_object_trailing_comment_slash_open" {
1281		try err(
1282		\\{"a":"b"}//
1283		);
1284		}
1285
1286		test "n_object_two_commas_in_a_row" {
1287		try err(
1288		\\{"a":"b",,"c":"d"}
1289		);
1290		}
1291
1292		test "n_object_unquoted_key" {
1293		try err(
1294		\\{a: "b"}
1295		);
1296		}
1297
1298		test "n_object_unterminated-value" {
1299		try err(
1300		\\{"a":"a
1301		);
1302		}
1303
1304		test "n_object_with_single_string" {
1305		try err(
1306		\\{ "foo" : "bar", "a" }
1307		);
1308		}
1309
1310		test "n_object_with_trailing_garbage" {
1311		try err(
1312		\\{"a":"b"}#
1313		);
1314		}
1315
1316		test "n_single_space" {
1317		try err(" ");
1318		}
1319
1320		test "n_string_1_surrogate_then_escape" {
1321		try err(
1322		\\["\uD800\"]
1323		);
1324		}
1325
1326		test "n_string_1_surrogate_then_escape_u1" {
1327		try err(
1328		\\["\uD800\u1"]
1329		);
1330		}
1331
1332		test "n_string_1_surrogate_then_escape_u1x" {
1333		try err(
1334		\\["\uD800\u1x"]
1335		);
1336		}
1337
1338		test "n_string_1_surrogate_then_escape_u" {
1339		try err(
1340		\\["\uD800\u"]
1341		);
1342		}
1343
1344		test "n_string_accentuated_char_no_quotes" {
1345		try err(
1346		\\[Ã©]
1347		);
1348		}
1349
1350		test "n_string_backslash_00" {
1351		try err("[\"\x00\"]");
1352		}
1353
1354		test "n_string_escaped_backslash_bad" {
1355		try err(
1356		\\["\\\"]
1357		);
1358		}
1359
1360		test "n_string_escaped_ctrl_char_tab" {
1361		try err("\x5b\x22\x5c\x09\x22\x5d");
1362		}
1363
1364		test "n_string_escaped_emoji" {
1365		try err("[\"\x5c\xc3\xb0\xc2\x9f\xc2\x8c\xc2\x80\"]");
1366		}
1367
1368		test "n_string_escape_x" {
1369		try err(
1370		\\["\x00"]
1371		);
1372		}
1373
1374		test "n_string_incomplete_escaped_character" {
1375		try err(
1376		\\["\u00A"]
1377		);
1378		}
1379
1380		test "n_string_incomplete_escape" {
1381		try err(
1382		\\["\"]
1383		);
1384		}
1385
1386		test "n_string_incomplete_surrogate_escape_invalid" {
1387		try err(
1388		\\["\uD800\uD800\x"]
1389		);
1390		}
1391
1392		test "n_string_incomplete_surrogate" {
1393		try err(
1394		\\["\uD834\uDd"]
1395		);
1396		}
1397
1398		test "n_string_invalid_backslash_esc" {
1399		try err(
1400		\\["\a"]
1401		);
1402		}
1403
1404		test "n_string_invalid_unicode_escape" {
1405		try err(
1406		\\["\uqqqq"]
1407		);
1408		}
1409
1410		test "n_string_invalid_utf8_after_escape" {
1411		try err("[\"\\\x75\xc3\xa5\"]");
1412		}
1413
1414		test "n_string_invalid-utf-8-in-escape" {
1415		try err(
1416		\\["\uå"]
1417		);
1418		}
1419
1420		test "n_string_leading_uescaped_thinspace" {
1421		try err(
1422		\\[\u0020"asd"]
1423		);
1424		}
1425
1426		test "n_string_no_quotes_with_bad_escape" {
1427		try err(
1428		\\[\n]
1429		);
1430		}
1431
1432		test "n_string_single_doublequote" {
1433		try err(
1434		\\"
1435		);
1436		}
1437
1438		test "n_string_single_quote" {
1439		try err(
1440		\\['single quote']
1441		);
1442		}
1443
1444		test "n_string_single_string_no_double_quotes" {
1445		try err(
1446		\\abc
1447		);
1448		}
1449
1450		test "n_string_start_escape_unclosed" {
1451		try err(
1452		\\["\
1453		);
1454		}
1455
1456		test "n_string_unescaped_crtl_char" {
1457		try err("[\"a\x00a\"]");
1458		}
1459
1460		test "n_string_unescaped_newline" {
1461		try err(
1462		\\["new
1463		\\line"]
1464		);
1465		}
1466
1467		test "n_string_unescaped_tab" {
1468		try err("[\"\t\"]");
1469		}
1470
1471		test "n_string_unicode_CapitalU" {
1472		try err(
1473		\\"\UA66D"
1474		);
1475		}
1476
1477		test "n_string_with_trailing_garbage" {
1478		try err(
1479		\\""x
1480		);
1481		}
1482
1483		test "n_structure_100000_opening_arrays" {
1484		try err("[" ** 100000);
1485		}
1486
1487		test "n_structure_angle_bracket_." {
1488		try err(
1489		\\<.>
1490		);
1491		}
1492
1493		test "n_structure_angle_bracket_null" {
1494		try err(
1495		\\[<null>]
1496		);
1497		}
1498
1499		test "n_structure_array_trailing_garbage" {
1500		try err(
1501		\\[1]x
1502		);
1503		}
1504
1505		test "n_structure_array_with_extra_array_close" {
1506		try err(
1507		\\[1]]
1508		);
1509		}
1510
1511		test "n_structure_array_with_unclosed_string" {
1512		try err(
1513		\\["asd]
1514		);
1515		}
1516
1517		test "n_structure_ascii-unicode-identifier" {
1518		try err(
1519		\\aÃ¥
1520		);
1521		}
1522
1523		test "n_structure_capitalized_True" {
1524		try err(
1525		\\[True]
1526		);
1527		}
1528
1529		test "n_structure_close_unopened_array" {
1530		try err(
1531		\\1]
1532		);
1533		}
1534
1535		test "n_structure_comma_instead_of_closing_brace" {
1536		try err(
1537		\\{"x": true,
1538		);
1539		}
1540
1541		test "n_structure_double_array" {
1542		try err(
1543		\\[][]
1544		);
1545		}
1546
1547		test "n_structure_end_array" {
1548		try err(
1549		\\]
1550		);
1551		}
1552
1553		test "n_structure_incomplete_UTF8_BOM" {
1554		try err(
1555		\\ï»{}
1556		);
1557		}
1558
1559		test "n_structure_lone-invalid-utf-8" {
1560		try err(
1561		\\å
1562		);
1563		}
1564
1565		test "n_structure_lone-open-bracket" {
1566		try err(
1567		\\[
1568		);
1569		}
1570
1571		test "n_structure_no_data" {
1572		try err(
1573		\\
1574		);
1575		}
1576
1577		test "n_structure_null-byte-outside-string" {
1578		try err("[\x00]");
1579		}
1580
1581		test "n_structure_number_with_trailing_garbage" {
1582		try err(
1583		\\2@
1584		);
1585		}
1586
1587		test "n_structure_object_followed_by_closing_object" {
1588		try err(
1589		\\{}}
1590		);
1591		}
1592
1593		test "n_structure_object_unclosed_no_value" {
1594		try err(
1595		\\{"":
1596		);
1597		}
1598
1599		test "n_structure_object_with_comment" {
1600		try err(
1601		\\{"a":/comment/"b"}
1602		);
1603		}
1604
1605		test "n_structure_object_with_trailing_garbage" {
1606		try err(
1607		\\{"a": true} "x"
1608		);
1609		}
1610
1611		test "n_structure_open_array_apostrophe" {
1612		try err(
1613		\\['
1614		);
1615		}
1616
1617		test "n_structure_open_array_comma" {
1618		try err(
1619		\\[,
1620		);
1621		}
1622
1623		test "n_structure_open_array_object" {
1624		try err("[{\"\":" ** 50000);
1625		}
1626
1627		test "n_structure_open_array_open_object" {
1628		try err(
1629		\\[{
1630		);
1631		}
1632
1633		test "n_structure_open_array_open_string" {
1634		try err(
1635		\\["a
1636		);
1637		}
1638
1639		test "n_structure_open_array_string" {
1640		try err(
1641		\\["a"
1642		);
1643		}
1644
1645		test "n_structure_open_object_close_array" {
1646		try err(
1647		\\{]
1648		);
1649		}
1650
1651		test "n_structure_open_object_comma" {
1652		try err(
1653		\\{,
1654		);
1655		}
1656
1657		test "n_structure_open_object" {
1658		try err(
1659		\\{
1660		);
1661		}
1662
1663		test "n_structure_open_object_open_array" {
1664		try err(
1665		\\{[
1666		);
1667		}
1668
1669		test "n_structure_open_object_open_string" {
1670		try err(
1671		\\{"a
1672		);
1673		}
1674
1675		test "n_structure_open_object_string_with_apostrophes" {
1676		try err(
1677		\\{'a'
1678		);
1679		}
1680
1681		test "n_structure_open_open" {
1682		try err(
1683		\\["\{["\{["\{["\{
1684		);
1685		}
1686
1687		test "n_structure_single_eacute" {
1688		try err(
1689		\\é
1690		);
1691		}
1692
1693		test "n_structure_single_star" {
1694		try err(
1695		\\*
1696		);
1697		}
1698
1699		test "n_structure_trailing_#" {
1700		try err(
1701		\\{"a":"b"}#{}
1702		);
1703		}
1704
1705		test "n_structure_U+2060_word_joined" {
1706		try err(
1707		\\[â ]
1708		);
1709		}
1710
1711		test "n_structure_uescaped_LF_before_string" {
1712		try err(
1713		\\[\u000A""]
1714		);
1715		}
1716
1717		test "n_structure_unclosed_array" {
1718		try err(
1719		\\[1
1720		);
1721		}
1722
1723		test "n_structure_unclosed_array_partial_null" {
1724		try err(
1725		\\[ false, nul
1726		);
1727		}
1728
1729		test "n_structure_unclosed_array_unfinished_false" {
1730		try err(
1731		\\[ true, fals
1732		);
1733		}
1734
1735		test "n_structure_unclosed_array_unfinished_true" {
1736		try err(
1737		\\[ false, tru
1738		);
1739		}
1740
1741		test "n_structure_unclosed_object" {
1742		try err(
1743		\\{"asd":"asd"
1744		);
1745		}
1746
1747		test "n_structure_unicode-identifier" {
1748		try err(
1749		\\Ã¥
1750		);
1751		}
1752
1753		test "n_structure_UTF8_BOM_no_data" {
1754		try err(
1755		\\ï»¿
1756		);
1757		}
1758
1759		test "n_structure_whitespace_formfeed" {
1760		try err("[\x0c]");
1761		}
1762
1763		test "n_structure_whitespace_U+2060_word_joiner" {
1764		try err(
1765		\\[â ]
1766		);
1767		}
1768
1769		////////////////////////////////////////////////////////////////////////////////////////////////////
1770
1771		test "i_number_double_huge_neg_exp" {
1772		try any(
1773		\\[123.456e-789]
1774		);
1775		}
1776
1777		test "i_number_huge_exp" {
1778		try any(
1779		\\[0.4e00669999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999969999999006]
1780		);
1781		}
1782
1783		test "i_number_neg_int_huge_exp" {
1784		try any(
1785		\\[-1e+9999]
1786		);
1787		}
1788
1789		test "i_number_pos_double_huge_exp" {
1790		try any(
1791		\\[1.5e+9999]
1792		);
1793		}
1794
1795		test "i_number_real_neg_overflow" {
1796		try any(
1797		\\[-123123e100000]
1798		);
1799		}
1800
1801		test "i_number_real_pos_overflow" {
1802		try any(
1803		\\[123123e100000]
1804		);
1805		}
1806
1807		test "i_number_real_underflow" {
1808		try any(
1809		\\[123e-10000000]
1810		);
1811		}
1812
1813		test "i_number_too_big_neg_int" {
1814		try any(
1815		\\[-123123123123123123123123123123]
1816		);
1817		}
1818
1819		test "i_number_too_big_pos_int" {
1820		try any(
1821		\\[100000000000000000000]
1822		);
1823		}
1824
1825		test "i_number_very_big_negative_int" {
1826		try any(
1827		\\[-237462374673276894279832749832423479823246327846]
1828		);
1829		}
1830
1831		test "i_object_key_lone_2nd_surrogate" {
1832		try anyStreamingErrNonStreaming(
1833		\\{"\uDFAA":0}
1834		);
1835		}
1836
1837		test "i_string_1st_surrogate_but_2nd_missing" {
1838		try anyStreamingErrNonStreaming(
1839		\\["\uDADA"]
1840		);
1841		}
1842
1843		test "i_string_1st_valid_surrogate_2nd_invalid" {
1844		try anyStreamingErrNonStreaming(
1845		\\["\uD888\u1234"]
1846		);
1847		}
1848
1849		test "i_string_incomplete_surrogate_and_escape_valid" {
1850		try anyStreamingErrNonStreaming(
1851		\\["\uD800\n"]
1852		);
1853		}
1854
1855		test "i_string_incomplete_surrogate_pair" {
1856		try anyStreamingErrNonStreaming(
1857		\\["\uDd1ea"]
1858		);
1859		}
1860
1861		test "i_string_incomplete_surrogates_escape_valid" {
1862		try anyStreamingErrNonStreaming(
1863		\\["\uD800\uD800\n"]
1864		);
1865		}
1866
1867		test "i_string_invalid_lonely_surrogate" {
1868		try anyStreamingErrNonStreaming(
1869		\\["\ud800"]
1870		);
1871		}
1872
1873		test "i_string_invalid_surrogate" {
1874		try anyStreamingErrNonStreaming(
1875		\\["\ud800abc"]
1876		);
1877		}
1878
1879		test "i_string_invalid_utf-8" {
1880		try any(
1881		\\["ÿ"]
1882		);
1883		}
1884
1885		test "i_string_inverted_surrogates_U+1D11E" {
1886		try anyStreamingErrNonStreaming(
1887		\\["\uDd1e\uD834"]
1888		);
1889		}
1890
1891		test "i_string_iso_latin_1" {
1892		try any(
1893		\\["é"]
1894		);
1895		}
1896
1897		test "i_string_lone_second_surrogate" {
1898		try anyStreamingErrNonStreaming(
1899		\\["\uDFAA"]
1900		);
1901		}
1902
1903		test "i_string_lone_utf8_continuation_byte" {
1904		try any(
1905		\\[""]
1906		);
1907		}
1908
1909		test "i_string_not_in_unicode_range" {
1910		try any(
1911		\\["ô¿¿¿"]
1912		);
1913		}
1914
1915		test "i_string_overlong_sequence_2_bytes" {
1916		try any(
1917		\\["À¯"]
1918		);
1919		}
1920
1921		test "i_string_overlong_sequence_6_bytes" {
1922		try any(
1923		\\["ü¿¿¿¿"]
1924		);
1925		}
1926
1927		test "i_string_overlong_sequence_6_bytes_null" {
1928		try any(
1929		\\["ü"]
1930		);
1931		}
1932
1933		test "i_string_truncated-utf-8" {
1934		try any(
1935		\\["àÿ"]
1936		);
1937		}
1938
1939		test "i_string_utf16BE_no_BOM" {
1940		try any("\x00\x5b\x00\x22\x00\xc3\xa9\x00\x22\x00\x5d");
1941		}
1942
1943		test "i_string_utf16LE_no_BOM" {
1944		try any("\x5b\x00\x22\x00\xc3\xa9\x00\x22\x00\x5d\x00");
1945		}
1946
1947		test "i_string_UTF-16LE_with_BOM" {
1948		try any("\xc3\xbf\xc3\xbe\x5b\x00\x22\x00\xc3\xa9\x00\x22\x00\x5d\x00");
1949		}
1950
1951		test "i_string_UTF-8_invalid_sequence" {
1952		try any(
1953		\\["æ¥Ñú"]
1954		);
1955		}
1956
1957		test "i_string_UTF8_surrogate_U+D800" {
1958		try any(
1959		\\["í "]
1960		);
1961		}
1962
1963		test "i_structure_500_nested_arrays" {
1964		try any(("[" 500) ++ ("]" 500));
1965		}
1966
1967		test "i_structure_UTF-8_BOM_empty_object" {
1968		try any(
1969		\\ï»¿{}
1970		);
1971		}
1972
1973		test "truncated UTF-8 sequence" {
1974		try utf8Error("\"\xc2\"");
1975		try utf8Error("\"\xdf\"");
1976		try utf8Error("\"\xed\xa0\"");
1977		try utf8Error("\"\xf0\x80\"");
1978		try utf8Error("\"\xf0\x80\x80\"");
1979		}
1980
1981		test "invalid continuation byte" {
1982		try utf8Error("\"\xc2\x00\"");
1983		try utf8Error("\"\xc2\x7f\"");
1984		try utf8Error("\"\xc2\xc0\"");
1985		try utf8Error("\"\xc3\xc1\"");
1986		try utf8Error("\"\xc4\xf5\"");
1987		try utf8Error("\"\xc5\xff\"");
1988		try utf8Error("\"\xe4\x80\x00\"");
1989		try utf8Error("\"\xe5\x80\x10\"");
1990		try utf8Error("\"\xe6\x80\xc0\"");
1991		try utf8Error("\"\xe7\x80\xf5\"");
1992		try utf8Error("\"\xe8\x00\x80\"");
1993		try utf8Error("\"\xf2\x00\x80\x80\"");
1994		try utf8Error("\"\xf0\x80\x00\x80\"");
1995		try utf8Error("\"\xf1\x80\xc0\x80\"");
1996		try utf8Error("\"\xf2\x80\x80\x00\"");
1997		try utf8Error("\"\xf3\x80\x80\xc0\"");
1998		try utf8Error("\"\xf4\x80\x80\xf5\"");
1999		}
2000
2001		test "disallowed overlong form" {
2002		try utf8Error("\"\xc0\x80\"");
2003		try utf8Error("\"\xc0\x90\"");
2004		try utf8Error("\"\xc1\x80\"");
2005		try utf8Error("\"\xc1\x90\"");
2006		try utf8Error("\"\xe0\x80\x80\"");
2007		try utf8Error("\"\xf0\x80\x80\x80\"");
	91	test "disallowed overlong form" {
	92	try err("\"\xc0\x80\"");
	93	try err("\"\xc0\x90\"");
	94	try err("\"\xc1\x80\"");
	95	try err("\"\xc1\x90\"");
	96	try err("\"\xe0\x80\x80\"");
	97	try err("\"\xf0\x80\x80\x80\"");
2008	98	}
2009	99
2010	100	test "out of UTF-16 range" {
2011		try utf8Error("\"\xf4\x90\x80\x80\"");
2012		try utf8Error("\"\xf5\x80\x80\x80\"");
2013		try utf8Error("\"\xf6\x80\x80\x80\"");
2014		try utf8Error("\"\xf7\x80\x80\x80\"");
2015		try utf8Error("\"\xf8\x80\x80\x80\"");
2016		try utf8Error("\"\xf9\x80\x80\x80\"");
2017		try utf8Error("\"\xfa\x80\x80\x80\"");
2018		try utf8Error("\"\xfb\x80\x80\x80\"");
2019		try utf8Error("\"\xfc\x80\x80\x80\"");
2020		try utf8Error("\"\xfd\x80\x80\x80\"");
2021		try utf8Error("\"\xfe\x80\x80\x80\"");
2022		try utf8Error("\"\xff\x80\x80\x80\"");
2023		}
2024
2025		test "parse" {
2026		var ts = TokenStream.init("false");
2027		try testing.expectEqual(false, try parse(bool, &ts, ParseOptions{}));
2028		ts = TokenStream.init("true");
2029		try testing.expectEqual(true, try parse(bool, &ts, ParseOptions{}));
2030		ts = TokenStream.init("1");
2031		try testing.expectEqual(@as(u1, 1), try parse(u1, &ts, ParseOptions{}));
2032		ts = TokenStream.init("50");
2033		try testing.expectError(error.Overflow, parse(u1, &ts, ParseOptions{}));
2034		ts = TokenStream.init("42");
2035		try testing.expectEqual(@as(u64, 42), try parse(u64, &ts, ParseOptions{}));
2036		ts = TokenStream.init("42.0");
2037		try testing.expectEqual(@as(f64, 42), try parse(f64, &ts, ParseOptions{}));
2038		ts = TokenStream.init("null");
2039		try testing.expectEqual(@as(?bool, null), try parse(?bool, &ts, ParseOptions{}));
2040		ts = TokenStream.init("true");
2041		try testing.expectEqual(@as(?bool, true), try parse(?bool, &ts, ParseOptions{}));
2042
2043		ts = TokenStream.init("\"foo\"");
2044		try testing.expectEqual(@as([3]u8, "foo".*), try parse([3]u8, &ts, ParseOptions{}));
2045		ts = TokenStream.init("[102, 111, 111]");
2046		try testing.expectEqual(@as([3]u8, "foo".*), try parse([3]u8, &ts, ParseOptions{}));
2047		ts = TokenStream.init("[]");
2048		try testing.expectEqual(@as([0]u8, undefined), try parse([0]u8, &ts, ParseOptions{}));
2049
2050		ts = TokenStream.init("\"12345678901234567890\"");
2051		try testing.expectEqual(@as(u64, 12345678901234567890), try parse(u64, &ts, ParseOptions{}));
2052		ts = TokenStream.init("\"123.456\"");
2053		try testing.expectEqual(@as(f64, 123.456), try parse(f64, &ts, ParseOptions{}));
2054		}
2055
2056		test "parse into enum" {
2057		const T = enum(u32) {
2058		Foo = 42,
2059		Bar,
2060		@"with\\escape",
2061		};
2062		var ts = TokenStream.init("\"Foo\"");
2063		try testing.expectEqual(@as(T, .Foo), try parse(T, &ts, ParseOptions{}));
2064		ts = TokenStream.init("42");
2065		try testing.expectEqual(@as(T, .Foo), try parse(T, &ts, ParseOptions{}));
2066		ts = TokenStream.init("\"with\\\\escape\"");
2067		try testing.expectEqual(@as(T, .@"with\\escape"), try parse(T, &ts, ParseOptions{}));
2068		ts = TokenStream.init("5");
2069		try testing.expectError(error.InvalidEnumTag, parse(T, &ts, ParseOptions{}));
2070		ts = TokenStream.init("\"Qux\"");
2071		try testing.expectError(error.InvalidEnumTag, parse(T, &ts, ParseOptions{}));
2072		}
2073
2074		test "parse with trailing data" {
2075		var ts = TokenStream.init("falsed");
2076		try testing.expectEqual(false, try parse(bool, &ts, ParseOptions{ .allow_trailing_data = true }));
2077		ts = TokenStream.init("falsed");
2078		try testing.expectError(error.InvalidTopLevelTrailing, parse(bool, &ts, ParseOptions{ .allow_trailing_data = false }));
2079		// trailing whitespace is okay
2080		ts = TokenStream.init("false \n");
2081		try testing.expectEqual(false, try parse(bool, &ts, ParseOptions{ .allow_trailing_data = false }));
2082		}
2083
2084		test "parse into that allocates a slice" {
2085		var ts = TokenStream.init("\"foo\"");
2086		try testing.expectError(error.AllocatorRequired, parse([]u8, &ts, ParseOptions{}));
2087
2088		const options = ParseOptions{ .allocator = testing.allocator };
2089		{
2090		ts = TokenStream.init("\"foo\"");
2091		const r = try parse([]u8, &ts, options);
2092		defer parseFree([]u8, r, options);
2093		try testing.expectEqualSlices(u8, "foo", r);
2094		}
2095		{
2096		ts = TokenStream.init("[102, 111, 111]");
2097		const r = try parse([]u8, &ts, options);
2098		defer parseFree([]u8, r, options);
2099		try testing.expectEqualSlices(u8, "foo", r);
2100		}
2101		{
2102		ts = TokenStream.init("\"with\\\\escape\"");
2103		const r = try parse([]u8, &ts, options);
2104		defer parseFree([]u8, r, options);
2105		try testing.expectEqualSlices(u8, "with\\escape", r);
2106		}
2107		}
2108
2109		test "parse into tagged union" {
2110		{
2111		const T = union(enum) {
2112		int: i32,
2113		float: f64,
2114		string: []const u8,
2115		};
2116		var ts = TokenStream.init("1.5");
2117		try testing.expectEqual(T{ .float = 1.5 }, try parse(T, &ts, ParseOptions{}));
2118		}
2119
2120		{ // failing allocations should be bubbled up instantly without trying next member
2121		var fail_alloc = testing.FailingAllocator.init(testing.allocator, 0);
2122		const options = ParseOptions{ .allocator = fail_alloc.allocator() };
2123		const T = union(enum) {
2124		// both fields here match the input
2125		string: []const u8,
2126		array: [3]u8,
2127		};
2128		var ts = TokenStream.init("[1,2,3]");
2129		try testing.expectError(error.OutOfMemory, parse(T, &ts, options));
2130		}
2131
2132		{
2133		// if multiple matches possible, takes first option
2134		const T = union(enum) {
2135		x: u8,
2136		y: u8,
2137		};
2138		var ts = TokenStream.init("42");
2139		try testing.expectEqual(T{ .x = 42 }, try parse(T, &ts, ParseOptions{}));
2140		}
2141
2142		{ // needs to back out when first union member doesn't match
2143		const T = union(enum) {
2144		A: struct { x: u32 },
2145		B: struct { y: u32 },
2146		};
2147		var ts = TokenStream.init("{\"y\":42}");
2148		try testing.expectEqual(T{ .B = .{ .y = 42 } }, try parse(T, &ts, ParseOptions{}));
2149		}
2150		}
2151
2152		test "parse union bubbles up AllocatorRequired" {
2153		{ // string member first in union (and not matching)
2154		const T = union(enum) {
2155		string: []const u8,
2156		int: i32,
2157		};
2158		var ts = TokenStream.init("42");
2159		try testing.expectError(error.AllocatorRequired, parse(T, &ts, ParseOptions{}));
2160		}
2161
2162		{ // string member not first in union (and matching)
2163		const T = union(enum) {
2164		int: i32,
2165		float: f64,
2166		string: []const u8,
2167		};
2168		var ts = TokenStream.init("\"foo\"");
2169		try testing.expectError(error.AllocatorRequired, parse(T, &ts, ParseOptions{}));
2170		}
2171		}
2172
2173		test "parseFree descends into tagged union" {
2174		var fail_alloc = testing.FailingAllocator.init(testing.allocator, 1);
2175		const options = ParseOptions{ .allocator = fail_alloc.allocator() };
2176		const T = union(enum) {
2177		int: i32,
2178		float: f64,
2179		string: []const u8,
2180		};
2181		// use a string with unicode escape so we know result can't be a reference to global constant
2182		var ts = TokenStream.init("\"with\\u0105unicode\"");
2183		const r = try parse(T, &ts, options);
2184		try testing.expectEqual(std.meta.Tag(T).string, @as(std.meta.Tag(T), r));
2185		try testing.expectEqualSlices(u8, "withąunicode", r.string);
2186		try testing.expectEqual(@as(usize, 0), fail_alloc.deallocations);
2187		parseFree(T, r, options);
2188		try testing.expectEqual(@as(usize, 1), fail_alloc.deallocations);
2189		}
2190
2191		test "parse with comptime field" {
2192		{
2193		const T = struct {
2194		comptime a: i32 = 0,
2195		b: bool,
2196		};
2197		var ts = TokenStream.init(
2198		\\{
2199		\\ "a": 0,
2200		\\ "b": true
2201		\\}
2202		);
2203		try testing.expectEqual(T{ .a = 0, .b = true }, try parse(T, &ts, ParseOptions{}));
2204		}
2205
2206		{ // string comptime values currently require an allocator
2207		const T = union(enum) {
2208		foo: struct {
2209		comptime kind: []const u8 = "boolean",
2210		b: bool,
2211		},
2212		bar: struct {
2213		comptime kind: []const u8 = "float",
2214		b: f64,
2215		},
2216		};
2217
2218		const options = ParseOptions{
2219		.allocator = std.testing.allocator,
2220		};
2221
2222		var ts = TokenStream.init(
2223		\\{
2224		\\ "kind": "float",
2225		\\ "b": 1.0
2226		\\}
2227		);
2228		const r = try parse(T, &ts, options);
2229
2230		// check that parseFree doesn't try to free comptime fields
2231		parseFree(T, r, options);
2232		}
2233		}
2234
2235		test "parse into struct with no fields" {
2236		const T = struct {};
2237		var ts = TokenStream.init("{}");
2238		try testing.expectEqual(T{}, try parse(T, &ts, ParseOptions{}));
2239		}
2240
2241		const test_const_value: usize = 123;
2242
2243		test "parse into struct with default const pointer field" {
2244		const T = struct { a: *const usize = &test_const_value };
2245		var ts = TokenStream.init("{}");
2246		try testing.expectEqual(T{}, try parse(T, &ts, .{}));
2247		}
2248
2249		const test_default_usize: usize = 123;
2250		const test_default_usize_ptr: *align(1) const usize = &test_default_usize;
2251		const test_default_str: []const u8 = "test str";
2252		const test_default_str_slice: [2][]const u8 = [_][]const u8{
2253		"test1",
2254		"test2",
2255		};
2256
2257		test "freeing parsed structs with pointers to default values" {
2258		const T = struct {
2259		int: *const usize = &test_default_usize,
2260		int_ptr: *allowzero align(1) const usize = test_default_usize_ptr,
2261		str: []const u8 = test_default_str,
2262		str_slice: []const []const u8 = &test_default_str_slice,
2263		};
2264
2265		var ts = json.TokenStream.init("{}");
2266		const options = .{ .allocator = std.heap.page_allocator };
2267		const parsed = try json.parse(T, &ts, options);
2268
2269		try testing.expectEqual(T{}, parsed);
2270
2271		json.parseFree(T, parsed, options);
2272		}
2273
2274		test "parse into struct where destination and source lengths mismatch" {
2275		const T = struct { a: [2]u8 };
2276		var ts = TokenStream.init("{\"a\": \"bbb\"}");
2277		try testing.expectError(error.LengthMismatch, parse(T, &ts, ParseOptions{}));
2278		}
2279
2280		test "parse into struct with misc fields" {
2281		@setEvalBranchQuota(10000);
2282		const options = ParseOptions{ .allocator = testing.allocator };
2283		const T = struct {
2284		int: i64,
2285		float: f64,
2286		@"with\\escape": bool,
2287		@"withąunicode😂": bool,
2288		language: []const u8,
2289		optional: ?bool,
2290		default_field: i32 = 42,
2291		static_array: [3]f64,
2292		dynamic_array: []f64,
2293
2294		complex: struct {
2295		nested: []const u8,
2296		},
2297
2298		veryComplex: []struct {
2299		foo: []const u8,
2300		},
2301
2302		a_union: Union,
2303		const Union = union(enum) {
2304		x: u8,
2305		float: f64,
2306		string: []const u8,
2307		};
2308		};
2309		var ts = TokenStream.init(
2310		\\{
2311		\\ "int": 420,
2312		\\ "float": 3.14,
2313		\\ "with\\escape": true,
2314		\\ "with\u0105unicode\ud83d\ude02": false,
2315		\\ "language": "zig",
2316		\\ "optional": null,
2317		\\ "static_array": [66.6, 420.420, 69.69],
2318		\\ "dynamic_array": [66.6, 420.420, 69.69],
2319		\\ "complex": {
2320		\\ "nested": "zig"
2321		\\ },
2322		\\ "veryComplex": [
2323		\\ {
2324		\\ "foo": "zig"
2325		\\ }, {
2326		\\ "foo": "rocks"
2327		\\ }
2328		\\ ],
2329		\\ "a_union": 100000
2330		\\}
2331		);
2332		const r = try parse(T, &ts, options);
2333		defer parseFree(T, r, options);
2334		try testing.expectEqual(@as(i64, 420), r.int);
2335		try testing.expectEqual(@as(f64, 3.14), r.float);
2336		try testing.expectEqual(true, r.@"with\\escape");
2337		try testing.expectEqual(false, r.@"withąunicode😂");
2338		try testing.expectEqualSlices(u8, "zig", r.language);
2339		try testing.expectEqual(@as(?bool, null), r.optional);
2340		try testing.expectEqual(@as(i32, 42), r.default_field);
2341		try testing.expectEqual(@as(f64, 66.6), r.static_array[0]);
2342		try testing.expectEqual(@as(f64, 420.420), r.static_array[1]);
2343		try testing.expectEqual(@as(f64, 69.69), r.static_array[2]);
2344		try testing.expectEqual(@as(usize, 3), r.dynamic_array.len);
2345		try testing.expectEqual(@as(f64, 66.6), r.dynamic_array[0]);
2346		try testing.expectEqual(@as(f64, 420.420), r.dynamic_array[1]);
2347		try testing.expectEqual(@as(f64, 69.69), r.dynamic_array[2]);
2348		try testing.expectEqualSlices(u8, r.complex.nested, "zig");
2349		try testing.expectEqualSlices(u8, "zig", r.veryComplex[0].foo);
2350		try testing.expectEqualSlices(u8, "rocks", r.veryComplex[1].foo);
2351		try testing.expectEqual(T.Union{ .float = 100000 }, r.a_union);
2352		}
2353
2354		test "parse into struct with strings and arrays with sentinels" {
2355		@setEvalBranchQuota(10000);
2356		const options = ParseOptions{ .allocator = testing.allocator };
2357		const T = struct {
2358		language: [:0]const u8,
2359		language_without_sentinel: []const u8,
2360		data: [:99]const i32,
2361		simple_data: []const i32,
2362		};
2363		var ts = TokenStream.init(
2364		\\{
2365		\\ "language": "zig",
2366		\\ "language_without_sentinel": "zig again!",
2367		\\ "data": [1, 2, 3],
2368		\\ "simple_data": [4, 5, 6]
2369		\\}
2370		);
2371		const r = try parse(T, &ts, options);
2372		defer parseFree(T, r, options);
2373
2374		try testing.expectEqualSentinel(u8, 0, "zig", r.language);
2375
2376		const data = [_:99]i32{ 1, 2, 3 };
2377		try testing.expectEqualSentinel(i32, 99, data[0..data.len], r.data);
2378
2379		// Make sure that arrays who aren't supposed to have a sentinel still parse without one.
2380		try testing.expectEqual(@as(?i32, null), std.meta.sentinel(@TypeOf(r.simple_data)));
2381		try testing.expectEqual(@as(?u8, null), std.meta.sentinel(@TypeOf(r.language_without_sentinel)));
2382		}
2383
2384		test "parse into struct with duplicate field" {
2385		// allow allocator to detect double frees by keeping bucket in use
2386		const ballast = try testing.allocator.alloc(u64, 1);
2387		defer testing.allocator.free(ballast);
2388
2389		const options_first = ParseOptions{ .allocator = testing.allocator, .duplicate_field_behavior = .UseFirst };
2390
2391		const options_last = ParseOptions{
2392		.allocator = testing.allocator,
2393		.duplicate_field_behavior = .UseLast,
2394		};
2395
2396		const str = "{ \"a\": 1, \"a\": 0.25 }";
2397
2398		const T1 = struct { a: *u64 };
2399		// both .UseFirst and .UseLast should fail because second "a" value isn't a u64
2400		var ts = TokenStream.init(str);
2401		try testing.expectError(error.InvalidNumber, parse(T1, &ts, options_first));
2402		ts = TokenStream.init(str);
2403		try testing.expectError(error.InvalidNumber, parse(T1, &ts, options_last));
2404
2405		const T2 = struct { a: f64 };
2406		ts = TokenStream.init(str);
2407		try testing.expectEqual(T2{ .a = 1.0 }, try parse(T2, &ts, options_first));
2408		ts = TokenStream.init(str);
2409		try testing.expectEqual(T2{ .a = 0.25 }, try parse(T2, &ts, options_last));
2410
2411		const T3 = struct { comptime a: f64 = 1.0 };
2412		// .UseFirst should succeed because second "a" value is unconditionally ignored (even though != 1.0)
2413		const t3 = T3{ .a = 1.0 };
2414		ts = TokenStream.init(str);
2415		try testing.expectEqual(t3, try parse(T3, &ts, options_first));
2416		// .UseLast should fail because second "a" value is 0.25 which is not equal to default value of 1.0
2417		ts = TokenStream.init(str);
2418		try testing.expectError(error.UnexpectedValue, parse(T3, &ts, options_last));
2419		}
2420
2421		test "parse into struct ignoring unknown fields" {
2422		const T = struct {
2423		int: i64,
2424		language: []const u8,
2425		};
2426
2427		const ops = ParseOptions{
2428		.allocator = testing.allocator,
2429		.ignore_unknown_fields = true,
2430		};
2431
2432		var ts = TokenStream.init(
2433		\\{
2434		\\ "int": 420,
2435		\\ "float": 3.14,
2436		\\ "with\\escape": true,
2437		\\ "with\u0105unicode\ud83d\ude02": false,
2438		\\ "optional": null,
2439		\\ "static_array": [66.6, 420.420, 69.69],
2440		\\ "dynamic_array": [66.6, 420.420, 69.69],
2441		\\ "complex": {
2442		\\ "nested": "zig"
2443		\\ },
2444		\\ "veryComplex": [
2445		\\ {
2446		\\ "foo": "zig"
2447		\\ }, {
2448		\\ "foo": "rocks"
2449		\\ }
2450		\\ ],
2451		\\ "a_union": 100000,
2452		\\ "language": "zig"
2453		\\}
2454		);
2455		const r = try parse(T, &ts, ops);
2456		defer parseFree(T, r, ops);
2457
2458		try testing.expectEqual(@as(i64, 420), r.int);
2459		try testing.expectEqualSlices(u8, "zig", r.language);
2460		}
2461
2462		test "parse into tuple" {
2463		const options = ParseOptions{ .allocator = testing.allocator };
2464		const Union = union(enum) {
2465		char: u8,
2466		float: f64,
2467		string: []const u8,
2468		};
2469		const T = std.meta.Tuple(&.{
2470		i64,
2471		f64,
2472		bool,
2473		[]const u8,
2474		?bool,
2475		struct {
2476		foo: i32,
2477		bar: []const u8,
2478		},
2479		std.meta.Tuple(&.{ u8, []const u8, u8 }),
2480		Union,
2481		});
2482		var ts = TokenStream.init(
2483		\\[
2484		\\ 420,
2485		\\ 3.14,
2486		\\ true,
2487		\\ "zig",
2488		\\ null,
2489		\\ {
2490		\\ "foo": 1,
2491		\\ "bar": "zero"
2492		\\ },
2493		\\ [4, "två", 42],
2494		\\ 12.34
2495		\\]
2496		);
2497		const r = try parse(T, &ts, options);
2498		defer parseFree(T, r, options);
2499		try testing.expectEqual(@as(i64, 420), r[0]);
2500		try testing.expectEqual(@as(f64, 3.14), r[1]);
2501		try testing.expectEqual(true, r[2]);
2502		try testing.expectEqualSlices(u8, "zig", r[3]);
2503		try testing.expectEqual(@as(?bool, null), r[4]);
2504		try testing.expectEqual(@as(i32, 1), r[5].foo);
2505		try testing.expectEqualSlices(u8, "zero", r[5].bar);
2506		try testing.expectEqual(@as(u8, 4), r[6][0]);
2507		try testing.expectEqualSlices(u8, "två", r[6][1]);
2508		try testing.expectEqual(@as(u8, 42), r[6][2]);
2509		try testing.expectEqual(Union{ .float = 12.34 }, r[7]);
2510		}
2511
2512		const ParseIntoRecursiveUnionDefinitionValue = union(enum) {
2513		integer: i64,
2514		array: []const ParseIntoRecursiveUnionDefinitionValue,
2515		};
2516
2517		test "parse into recursive union definition" {
2518		const T = struct {
2519		values: ParseIntoRecursiveUnionDefinitionValue,
2520		};
2521		const ops = ParseOptions{ .allocator = testing.allocator };
2522
2523		var ts = TokenStream.init("{\"values\":[58]}");
2524		const r = try parse(T, &ts, ops);
2525		defer parseFree(T, r, ops);
2526
2527		try testing.expectEqual(@as(i64, 58), r.values.array[0].integer);
2528		}
2529
2530		const ParseIntoDoubleRecursiveUnionValueFirst = union(enum) {
2531		integer: i64,
2532		array: []const ParseIntoDoubleRecursiveUnionValueSecond,
2533		};
2534
2535		const ParseIntoDoubleRecursiveUnionValueSecond = union(enum) {
2536		boolean: bool,
2537		array: []const ParseIntoDoubleRecursiveUnionValueFirst,
2538		};
2539
2540		test "parse into double recursive union definition" {
2541		const T = struct {
2542		values: ParseIntoDoubleRecursiveUnionValueFirst,
2543		};
2544		const ops = ParseOptions{ .allocator = testing.allocator };
2545
2546		var ts = TokenStream.init("{\"values\":[[58]]}");
2547		const r = try parse(T, &ts, ops);
2548		defer parseFree(T, r, ops);
2549
2550		try testing.expectEqual(@as(i64, 58), r.values.array[0].array[0].integer);
2551		}
2552
2553		test "parse into vector" {
2554		const options = ParseOptions{ .allocator = testing.allocator };
2555		const T = struct {
2556		vec_i32: @Vector(4, i32),
2557		vec_f32: @Vector(2, f32),
2558		};
2559		var ts = TokenStream.init(
2560		\\{
2561		\\ "vec_f32": [1.5, 2.5],
2562		\\ "vec_i32": [4, 5, 6, 7]
2563		\\}
2564		);
2565		const r = try parse(T, &ts, options);
2566		defer parseFree(T, r, options);
2567		try testing.expectApproxEqAbs(@as(f32, 1.5), r.vec_f32[0], 0.0000001);
2568		try testing.expectApproxEqAbs(@as(f32, 2.5), r.vec_f32[1], 0.0000001);
2569		try testing.expectEqual(@Vector(4, i32){ 4, 5, 6, 7 }, r.vec_i32);
2570		}
2571
2572		test "json.parser.dynamic" {
2573		var p = Parser.init(testing.allocator, false);
2574		defer p.deinit();
2575
2576		const s =
2577		\\{
2578		\\ "Image": {
2579		\\ "Width": 800,
2580		\\ "Height": 600,
2581		\\ "Title": "View from 15th Floor",
2582		\\ "Thumbnail": {
2583		\\ "Url": "http://www.example.com/image/481989943",
2584		\\ "Height": 125,
2585		\\ "Width": 100
2586		\\ },
2587		\\ "Animated" : false,
2588		\\ "IDs": [116, 943, 234, 38793],
2589		\\ "ArrayOfObject": [{"n": "m"}],
2590		\\ "double": 1.3412,
2591		\\ "LargeInt": 18446744073709551615
2592		\\ }
2593		\\}
2594		;
2595
2596		var tree = try p.parse(s);
2597		defer tree.deinit();
2598
2599		var root = tree.root;
2600
2601		var image = root.Object.get("Image").?;
2602
2603		const width = image.Object.get("Width").?;
2604		try testing.expect(width.Integer == 800);
2605
2606		const height = image.Object.get("Height").?;
2607		try testing.expect(height.Integer == 600);
2608
2609		const title = image.Object.get("Title").?;
2610		try testing.expect(mem.eql(u8, title.String, "View from 15th Floor"));
2611
2612		const animated = image.Object.get("Animated").?;
2613		try testing.expect(animated.Bool == false);
2614
2615		const array_of_object = image.Object.get("ArrayOfObject").?;
2616		try testing.expect(array_of_object.Array.items.len == 1);
2617
2618		const obj0 = array_of_object.Array.items[0].Object.get("n").?;
2619		try testing.expect(mem.eql(u8, obj0.String, "m"));
2620
2621		const double = image.Object.get("double").?;
2622		try testing.expect(double.Float == 1.3412);
2623
2624		const large_int = image.Object.get("LargeInt").?;
2625		try testing.expect(mem.eql(u8, large_int.NumberString, "18446744073709551615"));
2626		}
2627
2628		test "write json then parse it" {
2629		var out_buffer: [1000]u8 = undefined;
2630
2631		var fixed_buffer_stream = std.io.fixedBufferStream(&out_buffer);
2632		const out_stream = fixed_buffer_stream.writer();
2633		var jw = writeStream(out_stream, 4);
2634
2635		try jw.beginObject();
2636
2637		try jw.objectField("f");
2638		try jw.emitBool(false);
2639
2640		try jw.objectField("t");
2641		try jw.emitBool(true);
2642
2643		try jw.objectField("int");
2644		try jw.emitNumber(1234);
2645
2646		try jw.objectField("array");
2647		try jw.beginArray();
2648
2649		try jw.arrayElem();
2650		try jw.emitNull();
2651
2652		try jw.arrayElem();
2653		try jw.emitNumber(12.34);
2654
2655		try jw.endArray();
2656
2657		try jw.objectField("str");
2658		try jw.emitString("hello");
2659
2660		try jw.endObject();
2661
2662		var parser = Parser.init(testing.allocator, false);
2663		defer parser.deinit();
2664		var tree = try parser.parse(fixed_buffer_stream.getWritten());
2665		defer tree.deinit();
2666
2667		try testing.expect(tree.root.Object.get("f").?.Bool == false);
2668		try testing.expect(tree.root.Object.get("t").?.Bool == true);
2669		try testing.expect(tree.root.Object.get("int").?.Integer == 1234);
2670		try testing.expect(tree.root.Object.get("array").?.Array.items[0].Null == {});
2671		try testing.expect(tree.root.Object.get("array").?.Array.items[1].Float == 12.34);
2672		try testing.expect(mem.eql(u8, tree.root.Object.get("str").?.String, "hello"));
2673		}
2674
2675		fn testParse(arena_allocator: std.mem.Allocator, json_str: []const u8) !Value {
2676		var p = Parser.init(arena_allocator, false);
2677		return (try p.parse(json_str)).root;
2678		}
2679
2680		test "parsing empty string gives appropriate error" {
2681		var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
2682		defer arena_allocator.deinit();
2683		try testing.expectError(error.UnexpectedEndOfJson, testParse(arena_allocator.allocator(), ""));
2684		}
2685
2686		test "parse tree should not contain dangling pointers" {
2687		var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
2688		defer arena_allocator.deinit();
2689
2690		var p = json.Parser.init(arena_allocator.allocator(), false);
2691		defer p.deinit();
2692
2693		var tree = try p.parse("[]");
2694		defer tree.deinit();
2695
2696		// Allocation should succeed
2697		var i: usize = 0;
2698		while (i < 100) : (i += 1) {
2699		try tree.root.Array.append(std.json.Value{ .Integer = 100 });
2700		}
2701		try testing.expectEqual(tree.root.Array.items.len, 100);
2702		}
2703
2704		test "integer after float has proper type" {
2705		var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
2706		defer arena_allocator.deinit();
2707		const parsed = try testParse(arena_allocator.allocator(),
2708		\\{
2709		\\ "float": 3.14,
2710		\\ "ints": [1, 2, 3]
2711		\\}
2712		);
2713		try std.testing.expect(parsed.Object.get("ints").?.Array.items[0] == .Integer);
2714		}
2715
2716		test "parse exponential into int" {
2717		const T = struct { int: i64 };
2718		var ts = TokenStream.init("{ \"int\": 4.2e2 }");
2719		const r = try parse(T, &ts, ParseOptions{});
2720		try testing.expectEqual(@as(i64, 420), r.int);
2721		ts = TokenStream.init("{ \"int\": 0.042e2 }");
2722		try testing.expectError(error.InvalidNumber, parse(T, &ts, ParseOptions{}));
2723		ts = TokenStream.init("{ \"int\": 18446744073709551616.0 }");
2724		try testing.expectError(error.Overflow, parse(T, &ts, ParseOptions{}));
2725		}
2726
2727		test "escaped characters" {
2728		var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
2729		defer arena_allocator.deinit();
2730		const input =
2731		\\{
2732		\\ "backslash": "\\",
2733		\\ "forwardslash": "\/",
2734		\\ "newline": "\n",
2735		\\ "carriagereturn": "\r",
2736		\\ "tab": "\t",
2737		\\ "formfeed": "\f",
2738		\\ "backspace": "\b",
2739		\\ "doublequote": "\"",
2740		\\ "unicode": "\u0105",
2741		\\ "surrogatepair": "\ud83d\ude02"
2742		\\}
2743		;
2744
2745		const obj = (try testParse(arena_allocator.allocator(), input)).Object;
2746
2747		try testing.expectEqualSlices(u8, obj.get("backslash").?.String, "\\");
2748		try testing.expectEqualSlices(u8, obj.get("forwardslash").?.String, "/");
2749		try testing.expectEqualSlices(u8, obj.get("newline").?.String, "\n");
2750		try testing.expectEqualSlices(u8, obj.get("carriagereturn").?.String, "\r");
2751		try testing.expectEqualSlices(u8, obj.get("tab").?.String, "\t");
2752		try testing.expectEqualSlices(u8, obj.get("formfeed").?.String, "\x0C");
2753		try testing.expectEqualSlices(u8, obj.get("backspace").?.String, "\x08");
2754		try testing.expectEqualSlices(u8, obj.get("doublequote").?.String, "\"");
2755		try testing.expectEqualSlices(u8, obj.get("unicode").?.String, "ą");
2756		try testing.expectEqualSlices(u8, obj.get("surrogatepair").?.String, "😂");
2757		}
2758
2759		test "string copy option" {
2760		const input =
2761		\\{
2762		\\ "noescape": "aą😂",
2763		\\ "simple": "\\\/\n\r\t\f\b\"",
2764		\\ "unicode": "\u0105",
2765		\\ "surrogatepair": "\ud83d\ude02"
2766		\\}
2767		;
2768
2769		var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
2770		defer arena_allocator.deinit();
2771		const allocator = arena_allocator.allocator();
2772
2773		var parser = Parser.init(allocator, false);
2774		const tree_nocopy = try parser.parse(input);
2775		const obj_nocopy = tree_nocopy.root.Object;
2776
2777		parser = Parser.init(allocator, true);
2778		const tree_copy = try parser.parse(input);
2779		const obj_copy = tree_copy.root.Object;
2780
2781		for ([_][]const u8{ "noescape", "simple", "unicode", "surrogatepair" }) \|field_name\| {
2782		try testing.expectEqualSlices(u8, obj_nocopy.get(field_name).?.String, obj_copy.get(field_name).?.String);
2783		}
2784
2785		const nocopy_addr = &obj_nocopy.get("noescape").?.String[0];
2786		const copy_addr = &obj_copy.get("noescape").?.String[0];
2787
2788		var found_nocopy = false;
2789		for (input, 0..) \|_, index\| {
2790		try testing.expect(copy_addr != &input[index]);
2791		if (nocopy_addr == &input[index]) {
2792		found_nocopy = true;
2793		}
2794		}
2795		try testing.expect(found_nocopy);
2796		}
2797
2798		test "stringify alloc" {
2799		const allocator = std.testing.allocator;
2800		const expected =
2801		\\{"foo":"bar","answer":42,"my_friend":"sammy"}
2802		;
2803		const actual = try stringifyAlloc(allocator, .{ .foo = "bar", .answer = 42, .my_friend = "sammy" }, .{});
2804		defer allocator.free(actual);
2805
2806		try std.testing.expectEqualStrings(expected, actual);
2807		}
2808
2809		test "json.serialize issue #5959" {
2810		var parser: StreamingParser = undefined;
2811		// StreamingParser has multiple internal fields set to undefined. This causes issues when using
2812		// expectEqual so these are zeroed. We are testing for equality here only because this is a
2813		// known small test reproduction which hits the relevant LLVM issue.
2814		@memset(@ptrCast([*]u8, &parser)[0..@sizeOf(StreamingParser)], 0);
2815		try std.testing.expectEqual(parser, parser);
2816		}
2817
2818		fn checkNext(p: *TokenStream, id: std.meta.Tag(Token)) !void {
2819		const token = (p.next() catch unreachable).?;
2820		try testing.expect(std.meta.activeTag(token) == id);
2821		}
2822
2823		test "json.token" {
2824		const s =
2825		\\{
2826		\\ "Image": {
2827		\\ "Width": 800,
2828		\\ "Height": 600,
2829		\\ "Title": "View from 15th Floor",
2830		\\ "Thumbnail": {
2831		\\ "Url": "http://www.example.com/image/481989943",
2832		\\ "Height": 125,
2833		\\ "Width": 100
2834		\\ },
2835		\\ "Animated" : false,
2836		\\ "IDs": [116, 943, 234, 38793]
2837		\\ }
2838		\\}
2839		;
2840
2841		var p = TokenStream.init(s);
2842
2843		try checkNext(&p, .ObjectBegin);
2844		try checkNext(&p, .String); // Image
2845		try checkNext(&p, .ObjectBegin);
2846		try checkNext(&p, .String); // Width
2847		try checkNext(&p, .Number);
2848		try checkNext(&p, .String); // Height
2849		try checkNext(&p, .Number);
2850		try checkNext(&p, .String); // Title
2851		try checkNext(&p, .String);
2852		try checkNext(&p, .String); // Thumbnail
2853		try checkNext(&p, .ObjectBegin);
2854		try checkNext(&p, .String); // Url
2855		try checkNext(&p, .String);
2856		try checkNext(&p, .String); // Height
2857		try checkNext(&p, .Number);
2858		try checkNext(&p, .String); // Width
2859		try checkNext(&p, .Number);
2860		try checkNext(&p, .ObjectEnd);
2861		try checkNext(&p, .String); // Animated
2862		try checkNext(&p, .False);
2863		try checkNext(&p, .String); // IDs
2864		try checkNext(&p, .ArrayBegin);
2865		try checkNext(&p, .Number);
2866		try checkNext(&p, .Number);
2867		try checkNext(&p, .Number);
2868		try checkNext(&p, .Number);
2869		try checkNext(&p, .ArrayEnd);
2870		try checkNext(&p, .ObjectEnd);
2871		try checkNext(&p, .ObjectEnd);
2872
2873		try testing.expect((try p.next()) == null);
2874		}
2875
2876		test "json.token mismatched close" {
2877		var p = TokenStream.init("[102, 111, 111 }");
2878		try checkNext(&p, .ArrayBegin);
2879		try checkNext(&p, .Number);
2880		try checkNext(&p, .Number);
2881		try checkNext(&p, .Number);
2882		try testing.expectError(error.UnexpectedClosingBrace, p.next());
2883		}
2884
2885		test "json.token premature object close" {
2886		var p = TokenStream.init("{ \"key\": }");
2887		try checkNext(&p, .ObjectBegin);
2888		try checkNext(&p, .String);
2889		try testing.expectError(error.InvalidValueBegin, p.next());
2890		}
2891
2892		test "json.validate" {
2893		try testing.expectEqual(true, validate("{}"));
2894		try testing.expectEqual(true, validate("[]"));
2895		try testing.expectEqual(true, validate("[{[[[[{}]]]]}]"));
2896		try testing.expectEqual(false, validate("{]"));
2897		try testing.expectEqual(false, validate("[}"));
2898		try testing.expectEqual(false, validate("{{{{[]}}}]"));
2899		}
2900
2901		test "Value.jsonStringify" {
2902		{
2903		var buffer: [10]u8 = undefined;
2904		var fbs = std.io.fixedBufferStream(&buffer);
2905		try @as(Value, .Null).jsonStringify(.{}, fbs.writer());
2906		try testing.expectEqualSlices(u8, fbs.getWritten(), "null");
2907		}
2908		{
2909		var buffer: [10]u8 = undefined;
2910		var fbs = std.io.fixedBufferStream(&buffer);
2911		try (Value{ .Bool = true }).jsonStringify(.{}, fbs.writer());
2912		try testing.expectEqualSlices(u8, fbs.getWritten(), "true");
2913		}
2914		{
2915		var buffer: [10]u8 = undefined;
2916		var fbs = std.io.fixedBufferStream(&buffer);
2917		try (Value{ .Integer = 42 }).jsonStringify(.{}, fbs.writer());
2918		try testing.expectEqualSlices(u8, fbs.getWritten(), "42");
2919		}
2920		{
2921		var buffer: [10]u8 = undefined;
2922		var fbs = std.io.fixedBufferStream(&buffer);
2923		try (Value{ .NumberString = "43" }).jsonStringify(.{}, fbs.writer());
2924		try testing.expectEqualSlices(u8, fbs.getWritten(), "43");
2925		}
2926		{
2927		var buffer: [10]u8 = undefined;
2928		var fbs = std.io.fixedBufferStream(&buffer);
2929		try (Value{ .Float = 42 }).jsonStringify(.{}, fbs.writer());
2930		try testing.expectEqualSlices(u8, fbs.getWritten(), "4.2e+01");
2931		}
2932		{
2933		var buffer: [10]u8 = undefined;
2934		var fbs = std.io.fixedBufferStream(&buffer);
2935		try (Value{ .String = "weeee" }).jsonStringify(.{}, fbs.writer());
2936		try testing.expectEqualSlices(u8, fbs.getWritten(), "\"weeee\"");
2937		}
2938		{
2939		var buffer: [10]u8 = undefined;
2940		var fbs = std.io.fixedBufferStream(&buffer);
2941		var vals = [_]Value{
2942		.{ .Integer = 1 },
2943		.{ .Integer = 2 },
2944		.{ .NumberString = "3" },
2945		};
2946		try (Value{
2947		.Array = Array.fromOwnedSlice(undefined, &vals),
2948		}).jsonStringify(.{}, fbs.writer());
2949		try testing.expectEqualSlices(u8, fbs.getWritten(), "[1,2,3]");
2950		}
2951		{
2952		var buffer: [10]u8 = undefined;
2953		var fbs = std.io.fixedBufferStream(&buffer);
2954		var obj = ObjectMap.init(testing.allocator);
2955		defer obj.deinit();
2956		try obj.putNoClobber("a", .{ .String = "b" });
2957		try (Value{ .Object = obj }).jsonStringify(.{}, fbs.writer());
2958		try testing.expectEqualSlices(u8, fbs.getWritten(), "{\"a\":\"b\"}");
2959		}
	101	try err("\"\xf4\x90\x80\x80\"");
	102	try err("\"\xf5\x80\x80\x80\"");
	103	try err("\"\xf6\x80\x80\x80\"");
	104	try err("\"\xf7\x80\x80\x80\"");
	105	try err("\"\xf8\x80\x80\x80\"");
	106	try err("\"\xf9\x80\x80\x80\"");
	107	try err("\"\xfa\x80\x80\x80\"");
	108	try err("\"\xfb\x80\x80\x80\"");
	109	try err("\"\xfc\x80\x80\x80\"");
	110	try err("\"\xfd\x80\x80\x80\"");
	111	try err("\"\xfe\x80\x80\x80\"");
	112	try err("\"\xff\x80\x80\x80\"");
2960	113	}

lib/std/json/write_stream.zig+63-56

...	...	@@ -1,14 +1,19 @@
1		const std = @import("../std.zig");
	1	const std = @import("std");
2	2	const assert = std.debug.assert;
3	3	const maxInt = std.math.maxInt;
4	4
	5	const StringifyOptions = @import("./stringify.zig").StringifyOptions;
	6	const jsonStringify = @import("./stringify.zig").stringify;
	7
	8	const Value = @import("./dynamic.zig").Value;
	9
5	10	const State = enum {
6		Complete,
7		Value,
8		ArrayStart,
9		Array,
10		ObjectStart,
11		Object,
	11	complete,
	12	value,
	13	array_start,
	14	array,
	15	object_start,
	16	object,
12	17	};
13	18
14	19	/// Writes JSON ([RFC8259](https://tools.ietf.org/html/rfc8259)) formatted data
...	...	@@ -21,9 +26,9 @@ pub fn WriteStream(comptime OutStream: type, comptime max_depth: usize) type {
21	26
22	27	pub const Stream = OutStream;
23	28
24		whitespace: std.json.StringifyOptions.Whitespace = std.json.StringifyOptions.Whitespace{
	29	whitespace: StringifyOptions.Whitespace = StringifyOptions.Whitespace{
25	30	.indent_level = 0,
26		.indent = .{ .Space = 1 },
	31	.indent = .{ .space = 1 },
27	32	},
28	33
29	34	stream: OutStream,
...	...	@@ -36,38 +41,38 @@ pub fn WriteStream(comptime OutStream: type, comptime max_depth: usize) type {
36	41	.state_index = 1,
37	42	.state = undefined,
38	43	};
39		self.state[0] = .Complete;
40		self.state[1] = .Value;
	44	self.state[0] = .complete;
	45	self.state[1] = .value;
41	46	return self;
42	47	}
43	48
44	49	pub fn beginArray(self: *Self) !void {
45		assert(self.state[self.state_index] == State.Value); // need to call arrayElem or objectField
	50	assert(self.state[self.state_index] == State.value); // need to call arrayElem or objectField
46	51	try self.stream.writeByte('[');
47		self.state[self.state_index] = State.ArrayStart;
	52	self.state[self.state_index] = State.array_start;
48	53	self.whitespace.indent_level += 1;
49	54	}
50	55
51	56	pub fn beginObject(self: *Self) !void {
52		assert(self.state[self.state_index] == State.Value); // need to call arrayElem or objectField
	57	assert(self.state[self.state_index] == State.value); // need to call arrayElem or objectField
53	58	try self.stream.writeByte('{');
54		self.state[self.state_index] = State.ObjectStart;
	59	self.state[self.state_index] = State.object_start;
55	60	self.whitespace.indent_level += 1;
56	61	}
57	62
58	63	pub fn arrayElem(self: *Self) !void {
59	64	const state = self.state[self.state_index];
60	65	switch (state) {
61		.Complete => unreachable,
62		.Value => unreachable,
63		.ObjectStart => unreachable,
64		.Object => unreachable,
65		.Array, .ArrayStart => {
66		if (state == .Array) {
	66	.complete => unreachable,
	67	.value => unreachable,
	68	.object_start => unreachable,
	69	.object => unreachable,
	70	.array, .array_start => {
	71	if (state == .array) {
67	72	try self.stream.writeByte(',');
68	73	}
69		self.state[self.state_index] = .Array;
70		self.pushState(.Value);
	74	self.state[self.state_index] = .array;
	75	self.pushState(.value);
71	76	try self.indent();
72	77	},
73	78	}
...	...	@@ -76,16 +81,16 @@ pub fn WriteStream(comptime OutStream: type, comptime max_depth: usize) type {
76	81	pub fn objectField(self: *Self, name: []const u8) !void {
77	82	const state = self.state[self.state_index];
78	83	switch (state) {
79		.Complete => unreachable,
80		.Value => unreachable,
81		.ArrayStart => unreachable,
82		.Array => unreachable,
83		.Object, .ObjectStart => {
84		if (state == .Object) {
	84	.complete => unreachable,
	85	.value => unreachable,
	86	.array_start => unreachable,
	87	.array => unreachable,
	88	.object, .object_start => {
	89	if (state == .object) {
85	90	try self.stream.writeByte(',');
86	91	}
87		self.state[self.state_index] = .Object;
88		self.pushState(.Value);
	92	self.state[self.state_index] = .object;
	93	self.pushState(.value);
89	94	try self.indent();
90	95	try self.writeEscapedString(name);
91	96	try self.stream.writeByte(':');
...	...	@@ -98,16 +103,16 @@ pub fn WriteStream(comptime OutStream: type, comptime max_depth: usize) type {
98	103
99	104	pub fn endArray(self: *Self) !void {
100	105	switch (self.state[self.state_index]) {
101		.Complete => unreachable,
102		.Value => unreachable,
103		.ObjectStart => unreachable,
104		.Object => unreachable,
105		.ArrayStart => {
	106	.complete => unreachable,
	107	.value => unreachable,
	108	.object_start => unreachable,
	109	.object => unreachable,
	110	.array_start => {
106	111	self.whitespace.indent_level -= 1;
107	112	try self.stream.writeByte(']');
108	113	self.popState();
109	114	},
110		.Array => {
	115	.array => {
111	116	self.whitespace.indent_level -= 1;
112	117	try self.indent();
113	118	self.popState();
...	...	@@ -118,16 +123,16 @@ pub fn WriteStream(comptime OutStream: type, comptime max_depth: usize) type {
118	123
119	124	pub fn endObject(self: *Self) !void {
120	125	switch (self.state[self.state_index]) {
121		.Complete => unreachable,
122		.Value => unreachable,
123		.ArrayStart => unreachable,
124		.Array => unreachable,
125		.ObjectStart => {
	126	.complete => unreachable,
	127	.value => unreachable,
	128	.array_start => unreachable,
	129	.array => unreachable,
	130	.object_start => {
126	131	self.whitespace.indent_level -= 1;
127	132	try self.stream.writeByte('}');
128	133	self.popState();
129	134	},
130		.Object => {
	135	.object => {
131	136	self.whitespace.indent_level -= 1;
132	137	try self.indent();
133	138	self.popState();
...	...	@@ -137,13 +142,13 @@ pub fn WriteStream(comptime OutStream: type, comptime max_depth: usize) type {
137	142	}
138	143
139	144	pub fn emitNull(self: *Self) !void {
140		assert(self.state[self.state_index] == State.Value);
	145	assert(self.state[self.state_index] == State.value);
141	146	try self.stringify(null);
142	147	self.popState();
143	148	}
144	149
145	150	pub fn emitBool(self: *Self, value: bool) !void {
146		assert(self.state[self.state_index] == State.Value);
	151	assert(self.state[self.state_index] == State.value);
147	152	try self.stringify(value);
148	153	self.popState();
149	154	}
...	...	@@ -154,7 +159,7 @@ pub fn WriteStream(comptime OutStream: type, comptime max_depth: usize) type {
154	159	/// in a IEEE 754 double float, otherwise emitted as a string to the full precision.
155	160	value: anytype,
156	161	) !void {
157		assert(self.state[self.state_index] == State.Value);
	162	assert(self.state[self.state_index] == State.value);
158	163	switch (@typeInfo(@TypeOf(value))) {
159	164	.Int => \|info\| {
160	165	if (info.bits < 53) {
...	...	@@ -183,7 +188,7 @@ pub fn WriteStream(comptime OutStream: type, comptime max_depth: usize) type {
183	188	}
184	189
185	190	pub fn emitString(self: *Self, string: []const u8) !void {
186		assert(self.state[self.state_index] == State.Value);
	191	assert(self.state[self.state_index] == State.value);
187	192	try self.writeEscapedString(string);
188	193	self.popState();
189	194	}
...	...	@@ -194,9 +199,9 @@ pub fn WriteStream(comptime OutStream: type, comptime max_depth: usize) type {
194	199	}
195	200
196	201	/// Writes the complete json into the output stream
197		pub fn emitJson(self: *Self, json: std.json.Value) Stream.Error!void {
198		assert(self.state[self.state_index] == State.Value);
199		try self.stringify(json);
	202	pub fn emitJson(self: *Self, value: Value) Stream.Error!void {
	203	assert(self.state[self.state_index] == State.value);
	204	try self.stringify(value);
200	205	self.popState();
201	206	}
202	207
...	...	@@ -215,7 +220,7 @@ pub fn WriteStream(comptime OutStream: type, comptime max_depth: usize) type {
215	220	}
216	221
217	222	fn stringify(self: *Self, value: anytype) !void {
218		try std.json.stringify(value, std.json.StringifyOptions{
	223	try jsonStringify(value, StringifyOptions{
219	224	.whitespace = self.whitespace,
220	225	}, self.stream);
221	226	}
...	...	@@ -229,6 +234,8 @@ pub fn writeStream(
229	234	return WriteStream(@TypeOf(out_stream), max_depth).init(out_stream);
230	235	}
231	236
	237	const ObjectMap = @import("./dynamic.zig").ObjectMap;
	238
232	239	test "json write stream" {
233	240	var out_buf: [1024]u8 = undefined;
234	241	var slice_stream = std.io.fixedBufferStream(&out_buf);
...	...	@@ -237,7 +244,7 @@ test "json write stream" {
237	244	var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
238	245	defer arena_allocator.deinit();
239	246
240		var w = std.json.writeStream(out, 10);
	247	var w = writeStream(out, 10);
241	248
242	249	try w.beginObject();
243	250
...	...	@@ -285,9 +292,9 @@ test "json write stream" {
285	292	try std.testing.expect(std.mem.eql(u8, expected, result));
286	293	}
287	294
288		fn getJsonObject(allocator: std.mem.Allocator) !std.json.Value {
289		var value = std.json.Value{ .Object = std.json.ObjectMap.init(allocator) };
290		try value.Object.put("one", std.json.Value{ .Integer = @intCast(i64, 1) });
291		try value.Object.put("two", std.json.Value{ .Float = 2.0 });
	295	fn getJsonObject(allocator: std.mem.Allocator) !Value {
	296	var value = Value{ .object = ObjectMap.init(allocator) };
	297	try value.object.put("one", Value{ .integer = @intCast(i64, 1) });
	298	try value.object.put("two", Value{ .float = 2.0 });
292	299	return value;
293	300	}

src/Autodoc.zig+7-7

...	...	@@ -295,7 +295,7 @@ pub fn generateZirData(self: *Autodoc) !void {
295	295	try std.json.stringify(
296	296	data,
297	297	.{
298		.whitespace = .{ .indent = .None, .separator = false },
	298	.whitespace = .{ .indent = .none, .separator = false },
299	299	.emit_null_optional_fields = true,
300	300	},
301	301	out,
...	...	@@ -444,7 +444,7 @@ const DocData = struct {
444	444	w: anytype,
445	445	) !void {
446	446	var jsw = std.json.writeStream(w, 15);
447		if (opts.whitespace) \|ws\| jsw.whitespace = ws;
	447	jsw.whitespace = opts.whitespace;
448	448	try jsw.beginObject();
449	449	inline for (comptime std.meta.tags(std.meta.FieldEnum(DocData))) \|f\| {
450	450	const f_name = @tagName(f);
...	...	@@ -495,7 +495,7 @@ const DocData = struct {
495	495	w: anytype,
496	496	) !void {
497	497	var jsw = std.json.writeStream(w, 15);
498		if (opts.whitespace) \|ws\| jsw.whitespace = ws;
	498	jsw.whitespace = opts.whitespace;
499	499
500	500	try jsw.beginObject();
501	501	inline for (comptime std.meta.tags(std.meta.FieldEnum(DocModule))) \|f\| {
...	...	@@ -529,7 +529,7 @@ const DocData = struct {
529	529	w: anytype,
530	530	) !void {
531	531	var jsw = std.json.writeStream(w, 15);
532		if (opts.whitespace) \|ws\| jsw.whitespace = ws;
	532	jsw.whitespace = opts.whitespace;
533	533	try jsw.beginArray();
534	534	inline for (comptime std.meta.fields(Decl)) \|f\| {
535	535	try jsw.arrayElem();
...	...	@@ -556,7 +556,7 @@ const DocData = struct {
556	556	w: anytype,
557	557	) !void {
558	558	var jsw = std.json.writeStream(w, 15);
559		if (opts.whitespace) \|ws\| jsw.whitespace = ws;
	559	jsw.whitespace = opts.whitespace;
560	560	try jsw.beginArray();
561	561	inline for (comptime std.meta.fields(AstNode)) \|f\| {
562	562	try jsw.arrayElem();
...	...	@@ -689,7 +689,7 @@ const DocData = struct {
689	689	) !void {
690	690	const active_tag = std.meta.activeTag(self);
691	691	var jsw = std.json.writeStream(w, 15);
692		if (opts.whitespace) \|ws\| jsw.whitespace = ws;
	692	jsw.whitespace = opts.whitespace;
693	693	try jsw.beginArray();
694	694	try jsw.arrayElem();
695	695	try jsw.emitNumber(@enumToInt(active_tag));
...	...	@@ -831,7 +831,7 @@ const DocData = struct {
831	831	) @TypeOf(w).Error!void {
832	832	const active_tag = std.meta.activeTag(self);
833	833	var jsw = std.json.writeStream(w, 15);
834		if (opts.whitespace) \|ws\| jsw.whitespace = ws;
	834	jsw.whitespace = opts.whitespace;
835	835	try jsw.beginObject();
836	836	if (active_tag == .declIndex) {
837	837	try jsw.objectField("declRef");

src/print_env.zig+1-1

...	...	@@ -28,7 +28,7 @@ pub fn cmdEnv(gpa: Allocator, args: []const []const u8, stdout: std.fs.File.Writ
28	28	var bw = std.io.bufferedWriter(stdout);
29	29	const w = bw.writer();
30	30
31		var jws = std.json.WriteStream(@TypeOf(w), 6).init(w);
	31	var jws = std.json.writeStream(w, 6);
32	32	try jws.beginObject();
33	33
34	34	try jws.objectField("zig_exe");

src/print_targets.zig+1-1

...	...	@@ -40,7 +40,7 @@ pub fn cmdTargets(
40	40
41	41	var bw = io.bufferedWriter(stdout);
42	42	const w = bw.writer();
43		var jws = std.json.WriteStream(@TypeOf(w), 6).init(w);
	43	var jws = std.json.writeStream(w, 6);
44	44
45	45	try jws.beginObject();
46	46

tools/gen_spirv_spec.zig+1-2

...	...	@@ -20,8 +20,7 @@ pub fn main() !void {
20	20	// Required for json parsing.
21	21	@setEvalBranchQuota(10000);
22	22
23		var tokens = std.json.TokenStream.init(spec);
24		var registry = try std.json.parse(g.Registry, &tokens, .{ .allocator = allocator });
	23	var registry = try std.json.parseFromSlice(g.Registry, allocator, spec, .{});
25	24
26	25	const core_reg = switch (registry) {
27	26	.core => \|core_reg\| core_reg,

tools/generate_JSONTestSuite.zig created+79

...	...	@@ -0,0 +1,79 @@
	1	// zig run this file inside the test_parsing/ directory of this repo: https://github.com/nst/JSONTestSuite
	2
	3	const std = @import("std");
	4
	5	pub fn main() !void {
	6	var gpa = std.heap.GeneralPurposeAllocator(.{}){};
	7	var allocator = gpa.allocator();
	8
	9	var output = std.io.getStdOut().writer();
	10	try output.writeAll(
	11	\\// This file was generated by _generate_JSONTestSuite.zig
	12	\\// These test cases are sourced from: https://github.com/nst/JSONTestSuite
	13	\\const ok = @import("./test.zig").ok;
	14	\\const err = @import("./test.zig").err;
	15	\\const any = @import("./test.zig").any;
	16	\\
	17	\\
	18	);
	19
	20	var names = std.ArrayList([]const u8).init(allocator);
	21	var cwd = try std.fs.cwd().openIterableDir(".", .{});
	22	var it = cwd.iterate();
	23	while (try it.next()) \|entry\| {
	24	try names.append(try allocator.dupe(u8, entry.name));
	25	}
	26	std.sort.sort([]const u8, names.items, {}, (struct {
	27	fn lessThan(_: void, a: []const u8, b: []const u8) bool {
	28	return std.mem.lessThan(u8, a, b);
	29	}
	30	}).lessThan);
	31
	32	for (names.items) \|name\| {
	33	const contents = try std.fs.cwd().readFileAlloc(allocator, name, 250001);
	34	try output.writeAll("test ");
	35	try writeString(output, name);
	36	try output.writeAll(" {\n try ");
	37	switch (name[0]) {
	38	'y' => try output.writeAll("ok"),
	39	'n' => try output.writeAll("err"),
	40	'i' => try output.writeAll("any"),
	41	else => unreachable,
	42	}
	43	try output.writeByte('(');
	44	try writeString(output, contents);
	45	try output.writeAll(");\n}\n");
	46	}
	47	}
	48
	49	const i_structure_500_nested_arrays = "[" 500 ++ "]" 500;
	50	const n_structure_100000_opening_arrays = "[" ** 100000;
	51	const n_structure_open_array_object = "[{\"\":" ** 50000 ++ "\n";
	52
	53	fn writeString(writer: anytype, s: []const u8) !void {
	54	if (s.len > 200) {
	55	// There are a few of these we can compress with Zig expressions.
	56	if (std.mem.eql(u8, s, i_structure_500_nested_arrays)) {
	57	return writer.writeAll("\"[\" 500 ++ \"]\" 500");
	58	} else if (std.mem.eql(u8, s, n_structure_100000_opening_arrays)) {
	59	return writer.writeAll("\"[\" ** 100000");
	60	} else if (std.mem.eql(u8, s, n_structure_open_array_object)) {
	61	return writer.writeAll("\"[{\\\"\\\":\" ** 50000 ++ \"\\n\"");
	62	}
	63	unreachable;
	64	}
	65	try writer.writeByte('"');
	66	for (s) \|b\| {
	67	switch (b) {
	68	0...('\n' - 1),
	69	('\n' + 1)...0x1f,
	70	0x7f...0xff,
	71	=> try writer.print("\\x{x:0>2}", .{b}),
	72	'\n' => try writer.writeAll("\\n"),
	73	'"' => try writer.writeAll("\\\""),
	74	'\\' => try writer.writeAll("\\\\"),
	75	else => try writer.writeByte(b),
	76	}
	77	}
	78	try writer.writeByte('"');
	79	}

tools/update_clang_options.zig+22-22

...	...	@@ -624,9 +624,9 @@ pub fn main() anyerror!void {
624	624	},
625	625	};
626	626
627		var parser = json.Parser.init(allocator, false);
	627	var parser = json.Parser.init(allocator, .alloc_if_needed);
628	628	const tree = try parser.parse(json_text);
629		const root_map = &tree.root.Object;
	629	const root_map = &tree.root.object;
630	630
631	631	var all_objects = std.ArrayList(*json.ObjectMap).init(allocator);
632	632	{
...	...	@@ -634,14 +634,14 @@ pub fn main() anyerror!void {
634	634	it_map: while (it.next()) \|kv\| {
635	635	if (kv.key_ptr.len == 0) continue;
636	636	if (kv.key_ptr.*[0] == '!') continue;
637		if (kv.value_ptr.* != .Object) continue;
638		if (!kv.value_ptr.Object.contains("NumArgs")) continue;
639		if (!kv.value_ptr.Object.contains("Name")) continue;
	637	if (kv.value_ptr.* != .object) continue;
	638	if (!kv.value_ptr.object.contains("NumArgs")) continue;
	639	if (!kv.value_ptr.object.contains("Name")) continue;
640	640	for (blacklisted_options) \|blacklisted_key\| {
641	641	if (std.mem.eql(u8, blacklisted_key, kv.key_ptr.*)) continue :it_map;
642	642	}
643		if (kv.value_ptr.Object.get("Name").?.String.len == 0) continue;
644		try all_objects.append(&kv.value_ptr.Object);
	643	if (kv.value_ptr.object.get("Name").?.string.len == 0) continue;
	644	try all_objects.append(&kv.value_ptr.object);
645	645	}
646	646	}
647	647	// Some options have multiple matches. As an example, "-Wl,foo" matches both
...	...	@@ -666,12 +666,12 @@ pub fn main() anyerror!void {
666	666	);
667	667
668	668	for (all_objects.items) \|obj\| {
669		const name = obj.get("Name").?.String;
	669	const name = obj.get("Name").?.string;
670	670	var pd1 = false;
671	671	var pd2 = false;
672	672	var pslash = false;
673		for (obj.get("Prefixes").?.Array.items) \|prefix_json\| {
674		const prefix = prefix_json.String;
	673	for (obj.get("Prefixes").?.array.items) \|prefix_json\| {
	674	const prefix = prefix_json.string;
675	675	if (std.mem.eql(u8, prefix, "-")) {
676	676	pd1 = true;
677	677	} else if (std.mem.eql(u8, prefix, "--")) {
...	...	@@ -790,9 +790,9 @@ const Syntax = union(enum) {
790	790	};
791	791
792	792	fn objSyntax(obj: *json.ObjectMap) ?Syntax {
793		const num_args = @intCast(u8, obj.get("NumArgs").?.Integer);
794		for (obj.get("!superclasses").?.Array.items) \|superclass_json\| {
795		const superclass = superclass_json.String;
	793	const num_args = @intCast(u8, obj.get("NumArgs").?.integer);
	794	for (obj.get("!superclasses").?.array.items) \|superclass_json\| {
	795	const superclass = superclass_json.string;
796	796	if (std.mem.eql(u8, superclass, "Joined")) {
797	797	return .joined;
798	798	} else if (std.mem.eql(u8, superclass, "CLJoined")) {
...	...	@@ -831,20 +831,20 @@ fn objSyntax(obj: *json.ObjectMap) ?Syntax {
831	831	return .{ .multi_arg = num_args };
832	832	}
833	833	}
834		const name = obj.get("Name").?.String;
	834	const name = obj.get("Name").?.string;
835	835	if (std.mem.eql(u8, name, "<input>")) {
836	836	return .flag;
837	837	} else if (std.mem.eql(u8, name, "<unknown>")) {
838	838	return .flag;
839	839	}
840		const kind_def = obj.get("Kind").?.Object.get("def").?.String;
	840	const kind_def = obj.get("Kind").?.object.get("def").?.string;
841	841	if (std.mem.eql(u8, kind_def, "KIND_FLAG")) {
842	842	return .flag;
843	843	}
844		const key = obj.get("!name").?.String;
	844	const key = obj.get("!name").?.string;
845	845	std.debug.print("{s} (key {s}) has unrecognized superclasses:\n", .{ name, key });
846		for (obj.get("!superclasses").?.Array.items) \|superclass_json\| {
847		std.debug.print(" {s}\n", .{superclass_json.String});
	846	for (obj.get("!superclasses").?.array.items) \|superclass_json\| {
	847	std.debug.print(" {s}\n", .{superclass_json.string});
848	848	}
849	849	//std.process.exit(1);
850	850	return null;
...	...	@@ -883,15 +883,15 @@ fn objectLessThan(context: void, a: json.ObjectMap, b: json.ObjectMap) bool {
883	883	}
884	884
885	885	if (!a_match_with_eql and !b_match_with_eql) {
886		const a_name = a.get("Name").?.String;
887		const b_name = b.get("Name").?.String;
	886	const a_name = a.get("Name").?.string;
	887	const b_name = b.get("Name").?.string;
888	888	if (a_name.len != b_name.len) {
889	889	return a_name.len > b_name.len;
890	890	}
891	891	}
892	892
893		const a_key = a.get("!name").?.String;
894		const b_key = b.get("!name").?.String;
	893	const a_key = a.get("!name").?.string;
	894	const b_key = b.get("!name").?.string;
895	895	return std.mem.lessThan(u8, a_key, b_key);
896	896	}
897	897

tools/update_cpu_features.zig+22-22

...	...	@@ -1054,14 +1054,14 @@ fn processOneTarget(job: Job) anyerror!void {
1054	1054	var json_parse_progress = progress_node.start("parse JSON", 0);
1055	1055	json_parse_progress.activate();
1056	1056
1057		var parser = json.Parser.init(arena, false);
	1057	var parser = json.Parser.init(arena, .alloc_if_needed);
1058	1058	const tree = try parser.parse(json_text);
1059	1059	json_parse_progress.end();
1060	1060
1061	1061	var render_progress = progress_node.start("render zig code", 0);
1062	1062	render_progress.activate();
1063	1063
1064		const root_map = &tree.root.Object;
	1064	const root_map = &tree.root.object;
1065	1065	var features_table = std.StringHashMap(Feature).init(arena);
1066	1066	var all_features = std.ArrayList(Feature).init(arena);
1067	1067	var all_cpus = std.ArrayList(Cpu).init(arena);
...	...	@@ -1070,21 +1070,21 @@ fn processOneTarget(job: Job) anyerror!void {
1070	1070	root_it: while (it.next()) \|kv\| {
1071	1071	if (kv.key_ptr.len == 0) continue;
1072	1072	if (kv.key_ptr.*[0] == '!') continue;
1073		if (kv.value_ptr.* != .Object) continue;
1074		if (hasSuperclass(&kv.value_ptr.Object, "SubtargetFeature")) {
1075		const llvm_name = kv.value_ptr.Object.get("Name").?.String;
	1073	if (kv.value_ptr.* != .object) continue;
	1074	if (hasSuperclass(&kv.value_ptr.object, "SubtargetFeature")) {
	1075	const llvm_name = kv.value_ptr.object.get("Name").?.string;
1076	1076	if (llvm_name.len == 0) continue;
1077	1077
1078	1078	var zig_name = try llvmNameToZigName(arena, llvm_name);
1079		var desc = kv.value_ptr.Object.get("Desc").?.String;
	1079	var desc = kv.value_ptr.object.get("Desc").?.string;
1080	1080	var deps = std.ArrayList([]const u8).init(arena);
1081	1081	var omit = false;
1082	1082	var flatten = false;
1083		const implies = kv.value_ptr.Object.get("Implies").?.Array;
	1083	const implies = kv.value_ptr.object.get("Implies").?.array;
1084	1084	for (implies.items) \|imply\| {
1085		const other_key = imply.Object.get("def").?.String;
1086		const other_obj = &root_map.getPtr(other_key).?.Object;
1087		const other_llvm_name = other_obj.get("Name").?.String;
	1085	const other_key = imply.object.get("def").?.string;
	1086	const other_obj = &root_map.getPtr(other_key).?.object;
	1087	const other_llvm_name = other_obj.get("Name").?.string;
1088	1088	const other_zig_name = (try llvmNameToZigNameOmit(
1089	1089	arena,
1090	1090	llvm_target,
...	...	@@ -1126,17 +1126,17 @@ fn processOneTarget(job: Job) anyerror!void {
1126	1126	try all_features.append(feature);
1127	1127	}
1128	1128	}
1129		if (hasSuperclass(&kv.value_ptr.Object, "Processor")) {
1130		const llvm_name = kv.value_ptr.Object.get("Name").?.String;
	1129	if (hasSuperclass(&kv.value_ptr.object, "Processor")) {
	1130	const llvm_name = kv.value_ptr.object.get("Name").?.string;
1131	1131	if (llvm_name.len == 0) continue;
1132	1132
1133	1133	var zig_name = try llvmNameToZigName(arena, llvm_name);
1134	1134	var deps = std.ArrayList([]const u8).init(arena);
1135		const features = kv.value_ptr.Object.get("Features").?.Array;
	1135	const features = kv.value_ptr.object.get("Features").?.array;
1136	1136	for (features.items) \|feature\| {
1137		const feature_key = feature.Object.get("def").?.String;
1138		const feature_obj = &root_map.getPtr(feature_key).?.Object;
1139		const feature_llvm_name = feature_obj.get("Name").?.String;
	1137	const feature_key = feature.object.get("def").?.string;
	1138	const feature_obj = &root_map.getPtr(feature_key).?.object;
	1139	const feature_llvm_name = feature_obj.get("Name").?.string;
1140	1140	if (feature_llvm_name.len == 0) continue;
1141	1141	const feature_zig_name = (try llvmNameToZigNameOmit(
1142	1142	arena,
...	...	@@ -1145,11 +1145,11 @@ fn processOneTarget(job: Job) anyerror!void {
1145	1145	)) orelse continue;
1146	1146	try deps.append(feature_zig_name);
1147	1147	}
1148		const tune_features = kv.value_ptr.Object.get("TuneFeatures").?.Array;
	1148	const tune_features = kv.value_ptr.object.get("TuneFeatures").?.array;
1149	1149	for (tune_features.items) \|feature\| {
1150		const feature_key = feature.Object.get("def").?.String;
1151		const feature_obj = &root_map.getPtr(feature_key).?.Object;
1152		const feature_llvm_name = feature_obj.get("Name").?.String;
	1150	const feature_key = feature.object.get("def").?.string;
	1151	const feature_obj = &root_map.getPtr(feature_key).?.object;
	1152	const feature_llvm_name = feature_obj.get("Name").?.string;
1153	1153	if (feature_llvm_name.len == 0) continue;
1154	1154	const feature_zig_name = (try llvmNameToZigNameOmit(
1155	1155	arena,
...	...	@@ -1431,8 +1431,8 @@ fn llvmNameToZigNameOmit(
1431	1431
1432	1432	fn hasSuperclass(obj: *json.ObjectMap, class_name: []const u8) bool {
1433	1433	const superclasses_json = obj.get("!superclasses") orelse return false;
1434		for (superclasses_json.Array.items) \|superclass_json\| {
1435		const superclass = superclass_json.String;
	1434	for (superclasses_json.array.items) \|superclass_json\| {
	1435	const superclass = superclass_json.string;
1436	1436	if (std.mem.eql(u8, superclass, class_name)) {
1437	1437	return true;
1438	1438	}

tools/update_spirv_features.zig+1-2

...	...	@@ -74,8 +74,7 @@ pub fn main() !void {
74	74
75	75	const registry_path = try fs.path.join(allocator, &.{ spirv_headers_root, "include", "spirv", "unified1", "spirv.core.grammar.json" });
76	76	const registry_json = try std.fs.cwd().readFileAlloc(allocator, registry_path, std.math.maxInt(usize));
77		var tokens = std.json.TokenStream.init(registry_json);
78		const registry = try std.json.parse(g.CoreRegistry, &tokens, .{ .allocator = allocator });
	77	const registry = try std.json.parseFromSlice(g.CoreRegistry, allocator, registry_json, .{});
79	78
80	79	const capabilities = for (registry.operand_kinds) \|opkind\| {
81	80	if (std.mem.eql(u8, opkind.kind, "Capability"))