authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-12-30 18:13:20-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2019-12-30 18:13:20-05:00
log8f8a32d297e872acb9e917559228a4a7675cd680
treebd8bb74e3414d6e78684df604472f00992aebc51
parent28a8ded95a96b1e2af5d2a73f47db2c967233476
parent42727c73f92b7b3453de643ecd13eeb35ce00b72
signature Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #4007 from daurnimator/json-cleanup

std.json: cleanups

2 files changed, 468 insertions(+), 371 deletions(-)

lib/std/json.zig+467-368
...@@ -4,83 +4,63 @@...@@ -4,83 +4,63 @@
44
5const std = @import("std.zig");5const std = @import("std.zig");
6const debug = std.debug;6const debug = std.debug;
7const assert = debug.assert;
7const testing = std.testing;8const testing = std.testing;
8const mem = std.mem;9const mem = std.mem;
9const maxInt = std.math.maxInt;10const maxInt = std.math.maxInt;
1011
11pub const WriteStream = @import("json/write_stream.zig").WriteStream;12pub const WriteStream = @import("json/write_stream.zig").WriteStream;
1213
14const StringEscapes = union(enum) {
15 None,
16
17 Some: struct {
18 size_diff: isize,
19 },
20};
21
13/// A single token slice into the parent string.22/// A single token slice into the parent string.
14///23///
15/// Use `token.slice()` on the input at the current position to get the current slice.24/// Use `token.slice()` on the input at the current position to get the current slice.
16pub const Token = struct {25pub const Token = union(enum) {
17 id: Id,26 ObjectBegin,
18 /// How many bytes do we skip before counting27 ObjectEnd,
19 offset: u1,28 ArrayBegin,
20 /// Whether string contains an escape sequence and cannot be zero-copied29 ArrayEnd,
21 string_has_escape: bool,30 String: struct {
22 /// Whether number is simple and can be represented by an integer (i.e. no `.` or `e`)31 /// How many bytes the token is.
23 number_is_integer: bool,32 count: usize,
24 /// How many bytes from the current position behind the start of this token is.33
25 count: usize,34 /// Whether string contains an escape sequence and cannot be zero-copied
2635 escapes: StringEscapes,
27 pub const Id = enum {36
28 ObjectBegin,37 pub fn decodedLength(self: @This()) usize {
29 ObjectEnd,38 return self.count +% switch (self.escapes) {
30 ArrayBegin,39 .None => 0,
31 ArrayEnd,40 .Some => |s| @bitCast(usize, s.size_diff),
32 String,41 };
33 Number,42 }
34 True,
35 False,
36 Null,
37 };
38
39 pub fn init(id: Id, count: usize, offset: u1) Token {
40 return Token{
41 .id = id,
42 .offset = offset,
43 .string_has_escape = false,
44 .number_is_integer = true,
45 .count = count,
46 };
47 }
48
49 pub fn initString(count: usize, has_unicode_escape: bool) Token {
50 return Token{
51 .id = Id.String,
52 .offset = 0,
53 .string_has_escape = has_unicode_escape,
54 .number_is_integer = true,
55 .count = count,
56 };
57 }
5843
59 pub fn initNumber(count: usize, number_is_integer: bool) Token {44 /// Slice into the underlying input string.
60 return Token{45 pub fn slice(self: @This(), input: []const u8, i: usize) []const u8 {
61 .id = Id.Number,46 return input[i - self.count .. i];
62 .offset = 0,47 }
63 .string_has_escape = false,48 },
64 .number_is_integer = number_is_integer,49 Number: struct {
65 .count = count,50 /// How many bytes the token is.
66 };51 count: usize,
67 }
6852
69 /// A marker token is a zero-length53 /// Whether number is simple and can be represented by an integer (i.e. no `.` or `e`)
70 pub fn initMarker(id: Id) Token {54 is_integer: bool,
71 return Token{
72 .id = id,
73 .offset = 0,
74 .string_has_escape = false,
75 .number_is_integer = true,
76 .count = 0,
77 };
78 }
7955
80 /// Slice into the underlying input string.56 /// Slice into the underlying input string.
81 pub fn slice(self: Token, input: []const u8, i: usize) []const u8 {57 pub fn slice(self: @This(), input: []const u8, i: usize) []const u8 {
82 return input[i + self.offset - self.count .. i + self.offset];58 return input[i - self.count .. i];
83 }59 }
60 },
61 True,
62 False,
63 Null,
84};64};
8565
86/// A small streaming JSON parser. This accepts input one byte at a time and returns tokens as66/// A small streaming JSON parser. This accepts input one byte at a time and returns tokens as
...@@ -102,7 +82,12 @@ pub const StreamingParser = struct {...@@ -102,7 +82,12 @@ pub const StreamingParser = struct {
102 // If we stopped now, would the complete parsed string to now be a valid json string82 // If we stopped now, would the complete parsed string to now be a valid json string
103 complete: bool,83 complete: bool,
104 // Current token flags to pass through to the next generated, see Token.84 // Current token flags to pass through to the next generated, see Token.
105 string_has_escape: bool,85 string_escapes: StringEscapes,
86 // When in .String states, was the previous character a high surrogate?
87 string_last_was_high_surrogate: bool,
88 // Used inside of StringEscapeHexUnicode* states
89 string_unicode_codepoint: u21,
90 // When in .Number states, is the number a (still) valid integer?
106 number_is_integer: bool,91 number_is_integer: bool,
10792
108 // Bit-stack for nested object/map literals (max 255 nestings).93 // Bit-stack for nested object/map literals (max 255 nestings).
...@@ -120,16 +105,18 @@ pub const StreamingParser = struct {...@@ -120,16 +105,18 @@ pub const StreamingParser = struct {
120 }105 }
121106
122 pub fn reset(p: *StreamingParser) void {107 pub fn reset(p: *StreamingParser) void {
123 p.state = State.TopLevelBegin;108 p.state = .TopLevelBegin;
124 p.count = 0;109 p.count = 0;
125 // Set before ever read in main transition function110 // Set before ever read in main transition function
126 p.after_string_state = undefined;111 p.after_string_state = undefined;
127 p.after_value_state = State.ValueEnd; // handle end of values normally112 p.after_value_state = .ValueEnd; // handle end of values normally
128 p.stack = 0;113 p.stack = 0;
129 p.stack_used = 0;114 p.stack_used = 0;
130 p.complete = false;115 p.complete = false;
131 p.string_has_escape = false;116 p.string_escapes = undefined;
132 p.number_is_integer = true;117 p.string_last_was_high_surrogate = undefined;
118 p.string_unicode_codepoint = undefined;
119 p.number_is_integer = undefined;
133 }120 }
134121
135 pub const State = enum {122 pub const State = enum {
...@@ -223,66 +210,67 @@ pub const StreamingParser = struct {...@@ -223,66 +210,67 @@ pub const StreamingParser = struct {
223 // Perform a single transition on the state machine and return any possible token.210 // Perform a single transition on the state machine and return any possible token.
224 fn transition(p: *StreamingParser, c: u8, token: *?Token) Error!bool {211 fn transition(p: *StreamingParser, c: u8, token: *?Token) Error!bool {
225 switch (p.state) {212 switch (p.state) {
226 State.TopLevelBegin => switch (c) {213 .TopLevelBegin => switch (c) {
227 '{' => {214 '{' => {
228 p.stack <<= 1;215 p.stack <<= 1;
229 p.stack |= object_bit;216 p.stack |= object_bit;
230 p.stack_used += 1;217 p.stack_used += 1;
231218
232 p.state = State.ValueBegin;219 p.state = .ValueBegin;
233 p.after_string_state = State.ObjectSeparator;220 p.after_string_state = .ObjectSeparator;
234221
235 token.* = Token.initMarker(Token.Id.ObjectBegin);222 token.* = Token.ObjectBegin;
236 },223 },
237 '[' => {224 '[' => {
238 p.stack <<= 1;225 p.stack <<= 1;
239 p.stack |= array_bit;226 p.stack |= array_bit;
240 p.stack_used += 1;227 p.stack_used += 1;
241228
242 p.state = State.ValueBegin;229 p.state = .ValueBegin;
243 p.after_string_state = State.ValueEnd;230 p.after_string_state = .ValueEnd;
244231
245 token.* = Token.initMarker(Token.Id.ArrayBegin);232 token.* = Token.ArrayBegin;
246 },233 },
247 '-' => {234 '-' => {
248 p.number_is_integer = true;235 p.number_is_integer = true;
249 p.state = State.Number;236 p.state = .Number;
250 p.after_value_state = State.TopLevelEnd;237 p.after_value_state = .TopLevelEnd;
251 p.count = 0;238 p.count = 0;
252 },239 },
253 '0' => {240 '0' => {
254 p.number_is_integer = true;241 p.number_is_integer = true;
255 p.state = State.NumberMaybeDotOrExponent;242 p.state = .NumberMaybeDotOrExponent;
256 p.after_value_state = State.TopLevelEnd;243 p.after_value_state = .TopLevelEnd;
257 p.count = 0;244 p.count = 0;
258 },245 },
259 '1'...'9' => {246 '1'...'9' => {
260 p.number_is_integer = true;247 p.number_is_integer = true;
261 p.state = State.NumberMaybeDigitOrDotOrExponent;248 p.state = .NumberMaybeDigitOrDotOrExponent;
262 p.after_value_state = State.TopLevelEnd;249 p.after_value_state = .TopLevelEnd;
263 p.count = 0;250 p.count = 0;
264 },251 },
265 '"' => {252 '"' => {
266 p.state = State.String;253 p.state = .String;
267 p.after_value_state = State.TopLevelEnd;254 p.after_value_state = .TopLevelEnd;
268 // We don't actually need the following since after_value_state should override.255 // We don't actually need the following since after_value_state should override.
269 p.after_string_state = State.ValueEnd;256 p.after_string_state = .ValueEnd;
270 p.string_has_escape = false;257 p.string_escapes = .None;
258 p.string_last_was_high_surrogate = false;
271 p.count = 0;259 p.count = 0;
272 },260 },
273 't' => {261 't' => {
274 p.state = State.TrueLiteral1;262 p.state = .TrueLiteral1;
275 p.after_value_state = State.TopLevelEnd;263 p.after_value_state = .TopLevelEnd;
276 p.count = 0;264 p.count = 0;
277 },265 },
278 'f' => {266 'f' => {
279 p.state = State.FalseLiteral1;267 p.state = .FalseLiteral1;
280 p.after_value_state = State.TopLevelEnd;268 p.after_value_state = .TopLevelEnd;
281 p.count = 0;269 p.count = 0;
282 },270 },
283 'n' => {271 'n' => {
284 p.state = State.NullLiteral1;272 p.state = .NullLiteral1;
285 p.after_value_state = State.TopLevelEnd;273 p.after_value_state = .TopLevelEnd;
286 p.count = 0;274 p.count = 0;
287 },275 },
288 0x09, 0x0A, 0x0D, 0x20 => {276 0x09, 0x0A, 0x0D, 0x20 => {
...@@ -293,7 +281,7 @@ pub const StreamingParser = struct {...@@ -293,7 +281,7 @@ pub const StreamingParser = struct {
293 },281 },
294 },282 },
295283
296 State.TopLevelEnd => switch (c) {284 .TopLevelEnd => switch (c) {
297 0x09, 0x0A, 0x0D, 0x20 => {285 0x09, 0x0A, 0x0D, 0x20 => {
298 // whitespace286 // whitespace
299 },287 },
...@@ -302,7 +290,7 @@ pub const StreamingParser = struct {...@@ -302,7 +290,7 @@ pub const StreamingParser = struct {
302 },290 },
303 },291 },
304292
305 State.ValueBegin => switch (c) {293 .ValueBegin => switch (c) {
306 // NOTE: These are shared in ValueEnd as well, think we can reorder states to294 // NOTE: These are shared in ValueEnd as well, think we can reorder states to
307 // be a bit clearer and avoid this duplication.295 // be a bit clearer and avoid this duplication.
308 '}' => {296 '}' => {
...@@ -314,7 +302,7 @@ pub const StreamingParser = struct {...@@ -314,7 +302,7 @@ pub const StreamingParser = struct {
314 return error.TooManyClosingItems;302 return error.TooManyClosingItems;
315 }303 }
316304
317 p.state = State.ValueBegin;305 p.state = .ValueBegin;
318 p.after_string_state = State.fromInt(p.stack & 1);306 p.after_string_state = State.fromInt(p.stack & 1);
319307
320 p.stack >>= 1;308 p.stack >>= 1;
...@@ -323,14 +311,14 @@ pub const StreamingParser = struct {...@@ -323,14 +311,14 @@ pub const StreamingParser = struct {
323 switch (p.stack_used) {311 switch (p.stack_used) {
324 0 => {312 0 => {
325 p.complete = true;313 p.complete = true;
326 p.state = State.TopLevelEnd;314 p.state = .TopLevelEnd;
327 },315 },
328 else => {316 else => {
329 p.state = State.ValueEnd;317 p.state = .ValueEnd;
330 },318 },
331 }319 }
332320
333 token.* = Token.initMarker(Token.Id.ObjectEnd);321 token.* = Token.ObjectEnd;
334 },322 },
335 ']' => {323 ']' => {
336 if (p.stack & 1 != array_bit) {324 if (p.stack & 1 != array_bit) {
...@@ -340,7 +328,7 @@ pub const StreamingParser = struct {...@@ -340,7 +328,7 @@ pub const StreamingParser = struct {
340 return error.TooManyClosingItems;328 return error.TooManyClosingItems;
341 }329 }
342330
343 p.state = State.ValueBegin;331 p.state = .ValueBegin;
344 p.after_string_state = State.fromInt(p.stack & 1);332 p.after_string_state = State.fromInt(p.stack & 1);
345333
346 p.stack >>= 1;334 p.stack >>= 1;
...@@ -349,14 +337,14 @@ pub const StreamingParser = struct {...@@ -349,14 +337,14 @@ pub const StreamingParser = struct {
349 switch (p.stack_used) {337 switch (p.stack_used) {
350 0 => {338 0 => {
351 p.complete = true;339 p.complete = true;
352 p.state = State.TopLevelEnd;340 p.state = .TopLevelEnd;
353 },341 },
354 else => {342 else => {
355 p.state = State.ValueEnd;343 p.state = .ValueEnd;
356 },344 },
357 }345 }
358346
359 token.* = Token.initMarker(Token.Id.ArrayEnd);347 token.* = Token.ArrayEnd;
360 },348 },
361 '{' => {349 '{' => {
362 if (p.stack_used == max_stack_size) {350 if (p.stack_used == max_stack_size) {
...@@ -367,10 +355,10 @@ pub const StreamingParser = struct {...@@ -367,10 +355,10 @@ pub const StreamingParser = struct {
367 p.stack |= object_bit;355 p.stack |= object_bit;
368 p.stack_used += 1;356 p.stack_used += 1;
369357
370 p.state = State.ValueBegin;358 p.state = .ValueBegin;
371 p.after_string_state = State.ObjectSeparator;359 p.after_string_state = .ObjectSeparator;
372360
373 token.* = Token.initMarker(Token.Id.ObjectBegin);361 token.* = Token.ObjectBegin;
374 },362 },
375 '[' => {363 '[' => {
376 if (p.stack_used == max_stack_size) {364 if (p.stack_used == max_stack_size) {
...@@ -381,40 +369,42 @@ pub const StreamingParser = struct {...@@ -381,40 +369,42 @@ pub const StreamingParser = struct {
381 p.stack |= array_bit;369 p.stack |= array_bit;
382 p.stack_used += 1;370 p.stack_used += 1;
383371
384 p.state = State.ValueBegin;372 p.state = .ValueBegin;
385 p.after_string_state = State.ValueEnd;373 p.after_string_state = .ValueEnd;
386374
387 token.* = Token.initMarker(Token.Id.ArrayBegin);375 token.* = Token.ArrayBegin;
388 },376 },
389 '-' => {377 '-' => {
390 p.number_is_integer = true;378 p.number_is_integer = true;
391 p.state = State.Number;379 p.state = .Number;
392 p.count = 0;380 p.count = 0;
393 },381 },
394 '0' => {382 '0' => {
395 p.number_is_integer = true;383 p.number_is_integer = true;
396 p.state = State.NumberMaybeDotOrExponent;384 p.state = .NumberMaybeDotOrExponent;
397 p.count = 0;385 p.count = 0;
398 },386 },
399 '1'...'9' => {387 '1'...'9' => {
400 p.number_is_integer = true;388 p.number_is_integer = true;
401 p.state = State.NumberMaybeDigitOrDotOrExponent;389 p.state = .NumberMaybeDigitOrDotOrExponent;
402 p.count = 0;390 p.count = 0;
403 },391 },
404 '"' => {392 '"' => {
405 p.state = State.String;393 p.state = .String;
394 p.string_escapes = .None;
395 p.string_last_was_high_surrogate = false;
406 p.count = 0;396 p.count = 0;
407 },397 },
408 't' => {398 't' => {
409 p.state = State.TrueLiteral1;399 p.state = .TrueLiteral1;
410 p.count = 0;400 p.count = 0;
411 },401 },
412 'f' => {402 'f' => {
413 p.state = State.FalseLiteral1;403 p.state = .FalseLiteral1;
414 p.count = 0;404 p.count = 0;
415 },405 },
416 'n' => {406 'n' => {
417 p.state = State.NullLiteral1;407 p.state = .NullLiteral1;
418 p.count = 0;408 p.count = 0;
419 },409 },
420 0x09, 0x0A, 0x0D, 0x20 => {410 0x09, 0x0A, 0x0D, 0x20 => {
...@@ -426,7 +416,7 @@ pub const StreamingParser = struct {...@@ -426,7 +416,7 @@ pub const StreamingParser = struct {
426 },416 },
427417
428 // TODO: A bit of duplication here and in the following state, redo.418 // TODO: A bit of duplication here and in the following state, redo.
429 State.ValueBeginNoClosing => switch (c) {419 .ValueBeginNoClosing => switch (c) {
430 '{' => {420 '{' => {
431 if (p.stack_used == max_stack_size) {421 if (p.stack_used == max_stack_size) {
432 return error.TooManyNestedItems;422 return error.TooManyNestedItems;
...@@ -436,10 +426,10 @@ pub const StreamingParser = struct {...@@ -436,10 +426,10 @@ pub const StreamingParser = struct {
436 p.stack |= object_bit;426 p.stack |= object_bit;
437 p.stack_used += 1;427 p.stack_used += 1;
438428
439 p.state = State.ValueBegin;429 p.state = .ValueBegin;
440 p.after_string_state = State.ObjectSeparator;430 p.after_string_state = .ObjectSeparator;
441431
442 token.* = Token.initMarker(Token.Id.ObjectBegin);432 token.* = Token.ObjectBegin;
443 },433 },
444 '[' => {434 '[' => {
445 if (p.stack_used == max_stack_size) {435 if (p.stack_used == max_stack_size) {
...@@ -450,40 +440,42 @@ pub const StreamingParser = struct {...@@ -450,40 +440,42 @@ pub const StreamingParser = struct {
450 p.stack |= array_bit;440 p.stack |= array_bit;
451 p.stack_used += 1;441 p.stack_used += 1;
452442
453 p.state = State.ValueBegin;443 p.state = .ValueBegin;
454 p.after_string_state = State.ValueEnd;444 p.after_string_state = .ValueEnd;
455445
456 token.* = Token.initMarker(Token.Id.ArrayBegin);446 token.* = Token.ArrayBegin;
457 },447 },
458 '-' => {448 '-' => {
459 p.number_is_integer = true;449 p.number_is_integer = true;
460 p.state = State.Number;450 p.state = .Number;
461 p.count = 0;451 p.count = 0;
462 },452 },
463 '0' => {453 '0' => {
464 p.number_is_integer = true;454 p.number_is_integer = true;
465 p.state = State.NumberMaybeDotOrExponent;455 p.state = .NumberMaybeDotOrExponent;
466 p.count = 0;456 p.count = 0;
467 },457 },
468 '1'...'9' => {458 '1'...'9' => {
469 p.number_is_integer = true;459 p.number_is_integer = true;
470 p.state = State.NumberMaybeDigitOrDotOrExponent;460 p.state = .NumberMaybeDigitOrDotOrExponent;
471 p.count = 0;461 p.count = 0;
472 },462 },
473 '"' => {463 '"' => {
474 p.state = State.String;464 p.state = .String;
465 p.string_escapes = .None;
466 p.string_last_was_high_surrogate = false;
475 p.count = 0;467 p.count = 0;
476 },468 },
477 't' => {469 't' => {
478 p.state = State.TrueLiteral1;470 p.state = .TrueLiteral1;
479 p.count = 0;471 p.count = 0;
480 },472 },
481 'f' => {473 'f' => {
482 p.state = State.FalseLiteral1;474 p.state = .FalseLiteral1;
483 p.count = 0;475 p.count = 0;
484 },476 },
485 'n' => {477 'n' => {
486 p.state = State.NullLiteral1;478 p.state = .NullLiteral1;
487 p.count = 0;479 p.count = 0;
488 },480 },
489 0x09, 0x0A, 0x0D, 0x20 => {481 0x09, 0x0A, 0x0D, 0x20 => {
...@@ -494,17 +486,17 @@ pub const StreamingParser = struct {...@@ -494,17 +486,17 @@ pub const StreamingParser = struct {
494 },486 },
495 },487 },
496488
497 State.ValueEnd => switch (c) {489 .ValueEnd => switch (c) {
498 ',' => {490 ',' => {
499 p.after_string_state = State.fromInt(p.stack & 1);491 p.after_string_state = State.fromInt(p.stack & 1);
500 p.state = State.ValueBeginNoClosing;492 p.state = .ValueBeginNoClosing;
501 },493 },
502 ']' => {494 ']' => {
503 if (p.stack_used == 0) {495 if (p.stack_used == 0) {
504 return error.UnbalancedBrackets;496 return error.UnbalancedBrackets;
505 }497 }
506498
507 p.state = State.ValueEnd;499 p.state = .ValueEnd;
508 p.after_string_state = State.fromInt(p.stack & 1);500 p.after_string_state = State.fromInt(p.stack & 1);
509501
510 p.stack >>= 1;502 p.stack >>= 1;
...@@ -512,17 +504,17 @@ pub const StreamingParser = struct {...@@ -512,17 +504,17 @@ pub const StreamingParser = struct {
512504
513 if (p.stack_used == 0) {505 if (p.stack_used == 0) {
514 p.complete = true;506 p.complete = true;
515 p.state = State.TopLevelEnd;507 p.state = .TopLevelEnd;
516 }508 }
517509
518 token.* = Token.initMarker(Token.Id.ArrayEnd);510 token.* = Token.ArrayEnd;
519 },511 },
520 '}' => {512 '}' => {
521 if (p.stack_used == 0) {513 if (p.stack_used == 0) {
522 return error.UnbalancedBraces;514 return error.UnbalancedBraces;
523 }515 }
524516
525 p.state = State.ValueEnd;517 p.state = .ValueEnd;
526 p.after_string_state = State.fromInt(p.stack & 1);518 p.after_string_state = State.fromInt(p.stack & 1);
527519
528 p.stack >>= 1;520 p.stack >>= 1;
...@@ -530,10 +522,10 @@ pub const StreamingParser = struct {...@@ -530,10 +522,10 @@ pub const StreamingParser = struct {
530522
531 if (p.stack_used == 0) {523 if (p.stack_used == 0) {
532 p.complete = true;524 p.complete = true;
533 p.state = State.TopLevelEnd;525 p.state = .TopLevelEnd;
534 }526 }
535527
536 token.* = Token.initMarker(Token.Id.ObjectEnd);528 token.* = Token.ObjectEnd;
537 },529 },
538 0x09, 0x0A, 0x0D, 0x20 => {530 0x09, 0x0A, 0x0D, 0x20 => {
539 // whitespace531 // whitespace
...@@ -543,10 +535,10 @@ pub const StreamingParser = struct {...@@ -543,10 +535,10 @@ pub const StreamingParser = struct {
543 },535 },
544 },536 },
545537
546 State.ObjectSeparator => switch (c) {538 .ObjectSeparator => switch (c) {
547 ':' => {539 ':' => {
548 p.state = State.ValueBegin;540 p.state = .ValueBegin;
549 p.after_string_state = State.ValueEnd;541 p.after_string_state = .ValueEnd;
550 },542 },
551 0x09, 0x0A, 0x0D, 0x20 => {543 0x09, 0x0A, 0x0D, 0x20 => {
552 // whitespace544 // whitespace
...@@ -556,55 +548,72 @@ pub const StreamingParser = struct {...@@ -556,55 +548,72 @@ pub const StreamingParser = struct {
556 },548 },
557 },549 },
558550
559 State.String => switch (c) {551 .String => switch (c) {
560 0x00...0x1F => {552 0x00...0x1F => {
561 return error.InvalidControlCharacter;553 return error.InvalidControlCharacter;
562 },554 },
563 '"' => {555 '"' => {
564 p.state = p.after_string_state;556 p.state = p.after_string_state;
565 if (p.after_value_state == State.TopLevelEnd) {557 if (p.after_value_state == .TopLevelEnd) {
566 p.state = State.TopLevelEnd;558 p.state = .TopLevelEnd;
567 p.complete = true;559 p.complete = true;
568 }560 }
569561
570 token.* = Token.initString(p.count - 1, p.string_has_escape);562 token.* = .{
563 .String = .{
564 .count = p.count - 1,
565 .escapes = p.string_escapes,
566 },
567 };
568 p.string_escapes = undefined;
569 p.string_last_was_high_surrogate = undefined;
571 },570 },
572 '\\' => {571 '\\' => {
573 p.state = State.StringEscapeCharacter;572 p.state = .StringEscapeCharacter;
573 switch (p.string_escapes) {
574 .None => {
575 p.string_escapes = .{ .Some = .{ .size_diff = 0 } };
576 },
577 .Some => {},
578 }
574 },579 },
575 0x20, 0x21, 0x23...0x5B, 0x5D...0x7F => {580 0x20, 0x21, 0x23...0x5B, 0x5D...0x7F => {
576 // non-control ascii581 // non-control ascii
582 p.string_last_was_high_surrogate = false;
577 },583 },
578 0xC0...0xDF => {584 0xC0...0xDF => {
579 p.state = State.StringUtf8Byte1;585 p.state = .StringUtf8Byte1;
580 },586 },
581 0xE0...0xEF => {587 0xE0...0xEF => {
582 p.state = State.StringUtf8Byte2;588 p.state = .StringUtf8Byte2;
583 },589 },
584 0xF0...0xFF => {590 0xF0...0xFF => {
585 p.state = State.StringUtf8Byte3;591 p.state = .StringUtf8Byte3;
586 },592 },
587 else => {593 else => {
588 return error.InvalidUtf8Byte;594 return error.InvalidUtf8Byte;
589 },595 },
590 },596 },
591597
592 State.StringUtf8Byte3 => switch (c >> 6) {598 .StringUtf8Byte3 => switch (c >> 6) {
593 0b10 => p.state = State.StringUtf8Byte2,599 0b10 => p.state = .StringUtf8Byte2,
594 else => return error.InvalidUtf8Byte,600 else => return error.InvalidUtf8Byte,
595 },601 },
596602
597 State.StringUtf8Byte2 => switch (c >> 6) {603 .StringUtf8Byte2 => switch (c >> 6) {
598 0b10 => p.state = State.StringUtf8Byte1,604 0b10 => p.state = .StringUtf8Byte1,
599 else => return error.InvalidUtf8Byte,605 else => return error.InvalidUtf8Byte,
600 },606 },
601607
602 State.StringUtf8Byte1 => switch (c >> 6) {608 .StringUtf8Byte1 => switch (c >> 6) {
603 0b10 => p.state = State.String,609 0b10 => {
610 p.state = .String;
611 p.string_last_was_high_surrogate = false;
612 },
604 else => return error.InvalidUtf8Byte,613 else => return error.InvalidUtf8Byte,
605 },614 },
606615
607 State.StringEscapeCharacter => switch (c) {616 .StringEscapeCharacter => switch (c) {
608 // NOTE: '/' is allowed as an escaped character but it also is allowed617 // NOTE: '/' is allowed as an escaped character but it also is allowed
609 // as unescaped according to the RFC. There is a reported errata which suggests618 // as unescaped according to the RFC. There is a reported errata which suggests
610 // removing the non-escaped variant but it makes more sense to simply disallow619 // removing the non-escaped variant but it makes more sense to simply disallow
...@@ -614,54 +623,121 @@ pub const StreamingParser = struct {...@@ -614,54 +623,121 @@ pub const StreamingParser = struct {
614 // however, so we default to the status quo where both are accepted until this623 // however, so we default to the status quo where both are accepted until this
615 // is further clarified.624 // is further clarified.
616 '"', '\\', '/', 'b', 'f', 'n', 'r', 't' => {625 '"', '\\', '/', 'b', 'f', 'n', 'r', 't' => {
617 p.string_has_escape = true;626 p.string_escapes.Some.size_diff -= 1;
618 p.state = State.String;627 p.state = .String;
628 p.string_last_was_high_surrogate = false;
619 },629 },
620 'u' => {630 'u' => {
621 p.string_has_escape = true;631 p.state = .StringEscapeHexUnicode4;
622 p.state = State.StringEscapeHexUnicode4;
623 },632 },
624 else => {633 else => {
625 return error.InvalidEscapeCharacter;634 return error.InvalidEscapeCharacter;
626 },635 },
627 },636 },
628637
629 State.StringEscapeHexUnicode4 => switch (c) {638 .StringEscapeHexUnicode4 => {
630 '0'...'9', 'A'...'F', 'a'...'f' => {639 var codepoint: u21 = undefined;
631 p.state = State.StringEscapeHexUnicode3;640 switch (c) {
632 },641 else => return error.InvalidUnicodeHexSymbol,
633 else => return error.InvalidUnicodeHexSymbol,642 '0'...'9' => {
643 codepoint = c - '0';
644 },
645 'A'...'F' => {
646 codepoint = c - 'A' + 10;
647 },
648 'a'...'f' => {
649 codepoint = c - 'a' + 10;
650 },
651 }
652 p.state = .StringEscapeHexUnicode3;
653 p.string_unicode_codepoint = codepoint << 12;
634 },654 },
635655
636 State.StringEscapeHexUnicode3 => switch (c) {656 .StringEscapeHexUnicode3 => {
637 '0'...'9', 'A'...'F', 'a'...'f' => {657 var codepoint: u21 = undefined;
638 p.state = State.StringEscapeHexUnicode2;658 switch (c) {
639 },659 else => return error.InvalidUnicodeHexSymbol,
640 else => return error.InvalidUnicodeHexSymbol,660 '0'...'9' => {
661 codepoint = c - '0';
662 },
663 'A'...'F' => {
664 codepoint = c - 'A' + 10;
665 },
666 'a'...'f' => {
667 codepoint = c - 'a' + 10;
668 },
669 }
670 p.state = .StringEscapeHexUnicode2;
671 p.string_unicode_codepoint |= codepoint << 8;
641 },672 },
642673
643 State.StringEscapeHexUnicode2 => switch (c) {674 .StringEscapeHexUnicode2 => {
644 '0'...'9', 'A'...'F', 'a'...'f' => {675 var codepoint: u21 = undefined;
645 p.state = State.StringEscapeHexUnicode1;676 switch (c) {
646 },677 else => return error.InvalidUnicodeHexSymbol,
647 else => return error.InvalidUnicodeHexSymbol,678 '0'...'9' => {
679 codepoint = c - '0';
680 },
681 'A'...'F' => {
682 codepoint = c - 'A' + 10;
683 },
684 'a'...'f' => {
685 codepoint = c - 'a' + 10;
686 },
687 }
688 p.state = .StringEscapeHexUnicode1;
689 p.string_unicode_codepoint |= codepoint << 4;
648 },690 },
649691
650 State.StringEscapeHexUnicode1 => switch (c) {692 .StringEscapeHexUnicode1 => {
651 '0'...'9', 'A'...'F', 'a'...'f' => {693 var codepoint: u21 = undefined;
652 p.state = State.String;694 switch (c) {
653 },695 else => return error.InvalidUnicodeHexSymbol,
654 else => return error.InvalidUnicodeHexSymbol,696 '0'...'9' => {
697 codepoint = c - '0';
698 },
699 'A'...'F' => {
700 codepoint = c - 'A' + 10;
701 },
702 'a'...'f' => {
703 codepoint = c - 'a' + 10;
704 },
705 }
706 p.state = .String;
707 p.string_unicode_codepoint |= codepoint;
708 if (p.string_unicode_codepoint < 0xD800 or p.string_unicode_codepoint >= 0xE000) {
709 // not part of surrogate pair
710 p.string_escapes.Some.size_diff -= @as(isize, 6 - (std.unicode.utf8CodepointSequenceLength(p.string_unicode_codepoint) catch unreachable));
711 p.string_last_was_high_surrogate = false;
712 } else if (p.string_unicode_codepoint < 0xDC00) {
713 // 'high' surrogate
714 // takes 3 bytes to encode a half surrogate pair into wtf8
715 p.string_escapes.Some.size_diff -= 6 - 3;
716 p.string_last_was_high_surrogate = true;
717 } else {
718 // 'low' surrogate
719 p.string_escapes.Some.size_diff -= 6;
720 if (p.string_last_was_high_surrogate) {
721 // takes 4 bytes to encode a full surrogate pair into utf8
722 // 3 bytes are already reserved by high surrogate
723 p.string_escapes.Some.size_diff -= -1;
724 } else {
725 // takes 3 bytes to encode a half surrogate pair into wtf8
726 p.string_escapes.Some.size_diff -= -3;
727 }
728 p.string_last_was_high_surrogate = false;
729 }
730 p.string_unicode_codepoint = undefined;
655 },731 },
656732
657 State.Number => {733 .Number => {
658 p.complete = p.after_value_state == State.TopLevelEnd;734 p.complete = p.after_value_state == .TopLevelEnd;
659 switch (c) {735 switch (c) {
660 '0' => {736 '0' => {
661 p.state = State.NumberMaybeDotOrExponent;737 p.state = .NumberMaybeDotOrExponent;
662 },738 },
663 '1'...'9' => {739 '1'...'9' => {
664 p.state = State.NumberMaybeDigitOrDotOrExponent;740 p.state = .NumberMaybeDigitOrDotOrExponent;
665 },741 },
666 else => {742 else => {
667 return error.InvalidNumber;743 return error.InvalidNumber;
...@@ -669,52 +745,63 @@ pub const StreamingParser = struct {...@@ -669,52 +745,63 @@ pub const StreamingParser = struct {
669 }745 }
670 },746 },
671747
672 State.NumberMaybeDotOrExponent => {748 .NumberMaybeDotOrExponent => {
673 p.complete = p.after_value_state == State.TopLevelEnd;749 p.complete = p.after_value_state == .TopLevelEnd;
674 switch (c) {750 switch (c) {
675 '.' => {751 '.' => {
676 p.number_is_integer = false;752 p.number_is_integer = false;
677 p.state = State.NumberFractionalRequired;753 p.state = .NumberFractionalRequired;
678 },754 },
679 'e', 'E' => {755 'e', 'E' => {
680 p.number_is_integer = false;756 p.number_is_integer = false;
681 p.state = State.NumberExponent;757 p.state = .NumberExponent;
682 },758 },
683 else => {759 else => {
684 p.state = p.after_value_state;760 p.state = p.after_value_state;
685 token.* = Token.initNumber(p.count, p.number_is_integer);761 token.* = .{
762 .Number = .{
763 .count = p.count,
764 .is_integer = p.number_is_integer,
765 },
766 };
767 p.number_is_integer = undefined;
686 return true;768 return true;
687 },769 },
688 }770 }
689 },771 },
690772
691 State.NumberMaybeDigitOrDotOrExponent => {773 .NumberMaybeDigitOrDotOrExponent => {
692 p.complete = p.after_value_state == State.TopLevelEnd;774 p.complete = p.after_value_state == .TopLevelEnd;
693 switch (c) {775 switch (c) {
694 '.' => {776 '.' => {
695 p.number_is_integer = false;777 p.number_is_integer = false;
696 p.state = State.NumberFractionalRequired;778 p.state = .NumberFractionalRequired;
697 },779 },
698 'e', 'E' => {780 'e', 'E' => {
699 p.number_is_integer = false;781 p.number_is_integer = false;
700 p.state = State.NumberExponent;782 p.state = .NumberExponent;
701 },783 },
702 '0'...'9' => {784 '0'...'9' => {
703 // another digit785 // another digit
704 },786 },
705 else => {787 else => {
706 p.state = p.after_value_state;788 p.state = p.after_value_state;
707 token.* = Token.initNumber(p.count, p.number_is_integer);789 token.* = .{
790 .Number = .{
791 .count = p.count,
792 .is_integer = p.number_is_integer,
793 },
794 };
708 return true;795 return true;
709 },796 },
710 }797 }
711 },798 },
712799
713 State.NumberFractionalRequired => {800 .NumberFractionalRequired => {
714 p.complete = p.after_value_state == State.TopLevelEnd;801 p.complete = p.after_value_state == .TopLevelEnd;
715 switch (c) {802 switch (c) {
716 '0'...'9' => {803 '0'...'9' => {
717 p.state = State.NumberFractional;804 p.state = .NumberFractional;
718 },805 },
719 else => {806 else => {
720 return error.InvalidNumber;807 return error.InvalidNumber;
...@@ -722,139 +809,154 @@ pub const StreamingParser = struct {...@@ -722,139 +809,154 @@ pub const StreamingParser = struct {
722 }809 }
723 },810 },
724811
725 State.NumberFractional => {812 .NumberFractional => {
726 p.complete = p.after_value_state == State.TopLevelEnd;813 p.complete = p.after_value_state == .TopLevelEnd;
727 switch (c) {814 switch (c) {
728 '0'...'9' => {815 '0'...'9' => {
729 // another digit816 // another digit
730 },817 },
731 'e', 'E' => {818 'e', 'E' => {
732 p.number_is_integer = false;819 p.number_is_integer = false;
733 p.state = State.NumberExponent;820 p.state = .NumberExponent;
734 },821 },
735 else => {822 else => {
736 p.state = p.after_value_state;823 p.state = p.after_value_state;
737 token.* = Token.initNumber(p.count, p.number_is_integer);824 token.* = .{
825 .Number = .{
826 .count = p.count,
827 .is_integer = p.number_is_integer,
828 },
829 };
738 return true;830 return true;
739 },831 },
740 }832 }
741 },833 },
742834
743 State.NumberMaybeExponent => {835 .NumberMaybeExponent => {
744 p.complete = p.after_value_state == State.TopLevelEnd;836 p.complete = p.after_value_state == .TopLevelEnd;
745 switch (c) {837 switch (c) {
746 'e', 'E' => {838 'e', 'E' => {
747 p.number_is_integer = false;839 p.number_is_integer = false;
748 p.state = State.NumberExponent;840 p.state = .NumberExponent;
749 },841 },
750 else => {842 else => {
751 p.state = p.after_value_state;843 p.state = p.after_value_state;
752 token.* = Token.initNumber(p.count, p.number_is_integer);844 token.* = .{
845 .Number = .{
846 .count = p.count,
847 .is_integer = p.number_is_integer,
848 },
849 };
753 return true;850 return true;
754 },851 },
755 }852 }
756 },853 },
757854
758 State.NumberExponent => switch (c) {855 .NumberExponent => switch (c) {
759 '-', '+' => {856 '-', '+' => {
760 p.complete = false;857 p.complete = false;
761 p.state = State.NumberExponentDigitsRequired;858 p.state = .NumberExponentDigitsRequired;
762 },859 },
763 '0'...'9' => {860 '0'...'9' => {
764 p.complete = p.after_value_state == State.TopLevelEnd;861 p.complete = p.after_value_state == .TopLevelEnd;
765 p.state = State.NumberExponentDigits;862 p.state = .NumberExponentDigits;
766 },863 },
767 else => {864 else => {
768 return error.InvalidNumber;865 return error.InvalidNumber;
769 },866 },
770 },867 },
771868
772 State.NumberExponentDigitsRequired => switch (c) {869 .NumberExponentDigitsRequired => switch (c) {
773 '0'...'9' => {870 '0'...'9' => {
774 p.complete = p.after_value_state == State.TopLevelEnd;871 p.complete = p.after_value_state == .TopLevelEnd;
775 p.state = State.NumberExponentDigits;872 p.state = .NumberExponentDigits;
776 },873 },
777 else => {874 else => {
778 return error.InvalidNumber;875 return error.InvalidNumber;
779 },876 },
780 },877 },
781878
782 State.NumberExponentDigits => {879 .NumberExponentDigits => {
783 p.complete = p.after_value_state == State.TopLevelEnd;880 p.complete = p.after_value_state == .TopLevelEnd;
784 switch (c) {881 switch (c) {
785 '0'...'9' => {882 '0'...'9' => {
786 // another digit883 // another digit
787 },884 },
788 else => {885 else => {
789 p.state = p.after_value_state;886 p.state = p.after_value_state;
790 token.* = Token.initNumber(p.count, p.number_is_integer);887 token.* = .{
888 .Number = .{
889 .count = p.count,
890 .is_integer = p.number_is_integer,
891 },
892 };
791 return true;893 return true;
792 },894 },
793 }895 }
794 },896 },
795897
796 State.TrueLiteral1 => switch (c) {898 .TrueLiteral1 => switch (c) {
797 'r' => p.state = State.TrueLiteral2,899 'r' => p.state = .TrueLiteral2,
798 else => return error.InvalidLiteral,900 else => return error.InvalidLiteral,
799 },901 },
800902
801 State.TrueLiteral2 => switch (c) {903 .TrueLiteral2 => switch (c) {
802 'u' => p.state = State.TrueLiteral3,904 'u' => p.state = .TrueLiteral3,
803 else => return error.InvalidLiteral,905 else => return error.InvalidLiteral,
804 },906 },
805907
806 State.TrueLiteral3 => switch (c) {908 .TrueLiteral3 => switch (c) {
807 'e' => {909 'e' => {
808 p.state = p.after_value_state;910 p.state = p.after_value_state;
809 p.complete = p.state == State.TopLevelEnd;911 p.complete = p.state == .TopLevelEnd;
810 token.* = Token.init(Token.Id.True, p.count + 1, 1);912 token.* = Token.True;
811 },913 },
812 else => {914 else => {
813 return error.InvalidLiteral;915 return error.InvalidLiteral;
814 },916 },
815 },917 },
816918
817 State.FalseLiteral1 => switch (c) {919 .FalseLiteral1 => switch (c) {
818 'a' => p.state = State.FalseLiteral2,920 'a' => p.state = .FalseLiteral2,
819 else => return error.InvalidLiteral,921 else => return error.InvalidLiteral,
820 },922 },
821923
822 State.FalseLiteral2 => switch (c) {924 .FalseLiteral2 => switch (c) {
823 'l' => p.state = State.FalseLiteral3,925 'l' => p.state = .FalseLiteral3,
824 else => return error.InvalidLiteral,926 else => return error.InvalidLiteral,
825 },927 },
826928
827 State.FalseLiteral3 => switch (c) {929 .FalseLiteral3 => switch (c) {
828 's' => p.state = State.FalseLiteral4,930 's' => p.state = .FalseLiteral4,
829 else => return error.InvalidLiteral,931 else => return error.InvalidLiteral,
830 },932 },
831933
832 State.FalseLiteral4 => switch (c) {934 .FalseLiteral4 => switch (c) {
833 'e' => {935 'e' => {
834 p.state = p.after_value_state;936 p.state = p.after_value_state;
835 p.complete = p.state == State.TopLevelEnd;937 p.complete = p.state == .TopLevelEnd;
836 token.* = Token.init(Token.Id.False, p.count + 1, 1);938 token.* = Token.False;
837 },939 },
838 else => {940 else => {
839 return error.InvalidLiteral;941 return error.InvalidLiteral;
840 },942 },
841 },943 },
842944
843 State.NullLiteral1 => switch (c) {945 .NullLiteral1 => switch (c) {
844 'u' => p.state = State.NullLiteral2,946 'u' => p.state = .NullLiteral2,
845 else => return error.InvalidLiteral,947 else => return error.InvalidLiteral,
846 },948 },
847949
848 State.NullLiteral2 => switch (c) {950 .NullLiteral2 => switch (c) {
849 'l' => p.state = State.NullLiteral3,951 'l' => p.state = .NullLiteral3,
850 else => return error.InvalidLiteral,952 else => return error.InvalidLiteral,
851 },953 },
852954
853 State.NullLiteral3 => switch (c) {955 .NullLiteral3 => switch (c) {
854 'l' => {956 'l' => {
855 p.state = p.after_value_state;957 p.state = p.after_value_state;
856 p.complete = p.state == State.TopLevelEnd;958 p.complete = p.state == .TopLevelEnd;
857 token.* = Token.init(Token.Id.Null, p.count + 1, 1);959 token.* = Token.Null;
858 },960 },
859 else => {961 else => {
860 return error.InvalidLiteral;962 return error.InvalidLiteral;
...@@ -905,7 +1007,7 @@ pub const TokenStream = struct {...@@ -905,7 +1007,7 @@ pub const TokenStream = struct {
905 }1007 }
906 }1008 }
9071009
908 // Without this a bare number fails, becasue the streaming parser doesn't know it ended1010 // Without this a bare number fails, the streaming parser doesn't know the input ended
909 try self.parser.feed(' ', &t1, &t2);1011 try self.parser.feed(' ', &t1, &t2);
910 self.i += 1;1012 self.i += 1;
9111013
...@@ -919,9 +1021,9 @@ pub const TokenStream = struct {...@@ -919,9 +1021,9 @@ pub const TokenStream = struct {
919 }1021 }
920};1022};
9211023
922fn checkNext(p: *TokenStream, id: Token.Id) void {1024fn checkNext(p: *TokenStream, id: std.meta.TagType(Token)) void {
923 const token = (p.next() catch unreachable).?;1025 const token = (p.next() catch unreachable).?;
924 debug.assert(token.id == id);1026 debug.assert(std.meta.activeTag(token) == id);
925}1027}
9261028
927test "json.token" {1029test "json.token" {
...@@ -944,35 +1046,35 @@ test "json.token" {...@@ -944,35 +1046,35 @@ test "json.token" {
9441046
945 var p = TokenStream.init(s);1047 var p = TokenStream.init(s);
9461048
947 checkNext(&p, Token.Id.ObjectBegin);1049 checkNext(&p, .ObjectBegin);
948 checkNext(&p, Token.Id.String); // Image1050 checkNext(&p, .String); // Image
949 checkNext(&p, Token.Id.ObjectBegin);1051 checkNext(&p, .ObjectBegin);
950 checkNext(&p, Token.Id.String); // Width1052 checkNext(&p, .String); // Width
951 checkNext(&p, Token.Id.Number);1053 checkNext(&p, .Number);
952 checkNext(&p, Token.Id.String); // Height1054 checkNext(&p, .String); // Height
953 checkNext(&p, Token.Id.Number);1055 checkNext(&p, .Number);
954 checkNext(&p, Token.Id.String); // Title1056 checkNext(&p, .String); // Title
955 checkNext(&p, Token.Id.String);1057 checkNext(&p, .String);
956 checkNext(&p, Token.Id.String); // Thumbnail1058 checkNext(&p, .String); // Thumbnail
957 checkNext(&p, Token.Id.ObjectBegin);1059 checkNext(&p, .ObjectBegin);
958 checkNext(&p, Token.Id.String); // Url1060 checkNext(&p, .String); // Url
959 checkNext(&p, Token.Id.String);1061 checkNext(&p, .String);
960 checkNext(&p, Token.Id.String); // Height1062 checkNext(&p, .String); // Height
961 checkNext(&p, Token.Id.Number);1063 checkNext(&p, .Number);
962 checkNext(&p, Token.Id.String); // Width1064 checkNext(&p, .String); // Width
963 checkNext(&p, Token.Id.Number);1065 checkNext(&p, .Number);
964 checkNext(&p, Token.Id.ObjectEnd);1066 checkNext(&p, .ObjectEnd);
965 checkNext(&p, Token.Id.String); // Animated1067 checkNext(&p, .String); // Animated
966 checkNext(&p, Token.Id.False);1068 checkNext(&p, .False);
967 checkNext(&p, Token.Id.String); // IDs1069 checkNext(&p, .String); // IDs
968 checkNext(&p, Token.Id.ArrayBegin);1070 checkNext(&p, .ArrayBegin);
969 checkNext(&p, Token.Id.Number);1071 checkNext(&p, .Number);
970 checkNext(&p, Token.Id.Number);1072 checkNext(&p, .Number);
971 checkNext(&p, Token.Id.Number);1073 checkNext(&p, .Number);
972 checkNext(&p, Token.Id.Number);1074 checkNext(&p, .Number);
973 checkNext(&p, Token.Id.ArrayEnd);1075 checkNext(&p, .ArrayEnd);
974 checkNext(&p, Token.Id.ObjectEnd);1076 checkNext(&p, .ObjectEnd);
975 checkNext(&p, Token.Id.ObjectEnd);1077 checkNext(&p, .ObjectEnd);
9761078
977 testing.expect((try p.next()) == null);1079 testing.expect((try p.next()) == null);
978}1080}
...@@ -1081,7 +1183,7 @@ pub const Parser = struct {...@@ -1081,7 +1183,7 @@ pub const Parser = struct {
1081 pub fn init(allocator: *Allocator, copy_strings: bool) Parser {1183 pub fn init(allocator: *Allocator, copy_strings: bool) Parser {
1082 return Parser{1184 return Parser{
1083 .allocator = allocator,1185 .allocator = allocator,
1084 .state = State.Simple,1186 .state = .Simple,
1085 .copy_strings = copy_strings,1187 .copy_strings = copy_strings,
1086 .stack = Array.init(allocator),1188 .stack = Array.init(allocator),
1087 };1189 };
...@@ -1092,7 +1194,7 @@ pub const Parser = struct {...@@ -1092,7 +1194,7 @@ pub const Parser = struct {
1092 }1194 }
10931195
1094 pub fn reset(p: *Parser) void {1196 pub fn reset(p: *Parser) void {
1095 p.state = State.Simple;1197 p.state = .Simple;
1096 p.stack.shrink(0);1198 p.stack.shrink(0);
1097 }1199 }
10981200
...@@ -1118,8 +1220,8 @@ pub const Parser = struct {...@@ -1118,8 +1220,8 @@ pub const Parser = struct {
1118 // can be cleaned up on error correctly during a `parse` on call.1220 // can be cleaned up on error correctly during a `parse` on call.
1119 fn transition(p: *Parser, allocator: *Allocator, input: []const u8, i: usize, token: Token) !void {1221 fn transition(p: *Parser, allocator: *Allocator, input: []const u8, i: usize, token: Token) !void {
1120 switch (p.state) {1222 switch (p.state) {
1121 State.ObjectKey => switch (token.id) {1223 .ObjectKey => switch (token) {
1122 Token.Id.ObjectEnd => {1224 .ObjectEnd => {
1123 if (p.stack.len == 1) {1225 if (p.stack.len == 1) {
1124 return;1226 return;
1125 }1227 }
...@@ -1127,9 +1229,9 @@ pub const Parser = struct {...@@ -1127,9 +1229,9 @@ pub const Parser = struct {
1127 var value = p.stack.pop();1229 var value = p.stack.pop();
1128 try p.pushToParent(&value);1230 try p.pushToParent(&value);
1129 },1231 },
1130 Token.Id.String => {1232 .String => |s| {
1131 try p.stack.append(try p.parseString(allocator, token, input, i));1233 try p.stack.append(try p.parseString(allocator, s, input, i));
1132 p.state = State.ObjectValue;1234 p.state = .ObjectValue;
1133 },1235 },
1134 else => {1236 else => {
1135 // The streaming parser would return an error eventually.1237 // The streaming parser would return an error eventually.
...@@ -1138,54 +1240,54 @@ pub const Parser = struct {...@@ -1138,54 +1240,54 @@ pub const Parser = struct {
1138 return error.InvalidLiteral;1240 return error.InvalidLiteral;
1139 },1241 },
1140 },1242 },
1141 State.ObjectValue => {1243 .ObjectValue => {
1142 var object = &p.stack.items[p.stack.len - 2].Object;1244 var object = &p.stack.items[p.stack.len - 2].Object;
1143 var key = p.stack.items[p.stack.len - 1].String;1245 var key = p.stack.items[p.stack.len - 1].String;
11441246
1145 switch (token.id) {1247 switch (token) {
1146 Token.Id.ObjectBegin => {1248 .ObjectBegin => {
1147 try p.stack.append(Value{ .Object = ObjectMap.init(allocator) });1249 try p.stack.append(Value{ .Object = ObjectMap.init(allocator) });
1148 p.state = State.ObjectKey;1250 p.state = .ObjectKey;
1149 },1251 },
1150 Token.Id.ArrayBegin => {1252 .ArrayBegin => {
1151 try p.stack.append(Value{ .Array = Array.init(allocator) });1253 try p.stack.append(Value{ .Array = Array.init(allocator) });
1152 p.state = State.ArrayValue;1254 p.state = .ArrayValue;
1153 },1255 },
1154 Token.Id.String => {1256 .String => |s| {
1155 _ = try object.put(key, try p.parseString(allocator, token, input, i));1257 _ = try object.put(key, try p.parseString(allocator, s, input, i));
1156 _ = p.stack.pop();1258 _ = p.stack.pop();
1157 p.state = State.ObjectKey;1259 p.state = .ObjectKey;
1158 },1260 },
1159 Token.Id.Number => {1261 .Number => |n| {
1160 _ = try object.put(key, try p.parseNumber(token, input, i));1262 _ = try object.put(key, try p.parseNumber(n, input, i));
1161 _ = p.stack.pop();1263 _ = p.stack.pop();
1162 p.state = State.ObjectKey;1264 p.state = .ObjectKey;
1163 },1265 },
1164 Token.Id.True => {1266 .True => {
1165 _ = try object.put(key, Value{ .Bool = true });1267 _ = try object.put(key, Value{ .Bool = true });
1166 _ = p.stack.pop();1268 _ = p.stack.pop();
1167 p.state = State.ObjectKey;1269 p.state = .ObjectKey;
1168 },1270 },
1169 Token.Id.False => {1271 .False => {
1170 _ = try object.put(key, Value{ .Bool = false });1272 _ = try object.put(key, Value{ .Bool = false });
1171 _ = p.stack.pop();1273 _ = p.stack.pop();
1172 p.state = State.ObjectKey;1274 p.state = .ObjectKey;
1173 },1275 },
1174 Token.Id.Null => {1276 .Null => {
1175 _ = try object.put(key, Value.Null);1277 _ = try object.put(key, Value.Null);
1176 _ = p.stack.pop();1278 _ = p.stack.pop();
1177 p.state = State.ObjectKey;1279 p.state = .ObjectKey;
1178 },1280 },
1179 Token.Id.ObjectEnd, Token.Id.ArrayEnd => {1281 .ObjectEnd, .ArrayEnd => {
1180 unreachable;1282 unreachable;
1181 },1283 },
1182 }1284 }
1183 },1285 },
1184 State.ArrayValue => {1286 .ArrayValue => {
1185 var array = &p.stack.items[p.stack.len - 1].Array;1287 var array = &p.stack.items[p.stack.len - 1].Array;
11861288
1187 switch (token.id) {1289 switch (token) {
1188 Token.Id.ArrayEnd => {1290 .ArrayEnd => {
1189 if (p.stack.len == 1) {1291 if (p.stack.len == 1) {
1190 return;1292 return;
1191 }1293 }
...@@ -1193,59 +1295,59 @@ pub const Parser = struct {...@@ -1193,59 +1295,59 @@ pub const Parser = struct {
1193 var value = p.stack.pop();1295 var value = p.stack.pop();
1194 try p.pushToParent(&value);1296 try p.pushToParent(&value);
1195 },1297 },
1196 Token.Id.ObjectBegin => {1298 .ObjectBegin => {
1197 try p.stack.append(Value{ .Object = ObjectMap.init(allocator) });1299 try p.stack.append(Value{ .Object = ObjectMap.init(allocator) });
1198 p.state = State.ObjectKey;1300 p.state = .ObjectKey;
1199 },1301 },
1200 Token.Id.ArrayBegin => {1302 .ArrayBegin => {
1201 try p.stack.append(Value{ .Array = Array.init(allocator) });1303 try p.stack.append(Value{ .Array = Array.init(allocator) });
1202 p.state = State.ArrayValue;1304 p.state = .ArrayValue;
1203 },1305 },
1204 Token.Id.String => {1306 .String => |s| {
1205 try array.append(try p.parseString(allocator, token, input, i));1307 try array.append(try p.parseString(allocator, s, input, i));
1206 },1308 },
1207 Token.Id.Number => {1309 .Number => |n| {
1208 try array.append(try p.parseNumber(token, input, i));1310 try array.append(try p.parseNumber(n, input, i));
1209 },1311 },
1210 Token.Id.True => {1312 .True => {
1211 try array.append(Value{ .Bool = true });1313 try array.append(Value{ .Bool = true });
1212 },1314 },
1213 Token.Id.False => {1315 .False => {
1214 try array.append(Value{ .Bool = false });1316 try array.append(Value{ .Bool = false });
1215 },1317 },
1216 Token.Id.Null => {1318 .Null => {
1217 try array.append(Value.Null);1319 try array.append(Value.Null);
1218 },1320 },
1219 Token.Id.ObjectEnd => {1321 .ObjectEnd => {
1220 unreachable;1322 unreachable;
1221 },1323 },
1222 }1324 }
1223 },1325 },
1224 State.Simple => switch (token.id) {1326 .Simple => switch (token) {
1225 Token.Id.ObjectBegin => {1327 .ObjectBegin => {
1226 try p.stack.append(Value{ .Object = ObjectMap.init(allocator) });1328 try p.stack.append(Value{ .Object = ObjectMap.init(allocator) });
1227 p.state = State.ObjectKey;1329 p.state = .ObjectKey;
1228 },1330 },
1229 Token.Id.ArrayBegin => {1331 .ArrayBegin => {
1230 try p.stack.append(Value{ .Array = Array.init(allocator) });1332 try p.stack.append(Value{ .Array = Array.init(allocator) });
1231 p.state = State.ArrayValue;1333 p.state = .ArrayValue;
1232 },1334 },
1233 Token.Id.String => {1335 .String => |s| {
1234 try p.stack.append(try p.parseString(allocator, token, input, i));1336 try p.stack.append(try p.parseString(allocator, s, input, i));
1235 },1337 },
1236 Token.Id.Number => {1338 .Number => |n| {
1237 try p.stack.append(try p.parseNumber(token, input, i));1339 try p.stack.append(try p.parseNumber(n, input, i));
1238 },1340 },
1239 Token.Id.True => {1341 .True => {
1240 try p.stack.append(Value{ .Bool = true });1342 try p.stack.append(Value{ .Bool = true });
1241 },1343 },
1242 Token.Id.False => {1344 .False => {
1243 try p.stack.append(Value{ .Bool = false });1345 try p.stack.append(Value{ .Bool = false });
1244 },1346 },
1245 Token.Id.Null => {1347 .Null => {
1246 try p.stack.append(Value.Null);1348 try p.stack.append(Value.Null);
1247 },1349 },
1248 Token.Id.ObjectEnd, Token.Id.ArrayEnd => {1350 .ObjectEnd, .ArrayEnd => {
1249 unreachable;1351 unreachable;
1250 },1352 },
1251 },1353 },
...@@ -1260,12 +1362,12 @@ pub const Parser = struct {...@@ -1260,12 +1362,12 @@ pub const Parser = struct {
12601362
1261 var object = &p.stack.items[p.stack.len - 1].Object;1363 var object = &p.stack.items[p.stack.len - 1].Object;
1262 _ = try object.put(key, value.*);1364 _ = try object.put(key, value.*);
1263 p.state = State.ObjectKey;1365 p.state = .ObjectKey;
1264 },1366 },
1265 // Array Parent -> [ ..., <array>, value ]1367 // Array Parent -> [ ..., <array>, value ]
1266 Value.Array => |*array| {1368 Value.Array => |*array| {
1267 try array.append(value.*);1369 try array.append(value.*);
1268 p.state = State.ArrayValue;1370 p.state = .ArrayValue;
1269 },1371 },
1270 else => {1372 else => {
1271 unreachable;1373 unreachable;
...@@ -1273,80 +1375,78 @@ pub const Parser = struct {...@@ -1273,80 +1375,78 @@ pub const Parser = struct {
1273 }1375 }
1274 }1376 }
12751377
1276 fn parseString(p: *Parser, allocator: *Allocator, token: Token, input: []const u8, i: usize) !Value {1378 fn parseString(p: *Parser, allocator: *Allocator, s: std.meta.TagPayloadType(Token, Token.String), input: []const u8, i: usize) !Value {
1277 // TODO: We don't strictly have to copy values which do not contain any escape1379 // TODO: We don't strictly have to copy values which do not contain any escape
1278 // characters if flagged with the option.1380 // characters if flagged with the option.
1279 const slice = token.slice(input, i);1381 const slice = s.slice(input, i);
1280 return Value{ .String = try unescapeStringAlloc(allocator, slice) };1382 switch (s.escapes) {
1383 .None => return Value{ .String = try mem.dupe(allocator, u8, slice) },
1384 .Some => |some_escapes| {
1385 const output = try allocator.alloc(u8, s.decodedLength());
1386 errdefer allocator.free(output);
1387 try unescapeString(output, slice);
1388 return Value{ .String = output };
1389 },
1390 }
1281 }1391 }
12821392
1283 fn parseNumber(p: *Parser, token: Token, input: []const u8, i: usize) !Value {1393 fn parseNumber(p: *Parser, n: std.meta.TagPayloadType(Token, Token.Number), input: []const u8, i: usize) !Value {
1284 return if (token.number_is_integer)1394 return if (n.is_integer)
1285 Value{ .Integer = try std.fmt.parseInt(i64, token.slice(input, i), 10) }1395 Value{ .Integer = try std.fmt.parseInt(i64, n.slice(input, i), 10) }
1286 else1396 else
1287 Value{ .Float = try std.fmt.parseFloat(f64, token.slice(input, i)) };1397 Value{ .Float = try std.fmt.parseFloat(f64, n.slice(input, i)) };
1288 }1398 }
1289};1399};
12901400
1291// Unescape a JSON string1401// Unescape a JSON string
1292// Only to be used on strings already validated by the parser1402// Only to be used on strings already validated by the parser
1293// (note the unreachable statements and lack of bounds checking)1403// (note the unreachable statements and lack of bounds checking)
1294// Optimized for arena allocators, uses Allocator.shrink1404fn unescapeString(output: []u8, input: []const u8) !void {
1295//
1296// Idea: count how many bytes we will need to allocate in the streaming parser and store it
1297// in the token to avoid allocating too much memory or iterating through the string again
1298// Downside: need to find how many bytes a unicode escape sequence will produce twice
1299fn unescapeStringAlloc(alloc: *Allocator, input: []const u8) ![]u8 {
1300 const output = try alloc.alloc(u8, input.len);
1301 errdefer alloc.free(output);
1302
1303 var inIndex: usize = 0;1405 var inIndex: usize = 0;
1304 var outIndex: usize = 0;1406 var outIndex: usize = 0;
13051407
1306 while(inIndex < input.len) {1408 while (inIndex < input.len) {
1307 if(input[inIndex] != '\\'){1409 if (input[inIndex] != '\\') {
1308 // not an escape sequence1410 // not an escape sequence
1309 output[outIndex] = input[inIndex];1411 output[outIndex] = input[inIndex];
1310 inIndex += 1;1412 inIndex += 1;
1311 outIndex += 1;1413 outIndex += 1;
1312 } else if(input[inIndex + 1] != 'u'){1414 } else if (input[inIndex + 1] != 'u') {
1313 // a simple escape sequence1415 // a simple escape sequence
1314 output[outIndex] = @as(u8,1416 output[outIndex] = @as(u8, switch (input[inIndex + 1]) {
1315 switch(input[inIndex + 1]){1417 '\\' => '\\',
1316 '\\' => '\\',1418 '/' => '/',
1317 '/' => '/',1419 'n' => '\n',
1318 'n' => '\n',1420 'r' => '\r',
1319 'r' => '\r',1421 't' => '\t',
1320 't' => '\t',1422 'f' => 12,
1321 'f' => 12,1423 'b' => 8,
1322 'b' => 8,1424 '"' => '"',
1323 '"' => '"',1425 else => unreachable,
1324 else => unreachable1426 });
1325 }
1326 );
1327 inIndex += 2;1427 inIndex += 2;
1328 outIndex += 1;1428 outIndex += 1;
1329 } else {1429 } else {
1330 // a unicode escape sequence1430 // a unicode escape sequence
1331 const firstCodeUnit = std.fmt.parseInt(u16, input[inIndex+2 .. inIndex+6], 16) catch unreachable;1431 const firstCodeUnit = std.fmt.parseInt(u16, input[inIndex + 2 .. inIndex + 6], 16) catch unreachable;
13321432
1333 // guess optimistically that it's not a surrogate pair1433 // guess optimistically that it's not a surrogate pair
1334 if(std.unicode.utf8Encode(firstCodeUnit, output[outIndex..])) |byteCount| {1434 if (std.unicode.utf8Encode(firstCodeUnit, output[outIndex..])) |byteCount| {
1335 outIndex += byteCount;1435 outIndex += byteCount;
1336 inIndex += 6;1436 inIndex += 6;
1337 } else |err| {1437 } else |err| {
1338 // it might be a surrogate pair1438 // it might be a surrogate pair
1339 if(err != error.Utf8CannotEncodeSurrogateHalf) {1439 if (err != error.Utf8CannotEncodeSurrogateHalf) {
1340 return error.InvalidUnicodeHexSymbol;1440 return error.InvalidUnicodeHexSymbol;
1341 }1441 }
1342 // check if a second code unit is present1442 // check if a second code unit is present
1343 if(inIndex + 7 >= input.len or input[inIndex + 6] != '\\' or input[inIndex + 7] != 'u'){1443 if (inIndex + 7 >= input.len or input[inIndex + 6] != '\\' or input[inIndex + 7] != 'u') {
1344 return error.InvalidUnicodeHexSymbol;1444 return error.InvalidUnicodeHexSymbol;
1345 }1445 }
1346 1446
1347 const secondCodeUnit = std.fmt.parseInt(u16, input[inIndex+8 .. inIndex+12], 16) catch unreachable;1447 const secondCodeUnit = std.fmt.parseInt(u16, input[inIndex + 8 .. inIndex + 12], 16) catch unreachable;
1348 1448
1349 if(std.unicode.utf16leToUtf8(output[outIndex..], &[2]u16{ firstCodeUnit, secondCodeUnit })) |byteCount| {1449 if (std.unicode.utf16leToUtf8(output[outIndex..], &[2]u16{ firstCodeUnit, secondCodeUnit })) |byteCount| {
1350 outIndex += byteCount;1450 outIndex += byteCount;
1351 inIndex += 12;1451 inIndex += 12;
1352 } else |_| {1452 } else |_| {
...@@ -1355,8 +1455,7 @@ fn unescapeStringAlloc(alloc: *Allocator, input: []const u8) ![]u8 {...@@ -1355,8 +1455,7 @@ fn unescapeStringAlloc(alloc: *Allocator, input: []const u8) ![]u8 {
1355 }1455 }
1356 }1456 }
1357 }1457 }
13581458 assert(outIndex == output.len);
1359 return alloc.shrink(output, outIndex);
1360}1459}
13611460
1362test "json.parser.dynamic" {1461test "json.parser.dynamic" {
lib/std/meta.zig+1-3
...@@ -364,10 +364,8 @@ test "std.meta.activeTag" {...@@ -364,10 +364,8 @@ test "std.meta.activeTag" {
364364
365///Given a tagged union type, and an enum, return the type of the union365///Given a tagged union type, and an enum, return the type of the union
366/// field corresponding to the enum tag.366/// field corresponding to the enum tag.
367pub fn TagPayloadType(comptime U: type, tag: var) type {367pub fn TagPayloadType(comptime U: type, tag: @TagType(U)) type {
368 const Tag = @TypeOf(tag);
369 testing.expect(trait.is(builtin.TypeId.Union)(U));368 testing.expect(trait.is(builtin.TypeId.Union)(U));
370 testing.expect(trait.is(builtin.TypeId.Enum)(Tag));
371369
372 const info = @typeInfo(U).Union;370 const info = @typeInfo(U).Union;
373371