authorgravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2024-05-04 07:58:22-04:00
committergravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2024-05-29 08:22:01-04:00
log925e17879b851cb44d1a419eb08a2cb31d9e5eb7
treef22abad0bbbef153c6acaba3b4bc01a3c8a61554
parente60cd8058091e6c97743cd56206072ba5d1d8fce

second attempt

2 files changed, 84 insertions(+), 38 deletions(-)

lib/std/json/scanner.zig+77-28
......@@ -32,6 +32,7 @@ const std = @import("std");
3232
3333const Allocator = std.mem.Allocator;
3434const ArrayList = std.ArrayList;
35const ArrayListUnmanaged = std.ArrayListUnmanaged;
3536const assert = std.debug.assert;
3637const BitStack = std.BitStack;
3738
......@@ -193,15 +194,17 @@ pub const TokenType = enum {
193194/// At any time, notably just after an error, call `getLine()`, `getColumn()`, and/or `getByteOffset()`
194195/// to get meaningful information from this.
195196pub const Diagnostics = struct {
197 // continually updated by Scanner:
196198 line_number: u64 = 1,
197 line_start_cursor: usize = @as(usize, @bitCast(@as(isize, -1))), // Start just "before" the input buffer to get a 1-based column for line 1.
199 line_start_cursor: usize = @bitCast(@as(isize, -1)), // Start just "before" the input buffer to get a 1-based column for line 1.
198200 total_bytes_before_current_input: u64 = 0,
199 /// While the source is operational, this is a pointer into it.
200 /// If the source is destroyed, this becomes a literal value.
201 cursor: union(enum) {
202 pointer: *const usize,
203 value: usize,
204 } = undefined,
201
202 // updated by Scanner.saveDiagnostics:
203 cursor_in_current_input: usize = undefined,
204 current_input: []const u8 = undefined,
205
206 // updated by recordContext().
207 context_stack: ArrayListUnmanaged([]const u8) = .{},
205208
206209 /// Starts at 1.
207210 pub fn getLine(self: *const @This()) u64 {
......@@ -209,25 +212,68 @@ pub const Diagnostics = struct {
209212 }
210213 /// Starts at 1.
211214 pub fn getColumn(self: *const @This()) u64 {
212 return self.getCursor() -% self.line_start_cursor;
215 return self.cursor_in_current_input -% self.line_start_cursor;
213216 }
214217 /// Starts at 0. Measures the byte offset since the start of the input.
215218 pub fn getByteOffset(self: *const @This()) u64 {
216 return self.total_bytes_before_current_input + self.getCursor();
219 return self.total_bytes_before_current_input + self.cursor_in_current_input;
217220 }
218221
219 fn getCursor(self: *const @This()) usize {
220 return switch (self.cursor) {
221 .pointer => |p| p.*,
222 .value => |v| v,
223 };
222 pub fn recordContext(self: *@This(), allocator: Allocator, context: []const u8) Allocator.Error!void {
223 return self.context_stack.append(allocator, context);
224224 }
225 fn saveCursor(self: *@This()) void {
226 const value = self.getCursor();
227 self.cursor = .{ .value = value };
225
226 /// Pretty-print diagnostic information to the given writer, such as `std.io.getStdErr().writer()`.
227 /// file_name if non-null will be printed in a line with the line and column numbers;
228 /// it is purely aesthetic and is not touched on any actual file system.
229 pub fn dump(self: *const @This(), writer: anytype, err: anyerror, file_name: ?[]const u8) !void {
230 try writer.print("{s}:{}:{}: {s}\n", .{file_name orelse "<json>", self.getLine(), self.getColumn(), @errorName(err)});
231
232 // Show a "line" of context, or in case of very long lines, just an excerpt of the line.
233 // (Very long lines are common in minified JSON such as in an HTTP API or other machine-to-machine contexts.)
234 var start = self.cursor_in_current_input;
235 var start_elipsis: []const u8 = "";
236 while (true) {
237 if (start == 0 or self.current_input[start - 1] == '\n') break; // found start of line.
238 if (start + 40 <= self.cursor_in_current_input) {
239 // Too far into the line. Show part of the line.
240 start_elipsis = "...";
241 break;
242 }
243 start -= 1;
244 }
245 var end = start;
246 var end_elipsis: []const u8 = "";
247 while (true) {
248 if (end + 1 < self.current_input.len and self.current_input[end + 1] == '\n') break; // found end of line.
249 if (end == self.current_input.len) {
250 // found end of input.
251 // TODO: put elipsis when not is_end_of_input.
252 break;
253 }
254 if (end >= start + 70) {
255 // Line is too long. Show part of it.
256 end_elipsis = "...";
257 break;
258 }
259 end += 1;
260 }
261 try writer.print("{s}{s}{s}\n", .{start_elipsis, self.current_input[start..end], end_elipsis});
262 try writer.writeByteNTimes(' ', start_elipsis.len + self.cursor_in_current_input - start);
263 try writer.writeAll("^\n");
264
265 for (self.context_stack.items) |item| {
266 try writer.print(" in {s}\n", .{item});
267 }
228268 }
229269};
230270
271pub inline fn maybeRecordDiagnosticContext(allocator: Allocator, maybe_diagnostics: ?*Diagnostics, context: []const u8) Allocator.Error!void {
272 if (maybe_diagnostics) |diag| {
273 try diag.recordContext(allocator, context);
274 }
275}
276
231277/// See the documentation for `std.json.Token`.
232278pub const AllocWhen = enum { alloc_if_needed, alloc_always };
233279
......@@ -260,10 +306,6 @@ pub fn Reader(comptime buffer_size: usize, comptime ReaderType: type) type {
260306 pub fn enableDiagnostics(self: *@This(), diagnostics: *Diagnostics) void {
261307 self.scanner.enableDiagnostics(diagnostics);
262308 }
263 /// Calls `std.json.Scanner.saveDiagnostics`.
264 pub fn saveDiagnostics(self: *const @This()) void {
265 self.scanner.saveDiagnostics();
266 }
267309
268310 pub const NextError = ReaderType.Error || Error || Allocator.Error;
269311 pub const SkipError = NextError;
......@@ -466,18 +508,18 @@ pub const Scanner = struct {
466508 self.* = undefined;
467509 }
468510
469 /// See also `saveDiagnostics()`.
470511 pub fn enableDiagnostics(self: *@This(), diagnostics: *Diagnostics) void {
471 diagnostics.cursor = .{ .pointer = &self.cursor };
472 std.log.warn("cursor(enableDiagnostics): {}", .{diagnostics.getCursor()});
473512 self.diagnostics = diagnostics;
474513 }
475 /// Call this just before `deinit()` to make the diagnostics available after the `deinit()`.
514 /// For performance reasons, the diagnostics (see `enableDiagnostics`) are not kept up to date continually.
515 /// Call this method to update the diagnostics with the latest information.
516 /// Because diagnostics are usually consulted in case of an error, it is common to call this in an errdefer.
517 /// It is safe to call this regardless of whether diagnostics have been enabled.
518 /// This is already called in an errdefer block in every relevant public method of this class.
476519 pub fn saveDiagnostics(self: *const @This()) void {
477520 if (self.diagnostics) |diag| {
478 std.log.warn("cursor(deinit presave): {}", .{diag.getCursor()});
479 diag.saveCursor();
480 std.log.warn("cursor(deinit postsave): {}", .{diag.getCursor()});
521 diag.cursor_in_current_input = self.cursor;
522 diag.current_input = self.input;
481523 }
482524 }
483525
......@@ -520,6 +562,7 @@ pub const Scanner = struct {
520562 /// See also `std.json.Token` for documentation of `nextAlloc*()` function behavior.
521563 pub fn nextAllocMax(self: *@This(), allocator: Allocator, when: AllocWhen, max_value_len: usize) AllocError!Token {
522564 assert(self.is_end_of_input); // This function is not available in streaming mode.
565 errdefer self.saveDiagnostics();
523566 const token_type = self.peekNextTokenType() catch |e| switch (e) {
524567 error.BufferUnderrun => unreachable,
525568 else => |err| return err,
......@@ -577,6 +620,7 @@ pub const Scanner = struct {
577620 /// This method does not indicate whether the token content being returned is for a `.number` or `.string` token type;
578621 /// the caller of this method is expected to know which type of token is being processed.
579622 pub fn allocNextIntoArrayListMax(self: *@This(), value_list: *ArrayList(u8), when: AllocWhen, max_value_len: usize) AllocIntoArrayListError!?[]const u8 {
623 errdefer self.saveDiagnostics();
580624 while (true) {
581625 const token = try self.next();
582626 switch (token) {
......@@ -642,6 +686,7 @@ pub const Scanner = struct {
642686 /// see `peekNextTokenType()`.
643687 pub fn skipValue(self: *@This()) SkipError!void {
644688 assert(self.is_end_of_input); // This function is not available in streaming mode.
689 errdefer self.saveDiagnostics();
645690 switch (self.peekNextTokenType() catch |e| switch (e) {
646691 error.BufferUnderrun => unreachable,
647692 else => |err| return err,
......@@ -686,6 +731,7 @@ pub const Scanner = struct {
686731 /// Skip tokens until an `.object_end` or `.array_end` token results in a `stackHeight()` equal the given stack height.
687732 /// Unlike `skipValue()`, this function is available in streaming mode.
688733 pub fn skipUntilStackHeight(self: *@This(), terminal_stack_height: usize) NextError!void {
734 errdefer self.saveDiagnostics();
689735 while (true) {
690736 switch (try self.next()) {
691737 .object_end, .array_end => {
......@@ -705,11 +751,13 @@ pub const Scanner = struct {
705751 /// Pre allocate memory to hold the given number of nesting levels.
706752 /// `stackHeight()` up to the given number will not cause allocations.
707753 pub fn ensureTotalStackCapacity(self: *@This(), height: usize) Allocator.Error!void {
754 errdefer self.saveDiagnostics();
708755 try self.stack.ensureTotalCapacity(height);
709756 }
710757
711758 /// See `std.json.Token` for documentation of this function.
712759 pub fn next(self: *@This()) NextError!Token {
760 errdefer self.saveDiagnostics();
713761 state_loop: while (true) {
714762 switch (self.state) {
715763 .value => {
......@@ -1463,6 +1511,7 @@ pub const Scanner = struct {
14631511 /// determines which type of token will be returned from the next `next*()` call.
14641512 /// This function is idempotent, only advancing past commas, colons, and inter-token whitespace.
14651513 pub fn peekNextTokenType(self: *@This()) PeekError!TokenType {
1514 errdefer self.saveDiagnostics();
14661515 state_loop: while (true) {
14671516 switch (self.state) {
14681517 .value => {
lib/std/json/static.zig+7-10
......@@ -10,6 +10,7 @@ const AllocWhen = @import("./scanner.zig").AllocWhen;
1010const Diagnostics = @import("./scanner.zig").Diagnostics;
1111const default_max_value_len = @import("./scanner.zig").default_max_value_len;
1212const isNumberFormattedLikeAnInteger = @import("./scanner.zig").isNumberFormattedLikeAnInteger;
13const maybeRecordDiagnosticContext = @import("./scanner.zig").maybeRecordDiagnosticContext;
1314
1415const Value = @import("./dynamic.zig").Value;
1516const Array = @import("./dynamic.zig").Array;
......@@ -144,11 +145,6 @@ pub fn parseFromTokenSourceLeaky(
144145 if (resolved_options.diagnostics) |diag| {
145146 scanner_or_reader.enableDiagnostics(diag);
146147 }
147 defer {
148 if (resolved_options.diagnostics) |_| {
149 scanner_or_reader.saveDiagnostics();
150 }
151 }
152148
153149 const value = try innerParse(T, allocator, scanner_or_reader, resolved_options);
154150
......@@ -222,6 +218,8 @@ pub fn innerParse(
222218 source: anytype,
223219 options: ParseOptions,
224220) ParseError(@TypeOf(source.*))!T {
221 errdefer source.saveDiagnostics();
222 errdefer maybeRecordDiagnosticContext(allocator, options.diagnostics, @typeName(T)) catch {};
225223 switch (@typeInfo(T)) {
226224 .Bool => {
227225 return switch (try source.next()) {
......@@ -299,7 +297,7 @@ pub fn innerParse(
299297 if (u_field.type == void) {
300298 // void isn't really a json type, but we can support void payload union tags with {} as a value.
301299 if (.object_begin != try source.next()) return error.UnexpectedToken;
302 if (.object_end != try source.next()) return error.UnexpectedToken;
300 if (.object_end != try source.next()) return error.UnknownField;
303301 result = @unionInit(T, u_field.name, {});
304302 } else {
305303 // Recurse.
......@@ -347,9 +345,7 @@ pub fn innerParse(
347345 .object_end => { // No more fields.
348346 break;
349347 },
350 else => {
351 return error.UnexpectedToken;
352 },
348 else => unreachable, // Not possible while in an object.
353349 };
354350
355351 inline for (structInfo.fields, 0..) |field, i| {
......@@ -358,6 +354,7 @@ pub fn innerParse(
358354 // Free the name token now in case we're using an allocator that optimizes freeing the last allocated object.
359355 // (Recursing into innerParse() might trigger more allocations.)
360356 freeAllocated(allocator, name_token.?);
357 errdefer maybeRecordDiagnosticContext(allocator, options.diagnostics, @typeName(T) ++ "." ++ field.name) catch {};
361358 name_token = null;
362359 if (fields_seen[i]) {
363360 switch (options.duplicate_field_behavior) {
......@@ -624,7 +621,7 @@ pub fn innerParseFromValue(
624621 if (u_field.type == void) {
625622 // void isn't really a json type, but we can support void payload union tags with {} as a value.
626623 if (kv.value_ptr.* != .object) return error.UnexpectedToken;
627 if (kv.value_ptr.*.object.count() != 0) return error.UnexpectedToken;
624 if (kv.value_ptr.*.object.count() != 0) return error.UnknownField;
628625 return @unionInit(T, u_field.name, {});
629626 }
630627 // Recurse.