authorgravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2024-08-22 08:26:14-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2024-08-22 08:26:14-04:00
logfebfcbd49d0a4713dee9db38816c163fa1926ed4
tree9a926671f830cfe6aeb7d3e94f5c9bc8066d56a2
parent31220b50b5e718b862a8ce8e993619ecd2959101
signaturebadge-check Signed by PGP key B5690EEEBB952194

std.json.WriteStream supports streaming long values directly to the underlying stream (#21155)

2 files changed, 155 insertions(+), 30 deletions(-)

lib/std/json/stringify.zig+105-30
...@@ -156,36 +156,23 @@ pub fn writeStreamArbitraryDepth(...@@ -156,36 +156,23 @@ pub fn writeStreamArbitraryDepth(
156/// | <array>156/// | <array>
157/// | write157/// | write
158/// | print158/// | print
159/// | <writeRawStream>
159/// <object> = beginObject ( <field> <value> )* endObject160/// <object> = beginObject ( <field> <value> )* endObject
160/// <field> = objectField | objectFieldRaw161/// <field> = objectField | objectFieldRaw | <objectFieldRawStream>
161/// <array> = beginArray ( <value> )* endArray162/// <array> = beginArray ( <value> )* endArray
163/// <writeRawStream> = beginWriteRaw ( stream.writeAll )* endWriteRaw
164/// <objectFieldRawStream> = beginObjectFieldRaw ( stream.writeAll )* endObjectFieldRaw
162/// ```165/// ```
163///166///
164/// Supported types:167/// The `safety_checks_hint` parameter determines how much memory is used to enable assertions that the above grammar is being followed,
165/// * Zig `bool` -> JSON `true` or `false`.168/// e.g. tripping an assertion rather than allowing `endObject` to emit the final `}` in `[[[]]}`.
166/// * Zig `?T` -> `null` or the rendering of `T`.169/// "Depth" in this context means the depth of nested `[]` or `{}` expressions
167/// * Zig `i32`, `u64`, etc. -> JSON number or string.170/// (or equivalently the amount of recursion on the `<value>` grammar expression above).
168/// * When option `emit_nonportable_numbers_as_strings` is true, if the value is outside the range `+-1<<53` (the precise integer range of f64), it is rendered as a JSON string in base 10. Otherwise, it is rendered as JSON number.171/// For example, emitting the JSON `[[[]]]` requires a depth of 3.
169/// * Zig floats -> JSON number or string.172/// If `.checked_to_fixed_depth` is used, there is additionally an assertion that the nesting depth never exceeds the given limit.
170/// * If the value cannot be precisely represented by an f64, it is rendered as a JSON string. Otherwise, it is rendered as JSON number.173/// `.checked_to_arbitrary_depth` requires a runtime allocator for the memory.
171/// * TODO: Float rendering will likely change in the future, e.g. to remove the unnecessary "e+00".174/// `.checked_to_fixed_depth` embeds the storage required in the `WriteStream` struct.
172/// * Zig `[]const u8`, `[]u8`, `*[N]u8`, `@Vector(N, u8)`, and similar -> JSON string.175/// `.assumed_correct` requires no space and performs none of these assertions.
173/// * See `StringifyOptions.emit_strings_as_arrays`.
174/// * If the content is not valid UTF-8, rendered as an array of numbers instead.
175/// * Zig `[]T`, `[N]T`, `*[N]T`, `@Vector(N, T)`, and similar -> JSON array of the rendering of each item.
176/// * Zig tuple -> JSON array of the rendering of each item.
177/// * Zig `struct` -> JSON object with each field in declaration order.
178/// * If the struct declares a method `pub fn jsonStringify(self: *@This(), jw: anytype) !void`, it is called to do the serialization instead of the default behavior. The given `jw` is a pointer to this `WriteStream`. See `std.json.Value` for an example.
179/// * See `StringifyOptions.emit_null_optional_fields`.
180/// * Zig `union(enum)` -> JSON object with one field named for the active tag and a value representing the payload.
181/// * If the payload is `void`, then the emitted value is `{}`.
182/// * If the union declares a method `pub fn jsonStringify(self: *@This(), jw: anytype) !void`, it is called to do the serialization instead of the default behavior. The given `jw` is a pointer to this `WriteStream`.
183/// * Zig `enum` -> JSON string naming the active tag.
184/// * If the enum declares a method `pub fn jsonStringify(self: *@This(), jw: anytype) !void`, it is called to do the serialization instead of the default behavior. The given `jw` is a pointer to this `WriteStream`.
185/// * Zig untyped enum literal -> JSON string naming the active tag.
186/// * Zig error -> JSON string naming the error.
187/// * Zig `*T` -> the rendering of `T`. Note there is no guard against circular-reference infinite recursion.
188///
189/// In `ReleaseFast` and `ReleaseSmall` mode, the given `safety_checks_hint` is ignored and is always treated as `.assumed_correct`.176/// In `ReleaseFast` and `ReleaseSmall` mode, the given `safety_checks_hint` is ignored and is always treated as `.assumed_correct`.
190pub fn WriteStream(177pub fn WriteStream(
191 comptime OutStream: type,178 comptime OutStream: type,
...@@ -197,10 +184,14 @@ pub fn WriteStream(...@@ -197,10 +184,14 @@ pub fn WriteStream(
197) type {184) type {
198 return struct {185 return struct {
199 const Self = @This();186 const Self = @This();
200 const safety_checks: @TypeOf(safety_checks_hint) = switch (@import("builtin").mode) {187 const build_mode_has_safety = switch (@import("builtin").mode) {
201 .Debug, .ReleaseSafe => safety_checks_hint,188 .Debug, .ReleaseSafe => true,
202 .ReleaseFast, .ReleaseSmall => .assumed_correct,189 .ReleaseFast, .ReleaseSmall => false,
203 };190 };
191 const safety_checks: @TypeOf(safety_checks_hint) = if (build_mode_has_safety)
192 safety_checks_hint
193 else
194 .assumed_correct;
204195
205 pub const Stream = OutStream;196 pub const Stream = OutStream;
206 pub const Error = switch (safety_checks) {197 pub const Error = switch (safety_checks) {
...@@ -225,6 +216,11 @@ pub fn WriteStream(...@@ -225,6 +216,11 @@ pub fn WriteStream(
225 .assumed_correct => void,216 .assumed_correct => void,
226 },217 },
227218
219 raw_streaming_mode: if (build_mode_has_safety)
220 enum { none, value, objectField }
221 else
222 void = if (build_mode_has_safety) .none else {},
223
228 pub fn init(safety_allocator: Allocator, stream: OutStream, options: StringifyOptions) Self {224 pub fn init(safety_allocator: Allocator, stream: OutStream, options: StringifyOptions) Self {
229 return .{225 return .{
230 .options = options,226 .options = options,
...@@ -237,6 +233,7 @@ pub fn WriteStream(...@@ -237,6 +233,7 @@ pub fn WriteStream(
237 };233 };
238 }234 }
239235
236 /// Only necessary with .checked_to_arbitrary_depth.
240 pub fn deinit(self: *Self) void {237 pub fn deinit(self: *Self) void {
241 switch (safety_checks) {238 switch (safety_checks) {
242 .checked_to_arbitrary_depth => self.nesting_stack.deinit(),239 .checked_to_arbitrary_depth => self.nesting_stack.deinit(),
...@@ -246,6 +243,7 @@ pub fn WriteStream(...@@ -246,6 +243,7 @@ pub fn WriteStream(
246 }243 }
247244
248 pub fn beginArray(self: *Self) Error!void {245 pub fn beginArray(self: *Self) Error!void {
246 if (build_mode_has_safety) assert(self.raw_streaming_mode == .none);
249 try self.valueStart();247 try self.valueStart();
250 try self.stream.writeByte('[');248 try self.stream.writeByte('[');
251 try self.pushIndentation(ARRAY_MODE);249 try self.pushIndentation(ARRAY_MODE);
...@@ -253,6 +251,7 @@ pub fn WriteStream(...@@ -253,6 +251,7 @@ pub fn WriteStream(
253 }251 }
254252
255 pub fn beginObject(self: *Self) Error!void {253 pub fn beginObject(self: *Self) Error!void {
254 if (build_mode_has_safety) assert(self.raw_streaming_mode == .none);
256 try self.valueStart();255 try self.valueStart();
257 try self.stream.writeByte('{');256 try self.stream.writeByte('{');
258 try self.pushIndentation(OBJECT_MODE);257 try self.pushIndentation(OBJECT_MODE);
...@@ -260,6 +259,7 @@ pub fn WriteStream(...@@ -260,6 +259,7 @@ pub fn WriteStream(
260 }259 }
261260
262 pub fn endArray(self: *Self) Error!void {261 pub fn endArray(self: *Self) Error!void {
262 if (build_mode_has_safety) assert(self.raw_streaming_mode == .none);
263 self.popIndentation(ARRAY_MODE);263 self.popIndentation(ARRAY_MODE);
264 switch (self.next_punctuation) {264 switch (self.next_punctuation) {
265 .none => {},265 .none => {},
...@@ -273,6 +273,7 @@ pub fn WriteStream(...@@ -273,6 +273,7 @@ pub fn WriteStream(
273 }273 }
274274
275 pub fn endObject(self: *Self) Error!void {275 pub fn endObject(self: *Self) Error!void {
276 if (build_mode_has_safety) assert(self.raw_streaming_mode == .none);
276 self.popIndentation(OBJECT_MODE);277 self.popIndentation(OBJECT_MODE);
277 switch (self.next_punctuation) {278 switch (self.next_punctuation) {
278 .none => {},279 .none => {},
...@@ -389,16 +390,39 @@ pub fn WriteStream(...@@ -389,16 +390,39 @@ pub fn WriteStream(
389 /// e.g. `"1"`, `"[]"`, `"[1,2]"`, not `"1,2"`.390 /// e.g. `"1"`, `"[]"`, `"[1,2]"`, not `"1,2"`.
390 /// This function may be useful for doing your own number formatting.391 /// This function may be useful for doing your own number formatting.
391 pub fn print(self: *Self, comptime fmt: []const u8, args: anytype) Error!void {392 pub fn print(self: *Self, comptime fmt: []const u8, args: anytype) Error!void {
393 if (build_mode_has_safety) assert(self.raw_streaming_mode == .none);
392 try self.valueStart();394 try self.valueStart();
393 try self.stream.print(fmt, args);395 try self.stream.print(fmt, args);
394 self.valueDone();396 self.valueDone();
395 }397 }
396398
399 /// An alternative to calling `write` that allows you to write directly to the `.stream` field, e.g. with `.stream.writeAll()`.
400 /// Call `beginWriteRaw()`, then write a complete value (including any quotes if necessary) directly to the `.stream` field,
401 /// then call `endWriteRaw()`.
402 /// This can be useful for streaming very long strings into the output without needing it all buffered in memory.
403 pub fn beginWriteRaw(self: *Self) !void {
404 if (build_mode_has_safety) {
405 assert(self.raw_streaming_mode == .none);
406 self.raw_streaming_mode = .value;
407 }
408 try self.valueStart();
409 }
410
411 /// See `beginWriteRaw`.
412 pub fn endWriteRaw(self: *Self) void {
413 if (build_mode_has_safety) {
414 assert(self.raw_streaming_mode == .value);
415 self.raw_streaming_mode = .none;
416 }
417 self.valueDone();
418 }
419
397 /// See `WriteStream` for when to call this method.420 /// See `WriteStream` for when to call this method.
398 /// `key` is the string content of the property name.421 /// `key` is the string content of the property name.
399 /// Surrounding quotes will be added and any special characters will be escaped.422 /// Surrounding quotes will be added and any special characters will be escaped.
400 /// See also `objectFieldRaw`.423 /// See also `objectFieldRaw`.
401 pub fn objectField(self: *Self, key: []const u8) Error!void {424 pub fn objectField(self: *Self, key: []const u8) Error!void {
425 if (build_mode_has_safety) assert(self.raw_streaming_mode == .none);
402 try self.objectFieldStart();426 try self.objectFieldStart();
403 try encodeJsonString(key, self.options, self.stream);427 try encodeJsonString(key, self.options, self.stream);
404 self.next_punctuation = .colon;428 self.next_punctuation = .colon;
...@@ -408,14 +432,65 @@ pub fn WriteStream(...@@ -408,14 +432,65 @@ pub fn WriteStream(
408 /// A few assertions are performed on the given value to ensure that the caller of this function understands the API contract.432 /// A few assertions are performed on the given value to ensure that the caller of this function understands the API contract.
409 /// See also `objectField`.433 /// See also `objectField`.
410 pub fn objectFieldRaw(self: *Self, quoted_key: []const u8) Error!void {434 pub fn objectFieldRaw(self: *Self, quoted_key: []const u8) Error!void {
435 if (build_mode_has_safety) assert(self.raw_streaming_mode == .none);
411 assert(quoted_key.len >= 2 and quoted_key[0] == '"' and quoted_key[quoted_key.len - 1] == '"'); // quoted_key should be "quoted".436 assert(quoted_key.len >= 2 and quoted_key[0] == '"' and quoted_key[quoted_key.len - 1] == '"'); // quoted_key should be "quoted".
412 try self.objectFieldStart();437 try self.objectFieldStart();
413 try self.stream.writeAll(quoted_key);438 try self.stream.writeAll(quoted_key);
414 self.next_punctuation = .colon;439 self.next_punctuation = .colon;
415 }440 }
416441
417 /// See `WriteStream`.442 /// In the rare case that you need to write very long object field names,
443 /// this is an alternative to `objectField` and `objectFieldRaw` that allows you to write directly to the `.stream` field
444 /// similar to `beginWriteRaw`.
445 /// Call `endObjectFieldRaw()` when you're done.
446 pub fn beginObjectFieldRaw(self: *Self) !void {
447 if (build_mode_has_safety) {
448 assert(self.raw_streaming_mode == .none);
449 self.raw_streaming_mode = .objectField;
450 }
451 try self.objectFieldStart();
452 }
453
454 /// See `beginObjectFieldRaw`.
455 pub fn endObjectFieldRaw(self: *Self) void {
456 if (build_mode_has_safety) {
457 assert(self.raw_streaming_mode == .objectField);
458 self.raw_streaming_mode = .none;
459 }
460 self.next_punctuation = .colon;
461 }
462
463 /// Renders the given Zig value as JSON.
464 ///
465 /// Supported types:
466 /// * Zig `bool` -> JSON `true` or `false`.
467 /// * Zig `?T` -> `null` or the rendering of `T`.
468 /// * Zig `i32`, `u64`, etc. -> JSON number or string.
469 /// * When option `emit_nonportable_numbers_as_strings` is true, if the value is outside the range `+-1<<53` (the precise integer range of f64), it is rendered as a JSON string in base 10. Otherwise, it is rendered as JSON number.
470 /// * Zig floats -> JSON number or string.
471 /// * If the value cannot be precisely represented by an f64, it is rendered as a JSON string. Otherwise, it is rendered as JSON number.
472 /// * TODO: Float rendering will likely change in the future, e.g. to remove the unnecessary "e+00".
473 /// * Zig `[]const u8`, `[]u8`, `*[N]u8`, `@Vector(N, u8)`, and similar -> JSON string.
474 /// * See `StringifyOptions.emit_strings_as_arrays`.
475 /// * If the content is not valid UTF-8, rendered as an array of numbers instead.
476 /// * Zig `[]T`, `[N]T`, `*[N]T`, `@Vector(N, T)`, and similar -> JSON array of the rendering of each item.
477 /// * Zig tuple -> JSON array of the rendering of each item.
478 /// * Zig `struct` -> JSON object with each field in declaration order.
479 /// * If the struct declares a method `pub fn jsonStringify(self: *@This(), jw: anytype) !void`, it is called to do the serialization instead of the default behavior. The given `jw` is a pointer to this `WriteStream`. See `std.json.Value` for an example.
480 /// * See `StringifyOptions.emit_null_optional_fields`.
481 /// * Zig `union(enum)` -> JSON object with one field named for the active tag and a value representing the payload.
482 /// * If the payload is `void`, then the emitted value is `{}`.
483 /// * If the union declares a method `pub fn jsonStringify(self: *@This(), jw: anytype) !void`, it is called to do the serialization instead of the default behavior. The given `jw` is a pointer to this `WriteStream`.
484 /// * Zig `enum` -> JSON string naming the active tag.
485 /// * If the enum declares a method `pub fn jsonStringify(self: *@This(), jw: anytype) !void`, it is called to do the serialization instead of the default behavior. The given `jw` is a pointer to this `WriteStream`.
486 /// * Zig untyped enum literal -> JSON string naming the active tag.
487 /// * Zig error -> JSON string naming the error.
488 /// * Zig `*T` -> the rendering of `T`. Note there is no guard against circular-reference infinite recursion.
489 ///
490 /// See also alternative functions `print` and `beginWriteRaw`.
491 /// For writing object field names, use `objectField` instead.
418 pub fn write(self: *Self, value: anytype) Error!void {492 pub fn write(self: *Self, value: anytype) Error!void {
493 if (build_mode_has_safety) assert(self.raw_streaming_mode == .none);
419 const T = @TypeOf(value);494 const T = @TypeOf(value);
420 switch (@typeInfo(T)) {495 switch (@typeInfo(T)) {
421 .Int => {496 .Int => {
lib/std/json/stringify_test.zig+50
...@@ -443,3 +443,53 @@ test "nonportable numbers" {...@@ -443,3 +443,53 @@ test "nonportable numbers" {
443 try testStringify("9999999999999999", 9999999999999999, .{});443 try testStringify("9999999999999999", 9999999999999999, .{});
444 try testStringify("\"9999999999999999\"", 9999999999999999, .{ .emit_nonportable_numbers_as_strings = true });444 try testStringify("\"9999999999999999\"", 9999999999999999, .{ .emit_nonportable_numbers_as_strings = true });
445}445}
446
447test "stringify raw streaming" {
448 var out_buf: [1024]u8 = undefined;
449 var slice_stream = std.io.fixedBufferStream(&out_buf);
450 const out = slice_stream.writer();
451
452 {
453 var w = writeStream(out, .{ .whitespace = .indent_2 });
454 try testRawStreaming(&w, &slice_stream);
455 }
456
457 {
458 var w = writeStreamMaxDepth(out, .{ .whitespace = .indent_2 }, 8);
459 try testRawStreaming(&w, &slice_stream);
460 }
461
462 {
463 var w = writeStreamMaxDepth(out, .{ .whitespace = .indent_2 }, null);
464 try testRawStreaming(&w, &slice_stream);
465 }
466
467 {
468 var w = writeStreamArbitraryDepth(testing.allocator, out, .{ .whitespace = .indent_2 });
469 defer w.deinit();
470 try testRawStreaming(&w, &slice_stream);
471 }
472}
473
474fn testRawStreaming(w: anytype, slice_stream: anytype) !void {
475 slice_stream.reset();
476
477 try w.beginObject();
478 try w.beginObjectFieldRaw();
479 try w.stream.writeAll("\"long");
480 try w.stream.writeAll(" key\"");
481 w.endObjectFieldRaw();
482 try w.beginWriteRaw();
483 try w.stream.writeAll("\"long");
484 try w.stream.writeAll(" value\"");
485 w.endWriteRaw();
486 try w.endObject();
487
488 const result = slice_stream.getWritten();
489 const expected =
490 \\{
491 \\ "long key": "long value"
492 \\}
493 ;
494 try std.testing.expectEqualStrings(expected, result);
495}