authorgravatar for me@nurulhudaapon.comNurul Huda (Apon) <me@nurulhudaapon.com> 2026-08-12 23:43:31+02:00
committergravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2026-08-12 23:43:31+02:00
log013492ecb89334750f019fa4d70bcedccd31b300
treebd360d4353992e680cc6865415bc2d295462765f
parent89e0881f11c8fdb2dd981c4198618d071d323026

std.zon serialize: make escaping non-ascii optional, defaulting to false (#30710)

Currently std.zon.stringify.serialize always escapes Unicode characters, while std.json.stringify by default does not. This change adds an escape_non_ascii option that matches the JSON serializer's behavior. `escape_non_ascii` is false by default, and will only escape unicode once set to true. Closes [#23535](https://github.com/ziglang/zig/issues/23535) Reviewed-on: https://codeberg.org/ziglang/zig/pulls/30710 Reviewed-by: Ryan Liptak <squeek502@noreply.codeberg.org>

2 files changed, 133 insertions(+), 21 deletions(-)

lib/std/zon/Serializer.zig+110-6
...@@ -64,6 +64,7 @@ pub const ValueOptions = struct {...@@ -64,6 +64,7 @@ pub const ValueOptions = struct {
64 emit_codepoint_literals: EmitCodepointLiterals = .never,64 emit_codepoint_literals: EmitCodepointLiterals = .never,
65 emit_strings_as_containers: bool = false,65 emit_strings_as_containers: bool = false,
66 emit_default_optional_fields: bool = true,66 emit_default_optional_fields: bool = true,
67 escape_non_ascii: bool = false,
67};68};
6869
69/// Determines when to emit Unicode code point literals as opposed to integer literals.70/// Determines when to emit Unicode code point literals as opposed to integer literals.
...@@ -125,7 +126,7 @@ pub fn valueArbitraryDepth(self: *Serializer, val: anytype, options: ValueOption...@@ -125,7 +126,7 @@ pub fn valueArbitraryDepth(self: *Serializer, val: anytype, options: ValueOption
125 comptime assertCanSerializeType(@TypeOf(val));126 comptime assertCanSerializeType(@TypeOf(val));
126 switch (@typeInfo(@TypeOf(val))) {127 switch (@typeInfo(@TypeOf(val))) {
127 .int, .comptime_int => if (options.emit_codepoint_literals.emitAsCodepoint(val)) |c| {128 .int, .comptime_int => if (options.emit_codepoint_literals.emitAsCodepoint(val)) |c| {
128 self.codePoint(c) catch |err| switch (err) {129 self.codePoint(c, .{ .escape_non_ascii = options.escape_non_ascii }) catch |err| switch (err) {
129 error.InvalidCodepoint => unreachable, // Already validated130 error.InvalidCodepoint => unreachable, // Already validated
130 else => |e| return e,131 else => |e| return e,
131 };132 };
...@@ -146,7 +147,7 @@ pub fn valueArbitraryDepth(self: *Serializer, val: anytype, options: ValueOption...@@ -146,7 +147,7 @@ pub fn valueArbitraryDepth(self: *Serializer, val: anytype, options: ValueOption
146 (pointer.sentinel() == null or pointer.sentinel() == 0) and147 (pointer.sentinel() == null or pointer.sentinel() == 0) and
147 !options.emit_strings_as_containers)148 !options.emit_strings_as_containers)
148 {149 {
149 return try self.string(val);150 return try self.string(val, .{ .escape_non_ascii = options.escape_non_ascii });
150 }151 }
151152
152 // Serialize as either a tuple or as the child type153 // Serialize as either a tuple or as the child type
...@@ -285,12 +286,25 @@ pub fn ident(self: *Serializer, name: []const u8) Error!void {...@@ -285,12 +286,25 @@ pub fn ident(self: *Serializer, name: []const u8) Error!void {
285}286}
286287
287pub const CodePointError = Error || error{InvalidCodepoint};288pub const CodePointError = Error || error{InvalidCodepoint};
289/// Options for formatting code points.
290pub const CodePointOptions = struct {
291 escape_non_ascii: bool = false,
292};
288293
289/// Serialize `val` as a Unicode codepoint.294/// Serialize `val` as a Unicode codepoint.
290///295///
291/// Returns `error.InvalidCodepoint` if `val` is not a valid Unicode codepoint.296/// Returns `error.InvalidCodepoint` if `val` is not a valid Unicode codepoint.
292pub fn codePoint(self: *Serializer, val: u21) CodePointError!void {297pub fn codePoint(
293 try self.writer.print("'{f}'", .{std.zig.fmtChar(val)});298 self: *Serializer,
299 val: u21,
300 options: CodePointOptions,
301) CodePointError!void {
302 try self.writer.writeByte('\'');
303 try self.writeCodepoint(val, .{
304 .escape_non_ascii = options.escape_non_ascii,
305 .quote_style = .single,
306 });
307 try self.writer.writeByte('\'');
294}308}
295309
296/// Like `value`, but always serializes `val` as a tuple.310/// Like `value`, but always serializes `val` as a tuple.
...@@ -346,9 +360,99 @@ fn tupleImpl(self: *Serializer, val: anytype, options: ValueOptions) Error!void...@@ -346,9 +360,99 @@ fn tupleImpl(self: *Serializer, val: anytype, options: ValueOptions) Error!void
346 }360 }
347}361}
348362
363/// Options for writing a Unicode codepoint.
364const WriteCodepointOptions = struct {
365 escape_non_ascii: bool = false,
366 /// If single quote style then single quotes are escaped, otherwise double quotes are escaped.
367 quote_style: enum { single, double } = .single,
368};
369
370/// Write a Unicode codepoint to the writer using the given options.
371///
372/// Returns `error.InvalidCodepoint` if `codepoint` is not a valid Unicode codepoint.
373fn writeCodepoint(
374 self: *Serializer,
375 codepoint: u21,
376 options: WriteCodepointOptions,
377) CodePointError!void {
378 switch (codepoint) {
379 // Printable ASCII
380 ' ', '!', '#'...'&', '('...'[', ']'...'~' => try self.writer.writeByte(@intCast(codepoint)),
381 // Unprintable ASCII
382 0x00...0x08, 0x0B, 0x0C, 0x0E...0x1F, 0x7F => try self.writer.print("\\x{x:0>2}", .{codepoint}),
383 // ASCII with special escapes
384 '\n' => try self.writer.writeAll("\\n"),
385 '\r' => try self.writer.writeAll("\\r"),
386 '\t' => try self.writer.writeAll("\\t"),
387 '\\' => try self.writer.writeAll("\\\\"),
388 // Quotes need escaping if they conflict with the in-use quote character
389 '\'' => if (options.quote_style == .single) try self.writer.writeAll("\\'") else try self.writer.writeByte('\''),
390 '\"' => if (options.quote_style == .double) try self.writer.writeAll("\\\"") else try self.writer.writeByte('"'),
391
392 // Surrogates can only be written with an escape
393 0xD800...0xDFFF => try self.writer.print("\\u{{{x}}}", .{codepoint}),
394 // Other valid codepoints
395 0x80...0xD7FF, 0xE000...0x10FFFF => if (options.escape_non_ascii) {
396 try self.writer.print("\\u{{{x}}}", .{codepoint});
397 } else {
398 var buf: [7]u8 = undefined;
399 const len = std.unicode.utf8Encode(codepoint, &buf) catch unreachable;
400 try self.writer.writeAll(buf[0..len]);
401 },
402 // Invalid codepoints
403 0x110000...std.math.maxInt(u21) => return error.InvalidCodepoint,
404 }
405}
406
407pub const StringOptions = struct {
408 escape_non_ascii: bool = false,
409};
410
349/// Like `value`, but always serializes `val` as a string.411/// Like `value`, but always serializes `val` as a string.
350pub fn string(self: *Serializer, val: []const u8) Error!void {412pub fn string(self: *Serializer, val: []const u8, options: StringOptions) Writer.Error!void {
351 try self.writer.print("\"{f}\"", .{std.zig.fmtString(val)});413 try self.writer.writeByte('"');
414 // Batch write sequences of "raw" bytes (printable ASCII or non-escaped non-ASCII) for performance.
415 // `val[start..i]` contains pending raw bytes to write.
416 var start: usize = 0;
417 var i: usize = 0;
418 while (i < val.len) {
419 const byte = val[i];
420 // Check if this byte can be written as-is
421 const is_raw = switch (byte) {
422 ' ', '!', '#'...'[', ']'...'~' => true,
423 0x80...0xFF => !options.escape_non_ascii,
424 else => false,
425 };
426 if (is_raw) {
427 i += 1;
428 continue;
429 }
430 // Flush pending raw bytes
431 try self.writer.writeAll(val[start..i]);
432 // Handle the special character
433 if (byte >= 0x80) {
434 // Decode UTF-8 sequence and write the codepoint
435 const ulen = std.unicode.utf8ByteSequenceLength(byte) catch unreachable;
436 const codepoint = std.unicode.utf8Decode(val[i..][0..ulen]) catch unreachable;
437 // InvalidCodepoint cannot occur from valid UTF-8
438 self.writeCodepoint(codepoint, .{
439 .escape_non_ascii = options.escape_non_ascii,
440 .quote_style = .double,
441 }) catch unreachable;
442 i += ulen;
443 } else {
444 // ASCII character that needs escaping
445 self.writeCodepoint(byte, .{
446 .escape_non_ascii = options.escape_non_ascii,
447 .quote_style = .double,
448 }) catch unreachable; // InvalidCodepoint cannot occur for valid ASCII values
449 i += 1;
450 }
451 start = i;
452 }
453
454 try self.writer.writeAll(val[start..]);
455 try self.writer.writeByte('"');
352}456}
353457
354/// Options for formatting multiline strings.458/// Options for formatting multiline strings.
lib/std/zon/stringify.zig+23-15
...@@ -24,7 +24,7 @@...@@ -24,7 +24,7 @@
24const std = @import("std");24const std = @import("std");
25const assert = std.debug.assert;25const assert = std.debug.assert;
26const Writer = std.Io.Writer;26const Writer = std.Io.Writer;
27const Serializer = std.zon.Serializer;27const Serializer = @import("Serializer.zig");
2828
29pub const SerializeOptions = struct {29pub const SerializeOptions = struct {
30 /// If false, whitespace is omitted. Otherwise whitespace is emitted in standard Zig style.30 /// If false, whitespace is omitted. Otherwise whitespace is emitted in standard Zig style.
...@@ -37,6 +37,8 @@ pub const SerializeOptions = struct {...@@ -37,6 +37,8 @@ pub const SerializeOptions = struct {
37 /// If false, struct fields are not written if they are equal to their default value. Comparison37 /// If false, struct fields are not written if they are equal to their default value. Comparison
38 /// is done by `std.meta.eql`.38 /// is done by `std.meta.eql`.
39 emit_default_optional_fields: bool = true,39 emit_default_optional_fields: bool = true,
40 /// If true, non-ASCII unicode characters are escaped.
41 escape_non_ascii: bool = false,
40};42};
4143
42/// Serialize the given value as ZON.44/// Serialize the given value as ZON.
...@@ -51,6 +53,7 @@ pub fn serialize(val: anytype, options: SerializeOptions, writer: *Writer) Write...@@ -51,6 +53,7 @@ pub fn serialize(val: anytype, options: SerializeOptions, writer: *Writer) Write
51 .emit_codepoint_literals = options.emit_codepoint_literals,53 .emit_codepoint_literals = options.emit_codepoint_literals,
52 .emit_strings_as_containers = options.emit_strings_as_containers,54 .emit_strings_as_containers = options.emit_strings_as_containers,
53 .emit_default_optional_fields = options.emit_default_optional_fields,55 .emit_default_optional_fields = options.emit_default_optional_fields,
56 .escape_non_ascii = options.escape_non_ascii,
54 });57 });
55}58}
5659
...@@ -72,6 +75,7 @@ pub fn serializeMaxDepth(...@@ -72,6 +75,7 @@ pub fn serializeMaxDepth(
72 .emit_codepoint_literals = options.emit_codepoint_literals,75 .emit_codepoint_literals = options.emit_codepoint_literals,
73 .emit_strings_as_containers = options.emit_strings_as_containers,76 .emit_strings_as_containers = options.emit_strings_as_containers,
74 .emit_default_optional_fields = options.emit_default_optional_fields,77 .emit_default_optional_fields = options.emit_default_optional_fields,
78 .escape_non_ascii = options.escape_non_ascii,
75 }, depth);79 }, depth);
76}80}
7781
...@@ -91,6 +95,7 @@ pub fn serializeArbitraryDepth(...@@ -91,6 +95,7 @@ pub fn serializeArbitraryDepth(
91 .emit_codepoint_literals = options.emit_codepoint_literals,95 .emit_codepoint_literals = options.emit_codepoint_literals,
92 .emit_strings_as_containers = options.emit_strings_as_containers,96 .emit_strings_as_containers = options.emit_strings_as_containers,
93 .emit_default_optional_fields = options.emit_default_optional_fields,97 .emit_default_optional_fields = options.emit_default_optional_fields,
98 .escape_non_ascii = options.escape_non_ascii,
94 });99 });
95}100}
96101
...@@ -588,7 +593,7 @@ test "std.zon stringify utf8 codepoints" {...@@ -588,7 +593,7 @@ test "std.zon stringify utf8 codepoints" {
588 try std.testing.expectEqualStrings("97", aw.written());593 try std.testing.expectEqualStrings("97", aw.written());
589 aw.clearRetainingCapacity();594 aw.clearRetainingCapacity();
590595
591 try s.codePoint('a');596 try s.codePoint('a', .{});
592 try std.testing.expectEqualStrings("'a'", aw.written());597 try std.testing.expectEqualStrings("'a'", aw.written());
593 aw.clearRetainingCapacity();598 aw.clearRetainingCapacity();
594599
...@@ -609,7 +614,7 @@ test "std.zon stringify utf8 codepoints" {...@@ -609,7 +614,7 @@ test "std.zon stringify utf8 codepoints" {
609 try std.testing.expectEqualStrings("10", aw.written());614 try std.testing.expectEqualStrings("10", aw.written());
610 aw.clearRetainingCapacity();615 aw.clearRetainingCapacity();
611616
612 try s.codePoint('\n');617 try s.codePoint('\n', .{});
613 try std.testing.expectEqualStrings("'\\n'", aw.written());618 try std.testing.expectEqualStrings("'\\n'", aw.written());
614 aw.clearRetainingCapacity();619 aw.clearRetainingCapacity();
615620
...@@ -630,11 +635,11 @@ test "std.zon stringify utf8 codepoints" {...@@ -630,11 +635,11 @@ test "std.zon stringify utf8 codepoints" {
630 try std.testing.expectEqualStrings("9889", aw.written());635 try std.testing.expectEqualStrings("9889", aw.written());
631 aw.clearRetainingCapacity();636 aw.clearRetainingCapacity();
632637
633 try s.codePoint('⚡');638 try s.codePoint('⚡', .{ .escape_non_ascii = true });
634 try std.testing.expectEqualStrings("'\\u{26a1}'", aw.written());639 try std.testing.expectEqualStrings("'\\u{26a1}'", aw.written());
635 aw.clearRetainingCapacity();640 aw.clearRetainingCapacity();
636641
637 try s.value('⚡', .{ .emit_codepoint_literals = .always });642 try s.value('⚡', .{ .emit_codepoint_literals = .always, .escape_non_ascii = true });
638 try std.testing.expectEqualStrings("'\\u{26a1}'", aw.written());643 try std.testing.expectEqualStrings("'\\u{26a1}'", aw.written());
639 aw.clearRetainingCapacity();644 aw.clearRetainingCapacity();
640645
...@@ -647,8 +652,7 @@ test "std.zon stringify utf8 codepoints" {...@@ -647,8 +652,7 @@ test "std.zon stringify utf8 codepoints" {
647 aw.clearRetainingCapacity();652 aw.clearRetainingCapacity();
648653
649 // Invalid codepoint654 // Invalid codepoint
650 try s.codePoint(0x110000 + 1);655 try std.testing.expectError(error.InvalidCodepoint, s.codePoint(0x110000 + 1, .{ .escape_non_ascii = true }));
651 try std.testing.expectEqualStrings("'\\u{110001}'", aw.written());
652 aw.clearRetainingCapacity();656 aw.clearRetainingCapacity();
653657
654 try s.int(0x110000 + 1);658 try s.int(0x110000 + 1);
...@@ -681,7 +685,7 @@ test "std.zon stringify utf8 codepoints" {...@@ -681,7 +685,7 @@ test "std.zon stringify utf8 codepoints" {
681 aw.clearRetainingCapacity();685 aw.clearRetainingCapacity();
682686
683 // Make sure value options are passed to children687 // Make sure value options are passed to children
684 try s.value(.{ .c = '⚡' }, .{ .emit_codepoint_literals = .always });688 try s.value(.{ .c = '⚡' }, .{ .emit_codepoint_literals = .always, .escape_non_ascii = true });
685 try std.testing.expectEqualStrings(".{ .c = '\\u{26a1}' }", aw.written());689 try std.testing.expectEqualStrings(".{ .c = '\\u{26a1}' }", aw.written());
686 aw.clearRetainingCapacity();690 aw.clearRetainingCapacity();
687691
...@@ -696,11 +700,11 @@ test "std.zon stringify strings" {...@@ -696,11 +700,11 @@ test "std.zon stringify strings" {
696 defer aw.deinit();700 defer aw.deinit();
697701
698 // Minimal case702 // Minimal case
699 try s.string("abc⚡\n");703 try s.string("abc⚡ÿ\n", .{ .escape_non_ascii = true });
700 try std.testing.expectEqualStrings("\"abc\\xe2\\x9a\\xa1\\n\"", aw.written());704 try std.testing.expectEqualStrings("\"abc\\u{26a1}\\u{ff}\\n\"", aw.written());
701 aw.clearRetainingCapacity();705 aw.clearRetainingCapacity();
702706
703 try s.tuple("abc⚡\n", .{});707 try s.tuple("abc⚡ÿ\n", .{});
704 try std.testing.expectEqualStrings(708 try std.testing.expectEqualStrings(
705 \\.{709 \\.{
706 \\ 97,710 \\ 97,
...@@ -709,16 +713,18 @@ test "std.zon stringify strings" {...@@ -709,16 +713,18 @@ test "std.zon stringify strings" {
709 \\ 226,713 \\ 226,
710 \\ 154,714 \\ 154,
711 \\ 161,715 \\ 161,
716 \\ 195,
717 \\ 191,
712 \\ 10,718 \\ 10,
713 \\}719 \\}
714 , aw.written());720 , aw.written());
715 aw.clearRetainingCapacity();721 aw.clearRetainingCapacity();
716722
717 try s.value("abc⚡\n", .{});723 try s.value("abc⚡ÿ\n", .{ .escape_non_ascii = false });
718 try std.testing.expectEqualStrings("\"abc\\xe2\\x9a\\xa1\\n\"", aw.written());724 try std.testing.expectEqualStrings("\"abc⚡ÿ\\n\"", aw.written());
719 aw.clearRetainingCapacity();725 aw.clearRetainingCapacity();
720726
721 try s.value("abc⚡\n", .{ .emit_strings_as_containers = true });727 try s.value("abc⚡ÿ\n", .{ .emit_strings_as_containers = true });
722 try std.testing.expectEqualStrings(728 try std.testing.expectEqualStrings(
723 \\.{729 \\.{
724 \\ 97,730 \\ 97,
...@@ -727,6 +733,8 @@ test "std.zon stringify strings" {...@@ -727,6 +733,8 @@ test "std.zon stringify strings" {
727 \\ 226,733 \\ 226,
728 \\ 154,734 \\ 154,
729 \\ 161,735 \\ 161,
736 \\ 195,
737 \\ 191,
730 \\ 10,738 \\ 10,
731 \\}739 \\}
732 , aw.written());740 , aw.written());
...@@ -816,7 +824,7 @@ test "std.zon stringify multiline strings" {...@@ -816,7 +824,7 @@ test "std.zon stringify multiline strings" {
816824
817 {825 {
818 const str: []const u8 = &.{ 'a', '\r', 'c' };826 const str: []const u8 = &.{ 'a', '\r', 'c' };
819 try s.string(str);827 try s.string(str, .{ .escape_non_ascii = false });
820 try std.testing.expectEqualStrings("\"a\\rc\"", aw.written());828 try std.testing.expectEqualStrings("\"a\\rc\"", aw.written());
821 aw.clearRetainingCapacity();829 aw.clearRetainingCapacity();
822 }830 }