authorgravatar for me@nurulhudaapon.comNurul Huda (Apon) <me@nurulhudaapon.com> 2026-08-12 23:43:31+02:00
committergravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2026-08-12 23:43:31+02:00
log013492ecb89334750f019fa4d70bcedccd31b300
treebd360d4353992e680cc6865415bc2d295462765f
parent89e0881f11c8fdb2dd981c4198618d071d323026

std.zon serialize: make escaping non-ascii optional, defaulting to false (#30710)

Currently std.zon.stringify.serialize always escapes Unicode characters, while std.json.stringify by default does not. This change adds an escape_non_ascii option that matches the JSON serializer's behavior. `escape_non_ascii` is false by default, and will only escape unicode once set to true. Closes [#23535](https://github.com/ziglang/zig/issues/23535) Reviewed-on: https://codeberg.org/ziglang/zig/pulls/30710 Reviewed-by: Ryan Liptak <squeek502@noreply.codeberg.org>

2 files changed, 133 insertions(+), 21 deletions(-)

lib/std/zon/Serializer.zig+110-6
......@@ -64,6 +64,7 @@ pub const ValueOptions = struct {
6464 emit_codepoint_literals: EmitCodepointLiterals = .never,
6565 emit_strings_as_containers: bool = false,
6666 emit_default_optional_fields: bool = true,
67 escape_non_ascii: bool = false,
6768};
6869
6970/// Determines when to emit Unicode code point literals as opposed to integer literals.
......@@ -125,7 +126,7 @@ pub fn valueArbitraryDepth(self: *Serializer, val: anytype, options: ValueOption
125126 comptime assertCanSerializeType(@TypeOf(val));
126127 switch (@typeInfo(@TypeOf(val))) {
127128 .int, .comptime_int => if (options.emit_codepoint_literals.emitAsCodepoint(val)) |c| {
128 self.codePoint(c) catch |err| switch (err) {
129 self.codePoint(c, .{ .escape_non_ascii = options.escape_non_ascii }) catch |err| switch (err) {
129130 error.InvalidCodepoint => unreachable, // Already validated
130131 else => |e| return e,
131132 };
......@@ -146,7 +147,7 @@ pub fn valueArbitraryDepth(self: *Serializer, val: anytype, options: ValueOption
146147 (pointer.sentinel() == null or pointer.sentinel() == 0) and
147148 !options.emit_strings_as_containers)
148149 {
149 return try self.string(val);
150 return try self.string(val, .{ .escape_non_ascii = options.escape_non_ascii });
150151 }
151152
152153 // Serialize as either a tuple or as the child type
......@@ -285,12 +286,25 @@ pub fn ident(self: *Serializer, name: []const u8) Error!void {
285286}
286287
287288pub const CodePointError = Error || error{InvalidCodepoint};
289/// Options for formatting code points.
290pub const CodePointOptions = struct {
291 escape_non_ascii: bool = false,
292};
288293
289294/// Serialize `val` as a Unicode codepoint.
290295///
291296/// Returns `error.InvalidCodepoint` if `val` is not a valid Unicode codepoint.
292pub fn codePoint(self: *Serializer, val: u21) CodePointError!void {
293 try self.writer.print("'{f}'", .{std.zig.fmtChar(val)});
297pub fn codePoint(
298 self: *Serializer,
299 val: u21,
300 options: CodePointOptions,
301) CodePointError!void {
302 try self.writer.writeByte('\'');
303 try self.writeCodepoint(val, .{
304 .escape_non_ascii = options.escape_non_ascii,
305 .quote_style = .single,
306 });
307 try self.writer.writeByte('\'');
294308}
295309
296310/// Like `value`, but always serializes `val` as a tuple.
......@@ -346,9 +360,99 @@ fn tupleImpl(self: *Serializer, val: anytype, options: ValueOptions) Error!void
346360 }
347361}
348362
363/// Options for writing a Unicode codepoint.
364const WriteCodepointOptions = struct {
365 escape_non_ascii: bool = false,
366 /// If single quote style then single quotes are escaped, otherwise double quotes are escaped.
367 quote_style: enum { single, double } = .single,
368};
369
370/// Write a Unicode codepoint to the writer using the given options.
371///
372/// Returns `error.InvalidCodepoint` if `codepoint` is not a valid Unicode codepoint.
373fn writeCodepoint(
374 self: *Serializer,
375 codepoint: u21,
376 options: WriteCodepointOptions,
377) CodePointError!void {
378 switch (codepoint) {
379 // Printable ASCII
380 ' ', '!', '#'...'&', '('...'[', ']'...'~' => try self.writer.writeByte(@intCast(codepoint)),
381 // Unprintable ASCII
382 0x00...0x08, 0x0B, 0x0C, 0x0E...0x1F, 0x7F => try self.writer.print("\\x{x:0>2}", .{codepoint}),
383 // ASCII with special escapes
384 '\n' => try self.writer.writeAll("\\n"),
385 '\r' => try self.writer.writeAll("\\r"),
386 '\t' => try self.writer.writeAll("\\t"),
387 '\\' => try self.writer.writeAll("\\\\"),
388 // Quotes need escaping if they conflict with the in-use quote character
389 '\'' => if (options.quote_style == .single) try self.writer.writeAll("\\'") else try self.writer.writeByte('\''),
390 '\"' => if (options.quote_style == .double) try self.writer.writeAll("\\\"") else try self.writer.writeByte('"'),
391
392 // Surrogates can only be written with an escape
393 0xD800...0xDFFF => try self.writer.print("\\u{{{x}}}", .{codepoint}),
394 // Other valid codepoints
395 0x80...0xD7FF, 0xE000...0x10FFFF => if (options.escape_non_ascii) {
396 try self.writer.print("\\u{{{x}}}", .{codepoint});
397 } else {
398 var buf: [7]u8 = undefined;
399 const len = std.unicode.utf8Encode(codepoint, &buf) catch unreachable;
400 try self.writer.writeAll(buf[0..len]);
401 },
402 // Invalid codepoints
403 0x110000...std.math.maxInt(u21) => return error.InvalidCodepoint,
404 }
405}
406
407pub const StringOptions = struct {
408 escape_non_ascii: bool = false,
409};
410
349411/// Like `value`, but always serializes `val` as a string.
350pub fn string(self: *Serializer, val: []const u8) Error!void {
351 try self.writer.print("\"{f}\"", .{std.zig.fmtString(val)});
412pub fn string(self: *Serializer, val: []const u8, options: StringOptions) Writer.Error!void {
413 try self.writer.writeByte('"');
414 // Batch write sequences of "raw" bytes (printable ASCII or non-escaped non-ASCII) for performance.
415 // `val[start..i]` contains pending raw bytes to write.
416 var start: usize = 0;
417 var i: usize = 0;
418 while (i < val.len) {
419 const byte = val[i];
420 // Check if this byte can be written as-is
421 const is_raw = switch (byte) {
422 ' ', '!', '#'...'[', ']'...'~' => true,
423 0x80...0xFF => !options.escape_non_ascii,
424 else => false,
425 };
426 if (is_raw) {
427 i += 1;
428 continue;
429 }
430 // Flush pending raw bytes
431 try self.writer.writeAll(val[start..i]);
432 // Handle the special character
433 if (byte >= 0x80) {
434 // Decode UTF-8 sequence and write the codepoint
435 const ulen = std.unicode.utf8ByteSequenceLength(byte) catch unreachable;
436 const codepoint = std.unicode.utf8Decode(val[i..][0..ulen]) catch unreachable;
437 // InvalidCodepoint cannot occur from valid UTF-8
438 self.writeCodepoint(codepoint, .{
439 .escape_non_ascii = options.escape_non_ascii,
440 .quote_style = .double,
441 }) catch unreachable;
442 i += ulen;
443 } else {
444 // ASCII character that needs escaping
445 self.writeCodepoint(byte, .{
446 .escape_non_ascii = options.escape_non_ascii,
447 .quote_style = .double,
448 }) catch unreachable; // InvalidCodepoint cannot occur for valid ASCII values
449 i += 1;
450 }
451 start = i;
452 }
453
454 try self.writer.writeAll(val[start..]);
455 try self.writer.writeByte('"');
352456}
353457
354458/// Options for formatting multiline strings.
lib/std/zon/stringify.zig+23-15
......@@ -24,7 +24,7 @@
2424const std = @import("std");
2525const assert = std.debug.assert;
2626const Writer = std.Io.Writer;
27const Serializer = std.zon.Serializer;
27const Serializer = @import("Serializer.zig");
2828
2929pub const SerializeOptions = struct {
3030 /// If false, whitespace is omitted. Otherwise whitespace is emitted in standard Zig style.
......@@ -37,6 +37,8 @@ pub const SerializeOptions = struct {
3737 /// If false, struct fields are not written if they are equal to their default value. Comparison
3838 /// is done by `std.meta.eql`.
3939 emit_default_optional_fields: bool = true,
40 /// If true, non-ASCII unicode characters are escaped.
41 escape_non_ascii: bool = false,
4042};
4143
4244/// Serialize the given value as ZON.
......@@ -51,6 +53,7 @@ pub fn serialize(val: anytype, options: SerializeOptions, writer: *Writer) Write
5153 .emit_codepoint_literals = options.emit_codepoint_literals,
5254 .emit_strings_as_containers = options.emit_strings_as_containers,
5355 .emit_default_optional_fields = options.emit_default_optional_fields,
56 .escape_non_ascii = options.escape_non_ascii,
5457 });
5558}
5659
......@@ -72,6 +75,7 @@ pub fn serializeMaxDepth(
7275 .emit_codepoint_literals = options.emit_codepoint_literals,
7376 .emit_strings_as_containers = options.emit_strings_as_containers,
7477 .emit_default_optional_fields = options.emit_default_optional_fields,
78 .escape_non_ascii = options.escape_non_ascii,
7579 }, depth);
7680}
7781
......@@ -91,6 +95,7 @@ pub fn serializeArbitraryDepth(
9195 .emit_codepoint_literals = options.emit_codepoint_literals,
9296 .emit_strings_as_containers = options.emit_strings_as_containers,
9397 .emit_default_optional_fields = options.emit_default_optional_fields,
98 .escape_non_ascii = options.escape_non_ascii,
9499 });
95100}
96101
......@@ -588,7 +593,7 @@ test "std.zon stringify utf8 codepoints" {
588593 try std.testing.expectEqualStrings("97", aw.written());
589594 aw.clearRetainingCapacity();
590595
591 try s.codePoint('a');
596 try s.codePoint('a', .{});
592597 try std.testing.expectEqualStrings("'a'", aw.written());
593598 aw.clearRetainingCapacity();
594599
......@@ -609,7 +614,7 @@ test "std.zon stringify utf8 codepoints" {
609614 try std.testing.expectEqualStrings("10", aw.written());
610615 aw.clearRetainingCapacity();
611616
612 try s.codePoint('\n');
617 try s.codePoint('\n', .{});
613618 try std.testing.expectEqualStrings("'\\n'", aw.written());
614619 aw.clearRetainingCapacity();
615620
......@@ -630,11 +635,11 @@ test "std.zon stringify utf8 codepoints" {
630635 try std.testing.expectEqualStrings("9889", aw.written());
631636 aw.clearRetainingCapacity();
632637
633 try s.codePoint('⚡');
638 try s.codePoint('⚡', .{ .escape_non_ascii = true });
634639 try std.testing.expectEqualStrings("'\\u{26a1}'", aw.written());
635640 aw.clearRetainingCapacity();
636641
637 try s.value('⚡', .{ .emit_codepoint_literals = .always });
642 try s.value('⚡', .{ .emit_codepoint_literals = .always, .escape_non_ascii = true });
638643 try std.testing.expectEqualStrings("'\\u{26a1}'", aw.written());
639644 aw.clearRetainingCapacity();
640645
......@@ -647,8 +652,7 @@ test "std.zon stringify utf8 codepoints" {
647652 aw.clearRetainingCapacity();
648653
649654 // Invalid codepoint
650 try s.codePoint(0x110000 + 1);
651 try std.testing.expectEqualStrings("'\\u{110001}'", aw.written());
655 try std.testing.expectError(error.InvalidCodepoint, s.codePoint(0x110000 + 1, .{ .escape_non_ascii = true }));
652656 aw.clearRetainingCapacity();
653657
654658 try s.int(0x110000 + 1);
......@@ -681,7 +685,7 @@ test "std.zon stringify utf8 codepoints" {
681685 aw.clearRetainingCapacity();
682686
683687 // Make sure value options are passed to children
684 try s.value(.{ .c = '⚡' }, .{ .emit_codepoint_literals = .always });
688 try s.value(.{ .c = '⚡' }, .{ .emit_codepoint_literals = .always, .escape_non_ascii = true });
685689 try std.testing.expectEqualStrings(".{ .c = '\\u{26a1}' }", aw.written());
686690 aw.clearRetainingCapacity();
687691
......@@ -696,11 +700,11 @@ test "std.zon stringify strings" {
696700 defer aw.deinit();
697701
698702 // Minimal case
699 try s.string("abc⚡\n");
700 try std.testing.expectEqualStrings("\"abc\\xe2\\x9a\\xa1\\n\"", aw.written());
703 try s.string("abc⚡ÿ\n", .{ .escape_non_ascii = true });
704 try std.testing.expectEqualStrings("\"abc\\u{26a1}\\u{ff}\\n\"", aw.written());
701705 aw.clearRetainingCapacity();
702706
703 try s.tuple("abc⚡\n", .{});
707 try s.tuple("abc⚡ÿ\n", .{});
704708 try std.testing.expectEqualStrings(
705709 \\.{
706710 \\ 97,
......@@ -709,16 +713,18 @@ test "std.zon stringify strings" {
709713 \\ 226,
710714 \\ 154,
711715 \\ 161,
716 \\ 195,
717 \\ 191,
712718 \\ 10,
713719 \\}
714720 , aw.written());
715721 aw.clearRetainingCapacity();
716722
717 try s.value("abc⚡\n", .{});
718 try std.testing.expectEqualStrings("\"abc\\xe2\\x9a\\xa1\\n\"", aw.written());
723 try s.value("abc⚡ÿ\n", .{ .escape_non_ascii = false });
724 try std.testing.expectEqualStrings("\"abc⚡ÿ\\n\"", aw.written());
719725 aw.clearRetainingCapacity();
720726
721 try s.value("abc⚡\n", .{ .emit_strings_as_containers = true });
727 try s.value("abc⚡ÿ\n", .{ .emit_strings_as_containers = true });
722728 try std.testing.expectEqualStrings(
723729 \\.{
724730 \\ 97,
......@@ -727,6 +733,8 @@ test "std.zon stringify strings" {
727733 \\ 226,
728734 \\ 154,
729735 \\ 161,
736 \\ 195,
737 \\ 191,
730738 \\ 10,
731739 \\}
732740 , aw.written());
......@@ -816,7 +824,7 @@ test "std.zon stringify multiline strings" {
816824
817825 {
818826 const str: []const u8 = &.{ 'a', '\r', 'c' };
819 try s.string(str);
827 try s.string(str, .{ .escape_non_ascii = false });
820828 try std.testing.expectEqualStrings("\"a\\rc\"", aw.written());
821829 aw.clearRetainingCapacity();
822830 }