Zig source code is encoded in UTF-8. An invalid UTF-8 byte sequence results in a compile error.
-Throughout all zig source code (including in comments), some code points are never allowed:
+Zig source code is UTF-8 encoded. Invalid UTF-8 byte sequences are not allowed anywhere.
+Some code points are never allowed, even in {#link|Comments#}:
- LF (byte value 0x0a, code point U+000a, {#syntax#}'\n'{#endsyntax#}) is the line terminator in Zig source code. - This byte value terminates every line of zig source code except the last line of the file. - It is recommended that non-empty source files end with an empty line, which means the last byte would be 0x0a (LF). -
-- Each LF may be immediately preceded by a single CR (byte value 0x0d, code point U+000d, {#syntax#}'\r'{#endsyntax#}) - to form a Windows style line ending, but this is discouraged. Note that in multiline strings, CRLF sequences will - be encoded as LF when compiled into a zig program. - A CR in any other context is not allowed. -
-- HT hard tabs (byte value 0x09, code point U+0009, {#syntax#}'\t'{#endsyntax#}) are interchangeable with - SP spaces (byte value 0x20, code point U+0020, {#syntax#}' '{#endsyntax#}) as a token separator, - but use of hard tabs is discouraged. See {#link|Grammar#}. -
-- For compatibility with other tools, the compiler ignores a UTF-8-encoded byte order mark (U+FEFF) - if it is the first Unicode code point in the source text. A byte order mark is not allowed anywhere else in the source. -
-- Note that running zig fmt on a source file will implement all recommendations mentioned here. -
-
- Note that a tool reading Zig source code can make assumptions if the source code is assumed to be correct Zig code.
- For example, when identifying the ends of lines, a tool can use a naive search such as /\n/,
- or an advanced
- search such as /\r\n?|[\n\u0085\u2028\u2029]/, and in either case line endings will be correctly identified.
- For another example, when identifying the whitespace before the first token on a line,
- a tool can either use a naive search such as /[ \t]/,
- or an advanced search such as /\s/,
- and in either case whitespace will be correctly identified.
-
These conservative rules mean that third party tools reading + already-validated Zig source code may make simplifying assumptions, such + as naively separating lines based on {#syntax#}'\n'{#endsyntax#}. + However, tooling such as zig fmt provides convenience + functionality to convert invalid source encodings to valid source + encodings, for instance by stripping byte order marks and carriage + returns.
{#header_close#} {#header_open|Keyword Reference#} diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index b5132c6ad05787c9318d649beb2549b7282de07f..d32e3e15ad4f9bb37bf6fb9ccb358c9b1ea9f7c4 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -292,9 +292,7 @@ pub fn main(init: process.Init.Minimal) !void { try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len)); configure_argv.appendAssumeCapacity(arg); } else if (mem.eql(u8, arg, "--color")) { - const next_arg = nextArgOrFatal(args, &arg_i); - color = stringToEnum(Color, next_arg) orelse - fatalWithHint("expected [auto|on|off] found {q}", .{next_arg}); + color = nextEnumArg(args, &arg_i, Color); try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len)); configure_argv.appendAssumeCapacity(try arena.print("--color={t}", .{color})); @@ -401,25 +399,11 @@ pub fn main(init: process.Init.Minimal) !void { } else if (mem.eql(u8, arg, "--libc")) { graph.libc_file = nextArgOrFatal(args, &arg_i); } else if (mem.eql(u8, arg, "--error-style")) { - const next_arg = nextArg(args, &arg_i) orelse - fatalWithHint("expected style after {q}", .{arg}); - error_style = stringToEnum(ErrorStyle, next_arg) orelse { - fatalWithHint("expected style after {q}, found {q}", .{ arg, next_arg }); - }; + error_style = nextEnumArg(args, &arg_i, ErrorStyle); } else if (mem.eql(u8, arg, "--multiline-errors")) { - const next_arg = nextArg(args, &arg_i) orelse - fatalWithHint("expected style after {q}", .{arg}); - multiline_errors = stringToEnum(MultilineErrors, next_arg) orelse { - fatalWithHint("expected style after {q}, found {q}", .{ arg, next_arg }); - }; + multiline_errors = nextEnumArg(args, &arg_i, MultilineErrors); } else if (mem.eql(u8, arg, "--summary")) { - const next_arg = nextArg(args, &arg_i) orelse - fatalWithHint("expected [all|new|failures|line|none] after {q}", .{arg}); - summary = stringToEnum(Summary, next_arg) orelse { - fatalWithHint("expected [all|new|failures|line|none] after {q}, found {q}", .{ - arg, next_arg, - }); - }; + summary = nextEnumArg(args, &arg_i, Summary); } else if (mem.cutPrefix(u8, arg, "--seed=")) |rest| { graph.random_seed = parseRandomSeed(rest); } else if (mem.eql(u8, arg, "--build-id")) { @@ -4053,3 +4037,25 @@ fn confPathDepToCachePath( .install_include => @panic("TODO"), }; } + +fn fatalEnumHint(comptime E: type, arg: []const u8, param: ?[]const u8) noreturn { + var buf: [100]u8 = undefined; + var w: Io.Writer = .fixed(&buf); + for (@typeInfo(E).@"enum".field_names) |field_name| { + w.writeAll(field_name) catch unreachable; + w.writeByte('|') catch unreachable; + } + const buffered = w.buffered(); + const enum_options_text = buffered[0 .. buffered.len - 1]; + if (param) |p| { + fatalWithHint("expected [{s}] after {q}; found {q}", .{ enum_options_text, arg, p }); + } else { + fatalWithHint("expected [{s}] after {q}", .{ enum_options_text, arg }); + } +} + +fn nextEnumArg(args: []const []const u8, i: *usize, comptime E: type) E { + const arg = args[i.* - 1]; + const next_arg = nextArg(args, i) orelse fatalEnumHint(E, arg, null); + return stringToEnum(E, next_arg) orelse fatalEnumHint(E, arg, next_arg); +} diff --git a/lib/std/Io/Writer.zig b/lib/std/Io/Writer.zig index 7a6f4468964c4b5f8c8a1a6ff86fcd6d7a63a6b4..2bc12d27c00acd7b605409e02cadf88b09715197 100644 --- a/lib/std/Io/Writer.zig +++ b/lib/std/Io/Writer.zig @@ -584,36 +584,41 @@ pub fn writeAll(w: *Writer, bytes: []const u8) Error!void { /// required, otherwise the digit following ':' is interpreted as **width**. /// /// **specifier** supports: -/// - `x` and `X`: numeric value in hexadecimal notation, or string in hexadecimal bytes -/// - `s`: +/// - "x" and "X": numeric value in hexadecimal notation, or string in hexadecimal bytes +/// - "s": /// - for pointer-to-many and C pointers of u8, print as a C-string using zero-termination /// - for slices of u8, print the entire slice as a string without zero-termination -/// - `t`: +/// - "t": /// - for enums and tagged unions: prints the tag name /// - for error sets: prints the error name -/// - `b64`: string as standard base64 -/// - `e`: floating point value in scientific notation -/// - `d`: numeric value in decimal notation -/// - `b`: integer value in binary notation -/// - `o`: integer value in octal notation -/// - `c`: integer as an ASCII character. Integer type must have 8 bits at max. -/// - `u`: integer as an UTF-8 sequence. Integer type must have 21 bits at max. -/// - `B`: bytes in SI units (decimal) -/// - `Bi`: bytes in IEC units (binary) -/// - `?`: optional value as either the unwrapped value, or `null`; may be +/// - "b64": string as standard base64 +/// - "e": floating point value in scientific notation +/// - "d": numeric value in decimal notation +/// - "b": integer value in binary notation +/// - "o": integer value in octal notation +/// - "c": integer as an ASCII character. Integer type must have 8 bits at max. +/// - "u": integer as an UTF-8 sequence. Integer type must have 21 bits at max. +/// - "B": bytes in SI units (decimal) +/// - "Bi": bytes in IEC units (binary) +/// - "?": optional value as either the unwrapped value, or `null`; may be /// followed by a format specifier for the underlying value. -/// - `!`: error union value as either the unwrapped value, or the formatted +/// - "!": error union value as either the unwrapped value, or the formatted /// error value; may be followed by a format specifier for the underlying /// value. -/// - `*`: the address of the value instead of the value itself. -/// - `any`: a value of any type using its default format. -/// - `f`: delegates to the `format` method of the type, passing `*Writer` and +/// - "*": the address of the value instead of the value itself. +/// - "any": a value of any type using its default format. +/// - "f": delegates to the `format` method of the type, passing `*Writer` and /// expecting `Error!void` returned. -/// -/// A user type may be a struct, vector, union or enum type. +/// - "q": prints as a double-quote escaped string. Inside the double-quoted +/// string, everything is passed through unmodified, except for the following +/// transformations: +/// - escaped: '\n', '\r', '\t', '\\', '"' +/// - hex-encoded: ASCII control characters +/// - "qf": delegates to the `format` method of the type, while double-quote +/// escaping. /// /// Literal curly braces can be escaped in the format string via doubling, e.g. -/// `{{` or `}}`. +/// "{{" or "}}". pub fn print(w: *Writer, comptime fmt: []const u8, args: anytype) Error!void { const ArgsType = @TypeOf(args); const args_type_info = @typeInfo(ArgsType); @@ -1231,6 +1236,18 @@ pub fn printValue( }, else => {}, }, + 'q' => switch (fmt[1]) { + 'f' => { + try w.writeByte('"'); + var buffer: [64]u8 = undefined; + var escaping_writer: std.zig.StringEscapeWriter = .init(w, &buffer); + try value.format(&escaping_writer.writer); + try escaping_writer.writer.flush(); + try w.writeByte('"'); + return; + }, + else => {}, + }, else => {}, }, 3 => if (fmt[0] == 'b' and fmt[1] == '6' and fmt[2] == '4') switch (@typeInfo(T)) { @@ -2143,6 +2160,11 @@ test "{q} format string" { try testing.expectFmt("hello \"i\\tlike\\\"cheese\\x00\\x05cheese\" world", "hello {q} world", .{data}); } +test "{qf} format string" { + const data: []const u8 = "😎"; + try testing.expectFmt("hello \"@\\\"😎\\\"\" world", "hello {qf} world", .{std.zig.fmtId(data)}); +} + fn testPrintIntCase(expected: []const u8, value: anytype, base: u8, case: std.fmt.Case, options: std.fmt.Options) !void { var buffer: [100]u8 = undefined; var w: Writer = .fixed(&buffer); diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 81baffb36bc70dd6551b4acfd360d2a811db2629..e18a6414c6e9ba130640e061307c68b16ff02456 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -538,21 +538,83 @@ test fmtChar { } /// Print the string as escaped contents of a double quoted string. +/// +/// The following transformations are made: +/// * escaped: '\n', '\r', '\t', '\\', '"' +/// * hex-encoded: ascii control characters +/// +/// Everything else is passed through unmodified. pub fn stringEscape(bytes: []const u8, w: *Writer) Writer.Error!void { + _ = try stringEscapeCounting(bytes, w); +} + +pub fn stringEscapeCounting(bytes: []const u8, w: *Writer) Writer.Error!usize { + var n: usize = 0; for (bytes) |byte| switch (byte) { - '\n' => try w.writeAll("\\n"), - '\r' => try w.writeAll("\\r"), - '\t' => try w.writeAll("\\t"), - '\\' => try w.writeAll("\\\\"), - '"' => try w.writeAll("\\\""), - ' ', '!', '#'...'[', ']'...'~' => try w.writeByte(byte), - else => { + '\t' => { + try w.writeAll("\\t"); + n += 2; + }, + '\n' => { + try w.writeAll("\\n"); + n += 2; + }, + '\r' => { + try w.writeAll("\\r"); + n += 2; + }, + '\\' => { + try w.writeAll("\\\\"); + n += 2; + }, + '"' => { + try w.writeAll("\\\""); + n += 2; + }, + 0...8, 11, 12, 14...0x1f, 0x7f => { try w.writeAll("\\x"); try w.printInt(byte, 16, .lower, .{ .width = 2, .fill = '0' }); + n += 4; + }, + else => { + try w.writeByte(byte); + n += 1; }, }; + return n; } +pub const StringEscapeWriter = struct { + out: *Writer, + writer: Writer, + + pub fn init(out: *Writer, buffer: []u8) @This() { + return .{ + .out = out, + .writer = .{ + .vtable = &.{ .drain = @This().drain }, + .buffer = buffer, + }, + }; + } + + fn drain(w: *Writer, data: []const []const u8, splat: usize) Io.Writer.Error!usize { + const sew: *StringEscapeWriter = @alignCast(@fieldParentPtr("writer", w)); + const out = sew.out; + _ = try stringEscapeCounting(w.buffered(), out); + w.end = 0; + var n: usize = 0; + for (data[0 .. data.len - 1]) |bytes| { + n += try stringEscapeCounting(bytes, out); + } + const pattern = data[data.len - 1]; + for (0..splat) |_| { + n += try stringEscapeCounting(pattern, out); + } + return n; + } +}; + /// Print as escaped contents of a single-quoted string. pub fn charEscape(codepoint: u21, w: *Writer) Writer.Error!void { switch (codepoint) { diff --git a/lib/std/zig/parser_generated_oracle.zig b/lib/std/zig/parser_generated_oracle.zig index 1cd16d6ce1495e8587fd16feb019f23501c90f3d..79de64f7298ad6f141babcea868bdef5cae4c474 100644 --- a/lib/std/zig/parser_generated_oracle.zig +++ b/lib/std/zig/parser_generated_oracle.zig @@ -3073,7 +3073,7 @@ const Parser = struct { return blk_0: { const pos_0 = p.i; if (blk_1: { - if (std.mem.startsWith(u8, p.source[p.i..], "\xef\xbb\xbf")) { + if (std.mem.startsWith(u8, p.source[p.i..], "")) { p.i += 3; break :blk_1 true; } @@ -3145,7 +3145,7 @@ const Parser = struct { return blk_0: { const pos_0 = p.i; if (blk_1: { - if (std.mem.startsWith(u8, p.source[p.i..], "\xf4")) { + if (std.mem.startsWith(u8, p.source[p.i..], "ô")) { p.i += 1; break :blk_1 true; } @@ -3201,7 +3201,7 @@ const Parser = struct { return blk_0: { const pos_0 = p.i; if (blk_1: { - if (std.mem.startsWith(u8, p.source[p.i..], "\xf0")) { + if (std.mem.startsWith(u8, p.source[p.i..], "ð")) { p.i += 1; break :blk_1 true; } @@ -3257,7 +3257,7 @@ const Parser = struct { return blk_0: { const pos_0 = p.i; if (blk_1: { - if (std.mem.startsWith(u8, p.source[p.i..], "\xed")) { + if (std.mem.startsWith(u8, p.source[p.i..], "í")) { p.i += 1; break :blk_1 true; } @@ -3313,7 +3313,7 @@ const Parser = struct { return blk_0: { const pos_0 = p.i; if (blk_1: { - if (std.mem.startsWith(u8, p.source[p.i..], "\xe0")) { + if (std.mem.startsWith(u8, p.source[p.i..], "à")) { p.i += 1; break :blk_1 true; } diff --git a/tools/gen_parser_oracle.zig b/tools/gen_parser_oracle.zig index e20b3bfc15f9e376370778c21061d4363d1dd6ba..866c266d82b6284d506aaa2dbf0ae4d11136c987 100644 --- a/tools/gen_parser_oracle.zig +++ b/tools/gen_parser_oracle.zig @@ -267,17 +267,14 @@ const Generator = struct { const bytes = g.p.strings.items[literal.off..][0..literal.len]; try g.w.print( \\blk_{d}: {{ - \\if (std.mem.startsWith(u8, p.source[p.i..], " - , .{suffix}); - try std.zig.stringEscape(bytes, g.w); - try g.w.print( - \\")) {{ + \\if (std.mem.startsWith(u8, p.source[p.i..], {q})) {{ + \\ \\p.i += {d}; \\ break :blk_{d} true; \\}} \\break :blk_{d} false; \\}} - , .{ bytes.len, suffix, suffix }); + , .{ suffix, bytes, bytes.len, suffix, suffix }); }, .class => |ranges| { try g.w.writeAll("(p.i < p.source.len and switch (p.source[p.i]) {");