authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2021-03-18 14:30:11+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2021-03-18 14:30:11+02:00
logbcc97bc1ed781c261ece5ff9a33ee9c1e621b62a
treeda4f77d30bddfc5a5e6d83b4aed25c3301ec40d6
parentdfeca48e35ffd768813d8e595962005e80593e2b
parent36db4b7cc48e9e2b1bed5a977f36ef3a0f158ee8
signature Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #8247 from Ersikan/fmt-fix-encoding

zig fmt: fix non-UTF-8 encoding #2820

2 files changed, 65 insertions(+), 14 deletions(-)

src/main.zig+54-14
......@@ -2637,6 +2637,50 @@ fn argvCmd(allocator: *Allocator, argv: []const []const u8) ![]u8 {
26372637 return cmd.toOwnedSlice();
26382638}
26392639
2640fn readSourceFileToEndAlloc(allocator: *mem.Allocator, input: *const fs.File, size_hint: ?usize) ![]const u8 {
2641 const source_code = input.readToEndAllocOptions(
2642 allocator,
2643 max_src_size,
2644 size_hint,
2645 @alignOf(u16),
2646 null,
2647 ) catch |err| switch (err) {
2648 error.ConnectionResetByPeer => unreachable,
2649 error.ConnectionTimedOut => unreachable,
2650 error.NotOpenForReading => unreachable,
2651 else => |e| return e,
2652 };
2653 errdefer allocator.free(source_code);
2654
2655 // Detect unsupported file types with their Byte Order Mark
2656 const unsupported_boms = [_][]const u8{
2657 "\xff\xfe\x00\x00", // UTF-32 little endian
2658 "\xfe\xff\x00\x00", // UTF-32 big endian
2659 "\xfe\xff", // UTF-16 big endian
2660 };
2661 for (unsupported_boms) |bom| {
2662 if (mem.startsWith(u8, source_code, bom)) {
2663 return error.UnsupportedEncoding;
2664 }
2665 }
2666
2667 // If the file starts with a UTF-16 little endian BOM, translate it to UTF-8
2668 if (mem.startsWith(u8, source_code, "\xff\xfe")) {
2669 const source_code_utf16_le = mem.bytesAsSlice(u16, source_code);
2670 const source_code_utf8 = std.unicode.utf16leToUtf8Alloc(allocator, source_code_utf16_le) catch |err| switch (err) {
2671 error.DanglingSurrogateHalf => error.UnsupportedEncoding,
2672 error.ExpectedSecondSurrogateHalf => error.UnsupportedEncoding,
2673 error.UnexpectedSecondSurrogateHalf => error.UnsupportedEncoding,
2674 else => |e| return e,
2675 };
2676
2677 allocator.free(source_code);
2678 return source_code_utf8;
2679 }
2680
2681 return source_code;
2682}
2683
26402684pub const usage_fmt =
26412685 \\Usage: zig fmt [file]...
26422686 \\
......@@ -2708,9 +2752,10 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void {
27082752 fatal("cannot use --stdin with positional arguments", .{});
27092753 }
27102754
2711 const stdin = io.getStdIn().reader();
2712
2713 const source_code = try stdin.readAllAlloc(gpa, max_src_size);
2755 const stdin = io.getStdIn();
2756 const source_code = readSourceFileToEndAlloc(gpa, &stdin, null) catch |err| {
2757 fatal("unable to read stdin: {s}", .{err});
2758 };
27142759 defer gpa.free(source_code);
27152760
27162761 var tree = std.zig.parse(gpa, source_code) catch |err| {
......@@ -2785,6 +2830,7 @@ const FmtError = error{
27852830 EndOfStream,
27862831 Unseekable,
27872832 NotOpenForWriting,
2833 UnsupportedEncoding,
27882834} || fs.File.OpenError;
27892835
27902836fn fmtPath(fmt: *Fmt, file_path: []const u8, check_mode: bool, dir: fs.Dir, sub_path: []const u8) FmtError!void {
......@@ -2850,21 +2896,15 @@ fn fmtPathFile(
28502896 if (stat.kind == .Directory)
28512897 return error.IsDir;
28522898
2853 const source_code = source_file.readToEndAllocOptions(
2899 const source_code = try readSourceFileToEndAlloc(
28542900 fmt.gpa,
2855 max_src_size,
2901 &source_file,
28562902 std.math.cast(usize, stat.size) catch return error.FileTooBig,
2857 @alignOf(u8),
2858 null,
2859 ) catch |err| switch (err) {
2860 error.ConnectionResetByPeer => unreachable,
2861 error.ConnectionTimedOut => unreachable,
2862 error.NotOpenForReading => unreachable,
2863 else => |e| return e,
2864 };
2903 );
2904 defer fmt.gpa.free(source_code);
2905
28652906 source_file.close();
28662907 file_closed = true;
2867 defer fmt.gpa.free(source_code);
28682908
28692909 // Add to set after no longer possible to get error.IsDir.
28702910 if (try fmt.seen.fetchPut(stat.inode, {})) |_| return;
test/cli.zig+11
......@@ -28,6 +28,8 @@ pub fn main() !void {
2828 const zig_exe = try fs.path.resolve(a, &[_][]const u8{zig_exe_rel});
2929
3030 const dir_path = try fs.path.join(a, &[_][]const u8{ cache_root, "clitest" });
31 defer fs.cwd().deleteTree(dir_path) catch {};
32
3133 const TestFn = fn ([]const u8, []const u8) anyerror!void;
3234 const test_fns = [_]TestFn{
3335 testZigInitLib,
......@@ -174,4 +176,13 @@ fn testZigFmt(zig_exe: []const u8, dir_path: []const u8) !void {
174176 const run_result3 = try exec(dir_path, true, &[_][]const u8{ zig_exe, "fmt", dir_path });
175177 // both files have been formatted, nothing should change now
176178 testing.expect(run_result3.stdout.len == 0);
179
180 // Check UTF-16 decoding
181 const fmt4_zig_path = try fs.path.join(a, &[_][]const u8{ dir_path, "fmt4.zig" });
182 var unformatted_code_utf16 = "\xff\xfe \x00 \x00 \x00 \x00/\x00/\x00 \x00n\x00o\x00 \x00r\x00e\x00a\x00s\x00o\x00n\x00";
183 try fs.cwd().writeFile(fmt4_zig_path, unformatted_code_utf16);
184
185 const run_result4 = try exec(dir_path, true, &[_][]const u8{ zig_exe, "fmt", dir_path });
186 testing.expect(std.mem.startsWith(u8, run_result4.stdout, fmt4_zig_path));
187 testing.expect(run_result4.stdout.len == fmt4_zig_path.len + 1 and run_result4.stdout[run_result4.stdout.len - 1] == '\n');
177188}