authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2021-03-18 14:30:11+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2021-03-18 14:30:11+02:00
logbcc97bc1ed781c261ece5ff9a33ee9c1e621b62a
treeda4f77d30bddfc5a5e6d83b4aed25c3301ec40d6
parentdfeca48e35ffd768813d8e595962005e80593e2b
parent36db4b7cc48e9e2b1bed5a977f36ef3a0f158ee8
signature Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #8247 from Ersikan/fmt-fix-encoding

zig fmt: fix non-UTF-8 encoding #2820

2 files changed, 65 insertions(+), 14 deletions(-)

src/main.zig+54-14
...@@ -2637,6 +2637,50 @@ fn argvCmd(allocator: *Allocator, argv: []const []const u8) ![]u8 {...@@ -2637,6 +2637,50 @@ fn argvCmd(allocator: *Allocator, argv: []const []const u8) ![]u8 {
2637 return cmd.toOwnedSlice();2637 return cmd.toOwnedSlice();
2638}2638}
26392639
2640fn readSourceFileToEndAlloc(allocator: *mem.Allocator, input: *const fs.File, size_hint: ?usize) ![]const u8 {
2641 const source_code = input.readToEndAllocOptions(
2642 allocator,
2643 max_src_size,
2644 size_hint,
2645 @alignOf(u16),
2646 null,
2647 ) catch |err| switch (err) {
2648 error.ConnectionResetByPeer => unreachable,
2649 error.ConnectionTimedOut => unreachable,
2650 error.NotOpenForReading => unreachable,
2651 else => |e| return e,
2652 };
2653 errdefer allocator.free(source_code);
2654
2655 // Detect unsupported file types with their Byte Order Mark
2656 const unsupported_boms = [_][]const u8{
2657 "\xff\xfe\x00\x00", // UTF-32 little endian
2658 "\xfe\xff\x00\x00", // UTF-32 big endian
2659 "\xfe\xff", // UTF-16 big endian
2660 };
2661 for (unsupported_boms) |bom| {
2662 if (mem.startsWith(u8, source_code, bom)) {
2663 return error.UnsupportedEncoding;
2664 }
2665 }
2666
2667 // If the file starts with a UTF-16 little endian BOM, translate it to UTF-8
2668 if (mem.startsWith(u8, source_code, "\xff\xfe")) {
2669 const source_code_utf16_le = mem.bytesAsSlice(u16, source_code);
2670 const source_code_utf8 = std.unicode.utf16leToUtf8Alloc(allocator, source_code_utf16_le) catch |err| switch (err) {
2671 error.DanglingSurrogateHalf => error.UnsupportedEncoding,
2672 error.ExpectedSecondSurrogateHalf => error.UnsupportedEncoding,
2673 error.UnexpectedSecondSurrogateHalf => error.UnsupportedEncoding,
2674 else => |e| return e,
2675 };
2676
2677 allocator.free(source_code);
2678 return source_code_utf8;
2679 }
2680
2681 return source_code;
2682}
2683
2640pub const usage_fmt =2684pub const usage_fmt =
2641 \\Usage: zig fmt [file]...2685 \\Usage: zig fmt [file]...
2642 \\2686 \\
...@@ -2708,9 +2752,10 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void {...@@ -2708,9 +2752,10 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void {
2708 fatal("cannot use --stdin with positional arguments", .{});2752 fatal("cannot use --stdin with positional arguments", .{});
2709 }2753 }
27102754
2711 const stdin = io.getStdIn().reader();2755 const stdin = io.getStdIn();
27122756 const source_code = readSourceFileToEndAlloc(gpa, &stdin, null) catch |err| {
2713 const source_code = try stdin.readAllAlloc(gpa, max_src_size);2757 fatal("unable to read stdin: {s}", .{err});
2758 };
2714 defer gpa.free(source_code);2759 defer gpa.free(source_code);
27152760
2716 var tree = std.zig.parse(gpa, source_code) catch |err| {2761 var tree = std.zig.parse(gpa, source_code) catch |err| {
...@@ -2785,6 +2830,7 @@ const FmtError = error{...@@ -2785,6 +2830,7 @@ const FmtError = error{
2785 EndOfStream,2830 EndOfStream,
2786 Unseekable,2831 Unseekable,
2787 NotOpenForWriting,2832 NotOpenForWriting,
2833 UnsupportedEncoding,
2788} || fs.File.OpenError;2834} || fs.File.OpenError;
27892835
2790fn fmtPath(fmt: *Fmt, file_path: []const u8, check_mode: bool, dir: fs.Dir, sub_path: []const u8) FmtError!void {2836fn fmtPath(fmt: *Fmt, file_path: []const u8, check_mode: bool, dir: fs.Dir, sub_path: []const u8) FmtError!void {
...@@ -2850,21 +2896,15 @@ fn fmtPathFile(...@@ -2850,21 +2896,15 @@ fn fmtPathFile(
2850 if (stat.kind == .Directory)2896 if (stat.kind == .Directory)
2851 return error.IsDir;2897 return error.IsDir;
28522898
2853 const source_code = source_file.readToEndAllocOptions(2899 const source_code = try readSourceFileToEndAlloc(
2854 fmt.gpa,2900 fmt.gpa,
2855 max_src_size,2901 &source_file,
2856 std.math.cast(usize, stat.size) catch return error.FileTooBig,2902 std.math.cast(usize, stat.size) catch return error.FileTooBig,
2857 @alignOf(u8),2903 );
2858 null,2904 defer fmt.gpa.free(source_code);
2859 ) catch |err| switch (err) {2905
2860 error.ConnectionResetByPeer => unreachable,
2861 error.ConnectionTimedOut => unreachable,
2862 error.NotOpenForReading => unreachable,
2863 else => |e| return e,
2864 };
2865 source_file.close();2906 source_file.close();
2866 file_closed = true;2907 file_closed = true;
2867 defer fmt.gpa.free(source_code);
28682908
2869 // Add to set after no longer possible to get error.IsDir.2909 // Add to set after no longer possible to get error.IsDir.
2870 if (try fmt.seen.fetchPut(stat.inode, {})) |_| return;2910 if (try fmt.seen.fetchPut(stat.inode, {})) |_| return;
test/cli.zig+11
...@@ -28,6 +28,8 @@ pub fn main() !void {...@@ -28,6 +28,8 @@ pub fn main() !void {
28 const zig_exe = try fs.path.resolve(a, &[_][]const u8{zig_exe_rel});28 const zig_exe = try fs.path.resolve(a, &[_][]const u8{zig_exe_rel});
2929
30 const dir_path = try fs.path.join(a, &[_][]const u8{ cache_root, "clitest" });30 const dir_path = try fs.path.join(a, &[_][]const u8{ cache_root, "clitest" });
31 defer fs.cwd().deleteTree(dir_path) catch {};
32
31 const TestFn = fn ([]const u8, []const u8) anyerror!void;33 const TestFn = fn ([]const u8, []const u8) anyerror!void;
32 const test_fns = [_]TestFn{34 const test_fns = [_]TestFn{
33 testZigInitLib,35 testZigInitLib,
...@@ -174,4 +176,13 @@ fn testZigFmt(zig_exe: []const u8, dir_path: []const u8) !void {...@@ -174,4 +176,13 @@ fn testZigFmt(zig_exe: []const u8, dir_path: []const u8) !void {
174 const run_result3 = try exec(dir_path, true, &[_][]const u8{ zig_exe, "fmt", dir_path });176 const run_result3 = try exec(dir_path, true, &[_][]const u8{ zig_exe, "fmt", dir_path });
175 // both files have been formatted, nothing should change now177 // both files have been formatted, nothing should change now
176 testing.expect(run_result3.stdout.len == 0);178 testing.expect(run_result3.stdout.len == 0);
179
180 // Check UTF-16 decoding
181 const fmt4_zig_path = try fs.path.join(a, &[_][]const u8{ dir_path, "fmt4.zig" });
182 var unformatted_code_utf16 = "\xff\xfe \x00 \x00 \x00 \x00/\x00/\x00 \x00n\x00o\x00 \x00r\x00e\x00a\x00s\x00o\x00n\x00";
183 try fs.cwd().writeFile(fmt4_zig_path, unformatted_code_utf16);
184
185 const run_result4 = try exec(dir_path, true, &[_][]const u8{ zig_exe, "fmt", dir_path });
186 testing.expect(std.mem.startsWith(u8, run_result4.stdout, fmt4_zig_path));
187 testing.expect(run_result4.stdout.len == fmt4_zig_path.len + 1 and run_result4.stdout[run_result4.stdout.len - 1] == '\n');
177}188}