authorgravatar for julien.philippon@epitech.euErsikan <julien.philippon@epitech.eu> 2021-03-14 07:03:22+01:00
committergravatar for julien.philippon@epitech.euErsikan <julien.philippon@epitech.eu> 2021-03-17 10:26:47+01:00
loga3540000909bdc6a59ba07c85d21afeb3a7e54e2
treed671cf5efd931c7b89af4c6fb8612efae085d645
parentf76bd56588e556ea580c1faa63667cc9264cc218

zig fmt: fix non-UTF-8 encoding #2820

Fixes #2820 After reading the source code, the first two bytes are inspected, and if they correspond to a UTF-16 BOM in little-endian order, the source code is converted to UTF-8.

2 files changed, 57 insertions(+), 16 deletions(-)

src/main.zig+48-16
...@@ -2708,9 +2708,22 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void {...@@ -2708,9 +2708,22 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void {
2708 fatal("cannot use --stdin with positional arguments", .{});2708 fatal("cannot use --stdin with positional arguments", .{});
2709 }2709 }
27102710
2711 const stdin = io.getStdIn().reader();2711 const stdin = io.getStdIn();
27122712
2713 const source_code = try stdin.readAllAlloc(gpa, max_src_size);2713 const source_code = blk: {
2714 const source_code = try stdin.readToEndAllocOptions(gpa, max_src_size, null, @alignOf(u16), null);
2715 errdefer gpa.free(source_code);
2716
2717 // If the file starts with a UTF-16 BOM, translate it to UTF-8
2718 if (mem.startsWith(u8, source_code, "\xff\xfe")) {
2719 const source_code_utf16_le = mem.bytesAsSlice(u16, source_code);
2720 const source_code_utf8 = try std.unicode.utf16leToUtf8Alloc(gpa, source_code_utf16_le);
2721 gpa.free(source_code);
2722 break :blk source_code_utf8;
2723 } else {
2724 break :blk source_code;
2725 }
2726 };
2714 defer gpa.free(source_code);2727 defer gpa.free(source_code);
27152728
2716 var tree = std.zig.parse(gpa, source_code) catch |err| {2729 var tree = std.zig.parse(gpa, source_code) catch |err| {
...@@ -2785,6 +2798,7 @@ const FmtError = error{...@@ -2785,6 +2798,7 @@ const FmtError = error{
2785 EndOfStream,2798 EndOfStream,
2786 Unseekable,2799 Unseekable,
2787 NotOpenForWriting,2800 NotOpenForWriting,
2801 UnknownTextFormat,
2788} || fs.File.OpenError;2802} || fs.File.OpenError;
27892803
2790fn fmtPath(fmt: *Fmt, file_path: []const u8, check_mode: bool, dir: fs.Dir, sub_path: []const u8) FmtError!void {2804fn fmtPath(fmt: *Fmt, file_path: []const u8, check_mode: bool, dir: fs.Dir, sub_path: []const u8) FmtError!void {
...@@ -2850,20 +2864,38 @@ fn fmtPathFile(...@@ -2850,20 +2864,38 @@ fn fmtPathFile(
2850 if (stat.kind == .Directory)2864 if (stat.kind == .Directory)
2851 return error.IsDir;2865 return error.IsDir;
28522866
2853 const source_code = source_file.readToEndAllocOptions(2867 const source_code = blk: {
2854 fmt.gpa,2868 const source_code = source_file.readToEndAllocOptions(
2855 max_src_size,2869 fmt.gpa,
2856 std.math.cast(usize, stat.size) catch return error.FileTooBig,2870 max_src_size,
2857 @alignOf(u8),2871 std.math.cast(usize, stat.size) catch return error.FileTooBig,
2858 null,2872 @alignOf(u16),
2859 ) catch |err| switch (err) {2873 null,
2860 error.ConnectionResetByPeer => unreachable,2874 ) catch |err| switch (err) {
2861 error.ConnectionTimedOut => unreachable,2875 error.ConnectionResetByPeer => unreachable,
2862 error.NotOpenForReading => unreachable,2876 error.ConnectionTimedOut => unreachable,
2863 else => |e| return e,2877 error.NotOpenForReading => unreachable,
2878 else => |e| return e,
2879 };
2880 source_file.close();
2881 file_closed = true;
2882 errdefer fmt.gpa.free(source_code);
2883
2884 // If the file starts with a UTF-16 BOM, translate it to UTF-8
2885 if (mem.eql(u8, source_code[0..2], "\xff\xfe")) {
2886 const source_code_utf16_le = mem.bytesAsSlice(u16, source_code);
2887 const source_code_utf8 = std.unicode.utf16leToUtf8Alloc(fmt.gpa, source_code_utf16_le) catch |err| return switch (err) {
2888 error.DanglingSurrogateHalf => FmtError.UnknownTextFormat,
2889 error.ExpectedSecondSurrogateHalf => FmtError.UnknownTextFormat,
2890 error.UnexpectedSecondSurrogateHalf => FmtError.UnknownTextFormat,
2891 else => |e| e,
2892 };
2893 fmt.gpa.free(source_code);
2894 break :blk source_code_utf8;
2895 } else {
2896 break :blk source_code;
2897 }
2864 };2898 };
2865 source_file.close();
2866 file_closed = true;
2867 defer fmt.gpa.free(source_code);2899 defer fmt.gpa.free(source_code);
28682900
2869 // Add to set after no longer possible to get error.IsDir.2901 // Add to set after no longer possible to get error.IsDir.
test/cli.zig+9
...@@ -174,4 +174,13 @@ fn testZigFmt(zig_exe: []const u8, dir_path: []const u8) !void {...@@ -174,4 +174,13 @@ fn testZigFmt(zig_exe: []const u8, dir_path: []const u8) !void {
174 const run_result3 = try exec(dir_path, true, &[_][]const u8{ zig_exe, "fmt", dir_path });174 const run_result3 = try exec(dir_path, true, &[_][]const u8{ zig_exe, "fmt", dir_path });
175 // both files have been formatted, nothing should change now175 // both files have been formatted, nothing should change now
176 testing.expect(run_result3.stdout.len == 0);176 testing.expect(run_result3.stdout.len == 0);
177
178 // Check UTF-16 decoding
179 const fmt4_zig_path = try fs.path.join(a, &[_][]const u8{ dir_path, "fmt4.zig" });
180 var unformatted_code_utf16 = "\xff\xfe \x00 \x00 \x00 \x00/\x00/\x00 \x00n\x00o\x00 \x00r\x00e\x00a\x00s\x00o\x00n\x00";
181 try fs.cwd().writeFile(fmt4_zig_path, unformatted_code_utf16);
182
183 const run_result4 = try exec(dir_path, true, &[_][]const u8{ zig_exe, "fmt", dir_path });
184 testing.expect(std.mem.startsWith(u8, run_result4.stdout, fmt4_zig_path));
185 testing.expect(run_result4.stdout.len == fmt4_zig_path.len + 1 and run_result4.stdout[run_result4.stdout.len - 1] == '\n');
177}186}