authorgravatar for evan@lagerdata.comEvan Haas <evan@lagerdata.com> 2022-09-08 20:19:10-07:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2022-09-10 01:50:17+03:00
log8e631ee3e7b4e7b4466c0efafaffb4151447785f
tree1873bad537882c07d240b8ceed55e0221f81705c
parent9e070b653c89a9216f9dd9f78ed7c78c11460ac7

translate-c: Escape non-ASCII characters that appear in macros

Macro definitions are simply a slice of bytes, which may not be UTF-8 encoded. If they are not UTF-8 encoded, escape non-printable and non-ASCII characters as `\xNN`. Fixes #12784

3 files changed, 35 insertions(+), 2 deletions(-)

src/translate_c.zig+18-2
......@@ -5957,20 +5957,36 @@ fn zigifyEscapeSequences(ctx: *Context, m: *MacroCtx) ![]const u8 {
59575957 return bytes[0..i];
59585958}
59595959
5960/// non-ASCII characters (c > 127) are also treated as non-printable by fmtSliceEscapeLower.
5961/// If a C string literal or char literal in a macro is not valid UTF-8, we need to escape
5962/// non-ASCII characters so that the Zig source we output will itself be UTF-8.
5963fn escapeUnprintables(ctx: *Context, m: *MacroCtx) ![]const u8 {
5964 const zigified = try zigifyEscapeSequences(ctx, m);
5965 if (std.unicode.utf8ValidateSlice(zigified)) return zigified;
5966
5967 const formatter = std.fmt.fmtSliceEscapeLower(zigified);
5968 const encoded_size = @intCast(usize, std.fmt.count("{s}", .{formatter}));
5969 var output = try ctx.arena.alloc(u8, encoded_size);
5970 return std.fmt.bufPrint(output, "{s}", .{formatter}) catch |err| switch (err) {
5971 error.NoSpaceLeft => unreachable,
5972 else => |e| return e,
5973 };
5974}
5975
59605976fn parseCPrimaryExprInner(c: *Context, m: *MacroCtx, scope: *Scope) ParseError!Node {
59615977 const tok = m.next().?;
59625978 const slice = m.slice();
59635979 switch (tok) {
59645980 .CharLiteral => {
59655981 if (slice[0] != '\'' or slice[1] == '\\' or slice.len == 3) {
5966 return Tag.char_literal.create(c.arena, try zigifyEscapeSequences(c, m));
5982 return Tag.char_literal.create(c.arena, try escapeUnprintables(c, m));
59675983 } else {
59685984 const str = try std.fmt.allocPrint(c.arena, "0x{s}", .{std.fmt.fmtSliceHexLower(slice[1 .. slice.len - 1])});
59695985 return Tag.integer_literal.create(c.arena, str);
59705986 }
59715987 },
59725988 .StringLiteral => {
5973 return Tag.string_literal.create(c.arena, try zigifyEscapeSequences(c, m));
5989 return Tag.string_literal.create(c.arena, try escapeUnprintables(c, m));
59745990 },
59755991 .IntegerLiteral, .FloatLiteral => {
59765992 return parseCNumLit(c, m);
test/behavior/translate_c_macros.zig+12
......@@ -5,6 +5,7 @@ const expectEqual = std.testing.expectEqual;
55const expectEqualStrings = std.testing.expectEqualStrings;
66
77const h = @cImport(@cInclude("behavior/translate_c_macros.h"));
8const latin1 = @cImport(@cInclude("behavior/translate_c_macros_not_utf8.h"));
89
910test "casting to void with a macro" {
1011 h.IGNORE_ME_1(42);
......@@ -134,3 +135,14 @@ test "string literal macro with embedded tab character" {
134135
135136 try expectEqualStrings("hello\t", h.EMBEDDED_TAB);
136137}
138
139test "string and char literals that are not UTF-8 encoded. Issue #12784" {
140 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
141 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
142 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
143 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
144 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
145
146 try expectEqual(@as(u8, '\xA9'), latin1.UNPRINTABLE_CHAR);
147 try expectEqualStrings("\xA9\xA9\xA9", latin1.UNPRINTABLE_STRING);
148}
test/behavior/translate_c_macros_not_utf8.h created+5
......@@ -0,0 +1,5 @@
1// Note: This file is encoded with ISO/IEC 8859-1 (latin1), not UTF-8.
2// Do not change the encoding
3
4#define UNPRINTABLE_STRING "���"
5#define UNPRINTABLE_CHAR '�'