authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-06-01 14:00:54-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2021-06-01 14:00:54-04:00
logf6932472950e7dbb451d4cfef8e5f4a1cc506ac0
tree986353a4f281ca362f48e9aec31af691d85c8019
parentd496400cff8b025dea262a9544e1b20482233089
parentabd1c75c4aa70a83884e0509820dff8a6e51430c
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #8917 from ifreund/fix-float-tokenize2

stage1, stage2: disallow 1.e9 and 0x1.p9 as float literals

10 files changed, 32 insertions(+), 36 deletions(-)

lib/std/fmt.zig+9-6
...@@ -1121,9 +1121,12 @@ pub fn formatFloatHexadecimal(...@@ -1121,9 +1121,12 @@ pub fn formatFloatHexadecimal(
11211121
1122 try writer.writeAll("0x");1122 try writer.writeAll("0x");
1123 try writer.writeByte(buf[0]);1123 try writer.writeByte(buf[0]);
1124 if (options.precision != @as(usize, 0))
1125 try writer.writeAll(".");
1126 const trimmed = mem.trimRight(u8, buf[1..], "0");1124 const trimmed = mem.trimRight(u8, buf[1..], "0");
1125 if (options.precision) |precision| {
1126 if (precision > 0) try writer.writeAll(".");
1127 } else if (trimmed.len > 0) {
1128 try writer.writeAll(".");
1129 }
1127 try writer.writeAll(trimmed);1130 try writer.writeAll(trimmed);
1128 // Add trailing zeros if explicitly requested.1131 // Add trailing zeros if explicitly requested.
1129 if (options.precision) |precision| if (precision > 0) {1132 if (options.precision) |precision| if (precision > 0) {
...@@ -2048,10 +2051,10 @@ test "float.hexadecimal" {...@@ -2048,10 +2051,10 @@ test "float.hexadecimal" {
2048 try expectFmt("f64: 0x1.5555555555555p-2", "f64: {x}", .{@as(f64, 1.0 / 3.0)});2051 try expectFmt("f64: 0x1.5555555555555p-2", "f64: {x}", .{@as(f64, 1.0 / 3.0)});
2049 try expectFmt("f128: 0x1.5555555555555555555555555555p-2", "f128: {x}", .{@as(f128, 1.0 / 3.0)});2052 try expectFmt("f128: 0x1.5555555555555555555555555555p-2", "f128: {x}", .{@as(f128, 1.0 / 3.0)});
20502053
2051 try expectFmt("f16: 0x1.p-14", "f16: {x}", .{@as(f16, math.f16_min)});2054 try expectFmt("f16: 0x1p-14", "f16: {x}", .{@as(f16, math.f16_min)});
2052 try expectFmt("f32: 0x1.p-126", "f32: {x}", .{@as(f32, math.f32_min)});2055 try expectFmt("f32: 0x1p-126", "f32: {x}", .{@as(f32, math.f32_min)});
2053 try expectFmt("f64: 0x1.p-1022", "f64: {x}", .{@as(f64, math.f64_min)});2056 try expectFmt("f64: 0x1p-1022", "f64: {x}", .{@as(f64, math.f64_min)});
2054 try expectFmt("f128: 0x1.p-16382", "f128: {x}", .{@as(f128, math.f128_min)});2057 try expectFmt("f128: 0x1p-16382", "f128: {x}", .{@as(f128, math.f128_min)});
20552058
2056 try expectFmt("f16: 0x0.004p-14", "f16: {x}", .{@as(f16, math.f16_true_min)});2059 try expectFmt("f16: 0x0.004p-14", "f16: {x}", .{@as(f16, math.f16_true_min)});
2057 try expectFmt("f32: 0x0.000002p-126", "f32: {x}", .{@as(f32, math.f32_true_min)});2060 try expectFmt("f32: 0x0.000002p-126", "f32: {x}", .{@as(f32, math.f32_true_min)});
lib/std/special/compiler_rt/fixunstfdi_test.zig+1-1
...@@ -46,7 +46,7 @@ test "fixunstfdi" {...@@ -46,7 +46,7 @@ test "fixunstfdi" {
46 try test__fixunstfdi(0x1.0000000000000000p+63, 0x8000000000000000);46 try test__fixunstfdi(0x1.0000000000000000p+63, 0x8000000000000000);
47 try test__fixunstfdi(0x1.FFFFFFFFFFFFFFFCp+62, 0x7FFFFFFFFFFFFFFF);47 try test__fixunstfdi(0x1.FFFFFFFFFFFFFFFCp+62, 0x7FFFFFFFFFFFFFFF);
48 try test__fixunstfdi(0x1.FFFFFFFFFFFFFFF8p+62, 0x7FFFFFFFFFFFFFFE);48 try test__fixunstfdi(0x1.FFFFFFFFFFFFFFF8p+62, 0x7FFFFFFFFFFFFFFE);
49 try test__fixunstfdi(0x1.p+64, 0xFFFFFFFFFFFFFFFF);49 try test__fixunstfdi(0x1p+64, 0xFFFFFFFFFFFFFFFF);
5050
51 try test__fixunstfdi(-0x1.0000000000000000p+63, 0);51 try test__fixunstfdi(-0x1.0000000000000000p+63, 0);
52 try test__fixunstfdi(-0x1.FFFFFFFFFFFFFFFCp+62, 0);52 try test__fixunstfdi(-0x1.FFFFFFFFFFFFFFFCp+62, 0);
lib/std/special/compiler_rt/fixunstfsi_test.zig+1-1
...@@ -23,5 +23,5 @@ test "fixunstfsi" {...@@ -23,5 +23,5 @@ test "fixunstfsi" {
23 try test__fixunstfsi(0x1.23456789abcdefp+256, 0xffffffff);23 try test__fixunstfsi(0x1.23456789abcdefp+256, 0xffffffff);
24 try test__fixunstfsi(-0x1.23456789abcdefp+3, 0x0);24 try test__fixunstfsi(-0x1.23456789abcdefp+3, 0x0);
2525
26 try test__fixunstfsi(0x1.p+32, 0xFFFFFFFF);26 try test__fixunstfsi(0x1p+32, 0xFFFFFFFF);
27}27}
lib/std/special/compiler_rt/fixunstfti_test.zig+1-1
...@@ -28,7 +28,7 @@ test "fixunstfti" {...@@ -28,7 +28,7 @@ test "fixunstfti" {
28 try test__fixunstfti(-0.01, 0);28 try test__fixunstfti(-0.01, 0);
29 try test__fixunstfti(-0.99, 0);29 try test__fixunstfti(-0.99, 0);
3030
31 try test__fixunstfti(0x1.p+128, 0xffffffffffffffffffffffffffffffff);31 try test__fixunstfti(0x1p+128, 0xffffffffffffffffffffffffffffffff);
3232
33 try test__fixunstfti(0x1.FFFFFEp+126, 0x7fffff80000000000000000000000000);33 try test__fixunstfti(0x1.FFFFFEp+126, 0x7fffff80000000000000000000000000);
34 try test__fixunstfti(0x1.FFFFFEp+127, 0xffffff00000000000000000000000000);34 try test__fixunstfti(0x1.FFFFFEp+127, 0xffffff00000000000000000000000000);
lib/std/special/compiler_rt/mulXf3_test.zig+2-2
...@@ -92,12 +92,12 @@ test "multf3" {...@@ -92,12 +92,12 @@ test "multf3" {
92 // Denormal operands.92 // Denormal operands.
93 try test__multf3(93 try test__multf3(
94 0x0.0000000000000000000000000001p-16382,94 0x0.0000000000000000000000000001p-16382,
95 0x1.p16383,95 0x1p16383,
96 0x3f90000000000000,96 0x3f90000000000000,
97 0x0,97 0x0,
98 );98 );
99 try test__multf3(99 try test__multf3(
100 0x1.p16383,100 0x1p16383,
101 0x0.0000000000000000000000000001p-16382,101 0x0.0000000000000000000000000001p-16382,
102 0x3f90000000000000,102 0x3f90000000000000,
103 0x0,103 0x0,
lib/std/zig/parser_test.zig+4-4
...@@ -4152,13 +4152,13 @@ test "zig fmt: hex literals with underscore separators" {...@@ -4152,13 +4152,13 @@ test "zig fmt: hex literals with underscore separators" {
4152test "zig fmt: decimal float literals with underscore separators" {4152test "zig fmt: decimal float literals with underscore separators" {
4153 try testTransform(4153 try testTransform(
4154 \\pub fn main() void {4154 \\pub fn main() void {
4155 \\ const a:f64=(10.0e-0+(10.e+0))+10_00.00_00e-2+00_00.00_10e+4;4155 \\ const a:f64=(10.0e-0+(10.0e+0))+10_00.00_00e-2+00_00.00_10e+4;
4156 \\ const b:f64=010.0--0_10.0+0_1_0.0_0+1e2;4156 \\ const b:f64=010.0--0_10.0+0_1_0.0_0+1e2;
4157 \\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b });4157 \\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b });
4158 \\}4158 \\}
4159 ,4159 ,
4160 \\pub fn main() void {4160 \\pub fn main() void {
4161 \\ const a: f64 = (10.0e-0 + (10.e+0)) + 10_00.00_00e-2 + 00_00.00_10e+4;4161 \\ const a: f64 = (10.0e-0 + (10.0e+0)) + 10_00.00_00e-2 + 00_00.00_10e+4;
4162 \\ const b: f64 = 010.0 - -0_10.0 + 0_1_0.0_0 + 1e2;4162 \\ const b: f64 = 010.0 - -0_10.0 + 0_1_0.0_0 + 1e2;
4163 \\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b });4163 \\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b });
4164 \\}4164 \\}
...@@ -4169,13 +4169,13 @@ test "zig fmt: decimal float literals with underscore separators" {...@@ -4169,13 +4169,13 @@ test "zig fmt: decimal float literals with underscore separators" {
4169test "zig fmt: hexadeciaml float literals with underscore separators" {4169test "zig fmt: hexadeciaml float literals with underscore separators" {
4170 try testTransform(4170 try testTransform(
4171 \\pub fn main() void {4171 \\pub fn main() void {
4172 \\ const a: f64 = (0x10.0p-0+(0x10.p+0))+0x10_00.00_00p-8+0x00_00.00_10p+16;4172 \\ const a: f64 = (0x10.0p-0+(0x10.0p+0))+0x10_00.00_00p-8+0x00_00.00_10p+16;
4173 \\ const b: f64 = 0x0010.0--0x00_10.0+0x10.00+0x1p4;4173 \\ const b: f64 = 0x0010.0--0x00_10.0+0x10.00+0x1p4;
4174 \\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b });4174 \\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b });
4175 \\}4175 \\}
4176 ,4176 ,
4177 \\pub fn main() void {4177 \\pub fn main() void {
4178 \\ const a: f64 = (0x10.0p-0 + (0x10.p+0)) + 0x10_00.00_00p-8 + 0x00_00.00_10p+16;4178 \\ const a: f64 = (0x10.0p-0 + (0x10.0p+0)) + 0x10_00.00_00p-8 + 0x00_00.00_10p+16;
4179 \\ const b: f64 = 0x0010.0 - -0x00_10.0 + 0x10.00 + 0x1p4;4179 \\ const b: f64 = 0x0010.0 - -0x00_10.0 + 0x10.00 + 0x1p4;
4180 \\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b });4180 \\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b });
4181 \\}4181 \\}
lib/std/zig/tokenizer.zig+3-11
...@@ -1210,10 +1210,6 @@ pub const Tokenizer = struct {...@@ -1210,10 +1210,6 @@ pub const Tokenizer = struct {
1210 state = .start;1210 state = .start;
1211 break;1211 break;
1212 },1212 },
1213 'e', 'E' => {
1214 result.tag = .float_literal;
1215 state = .float_exponent_unsigned;
1216 },
1217 '0'...'9' => {1213 '0'...'9' => {
1218 result.tag = .float_literal;1214 result.tag = .float_literal;
1219 state = .float_fraction_dec;1215 state = .float_fraction_dec;
...@@ -1232,10 +1228,6 @@ pub const Tokenizer = struct {...@@ -1232,10 +1228,6 @@ pub const Tokenizer = struct {
1232 state = .start;1228 state = .start;
1233 break;1229 break;
1234 },1230 },
1235 'p', 'P' => {
1236 result.tag = .float_literal;
1237 state = .float_exponent_unsigned;
1238 },
1239 '0'...'9', 'a'...'f', 'A'...'F' => {1231 '0'...'9', 'a'...'f', 'A'...'F' => {
1240 result.tag = .float_literal;1232 result.tag = .float_literal;
1241 state = .float_fraction_hex;1233 state = .float_fraction_hex;
...@@ -1861,7 +1853,6 @@ test "tokenizer - number literals decimal" {...@@ -1861,7 +1853,6 @@ test "tokenizer - number literals decimal" {
1861 try testTokenize("0e0", &.{.float_literal});1853 try testTokenize("0e0", &.{.float_literal});
1862 try testTokenize("1e0", &.{.float_literal});1854 try testTokenize("1e0", &.{.float_literal});
1863 try testTokenize("1e100", &.{.float_literal});1855 try testTokenize("1e100", &.{.float_literal});
1864 try testTokenize("1.e100", &.{.float_literal});
1865 try testTokenize("1.0e100", &.{.float_literal});1856 try testTokenize("1.0e100", &.{.float_literal});
1866 try testTokenize("1.0e+100", &.{.float_literal});1857 try testTokenize("1.0e+100", &.{.float_literal});
1867 try testTokenize("1.0e-100", &.{.float_literal});1858 try testTokenize("1.0e-100", &.{.float_literal});
...@@ -1869,6 +1860,7 @@ test "tokenizer - number literals decimal" {...@@ -1869,6 +1860,7 @@ test "tokenizer - number literals decimal" {
18691860
1870 try testTokenize("1.", &.{.invalid});1861 try testTokenize("1.", &.{.invalid});
1871 try testTokenize("1e", &.{.invalid});1862 try testTokenize("1e", &.{.invalid});
1863 try testTokenize("1.e100", &.{ .invalid, .identifier });
1872 try testTokenize("1.0e1f0", &.{ .invalid, .identifier });1864 try testTokenize("1.0e1f0", &.{ .invalid, .identifier });
1873 try testTokenize("1.0p100", &.{ .invalid, .identifier });1865 try testTokenize("1.0p100", &.{ .invalid, .identifier });
1874 try testTokenize("1.0p-100", &.{ .invalid, .identifier, .minus, .integer_literal });1866 try testTokenize("1.0p-100", &.{ .invalid, .identifier, .minus, .integer_literal });
...@@ -2019,6 +2011,7 @@ test "tokenizer - number literals hexadecimal" {...@@ -2019,6 +2011,7 @@ test "tokenizer - number literals hexadecimal" {
2019 try testTokenize("0x1.", &.{.invalid});2011 try testTokenize("0x1.", &.{.invalid});
2020 try testTokenize("0xF.", &.{.invalid});2012 try testTokenize("0xF.", &.{.invalid});
2021 try testTokenize("0x1.+0xF.", &.{ .invalid, .plus, .invalid });2013 try testTokenize("0x1.+0xF.", &.{ .invalid, .plus, .invalid });
2014 try testTokenize("0xff.p10", &.{ .invalid, .identifier });
20222015
2023 try testTokenize("0x0123456.789ABCDEF", &.{.float_literal});2016 try testTokenize("0x0123456.789ABCDEF", &.{.float_literal});
2024 try testTokenize("0x0_123_456.789_ABC_DEF", &.{.float_literal});2017 try testTokenize("0x0_123_456.789_ABC_DEF", &.{.float_literal});
...@@ -2027,7 +2020,6 @@ test "tokenizer - number literals hexadecimal" {...@@ -2027,7 +2020,6 @@ test "tokenizer - number literals hexadecimal" {
2027 try testTokenize("0x0.0p0", &.{.float_literal});2020 try testTokenize("0x0.0p0", &.{.float_literal});
2028 try testTokenize("0xff.ffp10", &.{.float_literal});2021 try testTokenize("0xff.ffp10", &.{.float_literal});
2029 try testTokenize("0xff.ffP10", &.{.float_literal});2022 try testTokenize("0xff.ffP10", &.{.float_literal});
2030 try testTokenize("0xff.p10", &.{.float_literal});
2031 try testTokenize("0xffp10", &.{.float_literal});2023 try testTokenize("0xffp10", &.{.float_literal});
2032 try testTokenize("0xff_ff.ff_ffp1_0_0_0", &.{.float_literal});2024 try testTokenize("0xff_ff.ff_ffp1_0_0_0", &.{.float_literal});
2033 try testTokenize("0xf_f_f_f.f_f_f_fp+1_000", &.{.float_literal});2025 try testTokenize("0xf_f_f_f.f_f_f_fp+1_000", &.{.float_literal});
...@@ -2038,7 +2030,7 @@ test "tokenizer - number literals hexadecimal" {...@@ -2038,7 +2030,7 @@ test "tokenizer - number literals hexadecimal" {
2038 try testTokenize("0x1p", &.{.invalid});2030 try testTokenize("0x1p", &.{.invalid});
2039 try testTokenize("0xfp0z1", &.{ .invalid, .identifier });2031 try testTokenize("0xfp0z1", &.{ .invalid, .identifier });
2040 try testTokenize("0xff.ffpff", &.{ .invalid, .identifier });2032 try testTokenize("0xff.ffpff", &.{ .invalid, .identifier });
2041 try testTokenize("0x0.p", &.{.invalid});2033 try testTokenize("0x0.p", &.{ .invalid, .identifier });
2042 try testTokenize("0x0.z", &.{ .invalid, .identifier });2034 try testTokenize("0x0.z", &.{ .invalid, .identifier });
2043 try testTokenize("0x0._", &.{ .invalid, .identifier });2035 try testTokenize("0x0._", &.{ .invalid, .identifier });
2044 try testTokenize("0x0_.0", &.{ .invalid, .period, .integer_literal });2036 try testTokenize("0x0_.0", &.{ .invalid, .period, .integer_literal });
src/stage1/tokenizer.cpp-8
...@@ -1286,10 +1286,6 @@ void tokenize(const char *source, Tokenization *out) {...@@ -1286,10 +1286,6 @@ void tokenize(const char *source, Tokenization *out) {
1286 t.column -= 1;1286 t.column -= 1;
1287 t.state = TokenizeState_start;1287 t.state = TokenizeState_start;
1288 continue;1288 continue;
1289 case 'e':
1290 case 'E':
1291 t.state = TokenizeState_float_exponent_unsigned;
1292 break;
1293 case DIGIT:1289 case DIGIT:
1294 t.state = TokenizeState_float_fraction_dec;1290 t.state = TokenizeState_float_fraction_dec;
1295 break;1291 break;
...@@ -1308,10 +1304,6 @@ void tokenize(const char *source, Tokenization *out) {...@@ -1308,10 +1304,6 @@ void tokenize(const char *source, Tokenization *out) {
1308 t.column -= 1;1304 t.column -= 1;
1309 t.state = TokenizeState_start;1305 t.state = TokenizeState_start;
1310 continue;1306 continue;
1311 case 'p':
1312 case 'P':
1313 t.state = TokenizeState_float_exponent_unsigned;
1314 break;
1315 case HEXDIGIT:1307 case HEXDIGIT:
1316 t.out->ids.last() = TokenIdFloatLiteral;1308 t.out->ids.last() = TokenIdFloatLiteral;
1317 t.state = TokenizeState_float_fraction_hex;1309 t.state = TokenizeState_float_fraction_hex;
src/translate_c.zig+10-1
...@@ -4932,8 +4932,17 @@ fn parseCNumLit(c: *Context, m: *MacroCtx) ParseError!Node {...@@ -4932,8 +4932,17 @@ fn parseCNumLit(c: *Context, m: *MacroCtx) ParseError!Node {
4932 }4932 }
4933 },4933 },
4934 .FloatLiteral => |suffix| {4934 .FloatLiteral => |suffix| {
4935 if (lit_bytes[0] == '.')4935 const dot_index = mem.indexOfScalar(u8, lit_bytes, '.').?;
4936 if (dot_index == 0) {
4936 lit_bytes = try std.fmt.allocPrint(c.arena, "0{s}", .{lit_bytes});4937 lit_bytes = try std.fmt.allocPrint(c.arena, "0{s}", .{lit_bytes});
4938 } else if (dot_index + 1 == lit_bytes.len or !std.ascii.isDigit(lit_bytes[dot_index + 1])) {
4939 // If the literal lacks a digit after the `.`, we need to
4940 // add one since `1.` or `1.e10` would be invalid syntax in Zig.
4941 lit_bytes = try std.fmt.allocPrint(c.arena, "{s}0{s}", .{
4942 lit_bytes[0 .. dot_index + 1],
4943 lit_bytes[dot_index + 1 ..],
4944 });
4945 }
4937 if (suffix == .none) {4946 if (suffix == .none) {
4938 return transCreateNodeNumber(c, lit_bytes, .float);4947 return transCreateNodeNumber(c, lit_bytes, .float);
4939 }4948 }
test/translate_c.zig+1-1
...@@ -1110,7 +1110,7 @@ pub fn addCases(cases: *tests.TranslateCContext) void {...@@ -1110,7 +1110,7 @@ pub fn addCases(cases: *tests.TranslateCContext) void {
1110 \\#define BAR .123451110 \\#define BAR .12345
1111 , &[_][]const u8{1111 , &[_][]const u8{
1112 "pub const foo = @as(f32, 3.14);",1112 "pub const foo = @as(f32, 3.14);",
1113 "pub const bar = @as(c_longdouble, 16.e-2);",1113 "pub const bar = @as(c_longdouble, 16.0e-2);",
1114 "pub const FOO = 0.12345;",1114 "pub const FOO = 0.12345;",
1115 "pub const BAR = 0.12345;",1115 "pub const BAR = 0.12345;",
1116 });1116 });