authorgravatar for 57862114+momumi@users.noreply.github.commomumi <57862114+momumi@users.noreply.github.com> 2020-03-23 09:21:34+10:00
committergravatar for 57862114+momumi@users.noreply.github.commomumi <57862114+momumi@users.noreply.github.com> 2020-03-23 09:21:34+10:00
log2d18178c27060ff9b9b4a5b56941617dc47868d0
tree35f02fd50f5508f06ef1defcc20fbe69fc4be017
parent8de45e51438d7f593825ae2134ab7dfc46f3bab2

minor fixes and more tests for _ separators

* Make the tokenizer spit out an Invalid token on the first invalid character found in the number literal. * More parsing and tokenizer tests for number literals * fix invalid switch statement in ir.zig

3 files changed, 242 insertions(+), 112 deletions(-)

lib/std/zig/parser_test.zig+69
......@@ -2800,6 +2800,75 @@ test "zig fmt: extern without container keyword returns error" {
28002800 );
28012801}
28022802
2803test "zig fmt: integer literals with underscore separators" {
2804 try testTransform(
2805 \\const
2806 \\ x =
2807 \\ 1_234_567
2808 \\ +(0b0_1-0o7_0+0xff_FF ) + 0_0;
2809 ,
2810 \\const x = 1_234_567 + (0b0_1 - 0o7_0 + 0xff_FF) + 0_0;
2811 \\
2812 );
2813}
2814
2815test "zig fmt: hex literals with underscore separators" {
2816 try testTransform(
2817 \\pub fn orMask(a: [ 1_000 ]u64, b: [ 1_000] u64) [1_000]u64 {
2818 \\ var c: [1_000]u64 = [1]u64{ 0xFFFF_FFFF_FFFF_FFFF}**1_000;
2819 \\ for (c [ 0_0 .. ]) |_, i| {
2820 \\ c[i] = (a[i] | b[i]) & 0xCCAA_CCAA_CCAA_CCAA;
2821 \\ }
2822 \\ return c;
2823 \\}
2824 \\
2825 \\
2826 ,
2827 \\pub fn orMask(a: [1_000]u64, b: [1_000]u64) [1_000]u64 {
2828 \\ var c: [1_000]u64 = [1]u64{0xFFFF_FFFF_FFFF_FFFF} ** 1_000;
2829 \\ for (c[0_0..]) |_, i| {
2830 \\ c[i] = (a[i] | b[i]) & 0xCCAA_CCAA_CCAA_CCAA;
2831 \\ }
2832 \\ return c;
2833 \\}
2834 \\
2835 );
2836}
2837
2838test "zig fmt: decimal float literals with underscore separators" {
2839 try testTransform(
2840 \\pub fn main() void {
2841 \\ const a:f64=(10.0e-0+(10.e+0))+10_00.00_00e-2+00_00.00_10e+4;
2842 \\ const b:f64=010.0--0_10.+0_1_0.0_0+1e2;
2843 \\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b });
2844 \\}
2845 ,
2846 \\pub fn main() void {
2847 \\ const a: f64 = (10.0e-0 + (10.e+0)) + 10_00.00_00e-2 + 00_00.00_10e+4;
2848 \\ const b: f64 = 010.0 - -0_10. + 0_1_0.0_0 + 1e2;
2849 \\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b });
2850 \\}
2851 \\
2852 );
2853}
2854
2855test "zig fmt: hexadeciaml float literals with underscore separators" {
2856 try testTransform(
2857 \\pub fn main() void {
2858 \\ const a: f64 = (0x10.0p-0+(0x10.p+0))+0x10_00.00_00p-8+0x00_00.00_10p+16;
2859 \\ const b: f64 = 0x0010.0--0x00_10.+0x10.00+0x1p4;
2860 \\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b });
2861 \\}
2862 ,
2863 \\pub fn main() void {
2864 \\ const a: f64 = (0x10.0p-0 + (0x10.p+0)) + 0x10_00.00_00p-8 + 0x00_00.00_10p+16;
2865 \\ const b: f64 = 0x0010.0 - -0x00_10. + 0x10.00 + 0x1p4;
2866 \\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b });
2867 \\}
2868 \\
2869 );
2870}
2871
28032872const std = @import("std");
28042873const mem = std.mem;
28052874const warn = std.debug.warn;
lib/std/zig/tokenizer.zig+168-107
......@@ -418,6 +418,10 @@ pub const Tokenizer = struct {
418418 SawAtSign,
419419 };
420420
421 fn isIdentifierChar(char: u8) bool {
422 return std.ascii.isAlNum(char) or char == '_';
423 }
424
421425 pub fn next(self: *Tokenizer) Token {
422426 if (self.pending_invalid_token) |token| {
423427 self.pending_invalid_token = null;
......@@ -1063,11 +1067,17 @@ pub const Tokenizer = struct {
10631067 'x' => {
10641068 state = State.IntegerLiteralHexNoUnderscore;
10651069 },
1066 else => {
1067 // reinterpret as a normal number
1070 '0'...'9', '_', '.', 'e', 'E' => {
1071 // reinterpret as a decimal number
10681072 self.index -= 1;
10691073 state = State.IntegerLiteralDec;
10701074 },
1075 else => {
1076 if (isIdentifierChar(c)) {
1077 result.id = Token.Id.Invalid;
1078 }
1079 break;
1080 },
10711081 },
10721082 State.IntegerLiteralBinNoUnderscore => switch (c) {
10731083 '0'...'1' => {
......@@ -1075,6 +1085,7 @@ pub const Tokenizer = struct {
10751085 },
10761086 else => {
10771087 result.id = Token.Id.Invalid;
1088 break;
10781089 },
10791090 },
10801091 State.IntegerLiteralBin => switch (c) {
......@@ -1082,10 +1093,12 @@ pub const Tokenizer = struct {
10821093 state = State.IntegerLiteralBinNoUnderscore;
10831094 },
10841095 '0'...'1' => {},
1085 '2'...'9', 'a'...'z', 'A'...'Z' => {
1086 result.id = Token.Id.Invalid;
1096 else => {
1097 if (isIdentifierChar(c)) {
1098 result.id = Token.Id.Invalid;
1099 }
1100 break;
10871101 },
1088 else => break,
10891102 },
10901103 State.IntegerLiteralOctNoUnderscore => switch (c) {
10911104 '0'...'7' => {
......@@ -1093,6 +1106,7 @@ pub const Tokenizer = struct {
10931106 },
10941107 else => {
10951108 result.id = Token.Id.Invalid;
1109 break;
10961110 },
10971111 },
10981112 State.IntegerLiteralOct => switch (c) {
......@@ -1100,10 +1114,12 @@ pub const Tokenizer = struct {
11001114 state = State.IntegerLiteralOctNoUnderscore;
11011115 },
11021116 '0'...'7' => {},
1103 '8'...'9', 'a'...'z', 'A'...'Z' => {
1104 result.id = Token.Id.Invalid;
1117 else => {
1118 if (isIdentifierChar(c)) {
1119 result.id = Token.Id.Invalid;
1120 }
1121 break;
11051122 },
1106 else => break,
11071123 },
11081124 State.IntegerLiteralDecNoUnderscore => switch (c) {
11091125 '0'...'9' => {
......@@ -1111,6 +1127,7 @@ pub const Tokenizer = struct {
11111127 },
11121128 else => {
11131129 result.id = Token.Id.Invalid;
1130 break;
11141131 },
11151132 },
11161133 State.IntegerLiteralDec => switch (c) {
......@@ -1126,10 +1143,12 @@ pub const Tokenizer = struct {
11261143 result.id = Token.Id.FloatLiteral;
11271144 },
11281145 '0'...'9' => {},
1129 'a'...'d', 'f'...'z', 'A'...'D', 'F'...'Z' => {
1130 result.id = Token.Id.Invalid;
1146 else => {
1147 if (isIdentifierChar(c)) {
1148 result.id = Token.Id.Invalid;
1149 }
1150 break;
11311151 },
1132 else => break,
11331152 },
11341153 State.IntegerLiteralHexNoUnderscore => switch (c) {
11351154 '0'...'9', 'a'...'f', 'A'...'F' => {
......@@ -1137,6 +1156,7 @@ pub const Tokenizer = struct {
11371156 },
11381157 else => {
11391158 result.id = Token.Id.Invalid;
1159 break;
11401160 },
11411161 },
11421162 State.IntegerLiteralHex => switch (c) {
......@@ -1152,10 +1172,12 @@ pub const Tokenizer = struct {
11521172 result.id = Token.Id.FloatLiteral;
11531173 },
11541174 '0'...'9', 'a'...'f', 'A'...'F' => {},
1155 'g'...'o', 'q'...'z', 'G'...'O', 'Q'...'Z' => {
1156 result.id = Token.Id.Invalid;
1175 else => {
1176 if (isIdentifierChar(c)) {
1177 result.id = Token.Id.Invalid;
1178 }
1179 break;
11571180 },
1158 else => break,
11591181 },
11601182 State.NumberDotDec => switch (c) {
11611183 '.' => {
......@@ -1166,10 +1188,15 @@ pub const Tokenizer = struct {
11661188 'e', 'E' => {
11671189 state = State.FloatExponentUnsigned;
11681190 },
1169 else => {
1170 self.index -= 1;
1191 '0'...'9' => {
11711192 result.id = Token.Id.FloatLiteral;
1172 state = State.FloatFractionDecNoUnderscore;
1193 state = State.FloatFractionDec;
1194 },
1195 else => {
1196 if (isIdentifierChar(c)) {
1197 result.id = Token.Id.Invalid;
1198 }
1199 break;
11731200 },
11741201 },
11751202 State.NumberDotHex => switch (c) {
......@@ -1181,10 +1208,15 @@ pub const Tokenizer = struct {
11811208 'p', 'P' => {
11821209 state = State.FloatExponentUnsigned;
11831210 },
1184 else => {
1185 self.index -= 1;
1211 '0'...'9', 'a'...'f', 'A'...'F' => {
11861212 result.id = Token.Id.FloatLiteral;
1187 state = State.FloatFractionHexNoUnderscore;
1213 state = State.FloatFractionHex;
1214 },
1215 else => {
1216 if (isIdentifierChar(c)) {
1217 result.id = Token.Id.Invalid;
1218 }
1219 break;
11881220 },
11891221 },
11901222 State.FloatFractionDecNoUnderscore => switch (c) {
......@@ -1193,6 +1225,7 @@ pub const Tokenizer = struct {
11931225 },
11941226 else => {
11951227 result.id = Token.Id.Invalid;
1228 break;
11961229 },
11971230 },
11981231 State.FloatFractionDec => switch (c) {
......@@ -1203,10 +1236,12 @@ pub const Tokenizer = struct {
12031236 state = State.FloatExponentUnsigned;
12041237 },
12051238 '0'...'9' => {},
1206 'a'...'d', 'f'...'z', 'A'...'D', 'F'...'Z' => {
1207 result.id = Token.Id.Invalid;
1239 else => {
1240 if (isIdentifierChar(c)) {
1241 result.id = Token.Id.Invalid;
1242 }
1243 break;
12081244 },
1209 else => break,
12101245 },
12111246 State.FloatFractionHexNoUnderscore => switch (c) {
12121247 '0'...'9', 'a'...'f', 'A'...'F' => {
......@@ -1214,6 +1249,7 @@ pub const Tokenizer = struct {
12141249 },
12151250 else => {
12161251 result.id = Token.Id.Invalid;
1252 break;
12171253 },
12181254 },
12191255 State.FloatFractionHex => switch (c) {
......@@ -1224,10 +1260,12 @@ pub const Tokenizer = struct {
12241260 state = State.FloatExponentUnsigned;
12251261 },
12261262 '0'...'9', 'a'...'f', 'A'...'F' => {},
1227 'g'...'o', 'q'...'z', 'G'...'O', 'Q'...'Z' => {
1228 result.id = Token.Id.Invalid;
1263 else => {
1264 if (isIdentifierChar(c)) {
1265 result.id = Token.Id.Invalid;
1266 }
1267 break;
12291268 },
1230 else => break,
12311269 },
12321270 State.FloatExponentUnsigned => switch (c) {
12331271 '+', '-' => {
......@@ -1245,6 +1283,7 @@ pub const Tokenizer = struct {
12451283 },
12461284 else => {
12471285 result.id = Token.Id.Invalid;
1286 break;
12481287 },
12491288 },
12501289 State.FloatExponentNumber => switch (c) {
......@@ -1252,10 +1291,12 @@ pub const Tokenizer = struct {
12521291 state = State.FloatExponentNumberNoUnderscore;
12531292 },
12541293 '0'...'9' => {},
1255 'a'...'z', 'A'...'Z' => {
1256 result.id = Token.Id.Invalid;
1294 else => {
1295 if (isIdentifierChar(c)) {
1296 result.id = Token.Id.Invalid;
1297 }
1298 break;
12571299 },
1258 else => break,
12591300 },
12601301 }
12611302 } else if (self.index == self.buffer.len) {
......@@ -1706,11 +1747,11 @@ test "tokenizer - number literals decimal" {
17061747 testTokenize("7", &[_]Token.Id{.IntegerLiteral});
17071748 testTokenize("8", &[_]Token.Id{.IntegerLiteral});
17081749 testTokenize("9", &[_]Token.Id{.IntegerLiteral});
1709 testTokenize("0a", &[_]Token.Id{.Invalid});
1710 testTokenize("9b", &[_]Token.Id{.Invalid});
1711 testTokenize("1z", &[_]Token.Id{.Invalid});
1712 testTokenize("1z_1", &[_]Token.Id{.Invalid});
1713 testTokenize("9z3", &[_]Token.Id{.Invalid});
1750 testTokenize("0a", &[_]Token.Id{ .Invalid, .Identifier });
1751 testTokenize("9b", &[_]Token.Id{ .Invalid, .Identifier });
1752 testTokenize("1z", &[_]Token.Id{ .Invalid, .Identifier });
1753 testTokenize("1z_1", &[_]Token.Id{ .Invalid, .Identifier });
1754 testTokenize("9z3", &[_]Token.Id{ .Invalid, .Identifier });
17141755
17151756 testTokenize("0_0", &[_]Token.Id{.IntegerLiteral});
17161757 testTokenize("0001", &[_]Token.Id{.IntegerLiteral});
......@@ -1720,15 +1761,17 @@ test "tokenizer - number literals decimal" {
17201761
17211762 testTokenize("00_", &[_]Token.Id{.Invalid});
17221763 testTokenize("0_0_", &[_]Token.Id{.Invalid});
1723 testTokenize("0__0", &[_]Token.Id{.Invalid});
1724 testTokenize("0_0f", &[_]Token.Id{.Invalid});
1725 testTokenize("0_0_f", &[_]Token.Id{.Invalid});
1726 testTokenize("1_,", &[_]Token.Id{.Invalid});
1764 testTokenize("0__0", &[_]Token.Id{ .Invalid, .Identifier });
1765 testTokenize("0_0f", &[_]Token.Id{ .Invalid, .Identifier });
1766 testTokenize("0_0_f", &[_]Token.Id{ .Invalid, .Identifier });
1767 testTokenize("0_0_f_00", &[_]Token.Id{ .Invalid, .Identifier });
1768 testTokenize("1_,", &[_]Token.Id{ .Invalid, .Comma });
17271769
17281770 testTokenize("1.", &[_]Token.Id{.FloatLiteral});
17291771 testTokenize("0.0", &[_]Token.Id{.FloatLiteral});
17301772 testTokenize("1.0", &[_]Token.Id{.FloatLiteral});
17311773 testTokenize("10.0", &[_]Token.Id{.FloatLiteral});
1774 testTokenize("0e0", &[_]Token.Id{.FloatLiteral});
17321775 testTokenize("1e0", &[_]Token.Id{.FloatLiteral});
17331776 testTokenize("1e100", &[_]Token.Id{.FloatLiteral});
17341777 testTokenize("1.e100", &[_]Token.Id{.FloatLiteral});
......@@ -1736,34 +1779,47 @@ test "tokenizer - number literals decimal" {
17361779 testTokenize("1.0e+100", &[_]Token.Id{.FloatLiteral});
17371780 testTokenize("1.0e-100", &[_]Token.Id{.FloatLiteral});
17381781 testTokenize("1_0_0_0.0_0_0_0_0_1e1_0_0_0", &[_]Token.Id{.FloatLiteral});
1782 testTokenize("1.+", &[_]Token.Id{ .FloatLiteral, .Plus });
17391783
17401784 testTokenize("1e", &[_]Token.Id{.Invalid});
1741 testTokenize("1.0e1f0", &[_]Token.Id{.Invalid});
1742 testTokenize("1.0p100", &[_]Token.Id{.Invalid});
1743 testTokenize("1.0p-100", &[_]Token.Id{ .Invalid, .Minus, .IntegerLiteral });
1744 testTokenize("1.0p1f0", &[_]Token.Id{.Invalid});
1745 testTokenize("1.0_,", &[_]Token.Id{.Invalid});
1746 testTokenize("1.0e,", &[_]Token.Id{.Invalid});
1785 testTokenize("1.0e1f0", &[_]Token.Id{ .Invalid, .Identifier });
1786 testTokenize("1.0p100", &[_]Token.Id{ .Invalid, .Identifier });
1787 testTokenize("1.0p-100", &[_]Token.Id{ .Invalid, .Identifier, .Minus, .IntegerLiteral });
1788 testTokenize("1.0p1f0", &[_]Token.Id{ .Invalid, .Identifier });
1789 testTokenize("1.0_,", &[_]Token.Id{ .Invalid, .Comma });
1790 testTokenize("1_.0", &[_]Token.Id{ .Invalid, .Period, .IntegerLiteral });
1791 testTokenize("1._", &[_]Token.Id{ .Invalid, .Identifier });
1792 testTokenize("1.a", &[_]Token.Id{ .Invalid, .Identifier });
1793 testTokenize("1.z", &[_]Token.Id{ .Invalid, .Identifier });
1794 testTokenize("1._0", &[_]Token.Id{ .Invalid, .Identifier });
1795 testTokenize("1._+", &[_]Token.Id{ .Invalid, .Identifier, .Plus });
1796 testTokenize("1._e", &[_]Token.Id{ .Invalid, .Identifier });
1797 testTokenize("1.0e", &[_]Token.Id{.Invalid});
1798 testTokenize("1.0e,", &[_]Token.Id{ .Invalid, .Comma });
1799 testTokenize("1.0e_", &[_]Token.Id{ .Invalid, .Identifier });
1800 testTokenize("1.0e+_", &[_]Token.Id{ .Invalid, .Identifier });
1801 testTokenize("1.0e-_", &[_]Token.Id{ .Invalid, .Identifier });
1802 testTokenize("1.0e0_+", &[_]Token.Id{ .Invalid, .Plus });
17471803}
17481804
17491805test "tokenizer - number literals binary" {
17501806 testTokenize("0b0", &[_]Token.Id{.IntegerLiteral});
17511807 testTokenize("0b1", &[_]Token.Id{.IntegerLiteral});
1752 testTokenize("0b2", &[_]Token.Id{.Invalid});
1753 testTokenize("0b3", &[_]Token.Id{.Invalid});
1754 testTokenize("0b4", &[_]Token.Id{.Invalid});
1755 testTokenize("0b5", &[_]Token.Id{.Invalid});
1756 testTokenize("0b6", &[_]Token.Id{.Invalid});
1757 testTokenize("0b7", &[_]Token.Id{.Invalid});
1758 testTokenize("0b8", &[_]Token.Id{.Invalid});
1759 testTokenize("0b9", &[_]Token.Id{.Invalid});
1760 testTokenize("0ba", &[_]Token.Id{.Invalid});
1761 testTokenize("0bb", &[_]Token.Id{.Invalid});
1762 testTokenize("0bc", &[_]Token.Id{.Invalid});
1763 testTokenize("0bd", &[_]Token.Id{.Invalid});
1764 testTokenize("0be", &[_]Token.Id{.Invalid});
1765 testTokenize("0bf", &[_]Token.Id{.Invalid});
1766 testTokenize("0bz", &[_]Token.Id{.Invalid});
1808 testTokenize("0b2", &[_]Token.Id{ .Invalid, .IntegerLiteral });
1809 testTokenize("0b3", &[_]Token.Id{ .Invalid, .IntegerLiteral });
1810 testTokenize("0b4", &[_]Token.Id{ .Invalid, .IntegerLiteral });
1811 testTokenize("0b5", &[_]Token.Id{ .Invalid, .IntegerLiteral });
1812 testTokenize("0b6", &[_]Token.Id{ .Invalid, .IntegerLiteral });
1813 testTokenize("0b7", &[_]Token.Id{ .Invalid, .IntegerLiteral });
1814 testTokenize("0b8", &[_]Token.Id{ .Invalid, .IntegerLiteral });
1815 testTokenize("0b9", &[_]Token.Id{ .Invalid, .IntegerLiteral });
1816 testTokenize("0ba", &[_]Token.Id{ .Invalid, .Identifier });
1817 testTokenize("0bb", &[_]Token.Id{ .Invalid, .Identifier });
1818 testTokenize("0bc", &[_]Token.Id{ .Invalid, .Identifier });
1819 testTokenize("0bd", &[_]Token.Id{ .Invalid, .Identifier });
1820 testTokenize("0be", &[_]Token.Id{ .Invalid, .Identifier });
1821 testTokenize("0bf", &[_]Token.Id{ .Invalid, .Identifier });
1822 testTokenize("0bz", &[_]Token.Id{ .Invalid, .Identifier });
17671823
17681824 testTokenize("0b0000_0000", &[_]Token.Id{.IntegerLiteral});
17691825 testTokenize("0b1111_1111", &[_]Token.Id{.IntegerLiteral});
......@@ -1772,17 +1828,17 @@ test "tokenizer - number literals binary" {
17721828 testTokenize("0b1.", &[_]Token.Id{ .IntegerLiteral, .Period });
17731829 testTokenize("0b1.0", &[_]Token.Id{ .IntegerLiteral, .Period, .IntegerLiteral });
17741830
1775 testTokenize("0B0", &[_]Token.Id{.Invalid});
1776 testTokenize("0b_", &[_]Token.Id{.Invalid});
1777 testTokenize("0b_0", &[_]Token.Id{.Invalid});
1831 testTokenize("0B0", &[_]Token.Id{ .Invalid, .Identifier });
1832 testTokenize("0b_", &[_]Token.Id{ .Invalid, .Identifier });
1833 testTokenize("0b_0", &[_]Token.Id{ .Invalid, .Identifier });
17781834 testTokenize("0b1_", &[_]Token.Id{.Invalid});
1779 testTokenize("0b0__1", &[_]Token.Id{.Invalid});
1835 testTokenize("0b0__1", &[_]Token.Id{ .Invalid, .Identifier });
17801836 testTokenize("0b0_1_", &[_]Token.Id{.Invalid});
1781 testTokenize("0b1e", &[_]Token.Id{.Invalid});
1782 testTokenize("0b1p", &[_]Token.Id{.Invalid});
1783 testTokenize("0b1e0", &[_]Token.Id{.Invalid});
1784 testTokenize("0b1p0", &[_]Token.Id{.Invalid});
1785 testTokenize("0b1_,", &[_]Token.Id{.Invalid});
1837 testTokenize("0b1e", &[_]Token.Id{ .Invalid, .Identifier });
1838 testTokenize("0b1p", &[_]Token.Id{ .Invalid, .Identifier });
1839 testTokenize("0b1e0", &[_]Token.Id{ .Invalid, .Identifier });
1840 testTokenize("0b1p0", &[_]Token.Id{ .Invalid, .Identifier });
1841 testTokenize("0b1_,", &[_]Token.Id{ .Invalid, .Comma });
17861842}
17871843
17881844test "tokenizer - number literals octal" {
......@@ -1794,15 +1850,15 @@ test "tokenizer - number literals octal" {
17941850 testTokenize("0o5", &[_]Token.Id{.IntegerLiteral});
17951851 testTokenize("0o6", &[_]Token.Id{.IntegerLiteral});
17961852 testTokenize("0o7", &[_]Token.Id{.IntegerLiteral});
1797 testTokenize("0o8", &[_]Token.Id{.Invalid});
1798 testTokenize("0o9", &[_]Token.Id{.Invalid});
1799 testTokenize("0oa", &[_]Token.Id{.Invalid});
1800 testTokenize("0ob", &[_]Token.Id{.Invalid});
1801 testTokenize("0oc", &[_]Token.Id{.Invalid});
1802 testTokenize("0od", &[_]Token.Id{.Invalid});
1803 testTokenize("0oe", &[_]Token.Id{.Invalid});
1804 testTokenize("0of", &[_]Token.Id{.Invalid});
1805 testTokenize("0oz", &[_]Token.Id{.Invalid});
1853 testTokenize("0o8", &[_]Token.Id{ .Invalid, .IntegerLiteral });
1854 testTokenize("0o9", &[_]Token.Id{ .Invalid, .IntegerLiteral });
1855 testTokenize("0oa", &[_]Token.Id{ .Invalid, .Identifier });
1856 testTokenize("0ob", &[_]Token.Id{ .Invalid, .Identifier });
1857 testTokenize("0oc", &[_]Token.Id{ .Invalid, .Identifier });
1858 testTokenize("0od", &[_]Token.Id{ .Invalid, .Identifier });
1859 testTokenize("0oe", &[_]Token.Id{ .Invalid, .Identifier });
1860 testTokenize("0of", &[_]Token.Id{ .Invalid, .Identifier });
1861 testTokenize("0oz", &[_]Token.Id{ .Invalid, .Identifier });
18061862
18071863 testTokenize("0o01234567", &[_]Token.Id{.IntegerLiteral});
18081864 testTokenize("0o0123_4567", &[_]Token.Id{.IntegerLiteral});
......@@ -1811,17 +1867,17 @@ test "tokenizer - number literals octal" {
18111867 testTokenize("0o7.", &[_]Token.Id{ .IntegerLiteral, .Period });
18121868 testTokenize("0o7.0", &[_]Token.Id{ .IntegerLiteral, .Period, .IntegerLiteral });
18131869
1814 testTokenize("0O0", &[_]Token.Id{.Invalid});
1815 testTokenize("0o_", &[_]Token.Id{.Invalid});
1816 testTokenize("0o_0", &[_]Token.Id{.Invalid});
1870 testTokenize("0O0", &[_]Token.Id{ .Invalid, .Identifier });
1871 testTokenize("0o_", &[_]Token.Id{ .Invalid, .Identifier });
1872 testTokenize("0o_0", &[_]Token.Id{ .Invalid, .Identifier });
18171873 testTokenize("0o1_", &[_]Token.Id{.Invalid});
1818 testTokenize("0o0__1", &[_]Token.Id{.Invalid});
1874 testTokenize("0o0__1", &[_]Token.Id{ .Invalid, .Identifier });
18191875 testTokenize("0o0_1_", &[_]Token.Id{.Invalid});
1820 testTokenize("0o1e", &[_]Token.Id{.Invalid});
1821 testTokenize("0o1p", &[_]Token.Id{.Invalid});
1822 testTokenize("0o1e0", &[_]Token.Id{.Invalid});
1823 testTokenize("0o1p0", &[_]Token.Id{.Invalid});
1824 testTokenize("0o_,", &[_]Token.Id{.Invalid});
1876 testTokenize("0o1e", &[_]Token.Id{ .Invalid, .Identifier });
1877 testTokenize("0o1p", &[_]Token.Id{ .Invalid, .Identifier });
1878 testTokenize("0o1e0", &[_]Token.Id{ .Invalid, .Identifier });
1879 testTokenize("0o1p0", &[_]Token.Id{ .Invalid, .Identifier });
1880 testTokenize("0o_,", &[_]Token.Id{ .Invalid, .Identifier, .Comma });
18251881}
18261882
18271883test "tokenizer - number literals hexadeciaml" {
......@@ -1847,21 +1903,21 @@ test "tokenizer - number literals hexadeciaml" {
18471903 testTokenize("0xD", &[_]Token.Id{.IntegerLiteral});
18481904 testTokenize("0xE", &[_]Token.Id{.IntegerLiteral});
18491905 testTokenize("0xF", &[_]Token.Id{.IntegerLiteral});
1850 testTokenize("0x0z", &[_]Token.Id{.Invalid});
1851 testTokenize("0xz", &[_]Token.Id{.Invalid});
1906 testTokenize("0x0z", &[_]Token.Id{ .Invalid, .Identifier });
1907 testTokenize("0xz", &[_]Token.Id{ .Invalid, .Identifier });
18521908
18531909 testTokenize("0x0123456789ABCDEF", &[_]Token.Id{.IntegerLiteral});
18541910 testTokenize("0x0123_4567_89AB_CDEF", &[_]Token.Id{.IntegerLiteral});
18551911 testTokenize("0x01_23_45_67_89AB_CDE_F", &[_]Token.Id{.IntegerLiteral});
18561912 testTokenize("0x0_1_2_3_4_5_6_7_8_9_A_B_C_D_E_F", &[_]Token.Id{.IntegerLiteral});
18571913
1858 testTokenize("0X0", &[_]Token.Id{.Invalid});
1859 testTokenize("0x_", &[_]Token.Id{.Invalid});
1860 testTokenize("0x_1", &[_]Token.Id{.Invalid});
1914 testTokenize("0X0", &[_]Token.Id{ .Invalid, .Identifier });
1915 testTokenize("0x_", &[_]Token.Id{ .Invalid, .Identifier });
1916 testTokenize("0x_1", &[_]Token.Id{ .Invalid, .Identifier });
18611917 testTokenize("0x1_", &[_]Token.Id{.Invalid});
1862 testTokenize("0x0__1", &[_]Token.Id{.Invalid});
1918 testTokenize("0x0__1", &[_]Token.Id{ .Invalid, .Identifier });
18631919 testTokenize("0x0_1_", &[_]Token.Id{.Invalid});
1864 testTokenize("0x_,", &[_]Token.Id{.Invalid});
1920 testTokenize("0x_,", &[_]Token.Id{ .Invalid, .Identifier, .Comma });
18651921
18661922 testTokenize("0x1.", &[_]Token.Id{.FloatLiteral});
18671923 testTokenize("0x1.0", &[_]Token.Id{.FloatLiteral});
......@@ -1872,10 +1928,12 @@ test "tokenizer - number literals hexadeciaml" {
18721928 testTokenize("0xF.FP0", &[_]Token.Id{.FloatLiteral});
18731929 testTokenize("0x1p0", &[_]Token.Id{.FloatLiteral});
18741930 testTokenize("0xfp0", &[_]Token.Id{.FloatLiteral});
1931 testTokenize("0x1.+0xF.", &[_]Token.Id{ .FloatLiteral, .Plus, .FloatLiteral });
18751932
18761933 testTokenize("0x0123456.789ABCDEF", &[_]Token.Id{.FloatLiteral});
18771934 testTokenize("0x0_123_456.789_ABC_DEF", &[_]Token.Id{.FloatLiteral});
18781935 testTokenize("0x0_1_2_3_4_5_6.7_8_9_A_B_C_D_E_F", &[_]Token.Id{.FloatLiteral});
1936 testTokenize("0x0p0", &[_]Token.Id{.FloatLiteral});
18791937 testTokenize("0x0.0p0", &[_]Token.Id{.FloatLiteral});
18801938 testTokenize("0xff.ffp10", &[_]Token.Id{.FloatLiteral});
18811939 testTokenize("0xff.ffP10", &[_]Token.Id{.FloatLiteral});
......@@ -1888,21 +1946,24 @@ test "tokenizer - number literals hexadeciaml" {
18881946 testTokenize("0x1e", &[_]Token.Id{.IntegerLiteral});
18891947 testTokenize("0x1e0", &[_]Token.Id{.IntegerLiteral});
18901948 testTokenize("0x1p", &[_]Token.Id{.Invalid});
1891 testTokenize("0xfp0z1", &[_]Token.Id{.Invalid});
1892 testTokenize("0xff.ffpff", &[_]Token.Id{.Invalid});
1893 testTokenize("0x0_.0", &[_]Token.Id{.Invalid});
1894 testTokenize("0x0._0", &[_]Token.Id{.Invalid});
1949 testTokenize("0xfp0z1", &[_]Token.Id{ .Invalid, .Identifier });
1950 testTokenize("0xff.ffpff", &[_]Token.Id{ .Invalid, .Identifier });
1951 testTokenize("0x0.p", &[_]Token.Id{.Invalid});
1952 testTokenize("0x0.z", &[_]Token.Id{ .Invalid, .Identifier });
1953 testTokenize("0x0._", &[_]Token.Id{ .Invalid, .Identifier });
1954 testTokenize("0x0_.0", &[_]Token.Id{ .Invalid, .Period, .IntegerLiteral });
1955 testTokenize("0x0_.0.0", &[_]Token.Id{ .Invalid, .Period, .FloatLiteral });
1956 testTokenize("0x0._0", &[_]Token.Id{ .Invalid, .Identifier });
18951957 testTokenize("0x0.0_", &[_]Token.Id{.Invalid});
1896 testTokenize("0x0_p0", &[_]Token.Id{.Invalid});
1897 testTokenize("0x0_.p0", &[_]Token.Id{.Invalid});
1898 testTokenize("0x0._p0", &[_]Token.Id{.Invalid});
1899 testTokenize("0x0.0_p0", &[_]Token.Id{.Invalid});
1900 testTokenize("0x0._0p0", &[_]Token.Id{.Invalid});
1901 testTokenize("0x0.0_p0", &[_]Token.Id{.Invalid});
1902 testTokenize("0x0.0p_0", &[_]Token.Id{.Invalid});
1903 testTokenize("0x0.0p+_0", &[_]Token.Id{.Invalid});
1904 testTokenize("0x0.0p-_0", &[_]Token.Id{.Invalid});
1905 testTokenize("0x0.0p0_", &[_]Token.Id{.Invalid});
1958 testTokenize("0x0_p0", &[_]Token.Id{ .Invalid, .Identifier });
1959 testTokenize("0x0_.p0", &[_]Token.Id{ .Invalid, .Period, .Identifier });
1960 testTokenize("0x0._p0", &[_]Token.Id{ .Invalid, .Identifier });
1961 testTokenize("0x0.0_p0", &[_]Token.Id{ .Invalid, .Identifier });
1962 testTokenize("0x0._0p0", &[_]Token.Id{ .Invalid, .Identifier });
1963 testTokenize("0x0.0p_0", &[_]Token.Id{ .Invalid, .Identifier });
1964 testTokenize("0x0.0p+_0", &[_]Token.Id{ .Invalid, .Identifier });
1965 testTokenize("0x0.0p-_0", &[_]Token.Id{ .Invalid, .Identifier });
1966 testTokenize("0x0.0p0_", &[_]Token.Id{ .Invalid, .Eof });
19061967}
19071968
19081969fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
src-self-hosted/ir.zig+5-5
......@@ -1312,15 +1312,15 @@ pub const Builder = struct {
13121312 var rest: []const u8 = undefined;
13131313 if (int_token.len >= 3 and int_token[0] == '0') {
13141314 rest = int_token[2..];
1315 base = switch (int_token[1]) {
1316 'b' => 2,
1317 'o' => 8,
1318 'x' => 16,
1315 switch (int_token[1]) {
1316 'b' => base = 2,
1317 'o' => base = 8,
1318 'x' => base = 16,
13191319 else => {
13201320 base = 10;
13211321 rest = int_token;
13221322 },
1323 };
1323 }
13241324 } else {
13251325 base = 10;
13261326 rest = int_token;