authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-03-23 00:54:54-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2020-03-23 00:54:54-04:00
log13d04f9963be930360ab728edd47f1a6ecfb1777
treeaac3edc539e68e80ff60663709c256e4c8051154
parent7ffdf59c441380efd9bbb837de7ad5f2df747a6e
parent2d18178c27060ff9b9b4a5b56941617dc47868d0
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #4741 from momumi/master

allow `_` separators in number literals (stage 1)

10 files changed, 901 insertions(+), 158 deletions(-)

doc/langref.html.in+11
......@@ -885,6 +885,12 @@ const hex_int = 0xff;
885885const another_hex_int = 0xFF;
886886const octal_int = 0o755;
887887const binary_int = 0b11110000;
888
889// underscores may be placed between two digits as a visual separator
890const one_billion = 1_000_000_000;
891const binary_mask = 0b1_1111_1111;
892const permissions = 0o7_5_5;
893const big_address = 0xFF80_0000_0000_0000;
888894 {#code_end#}
889895 {#header_close#}
890896 {#header_open|Runtime Integer Values#}
......@@ -947,6 +953,11 @@ const yet_another = 123.0e+77;
947953const hex_floating_point = 0x103.70p-5;
948954const another_hex_float = 0x103.70;
949955const yet_another_hex_float = 0x103.70P-5;
956
957// underscores may be placed between two digits as a visual separator
958const lightspeed = 299_792_458.000_000;
959const nanosecond = 0.000_000_001;
960const more_hex = 0x1234_5678.9ABC_CDEFp-10;
950961 {#code_end#}
951962 <p>
952963 There is no syntax for NaN, infinity, or negative infinity. For these special values,
lib/std/math/big/int.zig+23-2
......@@ -373,6 +373,7 @@ pub const Int = struct {
373373 const d = switch (ch) {
374374 '0'...'9' => ch - '0',
375375 'a'...'f' => (ch - 'a') + 0xa,
376 'A'...'F' => (ch - 'A') + 0xa,
376377 else => return error.InvalidCharForDigit,
377378 };
378379
......@@ -393,8 +394,9 @@ pub const Int = struct {
393394
394395 /// Set self from the string representation `value`.
395396 ///
396 /// value must contain only digits <= `base`. Base prefixes are not allowed (e.g. 0x43 should
397 /// simply be 43).
397 /// `value` must contain only digits <= `base` and is case insensitive. Base prefixes are
398 /// not allowed (e.g. 0x43 should simply be 43). Underscores in the input string are
399 /// ignored and can be used as digit separators.
398400 ///
399401 /// Returns an error if memory could not be allocated or `value` has invalid digits for the
400402 /// requested base.
......@@ -415,6 +417,9 @@ pub const Int = struct {
415417 try self.set(0);
416418
417419 for (value[i..]) |ch| {
420 if (ch == '_') {
421 continue;
422 }
418423 const d = try charToDigit(ch, base);
419424
420425 const ap_d = Int.initFixed(([_]Limb{d})[0..]);
......@@ -1582,6 +1587,22 @@ test "big.int string negative" {
15821587 testing.expect((try a.to(i32)) == -1023);
15831588}
15841589
1590test "big.int string set number with underscores" {
1591 var a = try Int.init(testing.allocator);
1592 defer a.deinit();
1593
1594 try a.setString(10, "__1_2_0_3_1_7_2_4_1_2_0_____9_1__2__4_7_8_1_2_4_1_2_9_0_8_4_7_1_2_4___");
1595 testing.expect((try a.to(u128)) == 120317241209124781241290847124);
1596}
1597
1598test "big.int string set case insensitive number" {
1599 var a = try Int.init(testing.allocator);
1600 defer a.deinit();
1601
1602 try a.setString(16, "aB_cD_eF");
1603 testing.expect((try a.to(u32)) == 0xabcdef);
1604}
1605
15851606test "big.int string set bad char error" {
15861607 var a = try Int.init(testing.allocator);
15871608 defer a.deinit();
lib/std/special/compiler_rt/floatundisf.zig+19-19
......@@ -69,23 +69,23 @@ test "floatundisf" {
6969 test__floatundisf(0, 0.0);
7070 test__floatundisf(1, 1.0);
7171 test__floatundisf(2, 2.0);
72 test__floatundisf(0x7FFFFF8000000000, 0x1.FFFFFEp+62F);
73 test__floatundisf(0x7FFFFF0000000000, 0x1.FFFFFCp+62F);
74 test__floatundisf(0x8000008000000000, 0x1p+63F);
75 test__floatundisf(0x8000010000000000, 0x1.000002p+63F);
76 test__floatundisf(0x8000000000000000, 0x1p+63F);
77 test__floatundisf(0x8000000000000001, 0x1p+63F);
78 test__floatundisf(0xFFFFFFFFFFFFFFFE, 0x1p+64F);
79 test__floatundisf(0xFFFFFFFFFFFFFFFF, 0x1p+64F);
80 test__floatundisf(0x0007FB72E8000000, 0x1.FEDCBAp+50F);
81 test__floatundisf(0x0007FB72EA000000, 0x1.FEDCBAp+50F);
82 test__floatundisf(0x0007FB72EB000000, 0x1.FEDCBAp+50F);
83 test__floatundisf(0x0007FB72EBFFFFFF, 0x1.FEDCBAp+50F);
84 test__floatundisf(0x0007FB72EC000000, 0x1.FEDCBCp+50F);
85 test__floatundisf(0x0007FB72E8000001, 0x1.FEDCBAp+50F);
86 test__floatundisf(0x0007FB72E6000000, 0x1.FEDCBAp+50F);
87 test__floatundisf(0x0007FB72E7000000, 0x1.FEDCBAp+50F);
88 test__floatundisf(0x0007FB72E7FFFFFF, 0x1.FEDCBAp+50F);
89 test__floatundisf(0x0007FB72E4000001, 0x1.FEDCBAp+50F);
90 test__floatundisf(0x0007FB72E4000000, 0x1.FEDCB8p+50F);
72 test__floatundisf(0x7FFFFF8000000000, 0x1.FFFFFEp+62);
73 test__floatundisf(0x7FFFFF0000000000, 0x1.FFFFFCp+62);
74 test__floatundisf(0x8000008000000000, 0x1p+63);
75 test__floatundisf(0x8000010000000000, 0x1.000002p+63);
76 test__floatundisf(0x8000000000000000, 0x1p+63);
77 test__floatundisf(0x8000000000000001, 0x1p+63);
78 test__floatundisf(0xFFFFFFFFFFFFFFFE, 0x1p+64);
79 test__floatundisf(0xFFFFFFFFFFFFFFFF, 0x1p+64);
80 test__floatundisf(0x0007FB72E8000000, 0x1.FEDCBAp+50);
81 test__floatundisf(0x0007FB72EA000000, 0x1.FEDCBAp+50);
82 test__floatundisf(0x0007FB72EB000000, 0x1.FEDCBAp+50);
83 test__floatundisf(0x0007FB72EBFFFFFF, 0x1.FEDCBAp+50);
84 test__floatundisf(0x0007FB72EC000000, 0x1.FEDCBCp+50);
85 test__floatundisf(0x0007FB72E8000001, 0x1.FEDCBAp+50);
86 test__floatundisf(0x0007FB72E6000000, 0x1.FEDCBAp+50);
87 test__floatundisf(0x0007FB72E7000000, 0x1.FEDCBAp+50);
88 test__floatundisf(0x0007FB72E7FFFFFF, 0x1.FEDCBAp+50);
89 test__floatundisf(0x0007FB72E4000001, 0x1.FEDCBAp+50);
90 test__floatundisf(0x0007FB72E4000000, 0x1.FEDCB8p+50);
9191}
lib/std/zig/parser_test.zig+69
......@@ -2815,6 +2815,75 @@ test "zig fmt: extern without container keyword returns error" {
28152815 );
28162816}
28172817
2818test "zig fmt: integer literals with underscore separators" {
2819 try testTransform(
2820 \\const
2821 \\ x =
2822 \\ 1_234_567
2823 \\ +(0b0_1-0o7_0+0xff_FF ) + 0_0;
2824 ,
2825 \\const x = 1_234_567 + (0b0_1 - 0o7_0 + 0xff_FF) + 0_0;
2826 \\
2827 );
2828}
2829
2830test "zig fmt: hex literals with underscore separators" {
2831 try testTransform(
2832 \\pub fn orMask(a: [ 1_000 ]u64, b: [ 1_000] u64) [1_000]u64 {
2833 \\ var c: [1_000]u64 = [1]u64{ 0xFFFF_FFFF_FFFF_FFFF}**1_000;
2834 \\ for (c [ 0_0 .. ]) |_, i| {
2835 \\ c[i] = (a[i] | b[i]) & 0xCCAA_CCAA_CCAA_CCAA;
2836 \\ }
2837 \\ return c;
2838 \\}
2839 \\
2840 \\
2841 ,
2842 \\pub fn orMask(a: [1_000]u64, b: [1_000]u64) [1_000]u64 {
2843 \\ var c: [1_000]u64 = [1]u64{0xFFFF_FFFF_FFFF_FFFF} ** 1_000;
2844 \\ for (c[0_0..]) |_, i| {
2845 \\ c[i] = (a[i] | b[i]) & 0xCCAA_CCAA_CCAA_CCAA;
2846 \\ }
2847 \\ return c;
2848 \\}
2849 \\
2850 );
2851}
2852
2853test "zig fmt: decimal float literals with underscore separators" {
2854 try testTransform(
2855 \\pub fn main() void {
2856 \\ const a:f64=(10.0e-0+(10.e+0))+10_00.00_00e-2+00_00.00_10e+4;
2857 \\ const b:f64=010.0--0_10.+0_1_0.0_0+1e2;
2858 \\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b });
2859 \\}
2860 ,
2861 \\pub fn main() void {
2862 \\ const a: f64 = (10.0e-0 + (10.e+0)) + 10_00.00_00e-2 + 00_00.00_10e+4;
2863 \\ const b: f64 = 010.0 - -0_10. + 0_1_0.0_0 + 1e2;
2864 \\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b });
2865 \\}
2866 \\
2867 );
2868}
2869
2870test "zig fmt: hexadeciaml float literals with underscore separators" {
2871 try testTransform(
2872 \\pub fn main() void {
2873 \\ const a: f64 = (0x10.0p-0+(0x10.p+0))+0x10_00.00_00p-8+0x00_00.00_10p+16;
2874 \\ const b: f64 = 0x0010.0--0x00_10.+0x10.00+0x1p4;
2875 \\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b });
2876 \\}
2877 ,
2878 \\pub fn main() void {
2879 \\ const a: f64 = (0x10.0p-0 + (0x10.p+0)) + 0x10_00.00_00p-8 + 0x00_00.00_10p+16;
2880 \\ const b: f64 = 0x0010.0 - -0x00_10. + 0x10.00 + 0x1p4;
2881 \\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b });
2882 \\}
2883 \\
2884 );
2885}
2886
28182887const std = @import("std");
28192888const mem = std.mem;
28202889const warn = std.debug.warn;
lib/std/zig/tokenizer.zig+438-57
......@@ -387,17 +387,23 @@ pub const Tokenizer = struct {
387387 DocComment,
388388 ContainerDocComment,
389389 Zero,
390 IntegerLiteral,
391 IntegerLiteralWithRadix,
392 IntegerLiteralWithRadixHex,
393 NumberDot,
390 IntegerLiteralDec,
391 IntegerLiteralDecNoUnderscore,
392 IntegerLiteralBin,
393 IntegerLiteralBinNoUnderscore,
394 IntegerLiteralOct,
395 IntegerLiteralOctNoUnderscore,
396 IntegerLiteralHex,
397 IntegerLiteralHexNoUnderscore,
398 NumberDotDec,
394399 NumberDotHex,
395 FloatFraction,
400 FloatFractionDec,
401 FloatFractionDecNoUnderscore,
396402 FloatFractionHex,
403 FloatFractionHexNoUnderscore,
397404 FloatExponentUnsigned,
398 FloatExponentUnsignedHex,
399405 FloatExponentNumber,
400 FloatExponentNumberHex,
406 FloatExponentNumberNoUnderscore,
401407 Ampersand,
402408 Caret,
403409 Percent,
......@@ -412,6 +418,10 @@ pub const Tokenizer = struct {
412418 SawAtSign,
413419 };
414420
421 fn isIdentifierChar(char: u8) bool {
422 return std.ascii.isAlNum(char) or char == '_';
423 }
424
415425 pub fn next(self: *Tokenizer) Token {
416426 if (self.pending_invalid_token) |token| {
417427 self.pending_invalid_token = null;
......@@ -550,7 +560,7 @@ pub const Tokenizer = struct {
550560 result.id = Token.Id.IntegerLiteral;
551561 },
552562 '1'...'9' => {
553 state = State.IntegerLiteral;
563 state = State.IntegerLiteralDec;
554564 result.id = Token.Id.IntegerLiteral;
555565 },
556566 else => {
......@@ -1048,55 +1058,145 @@ pub const Tokenizer = struct {
10481058 else => self.checkLiteralCharacter(),
10491059 },
10501060 State.Zero => switch (c) {
1051 'b', 'o' => {
1052 state = State.IntegerLiteralWithRadix;
1061 'b' => {
1062 state = State.IntegerLiteralBinNoUnderscore;
1063 },
1064 'o' => {
1065 state = State.IntegerLiteralOctNoUnderscore;
10531066 },
10541067 'x' => {
1055 state = State.IntegerLiteralWithRadixHex;
1068 state = State.IntegerLiteralHexNoUnderscore;
10561069 },
1057 else => {
1058 // reinterpret as a normal number
1070 '0'...'9', '_', '.', 'e', 'E' => {
1071 // reinterpret as a decimal number
10591072 self.index -= 1;
1060 state = State.IntegerLiteral;
1073 state = State.IntegerLiteralDec;
1074 },
1075 else => {
1076 if (isIdentifierChar(c)) {
1077 result.id = Token.Id.Invalid;
1078 }
1079 break;
1080 },
1081 },
1082 State.IntegerLiteralBinNoUnderscore => switch (c) {
1083 '0'...'1' => {
1084 state = State.IntegerLiteralBin;
1085 },
1086 else => {
1087 result.id = Token.Id.Invalid;
1088 break;
1089 },
1090 },
1091 State.IntegerLiteralBin => switch (c) {
1092 '_' => {
1093 state = State.IntegerLiteralBinNoUnderscore;
1094 },
1095 '0'...'1' => {},
1096 else => {
1097 if (isIdentifierChar(c)) {
1098 result.id = Token.Id.Invalid;
1099 }
1100 break;
1101 },
1102 },
1103 State.IntegerLiteralOctNoUnderscore => switch (c) {
1104 '0'...'7' => {
1105 state = State.IntegerLiteralOct;
1106 },
1107 else => {
1108 result.id = Token.Id.Invalid;
1109 break;
1110 },
1111 },
1112 State.IntegerLiteralOct => switch (c) {
1113 '_' => {
1114 state = State.IntegerLiteralOctNoUnderscore;
1115 },
1116 '0'...'7' => {},
1117 else => {
1118 if (isIdentifierChar(c)) {
1119 result.id = Token.Id.Invalid;
1120 }
1121 break;
1122 },
1123 },
1124 State.IntegerLiteralDecNoUnderscore => switch (c) {
1125 '0'...'9' => {
1126 state = State.IntegerLiteralDec;
1127 },
1128 else => {
1129 result.id = Token.Id.Invalid;
1130 break;
10611131 },
10621132 },
1063 State.IntegerLiteral => switch (c) {
1133 State.IntegerLiteralDec => switch (c) {
1134 '_' => {
1135 state = State.IntegerLiteralDecNoUnderscore;
1136 },
10641137 '.' => {
1065 state = State.NumberDot;
1138 state = State.NumberDotDec;
1139 result.id = Token.Id.FloatLiteral;
10661140 },
1067 'p', 'P', 'e', 'E' => {
1141 'e', 'E' => {
10681142 state = State.FloatExponentUnsigned;
1143 result.id = Token.Id.FloatLiteral;
10691144 },
10701145 '0'...'9' => {},
1071 else => break,
1146 else => {
1147 if (isIdentifierChar(c)) {
1148 result.id = Token.Id.Invalid;
1149 }
1150 break;
1151 },
10721152 },
1073 State.IntegerLiteralWithRadix => switch (c) {
1074 '.' => {
1075 state = State.NumberDot;
1153 State.IntegerLiteralHexNoUnderscore => switch (c) {
1154 '0'...'9', 'a'...'f', 'A'...'F' => {
1155 state = State.IntegerLiteralHex;
1156 },
1157 else => {
1158 result.id = Token.Id.Invalid;
1159 break;
10761160 },
1077 '0'...'9' => {},
1078 else => break,
10791161 },
1080 State.IntegerLiteralWithRadixHex => switch (c) {
1162 State.IntegerLiteralHex => switch (c) {
1163 '_' => {
1164 state = State.IntegerLiteralHexNoUnderscore;
1165 },
10811166 '.' => {
10821167 state = State.NumberDotHex;
1168 result.id = Token.Id.FloatLiteral;
10831169 },
10841170 'p', 'P' => {
1085 state = State.FloatExponentUnsignedHex;
1171 state = State.FloatExponentUnsigned;
1172 result.id = Token.Id.FloatLiteral;
10861173 },
10871174 '0'...'9', 'a'...'f', 'A'...'F' => {},
1088 else => break,
1175 else => {
1176 if (isIdentifierChar(c)) {
1177 result.id = Token.Id.Invalid;
1178 }
1179 break;
1180 },
10891181 },
1090 State.NumberDot => switch (c) {
1182 State.NumberDotDec => switch (c) {
10911183 '.' => {
10921184 self.index -= 1;
10931185 state = State.Start;
10941186 break;
10951187 },
1096 else => {
1097 self.index -= 1;
1188 'e', 'E' => {
1189 state = State.FloatExponentUnsigned;
1190 },
1191 '0'...'9' => {
10981192 result.id = Token.Id.FloatLiteral;
1099 state = State.FloatFraction;
1193 state = State.FloatFractionDec;
1194 },
1195 else => {
1196 if (isIdentifierChar(c)) {
1197 result.id = Token.Id.Invalid;
1198 }
1199 break;
11001200 },
11011201 },
11021202 State.NumberDotHex => switch (c) {
......@@ -1105,65 +1205,112 @@ pub const Tokenizer = struct {
11051205 state = State.Start;
11061206 break;
11071207 },
1108 else => {
1109 self.index -= 1;
1208 'p', 'P' => {
1209 state = State.FloatExponentUnsigned;
1210 },
1211 '0'...'9', 'a'...'f', 'A'...'F' => {
11101212 result.id = Token.Id.FloatLiteral;
11111213 state = State.FloatFractionHex;
11121214 },
1215 else => {
1216 if (isIdentifierChar(c)) {
1217 result.id = Token.Id.Invalid;
1218 }
1219 break;
1220 },
11131221 },
1114 State.FloatFraction => switch (c) {
1222 State.FloatFractionDecNoUnderscore => switch (c) {
1223 '0'...'9' => {
1224 state = State.FloatFractionDec;
1225 },
1226 else => {
1227 result.id = Token.Id.Invalid;
1228 break;
1229 },
1230 },
1231 State.FloatFractionDec => switch (c) {
1232 '_' => {
1233 state = State.FloatFractionDecNoUnderscore;
1234 },
11151235 'e', 'E' => {
11161236 state = State.FloatExponentUnsigned;
11171237 },
11181238 '0'...'9' => {},
1119 else => break,
1239 else => {
1240 if (isIdentifierChar(c)) {
1241 result.id = Token.Id.Invalid;
1242 }
1243 break;
1244 },
1245 },
1246 State.FloatFractionHexNoUnderscore => switch (c) {
1247 '0'...'9', 'a'...'f', 'A'...'F' => {
1248 state = State.FloatFractionHex;
1249 },
1250 else => {
1251 result.id = Token.Id.Invalid;
1252 break;
1253 },
11201254 },
11211255 State.FloatFractionHex => switch (c) {
1256 '_' => {
1257 state = State.FloatFractionHexNoUnderscore;
1258 },
11221259 'p', 'P' => {
1123 state = State.FloatExponentUnsignedHex;
1260 state = State.FloatExponentUnsigned;
11241261 },
11251262 '0'...'9', 'a'...'f', 'A'...'F' => {},
1126 else => break,
1263 else => {
1264 if (isIdentifierChar(c)) {
1265 result.id = Token.Id.Invalid;
1266 }
1267 break;
1268 },
11271269 },
11281270 State.FloatExponentUnsigned => switch (c) {
11291271 '+', '-' => {
1130 state = State.FloatExponentNumber;
1272 state = State.FloatExponentNumberNoUnderscore;
11311273 },
11321274 else => {
11331275 // reinterpret as a normal exponent number
11341276 self.index -= 1;
1135 state = State.FloatExponentNumber;
1277 state = State.FloatExponentNumberNoUnderscore;
11361278 },
11371279 },
1138 State.FloatExponentUnsignedHex => switch (c) {
1139 '+', '-' => {
1140 state = State.FloatExponentNumberHex;
1280 State.FloatExponentNumberNoUnderscore => switch (c) {
1281 '0'...'9' => {
1282 state = State.FloatExponentNumber;
11411283 },
11421284 else => {
1143 // reinterpret as a normal exponent number
1144 self.index -= 1;
1145 state = State.FloatExponentNumberHex;
1285 result.id = Token.Id.Invalid;
1286 break;
11461287 },
11471288 },
11481289 State.FloatExponentNumber => switch (c) {
1290 '_' => {
1291 state = State.FloatExponentNumberNoUnderscore;
1292 },
11491293 '0'...'9' => {},
1150 else => break,
1151 },
1152 State.FloatExponentNumberHex => switch (c) {
1153 '0'...'9', 'a'...'f', 'A'...'F' => {},
1154 else => break,
1294 else => {
1295 if (isIdentifierChar(c)) {
1296 result.id = Token.Id.Invalid;
1297 }
1298 break;
1299 },
11551300 },
11561301 }
11571302 } else if (self.index == self.buffer.len) {
11581303 switch (state) {
11591304 State.Start,
1160 State.IntegerLiteral,
1161 State.IntegerLiteralWithRadix,
1162 State.IntegerLiteralWithRadixHex,
1163 State.FloatFraction,
1305 State.IntegerLiteralDec,
1306 State.IntegerLiteralBin,
1307 State.IntegerLiteralOct,
1308 State.IntegerLiteralHex,
1309 State.NumberDotDec,
1310 State.NumberDotHex,
1311 State.FloatFractionDec,
11641312 State.FloatFractionHex,
11651313 State.FloatExponentNumber,
1166 State.FloatExponentNumberHex,
11671314 State.StringLiteral, // find this error later
11681315 State.MultilineStringLiteralLine,
11691316 State.Builtin,
......@@ -1184,10 +1331,14 @@ pub const Tokenizer = struct {
11841331 result.id = Token.Id.ContainerDocComment;
11851332 },
11861333
1187 State.NumberDot,
1188 State.NumberDotHex,
1334 State.IntegerLiteralDecNoUnderscore,
1335 State.IntegerLiteralBinNoUnderscore,
1336 State.IntegerLiteralOctNoUnderscore,
1337 State.IntegerLiteralHexNoUnderscore,
1338 State.FloatFractionDecNoUnderscore,
1339 State.FloatFractionHexNoUnderscore,
1340 State.FloatExponentNumberNoUnderscore,
11891341 State.FloatExponentUnsigned,
1190 State.FloatExponentUnsignedHex,
11911342 State.SawAtSign,
11921343 State.Backslash,
11931344 State.CharLiteral,
......@@ -1585,6 +1736,236 @@ test "correctly parse pointer assignment" {
15851736 });
15861737}
15871738
1739test "tokenizer - number literals decimal" {
1740 testTokenize("0", &[_]Token.Id{.IntegerLiteral});
1741 testTokenize("1", &[_]Token.Id{.IntegerLiteral});
1742 testTokenize("2", &[_]Token.Id{.IntegerLiteral});
1743 testTokenize("3", &[_]Token.Id{.IntegerLiteral});
1744 testTokenize("4", &[_]Token.Id{.IntegerLiteral});
1745 testTokenize("5", &[_]Token.Id{.IntegerLiteral});
1746 testTokenize("6", &[_]Token.Id{.IntegerLiteral});
1747 testTokenize("7", &[_]Token.Id{.IntegerLiteral});
1748 testTokenize("8", &[_]Token.Id{.IntegerLiteral});
1749 testTokenize("9", &[_]Token.Id{.IntegerLiteral});
1750 testTokenize("0a", &[_]Token.Id{ .Invalid, .Identifier });
1751 testTokenize("9b", &[_]Token.Id{ .Invalid, .Identifier });
1752 testTokenize("1z", &[_]Token.Id{ .Invalid, .Identifier });
1753 testTokenize("1z_1", &[_]Token.Id{ .Invalid, .Identifier });
1754 testTokenize("9z3", &[_]Token.Id{ .Invalid, .Identifier });
1755
1756 testTokenize("0_0", &[_]Token.Id{.IntegerLiteral});
1757 testTokenize("0001", &[_]Token.Id{.IntegerLiteral});
1758 testTokenize("01234567890", &[_]Token.Id{.IntegerLiteral});
1759 testTokenize("012_345_6789_0", &[_]Token.Id{.IntegerLiteral});
1760 testTokenize("0_1_2_3_4_5_6_7_8_9_0", &[_]Token.Id{.IntegerLiteral});
1761
1762 testTokenize("00_", &[_]Token.Id{.Invalid});
1763 testTokenize("0_0_", &[_]Token.Id{.Invalid});
1764 testTokenize("0__0", &[_]Token.Id{ .Invalid, .Identifier });
1765 testTokenize("0_0f", &[_]Token.Id{ .Invalid, .Identifier });
1766 testTokenize("0_0_f", &[_]Token.Id{ .Invalid, .Identifier });
1767 testTokenize("0_0_f_00", &[_]Token.Id{ .Invalid, .Identifier });
1768 testTokenize("1_,", &[_]Token.Id{ .Invalid, .Comma });
1769
1770 testTokenize("1.", &[_]Token.Id{.FloatLiteral});
1771 testTokenize("0.0", &[_]Token.Id{.FloatLiteral});
1772 testTokenize("1.0", &[_]Token.Id{.FloatLiteral});
1773 testTokenize("10.0", &[_]Token.Id{.FloatLiteral});
1774 testTokenize("0e0", &[_]Token.Id{.FloatLiteral});
1775 testTokenize("1e0", &[_]Token.Id{.FloatLiteral});
1776 testTokenize("1e100", &[_]Token.Id{.FloatLiteral});
1777 testTokenize("1.e100", &[_]Token.Id{.FloatLiteral});
1778 testTokenize("1.0e100", &[_]Token.Id{.FloatLiteral});
1779 testTokenize("1.0e+100", &[_]Token.Id{.FloatLiteral});
1780 testTokenize("1.0e-100", &[_]Token.Id{.FloatLiteral});
1781 testTokenize("1_0_0_0.0_0_0_0_0_1e1_0_0_0", &[_]Token.Id{.FloatLiteral});
1782 testTokenize("1.+", &[_]Token.Id{ .FloatLiteral, .Plus });
1783
1784 testTokenize("1e", &[_]Token.Id{.Invalid});
1785 testTokenize("1.0e1f0", &[_]Token.Id{ .Invalid, .Identifier });
1786 testTokenize("1.0p100", &[_]Token.Id{ .Invalid, .Identifier });
1787 testTokenize("1.0p-100", &[_]Token.Id{ .Invalid, .Identifier, .Minus, .IntegerLiteral });
1788 testTokenize("1.0p1f0", &[_]Token.Id{ .Invalid, .Identifier });
1789 testTokenize("1.0_,", &[_]Token.Id{ .Invalid, .Comma });
1790 testTokenize("1_.0", &[_]Token.Id{ .Invalid, .Period, .IntegerLiteral });
1791 testTokenize("1._", &[_]Token.Id{ .Invalid, .Identifier });
1792 testTokenize("1.a", &[_]Token.Id{ .Invalid, .Identifier });
1793 testTokenize("1.z", &[_]Token.Id{ .Invalid, .Identifier });
1794 testTokenize("1._0", &[_]Token.Id{ .Invalid, .Identifier });
1795 testTokenize("1._+", &[_]Token.Id{ .Invalid, .Identifier, .Plus });
1796 testTokenize("1._e", &[_]Token.Id{ .Invalid, .Identifier });
1797 testTokenize("1.0e", &[_]Token.Id{.Invalid});
1798 testTokenize("1.0e,", &[_]Token.Id{ .Invalid, .Comma });
1799 testTokenize("1.0e_", &[_]Token.Id{ .Invalid, .Identifier });
1800 testTokenize("1.0e+_", &[_]Token.Id{ .Invalid, .Identifier });
1801 testTokenize("1.0e-_", &[_]Token.Id{ .Invalid, .Identifier });
1802 testTokenize("1.0e0_+", &[_]Token.Id{ .Invalid, .Plus });
1803}
1804
1805test "tokenizer - number literals binary" {
1806 testTokenize("0b0", &[_]Token.Id{.IntegerLiteral});
1807 testTokenize("0b1", &[_]Token.Id{.IntegerLiteral});
1808 testTokenize("0b2", &[_]Token.Id{ .Invalid, .IntegerLiteral });
1809 testTokenize("0b3", &[_]Token.Id{ .Invalid, .IntegerLiteral });
1810 testTokenize("0b4", &[_]Token.Id{ .Invalid, .IntegerLiteral });
1811 testTokenize("0b5", &[_]Token.Id{ .Invalid, .IntegerLiteral });
1812 testTokenize("0b6", &[_]Token.Id{ .Invalid, .IntegerLiteral });
1813 testTokenize("0b7", &[_]Token.Id{ .Invalid, .IntegerLiteral });
1814 testTokenize("0b8", &[_]Token.Id{ .Invalid, .IntegerLiteral });
1815 testTokenize("0b9", &[_]Token.Id{ .Invalid, .IntegerLiteral });
1816 testTokenize("0ba", &[_]Token.Id{ .Invalid, .Identifier });
1817 testTokenize("0bb", &[_]Token.Id{ .Invalid, .Identifier });
1818 testTokenize("0bc", &[_]Token.Id{ .Invalid, .Identifier });
1819 testTokenize("0bd", &[_]Token.Id{ .Invalid, .Identifier });
1820 testTokenize("0be", &[_]Token.Id{ .Invalid, .Identifier });
1821 testTokenize("0bf", &[_]Token.Id{ .Invalid, .Identifier });
1822 testTokenize("0bz", &[_]Token.Id{ .Invalid, .Identifier });
1823
1824 testTokenize("0b0000_0000", &[_]Token.Id{.IntegerLiteral});
1825 testTokenize("0b1111_1111", &[_]Token.Id{.IntegerLiteral});
1826 testTokenize("0b10_10_10_10", &[_]Token.Id{.IntegerLiteral});
1827 testTokenize("0b0_1_0_1_0_1_0_1", &[_]Token.Id{.IntegerLiteral});
1828 testTokenize("0b1.", &[_]Token.Id{ .IntegerLiteral, .Period });
1829 testTokenize("0b1.0", &[_]Token.Id{ .IntegerLiteral, .Period, .IntegerLiteral });
1830
1831 testTokenize("0B0", &[_]Token.Id{ .Invalid, .Identifier });
1832 testTokenize("0b_", &[_]Token.Id{ .Invalid, .Identifier });
1833 testTokenize("0b_0", &[_]Token.Id{ .Invalid, .Identifier });
1834 testTokenize("0b1_", &[_]Token.Id{.Invalid});
1835 testTokenize("0b0__1", &[_]Token.Id{ .Invalid, .Identifier });
1836 testTokenize("0b0_1_", &[_]Token.Id{.Invalid});
1837 testTokenize("0b1e", &[_]Token.Id{ .Invalid, .Identifier });
1838 testTokenize("0b1p", &[_]Token.Id{ .Invalid, .Identifier });
1839 testTokenize("0b1e0", &[_]Token.Id{ .Invalid, .Identifier });
1840 testTokenize("0b1p0", &[_]Token.Id{ .Invalid, .Identifier });
1841 testTokenize("0b1_,", &[_]Token.Id{ .Invalid, .Comma });
1842}
1843
1844test "tokenizer - number literals octal" {
1845 testTokenize("0o0", &[_]Token.Id{.IntegerLiteral});
1846 testTokenize("0o1", &[_]Token.Id{.IntegerLiteral});
1847 testTokenize("0o2", &[_]Token.Id{.IntegerLiteral});
1848 testTokenize("0o3", &[_]Token.Id{.IntegerLiteral});
1849 testTokenize("0o4", &[_]Token.Id{.IntegerLiteral});
1850 testTokenize("0o5", &[_]Token.Id{.IntegerLiteral});
1851 testTokenize("0o6", &[_]Token.Id{.IntegerLiteral});
1852 testTokenize("0o7", &[_]Token.Id{.IntegerLiteral});
1853 testTokenize("0o8", &[_]Token.Id{ .Invalid, .IntegerLiteral });
1854 testTokenize("0o9", &[_]Token.Id{ .Invalid, .IntegerLiteral });
1855 testTokenize("0oa", &[_]Token.Id{ .Invalid, .Identifier });
1856 testTokenize("0ob", &[_]Token.Id{ .Invalid, .Identifier });
1857 testTokenize("0oc", &[_]Token.Id{ .Invalid, .Identifier });
1858 testTokenize("0od", &[_]Token.Id{ .Invalid, .Identifier });
1859 testTokenize("0oe", &[_]Token.Id{ .Invalid, .Identifier });
1860 testTokenize("0of", &[_]Token.Id{ .Invalid, .Identifier });
1861 testTokenize("0oz", &[_]Token.Id{ .Invalid, .Identifier });
1862
1863 testTokenize("0o01234567", &[_]Token.Id{.IntegerLiteral});
1864 testTokenize("0o0123_4567", &[_]Token.Id{.IntegerLiteral});
1865 testTokenize("0o01_23_45_67", &[_]Token.Id{.IntegerLiteral});
1866 testTokenize("0o0_1_2_3_4_5_6_7", &[_]Token.Id{.IntegerLiteral});
1867 testTokenize("0o7.", &[_]Token.Id{ .IntegerLiteral, .Period });
1868 testTokenize("0o7.0", &[_]Token.Id{ .IntegerLiteral, .Period, .IntegerLiteral });
1869
1870 testTokenize("0O0", &[_]Token.Id{ .Invalid, .Identifier });
1871 testTokenize("0o_", &[_]Token.Id{ .Invalid, .Identifier });
1872 testTokenize("0o_0", &[_]Token.Id{ .Invalid, .Identifier });
1873 testTokenize("0o1_", &[_]Token.Id{.Invalid});
1874 testTokenize("0o0__1", &[_]Token.Id{ .Invalid, .Identifier });
1875 testTokenize("0o0_1_", &[_]Token.Id{.Invalid});
1876 testTokenize("0o1e", &[_]Token.Id{ .Invalid, .Identifier });
1877 testTokenize("0o1p", &[_]Token.Id{ .Invalid, .Identifier });
1878 testTokenize("0o1e0", &[_]Token.Id{ .Invalid, .Identifier });
1879 testTokenize("0o1p0", &[_]Token.Id{ .Invalid, .Identifier });
1880 testTokenize("0o_,", &[_]Token.Id{ .Invalid, .Identifier, .Comma });
1881}
1882
1883test "tokenizer - number literals hexadeciaml" {
1884 testTokenize("0x0", &[_]Token.Id{.IntegerLiteral});
1885 testTokenize("0x1", &[_]Token.Id{.IntegerLiteral});
1886 testTokenize("0x2", &[_]Token.Id{.IntegerLiteral});
1887 testTokenize("0x3", &[_]Token.Id{.IntegerLiteral});
1888 testTokenize("0x4", &[_]Token.Id{.IntegerLiteral});
1889 testTokenize("0x5", &[_]Token.Id{.IntegerLiteral});
1890 testTokenize("0x6", &[_]Token.Id{.IntegerLiteral});
1891 testTokenize("0x7", &[_]Token.Id{.IntegerLiteral});
1892 testTokenize("0x8", &[_]Token.Id{.IntegerLiteral});
1893 testTokenize("0x9", &[_]Token.Id{.IntegerLiteral});
1894 testTokenize("0xa", &[_]Token.Id{.IntegerLiteral});
1895 testTokenize("0xb", &[_]Token.Id{.IntegerLiteral});
1896 testTokenize("0xc", &[_]Token.Id{.IntegerLiteral});
1897 testTokenize("0xd", &[_]Token.Id{.IntegerLiteral});
1898 testTokenize("0xe", &[_]Token.Id{.IntegerLiteral});
1899 testTokenize("0xf", &[_]Token.Id{.IntegerLiteral});
1900 testTokenize("0xA", &[_]Token.Id{.IntegerLiteral});
1901 testTokenize("0xB", &[_]Token.Id{.IntegerLiteral});
1902 testTokenize("0xC", &[_]Token.Id{.IntegerLiteral});
1903 testTokenize("0xD", &[_]Token.Id{.IntegerLiteral});
1904 testTokenize("0xE", &[_]Token.Id{.IntegerLiteral});
1905 testTokenize("0xF", &[_]Token.Id{.IntegerLiteral});
1906 testTokenize("0x0z", &[_]Token.Id{ .Invalid, .Identifier });
1907 testTokenize("0xz", &[_]Token.Id{ .Invalid, .Identifier });
1908
1909 testTokenize("0x0123456789ABCDEF", &[_]Token.Id{.IntegerLiteral});
1910 testTokenize("0x0123_4567_89AB_CDEF", &[_]Token.Id{.IntegerLiteral});
1911 testTokenize("0x01_23_45_67_89AB_CDE_F", &[_]Token.Id{.IntegerLiteral});
1912 testTokenize("0x0_1_2_3_4_5_6_7_8_9_A_B_C_D_E_F", &[_]Token.Id{.IntegerLiteral});
1913
1914 testTokenize("0X0", &[_]Token.Id{ .Invalid, .Identifier });
1915 testTokenize("0x_", &[_]Token.Id{ .Invalid, .Identifier });
1916 testTokenize("0x_1", &[_]Token.Id{ .Invalid, .Identifier });
1917 testTokenize("0x1_", &[_]Token.Id{.Invalid});
1918 testTokenize("0x0__1", &[_]Token.Id{ .Invalid, .Identifier });
1919 testTokenize("0x0_1_", &[_]Token.Id{.Invalid});
1920 testTokenize("0x_,", &[_]Token.Id{ .Invalid, .Identifier, .Comma });
1921
1922 testTokenize("0x1.", &[_]Token.Id{.FloatLiteral});
1923 testTokenize("0x1.0", &[_]Token.Id{.FloatLiteral});
1924 testTokenize("0xF.", &[_]Token.Id{.FloatLiteral});
1925 testTokenize("0xF.0", &[_]Token.Id{.FloatLiteral});
1926 testTokenize("0xF.F", &[_]Token.Id{.FloatLiteral});
1927 testTokenize("0xF.Fp0", &[_]Token.Id{.FloatLiteral});
1928 testTokenize("0xF.FP0", &[_]Token.Id{.FloatLiteral});
1929 testTokenize("0x1p0", &[_]Token.Id{.FloatLiteral});
1930 testTokenize("0xfp0", &[_]Token.Id{.FloatLiteral});
1931 testTokenize("0x1.+0xF.", &[_]Token.Id{ .FloatLiteral, .Plus, .FloatLiteral });
1932
1933 testTokenize("0x0123456.789ABCDEF", &[_]Token.Id{.FloatLiteral});
1934 testTokenize("0x0_123_456.789_ABC_DEF", &[_]Token.Id{.FloatLiteral});
1935 testTokenize("0x0_1_2_3_4_5_6.7_8_9_A_B_C_D_E_F", &[_]Token.Id{.FloatLiteral});
1936 testTokenize("0x0p0", &[_]Token.Id{.FloatLiteral});
1937 testTokenize("0x0.0p0", &[_]Token.Id{.FloatLiteral});
1938 testTokenize("0xff.ffp10", &[_]Token.Id{.FloatLiteral});
1939 testTokenize("0xff.ffP10", &[_]Token.Id{.FloatLiteral});
1940 testTokenize("0xff.p10", &[_]Token.Id{.FloatLiteral});
1941 testTokenize("0xffp10", &[_]Token.Id{.FloatLiteral});
1942 testTokenize("0xff_ff.ff_ffp1_0_0_0", &[_]Token.Id{.FloatLiteral});
1943 testTokenize("0xf_f_f_f.f_f_f_fp+1_000", &[_]Token.Id{.FloatLiteral});
1944 testTokenize("0xf_f_f_f.f_f_f_fp-1_00_0", &[_]Token.Id{.FloatLiteral});
1945
1946 testTokenize("0x1e", &[_]Token.Id{.IntegerLiteral});
1947 testTokenize("0x1e0", &[_]Token.Id{.IntegerLiteral});
1948 testTokenize("0x1p", &[_]Token.Id{.Invalid});
1949 testTokenize("0xfp0z1", &[_]Token.Id{ .Invalid, .Identifier });
1950 testTokenize("0xff.ffpff", &[_]Token.Id{ .Invalid, .Identifier });
1951 testTokenize("0x0.p", &[_]Token.Id{.Invalid});
1952 testTokenize("0x0.z", &[_]Token.Id{ .Invalid, .Identifier });
1953 testTokenize("0x0._", &[_]Token.Id{ .Invalid, .Identifier });
1954 testTokenize("0x0_.0", &[_]Token.Id{ .Invalid, .Period, .IntegerLiteral });
1955 testTokenize("0x0_.0.0", &[_]Token.Id{ .Invalid, .Period, .FloatLiteral });
1956 testTokenize("0x0._0", &[_]Token.Id{ .Invalid, .Identifier });
1957 testTokenize("0x0.0_", &[_]Token.Id{.Invalid});
1958 testTokenize("0x0_p0", &[_]Token.Id{ .Invalid, .Identifier });
1959 testTokenize("0x0_.p0", &[_]Token.Id{ .Invalid, .Period, .Identifier });
1960 testTokenize("0x0._p0", &[_]Token.Id{ .Invalid, .Identifier });
1961 testTokenize("0x0.0_p0", &[_]Token.Id{ .Invalid, .Identifier });
1962 testTokenize("0x0._0p0", &[_]Token.Id{ .Invalid, .Identifier });
1963 testTokenize("0x0.0p_0", &[_]Token.Id{ .Invalid, .Identifier });
1964 testTokenize("0x0.0p+_0", &[_]Token.Id{ .Invalid, .Identifier });
1965 testTokenize("0x0.0p-_0", &[_]Token.Id{ .Invalid, .Identifier });
1966 testTokenize("0x0.0p0_", &[_]Token.Id{ .Invalid, .Eof });
1967}
1968
15881969fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
15891970 var tokenizer = Tokenizer.init(source);
15901971 for (expected_tokens) |expected_token_id| {
src-self-hosted/ir.zig+9-6
......@@ -1311,13 +1311,16 @@ pub const Builder = struct {
13111311 var base: u8 = undefined;
13121312 var rest: []const u8 = undefined;
13131313 if (int_token.len >= 3 and int_token[0] == '0') {
1314 base = switch (int_token[1]) {
1315 'b' => 2,
1316 'o' => 8,
1317 'x' => 16,
1318 else => unreachable,
1319 };
13201314 rest = int_token[2..];
1315 switch (int_token[1]) {
1316 'b' => base = 2,
1317 'o' => base = 8,
1318 'x' => base = 16,
1319 else => {
1320 base = 10;
1321 rest = int_token;
1322 },
1323 }
13211324 } else {
13221325 base = 10;
13231326 rest = int_token;
src/parse_f128.c+62-17
......@@ -165,22 +165,36 @@ static long long scanexp(struct MuslFILE *f, int pok)
165165 int x;
166166 long long y;
167167 int neg = 0;
168
168
169169 c = shgetc(f);
170170 if (c=='+' || c=='-') {
171171 neg = (c=='-');
172172 c = shgetc(f);
173173 if (c-'0'>=10U && pok) shunget(f);
174174 }
175 if (c-'0'>=10U) {
175 if (c-'0'>=10U && c!='_') {
176176 shunget(f);
177177 return LLONG_MIN;
178178 }
179 for (x=0; c-'0'<10U && x<INT_MAX/10; c = shgetc(f))
180 x = 10*x + c-'0';
181 for (y=x; c-'0'<10U && y<LLONG_MAX/100; c = shgetc(f))
182 y = 10*y + c-'0';
183 for (; c-'0'<10U; c = shgetc(f));
179 for (x=0; ; c = shgetc(f)) {
180 if (c=='_') {
181 continue;
182 } else if (c-'0'<10U && x<INT_MAX/10) {
183 x = 10*x + c-'0';
184 } else {
185 break;
186 }
187 }
188 for (y=x; ; c = shgetc(f)) {
189 if (c=='_') {
190 continue;
191 } else if (c-'0'<10U && y<LLONG_MAX/100) {
192 y = 10*y + c-'0';
193 } else {
194 break;
195 }
196 }
197 for (; c-'0'<10U || c=='_'; c = shgetc(f));
184198 shunget(f);
185199 return neg ? -y : y;
186200}
......@@ -450,16 +464,36 @@ static float128_t decfloat(struct MuslFILE *f, int c, int bits, int emin, int si
450464 j=0;
451465 k=0;
452466
453 /* Don't let leading zeros consume buffer space */
454 for (; c=='0'; c = shgetc(f)) gotdig=1;
467 /* Don't let leading zeros/underscores consume buffer space */
468 for (; ; c = shgetc(f)) {
469 if (c=='_') {
470 continue;
471 } else if (c=='0') {
472 gotdig=1;
473 } else {
474 break;
475 }
476 }
477
455478 if (c=='.') {
456479 gotrad = 1;
457 for (c = shgetc(f); c=='0'; c = shgetc(f)) gotdig=1, lrp--;
480 for (c = shgetc(f); ; c = shgetc(f)) {
481 if (c == '_') {
482 continue;
483 } else if (c=='0') {
484 gotdig=1;
485 lrp--;
486 } else {
487 break;
488 }
489 }
458490 }
459491
460492 x[0] = 0;
461 for (; c-'0'<10U || c=='.'; c = shgetc(f)) {
462 if (c == '.') {
493 for (; c-'0'<10U || c=='.' || c=='_'; c = shgetc(f)) {
494 if (c == '_') {
495 continue;
496 } else if (c == '.') {
463497 if (gotrad) break;
464498 gotrad = 1;
465499 lrp = dc;
......@@ -773,18 +807,29 @@ static float128_t hexfloat(struct MuslFILE *f, int bits, int emin, int sign, int
773807
774808 c = shgetc(f);
775809
776 /* Skip leading zeros */
777 for (; c=='0'; c = shgetc(f)) gotdig = 1;
810 /* Skip leading zeros/underscores */
811 for (; c=='0' || c=='_'; c = shgetc(f)) gotdig = 1;
778812
779813 if (c=='.') {
780814 gotrad = 1;
781815 c = shgetc(f);
782816 /* Count zeros after the radix point before significand */
783 for (rp=0; c=='0'; c = shgetc(f), rp--) gotdig = 1;
817 for (rp=0; ; c = shgetc(f)) {
818 if (c == '_') {
819 continue;
820 } else if (c == '0') {
821 gotdig = 1;
822 rp--;
823 } else {
824 break;
825 }
826 }
784827 }
785828
786 for (; c-'0'<10U || (c|32)-'a'<6U || c=='.'; c = shgetc(f)) {
787 if (c=='.') {
829 for (; c-'0'<10U || (c|32)-'a'<6U || c=='.' || c=='_'; c = shgetc(f)) {
830 if (c=='_') {
831 continue;
832 } else if (c=='.') {
788833 if (gotrad) break;
789834 rp = dc;
790835 gotrad = 1;
src/tokenizer.cpp+89-56
......@@ -177,10 +177,13 @@ enum TokenizeState {
177177 TokenizeStateSymbol,
178178 TokenizeStateZero, // "0", which might lead to "0x"
179179 TokenizeStateNumber, // "123", "0x123"
180 TokenizeStateNumberNoUnderscore, // "12_", "0x12_" next char must be digit
180181 TokenizeStateNumberDot,
181182 TokenizeStateFloatFraction, // "123.456", "0x123.456"
183 TokenizeStateFloatFractionNoUnderscore, // "123.45_", "0x123.45_"
182184 TokenizeStateFloatExponentUnsigned, // "123.456e", "123e", "0x123p"
183 TokenizeStateFloatExponentNumber, // "123.456e-", "123.456e5", "123.456e5e-5"
185 TokenizeStateFloatExponentNumber, // "123.456e7", "123.456e+7", "123.456e-7"
186 TokenizeStateFloatExponentNumberNoUnderscore, // "123.456e7_", "123.456e+7_", "123.456e-7_"
184187 TokenizeStateString,
185188 TokenizeStateStringEscape,
186189 TokenizeStateStringEscapeUnicodeStart,
......@@ -233,14 +236,10 @@ struct Tokenize {
233236 Token *cur_tok;
234237 Tokenization *out;
235238 uint32_t radix;
236 int32_t exp_add_amt;
237 bool is_exp_negative;
239 bool is_trailing_underscore;
238240 size_t char_code_index;
239241 bool unicode;
240242 uint32_t char_code;
241 int exponent_in_bin_or_dec;
242 BigInt specified_exponent;
243 BigInt significand;
244243 size_t remaining_code_units;
245244};
246245
......@@ -426,20 +425,16 @@ void tokenize(Buf *buf, Tokenization *out) {
426425 case '0':
427426 t.state = TokenizeStateZero;
428427 begin_token(&t, TokenIdIntLiteral);
428 t.is_trailing_underscore = false;
429429 t.radix = 10;
430 t.exp_add_amt = 1;
431 t.exponent_in_bin_or_dec = 0;
432430 bigint_init_unsigned(&t.cur_tok->data.int_lit.bigint, 0);
433 bigint_init_unsigned(&t.specified_exponent, 0);
434431 break;
435432 case DIGIT_NON_ZERO:
436433 t.state = TokenizeStateNumber;
437434 begin_token(&t, TokenIdIntLiteral);
435 t.is_trailing_underscore = false;
438436 t.radix = 10;
439 t.exp_add_amt = 1;
440 t.exponent_in_bin_or_dec = 0;
441437 bigint_init_unsigned(&t.cur_tok->data.int_lit.bigint, get_digit_value(c));
442 bigint_init_unsigned(&t.specified_exponent, 0);
443438 break;
444439 case '"':
445440 begin_token(&t, TokenIdStringLiteral);
......@@ -1189,17 +1184,15 @@ void tokenize(Buf *buf, Tokenization *out) {
11891184 switch (c) {
11901185 case 'b':
11911186 t.radix = 2;
1192 t.state = TokenizeStateNumber;
1187 t.state = TokenizeStateNumberNoUnderscore;
11931188 break;
11941189 case 'o':
11951190 t.radix = 8;
1196 t.exp_add_amt = 3;
1197 t.state = TokenizeStateNumber;
1191 t.state = TokenizeStateNumberNoUnderscore;
11981192 break;
11991193 case 'x':
12001194 t.radix = 16;
1201 t.exp_add_amt = 4;
1202 t.state = TokenizeStateNumber;
1195 t.state = TokenizeStateNumberNoUnderscore;
12031196 break;
12041197 default:
12051198 // reinterpret as normal number
......@@ -1208,9 +1201,27 @@ void tokenize(Buf *buf, Tokenization *out) {
12081201 continue;
12091202 }
12101203 break;
1204 case TokenizeStateNumberNoUnderscore:
1205 if (c == '_') {
1206 invalid_char_error(&t, c);
1207 break;
1208 } else if (get_digit_value(c) < t.radix) {
1209 t.is_trailing_underscore = false;
1210 t.state = TokenizeStateNumber;
1211 }
1212 // fall through
12111213 case TokenizeStateNumber:
12121214 {
1215 if (c == '_') {
1216 t.is_trailing_underscore = true;
1217 t.state = TokenizeStateNumberNoUnderscore;
1218 break;
1219 }
12131220 if (c == '.') {
1221 if (t.is_trailing_underscore) {
1222 invalid_char_error(&t, c);
1223 break;
1224 }
12141225 if (t.radix != 16 && t.radix != 10) {
12151226 invalid_char_error(&t, c);
12161227 }
......@@ -1218,17 +1229,26 @@ void tokenize(Buf *buf, Tokenization *out) {
12181229 break;
12191230 }
12201231 if (is_exponent_signifier(c, t.radix)) {
1232 if (t.is_trailing_underscore) {
1233 invalid_char_error(&t, c);
1234 break;
1235 }
12211236 if (t.radix != 16 && t.radix != 10) {
12221237 invalid_char_error(&t, c);
12231238 }
12241239 t.state = TokenizeStateFloatExponentUnsigned;
1240 t.radix = 10; // exponent is always base 10
12251241 assert(t.cur_tok->id == TokenIdIntLiteral);
1226 bigint_init_bigint(&t.significand, &t.cur_tok->data.int_lit.bigint);
12271242 set_token_id(&t, t.cur_tok, TokenIdFloatLiteral);
12281243 break;
12291244 }
12301245 uint32_t digit_value = get_digit_value(c);
12311246 if (digit_value >= t.radix) {
1247 if (t.is_trailing_underscore) {
1248 invalid_char_error(&t, c);
1249 break;
1250 }
1251
12321252 if (is_symbol_char(c)) {
12331253 invalid_char_error(&t, c);
12341254 }
......@@ -1259,20 +1279,41 @@ void tokenize(Buf *buf, Tokenization *out) {
12591279 continue;
12601280 }
12611281 t.pos -= 1;
1262 t.state = TokenizeStateFloatFraction;
1282 t.state = TokenizeStateFloatFractionNoUnderscore;
12631283 assert(t.cur_tok->id == TokenIdIntLiteral);
1264 bigint_init_bigint(&t.significand, &t.cur_tok->data.int_lit.bigint);
12651284 set_token_id(&t, t.cur_tok, TokenIdFloatLiteral);
12661285 continue;
12671286 }
1287 case TokenizeStateFloatFractionNoUnderscore:
1288 if (c == '_') {
1289 invalid_char_error(&t, c);
1290 } else if (get_digit_value(c) < t.radix) {
1291 t.is_trailing_underscore = false;
1292 t.state = TokenizeStateFloatFraction;
1293 }
1294 // fall through
12681295 case TokenizeStateFloatFraction:
12691296 {
1297 if (c == '_') {
1298 t.is_trailing_underscore = true;
1299 t.state = TokenizeStateFloatFractionNoUnderscore;
1300 break;
1301 }
12701302 if (is_exponent_signifier(c, t.radix)) {
1303 if (t.is_trailing_underscore) {
1304 invalid_char_error(&t, c);
1305 break;
1306 }
12711307 t.state = TokenizeStateFloatExponentUnsigned;
1308 t.radix = 10; // exponent is always base 10
12721309 break;
12731310 }
12741311 uint32_t digit_value = get_digit_value(c);
12751312 if (digit_value >= t.radix) {
1313 if (t.is_trailing_underscore) {
1314 invalid_char_error(&t, c);
1315 break;
1316 }
12761317 if (is_symbol_char(c)) {
12771318 invalid_char_error(&t, c);
12781319 }
......@@ -1282,46 +1323,47 @@ void tokenize(Buf *buf, Tokenization *out) {
12821323 t.state = TokenizeStateStart;
12831324 continue;
12841325 }
1285 t.exponent_in_bin_or_dec -= t.exp_add_amt;
1286 if (t.radix == 10) {
1287 // For now we use strtod to parse decimal floats, so we just have to get to the
1288 // end of the token.
1289 break;
1290 }
1291 BigInt digit_value_bi;
1292 bigint_init_unsigned(&digit_value_bi, digit_value);
1293
1294 BigInt radix_bi;
1295 bigint_init_unsigned(&radix_bi, t.radix);
1296
1297 BigInt multiplied;
1298 bigint_mul(&multiplied, &t.significand, &radix_bi);
12991326
1300 bigint_add(&t.significand, &multiplied, &digit_value_bi);
1301 break;
1327 // we use parse_f128 to generate the float literal, so just
1328 // need to get to the end of the token
13021329 }
1330 break;
13031331 case TokenizeStateFloatExponentUnsigned:
13041332 switch (c) {
13051333 case '+':
1306 t.is_exp_negative = false;
1307 t.state = TokenizeStateFloatExponentNumber;
1334 t.state = TokenizeStateFloatExponentNumberNoUnderscore;
13081335 break;
13091336 case '-':
1310 t.is_exp_negative = true;
1311 t.state = TokenizeStateFloatExponentNumber;
1337 t.state = TokenizeStateFloatExponentNumberNoUnderscore;
13121338 break;
13131339 default:
13141340 // reinterpret as normal exponent number
13151341 t.pos -= 1;
1316 t.is_exp_negative = false;
1317 t.state = TokenizeStateFloatExponentNumber;
1342 t.state = TokenizeStateFloatExponentNumberNoUnderscore;
13181343 continue;
13191344 }
13201345 break;
1346 case TokenizeStateFloatExponentNumberNoUnderscore:
1347 if (c == '_') {
1348 invalid_char_error(&t, c);
1349 } else if (get_digit_value(c) < t.radix) {
1350 t.is_trailing_underscore = false;
1351 t.state = TokenizeStateFloatExponentNumber;
1352 }
1353 // fall through
13211354 case TokenizeStateFloatExponentNumber:
13221355 {
1356 if (c == '_') {
1357 t.is_trailing_underscore = true;
1358 t.state = TokenizeStateFloatExponentNumberNoUnderscore;
1359 break;
1360 }
13231361 uint32_t digit_value = get_digit_value(c);
13241362 if (digit_value >= t.radix) {
1363 if (t.is_trailing_underscore) {
1364 invalid_char_error(&t, c);
1365 break;
1366 }
13251367 if (is_symbol_char(c)) {
13261368 invalid_char_error(&t, c);
13271369 }
......@@ -1331,21 +1373,9 @@ void tokenize(Buf *buf, Tokenization *out) {
13311373 t.state = TokenizeStateStart;
13321374 continue;
13331375 }
1334 if (t.radix == 10) {
1335 // For now we use strtod to parse decimal floats, so we just have to get to the
1336 // end of the token.
1337 break;
1338 }
1339 BigInt digit_value_bi;
1340 bigint_init_unsigned(&digit_value_bi, digit_value);
1341
1342 BigInt radix_bi;
1343 bigint_init_unsigned(&radix_bi, 10);
1344
1345 BigInt multiplied;
1346 bigint_mul(&multiplied, &t.specified_exponent, &radix_bi);
13471376
1348 bigint_add(&t.specified_exponent, &multiplied, &digit_value_bi);
1377 // we use parse_f128 to generate the float literal, so just
1378 // need to get to the end of the token
13491379 }
13501380 break;
13511381 case TokenizeStateSawDash:
......@@ -1399,6 +1429,9 @@ void tokenize(Buf *buf, Tokenization *out) {
13991429 case TokenizeStateStart:
14001430 case TokenizeStateError:
14011431 break;
1432 case TokenizeStateNumberNoUnderscore:
1433 case TokenizeStateFloatFractionNoUnderscore:
1434 case TokenizeStateFloatExponentNumberNoUnderscore:
14021435 case TokenizeStateNumberDot:
14031436 tokenize_error(&t, "unterminated number literal");
14041437 break;
test/compile_errors.zig+153-1
......@@ -395,11 +395,163 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {
395395 \\ var bad_float :f32 = 0.0;
396396 \\ bad_float = bad_float + .20;
397397 \\ std.debug.assert(bad_float < 1.0);
398 \\})
398 \\}
399399 , &[_][]const u8{
400400 "tmp.zig:5:29: error: invalid token: '.'",
401401 });
402402
403 cases.add("invalid exponent in float literal - 1",
404 \\fn main() void {
405 \\ var bad: f128 = 0x1.0p1ab1;
406 \\}
407 , &[_][]const u8{
408 "tmp.zig:2:28: error: invalid character: 'a'",
409 });
410
411 cases.add("invalid exponent in float literal - 2",
412 \\fn main() void {
413 \\ var bad: f128 = 0x1.0p50F;
414 \\}
415 , &[_][]const u8{
416 "tmp.zig:2:29: error: invalid character: 'F'",
417 });
418
419 cases.add("invalid underscore placement in float literal - 1",
420 \\fn main() void {
421 \\ var bad: f128 = 0._0;
422 \\}
423 , &[_][]const u8{
424 "tmp.zig:2:23: error: invalid character: '_'",
425 });
426
427 cases.add("invalid underscore placement in float literal - 2",
428 \\fn main() void {
429 \\ var bad: f128 = 0_.0;
430 \\}
431 , &[_][]const u8{
432 "tmp.zig:2:23: error: invalid character: '.'",
433 });
434
435 cases.add("invalid underscore placement in float literal - 3",
436 \\fn main() void {
437 \\ var bad: f128 = 0.0_;
438 \\}
439 , &[_][]const u8{
440 "tmp.zig:2:25: error: invalid character: ';'",
441 });
442
443 cases.add("invalid underscore placement in float literal - 4",
444 \\fn main() void {
445 \\ var bad: f128 = 1.0e_1;
446 \\}
447 , &[_][]const u8{
448 "tmp.zig:2:25: error: invalid character: '_'",
449 });
450
451 cases.add("invalid underscore placement in float literal - 5",
452 \\fn main() void {
453 \\ var bad: f128 = 1.0e+_1;
454 \\}
455 , &[_][]const u8{
456 "tmp.zig:2:26: error: invalid character: '_'",
457 });
458
459 cases.add("invalid underscore placement in float literal - 6",
460 \\fn main() void {
461 \\ var bad: f128 = 1.0e-_1;
462 \\}
463 , &[_][]const u8{
464 "tmp.zig:2:26: error: invalid character: '_'",
465 });
466
467 cases.add("invalid underscore placement in float literal - 7",
468 \\fn main() void {
469 \\ var bad: f128 = 1.0e-1_;
470 \\}
471 , &[_][]const u8{
472 "tmp.zig:2:28: error: invalid character: ';'",
473 });
474
475 cases.add("invalid underscore placement in float literal - 9",
476 \\fn main() void {
477 \\ var bad: f128 = 1__0.0e-1;
478 \\}
479 , &[_][]const u8{
480 "tmp.zig:2:23: error: invalid character: '_'",
481 });
482
483 cases.add("invalid underscore placement in float literal - 10",
484 \\fn main() void {
485 \\ var bad: f128 = 1.0__0e-1;
486 \\}
487 , &[_][]const u8{
488 "tmp.zig:2:25: error: invalid character: '_'",
489 });
490
491 cases.add("invalid underscore placement in float literal - 11",
492 \\fn main() void {
493 \\ var bad: f128 = 1.0e-1__0;
494 \\}
495 , &[_][]const u8{
496 "tmp.zig:2:28: error: invalid character: '_'",
497 });
498
499 cases.add("invalid underscore placement in float literal - 12",
500 \\fn main() void {
501 \\ var bad: f128 = 0_x0.0;
502 \\}
503 , &[_][]const u8{
504 "tmp.zig:2:23: error: invalid character: 'x'",
505 });
506
507 cases.add("invalid underscore placement in float literal - 13",
508 \\fn main() void {
509 \\ var bad: f128 = 0x_0.0;
510 \\}
511 , &[_][]const u8{
512 "tmp.zig:2:23: error: invalid character: '_'",
513 });
514
515 cases.add("invalid underscore placement in float literal - 14",
516 \\fn main() void {
517 \\ var bad: f128 = 0x0.0_p1;
518 \\}
519 , &[_][]const u8{
520 "tmp.zig:2:27: error: invalid character: 'p'",
521 });
522
523 cases.add("invalid underscore placement in int literal - 1",
524 \\fn main() void {
525 \\ var bad: u128 = 0010_;
526 \\}
527 , &[_][]const u8{
528 "tmp.zig:2:26: error: invalid character: ';'",
529 });
530
531 cases.add("invalid underscore placement in int literal - 2",
532 \\fn main() void {
533 \\ var bad: u128 = 0b0010_;
534 \\}
535 , &[_][]const u8{
536 "tmp.zig:2:28: error: invalid character: ';'",
537 });
538
539 cases.add("invalid underscore placement in int literal - 3",
540 \\fn main() void {
541 \\ var bad: u128 = 0o0010_;
542 \\}
543 , &[_][]const u8{
544 "tmp.zig:2:28: error: invalid character: ';'",
545 });
546
547 cases.add("invalid underscore placement in int literal - 4",
548 \\fn main() void {
549 \\ var bad: u128 = 0x0010_;
550 \\}
551 , &[_][]const u8{
552 "tmp.zig:2:28: error: invalid character: ';'",
553 });
554
403555 cases.add("var args without c calling conv",
404556 \\fn foo(args: ...) void {}
405557 \\comptime {
test/stage1/behavior/math.zig+28
......@@ -411,6 +411,34 @@ test "quad hex float literal parsing accurate" {
411411 comptime S.doTheTest();
412412}
413413
414test "underscore separator parsing" {
415 expect(0_0_0_0 == 0);
416 expect(1_234_567 == 1234567);
417 expect(001_234_567 == 1234567);
418 expect(0_0_1_2_3_4_5_6_7 == 1234567);
419
420 expect(0b0_0_0_0 == 0);
421 expect(0b1010_1010 == 0b10101010);
422 expect(0b0000_1010_1010 == 0b10101010);
423 expect(0b1_0_1_0_1_0_1_0 == 0b10101010);
424
425 expect(0o0_0_0_0 == 0);
426 expect(0o1010_1010 == 0o10101010);
427 expect(0o0000_1010_1010 == 0o10101010);
428 expect(0o1_0_1_0_1_0_1_0 == 0o10101010);
429
430 expect(0x0_0_0_0 == 0);
431 expect(0x1010_1010 == 0x10101010);
432 expect(0x0000_1010_1010 == 0x10101010);
433 expect(0x1_0_1_0_1_0_1_0 == 0x10101010);
434
435 expect(123_456.789_000e1_0 == 123456.789000e10);
436 expect(0_1_2_3_4_5_6.7_8_9_0_0_0e0_0_1_0 == 123456.789000e10);
437
438 expect(0x1234_5678.9ABC_DEF0p-1_0 == 0x12345678.9ABCDEF0p-10);
439 expect(0x1_2_3_4_5_6_7_8.9_A_B_C_D_E_F_0p-0_0_0_1_0 == 0x12345678.9ABCDEF0p-10);
440}
441
414442test "hex float literal within range" {
415443 const a = 0x1.0p16383;
416444 const b = 0x0.1p16387;