authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2017-06-16 14:34:38-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2017-06-16 14:34:38-04:00
logc0f9012bed2e0646d15d454cffb971f4b7368edf
tree643c04eac9c5f091c9f9b57c1114f05ca9953bb0
parent865b53f2860405a718262abf9a794d2bf9529dbc

parseh: fix not recognizing integer suffixes on hex numbers

4 files changed, 189 insertions(+), 71 deletions(-)

src/c_tokenizer.cpp+96-63
...@@ -107,8 +107,11 @@ static void begin_token(CTokenize *ctok, CTokId id) {...@@ -107,8 +107,11 @@ static void begin_token(CTokenize *ctok, CTokId id) {
107 memset(&ctok->cur_tok->data.symbol, 0, sizeof(Buf));107 memset(&ctok->cur_tok->data.symbol, 0, sizeof(Buf));
108 buf_resize(&ctok->cur_tok->data.symbol, 0);108 buf_resize(&ctok->cur_tok->data.symbol, 0);
109 break;109 break;
110 case CTokIdCharLit:
111 case CTokIdNumLitInt:110 case CTokIdNumLitInt:
111 ctok->cur_tok->data.num_lit_int.x = 0;
112 ctok->cur_tok->data.num_lit_int.suffix = CNumLitSuffixNone;
113 break;
114 case CTokIdCharLit:
112 case CTokIdNumLitFloat:115 case CTokIdNumLitFloat:
113 case CTokIdMinus:116 case CTokIdMinus:
114 break;117 break;
...@@ -138,9 +141,9 @@ static void add_char(CTokenize *ctok, uint8_t c) {...@@ -138,9 +141,9 @@ static void add_char(CTokenize *ctok, uint8_t c) {
138141
139static void hex_digit(CTokenize *ctok, uint8_t value) {142static void hex_digit(CTokenize *ctok, uint8_t value) {
140 // TODO @mul_with_overflow143 // TODO @mul_with_overflow
141 ctok->cur_tok->data.num_lit_int *= 16;144 ctok->cur_tok->data.num_lit_int.x *= 16;
142 // TODO @add_with_overflow145 // TODO @add_with_overflow
143 ctok->cur_tok->data.num_lit_int += value;146 ctok->cur_tok->data.num_lit_int.x += value;
144147
145 static const uint8_t hex_digit[] = "0123456789abcdef";148 static const uint8_t hex_digit[] = "0123456789abcdef";
146 buf_append_char(&ctok->buf, hex_digit[value]);149 buf_append_char(&ctok->buf, hex_digit[value]);
...@@ -194,19 +197,15 @@ void tokenize_c_macro(CTokenize *ctok, const uint8_t *c) {...@@ -194,19 +197,15 @@ void tokenize_c_macro(CTokenize *ctok, const uint8_t *c) {
194 break;197 break;
195 case DIGIT_NON_ZERO:198 case DIGIT_NON_ZERO:
196 ctok->state = CTokStateDecimal;199 ctok->state = CTokStateDecimal;
197 ctok->unsigned_suffix = false;
198 ctok->long_suffix = false;
199 begin_token(ctok, CTokIdNumLitInt);200 begin_token(ctok, CTokIdNumLitInt);
200 ctok->cur_tok->data.num_lit_int = *c - '0';201 ctok->cur_tok->data.num_lit_int.x = *c - '0';
201 buf_resize(&ctok->buf, 0);202 buf_resize(&ctok->buf, 0);
202 buf_append_char(&ctok->buf, *c);203 buf_append_char(&ctok->buf, *c);
203 break;204 break;
204 case '0':205 case '0':
205 ctok->state = CTokStateGotZero;206 ctok->state = CTokStateGotZero;
206 ctok->unsigned_suffix = false;
207 ctok->long_suffix = false;
208 begin_token(ctok, CTokIdNumLitInt);207 begin_token(ctok, CTokIdNumLitInt);
209 ctok->cur_tok->data.num_lit_int = 0;208 ctok->cur_tok->data.num_lit_int.x = 0;
210 buf_resize(&ctok->buf, 0);209 buf_resize(&ctok->buf, 0);
211 buf_append_char(&ctok->buf, '0');210 buf_append_char(&ctok->buf, '0');
212 break;211 break;
...@@ -289,21 +288,21 @@ void tokenize_c_macro(CTokenize *ctok, const uint8_t *c) {...@@ -289,21 +288,21 @@ void tokenize_c_macro(CTokenize *ctok, const uint8_t *c) {
289 buf_append_char(&ctok->buf, *c);288 buf_append_char(&ctok->buf, *c);
290289
291 // TODO @mul_with_overflow290 // TODO @mul_with_overflow
292 ctok->cur_tok->data.num_lit_int *= 10;291 ctok->cur_tok->data.num_lit_int.x *= 10;
293 // TODO @add_with_overflow292 // TODO @add_with_overflow
294 ctok->cur_tok->data.num_lit_int += *c - '0';293 ctok->cur_tok->data.num_lit_int.x += *c - '0';
295 break;294 break;
296 case '\'':295 case '\'':
297 break;296 break;
298 case 'u':297 case 'u':
299 case 'U':298 case 'U':
300 ctok->unsigned_suffix = true;299 ctok->state = CTokStateNumLitIntSuffixU;
301 ctok->state = CTokStateIntSuffix;300 ctok->cur_tok->data.num_lit_int.suffix = CNumLitSuffixU;
302 break;301 break;
303 case 'l':302 case 'l':
304 case 'L':303 case 'L':
305 ctok->long_suffix = true;304 ctok->state = CTokStateNumLitIntSuffixL;
306 ctok->state = CTokStateIntSuffixLong;305 ctok->cur_tok->data.num_lit_int.suffix = CNumLitSuffixL;
307 break;306 break;
308 case '.':307 case '.':
309 buf_append_char(&ctok->buf, '.');308 buf_append_char(&ctok->buf, '.');
...@@ -317,50 +316,6 @@ void tokenize_c_macro(CTokenize *ctok, const uint8_t *c) {...@@ -317,50 +316,6 @@ void tokenize_c_macro(CTokenize *ctok, const uint8_t *c) {
317 continue;316 continue;
318 }317 }
319 break;318 break;
320 case CTokStateIntSuffix:
321 switch (*c) {
322 case 'l':
323 case 'L':
324 if (ctok->long_suffix) {
325 return mark_error(ctok);
326 }
327 ctok->long_suffix = true;
328 ctok->state = CTokStateIntSuffixLong;
329 break;
330 case 'u':
331 case 'U':
332 if (ctok->unsigned_suffix) {
333 return mark_error(ctok);
334 }
335 ctok->unsigned_suffix = true;
336 break;
337 default:
338 c -= 1;
339 end_token(ctok);
340 ctok->state = CTokStateStart;
341 continue;
342 }
343 break;
344 case CTokStateIntSuffixLong:
345 switch (*c) {
346 case 'l':
347 case 'L':
348 ctok->state = CTokStateIntSuffix;
349 break;
350 case 'u':
351 case 'U':
352 if (ctok->unsigned_suffix) {
353 return mark_error(ctok);
354 }
355 ctok->unsigned_suffix = true;
356 break;
357 default:
358 c -= 1;
359 end_token(ctok);
360 ctok->state = CTokStateStart;
361 continue;
362 }
363 break;
364 case CTokStateGotZero:319 case CTokStateGotZero:
365 switch (*c) {320 switch (*c) {
366 case 'x':321 case 'x':
...@@ -389,9 +344,9 @@ void tokenize_c_macro(CTokenize *ctok, const uint8_t *c) {...@@ -389,9 +344,9 @@ void tokenize_c_macro(CTokenize *ctok, const uint8_t *c) {
389 case '6':344 case '6':
390 case '7':345 case '7':
391 // TODO @mul_with_overflow346 // TODO @mul_with_overflow
392 ctok->cur_tok->data.num_lit_int *= 8;347 ctok->cur_tok->data.num_lit_int.x *= 8;
393 // TODO @add_with_overflow348 // TODO @add_with_overflow
394 ctok->cur_tok->data.num_lit_int += *c - '0';349 ctok->cur_tok->data.num_lit_int.x += *c - '0';
395 break;350 break;
396 case '8':351 case '8':
397 case '9':352 case '9':
...@@ -466,6 +421,82 @@ void tokenize_c_macro(CTokenize *ctok, const uint8_t *c) {...@@ -466,6 +421,82 @@ void tokenize_c_macro(CTokenize *ctok, const uint8_t *c) {
466 ctok->cur_tok->id = CTokIdNumLitFloat;421 ctok->cur_tok->id = CTokIdNumLitFloat;
467 ctok->state = CTokStateExpSign;422 ctok->state = CTokStateExpSign;
468 break;423 break;
424 case 'u':
425 case 'U':
426 // marks the number literal as unsigned
427 ctok->state = CTokStateNumLitIntSuffixU;
428 ctok->cur_tok->data.num_lit_int.suffix = CNumLitSuffixU;
429 break;
430 case 'l':
431 case 'L':
432 // marks the number literal as long
433 ctok->state = CTokStateNumLitIntSuffixL;
434 ctok->cur_tok->data.num_lit_int.suffix = CNumLitSuffixL;
435 break;
436 default:
437 c -= 1;
438 end_token(ctok);
439 ctok->state = CTokStateStart;
440 continue;
441 }
442 break;
443 case CTokStateNumLitIntSuffixU:
444 switch (*c) {
445 case 'l':
446 case 'L':
447 ctok->cur_tok->data.num_lit_int.suffix = CNumLitSuffixLU;
448 ctok->state = CTokStateNumLitIntSuffixUL;
449 break;
450 default:
451 c -= 1;
452 end_token(ctok);
453 ctok->state = CTokStateStart;
454 continue;
455 }
456 break;
457 case CTokStateNumLitIntSuffixL:
458 switch (*c) {
459 case 'l':
460 case 'L':
461 ctok->cur_tok->data.num_lit_int.suffix = CNumLitSuffixLL;
462 ctok->state = CTokStateNumLitIntSuffixLL;
463 break;
464 case 'u':
465 case 'U':
466 ctok->cur_tok->data.num_lit_int.suffix = CNumLitSuffixLU;
467 end_token(ctok);
468 ctok->state = CTokStateStart;
469 break;
470 default:
471 c -= 1;
472 end_token(ctok);
473 ctok->state = CTokStateStart;
474 continue;
475 }
476 break;
477 case CTokStateNumLitIntSuffixLL:
478 switch (*c) {
479 case 'u':
480 case 'U':
481 ctok->cur_tok->data.num_lit_int.suffix = CNumLitSuffixLLU;
482 end_token(ctok);
483 ctok->state = CTokStateStart;
484 break;
485 default:
486 c -= 1;
487 end_token(ctok);
488 ctok->state = CTokStateStart;
489 continue;
490 }
491 break;
492 case CTokStateNumLitIntSuffixUL:
493 switch (*c) {
494 case 'l':
495 case 'L':
496 ctok->cur_tok->data.num_lit_int.suffix = CNumLitSuffixLLU;
497 end_token(ctok);
498 ctok->state = CTokStateStart;
499 break;
469 default:500 default:
470 c -= 1;501 c -= 1;
471 end_token(ctok);502 end_token(ctok);
...@@ -681,8 +712,10 @@ found_end_of_macro:...@@ -681,8 +712,10 @@ found_end_of_macro:
681 case CTokStateHex:712 case CTokStateHex:
682 case CTokStateOctal:713 case CTokStateOctal:
683 case CTokStateGotZero:714 case CTokStateGotZero:
684 case CTokStateIntSuffix:715 case CTokStateNumLitIntSuffixU:
685 case CTokStateIntSuffixLong:716 case CTokStateNumLitIntSuffixL:
717 case CTokStateNumLitIntSuffixUL:
718 case CTokStateNumLitIntSuffixLL:
686 end_token(ctok);719 end_token(ctok);
687 break;720 break;
688 case CTokStateFloat:721 case CTokStateFloat:
src/c_tokenizer.hpp+19-5
...@@ -20,12 +20,26 @@ enum CTokId {...@@ -20,12 +20,26 @@ enum CTokId {
20 CTokIdMinus,20 CTokIdMinus,
21};21};
2222
23enum CNumLitSuffix {
24 CNumLitSuffixNone,
25 CNumLitSuffixL,
26 CNumLitSuffixU,
27 CNumLitSuffixLU,
28 CNumLitSuffixLL,
29 CNumLitSuffixLLU,
30};
31
32struct CNumLitInt {
33 uint64_t x;
34 CNumLitSuffix suffix;
35};
36
23struct CTok {37struct CTok {
24 enum CTokId id;38 enum CTokId id;
25 union {39 union {
26 uint8_t char_lit;40 uint8_t char_lit;
27 Buf str_lit;41 Buf str_lit;
28 uint64_t num_lit_int;42 CNumLitInt num_lit_int;
29 double num_lit_float;43 double num_lit_float;
30 Buf symbol;44 Buf symbol;
31 } data;45 } data;
...@@ -47,13 +61,15 @@ enum CTokState {...@@ -47,13 +61,15 @@ enum CTokState {
47 CTokStateOctal,61 CTokStateOctal,
48 CTokStateGotZero,62 CTokStateGotZero,
49 CTokStateHex,63 CTokStateHex,
50 CTokStateIntSuffix,
51 CTokStateIntSuffixLong,
52 CTokStateFloat,64 CTokStateFloat,
53 CTokStateExpSign,65 CTokStateExpSign,
54 CTokStateFloatExp,66 CTokStateFloatExp,
55 CTokStateFloatExpFirst,67 CTokStateFloatExpFirst,
56 CTokStateStrOctal,68 CTokStateStrOctal,
69 CTokStateNumLitIntSuffixU,
70 CTokStateNumLitIntSuffixL,
71 CTokStateNumLitIntSuffixLL,
72 CTokStateNumLitIntSuffixUL,
57};73};
5874
59struct CTokenize {75struct CTokenize {
...@@ -62,8 +78,6 @@ struct CTokenize {...@@ -62,8 +78,6 @@ struct CTokenize {
62 bool error;78 bool error;
63 CTok *cur_tok;79 CTok *cur_tok;
64 Buf buf;80 Buf buf;
65 bool unsigned_suffix;
66 bool long_suffix;
67 uint8_t cur_char;81 uint8_t cur_char;
68 int octal_index;82 int octal_index;
69};83};
src/parseh.cpp+32-3
...@@ -162,12 +162,16 @@ static Tld *create_global_str_lit_var(Context *c, Buf *name, Buf *value) {...@@ -162,12 +162,16 @@ static Tld *create_global_str_lit_var(Context *c, Buf *name, Buf *value) {
162 return &tld_var->base;162 return &tld_var->base;
163}163}
164164
165static Tld *create_global_num_lit_unsigned_negative(Context *c, Buf *name, uint64_t x, bool negative) {165static Tld *create_global_num_lit_unsigned_negative_type(Context *c, Buf *name, uint64_t x, bool negative, TypeTableEntry *type_entry) {
166 ConstExprValue *var_val = create_const_unsigned_negative(c->codegen->builtin_types.entry_num_lit_int, x, negative);166 ConstExprValue *var_val = create_const_unsigned_negative(type_entry, x, negative);
167 TldVar *tld_var = create_global_var(c, name, var_val, true);167 TldVar *tld_var = create_global_var(c, name, var_val, true);
168 return &tld_var->base;168 return &tld_var->base;
169}169}
170170
171static Tld *create_global_num_lit_unsigned_negative(Context *c, Buf *name, uint64_t x, bool negative) {
172 return create_global_num_lit_unsigned_negative_type(c, name, x, negative, c->codegen->builtin_types.entry_num_lit_int);
173}
174
171static Tld *create_global_num_lit_float(Context *c, Buf *name, double value) {175static Tld *create_global_num_lit_float(Context *c, Buf *name, double value) {
172 ConstExprValue *var_val = create_const_float(c->codegen->builtin_types.entry_num_lit_float, value);176 ConstExprValue *var_val = create_const_float(c->codegen->builtin_types.entry_num_lit_float, value);
173 TldVar *tld_var = create_global_var(c, name, var_val, true);177 TldVar *tld_var = create_global_var(c, name, var_val, true);
...@@ -1149,7 +1153,32 @@ static void process_macro(Context *c, CTokenize *ctok, Buf *name, const char *ch...@@ -1149,7 +1153,32 @@ static void process_macro(Context *c, CTokenize *ctok, Buf *name, const char *ch
1149 return;1153 return;
1150 case CTokIdNumLitInt:1154 case CTokIdNumLitInt:
1151 if (is_last) {1155 if (is_last) {
1152 Tld *tld = create_global_num_lit_unsigned_negative(c, name, tok->data.num_lit_int, negate);1156 Tld *tld;
1157 switch (tok->data.num_lit_int.suffix) {
1158 case CNumLitSuffixNone:
1159 tld = create_global_num_lit_unsigned_negative(c, name, tok->data.num_lit_int.x, negate);
1160 break;
1161 case CNumLitSuffixL:
1162 tld = create_global_num_lit_unsigned_negative_type(c, name, tok->data.num_lit_int.x, negate,
1163 c->codegen->builtin_types.entry_c_int[CIntTypeLong]);
1164 break;
1165 case CNumLitSuffixU:
1166 tld = create_global_num_lit_unsigned_negative_type(c, name, tok->data.num_lit_int.x, negate,
1167 c->codegen->builtin_types.entry_c_int[CIntTypeUInt]);
1168 break;
1169 case CNumLitSuffixLU:
1170 tld = create_global_num_lit_unsigned_negative_type(c, name, tok->data.num_lit_int.x, negate,
1171 c->codegen->builtin_types.entry_c_int[CIntTypeULong]);
1172 break;
1173 case CNumLitSuffixLL:
1174 tld = create_global_num_lit_unsigned_negative_type(c, name, tok->data.num_lit_int.x, negate,
1175 c->codegen->builtin_types.entry_c_int[CIntTypeLongLong]);
1176 break;
1177 case CNumLitSuffixLLU:
1178 tld = create_global_num_lit_unsigned_negative_type(c, name, tok->data.num_lit_int.x, negate,
1179 c->codegen->builtin_types.entry_c_int[CIntTypeULongLong]);
1180 break;
1181 }
1153 c->macro_table.put(name, tld);1182 c->macro_table.put(name, tld);
1154 }1183 }
1155 return;1184 return;
test/parseh.zig+42
...@@ -217,6 +217,48 @@ pub fn addCases(cases: &tests.ParseHContext) {...@@ -217,6 +217,48 @@ pub fn addCases(cases: &tests.ParseHContext) {
217 \\pub const SDL_INIT_VIDEO = 32;217 \\pub const SDL_INIT_VIDEO = 32;
218 );218 );
219219
220 cases.add("u integer suffix after hex literal",
221 \\#define SDL_INIT_VIDEO 0x00000020u /**< SDL_INIT_VIDEO implies SDL_INIT_EVENTS */
222 ,
223 \\pub const SDL_INIT_VIDEO: c_uint = 32;
224 );
225
226 cases.add("l integer suffix after hex literal",
227 \\#define SDL_INIT_VIDEO 0x00000020l /**< SDL_INIT_VIDEO implies SDL_INIT_EVENTS */
228 ,
229 \\pub const SDL_INIT_VIDEO: c_long = 32;
230 );
231
232 cases.add("ul integer suffix after hex literal",
233 \\#define SDL_INIT_VIDEO 0x00000020ul /**< SDL_INIT_VIDEO implies SDL_INIT_EVENTS */
234 ,
235 \\pub const SDL_INIT_VIDEO: c_ulong = 32;
236 );
237
238 cases.add("lu integer suffix after hex literal",
239 \\#define SDL_INIT_VIDEO 0x00000020lu /**< SDL_INIT_VIDEO implies SDL_INIT_EVENTS */
240 ,
241 \\pub const SDL_INIT_VIDEO: c_ulong = 32;
242 );
243
244 cases.add("ll integer suffix after hex literal",
245 \\#define SDL_INIT_VIDEO 0x00000020ll /**< SDL_INIT_VIDEO implies SDL_INIT_EVENTS */
246 ,
247 \\pub const SDL_INIT_VIDEO: c_longlong = 32;
248 );
249
250 cases.add("ull integer suffix after hex literal",
251 \\#define SDL_INIT_VIDEO 0x00000020ull /**< SDL_INIT_VIDEO implies SDL_INIT_EVENTS */
252 ,
253 \\pub const SDL_INIT_VIDEO: c_ulonglong = 32;
254 );
255
256 cases.add("llu integer suffix after hex literal",
257 \\#define SDL_INIT_VIDEO 0x00000020llu /**< SDL_INIT_VIDEO implies SDL_INIT_EVENTS */
258 ,
259 \\pub const SDL_INIT_VIDEO: c_ulonglong = 32;
260 );
261
220 cases.add("zig keywords in C code",262 cases.add("zig keywords in C code",
221 \\struct comptime {263 \\struct comptime {
222 \\ int defer;264 \\ int defer;