authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-02-02 01:40:46+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-02-02 01:40:46+02:00
log3b23929be565a66a94441d107448860079c2847d
tree73125a48fa30d49df06a55d0c74ff5d55a9d464d
parent4f2652d504c796bbba6d7ccf6a699dc01055e3e7
signature Commit is signed but in an unrecognized format.

use std.c.tokenizer in translate-c

2 files changed, 281 insertions(+), 1069 deletions(-)

src-self-hosted/c_tokenizer.zig deleted-977
......@@ -1,977 +0,0 @@
1const std = @import("std");
2const expect = std.testing.expect;
3const ZigClangSourceLocation = @import("clang.zig").ZigClangSourceLocation;
4const Context = @import("translate_c.zig").Context;
5const failDecl = @import("translate_c.zig").failDecl;
6
7pub const TokenList = std.SegmentedList(CToken, 32);
8
9pub const CToken = struct {
10 id: Id,
11 bytes: []const u8 = "",
12 num_lit_suffix: NumLitSuffix = .None,
13
14 pub const Id = enum {
15 CharLit,
16 StrLit,
17 NumLitInt,
18 NumLitFloat,
19 Identifier,
20 Plus,
21 Minus,
22 Slash,
23 LParen,
24 RParen,
25 Eof,
26 Dot,
27 Asterisk, // *
28 Ampersand, // &
29 And, // &&
30 Assign, // =
31 Or, // ||
32 Bang, // !
33 Tilde, // ~
34 Shl, // <<
35 Shr, // >>
36 Lt, // <
37 Lte, // <=
38 Gt, // >
39 Gte, // >=
40 Eq, // ==
41 Ne, // !=
42 Increment, // ++
43 Decrement, // --
44 Comma,
45 Fn,
46 Arrow, // ->
47 LBrace,
48 RBrace,
49 Pipe,
50 QuestionMark,
51 Colon,
52 };
53
54 pub const NumLitSuffix = enum {
55 None,
56 F,
57 L,
58 U,
59 LU,
60 LL,
61 LLU,
62 };
63};
64
65pub fn tokenizeCMacro(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, tl: *TokenList, chars: [*:0]const u8) !void {
66 var index: usize = 0;
67 var first = true;
68 while (true) {
69 const tok = try next(ctx, loc, name, chars, &index);
70 if (tok.id == .StrLit or tok.id == .CharLit)
71 try tl.push(try zigifyEscapeSequences(ctx, loc, name, tl.allocator, tok))
72 else
73 try tl.push(tok);
74 if (tok.id == .Eof)
75 return;
76 if (first) {
77 // distinguish NAME (EXPR) from NAME(ARGS)
78 first = false;
79 if (chars[index] == '(') {
80 try tl.push(.{
81 .id = .Fn,
82 .bytes = "",
83 });
84 }
85 }
86 }
87}
88
89fn zigifyEscapeSequences(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, allocator: *std.mem.Allocator, tok: CToken) !CToken {
90 for (tok.bytes) |c| {
91 if (c == '\\') {
92 break;
93 }
94 } else return tok;
95 var bytes = try allocator.alloc(u8, tok.bytes.len * 2);
96 var state: enum {
97 Start,
98 Escape,
99 Hex,
100 Octal,
101 } = .Start;
102 var i: usize = 0;
103 var count: u8 = 0;
104 var num: u8 = 0;
105 for (tok.bytes) |c| {
106 switch (state) {
107 .Escape => {
108 switch (c) {
109 'n', 'r', 't', '\\', '\'', '\"' => {
110 bytes[i] = c;
111 },
112 '0'...'7' => {
113 count += 1;
114 num += c - '0';
115 state = .Octal;
116 bytes[i] = 'x';
117 },
118 'x' => {
119 state = .Hex;
120 bytes[i] = 'x';
121 },
122 'a' => {
123 bytes[i] = 'x';
124 i += 1;
125 bytes[i] = '0';
126 i += 1;
127 bytes[i] = '7';
128 },
129 'b' => {
130 bytes[i] = 'x';
131 i += 1;
132 bytes[i] = '0';
133 i += 1;
134 bytes[i] = '8';
135 },
136 'f' => {
137 bytes[i] = 'x';
138 i += 1;
139 bytes[i] = '0';
140 i += 1;
141 bytes[i] = 'C';
142 },
143 'v' => {
144 bytes[i] = 'x';
145 i += 1;
146 bytes[i] = '0';
147 i += 1;
148 bytes[i] = 'B';
149 },
150 '?' => {
151 i -= 1;
152 bytes[i] = '?';
153 },
154 'u', 'U' => {
155 try failDecl(ctx, loc, name, "macro tokenizing failed: TODO unicode escape sequences", .{});
156 return error.TokenizingFailed;
157 },
158 else => {
159 try failDecl(ctx, loc, name, "macro tokenizing failed: unknown escape sequence", .{});
160 return error.TokenizingFailed;
161 },
162 }
163 i += 1;
164 if (state == .Escape)
165 state = .Start;
166 },
167 .Start => {
168 if (c == '\\') {
169 state = .Escape;
170 }
171 bytes[i] = c;
172 i += 1;
173 },
174 .Hex => {
175 switch (c) {
176 '0'...'9' => {
177 num = std.math.mul(u8, num, 16) catch {
178 try failDecl(ctx, loc, name, "macro tokenizing failed: hex literal overflowed", .{});
179 return error.TokenizingFailed;
180 };
181 num += c - '0';
182 },
183 'a'...'f' => {
184 num = std.math.mul(u8, num, 16) catch {
185 try failDecl(ctx, loc, name, "macro tokenizing failed: hex literal overflowed", .{});
186 return error.TokenizingFailed;
187 };
188 num += c - 'a' + 10;
189 },
190 'A'...'F' => {
191 num = std.math.mul(u8, num, 16) catch {
192 try failDecl(ctx, loc, name, "macro tokenizing failed: hex literal overflowed", .{});
193 return error.TokenizingFailed;
194 };
195 num += c - 'A' + 10;
196 },
197 else => {
198 i += std.fmt.formatIntBuf(bytes[i..], num, 16, false, std.fmt.FormatOptions{ .fill = '0', .width = 2 });
199 num = 0;
200 if (c == '\\')
201 state = .Escape
202 else
203 state = .Start;
204 bytes[i] = c;
205 i += 1;
206 },
207 }
208 },
209 .Octal => {
210 const accept_digit = switch (c) {
211 // The maximum length of a octal literal is 3 digits
212 '0'...'7' => count < 3,
213 else => false,
214 };
215
216 if (accept_digit) {
217 count += 1;
218 num = std.math.mul(u8, num, 8) catch {
219 try failDecl(ctx, loc, name, "macro tokenizing failed: octal literal overflowed", .{});
220 return error.TokenizingFailed;
221 };
222 num += c - '0';
223 } else {
224 i += std.fmt.formatIntBuf(bytes[i..], num, 16, false, std.fmt.FormatOptions{ .fill = '0', .width = 2 });
225 num = 0;
226 count = 0;
227 if (c == '\\')
228 state = .Escape
229 else
230 state = .Start;
231 bytes[i] = c;
232 i += 1;
233 }
234 },
235 }
236 }
237 if (state == .Hex or state == .Octal)
238 i += std.fmt.formatIntBuf(bytes[i..], num, 16, false, std.fmt.FormatOptions{ .fill = '0', .width = 2 });
239 return CToken{
240 .id = tok.id,
241 .bytes = bytes[0..i],
242 };
243}
244
245fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:0]const u8, i: *usize) !CToken {
246 var state: enum {
247 Start,
248 SawLt,
249 SawGt,
250 SawPlus,
251 SawMinus,
252 SawAmpersand,
253 SawPipe,
254 SawBang,
255 SawEq,
256 CharLit,
257 OpenComment,
258 Comment,
259 CommentStar,
260 Backslash,
261 String,
262 Identifier,
263 Decimal,
264 Octal,
265 SawZero,
266 Hex,
267 Bin,
268 Float,
269 ExpSign,
270 FloatExp,
271 FloatExpFirst,
272 NumLitIntSuffixU,
273 NumLitIntSuffixL,
274 NumLitIntSuffixLL,
275 NumLitIntSuffixUL,
276 Done,
277 } = .Start;
278
279 var result = CToken{
280 .bytes = "",
281 .id = .Eof,
282 };
283 var begin_index: usize = 0;
284 var digits: u8 = 0;
285 var pre_escape = state;
286
287 while (true) {
288 const c = chars[i.*];
289 if (c == 0) {
290 switch (state) {
291 .Identifier,
292 .Decimal,
293 .Hex,
294 .Bin,
295 .Octal,
296 .SawZero,
297 .Float,
298 .FloatExp,
299 => {
300 result.bytes = chars[begin_index..i.*];
301 return result;
302 },
303 .Start,
304 .SawMinus,
305 .Done,
306 .NumLitIntSuffixU,
307 .NumLitIntSuffixL,
308 .NumLitIntSuffixUL,
309 .NumLitIntSuffixLL,
310 .SawLt,
311 .SawGt,
312 .SawPlus,
313 .SawAmpersand,
314 .SawPipe,
315 .SawBang,
316 .SawEq,
317 => {
318 return result;
319 },
320 .CharLit,
321 .OpenComment,
322 .Comment,
323 .CommentStar,
324 .Backslash,
325 .String,
326 .ExpSign,
327 .FloatExpFirst,
328 => {
329 try failDecl(ctx, loc, name, "macro tokenizing failed: unexpected EOF", .{});
330 return error.TokenizingFailed;
331 },
332 }
333 }
334 switch (state) {
335 .Start => {
336 switch (c) {
337 ' ', '\t', '\x0B', '\x0C' => {},
338 '\'' => {
339 state = .CharLit;
340 result.id = .CharLit;
341 begin_index = i.*;
342 },
343 '\"' => {
344 state = .String;
345 result.id = .StrLit;
346 begin_index = i.*;
347 },
348 '/' => {
349 state = .OpenComment;
350 },
351 '\\' => {
352 state = .Backslash;
353 },
354 '\n', '\r' => {
355 return result;
356 },
357 'a'...'z', 'A'...'Z', '_' => {
358 state = .Identifier;
359 result.id = .Identifier;
360 begin_index = i.*;
361 },
362 '1'...'9' => {
363 state = .Decimal;
364 result.id = .NumLitInt;
365 begin_index = i.*;
366 },
367 '0' => {
368 state = .SawZero;
369 result.id = .NumLitInt;
370 begin_index = i.*;
371 },
372 '.' => {
373 result.id = .Dot;
374 state = .Done;
375 },
376 '<' => {
377 result.id = .Lt;
378 state = .SawLt;
379 },
380 '>' => {
381 result.id = .Gt;
382 state = .SawGt;
383 },
384 '(' => {
385 result.id = .LParen;
386 state = .Done;
387 },
388 ')' => {
389 result.id = .RParen;
390 state = .Done;
391 },
392 '*' => {
393 result.id = .Asterisk;
394 state = .Done;
395 },
396 '+' => {
397 result.id = .Plus;
398 state = .SawPlus;
399 },
400 '-' => {
401 result.id = .Minus;
402 state = .SawMinus;
403 },
404 '!' => {
405 result.id = .Bang;
406 state = .SawBang;
407 },
408 '~' => {
409 result.id = .Tilde;
410 state = .Done;
411 },
412 '=' => {
413 result.id = .Assign;
414 state = .SawEq;
415 },
416 ',' => {
417 result.id = .Comma;
418 state = .Done;
419 },
420 '[' => {
421 result.id = .LBrace;
422 state = .Done;
423 },
424 ']' => {
425 result.id = .RBrace;
426 state = .Done;
427 },
428 '|' => {
429 result.id = .Pipe;
430 state = .SawPipe;
431 },
432 '&' => {
433 result.id = .Ampersand;
434 state = .SawAmpersand;
435 },
436 '?' => {
437 result.id = .QuestionMark;
438 state = .Done;
439 },
440 ':' => {
441 result.id = .Colon;
442 state = .Done;
443 },
444 else => {
445 try failDecl(ctx, loc, name, "macro tokenizing failed: unexpected character '{c}'", .{c});
446 return error.TokenizingFailed;
447 },
448 }
449 },
450 .Done => return result,
451 .SawMinus => {
452 switch (c) {
453 '>' => {
454 result.id = .Arrow;
455 state = .Done;
456 },
457 '-' => {
458 result.id = .Decrement;
459 state = .Done;
460 },
461 else => return result,
462 }
463 },
464 .SawPlus => {
465 switch (c) {
466 '+' => {
467 result.id = .Increment;
468 state = .Done;
469 },
470 else => return result,
471 }
472 },
473 .SawLt => {
474 switch (c) {
475 '<' => {
476 result.id = .Shl;
477 state = .Done;
478 },
479 '=' => {
480 result.id = .Lte;
481 state = .Done;
482 },
483 else => return result,
484 }
485 },
486 .SawGt => {
487 switch (c) {
488 '>' => {
489 result.id = .Shr;
490 state = .Done;
491 },
492 '=' => {
493 result.id = .Gte;
494 state = .Done;
495 },
496 else => return result,
497 }
498 },
499 .SawPipe => {
500 switch (c) {
501 '|' => {
502 result.id = .Or;
503 state = .Done;
504 },
505 else => return result,
506 }
507 },
508 .SawAmpersand => {
509 switch (c) {
510 '&' => {
511 result.id = .And;
512 state = .Done;
513 },
514 else => return result,
515 }
516 },
517 .SawBang => {
518 switch (c) {
519 '=' => {
520 result.id = .Ne;
521 state = .Done;
522 },
523 else => return result,
524 }
525 },
526 .SawEq => {
527 switch (c) {
528 '=' => {
529 result.id = .Eq;
530 state = .Done;
531 },
532 else => return result,
533 }
534 },
535 .Float => {
536 switch (c) {
537 '.', '0'...'9' => {},
538 'e', 'E' => {
539 state = .ExpSign;
540 },
541 'f',
542 'F',
543 => {
544 result.num_lit_suffix = .F;
545 result.bytes = chars[begin_index..i.*];
546 state = .Done;
547 },
548 'l', 'L' => {
549 result.num_lit_suffix = .L;
550 result.bytes = chars[begin_index..i.*];
551 state = .Done;
552 },
553 else => {
554 result.bytes = chars[begin_index..i.*];
555 return result;
556 },
557 }
558 },
559 .ExpSign => {
560 switch (c) {
561 '+', '-' => {
562 state = .FloatExpFirst;
563 },
564 '0'...'9' => {
565 state = .FloatExp;
566 },
567 else => {
568 try failDecl(ctx, loc, name, "macro tokenizing failed: expected a digit or '+' or '-'", .{});
569 return error.TokenizingFailed;
570 },
571 }
572 },
573 .FloatExpFirst => {
574 switch (c) {
575 '0'...'9' => {
576 state = .FloatExp;
577 },
578 else => {
579 try failDecl(ctx, loc, name, "macro tokenizing failed: expected a digit", .{});
580 return error.TokenizingFailed;
581 },
582 }
583 },
584 .FloatExp => {
585 switch (c) {
586 '0'...'9' => {},
587 'f', 'F' => {
588 result.num_lit_suffix = .F;
589 result.bytes = chars[begin_index..i.*];
590 state = .Done;
591 },
592 'l', 'L' => {
593 result.num_lit_suffix = .L;
594 result.bytes = chars[begin_index..i.*];
595 state = .Done;
596 },
597 else => {
598 result.bytes = chars[begin_index..i.*];
599 return result;
600 },
601 }
602 },
603 .Decimal => {
604 switch (c) {
605 '0'...'9' => {},
606 '\'' => {},
607 'u', 'U' => {
608 state = .NumLitIntSuffixU;
609 result.num_lit_suffix = .U;
610 result.bytes = chars[begin_index..i.*];
611 },
612 'l', 'L' => {
613 state = .NumLitIntSuffixL;
614 result.num_lit_suffix = .L;
615 result.bytes = chars[begin_index..i.*];
616 },
617 '.' => {
618 result.id = .NumLitFloat;
619 state = .Float;
620 },
621 else => {
622 result.bytes = chars[begin_index..i.*];
623 return result;
624 },
625 }
626 },
627 .SawZero => {
628 switch (c) {
629 'x', 'X' => {
630 state = .Hex;
631 },
632 'b', 'B' => {
633 state = .Bin;
634 },
635 '.' => {
636 state = .Float;
637 result.id = .NumLitFloat;
638 },
639 'u', 'U' => {
640 state = .NumLitIntSuffixU;
641 result.num_lit_suffix = .U;
642 result.bytes = chars[begin_index..i.*];
643 },
644 'l', 'L' => {
645 state = .NumLitIntSuffixL;
646 result.num_lit_suffix = .L;
647 result.bytes = chars[begin_index..i.*];
648 },
649 else => {
650 i.* -= 1;
651 state = .Octal;
652 },
653 }
654 },
655 .Octal => {
656 switch (c) {
657 '0'...'7' => {},
658 '8', '9' => {
659 try failDecl(ctx, loc, name, "macro tokenizing failed: invalid digit '{c}' in octal number", .{c});
660 return error.TokenizingFailed;
661 },
662 'u', 'U' => {
663 state = .NumLitIntSuffixU;
664 result.num_lit_suffix = .U;
665 result.bytes = chars[begin_index..i.*];
666 },
667 'l', 'L' => {
668 state = .NumLitIntSuffixL;
669 result.num_lit_suffix = .L;
670 result.bytes = chars[begin_index..i.*];
671 },
672 else => {
673 result.bytes = chars[begin_index..i.*];
674 return result;
675 },
676 }
677 },
678 .Hex => {
679 switch (c) {
680 '0'...'9', 'a'...'f', 'A'...'F' => {},
681 'u', 'U' => {
682 // marks the number literal as unsigned
683 state = .NumLitIntSuffixU;
684 result.num_lit_suffix = .U;
685 result.bytes = chars[begin_index..i.*];
686 },
687 'l', 'L' => {
688 // marks the number literal as long
689 state = .NumLitIntSuffixL;
690 result.num_lit_suffix = .L;
691 result.bytes = chars[begin_index..i.*];
692 },
693 else => {
694 result.bytes = chars[begin_index..i.*];
695 return result;
696 },
697 }
698 },
699 .Bin => {
700 switch (c) {
701 '0'...'1' => {},
702 '2'...'9' => {
703 try failDecl(ctx, loc, name, "macro tokenizing failed: invalid digit '{c}' in binary number", .{c});
704 return error.TokenizingFailed;
705 },
706 'u', 'U' => {
707 // marks the number literal as unsigned
708 state = .NumLitIntSuffixU;
709 result.num_lit_suffix = .U;
710 result.bytes = chars[begin_index..i.*];
711 },
712 'l', 'L' => {
713 // marks the number literal as long
714 state = .NumLitIntSuffixL;
715 result.num_lit_suffix = .L;
716 result.bytes = chars[begin_index..i.*];
717 },
718 else => {
719 result.bytes = chars[begin_index..i.*];
720 return result;
721 },
722 }
723 },
724 .NumLitIntSuffixU => {
725 switch (c) {
726 'l', 'L' => {
727 result.num_lit_suffix = .LU;
728 state = .NumLitIntSuffixUL;
729 },
730 else => {
731 return result;
732 },
733 }
734 },
735 .NumLitIntSuffixL => {
736 switch (c) {
737 'l', 'L' => {
738 result.num_lit_suffix = .LL;
739 state = .NumLitIntSuffixLL;
740 },
741 'u', 'U' => {
742 result.num_lit_suffix = .LU;
743 state = .Done;
744 },
745 else => {
746 return result;
747 },
748 }
749 },
750 .NumLitIntSuffixLL => {
751 switch (c) {
752 'u', 'U' => {
753 result.num_lit_suffix = .LLU;
754 state = .Done;
755 },
756 else => {
757 return result;
758 },
759 }
760 },
761 .NumLitIntSuffixUL => {
762 switch (c) {
763 'l', 'L' => {
764 result.num_lit_suffix = .LLU;
765 state = .Done;
766 },
767 else => {
768 return result;
769 },
770 }
771 },
772 .Identifier => {
773 switch (c) {
774 '_', 'a'...'z', 'A'...'Z', '0'...'9' => {},
775 else => {
776 result.bytes = chars[begin_index..i.*];
777 return result;
778 },
779 }
780 },
781 .String => {
782 switch (c) {
783 '\"' => {
784 result.bytes = chars[begin_index .. i.* + 1];
785 state = .Done;
786 },
787 else => {},
788 }
789 },
790 .CharLit => {
791 switch (c) {
792 '\'' => {
793 result.bytes = chars[begin_index .. i.* + 1];
794 state = .Done;
795 },
796 else => {},
797 }
798 },
799 .OpenComment => {
800 switch (c) {
801 '/' => {
802 return result;
803 },
804 '*' => {
805 state = .Comment;
806 },
807 else => {
808 result.id = .Slash;
809 state = .Done;
810 },
811 }
812 },
813 .Comment => {
814 switch (c) {
815 '*' => {
816 state = .CommentStar;
817 },
818 else => {},
819 }
820 },
821 .CommentStar => {
822 switch (c) {
823 '/' => {
824 state = .Start;
825 },
826 else => {
827 state = .Comment;
828 },
829 }
830 },
831 .Backslash => {
832 switch (c) {
833 ' ', '\t', '\x0B', '\x0C' => {},
834 '\n', '\r' => {
835 state = .Start;
836 },
837 else => {
838 try failDecl(ctx, loc, name, "macro tokenizing failed: expected whitespace", .{});
839 return error.TokenizingFailed;
840 },
841 }
842 },
843 }
844 i.* += 1;
845 }
846 unreachable;
847}
848
849fn expectTokens(tl: *TokenList, src: [*:0]const u8, expected: []CToken) void {
850 // these can be undefined since they are only used for error reporting
851 tokenizeCMacro(undefined, undefined, undefined, tl, src) catch unreachable;
852 var it = tl.iterator(0);
853 for (expected) |t| {
854 var tok = it.next().?;
855 std.testing.expectEqual(t.id, tok.id);
856 if (t.bytes.len > 0) {
857 //std.debug.warn(" {} = {}\n", .{tok.bytes, t.bytes});
858 std.testing.expectEqualSlices(u8, tok.bytes, t.bytes);
859 }
860 if (t.num_lit_suffix != .None) {
861 std.testing.expectEqual(t.num_lit_suffix, tok.num_lit_suffix);
862 }
863 }
864 std.testing.expect(it.next() == null);
865 tl.shrink(0);
866}
867
868test "tokenize macro" {
869 var tl = TokenList.init(std.testing.allocator);
870 defer tl.deinit();
871
872 expectTokens(&tl, "TEST(0\n", &[_]CToken{
873 .{ .id = .Identifier, .bytes = "TEST" },
874 .{ .id = .Fn },
875 .{ .id = .LParen },
876 .{ .id = .NumLitInt, .bytes = "0" },
877 .{ .id = .Eof },
878 });
879
880 expectTokens(&tl, "__FLT_MIN_10_EXP__ -37\n", &[_]CToken{
881 .{ .id = .Identifier, .bytes = "__FLT_MIN_10_EXP__" },
882 .{ .id = .Minus },
883 .{ .id = .NumLitInt, .bytes = "37" },
884 .{ .id = .Eof },
885 });
886
887 expectTokens(&tl, "__llvm__ 1\n#define", &[_]CToken{
888 .{ .id = .Identifier, .bytes = "__llvm__" },
889 .{ .id = .NumLitInt, .bytes = "1" },
890 .{ .id = .Eof },
891 });
892
893 expectTokens(&tl, "TEST 2", &[_]CToken{
894 .{ .id = .Identifier, .bytes = "TEST" },
895 .{ .id = .NumLitInt, .bytes = "2" },
896 .{ .id = .Eof },
897 });
898
899 expectTokens(&tl, "FOO 0ull", &[_]CToken{
900 .{ .id = .Identifier, .bytes = "FOO" },
901 .{ .id = .NumLitInt, .bytes = "0", .num_lit_suffix = .LLU },
902 .{ .id = .Eof },
903 });
904}
905
906test "tokenize macro ops" {
907 var tl = TokenList.init(std.testing.allocator);
908 defer tl.deinit();
909
910 expectTokens(&tl, "ADD A + B", &[_]CToken{
911 .{ .id = .Identifier, .bytes = "ADD" },
912 .{ .id = .Identifier, .bytes = "A" },
913 .{ .id = .Plus },
914 .{ .id = .Identifier, .bytes = "B" },
915 .{ .id = .Eof },
916 });
917
918 expectTokens(&tl, "ADD (A) + B", &[_]CToken{
919 .{ .id = .Identifier, .bytes = "ADD" },
920 .{ .id = .LParen },
921 .{ .id = .Identifier, .bytes = "A" },
922 .{ .id = .RParen },
923 .{ .id = .Plus },
924 .{ .id = .Identifier, .bytes = "B" },
925 .{ .id = .Eof },
926 });
927
928 expectTokens(&tl, "ADD (A) + B", &[_]CToken{
929 .{ .id = .Identifier, .bytes = "ADD" },
930 .{ .id = .LParen },
931 .{ .id = .Identifier, .bytes = "A" },
932 .{ .id = .RParen },
933 .{ .id = .Plus },
934 .{ .id = .Identifier, .bytes = "B" },
935 .{ .id = .Eof },
936 });
937}
938
939test "escape sequences" {
940 var buf: [1024]u8 = undefined;
941 var alloc = std.heap.FixedBufferAllocator.init(buf[0..]);
942 const a = &alloc.allocator;
943 // these can be undefined since they are only used for error reporting
944 expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
945 .id = .StrLit,
946 .bytes = "\\x0077",
947 })).bytes, "\\x77"));
948 expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
949 .id = .StrLit,
950 .bytes = "\\24500",
951 })).bytes, "\\xa500"));
952 expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
953 .id = .StrLit,
954 .bytes = "\\x0077 abc",
955 })).bytes, "\\x77 abc"));
956 expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
957 .id = .StrLit,
958 .bytes = "\\045abc",
959 })).bytes, "\\x25abc"));
960
961 expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
962 .id = .CharLit,
963 .bytes = "\\0",
964 })).bytes, "\\x00"));
965 expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
966 .id = .CharLit,
967 .bytes = "\\00",
968 })).bytes, "\\x00"));
969 expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
970 .id = .CharLit,
971 .bytes = "\\000\\001",
972 })).bytes, "\\x00\\x01"));
973 expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
974 .id = .CharLit,
975 .bytes = "\\000abc",
976 })).bytes, "\\x00abc"));
977}
src-self-hosted/translate_c.zig+281-92
......@@ -6,8 +6,9 @@ const assert = std.debug.assert;
66const ast = std.zig.ast;
77const Token = std.zig.Token;
88usingnamespace @import("clang.zig");
9const ctok = @import("c_tokenizer.zig");
10const CToken = ctok.CToken;
9const ctok = std.c.tokenizer;
10const CToken = std.c.Token;
11const CTokenList = std.c.tokenizer.Source.TokenList;
1112const mem = std.mem;
1213const math = std.math;
1314
......@@ -4818,7 +4819,7 @@ fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void {
48184819 // TODO if we see #undef, delete it from the table
48194820 var it = ZigClangASTUnit_getLocalPreprocessingEntities_begin(unit);
48204821 const it_end = ZigClangASTUnit_getLocalPreprocessingEntities_end(unit);
4821 var tok_list = ctok.TokenList.init(c.a());
4822 var tok_list = CTokenList.init(c.a());
48224823 const scope = c.global_scope;
48234824
48244825 while (it.I != it_end.I) : (it.I += 1) {
......@@ -4829,6 +4830,7 @@ fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void {
48294830 const macro = @ptrCast(*ZigClangMacroDefinitionRecord, entity);
48304831 const raw_name = ZigClangMacroDefinitionRecord_getName_getNameStart(macro);
48314832 const begin_loc = ZigClangMacroDefinitionRecord_getSourceRange_getBegin(macro);
4833 // const end_loc = ZigClangMacroDefinitionRecord_getSourceRange_getEnd(macro);
48324834
48334835 const name = try c.str(raw_name);
48344836 // TODO https://github.com/ziglang/zig/issues/3756
......@@ -4839,42 +4841,61 @@ fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void {
48394841 }
48404842
48414843 const begin_c = ZigClangSourceManager_getCharacterData(c.source_manager, begin_loc);
4842 ctok.tokenizeCMacro(c, begin_loc, mangled_name, &tok_list, begin_c) catch |err| switch (err) {
4843 error.OutOfMemory => |e| return e,
4844 else => {
4845 continue;
4844 // const end_c = ZigClangSourceManager_getCharacterData(c.source_manager, end_loc);
4845 // const slice = begin_c[0 .. @ptrToInt(end_c) - @ptrToInt(begin_c)];
4846 const slice = begin_c[0..mem.len(u8, begin_c)];
4847
4848 tok_list.shrink(0);
4849 var tokenizer = std.c.Tokenizer{
4850 .source = &std.c.tokenizer.Source{
4851 .buffer = slice,
4852 .file_name = undefined,
4853 .tokens = undefined,
48464854 },
48474855 };
4856 while (true) {
4857 const tok = tokenizer.next();
4858 switch (tok.id) {
4859 .Nl, .Eof => {
4860 try tok_list.push(tok);
4861 break;
4862 },
4863 .LineComment, .MultiLineComment => continue,
4864 else => {},
4865 }
4866 try tok_list.push(tok);
4867 }
48484868
48494869 var tok_it = tok_list.iterator(0);
48504870 const first_tok = tok_it.next().?;
4851 assert(first_tok.id == .Identifier and mem.eql(u8, first_tok.bytes, name));
4871 assert(first_tok.id == .Identifier and mem.eql(u8, slice[first_tok.start..first_tok.end], name));
4872
4873 var macro_fn = false;
48524874 const next = tok_it.peek().?;
48534875 switch (next.id) {
48544876 .Identifier => {
48554877 // if it equals itself, ignore. for example, from stdio.h:
48564878 // #define stdin stdin
4857 if (mem.eql(u8, name, next.bytes)) {
4879 if (mem.eql(u8, name, slice[next.start..next.end])) {
48584880 continue;
48594881 }
48604882 },
4861 .Eof => {
4883 .Nl, .Eof => {
48624884 // this means it is a macro without a value
48634885 // we don't care about such things
48644886 continue;
48654887 },
4888 .LParen => {
4889 // if the name is immediately followed by a '(' then it is a function
4890 macro_fn = first_tok.end == next.start;
4891 },
48664892 else => {},
48674893 }
48684894
4869 const macro_fn = if (tok_it.peek().?.id == .Fn) blk: {
4870 _ = tok_it.next();
4871 break :blk true;
4872 } else false;
4873
48744895 (if (macro_fn)
4875 transMacroFnDefine(c, &tok_it, mangled_name, begin_loc)
4896 transMacroFnDefine(c, &tok_it, slice, mangled_name, begin_loc)
48764897 else
4877 transMacroDefine(c, &tok_it, mangled_name, begin_loc)) catch |err| switch (err) {
4898 transMacroDefine(c, &tok_it, slice, mangled_name, begin_loc)) catch |err| switch (err) {
48784899 error.ParseError => continue,
48794900 error.OutOfMemory => |e| return e,
48804901 };
......@@ -4884,15 +4905,15 @@ fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void {
48844905 }
48854906}
48864907
4887fn transMacroDefine(c: *Context, it: *ctok.TokenList.Iterator, name: []const u8, source_loc: ZigClangSourceLocation) ParseError!void {
4908fn transMacroDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, name: []const u8, source_loc: ZigClangSourceLocation) ParseError!void {
48884909 const scope = &c.global_scope.base;
48894910
48904911 const node = try transCreateNodeVarDecl(c, true, true, name);
48914912 node.eq_token = try appendToken(c, .Equal, "=");
48924913
4893 node.init_node = try parseCExpr(c, it, source_loc, scope);
4914 node.init_node = try parseCExpr(c, it, source, source_loc, scope);
48944915 const last = it.next().?;
4895 if (last.id != .Eof)
4916 if (last.id != .Eof and last.id != .Nl)
48964917 return failDecl(
48974918 c,
48984919 source_loc,
......@@ -4905,7 +4926,7 @@ fn transMacroDefine(c: *Context, it: *ctok.TokenList.Iterator, name: []const u8,
49054926 _ = try c.global_scope.macro_table.put(name, &node.base);
49064927}
49074928
4908fn transMacroFnDefine(c: *Context, it: *ctok.TokenList.Iterator, name: []const u8, source_loc: ZigClangSourceLocation) ParseError!void {
4929fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, name: []const u8, source_loc: ZigClangSourceLocation) ParseError!void {
49094930 const block_scope = try Scope.Block.init(c, &c.global_scope.base, null);
49104931 const scope = &block_scope.base;
49114932
......@@ -4937,7 +4958,7 @@ fn transMacroFnDefine(c: *Context, it: *ctok.TokenList.Iterator, name: []const u
49374958 );
49384959 }
49394960
4940 const mangled_name = try block_scope.makeMangledName(c, param_tok.bytes);
4961 const mangled_name = try block_scope.makeMangledName(c, source[param_tok.start..param_tok.end]);
49414962 const param_name_tok = try appendIdentifier(c, mangled_name);
49424963 _ = try appendToken(c, .Colon, ":");
49434964
......@@ -5000,7 +5021,7 @@ fn transMacroFnDefine(c: *Context, it: *ctok.TokenList.Iterator, name: []const u
50005021 const block = try transCreateNodeBlock(c, null);
50015022
50025023 const return_expr = try transCreateNodeReturnExpr(c);
5003 const expr = try parseCExpr(c, it, source_loc, scope);
5024 const expr = try parseCExpr(c, it, source, source_loc, scope);
50045025 const last = it.next().?;
50055026 if (last.id != .Eof)
50065027 return failDecl(
......@@ -5022,27 +5043,28 @@ fn transMacroFnDefine(c: *Context, it: *ctok.TokenList.Iterator, name: []const u
50225043
50235044const ParseError = Error || error{ParseError};
50245045
5025fn parseCExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node {
5026 const node = try parseCPrefixOpExpr(c, it, source_loc, scope);
5046fn parseCExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node {
5047 const node = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
50275048 switch (it.next().?.id) {
50285049 .QuestionMark => {
50295050 // must come immediately after expr
50305051 _ = try appendToken(c, .RParen, ")");
50315052 const if_node = try transCreateNodeIf(c);
50325053 if_node.condition = node;
5033 if_node.body = try parseCPrimaryExpr(c, it, source_loc, scope);
5054 if_node.body = try parseCPrimaryExpr(c, it, source, source_loc, scope);
50345055 if (it.next().?.id != .Colon) {
5056 const first_tok = it.list.at(0);
50355057 try failDecl(
50365058 c,
50375059 source_loc,
5038 it.list.at(0).*.bytes,
5060 source[first_tok.start..first_tok.end],
50395061 "unable to translate C expr: expected ':'",
50405062 .{},
50415063 );
50425064 return error.ParseError;
50435065 }
50445066 if_node.@"else" = try transCreateNodeElse(c);
5045 if_node.@"else".?.body = try parseCPrimaryExpr(c, it, source_loc, scope);
5067 if_node.@"else".?.body = try parseCPrimaryExpr(c, it, source, source_loc, scope);
50465068 return &if_node.base;
50475069 },
50485070 else => {
......@@ -5052,30 +5074,30 @@ fn parseCExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: ZigClangSou
50525074 }
50535075}
50545076
5055fn parseCNumLit(c: *Context, tok: *CToken, source_loc: ZigClangSourceLocation) ParseError!*ast.Node {
5056 if (tok.id == .NumLitInt) {
5057 var lit_bytes = tok.bytes;
5077fn parseCNumLit(c: *Context, tok: *CToken, source: []const u8, source_loc: ZigClangSourceLocation) ParseError!*ast.Node {
5078 var lit_bytes = source[tok.start..tok.end];
50585079
5059 if (tok.bytes.len > 2 and tok.bytes[0] == '0') {
5060 switch (tok.bytes[1]) {
5080 if (tok.id == .IntegerLiteral) {
5081 if (lit_bytes.len > 2 and lit_bytes[0] == '0') {
5082 switch (lit_bytes[1]) {
50615083 '0'...'7' => {
50625084 // Octal
5063 lit_bytes = try std.fmt.allocPrint(c.a(), "0o{}", .{tok.bytes});
5085 lit_bytes = try std.fmt.allocPrint(c.a(), "0o{}", .{lit_bytes});
50645086 },
50655087 'X' => {
50665088 // Hexadecimal with capital X, valid in C but not in Zig
5067 lit_bytes = try std.fmt.allocPrint(c.a(), "0x{}", .{tok.bytes[2..]});
5089 lit_bytes = try std.fmt.allocPrint(c.a(), "0x{}", .{lit_bytes[2..]});
50685090 },
50695091 else => {},
50705092 }
50715093 }
50725094
5073 if (tok.num_lit_suffix == .None) {
5095 if (tok.id.IntegerLiteral == .None) {
50745096 return transCreateNodeInt(c, lit_bytes);
50755097 }
50765098
50775099 const cast_node = try transCreateNodeBuiltinFnCall(c, "@as");
5078 try cast_node.params.push(try transCreateNodeIdentifier(c, switch (tok.num_lit_suffix) {
5100 try cast_node.params.push(try transCreateNodeIdentifier(c, switch (tok.id.IntegerLiteral) {
50795101 .U => "c_uint",
50805102 .L => "c_long",
50815103 .LU => "c_ulong",
......@@ -5083,55 +5105,216 @@ fn parseCNumLit(c: *Context, tok: *CToken, source_loc: ZigClangSourceLocation) P
50835105 .LLU => "c_ulonglong",
50845106 else => unreachable,
50855107 }));
5108 lit_bytes = lit_bytes[0 .. lit_bytes.len - switch (tok.id.IntegerLiteral) {
5109 .U, .L => @as(u8, 1),
5110 .LU, .LL => 2,
5111 .LLU => 3,
5112 else => unreachable,
5113 }];
50865114 _ = try appendToken(c, .Comma, ",");
50875115 try cast_node.params.push(try transCreateNodeInt(c, lit_bytes));
50885116 cast_node.rparen_token = try appendToken(c, .RParen, ")");
50895117 return &cast_node.base;
5090 } else if (tok.id == .NumLitFloat) {
5091 if (tok.num_lit_suffix == .None) {
5092 return transCreateNodeFloat(c, tok.bytes);
5118 } else if (tok.id == .FloatLiteral) {
5119 if (tok.id.FloatLiteral == .None) {
5120 return transCreateNodeFloat(c, lit_bytes);
50935121 }
50945122 const cast_node = try transCreateNodeBuiltinFnCall(c, "@as");
5095 try cast_node.params.push(try transCreateNodeIdentifier(c, switch (tok.num_lit_suffix) {
5123 try cast_node.params.push(try transCreateNodeIdentifier(c, switch (tok.id.FloatLiteral) {
50965124 .F => "f32",
50975125 .L => "f64",
50985126 else => unreachable,
50995127 }));
51005128 _ = try appendToken(c, .Comma, ",");
5101 try cast_node.params.push(try transCreateNodeFloat(c, tok.bytes));
5129 try cast_node.params.push(try transCreateNodeFloat(c, lit_bytes[0 .. lit_bytes.len - 1]));
51025130 cast_node.rparen_token = try appendToken(c, .RParen, ")");
51035131 return &cast_node.base;
51045132 } else unreachable;
51055133}
51065134
5107fn parseCPrimaryExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node {
5135fn zigifyEscapeSequences(ctx: *Context, source: []const u8, name: []const u8, source_loc: ZigClangSourceLocation) ![]const u8 {
5136 for (source) |c| {
5137 if (c == '\\') {
5138 break;
5139 }
5140 } else return source;
5141 var bytes = try ctx.a().alloc(u8, source.len * 2);
5142 var state: enum {
5143 Start,
5144 Escape,
5145 Hex,
5146 Octal,
5147 } = .Start;
5148 var i: usize = 0;
5149 var count: u8 = 0;
5150 var num: u8 = 0;
5151 for (source) |c| {
5152 switch (state) {
5153 .Escape => {
5154 switch (c) {
5155 'n', 'r', 't', '\\', '\'', '\"' => {
5156 bytes[i] = c;
5157 },
5158 '0'...'7' => {
5159 count += 1;
5160 num += c - '0';
5161 state = .Octal;
5162 bytes[i] = 'x';
5163 },
5164 'x' => {
5165 state = .Hex;
5166 bytes[i] = 'x';
5167 },
5168 'a' => {
5169 bytes[i] = 'x';
5170 i += 1;
5171 bytes[i] = '0';
5172 i += 1;
5173 bytes[i] = '7';
5174 },
5175 'b' => {
5176 bytes[i] = 'x';
5177 i += 1;
5178 bytes[i] = '0';
5179 i += 1;
5180 bytes[i] = '8';
5181 },
5182 'f' => {
5183 bytes[i] = 'x';
5184 i += 1;
5185 bytes[i] = '0';
5186 i += 1;
5187 bytes[i] = 'C';
5188 },
5189 'v' => {
5190 bytes[i] = 'x';
5191 i += 1;
5192 bytes[i] = '0';
5193 i += 1;
5194 bytes[i] = 'B';
5195 },
5196 '?' => {
5197 i -= 1;
5198 bytes[i] = '?';
5199 },
5200 'u', 'U' => {
5201 try failDecl(ctx, source_loc, name, "macro tokenizing failed: TODO unicode escape sequences", .{});
5202 return error.ParseError;
5203 },
5204 else => {
5205 try failDecl(ctx, source_loc, name, "macro tokenizing failed: unknown escape sequence", .{});
5206 return error.ParseError;
5207 },
5208 }
5209 i += 1;
5210 if (state == .Escape)
5211 state = .Start;
5212 },
5213 .Start => {
5214 if (c == '\\') {
5215 state = .Escape;
5216 }
5217 bytes[i] = c;
5218 i += 1;
5219 },
5220 .Hex => {
5221 switch (c) {
5222 '0'...'9' => {
5223 num = std.math.mul(u8, num, 16) catch {
5224 try failDecl(ctx, source_loc, name, "macro tokenizing failed: hex literal overflowed", .{});
5225 return error.ParseError;
5226 };
5227 num += c - '0';
5228 },
5229 'a'...'f' => {
5230 num = std.math.mul(u8, num, 16) catch {
5231 try failDecl(ctx, source_loc, name, "macro tokenizing failed: hex literal overflowed", .{});
5232 return error.ParseError;
5233 };
5234 num += c - 'a' + 10;
5235 },
5236 'A'...'F' => {
5237 num = std.math.mul(u8, num, 16) catch {
5238 try failDecl(ctx, source_loc, name, "macro tokenizing failed: hex literal overflowed", .{});
5239 return error.ParseError;
5240 };
5241 num += c - 'A' + 10;
5242 },
5243 else => {
5244 i += std.fmt.formatIntBuf(bytes[i..], num, 16, false, std.fmt.FormatOptions{ .fill = '0', .width = 2 });
5245 num = 0;
5246 if (c == '\\')
5247 state = .Escape
5248 else
5249 state = .Start;
5250 bytes[i] = c;
5251 i += 1;
5252 },
5253 }
5254 },
5255 .Octal => {
5256 const accept_digit = switch (c) {
5257 // The maximum length of a octal literal is 3 digits
5258 '0'...'7' => count < 3,
5259 else => false,
5260 };
5261
5262 if (accept_digit) {
5263 count += 1;
5264 num = std.math.mul(u8, num, 8) catch {
5265 try failDecl(ctx, source_loc, name, "macro tokenizing failed: octal literal overflowed", .{});
5266 return error.ParseError;
5267 };
5268 num += c - '0';
5269 } else {
5270 i += std.fmt.formatIntBuf(bytes[i..], num, 16, false, std.fmt.FormatOptions{ .fill = '0', .width = 2 });
5271 num = 0;
5272 count = 0;
5273 if (c == '\\')
5274 state = .Escape
5275 else
5276 state = .Start;
5277 bytes[i] = c;
5278 i += 1;
5279 }
5280 },
5281 }
5282 }
5283 if (state == .Hex or state == .Octal)
5284 i += std.fmt.formatIntBuf(bytes[i..], num, 16, false, std.fmt.FormatOptions{ .fill = '0', .width = 2 });
5285 return bytes[0..i];
5286}
5287
5288fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node {
51085289 const tok = it.next().?;
51095290 switch (tok.id) {
5110 .CharLit => {
5111 const token = try appendToken(c, .CharLiteral, tok.bytes);
5291 .CharLiteral => {
5292 const first_tok = it.list.at(0);
5293 const token = try appendToken(c, .CharLiteral, try zigifyEscapeSequences(c, source[tok.start..tok.end], source[first_tok.start..first_tok.end], source_loc));
51125294 const node = try c.a().create(ast.Node.CharLiteral);
51135295 node.* = ast.Node.CharLiteral{
51145296 .token = token,
51155297 };
51165298 return &node.base;
51175299 },
5118 .StrLit => {
5119 const token = try appendToken(c, .StringLiteral, tok.bytes);
5300 .StringLiteral => {
5301 const first_tok = it.list.at(0);
5302 const token = try appendToken(c, .StringLiteral, try zigifyEscapeSequences(c, source[tok.start..tok.end], source[first_tok.start..first_tok.end], source_loc));
51205303 const node = try c.a().create(ast.Node.StringLiteral);
51215304 node.* = ast.Node.StringLiteral{
51225305 .token = token,
51235306 };
51245307 return &node.base;
51255308 },
5126 .NumLitInt, .NumLitFloat => {
5127 return parseCNumLit(c, tok, source_loc);
5309 .IntegerLiteral, .FloatLiteral => {
5310 return parseCNumLit(c, tok, source, source_loc);
51285311 },
51295312 .Identifier => {
5130 const mangled_name = scope.getAlias(tok.bytes);
5313 const mangled_name = scope.getAlias(source[tok.start..tok.end]);
51315314 return transCreateNodeIdentifier(c, mangled_name);
51325315 },
51335316 .LParen => {
5134 const inner_node = try parseCExpr(c, it, source_loc, scope);
5317 const inner_node = try parseCExpr(c, it, source, source_loc, scope);
51355318
51365319 if (it.peek().?.id == .RParen) {
51375320 _ = it.next();
......@@ -5144,13 +5327,14 @@ fn parseCPrimaryExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: ZigC
51445327 // hack to get zig fmt to render a comma in builtin calls
51455328 _ = try appendToken(c, .Comma, ",");
51465329
5147 const node_to_cast = try parseCExpr(c, it, source_loc, scope);
5330 const node_to_cast = try parseCExpr(c, it, source, source_loc, scope);
51485331
51495332 if (it.next().?.id != .RParen) {
5333 const first_tok = it.list.at(0);
51505334 try failDecl(
51515335 c,
51525336 source_loc,
5153 it.list.at(0).*.bytes,
5337 source[first_tok.start..first_tok.end],
51545338 "unable to translate C expr: expected ')''",
51555339 .{},
51565340 );
......@@ -5228,10 +5412,11 @@ fn parseCPrimaryExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: ZigC
52285412 return &if_1.base;
52295413 },
52305414 else => {
5415 const first_tok = it.list.at(0);
52315416 try failDecl(
52325417 c,
52335418 source_loc,
5234 it.list.at(0).*.bytes,
5419 source[first_tok.start..first_tok.end],
52355420 "unable to translate C expr: unexpected token {}",
52365421 .{tok.id},
52375422 );
......@@ -5240,33 +5425,35 @@ fn parseCPrimaryExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: ZigC
52405425 }
52415426}
52425427
5243fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node {
5244 var node = try parseCPrimaryExpr(c, it, source_loc, scope);
5428fn parseCSuffixOpExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node {
5429 var node = try parseCPrimaryExpr(c, it, source, source_loc, scope);
52455430 while (true) {
52465431 const tok = it.next().?;
52475432 switch (tok.id) {
5248 .Dot => {
5433 .Period => {
52495434 const name_tok = it.next().?;
52505435 if (name_tok.id != .Identifier) {
5436 const first_tok = it.list.at(0);
52515437 try failDecl(
52525438 c,
52535439 source_loc,
5254 it.list.at(0).*.bytes,
5440 source[first_tok.start..first_tok.end],
52555441 "unable to translate C expr: expected identifier",
52565442 .{},
52575443 );
52585444 return error.ParseError;
52595445 }
52605446
5261 node = try transCreateNodeFieldAccess(c, node, name_tok.bytes);
5447 node = try transCreateNodeFieldAccess(c, node, source[name_tok.start..name_tok.end]);
52625448 },
52635449 .Arrow => {
52645450 const name_tok = it.next().?;
52655451 if (name_tok.id != .Identifier) {
5452 const first_tok = it.list.at(0);
52665453 try failDecl(
52675454 c,
52685455 source_loc,
5269 it.list.at(0).*.bytes,
5456 source[first_tok.start..first_tok.end],
52705457 "unable to translate C expr: expected identifier",
52715458 .{},
52725459 );
......@@ -5274,7 +5461,7 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
52745461 }
52755462
52765463 const deref = try transCreateNodePtrDeref(c, node);
5277 node = try transCreateNodeFieldAccess(c, deref, name_tok.bytes);
5464 node = try transCreateNodeFieldAccess(c, deref, source[name_tok.start..name_tok.end]);
52785465 },
52795466 .Asterisk => {
52805467 if (it.peek().?.id == .RParen) {
......@@ -5289,7 +5476,7 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
52895476 } else {
52905477 // expr * expr
52915478 const op_token = try appendToken(c, .Asterisk, "*");
5292 const rhs = try parseCPrimaryExpr(c, it, source_loc, scope);
5479 const rhs = try parseCPrimaryExpr(c, it, source, source_loc, scope);
52935480 const mul_node = try c.a().create(ast.Node.InfixOp);
52945481 mul_node.* = .{
52955482 .op_token = op_token,
......@@ -5300,9 +5487,9 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
53005487 node = &mul_node.base;
53015488 }
53025489 },
5303 .Shl => {
5490 .AngleBracketAngleBracketLeft => {
53045491 const op_token = try appendToken(c, .AngleBracketAngleBracketLeft, "<<");
5305 const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
5492 const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
53065493 const bitshift_node = try c.a().create(ast.Node.InfixOp);
53075494 bitshift_node.* = .{
53085495 .op_token = op_token,
......@@ -5312,9 +5499,9 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
53125499 };
53135500 node = &bitshift_node.base;
53145501 },
5315 .Shr => {
5502 .AngleBracketAngleBracketRight => {
53165503 const op_token = try appendToken(c, .AngleBracketAngleBracketRight, ">>");
5317 const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
5504 const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
53185505 const bitshift_node = try c.a().create(ast.Node.InfixOp);
53195506 bitshift_node.* = .{
53205507 .op_token = op_token,
......@@ -5326,7 +5513,7 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
53265513 },
53275514 .Pipe => {
53285515 const op_token = try appendToken(c, .Pipe, "|");
5329 const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
5516 const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
53305517 const or_node = try c.a().create(ast.Node.InfixOp);
53315518 or_node.* = .{
53325519 .op_token = op_token,
......@@ -5338,7 +5525,7 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
53385525 },
53395526 .Ampersand => {
53405527 const op_token = try appendToken(c, .Ampersand, "&");
5341 const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
5528 const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
53425529 const bitand_node = try c.a().create(ast.Node.InfixOp);
53435530 bitand_node.* = .{
53445531 .op_token = op_token,
......@@ -5350,7 +5537,7 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
53505537 },
53515538 .Plus => {
53525539 const op_token = try appendToken(c, .Plus, "+");
5353 const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
5540 const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
53545541 const add_node = try c.a().create(ast.Node.InfixOp);
53555542 add_node.* = .{
53565543 .op_token = op_token,
......@@ -5362,7 +5549,7 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
53625549 },
53635550 .Minus => {
53645551 const op_token = try appendToken(c, .Minus, "-");
5365 const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
5552 const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
53665553 const sub_node = try c.a().create(ast.Node.InfixOp);
53675554 sub_node.* = .{
53685555 .op_token = op_token,
......@@ -5372,9 +5559,9 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
53725559 };
53735560 node = &sub_node.base;
53745561 },
5375 .And => {
5562 .AmpersandAmpersand => {
53765563 const op_token = try appendToken(c, .Keyword_and, "and");
5377 const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
5564 const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
53785565 const and_node = try c.a().create(ast.Node.InfixOp);
53795566 and_node.* = .{
53805567 .op_token = op_token,
......@@ -5384,9 +5571,9 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
53845571 };
53855572 node = &and_node.base;
53865573 },
5387 .Or => {
5574 .PipePipe => {
53885575 const op_token = try appendToken(c, .Keyword_or, "or");
5389 const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
5576 const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
53905577 const or_node = try c.a().create(ast.Node.InfixOp);
53915578 or_node.* = .{
53925579 .op_token = op_token,
......@@ -5396,9 +5583,9 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
53965583 };
53975584 node = &or_node.base;
53985585 },
5399 .Gt => {
5586 .AngleBracketRight => {
54005587 const op_token = try appendToken(c, .AngleBracketRight, ">");
5401 const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
5588 const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
54025589 const and_node = try c.a().create(ast.Node.InfixOp);
54035590 and_node.* = .{
54045591 .op_token = op_token,
......@@ -5408,9 +5595,9 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
54085595 };
54095596 node = &and_node.base;
54105597 },
5411 .Gte => {
5598 .AngleBracketRightEqual => {
54125599 const op_token = try appendToken(c, .AngleBracketRightEqual, ">=");
5413 const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
5600 const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
54145601 const and_node = try c.a().create(ast.Node.InfixOp);
54155602 and_node.* = .{
54165603 .op_token = op_token,
......@@ -5420,9 +5607,9 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
54205607 };
54215608 node = &and_node.base;
54225609 },
5423 .Lt => {
5610 .AngleBracketLeft => {
54245611 const op_token = try appendToken(c, .AngleBracketLeft, "<");
5425 const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
5612 const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
54265613 const and_node = try c.a().create(ast.Node.InfixOp);
54275614 and_node.* = .{
54285615 .op_token = op_token,
......@@ -5432,9 +5619,9 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
54325619 };
54335620 node = &and_node.base;
54345621 },
5435 .Lte => {
5622 .AngleBracketLeftEqual => {
54365623 const op_token = try appendToken(c, .AngleBracketLeftEqual, "<=");
5437 const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
5624 const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
54385625 const and_node = try c.a().create(ast.Node.InfixOp);
54395626 and_node.* = .{
54405627 .op_token = op_token,
......@@ -5446,14 +5633,15 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
54465633 },
54475634 .LBrace => {
54485635 const arr_node = try transCreateNodeArrayAccess(c, node);
5449 arr_node.op.ArrayAccess = try parseCPrefixOpExpr(c, it, source_loc, scope);
5636 arr_node.op.ArrayAccess = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
54505637 arr_node.rtoken = try appendToken(c, .RBrace, "]");
54515638 node = &arr_node.base;
54525639 if (it.next().?.id != .RBrace) {
5640 const first_tok = it.list.at(0);
54535641 try failDecl(
54545642 c,
54555643 source_loc,
5456 it.list.at(0).*.bytes,
5644 source[first_tok.start..first_tok.end],
54575645 "unable to translate C expr: expected ']'",
54585646 .{},
54595647 );
......@@ -5463,7 +5651,7 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
54635651 .LParen => {
54645652 const call_node = try transCreateNodeFnCall(c, node);
54655653 while (true) {
5466 const arg = try parseCPrefixOpExpr(c, it, source_loc, scope);
5654 const arg = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
54675655 try call_node.op.Call.params.push(arg);
54685656 const next = it.next().?;
54695657 if (next.id == .Comma)
......@@ -5471,10 +5659,11 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
54715659 else if (next.id == .RParen)
54725660 break
54735661 else {
5662 const first_tok = it.list.at(0);
54745663 try failDecl(
54755664 c,
54765665 source_loc,
5477 it.list.at(0).*.bytes,
5666 source[first_tok.start..first_tok.end],
54785667 "unable to translate C expr: expected ',' or ')'",
54795668 .{},
54805669 );
......@@ -5492,32 +5681,32 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
54925681 }
54935682}
54945683
5495fn parseCPrefixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node {
5684fn parseCPrefixOpExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node {
54965685 const op_tok = it.next().?;
54975686
54985687 switch (op_tok.id) {
54995688 .Bang => {
55005689 const node = try transCreateNodePrefixOp(c, .BoolNot, .Bang, "!");
5501 node.rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
5690 node.rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
55025691 return &node.base;
55035692 },
55045693 .Minus => {
55055694 const node = try transCreateNodePrefixOp(c, .Negation, .Minus, "-");
5506 node.rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
5695 node.rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
55075696 return &node.base;
55085697 },
55095698 .Tilde => {
55105699 const node = try transCreateNodePrefixOp(c, .BitNot, .Tilde, "~");
5511 node.rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
5700 node.rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
55125701 return &node.base;
55135702 },
55145703 .Asterisk => {
5515 const prefix_op_expr = try parseCPrefixOpExpr(c, it, source_loc, scope);
5704 const prefix_op_expr = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
55165705 return try transCreateNodePtrDeref(c, prefix_op_expr);
55175706 },
55185707 else => {
55195708 _ = it.prev();
5520 return try parseCSuffixOpExpr(c, it, source_loc, scope);
5709 return try parseCSuffixOpExpr(c, it, source, source_loc, scope);
55215710 },
55225711 }
55235712}