authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-07-27 15:19:07+03:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-07-27 15:38:56+03:00
loge7007fa7bd94b3290982b1c39933f700e4ee133f
treec054a9bce4e55f76106c98644b18e986d207a36a
parent9f6401c69275f5c80b92c2c4c18c9b73321b82aa
signature Commit is signed but in an unrecognized format.

translate-c: use ArrayList for macro tokens

3 files changed, 302 insertions(+), 323 deletions(-)

lib/std/c.zig+4
...@@ -8,6 +8,10 @@ pub const Tokenizer = tokenizer.Tokenizer;...@@ -8,6 +8,10 @@ pub const Tokenizer = tokenizer.Tokenizer;
8pub const parse = @import("c/parse.zig").parse;8pub const parse = @import("c/parse.zig").parse;
9pub const ast = @import("c/ast.zig");9pub const ast = @import("c/ast.zig");
1010
11test "" {
12 _ = tokenizer;
13}
14
11pub usingnamespace @import("os/bits.zig");15pub usingnamespace @import("os/bits.zig");
1216
13pub usingnamespace switch (std.Target.current.os.tag) {17pub usingnamespace switch (std.Target.current.os.tag) {
lib/std/c/tokenizer.zig+76-115
...@@ -1,19 +1,10 @@...@@ -1,19 +1,10 @@
1const std = @import("std");1const std = @import("std");
2const mem = std.mem;2const mem = std.mem;
33
4pub const Source = struct {
5 buffer: []const u8,
6 file_name: []const u8,
7 tokens: TokenList,
8
9 pub const TokenList = std.SegmentedList(Token, 64);
10};
11
12pub const Token = struct {4pub const Token = struct {
13 id: Id,5 id: Id,
14 start: usize,6 start: usize,
15 end: usize,7 end: usize,
16 source: *Source,
178
18 pub const Id = union(enum) {9 pub const Id = union(enum) {
19 Invalid,10 Invalid,
...@@ -251,31 +242,6 @@ pub const Token = struct {...@@ -251,31 +242,6 @@ pub const Token = struct {
251 }242 }
252 };243 };
253244
254 pub fn eql(a: Token, b: Token) bool {
255 // do we really need this cast here
256 if (@as(@TagType(Id), a.id) != b.id) return false;
257 return mem.eql(u8, a.slice(), b.slice());
258 }
259
260 pub fn slice(tok: Token) []const u8 {
261 return tok.source.buffer[tok.start..tok.end];
262 }
263
264 pub const Keyword = struct {
265 bytes: []const u8,
266 id: Id,
267 hash: u32,
268
269 fn init(bytes: []const u8, id: Id) Keyword {
270 @setEvalBranchQuota(2000);
271 return .{
272 .bytes = bytes,
273 .id = id,
274 .hash = std.hash_map.hashString(bytes),
275 };
276 }
277 };
278
279 // TODO extensions245 // TODO extensions
280 pub const keywords = std.ComptimeStringMap(Id, .{246 pub const keywords = std.ComptimeStringMap(Id, .{
281 .{ "auto", .Keyword_auto },247 .{ "auto", .Keyword_auto },
...@@ -355,26 +321,26 @@ pub const Token = struct {...@@ -355,26 +321,26 @@ pub const Token = struct {
355 }321 }
356322
357 pub const NumSuffix = enum {323 pub const NumSuffix = enum {
358 None,324 none,
359 F,325 f,
360 L,326 l,
361 U,327 u,
362 LU,328 lu,
363 LL,329 ll,
364 LLU,330 llu,
365 };331 };
366332
367 pub const StrKind = enum {333 pub const StrKind = enum {
368 None,334 none,
369 Wide,335 wide,
370 Utf8,336 utf_8,
371 Utf16,337 utf_16,
372 Utf32,338 utf_32,
373 };339 };
374};340};
375341
376pub const Tokenizer = struct {342pub const Tokenizer = struct {
377 source: *Source,343 buffer: []const u8,
378 index: usize = 0,344 index: usize = 0,
379 prev_tok_id: @TagType(Token.Id) = .Invalid,345 prev_tok_id: @TagType(Token.Id) = .Invalid,
380 pp_directive: bool = false,346 pp_directive: bool = false,
...@@ -385,7 +351,6 @@ pub const Tokenizer = struct {...@@ -385,7 +351,6 @@ pub const Tokenizer = struct {
385 .id = .Eof,351 .id = .Eof,
386 .start = self.index,352 .start = self.index,
387 .end = undefined,353 .end = undefined,
388 .source = self.source,
389 };354 };
390 var state: enum {355 var state: enum {
391 Start,356 Start,
...@@ -446,8 +411,8 @@ pub const Tokenizer = struct {...@@ -446,8 +411,8 @@ pub const Tokenizer = struct {
446 } = .Start;411 } = .Start;
447 var string = false;412 var string = false;
448 var counter: u32 = 0;413 var counter: u32 = 0;
449 while (self.index < self.source.buffer.len) : (self.index += 1) {414 while (self.index < self.buffer.len) : (self.index += 1) {
450 const c = self.source.buffer[self.index];415 const c = self.buffer[self.index];
451 switch (state) {416 switch (state) {
452 .Start => switch (c) {417 .Start => switch (c) {
453 '\n' => {418 '\n' => {
...@@ -460,11 +425,11 @@ pub const Tokenizer = struct {...@@ -460,11 +425,11 @@ pub const Tokenizer = struct {
460 state = .Cr;425 state = .Cr;
461 },426 },
462 '"' => {427 '"' => {
463 result.id = .{ .StringLiteral = .None };428 result.id = .{ .StringLiteral = .none };
464 state = .StringLiteral;429 state = .StringLiteral;
465 },430 },
466 '\'' => {431 '\'' => {
467 result.id = .{ .CharLiteral = .None };432 result.id = .{ .CharLiteral = .none };
468 state = .CharLiteralStart;433 state = .CharLiteralStart;
469 },434 },
470 'u' => {435 'u' => {
...@@ -641,11 +606,11 @@ pub const Tokenizer = struct {...@@ -641,11 +606,11 @@ pub const Tokenizer = struct {
641 state = .u8;606 state = .u8;
642 },607 },
643 '\'' => {608 '\'' => {
644 result.id = .{ .CharLiteral = .Utf16 };609 result.id = .{ .CharLiteral = .utf_16 };
645 state = .CharLiteralStart;610 state = .CharLiteralStart;
646 },611 },
647 '\"' => {612 '\"' => {
648 result.id = .{ .StringLiteral = .Utf16 };613 result.id = .{ .StringLiteral = .utf_16 };
649 state = .StringLiteral;614 state = .StringLiteral;
650 },615 },
651 else => {616 else => {
...@@ -655,7 +620,7 @@ pub const Tokenizer = struct {...@@ -655,7 +620,7 @@ pub const Tokenizer = struct {
655 },620 },
656 .u8 => switch (c) {621 .u8 => switch (c) {
657 '\"' => {622 '\"' => {
658 result.id = .{ .StringLiteral = .Utf8 };623 result.id = .{ .StringLiteral = .utf_8 };
659 state = .StringLiteral;624 state = .StringLiteral;
660 },625 },
661 else => {626 else => {
...@@ -665,11 +630,11 @@ pub const Tokenizer = struct {...@@ -665,11 +630,11 @@ pub const Tokenizer = struct {
665 },630 },
666 .U => switch (c) {631 .U => switch (c) {
667 '\'' => {632 '\'' => {
668 result.id = .{ .CharLiteral = .Utf32 };633 result.id = .{ .CharLiteral = .utf_32 };
669 state = .CharLiteralStart;634 state = .CharLiteralStart;
670 },635 },
671 '\"' => {636 '\"' => {
672 result.id = .{ .StringLiteral = .Utf32 };637 result.id = .{ .StringLiteral = .utf_32 };
673 state = .StringLiteral;638 state = .StringLiteral;
674 },639 },
675 else => {640 else => {
...@@ -679,11 +644,11 @@ pub const Tokenizer = struct {...@@ -679,11 +644,11 @@ pub const Tokenizer = struct {
679 },644 },
680 .L => switch (c) {645 .L => switch (c) {
681 '\'' => {646 '\'' => {
682 result.id = .{ .CharLiteral = .Wide };647 result.id = .{ .CharLiteral = .wide };
683 state = .CharLiteralStart;648 state = .CharLiteralStart;
684 },649 },
685 '\"' => {650 '\"' => {
686 result.id = .{ .StringLiteral = .Wide };651 result.id = .{ .StringLiteral = .wide };
687 state = .StringLiteral;652 state = .StringLiteral;
688 },653 },
689 else => {654 else => {
...@@ -808,7 +773,7 @@ pub const Tokenizer = struct {...@@ -808,7 +773,7 @@ pub const Tokenizer = struct {
808 .Identifier => switch (c) {773 .Identifier => switch (c) {
809 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},774 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
810 else => {775 else => {
811 result.id = Token.getKeyword(self.source.buffer[result.start..self.index], self.prev_tok_id == .Hash and !self.pp_directive) orelse .Identifier;776 result.id = Token.getKeyword(self.buffer[result.start..self.index], self.prev_tok_id == .Hash and !self.pp_directive) orelse .Identifier;
812 if (self.prev_tok_id == .Hash)777 if (self.prev_tok_id == .Hash)
813 self.pp_directive = true;778 self.pp_directive = true;
814 break;779 break;
...@@ -1137,7 +1102,7 @@ pub const Tokenizer = struct {...@@ -1137,7 +1102,7 @@ pub const Tokenizer = struct {
1137 state = .IntegerSuffixL;1102 state = .IntegerSuffixL;
1138 },1103 },
1139 else => {1104 else => {
1140 result.id = .{ .IntegerLiteral = .None };1105 result.id = .{ .IntegerLiteral = .none };
1141 break;1106 break;
1142 },1107 },
1143 },1108 },
...@@ -1146,7 +1111,7 @@ pub const Tokenizer = struct {...@@ -1146,7 +1111,7 @@ pub const Tokenizer = struct {
1146 state = .IntegerSuffixUL;1111 state = .IntegerSuffixUL;
1147 },1112 },
1148 else => {1113 else => {
1149 result.id = .{ .IntegerLiteral = .U };1114 result.id = .{ .IntegerLiteral = .u };
1150 break;1115 break;
1151 },1116 },
1152 },1117 },
...@@ -1155,34 +1120,34 @@ pub const Tokenizer = struct {...@@ -1155,34 +1120,34 @@ pub const Tokenizer = struct {
1155 state = .IntegerSuffixLL;1120 state = .IntegerSuffixLL;
1156 },1121 },
1157 'u', 'U' => {1122 'u', 'U' => {
1158 result.id = .{ .IntegerLiteral = .LU };1123 result.id = .{ .IntegerLiteral = .lu };
1159 self.index += 1;1124 self.index += 1;
1160 break;1125 break;
1161 },1126 },
1162 else => {1127 else => {
1163 result.id = .{ .IntegerLiteral = .L };1128 result.id = .{ .IntegerLiteral = .l };
1164 break;1129 break;
1165 },1130 },
1166 },1131 },
1167 .IntegerSuffixLL => switch (c) {1132 .IntegerSuffixLL => switch (c) {
1168 'u', 'U' => {1133 'u', 'U' => {
1169 result.id = .{ .IntegerLiteral = .LLU };1134 result.id = .{ .IntegerLiteral = .llu };
1170 self.index += 1;1135 self.index += 1;
1171 break;1136 break;
1172 },1137 },
1173 else => {1138 else => {
1174 result.id = .{ .IntegerLiteral = .LL };1139 result.id = .{ .IntegerLiteral = .ll };
1175 break;1140 break;
1176 },1141 },
1177 },1142 },
1178 .IntegerSuffixUL => switch (c) {1143 .IntegerSuffixUL => switch (c) {
1179 'l', 'L' => {1144 'l', 'L' => {
1180 result.id = .{ .IntegerLiteral = .LLU };1145 result.id = .{ .IntegerLiteral = .llu };
1181 self.index += 1;1146 self.index += 1;
1182 break;1147 break;
1183 },1148 },
1184 else => {1149 else => {
1185 result.id = .{ .IntegerLiteral = .LU };1150 result.id = .{ .IntegerLiteral = .lu };
1186 break;1151 break;
1187 },1152 },
1188 },1153 },
...@@ -1230,26 +1195,26 @@ pub const Tokenizer = struct {...@@ -1230,26 +1195,26 @@ pub const Tokenizer = struct {
1230 },1195 },
1231 .FloatSuffix => switch (c) {1196 .FloatSuffix => switch (c) {
1232 'l', 'L' => {1197 'l', 'L' => {
1233 result.id = .{ .FloatLiteral = .L };1198 result.id = .{ .FloatLiteral = .l };
1234 self.index += 1;1199 self.index += 1;
1235 break;1200 break;
1236 },1201 },
1237 'f', 'F' => {1202 'f', 'F' => {
1238 result.id = .{ .FloatLiteral = .F };1203 result.id = .{ .FloatLiteral = .f };
1239 self.index += 1;1204 self.index += 1;
1240 break;1205 break;
1241 },1206 },
1242 else => {1207 else => {
1243 result.id = .{ .FloatLiteral = .None };1208 result.id = .{ .FloatLiteral = .none };
1244 break;1209 break;
1245 },1210 },
1246 },1211 },
1247 }1212 }
1248 } else if (self.index == self.source.buffer.len) {1213 } else if (self.index == self.buffer.len) {
1249 switch (state) {1214 switch (state) {
1250 .Start => {},1215 .Start => {},
1251 .u, .u8, .U, .L, .Identifier => {1216 .u, .u8, .U, .L, .Identifier => {
1252 result.id = Token.getKeyword(self.source.buffer[result.start..self.index], self.prev_tok_id == .Hash and !self.pp_directive) orelse .Identifier;1217 result.id = Token.getKeyword(self.buffer[result.start..self.index], self.prev_tok_id == .Hash and !self.pp_directive) orelse .Identifier;
1253 },1218 },
12541219
1255 .Cr,1220 .Cr,
...@@ -1270,11 +1235,11 @@ pub const Tokenizer = struct {...@@ -1270,11 +1235,11 @@ pub const Tokenizer = struct {
1270 .MacroString,1235 .MacroString,
1271 => result.id = .Invalid,1236 => result.id = .Invalid,
12721237
1273 .FloatExponentDigits => result.id = if (counter == 0) .Invalid else .{ .FloatLiteral = .None },1238 .FloatExponentDigits => result.id = if (counter == 0) .Invalid else .{ .FloatLiteral = .none },
12741239
1275 .FloatFraction,1240 .FloatFraction,
1276 .FloatFractionHex,1241 .FloatFractionHex,
1277 => result.id = .{ .FloatLiteral = .None },1242 => result.id = .{ .FloatLiteral = .none },
12781243
1279 .IntegerLiteralOct,1244 .IntegerLiteralOct,
1280 .IntegerLiteralBinary,1245 .IntegerLiteralBinary,
...@@ -1282,13 +1247,13 @@ pub const Tokenizer = struct {...@@ -1282,13 +1247,13 @@ pub const Tokenizer = struct {
1282 .IntegerLiteral,1247 .IntegerLiteral,
1283 .IntegerSuffix,1248 .IntegerSuffix,
1284 .Zero,1249 .Zero,
1285 => result.id = .{ .IntegerLiteral = .None },1250 => result.id = .{ .IntegerLiteral = .none },
1286 .IntegerSuffixU => result.id = .{ .IntegerLiteral = .U },1251 .IntegerSuffixU => result.id = .{ .IntegerLiteral = .u },
1287 .IntegerSuffixL => result.id = .{ .IntegerLiteral = .L },1252 .IntegerSuffixL => result.id = .{ .IntegerLiteral = .l },
1288 .IntegerSuffixLL => result.id = .{ .IntegerLiteral = .LL },1253 .IntegerSuffixLL => result.id = .{ .IntegerLiteral = .ll },
1289 .IntegerSuffixUL => result.id = .{ .IntegerLiteral = .LU },1254 .IntegerSuffixUL => result.id = .{ .IntegerLiteral = .lu },
12901255
1291 .FloatSuffix => result.id = .{ .FloatLiteral = .None },1256 .FloatSuffix => result.id = .{ .FloatLiteral = .none },
1292 .Equal => result.id = .Equal,1257 .Equal => result.id = .Equal,
1293 .Bang => result.id = .Bang,1258 .Bang => result.id = .Bang,
1294 .Minus => result.id = .Minus,1259 .Minus => result.id = .Minus,
...@@ -1466,7 +1431,7 @@ test "preprocessor keywords" {...@@ -1466,7 +1431,7 @@ test "preprocessor keywords" {
1466 .Hash,1431 .Hash,
1467 .Identifier,1432 .Identifier,
1468 .AngleBracketLeft,1433 .AngleBracketLeft,
1469 .{ .IntegerLiteral = .None },1434 .{ .IntegerLiteral = .none },
1470 .Nl,1435 .Nl,
1471 .Hash,1436 .Hash,
1472 .Keyword_ifdef,1437 .Keyword_ifdef,
...@@ -1499,18 +1464,18 @@ test "line continuation" {...@@ -1499,18 +1464,18 @@ test "line continuation" {
1499 .Identifier,1464 .Identifier,
1500 .Identifier,1465 .Identifier,
1501 .Nl,1466 .Nl,
1502 .{ .StringLiteral = .None },1467 .{ .StringLiteral = .none },
1503 .Nl,1468 .Nl,
1504 .Hash,1469 .Hash,
1505 .Keyword_define,1470 .Keyword_define,
1506 .{ .StringLiteral = .None },1471 .{ .StringLiteral = .none },
1507 .Nl,1472 .Nl,
1508 .{ .StringLiteral = .None },1473 .{ .StringLiteral = .none },
1509 .Nl,1474 .Nl,
1510 .Hash,1475 .Hash,
1511 .Keyword_define,1476 .Keyword_define,
1512 .{ .StringLiteral = .None },1477 .{ .StringLiteral = .none },
1513 .{ .StringLiteral = .None },1478 .{ .StringLiteral = .none },
1514 });1479 });
1515}1480}
15161481
...@@ -1527,23 +1492,23 @@ test "string prefix" {...@@ -1527,23 +1492,23 @@ test "string prefix" {
1527 \\L'foo'1492 \\L'foo'
1528 \\1493 \\
1529 , &[_]Token.Id{1494 , &[_]Token.Id{
1530 .{ .StringLiteral = .None },1495 .{ .StringLiteral = .none },
1531 .Nl,1496 .Nl,
1532 .{ .StringLiteral = .Utf16 },1497 .{ .StringLiteral = .utf_16 },
1533 .Nl,1498 .Nl,
1534 .{ .StringLiteral = .Utf8 },1499 .{ .StringLiteral = .utf_8 },
1535 .Nl,1500 .Nl,
1536 .{ .StringLiteral = .Utf32 },1501 .{ .StringLiteral = .utf_32 },
1537 .Nl,1502 .Nl,
1538 .{ .StringLiteral = .Wide },1503 .{ .StringLiteral = .wide },
1539 .Nl,1504 .Nl,
1540 .{ .CharLiteral = .None },1505 .{ .CharLiteral = .none },
1541 .Nl,1506 .Nl,
1542 .{ .CharLiteral = .Utf16 },1507 .{ .CharLiteral = .utf_16 },
1543 .Nl,1508 .Nl,
1544 .{ .CharLiteral = .Utf32 },1509 .{ .CharLiteral = .utf_32 },
1545 .Nl,1510 .Nl,
1546 .{ .CharLiteral = .Wide },1511 .{ .CharLiteral = .wide },
1547 .Nl,1512 .Nl,
1548 });1513 });
1549}1514}
...@@ -1555,33 +1520,29 @@ test "num suffixes" {...@@ -1555,33 +1520,29 @@ test "num suffixes" {
1555 \\ 1u 1ul 1ull 11520 \\ 1u 1ul 1ull 1
1556 \\1521 \\
1557 , &[_]Token.Id{1522 , &[_]Token.Id{
1558 .{ .FloatLiteral = .F },1523 .{ .FloatLiteral = .f },
1559 .{ .FloatLiteral = .L },1524 .{ .FloatLiteral = .l },
1560 .{ .FloatLiteral = .None },1525 .{ .FloatLiteral = .none },
1561 .{ .FloatLiteral = .None },1526 .{ .FloatLiteral = .none },
1562 .{ .FloatLiteral = .None },1527 .{ .FloatLiteral = .none },
1563 .Nl,1528 .Nl,
1564 .{ .IntegerLiteral = .L },1529 .{ .IntegerLiteral = .l },
1565 .{ .IntegerLiteral = .LU },1530 .{ .IntegerLiteral = .lu },
1566 .{ .IntegerLiteral = .LL },1531 .{ .IntegerLiteral = .ll },
1567 .{ .IntegerLiteral = .LLU },1532 .{ .IntegerLiteral = .llu },
1568 .{ .IntegerLiteral = .None },1533 .{ .IntegerLiteral = .none },
1569 .Nl,1534 .Nl,
1570 .{ .IntegerLiteral = .U },1535 .{ .IntegerLiteral = .u },
1571 .{ .IntegerLiteral = .LU },1536 .{ .IntegerLiteral = .lu },
1572 .{ .IntegerLiteral = .LLU },1537 .{ .IntegerLiteral = .llu },
1573 .{ .IntegerLiteral = .None },1538 .{ .IntegerLiteral = .none },
1574 .Nl,1539 .Nl,
1575 });1540 });
1576}1541}
15771542
1578fn expectTokens(source: []const u8, expected_tokens: []const Token.Id) void {1543fn expectTokens(source: []const u8, expected_tokens: []const Token.Id) void {
1579 var tokenizer = Tokenizer{1544 var tokenizer = Tokenizer{
1580 .source = &Source{1545 .buffer = source,
1581 .buffer = source,
1582 .file_name = undefined,
1583 .tokens = undefined,
1584 },
1585 };1546 };
1586 for (expected_tokens) |expected_token_id| {1547 for (expected_tokens) |expected_token_id| {
1587 const token = tokenizer.next();1548 const token = tokenizer.next();
src-self-hosted/translate_c.zig+222-208
...@@ -8,7 +8,6 @@ const Token = std.zig.Token;...@@ -8,7 +8,6 @@ const Token = std.zig.Token;
8usingnamespace @import("clang.zig");8usingnamespace @import("clang.zig");
9const ctok = std.c.tokenizer;9const ctok = std.c.tokenizer;
10const CToken = std.c.Token;10const CToken = std.c.Token;
11const CTokenList = std.c.tokenizer.Source.TokenList;
12const mem = std.mem;11const mem = std.mem;
13const math = std.math;12const math = std.math;
1413
...@@ -5196,16 +5195,39 @@ pub fn freeErrors(errors: []ClangErrMsg) void {...@@ -5196,16 +5195,39 @@ pub fn freeErrors(errors: []ClangErrMsg) void {
5196 ZigClangErrorMsg_delete(errors.ptr, errors.len);5195 ZigClangErrorMsg_delete(errors.ptr, errors.len);
5197}5196}
51985197
5198const CTokIterator = struct {
5199 source: []const u8,
5200 list: []const CToken,
5201 i: usize = 0,
5202
5203 fn peek(self: *CTokIterator) ?CToken.Id {
5204 if (self.i >= self.list.len) return null;
5205 return self.list[self.i + 1].id;
5206 }
5207
5208 fn next(self: *CTokIterator) ?CToken.Id {
5209 if (self.i >= self.list.len) return null;
5210 self.i += 1;
5211 return self.list[self.i].id;
5212 }
5213
5214 fn slice(self: *CTokIterator, index: usize) []const u8 {
5215 const tok = self.list[index];
5216 return self.source[tok.start..tok.end];
5217 }
5218};
5219
5199fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void {5220fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void {
5200 // TODO if we see #undef, delete it from the table5221 // TODO if we see #undef, delete it from the table
5201 var it = ZigClangASTUnit_getLocalPreprocessingEntities_begin(unit);5222 var it = ZigClangASTUnit_getLocalPreprocessingEntities_begin(unit);
5202 const it_end = ZigClangASTUnit_getLocalPreprocessingEntities_end(unit);5223 const it_end = ZigClangASTUnit_getLocalPreprocessingEntities_end(unit);
5203 var tok_list = CTokenList.init(c.arena);5224 var tok_list = std.ArrayList(CToken).init(c.gpa);
5225 defer tok_list.deinit();
5204 const scope = c.global_scope;5226 const scope = c.global_scope;
52055227
5206 while (it.I != it_end.I) : (it.I += 1) {5228 while (it.I != it_end.I) : (it.I += 1) {
5207 const entity = ZigClangPreprocessingRecord_iterator_deref(it);5229 const entity = ZigClangPreprocessingRecord_iterator_deref(it);
5208 tok_list.shrink(0);5230 tok_list.items.len = 0;
5209 switch (ZigClangPreprocessedEntity_getKind(entity)) {5231 switch (ZigClangPreprocessedEntity_getKind(entity)) {
5210 .MacroDefinitionKind => {5232 .MacroDefinitionKind => {
5211 const macro = @ptrCast(*ZigClangMacroDefinitionRecord, entity);5233 const macro = @ptrCast(*ZigClangMacroDefinitionRecord, entity);
...@@ -5223,38 +5245,34 @@ fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void {...@@ -5223,38 +5245,34 @@ fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void {
5223 const begin_c = ZigClangSourceManager_getCharacterData(c.source_manager, begin_loc);5245 const begin_c = ZigClangSourceManager_getCharacterData(c.source_manager, begin_loc);
5224 const slice = begin_c[0..mem.len(begin_c)];5246 const slice = begin_c[0..mem.len(begin_c)];
52255247
5226 tok_list.shrink(0);
5227 var tokenizer = std.c.Tokenizer{5248 var tokenizer = std.c.Tokenizer{
5228 .source = &std.c.tokenizer.Source{5249 .buffer = slice,
5229 .buffer = slice,
5230 .file_name = undefined,
5231 .tokens = undefined,
5232 },
5233 };5250 };
5234 while (true) {5251 while (true) {
5235 const tok = tokenizer.next();5252 const tok = tokenizer.next();
5236 switch (tok.id) {5253 switch (tok.id) {
5237 .Nl, .Eof => {5254 .Nl, .Eof => {
5238 try tok_list.push(tok);5255 try tok_list.append(tok);
5239 break;5256 break;
5240 },5257 },
5241 .LineComment, .MultiLineComment => continue,5258 .LineComment, .MultiLineComment => continue,
5242 else => {},5259 else => {},
5243 }5260 }
5244 try tok_list.push(tok);5261 try tok_list.append(tok);
5245 }5262 }
52465263
5247 var tok_it = tok_list.iterator(0);5264 var tok_it = CTokIterator{
5248 const first_tok = tok_it.next().?;5265 .source = slice,
5249 assert(mem.eql(u8, slice[first_tok.start..first_tok.end], name));5266 .list = tok_list.items,
5267 };
5268 assert(mem.eql(u8, tok_it.slice(0), name));
52505269
5251 var macro_fn = false;5270 var macro_fn = false;
5252 const next = tok_it.peek().?;5271 switch (tok_it.peek().?) {
5253 switch (next.id) {
5254 .Identifier => {5272 .Identifier => {
5255 // if it equals itself, ignore. for example, from stdio.h:5273 // if it equals itself, ignore. for example, from stdio.h:
5256 // #define stdin stdin5274 // #define stdin stdin
5257 if (mem.eql(u8, name, slice[next.start..next.end])) {5275 if (mem.eql(u8, name, tok_it.slice(1))) {
5258 continue;5276 continue;
5259 }5277 }
5260 },5278 },
...@@ -5265,15 +5283,15 @@ fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void {...@@ -5265,15 +5283,15 @@ fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void {
5265 },5283 },
5266 .LParen => {5284 .LParen => {
5267 // if the name is immediately followed by a '(' then it is a function5285 // if the name is immediately followed by a '(' then it is a function
5268 macro_fn = first_tok.end == next.start;5286 macro_fn = tok_it.list[0].end == tok_it.list[1].start;
5269 },5287 },
5270 else => {},5288 else => {},
5271 }5289 }
52725290
5273 (if (macro_fn)5291 (if (macro_fn)
5274 transMacroFnDefine(c, &tok_it, slice, mangled_name, begin_loc)5292 transMacroFnDefine(c, &tok_it, mangled_name, begin_loc)
5275 else5293 else
5276 transMacroDefine(c, &tok_it, slice, mangled_name, begin_loc)) catch |err| switch (err) {5294 transMacroDefine(c, &tok_it, mangled_name, begin_loc)) catch |err| switch (err) {
5277 error.ParseError => continue,5295 error.ParseError => continue,
5278 error.OutOfMemory => |e| return e,5296 error.OutOfMemory => |e| return e,
5279 };5297 };
...@@ -5283,7 +5301,7 @@ fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void {...@@ -5283,7 +5301,7 @@ fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void {
5283 }5301 }
5284}5302}
52855303
5286fn transMacroDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, name: []const u8, source_loc: ZigClangSourceLocation) ParseError!void {5304fn transMacroDefine(c: *Context, it: *CTokIterator, name: []const u8, source_loc: ZigClangSourceLocation) ParseError!void {
5287 const scope = &c.global_scope.base;5305 const scope = &c.global_scope.base;
52885306
5289 const visib_tok = try appendToken(c, .Keyword_pub, "pub");5307 const visib_tok = try appendToken(c, .Keyword_pub, "pub");
...@@ -5291,15 +5309,15 @@ fn transMacroDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, n...@@ -5291,15 +5309,15 @@ fn transMacroDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, n
5291 const name_tok = try appendIdentifier(c, name);5309 const name_tok = try appendIdentifier(c, name);
5292 const eq_token = try appendToken(c, .Equal, "=");5310 const eq_token = try appendToken(c, .Equal, "=");
52935311
5294 const init_node = try parseCExpr(c, it, source, source_loc, scope);5312 const init_node = try parseCExpr(c, it, source_loc, scope);
5295 const last = it.next().?;5313 const last = it.next().?;
5296 if (last.id != .Eof and last.id != .Nl)5314 if (last != .Eof and last != .Nl)
5297 return failDecl(5315 return failDecl(
5298 c,5316 c,
5299 source_loc,5317 source_loc,
5300 name,5318 name,
5301 "unable to translate C expr: unexpected token .{}",5319 "unable to translate C expr: unexpected token .{}",
5302 .{@tagName(last.id)},5320 .{@tagName(last)},
5303 );5321 );
53045322
5305 const semicolon_token = try appendToken(c, .Semicolon, ";");5323 const semicolon_token = try appendToken(c, .Semicolon, ";");
...@@ -5315,7 +5333,7 @@ fn transMacroDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, n...@@ -5315,7 +5333,7 @@ fn transMacroDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, n
5315 _ = try c.global_scope.macro_table.put(name, &node.base);5333 _ = try c.global_scope.macro_table.put(name, &node.base);
5316}5334}
53175335
5318fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, name: []const u8, source_loc: ZigClangSourceLocation) ParseError!void {5336fn transMacroFnDefine(c: *Context, it: *CTokIterator, name: []const u8, source_loc: ZigClangSourceLocation) ParseError!void {
5319 var block_scope = try Scope.Block.init(c, &c.global_scope.base, null);5337 var block_scope = try Scope.Block.init(c, &c.global_scope.base, null);
5320 defer block_scope.deinit();5338 defer block_scope.deinit();
5321 const scope = &block_scope.base;5339 const scope = &block_scope.base;
...@@ -5326,7 +5344,7 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8,...@@ -5326,7 +5344,7 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8,
5326 const name_tok = try appendIdentifier(c, name);5344 const name_tok = try appendIdentifier(c, name);
5327 _ = try appendToken(c, .LParen, "(");5345 _ = try appendToken(c, .LParen, "(");
53285346
5329 if (it.next().?.id != .LParen) {5347 if (it.next().? != .LParen) {
5330 return failDecl(5348 return failDecl(
5331 c,5349 c,
5332 source_loc,5350 source_loc,
...@@ -5340,8 +5358,7 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8,...@@ -5340,8 +5358,7 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8,
5340 defer fn_params.deinit();5358 defer fn_params.deinit();
53415359
5342 while (true) {5360 while (true) {
5343 const param_tok = it.next().?;5361 if (it.next().? != .Identifier) {
5344 if (param_tok.id != .Identifier) {
5345 return failDecl(5362 return failDecl(
5346 c,5363 c,
5347 source_loc,5364 source_loc,
...@@ -5351,7 +5368,7 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8,...@@ -5351,7 +5368,7 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8,
5351 );5368 );
5352 }5369 }
53535370
5354 const mangled_name = try block_scope.makeMangledName(c, source[param_tok.start..param_tok.end]);5371 const mangled_name = try block_scope.makeMangledName(c, it.slice(it.i));
5355 const param_name_tok = try appendIdentifier(c, mangled_name);5372 const param_name_tok = try appendIdentifier(c, mangled_name);
5356 _ = try appendToken(c, .Colon, ":");5373 _ = try appendToken(c, .Colon, ":");
53575374
...@@ -5369,13 +5386,13 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8,...@@ -5369,13 +5386,13 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8,
5369 .param_type = .{ .any_type = &any_type.base },5386 .param_type = .{ .any_type = &any_type.base },
5370 };5387 };
53715388
5372 if (it.peek().?.id != .Comma)5389 if (it.peek().? != .Comma)
5373 break;5390 break;
5374 _ = it.next();5391 _ = it.next();
5375 _ = try appendToken(c, .Comma, ",");5392 _ = try appendToken(c, .Comma, ",");
5376 }5393 }
53775394
5378 if (it.next().?.id != .RParen) {5395 if (it.next().? != .RParen) {
5379 return failDecl(5396 return failDecl(
5380 c,5397 c,
5381 source_loc,5398 source_loc,
...@@ -5390,15 +5407,15 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8,...@@ -5390,15 +5407,15 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8,
5390 const type_of = try c.createBuiltinCall("@TypeOf", 1);5407 const type_of = try c.createBuiltinCall("@TypeOf", 1);
53915408
5392 const return_kw = try appendToken(c, .Keyword_return, "return");5409 const return_kw = try appendToken(c, .Keyword_return, "return");
5393 const expr = try parseCExpr(c, it, source, source_loc, scope);5410 const expr = try parseCExpr(c, it, source_loc, scope);
5394 const last = it.next().?;5411 const last = it.next().?;
5395 if (last.id != .Eof and last.id != .Nl)5412 if (last != .Eof and last != .Nl)
5396 return failDecl(5413 return failDecl(
5397 c,5414 c,
5398 source_loc,5415 source_loc,
5399 name,5416 name,
5400 "unable to translate C expr: unexpected token .{}",5417 "unable to translate C expr: unexpected token .{}",
5401 .{@tagName(last.id)},5418 .{@tagName(last)},
5402 );5419 );
5403 _ = try appendToken(c, .Semicolon, ";");5420 _ = try appendToken(c, .Semicolon, ";");
5404 const type_of_arg = if (expr.tag != .Block) expr else blk: {5421 const type_of_arg = if (expr.tag != .Block) expr else blk: {
...@@ -5435,28 +5452,27 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8,...@@ -5435,28 +5452,27 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8,
54355452
5436const ParseError = Error || error{ParseError};5453const ParseError = Error || error{ParseError};
54375454
5438fn parseCExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node {5455fn parseCExpr(c: *Context, it: *CTokIterator, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node {
5439 const node = try parseCPrefixOpExpr(c, it, source, source_loc, scope);5456 const node = try parseCPrefixOpExpr(c, it, source_loc, scope);
5440 switch (it.next().?.id) {5457 switch (it.next().?) {
5441 .QuestionMark => {5458 .QuestionMark => {
5442 // must come immediately after expr5459 // must come immediately after expr
5443 _ = try appendToken(c, .RParen, ")");5460 _ = try appendToken(c, .RParen, ")");
5444 const if_node = try transCreateNodeIf(c);5461 const if_node = try transCreateNodeIf(c);
5445 if_node.condition = node;5462 if_node.condition = node;
5446 if_node.body = try parseCPrimaryExpr(c, it, source, source_loc, scope);5463 if_node.body = try parseCPrimaryExpr(c, it, source_loc, scope);
5447 if (it.next().?.id != .Colon) {5464 if (it.next().? != .Colon) {
5448 const first_tok = it.list.at(0);
5449 try failDecl(5465 try failDecl(
5450 c,5466 c,
5451 source_loc,5467 source_loc,
5452 source[first_tok.start..first_tok.end],5468 it.slice(0),
5453 "unable to translate C expr: expected ':'",5469 "unable to translate C expr: expected ':'",
5454 .{},5470 .{},
5455 );5471 );
5456 return error.ParseError;5472 return error.ParseError;
5457 }5473 }
5458 if_node.@"else" = try transCreateNodeElse(c);5474 if_node.@"else" = try transCreateNodeElse(c);
5459 if_node.@"else".?.body = try parseCPrimaryExpr(c, it, source, source_loc, scope);5475 if_node.@"else".?.body = try parseCPrimaryExpr(c, it, source_loc, scope);
5460 return &if_node.base;5476 return &if_node.base;
5461 },5477 },
5462 .Comma => {5478 .Comma => {
...@@ -5479,10 +5495,10 @@ fn parseCExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, source_...@@ -5479,10 +5495,10 @@ fn parseCExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, source_
5479 };5495 };
5480 try block_scope.statements.append(&op_node.base);5496 try block_scope.statements.append(&op_node.base);
54815497
5482 last = try parseCPrefixOpExpr(c, it, source, source_loc, scope);5498 last = try parseCPrefixOpExpr(c, it, source_loc, scope);
5483 _ = try appendToken(c, .Semicolon, ";");5499 _ = try appendToken(c, .Semicolon, ";");
5484 if (it.next().?.id != .Comma) {5500 if (it.next().? != .Comma) {
5485 _ = it.prev();5501 it.i -= 1;
5486 break;5502 break;
5487 }5503 }
5488 }5504 }
...@@ -5493,70 +5509,74 @@ fn parseCExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, source_...@@ -5493,70 +5509,74 @@ fn parseCExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, source_
5493 return &block_node.base;5509 return &block_node.base;
5494 },5510 },
5495 else => {5511 else => {
5496 _ = it.prev();5512 it.i -= 1;
5497 return node;5513 return node;
5498 },5514 },
5499 }5515 }
5500}5516}
55015517
5502fn parseCNumLit(c: *Context, tok: *CToken, source: []const u8, source_loc: ZigClangSourceLocation) ParseError!*ast.Node {5518fn parseCNumLit(c: *Context, it: *CTokIterator, source_loc: ZigClangSourceLocation) ParseError!*ast.Node {
5503 var lit_bytes = source[tok.start..tok.end];5519 var lit_bytes = it.slice(it.i);
55045520
5505 if (tok.id == .IntegerLiteral) {5521 switch (it.list[it.i].id) {
5506 if (lit_bytes.len > 2 and lit_bytes[0] == '0') {5522 .IntegerLiteral => |suffix| {
5507 switch (lit_bytes[1]) {5523 if (lit_bytes.len > 2 and lit_bytes[0] == '0') {
5508 '0'...'7' => {5524 switch (lit_bytes[1]) {
5509 // Octal5525 '0'...'7' => {
5510 lit_bytes = try std.fmt.allocPrint(c.arena, "0o{}", .{lit_bytes});5526 // Octal
5511 },5527 lit_bytes = try std.fmt.allocPrint(c.arena, "0o{}", .{lit_bytes});
5512 'X' => {5528 },
5513 // Hexadecimal with capital X, valid in C but not in Zig5529 'X' => {
5514 lit_bytes = try std.fmt.allocPrint(c.arena, "0x{}", .{lit_bytes[2..]});5530 // Hexadecimal with capital X, valid in C but not in Zig
5515 },5531 lit_bytes = try std.fmt.allocPrint(c.arena, "0x{}", .{lit_bytes[2..]});
5516 else => {},5532 },
5533 else => {},
5534 }
5517 }5535 }
5518 }
55195536
5520 if (tok.id.IntegerLiteral == .None) {5537 if (suffix == .none) {
5521 return transCreateNodeInt(c, lit_bytes);5538 return transCreateNodeInt(c, lit_bytes);
5522 }5539 }
55235540
5524 const cast_node = try c.createBuiltinCall("@as", 2);5541 const cast_node = try c.createBuiltinCall("@as", 2);
5525 cast_node.params()[0] = try transCreateNodeIdentifier(c, switch (tok.id.IntegerLiteral) {5542 cast_node.params()[0] = try transCreateNodeIdentifier(c, switch (suffix) {
5526 .U => "c_uint",5543 .u => "c_uint",
5527 .L => "c_long",5544 .l => "c_long",
5528 .LU => "c_ulong",5545 .lu => "c_ulong",
5529 .LL => "c_longlong",5546 .ll => "c_longlong",
5530 .LLU => "c_ulonglong",5547 .llu => "c_ulonglong",
5531 else => unreachable,5548 else => unreachable,
5532 });5549 });
5533 lit_bytes = lit_bytes[0 .. lit_bytes.len - switch (tok.id.IntegerLiteral) {5550 lit_bytes = lit_bytes[0 .. lit_bytes.len - switch (suffix) {
5534 .U, .L => @as(u8, 1),5551 .u, .l => @as(u8, 1),
5535 .LU, .LL => 2,5552 .lu, .ll => 2,
5536 .LLU => 3,5553 .llu => 3,
5537 else => unreachable,5554 else => unreachable,
5538 }];5555 }];
5539 _ = try appendToken(c, .Comma, ",");5556 _ = try appendToken(c, .Comma, ",");
5540 cast_node.params()[1] = try transCreateNodeInt(c, lit_bytes);5557 cast_node.params()[1] = try transCreateNodeInt(c, lit_bytes);
5541 cast_node.rparen_token = try appendToken(c, .RParen, ")");5558 cast_node.rparen_token = try appendToken(c, .RParen, ")");
5542 return &cast_node.base;5559 return &cast_node.base;
5543 } else if (tok.id == .FloatLiteral) {5560 },
5544 if (lit_bytes[0] == '.')5561 .FloatLiteral => |suffix| {
5545 lit_bytes = try std.fmt.allocPrint(c.arena, "0{}", .{lit_bytes});5562 if (lit_bytes[0] == '.')
5546 if (tok.id.FloatLiteral == .None) {5563 lit_bytes = try std.fmt.allocPrint(c.arena, "0{}", .{lit_bytes});
5547 return transCreateNodeFloat(c, lit_bytes);5564 if (suffix == .none) {
5548 }5565 return transCreateNodeFloat(c, lit_bytes);
5549 const cast_node = try c.createBuiltinCall("@as", 2);5566 }
5550 cast_node.params()[0] = try transCreateNodeIdentifier(c, switch (tok.id.FloatLiteral) {5567 const cast_node = try c.createBuiltinCall("@as", 2);
5551 .F => "f32",5568 cast_node.params()[0] = try transCreateNodeIdentifier(c, switch (suffix) {
5552 .L => "c_longdouble",5569 .f => "f32",
5553 else => unreachable,5570 .l => "c_longdouble",
5554 });5571 else => unreachable,
5555 _ = try appendToken(c, .Comma, ",");5572 });
5556 cast_node.params()[1] = try transCreateNodeFloat(c, lit_bytes[0 .. lit_bytes.len - 1]);5573 _ = try appendToken(c, .Comma, ",");
5557 cast_node.rparen_token = try appendToken(c, .RParen, ")");5574 cast_node.params()[1] = try transCreateNodeFloat(c, lit_bytes[0 .. lit_bytes.len - 1]);
5558 return &cast_node.base;5575 cast_node.rparen_token = try appendToken(c, .RParen, ")");
5559 } else unreachable;5576 return &cast_node.base;
5577 },
5578 else => unreachable,
5579 }
5560}5580}
55615581
5562fn zigifyEscapeSequences(ctx: *Context, source_bytes: []const u8, name: []const u8, source_loc: ZigClangSourceLocation) ![]const u8 {5582fn zigifyEscapeSequences(ctx: *Context, source_bytes: []const u8, name: []const u8, source_loc: ZigClangSourceLocation) ![]const u8 {
...@@ -5719,13 +5739,13 @@ fn zigifyEscapeSequences(ctx: *Context, source_bytes: []const u8, name: []const...@@ -5719,13 +5739,13 @@ fn zigifyEscapeSequences(ctx: *Context, source_bytes: []const u8, name: []const
5719 return bytes[0..i];5739 return bytes[0..i];
5720}5740}
57215741
5722fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node {5742fn parseCPrimaryExpr(c: *Context, it: *CTokIterator, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node {
5723 const tok = it.next().?;5743 const tok = it.next().?;
5724 switch (tok.id) {5744 const slice = it.slice(it.i);
5745 switch (tok) {
5725 .CharLiteral => {5746 .CharLiteral => {
5726 const first_tok = it.list.at(0);5747 if (slice[0] != '\'' or slice[1] == '\\' or slice.len == 3) {
5727 if (source[tok.start] != '\'' or source[tok.start + 1] == '\\' or tok.end - tok.start == 3) {5748 const token = try appendToken(c, .CharLiteral, try zigifyEscapeSequences(c, slice, it.slice(0), source_loc));
5728 const token = try appendToken(c, .CharLiteral, try zigifyEscapeSequences(c, source[tok.start..tok.end], source[first_tok.start..first_tok.end], source_loc));
5729 const node = try c.arena.create(ast.Node.OneToken);5749 const node = try c.arena.create(ast.Node.OneToken);
5730 node.* = .{5750 node.* = .{
5731 .base = .{ .tag = .CharLiteral },5751 .base = .{ .tag = .CharLiteral },
...@@ -5733,7 +5753,7 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8,...@@ -5733,7 +5753,7 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8,
5733 };5753 };
5734 return &node.base;5754 return &node.base;
5735 } else {5755 } else {
5736 const token = try appendTokenFmt(c, .IntegerLiteral, "0x{x}", .{source[tok.start + 1 .. tok.end - 1]});5756 const token = try appendTokenFmt(c, .IntegerLiteral, "0x{x}", .{slice[1 .. slice.len - 1]});
5737 const node = try c.arena.create(ast.Node.OneToken);5757 const node = try c.arena.create(ast.Node.OneToken);
5738 node.* = .{5758 node.* = .{
5739 .base = .{ .tag = .IntegerLiteral },5759 .base = .{ .tag = .IntegerLiteral },
...@@ -5743,8 +5763,7 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8,...@@ -5743,8 +5763,7 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8,
5743 }5763 }
5744 },5764 },
5745 .StringLiteral => {5765 .StringLiteral => {
5746 const first_tok = it.list.at(0);5766 const token = try appendToken(c, .StringLiteral, try zigifyEscapeSequences(c, slice, it.slice(0), source_loc));
5747 const token = try appendToken(c, .StringLiteral, try zigifyEscapeSequences(c, source[tok.start..tok.end], source[first_tok.start..first_tok.end], source_loc));
5748 const node = try c.arena.create(ast.Node.OneToken);5767 const node = try c.arena.create(ast.Node.OneToken);
5749 node.* = .{5768 node.* = .{
5750 .base = .{ .tag = .StringLiteral },5769 .base = .{ .tag = .StringLiteral },
...@@ -5753,7 +5772,7 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8,...@@ -5753,7 +5772,7 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8,
5753 return &node.base;5772 return &node.base;
5754 },5773 },
5755 .IntegerLiteral, .FloatLiteral => {5774 .IntegerLiteral, .FloatLiteral => {
5756 return parseCNumLit(c, tok, source, source_loc);5775 return parseCNumLit(c, it, source_loc);
5757 },5776 },
5758 // eventually this will be replaced by std.c.parse which will handle these correctly5777 // eventually this will be replaced by std.c.parse which will handle these correctly
5759 .Keyword_void => return transCreateNodeIdentifierUnchecked(c, "c_void"),5778 .Keyword_void => return transCreateNodeIdentifierUnchecked(c, "c_void"),
...@@ -5763,37 +5782,50 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8,...@@ -5763,37 +5782,50 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8,
5763 .Keyword_int => return transCreateNodeIdentifierUnchecked(c, "c_int"),5782 .Keyword_int => return transCreateNodeIdentifierUnchecked(c, "c_int"),
5764 .Keyword_float => return transCreateNodeIdentifierUnchecked(c, "f32"),5783 .Keyword_float => return transCreateNodeIdentifierUnchecked(c, "f32"),
5765 .Keyword_short => return transCreateNodeIdentifierUnchecked(c, "c_short"),5784 .Keyword_short => return transCreateNodeIdentifierUnchecked(c, "c_short"),
5766 .Keyword_char => return transCreateNodeIdentifierUnchecked(c, "c_char"),5785 .Keyword_char => return transCreateNodeIdentifierUnchecked(c, "u8"),
5767 .Keyword_unsigned => if (it.next()) |t| {5786 .Keyword_unsigned => if (it.next()) |t| switch (t) {
5768 switch (t.id) {5787 .Keyword_char => return transCreateNodeIdentifierUnchecked(c, "u8"),
5769 .Keyword_short => return transCreateNodeIdentifierUnchecked(c, "c_ushort"),5788 .Keyword_short => return transCreateNodeIdentifierUnchecked(c, "c_ushort"),
5770 .Keyword_int => return transCreateNodeIdentifierUnchecked(c, "c_uint"),5789 .Keyword_int => return transCreateNodeIdentifierUnchecked(c, "c_uint"),
5771 .Keyword_long => if (it.peek() != null and it.peek().?.id == .Keyword_long) {5790 .Keyword_long => if (it.peek() != null and it.peek().? == .Keyword_long) {
5772 _ = it.next();5791 _ = it.next();
5773 return transCreateNodeIdentifierUnchecked(c, "c_ulonglong");5792 return transCreateNodeIdentifierUnchecked(c, "c_ulonglong");
5774 } else return transCreateNodeIdentifierUnchecked(c, "c_ulong"),5793 } else return transCreateNodeIdentifierUnchecked(c, "c_ulong"),
5775 else => {5794 else => {
5776 _ = it.prev();5795 it.i -= 1;
5777 return transCreateNodeIdentifierUnchecked(c, "c_uint");5796 return transCreateNodeIdentifierUnchecked(c, "c_uint");
5778 },5797 },
5779 }
5780 } else {5798 } else {
5781 return transCreateNodeIdentifierUnchecked(c, "c_uint");5799 return transCreateNodeIdentifierUnchecked(c, "c_uint");
5782 },5800 },
5801 .Keyword_signed => if (it.next()) |t| switch (t) {
5802 .Keyword_char => return transCreateNodeIdentifierUnchecked(c, "i8"),
5803 .Keyword_short => return transCreateNodeIdentifierUnchecked(c, "c_short"),
5804 .Keyword_int => return transCreateNodeIdentifierUnchecked(c, "c_int"),
5805 .Keyword_long => if (it.peek() != null and it.peek().? == .Keyword_long) {
5806 _ = it.next();
5807 return transCreateNodeIdentifierUnchecked(c, "c_longlong");
5808 } else return transCreateNodeIdentifierUnchecked(c, "c_long"),
5809 else => {
5810 it.i -= 1;
5811 return transCreateNodeIdentifierUnchecked(c, "c_int");
5812 },
5813 } else {
5814 return transCreateNodeIdentifierUnchecked(c, "c_int");
5815 },
5783 .Identifier => {5816 .Identifier => {
5784 const mangled_name = scope.getAlias(source[tok.start..tok.end]);5817 const mangled_name = scope.getAlias(it.slice(it.i));
5785 return transCreateNodeIdentifier(c, mangled_name);5818 return transCreateNodeIdentifier(c, mangled_name);
5786 },5819 },
5787 .LParen => {5820 .LParen => {
5788 const inner_node = try parseCExpr(c, it, source, source_loc, scope);5821 const inner_node = try parseCExpr(c, it, source_loc, scope);
57895822
5790 const next_id = it.next().?.id;5823 const next_id = it.next().?;
5791 if (next_id != .RParen) {5824 if (next_id != .RParen) {
5792 const first_tok = it.list.at(0);
5793 try failDecl(5825 try failDecl(
5794 c,5826 c,
5795 source_loc,5827 source_loc,
5796 source[first_tok.start..first_tok.end],5828 it.slice(0),
5797 "unable to translate C expr: expected ')'' instead got: {}",5829 "unable to translate C expr: expected ')'' instead got: {}",
5798 .{@tagName(next_id)},5830 .{@tagName(next_id)},
5799 );5831 );
...@@ -5801,7 +5833,7 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8,...@@ -5801,7 +5833,7 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8,
5801 }5833 }
5802 var saw_l_paren = false;5834 var saw_l_paren = false;
5803 var saw_integer_literal = false;5835 var saw_integer_literal = false;
5804 switch (it.peek().?.id) {5836 switch (it.peek().?) {
5805 // (type)(to_cast)5837 // (type)(to_cast)
5806 .LParen => {5838 .LParen => {
5807 saw_l_paren = true;5839 saw_l_paren = true;
...@@ -5819,14 +5851,13 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8,...@@ -5819,14 +5851,13 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8,
5819 // hack to get zig fmt to render a comma in builtin calls5851 // hack to get zig fmt to render a comma in builtin calls
5820 _ = try appendToken(c, .Comma, ",");5852 _ = try appendToken(c, .Comma, ",");
58215853
5822 const node_to_cast = try parseCExpr(c, it, source, source_loc, scope);5854 const node_to_cast = try parseCExpr(c, it, source_loc, scope);
58235855
5824 if (saw_l_paren and it.next().?.id != .RParen) {5856 if (saw_l_paren and it.next().? != .RParen) {
5825 const first_tok = it.list.at(0);
5826 try failDecl(5857 try failDecl(
5827 c,5858 c,
5828 source_loc,5859 source_loc,
5829 source[first_tok.start..first_tok.end],5860 it.slice(0),
5830 "unable to translate C expr: expected ')''",5861 "unable to translate C expr: expected ')''",
5831 .{},5862 .{},
5832 );5863 );
...@@ -5857,13 +5888,12 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8,...@@ -5857,13 +5888,12 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8,
5857 return &group_node.base;5888 return &group_node.base;
5858 },5889 },
5859 else => {5890 else => {
5860 const first_tok = it.list.at(0);
5861 try failDecl(5891 try failDecl(
5862 c,5892 c,
5863 source_loc,5893 source_loc,
5864 source[first_tok.start..first_tok.end],5894 it.slice(0),
5865 "unable to translate C expr: unexpected token .{}",5895 "unable to translate C expr: unexpected token .{}",
5866 .{@tagName(tok.id)},5896 .{@tagName(tok)},
5867 );5897 );
5868 return error.ParseError;5898 return error.ParseError;
5869 },5899 },
...@@ -5971,61 +6001,52 @@ fn macroIntToBool(c: *Context, node: *ast.Node) !*ast.Node {...@@ -5971,61 +6001,52 @@ fn macroIntToBool(c: *Context, node: *ast.Node) !*ast.Node {
5971 return &group_node.base;6001 return &group_node.base;
5972}6002}
59736003
5974fn parseCSuffixOpExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node {6004fn parseCSuffixOpExpr(c: *Context, it: *CTokIterator, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node {
5975 var node = try parseCPrimaryExpr(c, it, source, source_loc, scope);6005 var node = try parseCPrimaryExpr(c, it, source_loc, scope);
5976 while (true) {6006 while (true) {
5977 const tok = it.next().?;
5978 var op_token: ast.TokenIndex = undefined;6007 var op_token: ast.TokenIndex = undefined;
5979 var op_id: ast.Node.Tag = undefined;6008 var op_id: ast.Node.Tag = undefined;
5980 var bool_op = false;6009 var bool_op = false;
5981 switch (tok.id) {6010 switch (it.next().?) {
5982 .Period => {6011 .Period => {
5983 const name_tok = it.next().?;6012 if (it.next().? != .Identifier) {
5984 if (name_tok.id != .Identifier) {
5985 const first_tok = it.list.at(0);
5986 try failDecl(6013 try failDecl(
5987 c,6014 c,
5988 source_loc,6015 source_loc,
5989 source[first_tok.start..first_tok.end],6016 it.slice(0),
5990 "unable to translate C expr: expected identifier",6017 "unable to translate C expr: expected identifier",
5991 .{},6018 .{},
5992 );6019 );
5993 return error.ParseError;6020 return error.ParseError;
5994 }6021 }
59956022
5996 node = try transCreateNodeFieldAccess(c, node, source[name_tok.start..name_tok.end]);6023 node = try transCreateNodeFieldAccess(c, node, it.slice(it.i));
5997 continue;6024 continue;
5998 },6025 },
5999 .Arrow => {6026 .Arrow => {
6000 const name_tok = it.next().?;6027 if (it.next().? != .Identifier) {
6001 if (name_tok.id != .Identifier) {
6002 const first_tok = it.list.at(0);
6003 try failDecl(6028 try failDecl(
6004 c,6029 c,
6005 source_loc,6030 source_loc,
6006 source[first_tok.start..first_tok.end],6031 it.slice(0),
6007 "unable to translate C expr: expected identifier",6032 "unable to translate C expr: expected identifier",
6008 .{},6033 .{},
6009 );6034 );
6010 return error.ParseError;6035 return error.ParseError;
6011 }6036 }
6012 const deref = try transCreateNodePtrDeref(c, node);6037 const deref = try transCreateNodePtrDeref(c, node);
6013 node = try transCreateNodeFieldAccess(c, deref, source[name_tok.start..name_tok.end]);6038 node = try transCreateNodeFieldAccess(c, deref, it.slice(it.i));
6014 continue;6039 continue;
6015 },6040 },
6016 .Asterisk => {6041 .Asterisk => {
6017 if (it.peek().?.id == .RParen) {6042 if (it.peek().? == .RParen) {
6018 // type *)6043 // type *)
60196044
6020 // hack to get zig fmt to render a comma in builtin calls6045 // hack to get zig fmt to render a comma in builtin calls
6021 _ = try appendToken(c, .Comma, ",");6046 _ = try appendToken(c, .Comma, ",");
60226047
6023 // * token
6024 _ = it.prev();
6025 // last token of `node`6048 // last token of `node`
6026 const prev_id = it.prev().?.id;6049 const prev_id = it.list[it.i - 1].id;
6027 _ = it.next();
6028 _ = it.next();
60296050
6030 if (prev_id == .Keyword_void) {6051 if (prev_id == .Keyword_void) {
6031 const ptr = try transCreateNodePtrType(c, false, false, .Asterisk);6052 const ptr = try transCreateNodePtrType(c, false, false, .Asterisk);
...@@ -6096,15 +6117,14 @@ fn parseCSuffixOpExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8,...@@ -6096,15 +6117,14 @@ fn parseCSuffixOpExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8,
6096 },6117 },
6097 .LBracket => {6118 .LBracket => {
6098 const arr_node = try transCreateNodeArrayAccess(c, node);6119 const arr_node = try transCreateNodeArrayAccess(c, node);
6099 arr_node.index_expr = try parseCPrefixOpExpr(c, it, source, source_loc, scope);6120 arr_node.index_expr = try parseCPrefixOpExpr(c, it, source_loc, scope);
6100 arr_node.rtoken = try appendToken(c, .RBracket, "]");6121 arr_node.rtoken = try appendToken(c, .RBracket, "]");
6101 node = &arr_node.base;6122 node = &arr_node.base;
6102 if (it.next().?.id != .RBracket) {6123 if (it.next().? != .RBracket) {
6103 const first_tok = it.list.at(0);
6104 try failDecl(6124 try failDecl(
6105 c,6125 c,
6106 source_loc,6126 source_loc,
6107 source[first_tok.start..first_tok.end],6127 it.slice(0),
6108 "unable to translate C expr: expected ']'",6128 "unable to translate C expr: expected ']'",
6109 .{},6129 .{},
6110 );6130 );
...@@ -6117,23 +6137,21 @@ fn parseCSuffixOpExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8,...@@ -6117,23 +6137,21 @@ fn parseCSuffixOpExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8,
6117 var call_params = std.ArrayList(*ast.Node).init(c.gpa);6137 var call_params = std.ArrayList(*ast.Node).init(c.gpa);
6118 defer call_params.deinit();6138 defer call_params.deinit();
6119 while (true) {6139 while (true) {
6120 const arg = try parseCPrefixOpExpr(c, it, source, source_loc, scope);6140 const arg = try parseCPrefixOpExpr(c, it, source_loc, scope);
6121 try call_params.append(arg);6141 try call_params.append(arg);
6122 const next = it.next().?;6142 switch (it.next().?) {
6123 if (next.id == .Comma)6143 .Comma => _ = try appendToken(c, .Comma, ","),
6124 _ = try appendToken(c, .Comma, ",")6144 .RParen => break,
6125 else if (next.id == .RParen)6145 else => {
6126 break6146 try failDecl(
6127 else {6147 c,
6128 const first_tok = it.list.at(0);6148 source_loc,
6129 try failDecl(6149 it.slice(0),
6130 c,6150 "unable to translate C expr: expected ',' or ')'",
6131 source_loc,6151 .{},
6132 source[first_tok.start..first_tok.end],6152 );
6133 "unable to translate C expr: expected ',' or ')'",6153 return error.ParseError;
6134 .{},6154 },
6135 );
6136 return error.ParseError;
6137 }6155 }
6138 }6156 }
6139 const call_node = try ast.Node.Call.alloc(c.arena, call_params.items.len);6157 const call_node = try ast.Node.Call.alloc(c.arena, call_params.items.len);
...@@ -6158,23 +6176,21 @@ fn parseCSuffixOpExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8,...@@ -6158,23 +6176,21 @@ fn parseCSuffixOpExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8,
6158 defer init_vals.deinit();6176 defer init_vals.deinit();
61596177
6160 while (true) {6178 while (true) {
6161 const val = try parseCPrefixOpExpr(c, it, source, source_loc, scope);6179 const val = try parseCPrefixOpExpr(c, it, source_loc, scope);
6162 try init_vals.append(val);6180 try init_vals.append(val);
6163 const next = it.next().?;6181 switch (it.next().?) {
6164 if (next.id == .Comma)6182 .Comma => _ = try appendToken(c, .Comma, ","),
6165 _ = try appendToken(c, .Comma, ",")6183 .RBrace => break,
6166 else if (next.id == .RBrace)6184 else => {
6167 break6185 try failDecl(
6168 else {6186 c,
6169 const first_tok = it.list.at(0);6187 source_loc,
6170 try failDecl(6188 it.slice(0),
6171 c,6189 "unable to translate C expr: expected ',' or '}}'",
6172 source_loc,6190 .{},
6173 source[first_tok.start..first_tok.end],6191 );
6174 "unable to translate C expr: expected ',' or '}}'",6192 return error.ParseError;
6175 .{},6193 },
6176 );
6177 return error.ParseError;
6178 }6194 }
6179 }6195 }
6180 const tuple_node = try ast.Node.StructInitializerDot.alloc(c.arena, init_vals.items.len);6196 const tuple_node = try ast.Node.StructInitializerDot.alloc(c.arena, init_vals.items.len);
...@@ -6221,22 +6237,22 @@ fn parseCSuffixOpExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8,...@@ -6221,22 +6237,22 @@ fn parseCSuffixOpExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8,
6221 op_id = .ArrayCat;6237 op_id = .ArrayCat;
6222 op_token = try appendToken(c, .PlusPlus, "++");6238 op_token = try appendToken(c, .PlusPlus, "++");
62236239
6224 _ = it.prev();6240 it.i -= 1;
6225 },6241 },
6226 .Identifier => {6242 .Identifier => {
6227 op_id = .ArrayCat;6243 op_id = .ArrayCat;
6228 op_token = try appendToken(c, .PlusPlus, "++");6244 op_token = try appendToken(c, .PlusPlus, "++");
62296245
6230 _ = it.prev();6246 it.i -= 1;
6231 },6247 },
6232 else => {6248 else => {
6233 _ = it.prev();6249 it.i -= 1;
6234 return node;6250 return node;
6235 },6251 },
6236 }6252 }
6237 const cast_fn = if (bool_op) macroIntToBool else macroBoolToInt;6253 const cast_fn = if (bool_op) macroIntToBool else macroBoolToInt;
6238 const lhs_node = try cast_fn(c, node);6254 const lhs_node = try cast_fn(c, node);
6239 const rhs_node = try parseCPrefixOpExpr(c, it, source, source_loc, scope);6255 const rhs_node = try parseCPrefixOpExpr(c, it, source_loc, scope);
6240 const op_node = try c.arena.create(ast.Node.SimpleInfixOp);6256 const op_node = try c.arena.create(ast.Node.SimpleInfixOp);
6241 op_node.* = .{6257 op_node.* = .{
6242 .base = .{ .tag = op_id },6258 .base = .{ .tag = op_id },
...@@ -6248,38 +6264,36 @@ fn parseCSuffixOpExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8,...@@ -6248,38 +6264,36 @@ fn parseCSuffixOpExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8,
6248 }6264 }
6249}6265}
62506266
6251fn parseCPrefixOpExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node {6267fn parseCPrefixOpExpr(c: *Context, it: *CTokIterator, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node {
6252 const op_tok = it.next().?;6268 switch (it.next().?) {
6253
6254 switch (op_tok.id) {
6255 .Bang => {6269 .Bang => {
6256 const node = try transCreateNodeSimplePrefixOp(c, .BoolNot, .Bang, "!");6270 const node = try transCreateNodeSimplePrefixOp(c, .BoolNot, .Bang, "!");
6257 node.rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);6271 node.rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
6258 return &node.base;6272 return &node.base;
6259 },6273 },
6260 .Minus => {6274 .Minus => {
6261 const node = try transCreateNodeSimplePrefixOp(c, .Negation, .Minus, "-");6275 const node = try transCreateNodeSimplePrefixOp(c, .Negation, .Minus, "-");
6262 node.rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);6276 node.rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
6263 return &node.base;6277 return &node.base;
6264 },6278 },
6265 .Plus => return try parseCPrefixOpExpr(c, it, source, source_loc, scope),6279 .Plus => return try parseCPrefixOpExpr(c, it, source_loc, scope),
6266 .Tilde => {6280 .Tilde => {
6267 const node = try transCreateNodeSimplePrefixOp(c, .BitNot, .Tilde, "~");6281 const node = try transCreateNodeSimplePrefixOp(c, .BitNot, .Tilde, "~");
6268 node.rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);6282 node.rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
6269 return &node.base;6283 return &node.base;
6270 },6284 },
6271 .Asterisk => {6285 .Asterisk => {
6272 const node = try parseCPrefixOpExpr(c, it, source, source_loc, scope);6286 const node = try parseCPrefixOpExpr(c, it, source_loc, scope);
6273 return try transCreateNodePtrDeref(c, node);6287 return try transCreateNodePtrDeref(c, node);
6274 },6288 },
6275 .Ampersand => {6289 .Ampersand => {
6276 const node = try transCreateNodeSimplePrefixOp(c, .AddressOf, .Ampersand, "&");6290 const node = try transCreateNodeSimplePrefixOp(c, .AddressOf, .Ampersand, "&");
6277 node.rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);6291 node.rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
6278 return &node.base;6292 return &node.base;
6279 },6293 },
6280 else => {6294 else => {
6281 _ = it.prev();6295 it.i -= 1;
6282 return try parseCSuffixOpExpr(c, it, source, source_loc, scope);6296 return try parseCSuffixOpExpr(c, it, source_loc, scope);
6283 },6297 },
6284 }6298 }
6285}6299}