authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2019-12-29 20:01:40+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2019-12-29 20:04:19+02:00
logca211617bd6a780037430f170f1c1a730b8aac17
treebb01bea243fc1e6b5f072f64e4e76657a60399a9
parent6df9e9fe47ad79f362a57279751374b6966782ed
signature Commit is signed but in an unrecognized format.

update c_tokenzier tests to new api

1 files changed, 83 insertions(+), 89 deletions(-)

src-self-hosted/c_tokenizer.zig+83-89
...@@ -8,7 +8,7 @@ pub const TokenList = std.SegmentedList(CToken, 32);...@@ -8,7 +8,7 @@ pub const TokenList = std.SegmentedList(CToken, 32);
88
9pub const CToken = struct {9pub const CToken = struct {
10 id: Id,10 id: Id,
11 bytes: []const u8,11 bytes: []const u8 = "",
12 num_lit_suffix: NumLitSuffix = .None,12 num_lit_suffix: NumLitSuffix = .None,
1313
14 pub const Id = enum {14 pub const Id = enum {
...@@ -238,14 +238,14 @@ fn zigifyEscapeSequences(ctx: *Context, loc: ZigClangSourceLocation, name: []con...@@ -238,14 +238,14 @@ fn zigifyEscapeSequences(ctx: *Context, loc: ZigClangSourceLocation, name: []con
238fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:0]const u8, i: *usize) !CToken {238fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:0]const u8, i: *usize) !CToken {
239 var state: enum {239 var state: enum {
240 Start,240 Start,
241 GotLt,241 SawLt,
242 GotGt,242 SawGt,
243 GotPlus,243 SawPlus,
244 GotMinus,244 SawMinus,
245 GotAmpersand,245 SawAmpersand,
246 GotPipe,246 SawPipe,
247 GotBang,247 SawBang,
248 GotEq,248 SawEq,
249 CharLit,249 CharLit,
250 OpenComment,250 OpenComment,
251 Comment,251 Comment,
...@@ -255,7 +255,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:...@@ -255,7 +255,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
255 Identifier,255 Identifier,
256 Decimal,256 Decimal,
257 Octal,257 Octal,
258 GotZero,258 SawZero,
259 Hex,259 Hex,
260 Bin,260 Bin,
261 Float,261 Float,
...@@ -286,7 +286,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:...@@ -286,7 +286,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
286 .Hex,286 .Hex,
287 .Bin,287 .Bin,
288 .Octal,288 .Octal,
289 .GotZero,289 .SawZero,
290 .Float,290 .Float,
291 .FloatExp,291 .FloatExp,
292 => {292 => {
...@@ -294,19 +294,19 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:...@@ -294,19 +294,19 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
294 return result;294 return result;
295 },295 },
296 .Start,296 .Start,
297 .GotMinus,297 .SawMinus,
298 .Done,298 .Done,
299 .NumLitIntSuffixU,299 .NumLitIntSuffixU,
300 .NumLitIntSuffixL,300 .NumLitIntSuffixL,
301 .NumLitIntSuffixUL,301 .NumLitIntSuffixUL,
302 .NumLitIntSuffixLL,302 .NumLitIntSuffixLL,
303 .GotLt,303 .SawLt,
304 .GotGt,304 .SawGt,
305 .GotPlus,305 .SawPlus,
306 .GotAmpersand,306 .SawAmpersand,
307 .GotPipe,307 .SawPipe,
308 .GotBang,308 .SawBang,
309 .GotEq,309 .SawEq,
310 => {310 => {
311 return result;311 return result;
312 },312 },
...@@ -358,7 +358,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:...@@ -358,7 +358,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
358 begin_index = i.*;358 begin_index = i.*;
359 },359 },
360 '0' => {360 '0' => {
361 state = .GotZero;361 state = .SawZero;
362 result.id = .NumLitInt;362 result.id = .NumLitInt;
363 begin_index = i.*;363 begin_index = i.*;
364 },364 },
...@@ -368,11 +368,11 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:...@@ -368,11 +368,11 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
368 },368 },
369 '<' => {369 '<' => {
370 result.id = .Lt;370 result.id = .Lt;
371 state = .GotLt;371 state = .SawLt;
372 },372 },
373 '>' => {373 '>' => {
374 result.id = .Gt;374 result.id = .Gt;
375 state = .GotGt;375 state = .SawGt;
376 },376 },
377 '(' => {377 '(' => {
378 result.id = .LParen;378 result.id = .LParen;
...@@ -388,15 +388,15 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:...@@ -388,15 +388,15 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
388 },388 },
389 '+' => {389 '+' => {
390 result.id = .Plus;390 result.id = .Plus;
391 state = .GotPlus;391 state = .SawPlus;
392 },392 },
393 '-' => {393 '-' => {
394 result.id = .Minus;394 result.id = .Minus;
395 state = .GotMinus;395 state = .SawMinus;
396 },396 },
397 '!' => {397 '!' => {
398 result.id = .Bang;398 result.id = .Bang;
399 state = .GotBang;399 state = .SawBang;
400 },400 },
401 '~' => {401 '~' => {
402 result.id = .Tilde;402 result.id = .Tilde;
...@@ -404,7 +404,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:...@@ -404,7 +404,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
404 },404 },
405 '=' => {405 '=' => {
406 result.id = .Assign;406 result.id = .Assign;
407 state = .GotEq;407 state = .SawEq;
408 },408 },
409 ',' => {409 ',' => {
410 result.id = .Comma;410 result.id = .Comma;
...@@ -420,11 +420,11 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:...@@ -420,11 +420,11 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
420 },420 },
421 '|' => {421 '|' => {
422 result.id = .Pipe;422 result.id = .Pipe;
423 state = .GotPipe;423 state = .SawPipe;
424 },424 },
425 '&' => {425 '&' => {
426 result.id = .Ampersand;426 result.id = .Ampersand;
427 state = .GotAmpersand;427 state = .SawAmpersand;
428 },428 },
429 '?' => {429 '?' => {
430 result.id = .QuestionMark;430 result.id = .QuestionMark;
...@@ -441,7 +441,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:...@@ -441,7 +441,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
441 }441 }
442 },442 },
443 .Done => return result,443 .Done => return result,
444 .GotMinus => {444 .SawMinus => {
445 switch (c) {445 switch (c) {
446 '>' => {446 '>' => {
447 result.id = .Arrow;447 result.id = .Arrow;
...@@ -454,7 +454,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:...@@ -454,7 +454,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
454 else => return result,454 else => return result,
455 }455 }
456 },456 },
457 .GotPlus => {457 .SawPlus => {
458 switch (c) {458 switch (c) {
459 '+' => {459 '+' => {
460 result.id = .Increment;460 result.id = .Increment;
...@@ -463,7 +463,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:...@@ -463,7 +463,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
463 else => return result,463 else => return result,
464 }464 }
465 },465 },
466 .GotLt => {466 .SawLt => {
467 switch (c) {467 switch (c) {
468 '<' => {468 '<' => {
469 result.id = .Shl;469 result.id = .Shl;
...@@ -476,7 +476,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:...@@ -476,7 +476,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
476 else => return result,476 else => return result,
477 }477 }
478 },478 },
479 .GotGt => {479 .SawGt => {
480 switch (c) {480 switch (c) {
481 '>' => {481 '>' => {
482 result.id = .Shr;482 result.id = .Shr;
...@@ -489,7 +489,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:...@@ -489,7 +489,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
489 else => return result,489 else => return result,
490 }490 }
491 },491 },
492 .GotPipe => {492 .SawPipe => {
493 switch (c) {493 switch (c) {
494 '|' => {494 '|' => {
495 result.id = .Or;495 result.id = .Or;
...@@ -498,7 +498,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:...@@ -498,7 +498,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
498 else => return result,498 else => return result,
499 }499 }
500 },500 },
501 .GotAmpersand => {501 .SawAmpersand => {
502 switch (c) {502 switch (c) {
503 '&' => {503 '&' => {
504 result.id = .And;504 result.id = .And;
...@@ -507,7 +507,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:...@@ -507,7 +507,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
507 else => return result,507 else => return result,
508 }508 }
509 },509 },
510 .GotBang => {510 .SawBang => {
511 switch (c) {511 switch (c) {
512 '=' => {512 '=' => {
513 result.id = .Ne;513 result.id = .Ne;
...@@ -516,7 +516,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:...@@ -516,7 +516,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
516 else => return result,516 else => return result,
517 }517 }
518 },518 },
519 .GotEq => {519 .SawEq => {
520 switch (c) {520 switch (c) {
521 '=' => {521 '=' => {
522 result.id = .Eq;522 result.id = .Eq;
...@@ -557,7 +557,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:...@@ -557,7 +557,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
557 '0'...'9' => {557 '0'...'9' => {
558 state = .FloatExp;558 state = .FloatExp;
559 },559 },
560 else => {560 else => {
561 try failDecl(ctx, loc, name, "macro tokenizing failed: expected a digit or '+' or '-'", .{});561 try failDecl(ctx, loc, name, "macro tokenizing failed: expected a digit or '+' or '-'", .{});
562 return error.TokenizingFailed;562 return error.TokenizingFailed;
563 },563 },
...@@ -617,7 +617,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:...@@ -617,7 +617,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
617 },617 },
618 }618 }
619 },619 },
620 .GotZero => {620 .SawZero => {
621 switch (c) {621 switch (c) {
622 'x', 'X' => {622 'x', 'X' => {
623 state = .Hex;623 state = .Hex;
...@@ -829,9 +829,9 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:...@@ -829,9 +829,9 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
829 unreachable;829 unreachable;
830}830}
831831
832
833fn expectTokens(tl: *TokenList, src: [*:0]const u8, expected: []CToken) void {832fn expectTokens(tl: *TokenList, src: [*:0]const u8, expected: []CToken) void {
834 tokenizeCMacro(tl, src) catch unreachable;833 // these can be undefined since they are only used for error reporting
834 tokenizeCMacro(undefined, undefined, undefined, tl, src) catch unreachable;
835 var it = tl.iterator(0);835 var it = tl.iterator(0);
836 for (expected) |t| {836 for (expected) |t| {
837 var tok = it.next().?;837 var tok = it.next().?;
...@@ -848,81 +848,74 @@ fn expectTokens(tl: *TokenList, src: [*:0]const u8, expected: []CToken) void {...@@ -848,81 +848,74 @@ fn expectTokens(tl: *TokenList, src: [*:0]const u8, expected: []CToken) void {
848 tl.shrink(0);848 tl.shrink(0);
849}849}
850850
851
852test "tokenize macro" {851test "tokenize macro" {
853 var tl = TokenList.init(std.heap.page_allocator);852 var tl = TokenList.init(std.heap.page_allocator);
854 defer tl.deinit();853 defer tl.deinit();
855854
856 expectTokens(&tl, "TEST(0\n", &[_]CToken{855 expectTokens(&tl, "TEST(0\n", &[_]CToken{
857 ctoken(.Identifier, "TEST"),856 .{ .id = .Identifier, .bytes = "TEST" },
858 ctoken(.Fn, ""),857 .{ .id = .Fn },
859 ctoken(.LParen, ""),858 .{ .id = .LParen },
860 ctoken(.NumLitInt, "0"),859 .{ .id = .NumLitInt, .bytes = "0" },
861 ctoken(.Eof, ""),860 .{ .id = .Eof },
862 });861 });
863862
864 expectTokens(&tl, "__FLT_MIN_10_EXP__ -37\n", &[_]CToken{863 expectTokens(&tl, "__FLT_MIN_10_EXP__ -37\n", &[_]CToken{
865 ctoken(.Identifier, "__FLT_MIN_10_EXP__"),864 .{ .id = .Identifier, .bytes = "__FLT_MIN_10_EXP__" },
866 ctoken(.Minus, ""),865 .{ .id = .Minus },
867 ctoken(.NumLitInt, "37"),866 .{ .id = .NumLitInt, .bytes = "37" },
868 ctoken(.Eof, ""),867 .{ .id = .Eof },
869 });868 });
870869
871 expectTokens(&tl, "__llvm__ 1\n#define", &[_]CToken{870 expectTokens(&tl, "__llvm__ 1\n#define", &[_]CToken{
872 ctoken(.Identifier, "__llvm__"),871 .{ .id = .Identifier, .bytes = "__llvm__" },
873 ctoken(.NumLitInt, "1"),872 .{ .id = .NumLitInt, .bytes = "1" },
874 ctoken(.Eof, ""),873 .{ .id = .Eof },
875
876 });874 });
877875
878 expectTokens(&tl, "TEST 2", &[_]CToken{876 expectTokens(&tl, "TEST 2", &[_]CToken{
879 ctoken(.Identifier, "TEST"),877 .{ .id = .Identifier, .bytes = "TEST" },
880 ctoken(.NumLitInt, "2"),878 .{ .id = .NumLitInt, .bytes = "2" },
881 ctoken(.Eof, ""),879 .{ .id = .Eof },
882
883 });880 });
884881
885 expectTokens(&tl, "FOO 0ull", &[_]CToken{882 expectTokens(&tl, "FOO 0ull", &[_]CToken{
886 ctoken(.Identifier, "FOO"),883 .{ .id = .Identifier, .bytes = "FOO" },
887 cnumtoken(.LLU, "0"),884 .{ .id = .NumLitInt, .bytes = "0", .num_lit_suffix = .LLU },
888 ctoken(.Eof, ""),885 .{ .id = .Eof },
889
890 });886 });
891
892}887}
893888
894
895
896test "tokenize macro ops" {889test "tokenize macro ops" {
897 var tl = TokenList.init(std.heap.page_allocator);890 var tl = TokenList.init(std.heap.page_allocator);
898 defer tl.deinit();891 defer tl.deinit();
899892
900 expectTokens(&tl, "ADD A + B", &[_]CToken{893 expectTokens(&tl, "ADD A + B", &[_]CToken{
901 ctoken(.Identifier, "ADD"),894 .{ .id = .Identifier, .bytes = "ADD" },
902 ctoken(.Identifier, "A"),895 .{ .id = .Identifier, .bytes = "A" },
903 ctoken(.Plus, ""),896 .{ .id = .Plus },
904 ctoken(.Identifier, "B"),897 .{ .id = .Identifier, .bytes = "B" },
905 ctoken(.Eof, ""),898 .{ .id = .Eof },
906 });899 });
907900
908 expectTokens(&tl, "ADD (A) + B", &[_]CToken{901 expectTokens(&tl, "ADD (A) + B", &[_]CToken{
909 ctoken(.Identifier, "ADD"),902 .{ .id = .Identifier, .bytes = "ADD" },
910 ctoken(.LParen, ""),903 .{ .id = .LParen },
911 ctoken(.Identifier, "A"),904 .{ .id = .Identifier, .bytes = "A" },
912 ctoken(.RParen, ""),905 .{ .id = .RParen },
913 ctoken(.Plus, ""),906 .{ .id = .Plus },
914 ctoken(.Identifier, "B"),907 .{ .id = .Identifier, .bytes = "B" },
915 ctoken(.Eof, ""),908 .{ .id = .Eof },
916 });909 });
917910
918 expectTokens(&tl, "ADD (A) + B", &[_]CToken{911 expectTokens(&tl, "ADD (A) + B", &[_]CToken{
919 ctoken(.Identifier, "ADD"),912 .{ .id = .Identifier, .bytes = "ADD" },
920 ctoken(.LParen, ""),913 .{ .id = .LParen },
921 ctoken(.Identifier, "A"),914 .{ .id = .Identifier, .bytes = "A" },
922 ctoken(.RParen, ""),915 .{ .id = .RParen },
923 ctoken(.Plus, ""),916 .{ .id = .Plus },
924 ctoken(.Identifier, "B"),917 .{ .id = .Identifier, .bytes = "B" },
925 ctoken(.Eof, ""),918 .{ .id = .Eof },
926 });919 });
927}920}
928921
...@@ -930,19 +923,20 @@ test "escape sequences" {...@@ -930,19 +923,20 @@ test "escape sequences" {
930 var buf: [1024]u8 = undefined;923 var buf: [1024]u8 = undefined;
931 var alloc = std.heap.FixedBufferAllocator.init(buf[0..]);924 var alloc = std.heap.FixedBufferAllocator.init(buf[0..]);
932 const a = &alloc.allocator;925 const a = &alloc.allocator;
933 expect(std.mem.eql(u8, (try zigifyEscapeSequences(a, .{926 // these can be undefined since they are only used for error reporting
927 expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
934 .id = .StrLit,928 .id = .StrLit,
935 .bytes = "\\x0077",929 .bytes = "\\x0077",
936 })).bytes, "\\x77"));930 })).bytes, "\\x77"));
937 expect(std.mem.eql(u8, (try zigifyEscapeSequences(a, .{931 expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
938 .id = .StrLit,932 .id = .StrLit,
939 .bytes = "\\24500",933 .bytes = "\\24500",
940 })).bytes, "\\xa500"));934 })).bytes, "\\xa500"));
941 expect(std.mem.eql(u8, (try zigifyEscapeSequences(a, .{935 expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
942 .id = .StrLit,936 .id = .StrLit,
943 .bytes = "\\x0077 abc",937 .bytes = "\\x0077 abc",
944 })).bytes, "\\x77 abc"));938 })).bytes, "\\x77 abc"));
945 expect(std.mem.eql(u8, (try zigifyEscapeSequences(a, .{939 expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
946 .id = .StrLit,940 .id = .StrLit,
947 .bytes = "\\045abc",941 .bytes = "\\045abc",
948 })).bytes, "\\x25abc"));942 })).bytes, "\\x25abc"));