authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2019-12-29 20:01:40+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2019-12-29 20:04:19+02:00
logca211617bd6a780037430f170f1c1a730b8aac17
treebb01bea243fc1e6b5f072f64e4e76657a60399a9
parent6df9e9fe47ad79f362a57279751374b6966782ed
signature Commit is signed but in an unrecognized format.

update c_tokenzier tests to new api

1 files changed, 83 insertions(+), 89 deletions(-)

src-self-hosted/c_tokenizer.zig+83-89
......@@ -8,7 +8,7 @@ pub const TokenList = std.SegmentedList(CToken, 32);
88
99pub const CToken = struct {
1010 id: Id,
11 bytes: []const u8,
11 bytes: []const u8 = "",
1212 num_lit_suffix: NumLitSuffix = .None,
1313
1414 pub const Id = enum {
......@@ -238,14 +238,14 @@ fn zigifyEscapeSequences(ctx: *Context, loc: ZigClangSourceLocation, name: []con
238238fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:0]const u8, i: *usize) !CToken {
239239 var state: enum {
240240 Start,
241 GotLt,
242 GotGt,
243 GotPlus,
244 GotMinus,
245 GotAmpersand,
246 GotPipe,
247 GotBang,
248 GotEq,
241 SawLt,
242 SawGt,
243 SawPlus,
244 SawMinus,
245 SawAmpersand,
246 SawPipe,
247 SawBang,
248 SawEq,
249249 CharLit,
250250 OpenComment,
251251 Comment,
......@@ -255,7 +255,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
255255 Identifier,
256256 Decimal,
257257 Octal,
258 GotZero,
258 SawZero,
259259 Hex,
260260 Bin,
261261 Float,
......@@ -286,7 +286,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
286286 .Hex,
287287 .Bin,
288288 .Octal,
289 .GotZero,
289 .SawZero,
290290 .Float,
291291 .FloatExp,
292292 => {
......@@ -294,19 +294,19 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
294294 return result;
295295 },
296296 .Start,
297 .GotMinus,
297 .SawMinus,
298298 .Done,
299299 .NumLitIntSuffixU,
300300 .NumLitIntSuffixL,
301301 .NumLitIntSuffixUL,
302302 .NumLitIntSuffixLL,
303 .GotLt,
304 .GotGt,
305 .GotPlus,
306 .GotAmpersand,
307 .GotPipe,
308 .GotBang,
309 .GotEq,
303 .SawLt,
304 .SawGt,
305 .SawPlus,
306 .SawAmpersand,
307 .SawPipe,
308 .SawBang,
309 .SawEq,
310310 => {
311311 return result;
312312 },
......@@ -358,7 +358,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
358358 begin_index = i.*;
359359 },
360360 '0' => {
361 state = .GotZero;
361 state = .SawZero;
362362 result.id = .NumLitInt;
363363 begin_index = i.*;
364364 },
......@@ -368,11 +368,11 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
368368 },
369369 '<' => {
370370 result.id = .Lt;
371 state = .GotLt;
371 state = .SawLt;
372372 },
373373 '>' => {
374374 result.id = .Gt;
375 state = .GotGt;
375 state = .SawGt;
376376 },
377377 '(' => {
378378 result.id = .LParen;
......@@ -388,15 +388,15 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
388388 },
389389 '+' => {
390390 result.id = .Plus;
391 state = .GotPlus;
391 state = .SawPlus;
392392 },
393393 '-' => {
394394 result.id = .Minus;
395 state = .GotMinus;
395 state = .SawMinus;
396396 },
397397 '!' => {
398398 result.id = .Bang;
399 state = .GotBang;
399 state = .SawBang;
400400 },
401401 '~' => {
402402 result.id = .Tilde;
......@@ -404,7 +404,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
404404 },
405405 '=' => {
406406 result.id = .Assign;
407 state = .GotEq;
407 state = .SawEq;
408408 },
409409 ',' => {
410410 result.id = .Comma;
......@@ -420,11 +420,11 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
420420 },
421421 '|' => {
422422 result.id = .Pipe;
423 state = .GotPipe;
423 state = .SawPipe;
424424 },
425425 '&' => {
426426 result.id = .Ampersand;
427 state = .GotAmpersand;
427 state = .SawAmpersand;
428428 },
429429 '?' => {
430430 result.id = .QuestionMark;
......@@ -441,7 +441,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
441441 }
442442 },
443443 .Done => return result,
444 .GotMinus => {
444 .SawMinus => {
445445 switch (c) {
446446 '>' => {
447447 result.id = .Arrow;
......@@ -454,7 +454,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
454454 else => return result,
455455 }
456456 },
457 .GotPlus => {
457 .SawPlus => {
458458 switch (c) {
459459 '+' => {
460460 result.id = .Increment;
......@@ -463,7 +463,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
463463 else => return result,
464464 }
465465 },
466 .GotLt => {
466 .SawLt => {
467467 switch (c) {
468468 '<' => {
469469 result.id = .Shl;
......@@ -476,7 +476,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
476476 else => return result,
477477 }
478478 },
479 .GotGt => {
479 .SawGt => {
480480 switch (c) {
481481 '>' => {
482482 result.id = .Shr;
......@@ -489,7 +489,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
489489 else => return result,
490490 }
491491 },
492 .GotPipe => {
492 .SawPipe => {
493493 switch (c) {
494494 '|' => {
495495 result.id = .Or;
......@@ -498,7 +498,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
498498 else => return result,
499499 }
500500 },
501 .GotAmpersand => {
501 .SawAmpersand => {
502502 switch (c) {
503503 '&' => {
504504 result.id = .And;
......@@ -507,7 +507,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
507507 else => return result,
508508 }
509509 },
510 .GotBang => {
510 .SawBang => {
511511 switch (c) {
512512 '=' => {
513513 result.id = .Ne;
......@@ -516,7 +516,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
516516 else => return result,
517517 }
518518 },
519 .GotEq => {
519 .SawEq => {
520520 switch (c) {
521521 '=' => {
522522 result.id = .Eq;
......@@ -557,7 +557,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
557557 '0'...'9' => {
558558 state = .FloatExp;
559559 },
560 else => {
560 else => {
561561 try failDecl(ctx, loc, name, "macro tokenizing failed: expected a digit or '+' or '-'", .{});
562562 return error.TokenizingFailed;
563563 },
......@@ -617,7 +617,7 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
617617 },
618618 }
619619 },
620 .GotZero => {
620 .SawZero => {
621621 switch (c) {
622622 'x', 'X' => {
623623 state = .Hex;
......@@ -829,9 +829,9 @@ fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:
829829 unreachable;
830830}
831831
832
833832fn expectTokens(tl: *TokenList, src: [*:0]const u8, expected: []CToken) void {
834 tokenizeCMacro(tl, src) catch unreachable;
833 // these can be undefined since they are only used for error reporting
834 tokenizeCMacro(undefined, undefined, undefined, tl, src) catch unreachable;
835835 var it = tl.iterator(0);
836836 for (expected) |t| {
837837 var tok = it.next().?;
......@@ -848,81 +848,74 @@ fn expectTokens(tl: *TokenList, src: [*:0]const u8, expected: []CToken) void {
848848 tl.shrink(0);
849849}
850850
851
852851test "tokenize macro" {
853852 var tl = TokenList.init(std.heap.page_allocator);
854853 defer tl.deinit();
855854
856855 expectTokens(&tl, "TEST(0\n", &[_]CToken{
857 ctoken(.Identifier, "TEST"),
858 ctoken(.Fn, ""),
859 ctoken(.LParen, ""),
860 ctoken(.NumLitInt, "0"),
861 ctoken(.Eof, ""),
856 .{ .id = .Identifier, .bytes = "TEST" },
857 .{ .id = .Fn },
858 .{ .id = .LParen },
859 .{ .id = .NumLitInt, .bytes = "0" },
860 .{ .id = .Eof },
862861 });
863862
864863 expectTokens(&tl, "__FLT_MIN_10_EXP__ -37\n", &[_]CToken{
865 ctoken(.Identifier, "__FLT_MIN_10_EXP__"),
866 ctoken(.Minus, ""),
867 ctoken(.NumLitInt, "37"),
868 ctoken(.Eof, ""),
864 .{ .id = .Identifier, .bytes = "__FLT_MIN_10_EXP__" },
865 .{ .id = .Minus },
866 .{ .id = .NumLitInt, .bytes = "37" },
867 .{ .id = .Eof },
869868 });
870869
871870 expectTokens(&tl, "__llvm__ 1\n#define", &[_]CToken{
872 ctoken(.Identifier, "__llvm__"),
873 ctoken(.NumLitInt, "1"),
874 ctoken(.Eof, ""),
875
871 .{ .id = .Identifier, .bytes = "__llvm__" },
872 .{ .id = .NumLitInt, .bytes = "1" },
873 .{ .id = .Eof },
876874 });
877875
878876 expectTokens(&tl, "TEST 2", &[_]CToken{
879 ctoken(.Identifier, "TEST"),
880 ctoken(.NumLitInt, "2"),
881 ctoken(.Eof, ""),
882
877 .{ .id = .Identifier, .bytes = "TEST" },
878 .{ .id = .NumLitInt, .bytes = "2" },
879 .{ .id = .Eof },
883880 });
884881
885882 expectTokens(&tl, "FOO 0ull", &[_]CToken{
886 ctoken(.Identifier, "FOO"),
887 cnumtoken(.LLU, "0"),
888 ctoken(.Eof, ""),
889
883 .{ .id = .Identifier, .bytes = "FOO" },
884 .{ .id = .NumLitInt, .bytes = "0", .num_lit_suffix = .LLU },
885 .{ .id = .Eof },
890886 });
891
892887}
893888
894
895
896889test "tokenize macro ops" {
897890 var tl = TokenList.init(std.heap.page_allocator);
898891 defer tl.deinit();
899892
900893 expectTokens(&tl, "ADD A + B", &[_]CToken{
901 ctoken(.Identifier, "ADD"),
902 ctoken(.Identifier, "A"),
903 ctoken(.Plus, ""),
904 ctoken(.Identifier, "B"),
905 ctoken(.Eof, ""),
894 .{ .id = .Identifier, .bytes = "ADD" },
895 .{ .id = .Identifier, .bytes = "A" },
896 .{ .id = .Plus },
897 .{ .id = .Identifier, .bytes = "B" },
898 .{ .id = .Eof },
906899 });
907900
908 expectTokens(&tl, "ADD (A) + B", &[_]CToken{
909 ctoken(.Identifier, "ADD"),
910 ctoken(.LParen, ""),
911 ctoken(.Identifier, "A"),
912 ctoken(.RParen, ""),
913 ctoken(.Plus, ""),
914 ctoken(.Identifier, "B"),
915 ctoken(.Eof, ""),
901 expectTokens(&tl, "ADD (A) + B", &[_]CToken{
902 .{ .id = .Identifier, .bytes = "ADD" },
903 .{ .id = .LParen },
904 .{ .id = .Identifier, .bytes = "A" },
905 .{ .id = .RParen },
906 .{ .id = .Plus },
907 .{ .id = .Identifier, .bytes = "B" },
908 .{ .id = .Eof },
916909 });
917910
918911 expectTokens(&tl, "ADD (A) + B", &[_]CToken{
919 ctoken(.Identifier, "ADD"),
920 ctoken(.LParen, ""),
921 ctoken(.Identifier, "A"),
922 ctoken(.RParen, ""),
923 ctoken(.Plus, ""),
924 ctoken(.Identifier, "B"),
925 ctoken(.Eof, ""),
912 .{ .id = .Identifier, .bytes = "ADD" },
913 .{ .id = .LParen },
914 .{ .id = .Identifier, .bytes = "A" },
915 .{ .id = .RParen },
916 .{ .id = .Plus },
917 .{ .id = .Identifier, .bytes = "B" },
918 .{ .id = .Eof },
926919 });
927920}
928921
......@@ -930,19 +923,20 @@ test "escape sequences" {
930923 var buf: [1024]u8 = undefined;
931924 var alloc = std.heap.FixedBufferAllocator.init(buf[0..]);
932925 const a = &alloc.allocator;
933 expect(std.mem.eql(u8, (try zigifyEscapeSequences(a, .{
926 // these can be undefined since they are only used for error reporting
927 expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
934928 .id = .StrLit,
935929 .bytes = "\\x0077",
936930 })).bytes, "\\x77"));
937 expect(std.mem.eql(u8, (try zigifyEscapeSequences(a, .{
931 expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
938932 .id = .StrLit,
939933 .bytes = "\\24500",
940934 })).bytes, "\\xa500"));
941 expect(std.mem.eql(u8, (try zigifyEscapeSequences(a, .{
935 expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
942936 .id = .StrLit,
943937 .bytes = "\\x0077 abc",
944938 })).bytes, "\\x77 abc"));
945 expect(std.mem.eql(u8, (try zigifyEscapeSequences(a, .{
939 expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
946940 .id = .StrLit,
947941 .bytes = "\\045abc",
948942 })).bytes, "\\x25abc"));