authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-05-30 12:07:55-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-05-30 12:07:55-04:00
log7878f9660fcb8fa64fda0b2b67320fe9ddc2727b
treee9286a58127e707c0c06b302347e7d49079e343e
parent5954d5235fd0d029034d68f82bb7831ff47506f8
signaturelock-open Commit is signed but in an unrecognized format.

dep tokenizer: run zig fmt and move exports to canonical location

2 files changed, 75 insertions(+), 102 deletions(-)

src-self-hosted/dep_tokenizer.zig+16-101
......@@ -359,63 +359,6 @@ pub const Tokenizer = struct {
359359 };
360360};
361361
362export fn stage2_DepTokenizer_init(input: [*]const u8, len: usize) stage2_DepTokenizer {
363 const t = std.heap.c_allocator.create(Tokenizer) catch @panic("failed to create .d tokenizer");
364 t.* = Tokenizer.init(std.heap.c_allocator, input[0..len]);
365 return stage2_DepTokenizer{
366 .handle = t,
367 };
368}
369
370export fn stage2_DepTokenizer_deinit(self: *stage2_DepTokenizer) void {
371 self.handle.deinit();
372}
373
374export fn stage2_DepTokenizer_next(self: *stage2_DepTokenizer) stage2_DepNextResult {
375 const otoken = self.handle.next() catch {
376 const textz = std.Buffer.init(&self.handle.arena.allocator, self.handle.error_text) catch @panic("failed to create .d tokenizer error text");
377 return stage2_DepNextResult{
378 .type_id = .error_,
379 .textz = textz.toSlice().ptr,
380 };
381 };
382 const token = otoken orelse {
383 return stage2_DepNextResult{
384 .type_id = .null_,
385 .textz = undefined,
386 };
387 };
388 const textz = std.Buffer.init(&self.handle.arena.allocator, token.bytes) catch @panic("failed to create .d tokenizer token text");
389 return stage2_DepNextResult{
390 .type_id = switch (token.id) {
391 .target => stage2_DepNextResult.TypeId.target,
392 .prereq => stage2_DepNextResult.TypeId.prereq,
393 },
394 .textz = textz.toSlice().ptr,
395 };
396}
397
398export const stage2_DepTokenizer = extern struct {
399 handle: *Tokenizer,
400};
401
402export const stage2_DepNextResult = extern struct {
403 type_id: TypeId,
404
405 // when type_id == error --> error text
406 // when type_id == null --> undefined
407 // when type_id == target --> target pathname
408 // when type_id == prereq --> prereq pathname
409 textz: [*]const u8,
410
411 export const TypeId = extern enum {
412 error_,
413 null_,
414 target,
415 prereq,
416 };
417};
418
419362test "empty file" {
420363 try depTokenizer("", "");
421364}
......@@ -469,78 +412,54 @@ test "empty target linefeeds" {
469412 const expect = "target = {foo.o}";
470413 try depTokenizer(
471414 \\foo.o:
472 ,
473 expect
474 );
415 , expect);
475416 try depTokenizer(
476417 \\foo.o:
477418 \\
478 ,
479 expect
480 );
419 , expect);
481420 try depTokenizer(
482421 \\foo.o:
483 ,
484 expect
485 );
422 , expect);
486423 try depTokenizer(
487424 \\foo.o:
488425 \\
489 ,
490 expect
491 );
426 , expect);
492427}
493428
494429test "empty target linefeeds + continuations" {
495430 const expect = "target = {foo.o}";
496431 try depTokenizer(
497432 \\foo.o:\
498 ,
499 expect
500 );
433 , expect);
501434 try depTokenizer(
502435 \\foo.o:\
503436 \\
504 ,
505 expect
506 );
437 , expect);
507438 try depTokenizer(
508439 \\foo.o:\
509 ,
510 expect
511 );
440 , expect);
512441 try depTokenizer(
513442 \\foo.o:\
514443 \\
515 ,
516 expect
517 );
444 , expect);
518445}
519446
520447test "empty target linefeeds + hspace + continuations" {
521448 const expect = "target = {foo.o}";
522449 try depTokenizer(
523450 \\foo.o: \
524 ,
525 expect
526 );
451 , expect);
527452 try depTokenizer(
528453 \\foo.o: \
529454 \\
530 ,
531 expect
532 );
455 , expect);
533456 try depTokenizer(
534457 \\foo.o: \
535 ,
536 expect
537 );
458 , expect);
538459 try depTokenizer(
539460 \\foo.o: \
540461 \\
541 ,
542 expect
543 );
462 , expect);
544463}
545464
546465test "prereq" {
......@@ -572,15 +491,11 @@ test "prereq continuation" {
572491 try depTokenizer(
573492 \\foo.o: foo.h\
574493 \\bar.h
575 ,
576 expect
577 );
494 , expect);
578495 try depTokenizer(
579496 \\foo.o: foo.h\
580497 \\bar.h
581 ,
582 expect
583 );
498 , expect);
584499}
585500
586501test "multiple prereqs" {
......@@ -907,14 +822,14 @@ test "error target - continuation expecting end-of-line" {
907822 \\target = {foo.o}
908823 \\ERROR: illegal char 'x' at position 8: continuation expecting end-of-line
909824 );
910 try depTokenizer("foo.o: \\ x",
825 try depTokenizer("foo.o: \\\x0dx",
911826 \\target = {foo.o}
912827 \\ERROR: illegal char 'x' at position 9: continuation expecting end-of-line
913828 );
914829}
915830
916831test "error prereq - continuation expecting end-of-line" {
917 try depTokenizer("foo.o: foo.h\\ x",
832 try depTokenizer("foo.o: foo.h\\\x0dx",
918833 \\target = {foo.o}
919834 \\ERROR: illegal char 'x' at position 14: continuation expecting end-of-line
920835 );
src-self-hosted/stage1.zig+59-1
......@@ -15,13 +15,14 @@ const self_hosted_main = @import("main.zig");
1515const Args = arg.Args;
1616const Flag = arg.Flag;
1717const errmsg = @import("errmsg.zig");
18const DepTokenizer = @import("dep_tokenizer.zig").Tokenizer;
1819
1920var stderr_file: fs.File = undefined;
2021var stderr: *io.OutStream(fs.File.WriteError) = undefined;
2122var stdout: *io.OutStream(fs.File.WriteError) = undefined;
2223
2324comptime {
24 _ = @import("dep_tokenizer.zig");
25 _ = @import("dep_tokenizer.zig");
2526}
2627
2728// ABI warning
......@@ -397,3 +398,60 @@ fn printErrMsgToFile(
397398 try stream.writeByteNTimes('~', last_token.end - first_token.start);
398399 try stream.write("\n");
399400}
401
402export fn stage2_DepTokenizer_init(input: [*]const u8, len: usize) stage2_DepTokenizer {
403 const t = std.heap.c_allocator.create(DepTokenizer) catch @panic("failed to create .d tokenizer");
404 t.* = DepTokenizer.init(std.heap.c_allocator, input[0..len]);
405 return stage2_DepTokenizer{
406 .handle = t,
407 };
408}
409
410export fn stage2_DepTokenizer_deinit(self: *stage2_DepTokenizer) void {
411 self.handle.deinit();
412}
413
414export fn stage2_DepTokenizer_next(self: *stage2_DepTokenizer) stage2_DepNextResult {
415 const otoken = self.handle.next() catch {
416 const textz = std.Buffer.init(&self.handle.arena.allocator, self.handle.error_text) catch @panic("failed to create .d tokenizer error text");
417 return stage2_DepNextResult{
418 .type_id = .error_,
419 .textz = textz.toSlice().ptr,
420 };
421 };
422 const token = otoken orelse {
423 return stage2_DepNextResult{
424 .type_id = .null_,
425 .textz = undefined,
426 };
427 };
428 const textz = std.Buffer.init(&self.handle.arena.allocator, token.bytes) catch @panic("failed to create .d tokenizer token text");
429 return stage2_DepNextResult{
430 .type_id = switch (token.id) {
431 .target => stage2_DepNextResult.TypeId.target,
432 .prereq => stage2_DepNextResult.TypeId.prereq,
433 },
434 .textz = textz.toSlice().ptr,
435 };
436}
437
438export const stage2_DepTokenizer = extern struct {
439 handle: *DepTokenizer,
440};
441
442export const stage2_DepNextResult = extern struct {
443 type_id: TypeId,
444
445 // when type_id == error --> error text
446 // when type_id == null --> undefined
447 // when type_id == target --> target pathname
448 // when type_id == prereq --> prereq pathname
449 textz: [*]const u8,
450
451 export const TypeId = extern enum {
452 error_,
453 null_,
454 target,
455 prereq,
456 };
457};