authorgravatar for rpkak@noreply.codeberg.orgrpkak <rpkak@noreply.codeberg.org> 2025-03-14 12:18:10+01:00
committergravatar for alex@alexrp.comAlex Rønne Petersen <alex@alexrp.com> 2025-03-24 15:31:03+01:00
log9f8d938d3825bd5d4d2d7f76907075d13b04d8e1
treed8479ec4f95f55c258b2a8e6c1e70d543bb06997
parent677b2d62e5af4d7a97883994b8d8c45a06535ea0

DepTokenizer: allow space between target and colon

1 files changed, 60 insertions(+), 4 deletions(-)

lib/std/Build/Cache/DepTokenizer.zig+60-4
......@@ -25,7 +25,7 @@ pub fn next(self: *Tokenizer) ?Token {
2525 },
2626 },
2727 .target => switch (char) {
28 '\t', '\n', '\r', ' ' => {
28 '\n', '\r' => {
2929 return errorIllegalChar(.invalid_target, self.index, char);
3030 },
3131 '$' => {
......@@ -40,6 +40,15 @@ pub fn next(self: *Tokenizer) ?Token {
4040 self.state = .target_colon;
4141 self.index += 1;
4242 },
43 '\t', ' ' => {
44 self.state = .target_space;
45
46 const bytes = self.bytes[start..self.index];
47 std.debug.assert(bytes.len != 0);
48 self.index += 1;
49
50 return finishTarget(must_resolve, bytes);
51 },
4352 else => {
4453 self.index += 1;
4554 },
......@@ -110,6 +119,19 @@ pub fn next(self: *Tokenizer) ?Token {
110119 self.state = .target;
111120 },
112121 },
122 .target_space => switch (char) {
123 '\t', ' ' => {
124 // silently ignore additional horizontal whitespace
125 self.index += 1;
126 },
127 ':' => {
128 self.state = .rhs;
129 self.index += 1;
130 },
131 else => {
132 return errorIllegalChar(.expected_colon, self.index, char);
133 },
134 },
113135 .rhs => switch (char) {
114136 '\t', ' ' => {
115137 // silently ignore horizontal whitespace
......@@ -256,6 +278,10 @@ pub fn next(self: *Tokenizer) ?Token {
256278 self.state = .lhs;
257279 return null;
258280 },
281 .target_space => {
282 const idx = self.index - 1;
283 return errorIllegalChar(.expected_colon, idx, self.bytes[idx]);
284 },
259285 .prereq_quote => {
260286 return errorPosition(.incomplete_quoted_prerequisite, start, self.bytes[start..]);
261287 },
......@@ -299,6 +325,7 @@ const State = enum {
299325 target_dollar_sign,
300326 target_colon,
301327 target_colon_reverse_solidus,
328 target_space,
302329 rhs,
303330 rhs_continuation,
304331 rhs_continuation_linefeed,
......@@ -322,6 +349,7 @@ pub const Token = union(enum) {
322349 expected_dollar_sign: IndexAndChar,
323350 continuation_eol: IndexAndChar,
324351 incomplete_escape: IndexAndChar,
352 expected_colon: IndexAndChar,
325353
326354 pub const IndexAndChar = struct {
327355 index: usize,
......@@ -420,6 +448,7 @@ pub const Token = union(enum) {
420448 .expected_dollar_sign,
421449 .continuation_eol,
422450 .incomplete_escape,
451 .expected_colon,
423452 => |index_and_char| {
424453 try writer.writeAll("illegal char ");
425454 try printUnderstandableChar(writer, index_and_char.char);
......@@ -438,6 +467,7 @@ pub const Token = union(enum) {
438467 .expected_dollar_sign => "expecting '$'",
439468 .continuation_eol => "continuation expecting end-of-line",
440469 .incomplete_escape => "incomplete escape",
470 .expected_colon => "expecting ':'",
441471 };
442472 }
443473};
......@@ -545,6 +575,16 @@ test "empty target linefeeds + hspace + continuations" {
545575 , expect);
546576}
547577
578test "empty target + hspace + colon" {
579 const expect = "target = {foo.o}";
580
581 try depTokenizer("foo.o :", expect);
582 try depTokenizer("foo.o\t\t\t:", expect);
583 try depTokenizer("foo.o \t \t :", expect);
584 try depTokenizer("\r\nfoo.o :", expect);
585 try depTokenizer(" foo.o :", expect);
586}
587
548588test "prereq" {
549589 const expect =
550590 \\target = {foo.o}
......@@ -923,9 +963,6 @@ test "error illegal char at position - expecting dollar_sign" {
923963}
924964
925965test "error illegal char at position - invalid target" {
926 try depTokenizer("foo\t.o",
927 \\ERROR: illegal char \x09 at position 3: invalid target
928 );
929966 try depTokenizer("foo\n.o",
930967 \\ERROR: illegal char \x0A at position 3: invalid target
931968 );
......@@ -963,6 +1000,25 @@ test "error prereq - continuation expecting end-of-line" {
9631000 );
9641001}
9651002
1003test "error illegal char at position - expecting colon" {
1004 try depTokenizer("foo\t.o:",
1005 \\target = {foo}
1006 \\ERROR: illegal char '.' at position 4: expecting ':'
1007 );
1008 try depTokenizer("foo .o:",
1009 \\target = {foo}
1010 \\ERROR: illegal char '.' at position 4: expecting ':'
1011 );
1012 try depTokenizer("foo \n.o:",
1013 \\target = {foo}
1014 \\ERROR: illegal char \x0A at position 4: expecting ':'
1015 );
1016 try depTokenizer("foo.o\t\n:",
1017 \\target = {foo.o}
1018 \\ERROR: illegal char \x0A at position 6: expecting ':'
1019 );
1020}
1021
9661022// - tokenize input, emit textual representation, and compare to expect
9671023fn depTokenizer(input: []const u8, expect: []const u8) !void {
9681024 var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);