authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-09-19 00:28:17+03:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-21 15:23:41-04:00
logf27bc79121bd9971c9e3465fdba3cc7f7d6ff864
tree6775832b8913010330caf8be403aabc830908975
parent302e1565237a0f54101e9c50af34039321dfbc29

stage2: DepTokenizer add target resolver

1 files changed, 182 insertions(+), 135 deletions(-)

src-self-hosted/DepTokenizer.zig+182-135
...@@ -25,256 +25,256 @@ pub fn next(self: *Tokenizer) ?Token {...@@ -25,256 +25,256 @@ pub fn next(self: *Tokenizer) ?Token {
25 var must_resolve = false;25 var must_resolve = false;
26 while (self.index < self.bytes.len) {26 while (self.index < self.bytes.len) {
27 const char = self.bytes[self.index];27 const char = self.bytes[self.index];
28 switch (self.state) {28 switch (self.state) {
29 .lhs => switch (char) {29 .lhs => switch (char) {
30 '\t', '\n', '\r', ' ' => {30 '\t', '\n', '\r', ' ' => {
31 // silently ignore whitespace31 // silently ignore whitespace
32 self.index += 1;32 self.index += 1;
33 },33 },
34 else => {34 else => {
35 start = self.index;35 start = self.index;
36 self.state = .target;36 self.state = .target;
37 },
38 },37 },
38 },
39 .target => switch (char) {39 .target => switch (char) {
40 '\t', '\n', '\r', ' ' => {40 '\t', '\n', '\r', ' ' => {
41 return errorIllegalChar(.invalid_target, self.index, char);41 return errorIllegalChar(.invalid_target, self.index, char);
42 },42 },
43 '$' => {43 '$' => {
44 self.state = .target_dollar_sign;44 self.state = .target_dollar_sign;
45 self.index += 1;45 self.index += 1;
46 },46 },
47 '\\' => {47 '\\' => {
48 self.state = .target_reverse_solidus;48 self.state = .target_reverse_solidus;
49 self.index += 1;49 self.index += 1;
50 },50 },
51 ':' => {51 ':' => {
52 self.state = .target_colon;52 self.state = .target_colon;
53 self.index += 1;53 self.index += 1;
54 },54 },
55 else => {55 else => {
56 self.index += 1;56 self.index += 1;
57 },
58 },57 },
58 },
59 .target_reverse_solidus => switch (char) {59 .target_reverse_solidus => switch (char) {
60 '\t', '\n', '\r' => {60 '\t', '\n', '\r' => {
61 return errorIllegalChar(.bad_target_escape, self.index, char);61 return errorIllegalChar(.bad_target_escape, self.index, char);
62 },62 },
63 ' ', '#', '\\' => {63 ' ', '#', '\\' => {
64 must_resolve = true;64 must_resolve = true;
65 self.state = .target;65 self.state = .target;
66 self.index += 1;66 self.index += 1;
67 },67 },
68 '$' => {68 '$' => {
69 self.state = .target_dollar_sign;69 self.state = .target_dollar_sign;
70 self.index += 1;70 self.index += 1;
71 },71 },
72 else => {72 else => {
73 self.state = .target;73 self.state = .target;
74 self.index += 1;74 self.index += 1;
75 },
76 },75 },
76 },
77 .target_dollar_sign => switch (char) {77 .target_dollar_sign => switch (char) {
78 '$' => {78 '$' => {
79 must_resolve = true;79 must_resolve = true;
80 self.state = .target;80 self.state = .target;
81 self.index += 1;81 self.index += 1;
82 },82 },
83 else => {83 else => {
84 return errorIllegalChar(.expected_dollar_sign, self.index, char);84 return errorIllegalChar(.expected_dollar_sign, self.index, char);
85 },
86 },85 },
86 },
87 .target_colon => switch (char) {87 .target_colon => switch (char) {
88 '\n', '\r' => {88 '\n', '\r' => {
89 const bytes = self.bytes[start..self.index - 1];89 const bytes = self.bytes[start .. self.index - 1];
90 if (bytes.len != 0) {90 if (bytes.len != 0) {
91 self.state = .lhs;91 self.state = .lhs;
92 return finishTarget(must_resolve, bytes);92 return finishTarget(must_resolve, bytes);
93 }93 }
94 // silently ignore null target94 // silently ignore null target
95 self.state = .lhs;95 self.state = .lhs;
96 },96 },
97 '\\' => {97 '\\' => {
98 self.state = .target_colon_reverse_solidus;98 self.state = .target_colon_reverse_solidus;
99 self.index += 1;99 self.index += 1;
100 },100 },
101 else => {101 else => {
102 const bytes = self.bytes[start..self.index - 1];102 const bytes = self.bytes[start .. self.index - 1];
103 if (bytes.len != 0) {103 if (bytes.len != 0) {
104 self.state = .rhs;104 self.state = .rhs;
105 return finishTarget(must_resolve, bytes);105 return finishTarget(must_resolve, bytes);
106 }106 }
107 // silently ignore null target107 // silently ignore null target
108 self.state = .lhs;108 self.state = .lhs;
109 },
110 },109 },
110 },
111 .target_colon_reverse_solidus => switch (char) {111 .target_colon_reverse_solidus => switch (char) {
112 '\n', '\r' => {112 '\n', '\r' => {
113 const bytes = self.bytes[start .. self.index - 2];113 const bytes = self.bytes[start .. self.index - 2];
114 if (bytes.len != 0) {114 if (bytes.len != 0) {
115 self.state = .lhs;115 self.state = .lhs;
116 return finishTarget(must_resolve, bytes);116 return finishTarget(must_resolve, bytes);
117 }117 }
118 // silently ignore null target118 // silently ignore null target
119 self.state = .lhs;119 self.state = .lhs;
120 },120 },
121 else => {121 else => {
122 self.state = .target;122 self.state = .target;
123 },
124 },123 },
125 .rhs => switch (char) {124 },
126 '\t', ' ' => {125 .rhs => switch (char) {
127 // silently ignore horizontal whitespace126 '\t', ' ' => {
127 // silently ignore horizontal whitespace
128 self.index += 1;128 self.index += 1;
129 },129 },
130 '\n', '\r' => {130 '\n', '\r' => {
131 self.state = .lhs;131 self.state = .lhs;
132 },132 },
133 '\\' => {133 '\\' => {
134 self.state = .rhs_continuation;134 self.state = .rhs_continuation;
135 self.index += 1;135 self.index += 1;
136 },136 },
137 '"' => {137 '"' => {
138 self.state = .prereq_quote;138 self.state = .prereq_quote;
139 self.index += 1;139 self.index += 1;
140 start = self.index;140 start = self.index;
141 },141 },
142 else => {142 else => {
143 start = self.index;143 start = self.index;
144 self.state = .prereq;144 self.state = .prereq;
145 },
146 },145 },
147 .rhs_continuation => switch (char) {146 },
148 '\n' => {147 .rhs_continuation => switch (char) {
148 '\n' => {
149 self.state = .rhs;149 self.state = .rhs;
150 self.index += 1;150 self.index += 1;
151 },151 },
152 '\r' => {152 '\r' => {
153 self.state = .rhs_continuation_linefeed;153 self.state = .rhs_continuation_linefeed;
154 self.index += 1;154 self.index += 1;
155 },155 },
156 else => {156 else => {
157 return errorIllegalChar(.continuation_eol, self.index, char);157 return errorIllegalChar(.continuation_eol, self.index, char);
158 },
159 },158 },
160 .rhs_continuation_linefeed => switch (char) {159 },
161 '\n' => {160 .rhs_continuation_linefeed => switch (char) {
161 '\n' => {
162 self.state = .rhs;162 self.state = .rhs;
163 self.index += 1;163 self.index += 1;
164 },164 },
165 else => {165 else => {
166 return errorIllegalChar(.continuation_eol, self.index, char);166 return errorIllegalChar(.continuation_eol, self.index, char);
167 },
168 },167 },
168 },
169 .prereq_quote => switch (char) {169 .prereq_quote => switch (char) {
170 '"' => {170 '"' => {
171 self.index += 1;171 self.index += 1;
172 self.state = .rhs;172 self.state = .rhs;
173 return Token{ .prereq = self.bytes[start .. self.index - 1] };173 return Token{ .prereq = self.bytes[start .. self.index - 1] };
174 },174 },
175 else => {175 else => {
176 self.index += 1;176 self.index += 1;
177 },
178 },177 },
178 },
179 .prereq => switch (char) {179 .prereq => switch (char) {
180 '\t', ' ' => {180 '\t', ' ' => {
181 self.state = .rhs;181 self.state = .rhs;
182 return Token{ .prereq = self.bytes[start..self.index] };182 return Token{ .prereq = self.bytes[start..self.index] };
183 },183 },
184 '\n', '\r' => {184 '\n', '\r' => {
185 self.state = .lhs;185 self.state = .lhs;
186 return Token{ .prereq = self.bytes[start..self.index] };186 return Token{ .prereq = self.bytes[start..self.index] };
187 },187 },
188 '\\' => {188 '\\' => {
189 self.state = .prereq_continuation;189 self.state = .prereq_continuation;
190 self.index += 1;190 self.index += 1;
191 },191 },
192 else => {192 else => {
193 self.index += 1;193 self.index += 1;
194 },
195 },194 },
195 },
196 .prereq_continuation => switch (char) {196 .prereq_continuation => switch (char) {
197 '\n' => {197 '\n' => {
198 self.index += 1;198 self.index += 1;
199 self.state = .rhs;199 self.state = .rhs;
200 return Token{ .prereq = self.bytes[start .. self.index - 2] };200 return Token{ .prereq = self.bytes[start .. self.index - 2] };
201 },201 },
202 '\r' => {202 '\r' => {
203 self.state = .prereq_continuation_linefeed;203 self.state = .prereq_continuation_linefeed;
204 self.index += 1;204 self.index += 1;
205 },205 },
206 else => {206 else => {
207 // not continuation207 // not continuation
208 self.state = .prereq;208 self.state = .prereq;
209 self.index += 1;209 self.index += 1;
210 },
211 },210 },
211 },
212 .prereq_continuation_linefeed => switch (char) {212 .prereq_continuation_linefeed => switch (char) {
213 '\n' => {213 '\n' => {
214 self.index += 1;214 self.index += 1;
215 self.state = .rhs;215 self.state = .rhs;
216 return Token{ .prereq = self.bytes[start .. self.index - 1] };216 return Token{ .prereq = self.bytes[start .. self.index - 1] };
217 },217 },
218 else => {218 else => {
219 return errorIllegalChar(.continuation_eol, self.index, char);219 return errorIllegalChar(.continuation_eol, self.index, char);
220 },
221 },220 },
222 }221 },
222 }
223 } else {223 } else {
224 switch (self.state) {224 switch (self.state) {
225 .lhs,225 .lhs,
226 .rhs,226 .rhs,
227 .rhs_continuation,227 .rhs_continuation,
228 .rhs_continuation_linefeed,228 .rhs_continuation_linefeed,
229 => return null,229 => return null,
230 .target => {230 .target => {
231 return Token{ .incomplete_target = self.bytes[start..] };231 return Token{ .incomplete_target = self.bytes[start..] };
232 },232 },
233 .target_reverse_solidus,233 .target_reverse_solidus,
234 .target_dollar_sign,234 .target_dollar_sign,
235 => {235 => {
236 const idx = self.index - 1;236 const idx = self.index - 1;
237 return errorIllegalChar(.incomplete_escape, idx, self.bytes[idx]);237 return errorIllegalChar(.incomplete_escape, idx, self.bytes[idx]);
238 },238 },
239 .target_colon => {239 .target_colon => {
240 const bytes = self.bytes[start.. self.index - 1];240 const bytes = self.bytes[start .. self.index - 1];
241 if (bytes.len != 0) {241 if (bytes.len != 0) {
242 self.index += 1;242 self.index += 1;
243 self.state = .rhs;243 self.state = .rhs;
244 return finishTarget(must_resolve, bytes);244 return finishTarget(must_resolve, bytes);
245 }245 }
246 // silently ignore null target246 // silently ignore null target
247 self.state = .lhs;247 self.state = .lhs;
248 return null;248 return null;
249 },249 },
250 .target_colon_reverse_solidus => {250 .target_colon_reverse_solidus => {
251 const bytes = self.bytes[start..self.index - 2];251 const bytes = self.bytes[start .. self.index - 2];
252 if (bytes.len != 0) {252 if (bytes.len != 0) {
253 self.index += 1;253 self.index += 1;
254 self.state = .rhs;254 self.state = .rhs;
255 return finishTarget(must_resolve, bytes);255 return finishTarget(must_resolve, bytes);
256 }256 }
257 // silently ignore null target257 // silently ignore null target
258 self.state = .lhs;258 self.state = .lhs;
259 return null;259 return null;
260 },260 },
261 .prereq_quote => {261 .prereq_quote => {
262 return Token{ .incomplete_quoted_prerequisite = self.bytes[start..] };262 return Token{ .incomplete_quoted_prerequisite = self.bytes[start..] };
263 },263 },
264 .prereq => {264 .prereq => {
265 self.state = .lhs;265 self.state = .lhs;
266 return Token{ .prereq = self.bytes[start..] };266 return Token{ .prereq = self.bytes[start..] };
267 },267 },
268 .prereq_continuation => {268 .prereq_continuation => {
269 self.state = .lhs;269 self.state = .lhs;
270 return Token{ .prereq = self.bytes[start.. self.index - 1] };270 return Token{ .prereq = self.bytes[start .. self.index - 1] };
271 },271 },
272 .prereq_continuation_linefeed => {272 .prereq_continuation_linefeed => {
273 self.state = .lhs;273 self.state = .lhs;
274 return Token{ .prereq = self.bytes[start.. self.index - 2] };274 return Token{ .prereq = self.bytes[start .. self.index - 2] };
275 },275 },
276 }
276 }277 }
277}
278 unreachable;278 unreachable;
279}279}
280280
...@@ -321,6 +321,46 @@ pub const Token = union(enum) {...@@ -321,6 +321,46 @@ pub const Token = union(enum) {
321 index: usize,321 index: usize,
322 char: u8,322 char: u8,
323 };323 };
324
325 /// Resolve escapes in target. Only valid with .target_must_resolve.
326 pub fn resolve(self: Token, buf: *std.ArrayList(u8)) std.mem.Allocator.Error!void {
327 const bytes = self.target_must_resolve; // resolve called on incorrect token
328
329 try buf.ensureCapacity(bytes.len); // cannot be longer than the unescaped string
330 var state: enum { start, escape, dollar } = .start;
331 for (bytes) |c| {
332 switch (state) {
333 .start => {
334 switch (c) {
335 '\\' => state = .escape,
336 '$' => state = .dollar,
337 else => buf.appendAssumeCapacity(c),
338 }
339 },
340 .escape => {
341 switch (c) {
342 ' ', '#', '\\' => {},
343 '$' => {
344 buf.appendAssumeCapacity('\\');
345 state = .dollar;
346 continue;
347 },
348 else => buf.appendAssumeCapacity('\\'),
349 }
350 buf.appendAssumeCapacity(c);
351 state = .start;
352 },
353 .dollar => {
354 buf.appendAssumeCapacity('$');
355 switch (c) {
356 '$' => {},
357 else => buf.appendAssumeCapacity(c),
358 }
359 state = .start;
360 },
361 }
362 }
363 }
324};364};
325365
326test "empty file" {366test "empty file" {
...@@ -807,20 +847,27 @@ fn depTokenizer(input: []const u8, expect: []const u8) !void {...@@ -807,20 +847,27 @@ fn depTokenizer(input: []const u8, expect: []const u8) !void {
807847
808 var it = Tokenizer.init(arena, input);848 var it = Tokenizer.init(arena, input);
809 var buffer = try std.ArrayListSentineled(u8, 0).initSize(arena, 0);849 var buffer = try std.ArrayListSentineled(u8, 0).initSize(arena, 0);
850 var resolve_buf = std.ArrayList(u8).init(arena);
810 var i: usize = 0;851 var i: usize = 0;
811 while (it.next()) |token| {852 while (it.next()) |token| {
812 if (i != 0) try buffer.appendSlice("\n");853 if (i != 0) try buffer.appendSlice("\n");
813 switch (token) {854 switch (token) {
814 .target, .prereq => |bytes| {855 .target, .prereq => |bytes| {
815 try buffer.appendSlice(@tagName(token));856 try buffer.appendSlice(@tagName(token));
816 try buffer.appendSlice(" = {");857 try buffer.appendSlice(" = {");
817 for (bytes) |b| {858 for (bytes) |b| {
818 try buffer.append(printable_char_tab[b]);859 try buffer.append(printable_char_tab[b]);
819 }860 }
820 try buffer.appendSlice("}");861 try buffer.appendSlice("}");
821 },862 },
822 .target_must_resolve => {863 .target_must_resolve => {
823 @panic("TODO");864 try buffer.appendSlice("target = {");
865 try token.resolve(&resolve_buf);
866 for (resolve_buf.items) |b| {
867 try buffer.append(printable_char_tab[b]);
868 }
869 resolve_buf.items.len = 0;
870 try buffer.appendSlice("}");
824 },871 },
825 else => {872 else => {
826 @panic("TODO");873 @panic("TODO");