authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-09-19 00:28:17+03:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-21 15:23:41-04:00
logf27bc79121bd9971c9e3465fdba3cc7f7d6ff864
tree6775832b8913010330caf8be403aabc830908975
parent302e1565237a0f54101e9c50af34039321dfbc29

stage2: DepTokenizer add target resolver

1 files changed, 182 insertions(+), 135 deletions(-)

src-self-hosted/DepTokenizer.zig+182-135
......@@ -25,256 +25,256 @@ pub fn next(self: *Tokenizer) ?Token {
2525 var must_resolve = false;
2626 while (self.index < self.bytes.len) {
2727 const char = self.bytes[self.index];
28 switch (self.state) {
29 .lhs => switch (char) {
30 '\t', '\n', '\r', ' ' => {
31 // silently ignore whitespace
28 switch (self.state) {
29 .lhs => switch (char) {
30 '\t', '\n', '\r', ' ' => {
31 // silently ignore whitespace
3232 self.index += 1;
33 },
34 else => {
33 },
34 else => {
3535 start = self.index;
3636 self.state = .target;
37 },
3837 },
38 },
3939 .target => switch (char) {
40 '\t', '\n', '\r', ' ' => {
40 '\t', '\n', '\r', ' ' => {
4141 return errorIllegalChar(.invalid_target, self.index, char);
42 },
43 '$' => {
42 },
43 '$' => {
4444 self.state = .target_dollar_sign;
4545 self.index += 1;
46 },
47 '\\' => {
46 },
47 '\\' => {
4848 self.state = .target_reverse_solidus;
4949 self.index += 1;
50 },
51 ':' => {
50 },
51 ':' => {
5252 self.state = .target_colon;
5353 self.index += 1;
54 },
55 else => {
54 },
55 else => {
5656 self.index += 1;
57 },
5857 },
58 },
5959 .target_reverse_solidus => switch (char) {
60 '\t', '\n', '\r' => {
60 '\t', '\n', '\r' => {
6161 return errorIllegalChar(.bad_target_escape, self.index, char);
62 },
63 ' ', '#', '\\' => {
62 },
63 ' ', '#', '\\' => {
6464 must_resolve = true;
6565 self.state = .target;
6666 self.index += 1;
67 },
68 '$' => {
67 },
68 '$' => {
6969 self.state = .target_dollar_sign;
7070 self.index += 1;
71 },
72 else => {
71 },
72 else => {
7373 self.state = .target;
7474 self.index += 1;
75 },
7675 },
76 },
7777 .target_dollar_sign => switch (char) {
78 '$' => {
78 '$' => {
7979 must_resolve = true;
8080 self.state = .target;
8181 self.index += 1;
82 },
83 else => {
82 },
83 else => {
8484 return errorIllegalChar(.expected_dollar_sign, self.index, char);
85 },
8685 },
86 },
8787 .target_colon => switch (char) {
88 '\n', '\r' => {
89 const bytes = self.bytes[start..self.index - 1];
90 if (bytes.len != 0) {
88 '\n', '\r' => {
89 const bytes = self.bytes[start .. self.index - 1];
90 if (bytes.len != 0) {
9191 self.state = .lhs;
9292 return finishTarget(must_resolve, bytes);
93 }
94 // silently ignore null target
93 }
94 // silently ignore null target
9595 self.state = .lhs;
96 },
97 '\\' => {
96 },
97 '\\' => {
9898 self.state = .target_colon_reverse_solidus;
9999 self.index += 1;
100 },
101 else => {
102 const bytes = self.bytes[start..self.index - 1];
103 if (bytes.len != 0) {
100 },
101 else => {
102 const bytes = self.bytes[start .. self.index - 1];
103 if (bytes.len != 0) {
104104 self.state = .rhs;
105105 return finishTarget(must_resolve, bytes);
106 }
107 // silently ignore null target
106 }
107 // silently ignore null target
108108 self.state = .lhs;
109 },
110109 },
110 },
111111 .target_colon_reverse_solidus => switch (char) {
112 '\n', '\r' => {
112 '\n', '\r' => {
113113 const bytes = self.bytes[start .. self.index - 2];
114 if (bytes.len != 0) {
114 if (bytes.len != 0) {
115115 self.state = .lhs;
116116 return finishTarget(must_resolve, bytes);
117 }
118 // silently ignore null target
117 }
118 // silently ignore null target
119119 self.state = .lhs;
120 },
121 else => {
120 },
121 else => {
122122 self.state = .target;
123 },
124123 },
125 .rhs => switch (char) {
126 '\t', ' ' => {
127 // silently ignore horizontal whitespace
124 },
125 .rhs => switch (char) {
126 '\t', ' ' => {
127 // silently ignore horizontal whitespace
128128 self.index += 1;
129 },
130 '\n', '\r' => {
129 },
130 '\n', '\r' => {
131131 self.state = .lhs;
132 },
133 '\\' => {
132 },
133 '\\' => {
134134 self.state = .rhs_continuation;
135135 self.index += 1;
136 },
137 '"' => {
136 },
137 '"' => {
138138 self.state = .prereq_quote;
139139 self.index += 1;
140140 start = self.index;
141 },
142 else => {
141 },
142 else => {
143143 start = self.index;
144144 self.state = .prereq;
145 },
146145 },
147 .rhs_continuation => switch (char) {
148 '\n' => {
146 },
147 .rhs_continuation => switch (char) {
148 '\n' => {
149149 self.state = .rhs;
150150 self.index += 1;
151 },
152 '\r' => {
151 },
152 '\r' => {
153153 self.state = .rhs_continuation_linefeed;
154154 self.index += 1;
155 },
156 else => {
155 },
156 else => {
157157 return errorIllegalChar(.continuation_eol, self.index, char);
158 },
159158 },
160 .rhs_continuation_linefeed => switch (char) {
161 '\n' => {
159 },
160 .rhs_continuation_linefeed => switch (char) {
161 '\n' => {
162162 self.state = .rhs;
163163 self.index += 1;
164 },
165 else => {
164 },
165 else => {
166166 return errorIllegalChar(.continuation_eol, self.index, char);
167 },
168167 },
168 },
169169 .prereq_quote => switch (char) {
170 '"' => {
171 self.index += 1;
170 '"' => {
171 self.index += 1;
172172 self.state = .rhs;
173173 return Token{ .prereq = self.bytes[start .. self.index - 1] };
174 },
175 else => {
174 },
175 else => {
176176 self.index += 1;
177 },
178177 },
178 },
179179 .prereq => switch (char) {
180 '\t', ' ' => {
180 '\t', ' ' => {
181181 self.state = .rhs;
182182 return Token{ .prereq = self.bytes[start..self.index] };
183 },
184 '\n', '\r' => {
183 },
184 '\n', '\r' => {
185185 self.state = .lhs;
186186 return Token{ .prereq = self.bytes[start..self.index] };
187 },
188 '\\' => {
187 },
188 '\\' => {
189189 self.state = .prereq_continuation;
190190 self.index += 1;
191 },
192 else => {
191 },
192 else => {
193193 self.index += 1;
194 },
195194 },
195 },
196196 .prereq_continuation => switch (char) {
197 '\n' => {
198 self.index += 1;
197 '\n' => {
198 self.index += 1;
199199 self.state = .rhs;
200200 return Token{ .prereq = self.bytes[start .. self.index - 2] };
201 },
202 '\r' => {
201 },
202 '\r' => {
203203 self.state = .prereq_continuation_linefeed;
204204 self.index += 1;
205 },
206 else => {
207 // not continuation
205 },
206 else => {
207 // not continuation
208208 self.state = .prereq;
209209 self.index += 1;
210 },
211210 },
211 },
212212 .prereq_continuation_linefeed => switch (char) {
213 '\n' => {
214 self.index += 1;
213 '\n' => {
214 self.index += 1;
215215 self.state = .rhs;
216216 return Token{ .prereq = self.bytes[start .. self.index - 1] };
217 },
218 else => {
217 },
218 else => {
219219 return errorIllegalChar(.continuation_eol, self.index, char);
220 },
221220 },
222 }
221 },
222 }
223223 } else {
224 switch (self.state) {
225 .lhs,
226 .rhs,
227 .rhs_continuation,
228 .rhs_continuation_linefeed,
224 switch (self.state) {
225 .lhs,
226 .rhs,
227 .rhs_continuation,
228 .rhs_continuation_linefeed,
229229 => return null,
230230 .target => {
231231 return Token{ .incomplete_target = self.bytes[start..] };
232 },
233 .target_reverse_solidus,
234 .target_dollar_sign,
235 => {
232 },
233 .target_reverse_solidus,
234 .target_dollar_sign,
235 => {
236236 const idx = self.index - 1;
237237 return errorIllegalChar(.incomplete_escape, idx, self.bytes[idx]);
238 },
238 },
239239 .target_colon => {
240 const bytes = self.bytes[start.. self.index - 1];
241 if (bytes.len != 0) {
242 self.index += 1;
240 const bytes = self.bytes[start .. self.index - 1];
241 if (bytes.len != 0) {
242 self.index += 1;
243243 self.state = .rhs;
244244 return finishTarget(must_resolve, bytes);
245 }
246 // silently ignore null target
245 }
246 // silently ignore null target
247247 self.state = .lhs;
248248 return null;
249 },
249 },
250250 .target_colon_reverse_solidus => {
251 const bytes = self.bytes[start..self.index - 2];
252 if (bytes.len != 0) {
253 self.index += 1;
251 const bytes = self.bytes[start .. self.index - 2];
252 if (bytes.len != 0) {
253 self.index += 1;
254254 self.state = .rhs;
255255 return finishTarget(must_resolve, bytes);
256 }
257 // silently ignore null target
256 }
257 // silently ignore null target
258258 self.state = .lhs;
259259 return null;
260 },
260 },
261261 .prereq_quote => {
262262 return Token{ .incomplete_quoted_prerequisite = self.bytes[start..] };
263 },
263 },
264264 .prereq => {
265265 self.state = .lhs;
266266 return Token{ .prereq = self.bytes[start..] };
267 },
267 },
268268 .prereq_continuation => {
269269 self.state = .lhs;
270 return Token{ .prereq = self.bytes[start.. self.index - 1] };
271 },
270 return Token{ .prereq = self.bytes[start .. self.index - 1] };
271 },
272272 .prereq_continuation_linefeed => {
273273 self.state = .lhs;
274 return Token{ .prereq = self.bytes[start.. self.index - 2] };
275 },
274 return Token{ .prereq = self.bytes[start .. self.index - 2] };
275 },
276 }
276277 }
277}
278278 unreachable;
279279}
280280
......@@ -321,6 +321,46 @@ pub const Token = union(enum) {
321321 index: usize,
322322 char: u8,
323323 };
324
325 /// Resolve escapes in target. Only valid with .target_must_resolve.
326 pub fn resolve(self: Token, buf: *std.ArrayList(u8)) std.mem.Allocator.Error!void {
327 const bytes = self.target_must_resolve; // resolve called on incorrect token
328
329 try buf.ensureCapacity(bytes.len); // cannot be longer than the unescaped string
330 var state: enum { start, escape, dollar } = .start;
331 for (bytes) |c| {
332 switch (state) {
333 .start => {
334 switch (c) {
335 '\\' => state = .escape,
336 '$' => state = .dollar,
337 else => buf.appendAssumeCapacity(c),
338 }
339 },
340 .escape => {
341 switch (c) {
342 ' ', '#', '\\' => {},
343 '$' => {
344 buf.appendAssumeCapacity('\\');
345 state = .dollar;
346 continue;
347 },
348 else => buf.appendAssumeCapacity('\\'),
349 }
350 buf.appendAssumeCapacity(c);
351 state = .start;
352 },
353 .dollar => {
354 buf.appendAssumeCapacity('$');
355 switch (c) {
356 '$' => {},
357 else => buf.appendAssumeCapacity(c),
358 }
359 state = .start;
360 },
361 }
362 }
363 }
324364};
325365
326366test "empty file" {
......@@ -807,20 +847,27 @@ fn depTokenizer(input: []const u8, expect: []const u8) !void {
807847
808848 var it = Tokenizer.init(arena, input);
809849 var buffer = try std.ArrayListSentineled(u8, 0).initSize(arena, 0);
850 var resolve_buf = std.ArrayList(u8).init(arena);
810851 var i: usize = 0;
811852 while (it.next()) |token| {
812853 if (i != 0) try buffer.appendSlice("\n");
813854 switch (token) {
814855 .target, .prereq => |bytes| {
815856 try buffer.appendSlice(@tagName(token));
816 try buffer.appendSlice(" = {");
857 try buffer.appendSlice(" = {");
817858 for (bytes) |b| {
818 try buffer.append(printable_char_tab[b]);
819 }
820 try buffer.appendSlice("}");
859 try buffer.append(printable_char_tab[b]);
860 }
861 try buffer.appendSlice("}");
821862 },
822863 .target_must_resolve => {
823 @panic("TODO");
864 try buffer.appendSlice("target = {");
865 try token.resolve(&resolve_buf);
866 for (resolve_buf.items) |b| {
867 try buffer.append(printable_char_tab[b]);
868 }
869 resolve_buf.items.len = 0;
870 try buffer.appendSlice("}");
824871 },
825872 else => {
826873 @panic("TODO");