1const std = @import("std");
2const Allocator = std.mem.Allocator;
3const utils = @import("utils.zig");
4const UncheckedSliceWriter = utils.UncheckedSliceWriter;
5
6pub const ParseLineCommandsResult = struct {
7 result: []u8,
8 mappings: SourceMappings,
9};
10
11const CurrentMapping = struct {
12 line_num: usize = 1,
13 filename: std.ArrayList(u8) = .empty,
14 pending: bool = true,
15 ignore_contents: bool = false,
16};
17
18pub const ParseAndRemoveLineCommandsOptions = struct {
19 initial_filename: ?[]const u8 = null,
20};
21
22/// Parses and removes #line commands as well as all source code that is within a file
23/// with .c or .h extensions.
24///
25/// > RC treats files with the .c and .h extensions in a special manner. It
26/// > assumes that a file with one of these extensions does not contain
27/// > resources. If a file has the .c or .h file name extension, RC ignores all
28/// > lines in the file except the preprocessor directives. Therefore, to
29/// > include a file that contains resources in another resource script, give
30/// > the file to be included an extension other than .c or .h.
31/// from https://learn.microsoft.com/en-us/windows/win32/menurc/preprocessor-directives
32///
33/// Returns a slice of `buf` with the aforementioned stuff removed as well as a mapping
34/// between the lines and their corresponding lines in their original files.
35///
36/// `buf` must be at least as long as `source`
37/// In-place transformation is supported (i.e. `source` and `buf` can be the same slice)
38///
39/// If `options.initial_filename` is provided, that filename is guaranteed to be
40/// within the `mappings.files` table and `root_filename_offset` will be set appropriately.
41pub fn parseAndRemoveLineCommands(allocator: Allocator, source: []const u8, buf: []u8, options: ParseAndRemoveLineCommandsOptions) error{ OutOfMemory, InvalidLineCommand, LineNumberOverflow }!ParseLineCommandsResult {
42 var parse_result = ParseLineCommandsResult{
43 .result = undefined,
44 .mappings = .{},
45 };
46 errdefer parse_result.mappings.deinit(allocator);
47
48 var current_mapping: CurrentMapping = .{};
49 defer current_mapping.filename.deinit(allocator);
50
51 if (options.initial_filename) |initial_filename| {
52 try current_mapping.filename.appendSlice(allocator, initial_filename);
53 parse_result.mappings.root_filename_offset = try parse_result.mappings.files.put(allocator, initial_filename);
54 }
55
56 // This implementation attempts to be comment and string aware in order
57 // to avoid errant #line <num> "<filename>" within multiline comments
58 // leading to problems in the source mapping after comments are removed,
59 // but it is not a perfect implementation (intentionally).
60 //
61 // The current implementation does not handle cases like
62 // /* foo */ #line ...
63 // #line ... // foo
64 // #line ... /* foo ...
65 // etc
66 //
67 // (the first example will not be recognized as a #line command, the second
68 // and third will error with InvalidLineCommand)
69 //
70 // This is fine, though, since #line commands are generated by the
71 // preprocessor so in normal circumstances they will be well-formed and
72 // consistent. The only realistic way the imperfect implementation could
73 // affect a 'real' use-case would be someone taking the output of a
74 // preprocessor, editing it manually to add comments before/after #line
75 // commands, and then running it through resinator with /:no-preprocess.
76
77 std.debug.assert(buf.len >= source.len);
78 var result = UncheckedSliceWriter{ .slice = buf };
79 const State = enum {
80 line_start,
81 preprocessor,
82 non_preprocessor,
83 forward_slash,
84 line_comment,
85 multiline_comment,
86 multiline_comment_end,
87 single_quoted,
88 single_quoted_escape,
89 double_quoted,
90 double_quoted_escape,
91 };
92 var state: State = .line_start;
93 var index: usize = 0;
94 var pending_start: ?usize = null;
95 var preprocessor_start: usize = 0;
96 var line_number: usize = 1;
97 while (index < source.len) : (index += 1) {
98 var c = source[index];
99 state: switch (state) {
100 .line_start => switch (c) {
101 '#' => {
102 preprocessor_start = index;
103 state = .preprocessor;
104 if (pending_start == null) {
105 pending_start = index;
106 }
107 },
108 '\r', '\n' => {
109 const is_crlf = formsLineEndingPair(source, c, index + 1);
110 if (!current_mapping.ignore_contents) {
111 try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
112
113 result.write(c);
114 if (is_crlf) result.write(source[index + 1]);
115 line_number += 1;
116 }
117 if (is_crlf) index += 1;
118 pending_start = null;
119 },
120 ' ', '\t', '\x0b', '\x0c' => {
121 if (pending_start == null) {
122 pending_start = index;
123 }
124 },
125 '/' => {
126 if (!current_mapping.ignore_contents) {
127 result.writeSlice(source[pending_start orelse index .. index + 1]);
128 pending_start = null;
129 }
130 state = .forward_slash;
131 },
132 '\'' => {
133 if (!current_mapping.ignore_contents) {
134 result.writeSlice(source[pending_start orelse index .. index + 1]);
135 pending_start = null;
136 }
137 state = .single_quoted;
138 },
139 '"' => {
140 if (!current_mapping.ignore_contents) {
141 result.writeSlice(source[pending_start orelse index .. index + 1]);
142 pending_start = null;
143 }
144 state = .double_quoted;
145 },
146 else => {
147 state = .non_preprocessor;
148 if (pending_start != null) {
149 if (!current_mapping.ignore_contents) {
150 result.writeSlice(source[pending_start.? .. index + 1]);
151 }
152 pending_start = null;
153 continue;
154 }
155 if (!current_mapping.ignore_contents) {
156 result.write(c);
157 }
158 },
159 },
160 .forward_slash => switch (c) {
161 '\r', '\n' => {
162 const is_crlf = formsLineEndingPair(source, c, index + 1);
163 if (!current_mapping.ignore_contents) {
164 try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
165
166 result.write(c);
167 if (is_crlf) result.write(source[index + 1]);
168 line_number += 1;
169 }
170 if (is_crlf) index += 1;
171 state = .line_start;
172 pending_start = null;
173 },
174 '/' => {
175 if (!current_mapping.ignore_contents) {
176 result.write(c);
177 }
178 state = .line_comment;
179 },
180 '*' => {
181 if (!current_mapping.ignore_contents) {
182 result.write(c);
183 }
184 state = .multiline_comment;
185 },
186 else => {
187 if (!current_mapping.ignore_contents) {
188 result.write(c);
189 }
190 state = .non_preprocessor;
191 },
192 },
193 .line_comment => switch (c) {
194 '\r', '\n' => {
195 const is_crlf = formsLineEndingPair(source, c, index + 1);
196 if (!current_mapping.ignore_contents) {
197 try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
198
199 result.write(c);
200 if (is_crlf) result.write(source[index + 1]);
201 line_number += 1;
202 }
203 if (is_crlf) index += 1;
204 state = .line_start;
205 pending_start = null;
206 },
207 else => {
208 if (!current_mapping.ignore_contents) {
209 result.write(c);
210 }
211 },
212 },
213 .multiline_comment => switch (c) {
214 '\r', '\n' => {
215 const is_crlf = formsLineEndingPair(source, c, index + 1);
216 if (!current_mapping.ignore_contents) {
217 try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
218
219 result.write(c);
220 if (is_crlf) result.write(source[index + 1]);
221 line_number += 1;
222 }
223 if (is_crlf) index += 1;
224 pending_start = null;
225 },
226 '*' => {
227 if (!current_mapping.ignore_contents) {
228 result.write(c);
229 }
230 state = .multiline_comment_end;
231 },
232 else => {
233 if (!current_mapping.ignore_contents) {
234 result.write(c);
235 }
236 },
237 },
238 .multiline_comment_end => switch (c) {
239 '\r', '\n' => {
240 const is_crlf = formsLineEndingPair(source, c, index + 1);
241 if (!current_mapping.ignore_contents) {
242 try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
243
244 result.write(c);
245 if (is_crlf) result.write(source[index + 1]);
246 line_number += 1;
247 }
248 if (is_crlf) index += 1;
249 state = .multiline_comment;
250 pending_start = null;
251 },
252 '/' => {
253 if (!current_mapping.ignore_contents) {
254 result.write(c);
255 }
256 state = .non_preprocessor;
257 },
258 '*' => {
259 if (!current_mapping.ignore_contents) {
260 result.write(c);
261 }
262 // stay in multiline_comment_end state
263 },
264 else => {
265 if (!current_mapping.ignore_contents) {
266 result.write(c);
267 }
268 state = .multiline_comment;
269 },
270 },
271 .single_quoted => switch (c) {
272 '\r', '\n' => {
273 const is_crlf = formsLineEndingPair(source, c, index + 1);
274 if (!current_mapping.ignore_contents) {
275 try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
276
277 result.write(c);
278 if (is_crlf) result.write(source[index + 1]);
279 line_number += 1;
280 }
281 if (is_crlf) index += 1;
282 state = .line_start;
283 pending_start = null;
284 },
285 '\\' => {
286 if (!current_mapping.ignore_contents) {
287 result.write(c);
288 }
289 state = .single_quoted_escape;
290 },
291 '\'' => {
292 if (!current_mapping.ignore_contents) {
293 result.write(c);
294 }
295 state = .non_preprocessor;
296 },
297 else => {
298 if (!current_mapping.ignore_contents) {
299 result.write(c);
300 }
301 },
302 },
303 .single_quoted_escape => switch (c) {
304 '\r', '\n' => {
305 const is_crlf = formsLineEndingPair(source, c, index + 1);
306 if (!current_mapping.ignore_contents) {
307 try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
308
309 result.write(c);
310 if (is_crlf) result.write(source[index + 1]);
311 line_number += 1;
312 }
313 if (is_crlf) index += 1;
314 state = .line_start;
315 pending_start = null;
316 },
317 else => {
318 if (!current_mapping.ignore_contents) {
319 result.write(c);
320 }
321 state = .single_quoted;
322 },
323 },
324 .double_quoted => switch (c) {
325 '\r', '\n' => {
326 const is_crlf = formsLineEndingPair(source, c, index + 1);
327 if (!current_mapping.ignore_contents) {
328 try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
329
330 result.write(c);
331 if (is_crlf) result.write(source[index + 1]);
332 line_number += 1;
333 }
334 if (is_crlf) index += 1;
335 state = .line_start;
336 pending_start = null;
337 },
338 '\\' => {
339 if (!current_mapping.ignore_contents) {
340 result.write(c);
341 }
342 state = .double_quoted_escape;
343 },
344 '"' => {
345 if (!current_mapping.ignore_contents) {
346 result.write(c);
347 }
348 state = .non_preprocessor;
349 },
350 else => {
351 if (!current_mapping.ignore_contents) {
352 result.write(c);
353 }
354 },
355 },
356 .double_quoted_escape => switch (c) {
357 '\r', '\n' => {
358 const is_crlf = formsLineEndingPair(source, c, index + 1);
359 if (!current_mapping.ignore_contents) {
360 try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
361
362 result.write(c);
363 if (is_crlf) result.write(source[index + 1]);
364 line_number += 1;
365 }
366 if (is_crlf) index += 1;
367 state = .line_start;
368 pending_start = null;
369 },
370 else => {
371 if (!current_mapping.ignore_contents) {
372 result.write(c);
373 }
374 state = .double_quoted;
375 },
376 },
377 .preprocessor => switch (c) {
378 '\r', '\n' => {
379 // Now that we have the full line we can decide what to do with it
380 const preprocessor_str = source[preprocessor_start..index];
381 if (std.mem.startsWith(u8, preprocessor_str, "#line")) {
382 try handleLineCommand(allocator, preprocessor_str, &current_mapping);
383 const is_crlf = formsLineEndingPair(source, c, index + 1);
384 if (is_crlf) index += 1;
385 state = .line_start;
386 pending_start = null;
387 } else {
388 // Backtrack and reparse the line in the non_preprocessor state,
389 // since it's possible that this line contains a multiline comment
390 // start, etc.
391 state = .non_preprocessor;
392 index = pending_start.?;
393 pending_start = null;
394 // TODO: This is a hacky way to implement this, c needs to be
395 // updated since we're using continue :state here
396 c = source[index];
397 // continue to avoid the index += 1 of the while loop
398 continue :state .non_preprocessor;
399 }
400 },
401 else => {},
402 },
403 .non_preprocessor => switch (c) {
404 '\r', '\n' => {
405 const is_crlf = formsLineEndingPair(source, c, index + 1);
406 if (!current_mapping.ignore_contents) {
407 try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
408
409 result.write(c);
410 if (is_crlf) result.write(source[index + 1]);
411 line_number += 1;
412 }
413 if (is_crlf) index += 1;
414 state = .line_start;
415 pending_start = null;
416 },
417 '/' => {
418 if (!current_mapping.ignore_contents) {
419 result.write(c);
420 }
421 state = .forward_slash;
422 },
423 '\'' => {
424 if (!current_mapping.ignore_contents) {
425 result.write(c);
426 }
427 state = .single_quoted;
428 },
429 '"' => {
430 if (!current_mapping.ignore_contents) {
431 result.write(c);
432 }
433 state = .double_quoted;
434 },
435 else => {
436 if (!current_mapping.ignore_contents) {
437 result.write(c);
438 }
439 },
440 },
441 }
442 } else {
443 switch (state) {
444 .line_start => {},
445 .forward_slash,
446 .line_comment,
447 .multiline_comment,
448 .multiline_comment_end,
449 .single_quoted,
450 .single_quoted_escape,
451 .double_quoted,
452 .double_quoted_escape,
453 .non_preprocessor,
454 => {
455 try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
456 },
457 .preprocessor => {
458 // Now that we have the full line we can decide what to do with it
459 const preprocessor_str = source[preprocessor_start..index];
460 if (std.mem.startsWith(u8, preprocessor_str, "#line")) {
461 try handleLineCommand(allocator, preprocessor_str, &current_mapping);
462 } else {
463 try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
464 if (!current_mapping.ignore_contents) {
465 result.writeSlice(source[pending_start.?..index]);
466 }
467 }
468 },
469 }
470 }
471
472 parse_result.result = result.getWritten();
473
474 // Remove whitespace from the end of the result. This avoids issues when the
475 // preprocessor adds a newline to the end of the file, since then the
476 // post-preprocessed source could have more lines than the corresponding input source and
477 // the inserted line can't be mapped to any lines in the original file.
478 // There's no way that whitespace at the end of a file can affect the parsing
479 // of the RC script so this is okay to do unconditionally.
480 // TODO: There might be a better way around this
481 while (parse_result.result.len > 0 and std.ascii.isWhitespace(parse_result.result[parse_result.result.len - 1])) {
482 parse_result.result.len -= 1;
483 }
484
485 // If there have been no line mappings at all, then we're dealing with an empty file.
486 // In this case, we want to fake a line mapping just so that we return something
487 // that is useable in the same way that a non-empty mapping would be.
488 if (parse_result.mappings.sources.root == null) {
489 try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
490 }
491
492 return parse_result;
493}
494
495/// Note: This should function the same as lex.LineHandler.currentIndexFormsLineEndingPair
496pub fn formsLineEndingPair(source: []const u8, line_ending: u8, next_index: usize) bool {
497 if (next_index >= source.len) return false;
498
499 const next_ending = source[next_index];
500 return utils.isLineEndingPair(line_ending, next_ending);
501}
502
503pub fn handleLineEnd(allocator: Allocator, post_processed_line_number: usize, mapping: *SourceMappings, current_mapping: *CurrentMapping) !void {
504 const filename_offset = try mapping.files.put(allocator, current_mapping.filename.items);
505
506 try mapping.set(post_processed_line_number, current_mapping.line_num, filename_offset);
507
508 current_mapping.line_num = std.math.add(usize, current_mapping.line_num, 1) catch return error.LineNumberOverflow;
509 current_mapping.pending = false;
510}
511
512// TODO: Might want to provide diagnostics on invalid line commands instead of just returning
513pub fn handleLineCommand(allocator: Allocator, line_command: []const u8, current_mapping: *CurrentMapping) error{ OutOfMemory, InvalidLineCommand }!void {
514 // TODO: Are there other whitespace characters that should be included?
515 var tokenizer = std.mem.tokenizeAny(u8, line_command, " \t");
516 const line_directive = tokenizer.next() orelse return error.InvalidLineCommand; // #line
517 if (!std.mem.eql(u8, line_directive, "#line")) return error.InvalidLineCommand;
518 const linenum_str = tokenizer.next() orelse return error.InvalidLineCommand;
519 const linenum = std.fmt.parseUnsigned(usize, linenum_str, 10) catch return error.InvalidLineCommand;
520 if (linenum == 0) return error.InvalidLineCommand;
521
522 var filename_literal = tokenizer.rest();
523 while (filename_literal.len > 0 and std.ascii.isWhitespace(filename_literal[filename_literal.len - 1])) {
524 filename_literal.len -= 1;
525 }
526 if (filename_literal.len < 2) return error.InvalidLineCommand;
527 const is_quoted = filename_literal[0] == '"' and filename_literal[filename_literal.len - 1] == '"';
528 if (!is_quoted) return error.InvalidLineCommand;
529 const unquoted_filename = filename_literal[1 .. filename_literal.len - 1];
530
531 // Ignore <builtin> and <command line>
532 if (std.mem.eql(u8, unquoted_filename, "<builtin>") or std.mem.eql(u8, unquoted_filename, "<command line>")) return;
533
534 const filename = parseFilename(allocator, unquoted_filename) catch |err| switch (err) {
535 error.OutOfMemory => |e| return e,
536 else => return error.InvalidLineCommand,
537 };
538 defer allocator.free(filename);
539
540 // \x00 bytes in the filename is incompatible with how StringTable works
541 if (std.mem.findScalar(u8, filename, '\x00') != null) return error.InvalidLineCommand;
542
543 current_mapping.line_num = linenum;
544 current_mapping.filename.clearRetainingCapacity();
545 try current_mapping.filename.appendSlice(allocator, filename);
546 current_mapping.pending = true;
547 current_mapping.ignore_contents = std.ascii.endsWithIgnoreCase(filename, ".c") or std.ascii.endsWithIgnoreCase(filename, ".h");
548}
549
550pub fn parseAndRemoveLineCommandsAlloc(allocator: Allocator, source: []const u8, options: ParseAndRemoveLineCommandsOptions) !ParseLineCommandsResult {
551 const buf = try allocator.alloc(u8, source.len);
552 errdefer allocator.free(buf);
553 var result = try parseAndRemoveLineCommands(allocator, source, buf, options);
554 result.result = try allocator.realloc(buf, result.result.len);
555 return result;
556}
557
558/// C-style string parsing with a few caveats:
559/// - The str cannot contain newlines or carriage returns
560/// - Hex and octal escape are limited to u8
561/// - No handling/support for L, u, or U prefixed strings
562/// - The start and end double quotes should be omitted from the `str`
563/// - Other than the above, does not assume any validity of the strings (i.e. there
564/// may be unescaped double quotes within the str) and will return error.InvalidString
565/// on any problems found.
566///
567/// The result is a UTF-8 encoded string.
568fn parseFilename(allocator: Allocator, str: []const u8) error{ OutOfMemory, InvalidString }![]u8 {
569 const State = enum {
570 string,
571 escape,
572 escape_hex,
573 escape_octal,
574 escape_u,
575 };
576
577 var filename = try std.ArrayList(u8).initCapacity(allocator, str.len);
578 errdefer filename.deinit(allocator);
579 var state: State = .string;
580 var index: usize = 0;
581 var escape_len: usize = undefined;
582 var escape_val: u64 = undefined;
583 var escape_expected_len: u8 = undefined;
584 while (index < str.len) : (index += 1) {
585 const c = str[index];
586 switch (state) {
587 .string => switch (c) {
588 '\\' => state = .escape,
589 '"' => return error.InvalidString,
590 else => filename.appendAssumeCapacity(c),
591 },
592 .escape => switch (c) {
593 '\'', '"', '\\', '?', 'n', 'r', 't', 'a', 'b', 'e', 'f', 'v' => {
594 const escaped_c = switch (c) {
595 '\'', '"', '\\', '?' => c,
596 'n' => '\n',
597 'r' => '\r',
598 't' => '\t',
599 'a' => '\x07',
600 'b' => '\x08',
601 'e' => '\x1b', // non-standard
602 'f' => '\x0c',
603 'v' => '\x0b',
604 else => unreachable,
605 };
606 filename.appendAssumeCapacity(escaped_c);
607 state = .string;
608 },
609 'x' => {
610 escape_val = 0;
611 escape_len = 0;
612 state = .escape_hex;
613 },
614 '0'...'7' => {
615 escape_val = std.fmt.charToDigit(c, 8) catch unreachable;
616 escape_len = 1;
617 state = .escape_octal;
618 },
619 'u' => {
620 escape_val = 0;
621 escape_len = 0;
622 state = .escape_u;
623 escape_expected_len = 4;
624 },
625 'U' => {
626 escape_val = 0;
627 escape_len = 0;
628 state = .escape_u;
629 escape_expected_len = 8;
630 },
631 else => return error.InvalidString,
632 },
633 .escape_hex => switch (c) {
634 '0'...'9', 'a'...'f', 'A'...'F' => {
635 const digit = std.fmt.charToDigit(c, 16) catch unreachable;
636 if (escape_val != 0) escape_val = std.math.mul(u8, @as(u8, @intCast(escape_val)), 16) catch return error.InvalidString;
637 escape_val = std.math.add(u8, @as(u8, @intCast(escape_val)), digit) catch return error.InvalidString;
638 escape_len += 1;
639 if (escape_len == 2) {
640 filename.appendAssumeCapacity(@intCast(escape_val));
641 state = .string;
642 }
643 },
644 else => {
645 if (escape_len == 0) return error.InvalidString;
646 filename.appendAssumeCapacity(@intCast(escape_val));
647 state = .string;
648 index -= 1; // reconsume
649 },
650 },
651 .escape_octal => switch (c) {
652 '0'...'7' => {
653 const digit = std.fmt.charToDigit(c, 8) catch unreachable;
654 if (escape_val != 0) escape_val = std.math.mul(u8, @as(u8, @intCast(escape_val)), 8) catch return error.InvalidString;
655 escape_val = std.math.add(u8, @as(u8, @intCast(escape_val)), digit) catch return error.InvalidString;
656 escape_len += 1;
657 if (escape_len == 3) {
658 filename.appendAssumeCapacity(@intCast(escape_val));
659 state = .string;
660 }
661 },
662 else => {
663 if (escape_len == 0) return error.InvalidString;
664 filename.appendAssumeCapacity(@intCast(escape_val));
665 state = .string;
666 index -= 1; // reconsume
667 },
668 },
669 .escape_u => switch (c) {
670 '0'...'9', 'a'...'f', 'A'...'F' => {
671 const digit = std.fmt.charToDigit(c, 16) catch unreachable;
672 if (escape_val != 0) escape_val = std.math.mul(u21, @as(u21, @intCast(escape_val)), 16) catch return error.InvalidString;
673 escape_val = std.math.add(u21, @as(u21, @intCast(escape_val)), digit) catch return error.InvalidString;
674 escape_len += 1;
675 if (escape_len == escape_expected_len) {
676 var buf: [4]u8 = undefined;
677 const utf8_len = std.unicode.utf8Encode(@intCast(escape_val), &buf) catch return error.InvalidString;
678 filename.appendSliceAssumeCapacity(buf[0..utf8_len]);
679 state = .string;
680 }
681 },
682 // Requires escape_expected_len valid hex digits
683 else => return error.InvalidString,
684 },
685 }
686 } else {
687 switch (state) {
688 .string => {},
689 .escape, .escape_u => return error.InvalidString,
690 .escape_hex => {
691 if (escape_len == 0) return error.InvalidString;
692 filename.appendAssumeCapacity(@intCast(escape_val));
693 },
694 .escape_octal => {
695 filename.appendAssumeCapacity(@intCast(escape_val));
696 },
697 }
698 }
699
700 return filename.toOwnedSlice(allocator);
701}
702
703fn testParseFilename(expected: []const u8, input: []const u8) !void {
704 const parsed = try parseFilename(std.testing.allocator, input);
705 defer std.testing.allocator.free(parsed);
706
707 return std.testing.expectEqualSlices(u8, expected, parsed);
708}
709
710test parseFilename {
711 try testParseFilename("'\"?\\\t\n\r\x11", "\\'\\\"\\?\\\\\\t\\n\\r\\x11");
712 try testParseFilename("\xABz\x53", "\\xABz\\123");
713 try testParseFilename("\xABCDEF", "\\xABCDEF");
714 try testParseFilename("⚡⚡", "\\u26A1\\U000026A1");
715 try std.testing.expectError(error.InvalidString, parseFilename(std.testing.allocator, "\""));
716 try std.testing.expectError(error.InvalidString, parseFilename(std.testing.allocator, "\\"));
717 try std.testing.expectError(error.InvalidString, parseFilename(std.testing.allocator, "\\u"));
718 try std.testing.expectError(error.InvalidString, parseFilename(std.testing.allocator, "\\U"));
719 try std.testing.expectError(error.InvalidString, parseFilename(std.testing.allocator, "\\x"));
720 try std.testing.expectError(error.InvalidString, parseFilename(std.testing.allocator, "\\xZZ"));
721 try std.testing.expectError(error.InvalidString, parseFilename(std.testing.allocator, "\\777"));
722}
723
724pub const SourceMappings = struct {
725 sources: Sources = .{},
726 files: StringTable = .{},
727 /// The default assumes that the first filename added is the root file.
728 /// The value should be set to the correct offset if that assumption does not hold.
729 root_filename_offset: u32 = 0,
730 source_node_pool: std.heap.MemoryPool(Sources.Node) = .empty,
731 end_line: usize = 0,
732
733 const sourceCompare = struct {
734 fn compare(a: Source, b: Source) std.math.Order {
735 return std.math.order(a.start_line, b.start_line);
736 }
737 }.compare;
738 const Sources = std.Treap(Source, sourceCompare);
739
740 pub const Source = struct {
741 start_line: usize,
742 span: usize = 0,
743 corresponding_start_line: usize,
744 filename_offset: u32,
745 };
746
747 pub fn deinit(self: *SourceMappings, allocator: Allocator) void {
748 self.files.deinit(allocator);
749 self.source_node_pool.deinit(std.heap.page_allocator);
750 }
751
752 /// Find the node that 'contains' the `line`, i.e. the node's start_line is
753 /// >= `line`
754 fn findNode(self: SourceMappings, line: usize) ?*Sources.Node {
755 var node = self.sources.root;
756 var last_gt: ?*Sources.Node = null;
757
758 var search_key: Source = undefined;
759 search_key.start_line = line;
760 while (node) |current| {
761 const order = sourceCompare(search_key, current.key);
762 if (order == .eq) break;
763 if (order == .gt) last_gt = current;
764
765 node = current.children[@intFromBool(order == .gt)] orelse {
766 // Regardless of the current order, last_gt will contain the
767 // the node we want to return.
768 //
769 // If search key is > current node's key, then last_gt will be
770 // current which we now know is the closest node that is <=
771 // the search key.
772 //
773 //
774 // If the key is < current node's key, we want to jump back to the
775 // node that the search key was most recently greater than.
776 // This is necessary for scenarios like (where the search key is 2):
777 //
778 // 1
779 // \
780 // 6
781 // /
782 // 3
783 //
784 // In this example, we'll get down to the '3' node but ultimately want
785 // to return the '1' node.
786 //
787 // Note: If we've never seen a key that the search key is greater than,
788 // then we know that there's no valid node, so last_gt will be null.
789 return last_gt;
790 };
791 }
792
793 return node;
794 }
795
796 /// Note: `line_num` and `corresponding_line_num` start at 1
797 pub fn set(self: *SourceMappings, line_num: usize, corresponding_line_num: usize, filename_offset: u32) !void {
798 const maybe_node = self.findNode(line_num);
799
800 const need_new_node = need_new_node: {
801 if (maybe_node) |node| {
802 if (node.key.filename_offset != filename_offset) {
803 break :need_new_node true;
804 }
805 // TODO: These use i65 to avoid truncation when any of the line number values
806 // use all 64 bits of the usize. In reality, line numbers can't really
807 // get that large so limiting the line number and using a smaller iX
808 // type here might be a better solution.
809 const exist_delta = @as(i65, @intCast(node.key.corresponding_start_line)) - @as(i65, @intCast(node.key.start_line));
810 const cur_delta = @as(i65, @intCast(corresponding_line_num)) - @as(i65, @intCast(line_num));
811 if (exist_delta != cur_delta) {
812 break :need_new_node true;
813 }
814 break :need_new_node false;
815 }
816 break :need_new_node true;
817 };
818 if (need_new_node) {
819 // spans must not overlap
820 if (maybe_node) |node| {
821 std.debug.assert(node.key.start_line != line_num);
822 }
823
824 const key = Source{
825 .start_line = line_num,
826 .corresponding_start_line = corresponding_line_num,
827 .filename_offset = filename_offset,
828 };
829 var entry = self.sources.getEntryFor(key);
830 var new_node = try self.source_node_pool.create(std.heap.page_allocator);
831 new_node.key = key;
832 entry.set(new_node);
833 }
834 if (line_num > self.end_line) {
835 self.end_line = line_num;
836 }
837 }
838
839 /// Note: `line_num` starts at 1
840 pub fn get(self: SourceMappings, line_num: usize) ?Source {
841 const node = self.findNode(line_num) orelse return null;
842 return node.key;
843 }
844
845 pub const CorrespondingSpan = struct {
846 start_line: usize,
847 end_line: usize,
848 filename_offset: u32,
849 };
850
851 pub fn getCorrespondingSpan(self: SourceMappings, line_num: usize) ?CorrespondingSpan {
852 const source = self.get(line_num) orelse return null;
853 const diff = line_num - source.start_line;
854 const start_line = source.corresponding_start_line + (if (line_num == source.start_line) 0 else source.span + diff);
855 const end_line = start_line + (if (line_num == source.start_line) source.span else 0);
856 return CorrespondingSpan{
857 .start_line = start_line,
858 .end_line = end_line,
859 .filename_offset = source.filename_offset,
860 };
861 }
862
863 pub fn collapse(self: *SourceMappings, line_num: usize, num_following_lines_to_collapse: usize) !void {
864 std.debug.assert(num_following_lines_to_collapse > 0);
865 var node = self.findNode(line_num).?;
866 const span_diff = num_following_lines_to_collapse;
867 if (node.key.start_line != line_num) {
868 const offset = line_num - node.key.start_line;
869 const key = Source{
870 .start_line = line_num,
871 .span = num_following_lines_to_collapse,
872 .corresponding_start_line = node.key.corresponding_start_line + node.key.span + offset,
873 .filename_offset = node.key.filename_offset,
874 };
875 var entry = self.sources.getEntryFor(key);
876 var new_node = try self.source_node_pool.create(std.heap.page_allocator);
877 new_node.key = key;
878 entry.set(new_node);
879 node = new_node;
880 } else {
881 node.key.span += span_diff;
882 }
883
884 // now subtract the span diff from the start line number of all of
885 // the following nodes in order
886 var it = Sources.InorderIterator{ .current = node };
887 // skip past current, but store it
888 var prev = it.next().?;
889 while (it.next()) |inorder_node| {
890 inorder_node.key.start_line -= span_diff;
891
892 // This can only really happen if there are #line commands within
893 // a multiline comment, which should be skipped over.
894 std.debug.assert(prev.key.start_line <= inorder_node.key.start_line);
895 prev = inorder_node;
896 }
897 self.end_line -= span_diff;
898 }
899
900 /// Returns true if the line is from the main/root file (i.e. not a file that has been
901 /// `#include`d).
902 pub fn isRootFile(self: *const SourceMappings, line_num: usize) bool {
903 const source = self.get(line_num) orelse return false;
904 return source.filename_offset == self.root_filename_offset;
905 }
906};
907
908test "SourceMappings collapse" {
909 const allocator = std.testing.allocator;
910
911 var mappings = SourceMappings{};
912 defer mappings.deinit(allocator);
913 const filename_offset = try mappings.files.put(allocator, "test.rc");
914
915 try mappings.set(1, 1, filename_offset);
916 try mappings.set(5, 5, filename_offset);
917
918 try mappings.collapse(2, 2);
919
920 try std.testing.expectEqual(@as(usize, 3), mappings.end_line);
921 const span_1 = mappings.getCorrespondingSpan(1).?;
922 try std.testing.expectEqual(@as(usize, 1), span_1.start_line);
923 try std.testing.expectEqual(@as(usize, 1), span_1.end_line);
924 const span_2 = mappings.getCorrespondingSpan(2).?;
925 try std.testing.expectEqual(@as(usize, 2), span_2.start_line);
926 try std.testing.expectEqual(@as(usize, 4), span_2.end_line);
927 const span_3 = mappings.getCorrespondingSpan(3).?;
928 try std.testing.expectEqual(@as(usize, 5), span_3.start_line);
929 try std.testing.expectEqual(@as(usize, 5), span_3.end_line);
930}
931
932/// Same thing as StringTable in Zig's src/Wasm.zig
933pub const StringTable = struct {
934 data: std.ArrayList(u8) = .empty,
935 map: std.HashMapUnmanaged(u32, void, std.hash_map.StringIndexContext, std.hash_map.default_max_load_percentage) = .empty,
936
937 pub fn deinit(self: *StringTable, allocator: Allocator) void {
938 self.data.deinit(allocator);
939 self.map.deinit(allocator);
940 }
941
942 pub fn put(self: *StringTable, allocator: Allocator, value: []const u8) !u32 {
943 const result = try self.map.getOrPutContextAdapted(
944 allocator,
945 value,
946 std.hash_map.StringIndexAdapter{ .bytes = &self.data },
947 .{ .bytes = &self.data },
948 );
949 if (result.found_existing) {
950 return result.key_ptr.*;
951 }
952
953 try self.data.ensureUnusedCapacity(allocator, value.len + 1);
954 const offset: u32 = @intCast(self.data.items.len);
955
956 self.data.appendSliceAssumeCapacity(value);
957 self.data.appendAssumeCapacity(0);
958
959 result.key_ptr.* = offset;
960
961 return offset;
962 }
963
964 pub fn get(self: StringTable, offset: u32) []const u8 {
965 std.debug.assert(offset < self.data.items.len);
966 return std.mem.sliceTo(@as([*:0]const u8, @ptrCast(self.data.items.ptr + offset)), 0);
967 }
968
969 pub fn getOffset(self: *StringTable, value: []const u8) ?u32 {
970 return self.map.getKeyAdapted(
971 value,
972 std.hash_map.StringIndexAdapter{ .bytes = &self.data },
973 );
974 }
975};
976
977const ExpectedSourceSpan = struct {
978 start_line: usize,
979 end_line: usize,
980 filename: []const u8,
981};
982
983fn testParseAndRemoveLineCommands(
984 expected: []const u8,
985 comptime expected_spans: []const ExpectedSourceSpan,
986 source: []const u8,
987 options: ParseAndRemoveLineCommandsOptions,
988) !void {
989 var results = try parseAndRemoveLineCommandsAlloc(std.testing.allocator, source, options);
990 defer std.testing.allocator.free(results.result);
991 defer results.mappings.deinit(std.testing.allocator);
992
993 try std.testing.expectEqualStrings(expected, results.result);
994
995 expectEqualMappings(expected_spans, results.mappings) catch |err| {
996 std.debug.print("\nexpected mappings:\n", .{});
997 for (expected_spans, 0..) |span, i| {
998 const line_num = i + 1;
999 std.debug.print("{}: {s}:{}-{}\n", .{ line_num, span.filename, span.start_line, span.end_line });
1000 }
1001 std.debug.print("\nactual mappings:\n", .{});
1002 var i: usize = 1;
1003 while (i <= results.mappings.end_line) : (i += 1) {
1004 const span = results.mappings.getCorrespondingSpan(i).?;
1005 const filename = results.mappings.files.get(span.filename_offset);
1006 std.debug.print("{}: {s}:{}-{}\n", .{ i, filename, span.start_line, span.end_line });
1007 }
1008 std.debug.print("\n", .{});
1009 return err;
1010 };
1011}
1012
1013fn expectEqualMappings(expected_spans: []const ExpectedSourceSpan, mappings: SourceMappings) !void {
1014 try std.testing.expectEqual(expected_spans.len, mappings.end_line);
1015 for (expected_spans, 0..) |expected_span, i| {
1016 const line_num = i + 1;
1017 const span = mappings.getCorrespondingSpan(line_num) orelse return error.MissingLineNum;
1018 const filename = mappings.files.get(span.filename_offset);
1019 try std.testing.expectEqual(expected_span.start_line, span.start_line);
1020 try std.testing.expectEqual(expected_span.end_line, span.end_line);
1021 try std.testing.expectEqualStrings(expected_span.filename, filename);
1022 }
1023}
1024
1025test "basic" {
1026 try testParseAndRemoveLineCommands("", &[_]ExpectedSourceSpan{
1027 .{ .start_line = 1, .end_line = 1, .filename = "blah.rc" },
1028 }, "#line 1 \"blah.rc\"", .{});
1029}
1030
1031test "only removes line commands" {
1032 try testParseAndRemoveLineCommands(
1033 \\#pragma code_page(65001)
1034 , &[_]ExpectedSourceSpan{
1035 .{ .start_line = 1, .end_line = 1, .filename = "blah.rc" },
1036 },
1037 \\#line 1 "blah.rc"
1038 \\#pragma code_page(65001)
1039 , .{});
1040}
1041
1042test "whitespace and line endings" {
1043 try testParseAndRemoveLineCommands("", &[_]ExpectedSourceSpan{
1044 .{ .start_line = 1, .end_line = 1, .filename = "blah.rc" },
1045 }, "#line \t 1 \t \"blah.rc\"\r\n", .{});
1046}
1047
1048test "example" {
1049 try testParseAndRemoveLineCommands(
1050 \\
1051 \\included RCDATA {"hello"}
1052 , &[_]ExpectedSourceSpan{
1053 .{ .start_line = 1, .end_line = 1, .filename = "./included.rc" },
1054 .{ .start_line = 2, .end_line = 2, .filename = "./included.rc" },
1055 },
1056 \\#line 1 "rcdata.rc"
1057 \\#line 1 "<built-in>"
1058 \\#line 1 "<built-in>"
1059 \\#line 355 "<built-in>"
1060 \\#line 1 "<command line>"
1061 \\#line 1 "<built-in>"
1062 \\#line 1 "rcdata.rc"
1063 \\#line 1 "./header.h"
1064 \\
1065 \\
1066 \\2 RCDATA {"blah"}
1067 \\
1068 \\
1069 \\#line 1 "./included.rc"
1070 \\
1071 \\included RCDATA {"hello"}
1072 \\#line 7 "./header.h"
1073 \\#line 1 "rcdata.rc"
1074 , .{});
1075}
1076
1077test "CRLF and other line endings" {
1078 try testParseAndRemoveLineCommands(
1079 "hello\r\n#pragma code_page(65001)\r\nworld",
1080 &[_]ExpectedSourceSpan{
1081 .{ .start_line = 1, .end_line = 1, .filename = "crlf.rc" },
1082 .{ .start_line = 2, .end_line = 2, .filename = "crlf.rc" },
1083 .{ .start_line = 3, .end_line = 3, .filename = "crlf.rc" },
1084 },
1085 "#line 1 \"crlf.rc\"\r\n#line 1 \"<built-in>\"\r#line 1 \"crlf.rc\"\n\rhello\r\n#pragma code_page(65001)\r\nworld\r\n",
1086 .{},
1087 );
1088}
1089
1090test "no line commands" {
1091 try testParseAndRemoveLineCommands(
1092 \\1 RCDATA {"blah"}
1093 \\2 RCDATA {"blah"}
1094 , &[_]ExpectedSourceSpan{
1095 .{ .start_line = 1, .end_line = 1, .filename = "blah.rc" },
1096 .{ .start_line = 2, .end_line = 2, .filename = "blah.rc" },
1097 },
1098 \\1 RCDATA {"blah"}
1099 \\2 RCDATA {"blah"}
1100 , .{ .initial_filename = "blah.rc" });
1101}
1102
1103test "in place" {
1104 var mut_source = "#line 1 \"blah.rc\"".*;
1105 var result = try parseAndRemoveLineCommands(std.testing.allocator, &mut_source, &mut_source, .{});
1106 defer result.mappings.deinit(std.testing.allocator);
1107 try std.testing.expectEqualStrings("", result.result);
1108}
1109
1110test "line command within a multiline comment" {
1111 try testParseAndRemoveLineCommands(
1112 \\/*
1113 \\#line 1 "irrelevant.rc"
1114 \\
1115 \\
1116 \\*/
1117 , &[_]ExpectedSourceSpan{
1118 .{ .start_line = 1, .end_line = 1, .filename = "blah.rc" },
1119 .{ .start_line = 2, .end_line = 2, .filename = "blah.rc" },
1120 .{ .start_line = 3, .end_line = 3, .filename = "blah.rc" },
1121 .{ .start_line = 4, .end_line = 4, .filename = "blah.rc" },
1122 .{ .start_line = 5, .end_line = 5, .filename = "blah.rc" },
1123 },
1124 \\/*
1125 \\#line 1 "irrelevant.rc"
1126 \\
1127 \\
1128 \\*/
1129 , .{ .initial_filename = "blah.rc" });
1130
1131 // * but without / directly after
1132 try testParseAndRemoveLineCommands(
1133 \\/** /
1134 \\#line 1 "irrelevant.rc"
1135 \\*/
1136 , &[_]ExpectedSourceSpan{
1137 .{ .start_line = 1, .end_line = 1, .filename = "blah.rc" },
1138 .{ .start_line = 2, .end_line = 2, .filename = "blah.rc" },
1139 .{ .start_line = 3, .end_line = 3, .filename = "blah.rc" },
1140 },
1141 \\/** /
1142 \\#line 1 "irrelevant.rc"
1143 \\*/
1144 , .{ .initial_filename = "blah.rc" });
1145
1146 // /** and **/
1147 try testParseAndRemoveLineCommands(
1148 \\/**
1149 \\#line 1 "irrelevant.rc"
1150 \\**/
1151 \\foo
1152 , &[_]ExpectedSourceSpan{
1153 .{ .start_line = 1, .end_line = 1, .filename = "blah.rc" },
1154 .{ .start_line = 2, .end_line = 2, .filename = "blah.rc" },
1155 .{ .start_line = 3, .end_line = 3, .filename = "blah.rc" },
1156 .{ .start_line = 20, .end_line = 20, .filename = "blah.rc" },
1157 },
1158 \\/**
1159 \\#line 1 "irrelevant.rc"
1160 \\**/
1161 \\#line 20 "blah.rc"
1162 \\foo
1163 , .{ .initial_filename = "blah.rc" });
1164}
1165
1166test "whitespace preservation" {
1167 try testParseAndRemoveLineCommands(
1168 \\ /
1169 \\/
1170 , &[_]ExpectedSourceSpan{
1171 .{ .start_line = 1, .end_line = 1, .filename = "blah.rc" },
1172 .{ .start_line = 2, .end_line = 2, .filename = "blah.rc" },
1173 },
1174 \\ /
1175 \\/
1176 , .{ .initial_filename = "blah.rc" });
1177}
1178
1179test "preprocessor line with a multiline comment after" {
1180 try testParseAndRemoveLineCommands(
1181 \\#pragma test /*
1182 \\
1183 \\*/
1184 , &[_]ExpectedSourceSpan{
1185 .{ .start_line = 1, .end_line = 1, .filename = "blah.rc" },
1186 .{ .start_line = 2, .end_line = 2, .filename = "blah.rc" },
1187 .{ .start_line = 3, .end_line = 3, .filename = "blah.rc" },
1188 },
1189 \\#pragma test /*
1190 \\
1191 \\*/
1192 , .{ .initial_filename = "blah.rc" });
1193}
1194
1195test "comment after line command" {
1196 var mut_source = "#line 1 \"blah.rc\" /*".*;
1197 try std.testing.expectError(error.InvalidLineCommand, parseAndRemoveLineCommands(std.testing.allocator, &mut_source, &mut_source, .{}));
1198}
1199
1200test "line command with 0 as line number" {
1201 var mut_source = "#line 0 \"blah.rc\"".*;
1202 try std.testing.expectError(error.InvalidLineCommand, parseAndRemoveLineCommands(std.testing.allocator, &mut_source, &mut_source, .{}));
1203}
1204
1205test "line number limits" {
1206 // TODO: Avoid usize for line numbers
1207 if (@sizeOf(usize) != 8) return error.SkipZigTest;
1208
1209 // greater than i64 max
1210 try testParseAndRemoveLineCommands(
1211 \\
1212 , &[_]ExpectedSourceSpan{
1213 .{ .start_line = 11111111111111111111, .end_line = 11111111111111111111, .filename = "blah.rc" },
1214 },
1215 \\#line 11111111111111111111 "blah.rc"
1216 , .{ .initial_filename = "blah.rc" });
1217
1218 // equal to u64 max, overflows on line number increment
1219 {
1220 var mut_source = "#line 18446744073709551615 \"blah.rc\"".*;
1221 try std.testing.expectError(error.LineNumberOverflow, parseAndRemoveLineCommands(std.testing.allocator, &mut_source, &mut_source, .{}));
1222 }
1223
1224 // greater than u64 max
1225 {
1226 var mut_source = "#line 18446744073709551616 \"blah.rc\"".*;
1227 try std.testing.expectError(error.InvalidLineCommand, parseAndRemoveLineCommands(std.testing.allocator, &mut_source, &mut_source, .{}));
1228 }
1229}