1const std = @import("std");
2const Tokenizer = @import("./Tokenizer.zig");
3const Allocator = std.mem.Allocator;
4const Token = Tokenizer.Token;
5const mem = std.mem;
6const assert = std.debug.assert;
7const Path = std.Build.Cache.Path;
8
9test {
10 _ = Tokenizer;
11}
12
13const TokenList = std.MultiArrayList(Token);
14const RawTokenList = std.ArrayList(Token);
15
16const ExpandBuf = std.ArrayList(Token);
17
18const Preprocessor = @This();
19const DefineMap = std.array_hash_map.String(Macro);
20
21const GeneratedTokens = std.ArrayList(u8);
22
23const MacroArgument = []const Token;
24
25pub const Source = struct {
26 pub const generated: Source.Id = std.math.maxInt(usize);
27 pub const Id = usize;
28 id: Id = generated,
29 path: Path,
30 buf: []const u8,
31};
32
33sources: std.array_hash_map.Custom(Path, Source, Path.TableAdapter, false) = .empty,
34
35arena: Allocator,
36io: std.Io,
37include_dir: Path,
38
39top_expansion_buf: ExpandBuf = .empty,
40add_expansion_nl: usize = 0,
41token_buf: RawTokenList = .empty,
42generated_tokens: GeneratedTokens = .empty,
43generated_line: u32 = 1,
44defines: DefineMap = .empty,
45tokens: TokenList = .empty,
46target: *const std.Target,
47
48const Macro = struct {
49 param: []const u8,
50 tokens: []const Token,
51 is_func: bool,
52};
53
54const IfContext = struct {
55 const Backing = u2;
56 const Nesting = enum(Backing) {
57 until_else,
58 until_endif,
59 until_endif_seen_else,
60 };
61
62 const buf_size_bits = @bitSizeOf(Backing) * 256;
63 kind: [buf_size_bits / std.mem.byte_size_in_bits]u8,
64 level: u8,
65
66 fn get(self: *const IfContext) Nesting {
67 return @fromBackingInt(@intCast(std.mem.readPackedInt(Backing, &self.kind, @as(usize, self.level) * 2, .native)));
68 }
69
70 fn set(self: *IfContext, context: Nesting) void {
71 std.mem.writePackedInt(Backing, &self.kind, @as(usize, self.level) * 2, @backingInt(context), .native);
72 }
73
74 fn increment(self: *IfContext) void {
75 self.level += 1;
76 }
77
78 fn decrement(self: *IfContext) void {
79 self.level -= 1;
80 }
81
82 const default: IfContext = .{ .kind = @splat(0xFF), .level = 0 };
83};
84
85fn addToken(pp: *Preprocessor, tok: Token) !void {
86 try pp.tokens.append(pp.arena, tok);
87}
88
89fn addTokenAssumeCapacity(pp: *Preprocessor, tok: Token) void {
90 pp.tokens.appendAssumeCapacity(tok);
91}
92
93fn defineBuiltins(pp: *Preprocessor) !void {
94 var buf: [5]u8 = undefined;
95 var val = std.mem.print(&buf, "{d}", .{pp.target.cTypeByteSize(.longdouble).?}) catch unreachable;
96 try pp.defineBuiltinValue("__SIZEOF_LONG_DOUBLE__", val, .pp_num);
97 val = std.mem.print(&buf, "{d}", .{pp.target.cTypeByteSize(.double).?}) catch unreachable;
98 try pp.defineBuiltinValue("__SIZEOF_DOUBLE__", val, .pp_num);
99
100 if (pp.target.abi.isGnu()) {
101 try pp.defineBuiltinValue("__cdecl", "__attribute__((__cdecl__))", .identifier);
102 }
103
104 const arch = switch (pp.target.cpu.arch) {
105 .aarch64 => "__aarch64__",
106 .x86 => "__i386__",
107 .x86_64 => "__x86_64__",
108 .arm, .thumb => "__arm__",
109 else => return error.ArchitectureNotSupported,
110 };
111 try pp.defineBuiltin(arch);
112}
113
114fn defineBuiltinValue(pp: *Preprocessor, name: []const u8, value: []const u8, id: Token.Id) !void {
115 const start = pp.generated_tokens.items.len;
116 try pp.generated_tokens.appendSlice(pp.arena, value);
117 const end = pp.generated_tokens.items.len;
118
119 const token_list = try pp.arena.alloc(Token, 1);
120 token_list[0] = .{ .source = Source.generated, .id = id, .start = @intCast(start), .end = @intCast(end) };
121 try pp.defines.putNoClobber(pp.arena, name, .{
122 .is_func = false,
123 .param = "",
124 .tokens = token_list,
125 });
126}
127
128fn defineBuiltin(pp: *Preprocessor, name: []const u8) !void {
129 return pp.defines.putNoClobber(pp.arena, name, .{
130 .tokens = &.{},
131 .param = "",
132 .is_func = false,
133 });
134}
135
136pub fn preprocess(pp: *Preprocessor, file_path: Path) !void {
137 const source = try pp.addSourceFromPath(file_path);
138 try pp.preprocessFile(source);
139}
140
141fn preprocessFile(pp: *Preprocessor, src: Source) !void {
142 try pp.defineBuiltins();
143 const eof = try pp.preprocessFileExtra(src);
144 try pp.addToken(eof);
145}
146
147fn preprocessFileExtra(pp: *Preprocessor, src: Source) !Token {
148 var tokenizer: Tokenizer = .init(src.buf, src.id);
149 var if_context: IfContext = .default;
150
151 while (true) {
152 var tok = tokenizer.next();
153 switch (tok.id) {
154 .hash => {
155 const directive = tokenizer.nextNoWS();
156 switch (directive.id) {
157 .keyword_define => try pp.define(&tokenizer),
158 .keyword_if => {
159 if_context.increment();
160 if (try pp.expr(&tokenizer)) {
161 if_context.set(.until_endif);
162 } else {
163 if_context.set(.until_else);
164 try pp.skip(&tokenizer, .until_else);
165 }
166 },
167 .keyword_ifdef => {
168 if_context.increment();
169 const macro_name = pp.expectMacroName(&tokenizer);
170 skipToNl(&tokenizer);
171 if (pp.defines.get(macro_name) != null) {
172 if_context.set(.until_endif);
173 } else {
174 if_context.set(.until_else);
175 try pp.skip(&tokenizer, .until_else);
176 }
177 },
178 .keyword_ifndef => {
179 if_context.increment();
180 const macro_name = pp.expectMacroName(&tokenizer);
181 skipToNl(&tokenizer);
182 if (pp.defines.get(macro_name) == null) {
183 if_context.set(.until_endif);
184 } else {
185 if_context.set(.until_else);
186 try pp.skip(&tokenizer, .until_else);
187 }
188 },
189 .keyword_elif => {
190 assert(if_context.level > 0);
191 switch (if_context.get()) {
192 .until_else => if (try pp.expr(&tokenizer)) {
193 if_context.set(.until_endif);
194 } else {
195 try pp.skip(&tokenizer, .until_else);
196 },
197 .until_endif => try pp.skip(&tokenizer, .until_endif),
198 .until_endif_seen_else => unreachable, //elif after endif
199 }
200 },
201 .keyword_else => {
202 skipToNl(&tokenizer);
203 assert(if_context.level > 0);
204 switch (if_context.get()) {
205 .until_else => if_context.set(.until_endif_seen_else),
206 .until_endif => try pp.skip(&tokenizer, .until_endif),
207 .until_endif_seen_else => unreachable, // else after else
208 }
209 },
210 .keyword_endif => {
211 skipToNl(&tokenizer);
212 assert(if_context.level > 0);
213 if_context.decrement();
214 },
215 .keyword_undef => {
216 const macro_name = tokenizer.nextNoWS();
217 assert(macro_name.id == .identifier);
218 pp.undefineMacro(macro_name);
219 skipToNl(&tokenizer);
220 },
221 .keyword_include => {
222 try pp.include(&tokenizer);
223 continue;
224 },
225 .keyword_defined, .keyword_error => {},
226 else => unreachable,
227 }
228 tok.id = .nl;
229 try pp.addToken(tok);
230 },
231 .whitespace, .nl => try pp.addToken(tok),
232 .eof => {
233 assert(if_context.level == 0);
234 return tok;
235 },
236 else => try pp.expandMacro(&tokenizer, tok),
237 }
238 }
239}
240
241fn include(pp: *Preprocessor, tokenizer: *Tokenizer) anyerror!void {
242 const first = tokenizer.nextNoWS();
243 const src = try findIncludeSource(pp, tokenizer, first);
244
245 _ = try pp.preprocessFileExtra(src);
246 if (pp.tokens.items(.id)[pp.tokens.len - 1] != .nl) {
247 try pp.addToken(.{ .id = .nl, .source = Source.generated });
248 }
249}
250
251fn findIncludeSource(
252 pp: *Preprocessor,
253 tokenizer: *Tokenizer,
254 first: Token,
255) !Source {
256 const filename_tok = first;
257 skipToNl(tokenizer);
258 const tok_slice = pp.expandToken(filename_tok);
259 assert(tok_slice.len >= 3);
260 const filename = tok_slice[1 .. tok_slice.len - 1];
261 return (try pp.findInclude(filename, first)) orelse @panic("include not found");
262}
263
264fn expectMacroName(pp: *const Preprocessor, tokenizer: *Tokenizer) []const u8 {
265 const macro_name = tokenizer.nextNoWS();
266 assert(macro_name.id.isMacroIdentifier());
267 return pp.expandToken(macro_name);
268}
269
270fn skipToNl(tokenizer: *Tokenizer) void {
271 while (true) {
272 const tok = tokenizer.next();
273 if (tok.id == .nl or tok.id == .eof) return;
274 if (tok.id == .whitespace) continue;
275 }
276}
277
278fn define(pp: *Preprocessor, tokenizer: *Tokenizer) !void {
279 const macro_name = tokenizer.nextNoWS();
280 assert(macro_name.id == .identifier);
281
282 const first = tokenizer.nextNoWS();
283 switch (first.id) {
284 .nl, .eof => return pp.defineMacro(macro_name, .{
285 .is_func = false,
286 .tokens = &.{},
287 .param = "",
288 }),
289 .l_paren => return pp.defineFn(tokenizer, macro_name),
290 else => {},
291 }
292}
293
294fn defineMacro(pp: *Preprocessor, tok: Token, macro: Macro) !void {
295 const token_value = pp.expandToken(tok);
296 try pp.defines.putNoClobber(pp.arena, token_value, macro);
297}
298
299fn undefineMacro(pp: *Preprocessor, tok: Token) void {
300 const token_value = pp.expandToken(tok);
301 _ = pp.defines.orderedRemove(token_value);
302}
303
304fn defineFn(
305 pp: *Preprocessor,
306 tokenizer: *Tokenizer,
307 macro_name: Token,
308) !void {
309 var tok = tokenizer.nextNoWS();
310 assert(tok.id == .identifier);
311 const param = pp.expandToken(tok);
312 tok = tokenizer.nextNoWS();
313 assert(tok.id == .r_paren);
314
315 pp.token_buf.items.len = 0;
316 var need_ws = false;
317 while (true) {
318 tok = tokenizer.next();
319 switch (tok.id) {
320 .nl, .eof => break,
321 .whitespace => need_ws = pp.token_buf.items.len != 0,
322 .hash => unreachable,
323 .hash_hash => {
324 need_ws = false;
325 try pp.token_buf.append(pp.arena, tok);
326 },
327 else => {
328 if (need_ws) {
329 need_ws = false;
330 try pp.token_buf.append(pp.arena, .{ .id = .whitespace, .source = Source.generated });
331 }
332
333 if (tok.id.isMacroIdentifier()) {
334 tok.id = .identifier;
335 const s = pp.expandToken(tok);
336 if (mem.eql(u8, param, s)) {
337 tok.id = .macro_param;
338 tok.end = 0;
339 }
340 }
341 try pp.token_buf.append(pp.arena, tok);
342 },
343 }
344 }
345
346 const token_list = try pp.arena.dupe(Token, pp.token_buf.items);
347 try pp.defineMacro(macro_name, .{
348 .tokens = token_list,
349 .is_func = true,
350 .param = param,
351 });
352}
353
354fn expandToken(pp: *const Preprocessor, tok: Token) []const u8 {
355 return switch (tok.source) {
356 Source.generated => pp.generated_tokens.items,
357 else => blk: {
358 const src = pp.sources.values()[tok.source];
359 break :blk src.buf;
360 },
361 }[@intCast(tok.start)..@intCast(tok.end)];
362}
363
364fn skip(
365 pp: *Preprocessor,
366 tokenizer: *Tokenizer,
367 cont: IfContext.Nesting,
368) !void {
369 var ifs_seen: u32 = 0;
370 var line_start = true;
371 while (tokenizer.index < tokenizer.buf.len) {
372 if (line_start) {
373 const tokenizer_bkp = tokenizer.*;
374 const hash = tokenizer.nextNoWS();
375 if (hash.id == .nl) continue;
376 line_start = false;
377 if (hash.id != .hash) continue;
378 const directive = tokenizer.nextNoWS();
379 switch (directive.id) {
380 .keyword_else => {
381 if (ifs_seen != 0) continue;
382 assert(cont != .until_endif_seen_else); // else after else;
383 tokenizer.* = tokenizer_bkp;
384 return;
385 },
386 .keyword_elif => {
387 if (ifs_seen != 0 or cont == .until_endif) continue;
388 assert(cont != .until_endif_seen_else); // elif after else;
389 tokenizer.* = tokenizer_bkp;
390 return;
391 },
392 .keyword_endif => {
393 if (ifs_seen == 0) {
394 tokenizer.* = tokenizer_bkp;
395 return;
396 }
397 ifs_seen -= 1;
398 },
399 .keyword_if, .keyword_ifdef, .keyword_ifndef => ifs_seen += 1,
400 else => {},
401 }
402 } else if (tokenizer.buf[tokenizer.index] == '\n') {
403 line_start = true;
404 tokenizer.index += 1;
405 try pp.addToken(.{ .id = .nl, .source = Source.generated });
406 } else {
407 line_start = false;
408 tokenizer.index += 1;
409 }
410 }
411}
412
413fn ensureUnusedTokenCapacity(pp: *Preprocessor, capacity: usize) !void {
414 try pp.tokens.ensureUnusedCapacity(pp.arena, capacity);
415}
416
417fn expr(pp: *Preprocessor, tokenizer: *Tokenizer) !bool {
418 const token_state = pp.tokens.len;
419 defer pp.tokens.len = token_state;
420
421 pp.top_expansion_buf.items.len = 0;
422 while (true) {
423 const tok = tokenizer.next();
424 switch (tok.id) {
425 .nl, .eof => break,
426 .whitespace => if (pp.top_expansion_buf.items.len == 0) continue,
427 else => {},
428 }
429 try pp.top_expansion_buf.append(pp.arena, tok);
430 } else unreachable;
431 if (pp.top_expansion_buf.items.len != 0) {
432 try pp.expandMacroExhaustive(tokenizer, &pp.top_expansion_buf, 0, pp.top_expansion_buf.items.len, false, .expr);
433 }
434 try pp.ensureUnusedTokenCapacity(pp.top_expansion_buf.items.len);
435 var i: usize = 0;
436 const items = pp.top_expansion_buf.items;
437 while (i < items.len) : (i += 1) {
438 var tok = items[i];
439 switch (tok.id) {
440 .string_literal,
441 .semicolon,
442 .hash_hash,
443 => unreachable,
444 .whitespace => continue,
445 else => if (tok.id == .keyword_defined) {
446 i += try pp.handleKeywordDefined(&tok, items[i + 1 ..]);
447 },
448 }
449 pp.addTokenAssumeCapacity(tok);
450 }
451
452 try pp.addToken(.{ .id = .eof, .source = Source.generated });
453 return pp.evalExpression(token_state);
454}
455
456fn handleKeywordDefined(
457 pp: *Preprocessor,
458 macro_tok: *Token,
459 tokens: []const Token,
460) !usize {
461 assert(macro_tok.id == .keyword_defined);
462 var it = TokenIterator.init(tokens);
463
464 _ = it.expectNoWS(.l_paren);
465 const second = it.expectNoWS(.identifier);
466 _ = it.expectNoWS(.r_paren);
467
468 macro_tok.id = if (pp.defines.contains(pp.expandToken(second))) .one else .zero;
469
470 return it.i;
471}
472
473const TokenIterator = struct {
474 toks: []const Token,
475 i: usize,
476
477 fn init(toks: []const Token) TokenIterator {
478 return .{ .toks = toks, .i = 0 };
479 }
480
481 fn nextNoWS(self: *TokenIterator) ?Token {
482 while (self.i < self.toks.len) : (self.i += 1) {
483 const tok = self.toks[self.i];
484 if (tok.id == .whitespace) continue;
485
486 self.i += 1;
487 return tok;
488 }
489 return null;
490 }
491
492 fn expectNext(self: *TokenIterator) Token {
493 assert(self.i < self.toks.len);
494 const t = self.toks[self.i];
495 self.i += 1;
496 return t;
497 }
498
499 fn expectNoWS(self: *TokenIterator, expected: Token.Id) Token {
500 if (self.nextNoWS()) |tok| {
501 if (tok.id != expected) {
502 std.debug.panic("expected token {any} but got {any}\n", .{ expected, tok.id });
503 }
504 return tok;
505 }
506 std.debug.panic("expected token {any} but got null\n", .{expected});
507 }
508};
509
510fn expandMacro(pp: *Preprocessor, tokenizer: *Tokenizer, tok: Token) !void {
511 if (!tok.id.isMacroIdentifier()) {
512 return pp.addToken(tok);
513 }
514 pp.top_expansion_buf.items.len = 0;
515 try pp.top_expansion_buf.append(pp.arena, tok);
516 try pp.expandMacroExhaustive(tokenizer, &pp.top_expansion_buf, 0, 1, true, .non_expr);
517 try pp.addTokensFromExpandBuf(pp.top_expansion_buf.items, .{ .id = .nl, .source = Source.generated });
518}
519
520fn addTokensFromExpandBuf(pp: *Preprocessor, tokens: []Token, tokenizer_nl: Token) !void {
521 try pp.ensureUnusedTokenCapacity(tokens.len);
522 for (tokens) |tok| {
523 pp.addTokenAssumeCapacity(tok);
524 }
525 try pp.ensureUnusedTokenCapacity(pp.add_expansion_nl);
526 while (pp.add_expansion_nl > 0) : (pp.add_expansion_nl -= 1) {
527 pp.addTokenAssumeCapacity(tokenizer_nl);
528 }
529}
530
531const EvalContext = enum {
532 expr,
533 non_expr,
534};
535
536fn expandMacroExhaustive(
537 pp: *Preprocessor,
538 tokenizer: *Tokenizer,
539 buf: *ExpandBuf,
540 start_idx: usize,
541 end_idx: usize,
542 extend_buf: bool,
543 eval_ctx: EvalContext,
544) !void {
545 var moving_end_idx = end_idx;
546 var advance_index: usize = 0;
547 var do_rescan = true;
548 while (do_rescan) {
549 do_rescan = false;
550 var idx: usize = start_idx + advance_index;
551 while (idx < moving_end_idx) {
552 const macro_tok = buf.items[idx];
553 if (macro_tok.id == .keyword_defined and eval_ctx == .expr) {
554 idx += 1;
555 var it = TokenIterator.init(buf.items[idx..moving_end_idx]);
556 if (it.nextNoWS()) |tok| {
557 switch (tok.id) {
558 .l_paren => {
559 _ = it.nextNoWS();
560 _ = it.nextNoWS();
561 },
562 else => {},
563 }
564 }
565 idx += it.i;
566 continue;
567 }
568 if (!macro_tok.id.isMacroIdentifier()) {
569 idx += 1;
570 continue;
571 }
572 const expanded = pp.expandToken(macro_tok);
573 const macro = pp.defines.getPtr(expanded) orelse {
574 idx += 1;
575 continue;
576 };
577
578 if (macro.is_func) {
579 var macro_scan_idx = idx;
580 const arg = try pp.collectMacroArgument(
581 tokenizer,
582 buf,
583 &macro_scan_idx,
584 &moving_end_idx,
585 extend_buf,
586 );
587 const expanded_arg = arg: {
588 var expand_buf: ExpandBuf = .empty;
589 errdefer expand_buf.deinit(pp.arena);
590 try expand_buf.appendSlice(pp.arena, arg);
591 try pp.expandMacroExhaustive(tokenizer, &expand_buf, 0, expand_buf.items.len, false, eval_ctx);
592 break :arg try expand_buf.toOwnedSlice(pp.arena);
593 };
594
595 const res = try pp.expandFuncMacro(macro, arg, expanded_arg);
596 const tokens_added = res.items.len;
597 const tokens_removed = macro_scan_idx - idx + 1;
598 try buf.replaceRange(pp.arena, idx, tokens_removed, res.items);
599
600 moving_end_idx += tokens_added;
601 moving_end_idx -|= tokens_removed;
602 idx += tokens_added;
603 do_rescan = true;
604 } else {
605 var res = try pp.expandObjMacro(macro);
606 defer res.deinit(pp.arena);
607 var increment_idx_by = res.items.len;
608
609 for (res.items, 0..) |*tok, i| {
610 if (i < increment_idx_by and pp.defines.contains(pp.expandToken(tok.*))) {
611 increment_idx_by = i;
612 }
613 }
614 try buf.replaceRange(pp.arena, idx, 1, res.items);
615 idx += res.items.len;
616 moving_end_idx = moving_end_idx + res.items.len - 1;
617 do_rescan = true;
618 }
619 if (idx - start_idx == advance_index + 1 and !do_rescan) {
620 advance_index += 1;
621 }
622 }
623 }
624 buf.items.len = moving_end_idx;
625}
626
627fn collectMacroArgument(
628 pp: *Preprocessor,
629 tokenizer: *Tokenizer,
630 buf: *ExpandBuf,
631 start_idx: *usize,
632 end_idx: *usize,
633 extend_buf: bool,
634) !MacroArgument {
635 var parens: u32 = 0;
636 var argument: std.ArrayList(Token) = .empty;
637 defer argument.deinit(pp.arena);
638
639 while (true) {
640 const tok = try nextBufToken(pp, tokenizer, buf, start_idx, end_idx, extend_buf);
641 switch (tok.id) {
642 .nl, .whitespace => {},
643 .l_paren => break,
644 else => unreachable,
645 }
646 }
647
648 while (true) {
649 const tok = try nextBufToken(pp, tokenizer, buf, start_idx, end_idx, extend_buf);
650 switch (tok.id) {
651 .l_paren => {
652 try argument.append(pp.arena, tok);
653 parens += 1;
654 },
655 .r_paren => {
656 if (parens == 0) {
657 return try argument.toOwnedSlice(pp.arena);
658 } else {
659 try argument.append(pp.arena, tok);
660 parens -= 1;
661 }
662 },
663 .nl, .whitespace => try argument.append(pp.arena, .{ .id = .whitespace, .source = Source.generated }),
664 .eof => unreachable,
665 else => try argument.append(pp.arena, tok),
666 }
667 }
668}
669
670fn expandObjMacro(pp: *Preprocessor, simple_macro: *const Macro) !ExpandBuf {
671 var buf: ExpandBuf = .empty;
672 errdefer buf.deinit(pp.arena);
673 try buf.appendSlice(pp.arena, simple_macro.tokens);
674 return buf;
675}
676
677fn expandFuncMacro(
678 pp: *Preprocessor,
679 func_macro: *const Macro,
680 arg: MacroArgument,
681 expanded_arg: MacroArgument,
682) !ExpandBuf {
683 var buf: ExpandBuf = .empty;
684 errdefer buf.deinit(pp.arena);
685 try buf.ensureTotalCapacity(pp.arena, func_macro.tokens.len);
686
687 var tok_i: usize = 0;
688 while (tok_i < func_macro.tokens.len) : (tok_i += 1) {
689 const tok = func_macro.tokens[tok_i];
690 switch (tok.id) {
691 .hash_hash => while (tok_i + 1 < func_macro.tokens.len) {
692 tok_i += 1;
693 const tok_next = func_macro.tokens[tok_i];
694 const next = switch (tok_next.id) {
695 .whitespace => continue,
696 .hash_hash => continue,
697 .macro_param => arg,
698 else => &[1]Token{tok_next},
699 };
700 try pp.pasteTokens(&buf, next);
701 if (next.len != 0) break;
702 },
703 .macro_param => {
704 try buf.appendSlice(pp.arena, expanded_arg);
705 },
706 else => try buf.append(pp.arena, tok),
707 }
708 }
709
710 return buf;
711}
712
713fn pasteTokens(
714 pp: *Preprocessor,
715 lhs_toks: *ExpandBuf,
716 rhs_toks: []const Token,
717) !void {
718 const lhs = while (lhs_toks.pop()) |lhs| {
719 if (lhs.id != .whitespace) break lhs;
720 } else {
721 return lhs_toks.appendSlice(pp.arena, rhs_toks);
722 };
723
724 var rhs_rest: u32 = 1;
725 const rhs = for (rhs_toks) |rhs| {
726 if (rhs.id != .whitespace) break rhs;
727 rhs_rest += 1;
728 } else {
729 return lhs_toks.appendAssumeCapacity(lhs);
730 };
731
732 const start = pp.generated_tokens.items.len;
733 const end = start + pp.expandToken(lhs).len + pp.expandToken(rhs).len;
734 try pp.generated_tokens.ensureTotalCapacity(pp.arena, end + 1);
735 pp.generated_tokens.appendSliceAssumeCapacity(pp.expandToken(lhs));
736 pp.generated_tokens.appendSliceAssumeCapacity(pp.expandToken(rhs));
737 pp.generated_tokens.appendAssumeCapacity('\n');
738
739 var tmp_tokenizer: Tokenizer = .{
740 .index = @intCast(start),
741 .buf = pp.generated_tokens.items,
742 .source = Source.generated,
743 };
744 const pasted_token = tmp_tokenizer.nextNoWS();
745 const next = tmp_tokenizer.nextNoWS();
746
747 try lhs_toks.append(pp.arena, pp.makeGeneratedToken(start, end, pasted_token.id));
748 assert(next.id == .nl or next.id == .eof);
749
750 return lhs_toks.appendSlice(pp.arena, rhs_toks[rhs_rest..]);
751}
752
753fn nextBufToken(
754 pp: *Preprocessor,
755 tokenizer: *Tokenizer,
756 buf: *ExpandBuf,
757 start_idx: *usize,
758 end_idx: *usize,
759 extend_buf: bool,
760) !Token {
761 start_idx.* += 1;
762 if (start_idx.* == buf.items.len and start_idx.* >= end_idx.*) {
763 if (extend_buf) {
764 const tok = tokenizer.next();
765 if (tok.id == .nl) pp.add_expansion_nl += 1;
766
767 end_idx.* += 1;
768 try buf.append(pp.arena, tok);
769 return tok;
770 }
771 return .{ .id = .eof, .source = Source.generated };
772 }
773
774 return buf.items[start_idx.*];
775}
776
777fn makeGeneratedToken(
778 pp: *Preprocessor,
779 start: usize,
780 end: usize,
781 id: Token.Id,
782) Token {
783 const pasted_token: Token = .{
784 .id = id,
785 .source = Source.generated,
786 .start = @intCast(start),
787 .end = @intCast(end),
788 };
789 pp.generated_line += 1;
790 return pasted_token;
791}
792
793fn findInclude(pp: *Preprocessor, filename: []const u8, includer_token: Token) !?Source {
794 const other_file = pp.sources.values()[includer_token.source].path;
795 const dir: Path = other_file.dirname() orelse .cwd();
796 if (try pp.checkIncludeDir(filename, dir)) |res| return res;
797
798 return pp.checkIncludeDir(filename, pp.include_dir);
799}
800
801fn checkIncludeDir(
802 pp: *Preprocessor,
803 include_path: []const u8,
804 include_dir: Path,
805) !?Source {
806 var bfa_buf: [1024]u8 = undefined;
807 var bfa_state: std.heap.BufferFirstAllocator = .init(&bfa_buf, pp.arena);
808 const bfa = bfa_state.allocator();
809 const header_path = try include_dir.join(bfa, include_path);
810 return pp.addSourceFromPath(header_path) catch |err| switch (err) {
811 error.OutOfMemory => |e| return e,
812 else => return null,
813 };
814}
815
816pub fn addSourceFromPath(pp: *Preprocessor, path: Path) !Source {
817 if (pp.sources.get(path)) |src| return src;
818 try pp.sources.ensureUnusedCapacity(pp.arena, 1);
819
820 const contents = try path.root_dir.handle.readFileAlloc(pp.io, path.sub_path, pp.arena, .limited(std.math.maxInt(u32)));
821 const duped_path = try path.clone(pp.arena);
822
823 const src: Source = .{
824 .buf = contents,
825 .path = duped_path,
826 .id = pp.sources.count(),
827 };
828
829 pp.sources.putAssumeCapacityNoClobber(duped_path, src);
830 return src;
831}
832
833fn evalExpression(
834 pp: *Preprocessor,
835 start: usize,
836) !bool {
837 const s = pp.tokens.slice();
838 const len = s.len - start;
839 const ss = s.subslice(start, len);
840
841 const ids: []Token.Id = ss.items(.id);
842 const starts: []u32 = ss.items(.start);
843 const ends: []u32 = ss.items(.end);
844 const srcs: []usize = ss.items(.source);
845
846 var toks = try pp.arena.alloc(Token, len);
847 defer pp.arena.free(toks);
848
849 for (0..len) |i| {
850 toks[i] = .{
851 .id = ids[i],
852 .source = srcs[i],
853 .start = starts[i],
854 .end = ends[i],
855 };
856 }
857
858 return pp.evaluateExpressionTokens(toks);
859}
860
861fn evaluateExpressionTokens(
862 pp: *const Preprocessor,
863 toks: []const Token,
864) bool {
865 var it = TokenIterator.init(toks);
866
867 const left = evalToken(&it);
868 assert(!left.id.isInfix());
869
870 const op = evalToken(&it);
871 if (op.id == .eof) return left.id == .one;
872
873 assert(op.id.isInfix());
874 const right = evalToken(&it);
875
876 return pp.evalInfix(left, op, right);
877}
878
879fn evalToken(it: *TokenIterator) Token {
880 const tok = it.expectNext();
881 if (tok.id != .bang) {
882 return tok;
883 }
884
885 var op = it.expectNext();
886 const flipped: Token.Id = switch (op.id) {
887 .one => .zero,
888 .zero => .one,
889 else => unreachable,
890 };
891 op.id = flipped;
892 return op;
893}
894
895fn evalInfix(
896 pp: *const Preprocessor,
897 left: Token,
898 op: Token,
899 right: Token,
900) bool {
901 switch (op.id) {
902 .pipe_pipe => return (left.id == .one) or (right.id == .one),
903 .equal_equal => {
904 switch (left.id) {
905 .one, .zero => {
906 assert(right.id == .one or right.id == .zero);
907 return left.id == right.id;
908 },
909 .pp_num => {
910 assert(right.id == .pp_num);
911 const lval = pp.expandToken(left);
912 const rval = pp.expandToken(right);
913 return std.mem.eql(u8, lval, rval);
914 },
915 else => unreachable,
916 }
917 },
918 else => unreachable,
919 }
920}
921
922pub fn prettyPrintTokens(pp: *Preprocessor, w: *std.Io.Writer) !void {
923 const tok_ids = pp.tokens.items(.id);
924 var i: usize = 0;
925 var last_nl = true;
926 outer: while (true) : (i += 1) {
927 const cur: Token = pp.tokens.get(i);
928 switch (cur.id) {
929 .eof => {
930 if (!last_nl) try w.writeByte('\n');
931 try w.flush();
932 return;
933 },
934 .nl => {
935 var newlines: u32 = 0;
936 for (tok_ids[i..], i..) |id, j| {
937 if (id == .nl) {
938 newlines += 1;
939 } else if (id == .eof) {
940 if (!last_nl) try w.writeByte('\n');
941 try w.flush();
942 return;
943 } else if (id != .whitespace) {
944 if (newlines < 2) break;
945
946 i = @intCast((j - 1) - @intFromBool(tok_ids[j - 1] == .whitespace));
947 if (!last_nl) try w.writeAll("\n");
948 continue :outer;
949 }
950 }
951 last_nl = true;
952 try w.writeAll("\n");
953 },
954 .whitespace => {
955 try w.writeByte(' ');
956 last_nl = false;
957 },
958 else => {
959 const slice = pp.expandToken(cur);
960 try w.writeAll(slice);
961 last_nl = false;
962 },
963 }
964 }
965}