Commit 90ab8ea9e681a4ffac0b4dc500e3ec489014e12f | zig

.gitattributes+1-1

...	...	@@ -12,4 +12,4 @@ lib/libcxx/** linguist-vendored
12	12	lib/libcxxabi/** linguist-vendored
13	13	lib/libunwind/** linguist-vendored
14	14	lib/tsan/** linguist-vendored
15		deps/** linguist-vendored
	15	lib/compiler/aro/** linguist-vendored

lib/compiler/aro/README.md+1-2

...	...	@@ -20,8 +20,7 @@ int main(void) {
20	20	printf("Hello, world!\n");
21	21	return 0;
22	22	}
23		$ zig build run -- hello.c -o hello
	23	$ zig build && ./zig-out/bin/arocc hello.c -o hello
24	24	$ ./hello
25	25	Hello, world!
26		$
27	26	```

lib/compiler/aro/aro/Attribute/names.zig+6-16

...	...	@@ -1,4 +1,4 @@
1		//! Autogenerated by GenerateDef from deps/aro/aro/Attribute/names.def, do not edit
	1	//! Autogenerated by GenerateDef from src/aro/Attribute/names.def, do not edit
2	2	// zig fmt: off
3	3
4	4	const std = @import("std");
...	...	@@ -142,15 +142,7 @@ pub fn nameFromUniqueIndex(index: u16, buf: []u8) []u8 {
142	142	return fbs.getWritten();
143	143	}
144	144
145		/// We're 1 bit shy of being able to fit this in a u32:
146		/// - char only contains 0-9, a-z, A-Z, and _, so it could use a enum(u6) with a way to convert <-> u8
147		/// (note: this would have a performance cost that may make the u32 not worth it)
148		/// - number has a max value of > 2047 and < 4095 (the first _ node has the largest number),
149		/// so it could fit into a u12
150		/// - child_index currently has a max of > 4095 and < 8191, so it could fit into a u13
151		///
152		/// with the end_of_word/end_of_list 2 bools, that makes 33 bits total
153		const Node = packed struct(u64) {
	145	const Node = packed struct(u32) {
154	146	char: u8,
155	147	/// Nodes are numbered with "an integer which gives the number of words that
156	148	/// would be accepted by the automaton starting from that state." This numbering
...	...	@@ -158,18 +150,16 @@ const Node = packed struct(u64) {
158	150	/// (L is the number of words accepted by the automaton) and the words themselves."
159	151	///
160	152	/// Essentially, this allows us to have a minimal perfect hashing scheme such that
161		/// it's possible to store & lookup the properties of each builtin using a separate array.
162		number: u16,
163		/// If true, this node is the end of a valid builtin.
	153	/// it's possible to store & lookup the properties of each name using a separate array.
	154	number: u8,
	155	/// If true, this node is the end of a valid name.
164	156	/// Note: This does not necessarily mean that this node does not have child nodes.
165	157	end_of_word: bool,
166	158	/// If true, this node is the end of a sibling list.
167	159	/// If false, then (index + 1) will contain the next sibling.
168	160	end_of_list: bool,
169		/// Padding bits to get to u64, unsure if there's some way to use these to improve something.
170		_extra: u22 = 0,
171	161	/// Index of the first child of this node.
172		child_index: u16,
	162	child_index: u14,
173	163	};
174	164
175	165	const dafsa = [_]Node{

lib/compiler/aro/aro/Builtins.zig+1-4

...	...	@@ -99,10 +99,7 @@ fn createType(desc: TypeDescription, it: TypeDescription.TypeIterator, comp: c
99	99	}
100	100	},
101	101	.h => builder.combine(undefined, .fp16, 0) catch unreachable,
102		.x => {
103		// Todo: _Float16
104		return .{ .specifier = .invalid };
105		},
	102	.x => builder.combine(undefined, .float16, 0) catch unreachable,
106	103	.y => {
107	104	// Todo: __bf16
108	105	return .{ .specifier = .invalid };

lib/compiler/aro/aro/Builtins/Builtin.zig+1-1

...	...	@@ -1,4 +1,4 @@
1		//! Autogenerated by GenerateDef from deps/aro/aro/Builtins/Builtin.def, do not edit
	1	//! Autogenerated by GenerateDef from src/aro/Builtins/Builtin.def, do not edit
2	2	// zig fmt: off
3	3
4	4	const std = @import("std");

lib/compiler/aro/aro/Compilation.zig+39-20

...	...	@@ -241,6 +241,12 @@ pub const SystemDefinesMode = enum {
241	241	fn generateSystemDefines(comp: *Compilation, w: anytype) !void {
242	242	const ptr_width = comp.target.ptrBitWidth();
243	243
	244	if (comp.langopts.gnuc_version > 0) {
	245	try w.print("#define __GNUC__ {d}\n", .{comp.langopts.gnuc_version / 10_000});
	246	try w.print("#define __GNUC_MINOR__ {d}\n", .{comp.langopts.gnuc_version / 100 % 100});
	247	try w.print("#define __GNUC_PATCHLEVEL__ {d}\n", .{comp.langopts.gnuc_version % 100});
	248	}
	249
244	250	// os macros
245	251	switch (comp.target.os.tag) {
246	252	.linux => try w.writeAll(
...	...	@@ -419,6 +425,25 @@ fn generateSystemDefines(comp: *Compilation, w: anytype) !void {
419	425	\\
420	426	);
421	427
	428	// TODO: Set these to target-specific constants depending on backend capabilities
	429	// For now they are just set to the "may be lock-free" value
	430	try w.writeAll(
	431	\\#define __ATOMIC_BOOL_LOCK_FREE 1
	432	\\#define __ATOMIC_CHAR_LOCK_FREE 1
	433	\\#define __ATOMIC_CHAR16_T_LOCK_FREE 1
	434	\\#define __ATOMIC_CHAR32_T_LOCK_FREE 1
	435	\\#define __ATOMIC_WCHAR_T_LOCK_FREE 1
	436	\\#define __ATOMIC_SHORT_LOCK_FREE 1
	437	\\#define __ATOMIC_INT_LOCK_FREE 1
	438	\\#define __ATOMIC_LONG_LOCK_FREE 1
	439	\\#define __ATOMIC_LLONG_LOCK_FREE 1
	440	\\#define __ATOMIC_POINTER_LOCK_FREE 1
	441	\\
	442	);
	443	if (comp.langopts.hasChar8_T()) {
	444	try w.writeAll("#define __ATOMIC_CHAR8_T_LOCK_FREE 1\n");
	445	}
	446
422	447	// types
423	448	if (comp.getCharSignedness() == .unsigned) try w.writeAll("#define __CHAR_UNSIGNED__ 1\n");
424	449	try w.writeAll("#define __CHAR_BIT__ 8\n");
...	...	@@ -438,6 +463,7 @@ fn generateSystemDefines(comp: *Compilation, w: anytype) !void {
438	463	try comp.generateIntMaxAndWidth(w, "PTRDIFF", comp.types.ptrdiff);
439	464	try comp.generateIntMaxAndWidth(w, "INTPTR", comp.types.intptr);
440	465	try comp.generateIntMaxAndWidth(w, "UINTPTR", comp.types.intptr.makeIntegerUnsigned());
	466	try comp.generateIntMaxAndWidth(w, "SIG_ATOMIC", target_util.sigAtomicType(comp.target));
441	467
442	468	// int widths
443	469	try w.print("#define __BITINT_MAXWIDTH__ {d}\n", .{bit_int_max_bits});
...	...	@@ -474,6 +500,8 @@ fn generateSystemDefines(comp: *Compilation, w: anytype) !void {
474	500	try generateTypeMacro(w, mapper, "__PTRDIFF_TYPE__", comp.types.ptrdiff, comp.langopts);
475	501	try generateTypeMacro(w, mapper, "__SIZE_TYPE__", comp.types.size, comp.langopts);
476	502	try generateTypeMacro(w, mapper, "__WCHAR_TYPE__", comp.types.wchar, comp.langopts);
	503	try generateTypeMacro(w, mapper, "__CHAR16_TYPE__", comp.types.uint_least16_t, comp.langopts);
	504	try generateTypeMacro(w, mapper, "__CHAR32_TYPE__", comp.types.uint_least32_t, comp.langopts);
477	505
478	506	try comp.generateExactWidthTypes(w, mapper);
479	507	try comp.generateFastAndLeastWidthTypes(w, mapper);
...	...	@@ -518,7 +546,6 @@ pub fn generateBuiltinMacros(comp: *Compilation, system_defines_mode: SystemDefi
518	546
519	547	// standard macros
520	548	try buf.appendSlice(
521		\\#define __STDC_NO_ATOMICS__ 1
522	549	\\#define __STDC_NO_COMPLEX__ 1
523	550	\\#define __STDC_NO_THREADS__ 1
524	551	\\#define __STDC_NO_VLA__ 1
...	...	@@ -1030,9 +1057,8 @@ pub fn getCharSignedness(comp: *const Compilation) std.builtin.Signedness {
1030	1057	return comp.langopts.char_signedness_override orelse comp.target.charSignedness();
1031	1058	}
1032	1059
1033		pub fn defineSystemIncludes(comp: *Compilation, aro_dir: []const u8) !void {
1034		var stack_fallback = std.heap.stackFallback(path_buf_stack_limit, comp.gpa);
1035		const allocator = stack_fallback.get();
	1060	/// Add built-in aro headers directory to system include paths
	1061	pub fn addBuiltinIncludeDir(comp: *Compilation, aro_dir: []const u8) !void {
1036	1062	var search_path = aro_dir;
1037	1063	while (std.fs.path.dirname(search_path)) \|dirname\| : (search_path = dirname) {
1038	1064	var base_dir = std.fs.cwd().openDir(dirname, .{}) catch continue;
...	...	@@ -1044,23 +1070,12 @@ pub fn defineSystemIncludes(comp: *Compilation, aro_dir: []const u8) !void {
1044	1070	try comp.system_include_dirs.append(comp.gpa, path);
1045	1071	break;
1046	1072	} else return error.AroIncludeNotFound;
	1073	}
1047	1074
1048		if (comp.target.os.tag == .linux) {
1049		const triple_str = try comp.target.linuxTriple(allocator);
1050		defer allocator.free(triple_str);
1051
1052		const multiarch_path = try std.fs.path.join(allocator, &.{ "/usr/include", triple_str });
1053		defer allocator.free(multiarch_path);
1054
1055		if (!std.meta.isError(std.fs.accessAbsolute(multiarch_path, .{}))) {
1056		const duped = try comp.gpa.dupe(u8, multiarch_path);
1057		errdefer comp.gpa.free(duped);
1058		try comp.system_include_dirs.append(comp.gpa, duped);
1059		}
1060		}
1061		const usr_include = try comp.gpa.dupe(u8, "/usr/include");
1062		errdefer comp.gpa.free(usr_include);
1063		try comp.system_include_dirs.append(comp.gpa, usr_include);
	1075	pub fn addSystemIncludeDir(comp: *Compilation, path: []const u8) !void {
	1076	const duped = try comp.gpa.dupe(u8, path);
	1077	errdefer comp.gpa.free(duped);
	1078	try comp.system_include_dirs.append(comp.gpa, duped);
1064	1079	}
1065	1080
1066	1081	pub fn getSource(comp: *const Compilation, id: Source.Id) Source {
...	...	@@ -1331,6 +1346,10 @@ pub fn hasInclude(
1331	1346	/// __has_include vs __has_include_next
1332	1347	which: WhichInclude,
1333	1348	) !bool {
	1349	if (mem.indexOfScalar(u8, filename, 0) != null) {
	1350	return false;
	1351	}
	1352
1334	1353	const cwd = std.fs.cwd();
1335	1354	if (std.fs.path.isAbsolute(filename)) {
1336	1355	if (which == .next) return false;

lib/compiler/aro/aro/Diagnostics.zig+3-1

...	...	@@ -208,6 +208,8 @@ pub const Options = struct {
208	208	@"unsupported-embed-param": Kind = .default,
209	209	@"unused-result": Kind = .default,
210	210	normalized: Kind = .default,
	211	@"shift-count-negative": Kind = .default,
	212	@"shift-count-overflow": Kind = .default,
211	213	};
212	214
213	215	const Diagnostics = @This();
...	...	@@ -291,7 +293,7 @@ pub fn addExtra(
291	293	.kind = .note,
292	294	.extra = .{ .unsigned = expansion_locs.len - d.macro_backtrace_limit },
293	295	});
294		i = half - 1;
	296	i = half -\| 1;
295	297	while (i > 0) {
296	298	i -= 1;
297	299	d.list.appendAssumeCapacity(.{

lib/compiler/aro/aro/Diagnostics/messages.zig+11-1

...	...	@@ -1,4 +1,4 @@
1		//! Autogenerated by GenerateDef from deps/aro/aro/Diagnostics/messages.def, do not edit
	1	//! Autogenerated by GenerateDef from src/aro/Diagnostics/messages.def, do not edit
2	2	// zig fmt: off
3	3
4	4	const std = @import("std");
...	...	@@ -504,6 +504,11 @@ pub const Tag = enum {
504	504	c23_auto_single_declarator,
505	505	c32_auto_requires_initializer,
506	506	c23_auto_scalar_init,
	507	negative_shift_count,
	508	too_big_shift_count,
	509	complex_conj,
	510	overflow_builtin_requires_int,
	511	overflow_result_requires_ptr,
507	512
508	513	pub fn property(tag: Tag) Properties {
509	514	return named_data[@intFromEnum(tag)];
...	...	@@ -1005,6 +1010,11 @@ pub const Tag = enum {
1005	1010	.{ .msg = "'auto' can only be used with a single declarator", .kind = .@"error" },
1006	1011	.{ .msg = "'auto' requires an initializer", .kind = .@"error" },
1007	1012	.{ .msg = "'auto' requires a scalar initializer", .kind = .@"error" },
	1013	.{ .msg = "shift count is negative", .opt = W("shift-count-negative"), .kind = .warning, .all = true },
	1014	.{ .msg = "shift count >= width of type", .opt = W("shift-count-overflow"), .kind = .warning, .all = true },
	1015	.{ .msg = "ISO C does not support '~' for complex conjugation of '{s}'", .opt = W("pedantic"), .extra = .str, .kind = .off },
	1016	.{ .msg = "operand argument to overflow builtin must be an integer ('{s}' invalid)", .extra = .str, .kind = .@"error" },
	1017	.{ .msg = "result argument to overflow builtin must be a pointer to a non-const integer ('{s}' invalid)", .extra = .str, .kind = .@"error" },
1008	1018	};
1009	1019	};
1010	1020	};

lib/compiler/aro/aro/Driver.zig+27-3

...	...	@@ -12,6 +12,7 @@ const Preprocessor = @import("Preprocessor.zig");
12	12	const Source = @import("Source.zig");
13	13	const Toolchain = @import("Toolchain.zig");
14	14	const target_util = @import("target.zig");
	15	const GCCVersion = @import("Driver/GCCVersion.zig");
15	16
16	17	pub const Linker = enum {
17	18	ld,
...	...	@@ -43,6 +44,9 @@ verbose_pp: bool = false,
43	44	verbose_ir: bool = false,
44	45	verbose_linker_args: bool = false,
45	46	color: ?bool = null,
	47	nobuiltininc: bool = false,
	48	nostdinc: bool = false,
	49	nostdlibinc: bool = false,
46	50
47	51	/// Full path to the aro executable
48	52	aro_name: []const u8 = "",
...	...	@@ -95,6 +99,7 @@ pub const usage =
95	99	\\ -fcolor-diagnostics Enable colors in diagnostics
96	100	\\ -fno-color-diagnostics Disable colors in diagnostics
97	101	\\ -fdeclspec Enable support for __declspec attributes
	102	\\ -fgnuc-version=<value> Controls value of __GNUC__ and related macros. Set to 0 or empty to disable them.
98	103	\\ -fno-declspec Disable support for __declspec attributes
99	104	\\ -ffp-eval-method=[source\|double\|extended]
100	105	\\ Evaluation method to use for floating-point arithmetic
...	...	@@ -127,6 +132,10 @@ pub const usage =
127	132	\\ -isystem Add directory to SYSTEM include search path
128	133	\\ --emulate=[clang\|gcc\|msvc]
129	134	\\ Select which C compiler to emulate (default clang)
	135	\\ -nobuiltininc Do not search the compiler's builtin directory for include files
	136	\\ -nostdinc, --no-standard-includes
	137	\\ Do not search the standard system directories or compiler builtin directories for include files.
	138	\\ -nostdlibinc Do not search the standard system directories for include files, but do search compiler builtin include directories
130	139	\\ -o <file> Write output to <file>
131	140	\\ -P, --no-line-commands Disable linemarker output in -E mode
132	141	\\ -pedantic Warn on language extensions
...	...	@@ -180,6 +189,7 @@ pub fn parseArgs(
180	189	var i: usize = 1;
181	190	var comment_arg: []const u8 = "";
182	191	var hosted: ?bool = null;
	192	var gnuc_version: []const u8 = "4.2.1"; // default value set by clang
183	193	while (i < args.len) : (i += 1) {
184	194	const arg = args[i];
185	195	if (mem.startsWith(u8, arg, "-") and arg.len > 1) {
...	...	@@ -303,6 +313,10 @@ pub fn parseArgs(
303	313	d.only_syntax = true;
304	314	} else if (mem.startsWith(u8, arg, "-fno-syntax-only")) {
305	315	d.only_syntax = false;
	316	} else if (mem.eql(u8, arg, "-fgnuc-version=")) {
	317	gnuc_version = "0";
	318	} else if (option(arg, "-fgnuc-version=")) \|version\| {
	319	gnuc_version = version;
306	320	} else if (mem.startsWith(u8, arg, "-isystem")) {
307	321	var path = arg["-isystem".len..];
308	322	if (path.len == 0) {
...	...	@@ -421,6 +435,12 @@ pub fn parseArgs(
421	435	d.nodefaultlibs = true;
422	436	} else if (mem.eql(u8, arg, "-nolibc")) {
423	437	d.nolibc = true;
	438	} else if (mem.eql(u8, arg, "-nobuiltininc")) {
	439	d.nobuiltininc = true;
	440	} else if (mem.eql(u8, arg, "-nostdinc") or mem.eql(u8, arg, "--no-standard-includes")) {
	441	d.nostdinc = true;
	442	} else if (mem.eql(u8, arg, "-nostdlibinc")) {
	443	d.nostdlibinc = true;
424	444	} else if (mem.eql(u8, arg, "-nostdlib")) {
425	445	d.nostdlib = true;
426	446	} else if (mem.eql(u8, arg, "-nostartfiles")) {
...	...	@@ -459,6 +479,11 @@ pub fn parseArgs(
459	479	d.comp.target.os.tag = .freestanding;
460	480	}
461	481	}
	482	const version = GCCVersion.parse(gnuc_version);
	483	if (version.major == -1) {
	484	return d.fatal("invalid value '{0s}' in '-fgnuc-version={0s}'", .{gnuc_version});
	485	}
	486	d.comp.langopts.gnuc_version = version.toUnsigned();
462	487	return false;
463	488	}
464	489
...	...	@@ -558,7 +583,8 @@ pub fn main(d: Driver, tc: Toolchain, args: []const []const u8, comptime fast_
558	583	try d.comp.addDiagnostic(.{ .tag = .cli_unused_link_object, .extra = .{ .str = obj } }, &.{});
559	584	};
560	585
561		d.comp.defineSystemIncludes(d.aro_name) catch \|er\| switch (er) {
	586	try tc.discover();
	587	tc.defineSystemIncludes() catch \|er\| switch (er) {
562	588	error.OutOfMemory => return error.OutOfMemory,
563	589	error.AroIncludeNotFound => return d.fatal("unable to find Aro builtin headers", .{}),
564	590	};
...	...	@@ -763,8 +789,6 @@ fn dumpLinkerArgs(items: []const []const u8) !void {
763	789	/// The entry point of the Aro compiler.
764	790	/// MAY call `exit` if `fast_exit` is set.
765	791	pub fn invokeLinker(d: Driver, tc: Toolchain, comptime fast_exit: bool) !void {
766		try tc.discover();
767
768	792	var argv = std.ArrayList([]const u8).init(d.comp.gpa);
769	793	defer argv.deinit();
770	794

lib/compiler/aro/aro/Driver/GCCVersion.zig+10

...	...	@@ -98,6 +98,16 @@ pub fn order(a: GCCVersion, b: GCCVersion) Order {
98	98	return .eq;
99	99	}
100	100
	101	/// Used for determining __GNUC__ macro values
	102	/// This matches clang's logic for overflowing values
	103	pub fn toUnsigned(self: GCCVersion) u32 {
	104	var result: u32 = 0;
	105	if (self.major > 0) result = @as(u32, @intCast(self.major)) *% 10_000;
	106	if (self.minor > 0) result +%= @as(u32, @intCast(self.minor)) *% 100;
	107	if (self.patch > 0) result +%= @as(u32, @intCast(self.patch));
	108	return result;
	109	}
	110
101	111	test parse {
102	112	const versions = [10]GCCVersion{
103	113	parse("5"),

lib/compiler/aro/aro/Hideset.zig created+191

...	...	@@ -0,0 +1,191 @@
	1	//! A hideset is a linked list (implemented as an array so that elements are identified by 4-byte indices)
	2	//! of the set of identifiers from which a token was expanded.
	3	//! During macro expansion, if a token would otherwise be expanded, but its hideset contains
	4	//! the token itself, then it is not expanded
	5	//! Most tokens have an empty hideset, and the hideset is not needed once expansion is complete,
	6	//! so we use a hash map to store them instead of directly storing them with the token.
	7	//! The C standard underspecifies the algorithm for updating a token's hideset;
	8	//! we use the one here: https://www.spinellis.gr/blog/20060626/cpp.algo.pdf
	9
	10	const std = @import("std");
	11	const mem = std.mem;
	12	const Allocator = mem.Allocator;
	13	const Source = @import("Source.zig");
	14	const Compilation = @import("Compilation.zig");
	15	const Tokenizer = @import("Tokenizer.zig");
	16
	17	pub const Hideset = @This();
	18
	19	const Identifier = struct {
	20	id: Source.Id = .unused,
	21	byte_offset: u32 = 0,
	22
	23	fn slice(self: Identifier, comp: *const Compilation) []const u8 {
	24	var tmp_tokenizer = Tokenizer{
	25	.buf = comp.getSource(self.id).buf,
	26	.langopts = comp.langopts,
	27	.index = self.byte_offset,
	28	.source = .generated,
	29	};
	30	const res = tmp_tokenizer.next();
	31	return tmp_tokenizer.buf[res.start..res.end];
	32	}
	33
	34	fn fromLocation(loc: Source.Location) Identifier {
	35	return .{
	36	.id = loc.id,
	37	.byte_offset = loc.byte_offset,
	38	};
	39	}
	40	};
	41
	42	const Item = struct {
	43	identifier: Identifier = .{},
	44	next: Index = .none,
	45
	46	const List = std.MultiArrayList(Item);
	47	};
	48
	49	const Index = enum(u32) {
	50	none = std.math.maxInt(u32),
	51	_,
	52	};
	53
	54	map: std.AutoHashMapUnmanaged(Identifier, Index) = .{},
	55	/// Used for computing intersection of two lists; stored here so that allocations can be retained
	56	/// until hideset is deinit'ed
	57	intersection_map: std.AutoHashMapUnmanaged(Identifier, void) = .{},
	58	linked_list: Item.List = .{},
	59	comp: *const Compilation,
	60
	61	/// Invalidated if the underlying MultiArrayList slice is reallocated due to resize
	62	const Iterator = struct {
	63	slice: Item.List.Slice,
	64	i: Index,
	65
	66	fn next(self: *Iterator) ?Identifier {
	67	if (self.i == .none) return null;
	68	defer self.i = self.slice.items(.next)[@intFromEnum(self.i)];
	69	return self.slice.items(.identifier)[@intFromEnum(self.i)];
	70	}
	71	};
	72
	73	pub fn deinit(self: *Hideset) void {
	74	self.map.deinit(self.comp.gpa);
	75	self.intersection_map.deinit(self.comp.gpa);
	76	self.linked_list.deinit(self.comp.gpa);
	77	}
	78
	79	pub fn clearRetainingCapacity(self: *Hideset) void {
	80	self.linked_list.shrinkRetainingCapacity(0);
	81	self.map.clearRetainingCapacity();
	82	}
	83
	84	pub fn clearAndFree(self: *Hideset) void {
	85	self.map.clearAndFree(self.comp.gpa);
	86	self.intersection_map.clearAndFree(self.comp.gpa);
	87	self.linked_list.shrinkAndFree(self.comp.gpa, 0);
	88	}
	89
	90	/// Iterator is invalidated if the underlying MultiArrayList slice is reallocated due to resize
	91	fn iterator(self: *const Hideset, idx: Index) Iterator {
	92	return Iterator{
	93	.slice = self.linked_list.slice(),
	94	.i = idx,
	95	};
	96	}
	97
	98	pub fn get(self: *const Hideset, loc: Source.Location) Index {
	99	return self.map.get(Identifier.fromLocation(loc)) orelse .none;
	100	}
	101
	102	pub fn put(self: *Hideset, loc: Source.Location, value: Index) !void {
	103	try self.map.put(self.comp.gpa, Identifier.fromLocation(loc), value);
	104	}
	105
	106	fn ensureUnusedCapacity(self: *Hideset, new_size: usize) !void {
	107	try self.linked_list.ensureUnusedCapacity(self.comp.gpa, new_size);
	108	}
	109
	110	/// Creates a one-item list with contents `identifier`
	111	fn createNodeAssumeCapacity(self: *Hideset, identifier: Identifier) Index {
	112	const next_idx = self.linked_list.len;
	113	self.linked_list.appendAssumeCapacity(.{ .identifier = identifier });
	114	return @enumFromInt(next_idx);
	115	}
	116
	117	/// Create a new list with `identifier` at the front followed by `tail`
	118	pub fn prepend(self: *Hideset, loc: Source.Location, tail: Index) !Index {
	119	const new_idx = self.linked_list.len;
	120	try self.linked_list.append(self.comp.gpa, .{ .identifier = Identifier.fromLocation(loc), .next = tail });
	121	return @enumFromInt(new_idx);
	122	}
	123
	124	/// Copy a, then attach b at the end
	125	pub fn @"union"(self: *Hideset, a: Index, b: Index) !Index {
	126	var cur: Index = .none;
	127	var head: Index = b;
	128	try self.ensureUnusedCapacity(self.len(a));
	129	var it = self.iterator(a);
	130	while (it.next()) \|identifier\| {
	131	const new_idx = self.createNodeAssumeCapacity(identifier);
	132	if (head == b) {
	133	head = new_idx;
	134	}
	135	if (cur != .none) {
	136	self.linked_list.items(.next)[@intFromEnum(cur)] = new_idx;
	137	}
	138	cur = new_idx;
	139	}
	140	if (cur != .none) {
	141	self.linked_list.items(.next)[@intFromEnum(cur)] = b;
	142	}
	143	return head;
	144	}
	145
	146	pub fn contains(self: *const Hideset, list: Index, str: []const u8) bool {
	147	var it = self.iterator(list);
	148	while (it.next()) \|identifier\| {
	149	if (mem.eql(u8, str, identifier.slice(self.comp))) return true;
	150	}
	151	return false;
	152	}
	153
	154	fn len(self: *const Hideset, list: Index) usize {
	155	const nexts = self.linked_list.items(.next);
	156	var cur = list;
	157	var count: usize = 0;
	158	while (cur != .none) : (count += 1) {
	159	cur = nexts[@intFromEnum(cur)];
	160	}
	161	return count;
	162	}
	163
	164	pub fn intersection(self: *Hideset, a: Index, b: Index) !Index {
	165	if (a == .none or b == .none) return .none;
	166	self.intersection_map.clearRetainingCapacity();
	167
	168	var cur: Index = .none;
	169	var head: Index = .none;
	170	var it = self.iterator(a);
	171	var a_len: usize = 0;
	172	while (it.next()) \|identifier\| : (a_len += 1) {
	173	try self.intersection_map.put(self.comp.gpa, identifier, {});
	174	}
	175	try self.ensureUnusedCapacity(@min(a_len, self.len(b)));
	176
	177	it = self.iterator(b);
	178	while (it.next()) \|identifier\| {
	179	if (self.intersection_map.contains(identifier)) {
	180	const new_idx = self.createNodeAssumeCapacity(identifier);
	181	if (head == .none) {
	182	head = new_idx;
	183	}
	184	if (cur != .none) {
	185	self.linked_list.items(.next)[@intFromEnum(cur)] = new_idx;
	186	}
	187	cur = new_idx;
	188	}
	189	}
	190	return head;
	191	}

lib/compiler/aro/aro/LangOpts.zig+5

...	...	@@ -135,6 +135,11 @@ preserve_comments: bool = false,
135	135	/// Preserve comments in macros when preprocessing
136	136	preserve_comments_in_macros: bool = false,
137	137
	138	/// Used ONLY for generating __GNUC__ and related macros. Does not control the presence/absence of any features
	139	/// Encoded as major * 10,000 + minor * 100 + patch
	140	/// e.g. 4.2.1 == 40201
	141	gnuc_version: u32 = 0,
	142
138	143	pub fn setStandard(self: *LangOpts, name: []const u8) error{InvalidStandard}!void {
139	144	self.standard = Standard.NameMap.get(name) orelse return error.InvalidStandard;
140	145	}

lib/compiler/aro/aro/Parser.zig+171-30

...	...	@@ -403,7 +403,7 @@ pub fn errExtra(p: *Parser, tag: Diagnostics.Tag, tok_i: TokenIndex, extra: Diag
403	403	.tag = tag,
404	404	.loc = loc,
405	405	.extra = extra,
406		}, tok.expansionSlice());
	406	}, p.pp.expansionSlice(tok_i));
407	407	}
408	408
409	409	pub fn errTok(p: *Parser, tag: Diagnostics.Tag, tok_i: TokenIndex) Compilation.Error!void {
...	...	@@ -432,6 +432,11 @@ pub fn removeNull(p: *Parser, str: Value) !Value {
432	432	}
433	433
434	434	pub fn typeStr(p: *Parser, ty: Type) ![]const u8 {
	435	if (@import("builtin").mode != .Debug) {
	436	if (ty.is(.invalid)) {
	437	return "Tried to render invalid type - this is an aro bug.";
	438	}
	439	}
435	440	if (Type.Builder.fromType(ty).str(p.comp.langopts)) \|str\| return str;
436	441	const strings_top = p.strings.items.len;
437	442	defer p.strings.items.len = strings_top;
...	...	@@ -446,6 +451,11 @@ pub fn typePairStr(p: *Parser, a: Type, b: Type) ![]const u8 {
446	451	}
447	452
448	453	pub fn typePairStrExtra(p: *Parser, a: Type, msg: []const u8, b: Type) ![]const u8 {
	454	if (@import("builtin").mode != .Debug) {
	455	if (a.is(.invalid) or b.is(.invalid)) {
	456	return "Tried to render invalid type - this is an aro bug.";
	457	}
	458	}
449	459	const strings_top = p.strings.items.len;
450	460	defer p.strings.items.len = strings_top;
451	461
...	...	@@ -635,7 +645,6 @@ fn diagnoseIncompleteDefinitions(p: *Parser) !void {
635	645	const tys = node_slices.items(.ty);
636	646	const data = node_slices.items(.data);
637	647
638		const err_start = p.comp.diagnostics.list.items.len;
639	648	for (p.decl_buf.items) \|decl_node\| {
640	649	const idx = @intFromEnum(decl_node);
641	650	switch (tags[idx]) {
...	...	@@ -656,8 +665,6 @@ fn diagnoseIncompleteDefinitions(p: *Parser) !void {
656	665	try p.errStr(.tentative_definition_incomplete, tentative_def_tok, type_str);
657	666	try p.errStr(.forward_declaration_here, data[idx].decl_ref, type_str);
658	667	}
659		const errors_added = p.comp.diagnostics.list.items.len - err_start;
660		assert(errors_added == 2 * p.tentative_defs.count()); // Each tentative def should add an error + note
661	668	}
662	669
663	670	/// root : (decl \| assembly ';' \| staticAssert)*
...	...	@@ -2201,7 +2208,15 @@ fn recordSpec(p: *Parser) Error!Type {
2201	2208	} else {
2202	2209	record_ty.fields = try p.arena.dupe(Type.Record.Field, p.record_buf.items[record_buf_top..]);
2203	2210	}
2204		if (old_field_attr_start < p.field_attr_buf.items.len) {
	2211	const attr_count = p.field_attr_buf.items.len - old_field_attr_start;
	2212	const record_decls = p.decl_buf.items[decl_buf_top..];
	2213	if (attr_count > 0) {
	2214	if (attr_count != record_decls.len) {
	2215	// A mismatch here means that non-field decls were parsed. This can happen if there were
	2216	// parse errors during attribute parsing. Bail here because if there are any field attributes,
	2217	// there must be exactly one per field.
	2218	return error.ParsingFailed;
	2219	}
2205	2220	const field_attr_slice = p.field_attr_buf.items[old_field_attr_start..];
2206	2221	const duped = try p.arena.dupe([]const Attribute, field_attr_slice);
2207	2222	record_ty.field_attributes = duped.ptr;
...	...	@@ -2242,7 +2257,6 @@ fn recordSpec(p: *Parser) Error!Type {
2242	2257	.ty = ty,
2243	2258	.data = .{ .bin = .{ .lhs = .none, .rhs = .none } },
2244	2259	};
2245		const record_decls = p.decl_buf.items[decl_buf_top..];
2246	2260	switch (record_decls.len) {
2247	2261	0 => {},
2248	2262	1 => node.data = .{ .bin = .{ .lhs = record_decls[0], .rhs = .none } },
...	...	@@ -2560,6 +2574,7 @@ fn enumSpec(p: *Parser) Error!Type {
2560	2574	if (field.ty.eql(Type.int, p.comp, false)) continue;
2561	2575
2562	2576	const sym = p.syms.get(field.name, .vars) orelse continue;
	2577	if (sym.kind != .enumeration) continue; // already an error
2563	2578
2564	2579	var res = Result{ .node = field.node, .ty = field.ty, .val = sym.val };
2565	2580	const dest_ty = if (p.comp.fixedEnumTagSpecifier()) \|some\|
...	...	@@ -4603,24 +4618,31 @@ fn nodeIsNoreturn(p: *Parser, node: NodeIndex) NoreturnKind {
4603	4618	},
4604	4619	.compound_stmt_two => {
4605	4620	const data = p.nodes.items(.data)[@intFromEnum(node)];
4606		if (data.bin.rhs != .none) return p.nodeIsNoreturn(data.bin.rhs);
4607		if (data.bin.lhs != .none) return p.nodeIsNoreturn(data.bin.lhs);
	4621	const lhs_type = if (data.bin.lhs != .none) p.nodeIsNoreturn(data.bin.lhs) else .no;
	4622	const rhs_type = if (data.bin.rhs != .none) p.nodeIsNoreturn(data.bin.rhs) else .no;
	4623	if (lhs_type == .complex or rhs_type == .complex) return .complex;
	4624	if (lhs_type == .yes or rhs_type == .yes) return .yes;
4608	4625	return .no;
4609	4626	},
4610	4627	.compound_stmt => {
4611	4628	const data = p.nodes.items(.data)[@intFromEnum(node)];
4612		return p.nodeIsNoreturn(p.data.items[data.range.end - 1]);
	4629	var it = data.range.start;
	4630	while (it != data.range.end) : (it += 1) {
	4631	const kind = p.nodeIsNoreturn(p.data.items[it]);
	4632	if (kind != .no) return kind;
	4633	}
	4634	return .no;
4613	4635	},
4614	4636	.labeled_stmt => {
4615	4637	const data = p.nodes.items(.data)[@intFromEnum(node)];
4616	4638	return p.nodeIsNoreturn(data.decl.node);
4617	4639	},
4618		.switch_stmt => {
	4640	.default_stmt => {
4619	4641	const data = p.nodes.items(.data)[@intFromEnum(node)];
4620		if (data.bin.rhs == .none) return .complex;
4621		if (p.nodeIsNoreturn(data.bin.rhs) == .yes) return .yes;
4622		return .complex;
	4642	if (data.un == .none) return .no;
	4643	return p.nodeIsNoreturn(data.un);
4623	4644	},
	4645	.while_stmt, .do_while_stmt, .for_decl_stmt, .forever_stmt, .for_stmt, .switch_stmt => return .complex,
4624	4646	else => return .no,
4625	4647	}
4626	4648	}
...	...	@@ -4787,7 +4809,11 @@ const CallExpr = union(enum) {
4787	4809	Builtin.tagFromName("__va_start").?,
4788	4810	Builtin.tagFromName("va_start").?,
4789	4811	=> arg_idx != 1,
4790		Builtin.tagFromName("__builtin_complex").? => false,
	4812	Builtin.tagFromName("__builtin_complex").?,
	4813	Builtin.tagFromName("__builtin_add_overflow").?,
	4814	Builtin.tagFromName("__builtin_sub_overflow").?,
	4815	Builtin.tagFromName("__builtin_mul_overflow").?,
	4816	=> false,
4791	4817	else => true,
4792	4818	},
4793	4819	};
...	...	@@ -4800,6 +4826,7 @@ const CallExpr = union(enum) {
4800	4826	}
4801	4827
4802	4828	fn checkVarArg(self: CallExpr, p: Parser, first_after: TokenIndex, param_tok: TokenIndex, arg: Result, arg_idx: u32) !void {
	4829	@setEvalBranchQuota(10_000);
4803	4830	if (self == .standard) return;
4804	4831
4805	4832	const builtin_tok = p.nodes.items(.data)[@intFromEnum(self.builtin.node)].decl.name;
...	...	@@ -4809,6 +4836,11 @@ const CallExpr = union(enum) {
4809	4836	Builtin.tagFromName("va_start").?,
4810	4837	=> return p.checkVaStartArg(builtin_tok, first_after, param_tok, arg, arg_idx),
4811	4838	Builtin.tagFromName("__builtin_complex").? => return p.checkComplexArg(builtin_tok, first_after, param_tok, arg, arg_idx),
	4839	Builtin.tagFromName("__builtin_add_overflow").?,
	4840	Builtin.tagFromName("__builtin_sub_overflow").?,
	4841	Builtin.tagFromName("__builtin_mul_overflow").?,
	4842	=> return p.checkArithOverflowArg(builtin_tok, first_after, param_tok, arg, arg_idx),
	4843
4812	4844	else => {},
4813	4845	}
4814	4846	}
...	...	@@ -4823,16 +4855,44 @@ const CallExpr = union(enum) {
4823	4855	return switch (self) {
4824	4856	.standard => null,
4825	4857	.builtin => \|builtin\| switch (builtin.tag) {
4826		Builtin.tagFromName("__builtin_complex").? => 2,
4827
	4858	Builtin.tagFromName("__c11_atomic_thread_fence").?,
	4859	Builtin.tagFromName("__c11_atomic_signal_fence").?,
	4860	Builtin.tagFromName("__c11_atomic_is_lock_free").?,
	4861	=> 1,
	4862
	4863	Builtin.tagFromName("__builtin_complex").?,
	4864	Builtin.tagFromName("__c11_atomic_load").?,
	4865	Builtin.tagFromName("__c11_atomic_init").?,
	4866	=> 2,
	4867
	4868	Builtin.tagFromName("__c11_atomic_store").?,
	4869	Builtin.tagFromName("__c11_atomic_exchange").?,
	4870	Builtin.tagFromName("__c11_atomic_fetch_add").?,
	4871	Builtin.tagFromName("__c11_atomic_fetch_sub").?,
	4872	Builtin.tagFromName("__c11_atomic_fetch_or").?,
	4873	Builtin.tagFromName("__c11_atomic_fetch_xor").?,
	4874	Builtin.tagFromName("__c11_atomic_fetch_and").?,
4828	4875	Builtin.tagFromName("__atomic_fetch_add").?,
4829	4876	Builtin.tagFromName("__atomic_fetch_sub").?,
4830	4877	Builtin.tagFromName("__atomic_fetch_and").?,
4831	4878	Builtin.tagFromName("__atomic_fetch_xor").?,
4832	4879	Builtin.tagFromName("__atomic_fetch_or").?,
4833	4880	Builtin.tagFromName("__atomic_fetch_nand").?,
	4881	Builtin.tagFromName("__atomic_add_fetch").?,
	4882	Builtin.tagFromName("__atomic_sub_fetch").?,
	4883	Builtin.tagFromName("__atomic_and_fetch").?,
	4884	Builtin.tagFromName("__atomic_xor_fetch").?,
	4885	Builtin.tagFromName("__atomic_or_fetch").?,
	4886	Builtin.tagFromName("__atomic_nand_fetch").?,
	4887	Builtin.tagFromName("__builtin_add_overflow").?,
	4888	Builtin.tagFromName("__builtin_sub_overflow").?,
	4889	Builtin.tagFromName("__builtin_mul_overflow").?,
4834	4890	=> 3,
4835	4891
	4892	Builtin.tagFromName("__c11_atomic_compare_exchange_strong").?,
	4893	Builtin.tagFromName("__c11_atomic_compare_exchange_weak").?,
	4894	=> 5,
	4895
4836	4896	Builtin.tagFromName("__atomic_compare_exchange").?,
4837	4897	Builtin.tagFromName("__atomic_compare_exchange_n").?,
4838	4898	=> 6,
...	...	@@ -4845,15 +4905,45 @@ const CallExpr = union(enum) {
4845	4905	return switch (self) {
4846	4906	.standard => callable_ty.returnType(),
4847	4907	.builtin => \|builtin\| switch (builtin.tag) {
	4908	Builtin.tagFromName("__c11_atomic_exchange").? => {
	4909	if (p.list_buf.items.len != 4) return Type.invalid; // wrong number of arguments; already an error
	4910	const second_param = p.list_buf.items[2];
	4911	return p.nodes.items(.ty)[@intFromEnum(second_param)];
	4912	},
	4913	Builtin.tagFromName("__c11_atomic_load").? => {
	4914	if (p.list_buf.items.len != 3) return Type.invalid; // wrong number of arguments; already an error
	4915	const first_param = p.list_buf.items[1];
	4916	const ty = p.nodes.items(.ty)[@intFromEnum(first_param)];
	4917	if (!ty.isPtr()) return Type.invalid;
	4918	return ty.elemType();
	4919	},
	4920
4848	4921	Builtin.tagFromName("__atomic_fetch_add").?,
	4922	Builtin.tagFromName("__atomic_add_fetch").?,
	4923	Builtin.tagFromName("__c11_atomic_fetch_add").?,
	4924
4849	4925	Builtin.tagFromName("__atomic_fetch_sub").?,
	4926	Builtin.tagFromName("__atomic_sub_fetch").?,
	4927	Builtin.tagFromName("__c11_atomic_fetch_sub").?,
	4928
4850	4929	Builtin.tagFromName("__atomic_fetch_and").?,
	4930	Builtin.tagFromName("__atomic_and_fetch").?,
	4931	Builtin.tagFromName("__c11_atomic_fetch_and").?,
	4932
4851	4933	Builtin.tagFromName("__atomic_fetch_xor").?,
	4934	Builtin.tagFromName("__atomic_xor_fetch").?,
	4935	Builtin.tagFromName("__c11_atomic_fetch_xor").?,
	4936
4852	4937	Builtin.tagFromName("__atomic_fetch_or").?,
	4938	Builtin.tagFromName("__atomic_or_fetch").?,
	4939	Builtin.tagFromName("__c11_atomic_fetch_or").?,
	4940
4853	4941	Builtin.tagFromName("__atomic_fetch_nand").?,
	4942	Builtin.tagFromName("__atomic_nand_fetch").?,
	4943	Builtin.tagFromName("__c11_atomic_fetch_nand").?,
4854	4944	=> {
4855		if (p.list_buf.items.len < 2) return Type.invalid; // not enough arguments; already an error
4856		const second_param = p.list_buf.items[p.list_buf.items.len - 2];
	4945	if (p.list_buf.items.len != 3) return Type.invalid; // wrong number of arguments; already an error
	4946	const second_param = p.list_buf.items[2];
4857	4947	return p.nodes.items(.ty)[@intFromEnum(second_param)];
4858	4948	},
4859	4949	Builtin.tagFromName("__builtin_complex").? => {
...	...	@@ -4863,8 +4953,17 @@ const CallExpr = union(enum) {
4863	4953	},
4864	4954	Builtin.tagFromName("__atomic_compare_exchange").?,
4865	4955	Builtin.tagFromName("__atomic_compare_exchange_n").?,
	4956	Builtin.tagFromName("__c11_atomic_is_lock_free").?,
4866	4957	=> .{ .specifier = .bool },
4867	4958	else => callable_ty.returnType(),
	4959
	4960	Builtin.tagFromName("__c11_atomic_compare_exchange_strong").?,
	4961	Builtin.tagFromName("__c11_atomic_compare_exchange_weak").?,
	4962	=> {
	4963	if (p.list_buf.items.len != 6) return Type.invalid; // wrong number of arguments
	4964	const third_param = p.list_buf.items[3];
	4965	return p.nodes.items(.ty)[@intFromEnum(third_param)];
	4966	},
4868	4967	},
4869	4968	};
4870	4969	}
...	...	@@ -4975,15 +5074,19 @@ pub const Result = struct {
4975	5074	.call_expr_one => {
4976	5075	const fn_ptr = p.nodes.items(.data)[@intFromEnum(cur_node)].bin.lhs;
4977	5076	const fn_ty = p.nodes.items(.ty)[@intFromEnum(fn_ptr)].elemType();
4978		if (fn_ty.hasAttribute(.nodiscard)) try p.errStr(.nodiscard_unused, expr_start, "TODO get name");
4979		if (fn_ty.hasAttribute(.warn_unused_result)) try p.errStr(.warn_unused_result, expr_start, "TODO get name");
	5077	const cast_info = p.nodes.items(.data)[@intFromEnum(fn_ptr)].cast.operand;
	5078	const decl_ref = p.nodes.items(.data)[@intFromEnum(cast_info)].decl_ref;
	5079	if (fn_ty.hasAttribute(.nodiscard)) try p.errStr(.nodiscard_unused, expr_start, p.tokSlice(decl_ref));
	5080	if (fn_ty.hasAttribute(.warn_unused_result)) try p.errStr(.warn_unused_result, expr_start, p.tokSlice(decl_ref));
4980	5081	return;
4981	5082	},
4982	5083	.call_expr => {
4983	5084	const fn_ptr = p.data.items[p.nodes.items(.data)[@intFromEnum(cur_node)].range.start];
4984	5085	const fn_ty = p.nodes.items(.ty)[@intFromEnum(fn_ptr)].elemType();
4985		if (fn_ty.hasAttribute(.nodiscard)) try p.errStr(.nodiscard_unused, expr_start, "TODO get name");
4986		if (fn_ty.hasAttribute(.warn_unused_result)) try p.errStr(.warn_unused_result, expr_start, "TODO get name");
	5086	const cast_info = p.nodes.items(.data)[@intFromEnum(fn_ptr)].cast.operand;
	5087	const decl_ref = p.nodes.items(.data)[@intFromEnum(cast_info)].decl_ref;
	5088	if (fn_ty.hasAttribute(.nodiscard)) try p.errStr(.nodiscard_unused, expr_start, p.tokSlice(decl_ref));
	5089	if (fn_ty.hasAttribute(.warn_unused_result)) try p.errStr(.warn_unused_result, expr_start, p.tokSlice(decl_ref));
4987	5090	return;
4988	5091	},
4989	5092	.stmt_expr => {
...	...	@@ -6356,8 +6459,15 @@ fn shiftExpr(p: *Parser) Error!Result {
6356	6459	try rhs.expect(p);
6357	6460
6358	6461	if (try lhs.adjustTypes(shr.?, &rhs, p, .integer)) {
	6462	if (rhs.val.compare(.lt, Value.zero, p.comp)) {
	6463	try p.errStr(.negative_shift_count, shl orelse shr.?, try rhs.str(p));
	6464	}
	6465	if (rhs.val.compare(.gte, try Value.int(lhs.ty.bitSizeof(p.comp).?, p.comp), p.comp)) {
	6466	try p.errStr(.too_big_shift_count, shl orelse shr.?, try rhs.str(p));
	6467	}
6359	6468	if (shl != null) {
6360		if (try lhs.val.shl(lhs.val, rhs.val, lhs.ty, p.comp)) try p.errOverflow(shl.?, lhs);
	6469	if (try lhs.val.shl(lhs.val, rhs.val, lhs.ty, p.comp) and
	6470	lhs.ty.signedness(p.comp) != .unsigned) try p.errOverflow(shl.?, lhs);
6361	6471	} else {
6362	6472	lhs.val = try lhs.val.shr(rhs.val, lhs.ty, p.comp);
6363	6473	}
...	...	@@ -6381,9 +6491,11 @@ fn addExpr(p: *Parser) Error!Result {
6381	6491	const lhs_ty = lhs.ty;
6382	6492	if (try lhs.adjustTypes(minus.?, &rhs, p, if (plus != null) .add else .sub)) {
6383	6493	if (plus != null) {
6384		if (try lhs.val.add(lhs.val, rhs.val, lhs.ty, p.comp)) try p.errOverflow(plus.?, lhs);
	6494	if (try lhs.val.add(lhs.val, rhs.val, lhs.ty, p.comp) and
	6495	lhs.ty.signedness(p.comp) != .unsigned) try p.errOverflow(plus.?, lhs);
6385	6496	} else {
6386		if (try lhs.val.sub(lhs.val, rhs.val, lhs.ty, p.comp)) try p.errOverflow(minus.?, lhs);
	6497	if (try lhs.val.sub(lhs.val, rhs.val, lhs.ty, p.comp) and
	6498	lhs.ty.signedness(p.comp) != .unsigned) try p.errOverflow(minus.?, lhs);
6387	6499	}
6388	6500	}
6389	6501	if (lhs.ty.specifier != .invalid and lhs_ty.isPtr() and !lhs_ty.isVoidStar() and lhs_ty.elemType().hasIncompleteSize()) {
...	...	@@ -6420,9 +6532,11 @@ fn mulExpr(p: *Parser) Error!Result {
6420	6532
6421	6533	if (try lhs.adjustTypes(percent.?, &rhs, p, if (tag == .mod_expr) .integer else .arithmetic)) {
6422	6534	if (mul != null) {
6423		if (try lhs.val.mul(lhs.val, rhs.val, lhs.ty, p.comp)) try p.errOverflow(mul.?, lhs);
	6535	if (try lhs.val.mul(lhs.val, rhs.val, lhs.ty, p.comp) and
	6536	lhs.ty.signedness(p.comp) != .unsigned) try p.errOverflow(mul.?, lhs);
6424	6537	} else if (div != null) {
6425		if (try lhs.val.div(lhs.val, rhs.val, lhs.ty, p.comp)) try p.errOverflow(mul.?, lhs);
	6538	if (try lhs.val.div(lhs.val, rhs.val, lhs.ty, p.comp) and
	6539	lhs.ty.signedness(p.comp) != .unsigned) try p.errOverflow(mul.?, lhs);
6426	6540	} else {
6427	6541	var res = try Value.rem(lhs.val, rhs.val, lhs.ty, p.comp);
6428	6542	if (res.opt_ref == .none) {
...	...	@@ -6827,7 +6941,7 @@ fn unExpr(p: *Parser) Error!Result {
6827	6941	try p.errStr(.invalid_argument_un, tok, try p.typeStr(operand.ty));
6828	6942
6829	6943	try operand.usualUnaryConversion(p, tok);
6830		if (operand.val.is(.int, p.comp)) {
	6944	if (operand.val.is(.int, p.comp) or operand.val.is(.float, p.comp)) {
6831	6945	_ = try operand.val.sub(Value.zero, operand.val, operand.ty, p.comp);
6832	6946	} else {
6833	6947	operand.val = .{};
...	...	@@ -6898,6 +7012,8 @@ fn unExpr(p: *Parser) Error!Result {
6898	7012	if (operand.val.is(.int, p.comp)) {
6899	7013	operand.val = try operand.val.bitNot(operand.ty, p.comp);
6900	7014	}
	7015	} else if (operand.ty.isComplex()) {
	7016	try p.errStr(.complex_conj, tok, try p.typeStr(operand.ty));
6901	7017	} else {
6902	7018	try p.errStr(.invalid_argument_un, tok, try p.typeStr(operand.ty));
6903	7019	operand.val = .{};
...	...	@@ -7334,6 +7450,20 @@ fn checkVaStartArg(p: *Parser, builtin_tok: TokenIndex, first_after: TokenIndex,
7334	7450	}
7335	7451	}
7336	7452
	7453	fn checkArithOverflowArg(p: Parser, builtin_tok: TokenIndex, first_after: TokenIndex, param_tok: TokenIndex, arg: Result, idx: u32) !void {
	7454	_ = builtin_tok;
	7455	_ = first_after;
	7456	if (idx <= 1) {
	7457	if (!arg.ty.isInt()) {
	7458	return p.errStr(.overflow_builtin_requires_int, param_tok, try p.typeStr(arg.ty));
	7459	}
	7460	} else if (idx == 2) {
	7461	if (!arg.ty.isPtr()) return p.errStr(.overflow_result_requires_ptr, param_tok, try p.typeStr(arg.ty));
	7462	const child = arg.ty.elemType();
	7463	if (!child.isInt() or child.is(.bool) or child.is(.@"enum") or child.qual.@"const") return p.errStr(.overflow_result_requires_ptr, param_tok, try p.typeStr(arg.ty));
	7464	}
	7465	}
	7466
7337	7467	fn checkComplexArg(p: Parser, builtin_tok: TokenIndex, first_after: TokenIndex, param_tok: TokenIndex, arg: Result, idx: u32) !void {
7338	7468	_ = builtin_tok;
7339	7469	_ = first_after;
...	...	@@ -7880,6 +8010,7 @@ fn charLiteral(p: *Parser) Error!Result {
7880	8010
7881	8011	const slice = char_kind.contentSlice(p.tokSlice(p.tok_i));
7882	8012
	8013	var is_multichar = false;
7883	8014	if (slice.len == 1 and std.ascii.isASCII(slice[0])) {
7884	8015	// fast path: single unescaped ASCII char
7885	8016	val = slice[0];
...	...	@@ -7913,7 +8044,7 @@ fn charLiteral(p: *Parser) Error!Result {
7913	8044	},
7914	8045	};
7915	8046
7916		const is_multichar = chars.items.len > 1;
	8047	is_multichar = chars.items.len > 1;
7917	8048	if (is_multichar) {
7918	8049	if (char_kind == .char and chars.items.len == 4) {
7919	8050	char_literal_parser.warn(.four_char_char_literal, .{ .none = {} });
...	...	@@ -7956,9 +8087,19 @@ fn charLiteral(p: *Parser) Error!Result {
7956	8087	else
7957	8088	p.comp.types.intmax;
7958	8089
	8090	var value = try Value.int(val, p.comp);
	8091	// C99 6.4.4.4.10
	8092	// > If an integer character constant contains a single character or escape sequence,
	8093	// > its value is the one that results when an object with type char whose value is
	8094	// > that of the single character or escape sequence is converted to type int.
	8095	// This conversion only matters if `char` is signed and has a high-order bit of `1`
	8096	if (char_kind == .char and !is_multichar and val > 0x7F and p.comp.getCharSignedness() == .signed) {
	8097	try value.intCast(.{ .specifier = .char }, p.comp);
	8098	}
	8099
7959	8100	const res = Result{
7960	8101	.ty = if (p.in_macro) macro_ty else ty,
7961		.val = try Value.int(val, p.comp),
	8102	.val = value,
7962	8103	.node = try p.addNode(.{ .tag = .char_literal, .ty = ty, .data = undefined }),
7963	8104	};
7964	8105	if (!p.in_macro) try p.value_map.put(res.node, res.val);

lib/compiler/aro/aro/Preprocessor.zig+269-164

...	...	@@ -9,9 +9,12 @@ const Tokenizer = @import("Tokenizer.zig");
9	9	const RawToken = Tokenizer.Token;
10	10	const Parser = @import("Parser.zig");
11	11	const Diagnostics = @import("Diagnostics.zig");
12		const Token = @import("Tree.zig").Token;
	12	const Tree = @import("Tree.zig");
	13	const Token = Tree.Token;
	14	const TokenWithExpansionLocs = Tree.TokenWithExpansionLocs;
13	15	const Attribute = @import("Attribute.zig");
14	16	const features = @import("features.zig");
	17	const Hideset = @import("Hideset.zig");
15	18
16	19	const DefineMap = std.StringHashMapUnmanaged(Macro);
17	20	const RawTokenList = std.ArrayList(RawToken);
...	...	@@ -40,8 +43,6 @@ const Macro = struct {
40	43
41	44	/// Location of macro in the source
42	45	loc: Source.Location,
43		start: u32,
44		end: u32,
45	46
46	47	fn eql(a: Macro, b: Macro, pp: *Preprocessor) bool {
47	48	if (a.tokens.len != b.tokens.len) return false;
...	...	@@ -64,11 +65,24 @@ const Macro = struct {
64	65
65	66	const Preprocessor = @This();
66	67
	68	const ExpansionEntry = struct {
	69	idx: Tree.TokenIndex,
	70	locs: [*]Source.Location,
	71	};
	72
	73	const TokenState = struct {
	74	tokens_len: usize,
	75	expansion_entries_len: usize,
	76	};
	77
67	78	comp: *Compilation,
68	79	gpa: mem.Allocator,
69	80	arena: std.heap.ArenaAllocator,
70	81	defines: DefineMap = .{},
	82	/// Do not directly mutate this; use addToken / addTokenAssumeCapacity / ensureTotalTokenCapacity / ensureUnusedTokenCapacity
71	83	tokens: Token.List = .{},
	84	/// Do not directly mutate this; must be kept in sync with `tokens`
	85	expansion_entries: std.MultiArrayList(ExpansionEntry) = .{},
72	86	token_buf: RawTokenList,
73	87	char_buf: std.ArrayList(u8),
74	88	/// Counter that is incremented each time preprocess() is called
...	...	@@ -93,6 +107,8 @@ preserve_whitespace: bool = false,
93	107	/// linemarker tokens. Must be .none unless in -E mode (parser does not handle linemarkers)
94	108	linemarkers: Linemarkers = .none,
95	109
	110	hideset: Hideset,
	111
96	112	pub const parse = Parser.parse;
97	113
98	114	pub const Linemarkers = enum {
...	...	@@ -113,6 +129,7 @@ pub fn init(comp: *Compilation) Preprocessor {
113	129	.char_buf = std.ArrayList(u8).init(comp.gpa),
114	130	.poisoned_identifiers = std.StringHashMap(void).init(comp.gpa),
115	131	.top_expansion_buf = ExpandBuf.init(comp.gpa),
	132	.hideset = .{ .comp = comp },
116	133	};
117	134	comp.pragmaEvent(.before_preprocess);
118	135	return pp;
...	...	@@ -201,8 +218,6 @@ fn addBuiltinMacro(pp: *Preprocessor, name: []const u8, is_func: bool, tokens: [
201	218	.var_args = false,
202	219	.is_func = is_func,
203	220	.loc = .{ .id = .generated },
204		.start = 0,
205		.end = 0,
206	221	.is_builtin = true,
207	222	});
208	223	}
...	...	@@ -228,7 +243,6 @@ pub fn addBuiltinMacros(pp: *Preprocessor) !void {
228	243
229	244	pub fn deinit(pp: *Preprocessor) void {
230	245	pp.defines.deinit(pp.gpa);
231		for (pp.tokens.items(.expansion_locs)) \|loc\| Token.free(loc, pp.gpa);
232	246	pp.tokens.deinit(pp.gpa);
233	247	pp.arena.deinit();
234	248	pp.token_buf.deinit();
...	...	@@ -236,6 +250,33 @@ pub fn deinit(pp: *Preprocessor) void {
236	250	pp.poisoned_identifiers.deinit();
237	251	pp.include_guards.deinit(pp.gpa);
238	252	pp.top_expansion_buf.deinit();
	253	pp.hideset.deinit();
	254	for (pp.expansion_entries.items(.locs)) \|locs\| TokenWithExpansionLocs.free(locs, pp.gpa);
	255	pp.expansion_entries.deinit(pp.gpa);
	256	}
	257
	258	/// Free buffers that are not needed after preprocessing
	259	fn clearBuffers(pp: *Preprocessor) void {
	260	pp.token_buf.clearAndFree();
	261	pp.char_buf.clearAndFree();
	262	pp.top_expansion_buf.clearAndFree();
	263	pp.hideset.clearAndFree();
	264	}
	265
	266	pub fn expansionSlice(pp: *Preprocessor, tok: Tree.TokenIndex) []Source.Location {
	267	const S = struct {
	268	fn order_token_index(context: void, lhs: Tree.TokenIndex, rhs: Tree.TokenIndex) std.math.Order {
	269	_ = context;
	270	return std.math.order(lhs, rhs);
	271	}
	272	};
	273
	274	const indices = pp.expansion_entries.items(.idx);
	275	const idx = std.sort.binarySearch(Tree.TokenIndex, tok, indices, {}, S.order_token_index) orelse return &.{};
	276	const locs = pp.expansion_entries.items(.locs)[idx];
	277	var i: usize = 0;
	278	while (locs[i].id != .unused) : (i += 1) {}
	279	return locs[0..i];
239	280	}
240	281
241	282	/// Preprocess a compilation unit of sources into a parsable list of tokens.
...	...	@@ -247,13 +288,14 @@ pub fn preprocessSources(pp: *Preprocessor, sources: []const Source) Error!void
247	288	try pp.addIncludeStart(header);
248	289	_ = try pp.preprocess(header);
249	290	}
250		try pp.addIncludeResume(first.id, 0, 0);
	291	try pp.addIncludeResume(first.id, 0, 1);
251	292	const eof = try pp.preprocess(first);
252		try pp.tokens.append(pp.comp.gpa, eof);
	293	try pp.addToken(eof);
	294	pp.clearBuffers();
253	295	}
254	296
255	297	/// Preprocess a source file, returns eof token.
256		pub fn preprocess(pp: *Preprocessor, source: Source) Error!Token {
	298	pub fn preprocess(pp: *Preprocessor, source: Source) Error!TokenWithExpansionLocs {
257	299	const eof = pp.preprocessExtra(source) catch \|er\| switch (er) {
258	300	// This cannot occur in the main file and is handled in `include`.
259	301	error.StopPreprocessing => unreachable,
...	...	@@ -275,27 +317,27 @@ pub fn tokenize(pp: *Preprocessor, source: Source) Error!Token {
275	317
276	318	// Estimate how many new tokens this source will contain.
277	319	const estimated_token_count = source.buf.len / 8;
278		try pp.tokens.ensureTotalCapacity(pp.gpa, pp.tokens.len + estimated_token_count);
	320	try pp.ensureTotalTokenCapacity(pp.tokens.len + estimated_token_count);
279	321
280	322	while (true) {
281	323	const tok = tokenizer.next();
282	324	if (tok.id == .eof) return tokFromRaw(tok);
283		try pp.tokens.append(pp.gpa, tokFromRaw(tok));
	325	try pp.addToken(tokFromRaw(tok));
284	326	}
285	327	}
286	328
287	329	pub fn addIncludeStart(pp: *Preprocessor, source: Source) !void {
288	330	if (pp.linemarkers == .none) return;
289		try pp.tokens.append(pp.gpa, .{ .id = .include_start, .loc = .{
	331	try pp.addToken(.{ .id = .include_start, .loc = .{
290	332	.id = source.id,
291	333	.byte_offset = std.math.maxInt(u32),
292		.line = 0,
	334	.line = 1,
293	335	} });
294	336	}
295	337
296	338	pub fn addIncludeResume(pp: *Preprocessor, source: Source.Id, offset: u32, line: u32) !void {
297	339	if (pp.linemarkers == .none) return;
298		try pp.tokens.append(pp.gpa, .{ .id = .include_resume, .loc = .{
	340	try pp.addToken(.{ .id = .include_resume, .loc = .{
299	341	.id = source,
300	342	.byte_offset = offset,
301	343	.line = line,
...	...	@@ -328,7 +370,7 @@ fn findIncludeGuard(pp: *Preprocessor, source: Source) ?[]const u8 {
328	370	return pp.tokSlice(guard);
329	371	}
330	372
331		fn preprocessExtra(pp: *Preprocessor, source: Source) MacroError!Token {
	373	fn preprocessExtra(pp: *Preprocessor, source: Source) MacroError!TokenWithExpansionLocs {
332	374	var guard_name = pp.findIncludeGuard(source);
333	375
334	376	pp.preprocess_count += 1;
...	...	@@ -340,7 +382,7 @@ fn preprocessExtra(pp: *Preprocessor, source: Source) MacroError!Token {
340	382
341	383	// Estimate how many new tokens this source will contain.
342	384	const estimated_token_count = source.buf.len / 8;
343		try pp.tokens.ensureTotalCapacity(pp.gpa, pp.tokens.len + estimated_token_count);
	385	try pp.ensureTotalTokenCapacity(pp.tokens.len + estimated_token_count);
344	386
345	387	var if_level: u8 = 0;
346	388	var if_kind = std.PackedIntArray(u2, 256).init([1]u2{0} ** 256);
...	...	@@ -352,7 +394,7 @@ fn preprocessExtra(pp: *Preprocessor, source: Source) MacroError!Token {
352	394	while (true) {
353	395	var tok = tokenizer.next();
354	396	switch (tok.id) {
355		.hash => if (!start_of_line) try pp.tokens.append(pp.gpa, tokFromRaw(tok)) else {
	397	.hash => if (!start_of_line) try pp.addToken(tokFromRaw(tok)) else {
356	398	const directive = tokenizer.nextNoWS();
357	399	switch (directive.id) {
358	400	.keyword_error, .keyword_warning => {
...	...	@@ -654,13 +696,13 @@ fn preprocessExtra(pp: *Preprocessor, source: Source) MacroError!Token {
654	696	}
655	697	if (pp.preserve_whitespace) {
656	698	tok.id = .nl;
657		try pp.tokens.append(pp.gpa, tokFromRaw(tok));
	699	try pp.addToken(tokFromRaw(tok));
658	700	}
659	701	},
660		.whitespace => if (pp.preserve_whitespace) try pp.tokens.append(pp.gpa, tokFromRaw(tok)),
	702	.whitespace => if (pp.preserve_whitespace) try pp.addToken(tokFromRaw(tok)),
661	703	.nl => {
662	704	start_of_line = true;
663		if (pp.preserve_whitespace) try pp.tokens.append(pp.gpa, tokFromRaw(tok));
	705	if (pp.preserve_whitespace) try pp.addToken(tokFromRaw(tok));
664	706	},
665	707	.eof => {
666	708	if (if_level != 0) try pp.err(tok, .unterminated_conditional_directive);
...	...	@@ -696,14 +738,14 @@ fn preprocessExtra(pp: *Preprocessor, source: Source) MacroError!Token {
696	738
697	739	/// Get raw token source string.
698	740	/// Returned slice is invalidated when comp.generated_buf is updated.
699		pub fn tokSlice(pp: *Preprocessor, token: RawToken) []const u8 {
	741	pub fn tokSlice(pp: *Preprocessor, token: anytype) []const u8 {
700	742	if (token.id.lexeme()) \|some\| return some;
701	743	const source = pp.comp.getSource(token.source);
702	744	return source.buf[token.start..token.end];
703	745	}
704	746
705	747	/// Convert a token from the Tokenizer into a token used by the parser.
706		fn tokFromRaw(raw: RawToken) Token {
	748	fn tokFromRaw(raw: RawToken) TokenWithExpansionLocs {
707	749	return .{
708	750	.id = raw.id,
709	751	.loc = .{
...	...	@@ -725,7 +767,7 @@ fn err(pp: *Preprocessor, raw: RawToken, tag: Diagnostics.Tag) !void {
725	767	}, &.{});
726	768	}
727	769
728		fn errStr(pp: *Preprocessor, tok: Token, tag: Diagnostics.Tag, str: []const u8) !void {
	770	fn errStr(pp: *Preprocessor, tok: TokenWithExpansionLocs, tag: Diagnostics.Tag, str: []const u8) !void {
729	771	try pp.comp.addDiagnostic(.{
730	772	.tag = tag,
731	773	.loc = tok.loc,
...	...	@@ -747,7 +789,7 @@ fn fatal(pp: *Preprocessor, raw: RawToken, comptime fmt: []const u8, args: anyty
747	789	return error.FatalError;
748	790	}
749	791
750		fn fatalNotFound(pp: *Preprocessor, tok: Token, filename: []const u8) Compilation.Error {
	792	fn fatalNotFound(pp: *Preprocessor, tok: TokenWithExpansionLocs, filename: []const u8) Compilation.Error {
751	793	const old = pp.comp.diagnostics.fatal_errors;
752	794	pp.comp.diagnostics.fatal_errors = true;
753	795	defer pp.comp.diagnostics.fatal_errors = old;
...	...	@@ -790,7 +832,7 @@ fn expectNl(pp: Preprocessor, tokenizer: Tokenizer) Error!void {
790	832	while (true) {
791	833	const tok = tokenizer.next();
792	834	if (tok.id == .nl or tok.id == .eof) return;
793		if (tok.id == .whitespace) continue;
	835	if (tok.id == .whitespace or tok.id == .comment) continue;
794	836	if (!sent_err) {
795	837	sent_err = true;
796	838	try pp.err(tok, .extra_tokens_directive_end);
...	...	@@ -798,12 +840,24 @@ fn expectNl(pp: Preprocessor, tokenizer: Tokenizer) Error!void {
798	840	}
799	841	}
800	842
	843	fn getTokenState(pp: *const Preprocessor) TokenState {
	844	return .{
	845	.tokens_len = pp.tokens.len,
	846	.expansion_entries_len = pp.expansion_entries.len,
	847	};
	848	}
	849
	850	fn restoreTokenState(pp: *Preprocessor, state: TokenState) void {
	851	pp.tokens.len = state.tokens_len;
	852	pp.expansion_entries.len = state.expansion_entries_len;
	853	}
	854
801	855	/// Consume all tokens until a newline and parse the result into a boolean.
802	856	fn expr(pp: Preprocessor, tokenizer: Tokenizer) MacroError!bool {
803		const start = pp.tokens.len;
	857	const token_state = pp.getTokenState();
804	858	defer {
805		for (pp.top_expansion_buf.items) \|tok\| Token.free(tok.expansion_locs, pp.gpa);
806		pp.tokens.len = start;
	859	for (pp.top_expansion_buf.items) \|tok\| TokenWithExpansionLocs.free(tok.expansion_locs, pp.gpa);
	860	pp.restoreTokenState(token_state);
807	861	}
808	862
809	863	pp.top_expansion_buf.items.len = 0;
...	...	@@ -818,6 +872,7 @@ fn expr(pp: Preprocessor, tokenizer: Tokenizer) MacroError!bool {
818	872	} else unreachable;
819	873	if (pp.top_expansion_buf.items.len != 0) {
820	874	pp.expansion_source_loc = pp.top_expansion_buf.items[0].loc;
	875	pp.hideset.clearRetainingCapacity();
821	876	try pp.expandMacroExhaustive(tokenizer, &pp.top_expansion_buf, 0, pp.top_expansion_buf.items.len, false, .expr);
822	877	}
823	878	for (pp.top_expansion_buf.items) \|tok\| {
...	...	@@ -836,7 +891,7 @@ fn expr(pp: Preprocessor, tokenizer: Tokenizer) MacroError!bool {
836	891	}
837	892
838	893	// validate the tokens in the expression
839		try pp.tokens.ensureUnusedCapacity(pp.gpa, pp.top_expansion_buf.items.len);
	894	try pp.ensureUnusedTokenCapacity(pp.top_expansion_buf.items.len);
840	895	var i: usize = 0;
841	896	const items = pp.top_expansion_buf.items;
842	897	while (i < items.len) : (i += 1) {
...	...	@@ -905,9 +960,9 @@ fn expr(pp: Preprocessor, tokenizer: Tokenizer) MacroError!bool {
905	960	}
906	961	},
907	962	}
908		pp.tokens.appendAssumeCapacity(tok);
	963	pp.addTokenAssumeCapacity(tok);
909	964	}
910		try pp.tokens.append(pp.gpa, .{
	965	try pp.addToken(.{
911	966	.id = .eof,
912	967	.loc = tokFromRaw(eof).loc,
913	968	});
...	...	@@ -918,7 +973,7 @@ fn expr(pp: Preprocessor, tokenizer: Tokenizer) MacroError!bool {
918	973	.comp = pp.comp,
919	974	.gpa = pp.gpa,
920	975	.tok_ids = pp.tokens.items(.id),
921		.tok_i = @intCast(start),
	976	.tok_i = @intCast(token_state.tokens_len),
922	977	.arena = pp.arena.allocator(),
923	978	.in_macro = true,
924	979	.strings = std.ArrayList(u8).init(pp.comp.gpa),
...	...	@@ -941,7 +996,7 @@ fn expr(pp: Preprocessor, tokenizer: Tokenizer) MacroError!bool {
941	996
942	997	/// Turns macro_tok from .keyword_defined into .zero or .one depending on whether the argument is defined
943	998	/// Returns the number of tokens consumed
944		fn handleKeywordDefined(pp: Preprocessor, macro_tok: Token, tokens: []const Token, eof: RawToken) !usize {
	999	fn handleKeywordDefined(pp: Preprocessor, macro_tok: TokenWithExpansionLocs, tokens: []const TokenWithExpansionLocs, eof: RawToken) !usize {
945	1000	std.debug.assert(macro_tok.id == .keyword_defined);
946	1001	var it = TokenIterator.init(tokens);
947	1002	const first = it.nextNoWS() orelse {
...	...	@@ -1056,7 +1111,7 @@ fn skip(
1056	1111	tokenizer.index += 1;
1057	1112	tokenizer.line += 1;
1058	1113	if (pp.preserve_whitespace) {
1059		try pp.tokens.append(pp.gpa, .{ .id = .nl, .loc = .{
	1114	try pp.addToken(.{ .id = .nl, .loc = .{
1060	1115	.id = tokenizer.source,
1061	1116	.line = tokenizer.line,
1062	1117	} });
...	...	@@ -1079,21 +1134,21 @@ fn skipToNl(tokenizer: *Tokenizer) void {
1079	1134	}
1080	1135	}
1081	1136
1082		const ExpandBuf = std.ArrayList(Token);
	1137	const ExpandBuf = std.ArrayList(TokenWithExpansionLocs);
1083	1138	fn removePlacemarkers(buf: *ExpandBuf) void {
1084	1139	var i: usize = buf.items.len -% 1;
1085	1140	while (i < buf.items.len) : (i -%= 1) {
1086	1141	if (buf.items[i].id == .placemarker) {
1087	1142	const placemarker = buf.orderedRemove(i);
1088		Token.free(placemarker.expansion_locs, buf.allocator);
	1143	TokenWithExpansionLocs.free(placemarker.expansion_locs, buf.allocator);
1089	1144	}
1090	1145	}
1091	1146	}
1092	1147
1093		const MacroArguments = std.ArrayList([]const Token);
	1148	const MacroArguments = std.ArrayList([]const TokenWithExpansionLocs);
1094	1149	fn deinitMacroArguments(allocator: Allocator, args: *const MacroArguments) void {
1095	1150	for (args.items) \|item\| {
1096		for (item) \|tok\| Token.free(tok.expansion_locs, allocator);
	1151	for (item) \|tok\| TokenWithExpansionLocs.free(tok.expansion_locs, allocator);
1097	1152	allocator.free(item);
1098	1153	}
1099	1154	args.deinit();
...	...	@@ -1102,6 +1157,10 @@ fn deinitMacroArguments(allocator: Allocator, args: *const MacroArguments) void
1102	1157	fn expandObjMacro(pp: Preprocessor, simple_macro: const Macro) Error!ExpandBuf {
1103	1158	var buf = ExpandBuf.init(pp.gpa);
1104	1159	errdefer buf.deinit();
	1160	if (simple_macro.tokens.len == 0) {
	1161	try buf.append(.{ .id = .placemarker, .loc = .{ .id = .generated } });
	1162	return buf;
	1163	}
1105	1164	try buf.ensureTotalCapacity(simple_macro.tokens.len);
1106	1165
1107	1166	// Add all of the simple_macros tokens to the new buffer handling any concats.
...	...	@@ -1161,7 +1220,7 @@ fn expandObjMacro(pp: Preprocessor, simple_macro: const Macro) Error!ExpandBuf
1161	1220	/// Returns error.ExpectedStringLiteral if parentheses are not balanced, a non-string-literal
1162	1221	/// is encountered, or if no string literals are encountered
1163	1222	/// TODO: destringize (replace all '\\' with a single `\` and all '\"' with a '"')
1164		fn pasteStringsUnsafe(pp: *Preprocessor, toks: []const Token) ![]const u8 {
	1223	fn pasteStringsUnsafe(pp: *Preprocessor, toks: []const TokenWithExpansionLocs) ![]const u8 {
1165	1224	const char_top = pp.char_buf.items.len;
1166	1225	defer pp.char_buf.items.len = char_top;
1167	1226	var unwrapped = toks;
...	...	@@ -1180,7 +1239,7 @@ fn pasteStringsUnsafe(pp: *Preprocessor, toks: []const Token) ![]const u8 {
1180	1239	}
1181	1240
1182	1241	/// Handle the _Pragma operator (implemented as a builtin macro)
1183		fn pragmaOperator(pp: *Preprocessor, arg_tok: Token, operator_loc: Source.Location) !void {
	1242	fn pragmaOperator(pp: *Preprocessor, arg_tok: TokenWithExpansionLocs, operator_loc: Source.Location) !void {
1184	1243	const arg_slice = pp.expandedSlice(arg_tok);
1185	1244	const content = arg_slice[1 .. arg_slice.len - 1];
1186	1245	const directive = "#pragma ";
...	...	@@ -1234,7 +1293,7 @@ fn destringify(pp: *Preprocessor, str: []const u8) void {
1234	1293
1235	1294	/// Stringify `tokens` into pp.char_buf.
1236	1295	/// See https://gcc.gnu.org/onlinedocs/gcc-11.2.0/cpp/Stringizing.html#Stringizing
1237		fn stringify(pp: *Preprocessor, tokens: []const Token) !void {
	1296	fn stringify(pp: *Preprocessor, tokens: []const TokenWithExpansionLocs) !void {
1238	1297	try pp.char_buf.append('"');
1239	1298	var ws_state: enum { start, need, not_needed } = .start;
1240	1299	for (tokens) \|tok\| {
...	...	@@ -1281,7 +1340,8 @@ fn stringify(pp: *Preprocessor, tokens: []const Token) !void {
1281	1340	try pp.char_buf.appendSlice("\"\n");
1282	1341	}
1283	1342
1284		fn reconstructIncludeString(pp: Preprocessor, param_toks: []const Token, embed_args: ?[]const Token) !?[]const u8 {
	1343	fn reconstructIncludeString(pp: Preprocessor, param_toks: []const TokenWithExpansionLocs, embed_args: ?[]const TokenWithExpansionLocs, first: TokenWithExpansionLocs) !?[]const u8 {
	1344	assert(param_toks.len != 0);
1285	1345	const char_top = pp.char_buf.items.len;
1286	1346	defer pp.char_buf.items.len = char_top;
1287	1347
...	...	@@ -1295,8 +1355,8 @@ fn reconstructIncludeString(pp: *Preprocessor, param_toks: []const Token, embed_
1295	1355	if (params.len == 0) {
1296	1356	try pp.comp.addDiagnostic(.{
1297	1357	.tag = .expected_filename,
1298		.loc = param_toks[0].loc,
1299		}, param_toks[0].expansionSlice());
	1358	.loc = first.loc,
	1359	}, first.expansionSlice());
1300	1360	return null;
1301	1361	}
1302	1362	// no string pasting
...	...	@@ -1321,6 +1381,13 @@ fn reconstructIncludeString(pp: *Preprocessor, param_toks: []const Token, embed_
1321	1381
1322	1382	const include_str = pp.char_buf.items[char_top..];
1323	1383	if (include_str.len < 3) {
	1384	if (include_str.len == 0) {
	1385	try pp.comp.addDiagnostic(.{
	1386	.tag = .expected_filename,
	1387	.loc = first.loc,
	1388	}, first.expansionSlice());
	1389	return null;
	1390	}
1324	1391	try pp.comp.addDiagnostic(.{
1325	1392	.tag = .empty_filename,
1326	1393	.loc = params[0].loc,
...	...	@@ -1356,7 +1423,7 @@ fn reconstructIncludeString(pp: *Preprocessor, param_toks: []const Token, embed_
1356	1423	}
1357	1424	}
1358	1425
1359		fn handleBuiltinMacro(pp: *Preprocessor, builtin: RawToken.Id, param_toks: []const Token, src_loc: Source.Location) Error!bool {
	1426	fn handleBuiltinMacro(pp: *Preprocessor, builtin: RawToken.Id, param_toks: []const TokenWithExpansionLocs, src_loc: Source.Location) Error!bool {
1360	1427	switch (builtin) {
1361	1428	.macro_param_has_attribute,
1362	1429	.macro_param_has_declspec_attribute,
...	...	@@ -1364,8 +1431,8 @@ fn handleBuiltinMacro(pp: *Preprocessor, builtin: RawToken.Id, param_toks: []con
1364	1431	.macro_param_has_extension,
1365	1432	.macro_param_has_builtin,
1366	1433	=> {
1367		var invalid: ?Token = null;
1368		var identifier: ?Token = null;
	1434	var invalid: ?TokenWithExpansionLocs = null;
	1435	var identifier: ?TokenWithExpansionLocs = null;
1369	1436	for (param_toks) \|tok\| {
1370	1437	if (tok.id == .macro_ws) continue;
1371	1438	if (tok.id == .comment) continue;
...	...	@@ -1415,8 +1482,8 @@ fn handleBuiltinMacro(pp: *Preprocessor, builtin: RawToken.Id, param_toks: []con
1415	1482	return Diagnostics.warningExists(warning_name);
1416	1483	},
1417	1484	.macro_param_is_identifier => {
1418		var invalid: ?Token = null;
1419		var identifier: ?Token = null;
	1485	var invalid: ?TokenWithExpansionLocs = null;
	1486	var identifier: ?TokenWithExpansionLocs = null;
1420	1487	for (param_toks) \|tok\| switch (tok.id) {
1421	1488	.macro_ws => continue,
1422	1489	.comment => continue,
...	...	@@ -1438,7 +1505,7 @@ fn handleBuiltinMacro(pp: *Preprocessor, builtin: RawToken.Id, param_toks: []con
1438	1505	return id == .identifier or id == .extended_identifier;
1439	1506	},
1440	1507	.macro_param_has_include, .macro_param_has_include_next => {
1441		const include_str = (try pp.reconstructIncludeString(param_toks, null)) orelse return false;
	1508	const include_str = (try pp.reconstructIncludeString(param_toks, null, param_toks[0])) orelse return false;
1442	1509	const include_type: Compilation.IncludeType = switch (include_str[0]) {
1443	1510	'"' => .quotes,
1444	1511	'<' => .angle_brackets,
...	...	@@ -1460,6 +1527,17 @@ fn handleBuiltinMacro(pp: *Preprocessor, builtin: RawToken.Id, param_toks: []con
1460	1527	}
1461	1528	}
1462	1529
	1530	/// Treat whitespace-only paste arguments as empty
	1531	fn getPasteArgs(args: []const TokenWithExpansionLocs) []const TokenWithExpansionLocs {
	1532	for (args) \|tok\| {
	1533	if (tok.id != .macro_ws) return args;
	1534	}
	1535	return &[1]TokenWithExpansionLocs{.{
	1536	.id = .placemarker,
	1537	.loc = .{ .id = .generated, .byte_offset = 0, .line = 0 },
	1538	}};
	1539	}
	1540
1463	1541	fn expandFuncMacro(
1464	1542	pp: *Preprocessor,
1465	1543	loc: Source.Location,
...	...	@@ -1482,7 +1560,7 @@ fn expandFuncMacro(
1482	1560	try variable_arguments.appendSlice(args.items[i]);
1483	1561	try expanded_variable_arguments.appendSlice(expanded_args.items[i]);
1484	1562	if (i != expanded_args.items.len - 1) {
1485		const comma = Token{ .id = .comma, .loc = .{ .id = .generated } };
	1563	const comma = TokenWithExpansionLocs{ .id = .comma, .loc = .{ .id = .generated } };
1486	1564	try variable_arguments.append(comma);
1487	1565	try expanded_variable_arguments.append(comma);
1488	1566	}
...	...	@@ -1507,28 +1585,22 @@ fn expandFuncMacro(
1507	1585	.comment => if (!pp.comp.langopts.preserve_comments_in_macros)
1508	1586	continue
1509	1587	else
1510		&[1]Token{tokFromRaw(raw_next)},
1511		.macro_param, .macro_param_no_expand => if (args.items[raw_next.end].len > 0)
1512		args.items[raw_next.end]
1513		else
1514		&[1]Token{tokFromRaw(.{ .id = .placemarker, .source = .generated })},
	1588	&[1]TokenWithExpansionLocs{tokFromRaw(raw_next)},
	1589	.macro_param, .macro_param_no_expand => getPasteArgs(args.items[raw_next.end]),
1515	1590	.keyword_va_args => variable_arguments.items,
1516	1591	.keyword_va_opt => blk: {
1517	1592	try pp.expandVaOpt(&va_opt_buf, raw_next, variable_arguments.items.len != 0);
1518	1593	if (va_opt_buf.items.len == 0) break;
1519	1594	break :blk va_opt_buf.items;
1520	1595	},
1521		else => &[1]Token{tokFromRaw(raw_next)},
	1596	else => &[1]TokenWithExpansionLocs{tokFromRaw(raw_next)},
1522	1597	};
1523	1598
1524	1599	try pp.pasteTokens(&buf, next);
1525	1600	if (next.len != 0) break;
1526	1601	},
1527	1602	.macro_param_no_expand => {
1528		const slice = if (args.items[raw.end].len > 0)
1529		args.items[raw.end]
1530		else
1531		&[1]Token{tokFromRaw(.{ .id = .placemarker, .source = .generated })};
	1603	const slice = getPasteArgs(args.items[raw.end]);
1532	1604	const raw_loc = Source.Location{ .id = raw.source, .byte_offset = raw.start, .line = raw.line };
1533	1605	try bufCopyTokens(&buf, slice, &.{raw_loc});
1534	1606	},
...	...	@@ -1587,10 +1659,10 @@ fn expandFuncMacro(
1587	1659	try pp.comp.addDiagnostic(.{ .tag = .expected_arguments, .loc = loc, .extra = extra }, &.{});
1588	1660	break :blk not_found;
1589	1661	} else res: {
1590		var invalid: ?Token = null;
1591		var vendor_ident: ?Token = null;
1592		var colon_colon: ?Token = null;
1593		var attr_ident: ?Token = null;
	1662	var invalid: ?TokenWithExpansionLocs = null;
	1663	var vendor_ident: ?TokenWithExpansionLocs = null;
	1664	var colon_colon: ?TokenWithExpansionLocs = null;
	1665	var attr_ident: ?TokenWithExpansionLocs = null;
1594	1666	for (arg) \|tok\| {
1595	1667	if (tok.id == .macro_ws) continue;
1596	1668	if (tok.id == .comment) continue;
...	...	@@ -1663,17 +1735,17 @@ fn expandFuncMacro(
1663	1735	try pp.comp.addDiagnostic(.{ .tag = .expected_arguments, .loc = loc, .extra = extra }, &.{});
1664	1736	break :blk not_found;
1665	1737	} else res: {
1666		var embed_args: []const Token = &.{};
1667		const include_str = (try pp.reconstructIncludeString(arg, &embed_args)) orelse
	1738	var embed_args: []const TokenWithExpansionLocs = &.{};
	1739	const include_str = (try pp.reconstructIncludeString(arg, &embed_args, arg[0])) orelse
1668	1740	break :res not_found;
1669	1741
1670	1742	var prev = tokFromRaw(raw);
1671	1743	prev.id = .eof;
1672	1744	var it: struct {
1673	1745	i: u32 = 0,
1674		slice: []const Token,
1675		prev: Token,
1676		fn next(it: *@This()) Token {
	1746	slice: []const TokenWithExpansionLocs,
	1747	prev: TokenWithExpansionLocs,
	1748	fn next(it: *@This()) TokenWithExpansionLocs {
1677	1749	while (it.i < it.slice.len) switch (it.slice[it.i].id) {
1678	1750	.macro_ws, .whitespace => it.i += 1,
1679	1751	else => break,
...	...	@@ -1732,7 +1804,7 @@ fn expandFuncMacro(
1732	1804	};
1733	1805
1734	1806	var arg_count: u32 = 0;
1735		var first_arg: Token = undefined;
	1807	var first_arg: TokenWithExpansionLocs = undefined;
1736	1808	while (true) {
1737	1809	const next = it.next();
1738	1810	if (next.id == .eof) {
...	...	@@ -1793,8 +1865,8 @@ fn expandFuncMacro(
1793	1865	// Clang and GCC require exactly one token (so, no parentheses or string pasting)
1794	1866	// even though their error messages indicate otherwise. Ours is slightly more
1795	1867	// descriptive.
1796		var invalid: ?Token = null;
1797		var string: ?Token = null;
	1868	var invalid: ?TokenWithExpansionLocs = null;
	1869	var string: ?TokenWithExpansionLocs = null;
1798	1870	for (param_toks) \|tok\| switch (tok.id) {
1799	1871	.string_literal => {
1800	1872	if (string) \|_\| invalid = tok else string = tok;
...	...	@@ -1884,27 +1956,11 @@ fn expandVaOpt(
1884	1956	}
1885	1957	}
1886	1958
1887		fn shouldExpand(tok: Token, macro: *Macro) bool {
1888		if (tok.loc.id == macro.loc.id and
1889		tok.loc.byte_offset >= macro.start and
1890		tok.loc.byte_offset <= macro.end)
1891		return false;
1892		for (tok.expansionSlice()) \|loc\| {
1893		if (loc.id == macro.loc.id and
1894		loc.byte_offset >= macro.start and
1895		loc.byte_offset <= macro.end)
1896		return false;
1897		}
1898		if (tok.flags.expansion_disabled) return false;
1899
1900		return true;
1901		}
1902
1903		fn bufCopyTokens(buf: *ExpandBuf, tokens: []const Token, src: []const Source.Location) !void {
	1959	fn bufCopyTokens(buf: *ExpandBuf, tokens: []const TokenWithExpansionLocs, src: []const Source.Location) !void {
1904	1960	try buf.ensureUnusedCapacity(tokens.len);
1905	1961	for (tokens) \|tok\| {
1906	1962	var copy = try tok.dupe(buf.allocator);
1907		errdefer Token.free(copy.expansion_locs, buf.allocator);
	1963	errdefer TokenWithExpansionLocs.free(copy.expansion_locs, buf.allocator);
1908	1964	try copy.addExpansionLocation(buf.allocator, src);
1909	1965	buf.appendAssumeCapacity(copy);
1910	1966	}
...	...	@@ -1917,7 +1973,7 @@ fn nextBufToken(
1917	1973	start_idx: *usize,
1918	1974	end_idx: *usize,
1919	1975	extend_buf: bool,
1920		) Error!Token {
	1976	) Error!TokenWithExpansionLocs {
1921	1977	start_idx.* += 1;
1922	1978	if (start_idx.* == buf.items.len and start_idx.* >= end_idx.*) {
1923	1979	if (extend_buf) {
...	...	@@ -1933,7 +1989,7 @@ fn nextBufToken(
1933	1989	try buf.append(new_tok);
1934	1990	return new_tok;
1935	1991	} else {
1936		return Token{ .id = .eof, .loc = .{ .id = .generated } };
	1992	return TokenWithExpansionLocs{ .id = .eof, .loc = .{ .id = .generated } };
1937	1993	}
1938	1994	} else {
1939	1995	return buf.items[start_idx.*];
...	...	@@ -1948,6 +2004,7 @@ fn collectMacroFuncArguments(
1948	2004	end_idx: *usize,
1949	2005	extend_buf: bool,
1950	2006	is_builtin: bool,
	2007	r_paren: *TokenWithExpansionLocs,
1951	2008	) !MacroArguments {
1952	2009	const name_tok = buf.items[start_idx.*];
1953	2010	const saved_tokenizer = tokenizer.*;
...	...	@@ -1974,7 +2031,7 @@ fn collectMacroFuncArguments(
1974	2031	var parens: u32 = 0;
1975	2032	var args = MacroArguments.init(pp.gpa);
1976	2033	errdefer deinitMacroArguments(pp.gpa, &args);
1977		var curArgument = std.ArrayList(Token).init(pp.gpa);
	2034	var curArgument = std.ArrayList(TokenWithExpansionLocs).init(pp.gpa);
1978	2035	defer curArgument.deinit();
1979	2036	while (true) {
1980	2037	var tok = try nextBufToken(pp, tokenizer, buf, start_idx, end_idx, extend_buf);
...	...	@@ -1987,13 +2044,13 @@ fn collectMacroFuncArguments(
1987	2044	try args.append(owned);
1988	2045	} else {
1989	2046	const duped = try tok.dupe(pp.gpa);
1990		errdefer Token.free(duped.expansion_locs, pp.gpa);
	2047	errdefer TokenWithExpansionLocs.free(duped.expansion_locs, pp.gpa);
1991	2048	try curArgument.append(duped);
1992	2049	}
1993	2050	},
1994	2051	.l_paren => {
1995	2052	const duped = try tok.dupe(pp.gpa);
1996		errdefer Token.free(duped.expansion_locs, pp.gpa);
	2053	errdefer TokenWithExpansionLocs.free(duped.expansion_locs, pp.gpa);
1997	2054	try curArgument.append(duped);
1998	2055	parens += 1;
1999	2056	},
...	...	@@ -2002,10 +2059,11 @@ fn collectMacroFuncArguments(
2002	2059	const owned = try curArgument.toOwnedSlice();
2003	2060	errdefer pp.gpa.free(owned);
2004	2061	try args.append(owned);
	2062	r_paren.* = tok;
2005	2063	break;
2006	2064	} else {
2007	2065	const duped = try tok.dupe(pp.gpa);
2008		errdefer Token.free(duped.expansion_locs, pp.gpa);
	2066	errdefer TokenWithExpansionLocs.free(duped.expansion_locs, pp.gpa);
2009	2067	try curArgument.append(duped);
2010	2068	parens -= 1;
2011	2069	}
...	...	@@ -2028,7 +2086,7 @@ fn collectMacroFuncArguments(
2028	2086	},
2029	2087	else => {
2030	2088	const duped = try tok.dupe(pp.gpa);
2031		errdefer Token.free(duped.expansion_locs, pp.gpa);
	2089	errdefer TokenWithExpansionLocs.free(duped.expansion_locs, pp.gpa);
2032	2090	try curArgument.append(duped);
2033	2091	},
2034	2092	}
...	...	@@ -2038,7 +2096,7 @@ fn collectMacroFuncArguments(
2038	2096	}
2039	2097
2040	2098	fn removeExpandedTokens(pp: Preprocessor, buf: ExpandBuf, start: usize, len: usize, moving_end_idx: *usize) !void {
2041		for (buf.items[start .. start + len]) \|tok\| Token.free(tok.expansion_locs, pp.gpa);
	2099	for (buf.items[start .. start + len]) \|tok\| TokenWithExpansionLocs.free(tok.expansion_locs, pp.gpa);
2042	2100	try buf.replaceRange(start, len, &.{});
2043	2101	moving_end_idx.* -\|= len;
2044	2102	}
...	...	@@ -2054,14 +2112,14 @@ const EvalContext = enum {
2054	2112
2055	2113	/// Helper for safely iterating over a slice of tokens while skipping whitespace
2056	2114	const TokenIterator = struct {
2057		toks: []const Token,
	2115	toks: []const TokenWithExpansionLocs,
2058	2116	i: usize,
2059	2117
2060		fn init(toks: []const Token) TokenIterator {
	2118	fn init(toks: []const TokenWithExpansionLocs) TokenIterator {
2061	2119	return .{ .toks = toks, .i = 0 };
2062	2120	}
2063	2121
2064		fn nextNoWS(self: *TokenIterator) ?Token {
	2122	fn nextNoWS(self: *TokenIterator) ?TokenWithExpansionLocs {
2065	2123	while (self.i < self.toks.len) : (self.i += 1) {
2066	2124	const tok = self.toks[self.i];
2067	2125	if (tok.id == .whitespace or tok.id == .macro_ws) continue;
...	...	@@ -2108,13 +2166,24 @@ fn expandMacroExhaustive(
2108	2166	idx += it.i;
2109	2167	continue;
2110	2168	}
2111		const macro_entry = pp.defines.getPtr(pp.expandedSlice(macro_tok));
2112		if (macro_entry == null or !shouldExpand(buf.items[idx], macro_entry.?)) {
	2169	if (!macro_tok.id.isMacroIdentifier() or macro_tok.flags.expansion_disabled) {
2113	2170	idx += 1;
2114	2171	continue;
2115	2172	}
2116		if (macro_entry) \|macro\| macro_handler: {
	2173	const expanded = pp.expandedSlice(macro_tok);
	2174	const macro = pp.defines.getPtr(expanded) orelse {
	2175	idx += 1;
	2176	continue;
	2177	};
	2178	const macro_hidelist = pp.hideset.get(macro_tok.loc);
	2179	if (pp.hideset.contains(macro_hidelist, expanded)) {
	2180	idx += 1;
	2181	continue;
	2182	}
	2183
	2184	macro_handler: {
2117	2185	if (macro.is_func) {
	2186	var r_paren: TokenWithExpansionLocs = undefined;
2118	2187	var macro_scan_idx = idx;
2119	2188	// to be saved in case this doesn't turn out to be a call
2120	2189	const args = pp.collectMacroFuncArguments(
...	...	@@ -2124,6 +2193,7 @@ fn expandMacroExhaustive(
2124	2193	&moving_end_idx,
2125	2194	extend_buf,
2126	2195	macro.is_builtin,
	2196	&r_paren,
2127	2197	) catch \|er\| switch (er) {
2128	2198	error.MissingLParen => {
2129	2199	if (!buf.items[idx].flags.is_macro_arg) buf.items[idx].flags.expansion_disabled = true;
...	...	@@ -2137,12 +2207,16 @@ fn expandMacroExhaustive(
2137	2207	},
2138	2208	else => \|e\| return e,
2139	2209	};
	2210	assert(r_paren.id == .r_paren);
2140	2211	defer {
2141	2212	for (args.items) \|item\| {
2142	2213	pp.gpa.free(item);
2143	2214	}
2144	2215	args.deinit();
2145	2216	}
	2217	const r_paren_hidelist = pp.hideset.get(r_paren.loc);
	2218	var hs = try pp.hideset.intersection(macro_hidelist, r_paren_hidelist);
	2219	hs = try pp.hideset.prepend(macro_tok.loc, hs);
2146	2220
2147	2221	var args_count: u32 = @intCast(args.items.len);
2148	2222	// if the macro has zero arguments g() args_count is still 1
...	...	@@ -2199,10 +2273,13 @@ fn expandMacroExhaustive(
2199	2273	for (res.items) \|*tok\| {
2200	2274	try tok.addExpansionLocation(pp.gpa, &.{macro_tok.loc});
2201	2275	try tok.addExpansionLocation(pp.gpa, macro_expansion_locs);
	2276	const tok_hidelist = pp.hideset.get(tok.loc);
	2277	const new_hidelist = try pp.hideset.@"union"(tok_hidelist, hs);
	2278	try pp.hideset.put(tok.loc, new_hidelist);
2202	2279	}
2203	2280
2204	2281	const tokens_removed = macro_scan_idx - idx + 1;
2205		for (buf.items[idx .. idx + tokens_removed]) \|tok\| Token.free(tok.expansion_locs, pp.gpa);
	2282	for (buf.items[idx .. idx + tokens_removed]) \|tok\| TokenWithExpansionLocs.free(tok.expansion_locs, pp.gpa);
2206	2283	try buf.replaceRange(idx, tokens_removed, res.items);
2207	2284
2208	2285	moving_end_idx += tokens_added;
...	...	@@ -2215,12 +2292,19 @@ fn expandMacroExhaustive(
2215	2292	const res = try pp.expandObjMacro(macro);
2216	2293	defer res.deinit();
2217	2294
	2295	const hs = try pp.hideset.prepend(macro_tok.loc, macro_hidelist);
	2296
2218	2297	const macro_expansion_locs = macro_tok.expansionSlice();
2219	2298	var increment_idx_by = res.items.len;
2220	2299	for (res.items, 0..) \|*tok, i\| {
2221	2300	tok.flags.is_macro_arg = macro_tok.flags.is_macro_arg;
2222	2301	try tok.addExpansionLocation(pp.gpa, &.{macro_tok.loc});
2223	2302	try tok.addExpansionLocation(pp.gpa, macro_expansion_locs);
	2303
	2304	const tok_hidelist = pp.hideset.get(tok.loc);
	2305	const new_hidelist = try pp.hideset.@"union"(tok_hidelist, hs);
	2306	try pp.hideset.put(tok.loc, new_hidelist);
	2307
2224	2308	if (tok.id == .keyword_defined and eval_ctx == .expr) {
2225	2309	try pp.comp.addDiagnostic(.{
2226	2310	.tag = .expansion_to_defined,
...	...	@@ -2233,7 +2317,7 @@ fn expandMacroExhaustive(
2233	2317	}
2234	2318	}
2235	2319
2236		Token.free(buf.items[idx].expansion_locs, pp.gpa);
	2320	TokenWithExpansionLocs.free(buf.items[idx].expansion_locs, pp.gpa);
2237	2321	try buf.replaceRange(idx, 1, res.items);
2238	2322	idx += increment_idx_by;
2239	2323	moving_end_idx = moving_end_idx + res.items.len - 1;
...	...	@@ -2249,7 +2333,7 @@ fn expandMacroExhaustive(
2249	2333
2250	2334	// trim excess buffer
2251	2335	for (buf.items[moving_end_idx..]) \|item\| {
2252		Token.free(item.expansion_locs, pp.gpa);
	2336	TokenWithExpansionLocs.free(item.expansion_locs, pp.gpa);
2253	2337	}
2254	2338	buf.items.len = moving_end_idx;
2255	2339	}
...	...	@@ -2260,30 +2344,35 @@ fn expandMacro(pp: Preprocessor, tokenizer: Tokenizer, raw: RawToken) MacroErr
2260	2344	var source_tok = tokFromRaw(raw);
2261	2345	if (!raw.id.isMacroIdentifier()) {
2262	2346	source_tok.id.simplifyMacroKeyword();
2263		return pp.tokens.append(pp.gpa, source_tok);
	2347	return pp.addToken(source_tok);
2264	2348	}
2265	2349	pp.top_expansion_buf.items.len = 0;
2266	2350	try pp.top_expansion_buf.append(source_tok);
2267	2351	pp.expansion_source_loc = source_tok.loc;
2268	2352
	2353	pp.hideset.clearRetainingCapacity();
2269	2354	try pp.expandMacroExhaustive(tokenizer, &pp.top_expansion_buf, 0, 1, true, .non_expr);
2270		try pp.tokens.ensureUnusedCapacity(pp.gpa, pp.top_expansion_buf.items.len);
	2355	try pp.ensureUnusedTokenCapacity(pp.top_expansion_buf.items.len);
2271	2356	for (pp.top_expansion_buf.items) \|*tok\| {
2272	2357	if (tok.id == .macro_ws and !pp.preserve_whitespace) {
2273		Token.free(tok.expansion_locs, pp.gpa);
	2358	TokenWithExpansionLocs.free(tok.expansion_locs, pp.gpa);
2274	2359	continue;
2275	2360	}
2276	2361	if (tok.id == .comment and !pp.comp.langopts.preserve_comments_in_macros) {
2277		Token.free(tok.expansion_locs, pp.gpa);
	2362	TokenWithExpansionLocs.free(tok.expansion_locs, pp.gpa);
	2363	continue;
	2364	}
	2365	if (tok.id == .placemarker) {
	2366	TokenWithExpansionLocs.free(tok.expansion_locs, pp.gpa);
2278	2367	continue;
2279	2368	}
2280	2369	tok.id.simplifyMacroKeywordExtra(true);
2281		pp.tokens.appendAssumeCapacity(tok.*);
	2370	pp.addTokenAssumeCapacity(tok.*);
2282	2371	}
2283	2372	if (pp.preserve_whitespace) {
2284		try pp.tokens.ensureUnusedCapacity(pp.gpa, pp.add_expansion_nl);
	2373	try pp.ensureUnusedTokenCapacity(pp.add_expansion_nl);
2285	2374	while (pp.add_expansion_nl > 0) : (pp.add_expansion_nl -= 1) {
2286		pp.tokens.appendAssumeCapacity(.{ .id = .nl, .loc = .{
	2375	pp.addTokenAssumeCapacity(.{ .id = .nl, .loc = .{
2287	2376	.id = tokenizer.source,
2288	2377	.line = tokenizer.line,
2289	2378	} });
...	...	@@ -2291,7 +2380,7 @@ fn expandMacro(pp: Preprocessor, tokenizer: Tokenizer, raw: RawToken) MacroErr
2291	2380	}
2292	2381	}
2293	2382
2294		fn expandedSliceExtra(pp: *const Preprocessor, tok: Token, macro_ws_handling: enum { single_macro_ws, preserve_macro_ws }) []const u8 {
	2383	fn expandedSliceExtra(pp: *const Preprocessor, tok: anytype, macro_ws_handling: enum { single_macro_ws, preserve_macro_ws }) []const u8 {
2295	2384	if (tok.id.lexeme()) \|some\| {
2296	2385	if (!tok.id.allowsDigraphs(pp.comp.langopts) and !(tok.id == .macro_ws and macro_ws_handling == .preserve_macro_ws)) return some;
2297	2386	}
...	...	@@ -2312,18 +2401,18 @@ fn expandedSliceExtra(pp: *const Preprocessor, tok: Token, macro_ws_handling: en
2312	2401	}
2313	2402
2314	2403	/// Get expanded token source string.
2315		pub fn expandedSlice(pp: *Preprocessor, tok: Token) []const u8 {
	2404	pub fn expandedSlice(pp: *const Preprocessor, tok: anytype) []const u8 {
2316	2405	return pp.expandedSliceExtra(tok, .single_macro_ws);
2317	2406	}
2318	2407
2319	2408	/// Concat two tokens and add the result to pp.generated
2320		fn pasteTokens(pp: Preprocessor, lhs_toks: ExpandBuf, rhs_toks: []const Token) Error!void {
	2409	fn pasteTokens(pp: Preprocessor, lhs_toks: ExpandBuf, rhs_toks: []const TokenWithExpansionLocs) Error!void {
2321	2410	const lhs = while (lhs_toks.popOrNull()) \|lhs\| {
2322	2411	if ((pp.comp.langopts.preserve_comments_in_macros and lhs.id == .comment) or
2323	2412	(lhs.id != .macro_ws and lhs.id != .comment))
2324	2413	break lhs;
2325	2414
2326		Token.free(lhs.expansion_locs, pp.gpa);
	2415	TokenWithExpansionLocs.free(lhs.expansion_locs, pp.gpa);
2327	2416	} else {
2328	2417	return bufCopyTokens(lhs_toks, rhs_toks, &.{});
2329	2418	};
...	...	@@ -2338,7 +2427,7 @@ fn pasteTokens(pp: Preprocessor, lhs_toks: ExpandBuf, rhs_toks: []const Token)
2338	2427	} else {
2339	2428	return lhs_toks.appendAssumeCapacity(lhs);
2340	2429	};
2341		defer Token.free(lhs.expansion_locs, pp.gpa);
	2430	defer TokenWithExpansionLocs.free(lhs.expansion_locs, pp.gpa);
2342	2431
2343	2432	const start = pp.comp.generated_buf.items.len;
2344	2433	const end = start + pp.expandedSlice(lhs).len + pp.expandedSlice(rhs).len;
...	...	@@ -2375,8 +2464,8 @@ fn pasteTokens(pp: Preprocessor, lhs_toks: ExpandBuf, rhs_toks: []const Token)
2375	2464	try bufCopyTokens(lhs_toks, rhs_toks[rhs_rest..], &.{});
2376	2465	}
2377	2466
2378		fn makeGeneratedToken(pp: *Preprocessor, start: usize, id: Token.Id, source: Token) !Token {
2379		var pasted_token = Token{ .id = id, .loc = .{
	2467	fn makeGeneratedToken(pp: *Preprocessor, start: usize, id: Token.Id, source: TokenWithExpansionLocs) !TokenWithExpansionLocs {
	2468	var pasted_token = TokenWithExpansionLocs{ .id = id, .loc = .{
2380	2469	.id = .generated,
2381	2470	.byte_offset = @intCast(start),
2382	2471	.line = pp.generated_line,
...	...	@@ -2441,8 +2530,6 @@ fn define(pp: Preprocessor, tokenizer: Tokenizer) Error!void {
2441	2530	.tokens = &.{},
2442	2531	.var_args = false,
2443	2532	.loc = tokFromRaw(macro_name).loc,
2444		.start = 0,
2445		.end = 0,
2446	2533	.is_func = false,
2447	2534	}),
2448	2535	.whitespace => first = tokenizer.next(),
...	...	@@ -2460,7 +2547,7 @@ fn define(pp: Preprocessor, tokenizer: Tokenizer) Error!void {
2460	2547	var need_ws = false;
2461	2548	// Collect the token body and validate any ## found.
2462	2549	var tok = first;
2463		const end_index = while (true) {
	2550	while (true) {
2464	2551	tok.id.simplifyMacroKeyword();
2465	2552	switch (tok.id) {
2466	2553	.hash_hash => {
...	...	@@ -2479,7 +2566,7 @@ fn define(pp: Preprocessor, tokenizer: Tokenizer) Error!void {
2479	2566	try pp.token_buf.append(tok);
2480	2567	try pp.token_buf.append(next);
2481	2568	},
2482		.nl, .eof => break tok.start,
	2569	.nl, .eof => break,
2483	2570	.comment => if (pp.comp.langopts.preserve_comments_in_macros) {
2484	2571	if (need_ws) {
2485	2572	need_ws = false;
...	...	@@ -2502,13 +2589,11 @@ fn define(pp: Preprocessor, tokenizer: Tokenizer) Error!void {
2502	2589	},
2503	2590	}
2504	2591	tok = tokenizer.next();
2505		} else unreachable;
	2592	}
2506	2593
2507	2594	const list = try pp.arena.allocator().dupe(RawToken, pp.token_buf.items);
2508	2595	try pp.defineMacro(macro_name, .{
2509	2596	.loc = tokFromRaw(macro_name).loc,
2510		.start = first.start,
2511		.end = end_index,
2512	2597	.tokens = list,
2513	2598	.params = undefined,
2514	2599	.is_func = false,
...	...	@@ -2525,9 +2610,9 @@ fn defineFn(pp: Preprocessor, tokenizer: Tokenizer, macro_name: RawToken, l_pa
2525	2610	// Parse the parameter list.
2526	2611	var gnu_var_args: []const u8 = "";
2527	2612	var var_args = false;
2528		const start_index = while (true) {
	2613	while (true) {
2529	2614	var tok = tokenizer.nextNoWS();
2530		if (tok.id == .r_paren) break tok.end;
	2615	if (tok.id == .r_paren) break;
2531	2616	if (tok.id == .eof) return pp.err(tok, .unterminated_macro_param_list);
2532	2617	if (tok.id == .ellipsis) {
2533	2618	var_args = true;
...	...	@@ -2537,7 +2622,7 @@ fn defineFn(pp: Preprocessor, tokenizer: Tokenizer, macro_name: RawToken, l_pa
2537	2622	try pp.err(l_paren, .to_match_paren);
2538	2623	return skipToNl(tokenizer);
2539	2624	}
2540		break r_paren.end;
	2625	break;
2541	2626	}
2542	2627	if (!tok.id.isMacroIdentifier()) {
2543	2628	try pp.err(tok, .invalid_token_param_list);
...	...	@@ -2556,22 +2641,22 @@ fn defineFn(pp: Preprocessor, tokenizer: Tokenizer, macro_name: RawToken, l_pa
2556	2641	try pp.err(l_paren, .to_match_paren);
2557	2642	return skipToNl(tokenizer);
2558	2643	}
2559		break r_paren.end;
	2644	break;
2560	2645	} else if (tok.id == .r_paren) {
2561		break tok.end;
	2646	break;
2562	2647	} else if (tok.id != .comma) {
2563	2648	try pp.err(tok, .expected_comma_param_list);
2564	2649	return skipToNl(tokenizer);
2565	2650	}
2566		} else unreachable;
	2651	}
2567	2652
2568	2653	var need_ws = false;
2569	2654	// Collect the body tokens and validate # and ##'s found.
2570	2655	pp.token_buf.items.len = 0; // Safe to use since we can only be in one directive at a time.
2571		const end_index = tok_loop: while (true) {
	2656	tok_loop: while (true) {
2572	2657	var tok = tokenizer.next();
2573	2658	switch (tok.id) {
2574		.nl, .eof => break tok.start,
	2659	.nl, .eof => break,
2575	2660	.whitespace => need_ws = pp.token_buf.items.len != 0,
2576	2661	.comment => if (!pp.comp.langopts.preserve_comments_in_macros) continue else {
2577	2662	if (need_ws) {
...	...	@@ -2690,7 +2775,7 @@ fn defineFn(pp: Preprocessor, tokenizer: Tokenizer, macro_name: RawToken, l_pa
2690	2775	try pp.token_buf.append(tok);
2691	2776	},
2692	2777	}
2693		} else unreachable;
	2778	}
2694	2779
2695	2780	const param_list = try pp.arena.allocator().dupe([]const u8, params.items);
2696	2781	const token_list = try pp.arena.allocator().dupe(RawToken, pp.token_buf.items);
...	...	@@ -2700,8 +2785,6 @@ fn defineFn(pp: Preprocessor, tokenizer: Tokenizer, macro_name: RawToken, l_pa
2700	2785	.var_args = var_args or gnu_var_args.len != 0,
2701	2786	.tokens = token_list,
2702	2787	.loc = tokFromRaw(macro_name).loc,
2703		.start = start_index,
2704		.end = end_index,
2705	2788	});
2706	2789	}
2707	2790
...	...	@@ -2714,7 +2797,7 @@ fn embed(pp: Preprocessor, tokenizer: Tokenizer) MacroError!void {
2714	2797	error.InvalidInclude => return,
2715	2798	else => \|e\| return e,
2716	2799	};
2717		defer Token.free(filename_tok.expansion_locs, pp.gpa);
	2800	defer TokenWithExpansionLocs.free(filename_tok.expansion_locs, pp.gpa);
2718	2801
2719	2802	// Check for empty filename.
2720	2803	const tok_slice = pp.expandedSliceExtra(filename_tok, .single_macro_ws);
...	...	@@ -2859,7 +2942,7 @@ fn embed(pp: Preprocessor, tokenizer: Tokenizer) MacroError!void {
2859	2942	return;
2860	2943	}
2861	2944
2862		try pp.tokens.ensureUnusedCapacity(pp.comp.gpa, 2 * embed_bytes.len - 1); // N bytes and N-1 commas
	2945	try pp.ensureUnusedTokenCapacity(2 * embed_bytes.len - 1); // N bytes and N-1 commas
2863	2946
2864	2947	// TODO: We currently only support systems with CHAR_BIT == 8
2865	2948	// If the target's CHAR_BIT is not 8, we need to write out correctly-sized embed_bytes
...	...	@@ -2870,14 +2953,14 @@ fn embed(pp: Preprocessor, tokenizer: Tokenizer) MacroError!void {
2870	2953	const byte = embed_bytes[0];
2871	2954	const start = pp.comp.generated_buf.items.len;
2872	2955	try writer.print("{d}", .{byte});
2873		pp.tokens.appendAssumeCapacity(try pp.makeGeneratedToken(start, .embed_byte, filename_tok));
	2956	pp.addTokenAssumeCapacity(try pp.makeGeneratedToken(start, .embed_byte, filename_tok));
2874	2957	}
2875	2958
2876	2959	for (embed_bytes[1..]) \|byte\| {
2877	2960	const start = pp.comp.generated_buf.items.len;
2878	2961	try writer.print(",{d}", .{byte});
2879		pp.tokens.appendAssumeCapacity(.{ .id = .comma, .loc = .{ .id = .generated, .byte_offset = @intCast(start) } });
2880		pp.tokens.appendAssumeCapacity(try pp.makeGeneratedToken(start + 1, .embed_byte, filename_tok));
	2962	pp.addTokenAssumeCapacity(.{ .id = .comma, .loc = .{ .id = .generated, .byte_offset = @intCast(start) } });
	2963	pp.addTokenAssumeCapacity(try pp.makeGeneratedToken(start + 1, .embed_byte, filename_tok));
2881	2964	}
2882	2965	try pp.comp.generated_buf.append(pp.gpa, '\n');
2883	2966
...	...	@@ -2911,19 +2994,19 @@ fn include(pp: Preprocessor, tokenizer: Tokenizer, which: Compilation.WhichInc
2911	2994	pp.verboseLog(first, "include file {s}", .{new_source.path});
2912	2995	}
2913	2996
2914		const tokens_start = pp.tokens.len;
	2997	const token_state = pp.getTokenState();
2915	2998	try pp.addIncludeStart(new_source);
2916	2999	const eof = pp.preprocessExtra(new_source) catch \|er\| switch (er) {
2917	3000	error.StopPreprocessing => {
2918		for (pp.tokens.items(.expansion_locs)[tokens_start..]) \|loc\| Token.free(loc, pp.gpa);
2919		pp.tokens.len = tokens_start;
	3001	for (pp.expansion_entries.items(.locs)[token_state.expansion_entries_len..]) \|loc\| TokenWithExpansionLocs.free(loc, pp.gpa);
	3002	pp.restoreTokenState(token_state);
2920	3003	return;
2921	3004	},
2922	3005	else => \|e\| return e,
2923	3006	};
2924	3007	try eof.checkMsEof(new_source, pp.comp);
2925	3008	if (pp.preserve_whitespace and pp.tokens.items(.id)[pp.tokens.len - 1] != .nl) {
2926		try pp.tokens.append(pp.gpa, .{ .id = .nl, .loc = .{
	3009	try pp.addToken(.{ .id = .nl, .loc = .{
2927	3010	.id = tokenizer.source,
2928	3011	.line = tokenizer.line,
2929	3012	} });
...	...	@@ -2945,7 +3028,7 @@ fn include(pp: Preprocessor, tokenizer: Tokenizer, which: Compilation.WhichInc
2945	3028	/// 3. Via a stringified macro argument which is used as an argument to `_Pragma`
2946	3029	/// operator_loc: Location of `_Pragma`; null if this is from #pragma
2947	3030	/// arg_locs: expansion locations of the argument to _Pragma. empty if #pragma or a raw string literal was used
2948		fn makePragmaToken(pp: *Preprocessor, raw: RawToken, operator_loc: ?Source.Location, arg_locs: []const Source.Location) !Token {
	3031	fn makePragmaToken(pp: *Preprocessor, raw: RawToken, operator_loc: ?Source.Location, arg_locs: []const Source.Location) !TokenWithExpansionLocs {
2949	3032	var tok = tokFromRaw(raw);
2950	3033	if (operator_loc) \|loc\| {
2951	3034	try tok.addExpansionLocation(pp.gpa, &.{loc});
...	...	@@ -2954,28 +3037,52 @@ fn makePragmaToken(pp: *Preprocessor, raw: RawToken, operator_loc: ?Source.Locat
2954	3037	return tok;
2955	3038	}
2956	3039
	3040	pub fn addToken(pp: *Preprocessor, tok: TokenWithExpansionLocs) !void {
	3041	if (tok.expansion_locs) \|expansion_locs\| {
	3042	try pp.expansion_entries.append(pp.gpa, .{ .idx = @intCast(pp.tokens.len), .locs = expansion_locs });
	3043	}
	3044	try pp.tokens.append(pp.gpa, .{ .id = tok.id, .loc = tok.loc });
	3045	}
	3046
	3047	pub fn addTokenAssumeCapacity(pp: *Preprocessor, tok: TokenWithExpansionLocs) void {
	3048	if (tok.expansion_locs) \|expansion_locs\| {
	3049	pp.expansion_entries.appendAssumeCapacity(.{ .idx = @intCast(pp.tokens.len), .locs = expansion_locs });
	3050	}
	3051	pp.tokens.appendAssumeCapacity(.{ .id = tok.id, .loc = tok.loc });
	3052	}
	3053
	3054	pub fn ensureTotalTokenCapacity(pp: *Preprocessor, capacity: usize) !void {
	3055	try pp.tokens.ensureTotalCapacity(pp.gpa, capacity);
	3056	try pp.expansion_entries.ensureTotalCapacity(pp.gpa, capacity);
	3057	}
	3058
	3059	pub fn ensureUnusedTokenCapacity(pp: *Preprocessor, capacity: usize) !void {
	3060	try pp.tokens.ensureUnusedCapacity(pp.gpa, capacity);
	3061	try pp.expansion_entries.ensureUnusedCapacity(pp.gpa, capacity);
	3062	}
	3063
2957	3064	/// Handle a pragma directive
2958	3065	fn pragma(pp: Preprocessor, tokenizer: Tokenizer, pragma_tok: RawToken, operator_loc: ?Source.Location, arg_locs: []const Source.Location) !void {
2959	3066	const name_tok = tokenizer.nextNoWS();
2960	3067	if (name_tok.id == .nl or name_tok.id == .eof) return;
2961	3068
2962	3069	const name = pp.tokSlice(name_tok);
2963		try pp.tokens.append(pp.gpa, try pp.makePragmaToken(pragma_tok, operator_loc, arg_locs));
	3070	try pp.addToken(try pp.makePragmaToken(pragma_tok, operator_loc, arg_locs));
2964	3071	const pragma_start: u32 = @intCast(pp.tokens.len);
2965	3072
2966	3073	const pragma_name_tok = try pp.makePragmaToken(name_tok, operator_loc, arg_locs);
2967		try pp.tokens.append(pp.gpa, pragma_name_tok);
	3074	try pp.addToken(pragma_name_tok);
2968	3075	while (true) {
2969	3076	const next_tok = tokenizer.next();
2970	3077	if (next_tok.id == .whitespace) continue;
2971	3078	if (next_tok.id == .eof) {
2972		try pp.tokens.append(pp.gpa, .{
	3079	try pp.addToken(.{
2973	3080	.id = .nl,
2974	3081	.loc = .{ .id = .generated },
2975	3082	});
2976	3083	break;
2977	3084	}
2978		try pp.tokens.append(pp.gpa, try pp.makePragmaToken(next_tok, operator_loc, arg_locs));
	3085	try pp.addToken(try pp.makePragmaToken(next_tok, operator_loc, arg_locs));
2979	3086	if (next_tok.id == .nl) break;
2980	3087	}
2981	3088	if (pp.comp.getPragma(name)) \|prag\| unknown: {
...	...	@@ -2995,7 +3102,7 @@ fn findIncludeFilenameToken(
2995	3102	first_token: RawToken,
2996	3103	tokenizer: *Tokenizer,
2997	3104	trailing_token_behavior: enum { ignore_trailing_tokens, expect_nl_eof },
2998		) !Token {
	3105	) !TokenWithExpansionLocs {
2999	3106	var first = first_token;
3000	3107
3001	3108	if (first.id == .angle_bracket_left) to_end: {
...	...	@@ -3025,14 +3132,13 @@ fn findIncludeFilenameToken(
3025	3132	else => expanded: {
3026	3133	// Try to expand if the argument is a macro.
3027	3134	pp.top_expansion_buf.items.len = 0;
3028		defer for (pp.top_expansion_buf.items) \|tok\| Token.free(tok.expansion_locs, pp.gpa);
	3135	defer for (pp.top_expansion_buf.items) \|tok\| TokenWithExpansionLocs.free(tok.expansion_locs, pp.gpa);
3029	3136	try pp.top_expansion_buf.append(source_tok);
3030	3137	pp.expansion_source_loc = source_tok.loc;
3031	3138
3032	3139	try pp.expandMacroExhaustive(tokenizer, &pp.top_expansion_buf, 0, 1, true, .non_expr);
3033		var trailing_toks: []const Token = &.{};
3034		const include_str = (try pp.reconstructIncludeString(pp.top_expansion_buf.items, &trailing_toks)) orelse {
3035		try pp.err(first, .expected_filename);
	3140	var trailing_toks: []const TokenWithExpansionLocs = &.{};
	3141	const include_str = (try pp.reconstructIncludeString(pp.top_expansion_buf.items, &trailing_toks, tokFromRaw(first))) orelse {
3036	3142	try pp.expectNl(tokenizer);
3037	3143	return error.InvalidInclude;
3038	3144	};
...	...	@@ -3071,7 +3177,7 @@ fn findIncludeFilenameToken(
3071	3177
3072	3178	fn findIncludeSource(pp: Preprocessor, tokenizer: Tokenizer, first: RawToken, which: Compilation.WhichInclude) !Source {
3073	3179	const filename_tok = try pp.findIncludeFilenameToken(first, tokenizer, .expect_nl_eof);
3074		defer Token.free(filename_tok.expansion_locs, pp.gpa);
	3180	defer TokenWithExpansionLocs.free(filename_tok.expansion_locs, pp.gpa);
3075	3181
3076	3182	// Check for empty filename.
3077	3183	const tok_slice = pp.expandedSliceExtra(filename_tok, .single_macro_ws);
...	...	@@ -3101,8 +3207,7 @@ fn printLinemarker(
3101	3207	) !void {
3102	3208	try w.writeByte('#');
3103	3209	if (pp.linemarkers == .line_directives) try w.writeAll("line");
3104		// line_no is 0 indexed
3105		try w.print(" {d} \"", .{line_no + 1});
	3210	try w.print(" {d} \"", .{line_no});
3106	3211	for (source.path) \|byte\| switch (byte) {
3107	3212	'\n' => try w.writeAll("\\n"),
3108	3213	'\r' => try w.writeAll("\\r"),
...	...	@@ -3219,7 +3324,7 @@ pub fn prettyPrintTokens(pp: *Preprocessor, w: anytype) !void {
3219	3324	.include_start => {
3220	3325	const source = pp.comp.getSource(cur.loc.id);
3221	3326
3222		try pp.printLinemarker(w, 0, source, .start);
	3327	try pp.printLinemarker(w, 1, source, .start);
3223	3328	last_nl = true;
3224	3329	},
3225	3330	.include_resume => {
...	...	@@ -3259,7 +3364,7 @@ test "Preserve pragma tokens sometimes" {
3259	3364
3260	3365	const test_runner_macros = try comp.addSourceFromBuffer("<test_runner>", source_text);
3261	3366	const eof = try pp.preprocess(test_runner_macros);
3262		try pp.tokens.append(pp.gpa, eof);
	3367	try pp.addToken(eof);
3263	3368	try pp.prettyPrintTokens(buf.writer());
3264	3369	return allocator.dupe(u8, buf.items);
3265	3370	}

lib/compiler/aro/aro/Toolchain.zig+19

...	...	@@ -487,3 +487,22 @@ pub fn addRuntimeLibs(tc: const Toolchain, argv: std.ArrayList([]const u8)) !v
487	487	try argv.append("-ldl");
488	488	}
489	489	}
	490
	491	pub fn defineSystemIncludes(tc: *Toolchain) !void {
	492	return switch (tc.inner) {
	493	.uninitialized => unreachable,
	494	.linux => \|*linux\| linux.defineSystemIncludes(tc),
	495	.unknown => {
	496	if (tc.driver.nostdinc) return;
	497
	498	const comp = tc.driver.comp;
	499	if (!tc.driver.nobuiltininc) {
	500	try comp.addBuiltinIncludeDir(tc.driver.aro_name);
	501	}
	502
	503	if (!tc.driver.nostdlibinc) {
	504	try comp.addSystemIncludeDir("/usr/include");
	505	}
	506	},
	507	};
	508	}

lib/compiler/aro/aro/Tree.zig+15-10

...	...	@@ -12,6 +12,16 @@ const StringInterner = @import("StringInterner.zig");
12	12
13	13	pub const Token = struct {
14	14	id: Id,
	15	loc: Source.Location,
	16
	17	pub const List = std.MultiArrayList(Token);
	18	pub const Id = Tokenizer.Token.Id;
	19	pub const NumberPrefix = number_affixes.Prefix;
	20	pub const NumberSuffix = number_affixes.Suffix;
	21	};
	22
	23	pub const TokenWithExpansionLocs = struct {
	24	id: Token.Id,
15	25	flags: packed struct {
16	26	expansion_disabled: bool = false,
17	27	is_macro_arg: bool = false,
...	...	@@ -22,15 +32,15 @@ pub const Token = struct {
22	32	loc: Source.Location,
23	33	expansion_locs: ?[*]Source.Location = null,
24	34
25		pub fn expansionSlice(tok: Token) []const Source.Location {
	35	pub fn expansionSlice(tok: TokenWithExpansionLocs) []const Source.Location {
26	36	const locs = tok.expansion_locs orelse return &[0]Source.Location{};
27	37	var i: usize = 0;
28	38	while (locs[i].id != .unused) : (i += 1) {}
29	39	return locs[0..i];
30	40	}
31	41
32		pub fn addExpansionLocation(tok: *Token, gpa: std.mem.Allocator, new: []const Source.Location) !void {
33		if (new.len == 0 or tok.id == .whitespace) return;
	42	pub fn addExpansionLocation(tok: *TokenWithExpansionLocs, gpa: std.mem.Allocator, new: []const Source.Location) !void {
	43	if (new.len == 0 or tok.id == .whitespace or tok.id == .macro_ws or tok.id == .placemarker) return;
34	44	var list = std.ArrayList(Source.Location).init(gpa);
35	45	defer {
36	46	@memset(list.items.ptr[list.items.len..list.capacity], .{});
...	...	@@ -70,14 +80,14 @@ pub const Token = struct {
70	80	gpa.free(locs[0 .. i + 1]);
71	81	}
72	82
73		pub fn dupe(tok: Token, gpa: std.mem.Allocator) !Token {
	83	pub fn dupe(tok: TokenWithExpansionLocs, gpa: std.mem.Allocator) !TokenWithExpansionLocs {
74	84	var copy = tok;
75	85	copy.expansion_locs = null;
76	86	try copy.addExpansionLocation(gpa, tok.expansionSlice());
77	87	return copy;
78	88	}
79	89
80		pub fn checkMsEof(tok: Token, source: Source, comp: *Compilation) !void {
	90	pub fn checkMsEof(tok: TokenWithExpansionLocs, source: Source, comp: *Compilation) !void {
81	91	std.debug.assert(tok.id == .eof);
82	92	if (source.buf.len > tok.loc.byte_offset and source.buf[tok.loc.byte_offset] == 0x1A) {
83	93	try comp.addDiagnostic(.{
...	...	@@ -90,11 +100,6 @@ pub const Token = struct {
90	100	}, &.{});
91	101	}
92	102	}
93
94		pub const List = std.MultiArrayList(Token);
95		pub const Id = Tokenizer.Token.Id;
96		pub const NumberPrefix = number_affixes.Prefix;
97		pub const NumberSuffix = number_affixes.Suffix;
98	103	};
99	104
100	105	pub const TokenIndex = u32;

lib/compiler/aro/aro/Type.zig+5-2

...	...	@@ -105,6 +105,7 @@ pub const Func = struct {
105	105	fn eql(a: const Func, b: const Func, a_spec: Specifier, b_spec: Specifier, comp: *const Compilation) bool {
106	106	// return type cannot have qualifiers
107	107	if (!a.return_type.eql(b.return_type, comp, false)) return false;
	108	if (a.params.len == 0 and b.params.len == 0) return true;
108	109
109	110	if (a.params.len != b.params.len) {
110	111	if (a_spec == .old_style_func or b_spec == .old_style_func) {
...	...	@@ -114,6 +115,7 @@ pub const Func = struct {
114	115	}
115	116	return true;
116	117	}
	118	return false;
117	119	}
118	120	if ((a_spec == .func) != (b_spec == .func)) return false;
119	121	// TODO validate this
...	...	@@ -887,7 +889,8 @@ pub fn hasIncompleteSize(ty: Type) bool {
887	889	.@"struct", .@"union" => ty.data.record.isIncomplete(),
888	890	.array, .static_array => ty.data.array.elem.hasIncompleteSize(),
889	891	.typeof_type => ty.data.sub_type.hasIncompleteSize(),
890		.typeof_expr => ty.data.expr.ty.hasIncompleteSize(),
	892	.typeof_expr, .variable_len_array => ty.data.expr.ty.hasIncompleteSize(),
	893	.unspecified_variable_len_array => ty.data.sub_type.hasIncompleteSize(),
891	894	.attributed => ty.data.attributed.base.hasIncompleteSize(),
892	895	else => false,
893	896	};
...	...	@@ -1053,7 +1056,7 @@ pub fn bitSizeof(ty: Type, comp: *const Compilation) ?u64 {
1053	1056	}
1054	1057
1055	1058	pub fn alignable(ty: Type) bool {
1056		return ty.isArray() or !ty.hasIncompleteSize() or ty.is(.void);
	1059	return (ty.isArray() or !ty.hasIncompleteSize() or ty.is(.void)) and !ty.is(.invalid);
1057	1060	}
1058	1061
1059	1062	/// Get the alignment of a type

lib/compiler/aro/aro/Value.zig+5-3

...	...	@@ -60,7 +60,8 @@ test "minUnsignedBits" {
60	60
61	61	var comp = Compilation.init(std.testing.allocator);
62	62	defer comp.deinit();
63		comp.target = (try std.zig.CrossTarget.parse(.{ .arch_os_abi = "x86_64-linux-gnu" })).toTarget();
	63	const target_query = try std.Target.Query.parse(.{ .arch_os_abi = "x86_64-linux-gnu" });
	64	comp.target = try std.zig.system.resolveTargetQuery(target_query);
64	65
65	66	try Test.checkIntBits(&comp, 0, 0);
66	67	try Test.checkIntBits(&comp, 1, 1);
...	...	@@ -94,7 +95,8 @@ test "minSignedBits" {
94	95
95	96	var comp = Compilation.init(std.testing.allocator);
96	97	defer comp.deinit();
97		comp.target = (try std.zig.CrossTarget.parse(.{ .arch_os_abi = "x86_64-linux-gnu" })).toTarget();
	98	const target_query = try std.Target.Query.parse(.{ .arch_os_abi = "x86_64-linux-gnu" });
	99	comp.target = try std.zig.system.resolveTargetQuery(target_query);
98	100
99	101	try Test.checkIntBits(&comp, -1, 1);
100	102	try Test.checkIntBits(&comp, -2, 2);
...	...	@@ -224,7 +226,7 @@ pub fn intCast(v: Value, dest_ty: Type, comp: Compilation) !void {
224	226	v.* = try intern(comp, .{ .int = .{ .big_int = result_bigint.toConst() } });
225	227	}
226	228
227		/// Converts the stored value from an integer to a float.
	229	/// Converts the stored value to a float of the specified type
228	230	/// `.none` value remains unchanged.
229	231	pub fn floatCast(v: Value, dest_ty: Type, comp: Compilation) !void {
230	232	if (v.opt_ref == .none) return;

lib/compiler/aro/aro/pragmas/gcc.zig+9-9

...	...	@@ -80,7 +80,7 @@ fn diagnosticHandler(self: GCC, pp: Preprocessor, start_idx: TokenIndex) Pragm
80	80	.tag = .pragma_requires_string_literal,
81	81	.loc = diagnostic_tok.loc,
82	82	.extra = .{ .str = "GCC diagnostic" },
83		}, diagnostic_tok.expansionSlice());
	83	}, pp.expansionSlice(start_idx));
84	84	},
85	85	else => \|e\| return e,
86	86	};
...	...	@@ -90,7 +90,7 @@ fn diagnosticHandler(self: GCC, pp: Preprocessor, start_idx: TokenIndex) Pragm
90	90	.tag = .malformed_warning_check,
91	91	.loc = next.loc,
92	92	.extra = .{ .str = "GCC diagnostic" },
93		}, next.expansionSlice());
	93	}, pp.expansionSlice(start_idx + 1));
94	94	}
95	95	const new_kind: Diagnostics.Kind = switch (diagnostic) {
96	96	.ignored => .off,
...	...	@@ -116,7 +116,7 @@ fn preprocessorHandler(pragma: Pragma, pp: Preprocessor, start_idx: TokenIndex
116	116	return pp.comp.addDiagnostic(.{
117	117	.tag = .unknown_gcc_pragma,
118	118	.loc = directive_tok.loc,
119		}, directive_tok.expansionSlice());
	119	}, pp.expansionSlice(start_idx + 1));
120	120
121	121	switch (gcc_pragma) {
122	122	.warning, .@"error" => {
...	...	@@ -126,7 +126,7 @@ fn preprocessorHandler(pragma: Pragma, pp: Preprocessor, start_idx: TokenIndex
126	126	.tag = .pragma_requires_string_literal,
127	127	.loc = directive_tok.loc,
128	128	.extra = .{ .str = @tagName(gcc_pragma) },
129		}, directive_tok.expansionSlice());
	129	}, pp.expansionSlice(start_idx + 1));
130	130	},
131	131	else => \|e\| return e,
132	132	};
...	...	@@ -134,7 +134,7 @@ fn preprocessorHandler(pragma: Pragma, pp: Preprocessor, start_idx: TokenIndex
134	134	const diagnostic_tag: Diagnostics.Tag = if (gcc_pragma == .warning) .pragma_warning_message else .pragma_error_message;
135	135	return pp.comp.addDiagnostic(
136	136	.{ .tag = diagnostic_tag, .loc = directive_tok.loc, .extra = extra },
137		directive_tok.expansionSlice(),
	137	pp.expansionSlice(start_idx + 1),
138	138	);
139	139	},
140	140	.diagnostic => return self.diagnosticHandler(pp, start_idx + 2) catch \|err\| switch (err) {
...	...	@@ -143,12 +143,12 @@ fn preprocessorHandler(pragma: Pragma, pp: Preprocessor, start_idx: TokenIndex
143	143	return pp.comp.addDiagnostic(.{
144	144	.tag = .unknown_gcc_pragma_directive,
145	145	.loc = tok.loc,
146		}, tok.expansionSlice());
	146	}, pp.expansionSlice(start_idx + 2));
147	147	},
148	148	else => \|e\| return e,
149	149	},
150	150	.poison => {
151		var i: usize = 2;
	151	var i: u32 = 2;
152	152	while (true) : (i += 1) {
153	153	const tok = pp.tokens.get(start_idx + i);
154	154	if (tok.id == .nl) break;
...	...	@@ -157,14 +157,14 @@ fn preprocessorHandler(pragma: Pragma, pp: Preprocessor, start_idx: TokenIndex
157	157	return pp.comp.addDiagnostic(.{
158	158	.tag = .pragma_poison_identifier,
159	159	.loc = tok.loc,
160		}, tok.expansionSlice());
	160	}, pp.expansionSlice(start_idx + i));
161	161	}
162	162	const str = pp.expandedSlice(tok);
163	163	if (pp.defines.get(str) != null) {
164	164	try pp.comp.addDiagnostic(.{
165	165	.tag = .pragma_poison_macro,
166	166	.loc = tok.loc,
167		}, tok.expansionSlice());
	167	}, pp.expansionSlice(start_idx + i));
168	168	}
169	169	try pp.poisoned_identifiers.put(str, {});
170	170	}

lib/compiler/aro/aro/pragmas/message.zig+1-1

...	...	@@ -28,7 +28,7 @@ fn deinit(pragma: Pragma, comp: Compilation) void {
28	28
29	29	fn preprocessorHandler(_: Pragma, pp: Preprocessor, start_idx: TokenIndex) Pragma.Error!void {
30	30	const message_tok = pp.tokens.get(start_idx);
31		const message_expansion_locs = message_tok.expansionSlice();
	31	const message_expansion_locs = pp.expansionSlice(start_idx);
32	32
33	33	const str = Pragma.pasteTokens(pp, start_idx + 1) catch \|err\| switch (err) {
34	34	error.ExpectedStringLiteral => {

lib/compiler/aro/aro/pragmas/once.zig+1-1

...	...	@@ -45,7 +45,7 @@ fn preprocessorHandler(pragma: Pragma, pp: Preprocessor, start_idx: TokenIndex
45	45	try pp.comp.addDiagnostic(.{
46	46	.tag = .extra_tokens_directive_end,
47	47	.loc = name_tok.loc,
48		}, next.expansionSlice());
	48	}, pp.expansionSlice(start_idx + 1));
49	49	}
50	50	const seen = self.preprocess_count == pp.preprocess_count;
51	51	const prev = try self.pragma_once.fetchPut(name_tok.loc.id, {});

lib/compiler/aro/aro/pragmas/pack.zig+1-1

...	...	@@ -37,7 +37,7 @@ fn parserHandler(pragma: Pragma, p: Parser, start_idx: TokenIndex) Compilation
37	37	return p.comp.addDiagnostic(.{
38	38	.tag = .pragma_pack_lparen,
39	39	.loc = l_paren.loc,
40		}, l_paren.expansionSlice());
	40	}, p.pp.expansionSlice(idx));
41	41	}
42	42	idx += 1;
43	43

lib/compiler/aro/aro/target.zig+10

...	...	@@ -102,6 +102,16 @@ pub fn int16Type(target: std.Target) Type {
102	102	};
103	103	}
104	104
	105	/// sig_atomic_t for this target
	106	pub fn sigAtomicType(target: std.Target) Type {
	107	if (target.cpu.arch.isWasm()) return .{ .specifier = .long };
	108	return switch (target.cpu.arch) {
	109	.avr => .{ .specifier = .schar },
	110	.msp430 => .{ .specifier = .long },
	111	else => .{ .specifier = .int },
	112	};
	113	}
	114
105	115	/// int64_t for this target
106	116	pub fn int64Type(target: std.Target) Type {
107	117	switch (target.cpu.arch) {

lib/compiler/aro/aro/toolchains/Linux.zig+46-2

...	...	@@ -373,6 +373,50 @@ fn getOSLibDir(target: std.Target) []const u8 {
373	373	return "lib64";
374	374	}
375	375
	376	pub fn defineSystemIncludes(self: const Linux, tc: const Toolchain) !void {
	377	if (tc.driver.nostdinc) return;
	378
	379	const comp = tc.driver.comp;
	380	const target = tc.getTarget();
	381
	382	// musl prefers /usr/include before builtin includes, so musl targets will add builtins
	383	// at the end of this function (unless disabled with nostdlibinc)
	384	if (!tc.driver.nobuiltininc and (!target.isMusl() or tc.driver.nostdlibinc)) {
	385	try comp.addBuiltinIncludeDir(tc.driver.aro_name);
	386	}
	387
	388	if (tc.driver.nostdlibinc) return;
	389
	390	const sysroot = tc.getSysroot();
	391	const local_include = try std.fmt.allocPrint(comp.gpa, "{s}{s}", .{ sysroot, "/usr/local/include" });
	392	defer comp.gpa.free(local_include);
	393	try comp.addSystemIncludeDir(local_include);
	394
	395	if (self.gcc_detector.is_valid) {
	396	const gcc_include_path = try std.fs.path.join(comp.gpa, &.{ self.gcc_detector.parent_lib_path, "..", self.gcc_detector.gcc_triple, "include" });
	397	defer comp.gpa.free(gcc_include_path);
	398	try comp.addSystemIncludeDir(gcc_include_path);
	399	}
	400
	401	if (getMultiarchTriple(target)) \|triple\| {
	402	const joined = try std.fs.path.join(comp.gpa, &.{ sysroot, "usr", "include", triple });
	403	defer comp.gpa.free(joined);
	404	if (tc.filesystem.exists(joined)) {
	405	try comp.addSystemIncludeDir(joined);
	406	}
	407	}
	408
	409	if (target.os.tag == .rtems) return;
	410
	411	try comp.addSystemIncludeDir("/include");
	412	try comp.addSystemIncludeDir("/usr/include");
	413
	414	std.debug.assert(!tc.driver.nostdlibinc);
	415	if (!tc.driver.nobuiltininc and target.isMusl()) {
	416	try comp.addBuiltinIncludeDir(tc.driver.aro_name);
	417	}
	418	}
	419
376	420	test Linux {
377	421	if (@import("builtin").os.tag == .windows) return error.SkipZigTest;
378	422
...	...	@@ -388,8 +432,8 @@ test Linux {
388	432	defer comp.environment = .{};
389	433
390	434	const raw_triple = "x86_64-linux-gnu";
391		const cross = std.zig.CrossTarget.parse(.{ .arch_os_abi = raw_triple }) catch unreachable;
392		comp.target = cross.toTarget(); // TODO deprecated
	435	const target_query = try std.Target.Query.parse(.{ .arch_os_abi = raw_triple });
	436	comp.target = try std.zig.system.resolveTargetQuery(target_query);
393	437	comp.langopts.setEmulatedCompiler(.gcc);
394	438
395	439	var driver: Driver = .{ .comp = &comp };

lib/compiler/aro/backend/Interner.zig+2-2

...	...	@@ -485,11 +485,11 @@ pub fn put(i: *Interner, gpa: Allocator, key: Key) !Ref {
485	485	.data = try i.addExtra(gpa, Tag.F64.pack(data)),
486	486	}),
487	487	.f80 => \|data\| i.items.appendAssumeCapacity(.{
488		.tag = .f64,
	488	.tag = .f80,
489	489	.data = try i.addExtra(gpa, Tag.F80.pack(data)),
490	490	}),
491	491	.f128 => \|data\| i.items.appendAssumeCapacity(.{
492		.tag = .f64,
	492	.tag = .f128,
493	493	.data = try i.addExtra(gpa, Tag.F128.pack(data)),
494	494	}),
495	495	},

lib/compiler/aro/backend/Ir.zig+1-1

...	...	@@ -649,7 +649,7 @@ fn writeValue(ir: Ir, val: Interner.Ref, config: std.io.tty.Config, w: anytype)
649	649	.float => \|repr\| switch (repr) {
650	650	inline else => \|x\| return w.print("{d}", .{@as(f64, @floatCast(x))}),
651	651	},
652		.bytes => \|b\| return std.zig.fmt.stringEscape(b, "", .{}, w),
	652	.bytes => \|b\| return std.zig.stringEscape(b, "", .{}, w),
653	653	else => unreachable, // not a value
654	654	}
655	655	}

author	Veikka Tuominen <git@vexu.eu> 2024-03-06 21:17:41+02:00
committer	GitHub <noreply@github.com> 2024-03-06 14:17:41-05:00
log	90ab8ea9e681a4ffac0b4dc500e3ec489014e12f
tree	5c6d1d6cd24fc5dfca968e11df234cb0db8e83bf
parent	1e67f5021159ed1d9888cf2d0b9f04ef73222f7d
signature	Signed by PGP key B5690EEEBB952194

Sync Aro sources (#19199)