Commit f4c4c04f1c6f48b4d414a6ef88ff76c26f2d4f8a | zig

author	Ryan Liptak <squeek502@hotmail.com> 2024-04-15 02:09:19-07:00
committer	Ryan Liptak <squeek502@hotmail.com> 2024-04-15 02:09:48-07:00
log	f4c4c04f1c6f48b4d414a6ef88ff76c26f2d4f8a
tree	81723797823c361fa2c97de34b35d6a15b55d103
parent	d979df585d05de8d7385495fe6aee2b1d4e1380f

ArgIteratorWindows: Match post-2008 C runtime rather than CommandLineToArgvW

On Windows, the command line arguments of a program are a single WTF-16 encoded string and it's up to the program to split it into an array of strings. In C/C++, the entry point of the C runtime takes care of splitting the command line and passing argc/argv to the main function.

https://github.com/ziglang/zig/pull/18309 updated ArgIteratorWindows to match the behavior of CommandLineToArgvW, but it turns out that CommandLineToArgvW's behavior does not match the behavior of the C runtime post-2008. In 2008, the C runtime argv splitting changed how it handles consecutive double quotes within a quoted argument (it's now considered an escaped quote, e.g. `"foo""bar"` post-2008 would get parsed into `foo"bar`), and the rules around argv[0] were also changed.

This commit makes ArgIteratorWindows match the behavior of the post-2008 C runtime, and adds a standalone test that verifies the behavior matches both the MSVC and MinGW argv splitting exactly in all cases (it checks that randomly generated command line strings get split the same way).

The motivation here is roughly the same as when the same change was made in Rust (https://github.com/rust-lang/rust/pull/87580), that is (paraphrased):

- Consistent behavior between Zig and modern C/C++ programs
- Allows users to escape double quotes in a way that can be more straightforward

Additionally, the suggested mitigation for BatBadBut (https://flatt.tech/research/posts/batbadbut-you-cant-securely-execute-commands-on-windows/) relies on the post-2008 argv splitting behavior for roundtripping of the arguments given to `cmd.exe`. Note: it's not necessary for the suggested mitigation to work, but it is necessary for the suggested escaping to be parsed back into the intended argv by ArgIteratorWindows after being run through a `.bat` file.

8 files changed, 490 insertions(+), 84 deletions(-)

lib
- std
  - process.zig
test
- standalone
  - build.zig.zon
  - windows_argv
    README.md
    build.zig
    fuzz.zig
    lib.h
    lib.zig
    verify.c

lib/std/process.zig+147-84

...	@@ -625,11 +625,22 @@ pub const ArgIteratorWasi = struct {	...	@@ -625,11 +625,22 @@ pub const ArgIteratorWasi = struct {
625	};	625	};
626		626
627	/// Iterator that implements the Windows command-line parsing algorithm.	627	/// Iterator that implements the Windows command-line parsing algorithm.
		628	/// The implementation is intended to be compatible with the post-2008 C runtime,
		629	/// but is not intended to be compatible with `CommandLineToArgvW` since
		630	/// `CommandLineToArgvW` uses the pre-2008 parsing rules.
628	///	631	///
629	/// This iterator faithfully implements the parsing behavior observed in `CommandLineToArgvW` with	632	/// This iterator faithfully implements the parsing behavior observed from the C runtime with
630	/// one exception: if the command-line string is empty, the iterator will immediately complete	633	/// one exception: if the command-line string is empty, the iterator will immediately complete
631	/// without returning any arguments (whereas `CommandLineArgvW` will return a single argument	634	/// without returning any arguments (whereas the C runtime will return a single argument
632	/// representing the name of the current executable).	635	/// representing the name of the current executable).
		636	///
		637	/// The essential parts of the algorithm are described in Microsoft's documentation:
		638	///
		639	/// - https://learn.microsoft.com/en-us/cpp/cpp/main-function-command-line-args?view=msvc-170#parsing-c-command-line-arguments
		640	///
		641	/// David Deley explains some additional undocumented quirks in great detail:
		642	///
		643	/// - https://daviddeley.com/autohotkey/parameters/parameters.htm#WINCRULES
633	pub const ArgIteratorWindows = struct {	644	pub const ArgIteratorWindows = struct {
634	allocator: Allocator,	645	allocator: Allocator,
635	/// Owned by the iterator.	646	/// Owned by the iterator.
...	@@ -686,6 +697,51 @@ pub const ArgIteratorWindows = struct {	...	@@ -686,6 +697,51 @@ pub const ArgIteratorWindows = struct {
686	fn emitCharacter(self: *ArgIteratorWindows, char: u8) void {	697	fn emitCharacter(self: *ArgIteratorWindows, char: u8) void {
687	self.buffer[self.end] = char;	698	self.buffer[self.end] = char;
688	self.end += 1;	699	self.end += 1;
		700
		701	// Because we are emitting WTF-8 byte-by-byte, we need to
		702	// check to see if we've emitted two consecutive surrogate
		703	// codepoints that form a valid surrogate pair in order
		704	// to ensure that we're always emitting well-formed WTF-8
		705	// (https://simonsapin.github.io/wtf-8/#concatenating).
		706	//
		707	// If we do have a valid surrogate pair, we need to emit
		708	// the UTF-8 sequence for the codepoint that they encode
		709	// instead of the WTF-8 encoding for the two surrogate pairs
		710	// separately.
		711	//
		712	// This is relevant when dealing with a WTF-16 encoded
		713	// command line like this:
		714	// "<0xD801>"<0xDC37>
		715	// which would get converted to WTF-8 in `cmd_line` as:
		716	// "<0xED><0xA0><0x81>"<0xED><0xB0><0xB7>
		717	// and then after parsing it'd naively get emitted as:
		718	// <0xED><0xA0><0x81><0xED><0xB0><0xB7>
		719	// but instead, we need to recognize the surrogate pair
		720	// and emit the codepoint it encodes, which in this
		721	// example is U+10437 (𐐷), which is encoded in UTF-8 as:
		722	// <0xF0><0x90><0x90><0xB7>
		723	concatSurrogatePair(self);
		724	}
		725
		726	fn concatSurrogatePair(self: *ArgIteratorWindows) void {
		727	// Surrogate codepoints are always encoded as 3 bytes, so there
		728	// must be 6 bytes for a surrogate pair to exist.
		729	if (self.end - self.start >= 6) {
		730	const window = self.buffer[self.end - 6 .. self.end];
		731	const view = std.unicode.Wtf8View.init(window) catch return;
		732	var it = view.iterator();
		733	var pair: [2]u16 = undefined;
		734	pair[0] = std.mem.nativeToLittle(u16, std.math.cast(u16, it.nextCodepoint().?) orelse return);
		735	if (!std.unicode.utf16IsHighSurrogate(std.mem.littleToNative(u16, pair[0]))) return;
		736	pair[1] = std.mem.nativeToLittle(u16, std.math.cast(u16, it.nextCodepoint().?) orelse return);
		737	if (!std.unicode.utf16IsLowSurrogate(std.mem.littleToNative(u16, pair[1]))) return;
		738	// We know we have a valid surrogate pair, so convert
		739	// it to UTF-8, overwriting the surrogate pair's bytes
		740	// and then chop off the extra bytes.
		741	const len = std.unicode.utf16LeToUtf8(window, &pair) catch unreachable;
		742	const delta = 6 - len;
		743	self.end -= delta;
		744	}
689	}	745	}
690		746
691	fn yieldArg(self: *ArgIteratorWindows) [:0]const u8 {	747	fn yieldArg(self: *ArgIteratorWindows) [:0]const u8 {
...	@@ -711,69 +767,37 @@ pub const ArgIteratorWindows = struct {	...	@@ -711,69 +767,37 @@ pub const ArgIteratorWindows = struct {
711	}	767	}
712	};	768	};
713		769
714	// The essential parts of the algorithm are described in Microsoft's documentation:
715	//
716	// - <https://learn.microsoft.com/en-us/cpp/cpp/main-function-command-line-args?view=msvc-170#parsing-c-command-line-arguments>
717	// - <https://learn.microsoft.com/en-us/windows/win32/api/shellapi/nf-shellapi-commandlinetoargvw>
718	//
719	// David Deley explains some additional undocumented quirks in great detail:
720	//
721	// - <https://daviddeley.com/autohotkey/parameters/parameters.htm#WINCRULES>
722	//
723	// Code points <= U+0020 terminating an unquoted first argument was discovered independently by
724	// testing and observing the behavior of 'CommandLineToArgvW' on Windows 10.
725
726	fn nextWithStrategy(self: *ArgIteratorWindows, comptime strategy: type) strategy.T {	770	fn nextWithStrategy(self: *ArgIteratorWindows, comptime strategy: type) strategy.T {
727	// The first argument (the executable name) uses different parsing rules.	771	// The first argument (the executable name) uses different parsing rules.
728	if (self.index == 0) {	772	if (self.index == 0) {
729	var char = if (self.cmd_line.len != 0) self.cmd_line[0] else 0;	773	if (self.cmd_line.len == 0 or self.cmd_line[0] == 0) {
730	switch (char) {	774	// Immediately complete the iterator.
731	0 => {	775	// The C runtime would return the name of the current executable here.
732	// Immediately complete the iterator.	776	return strategy.eof;
733	// 'CommandLineToArgvW' would return the name of the current executable here.	777	}
734	return strategy.eof;	778
735	},	779	var inside_quotes = false;
736	'"' => {	780	while (true) : (self.index += 1) {
737	// If the first character is a quote, read everything until the next quote (then	781	const char = if (self.index != self.cmd_line.len) self.cmd_line[self.index] else 0;
738	// skip that quote), or until the end of the string.	782	switch (char) {
739	self.index += 1;	783	0 => {
740	while (true) : (self.index += 1) {	784	return strategy.yieldArg(self);
741	char = if (self.index != self.cmd_line.len) self.cmd_line[self.index] else 0;	785	},
742	switch (char) {	786	'"' => {
743	0 => {	787	inside_quotes = !inside_quotes;
744	return strategy.yieldArg(self);	788	},
745	},	789	' ', '\t' => {
746	'"' => {	790	if (inside_quotes)
747	self.index += 1;	791	strategy.emitCharacter(self, char)
748	return strategy.yieldArg(self);	792	else {
749	},	793	self.index += 1;
750	else => {	794	return strategy.yieldArg(self);
751	strategy.emitCharacter(self, char);
752	},
753	}
754	}
755	},
756	else => {
757	// Otherwise, read everything until the next space or ASCII control character
758	// (not including DEL) (then skip that character), or until the end of the
759	// string. This means that if the command-line string starts with one of these
760	// characters, the first returned argument will be the empty string.
761	while (true) : (self.index += 1) {
762	char = if (self.index != self.cmd_line.len) self.cmd_line[self.index] else 0;
763	switch (char) {
764	0 => {
765	return strategy.yieldArg(self);
766	},
767	'\x01'...' ' => {
768	self.index += 1;
769	return strategy.yieldArg(self);
770	},
771	else => {
772	strategy.emitCharacter(self, char);
773	},
774	}	795	}
775	}	796	},
776	},	797	else => {
		798	strategy.emitCharacter(self, char);
		799	},
		800	}
777	}	801	}
778	}	802	}
779		803
...	@@ -791,9 +815,10 @@ pub const ArgIteratorWindows = struct {	...	@@ -791,9 +815,10 @@ pub const ArgIteratorWindows = struct {
791	//	815	//
792	// - The end of the string always terminates the current argument.	816	// - The end of the string always terminates the current argument.
793	// - When not in 'inside_quotes' mode, a space or tab terminates the current argument.	817	// - When not in 'inside_quotes' mode, a space or tab terminates the current argument.
794	// - 2n backslashes followed by a quote emit n backslashes. If in 'inside_quotes' and the	818	// - 2n backslashes followed by a quote emit n backslashes (note: n can be zero).
795	// quote is immediately followed by a second quote, one quote is emitted and the other is	819	// If in 'inside_quotes' and the quote is immediately followed by a second quote,
796	// skipped, otherwise, the quote is skipped. Finally, 'inside_quotes' is toggled.	820	// one quote is emitted and the other is skipped, otherwise, the quote is skipped
		821	// and 'inside_quotes' is toggled.
797	// - 2n + 1 backslashes followed by a quote emit n backslashes followed by a quote.	822	// - 2n + 1 backslashes followed by a quote emit n backslashes followed by a quote.
798	// - n backslashes not followed by a quote emit n backslashes.	823	// - n backslashes not followed by a quote emit n backslashes.
799	var backslash_count: usize = 0;	824	var backslash_count: usize = 0;
...	@@ -826,8 +851,9 @@ pub const ArgIteratorWindows = struct {	...	@@ -826,8 +851,9 @@ pub const ArgIteratorWindows = struct {
826	{	851	{
827	strategy.emitCharacter(self, '"');	852	strategy.emitCharacter(self, '"');
828	self.index += 1;	853	self.index += 1;
		854	} else {
		855	inside_quotes = !inside_quotes;
829	}	856	}
830	inside_quotes = !inside_quotes;
831	}	857	}
832	},	858	},
833	'\\' => {	859	'\\' => {
...	@@ -1215,10 +1241,10 @@ test ArgIteratorWindows {	...	@@ -1215,10 +1241,10 @@ test ArgIteratorWindows {
1215	// Separators	1241	// Separators
1216	try t("aa bb cc", &.{ "aa", "bb", "cc" });	1242	try t("aa bb cc", &.{ "aa", "bb", "cc" });
1217	try t("aa\tbb\tcc", &.{ "aa", "bb", "cc" });	1243	try t("aa\tbb\tcc", &.{ "aa", "bb", "cc" });
1218	try t("aa\nbb\ncc", &.{ "aa", "bb\ncc" });	1244	try t("aa\nbb\ncc", &.{"aa\nbb\ncc"});
1219	try t("aa\r\nbb\r\ncc", &.{ "aa", "\nbb\r\ncc" });	1245	try t("aa\r\nbb\r\ncc", &.{"aa\r\nbb\r\ncc"});
1220	try t("aa\rbb\rcc", &.{ "aa", "bb\rcc" });	1246	try t("aa\rbb\rcc", &.{"aa\rbb\rcc"});
1221	try t("aa\x07bb\x07cc", &.{ "aa", "bb\x07cc" });	1247	try t("aa\x07bb\x07cc", &.{"aa\x07bb\x07cc"});
1222	try t("aa\x7Fbb\x7Fcc", &.{"aa\x7Fbb\x7Fcc"});	1248	try t("aa\x7Fbb\x7Fcc", &.{"aa\x7Fbb\x7Fcc"});
1223	try t("aa🦎bb🦎cc", &.{"aa🦎bb🦎cc"});	1249	try t("aa🦎bb🦎cc", &.{"aa🦎bb🦎cc"});
1224		1250
...	@@ -1227,22 +1253,22 @@ test ArgIteratorWindows {	...	@@ -1227,22 +1253,22 @@ test ArgIteratorWindows {
1227	try t(" aa bb ", &.{ "", "aa", "bb" });	1253	try t(" aa bb ", &.{ "", "aa", "bb" });
1228	try t("\t\t", &.{""});	1254	try t("\t\t", &.{""});
1229	try t("\t\taa\t\tbb\t\t", &.{ "", "aa", "bb" });	1255	try t("\t\taa\t\tbb\t\t", &.{ "", "aa", "bb" });
1230	try t("\n\n", &.{ "", "\n" });	1256	try t("\n\n", &.{"\n\n"});
1231	try t("\n\naa\n\nbb\n\n", &.{ "", "\naa\n\nbb\n\n" });	1257	try t("\n\naa\n\nbb\n\n", &.{"\n\naa\n\nbb\n\n"});
1232		1258
1233	// Executable name with quotes/backslashes	1259	// Executable name with quotes/backslashes
1234	try t("\"aa bb\tcc\ndd\"", &.{"aa bb\tcc\ndd"});	1260	try t("\"aa bb\tcc\ndd\"", &.{"aa bb\tcc\ndd"});
1235	try t("\"", &.{""});	1261	try t("\"", &.{""});
1236	try t("\"\"", &.{""});	1262	try t("\"\"", &.{""});
1237	try t("\"\"\"", &.{ "", "" });	1263	try t("\"\"\"", &.{""});
1238	try t("\"\"\"\"", &.{ "", "" });	1264	try t("\"\"\"\"", &.{""});
1239	try t("\"\"\"\"\"", &.{ "", "\"" });	1265	try t("\"\"\"\"\"", &.{""});
1240	try t("aa\"bb\"cc\"dd", &.{"aa\"bb\"cc\"dd"});	1266	try t("aa\"bb\"cc\"dd", &.{"aabbccdd"});
1241	try t("aa\"bb cc\"dd", &.{ "aa\"bb", "ccdd" });	1267	try t("aa\"bb cc\"dd", &.{"aabb ccdd"});
1242	try t("\"aa\\\"bb\"", &.{ "aa\\", "bb" });	1268	try t("\"aa\\\"bb\"", &.{"aa\\bb"});
1243	try t("\"aa\\\\\"", &.{"aa\\\\"});	1269	try t("\"aa\\\\\"", &.{"aa\\\\"});
1244	try t("aa\\\"bb", &.{"aa\\\"bb"});	1270	try t("aa\\\"bb", &.{"aa\\bb"});
1245	try t("aa\\\\\"bb", &.{"aa\\\\\"bb"});	1271	try t("aa\\\\\"bb", &.{"aa\\\\bb"});
1246		1272
1247	// Arguments with quotes/backslashes	1273	// Arguments with quotes/backslashes
1248	try t(". \"aa bb\tcc\ndd\"", &.{ ".", "aa bb\tcc\ndd" });	1274	try t(". \"aa bb\tcc\ndd\"", &.{ ".", "aa bb\tcc\ndd" });
...	@@ -1252,29 +1278,66 @@ test ArgIteratorWindows {	...	@@ -1252,29 +1278,66 @@ test ArgIteratorWindows {
1252	try t(". \"\"", &.{ ".", "" });	1278	try t(". \"\"", &.{ ".", "" });
1253	try t(". \"\"\"", &.{ ".", "\"" });	1279	try t(". \"\"\"", &.{ ".", "\"" });
1254	try t(". \"\"\"\"", &.{ ".", "\"" });	1280	try t(". \"\"\"\"", &.{ ".", "\"" });
1255	try t(". \"\"\"\"\"", &.{ ".", "\"" });	1281	try t(". \"\"\"\"\"", &.{ ".", "\"\"" });
1256	try t(". \"\"\"\"\"\"", &.{ ".", "\"\"" });	1282	try t(". \"\"\"\"\"\"", &.{ ".", "\"\"" });
1257	try t(". \" \"", &.{ ".", " " });	1283	try t(". \" \"", &.{ ".", " " });
1258	try t(". \" \"\"", &.{ ".", " \"" });	1284	try t(". \" \"\"", &.{ ".", " \"" });
1259	try t(". \" \"\"\"", &.{ ".", " \"" });	1285	try t(". \" \"\"\"", &.{ ".", " \"" });
1260	try t(". \" \"\"\"\"", &.{ ".", " \"" });	1286	try t(". \" \"\"\"\"", &.{ ".", " \"\"" });
1261	try t(". \" \"\"\"\"\"", &.{ ".", " \"\"" });	1287	try t(". \" \"\"\"\"\"", &.{ ".", " \"\"" });
1262	try t(". \" \"\"\"\"\"\"", &.{ ".", " \"\"" });	1288	try t(". \" \"\"\"\"\"\"", &.{ ".", " \"\"\"" });
1263	try t(". \\\"", &.{ ".", "\"" });	1289	try t(". \\\"", &.{ ".", "\"" });
1264	try t(". \\\"\"", &.{ ".", "\"" });	1290	try t(". \\\"\"", &.{ ".", "\"" });
1265	try t(". \\\"\"\"", &.{ ".", "\"" });	1291	try t(". \\\"\"\"", &.{ ".", "\"" });
1266	try t(". \\\"\"\"\"", &.{ ".", "\"\"" });	1292	try t(". \\\"\"\"\"", &.{ ".", "\"\"" });
1267	try t(". \\\"\"\"\"\"", &.{ ".", "\"\"" });	1293	try t(". \\\"\"\"\"\"", &.{ ".", "\"\"" });
1268	try t(". \\\"\"\"\"\"\"", &.{ ".", "\"\"" });	1294	try t(". \\\"\"\"\"\"\"", &.{ ".", "\"\"\"" });
1269	try t(". \" \\\"", &.{ ".", " \"" });	1295	try t(". \" \\\"", &.{ ".", " \"" });
1270	try t(". \" \\\"\"", &.{ ".", " \"" });	1296	try t(". \" \\\"\"", &.{ ".", " \"" });
1271	try t(". \" \\\"\"\"", &.{ ".", " \"\"" });	1297	try t(". \" \\\"\"\"", &.{ ".", " \"\"" });
1272	try t(". \" \\\"\"\"\"", &.{ ".", " \"\"" });	1298	try t(". \" \\\"\"\"\"", &.{ ".", " \"\"" });
1273	try t(". \" \\\"\"\"\"\"", &.{ ".", " \"\"" });	1299	try t(". \" \\\"\"\"\"\"", &.{ ".", " \"\"\"" });
1274	try t(". \" \\\"\"\"\"\"\"", &.{ ".", " \"\"\"" });	1300	try t(". \" \\\"\"\"\"\"\"", &.{ ".", " \"\"\"" });
1275	try t(". aa\\bb\\\\cc\\\\\\dd", &.{ ".", "aa\\bb\\\\cc\\\\\\dd" });	1301	try t(". aa\\bb\\\\cc\\\\\\dd", &.{ ".", "aa\\bb\\\\cc\\\\\\dd" });
1276	try t(". \\\\\\\"aa bb\"", &.{ ".", "\\\"aa", "bb" });	1302	try t(". \\\\\\\"aa bb\"", &.{ ".", "\\\"aa", "bb" });
1277	try t(". \\\\\\\\\"aa bb\"", &.{ ".", "\\\\aa bb" });	1303	try t(". \\\\\\\\\"aa bb\"", &.{ ".", "\\\\aa bb" });
		1304
		1305	// From https://learn.microsoft.com/en-us/cpp/cpp/main-function-command-line-args#results-of-parsing-command-lines
		1306	try t(
		1307	\\foo.exe "abc" d e
		1308	, &.{ "foo.exe", "abc", "d", "e" });
		1309	try t(
		1310	\\foo.exe a\\b d"e f"g h
		1311	, &.{ "foo.exe", "a\\\\b", "de fg", "h" });
		1312	try t(
		1313	\\foo.exe a\\\"b c d
		1314	, &.{ "foo.exe", "a\\\"b", "c", "d" });
		1315	try t(
		1316	\\foo.exe a\\\\"b c" d e
		1317	, &.{ "foo.exe", "a\\\\b c", "d", "e" });
		1318	try t(
		1319	\\foo.exe a"b"" c d
		1320	, &.{ "foo.exe", "ab\" c d" });
		1321
		1322	// From https://daviddeley.com/autohotkey/parameters/parameters.htm#WINCRULESEX
		1323	try t("foo.exe CallMeIshmael", &.{ "foo.exe", "CallMeIshmael" });
		1324	try t("foo.exe \"Call Me Ishmael\"", &.{ "foo.exe", "Call Me Ishmael" });
		1325	try t("foo.exe Cal\"l Me I\"shmael", &.{ "foo.exe", "Call Me Ishmael" });
		1326	try t("foo.exe CallMe\\\"Ishmael", &.{ "foo.exe", "CallMe\"Ishmael" });
		1327	try t("foo.exe \"CallMe\\\"Ishmael\"", &.{ "foo.exe", "CallMe\"Ishmael" });
		1328	try t("foo.exe \"Call Me Ishmael\\\\\"", &.{ "foo.exe", "Call Me Ishmael\\" });
		1329	try t("foo.exe \"CallMe\\\\\\\"Ishmael\"", &.{ "foo.exe", "CallMe\\\"Ishmael" });
		1330	try t("foo.exe a\\\\\\b", &.{ "foo.exe", "a\\\\\\b" });
		1331	try t("foo.exe \"a\\\\\\b\"", &.{ "foo.exe", "a\\\\\\b" });
		1332
		1333	// Surrogate pair encoding of 𐐷 separated by quotes.
		1334	// Encoded as WTF-16:
		1335	// "<0xD801>"<0xDC37>
		1336	// Encoded as WTF-8:
		1337	// "<0xED><0xA0><0x81>"<0xED><0xB0><0xB7>
		1338	// During parsing, the quotes drop out and the surrogate pair
		1339	// should end up encoded as its normal UTF-8 representation.
		1340	try t("foo.exe \"\xed\xa0\x81\"\xed\xb0\xb7", &.{ "foo.exe", "𐐷" });
1278	}	1341	}
1279		1342
1280	fn testArgIteratorWindows(cmd_line: []const u8, expected_args: []const []const u8) !void {	1343	fn testArgIteratorWindows(cmd_line: []const u8, expected_args: []const []const u8) !void {

test/standalone/build.zig.zon+3

 @@ -104,6 +104,9 @@
         .windows_spawn = .{
             .path = "windows_spawn",
         },
+        .windows_argv = .{
+            .path = "windows_argv",
+        },
         .self_exe_symlink = .{
             .path = "self_exe_symlink",
         },

test/standalone/windows_argv/README.md created+19

...	@@ -0,0 +1,19 @@
1	Tests that Zig's `std.process.ArgIteratorWindows` is compatible with both the MSVC and MinGW C runtimes' argv splitting algorithms.
2
3	The method of testing is:
4	- Compile a C file with `wmain` as its entry point
5	- The C `wmain` calls a Zig-implemented `verify` function that takes the `argv` from `wmain` and compares it to the argv gotten from `std.proccess.argsAlloc` (which takes `kernel32.GetCommandLineW()` and splits it)
6	- The compiled C program is spawned continuously as a child process by the implementation in `fuzz.zig` with randomly generated command lines
7	+ On Windows, the 'application name' and the 'command line' are disjoint concepts. That is, you can spawn `foo.exe` but set the command line to `bar.exe`, and `CreateProcessW` will spawn `foo.exe` but `argv[0]` will be `bar.exe`. This quirk allows us to test arbitrary `argv[0]` values as well which otherwise wouldn't be possible.
8
9	Note: This is intentionally testing against the C runtime argv splitting and not [`CommandLineToArgvW`](https://learn.microsoft.com/en-us/windows/win32/api/shellapi/nf-shellapi-commandlinetoargvw), since the C runtime argv splitting was updated in 2008 but `CommandLineToArgvW` still uses the pre-2008 algorithm (which differs in both `argv[0]` rules and `""`; see [here](https://daviddeley.com/autohotkey/parameters/parameters.htm#WINCRULESDOC) for details)
10
11	---
12
13	In addition to being run during `zig build test-standalone`, this test can be run on its own via `zig build test` from within this directory.
14
15	When run on its own:
16	- `-Diterations=<num>` can be used to set the max fuzzing iterations, and `-Diterations=0` can be used to fuzz indefinitely
17	- `-Dseed=<num>` can be used to set the PRNG seed for fuzz testing. If not provided, then the seed is chosen at random during `build.zig` compilation.
18
19	On failure, the number of iterations and the seed can be seen in the failing command, e.g. in `path\to\fuzz.exe path\to\verify-msvc.exe 100 2780392459403250529`, the iterations is `100` and the seed is `2780392459403250529`.

test/standalone/windows_argv/build.zig created+88

...	@@ -0,0 +1,88 @@
1	const std = @import("std");
2	const builtin = @import("builtin");
3
4	pub fn build(b: *std.Build) !void {
5	const test_step = b.step("test", "Test it");
6	b.default_step = test_step;
7
8	if (builtin.os.tag != .windows) return;
9
10	const optimize: std.builtin.OptimizeMode = .Debug;
11
12	const lib_msvc = b.addStaticLibrary(.{
13	.name = "toargv-msvc",
14	.root_source_file = .{ .path = "lib.zig" },
15	.target = b.resolveTargetQuery(.{
16	.abi = .msvc,
17	}),
18	.optimize = optimize,
19	});
20	const verify_msvc = b.addExecutable(.{
21	.name = "verify-msvc",
22	.target = b.resolveTargetQuery(.{
23	.abi = .msvc,
24	}),
25	.optimize = optimize,
26	});
27	verify_msvc.addCSourceFile(.{
28	.file = .{ .path = "verify.c" },
29	.flags = &.{ "-DUNICODE", "-D_UNICODE" },
30	});
31	verify_msvc.linkLibrary(lib_msvc);
32	verify_msvc.linkLibC();
33
34	const lib_gnu = b.addStaticLibrary(.{
35	.name = "toargv-gnu",
36	.root_source_file = .{ .path = "lib.zig" },
37	.target = b.resolveTargetQuery(.{
38	.abi = .gnu,
39	}),
40	.optimize = optimize,
41	});
42	const verify_gnu = b.addExecutable(.{
43	.name = "verify-gnu",
44	.target = b.resolveTargetQuery(.{
45	.abi = .gnu,
46	}),
47	.optimize = optimize,
48	});
49	verify_gnu.addCSourceFile(.{
50	.file = .{ .path = "verify.c" },
51	.flags = &.{ "-DUNICODE", "-D_UNICODE" },
52	});
53	verify_gnu.mingw_unicode_entry_point = true;
54	verify_gnu.linkLibrary(lib_gnu);
55	verify_gnu.linkLibC();
56
57	const fuzz = b.addExecutable(.{
58	.name = "fuzz",
59	.root_source_file = .{ .path = "fuzz.zig" },
60	.target = b.host,
61	.optimize = optimize,
62	});
63
64	const fuzz_max_iterations = b.option(u64, "iterations", "The max fuzz iterations (default: 100)") orelse 100;
65	const fuzz_iterations_arg = std.fmt.allocPrint(b.allocator, "{}", .{fuzz_max_iterations}) catch @panic("oom");
66
67	const fuzz_seed = b.option(u64, "seed", "Seed to use for the PRNG (default: random)") orelse seed: {
68	var buf: [8]u8 = undefined;
69	try std.posix.getrandom(&buf);
70	break :seed std.mem.readInt(u64, &buf, builtin.cpu.arch.endian());
71	};
72	const fuzz_seed_arg = std.fmt.allocPrint(b.allocator, "{}", .{fuzz_seed}) catch @panic("oom");
73
74	const run_msvc = b.addRunArtifact(fuzz);
75	run_msvc.setName("fuzz-msvc");
76	run_msvc.addArtifactArg(verify_msvc);
77	run_msvc.addArgs(&.{ fuzz_iterations_arg, fuzz_seed_arg });
78	run_msvc.expectExitCode(0);
79
80	const run_gnu = b.addRunArtifact(fuzz);
81	run_gnu.setName("fuzz-gnu");
82	run_gnu.addArtifactArg(verify_gnu);
83	run_gnu.addArgs(&.{ fuzz_iterations_arg, fuzz_seed_arg });
84	run_gnu.expectExitCode(0);
85
86	test_step.dependOn(&run_msvc.step);
87	test_step.dependOn(&run_gnu.step);
88	}

test/standalone/windows_argv/fuzz.zig created+159

...	@@ -0,0 +1,159 @@
1	const std = @import("std");
2	const builtin = @import("builtin");
3	const windows = std.os.windows;
4	const Allocator = std.mem.Allocator;
5
6	pub fn main() !void {
7	var gpa = std.heap.GeneralPurposeAllocator(.{}){};
8	defer std.debug.assert(gpa.deinit() == .ok);
9	const allocator = gpa.allocator();
10
11	const args = try std.process.argsAlloc(allocator);
12	defer std.process.argsFree(allocator, args);
13
14	if (args.len < 2) return error.MissingArgs;
15
16	const verify_path_wtf8 = args[1];
17	const verify_path_w = try std.unicode.wtf8ToWtf16LeAllocZ(allocator, verify_path_wtf8);
18	defer allocator.free(verify_path_w);
19
20	const iterations: u64 = iterations: {
21	if (args.len < 3) break :iterations 0;
22	break :iterations try std.fmt.parseUnsigned(u64, args[2], 10);
23	};
24
25	var rand_seed = false;
26	const seed: u64 = seed: {
27	if (args.len < 4) {
28	rand_seed = true;
29	var buf: [8]u8 = undefined;
30	try std.posix.getrandom(&buf);
31	break :seed std.mem.readInt(u64, &buf, builtin.cpu.arch.endian());
32	}
33	break :seed try std.fmt.parseUnsigned(u64, args[3], 10);
34	};
35	var random = std.rand.DefaultPrng.init(seed);
36	const rand = random.random();
37
38	// If the seed was not given via the CLI, then output the
39	// randomly chosen seed so that this run can be reproduced
40	if (rand_seed) {
41	std.debug.print("rand seed: {}\n", .{seed});
42	}
43
44	var cmd_line_w_buf = std.ArrayList(u16).init(allocator);
45	defer cmd_line_w_buf.deinit();
46
47	var i: u64 = 0;
48	var errors: u64 = 0;
49	while (iterations == 0 or i < iterations) {
50	const cmd_line_w = try randomCommandLineW(allocator, rand);
51	defer allocator.free(cmd_line_w);
52
53	// avoid known difference for 0-length command lines
54	if (cmd_line_w.len == 0 or cmd_line_w[0] == '\x00') continue;
55
56	const exit_code = try spawnVerify(verify_path_w, cmd_line_w);
57	if (exit_code != 0) {
58	std.debug.print(">>> found discrepancy <<<\n", .{});
59	const cmd_line_wtf8 = try std.unicode.wtf16LeToWtf8Alloc(allocator, cmd_line_w);
60	defer allocator.free(cmd_line_wtf8);
61	std.debug.print("\"{}\"\n\n", .{std.zig.fmtEscapes(cmd_line_wtf8)});
62
63	errors += 1;
64	}
65
66	i += 1;
67	}
68	if (errors > 0) {
69	// we never get here if iterations is 0 so we don't have to worry about that case
70	std.debug.print("found {} discrepancies in {} iterations\n", .{ errors, iterations });
71	return error.FoundDiscrepancies;
72	}
73	}
74
75	fn randomCommandLineW(allocator: Allocator, rand: std.rand.Random) ![:0]const u16 {
76	const Choice = enum {
77	backslash,
78	quote,
79	space,
80	tab,
81	control,
82	printable,
83	non_ascii,
84	};
85
86	const choices = rand.uintAtMostBiased(u16, 256);
87	var buf = try std.ArrayList(u16).initCapacity(allocator, choices);
88	errdefer buf.deinit();
89
90	for (0..choices) \|_\| {
91	const choice = rand.enumValue(Choice);
92	const code_unit = switch (choice) {
93	.backslash => '\\',
94	.quote => '"',
95	.space => ' ',
96	.tab => '\t',
97	.control => switch (rand.uintAtMostBiased(u8, 0x21)) {
98	0x21 => '\x7F',
99	else => \|b\| b,
100	},
101	.printable => '!' + rand.uintAtMostBiased(u8, '~' - '!'),
102	.non_ascii => rand.intRangeAtMostBiased(u16, 0x80, 0xFFFF),
103	};
104	try buf.append(std.mem.nativeToLittle(u16, code_unit));
105	}
106
107	return buf.toOwnedSliceSentinel(0);
108	}
109
110	/// Returns the exit code of the verify process
111	fn spawnVerify(verify_path: [:0]const u16, cmd_line: [:0]const u16) !windows.DWORD {
112	const child_proc = spawn: {
113	var startup_info: windows.STARTUPINFOW = .{
114	.cb = @sizeOf(windows.STARTUPINFOW),
115	.lpReserved = null,
116	.lpDesktop = null,
117	.lpTitle = null,
118	.dwX = 0,
119	.dwY = 0,
120	.dwXSize = 0,
121	.dwYSize = 0,
122	.dwXCountChars = 0,
123	.dwYCountChars = 0,
124	.dwFillAttribute = 0,
125	.dwFlags = windows.STARTF_USESTDHANDLES,
126	.wShowWindow = 0,
127	.cbReserved2 = 0,
128	.lpReserved2 = null,
129	.hStdInput = null,
130	.hStdOutput = null,
131	.hStdError = windows.GetStdHandle(windows.STD_ERROR_HANDLE) catch null,
132	};
133	var proc_info: windows.PROCESS_INFORMATION = undefined;
134
135	try windows.CreateProcessW(
136	@constCast(verify_path.ptr),
137	@constCast(cmd_line.ptr),
138	null,
139	null,
140	windows.TRUE,
141	0,
142	null,
143	null,
144	&startup_info,
145	&proc_info,
146	);
147	windows.CloseHandle(proc_info.hThread);
148
149	break :spawn proc_info.hProcess;
150	};
151	defer windows.CloseHandle(child_proc);
152	try windows.WaitForSingleObjectEx(child_proc, windows.INFINITE, false);
153
154	var exit_code: windows.DWORD = undefined;
155	if (windows.kernel32.GetExitCodeProcess(child_proc, &exit_code) == 0) {
156	return error.UnableToGetExitCode;
157	}
158	return exit_code;
159	}

test/standalone/windows_argv/lib.h created+8

...	@@ -0,0 +1,8 @@
1	#ifndef _LIB_H_
2	#define _LIB_H_
3
4	#include <windows.h>
5
6	int verify(int argc, wchar_t *argv[]);
7
8	#endif
	\ No newline at end of file

test/standalone/windows_argv/lib.zig created+59

...	@@ -0,0 +1,59 @@
1	const std = @import("std");
2
3	/// Returns 1 on success, 0 on failure
4	export fn verify(argc: c_int, argv: []const [:0]const u16) c_int {
5	const argv_slice = argv[0..@intCast(argc)];
6	testArgv(argv_slice) catch \|err\| switch (err) {
7	error.OutOfMemory => @panic("oom"),
8	error.Overflow => @panic("bytes needed to contain args would overflow usize"),
9	error.ArgvMismatch => return 0,
10	};
11	return 1;
12	}
13
14	fn testArgv(expected_args: []const [*:0]const u16) !void {
15	var arena_state = std.heap.ArenaAllocator.init(std.heap.page_allocator);
16	defer arena_state.deinit();
17	const allocator = arena_state.allocator();
18
19	const args = try std.process.argsAlloc(allocator);
20	var wtf8_buf = std.ArrayList(u8).init(allocator);
21
22	var eql = true;
23	if (args.len != expected_args.len) eql = false;
24
25	const min_len = @min(expected_args.len, args.len);
26	for (expected_args[0..min_len], args[0..min_len], 0..) \|expected_arg, arg_wtf8, i\| {
27	wtf8_buf.clearRetainingCapacity();
28	try std.unicode.wtf16LeToWtf8ArrayList(&wtf8_buf, std.mem.span(expected_arg));
29	if (!std.mem.eql(u8, wtf8_buf.items, arg_wtf8)) {
30	std.debug.print("{}: expected: \"{}\"\n", .{ i, std.zig.fmtEscapes(wtf8_buf.items) });
31	std.debug.print("{}: actual: \"{}\"\n", .{ i, std.zig.fmtEscapes(arg_wtf8) });
32	eql = false;
33	}
34	}
35	if (!eql) {
36	for (expected_args[min_len..], min_len..) \|arg, i\| {
37	wtf8_buf.clearRetainingCapacity();
38	try std.unicode.wtf16LeToWtf8ArrayList(&wtf8_buf, std.mem.span(arg));
39	std.debug.print("{}: expected: \"{}\"\n", .{ i, std.zig.fmtEscapes(wtf8_buf.items) });
40	}
41	for (args[min_len..], min_len..) \|arg, i\| {
42	std.debug.print("{}: actual: \"{}\"\n", .{ i, std.zig.fmtEscapes(arg) });
43	}
44	const peb = std.os.windows.peb();
45	const lpCmdLine: [*:0]u16 = @ptrCast(peb.ProcessParameters.CommandLine.Buffer);
46	wtf8_buf.clearRetainingCapacity();
47	try std.unicode.wtf16LeToWtf8ArrayList(&wtf8_buf, std.mem.span(lpCmdLine));
48	std.debug.print("command line: \"{}\"\n", .{std.zig.fmtEscapes(wtf8_buf.items)});
49	std.debug.print("expected argv:\n", .{});
50	std.debug.print("&.{{\n", .{});
51	for (expected_args) \|arg\| {
52	wtf8_buf.clearRetainingCapacity();
53	try std.unicode.wtf16LeToWtf8ArrayList(&wtf8_buf, std.mem.span(arg));
54	std.debug.print(" \"{}\",\n", .{std.zig.fmtEscapes(wtf8_buf.items)});
55	}
56	std.debug.print("}}\n", .{});
57	return error.ArgvMismatch;
58	}
59	}

test/standalone/windows_argv/verify.c created+7

...	@@ -0,0 +1,7 @@
1	#include <windows.h>
2	#include "lib.h"
3
4	int wmain(int argc, wchar_t *argv[]) {
5	if (!verify(argc, argv)) return 1;
6	return 0;
7	}
	\ No newline at end of file