authorgravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2024-07-13 16:54:00-07:00
committergravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2024-07-13 18:19:19-07:00
log1418c8a5d49ecc38a20df07151a7fe942992aea4
tree4386161d9e439aac1b609f8902f0e07e12c44565
parent10914dc31000bc5c3b7164a4101808719d3da07f

ArgIteratorWindows: Store last emitted code unit instead of checking the last 6 emitted bytes

Previously, to ensure args were encoded as well-formed WTF-8 (i.e. no encoded surrogate pairs), the code unit would be encoded and then the last 6 emitted bytes would be checked to see if they were a surrogate pair, and this was done for any emitted code unit (although this was not necessary, it should have only been done when emitting a low surrogate). After this commit, we still want to ensure well-formed WTF-8, but, to do so, the last emitted code point is stored, meaning we can just directly check that the last code unit is a high surrogate and the current code unit is a low surrogate to determine if we have a surrogate pair. This provides some performance benefit over and above a "use the same strategy as before but only check when we're emitting a low surrogate" implementation: Benchmark 1 (111 runs): benchargv-master.exe measurement mean ± σ min … max outliers delta wall_time 45.2ms ± 532us 44.5ms … 49.4ms 2 ( 2%) 0% peak_rss 6.49MB ± 3.94KB 6.46MB … 6.49MB 10 ( 9%) 0% Benchmark 2 (154 runs): benchargv-storelast.exe measurement mean ± σ min … max outliers delta wall_time 32.6ms ± 293us 32.2ms … 34.2ms 8 ( 5%) ⚡- 27.8% ± 0.2% peak_rss 6.49MB ± 5.15KB 6.46MB … 6.49MB 15 (10%) - 0.0% ± 0.0% Benchmark 3 (131 runs): benchargv-onlylow.exe measurement mean ± σ min … max outliers delta wall_time 38.4ms ± 257us 37.9ms … 39.6ms 5 ( 4%) ⚡- 15.1% ± 0.2% peak_rss 6.49MB ± 5.70KB 6.46MB … 6.49MB 9 ( 7%) - 0.0% ± 0.0%

1 files changed, 50 insertions(+), 44 deletions(-)

lib/std/process.zig+50-44
...@@ -717,14 +717,20 @@ pub const ArgIteratorWindows = struct {...@@ -717,14 +717,20 @@ pub const ArgIteratorWindows = struct {
717717
718 const eof = null;718 const eof = null;
719719
720 fn emitBackslashes(self: *ArgIteratorWindows, count: usize) void {720 /// Returns '\' if any backslashes are emitted, otherwise returns `last_emitted_code_unit`.
721 for (0..count) |_| emitCharacter(self, '\\');721 fn emitBackslashes(self: *ArgIteratorWindows, count: usize, last_emitted_code_unit: ?u16) ?u16 {
722 for (0..count) |_| {
723 self.buffer[self.end] = '\\';
724 self.end += 1;
725 }
726 return if (count != 0) '\\' else last_emitted_code_unit;
722 }727 }
723728
724 fn emitCharacter(self: *ArgIteratorWindows, code_unit: u16) void {729 /// If `last_emitted_code_unit` and `code_unit` form a surrogate pair, then
725 const wtf8_len = std.unicode.wtf8Encode(code_unit, self.buffer[self.end..]) catch unreachable;730 /// the previously emitted high surrogate is overwritten by the codepoint encoded
726 self.end += wtf8_len;731 /// by the surrogate pair, and `null` is returned.
727732 /// Otherwise, `code_unit` is emitted and returned.
733 fn emitCharacter(self: *ArgIteratorWindows, code_unit: u16, last_emitted_code_unit: ?u16) ?u16 {
728 // Because we are emitting WTF-8, we need to734 // Because we are emitting WTF-8, we need to
729 // check to see if we've emitted two consecutive surrogate735 // check to see if we've emitted two consecutive surrogate
730 // codepoints that form a valid surrogate pair in order736 // codepoints that form a valid surrogate pair in order
...@@ -745,28 +751,24 @@ pub const ArgIteratorWindows = struct {...@@ -745,28 +751,24 @@ pub const ArgIteratorWindows = struct {
745 // and emit the codepoint it encodes, which in this751 // and emit the codepoint it encodes, which in this
746 // example is U+10437 (𐐷), which is encoded in UTF-8 as:752 // example is U+10437 (𐐷), which is encoded in UTF-8 as:
747 // <0xF0><0x90><0x90><0xB7>753 // <0xF0><0x90><0x90><0xB7>
748 concatSurrogatePair(self);754 if (last_emitted_code_unit != null and
749 }755 std.unicode.utf16IsLowSurrogate(code_unit) and
750756 std.unicode.utf16IsHighSurrogate(last_emitted_code_unit.?))
751 fn concatSurrogatePair(self: *ArgIteratorWindows) void {757 {
752 // Surrogate codepoints are always encoded as 3 bytes, so there758 const codepoint = std.unicode.utf16DecodeSurrogatePair(&.{ last_emitted_code_unit.?, code_unit }) catch unreachable;
753 // must be 6 bytes for a surrogate pair to exist.759
754 if (self.end - self.start >= 6) {760 // Unpaired surrogate is 3 bytes long
755 const window = self.buffer[self.end - 6 .. self.end];761 const dest = self.buffer[self.end - 3 ..];
756 const view = unicode.Wtf8View.init(window) catch return;762 const len = unicode.utf8Encode(codepoint, dest) catch unreachable;
757 var it = view.iterator();763 // All codepoints that require a surrogate pair (> U+FFFF) are encoded as 4 bytes
758 var pair: [2]u16 = undefined;764 assert(len == 4);
759 pair[0] = std.mem.nativeToLittle(u16, std.math.cast(u16, it.nextCodepoint().?) orelse return);765 self.end += 1;
760 if (!unicode.utf16IsHighSurrogate(std.mem.littleToNative(u16, pair[0]))) return;766 return null;
761 pair[1] = std.mem.nativeToLittle(u16, std.math.cast(u16, it.nextCodepoint().?) orelse return);
762 if (!unicode.utf16IsLowSurrogate(std.mem.littleToNative(u16, pair[1]))) return;
763 // We know we have a valid surrogate pair, so convert
764 // it to UTF-8, overwriting the surrogate pair's bytes
765 // and then chop off the extra bytes.
766 const len = unicode.utf16LeToUtf8(window, &pair) catch unreachable;
767 const delta = 6 - len;
768 self.end -= delta;
769 }767 }
768
769 const wtf8_len = std.unicode.wtf8Encode(code_unit, self.buffer[self.end..]) catch unreachable;
770 self.end += wtf8_len;
771 return code_unit;
770 }772 }
771773
772 fn yieldArg(self: *ArgIteratorWindows) [:0]const u8 {774 fn yieldArg(self: *ArgIteratorWindows) [:0]const u8 {
...@@ -783,9 +785,13 @@ pub const ArgIteratorWindows = struct {...@@ -783,9 +785,13 @@ pub const ArgIteratorWindows = struct {
783785
784 const eof = false;786 const eof = false;
785787
786 fn emitBackslashes(_: *ArgIteratorWindows, _: usize) void {}788 fn emitBackslashes(_: *ArgIteratorWindows, _: usize, last_emitted_code_unit: ?u16) ?u16 {
789 return last_emitted_code_unit;
790 }
787791
788 fn emitCharacter(_: *ArgIteratorWindows, _: u16) void {}792 fn emitCharacter(_: *ArgIteratorWindows, _: u16, last_emitted_code_unit: ?u16) ?u16 {
793 return last_emitted_code_unit;
794 }
789795
790 fn yieldArg(_: *ArgIteratorWindows) bool {796 fn yieldArg(_: *ArgIteratorWindows) bool {
791 return true;797 return true;
...@@ -793,6 +799,7 @@ pub const ArgIteratorWindows = struct {...@@ -793,6 +799,7 @@ pub const ArgIteratorWindows = struct {
793 };799 };
794800
795 fn nextWithStrategy(self: *ArgIteratorWindows, comptime strategy: type) strategy.T {801 fn nextWithStrategy(self: *ArgIteratorWindows, comptime strategy: type) strategy.T {
802 var last_emitted_code_unit: ?u16 = null;
796 // The first argument (the executable name) uses different parsing rules.803 // The first argument (the executable name) uses different parsing rules.
797 if (self.index == 0) {804 if (self.index == 0) {
798 if (self.cmd_line.len == 0 or self.cmd_line[0] == 0) {805 if (self.cmd_line.len == 0 or self.cmd_line[0] == 0) {
...@@ -815,15 +822,15 @@ pub const ArgIteratorWindows = struct {...@@ -815,15 +822,15 @@ pub const ArgIteratorWindows = struct {
815 inside_quotes = !inside_quotes;822 inside_quotes = !inside_quotes;
816 },823 },
817 ' ', '\t' => {824 ' ', '\t' => {
818 if (inside_quotes)825 if (inside_quotes) {
819 strategy.emitCharacter(self, char)826 last_emitted_code_unit = strategy.emitCharacter(self, char, last_emitted_code_unit);
820 else {827 } else {
821 self.index += 1;828 self.index += 1;
822 return strategy.yieldArg(self);829 return strategy.yieldArg(self);
823 }830 }
824 },831 },
825 else => {832 else => {
826 strategy.emitCharacter(self, char);833 last_emitted_code_unit = strategy.emitCharacter(self, char, last_emitted_code_unit);
827 },834 },
828 }835 }
829 }836 }
...@@ -861,29 +868,28 @@ pub const ArgIteratorWindows = struct {...@@ -861,29 +868,28 @@ pub const ArgIteratorWindows = struct {
861 0;868 0;
862 switch (char) {869 switch (char) {
863 0 => {870 0 => {
864 strategy.emitBackslashes(self, backslash_count);871 last_emitted_code_unit = strategy.emitBackslashes(self, backslash_count, last_emitted_code_unit);
865 return strategy.yieldArg(self);872 return strategy.yieldArg(self);
866 },873 },
867 ' ', '\t' => {874 ' ', '\t' => {
868 strategy.emitBackslashes(self, backslash_count);875 last_emitted_code_unit = strategy.emitBackslashes(self, backslash_count, last_emitted_code_unit);
869 backslash_count = 0;876 backslash_count = 0;
870 if (inside_quotes)877 if (inside_quotes) {
871 strategy.emitCharacter(self, char)878 last_emitted_code_unit = strategy.emitCharacter(self, char, last_emitted_code_unit);
872 else879 } else return strategy.yieldArg(self);
873 return strategy.yieldArg(self);
874 },880 },
875 '"' => {881 '"' => {
876 const char_is_escaped_quote = backslash_count % 2 != 0;882 const char_is_escaped_quote = backslash_count % 2 != 0;
877 strategy.emitBackslashes(self, backslash_count / 2);883 last_emitted_code_unit = strategy.emitBackslashes(self, backslash_count / 2, last_emitted_code_unit);
878 backslash_count = 0;884 backslash_count = 0;
879 if (char_is_escaped_quote) {885 if (char_is_escaped_quote) {
880 strategy.emitCharacter(self, '"');886 last_emitted_code_unit = strategy.emitCharacter(self, '"', last_emitted_code_unit);
881 } else {887 } else {
882 if (inside_quotes and888 if (inside_quotes and
883 self.index + 1 != self.cmd_line.len and889 self.index + 1 != self.cmd_line.len and
884 mem.littleToNative(u16, self.cmd_line[self.index + 1]) == '"')890 mem.littleToNative(u16, self.cmd_line[self.index + 1]) == '"')
885 {891 {
886 strategy.emitCharacter(self, '"');892 last_emitted_code_unit = strategy.emitCharacter(self, '"', last_emitted_code_unit);
887 self.index += 1;893 self.index += 1;
888 } else {894 } else {
889 inside_quotes = !inside_quotes;895 inside_quotes = !inside_quotes;
...@@ -894,9 +900,9 @@ pub const ArgIteratorWindows = struct {...@@ -894,9 +900,9 @@ pub const ArgIteratorWindows = struct {
894 backslash_count += 1;900 backslash_count += 1;
895 },901 },
896 else => {902 else => {
897 strategy.emitBackslashes(self, backslash_count);903 last_emitted_code_unit = strategy.emitBackslashes(self, backslash_count, last_emitted_code_unit);
898 backslash_count = 0;904 backslash_count = 0;
899 strategy.emitCharacter(self, char);905 last_emitted_code_unit = strategy.emitCharacter(self, char, last_emitted_code_unit);
900 },906 },
901 }907 }
902 }908 }