authorgravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2023-05-04 17:49:23-07:00
committergravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2023-05-13 13:43:50-07:00
log9da3a9733db037d8391eb2b4ccb6c7db774a5dd5
treed89ffa89da45157daaebd9ddd34a5a5af4749d5f
parentbda645d911fd79c5aba1f93c0877c7eb915ed709

std.mem: Split `tokenize` into 3 versions by delimiter type: full, any, and scalar

This allows users to choose which version they need for their particular use case, as the previous default (now the 'any' version) was (1) not always the desired type of delimiter and (2) performed worse than the scalar version if the delimiter was a single item.

1 files changed, 169 insertions(+), 54 deletions(-)

lib/std/mem.zig+169-54
......@@ -1910,72 +1910,117 @@ test "byteSwapAllFields" {
19101910 }, s);
19111911}
19121912
1913/// Deprecated: use `tokenizeAny`, `tokenizeFull`, or `tokenizeScalar`
1914pub const tokenize = tokenizeAny;
1915
19131916/// Returns an iterator that iterates over the slices of `buffer` that are not
1914/// any of the bytes in `delimiter_bytes`.
1917/// any of the items in `delimiters`.
19151918///
1916/// `tokenize(u8, " abc def ghi ", " ")` will return slices
1919/// `tokenizeAny(u8, " abc|def || ghi ", " |")` will return slices
19171920/// for "abc", "def", "ghi", null, in that order.
19181921///
19191922/// If `buffer` is empty, the iterator will return null.
1920/// If `delimiter_bytes` does not exist in buffer,
1923/// If none of `delimiters` exist in buffer,
1924/// the iterator will return `buffer`, null, in that order.
1925///
1926/// See also: `tokenizeFull`, `tokenizeScalar`,
1927/// `splitFull`,`splitAny`, `splitScalar`,
1928/// `splitBackwardsFull`, `splitBackwardsAny`, and `splitBackwardsScalar`
1929pub fn tokenizeAny(comptime T: type, buffer: []const T, delimiters: []const T) TokenIterator(T, .any) {
1930 return .{
1931 .index = 0,
1932 .buffer = buffer,
1933 .delimiter = delimiters,
1934 };
1935}
1936
1937/// Returns an iterator that iterates over the slices of `buffer` that are not
1938/// the sequence in `delimiter`.
1939///
1940/// `tokenizeFull(u8, "<>abc><def<><>ghi", "<>")` will return slices
1941/// for "abc><def", "ghi", null, in that order.
1942///
1943/// If `buffer` is empty, the iterator will return null.
1944/// If `delimiter` does not exist in buffer,
19211945/// the iterator will return `buffer`, null, in that order.
1946/// The delimiter length must not be zero.
19221947///
1923/// See also: `split` and `splitBackwards`.
1924pub fn tokenize(comptime T: type, buffer: []const T, delimiter_bytes: []const T) TokenIterator(T) {
1948/// See also: `tokenizeAny`, `tokenizeScalar`,
1949/// `splitFull`,`splitAny`, and `splitScalar`
1950/// `splitBackwardsFull`, `splitBackwardsAny`, and `splitBackwardsScalar`
1951pub fn tokenizeFull(comptime T: type, buffer: []const T, delimiter: []const T) TokenIterator(T, .full) {
1952 assert(delimiter.len != 0);
19251953 return .{
19261954 .index = 0,
19271955 .buffer = buffer,
1928 .delimiter_bytes = delimiter_bytes,
1956 .delimiter = delimiter,
19291957 };
19301958}
19311959
1932test "tokenize" {
1933 var it = tokenize(u8, " abc def ghi ", " ");
1960/// Returns an iterator that iterates over the slices of `buffer` that are not
1961/// `delimiter`.
1962///
1963/// `tokenizeScalar(u8, " abc def ghi ", ' ')` will return slices
1964/// for "abc", "def", "ghi", null, in that order.
1965///
1966/// If `buffer` is empty, the iterator will return null.
1967/// If `delimiter` does not exist in buffer,
1968/// the iterator will return `buffer`, null, in that order.
1969///
1970/// See also: `tokenizeAny`, `tokenizeFull`,
1971/// `splitFull`,`splitAny`, and `splitScalar`
1972/// `splitBackwardsFull`, `splitBackwardsAny`, and `splitBackwardsScalar`
1973pub fn tokenizeScalar(comptime T: type, buffer: []const T, delimiter: T) TokenIterator(T, .scalar) {
1974 return .{
1975 .index = 0,
1976 .buffer = buffer,
1977 .delimiter = delimiter,
1978 };
1979}
1980
1981test "tokenizeScalar" {
1982 var it = tokenizeScalar(u8, " abc def ghi ", ' ');
19341983 try testing.expect(eql(u8, it.next().?, "abc"));
19351984 try testing.expect(eql(u8, it.peek().?, "def"));
19361985 try testing.expect(eql(u8, it.next().?, "def"));
19371986 try testing.expect(eql(u8, it.next().?, "ghi"));
19381987 try testing.expect(it.next() == null);
19391988
1940 it = tokenize(u8, "..\\bob", "\\");
1989 it = tokenizeScalar(u8, "..\\bob", '\\');
19411990 try testing.expect(eql(u8, it.next().?, ".."));
19421991 try testing.expect(eql(u8, "..", "..\\bob"[0..it.index]));
19431992 try testing.expect(eql(u8, it.next().?, "bob"));
19441993 try testing.expect(it.next() == null);
19451994
1946 it = tokenize(u8, "//a/b", "/");
1995 it = tokenizeScalar(u8, "//a/b", '/');
19471996 try testing.expect(eql(u8, it.next().?, "a"));
19481997 try testing.expect(eql(u8, it.next().?, "b"));
19491998 try testing.expect(eql(u8, "//a/b", "//a/b"[0..it.index]));
19501999 try testing.expect(it.next() == null);
19512000
1952 it = tokenize(u8, "|", "|");
2001 it = tokenizeScalar(u8, "|", '|');
19532002 try testing.expect(it.next() == null);
19542003 try testing.expect(it.peek() == null);
19552004
1956 it = tokenize(u8, "", "|");
2005 it = tokenizeScalar(u8, "", '|');
19572006 try testing.expect(it.next() == null);
19582007 try testing.expect(it.peek() == null);
19592008
1960 it = tokenize(u8, "hello", "");
1961 try testing.expect(eql(u8, it.next().?, "hello"));
1962 try testing.expect(it.next() == null);
1963
1964 it = tokenize(u8, "hello", " ");
2009 it = tokenizeScalar(u8, "hello", ' ');
19652010 try testing.expect(eql(u8, it.next().?, "hello"));
19662011 try testing.expect(it.next() == null);
19672012
1968 var it16 = tokenize(
2013 var it16 = tokenizeScalar(
19692014 u16,
19702015 std.unicode.utf8ToUtf16LeStringLiteral("hello"),
1971 std.unicode.utf8ToUtf16LeStringLiteral(" "),
2016 ' ',
19722017 );
19732018 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("hello")));
19742019 try testing.expect(it16.next() == null);
19752020}
19762021
1977test "tokenize (multibyte)" {
1978 var it = tokenize(u8, "a|b,c/d e", " /,|");
2022test "tokenizeAny (multibyte)" {
2023 var it = tokenizeAny(u8, "a|b,c/d e", " /,|");
19792024 try testing.expect(eql(u8, it.next().?, "a"));
19802025 try testing.expect(eql(u8, it.peek().?, "b"));
19812026 try testing.expect(eql(u8, it.next().?, "b"));
......@@ -1985,7 +2030,11 @@ test "tokenize (multibyte)" {
19852030 try testing.expect(it.next() == null);
19862031 try testing.expect(it.peek() == null);
19872032
1988 var it16 = tokenize(
2033 it = tokenizeAny(u8, "hello", "");
2034 try testing.expect(eql(u8, it.next().?, "hello"));
2035 try testing.expect(it.next() == null);
2036
2037 var it16 = tokenizeAny(
19892038 u16,
19902039 std.unicode.utf8ToUtf16LeStringLiteral("a|b,c/d e"),
19912040 std.unicode.utf8ToUtf16LeStringLiteral(" /,|"),
......@@ -1998,18 +2047,68 @@ test "tokenize (multibyte)" {
19982047 try testing.expect(it16.next() == null);
19992048}
20002049
2050test "tokenizeFull" {
2051 var it = tokenizeFull(u8, "a<>b<><>c><>d><", "<>");
2052 try testing.expectEqualStrings("a", it.next().?);
2053 try testing.expectEqualStrings("b", it.peek().?);
2054 try testing.expectEqualStrings("b", it.next().?);
2055 try testing.expectEqualStrings("c>", it.next().?);
2056 try testing.expectEqualStrings("d><", it.next().?);
2057 try testing.expect(it.next() == null);
2058 try testing.expect(it.peek() == null);
2059
2060 var it16 = tokenizeFull(
2061 u16,
2062 std.unicode.utf8ToUtf16LeStringLiteral("a<>b<><>c><>d><"),
2063 std.unicode.utf8ToUtf16LeStringLiteral("<>"),
2064 );
2065 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("a")));
2066 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("b")));
2067 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("c>")));
2068 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("d><")));
2069 try testing.expect(it16.next() == null);
2070}
2071
20012072test "tokenize (reset)" {
2002 var it = tokenize(u8, " abc def ghi ", " ");
2003 try testing.expect(eql(u8, it.next().?, "abc"));
2004 try testing.expect(eql(u8, it.next().?, "def"));
2005 try testing.expect(eql(u8, it.next().?, "ghi"));
2073 {
2074 var it = tokenizeAny(u8, " abc def ghi ", " ");
2075 try testing.expect(eql(u8, it.next().?, "abc"));
2076 try testing.expect(eql(u8, it.next().?, "def"));
2077 try testing.expect(eql(u8, it.next().?, "ghi"));
20062078
2007 it.reset();
2079 it.reset();
20082080
2009 try testing.expect(eql(u8, it.next().?, "abc"));
2010 try testing.expect(eql(u8, it.next().?, "def"));
2011 try testing.expect(eql(u8, it.next().?, "ghi"));
2012 try testing.expect(it.next() == null);
2081 try testing.expect(eql(u8, it.next().?, "abc"));
2082 try testing.expect(eql(u8, it.next().?, "def"));
2083 try testing.expect(eql(u8, it.next().?, "ghi"));
2084 try testing.expect(it.next() == null);
2085 }
2086 {
2087 var it = tokenizeFull(u8, "<><>abc<>def<><>ghi<>", "<>");
2088 try testing.expect(eql(u8, it.next().?, "abc"));
2089 try testing.expect(eql(u8, it.next().?, "def"));
2090 try testing.expect(eql(u8, it.next().?, "ghi"));
2091
2092 it.reset();
2093
2094 try testing.expect(eql(u8, it.next().?, "abc"));
2095 try testing.expect(eql(u8, it.next().?, "def"));
2096 try testing.expect(eql(u8, it.next().?, "ghi"));
2097 try testing.expect(it.next() == null);
2098 }
2099 {
2100 var it = tokenizeScalar(u8, " abc def ghi ", ' ');
2101 try testing.expect(eql(u8, it.next().?, "abc"));
2102 try testing.expect(eql(u8, it.next().?, "def"));
2103 try testing.expect(eql(u8, it.next().?, "ghi"));
2104
2105 it.reset();
2106
2107 try testing.expect(eql(u8, it.next().?, "abc"));
2108 try testing.expect(eql(u8, it.next().?, "def"));
2109 try testing.expect(eql(u8, it.next().?, "ghi"));
2110 try testing.expect(it.next() == null);
2111 }
20132112}
20142113
20152114/// Deprecated: use `splitFull`, `splitAny`, or `splitScalar`
......@@ -2026,8 +2125,8 @@ pub const split = splitFull;
20262125/// The delimiter length must not be zero.
20272126///
20282127/// See also: `splitAny`, `splitScalar`, `splitBackwardsFull`,
2029/// `splitBackwardsAny`,`splitBackwardsScalar`, and
2030/// `tokenize`.
2128/// `splitBackwardsAny`,`splitBackwardsScalar`,
2129/// `tokenizeAny`, `tokenizeFull`, and `tokenizeScalar`.
20312130pub fn splitFull(comptime T: type, buffer: []const T, delimiter: []const T) SplitIterator(T, .full) {
20322131 assert(delimiter.len != 0);
20332132 return .{
......@@ -2047,8 +2146,8 @@ pub fn splitFull(comptime T: type, buffer: []const T, delimiter: []const T) Spli
20472146/// the iterator will return `buffer`, null, in that order.
20482147///
20492148/// See also: `splitFull`, `splitScalar`, `splitBackwardsFull`,
2050/// `splitBackwardsAny`,`splitBackwardsScalar`, and
2051/// `tokenize`.
2149/// `splitBackwardsAny`,`splitBackwardsScalar`,
2150/// `tokenizeAny`, `tokenizeFull`, and `tokenizeScalar`.
20522151pub fn splitAny(comptime T: type, buffer: []const T, delimiters: []const T) SplitIterator(T, .any) {
20532152 return .{
20542153 .index = 0,
......@@ -2067,8 +2166,8 @@ pub fn splitAny(comptime T: type, buffer: []const T, delimiters: []const T) Spli
20672166/// the iterator will return `buffer`, null, in that order.
20682167///
20692168/// See also: `splitFull`, `splitAny`, `splitBackwardsFull`,
2070/// `splitBackwardsAny`,`splitBackwardsScalar`, and
2071/// `tokenize`.
2169/// `splitBackwardsAny`,`splitBackwardsScalar`,
2170/// `tokenizeAny`, `tokenizeFull`, and `tokenizeScalar`.
20722171pub fn splitScalar(comptime T: type, buffer: []const T, delimiter: T) SplitIterator(T, .scalar) {
20732172 return .{
20742173 .index = 0,
......@@ -2224,8 +2323,8 @@ pub const splitBackwards = splitBackwardsFull;
22242323/// The delimiter length must not be zero.
22252324///
22262325/// See also: `splitBackwardsAny`, `splitBackwardsScalar`,
2227/// `splitFull`, `splitAny`,`splitScalar`, and
2228/// `tokenize`.
2326/// `splitFull`, `splitAny`,`splitScalar`,
2327/// `tokenizeAny`, `tokenizeFull`, and `tokenizeScalar`.
22292328pub fn splitBackwardsFull(comptime T: type, buffer: []const T, delimiter: []const T) SplitBackwardsIterator(T, .full) {
22302329 assert(delimiter.len != 0);
22312330 return .{
......@@ -2245,8 +2344,8 @@ pub fn splitBackwardsFull(comptime T: type, buffer: []const T, delimiter: []cons
22452344/// the iterator will return `buffer`, null, in that order.
22462345///
22472346/// See also: `splitBackwardsFull`, `splitBackwardsScalar`,
2248/// `splitFull`, `splitAny`,`splitScalar`, and
2249/// `tokenize`.
2347/// `splitFull`, `splitAny`,`splitScalar`,
2348/// `tokenizeAny`, `tokenizeFull`, and `tokenizeScalar`.
22502349pub fn splitBackwardsAny(comptime T: type, buffer: []const T, delimiters: []const T) SplitBackwardsIterator(T, .any) {
22512350 return .{
22522351 .index = buffer.len,
......@@ -2265,8 +2364,8 @@ pub fn splitBackwardsAny(comptime T: type, buffer: []const T, delimiters: []cons
22652364/// the iterator will return `buffer`, null, in that order.
22662365///
22672366/// See also: `splitBackwardsFull`, `splitBackwardsAny`,
2268/// `splitFull`, `splitAny`,`splitScalar`, and
2269/// `tokenize`.
2367/// `splitFull`, `splitAny`,`splitScalar`,
2368/// `tokenizeAny`, `tokenizeFull`, and `tokenizeScalar`.
22702369pub fn splitBackwardsScalar(comptime T: type, buffer: []const T, delimiter: T) SplitBackwardsIterator(T, .scalar) {
22712370 return .{
22722371 .index = buffer.len,
......@@ -2596,10 +2695,13 @@ test "endsWith" {
25962695
25972696pub const DelimiterType = enum { full, any, scalar };
25982697
2599pub fn TokenIterator(comptime T: type) type {
2698pub fn TokenIterator(comptime T: type, comptime delimiter_type: DelimiterType) type {
26002699 return struct {
26012700 buffer: []const T,
2602 delimiter_bytes: []const T,
2701 delimiter: switch (delimiter_type) {
2702 .full, .any => []const T,
2703 .scalar => T,
2704 },
26032705 index: usize,
26042706
26052707 const Self = @This();
......@@ -2616,7 +2718,10 @@ pub fn TokenIterator(comptime T: type) type {
26162718 /// complete. Does not advance to the next token.
26172719 pub fn peek(self: *Self) ?[]const T {
26182720 // move to beginning of token
2619 while (self.index < self.buffer.len and self.isSplitByte(self.buffer[self.index])) : (self.index += 1) {}
2721 while (self.index < self.buffer.len and self.isDelimiter(self.index)) : (self.index += switch (delimiter_type) {
2722 .full => self.delimiter.len,
2723 .any, .scalar => 1,
2724 }) {}
26202725 const start = self.index;
26212726 if (start == self.buffer.len) {
26222727 return null;
......@@ -2624,7 +2729,7 @@ pub fn TokenIterator(comptime T: type) type {
26242729
26252730 // move to end of token
26262731 var end = start;
2627 while (end < self.buffer.len and !self.isSplitByte(self.buffer[end])) : (end += 1) {}
2732 while (end < self.buffer.len and !self.isDelimiter(end)) : (end += 1) {}
26282733
26292734 return self.buffer[start..end];
26302735 }
......@@ -2633,7 +2738,10 @@ pub fn TokenIterator(comptime T: type) type {
26332738 pub fn rest(self: Self) []const T {
26342739 // move to beginning of token
26352740 var index: usize = self.index;
2636 while (index < self.buffer.len and self.isSplitByte(self.buffer[index])) : (index += 1) {}
2741 while (index < self.buffer.len and self.isDelimiter(index)) : (index += switch (delimiter_type) {
2742 .full => self.delimiter.len,
2743 .any, .scalar => 1,
2744 }) {}
26372745 return self.buffer[index..];
26382746 }
26392747
......@@ -2642,13 +2750,20 @@ pub fn TokenIterator(comptime T: type) type {
26422750 self.index = 0;
26432751 }
26442752
2645 fn isSplitByte(self: Self, byte: T) bool {
2646 for (self.delimiter_bytes) |delimiter_byte| {
2647 if (byte == delimiter_byte) {
2648 return true;
2649 }
2753 fn isDelimiter(self: Self, index: usize) bool {
2754 switch (delimiter_type) {
2755 .full => return startsWith(T, self.buffer[index..], self.delimiter),
2756 .any => {
2757 const item = self.buffer[index];
2758 for (self.delimiter) |delimiter_item| {
2759 if (item == delimiter_item) {
2760 return true;
2761 }
2762 }
2763 return false;
2764 },
2765 .scalar => return self.buffer[index] == self.delimiter,
26502766 }
2651 return false;
26522767 }
26532768 };
26542769}