authorgravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2021-08-06 01:53:07-07:00
committergravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2021-08-06 01:53:07-07:00
log05fd20dc104b3654ce9c5d7a22a2bff66a940dba
treec4e88a28618da033cbad7832a7a86d57159b9fc4
parentea7bdeb67d474526732b117992971603e4065f98

Make mem.split and mem.tokenize generic instead of assuming u8

This allows these functions to work on slices of u16, etc

1 files changed, 125 insertions(+), 77 deletions(-)

lib/std/mem.zig+125-77
...@@ -1575,8 +1575,8 @@ test "bswapAllFields" {...@@ -1575,8 +1575,8 @@ test "bswapAllFields" {
1575/// If `delimiter_bytes` does not exist in buffer,1575/// If `delimiter_bytes` does not exist in buffer,
1576/// the iterator will return `buffer`, null, in that order.1576/// the iterator will return `buffer`, null, in that order.
1577/// See also the related function `split`.1577/// See also the related function `split`.
1578pub fn tokenize(buffer: []const u8, delimiter_bytes: []const u8) TokenIterator {1578pub fn tokenize(comptime T: type, buffer: []const T, delimiter_bytes: []const T) TokenIterator(T) {
1579 return TokenIterator{1579 return .{
1580 .index = 0,1580 .index = 0,
1581 .buffer = buffer,1581 .buffer = buffer,
1582 .delimiter_bytes = delimiter_bytes,1582 .delimiter_bytes = delimiter_bytes,
...@@ -1584,51 +1584,71 @@ pub fn tokenize(buffer: []const u8, delimiter_bytes: []const u8) TokenIterator {...@@ -1584,51 +1584,71 @@ pub fn tokenize(buffer: []const u8, delimiter_bytes: []const u8) TokenIterator {
1584}1584}
15851585
1586test "mem.tokenize" {1586test "mem.tokenize" {
1587 var it = tokenize(" abc def ghi ", " ");1587 var it = tokenize(u8, " abc def ghi ", " ");
1588 try testing.expect(eql(u8, it.next().?, "abc"));1588 try testing.expect(eql(u8, it.next().?, "abc"));
1589 try testing.expect(eql(u8, it.next().?, "def"));1589 try testing.expect(eql(u8, it.next().?, "def"));
1590 try testing.expect(eql(u8, it.next().?, "ghi"));1590 try testing.expect(eql(u8, it.next().?, "ghi"));
1591 try testing.expect(it.next() == null);1591 try testing.expect(it.next() == null);
15921592
1593 it = tokenize("..\\bob", "\\");1593 it = tokenize(u8, "..\\bob", "\\");
1594 try testing.expect(eql(u8, it.next().?, ".."));1594 try testing.expect(eql(u8, it.next().?, ".."));
1595 try testing.expect(eql(u8, "..", "..\\bob"[0..it.index]));1595 try testing.expect(eql(u8, "..", "..\\bob"[0..it.index]));
1596 try testing.expect(eql(u8, it.next().?, "bob"));1596 try testing.expect(eql(u8, it.next().?, "bob"));
1597 try testing.expect(it.next() == null);1597 try testing.expect(it.next() == null);
15981598
1599 it = tokenize("//a/b", "/");1599 it = tokenize(u8, "//a/b", "/");
1600 try testing.expect(eql(u8, it.next().?, "a"));1600 try testing.expect(eql(u8, it.next().?, "a"));
1601 try testing.expect(eql(u8, it.next().?, "b"));1601 try testing.expect(eql(u8, it.next().?, "b"));
1602 try testing.expect(eql(u8, "//a/b", "//a/b"[0..it.index]));1602 try testing.expect(eql(u8, "//a/b", "//a/b"[0..it.index]));
1603 try testing.expect(it.next() == null);1603 try testing.expect(it.next() == null);
16041604
1605 it = tokenize("|", "|");1605 it = tokenize(u8, "|", "|");
1606 try testing.expect(it.next() == null);1606 try testing.expect(it.next() == null);
16071607
1608 it = tokenize("", "|");1608 it = tokenize(u8, "", "|");
1609 try testing.expect(it.next() == null);1609 try testing.expect(it.next() == null);
16101610
1611 it = tokenize("hello", "");1611 it = tokenize(u8, "hello", "");
1612 try testing.expect(eql(u8, it.next().?, "hello"));1612 try testing.expect(eql(u8, it.next().?, "hello"));
1613 try testing.expect(it.next() == null);1613 try testing.expect(it.next() == null);
16141614
1615 it = tokenize("hello", " ");1615 it = tokenize(u8, "hello", " ");
1616 try testing.expect(eql(u8, it.next().?, "hello"));1616 try testing.expect(eql(u8, it.next().?, "hello"));
1617 try testing.expect(it.next() == null);1617 try testing.expect(it.next() == null);
1618
1619 var it16 = tokenize(
1620 u16,
1621 std.unicode.utf8ToUtf16LeStringLiteral("hello"),
1622 std.unicode.utf8ToUtf16LeStringLiteral(" "),
1623 );
1624 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("hello")));
1625 try testing.expect(it16.next() == null);
1618}1626}
16191627
1620test "mem.tokenize (multibyte)" {1628test "mem.tokenize (multibyte)" {
1621 var it = tokenize("a|b,c/d e", " /,|");1629 var it = tokenize(u8, "a|b,c/d e", " /,|");
1622 try testing.expect(eql(u8, it.next().?, "a"));1630 try testing.expect(eql(u8, it.next().?, "a"));
1623 try testing.expect(eql(u8, it.next().?, "b"));1631 try testing.expect(eql(u8, it.next().?, "b"));
1624 try testing.expect(eql(u8, it.next().?, "c"));1632 try testing.expect(eql(u8, it.next().?, "c"));
1625 try testing.expect(eql(u8, it.next().?, "d"));1633 try testing.expect(eql(u8, it.next().?, "d"));
1626 try testing.expect(eql(u8, it.next().?, "e"));1634 try testing.expect(eql(u8, it.next().?, "e"));
1627 try testing.expect(it.next() == null);1635 try testing.expect(it.next() == null);
1636
1637 var it16 = tokenize(
1638 u16,
1639 std.unicode.utf8ToUtf16LeStringLiteral("a|b,c/d e"),
1640 std.unicode.utf8ToUtf16LeStringLiteral(" /,|"),
1641 );
1642 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("a")));
1643 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("b")));
1644 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("c")));
1645 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("d")));
1646 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("e")));
1647 try testing.expect(it16.next() == null);
1628}1648}
16291649
1630test "mem.tokenize (reset)" {1650test "mem.tokenize (reset)" {
1631 var it = tokenize(" abc def ghi ", " ");1651 var it = tokenize(u8, " abc def ghi ", " ");
1632 try testing.expect(eql(u8, it.next().?, "abc"));1652 try testing.expect(eql(u8, it.next().?, "abc"));
1633 try testing.expect(eql(u8, it.next().?, "def"));1653 try testing.expect(eql(u8, it.next().?, "def"));
1634 try testing.expect(eql(u8, it.next().?, "ghi"));1654 try testing.expect(eql(u8, it.next().?, "ghi"));
...@@ -1649,9 +1669,9 @@ test "mem.tokenize (reset)" {...@@ -1649,9 +1669,9 @@ test "mem.tokenize (reset)" {
1649/// the iterator will return `buffer`, null, in that order.1669/// the iterator will return `buffer`, null, in that order.
1650/// The delimiter length must not be zero.1670/// The delimiter length must not be zero.
1651/// See also the related function `tokenize`.1671/// See also the related function `tokenize`.
1652pub fn split(buffer: []const u8, delimiter: []const u8) SplitIterator {1672pub fn split(comptime T: type, buffer: []const T, delimiter: []const T) SplitIterator(T) {
1653 assert(delimiter.len != 0);1673 assert(delimiter.len != 0);
1654 return SplitIterator{1674 return .{
1655 .index = 0,1675 .index = 0,
1656 .buffer = buffer,1676 .buffer = buffer,
1657 .delimiter = delimiter,1677 .delimiter = delimiter,
...@@ -1661,35 +1681,55 @@ pub fn split(buffer: []const u8, delimiter: []const u8) SplitIterator {...@@ -1661,35 +1681,55 @@ pub fn split(buffer: []const u8, delimiter: []const u8) SplitIterator {
1661pub const separate = @compileError("deprecated: renamed to split (behavior remains unchanged)");1681pub const separate = @compileError("deprecated: renamed to split (behavior remains unchanged)");
16621682
1663test "mem.split" {1683test "mem.split" {
1664 var it = split("abc|def||ghi", "|");1684 var it = split(u8, "abc|def||ghi", "|");
1665 try testing.expect(eql(u8, it.next().?, "abc"));1685 try testing.expect(eql(u8, it.next().?, "abc"));
1666 try testing.expect(eql(u8, it.next().?, "def"));1686 try testing.expect(eql(u8, it.next().?, "def"));
1667 try testing.expect(eql(u8, it.next().?, ""));1687 try testing.expect(eql(u8, it.next().?, ""));
1668 try testing.expect(eql(u8, it.next().?, "ghi"));1688 try testing.expect(eql(u8, it.next().?, "ghi"));
1669 try testing.expect(it.next() == null);1689 try testing.expect(it.next() == null);
16701690
1671 it = split("", "|");1691 it = split(u8, "", "|");
1672 try testing.expect(eql(u8, it.next().?, ""));1692 try testing.expect(eql(u8, it.next().?, ""));
1673 try testing.expect(it.next() == null);1693 try testing.expect(it.next() == null);
16741694
1675 it = split("|", "|");1695 it = split(u8, "|", "|");
1676 try testing.expect(eql(u8, it.next().?, ""));1696 try testing.expect(eql(u8, it.next().?, ""));
1677 try testing.expect(eql(u8, it.next().?, ""));1697 try testing.expect(eql(u8, it.next().?, ""));
1678 try testing.expect(it.next() == null);1698 try testing.expect(it.next() == null);
16791699
1680 it = split("hello", " ");1700 it = split(u8, "hello", " ");
1681 try testing.expect(eql(u8, it.next().?, "hello"));1701 try testing.expect(eql(u8, it.next().?, "hello"));
1682 try testing.expect(it.next() == null);1702 try testing.expect(it.next() == null);
1703
1704 var it16 = split(
1705 u16,
1706 std.unicode.utf8ToUtf16LeStringLiteral("hello"),
1707 std.unicode.utf8ToUtf16LeStringLiteral(" "),
1708 );
1709 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("hello")));
1710 try testing.expect(it16.next() == null);
1683}1711}
16841712
1685test "mem.split (multibyte)" {1713test "mem.split (multibyte)" {
1686 var it = split("a, b ,, c, d, e", ", ");1714 var it = split(u8, "a, b ,, c, d, e", ", ");
1687 try testing.expect(eql(u8, it.next().?, "a"));1715 try testing.expect(eql(u8, it.next().?, "a"));
1688 try testing.expect(eql(u8, it.next().?, "b ,"));1716 try testing.expect(eql(u8, it.next().?, "b ,"));
1689 try testing.expect(eql(u8, it.next().?, "c"));1717 try testing.expect(eql(u8, it.next().?, "c"));
1690 try testing.expect(eql(u8, it.next().?, "d"));1718 try testing.expect(eql(u8, it.next().?, "d"));
1691 try testing.expect(eql(u8, it.next().?, "e"));1719 try testing.expect(eql(u8, it.next().?, "e"));
1692 try testing.expect(it.next() == null);1720 try testing.expect(it.next() == null);
1721
1722 var it16 = split(
1723 u16,
1724 std.unicode.utf8ToUtf16LeStringLiteral("a, b ,, c, d, e"),
1725 std.unicode.utf8ToUtf16LeStringLiteral(", "),
1726 );
1727 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("a")));
1728 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("b ,")));
1729 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("c")));
1730 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("d")));
1731 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("e")));
1732 try testing.expect(it16.next() == null);
1693}1733}
16941734
1695pub fn startsWith(comptime T: type, haystack: []const T, needle: []const T) bool {1735pub fn startsWith(comptime T: type, haystack: []const T, needle: []const T) bool {
...@@ -1710,75 +1750,83 @@ test "mem.endsWith" {...@@ -1710,75 +1750,83 @@ test "mem.endsWith" {
1710 try testing.expect(!endsWith(u8, "Bob", "Bo"));1750 try testing.expect(!endsWith(u8, "Bob", "Bo"));
1711}1751}
17121752
1713pub const TokenIterator = struct {1753pub fn TokenIterator(comptime T: type) type {
1714 buffer: []const u8,1754 return struct {
1715 delimiter_bytes: []const u8,1755 buffer: []const T,
1716 index: usize,1756 delimiter_bytes: []const T,
1757 index: usize,
17171758
1718 /// Returns a slice of the next token, or null if tokenization is complete.1759 const Self = @This();
1719 pub fn next(self: *TokenIterator) ?[]const u8 {
1720 // move to beginning of token
1721 while (self.index < self.buffer.len and self.isSplitByte(self.buffer[self.index])) : (self.index += 1) {}
1722 const start = self.index;
1723 if (start == self.buffer.len) {
1724 return null;
1725 }
17261760
1727 // move to end of token1761 /// Returns a slice of the next token, or null if tokenization is complete.
1728 while (self.index < self.buffer.len and !self.isSplitByte(self.buffer[self.index])) : (self.index += 1) {}1762 pub fn next(self: *Self) ?[]const T {
1729 const end = self.index;1763 // move to beginning of token
1764 while (self.index < self.buffer.len and self.isSplitByte(self.buffer[self.index])) : (self.index += 1) {}
1765 const start = self.index;
1766 if (start == self.buffer.len) {
1767 return null;
1768 }
17301769
1731 return self.buffer[start..end];1770 // move to end of token
1732 }1771 while (self.index < self.buffer.len and !self.isSplitByte(self.buffer[self.index])) : (self.index += 1) {}
1772 const end = self.index;
17331773
1734 /// Returns a slice of the remaining bytes. Does not affect iterator state.1774 return self.buffer[start..end];
1735 pub fn rest(self: TokenIterator) []const u8 {1775 }
1736 // move to beginning of token
1737 var index: usize = self.index;
1738 while (index < self.buffer.len and self.isSplitByte(self.buffer[index])) : (index += 1) {}
1739 return self.buffer[index..];
1740 }
17411776
1742 /// Resets the iterator to the initial token.1777 /// Returns a slice of the remaining bytes. Does not affect iterator state.
1743 pub fn reset(self: *TokenIterator) void {1778 pub fn rest(self: Self) []const T {
1744 self.index = 0;1779 // move to beginning of token
1745 }1780 var index: usize = self.index;
1781 while (index < self.buffer.len and self.isSplitByte(self.buffer[index])) : (index += 1) {}
1782 return self.buffer[index..];
1783 }
1784
1785 /// Resets the iterator to the initial token.
1786 pub fn reset(self: *Self) void {
1787 self.index = 0;
1788 }
17461789
1747 fn isSplitByte(self: TokenIterator, byte: u8) bool {1790 fn isSplitByte(self: Self, byte: T) bool {
1748 for (self.delimiter_bytes) |delimiter_byte| {1791 for (self.delimiter_bytes) |delimiter_byte| {
1749 if (byte == delimiter_byte) {1792 if (byte == delimiter_byte) {
1750 return true;1793 return true;
1794 }
1751 }1795 }
1796 return false;
1752 }1797 }
1753 return false;1798 };
1754 }1799}
1755};
17561800
1757pub const SplitIterator = struct {1801pub fn SplitIterator(comptime T: type) type {
1758 buffer: []const u8,1802 return struct {
1759 index: ?usize,1803 buffer: []const T,
1760 delimiter: []const u8,1804 index: ?usize,
17611805 delimiter: []const T,
1762 /// Returns a slice of the next field, or null if splitting is complete.
1763 pub fn next(self: *SplitIterator) ?[]const u8 {
1764 const start = self.index orelse return null;
1765 const end = if (indexOfPos(u8, self.buffer, start, self.delimiter)) |delim_start| blk: {
1766 self.index = delim_start + self.delimiter.len;
1767 break :blk delim_start;
1768 } else blk: {
1769 self.index = null;
1770 break :blk self.buffer.len;
1771 };
1772 return self.buffer[start..end];
1773 }
17741806
1775 /// Returns a slice of the remaining bytes. Does not affect iterator state.1807 const Self = @This();
1776 pub fn rest(self: SplitIterator) []const u8 {1808
1777 const end = self.buffer.len;1809 /// Returns a slice of the next field, or null if splitting is complete.
1778 const start = self.index orelse end;1810 pub fn next(self: *Self) ?[]const T {
1779 return self.buffer[start..end];1811 const start = self.index orelse return null;
1780 }1812 const end = if (indexOfPos(T, self.buffer, start, self.delimiter)) |delim_start| blk: {
1781};1813 self.index = delim_start + self.delimiter.len;
1814 break :blk delim_start;
1815 } else blk: {
1816 self.index = null;
1817 break :blk self.buffer.len;
1818 };
1819 return self.buffer[start..end];
1820 }
1821
1822 /// Returns a slice of the remaining bytes. Does not affect iterator state.
1823 pub fn rest(self: Self) []const T {
1824 const end = self.buffer.len;
1825 const start = self.index orelse end;
1826 return self.buffer[start..end];
1827 }
1828 };
1829}
17821830
1783/// Naively combines a series of slices with a separator.1831/// Naively combines a series of slices with a separator.
1784/// Allocates memory for the result, which must be freed by the caller.1832/// Allocates memory for the result, which must be freed by the caller.