authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-06-03 13:51:02-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-06-03 13:51:02-07:00
log629f0d23b5c0768b5957688591f6fa6216ae4dd3
tree8952bf92a1069fa9dfee49d3fcf8f3c2abafbd4b
parent3add9d8257d9414421acf91823917d9d49b28c6f
parent104f4053a2c3c6a1a2bf801ca5bf88ce4fee7a2a
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #15579 from squeek502/mem-delimiters

Split `std.mem.split` and `tokenize` into `sequence`, `any`, and `scalar` versions

42 files changed, 587 insertions(+), 235 deletions(-)

build.zig+7-7
...@@ -235,7 +235,7 @@ pub fn build(b: *std.Build) !void {...@@ -235,7 +235,7 @@ pub fn build(b: *std.Build) !void {
235 },235 },
236 2 => {236 2 => {
237 // Untagged development build (e.g. 0.10.0-dev.2025+ecf0050a9).237 // Untagged development build (e.g. 0.10.0-dev.2025+ecf0050a9).
238 var it = mem.split(u8, git_describe, "-");238 var it = mem.splitScalar(u8, git_describe, '-');
239 const tagged_ancestor = it.first();239 const tagged_ancestor = it.first();
240 const commit_height = it.next().?;240 const commit_height = it.next().?;
241 const commit_id = it.next().?;241 const commit_id = it.next().?;
...@@ -280,7 +280,7 @@ pub fn build(b: *std.Build) !void {...@@ -280,7 +280,7 @@ pub fn build(b: *std.Build) !void {
280 // That means we also have to rely on stage1 compiled c++ files. We parse config.h to find280 // That means we also have to rely on stage1 compiled c++ files. We parse config.h to find
281 // the information passed on to us from cmake.281 // the information passed on to us from cmake.
282 if (cfg.cmake_prefix_path.len > 0) {282 if (cfg.cmake_prefix_path.len > 0) {
283 var it = mem.tokenize(u8, cfg.cmake_prefix_path, ";");283 var it = mem.tokenizeScalar(u8, cfg.cmake_prefix_path, ';');
284 while (it.next()) |path| {284 while (it.next()) |path| {
285 b.addSearchPrefix(path);285 b.addSearchPrefix(path);
286 }286 }
...@@ -682,7 +682,7 @@ fn addCxxKnownPath(...@@ -682,7 +682,7 @@ fn addCxxKnownPath(
682 if (!std.process.can_spawn)682 if (!std.process.can_spawn)
683 return error.RequiredLibraryNotFound;683 return error.RequiredLibraryNotFound;
684 const path_padded = b.exec(&.{ ctx.cxx_compiler, b.fmt("-print-file-name={s}", .{objname}) });684 const path_padded = b.exec(&.{ ctx.cxx_compiler, b.fmt("-print-file-name={s}", .{objname}) });
685 var tokenizer = mem.tokenize(u8, path_padded, "\r\n");685 var tokenizer = mem.tokenizeAny(u8, path_padded, "\r\n");
686 const path_unpadded = tokenizer.next().?;686 const path_unpadded = tokenizer.next().?;
687 if (mem.eql(u8, path_unpadded, objname)) {687 if (mem.eql(u8, path_unpadded, objname)) {
688 if (errtxt) |msg| {688 if (errtxt) |msg| {
...@@ -705,7 +705,7 @@ fn addCxxKnownPath(...@@ -705,7 +705,7 @@ fn addCxxKnownPath(
705}705}
706706
707fn addCMakeLibraryList(exe: *std.Build.Step.Compile, list: []const u8) void {707fn addCMakeLibraryList(exe: *std.Build.Step.Compile, list: []const u8) void {
708 var it = mem.tokenize(u8, list, ";");708 var it = mem.tokenizeScalar(u8, list, ';');
709 while (it.next()) |lib| {709 while (it.next()) |lib| {
710 if (mem.startsWith(u8, lib, "-l")) {710 if (mem.startsWith(u8, lib, "-l")) {
711 exe.linkSystemLibrary(lib["-l".len..]);711 exe.linkSystemLibrary(lib["-l".len..]);
...@@ -850,18 +850,18 @@ fn parseConfigH(b: *std.Build, config_h_text: []const u8) ?CMakeConfig {...@@ -850,18 +850,18 @@ fn parseConfigH(b: *std.Build, config_h_text: []const u8) ?CMakeConfig {
850 // .prefix = ZIG_LLVM_LINK_MODE parsed manually below850 // .prefix = ZIG_LLVM_LINK_MODE parsed manually below
851 };851 };
852852
853 var lines_it = mem.tokenize(u8, config_h_text, "\r\n");853 var lines_it = mem.tokenizeAny(u8, config_h_text, "\r\n");
854 while (lines_it.next()) |line| {854 while (lines_it.next()) |line| {
855 inline for (mappings) |mapping| {855 inline for (mappings) |mapping| {
856 if (mem.startsWith(u8, line, mapping.prefix)) {856 if (mem.startsWith(u8, line, mapping.prefix)) {
857 var it = mem.split(u8, line, "\"");857 var it = mem.splitScalar(u8, line, '"');
858 _ = it.first(); // skip the stuff before the quote858 _ = it.first(); // skip the stuff before the quote
859 const quoted = it.next().?; // the stuff inside the quote859 const quoted = it.next().?; // the stuff inside the quote
860 @field(ctx, mapping.field) = toNativePathSep(b, quoted);860 @field(ctx, mapping.field) = toNativePathSep(b, quoted);
861 }861 }
862 }862 }
863 if (mem.startsWith(u8, line, "#define ZIG_LLVM_LINK_MODE ")) {863 if (mem.startsWith(u8, line, "#define ZIG_LLVM_LINK_MODE ")) {
864 var it = mem.split(u8, line, "\"");864 var it = mem.splitScalar(u8, line, '"');
865 _ = it.next().?; // skip the stuff before the quote865 _ = it.next().?; // skip the stuff before the quote
866 const quoted = it.next().?; // the stuff inside the quote866 const quoted = it.next().?; // the stuff inside the quote
867 ctx.llvm_linkage = if (mem.eql(u8, quoted, "shared")) .dynamic else .static;867 ctx.llvm_linkage = if (mem.eql(u8, quoted, "shared")) .dynamic else .static;
doc/docgen.zig+1-1
...@@ -1223,7 +1223,7 @@ fn printShell(out: anytype, shell_content: []const u8, escape: bool) !void {...@@ -1223,7 +1223,7 @@ fn printShell(out: anytype, shell_content: []const u8, escape: bool) !void {
1223 const trimmed_shell_content = mem.trim(u8, shell_content, " \n");1223 const trimmed_shell_content = mem.trim(u8, shell_content, " \n");
1224 try out.writeAll("<figure><figcaption class=\"shell-cap\">Shell</figcaption><pre><samp>");1224 try out.writeAll("<figure><figcaption class=\"shell-cap\">Shell</figcaption><pre><samp>");
1225 var cmd_cont: bool = false;1225 var cmd_cont: bool = false;
1226 var iter = std.mem.split(u8, trimmed_shell_content, "\n");1226 var iter = std.mem.splitScalar(u8, trimmed_shell_content, '\n');
1227 while (iter.next()) |orig_line| {1227 while (iter.next()) |orig_line| {
1228 const line = mem.trimRight(u8, orig_line, " ");1228 const line = mem.trimRight(u8, orig_line, " ");
1229 if (!cmd_cont and line.len > 1 and mem.eql(u8, line[0..2], "$ ") and line[line.len - 1] != '\\') {1229 if (!cmd_cont and line.len > 1 and mem.eql(u8, line[0..2], "$ ") and line[line.len - 1] != '\\') {
lib/std/Build.zig+1-1
...@@ -1388,7 +1388,7 @@ pub fn findProgram(self: *Build, names: []const []const u8, paths: []const []con...@@ -1388,7 +1388,7 @@ pub fn findProgram(self: *Build, names: []const []const u8, paths: []const []con
1388 if (fs.path.isAbsolute(name)) {1388 if (fs.path.isAbsolute(name)) {
1389 return name;1389 return name;
1390 }1390 }
1391 var it = mem.tokenize(u8, PATH, &[_]u8{fs.path.delimiter});1391 var it = mem.tokenizeScalar(u8, PATH, fs.path.delimiter);
1392 while (it.next()) |path| {1392 while (it.next()) |path| {
1393 const full_path = self.pathJoin(&.{1393 const full_path = self.pathJoin(&.{
1394 path,1394 path,
lib/std/Build/Cache.zig+2-2
...@@ -438,7 +438,7 @@ pub const Manifest = struct {...@@ -438,7 +438,7 @@ pub const Manifest = struct {
438438
439 const input_file_count = self.files.items.len;439 const input_file_count = self.files.items.len;
440 var any_file_changed = false;440 var any_file_changed = false;
441 var line_iter = mem.tokenize(u8, file_contents, "\n");441 var line_iter = mem.tokenizeScalar(u8, file_contents, '\n');
442 var idx: usize = 0;442 var idx: usize = 0;
443 if (if (line_iter.next()) |line| !std.mem.eql(u8, line, manifest_header) else true) {443 if (if (line_iter.next()) |line| !std.mem.eql(u8, line, manifest_header) else true) {
444 if (try self.upgradeToExclusiveLock()) continue;444 if (try self.upgradeToExclusiveLock()) continue;
...@@ -467,7 +467,7 @@ pub const Manifest = struct {...@@ -467,7 +467,7 @@ pub const Manifest = struct {
467 break :blk new;467 break :blk new;
468 };468 };
469469
470 var iter = mem.tokenize(u8, line, " ");470 var iter = mem.tokenizeScalar(u8, line, ' ');
471 const size = iter.next() orelse return error.InvalidFormat;471 const size = iter.next() orelse return error.InvalidFormat;
472 const inode = iter.next() orelse return error.InvalidFormat;472 const inode = iter.next() orelse return error.InvalidFormat;
473 const mtime_nsec_str = iter.next() orelse return error.InvalidFormat;473 const mtime_nsec_str = iter.next() orelse return error.InvalidFormat;
lib/std/Build/Step/CheckObject.zig+4-4
...@@ -103,8 +103,8 @@ const Action = struct {...@@ -103,8 +103,8 @@ const Action = struct {
103 assert(act.tag == .match or act.tag == .not_present);103 assert(act.tag == .match or act.tag == .not_present);
104 const phrase = act.phrase.resolve(b, step);104 const phrase = act.phrase.resolve(b, step);
105 var candidate_var: ?struct { name: []const u8, value: u64 } = null;105 var candidate_var: ?struct { name: []const u8, value: u64 } = null;
106 var hay_it = mem.tokenize(u8, mem.trim(u8, haystack, " "), " ");106 var hay_it = mem.tokenizeScalar(u8, mem.trim(u8, haystack, " "), ' ');
107 var needle_it = mem.tokenize(u8, mem.trim(u8, phrase, " "), " ");107 var needle_it = mem.tokenizeScalar(u8, mem.trim(u8, phrase, " "), ' ');
108108
109 while (needle_it.next()) |needle_tok| {109 while (needle_it.next()) |needle_tok| {
110 const hay_tok = hay_it.next() orelse return false;110 const hay_tok = hay_it.next() orelse return false;
...@@ -155,7 +155,7 @@ const Action = struct {...@@ -155,7 +155,7 @@ const Action = struct {
155 var op_stack = std.ArrayList(enum { add, sub, mod, mul }).init(gpa);155 var op_stack = std.ArrayList(enum { add, sub, mod, mul }).init(gpa);
156 var values = std.ArrayList(u64).init(gpa);156 var values = std.ArrayList(u64).init(gpa);
157157
158 var it = mem.tokenize(u8, phrase, " ");158 var it = mem.tokenizeScalar(u8, phrase, ' ');
159 while (it.next()) |next| {159 while (it.next()) |next| {
160 if (mem.eql(u8, next, "+")) {160 if (mem.eql(u8, next, "+")) {
161 try op_stack.append(.add);161 try op_stack.append(.add);
...@@ -365,7 +365,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {...@@ -365,7 +365,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {
365 var vars = std.StringHashMap(u64).init(gpa);365 var vars = std.StringHashMap(u64).init(gpa);
366366
367 for (self.checks.items) |chk| {367 for (self.checks.items) |chk| {
368 var it = mem.tokenize(u8, output, "\r\n");368 var it = mem.tokenizeAny(u8, output, "\r\n");
369 for (chk.actions.items) |act| {369 for (chk.actions.items) |act| {
370 switch (act.tag) {370 switch (act.tag) {
371 .match => {371 .match => {
lib/std/Build/Step/Compile.zig+4-4
...@@ -853,7 +853,7 @@ fn runPkgConfig(self: *Compile, lib_name: []const u8) ![]const []const u8 {...@@ -853,7 +853,7 @@ fn runPkgConfig(self: *Compile, lib_name: []const u8) ![]const []const u8 {
853 var zig_args = ArrayList([]const u8).init(b.allocator);853 var zig_args = ArrayList([]const u8).init(b.allocator);
854 defer zig_args.deinit();854 defer zig_args.deinit();
855855
856 var it = mem.tokenize(u8, stdout, " \r\n\t");856 var it = mem.tokenizeAny(u8, stdout, " \r\n\t");
857 while (it.next()) |tok| {857 while (it.next()) |tok| {
858 if (mem.eql(u8, tok, "-I")) {858 if (mem.eql(u8, tok, "-I")) {
859 const dir = it.next() orelse return error.PkgConfigInvalidOutput;859 const dir = it.next() orelse return error.PkgConfigInvalidOutput;
...@@ -2101,10 +2101,10 @@ fn execPkgConfigList(self: *std.Build, out_code: *u8) (PkgConfigError || ExecErr...@@ -2101,10 +2101,10 @@ fn execPkgConfigList(self: *std.Build, out_code: *u8) (PkgConfigError || ExecErr
2101 const stdout = try self.execAllowFail(&[_][]const u8{ "pkg-config", "--list-all" }, out_code, .Ignore);2101 const stdout = try self.execAllowFail(&[_][]const u8{ "pkg-config", "--list-all" }, out_code, .Ignore);
2102 var list = ArrayList(PkgConfigPkg).init(self.allocator);2102 var list = ArrayList(PkgConfigPkg).init(self.allocator);
2103 errdefer list.deinit();2103 errdefer list.deinit();
2104 var line_it = mem.tokenize(u8, stdout, "\r\n");2104 var line_it = mem.tokenizeAny(u8, stdout, "\r\n");
2105 while (line_it.next()) |line| {2105 while (line_it.next()) |line| {
2106 if (mem.trim(u8, line, " \t").len == 0) continue;2106 if (mem.trim(u8, line, " \t").len == 0) continue;
2107 var tok_it = mem.tokenize(u8, line, " \t");2107 var tok_it = mem.tokenizeAny(u8, line, " \t");
2108 try list.append(PkgConfigPkg{2108 try list.append(PkgConfigPkg{
2109 .name = tok_it.next() orelse return error.PkgConfigInvalidOutput,2109 .name = tok_it.next() orelse return error.PkgConfigInvalidOutput,
2110 .desc = tok_it.rest(),2110 .desc = tok_it.rest(),
...@@ -2224,7 +2224,7 @@ fn checkCompileErrors(self: *Compile) !void {...@@ -2224,7 +2224,7 @@ fn checkCompileErrors(self: *Compile) !void {
2224 // Render the expected lines into a string that we can compare verbatim.2224 // Render the expected lines into a string that we can compare verbatim.
2225 var expected_generated = std.ArrayList(u8).init(arena);2225 var expected_generated = std.ArrayList(u8).init(arena);
22262226
2227 var actual_line_it = mem.split(u8, actual_stderr, "\n");2227 var actual_line_it = mem.splitScalar(u8, actual_stderr, '\n');
2228 for (self.expect_errors) |expect_line| {2228 for (self.expect_errors) |expect_line| {
2229 const actual_line = actual_line_it.next() orelse {2229 const actual_line = actual_line_it.next() orelse {
2230 try expected_generated.appendSlice(expect_line);2230 try expected_generated.appendSlice(expect_line);
lib/std/Build/Step/ConfigHeader.zig+4-4
...@@ -250,14 +250,14 @@ fn render_autoconf(...@@ -250,14 +250,14 @@ fn render_autoconf(
250250
251 var any_errors = false;251 var any_errors = false;
252 var line_index: u32 = 0;252 var line_index: u32 = 0;
253 var line_it = std.mem.split(u8, contents, "\n");253 var line_it = std.mem.splitScalar(u8, contents, '\n');
254 while (line_it.next()) |line| : (line_index += 1) {254 while (line_it.next()) |line| : (line_index += 1) {
255 if (!std.mem.startsWith(u8, line, "#")) {255 if (!std.mem.startsWith(u8, line, "#")) {
256 try output.appendSlice(line);256 try output.appendSlice(line);
257 try output.appendSlice("\n");257 try output.appendSlice("\n");
258 continue;258 continue;
259 }259 }
260 var it = std.mem.tokenize(u8, line[1..], " \t\r");260 var it = std.mem.tokenizeAny(u8, line[1..], " \t\r");
261 const undef = it.next().?;261 const undef = it.next().?;
262 if (!std.mem.eql(u8, undef, "undef")) {262 if (!std.mem.eql(u8, undef, "undef")) {
263 try output.appendSlice(line);263 try output.appendSlice(line);
...@@ -297,14 +297,14 @@ fn render_cmake(...@@ -297,14 +297,14 @@ fn render_cmake(
297297
298 var any_errors = false;298 var any_errors = false;
299 var line_index: u32 = 0;299 var line_index: u32 = 0;
300 var line_it = std.mem.split(u8, contents, "\n");300 var line_it = std.mem.splitScalar(u8, contents, '\n');
301 while (line_it.next()) |line| : (line_index += 1) {301 while (line_it.next()) |line| : (line_index += 1) {
302 if (!std.mem.startsWith(u8, line, "#")) {302 if (!std.mem.startsWith(u8, line, "#")) {
303 try output.appendSlice(line);303 try output.appendSlice(line);
304 try output.appendSlice("\n");304 try output.appendSlice("\n");
305 continue;305 continue;
306 }306 }
307 var it = std.mem.tokenize(u8, line[1..], " \t\r");307 var it = std.mem.tokenizeAny(u8, line[1..], " \t\r");
308 const cmakedefine = it.next().?;308 const cmakedefine = it.next().?;
309 if (!std.mem.eql(u8, cmakedefine, "cmakedefine") and309 if (!std.mem.eql(u8, cmakedefine, "cmakedefine") and
310 !std.mem.eql(u8, cmakedefine, "cmakedefine01"))310 !std.mem.eql(u8, cmakedefine, "cmakedefine01"))
lib/std/SemanticVersion.zig+5-5
...@@ -42,8 +42,8 @@ pub fn order(lhs: Version, rhs: Version) std.math.Order {...@@ -42,8 +42,8 @@ pub fn order(lhs: Version, rhs: Version) std.math.Order {
42 if (lhs.pre == null and rhs.pre != null) return .gt;42 if (lhs.pre == null and rhs.pre != null) return .gt;
4343
44 // Iterate over pre-release identifiers until a difference is found.44 // Iterate over pre-release identifiers until a difference is found.
45 var lhs_pre_it = std.mem.split(u8, lhs.pre.?, ".");45 var lhs_pre_it = std.mem.splitScalar(u8, lhs.pre.?, '.');
46 var rhs_pre_it = std.mem.split(u8, rhs.pre.?, ".");46 var rhs_pre_it = std.mem.splitScalar(u8, rhs.pre.?, '.');
47 while (true) {47 while (true) {
48 const next_lid = lhs_pre_it.next();48 const next_lid = lhs_pre_it.next();
49 const next_rid = rhs_pre_it.next();49 const next_rid = rhs_pre_it.next();
...@@ -86,7 +86,7 @@ pub fn parse(text: []const u8) !Version {...@@ -86,7 +86,7 @@ pub fn parse(text: []const u8) !Version {
86 // Parse the required major, minor, and patch numbers.86 // Parse the required major, minor, and patch numbers.
87 const extra_index = std.mem.indexOfAny(u8, text, "-+");87 const extra_index = std.mem.indexOfAny(u8, text, "-+");
88 const required = text[0..(extra_index orelse text.len)];88 const required = text[0..(extra_index orelse text.len)];
89 var it = std.mem.split(u8, required, ".");89 var it = std.mem.splitScalar(u8, required, '.');
90 var ver = Version{90 var ver = Version{
91 .major = try parseNum(it.first()),91 .major = try parseNum(it.first()),
92 .minor = try parseNum(it.next() orelse return error.InvalidVersion),92 .minor = try parseNum(it.next() orelse return error.InvalidVersion),
...@@ -108,7 +108,7 @@ pub fn parse(text: []const u8) !Version {...@@ -108,7 +108,7 @@ pub fn parse(text: []const u8) !Version {
108 // Check validity of optional pre-release identifiers.108 // Check validity of optional pre-release identifiers.
109 // See: https://semver.org/#spec-item-9109 // See: https://semver.org/#spec-item-9
110 if (ver.pre) |pre| {110 if (ver.pre) |pre| {
111 it = std.mem.split(u8, pre, ".");111 it = std.mem.splitScalar(u8, pre, '.');
112 while (it.next()) |id| {112 while (it.next()) |id| {
113 // Identifiers MUST NOT be empty.113 // Identifiers MUST NOT be empty.
114 if (id.len == 0) return error.InvalidVersion;114 if (id.len == 0) return error.InvalidVersion;
...@@ -127,7 +127,7 @@ pub fn parse(text: []const u8) !Version {...@@ -127,7 +127,7 @@ pub fn parse(text: []const u8) !Version {
127 // Check validity of optional build metadata identifiers.127 // Check validity of optional build metadata identifiers.
128 // See: https://semver.org/#spec-item-10128 // See: https://semver.org/#spec-item-10
129 if (ver.build) |build| {129 if (ver.build) |build| {
130 it = std.mem.split(u8, build, ".");130 it = std.mem.splitScalar(u8, build, '.');
131 while (it.next()) |id| {131 while (it.next()) |id| {
132 // Identifiers MUST NOT be empty.132 // Identifiers MUST NOT be empty.
133 if (id.len == 0) return error.InvalidVersion;133 if (id.len == 0) return error.InvalidVersion;
lib/std/builtin.zig+1-1
...@@ -531,7 +531,7 @@ pub const Version = struct {...@@ -531,7 +531,7 @@ pub const Version = struct {
531 // found no digits or '.' before unexpected character531 // found no digits or '.' before unexpected character
532 if (end == 0) return error.InvalidVersion;532 if (end == 0) return error.InvalidVersion;
533533
534 var it = std.mem.split(u8, text[0..end], ".");534 var it = std.mem.splitScalar(u8, text[0..end], '.');
535 // substring is not empty, first call will succeed535 // substring is not empty, first call will succeed
536 const major = it.first();536 const major = it.first();
537 if (major.len == 0) return error.InvalidVersion;537 if (major.len == 0) return error.InvalidVersion;
lib/std/child_process.zig+2-2
...@@ -850,7 +850,7 @@ pub const ChildProcess = struct {...@@ -850,7 +850,7 @@ pub const ChildProcess = struct {
850 return original_err;850 return original_err;
851 }851 }
852852
853 var it = mem.tokenize(u16, PATH, &[_]u16{';'});853 var it = mem.tokenizeScalar(u16, PATH, ';');
854 while (it.next()) |search_path| {854 while (it.next()) |search_path| {
855 dir_buf.clearRetainingCapacity();855 dir_buf.clearRetainingCapacity();
856 try dir_buf.appendSlice(self.allocator, search_path);856 try dir_buf.appendSlice(self.allocator, search_path);
...@@ -1064,7 +1064,7 @@ fn windowsCreateProcessPathExt(...@@ -1064,7 +1064,7 @@ fn windowsCreateProcessPathExt(
1064 // Now we know that at least *a* file matching the wildcard exists, we can loop1064 // Now we know that at least *a* file matching the wildcard exists, we can loop
1065 // through PATHEXT in order and exec any that exist1065 // through PATHEXT in order and exec any that exist
10661066
1067 var ext_it = mem.tokenize(u16, pathext, &[_]u16{';'});1067 var ext_it = mem.tokenizeScalar(u16, pathext, ';');
1068 while (ext_it.next()) |ext| {1068 while (ext_it.next()) |ext| {
1069 if (!windowsCreateProcessSupportsExtension(ext)) continue;1069 if (!windowsCreateProcessSupportsExtension(ext)) continue;
10701070
lib/std/crypto/Certificate.zig+2-2
...@@ -337,8 +337,8 @@ pub const Parsed = struct {...@@ -337,8 +337,8 @@ pub const Parsed = struct {
337 return true; // exact match337 return true; // exact match
338 }338 }
339339
340 var it_host = std.mem.split(u8, host_name, ".");340 var it_host = std.mem.splitScalar(u8, host_name, '.');
341 var it_dns = std.mem.split(u8, dns_name, ".");341 var it_dns = std.mem.splitScalar(u8, dns_name, '.');
342342
343 const len_match = while (true) {343 const len_match = while (true) {
344 const host = it_host.next();344 const host = it_host.next();
lib/std/crypto/phc_encoding.zig+6-3
...@@ -7,9 +7,12 @@ const mem = std.mem;...@@ -7,9 +7,12 @@ const mem = std.mem;
7const meta = std.meta;7const meta = std.meta;
88
9const fields_delimiter = "$";9const fields_delimiter = "$";
10const fields_delimiter_scalar = '$';
10const version_param_name = "v";11const version_param_name = "v";
11const params_delimiter = ",";12const params_delimiter = ",";
13const params_delimiter_scalar = ',';
12const kv_delimiter = "=";14const kv_delimiter = "=";
15const kv_delimiter_scalar = '=';
1316
14pub const Error = std.crypto.errors.EncodingError || error{NoSpaceLeft};17pub const Error = std.crypto.errors.EncodingError || error{NoSpaceLeft};
1518
...@@ -73,7 +76,7 @@ pub fn BinValue(comptime max_len: usize) type {...@@ -73,7 +76,7 @@ pub fn BinValue(comptime max_len: usize) type {
73/// Other fields will also be deserialized from the function parameters section.76/// Other fields will also be deserialized from the function parameters section.
74pub fn deserialize(comptime HashResult: type, str: []const u8) Error!HashResult {77pub fn deserialize(comptime HashResult: type, str: []const u8) Error!HashResult {
75 var out = mem.zeroes(HashResult);78 var out = mem.zeroes(HashResult);
76 var it = mem.split(u8, str, fields_delimiter);79 var it = mem.splitScalar(u8, str, fields_delimiter_scalar);
77 var set_fields: usize = 0;80 var set_fields: usize = 0;
7881
79 while (true) {82 while (true) {
...@@ -104,7 +107,7 @@ pub fn deserialize(comptime HashResult: type, str: []const u8) Error!HashResult...@@ -104,7 +107,7 @@ pub fn deserialize(comptime HashResult: type, str: []const u8) Error!HashResult
104107
105 // Read optional parameters108 // Read optional parameters
106 var has_params = false;109 var has_params = false;
107 var it_params = mem.split(u8, field, params_delimiter);110 var it_params = mem.splitScalar(u8, field, params_delimiter_scalar);
108 while (it_params.next()) |params| {111 while (it_params.next()) |params| {
109 const param = kvSplit(params) catch break;112 const param = kvSplit(params) catch break;
110 var found = false;113 var found = false;
...@@ -252,7 +255,7 @@ fn serializeTo(params: anytype, out: anytype) !void {...@@ -252,7 +255,7 @@ fn serializeTo(params: anytype, out: anytype) !void {
252255
253// Split a `key=value` string into `key` and `value`256// Split a `key=value` string into `key` and `value`
254fn kvSplit(str: []const u8) !struct { key: []const u8, value: []const u8 } {257fn kvSplit(str: []const u8) !struct { key: []const u8, value: []const u8 } {
255 var it = mem.split(u8, str, kv_delimiter);258 var it = mem.splitScalar(u8, str, kv_delimiter_scalar);
256 const key = it.first();259 const key = it.first();
257 const value = it.next() orelse return Error.InvalidEncoding;260 const value = it.next() orelse return Error.InvalidEncoding;
258 const ret = .{ .key = key, .value = value };261 const ret = .{ .key = key, .value = value };
lib/std/crypto/scrypt.zig+1-1
...@@ -287,7 +287,7 @@ const crypt_format = struct {...@@ -287,7 +287,7 @@ const crypt_format = struct {
287 out.r = try Codec.intDecode(u30, str[4..9]);287 out.r = try Codec.intDecode(u30, str[4..9]);
288 out.p = try Codec.intDecode(u30, str[9..14]);288 out.p = try Codec.intDecode(u30, str[9..14]);
289289
290 var it = mem.split(u8, str[14..], "$");290 var it = mem.splitScalar(u8, str[14..], '$');
291291
292 const salt = it.first();292 const salt = it.first();
293 if (@hasField(T, "salt")) out.salt = salt;293 if (@hasField(T, "salt")) out.salt = salt;
lib/std/fs.zig+1-1
...@@ -3021,7 +3021,7 @@ pub fn selfExePath(out_buffer: []u8) SelfExePathError![]u8 {...@@ -3021,7 +3021,7 @@ pub fn selfExePath(out_buffer: []u8) SelfExePathError![]u8 {
3021 } else if (argv0.len != 0) {3021 } else if (argv0.len != 0) {
3022 // argv[0] is not empty (and not a path): search it inside PATH3022 // argv[0] is not empty (and not a path): search it inside PATH
3023 const PATH = std.os.getenvZ("PATH") orelse return error.FileNotFound;3023 const PATH = std.os.getenvZ("PATH") orelse return error.FileNotFound;
3024 var path_it = mem.tokenize(u8, PATH, &[_]u8{path.delimiter});3024 var path_it = mem.tokenizeScalar(u8, PATH, path.delimiter);
3025 while (path_it.next()) |a_path| {3025 while (path_it.next()) |a_path| {
3026 var resolved_path_buf: [MAX_PATH_BYTES - 1:0]u8 = undefined;3026 var resolved_path_buf: [MAX_PATH_BYTES - 1:0]u8 = undefined;
3027 const resolved_path = std.fmt.bufPrintZ(&resolved_path_buf, "{s}/{s}", .{3027 const resolved_path = std.fmt.bufPrintZ(&resolved_path_buf, "{s}/{s}", .{
lib/std/fs/path.zig+13-13
...@@ -358,7 +358,7 @@ pub fn windowsParsePath(path: []const u8) WindowsPath {...@@ -358,7 +358,7 @@ pub fn windowsParsePath(path: []const u8) WindowsPath {
358 return relative_path;358 return relative_path;
359 }359 }
360360
361 var it = mem.tokenize(u8, path, &[_]u8{this_sep});361 var it = mem.tokenizeScalar(u8, path, this_sep);
362 _ = (it.next() orelse return relative_path);362 _ = (it.next() orelse return relative_path);
363 _ = (it.next() orelse return relative_path);363 _ = (it.next() orelse return relative_path);
364 return WindowsPath{364 return WindowsPath{
...@@ -420,8 +420,8 @@ fn networkShareServersEql(ns1: []const u8, ns2: []const u8) bool {...@@ -420,8 +420,8 @@ fn networkShareServersEql(ns1: []const u8, ns2: []const u8) bool {
420 const sep1 = ns1[0];420 const sep1 = ns1[0];
421 const sep2 = ns2[0];421 const sep2 = ns2[0];
422422
423 var it1 = mem.tokenize(u8, ns1, &[_]u8{sep1});423 var it1 = mem.tokenizeScalar(u8, ns1, sep1);
424 var it2 = mem.tokenize(u8, ns2, &[_]u8{sep2});424 var it2 = mem.tokenizeScalar(u8, ns2, sep2);
425425
426 // TODO ASCII is wrong, we actually need full unicode support to compare paths.426 // TODO ASCII is wrong, we actually need full unicode support to compare paths.
427 return ascii.eqlIgnoreCase(it1.next().?, it2.next().?);427 return ascii.eqlIgnoreCase(it1.next().?, it2.next().?);
...@@ -441,8 +441,8 @@ fn compareDiskDesignators(kind: WindowsPath.Kind, p1: []const u8, p2: []const u8...@@ -441,8 +441,8 @@ fn compareDiskDesignators(kind: WindowsPath.Kind, p1: []const u8, p2: []const u8
441 const sep1 = p1[0];441 const sep1 = p1[0];
442 const sep2 = p2[0];442 const sep2 = p2[0];
443443
444 var it1 = mem.tokenize(u8, p1, &[_]u8{sep1});444 var it1 = mem.tokenizeScalar(u8, p1, sep1);
445 var it2 = mem.tokenize(u8, p2, &[_]u8{sep2});445 var it2 = mem.tokenizeScalar(u8, p2, sep2);
446446
447 // TODO ASCII is wrong, we actually need full unicode support to compare paths.447 // TODO ASCII is wrong, we actually need full unicode support to compare paths.
448 return ascii.eqlIgnoreCase(it1.next().?, it2.next().?) and ascii.eqlIgnoreCase(it1.next().?, it2.next().?);448 return ascii.eqlIgnoreCase(it1.next().?, it2.next().?) and ascii.eqlIgnoreCase(it1.next().?, it2.next().?);
...@@ -535,7 +535,7 @@ pub fn resolveWindows(allocator: Allocator, paths: []const []const u8) ![]u8 {...@@ -535,7 +535,7 @@ pub fn resolveWindows(allocator: Allocator, paths: []const []const u8) ![]u8 {
535 break :l disk_designator.len;535 break :l disk_designator.len;
536 },536 },
537 .NetworkShare => {537 .NetworkShare => {
538 var it = mem.tokenize(u8, paths[first_index], "/\\");538 var it = mem.tokenizeAny(u8, paths[first_index], "/\\");
539 const server_name = it.next().?;539 const server_name = it.next().?;
540 const other_name = it.next().?;540 const other_name = it.next().?;
541541
...@@ -570,7 +570,7 @@ pub fn resolveWindows(allocator: Allocator, paths: []const []const u8) ![]u8 {...@@ -570,7 +570,7 @@ pub fn resolveWindows(allocator: Allocator, paths: []const []const u8) ![]u8 {
570 if (!correct_disk_designator) {570 if (!correct_disk_designator) {
571 continue;571 continue;
572 }572 }
573 var it = mem.tokenize(u8, p[parsed.disk_designator.len..], "/\\");573 var it = mem.tokenizeAny(u8, p[parsed.disk_designator.len..], "/\\");
574 while (it.next()) |component| {574 while (it.next()) |component| {
575 if (mem.eql(u8, component, ".")) {575 if (mem.eql(u8, component, ".")) {
576 continue;576 continue;
...@@ -657,7 +657,7 @@ pub fn resolvePosix(allocator: Allocator, paths: []const []const u8) Allocator.E...@@ -657,7 +657,7 @@ pub fn resolvePosix(allocator: Allocator, paths: []const []const u8) Allocator.E
657 negative_count = 0;657 negative_count = 0;
658 result.clearRetainingCapacity();658 result.clearRetainingCapacity();
659 }659 }
660 var it = mem.tokenize(u8, p, "/");660 var it = mem.tokenizeScalar(u8, p, '/');
661 while (it.next()) |component| {661 while (it.next()) |component| {
662 if (mem.eql(u8, component, ".")) {662 if (mem.eql(u8, component, ".")) {
663 continue;663 continue;
...@@ -1078,8 +1078,8 @@ pub fn relativeWindows(allocator: Allocator, from: []const u8, to: []const u8) !...@@ -1078,8 +1078,8 @@ pub fn relativeWindows(allocator: Allocator, from: []const u8, to: []const u8) !
1078 return resolved_to;1078 return resolved_to;
1079 }1079 }
10801080
1081 var from_it = mem.tokenize(u8, resolved_from, "/\\");1081 var from_it = mem.tokenizeAny(u8, resolved_from, "/\\");
1082 var to_it = mem.tokenize(u8, resolved_to, "/\\");1082 var to_it = mem.tokenizeAny(u8, resolved_to, "/\\");
1083 while (true) {1083 while (true) {
1084 const from_component = from_it.next() orelse return allocator.dupe(u8, to_it.rest());1084 const from_component = from_it.next() orelse return allocator.dupe(u8, to_it.rest());
1085 const to_rest = to_it.rest();1085 const to_rest = to_it.rest();
...@@ -1102,7 +1102,7 @@ pub fn relativeWindows(allocator: Allocator, from: []const u8, to: []const u8) !...@@ -1102,7 +1102,7 @@ pub fn relativeWindows(allocator: Allocator, from: []const u8, to: []const u8) !
1102 result_index += 3;1102 result_index += 3;
1103 }1103 }
11041104
1105 var rest_it = mem.tokenize(u8, to_rest, "/\\");1105 var rest_it = mem.tokenizeAny(u8, to_rest, "/\\");
1106 while (rest_it.next()) |to_component| {1106 while (rest_it.next()) |to_component| {
1107 result[result_index] = '\\';1107 result[result_index] = '\\';
1108 result_index += 1;1108 result_index += 1;
...@@ -1124,8 +1124,8 @@ pub fn relativePosix(allocator: Allocator, from: []const u8, to: []const u8) ![]...@@ -1124,8 +1124,8 @@ pub fn relativePosix(allocator: Allocator, from: []const u8, to: []const u8) ![]
1124 const resolved_to = try resolvePosix(allocator, &[_][]const u8{ cwd, to });1124 const resolved_to = try resolvePosix(allocator, &[_][]const u8{ cwd, to });
1125 defer allocator.free(resolved_to);1125 defer allocator.free(resolved_to);
11261126
1127 var from_it = mem.tokenize(u8, resolved_from, "/");1127 var from_it = mem.tokenizeScalar(u8, resolved_from, '/');
1128 var to_it = mem.tokenize(u8, resolved_to, "/");1128 var to_it = mem.tokenizeScalar(u8, resolved_to, '/');
1129 while (true) {1129 while (true) {
1130 const from_component = from_it.next() orelse return allocator.dupe(u8, to_it.rest());1130 const from_component = from_it.next() orelse return allocator.dupe(u8, to_it.rest());
1131 const to_rest = to_it.rest();1131 const to_rest = to_it.rest();
lib/std/http/Client.zig+3-3
...@@ -331,7 +331,7 @@ pub const Response = struct {...@@ -331,7 +331,7 @@ pub const Response = struct {
331 };331 };
332332
333 pub fn parse(res: *Response, bytes: []const u8, trailing: bool) ParseError!void {333 pub fn parse(res: *Response, bytes: []const u8, trailing: bool) ParseError!void {
334 var it = mem.tokenize(u8, bytes[0 .. bytes.len - 4], "\r\n");334 var it = mem.tokenizeAny(u8, bytes[0 .. bytes.len - 4], "\r\n");
335335
336 const first_line = it.next() orelse return error.HttpHeadersInvalid;336 const first_line = it.next() orelse return error.HttpHeadersInvalid;
337 if (first_line.len < 12)337 if (first_line.len < 12)
...@@ -357,7 +357,7 @@ pub const Response = struct {...@@ -357,7 +357,7 @@ pub const Response = struct {
357 else => {},357 else => {},
358 }358 }
359359
360 var line_it = mem.tokenize(u8, line, ": ");360 var line_it = mem.tokenizeAny(u8, line, ": ");
361 const header_name = line_it.next() orelse return error.HttpHeadersInvalid;361 const header_name = line_it.next() orelse return error.HttpHeadersInvalid;
362 const header_value = line_it.rest();362 const header_value = line_it.rest();
363363
...@@ -371,7 +371,7 @@ pub const Response = struct {...@@ -371,7 +371,7 @@ pub const Response = struct {
371 } else if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) {371 } else if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) {
372 // Transfer-Encoding: second, first372 // Transfer-Encoding: second, first
373 // Transfer-Encoding: deflate, chunked373 // Transfer-Encoding: deflate, chunked
374 var iter = mem.splitBackwards(u8, header_value, ",");374 var iter = mem.splitBackwardsScalar(u8, header_value, ',');
375375
376 if (iter.next()) |first| {376 if (iter.next()) |first| {
377 const trimmed = mem.trim(u8, first, " ");377 const trimmed = mem.trim(u8, first, " ");
lib/std/http/Server.zig+3-3
...@@ -178,7 +178,7 @@ pub const Request = struct {...@@ -178,7 +178,7 @@ pub const Request = struct {
178 };178 };
179179
180 pub fn parse(req: *Request, bytes: []const u8) ParseError!void {180 pub fn parse(req: *Request, bytes: []const u8) ParseError!void {
181 var it = mem.tokenize(u8, bytes[0 .. bytes.len - 4], "\r\n");181 var it = mem.tokenizeAny(u8, bytes[0 .. bytes.len - 4], "\r\n");
182182
183 const first_line = it.next() orelse return error.HttpHeadersInvalid;183 const first_line = it.next() orelse return error.HttpHeadersInvalid;
184 if (first_line.len < 10)184 if (first_line.len < 10)
...@@ -212,7 +212,7 @@ pub const Request = struct {...@@ -212,7 +212,7 @@ pub const Request = struct {
212 else => {},212 else => {},
213 }213 }
214214
215 var line_it = mem.tokenize(u8, line, ": ");215 var line_it = mem.tokenizeAny(u8, line, ": ");
216 const header_name = line_it.next() orelse return error.HttpHeadersInvalid;216 const header_name = line_it.next() orelse return error.HttpHeadersInvalid;
217 const header_value = line_it.rest();217 const header_value = line_it.rest();
218218
...@@ -224,7 +224,7 @@ pub const Request = struct {...@@ -224,7 +224,7 @@ pub const Request = struct {
224 } else if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) {224 } else if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) {
225 // Transfer-Encoding: second, first225 // Transfer-Encoding: second, first
226 // Transfer-Encoding: deflate, chunked226 // Transfer-Encoding: deflate, chunked
227 var iter = mem.splitBackwards(u8, header_value, ",");227 var iter = mem.splitBackwardsScalar(u8, header_value, ',');
228228
229 if (iter.next()) |first| {229 if (iter.next()) |first| {
230 const trimmed = mem.trim(u8, first, " ");230 const trimmed = mem.trim(u8, first, " ");
lib/std/mem.zig+447-98
...@@ -1958,72 +1958,117 @@ test "byteSwapAllFields" {...@@ -1958,72 +1958,117 @@ test "byteSwapAllFields" {
1958 }, k);1958 }, k);
1959}1959}
19601960
1961/// Deprecated: use `tokenizeAny`, `tokenizeSequence`, or `tokenizeScalar`
1962pub const tokenize = tokenizeAny;
1963
1961/// Returns an iterator that iterates over the slices of `buffer` that are not1964/// Returns an iterator that iterates over the slices of `buffer` that are not
1962/// any of the bytes in `delimiter_bytes`.1965/// any of the items in `delimiters`.
1963///1966///
1964/// `tokenize(u8, " abc def ghi ", " ")` will return slices1967/// `tokenizeAny(u8, " abc|def || ghi ", " |")` will return slices
1965/// for "abc", "def", "ghi", null, in that order.1968/// for "abc", "def", "ghi", null, in that order.
1966///1969///
1967/// If `buffer` is empty, the iterator will return null.1970/// If `buffer` is empty, the iterator will return null.
1968/// If `delimiter_bytes` does not exist in buffer,1971/// If none of `delimiters` exist in buffer,
1969/// the iterator will return `buffer`, null, in that order.1972/// the iterator will return `buffer`, null, in that order.
1970///1973///
1971/// See also: `split` and `splitBackwards`.1974/// See also: `tokenizeSequence`, `tokenizeScalar`,
1972pub fn tokenize(comptime T: type, buffer: []const T, delimiter_bytes: []const T) TokenIterator(T) {1975/// `splitSequence`,`splitAny`, `splitScalar`,
1976/// `splitBackwardsSequence`, `splitBackwardsAny`, and `splitBackwardsScalar`
1977pub fn tokenizeAny(comptime T: type, buffer: []const T, delimiters: []const T) TokenIterator(T, .any) {
1973 return .{1978 return .{
1974 .index = 0,1979 .index = 0,
1975 .buffer = buffer,1980 .buffer = buffer,
1976 .delimiter_bytes = delimiter_bytes,1981 .delimiter = delimiters,
1977 };1982 };
1978}1983}
19791984
1980test "tokenize" {1985/// Returns an iterator that iterates over the slices of `buffer` that are not
1981 var it = tokenize(u8, " abc def ghi ", " ");1986/// the sequence in `delimiter`.
1987///
1988/// `tokenizeSequence(u8, "<>abc><def<><>ghi", "<>")` will return slices
1989/// for "abc><def", "ghi", null, in that order.
1990///
1991/// If `buffer` is empty, the iterator will return null.
1992/// If `delimiter` does not exist in buffer,
1993/// the iterator will return `buffer`, null, in that order.
1994/// The delimiter length must not be zero.
1995///
1996/// See also: `tokenizeAny`, `tokenizeScalar`,
1997/// `splitSequence`,`splitAny`, and `splitScalar`
1998/// `splitBackwardsSequence`, `splitBackwardsAny`, and `splitBackwardsScalar`
1999pub fn tokenizeSequence(comptime T: type, buffer: []const T, delimiter: []const T) TokenIterator(T, .sequence) {
2000 assert(delimiter.len != 0);
2001 return .{
2002 .index = 0,
2003 .buffer = buffer,
2004 .delimiter = delimiter,
2005 };
2006}
2007
2008/// Returns an iterator that iterates over the slices of `buffer` that are not
2009/// `delimiter`.
2010///
2011/// `tokenizeScalar(u8, " abc def ghi ", ' ')` will return slices
2012/// for "abc", "def", "ghi", null, in that order.
2013///
2014/// If `buffer` is empty, the iterator will return null.
2015/// If `delimiter` does not exist in buffer,
2016/// the iterator will return `buffer`, null, in that order.
2017///
2018/// See also: `tokenizeAny`, `tokenizeSequence`,
2019/// `splitSequence`,`splitAny`, and `splitScalar`
2020/// `splitBackwardsSequence`, `splitBackwardsAny`, and `splitBackwardsScalar`
2021pub fn tokenizeScalar(comptime T: type, buffer: []const T, delimiter: T) TokenIterator(T, .scalar) {
2022 return .{
2023 .index = 0,
2024 .buffer = buffer,
2025 .delimiter = delimiter,
2026 };
2027}
2028
2029test "tokenizeScalar" {
2030 var it = tokenizeScalar(u8, " abc def ghi ", ' ');
1982 try testing.expect(eql(u8, it.next().?, "abc"));2031 try testing.expect(eql(u8, it.next().?, "abc"));
1983 try testing.expect(eql(u8, it.peek().?, "def"));2032 try testing.expect(eql(u8, it.peek().?, "def"));
1984 try testing.expect(eql(u8, it.next().?, "def"));2033 try testing.expect(eql(u8, it.next().?, "def"));
1985 try testing.expect(eql(u8, it.next().?, "ghi"));2034 try testing.expect(eql(u8, it.next().?, "ghi"));
1986 try testing.expect(it.next() == null);2035 try testing.expect(it.next() == null);
19872036
1988 it = tokenize(u8, "..\\bob", "\\");2037 it = tokenizeScalar(u8, "..\\bob", '\\');
1989 try testing.expect(eql(u8, it.next().?, ".."));2038 try testing.expect(eql(u8, it.next().?, ".."));
1990 try testing.expect(eql(u8, "..", "..\\bob"[0..it.index]));2039 try testing.expect(eql(u8, "..", "..\\bob"[0..it.index]));
1991 try testing.expect(eql(u8, it.next().?, "bob"));2040 try testing.expect(eql(u8, it.next().?, "bob"));
1992 try testing.expect(it.next() == null);2041 try testing.expect(it.next() == null);
19932042
1994 it = tokenize(u8, "//a/b", "/");2043 it = tokenizeScalar(u8, "//a/b", '/');
1995 try testing.expect(eql(u8, it.next().?, "a"));2044 try testing.expect(eql(u8, it.next().?, "a"));
1996 try testing.expect(eql(u8, it.next().?, "b"));2045 try testing.expect(eql(u8, it.next().?, "b"));
1997 try testing.expect(eql(u8, "//a/b", "//a/b"[0..it.index]));2046 try testing.expect(eql(u8, "//a/b", "//a/b"[0..it.index]));
1998 try testing.expect(it.next() == null);2047 try testing.expect(it.next() == null);
19992048
2000 it = tokenize(u8, "|", "|");2049 it = tokenizeScalar(u8, "|", '|');
2001 try testing.expect(it.next() == null);2050 try testing.expect(it.next() == null);
2002 try testing.expect(it.peek() == null);2051 try testing.expect(it.peek() == null);
20032052
2004 it = tokenize(u8, "", "|");2053 it = tokenizeScalar(u8, "", '|');
2005 try testing.expect(it.next() == null);2054 try testing.expect(it.next() == null);
2006 try testing.expect(it.peek() == null);2055 try testing.expect(it.peek() == null);
20072056
2008 it = tokenize(u8, "hello", "");2057 it = tokenizeScalar(u8, "hello", ' ');
2009 try testing.expect(eql(u8, it.next().?, "hello"));2058 try testing.expect(eql(u8, it.next().?, "hello"));
2010 try testing.expect(it.next() == null);2059 try testing.expect(it.next() == null);
20112060
2012 it = tokenize(u8, "hello", " ");2061 var it16 = tokenizeScalar(
2013 try testing.expect(eql(u8, it.next().?, "hello"));
2014 try testing.expect(it.next() == null);
2015
2016 var it16 = tokenize(
2017 u16,2062 u16,
2018 std.unicode.utf8ToUtf16LeStringLiteral("hello"),2063 std.unicode.utf8ToUtf16LeStringLiteral("hello"),
2019 std.unicode.utf8ToUtf16LeStringLiteral(" "),2064 ' ',
2020 );2065 );
2021 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("hello")));2066 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("hello")));
2022 try testing.expect(it16.next() == null);2067 try testing.expect(it16.next() == null);
2023}2068}
20242069
2025test "tokenize (multibyte)" {2070test "tokenizeAny" {
2026 var it = tokenize(u8, "a|b,c/d e", " /,|");2071 var it = tokenizeAny(u8, "a|b,c/d e", " /,|");
2027 try testing.expect(eql(u8, it.next().?, "a"));2072 try testing.expect(eql(u8, it.next().?, "a"));
2028 try testing.expect(eql(u8, it.peek().?, "b"));2073 try testing.expect(eql(u8, it.peek().?, "b"));
2029 try testing.expect(eql(u8, it.next().?, "b"));2074 try testing.expect(eql(u8, it.next().?, "b"));
...@@ -2033,7 +2078,11 @@ test "tokenize (multibyte)" {...@@ -2033,7 +2078,11 @@ test "tokenize (multibyte)" {
2033 try testing.expect(it.next() == null);2078 try testing.expect(it.next() == null);
2034 try testing.expect(it.peek() == null);2079 try testing.expect(it.peek() == null);
20352080
2036 var it16 = tokenize(2081 it = tokenizeAny(u8, "hello", "");
2082 try testing.expect(eql(u8, it.next().?, "hello"));
2083 try testing.expect(it.next() == null);
2084
2085 var it16 = tokenizeAny(
2037 u16,2086 u16,
2038 std.unicode.utf8ToUtf16LeStringLiteral("a|b,c/d e"),2087 std.unicode.utf8ToUtf16LeStringLiteral("a|b,c/d e"),
2039 std.unicode.utf8ToUtf16LeStringLiteral(" /,|"),2088 std.unicode.utf8ToUtf16LeStringLiteral(" /,|"),
...@@ -2046,32 +2095,87 @@ test "tokenize (multibyte)" {...@@ -2046,32 +2095,87 @@ test "tokenize (multibyte)" {
2046 try testing.expect(it16.next() == null);2095 try testing.expect(it16.next() == null);
2047}2096}
20482097
2098test "tokenizeSequence" {
2099 var it = tokenizeSequence(u8, "a<>b<><>c><>d><", "<>");
2100 try testing.expectEqualStrings("a", it.next().?);
2101 try testing.expectEqualStrings("b", it.peek().?);
2102 try testing.expectEqualStrings("b", it.next().?);
2103 try testing.expectEqualStrings("c>", it.next().?);
2104 try testing.expectEqualStrings("d><", it.next().?);
2105 try testing.expect(it.next() == null);
2106 try testing.expect(it.peek() == null);
2107
2108 var it16 = tokenizeSequence(
2109 u16,
2110 std.unicode.utf8ToUtf16LeStringLiteral("a<>b<><>c><>d><"),
2111 std.unicode.utf8ToUtf16LeStringLiteral("<>"),
2112 );
2113 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("a")));
2114 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("b")));
2115 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("c>")));
2116 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("d><")));
2117 try testing.expect(it16.next() == null);
2118}
2119
2049test "tokenize (reset)" {2120test "tokenize (reset)" {
2050 var it = tokenize(u8, " abc def ghi ", " ");2121 {
2051 try testing.expect(eql(u8, it.next().?, "abc"));2122 var it = tokenizeAny(u8, " abc def ghi ", " ");
2052 try testing.expect(eql(u8, it.next().?, "def"));2123 try testing.expect(eql(u8, it.next().?, "abc"));
2053 try testing.expect(eql(u8, it.next().?, "ghi"));2124 try testing.expect(eql(u8, it.next().?, "def"));
2125 try testing.expect(eql(u8, it.next().?, "ghi"));
2126
2127 it.reset();
20542128
2055 it.reset();2129 try testing.expect(eql(u8, it.next().?, "abc"));
2130 try testing.expect(eql(u8, it.next().?, "def"));
2131 try testing.expect(eql(u8, it.next().?, "ghi"));
2132 try testing.expect(it.next() == null);
2133 }
2134 {
2135 var it = tokenizeSequence(u8, "<><>abc<>def<><>ghi<>", "<>");
2136 try testing.expect(eql(u8, it.next().?, "abc"));
2137 try testing.expect(eql(u8, it.next().?, "def"));
2138 try testing.expect(eql(u8, it.next().?, "ghi"));
20562139
2057 try testing.expect(eql(u8, it.next().?, "abc"));2140 it.reset();
2058 try testing.expect(eql(u8, it.next().?, "def"));2141
2059 try testing.expect(eql(u8, it.next().?, "ghi"));2142 try testing.expect(eql(u8, it.next().?, "abc"));
2060 try testing.expect(it.next() == null);2143 try testing.expect(eql(u8, it.next().?, "def"));
2144 try testing.expect(eql(u8, it.next().?, "ghi"));
2145 try testing.expect(it.next() == null);
2146 }
2147 {
2148 var it = tokenizeScalar(u8, " abc def ghi ", ' ');
2149 try testing.expect(eql(u8, it.next().?, "abc"));
2150 try testing.expect(eql(u8, it.next().?, "def"));
2151 try testing.expect(eql(u8, it.next().?, "ghi"));
2152
2153 it.reset();
2154
2155 try testing.expect(eql(u8, it.next().?, "abc"));
2156 try testing.expect(eql(u8, it.next().?, "def"));
2157 try testing.expect(eql(u8, it.next().?, "ghi"));
2158 try testing.expect(it.next() == null);
2159 }
2061}2160}
20622161
2162/// Deprecated: use `splitSequence`, `splitAny`, or `splitScalar`
2163pub const split = splitSequence;
2164
2063/// Returns an iterator that iterates over the slices of `buffer` that2165/// Returns an iterator that iterates over the slices of `buffer` that
2064/// are separated by bytes in `delimiter`.2166/// are separated by the byte sequence in `delimiter`.
2065///2167///
2066/// `split(u8, "abc|def||ghi", "|")` will return slices2168/// `splitSequence(u8, "abc||def||||ghi", "||")` will return slices
2067/// for "abc", "def", "", "ghi", null, in that order.2169/// for "abc", "def", "", "ghi", null, in that order.
2068///2170///
2069/// If `delimiter` does not exist in buffer,2171/// If `delimiter` does not exist in buffer,
2070/// the iterator will return `buffer`, null, in that order.2172/// the iterator will return `buffer`, null, in that order.
2071/// The delimiter length must not be zero.2173/// The delimiter length must not be zero.
2072///2174///
2073/// See also: `tokenize` and `splitBackwards`.2175/// See also: `splitAny`, `splitScalar`, `splitBackwardsSequence`,
2074pub fn split(comptime T: type, buffer: []const T, delimiter: []const T) SplitIterator(T) {2176/// `splitBackwardsAny`,`splitBackwardsScalar`,
2177/// `tokenizeAny`, `tokenizeSequence`, and `tokenizeScalar`.
2178pub fn splitSequence(comptime T: type, buffer: []const T, delimiter: []const T) SplitIterator(T, .sequence) {
2075 assert(delimiter.len != 0);2179 assert(delimiter.len != 0);
2076 return .{2180 return .{
2077 .index = 0,2181 .index = 0,
...@@ -2080,8 +2184,48 @@ pub fn split(comptime T: type, buffer: []const T, delimiter: []const T) SplitIte...@@ -2080,8 +2184,48 @@ pub fn split(comptime T: type, buffer: []const T, delimiter: []const T) SplitIte
2080 };2184 };
2081}2185}
20822186
2083test "split" {2187/// Returns an iterator that iterates over the slices of `buffer` that
2084 var it = split(u8, "abc|def||ghi", "|");2188/// are separated by any item in `delimiters`.
2189///
2190/// `splitAny(u8, "abc,def||ghi", "|,")` will return slices
2191/// for "abc", "def", "", "ghi", null, in that order.
2192///
2193/// If none of `delimiters` exist in buffer,
2194/// the iterator will return `buffer`, null, in that order.
2195///
2196/// See also: `splitSequence`, `splitScalar`, `splitBackwardsSequence`,
2197/// `splitBackwardsAny`,`splitBackwardsScalar`,
2198/// `tokenizeAny`, `tokenizeSequence`, and `tokenizeScalar`.
2199pub fn splitAny(comptime T: type, buffer: []const T, delimiters: []const T) SplitIterator(T, .any) {
2200 return .{
2201 .index = 0,
2202 .buffer = buffer,
2203 .delimiter = delimiters,
2204 };
2205}
2206
2207/// Returns an iterator that iterates over the slices of `buffer` that
2208/// are separated by `delimiter`.
2209///
2210/// `splitScalar(u8, "abc|def||ghi", '|')` will return slices
2211/// for "abc", "def", "", "ghi", null, in that order.
2212///
2213/// If `delimiter` does not exist in buffer,
2214/// the iterator will return `buffer`, null, in that order.
2215///
2216/// See also: `splitSequence`, `splitAny`, `splitBackwardsSequence`,
2217/// `splitBackwardsAny`,`splitBackwardsScalar`,
2218/// `tokenizeAny`, `tokenizeSequence`, and `tokenizeScalar`.
2219pub fn splitScalar(comptime T: type, buffer: []const T, delimiter: T) SplitIterator(T, .scalar) {
2220 return .{
2221 .index = 0,
2222 .buffer = buffer,
2223 .delimiter = delimiter,
2224 };
2225}
2226
2227test "splitScalar" {
2228 var it = splitScalar(u8, "abc|def||ghi", '|');
2085 try testing.expectEqualSlices(u8, it.rest(), "abc|def||ghi");2229 try testing.expectEqualSlices(u8, it.rest(), "abc|def||ghi");
2086 try testing.expectEqualSlices(u8, it.first(), "abc");2230 try testing.expectEqualSlices(u8, it.first(), "abc");
20872231
...@@ -2097,30 +2241,30 @@ test "split" {...@@ -2097,30 +2241,30 @@ test "split" {
2097 try testing.expectEqualSlices(u8, it.rest(), "");2241 try testing.expectEqualSlices(u8, it.rest(), "");
2098 try testing.expect(it.next() == null);2242 try testing.expect(it.next() == null);
20992243
2100 it = split(u8, "", "|");2244 it = splitScalar(u8, "", '|');
2101 try testing.expectEqualSlices(u8, it.first(), "");2245 try testing.expectEqualSlices(u8, it.first(), "");
2102 try testing.expect(it.next() == null);2246 try testing.expect(it.next() == null);
21032247
2104 it = split(u8, "|", "|");2248 it = splitScalar(u8, "|", '|');
2105 try testing.expectEqualSlices(u8, it.first(), "");2249 try testing.expectEqualSlices(u8, it.first(), "");
2106 try testing.expectEqualSlices(u8, it.next().?, "");2250 try testing.expectEqualSlices(u8, it.next().?, "");
2107 try testing.expect(it.next() == null);2251 try testing.expect(it.next() == null);
21082252
2109 it = split(u8, "hello", " ");2253 it = splitScalar(u8, "hello", ' ');
2110 try testing.expectEqualSlices(u8, it.first(), "hello");2254 try testing.expectEqualSlices(u8, it.first(), "hello");
2111 try testing.expect(it.next() == null);2255 try testing.expect(it.next() == null);
21122256
2113 var it16 = split(2257 var it16 = splitScalar(
2114 u16,2258 u16,
2115 std.unicode.utf8ToUtf16LeStringLiteral("hello"),2259 std.unicode.utf8ToUtf16LeStringLiteral("hello"),
2116 std.unicode.utf8ToUtf16LeStringLiteral(" "),2260 ' ',
2117 );2261 );
2118 try testing.expectEqualSlices(u16, it16.first(), std.unicode.utf8ToUtf16LeStringLiteral("hello"));2262 try testing.expectEqualSlices(u16, it16.first(), std.unicode.utf8ToUtf16LeStringLiteral("hello"));
2119 try testing.expect(it16.next() == null);2263 try testing.expect(it16.next() == null);
2120}2264}
21212265
2122test "split (multibyte)" {2266test "splitSequence" {
2123 var it = split(u8, "a, b ,, c, d, e", ", ");2267 var it = splitSequence(u8, "a, b ,, c, d, e", ", ");
2124 try testing.expectEqualSlices(u8, it.first(), "a");2268 try testing.expectEqualSlices(u8, it.first(), "a");
2125 try testing.expectEqualSlices(u8, it.rest(), "b ,, c, d, e");2269 try testing.expectEqualSlices(u8, it.rest(), "b ,, c, d, e");
2126 try testing.expectEqualSlices(u8, it.next().?, "b ,");2270 try testing.expectEqualSlices(u8, it.next().?, "b ,");
...@@ -2129,7 +2273,7 @@ test "split (multibyte)" {...@@ -2129,7 +2273,7 @@ test "split (multibyte)" {
2129 try testing.expectEqualSlices(u8, it.next().?, "e");2273 try testing.expectEqualSlices(u8, it.next().?, "e");
2130 try testing.expect(it.next() == null);2274 try testing.expect(it.next() == null);
21312275
2132 var it16 = split(2276 var it16 = splitSequence(
2133 u16,2277 u16,
2134 std.unicode.utf8ToUtf16LeStringLiteral("a, b ,, c, d, e"),2278 std.unicode.utf8ToUtf16LeStringLiteral("a, b ,, c, d, e"),
2135 std.unicode.utf8ToUtf16LeStringLiteral(", "),2279 std.unicode.utf8ToUtf16LeStringLiteral(", "),
...@@ -2142,42 +2286,144 @@ test "split (multibyte)" {...@@ -2142,42 +2286,144 @@ test "split (multibyte)" {
2142 try testing.expect(it16.next() == null);2286 try testing.expect(it16.next() == null);
2143}2287}
21442288
2289test "splitAny" {
2290 var it = splitAny(u8, "a,b, c d e", ", ");
2291 try testing.expectEqualSlices(u8, it.first(), "a");
2292 try testing.expectEqualSlices(u8, it.rest(), "b, c d e");
2293 try testing.expectEqualSlices(u8, it.next().?, "b");
2294 try testing.expectEqualSlices(u8, it.next().?, "");
2295 try testing.expectEqualSlices(u8, it.next().?, "c");
2296 try testing.expectEqualSlices(u8, it.next().?, "d");
2297 try testing.expectEqualSlices(u8, it.next().?, "e");
2298 try testing.expect(it.next() == null);
2299
2300 it = splitAny(u8, "hello", "");
2301 try testing.expect(eql(u8, it.next().?, "hello"));
2302 try testing.expect(it.next() == null);
2303
2304 var it16 = splitAny(
2305 u16,
2306 std.unicode.utf8ToUtf16LeStringLiteral("a,b, c d e"),
2307 std.unicode.utf8ToUtf16LeStringLiteral(", "),
2308 );
2309 try testing.expectEqualSlices(u16, it16.first(), std.unicode.utf8ToUtf16LeStringLiteral("a"));
2310 try testing.expectEqualSlices(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("b"));
2311 try testing.expectEqualSlices(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral(""));
2312 try testing.expectEqualSlices(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("c"));
2313 try testing.expectEqualSlices(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("d"));
2314 try testing.expectEqualSlices(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("e"));
2315 try testing.expect(it16.next() == null);
2316}
2317
2145test "split (reset)" {2318test "split (reset)" {
2146 var it = split(u8, "abc def ghi", " ");2319 {
2147 try testing.expect(eql(u8, it.first(), "abc"));2320 var it = splitSequence(u8, "abc def ghi", " ");
2148 try testing.expect(eql(u8, it.next().?, "def"));2321 try testing.expect(eql(u8, it.first(), "abc"));
2149 try testing.expect(eql(u8, it.next().?, "ghi"));2322 try testing.expect(eql(u8, it.next().?, "def"));
2323 try testing.expect(eql(u8, it.next().?, "ghi"));
21502324
2151 it.reset();2325 it.reset();
21522326
2153 try testing.expect(eql(u8, it.first(), "abc"));2327 try testing.expect(eql(u8, it.first(), "abc"));
2154 try testing.expect(eql(u8, it.next().?, "def"));2328 try testing.expect(eql(u8, it.next().?, "def"));
2155 try testing.expect(eql(u8, it.next().?, "ghi"));2329 try testing.expect(eql(u8, it.next().?, "ghi"));
2156 try testing.expect(it.next() == null);2330 try testing.expect(it.next() == null);
2331 }
2332 {
2333 var it = splitAny(u8, "abc def,ghi", " ,");
2334 try testing.expect(eql(u8, it.first(), "abc"));
2335 try testing.expect(eql(u8, it.next().?, "def"));
2336 try testing.expect(eql(u8, it.next().?, "ghi"));
2337
2338 it.reset();
2339
2340 try testing.expect(eql(u8, it.first(), "abc"));
2341 try testing.expect(eql(u8, it.next().?, "def"));
2342 try testing.expect(eql(u8, it.next().?, "ghi"));
2343 try testing.expect(it.next() == null);
2344 }
2345 {
2346 var it = splitScalar(u8, "abc def ghi", ' ');
2347 try testing.expect(eql(u8, it.first(), "abc"));
2348 try testing.expect(eql(u8, it.next().?, "def"));
2349 try testing.expect(eql(u8, it.next().?, "ghi"));
2350
2351 it.reset();
2352
2353 try testing.expect(eql(u8, it.first(), "abc"));
2354 try testing.expect(eql(u8, it.next().?, "def"));
2355 try testing.expect(eql(u8, it.next().?, "ghi"));
2356 try testing.expect(it.next() == null);
2357 }
2157}2358}
21582359
2159/// Returns an iterator that iterates backwards over the slices of `buffer`2360/// Deprecated: use `splitBackwardsSequence`, `splitBackwardsAny`, or `splitBackwardsScalar`
2160/// that are separated by bytes in `delimiter`.2361pub const splitBackwards = splitBackwardsSequence;
2362
2363/// Returns an iterator that iterates backwards over the slices of `buffer` that
2364/// are separated by the sequence in `delimiter`.
2161///2365///
2162/// `splitBackwards(u8, "abc|def||ghi", "|")` will return slices2366/// `splitBackwardsSequence(u8, "abc||def||||ghi", "||")` will return slices
2163/// for "ghi", "", "def", "abc", null, in that order.2367/// for "ghi", "", "def", "abc", null, in that order.
2164///2368///
2165/// If `delimiter` does not exist in buffer,2369/// If `delimiter` does not exist in buffer,
2166/// the iterator will return `buffer`, null, in that order.2370/// the iterator will return `buffer`, null, in that order.
2167/// The delimiter length must not be zero.2371/// The delimiter length must not be zero.
2168///2372///
2169/// See also: `tokenize` and `split`.2373/// See also: `splitBackwardsAny`, `splitBackwardsScalar`,
2170pub fn splitBackwards(comptime T: type, buffer: []const T, delimiter: []const T) SplitBackwardsIterator(T) {2374/// `splitSequence`, `splitAny`,`splitScalar`,
2375/// `tokenizeAny`, `tokenizeSequence`, and `tokenizeScalar`.
2376pub fn splitBackwardsSequence(comptime T: type, buffer: []const T, delimiter: []const T) SplitBackwardsIterator(T, .sequence) {
2171 assert(delimiter.len != 0);2377 assert(delimiter.len != 0);
2172 return SplitBackwardsIterator(T){2378 return .{
2379 .index = buffer.len,
2380 .buffer = buffer,
2381 .delimiter = delimiter,
2382 };
2383}
2384
2385/// Returns an iterator that iterates backwards over the slices of `buffer` that
2386/// are separated by any item in `delimiters`.
2387///
2388/// `splitBackwardsAny(u8, "abc,def||ghi", "|,")` will return slices
2389/// for "ghi", "", "def", "abc", null, in that order.
2390///
2391/// If none of `delimiters` exist in buffer,
2392/// the iterator will return `buffer`, null, in that order.
2393///
2394/// See also: `splitBackwardsSequence`, `splitBackwardsScalar`,
2395/// `splitSequence`, `splitAny`,`splitScalar`,
2396/// `tokenizeAny`, `tokenizeSequence`, and `tokenizeScalar`.
2397pub fn splitBackwardsAny(comptime T: type, buffer: []const T, delimiters: []const T) SplitBackwardsIterator(T, .any) {
2398 return .{
2399 .index = buffer.len,
2400 .buffer = buffer,
2401 .delimiter = delimiters,
2402 };
2403}
2404
2405/// Returns an iterator that iterates backwards over the slices of `buffer` that
2406/// are separated by `delimiter`.
2407///
2408/// `splitBackwardsScalar(u8, "abc|def||ghi", '|')` will return slices
2409/// for "ghi", "", "def", "abc", null, in that order.
2410///
2411/// If `delimiter` does not exist in buffer,
2412/// the iterator will return `buffer`, null, in that order.
2413///
2414/// See also: `splitBackwardsSequence`, `splitBackwardsAny`,
2415/// `splitSequence`, `splitAny`,`splitScalar`,
2416/// `tokenizeAny`, `tokenizeSequence`, and `tokenizeScalar`.
2417pub fn splitBackwardsScalar(comptime T: type, buffer: []const T, delimiter: T) SplitBackwardsIterator(T, .scalar) {
2418 return .{
2173 .index = buffer.len,2419 .index = buffer.len,
2174 .buffer = buffer,2420 .buffer = buffer,
2175 .delimiter = delimiter,2421 .delimiter = delimiter,
2176 };2422 };
2177}2423}
21782424
2179test "splitBackwards" {2425test "splitBackwardsScalar" {
2180 var it = splitBackwards(u8, "abc|def||ghi", "|");2426 var it = splitBackwardsScalar(u8, "abc|def||ghi", '|');
2181 try testing.expectEqualSlices(u8, it.rest(), "abc|def||ghi");2427 try testing.expectEqualSlices(u8, it.rest(), "abc|def||ghi");
2182 try testing.expectEqualSlices(u8, it.first(), "ghi");2428 try testing.expectEqualSlices(u8, it.first(), "ghi");
21832429
...@@ -2193,30 +2439,30 @@ test "splitBackwards" {...@@ -2193,30 +2439,30 @@ test "splitBackwards" {
2193 try testing.expectEqualSlices(u8, it.rest(), "");2439 try testing.expectEqualSlices(u8, it.rest(), "");
2194 try testing.expect(it.next() == null);2440 try testing.expect(it.next() == null);
21952441
2196 it = splitBackwards(u8, "", "|");2442 it = splitBackwardsScalar(u8, "", '|');
2197 try testing.expectEqualSlices(u8, it.first(), "");2443 try testing.expectEqualSlices(u8, it.first(), "");
2198 try testing.expect(it.next() == null);2444 try testing.expect(it.next() == null);
21992445
2200 it = splitBackwards(u8, "|", "|");2446 it = splitBackwardsScalar(u8, "|", '|');
2201 try testing.expectEqualSlices(u8, it.first(), "");2447 try testing.expectEqualSlices(u8, it.first(), "");
2202 try testing.expectEqualSlices(u8, it.next().?, "");2448 try testing.expectEqualSlices(u8, it.next().?, "");
2203 try testing.expect(it.next() == null);2449 try testing.expect(it.next() == null);
22042450
2205 it = splitBackwards(u8, "hello", " ");2451 it = splitBackwardsScalar(u8, "hello", ' ');
2206 try testing.expectEqualSlices(u8, it.first(), "hello");2452 try testing.expectEqualSlices(u8, it.first(), "hello");
2207 try testing.expect(it.next() == null);2453 try testing.expect(it.next() == null);
22082454
2209 var it16 = splitBackwards(2455 var it16 = splitBackwardsScalar(
2210 u16,2456 u16,
2211 std.unicode.utf8ToUtf16LeStringLiteral("hello"),2457 std.unicode.utf8ToUtf16LeStringLiteral("hello"),
2212 std.unicode.utf8ToUtf16LeStringLiteral(" "),2458 ' ',
2213 );2459 );
2214 try testing.expectEqualSlices(u16, it16.first(), std.unicode.utf8ToUtf16LeStringLiteral("hello"));2460 try testing.expectEqualSlices(u16, it16.first(), std.unicode.utf8ToUtf16LeStringLiteral("hello"));
2215 try testing.expect(it16.next() == null);2461 try testing.expect(it16.next() == null);
2216}2462}
22172463
2218test "splitBackwards (multibyte)" {2464test "splitBackwardsSequence" {
2219 var it = splitBackwards(u8, "a, b ,, c, d, e", ", ");2465 var it = splitBackwardsSequence(u8, "a, b ,, c, d, e", ", ");
2220 try testing.expectEqualSlices(u8, it.rest(), "a, b ,, c, d, e");2466 try testing.expectEqualSlices(u8, it.rest(), "a, b ,, c, d, e");
2221 try testing.expectEqualSlices(u8, it.first(), "e");2467 try testing.expectEqualSlices(u8, it.first(), "e");
22222468
...@@ -2235,7 +2481,7 @@ test "splitBackwards (multibyte)" {...@@ -2235,7 +2481,7 @@ test "splitBackwards (multibyte)" {
2235 try testing.expectEqualSlices(u8, it.rest(), "");2481 try testing.expectEqualSlices(u8, it.rest(), "");
2236 try testing.expect(it.next() == null);2482 try testing.expect(it.next() == null);
22372483
2238 var it16 = splitBackwards(2484 var it16 = splitBackwardsSequence(
2239 u16,2485 u16,
2240 std.unicode.utf8ToUtf16LeStringLiteral("a, b ,, c, d, e"),2486 std.unicode.utf8ToUtf16LeStringLiteral("a, b ,, c, d, e"),
2241 std.unicode.utf8ToUtf16LeStringLiteral(", "),2487 std.unicode.utf8ToUtf16LeStringLiteral(", "),
...@@ -2248,18 +2494,83 @@ test "splitBackwards (multibyte)" {...@@ -2248,18 +2494,83 @@ test "splitBackwards (multibyte)" {
2248 try testing.expect(it16.next() == null);2494 try testing.expect(it16.next() == null);
2249}2495}
22502496
2251test "splitBackwards (reset)" {2497test "splitBackwardsAny" {
2252 var it = splitBackwards(u8, "abc def ghi", " ");2498 var it = splitBackwardsAny(u8, "a,b, c d e", ", ");
2253 try testing.expect(eql(u8, it.first(), "ghi"));2499 try testing.expectEqualSlices(u8, it.rest(), "a,b, c d e");
2254 try testing.expect(eql(u8, it.next().?, "def"));2500 try testing.expectEqualSlices(u8, it.first(), "e");
2255 try testing.expect(eql(u8, it.next().?, "abc"));
22562501
2257 it.reset();2502 try testing.expectEqualSlices(u8, it.rest(), "a,b, c d");
2503 try testing.expectEqualSlices(u8, it.next().?, "d");
22582504
2259 try testing.expect(eql(u8, it.first(), "ghi"));2505 try testing.expectEqualSlices(u8, it.rest(), "a,b, c");
2260 try testing.expect(eql(u8, it.next().?, "def"));2506 try testing.expectEqualSlices(u8, it.next().?, "c");
2261 try testing.expect(eql(u8, it.next().?, "abc"));2507
2508 try testing.expectEqualSlices(u8, it.rest(), "a,b,");
2509 try testing.expectEqualSlices(u8, it.next().?, "");
2510
2511 try testing.expectEqualSlices(u8, it.rest(), "a,b");
2512 try testing.expectEqualSlices(u8, it.next().?, "b");
2513
2514 try testing.expectEqualSlices(u8, it.rest(), "a");
2515 try testing.expectEqualSlices(u8, it.next().?, "a");
2516
2517 try testing.expectEqualSlices(u8, it.rest(), "");
2262 try testing.expect(it.next() == null);2518 try testing.expect(it.next() == null);
2519
2520 var it16 = splitBackwardsAny(
2521 u16,
2522 std.unicode.utf8ToUtf16LeStringLiteral("a,b, c d e"),
2523 std.unicode.utf8ToUtf16LeStringLiteral(", "),
2524 );
2525 try testing.expectEqualSlices(u16, it16.first(), std.unicode.utf8ToUtf16LeStringLiteral("e"));
2526 try testing.expectEqualSlices(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("d"));
2527 try testing.expectEqualSlices(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("c"));
2528 try testing.expectEqualSlices(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral(""));
2529 try testing.expectEqualSlices(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("b"));
2530 try testing.expectEqualSlices(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("a"));
2531 try testing.expect(it16.next() == null);
2532}
2533
2534test "splitBackwards (reset)" {
2535 {
2536 var it = splitBackwardsSequence(u8, "abc def ghi", " ");
2537 try testing.expect(eql(u8, it.first(), "ghi"));
2538 try testing.expect(eql(u8, it.next().?, "def"));
2539 try testing.expect(eql(u8, it.next().?, "abc"));
2540
2541 it.reset();
2542
2543 try testing.expect(eql(u8, it.first(), "ghi"));
2544 try testing.expect(eql(u8, it.next().?, "def"));
2545 try testing.expect(eql(u8, it.next().?, "abc"));
2546 try testing.expect(it.next() == null);
2547 }
2548 {
2549 var it = splitBackwardsAny(u8, "abc def,ghi", " ,");
2550 try testing.expect(eql(u8, it.first(), "ghi"));
2551 try testing.expect(eql(u8, it.next().?, "def"));
2552 try testing.expect(eql(u8, it.next().?, "abc"));
2553
2554 it.reset();
2555
2556 try testing.expect(eql(u8, it.first(), "ghi"));
2557 try testing.expect(eql(u8, it.next().?, "def"));
2558 try testing.expect(eql(u8, it.next().?, "abc"));
2559 try testing.expect(it.next() == null);
2560 }
2561 {
2562 var it = splitBackwardsScalar(u8, "abc def ghi", ' ');
2563 try testing.expect(eql(u8, it.first(), "ghi"));
2564 try testing.expect(eql(u8, it.next().?, "def"));
2565 try testing.expect(eql(u8, it.next().?, "abc"));
2566
2567 it.reset();
2568
2569 try testing.expect(eql(u8, it.first(), "ghi"));
2570 try testing.expect(eql(u8, it.next().?, "def"));
2571 try testing.expect(eql(u8, it.next().?, "abc"));
2572 try testing.expect(it.next() == null);
2573 }
2263}2574}
22642575
2265/// Returns an iterator with a sliding window of slices for `buffer`.2576/// Returns an iterator with a sliding window of slices for `buffer`.
...@@ -2430,10 +2741,15 @@ test "endsWith" {...@@ -2430,10 +2741,15 @@ test "endsWith" {
2430 try testing.expect(!endsWith(u8, "Bob", "Bo"));2741 try testing.expect(!endsWith(u8, "Bob", "Bo"));
2431}2742}
24322743
2433pub fn TokenIterator(comptime T: type) type {2744pub const DelimiterType = enum { sequence, any, scalar };
2745
2746pub fn TokenIterator(comptime T: type, comptime delimiter_type: DelimiterType) type {
2434 return struct {2747 return struct {
2435 buffer: []const T,2748 buffer: []const T,
2436 delimiter_bytes: []const T,2749 delimiter: switch (delimiter_type) {
2750 .sequence, .any => []const T,
2751 .scalar => T,
2752 },
2437 index: usize,2753 index: usize,
24382754
2439 const Self = @This();2755 const Self = @This();
...@@ -2450,7 +2766,10 @@ pub fn TokenIterator(comptime T: type) type {...@@ -2450,7 +2766,10 @@ pub fn TokenIterator(comptime T: type) type {
2450 /// complete. Does not advance to the next token.2766 /// complete. Does not advance to the next token.
2451 pub fn peek(self: *Self) ?[]const T {2767 pub fn peek(self: *Self) ?[]const T {
2452 // move to beginning of token2768 // move to beginning of token
2453 while (self.index < self.buffer.len and self.isSplitByte(self.buffer[self.index])) : (self.index += 1) {}2769 while (self.index < self.buffer.len and self.isDelimiter(self.index)) : (self.index += switch (delimiter_type) {
2770 .sequence => self.delimiter.len,
2771 .any, .scalar => 1,
2772 }) {}
2454 const start = self.index;2773 const start = self.index;
2455 if (start == self.buffer.len) {2774 if (start == self.buffer.len) {
2456 return null;2775 return null;
...@@ -2458,7 +2777,7 @@ pub fn TokenIterator(comptime T: type) type {...@@ -2458,7 +2777,7 @@ pub fn TokenIterator(comptime T: type) type {
24582777
2459 // move to end of token2778 // move to end of token
2460 var end = start;2779 var end = start;
2461 while (end < self.buffer.len and !self.isSplitByte(self.buffer[end])) : (end += 1) {}2780 while (end < self.buffer.len and !self.isDelimiter(end)) : (end += 1) {}
24622781
2463 return self.buffer[start..end];2782 return self.buffer[start..end];
2464 }2783 }
...@@ -2467,7 +2786,10 @@ pub fn TokenIterator(comptime T: type) type {...@@ -2467,7 +2786,10 @@ pub fn TokenIterator(comptime T: type) type {
2467 pub fn rest(self: Self) []const T {2786 pub fn rest(self: Self) []const T {
2468 // move to beginning of token2787 // move to beginning of token
2469 var index: usize = self.index;2788 var index: usize = self.index;
2470 while (index < self.buffer.len and self.isSplitByte(self.buffer[index])) : (index += 1) {}2789 while (index < self.buffer.len and self.isDelimiter(index)) : (index += switch (delimiter_type) {
2790 .sequence => self.delimiter.len,
2791 .any, .scalar => 1,
2792 }) {}
2471 return self.buffer[index..];2793 return self.buffer[index..];
2472 }2794 }
24732795
...@@ -2476,22 +2798,32 @@ pub fn TokenIterator(comptime T: type) type {...@@ -2476,22 +2798,32 @@ pub fn TokenIterator(comptime T: type) type {
2476 self.index = 0;2798 self.index = 0;
2477 }2799 }
24782800
2479 fn isSplitByte(self: Self, byte: T) bool {2801 fn isDelimiter(self: Self, index: usize) bool {
2480 for (self.delimiter_bytes) |delimiter_byte| {2802 switch (delimiter_type) {
2481 if (byte == delimiter_byte) {2803 .sequence => return startsWith(T, self.buffer[index..], self.delimiter),
2482 return true;2804 .any => {
2483 }2805 const item = self.buffer[index];
2806 for (self.delimiter) |delimiter_item| {
2807 if (item == delimiter_item) {
2808 return true;
2809 }
2810 }
2811 return false;
2812 },
2813 .scalar => return self.buffer[index] == self.delimiter,
2484 }2814 }
2485 return false;
2486 }2815 }
2487 };2816 };
2488}2817}
24892818
2490pub fn SplitIterator(comptime T: type) type {2819pub fn SplitIterator(comptime T: type, comptime delimiter_type: DelimiterType) type {
2491 return struct {2820 return struct {
2492 buffer: []const T,2821 buffer: []const T,
2493 index: ?usize,2822 index: ?usize,
2494 delimiter: []const T,2823 delimiter: switch (delimiter_type) {
2824 .sequence, .any => []const T,
2825 .scalar => T,
2826 },
24952827
2496 const Self = @This();2828 const Self = @This();
24972829
...@@ -2505,8 +2837,15 @@ pub fn SplitIterator(comptime T: type) type {...@@ -2505,8 +2837,15 @@ pub fn SplitIterator(comptime T: type) type {
2505 /// Returns a slice of the next field, or null if splitting is complete.2837 /// Returns a slice of the next field, or null if splitting is complete.
2506 pub fn next(self: *Self) ?[]const T {2838 pub fn next(self: *Self) ?[]const T {
2507 const start = self.index orelse return null;2839 const start = self.index orelse return null;
2508 const end = if (indexOfPos(T, self.buffer, start, self.delimiter)) |delim_start| blk: {2840 const end = if (switch (delimiter_type) {
2509 self.index = delim_start + self.delimiter.len;2841 .sequence => indexOfPos(T, self.buffer, start, self.delimiter),
2842 .any => indexOfAnyPos(T, self.buffer, start, self.delimiter),
2843 .scalar => indexOfScalarPos(T, self.buffer, start, self.delimiter),
2844 }) |delim_start| blk: {
2845 self.index = delim_start + switch (delimiter_type) {
2846 .sequence => self.delimiter.len,
2847 .any, .scalar => 1,
2848 };
2510 break :blk delim_start;2849 break :blk delim_start;
2511 } else blk: {2850 } else blk: {
2512 self.index = null;2851 self.index = null;
...@@ -2529,11 +2868,14 @@ pub fn SplitIterator(comptime T: type) type {...@@ -2529,11 +2868,14 @@ pub fn SplitIterator(comptime T: type) type {
2529 };2868 };
2530}2869}
25312870
2532pub fn SplitBackwardsIterator(comptime T: type) type {2871pub fn SplitBackwardsIterator(comptime T: type, comptime delimiter_type: DelimiterType) type {
2533 return struct {2872 return struct {
2534 buffer: []const T,2873 buffer: []const T,
2535 index: ?usize,2874 index: ?usize,
2536 delimiter: []const T,2875 delimiter: switch (delimiter_type) {
2876 .sequence, .any => []const T,
2877 .scalar => T,
2878 },
25372879
2538 const Self = @This();2880 const Self = @This();
25392881
...@@ -2547,9 +2889,16 @@ pub fn SplitBackwardsIterator(comptime T: type) type {...@@ -2547,9 +2889,16 @@ pub fn SplitBackwardsIterator(comptime T: type) type {
2547 /// Returns a slice of the next field, or null if splitting is complete.2889 /// Returns a slice of the next field, or null if splitting is complete.
2548 pub fn next(self: *Self) ?[]const T {2890 pub fn next(self: *Self) ?[]const T {
2549 const end = self.index orelse return null;2891 const end = self.index orelse return null;
2550 const start = if (lastIndexOf(T, self.buffer[0..end], self.delimiter)) |delim_start| blk: {2892 const start = if (switch (delimiter_type) {
2893 .sequence => lastIndexOf(T, self.buffer[0..end], self.delimiter),
2894 .any => lastIndexOfAny(T, self.buffer[0..end], self.delimiter),
2895 .scalar => lastIndexOfScalar(T, self.buffer[0..end], self.delimiter),
2896 }) |delim_start| blk: {
2551 self.index = delim_start;2897 self.index = delim_start;
2552 break :blk delim_start + self.delimiter.len;2898 break :blk delim_start + switch (delimiter_type) {
2899 .sequence => self.delimiter.len,
2900 .any, .scalar => 1,
2901 };
2553 } else blk: {2902 } else blk: {
2554 self.index = null;2903 self.index = null;
2555 break :blk 0;2904 break :blk 0;
lib/std/net.zig+6-6
...@@ -1263,10 +1263,10 @@ fn linuxLookupNameFromHosts(...@@ -1263,10 +1263,10 @@ fn linuxLookupNameFromHosts(
1263 },1263 },
1264 else => |e| return e,1264 else => |e| return e,
1265 }) |line| {1265 }) |line| {
1266 var split_it = mem.split(u8, line, "#");1266 var split_it = mem.splitScalar(u8, line, '#');
1267 const no_comment_line = split_it.first();1267 const no_comment_line = split_it.first();
12681268
1269 var line_it = mem.tokenize(u8, no_comment_line, " \t");1269 var line_it = mem.tokenizeAny(u8, no_comment_line, " \t");
1270 const ip_text = line_it.next() orelse continue;1270 const ip_text = line_it.next() orelse continue;
1271 var first_name_text: ?[]const u8 = null;1271 var first_name_text: ?[]const u8 = null;
1272 while (line_it.next()) |name_text| {1272 while (line_it.next()) |name_text| {
...@@ -1346,7 +1346,7 @@ fn linuxLookupNameFromDnsSearch(...@@ -1346,7 +1346,7 @@ fn linuxLookupNameFromDnsSearch(
1346 @memcpy(canon.items, canon_name);1346 @memcpy(canon.items, canon_name);
1347 try canon.append('.');1347 try canon.append('.');
13481348
1349 var tok_it = mem.tokenize(u8, search, " \t");1349 var tok_it = mem.tokenizeAny(u8, search, " \t");
1350 while (tok_it.next()) |tok| {1350 while (tok_it.next()) |tok| {
1351 canon.shrinkRetainingCapacity(canon_name.len + 1);1351 canon.shrinkRetainingCapacity(canon_name.len + 1);
1352 try canon.appendSlice(tok);1352 try canon.appendSlice(tok);
...@@ -1465,15 +1465,15 @@ fn getResolvConf(allocator: mem.Allocator, rc: *ResolvConf) !void {...@@ -1465,15 +1465,15 @@ fn getResolvConf(allocator: mem.Allocator, rc: *ResolvConf) !void {
1465 else => |e| return e,1465 else => |e| return e,
1466 }) |line| {1466 }) |line| {
1467 const no_comment_line = no_comment_line: {1467 const no_comment_line = no_comment_line: {
1468 var split = mem.split(u8, line, "#");1468 var split = mem.splitScalar(u8, line, '#');
1469 break :no_comment_line split.first();1469 break :no_comment_line split.first();
1470 };1470 };
1471 var line_it = mem.tokenize(u8, no_comment_line, " \t");1471 var line_it = mem.tokenizeAny(u8, no_comment_line, " \t");
14721472
1473 const token = line_it.next() orelse continue;1473 const token = line_it.next() orelse continue;
1474 if (mem.eql(u8, token, "options")) {1474 if (mem.eql(u8, token, "options")) {
1475 while (line_it.next()) |sub_tok| {1475 while (line_it.next()) |sub_tok| {
1476 var colon_it = mem.split(u8, sub_tok, ":");1476 var colon_it = mem.splitScalar(u8, sub_tok, ':');
1477 const name = colon_it.first();1477 const name = colon_it.first();
1478 const value_txt = colon_it.next() orelse continue;1478 const value_txt = colon_it.next() orelse continue;
1479 const value = std.fmt.parseInt(u8, value_txt, 10) catch |err| switch (err) {1479 const value = std.fmt.parseInt(u8, value_txt, 10) catch |err| switch (err) {
lib/std/os.zig+1-1
...@@ -1878,7 +1878,7 @@ pub fn execvpeZ_expandArg0(...@@ -1878,7 +1878,7 @@ pub fn execvpeZ_expandArg0(
1878 // Use of MAX_PATH_BYTES here is valid as the path_buf will be passed1878 // Use of MAX_PATH_BYTES here is valid as the path_buf will be passed
1879 // directly to the operating system in execveZ.1879 // directly to the operating system in execveZ.
1880 var path_buf: [MAX_PATH_BYTES]u8 = undefined;1880 var path_buf: [MAX_PATH_BYTES]u8 = undefined;
1881 var it = mem.tokenize(u8, PATH, ":");1881 var it = mem.tokenizeScalar(u8, PATH, ':');
1882 var seen_eacces = false;1882 var seen_eacces = false;
1883 var err: ExecveError = error.FileNotFound;1883 var err: ExecveError = error.FileNotFound;
18841884
lib/std/process.zig+2-2
...@@ -310,7 +310,7 @@ pub fn getEnvMap(allocator: Allocator) !EnvMap {...@@ -310,7 +310,7 @@ pub fn getEnvMap(allocator: Allocator) !EnvMap {
310310
311 for (environ) |env| {311 for (environ) |env| {
312 const pair = mem.sliceTo(env, 0);312 const pair = mem.sliceTo(env, 0);
313 var parts = mem.split(u8, pair, "=");313 var parts = mem.splitScalar(u8, pair, '=');
314 const key = parts.first();314 const key = parts.first();
315 const value = parts.rest();315 const value = parts.rest();
316 try result.put(key, value);316 try result.put(key, value);
...@@ -1200,7 +1200,7 @@ fn totalSystemMemoryLinux() !usize {...@@ -1200,7 +1200,7 @@ fn totalSystemMemoryLinux() !usize {
1200 var buf: [50]u8 = undefined;1200 var buf: [50]u8 = undefined;
1201 const amt = try file.read(&buf);1201 const amt = try file.read(&buf);
1202 if (amt != 50) return error.Unexpected;1202 if (amt != 50) return error.Unexpected;
1203 var it = std.mem.tokenize(u8, buf[0..amt], " \n");1203 var it = std.mem.tokenizeAny(u8, buf[0..amt], " \n");
1204 const label = it.next().?;1204 const label = it.next().?;
1205 if (!std.mem.eql(u8, label, "MemTotal:")) return error.Unexpected;1205 if (!std.mem.eql(u8, label, "MemTotal:")) return error.Unexpected;
1206 const int_text = it.next() orelse return error.Unexpected;1206 const int_text = it.next() orelse return error.Unexpected;
lib/std/zig/CrossTarget.zig+6-6
...@@ -239,7 +239,7 @@ pub fn parse(args: ParseOptions) !CrossTarget {...@@ -239,7 +239,7 @@ pub fn parse(args: ParseOptions) !CrossTarget {
239 .dynamic_linker = DynamicLinker.init(args.dynamic_linker),239 .dynamic_linker = DynamicLinker.init(args.dynamic_linker),
240 };240 };
241241
242 var it = mem.split(u8, args.arch_os_abi, "-");242 var it = mem.splitScalar(u8, args.arch_os_abi, '-');
243 const arch_name = it.first();243 const arch_name = it.first();
244 const arch_is_native = mem.eql(u8, arch_name, "native");244 const arch_is_native = mem.eql(u8, arch_name, "native");
245 if (!arch_is_native) {245 if (!arch_is_native) {
...@@ -257,7 +257,7 @@ pub fn parse(args: ParseOptions) !CrossTarget {...@@ -257,7 +257,7 @@ pub fn parse(args: ParseOptions) !CrossTarget {
257257
258 const opt_abi_text = it.next();258 const opt_abi_text = it.next();
259 if (opt_abi_text) |abi_text| {259 if (opt_abi_text) |abi_text| {
260 var abi_it = mem.split(u8, abi_text, ".");260 var abi_it = mem.splitScalar(u8, abi_text, '.');
261 const abi = std.meta.stringToEnum(Target.Abi, abi_it.first()) orelse261 const abi = std.meta.stringToEnum(Target.Abi, abi_it.first()) orelse
262 return error.UnknownApplicationBinaryInterface;262 return error.UnknownApplicationBinaryInterface;
263 result.abi = abi;263 result.abi = abi;
...@@ -343,7 +343,7 @@ pub fn parse(args: ParseOptions) !CrossTarget {...@@ -343,7 +343,7 @@ pub fn parse(args: ParseOptions) !CrossTarget {
343/// This is intended to be used if the API user of CrossTarget needs to learn the343/// This is intended to be used if the API user of CrossTarget needs to learn the
344/// target CPU architecture in order to fully populate `ParseOptions`.344/// target CPU architecture in order to fully populate `ParseOptions`.
345pub fn parseCpuArch(args: ParseOptions) ?Target.Cpu.Arch {345pub fn parseCpuArch(args: ParseOptions) ?Target.Cpu.Arch {
346 var it = mem.split(u8, args.arch_os_abi, "-");346 var it = mem.splitScalar(u8, args.arch_os_abi, '-');
347 const arch_name = it.first();347 const arch_name = it.first();
348 const arch_is_native = mem.eql(u8, arch_name, "native");348 const arch_is_native = mem.eql(u8, arch_name, "native");
349 if (arch_is_native) {349 if (arch_is_native) {
...@@ -645,7 +645,7 @@ pub fn updateCpuFeatures(self: CrossTarget, set: *Target.Cpu.Feature.Set) void {...@@ -645,7 +645,7 @@ pub fn updateCpuFeatures(self: CrossTarget, set: *Target.Cpu.Feature.Set) void {
645}645}
646646
647fn parseOs(result: *CrossTarget, diags: *ParseOptions.Diagnostics, text: []const u8) !void {647fn parseOs(result: *CrossTarget, diags: *ParseOptions.Diagnostics, text: []const u8) !void {
648 var it = mem.split(u8, text, ".");648 var it = mem.splitScalar(u8, text, '.');
649 const os_name = it.first();649 const os_name = it.first();
650 diags.os_name = os_name;650 diags.os_name = os_name;
651 const os_is_native = mem.eql(u8, os_name, "native");651 const os_is_native = mem.eql(u8, os_name, "native");
...@@ -706,7 +706,7 @@ fn parseOs(result: *CrossTarget, diags: *ParseOptions.Diagnostics, text: []const...@@ -706,7 +706,7 @@ fn parseOs(result: *CrossTarget, diags: *ParseOptions.Diagnostics, text: []const
706 .linux,706 .linux,
707 .dragonfly,707 .dragonfly,
708 => {708 => {
709 var range_it = mem.split(u8, version_text, "...");709 var range_it = mem.splitSequence(u8, version_text, "...");
710710
711 const min_text = range_it.next().?;711 const min_text = range_it.next().?;
712 const min_ver = SemVer.parse(min_text) catch |err| switch (err) {712 const min_ver = SemVer.parse(min_text) catch |err| switch (err) {
...@@ -726,7 +726,7 @@ fn parseOs(result: *CrossTarget, diags: *ParseOptions.Diagnostics, text: []const...@@ -726,7 +726,7 @@ fn parseOs(result: *CrossTarget, diags: *ParseOptions.Diagnostics, text: []const
726 },726 },
727727
728 .windows => {728 .windows => {
729 var range_it = mem.split(u8, version_text, "...");729 var range_it = mem.splitSequence(u8, version_text, "...");
730730
731 const min_text = range_it.first();731 const min_text = range_it.first();
732 const min_ver = std.meta.stringToEnum(Target.Os.WindowsVersion, min_text) orelse732 const min_ver = std.meta.stringToEnum(Target.Os.WindowsVersion, min_text) orelse
lib/std/zig/ErrorBundle.zig+1-1
...@@ -294,7 +294,7 @@ fn renderErrorMessageToWriter(...@@ -294,7 +294,7 @@ fn renderErrorMessageToWriter(
294///294///
295/// This is used to split the message in `@compileError("hello\nworld")` for example.295/// This is used to split the message in `@compileError("hello\nworld")` for example.
296fn writeMsg(eb: ErrorBundle, err_msg: ErrorMessage, stderr: anytype, indent: usize) !void {296fn writeMsg(eb: ErrorBundle, err_msg: ErrorMessage, stderr: anytype, indent: usize) !void {
297 var lines = std.mem.split(u8, eb.nullTerminatedString(err_msg.msg), "\n");297 var lines = std.mem.splitScalar(u8, eb.nullTerminatedString(err_msg.msg), '\n');
298 while (lines.next()) |line| {298 while (lines.next()) |line| {
299 try stderr.writeAll(line);299 try stderr.writeAll(line);
300 if (lines.index == null) break;300 if (lines.index == null) break;
lib/std/zig/render.zig+1-1
...@@ -1995,7 +1995,7 @@ fn renderArrayInit(...@@ -1995,7 +1995,7 @@ fn renderArrayInit(
1995 if (!expr_newlines[i]) {1995 if (!expr_newlines[i]) {
1996 try ais.writer().writeAll(expr_text);1996 try ais.writer().writeAll(expr_text);
1997 } else {1997 } else {
1998 var by_line = std.mem.split(u8, expr_text, "\n");1998 var by_line = std.mem.splitScalar(u8, expr_text, '\n');
1999 var last_line_was_empty = false;1999 var last_line_was_empty = false;
2000 try ais.writer().writeAll(by_line.first());2000 try ais.writer().writeAll(by_line.first());
2001 while (by_line.next()) |line| {2001 while (by_line.next()) |line| {
lib/std/zig/system/NativePaths.zig+5-5
...@@ -31,7 +31,7 @@ pub fn detect(allocator: Allocator, native_info: NativeTargetInfo) !NativePaths...@@ -31,7 +31,7 @@ pub fn detect(allocator: Allocator, native_info: NativeTargetInfo) !NativePaths
31 defer allocator.free(nix_cflags_compile);31 defer allocator.free(nix_cflags_compile);
3232
33 is_nix = true;33 is_nix = true;
34 var it = mem.tokenize(u8, nix_cflags_compile, " ");34 var it = mem.tokenizeScalar(u8, nix_cflags_compile, ' ');
35 while (true) {35 while (true) {
36 const word = it.next() orelse break;36 const word = it.next() orelse break;
37 if (mem.eql(u8, word, "-isystem")) {37 if (mem.eql(u8, word, "-isystem")) {
...@@ -62,7 +62,7 @@ pub fn detect(allocator: Allocator, native_info: NativeTargetInfo) !NativePaths...@@ -62,7 +62,7 @@ pub fn detect(allocator: Allocator, native_info: NativeTargetInfo) !NativePaths
62 defer allocator.free(nix_ldflags);62 defer allocator.free(nix_ldflags);
6363
64 is_nix = true;64 is_nix = true;
65 var it = mem.tokenize(u8, nix_ldflags, " ");65 var it = mem.tokenizeScalar(u8, nix_ldflags, ' ');
66 while (true) {66 while (true) {
67 const word = it.next() orelse break;67 const word = it.next() orelse break;
68 if (mem.eql(u8, word, "-rpath")) {68 if (mem.eql(u8, word, "-rpath")) {
...@@ -147,21 +147,21 @@ pub fn detect(allocator: Allocator, native_info: NativeTargetInfo) !NativePaths...@@ -147,21 +147,21 @@ pub fn detect(allocator: Allocator, native_info: NativeTargetInfo) !NativePaths
147 // We use os.getenv here since this part won't be executed on147 // We use os.getenv here since this part won't be executed on
148 // windows, to get rid of unnecessary error handling.148 // windows, to get rid of unnecessary error handling.
149 if (std.os.getenv("C_INCLUDE_PATH")) |c_include_path| {149 if (std.os.getenv("C_INCLUDE_PATH")) |c_include_path| {
150 var it = mem.tokenize(u8, c_include_path, ":");150 var it = mem.tokenizeScalar(u8, c_include_path, ':');
151 while (it.next()) |dir| {151 while (it.next()) |dir| {
152 try self.addIncludeDir(dir);152 try self.addIncludeDir(dir);
153 }153 }
154 }154 }
155155
156 if (std.os.getenv("CPLUS_INCLUDE_PATH")) |cplus_include_path| {156 if (std.os.getenv("CPLUS_INCLUDE_PATH")) |cplus_include_path| {
157 var it = mem.tokenize(u8, cplus_include_path, ":");157 var it = mem.tokenizeScalar(u8, cplus_include_path, ':');
158 while (it.next()) |dir| {158 while (it.next()) |dir| {
159 try self.addIncludeDir(dir);159 try self.addIncludeDir(dir);
160 }160 }
161 }161 }
162162
163 if (std.os.getenv("LIBRARY_PATH")) |library_path| {163 if (std.os.getenv("LIBRARY_PATH")) |library_path| {
164 var it = mem.tokenize(u8, library_path, ":");164 var it = mem.tokenizeScalar(u8, library_path, ':');
165 while (it.next()) |dir| {165 while (it.next()) |dir| {
166 try self.addLibDir(dir);166 try self.addLibDir(dir);
167 }167 }
lib/std/zig/system/NativeTargetInfo.zig+3-3
...@@ -354,7 +354,7 @@ fn detectAbiAndDynamicLinker(...@@ -354,7 +354,7 @@ fn detectAbiAndDynamicLinker(
354 const newline = mem.indexOfScalar(u8, buffer[0..len], '\n') orelse break :blk file;354 const newline = mem.indexOfScalar(u8, buffer[0..len], '\n') orelse break :blk file;
355 const line = buffer[0..newline];355 const line = buffer[0..newline];
356 if (!mem.startsWith(u8, line, "#!")) break :blk file;356 if (!mem.startsWith(u8, line, "#!")) break :blk file;
357 var it = mem.tokenize(u8, line[2..], " ");357 var it = mem.tokenizeScalar(u8, line[2..], ' ');
358 file_name = it.next() orelse return defaultAbiAndDynamicLinker(cpu, os, cross_target);358 file_name = it.next() orelse return defaultAbiAndDynamicLinker(cpu, os, cross_target);
359 file.close();359 file.close();
360 }360 }
...@@ -556,7 +556,7 @@ fn glibcVerFromSoFile(file: fs.File) !std.builtin.Version {...@@ -556,7 +556,7 @@ fn glibcVerFromSoFile(file: fs.File) !std.builtin.Version {
556 const dynstr_size = @intCast(usize, dynstr.size);556 const dynstr_size = @intCast(usize, dynstr.size);
557 const dynstr_bytes = buf[0..dynstr_size];557 const dynstr_bytes = buf[0..dynstr_size];
558 _ = try preadMin(file, dynstr_bytes, dynstr.offset, dynstr_bytes.len);558 _ = try preadMin(file, dynstr_bytes, dynstr.offset, dynstr_bytes.len);
559 var it = mem.split(u8, dynstr_bytes, &.{0});559 var it = mem.splitScalar(u8, dynstr_bytes, 0);
560 var max_ver: std.builtin.Version = .{ .major = 2, .minor = 2, .patch = 5 };560 var max_ver: std.builtin.Version = .{ .major = 2, .minor = 2, .patch = 5 };
561 while (it.next()) |s| {561 while (it.next()) |s| {
562 if (mem.startsWith(u8, s, "GLIBC_2.")) {562 if (mem.startsWith(u8, s, "GLIBC_2.")) {
...@@ -811,7 +811,7 @@ pub fn abiAndDynamicLinkerFromFile(...@@ -811,7 +811,7 @@ pub fn abiAndDynamicLinkerFromFile(
811 const strtab = strtab_buf[0..strtab_read_len];811 const strtab = strtab_buf[0..strtab_read_len];
812812
813 const rpath_list = mem.sliceTo(strtab, 0);813 const rpath_list = mem.sliceTo(strtab, 0);
814 var it = mem.tokenize(u8, rpath_list, ":");814 var it = mem.tokenizeScalar(u8, rpath_list, ':');
815 while (it.next()) |rpath| {815 while (it.next()) |rpath| {
816 if (glibcVerFromRPath(rpath)) |ver| {816 if (glibcVerFromRPath(rpath)) |ver| {
817 result.target.os.version_range.linux.glibc = ver;817 result.target.os.version_range.linux.glibc = ver;
src/Autodoc.zig+1-1
...@@ -4950,7 +4950,7 @@ fn findGuidePaths(self: *Autodoc, file: *File, str: []const u8) ![]const u8 {...@@ -4950,7 +4950,7 @@ fn findGuidePaths(self: *Autodoc, file: *File, str: []const u8) ![]const u8 {
49504950
4951 // TODO: this algo is kinda inefficient4951 // TODO: this algo is kinda inefficient
49524952
4953 var it = std.mem.split(u8, str, "\n");4953 var it = std.mem.splitScalar(u8, str, '\n');
4954 while (it.next()) |line| {4954 while (it.next()) |line| {
4955 const trimmed_line = std.mem.trim(u8, line, " ");4955 const trimmed_line = std.mem.trim(u8, line, " ");
4956 if (std.mem.startsWith(u8, trimmed_line, guide_prefix)) {4956 if (std.mem.startsWith(u8, trimmed_line, guide_prefix)) {
src/Compilation.zig+3-3
...@@ -4671,7 +4671,7 @@ pub fn hasSharedLibraryExt(filename: []const u8) bool {...@@ -4671,7 +4671,7 @@ pub fn hasSharedLibraryExt(filename: []const u8) bool {
4671 return true;4671 return true;
4672 }4672 }
4673 // Look for .so.X, .so.X.Y, .so.X.Y.Z4673 // Look for .so.X, .so.X.Y, .so.X.Y.Z
4674 var it = mem.split(u8, filename, ".");4674 var it = mem.splitScalar(u8, filename, '.');
4675 _ = it.first();4675 _ = it.first();
4676 var so_txt = it.next() orelse return false;4676 var so_txt = it.next() orelse return false;
4677 while (!mem.eql(u8, so_txt, "so")) {4677 while (!mem.eql(u8, so_txt, "so")) {
...@@ -5051,14 +5051,14 @@ fn parseLldStderr(comp: *Compilation, comptime prefix: []const u8, stderr: []con...@@ -5051,14 +5051,14 @@ fn parseLldStderr(comp: *Compilation, comptime prefix: []const u8, stderr: []con
5051 defer context_lines.deinit();5051 defer context_lines.deinit();
50525052
5053 var current_err: ?*LldError = null;5053 var current_err: ?*LldError = null;
5054 var lines = mem.split(u8, stderr, std.cstr.line_sep);5054 var lines = mem.splitSequence(u8, stderr, std.cstr.line_sep);
5055 while (lines.next()) |line| {5055 while (lines.next()) |line| {
5056 if (mem.startsWith(u8, line, prefix ++ ":")) {5056 if (mem.startsWith(u8, line, prefix ++ ":")) {
5057 if (current_err) |err| {5057 if (current_err) |err| {
5058 err.context_lines = try context_lines.toOwnedSlice();5058 err.context_lines = try context_lines.toOwnedSlice();
5059 }5059 }
50605060
5061 var split = std.mem.split(u8, line, "error: ");5061 var split = std.mem.splitSequence(u8, line, "error: ");
5062 _ = split.first();5062 _ = split.first();
50635063
5064 const duped_msg = try std.fmt.allocPrint(comp.gpa, "{s}: {s}", .{ prefix, split.rest() });5064 const duped_msg = try std.fmt.allocPrint(comp.gpa, "{s}: {s}", .{ prefix, split.rest() });
src/arch/x86_64/CodeGen.zig+3-3
...@@ -9232,9 +9232,9 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -9232,9 +9232,9 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
9232 }9232 }
92339233
9234 const asm_source = mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len];9234 const asm_source = mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len];
9235 var line_it = mem.tokenize(u8, asm_source, "\n\r;");9235 var line_it = mem.tokenizeAny(u8, asm_source, "\n\r;");
9236 while (line_it.next()) |line| {9236 while (line_it.next()) |line| {
9237 var mnem_it = mem.tokenize(u8, line, " \t");9237 var mnem_it = mem.tokenizeAny(u8, line, " \t");
9238 const mnem_str = mnem_it.next() orelse continue;9238 const mnem_str = mnem_it.next() orelse continue;
9239 if (mem.startsWith(u8, mnem_str, "#")) continue;9239 if (mem.startsWith(u8, mnem_str, "#")) continue;
92409240
...@@ -9258,7 +9258,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {...@@ -9258,7 +9258,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
9258 return self.fail("Invalid mnemonic: '{s}'", .{mnem_str});9258 return self.fail("Invalid mnemonic: '{s}'", .{mnem_str});
9259 } };9259 } };
92609260
9261 var op_it = mem.tokenize(u8, mnem_it.rest(), ",");9261 var op_it = mem.tokenizeScalar(u8, mnem_it.rest(), ',');
9262 var ops = [1]encoder.Instruction.Operand{.none} ** 4;9262 var ops = [1]encoder.Instruction.Operand{.none} ** 4;
9263 for (&ops) |*op| {9263 for (&ops) |*op| {
9264 const op_str = mem.trim(u8, op_it.next() orelse break, " \t");9264 const op_str = mem.trim(u8, op_it.next() orelse break, " \t");
src/glibc.zig+1-1
...@@ -109,7 +109,7 @@ pub fn loadMetaData(gpa: Allocator, contents: []const u8) LoadMetaDataError!*ABI...@@ -109,7 +109,7 @@ pub fn loadMetaData(gpa: Allocator, contents: []const u8) LoadMetaDataError!*ABI
109 const target_name = mem.sliceTo(contents[index..], 0);109 const target_name = mem.sliceTo(contents[index..], 0);
110 index += target_name.len + 1;110 index += target_name.len + 1;
111111
112 var component_it = mem.tokenize(u8, target_name, "-");112 var component_it = mem.tokenizeScalar(u8, target_name, '-');
113 const arch_name = component_it.next() orelse {113 const arch_name = component_it.next() orelse {
114 log.err("abilists: expected arch name", .{});114 log.err("abilists: expected arch name", .{});
115 return error.ZigInstallationCorrupt;115 return error.ZigInstallationCorrupt;
src/libc_installation.zig+5-5
...@@ -60,10 +60,10 @@ pub const LibCInstallation = struct {...@@ -60,10 +60,10 @@ pub const LibCInstallation = struct {
60 const contents = try std.fs.cwd().readFileAlloc(allocator, libc_file, std.math.maxInt(usize));60 const contents = try std.fs.cwd().readFileAlloc(allocator, libc_file, std.math.maxInt(usize));
61 defer allocator.free(contents);61 defer allocator.free(contents);
6262
63 var it = std.mem.tokenize(u8, contents, "\n");63 var it = std.mem.tokenizeScalar(u8, contents, '\n');
64 while (it.next()) |line| {64 while (it.next()) |line| {
65 if (line.len == 0 or line[0] == '#') continue;65 if (line.len == 0 or line[0] == '#') continue;
66 var line_it = std.mem.split(u8, line, "=");66 var line_it = std.mem.splitScalar(u8, line, '=');
67 const name = line_it.first();67 const name = line_it.first();
68 const value = line_it.rest();68 const value = line_it.rest();
69 inline for (fields, 0..) |field, i| {69 inline for (fields, 0..) |field, i| {
...@@ -293,7 +293,7 @@ pub const LibCInstallation = struct {...@@ -293,7 +293,7 @@ pub const LibCInstallation = struct {
293 },293 },
294 }294 }
295295
296 var it = std.mem.tokenize(u8, exec_res.stderr, "\n\r");296 var it = std.mem.tokenizeAny(u8, exec_res.stderr, "\n\r");
297 var search_paths = std.ArrayList([]const u8).init(allocator);297 var search_paths = std.ArrayList([]const u8).init(allocator);
298 defer search_paths.deinit();298 defer search_paths.deinit();
299 while (it.next()) |line| {299 while (it.next()) |line| {
...@@ -613,7 +613,7 @@ fn ccPrintFileName(args: CCPrintFileNameOptions) ![:0]u8 {...@@ -613,7 +613,7 @@ fn ccPrintFileName(args: CCPrintFileNameOptions) ![:0]u8 {
613 },613 },
614 }614 }
615615
616 var it = std.mem.tokenize(u8, exec_res.stdout, "\n\r");616 var it = std.mem.tokenizeAny(u8, exec_res.stdout, "\n\r");
617 const line = it.next() orelse return error.LibCRuntimeNotFound;617 const line = it.next() orelse return error.LibCRuntimeNotFound;
618 // When this command fails, it returns exit code 0 and duplicates the input file name.618 // When this command fails, it returns exit code 0 and duplicates the input file name.
619 // So we detect failure by checking if the output matches exactly the input.619 // So we detect failure by checking if the output matches exactly the input.
...@@ -692,7 +692,7 @@ fn appendCcExe(args: *std.ArrayList([]const u8), skip_cc_env_var: bool) !void {...@@ -692,7 +692,7 @@ fn appendCcExe(args: *std.ArrayList([]const u8), skip_cc_env_var: bool) !void {
692 return;692 return;
693 };693 };
694 // Respect space-separated flags to the C compiler.694 // Respect space-separated flags to the C compiler.
695 var it = std.mem.tokenize(u8, cc_env_var, " ");695 var it = std.mem.tokenizeScalar(u8, cc_env_var, ' ');
696 while (it.next()) |arg| {696 while (it.next()) |arg| {
697 try args.append(arg);697 try args.append(arg);
698 }698 }
src/link/MachO/Dylib.zig+1-1
...@@ -91,7 +91,7 @@ pub const Id = struct {...@@ -91,7 +91,7 @@ pub const Id = struct {
91 var out: u32 = 0;91 var out: u32 = 0;
92 var values: [3][]const u8 = undefined;92 var values: [3][]const u8 = undefined;
9393
94 var split = mem.split(u8, string, ".");94 var split = mem.splitScalar(u8, string, '.');
95 var count: u4 = 0;95 var count: u4 = 0;
96 while (split.next()) |value| {96 while (split.next()) |value| {
97 if (count > 2) {97 if (count > 2) {
src/link/Plan9.zig+1-1
...@@ -264,7 +264,7 @@ fn putFn(self: *Plan9, decl_index: Module.Decl.Index, out: FnDeclOutput) !void {...@@ -264,7 +264,7 @@ fn putFn(self: *Plan9, decl_index: Module.Decl.Index, out: FnDeclOutput) !void {
264264
265fn addPathComponents(self: *Plan9, path: []const u8, a: *std.ArrayList(u8)) !void {265fn addPathComponents(self: *Plan9, path: []const u8, a: *std.ArrayList(u8)) !void {
266 const sep = std.fs.path.sep;266 const sep = std.fs.path.sep;
267 var it = std.mem.tokenize(u8, path, &.{sep});267 var it = std.mem.tokenizeScalar(u8, path, sep);
268 while (it.next()) |component| {268 while (it.next()) |component| {
269 if (self.file_segments.get(component)) |num| {269 if (self.file_segments.get(component)) |num| {
270 try a.writer().writeIntBig(u16, num);270 try a.writer().writeIntBig(u16, num);
src/main.zig+9-9
...@@ -973,7 +973,7 @@ fn buildOutputType(...@@ -973,7 +973,7 @@ fn buildOutputType(
973 }973 }
974 } else if (mem.eql(u8, arg, "--mod")) {974 } else if (mem.eql(u8, arg, "--mod")) {
975 const info = args_iter.nextOrFatal();975 const info = args_iter.nextOrFatal();
976 var info_it = mem.split(u8, info, ":");976 var info_it = mem.splitScalar(u8, info, ':');
977 const mod_name = info_it.next() orelse fatal("expected non-empty argument after {s}", .{arg});977 const mod_name = info_it.next() orelse fatal("expected non-empty argument after {s}", .{arg});
978 const deps_str = info_it.next() orelse fatal("expected 'name:deps:path' after {s}", .{arg});978 const deps_str = info_it.next() orelse fatal("expected 'name:deps:path' after {s}", .{arg});
979 const root_src_orig = info_it.rest();979 const root_src_orig = info_it.rest();
...@@ -1173,7 +1173,7 @@ fn buildOutputType(...@@ -1173,7 +1173,7 @@ fn buildOutputType(
1173 } else {1173 } else {
1174 if (build_options.only_core_functionality) unreachable;1174 if (build_options.only_core_functionality) unreachable;
1175 // example: --listen 127.0.0.1:90001175 // example: --listen 127.0.0.1:9000
1176 var it = std.mem.split(u8, next_arg, ":");1176 var it = std.mem.splitScalar(u8, next_arg, ':');
1177 const host = it.next().?;1177 const host = it.next().?;
1178 const port_text = it.next() orelse "14735";1178 const port_text = it.next() orelse "14735";
1179 const port = std.fmt.parseInt(u16, port_text, 10) catch |err|1179 const port = std.fmt.parseInt(u16, port_text, 10) catch |err|
...@@ -1676,7 +1676,7 @@ fn buildOutputType(...@@ -1676,7 +1676,7 @@ fn buildOutputType(
1676 },1676 },
1677 .rdynamic => rdynamic = true,1677 .rdynamic => rdynamic = true,
1678 .wl => {1678 .wl => {
1679 var split_it = mem.split(u8, it.only_arg, ",");1679 var split_it = mem.splitScalar(u8, it.only_arg, ',');
1680 while (split_it.next()) |linker_arg| {1680 while (split_it.next()) |linker_arg| {
1681 // Handle nested-joined args like `-Wl,-rpath=foo`.1681 // Handle nested-joined args like `-Wl,-rpath=foo`.
1682 // Must be prefixed with 1 or 2 dashes.1682 // Must be prefixed with 1 or 2 dashes.
...@@ -2191,17 +2191,17 @@ fn buildOutputType(...@@ -2191,17 +2191,17 @@ fn buildOutputType(
2191 const next_arg = linker_args_it.nextOrFatal();2191 const next_arg = linker_args_it.nextOrFatal();
2192 try symbol_wrap_set.put(arena, next_arg, {});2192 try symbol_wrap_set.put(arena, next_arg, {});
2193 } else if (mem.startsWith(u8, arg, "/subsystem:")) {2193 } else if (mem.startsWith(u8, arg, "/subsystem:")) {
2194 var split_it = mem.splitBackwards(u8, arg, ":");2194 var split_it = mem.splitBackwardsScalar(u8, arg, ':');
2195 subsystem = try parseSubSystem(split_it.first());2195 subsystem = try parseSubSystem(split_it.first());
2196 } else if (mem.startsWith(u8, arg, "/implib:")) {2196 } else if (mem.startsWith(u8, arg, "/implib:")) {
2197 var split_it = mem.splitBackwards(u8, arg, ":");2197 var split_it = mem.splitBackwardsScalar(u8, arg, ':');
2198 emit_implib = .{ .yes = split_it.first() };2198 emit_implib = .{ .yes = split_it.first() };
2199 emit_implib_arg_provided = true;2199 emit_implib_arg_provided = true;
2200 } else if (mem.startsWith(u8, arg, "/pdb:")) {2200 } else if (mem.startsWith(u8, arg, "/pdb:")) {
2201 var split_it = mem.splitBackwards(u8, arg, ":");2201 var split_it = mem.splitBackwardsScalar(u8, arg, ':');
2202 pdb_out_path = split_it.first();2202 pdb_out_path = split_it.first();
2203 } else if (mem.startsWith(u8, arg, "/version:")) {2203 } else if (mem.startsWith(u8, arg, "/version:")) {
2204 var split_it = mem.splitBackwards(u8, arg, ":");2204 var split_it = mem.splitBackwardsScalar(u8, arg, ':');
2205 const version_arg = split_it.first();2205 const version_arg = split_it.first();
2206 version = std.builtin.Version.parse(version_arg) catch |err| {2206 version = std.builtin.Version.parse(version_arg) catch |err| {
2207 fatal("unable to parse /version '{s}': {s}", .{ arg, @errorName(err) });2207 fatal("unable to parse /version '{s}': {s}", .{ arg, @errorName(err) });
...@@ -3541,10 +3541,10 @@ fn serveUpdateResults(s: *Server, comp: *Compilation) !void {...@@ -3541,10 +3541,10 @@ fn serveUpdateResults(s: *Server, comp: *Compilation) !void {
3541}3541}
35423542
3543const ModuleDepIterator = struct {3543const ModuleDepIterator = struct {
3544 split: mem.SplitIterator(u8),3544 split: mem.SplitIterator(u8, .scalar),
35453545
3546 fn init(deps_str: []const u8) ModuleDepIterator {3546 fn init(deps_str: []const u8) ModuleDepIterator {
3547 return .{ .split = mem.split(u8, deps_str, ",") };3547 return .{ .split = mem.splitScalar(u8, deps_str, ',') };
3548 }3548 }
35493549
3550 const Dependency = struct {3550 const Dependency = struct {
src/print_zir.zig+1-1
...@@ -2588,7 +2588,7 @@ const Writer = struct {...@@ -2588,7 +2588,7 @@ const Writer = struct {
2588 fn writeDocComment(self: *Writer, stream: anytype, doc_comment_index: u32) !void {2588 fn writeDocComment(self: *Writer, stream: anytype, doc_comment_index: u32) !void {
2589 if (doc_comment_index != 0) {2589 if (doc_comment_index != 0) {
2590 const doc_comment = self.code.nullTerminatedString(doc_comment_index);2590 const doc_comment = self.code.nullTerminatedString(doc_comment_index);
2591 var it = std.mem.tokenize(u8, doc_comment, "\n");2591 var it = std.mem.tokenizeScalar(u8, doc_comment, '\n');
2592 while (it.next()) |doc_line| {2592 while (it.next()) |doc_line| {
2593 try stream.writeByteNTimes(' ', self.indent);2593 try stream.writeByteNTimes(' ', self.indent);
2594 try stream.print("///{s}\n", .{doc_line});2594 try stream.print("///{s}\n", .{doc_line});
test/behavior/bugs/6456.zig+1-1
...@@ -18,7 +18,7 @@ test "issue 6456" {...@@ -18,7 +18,7 @@ test "issue 6456" {
18 comptime {18 comptime {
19 var fields: []const StructField = &[0]StructField{};19 var fields: []const StructField = &[0]StructField{};
2020
21 var it = std.mem.tokenize(u8, text, "\n");21 var it = std.mem.tokenizeScalar(u8, text, '\n');
22 while (it.next()) |name| {22 while (it.next()) |name| {
23 fields = fields ++ &[_]StructField{StructField{23 fields = fields ++ &[_]StructField{StructField{
24 .alignment = 0,24 .alignment = 0,
test/src/Cases.zig+7-7
...@@ -804,7 +804,7 @@ const TestManifest = struct {...@@ -804,7 +804,7 @@ const TestManifest = struct {
804 };804 };
805805
806 const TrailingIterator = struct {806 const TrailingIterator = struct {
807 inner: std.mem.TokenIterator(u8),807 inner: std.mem.TokenIterator(u8, .any),
808808
809 fn next(self: *TrailingIterator) ?[]const u8 {809 fn next(self: *TrailingIterator) ?[]const u8 {
810 const next_inner = self.inner.next() orelse return null;810 const next_inner = self.inner.next() orelse return null;
...@@ -814,7 +814,7 @@ const TestManifest = struct {...@@ -814,7 +814,7 @@ const TestManifest = struct {
814814
815 fn ConfigValueIterator(comptime T: type) type {815 fn ConfigValueIterator(comptime T: type) type {
816 return struct {816 return struct {
817 inner: std.mem.SplitIterator(u8),817 inner: std.mem.SplitIterator(u8, .scalar),
818818
819 fn next(self: *@This()) !?T {819 fn next(self: *@This()) !?T {
820 const next_raw = self.inner.next() orelse return null;820 const next_raw = self.inner.next() orelse return null;
...@@ -855,7 +855,7 @@ const TestManifest = struct {...@@ -855,7 +855,7 @@ const TestManifest = struct {
855 const actual_start = start orelse return error.MissingTestManifest;855 const actual_start = start orelse return error.MissingTestManifest;
856 const manifest_bytes = bytes[actual_start..end];856 const manifest_bytes = bytes[actual_start..end];
857857
858 var it = std.mem.tokenize(u8, manifest_bytes, "\r\n");858 var it = std.mem.tokenizeAny(u8, manifest_bytes, "\r\n");
859859
860 // First line is the test type860 // First line is the test type
861 const tt: Type = blk: {861 const tt: Type = blk: {
...@@ -886,7 +886,7 @@ const TestManifest = struct {...@@ -886,7 +886,7 @@ const TestManifest = struct {
886 if (trimmed.len == 0) break;886 if (trimmed.len == 0) break;
887887
888 // Parse key=value(s)888 // Parse key=value(s)
889 var kv_it = std.mem.split(u8, trimmed, "=");889 var kv_it = std.mem.splitScalar(u8, trimmed, '=');
890 const key = kv_it.first();890 const key = kv_it.first();
891 try manifest.config_map.putNoClobber(key, kv_it.next() orelse return error.MissingValuesForConfig);891 try manifest.config_map.putNoClobber(key, kv_it.next() orelse return error.MissingValuesForConfig);
892 }892 }
...@@ -904,7 +904,7 @@ const TestManifest = struct {...@@ -904,7 +904,7 @@ const TestManifest = struct {
904 ) ConfigValueIterator(T) {904 ) ConfigValueIterator(T) {
905 const bytes = self.config_map.get(key) orelse TestManifestConfigDefaults.get(self.type, key);905 const bytes = self.config_map.get(key) orelse TestManifestConfigDefaults.get(self.type, key);
906 return ConfigValueIterator(T){906 return ConfigValueIterator(T){
907 .inner = std.mem.split(u8, bytes, ","),907 .inner = std.mem.splitScalar(u8, bytes, ','),
908 };908 };
909 }909 }
910910
...@@ -932,7 +932,7 @@ const TestManifest = struct {...@@ -932,7 +932,7 @@ const TestManifest = struct {
932932
933 fn trailing(self: TestManifest) TrailingIterator {933 fn trailing(self: TestManifest) TrailingIterator {
934 return .{934 return .{
935 .inner = std.mem.tokenize(u8, self.trailing_bytes, "\r\n"),935 .inner = std.mem.tokenizeAny(u8, self.trailing_bytes, "\r\n"),
936 };936 };
937 }937 }
938938
...@@ -1408,7 +1408,7 @@ fn runOneCase(...@@ -1408,7 +1408,7 @@ fn runOneCase(
1408 // Render the expected lines into a string that we can compare verbatim.1408 // Render the expected lines into a string that we can compare verbatim.
1409 var expected_generated = std.ArrayList(u8).init(arena);1409 var expected_generated = std.ArrayList(u8).init(arena);
14101410
1411 var actual_line_it = std.mem.split(u8, actual_stderr.items, "\n");1411 var actual_line_it = std.mem.splitScalar(u8, actual_stderr.items, '\n');
1412 for (expected_errors) |expect_line| {1412 for (expected_errors) |expect_line| {
1413 const actual_line = actual_line_it.next() orelse {1413 const actual_line = actual_line_it.next() orelse {
1414 try expected_generated.appendSlice(expect_line);1414 try expected_generated.appendSlice(expect_line);
test/src/check-stack-trace.zig+1-1
...@@ -27,7 +27,7 @@ pub fn main() !void {...@@ -27,7 +27,7 @@ pub fn main() !void {
27 var buf = std.ArrayList(u8).init(arena);27 var buf = std.ArrayList(u8).init(arena);
28 defer buf.deinit();28 defer buf.deinit();
29 if (stderr.len != 0 and stderr[stderr.len - 1] == '\n') stderr = stderr[0 .. stderr.len - 1];29 if (stderr.len != 0 and stderr[stderr.len - 1] == '\n') stderr = stderr[0 .. stderr.len - 1];
30 var it = mem.split(u8, stderr, "\n");30 var it = mem.splitScalar(u8, stderr, '\n');
31 process_lines: while (it.next()) |line| {31 process_lines: while (it.next()) |line| {
32 if (line.len == 0) continue;32 if (line.len == 0) continue;
3333
tools/gen_outline_atomics.zig+1-1
...@@ -88,7 +88,7 @@ fn writeFunction(...@@ -88,7 +88,7 @@ fn writeFunction(
88 \\ asm volatile (88 \\ asm volatile (
89 \\89 \\
90 );90 );
91 var iter = std.mem.split(u8, body, "\n");91 var iter = std.mem.splitScalar(u8, body, '\n');
92 while (iter.next()) |line| {92 while (iter.next()) |line| {
93 try w.writeAll(" \\\\");93 try w.writeAll(" \\\\");
94 try w.writeAll(line);94 try w.writeAll(line);
tools/generate_linux_syscalls.zig+18-18
...@@ -51,11 +51,11 @@ pub fn main() !void {...@@ -51,11 +51,11 @@ pub fn main() !void {
51 try writer.writeAll("pub const X86 = enum(usize) {\n");51 try writer.writeAll("pub const X86 = enum(usize) {\n");
5252
53 const table = try linux_dir.readFile("arch/x86/entry/syscalls/syscall_32.tbl", buf);53 const table = try linux_dir.readFile("arch/x86/entry/syscalls/syscall_32.tbl", buf);
54 var lines = mem.tokenize(u8, table, "\n");54 var lines = mem.tokenizeScalar(u8, table, '\n');
55 while (lines.next()) |line| {55 while (lines.next()) |line| {
56 if (line[0] == '#') continue;56 if (line[0] == '#') continue;
5757
58 var fields = mem.tokenize(u8, line, " \t");58 var fields = mem.tokenizeAny(u8, line, " \t");
59 const number = fields.next() orelse return error.Incomplete;59 const number = fields.next() orelse return error.Incomplete;
60 // abi is always i38660 // abi is always i386
61 _ = fields.next() orelse return error.Incomplete;61 _ = fields.next() orelse return error.Incomplete;
...@@ -70,11 +70,11 @@ pub fn main() !void {...@@ -70,11 +70,11 @@ pub fn main() !void {
70 try writer.writeAll("pub const X64 = enum(usize) {\n");70 try writer.writeAll("pub const X64 = enum(usize) {\n");
7171
72 const table = try linux_dir.readFile("arch/x86/entry/syscalls/syscall_64.tbl", buf);72 const table = try linux_dir.readFile("arch/x86/entry/syscalls/syscall_64.tbl", buf);
73 var lines = mem.tokenize(u8, table, "\n");73 var lines = mem.tokenizeScalar(u8, table, '\n');
74 while (lines.next()) |line| {74 while (lines.next()) |line| {
75 if (line[0] == '#') continue;75 if (line[0] == '#') continue;
7676
77 var fields = mem.tokenize(u8, line, " \t");77 var fields = mem.tokenizeAny(u8, line, " \t");
78 const number = fields.next() orelse return error.Incomplete;78 const number = fields.next() orelse return error.Incomplete;
79 const abi = fields.next() orelse return error.Incomplete;79 const abi = fields.next() orelse return error.Incomplete;
80 // The x32 abi syscalls are always at the end.80 // The x32 abi syscalls are always at the end.
...@@ -96,11 +96,11 @@ pub fn main() !void {...@@ -96,11 +96,11 @@ pub fn main() !void {
96 );96 );
9797
98 const table = try linux_dir.readFile("arch/arm/tools/syscall.tbl", buf);98 const table = try linux_dir.readFile("arch/arm/tools/syscall.tbl", buf);
99 var lines = mem.tokenize(u8, table, "\n");99 var lines = mem.tokenizeScalar(u8, table, '\n');
100 while (lines.next()) |line| {100 while (lines.next()) |line| {
101 if (line[0] == '#') continue;101 if (line[0] == '#') continue;
102102
103 var fields = mem.tokenize(u8, line, " \t");103 var fields = mem.tokenizeAny(u8, line, " \t");
104 const number = fields.next() orelse return error.Incomplete;104 const number = fields.next() orelse return error.Incomplete;
105 const abi = fields.next() orelse return error.Incomplete;105 const abi = fields.next() orelse return error.Incomplete;
106 if (mem.eql(u8, abi, "oabi")) continue;106 if (mem.eql(u8, abi, "oabi")) continue;
...@@ -127,11 +127,11 @@ pub fn main() !void {...@@ -127,11 +127,11 @@ pub fn main() !void {
127 {127 {
128 try writer.writeAll("pub const Sparc64 = enum(usize) {\n");128 try writer.writeAll("pub const Sparc64 = enum(usize) {\n");
129 const table = try linux_dir.readFile("arch/sparc/kernel/syscalls/syscall.tbl", buf);129 const table = try linux_dir.readFile("arch/sparc/kernel/syscalls/syscall.tbl", buf);
130 var lines = mem.tokenize(u8, table, "\n");130 var lines = mem.tokenizeScalar(u8, table, '\n');
131 while (lines.next()) |line| {131 while (lines.next()) |line| {
132 if (line[0] == '#') continue;132 if (line[0] == '#') continue;
133133
134 var fields = mem.tokenize(u8, line, " \t");134 var fields = mem.tokenizeAny(u8, line, " \t");
135 const number = fields.next() orelse return error.Incomplete;135 const number = fields.next() orelse return error.Incomplete;
136 const abi = fields.next() orelse return error.Incomplete;136 const abi = fields.next() orelse return error.Incomplete;
137 if (mem.eql(u8, abi, "32")) continue;137 if (mem.eql(u8, abi, "32")) continue;
...@@ -151,11 +151,11 @@ pub fn main() !void {...@@ -151,11 +151,11 @@ pub fn main() !void {
151 );151 );
152152
153 const table = try linux_dir.readFile("arch/mips/kernel/syscalls/syscall_o32.tbl", buf);153 const table = try linux_dir.readFile("arch/mips/kernel/syscalls/syscall_o32.tbl", buf);
154 var lines = mem.tokenize(u8, table, "\n");154 var lines = mem.tokenizeScalar(u8, table, '\n');
155 while (lines.next()) |line| {155 while (lines.next()) |line| {
156 if (line[0] == '#') continue;156 if (line[0] == '#') continue;
157157
158 var fields = mem.tokenize(u8, line, " \t");158 var fields = mem.tokenizeAny(u8, line, " \t");
159 const number = fields.next() orelse return error.Incomplete;159 const number = fields.next() orelse return error.Incomplete;
160 // abi is always o32160 // abi is always o32
161 _ = fields.next() orelse return error.Incomplete;161 _ = fields.next() orelse return error.Incomplete;
...@@ -176,11 +176,11 @@ pub fn main() !void {...@@ -176,11 +176,11 @@ pub fn main() !void {
176 );176 );
177177
178 const table = try linux_dir.readFile("arch/mips/kernel/syscalls/syscall_n64.tbl", buf);178 const table = try linux_dir.readFile("arch/mips/kernel/syscalls/syscall_n64.tbl", buf);
179 var lines = mem.tokenize(u8, table, "\n");179 var lines = mem.tokenizeScalar(u8, table, '\n');
180 while (lines.next()) |line| {180 while (lines.next()) |line| {
181 if (line[0] == '#') continue;181 if (line[0] == '#') continue;
182182
183 var fields = mem.tokenize(u8, line, " \t");183 var fields = mem.tokenizeAny(u8, line, " \t");
184 const number = fields.next() orelse return error.Incomplete;184 const number = fields.next() orelse return error.Incomplete;
185 // abi is always n64185 // abi is always n64
186 _ = fields.next() orelse return error.Incomplete;186 _ = fields.next() orelse return error.Incomplete;
...@@ -197,11 +197,11 @@ pub fn main() !void {...@@ -197,11 +197,11 @@ pub fn main() !void {
197197
198 const table = try linux_dir.readFile("arch/powerpc/kernel/syscalls/syscall.tbl", buf);198 const table = try linux_dir.readFile("arch/powerpc/kernel/syscalls/syscall.tbl", buf);
199 var list_64 = std.ArrayList(u8).init(allocator);199 var list_64 = std.ArrayList(u8).init(allocator);
200 var lines = mem.tokenize(u8, table, "\n");200 var lines = mem.tokenizeScalar(u8, table, '\n');
201 while (lines.next()) |line| {201 while (lines.next()) |line| {
202 if (line[0] == '#') continue;202 if (line[0] == '#') continue;
203203
204 var fields = mem.tokenize(u8, line, " \t");204 var fields = mem.tokenizeAny(u8, line, " \t");
205 const number = fields.next() orelse return error.Incomplete;205 const number = fields.next() orelse return error.Incomplete;
206 const abi = fields.next() orelse return error.Incomplete;206 const abi = fields.next() orelse return error.Incomplete;
207 const name = fields.next() orelse return error.Incomplete;207 const name = fields.next() orelse return error.Incomplete;
...@@ -277,9 +277,9 @@ pub fn main() !void {...@@ -277,9 +277,9 @@ pub fn main() !void {
277 },277 },
278 };278 };
279279
280 var lines = mem.tokenize(u8, defines, "\n");280 var lines = mem.tokenizeScalar(u8, defines, '\n');
281 loop: while (lines.next()) |line| {281 loop: while (lines.next()) |line| {
282 var fields = mem.tokenize(u8, line, " \t");282 var fields = mem.tokenizeAny(u8, line, " \t");
283 const cmd = fields.next() orelse return error.Incomplete;283 const cmd = fields.next() orelse return error.Incomplete;
284 if (!mem.eql(u8, cmd, "#define")) continue;284 if (!mem.eql(u8, cmd, "#define")) continue;
285 const define = fields.next() orelse return error.Incomplete;285 const define = fields.next() orelse return error.Incomplete;
...@@ -339,9 +339,9 @@ pub fn main() !void {...@@ -339,9 +339,9 @@ pub fn main() !void {
339 },339 },
340 };340 };
341341
342 var lines = mem.tokenize(u8, defines, "\n");342 var lines = mem.tokenizeScalar(u8, defines, '\n');
343 loop: while (lines.next()) |line| {343 loop: while (lines.next()) |line| {
344 var fields = mem.tokenize(u8, line, " \t");344 var fields = mem.tokenizeAny(u8, line, " \t");
345 const cmd = fields.next() orelse return error.Incomplete;345 const cmd = fields.next() orelse return error.Incomplete;
346 if (!mem.eql(u8, cmd, "#define")) continue;346 if (!mem.eql(u8, cmd, "#define")) continue;
347 const define = fields.next() orelse return error.Incomplete;347 const define = fields.next() orelse return error.Incomplete;
tools/update_crc_catalog.zig+1-1
...@@ -78,7 +78,7 @@ pub fn main() anyerror!void {...@@ -78,7 +78,7 @@ pub fn main() anyerror!void {
78 var residue: []const u8 = undefined;78 var residue: []const u8 = undefined;
79 var name: []const u8 = undefined;79 var name: []const u8 = undefined;
8080
81 var it = mem.split(u8, line, " ");81 var it = mem.splitSequence(u8, line, " ");
82 while (it.next()) |property| {82 while (it.next()) |property| {
83 const i = mem.indexOf(u8, property, "=").?;83 const i = mem.indexOf(u8, property, "=").?;
84 const key = property[0..i];84 const key = property[0..i];
tools/update_spirv_features.zig+1-1
...@@ -19,7 +19,7 @@ const Version = struct {...@@ -19,7 +19,7 @@ const Version = struct {
19 minor: u32,19 minor: u32,
2020
21 fn parse(str: []const u8) !Version {21 fn parse(str: []const u8) !Version {
22 var it = std.mem.split(u8, str, ".");22 var it = std.mem.splitScalar(u8, str, '.');
2323
24 const major = it.first();24 const major = it.first();
25 const minor = it.next() orelse return error.InvalidVersion;25 const minor = it.next() orelse return error.InvalidVersion;