| ... | @@ -1126,63 +1126,6 @@ pub const indexOfSentinel = findSentinel; | ... | @@ -1126,63 +1126,6 @@ pub const indexOfSentinel = findSentinel; |
| 1126 | /// Linear search through memory until the sentinel is found. | 1126 | /// Linear search through memory until the sentinel is found. |
| 1127 | pub fn findSentinel(comptime T: type, comptime sentinel: T, p: [*:sentinel]const T) usize { | 1127 | pub fn findSentinel(comptime T: type, comptime sentinel: T, p: [*:sentinel]const T) usize { |
| 1128 | var i: usize = 0; | 1128 | var i: usize = 0; |
| 1129 | | | |
| 1130 | if (use_vectors_for_comparison and | | |
| 1131 | !std.debug.inValgrind() and // https://github.com/ziglang/zig/issues/17717 | | |
| 1132 | !@inComptime() and | | |
| 1133 | (@typeInfo(T) == .int or @typeInfo(T) == .float) and std.math.isPowerOfTwo(@bitSizeOf(T))) | | |
| 1134 | { | | |
| 1135 | switch (@import("builtin").cpu.arch) { | | |
| 1136 | // The below branch assumes that reading past the end of the buffer is valid, as long | | |
| 1137 | // as we don't read into a new page. This should be the case for most architectures | | |
| 1138 | // which use paged memory, however should be confirmed before adding a new arch below. | | |
| 1139 | .aarch64, .x86, .x86_64 => if (std.simd.suggestVectorLength(T)) |block_len| { | | |
| 1140 | const page_size = std.heap.page_size_min; | | |
| 1141 | const block_size = @sizeOf(T) * block_len; | | |
| 1142 | const Block = @Vector(block_len, T); | | |
| 1143 | const mask: Block = @splat(sentinel); | | |
| 1144 | | | |
| 1145 | comptime assert(std.heap.page_size_min % @sizeOf(Block) == 0); | | |
| 1146 | assert(page_size % @sizeOf(Block) == 0); | | |
| 1147 | | | |
| 1148 | // First block may be unaligned | | |
| 1149 | const start_addr = @intFromPtr(&p[i]); | | |
| 1150 | const offset_in_page = start_addr & (page_size - 1); | | |
| 1151 | if (offset_in_page <= page_size - @sizeOf(Block)) { | | |
| 1152 | // Will not read past the end of a page, full block. | | |
| 1153 | const block: Block = p[i..][0..block_len].*; | | |
| 1154 | const matches = block == mask; | | |
| 1155 | if (@reduce(.Or, matches)) { | | |
| 1156 | return i + std.simd.firstTrue(matches).?; | | |
| 1157 | } | | |
| 1158 | | | |
| 1159 | i += @divExact(std.mem.alignForward(usize, start_addr, block_size) - start_addr, @sizeOf(T)); | | |
| 1160 | } else { | | |
| 1161 | @branchHint(.unlikely); | | |
| 1162 | // Would read over a page boundary. Per-byte at a time until aligned or found. | | |
| 1163 | // 0.39% chance this branch is taken for 4K pages at 16b block length. | | |
| 1164 | // | | |
| 1165 | // An alternate strategy is to do read a full block (the last in the page) and | | |
| 1166 | // mask the entries before the pointer. | | |
| 1167 | while ((@intFromPtr(&p[i]) & (block_size - 1)) != 0) : (i += 1) { | | |
| 1168 | if (p[i] == sentinel) return i; | | |
| 1169 | } | | |
| 1170 | } | | |
| 1171 | | | |
| 1172 | std.debug.assertAligned(&p[i], .fromByteUnits(block_size)); | | |
| 1173 | while (true) { | | |
| 1174 | const block: Block = p[i..][0..block_len].*; | | |
| 1175 | const matches = block == mask; | | |
| 1176 | if (@reduce(.Or, matches)) { | | |
| 1177 | return i + std.simd.firstTrue(matches).?; | | |
| 1178 | } | | |
| 1179 | i += block_len; | | |
| 1180 | } | | |
| 1181 | }, | | |
| 1182 | else => {}, | | |
| 1183 | } | | |
| 1184 | } | | |
| 1185 | | | |
| 1186 | while (p[i] != sentinel) { | 1129 | while (p[i] != sentinel) { |
| 1187 | i += 1; | 1130 | i += 1; |
| 1188 | } | 1131 | } |