| author |  Andrew Kelley <andrew@ziglang.org> 2025-09-21 17:26:00-07:00 |
| committer | Andrew Kelley <andrew@ziglang.org> 2025-10-02 16:30:59-07:00 |
| log | 0f2427c5d2e16626c6c88621e8aa3aa1b4aa8785 |
| tree | 9f5c876b04e539411411522bf3f89d85dcb9d232 |
| parent | 43eea7beecaeb29d048cf971d6d08d0297109ce0 |
std.Io: rename ThreadPool to Threaded
3 files changed, 1263 insertions(+), 1263 deletions(-)
lib
lib/std/Io.zig+3-3
| ... | ... | @@ -558,7 +558,7 @@ test { |
| 558 | 558 | const Io = @This(); |
| 559 | 559 | |
| 560 | 560 | pub const EventLoop = @import("Io/EventLoop.zig"); |
| 561 | pub const ThreadPool = @import("Io/ThreadPool.zig"); | |
| 561 | pub const Threaded = @import("Io/Threaded.zig"); | |
| 562 | 562 | pub const net = @import("Io/net.zig"); |
| 563 | 563 | |
| 564 | 564 | userdata: ?*anyopaque, |
| ... | ... | @@ -668,8 +668,8 @@ pub const VTable = struct { |
| 668 | 668 | netRead: *const fn (?*anyopaque, src: net.Stream, data: [][]u8) net.Stream.Reader.Error!usize, |
| 669 | 669 | netWrite: *const fn (?*anyopaque, dest: net.Stream, header: []const u8, data: []const []const u8, splat: usize) net.Stream.Writer.Error!usize, |
| 670 | 670 | netClose: *const fn (?*anyopaque, stream: net.Stream) void, |
| 671 | /// Equivalent to libc "if_nametoindex". | |
| 672 | netInterfaceIndex: *const fn (?*anyopaque, name: []const u8) net.InterfaceIndexError!u32, | |
| 671 | netInterfaceNameResolve: *const fn (?*anyopaque, net.Interface.Name) net.Interface.Name.ResolveError!net.Interface, | |
| 672 | netInterfaceName: *const fn (?*anyopaque, net.Interface) net.Interface.NameError!net.Interface.Name, | |
| 673 | 673 | }; |
| 674 | 674 | |
| 675 | 675 | pub const Cancelable = error{ |
lib/std/Io/ThreadPool.zig deleted-1260
| ... | ... | @@ -1,1260 +0,0 @@ |
| 1 | const Pool = @This(); | |
| 2 | ||
| 3 | const builtin = @import("builtin"); | |
| 4 | const native_os = builtin.os.tag; | |
| 5 | const is_windows = native_os == .windows; | |
| 6 | const windows = std.os.windows; | |
| 7 | ||
| 8 | const std = @import("../std.zig"); | |
| 9 | const Allocator = std.mem.Allocator; | |
| 10 | const assert = std.debug.assert; | |
| 11 | const WaitGroup = std.Thread.WaitGroup; | |
| 12 | const posix = std.posix; | |
| 13 | const Io = std.Io; | |
| 14 | ||
| 15 | /// Thread-safe. | |
| 16 | allocator: Allocator, | |
| 17 | mutex: std.Thread.Mutex = .{}, | |
| 18 | cond: std.Thread.Condition = .{}, | |
| 19 | run_queue: std.SinglyLinkedList = .{}, | |
| 20 | join_requested: bool = false, | |
| 21 | threads: std.ArrayListUnmanaged(std.Thread), | |
| 22 | stack_size: usize, | |
| 23 | cpu_count: std.Thread.CpuCountError!usize, | |
| 24 | parallel_count: usize, | |
| 25 | ||
| 26 | threadlocal var current_closure: ?*AsyncClosure = null; | |
| 27 | ||
| 28 | const max_iovecs_len = 8; | |
| 29 | const splat_buffer_size = 64; | |
| 30 | ||
| 31 | comptime { | |
| 32 | assert(max_iovecs_len <= posix.IOV_MAX); | |
| 33 | } | |
| 34 | ||
| 35 | pub const Runnable = struct { | |
| 36 | start: Start, | |
| 37 | node: std.SinglyLinkedList.Node = .{}, | |
| 38 | is_parallel: bool, | |
| 39 | ||
| 40 | pub const Start = *const fn (*Runnable) void; | |
| 41 | }; | |
| 42 | ||
| 43 | pub const InitError = std.Thread.CpuCountError || Allocator.Error; | |
| 44 | ||
| 45 | pub fn init(gpa: Allocator) Pool { | |
| 46 | var pool: Pool = .{ | |
| 47 | .allocator = gpa, | |
| 48 | .threads = .empty, | |
| 49 | .stack_size = std.Thread.SpawnConfig.default_stack_size, | |
| 50 | .cpu_count = std.Thread.getCpuCount(), | |
| 51 | .parallel_count = 0, | |
| 52 | }; | |
| 53 | if (pool.cpu_count) |n| { | |
| 54 | pool.threads.ensureTotalCapacityPrecise(gpa, n - 1) catch {}; | |
| 55 | } else |_| {} | |
| 56 | return pool; | |
| 57 | } | |
| 58 | ||
| 59 | pub fn deinit(pool: *Pool) void { | |
| 60 | const gpa = pool.allocator; | |
| 61 | pool.join(); | |
| 62 | pool.threads.deinit(gpa); | |
| 63 | pool.* = undefined; | |
| 64 | } | |
| 65 | ||
| 66 | fn join(pool: *Pool) void { | |
| 67 | if (builtin.single_threaded) return; | |
| 68 | { | |
| 69 | pool.mutex.lock(); | |
| 70 | defer pool.mutex.unlock(); | |
| 71 | pool.join_requested = true; | |
| 72 | } | |
| 73 | pool.cond.broadcast(); | |
| 74 | for (pool.threads.items) |thread| thread.join(); | |
| 75 | } | |
| 76 | ||
| 77 | fn worker(pool: *Pool) void { | |
| 78 | pool.mutex.lock(); | |
| 79 | defer pool.mutex.unlock(); | |
| 80 | ||
| 81 | while (true) { | |
| 82 | while (pool.run_queue.popFirst()) |run_node| { | |
| 83 | pool.mutex.unlock(); | |
| 84 | const runnable: *Runnable = @fieldParentPtr("node", run_node); | |
| 85 | runnable.start(runnable); | |
| 86 | pool.mutex.lock(); | |
| 87 | if (runnable.is_parallel) { | |
| 88 | // TODO also pop thread and join sometimes | |
| 89 | pool.parallel_count -= 1; | |
| 90 | } | |
| 91 | } | |
| 92 | if (pool.join_requested) break; | |
| 93 | pool.cond.wait(&pool.mutex); | |
| 94 | } | |
| 95 | } | |
| 96 | ||
| 97 | pub fn io(pool: *Pool) Io { | |
| 98 | return .{ | |
| 99 | .userdata = pool, | |
| 100 | .vtable = &.{ | |
| 101 | .async = async, | |
| 102 | .asyncConcurrent = asyncConcurrent, | |
| 103 | .await = await, | |
| 104 | .asyncDetached = asyncDetached, | |
| 105 | .cancel = cancel, | |
| 106 | .cancelRequested = cancelRequested, | |
| 107 | .select = select, | |
| 108 | ||
| 109 | .mutexLock = mutexLock, | |
| 110 | .mutexUnlock = mutexUnlock, | |
| 111 | ||
| 112 | .conditionWait = conditionWait, | |
| 113 | .conditionWake = conditionWake, | |
| 114 | ||
| 115 | .createFile = createFile, | |
| 116 | .fileOpen = fileOpen, | |
| 117 | .fileClose = fileClose, | |
| 118 | .pwrite = pwrite, | |
| 119 | .fileReadStreaming = fileReadStreaming, | |
| 120 | .fileReadPositional = fileReadPositional, | |
| 121 | .fileSeekBy = fileSeekBy, | |
| 122 | .fileSeekTo = fileSeekTo, | |
| 123 | ||
| 124 | .now = now, | |
| 125 | .sleep = sleep, | |
| 126 | ||
| 127 | .listen = listen, | |
| 128 | .accept = accept, | |
| 129 | .netRead = switch (builtin.os.tag) { | |
| 130 | .windows => @panic("TODO"), | |
| 131 | else => netReadPosix, | |
| 132 | }, | |
| 133 | .netWrite = switch (builtin.os.tag) { | |
| 134 | .windows => @panic("TODO"), | |
| 135 | else => netWritePosix, | |
| 136 | }, | |
| 137 | .netClose = netClose, | |
| 138 | .netInterfaceIndex = netInterfaceIndex, | |
| 139 | }, | |
| 140 | }; | |
| 141 | } | |
| 142 | ||
| 143 | const AsyncClosure = struct { | |
| 144 | func: *const fn (context: *anyopaque, result: *anyopaque) void, | |
| 145 | runnable: Runnable, | |
| 146 | reset_event: std.Thread.ResetEvent, | |
| 147 | select_condition: ?*std.Thread.ResetEvent, | |
| 148 | cancel_tid: std.Thread.Id, | |
| 149 | context_offset: usize, | |
| 150 | result_offset: usize, | |
| 151 | ||
| 152 | const done_reset_event: *std.Thread.ResetEvent = @ptrFromInt(@alignOf(std.Thread.ResetEvent)); | |
| 153 | ||
| 154 | const canceling_tid: std.Thread.Id = switch (@typeInfo(std.Thread.Id)) { | |
| 155 | .int => |int_info| switch (int_info.signedness) { | |
| 156 | .signed => -1, | |
| 157 | .unsigned => std.math.maxInt(std.Thread.Id), | |
| 158 | }, | |
| 159 | .pointer => @ptrFromInt(std.math.maxInt(usize)), | |
| 160 | else => @compileError("unsupported std.Thread.Id: " ++ @typeName(std.Thread.Id)), | |
| 161 | }; | |
| 162 | ||
| 163 | fn start(runnable: *Runnable) void { | |
| 164 | const closure: *AsyncClosure = @alignCast(@fieldParentPtr("runnable", runnable)); | |
| 165 | const tid = std.Thread.getCurrentId(); | |
| 166 | if (@cmpxchgStrong( | |
| 167 | std.Thread.Id, | |
| 168 | &closure.cancel_tid, | |
| 169 | 0, | |
| 170 | tid, | |
| 171 | .acq_rel, | |
| 172 | .acquire, | |
| 173 | )) |cancel_tid| { | |
| 174 | assert(cancel_tid == canceling_tid); | |
| 175 | closure.reset_event.set(); | |
| 176 | return; | |
| 177 | } | |
| 178 | current_closure = closure; | |
| 179 | closure.func(closure.contextPointer(), closure.resultPointer()); | |
| 180 | current_closure = null; | |
| 181 | if (@cmpxchgStrong( | |
| 182 | std.Thread.Id, | |
| 183 | &closure.cancel_tid, | |
| 184 | tid, | |
| 185 | 0, | |
| 186 | .acq_rel, | |
| 187 | .acquire, | |
| 188 | )) |cancel_tid| assert(cancel_tid == canceling_tid); | |
| 189 | ||
| 190 | if (@atomicRmw( | |
| 191 | ?*std.Thread.ResetEvent, | |
| 192 | &closure.select_condition, | |
| 193 | .Xchg, | |
| 194 | done_reset_event, | |
| 195 | .release, | |
| 196 | )) |select_reset| { | |
| 197 | assert(select_reset != done_reset_event); | |
| 198 | select_reset.set(); | |
| 199 | } | |
| 200 | closure.reset_event.set(); | |
| 201 | } | |
| 202 | ||
| 203 | fn contextOffset(context_alignment: std.mem.Alignment) usize { | |
| 204 | return context_alignment.forward(@sizeOf(AsyncClosure)); | |
| 205 | } | |
| 206 | ||
| 207 | fn resultOffset( | |
| 208 | context_alignment: std.mem.Alignment, | |
| 209 | context_len: usize, | |
| 210 | result_alignment: std.mem.Alignment, | |
| 211 | ) usize { | |
| 212 | return result_alignment.forward(contextOffset(context_alignment) + context_len); | |
| 213 | } | |
| 214 | ||
| 215 | fn resultPointer(closure: *AsyncClosure) [*]u8 { | |
| 216 | const base: [*]u8 = @ptrCast(closure); | |
| 217 | return base + closure.result_offset; | |
| 218 | } | |
| 219 | ||
| 220 | fn contextPointer(closure: *AsyncClosure) [*]u8 { | |
| 221 | const base: [*]u8 = @ptrCast(closure); | |
| 222 | return base + closure.context_offset; | |
| 223 | } | |
| 224 | ||
| 225 | fn waitAndFree(closure: *AsyncClosure, gpa: Allocator, result: []u8) void { | |
| 226 | closure.reset_event.wait(); | |
| 227 | @memcpy(result, closure.resultPointer()[0..result.len]); | |
| 228 | free(closure, gpa, result.len); | |
| 229 | } | |
| 230 | ||
| 231 | fn free(closure: *AsyncClosure, gpa: Allocator, result_len: usize) void { | |
| 232 | const base: [*]align(@alignOf(AsyncClosure)) u8 = @ptrCast(closure); | |
| 233 | gpa.free(base[0 .. closure.result_offset + result_len]); | |
| 234 | } | |
| 235 | }; | |
| 236 | ||
| 237 | fn async( | |
| 238 | userdata: ?*anyopaque, | |
| 239 | result: []u8, | |
| 240 | result_alignment: std.mem.Alignment, | |
| 241 | context: []const u8, | |
| 242 | context_alignment: std.mem.Alignment, | |
| 243 | start: *const fn (context: *const anyopaque, result: *anyopaque) void, | |
| 244 | ) ?*Io.AnyFuture { | |
| 245 | if (builtin.single_threaded) { | |
| 246 | start(context.ptr, result.ptr); | |
| 247 | return null; | |
| 248 | } | |
| 249 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 250 | const cpu_count = pool.cpu_count catch { | |
| 251 | return asyncConcurrent(userdata, result.len, result_alignment, context, context_alignment, start) catch { | |
| 252 | start(context.ptr, result.ptr); | |
| 253 | return null; | |
| 254 | }; | |
| 255 | }; | |
| 256 | const gpa = pool.allocator; | |
| 257 | const context_offset = context_alignment.forward(@sizeOf(AsyncClosure)); | |
| 258 | const result_offset = result_alignment.forward(context_offset + context.len); | |
| 259 | const n = result_offset + result.len; | |
| 260 | const closure: *AsyncClosure = @ptrCast(@alignCast(gpa.alignedAlloc(u8, .of(AsyncClosure), n) catch { | |
| 261 | start(context.ptr, result.ptr); | |
| 262 | return null; | |
| 263 | })); | |
| 264 | ||
| 265 | closure.* = .{ | |
| 266 | .func = start, | |
| 267 | .context_offset = context_offset, | |
| 268 | .result_offset = result_offset, | |
| 269 | .reset_event = .{}, | |
| 270 | .cancel_tid = 0, | |
| 271 | .select_condition = null, | |
| 272 | .runnable = .{ | |
| 273 | .start = AsyncClosure.start, | |
| 274 | .is_parallel = false, | |
| 275 | }, | |
| 276 | }; | |
| 277 | ||
| 278 | @memcpy(closure.contextPointer()[0..context.len], context); | |
| 279 | ||
| 280 | pool.mutex.lock(); | |
| 281 | ||
| 282 | const thread_capacity = cpu_count - 1 + pool.parallel_count; | |
| 283 | ||
| 284 | pool.threads.ensureTotalCapacityPrecise(gpa, thread_capacity) catch { | |
| 285 | pool.mutex.unlock(); | |
| 286 | closure.free(gpa, result.len); | |
| 287 | start(context.ptr, result.ptr); | |
| 288 | return null; | |
| 289 | }; | |
| 290 | ||
| 291 | pool.run_queue.prepend(&closure.runnable.node); | |
| 292 | ||
| 293 | if (pool.threads.items.len < thread_capacity) { | |
| 294 | const thread = std.Thread.spawn(.{ .stack_size = pool.stack_size }, worker, .{pool}) catch { | |
| 295 | if (pool.threads.items.len == 0) { | |
| 296 | assert(pool.run_queue.popFirst() == &closure.runnable.node); | |
| 297 | pool.mutex.unlock(); | |
| 298 | closure.free(gpa, result.len); | |
| 299 | start(context.ptr, result.ptr); | |
| 300 | return null; | |
| 301 | } | |
| 302 | // Rely on other workers to do it. | |
| 303 | pool.mutex.unlock(); | |
| 304 | pool.cond.signal(); | |
| 305 | return @ptrCast(closure); | |
| 306 | }; | |
| 307 | pool.threads.appendAssumeCapacity(thread); | |
| 308 | } | |
| 309 | ||
| 310 | pool.mutex.unlock(); | |
| 311 | pool.cond.signal(); | |
| 312 | return @ptrCast(closure); | |
| 313 | } | |
| 314 | ||
| 315 | fn asyncConcurrent( | |
| 316 | userdata: ?*anyopaque, | |
| 317 | result_len: usize, | |
| 318 | result_alignment: std.mem.Alignment, | |
| 319 | context: []const u8, | |
| 320 | context_alignment: std.mem.Alignment, | |
| 321 | start: *const fn (context: *const anyopaque, result: *anyopaque) void, | |
| 322 | ) error{OutOfMemory}!*Io.AnyFuture { | |
| 323 | if (builtin.single_threaded) unreachable; | |
| 324 | ||
| 325 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 326 | const cpu_count = pool.cpu_count catch 1; | |
| 327 | const gpa = pool.allocator; | |
| 328 | const context_offset = context_alignment.forward(@sizeOf(AsyncClosure)); | |
| 329 | const result_offset = result_alignment.forward(context_offset + context.len); | |
| 330 | const n = result_offset + result_len; | |
| 331 | const closure: *AsyncClosure = @ptrCast(@alignCast(try gpa.alignedAlloc(u8, .of(AsyncClosure), n))); | |
| 332 | ||
| 333 | closure.* = .{ | |
| 334 | .func = start, | |
| 335 | .context_offset = context_offset, | |
| 336 | .result_offset = result_offset, | |
| 337 | .reset_event = .{}, | |
| 338 | .cancel_tid = 0, | |
| 339 | .select_condition = null, | |
| 340 | .runnable = .{ | |
| 341 | .start = AsyncClosure.start, | |
| 342 | .is_parallel = true, | |
| 343 | }, | |
| 344 | }; | |
| 345 | @memcpy(closure.contextPointer()[0..context.len], context); | |
| 346 | ||
| 347 | pool.mutex.lock(); | |
| 348 | ||
| 349 | pool.parallel_count += 1; | |
| 350 | const thread_capacity = cpu_count - 1 + pool.parallel_count; | |
| 351 | ||
| 352 | pool.threads.ensureTotalCapacity(gpa, thread_capacity) catch { | |
| 353 | pool.mutex.unlock(); | |
| 354 | closure.free(gpa, result_len); | |
| 355 | return error.OutOfMemory; | |
| 356 | }; | |
| 357 | ||
| 358 | pool.run_queue.prepend(&closure.runnable.node); | |
| 359 | ||
| 360 | if (pool.threads.items.len < thread_capacity) { | |
| 361 | const thread = std.Thread.spawn(.{ .stack_size = pool.stack_size }, worker, .{pool}) catch { | |
| 362 | assert(pool.run_queue.popFirst() == &closure.runnable.node); | |
| 363 | pool.mutex.unlock(); | |
| 364 | closure.free(gpa, result_len); | |
| 365 | return error.OutOfMemory; | |
| 366 | }; | |
| 367 | pool.threads.appendAssumeCapacity(thread); | |
| 368 | } | |
| 369 | ||
| 370 | pool.mutex.unlock(); | |
| 371 | pool.cond.signal(); | |
| 372 | return @ptrCast(closure); | |
| 373 | } | |
| 374 | ||
| 375 | const DetachedClosure = struct { | |
| 376 | pool: *Pool, | |
| 377 | func: *const fn (context: *anyopaque) void, | |
| 378 | runnable: Runnable, | |
| 379 | context_alignment: std.mem.Alignment, | |
| 380 | context_len: usize, | |
| 381 | ||
| 382 | fn start(runnable: *Runnable) void { | |
| 383 | const closure: *DetachedClosure = @alignCast(@fieldParentPtr("runnable", runnable)); | |
| 384 | closure.func(closure.contextPointer()); | |
| 385 | const gpa = closure.pool.allocator; | |
| 386 | free(closure, gpa); | |
| 387 | } | |
| 388 | ||
| 389 | fn free(closure: *DetachedClosure, gpa: Allocator) void { | |
| 390 | const base: [*]align(@alignOf(DetachedClosure)) u8 = @ptrCast(closure); | |
| 391 | gpa.free(base[0..contextEnd(closure.context_alignment, closure.context_len)]); | |
| 392 | } | |
| 393 | ||
| 394 | fn contextOffset(context_alignment: std.mem.Alignment) usize { | |
| 395 | return context_alignment.forward(@sizeOf(DetachedClosure)); | |
| 396 | } | |
| 397 | ||
| 398 | fn contextEnd(context_alignment: std.mem.Alignment, context_len: usize) usize { | |
| 399 | return contextOffset(context_alignment) + context_len; | |
| 400 | } | |
| 401 | ||
| 402 | fn contextPointer(closure: *DetachedClosure) [*]u8 { | |
| 403 | const base: [*]u8 = @ptrCast(closure); | |
| 404 | return base + contextOffset(closure.context_alignment); | |
| 405 | } | |
| 406 | }; | |
| 407 | ||
| 408 | fn asyncDetached( | |
| 409 | userdata: ?*anyopaque, | |
| 410 | context: []const u8, | |
| 411 | context_alignment: std.mem.Alignment, | |
| 412 | start: *const fn (context: *const anyopaque) void, | |
| 413 | ) void { | |
| 414 | if (builtin.single_threaded) return start(context.ptr); | |
| 415 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 416 | const cpu_count = pool.cpu_count catch 1; | |
| 417 | const gpa = pool.allocator; | |
| 418 | const n = DetachedClosure.contextEnd(context_alignment, context.len); | |
| 419 | const closure: *DetachedClosure = @ptrCast(@alignCast(gpa.alignedAlloc(u8, .of(DetachedClosure), n) catch { | |
| 420 | return start(context.ptr); | |
| 421 | })); | |
| 422 | closure.* = .{ | |
| 423 | .pool = pool, | |
| 424 | .func = start, | |
| 425 | .context_alignment = context_alignment, | |
| 426 | .context_len = context.len, | |
| 427 | .runnable = .{ | |
| 428 | .start = DetachedClosure.start, | |
| 429 | .is_parallel = false, | |
| 430 | }, | |
| 431 | }; | |
| 432 | @memcpy(closure.contextPointer()[0..context.len], context); | |
| 433 | ||
| 434 | pool.mutex.lock(); | |
| 435 | ||
| 436 | const thread_capacity = cpu_count - 1 + pool.parallel_count; | |
| 437 | ||
| 438 | pool.threads.ensureTotalCapacityPrecise(gpa, thread_capacity) catch { | |
| 439 | pool.mutex.unlock(); | |
| 440 | closure.free(gpa); | |
| 441 | return start(context.ptr); | |
| 442 | }; | |
| 443 | ||
| 444 | pool.run_queue.prepend(&closure.runnable.node); | |
| 445 | ||
| 446 | if (pool.threads.items.len < thread_capacity) { | |
| 447 | const thread = std.Thread.spawn(.{ .stack_size = pool.stack_size }, worker, .{pool}) catch { | |
| 448 | assert(pool.run_queue.popFirst() == &closure.runnable.node); | |
| 449 | pool.mutex.unlock(); | |
| 450 | closure.free(gpa); | |
| 451 | return start(context.ptr); | |
| 452 | }; | |
| 453 | pool.threads.appendAssumeCapacity(thread); | |
| 454 | } | |
| 455 | ||
| 456 | pool.mutex.unlock(); | |
| 457 | pool.cond.signal(); | |
| 458 | } | |
| 459 | ||
| 460 | fn await( | |
| 461 | userdata: ?*anyopaque, | |
| 462 | any_future: *std.Io.AnyFuture, | |
| 463 | result: []u8, | |
| 464 | result_alignment: std.mem.Alignment, | |
| 465 | ) void { | |
| 466 | _ = result_alignment; | |
| 467 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 468 | const closure: *AsyncClosure = @ptrCast(@alignCast(any_future)); | |
| 469 | closure.waitAndFree(pool.allocator, result); | |
| 470 | } | |
| 471 | ||
| 472 | fn cancel( | |
| 473 | userdata: ?*anyopaque, | |
| 474 | any_future: *Io.AnyFuture, | |
| 475 | result: []u8, | |
| 476 | result_alignment: std.mem.Alignment, | |
| 477 | ) void { | |
| 478 | _ = result_alignment; | |
| 479 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 480 | const closure: *AsyncClosure = @ptrCast(@alignCast(any_future)); | |
| 481 | switch (@atomicRmw( | |
| 482 | std.Thread.Id, | |
| 483 | &closure.cancel_tid, | |
| 484 | .Xchg, | |
| 485 | AsyncClosure.canceling_tid, | |
| 486 | .acq_rel, | |
| 487 | )) { | |
| 488 | 0, AsyncClosure.canceling_tid => {}, | |
| 489 | else => |cancel_tid| switch (builtin.os.tag) { | |
| 490 | .linux => _ = std.os.linux.tgkill( | |
| 491 | std.os.linux.getpid(), | |
| 492 | @bitCast(cancel_tid), | |
| 493 | posix.SIG.IO, | |
| 494 | ), | |
| 495 | else => {}, | |
| 496 | }, | |
| 497 | } | |
| 498 | closure.waitAndFree(pool.allocator, result); | |
| 499 | } | |
| 500 | ||
| 501 | fn cancelRequested(userdata: ?*anyopaque) bool { | |
| 502 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 503 | _ = pool; | |
| 504 | const closure = current_closure orelse return false; | |
| 505 | return @atomicLoad(std.Thread.Id, &closure.cancel_tid, .acquire) == AsyncClosure.canceling_tid; | |
| 506 | } | |
| 507 | ||
| 508 | fn checkCancel(pool: *Pool) error{Canceled}!void { | |
| 509 | if (cancelRequested(pool)) return error.Canceled; | |
| 510 | } | |
| 511 | ||
| 512 | fn mutexLock(userdata: ?*anyopaque, prev_state: Io.Mutex.State, mutex: *Io.Mutex) error{Canceled}!void { | |
| 513 | _ = userdata; | |
| 514 | if (prev_state == .contended) { | |
| 515 | std.Thread.Futex.wait(@ptrCast(&mutex.state), @intFromEnum(Io.Mutex.State.contended)); | |
| 516 | } | |
| 517 | while (@atomicRmw( | |
| 518 | Io.Mutex.State, | |
| 519 | &mutex.state, | |
| 520 | .Xchg, | |
| 521 | .contended, | |
| 522 | .acquire, | |
| 523 | ) != .unlocked) { | |
| 524 | std.Thread.Futex.wait(@ptrCast(&mutex.state), @intFromEnum(Io.Mutex.State.contended)); | |
| 525 | } | |
| 526 | } | |
| 527 | fn mutexUnlock(userdata: ?*anyopaque, prev_state: Io.Mutex.State, mutex: *Io.Mutex) void { | |
| 528 | _ = userdata; | |
| 529 | _ = prev_state; | |
| 530 | if (@atomicRmw(Io.Mutex.State, &mutex.state, .Xchg, .unlocked, .release) == .contended) { | |
| 531 | std.Thread.Futex.wake(@ptrCast(&mutex.state), 1); | |
| 532 | } | |
| 533 | } | |
| 534 | ||
| 535 | fn conditionWait(userdata: ?*anyopaque, cond: *Io.Condition, mutex: *Io.Mutex) Io.Cancelable!void { | |
| 536 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 537 | comptime assert(@TypeOf(cond.state) == u64); | |
| 538 | const ints: *[2]std.atomic.Value(u32) = @ptrCast(&cond.state); | |
| 539 | const cond_state = &ints[0]; | |
| 540 | const cond_epoch = &ints[1]; | |
| 541 | const one_waiter = 1; | |
| 542 | const waiter_mask = 0xffff; | |
| 543 | const one_signal = 1 << 16; | |
| 544 | const signal_mask = 0xffff << 16; | |
| 545 | // Observe the epoch, then check the state again to see if we should wake up. | |
| 546 | // The epoch must be observed before we check the state or we could potentially miss a wake() and deadlock: | |
| 547 | // | |
| 548 | // - T1: s = LOAD(&state) | |
| 549 | // - T2: UPDATE(&s, signal) | |
| 550 | // - T2: UPDATE(&epoch, 1) + FUTEX_WAKE(&epoch) | |
| 551 | // - T1: e = LOAD(&epoch) (was reordered after the state load) | |
| 552 | // - T1: s & signals == 0 -> FUTEX_WAIT(&epoch, e) (missed the state update + the epoch change) | |
| 553 | // | |
| 554 | // Acquire barrier to ensure the epoch load happens before the state load. | |
| 555 | var epoch = cond_epoch.load(.acquire); | |
| 556 | var state = cond_state.fetchAdd(one_waiter, .monotonic); | |
| 557 | assert(state & waiter_mask != waiter_mask); | |
| 558 | state += one_waiter; | |
| 559 | ||
| 560 | mutex.unlock(pool.io()); | |
| 561 | defer mutex.lock(pool.io()) catch @panic("TODO"); | |
| 562 | ||
| 563 | var futex_deadline = std.Thread.Futex.Deadline.init(null); | |
| 564 | ||
| 565 | while (true) { | |
| 566 | futex_deadline.wait(cond_epoch, epoch) catch |err| switch (err) { | |
| 567 | error.Timeout => unreachable, | |
| 568 | }; | |
| 569 | ||
| 570 | epoch = cond_epoch.load(.acquire); | |
| 571 | state = cond_state.load(.monotonic); | |
| 572 | ||
| 573 | // Try to wake up by consuming a signal and decremented the waiter we added previously. | |
| 574 | // Acquire barrier ensures code before the wake() which added the signal happens before we decrement it and return. | |
| 575 | while (state & signal_mask != 0) { | |
| 576 | const new_state = state - one_waiter - one_signal; | |
| 577 | state = cond_state.cmpxchgWeak(state, new_state, .acquire, .monotonic) orelse return; | |
| 578 | } | |
| 579 | } | |
| 580 | } | |
| 581 | ||
| 582 | fn conditionWake(userdata: ?*anyopaque, cond: *Io.Condition, wake: Io.Condition.Wake) void { | |
| 583 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 584 | _ = pool; | |
| 585 | comptime assert(@TypeOf(cond.state) == u64); | |
| 586 | const ints: *[2]std.atomic.Value(u32) = @ptrCast(&cond.state); | |
| 587 | const cond_state = &ints[0]; | |
| 588 | const cond_epoch = &ints[1]; | |
| 589 | const one_waiter = 1; | |
| 590 | const waiter_mask = 0xffff; | |
| 591 | const one_signal = 1 << 16; | |
| 592 | const signal_mask = 0xffff << 16; | |
| 593 | var state = cond_state.load(.monotonic); | |
| 594 | while (true) { | |
| 595 | const waiters = (state & waiter_mask) / one_waiter; | |
| 596 | const signals = (state & signal_mask) / one_signal; | |
| 597 | ||
| 598 | // Reserves which waiters to wake up by incrementing the signals count. | |
| 599 | // Therefore, the signals count is always less than or equal to the waiters count. | |
| 600 | // We don't need to Futex.wake if there's nothing to wake up or if other wake() threads have reserved to wake up the current waiters. | |
| 601 | const wakeable = waiters - signals; | |
| 602 | if (wakeable == 0) { | |
| 603 | return; | |
| 604 | } | |
| 605 | ||
| 606 | const to_wake = switch (wake) { | |
| 607 | .one => 1, | |
| 608 | .all => wakeable, | |
| 609 | }; | |
| 610 | ||
| 611 | // Reserve the amount of waiters to wake by incrementing the signals count. | |
| 612 | // Release barrier ensures code before the wake() happens before the signal it posted and consumed by the wait() threads. | |
| 613 | const new_state = state + (one_signal * to_wake); | |
| 614 | state = cond_state.cmpxchgWeak(state, new_state, .release, .monotonic) orelse { | |
| 615 | // Wake up the waiting threads we reserved above by changing the epoch value. | |
| 616 | // NOTE: a waiting thread could miss a wake up if *exactly* ((1<<32)-1) wake()s happen between it observing the epoch and sleeping on it. | |
| 617 | // This is very unlikely due to how many precise amount of Futex.wake() calls that would be between the waiting thread's potential preemption. | |
| 618 | // | |
| 619 | // Release barrier ensures the signal being added to the state happens before the epoch is changed. | |
| 620 | // If not, the waiting thread could potentially deadlock from missing both the state and epoch change: | |
| 621 | // | |
| 622 | // - T2: UPDATE(&epoch, 1) (reordered before the state change) | |
| 623 | // - T1: e = LOAD(&epoch) | |
| 624 | // - T1: s = LOAD(&state) | |
| 625 | // - T2: UPDATE(&state, signal) + FUTEX_WAKE(&epoch) | |
| 626 | // - T1: s & signals == 0 -> FUTEX_WAIT(&epoch, e) (missed both epoch change and state change) | |
| 627 | _ = cond_epoch.fetchAdd(1, .release); | |
| 628 | std.Thread.Futex.wake(cond_epoch, to_wake); | |
| 629 | return; | |
| 630 | }; | |
| 631 | } | |
| 632 | } | |
| 633 | ||
| 634 | fn createFile( | |
| 635 | userdata: ?*anyopaque, | |
| 636 | dir: Io.Dir, | |
| 637 | sub_path: []const u8, | |
| 638 | flags: Io.File.CreateFlags, | |
| 639 | ) Io.File.OpenError!Io.File { | |
| 640 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 641 | try pool.checkCancel(); | |
| 642 | const fs_dir: std.fs.Dir = .{ .fd = dir.handle }; | |
| 643 | const fs_file = try fs_dir.createFile(sub_path, flags); | |
| 644 | return .{ .handle = fs_file.handle }; | |
| 645 | } | |
| 646 | ||
| 647 | fn fileOpen( | |
| 648 | userdata: ?*anyopaque, | |
| 649 | dir: Io.Dir, | |
| 650 | sub_path: []const u8, | |
| 651 | flags: Io.File.OpenFlags, | |
| 652 | ) Io.File.OpenError!Io.File { | |
| 653 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 654 | try pool.checkCancel(); | |
| 655 | const fs_dir: std.fs.Dir = .{ .fd = dir.handle }; | |
| 656 | const fs_file = try fs_dir.openFile(sub_path, flags); | |
| 657 | return .{ .handle = fs_file.handle }; | |
| 658 | } | |
| 659 | ||
| 660 | fn fileClose(userdata: ?*anyopaque, file: Io.File) void { | |
| 661 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 662 | _ = pool; | |
| 663 | const fs_file: std.fs.File = .{ .handle = file.handle }; | |
| 664 | return fs_file.close(); | |
| 665 | } | |
| 666 | ||
| 667 | fn fileReadStreaming(userdata: ?*anyopaque, file: Io.File, data: [][]u8) Io.File.ReadStreamingError!usize { | |
| 668 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 669 | ||
| 670 | if (is_windows) { | |
| 671 | const DWORD = windows.DWORD; | |
| 672 | var index: usize = 0; | |
| 673 | var truncate: usize = 0; | |
| 674 | var total: usize = 0; | |
| 675 | while (index < data.len) { | |
| 676 | try pool.checkCancel(); | |
| 677 | { | |
| 678 | const untruncated = data[index]; | |
| 679 | data[index] = untruncated[truncate..]; | |
| 680 | defer data[index] = untruncated; | |
| 681 | const buffer = data[index..]; | |
| 682 | const want_read_count: DWORD = @min(std.math.maxInt(DWORD), buffer.len); | |
| 683 | var n: DWORD = undefined; | |
| 684 | if (windows.kernel32.ReadFile(file.handle, buffer.ptr, want_read_count, &n, null) == 0) { | |
| 685 | switch (windows.GetLastError()) { | |
| 686 | .IO_PENDING => unreachable, | |
| 687 | .OPERATION_ABORTED => continue, | |
| 688 | .BROKEN_PIPE => return 0, | |
| 689 | .HANDLE_EOF => return 0, | |
| 690 | .NETNAME_DELETED => return error.ConnectionResetByPeer, | |
| 691 | .LOCK_VIOLATION => return error.LockViolation, | |
| 692 | .ACCESS_DENIED => return error.AccessDenied, | |
| 693 | .INVALID_HANDLE => return error.NotOpenForReading, | |
| 694 | else => |err| return windows.unexpectedError(err), | |
| 695 | } | |
| 696 | } | |
| 697 | total += n; | |
| 698 | truncate += n; | |
| 699 | } | |
| 700 | while (index < data.len and truncate >= data[index].len) { | |
| 701 | truncate -= data[index].len; | |
| 702 | index += 1; | |
| 703 | } | |
| 704 | } | |
| 705 | return total; | |
| 706 | } | |
| 707 | ||
| 708 | var iovecs_buffer: [max_iovecs_len]posix.iovec = undefined; | |
| 709 | var i: usize = 0; | |
| 710 | for (data) |buf| { | |
| 711 | if (iovecs_buffer.len - i == 0) break; | |
| 712 | if (buf.len != 0) { | |
| 713 | iovecs_buffer[i] = .{ .base = buf.ptr, .len = buf.len }; | |
| 714 | i += 1; | |
| 715 | } | |
| 716 | } | |
| 717 | const dest = iovecs_buffer[0..i]; | |
| 718 | assert(dest[0].len > 0); | |
| 719 | ||
| 720 | if (native_os == .wasi and !builtin.link_libc) { | |
| 721 | try pool.checkCancel(); | |
| 722 | var nread: usize = undefined; | |
| 723 | switch (std.os.wasi.fd_read(file.handle, dest.ptr, dest.len, &nread)) { | |
| 724 | .SUCCESS => return nread, | |
| 725 | .INTR => unreachable, | |
| 726 | .INVAL => unreachable, | |
| 727 | .FAULT => unreachable, | |
| 728 | .AGAIN => unreachable, // currently not support in WASI | |
| 729 | .BADF => return error.NotOpenForReading, // can be a race condition | |
| 730 | .IO => return error.InputOutput, | |
| 731 | .ISDIR => return error.IsDir, | |
| 732 | .NOBUFS => return error.SystemResources, | |
| 733 | .NOMEM => return error.SystemResources, | |
| 734 | .NOTCONN => return error.SocketNotConnected, | |
| 735 | .CONNRESET => return error.ConnectionResetByPeer, | |
| 736 | .TIMEDOUT => return error.ConnectionTimedOut, | |
| 737 | .NOTCAPABLE => return error.AccessDenied, | |
| 738 | else => |err| return posix.unexpectedErrno(err), | |
| 739 | } | |
| 740 | } | |
| 741 | ||
| 742 | while (true) { | |
| 743 | try pool.checkCancel(); | |
| 744 | const rc = posix.system.readv(file.handle, dest.ptr, dest.len); | |
| 745 | switch (posix.errno(rc)) { | |
| 746 | .SUCCESS => return @intCast(rc), | |
| 747 | .INTR => continue, | |
| 748 | .INVAL => unreachable, | |
| 749 | .FAULT => unreachable, | |
| 750 | .SRCH => return error.ProcessNotFound, | |
| 751 | .AGAIN => return error.WouldBlock, | |
| 752 | .BADF => return error.NotOpenForReading, // can be a race condition | |
| 753 | .IO => return error.InputOutput, | |
| 754 | .ISDIR => return error.IsDir, | |
| 755 | .NOBUFS => return error.SystemResources, | |
| 756 | .NOMEM => return error.SystemResources, | |
| 757 | .NOTCONN => return error.SocketNotConnected, | |
| 758 | .CONNRESET => return error.ConnectionResetByPeer, | |
| 759 | .TIMEDOUT => return error.ConnectionTimedOut, | |
| 760 | else => |err| return posix.unexpectedErrno(err), | |
| 761 | } | |
| 762 | } | |
| 763 | } | |
| 764 | ||
| 765 | fn fileReadPositional(userdata: ?*anyopaque, file: Io.File, data: [][]u8, offset: u64) Io.File.ReadPositionalError!usize { | |
| 766 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 767 | ||
| 768 | const have_pread_but_not_preadv = switch (native_os) { | |
| 769 | .windows, .macos, .ios, .watchos, .tvos, .visionos, .haiku, .serenity => true, | |
| 770 | else => false, | |
| 771 | }; | |
| 772 | if (have_pread_but_not_preadv) { | |
| 773 | @compileError("TODO"); | |
| 774 | } | |
| 775 | ||
| 776 | if (is_windows) { | |
| 777 | const DWORD = windows.DWORD; | |
| 778 | const OVERLAPPED = windows.OVERLAPPED; | |
| 779 | var index: usize = 0; | |
| 780 | var truncate: usize = 0; | |
| 781 | var total: usize = 0; | |
| 782 | while (true) { | |
| 783 | try pool.checkCancel(); | |
| 784 | { | |
| 785 | const untruncated = data[index]; | |
| 786 | data[index] = untruncated[truncate..]; | |
| 787 | defer data[index] = untruncated; | |
| 788 | const buffer = data[index..]; | |
| 789 | const want_read_count: DWORD = @min(std.math.maxInt(DWORD), buffer.len); | |
| 790 | var n: DWORD = undefined; | |
| 791 | var overlapped_data: OVERLAPPED = undefined; | |
| 792 | const overlapped: ?*OVERLAPPED = if (offset) |off| blk: { | |
| 793 | overlapped_data = .{ | |
| 794 | .Internal = 0, | |
| 795 | .InternalHigh = 0, | |
| 796 | .DUMMYUNIONNAME = .{ | |
| 797 | .DUMMYSTRUCTNAME = .{ | |
| 798 | .Offset = @as(u32, @truncate(off)), | |
| 799 | .OffsetHigh = @as(u32, @truncate(off >> 32)), | |
| 800 | }, | |
| 801 | }, | |
| 802 | .hEvent = null, | |
| 803 | }; | |
| 804 | break :blk &overlapped_data; | |
| 805 | } else null; | |
| 806 | if (windows.kernel32.ReadFile(file.handle, buffer.ptr, want_read_count, &n, overlapped) == 0) { | |
| 807 | switch (windows.GetLastError()) { | |
| 808 | .IO_PENDING => unreachable, | |
| 809 | .OPERATION_ABORTED => continue, | |
| 810 | .BROKEN_PIPE => return 0, | |
| 811 | .HANDLE_EOF => return 0, | |
| 812 | .NETNAME_DELETED => return error.ConnectionResetByPeer, | |
| 813 | .LOCK_VIOLATION => return error.LockViolation, | |
| 814 | .ACCESS_DENIED => return error.AccessDenied, | |
| 815 | .INVALID_HANDLE => return error.NotOpenForReading, | |
| 816 | else => |err| return windows.unexpectedError(err), | |
| 817 | } | |
| 818 | } | |
| 819 | total += n; | |
| 820 | truncate += n; | |
| 821 | } | |
| 822 | while (index < data.len and truncate >= data[index].len) { | |
| 823 | truncate -= data[index].len; | |
| 824 | index += 1; | |
| 825 | } | |
| 826 | } | |
| 827 | return total; | |
| 828 | } | |
| 829 | ||
| 830 | var iovecs_buffer: [max_iovecs_len]posix.iovec = undefined; | |
| 831 | var i: usize = 0; | |
| 832 | for (data) |buf| { | |
| 833 | if (iovecs_buffer.len - i == 0) break; | |
| 834 | if (buf.len != 0) { | |
| 835 | iovecs_buffer[i] = .{ .base = buf.ptr, .len = buf.len }; | |
| 836 | i += 1; | |
| 837 | } | |
| 838 | } | |
| 839 | const dest = iovecs_buffer[0..i]; | |
| 840 | assert(dest[0].len > 0); | |
| 841 | ||
| 842 | if (native_os == .wasi and !builtin.link_libc) { | |
| 843 | try pool.checkCancel(); | |
| 844 | var nread: usize = undefined; | |
| 845 | switch (std.os.wasi.fd_pread(file.handle, dest.ptr, dest.len, offset, &nread)) { | |
| 846 | .SUCCESS => return nread, | |
| 847 | .INTR => unreachable, | |
| 848 | .INVAL => unreachable, | |
| 849 | .FAULT => unreachable, | |
| 850 | .AGAIN => unreachable, | |
| 851 | .BADF => return error.NotOpenForReading, // can be a race condition | |
| 852 | .IO => return error.InputOutput, | |
| 853 | .ISDIR => return error.IsDir, | |
| 854 | .NOBUFS => return error.SystemResources, | |
| 855 | .NOMEM => return error.SystemResources, | |
| 856 | .NOTCONN => return error.SocketNotConnected, | |
| 857 | .CONNRESET => return error.ConnectionResetByPeer, | |
| 858 | .TIMEDOUT => return error.ConnectionTimedOut, | |
| 859 | .NXIO => return error.Unseekable, | |
| 860 | .SPIPE => return error.Unseekable, | |
| 861 | .OVERFLOW => return error.Unseekable, | |
| 862 | .NOTCAPABLE => return error.AccessDenied, | |
| 863 | else => |err| return posix.unexpectedErrno(err), | |
| 864 | } | |
| 865 | } | |
| 866 | ||
| 867 | const preadv_sym = if (posix.lfs64_abi) posix.system.preadv64 else posix.system.preadv; | |
| 868 | while (true) { | |
| 869 | try pool.checkCancel(); | |
| 870 | const rc = preadv_sym(file.handle, dest.ptr, dest.len, @bitCast(offset)); | |
| 871 | switch (posix.errno(rc)) { | |
| 872 | .SUCCESS => return @bitCast(rc), | |
| 873 | .INTR => continue, | |
| 874 | .INVAL => unreachable, | |
| 875 | .FAULT => unreachable, | |
| 876 | .SRCH => return error.ProcessNotFound, | |
| 877 | .AGAIN => return error.WouldBlock, | |
| 878 | .BADF => return error.NotOpenForReading, // can be a race condition | |
| 879 | .IO => return error.InputOutput, | |
| 880 | .ISDIR => return error.IsDir, | |
| 881 | .NOBUFS => return error.SystemResources, | |
| 882 | .NOMEM => return error.SystemResources, | |
| 883 | .NOTCONN => return error.SocketNotConnected, | |
| 884 | .CONNRESET => return error.ConnectionResetByPeer, | |
| 885 | .TIMEDOUT => return error.ConnectionTimedOut, | |
| 886 | .NXIO => return error.Unseekable, | |
| 887 | .SPIPE => return error.Unseekable, | |
| 888 | .OVERFLOW => return error.Unseekable, | |
| 889 | else => |err| return posix.unexpectedErrno(err), | |
| 890 | } | |
| 891 | } | |
| 892 | } | |
| 893 | ||
| 894 | fn fileSeekBy(userdata: ?*anyopaque, file: Io.File, offset: i64) Io.File.SeekError!void { | |
| 895 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 896 | try pool.checkCancel(); | |
| 897 | ||
| 898 | _ = file; | |
| 899 | _ = offset; | |
| 900 | @panic("TODO"); | |
| 901 | } | |
| 902 | ||
| 903 | fn fileSeekTo(userdata: ?*anyopaque, file: Io.File, offset: u64) Io.File.SeekError!void { | |
| 904 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 905 | try pool.checkCancel(); | |
| 906 | ||
| 907 | _ = file; | |
| 908 | _ = offset; | |
| 909 | @panic("TODO"); | |
| 910 | } | |
| 911 | ||
| 912 | fn pwrite(userdata: ?*anyopaque, file: Io.File, buffer: []const u8, offset: posix.off_t) Io.File.PWriteError!usize { | |
| 913 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 914 | try pool.checkCancel(); | |
| 915 | const fs_file: std.fs.File = .{ .handle = file.handle }; | |
| 916 | return switch (offset) { | |
| 917 | -1 => fs_file.write(buffer), | |
| 918 | else => fs_file.pwrite(buffer, @bitCast(offset)), | |
| 919 | }; | |
| 920 | } | |
| 921 | ||
| 922 | fn now(userdata: ?*anyopaque, clockid: posix.clockid_t) Io.ClockGetTimeError!Io.Timestamp { | |
| 923 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 924 | try pool.checkCancel(); | |
| 925 | const timespec = try posix.clock_gettime(clockid); | |
| 926 | return @enumFromInt(@as(i128, timespec.sec) * std.time.ns_per_s + timespec.nsec); | |
| 927 | } | |
| 928 | ||
| 929 | fn sleep(userdata: ?*anyopaque, clockid: posix.clockid_t, deadline: Io.Deadline) Io.SleepError!void { | |
| 930 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 931 | const deadline_nanoseconds: i96 = switch (deadline) { | |
| 932 | .duration => |duration| duration.nanoseconds, | |
| 933 | .timestamp => |timestamp| @intFromEnum(timestamp), | |
| 934 | }; | |
| 935 | var timespec: posix.timespec = .{ | |
| 936 | .sec = @intCast(@divFloor(deadline_nanoseconds, std.time.ns_per_s)), | |
| 937 | .nsec = @intCast(@mod(deadline_nanoseconds, std.time.ns_per_s)), | |
| 938 | }; | |
| 939 | while (true) { | |
| 940 | try pool.checkCancel(); | |
| 941 | switch (std.os.linux.E.init(std.os.linux.clock_nanosleep(clockid, .{ .ABSTIME = switch (deadline) { | |
| 942 | .duration => false, | |
| 943 | .timestamp => true, | |
| 944 | } }, ×pec, ×pec))) { | |
| 945 | .SUCCESS => return, | |
| 946 | .FAULT => unreachable, | |
| 947 | .INTR => {}, | |
| 948 | .INVAL => return error.UnsupportedClock, | |
| 949 | else => |err| return posix.unexpectedErrno(err), | |
| 950 | } | |
| 951 | } | |
| 952 | } | |
| 953 | ||
| 954 | fn select(userdata: ?*anyopaque, futures: []const *Io.AnyFuture) usize { | |
| 955 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 956 | _ = pool; | |
| 957 | ||
| 958 | var reset_event: std.Thread.ResetEvent = .{}; | |
| 959 | ||
| 960 | for (futures, 0..) |future, i| { | |
| 961 | const closure: *AsyncClosure = @ptrCast(@alignCast(future)); | |
| 962 | if (@atomicRmw(?*std.Thread.ResetEvent, &closure.select_condition, .Xchg, &reset_event, .seq_cst) == AsyncClosure.done_reset_event) { | |
| 963 | for (futures[0..i]) |cleanup_future| { | |
| 964 | const cleanup_closure: *AsyncClosure = @ptrCast(@alignCast(cleanup_future)); | |
| 965 | if (@atomicRmw(?*std.Thread.ResetEvent, &cleanup_closure.select_condition, .Xchg, null, .seq_cst) == AsyncClosure.done_reset_event) { | |
| 966 | cleanup_closure.reset_event.wait(); // Ensure no reference to our stack-allocated reset_event. | |
| 967 | } | |
| 968 | } | |
| 969 | return i; | |
| 970 | } | |
| 971 | } | |
| 972 | ||
| 973 | reset_event.wait(); | |
| 974 | ||
| 975 | var result: ?usize = null; | |
| 976 | for (futures, 0..) |future, i| { | |
| 977 | const closure: *AsyncClosure = @ptrCast(@alignCast(future)); | |
| 978 | if (@atomicRmw(?*std.Thread.ResetEvent, &closure.select_condition, .Xchg, null, .seq_cst) == AsyncClosure.done_reset_event) { | |
| 979 | closure.reset_event.wait(); // Ensure no reference to our stack-allocated reset_event. | |
| 980 | if (result == null) result = i; // In case multiple are ready, return first. | |
| 981 | } | |
| 982 | } | |
| 983 | return result.?; | |
| 984 | } | |
| 985 | ||
| 986 | fn listen(userdata: ?*anyopaque, address: Io.net.IpAddress, options: Io.net.ListenOptions) Io.net.ListenError!Io.net.Server { | |
| 987 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 988 | try pool.checkCancel(); | |
| 989 | ||
| 990 | const nonblock: u32 = if (options.force_nonblocking) posix.SOCK.NONBLOCK else 0; | |
| 991 | const sock_flags = posix.SOCK.STREAM | posix.SOCK.CLOEXEC | nonblock; | |
| 992 | const proto: u32 = posix.IPPROTO.TCP; | |
| 993 | const family = posixAddressFamily(address); | |
| 994 | const sockfd = try posix.socket(family, sock_flags, proto); | |
| 995 | const stream: std.net.Stream = .{ .handle = sockfd }; | |
| 996 | errdefer stream.close(); | |
| 997 | ||
| 998 | if (options.reuse_address) { | |
| 999 | try posix.setsockopt( | |
| 1000 | sockfd, | |
| 1001 | posix.SOL.SOCKET, | |
| 1002 | posix.SO.REUSEADDR, | |
| 1003 | &std.mem.toBytes(@as(c_int, 1)), | |
| 1004 | ); | |
| 1005 | if (@hasDecl(posix.SO, "REUSEPORT") and family != posix.AF.UNIX) { | |
| 1006 | try posix.setsockopt( | |
| 1007 | sockfd, | |
| 1008 | posix.SOL.SOCKET, | |
| 1009 | posix.SO.REUSEPORT, | |
| 1010 | &std.mem.toBytes(@as(c_int, 1)), | |
| 1011 | ); | |
| 1012 | } | |
| 1013 | } | |
| 1014 | ||
| 1015 | var storage: PosixAddress = undefined; | |
| 1016 | var socklen = addressToPosix(address, &storage); | |
| 1017 | try posix.bind(sockfd, &storage.any, socklen); | |
| 1018 | try posix.listen(sockfd, options.kernel_backlog); | |
| 1019 | try posix.getsockname(sockfd, &storage.any, &socklen); | |
| 1020 | return .{ | |
| 1021 | .listen_address = addressFromPosix(&storage), | |
| 1022 | .stream = .{ .handle = stream.handle }, | |
| 1023 | }; | |
| 1024 | } | |
| 1025 | ||
| 1026 | fn accept(userdata: ?*anyopaque, server: *Io.net.Server) Io.net.Server.AcceptError!Io.net.Server.Connection { | |
| 1027 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 1028 | try pool.checkCancel(); | |
| 1029 | ||
| 1030 | var storage: PosixAddress = undefined; | |
| 1031 | var addr_len: posix.socklen_t = @sizeOf(PosixAddress); | |
| 1032 | const fd = try posix.accept(server.stream.handle, &storage.any, &addr_len, posix.SOCK.CLOEXEC); | |
| 1033 | return .{ | |
| 1034 | .stream = .{ .handle = fd }, | |
| 1035 | .address = addressFromPosix(&storage), | |
| 1036 | }; | |
| 1037 | } | |
| 1038 | ||
| 1039 | fn netReadPosix(userdata: ?*anyopaque, stream: Io.net.Stream, data: [][]u8) Io.net.Stream.Reader.Error!usize { | |
| 1040 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 1041 | try pool.checkCancel(); | |
| 1042 | ||
| 1043 | var iovecs_buffer: [max_iovecs_len]posix.iovec = undefined; | |
| 1044 | var i: usize = 0; | |
| 1045 | for (data) |buf| { | |
| 1046 | if (iovecs_buffer.len - i == 0) break; | |
| 1047 | if (buf.len != 0) { | |
| 1048 | iovecs_buffer[i] = .{ .base = buf.ptr, .len = buf.len }; | |
| 1049 | i += 1; | |
| 1050 | } | |
| 1051 | } | |
| 1052 | const dest = iovecs_buffer[0..i]; | |
| 1053 | assert(dest[0].len > 0); | |
| 1054 | const n = try posix.readv(stream.handle, dest); | |
| 1055 | if (n == 0) return error.EndOfStream; | |
| 1056 | return n; | |
| 1057 | } | |
| 1058 | ||
| 1059 | fn netWritePosix( | |
| 1060 | userdata: ?*anyopaque, | |
| 1061 | stream: Io.net.Stream, | |
| 1062 | header: []const u8, | |
| 1063 | data: []const []const u8, | |
| 1064 | splat: usize, | |
| 1065 | ) Io.net.Stream.Writer.Error!usize { | |
| 1066 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 1067 | try pool.checkCancel(); | |
| 1068 | ||
| 1069 | var iovecs: [max_iovecs_len]posix.iovec_const = undefined; | |
| 1070 | var msg: posix.msghdr_const = .{ | |
| 1071 | .name = null, | |
| 1072 | .namelen = 0, | |
| 1073 | .iov = &iovecs, | |
| 1074 | .iovlen = 0, | |
| 1075 | .control = null, | |
| 1076 | .controllen = 0, | |
| 1077 | .flags = 0, | |
| 1078 | }; | |
| 1079 | addBuf(&iovecs, &msg.iovlen, header); | |
| 1080 | for (data[0 .. data.len - 1]) |bytes| addBuf(&iovecs, &msg.iovlen, bytes); | |
| 1081 | const pattern = data[data.len - 1]; | |
| 1082 | if (iovecs.len - msg.iovlen != 0) switch (splat) { | |
| 1083 | 0 => {}, | |
| 1084 | 1 => addBuf(&iovecs, &msg.iovlen, pattern), | |
| 1085 | else => switch (pattern.len) { | |
| 1086 | 0 => {}, | |
| 1087 | 1 => { | |
| 1088 | var backup_buffer: [splat_buffer_size]u8 = undefined; | |
| 1089 | const splat_buffer = &backup_buffer; | |
| 1090 | const memset_len = @min(splat_buffer.len, splat); | |
| 1091 | const buf = splat_buffer[0..memset_len]; | |
| 1092 | @memset(buf, pattern[0]); | |
| 1093 | addBuf(&iovecs, &msg.iovlen, buf); | |
| 1094 | var remaining_splat = splat - buf.len; | |
| 1095 | while (remaining_splat > splat_buffer.len and iovecs.len - msg.iovlen != 0) { | |
| 1096 | assert(buf.len == splat_buffer.len); | |
| 1097 | addBuf(&iovecs, &msg.iovlen, splat_buffer); | |
| 1098 | remaining_splat -= splat_buffer.len; | |
| 1099 | } | |
| 1100 | addBuf(&iovecs, &msg.iovlen, splat_buffer[0..remaining_splat]); | |
| 1101 | }, | |
| 1102 | else => for (0..@min(splat, iovecs.len - msg.iovlen)) |_| { | |
| 1103 | addBuf(&iovecs, &msg.iovlen, pattern); | |
| 1104 | }, | |
| 1105 | }, | |
| 1106 | }; | |
| 1107 | const flags = posix.MSG.NOSIGNAL; | |
| 1108 | return posix.sendmsg(stream.handle, &msg, flags); | |
| 1109 | } | |
| 1110 | ||
| 1111 | fn addBuf(v: []posix.iovec_const, i: *@FieldType(posix.msghdr_const, "iovlen"), bytes: []const u8) void { | |
| 1112 | // OS checks ptr addr before length so zero length vectors must be omitted. | |
| 1113 | if (bytes.len == 0) return; | |
| 1114 | if (v.len - i.* == 0) return; | |
| 1115 | v[i.*] = .{ .base = bytes.ptr, .len = bytes.len }; | |
| 1116 | i.* += 1; | |
| 1117 | } | |
| 1118 | ||
| 1119 | fn netClose(userdata: ?*anyopaque, stream: Io.net.Stream) void { | |
| 1120 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 1121 | _ = pool; | |
| 1122 | const net_stream: std.net.Stream = .{ .handle = stream.handle }; | |
| 1123 | return net_stream.close(); | |
| 1124 | } | |
| 1125 | ||
| 1126 | fn netInterfaceIndex(userdata: ?*anyopaque, name: []const u8) Io.net.InterfaceIndexError!u32 { | |
| 1127 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 1128 | try pool.checkCancel(); | |
| 1129 | ||
| 1130 | if (native_os == .linux) { | |
| 1131 | if (name.len >= posix.IFNAMESIZE) return error.InterfaceNotFound; | |
| 1132 | var ifr: posix.ifreq = undefined; | |
| 1133 | @memcpy(ifr.ifrn.name[0..name.len], name); | |
| 1134 | ifr.ifrn.name[name.len] = 0; | |
| 1135 | ||
| 1136 | const rc = posix.system.socket(posix.AF.UNIX, posix.SOCK.DGRAM | posix.SOCK.CLOEXEC, 0); | |
| 1137 | const sock_fd: posix.fd_t = switch (posix.errno(rc)) { | |
| 1138 | .SUCCESS => @intCast(rc), | |
| 1139 | .ACCES => return error.AccessDenied, | |
| 1140 | .MFILE => return error.SystemResources, | |
| 1141 | .NFILE => return error.SystemResources, | |
| 1142 | .NOBUFS => return error.SystemResources, | |
| 1143 | .NOMEM => return error.SystemResources, | |
| 1144 | else => |err| return posix.unexpectedErrno(err), | |
| 1145 | }; | |
| 1146 | defer posix.close(sock_fd); | |
| 1147 | ||
| 1148 | while (true) { | |
| 1149 | try pool.checkCancel(); | |
| 1150 | switch (posix.errno(posix.system.ioctl(sock_fd, posix.SIOCGIFINDEX, @intFromPtr(&ifr)))) { | |
| 1151 | .SUCCESS => return @bitCast(ifr.ifru.ivalue), | |
| 1152 | .INVAL => |err| return badErrno(err), // Bad parameters. | |
| 1153 | .NOTTY => |err| return badErrno(err), | |
| 1154 | .NXIO => |err| return badErrno(err), | |
| 1155 | .BADF => |err| return badErrno(err), // Always a race condition. | |
| 1156 | .FAULT => |err| return badErrno(err), // Bad pointer parameter. | |
| 1157 | .INTR => continue, | |
| 1158 | .IO => |err| return badErrno(err), // sock_fd is not a file descriptor | |
| 1159 | .NODEV => return error.InterfaceNotFound, | |
| 1160 | else => |err| return posix.unexpectedErrno(err), | |
| 1161 | } | |
| 1162 | } | |
| 1163 | } | |
| 1164 | ||
| 1165 | if (native_os.isDarwin()) { | |
| 1166 | if (name.len >= posix.IFNAMESIZE) return error.InterfaceNotFound; | |
| 1167 | var if_name: [posix.IFNAMESIZE:0]u8 = undefined; | |
| 1168 | @memcpy(if_name[0..name.len], name); | |
| 1169 | if_name[name.len] = 0; | |
| 1170 | const if_slice = if_name[0..name.len :0]; | |
| 1171 | const index = std.c.if_nametoindex(if_slice); | |
| 1172 | if (index == 0) return error.InterfaceNotFound; | |
| 1173 | return @bitCast(index); | |
| 1174 | } | |
| 1175 | ||
| 1176 | if (native_os == .windows) { | |
| 1177 | if (name.len >= posix.IFNAMESIZE) return error.InterfaceNotFound; | |
| 1178 | var interface_name: [posix.IFNAMESIZE:0]u8 = undefined; | |
| 1179 | @memcpy(interface_name[0..name.len], name); | |
| 1180 | interface_name[name.len] = 0; | |
| 1181 | const index = std.os.windows.ws2_32.if_nametoindex(@as([*:0]const u8, &interface_name)); | |
| 1182 | if (index == 0) return error.InterfaceNotFound; | |
| 1183 | return index; | |
| 1184 | } | |
| 1185 | ||
| 1186 | @compileError("std.net.if_nametoindex unimplemented for this OS"); | |
| 1187 | } | |
| 1188 | ||
| 1189 | const PosixAddress = extern union { | |
| 1190 | any: posix.sockaddr, | |
| 1191 | in: posix.sockaddr.in, | |
| 1192 | in6: posix.sockaddr.in6, | |
| 1193 | }; | |
| 1194 | ||
| 1195 | fn posixAddressFamily(a: Io.net.IpAddress) posix.sa_family_t { | |
| 1196 | return switch (a) { | |
| 1197 | .ip4 => posix.AF.INET, | |
| 1198 | .ip6 => posix.AF.INET6, | |
| 1199 | }; | |
| 1200 | } | |
| 1201 | ||
| 1202 | fn addressFromPosix(posix_address: *PosixAddress) Io.net.IpAddress { | |
| 1203 | return switch (posix_address.any.family) { | |
| 1204 | posix.AF.INET => .{ .ip4 = address4FromPosix(&posix_address.in) }, | |
| 1205 | posix.AF.INET6 => .{ .ip6 = address6FromPosix(&posix_address.in6) }, | |
| 1206 | else => unreachable, | |
| 1207 | }; | |
| 1208 | } | |
| 1209 | ||
| 1210 | fn addressToPosix(a: Io.net.IpAddress, storage: *PosixAddress) posix.socklen_t { | |
| 1211 | return switch (a) { | |
| 1212 | .ip4 => |ip4| { | |
| 1213 | storage.in = address4ToPosix(ip4); | |
| 1214 | return @sizeOf(posix.sockaddr.in); | |
| 1215 | }, | |
| 1216 | .ip6 => |ip6| { | |
| 1217 | storage.in6 = address6ToPosix(ip6); | |
| 1218 | return @sizeOf(posix.sockaddr.in6); | |
| 1219 | }, | |
| 1220 | }; | |
| 1221 | } | |
| 1222 | ||
| 1223 | fn address4FromPosix(in: *posix.sockaddr.in) Io.net.Ip4Address { | |
| 1224 | return .{ | |
| 1225 | .port = std.mem.bigToNative(u16, in.port), | |
| 1226 | .bytes = @bitCast(in.addr), | |
| 1227 | }; | |
| 1228 | } | |
| 1229 | ||
| 1230 | fn address6FromPosix(in6: *posix.sockaddr.in6) Io.net.Ip6Address { | |
| 1231 | return .{ | |
| 1232 | .port = std.mem.bigToNative(u16, in6.port), | |
| 1233 | .bytes = in6.addr, | |
| 1234 | .flowinfo = in6.flowinfo, | |
| 1235 | .scope_id = in6.scope_id, | |
| 1236 | }; | |
| 1237 | } | |
| 1238 | ||
| 1239 | fn address4ToPosix(a: Io.net.Ip4Address) posix.sockaddr.in { | |
| 1240 | return .{ | |
| 1241 | .port = std.mem.nativeToBig(u16, a.port), | |
| 1242 | .addr = @bitCast(a.bytes), | |
| 1243 | }; | |
| 1244 | } | |
| 1245 | ||
| 1246 | fn address6ToPosix(a: Io.net.Ip6Address) posix.sockaddr.in6 { | |
| 1247 | return .{ | |
| 1248 | .port = std.mem.nativeToBig(u16, a.port), | |
| 1249 | .flowinfo = a.flowinfo, | |
| 1250 | .addr = a.bytes, | |
| 1251 | .scope_id = a.scope_id, | |
| 1252 | }; | |
| 1253 | } | |
| 1254 | ||
| 1255 | fn badErrno(err: posix.E) Io.UnexpectedError { | |
| 1256 | switch (builtin.mode) { | |
| 1257 | .Debug => std.debug.panic("programmer bug caused syscall error: {t}", .{err}), | |
| 1258 | else => return error.Unexpected, | |
| 1259 | } | |
| 1260 | } |
lib/std/Io/Threaded.zig created+1260
| ... | ... | @@ -0,0 +1,1260 @@ |
| 1 | const Pool = @This(); | |
| 2 | ||
| 3 | const builtin = @import("builtin"); | |
| 4 | const native_os = builtin.os.tag; | |
| 5 | const is_windows = native_os == .windows; | |
| 6 | const windows = std.os.windows; | |
| 7 | ||
| 8 | const std = @import("../std.zig"); | |
| 9 | const Allocator = std.mem.Allocator; | |
| 10 | const assert = std.debug.assert; | |
| 11 | const WaitGroup = std.Thread.WaitGroup; | |
| 12 | const posix = std.posix; | |
| 13 | const Io = std.Io; | |
| 14 | ||
| 15 | /// Thread-safe. | |
| 16 | allocator: Allocator, | |
| 17 | mutex: std.Thread.Mutex = .{}, | |
| 18 | cond: std.Thread.Condition = .{}, | |
| 19 | run_queue: std.SinglyLinkedList = .{}, | |
| 20 | join_requested: bool = false, | |
| 21 | threads: std.ArrayListUnmanaged(std.Thread), | |
| 22 | stack_size: usize, | |
| 23 | cpu_count: std.Thread.CpuCountError!usize, | |
| 24 | parallel_count: usize, | |
| 25 | ||
| 26 | threadlocal var current_closure: ?*AsyncClosure = null; | |
| 27 | ||
| 28 | const max_iovecs_len = 8; | |
| 29 | const splat_buffer_size = 64; | |
| 30 | ||
| 31 | comptime { | |
| 32 | assert(max_iovecs_len <= posix.IOV_MAX); | |
| 33 | } | |
| 34 | ||
| 35 | pub const Runnable = struct { | |
| 36 | start: Start, | |
| 37 | node: std.SinglyLinkedList.Node = .{}, | |
| 38 | is_parallel: bool, | |
| 39 | ||
| 40 | pub const Start = *const fn (*Runnable) void; | |
| 41 | }; | |
| 42 | ||
| 43 | pub const InitError = std.Thread.CpuCountError || Allocator.Error; | |
| 44 | ||
| 45 | pub fn init(gpa: Allocator) Pool { | |
| 46 | var pool: Pool = .{ | |
| 47 | .allocator = gpa, | |
| 48 | .threads = .empty, | |
| 49 | .stack_size = std.Thread.SpawnConfig.default_stack_size, | |
| 50 | .cpu_count = std.Thread.getCpuCount(), | |
| 51 | .parallel_count = 0, | |
| 52 | }; | |
| 53 | if (pool.cpu_count) |n| { | |
| 54 | pool.threads.ensureTotalCapacityPrecise(gpa, n - 1) catch {}; | |
| 55 | } else |_| {} | |
| 56 | return pool; | |
| 57 | } | |
| 58 | ||
| 59 | pub fn deinit(pool: *Pool) void { | |
| 60 | const gpa = pool.allocator; | |
| 61 | pool.join(); | |
| 62 | pool.threads.deinit(gpa); | |
| 63 | pool.* = undefined; | |
| 64 | } | |
| 65 | ||
| 66 | fn join(pool: *Pool) void { | |
| 67 | if (builtin.single_threaded) return; | |
| 68 | { | |
| 69 | pool.mutex.lock(); | |
| 70 | defer pool.mutex.unlock(); | |
| 71 | pool.join_requested = true; | |
| 72 | } | |
| 73 | pool.cond.broadcast(); | |
| 74 | for (pool.threads.items) |thread| thread.join(); | |
| 75 | } | |
| 76 | ||
| 77 | fn worker(pool: *Pool) void { | |
| 78 | pool.mutex.lock(); | |
| 79 | defer pool.mutex.unlock(); | |
| 80 | ||
| 81 | while (true) { | |
| 82 | while (pool.run_queue.popFirst()) |run_node| { | |
| 83 | pool.mutex.unlock(); | |
| 84 | const runnable: *Runnable = @fieldParentPtr("node", run_node); | |
| 85 | runnable.start(runnable); | |
| 86 | pool.mutex.lock(); | |
| 87 | if (runnable.is_parallel) { | |
| 88 | // TODO also pop thread and join sometimes | |
| 89 | pool.parallel_count -= 1; | |
| 90 | } | |
| 91 | } | |
| 92 | if (pool.join_requested) break; | |
| 93 | pool.cond.wait(&pool.mutex); | |
| 94 | } | |
| 95 | } | |
| 96 | ||
| 97 | pub fn io(pool: *Pool) Io { | |
| 98 | return .{ | |
| 99 | .userdata = pool, | |
| 100 | .vtable = &.{ | |
| 101 | .async = async, | |
| 102 | .asyncConcurrent = asyncConcurrent, | |
| 103 | .await = await, | |
| 104 | .asyncDetached = asyncDetached, | |
| 105 | .cancel = cancel, | |
| 106 | .cancelRequested = cancelRequested, | |
| 107 | .select = select, | |
| 108 | ||
| 109 | .mutexLock = mutexLock, | |
| 110 | .mutexUnlock = mutexUnlock, | |
| 111 | ||
| 112 | .conditionWait = conditionWait, | |
| 113 | .conditionWake = conditionWake, | |
| 114 | ||
| 115 | .createFile = createFile, | |
| 116 | .fileOpen = fileOpen, | |
| 117 | .fileClose = fileClose, | |
| 118 | .pwrite = pwrite, | |
| 119 | .fileReadStreaming = fileReadStreaming, | |
| 120 | .fileReadPositional = fileReadPositional, | |
| 121 | .fileSeekBy = fileSeekBy, | |
| 122 | .fileSeekTo = fileSeekTo, | |
| 123 | ||
| 124 | .now = now, | |
| 125 | .sleep = sleep, | |
| 126 | ||
| 127 | .listen = listen, | |
| 128 | .accept = accept, | |
| 129 | .netRead = switch (builtin.os.tag) { | |
| 130 | .windows => @panic("TODO"), | |
| 131 | else => netReadPosix, | |
| 132 | }, | |
| 133 | .netWrite = switch (builtin.os.tag) { | |
| 134 | .windows => @panic("TODO"), | |
| 135 | else => netWritePosix, | |
| 136 | }, | |
| 137 | .netClose = netClose, | |
| 138 | .netInterfaceIndex = netInterfaceIndex, | |
| 139 | }, | |
| 140 | }; | |
| 141 | } | |
| 142 | ||
| 143 | const AsyncClosure = struct { | |
| 144 | func: *const fn (context: *anyopaque, result: *anyopaque) void, | |
| 145 | runnable: Runnable, | |
| 146 | reset_event: std.Thread.ResetEvent, | |
| 147 | select_condition: ?*std.Thread.ResetEvent, | |
| 148 | cancel_tid: std.Thread.Id, | |
| 149 | context_offset: usize, | |
| 150 | result_offset: usize, | |
| 151 | ||
| 152 | const done_reset_event: *std.Thread.ResetEvent = @ptrFromInt(@alignOf(std.Thread.ResetEvent)); | |
| 153 | ||
| 154 | const canceling_tid: std.Thread.Id = switch (@typeInfo(std.Thread.Id)) { | |
| 155 | .int => |int_info| switch (int_info.signedness) { | |
| 156 | .signed => -1, | |
| 157 | .unsigned => std.math.maxInt(std.Thread.Id), | |
| 158 | }, | |
| 159 | .pointer => @ptrFromInt(std.math.maxInt(usize)), | |
| 160 | else => @compileError("unsupported std.Thread.Id: " ++ @typeName(std.Thread.Id)), | |
| 161 | }; | |
| 162 | ||
| 163 | fn start(runnable: *Runnable) void { | |
| 164 | const closure: *AsyncClosure = @alignCast(@fieldParentPtr("runnable", runnable)); | |
| 165 | const tid = std.Thread.getCurrentId(); | |
| 166 | if (@cmpxchgStrong( | |
| 167 | std.Thread.Id, | |
| 168 | &closure.cancel_tid, | |
| 169 | 0, | |
| 170 | tid, | |
| 171 | .acq_rel, | |
| 172 | .acquire, | |
| 173 | )) |cancel_tid| { | |
| 174 | assert(cancel_tid == canceling_tid); | |
| 175 | closure.reset_event.set(); | |
| 176 | return; | |
| 177 | } | |
| 178 | current_closure = closure; | |
| 179 | closure.func(closure.contextPointer(), closure.resultPointer()); | |
| 180 | current_closure = null; | |
| 181 | if (@cmpxchgStrong( | |
| 182 | std.Thread.Id, | |
| 183 | &closure.cancel_tid, | |
| 184 | tid, | |
| 185 | 0, | |
| 186 | .acq_rel, | |
| 187 | .acquire, | |
| 188 | )) |cancel_tid| assert(cancel_tid == canceling_tid); | |
| 189 | ||
| 190 | if (@atomicRmw( | |
| 191 | ?*std.Thread.ResetEvent, | |
| 192 | &closure.select_condition, | |
| 193 | .Xchg, | |
| 194 | done_reset_event, | |
| 195 | .release, | |
| 196 | )) |select_reset| { | |
| 197 | assert(select_reset != done_reset_event); | |
| 198 | select_reset.set(); | |
| 199 | } | |
| 200 | closure.reset_event.set(); | |
| 201 | } | |
| 202 | ||
| 203 | fn contextOffset(context_alignment: std.mem.Alignment) usize { | |
| 204 | return context_alignment.forward(@sizeOf(AsyncClosure)); | |
| 205 | } | |
| 206 | ||
| 207 | fn resultOffset( | |
| 208 | context_alignment: std.mem.Alignment, | |
| 209 | context_len: usize, | |
| 210 | result_alignment: std.mem.Alignment, | |
| 211 | ) usize { | |
| 212 | return result_alignment.forward(contextOffset(context_alignment) + context_len); | |
| 213 | } | |
| 214 | ||
| 215 | fn resultPointer(closure: *AsyncClosure) [*]u8 { | |
| 216 | const base: [*]u8 = @ptrCast(closure); | |
| 217 | return base + closure.result_offset; | |
| 218 | } | |
| 219 | ||
| 220 | fn contextPointer(closure: *AsyncClosure) [*]u8 { | |
| 221 | const base: [*]u8 = @ptrCast(closure); | |
| 222 | return base + closure.context_offset; | |
| 223 | } | |
| 224 | ||
| 225 | fn waitAndFree(closure: *AsyncClosure, gpa: Allocator, result: []u8) void { | |
| 226 | closure.reset_event.wait(); | |
| 227 | @memcpy(result, closure.resultPointer()[0..result.len]); | |
| 228 | free(closure, gpa, result.len); | |
| 229 | } | |
| 230 | ||
| 231 | fn free(closure: *AsyncClosure, gpa: Allocator, result_len: usize) void { | |
| 232 | const base: [*]align(@alignOf(AsyncClosure)) u8 = @ptrCast(closure); | |
| 233 | gpa.free(base[0 .. closure.result_offset + result_len]); | |
| 234 | } | |
| 235 | }; | |
| 236 | ||
| 237 | fn async( | |
| 238 | userdata: ?*anyopaque, | |
| 239 | result: []u8, | |
| 240 | result_alignment: std.mem.Alignment, | |
| 241 | context: []const u8, | |
| 242 | context_alignment: std.mem.Alignment, | |
| 243 | start: *const fn (context: *const anyopaque, result: *anyopaque) void, | |
| 244 | ) ?*Io.AnyFuture { | |
| 245 | if (builtin.single_threaded) { | |
| 246 | start(context.ptr, result.ptr); | |
| 247 | return null; | |
| 248 | } | |
| 249 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 250 | const cpu_count = pool.cpu_count catch { | |
| 251 | return asyncConcurrent(userdata, result.len, result_alignment, context, context_alignment, start) catch { | |
| 252 | start(context.ptr, result.ptr); | |
| 253 | return null; | |
| 254 | }; | |
| 255 | }; | |
| 256 | const gpa = pool.allocator; | |
| 257 | const context_offset = context_alignment.forward(@sizeOf(AsyncClosure)); | |
| 258 | const result_offset = result_alignment.forward(context_offset + context.len); | |
| 259 | const n = result_offset + result.len; | |
| 260 | const closure: *AsyncClosure = @ptrCast(@alignCast(gpa.alignedAlloc(u8, .of(AsyncClosure), n) catch { | |
| 261 | start(context.ptr, result.ptr); | |
| 262 | return null; | |
| 263 | })); | |
| 264 | ||
| 265 | closure.* = .{ | |
| 266 | .func = start, | |
| 267 | .context_offset = context_offset, | |
| 268 | .result_offset = result_offset, | |
| 269 | .reset_event = .{}, | |
| 270 | .cancel_tid = 0, | |
| 271 | .select_condition = null, | |
| 272 | .runnable = .{ | |
| 273 | .start = AsyncClosure.start, | |
| 274 | .is_parallel = false, | |
| 275 | }, | |
| 276 | }; | |
| 277 | ||
| 278 | @memcpy(closure.contextPointer()[0..context.len], context); | |
| 279 | ||
| 280 | pool.mutex.lock(); | |
| 281 | ||
| 282 | const thread_capacity = cpu_count - 1 + pool.parallel_count; | |
| 283 | ||
| 284 | pool.threads.ensureTotalCapacityPrecise(gpa, thread_capacity) catch { | |
| 285 | pool.mutex.unlock(); | |
| 286 | closure.free(gpa, result.len); | |
| 287 | start(context.ptr, result.ptr); | |
| 288 | return null; | |
| 289 | }; | |
| 290 | ||
| 291 | pool.run_queue.prepend(&closure.runnable.node); | |
| 292 | ||
| 293 | if (pool.threads.items.len < thread_capacity) { | |
| 294 | const thread = std.Thread.spawn(.{ .stack_size = pool.stack_size }, worker, .{pool}) catch { | |
| 295 | if (pool.threads.items.len == 0) { | |
| 296 | assert(pool.run_queue.popFirst() == &closure.runnable.node); | |
| 297 | pool.mutex.unlock(); | |
| 298 | closure.free(gpa, result.len); | |
| 299 | start(context.ptr, result.ptr); | |
| 300 | return null; | |
| 301 | } | |
| 302 | // Rely on other workers to do it. | |
| 303 | pool.mutex.unlock(); | |
| 304 | pool.cond.signal(); | |
| 305 | return @ptrCast(closure); | |
| 306 | }; | |
| 307 | pool.threads.appendAssumeCapacity(thread); | |
| 308 | } | |
| 309 | ||
| 310 | pool.mutex.unlock(); | |
| 311 | pool.cond.signal(); | |
| 312 | return @ptrCast(closure); | |
| 313 | } | |
| 314 | ||
| 315 | fn asyncConcurrent( | |
| 316 | userdata: ?*anyopaque, | |
| 317 | result_len: usize, | |
| 318 | result_alignment: std.mem.Alignment, | |
| 319 | context: []const u8, | |
| 320 | context_alignment: std.mem.Alignment, | |
| 321 | start: *const fn (context: *const anyopaque, result: *anyopaque) void, | |
| 322 | ) error{OutOfMemory}!*Io.AnyFuture { | |
| 323 | if (builtin.single_threaded) unreachable; | |
| 324 | ||
| 325 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 326 | const cpu_count = pool.cpu_count catch 1; | |
| 327 | const gpa = pool.allocator; | |
| 328 | const context_offset = context_alignment.forward(@sizeOf(AsyncClosure)); | |
| 329 | const result_offset = result_alignment.forward(context_offset + context.len); | |
| 330 | const n = result_offset + result_len; | |
| 331 | const closure: *AsyncClosure = @ptrCast(@alignCast(try gpa.alignedAlloc(u8, .of(AsyncClosure), n))); | |
| 332 | ||
| 333 | closure.* = .{ | |
| 334 | .func = start, | |
| 335 | .context_offset = context_offset, | |
| 336 | .result_offset = result_offset, | |
| 337 | .reset_event = .{}, | |
| 338 | .cancel_tid = 0, | |
| 339 | .select_condition = null, | |
| 340 | .runnable = .{ | |
| 341 | .start = AsyncClosure.start, | |
| 342 | .is_parallel = true, | |
| 343 | }, | |
| 344 | }; | |
| 345 | @memcpy(closure.contextPointer()[0..context.len], context); | |
| 346 | ||
| 347 | pool.mutex.lock(); | |
| 348 | ||
| 349 | pool.parallel_count += 1; | |
| 350 | const thread_capacity = cpu_count - 1 + pool.parallel_count; | |
| 351 | ||
| 352 | pool.threads.ensureTotalCapacity(gpa, thread_capacity) catch { | |
| 353 | pool.mutex.unlock(); | |
| 354 | closure.free(gpa, result_len); | |
| 355 | return error.OutOfMemory; | |
| 356 | }; | |
| 357 | ||
| 358 | pool.run_queue.prepend(&closure.runnable.node); | |
| 359 | ||
| 360 | if (pool.threads.items.len < thread_capacity) { | |
| 361 | const thread = std.Thread.spawn(.{ .stack_size = pool.stack_size }, worker, .{pool}) catch { | |
| 362 | assert(pool.run_queue.popFirst() == &closure.runnable.node); | |
| 363 | pool.mutex.unlock(); | |
| 364 | closure.free(gpa, result_len); | |
| 365 | return error.OutOfMemory; | |
| 366 | }; | |
| 367 | pool.threads.appendAssumeCapacity(thread); | |
| 368 | } | |
| 369 | ||
| 370 | pool.mutex.unlock(); | |
| 371 | pool.cond.signal(); | |
| 372 | return @ptrCast(closure); | |
| 373 | } | |
| 374 | ||
| 375 | const DetachedClosure = struct { | |
| 376 | pool: *Pool, | |
| 377 | func: *const fn (context: *anyopaque) void, | |
| 378 | runnable: Runnable, | |
| 379 | context_alignment: std.mem.Alignment, | |
| 380 | context_len: usize, | |
| 381 | ||
| 382 | fn start(runnable: *Runnable) void { | |
| 383 | const closure: *DetachedClosure = @alignCast(@fieldParentPtr("runnable", runnable)); | |
| 384 | closure.func(closure.contextPointer()); | |
| 385 | const gpa = closure.pool.allocator; | |
| 386 | free(closure, gpa); | |
| 387 | } | |
| 388 | ||
| 389 | fn free(closure: *DetachedClosure, gpa: Allocator) void { | |
| 390 | const base: [*]align(@alignOf(DetachedClosure)) u8 = @ptrCast(closure); | |
| 391 | gpa.free(base[0..contextEnd(closure.context_alignment, closure.context_len)]); | |
| 392 | } | |
| 393 | ||
| 394 | fn contextOffset(context_alignment: std.mem.Alignment) usize { | |
| 395 | return context_alignment.forward(@sizeOf(DetachedClosure)); | |
| 396 | } | |
| 397 | ||
| 398 | fn contextEnd(context_alignment: std.mem.Alignment, context_len: usize) usize { | |
| 399 | return contextOffset(context_alignment) + context_len; | |
| 400 | } | |
| 401 | ||
| 402 | fn contextPointer(closure: *DetachedClosure) [*]u8 { | |
| 403 | const base: [*]u8 = @ptrCast(closure); | |
| 404 | return base + contextOffset(closure.context_alignment); | |
| 405 | } | |
| 406 | }; | |
| 407 | ||
| 408 | fn asyncDetached( | |
| 409 | userdata: ?*anyopaque, | |
| 410 | context: []const u8, | |
| 411 | context_alignment: std.mem.Alignment, | |
| 412 | start: *const fn (context: *const anyopaque) void, | |
| 413 | ) void { | |
| 414 | if (builtin.single_threaded) return start(context.ptr); | |
| 415 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 416 | const cpu_count = pool.cpu_count catch 1; | |
| 417 | const gpa = pool.allocator; | |
| 418 | const n = DetachedClosure.contextEnd(context_alignment, context.len); | |
| 419 | const closure: *DetachedClosure = @ptrCast(@alignCast(gpa.alignedAlloc(u8, .of(DetachedClosure), n) catch { | |
| 420 | return start(context.ptr); | |
| 421 | })); | |
| 422 | closure.* = .{ | |
| 423 | .pool = pool, | |
| 424 | .func = start, | |
| 425 | .context_alignment = context_alignment, | |
| 426 | .context_len = context.len, | |
| 427 | .runnable = .{ | |
| 428 | .start = DetachedClosure.start, | |
| 429 | .is_parallel = false, | |
| 430 | }, | |
| 431 | }; | |
| 432 | @memcpy(closure.contextPointer()[0..context.len], context); | |
| 433 | ||
| 434 | pool.mutex.lock(); | |
| 435 | ||
| 436 | const thread_capacity = cpu_count - 1 + pool.parallel_count; | |
| 437 | ||
| 438 | pool.threads.ensureTotalCapacityPrecise(gpa, thread_capacity) catch { | |
| 439 | pool.mutex.unlock(); | |
| 440 | closure.free(gpa); | |
| 441 | return start(context.ptr); | |
| 442 | }; | |
| 443 | ||
| 444 | pool.run_queue.prepend(&closure.runnable.node); | |
| 445 | ||
| 446 | if (pool.threads.items.len < thread_capacity) { | |
| 447 | const thread = std.Thread.spawn(.{ .stack_size = pool.stack_size }, worker, .{pool}) catch { | |
| 448 | assert(pool.run_queue.popFirst() == &closure.runnable.node); | |
| 449 | pool.mutex.unlock(); | |
| 450 | closure.free(gpa); | |
| 451 | return start(context.ptr); | |
| 452 | }; | |
| 453 | pool.threads.appendAssumeCapacity(thread); | |
| 454 | } | |
| 455 | ||
| 456 | pool.mutex.unlock(); | |
| 457 | pool.cond.signal(); | |
| 458 | } | |
| 459 | ||
| 460 | fn await( | |
| 461 | userdata: ?*anyopaque, | |
| 462 | any_future: *std.Io.AnyFuture, | |
| 463 | result: []u8, | |
| 464 | result_alignment: std.mem.Alignment, | |
| 465 | ) void { | |
| 466 | _ = result_alignment; | |
| 467 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 468 | const closure: *AsyncClosure = @ptrCast(@alignCast(any_future)); | |
| 469 | closure.waitAndFree(pool.allocator, result); | |
| 470 | } | |
| 471 | ||
| 472 | fn cancel( | |
| 473 | userdata: ?*anyopaque, | |
| 474 | any_future: *Io.AnyFuture, | |
| 475 | result: []u8, | |
| 476 | result_alignment: std.mem.Alignment, | |
| 477 | ) void { | |
| 478 | _ = result_alignment; | |
| 479 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 480 | const closure: *AsyncClosure = @ptrCast(@alignCast(any_future)); | |
| 481 | switch (@atomicRmw( | |
| 482 | std.Thread.Id, | |
| 483 | &closure.cancel_tid, | |
| 484 | .Xchg, | |
| 485 | AsyncClosure.canceling_tid, | |
| 486 | .acq_rel, | |
| 487 | )) { | |
| 488 | 0, AsyncClosure.canceling_tid => {}, | |
| 489 | else => |cancel_tid| switch (builtin.os.tag) { | |
| 490 | .linux => _ = std.os.linux.tgkill( | |
| 491 | std.os.linux.getpid(), | |
| 492 | @bitCast(cancel_tid), | |
| 493 | posix.SIG.IO, | |
| 494 | ), | |
| 495 | else => {}, | |
| 496 | }, | |
| 497 | } | |
| 498 | closure.waitAndFree(pool.allocator, result); | |
| 499 | } | |
| 500 | ||
| 501 | fn cancelRequested(userdata: ?*anyopaque) bool { | |
| 502 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 503 | _ = pool; | |
| 504 | const closure = current_closure orelse return false; | |
| 505 | return @atomicLoad(std.Thread.Id, &closure.cancel_tid, .acquire) == AsyncClosure.canceling_tid; | |
| 506 | } | |
| 507 | ||
| 508 | fn checkCancel(pool: *Pool) error{Canceled}!void { | |
| 509 | if (cancelRequested(pool)) return error.Canceled; | |
| 510 | } | |
| 511 | ||
| 512 | fn mutexLock(userdata: ?*anyopaque, prev_state: Io.Mutex.State, mutex: *Io.Mutex) error{Canceled}!void { | |
| 513 | _ = userdata; | |
| 514 | if (prev_state == .contended) { | |
| 515 | std.Thread.Futex.wait(@ptrCast(&mutex.state), @intFromEnum(Io.Mutex.State.contended)); | |
| 516 | } | |
| 517 | while (@atomicRmw( | |
| 518 | Io.Mutex.State, | |
| 519 | &mutex.state, | |
| 520 | .Xchg, | |
| 521 | .contended, | |
| 522 | .acquire, | |
| 523 | ) != .unlocked) { | |
| 524 | std.Thread.Futex.wait(@ptrCast(&mutex.state), @intFromEnum(Io.Mutex.State.contended)); | |
| 525 | } | |
| 526 | } | |
| 527 | fn mutexUnlock(userdata: ?*anyopaque, prev_state: Io.Mutex.State, mutex: *Io.Mutex) void { | |
| 528 | _ = userdata; | |
| 529 | _ = prev_state; | |
| 530 | if (@atomicRmw(Io.Mutex.State, &mutex.state, .Xchg, .unlocked, .release) == .contended) { | |
| 531 | std.Thread.Futex.wake(@ptrCast(&mutex.state), 1); | |
| 532 | } | |
| 533 | } | |
| 534 | ||
| 535 | fn conditionWait(userdata: ?*anyopaque, cond: *Io.Condition, mutex: *Io.Mutex) Io.Cancelable!void { | |
| 536 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 537 | comptime assert(@TypeOf(cond.state) == u64); | |
| 538 | const ints: *[2]std.atomic.Value(u32) = @ptrCast(&cond.state); | |
| 539 | const cond_state = &ints[0]; | |
| 540 | const cond_epoch = &ints[1]; | |
| 541 | const one_waiter = 1; | |
| 542 | const waiter_mask = 0xffff; | |
| 543 | const one_signal = 1 << 16; | |
| 544 | const signal_mask = 0xffff << 16; | |
| 545 | // Observe the epoch, then check the state again to see if we should wake up. | |
| 546 | // The epoch must be observed before we check the state or we could potentially miss a wake() and deadlock: | |
| 547 | // | |
| 548 | // - T1: s = LOAD(&state) | |
| 549 | // - T2: UPDATE(&s, signal) | |
| 550 | // - T2: UPDATE(&epoch, 1) + FUTEX_WAKE(&epoch) | |
| 551 | // - T1: e = LOAD(&epoch) (was reordered after the state load) | |
| 552 | // - T1: s & signals == 0 -> FUTEX_WAIT(&epoch, e) (missed the state update + the epoch change) | |
| 553 | // | |
| 554 | // Acquire barrier to ensure the epoch load happens before the state load. | |
| 555 | var epoch = cond_epoch.load(.acquire); | |
| 556 | var state = cond_state.fetchAdd(one_waiter, .monotonic); | |
| 557 | assert(state & waiter_mask != waiter_mask); | |
| 558 | state += one_waiter; | |
| 559 | ||
| 560 | mutex.unlock(pool.io()); | |
| 561 | defer mutex.lock(pool.io()) catch @panic("TODO"); | |
| 562 | ||
| 563 | var futex_deadline = std.Thread.Futex.Deadline.init(null); | |
| 564 | ||
| 565 | while (true) { | |
| 566 | futex_deadline.wait(cond_epoch, epoch) catch |err| switch (err) { | |
| 567 | error.Timeout => unreachable, | |
| 568 | }; | |
| 569 | ||
| 570 | epoch = cond_epoch.load(.acquire); | |
| 571 | state = cond_state.load(.monotonic); | |
| 572 | ||
| 573 | // Try to wake up by consuming a signal and decremented the waiter we added previously. | |
| 574 | // Acquire barrier ensures code before the wake() which added the signal happens before we decrement it and return. | |
| 575 | while (state & signal_mask != 0) { | |
| 576 | const new_state = state - one_waiter - one_signal; | |
| 577 | state = cond_state.cmpxchgWeak(state, new_state, .acquire, .monotonic) orelse return; | |
| 578 | } | |
| 579 | } | |
| 580 | } | |
| 581 | ||
| 582 | fn conditionWake(userdata: ?*anyopaque, cond: *Io.Condition, wake: Io.Condition.Wake) void { | |
| 583 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 584 | _ = pool; | |
| 585 | comptime assert(@TypeOf(cond.state) == u64); | |
| 586 | const ints: *[2]std.atomic.Value(u32) = @ptrCast(&cond.state); | |
| 587 | const cond_state = &ints[0]; | |
| 588 | const cond_epoch = &ints[1]; | |
| 589 | const one_waiter = 1; | |
| 590 | const waiter_mask = 0xffff; | |
| 591 | const one_signal = 1 << 16; | |
| 592 | const signal_mask = 0xffff << 16; | |
| 593 | var state = cond_state.load(.monotonic); | |
| 594 | while (true) { | |
| 595 | const waiters = (state & waiter_mask) / one_waiter; | |
| 596 | const signals = (state & signal_mask) / one_signal; | |
| 597 | ||
| 598 | // Reserves which waiters to wake up by incrementing the signals count. | |
| 599 | // Therefore, the signals count is always less than or equal to the waiters count. | |
| 600 | // We don't need to Futex.wake if there's nothing to wake up or if other wake() threads have reserved to wake up the current waiters. | |
| 601 | const wakeable = waiters - signals; | |
| 602 | if (wakeable == 0) { | |
| 603 | return; | |
| 604 | } | |
| 605 | ||
| 606 | const to_wake = switch (wake) { | |
| 607 | .one => 1, | |
| 608 | .all => wakeable, | |
| 609 | }; | |
| 610 | ||
| 611 | // Reserve the amount of waiters to wake by incrementing the signals count. | |
| 612 | // Release barrier ensures code before the wake() happens before the signal it posted and consumed by the wait() threads. | |
| 613 | const new_state = state + (one_signal * to_wake); | |
| 614 | state = cond_state.cmpxchgWeak(state, new_state, .release, .monotonic) orelse { | |
| 615 | // Wake up the waiting threads we reserved above by changing the epoch value. | |
| 616 | // NOTE: a waiting thread could miss a wake up if *exactly* ((1<<32)-1) wake()s happen between it observing the epoch and sleeping on it. | |
| 617 | // This is very unlikely due to how many precise amount of Futex.wake() calls that would be between the waiting thread's potential preemption. | |
| 618 | // | |
| 619 | // Release barrier ensures the signal being added to the state happens before the epoch is changed. | |
| 620 | // If not, the waiting thread could potentially deadlock from missing both the state and epoch change: | |
| 621 | // | |
| 622 | // - T2: UPDATE(&epoch, 1) (reordered before the state change) | |
| 623 | // - T1: e = LOAD(&epoch) | |
| 624 | // - T1: s = LOAD(&state) | |
| 625 | // - T2: UPDATE(&state, signal) + FUTEX_WAKE(&epoch) | |
| 626 | // - T1: s & signals == 0 -> FUTEX_WAIT(&epoch, e) (missed both epoch change and state change) | |
| 627 | _ = cond_epoch.fetchAdd(1, .release); | |
| 628 | std.Thread.Futex.wake(cond_epoch, to_wake); | |
| 629 | return; | |
| 630 | }; | |
| 631 | } | |
| 632 | } | |
| 633 | ||
| 634 | fn createFile( | |
| 635 | userdata: ?*anyopaque, | |
| 636 | dir: Io.Dir, | |
| 637 | sub_path: []const u8, | |
| 638 | flags: Io.File.CreateFlags, | |
| 639 | ) Io.File.OpenError!Io.File { | |
| 640 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 641 | try pool.checkCancel(); | |
| 642 | const fs_dir: std.fs.Dir = .{ .fd = dir.handle }; | |
| 643 | const fs_file = try fs_dir.createFile(sub_path, flags); | |
| 644 | return .{ .handle = fs_file.handle }; | |
| 645 | } | |
| 646 | ||
| 647 | fn fileOpen( | |
| 648 | userdata: ?*anyopaque, | |
| 649 | dir: Io.Dir, | |
| 650 | sub_path: []const u8, | |
| 651 | flags: Io.File.OpenFlags, | |
| 652 | ) Io.File.OpenError!Io.File { | |
| 653 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 654 | try pool.checkCancel(); | |
| 655 | const fs_dir: std.fs.Dir = .{ .fd = dir.handle }; | |
| 656 | const fs_file = try fs_dir.openFile(sub_path, flags); | |
| 657 | return .{ .handle = fs_file.handle }; | |
| 658 | } | |
| 659 | ||
| 660 | fn fileClose(userdata: ?*anyopaque, file: Io.File) void { | |
| 661 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 662 | _ = pool; | |
| 663 | const fs_file: std.fs.File = .{ .handle = file.handle }; | |
| 664 | return fs_file.close(); | |
| 665 | } | |
| 666 | ||
| 667 | fn fileReadStreaming(userdata: ?*anyopaque, file: Io.File, data: [][]u8) Io.File.ReadStreamingError!usize { | |
| 668 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 669 | ||
| 670 | if (is_windows) { | |
| 671 | const DWORD = windows.DWORD; | |
| 672 | var index: usize = 0; | |
| 673 | var truncate: usize = 0; | |
| 674 | var total: usize = 0; | |
| 675 | while (index < data.len) { | |
| 676 | try pool.checkCancel(); | |
| 677 | { | |
| 678 | const untruncated = data[index]; | |
| 679 | data[index] = untruncated[truncate..]; | |
| 680 | defer data[index] = untruncated; | |
| 681 | const buffer = data[index..]; | |
| 682 | const want_read_count: DWORD = @min(std.math.maxInt(DWORD), buffer.len); | |
| 683 | var n: DWORD = undefined; | |
| 684 | if (windows.kernel32.ReadFile(file.handle, buffer.ptr, want_read_count, &n, null) == 0) { | |
| 685 | switch (windows.GetLastError()) { | |
| 686 | .IO_PENDING => unreachable, | |
| 687 | .OPERATION_ABORTED => continue, | |
| 688 | .BROKEN_PIPE => return 0, | |
| 689 | .HANDLE_EOF => return 0, | |
| 690 | .NETNAME_DELETED => return error.ConnectionResetByPeer, | |
| 691 | .LOCK_VIOLATION => return error.LockViolation, | |
| 692 | .ACCESS_DENIED => return error.AccessDenied, | |
| 693 | .INVALID_HANDLE => return error.NotOpenForReading, | |
| 694 | else => |err| return windows.unexpectedError(err), | |
| 695 | } | |
| 696 | } | |
| 697 | total += n; | |
| 698 | truncate += n; | |
| 699 | } | |
| 700 | while (index < data.len and truncate >= data[index].len) { | |
| 701 | truncate -= data[index].len; | |
| 702 | index += 1; | |
| 703 | } | |
| 704 | } | |
| 705 | return total; | |
| 706 | } | |
| 707 | ||
| 708 | var iovecs_buffer: [max_iovecs_len]posix.iovec = undefined; | |
| 709 | var i: usize = 0; | |
| 710 | for (data) |buf| { | |
| 711 | if (iovecs_buffer.len - i == 0) break; | |
| 712 | if (buf.len != 0) { | |
| 713 | iovecs_buffer[i] = .{ .base = buf.ptr, .len = buf.len }; | |
| 714 | i += 1; | |
| 715 | } | |
| 716 | } | |
| 717 | const dest = iovecs_buffer[0..i]; | |
| 718 | assert(dest[0].len > 0); | |
| 719 | ||
| 720 | if (native_os == .wasi and !builtin.link_libc) { | |
| 721 | try pool.checkCancel(); | |
| 722 | var nread: usize = undefined; | |
| 723 | switch (std.os.wasi.fd_read(file.handle, dest.ptr, dest.len, &nread)) { | |
| 724 | .SUCCESS => return nread, | |
| 725 | .INTR => unreachable, | |
| 726 | .INVAL => unreachable, | |
| 727 | .FAULT => unreachable, | |
| 728 | .AGAIN => unreachable, // currently not support in WASI | |
| 729 | .BADF => return error.NotOpenForReading, // can be a race condition | |
| 730 | .IO => return error.InputOutput, | |
| 731 | .ISDIR => return error.IsDir, | |
| 732 | .NOBUFS => return error.SystemResources, | |
| 733 | .NOMEM => return error.SystemResources, | |
| 734 | .NOTCONN => return error.SocketNotConnected, | |
| 735 | .CONNRESET => return error.ConnectionResetByPeer, | |
| 736 | .TIMEDOUT => return error.ConnectionTimedOut, | |
| 737 | .NOTCAPABLE => return error.AccessDenied, | |
| 738 | else => |err| return posix.unexpectedErrno(err), | |
| 739 | } | |
| 740 | } | |
| 741 | ||
| 742 | while (true) { | |
| 743 | try pool.checkCancel(); | |
| 744 | const rc = posix.system.readv(file.handle, dest.ptr, dest.len); | |
| 745 | switch (posix.errno(rc)) { | |
| 746 | .SUCCESS => return @intCast(rc), | |
| 747 | .INTR => continue, | |
| 748 | .INVAL => unreachable, | |
| 749 | .FAULT => unreachable, | |
| 750 | .SRCH => return error.ProcessNotFound, | |
| 751 | .AGAIN => return error.WouldBlock, | |
| 752 | .BADF => return error.NotOpenForReading, // can be a race condition | |
| 753 | .IO => return error.InputOutput, | |
| 754 | .ISDIR => return error.IsDir, | |
| 755 | .NOBUFS => return error.SystemResources, | |
| 756 | .NOMEM => return error.SystemResources, | |
| 757 | .NOTCONN => return error.SocketNotConnected, | |
| 758 | .CONNRESET => return error.ConnectionResetByPeer, | |
| 759 | .TIMEDOUT => return error.ConnectionTimedOut, | |
| 760 | else => |err| return posix.unexpectedErrno(err), | |
| 761 | } | |
| 762 | } | |
| 763 | } | |
| 764 | ||
| 765 | fn fileReadPositional(userdata: ?*anyopaque, file: Io.File, data: [][]u8, offset: u64) Io.File.ReadPositionalError!usize { | |
| 766 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 767 | ||
| 768 | const have_pread_but_not_preadv = switch (native_os) { | |
| 769 | .windows, .macos, .ios, .watchos, .tvos, .visionos, .haiku, .serenity => true, | |
| 770 | else => false, | |
| 771 | }; | |
| 772 | if (have_pread_but_not_preadv) { | |
| 773 | @compileError("TODO"); | |
| 774 | } | |
| 775 | ||
| 776 | if (is_windows) { | |
| 777 | const DWORD = windows.DWORD; | |
| 778 | const OVERLAPPED = windows.OVERLAPPED; | |
| 779 | var index: usize = 0; | |
| 780 | var truncate: usize = 0; | |
| 781 | var total: usize = 0; | |
| 782 | while (true) { | |
| 783 | try pool.checkCancel(); | |
| 784 | { | |
| 785 | const untruncated = data[index]; | |
| 786 | data[index] = untruncated[truncate..]; | |
| 787 | defer data[index] = untruncated; | |
| 788 | const buffer = data[index..]; | |
| 789 | const want_read_count: DWORD = @min(std.math.maxInt(DWORD), buffer.len); | |
| 790 | var n: DWORD = undefined; | |
| 791 | var overlapped_data: OVERLAPPED = undefined; | |
| 792 | const overlapped: ?*OVERLAPPED = if (offset) |off| blk: { | |
| 793 | overlapped_data = .{ | |
| 794 | .Internal = 0, | |
| 795 | .InternalHigh = 0, | |
| 796 | .DUMMYUNIONNAME = .{ | |
| 797 | .DUMMYSTRUCTNAME = .{ | |
| 798 | .Offset = @as(u32, @truncate(off)), | |
| 799 | .OffsetHigh = @as(u32, @truncate(off >> 32)), | |
| 800 | }, | |
| 801 | }, | |
| 802 | .hEvent = null, | |
| 803 | }; | |
| 804 | break :blk &overlapped_data; | |
| 805 | } else null; | |
| 806 | if (windows.kernel32.ReadFile(file.handle, buffer.ptr, want_read_count, &n, overlapped) == 0) { | |
| 807 | switch (windows.GetLastError()) { | |
| 808 | .IO_PENDING => unreachable, | |
| 809 | .OPERATION_ABORTED => continue, | |
| 810 | .BROKEN_PIPE => return 0, | |
| 811 | .HANDLE_EOF => return 0, | |
| 812 | .NETNAME_DELETED => return error.ConnectionResetByPeer, | |
| 813 | .LOCK_VIOLATION => return error.LockViolation, | |
| 814 | .ACCESS_DENIED => return error.AccessDenied, | |
| 815 | .INVALID_HANDLE => return error.NotOpenForReading, | |
| 816 | else => |err| return windows.unexpectedError(err), | |
| 817 | } | |
| 818 | } | |
| 819 | total += n; | |
| 820 | truncate += n; | |
| 821 | } | |
| 822 | while (index < data.len and truncate >= data[index].len) { | |
| 823 | truncate -= data[index].len; | |
| 824 | index += 1; | |
| 825 | } | |
| 826 | } | |
| 827 | return total; | |
| 828 | } | |
| 829 | ||
| 830 | var iovecs_buffer: [max_iovecs_len]posix.iovec = undefined; | |
| 831 | var i: usize = 0; | |
| 832 | for (data) |buf| { | |
| 833 | if (iovecs_buffer.len - i == 0) break; | |
| 834 | if (buf.len != 0) { | |
| 835 | iovecs_buffer[i] = .{ .base = buf.ptr, .len = buf.len }; | |
| 836 | i += 1; | |
| 837 | } | |
| 838 | } | |
| 839 | const dest = iovecs_buffer[0..i]; | |
| 840 | assert(dest[0].len > 0); | |
| 841 | ||
| 842 | if (native_os == .wasi and !builtin.link_libc) { | |
| 843 | try pool.checkCancel(); | |
| 844 | var nread: usize = undefined; | |
| 845 | switch (std.os.wasi.fd_pread(file.handle, dest.ptr, dest.len, offset, &nread)) { | |
| 846 | .SUCCESS => return nread, | |
| 847 | .INTR => unreachable, | |
| 848 | .INVAL => unreachable, | |
| 849 | .FAULT => unreachable, | |
| 850 | .AGAIN => unreachable, | |
| 851 | .BADF => return error.NotOpenForReading, // can be a race condition | |
| 852 | .IO => return error.InputOutput, | |
| 853 | .ISDIR => return error.IsDir, | |
| 854 | .NOBUFS => return error.SystemResources, | |
| 855 | .NOMEM => return error.SystemResources, | |
| 856 | .NOTCONN => return error.SocketNotConnected, | |
| 857 | .CONNRESET => return error.ConnectionResetByPeer, | |
| 858 | .TIMEDOUT => return error.ConnectionTimedOut, | |
| 859 | .NXIO => return error.Unseekable, | |
| 860 | .SPIPE => return error.Unseekable, | |
| 861 | .OVERFLOW => return error.Unseekable, | |
| 862 | .NOTCAPABLE => return error.AccessDenied, | |
| 863 | else => |err| return posix.unexpectedErrno(err), | |
| 864 | } | |
| 865 | } | |
| 866 | ||
| 867 | const preadv_sym = if (posix.lfs64_abi) posix.system.preadv64 else posix.system.preadv; | |
| 868 | while (true) { | |
| 869 | try pool.checkCancel(); | |
| 870 | const rc = preadv_sym(file.handle, dest.ptr, dest.len, @bitCast(offset)); | |
| 871 | switch (posix.errno(rc)) { | |
| 872 | .SUCCESS => return @bitCast(rc), | |
| 873 | .INTR => continue, | |
| 874 | .INVAL => unreachable, | |
| 875 | .FAULT => unreachable, | |
| 876 | .SRCH => return error.ProcessNotFound, | |
| 877 | .AGAIN => return error.WouldBlock, | |
| 878 | .BADF => return error.NotOpenForReading, // can be a race condition | |
| 879 | .IO => return error.InputOutput, | |
| 880 | .ISDIR => return error.IsDir, | |
| 881 | .NOBUFS => return error.SystemResources, | |
| 882 | .NOMEM => return error.SystemResources, | |
| 883 | .NOTCONN => return error.SocketNotConnected, | |
| 884 | .CONNRESET => return error.ConnectionResetByPeer, | |
| 885 | .TIMEDOUT => return error.ConnectionTimedOut, | |
| 886 | .NXIO => return error.Unseekable, | |
| 887 | .SPIPE => return error.Unseekable, | |
| 888 | .OVERFLOW => return error.Unseekable, | |
| 889 | else => |err| return posix.unexpectedErrno(err), | |
| 890 | } | |
| 891 | } | |
| 892 | } | |
| 893 | ||
| 894 | fn fileSeekBy(userdata: ?*anyopaque, file: Io.File, offset: i64) Io.File.SeekError!void { | |
| 895 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 896 | try pool.checkCancel(); | |
| 897 | ||
| 898 | _ = file; | |
| 899 | _ = offset; | |
| 900 | @panic("TODO"); | |
| 901 | } | |
| 902 | ||
| 903 | fn fileSeekTo(userdata: ?*anyopaque, file: Io.File, offset: u64) Io.File.SeekError!void { | |
| 904 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 905 | try pool.checkCancel(); | |
| 906 | ||
| 907 | _ = file; | |
| 908 | _ = offset; | |
| 909 | @panic("TODO"); | |
| 910 | } | |
| 911 | ||
| 912 | fn pwrite(userdata: ?*anyopaque, file: Io.File, buffer: []const u8, offset: posix.off_t) Io.File.PWriteError!usize { | |
| 913 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 914 | try pool.checkCancel(); | |
| 915 | const fs_file: std.fs.File = .{ .handle = file.handle }; | |
| 916 | return switch (offset) { | |
| 917 | -1 => fs_file.write(buffer), | |
| 918 | else => fs_file.pwrite(buffer, @bitCast(offset)), | |
| 919 | }; | |
| 920 | } | |
| 921 | ||
| 922 | fn now(userdata: ?*anyopaque, clockid: posix.clockid_t) Io.ClockGetTimeError!Io.Timestamp { | |
| 923 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 924 | try pool.checkCancel(); | |
| 925 | const timespec = try posix.clock_gettime(clockid); | |
| 926 | return @enumFromInt(@as(i128, timespec.sec) * std.time.ns_per_s + timespec.nsec); | |
| 927 | } | |
| 928 | ||
| 929 | fn sleep(userdata: ?*anyopaque, clockid: posix.clockid_t, deadline: Io.Deadline) Io.SleepError!void { | |
| 930 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 931 | const deadline_nanoseconds: i96 = switch (deadline) { | |
| 932 | .duration => |duration| duration.nanoseconds, | |
| 933 | .timestamp => |timestamp| @intFromEnum(timestamp), | |
| 934 | }; | |
| 935 | var timespec: posix.timespec = .{ | |
| 936 | .sec = @intCast(@divFloor(deadline_nanoseconds, std.time.ns_per_s)), | |
| 937 | .nsec = @intCast(@mod(deadline_nanoseconds, std.time.ns_per_s)), | |
| 938 | }; | |
| 939 | while (true) { | |
| 940 | try pool.checkCancel(); | |
| 941 | switch (std.os.linux.E.init(std.os.linux.clock_nanosleep(clockid, .{ .ABSTIME = switch (deadline) { | |
| 942 | .duration => false, | |
| 943 | .timestamp => true, | |
| 944 | } }, ×pec, ×pec))) { | |
| 945 | .SUCCESS => return, | |
| 946 | .FAULT => unreachable, | |
| 947 | .INTR => {}, | |
| 948 | .INVAL => return error.UnsupportedClock, | |
| 949 | else => |err| return posix.unexpectedErrno(err), | |
| 950 | } | |
| 951 | } | |
| 952 | } | |
| 953 | ||
| 954 | fn select(userdata: ?*anyopaque, futures: []const *Io.AnyFuture) usize { | |
| 955 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 956 | _ = pool; | |
| 957 | ||
| 958 | var reset_event: std.Thread.ResetEvent = .{}; | |
| 959 | ||
| 960 | for (futures, 0..) |future, i| { | |
| 961 | const closure: *AsyncClosure = @ptrCast(@alignCast(future)); | |
| 962 | if (@atomicRmw(?*std.Thread.ResetEvent, &closure.select_condition, .Xchg, &reset_event, .seq_cst) == AsyncClosure.done_reset_event) { | |
| 963 | for (futures[0..i]) |cleanup_future| { | |
| 964 | const cleanup_closure: *AsyncClosure = @ptrCast(@alignCast(cleanup_future)); | |
| 965 | if (@atomicRmw(?*std.Thread.ResetEvent, &cleanup_closure.select_condition, .Xchg, null, .seq_cst) == AsyncClosure.done_reset_event) { | |
| 966 | cleanup_closure.reset_event.wait(); // Ensure no reference to our stack-allocated reset_event. | |
| 967 | } | |
| 968 | } | |
| 969 | return i; | |
| 970 | } | |
| 971 | } | |
| 972 | ||
| 973 | reset_event.wait(); | |
| 974 | ||
| 975 | var result: ?usize = null; | |
| 976 | for (futures, 0..) |future, i| { | |
| 977 | const closure: *AsyncClosure = @ptrCast(@alignCast(future)); | |
| 978 | if (@atomicRmw(?*std.Thread.ResetEvent, &closure.select_condition, .Xchg, null, .seq_cst) == AsyncClosure.done_reset_event) { | |
| 979 | closure.reset_event.wait(); // Ensure no reference to our stack-allocated reset_event. | |
| 980 | if (result == null) result = i; // In case multiple are ready, return first. | |
| 981 | } | |
| 982 | } | |
| 983 | return result.?; | |
| 984 | } | |
| 985 | ||
| 986 | fn listen(userdata: ?*anyopaque, address: Io.net.IpAddress, options: Io.net.ListenOptions) Io.net.ListenError!Io.net.Server { | |
| 987 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 988 | try pool.checkCancel(); | |
| 989 | ||
| 990 | const nonblock: u32 = if (options.force_nonblocking) posix.SOCK.NONBLOCK else 0; | |
| 991 | const sock_flags = posix.SOCK.STREAM | posix.SOCK.CLOEXEC | nonblock; | |
| 992 | const proto: u32 = posix.IPPROTO.TCP; | |
| 993 | const family = posixAddressFamily(address); | |
| 994 | const sockfd = try posix.socket(family, sock_flags, proto); | |
| 995 | const stream: std.net.Stream = .{ .handle = sockfd }; | |
| 996 | errdefer stream.close(); | |
| 997 | ||
| 998 | if (options.reuse_address) { | |
| 999 | try posix.setsockopt( | |
| 1000 | sockfd, | |
| 1001 | posix.SOL.SOCKET, | |
| 1002 | posix.SO.REUSEADDR, | |
| 1003 | &std.mem.toBytes(@as(c_int, 1)), | |
| 1004 | ); | |
| 1005 | if (@hasDecl(posix.SO, "REUSEPORT") and family != posix.AF.UNIX) { | |
| 1006 | try posix.setsockopt( | |
| 1007 | sockfd, | |
| 1008 | posix.SOL.SOCKET, | |
| 1009 | posix.SO.REUSEPORT, | |
| 1010 | &std.mem.toBytes(@as(c_int, 1)), | |
| 1011 | ); | |
| 1012 | } | |
| 1013 | } | |
| 1014 | ||
| 1015 | var storage: PosixAddress = undefined; | |
| 1016 | var socklen = addressToPosix(address, &storage); | |
| 1017 | try posix.bind(sockfd, &storage.any, socklen); | |
| 1018 | try posix.listen(sockfd, options.kernel_backlog); | |
| 1019 | try posix.getsockname(sockfd, &storage.any, &socklen); | |
| 1020 | return .{ | |
| 1021 | .listen_address = addressFromPosix(&storage), | |
| 1022 | .stream = .{ .handle = stream.handle }, | |
| 1023 | }; | |
| 1024 | } | |
| 1025 | ||
| 1026 | fn accept(userdata: ?*anyopaque, server: *Io.net.Server) Io.net.Server.AcceptError!Io.net.Server.Connection { | |
| 1027 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 1028 | try pool.checkCancel(); | |
| 1029 | ||
| 1030 | var storage: PosixAddress = undefined; | |
| 1031 | var addr_len: posix.socklen_t = @sizeOf(PosixAddress); | |
| 1032 | const fd = try posix.accept(server.stream.handle, &storage.any, &addr_len, posix.SOCK.CLOEXEC); | |
| 1033 | return .{ | |
| 1034 | .stream = .{ .handle = fd }, | |
| 1035 | .address = addressFromPosix(&storage), | |
| 1036 | }; | |
| 1037 | } | |
| 1038 | ||
| 1039 | fn netReadPosix(userdata: ?*anyopaque, stream: Io.net.Stream, data: [][]u8) Io.net.Stream.Reader.Error!usize { | |
| 1040 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 1041 | try pool.checkCancel(); | |
| 1042 | ||
| 1043 | var iovecs_buffer: [max_iovecs_len]posix.iovec = undefined; | |
| 1044 | var i: usize = 0; | |
| 1045 | for (data) |buf| { | |
| 1046 | if (iovecs_buffer.len - i == 0) break; | |
| 1047 | if (buf.len != 0) { | |
| 1048 | iovecs_buffer[i] = .{ .base = buf.ptr, .len = buf.len }; | |
| 1049 | i += 1; | |
| 1050 | } | |
| 1051 | } | |
| 1052 | const dest = iovecs_buffer[0..i]; | |
| 1053 | assert(dest[0].len > 0); | |
| 1054 | const n = try posix.readv(stream.handle, dest); | |
| 1055 | if (n == 0) return error.EndOfStream; | |
| 1056 | return n; | |
| 1057 | } | |
| 1058 | ||
| 1059 | fn netWritePosix( | |
| 1060 | userdata: ?*anyopaque, | |
| 1061 | stream: Io.net.Stream, | |
| 1062 | header: []const u8, | |
| 1063 | data: []const []const u8, | |
| 1064 | splat: usize, | |
| 1065 | ) Io.net.Stream.Writer.Error!usize { | |
| 1066 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 1067 | try pool.checkCancel(); | |
| 1068 | ||
| 1069 | var iovecs: [max_iovecs_len]posix.iovec_const = undefined; | |
| 1070 | var msg: posix.msghdr_const = .{ | |
| 1071 | .name = null, | |
| 1072 | .namelen = 0, | |
| 1073 | .iov = &iovecs, | |
| 1074 | .iovlen = 0, | |
| 1075 | .control = null, | |
| 1076 | .controllen = 0, | |
| 1077 | .flags = 0, | |
| 1078 | }; | |
| 1079 | addBuf(&iovecs, &msg.iovlen, header); | |
| 1080 | for (data[0 .. data.len - 1]) |bytes| addBuf(&iovecs, &msg.iovlen, bytes); | |
| 1081 | const pattern = data[data.len - 1]; | |
| 1082 | if (iovecs.len - msg.iovlen != 0) switch (splat) { | |
| 1083 | 0 => {}, | |
| 1084 | 1 => addBuf(&iovecs, &msg.iovlen, pattern), | |
| 1085 | else => switch (pattern.len) { | |
| 1086 | 0 => {}, | |
| 1087 | 1 => { | |
| 1088 | var backup_buffer: [splat_buffer_size]u8 = undefined; | |
| 1089 | const splat_buffer = &backup_buffer; | |
| 1090 | const memset_len = @min(splat_buffer.len, splat); | |
| 1091 | const buf = splat_buffer[0..memset_len]; | |
| 1092 | @memset(buf, pattern[0]); | |
| 1093 | addBuf(&iovecs, &msg.iovlen, buf); | |
| 1094 | var remaining_splat = splat - buf.len; | |
| 1095 | while (remaining_splat > splat_buffer.len and iovecs.len - msg.iovlen != 0) { | |
| 1096 | assert(buf.len == splat_buffer.len); | |
| 1097 | addBuf(&iovecs, &msg.iovlen, splat_buffer); | |
| 1098 | remaining_splat -= splat_buffer.len; | |
| 1099 | } | |
| 1100 | addBuf(&iovecs, &msg.iovlen, splat_buffer[0..remaining_splat]); | |
| 1101 | }, | |
| 1102 | else => for (0..@min(splat, iovecs.len - msg.iovlen)) |_| { | |
| 1103 | addBuf(&iovecs, &msg.iovlen, pattern); | |
| 1104 | }, | |
| 1105 | }, | |
| 1106 | }; | |
| 1107 | const flags = posix.MSG.NOSIGNAL; | |
| 1108 | return posix.sendmsg(stream.handle, &msg, flags); | |
| 1109 | } | |
| 1110 | ||
| 1111 | fn addBuf(v: []posix.iovec_const, i: *@FieldType(posix.msghdr_const, "iovlen"), bytes: []const u8) void { | |
| 1112 | // OS checks ptr addr before length so zero length vectors must be omitted. | |
| 1113 | if (bytes.len == 0) return; | |
| 1114 | if (v.len - i.* == 0) return; | |
| 1115 | v[i.*] = .{ .base = bytes.ptr, .len = bytes.len }; | |
| 1116 | i.* += 1; | |
| 1117 | } | |
| 1118 | ||
| 1119 | fn netClose(userdata: ?*anyopaque, stream: Io.net.Stream) void { | |
| 1120 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 1121 | _ = pool; | |
| 1122 | const net_stream: std.net.Stream = .{ .handle = stream.handle }; | |
| 1123 | return net_stream.close(); | |
| 1124 | } | |
| 1125 | ||
| 1126 | fn netInterfaceIndex(userdata: ?*anyopaque, name: []const u8) Io.net.InterfaceIndexError!u32 { | |
| 1127 | const pool: *Pool = @ptrCast(@alignCast(userdata)); | |
| 1128 | try pool.checkCancel(); | |
| 1129 | ||
| 1130 | if (native_os == .linux) { | |
| 1131 | if (name.len >= posix.IFNAMESIZE) return error.InterfaceNotFound; | |
| 1132 | var ifr: posix.ifreq = undefined; | |
| 1133 | @memcpy(ifr.ifrn.name[0..name.len], name); | |
| 1134 | ifr.ifrn.name[name.len] = 0; | |
| 1135 | ||
| 1136 | const rc = posix.system.socket(posix.AF.UNIX, posix.SOCK.DGRAM | posix.SOCK.CLOEXEC, 0); | |
| 1137 | const sock_fd: posix.fd_t = switch (posix.errno(rc)) { | |
| 1138 | .SUCCESS => @intCast(rc), | |
| 1139 | .ACCES => return error.AccessDenied, | |
| 1140 | .MFILE => return error.SystemResources, | |
| 1141 | .NFILE => return error.SystemResources, | |
| 1142 | .NOBUFS => return error.SystemResources, | |
| 1143 | .NOMEM => return error.SystemResources, | |
| 1144 | else => |err| return posix.unexpectedErrno(err), | |
| 1145 | }; | |
| 1146 | defer posix.close(sock_fd); | |
| 1147 | ||
| 1148 | while (true) { | |
| 1149 | try pool.checkCancel(); | |
| 1150 | switch (posix.errno(posix.system.ioctl(sock_fd, posix.SIOCGIFINDEX, @intFromPtr(&ifr)))) { | |
| 1151 | .SUCCESS => return @bitCast(ifr.ifru.ivalue), | |
| 1152 | .INVAL => |err| return badErrno(err), // Bad parameters. | |
| 1153 | .NOTTY => |err| return badErrno(err), | |
| 1154 | .NXIO => |err| return badErrno(err), | |
| 1155 | .BADF => |err| return badErrno(err), // Always a race condition. | |
| 1156 | .FAULT => |err| return badErrno(err), // Bad pointer parameter. | |
| 1157 | .INTR => continue, | |
| 1158 | .IO => |err| return badErrno(err), // sock_fd is not a file descriptor | |
| 1159 | .NODEV => return error.InterfaceNotFound, | |
| 1160 | else => |err| return posix.unexpectedErrno(err), | |
| 1161 | } | |
| 1162 | } | |
| 1163 | } | |
| 1164 | ||
| 1165 | if (native_os.isDarwin()) { | |
| 1166 | if (name.len >= posix.IFNAMESIZE) return error.InterfaceNotFound; | |
| 1167 | var if_name: [posix.IFNAMESIZE:0]u8 = undefined; | |
| 1168 | @memcpy(if_name[0..name.len], name); | |
| 1169 | if_name[name.len] = 0; | |
| 1170 | const if_slice = if_name[0..name.len :0]; | |
| 1171 | const index = std.c.if_nametoindex(if_slice); | |
| 1172 | if (index == 0) return error.InterfaceNotFound; | |
| 1173 | return @bitCast(index); | |
| 1174 | } | |
| 1175 | ||
| 1176 | if (native_os == .windows) { | |
| 1177 | if (name.len >= posix.IFNAMESIZE) return error.InterfaceNotFound; | |
| 1178 | var interface_name: [posix.IFNAMESIZE:0]u8 = undefined; | |
| 1179 | @memcpy(interface_name[0..name.len], name); | |
| 1180 | interface_name[name.len] = 0; | |
| 1181 | const index = std.os.windows.ws2_32.if_nametoindex(@as([*:0]const u8, &interface_name)); | |
| 1182 | if (index == 0) return error.InterfaceNotFound; | |
| 1183 | return index; | |
| 1184 | } | |
| 1185 | ||
| 1186 | @compileError("std.net.if_nametoindex unimplemented for this OS"); | |
| 1187 | } | |
| 1188 | ||
| 1189 | const PosixAddress = extern union { | |
| 1190 | any: posix.sockaddr, | |
| 1191 | in: posix.sockaddr.in, | |
| 1192 | in6: posix.sockaddr.in6, | |
| 1193 | }; | |
| 1194 | ||
| 1195 | fn posixAddressFamily(a: Io.net.IpAddress) posix.sa_family_t { | |
| 1196 | return switch (a) { | |
| 1197 | .ip4 => posix.AF.INET, | |
| 1198 | .ip6 => posix.AF.INET6, | |
| 1199 | }; | |
| 1200 | } | |
| 1201 | ||
| 1202 | fn addressFromPosix(posix_address: *PosixAddress) Io.net.IpAddress { | |
| 1203 | return switch (posix_address.any.family) { | |
| 1204 | posix.AF.INET => .{ .ip4 = address4FromPosix(&posix_address.in) }, | |
| 1205 | posix.AF.INET6 => .{ .ip6 = address6FromPosix(&posix_address.in6) }, | |
| 1206 | else => unreachable, | |
| 1207 | }; | |
| 1208 | } | |
| 1209 | ||
| 1210 | fn addressToPosix(a: Io.net.IpAddress, storage: *PosixAddress) posix.socklen_t { | |
| 1211 | return switch (a) { | |
| 1212 | .ip4 => |ip4| { | |
| 1213 | storage.in = address4ToPosix(ip4); | |
| 1214 | return @sizeOf(posix.sockaddr.in); | |
| 1215 | }, | |
| 1216 | .ip6 => |ip6| { | |
| 1217 | storage.in6 = address6ToPosix(ip6); | |
| 1218 | return @sizeOf(posix.sockaddr.in6); | |
| 1219 | }, | |
| 1220 | }; | |
| 1221 | } | |
| 1222 | ||
| 1223 | fn address4FromPosix(in: *posix.sockaddr.in) Io.net.Ip4Address { | |
| 1224 | return .{ | |
| 1225 | .port = std.mem.bigToNative(u16, in.port), | |
| 1226 | .bytes = @bitCast(in.addr), | |
| 1227 | }; | |
| 1228 | } | |
| 1229 | ||
| 1230 | fn address6FromPosix(in6: *posix.sockaddr.in6) Io.net.Ip6Address { | |
| 1231 | return .{ | |
| 1232 | .port = std.mem.bigToNative(u16, in6.port), | |
| 1233 | .bytes = in6.addr, | |
| 1234 | .flowinfo = in6.flowinfo, | |
| 1235 | .scope_id = in6.scope_id, | |
| 1236 | }; | |
| 1237 | } | |
| 1238 | ||
| 1239 | fn address4ToPosix(a: Io.net.Ip4Address) posix.sockaddr.in { | |
| 1240 | return .{ | |
| 1241 | .port = std.mem.nativeToBig(u16, a.port), | |
| 1242 | .addr = @bitCast(a.bytes), | |
| 1243 | }; | |
| 1244 | } | |
| 1245 | ||
| 1246 | fn address6ToPosix(a: Io.net.Ip6Address) posix.sockaddr.in6 { | |
| 1247 | return .{ | |
| 1248 | .port = std.mem.nativeToBig(u16, a.port), | |
| 1249 | .flowinfo = a.flowinfo, | |
| 1250 | .addr = a.bytes, | |
| 1251 | .scope_id = a.scope_id, | |
| 1252 | }; | |
| 1253 | } | |
| 1254 | ||
| 1255 | fn badErrno(err: posix.E) Io.UnexpectedError { | |
| 1256 | switch (builtin.mode) { | |
| 1257 | .Debug => std.debug.panic("programmer bug caused syscall error: {t}", .{err}), | |
| 1258 | else => return error.Unexpected, | |
| 1259 | } | |
| 1260 | } |