authorgravatar for robin@voetter.nlRobin Voetter <robin@voetter.nl> 2022-09-18 15:27:20+02:00
committergravatar for robin@voetter.nlRobin Voetter <robin@voetter.nl> 2022-10-12 20:36:14+02:00
log9006cd9d09da083e89b58e19c0091924e1e4849f
tree5f268055e142c85b72a57480ba56d1c5d81817b9
parent8894d1c45eb01fa3fbcc9173bac729e5812307ed
signaturelock-open Commit is signed but in an unrecognized format.

compiler_rt: cmpxchg-based atomic fetch/exchange for small types

Some architectures (AMDGPU) do not support atomic exchange/fetch for small types (for AMDGPU: 8- and 16-bit ints). For these types atomic fetch and atomic exchange needs to be implemeted using atomic operations on a wider type using cmpxchg.

1 files changed, 68 insertions(+), 0 deletions(-)

lib/compiler_rt/atomics.zig+68
...@@ -35,6 +35,17 @@ const largest_atomic_size = switch (arch) {...@@ -35,6 +35,17 @@ const largest_atomic_size = switch (arch) {
35 else => @sizeOf(usize),35 else => @sizeOf(usize),
36};36};
3737
38// The size (in bytes) of the smallest atomic object that the architecture can
39// perform fetch/exchange atomically. Note, this does not encompass load and store.
40// Objects smaller than this threshold are implemented in terms of compare-exchange
41// of a larger value.
42const smallest_atomic_fetch_exch_size = switch (arch) {
43 // On AMDGPU, there are no instructions for atomic operations other than load and store
44 // (as of LLVM 15), and so these need to be implemented in terms of atomic CAS.
45 .amdgcn => @sizeOf(u32),
46 else => @sizeOf(u8),
47};
48
38const cache_line_size = 64;49const cache_line_size = 64;
3950
40const SpinlockTable = struct {51const SpinlockTable = struct {
...@@ -214,6 +225,31 @@ inline fn atomic_exchange_N(comptime T: type, ptr: *T, val: T, model: i32) T {...@@ -214,6 +225,31 @@ inline fn atomic_exchange_N(comptime T: type, ptr: *T, val: T, model: i32) T {
214 const value = ptr.*;225 const value = ptr.*;
215 ptr.* = val;226 ptr.* = val;
216 return value;227 return value;
228 } else if (@sizeOf(T) < smallest_atomic_fetch_exch_size) {
229 // Machine does not support this type, but it does support a larger type.
230 const WideAtomic = std.meta.Int(.unsigned, smallest_atomic_fetch_exch_size * 8);
231
232 const addr = @ptrToInt(ptr);
233 const wide_addr = addr & ~(@as(T, smallest_atomic_fetch_exch_size) - 1);
234 const wide_ptr = @alignCast(smallest_atomic_fetch_exch_size, @intToPtr(*WideAtomic, wide_addr));
235
236 const inner_offset = addr & (@as(T, smallest_atomic_fetch_exch_size) - 1);
237 const inner_shift = @intCast(std.math.Log2Int(T), inner_offset * 8);
238
239 // Put the interesting bits at the right position (branch has dynamic RHS).
240 const shifted_value = @as(WideAtomic, val) << inner_shift;
241 // Mask that guards the bits we care about
242 const mask = @as(WideAtomic, std.math.maxInt(T)) << inner_shift;
243 while (true) {
244 const wide_old = @atomicLoad(WideAtomic, wide_ptr, .Acquire);
245 // Insert new bytes in old value.
246 const wide_new = wide_old & ~mask | shifted_value;
247 // CAS the new value until the result stabilizes.
248 if (@cmpxchgWeak(WideAtomic, wide_ptr, wide_old, wide_new, .SeqCst, .SeqCst) == null) {
249 // Mask-and-Shift back the old bits to get the old value.
250 return @truncate(T, (wide_old & mask) >> inner_shift);
251 }
252 }
217 } else {253 } else {
218 return @atomicRmw(T, ptr, .Xchg, val, .SeqCst);254 return @atomicRmw(T, ptr, .Xchg, val, .SeqCst);
219 }255 }
...@@ -298,6 +334,38 @@ inline fn fetch_op_N(comptime T: type, comptime op: std.builtin.AtomicRmwOp, ptr...@@ -298,6 +334,38 @@ inline fn fetch_op_N(comptime T: type, comptime op: std.builtin.AtomicRmwOp, ptr
298 };334 };
299335
300 return value;336 return value;
337 } else if (@sizeOf(T) < smallest_atomic_fetch_exch_size) {
338 // Machine does not support this type, but it does support a larger type.
339 const WideAtomic = std.meta.Int(.unsigned, smallest_atomic_fetch_exch_size * 8);
340
341 const addr = @ptrToInt(ptr);
342 const wide_addr = addr & ~(@as(T, smallest_atomic_fetch_exch_size) - 1);
343 const wide_ptr = @alignCast(smallest_atomic_fetch_exch_size, @intToPtr(*WideAtomic, wide_addr));
344
345 const inner_offset = addr & (@as(T, smallest_atomic_fetch_exch_size) - 1);
346 const inner_shift = @intCast(std.math.Log2Int(T), inner_offset * 8);
347
348 const mask = @as(WideAtomic, std.math.maxInt(T)) << inner_shift;
349
350 while (true) {
351 // Compute new wide value with updated bits.
352 const wide_old = @atomicLoad(WideAtomic, wide_ptr, .Acquire);
353 const old = @truncate(T, (wide_old & mask) >> inner_shift);
354 const new = switch (op) {
355 .Add => old +% val,
356 .Sub => old -% val,
357 .And => old & val,
358 .Nand => ~(old & val),
359 .Or => old | val,
360 .Xor => old ^ val,
361 else => @compileError("unsupported atomic op"),
362 };
363 const wide_new = wide_old & ~mask | (@as(WideAtomic, new) << inner_shift);
364 // CAS the new value until the result stabilizes.
365 if (@cmpxchgWeak(WideAtomic, wide_ptr, wide_old, wide_new, .SeqCst, .SeqCst) == null) {
366 return old;
367 }
368 }
301 }369 }
302370
303 return @atomicRmw(T, ptr, op, val, .SeqCst);371 return @atomicRmw(T, ptr, op, val, .SeqCst);