authorgravatar for robin@voetter.nlRobin Voetter <robin@voetter.nl> 2022-09-18 16:19:34+02:00
committergravatar for robin@voetter.nlRobin Voetter <robin@voetter.nl> 2022-10-12 20:36:15+02:00
logaa20295d24e77a3de586b203182d4e1cad69d475
treec207e1bd2679ec58f6c9bb7999d5907074ccf431
parent86f40d3ff6529baaeba1f0e7e5d7d2f0bfce00a1
signature Commit is signed but in an unrecognized format.

compiler_rt: apply protty suggestions

1 files changed, 47 insertions(+), 62 deletions(-)

lib/compiler_rt/atomics.zig+47-62
......@@ -217,6 +217,31 @@ fn __atomic_store_8(dst: *u64, value: u64, model: i32) callconv(.C) void {
217217 return atomic_store_N(u64, dst, value, model);
218218}
219219
220fn wideUpdate(comptime T: type, ptr: *T, val: T, update: anytype) T {
221 const WideAtomic = std.meta.Int(.unsigned, smallest_atomic_fetch_exch_size * 8);
222
223 const addr = @ptrToInt(ptr);
224 const wide_addr = addr & ~(@as(T, smallest_atomic_fetch_exch_size) - 1);
225 const wide_ptr = @alignCast(smallest_atomic_fetch_exch_size, @intToPtr(*WideAtomic, wide_addr));
226
227 const inner_offset = addr & (@as(T, smallest_atomic_fetch_exch_size) - 1);
228 const inner_shift = @intCast(std.math.Log2Int(T), inner_offset * 8);
229
230 const mask = @as(WideAtomic, std.math.maxInt(T)) << inner_shift;
231
232 var wide_old = @atomicLoad(WideAtomic, wide_ptr, .SeqCst);
233 while (true) {
234 const old = @truncate(T, (wide_old & mask) >> inner_shift);
235 const new = update(val, old);
236 const wide_new = wide_old & ~mask | (@as(WideAtomic, new) << inner_shift);
237 if (@cmpxchgWeak(WideAtomic, wide_ptr, wide_old, wide_new, .SeqCst, .SeqCst)) |new_wide_old| {
238 wide_old = new_wide_old;
239 } else {
240 return old;
241 }
242 }
243}
244
220245inline fn atomic_exchange_N(comptime T: type, ptr: *T, val: T, model: i32) T {
221246 _ = model;
222247 if (@sizeOf(T) > largest_atomic_size) {
......@@ -227,29 +252,13 @@ inline fn atomic_exchange_N(comptime T: type, ptr: *T, val: T, model: i32) T {
227252 return value;
228253 } else if (@sizeOf(T) < smallest_atomic_fetch_exch_size) {
229254 // Machine does not support this type, but it does support a larger type.
230 const WideAtomic = std.meta.Int(.unsigned, smallest_atomic_fetch_exch_size * 8);
231
232 const addr = @ptrToInt(ptr);
233 const wide_addr = addr & ~(@as(T, smallest_atomic_fetch_exch_size) - 1);
234 const wide_ptr = @alignCast(smallest_atomic_fetch_exch_size, @intToPtr(*WideAtomic, wide_addr));
235
236 const inner_offset = addr & (@as(T, smallest_atomic_fetch_exch_size) - 1);
237 const inner_shift = @intCast(std.math.Log2Int(T), inner_offset * 8);
238
239 // Put the interesting bits at the right position (branch has dynamic RHS).
240 const shifted_value = @as(WideAtomic, val) << inner_shift;
241 // Mask that guards the bits we care about
242 const mask = @as(WideAtomic, std.math.maxInt(T)) << inner_shift;
243 while (true) {
244 const wide_old = @atomicLoad(WideAtomic, wide_ptr, .Acquire);
245 // Insert new bytes in old value.
246 const wide_new = wide_old & ~mask | shifted_value;
247 // CAS the new value until the result stabilizes.
248 if (@cmpxchgWeak(WideAtomic, wide_ptr, wide_old, wide_new, .SeqCst, .SeqCst) == null) {
249 // Mask-and-Shift back the old bits to get the old value.
250 return @truncate(T, (wide_old & mask) >> inner_shift);
255 const Updater = struct {
256 fn update(new: T, old: T) T {
257 _ = old;
258 return new;
251259 }
252 }
260 };
261 return wideUpdate(T, ptr, val, Updater.update);
253262 } else {
254263 return @atomicRmw(T, ptr, .Xchg, val, .SeqCst);
255264 }
......@@ -318,54 +327,30 @@ fn __atomic_compare_exchange_8(ptr: *u64, expected: *u64, desired: u64, success:
318327
319328inline fn fetch_op_N(comptime T: type, comptime op: std.builtin.AtomicRmwOp, ptr: *T, val: T, model: i32) T {
320329 _ = model;
330 const Updater = struct {
331 fn update(new: T, old: T) T {
332 return switch (op) {
333 .Add => old +% new,
334 .Sub => old -% new,
335 .And => old & new,
336 .Nand => ~(old & new),
337 .Or => old | new,
338 .Xor => old ^ new,
339 else => @compileError("unsupported atomic op"),
340 };
341 }
342 };
343
321344 if (@sizeOf(T) > largest_atomic_size) {
322345 var sl = spinlocks.get(@ptrToInt(ptr));
323346 defer sl.release();
324347
325348 const value = ptr.*;
326 ptr.* = switch (op) {
327 .Add => value +% val,
328 .Sub => value -% val,
329 .And => value & val,
330 .Nand => ~(value & val),
331 .Or => value | val,
332 .Xor => value ^ val,
333 else => @compileError("unsupported atomic op"),
334 };
335
349 ptr.* = Updater.update(val, value);
336350 return value;
337351 } else if (@sizeOf(T) < smallest_atomic_fetch_exch_size) {
338352 // Machine does not support this type, but it does support a larger type.
339 const WideAtomic = std.meta.Int(.unsigned, smallest_atomic_fetch_exch_size * 8);
340
341 const addr = @ptrToInt(ptr);
342 const wide_addr = addr & ~(@as(T, smallest_atomic_fetch_exch_size) - 1);
343 const wide_ptr = @alignCast(smallest_atomic_fetch_exch_size, @intToPtr(*WideAtomic, wide_addr));
344
345 const inner_offset = addr & (@as(T, smallest_atomic_fetch_exch_size) - 1);
346 const inner_shift = @intCast(std.math.Log2Int(T), inner_offset * 8);
347
348 const mask = @as(WideAtomic, std.math.maxInt(T)) << inner_shift;
349
350 while (true) {
351 // Compute new wide value with updated bits.
352 const wide_old = @atomicLoad(WideAtomic, wide_ptr, .Acquire);
353 const old = @truncate(T, (wide_old & mask) >> inner_shift);
354 const new = switch (op) {
355 .Add => old +% val,
356 .Sub => old -% val,
357 .And => old & val,
358 .Nand => ~(old & val),
359 .Or => old | val,
360 .Xor => old ^ val,
361 else => @compileError("unsupported atomic op"),
362 };
363 const wide_new = wide_old & ~mask | (@as(WideAtomic, new) << inner_shift);
364 // CAS the new value until the result stabilizes.
365 if (@cmpxchgWeak(WideAtomic, wide_ptr, wide_old, wide_new, .SeqCst, .SeqCst) == null) {
366 return old;
367 }
368 }
353 return wideUpdate(T, ptr, val, Updater.update);
369354 }
370355
371356 return @atomicRmw(T, ptr, op, val, .SeqCst);