| ... | ... | @@ -0,0 +1,395 @@ |
| 1 | // SPDX-License-Identifier: MIT |
| 2 | // Copyright (c) 2015-2018 LLVM Compiler Infrastructure |
| 3 | // Copyright (c) 2020 Sebastien Marie <semarie@online.fr> |
| 4 | // This file is part of [zig](https://ziglang.org/), which is MIT licensed. |
| 5 | // The MIT license requires this copyright notice to be included in all copies |
| 6 | // and substantial portions of the software. |
| 7 | // __emutls_get_address specific builtin |
| 8 | // |
| 9 | // derived work from LLVM Compiler Infrastructure - release 8.0 (MIT) |
| 10 | // https://github.com/llvm-mirror/compiler-rt/blob/release_80/lib/builtins/emutls.c |
| 11 | // |
| 12 | |
| 13 | const std = @import("std"); |
| 14 | |
| 15 | const abort = std.os.abort; |
| 16 | const assert = std.debug.assert; |
| 17 | const expect = std.testing.expect; |
| 18 | |
| 19 | // defined in C as: |
| 20 | // typedef unsigned int gcc_word __attribute__((mode(word))); |
| 21 | const gcc_word = usize; |
| 22 | |
| 23 | comptime { |
| 24 | assert(std.builtin.link_libc); |
| 25 | } |
| 26 | |
| 27 | /// public entrypoint for generated code using EmulatedTLS |
| 28 | pub fn __emutls_get_address(control: *emutls_control) callconv(.C) *c_void { |
| 29 | return control.getPointer(); |
| 30 | } |
| 31 | |
| 32 | /// Simple allocator interface, to avoid pulling in the while |
| 33 | /// std allocator implementation. |
| 34 | const simple_allocator = struct { |
| 35 | /// Allocate a memory chunk for requested type. Return a pointer on the data. |
| 36 | pub fn alloc(comptime T: type) *T { |
| 37 | return @ptrCast(*T, @alignCast( |
| 38 | @alignOf(T), |
| 39 | advancedAlloc(@alignOf(T), @sizeOf(T)), |
| 40 | )); |
| 41 | } |
| 42 | |
| 43 | /// Allocate a slice of T, with len elements. |
| 44 | pub fn allocSlice(comptime T: type, len: usize) []T { |
| 45 | return @ptrCast([*]T, @alignCast( |
| 46 | @alignOf(T), |
| 47 | advancedAlloc(@alignOf(T), @sizeOf(T) * len), |
| 48 | ))[0 .. len - 1]; |
| 49 | } |
| 50 | |
| 51 | /// Allocate a memory chunk. |
| 52 | pub fn advancedAlloc(alignment: u29, size: usize) [*]u8 { |
| 53 | const minimal_alignment = std.math.max(@alignOf(usize), alignment); |
| 54 | |
| 55 | var aligned_ptr: ?*c_void = undefined; |
| 56 | if (std.c.posix_memalign(&aligned_ptr, minimal_alignment, size) != 0) { |
| 57 | abort(); |
| 58 | } |
| 59 | |
| 60 | return @ptrCast([*]u8, aligned_ptr); |
| 61 | } |
| 62 | |
| 63 | /// Resize a slice. |
| 64 | pub fn reallocSlice(comptime T: type, slice: []T, len: usize) []T { |
| 65 | var c_ptr: *c_void = @ptrCast(*c_void, slice.ptr); |
| 66 | var new_array: [*]T = @ptrCast([*]T, @alignCast( |
| 67 | @alignOf(T), |
| 68 | std.c.realloc(c_ptr, @sizeOf(T) * len) orelse abort(), |
| 69 | )); |
| 70 | return new_array[0..len]; |
| 71 | } |
| 72 | |
| 73 | /// Free a memory chunk allocated with simple_allocator. |
| 74 | pub fn free(ptr: anytype) void { |
| 75 | std.c.free(@ptrCast(*c_void, ptr)); |
| 76 | } |
| 77 | }; |
| 78 | |
| 79 | /// Simple array of ?ObjectPointer with automatic resizing and |
| 80 | /// automatic storage allocation. |
| 81 | const ObjectArray = struct { |
| 82 | const ObjectPointer = *c_void; |
| 83 | |
| 84 | // content of the array |
| 85 | slots: []?ObjectPointer, |
| 86 | |
| 87 | /// create a new ObjectArray with n slots. must call deinit() to deallocate. |
| 88 | pub fn init(n: usize) *ObjectArray { |
| 89 | var array = simple_allocator.alloc(ObjectArray); |
| 90 | errdefer simple_allocator.free(array); |
| 91 | |
| 92 | array.* = ObjectArray{ |
| 93 | .slots = simple_allocator.allocSlice(?ObjectPointer, n), |
| 94 | }; |
| 95 | errdefer simple_allocator.free(array.slots); |
| 96 | |
| 97 | for (array.slots) |*object| { |
| 98 | object.* = null; |
| 99 | } |
| 100 | |
| 101 | return array; |
| 102 | } |
| 103 | |
| 104 | /// deallocate the ObjectArray. |
| 105 | pub fn deinit(self: *ObjectArray) void { |
| 106 | // deallocated used objects in the array |
| 107 | for (self.slots) |*object| { |
| 108 | simple_allocator.free(object.*); |
| 109 | } |
| 110 | simple_allocator.free(self.slots); |
| 111 | simple_allocator.free(self); |
| 112 | } |
| 113 | |
| 114 | /// resize the ObjectArray if needed. |
| 115 | pub fn ensureLength(self: *ObjectArray, new_len: usize) *ObjectArray { |
| 116 | const old_len = self.slots.len; |
| 117 | |
| 118 | if (old_len > new_len) { |
| 119 | return self; |
| 120 | } |
| 121 | |
| 122 | // reallocate |
| 123 | self.slots = simple_allocator.reallocSlice(?ObjectPointer, self.slots, new_len); |
| 124 | |
| 125 | // init newly added slots |
| 126 | for (self.slots[old_len..]) |*object| { |
| 127 | object.* = null; |
| 128 | } |
| 129 | |
| 130 | return self; |
| 131 | } |
| 132 | |
| 133 | /// Retrieve the pointer at request index, using control to initialize it if needed. |
| 134 | pub fn getPointer(self: *ObjectArray, index: usize, control: *emutls_control) ObjectPointer { |
| 135 | if (self.slots[index] == null) { |
| 136 | // initialize the slot |
| 137 | const size = control.size; |
| 138 | const alignment = @truncate(u29, control.alignment); |
| 139 | |
| 140 | var data = simple_allocator.advancedAlloc(alignment, size); |
| 141 | errdefer simple_allocator.free(data); |
| 142 | |
| 143 | if (control.default_value) |value| { |
| 144 | // default value: copy the content to newly allocated object. |
| 145 | @memcpy(data, @ptrCast([*]u8, value), size); |
| 146 | } else { |
| 147 | // no default: return zeroed memory. |
| 148 | @memset(data, 0, size); |
| 149 | } |
| 150 | |
| 151 | self.slots[index] = @ptrCast(*c_void, data); |
| 152 | } |
| 153 | |
| 154 | return self.slots[index].?; |
| 155 | } |
| 156 | }; |
| 157 | |
| 158 | // Global stucture for Thread Storage. |
| 159 | // It provides thread-safety for on-demand storage of Thread Objects. |
| 160 | const current_thread_storage = struct { |
| 161 | var key: std.c.pthread_key_t = undefined; |
| 162 | var init_once = std.once(current_thread_storage.init); |
| 163 | |
| 164 | /// Return a per thread ObjectArray with at least the expected index. |
| 165 | pub fn getArray(index: usize) *ObjectArray { |
| 166 | if (current_thread_storage.getspecific()) |array| { |
| 167 | // we already have a specific. just ensure the array is |
| 168 | // big enough for the wanted index. |
| 169 | return array.ensureLength(index); |
| 170 | } |
| 171 | |
| 172 | // no specific. we need to create a new array. |
| 173 | |
| 174 | // make it to contains at least 16 objects (to avoid too much |
| 175 | // reallocation at startup). |
| 176 | const size = std.math.max(16, index); |
| 177 | |
| 178 | // create a new array and store it. |
| 179 | var array: *ObjectArray = ObjectArray.init(size); |
| 180 | current_thread_storage.setspecific(array); |
| 181 | return array; |
| 182 | } |
| 183 | |
| 184 | /// Return casted thread specific value. |
| 185 | fn getspecific() ?*ObjectArray { |
| 186 | return @ptrCast( |
| 187 | ?*ObjectArray, |
| 188 | @alignCast( |
| 189 | @alignOf(ObjectArray), |
| 190 | std.c.pthread_getspecific(current_thread_storage.key), |
| 191 | ), |
| 192 | ); |
| 193 | } |
| 194 | |
| 195 | /// Set casted thread specific value. |
| 196 | fn setspecific(new: ?*ObjectArray) void { |
| 197 | if (std.c.pthread_setspecific(current_thread_storage.key, @ptrCast(*c_void, new)) != 0) { |
| 198 | abort(); |
| 199 | } |
| 200 | } |
| 201 | |
| 202 | /// Initialize pthread_key_t. |
| 203 | fn init() void { |
| 204 | if (std.c.pthread_key_create(¤t_thread_storage.key, current_thread_storage.deinit) != 0) { |
| 205 | abort(); |
| 206 | } |
| 207 | } |
| 208 | |
| 209 | /// Invoked by pthread specific destructor. the passed argument is the ObjectArray pointer. |
| 210 | fn deinit(arrayPtr: *c_void) callconv(.C) void { |
| 211 | var array = @ptrCast( |
| 212 | *ObjectArray, |
| 213 | @alignCast(@alignOf(ObjectArray), arrayPtr), |
| 214 | ); |
| 215 | array.deinit(); |
| 216 | } |
| 217 | }; |
| 218 | |
| 219 | const emutls_control = extern struct { |
| 220 | // A emutls_control value is a global value across all |
| 221 | // threads. The threads shares the index of TLS variable. The data |
| 222 | // array (containing address of allocated variables) is thread |
| 223 | // specific and stored using pthread_setspecific(). |
| 224 | |
| 225 | // size of the object in bytes |
| 226 | size: gcc_word, |
| 227 | |
| 228 | // alignment of the object in bytes |
| 229 | alignment: gcc_word, |
| 230 | |
| 231 | object: extern union { |
| 232 | // data[index-1] is the object address / 0 = uninit |
| 233 | index: usize, |
| 234 | |
| 235 | // object address, when in single thread env (not used) |
| 236 | address: *c_void, |
| 237 | }, |
| 238 | |
| 239 | // null or non-zero initial value for the object |
| 240 | default_value: ?*c_void, |
| 241 | |
| 242 | // global Mutex used to serialize control.index initialization. |
| 243 | var mutex: std.c.pthread_mutex_t = std.c.PTHREAD_MUTEX_INITIALIZER; |
| 244 | |
| 245 | // global counter for keeping track of requested indexes. |
| 246 | // access should be done with mutex held. |
| 247 | var next_index: usize = 1; |
| 248 | |
| 249 | /// Simple wrapper for global lock. |
| 250 | fn lock() void { |
| 251 | if (std.c.pthread_mutex_lock(&emutls_control.mutex) != 0) { |
| 252 | abort(); |
| 253 | } |
| 254 | } |
| 255 | |
| 256 | /// Simple wrapper for global unlock. |
| 257 | fn unlock() void { |
| 258 | if (std.c.pthread_mutex_unlock(&emutls_control.mutex) != 0) { |
| 259 | abort(); |
| 260 | } |
| 261 | } |
| 262 | |
| 263 | /// Helper to retrieve nad initialize global unique index per emutls variable. |
| 264 | pub fn getIndex(self: *emutls_control) usize { |
| 265 | // Two threads could race against the same emutls_control. |
| 266 | |
| 267 | // Use atomic for reading coherent value lockless. |
| 268 | const index_lockless = @atomicLoad(usize, &self.object.index, .Acquire); |
| 269 | |
| 270 | if (index_lockless != 0) { |
| 271 | // index is already initialized, return it. |
| 272 | return index_lockless; |
| 273 | } |
| 274 | |
| 275 | // index is uninitialized: take global lock to avoid possible race. |
| 276 | emutls_control.lock(); |
| 277 | defer emutls_control.unlock(); |
| 278 | |
| 279 | const index_locked = self.object.index; |
| 280 | if (index_locked != 0) { |
| 281 | // we lost a race, but index is already initialized: nothing particular to do. |
| 282 | return index_locked; |
| 283 | } |
| 284 | |
| 285 | // Store a new index atomically (for having coherent index_lockless reading). |
| 286 | @atomicStore(usize, &self.object.index, emutls_control.next_index, .Release); |
| 287 | |
| 288 | // Increment the next available index |
| 289 | emutls_control.next_index += 1; |
| 290 | |
| 291 | return self.object.index; |
| 292 | } |
| 293 | |
| 294 | /// Simple helper for testing purpose. |
| 295 | pub fn init(comptime T: type, default_value: ?*T) emutls_control { |
| 296 | return emutls_control{ |
| 297 | .size = @sizeOf(T), |
| 298 | .alignment = @alignOf(T), |
| 299 | .object = .{ .index = 0 }, |
| 300 | .default_value = @ptrCast(?*c_void, default_value), |
| 301 | }; |
| 302 | } |
| 303 | |
| 304 | /// Get the pointer on allocated storage for emutls variable. |
| 305 | pub fn getPointer(self: *emutls_control) *c_void { |
| 306 | // ensure current_thread_storage initialization is done |
| 307 | current_thread_storage.init_once.call(); |
| 308 | |
| 309 | const index = self.getIndex(); |
| 310 | var array = current_thread_storage.getArray(index); |
| 311 | |
| 312 | return array.getPointer(index - 1, self); |
| 313 | } |
| 314 | |
| 315 | /// Testing helper for retrieving typed pointer. |
| 316 | pub fn get_typed_pointer(self: *emutls_control, comptime T: type) *T { |
| 317 | assert(self.size == @sizeOf(T)); |
| 318 | assert(self.alignment == @alignOf(T)); |
| 319 | return @ptrCast( |
| 320 | *T, |
| 321 | @alignCast(@alignOf(T), self.getPointer()), |
| 322 | ); |
| 323 | } |
| 324 | }; |
| 325 | |
| 326 | test "simple_allocator" { |
| 327 | var data1: *[64]u8 = simple_allocator.alloc([64]u8); |
| 328 | defer simple_allocator.free(data1); |
| 329 | for (data1) |*c| { |
| 330 | c.* = 0xff; |
| 331 | } |
| 332 | |
| 333 | var data2: [*]u8 = simple_allocator.advancedAlloc(@alignOf(u8), 64); |
| 334 | defer simple_allocator.free(data2); |
| 335 | for (data2[0..63]) |*c| { |
| 336 | c.* = 0xff; |
| 337 | } |
| 338 | } |
| 339 | |
| 340 | test "__emutls_get_address zeroed" { |
| 341 | var ctl = emutls_control.init(usize, null); |
| 342 | expect(ctl.object.index == 0); |
| 343 | |
| 344 | // retrieve a variable from ctl |
| 345 | var x = @ptrCast(*usize, @alignCast(@alignOf(usize), __emutls_get_address(&ctl))); |
| 346 | expect(ctl.object.index != 0); // index has been allocated for this ctl |
| 347 | expect(x.* == 0); // storage has been zeroed |
| 348 | |
| 349 | // modify the storage |
| 350 | x.* = 1234; |
| 351 | |
| 352 | // retrieve a variable from ctl (same ctl) |
| 353 | var y = @ptrCast(*usize, @alignCast(@alignOf(usize), __emutls_get_address(&ctl))); |
| 354 | |
| 355 | expect(y.* == 1234); // same content that x.* |
| 356 | expect(x == y); // same pointer |
| 357 | } |
| 358 | |
| 359 | test "__emutls_get_address with default_value" { |
| 360 | var value: usize = 5678; // default value |
| 361 | var ctl = emutls_control.init(usize, &value); |
| 362 | expect(ctl.object.index == 0); |
| 363 | |
| 364 | var x: *usize = @ptrCast(*usize, @alignCast(@alignOf(usize), __emutls_get_address(&ctl))); |
| 365 | expect(ctl.object.index != 0); |
| 366 | expect(x.* == 5678); // storage initialized with default value |
| 367 | |
| 368 | // modify the storage |
| 369 | x.* = 9012; |
| 370 | |
| 371 | expect(value == 5678); // the default value didn't change |
| 372 | |
| 373 | var y = @ptrCast(*usize, @alignCast(@alignOf(usize), __emutls_get_address(&ctl))); |
| 374 | expect(y.* == 9012); // the modified storage persists |
| 375 | } |
| 376 | |
| 377 | test "test default_value with differents sizes" { |
| 378 | const testType = struct { |
| 379 | fn _testType(comptime T: type, value: T) void { |
| 380 | var def: T = value; |
| 381 | var ctl = emutls_control.init(T, &def); |
| 382 | var x = ctl.get_typed_pointer(T); |
| 383 | expect(x.* == value); |
| 384 | } |
| 385 | }._testType; |
| 386 | |
| 387 | testType(usize, 1234); |
| 388 | testType(u32, 1234); |
| 389 | testType(i16, -12); |
| 390 | testType(f64, -12.0); |
| 391 | testType( |
| 392 | @TypeOf("012345678901234567890123456789"), |
| 393 | "012345678901234567890123456789", |
| 394 | ); |
| 395 | } |
Andrew Kelley <andrew@ziglang.org> 2021-01-12 17:54:02-08:00
GitHub <noreply@github.com> 2021-01-12 17:54:02-08:00