authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-02-22 17:14:06-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2025-02-22 17:14:06-05:00
log61ee9f94158bdd595a3cec632c6690672127ad71
tree94333e7fb97e6852ef9bc0202b53b804fdb4051e
parent42438ce8b226d9c7421ef4a73414a32fcc6e6264
parent3294ef792f4f40a32948268ba95d34bdba058e06
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #22606 from dweiller/memmove-opt

compiler-rt: memmove optimisation

3 files changed, 198 insertions(+), 116 deletions(-)

lib/compiler_rt/common.zig+11
......@@ -16,6 +16,17 @@ else
1616pub const visibility: std.builtin.SymbolVisibility =
1717 if (builtin.target.cpu.arch.isWasm() and linkage != .internal) .hidden else .default;
1818
19pub const PreferredLoadStoreElement = element: {
20 if (std.simd.suggestVectorLength(u8)) |vec_size| {
21 const Vec = @Vector(vec_size, u8);
22
23 if (@sizeOf(Vec) == vec_size and std.math.isPowerOfTwo(vec_size)) {
24 break :element Vec;
25 }
26 }
27 break :element usize;
28};
29
1930pub const want_aeabi = switch (builtin.abi) {
2031 .eabi,
2132 .eabihf,
lib/compiler_rt/memcpy.zig+1-10
......@@ -18,16 +18,7 @@ comptime {
1818 }
1919}
2020
21const Element = Element: {
22 if (std.simd.suggestVectorLength(u8)) |vec_size| {
23 const Vec = @Vector(vec_size, u8);
24
25 if (@sizeOf(Vec) == vec_size and std.math.isPowerOfTwo(vec_size)) {
26 break :Element Vec;
27 }
28 }
29 break :Element usize;
30};
21const Element = common.PreferredLoadStoreElement;
3122
3223comptime {
3324 assert(std.math.isPowerOfTwo(@sizeOf(Element)));
lib/compiler_rt/memmove.zig+186-106
......@@ -1,6 +1,10 @@
11const std = @import("std");
22const common = @import("./common.zig");
33const builtin = @import("builtin");
4const assert = std.debug.assert;
5const memcpy = @import("memcpy.zig");
6
7const Element = common.PreferredLoadStoreElement;
48
59comptime {
610 if (builtin.object_format != .c) {
......@@ -34,137 +38,213 @@ fn memmoveSmall(opt_dest: ?[*]u8, opt_src: ?[*]const u8, len: usize) callconv(.C
3438 return dest;
3539}
3640
37pub fn memmoveFast(opt_dest: ?[*]u8, opt_src: ?[*]const u8, len: usize) callconv(.C) ?[*]u8 {
38 // a port of https://github.com/facebook/folly/blob/1c8bc50e88804e2a7361a57cd9b551dd10f6c5fd/folly/memcpy.S
39 if (len == 0) {
40 @branchHint(.unlikely);
41 return opt_dest;
42 }
41fn memmoveFast(dest: ?[*]u8, src: ?[*]u8, len: usize) callconv(.C) ?[*]u8 {
42 @setRuntimeSafety(builtin.is_test);
43 const small_limit = @max(2 * @sizeOf(Element), @sizeOf(Element));
4344
44 const dest = opt_dest.?;
45 const src = opt_src.?;
45 if (copySmallLength(small_limit, dest.?, src.?, len)) return dest;
4646
47 if (len < 8) {
48 @branchHint(.unlikely);
49 if (len == 1) {
50 @branchHint(.unlikely);
51 dest[0] = src[0];
52 } else if (len >= 4) {
53 @branchHint(.unlikely);
54 blockCopy(dest, src, 4, len);
55 } else {
56 blockCopy(dest, src, 2, len);
57 }
58 return dest;
59 }
47 const dest_address = @intFromPtr(dest);
48 const src_address = @intFromPtr(src);
6049
61 if (len > 32) {
62 @branchHint(.unlikely);
63 if (len > 256) {
64 @branchHint(.unlikely);
65 copyMove(dest, src, len);
66 return dest;
67 }
68 copyLong(dest, src, len);
69 return dest;
50 if (src_address < dest_address) {
51 copyBackwards(dest.?, src.?, len);
52 } else {
53 copyForwards(dest.?, src.?, len);
7054 }
7155
72 if (len > 16) {
73 @branchHint(.unlikely);
74 blockCopy(dest, src, 16, len);
75 return dest;
56 return dest;
57}
58
59inline fn copySmallLength(
60 comptime small_limit: comptime_int,
61 dest: [*]u8,
62 src: [*]const u8,
63 len: usize,
64) bool {
65 if (len < 16) {
66 copyLessThan16(dest, src, len);
67 return true;
7668 }
7769
78 blockCopy(dest, src, 8, len);
70 if (comptime 2 < (std.math.log2(small_limit) + 1) / 2) {
71 if (copy16ToSmallLimit(small_limit, dest, src, len)) return true;
72 }
7973
80 return dest;
74 return false;
8175}
8276
83inline fn blockCopy(dest: [*]u8, src: [*]const u8, block_size: comptime_int, len: usize) void {
84 const first = @as(*align(1) const @Vector(block_size, u8), src[0..block_size]).*;
85 const second = @as(*align(1) const @Vector(block_size, u8), src[len - block_size ..][0..block_size]).*;
86 dest[0..block_size].* = first;
87 dest[len - block_size ..][0..block_size].* = second;
77inline fn copyLessThan16(
78 dest: [*]u8,
79 src: [*]const u8,
80 len: usize,
81) void {
82 @setRuntimeSafety(builtin.is_test);
83 if (len < 4) {
84 if (len == 0) return;
85 const b = len / 2;
86 const d0 = src[0];
87 const db = src[b];
88 const de = src[len - 1];
89 dest[0] = d0;
90 dest[b] = db;
91 dest[len - 1] = de;
92 return;
93 }
94 copyRange4(4, dest, src, len);
8895}
8996
90inline fn copyLong(dest: [*]u8, src: [*]const u8, len: usize) void {
91 var array: [8]@Vector(32, u8) = undefined;
92
93 inline for (.{ 64, 128, 192, 256 }, 0..) |N, i| {
94 array[i * 2] = src[(N / 2) - 32 ..][0..32].*;
95 array[(i * 2) + 1] = src[len - N / 2 ..][0..32].*;
96
97 if (len <= N) {
98 @branchHint(.unlikely);
99 for (0..i + 1) |j| {
100 dest[j * 32 ..][0..32].* = array[j * 2];
101 dest[len - ((j * 32) + 32) ..][0..32].* = array[(j * 2) + 1];
102 }
103 return;
97inline fn copy16ToSmallLimit(
98 comptime small_limit: comptime_int,
99 dest: [*]u8,
100 src: [*]const u8,
101 len: usize,
102) bool {
103 @setRuntimeSafety(builtin.is_test);
104 inline for (2..(std.math.log2(small_limit) + 1) / 2 + 1) |p| {
105 const limit = 1 << (2 * p);
106 if (len < limit) {
107 copyRange4(limit / 4, dest, src, len);
108 return true;
104109 }
105110 }
111 return false;
106112}
107113
108inline fn copyMove(dest: [*]u8, src: [*]const u8, len: usize) void {
109 if (@intFromPtr(src) >= @intFromPtr(dest)) {
110 @branchHint(.unlikely);
111 copyForward(dest, src, len);
112 } else if (@intFromPtr(src) + len > @intFromPtr(dest)) {
113 @branchHint(.unlikely);
114 overlapBwd(dest, src, len);
115 } else {
116 copyForward(dest, src, len);
117 }
114/// copy `len` bytes from `src` to `dest`; `len` must be in the range
115/// `[copy_len, 4 * copy_len)`.
116inline fn copyRange4(
117 comptime copy_len: comptime_int,
118 dest: [*]u8,
119 src: [*]const u8,
120 len: usize,
121) void {
122 @setRuntimeSafety(builtin.is_test);
123 comptime assert(std.math.isPowerOfTwo(copy_len));
124 assert(len >= copy_len);
125 assert(len < 4 * copy_len);
126
127 const a = len & (copy_len * 2);
128 const b = a / 2;
129
130 const last = len - copy_len;
131 const pen = last - b;
132
133 const d0 = src[0..copy_len].*;
134 const d1 = src[b..][0..copy_len].*;
135 const d2 = src[pen..][0..copy_len].*;
136 const d3 = src[last..][0..copy_len].*;
137
138 // the slice dest[0..len] is needed to workaround -ODebug miscompilation
139 dest[0..len][0..copy_len].* = d0;
140 dest[b..][0..copy_len].* = d1;
141 dest[pen..][0..copy_len].* = d2;
142 dest[last..][0..copy_len].* = d3;
143}
144
145inline fn copyForwards(
146 dest: [*]u8,
147 src: [*]const u8,
148 len: usize,
149) void {
150 @setRuntimeSafety(builtin.is_test);
151 assert(len >= 2 * @sizeOf(Element));
152
153 const head = src[0..@sizeOf(Element)].*;
154 const tail = src[len - @sizeOf(Element) ..][0..@sizeOf(Element)].*;
155 const alignment_offset = @alignOf(Element) - @intFromPtr(src) % @alignOf(Element);
156 const n = len - alignment_offset;
157 const d = dest + alignment_offset;
158 const s = src + alignment_offset;
159
160 copyBlocksAlignedSource(@ptrCast(d), @alignCast(@ptrCast(s)), n);
161
162 // copy last `copy_size` bytes unconditionally, since block copy
163 // methods only copy a multiple of `copy_size` bytes.
164 dest[len - @sizeOf(Element) ..][0..@sizeOf(Element)].* = tail;
165 dest[0..@sizeOf(Element)].* = head;
166}
167
168inline fn copyBlocksAlignedSource(
169 dest: [*]align(1) Element,
170 src: [*]const Element,
171 max_bytes: usize,
172) void {
173 copyBlocks(dest, src, max_bytes);
118174}
119175
120inline fn copyForward(dest: [*]u8, src: [*]const u8, len: usize) void {
121 const tail: @Vector(32, u8) = src[len - 32 ..][0..32].*;
176/// Copies the largest multiple of `@sizeOf(T)` bytes from `src` to `dest`,
177/// that is less than `max_bytes` where `T` is the child type of `src` and
178/// `dest`; `max_bytes` must be at least `@sizeOf(T)`.
179inline fn copyBlocks(
180 dest: anytype,
181 src: anytype,
182 max_bytes: usize,
183) void {
184 @setRuntimeSafety(builtin.is_test);
122185
123 const N: usize = len & ~@as(usize, 127);
124 var i: usize = 0;
186 const T = @typeInfo(@TypeOf(dest)).pointer.child;
187 comptime assert(T == @typeInfo(@TypeOf(src)).pointer.child);
125188
126 while (i < N) : (i += 128) {
127 dest[i..][0..32].* = src[i..][0..32].*;
128 dest[i + 32 ..][0..32].* = src[i + 32 ..][0..32].*;
129 dest[i + 64 ..][0..32].* = src[i + 64 ..][0..32].*;
130 dest[i + 96 ..][0..32].* = src[i + 96 ..][0..32].*;
189 const loop_count = max_bytes / @sizeOf(T);
190
191 for (dest[0..loop_count], src[0..loop_count]) |*d, s| {
192 d.* = s;
131193 }
194}
132195
133 if (len - i <= 32) {
134 @branchHint(.unlikely);
135 dest[len - 32 ..][0..32].* = tail;
136 } else {
137 copyLong(dest[i..], src[i..], len - i);
196inline fn copyBackwards(
197 dest: [*]u8,
198 src: [*]const u8,
199 len: usize,
200) void {
201 const end_bytes = src[len - @sizeOf(Element) ..][0..@sizeOf(Element)].*;
202 const start_bytes = src[0..@sizeOf(Element)].*;
203
204 const d_addr: usize = std.mem.alignBackward(usize, @intFromPtr(dest) + len, @alignOf(Element));
205 const d: [*]Element = @ptrFromInt(d_addr);
206 const n = d_addr - @intFromPtr(dest);
207 const s: [*]align(1) const Element = @ptrCast(src + n);
208
209 const loop_count = n / @sizeOf(Element);
210 var i: usize = 1;
211 while (i < loop_count + 1) : (i += 1) {
212 (d - i)[0] = (s - i)[0];
138213 }
214
215 dest[0..@sizeOf(Element)].* = start_bytes;
216 dest[len - @sizeOf(Element) ..][0..@sizeOf(Element)].* = end_bytes;
139217}
140218
141inline fn overlapBwd(dest: [*]u8, src: [*]const u8, len: usize) void {
142 var array: [5]@Vector(32, u8) = undefined;
143 array[0] = src[len - 32 ..][0..32].*;
144 inline for (1..5) |i| array[i] = src[(i - 1) << 5 ..][0..32].*;
145
146 const end: usize = (@intFromPtr(dest) + len - 32) & 31;
147 const range = len - end;
148 var s = src + range;
149 var d = dest + range;
150
151 while (@intFromPtr(s) > @intFromPtr(src + 128)) {
152 // zig fmt: off
153 const first = @as(*align(1) const @Vector(32, u8), @ptrCast(s - 32)).*;
154 const second = @as(*align(1) const @Vector(32, u8), @ptrCast(s - 64)).*;
155 const third = @as(*align(1) const @Vector(32, u8), @ptrCast(s - 96)).*;
156 const fourth = @as(*align(1) const @Vector(32, u8), @ptrCast(s - 128)).*;
157
158 @as(*align(32) @Vector(32, u8), @alignCast(@ptrCast(d - 32))).* = first;
159 @as(*align(32) @Vector(32, u8), @alignCast(@ptrCast(d - 64))).* = second;
160 @as(*align(32) @Vector(32, u8), @alignCast(@ptrCast(d - 96))).* = third;
161 @as(*align(32) @Vector(32, u8), @alignCast(@ptrCast(d - 128))).* = fourth;
162 // zig fmt: on
163
164 s -= 128;
165 d -= 128;
219test memmoveFast {
220 const max_len = 1024;
221 var buffer: [max_len + @alignOf(Element) - 1]u8 = undefined;
222 for (&buffer, 0..) |*b, i| {
223 b.* = @intCast(i % 97);
166224 }
167225
168 inline for (array[1..], 0..) |vec, i| dest[i * 32 ..][0..32].* = vec;
169 dest[len - 32 ..][0..32].* = array[0];
226 var move_buffer: [max_len + @alignOf(Element) - 1]u8 align(@alignOf(Element)) = undefined;
227
228 for (0..max_len) |copy_len| {
229 for (0..@alignOf(Element)) |s_offset| {
230 for (0..@alignOf(Element)) |d_offset| {
231 for (&move_buffer, buffer) |*d, s| {
232 d.* = s;
233 }
234 const dest = move_buffer[d_offset..][0..copy_len];
235 const src = move_buffer[s_offset..][0..copy_len];
236 _ = memmoveFast(dest.ptr, src.ptr, copy_len);
237 std.testing.expectEqualSlices(u8, buffer[s_offset..][0..copy_len], dest) catch |e| {
238 std.debug.print(
239 "error occured with source offset {d} and destination offset {d}\n",
240 .{
241 s_offset,
242 d_offset,
243 },
244 );
245 return e;
246 };
247 }
248 }
249 }
170250}