1/* $NetBSD: lock.h,v 1.39 2021/05/30 02:28:59 joerg Exp $ */
2
3/*-
4 * Copyright (c) 2000, 2001 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Jason R. Thorpe.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32/*
33 * Machine-dependent spin lock operations.
34 *
35 * NOTE: The SWP insn used here is available only on ARM architecture
36 * version 3 and later (as well as 2a). What we are going to do is
37 * expect that the kernel will trap and emulate the insn. That will
38 * be slow, but give us the atomicity that we need.
39 */
40
41#ifndef _ARM_LOCK_H_
42#define _ARM_LOCK_H_
43
44static __inline int
45__SIMPLELOCK_LOCKED_P(const __cpu_simple_lock_t *__ptr)
46{
47 return *__ptr == __SIMPLELOCK_LOCKED;
48}
49
50static __inline int
51__SIMPLELOCK_UNLOCKED_P(const __cpu_simple_lock_t *__ptr)
52{
53 return *__ptr == __SIMPLELOCK_UNLOCKED;
54}
55
56static __inline void
57__cpu_simple_lock_clear(__cpu_simple_lock_t *__ptr)
58{
59 *__ptr = __SIMPLELOCK_UNLOCKED;
60}
61
62static __inline void
63__cpu_simple_lock_set(__cpu_simple_lock_t *__ptr)
64{
65 *__ptr = __SIMPLELOCK_LOCKED;
66}
67
68#if defined(_ARM_ARCH_6)
69static __inline unsigned int
70__arm_load_exclusive(__cpu_simple_lock_t *__alp)
71{
72 unsigned int __rv;
73 if (/*CONSTCOND*/sizeof(*__alp) == 1) {
74 __asm __volatile("ldrexb\t%0,[%1]" : "=r"(__rv) : "r"(__alp));
75 } else {
76 __asm __volatile("ldrex\t%0,[%1]" : "=r"(__rv) : "r"(__alp));
77 }
78 return __rv;
79}
80
81/* returns 0 on success and 1 on failure */
82static __inline unsigned int
83__arm_store_exclusive(__cpu_simple_lock_t *__alp, unsigned int __val)
84{
85 unsigned int __rv;
86 if (/*CONSTCOND*/sizeof(*__alp) == 1) {
87 __asm __volatile("strexb\t%0,%1,[%2]"
88 : "=&r"(__rv) : "r"(__val), "r"(__alp) : "cc", "memory");
89 } else {
90 __asm __volatile("strex\t%0,%1,[%2]"
91 : "=&r"(__rv) : "r"(__val), "r"(__alp) : "cc", "memory");
92 }
93 return __rv;
94}
95#elif defined(_KERNEL)
96static __inline unsigned char
97__swp(unsigned char __val, __cpu_simple_lock_t *__ptr)
98{
99 uint32_t __val32;
100 __asm volatile("swpb %0, %1, [%2]"
101 : "=&r" (__val32) : "r" (__val), "r" (__ptr) : "memory");
102 return __val32;
103}
104#else
105/*
106 * On MP Cortex, SWP no longer guarantees atomic results. Thus we pad
107 * out SWP so that when the cpu generates an undefined exception we can replace
108 * the SWP/MOV instructions with the right LDREX/STREX instructions.
109 *
110 * This is why we force the SWP into the template needed for LDREX/STREX
111 * including the extra instructions and extra register for testing the result.
112 */
113static __inline int
114__swp(int __val, __cpu_simple_lock_t *__ptr)
115{
116 int __tmp, __rv;
117 __asm volatile(
118#if 1
119 "1:\t" "swp %[__rv], %[__val], [%[__ptr]]"
120 "\n\t" "b 2f"
121#else
122 "1:\t" "ldrex %[__rv],[%[__ptr]]"
123 "\n\t" "strex %[__tmp],%[__val],[%[__ptr]]"
124#endif
125 "\n\t" "cmp %[__tmp],#0"
126 "\n\t" "bne 1b"
127 "\n" "2:"
128 : [__rv] "=&r" (__rv), [__tmp] "=&r" (__tmp)
129 : [__val] "r" (__val), [__ptr] "r" (__ptr) : "cc", "memory");
130 return __rv;
131}
132#endif /* !_ARM_ARCH_6 */
133
134/* load/dmb implies load-acquire */
135static __inline void
136__arm_load_dmb(void)
137{
138#if defined(_ARM_ARCH_7)
139 __asm __volatile("dmb ish" ::: "memory");
140#elif defined(_ARM_ARCH_6)
141 __asm __volatile("mcr\tp15,0,%0,c7,c10,5" :: "r"(0) : "memory");
142#endif
143}
144
145/* dmb/store implies store-release */
146static __inline void
147__arm_dmb_store(void)
148{
149#if defined(_ARM_ARCH_7)
150 __asm __volatile("dmb ish" ::: "memory");
151#elif defined(_ARM_ARCH_6)
152 __asm __volatile("mcr\tp15,0,%0,c7,c10,5" :: "r"(0) : "memory");
153#endif
154}
155
156
157static __inline void __unused
158__cpu_simple_lock_init(__cpu_simple_lock_t *__alp)
159{
160
161 *__alp = __SIMPLELOCK_UNLOCKED;
162}
163
164#if !defined(__thumb__) || defined(_ARM_ARCH_T2)
165static __inline void __unused
166__cpu_simple_lock(__cpu_simple_lock_t *__alp)
167{
168#if defined(_ARM_ARCH_6)
169 do {
170 /* spin */
171 } while (__arm_load_exclusive(__alp) != __SIMPLELOCK_UNLOCKED
172 || __arm_store_exclusive(__alp, __SIMPLELOCK_LOCKED));
173 __arm_load_dmb();
174#else
175 while (__swp(__SIMPLELOCK_LOCKED, __alp) != __SIMPLELOCK_UNLOCKED)
176 continue;
177#endif
178}
179#else
180void __cpu_simple_lock(__cpu_simple_lock_t *);
181#endif
182
183#if !defined(__thumb__) || defined(_ARM_ARCH_T2)
184static __inline int __unused
185__cpu_simple_lock_try(__cpu_simple_lock_t *__alp)
186{
187#if defined(_ARM_ARCH_6)
188 do {
189 if (__arm_load_exclusive(__alp) != __SIMPLELOCK_UNLOCKED) {
190 return 0;
191 }
192 } while (__arm_store_exclusive(__alp, __SIMPLELOCK_LOCKED));
193 __arm_load_dmb();
194 return 1;
195#else
196 return (__swp(__SIMPLELOCK_LOCKED, __alp) == __SIMPLELOCK_UNLOCKED);
197#endif
198}
199#else
200int __cpu_simple_lock_try(__cpu_simple_lock_t *);
201#endif
202
203static __inline void __unused
204__cpu_simple_unlock(__cpu_simple_lock_t *__alp)
205{
206
207#if defined(_ARM_ARCH_8) && defined(__LP64__)
208 if (sizeof(*__alp) == 1) {
209 __asm __volatile("stlrb\t%w0, [%1]"
210 :: "r"(__SIMPLELOCK_UNLOCKED), "r"(__alp) : "memory");
211 } else {
212 __asm __volatile("stlr\t%0, [%1]"
213 :: "r"(__SIMPLELOCK_UNLOCKED), "r"(__alp) : "memory");
214 }
215#else
216 __arm_dmb_store();
217 *__alp = __SIMPLELOCK_UNLOCKED;
218#endif
219}
220
221#endif /* _ARM_LOCK_H_ */