spinlock.h 8.2 KB
Newer Older
C
Catalin Marinas 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
/*
 * Copyright (C) 2012 ARM Ltd.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
#ifndef __ASM_SPINLOCK_H
#define __ASM_SPINLOCK_H

19
#include <asm/lse.h>
C
Catalin Marinas 已提交
20 21 22 23 24 25 26 27 28
#include <asm/spinlock_types.h>
#include <asm/processor.h>

/*
 * Spinlock implementation.
 *
 * The memory barriers are implicit with the load-acquire and store-release
 * instructions.
 */
29 30 31 32
static inline void arch_spin_unlock_wait(arch_spinlock_t *lock)
{
	unsigned int tmp;
	arch_spinlock_t lockval;
33
	u32 owner;
C
Catalin Marinas 已提交
34

35 36 37 38 39
	/*
	 * Ensure prior spin_lock operations to other locks have completed
	 * on this CPU before we test whether "lock" is locked.
	 */
	smp_mb();
40
	owner = READ_ONCE(lock->owner) << 16;
41

42 43 44 45
	asm volatile(
"	sevl\n"
"1:	wfe\n"
"2:	ldaxr	%w0, %2\n"
46
	/* Is the lock free? */
47
"	eor	%w1, %w0, %w0, ror #16\n"
48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65
"	cbz	%w1, 3f\n"
	/* Lock taken -- has there been a subsequent unlock->lock transition? */
"	eor	%w1, %w3, %w0, lsl #16\n"
"	cbz	%w1, 1b\n"
	/*
	 * The owner has been updated, so there was an unlock->lock
	 * transition that we missed. That means we can rely on the
	 * store-release of the unlock operation paired with the
	 * load-acquire of the lock operation to publish any of our
	 * previous stores to the new lock owner and therefore don't
	 * need to bother with the writeback below.
	 */
"	b	4f\n"
"3:\n"
	/*
	 * Serialise against any concurrent lockers by writing back the
	 * unlocked lock value
	 */
66 67 68 69
	ARM64_LSE_ATOMIC_INSN(
	/* LL/SC */
"	stxr	%w1, %w0, %2\n"
"	nop\n"
70 71 72 73 74
"	nop\n",
	/* LSE atomics */
"	mov	%w1, %w0\n"
"	cas	%w0, %w0, %2\n"
"	eor	%w1, %w1, %w0\n")
75
	/* Somebody else wrote to the lock, GOTO 10 and reload the value */
76
"	cbnz	%w1, 2b\n"
77
"4:"
78
	: "=&r" (lockval), "=&r" (tmp), "+Q" (*lock)
79
	: "r" (owner)
80 81
	: "memory");
}
C
Catalin Marinas 已提交
82 83 84 85 86 87

#define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock)

static inline void arch_spin_lock(arch_spinlock_t *lock)
{
	unsigned int tmp;
88
	arch_spinlock_t lockval, newval;
C
Catalin Marinas 已提交
89 90

	asm volatile(
91
	/* Atomically increment the next ticket. */
92 93
	ARM64_LSE_ATOMIC_INSN(
	/* LL/SC */
94 95 96 97
"	prfm	pstl1strm, %3\n"
"1:	ldaxr	%w0, %3\n"
"	add	%w1, %w0, %w5\n"
"	stxr	%w2, %w1, %3\n"
98 99 100 101 102 103 104 105 106
"	cbnz	%w2, 1b\n",
	/* LSE atomics */
"	mov	%w2, %w5\n"
"	ldadda	%w2, %w0, %3\n"
"	nop\n"
"	nop\n"
"	nop\n"
	)

107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123
	/* Did we get the lock? */
"	eor	%w1, %w0, %w0, ror #16\n"
"	cbz	%w1, 3f\n"
	/*
	 * No: spin on the owner. Send a local event to avoid missing an
	 * unlock before the exclusive load.
	 */
"	sevl\n"
"2:	wfe\n"
"	ldaxrh	%w2, %4\n"
"	eor	%w1, %w2, %w0, lsr #16\n"
"	cbnz	%w1, 2b\n"
	/* We got the lock. Critical section starts here. */
"3:"
	: "=&r" (lockval), "=&r" (newval), "=&r" (tmp), "+Q" (*lock)
	: "Q" (lock->owner), "I" (1 << TICKET_SHIFT)
	: "memory");
C
Catalin Marinas 已提交
124 125 126 127 128
}

static inline int arch_spin_trylock(arch_spinlock_t *lock)
{
	unsigned int tmp;
129
	arch_spinlock_t lockval;
C
Catalin Marinas 已提交
130

131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149
	asm volatile(ARM64_LSE_ATOMIC_INSN(
	/* LL/SC */
	"	prfm	pstl1strm, %2\n"
	"1:	ldaxr	%w0, %2\n"
	"	eor	%w1, %w0, %w0, ror #16\n"
	"	cbnz	%w1, 2f\n"
	"	add	%w0, %w0, %3\n"
	"	stxr	%w1, %w0, %2\n"
	"	cbnz	%w1, 1b\n"
	"2:",
	/* LSE atomics */
	"	ldr	%w0, %2\n"
	"	eor	%w1, %w0, %w0, ror #16\n"
	"	cbnz	%w1, 1f\n"
	"	add	%w1, %w0, %3\n"
	"	casa	%w0, %w1, %2\n"
	"	and	%w1, %w1, #0xffff\n"
	"	eor	%w1, %w1, %w0, lsr #16\n"
	"1:")
150 151 152
	: "=&r" (lockval), "=&r" (tmp), "+Q" (*lock)
	: "I" (1 << TICKET_SHIFT)
	: "memory");
C
Catalin Marinas 已提交
153 154 155 156 157 158

	return !tmp;
}

static inline void arch_spin_unlock(arch_spinlock_t *lock)
{
159 160 161 162
	unsigned long tmp;

	asm volatile(ARM64_LSE_ATOMIC_INSN(
	/* LL/SC */
163
	"	ldrh	%w1, %0\n"
164 165 166 167 168 169 170 171
	"	add	%w1, %w1, #1\n"
	"	stlrh	%w1, %0",
	/* LSE atomics */
	"	mov	%w1, #1\n"
	"	nop\n"
	"	staddlh	%w1, %0")
	: "=Q" (lock->owner), "=&r" (tmp)
	:
172 173 174
	: "memory");
}

175 176 177 178 179
static inline int arch_spin_value_unlocked(arch_spinlock_t lock)
{
	return lock.owner == lock.next;
}

180 181
static inline int arch_spin_is_locked(arch_spinlock_t *lock)
{
182
	smp_mb(); /* See arch_spin_unlock_wait */
183
	return !arch_spin_value_unlocked(READ_ONCE(*lock));
184 185 186 187
}

static inline int arch_spin_is_contended(arch_spinlock_t *lock)
{
188
	arch_spinlock_t lockval = READ_ONCE(*lock);
189
	return (lockval.next - lockval.owner) > 1;
C
Catalin Marinas 已提交
190
}
191
#define arch_spin_is_contended	arch_spin_is_contended
C
Catalin Marinas 已提交
192 193 194 195 196 197 198 199 200 201 202 203 204 205 206

/*
 * Write lock implementation.
 *
 * Write locks set bit 31. Unlocking, is done by writing 0 since the lock is
 * exclusively held.
 *
 * The memory barriers are implicit with the load-acquire and store-release
 * instructions.
 */

static inline void arch_write_lock(arch_rwlock_t *rw)
{
	unsigned int tmp;

207 208
	asm volatile(ARM64_LSE_ATOMIC_INSN(
	/* LL/SC */
C
Catalin Marinas 已提交
209 210
	"	sevl\n"
	"1:	wfe\n"
211
	"2:	ldaxr	%w0, %1\n"
C
Catalin Marinas 已提交
212
	"	cbnz	%w0, 1b\n"
213
	"	stxr	%w0, %w2, %1\n"
C
Catalin Marinas 已提交
214
	"	cbnz	%w0, 2b\n"
215 216 217 218 219 220 221 222 223 224
	"	nop",
	/* LSE atomics */
	"1:	mov	%w0, wzr\n"
	"2:	casa	%w0, %w2, %1\n"
	"	cbz	%w0, 3f\n"
	"	ldxr	%w0, %1\n"
	"	cbz	%w0, 2b\n"
	"	wfe\n"
	"	b	1b\n"
	"3:")
225 226
	: "=&r" (tmp), "+Q" (rw->lock)
	: "r" (0x80000000)
227
	: "memory");
C
Catalin Marinas 已提交
228 229 230 231 232 233
}

static inline int arch_write_trylock(arch_rwlock_t *rw)
{
	unsigned int tmp;

234 235
	asm volatile(ARM64_LSE_ATOMIC_INSN(
	/* LL/SC */
236 237
	"1:	ldaxr	%w0, %1\n"
	"	cbnz	%w0, 2f\n"
238
	"	stxr	%w0, %w2, %1\n"
239
	"	cbnz	%w0, 1b\n"
240 241 242 243 244 245
	"2:",
	/* LSE atomics */
	"	mov	%w0, wzr\n"
	"	casa	%w0, %w2, %1\n"
	"	nop\n"
	"	nop")
246 247
	: "=&r" (tmp), "+Q" (rw->lock)
	: "r" (0x80000000)
248
	: "memory");
C
Catalin Marinas 已提交
249 250 251 252 253 254

	return !tmp;
}

static inline void arch_write_unlock(arch_rwlock_t *rw)
{
255 256 257 258
	asm volatile(ARM64_LSE_ATOMIC_INSN(
	"	stlr	wzr, %0",
	"	swpl	wzr, wzr, %0")
	: "=Q" (rw->lock) :: "memory");
C
Catalin Marinas 已提交
259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274
}

/* write_can_lock - would write_trylock() succeed? */
#define arch_write_can_lock(x)		((x)->lock == 0)

/*
 * Read lock implementation.
 *
 * It exclusively loads the lock value, increments it and stores the new value
 * back if positive and the CPU still exclusively owns the location. If the
 * value is negative, the lock is already held.
 *
 * During unlocking there may be multiple active read locks but no write lock.
 *
 * The memory barriers are implicit with the load-acquire and store-release
 * instructions.
275 276 277 278
 *
 * Note that in UNDEFINED cases, such as unlocking a lock twice, the LL/SC
 * and LSE implementations may exhibit different behaviour (although this
 * will have no effect on lockdep).
C
Catalin Marinas 已提交
279 280 281 282 283 284 285
 */
static inline void arch_read_lock(arch_rwlock_t *rw)
{
	unsigned int tmp, tmp2;

	asm volatile(
	"	sevl\n"
286 287
	ARM64_LSE_ATOMIC_INSN(
	/* LL/SC */
C
Catalin Marinas 已提交
288
	"1:	wfe\n"
289
	"2:	ldaxr	%w0, %2\n"
C
Catalin Marinas 已提交
290 291
	"	add	%w0, %w0, #1\n"
	"	tbnz	%w0, #31, 1b\n"
292
	"	stxr	%w1, %w0, %2\n"
293 294 295 296 297 298 299 300 301 302
	"	nop\n"
	"	cbnz	%w1, 2b",
	/* LSE atomics */
	"1:	wfe\n"
	"2:	ldxr	%w0, %2\n"
	"	adds	%w1, %w0, #1\n"
	"	tbnz	%w1, #31, 1b\n"
	"	casa	%w0, %w1, %2\n"
	"	sbc	%w0, %w1, %w0\n"
	"	cbnz	%w0, 2b")
303 304
	: "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock)
	:
305
	: "cc", "memory");
C
Catalin Marinas 已提交
306 307 308 309 310 311
}

static inline void arch_read_unlock(arch_rwlock_t *rw)
{
	unsigned int tmp, tmp2;

312 313
	asm volatile(ARM64_LSE_ATOMIC_INSN(
	/* LL/SC */
314
	"1:	ldxr	%w0, %2\n"
C
Catalin Marinas 已提交
315
	"	sub	%w0, %w0, #1\n"
316
	"	stlxr	%w1, %w0, %2\n"
317 318 319 320 321 322
	"	cbnz	%w1, 1b",
	/* LSE atomics */
	"	movn	%w0, #0\n"
	"	nop\n"
	"	nop\n"
	"	staddl	%w0, %2")
323 324
	: "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock)
	:
325
	: "memory");
C
Catalin Marinas 已提交
326 327 328 329
}

static inline int arch_read_trylock(arch_rwlock_t *rw)
{
330
	unsigned int tmp, tmp2;
C
Catalin Marinas 已提交
331

332 333 334
	asm volatile(ARM64_LSE_ATOMIC_INSN(
	/* LL/SC */
	"	mov	%w1, #1\n"
335
	"1:	ldaxr	%w0, %2\n"
C
Catalin Marinas 已提交
336
	"	add	%w0, %w0, #1\n"
337
	"	tbnz	%w0, #31, 2f\n"
338
	"	stxr	%w1, %w0, %2\n"
339
	"	cbnz	%w1, 1b\n"
340 341 342 343 344 345 346 347 348 349
	"2:",
	/* LSE atomics */
	"	ldr	%w0, %2\n"
	"	adds	%w1, %w0, #1\n"
	"	tbnz	%w1, #31, 1f\n"
	"	casa	%w0, %w1, %2\n"
	"	sbc	%w1, %w1, %w0\n"
	"	nop\n"
	"1:")
	: "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock)
350
	:
351
	: "cc", "memory");
C
Catalin Marinas 已提交
352 353 354 355 356 357 358 359 360 361 362 363 364 365 366

	return !tmp2;
}

/* read_can_lock - would read_trylock() succeed? */
#define arch_read_can_lock(x)		((x)->lock < 0x80000000)

#define arch_read_lock_flags(lock, flags) arch_read_lock(lock)
#define arch_write_lock_flags(lock, flags) arch_write_lock(lock)

#define arch_spin_relax(lock)	cpu_relax()
#define arch_read_relax(lock)	cpu_relax()
#define arch_write_relax(lock)	cpu_relax()

#endif /* __ASM_SPINLOCK_H */