atomic64_64.h 5.8 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0 */
2 3 4 5 6 7 8 9 10 11 12 13
#ifndef _ASM_X86_ATOMIC64_64_H
#define _ASM_X86_ATOMIC64_64_H

#include <linux/types.h>
#include <asm/alternative.h>
#include <asm/cmpxchg.h>

/* The 64-bit atomic type */

#define ATOMIC64_INIT(i)	{ (i) }

/**
14
 * arch_atomic64_read - read atomic64 variable
15 16 17 18 19
 * @v: pointer of type atomic64_t
 *
 * Atomically reads the value of @v.
 * Doesn't imply a read memory barrier.
 */
20
static inline long arch_atomic64_read(const atomic64_t *v)
21
{
22
	return READ_ONCE((v)->counter);
23 24 25
}

/**
26
 * arch_atomic64_set - set atomic64 variable
27 28 29 30 31
 * @v: pointer to type atomic64_t
 * @i: required value
 *
 * Atomically sets the value of @v to @i.
 */
32
static inline void arch_atomic64_set(atomic64_t *v, long i)
33
{
34
	WRITE_ONCE(v->counter, i);
35 36 37
}

/**
38
 * arch_atomic64_add - add integer to atomic64 variable
39 40 41 42 43
 * @i: integer value to add
 * @v: pointer to type atomic64_t
 *
 * Atomically adds @i to @v.
 */
44
static __always_inline void arch_atomic64_add(long i, atomic64_t *v)
45 46 47 48 49 50 51
{
	asm volatile(LOCK_PREFIX "addq %1,%0"
		     : "=m" (v->counter)
		     : "er" (i), "m" (v->counter));
}

/**
52
 * arch_atomic64_sub - subtract the atomic64 variable
53 54 55 56 57
 * @i: integer value to subtract
 * @v: pointer to type atomic64_t
 *
 * Atomically subtracts @i from @v.
 */
58
static inline void arch_atomic64_sub(long i, atomic64_t *v)
59 60 61 62 63 64 65
{
	asm volatile(LOCK_PREFIX "subq %1,%0"
		     : "=m" (v->counter)
		     : "er" (i), "m" (v->counter));
}

/**
66
 * arch_atomic64_sub_and_test - subtract value from variable and test result
67 68 69 70 71 72 73
 * @i: integer value to subtract
 * @v: pointer to type atomic64_t
 *
 * Atomically subtracts @i from @v and returns
 * true if the result is zero, or false for all
 * other cases.
 */
74
#define arch_atomic64_sub_and_test arch_atomic64_sub_and_test
75
static inline bool arch_atomic64_sub_and_test(long i, atomic64_t *v)
76
{
77
	GEN_BINARY_RMWcc(LOCK_PREFIX "subq", v->counter, "er", i, "%0", e);
78 79 80
}

/**
81
 * arch_atomic64_inc - increment atomic64 variable
82 83 84 85
 * @v: pointer to type atomic64_t
 *
 * Atomically increments @v by 1.
 */
86
#define arch_atomic64_inc arch_atomic64_inc
87
static __always_inline void arch_atomic64_inc(atomic64_t *v)
88 89 90 91 92 93 94
{
	asm volatile(LOCK_PREFIX "incq %0"
		     : "=m" (v->counter)
		     : "m" (v->counter));
}

/**
95
 * arch_atomic64_dec - decrement atomic64 variable
96 97 98 99
 * @v: pointer to type atomic64_t
 *
 * Atomically decrements @v by 1.
 */
100
#define arch_atomic64_dec arch_atomic64_dec
101
static __always_inline void arch_atomic64_dec(atomic64_t *v)
102 103 104 105 106 107 108
{
	asm volatile(LOCK_PREFIX "decq %0"
		     : "=m" (v->counter)
		     : "m" (v->counter));
}

/**
109
 * arch_atomic64_dec_and_test - decrement and test
110 111 112 113 114 115
 * @v: pointer to type atomic64_t
 *
 * Atomically decrements @v by 1 and
 * returns true if the result is 0, or false for all other
 * cases.
 */
116
#define arch_atomic64_dec_and_test arch_atomic64_dec_and_test
117
static inline bool arch_atomic64_dec_and_test(atomic64_t *v)
118
{
119
	GEN_UNARY_RMWcc(LOCK_PREFIX "decq", v->counter, "%0", e);
120 121 122
}

/**
123
 * arch_atomic64_inc_and_test - increment and test
124 125 126 127 128 129
 * @v: pointer to type atomic64_t
 *
 * Atomically increments @v by 1
 * and returns true if the result is zero, or false for all
 * other cases.
 */
130
#define arch_atomic64_inc_and_test arch_atomic64_inc_and_test
131
static inline bool arch_atomic64_inc_and_test(atomic64_t *v)
132
{
133
	GEN_UNARY_RMWcc(LOCK_PREFIX "incq", v->counter, "%0", e);
134 135 136
}

/**
137
 * arch_atomic64_add_negative - add and test if negative
138 139 140 141 142 143 144
 * @i: integer value to add
 * @v: pointer to type atomic64_t
 *
 * Atomically adds @i to @v and returns true
 * if the result is negative, or false when
 * result is greater than or equal to zero.
 */
145
#define arch_atomic64_add_negative arch_atomic64_add_negative
146
static inline bool arch_atomic64_add_negative(long i, atomic64_t *v)
147
{
148
	GEN_BINARY_RMWcc(LOCK_PREFIX "addq", v->counter, "er", i, "%0", s);
149 150 151
}

/**
152
 * arch_atomic64_add_return - add and return
153 154 155 156 157
 * @i: integer value to add
 * @v: pointer to type atomic64_t
 *
 * Atomically adds @i to @v and returns @i + @v
 */
158
static __always_inline long arch_atomic64_add_return(long i, atomic64_t *v)
159
{
160
	return i + xadd(&v->counter, i);
161 162
}

163
static inline long arch_atomic64_sub_return(long i, atomic64_t *v)
164
{
165
	return arch_atomic64_add_return(-i, v);
166 167
}

168
static inline long arch_atomic64_fetch_add(long i, atomic64_t *v)
169 170 171 172
{
	return xadd(&v->counter, i);
}

173
static inline long arch_atomic64_fetch_sub(long i, atomic64_t *v)
174 175 176 177
{
	return xadd(&v->counter, -i);
}

178
static inline long arch_atomic64_cmpxchg(atomic64_t *v, long old, long new)
179
{
180
	return arch_cmpxchg(&v->counter, old, new);
181 182
}

183 184
#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
static __always_inline bool arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, long new)
185 186 187 188
{
	return try_cmpxchg(&v->counter, old, new);
}

189
static inline long arch_atomic64_xchg(atomic64_t *v, long new)
190
{
191
	return arch_xchg(&v->counter, new);
192 193
}

194
static inline void arch_atomic64_and(long i, atomic64_t *v)
195 196 197 198 199
{
	asm volatile(LOCK_PREFIX "andq %1,%0"
			: "+m" (v->counter)
			: "er" (i)
			: "memory");
200 201
}

202
static inline long arch_atomic64_fetch_and(long i, atomic64_t *v)
203
{
204
	s64 val = arch_atomic64_read(v);
205 206

	do {
207
	} while (!arch_atomic64_try_cmpxchg(v, &val, val & i));
208
	return val;
209 210
}

211
static inline void arch_atomic64_or(long i, atomic64_t *v)
212 213 214 215 216 217
{
	asm volatile(LOCK_PREFIX "orq %1,%0"
			: "+m" (v->counter)
			: "er" (i)
			: "memory");
}
218

219
static inline long arch_atomic64_fetch_or(long i, atomic64_t *v)
220
{
221
	s64 val = arch_atomic64_read(v);
222

223
	do {
224
	} while (!arch_atomic64_try_cmpxchg(v, &val, val | i));
225 226 227
	return val;
}

228
static inline void arch_atomic64_xor(long i, atomic64_t *v)
229 230 231 232 233 234 235
{
	asm volatile(LOCK_PREFIX "xorq %1,%0"
			: "+m" (v->counter)
			: "er" (i)
			: "memory");
}

236
static inline long arch_atomic64_fetch_xor(long i, atomic64_t *v)
237
{
238
	s64 val = arch_atomic64_read(v);
239 240

	do {
241
	} while (!arch_atomic64_try_cmpxchg(v, &val, val ^ i));
242 243
	return val;
}
244

245
#endif /* _ASM_X86_ATOMIC64_64_H */