| @@ -1,14 +1,14 @@ | | | @@ -1,14 +1,14 @@ |
1 | /* $NetBSD: atomic.h,v 1.25 2021/12/19 01:39:20 riastradh Exp $ */ | | 1 | /* $NetBSD: atomic.h,v 1.26 2021/12/19 01:39:27 riastradh Exp $ */ |
2 | | | 2 | |
3 | /*- | | 3 | /*- |
4 | * Copyright (c) 2013 The NetBSD Foundation, Inc. | | 4 | * Copyright (c) 2013 The NetBSD Foundation, Inc. |
5 | * All rights reserved. | | 5 | * All rights reserved. |
6 | * | | 6 | * |
7 | * This code is derived from software contributed to The NetBSD Foundation | | 7 | * This code is derived from software contributed to The NetBSD Foundation |
8 | * by Taylor R. Campbell. | | 8 | * by Taylor R. Campbell. |
9 | * | | 9 | * |
10 | * Redistribution and use in source and binary forms, with or without | | 10 | * Redistribution and use in source and binary forms, with or without |
11 | * modification, are permitted provided that the following conditions | | 11 | * modification, are permitted provided that the following conditions |
12 | * are met: | | 12 | * are met: |
13 | * 1. Redistributions of source code must retain the above copyright | | 13 | * 1. Redistributions of source code must retain the above copyright |
14 | * notice, this list of conditions and the following disclaimer. | | 14 | * notice, this list of conditions and the following disclaimer. |
| @@ -197,26 +197,27 @@ atomic_clear_mask(unsigned long mask, at | | | @@ -197,26 +197,27 @@ atomic_clear_mask(unsigned long mask, at |
197 | { | | 197 | { |
198 | /* no membar */ | | 198 | /* no membar */ |
199 | atomic_and_uint(&atomic->a_u.au_uint, ~mask); | | 199 | atomic_and_uint(&atomic->a_u.au_uint, ~mask); |
200 | } | | 200 | } |
201 | | | 201 | |
202 | static inline int | | 202 | static inline int |
203 | atomic_add_unless(atomic_t *atomic, int addend, int zero) | | 203 | atomic_add_unless(atomic_t *atomic, int addend, int zero) |
204 | { | | 204 | { |
205 | int value; | | 205 | int value; |
206 | | | 206 | |
207 | smp_mb__before_atomic(); | | 207 | smp_mb__before_atomic(); |
208 | do { | | 208 | do { |
209 | value = atomic->a_u.au_int; | | 209 | value = atomic->a_u.au_int; |
| | | 210 | __insn_barrier(); |
210 | if (value == zero) | | 211 | if (value == zero) |
211 | break; | | 212 | break; |
212 | } while (atomic_cas_uint(&atomic->a_u.au_uint, value, (value + addend)) | | 213 | } while (atomic_cas_uint(&atomic->a_u.au_uint, value, (value + addend)) |
213 | != (unsigned)value); | | 214 | != (unsigned)value); |
214 | smp_mb__after_atomic(); | | 215 | smp_mb__after_atomic(); |
215 | | | 216 | |
216 | return value != zero; | | 217 | return value != zero; |
217 | } | | 218 | } |
218 | | | 219 | |
219 | static inline int | | 220 | static inline int |
220 | atomic_inc_not_zero(atomic_t *atomic) | | 221 | atomic_inc_not_zero(atomic_t *atomic) |
221 | { | | 222 | { |
222 | /* membar implied by atomic_add_unless */ | | 223 | /* membar implied by atomic_add_unless */ |
| @@ -379,26 +380,27 @@ atomic_long_set(struct atomic_long *a, l | | | @@ -379,26 +380,27 @@ atomic_long_set(struct atomic_long *a, l |
379 | { | | 380 | { |
380 | /* no membar */ | | 381 | /* no membar */ |
381 | a->al_v = v; | | 382 | a->al_v = v; |
382 | } | | 383 | } |
383 | | | 384 | |
384 | static inline long | | 385 | static inline long |
385 | atomic_long_add_unless(struct atomic_long *a, long addend, long zero) | | 386 | atomic_long_add_unless(struct atomic_long *a, long addend, long zero) |
386 | { | | 387 | { |
387 | long value; | | 388 | long value; |
388 | | | 389 | |
389 | smp_mb__before_atomic(); | | 390 | smp_mb__before_atomic(); |
390 | do { | | 391 | do { |
391 | value = (long)a->al_v; | | 392 | value = (long)a->al_v; |
| | | 393 | __insn_barrier(); |
392 | if (value == zero) | | 394 | if (value == zero) |
393 | break; | | 395 | break; |
394 | } while (atomic_cas_ulong(&a->al_v, (unsigned long)value, | | 396 | } while (atomic_cas_ulong(&a->al_v, (unsigned long)value, |
395 | (unsigned long)(value + addend)) != (unsigned long)value); | | 397 | (unsigned long)(value + addend)) != (unsigned long)value); |
396 | smp_mb__after_atomic(); | | 398 | smp_mb__after_atomic(); |
397 | | | 399 | |
398 | return value != zero; | | 400 | return value != zero; |
399 | } | | 401 | } |
400 | | | 402 | |
401 | static inline long | | 403 | static inline long |
402 | atomic_long_inc_not_zero(struct atomic_long *a) | | 404 | atomic_long_inc_not_zero(struct atomic_long *a) |
403 | { | | 405 | { |
404 | /* membar implied by atomic_long_add_unless */ | | 406 | /* membar implied by atomic_long_add_unless */ |