| @@ -1,157 +1,196 @@ | | | @@ -1,157 +1,196 @@ |
1 | /* $NetBSD: atomic.h,v 1.1.2.8 2013/07/24 02:28:36 riastradh Exp $ */ | | 1 | /* $NetBSD: atomic.h,v 1.1.2.9 2013/07/24 03:35:50 riastradh Exp $ */ |
2 | | | 2 | |
3 | /*- | | 3 | /*- |
4 | * Copyright (c) 2013 The NetBSD Foundation, Inc. | | 4 | * Copyright (c) 2013 The NetBSD Foundation, Inc. |
5 | * All rights reserved. | | 5 | * All rights reserved. |
6 | * | | 6 | * |
7 | * This code is derived from software contributed to The NetBSD Foundation | | 7 | * This code is derived from software contributed to The NetBSD Foundation |
8 | * by Taylor R. Campbell. | | 8 | * by Taylor R. Campbell. |
9 | * | | 9 | * |
10 | * Redistribution and use in source and binary forms, with or without | | 10 | * Redistribution and use in source and binary forms, with or without |
11 | * modification, are permitted provided that the following conditions | | 11 | * modification, are permitted provided that the following conditions |
12 | * are met: | | 12 | * are met: |
13 | * 1. Redistributions of source code must retain the above copyright | | 13 | * 1. Redistributions of source code must retain the above copyright |
14 | * notice, this list of conditions and the following disclaimer. | | 14 | * notice, this list of conditions and the following disclaimer. |
15 | * 2. Redistributions in binary form must reproduce the above copyright | | 15 | * 2. Redistributions in binary form must reproduce the above copyright |
16 | * notice, this list of conditions and the following disclaimer in the | | 16 | * notice, this list of conditions and the following disclaimer in the |
17 | * documentation and/or other materials provided with the distribution. | | 17 | * documentation and/or other materials provided with the distribution. |
18 | * | | 18 | * |
19 | * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS | | 19 | * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS |
20 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED | | 20 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
21 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | | 21 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
22 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS | | 22 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS |
23 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | | 23 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
24 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | | 24 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
25 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | | 25 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
26 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | | 26 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
27 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | | 27 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
28 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | | 28 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
29 | * POSSIBILITY OF SUCH DAMAGE. | | 29 | * POSSIBILITY OF SUCH DAMAGE. |
30 | */ | | 30 | */ |
31 | | | 31 | |
32 | #ifndef _LINUX_ATOMIC_H_ | | 32 | #ifndef _LINUX_ATOMIC_H_ |
33 | #define _LINUX_ATOMIC_H_ | | 33 | #define _LINUX_ATOMIC_H_ |
34 | | | 34 | |
35 | #include <sys/atomic.h> | | 35 | #include <sys/atomic.h> |
36 | | | 36 | |
37 | struct atomic { | | 37 | struct atomic { |
38 | union { | | 38 | union { |
39 | int au_int; | | 39 | int au_int; |
40 | unsigned int au_uint; | | 40 | unsigned int au_uint; |
41 | } a_u; | | 41 | } a_u; |
42 | }; | | 42 | }; |
43 | | | 43 | |
44 | typedef struct atomic atomic_t; | | 44 | typedef struct atomic atomic_t; |
45 | | | 45 | |
46 | static inline int | | 46 | static inline int |
47 | atomic_read(atomic_t *atomic) | | 47 | atomic_read(atomic_t *atomic) |
48 | { | | 48 | { |
49 | return *(volatile int *)&atomic->a_u.au_int; | | 49 | return *(volatile int *)&atomic->a_u.au_int; |
50 | } | | 50 | } |
51 | | | 51 | |
52 | static inline void | | 52 | static inline void |
53 | atomic_set(atomic_t *atomic, int value) | | 53 | atomic_set(atomic_t *atomic, int value) |
54 | { | | 54 | { |
55 | atomic->a_u.au_int = value; | | 55 | atomic->a_u.au_int = value; |
56 | } | | 56 | } |
57 | | | 57 | |
58 | static inline void | | 58 | static inline void |
59 | atomic_add(int addend, atomic_t *atomic) | | 59 | atomic_add(int addend, atomic_t *atomic) |
60 | { | | 60 | { |
61 | atomic_add_int(&atomic->a_u.au_uint, addend); | | 61 | atomic_add_int(&atomic->a_u.au_uint, addend); |
62 | } | | 62 | } |
63 | | | 63 | |
| | | 64 | static inline void |
| | | 65 | atomic_sub(int subtrahend, atomic_t *atomic) |
| | | 66 | { |
| | | 67 | atomic_add_int(&atomic->a_u.au_uint, -subtrahend); |
| | | 68 | } |
| | | 69 | |
64 | static inline int | | 70 | static inline int |
65 | atomic_add_return(int addend, atomic_t *atomic) | | 71 | atomic_add_return(int addend, atomic_t *atomic) |
66 | { | | 72 | { |
67 | return (int)atomic_add_int_nv(&atomic->a_u.au_uint, addend); | | 73 | return (int)atomic_add_int_nv(&atomic->a_u.au_uint, addend); |
68 | } | | 74 | } |
69 | | | 75 | |
70 | static inline void | | 76 | static inline void |
71 | atomic_inc(atomic_t *atomic) | | 77 | atomic_inc(atomic_t *atomic) |
72 | { | | 78 | { |
73 | atomic_inc_uint(&atomic->a_u.au_uint); | | 79 | atomic_inc_uint(&atomic->a_u.au_uint); |
74 | } | | 80 | } |
75 | | | 81 | |
76 | static inline void | | 82 | static inline void |
77 | atomic_dec(atomic_t *atomic) | | 83 | atomic_dec(atomic_t *atomic) |
78 | { | | 84 | { |
79 | atomic_dec_uint(&atomic->a_u.au_uint); | | 85 | atomic_dec_uint(&atomic->a_u.au_uint); |
80 | } | | 86 | } |
81 | | | 87 | |
82 | static inline int | | 88 | static inline int |
83 | atomic_dec_and_test(atomic_t *atomic) | | 89 | atomic_dec_and_test(atomic_t *atomic) |
84 | { | | 90 | { |
85 | return (-1 == (int)atomic_dec_uint_nv(&atomic->a_u.au_uint)); | | 91 | return (-1 == (int)atomic_dec_uint_nv(&atomic->a_u.au_uint)); |
86 | } | | 92 | } |
87 | | | 93 | |
88 | static inline void | | 94 | static inline void |
| | | 95 | atomic_set_mask(unsigned long mask, atomic_t *atomic) |
| | | 96 | { |
| | | 97 | atomic_or_uint(&atomic->a_u.au_uint, mask); |
| | | 98 | } |
| | | 99 | |
| | | 100 | static inline void |
| | | 101 | atomic_clear_mask(unsigned long mask, atomic_t *atomic) |
| | | 102 | { |
| | | 103 | atomic_and_uint(&atomic->a_u.au_uint, ~mask); |
| | | 104 | } |
| | | 105 | |
| | | 106 | static inline int |
| | | 107 | atomic_add_unless(atomic_t *atomic, int addend, int zero) |
| | | 108 | { |
| | | 109 | int value; |
| | | 110 | |
| | | 111 | do { |
| | | 112 | value = atomic->a_u.au_int; |
| | | 113 | if (value == zero) |
| | | 114 | return 0; |
| | | 115 | } while (atomic_cas_uint(&atomic->a_u.au_uint, value, (value + addend)) |
| | | 116 | != value); |
| | | 117 | |
| | | 118 | return 1; |
| | | 119 | } |
| | | 120 | |
| | | 121 | static inline int |
| | | 122 | atomic_inc_not_zero(atomic_t *atomic) |
| | | 123 | { |
| | | 124 | return atomic_add_unless(atomic, 1, 0); |
| | | 125 | } |
| | | 126 | |
| | | 127 | static inline void |
89 | set_bit(unsigned long bit, volatile unsigned long *ptr) | | 128 | set_bit(unsigned long bit, volatile unsigned long *ptr) |
90 | { | | 129 | { |
91 | atomic_or_ulong(ptr, (1 << bit)); | | 130 | atomic_or_ulong(ptr, (1 << bit)); |
92 | } | | 131 | } |
93 | | | 132 | |
94 | static inline void | | 133 | static inline void |
95 | clear_bit(unsigned long bit, volatile unsigned long *ptr) | | 134 | clear_bit(unsigned long bit, volatile unsigned long *ptr) |
96 | { | | 135 | { |
97 | atomic_and_ulong(ptr, ~(1 << bit)); | | 136 | atomic_and_ulong(ptr, ~(1 << bit)); |
98 | } | | 137 | } |
99 | | | 138 | |
100 | static inline void | | 139 | static inline void |
101 | change_bit(unsigned long bit, volatile unsigned long *ptr) | | 140 | change_bit(unsigned long bit, volatile unsigned long *ptr) |
102 | { | | 141 | { |
103 | unsigned long v; | | 142 | unsigned long v; |
104 | | | 143 | |
105 | do v = *ptr; while (atomic_cas_ulong(ptr, v, v ^ (1 << bit)) != v); | | 144 | do v = *ptr; while (atomic_cas_ulong(ptr, v, v ^ (1 << bit)) != v); |
106 | } | | 145 | } |
107 | | | 146 | |
108 | static inline unsigned long | | 147 | static inline unsigned long |
109 | test_and_set_bit(unsigned long bit, volatile unsigned long *ptr) | | 148 | test_and_set_bit(unsigned long bit, volatile unsigned long *ptr) |
110 | { | | 149 | { |
111 | unsigned long v; | | 150 | unsigned long v; |
112 | | | 151 | |
113 | do v = *ptr; while (atomic_cas_ulong(ptr, v, v | (1 << bit)) != v); | | 152 | do v = *ptr; while (atomic_cas_ulong(ptr, v, v | (1 << bit)) != v); |
114 | | | 153 | |
115 | return (v & (1 << bit)); | | 154 | return (v & (1 << bit)); |
116 | } | | 155 | } |
117 | | | 156 | |
118 | static inline unsigned long | | 157 | static inline unsigned long |
119 | test_and_clear_bit(unsigned long bit, volatile unsigned long *ptr) | | 158 | test_and_clear_bit(unsigned long bit, volatile unsigned long *ptr) |
120 | { | | 159 | { |
121 | unsigned long v; | | 160 | unsigned long v; |
122 | | | 161 | |
123 | do v = *ptr; while (atomic_cas_ulong(ptr, v, v &~ (1 << bit)) != v); | | 162 | do v = *ptr; while (atomic_cas_ulong(ptr, v, v &~ (1 << bit)) != v); |
124 | | | 163 | |
125 | return (v & (1 << bit)); | | 164 | return (v & (1 << bit)); |
126 | } | | 165 | } |
127 | | | 166 | |
128 | static inline unsigned long | | 167 | static inline unsigned long |
129 | test_and_change_bit(unsigned long bit, volatile unsigned long *ptr) | | 168 | test_and_change_bit(unsigned long bit, volatile unsigned long *ptr) |
130 | { | | 169 | { |
131 | unsigned long v; | | 170 | unsigned long v; |
132 | | | 171 | |
133 | do v = *ptr; while (atomic_cas_ulong(ptr, v, v ^ (1 << bit)) != v); | | 172 | do v = *ptr; while (atomic_cas_ulong(ptr, v, v ^ (1 << bit)) != v); |
134 | | | 173 | |
135 | return (v & (1 << bit)); | | 174 | return (v & (1 << bit)); |
136 | } | | 175 | } |
137 | | | 176 | |
138 | #if defined(MULTIPROCESSOR) && !defined(__HAVE_ATOMIC_AS_MEMBAR) | | 177 | #if defined(MULTIPROCESSOR) && !defined(__HAVE_ATOMIC_AS_MEMBAR) |
139 | /* | | 178 | /* |
140 | * XXX These memory barriers are doubtless overkill, but I am having | | 179 | * XXX These memory barriers are doubtless overkill, but I am having |
141 | * trouble understanding the intent and use of the Linux atomic membar | | 180 | * trouble understanding the intent and use of the Linux atomic membar |
142 | * API. I think that for reference counting purposes, the sequences | | 181 | * API. I think that for reference counting purposes, the sequences |
143 | * should be insn/inc/enter and exit/dec/insn, but the use of the | | 182 | * should be insn/inc/enter and exit/dec/insn, but the use of the |
144 | * before/after memory barriers is not consistent throughout Linux. | | 183 | * before/after memory barriers is not consistent throughout Linux. |
145 | */ | | 184 | */ |
146 | # define smp_mb__before_atomic_inc() membar_sync() | | 185 | # define smp_mb__before_atomic_inc() membar_sync() |
147 | # define smp_mb__after_atomic_inc() membar_sync() | | 186 | # define smp_mb__after_atomic_inc() membar_sync() |
148 | # define smp_mb__before_atomic_dec() membar_sync() | | 187 | # define smp_mb__before_atomic_dec() membar_sync() |
149 | # define smp_mb__after_atomic_dec() membar_sync() | | 188 | # define smp_mb__after_atomic_dec() membar_sync() |
150 | #else | | 189 | #else |
151 | # define smp_mb__before_atomic_inc() __insn_barrier() | | 190 | # define smp_mb__before_atomic_inc() __insn_barrier() |
152 | # define smp_mb__after_atomic_inc() __insn_barrier() | | 191 | # define smp_mb__after_atomic_inc() __insn_barrier() |
153 | # define smp_mb__before_atomic_dec() __insn_barrier() | | 192 | # define smp_mb__before_atomic_dec() __insn_barrier() |
154 | # define smp_mb__after_atomic_dec() __insn_barrier() | | 193 | # define smp_mb__after_atomic_dec() __insn_barrier() |
155 | #endif | | 194 | #endif |
156 | | | 195 | |
157 | #endif /* _LINUX_ATOMIC_H_ */ | | 196 | #endif /* _LINUX_ATOMIC_H_ */ |