| @@ -1,509 +1,500 @@ | | | @@ -1,509 +1,500 @@ |
1 | /* $NetBSD: subr_kmem.c,v 1.78 2020/01/25 15:08:40 ad Exp $ */ | | 1 | /* $NetBSD: subr_kmem.c,v 1.79 2020/03/08 00:31:19 ad Exp $ */ |
2 | | | 2 | |
3 | /* | | 3 | /* |
4 | * Copyright (c) 2009-2020 The NetBSD Foundation, Inc. | | 4 | * Copyright (c) 2009-2020 The NetBSD Foundation, Inc. |
5 | * All rights reserved. | | 5 | * All rights reserved. |
6 | * | | 6 | * |
7 | * This code is derived from software contributed to The NetBSD Foundation | | 7 | * This code is derived from software contributed to The NetBSD Foundation |
8 | * by Andrew Doran and Maxime Villard. | | 8 | * by Andrew Doran and Maxime Villard. |
9 | * | | 9 | * |
10 | * Redistribution and use in source and binary forms, with or without | | 10 | * Redistribution and use in source and binary forms, with or without |
11 | * modification, are permitted provided that the following conditions | | 11 | * modification, are permitted provided that the following conditions |
12 | * are met: | | 12 | * are met: |
13 | * 1. Redistributions of source code must retain the above copyright | | 13 | * 1. Redistributions of source code must retain the above copyright |
14 | * notice, this list of conditions and the following disclaimer. | | 14 | * notice, this list of conditions and the following disclaimer. |
15 | * 2. Redistributions in binary form must reproduce the above copyright | | 15 | * 2. Redistributions in binary form must reproduce the above copyright |
16 | * notice, this list of conditions and the following disclaimer in the | | 16 | * notice, this list of conditions and the following disclaimer in the |
17 | * documentation and/or other materials provided with the distribution. | | 17 | * documentation and/or other materials provided with the distribution. |
18 | * | | 18 | * |
19 | * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS | | 19 | * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS |
20 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED | | 20 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
21 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | | 21 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
22 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS | | 22 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS |
23 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | | 23 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
24 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | | 24 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
25 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | | 25 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
26 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | | 26 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
27 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | | 27 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
28 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | | 28 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
29 | * POSSIBILITY OF SUCH DAMAGE. | | 29 | * POSSIBILITY OF SUCH DAMAGE. |
30 | */ | | 30 | */ |
31 | | | 31 | |
32 | /* | | 32 | /* |
33 | * Copyright (c)2006 YAMAMOTO Takashi, | | 33 | * Copyright (c)2006 YAMAMOTO Takashi, |
34 | * All rights reserved. | | 34 | * All rights reserved. |
35 | * | | 35 | * |
36 | * Redistribution and use in source and binary forms, with or without | | 36 | * Redistribution and use in source and binary forms, with or without |
37 | * modification, are permitted provided that the following conditions | | 37 | * modification, are permitted provided that the following conditions |
38 | * are met: | | 38 | * are met: |
39 | * 1. Redistributions of source code must retain the above copyright | | 39 | * 1. Redistributions of source code must retain the above copyright |
40 | * notice, this list of conditions and the following disclaimer. | | 40 | * notice, this list of conditions and the following disclaimer. |
41 | * 2. Redistributions in binary form must reproduce the above copyright | | 41 | * 2. Redistributions in binary form must reproduce the above copyright |
42 | * notice, this list of conditions and the following disclaimer in the | | 42 | * notice, this list of conditions and the following disclaimer in the |
43 | * documentation and/or other materials provided with the distribution. | | 43 | * documentation and/or other materials provided with the distribution. |
44 | * | | 44 | * |
45 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND | | 45 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND |
46 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | | 46 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
47 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | | 47 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
48 | * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE | | 48 | * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE |
49 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | | 49 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
50 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS | | 50 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS |
51 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) | | 51 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
52 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT | | 52 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
53 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY | | 53 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
54 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | | 54 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
55 | * SUCH DAMAGE. | | 55 | * SUCH DAMAGE. |
56 | */ | | 56 | */ |
57 | | | 57 | |
58 | /* | | 58 | /* |
59 | * Allocator of kernel wired memory. This allocator has some debug features | | 59 | * Allocator of kernel wired memory. This allocator has some debug features |
60 | * enabled with "option DIAGNOSTIC" and "option DEBUG". | | 60 | * enabled with "option DIAGNOSTIC" and "option DEBUG". |
61 | */ | | 61 | */ |
62 | | | 62 | |
63 | /* | | 63 | /* |
64 | * KMEM_SIZE: detect alloc/free size mismatch bugs. | | 64 | * KMEM_SIZE: detect alloc/free size mismatch bugs. |
65 | * Prefix each allocations with a fixed-sized, aligned header and record | | 65 | * Append to each allocation a fixed-sized footer and record the exact |
66 | * the exact user-requested allocation size in it. When freeing, compare | | 66 | * user-requested allocation size in it. When freeing, compare it with |
67 | * it with kmem_free's "size" argument. | | 67 | * kmem_free's "size" argument. |
68 | * | | 68 | * |
69 | * This option is enabled on DIAGNOSTIC. | | 69 | * This option is enabled on DIAGNOSTIC. |
70 | * | | 70 | * |
71 | * |CHUNK|CHUNK|CHUNK|CHUNK|CHUNK|CHUNK|CHUNK|CHUNK|CHUNK|CHUNK| | | 71 | * |CHUNK|CHUNK|CHUNK|CHUNK|CHUNK|CHUNK|CHUNK|CHUNK|CHUNK| | |
72 | * +-----+-----+-----+-----+-----+-----+-----+-----+-----+---+-+ | | 72 | * +-----+-----+-----+-----+-----+-----+-----+-----+-----+-+ |
73 | * |/////| | | | | | | | | |U| | | 73 | * | | | | | | | | |/////|U| |
74 | * |/HSZ/| | | | | | | | | |U| | | 74 | * | | | | | | | | |/HSZ/|U| |
75 | * |/////| | | | | | | | | |U| | | 75 | * | | | | | | | | |/////|U| |
76 | * +-----+-----+-----+-----+-----+-----+-----+-----+-----+---+-+ | | 76 | * +-----+-----+-----+-----+-----+-----+-----+-----+-----+-+ |
77 | * |Size | Buffer usable by the caller (requested size) |Unused\ | | 77 | * | Buffer usable by the caller (requested size) |Size |Unused |
78 | */ | | 78 | */ |
79 | | | 79 | |
80 | #include <sys/cdefs.h> | | 80 | #include <sys/cdefs.h> |
81 | __KERNEL_RCSID(0, "$NetBSD: subr_kmem.c,v 1.78 2020/01/25 15:08:40 ad Exp $"); | | 81 | __KERNEL_RCSID(0, "$NetBSD: subr_kmem.c,v 1.79 2020/03/08 00:31:19 ad Exp $"); |
82 | | | 82 | |
83 | #ifdef _KERNEL_OPT | | 83 | #ifdef _KERNEL_OPT |
84 | #include "opt_kmem.h" | | 84 | #include "opt_kmem.h" |
85 | #endif | | 85 | #endif |
86 | | | 86 | |
87 | #include <sys/param.h> | | 87 | #include <sys/param.h> |
88 | #include <sys/callback.h> | | 88 | #include <sys/callback.h> |
89 | #include <sys/kmem.h> | | 89 | #include <sys/kmem.h> |
90 | #include <sys/pool.h> | | 90 | #include <sys/pool.h> |
91 | #include <sys/debug.h> | | 91 | #include <sys/debug.h> |
92 | #include <sys/lockdebug.h> | | 92 | #include <sys/lockdebug.h> |
93 | #include <sys/cpu.h> | | 93 | #include <sys/cpu.h> |
94 | #include <sys/asan.h> | | 94 | #include <sys/asan.h> |
95 | #include <sys/msan.h> | | 95 | #include <sys/msan.h> |
96 | | | 96 | |
97 | #include <uvm/uvm_extern.h> | | 97 | #include <uvm/uvm_extern.h> |
98 | #include <uvm/uvm_map.h> | | 98 | #include <uvm/uvm_map.h> |
99 | | | 99 | |
100 | #include <lib/libkern/libkern.h> | | 100 | #include <lib/libkern/libkern.h> |
101 | | | 101 | |
102 | struct kmem_cache_info { | | 102 | struct kmem_cache_info { |
103 | size_t kc_size; | | 103 | size_t kc_size; |
104 | const char * kc_name; | | 104 | const char * kc_name; |
105 | }; | | 105 | }; |
106 | | | 106 | |
107 | static const struct kmem_cache_info kmem_cache_sizes[] = { | | 107 | static const struct kmem_cache_info kmem_cache_sizes[] = { |
108 | { 8, "kmem-00008" }, | | 108 | { 8, "kmem-00008" }, |
109 | { 16, "kmem-00016" }, | | 109 | { 16, "kmem-00016" }, |
110 | { 24, "kmem-00024" }, | | 110 | { 24, "kmem-00024" }, |
111 | { 32, "kmem-00032" }, | | 111 | { 32, "kmem-00032" }, |
112 | { 40, "kmem-00040" }, | | 112 | { 40, "kmem-00040" }, |
113 | { 48, "kmem-00048" }, | | 113 | { 48, "kmem-00048" }, |
114 | { 56, "kmem-00056" }, | | 114 | { 56, "kmem-00056" }, |
115 | { 64, "kmem-00064" }, | | 115 | { 64, "kmem-00064" }, |
116 | { 80, "kmem-00080" }, | | 116 | { 80, "kmem-00080" }, |
117 | { 96, "kmem-00096" }, | | 117 | { 96, "kmem-00096" }, |
118 | { 112, "kmem-00112" }, | | 118 | { 112, "kmem-00112" }, |
119 | { 128, "kmem-00128" }, | | 119 | { 128, "kmem-00128" }, |
120 | { 160, "kmem-00160" }, | | 120 | { 160, "kmem-00160" }, |
121 | { 192, "kmem-00192" }, | | 121 | { 192, "kmem-00192" }, |
122 | { 224, "kmem-00224" }, | | 122 | { 224, "kmem-00224" }, |
123 | { 256, "kmem-00256" }, | | 123 | { 256, "kmem-00256" }, |
124 | { 320, "kmem-00320" }, | | 124 | { 320, "kmem-00320" }, |
125 | { 384, "kmem-00384" }, | | 125 | { 384, "kmem-00384" }, |
126 | { 448, "kmem-00448" }, | | 126 | { 448, "kmem-00448" }, |
127 | { 512, "kmem-00512" }, | | 127 | { 512, "kmem-00512" }, |
128 | { 768, "kmem-00768" }, | | 128 | { 768, "kmem-00768" }, |
129 | { 1024, "kmem-01024" }, | | 129 | { 1024, "kmem-01024" }, |
130 | { 0, NULL } | | 130 | { 0, NULL } |
131 | }; | | 131 | }; |
132 | | | 132 | |
133 | static const struct kmem_cache_info kmem_cache_big_sizes[] = { | | 133 | static const struct kmem_cache_info kmem_cache_big_sizes[] = { |
134 | { 2048, "kmem-02048" }, | | 134 | { 2048, "kmem-02048" }, |
135 | { 4096, "kmem-04096" }, | | 135 | { 4096, "kmem-04096" }, |
136 | { 8192, "kmem-08192" }, | | 136 | { 8192, "kmem-08192" }, |
137 | { 16384, "kmem-16384" }, | | 137 | { 16384, "kmem-16384" }, |
138 | { 0, NULL } | | 138 | { 0, NULL } |
139 | }; | | 139 | }; |
140 | | | 140 | |
141 | /* | | 141 | /* |
142 | * KMEM_ALIGN is the smallest guaranteed alignment and also the | | 142 | * KMEM_ALIGN is the smallest guaranteed alignment and also the |
143 | * smallest allocateable quantum. | | 143 | * smallest allocateable quantum. |
144 | * Every cache size >= CACHE_LINE_SIZE gets CACHE_LINE_SIZE alignment. | | 144 | * Every cache size >= CACHE_LINE_SIZE gets CACHE_LINE_SIZE alignment. |
145 | */ | | 145 | */ |
146 | #define KMEM_ALIGN 8 | | 146 | #define KMEM_ALIGN 8 |
147 | #define KMEM_SHIFT 3 | | 147 | #define KMEM_SHIFT 3 |
148 | #define KMEM_MAXSIZE 1024 | | 148 | #define KMEM_MAXSIZE 1024 |
149 | #define KMEM_CACHE_COUNT (KMEM_MAXSIZE >> KMEM_SHIFT) | | 149 | #define KMEM_CACHE_COUNT (KMEM_MAXSIZE >> KMEM_SHIFT) |
150 | | | 150 | |
151 | static pool_cache_t kmem_cache[KMEM_CACHE_COUNT] __cacheline_aligned; | | 151 | static pool_cache_t kmem_cache[KMEM_CACHE_COUNT] __cacheline_aligned; |
152 | static size_t kmem_cache_maxidx __read_mostly; | | 152 | static size_t kmem_cache_maxidx __read_mostly; |
153 | | | 153 | |
154 | #define KMEM_BIG_ALIGN 2048 | | 154 | #define KMEM_BIG_ALIGN 2048 |
155 | #define KMEM_BIG_SHIFT 11 | | 155 | #define KMEM_BIG_SHIFT 11 |
156 | #define KMEM_BIG_MAXSIZE 16384 | | 156 | #define KMEM_BIG_MAXSIZE 16384 |
157 | #define KMEM_CACHE_BIG_COUNT (KMEM_BIG_MAXSIZE >> KMEM_BIG_SHIFT) | | 157 | #define KMEM_CACHE_BIG_COUNT (KMEM_BIG_MAXSIZE >> KMEM_BIG_SHIFT) |
158 | | | 158 | |
159 | static pool_cache_t kmem_cache_big[KMEM_CACHE_BIG_COUNT] __cacheline_aligned; | | 159 | static pool_cache_t kmem_cache_big[KMEM_CACHE_BIG_COUNT] __cacheline_aligned; |
160 | static size_t kmem_cache_big_maxidx __read_mostly; | | 160 | static size_t kmem_cache_big_maxidx __read_mostly; |
161 | | | 161 | |
162 | #if defined(DIAGNOSTIC) && defined(_HARDKERNEL) | | 162 | #if defined(DIAGNOSTIC) && defined(_HARDKERNEL) |
163 | #define KMEM_SIZE | | 163 | #define KMEM_SIZE |
164 | #endif | | 164 | #endif |
165 | | | 165 | |
166 | #if defined(DEBUG) && defined(_HARDKERNEL) | | 166 | #if defined(DEBUG) && defined(_HARDKERNEL) |
167 | static void *kmem_freecheck; | | 167 | static void *kmem_freecheck; |
168 | #endif | | 168 | #endif |
169 | | | 169 | |
170 | #if defined(KMEM_SIZE) | | 170 | #if defined(KMEM_SIZE) |
171 | struct kmem_header { | | 171 | #define SIZE_SIZE sizeof(size_t) |
172 | size_t size; | | | |
173 | } __aligned(KMEM_ALIGN); | | | |
174 | #define SIZE_SIZE sizeof(struct kmem_header) | | | |
175 | static void kmem_size_set(void *, size_t); | | 172 | static void kmem_size_set(void *, size_t); |
176 | static void kmem_size_check(void *, size_t); | | 173 | static void kmem_size_check(void *, size_t); |
177 | #else | | 174 | #else |
178 | #define SIZE_SIZE 0 | | 175 | #define SIZE_SIZE 0 |
179 | #define kmem_size_set(p, sz) /* nothing */ | | 176 | #define kmem_size_set(p, sz) /* nothing */ |
180 | #define kmem_size_check(p, sz) /* nothing */ | | 177 | #define kmem_size_check(p, sz) /* nothing */ |
181 | #endif | | 178 | #endif |
182 | | | 179 | |
183 | CTASSERT(KM_SLEEP == PR_WAITOK); | | 180 | CTASSERT(KM_SLEEP == PR_WAITOK); |
184 | CTASSERT(KM_NOSLEEP == PR_NOWAIT); | | 181 | CTASSERT(KM_NOSLEEP == PR_NOWAIT); |
185 | | | 182 | |
186 | /* | | 183 | /* |
187 | * kmem_intr_alloc: allocate wired memory. | | 184 | * kmem_intr_alloc: allocate wired memory. |
188 | */ | | 185 | */ |
189 | void * | | 186 | void * |
190 | kmem_intr_alloc(size_t requested_size, km_flag_t kmflags) | | 187 | kmem_intr_alloc(size_t requested_size, km_flag_t kmflags) |
191 | { | | 188 | { |
192 | #ifdef KASAN | | 189 | #ifdef KASAN |
193 | const size_t origsize = requested_size; | | 190 | const size_t origsize = requested_size; |
194 | #endif | | 191 | #endif |
195 | size_t allocsz, index; | | 192 | size_t allocsz, index; |
196 | size_t size; | | 193 | size_t size; |
197 | pool_cache_t pc; | | 194 | pool_cache_t pc; |
198 | uint8_t *p; | | 195 | uint8_t *p; |
199 | | | 196 | |
200 | KASSERT(requested_size > 0); | | 197 | KASSERT(requested_size > 0); |
201 | | | 198 | |
202 | KASSERT((kmflags & KM_SLEEP) || (kmflags & KM_NOSLEEP)); | | 199 | KASSERT((kmflags & KM_SLEEP) || (kmflags & KM_NOSLEEP)); |
203 | KASSERT(!(kmflags & KM_SLEEP) || !(kmflags & KM_NOSLEEP)); | | 200 | KASSERT(!(kmflags & KM_SLEEP) || !(kmflags & KM_NOSLEEP)); |
204 | | | 201 | |
205 | kasan_add_redzone(&requested_size); | | 202 | kasan_add_redzone(&requested_size); |
206 | size = kmem_roundup_size(requested_size); | | 203 | size = kmem_roundup_size(requested_size); |
207 | allocsz = size + SIZE_SIZE; | | 204 | allocsz = size + SIZE_SIZE; |
208 | | | 205 | |
209 | if ((index = ((allocsz -1) >> KMEM_SHIFT)) | | 206 | if ((index = ((allocsz -1) >> KMEM_SHIFT)) |
210 | < kmem_cache_maxidx) { | | 207 | < kmem_cache_maxidx) { |
211 | pc = kmem_cache[index]; | | 208 | pc = kmem_cache[index]; |
212 | } else if ((index = ((allocsz - 1) >> KMEM_BIG_SHIFT)) | | 209 | } else if ((index = ((allocsz - 1) >> KMEM_BIG_SHIFT)) |
213 | < kmem_cache_big_maxidx) { | | 210 | < kmem_cache_big_maxidx) { |
214 | pc = kmem_cache_big[index]; | | 211 | pc = kmem_cache_big[index]; |
215 | } else { | | 212 | } else { |
216 | int ret = uvm_km_kmem_alloc(kmem_va_arena, | | 213 | int ret = uvm_km_kmem_alloc(kmem_va_arena, |
217 | (vsize_t)round_page(size), | | 214 | (vsize_t)round_page(size), |
218 | ((kmflags & KM_SLEEP) ? VM_SLEEP : VM_NOSLEEP) | | 215 | ((kmflags & KM_SLEEP) ? VM_SLEEP : VM_NOSLEEP) |
219 | | VM_INSTANTFIT, (vmem_addr_t *)&p); | | 216 | | VM_INSTANTFIT, (vmem_addr_t *)&p); |
220 | if (ret) { | | 217 | if (ret) { |
221 | return NULL; | | 218 | return NULL; |
222 | } | | 219 | } |
223 | FREECHECK_OUT(&kmem_freecheck, p); | | 220 | FREECHECK_OUT(&kmem_freecheck, p); |
224 | return p; | | 221 | return p; |
225 | } | | 222 | } |
226 | | | 223 | |
227 | p = pool_cache_get(pc, kmflags); | | 224 | p = pool_cache_get(pc, kmflags); |
228 | | | 225 | |
229 | if (__predict_true(p != NULL)) { | | 226 | if (__predict_true(p != NULL)) { |
230 | FREECHECK_OUT(&kmem_freecheck, p); | | 227 | FREECHECK_OUT(&kmem_freecheck, p); |
231 | kmem_size_set(p, requested_size); | | 228 | kmem_size_set(p, requested_size); |
232 | p += SIZE_SIZE; | | | |
233 | kasan_mark(p, origsize, size, KASAN_KMEM_REDZONE); | | 229 | kasan_mark(p, origsize, size, KASAN_KMEM_REDZONE); |
234 | return p; | | 230 | return p; |
235 | } | | 231 | } |
236 | return p; | | 232 | return p; |
237 | } | | 233 | } |
238 | | | 234 | |
239 | /* | | 235 | /* |
240 | * kmem_intr_zalloc: allocate zeroed wired memory. | | 236 | * kmem_intr_zalloc: allocate zeroed wired memory. |
241 | */ | | 237 | */ |
242 | void * | | 238 | void * |
243 | kmem_intr_zalloc(size_t size, km_flag_t kmflags) | | 239 | kmem_intr_zalloc(size_t size, km_flag_t kmflags) |
244 | { | | 240 | { |
245 | void *p; | | 241 | void *p; |
246 | | | 242 | |
247 | p = kmem_intr_alloc(size, kmflags); | | 243 | p = kmem_intr_alloc(size, kmflags); |
248 | if (p != NULL) { | | 244 | if (p != NULL) { |
249 | memset(p, 0, size); | | 245 | memset(p, 0, size); |
250 | } | | 246 | } |
251 | return p; | | 247 | return p; |
252 | } | | 248 | } |
253 | | | 249 | |
254 | /* | | 250 | /* |
255 | * kmem_intr_free: free wired memory allocated by kmem_alloc. | | 251 | * kmem_intr_free: free wired memory allocated by kmem_alloc. |
256 | */ | | 252 | */ |
257 | void | | 253 | void |
258 | kmem_intr_free(void *p, size_t requested_size) | | 254 | kmem_intr_free(void *p, size_t requested_size) |
259 | { | | 255 | { |
260 | size_t allocsz, index; | | 256 | size_t allocsz, index; |
261 | size_t size; | | 257 | size_t size; |
262 | pool_cache_t pc; | | 258 | pool_cache_t pc; |
263 | | | 259 | |
264 | KASSERT(p != NULL); | | 260 | KASSERT(p != NULL); |
265 | KASSERT(requested_size > 0); | | 261 | KASSERT(requested_size > 0); |
266 | | | 262 | |
267 | kasan_add_redzone(&requested_size); | | 263 | kasan_add_redzone(&requested_size); |
268 | size = kmem_roundup_size(requested_size); | | 264 | size = kmem_roundup_size(requested_size); |
269 | allocsz = size + SIZE_SIZE; | | 265 | allocsz = size + SIZE_SIZE; |
270 | | | 266 | |
271 | if ((index = ((allocsz -1) >> KMEM_SHIFT)) | | 267 | if ((index = ((allocsz -1) >> KMEM_SHIFT)) |
272 | < kmem_cache_maxidx) { | | 268 | < kmem_cache_maxidx) { |
273 | pc = kmem_cache[index]; | | 269 | pc = kmem_cache[index]; |
274 | } else if ((index = ((allocsz - 1) >> KMEM_BIG_SHIFT)) | | 270 | } else if ((index = ((allocsz - 1) >> KMEM_BIG_SHIFT)) |
275 | < kmem_cache_big_maxidx) { | | 271 | < kmem_cache_big_maxidx) { |
276 | pc = kmem_cache_big[index]; | | 272 | pc = kmem_cache_big[index]; |
277 | } else { | | 273 | } else { |
278 | FREECHECK_IN(&kmem_freecheck, p); | | 274 | FREECHECK_IN(&kmem_freecheck, p); |
279 | uvm_km_kmem_free(kmem_va_arena, (vaddr_t)p, | | 275 | uvm_km_kmem_free(kmem_va_arena, (vaddr_t)p, |
280 | round_page(size)); | | 276 | round_page(size)); |
281 | return; | | 277 | return; |
282 | } | | 278 | } |
283 | | | 279 | |
284 | kasan_mark(p, size, size, 0); | | 280 | kasan_mark(p, size, size, 0); |
285 | | | 281 | |
286 | p = (uint8_t *)p - SIZE_SIZE; | | | |
287 | kmem_size_check(p, requested_size); | | 282 | kmem_size_check(p, requested_size); |
288 | FREECHECK_IN(&kmem_freecheck, p); | | 283 | FREECHECK_IN(&kmem_freecheck, p); |
289 | LOCKDEBUG_MEM_CHECK(p, size); | | 284 | LOCKDEBUG_MEM_CHECK(p, size); |
290 | | | 285 | |
291 | pool_cache_put(pc, p); | | 286 | pool_cache_put(pc, p); |
292 | } | | 287 | } |
293 | | | 288 | |
294 | /* -------------------------------- Kmem API -------------------------------- */ | | 289 | /* -------------------------------- Kmem API -------------------------------- */ |
295 | | | 290 | |
296 | /* | | 291 | /* |
297 | * kmem_alloc: allocate wired memory. | | 292 | * kmem_alloc: allocate wired memory. |
298 | * => must not be called from interrupt context. | | 293 | * => must not be called from interrupt context. |
299 | */ | | 294 | */ |
300 | void * | | 295 | void * |
301 | kmem_alloc(size_t size, km_flag_t kmflags) | | 296 | kmem_alloc(size_t size, km_flag_t kmflags) |
302 | { | | 297 | { |
303 | void *v; | | 298 | void *v; |
304 | | | 299 | |
305 | KASSERTMSG((!cpu_intr_p() && !cpu_softintr_p()), | | 300 | KASSERTMSG((!cpu_intr_p() && !cpu_softintr_p()), |
306 | "kmem(9) should not be used from the interrupt context"); | | 301 | "kmem(9) should not be used from the interrupt context"); |
307 | v = kmem_intr_alloc(size, kmflags); | | 302 | v = kmem_intr_alloc(size, kmflags); |
308 | if (__predict_true(v != NULL)) { | | 303 | if (__predict_true(v != NULL)) { |
309 | kmsan_mark(v, size, KMSAN_STATE_UNINIT); | | 304 | kmsan_mark(v, size, KMSAN_STATE_UNINIT); |
310 | kmsan_orig(v, size, KMSAN_TYPE_KMEM, __RET_ADDR); | | 305 | kmsan_orig(v, size, KMSAN_TYPE_KMEM, __RET_ADDR); |
311 | } | | 306 | } |
312 | KASSERT(v || (kmflags & KM_NOSLEEP) != 0); | | 307 | KASSERT(v || (kmflags & KM_NOSLEEP) != 0); |
313 | return v; | | 308 | return v; |
314 | } | | 309 | } |
315 | | | 310 | |
316 | /* | | 311 | /* |
317 | * kmem_zalloc: allocate zeroed wired memory. | | 312 | * kmem_zalloc: allocate zeroed wired memory. |
318 | * => must not be called from interrupt context. | | 313 | * => must not be called from interrupt context. |
319 | */ | | 314 | */ |
320 | void * | | 315 | void * |
321 | kmem_zalloc(size_t size, km_flag_t kmflags) | | 316 | kmem_zalloc(size_t size, km_flag_t kmflags) |
322 | { | | 317 | { |
323 | void *v; | | 318 | void *v; |
324 | | | 319 | |
325 | KASSERTMSG((!cpu_intr_p() && !cpu_softintr_p()), | | 320 | KASSERTMSG((!cpu_intr_p() && !cpu_softintr_p()), |
326 | "kmem(9) should not be used from the interrupt context"); | | 321 | "kmem(9) should not be used from the interrupt context"); |
327 | v = kmem_intr_zalloc(size, kmflags); | | 322 | v = kmem_intr_zalloc(size, kmflags); |
328 | KASSERT(v || (kmflags & KM_NOSLEEP) != 0); | | 323 | KASSERT(v || (kmflags & KM_NOSLEEP) != 0); |
329 | return v; | | 324 | return v; |
330 | } | | 325 | } |
331 | | | 326 | |
332 | /* | | 327 | /* |
333 | * kmem_free: free wired memory allocated by kmem_alloc. | | 328 | * kmem_free: free wired memory allocated by kmem_alloc. |
334 | * => must not be called from interrupt context. | | 329 | * => must not be called from interrupt context. |
335 | */ | | 330 | */ |
336 | void | | 331 | void |
337 | kmem_free(void *p, size_t size) | | 332 | kmem_free(void *p, size_t size) |
338 | { | | 333 | { |
339 | KASSERT(!cpu_intr_p()); | | 334 | KASSERT(!cpu_intr_p()); |
340 | KASSERT(!cpu_softintr_p()); | | 335 | KASSERT(!cpu_softintr_p()); |
341 | kmem_intr_free(p, size); | | 336 | kmem_intr_free(p, size); |
342 | kmsan_mark(p, size, KMSAN_STATE_INITED); | | 337 | kmsan_mark(p, size, KMSAN_STATE_INITED); |
343 | } | | 338 | } |
344 | | | 339 | |
345 | static size_t | | 340 | static size_t |
346 | kmem_create_caches(const struct kmem_cache_info *array, | | 341 | kmem_create_caches(const struct kmem_cache_info *array, |
347 | pool_cache_t alloc_table[], size_t maxsize, int shift, int ipl) | | 342 | pool_cache_t alloc_table[], size_t maxsize, int shift, int ipl) |
348 | { | | 343 | { |
349 | size_t maxidx = 0; | | 344 | size_t maxidx = 0; |
350 | size_t table_unit = (1 << shift); | | 345 | size_t table_unit = (1 << shift); |
351 | size_t size = table_unit; | | 346 | size_t size = table_unit; |
352 | int i; | | 347 | int i; |
353 | | | 348 | |
354 | for (i = 0; array[i].kc_size != 0 ; i++) { | | 349 | for (i = 0; array[i].kc_size != 0 ; i++) { |
355 | const char *name = array[i].kc_name; | | 350 | const char *name = array[i].kc_name; |
356 | size_t cache_size = array[i].kc_size; | | 351 | size_t cache_size = array[i].kc_size; |
357 | struct pool_allocator *pa; | | 352 | struct pool_allocator *pa; |
358 | int flags = 0; | | 353 | int flags = 0; |
359 | pool_cache_t pc; | | 354 | pool_cache_t pc; |
360 | size_t align; | | 355 | size_t align; |
361 | | | 356 | |
362 | /* check if we reached the requested size */ | | 357 | /* check if we reached the requested size */ |
363 | if (cache_size > maxsize || cache_size > PAGE_SIZE) { | | 358 | if (cache_size > maxsize || cache_size > PAGE_SIZE) { |
364 | break; | | 359 | break; |
365 | } | | 360 | } |
366 | | | 361 | |
367 | /* | | 362 | /* |
368 | * Exclude caches with size not a factor or multiple of the | | 363 | * Exclude caches with size not a factor or multiple of the |
369 | * coherency unit. | | 364 | * coherency unit. |
370 | */ | | 365 | */ |
371 | if (cache_size < COHERENCY_UNIT) { | | 366 | if (cache_size < COHERENCY_UNIT) { |
372 | if (COHERENCY_UNIT % cache_size > 0) { | | 367 | if (COHERENCY_UNIT % cache_size > 0) { |
373 | continue; | | 368 | continue; |
374 | } | | 369 | } |
375 | flags |= PR_NOTOUCH; | | 370 | flags |= PR_NOTOUCH; |
376 | align = KMEM_ALIGN; | | 371 | align = KMEM_ALIGN; |
377 | } else if ((cache_size & (PAGE_SIZE - 1)) == 0) { | | 372 | } else if ((cache_size & (PAGE_SIZE - 1)) == 0) { |
378 | align = PAGE_SIZE; | | 373 | align = PAGE_SIZE; |
379 | } else { | | 374 | } else { |
380 | if ((cache_size % COHERENCY_UNIT) > 0) { | | 375 | if ((cache_size % COHERENCY_UNIT) > 0) { |
381 | continue; | | 376 | continue; |
382 | } | | 377 | } |
383 | align = COHERENCY_UNIT; | | 378 | align = COHERENCY_UNIT; |
384 | } | | 379 | } |
385 | | | 380 | |
386 | if ((cache_size >> shift) > maxidx) { | | 381 | if ((cache_size >> shift) > maxidx) { |
387 | maxidx = cache_size >> shift; | | 382 | maxidx = cache_size >> shift; |
388 | } | | 383 | } |
389 | | | 384 | |
390 | pa = &pool_allocator_kmem; | | 385 | pa = &pool_allocator_kmem; |
391 | pc = pool_cache_init(cache_size, align, 0, flags, | | 386 | pc = pool_cache_init(cache_size, align, 0, flags, |
392 | name, pa, ipl, NULL, NULL, NULL); | | 387 | name, pa, ipl, NULL, NULL, NULL); |
393 | | | 388 | |
394 | while (size <= cache_size) { | | 389 | while (size <= cache_size) { |
395 | alloc_table[(size - 1) >> shift] = pc; | | 390 | alloc_table[(size - 1) >> shift] = pc; |
396 | size += table_unit; | | 391 | size += table_unit; |
397 | } | | 392 | } |
398 | } | | 393 | } |
399 | return maxidx; | | 394 | return maxidx; |
400 | } | | 395 | } |
401 | | | 396 | |
402 | void | | 397 | void |
403 | kmem_init(void) | | 398 | kmem_init(void) |
404 | { | | 399 | { |
405 | kmem_cache_maxidx = kmem_create_caches(kmem_cache_sizes, | | 400 | kmem_cache_maxidx = kmem_create_caches(kmem_cache_sizes, |
406 | kmem_cache, KMEM_MAXSIZE, KMEM_SHIFT, IPL_VM); | | 401 | kmem_cache, KMEM_MAXSIZE, KMEM_SHIFT, IPL_VM); |
407 | kmem_cache_big_maxidx = kmem_create_caches(kmem_cache_big_sizes, | | 402 | kmem_cache_big_maxidx = kmem_create_caches(kmem_cache_big_sizes, |
408 | kmem_cache_big, PAGE_SIZE, KMEM_BIG_SHIFT, IPL_VM); | | 403 | kmem_cache_big, PAGE_SIZE, KMEM_BIG_SHIFT, IPL_VM); |
409 | } | | 404 | } |
410 | | | 405 | |
411 | size_t | | 406 | size_t |
412 | kmem_roundup_size(size_t size) | | 407 | kmem_roundup_size(size_t size) |
413 | { | | 408 | { |
414 | return (size + (KMEM_ALIGN - 1)) & ~(KMEM_ALIGN - 1); | | 409 | return (size + (KMEM_ALIGN - 1)) & ~(KMEM_ALIGN - 1); |
415 | } | | 410 | } |
416 | | | 411 | |
417 | /* | | 412 | /* |
418 | * Used to dynamically allocate string with kmem accordingly to format. | | 413 | * Used to dynamically allocate string with kmem accordingly to format. |
419 | */ | | 414 | */ |
420 | char * | | 415 | char * |
421 | kmem_asprintf(const char *fmt, ...) | | 416 | kmem_asprintf(const char *fmt, ...) |
422 | { | | 417 | { |
423 | int size __diagused, len; | | 418 | int size __diagused, len; |
424 | va_list va; | | 419 | va_list va; |
425 | char *str; | | 420 | char *str; |
426 | | | 421 | |
427 | va_start(va, fmt); | | 422 | va_start(va, fmt); |
428 | len = vsnprintf(NULL, 0, fmt, va); | | 423 | len = vsnprintf(NULL, 0, fmt, va); |
429 | va_end(va); | | 424 | va_end(va); |
430 | | | 425 | |
431 | str = kmem_alloc(len + 1, KM_SLEEP); | | 426 | str = kmem_alloc(len + 1, KM_SLEEP); |
432 | | | 427 | |
433 | va_start(va, fmt); | | 428 | va_start(va, fmt); |
434 | size = vsnprintf(str, len + 1, fmt, va); | | 429 | size = vsnprintf(str, len + 1, fmt, va); |
435 | va_end(va); | | 430 | va_end(va); |
436 | | | 431 | |
437 | KASSERT(size == len); | | 432 | KASSERT(size == len); |
438 | | | 433 | |
439 | return str; | | 434 | return str; |
440 | } | | 435 | } |
441 | | | 436 | |
442 | char * | | 437 | char * |
443 | kmem_strdupsize(const char *str, size_t *lenp, km_flag_t flags) | | 438 | kmem_strdupsize(const char *str, size_t *lenp, km_flag_t flags) |
444 | { | | 439 | { |
445 | size_t len = strlen(str) + 1; | | 440 | size_t len = strlen(str) + 1; |
446 | char *ptr = kmem_alloc(len, flags); | | 441 | char *ptr = kmem_alloc(len, flags); |
447 | if (ptr == NULL) | | 442 | if (ptr == NULL) |
448 | return NULL; | | 443 | return NULL; |
449 | | | 444 | |
450 | if (lenp) | | 445 | if (lenp) |
451 | *lenp = len; | | 446 | *lenp = len; |
452 | memcpy(ptr, str, len); | | 447 | memcpy(ptr, str, len); |
453 | return ptr; | | 448 | return ptr; |
454 | } | | 449 | } |
455 | | | 450 | |
456 | char * | | 451 | char * |
457 | kmem_strndup(const char *str, size_t maxlen, km_flag_t flags) | | 452 | kmem_strndup(const char *str, size_t maxlen, km_flag_t flags) |
458 | { | | 453 | { |
459 | KASSERT(str != NULL); | | 454 | KASSERT(str != NULL); |
460 | KASSERT(maxlen != 0); | | 455 | KASSERT(maxlen != 0); |
461 | | | 456 | |
462 | size_t len = strnlen(str, maxlen); | | 457 | size_t len = strnlen(str, maxlen); |
463 | char *ptr = kmem_alloc(len + 1, flags); | | 458 | char *ptr = kmem_alloc(len + 1, flags); |
464 | if (ptr == NULL) | | 459 | if (ptr == NULL) |
465 | return NULL; | | 460 | return NULL; |
466 | | | 461 | |
467 | memcpy(ptr, str, len); | | 462 | memcpy(ptr, str, len); |
468 | ptr[len] = '\0'; | | 463 | ptr[len] = '\0'; |
469 | | | 464 | |
470 | return ptr; | | 465 | return ptr; |
471 | } | | 466 | } |
472 | | | 467 | |
473 | void | | 468 | void |
474 | kmem_strfree(char *str) | | 469 | kmem_strfree(char *str) |
475 | { | | 470 | { |
476 | if (str == NULL) | | 471 | if (str == NULL) |
477 | return; | | 472 | return; |
478 | | | 473 | |
479 | kmem_free(str, strlen(str) + 1); | | 474 | kmem_free(str, strlen(str) + 1); |
480 | } | | 475 | } |
481 | | | 476 | |
482 | /* --------------------------- DEBUG / DIAGNOSTIC --------------------------- */ | | 477 | /* --------------------------- DEBUG / DIAGNOSTIC --------------------------- */ |
483 | | | 478 | |
484 | #if defined(KMEM_SIZE) | | 479 | #if defined(KMEM_SIZE) |
485 | static void | | 480 | static void |
486 | kmem_size_set(void *p, size_t sz) | | 481 | kmem_size_set(void *p, size_t sz) |
487 | { | | 482 | { |
488 | struct kmem_header *hd; | | 483 | memcpy((size_t *)((uintptr_t)p + sz), &sz, sizeof(size_t)); |
489 | hd = (struct kmem_header *)p; | | | |
490 | hd->size = sz; | | | |
491 | } | | 484 | } |
492 | | | 485 | |
493 | static void | | 486 | static void |
494 | kmem_size_check(void *p, size_t sz) | | 487 | kmem_size_check(void *p, size_t sz) |
495 | { | | 488 | { |
496 | struct kmem_header *hd; | | | |
497 | size_t hsz; | | 489 | size_t hsz; |
498 | | | 490 | |
499 | hd = (struct kmem_header *)p; | | 491 | memcpy(&hsz, (size_t *)((uintptr_t)p + sz), sizeof(size_t)); |
500 | hsz = hd->size; | | | |
501 | | | 492 | |
502 | if (hsz != sz) { | | 493 | if (hsz != sz) { |
503 | panic("kmem_free(%p, %zu) != allocated size %zu", | | 494 | panic("kmem_free(%p, %zu) != allocated size %zu; overwrote?", |
504 | (const uint8_t *)p + SIZE_SIZE, sz, hsz); | | 495 | p, sz, hsz); |
505 | } | | 496 | } |
506 | | | 497 | |
507 | hd->size = -1; | | 498 | memset((size_t *)((uintptr_t)p + sz), 0xff, sizeof(size_t)); |
508 | } | | 499 | } |
509 | #endif /* defined(KMEM_SIZE) */ | | 500 | #endif /* defined(KMEM_SIZE) */ |