Sun Apr 19 20:47:04 2020 UTC ()
Improve TSD behavior

Optimistically check whether the key has been used by this thread
already and avoid locking in that case. This avoids the atomic operation
in the hot path. When the value is set to non-NULL for the first time,
put the entry on the to-be-freed list and keep it their until
destruction or thread exit. Setting the key to NULL and back is common
enough and updating the list is more expensive than the extra check on
the final round.


(joerg)
diff -r1.21 -r1.22 src/lib/libpthread/pthread_tsd.c

cvs diff -r1.21 -r1.22 src/lib/libpthread/pthread_tsd.c (expand / switch to unified diff)

--- src/lib/libpthread/pthread_tsd.c 2020/04/19 20:46:04 1.21
+++ src/lib/libpthread/pthread_tsd.c 2020/04/19 20:47:03 1.22
@@ -1,14 +1,14 @@ @@ -1,14 +1,14 @@
1/* $NetBSD: pthread_tsd.c,v 1.21 2020/04/19 20:46:04 joerg Exp $ */ 1/* $NetBSD: pthread_tsd.c,v 1.22 2020/04/19 20:47:03 joerg Exp $ */
2 2
3/*- 3/*-
4 * Copyright (c) 2001, 2007 The NetBSD Foundation, Inc. 4 * Copyright (c) 2001, 2007 The NetBSD Foundation, Inc.
5 * All rights reserved. 5 * All rights reserved.
6 * 6 *
7 * This code is derived from software contributed to The NetBSD Foundation 7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Nathan J. Williams, by Andrew Doran, and by Christos Zoulas. 8 * by Nathan J. Williams, by Andrew Doran, and by Christos Zoulas.
9 * 9 *
10 * Redistribution and use in source and binary forms, with or without 10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions 11 * modification, are permitted provided that the following conditions
12 * are met: 12 * are met:
13 * 1. Redistributions of source code must retain the above copyright 13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer. 14 * notice, this list of conditions and the following disclaimer.
@@ -20,27 +20,27 @@ @@ -20,27 +20,27 @@
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE. 29 * POSSIBILITY OF SUCH DAMAGE.
30 */ 30 */
31 31
32#include <sys/cdefs.h> 32#include <sys/cdefs.h>
33__RCSID("$NetBSD: pthread_tsd.c,v 1.21 2020/04/19 20:46:04 joerg Exp $"); 33__RCSID("$NetBSD: pthread_tsd.c,v 1.22 2020/04/19 20:47:03 joerg Exp $");
34 34
35/* Functions and structures dealing with thread-specific data */ 35/* Functions and structures dealing with thread-specific data */
36#include <errno.h> 36#include <errno.h>
37#include <sys/mman.h> 37#include <sys/mman.h>
38 38
39#include "pthread.h" 39#include "pthread.h"
40#include "pthread_int.h" 40#include "pthread_int.h"
41#include "reentrant.h" 41#include "reentrant.h"
42#include "tsd.h" 42#include "tsd.h"
43 43
44int pthread_keys_max; 44int pthread_keys_max;
45static pthread_mutex_t tsd_mutex = PTHREAD_MUTEX_INITIALIZER; 45static pthread_mutex_t tsd_mutex = PTHREAD_MUTEX_INITIALIZER;
46static int nextkey; 46static int nextkey;
@@ -163,61 +163,58 @@ pthread_key_create(pthread_key_t *key, v @@ -163,61 +163,58 @@ pthread_key_create(pthread_key_t *key, v
163 163
164 nextkey = (i + 1) % pthread_keys_max; 164 nextkey = (i + 1) % pthread_keys_max;
165 pthread_mutex_unlock(&tsd_mutex); 165 pthread_mutex_unlock(&tsd_mutex);
166 *key = i; 166 *key = i;
167 167
168 return 0; 168 return 0;
169} 169}
170 170
171/* 171/*
172 * Each thread holds an array of pthread_keys_max pt_specific list 172 * Each thread holds an array of pthread_keys_max pt_specific list
173 * elements. When an element is used it is inserted into the appropriate 173 * elements. When an element is used it is inserted into the appropriate
174 * key bucket of pthread__tsd_list. This means that ptqe_prev == NULL, 174 * key bucket of pthread__tsd_list. This means that ptqe_prev == NULL,
175 * means that the element is not threaded, ptqe_prev != NULL it is 175 * means that the element is not threaded, ptqe_prev != NULL it is
176 * already part of the list. When we set to a NULL value we delete from the 176 * already part of the list. If a key is set to a non-NULL value for the
177 * list if it was in the list, and when we set to non-NULL value, we insert 177 * first time, it is added to the list.
178 * in the list if it was not already there. 
179 * 178 *
180 * We keep this global array of lists of threads that have called 179 * We keep this global array of lists of threads that have called
181 * pthread_set_specific with non-null values, for each key so that 180 * pthread_set_specific with non-null values, for each key so that
182 * we don't have to check all threads for non-NULL values in 181 * we don't have to check all threads for non-NULL values in
183 * pthread_key_destroy 182 * pthread_key_destroy.
 183 *
 184 * The assumption here is that a concurrent pthread_key_delete is already
 185 * undefined behavior. The mutex is taken only once per thread/key
 186 * combination.
184 * 187 *
185 * We could keep an accounting of the number of specific used 188 * We could keep an accounting of the number of specific used
186 * entries per thread, so that we can update pt_havespecific when we delete 189 * entries per thread, so that we can update pt_havespecific when we delete
187 * the last one, but we don't bother for now 190 * the last one, but we don't bother for now
188 */ 191 */
189int 192int
190pthread__add_specific(pthread_t self, pthread_key_t key, const void *value) 193pthread__add_specific(pthread_t self, pthread_key_t key, const void *value)
191{ 194{
192 struct pt_specific *pt; 195 struct pt_specific *pt;
193 196
194 pthread__assert(key >= 0 && key < pthread_keys_max); 197 pthread__assert(key >= 0 && key < pthread_keys_max);
195 198
196 pthread_mutex_lock(&tsd_mutex); 
197 pthread__assert(pthread__tsd_destructors[key] != NULL); 199 pthread__assert(pthread__tsd_destructors[key] != NULL);
198 pt = &self->pt_specific[key]; 200 pt = &self->pt_specific[key];
199 self->pt_havespecific = 1; 201 self->pt_havespecific = 1;
200 if (value) { 202 if (value && !pt->pts_next.ptqe_prev) {
201 if (pt->pts_next.ptqe_prev == NULL) 203 pthread_mutex_lock(&tsd_mutex);
202 PTQ_INSERT_HEAD(&pthread__tsd_list[key], pt, pts_next); 204 PTQ_INSERT_HEAD(&pthread__tsd_list[key], pt, pts_next);
203 } else { 205 pthread_mutex_unlock(&tsd_mutex);
204 if (pt->pts_next.ptqe_prev != NULL) { 
205 PTQ_REMOVE(&pthread__tsd_list[key], pt, pts_next); 
206 pt->pts_next.ptqe_prev = NULL; 
207 } 
208 } 206 }
209 pt->pts_value = __UNCONST(value); 207 pt->pts_value = __UNCONST(value);
210 pthread_mutex_unlock(&tsd_mutex); 
211 208
212 return 0; 209 return 0;
213} 210}
214 211
215int 212int
216pthread_key_delete(pthread_key_t key) 213pthread_key_delete(pthread_key_t key)
217{ 214{
218 /* 215 /*
219 * This is tricky. The standard says of pthread_key_create() 216 * This is tricky. The standard says of pthread_key_create()
220 * that new keys have the value NULL associated with them in 217 * that new keys have the value NULL associated with them in
221 * all threads. According to people who were present at the 218 * all threads. According to people who were present at the
222 * standardization meeting, that requirement was written 219 * standardization meeting, that requirement was written
223 * before pthread_key_delete() was introduced, and not 220 * before pthread_key_delete() was introduced, and not
@@ -363,27 +360,27 @@ pthread__destroy_tsd(pthread_t self) @@ -363,27 +360,27 @@ pthread__destroy_tsd(pthread_t self)
363 continue; 360 continue;
364 pthread_mutex_lock(&tsd_mutex); 361 pthread_mutex_lock(&tsd_mutex);
365 362
366 if (pt->pts_next.ptqe_prev != NULL) { 363 if (pt->pts_next.ptqe_prev != NULL) {
367 PTQ_REMOVE(&pthread__tsd_list[i], pt, pts_next); 364 PTQ_REMOVE(&pthread__tsd_list[i], pt, pts_next);
368 val = pt->pts_value; 365 val = pt->pts_value;
369 pt->pts_value = NULL; 366 pt->pts_value = NULL;
370 pt->pts_next.ptqe_prev = NULL; 367 pt->pts_next.ptqe_prev = NULL;
371 destructor = pthread__tsd_destructors[i]; 368 destructor = pthread__tsd_destructors[i];
372 } else 369 } else
373 destructor = NULL; 370 destructor = NULL;
374 371
375 pthread_mutex_unlock(&tsd_mutex); 372 pthread_mutex_unlock(&tsd_mutex);
376 if (destructor != NULL) { 373 if (destructor != NULL && val != NULL) {
377 done = 0; 374 done = 0;
378 (*destructor)(val); 375 (*destructor)(val);
379 } 376 }
380 } 377 }
381 } while (!done && --iterations); 378 } while (!done && --iterations);
382 379
383 self->pt_havespecific = 0; 380 self->pt_havespecific = 0;
384 pthread_mutex_lock(&self->pt_lock); 381 pthread_mutex_lock(&self->pt_lock);
385} 382}
386 383
387void 384void
388pthread__copy_tsd(pthread_t self) 385pthread__copy_tsd(pthread_t self)
389{ 386{