 @@ -1,14 +1,14 @@
-/*	$NetBSD: kern_threadpool.c,v 1.4 2018/12/26 18:54:19 thorpej Exp $	*/
+/*	$NetBSD: kern_threadpool.c,v 1.5 2018/12/26 20:08:22 thorpej Exp $	*/
 /*-
  * Copyright (c) 2014, 2018 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Taylor R. Campbell and Jason R. Thorpe.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -71,27 +71,27 @@
  * |            | <running (n+1)b> |                        |
  * |            +------------------+                        |
  * +--------------------------------------------------------+
+ *
  * XXX Why one overseer per CPU?  I did that originally to avoid
  * touching remote CPUs' memory when scheduling a job, but that still
  * requires interprocessor synchronization.  Perhaps we could get by
  * with a single overseer thread, at the expense of another pointer in
  * struct threadpool_job to identify the CPU on which it must run
  * in order for the overseer to schedule it correctly.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_threadpool.c,v 1.4 2018/12/26 18:54:19 thorpej Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_threadpool.c,v 1.5 2018/12/26 20:08:22 thorpej Exp $");
 #include <sys/types.h>
 #include <sys/param.h>
 #include <sys/atomic.h>
 #include <sys/condvar.h>
 #include <sys/cpu.h>
 #include <sys/kernel.h>
 #include <sys/kmem.h>
 #include <sys/kthread.h>
 #include <sys/mutex.h>
 #include <sys/once.h>
 #include <sys/percpu.h>
 #include <sys/pool.h>
 @@ -151,27 +151,27 @@ static void	threadpool_thread(void *) __
 static pool_cache_t	threadpool_thread_pc __read_mostly;
 static kmutex_t		threadpools_lock __cacheline_aligned;
 	/* Idle out threads after 30 seconds */
 #define	THREADPOOL_IDLE_TICKS	mstohz(30 * 1000)
 struct threadpool_unbound {
 	/* must be first; see threadpool_create() */
 	struct threadpool		tpu_pool;
 	/* protected by threadpools_lock */
 	LIST_ENTRY(threadpool_unbound)	tpu_link;
-	unsigned int			tpu_refcnt;
+	uint64_t			tpu_refcnt;
 };
 static LIST_HEAD(, threadpool_unbound) unbound_threadpools;
 static struct threadpool_unbound *
 threadpool_lookup_unbound(pri_t pri)
+{
 	struct threadpool_unbound *tpu;
 	LIST_FOREACH(tpu, &unbound_threadpools, tpu_link) {
 		if (tpu->tpu_pool.tp_pri == pri)
 			return tpu;
+	}
 @@ -188,27 +188,27 @@ threadpool_insert_unbound(struct threadp
 static void
 threadpool_remove_unbound(struct threadpool_unbound *tpu)
+{
 	KASSERT(threadpool_lookup_unbound(tpu->tpu_pool.tp_pri) == tpu);
 	LIST_REMOVE(tpu, tpu_link);
+}
 struct threadpool_percpu {
 	percpu_t *			tpp_percpu;
 	pri_t				tpp_pri;
 	/* protected by threadpools_lock */
 	LIST_ENTRY(threadpool_percpu)	tpp_link;
-	unsigned int			tpp_refcnt;
+	uint64_t			tpp_refcnt;
 };
 static LIST_HEAD(, threadpool_percpu) percpu_threadpools;
 static struct threadpool_percpu *
 threadpool_lookup_percpu(pri_t pri)
+{
 	struct threadpool_percpu *tpp;
 	LIST_FOREACH(tpp, &percpu_threadpools, tpp_link) {
 		if (tpp->tpp_pri == pri)
 			return tpp;
+	}
 @@ -418,33 +418,28 @@ threadpool_get(struct threadpool **poolp
 		tmp = container_of(new_pool, struct threadpool_unbound,
 		    tpu_pool);
 		mutex_enter(&threadpools_lock);
 		tpu = threadpool_lookup_unbound(pri);
 		if (tpu == NULL) {
 			TP_LOG(("%s: Won the creation race for pri=%d.\n",
 				__func__, (int)pri));
 			tpu = tmp;
 			tmp = NULL;
 			threadpool_insert_unbound(tpu);
+		}
+	}
 	KASSERT(tpu != NULL);
 	if (tpu->tpu_refcnt == UINT_MAX) {
 		mutex_exit(&threadpools_lock);
 		if (tmp != NULL)
 			threadpool_destroy(&tmp->tpu_pool, sizeof(*tpu));
 		return EBUSY;
 	tpu->tpu_refcnt++;
 	KASSERT(tpu->tpu_refcnt != 0);
 	mutex_exit(&threadpools_lock);
 	if (tmp != NULL)
 		threadpool_destroy((struct threadpool *)tmp, sizeof(*tpu));
 	KASSERT(tpu != NULL);
 	*poolp = &tpu->tpu_pool;
 	return 0;
+}
 void
 threadpool_put(struct threadpool *pool, pri_t pri)
+{
 	struct threadpool_unbound *tpu =
 @@ -453,28 +448,29 @@ threadpool_put(struct threadpool *pool,
 	THREADPOOL_INIT();
 	ASSERT_SLEEPABLE();
 	KASSERT(threadpool_pri_is_valid(pri));
 	mutex_enter(&threadpools_lock);
 	KASSERT(tpu == threadpool_lookup_unbound(pri));
 	KASSERT(0 < tpu->tpu_refcnt);
 	if (--tpu->tpu_refcnt == 0) {
 		TP_LOG(("%s: Last reference for pri=%d, destroying pool.\n",
 			__func__, (int)pri));
 		threadpool_remove_unbound(tpu);
-	} else
+	} else {
 		tpu = NULL;
+	}
 	mutex_exit(&threadpools_lock);
 	if (tpu)
 		threadpool_destroy(pool, sizeof(*tpu));
+}
 /* Per-CPU thread pools */
 int
 threadpool_percpu_get(struct threadpool_percpu **pool_percpup, pri_t pri)
+{
 	struct threadpool_percpu *pool_percpu, *tmp = NULL;
 	int error;
 @@ -497,61 +493,57 @@ threadpool_percpu_get(struct threadpool_
 			return error;
 		KASSERT(tmp != NULL);
 		mutex_enter(&threadpools_lock);
 		pool_percpu = threadpool_lookup_percpu(pri);
 		if (pool_percpu == NULL) {
 			TP_LOG(("%s: Won the creation race for pri=%d.\n",
 				__func__, (int)pri));
 			pool_percpu = tmp;
 			tmp = NULL;
 			threadpool_insert_percpu(pool_percpu);
+		}
+	}
 	KASSERT(pool_percpu != NULL);
 	if (pool_percpu->tpp_refcnt == UINT_MAX) {
 		mutex_exit(&threadpools_lock);
 		if (tmp != NULL)
 			threadpool_percpu_destroy(tmp);
 		return EBUSY;
 	pool_percpu->tpp_refcnt++;
 	KASSERT(pool_percpu->tpp_refcnt != 0);
 	mutex_exit(&threadpools_lock);
 	if (tmp != NULL)
 		threadpool_percpu_destroy(tmp);
 	KASSERT(pool_percpu != NULL);
 	*pool_percpup = pool_percpu;
 	return 0;
+}
 void
 threadpool_percpu_put(struct threadpool_percpu *pool_percpu, pri_t pri)
+{
 	THREADPOOL_INIT();
 	ASSERT_SLEEPABLE();
 	KASSERT(threadpool_pri_is_valid(pri));
 	mutex_enter(&threadpools_lock);
 	KASSERT(pool_percpu == threadpool_lookup_percpu(pri));
 	KASSERT(0 < pool_percpu->tpp_refcnt);
 	if (--pool_percpu->tpp_refcnt == 0) {
 		TP_LOG(("%s: Last reference for pri=%d, destroying pool.\n",
 			__func__, (int)pri));
 		threadpool_remove_percpu(pool_percpu);
-	} else
+	} else {
 		pool_percpu = NULL;
+	}
 	mutex_exit(&threadpools_lock);
 	if (pool_percpu)
 		threadpool_percpu_destroy(pool_percpu);
+}
 struct threadpool *
 threadpool_percpu_ref(struct threadpool_percpu *pool_percpu)
+{
 	struct threadpool **poolp, *pool;
 	poolp = percpu_getref(pool_percpu->tpp_percpu);
 	pool = *poolp;