 @@ -1,14 +1,14 @@
-/*	$NetBSD: kern_threadpool.c,v 1.5 2018/12/26 20:08:22 thorpej Exp $	*/
+/*	$NetBSD: kern_threadpool.c,v 1.6 2018/12/26 20:30:36 thorpej Exp $	*/
 /*-
  * Copyright (c) 2014, 2018 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Taylor R. Campbell and Jason R. Thorpe.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -71,27 +71,27 @@
  * |            | <running (n+1)b> |                        |
  * |            +------------------+                        |
  * +--------------------------------------------------------+
+ *
  * XXX Why one overseer per CPU?  I did that originally to avoid
  * touching remote CPUs' memory when scheduling a job, but that still
  * requires interprocessor synchronization.  Perhaps we could get by
  * with a single overseer thread, at the expense of another pointer in
  * struct threadpool_job to identify the CPU on which it must run
  * in order for the overseer to schedule it correctly.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_threadpool.c,v 1.5 2018/12/26 20:08:22 thorpej Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_threadpool.c,v 1.6 2018/12/26 20:30:36 thorpej Exp $");
 #include <sys/types.h>
 #include <sys/param.h>
 #include <sys/atomic.h>
 #include <sys/condvar.h>
 #include <sys/cpu.h>
 #include <sys/kernel.h>
 #include <sys/kmem.h>
 #include <sys/kthread.h>
 #include <sys/mutex.h>
 #include <sys/once.h>
 #include <sys/percpu.h>
 #include <sys/pool.h>
 @@ -146,27 +146,26 @@ static int	threadpool_job_hold(struct th
 static void	threadpool_job_rele(struct threadpool_job *);
 static void	threadpool_overseer_thread(void *) __dead;
 static void	threadpool_thread(void *) __dead;
 static pool_cache_t	threadpool_thread_pc __read_mostly;
 static kmutex_t		threadpools_lock __cacheline_aligned;
 	/* Idle out threads after 30 seconds */
 #define	THREADPOOL_IDLE_TICKS	mstohz(30 * 1000)
 struct threadpool_unbound {
 	/* must be first; see threadpool_create() */
 	struct threadpool		tpu_pool;
 	/* protected by threadpools_lock */
 	LIST_ENTRY(threadpool_unbound)	tpu_link;
 	uint64_t			tpu_refcnt;
 };
 static LIST_HEAD(, threadpool_unbound) unbound_threadpools;
 static struct threadpool_unbound *
 threadpool_lookup_unbound(pri_t pri)
+{
 	struct threadpool_unbound *tpu;
 @@ -252,30 +251,29 @@ threadpools_init(void)
 	return 0;
+}
 /* Thread pool creation */
 static bool
 threadpool_pri_is_valid(pri_t pri)
+{
 	return (pri == PRI_NONE || (pri >= PRI_USER && pri < PRI_COUNT));
+}
 static int
-threadpool_create(struct threadpool **poolp, struct cpu_info *ci, pri_t pri,
+threadpool_create(struct threadpool *const pool, struct cpu_info *ci,
-    size_t size)
+    pri_t pri)
+{
 	struct threadpool *const pool = kmem_zalloc(size, KM_SLEEP);
 	struct lwp *lwp;
 	int ktflags;
 	int error;
 	KASSERT(threadpool_pri_is_valid(pri));
 	mutex_init(&pool->tp_lock, MUTEX_DEFAULT, IPL_VM);
 	/* XXX overseer */
 	TAILQ_INIT(&pool->tp_jobs);
 	TAILQ_INIT(&pool->tp_idle_threads);
 	pool->tp_refcnt = 0;
 	pool->tp_flags = 0;
 	pool->tp_cpu = ci;
 @@ -293,74 +291,71 @@ threadpool_create(struct threadpool **po
 	if (pri < PRI_KERNEL)
 		ktflags |= KTHREAD_TS;
 	error = kthread_create(pri, ktflags, ci, &threadpool_overseer_thread,
 	    &pool->tp_overseer, &lwp,
 	    "pooloverseer/%d@%d", (ci ? cpu_index(ci) : -1), (int)pri);
 	if (error)
 		goto fail0;
 	mutex_spin_enter(&pool->tp_lock);
 	pool->tp_overseer.tpt_lwp = lwp;
 	cv_broadcast(&pool->tp_overseer.tpt_cv);
 	mutex_spin_exit(&pool->tp_lock);
 	*poolp = pool;
 	return 0;
 fail0:	KASSERT(error);
 	KASSERT(pool->tp_overseer.tpt_job == NULL);
 	KASSERT(pool->tp_overseer.tpt_pool == pool);
 	KASSERT(pool->tp_flags == 0);
 	KASSERT(pool->tp_refcnt == 0);
 	KASSERT(TAILQ_EMPTY(&pool->tp_idle_threads));
 	KASSERT(TAILQ_EMPTY(&pool->tp_jobs));
 	KASSERT(!cv_has_waiters(&pool->tp_overseer.tpt_cv));
 	cv_destroy(&pool->tp_overseer.tpt_cv);
 	mutex_destroy(&pool->tp_lock);
 	kmem_free(pool, size);
 	return error;
+}
 /* Thread pool destruction */
 static void
-threadpool_destroy(struct threadpool *pool, size_t size)
+threadpool_destroy(struct threadpool *pool)
+{
 	struct threadpool_thread *thread;
 	/* Mark the pool dying and wait for threads to commit suicide.  */
 	mutex_spin_enter(&pool->tp_lock);
 	KASSERT(TAILQ_EMPTY(&pool->tp_jobs));
 	pool->tp_flags |= THREADPOOL_DYING;
 	cv_broadcast(&pool->tp_overseer.tpt_cv);
 	TAILQ_FOREACH(thread, &pool->tp_idle_threads, tpt_entry)
 		cv_broadcast(&thread->tpt_cv);
 	while (0 < pool->tp_refcnt) {
 		TP_LOG(("%s: draining %u references...\n", __func__,
 		    pool->tp_refcnt));
 		cv_wait(&pool->tp_overseer.tpt_cv, &pool->tp_lock);
+	}
 	mutex_spin_exit(&pool->tp_lock);
 	KASSERT(pool->tp_overseer.tpt_job == NULL);
 	KASSERT(pool->tp_overseer.tpt_pool == pool);
 	KASSERT(pool->tp_flags == THREADPOOL_DYING);
 	KASSERT(pool->tp_refcnt == 0);
 	KASSERT(TAILQ_EMPTY(&pool->tp_idle_threads));
 	KASSERT(TAILQ_EMPTY(&pool->tp_jobs));
 	KASSERT(!cv_has_waiters(&pool->tp_overseer.tpt_cv));
 	cv_destroy(&pool->tp_overseer.tpt_cv);
 	mutex_destroy(&pool->tp_lock);
 	kmem_free(pool, size);
+}
 static int
 threadpool_hold(struct threadpool *pool)
+{
 	unsigned int refcnt;
 	do {
 		refcnt = pool->tp_refcnt;
 		if (refcnt == UINT_MAX)
 			return EBUSY;
 	} while (atomic_cas_uint(&pool->tp_refcnt, refcnt, (refcnt + 1))
 	    != refcnt);
 @@ -397,53 +392,54 @@ threadpool_get(struct threadpool **poolp
 	struct threadpool_unbound *tpu, *tmp = NULL;
 	int error;
 	THREADPOOL_INIT();
 	ASSERT_SLEEPABLE();
 	if (! threadpool_pri_is_valid(pri))
 		return EINVAL;
 	mutex_enter(&threadpools_lock);
 	tpu = threadpool_lookup_unbound(pri);
 	if (tpu == NULL) {
 		struct threadpool *new_pool;
 		mutex_exit(&threadpools_lock);
 		TP_LOG(("%s: No pool for pri=%d, creating one.\n",
 			__func__, (int)pri));
-		error = threadpool_create(&new_pool, NULL, pri, sizeof(*tpu));
+		tmp = kmem_zalloc(sizeof(*tmp), KM_SLEEP);
-		if (error)
+		error = threadpool_create(&tmp->tpu_pool, NULL, pri);
 		if (error) {
 			kmem_free(tmp, sizeof(*tmp));
 			return error;
 		KASSERT(new_pool != NULL);
 		tmp = container_of(new_pool, struct threadpool_unbound,
 		    tpu_pool);
 		mutex_enter(&threadpools_lock);
 		tpu = threadpool_lookup_unbound(pri);
 		if (tpu == NULL) {
 			TP_LOG(("%s: Won the creation race for pri=%d.\n",
 				__func__, (int)pri));
 			tpu = tmp;
 			tmp = NULL;
 			threadpool_insert_unbound(tpu);
+		}
+	}
 	KASSERT(tpu != NULL);
 	tpu->tpu_refcnt++;
 	KASSERT(tpu->tpu_refcnt != 0);
 	mutex_exit(&threadpools_lock);
-	if (tmp != NULL)
+	if (tmp != NULL) {
-		threadpool_destroy((struct threadpool *)tmp, sizeof(*tpu));
+		threadpool_destroy(&tmp->tpu_pool);
 		kmem_free(tmp, sizeof(*tmp));
+	}
 	KASSERT(tpu != NULL);
 	*poolp = &tpu->tpu_pool;
 	return 0;
+}
 void
 threadpool_put(struct threadpool *pool, pri_t pri)
+{
 	struct threadpool_unbound *tpu =
 	    container_of(pool, struct threadpool_unbound, tpu_pool);
 	THREADPOOL_INIT();
 @@ -453,28 +449,30 @@ threadpool_put(struct threadpool *pool,
 	mutex_enter(&threadpools_lock);
 	KASSERT(tpu == threadpool_lookup_unbound(pri));
 	KASSERT(0 < tpu->tpu_refcnt);
 	if (--tpu->tpu_refcnt == 0) {
 		TP_LOG(("%s: Last reference for pri=%d, destroying pool.\n",
 			__func__, (int)pri));
 		threadpool_remove_unbound(tpu);
 	} else {
 		tpu = NULL;
+	}
 	mutex_exit(&threadpools_lock);
-	if (tpu)
+	if (tpu) {
-		threadpool_destroy(pool, sizeof(*tpu));
+		threadpool_destroy(&tpu->tpu_pool);
 		kmem_free(tpu, sizeof(*tpu));
+	}
+}
 /* Per-CPU thread pools */
 int
 threadpool_percpu_get(struct threadpool_percpu **pool_percpup, pri_t pri)
+{
 	struct threadpool_percpu *pool_percpu, *tmp = NULL;
 	int error;
 	THREADPOOL_INIT();
 	ASSERT_SLEEPABLE();
 @@ -581,68 +579,73 @@ threadpool_percpu_create(struct threadpo
 		goto fail0;
+	}
 	pool_percpu->tpp_pri = pri;
 	pool_percpu->tpp_percpu = percpu_alloc(sizeof(struct threadpool *));
 	if (pool_percpu->tpp_percpu == NULL) {
 		error = ENOMEM;
 		goto fail1;
+	}
 	for (i = 0, CPU_INFO_FOREACH(cii, ci), i++) {
 		struct threadpool *pool;
-		error = threadpool_create(&pool, ci, pri, sizeof(*pool));
+		pool = kmem_zalloc(sizeof(*pool), KM_SLEEP);
-		if (error)
+		error = threadpool_create(pool, ci, pri);
 		if (error) {
 			kmem_free(pool, sizeof(*pool));
 			goto fail2;
+		}
 		percpu_traverse_enter();
 		struct threadpool **const poolp =
 		    percpu_getptr_remote(pool_percpu->tpp_percpu, ci);
 		*poolp = pool;
 		percpu_traverse_exit();
+	}
 	/* Success!  */
 	*pool_percpup = (struct threadpool_percpu *)pool_percpu;
 	return 0;
 fail2:	for (j = 0, CPU_INFO_FOREACH(cii, ci), j++) {
 		if (i <= j)
 			break;
 		percpu_traverse_enter();
 		struct threadpool **const poolp =
 		    percpu_getptr_remote(pool_percpu->tpp_percpu, ci);
 		struct threadpool *const pool = *poolp;
 		percpu_traverse_exit();
-		threadpool_destroy(pool, sizeof(*pool));
+		threadpool_destroy(pool);
 		kmem_free(pool, sizeof(*pool));
+	}
 	percpu_free(pool_percpu->tpp_percpu, sizeof(struct taskthread_pool *));
 fail1:	kmem_free(pool_percpu, sizeof(*pool_percpu));
 fail0:	return error;
+}
 static void
 threadpool_percpu_destroy(struct threadpool_percpu *pool_percpu)
+{
 	struct cpu_info *ci;
 	CPU_INFO_ITERATOR cii;
 	for (CPU_INFO_FOREACH(cii, ci)) {
 		percpu_traverse_enter();
 		struct threadpool **const poolp =
 		    percpu_getptr_remote(pool_percpu->tpp_percpu, ci);
 		struct threadpool *const pool = *poolp;
 		percpu_traverse_exit();
-		threadpool_destroy(pool, sizeof(*pool));
+		threadpool_destroy(pool);
 		kmem_free(pool, sizeof(*pool));
+	}
 	percpu_free(pool_percpu->tpp_percpu, sizeof(struct threadpool *));
 	kmem_free(pool_percpu, sizeof(*pool_percpu));
+}
 /* Thread pool jobs */
 void __printflike(4,5)
 threadpool_job_init(struct threadpool_job *job, threadpool_job_fn_t fn,
     kmutex_t *lock, const char *fmt, ...)
+{
 	va_list ap;