 @@ -1,14 +1,14 @@
-/*	$NetBSD: kern_threadpool.c,v 1.8 2018/12/26 21:18:51 thorpej Exp $	*/
+/*	$NetBSD: kern_threadpool.c,v 1.9 2018/12/26 21:25:51 thorpej Exp $	*/
 /*-
  * Copyright (c) 2014, 2018 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Taylor R. Campbell and Jason R. Thorpe.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -71,27 +71,27 @@
  * |            | <running (n+1)b> |                        |
  * |            +------------------+                        |
  * +--------------------------------------------------------+
+ *
  * XXX Why one overseer per CPU?  I did that originally to avoid
  * touching remote CPUs' memory when scheduling a job, but that still
  * requires interprocessor synchronization.  Perhaps we could get by
  * with a single overseer thread, at the expense of another pointer in
  * struct threadpool_job to identify the CPU on which it must run
  * in order for the overseer to schedule it correctly.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_threadpool.c,v 1.8 2018/12/26 21:18:51 thorpej Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_threadpool.c,v 1.9 2018/12/26 21:25:51 thorpej Exp $");
 #include <sys/types.h>
 #include <sys/param.h>
 #include <sys/atomic.h>
 #include <sys/condvar.h>
 #include <sys/cpu.h>
 #include <sys/kernel.h>
 #include <sys/kmem.h>
 #include <sys/kthread.h>
 #include <sys/mutex.h>
 #include <sys/once.h>
 #include <sys/percpu.h>
 #include <sys/pool.h>
 @@ -375,38 +375,38 @@ threadpool_get(struct threadpool **poolp
 	THREADPOOL_INIT();
 	ASSERT_SLEEPABLE();
 	if (! threadpool_pri_is_valid(pri))
 		return EINVAL;
 	mutex_enter(&threadpools_lock);
 	tpu = threadpool_lookup_unbound(pri);
 	if (tpu == NULL) {
 		mutex_exit(&threadpools_lock);
 		TP_LOG(("%s: No pool for pri=%d, creating one.\n",
-			__func__, (int)pri));
+		    __func__, (int)pri));
 		tmp = kmem_zalloc(sizeof(*tmp), KM_SLEEP);
 		error = threadpool_create(&tmp->tpu_pool, NULL, pri);
 		if (error) {
 			kmem_free(tmp, sizeof(*tmp));
 			return error;
+		}
 		mutex_enter(&threadpools_lock);
 		tpu = threadpool_lookup_unbound(pri);
 		if (tpu == NULL) {
 			TP_LOG(("%s: Won the creation race for pri=%d.\n",
-				__func__, (int)pri));
+			    __func__, (int)pri));
 			tpu = tmp;
 			tmp = NULL;
 			threadpool_insert_unbound(tpu);
+		}
+	}
 	KASSERT(tpu != NULL);
 	tpu->tpu_refcnt++;
 	KASSERT(tpu->tpu_refcnt != 0);
 	mutex_exit(&threadpools_lock);
 	if (tmp != NULL) {
 		threadpool_destroy(&tmp->tpu_pool);
 		kmem_free(tmp, sizeof(*tmp));
 @@ -423,27 +423,27 @@ threadpool_put(struct threadpool *pool,
 	    container_of(pool, struct threadpool_unbound, tpu_pool);
 	THREADPOOL_INIT();
 	ASSERT_SLEEPABLE();
 	KASSERT(threadpool_pri_is_valid(pri));
 	mutex_enter(&threadpools_lock);
 	KASSERT(tpu == threadpool_lookup_unbound(pri));
 	KASSERT(0 < tpu->tpu_refcnt);
 	if (--tpu->tpu_refcnt == 0) {
 		TP_LOG(("%s: Last reference for pri=%d, destroying pool.\n",
-			__func__, (int)pri));
+		    __func__, (int)pri));
 		threadpool_remove_unbound(tpu);
 	} else {
 		tpu = NULL;
+	}
 	mutex_exit(&threadpools_lock);
 	if (tpu) {
 		threadpool_destroy(&tpu->tpu_pool);
 		kmem_free(tpu, sizeof(*tpu));
+	}
+}
 /* Per-CPU thread pools */
 @@ -456,36 +456,36 @@ threadpool_percpu_get(struct threadpool_
 	THREADPOOL_INIT();
 	ASSERT_SLEEPABLE();
 	if (! threadpool_pri_is_valid(pri))
 		return EINVAL;
 	mutex_enter(&threadpools_lock);
 	pool_percpu = threadpool_lookup_percpu(pri);
 	if (pool_percpu == NULL) {
 		mutex_exit(&threadpools_lock);
 		TP_LOG(("%s: No pool for pri=%d, creating one.\n",
-			__func__, (int)pri));
+		    __func__, (int)pri));
 		error = threadpool_percpu_create(&tmp, pri);
 		if (error)
 			return error;
 		KASSERT(tmp != NULL);
 		mutex_enter(&threadpools_lock);
 		pool_percpu = threadpool_lookup_percpu(pri);
 		if (pool_percpu == NULL) {
 			TP_LOG(("%s: Won the creation race for pri=%d.\n",
-				__func__, (int)pri));
+			    __func__, (int)pri));
 			pool_percpu = tmp;
 			tmp = NULL;
 			threadpool_insert_percpu(pool_percpu);
+		}
+	}
 	KASSERT(pool_percpu != NULL);
 	pool_percpu->tpp_refcnt++;
 	KASSERT(pool_percpu->tpp_refcnt != 0);
 	mutex_exit(&threadpools_lock);
 	if (tmp != NULL)
 		threadpool_percpu_destroy(tmp);
 	KASSERT(pool_percpu != NULL);
 @@ -498,27 +498,27 @@ threadpool_percpu_put(struct threadpool_
+{
 	THREADPOOL_INIT();
 	ASSERT_SLEEPABLE();
 	KASSERT(threadpool_pri_is_valid(pri));
 	mutex_enter(&threadpools_lock);
 	KASSERT(pool_percpu == threadpool_lookup_percpu(pri));
 	KASSERT(0 < pool_percpu->tpp_refcnt);
 	if (--pool_percpu->tpp_refcnt == 0) {
 		TP_LOG(("%s: Last reference for pri=%d, destroying pool.\n",
-			__func__, (int)pri));
+		    __func__, (int)pri));
 		threadpool_remove_percpu(pool_percpu);
 	} else {
 		pool_percpu = NULL;
+	}
 	mutex_exit(&threadpools_lock);
 	if (pool_percpu)
 		threadpool_percpu_destroy(pool_percpu);
+}
 struct threadpool *
 threadpool_percpu_ref(struct threadpool_percpu *pool_percpu)
+{
 @@ -665,33 +665,34 @@ threadpool_job_destroy(struct threadpool
 	job->job_lock = NULL;
 	KASSERT(job->job_thread == NULL);
 	KASSERT(job->job_refcnt == 0);
 	KASSERT(!cv_has_waiters(&job->job_cv));
 	cv_destroy(&job->job_cv);
 	job->job_fn = threadpool_job_dead;
 	(void)strlcpy(job->job_name, "deadjob", sizeof(job->job_name));
+}
 static int
 threadpool_job_hold(struct threadpool_job *job)
+{
 	unsigned int refcnt;
 	do {
 		refcnt = job->job_refcnt;
 		if (refcnt == UINT_MAX)
 			return EBUSY;
 	} while (atomic_cas_uint(&job->job_refcnt, refcnt, (refcnt + 1))
 	    != refcnt);
 	return 0;
+}
 static void
 threadpool_job_rele(struct threadpool_job *job)
+{
 	unsigned int refcnt;
 	do {
 		refcnt = job->job_refcnt;
 		KASSERT(0 < refcnt);
 		if (refcnt == 1) {
 			mutex_enter(job->job_lock);
 @@ -722,43 +723,43 @@ void
 threadpool_schedule_job(struct threadpool *pool, struct threadpool_job *job)
+{
 	KASSERT(mutex_owned(job->job_lock));
 	/*
 	 * If the job's already running, let it keep running.  The job
 	 * is guaranteed by the interlock not to end early -- if it had
 	 * ended early, threadpool_job_done would have set job_thread
 	 * to NULL under the interlock.
 	 */
 	if (__predict_true(job->job_thread != NULL)) {
 		TP_LOG(("%s: job '%s' already runnining.\n",
-			__func__, job->job_name));
+		    __func__, job->job_name));
 		return;
+	}
 	/* Otherwise, try to assign a thread to the job.  */
 	mutex_spin_enter(&pool->tp_lock);
 	if (__predict_false(TAILQ_EMPTY(&pool->tp_idle_threads))) {
 		/* Nobody's idle.  Give it to the overseer.  */
 		TP_LOG(("%s: giving job '%s' to overseer.\n",
-			__func__, job->job_name));
+		    __func__, job->job_name));
 		job->job_thread = &pool->tp_overseer;
 		TAILQ_INSERT_TAIL(&pool->tp_jobs, job, job_entry);
 	} else {
 		/* Assign it to the first idle thread.  */
 		job->job_thread = TAILQ_FIRST(&pool->tp_idle_threads);
 		TP_LOG(("%s: giving job '%s' to idle thread %p.\n",
-			__func__, job->job_name, job->job_thread));
+		    __func__, job->job_name, job->job_thread));
 		TAILQ_REMOVE(&pool->tp_idle_threads, job->job_thread,
 		    tpt_entry);
 		threadpool_job_hold(job);
 		job->job_thread->tpt_job = job;
+	}
 	/* Notify whomever we gave it to, overseer or idle thread.  */
 	KASSERT(job->job_thread != NULL);
 	cv_broadcast(&job->job_thread->tpt_cv);
 	mutex_spin_exit(&pool->tp_lock);
+}
 bool
 @@ -835,38 +836,38 @@ threadpool_overseer_thread(void *arg)
 	/* Wait until we're initialized.  */
 	mutex_spin_enter(&pool->tp_lock);
 	while (overseer->tpt_lwp == NULL)
 		cv_wait(&overseer->tpt_cv, &pool->tp_lock);
 	TP_LOG(("%s: starting.\n", __func__));
 	for (;;) {
 		/* Wait until there's a job.  */
 		while (TAILQ_EMPTY(&pool->tp_jobs)) {
 			if (ISSET(pool->tp_flags, THREADPOOL_DYING)) {
 				TP_LOG(("%s: THREADPOOL_DYING\n",
-					__func__));
+				    __func__));
 				break;
+			}
 			cv_wait(&overseer->tpt_cv, &pool->tp_lock);
+		}
 		if (__predict_false(TAILQ_EMPTY(&pool->tp_jobs)))
 			break;
 		/* If there are no threads, we'll have to try to start one.  */
 		if (TAILQ_EMPTY(&pool->tp_idle_threads)) {
 			TP_LOG(("%s: Got a job, need to create a thread.\n",
-				__func__));
+			    __func__));
 			threadpool_hold(pool);
 			mutex_spin_exit(&pool->tp_lock);
 			struct threadpool_thread *const thread =
 			    pool_cache_get(threadpool_thread_pc, PR_WAITOK);
 			thread->tpt_lwp = NULL;
 			thread->tpt_pool = pool;
 			thread->tpt_job = NULL;
 			cv_init(&thread->tpt_cv, "poolthrd");
 			ktflags = 0;
 			ktflags |= KTHREAD_MPSAFE;
 			if (pool->tp_pri < PRI_KERNEL)
 @@ -912,39 +913,39 @@ threadpool_overseer_thread(void *arg)
 		mutex_spin_exit(&pool->tp_lock);
 		mutex_enter(job->job_lock);
 		/* If the job was cancelled, we'll no longer be its thread.  */
 		if (__predict_true(job->job_thread == overseer)) {
 			mutex_spin_enter(&pool->tp_lock);
 			if (__predict_false(
 				    TAILQ_EMPTY(&pool->tp_idle_threads))) {
 				/*
 				 * Someone else snagged the thread
 				 * first.  We'll have to try again.
 				 */
 				TP_LOG(("%s: '%s' lost race to use idle thread.\n",
-					__func__, job->job_name));
+				    __func__, job->job_name));
 				TAILQ_INSERT_HEAD(&pool->tp_jobs, job,
 				    job_entry);
 			} else {
 				/*
 				 * Assign the job to the thread and
 				 * wake the thread so it starts work.
 				 */
 				struct threadpool_thread *const thread =
 				    TAILQ_FIRST(&pool->tp_idle_threads);
 				TP_LOG(("%s: '%s' gets thread %p\n",
-					__func__, job->job_name, thread));
+				    __func__, job->job_name, thread));
 				KASSERT(thread->tpt_job == NULL);
 				TAILQ_REMOVE(&pool->tp_idle_threads, thread,
 				    tpt_entry);
 				thread->tpt_job = job;
 				job->job_thread = thread;
 				cv_broadcast(&thread->tpt_cv);
 				/* Gave the thread our job reference.  */
 				rele_job = false;
+			}
 			mutex_spin_exit(&pool->tp_lock);
+		}
 		mutex_exit(job->job_lock);
 		if (__predict_false(rele_job))
 @@ -973,45 +974,45 @@ threadpool_thread(void *arg)
 	/* Wait until we're initialized and on the queue.  */
 	mutex_spin_enter(&pool->tp_lock);
 	while (thread->tpt_lwp == NULL)
 		cv_wait(&thread->tpt_cv, &pool->tp_lock);
 	TP_LOG(("%s: starting.\n", __func__));
 	KASSERT(thread->tpt_lwp == curlwp);
 	for (;;) {
 		/* Wait until we are assigned a job.  */
 		while (thread->tpt_job == NULL) {
 			if (ISSET(pool->tp_flags, THREADPOOL_DYING)) {
 				TP_LOG(("%s: THREADPOOL_DYING\n",
-					__func__));
+				    __func__));
 				break;
+			}
 			if (cv_timedwait(&thread->tpt_cv, &pool->tp_lock,
-					 THREADPOOL_IDLE_TICKS))
+				THREADPOOL_IDLE_TICKS))
 				break;
+		}
 		if (__predict_false(thread->tpt_job == NULL)) {
 			TAILQ_REMOVE(&pool->tp_idle_threads, thread,
 			    tpt_entry);
 			break;
+		}
 		struct threadpool_job *const job = thread->tpt_job;
 		KASSERT(job != NULL);
 		mutex_spin_exit(&pool->tp_lock);
 		TP_LOG(("%s: running job '%s' on thread %p.\n",
-			__func__, job->job_name, thread));
+		    __func__, job->job_name, thread));
 		/* Set our lwp name to reflect what job we're doing.  */
 		lwp_lock(curlwp);
 		char *const lwp_name = curlwp->l_name;
 		curlwp->l_name = job->job_name;
 		lwp_unlock(curlwp);
 		/* Run the job.  */
 		(*job->job_fn)(job);
 		/* Restore our lwp name.  */
 		lwp_lock(curlwp);
 		curlwp->l_name = lwp_name;