 @@ -1,14 +1,14 @@
-/*	$NetBSD: kern_physio.c,v 1.87 2008/02/15 13:46:04 ad Exp $	*/
+/*	$NetBSD: kern_physio.c,v 1.88 2008/09/24 08:19:19 hannken Exp $	*/
 /*-
  * Copyright (c) 1982, 1986, 1990, 1993
  *	The Regents of the University of California.  All rights reserved.
  * (c) UNIX System Laboratories, Inc.
  * All or some portions of this file are derived from material licensed
  * to the University of California by American Telephone and Telegraph
  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
  * the permission of UNIX System Laboratories, Inc.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
 @@ -61,103 +61,65 @@
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
+ *
  *	@(#)kern_physio.c	8.1 (Berkeley) 6/10/93
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_physio.c,v 1.87 2008/02/15 13:46:04 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_physio.c,v 1.88 2008/09/24 08:19:19 hannken Exp $");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/buf.h>
 #include <sys/proc.h>
 #include <sys/once.h>
 #include <sys/workqueue.h>
 #include <sys/kmem.h>
 #include <uvm/uvm_extern.h>
 ONCE_DECL(physio_initialized);
 struct workqueue *physio_workqueue;
 /*
  * The routines implemented in this file are described in:
  *	Leffler, et al.: The Design and Implementation of the 4.3BSD
  *	    UNIX Operating System (Addison Welley, 1989)
  * on pages 231-233.
  * The routines "getphysbuf" and "putphysbuf" steal and return a swap
  * buffer.  Leffler, et al., says that swap buffers are used to do the
  * I/O, so raw I/O requests don't have to be single-threaded.  Of course,
  * NetBSD doesn't use "swap buffers" -- we have our own memory pool for
  * buffer descriptors.
  */
 /* #define	PHYSIO_DEBUG */
 #if defined(PHYSIO_DEBUG)
 #define	DPRINTF(a)	printf a
 #else /* defined(PHYSIO_DEBUG) */
 #define	DPRINTF(a)	/* nothing */
 #endif /* defined(PHYSIO_DEBUG) */
 struct physio_stat {
 	int ps_running;
 	int ps_error;
 	int ps_failed;
 	off_t ps_endoffset;
 	buf_t *ps_orig_bp;
 	kmutex_t ps_lock;
 	kcondvar_t ps_cv;
 };
 /* abuse these flags of struct buf */
 #define	BC_DONTFREE	BC_AGE
 /*
  * allocate a buffer structure for use in physical I/O.
  */
 static struct buf *
 getphysbuf(void)
 	struct buf *bp;
 	bp = getiobuf(NULL, true);
 	bp->b_error = 0;
 	bp->b_cflags = BC_BUSY;
 	return(bp);
 /*
  * get rid of a swap buffer structure which has been used in physical I/O.
  */
 static void
 putphysbuf(struct buf *bp)
 	if ((bp->b_cflags & BC_DONTFREE) != 0) {
 		return;
 	if (__predict_false(bp->b_cflags & BC_WANTED))
 		panic("putphysbuf: private buf BC_WANTED");
 	putiobuf(bp);
 static void
 physio_done(struct work *wk, void *dummy)
+{
 	struct buf *bp = (void *)wk;
 	size_t todo = bp->b_bufsize;
 	size_t done = bp->b_bcount - bp->b_resid;
 	struct physio_stat *ps = bp->b_private;
 	KASSERT(&bp->b_work == wk);
 	KASSERT(bp->b_bcount <= todo);
 	KASSERT(bp->b_resid <= bp->b_bcount);
 	KASSERT((bp->b_flags & B_PHYS) != 0);
 	KASSERT(dummy == NULL);
 @@ -191,27 +153,28 @@ physio_done(struct work *wk, void *dummy
 			ps->ps_endoffset = endoffset;
 			ps->ps_error = bp->b_error;
+		}
 		ps->ps_failed++;
 	} else {
 		KASSERT(bp->b_error == 0);
+	}
 	ps->ps_running--;
 	cv_signal(&ps->ps_cv);
 	mutex_exit(&ps->ps_lock);
-	putphysbuf(bp);
+	if (bp != ps->ps_orig_bp)
 		putiobuf(bp);
+}
 static void
 physio_biodone(struct buf *bp)
+{
 #if defined(DIAGNOSTIC)
 	struct physio_stat *ps = bp->b_private;
 	size_t todo = bp->b_bufsize;
 	KASSERT(ps->ps_running > 0);
 	KASSERT(bp->b_bcount <= todo);
 	KASSERT(bp->b_resid <= bp->b_bcount);
 #endif /* defined(DIAGNOSTIC) */
 @@ -267,38 +230,38 @@ physio(void (*strategy)(struct buf *), s
 		return error;
+	}
 	DPRINTF(("%s: called: off=%" PRIu64 ", resid=%zu\n",
 	    __func__, uio->uio_offset, uio->uio_resid));
 	flags &= B_READ | B_WRITE;
 	if ((ps = kmem_zalloc(sizeof(*ps), KM_SLEEP)) == NULL)
 		return ENOMEM;
 	/* ps->ps_running = 0; */
 	/* ps->ps_error = 0; */
 	/* ps->ps_failed = 0; */
 	ps->ps_orig_bp = obp;
 	ps->ps_endoffset = -1;
 	mutex_init(&ps->ps_lock, MUTEX_DEFAULT, IPL_NONE);
 	cv_init(&ps->ps_cv, "physio");
 	/* Make sure we have a buffer, creating one if necessary. */
 	if (obp != NULL) {
 		/* [raise the processor priority level to splbio;] */
 		mutex_enter(&bufcache_lock);
 		/* Mark it busy, so nobody else will use it. */
 		while (bbusy(obp, false, 0, NULL) == EPASSTHROUGH)
+			;
 		/* Mark it busy, so nobody else will use it. */
 		obp->b_cflags |= BC_DONTFREE;
 		mutex_exit(&bufcache_lock);
 		concurrency = 0; /* see "XXXkludge" comment below */
+	}
 	uvm_lwp_hold(l);
 	for (i = 0; i < uio->uio_iovcnt; i++) {
 		bool sync = true;
 		iovp = &uio->uio_iov[i];
 		while (iovp->iov_len > 0) {
 			size_t todo;
 			vaddr_t endp;
 @@ -306,41 +269,42 @@ physio(void (*strategy)(struct buf *), s
 			mutex_enter(&ps->ps_lock);
 			if (ps->ps_failed != 0) {
 				goto done_locked;
+			}
 			physio_wait(ps, sync ? 0 : concurrency);
 			mutex_exit(&ps->ps_lock);
 			if (obp != NULL) {
 				/*
 				 * XXXkludge
 				 * some drivers use "obp" as an identifier.
 				 */
 				bp = obp;
 			} else {
-				bp = getphysbuf();
+				bp = getiobuf(NULL, true);
 				bp->b_cflags = BC_BUSY;
+			}
 			bp->b_dev = dev;
 			bp->b_proc = p;
 			bp->b_private = ps;
 			/*
 			 * [mark the buffer busy for physical I/O]
 			 * (i.e. set B_PHYS (because it's an I/O to user
 			 * memory, and B_RAW, because B_RAW is to be
 			 * "Set by physio for raw transfers.", in addition
 			 * to the "busy" and read/write flag.)
 			 */
 			bp->b_oflags = 0;
-			bp->b_cflags = (bp->b_cflags & BC_DONTFREE) | BC_BUSY;
+			bp->b_cflags = BC_BUSY;
 			bp->b_flags = flags | B_PHYS | B_RAW;
 			bp->b_iodone = physio_biodone;
 			/* [set up the buffer for a maximum-sized transfer] */
 			bp->b_blkno = btodb(uio->uio_offset);
 			if (dbtob(bp->b_blkno) != uio->uio_offset) {
 				error = EINVAL;
 				goto done;
+			}
 			bp->b_bcount = MIN(MAXPHYS, iovp->iov_len);
 			bp->b_data = iovp->iov_base;
 			/*
 @@ -403,51 +367,50 @@ done_locked:
 	physio_wait(ps, 0);
 	mutex_exit(&ps->ps_lock);
 	if (ps->ps_failed != 0) {
 		off_t delta;
 		delta = uio->uio_offset - ps->ps_endoffset;
 		KASSERT(delta > 0);
 		uio->uio_resid += delta;
 		/* uio->uio_offset = ps->ps_endoffset; */
 	} else {
 		KASSERT(ps->ps_endoffset == -1);
+	}
-	if (bp != NULL) {
+	if (bp != NULL && bp != obp) {
-		putphysbuf(bp);
+		putiobuf(bp);
+	}
 	if (error == 0) {
 		error = ps->ps_error;
+	}
 	mutex_destroy(&ps->ps_lock);
 	cv_destroy(&ps->ps_cv);
 	kmem_free(ps, sizeof(*ps));
 	/*
 	 * [clean up the state of the buffer]
 	 * Remember if somebody wants it, so we can wake them up below.
 	 * Also, if we had to steal it, give it back.
 	 */
 	if (obp != NULL) {
 		KASSERT((obp->b_cflags & BC_BUSY) != 0);
 		KASSERT((obp->b_cflags & BC_DONTFREE) != 0);
 		/*
 		 * [if another process is waiting for the raw I/O buffer,
 		 *    wake up processes waiting to do physical I/O;
 		 */
 		mutex_enter(&bufcache_lock);
-		obp->b_cflags &= ~(BC_DONTFREE | BC_BUSY | BC_WANTED);
+		obp->b_cflags &= ~(BC_BUSY | BC_WANTED);
 		obp->b_flags &= ~(B_PHYS | B_RAW);
 		obp->b_iodone = NULL;
 		cv_broadcast(&obp->b_busy);
 		mutex_exit(&bufcache_lock);
+	}
 	uvm_lwp_rele(l);
 	DPRINTF(("%s: done: off=%" PRIu64 ", resid=%zu\n",
 	    __func__, uio->uio_offset, uio->uio_resid));
 	return error;
+}