 @@ -1,14 +1,14 @@
-/*	$NetBSD: ccd.c,v 1.189 2022/03/28 12:48:35 riastradh Exp $	*/
+/*	$NetBSD: ccd.c,v 1.189.4.1 2024/04/18 18:24:31 martin Exp $	*/
 /*-
  * Copyright (c) 1996, 1997, 1998, 1999, 2007, 2009 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Jason R. Thorpe, and by Andrew Doran.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -78,27 +78,27 @@
  *    sc_stop, sc_bufq and b_resid from master buffers.
+ *
  * => a combination of CCDF_INITED, sc_inflight, and sc_iolock is used to
  *    serialize I/O and configuration changes.
+ *
  * => the in-core disk label does not change while the device is open.
+ *
  * On memory consumption: ccd fans out I/O requests and so needs to
  * allocate memory.  If the system is desperately low on memory, we
  * single thread I/O.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ccd.c,v 1.189 2022/03/28 12:48:35 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ccd.c,v 1.189.4.1 2024/04/18 18:24:31 martin Exp $");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/proc.h>
 #include <sys/errno.h>
 #include <sys/buf.h>
 #include <sys/kmem.h>
 #include <sys/pool.h>
 #include <sys/module.h>
 #include <sys/namei.h>
 #include <sys/stat.h>
 #include <sys/ioctl.h>
 @@ -142,47 +142,47 @@ int ccddebug = 0x00;
 #define	ccdunit(x)	DISKUNIT(x)
 struct ccdbuf {
 	struct buf	cb_buf;		/* new I/O buf */
 	struct buf	*cb_obp;	/* ptr. to original I/O buf */
 	struct ccd_softc *cb_sc;	/* pointer to ccd softc */
 	int		cb_comp;	/* target component */
 	SIMPLEQ_ENTRY(ccdbuf) cb_q;	/* fifo of component buffers */
 };
 /* component buffer pool */
 static pool_cache_t ccd_cache;
-#define	CCD_GETBUF()		pool_cache_get(ccd_cache, PR_WAITOK)
+#define	CCD_GETBUF(wait)	pool_cache_get(ccd_cache, wait)
 #define	CCD_PUTBUF(cbp)		pool_cache_put(ccd_cache, cbp)
 #define CCDLABELDEV(dev)	\
 	(MAKEDISKDEV(major((dev)), ccdunit((dev)), RAW_PART))
 /* called by main() at boot time */
 void	ccddetach(void);
 /* called by biodone() at interrupt time */
 static void	ccdiodone(struct buf *);
 static void	ccdinterleave(struct ccd_softc *);
 static int	ccdinit(struct ccd_softc *, char **, struct vnode **,
 		    struct lwp *);
 static struct ccdbuf *ccdbuffer(struct ccd_softc *, struct buf *,
-		    daddr_t, void *, long);
+		    daddr_t, void *, long, int);
 static void	ccdgetdefaultlabel(struct ccd_softc *, struct disklabel *);
 static void	ccdgetdisklabel(dev_t);
 static void	ccdmakedisklabel(struct ccd_softc *);
-static void	ccdstart(struct ccd_softc *);
+static int	ccdstart(struct ccd_softc *, struct buf *, int);
 static void	ccdthread(void *);
 static dev_type_open(ccdopen);
 static dev_type_close(ccdclose);
 static dev_type_read(ccdread);
 static dev_type_write(ccdwrite);
 static dev_type_ioctl(ccdioctl);
 static dev_type_strategy(ccdstrategy);
 static dev_type_size(ccdsize);
 const struct bdevsw ccd_bdevsw = {
 	.d_open = ccdopen,
 	.d_close = ccdclose,
 @@ -692,64 +692,54 @@ ccdclose(dev_t dev, int flags, int fmt,
+	}
 	cs->sc_dkdev.dk_openmask =
 	    cs->sc_dkdev.dk_copenmask | cs->sc_dkdev.dk_bopenmask;
 	if (cs->sc_dkdev.dk_openmask == 0) {
 		if ((cs->sc_flags & CCDF_KLABEL) == 0)
 			cs->sc_flags &= ~CCDF_VLABEL;
+	}
 	mutex_exit(&cs->sc_dvlock);
 	return (0);
+}
 static bool
 ccdbackoff(struct ccd_softc *cs)
 	/* XXX Arbitrary, should be a uvm call. */
 	return uvm_availmem(true) < (uvmexp.freemin >> 1) &&
 	    disk_isbusy(&cs->sc_dkdev);
 static void
 ccdthread(void *cookie)
+{
 	int error;
 	struct ccd_softc *cs;
 	struct buf *bp;
 	cs = cookie;
 #ifdef DEBUG
  	if (ccddebug & CCDB_FOLLOW)
  		printf("ccdthread: hello\n");
 #endif
 	mutex_enter(cs->sc_iolock);
 	while (__predict_true(!cs->sc_zap)) {
-		if (bufq_peek(cs->sc_bufq) == NULL) {
+		bp = bufq_get(cs->sc_bufq);
 		if (bp == NULL) {
 			/* Nothing to do. */
 			cv_wait(&cs->sc_push, cs->sc_iolock);
 			continue;
+		}
 		if (ccdbackoff(cs)) {
 			/* Wait for memory to become available. */
 			(void)cv_timedwait(&cs->sc_push, cs->sc_iolock, 1);
 			continue;
 #ifdef DEBUG
  		if (ccddebug & CCDB_FOLLOW)
  			printf("ccdthread: dispatching I/O\n");
 #endif
-		ccdstart(cs);
+		error = ccdstart(cs, bp, PR_WAITOK);
 		KASSERT(error == 0);
 		mutex_enter(cs->sc_iolock);
+	}
 	cs->sc_thread = NULL;
 	mutex_exit(cs->sc_iolock);
 #ifdef DEBUG
  	if (ccddebug & CCDB_FOLLOW)
  		printf("ccdthread: goodbye\n");
 #endif
 	kthread_exit(0);
+}
 static void
 ccdstrategy(struct buf *bp)
 @@ -767,55 +757,48 @@ ccdstrategy(struct buf *bp)
 	/* Synchronize with device init/uninit. */
 	if (__predict_false((cs->sc_flags & CCDF_INITED) == 0)) {
 		mutex_exit(cs->sc_iolock);
 #ifdef DEBUG
  		if (ccddebug & CCDB_FOLLOW)
  			printf("ccdstrategy: unit %d: not inited\n", unit);
 #endif
  		bp->b_error = ENXIO;
  		bp->b_resid = bp->b_bcount;
  		biodone(bp);
 		return;
+	}
-	/* Defer to thread if system is low on memory. */
+	if (ccdstart(cs, bp, PR_NOWAIT) != 0) {
-	bufq_put(cs->sc_bufq, bp);
+		/* Defer to thread if system is low on memory. */
-	if (__predict_false(ccdbackoff(cs))) {
+		bufq_put(cs->sc_bufq, bp);
 		cv_broadcast(&cs->sc_push);
 		mutex_exit(cs->sc_iolock);
 #ifdef DEBUG
  		if (ccddebug & CCDB_FOLLOW)
  			printf("ccdstrategy: holding off on I/O\n");
 #endif
 		return;
+	}
 	ccdstart(cs);
+}
-static void
+static int
-ccdstart(struct ccd_softc *cs)
+ccdstart(struct ccd_softc *cs, struct buf *bp, int wait)
+{
 	daddr_t blkno;
 	int wlabel;
 	struct disklabel *lp;
 	long bcount, rcount;
 	struct ccdbuf *cbp;
 	char *addr;
 	daddr_t bn;
 	vnode_t *vp;
-	buf_t *bp;
+	SIMPLEQ_HEAD(, ccdbuf) cbufq;
 	KASSERT(mutex_owned(cs->sc_iolock));
 	bp = bufq_get(cs->sc_bufq);
 	KASSERT(bp != NULL);
 	disk_busy(&cs->sc_dkdev);
 #ifdef DEBUG
 	if (ccddebug & CCDB_FOLLOW)
 		printf("ccdstart(%s, %p)\n", cs->sc_xname, bp);
 #endif
 	/* If it's a nil transfer, wake up the top half now. */
 	if (bp->b_bcount == 0)
 		goto done;
 @@ -826,60 +809,78 @@ ccdstart(struct ccd_softc *cs)
 	 * error, the bounds check will flag that for us.  Convert
 	 * the partition relative block number to an absolute.
 	 */
 	blkno = bp->b_blkno;
 	wlabel = cs->sc_flags & (CCDF_WLABEL|CCDF_LABELLING);
 	if (DISKPART(bp->b_dev) != RAW_PART) {
 		if (bounds_check_with_label(&cs->sc_dkdev, bp, wlabel) <= 0)
 			goto done;
 		blkno += lp->d_partitions[DISKPART(bp->b_dev)].p_offset;
+	}
 	mutex_exit(cs->sc_iolock);
 	bp->b_rawblkno = blkno;
-	/* Allocate the component buffers and start I/O! */
+	/* Allocate the component buffers. */
 	SIMPLEQ_INIT(&cbufq);
 	bp->b_resid = bp->b_bcount;
 	bn = bp->b_rawblkno;
 	addr = bp->b_data;
 	for (bcount = bp->b_bcount; bcount > 0; bcount -= rcount) {
-		cbp = ccdbuffer(cs, bp, bn, addr, bcount);
+		cbp = ccdbuffer(cs, bp, bn, addr, bcount, wait);
 		KASSERT(cbp != NULL || wait == PR_NOWAIT);
 		if (cbp == NULL) {
 			while ((cbp = SIMPLEQ_FIRST(&cbufq)) != NULL) {
 				SIMPLEQ_REMOVE_HEAD(&cbufq, cb_q);
 				CCD_PUTBUF(cbp);
+			}
 			mutex_enter(cs->sc_iolock);
 			disk_unbusy(&cs->sc_dkdev, 0, 0);
 			return ENOMEM;
+		}
 		SIMPLEQ_INSERT_TAIL(&cbufq, cbp, cb_q);
 		rcount = cbp->cb_buf.b_bcount;
 		bn += btodb(rcount);
 		addr += rcount;
+	}
 	/* All buffers set up, now fire off the requests. */
 	while ((cbp = SIMPLEQ_FIRST(&cbufq)) != NULL) {
 		SIMPLEQ_REMOVE_HEAD(&cbufq, cb_q);
 		vp = cbp->cb_buf.b_vp;
 		if ((cbp->cb_buf.b_flags & B_READ) == 0) {
 			mutex_enter(vp->v_interlock);
 			vp->v_numoutput++;
 			mutex_exit(vp->v_interlock);
+		}
 		(void)VOP_STRATEGY(vp, &cbp->cb_buf);
+	}
-	return;
+	return 0;
  done:
 	disk_unbusy(&cs->sc_dkdev, 0, 0);
 	cv_broadcast(&cs->sc_stop);
 	cv_broadcast(&cs->sc_push);
 	mutex_exit(cs->sc_iolock);
 	bp->b_resid = bp->b_bcount;
 	biodone(bp);
 	return 0;
+}
 /*
  * Build a component buffer header.
  */
 static struct ccdbuf *
 ccdbuffer(struct ccd_softc *cs, struct buf *bp, daddr_t bn, void *addr,
-    long bcount)
+    long bcount, int wait)
+{
 	struct ccdcinfo *ci;
 	struct ccdbuf *cbp;
 	daddr_t cbn, cboff;
 	u_int64_t cbc;
 	int ccdisk;
 #ifdef DEBUG
 	if (ccddebug & CCDB_IO)
 		printf("ccdbuffer(%p, %p, %" PRId64 ", %p, %ld)\n",
 		       cs, bp, bn, addr, bcount);
 #endif
 	/*
 @@ -919,28 +920,29 @@ ccdbuffer(struct ccd_softc *cs, struct b
 			ccdisk = ii->ii_index[0];
 			cbn = ii->ii_startoff + off;
 		} else {
 			ccdisk = ii->ii_index[off % ii->ii_ndisk];
 			cbn = ii->ii_startoff + off / ii->ii_ndisk;
+		}
 		cbn *= cs->sc_ileave;
 		ci = &cs->sc_cinfo[ccdisk];
+	}
 	/*
 	 * Fill in the component buf structure.
 	 */
-	cbp = CCD_GETBUF();
+	cbp = CCD_GETBUF(wait);
-	KASSERT(cbp != NULL);
+	if (cbp == NULL)
 		return NULL;
 	buf_init(&cbp->cb_buf);
 	cbp->cb_buf.b_flags = bp->b_flags;
 	cbp->cb_buf.b_oflags = bp->b_oflags;
 	cbp->cb_buf.b_cflags = bp->b_cflags;
 	cbp->cb_buf.b_iodone = ccdiodone;
 	cbp->cb_buf.b_proc = bp->b_proc;
 	cbp->cb_buf.b_dev = ci->ci_dev;
 	cbp->cb_buf.b_blkno = cbn + cboff;
 	cbp->cb_buf.b_data = addr;
 	cbp->cb_buf.b_vp = ci->ci_vp;
 	cbp->cb_buf.b_objlock = ci->ci_vp->v_interlock;
 	if (cs->sc_ileave == 0)
 		cbc = dbtob((u_int64_t)(ci->ci_size - cbn));