 @@ -1,14 +1,14 @@
-/*	$NetBSD: xhci.c,v 1.139 2021/05/23 11:49:45 riastradh Exp $	*/
+/*	$NetBSD: xhci.c,v 1.140 2021/05/23 21:12:28 riastradh Exp $	*/
 /*
  * Copyright (c) 2013 Jonathan A. Kollasch
  * All rights reserved.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
 @@ -24,27 +24,27 @@
  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 /*
  * USB rev 2.0 and rev 3.1 specification
  *  http://www.usb.org/developers/docs/
  * xHCI rev 1.1 specification
  *  http://www.intel.com/technology/usb/spec.htm
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: xhci.c,v 1.139 2021/05/23 11:49:45 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: xhci.c,v 1.140 2021/05/23 21:12:28 riastradh Exp $");
 #ifdef _KERNEL_OPT
 #include "opt_usb.h"
 #endif
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/kmem.h>
 #include <sys/device.h>
 #include <sys/select.h>
 #include <sys/proc.h>
 #include <sys/queue.h>
 @@ -147,26 +147,28 @@ static void xhci_poll(struct usbd_bus *)
 static struct usbd_xfer *xhci_allocx(struct usbd_bus *, unsigned int);
 static void xhci_freex(struct usbd_bus *, struct usbd_xfer *);
 static void xhci_abortx(struct usbd_xfer *);
 static bool xhci_dying(struct usbd_bus *);
 static void xhci_get_lock(struct usbd_bus *, kmutex_t **);
 static usbd_status xhci_new_device(device_t, struct usbd_bus *, int, int, int,
     struct usbd_port *);
 static int xhci_roothub_ctrl(struct usbd_bus *, usb_device_request_t *,
     void *, int);
 static usbd_status xhci_configure_endpoint(struct usbd_pipe *);
 //static usbd_status xhci_unconfigure_endpoint(struct usbd_pipe *);
 static usbd_status xhci_reset_endpoint(struct usbd_pipe *);
 static usbd_status xhci_stop_endpoint_cmd(struct xhci_softc *,
     struct xhci_slot *, u_int, uint32_t);
 static usbd_status xhci_stop_endpoint(struct usbd_pipe *);
 static void xhci_host_dequeue(struct xhci_ring * const);
 static usbd_status xhci_set_dequeue(struct usbd_pipe *);
 static usbd_status xhci_do_command(struct xhci_softc * const,
     struct xhci_soft_trb * const, int);
 static usbd_status xhci_do_command_locked(struct xhci_softc * const,
     struct xhci_soft_trb * const, int);
 static usbd_status xhci_init_slot(struct usbd_device *, uint32_t);
 static void xhci_free_slot(struct xhci_softc *, struct xhci_slot *);
 static usbd_status xhci_set_address(struct usbd_device *, uint32_t, bool);
 static usbd_status xhci_enable_slot(struct xhci_softc * const,
 @@ -689,34 +691,90 @@ xhci_activate(device_t self, enum devact
 	switch (act) {
 	case DVACT_DEACTIVATE:
 		sc->sc_dying = true;
 		return 0;
 	default:
 		return EOPNOTSUPP;
+	}
+}
 bool
 xhci_suspend(device_t self, const pmf_qual_t *qual)
+{
 	struct xhci_softc * const sc = device_private(self);
-	size_t i, j, bn;
+	size_t i, j, bn, dci;
 	int port;
 	uint32_t v;
 	usbd_status err;
 	bool ok = false;
 	XHCIHIST_FUNC(); XHCIHIST_CALLED();
 	mutex_enter(&sc->sc_lock);
 	/*
 	 * Block issuance of new commands, and wait for all pending
 	 * commands to complete.
 	 */
 	KASSERT(sc->sc_suspender == NULL);
 	sc->sc_suspender = curlwp;
 	while (sc->sc_command_addr != 0)
 		cv_wait(&sc->sc_cmdbusy_cv, &sc->sc_lock);
 	/*
 	 * xHCI Requirements Specification 1.2, May 2019, Sec. 4.23.2:
 	 * xHCI Power Management, p. 342
 	 * https://www.intel.com/content/dam/www/public/us/en/documents/technical-specifications/extensible-host-controler-interface-usb-xhci.pdf#page=342
 	 */
 	/*
 	 * `1. Stop all USB activity by issuing Stop Endpoint Commands
 	 *     for Busy endpoints in the Running state.  If the Force
 	 *     Save Context Capability (FSC = ``0'') is not supported,
 	 *     then Stop Endpoint Commands shall be issued for all idle
 	 *     endpoints in the Running state as well.  The Stop
 	 *     Endpoint Command causes the xHC to update the respective
 	 *     Endpoint or Stream Contexts in system memory, e.g. the
 	 *     TR Dequeue Pointer, DCS, etc. fields.  Refer to
 	 *     Implementation Note "0".'
 	 */
 	for (i = 0; i < sc->sc_maxslots; i++) {
 		struct xhci_slot *xs = &sc->sc_slots[i];
 		/* Skip if the slot is not in use.  */
 		if (xs->xs_idx == 0)
 			continue;
 		for (dci = XHCI_DCI_SLOT; dci <= XHCI_MAX_DCI; dci++) {
 			/* Skip if the endpoint is not Running.  */
 			/* XXX What about Busy?  */
 			if (xhci_get_epstate(sc, xs, dci) !=
 			    XHCI_EPSTATE_RUNNING)
 				continue;
 			/* Stop endpoint.  */
 			err = xhci_stop_endpoint_cmd(sc, xs, dci,
 			    XHCI_TRB_3_SUSP_EP_BIT);
 			if (err) {
 				device_printf(self, "failed to stop endpoint"
 				    " slot %zu dci %zu err %d\n",
 				    i, dci, err);
 				goto out;
+			}
+		}
+	}
 	/*
-	 * First, suspend all the ports:
+	 * Next, suspend all the ports:
+	 *
 	 * xHCI Requirements Specification 1.2, May 2019, Sec. 4.15:
 	 * Suspend-Resume, pp. 276-283
 	 * https://www.intel.com/content/dam/www/public/us/en/documents/technical-specifications/extensible-host-controler-interface-usb-xhci.pdf#page=276
 	 */
 	for (bn = 0; bn < 2; bn++) {
 		for (i = 1; i <= sc->sc_rhportcount[bn]; i++) {
 			/* 4.15.1: Port Suspend.  */
 			port = XHCI_PORTSC(xhci_rhport2ctlrport(sc, bn, i));
 			/*
 			 * `System software places individual ports
 			 *  into suspend mode by writing a ``3'' into
 @@ -757,53 +815,32 @@ xhci_suspend(device_t self, const pmf_qu
 			 * reduce polling on host controllers that
 			 * support the U3C capability.
 			 */
 			for (j = 0; j < XHCI_WAIT_PLS_U3; j++) {
 				v = xhci_op_read_4(sc, port);
 				if (XHCI_PS_PLS_GET(v) == XHCI_PS_PLS_U3)
 					break;
 				usb_delay_ms(&sc->sc_bus, 1);
+			}
 			if (j == XHCI_WAIT_PLS_U3) {
 				device_printf(self,
 				    "suspend timeout on bus %zu port %zu\n",
 				    bn, i);
-				return false;
+				goto out;
+			}
+		}
+	}
 	/*
 	 * xHCI Requirements Specification 1.2, May 2019, Sec. 4.23.2:
 	 * xHCI Power Management, p. 342
 	 * https://www.intel.com/content/dam/www/public/us/en/documents/technical-specifications/extensible-host-controler-interface-usb-xhci.pdf#page=342
 	 */
 	/*
 	 * `1. Stop all USB activity by issuing Stop Endpoint Commands
 	 *     for Busy endpoints in the Running state.  If the Force
 	 *     Save Context Capability (FSC = ``0'') is not supported,
 	 *     then Stop Endpoint Commands shall be issued for all Idle
 	 *     endpoints in the Running state as well.  The Stop
 	 *     Endpoint Command causes the xHC to update the respective
 	 *     Endpoint or Stream Contexts in system memory, e.g. the
 	 *     TR Dequeue Pointer, DCS, etc. fields.  Refer to
 	 *     Implementation Note "0".'
 	 * XXX Not entirely sure if this is necessary for us; also it
 	 * probably has to happen before suspending the ports.
 	 */
 	/*
 	 * `2. Ensure that the Command Ring is in the Stopped state
 	 *     (CRR = ``0'') or Idle (i.e. the Command Transfer Ring is
 	 *     empty), and all Command Completion Events associated
 	 *     with them have been received.'
+	 *
 	 * XXX
 	 */
 	/* `3. Stop the controller by setting Run/Stop (R/S) = ``0''.'  */
 	xhci_op_write_4(sc, XHCI_USBCMD,
 	    xhci_op_read_4(sc, XHCI_USBCMD) & ~XHCI_CMD_RS);
 	/*
 @@ -846,42 +883,50 @@ xhci_suspend(device_t self, const pmf_qu
 		 * Just optimistically go on and check SRE anyway --
 		 * what's the worst that could happen?
 		 */
+	}
 	/*
 	 * `Note: After a Save or Restore operation completes, the
 	 *  Save/Restore Error (SRE) flag in the USBSTS register should
 	 *  be checked to ensure that the operation completed
 	 *  successfully.'
 	 */
 	if (xhci_op_read_4(sc, XHCI_USBSTS) & XHCI_STS_SRE) {
 		device_printf(self, "suspend error, USBSTS.SRE\n");
-		return false;
+		goto out;
+	}
-	return true;
+	/* Success!  */
 	ok = true;
 out:	mutex_exit(&sc->sc_lock);
 	return ok;
+}
 bool
 xhci_resume(device_t self, const pmf_qual_t *qual)
+{
 	struct xhci_softc * const sc = device_private(self);
 	size_t i, j, bn, dci;
 	int port;
 	uint32_t v;
 	bool ok = false;
 	XHCIHIST_FUNC(); XHCIHIST_CALLED();
 	mutex_enter(&sc->sc_lock);
 	KASSERT(sc->sc_suspender);
 	/*
 	 * xHCI Requirements Specification 1.2, May 2019, Sec. 4.23.2:
 	 * xHCI Power Management, p. 343
 	 * https://www.intel.com/content/dam/www/public/us/en/documents/technical-specifications/extensible-host-controler-interface-usb-xhci.pdf#page=343
 	 */
 	/*
 	 * `4. Restore the Operational Runtime, and VTIO registers with
 	 *     their previously saved state in the following order:
 	 *     DNCTRL, DCBAAP, CONFIG, ERSTSZ, ERSTBA, ERDP, IMAN,
 	 *     IMOD, and VTIO.'
+	 *
 	 * (We don't use VTIO here (for now?).)
 @@ -906,27 +951,27 @@ xhci_resume(device_t self, const pmf_qua
 	    xhci_op_read_4(sc, XHCI_USBCMD) | XHCI_CMD_CRS);
 	/*
 	 *    `...and wait for the Restore State Status (RSS) in the
 	 *     USBSTS register (5.4.2) to transition to ``0''.'
 	 */
 	for (i = 0; i < XHCI_WAIT_RSS; i++) {
 		if ((xhci_op_read_4(sc, XHCI_USBSTS) & XHCI_STS_RSS) == 0)
 			break;
 		usb_delay_ms(&sc->sc_bus, 1);
+	}
 	if (i >= XHCI_WAIT_RSS) {
 		device_printf(self, "suspend timeout, USBSTS.RSS\n");
-		return false;
+		goto out;
+	}
 	/*
 	 * `6. Reinitialize the Command Ring, i.e. so its Cycle bits
 	 *     are consistent with the RCS values to be written to the
 	 *     CRCR.'
+	 *
 	 * XXX Hope just zeroing it is good enough!
 	 */
 	xhci_host_dequeue(sc->sc_cr);
 	/*
 	 * `7. Write the CRCR with the address and RCS value of the
 @@ -998,27 +1043,27 @@ xhci_resume(device_t self, const pmf_qua
 			v |= XHCI_PS_LWS | XHCI_PS_PLS_SET(XHCI_PS_PLS_SETU0);
 			xhci_op_write_4(sc, port, v);
 			for (j = 0; j < XHCI_WAIT_PLS_U0; j++) {
 				v = xhci_op_read_4(sc, port);
 				if (XHCI_PS_PLS_GET(v) == XHCI_PS_PLS_U0)
 					break;
 				usb_delay_ms(&sc->sc_bus, 1);
+			}
 			if (j == XHCI_WAIT_PLS_U0) {
 				device_printf(self,
 				    "resume timeout on bus %zu port %zu\n",
 				    bn, i);
-				return false;
+				goto out;
+			}
+		}
+	}
 	/*
 	 * `10. Restart each of the previously Running endpoints by
 	 *      ringing their doorbells.'
 	 */
 	for (i = 0; i < sc->sc_maxslots; i++) {
 		struct xhci_slot *xs = &sc->sc_slots[i];
 		/* Skip if the slot is not in use.  */
 		if (xs->xs_idx == 0)
 @@ -1033,30 +1078,38 @@ xhci_resume(device_t self, const pmf_qua
 			/* Ring the doorbell.  */
 			xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci);
+		}
+	}
 	/*
 	 * `Note: After a Save or Restore operation completes, the
 	 *  Save/Restore Error (SRE) flag in the USBSTS register should
 	 *  be checked to ensure that the operation completed
 	 *  successfully.'
 	 */
 	if (xhci_op_read_4(sc, XHCI_USBSTS) & XHCI_STS_SRE) {
 		device_printf(self, "resume error, USBSTS.SRE\n");
-		return false;
+		goto out;
+	}
-	return true;
+	/* Resume command issuance.  */
 	sc->sc_suspender = NULL;
 	cv_broadcast(&sc->sc_cmdbusy_cv);
 	/* Success!  */
 	ok = true;
 out:	mutex_exit(&sc->sc_lock);
 	return ok;
+}
 bool
 xhci_shutdown(device_t self, int flags)
+{
 	return false;
+}
 static int
 xhci_hc_reset(struct xhci_softc * const sc)
+{
 	uint32_t usbcmd, usbsts;
 	int i;
 @@ -1853,50 +1906,64 @@ xhci_reset_endpoint(struct usbd_pipe *pi
 	mutex_enter(&sc->sc_lock);
 	usbd_status ret = xhci_reset_endpoint_locked(pipe);
 	mutex_exit(&sc->sc_lock);
 	return ret;
+}
 /*
  * 4.6.9, 6.4.3.8
  * Stop execution of TDs on xfer ring.
  * Should be called with sc_lock held.
  */
 static usbd_status
-xhci_stop_endpoint(struct usbd_pipe *pipe)
+xhci_stop_endpoint_cmd(struct xhci_softc *sc, struct xhci_slot *xs, u_int dci,
     uint32_t trb3flags)
+{
 	struct xhci_softc * const sc = XHCI_PIPE2SC(pipe);
 	struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv;
 	struct xhci_soft_trb trb;
 	usbd_status err;
 	const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc);
 	XHCIHIST_FUNC();
 	XHCIHIST_CALLARGS("slot %ju dci %ju", xs->xs_idx, dci, 0, 0);
 	KASSERT(mutex_owned(&sc->sc_lock));
 	trb.trb_0 = 0;
 	trb.trb_2 = 0;
 	trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) |
 	    XHCI_TRB_3_EP_SET(dci) |
-	    XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STOP_EP);
+	    XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STOP_EP) |
 	    trb3flags;
 	err = xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT);
 	return err;
+}
 static usbd_status
 xhci_stop_endpoint(struct usbd_pipe *pipe)
+{
 	struct xhci_softc * const sc = XHCI_PIPE2SC(pipe);
 	struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv;
 	const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc);
 	XHCIHIST_FUNC();
 	XHCIHIST_CALLARGS("slot %ju dci %ju", xs->xs_idx, dci, 0, 0);
 	KASSERT(mutex_owned(&sc->sc_lock));
 	return xhci_stop_endpoint_cmd(sc, xs, dci, 0);
+}
 /*
  * Set TR Dequeue Pointer.
  * xHCI 1.1  4.6.10  6.4.3.9
  * Purge all of the TRBs on ring and reinitialize ring.
  * Set TR dequeue Pointr to 0 and Cycle State to 1.
  * EPSTATE of endpoint must be ERROR or STOPPED, otherwise CONTEXT_STATE
  * error will be generated.
  */
 static usbd_status
 xhci_set_dequeue_locked(struct usbd_pipe *pipe)
+{
 	struct xhci_softc * const sc = XHCI_PIPE2SC(pipe);
 	struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv;
 @@ -3117,27 +3184,29 @@ static usbd_status
 xhci_do_command_locked(struct xhci_softc * const sc,
     struct xhci_soft_trb * const trb, int timeout)
+{
 	struct xhci_ring * const cr = sc->sc_cr;
 	usbd_status err;
 	XHCIHIST_FUNC();
 	XHCIHIST_CALLARGS("input: 0x%016jx 0x%08jx 0x%08jx",
 	    trb->trb_0, trb->trb_2, trb->trb_3, 0);
 	KASSERTMSG(!cpu_intr_p() && !cpu_softintr_p(), "called from intr ctx");
 	KASSERT(mutex_owned(&sc->sc_lock));
-	while (sc->sc_command_addr != 0)
+	while (sc->sc_command_addr != 0 &&
 	    sc->sc_suspender != NULL &&
 	    sc->sc_suspender != curlwp)
 		cv_wait(&sc->sc_cmdbusy_cv, &sc->sc_lock);
 	/*
 	 * If enqueue pointer points at last of ring, it's Link TRB,
 	 * command TRB will be stored in 0th TRB.
 	 */
 	if (cr->xr_ep == cr->xr_ntrb - 1)
 		sc->sc_command_addr = xhci_ring_trbp(cr, 0);
 	else
 		sc->sc_command_addr = xhci_ring_trbp(cr, cr->xr_ep);
 	sc->sc_resultpending = true;

 @@ -1,14 +1,14 @@
-/*	$NetBSD: xhcivar.h,v 1.18 2021/05/23 11:49:45 riastradh Exp $	*/
+/*	$NetBSD: xhcivar.h,v 1.19 2021/05/23 21:12:28 riastradh Exp $	*/
 /*
  * Copyright (c) 2013 Jonathan A. Kollasch
  * All rights reserved.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
 @@ -118,26 +118,27 @@ struct xhci_softc {
 	usb_dma_t sc_eventst_dma;
 	usb_dma_t sc_dcbaa_dma;
 	usb_dma_t sc_spbufarray_dma;
 	usb_dma_t *sc_spbuf_dma;
 	kcondvar_t sc_cmdbusy_cv;
 	kcondvar_t sc_command_cv;
 	bus_addr_t sc_command_addr;
 	struct xhci_soft_trb sc_result_trb;
 	bool sc_resultpending;
 	bool sc_dying;
 	struct lwp *sc_suspender;
 	void (*sc_vendor_init)(struct xhci_softc *);
 	int (*sc_vendor_port_status)(struct xhci_softc *, uint32_t, int);
 	int sc_quirks;
 #define XHCI_QUIRK_INTEL	__BIT(0) /* Intel xhci chip */
 #define XHCI_DEFERRED_START	__BIT(1)
 	uint32_t sc_hcc;		/* copy of HCCPARAMS1 */
 	uint32_t sc_hcc2;		/* copy of HCCPARAMS2 */
 	struct xhci_registers {
 		uint32_t	usbcmd;
 		uint32_t	dnctrl;