---
- branch: MAIN
  date: Sun Dec  1 20:56:40 UTC 2019
  files:
  - new: '1.32'
    old: '1.31'
    path: src/sys/kern/subr_xcall.c
    pathrev: src/sys/kern/subr_xcall.c@1.32
    type: modified
  id: 20191201T205640Z.be79e2427534fa40ef453e095b0140f74cf33c0e
  log: "Restore xcall(9) fast path using atomic_load/store_*.\n\nWhile here, fix a
    bug that was formerly in xcall(9): a missing\nacquire operation in the xc_wait
    fast path so that all memory\noperations in the xcall on remote CPUs will happen
    before any memory\noperations on the issuing CPU after xc_wait returns.\n\nAll
    stores of xc->xc_donep are done with atomic_store_release so that\nwe can safely
    use atomic_load_acquire to read it outside the lock.\nHowever, this fast path
    only works on platforms with cheap 64-bit\natomic load/store, so conditionalize
    it on __HAVE_ATOMIC64_LOADSTORE.\n(Under the lock, no need for atomic loads since
    nobody else will be\nissuing stores.)\n\nFor review, here's the relevant diff
    from the old version of the fast\npath, from before it was removed and some other
    things changed in the\nfile:\n\ndiff --git a/sys/kern/subr_xcall.c b/sys/kern/subr_xcall.c\nindex
    45a877aa90e0..b6bfb6455291 100644\n--- a/sys/kern/subr_xcall.c\n+++ b/sys/kern/subr_xcall.c\n@@
    -84,6 +84,7 @@ __KERNEL_RCSID(0, \"$NetBSD: subr_xcall.c,v 1.27 2019/10/06 15:11:17
    uwe Exp $\");\n #include <sys/evcnt.h>\n #include <sys/kthread.h>\n #include <sys/cpu.h>\n+#include
    <sys/atomic.h>\n\n #ifdef _RUMPKERNEL\n #include \"rump_private.h\"\n@@ -334,10
    +353,12 @@ xc_wait(uint64_t where)\n \t\txc = &xc_low_pri;\n \t}\n\n+#ifdef __HAVE_ATOMIC64_LOADSTORE\n
    \t/* Fast path, if already done. */\n-\tif (xc->xc_donep >= where) {\n+\tif (atomic_load_acquire(&xc->xc_donep)
    >= where) {\n \t\treturn;\n \t}\n+#endif\n\n \t/* Slow path: block until awoken.
    */\n \tmutex_enter(&xc->xc_lock);\n@@ -422,7 +443,11 @@ xc_thread(void *cookie)\n
    \t\t(*func)(arg1, arg2);\n\n \t\tmutex_enter(&xc->xc_lock);\n+#ifdef __HAVE_ATOMIC64_LOADSTORE\n+\t\tatomic_store_release(&xc->xc_donep,
    xc->xc_donep + 1);\n+#else\n \t\txc->xc_donep++;\n+#endif\n \t}\n \t/* NOTREACHED
    */\n }\n@@ -462,7 +487,6 @@ xc__highpri_intr(void *dummy)\n \t * Lock-less fetch
    of function and its arguments.\n \t * Safe since it cannot change at this point.\n
    \t */\n-\tKASSERT(xc->xc_donep < xc->xc_headp);\n \tfunc = xc->xc_func;\n \targ1
    = xc->xc_arg1;\n \targ2 = xc->xc_arg2;\n@@ -475,7 +499,13 @@ xc__highpri_intr(void
    *dummy)\n \t * cross-call has been processed - notify waiters, if any.\n \t */\n
    \tmutex_enter(&xc->xc_lock);\n-\tif (++xc->xc_donep == xc->xc_headp) {\n+\tKASSERT(xc->xc_donep
    < xc->xc_headp);\n+#ifdef __HAVE_ATOMIC64_LOADSTORE\n+\tatomic_store_release(&xc->xc_donep,
    xc->xc_donep + 1);\n+#else\n+\txc->xc_donep++;\n+#endif\n+\tif (xc->xc_donep ==
    xc->xc_headp) {\n \t\tcv_broadcast(&xc->xc_busy);\n \t}\n \tmutex_exit(&xc->xc_lock);\n"
  module: src
  subject: 'CVS commit: src/sys/kern'
  unixtime: '1575233800'
  user: riastradh