 @@ -1,14 +1,14 @@
-/* $NetBSD: cpu.c,v 1.14 2018/11/28 09:16:19 ryo Exp $ */
+/* $NetBSD: cpu.c,v 1.15 2018/12/21 08:01:01 ryo Exp $ */
 /*
  * Copyright (c) 2017 Ryo Shimizu <ryo@nerv.org>
  * All rights reserved.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
 @@ -17,27 +17,27 @@
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(1, "$NetBSD: cpu.c,v 1.14 2018/11/28 09:16:19 ryo Exp $");
+__KERNEL_RCSID(1, "$NetBSD: cpu.c,v 1.15 2018/12/21 08:01:01 ryo Exp $");
 #include "locators.h"
 #include "opt_arm_debug.h"
 #include "opt_fdt.h"
 #include "opt_multiprocessor.h"
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/atomic.h>
 #include <sys/device.h>
 #include <sys/cpu.h>
 #include <sys/kmem.h>
 #include <sys/reboot.h>
 @@ -153,26 +153,27 @@ cpu_attach(device_t dv, cpuid_t id)
+	}
 	ci->ci_dev = dv;
 	dv->dv_private = ci;
 	cpu_identify(ci->ci_dev, ci);
 #ifdef MULTIPROCESSOR
 	if (unit != 0) {
 		mi_cpu_attach(ci);
 		return;
+	}
 #endif /* MULTIPROCESSOR */
 	set_cpufuncs();
 	fpu_attach(ci);
 	cpu_identify1(dv, ci);
 	aarch64_getcacheinfo();
 	aarch64_printcacheinfo(dv);
 	cpu_identify2(dv, ci);
 	cpu_setup_sysctl(dv, ci);
+}
 struct cpuidtab {
 	uint32_t cpu_partnum;
 	const char *cpu_name;
 @@ -512,26 +513,27 @@ cpu_boot_secondary_processors(void)
 	/* add available processors to kcpuset */
 	uint32_t mbox = arm_cpu_hatched;
 	kcpuset_export_u32(kcpuset_attached, &mbox, sizeof(mbox));
+}
 void
 cpu_hatch(struct cpu_info *ci)
+{
 	KASSERT(curcpu() == ci);
 	mutex_enter(&cpu_hatch_lock);
 	set_cpufuncs();
 	fpu_attach(ci);
 	cpu_identify1(ci->ci_dev, ci);
 	aarch64_getcacheinfo();
 	aarch64_printcacheinfo(ci->ci_dev);
 	cpu_identify2(ci->ci_dev, ci);
 	mutex_exit(&cpu_hatch_lock);
 	intr_cpu_init(ci);
 #ifdef FDT
 	arm_fdt_cpu_hatch(ci);

 @@ -1,14 +1,14 @@
-/*	$NetBSD: cpufunc.c,v 1.4 2018/08/29 06:16:40 ryo Exp $	*/
+/*	$NetBSD: cpufunc.c,v 1.5 2018/12/21 08:01:01 ryo Exp $	*/
 /*
  * Copyright (c) 2017 Ryo Shimizu <ryo@nerv.org>
  * All rights reserved.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
 @@ -17,27 +17,27 @@
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpufunc.c,v 1.4 2018/08/29 06:16:40 ryo Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpufunc.c,v 1.5 2018/12/21 08:01:01 ryo Exp $");
 #include <sys/param.h>
 #include <sys/types.h>
 #include <sys/kmem.h>
 #include <aarch64/cpu.h>
 #include <aarch64/cpufunc.h>
 u_int cputype;			/* compat arm */
 u_int arm_dcache_align;		/* compat arm */
 u_int arm_dcache_align_mask;	/* compat arm */
 u_int arm_dcache_maxline;
 @@ -394,13 +394,37 @@ aarch64_dcache_wb_all(void)
 	int level;
 	cinfo = curcpu()->ci_cacheinfo;
 	for (level = 0; level < MAX_CACHE_LEVEL; level++) {
 		if (cinfo[level].cacheable == CACHE_CACHEABLE_NONE)
 			break;
 		__asm __volatile ("dsb ish");
 		ln_dcache_wb_all(level, &cinfo[level].dcache);
+	}
 	__asm __volatile ("dsb ish");
+}
 int
 set_cpufuncs(void)
+{
 	struct cpu_info * const ci = curcpu();
 	const uint32_t midr __unused = reg_midr_el1_read();
 	/* install default functions */
 	ci->ci_cpufuncs.cf_set_ttbr0 = aarch64_set_ttbr0;
 	/* install core/cluster specific functions */
 #ifdef CPU_THUNDERX
 	/* Cavium erratum 27456 */
 	if ((midr == CPU_ID_THUNDERXP1d0) ||
 	    (midr == CPU_ID_THUNDERXP1d1) ||
 	    (midr == CPU_ID_THUNDERXP2d1) ||
 	    (midr == CPU_ID_THUNDERX81XXRX)) {
 		ci->ci_cpufuncs.cf_set_ttbr0 = aarch64_set_ttbr0_thunderx;
+	}
 #endif
 	return 0;
+}

 @@ -1,14 +1,14 @@
-/*	$NetBSD: cpufunc_asm_armv8.S,v 1.2 2018/07/23 22:51:39 ryo Exp $	*/
+/*	$NetBSD: cpufunc_asm_armv8.S,v 1.3 2018/12/21 08:01:01 ryo Exp $	*/
 /*-
  * Copyright (c) 2014 Robin Randhawa
  * Copyright (c) 2015 The FreeBSD Foundation
  * All rights reserved.
+ *
  * Portions of this software were developed by Andrew Turner
  * under sponsorship from the FreeBSD Foundation
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
 @@ -22,26 +22,27 @@
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
+ *
  * $FreeBSD: head/sys/arm64/arm64/cpufunc_asm.S 313347 2017-02-06 17:50:09Z andrew $
  */
 #include "opt_cputypes.h"
 #include "opt_multiprocessor.h"
 #include <aarch64/asm.h>
 	.text
 	.align	2
 /*
  * Macro to handle the cache. This takes the start address in x0, length
  * in x1. It will corrupt x0, x1, x2, and x3.
  */
 .macro cache_handle_range dcop = 0, ic = 0, icop = 0
 .if \ic == 0
 	mrs	x3, ctr_el0
 @@ -153,26 +154,42 @@ END(aarch64_drain_writebuf)
 /*
  * TLB ops
  */
 /* void aarch64_set_ttbr0(uint64_t ttbr0) */
 ENTRY(aarch64_set_ttbr0)
 	dsb	ish
 	msr	ttbr0_el1, x0
 	dsb	ish
 	isb
 	ret
 END(aarch64_set_ttbr0)
 #ifdef CPU_THUNDERX
 /*
  * Cavium erratum 27456
  * void aarch64_set_ttbr0_thunderx(uint64_t ttbr0)
  */
 ENTRY(aarch64_set_ttbr0_thunderx)
 	dsb	ish
 	msr	ttbr0_el1, x0
 	isb
 	ic	iallu
 	dsb	nsh
 	isb
 	ret
 END(aarch64_set_ttbr0_thunderx)
 #endif /* CPU_THUNDERX */
 /* void aarch64_tlbi_all(void) */
 ENTRY(aarch64_tlbi_all)
 	dsb	ishst
 #ifdef MULTIPROCESSOR
 	tlbi	vmalle1is
 #else
 	tlbi	vmalle1
 #endif
 	dsb	ish
 	isb
 	ret
 END(aarch64_tlbi_all)

 @@ -1,14 +1,14 @@
-/*	$NetBSD: pmap.c,v 1.33 2018/11/01 20:34:49 maxv Exp $	*/
+/*	$NetBSD: pmap.c,v 1.34 2018/12/21 08:01:01 ryo Exp $	*/
 /*
  * Copyright (c) 2017 Ryo Shimizu <ryo@nerv.org>
  * All rights reserved.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
 @@ -17,27 +17,27 @@
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.33 2018/11/01 20:34:49 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.34 2018/12/21 08:01:01 ryo Exp $");
 #include "opt_arm_debug.h"
 #include "opt_ddb.h"
 #include "opt_kasan.h"
 #include "opt_multiprocessor.h"
 #include "opt_pmap.h"
 #include "opt_uvmhist.h"
 #include <sys/param.h>
 #include <sys/types.h>
 #include <sys/kmem.h>
 #include <sys/vmem.h>
 #include <sys/atomic.h>
 @@ -1196,27 +1196,27 @@ pmap_activate(struct lwp *l)
 	if (l != curlwp)
 		return;
 	KASSERT(pm->pm_l0table != NULL);
 	UVMHIST_LOG(pmaphist, "lwp=%p (pid=%d)", l, l->l_proc->p_pid, 0, 0);
 	/* XXX */
 	CTASSERT(PID_MAX <= 65535);	/* 16bit ASID */
 	if (pm->pm_asid == -1)
 		pm->pm_asid = l->l_proc->p_pid;
 	ttbr0 = ((uint64_t)pm->pm_asid << 48) | pm->pm_l0table_pa;
-	aarch64_set_ttbr0(ttbr0);
+	cpu_set_ttbr0(ttbr0);
 	pm->pm_activated = true;
 	PMAP_COUNT(activate);
+}
 void
 pmap_deactivate(struct lwp *l)
+{
 	struct pmap *pm = l->l_proc->p_vmspace->vm_map.pmap;
 	UVMHIST_FUNC(__func__);
 	UVMHIST_CALLED(pmaphist);

 @@ -1,14 +1,14 @@
-/* $NetBSD: cpu.h,v 1.12 2018/11/24 22:49:35 skrll Exp $ */
+/* $NetBSD: cpu.h,v 1.13 2018/12/21 08:01:01 ryo Exp $ */
 /*-
  * Copyright (c) 2014 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Matt Thomas of 3am Software Foundry.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -54,26 +54,30 @@ struct clockframe {
 #define CLKF_USERMODE(cf)	((((cf)->cf_tf.tf_spsr) & 0x0f) == 0)
 #define CLKF_PC(cf)		((cf)->cf_tf.tf_pc)
 #define CLKF_INTR(cf)		((void)(cf), curcpu()->ci_intr_depth > 1)
 /*
  * LWP_PC: Find out the program counter for the given lwp.
  */
 #define LWP_PC(l)		((l)->l_md.md_utf->tf_pc)
 #include <sys/cpu_data.h>
 #include <sys/device_if.h>
 #include <sys/intr.h>
 struct aarch64_cpufuncs {
 	void (*cf_set_ttbr0)(uint64_t);
 };
 struct cpu_info {
 	struct cpu_data ci_data;
 	device_t ci_dev;
 	cpuid_t ci_cpuid;
 	struct lwp *ci_curlwp;
 	struct lwp *ci_softlwps[SOFTINT_COUNT];
 	uint64_t ci_lastintr;
 	int ci_mtx_oldspl;
 	int ci_mtx_count;
 	int ci_want_resched;
 @@ -88,26 +92,27 @@ struct cpu_info {
 	struct evcnt ci_vfp_save;
 	struct evcnt ci_vfp_release;
 	/* interrupt controller */
 	u_int ci_gic_redist;	/* GICv3 redistributor index */
 	uint64_t ci_gic_sgir;	/* GICv3 SGIR target */
 	/* ACPI */
 	uint64_t ci_acpiid;	/* ACPI Processor Unique ID */
 	struct aarch64_sysctl_cpu_id ci_id;
 	struct aarch64_cache_info *ci_cacheinfo;
 	struct aarch64_cpufuncs ci_cpufuncs;
 } __aligned(COHERENCY_UNIT);
 static inline struct cpu_info *
 curcpu(void)
+{
 	struct cpu_info *ci;
 	__asm __volatile ("mrs %0, tpidr_el1" : "=r"(ci));
 	return ci;
+}
 #define curlwp			(curcpu()->ci_curlwp)
 #define setsoftast(ci)		atomic_or_uint(&(ci)->ci_astpending, __BIT(0))

 @@ -1,14 +1,14 @@
-/*	$NetBSD: cpufunc.h,v 1.4 2018/12/15 16:54:30 alnsn Exp $	*/
+/*	$NetBSD: cpufunc.h,v 1.5 2018/12/21 08:01:01 ryo Exp $	*/
 /*
  * Copyright (c) 2017 Ryo Shimizu <ryo@nerv.org>
  * All rights reserved.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
 @@ -24,32 +24,26 @@
  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 #ifndef _AARCH64_CPUFUNC_H_
 #define _AARCH64_CPUFUNC_H_
 #ifdef _KERNEL
 #include <arm/armreg.h>
 #include <sys/device_if.h>
 static inline int
 set_cpufuncs(void)
 	return 0;
 struct aarch64_cache_unit {
 	u_int cache_type;
 #define CACHE_TYPE_UNKNOWN	0
 #define CACHE_TYPE_VIVT		1	/* ASID-tagged VIVT */
 #define CACHE_TYPE_VIPT		2
 #define CACHE_TYPE_PIPT		3
 	u_int cache_line_size;
 	u_int cache_ways;
 	u_int cache_sets;
 	u_int cache_way_size;
 	u_int cache_size;
 	u_int cache_purging;
 #define CACHE_PURGING_WB	0x01
 @@ -65,47 +59,50 @@ struct aarch64_cache_info {
 #define CACHE_CACHEABLE_DCACHE	2	/* data cache only */
 #define CACHE_CACHEABLE_IDCACHE	3	/* instruction and data caches */
 #define CACHE_CACHEABLE_UNIFIED	4	/* unified cache */
 	struct aarch64_cache_unit icache;
 	struct aarch64_cache_unit dcache;
 };
 #define MAX_CACHE_LEVEL	8		/* ARMv8 has maximum 8 level cache */
 extern struct aarch64_cache_info aarch64_cache_info[MAX_CACHE_LEVEL];
 extern u_int aarch64_cache_vindexsize;	/* cachesize/way (VIVT/VIPT) */
 extern u_int aarch64_cache_prefer_mask;
 extern u_int cputype;			/* compat arm */
 int set_cpufuncs(void);
 void aarch64_getcacheinfo(void);
 void aarch64_printcacheinfo(device_t);
 void aarch64_dcache_wbinv_all(void);
 void aarch64_dcache_inv_all(void);
 void aarch64_dcache_wb_all(void);
 void aarch64_icache_inv_all(void);
 /* cache op in cpufunc_asm_armv8.S */
 void aarch64_nullop(void);
 uint32_t aarch64_cpuid(void);
 void aarch64_icache_sync_range(vaddr_t, vsize_t);
 void aarch64_idcache_wbinv_range(vaddr_t, vsize_t);
 void aarch64_dcache_wbinv_range(vaddr_t, vsize_t);
 void aarch64_dcache_inv_range(vaddr_t, vsize_t);
 void aarch64_dcache_wb_range(vaddr_t, vsize_t);
 void aarch64_icache_inv_all(void);
 void aarch64_drain_writebuf(void);
 /* tlb op in cpufunc_asm_armv8.S */
 #define cpu_set_ttbr0(t)		curcpu()->ci_cpufuncs.cf_set_ttbr0((t))
 void aarch64_set_ttbr0(uint64_t);
 void aarch64_set_ttbr0_thunderx(uint64_t);
 void aarch64_tlbi_all(void);			/* all ASID, all VA */
 void aarch64_tlbi_by_asid(int);			/*  an ASID, all VA */
 void aarch64_tlbi_by_va(vaddr_t);		/* all ASID, a VA */
 void aarch64_tlbi_by_va_ll(vaddr_t);		/* all ASID, a VA, lastlevel */
 void aarch64_tlbi_by_asid_va(int, vaddr_t);	/*  an ASID, a VA */
 void aarch64_tlbi_by_asid_va_ll(int, vaddr_t);	/*  an ASID, a VA, lastlevel */
 /* misc */
 #define cpu_idnum()			aarch64_cpuid()
 /* cache op */