Fri Jan 1 12:38:50 2021 UTC ()
Pull up following revision(s) (requested by rin in ticket #1170):

	sys/arch/aarch64/aarch64/cpufunc.c: revision 1.22 (patch)
	sys/arch/aarch64/aarch64/cpufunc.c: revision 1.23 (patch)
	sys/arch/aarch64/aarch64/pmap.c: revision 1.81

Set uvmexp.ncolors appropriately, which is required for some CPU
models with VIPT icache.

Otherwise, alias in virtual address results in inconsistent results,
at least for applications that rewrite text of other process, e.g.,
GDB for arm32.

Also, this hopefully fixes other unexpected failures due to alias.
Confirmed that there's no observable regression in performance;
difference in ``time make -j8'' for GENERIC64 kernel on BCM2837
with and without setting uvmexp.ncolors is within 0.1%.

Thanks to ryo@ for discussion.

Fix uvmexp.ncolors for some big.LITTLE configuration; it is uncertain
which CPU is used as primary, and as a result, secondary CPUs can
require larger number of colors.

In order to solve this problem, update uvmexp.ncolors via
uvm_page_recolor(9) when secondary CPUs are attached, as done for
other ports like x86.

Pointed out by jmcneill@, and discussed on port-arm@:
http://mail-index.netbsd.org/port-arm/2020/07/03/msg006837.html
Tested and OK'd by ryo@.

Fix previous; add missing <uvm/uvm.h> include.


(martin)
diff -r1.5.4.1 -r1.5.4.2 src/sys/arch/aarch64/aarch64/cpufunc.c
diff -r1.41.2.6 -r1.41.2.7 src/sys/arch/aarch64/aarch64/pmap.c
cvs diff -r1.5.4.1 -r1.5.4.2 src/sys/arch/aarch64/aarch64/cpufunc.c (expand / switch to unified diff)

--- src/sys/arch/aarch64/aarch64/cpufunc.c 2019/09/22 12:27:22 1.5.4.1
+++ src/sys/arch/aarch64/aarch64/cpufunc.c 2021/01/01 12:38:49 1.5.4.2
@@ -1,14 +1,14 @@ @@ -1,14 +1,14 @@
1/* $NetBSD: cpufunc.c,v 1.5.4.1 2019/09/22 12:27:22 martin Exp $ */ 1/* $NetBSD: cpufunc.c,v 1.5.4.2 2021/01/01 12:38:49 martin Exp $ */
2 2
3/* 3/*
4 * Copyright (c) 2017 Ryo Shimizu <ryo@nerv.org> 4 * Copyright (c) 2017 Ryo Shimizu <ryo@nerv.org>
5 * All rights reserved. 5 * All rights reserved.
6 * 6 *
7 * Redistribution and use in source and binary forms, with or without 7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions 8 * modification, are permitted provided that the following conditions
9 * are met: 9 * are met:
10 * 1. Redistributions of source code must retain the above copyright 10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer. 11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright 12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the 13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution. 14 * documentation and/or other materials provided with the distribution.
@@ -19,32 +19,35 @@ @@ -19,32 +19,35 @@
19 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 19 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
20 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
24 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 24 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
25 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 * POSSIBILITY OF SUCH DAMAGE. 26 * POSSIBILITY OF SUCH DAMAGE.
27 */ 27 */
28 28
29#include "opt_multiprocessor.h" 29#include "opt_multiprocessor.h"
30 30
31#include <sys/cdefs.h> 31#include <sys/cdefs.h>
32__KERNEL_RCSID(0, "$NetBSD: cpufunc.c,v 1.5.4.1 2019/09/22 12:27:22 martin Exp $"); 32__KERNEL_RCSID(0, "$NetBSD: cpufunc.c,v 1.5.4.2 2021/01/01 12:38:49 martin Exp $");
33 33
34#include <sys/param.h> 34#include <sys/param.h>
35#include <sys/types.h> 35#include <sys/types.h>
36#include <sys/kmem.h> 36#include <sys/kmem.h>
37 37
 38#include <uvm/uvm.h>
 39#include <uvm/uvm_page.h>
 40
38#include <aarch64/cpu.h> 41#include <aarch64/cpu.h>
39#include <aarch64/cpufunc.h> 42#include <aarch64/cpufunc.h>
40 43
41u_int cputype; /* compat arm */ 44u_int cputype; /* compat arm */
42u_int arm_dcache_align; /* compat arm */ 45u_int arm_dcache_align; /* compat arm */
43u_int arm_dcache_align_mask; /* compat arm */ 46u_int arm_dcache_align_mask; /* compat arm */
44u_int arm_dcache_maxline; 47u_int arm_dcache_maxline;
45 48
46u_int aarch64_cache_vindexsize; 49u_int aarch64_cache_vindexsize;
47u_int aarch64_cache_prefer_mask; 50u_int aarch64_cache_prefer_mask;
48 51
49/* cache info per cluster. the same cluster has the same cache configuration? */ 52/* cache info per cluster. the same cluster has the same cache configuration? */
50#define MAXCPUPACKAGES MAXCPUS /* maximum of ci->ci_package_id */ 53#define MAXCPUPACKAGES MAXCPUS /* maximum of ci->ci_package_id */
@@ -81,26 +84,27 @@ extract_cacheunit(int level, bool insn,  @@ -81,26 +84,27 @@ extract_cacheunit(int level, bool insn,
81 cunit->cache_size = cunit->cache_way_size * cunit->cache_ways; 84 cunit->cache_size = cunit->cache_way_size * cunit->cache_ways;
82 85
83 /* cache purging */ 86 /* cache purging */
84 cunit->cache_purging = (ccsidr & CCSIDR_WT) ? CACHE_PURGING_WT : 0; 87 cunit->cache_purging = (ccsidr & CCSIDR_WT) ? CACHE_PURGING_WT : 0;
85 cunit->cache_purging |= (ccsidr & CCSIDR_WB) ? CACHE_PURGING_WB : 0; 88 cunit->cache_purging |= (ccsidr & CCSIDR_WB) ? CACHE_PURGING_WB : 0;
86 cunit->cache_purging |= (ccsidr & CCSIDR_RA) ? CACHE_PURGING_RA : 0; 89 cunit->cache_purging |= (ccsidr & CCSIDR_RA) ? CACHE_PURGING_RA : 0;
87 cunit->cache_purging |= (ccsidr & CCSIDR_WA) ? CACHE_PURGING_WA : 0; 90 cunit->cache_purging |= (ccsidr & CCSIDR_WA) ? CACHE_PURGING_WA : 0;
88} 91}
89 92
90void 93void
91aarch64_getcacheinfo(void) 94aarch64_getcacheinfo(void)
92{ 95{
93 uint32_t clidr, ctr; 96 uint32_t clidr, ctr;
 97 u_int vindexsize;
94 int level, cachetype; 98 int level, cachetype;
95 struct aarch64_cache_info *cinfo; 99 struct aarch64_cache_info *cinfo;
96 100
97 if (cputype == 0) 101 if (cputype == 0)
98 cputype = aarch64_cpuid(); 102 cputype = aarch64_cpuid();
99 103
100 /* already extract about this cluster? */ 104 /* already extract about this cluster? */
101 KASSERT(curcpu()->ci_package_id < MAXCPUPACKAGES); 105 KASSERT(curcpu()->ci_package_id < MAXCPUPACKAGES);
102 cinfo = aarch64_cacheinfo[curcpu()->ci_package_id]; 106 cinfo = aarch64_cacheinfo[curcpu()->ci_package_id];
103 if (cinfo != NULL) { 107 if (cinfo != NULL) {
104 curcpu()->ci_cacheinfo = cinfo; 108 curcpu()->ci_cacheinfo = cinfo;
105 return; 109 return;
106 } 110 }
@@ -191,34 +195,40 @@ aarch64_getcacheinfo(void) @@ -191,34 +195,40 @@ aarch64_getcacheinfo(void)
191 /* 195 /*
192 * L1 insn cachetype is CTR_EL0:L1IP, 196 * L1 insn cachetype is CTR_EL0:L1IP,
193 * all other cachetype is PIPT. 197 * all other cachetype is PIPT.
194 */ 198 */
195 cachetype = CACHE_TYPE_PIPT; 199 cachetype = CACHE_TYPE_PIPT;
196 } 200 }
197 201
198 /* calculate L1 icache virtual index size */ 202 /* calculate L1 icache virtual index size */
199 if (((cinfo[0].icache.cache_type == CACHE_TYPE_VIVT) || 203 if (((cinfo[0].icache.cache_type == CACHE_TYPE_VIVT) ||
200 (cinfo[0].icache.cache_type == CACHE_TYPE_VIPT)) && 204 (cinfo[0].icache.cache_type == CACHE_TYPE_VIPT)) &&
201 ((cinfo[0].cacheable == CACHE_CACHEABLE_ICACHE) || 205 ((cinfo[0].cacheable == CACHE_CACHEABLE_ICACHE) ||
202 (cinfo[0].cacheable == CACHE_CACHEABLE_IDCACHE))) { 206 (cinfo[0].cacheable == CACHE_CACHEABLE_IDCACHE))) {
203 207
204 aarch64_cache_vindexsize = 208 vindexsize =
205 cinfo[0].icache.cache_size / 209 cinfo[0].icache.cache_size /
206 cinfo[0].icache.cache_ways; 210 cinfo[0].icache.cache_ways;
207 211
208 KASSERT(aarch64_cache_vindexsize != 0); 212 KASSERT(vindexsize != 0);
209 aarch64_cache_prefer_mask = aarch64_cache_vindexsize - 1; 
210 } else { 213 } else {
211 aarch64_cache_vindexsize = 0; 214 vindexsize = 0;
 215 }
 216
 217 if (vindexsize > aarch64_cache_vindexsize) {
 218 aarch64_cache_vindexsize = vindexsize;
 219 aarch64_cache_prefer_mask = vindexsize - 1;
 220 if (uvm.page_init_done)
 221 uvm_page_recolor(vindexsize / PAGE_SIZE);
212 } 222 }
213} 223}
214 224
215static int 225static int
216prt_cache(device_t self, struct aarch64_cache_info *cinfo, int level) 226prt_cache(device_t self, struct aarch64_cache_info *cinfo, int level)
217{ 227{
218 struct aarch64_cache_unit *cunit; 228 struct aarch64_cache_unit *cunit;
219 u_int purging; 229 u_int purging;
220 int i; 230 int i;
221 const char *cacheable, *cachetype; 231 const char *cacheable, *cachetype;
222 232
223 if (cinfo[level].cacheable == CACHE_CACHEABLE_NONE) 233 if (cinfo[level].cacheable == CACHE_CACHEABLE_NONE)
224 return -1; 234 return -1;
cvs diff -r1.41.2.6 -r1.41.2.7 src/sys/arch/aarch64/aarch64/pmap.c (expand / switch to unified diff)

--- src/sys/arch/aarch64/aarch64/pmap.c 2020/06/30 18:39:37 1.41.2.6
+++ src/sys/arch/aarch64/aarch64/pmap.c 2021/01/01 12:38:49 1.41.2.7
@@ -1,14 +1,14 @@ @@ -1,14 +1,14 @@
1/* $NetBSD: pmap.c,v 1.41.2.6 2020/06/30 18:39:37 martin Exp $ */ 1/* $NetBSD: pmap.c,v 1.41.2.7 2021/01/01 12:38:49 martin Exp $ */
2 2
3/* 3/*
4 * Copyright (c) 2017 Ryo Shimizu <ryo@nerv.org> 4 * Copyright (c) 2017 Ryo Shimizu <ryo@nerv.org>
5 * All rights reserved. 5 * All rights reserved.
6 * 6 *
7 * Redistribution and use in source and binary forms, with or without 7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions 8 * modification, are permitted provided that the following conditions
9 * are met: 9 * are met:
10 * 1. Redistributions of source code must retain the above copyright 10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer. 11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright 12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the 13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution. 14 * documentation and/or other materials provided with the distribution.
@@ -17,27 +17,27 @@ @@ -17,27 +17,27 @@
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 19 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
20 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
24 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 24 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
25 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 * POSSIBILITY OF SUCH DAMAGE. 26 * POSSIBILITY OF SUCH DAMAGE.
27 */ 27 */
28 28
29#include <sys/cdefs.h> 29#include <sys/cdefs.h>
30__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.41.2.6 2020/06/30 18:39:37 martin Exp $"); 30__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.41.2.7 2021/01/01 12:38:49 martin Exp $");
31 31
32#include "opt_arm_debug.h" 32#include "opt_arm_debug.h"
33#include "opt_ddb.h" 33#include "opt_ddb.h"
34#include "opt_kasan.h" 34#include "opt_kasan.h"
35#include "opt_multiprocessor.h" 35#include "opt_multiprocessor.h"
36#include "opt_pmap.h" 36#include "opt_pmap.h"
37#include "opt_uvmhist.h" 37#include "opt_uvmhist.h"
38 38
39#include <sys/param.h> 39#include <sys/param.h>
40#include <sys/types.h> 40#include <sys/types.h>
41#include <sys/kmem.h> 41#include <sys/kmem.h>
42#include <sys/vmem.h> 42#include <sys/vmem.h>
43#include <sys/atomic.h> 43#include <sys/atomic.h>
@@ -408,30 +408,27 @@ pmap_devmap_vtophys(paddr_t va) @@ -408,30 +408,27 @@ pmap_devmap_vtophys(paddr_t va)
408 408
409void 409void
410pmap_bootstrap(vaddr_t vstart, vaddr_t vend) 410pmap_bootstrap(vaddr_t vstart, vaddr_t vend)
411{ 411{
412 struct pmap *kpm; 412 struct pmap *kpm;
413 pd_entry_t *l0; 413 pd_entry_t *l0;
414 paddr_t l0pa; 414 paddr_t l0pa;
415 415
416 PMAP_HIST_INIT(); /* init once */ 416 PMAP_HIST_INIT(); /* init once */
417 417
418 UVMHIST_FUNC(__func__); 418 UVMHIST_FUNC(__func__);
419 UVMHIST_CALLED(pmaphist); 419 UVMHIST_CALLED(pmaphist);
420 420
421#if 0 
422 /* uvmexp.ncolors = icachesize / icacheways / PAGE_SIZE; */ 
423 uvmexp.ncolors = aarch64_cache_vindexsize / PAGE_SIZE; 421 uvmexp.ncolors = aarch64_cache_vindexsize / PAGE_SIZE;
424#endif 
425 422
426 /* devmap already uses last of va? */ 423 /* devmap already uses last of va? */
427 if ((virtual_devmap_addr != 0) && (virtual_devmap_addr < vend)) 424 if ((virtual_devmap_addr != 0) && (virtual_devmap_addr < vend))
428 vend = virtual_devmap_addr; 425 vend = virtual_devmap_addr;
429 426
430 virtual_avail = vstart; 427 virtual_avail = vstart;
431 virtual_end = vend; 428 virtual_end = vend;
432 pmap_maxkvaddr = vstart; 429 pmap_maxkvaddr = vstart;
433 430
434 aarch64_tlbi_all(); 431 aarch64_tlbi_all();
435 432
436 l0pa = reg_ttbr1_el1_read(); 433 l0pa = reg_ttbr1_el1_read();
437 l0 = (void *)AARCH64_PA_TO_KVA(l0pa); 434 l0 = (void *)AARCH64_PA_TO_KVA(l0pa);