| @@ -1,14 +1,14 @@ | | | @@ -1,14 +1,14 @@ |
1 | /* $NetBSD: locore.S,v 1.70 2020/08/12 13:19:35 skrll Exp $ */ | | 1 | /* $NetBSD: locore.S,v 1.71 2020/08/16 10:08:42 skrll Exp $ */ |
2 | | | 2 | |
3 | /* | | 3 | /* |
4 | * Copyright (c) 2017 Ryo Shimizu <ryo@nerv.org> | | 4 | * Copyright (c) 2017 Ryo Shimizu <ryo@nerv.org> |
5 | * All rights reserved. | | 5 | * All rights reserved. |
6 | * | | 6 | * |
7 | * Redistribution and use in source and binary forms, with or without | | 7 | * Redistribution and use in source and binary forms, with or without |
8 | * modification, are permitted provided that the following conditions | | 8 | * modification, are permitted provided that the following conditions |
9 | * are met: | | 9 | * are met: |
10 | * 1. Redistributions of source code must retain the above copyright | | 10 | * 1. Redistributions of source code must retain the above copyright |
11 | * notice, this list of conditions and the following disclaimer. | | 11 | * notice, this list of conditions and the following disclaimer. |
12 | * 2. Redistributions in binary form must reproduce the above copyright | | 12 | * 2. Redistributions in binary form must reproduce the above copyright |
13 | * notice, this list of conditions and the following disclaimer in the | | 13 | * notice, this list of conditions and the following disclaimer in the |
14 | * documentation and/or other materials provided with the distribution. | | 14 | * documentation and/or other materials provided with the distribution. |
| @@ -28,27 +28,27 @@ | | | @@ -28,27 +28,27 @@ |
28 | | | 28 | |
29 | #include "opt_arm_debug.h" | | 29 | #include "opt_arm_debug.h" |
30 | #include "opt_console.h" | | 30 | #include "opt_console.h" |
31 | #include "opt_cpuoptions.h" | | 31 | #include "opt_cpuoptions.h" |
32 | #include "opt_ddb.h" | | 32 | #include "opt_ddb.h" |
33 | #include "opt_fdt.h" | | 33 | #include "opt_fdt.h" |
34 | #include "opt_kasan.h" | | 34 | #include "opt_kasan.h" |
35 | #include "opt_multiprocessor.h" | | 35 | #include "opt_multiprocessor.h" |
36 | | | 36 | |
37 | #include <aarch64/asm.h> | | 37 | #include <aarch64/asm.h> |
38 | #include <aarch64/hypervisor.h> | | 38 | #include <aarch64/hypervisor.h> |
39 | #include "assym.h" | | 39 | #include "assym.h" |
40 | | | 40 | |
41 | RCSID("$NetBSD: locore.S,v 1.70 2020/08/12 13:19:35 skrll Exp $") | | 41 | RCSID("$NetBSD: locore.S,v 1.71 2020/08/16 10:08:42 skrll Exp $") |
42 | | | 42 | |
43 | #ifdef AARCH64_DEVICE_MEM_STRONGLY_ORDERED | | 43 | #ifdef AARCH64_DEVICE_MEM_STRONGLY_ORDERED |
44 | #define MAIR_DEVICE_MEM MAIR_DEVICE_nGnRnE | | 44 | #define MAIR_DEVICE_MEM MAIR_DEVICE_nGnRnE |
45 | #else | | 45 | #else |
46 | #define MAIR_DEVICE_MEM MAIR_DEVICE_nGnRE | | 46 | #define MAIR_DEVICE_MEM MAIR_DEVICE_nGnRE |
47 | #endif | | 47 | #endif |
48 | #define MAIR_DEVICE_MEM_SO MAIR_DEVICE_nGnRnE | | 48 | #define MAIR_DEVICE_MEM_SO MAIR_DEVICE_nGnRnE |
49 | | | 49 | |
50 | /*#define DEBUG_LOCORE // debug print */ | | 50 | /*#define DEBUG_LOCORE // debug print */ |
51 | /*#define DEBUG_LOCORE_PRINT_LOCK // avoid mixing AP's output */ | | 51 | /*#define DEBUG_LOCORE_PRINT_LOCK // avoid mixing AP's output */ |
52 | | | 52 | |
53 | #define LOCORE_EL2 | | 53 | #define LOCORE_EL2 |
54 | | | 54 | |
| @@ -444,33 +444,35 @@ ENTRY_NP(cpu_mpstart) | | | @@ -444,33 +444,35 @@ ENTRY_NP(cpu_mpstart) |
444 | | | 444 | |
445 | mp_vstart: | | 445 | mp_vstart: |
446 | hint 0x24 /* bti j */ | | 446 | hint 0x24 /* bti j */ |
447 | CPU_DPRINTREG("PC = ", x20) | | 447 | CPU_DPRINTREG("PC = ", x20) |
448 | | | 448 | |
449 | /* set exception vector */ | | 449 | /* set exception vector */ |
450 | adrl x0, _C_LABEL(el1_vectors) | | 450 | adrl x0, _C_LABEL(el1_vectors) |
451 | msr vbar_el1, x0 | | 451 | msr vbar_el1, x0 |
452 | | | 452 | |
453 | /* lwp-private = NULL */ | | 453 | /* lwp-private = NULL */ |
454 | msr tpidr_el0, xzr | | 454 | msr tpidr_el0, xzr |
455 | msr tpidrro_el0, xzr | | 455 | msr tpidrro_el0, xzr |
456 | | | 456 | |
457 | /* set curcpu(), and fill curcpu()->ci_{midr,mpidr} */ | | | |
458 | mov x0, #CPU_INFO_SIZE | | 457 | mov x0, #CPU_INFO_SIZE |
459 | mul x0, x27, x0 | | 458 | mul x0, x27, x0 |
460 | adrl x1, _C_LABEL(cpu_info_store) | | 459 | adrl x1, _C_LABEL(cpu_info_store) |
461 | add x0, x0, x1 /* x0 = &cpu_info_store[cpuindex] */ | | 460 | add x0, x0, x1 /* x0 = &cpu_info_store[cpuindex] */ |
| | | 461 | |
| | | 462 | /* temporarily set tpidr_el1 to curcpu until the idle lwp is setup */ |
462 | msr tpidr_el1, x0 /* tpidr_el1 = curcpu = x0 */ | | 463 | msr tpidr_el1, x0 /* tpidr_el1 = curcpu = x0 */ |
463 | | | 464 | |
| | | 465 | /* fill curcpu()->ci_{midr,mpidr} */ |
464 | mrs x1, midr_el1 | | 466 | mrs x1, midr_el1 |
465 | str x1, [x0, #CI_MIDR] /* curcpu()->ci_cpuid = midr_el1 */ | | 467 | str x1, [x0, #CI_MIDR] /* curcpu()->ci_cpuid = midr_el1 */ |
466 | mrs x1, mpidr_el1 | | 468 | mrs x1, mpidr_el1 |
467 | str x1, [x0, #CI_MPIDR] /* curcpu()->ci_mpidr = mpidr_el1 */ | | 469 | str x1, [x0, #CI_MPIDR] /* curcpu()->ci_mpidr = mpidr_el1 */ |
468 | | | 470 | |
469 | /* set topology information */ | | 471 | /* set topology information */ |
470 | mov x2, #0 | | 472 | mov x2, #0 |
471 | bl arm_cpu_topology_set | | 473 | bl arm_cpu_topology_set |
472 | | | 474 | |
473 | /* x28 = &arm_cpu_hatched[cpuindex / (sizeof(u_long) * NBBY)] */ | | 475 | /* x28 = &arm_cpu_hatched[cpuindex / (sizeof(u_long) * NBBY)] */ |
474 | adrl x0, _C_LABEL(arm_cpu_hatched) | | 476 | adrl x0, _C_LABEL(arm_cpu_hatched) |
475 | // Appease clang - mov x1, x27, lsr #6 | | 477 | // Appease clang - mov x1, x27, lsr #6 |
476 | orr x1, xzr, x27, lsr #6 | | 478 | orr x1, xzr, x27, lsr #6 |
| @@ -498,28 +500,31 @@ mp_vstart: | | | @@ -498,28 +500,31 @@ mp_vstart: |
498 | orr x1, xzr, x27, lsr #6 | | 500 | orr x1, xzr, x27, lsr #6 |
499 | add x28, x0, x1, lsl #3 | | 501 | add x28, x0, x1, lsl #3 |
500 | | | 502 | |
501 | /* wait for the mailbox start bit to become true */ | | 503 | /* wait for the mailbox start bit to become true */ |
502 | 1: | | 504 | 1: |
503 | dmb sy | | 505 | dmb sy |
504 | ldr x20, [x28] | | 506 | ldr x20, [x28] |
505 | tst x20, x29 | | 507 | tst x20, x29 |
506 | bne 9f | | 508 | bne 9f |
507 | wfe | | 509 | wfe |
508 | b 1b | | 510 | b 1b |
509 | 9: | | 511 | 9: |
510 | | | 512 | |
511 | /* fill my cpu_info */ | | 513 | /* |
512 | mrs x0, tpidr_el1 /* curcpu */ | | 514 | * set curlwp (tpidr_el1 and curcpu()->ci_curlwp) now we know the |
| | | 515 | * idle lwp from curcpu()->ci_idlelwp |
| | | 516 | */ |
| | | 517 | mrs x0, tpidr_el1 /* curcpu (temporarily) */ |
513 | ldr x1, [x0, #CI_IDLELWP] /* x0 = curcpu()->ci_idlelwp */ | | 518 | ldr x1, [x0, #CI_IDLELWP] /* x0 = curcpu()->ci_idlelwp */ |
514 | msr tpidr_el1, x1 /* tpidr_el1 = curlwp = x1 */ | | 519 | msr tpidr_el1, x1 /* tpidr_el1 = curlwp = x1 */ |
515 | str x1, [x0, #CI_CURLWP] /* curlwp is idlelwp */ | | 520 | str x1, [x0, #CI_CURLWP] /* curlwp is idlelwp */ |
516 | | | 521 | |
517 | /* get my stack from lwp */ | | 522 | /* get my stack from lwp */ |
518 | ldr x2, [x1, #L_PCB] /* x2 = lwp_getpcb(idlelwp) */ | | 523 | ldr x2, [x1, #L_PCB] /* x2 = lwp_getpcb(idlelwp) */ |
519 | add x2, x2, #(UPAGES * PAGE_SIZE) | | 524 | add x2, x2, #(UPAGES * PAGE_SIZE) |
520 | sub sp, x2, #TF_SIZE /* sp = pcb + USPACE - TF_SIZE */ | | 525 | sub sp, x2, #TF_SIZE /* sp = pcb + USPACE - TF_SIZE */ |
521 | | | 526 | |
522 | /* init PAN if supported */ | | 527 | /* init PAN if supported */ |
523 | mov x0, #0 | | 528 | mov x0, #0 |
524 | bl aarch64_pan_init | | 529 | bl aarch64_pan_init |
525 | | | 530 | |