Thu Apr 16 08:59:16 2020 UTC ()
Shave off 3 instructions per trap


(skrll)
diff -r1.14 -r1.15 src/sys/arch/aarch64/aarch64/vectors.S
cvs diff -r1.14 -r1.15 src/sys/arch/aarch64/aarch64/vectors.S (switch to unified diff)

--- src/sys/arch/aarch64/aarch64/vectors.S 2020/04/13 05:40:25 1.14
+++ src/sys/arch/aarch64/aarch64/vectors.S 2020/04/16 08:59:16 1.15
@@ -1,182 +1,183 @@ @@ -1,182 +1,183 @@
1/* $NetBSD: vectors.S,v 1.14 2020/04/13 05:40:25 maxv Exp $ */ 1/* $NetBSD: vectors.S,v 1.15 2020/04/16 08:59:16 skrll Exp $ */
2 2
3#include <aarch64/asm.h> 3#include <aarch64/asm.h>
4#include "assym.h" 4#include "assym.h"
5#include "opt_cpuoptions.h" 5#include "opt_cpuoptions.h"
6#include "opt_ddb.h" 6#include "opt_ddb.h"
7#include "opt_dtrace.h" 7#include "opt_dtrace.h"
8 8
9 ARMV8_DEFINE_OPTIONS 9 ARMV8_DEFINE_OPTIONS
10 10
11#ifdef KDTRACE_HOOKS 11#ifdef KDTRACE_HOOKS
12/* 12/*
13 * dtrace needs to emulate stp x29,x30,[sp,#-FRAMESIZE]! where 13 * dtrace needs to emulate stp x29,x30,[sp,#-FRAMESIZE]! where
14 * FRAMESIZE can be as large as 512, so create a 512-byte buffer 14 * FRAMESIZE can be as large as 512, so create a 512-byte buffer
15 * between the interrupted code's frame and our struct trapframe. 15 * between the interrupted code's frame and our struct trapframe.
16 */ 16 */
17#define TRAP_FRAMESIZE (TF_SIZE + 512) 17#define TRAP_FRAMESIZE (TF_SIZE + 512)
18#else 18#else
19#define TRAP_FRAMESIZE TF_SIZE 19#define TRAP_FRAMESIZE TF_SIZE
20#endif 20#endif
21 21
22/* 22/*
23 * Template for the handler functions. 23 * Template for the handler functions.
24 */ 24 */
25.macro vector_func, func, el, label, tpidr 25.macro vector_func, func, el, label, tpidr
26ENTRY_NBTI(\func) 
27 .align 7 /* cacheline-aligned */ 26 .align 7 /* cacheline-aligned */
28 27
 28ENTRY_NBTI(\func)
 29
29 .if \el == 1 30 .if \el == 1
30 /* need to allocate stack on el1 */ 31 /* need to allocate stack on el1 */
31 sub sp, sp, #TRAP_FRAMESIZE 32 sub sp, sp, #TRAP_FRAMESIZE
32 .endif 33 .endif
33 34
34 stp x0, x1, [sp, #TF_X0] 35 stp x0, x1, [sp, #TF_X0]
35 stp x2, x3, [sp, #TF_X2] 36 stp x2, x3, [sp, #TF_X2]
36 stp x4, x5, [sp, #TF_X4] 37 stp x4, x5, [sp, #TF_X4]
37 stp x6, x7, [sp, #TF_X6] 38 stp x6, x7, [sp, #TF_X6]
38 stp x8, x9, [sp, #TF_X8] 39 stp x8, x9, [sp, #TF_X8]
39 stp x10, x11, [sp, #TF_X10] 40 stp x10, x11, [sp, #TF_X10]
40 stp x12, x13, [sp, #TF_X12] 41 stp x12, x13, [sp, #TF_X12]
41 stp x14, x15, [sp, #TF_X14] 42 stp x14, x15, [sp, #TF_X14]
42 stp x16, x17, [sp, #TF_X16] 43 stp x16, x17, [sp, #TF_X16]
43 str x18, [sp, #TF_X18] 44 str x18, [sp, #TF_X18]
44 stp x19, x20, [sp, #TF_X19] 45 stp x19, x20, [sp, #TF_X19]
45 stp x21, x22, [sp, #TF_X21] 46 stp x21, x22, [sp, #TF_X21]
46 stp x23, x24, [sp, #TF_X23] 47 stp x23, x24, [sp, #TF_X23]
47 stp x25, x26, [sp, #TF_X25] 48 stp x25, x26, [sp, #TF_X25]
48 stp x27, x28, [sp, #TF_X27] 49 stp x27, x28, [sp, #TF_X27]
49 stp x29, x30, [sp, #TF_X29] 50 stp x29, x30, [sp, #TF_X29]
50 51
51 /* get sp and elr */ 52 /* get sp and elr */
52 .if \el == 0 53 .if \el == 0
53 mrs x20, sp_el0 54 mrs x20, sp_el0
54 .else 55 .else
55 /* sp was already adjusted, so adjust x20 back */ 56 /* sp was already adjusted, so adjust x20 back */
56 add x20, sp, #TRAP_FRAMESIZE 57 add x20, sp, #TRAP_FRAMESIZE
57 .endif 58 .endif
58 mrs x21, elr_el1 59 mrs x21, elr_el1
59 60
60 /* store sp and elr */ 61 /* store sp and elr */
61 .if TF_SP + 8 == TF_PC 62 .if TF_SP + 8 == TF_PC
62 stp x20, x21, [sp, #TF_SP] 63 stp x20, x21, [sp, #TF_SP]
63 .else 64 .else
64 str x20, [sp, #TF_SP] 65 str x20, [sp, #TF_SP]
65 str x21, [sp, #TF_PC] 66 str x21, [sp, #TF_PC]
66 .endif 67 .endif
67 68
68 mrs x22, spsr_el1 69 mrs x22, spsr_el1
69 str x22, [sp, #TF_SPSR] 70 str x22, [sp, #TF_SPSR]
70 71
71 mrs x23, esr_el1 72 mrs x23, esr_el1
72 mrs x24, far_el1 73 mrs x24, far_el1
73 74
74 .if TF_ESR + 8 == TF_FAR 75 .if TF_ESR + 8 == TF_FAR
75 stp x23, x24, [sp, #TF_ESR] 76 stp x23, x24, [sp, #TF_ESR]
76 .else 77 .else
77 str x23, [sp, #TF_ESR] 78 str x23, [sp, #TF_ESR]
78 str x24, [sp, #TF_FAR] 79 str x24, [sp, #TF_FAR]
79 .endif 80 .endif
80 81
81 .if \el == 0 82 .if \el == 0
82 /* curlwp->l_private = tpidr{,ro}_el0 */ 83 /* curlwp->l_private = tpidr{,ro}_el0 */
83 mrs x1, tpidr_el1 /* curcpu() */ 84 mrs x1, tpidr_el1 /* curcpu() */
84 ldr x1, [x1, #CI_CURLWP] /* x1 = curcpu()->ci_curlwp */ 85 ldr x1, [x1, #CI_CURLWP] /* x1 = curcpu()->ci_curlwp */
85 mrs x0, tpidr\tpidr\()_el0 86 mrs x0, tpidr\tpidr\()_el0
86 str x0, [x1, #L_PRIVATE] /* curlwp->l_private = tpidr{,ro}_el0 */ 87 str x0, [x1, #L_PRIVATE] /* curlwp->l_private = tpidr{,ro}_el0 */
87 88
88#ifdef ARMV83_PAC 89#ifdef ARMV83_PAC
89 /* Switch to the kern PAC key. */ 90 /* Switch to the kern PAC key. */
90 adrl x4, _C_LABEL(aarch64_pac_enabled) 91 adrl x4, _C_LABEL(aarch64_pac_enabled)
91 ldr w4, [x4] 92 ldr w4, [x4]
92 cbz w4, 1f 93 cbz w4, 1f
93 ldr x5, [x1, #L_MD_IA_KERN_LO] 94 ldr x5, [x1, #L_MD_IA_KERN_LO]
94 ldr x6, [x1, #L_MD_IA_KERN_HI] 95 ldr x6, [x1, #L_MD_IA_KERN_HI]
95 msr APIAKeyLo_EL1, x5 96 msr APIAKeyLo_EL1, x5
96 msr APIAKeyHi_EL1, x6 97 msr APIAKeyHi_EL1, x6
971: 981:
98#endif 99#endif
99 .endif 100 .endif
100 101
101 adr x30, el\el\()_trap_exit /* el[01]_trap_exit */ 102 adr x30, el\el\()_trap_exit /* el[01]_trap_exit */
102 mov x0, sp 103 mov x0, sp
103#ifdef DDB 104#ifdef DDB
104 mov x29, sp /* for backtrace */ 105 mov x29, sp /* for backtrace */
105#endif 106#endif
106 b \label 107 b \label
107END(\func) 108END(\func)
108.endm 109.endm
109 110
110/* 111/*
111 * The vector_entry macro must be small enough to fit 0x80 bytes! We just jump 112 * The vector_entry macro must be small enough to fit 0x80 bytes! We just jump
112 * into the proper function, so this constraint is always respected. 113 * into the proper function, so this constraint is always respected.
113 */ 114 */
114.macro vector_entry, func 115.macro vector_entry, func
115 .align 7 /* aligned 0x80 */ 116 .align 7 /* aligned 0x80 */
116 b \func 117 b \func
117.endm 118.endm
118 119
119/* 120/*
120 * The functions. 121 * The functions.
121 */ 122 */
122vector_func el1t_sync_handler, 1, trap_el1t_sync 123vector_func el1t_sync_handler, 1, trap_el1t_sync
123vector_func el1t_irq_handler, 1, trap_el1t_irq 124vector_func el1t_irq_handler, 1, trap_el1t_irq
124vector_func el1t_fiq_handler, 1, trap_el1t_fiq 125vector_func el1t_fiq_handler, 1, trap_el1t_fiq
125vector_func el1t_error_handler, 1, trap_el1t_error 126vector_func el1t_error_handler, 1, trap_el1t_error
126 127
127vector_func el1h_sync_handler, 1, trap_el1h_sync 128vector_func el1h_sync_handler, 1, trap_el1h_sync
128vector_func el1h_intr_handler, 1, interrupt 129vector_func el1h_intr_handler, 1, interrupt
129vector_func el1h_fiq_handler, 1, trap_el1h_fiq 130vector_func el1h_fiq_handler, 1, trap_el1h_fiq
130vector_func el1h_error_handler, 1, trap_el1h_error 131vector_func el1h_error_handler, 1, trap_el1h_error
131 132
132vector_func el0_sync_handler, 0, trap_el0_sync 133vector_func el0_sync_handler, 0, trap_el0_sync
133vector_func el0_intr_handler, 0, interrupt 134vector_func el0_intr_handler, 0, interrupt
134vector_func el0_fiq_handler, 0, trap_el0_fiq 135vector_func el0_fiq_handler, 0, trap_el0_fiq
135vector_func el0_error_handler, 0, trap_el0_error 136vector_func el0_error_handler, 0, trap_el0_error
136 137
137vector_func el0_32sync_handler, 0, trap_el0_32sync, ro 138vector_func el0_32sync_handler, 0, trap_el0_32sync, ro
138vector_func el0_32intr_handler, 0, interrupt, ro 139vector_func el0_32intr_handler, 0, interrupt, ro
139vector_func el0_32fiq_handler, 0, trap_el0_32fiq, ro 140vector_func el0_32fiq_handler, 0, trap_el0_32fiq, ro
140vector_func el0_32error_handler, 0, trap_el0_32error, ro 141vector_func el0_32error_handler, 0, trap_el0_32error, ro
141 142
142/* 143/*
143 * The vector table. Must be aligned to 2048. 144 * The vector table. Must be aligned to 2048.
144 */ 145 */
145 .align 11 146 .align 11
146ENTRY_NBTI(el1_vectors) 147ENTRY_NBTI(el1_vectors)
147 /* 148 /*
148 * Exception taken from current Exception Level with SP_EL0. 149 * Exception taken from current Exception Level with SP_EL0.
149 * (These shouldn't happen) 150 * (These shouldn't happen)
150 */ 151 */
151 vector_entry el1t_sync_handler 152 vector_entry el1t_sync_handler
152 vector_entry el1t_irq_handler 153 vector_entry el1t_irq_handler
153 vector_entry el1t_fiq_handler 154 vector_entry el1t_fiq_handler
154 vector_entry el1t_error_handler 155 vector_entry el1t_error_handler
155 156
156 /* 157 /*
157 * Exception taken from current Exception Level with SP_EL1. 158 * Exception taken from current Exception Level with SP_EL1.
158 * There are entries for exceptions caused in EL1 (kernel exceptions). 159 * There are entries for exceptions caused in EL1 (kernel exceptions).
159 */ 160 */
160 vector_entry el1h_sync_handler 161 vector_entry el1h_sync_handler
161 vector_entry el1h_intr_handler 162 vector_entry el1h_intr_handler
162 vector_entry el1h_fiq_handler 163 vector_entry el1h_fiq_handler
163 vector_entry el1h_error_handler 164 vector_entry el1h_error_handler
164 165
165 /* 166 /*
166 * Exception taken from lower Exception Level which is using AArch64. 167 * Exception taken from lower Exception Level which is using AArch64.
167 * There are entries for exceptions caused in EL0 (native user exceptions). 168 * There are entries for exceptions caused in EL0 (native user exceptions).
168 */ 169 */
169 vector_entry el0_sync_handler 170 vector_entry el0_sync_handler
170 vector_entry el0_intr_handler 171 vector_entry el0_intr_handler
171 vector_entry el0_fiq_handler 172 vector_entry el0_fiq_handler
172 vector_entry el0_error_handler 173 vector_entry el0_error_handler
173 174
174 /* 175 /*
175 * Exception taken from lower Exception Level which is using AArch32. 176 * Exception taken from lower Exception Level which is using AArch32.
176 * There are entries for exceptions caused in EL0 (compat user exceptions). 177 * There are entries for exceptions caused in EL0 (compat user exceptions).
177 */ 178 */
178 vector_entry el0_32sync_handler 179 vector_entry el0_32sync_handler
179 vector_entry el0_32intr_handler 180 vector_entry el0_32intr_handler
180 vector_entry el0_32fiq_handler 181 vector_entry el0_32fiq_handler
181 vector_entry el0_32error_handler 182 vector_entry el0_32error_handler
182END(el1_vectors) 183END(el1_vectors)