 @@ -1,1000 +1,1000 @@
-/*	$NetBSD: locore.s,v 1.297 2009/11/30 01:45:04 mrg Exp $	*/
+/*	$NetBSD: locore.s,v 1.298 2009/11/30 01:58:49 mrg Exp $	*/
 /*
  * Copyright (c) 1996-2002 Eduardo Horvath
  * Copyright (c) 1996 Paul Kranenburg
  * Copyright (c) 1996
  * 	The President and Fellows of Harvard College.
  *	All rights reserved.
  * Copyright (c) 1992, 1993
  *	The Regents of the University of California.
  *	All rights reserved.
+ *
  * This software was developed by the Computer Systems Engineering group
  * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and
  * contributed to Berkeley.
+ *
  * All advertising materials mentioning features or use of this software
  * must display the following acknowledgement:
  *	This product includes software developed by the University of
  *	California, Lawrence Berkeley Laboratory.
  *	This product includes software developed by Harvard University.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the
  *    distribution.
  * 3. All advertising materials mentioning features or use of this
  *    software must display the following acknowledgement:
  *	This product includes software developed by the University of
  *	California, Berkeley and its contributors.
  *	This product includes software developed by Harvard University.
  *	This product includes software developed by Paul Kranenburg.
  * 4. Neither the name of the University nor the names of its
  *    contributors may be used to endorse or promote products derived
  *    from this software without specific prior written permission.
+ *
  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS''
  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
  * PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR
  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
  * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
  * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
  * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
  * DAMAGE.
+ *
  *	@(#)locore.s	8.4 (Berkeley) 12/10/93
  */
 #ifndef SCHIZO_BUS_SPACE_BROKEN /* Need phys access for USIII so far */
 #define	SPITFIRE		/* We don't support Cheetah (USIII) yet */
 #endif
 #undef	PARANOID		/* Extremely expensive consistency checks */
 #undef	NO_VCACHE		/* Map w/D$ disabled */
 #undef	TRAPSTATS		/* Count traps */
 #undef	TRAPS_USE_IG		/* Use Interrupt Globals for all traps */
 #define	HWREF			/* Track ref/mod bits in trap handlers */
 #undef	DCACHE_BUG		/* Flush D$ around ASI_PHYS accesses */
 #undef	NO_TSB			/* Don't use TSB */
 #define	USE_BLOCK_STORE_LOAD	/* enable block load/store ops */
 #define	BB_ERRATA_1		/* writes to TICK_CMPR may fail */
 #include "opt_ddb.h"
 #include "opt_kgdb.h"
 #include "opt_multiprocessor.h"
 #include "opt_compat_netbsd.h"
 #include "opt_compat_netbsd32.h"
 #include "opt_lockdebug.h"
 #include "assym.h"
 #include <machine/param.h>
 #include <sparc64/sparc64/intreg.h>
 #include <sparc64/sparc64/timerreg.h>
 #include <machine/ctlreg.h>
 #include <machine/psl.h>
 #include <machine/signal.h>
 #include <machine/trap.h>
 #include <machine/frame.h>
 #include <machine/pte.h>
 #include <machine/pmap.h>
 #include <machine/intr.h>
 #include <machine/asm.h>
 #include <sys/syscall.h>
 #include "ksyms.h"
 /* A few convenient abbreviations for trapframe fields. */
 #define	TF_G	TF_GLOBAL
 #define	TF_O	TF_OUT
 #define	TF_L	TF_LOCAL
 #define	TF_I	TF_IN
 #undef	CURLWP
 #undef	CPCB
 #undef	FPLWP
 #define	CURLWP	(CPUINFO_VA + CI_CURLWP)
 #define	CPCB	(CPUINFO_VA + CI_CPCB)
 #define	FPLWP	(CPUINFO_VA + CI_FPLWP)
 /* Let us use same syntax as C code */
 #define Debugger()	ta	1; nop
 #if 1
 /*
  * Try to issue an elf note to ask the Solaris
  * bootloader to align the kernel properly.
  */
 	.section	.note
 	.word	0x0d
 	.word	4		! Dunno why
 	.word	1
 :	.asciz	"SUNW Solaris"
 :
 	.align	4
 	.word	0x0400000
 #endif
 	.register	%g2,#scratch
 	.register	%g3,#scratch
 /*
  * Here are some defines to try to maintain consistency but still
  * support 32-and 64-bit compilers.
  */
 #ifdef _LP64
 /* reg that points to base of data/text segment */
 #define	BASEREG	%g4
 /* first constants for storage allocation */
 #define LNGSZ		8
 #define LNGSHFT		3
 #define PTRSZ		8
 #define PTRSHFT		3
 #define POINTER		.xword
 #define ULONG		.xword
 /* Now instructions to load/store pointers & long ints */
 #define LDLNG		ldx
 #define LDULNG		ldx
 #define STLNG		stx
 #define STULNG		stx
 #define LDPTR		ldx
 #define LDPTRA		ldxa
 #define STPTR		stx
 #define STPTRA		stxa
 #define	CASPTR		casxa
 /* Now something to calculate the stack bias */
 #define STKB		BIAS
 #define	CCCR		%xcc
 #else
 #define	BASEREG		%g0
 #define LNGSZ		4
 #define LNGSHFT		2
 #define PTRSZ		4
 #define PTRSHFT		2
 #define POINTER		.word
 #define ULONG		.word
 /* Instructions to load/store pointers & long ints */
 #define LDLNG		ldsw
 #define LDULNG		lduw
 #define STLNG		stw
 #define STULNG		stw
 #define LDPTR		lduw
 #define LDPTRA		lduwa
 #define STPTR		stw
 #define STPTRA		stwa
 #define	CASPTR		casa
 #define STKB		0
 #define	CCCR		%icc
 #endif
 /*
  * GNU assembler does not understand `.empty' directive; Sun assembler
  * gripes about labels without it.  To allow cross-compilation using
  * the Sun assembler, and because .empty directives are useful
  * documentation, we use this trick.
  */
 #ifdef SUN_AS
 #define	EMPTY	.empty
 #else
 #define	EMPTY	/* .empty */
 #endif
 /* use as needed to align things on longword boundaries */
 #define	_ALIGN	.align 8
 #define ICACHE_ALIGN	.align	32
 /* Give this real authority: reset the machine */
 #define NOTREACHED	sir
 /*
  * This macro will clear out a cache line before an explicit
  * access to that location.  It's mostly used to make certain
  * loads bypassing the D$ do not get stale D$ data.
+ *
  * It uses a register with the address to clear and a temporary
  * which is destroyed.
  */
 #ifdef DCACHE_BUG
 #define DLFLUSH(a,t) \
 	andn	a, 0x1f, t; \
 	stxa	%g0, [ t ] ASI_DCACHE_TAG; \
 	membar	#Sync
 /* The following can be used if the pointer is 16-byte aligned */
 #define DLFLUSH2(t) \
 	stxa	%g0, [ t ] ASI_DCACHE_TAG; \
 	membar	#Sync
 #else
 #define DLFLUSH(a,t)
 #define DLFLUSH2(t)
 #endif
 /*
  * Combine 2 regs -- used to convert 64-bit ILP32
  * values to LP64.
  */
 #define	COMBINE(r1, r2, d)	\
 	sllx	r1, 32, d;	\
 	or	d, r2, d
 /*
  * Split 64-bit value in 1 reg into high and low halves.
  * Used for ILP32 return values.
  */
 #define	SPLIT(r0, r1)		\
 	srl	r0, 0, r1;	\
 	srlx	r0, 32, r0
 /*
  * A handy macro for maintaining instrumentation counters.
  * Note that this clobbers %o0, %o1 and %o2.  Normal usage is
  * something like:
  *	foointr:
  *		TRAP_SETUP(...)		! makes %o registers safe
  *		INCR(_C_LABEL(cnt)+V_FOO)	! count a foo
  */
 #define INCR(what) \
 	sethi	%hi(what), %o0; \
 	or	%o0, %lo(what), %o0; \
 :	\
 	lduw	[%o0], %o1; \
 	add	%o1, 1, %o2; \
 	casa	[%o0] ASI_P, %o1, %o2; \
 	cmp	%o1, %o2; \
 	bne,pn	%icc, 99b; \
 	 nop
 /*
  * A couple of handy macros to save and restore globals to/from
  * locals.  Since udivrem uses several globals, and it's called
  * from vsprintf, we need to do this before and after doing a printf.
  */
 #define GLOBTOLOC \
 	mov	%g1, %l1; \
 	mov	%g2, %l2; \
 	mov	%g3, %l3; \
 	mov	%g4, %l4; \
 	mov	%g5, %l5; \
 	mov	%g6, %l6; \
 	mov	%g7, %l7
 #define LOCTOGLOB \
 	mov	%l1, %g1; \
 	mov	%l2, %g2; \
 	mov	%l3, %g3; \
 	mov	%l4, %g4; \
 	mov	%l5, %g5; \
 	mov	%l6, %g6; \
 	mov	%l7, %g7
 /* Load strings address into register; NOTE: hidden local label 99 */
 #define LOAD_ASCIZ(reg, s)	\
 	set	99f, reg ;	\
 	.data ;			\
 :	.asciz	s ;		\
 	_ALIGN ;		\
 	.text
 /*
  * Handy stack conversion macros.
  * They correctly switch to requested stack type
  * regardless of the current stack.
  */
 #define TO_STACK64(size)					\
 	save	%sp, size, %sp;					\
 	add	%sp, -BIAS, %o0; /* Convert to 64-bits */	\
 	andcc	%sp, 1, %g0; /* 64-bit stack? */		\
 	movz	%icc, %o0, %sp
 #define TO_STACK32(size)					\
 	save	%sp, size, %sp;					\
 	add	%sp, +BIAS, %o0; /* Convert to 32-bits */	\
 	andcc	%sp, 1, %g0; /* 64-bit stack? */		\
 	movnz	%icc, %o0, %sp
 #ifdef _LP64
 #define	STACKFRAME(size)	TO_STACK64(size)
 #else
 #define	STACKFRAME(size)	TO_STACK32(size)
 #endif
 #ifdef USE_BLOCK_STORE_LOAD
 /*
  * The following routines allow fpu use in the kernel.
+ *
  * They allocate a stack frame and use all local regs.  Extra
  * local storage can be requested by setting the siz parameter,
  * and can be accessed at %sp+CC64FSZ.
  */
 #define ENABLE_FPU(siz)									     \
 	save	%sp, -(CC64FSZ), %sp;		/* Allocate a stack frame */		     \
 	sethi	%hi(FPLWP), %l1;							     \
 	add	%fp, STKB-FS_SIZE, %l0;		/* Allocate a fpstate */		     \
 	LDPTR	[%l1 + %lo(FPLWP)], %l2;	/* Load fplwp */			     \
 	andn	%l0, BLOCK_ALIGN, %l0;		/* Align it */				     \
 	clr	%l3;				/* NULL fpstate */			     \
 	brz,pt	%l2, 1f;			/* fplwp == NULL? */			     \
 	 add	%l0, -STKB-CC64FSZ-(siz), %sp;	/* Set proper %sp */			     \
 	LDPTR	[%l2 + L_FPSTATE], %l3;						    	     \
 	brz,pn	%l3, 1f;			/* Make sure we have an fpstate */	     \
 	 mov	%l3, %o0;								     \
 	call	_C_LABEL(savefpstate);		/* Save the old fpstate */		     \
 :											     \
 	 set	EINTSTACK-STKB, %l4;		/* Are we on intr stack? */		     \
 	cmp	%sp, %l4;								     \
 	bgu,pt	%xcc, 1f;								     \
 	 set	INTSTACK-STKB, %l4;							     \
 	cmp	%sp, %l4;								     \
 	blu	%xcc, 1f;								     \
 :											     \
 	 sethi	%hi(_C_LABEL(lwp0)), %l4;	/* Yes, use lpw0 */ 			     \
 	ba,pt	%xcc, 2f;			/* XXXX needs to change to CPUs idle proc */ \
 	 or	%l4, %lo(_C_LABEL(lwp0)), %l5;						     \
 :											     \
 	sethi	%hi(CURLWP), %l4;		/* Use curlwp */			     \
 	LDPTR	[%l4 + %lo(CURLWP)], %l5;						     \
 	brz,pn	%l5, 0b; nop;			/* If curlwp is NULL need to use lwp0 */     \
 :											     \
 	LDPTR	[%l5 + L_FPSTATE], %l6;		/* Save old fpstate */			     \
 	STPTR	%l0, [%l5 + L_FPSTATE];		/* Insert new fpstate */		     \
 	STPTR	%l5, [%l1 + %lo(FPLWP)];	/* Set new fplwp */			     \
 	wr	%g0, FPRS_FEF, %fprs		/* Enable FPU */
 /*
  * Weve saved our possible fpstate, now disable the fpu
  * and continue with life.
  */
 #ifdef DEBUG
 #define __CHECK_FPU				\
 	LDPTR	[%l5 + L_FPSTATE], %l7;		\
 	cmp	%l7, %l0;			\
 	tnz	1;
 #else
 #define	__CHECK_FPU
 #endif
 #define RESTORE_FPU							     \
 	__CHECK_FPU							     \
 	STPTR	%l2, [%l1 + %lo(FPLWP)];	/* Restore old fproc */	     \
 	wr	%g0, 0, %fprs;			/* Disable fpu */	     \
 	brz,pt	%l3, 1f;			/* Skip if no fpstate */     \
 	 STPTR	%l6, [%l5 + L_FPSTATE];		/* Restore old fpstate */    \
+									     \
 	mov	%l3, %o0;						     \
 	call	_C_LABEL(loadfpstate);		/* Re-load orig fpstate */   \
 : \
 	 membar	#Sync;				/* Finish all FP ops */
 #endif	/* USE_BLOCK_STORE_LOAD */
 	.data
 	.globl	_C_LABEL(data_start)
 _C_LABEL(data_start):					! Start of data segment
 #define DATA_START	_C_LABEL(data_start)
 #if 1
 /* XXX this shouldn't be needed... but kernel usually hangs without it */
 	.space	USPACE
 #endif
 #ifdef KGDB
 /*
  * Another item that must be aligned, easiest to put it here.
  */
 KGDB_STACK_SIZE = 2048
 	.globl	_C_LABEL(kgdb_stack)
 _C_LABEL(kgdb_stack):
 	.space	KGDB_STACK_SIZE		! hope this is enough
 #endif
 #ifdef NOTDEF_DEBUG
 /*
  * This stack is used when we detect kernel stack corruption.
  */
 	.space	USPACE
 	.align	16
 panicstack:
 #endif
 /*
  * romp is the prom entry pointer
  * romtba is the prom trap table base address
  */
 	.globl	romp
 romp:	POINTER	0
 	.globl	romtba
 romtba:	POINTER	0
 	_ALIGN
 	.text
 /*
  * The v9 trap frame is stored in the special trap registers.  The
  * register window is only modified on window overflow, underflow,
  * and clean window traps, where it points to the register window
  * needing service.  Traps have space for 8 instructions, except for
  * the window overflow, underflow, and clean window traps which are
  * 32 instructions long, large enough to in-line.
+ *
  * The spitfire CPU (Ultra I) has 4 different sets of global registers.
  * (blah blah...)
+ *
  * I used to generate these numbers by address arithmetic, but gas's
  * expression evaluator has about as much sense as your average slug
  * (oddly enough, the code looks about as slimy too).  Thus, all the
  * trap numbers are given as arguments to the trap macros.  This means
  * there is one line per trap.  Sigh.
+ *
  * Hardware interrupt vectors can be `linked'---the linkage is to regular
  * C code---or rewired to fast in-window handlers.  The latter are good
  * for unbuffered hardware like the Zilog serial chip and the AMD audio
  * chip, where many interrupts can be handled trivially with pseudo-DMA
  * or similar.  Only one `fast' interrupt can be used per level, however,
  * and direct and `fast' interrupts are incompatible.  Routines in intr.c
  * handle setting these, with optional paranoia.
  */
 /*
  *	TA8 -- trap align for 8 instruction traps
  *	TA32 -- trap align for 32 instruction traps
  */
 #define TA8	.align 32
 #define TA32	.align 128
 /*
  * v9 trap macros:
+ *
  *	We have a problem with v9 traps; we have no registers to put the
  *	trap type into.  But we do have a %tt register which already has
  *	that information.  Trap types in these macros are all dummys.
  */
 	/* regular vectored traps */
 #if KTR_COMPILE & KTR_TRAP
 #if 0
 #define TRACEWIN	wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate;\
 			sethi %hi(9f), %g1; ba,pt %icc,ktr_trap_gen; or %g1, %lo(9f), %g1; 9:
 #else
 #define TRACEWIN
 #endif
 #define TRACEFLT 	sethi %hi(1f), %g1; ba,pt %icc,ktr_trap_gen;\
 			or %g1, %lo(1f), %g1; 1:
 #define	VTRAP(type, label) \
 	sethi %hi(label), %g1; ba,pt %icc,ktr_trap_gen;\
 	or %g1, %lo(label), %g1; NOTREACHED; TA8
 #else
 #define TRACEWIN
 #define TRACEFLT
 #define	VTRAP(type, label) \
 	ba,a,pt	%icc,label; nop; NOTREACHED; TA8
 #endif
 	/* hardware interrupts (can be linked or made `fast') */
 #define	HARDINT4U(lev) \
 	VTRAP(lev, _C_LABEL(sparc_interrupt))
 	/* software interrupts (may not be made direct, sorry---but you
 	   should not be using them trivially anyway) */
 #define	SOFTINT4U(lev, bit) \
 	HARDINT4U(lev)
 	/* traps that just call trap() */
 #define	TRAP(type)	VTRAP(type, slowtrap)
 	/* architecturally undefined traps (cause panic) */
 #ifndef DEBUG
 #define	UTRAP(type)	sir; VTRAP(type, slowtrap)
 #else
 #define	UTRAP(type)	VTRAP(type, slowtrap)
 #endif
 	/* software undefined traps (may be replaced) */
 #define	STRAP(type)	VTRAP(type, slowtrap)
 /* breakpoint acts differently under kgdb */
 #ifdef KGDB
 #define	BPT		VTRAP(T_BREAKPOINT, bpt)
 #define	BPT_KGDB_EXEC	VTRAP(T_KGDB_EXEC, bpt)
 #else
 #define	BPT		TRAP(T_BREAKPOINT)
 #define	BPT_KGDB_EXEC	TRAP(T_KGDB_EXEC)
 #endif
 #define	SYSCALL		VTRAP(0x100, syscall_setup)
 #ifdef notyet
 #define	ZS_INTERRUPT	ba,a,pt %icc, zshard; nop; TA8
 #else
 #define	ZS_INTERRUPT4U	HARDINT4U(12)
 #endif
 /*
  * Macro to clear %tt so we don't get confused with old traps.
  */
 #ifdef DEBUG
 #define CLRTT	wrpr	%g0,0x1ff,%tt
 #else
 #define CLRTT
 #endif
 /*
  * Here are some oft repeated traps as macros.
  */
 	/* spill a 64-bit register window */
 #define SPILL64(label,as) \
 	TRACEWIN; \
 label:	\
 	wr	%g0, as, %asi; \
 	stxa	%l0, [%sp+BIAS+0x00]%asi; \
 	stxa	%l1, [%sp+BIAS+0x08]%asi; \
 	stxa	%l2, [%sp+BIAS+0x10]%asi; \
 	stxa	%l3, [%sp+BIAS+0x18]%asi; \
 	stxa	%l4, [%sp+BIAS+0x20]%asi; \
 	stxa	%l5, [%sp+BIAS+0x28]%asi; \
 	stxa	%l6, [%sp+BIAS+0x30]%asi; \
+	\
 	stxa	%l7, [%sp+BIAS+0x38]%asi; \
 	stxa	%i0, [%sp+BIAS+0x40]%asi; \
 	stxa	%i1, [%sp+BIAS+0x48]%asi; \
 	stxa	%i2, [%sp+BIAS+0x50]%asi; \
 	stxa	%i3, [%sp+BIAS+0x58]%asi; \
 	stxa	%i4, [%sp+BIAS+0x60]%asi; \
 	stxa	%i5, [%sp+BIAS+0x68]%asi; \
 	stxa	%i6, [%sp+BIAS+0x70]%asi; \
+	\
 	stxa	%i7, [%sp+BIAS+0x78]%asi; \
 	saved; \
 	CLRTT; \
 	retry; \
 	NOTREACHED; \
 	TA32
 	/* spill a 32-bit register window */
 #define SPILL32(label,as) \
 	TRACEWIN; \
 label:	\
 	wr	%g0, as, %asi; \
 	srl	%sp, 0, %sp; /* fixup 32-bit pointers */ \
 	stwa	%l0, [%sp+0x00]%asi; \
 	stwa	%l1, [%sp+0x04]%asi; \
 	stwa	%l2, [%sp+0x08]%asi; \
 	stwa	%l3, [%sp+0x0c]%asi; \
 	stwa	%l4, [%sp+0x10]%asi; \
 	stwa	%l5, [%sp+0x14]%asi; \
+	\
 	stwa	%l6, [%sp+0x18]%asi; \
 	stwa	%l7, [%sp+0x1c]%asi; \
 	stwa	%i0, [%sp+0x20]%asi; \
 	stwa	%i1, [%sp+0x24]%asi; \
 	stwa	%i2, [%sp+0x28]%asi; \
 	stwa	%i3, [%sp+0x2c]%asi; \
 	stwa	%i4, [%sp+0x30]%asi; \
 	stwa	%i5, [%sp+0x34]%asi; \
+	\
 	stwa	%i6, [%sp+0x38]%asi; \
 	stwa	%i7, [%sp+0x3c]%asi; \
 	saved; \
 	CLRTT; \
 	retry; \
 	NOTREACHED; \
 	TA32
 	/* Spill either 32-bit or 64-bit register window. */
 #define SPILLBOTH(label64,label32,as) \
 	TRACEWIN; \
 	andcc	%sp, 1, %g0; \
 	bnz,pt	%xcc, label64+4;	/* Is it a v9 or v8 stack? */ \
 	 wr	%g0, as, %asi; \
 	ba,pt	%xcc, label32+8; \
 	 srl	%sp, 0, %sp; /* fixup 32-bit pointers */ \
 	NOTREACHED; \
 	TA32
 	/* fill a 64-bit register window */
 #define FILL64(label,as) \
 	TRACEWIN; \
 label: \
 	wr	%g0, as, %asi; \
 	ldxa	[%sp+BIAS+0x00]%asi, %l0; \
 	ldxa	[%sp+BIAS+0x08]%asi, %l1; \
 	ldxa	[%sp+BIAS+0x10]%asi, %l2; \
 	ldxa	[%sp+BIAS+0x18]%asi, %l3; \
 	ldxa	[%sp+BIAS+0x20]%asi, %l4; \
 	ldxa	[%sp+BIAS+0x28]%asi, %l5; \
 	ldxa	[%sp+BIAS+0x30]%asi, %l6; \
+	\
 	ldxa	[%sp+BIAS+0x38]%asi, %l7; \
 	ldxa	[%sp+BIAS+0x40]%asi, %i0; \
 	ldxa	[%sp+BIAS+0x48]%asi, %i1; \
 	ldxa	[%sp+BIAS+0x50]%asi, %i2; \
 	ldxa	[%sp+BIAS+0x58]%asi, %i3; \
 	ldxa	[%sp+BIAS+0x60]%asi, %i4; \
 	ldxa	[%sp+BIAS+0x68]%asi, %i5; \
 	ldxa	[%sp+BIAS+0x70]%asi, %i6; \
+	\
 	ldxa	[%sp+BIAS+0x78]%asi, %i7; \
 	restored; \
 	CLRTT; \
 	retry; \
 	NOTREACHED; \
 	TA32
 	/* fill a 32-bit register window */
 #define FILL32(label,as) \
 	TRACEWIN; \
 label:	\
 	wr	%g0, as, %asi; \
 	srl	%sp, 0, %sp; /* fixup 32-bit pointers */ \
 	lda	[%sp+0x00]%asi, %l0; \
 	lda	[%sp+0x04]%asi, %l1; \
 	lda	[%sp+0x08]%asi, %l2; \
 	lda	[%sp+0x0c]%asi, %l3; \
 	lda	[%sp+0x10]%asi, %l4; \
 	lda	[%sp+0x14]%asi, %l5; \
+	\
 	lda	[%sp+0x18]%asi, %l6; \
 	lda	[%sp+0x1c]%asi, %l7; \
 	lda	[%sp+0x20]%asi, %i0; \
 	lda	[%sp+0x24]%asi, %i1; \
 	lda	[%sp+0x28]%asi, %i2; \
 	lda	[%sp+0x2c]%asi, %i3; \
 	lda	[%sp+0x30]%asi, %i4; \
 	lda	[%sp+0x34]%asi, %i5; \
+	\
 	lda	[%sp+0x38]%asi, %i6; \
 	lda	[%sp+0x3c]%asi, %i7; \
 	restored; \
 	CLRTT; \
 	retry; \
 	NOTREACHED; \
 	TA32
 	/* fill either 32-bit or 64-bit register window. */
 #define FILLBOTH(label64,label32,as) \
 	TRACEWIN; \
 	andcc	%sp, 1, %i0; \
 	bnz	(label64)+4; /* See if it's a v9 stack or v8 */ \
 	 wr	%g0, as, %asi; \
 	ba	(label32)+8; \
 	 srl	%sp, 0, %sp; /* fixup 32-bit pointers */ \
 	NOTREACHED; \
 	TA32
 	.globl	start, _C_LABEL(kernel_text)
 	_C_LABEL(kernel_text) = kernel_start		! for kvm_mkdb(8)
 kernel_start:
 	/* Traps from TL=0 -- traps from user mode */
 #ifdef __STDC__
 #define TABLE(name)	user_ ## name
 #else
 #define	TABLE(name)	user_/**/name
 #endif
 	.globl	_C_LABEL(trapbase)
 _C_LABEL(trapbase):
 	b dostart; nop; TA8	! 000 = reserved -- Use it to boot
 	/* We should not get the next 5 traps */
 	UTRAP(0x001)		! 001 = POR Reset -- ROM should get this
 	UTRAP(0x002)		! 002 = WDR -- ROM should get this
 	UTRAP(0x003)		! 003 = XIR -- ROM should get this
 	UTRAP(0x004)		! 004 = SIR -- ROM should get this
 	UTRAP(0x005)		! 005 = RED state exception
 	UTRAP(0x006); UTRAP(0x007)
 	VTRAP(T_INST_EXCEPT, textfault)	! 008 = instr. access exept
 	VTRAP(T_TEXTFAULT, textfault)	! 009 = instr access MMU miss
 	VTRAP(T_INST_ERROR, textfault)	! 00a = instr. access err
 	UTRAP(0x00b); UTRAP(0x00c); UTRAP(0x00d); UTRAP(0x00e); UTRAP(0x00f)
 	TRAP(T_ILLINST)			! 010 = illegal instruction
 	TRAP(T_PRIVINST)		! 011 = privileged instruction
 	UTRAP(0x012)			! 012 = unimplemented LDD
 	UTRAP(0x013)			! 013 = unimplemented STD
 	UTRAP(0x014); UTRAP(0x015); UTRAP(0x016); UTRAP(0x017); UTRAP(0x018)
 	UTRAP(0x019); UTRAP(0x01a); UTRAP(0x01b); UTRAP(0x01c); UTRAP(0x01d)
 	UTRAP(0x01e); UTRAP(0x01f)
 	TRAP(T_FPDISABLED)		! 020 = fp instr, but EF bit off in psr
 	TRAP(T_FP_IEEE_754)		! 021 = ieee 754 exception
 	TRAP(T_FP_OTHER)		! 022 = other fp exception
 	TRAP(T_TAGOF)			! 023 = tag overflow
 	TRACEWIN			! DEBUG -- 4 insns
 	rdpr %cleanwin, %o7		! 024-027 = clean window trap
 	inc %o7				!	This handler is in-lined and cannot fault
 #ifdef DEBUG
 	set	0xbadcafe, %l0		! DEBUG -- compiler should not rely on zero-ed registers.
 #else
 	clr	%l0
 #endif
 	wrpr %g0, %o7, %cleanwin	!       Nucleus (trap&IRQ) code does not need clean windows
 	mov %l0,%l1; mov %l0,%l2	!	Clear out %l0-%l8 and %o0-%o8 and inc %cleanwin and done
 	mov %l0,%l3; mov %l0,%l4
 #if 0
 #ifdef DIAGNOSTIC
 	!!
 	!! Check the sp redzone
 	!!
 	!! Since we can't spill the current window, we'll just keep
 	!! track of the frame pointer.  Problems occur when the routine
 	!! allocates and uses stack storage.
 	!!
 !	rdpr	%wstate, %l5	! User stack?
 !	cmp	%l5, WSTATE_KERN
 !	bne,pt	%icc, 7f
 	 sethi	%hi(CPCB), %l5
 	LDPTR	[%l5 + %lo(CPCB)], %l5	! If pcb < fp < pcb+sizeof(pcb)
 	inc	PCB_SIZE, %l5		! then we have a stack overflow
 	btst	%fp, 1			! 64-bit stack?
 	sub	%fp, %l5, %l7
 	bnz,a,pt	%icc, 1f
 	 inc	BIAS, %l7		! Remove BIAS
 :
 	cmp	%l7, PCB_SIZE
 	blu	%xcc, cleanwin_overflow
 #endif
 #endif
 	mov %l0, %l5
 	mov %l0, %l6; mov %l0, %l7; mov %l0, %o0; mov %l0, %o1
 	mov %l0, %o2; mov %l0, %o3; mov %l0, %o4; mov %l0, %o5;
 	mov %l0, %o6; mov %l0, %o7
 	CLRTT
 	retry; nop; NOTREACHED; TA32
 	TRAP(T_DIV0)			! 028 = divide by zero
 	UTRAP(0x029)			! 029 = internal processor error
 	UTRAP(0x02a); UTRAP(0x02b); UTRAP(0x02c); UTRAP(0x02d); UTRAP(0x02e); UTRAP(0x02f)
 	VTRAP(T_DATAFAULT, winfault)	! 030 = data fetch fault
 	UTRAP(0x031)			! 031 = data MMU miss -- no MMU
 	VTRAP(T_DATA_ERROR, winfault)	! 032 = data access error
 	VTRAP(T_DATA_PROT, winfault)	! 033 = data protection fault
 	TRAP(T_ALIGN)			! 034 = address alignment error -- we could fix it inline...
 	TRAP(T_LDDF_ALIGN)		! 035 = LDDF address alignment error -- we could fix it inline...
 	TRAP(T_STDF_ALIGN)		! 036 = STDF address alignment error -- we could fix it inline...
 	TRAP(T_PRIVACT)			! 037 = privileged action
 	UTRAP(0x038); UTRAP(0x039); UTRAP(0x03a); UTRAP(0x03b); UTRAP(0x03c);
 	UTRAP(0x03d); UTRAP(0x03e); UTRAP(0x03f);
 	VTRAP(T_ASYNC_ERROR, winfault)	! 040 = data fetch fault
 	SOFTINT4U(1, IE_L1)		! 041 = level 1 interrupt
 	HARDINT4U(2)			! 042 = level 2 interrupt
 	HARDINT4U(3)			! 043 = level 3 interrupt
 	SOFTINT4U(4, IE_L4)		! 044 = level 4 interrupt
 	HARDINT4U(5)			! 045 = level 5 interrupt
 	SOFTINT4U(6, IE_L6)		! 046 = level 6 interrupt
 	HARDINT4U(7)			! 047 = level 7 interrupt
 	HARDINT4U(8)			! 048 = level 8 interrupt
 	HARDINT4U(9)			! 049 = level 9 interrupt
 	HARDINT4U(10)			! 04a = level 10 interrupt
 	HARDINT4U(11)			! 04b = level 11 interrupt
 	ZS_INTERRUPT4U			! 04c = level 12 (zs) interrupt
 	HARDINT4U(13)			! 04d = level 13 interrupt
 	HARDINT4U(14)			! 04e = level 14 interrupt
 	HARDINT4U(15)			! 04f = nonmaskable interrupt
 	UTRAP(0x050); UTRAP(0x051); UTRAP(0x052); UTRAP(0x053); UTRAP(0x054); UTRAP(0x055)
 	UTRAP(0x056); UTRAP(0x057); UTRAP(0x058); UTRAP(0x059); UTRAP(0x05a); UTRAP(0x05b)
 	UTRAP(0x05c); UTRAP(0x05d); UTRAP(0x05e); UTRAP(0x05f)
 	VTRAP(0x060, interrupt_vector); ! 060 = interrupt vector
 	TRAP(T_PA_WATCHPT)		! 061 = physical address data watchpoint
 	TRAP(T_VA_WATCHPT)		! 062 = virtual address data watchpoint
 	UTRAP(T_ECCERR)			! We'll implement this one later
 ufast_IMMU_miss:			! 064 = fast instr access MMU miss
 	TRACEFLT			! DEBUG
 	ldxa	[%g0] ASI_IMMU_8KPTR, %g2 ! Load IMMU 8K TSB pointer
 #ifdef NO_TSB
 	ba,a	%icc, instr_miss
 #endif
 	ldxa	[%g0] ASI_IMMU, %g1	! Load IMMU tag target register
 	ldda	[%g2] ASI_NUCLEUS_QUAD_LDD, %g4	! Load TSB tag:data into %g4:%g5
 	brgez,pn %g5, instr_miss	! Entry invalid?  Punt
 	 cmp	%g1, %g4		! Compare TLB tags
 	bne,pn %xcc, instr_miss		! Got right tag?
 	 nop
 	CLRTT
 	stxa	%g5, [%g0] ASI_IMMU_DATA_IN ! Enter new mapping
 	retry				! Try new mapping
 :
 	sir
 	TA32
 ufast_DMMU_miss:			! 068 = fast data access MMU miss
 	TRACEFLT			! DEBUG
 	ldxa	[%g0] ASI_DMMU_8KPTR, %g2! Load DMMU 8K TSB pointer
 #ifdef NO_TSB
 	ba,a	%icc, data_miss
 #endif
 	ldxa	[%g0] ASI_DMMU, %g1	! Load DMMU tag target register
 	ldda	[%g2] ASI_NUCLEUS_QUAD_LDD, %g4	! Load TSB tag and data into %g4 and %g5
 	brgez,pn %g5, data_miss		! Entry invalid?  Punt
 	 cmp	%g1, %g4		! Compare TLB tags
 	bnz,pn	%xcc, data_miss		! Got right tag?
 	 nop
 	CLRTT
 #ifdef TRAPSTATS
 	sethi	%hi(_C_LABEL(udhit)), %g1
 	lduw	[%g1+%lo(_C_LABEL(udhit))], %g2
 	inc	%g2
 	stw	%g2, [%g1+%lo(_C_LABEL(udhit))]
 #endif
 	stxa	%g5, [%g0] ASI_DMMU_DATA_IN ! Enter new mapping
 	retry				! Try new mapping
 :
 	sir
 	TA32
 ufast_DMMU_protection:			! 06c = fast data access MMU protection
 	TRACEFLT			! DEBUG -- we're perilously close to 32 insns
 #ifdef TRAPSTATS
 	sethi	%hi(_C_LABEL(udprot)), %g1
 	lduw	[%g1+%lo(_C_LABEL(udprot))], %g2
 	inc	%g2
 	stw	%g2, [%g1+%lo(_C_LABEL(udprot))]
 #endif
 #ifdef HWREF
 	ba,a,pt	%xcc, dmmu_write_fault
 #else
 	ba,a,pt	%xcc, winfault
 #endif
 	nop
 	TA32
 	UTRAP(0x070)			! Implementation dependent traps
 	UTRAP(0x071); UTRAP(0x072); UTRAP(0x073); UTRAP(0x074); UTRAP(0x075); UTRAP(0x076)
 	UTRAP(0x077); UTRAP(0x078); UTRAP(0x079); UTRAP(0x07a); UTRAP(0x07b); UTRAP(0x07c)
 	UTRAP(0x07d); UTRAP(0x07e); UTRAP(0x07f)
 TABLE(uspill):
 	SPILL64(uspill8,ASI_AIUS)	! 0x080 spill_0_normal -- used to save user windows in user mode
 	SPILL32(uspill4,ASI_AIUS)	! 0x084 spill_1_normal
 	SPILLBOTH(uspill8,uspill4,ASI_AIUS)	 ! 0x088 spill_2_normal
 	UTRAP(0x08c); TA32		! 0x08c spill_3_normal
 TABLE(kspill):
 	SPILL64(kspill8,ASI_N)		! 0x090 spill_4_normal -- used to save supervisor windows
 	SPILL32(kspill4,ASI_N)		! 0x094 spill_5_normal
 	SPILLBOTH(kspill8,kspill4,ASI_N) ! 0x098 spill_6_normal
 	UTRAP(0x09c); TA32		! 0x09c spill_7_normal
 TABLE(uspillk):
 	SPILL64(uspillk8,ASI_AIUS)	! 0x0a0 spill_0_other -- used to save user windows in supervisor mode
 	SPILL32(uspillk4,ASI_AIUS)	! 0x0a4 spill_1_other
 	SPILLBOTH(uspillk8,uspillk4,ASI_AIUS) ! 0x0a8 spill_2_other
 	UTRAP(0x0ac); TA32		! 0x0ac spill_3_other
 	UTRAP(0x0b0); TA32		! 0x0b0 spill_4_other
 	UTRAP(0x0b4); TA32		! 0x0b4 spill_5_other
 	UTRAP(0x0b8); TA32		! 0x0b8 spill_6_other
 	UTRAP(0x0bc); TA32		! 0x0bc spill_7_other
 TABLE(ufill):
 	FILL64(ufill8,ASI_AIUS)		! 0x0c0 fill_0_normal -- used to fill windows when running user mode
 	FILL32(ufill4,ASI_AIUS)		! 0x0c4 fill_1_normal
 	FILLBOTH(ufill8,ufill4,ASI_AIUS) ! 0x0c8 fill_2_normal
 	UTRAP(0x0cc); TA32		! 0x0cc fill_3_normal
 TABLE(kfill):
 	FILL64(kfill8,ASI_N)		! 0x0d0 fill_4_normal -- used to fill windows when running supervisor mode
 	FILL32(kfill4,ASI_N)		! 0x0d4 fill_5_normal
 	FILLBOTH(kfill8,kfill4,ASI_N)	! 0x0d8 fill_6_normal
 	UTRAP(0x0dc); TA32		! 0x0dc fill_7_normal
 TABLE(ufillk):
 	FILL64(ufillk8,ASI_AIUS)	! 0x0e0 fill_0_other
 	FILL32(ufillk4,ASI_AIUS)	! 0x0e4 fill_1_other
 	FILLBOTH(ufillk8,ufillk4,ASI_AIUS) ! 0x0e8 fill_2_other
 	UTRAP(0x0ec); TA32		! 0x0ec fill_3_other
 	UTRAP(0x0f0); TA32		! 0x0f0 fill_4_other
 	UTRAP(0x0f4); TA32		! 0x0f4 fill_5_other
 	UTRAP(0x0f8); TA32		! 0x0f8 fill_6_other
 	UTRAP(0x0fc); TA32		! 0x0fc fill_7_other
 TABLE(syscall):
 	SYSCALL				! 0x100 = sun syscall
 	BPT				! 0x101 = pseudo breakpoint instruction
 	STRAP(0x102); STRAP(0x103); STRAP(0x104); STRAP(0x105); STRAP(0x106); STRAP(0x107)
 	SYSCALL				! 0x108 = svr4 syscall
 	SYSCALL				! 0x109 = bsd syscall
 	BPT_KGDB_EXEC			! 0x10a = enter kernel gdb on kernel startup
 	STRAP(0x10b); STRAP(0x10c); STRAP(0x10d); STRAP(0x10e); STRAP(0x10f);
 	STRAP(0x110); STRAP(0x111); STRAP(0x112); STRAP(0x113); STRAP(0x114); STRAP(0x115); STRAP(0x116); STRAP(0x117)
 	STRAP(0x118); STRAP(0x119); STRAP(0x11a); STRAP(0x11b); STRAP(0x11c); STRAP(0x11d); STRAP(0x11e); STRAP(0x11f)
 	STRAP(0x120); STRAP(0x121); STRAP(0x122); STRAP(0x123); STRAP(0x124); STRAP(0x125); STRAP(0x126); STRAP(0x127)
 	STRAP(0x128); STRAP(0x129); STRAP(0x12a); STRAP(0x12b); STRAP(0x12c); STRAP(0x12d); STRAP(0x12e); STRAP(0x12f)
 	STRAP(0x130); STRAP(0x131); STRAP(0x132); STRAP(0x133); STRAP(0x134); STRAP(0x135); STRAP(0x136); STRAP(0x137)
 	STRAP(0x138); STRAP(0x139); STRAP(0x13a); STRAP(0x13b); STRAP(0x13c); STRAP(0x13d); STRAP(0x13e); STRAP(0x13f)
 	SYSCALL				! 0x140 SVID syscall (Solaris 2.7)
 	SYSCALL				! 0x141 SPARC International syscall
 	SYSCALL				! 0x142	OS Vendor syscall
 	SYSCALL				! 0x143 HW OEM syscall
 	STRAP(0x144); STRAP(0x145); STRAP(0x146); STRAP(0x147)
 	STRAP(0x148); STRAP(0x149); STRAP(0x14a); STRAP(0x14b); STRAP(0x14c); STRAP(0x14d); STRAP(0x14e); STRAP(0x14f)
 	STRAP(0x150); STRAP(0x151); STRAP(0x152); STRAP(0x153); STRAP(0x154); STRAP(0x155); STRAP(0x156); STRAP(0x157)
 	STRAP(0x158); STRAP(0x159); STRAP(0x15a); STRAP(0x15b); STRAP(0x15c); STRAP(0x15d); STRAP(0x15e); STRAP(0x15f)
 	STRAP(0x160); STRAP(0x161); STRAP(0x162); STRAP(0x163); STRAP(0x164); STRAP(0x165); STRAP(0x166); STRAP(0x167)
 	STRAP(0x168); STRAP(0x169); STRAP(0x16a); STRAP(0x16b); STRAP(0x16c); STRAP(0x16d); STRAP(0x16e); STRAP(0x16f)
 	STRAP(0x170); STRAP(0x171); STRAP(0x172); STRAP(0x173); STRAP(0x174); STRAP(0x175); STRAP(0x176); STRAP(0x177)
 	STRAP(0x178); STRAP(0x179); STRAP(0x17a); STRAP(0x17b); STRAP(0x17c); STRAP(0x17d); STRAP(0x17e); STRAP(0x17f)
 	! Traps beyond 0x17f are reserved
 	UTRAP(0x180); UTRAP(0x181); UTRAP(0x182); UTRAP(0x183); UTRAP(0x184); UTRAP(0x185); UTRAP(0x186); UTRAP(0x187)
 	UTRAP(0x188); UTRAP(0x189); UTRAP(0x18a); UTRAP(0x18b); UTRAP(0x18c); UTRAP(0x18d); UTRAP(0x18e); UTRAP(0x18f)
 	UTRAP(0x190); UTRAP(0x191); UTRAP(0x192); UTRAP(0x193); UTRAP(0x194); UTRAP(0x195); UTRAP(0x196); UTRAP(0x197)
 	UTRAP(0x198); UTRAP(0x199); UTRAP(0x19a); UTRAP(0x19b); UTRAP(0x19c); UTRAP(0x19d); UTRAP(0x19e); UTRAP(0x19f)
 	UTRAP(0x1a0); UTRAP(0x1a1); UTRAP(0x1a2); UTRAP(0x1a3); UTRAP(0x1a4); UTRAP(0x1a5); UTRAP(0x1a6); UTRAP(0x1a7)
 	UTRAP(0x1a8); UTRAP(0x1a9); UTRAP(0x1aa); UTRAP(0x1ab); UTRAP(0x1ac); UTRAP(0x1ad); UTRAP(0x1ae); UTRAP(0x1af)
 	UTRAP(0x1b0); UTRAP(0x1b1); UTRAP(0x1b2); UTRAP(0x1b3); UTRAP(0x1b4); UTRAP(0x1b5); UTRAP(0x1b6); UTRAP(0x1b7)
 	UTRAP(0x1b8); UTRAP(0x1b9); UTRAP(0x1ba); UTRAP(0x1bb); UTRAP(0x1bc); UTRAP(0x1bd); UTRAP(0x1be); UTRAP(0x1bf)
 	UTRAP(0x1c0); UTRAP(0x1c1); UTRAP(0x1c2); UTRAP(0x1c3); UTRAP(0x1c4); UTRAP(0x1c5); UTRAP(0x1c6); UTRAP(0x1c7)
 	UTRAP(0x1c8); UTRAP(0x1c9); UTRAP(0x1ca); UTRAP(0x1cb); UTRAP(0x1cc); UTRAP(0x1cd); UTRAP(0x1ce); UTRAP(0x1cf)
 	UTRAP(0x1d0); UTRAP(0x1d1); UTRAP(0x1d2); UTRAP(0x1d3); UTRAP(0x1d4); UTRAP(0x1d5); UTRAP(0x1d6); UTRAP(0x1d7)
 	UTRAP(0x1d8); UTRAP(0x1d9); UTRAP(0x1da); UTRAP(0x1db); UTRAP(0x1dc); UTRAP(0x1dd); UTRAP(0x1de); UTRAP(0x1df)
 	UTRAP(0x1e0); UTRAP(0x1e1); UTRAP(0x1e2); UTRAP(0x1e3); UTRAP(0x1e4); UTRAP(0x1e5); UTRAP(0x1e6); UTRAP(0x1e7)
 	UTRAP(0x1e8); UTRAP(0x1e9); UTRAP(0x1ea); UTRAP(0x1eb); UTRAP(0x1ec); UTRAP(0x1ed); UTRAP(0x1ee); UTRAP(0x1ef)
 	UTRAP(0x1f0); UTRAP(0x1f1); UTRAP(0x1f2); UTRAP(0x1f3); UTRAP(0x1f4); UTRAP(0x1f5); UTRAP(0x1f6); UTRAP(0x1f7)
 	UTRAP(0x1f8); UTRAP(0x1f9); UTRAP(0x1fa); UTRAP(0x1fb); UTRAP(0x1fc); UTRAP(0x1fd); UTRAP(0x1fe); UTRAP(0x1ff)
 	/* Traps from TL>0 -- traps from supervisor mode */
 #undef TABLE
 #ifdef __STDC__
 #define	TABLE(name)	nucleus_ ## name
 #else
 #define	TABLE(name)	nucleus_/**/name
 #endif
 trapbase_priv:
 	UTRAP(0x000)			! 000 = reserved -- Use it to boot
 	/* We should not get the next 5 traps */
 	UTRAP(0x001)			! 001 = POR Reset -- ROM should get this
 	UTRAP(0x002)			! 002 = WDR Watchdog -- ROM should get this
 	UTRAP(0x003)			! 003 = XIR -- ROM should get this
 	UTRAP(0x004)			! 004 = SIR -- ROM should get this
 	UTRAP(0x005)			! 005 = RED state exception
 	UTRAP(0x006); UTRAP(0x007)
 ktextfault:
 	VTRAP(T_INST_EXCEPT, textfault)	! 008 = instr. access exept
 	VTRAP(T_TEXTFAULT, textfault)	! 009 = instr access MMU miss -- no MMU
 	VTRAP(T_INST_ERROR, textfault)	! 00a = instr. access err
 	UTRAP(0x00b); UTRAP(0x00c); UTRAP(0x00d); UTRAP(0x00e); UTRAP(0x00f)
 	TRAP(T_ILLINST)			! 010 = illegal instruction
 	TRAP(T_PRIVINST)		! 011 = privileged instruction
 	UTRAP(0x012)			! 012 = unimplemented LDD
 	UTRAP(0x013)			! 013 = unimplemented STD
 	UTRAP(0x014); UTRAP(0x015); UTRAP(0x016); UTRAP(0x017); UTRAP(0x018)
 	UTRAP(0x019); UTRAP(0x01a); UTRAP(0x01b); UTRAP(0x01c); UTRAP(0x01d)
 	UTRAP(0x01e); UTRAP(0x01f)
 	TRAP(T_FPDISABLED)		! 020 = fp instr, but EF bit off in psr
 	TRAP(T_FP_IEEE_754)		! 021 = ieee 754 exception
 	TRAP(T_FP_OTHER)		! 022 = other fp exception
 	TRAP(T_TAGOF)			! 023 = tag overflow
 	TRACEWIN			! DEBUG
 	clr	%l0
 #ifdef DEBUG
 	set	0xbadbeef, %l0		! DEBUG
 #endif
 	mov %l0, %l1; mov %l0, %l2	! 024-027 = clean window trap
 	rdpr %cleanwin, %o7		!	This handler is in-lined and cannot fault
 	inc %o7; mov %l0, %l3		!       Nucleus (trap&IRQ) code does not need clean windows
 	wrpr %g0, %o7, %cleanwin	!	Clear out %l0-%l8 and %o0-%o8 and inc %cleanwin and done
 #ifdef NOT_DEBUG
 	!!
 	!! Check the sp redzone
 	!!
 	rdpr	%wstate, t1
 	cmp	t1, WSTATE_KERN
 	bne,pt	icc, 7f
 	 sethi	%hi(_C_LABEL(redzone)), t1
 	ldx	[t1 + %lo(_C_LABEL(redzone))], t2
 	cmp	%sp, t2			! if sp >= t2, not in red zone
 	blu	panic_red		! and can continue normally
 :
 #endif
 	mov %l0, %l4; mov %l0, %l5; mov %l0, %l6; mov %l0, %l7
 	mov %l0, %o0; mov %l0, %o1; mov %l0, %o2; mov %l0, %o3
 	mov %l0, %o4; mov %l0, %o5; mov %l0, %o6; mov %l0, %o7
 	CLRTT
 	retry; nop; TA32
 	TRAP(T_DIV0)			! 028 = divide by zero
 	UTRAP(0x029)			! 029 = internal processor error
 	UTRAP(0x02a); UTRAP(0x02b); UTRAP(0x02c); UTRAP(0x02d); UTRAP(0x02e); UTRAP(0x02f)
 kdatafault:
 	VTRAP(T_DATAFAULT, winfault)	! 030 = data fetch fault
 	UTRAP(0x031)			! 031 = data MMU miss -- no MMU
 	VTRAP(T_DATA_ERROR, winfault)	! 032 = data fetch fault
 @@ -2821,3619 +2821,3651 @@ instr_miss:
 	nop
 	sll	%g5, 3, %g5
 	add	%g6, %g4, %g4
 	ldxa	[%g4] ASI_PHYS_CACHED, %g4
 	srlx	%g3, PTSHIFT, %g6			! Convert to ptab offset
 	and	%g6, PTMASK, %g6
 	add	%g5, %g4, %g5
 	brz,pn	%g4, textfault				! NULL entry? check somewhere else
 	 nop
 	ldxa	[%g5] ASI_PHYS_CACHED, %g4
 	sll	%g6, 3, %g6
 	brz,pn	%g4, textfault				! NULL entry? check somewhere else
 	 add	%g6, %g4, %g6
 :
 	ldxa	[%g6] ASI_PHYS_CACHED, %g4
 	brgez,pn %g4, textfault
 	 nop
 	/* Check if it's an executable mapping. */
 	andcc	%g4, TTE_EXEC, %g0
 	bz,pn	%xcc, textfault
 	 nop
 	or	%g4, TTE_ACCESS, %g7			! Update accessed bit
 	btst	TTE_ACCESS, %g4				! Need to update access git?
 	bne,pt	%xcc, 1f
 	 nop
 	casxa	[%g6] ASI_PHYS_CACHED, %g4, %g7		!  and store it
 	cmp	%g4, %g7
 	bne,pn	%xcc, 1b
 	 or	%g4, TTE_ACCESS, %g4			! Update accessed bit
 :
 	stx	%g1, [%g2]				! Update TSB entry tag
 	stx	%g4, [%g2+8]				! Update TSB entry data
 #ifdef DEBUG
 	set	DATA_START, %g6	! debug
 	stx	%g3, [%g6+8]	! debug
 	set	0xaa, %g3	! debug
 	stx	%g4, [%g6]	! debug -- what we tried to enter in TLB
 	stb	%g3, [%g6+0x20]	! debug
 #endif
 	stxa	%g4, [%g0] ASI_IMMU_DATA_IN		! Enter new mapping
 	membar	#Sync
 	CLRTT
 	retry
 	NOTREACHED
 	!!
 	!!  Check our prom mappings -- temporary
 	!!
 /*
  * Each memory text access fault, from user or kernel mode,
  * comes here.
+ *
  * We will assume that %pil is not lost so we won't bother to save it
  * unless we're in an interrupt handler.
+ *
  * On entry:
  *	We are on one of the alternate set of globals
  *	%g1 = MMU tag target
  *	%g2 = %tl
  *	%g3 = %tl - 1
+ *
  * On return:
+ *
  */
 textfault:
 	wrpr	%g0, PSTATE_KERN|PSTATE_AG, %pstate	! We need to save volatile stuff to AG regs
 #ifdef TRAPS_USE_IG
 	wrpr	%g0, PSTATE_KERN|PSTATE_IG, %pstate	! We need to save volatile stuff to AG regs
 #endif
 	wr	%g0, ASI_IMMU, %asi
 	ldxa	[%g0 + TLB_TAG_ACCESS] %asi, %g1	! Get fault address from tag access register
 	ldxa	[SFSR] %asi, %g3			! get sync fault status register
 	membar	#LoadStore
 	stxa	%g0, [SFSR] %asi			! Clear out old info
 	TRAP_SETUP(-CC64FSZ-TF_SIZE)
 	INCR(_C_LABEL(uvmexp)+V_FAULTS)			! cnt.v_faults++ (clobbers %o0,%o1,%o2)
 	mov	%g3, %o3
 	wrpr	%g0, PSTATE_KERN, %pstate		! Switch to normal globals
 	ldxa	[%g0] ASI_AFSR, %o4			! get async fault status
 	ldxa	[%g0] ASI_AFAR, %o5			! get async fault address
 	mov	-1, %o0
 	stxa	%o0, [%g0] ASI_AFSR			! Clear this out
 	stx	%g1, [%sp + CC64FSZ + STKB + TF_G + (1*8)]	! save g1
 	stx	%g2, [%sp + CC64FSZ + STKB + TF_G + (2*8)]	! save g2
 	stx	%g3, [%sp + CC64FSZ + STKB + TF_G + (3*8)]	! (sneak g3 in here)
 	rdpr	%tt, %o1					! Find out what caused this trap
 	stx	%g4, [%sp + CC64FSZ + STKB + TF_G + (4*8)]	! sneak in g4
 	rdpr	%tstate, %g1
 	stx	%g5, [%sp + CC64FSZ + STKB + TF_G + (5*8)]	! sneak in g5
 	rdpr	%tpc, %o2					! sync virt addr; must be read first
 	stx	%g6, [%sp + CC64FSZ + STKB + TF_G + (6*8)]	! sneak in g6
 	rdpr	%tnpc, %g3
 	stx	%g7, [%sp + CC64FSZ + STKB + TF_G + (7*8)]	! sneak in g7
 	rd	%y, %g5						! save y
 	/* Finish stackframe, call C trap handler */
 	stx	%g1, [%sp + CC64FSZ + STKB + TF_TSTATE]		! set tf.tf_psr, tf.tf_pc
 	sth	%o1, [%sp + CC64FSZ + STKB + TF_TT]		! debug
 	stx	%o2, [%sp + CC64FSZ + STKB + TF_PC]
 	stx	%g3, [%sp + CC64FSZ + STKB + TF_NPC]		! set tf.tf_npc
 	rdpr	%pil, %g4
 	stb	%g4, [%sp + CC64FSZ + STKB + TF_PIL]
 	stb	%g4, [%sp + CC64FSZ + STKB + TF_OLDPIL]
 	rdpr	%tl, %g7
 	dec	%g7
 	movrlz	%g7, %g0, %g7
 	CHKPT(%g1,%g3,0x22)
 	wrpr	%g0, %g7, %tl		! Revert to kernel mode
 	wr	%g0, ASI_PRIMARY_NOFAULT, %asi		! Restore default ASI
 	flushw						! Get rid of any user windows so we don't deadlock
 	!! In the EMBEDANY memory model %g4 points to the start of the data segment.
 	!! In our case we need to clear it before calling any C-code
 	clr	%g4
 	/* Use trap type to see what handler to call */
 	cmp	%o1, T_INST_ERROR
 	be,pn	%xcc, text_error
 	 st	%g5, [%sp + CC64FSZ + STKB + TF_Y]		! set tf.tf_y
 	wrpr	%g0, PSTATE_INTR, %pstate	! reenable interrupts
 	call	_C_LABEL(text_access_fault)	! mem_access_fault(&tf, type, pc, sfsr)
 	 add	%sp, CC64FSZ + STKB, %o0	! (argument: &tf)
 text_recover:
 	CHKPT(%o1,%o2,2)
 	wrpr	%g0, PSTATE_KERN, %pstate	! disable interrupts
 	b	return_from_trap		! go return
 	 ldx	[%sp + CC64FSZ + STKB + TF_TSTATE], %g1	! Load this for return_from_trap
 	NOTREACHED
 text_error:
 	wrpr	%g0, PSTATE_INTR, %pstate	! reenable interrupts
 	call	_C_LABEL(text_access_error)	! mem_access_fault(&tfm type, sfva [pc], sfsr,
 						!		afva, afsr);
 	 add	%sp, CC64FSZ + STKB, %o0	! (argument: &tf)
 	ba	text_recover
 	 nop
 	NOTREACHED
 /*
  * We're here because we took an alignment fault in NUCLEUS context.
  * This could be a kernel bug or it could be due to saving a user
  * window to an invalid stack pointer.
+ *
  * If the latter is the case, we could try to emulate unaligned accesses,
  * but we really don't know where to store the registers since we can't
  * determine if there's a stack bias.  Or we could store all the regs
  * into the PCB and punt, until the user program uses up all the CPU's
  * register windows and we run out of places to store them.  So for
  * simplicity we'll just blow them away and enter the trap code which
  * will generate a bus error.  Debugging the problem will be a bit
  * complicated since lots of register windows will be lost, but what
  * can we do?
  */
 checkalign:
 	rdpr	%tl, %g2
 	subcc	%g2, 1, %g1
 	bneg,pn	%icc, slowtrap		! Huh?
 	 sethi	%hi(CPCB), %g6		! get current pcb
 	wrpr	%g1, 0, %tl
 	rdpr	%tt, %g7
 	rdpr	%tstate, %g4
 	andn	%g7, 0x3f, %g5
 	cmp	%g5, 0x080		!   window spill traps are all 0b 0000 10xx xxxx
 	bne,a,pn	%icc, slowtrap
 	 wrpr	%g1, 0, %tl		! Revert TL  XXX wrpr in a delay slot...
 #ifdef DEBUG
 	cmp	%g7, 0x34		! If we took a datafault just before this trap
 	bne,pt	%icc, checkalignspill	! our stack's probably bad so we need to switch somewhere else
 	 nop
 	!!
 	!! Double data fault -- bad stack?
 	!!
 	wrpr	%g2, %tl		! Restore trap level.
 	sir				! Just issue a reset and don't try to recover.
 	mov	%fp, %l6		! Save the frame pointer
 	set	EINTSTACK+USPACE+CC64FSZ-STKB, %fp ! Set the frame pointer to the middle of the idle stack
 	add	%fp, -CC64FSZ, %sp	! Create a stackframe
 	wrpr	%g0, 15, %pil		! Disable interrupts, too
 	wrpr	%g0, %g0, %canrestore	! Our stack is hozed and our PCB
 	wrpr	%g0, 7, %cansave	!  probably is too, so blow away
 	ba	slowtrap		!  all our register windows.
 	 wrpr	%g0, 0x101, %tt
 #endif
 checkalignspill:
 	/*
          * %g1 -- current tl
 	 * %g2 -- original tl
 	 * %g4 -- tstate
          * %g7 -- tt
 	 */
 	and	%g4, CWP, %g5
 	wrpr	%g5, %cwp		! Go back to the original register win
 	/*
 	 * Remember:
+	 *
 	 * %otherwin = 0
 	 * %cansave = NWINDOWS - 2 - %canrestore
 	 */
 	rdpr	%otherwin, %g6
 	rdpr	%canrestore, %g3
 	rdpr	%ver, %g5
 	sub	%g3, %g6, %g3		! Calculate %canrestore - %g7
 	and	%g5, CWP, %g5		! NWINDOWS-1
 	movrlz	%g3, %g0, %g3		! Clamp at zero
 	wrpr	%g0, 0, %otherwin
 	wrpr	%g3, 0, %canrestore	! This is the new canrestore
 	dec	%g5			! NWINDOWS-2
 	wrpr	%g5, 0, %cleanwin	! Set cleanwin to max, since we're in-kernel
 	sub	%g5, %g3, %g5		! NWINDOWS-2-%canrestore
 	wrpr	%g5, 0, %cansave
 	wrpr	%g0, T_ALIGN, %tt	! This was an alignment fault
 	/*
 	 * Now we need to determine if this was a userland store or not.
 	 * Userland stores occur in anything other than the kernel spill
 	 * handlers (trap type 09x).
 	 */
 	and	%g7, 0xff0, %g5
 	cmp	%g5, 0x90
 	bz,pn	%icc, slowtrap
 	 nop
 	bclr	TSTATE_PRIV, %g4
 	wrpr	%g4, 0, %tstate
 	ba,a,pt	%icc, slowtrap
 	 nop
 /*
  * slowtrap() builds a trap frame and calls trap().
  * This is called `slowtrap' because it *is*....
  * We have to build a full frame for ptrace(), for instance.
+ *
  * Registers:
+ *
  */
 slowtrap:
 #ifdef TRAPS_USE_IG
 	wrpr	%g0, PSTATE_KERN|PSTATE_IG, %pstate	! DEBUG
 #endif
 #ifdef DIAGNOSTIC
 	/* Make sure kernel stack is aligned */
 	btst	0x03, %sp		! 32-bit stack OK?
 	 and	%sp, 0x07, %g4		! 64-bit stack OK?
 	bz,pt	%icc, 1f
 	cmp	%g4, 0x1		! Must end in 0b001
 	be,pt	%icc, 1f
 	 rdpr	%wstate, %g7
 	cmp	%g7, WSTATE_KERN
 	bnz,pt	%icc, 1f		! User stack -- we'll blow it away
 	 nop
 	sethi	%hi(PANICSTACK), %sp
 	LDPTR	[%sp + %lo(PANICSTACK)], %sp
 	add	%sp, -CC64FSZ-STKB, %sp
 :
 #endif
 	rdpr	%tt, %g4
 	rdpr	%tstate, %g1
 	rdpr	%tpc, %g2
 	rdpr	%tnpc, %g3
 	TRAP_SETUP(-CC64FSZ-TF_SIZE)
 Lslowtrap_reenter:
 	stx	%g1, [%sp + CC64FSZ + STKB + TF_TSTATE]
 	mov	%g4, %o1		! (type)
 	stx	%g2, [%sp + CC64FSZ + STKB + TF_PC]
 	rd	%y, %g5
 	stx	%g3, [%sp + CC64FSZ + STKB + TF_NPC]
 	mov	%g1, %o3		! (pstate)
 	st	%g5, [%sp + CC64FSZ + STKB + TF_Y]
 	mov	%g2, %o2		! (pc)
 	sth	%o1, [%sp + CC64FSZ + STKB + TF_TT]! debug
 	wrpr	%g0, PSTATE_KERN, %pstate		! Get back to normal globals
 	stx	%g1, [%sp + CC64FSZ + STKB + TF_G + (1*8)]
 	stx	%g2, [%sp + CC64FSZ + STKB + TF_G + (2*8)]
 	add	%sp, CC64FSZ + STKB, %o0		! (&tf)
 	stx	%g3, [%sp + CC64FSZ + STKB + TF_G + (3*8)]
 	stx	%g4, [%sp + CC64FSZ + STKB + TF_G + (4*8)]
 	stx	%g5, [%sp + CC64FSZ + STKB + TF_G + (5*8)]
 	rdpr	%pil, %g5
 	stx	%g6, [%sp + CC64FSZ + STKB + TF_G + (6*8)]
 	stx	%g7, [%sp + CC64FSZ + STKB + TF_G + (7*8)]
 	stb	%g5, [%sp + CC64FSZ + STKB + TF_PIL]
 	stb	%g5, [%sp + CC64FSZ + STKB + TF_OLDPIL]
 	/*
 	 * Phew, ready to enable traps and call C code.
 	 */
 	rdpr	%tl, %g1
 	dec	%g1
 	movrlz	%g1, %g0, %g1
 	CHKPT(%g2,%g3,0x24)
 	wrpr	%g0, %g1, %tl		! Revert to kernel mode
 	!! In the EMBEDANY memory model %g4 points to the start of the data segment.
 	!! In our case we need to clear it before calling any C-code
 	clr	%g4
 	wr	%g0, ASI_PRIMARY_NOFAULT, %asi		! Restore default ASI
 	wrpr	%g0, PSTATE_INTR, %pstate	! traps on again
 	call	_C_LABEL(trap)			! trap(tf, type, pc, pstate)
 	 nop
 	CHKPT(%o1,%o2,3)
 	ba,a,pt	%icc, return_from_trap
 	 nop
 	NOTREACHED
 #if 1
 /*
  * This code is no longer needed.
  */
 /*
  * Do a `software' trap by re-entering the trap code, possibly first
  * switching from interrupt stack to kernel stack.  This is used for
  * scheduling and signal ASTs (which generally occur from softclock or
  * tty or net interrupts).
+ *
  * We enter with the trap type in %g1.  All we have to do is jump to
  * Lslowtrap_reenter above, but maybe after switching stacks....
+ *
  * We should be running alternate globals.  The normal globals and
  * out registers were just loaded from the old trap frame.
+ *
  *	Input Params:
  *	%g1 = tstate
  *	%g2 = tpc
  *	%g3 = tnpc
  *	%g4 = tt == T_AST
  */
 softtrap:
 	sethi	%hi(EINTSTACK-STKB), %g5
 	sethi	%hi(EINTSTACK-INTSTACK), %g7
 	or	%g5, %lo(EINTSTACK-STKB), %g5
 	dec	%g7
 	sub	%g5, %sp, %g5
 	sethi	%hi(CPCB), %g6
 	andncc	%g5, %g7, %g0
 	bnz,pt	%xcc, Lslowtrap_reenter
 	 LDPTR	[%g6 + %lo(CPCB)], %g7
 	set	USPACE-CC64FSZ-TF_SIZE-STKB, %g5
 	add	%g7, %g5, %g6
 	SET_SP_REDZONE(%g7, %g5)
 #ifdef DEBUG
 	stx	%g1, [%g6 + CC64FSZ + STKB + TF_FAULT]		! Generate a new trapframe
 #endif
 	stx	%i0, [%g6 + CC64FSZ + STKB + TF_O + (0*8)]	!	but don't bother with
 	stx	%i1, [%g6 + CC64FSZ + STKB + TF_O + (1*8)]	!	locals and ins
 	stx	%i2, [%g6 + CC64FSZ + STKB + TF_O + (2*8)]
 	stx	%i3, [%g6 + CC64FSZ + STKB + TF_O + (3*8)]
 	stx	%i4, [%g6 + CC64FSZ + STKB + TF_O + (4*8)]
 	stx	%i5, [%g6 + CC64FSZ + STKB + TF_O + (5*8)]
 	stx	%i6, [%g6 + CC64FSZ + STKB + TF_O + (6*8)]
 	stx	%i7, [%g6 + CC64FSZ + STKB + TF_O + (7*8)]
 #ifdef DEBUG
 	ldx	[%sp + CC64FSZ + STKB + TF_I + (0*8)], %l0	! Copy over the rest of the regs
 	ldx	[%sp + CC64FSZ + STKB + TF_I + (1*8)], %l1	! But just dirty the locals
 	ldx	[%sp + CC64FSZ + STKB + TF_I + (2*8)], %l2
 	ldx	[%sp + CC64FSZ + STKB + TF_I + (3*8)], %l3
 	ldx	[%sp + CC64FSZ + STKB + TF_I + (4*8)], %l4
 	ldx	[%sp + CC64FSZ + STKB + TF_I + (5*8)], %l5
 	ldx	[%sp + CC64FSZ + STKB + TF_I + (6*8)], %l6
 	ldx	[%sp + CC64FSZ + STKB + TF_I + (7*8)], %l7
 	stx	%l0, [%g6 + CC64FSZ + STKB + TF_I + (0*8)]
 	stx	%l1, [%g6 + CC64FSZ + STKB + TF_I + (1*8)]
 	stx	%l2, [%g6 + CC64FSZ + STKB + TF_I + (2*8)]
 	stx	%l3, [%g6 + CC64FSZ + STKB + TF_I + (3*8)]
 	stx	%l4, [%g6 + CC64FSZ + STKB + TF_I + (4*8)]
 	stx	%l5, [%g6 + CC64FSZ + STKB + TF_I + (5*8)]
 	stx	%l6, [%g6 + CC64FSZ + STKB + TF_I + (6*8)]
 	stx	%l7, [%g6 + CC64FSZ + STKB + TF_I + (7*8)]
 	ldx	[%sp + CC64FSZ + STKB + TF_L + (0*8)], %l0
 	ldx	[%sp + CC64FSZ + STKB + TF_L + (1*8)], %l1
 	ldx	[%sp + CC64FSZ + STKB + TF_L + (2*8)], %l2
 	ldx	[%sp + CC64FSZ + STKB + TF_L + (3*8)], %l3
 	ldx	[%sp + CC64FSZ + STKB + TF_L + (4*8)], %l4
 	ldx	[%sp + CC64FSZ + STKB + TF_L + (5*8)], %l5
 	ldx	[%sp + CC64FSZ + STKB + TF_L + (6*8)], %l6
 	ldx	[%sp + CC64FSZ + STKB + TF_L + (7*8)], %l7
 	stx	%l0, [%g6 + CC64FSZ + STKB + TF_L + (0*8)]
 	stx	%l1, [%g6 + CC64FSZ + STKB + TF_L + (1*8)]
 	stx	%l2, [%g6 + CC64FSZ + STKB + TF_L + (2*8)]
 	stx	%l3, [%g6 + CC64FSZ + STKB + TF_L + (3*8)]
 	stx	%l4, [%g6 + CC64FSZ + STKB + TF_L + (4*8)]
 	stx	%l5, [%g6 + CC64FSZ + STKB + TF_L + (5*8)]
 	stx	%l6, [%g6 + CC64FSZ + STKB + TF_L + (6*8)]
 	stx	%l7, [%g6 + CC64FSZ + STKB + TF_L + (7*8)]
 #endif
 	ba,pt	%xcc, Lslowtrap_reenter
 	 mov	%g6, %sp
 #endif
 #if 0
 /*
  * breakpoint:	capture as much info as possible and then call DDB
  * or trap, as the case may be.
+ *
  * First, we switch to interrupt globals, and blow away %g7.  Then
  * switch down one stackframe -- just fiddle w/cwp, don't save or
  * we'll trap.  Then slowly save all the globals into our static
  * register buffer.  etc. etc.
  */
 breakpoint:
 	wrpr	%g0, PSTATE_KERN|PSTATE_IG, %pstate	! Get IG to use
 	rdpr	%cwp, %g7
 	inc	1, %g7					! Equivalent of save
 	wrpr	%g7, 0, %cwp				! Now we have some unused locals to fiddle with
 XXX ddb_regs is now ddb-regp and is a pointer not a symbol.
 	set	_C_LABEL(ddb_regs), %l0
 	stx	%g1, [%l0+DBR_IG+(1*8)]			! Save IGs
 	stx	%g2, [%l0+DBR_IG+(2*8)]
 	stx	%g3, [%l0+DBR_IG+(3*8)]
 	stx	%g4, [%l0+DBR_IG+(4*8)]
 	stx	%g5, [%l0+DBR_IG+(5*8)]
 	stx	%g6, [%l0+DBR_IG+(6*8)]
 	stx	%g7, [%l0+DBR_IG+(7*8)]
 	wrpr	%g0, PSTATE_KERN|PSTATE_MG, %pstate	! Get MG to use
 	stx	%g1, [%l0+DBR_MG+(1*8)]			! Save MGs
 	stx	%g2, [%l0+DBR_MG+(2*8)]
 	stx	%g3, [%l0+DBR_MG+(3*8)]
 	stx	%g4, [%l0+DBR_MG+(4*8)]
 	stx	%g5, [%l0+DBR_MG+(5*8)]
 	stx	%g6, [%l0+DBR_MG+(6*8)]
 	stx	%g7, [%l0+DBR_MG+(7*8)]
 	wrpr	%g0, PSTATE_KERN|PSTATE_AG, %pstate	! Get AG to use
 	stx	%g1, [%l0+DBR_AG+(1*8)]			! Save AGs
 	stx	%g2, [%l0+DBR_AG+(2*8)]
 	stx	%g3, [%l0+DBR_AG+(3*8)]
 	stx	%g4, [%l0+DBR_AG+(4*8)]
 	stx	%g5, [%l0+DBR_AG+(5*8)]
 	stx	%g6, [%l0+DBR_AG+(6*8)]
 	stx	%g7, [%l0+DBR_AG+(7*8)]
 	wrpr	%g0, PSTATE_KERN, %pstate	! Get G to use
 	stx	%g1, [%l0+DBR_G+(1*8)]			! Save Gs
 	stx	%g2, [%l0+DBR_G+(2*8)]
 	stx	%g3, [%l0+DBR_G+(3*8)]
 	stx	%g4, [%l0+DBR_G+(4*8)]
 	stx	%g5, [%l0+DBR_G+(5*8)]
 	stx	%g6, [%l0+DBR_G+(6*8)]
 	stx	%g7, [%l0+DBR_G+(7*8)]
 	rdpr	%canrestore, %l1
 	stb	%l1, [%l0+DBR_CANRESTORE]
 	rdpr	%cansave, %l2
 	stb	%l2, [%l0+DBR_CANSAVE]
 	rdpr	%cleanwin, %l3
 	stb	%l3, [%l0+DBR_CLEANWIN]
 	rdpr	%wstate, %l4
 	stb	%l4, [%l0+DBR_WSTATE]
 	rd	%y, %l5
 	stw	%l5, [%l0+DBR_Y]
 	rdpr	%tl, %l6
 	stb	%l6, [%l0+DBR_TL]
 	dec	1, %g7
 #endif
 /*
  * I will not touch any of the DDB or KGDB stuff until I know what's going
  * on with the symbol table.  This is all still v7/v8 code and needs to be fixed.
  */
 #ifdef KGDB
 /*
  * bpt is entered on all breakpoint traps.
  * If this is a kernel breakpoint, we do not want to call trap().
  * Among other reasons, this way we can set breakpoints in trap().
  */
 bpt:
 	set	TSTATE_PRIV, %l4
 	andcc	%l4, %l0, %g0		! breakpoint from kernel?
 	bz	slowtrap		! no, go do regular trap
 	 nop
 	/*
 	 * Build a trap frame for kgdb_trap_glue to copy.
 	 * Enable traps but set ipl high so that we will not
 	 * see interrupts from within breakpoints.
 	 */
 	save	%sp, -CCFSZ-TF_SIZE, %sp		! allocate a trap frame
 	TRAP_SETUP(-CCFSZ-TF_SIZE)
 	or	%l0, PSR_PIL, %l4	! splhigh()
 	wr	%l4, 0, %psr		! the manual claims that this
 	wr	%l4, PSR_ET, %psr	! song and dance is necessary
 	std	%l0, [%sp + CCFSZ + 0]	! tf.tf_psr, tf.tf_pc
 	mov	%l3, %o0		! trap type arg for kgdb_trap_glue
 	rd	%y, %l3
 	std	%l2, [%sp + CCFSZ + 8]	! tf.tf_npc, tf.tf_y
 	rd	%wim, %l3
 	st	%l3, [%sp + CCFSZ + 16]	! tf.tf_wim (a kgdb-only r/o field)
 	st	%g1, [%sp + CCFSZ + 20]	! tf.tf_global[1]
 	std	%g2, [%sp + CCFSZ + 24]	! etc
 	std	%g4, [%sp + CCFSZ + 32]
 	std	%g6, [%sp + CCFSZ + 40]
 	std	%i0, [%sp + CCFSZ + 48]	! tf.tf_in[0..1]
 	std	%i2, [%sp + CCFSZ + 56]	! etc
 	std	%i4, [%sp + CCFSZ + 64]
 	std	%i6, [%sp + CCFSZ + 72]
 	/*
 	 * Now call kgdb_trap_glue(); if it returns, call trap().
 	 */
 	mov	%o0, %l3		! gotta save trap type
 	call	_C_LABEL(kgdb_trap_glue)		! kgdb_trap_glue(type, &trapframe)
 	 add	%sp, CCFSZ, %o1		! (&trapframe)
 	/*
 	 * Use slowtrap to call trap---but first erase our tracks
 	 * (put the registers back the way they were).
 	 */
 	mov	%l3, %o0		! slowtrap will need trap type
 	ld	[%sp + CCFSZ + 12], %l3
 	wr	%l3, 0, %y
 	ld	[%sp + CCFSZ + 20], %g1
 	ldd	[%sp + CCFSZ + 24], %g2
 	ldd	[%sp + CCFSZ + 32], %g4
 	b	Lslowtrap_reenter
 	 ldd	[%sp + CCFSZ + 40], %g6
 /*
  * Enter kernel breakpoint.  Write all the windows (not including the
  * current window) into the stack, so that backtrace works.  Copy the
  * supplied trap frame to the kgdb stack and switch stacks.
+ *
  * kgdb_trap_glue(type, tf0)
  *	int type;
  *	struct trapframe *tf0;
  */
 ENTRY_NOPROFILE(kgdb_trap_glue)
 	save	%sp, -CCFSZ, %sp
 	flushw				! flush all windows
 	mov	%sp, %l4		! %l4 = current %sp
 	/* copy trapframe to top of kgdb stack */
 	set	_C_LABEL(kgdb_stack) + KGDB_STACK_SIZE - 80, %l0
 					! %l0 = tfcopy -> end_of_kgdb_stack
 	mov	80, %l1
 :	ldd	[%i1], %l2
 	inc	8, %i1
 	deccc	8, %l1
 	std	%l2, [%l0]
 	bg	1b
 	 inc	8, %l0
 #ifdef NOTDEF_DEBUG
 	/* save old red zone and then turn it off */
 	sethi	%hi(_C_LABEL(redzone)), %l7
 	ld	[%l7 + %lo(_C_LABEL(redzone))], %l6
 	st	%g0, [%l7 + %lo(_C_LABEL(redzone))]
 #endif
 	/* switch to kgdb stack */
 	add	%l0, -CCFSZ-TF_SIZE, %sp
 	/* if (kgdb_trap(type, tfcopy)) kgdb_rett(tfcopy); */
 	mov	%i0, %o0
 	call	_C_LABEL(kgdb_trap)
 	add	%l0, -80, %o1
 	tst	%o0
 	bnz,a	kgdb_rett
 	 add	%l0, -80, %g1
 	/*
 	 * kgdb_trap() did not handle the trap at all so the stack is
 	 * still intact.  A simple `restore' will put everything back,
 	 * after we reset the stack pointer.
 	 */
 	mov	%l4, %sp
 #ifdef NOTDEF_DEBUG
 	st	%l6, [%l7 + %lo(_C_LABEL(redzone))]	! restore red zone
 #endif
 	ret
 	 restore
 /*
  * Return from kgdb trap.  This is sort of special.
+ *
  * We know that kgdb_trap_glue wrote the window above it, so that we will
  * be able to (and are sure to have to) load it up.  We also know that we
  * came from kernel land and can assume that the %fp (%i6) we load here
  * is proper.  We must also be sure not to lower ipl (it is at splhigh())
  * until we have traps disabled, due to the SPARC taking traps at the
  * new ipl before noticing that PSR_ET has been turned off.  We are on
  * the kgdb stack, so this could be disastrous.
+ *
  * Note that the trapframe argument in %g1 points into the current stack
  * frame (current window).  We abandon this window when we move %g1->tf_psr
  * into %psr, but we will not have loaded the new %sp yet, so again traps
  * must be disabled.
  */
 kgdb_rett:
 	rd	%psr, %g4		! turn off traps
 	wr	%g4, PSR_ET, %psr
 	/* use the three-instruction delay to do something useful */
 	ld	[%g1], %g2		! pick up new %psr
 	ld	[%g1 + 12], %g3		! set %y
 	wr	%g3, 0, %y
 #ifdef NOTDEF_DEBUG
 	st	%l6, [%l7 + %lo(_C_LABEL(redzone))] ! and restore red zone
 #endif
 	wr	%g0, 0, %wim		! enable window changes
 	nop; nop; nop
 	/* now safe to set the new psr (changes CWP, leaves traps disabled) */
 	wr	%g2, 0, %psr		! set rett psr (including cond codes)
 	/* 3 instruction delay before we can use the new window */
 /*1*/	ldd	[%g1 + 24], %g2		! set new %g2, %g3
 /*2*/	ldd	[%g1 + 32], %g4		! set new %g4, %g5
 /*3*/	ldd	[%g1 + 40], %g6		! set new %g6, %g7
 	/* now we can use the new window */
 	mov	%g1, %l4
 	ld	[%l4 + 4], %l1		! get new pc
 	ld	[%l4 + 8], %l2		! get new npc
 	ld	[%l4 + 20], %g1		! set new %g1
 	/* set up returnee's out registers, including its %sp */
 	ldd	[%l4 + 48], %i0
 	ldd	[%l4 + 56], %i2
 	ldd	[%l4 + 64], %i4
 	ldd	[%l4 + 72], %i6
 	/* load returnee's window, making the window above it be invalid */
 	restore
 	restore	%g0, 1, %l1		! move to inval window and set %l1 = 1
 	rd	%psr, %l0
 	srl	%l1, %l0, %l1
 	wr	%l1, 0, %wim		! %wim = 1 << (%psr & 31)
 	sethi	%hi(CPCB), %l1
 	LDPTR	[%l1 + %lo(CPCB)], %l1
 	and	%l0, 31, %l0		! CWP = %psr & 31;
 !	st	%l0, [%l1 + PCB_WIM]	! cpcb->pcb_wim = CWP;
 	save	%g0, %g0, %g0		! back to window to reload
 !	LOADWIN(%sp)
 	save	%g0, %g0, %g0		! back to trap window
 	/* note, we have not altered condition codes; safe to just rett */
 	RETT
 #endif
 /*
  * syscall_setup() builds a trap frame and calls syscall().
  * sun_syscall is same but delivers sun system call number
  * XXX	should not have to save&reload ALL the registers just for
  *	ptrace...
  */
 syscall_setup:
 #ifdef TRAPS_USE_IG
 	wrpr	%g0, PSTATE_KERN|PSTATE_IG, %pstate	! DEBUG
 #endif
 	TRAP_SETUP(-CC64FSZ-TF_SIZE)
 #ifdef DEBUG
 	rdpr	%tt, %o1	! debug
 	sth	%o1, [%sp + CC64FSZ + STKB + TF_TT]! debug
 #endif
 	wrpr	%g0, PSTATE_KERN, %pstate	! Get back to normal globals
 	stx	%g1, [%sp + CC64FSZ + STKB + TF_G + ( 1*8)]
 	mov	%g1, %o1			! code
 	rdpr	%tpc, %o2			! (pc)
 	stx	%g2, [%sp + CC64FSZ + STKB + TF_G + ( 2*8)]
 	rdpr	%tstate, %g1
 	stx	%g3, [%sp + CC64FSZ + STKB + TF_G + ( 3*8)]
 	rdpr	%tnpc, %o3
 	stx	%g4, [%sp + CC64FSZ + STKB + TF_G + ( 4*8)]
 	rd	%y, %o4
 	stx	%g5, [%sp + CC64FSZ + STKB + TF_G + ( 5*8)]
 	stx	%g6, [%sp + CC64FSZ + STKB + TF_G + ( 6*8)]
 	CHKPT(%g5,%g6,0x31)
 	wrpr	%g0, 0, %tl			! return to tl=0
 	stx	%g7, [%sp + CC64FSZ + STKB + TF_G + ( 7*8)]
 	add	%sp, CC64FSZ + STKB, %o0	! (&tf)
 	stx	%g1, [%sp + CC64FSZ + STKB + TF_TSTATE]
 	stx	%o2, [%sp + CC64FSZ + STKB + TF_PC]
 	stx	%o3, [%sp + CC64FSZ + STKB + TF_NPC]
 	st	%o4, [%sp + CC64FSZ + STKB + TF_Y]
 	rdpr	%pil, %g5
 	stb	%g5, [%sp + CC64FSZ + STKB + TF_PIL]
 	stb	%g5, [%sp + CC64FSZ + STKB + TF_OLDPIL]
 	!! In the EMBEDANY memory model %g4 points to the start of the data segment.
 	!! In our case we need to clear it before calling any C-code
 	clr	%g4
 	wr	%g0, ASI_PRIMARY_NOFAULT, %asi	! Restore default ASI
 	sethi	%hi(CURLWP), %l1
 	LDPTR	[%l1 + %lo(CURLWP)], %l1
 	LDPTR	[%l1 + L_PROC], %l1		! now %l1 points to p
 	LDPTR	[%l1 + P_MD_SYSCALL], %l1
 	call	%l1
 	 wrpr	%g0, PSTATE_INTR, %pstate	! turn on interrupts
 	/* see `lwp_trampoline' for the reason for this label */
 return_from_syscall:
 	wrpr	%g0, PSTATE_KERN, %pstate	! Disable intterrupts
 	CHKPT(%o1,%o2,0x32)
 	wrpr	%g0, 0, %tl			! Return to tl==0
 	CHKPT(%o1,%o2,4)
 	ba,a,pt	%icc, return_from_trap
 	 nop
 	NOTREACHED
 /*
  * interrupt_vector:
+ *
  * Spitfire chips never get level interrupts directly from H/W.
  * Instead, all interrupts come in as interrupt_vector traps.
  * The interrupt number or handler address is an 11 bit number
  * encoded in the first interrupt data word.  Additional words
  * are application specific and used primarily for cross-calls.
+ *
  * The interrupt vector handler then needs to identify the
  * interrupt source from the interrupt number and arrange to
  * invoke the interrupt handler.  This can either be done directly
  * from here, or a softint at a particular level can be issued.
+ *
  * To call an interrupt directly and not overflow the trap stack,
  * the trap registers should be saved on the stack, registers
  * cleaned, trap-level decremented, the handler called, and then
  * the process must be reversed.
+ *
  * To simplify life all we do here is issue an appropriate softint.
+ *
  * Note:	It is impossible to identify or change a device's
  *		interrupt number until it is probed.  That's the
  *		purpose for all the funny interrupt acknowledge
  *		code.
+ *
  */
 /*
  * Vectored interrupts:
+ *
  * When an interrupt comes in, interrupt_vector uses the interrupt
  * vector number to lookup the appropriate intrhand from the intrlev
  * array.  It then looks up the interrupt level from the intrhand
  * structure.  It uses the level to index the intrpending array,
  * which is 8 slots for each possible interrupt level (so we can
  * shift instead of multiply for address calculation).  It hunts for
  * any available slot at that level.  Available slots are NULL.
+ *
  * Then interrupt_vector uses the interrupt level in the intrhand
  * to issue a softint of the appropriate level.  The softint handler
  * figures out what level interrupt it's handling and pulls the first
  * intrhand pointer out of the intrpending array for that interrupt
  * level, puts a NULL in its place, clears the interrupt generator,
  * and invokes the interrupt handler.
  */
 /* intrpending array is now in per-CPU structure. */
 #ifdef DEBUG
 #define INTRDEBUG_VECTOR	0x1
 #define INTRDEBUG_LEVEL		0x2
 #define INTRDEBUG_FUNC		0x4
 #define INTRDEBUG_SPUR		0x8
 	.data
 	.globl	_C_LABEL(intrdebug)
 _C_LABEL(intrdebug):	.word 0x0
 /*
  * Note: we use the local label `97' to branch forward to, to skip
  * actual debugging code following a `intrdebug' bit test.
  */
 #endif
 	.text
 interrupt_vector:
 #ifdef TRAPSTATS
 	set	_C_LABEL(kiveccnt), %g1
 	set	_C_LABEL(iveccnt), %g2
 	rdpr	%tl, %g3
 	dec	%g3
 	movrz	%g3, %g2, %g1
 	lduw	[%g1], %g2
 	inc	%g2
 	stw	%g2, [%g1]
 #endif
 	ldxa	[%g0] ASI_IRSR, %g1
 	mov	IRDR_0H, %g7
 	ldxa	[%g7] ASI_IRDR, %g7	! Get interrupt number
 	membar	#Sync
 #if KTR_COMPILE & KTR_INTR
 	CATR(KTR_TRAP, "interrupt_vector: tl %d ASI_IRSR %p ASI_IRDR %p",
 		 %g3, %g5, %g6, 10, 11, 12)
 	rdpr	%tl, %g5
 	stx	%g5, [%g3 + KTR_PARM1]
 	stx	%g1, [%g3 + KTR_PARM2]
 	stx	%g7, [%g3 + KTR_PARM3]
 :
 #endif
 	btst	IRSR_BUSY, %g1
 	bz,pn	%icc, 3f		! spurious interrupt
 #ifdef MULTIPROCESSOR
 	 sethi	%hi(KERNBASE), %g1
 	cmp	%g7, %g1
 	bl,pt	%xcc, Lsoftint_regular	! >= KERNBASE is a fast cross-call
 	 cmp	%g7, MAXINTNUM
 	mov	IRDR_1H, %g2
 	ldxa	[%g2] ASI_IRDR, %g2	! Get IPI handler argument 1
 	mov	IRDR_2H, %g3
 	ldxa	[%g3] ASI_IRDR, %g3	! Get IPI handler argument 2
 	stxa	%g0, [%g0] ASI_IRSR	! Ack IRQ
 	membar	#Sync			! Should not be needed due to retry
 	jmpl	%g7, %g0
 	 nop
 #else
 	 cmp	%g7, MAXINTNUM
 #endif
 Lsoftint_regular:
 	stxa	%g0, [%g0] ASI_IRSR	! Ack IRQ
 	membar	#Sync			! Should not be needed due to retry
 	sllx	%g7, PTRSHFT, %g5	! Calculate entry number
 	sethi	%hi(_C_LABEL(intrlev)), %g3
 	bgeu,pn	%xcc, 3f
 	 or	%g3, %lo(_C_LABEL(intrlev)), %g3
 	LDPTR	[%g3 + %g5], %g5	! We have a pointer to the handler
 	brz,pn	%g5, 3f			! NULL means it isn't registered yet.  Skip it.
 	 nop
 setup_sparcintr:
 	LDPTR	[%g5+IH_PEND], %g6	! Read pending flag
 	brnz,pn	%g6, ret_from_intr_vector ! Skip it if it's running
 	 ldub	[%g5+IH_PIL], %g6	! Read interrupt mask
 	sethi	%hi(CPUINFO_VA+CI_INTRPENDING), %g1
 	sll	%g6, PTRSHFT, %g3	! Find start of table for this IPL
 	or	%g1, %lo(CPUINFO_VA+CI_INTRPENDING), %g1
 	add	%g1, %g3, %g1
 :
 	LDPTR	[%g1], %g3		! Load list head
 	STPTR	%g3, [%g5+IH_PEND]	! Link our intrhand node in
 	mov	%g5, %g7
 	CASPTR	[%g1] ASI_N, %g3, %g7
 	cmp	%g7, %g3		! Did it work?
 	bne,pn	CCCR, 1b		! No, try again
 	 EMPTY
 :
 #ifdef NOT_DEBUG
 	set	_C_LABEL(intrdebug), %g7
 	ld	[%g7], %g7
 	btst	INTRDEBUG_VECTOR, %g7
 	bz,pt	%icc, 97f
 	 nop
 	cmp	%g6, 0xa		! ignore clock interrupts?
 	bz,pt	%icc, 97f
 	 nop
 	STACKFRAME(-CC64FSZ)		! Get a clean register window
 	LOAD_ASCIZ(%o0,\
 	    "interrupt_vector: number %lx softint mask %lx pil %lu slot %p\r\n")
 	mov	%g2, %o1
 	rdpr	%pil, %o3
 	mov	%g1, %o4
 	GLOBTOLOC
 	clr	%g4
 	call	prom_printf
 	 mov	%g6, %o2
 	LOCTOGLOB
 	restore
 :
 #endif
 	mov	1, %g7
 	sll	%g7, %g6, %g6
 	wr	%g6, 0, SET_SOFTINT	! Invoke a softint
 ret_from_intr_vector:
 #if KTR_COMPILE & KTR_INTR
 	CATR(KTR_TRAP, "ret_from_intr_vector: tl %d, tstate %p, tpc %p",
 		 %g3, %g4, %g5, 10, 11, 12)
 	rdpr	%tl, %g5
 	stx	%g5, [%g3 + KTR_PARM1]
 	rdpr	%tstate, %g5
 	stx	%g5, [%g3 + KTR_PARM2]
 	rdpr	%tpc, %g5
 	stx	%g5, [%g3 + KTR_PARM3]
 :
 #endif
 	retry
 	NOTREACHED
 :
 #ifdef NOT_DEBUG	/* always do this */
 	set	_C_LABEL(intrdebug), %g6
 	ld	[%g6], %g6
 	btst	INTRDEBUG_SPUR, %g6
 	bz,pt	%icc, 97f
 	 nop
 #endif
 #if 1
 	STACKFRAME(-CC64FSZ)		! Get a clean register window
 	LOAD_ASCIZ(%o0, "interrupt_vector: spurious vector %lx at pil %d\r\n")
 	mov	%g7, %o1
 	GLOBTOLOC
 	clr	%g4
 	call	prom_printf
 	 rdpr	%pil, %o2
 	LOCTOGLOB
 	restore
 :
 #endif
 	ba,a	ret_from_intr_vector
 	 nop				! XXX spitfire bug?
 #if defined(MULTIPROCESSOR)
 /*
  * IPI handler to do nothing, but causes rescheduling..
  * void sparc64_ipi_nop(void *);
  */
 ENTRY(sparc64_ipi_nop)
 	ba,a	ret_from_intr_vector
 	 nop
 /*
  * IPI handler to halt the CPU.  Just calls the C vector.
  * void sparc64_ipi_halt(void *);
  */
 ENTRY(sparc64_ipi_halt)
 	call	_C_LABEL(sparc64_ipi_halt_thiscpu)
 	 clr	%g4
 	sir
 /*
  * IPI handler to pause the CPU.  We just trap to the debugger if it
  * is configured, otherwise just return.
  */
 ENTRY(sparc64_ipi_pause)
 #if defined(DDB)
 sparc64_ipi_pause_trap_point:
 	ta	1
 	 nop
 #endif
 	ba,a	ret_from_intr_vector
 	 nop
 /*
  * Increment IPI event counter, defined in machine/{cpu,intr}.h.
  */
 #define IPIEVC_INC(n,r1,r2)						\
 	sethi	%hi(CPUINFO_VA+CI_IPIEVC+EVC_SIZE*n), r2;		\
 	ldx	[r2 + %lo(CPUINFO_VA+CI_IPIEVC+EVC_SIZE*n)], r1;	\
 	inc	r1;							\
 	stx	r1, [r2 + %lo(CPUINFO_VA+CI_IPIEVC+EVC_SIZE*n)]
 /*
  * IPI handler to flush single pte.
  * void sparc64_ipi_flush_pte(void *);
+ *
  * On entry:
  *	%g2 = vaddr_t va
  *	%g3 = int ctx
  */
 ENTRY(sparc64_ipi_flush_pte)
 #if  KTR_COMPILE & KTR_PMAP
 	CATR(KTR_TRAP, "sparc64_ipi_flush_pte:",
 		 %g1, %g3, %g4, 10, 11, 12)
 :
 #endif
 #ifdef SPITFIRE
 	srlx	%g2, PG_SHIFT4U, %g2		! drop unused va bits
 	mov	CTX_SECONDARY, %g5
 	sllx	%g2, PG_SHIFT4U, %g2
 	ldxa	[%g5] ASI_DMMU, %g6		! Save secondary context
 	sethi	%hi(KERNBASE), %g7
 	membar	#LoadStore
 	stxa	%g3, [%g5] ASI_DMMU		! Insert context to demap
 	membar	#Sync
 	or	%g2, DEMAP_PAGE_SECONDARY, %g2	! Demap page from secondary context only
 	stxa	%g2, [%g2] ASI_DMMU_DEMAP	! Do the demap
 	stxa	%g2, [%g2] ASI_IMMU_DEMAP	! to both TLBs
 #ifdef _LP64
 	srl	%g2, 0, %g2			! and make sure it's both 32- and 64-bit entries
 	stxa	%g2, [%g2] ASI_DMMU_DEMAP	! Do the demap
 	stxa	%g2, [%g2] ASI_IMMU_DEMAP	! Do the demap
 #endif
 	flush	%g7
 	stxa	%g6, [%g5] ASI_DMMU		! Restore secondary context
 	membar	#Sync
 	IPIEVC_INC(IPI_EVCNT_TLB_PTE,%g2,%g3)
 #else
 	WRITEME
 	andn	%g2, 0xfff, %g2			! drop unused va bits
 	mov	CTX_PRIMARY, %g5
 	ldxa	[%g5] ASI_DMMU, %g6		! Save secondary context
 	sethi	%hi(KERNBASE), %g7
 	membar	#LoadStore
 	stxa	%g3, [%g5] ASI_DMMU		! Insert context to demap
 	membar	#Sync
 	or	%g2, DEMAP_PAGE_PRIMARY, %g2
 	stxa	%g2, [%g2] ASI_DMMU_DEMAP	! Do the demap
 	stxa	%g2, [%g2] ASI_IMMU_DEMAP	! to both TLBs
 #ifdef _LP64
 	srl	%g2, 0, %g2			! and make sure it's both 32- and 64-bit entries
 	stxa	%g2, [%g2] ASI_DMMU_DEMAP	! Do the demap
 	stxa	%g2, [%g2] ASI_IMMU_DEMAP	! Do the demap
 #endif
 	flush	%g7
 	stxa	%g6, [%g5] ASI_DMMU		! Restore primary context
 	membar	#Sync
 	IPIEVC_INC(IPI_EVCNT_TLB_PTE,%g2,%g3)
 #endif
 	ba,a	ret_from_intr_vector
 	 nop
 /*
  * Secondary CPU bootstrap code.
  */
 	.text
 	.align 32
 :	rd	%pc, %l0
 	LDULNG	[%l0 + (4f-1b)], %l1
 	add	%l0, (6f-1b), %l2
 	clr	%l3
 :	cmp	%l3, %l1
 	be	CCCR, 3f
 	 nop
 	ldx	[%l2 + TTE_VPN], %l4
 	ldx	[%l2 + TTE_DATA], %l5
 	wr	%g0, ASI_DMMU, %asi
 	stxa	%l4, [%g0 + TLB_TAG_ACCESS] %asi
 	stxa	%l5, [%g0] ASI_DMMU_DATA_IN
 	wr	%g0, ASI_IMMU, %asi
 	stxa	%l4, [%g0 + TLB_TAG_ACCESS] %asi
 	stxa	%l5, [%g0] ASI_IMMU_DATA_IN
 	membar	#Sync
 	flush	%l4
 	add	%l2, PTE_SIZE, %l2
 	add	%l3, 1, %l3
 	ba	%xcc, 2b
 	 nop
 :	LDULNG	[%l0 + (5f-1b)], %l1
 	LDULNG	[%l0 + (7f-1b)], %g2	! Load cpu_info address.
 	jmpl	%l1, %g0
 	 nop
 	.align PTRSZ
 :	ULONG	0x0
 :	ULONG	0x0
 :	ULONG	0x0
 	_ALIGN
 :
 #define DATA(name) \
         .data ; \
         .align PTRSZ ; \
         .globl  name ; \
 name:
 DATA(mp_tramp_code)
 	POINTER	1b
 DATA(mp_tramp_code_len)
 	ULONG	6b-1b
 DATA(mp_tramp_tlb_slots)
 	ULONG	4b-1b
 DATA(mp_tramp_func)
 	ULONG	5b-1b
 DATA(mp_tramp_ci)
 	ULONG	7b-1b
 	.text
 	.align 32
 #endif				/* MULTIPROCESSOR */
 /*
  * Ultra1 and Ultra2 CPUs use soft interrupts for everything.  What we do
  * on a soft interrupt, is we should check which bits in ASR_SOFTINT(0x16)
  * are set, handle those interrupts, then clear them by setting the
  * appropriate bits in ASR_CLEAR_SOFTINT(0x15).
+ *
  * We have an array of 8 interrupt vector slots for each of 15 interrupt
  * levels.  If a vectored interrupt can be dispatched, the dispatch
  * routine will place a pointer to an intrhand structure in one of
  * the slots.  The interrupt handler will go through the list to look
  * for an interrupt to dispatch.  If it finds one it will pull it off
  * the list, free the entry, and call the handler.  The code is like
  * this:
+ *
  *	for (i=0; i<8; i++)
  *		if (ih = intrpending[intlev][i]) {
  *			intrpending[intlev][i] = NULL;
  *			if ((*ih->ih_fun)(ih->ih_arg ? ih->ih_arg : &frame))
  *				return;
  *			strayintr(&frame);
  *			return;
  *		}
+ *
  * Otherwise we go back to the old style of polled interrupts.
+ *
  * After preliminary setup work, the interrupt is passed to each
  * registered handler in turn.  These are expected to return nonzero if
  * they took care of the interrupt.  If a handler claims the interrupt,
  * we exit (hardware interrupts are latched in the requestor so we'll
  * just take another interrupt in the unlikely event of simultaneous
  * interrupts from two different devices at the same level).  If we go
  * through all the registered handlers and no one claims it, we report a
  * stray interrupt.  This is more or less done as:
+ *
  *	for (ih = intrhand[intlev]; ih; ih = ih->ih_next)
  *		if ((*ih->ih_fun)(ih->ih_arg ? ih->ih_arg : &frame))
  *			return;
  *	strayintr(&frame);
+ *
  * Inputs:
  *	%l0 = %tstate
  *	%l1 = return pc
  *	%l2 = return npc
  *	%l3 = interrupt level
  *	(software interrupt only) %l4 = bits to clear in interrupt register
+ *
  * Internal:
  *	%l4, %l5: local variables
  *	%l6 = %y
  *	%l7 = %g1
  *	%g2..%g7 go to stack
+ *
  * An interrupt frame is built in the space for a full trapframe;
  * this contains the psr, pc, npc, and interrupt level.
+ *
  * The level of this interrupt is determined by:
+ *
  *       IRQ# = %tt - 0x40
  */
 ENTRY_NOPROFILE(sparc_interrupt)
 #ifdef TRAPS_USE_IG
 	! This is for interrupt debugging
 	wrpr	%g0, PSTATE_KERN|PSTATE_IG, %pstate	! DEBUG
 #endif
 	/*
 	 * If this is a %tick softint, clear it then call interrupt_vector.
 	 */
 	rd	SOFTINT, %g1
 	btst	1, %g1
 	bz,pt	%icc, 0f
 	 sethi	%hi(CPUINFO_VA+CI_TICK_IH), %g3
 	wr	%g0, 1, CLEAR_SOFTINT
 	ba,pt	%icc, setup_sparcintr
 	 LDPTR	[%g3 + %lo(CPUINFO_VA+CI_TICK_IH)], %g5
 :
 	! Increment the per-cpu interrupt level
 	sethi	%hi(CPUINFO_VA+CI_IDEPTH), %g1
 	ld	[%g1 + %lo(CPUINFO_VA+CI_IDEPTH)], %g2
 	inc	%g2
 	st	%g2, [%g1 + %lo(CPUINFO_VA+CI_IDEPTH)]
 #ifdef TRAPSTATS
 	sethi	%hi(_C_LABEL(kintrcnt)), %g1
 	sethi	%hi(_C_LABEL(uintrcnt)), %g2
 	or	%g1, %lo(_C_LABEL(kintrcnt)), %g1
 	or	%g1, %lo(_C_LABEL(uintrcnt)), %g2
 	rdpr	%tl, %g3
 	dec	%g3
 	movrz	%g3, %g2, %g1
 	lduw	[%g1], %g2
 	inc	%g2
 	stw	%g2, [%g1]
 	/* See if we're on the interrupt stack already. */
 	set	EINTSTACK, %g2
 	set	(EINTSTACK-INTSTACK), %g1
 	btst	1, %sp
 	add	%sp, BIAS, %g3
 	movz	%icc, %sp, %g3
 	srl	%g3, 0, %g3
 	sub	%g2, %g3, %g3
 	cmp	%g3, %g1
 	bgu	1f
 	 set	_C_LABEL(intristk), %g1
 	lduw	[%g1], %g2
 	inc	%g2
 	stw	%g2, [%g1]
 :
 #endif
 	INTR_SETUP(-CC64FSZ-TF_SIZE)
 	! Switch to normal globals so we can save them
 	wrpr	%g0, PSTATE_KERN, %pstate
 	stx	%g1, [%sp + CC64FSZ + STKB + TF_G + ( 1*8)]
 	stx	%g2, [%sp + CC64FSZ + STKB + TF_G + ( 2*8)]
 	stx	%g3, [%sp + CC64FSZ + STKB + TF_G + ( 3*8)]
 	stx	%g4, [%sp + CC64FSZ + STKB + TF_G + ( 4*8)]
 	stx	%g5, [%sp + CC64FSZ + STKB + TF_G + ( 5*8)]
 	stx	%g6, [%sp + CC64FSZ + STKB + TF_G + ( 6*8)]
 	stx	%g7, [%sp + CC64FSZ + STKB + TF_G + ( 7*8)]
 	/*
 	 * In the EMBEDANY memory model %g4 points to the start of the
 	 * data segment.  In our case we need to clear it before calling
 	 * any C-code.
 	 */
 	clr	%g4
 	flushw			! Do not remove this insn -- causes interrupt loss
 	rd	%y, %l6
 	INCR(_C_LABEL(uvmexp)+V_INTR)	! cnt.v_intr++; (clobbers %o0,%o1,%o2)
 	rdpr	%tt, %l5		! Find out our current IPL
 	rdpr	%tstate, %l0
 	rdpr	%tpc, %l1
 	rdpr	%tnpc, %l2
 	rdpr	%tl, %l3		! Dump our trap frame now we have taken the IRQ
 	stw	%l6, [%sp + CC64FSZ + STKB + TF_Y]	! Silly, but we need to save this for rft
 	dec	%l3
 	CHKPT(%l4,%l7,0x26)
 	wrpr	%g0, %l3, %tl
 	sth	%l5, [%sp + CC64FSZ + STKB + TF_TT]! debug
 	stx	%l0, [%sp + CC64FSZ + STKB + TF_TSTATE]	! set up intrframe/clockframe
 	stx	%l1, [%sp + CC64FSZ + STKB + TF_PC]
 	btst	TSTATE_PRIV, %l0		! User mode?
 	stx	%l2, [%sp + CC64FSZ + STKB + TF_NPC]
 	sub	%l5, 0x40, %l6			! Convert to interrupt level
 	sethi	%hi(_C_LABEL(intr_evcnts)), %l4
 	stb	%l6, [%sp + CC64FSZ + STKB + TF_PIL]	! set up intrframe/clockframe
 	rdpr	%pil, %o1
 	mulx	%l6, EVC_SIZE, %l3
 	or	%l4, %lo(_C_LABEL(intr_evcnts)), %l4	! intrcnt[intlev]++;
 	stb	%o1, [%sp + CC64FSZ + STKB + TF_OLDPIL]	! old %pil
 	ldx	[%l4 + %l3], %o0
 	add	%l4, %l3, %l4
 	clr	%l5			! Zero handled count
 #ifdef MULTIPROCESSOR
 	mov	1, %l3			! Ack softint
 :	add	%o0, 1, %l7
 	casxa	[%l4] ASI_N, %o0, %l7
 	cmp	%o0, %l7
 	bne,a,pn %xcc, 1b		! retry if changed
 	 mov	%l7, %o0
 #else
 	inc	%o0
 	mov	1, %l3			! Ack softint
 	stx	%o0, [%l4]
 #endif
 	sll	%l3, %l6, %l3		! Generate IRQ mask
 	wrpr	%l6, %pil
 sparc_intr_retry:
 	wr	%l3, 0, CLEAR_SOFTINT	! (don't clear possible %tick IRQ)
 	sethi	%hi(CPUINFO_VA+CI_INTRPENDING), %l4
 	sll	%l6, PTRSHFT, %l2
 	or	%l4, %lo(CPUINFO_VA+CI_INTRPENDING), %l4
 	add	%l2, %l4, %l4
 :
 	membar	#StoreLoad		! Make sure any failed casxa insns complete
 	LDPTR	[%l4], %l2		! Check a slot
 	cmp	%l2, -1
 	beq,pn	CCCR, intrcmplt		! Empty list?
 	 mov	-1, %l7
 	membar	#LoadStore
 	CASPTR	[%l4] ASI_N, %l2, %l7	! Grab the entire list
 	cmp	%l7, %l2
 	bne,pn	CCCR, 1b
 	 EMPTY
 :
 	add	%sp, CC64FSZ+STKB, %o2	! tf = %sp + CC64FSZ + STKB
 	LDPTR	[%l2 + IH_PEND], %l7	! save ih->ih_pending
 	membar	#LoadStore
 	STPTR	%g0, [%l2 + IH_PEND]	! Clear pending flag
 	membar	#Sync
 	LDPTR	[%l2 + IH_FUN], %o4	! ih->ih_fun
 	LDPTR	[%l2 + IH_ARG], %o0	! ih->ih_arg
 #ifdef NOT_DEBUG
 	set	_C_LABEL(intrdebug), %o3
 	ld	[%o2], %o3
 	btst	INTRDEBUG_FUNC, %o3
 	bz,a,pt	%icc, 97f
 	 nop
 	cmp	%l6, 0xa		! ignore clock interrupts?
 	bz,pt	%icc, 97f
 	 nop
 	STACKFRAME(-CC64FSZ)		! Get a clean register window
 	LOAD_ASCIZ(%o0, "sparc_interrupt: func %p arg %p\r\n")
 	mov	%i0, %o2		! arg
 	GLOBTOLOC
 	call	prom_printf
 	 mov	%i4, %o1		! func
 	LOCTOGLOB
 	restore
 :
 	mov	%l4, %o1
 #endif
 	wrpr	%g0, PSTATE_INTR, %pstate	! Reenable interrupts
 	jmpl	%o4, %o7		! handled = (*ih->ih_fun)(...)
 	 movrz	%o0, %o2, %o0		! arg = (arg == 0) ? arg : tf
 	wrpr	%g0, PSTATE_KERN, %pstate	! Disable interrupts
 	LDPTR	[%l2 + IH_CLR], %l1
 	membar	#Sync
 	brz,pn	%l1, 0f
 	 add	%l5, %o0, %l5
 #ifdef SCHIZO_BUS_SPACE_BROKEN
 	stxa	%g0, [%l1] ASI_PHYS_NON_CACHED		! Clear intr source
 #else
 	stx	%g0, [%l1]		! Clear intr source
 #endif
 	membar	#Sync			! Should not be needed
 :
 	cmp	%l7, -1
 	bne,pn	CCCR, 2b		! 'Nother?
 	 mov	%l7, %l2
 intrcmplt:
 	/*
 	 * Re-read SOFTINT to see if any new  pending interrupts
 	 * at this level.
 	 */
 	mov	1, %l3			! Ack softint
 	rd	SOFTINT, %l7		! %l5 contains #intr handled.
 	sll	%l3, %l6, %l3		! Generate IRQ mask
 	btst	%l3, %l7		! leave mask in %l3 for retry code
 	bnz,pn	%icc, sparc_intr_retry
 	 mov	1, %l5			! initialize intr count for next run
 	! Decrement this cpu's interrupt depth
 	sethi	%hi(CPUINFO_VA+CI_IDEPTH), %l4
 	ld	[%l4 + %lo(CPUINFO_VA+CI_IDEPTH)], %l5
 	dec	%l5
 	st	%l5, [%l4 + %lo(CPUINFO_VA+CI_IDEPTH)]
 #ifdef NOT_DEBUG
 	set	_C_LABEL(intrdebug), %o2
 	ld	[%o2], %o2
 	btst	INTRDEBUG_FUNC, %o2
 	bz,a,pt	%icc, 97f
 	 nop
 	cmp	%l6, 0xa		! ignore clock interrupts?
 	bz,pt	%icc, 97f
 	 nop
 	STACKFRAME(-CC64FSZ)		! Get a clean register window
 	LOAD_ASCIZ(%o0, "sparc_interrupt:  done\r\n")
 	GLOBTOLOC
 	call	prom_printf
 	 nop
 	LOCTOGLOB
 	restore
 :
 #endif
 	ldub	[%sp + CC64FSZ + STKB + TF_OLDPIL], %l3	! restore old %pil
 	wrpr	%l3, 0, %pil
 	CHKPT(%o1,%o2,5)
 	ba,a,pt	%icc, return_from_trap
 	 nop
 #ifdef notyet
 /*
  * Level 12 (ZS serial) interrupt.  Handle it quickly, schedule a
  * software interrupt, and get out.  Do the software interrupt directly
  * if we would just take it on the way out.
+ *
  * Input:
  *	%l0 = %psr
  *	%l1 = return pc
  *	%l2 = return npc
  * Internal:
  *	%l3 = zs device
  *	%l4, %l5 = temporary
  *	%l6 = rr3 (or temporary data) + 0x100 => need soft int
  *	%l7 = zs soft status
  */
 zshard:
 #endif /* notyet */
 	.globl	return_from_trap, rft_kernel, rft_user
 	.globl	softtrap, slowtrap
 /*
  * Various return-from-trap routines (see return_from_trap).
  */
 /*
  * Return from trap.
  * registers are:
+ *
  *	[%sp + CC64FSZ + STKB] => trap frame
+ *
  * We must load all global, out, and trap registers from the trap frame.
+ *
  * If returning to kernel, we should be at the proper trap level because
  * we don't touch %tl.
+ *
  * When returning to user mode, the trap level does not matter, as it
  * will be set explicitly.
+ *
  * If we are returning to user code, we must:
  *  1.  Check for register windows in the pcb that belong on the stack.
  *	If there are any, reload them
  */
 return_from_trap:
 #ifdef DEBUG
 	!! Make sure we don't have pc == npc == 0 or we suck.
 	ldx	[%sp + CC64FSZ + STKB + TF_PC], %g2
 	ldx	[%sp + CC64FSZ + STKB + TF_NPC], %g3
 	orcc	%g2, %g3, %g0
 	tz	%icc, 1
 #endif
 #if KTR_COMPILE & KTR_TRAP
 	CATR(KTR_TRAP, "rft: sp=%p pc=%p npc=%p tstate=%p",
 		 %g2, %g3, %g4, 10, 11, 12)
 	stx	%i6, [%g2 + KTR_PARM1]
 	ldx	[%sp + CC64FSZ + STKB + TF_PC], %g3
 	stx	%g3, [%g2 + KTR_PARM2]
 	ldx	[%sp + CC64FSZ + STKB + TF_NPC], %g3
 	stx	%g3, [%g2 + KTR_PARM3]
 	ldx	[%sp + CC64FSZ + STKB + TF_TSTATE], %g3
 	stx	%g3, [%g2 + KTR_PARM4]
 :
 #endif
 	!!
 	!! We'll make sure we flush our pcb here, rather than later.
 	!!
 	ldx	[%sp + CC64FSZ + STKB + TF_TSTATE], %g1
 	btst	TSTATE_PRIV, %g1			! returning to userland?
 	!!
 	!! Let all pending interrupts drain before returning to userland
 	!!
 	bnz,pn	%icc, 1f				! Returning to userland?
 	 nop
 	wrpr	%g0, PSTATE_INTR, %pstate
 	wrpr	%g0, %g0, %pil				! Lower IPL
 :
 	wrpr	%g0, PSTATE_KERN, %pstate		! Make sure we have normal globals & no IRQs
 	/* Restore normal globals */
 	ldx	[%sp + CC64FSZ + STKB + TF_G + (1*8)], %g1
 	ldx	[%sp + CC64FSZ + STKB + TF_G + (2*8)], %g2
 	ldx	[%sp + CC64FSZ + STKB + TF_G + (3*8)], %g3
 	ldx	[%sp + CC64FSZ + STKB + TF_G + (4*8)], %g4
 	ldx	[%sp + CC64FSZ + STKB + TF_G + (5*8)], %g5
 	ldx	[%sp + CC64FSZ + STKB + TF_G + (6*8)], %g6
 	ldx	[%sp + CC64FSZ + STKB + TF_G + (7*8)], %g7
 	/* Switch to alternate globals and load outs */
 	wrpr	%g0, PSTATE_KERN|PSTATE_AG, %pstate
 #ifdef TRAPS_USE_IG
 	wrpr	%g0, PSTATE_KERN|PSTATE_IG, %pstate	! DEBUG
 #endif
 	ldx	[%sp + CC64FSZ + STKB + TF_O + (0*8)], %i0
 	ldx	[%sp + CC64FSZ + STKB + TF_O + (1*8)], %i1
 	ldx	[%sp + CC64FSZ + STKB + TF_O + (2*8)], %i2
 	ldx	[%sp + CC64FSZ + STKB + TF_O + (3*8)], %i3
 	ldx	[%sp + CC64FSZ + STKB + TF_O + (4*8)], %i4
 	ldx	[%sp + CC64FSZ + STKB + TF_O + (5*8)], %i5
 	ldx	[%sp + CC64FSZ + STKB + TF_O + (6*8)], %i6
 	ldx	[%sp + CC64FSZ + STKB + TF_O + (7*8)], %i7
 	/* Now load trap registers into alternate globals */
 	ld	[%sp + CC64FSZ + STKB + TF_Y], %g4
 	ldx	[%sp + CC64FSZ + STKB + TF_TSTATE], %g1		! load new values
 	wr	%g4, 0, %y
 	ldx	[%sp + CC64FSZ + STKB + TF_PC], %g2
 	ldx	[%sp + CC64FSZ + STKB + TF_NPC], %g3
 #ifdef NOTDEF_DEBUG
 	ldub	[%sp + CC64FSZ + STKB + TF_PIL], %g5		! restore %pil
 	wrpr	%g5, %pil				! DEBUG
 #endif
 	/* Returning to user mode or kernel mode? */
 	btst	TSTATE_PRIV, %g1		! returning to userland?
 	CHKPT(%g4, %g7, 6)
 	bz,pt	%icc, rft_user
 	 sethi	%hi(CPUINFO_VA+CI_WANT_AST), %g7	! first instr of rft_user
 /*
  * Return from trap, to kernel.
+ *
  * We will assume, for the moment, that all kernel traps are properly stacked
  * in the trap registers, so all we have to do is insert the (possibly modified)
  * register values into the trap registers then do a retry.
+ *
  */
 rft_kernel:
 	rdpr	%tl, %g4				! Grab a set of trap registers
 	inc	%g4
 	wrpr	%g4, %g0, %tl
 	wrpr	%g3, 0, %tnpc
 	wrpr	%g2, 0, %tpc
 	wrpr	%g1, 0, %tstate
 	CHKPT(%g1,%g2,7)
 	restore
 	CHKPT(%g1,%g2,0)			! Clear this out
 	rdpr	%tstate, %g1			! Since we may have trapped our regs may be toast
 	rdpr	%cwp, %g2
 	andn	%g1, CWP, %g1
 	wrpr	%g1, %g2, %tstate		! Put %cwp in %tstate
 	CLRTT
 #ifdef TRAPSTATS
 	rdpr	%tl, %g2
 	set	_C_LABEL(rftkcnt), %g1
 	sllx	%g2, 2, %g2
 	add	%g1, %g2, %g1
 	lduw	[%g1], %g2
 	inc	%g2
 	stw	%g2, [%g1]
 #endif
 #if	0
 	wrpr	%g0, 0, %cleanwin	! DEBUG
 #endif
 #if defined(DDB) && defined(MULTIPROCESSOR)
 	set	sparc64_ipi_pause_trap_point, %g1
 	rdpr	%tpc, %g2
 	cmp	%g1, %g2
 	bne,pt	%icc, 0f
 	 nop
 	done
 :
 #endif
 	retry
 	NOTREACHED
 /*
  * Return from trap, to user.  Checks for scheduling trap (`ast') first;
  * will re-enter trap() if set.  Note that we may have to switch from
  * the interrupt stack to the kernel stack in this case.
  *	%g1 = %tstate
  *	%g2 = return %pc
  *	%g3 = return %npc
  * If returning to a valid window, just set psr and return.
  */
 	.data
 rft_wcnt:	.word 0
 	.text
 rft_user:
 !	sethi	%hi(CPUINFO_VA+CI_WANT_AST), %g7	! (done above)
 	lduw	[%g7 + %lo(CPUINFO_VA+CI_WANT_AST)], %g7! want AST trap?
 	brnz,pn	%g7, softtrap			! yes, re-enter trap with type T_AST
 	 mov	T_AST, %g4
 	CHKPT(%g4,%g7,8)
 #ifdef NOTDEF_DEBUG
 	sethi	%hi(CPCB), %g4
 	LDPTR	[%g4 + %lo(CPCB)], %g4
 	ldub	[%g4 + PCB_NSAVED], %g4		! nsaved
 	brz,pt	%g4, 2f		! Only print if nsaved <> 0
 	 nop
 	set	1f, %o0
 	mov	%g4, %o1
 	mov	%g2, %o2			! pc
 	wr	%g0, ASI_DMMU, %asi		! restore the user context
 	ldxa	[CTX_SECONDARY] %asi, %o3	! ctx
 	GLOBTOLOC
 	mov	%g3, %o5
 	call	printf
 	 mov	%i6, %o4			! sp
 !	wrpr	%g0, PSTATE_INTR, %pstate		! Allow IRQ service
 !	wrpr	%g0, PSTATE_KERN, %pstate		! DenyIRQ service
 	LOCTOGLOB
 :
 	.data
 	.asciz	"rft_user: nsaved=%x pc=%d ctx=%x sp=%x npc=%p\n"
 	_ALIGN
 	.text
 #endif
 	/*
 	 * NB: only need to do this after a cache miss
 	 */
 #ifdef TRAPSTATS
 	set	_C_LABEL(rftucnt), %g6
 	lduw	[%g6], %g7
 	inc	%g7
 	stw	%g7, [%g6]
 #endif
 	/*
 	 * Now check to see if any regs are saved in the pcb and restore them.
+	 *
 	 * Here we need to undo the damage caused by switching to a kernel
 	 * stack.
+	 *
 	 * We will use alternate globals %g4..%g7 because %g1..%g3 are used
 	 * by the data fault trap handlers and we don't want possible conflict.
 	 */
 	sethi	%hi(CPCB), %g6
 	rdpr	%otherwin, %g7			! restore register window controls
 #ifdef DEBUG
 	rdpr	%canrestore, %g5		! DEBUG
 	tst	%g5				! DEBUG
 	tnz	%icc, 1; nop			! DEBUG
 !	mov	%g0, %g5			! There should be *NO* %canrestore
 	add	%g7, %g5, %g7			! DEBUG
 #endif
 	wrpr	%g0, %g7, %canrestore
 	LDPTR	[%g6 + %lo(CPCB)], %g6
 	wrpr	%g0, 0, %otherwin
 	CHKPT(%g4,%g7,9)
 	ldub	[%g6 + PCB_NSAVED], %g7		! Any saved reg windows?
 	wrpr	%g0, WSTATE_USER, %wstate	! Need to know where our sp points
 #ifdef DEBUG
 	set	rft_wcnt, %g4	! Keep track of all the windows we restored
 	stw	%g7, [%g4]
 #endif
 	brz,pt	%g7, 5f				! No saved reg wins
 	 nop
 	dec	%g7				! We can do this now or later.  Move to last entry
 #ifdef DEBUG
 	rdpr	%canrestore, %g4			! DEBUG Make sure we've restored everything
 	brnz,a,pn	%g4, 0f				! DEBUG
 	 sir						! DEBUG we should NOT have any usable windows here
 :							! DEBUG
 	wrpr	%g0, 5, %tl
 #endif
 	rdpr	%otherwin, %g4
 	sll	%g7, 7, %g5			! calculate ptr into rw64 array 8*16 == 128 or 7 bits
 	brz,pt	%g4, 6f				! We should not have any user windows left
 	 add	%g5, %g6, %g5
 	set	1f, %o0
 	mov	%g7, %o1
 	mov	%g4, %o2
 	call	printf
 	 wrpr	%g0, PSTATE_KERN, %pstate
 	set	2f, %o0
 	call	panic
 	 nop
 	NOTREACHED
 	.data
 :	.asciz	"pcb_nsaved=%x and otherwin=%x\n"
 :	.asciz	"rft_user\n"
 	_ALIGN
 	.text
 :
 :
 	restored					! Load in the window
 	restore						! This should not trap!
 	ldx	[%g5 + PCB_RW + ( 0*8)], %l0		! Load the window from the pcb
 	ldx	[%g5 + PCB_RW + ( 1*8)], %l1
 	ldx	[%g5 + PCB_RW + ( 2*8)], %l2
 	ldx	[%g5 + PCB_RW + ( 3*8)], %l3
 	ldx	[%g5 + PCB_RW + ( 4*8)], %l4
 	ldx	[%g5 + PCB_RW + ( 5*8)], %l5
 	ldx	[%g5 + PCB_RW + ( 6*8)], %l6
 	ldx	[%g5 + PCB_RW + ( 7*8)], %l7
 	ldx	[%g5 + PCB_RW + ( 8*8)], %i0
 	ldx	[%g5 + PCB_RW + ( 9*8)], %i1
 	ldx	[%g5 + PCB_RW + (10*8)], %i2
 	ldx	[%g5 + PCB_RW + (11*8)], %i3
 	ldx	[%g5 + PCB_RW + (12*8)], %i4
 	ldx	[%g5 + PCB_RW + (13*8)], %i5
 	ldx	[%g5 + PCB_RW + (14*8)], %i6
 	ldx	[%g5 + PCB_RW + (15*8)], %i7
 #ifdef DEBUG
 	stx	%g0, [%g5 + PCB_RW + (14*8)]		! DEBUG mark that we've saved this one
 #endif
 	cmp	%g5, %g6
 	bgu,pt	%xcc, 3b				! Next one?
 	 dec	8*16, %g5
 	rdpr	%ver, %g5
 	stb	%g0, [%g6 + PCB_NSAVED]			! Clear them out so we won't do this again
 	and	%g5, CWP, %g5
 	add	%g5, %g7, %g4
 	dec	1, %g5					! NWINDOWS-1-1
 	wrpr	%g5, 0, %cansave
 	wrpr	%g0, 0, %canrestore			! Make sure we have no freeloaders XXX
 	wrpr	%g0, WSTATE_USER, %wstate		! Save things to user space
 	mov	%g7, %g5				! We already did one restore
 :
 	rdpr	%canrestore, %g4
 	inc	%g4
 	deccc	%g5
 	wrpr	%g4, 0, %cleanwin			! Make *sure* we don't trap to cleanwin
 	bge,a,pt	%xcc, 4b				! return to starting regwin
 	 save	%g0, %g0, %g0				! This may force a datafault
 #ifdef DEBUG
 	wrpr	%g0, 0, %tl
 #endif
 #ifdef TRAPSTATS
 	set	_C_LABEL(rftuld), %g5
 	lduw	[%g5], %g4
 	inc	%g4
 	stw	%g4, [%g5]
 #endif
 	!!
 	!! We can't take any save faults in here 'cause they will never be serviced
 	!!
 #ifdef DEBUG
 	sethi	%hi(CPCB), %g5
 	LDPTR	[%g5 + %lo(CPCB)], %g5
 	ldub	[%g5 + PCB_NSAVED], %g5		! Any saved reg windows?
 	tst	%g5
 	tnz	%icc, 1; nop			! Debugger if we still have saved windows
 	bne,a	rft_user			! Try starting over again
 	 sethi	%hi(CPUINFO_VA+CI_WANT_AST), %g7
 #endif
 	/*
 	 * Set up our return trapframe so we can recover if we trap from here
 	 * on in.
 	 */
 	wrpr	%g0, 1, %tl			! Set up the trap state
 	wrpr	%g2, 0, %tpc
 	wrpr	%g3, 0, %tnpc
 	ba,pt	%icc, 6f
 	 wrpr	%g1, %g0, %tstate
 :
 	/*
 	 * Set up our return trapframe so we can recover if we trap from here
 	 * on in.
 	 */
 	wrpr	%g0, 1, %tl			! Set up the trap state
 	wrpr	%g2, 0, %tpc
 	wrpr	%g3, 0, %tnpc
 	wrpr	%g1, %g0, %tstate
 	restore
 :
 	CHKPT(%g4,%g7,0xa)
 	rdpr	%canrestore, %g5
 	wrpr	%g5, 0, %cleanwin			! Force cleanup of kernel windows
 #ifdef NOTDEF_DEBUG
 	ldx	[%g6 + CC64FSZ + STKB + TF_L + (0*8)], %g5! DEBUG -- get proper value for %l0
 	cmp	%l0, %g5
 	be,a,pt %icc, 1f
 	 nop
 !	sir			! WATCHDOG
 	set	badregs, %g1	! Save the suspect regs
 	stw	%l0, [%g1+(4*0)]
 	stw	%l1, [%g1+(4*1)]
 	stw	%l2, [%g1+(4*2)]
 	stw	%l3, [%g1+(4*3)]
 	stw	%l4, [%g1+(4*4)]
 	stw	%l5, [%g1+(4*5)]
 	stw	%l6, [%g1+(4*6)]
 	stw	%l7, [%g1+(4*7)]
 	stw	%i0, [%g1+(4*8)+(4*0)]
 	stw	%i1, [%g1+(4*8)+(4*1)]
 	stw	%i2, [%g1+(4*8)+(4*2)]
 	stw	%i3, [%g1+(4*8)+(4*3)]
 	stw	%i4, [%g1+(4*8)+(4*4)]
 	stw	%i5, [%g1+(4*8)+(4*5)]
 	stw	%i6, [%g1+(4*8)+(4*6)]
 	stw	%i7, [%g1+(4*8)+(4*7)]
 	save
 	inc	%g7
 	wrpr	%g7, 0, %otherwin
 	wrpr	%g0, 0, %canrestore
 	wrpr	%g0, WSTATE_KERN, %wstate	! Need to know where our sp points
 	set	rft_wcnt, %g4	! Restore nsaved before trapping
 	sethi	%hi(CPCB), %g6
 	LDPTR	[%g6 + %lo(CPCB)], %g6
 	lduw	[%g4], %g4
 	stb	%g4, [%g6 + PCB_NSAVED]
 	ta	1
 	sir
 	.data
 badregs:
 	.space	16*4
 	.text
 :
 #endif
 	rdpr	%tstate, %g1
 	rdpr	%cwp, %g7			! Find our cur window
 	andn	%g1, CWP, %g1			! Clear it from %tstate
 	wrpr	%g1, %g7, %tstate		! Set %tstate with %cwp
 	CHKPT(%g4,%g7,0xb)
 	wr	%g0, ASI_DMMU, %asi		! restore the user context
 	ldxa	[CTX_SECONDARY] %asi, %g4
 	sethi	%hi(KERNBASE), %g7		! Should not be needed due to retry
 	stxa	%g4, [CTX_PRIMARY] %asi
 	membar	#Sync				! Should not be needed due to retry
 	flush	%g7				! Should not be needed due to retry
 	CLRTT
 	CHKPT(%g4,%g7,0xd)
 #ifdef TRAPSTATS
 	set	_C_LABEL(rftudone), %g1
 	lduw	[%g1], %g2
 	inc	%g2
 	stw	%g2, [%g1]
 #endif
 #ifdef DEBUG
 	sethi	%hi(CPCB), %g5
 	LDPTR	[%g5 + %lo(CPCB)], %g5
 	ldub	[%g5 + PCB_NSAVED], %g5		! Any saved reg windows?
 	tst	%g5
 	tnz	%icc, 1; nop			! Debugger if we still have saved windows!
 #endif
 	wrpr	%g0, 0, %pil			! Enable all interrupts
 	retry
 ! exported end marker for kernel gdb
 	.globl	_C_LABEL(endtrapcode)
 _C_LABEL(endtrapcode):
 #ifdef DDB
 !!!
 !!! Dump the DTLB to phys address in %o0 and print it
 !!!
 !!! Only toast a few %o registers
 !!!
 ENTRY_NOPROFILE(dump_dtlb)
 	clr	%o1
 	add	%o1, (64 * 8), %o3
 :
 	ldxa	[%o1] ASI_DMMU_TLB_TAG, %o2
 	membar	#Sync
 	stx	%o2, [%o0]
 	membar	#Sync
 	inc	8, %o0
 	ldxa	[%o1] ASI_DMMU_TLB_DATA, %o4
 	membar	#Sync
 	inc	8, %o1
 	stx	%o4, [%o0]
 	cmp	%o1, %o3
 	membar	#Sync
 	bl	1b
 	 inc	8, %o0
 	retl
 	 nop
 ENTRY_NOPROFILE(dump_itlb)
 	clr	%o1
 	add	%o1, (64 * 8), %o3
 :
 	ldxa	[%o1] ASI_IMMU_TLB_TAG, %o2
 	membar	#Sync
 	stx	%o2, [%o0]
 	membar	#Sync
 	inc	8, %o0
 	ldxa	[%o1] ASI_IMMU_TLB_DATA, %o4
 	membar	#Sync
 	inc	8, %o1
 	stx	%o4, [%o0]
 	cmp	%o1, %o3
 	membar	#Sync
 	bl	1b
 	 inc	8, %o0
 	retl
 	 nop
 #ifdef _LP64
 ENTRY_NOPROFILE(print_dtlb)
 	save	%sp, -CC64FSZ, %sp
 	clr	%l1
 	add	%l1, (64 * 8), %l3
 	clr	%l2
 :
 	ldxa	[%l1] ASI_DMMU_TLB_TAG, %o2
 	membar	#Sync
 	mov	%l2, %o1
 	ldxa	[%l1] ASI_DMMU_TLB_DATA, %o3
 	membar	#Sync
 	inc	%l2
 	set	2f, %o0
 	call	_C_LABEL(db_printf)
 	 inc	8, %l1
 	ldxa	[%l1] ASI_DMMU_TLB_TAG, %o2
 	membar	#Sync
 	mov	%l2, %o1
 	ldxa	[%l1] ASI_DMMU_TLB_DATA, %o3
 	membar	#Sync
 	inc	%l2
 	set	3f, %o0
 	call	_C_LABEL(db_printf)
 	 inc	8, %l1
 	cmp	%l1, %l3
 	bl	1b
 	 inc	8, %l0
 	ret
 	 restore
 ENTRY_NOPROFILE(print_itlb)
 	save	%sp, -CC64FSZ, %sp
 	clr	%l1
 	add	%l1, (64 * 8), %l3
 	clr	%l2
 :
 	ldxa	[%l1] ASI_IMMU_TLB_TAG, %o2
 	membar	#Sync
 	mov	%l2, %o1
 	ldxa	[%l1] ASI_IMMU_TLB_DATA, %o3
 	membar	#Sync
 	inc	%l2
 	set	2f, %o0
 	call	_C_LABEL(db_printf)
 	 inc	8, %l1
 	ldxa	[%l1] ASI_IMMU_TLB_TAG, %o2
 	membar	#Sync
 	mov	%l2, %o1
 	ldxa	[%l1] ASI_IMMU_TLB_DATA, %o3
 	membar	#Sync
 	inc	%l2
 	set	3f, %o0
 	call	_C_LABEL(db_printf)
 	 inc	8, %l1
 	cmp	%l1, %l3
 	bl	1b
 	 inc	8, %l0
 	ret
 	 restore
 	.data
 :
 	.asciz	"%2d:%016lx %016lx "
 :
 	.asciz	"%2d:%016lx %016lx\r\n"
 	.text
 #else
 ENTRY_NOPROFILE(print_dtlb)
 	save	%sp, -CC64FSZ, %sp
 	clr	%l1
 	add	%l1, (64 * 8), %l3
 	clr	%l2
 :
 	ldxa	[%l1] ASI_DMMU_TLB_TAG, %o2
 	membar	#Sync
 	srl	%o2, 0, %o3
 	mov	%l2, %o1
 	srax	%o2, 32, %o2
 	ldxa	[%l1] ASI_DMMU_TLB_DATA, %o4
 	membar	#Sync
 	srl	%o4, 0, %o5
 	inc	%l2
 	srax	%o4, 32, %o4
 	set	2f, %o0
 	call	_C_LABEL(db_printf)
 	 inc	8, %l1
 	ldxa	[%l1] ASI_DMMU_TLB_TAG, %o2
 	membar	#Sync
 	srl	%o2, 0, %o3
 	mov	%l2, %o1
 	srax	%o2, 32, %o2
 	ldxa	[%l1] ASI_DMMU_TLB_DATA, %o4
 	membar	#Sync
 	srl	%o4, 0, %o5
 	inc	%l2
 	srax	%o4, 32, %o4
 	set	3f, %o0
 	call	_C_LABEL(db_printf)
 	 inc	8, %l1
 	cmp	%l1, %l3
 	bl	1b
 	 inc	8, %l0
 	ret
 	 restore
 ENTRY_NOPROFILE(print_itlb)
 	save	%sp, -CC64FSZ, %sp
 	clr	%l1
 	add	%l1, (64 * 8), %l3
 	clr	%l2
 :
 	ldxa	[%l1] ASI_IMMU_TLB_TAG, %o2
 	membar	#Sync
 	srl	%o2, 0, %o3
 	mov	%l2, %o1
 	srax	%o2, 32, %o2
 	ldxa	[%l1] ASI_IMMU_TLB_DATA, %o4
 	membar	#Sync
 	srl	%o4, 0, %o5
 	inc	%l2
 	srax	%o4, 32, %o4
 	set	2f, %o0
 	call	_C_LABEL(db_printf)
 	 inc	8, %l1
 	ldxa	[%l1] ASI_IMMU_TLB_TAG, %o2
 	membar	#Sync
 	srl	%o2, 0, %o3
 	mov	%l2, %o1
 	srax	%o2, 32, %o2
 	ldxa	[%l1] ASI_IMMU_TLB_DATA, %o4
 	membar	#Sync
 	srl	%o4, 0, %o5
 	inc	%l2
 	srax	%o4, 32, %o4
 	set	3f, %o0
 	call	_C_LABEL(db_printf)
 	 inc	8, %l1
 	cmp	%l1, %l3
 	bl	1b
 	 inc	8, %l0
 	ret
 	 restore
 	.data
 :
 	.asciz	"%2d:%08x:%08x %08x:%08x "
 :
 	.asciz	"%2d:%08x:%08x %08x:%08x\r\n"
 	.text
 #endif
 #endif
 /*
  * Kernel entry point.
+ *
  * The contract between bootloader and kernel is:
+ *
  * %o0		OpenFirmware entry point, to keep Sun's updaters happy
  * %o1		Address of boot information vector (see bootinfo.h)
  * %o2		Length of the vector, in bytes
  * %o3		OpenFirmware entry point, to mimic Sun bootloader behavior
  * %o4		OpenFirmware, to meet earlier NetBSD kernels expectations
  */
 	.align	8
 start:
 dostart:
 	mov	1, %g1
 	sllx	%g1, 63, %g1
 	wr	%g1, TICK_CMPR	! XXXXXXX clear and disable %tick_cmpr for now
 	/*
 	 * Startup.
+	 *
 	 * The Sun FCODE bootloader is nice and loads us where we want
 	 * to be.  We have a full set of mappings already set up for us.
+	 *
 	 * I think we end up having an entire 16M allocated to us.
+	 *
 	 * We enter with the prom entry vector in %o0, dvec in %o1,
 	 * and the bootops vector in %o2.
+	 *
 	 * All we need to do is:
+	 *
 	 *	1:	Save the prom vector
+	 *
 	 *	2:	Create a decent stack for ourselves
+	 *
 	 *	3:	Install the permanent 4MB kernel mapping
+	 *
 	 *	4:	Call the C language initialization code
+	 *
 	 */
 	/*
 	 * Set the psr into a known state:
 	 * Set supervisor mode, interrupt level >= 13, traps enabled
 	 */
 	wrpr	%g0, 13, %pil
 	wrpr	%g0, PSTATE_INTR|PSTATE_PEF, %pstate
 	wr	%g0, FPRS_FEF, %fprs		! Turn on FPU
 	/*
 	 * Step 2: Set up a v8-like stack if we need to
 	 */
 #ifdef _LP64
 	btst	1, %sp
 	bnz,pt	%icc, 0f
 	 nop
 	add	%sp, -BIAS, %sp
 #else
 	btst	1, %sp
 	bz,pt	%icc, 0f
 	 nop
 	add	%sp, BIAS, %sp
 #endif
 :
 	call	_C_LABEL(bootstrap)
 	 clr	%g4				! Clear data segment pointer
 /*
  * Initialize the boot CPU.  Basically:
+ *
  *	Locate the cpu_info structure for this CPU.
  *	Establish a locked mapping for interrupt stack.
  *	Switch to the initial stack.
  *	Call the routine passed in in cpu_info->ci_spinup
  */
 #ifdef NO_VCACHE
 #define	TTE_DATABITS	TTE_L|TTE_CP|TTE_P|TTE_W
 #else
 #define	TTE_DATABITS	TTE_L|TTE_CP|TTE_CV|TTE_P|TTE_W
 #endif
 ENTRY_NOPROFILE(cpu_initialize)	/* for cosmetic reasons - nicer backtrace */
 	/*
 	 * Step 5: is no more.
 	 */
 	/*
 	 * Step 6: hunt through cpus list and find the one that
 	 * matches our UPAID.
 	 */
 	sethi	%hi(_C_LABEL(cpus)), %l1
 	ldxa	[%g0] ASI_MID_REG, %l2
 	LDPTR	[%l1 + %lo(_C_LABEL(cpus))], %l1
 	srax	%l2, 17, %l2			! Isolate UPAID from CPU reg
 	and	%l2, 0x1f, %l2
 :
 	ld	[%l1 + CI_UPAID], %l3		! Load UPAID
 	cmp	%l3, %l2			! Does it match?
 	bne,a,pt	%icc, 0b		! no
 	 LDPTR	[%l1 + CI_NEXT], %l1		! Load next cpu_info pointer
 	/*
 	 * Get pointer to our cpu_info struct
 	 */
 	ldx	[%l1 + CI_PADDR], %l1		! Load the interrupt stack's PA
 	sethi	%hi(0xa0000000), %l2		! V=1|SZ=01|NFO=0|IE=0
 	sllx	%l2, 32, %l2			! Shift it into place
 	mov	-1, %l3				! Create a nice mask
 	sllx	%l3, 41, %l4			! Mask off high bits
 	or	%l4, 0xfff, %l4			! We can just load this in 12 (of 13) bits
 	andn	%l1, %l4, %l1			! Mask the phys page number
 	or	%l2, %l1, %l1			! Now take care of the high bits
 	or	%l1, TTE_DATABITS, %l2		! And low bits:	L=1|CP=1|CV=?|E=0|P=1|W=1|G=0
 	!!
 	!!  Now, map in the interrupt stack as context==0
 	!!
 	set	TLB_TAG_ACCESS, %l5
 	set	INTSTACK, %l0
 	stxa	%l0, [%l5] ASI_DMMU		! Make DMMU point to it
 	stxa	%l2, [%g0] ASI_DMMU_DATA_IN	! Store it
 	membar	#Sync
 	!! Setup kernel stack (we rely on curlwp on this cpu
 	!! being lwp0 here and it's uarea is mapped special
 	!! and already accessible here)
 	flushw
 	sethi	%hi(CPUINFO_VA+CI_CURLWP), %l0
 	LDPTR	[%l0 + %lo(CPUINFO_VA+CI_CURLWP)], %l0
 	set	USPACE - TF_SIZE - CC64FSZ, %l1
 	LDPTR	[%l0 + L_PCB], %l0
  	add	%l1, %l0, %l0
 #ifdef _LP64
 	andn	%l0, 0x0f, %l0			! Needs to be 16-byte aligned
 	sub	%l0, BIAS, %l0			! and biased
 #endif
 	mov	%l0, %sp
 	flushw
 #ifdef DEBUG
 	set	_C_LABEL(pmapdebug), %o1
 	ld	[%o1], %o1
 	sethi	%hi(0x40000), %o2
 	btst	%o2, %o1
 	bz	0f
 	set	1f, %o0		! Debug printf
 	call	_C_LABEL(prom_printf)
 	.data
 :
 	.asciz	"Setting trap base...\r\n"
 	_ALIGN
 	.text
 :
 #endif
 	/*
 	 * Step 7: change the trap base register, and install our TSB pointers
 	 */
 	/*
 	 * install our TSB pointers
 	 */
 	sethi	%hi(CPUINFO_VA+CI_TSB_DMMU), %l0
 	sethi	%hi(CPUINFO_VA+CI_TSB_IMMU), %l1
 	sethi	%hi(_C_LABEL(tsbsize)), %l2
 	sethi	%hi(0x1fff), %l3
 	sethi	%hi(TSB), %l4
 	LDPTR	[%l0 + %lo(CPUINFO_VA+CI_TSB_DMMU)], %l0
 	LDPTR	[%l1 + %lo(CPUINFO_VA+CI_TSB_IMMU)], %l1
 	ld	[%l2 + %lo(_C_LABEL(tsbsize))], %l2
 	or	%l3, %lo(0x1fff), %l3
 	or	%l4, %lo(TSB), %l4
 	andn	%l0, %l3, %l0			! Mask off size and split bits
 	or	%l0, %l2, %l0			! Make a TSB pointer
 	stxa	%l0, [%l4] ASI_DMMU		! Install data TSB pointer
 	andn	%l1, %l3, %l1			! Mask off size and split bits
 	or	%l1, %l2, %l1			! Make a TSB pointer
 	stxa	%l1, [%l4] ASI_IMMU		! Install instruction TSB pointer
 	membar	#Sync
 	set	1f, %l1
 	flush	%l1
 :
 	/* set trap table */
 	set	_C_LABEL(trapbase), %l1
 	call	_C_LABEL(prom_set_trap_table)	! Now we should be running 100% from our handlers
 	 mov	%l1, %o0
 	wrpr	%l1, 0, %tba			! Make sure the PROM didn't foul up.
 	/*
 	 * Switch to the kernel mode and run away.
 	 */
 	wrpr	%g0, WSTATE_KERN, %wstate
 #ifdef DEBUG
 	wrpr	%g0, 1, %tl			! Debug -- start at tl==3 so we'll watchdog
 	wrpr	%g0, 0x1ff, %tt			! Debug -- clear out unused trap regs
 	wrpr	%g0, 0, %tpc
 	wrpr	%g0, 0, %tnpc
 	wrpr	%g0, 0, %tstate
 	wrpr	%g0, 0, %tl
 #endif
 #ifdef DEBUG
 	set	_C_LABEL(pmapdebug), %o1
 	ld	[%o1], %o1
 	sethi	%hi(0x40000), %o2
 	btst	%o2, %o1
 	bz	0f
 	set	1f, %o0		! Debug printf
 	call	_C_LABEL(prom_printf)
 	.data
 :
 	.asciz	"Calling startup routine...\r\n"
 	_ALIGN
 	.text
 :
 #endif
 	/*
 	 * Call our startup routine.
 	 */
 	sethi	%hi(CPUINFO_VA+CI_SPINUP), %l0
 	LDPTR	[%l0 + %lo(CPUINFO_VA+CI_SPINUP)], %o1
 	call	%o1				! Call routine
 	 clr	%o0				! our frame arg is ignored
 	set	1f, %o0				! Main should never come back here
 	call	_C_LABEL(panic)
 	 nop
 	.data
 :
 	.asciz	"main() returned\r\n"
 	_ALIGN
 	.text
 #if defined(MULTIPROCESSOR)
 	/*
 	 * cpu_mp_startup is called with:
+	 *
 	 *	%g2 = cpu_args
 	 */
 ENTRY(cpu_mp_startup)
 	mov	1, %o0
 	sllx	%o0, 63, %o0
 	wr	%o0, TICK_CMPR	! XXXXXXX clear and disable %tick_cmpr for now
 	wrpr    %g0, 0, %cleanwin
 	wrpr	%g0, 0, %tl			! Make sure we're not in NUCLEUS mode
 	wrpr	%g0, WSTATE_KERN, %wstate
 	wrpr	%g0, PSTATE_KERN, %pstate
 	flushw
 	/*
 	 * Get pointer to our cpu_info struct
 	 */
 	ldx	[%g2 + CBA_CPUINFO], %l1	! Load the interrupt stack's PA
 	sethi	%hi(0xa0000000), %l2		! V=1|SZ=01|NFO=0|IE=0
 	sllx	%l2, 32, %l2			! Shift it into place
 	mov	-1, %l3				! Create a nice mask
 	sllx	%l3, 41, %l4			! Mask off high bits
 	or	%l4, 0xfff, %l4			! We can just load this in 12 (of 13) bits
 	andn	%l1, %l4, %l1			! Mask the phys page number
 	or	%l2, %l1, %l1			! Now take care of the high bits
 	or	%l1, TTE_DATABITS, %l2		! And low bits:	L=1|CP=1|CV=?|E=0|P=1|W=1|G=0
 	/*
 	 *  Now, map in the interrupt stack & cpu_info as context==0
 	 */
 	set	TLB_TAG_ACCESS, %l5
 	set	INTSTACK, %l0
 	stxa	%l0, [%l5] ASI_DMMU		! Make DMMU point to it
 	stxa	%l2, [%g0] ASI_DMMU_DATA_IN	! Store it
 	/*
 	 * Set 0 as primary context XXX
 	 */
 	mov	CTX_PRIMARY, %o0
 	stxa	%g0, [%o0] ASI_DMMU
 	membar	#Sync
 	/*
 	 * Temporarily use the interrupt stack
 	 */
 #ifdef _LP64
 	set	((EINTSTACK - CC64FSZ - TF_SIZE)) & ~0x0f - BIAS, %sp
 #else
 	set	EINTSTACK - CC64FSZ - TF_SIZE, %sp
 #endif
 	set	1, %fp
 	clr	%i7
 	/*
 	 * install our TSB pointers
 	 */
 	sethi	%hi(CPUINFO_VA+CI_TSB_DMMU), %l0
 	sethi	%hi(CPUINFO_VA+CI_TSB_IMMU), %l1
 	sethi	%hi(_C_LABEL(tsbsize)), %l2
 	sethi	%hi(0x1fff), %l3
 	sethi	%hi(TSB), %l4
 	LDPTR	[%l0 + %lo(CPUINFO_VA+CI_TSB_DMMU)], %l0
 	LDPTR	[%l1 + %lo(CPUINFO_VA+CI_TSB_IMMU)], %l1
 	ld	[%l2 + %lo(_C_LABEL(tsbsize))], %l2
 	or	%l3, %lo(0x1fff), %l3
 	or	%l4, %lo(TSB), %l4
 	andn	%l0, %l3, %l0			! Mask off size and split bits
 	or	%l0, %l2, %l0			! Make a TSB pointer
 	stxa	%l0, [%l4] ASI_DMMU		! Install data TSB pointer
 	membar	#Sync
 	andn	%l1, %l3, %l1			! Mask off size and split bits
 	or	%l1, %l2, %l1			! Make a TSB pointer
 	stxa	%l1, [%l4] ASI_IMMU		! Install instruction TSB pointer
 	membar	#Sync
 	set	1f, %o0
 	flush	%o0
 :
 	/* set trap table */
 	set	_C_LABEL(trapbase), %l1
 	call	_C_LABEL(prom_set_trap_table)
 	 mov	%l1, %o0
 	wrpr	%l1, 0, %tba			! Make sure the PROM didn't
 						! foul up.
 	/*
 	 * Use this CPUs idlelewp's uarea stack
 	 */
 	sethi	%hi(CPUINFO_VA+CI_IDLELWP), %l0
 	LDPTR	[%l0 + %lo(CPUINFO_VA+CI_IDLELWP)], %l0
 	set	USPACE - TF_SIZE - CC64FSZ, %l1
 	LDPTR	[%l0 + L_PCB], %l0
 	add	%l0, %l1, %l0
 #ifdef _LP64
 	andn	%l0, 0x0f, %l0			! Needs to be 16-byte aligned
 	sub	%l0, BIAS, %l0			! and biased
 #endif
 	mov	%l0, %sp
 	flushw
 	/*
 	 * Switch to the kernel mode and run away.
 	 */
 	wrpr	%g0, 13, %pil
 	wrpr	%g0, PSTATE_INTR|PSTATE_PEF, %pstate
 	wr	%g0, FPRS_FEF, %fprs			! Turn on FPU
 	call	_C_LABEL(cpu_hatch)
 	 clr %g4
 	b	_C_LABEL(idle_loop)
 	 clr	%o0
 	NOTREACHED
 	.globl cpu_mp_startup_end
 cpu_mp_startup_end:
 #endif	/* MULTIPROCESSOR */
 	.align 8
 ENTRY(get_romtba)
 	retl
 	 rdpr	%tba, %o0
 /*
  * openfirmware(cell* param);
+ *
  * OpenFirmware entry point
+ *
  * If we're running in 32-bit mode we need to convert to a 64-bit stack
  * and 64-bit cells.  The cells we'll allocate off the stack for simplicity.
  */
 	.align 8
 ENTRY(openfirmware)
 	sethi	%hi(romp), %o4
 	andcc	%sp, 1, %g0
 	bz,pt	%icc, 1f
 	 LDPTR	[%o4+%lo(romp)], %o4		! v9 stack, just load the addr and callit
 	save	%sp, -CC64FSZ, %sp
 	rdpr	%pil, %i2
 	mov	PIL_HIGH, %i3
 	cmp	%i3, %i2
 	movle	%icc, %i2, %i3
 	wrpr	%g0, %i3, %pil
 	mov	%i0, %o0
 	mov	%g1, %l1
 	mov	%g2, %l2
 	mov	%g3, %l3
 	mov	%g4, %l4
 	mov	%g5, %l5
 	mov	%g6, %l6
 	mov	%g7, %l7
 	rdpr	%pstate, %l0
 	jmpl	%i4, %o7
 #if !defined(_LP64)
 	 wrpr	%g0, PSTATE_PROM, %pstate
 #else
 	 wrpr	%g0, PSTATE_PROM|PSTATE_IE, %pstate
 #endif
 	wrpr	%l0, %g0, %pstate
 	mov	%l1, %g1
 	mov	%l2, %g2
 	mov	%l3, %g3
 	mov	%l4, %g4
 	mov	%l5, %g5
 	mov	%l6, %g6
 	mov	%l7, %g7
 	wrpr	%i2, 0, %pil
 	ret
 	 restore	%o0, %g0, %o0
 :	! v8 -- need to screw with stack & params
 #ifdef NOTDEF_DEBUG
 	mov	%o7, %o5
 	call	globreg_check
 	 nop
 	mov	%o5, %o7
 #endif
 	save	%sp, -CC64FSZ, %sp		! Get a new 64-bit stack frame
 	add	%sp, -BIAS, %sp
 	rdpr	%pstate, %l0
 	srl	%sp, 0, %sp
 	rdpr	%pil, %i2	! s = splx(level)
 	mov	%i0, %o0
 	mov	PIL_HIGH, %i3
 	mov	%g1, %l1
 	mov	%g2, %l2
 	cmp	%i3, %i2
 	mov	%g3, %l3
 	mov	%g4, %l4
 	mov	%g5, %l5
 	movle	%icc, %i2, %i3
 	mov	%g6, %l6
 	mov	%g7, %l7
 	wrpr	%i3, %g0, %pil
 	jmpl	%i4, %o7
 	! Enable 64-bit addresses for the prom
 #if defined(_LP64)
 	 wrpr	%g0, PSTATE_PROM, %pstate
 #else
 	 wrpr	%g0, PSTATE_PROM|PSTATE_IE, %pstate
 #endif
 	wrpr	%l0, 0, %pstate
 	wrpr	%i2, 0, %pil
 	mov	%l1, %g1
 	mov	%l2, %g2
 	mov	%l3, %g3
 	mov	%l4, %g4
 	mov	%l5, %g5
 	mov	%l6, %g6
 	mov	%l7, %g7
 	ret
 	 restore	%o0, %g0, %o0
 /*
  * void ofw_exit(cell_t args[])
  */
 ENTRY(openfirmware_exit)
 	STACKFRAME(-CC64FSZ)
 	flushw					! Flush register windows
 	wrpr	%g0, PIL_HIGH, %pil		! Disable interrupts
 	sethi	%hi(romtba), %l5
 	LDPTR	[%l5 + %lo(romtba)], %l5
 	wrpr	%l5, 0, %tba			! restore the ofw trap table
 	/* Arrange locked kernel stack as PROM stack */
 	set	EINTSTACK  - CC64FSZ, %l5
 	andn	%l5, 0x0f, %l5			! Needs to be 16-byte aligned
 	sub	%l5, BIAS, %l5			! and biased
 	mov	%l5, %sp
 	flushw
 	sethi	%hi(romp), %l6
 	LDPTR	[%l6 + %lo(romp)], %l6
 	mov     CTX_PRIMARY, %l3		! set context 0
 	stxa    %g0, [%l3] ASI_DMMU
 	membar	#Sync
 	wrpr	%g0, PSTATE_PROM, %pstate	! Disable interrupts
 						! and enable 64-bit addresses
 	wrpr	%g0, 0, %tl			! force trap level 0
 	call	%l6
 	 mov	%i0, %o0
 	NOTREACHED
 /*
  * sp_tlb_flush_pte(vaddr_t va, int ctx)
+ *
  * Flush tte from both IMMU and DMMU.
+ *
  * This uses %o0-%o5
  */
 	.align 8
 ENTRY(sp_tlb_flush_pte)
 #ifdef DEBUG
 	set	DATA_START, %o4				! Forget any recent TLB misses
 	stx	%g0, [%o4]
 	stx	%g0, [%o4+16]
 #endif
 #ifdef DEBUG
 	set	pmapdebug, %o3
 	lduw	[%o3], %o3
 !	movrz	%o1, -1, %o3				! Print on either pmapdebug & PDB_DEMAP or ctx == 0
 	btst	0x0020, %o3
 	bz,pt	%icc, 2f
 	 nop
 	save	%sp, -CC64FSZ, %sp
 	set	1f, %o0
 	mov	%i1, %o1
 	andn	%i0, 0xfff, %o3
 	or	%o3, 0x010, %o3
 	call	_C_LABEL(printf)
 	 mov	%i0, %o2
 	restore
 	.data
 :
 	.asciz	"sp_tlb_flush_pte:	demap ctx=%x va=%08x res=%x\r\n"
 	_ALIGN
 	.text
 :
 #endif
 #ifdef	SPITFIRE
 #ifdef MULTIPROCESSOR
 	rdpr	%pstate, %o3
 	andn	%o3, PSTATE_IE, %o4			! disable interrupts
 	wrpr	%o4, 0, %pstate
 #endif
 	srlx	%o0, PG_SHIFT4U, %o0			! drop unused va bits
 	mov	CTX_SECONDARY, %o2
 	sllx	%o0, PG_SHIFT4U, %o0
 	ldxa	[%o2] ASI_DMMU, %o5			! Save secondary context
 	sethi	%hi(KERNBASE), %o4
 	membar	#LoadStore
 	stxa	%o1, [%o2] ASI_DMMU			! Insert context to demap
 	membar	#Sync
 	or	%o0, DEMAP_PAGE_SECONDARY, %o0		! Demap page from secondary context only
 	stxa	%o0, [%o0] ASI_DMMU_DEMAP		! Do the demap
 	stxa	%o0, [%o0] ASI_IMMU_DEMAP		! to both TLBs
 #ifdef _LP64
 	srl	%o0, 0, %o0				! and make sure it's both 32- and 64-bit entries
 	stxa	%o0, [%o0] ASI_DMMU_DEMAP		! Do the demap
 	stxa	%o0, [%o0] ASI_IMMU_DEMAP		! Do the demap
 #endif
 	flush	%o4
 	stxa	%o5, [%o2] ASI_DMMU			! Restore secondary context
 	membar	#Sync
 	retl
 #ifdef MULTIPROCESSOR
 	 wrpr	%o3, %pstate				! restore interrupts
 #else
 	 nop
 #endif
 #else
 #ifdef MULTIPROCESSOR
-	WRITEME
+	! %o0 = VA [in]
-#endif
+	! %o1 = ctx value [in] / KERNBASE
 	! %o2 = CTX_PRIMARY
 	! %o3 = saved %tl
 	! %o4 = saved %pstate
 	! %o5 = saved primary ctx
 	! Need this for UP as well
 	rdpr	%pstate, %o4
 	andn	%o4, PSTATE_IE, %o3			! disable interrupts
 	wrpr	%o3, 0, %pstate
 	!!
 	!! Cheetahs do not support flushing the IMMU from secondary context
 	!!
 	rdpr	%tl, %o3
 	mov	CTX_PRIMARY, %o2
 	brnz,pt	%o3, 1f
 	 andn	%o0, 0xfff, %o0				! drop unused va bits
 	wrpr	%g0, 1, %tl				! Make sure we're NUCLEUS
 :
 	ldxa	[%o2] ASI_DMMU, %o5			! Save primary context
 	sethi	%hi(KERNBASE), %o4
 	membar	#LoadStore
 	stxa	%o1, [%o2] ASI_DMMU			! Insert context to demap
 	sethi	%hi(KERNBASE), %o1
 	membar	#Sync
 	or	%o0, DEMAP_PAGE_PRIMARY, %o0
 	stxa	%o0, [%o0] ASI_DMMU_DEMAP		! Do the demap
 	stxa	%o0, [%o0] ASI_IMMU_DEMAP		! to both TLBs
 #ifdef _LP64
 	srl	%o0, 0, %o0				! and make sure it's both 32- and 64-bit entries
 	stxa	%o0, [%o0] ASI_DMMU_DEMAP		! Do the demap
 	stxa	%o0, [%o0] ASI_IMMU_DEMAP		! Do the demap
-	flush	%o4
+#endif
 	flush	%o1
 	stxa	%o5, [%o2] ASI_DMMU			! Restore primary context
 	brz,pt	%o3, 1f
-	 flush	%o4
+	 flush	%o1
 	retl
 	 nop
 :
 	wrpr	%o4, %pstate				! restore interrupts
 	retl
 	 wrpr	%g0, %o3, %tl				! Return to kernel mode.
 #endif
 /*
  * sp_tlb_flush_all(void)
+ *
  * Flush all user TLB entries from both IMMU and DMMU.
  */
 	.align 8
 ENTRY(sp_tlb_flush_all)
 #ifdef SPITFIRE
 	rdpr	%pstate, %o3
 	andn	%o3, PSTATE_IE, %o4			! disable interrupts
 	wrpr	%o4, 0, %pstate
 	set	(63 * 8), %o0				! last TLB entry
 	set	CTX_SECONDARY, %o4
 	ldxa	[%o4] ASI_DMMU, %o4			! save secondary context
 	set	CTX_MASK, %o5
 	membar	#Sync
 	! %o0 = loop counter
 	! %o1 = ctx value
 	! %o2 = TLB tag value
 	! %o3 = saved %pstate
 	! %o4 = saved primary ctx
 	! %o5 = CTX_MASK
 	! %xx = saved %tl
 :
 	ldxa	[%o0] ASI_DMMU_TLB_TAG, %o2		! fetch the TLB tag
 	andcc	%o2, %o5, %o1				! context 0?
 	bz,pt	%xcc, 1f				! if so, skip
 	 mov	CTX_SECONDARY, %o2
 	stxa	%o1, [%o2] ASI_DMMU			! set the context
 	set	DEMAP_CTX_SECONDARY, %o2
 	membar	#Sync
 	stxa	%o2, [%o2] ASI_DMMU_DEMAP		! do the demap
 	membar	#Sync
 :
 	dec	8, %o0
 	brgz,pt %o0, 0b					! loop over all entries
 	 nop
 /*
  * now do the IMMU
  */
 	set	(63 * 8), %o0				! last TLB entry
 :
 	ldxa	[%o0] ASI_IMMU_TLB_TAG, %o2		! fetch the TLB tag
 	andcc	%o2, %o5, %o1				! context 0?
 	bz,pt	%xcc, 1f				! if so, skip
 	 mov	CTX_SECONDARY, %o2
 	stxa	%o1, [%o2] ASI_DMMU			! set the context
 	set	DEMAP_CTX_SECONDARY, %o2
 	membar	#Sync
 	stxa	%o2, [%o2] ASI_IMMU_DEMAP		! do the demap
 	membar	#Sync
 :
 	dec	8, %o0
 	brgz,pt %o0, 0b					! loop over all entries
 	 nop
 	set	CTX_SECONDARY, %o2
 	stxa	%o4, [%o2] ASI_DMMU			! restore secondary ctx
 	sethi	%hi(KERNBASE), %o4
 	membar	#Sync
 	flush	%o4
 	retl
 	 wrpr	%o3, %pstate
 #else
 	! XXX bump up %tl around this call always
 	rdpr	%tl, %o4
 	inc	%o4
 	wrpr	%o4, 0, %tl
 	rdpr	%pstate, %o3
 	andn	%o3, PSTATE_IE, %o4			! disable interrupts
 	wrpr	%o4, 0, %pstate
 	set	(63 * 8), %o0				! last TLB entry
 	set	CTX_PRIMARY, %o4
 	ldxa	[%o4] ASI_DMMU, %o4			! save secondary context
 	set	CTX_MASK, %o5
 	membar	#Sync
 	! %o0 = loop counter
 	! %o1 = ctx value
 	! %o2 = TLB tag value
 	! %o3 = saved %pstate
 	! %o4 = saved primary ctx
 	! %o5 = CTX_MASK
 	! %xx = saved %tl
 :
 	ldxa	[%o0] ASI_DMMU_TLB_TAG, %o2		! fetch the TLB tag
 	andcc	%o2, %o5, %o1				! context 0?
 	bz,pt	%xcc, 1f				! if so, skip
 	 mov	CTX_PRIMARY, %o2
 	stxa	%o1, [%o2] ASI_DMMU			! set the context
 	set	DEMAP_CTX_PRIMARY, %o2
 	membar	#Sync
 	stxa	%o2, [%o2] ASI_DMMU_DEMAP		! do the demap
 	membar	#Sync
 :
 	dec	8, %o0
 	brgz,pt %o0, 0b					! loop over all entries
 	 nop
 /*
  * now do the IMMU
  */
 	set	(63 * 8), %o0				! last TLB entry
 :
 	ldxa	[%o0] ASI_IMMU_TLB_TAG, %o2		! fetch the TLB tag
 	andcc	%o2, %o5, %o1				! context 0?
 	bz,pt	%xcc, 1f				! if so, skip
 	 mov	CTX_PRIMARY, %o2
 	stxa	%o1, [%o2] ASI_DMMU			! set the context
 	set	DEMAP_CTX_PRIMARY, %o2
 	membar	#Sync
 	stxa	%o2, [%o2] ASI_IMMU_DEMAP		! do the demap
 	membar	#Sync
 :
 	dec	8, %o0
 	brgz,pt %o0, 0b					! loop over all entries
 	 nop
 	set	CTX_PRIMARY, %o2
 	stxa	%o4, [%o2] ASI_DMMU			! restore secondary ctx
 	sethi	%hi(KERNBASE), %o4
 	membar	#Sync
 	flush	%o4
 	! XXX bump up %tl around this call always
 	rdpr	%tl, %o4
 	dec	%o4
 	wrpr	%o4, 0, %tl
 	retl
 	 wrpr	%o3, %pstate
 #endif
 /*
  * blast_dcache()
+ *
  * Clear out all of D$ regardless of contents
  * Does not modify %o0
+ *
  */
 	.align 8
 ENTRY(blast_dcache)
 /*
  * We turn off interrupts for the duration to prevent RED exceptions.
  */
 #ifdef PROF
 	save	%sp, -CC64FSZ, %sp
 #endif
 	rdpr	%pstate, %o3
 	set	(2 * NBPG) - 32, %o1
 	andn	%o3, PSTATE_IE, %o4			! Turn off PSTATE_IE bit
 	wrpr	%o4, 0, %pstate
 :
 #ifdef SPITFIRE
 	stxa	%g0, [%o1] ASI_DCACHE_TAG
 #else
 	stxa	%g0, [%o1] ASI_DCACHE_INVALIDATE
 #endif
 	brnz,pt	%o1, 1b
 	 dec	32, %o1
 	sethi	%hi(KERNBASE), %o2
 	flush	%o2
 	membar	#Sync
 #ifdef PROF
 	wrpr	%o3, %pstate
 	ret
 	 restore
 #else
 	retl
 	 wrpr	%o3, %pstate
 #endif
 /*
  * blast_icache()
+ *
  * Clear out all of I$ regardless of contents
  * Does not modify %o0
+ *
  */
 	.align 8
 ENTRY(blast_icache)
 /*
  * We turn off interrupts for the duration to prevent RED exceptions.
  */
 	rdpr	%pstate, %o3
 	set	(2 * NBPG) - 32, %o1
 	andn	%o3, PSTATE_IE, %o4			! Turn off PSTATE_IE bit
 	wrpr	%o4, 0, %pstate
 :
 	stxa	%g0, [%o1] ASI_ICACHE_TAG
 	brnz,pt	%o1, 1b
 	 dec	32, %o1
 	sethi	%hi(KERNBASE), %o2
 	flush	%o2
 	membar	#Sync
 	retl
 	 wrpr	%o3, %pstate
 /*
  * dcache_flush_page(paddr_t pa)
+ *
  * Clear one page from D$.
+ *
  */
 	.align 8
 ENTRY(dcache_flush_page)
 #ifndef _LP64
 	COMBINE(%o0, %o1, %o0)
 #endif
 	mov	-1, %o1		! Generate mask for tag: bits [29..2]
 	srlx	%o0, 13-2, %o2	! Tag is PA bits <40:13> in bits <29:2>
 	clr	%o4
 	srl	%o1, 2, %o1	! Now we have bits <29:0> set
 	set	(2*NBPG), %o5
 	ba,pt	%icc, 1f
 	 andn	%o1, 3, %o1	! Now we have bits <29:2> set
 	.align 8
 :
 	ldxa	[%o4] ASI_DCACHE_TAG, %o3
 	mov	%o4, %o0
 	deccc	32, %o5
 	bl,pn	%icc, 2f
 	 inc	32, %o4
 	xor	%o3, %o2, %o3
 	andcc	%o3, %o1, %g0
 	bne,pt	%xcc, 1b
 	 membar	#LoadStore
 #ifdef SPITFIRE
 	stxa	%g0, [%o0] ASI_DCACHE_TAG
 #else
 	stxa	%g0, [%o0] ASI_DCACHE_INVALIDATE
 #endif
 	ba,pt	%icc, 1b
 	 membar	#StoreLoad
 :
 	sethi	%hi(KERNBASE), %o5
 	flush	%o5
 	retl
 	 membar	#Sync
 /*
  * icache_flush_page(paddr_t pa)
+ *
  * Clear one page from I$.
+ *
  */
 	.align 8
 ENTRY(icache_flush_page)
 #ifndef _LP64
 	COMBINE(%o0, %o1, %o0)
 #endif
 #ifdef SPITFIRE
 	!!
 	!! Linux sez that I$ flushes are not needed for cheetah.
 	!!
 	!! Now do the I$
 	srlx	%o0, 13-8, %o2
 	mov	-1, %o1		! Generate mask for tag: bits [35..8]
 	srl	%o1, 32-35+7, %o1
 	clr	%o4
 	sll	%o1, 7, %o1	! Mask
 	set	(2*NBPG), %o5
 :
 	ldda	[%o4] ASI_ICACHE_TAG, %g0	! Tag goes in %g1
 	dec	32, %o5
 	xor	%g1, %o2, %g1
 	andcc	%g1, %o1, %g0
 	bne,pt	%xcc, 2f
 	 membar	#LoadStore
 	stxa	%g0, [%o4] ASI_ICACHE_TAG
 	membar	#StoreLoad
 :
 	brnz,pt	%o5, 1b
 	 inc	32, %o4
 #endif
 	sethi	%hi(KERNBASE), %o5
 	flush	%o5
 	membar	#Sync
 	retl
 	 nop
 /*
  *	cache_flush_phys(paddr_t, psize_t, int);
+ *
  *	Clear a set of paddrs from the D$, I$ and if param3 is
  *	non-zero, E$.  (E$ is not supported yet).
  */
 	.align 8
 ENTRY(cache_flush_phys)
 #ifndef _LP64
 	COMBINE(%o0, %o1, %o0)
 	COMBINE(%o2, %o3, %o1)
 	mov	%o4, %o2
 #endif
 #ifdef DEBUG
 	tst	%o2		! Want to clear E$?
 	tnz	1		! Error!
 #endif
 	add	%o0, %o1, %o1	! End PA
 	dec	%o1
 	!!
 	!! Both D$ and I$ tags match pa bits 40-13, but
 	!! they are shifted different amounts.  So we'll
 	!! generate a mask for bits 40-13.
 	!!
 	mov	-1, %o2		! Generate mask for tag: bits [40..13]
 	srl	%o2, 5, %o2	! 32-5 = [27..0]
 	sllx	%o2, 13, %o2	! 27+13 = [40..13]
 	and	%o2, %o0, %o0	! Mask away uninteresting bits
 	and	%o2, %o1, %o1	! (probably not necessary)
 	set	(2*NBPG), %o5
 	clr	%o4
 :
 	ldxa	[%o4] ASI_DCACHE_TAG, %o3
 #ifdef SPITFIRE
 	ldda	[%o4] ASI_ICACHE_TAG, %g0	! Tag goes in %g1 -- not on cheetah
 #endif
 	sllx	%o3, 40-29, %o3	! Shift D$ tag into place
 	and	%o3, %o2, %o3	! Mask out trash
 	cmp	%o0, %o3
 	blt,pt	%xcc, 2f	! Too low
 #ifdef SPITFIRE
 	 sllx	%g1, 40-35, %g1	! Shift I$ tag into place
 #endif
 	cmp	%o1, %o3
 	bgt,pt	%xcc, 2f	! Too high
 	 nop
 	membar	#LoadStore
 #ifdef SPITFIRE
 	stxa	%g0, [%o4] ASI_DCACHE_TAG ! Just right
 #else
 	stxa	%g0, [%o4] ASI_DCACHE_INVALIDATE ! Just right
 #endif
 :
 #ifdef SPITFIRE
 	and	%g1, %o2, %g1	! Mask out trash
 	cmp	%o0, %g1
 	blt,pt	%xcc, 3f
 	 cmp	%o1, %g1
 	bgt,pt	%xcc, 3f
 	 nop
 	stxa	%g0, [%o4] ASI_ICACHE_TAG
 :
 #endif
 	membar	#StoreLoad
 	dec	32, %o5
 	brgz,pt	%o5, 1b
 	 inc	32, %o4
 	sethi	%hi(KERNBASE), %o5
 	flush	%o5
 	membar	#Sync
 	retl
 	 nop
 #ifdef COMPAT_16
 #ifdef _LP64
 /*
  * XXXXX Still needs lotsa cleanup after sendsig is complete and offsets are known
+ *
  * The following code is copied to the top of the user stack when each
  * process is exec'ed, and signals are `trampolined' off it.
+ *
  * When this code is run, the stack looks like:
  *	[%sp]			128 bytes to which registers can be dumped
  *	[%sp + 128]		signal number (goes in %o0)
  *	[%sp + 128 + 4]		signal code (goes in %o1)
  *	[%sp + 128 + 8]		first word of saved state (sigcontext)
  *	    .
  *	    .
  *	    .
  *	[%sp + NNN]	last word of saved state
  * (followed by previous stack contents or top of signal stack).
  * The address of the function to call is in %g1; the old %g1 and %o0
  * have already been saved in the sigcontext.  We are running in a clean
  * window, all previous windows now being saved to the stack.
+ *
  * Note that [%sp + 128 + 8] == %sp + 128 + 16.  The copy at %sp+128+8
  * will eventually be removed, with a hole left in its place, if things
  * work out.
  */
 ENTRY_NOPROFILE(sigcode)
 	/*
 	 * XXX  the `save' and `restore' below are unnecessary: should
 	 *	replace with simple arithmetic on %sp
+	 *
 	 * Make room on the stack for 64 %f registers + %fsr.  This comes
 	 * out to 64*4+8 or 264 bytes, but this must be aligned to a multiple
 	 * of 64, or 320 bytes.
 	 */
 	save	%sp, -CC64FSZ - 320, %sp
 	mov	%g2, %l2		! save globals in %l registers
 	mov	%g3, %l3
 	mov	%g4, %l4
 	mov	%g5, %l5
 	mov	%g6, %l6
 	mov	%g7, %l7
 	/*
 	 * Saving the fpu registers is expensive, so do it iff it is
 	 * enabled and dirty.
 	 */
 	rd	%fprs, %l0
 	btst	FPRS_DL|FPRS_DU, %l0	! All clean?
 	bz,pt	%icc, 2f
 	 btst	FPRS_DL, %l0		! test dl
 	bz,pt	%icc, 1f
 	 btst	FPRS_DU, %l0		! test du
 	! fpu is enabled, oh well
 	stx	%fsr, [%sp + CC64FSZ + BIAS + 0]
 	add	%sp, BIAS+CC64FSZ+BLOCK_SIZE, %l0	! Generate a pointer so we can
 	andn	%l0, BLOCK_ALIGN, %l0	! do a block store
 	stda	%f0, [%l0] ASI_BLK_P
 	inc	BLOCK_SIZE, %l0
 	stda	%f16, [%l0] ASI_BLK_P
 :
 	bz,pt	%icc, 2f
 	 add	%sp, BIAS+CC64FSZ+BLOCK_SIZE, %l0	! Generate a pointer so we can
 	andn	%l0, BLOCK_ALIGN, %l0	! do a block store
 	add	%l0, 2*BLOCK_SIZE, %l0	! and skip what we already stored
 	stda	%f32, [%l0] ASI_BLK_P
 	inc	BLOCK_SIZE, %l0
 	stda	%f48, [%l0] ASI_BLK_P
 :
 	membar	#Sync
 	rd	%fprs, %l0		! reload fprs copy, for checking after
 	rd	%y, %l1			! in any case, save %y
 	lduw	[%fp + BIAS + 128], %o0	! sig
 	lduw	[%fp + BIAS + 128 + 4], %o1	! code
 	call	%g1			! (*sa->sa_handler)(sig,code,scp)
 	 add	%fp, BIAS + 128 + 8, %o2	! scp
 	wr	%l1, %g0, %y		! in any case, restore %y
 	/*
 	 * Now that the handler has returned, re-establish all the state
 	 * we just saved above, then do a sigreturn.
 	 */
 	btst	FPRS_DL|FPRS_DU, %l0	! All clean?
 	bz,pt	%icc, 2f
 	 btst	FPRS_DL, %l0		! test dl
 	bz,pt	%icc, 1f
 	 btst	FPRS_DU, %l0		! test du
 	ldx	[%sp + CC64FSZ + BIAS + 0], %fsr
 	add	%sp, BIAS+CC64FSZ+BLOCK_SIZE, %l0	! Generate a pointer so we can
 	andn	%l0, BLOCK_ALIGN, %l0	! do a block load
 	ldda	[%l0] ASI_BLK_P, %f0
 	inc	BLOCK_SIZE, %l0
 	ldda	[%l0] ASI_BLK_P, %f16
 :
 	bz,pt	%icc, 2f
 	 nop
 	add	%sp, BIAS+CC64FSZ+BLOCK_SIZE, %l0	! Generate a pointer so we can
 	andn	%l0, BLOCK_ALIGN, %l0	! do a block load
 	inc	2*BLOCK_SIZE, %l0	! and skip what we already loaded
 	ldda	[%l0] ASI_BLK_P, %f32
 	inc	BLOCK_SIZE, %l0
 	ldda	[%l0] ASI_BLK_P, %f48
 :
 	mov	%l2, %g2
 	mov	%l3, %g3
 	mov	%l4, %g4
 	mov	%l5, %g5
 	mov	%l6, %g6
 	mov	%l7, %g7
 	membar	#Sync
 	restore	%g0, SYS_compat_16___sigreturn14, %g1 ! get registers back & set syscall #
 	add	%sp, BIAS + 128 + 8, %o0! compute scp
 !	andn	%o0, 0x0f, %o0
 	t	ST_SYSCALL		! sigreturn(scp)
 	! sigreturn does not return unless it fails
 	mov	SYS_exit, %g1		! exit(errno)
 	t	ST_SYSCALL
 	/* NOTREACHED */
 	.globl	_C_LABEL(esigcode)
 _C_LABEL(esigcode):
 #endif
 #if !defined(_LP64)
 #define SIGCODE_NAME		sigcode
 #define ESIGCODE_NAME		esigcode
 #define SIGRETURN_NAME		SYS_compat_16___sigreturn14
 #define EXIT_NAME		SYS_exit
 #include "sigcode32.s"
 #endif
 #endif
 /*
  * Primitives
  */
 #ifdef ENTRY
 #undef ENTRY
 #endif
 #ifdef GPROF
 	.globl	_mcount
 #define	ENTRY(x) \
 	.globl _C_LABEL(x); .proc 1; .type _C_LABEL(x),@function; \
 _C_LABEL(x): ; \
 	.data; \
 	.align 8; \
 :	.uaword 0; .uaword 0; \
 	.text;	\
 	save	%sp, -CC64FSZ, %sp; \
 	sethi	%hi(0b), %o0; \
 	call	_mcount; \
 	or	%o0, %lo(0b), %o0; \
 	restore
 #else
 #define	ENTRY(x)	.globl _C_LABEL(x); .proc 1; \
 	.type _C_LABEL(x),@function; _C_LABEL(x):
 #endif
 #define	ALTENTRY(x)	.globl _C_LABEL(x); _C_LABEL(x):
 /*
  * getfp() - get stack frame pointer
  */
 ENTRY(getfp)
 	retl
 	 mov %fp, %o0
 /*
  * copyinstr(fromaddr, toaddr, maxlength, &lencopied)
+ *
  * Copy a null terminated string from the user address space into
  * the kernel address space.
  */
 ENTRY(copyinstr)
 	! %o0 = fromaddr, %o1 = toaddr, %o2 = maxlen, %o3 = &lencopied
 #ifdef NOTDEF_DEBUG
 	save	%sp, -CC64FSZ, %sp
 	set	8f, %o0
 	mov	%i0, %o1
 	mov	%i1, %o2
 	mov	%i2, %o3
 	call	printf
 	 mov	%i3, %o4
 	restore
 	.data
 :	.asciz	"copyinstr: from=%x to=%x max=%x &len=%x\n"
 	_ALIGN
 	.text
 #endif
 	brgz,pt	%o2, 1f					! Make sure len is valid
 	 sethi	%hi(CPCB), %o4		! (first instr of copy)
 	retl
 	 mov	ENAMETOOLONG, %o0
 :
 	LDPTR	[%o4 + %lo(CPCB)], %o4	! catch faults
 	set	Lcsfault, %o5
 	membar	#Sync
 	STPTR	%o5, [%o4 + PCB_ONFAULT]
 	mov	%o1, %o5		!	save = toaddr;
 ! XXX should do this in bigger chunks when possible
 :					! loop:
 	ldsba	[%o0] ASI_AIUS, %g1	!	c = *fromaddr;
 	stb	%g1, [%o1]		!	*toaddr++ = c;
 	inc	%o1
 	brz,a,pn	%g1, Lcsdone	!	if (c == NULL)
 	 clr	%o0			!		{ error = 0; done; }
 	deccc	%o2			!	if (--len > 0) {
 	bg,pt	%icc, 0b		!		fromaddr++;
 	 inc	%o0			!		goto loop;
 	ba,pt	%xcc, Lcsdone		!	}
 	 mov	ENAMETOOLONG, %o0	!	error = ENAMETOOLONG;
 	NOTREACHED
 /*
  * copyoutstr(fromaddr, toaddr, maxlength, &lencopied)
+ *
  * Copy a null terminated string from the kernel
  * address space to the user address space.
  */
 ENTRY(copyoutstr)
 	! %o0 = fromaddr, %o1 = toaddr, %o2 = maxlen, %o3 = &lencopied
 #ifdef NOTDEF_DEBUG
 	save	%sp, -CC64FSZ, %sp
 	set	8f, %o0
 	mov	%i0, %o1
 	mov	%i1, %o2
 	mov	%i2, %o3
 	call	printf
 	 mov	%i3, %o4
 	restore
 	.data
 :	.asciz	"copyoutstr: from=%x to=%x max=%x &len=%x\n"
 	_ALIGN
 	.text
 #endif
 	brgz,pt	%o2, 1f					! Make sure len is valid
 	 sethi	%hi(CPCB), %o4		! (first instr of copy)
 	retl
 	 mov	ENAMETOOLONG, %o0
 :
 	LDPTR	[%o4 + %lo(CPCB)], %o4	! catch faults
 	set	Lcsfault, %o5
 	membar	#Sync
 	STPTR	%o5, [%o4 + PCB_ONFAULT]
 	mov	%o1, %o5		!	save = toaddr;
 ! XXX should do this in bigger chunks when possible
 :					! loop:
 	ldsb	[%o0], %g1		!	c = *fromaddr;
 	stba	%g1, [%o1] ASI_AIUS	!	*toaddr++ = c;
 	inc	%o1
 	brz,a,pn	%g1, Lcsdone	!	if (c == NULL)
 	 clr	%o0			!		{ error = 0; done; }
 	deccc	%o2			!	if (--len > 0) {
 	bg,pt	%icc, 0b		!		fromaddr++;
 	 inc	%o0			!		goto loop;
 					!	}
 	mov	ENAMETOOLONG, %o0	!	error = ENAMETOOLONG;
 Lcsdone:				! done:
 	sub	%o1, %o5, %o1		!	len = to - save;
 	brnz,a	%o3, 1f			!	if (lencopied)
 	 STPTR	%o1, [%o3]		!		*lencopied = len;
 :
 	retl				! cpcb->pcb_onfault = 0;
 	 STPTR	%g0, [%o4 + PCB_ONFAULT]! return (error);
 Lcsfault:
 #ifdef NOTDEF_DEBUG
 	save	%sp, -CC64FSZ, %sp
 	set	5f, %o0
 	call	printf
 	 nop
 	restore
 	.data
 :	.asciz	"Lcsfault: recovering\n"
 	_ALIGN
 	.text
 #endif
 	b	Lcsdone			! error = EFAULT;
 	 mov	EFAULT, %o0		! goto ret;
 /*
  * copystr(fromaddr, toaddr, maxlength, &lencopied)
+ *
  * Copy a null terminated string from one point to another in
  * the kernel address space.  (This is a leaf procedure, but
  * it does not seem that way to the C compiler.)
  */
 ENTRY(copystr)
 	brgz,pt	%o2, 0f	! Make sure len is valid
 	 mov	%o1, %o5		!	to0 = to;
 	retl
 	 mov	ENAMETOOLONG, %o0
 :					! loop:
 	ldsb	[%o0], %o4		!	c = *from;
 	tst	%o4
 	stb	%o4, [%o1]		!	*to++ = c;
 	be	1f			!	if (c == 0)
 	 inc	%o1			!		goto ok;
 	deccc	%o2			!	if (--len > 0) {
 	bg,a	0b			!		from++;
 	 inc	%o0			!		goto loop;
 	b	2f			!	}
 	 mov	ENAMETOOLONG, %o0	!	ret = ENAMETOOLONG; goto done;
 :					! ok:
 	clr	%o0			!	ret = 0;
 :
 	sub	%o1, %o5, %o1		!	len = to - to0;
 	tst	%o3			!	if (lencopied)
 	bnz,a	3f
 	 STPTR	%o1, [%o3]		!		*lencopied = len;
 :
 	retl
 	 nop
 #ifdef DIAGNOSTIC
 :
 	sethi	%hi(5f), %o0
 	call	_C_LABEL(panic)
 	 or	%lo(5f), %o0, %o0
 	.data
 :
 	.asciz	"copystr"
 	_ALIGN
 	.text
 #endif
 /*
  * copyin(src, dst, len)
+ *
  * Copy specified amount of data from user space into the kernel.
+ *
  * This is a modified version of memcpy that uses ASI_AIUS.  When
  * memcpy is optimized to use block copy ASIs, this should be also.
  */
 #define	BCOPY_SMALL	32	/* if < 32, copy by bytes */
 ENTRY(copyin)
 !	flushw			! Make sure we don't have stack probs & lose hibits of %o
 #ifdef NOTDEF_DEBUG
 	save	%sp, -CC64FSZ, %sp
 	set	1f, %o0
 	mov	%i0, %o1
 	mov	%i1, %o2
 	call	printf
 	 mov	%i2, %o3
 	restore
 	.data
 :	.asciz	"copyin: src=%x dest=%x len=%x\n"
 	_ALIGN
 	.text
 #endif
 	sethi	%hi(CPCB), %o3
 	wr	%g0, ASI_AIUS, %asi
 	LDPTR	[%o3 + %lo(CPCB)], %o3
 	set	Lcopyfault, %o4
 !	mov	%o7, %g7		! save return address
 	membar	#Sync
 	STPTR	%o4, [%o3 + PCB_ONFAULT]
 	cmp	%o2, BCOPY_SMALL
 Lcopyin_start:
 	bge,a	Lcopyin_fancy	! if >= this many, go be fancy.
 	 btst	7, %o0		! (part of being fancy)
 	/*
 	 * Not much to copy, just do it a byte at a time.
 	 */
 	deccc	%o2		! while (--len >= 0)
 	bl	1f
 :
 	 inc	%o0
 	ldsba	[%o0 - 1] %asi, %o4!	*dst++ = (++src)[-1];
 	stb	%o4, [%o1]
 	deccc	%o2
 	bge	0b
 	 inc	%o1
 :
 	ba	Lcopyin_done
 	 clr	%o0
 	NOTREACHED
 	/*
 	 * Plenty of data to copy, so try to do it optimally.
 	 */
 Lcopyin_fancy:
 	! check for common case first: everything lines up.
 !	btst	7, %o0		! done already
 	bne	1f
 	 EMPTY
 	btst	7, %o1
 	be,a	Lcopyin_doubles
 	 dec	8, %o2		! if all lined up, len -= 8, goto copyin_doubes
 	! If the low bits match, we can make these line up.
 :
 	xor	%o0, %o1, %o3	! t = src ^ dst;
 	btst	1, %o3		! if (t & 1) {
 	be,a	1f
 	 btst	1, %o0		! [delay slot: if (src & 1)]
 	! low bits do not match, must copy by bytes.
 :
 	ldsba	[%o0] %asi, %o4	!	do {
 	inc	%o0		!		(++dst)[-1] = *src++;
 	inc	%o1
 	deccc	%o2
 	bnz	0b		!	} while (--len != 0);
 	 stb	%o4, [%o1 - 1]
 	ba	Lcopyin_done
 	 clr	%o0
 	NOTREACHED
 	! lowest bit matches, so we can copy by words, if nothing else
 :
 	be,a	1f		! if (src & 1) {
 	 btst	2, %o3		! [delay slot: if (t & 2)]
 	! although low bits match, both are 1: must copy 1 byte to align
 	ldsba	[%o0] %asi, %o4	!	*dst++ = *src++;
 	stb	%o4, [%o1]
 	inc	%o0
 	inc	%o1
 	dec	%o2		!	len--;
 	btst	2, %o3		! } [if (t & 2)]
 :
 	be,a	1f		! if (t & 2) {
 	 btst	2, %o0		! [delay slot: if (src & 2)]
 	dec	2, %o2		!	len -= 2;
 :
 	ldsha	[%o0] %asi, %o4	!	do {
 	sth	%o4, [%o1]	!		*(short *)dst = *(short *)src;
 	inc	2, %o0		!		dst += 2, src += 2;
 	deccc	2, %o2		!	} while ((len -= 2) >= 0);
 	bge	0b
 	 inc	2, %o1
 	b	Lcopyin_mopb	!	goto mop_up_byte;
 	 btst	1, %o2		! } [delay slot: if (len & 1)]
 	NOTREACHED
 	! low two bits match, so we can copy by longwords
 :
 	be,a	1f		! if (src & 2) {
 	 btst	4, %o3		! [delay slot: if (t & 4)]
 	! although low 2 bits match, they are 10: must copy one short to align
 	ldsha	[%o0] %asi, %o4	!	(*short *)dst = *(short *)src;
 	sth	%o4, [%o1]
 	inc	2, %o0		!	dst += 2;
 	inc	2, %o1		!	src += 2;
 	dec	2, %o2		!	len -= 2;
 	btst	4, %o3		! } [if (t & 4)]
 :
 	be,a	1f		! if (t & 4) {
 	 btst	4, %o0		! [delay slot: if (src & 4)]
 	dec	4, %o2		!	len -= 4;
 :
 	lduwa	[%o0] %asi, %o4	!	do {
 	st	%o4, [%o1]	!		*(int *)dst = *(int *)src;
 	inc	4, %o0		!		dst += 4, src += 4;
 	deccc	4, %o2		!	} while ((len -= 4) >= 0);
 	bge	0b
 	 inc	4, %o1
 	b	Lcopyin_mopw	!	goto mop_up_word_and_byte;
 	 btst	2, %o2		! } [delay slot: if (len & 2)]
 	NOTREACHED
 	! low three bits match, so we can copy by doublewords
 :
 	be	1f		! if (src & 4) {
 	 dec	8, %o2		! [delay slot: len -= 8]
 	lduwa	[%o0] %asi, %o4	!	*(int *)dst = *(int *)src;
 	st	%o4, [%o1]
 	inc	4, %o0		!	dst += 4, src += 4, len -= 4;
 	inc	4, %o1
 	dec	4, %o2		! }
 :
 Lcopyin_doubles:
 	ldxa	[%o0] %asi, %g1	! do {
 	stx	%g1, [%o1]	!	*(double *)dst = *(double *)src;
 	inc	8, %o0		!	dst += 8, src += 8;
 	deccc	8, %o2		! } while ((len -= 8) >= 0);
 	bge	Lcopyin_doubles
 	 inc	8, %o1
 	! check for a usual case again (save work)
 	btst	7, %o2		! if ((len & 7) == 0)
 	be	Lcopyin_done	!	goto copyin_done;
 	 btst	4, %o2		! if ((len & 4)) == 0)
 	be,a	Lcopyin_mopw	!	goto mop_up_word_and_byte;
 	 btst	2, %o2		! [delay slot: if (len & 2)]
 	lduwa	[%o0] %asi, %o4	!	*(int *)dst = *(int *)src;
 	st	%o4, [%o1]
 	inc	4, %o0		!	dst += 4;
 	inc	4, %o1		!	src += 4;
 	btst	2, %o2		! } [if (len & 2)]
 :
 	! mop up trailing word (if present) and byte (if present).
 Lcopyin_mopw:
 	be	Lcopyin_mopb	! no word, go mop up byte
 	 btst	1, %o2		! [delay slot: if (len & 1)]
 	ldsha	[%o0] %asi, %o4	! *(short *)dst = *(short *)src;
 	be	Lcopyin_done	! if ((len & 1) == 0) goto done;
 	 sth	%o4, [%o1]
 	ldsba	[%o0 + 2] %asi, %o4	! dst[2] = src[2];
 	stb	%o4, [%o1 + 2]
 	ba	Lcopyin_done
 	 clr	%o0
 	NOTREACHED
 	! mop up trailing byte (if present).
 Lcopyin_mopb:
 	be,a	Lcopyin_done
 	 nop
 	ldsba	[%o0] %asi, %o4
 	stb	%o4, [%o1]
 Lcopyin_done:
 	sethi	%hi(CPCB), %o3
 !	stb	%o4,[%o1]	! Store last byte -- should not be needed
 	LDPTR	[%o3 + %lo(CPCB)], %o3
 	membar	#Sync
 	STPTR	%g0, [%o3 + PCB_ONFAULT]
 	wr	%g0, ASI_PRIMARY_NOFAULT, %asi		! Restore ASI
 	retl
 	 clr	%o0			! return 0
 /*
  * copyout(src, dst, len)
+ *
  * Copy specified amount of data from kernel to user space.
  * Just like copyin, except that the `dst' addresses are user space
  * rather than the `src' addresses.
+ *
  * This is a modified version of memcpy that uses ASI_AIUS.  When
  * memcpy is optimized to use block copy ASIs, this should be also.
  */
  /*
   * This needs to be reimplemented to really do the copy.
   */
 ENTRY(copyout)
 	/*
 	 * ******NOTE****** this depends on memcpy() not using %g7
 	 */
 #ifdef NOTDEF_DEBUG
 	save	%sp, -CC64FSZ, %sp
 	set	1f, %o0
 	mov	%i0, %o1
 	set	CTX_SECONDARY, %o4
 	mov	%i1, %o2
 	ldxa	[%o4] ASI_DMMU, %o4
 	call	printf
 	 mov	%i2, %o3
 	restore
 	.data
 :	.asciz	"copyout: src=%x dest=%x len=%x ctx=%d\n"
 	_ALIGN
 	.text
 #endif
 Ldocopy:
 	sethi	%hi(CPCB), %o3
 	wr	%g0, ASI_AIUS, %asi
 	LDPTR	[%o3 + %lo(CPCB)], %o3
 	set	Lcopyfault, %o4
 !	mov	%o7, %g7		! save return address
 	membar	#Sync
 	STPTR	%o4, [%o3 + PCB_ONFAULT]
 	cmp	%o2, BCOPY_SMALL
 Lcopyout_start:
 	membar	#StoreStore
 	bge,a	Lcopyout_fancy	! if >= this many, go be fancy.
 	 btst	7, %o0		! (part of being fancy)
 	/*
 	 * Not much to copy, just do it a byte at a time.
 	 */
 	deccc	%o2		! while (--len >= 0)
 	bl	1f
 	 EMPTY
 :
 	inc	%o0
 	ldsb	[%o0 - 1], %o4!	(++dst)[-1] = *src++;
 	stba	%o4, [%o1] %asi
 	deccc	%o2
 	bge	0b
 	 inc	%o1
 :
 	ba	Lcopyout_done
 	 clr	%o0
 	NOTREACHED
 	/*
 	 * Plenty of data to copy, so try to do it optimally.
 	 */
 Lcopyout_fancy:
 	! check for common case first: everything lines up.
 !	btst	7, %o0		! done already
 	bne	1f
 	 EMPTY
 	btst	7, %o1
 	be,a	Lcopyout_doubles