 @@ -1,14 +1,14 @@
-/*	$NetBSD: pmap.c,v 1.210 2016/07/09 09:33:21 maxv Exp $	*/
+/*	$NetBSD: pmap.c,v 1.211 2016/07/11 14:18:16 maxv Exp $	*/
 /*-
  * Copyright (c) 2008, 2010, 2016 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Andrew Doran, and by Maxime Villard.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -161,27 +161,27 @@
  *     Hibler/Jolitz pmap, as modified for FreeBSD by John S. Dyson
  *     and David Greenman.
+ *
  * [3] the Mach pmap.   this pmap, from CMU, seems to have migrated
  *     between several processors.   the VAX version was done by
  *     Avadis Tevanian, Jr., and Michael Wayne Young.    the i386
  *     version was done by Lance Berc, Mike Kupfer, Bob Baron,
  *     David Golub, and Richard Draves.    the alpha version was
  *     done by Alessandro Forin (CMU/Mach) and Chris Demetriou
  *     (NetBSD/alpha).
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.210 2016/07/09 09:33:21 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.211 2016/07/11 14:18:16 maxv Exp $");
 #include "opt_user_ldt.h"
 #include "opt_lockdebug.h"
 #include "opt_multiprocessor.h"
 #include "opt_xen.h"
 #if !defined(__x86_64__)
 #include "opt_kstack_dr0.h"
 #endif /* !defined(__x86_64__) */
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/proc.h>
 #include <sys/pool.h>
 @@ -1646,27 +1646,26 @@ pmap_prealloc_lowmem_ptps(void)
 		pmap_pte_flush();
 		pmap_update_pg((vaddr_t)early_zerop);
 		memset(early_zerop, 0, PAGE_SIZE);
 #endif
 #ifdef XEN
 		/* Mark R/O before installing */
 		HYPERVISOR_update_va_mapping ((vaddr_t)early_zerop,
 		    xpmap_ptom_masked(newp) | PG_u | PG_V, UVMF_INVLPG);
 		if (newp < (NKL2_KIMG_ENTRIES * NBPD_L2))
 			HYPERVISOR_update_va_mapping (newp + KERNBASE,
 			    xpmap_ptom_masked(newp) | PG_u | PG_V, UVMF_INVLPG);
 		if (level == PTP_LEVELS) { /* Top level pde is per-cpu */
 			pd_entry_t *kpm_pdir;
 			/* Reach it via recursive mapping */
 			kpm_pdir = normal_pdes[PTP_LEVELS - 2];
 			/* Set it as usual. We can't defer this
 			 * outside the loop since recursive
 			 * pte entries won't be accessible during
 			 * further iterations at lower levels
 			 * otherwise.
 			 */
 			pmap_pte_set(&kpm_pdir[pl_i(0, PTP_LEVELS)],
 			    pmap_pa2pte(newp) | pteflags);
 @@ -4295,83 +4294,77 @@ pmap_get_physpage(void)
 /*
  * Allocate the amount of specified ptps for a ptp level, and populate
  * all levels below accordingly, mapping virtual addresses starting at
  * kva.
+ *
  * Used by pmap_growkernel.
  */
 static void
 pmap_alloc_level(pd_entry_t * const *pdes, vaddr_t kva, int lvl,
     long *needed_ptps)
+{
 	unsigned long i;
 	vaddr_t va;
 	paddr_t pa;
 	unsigned long index, endindex;
 	int level;
 	pd_entry_t *pdep;
 #ifdef XEN
 	int s = splvm(); /* protect xpq_* */
 #endif
 	for (level = lvl; level > 1; level--) {
 		if (level == PTP_LEVELS)
 			pdep = pmap_kernel()->pm_pdir;
 		else
 			pdep = pdes[level - 2];
 		va = kva;
 		index = pl_i_roundup(kva, level);
 		endindex = index + needed_ptps[level - 1] - 1;
 		for (i = index; i <= endindex; i++) {
 			pt_entry_t pte;
 			KASSERT(!pmap_valid_entry(pdep[i]));
 			pa = pmap_get_physpage();
 			pte = pmap_pa2pte(pa) | PG_k | PG_V | PG_RW;
 #ifdef XEN
 			pmap_pte_set(&pdep[i], pte);
 #if defined(PAE) || defined(__x86_64__)
 #if defined(XEN) && (defined(PAE) || defined(__x86_64__))
 			if (level == PTP_LEVELS && i >= PDIR_SLOT_KERN) {
 				if (__predict_true(
 				    cpu_info_primary.ci_flags & CPUF_PRESENT)) {
 					/* update per-cpu PMDs on all cpus */
 					xen_kpm_sync(pmap_kernel(), i);
 				} else {
 					/*
 					 * too early; update primary CPU
 					 * PMD only (without locks)
 					 */
 #ifdef PAE
 					pd_entry_t *cpu_pdep =
 					    &cpu_info_primary.ci_kpm_pdir[l2tol2(i)];
 #endif
 #ifdef __x86_64__
 					pd_entry_t *cpu_pdep =
 						&cpu_info_primary.ci_kpm_pdir[i];
 #endif
 					pmap_pte_set(cpu_pdep, pte);
+				}
+			}
-#endif /* PAE || __x86_64__ */
+#endif /* XEN && (PAE || __x86_64__) */
 #else /* XEN */
 			pdep[i] = pte;
 #endif /* XEN */
 			KASSERT(level != PTP_LEVELS || nkptp[level - 1] +
 			    pl_i(VM_MIN_KERNEL_ADDRESS, level) == i);
 			nkptp[level - 1]++;
 			va += nbpd[level - 1];
+		}
 		pmap_pte_flush();
+	}
 #ifdef XEN
 	splx(s);
 #endif
+}
 /*
  * pmap_growkernel: increase usage of KVM space
+ *
  * => we allocate new PTPs for the kernel and install them in all
  *	the pmaps on the system.
 @@ -4393,74 +4386,68 @@ pmap_growkernel(vaddr_t maxkvaddr)
 	mutex_enter(kpm->pm_lock);
 	if (maxkvaddr <= pmap_maxkvaddr) {
 		mutex_exit(kpm->pm_lock);
 		splx(s);
 		return pmap_maxkvaddr;
+	}
 	maxkvaddr = x86_round_pdr(maxkvaddr);
 #if !defined(XEN) || !defined(__x86_64__)
 	old = nkptp[PTP_LEVELS - 1];
 #endif
-	/*
+	/* Initialize needed_kptp. */
 	 * This loop could be optimized more, but pmap_growkernel()
 	 * is called infrequently.
 	 */
 	for (i = PTP_LEVELS - 1; i >= 1; i--) {
 		target_nptp = pl_i_roundup(maxkvaddr, i + 1) -
 		    pl_i_roundup(VM_MIN_KERNEL_ADDRESS, i + 1);
 		/*
 		 * XXX only need to check toplevel.
 		 */
 		if (target_nptp > nkptpmax[i])
 			panic("out of KVA space");
 		KASSERT(target_nptp >= nkptp[i]);
 		needed_kptp[i] = target_nptp - nkptp[i];
+	}
 	pmap_alloc_level(normal_pdes, pmap_maxkvaddr, PTP_LEVELS, needed_kptp);
 	/*
-	 * If the number of top level entries changed, update all
+	 * If the number of top level entries changed, update all pmaps.
 	 * pmaps.
 	 */
 	if (needed_kptp[PTP_LEVELS - 1] != 0) {
 #ifdef XEN
 #ifdef __x86_64__
 		/* nothing, kernel entries are never entered in user pmap */
 #else /* __x86_64__ */
 		mutex_enter(&pmaps_lock);
 		LIST_FOREACH(pm, &pmaps, pm_list) {
 			int pdkidx;
-			for (pdkidx =  PDIR_SLOT_KERN + old;
+			for (pdkidx = PDIR_SLOT_KERN + old;
 			    pdkidx < PDIR_SLOT_KERN + nkptp[PTP_LEVELS - 1];
 			    pdkidx++) {
 				pmap_pte_set(&pm->pm_pdir[pdkidx],
 				    kpm->pm_pdir[pdkidx]);
+			}
 			pmap_pte_flush();
+		}
 		mutex_exit(&pmaps_lock);
 #endif /* __x86_64__ */
 #else /* XEN */
 		unsigned newpdes;
 		newpdes = nkptp[PTP_LEVELS - 1] - old;
 		mutex_enter(&pmaps_lock);
 		LIST_FOREACH(pm, &pmaps, pm_list) {
 			memcpy(&pm->pm_pdir[PDIR_SLOT_KERN + old],
-			       &kpm->pm_pdir[PDIR_SLOT_KERN + old],
+			    &kpm->pm_pdir[PDIR_SLOT_KERN + old],
-			       newpdes * sizeof (pd_entry_t));
+			    newpdes * sizeof (pd_entry_t));
+		}
 		mutex_exit(&pmaps_lock);
 #endif
 		invalidate = true;
+	}
 	pmap_maxkvaddr = maxkvaddr;
 	mutex_exit(kpm->pm_lock);
 	splx(s);
 	if (invalidate && pmap_initialized) {
 		/* Invalidate the PDP cache. */
 		pool_cache_invalidate(&pmap_pdp_cache);
+	}