| @@ -1,14 +1,14 @@ | | | @@ -1,14 +1,14 @@ |
1 | /* $NetBSD: pmap.c,v 1.211.2.6 2010/02/10 15:37:48 uebayasi Exp $ */ | | 1 | /* $NetBSD: pmap.c,v 1.211.2.7 2010/02/10 15:48:28 uebayasi Exp $ */ |
2 | | | 2 | |
3 | /* | | 3 | /* |
4 | * Copyright 2003 Wasabi Systems, Inc. | | 4 | * Copyright 2003 Wasabi Systems, Inc. |
5 | * All rights reserved. | | 5 | * All rights reserved. |
6 | * | | 6 | * |
7 | * Written by Steve C. Woodford for Wasabi Systems, Inc. | | 7 | * Written by Steve C. Woodford for Wasabi Systems, Inc. |
8 | * | | 8 | * |
9 | * Redistribution and use in source and binary forms, with or without | | 9 | * Redistribution and use in source and binary forms, with or without |
10 | * modification, are permitted provided that the following conditions | | 10 | * modification, are permitted provided that the following conditions |
11 | * are met: | | 11 | * are met: |
12 | * 1. Redistributions of source code must retain the above copyright | | 12 | * 1. Redistributions of source code must retain the above copyright |
13 | * notice, this list of conditions and the following disclaimer. | | 13 | * notice, this list of conditions and the following disclaimer. |
14 | * 2. Redistributions in binary form must reproduce the above copyright | | 14 | * 2. Redistributions in binary form must reproduce the above copyright |
| @@ -201,27 +201,27 @@ | | | @@ -201,27 +201,27 @@ |
201 | #include <sys/pool.h> | | 201 | #include <sys/pool.h> |
202 | #include <sys/cdefs.h> | | 202 | #include <sys/cdefs.h> |
203 | #include <sys/cpu.h> | | 203 | #include <sys/cpu.h> |
204 | #include <sys/sysctl.h> | | 204 | #include <sys/sysctl.h> |
205 | | | 205 | |
206 | #include <uvm/uvm.h> | | 206 | #include <uvm/uvm.h> |
207 | | | 207 | |
208 | #include <machine/bus.h> | | 208 | #include <machine/bus.h> |
209 | #include <machine/pmap.h> | | 209 | #include <machine/pmap.h> |
210 | #include <machine/pcb.h> | | 210 | #include <machine/pcb.h> |
211 | #include <machine/param.h> | | 211 | #include <machine/param.h> |
212 | #include <arm/arm32/katelib.h> | | 212 | #include <arm/arm32/katelib.h> |
213 | | | 213 | |
214 | __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.211.2.6 2010/02/10 15:37:48 uebayasi Exp $"); | | 214 | __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.211.2.7 2010/02/10 15:48:28 uebayasi Exp $"); |
215 | | | 215 | |
216 | #ifdef PMAP_DEBUG | | 216 | #ifdef PMAP_DEBUG |
217 | | | 217 | |
218 | /* XXX need to get rid of all refs to this */ | | 218 | /* XXX need to get rid of all refs to this */ |
219 | int pmap_debug_level = 0; | | 219 | int pmap_debug_level = 0; |
220 | | | 220 | |
221 | /* | | 221 | /* |
222 | * for switching to potentially finer grained debugging | | 222 | * for switching to potentially finer grained debugging |
223 | */ | | 223 | */ |
224 | #define PDB_FOLLOW 0x0001 | | 224 | #define PDB_FOLLOW 0x0001 |
225 | #define PDB_INIT 0x0002 | | 225 | #define PDB_INIT 0x0002 |
226 | #define PDB_ENTER 0x0004 | | 226 | #define PDB_ENTER 0x0004 |
227 | #define PDB_REMOVE 0x0008 | | 227 | #define PDB_REMOVE 0x0008 |
| @@ -2559,36 +2559,37 @@ pmap_flush_page(struct vm_page_md *md, p | | | @@ -2559,36 +2559,37 @@ pmap_flush_page(struct vm_page_md *md, p |
2559 | #endif /* PMAP_CACHE_VIPT */ | | 2559 | #endif /* PMAP_CACHE_VIPT */ |
2560 | | | 2560 | |
2561 | /* | | 2561 | /* |
2562 | * Routine: pmap_page_remove | | 2562 | * Routine: pmap_page_remove |
2563 | * Function: | | 2563 | * Function: |
2564 | * Removes this physical page from | | 2564 | * Removes this physical page from |
2565 | * all physical maps in which it resides. | | 2565 | * all physical maps in which it resides. |
2566 | * Reflects back modify bits to the pager. | | 2566 | * Reflects back modify bits to the pager. |
2567 | */ | | 2567 | */ |
2568 | static void | | 2568 | static void |
2569 | pmap_page_remove(struct vm_page *pg) | | 2569 | pmap_page_remove(struct vm_page *pg) |
2570 | { | | 2570 | { |
2571 | struct vm_page_md *md = VM_PAGE_TO_MD(pg); | | 2571 | struct vm_page_md *md = VM_PAGE_TO_MD(pg); |
| | | 2572 | paddr_t pa = VM_PAGE_TO_PHYS(pg); |
2572 | struct l2_bucket *l2b; | | 2573 | struct l2_bucket *l2b; |
2573 | struct pv_entry *pv, *npv, **pvp; | | 2574 | struct pv_entry *pv, *npv, **pvp; |
2574 | pmap_t pm; | | 2575 | pmap_t pm; |
2575 | pt_entry_t *ptep; | | 2576 | pt_entry_t *ptep; |
2576 | bool flush; | | 2577 | bool flush; |
2577 | u_int flags; | | 2578 | u_int flags; |
2578 | | | 2579 | |
2579 | NPDEBUG(PDB_FOLLOW, | | 2580 | NPDEBUG(PDB_FOLLOW, |
2580 | printf("pmap_page_remove: pg %p (0x%08lx)\n", pg, | | 2581 | printf("pmap_page_remove: pg %p (0x%08lx)\n", pg, |
2581 | VM_PAGE_TO_PHYS(pg))); | | 2582 | pa)); |
2582 | | | 2583 | |
2583 | PMAP_HEAD_TO_MAP_LOCK(); | | 2584 | PMAP_HEAD_TO_MAP_LOCK(); |
2584 | simple_lock(&md->pvh_slock); | | 2585 | simple_lock(&md->pvh_slock); |
2585 | | | 2586 | |
2586 | pv = SLIST_FIRST(&md->pvh_list); | | 2587 | pv = SLIST_FIRST(&md->pvh_list); |
2587 | if (pv == NULL) { | | 2588 | if (pv == NULL) { |
2588 | #ifdef PMAP_CACHE_VIPT | | 2589 | #ifdef PMAP_CACHE_VIPT |
2589 | /* | | 2590 | /* |
2590 | * We *know* the page contents are about to be replaced. | | 2591 | * We *know* the page contents are about to be replaced. |
2591 | * Discard the exec contents | | 2592 | * Discard the exec contents |
2592 | */ | | 2593 | */ |
2593 | if (PV_IS_EXEC_P(md->pvh_attrs)) | | 2594 | if (PV_IS_EXEC_P(md->pvh_attrs)) |
2594 | PMAPCOUNT(exec_discarded_page_protect); | | 2595 | PMAPCOUNT(exec_discarded_page_protect); |
| @@ -2671,27 +2672,27 @@ pmap_page_remove(struct vm_page *pg) | | | @@ -2671,27 +2672,27 @@ pmap_page_remove(struct vm_page *pg) |
2671 | *ptep = 0; | | 2672 | *ptep = 0; |
2672 | PTE_SYNC_CURRENT(pm, ptep); | | 2673 | PTE_SYNC_CURRENT(pm, ptep); |
2673 | pmap_free_l2_bucket(pm, l2b, 1); | | 2674 | pmap_free_l2_bucket(pm, l2b, 1); |
2674 | | | 2675 | |
2675 | pool_put(&pmap_pv_pool, pv); | | 2676 | pool_put(&pmap_pv_pool, pv); |
2676 | pv = npv; | | 2677 | pv = npv; |
2677 | /* | | 2678 | /* |
2678 | * if we reach the end of the list and there are still | | 2679 | * if we reach the end of the list and there are still |
2679 | * mappings, they might be able to be cached now. | | 2680 | * mappings, they might be able to be cached now. |
2680 | */ | | 2681 | */ |
2681 | if (pv == NULL) { | | 2682 | if (pv == NULL) { |
2682 | *pvp = NULL; | | 2683 | *pvp = NULL; |
2683 | if (!SLIST_EMPTY(&md->pvh_list)) | | 2684 | if (!SLIST_EMPTY(&md->pvh_list)) |
2684 | pmap_vac_me_harder(md, VM_PAGE_TO_PHYS(pg), pm, 0); | | 2685 | pmap_vac_me_harder(md, pa, pm, 0); |
2685 | } | | 2686 | } |
2686 | pmap_release_pmap_lock(pm); | | 2687 | pmap_release_pmap_lock(pm); |
2687 | } | | 2688 | } |
2688 | #ifdef PMAP_CACHE_VIPT | | 2689 | #ifdef PMAP_CACHE_VIPT |
2689 | /* | | 2690 | /* |
2690 | * Its EXEC cache is now gone. | | 2691 | * Its EXEC cache is now gone. |
2691 | */ | | 2692 | */ |
2692 | if (PV_IS_EXEC_P(md->pvh_attrs)) | | 2693 | if (PV_IS_EXEC_P(md->pvh_attrs)) |
2693 | PMAPCOUNT(exec_discarded_page_protect); | | 2694 | PMAPCOUNT(exec_discarded_page_protect); |
2694 | md->pvh_attrs &= ~PVF_EXEC; | | 2695 | md->pvh_attrs &= ~PVF_EXEC; |
2695 | KASSERT(md->urw_mappings == 0); | | 2696 | KASSERT(md->urw_mappings == 0); |
2696 | KASSERT(md->uro_mappings == 0); | | 2697 | KASSERT(md->uro_mappings == 0); |
2697 | if (md->krw_mappings == 0) | | 2698 | if (md->krw_mappings == 0) |
| @@ -3461,28 +3462,28 @@ pmap_kremove(vaddr_t va, vsize_t len) | | | @@ -3461,28 +3462,28 @@ pmap_kremove(vaddr_t va, vsize_t len) |
3461 | next_bucket = L2_NEXT_BUCKET(va); | | 3462 | next_bucket = L2_NEXT_BUCKET(va); |
3462 | if (next_bucket > eva) | | 3463 | if (next_bucket > eva) |
3463 | next_bucket = eva; | | 3464 | next_bucket = eva; |
3464 | | | 3465 | |
3465 | l2b = pmap_get_l2_bucket(pmap_kernel(), va); | | 3466 | l2b = pmap_get_l2_bucket(pmap_kernel(), va); |
3466 | KDASSERT(l2b != NULL); | | 3467 | KDASSERT(l2b != NULL); |
3467 | | | 3468 | |
3468 | sptep = ptep = &l2b->l2b_kva[l2pte_index(va)]; | | 3469 | sptep = ptep = &l2b->l2b_kva[l2pte_index(va)]; |
3469 | mappings = 0; | | 3470 | mappings = 0; |
3470 | | | 3471 | |
3471 | while (va < next_bucket) { | | 3472 | while (va < next_bucket) { |
3472 | opte = *ptep; | | 3473 | opte = *ptep; |
3473 | opg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); | | 3474 | opg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); |
3474 | struct vm_page_md *omd = VM_PAGE_TO_MD(opg); | | | |
3475 | if (opg) { | | 3475 | if (opg) { |
| | | 3476 | struct vm_page_md *omd = VM_PAGE_TO_MD(opg); |
3476 | if (omd->pvh_attrs & PVF_KMPAGE) { | | 3477 | if (omd->pvh_attrs & PVF_KMPAGE) { |
3477 | simple_lock(&omd->pvh_slock); | | 3478 | simple_lock(&omd->pvh_slock); |
3478 | KASSERT(omd->urw_mappings == 0); | | 3479 | KASSERT(omd->urw_mappings == 0); |
3479 | KASSERT(omd->uro_mappings == 0); | | 3480 | KASSERT(omd->uro_mappings == 0); |
3480 | KASSERT(omd->krw_mappings == 0); | | 3481 | KASSERT(omd->krw_mappings == 0); |
3481 | KASSERT(omd->kro_mappings == 0); | | 3482 | KASSERT(omd->kro_mappings == 0); |
3482 | omd->pvh_attrs &= ~PVF_KMPAGE; | | 3483 | omd->pvh_attrs &= ~PVF_KMPAGE; |
3483 | #ifdef PMAP_CACHE_VIPT | | 3484 | #ifdef PMAP_CACHE_VIPT |
3484 | omd->pvh_attrs &= ~PVF_WRITE; | | 3485 | omd->pvh_attrs &= ~PVF_WRITE; |
3485 | #endif | | 3486 | #endif |
3486 | pmap_kmpages--; | | 3487 | pmap_kmpages--; |
3487 | simple_unlock(&omd->pvh_slock); | | 3488 | simple_unlock(&omd->pvh_slock); |
3488 | #ifdef PMAP_CACHE_VIPT | | 3489 | #ifdef PMAP_CACHE_VIPT |
| @@ -3630,28 +3631,28 @@ pmap_protect(pmap_t pm, vaddr_t sva, vad | | | @@ -3630,28 +3631,28 @@ pmap_protect(pmap_t pm, vaddr_t sva, vad |
3630 | * OK, at this point, we know we're doing | | 3631 | * OK, at this point, we know we're doing |
3631 | * write-protect operation. If the pmap is | | 3632 | * write-protect operation. If the pmap is |
3632 | * active, write-back the page. | | 3633 | * active, write-back the page. |
3633 | */ | | 3634 | */ |
3634 | pmap_dcache_wb_range(pm, sva, PAGE_SIZE, | | 3635 | pmap_dcache_wb_range(pm, sva, PAGE_SIZE, |
3635 | false, false); | | 3636 | false, false); |
3636 | #endif | | 3637 | #endif |
3637 | | | 3638 | |
3638 | pg = PHYS_TO_VM_PAGE(l2pte_pa(pte)); | | 3639 | pg = PHYS_TO_VM_PAGE(l2pte_pa(pte)); |
3639 | pte &= ~L2_S_PROT_W; | | 3640 | pte &= ~L2_S_PROT_W; |
3640 | *ptep = pte; | | 3641 | *ptep = pte; |
3641 | PTE_SYNC(ptep); | | 3642 | PTE_SYNC(ptep); |
3642 | | | 3643 | |
3643 | struct vm_page_md *md = VM_PAGE_TO_MD(pg); | | | |
3644 | if (pg != NULL) { | | 3644 | if (pg != NULL) { |
| | | 3645 | struct vm_page_md *md = VM_PAGE_TO_MD(pg); |
3645 | simple_lock(&md->pvh_slock); | | 3646 | simple_lock(&md->pvh_slock); |
3646 | f = pmap_modify_pv(md, VM_PAGE_TO_PHYS(pg), pm, sva, | | 3647 | f = pmap_modify_pv(md, VM_PAGE_TO_PHYS(pg), pm, sva, |
3647 | clr_mask, 0); | | 3648 | clr_mask, 0); |
3648 | pmap_vac_me_harder(md, VM_PAGE_TO_PHYS(pg), pm, sva); | | 3649 | pmap_vac_me_harder(md, VM_PAGE_TO_PHYS(pg), pm, sva); |
3649 | simple_unlock(&md->pvh_slock); | | 3650 | simple_unlock(&md->pvh_slock); |
3650 | } else | | 3651 | } else |
3651 | f = PVF_REF | PVF_EXEC; | | 3652 | f = PVF_REF | PVF_EXEC; |
3652 | | | 3653 | |
3653 | if (flush >= 0) { | | 3654 | if (flush >= 0) { |
3654 | flush++; | | 3655 | flush++; |
3655 | flags |= f; | | 3656 | flags |= f; |
3656 | } else | | 3657 | } else |
3657 | if (PV_BEEN_EXECD(f)) | | 3658 | if (PV_BEEN_EXECD(f)) |