Wed Mar 4 18:19:27 2015 UTC ()
CLFLUSH needs to be surrounded by membars.

While here, round everything to a multiple of the cache line size.


(riastradh)
diff -r1.3 -r1.4 src/sys/external/bsd/drm2/drm/drm_cache.c
cvs diff -r1.3 -r1.4 src/sys/external/bsd/drm2/drm/drm_cache.c (switch to unified diff)

--- src/sys/external/bsd/drm2/drm/drm_cache.c 2014/07/16 20:56:25 1.3
+++ src/sys/external/bsd/drm2/drm/drm_cache.c 2015/03/04 18:19:27 1.4
@@ -1,160 +1,164 @@ @@ -1,160 +1,164 @@
1/* $NetBSD: drm_cache.c,v 1.3 2014/07/16 20:56:25 riastradh Exp $ */ 1/* $NetBSD: drm_cache.c,v 1.4 2015/03/04 18:19:27 riastradh Exp $ */
2 2
3/*- 3/*-
4 * Copyright (c) 2013 The NetBSD Foundation, Inc. 4 * Copyright (c) 2013 The NetBSD Foundation, Inc.
5 * All rights reserved. 5 * All rights reserved.
6 * 6 *
7 * This code is derived from software contributed to The NetBSD Foundation 7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Taylor R. Campbell. 8 * by Taylor R. Campbell.
9 * 9 *
10 * Redistribution and use in source and binary forms, with or without 10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions 11 * modification, are permitted provided that the following conditions
12 * are met: 12 * are met:
13 * 1. Redistributions of source code must retain the above copyright 13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer. 14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright 15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the 16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution. 17 * documentation and/or other materials provided with the distribution.
18 * 18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE. 29 * POSSIBILITY OF SUCH DAMAGE.
30 */ 30 */
31 31
32#include <sys/cdefs.h> 32#include <sys/cdefs.h>
33__KERNEL_RCSID(0, "$NetBSD: drm_cache.c,v 1.3 2014/07/16 20:56:25 riastradh Exp $"); 33__KERNEL_RCSID(0, "$NetBSD: drm_cache.c,v 1.4 2015/03/04 18:19:27 riastradh Exp $");
34 34
35#include <sys/types.h> 35#include <sys/types.h>
36#include <sys/xcall.h> 36#include <sys/xcall.h>
37 37
38#include <uvm/uvm_extern.h> 38#include <uvm/uvm_extern.h>
39 39
40#include <machine/cpufunc.h> 40#include <machine/cpufunc.h>
41 41
42#include <linux/mm_types.h> 42#include <linux/mm_types.h>
43 43
44#include <drm/drmP.h> 44#include <drm/drmP.h>
45 45
46static bool drm_md_clflush_finegrained_p(void); 46static bool drm_md_clflush_finegrained_p(void);
47static void drm_md_clflush_all(void); 47static void drm_md_clflush_all(void);
48static void drm_md_clflush_page(struct page *); 48static void drm_md_clflush_page(struct page *);
49static void drm_md_clflush_virt_range(const void *, size_t); 49static void drm_md_clflush_virt_range(const void *, size_t);
50 50
51void 51void
52drm_clflush_pages(struct page **pages, unsigned long npages) 52drm_clflush_pages(struct page **pages, unsigned long npages)
53{ 53{
54 54
55 if (drm_md_clflush_finegrained_p()) { 55 if (drm_md_clflush_finegrained_p()) {
56 while (npages--) 56 while (npages--)
57 drm_md_clflush_page(pages[npages]); 57 drm_md_clflush_page(pages[npages]);
58 } else { 58 } else {
59 drm_md_clflush_all(); 59 drm_md_clflush_all();
60 } 60 }
61} 61}
62 62
63void 63void
64drm_clflush_pglist(struct pglist *list) 64drm_clflush_pglist(struct pglist *list)
65{ 65{
66 66
67 if (drm_md_clflush_finegrained_p()) { 67 if (drm_md_clflush_finegrained_p()) {
68 struct vm_page *page; 68 struct vm_page *page;
69 69
70 TAILQ_FOREACH(page, list, pageq.queue) 70 TAILQ_FOREACH(page, list, pageq.queue)
71 drm_md_clflush_page(container_of(page, struct page, 71 drm_md_clflush_page(container_of(page, struct page,
72 p_vmp)); 72 p_vmp));
73 } else { 73 } else {
74 drm_md_clflush_all(); 74 drm_md_clflush_all();
75 } 75 }
76} 76}
77 77
78void 78void
79drm_clflush_page(struct page *page) 79drm_clflush_page(struct page *page)
80{ 80{
81 81
82 if (drm_md_clflush_finegrained_p()) 82 if (drm_md_clflush_finegrained_p())
83 drm_md_clflush_page(page); 83 drm_md_clflush_page(page);
84 else 84 else
85 drm_md_clflush_all(); 85 drm_md_clflush_all();
86} 86}
87 87
88void 88void
89drm_clflush_virt_range(const void *vaddr, size_t nbytes) 89drm_clflush_virt_range(const void *vaddr, size_t nbytes)
90{ 90{
91 91
92 if (drm_md_clflush_finegrained_p()) 92 if (drm_md_clflush_finegrained_p())
93 drm_md_clflush_virt_range(vaddr, nbytes); 93 drm_md_clflush_virt_range(vaddr, nbytes);
94 else 94 else
95 drm_md_clflush_all(); 95 drm_md_clflush_all();
96} 96}
97 97
98#if defined(__i386__) || defined(__x86_64__) 98#if defined(__i386__) || defined(__x86_64__)
99 99
100static bool 100static bool
101drm_md_clflush_finegrained_p(void) 101drm_md_clflush_finegrained_p(void)
102{ 102{
103 return ISSET(cpu_info_primary.ci_feat_val[0], CPUID_CFLUSH); 103 return ISSET(cpu_info_primary.ci_feat_val[0], CPUID_CFLUSH);
104} 104}
105 105
106static void 106static void
107drm_x86_clflush_cpu(void) 107drm_x86_clflush_cpu(void)
108{ 108{
109 asm volatile ("wbinvd"); 109 asm volatile ("wbinvd");
110} 110}
111 111
112static void 112static void
113drm_x86_clflush(const void *vaddr) 113drm_x86_clflush(const void *vaddr)
114{ 114{
115 asm volatile ("clflush %0" : : "m" (*(const char *)vaddr)); 115 asm volatile ("clflush %0" : : "m" (*(const char *)vaddr));
116} 116}
117 117
118static size_t 118static size_t
119drm_x86_clflush_size(void) 119drm_x86_clflush_size(void)
120{ 120{
121 KASSERT(drm_md_clflush_finegrained_p()); 121 KASSERT(drm_md_clflush_finegrained_p());
122 return cpu_info_primary.ci_cflush_lsize; 122 return cpu_info_primary.ci_cflush_lsize;
123} 123}
124 124
125static void 125static void
126drm_x86_clflush_xc(void *arg0 __unused, void *arg1 __unused) 126drm_x86_clflush_xc(void *arg0 __unused, void *arg1 __unused)
127{ 127{
128 drm_x86_clflush_cpu(); 128 drm_x86_clflush_cpu();
129} 129}
130 130
131static void 131static void
132drm_md_clflush_all(void) 132drm_md_clflush_all(void)
133{ 133{
134 xc_wait(xc_broadcast(0, &drm_x86_clflush_xc, NULL, NULL)); 134 xc_wait(xc_broadcast(0, &drm_x86_clflush_xc, NULL, NULL));
135} 135}
136 136
137static void 137static void
138drm_md_clflush_page(struct page *page) 138drm_md_clflush_page(struct page *page)
139{ 139{
140 void *const vaddr = kmap_atomic(page); 140 void *const vaddr = kmap_atomic(page);
141 141
142 drm_md_clflush_virt_range(vaddr, PAGE_SIZE); 142 drm_md_clflush_virt_range(vaddr, PAGE_SIZE);
143 143
144 kunmap_atomic(vaddr); 144 kunmap_atomic(vaddr);
145} 145}
146 146
147static void 147static void
148drm_md_clflush_virt_range(const void *vaddr, size_t nbytes) 148drm_md_clflush_virt_range(const void *vaddr, size_t nbytes)
149 
150{ 149{
151 const char *const start = vaddr, *const end = (start + nbytes); 150 const unsigned clflush_size = drm_x86_clflush_size();
 151 const vaddr_t va = (vaddr_t)vaddr;
 152 const char *const start = (const void *)rounddown(va, clflush_size);
 153 const char *const end = (const void *)roundup(va + nbytes,
 154 clflush_size);
152 const char *p; 155 const char *p;
153 const unsigned int clflush_size = drm_x86_clflush_size(); 
154 156
155 KASSERT(drm_md_clflush_finegrained_p()); 157 KASSERT(drm_md_clflush_finegrained_p());
 158 membar_sync();
156 for (p = start; p < end; p += clflush_size) 159 for (p = start; p < end; p += clflush_size)
157 drm_x86_clflush(p); 160 drm_x86_clflush(p);
 161 membar_sync();
158} 162}
159 163
160#endif /* defined(__i386__) || defined(__x86_64__) */ 164#endif /* defined(__i386__) || defined(__x86_64__) */