flush.c 10.3 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12
/*
 *  linux/arch/arm/mm/flush.c
 *
 *  Copyright (C) 1995-2002 Russell King
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/pagemap.h>
13
#include <linux/highmem.h>
L
Linus Torvalds 已提交
14 15

#include <asm/cacheflush.h>
16
#include <asm/cachetype.h>
17
#include <asm/highmem.h>
R
Russell King 已提交
18
#include <asm/smp_plat.h>
19
#include <asm/tlbflush.h>
20
#include <linux/hugetlb.h>
21

22 23
#include "mm.h"

24
#ifdef CONFIG_CPU_CACHE_VIPT
R
Russell King 已提交
25

26 27
static void flush_pfn_alias(unsigned long pfn, unsigned long vaddr)
{
28
	unsigned long to = FLUSH_ALIAS_START + (CACHE_COLOUR(vaddr) << PAGE_SHIFT);
29
	const int zero = 0;
30

31
	set_top_pte(to, pfn_pte(pfn, PAGE_KERNEL));
32 33

	asm(	"mcrr	p15, 0, %1, %0, c14\n"
R
Russell King 已提交
34
	"	mcr	p15, 0, %2, c7, c10, 4"
35
	    :
36
	    : "r" (to), "r" (to + PAGE_SIZE - L1_CACHE_BYTES), "r" (zero)
37 38 39
	    : "cc");
}

40 41
static void flush_icache_alias(unsigned long pfn, unsigned long vaddr, unsigned long len)
{
42
	unsigned long va = FLUSH_ALIAS_START + (CACHE_COLOUR(vaddr) << PAGE_SHIFT);
43 44 45
	unsigned long offset = vaddr & (PAGE_SIZE - 1);
	unsigned long to;

46 47
	set_top_pte(va, pfn_pte(pfn, PAGE_KERNEL));
	to = va + offset;
48 49 50
	flush_icache_range(to, to + len);
}

R
Russell King 已提交
51 52 53
void flush_cache_mm(struct mm_struct *mm)
{
	if (cache_is_vivt()) {
54
		vivt_flush_cache_mm(mm);
R
Russell King 已提交
55 56 57 58 59
		return;
	}

	if (cache_is_vipt_aliasing()) {
		asm(	"mcr	p15, 0, %0, c7, c14, 0\n"
R
Russell King 已提交
60
		"	mcr	p15, 0, %0, c7, c10, 4"
R
Russell King 已提交
61 62 63 64 65 66 67 68 69
		    :
		    : "r" (0)
		    : "cc");
	}
}

void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
{
	if (cache_is_vivt()) {
70
		vivt_flush_cache_range(vma, start, end);
R
Russell King 已提交
71 72 73 74 75
		return;
	}

	if (cache_is_vipt_aliasing()) {
		asm(	"mcr	p15, 0, %0, c7, c14, 0\n"
R
Russell King 已提交
76
		"	mcr	p15, 0, %0, c7, c10, 4"
R
Russell King 已提交
77 78 79 80
		    :
		    : "r" (0)
		    : "cc");
	}
81

82
	if (vma->vm_flags & VM_EXEC)
83
		__flush_icache_all();
R
Russell King 已提交
84 85 86 87 88
}

void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsigned long pfn)
{
	if (cache_is_vivt()) {
89
		vivt_flush_cache_page(vma, user_addr, pfn);
R
Russell King 已提交
90 91 92
		return;
	}

93
	if (cache_is_vipt_aliasing()) {
R
Russell King 已提交
94
		flush_pfn_alias(pfn, user_addr);
95 96
		__flush_icache_all();
	}
97 98 99

	if (vma->vm_flags & VM_EXEC && icache_is_vivt_asid_tagged())
		__flush_icache_all();
R
Russell King 已提交
100
}
101

R
Russell King 已提交
102
#else
103 104
#define flush_pfn_alias(pfn,vaddr)		do { } while (0)
#define flush_icache_alias(pfn,vaddr,len)	do { } while (0)
R
Russell King 已提交
105
#endif
106

107 108 109
#define FLAG_PA_IS_EXEC 1
#define FLAG_PA_CORE_IN_MM 2

R
Russell King 已提交
110 111 112 113 114
static void flush_ptrace_access_other(void *args)
{
	__flush_icache_all();
}

115 116 117
static inline
void __flush_ptrace_access(struct page *page, unsigned long uaddr, void *kaddr,
			   unsigned long len, unsigned int flags)
118 119
{
	if (cache_is_vivt()) {
120
		if (flags & FLAG_PA_CORE_IN_MM) {
R
Russell King 已提交
121 122 123
			unsigned long addr = (unsigned long)kaddr;
			__cpuc_coherent_kern_range(addr, addr + len);
		}
124 125 126 127 128
		return;
	}

	if (cache_is_vipt_aliasing()) {
		flush_pfn_alias(page_to_pfn(page), uaddr);
129
		__flush_icache_all();
130 131 132
		return;
	}

133
	/* VIPT non-aliasing D-cache */
134
	if (flags & FLAG_PA_IS_EXEC) {
135
		unsigned long addr = (unsigned long)kaddr;
136 137 138 139
		if (icache_is_vipt_aliasing())
			flush_icache_alias(page_to_pfn(page), uaddr, len);
		else
			__cpuc_coherent_kern_range(addr, addr + len);
R
Russell King 已提交
140 141 142
		if (cache_ops_need_broadcast())
			smp_call_function(flush_ptrace_access_other,
					  NULL, 1);
143 144
	}
}
R
Russell King 已提交
145

146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165
static
void flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
			 unsigned long uaddr, void *kaddr, unsigned long len)
{
	unsigned int flags = 0;
	if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm)))
		flags |= FLAG_PA_CORE_IN_MM;
	if (vma->vm_flags & VM_EXEC)
		flags |= FLAG_PA_IS_EXEC;
	__flush_ptrace_access(page, uaddr, kaddr, len, flags);
}

void flush_uprobe_xol_access(struct page *page, unsigned long uaddr,
			     void *kaddr, unsigned long len)
{
	unsigned int flags = FLAG_PA_CORE_IN_MM|FLAG_PA_IS_EXEC;

	__flush_ptrace_access(page, uaddr, kaddr, len, flags);
}

R
Russell King 已提交
166 167 168 169 170 171 172 173 174 175 176 177 178
/*
 * Copy user data from/to a page which is mapped into a different
 * processes address space.  Really, we want to allow our "user
 * space" model to handle this.
 *
 * Note that this code needs to run on the current CPU.
 */
void copy_to_user_page(struct vm_area_struct *vma, struct page *page,
		       unsigned long uaddr, void *dst, const void *src,
		       unsigned long len)
{
#ifdef CONFIG_SMP
	preempt_disable();
179
#endif
R
Russell King 已提交
180 181 182 183 184 185
	memcpy(dst, src, len);
	flush_ptrace_access(vma, page, uaddr, dst, len);
#ifdef CONFIG_SMP
	preempt_enable();
#endif
}
L
Linus Torvalds 已提交
186

187
void __flush_dcache_page(struct address_space *mapping, struct page *page)
L
Linus Torvalds 已提交
188 189 190 191 192 193
{
	/*
	 * Writeback any data associated with the kernel mapping of this
	 * page.  This ensures that data in the physical page is mutually
	 * coherent with the kernels mapping.
	 */
194
	if (!PageHighMem(page)) {
195 196
		size_t page_size = PAGE_SIZE << compound_order(page);
		__cpuc_flush_dcache_area(page_address(page), page_size);
197
	} else {
198
		unsigned long i;
199
		if (cache_is_vipt_nonaliasing()) {
200
			for (i = 0; i < (1 << compound_order(page)); i++) {
201
				void *addr = kmap_atomic(page + i);
202
				__cpuc_flush_dcache_area(addr, PAGE_SIZE);
203 204 205 206
				kunmap_atomic(addr);
			}
		} else {
			for (i = 0; i < (1 << compound_order(page)); i++) {
207
				void *addr = kmap_high_get(page + i);
208 209
				if (addr) {
					__cpuc_flush_dcache_area(addr, PAGE_SIZE);
210
					kunmap_high(page + i);
211
				}
212
			}
213 214
		}
	}
L
Linus Torvalds 已提交
215 216

	/*
217 218
	 * If this is a page cache page, and we have an aliasing VIPT cache,
	 * we only need to do one flush - which would be at the relevant
219 220
	 * userspace colour, which is congruent with page->index.
	 */
221
	if (mapping && cache_is_vipt_aliasing())
222 223 224 225 226 227 228 229 230
		flush_pfn_alias(page_to_pfn(page),
				page->index << PAGE_CACHE_SHIFT);
}

static void __flush_dcache_aliases(struct address_space *mapping, struct page *page)
{
	struct mm_struct *mm = current->active_mm;
	struct vm_area_struct *mpnt;
	pgoff_t pgoff;
231

L
Linus Torvalds 已提交
232 233 234 235 236 237 238 239 240
	/*
	 * There are possible user space mappings of this page:
	 * - VIVT cache: we need to also write back and invalidate all user
	 *   data in the current VM view associated with this page.
	 * - aliasing VIPT: we only need to find one mapping of this page.
	 */
	pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);

	flush_dcache_mmap_lock(mapping);
241
	vma_interval_tree_foreach(mpnt, &mapping->i_mmap, pgoff, pgoff) {
L
Linus Torvalds 已提交
242 243 244 245 246 247 248 249 250 251 252 253 254 255 256
		unsigned long offset;

		/*
		 * If this VMA is not in our MM, we can ignore it.
		 */
		if (mpnt->vm_mm != mm)
			continue;
		if (!(mpnt->vm_flags & VM_MAYSHARE))
			continue;
		offset = (pgoff - mpnt->vm_pgoff) << PAGE_SHIFT;
		flush_cache_page(mpnt, mpnt->vm_start + offset, page_to_pfn(page));
	}
	flush_dcache_mmap_unlock(mapping);
}

257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278
#if __LINUX_ARM_ARCH__ >= 6
void __sync_icache_dcache(pte_t pteval)
{
	unsigned long pfn;
	struct page *page;
	struct address_space *mapping;

	if (cache_is_vipt_nonaliasing() && !pte_exec(pteval))
		/* only flush non-aliasing VIPT caches for exec mappings */
		return;
	pfn = pte_pfn(pteval);
	if (!pfn_valid(pfn))
		return;

	page = pfn_to_page(pfn);
	if (cache_is_vipt_aliasing())
		mapping = page_mapping(page);
	else
		mapping = NULL;

	if (!test_and_set_bit(PG_dcache_clean, &page->flags))
		__flush_dcache_page(mapping, page);
279 280

	if (pte_exec(pteval))
281 282 283 284
		__flush_icache_all();
}
#endif

L
Linus Torvalds 已提交
285 286 287 288 289 290 291 292 293 294 295 296 297 298 299
/*
 * Ensure cache coherency between kernel mapping and userspace mapping
 * of this page.
 *
 * We have three cases to consider:
 *  - VIPT non-aliasing cache: fully coherent so nothing required.
 *  - VIVT: fully aliasing, so we need to handle every alias in our
 *          current VM view.
 *  - VIPT aliasing: need to handle one alias in our current VM view.
 *
 * If we need to handle aliasing:
 *  If the page only exists in the page cache and there are no user
 *  space mappings, we can be lazy and remember that we may have dirty
 *  kernel cache lines for later.  Otherwise, we assume we have
 *  aliasing mappings.
300
 *
301 302
 * Note that we disable the lazy flush for SMP configurations where
 * the cache maintenance operations are not automatically broadcasted.
L
Linus Torvalds 已提交
303 304 305
 */
void flush_dcache_page(struct page *page)
{
306 307 308 309 310 311 312 313 314 315
	struct address_space *mapping;

	/*
	 * The zero page is never written to, so never has any dirty
	 * cache lines, and therefore never needs to be flushed.
	 */
	if (page == ZERO_PAGE(0))
		return;

	mapping = page_mapping(page);
L
Linus Torvalds 已提交
316

317
	if (!cache_ops_need_broadcast() &&
318
	    mapping && !page_mapped(page))
319
		clear_bit(PG_dcache_clean, &page->flags);
320
	else {
L
Linus Torvalds 已提交
321
		__flush_dcache_page(mapping, page);
322 323
		if (mapping && cache_is_vivt())
			__flush_dcache_aliases(mapping, page);
324 325
		else if (mapping)
			__flush_icache_all();
326
		set_bit(PG_dcache_clean, &page->flags);
327
	}
L
Linus Torvalds 已提交
328 329
}
EXPORT_SYMBOL(flush_dcache_page);
330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362

/*
 * Ensure cache coherency for the kernel mapping of this page. We can
 * assume that the page is pinned via kmap.
 *
 * If the page only exists in the page cache and there are no user
 * space mappings, this is a no-op since the page was already marked
 * dirty at creation.  Otherwise, we need to flush the dirty kernel
 * cache lines directly.
 */
void flush_kernel_dcache_page(struct page *page)
{
	if (cache_is_vivt() || cache_is_vipt_aliasing()) {
		struct address_space *mapping;

		mapping = page_mapping(page);

		if (!mapping || mapping_mapped(mapping)) {
			void *addr;

			addr = page_address(page);
			/*
			 * kmap_atomic() doesn't set the page virtual
			 * address for highmem pages, and
			 * kunmap_atomic() takes care of cache
			 * flushing already.
			 */
			if (!IS_ENABLED(CONFIG_HIGHMEM) || addr)
				__cpuc_flush_dcache_area(addr, PAGE_SIZE);
		}
	}
}
EXPORT_SYMBOL(flush_kernel_dcache_page);
363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392

/*
 * Flush an anonymous page so that users of get_user_pages()
 * can safely access the data.  The expected sequence is:
 *
 *  get_user_pages()
 *    -> flush_anon_page
 *  memcpy() to/from page
 *  if written to page, flush_dcache_page()
 */
void __flush_anon_page(struct vm_area_struct *vma, struct page *page, unsigned long vmaddr)
{
	unsigned long pfn;

	/* VIPT non-aliasing caches need do nothing */
	if (cache_is_vipt_nonaliasing())
		return;

	/*
	 * Write back and invalidate userspace mapping.
	 */
	pfn = page_to_pfn(page);
	if (cache_is_vivt()) {
		flush_cache_page(vma, vmaddr, pfn);
	} else {
		/*
		 * For aliasing VIPT, we can flush an alias of the
		 * userspace address only.
		 */
		flush_pfn_alias(pfn, vmaddr);
393
		__flush_icache_all();
394 395 396 397 398 399 400
	}

	/*
	 * Invalidate kernel mapping.  No data should be contained
	 * in this mapping of the page.  FIXME: this is overkill
	 * since we actually ask for a write-back and invalidate.
	 */
401
	__cpuc_flush_dcache_area(page_address(page), PAGE_SIZE);
402
}