p2m.c 37.1 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24
/*
 * Xen leaves the responsibility for maintaining p2m mappings to the
 * guests themselves, but it must also access and update the p2m array
 * during suspend/resume when all the pages are reallocated.
 *
 * The p2m table is logically a flat array, but we implement it as a
 * three-level tree to allow the address space to be sparse.
 *
 *                               Xen
 *                                |
 *     p2m_top              p2m_top_mfn
 *       /  \                   /   \
 * p2m_mid p2m_mid	p2m_mid_mfn p2m_mid_mfn
 *    / \      / \         /           /
 *  p2m p2m p2m p2m p2m p2m p2m ...
 *
 * The p2m_mid_mfn pages are mapped by p2m_top_mfn_p.
 *
 * The p2m_top and p2m_top_mfn levels are limited to 1 page, so the
 * maximum representable pseudo-physical address space is:
 *  P2M_TOP_PER_PAGE * P2M_MID_PER_PAGE * P2M_PER_PAGE pages
 *
 * P2M_PER_PAGE depends on the architecture, as a mfn is always
 * unsigned long (8 bytes on 64-bit, 4 bytes on 32), leading to
25
 * 512 and 1024 entries respectively.
26 27 28 29 30 31 32 33 34 35 36 37 38
 *
 * In short, these structures contain the Machine Frame Number (MFN) of the PFN.
 *
 * However not all entries are filled with MFNs. Specifically for all other
 * leaf entries, or for the top  root, or middle one, for which there is a void
 * entry, we assume it is  "missing". So (for example)
 *  pfn_to_mfn(0x90909090)=INVALID_P2M_ENTRY.
 *
 * We also have the possibility of setting 1-1 mappings on certain regions, so
 * that:
 *  pfn_to_mfn(0xc0000)=0xc0000
 *
 * The benefit of this is, that we can assume for non-RAM regions (think
39
 * PCI BARs, or ACPI spaces), we can create mappings easily because we
40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62
 * get the PFN value to match the MFN.
 *
 * For this to work efficiently we have one new page p2m_identity and
 * allocate (via reserved_brk) any other pages we need to cover the sides
 * (1GB or 4MB boundary violations). All entries in p2m_identity are set to
 * INVALID_P2M_ENTRY type (Xen toolstack only recognizes that and MFNs,
 * no other fancy value).
 *
 * On lookup we spot that the entry points to p2m_identity and return the
 * identity value instead of dereferencing and returning INVALID_P2M_ENTRY.
 * If the entry points to an allocated page, we just proceed as before and
 * return the PFN.  If the PFN has IDENTITY_FRAME_BIT set we unmask that in
 * appropriate functions (pfn_to_mfn).
 *
 * The reason for having the IDENTITY_FRAME_BIT instead of just returning the
 * PFN is that we could find ourselves where pfn_to_mfn(pfn)==pfn for a
 * non-identity pfn. To protect ourselves against we elect to set (and get) the
 * IDENTITY_FRAME_BIT on all identity mapped PFNs.
 *
 * This simplistic diagram is used to explain the more subtle piece of code.
 * There is also a digram of the P2M at the end that can help.
 * Imagine your E820 looking as so:
 *
63
 *                    1GB                                           2GB    4GB
64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79
 * /-------------------+---------\/----\         /----------\    /---+-----\
 * | System RAM        | Sys RAM ||ACPI|         | reserved |    | Sys RAM |
 * \-------------------+---------/\----/         \----------/    \---+-----/
 *                               ^- 1029MB                       ^- 2001MB
 *
 * [1029MB = 263424 (0x40500), 2001MB = 512256 (0x7D100),
 *  2048MB = 524288 (0x80000)]
 *
 * And dom0_mem=max:3GB,1GB is passed in to the guest, meaning memory past 1GB
 * is actually not present (would have to kick the balloon driver to put it in).
 *
 * When we are told to set the PFNs for identity mapping (see patch: "xen/setup:
 * Set identity mapping for non-RAM E820 and E820 gaps.") we pass in the start
 * of the PFN and the end PFN (263424 and 512256 respectively). The first step
 * is to reserve_brk a top leaf page if the p2m[1] is missing. The top leaf page
 * covers 512^2 of page estate (1GB) and in case the start or end PFN is not
80 81
 * aligned on 512^2*PAGE_SIZE (1GB) we reserve_brk new middle and leaf pages as
 * required to split any existing p2m_mid_missing middle pages.
82 83 84 85 86 87 88 89
 *
 * With the E820 example above, 263424 is not 1GB aligned so we allocate a
 * reserve_brk page which will cover the PFNs estate from 0x40000 to 0x80000.
 * Each entry in the allocate page is "missing" (points to p2m_missing).
 *
 * Next stage is to determine if we need to do a more granular boundary check
 * on the 4MB (or 2MB depending on architecture) off the start and end pfn's.
 * We check if the start pfn and end pfn violate that boundary check, and if
90
 * so reserve_brk a (p2m[x][y]) leaf page. This way we have a much finer
91 92 93 94 95 96 97 98 99 100 101 102 103
 * granularity of setting which PFNs are missing and which ones are identity.
 * In our example 263424 and 512256 both fail the check so we reserve_brk two
 * pages. Populate them with INVALID_P2M_ENTRY (so they both have "missing"
 * values) and assign them to p2m[1][2] and p2m[1][488] respectively.
 *
 * At this point we would at minimum reserve_brk one page, but could be up to
 * three. Each call to set_phys_range_identity has at maximum a three page
 * cost. If we were to query the P2M at this stage, all those entries from
 * start PFN through end PFN (so 1029MB -> 2001MB) would return
 * INVALID_P2M_ENTRY ("missing").
 *
 * The next step is to walk from the start pfn to the end pfn setting
 * the IDENTITY_FRAME_BIT on each PFN. This is done in set_phys_range_identity.
104 105 106 107
 * If we find that the middle entry is pointing to p2m_missing we can swap it
 * over to p2m_identity - this way covering 4MB (or 2MB) PFN space (and
 * similarly swapping p2m_mid_missing for p2m_mid_identity for larger regions).
 * At this point we do not need to worry about boundary aligment (so no need to
108 109 110 111 112 113 114 115 116 117 118 119 120
 * reserve_brk a middle page, figure out which PFNs are "missing" and which
 * ones are identity), as that has been done earlier.  If we find that the
 * middle leaf is not occupied by p2m_identity or p2m_missing, we dereference
 * that page (which covers 512 PFNs) and set the appropriate PFN with
 * IDENTITY_FRAME_BIT. In our example 263424 and 512256 end up there, and we
 * set from p2m[1][2][256->511] and p2m[1][488][0->256] with
 * IDENTITY_FRAME_BIT set.
 *
 * All other regions that are void (or not filled) either point to p2m_missing
 * (considered missing) or have the default value of INVALID_P2M_ENTRY (also
 * considered missing). In our case, p2m[1][2][0->255] and p2m[1][488][257->511]
 * contain the INVALID_P2M_ENTRY value and are considered "missing."
 *
121 122 123
 * Finally, the region beyond the end of of the E820 (4 GB in this example)
 * is set to be identity (in case there are MMIO regions placed here).
 *
124 125 126 127 128 129 130 131 132 133 134
 * This is what the p2m ends up looking (for the E820 above) with this
 * fabulous drawing:
 *
 *    p2m         /--------------\
 *  /-----\       | &mfn_list[0],|                           /-----------------\
 *  |  0  |------>| &mfn_list[1],|    /---------------\      | ~0, ~0, ..      |
 *  |-----|       |  ..., ~0, ~0 |    | ~0, ~0, [x]---+----->| IDENTITY [@256] |
 *  |  1  |---\   \--------------/    | [p2m_identity]+\     | IDENTITY [@257] |
 *  |-----|    \                      | [p2m_identity]+\\    | ....            |
 *  |  2  |--\  \-------------------->|  ...          | \\   \----------------/
 *  |-----|   \                       \---------------/  \\
135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155
 *  |  3  |-\  \                                          \\  p2m_identity [1]
 *  |-----|  \  \-------------------->/---------------\   /-----------------\
 *  | ..  |\  |                       | [p2m_identity]+-->| ~0, ~0, ~0, ... |
 *  \-----/ | |                       | [p2m_identity]+-->| ..., ~0         |
 *          | |                       | ....          |   \-----------------/
 *          | |                       +-[x], ~0, ~0.. +\
 *          | |                       \---------------/ \
 *          | |                                          \-> /---------------\
 *          | V  p2m_mid_missing       p2m_missing           | IDENTITY[@0]  |
 *          | /-----------------\     /------------\         | IDENTITY[@256]|
 *          | | [p2m_missing]   +---->| ~0, ~0, ...|         | ~0, ~0, ....  |
 *          | | [p2m_missing]   +---->| ..., ~0    |         \---------------/
 *          | | ...             |     \------------/
 *          | \-----------------/
 *          |
 *          |     p2m_mid_identity
 *          |   /-----------------\
 *          \-->| [p2m_identity]  +---->[1]
 *              | [p2m_identity]  +---->[1]
 *              | ...             |
 *              \-----------------/
156 157
 *
 * where ~0 is INVALID_P2M_ENTRY. IDENTITY is (PFN | IDENTITY_BIT)
158 159 160 161
 */

#include <linux/init.h>
#include <linux/module.h>
162 163
#include <linux/list.h>
#include <linux/hash.h>
164
#include <linux/sched.h>
165
#include <linux/seq_file.h>
166
#include <linux/bootmem.h>
167 168 169 170 171 172 173

#include <asm/cache.h>
#include <asm/setup.h>

#include <asm/xen/page.h>
#include <asm/xen/hypercall.h>
#include <asm/xen/hypervisor.h>
174
#include <xen/balloon.h>
175
#include <xen/grant_table.h>
176

177
#include "p2m.h"
178
#include "multicalls.h"
179 180
#include "xen-ops.h"

181 182
static void __init m2p_override_init(void);

183 184
unsigned long xen_max_p2m_pfn __read_mostly;

185 186 187 188
static unsigned long *p2m_mid_missing_mfn;
static unsigned long *p2m_top_mfn;
static unsigned long **p2m_top_mfn_p;

189 190 191 192 193 194
/* Placeholders for holes in the address space */
static RESERVE_BRK_ARRAY(unsigned long, p2m_missing, P2M_PER_PAGE);
static RESERVE_BRK_ARRAY(unsigned long *, p2m_mid_missing, P2M_MID_PER_PAGE);

static RESERVE_BRK_ARRAY(unsigned long **, p2m_top, P2M_TOP_PER_PAGE);

195
static RESERVE_BRK_ARRAY(unsigned long, p2m_identity, P2M_PER_PAGE);
196
static RESERVE_BRK_ARRAY(unsigned long *, p2m_mid_identity, P2M_MID_PER_PAGE);
197

198 199
RESERVE_BRK(p2m_mid, PAGE_SIZE * (MAX_DOMAIN_PAGES / (P2M_PER_PAGE * P2M_MID_PER_PAGE)));

200 201 202 203 204 205
/* For each I/O range remapped we may lose up to two leaf pages for the boundary
 * violations and three mid pages to cover up to 3GB. With
 * early_can_reuse_p2m_middle() most of the leaf pages will be reused by the
 * remapped region.
 */
RESERVE_BRK(p2m_identity_remap, PAGE_SIZE * 2 * 3 * MAX_REMAP_RANGES);
206

207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246
static inline unsigned p2m_top_index(unsigned long pfn)
{
	BUG_ON(pfn >= MAX_P2M_PFN);
	return pfn / (P2M_MID_PER_PAGE * P2M_PER_PAGE);
}

static inline unsigned p2m_mid_index(unsigned long pfn)
{
	return (pfn / P2M_PER_PAGE) % P2M_MID_PER_PAGE;
}

static inline unsigned p2m_index(unsigned long pfn)
{
	return pfn % P2M_PER_PAGE;
}

static void p2m_top_init(unsigned long ***top)
{
	unsigned i;

	for (i = 0; i < P2M_TOP_PER_PAGE; i++)
		top[i] = p2m_mid_missing;
}

static void p2m_top_mfn_init(unsigned long *top)
{
	unsigned i;

	for (i = 0; i < P2M_TOP_PER_PAGE; i++)
		top[i] = virt_to_mfn(p2m_mid_missing_mfn);
}

static void p2m_top_mfn_p_init(unsigned long **top)
{
	unsigned i;

	for (i = 0; i < P2M_TOP_PER_PAGE; i++)
		top[i] = p2m_mid_missing_mfn;
}

247
static void p2m_mid_init(unsigned long **mid, unsigned long *leaf)
248 249 250 251
{
	unsigned i;

	for (i = 0; i < P2M_MID_PER_PAGE; i++)
252
		mid[i] = leaf;
253 254
}

255
static void p2m_mid_mfn_init(unsigned long *mid, unsigned long *leaf)
256 257 258 259
{
	unsigned i;

	for (i = 0; i < P2M_MID_PER_PAGE; i++)
260
		mid[i] = virt_to_mfn(leaf);
261 262 263 264 265 266 267 268 269 270 271 272 273 274
}

static void p2m_init(unsigned long *p2m)
{
	unsigned i;

	for (i = 0; i < P2M_MID_PER_PAGE; i++)
		p2m[i] = INVALID_P2M_ENTRY;
}

/*
 * Build the parallel p2m_top_mfn and p2m_mid_mfn structures
 *
 * This is called both at boot time, and after resuming from suspend:
275
 * - At boot time we're called rather early, and must use alloc_bootmem*()
276 277 278
 *   to allocate memory.
 *
 * - After resume we're called from within stop_machine, but the mfn
279
 *   tree should already be completely allocated.
280
 */
281
void __ref xen_build_mfn_list_list(void)
282 283 284
{
	unsigned long pfn;

285 286 287
	if (xen_feature(XENFEAT_auto_translated_physmap))
		return;

288 289
	/* Pre-initialize p2m_top_mfn to be completely missing */
	if (p2m_top_mfn == NULL) {
290
		p2m_mid_missing_mfn = alloc_bootmem_align(PAGE_SIZE, PAGE_SIZE);
291
		p2m_mid_mfn_init(p2m_mid_missing_mfn, p2m_missing);
292

293
		p2m_top_mfn_p = alloc_bootmem_align(PAGE_SIZE, PAGE_SIZE);
294 295
		p2m_top_mfn_p_init(p2m_top_mfn_p);

296
		p2m_top_mfn = alloc_bootmem_align(PAGE_SIZE, PAGE_SIZE);
297 298 299
		p2m_top_mfn_init(p2m_top_mfn);
	} else {
		/* Reinitialise, mfn's all change after migration */
300
		p2m_mid_mfn_init(p2m_mid_missing_mfn, p2m_missing);
301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327
	}

	for (pfn = 0; pfn < xen_max_p2m_pfn; pfn += P2M_PER_PAGE) {
		unsigned topidx = p2m_top_index(pfn);
		unsigned mididx = p2m_mid_index(pfn);
		unsigned long **mid;
		unsigned long *mid_mfn_p;

		mid = p2m_top[topidx];
		mid_mfn_p = p2m_top_mfn_p[topidx];

		/* Don't bother allocating any mfn mid levels if
		 * they're just missing, just update the stored mfn,
		 * since all could have changed over a migrate.
		 */
		if (mid == p2m_mid_missing) {
			BUG_ON(mididx);
			BUG_ON(mid_mfn_p != p2m_mid_missing_mfn);
			p2m_top_mfn[topidx] = virt_to_mfn(p2m_mid_missing_mfn);
			pfn += (P2M_MID_PER_PAGE - 1) * P2M_PER_PAGE;
			continue;
		}

		if (mid_mfn_p == p2m_mid_missing_mfn) {
			/*
			 * XXX boot-time only!  We should never find
			 * missing parts of the mfn tree after
328
			 * runtime.
329
			 */
330
			mid_mfn_p = alloc_bootmem_align(PAGE_SIZE, PAGE_SIZE);
331
			p2m_mid_mfn_init(mid_mfn_p, p2m_missing);
332 333 334 335 336 337 338 339 340 341 342

			p2m_top_mfn_p[topidx] = mid_mfn_p;
		}

		p2m_top_mfn[topidx] = virt_to_mfn(mid_mfn_p);
		mid_mfn_p[mididx] = virt_to_mfn(mid[mididx]);
	}
}

void xen_setup_mfn_list_list(void)
{
M
Mukesh Rathor 已提交
343 344 345
	if (xen_feature(XENFEAT_auto_translated_physmap))
		return;

346 347 348 349 350 351 352 353 354 355
	BUG_ON(HYPERVISOR_shared_info == &xen_dummy_shared_info);

	HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list =
		virt_to_mfn(p2m_top_mfn);
	HYPERVISOR_shared_info->arch.max_pfn = xen_max_p2m_pfn;
}

/* Set up p2m_top to point to the domain-builder provided p2m pages */
void __init xen_build_dynamic_phys_to_machine(void)
{
356 357
	unsigned long *mfn_list;
	unsigned long max_pfn;
358 359
	unsigned long pfn;

360 361 362 363 364
	 if (xen_feature(XENFEAT_auto_translated_physmap))
		return;

	mfn_list = (unsigned long *)xen_start_info->mfn_list;
	max_pfn = min(MAX_DOMAIN_PAGES, xen_start_info->nr_pages);
365 366 367 368
	xen_max_p2m_pfn = max_pfn;

	p2m_missing = extend_brk(PAGE_SIZE, PAGE_SIZE);
	p2m_init(p2m_missing);
369 370
	p2m_identity = extend_brk(PAGE_SIZE, PAGE_SIZE);
	p2m_init(p2m_identity);
371 372

	p2m_mid_missing = extend_brk(PAGE_SIZE, PAGE_SIZE);
373 374 375
	p2m_mid_init(p2m_mid_missing, p2m_missing);
	p2m_mid_identity = extend_brk(PAGE_SIZE, PAGE_SIZE);
	p2m_mid_init(p2m_mid_identity, p2m_identity);
376 377 378 379 380 381 382 383 384 385 386 387 388 389 390

	p2m_top = extend_brk(PAGE_SIZE, PAGE_SIZE);
	p2m_top_init(p2m_top);

	/*
	 * The domain builder gives us a pre-constructed p2m array in
	 * mfn_list for all the pages initially given to us, so we just
	 * need to graft that into our tree structure.
	 */
	for (pfn = 0; pfn < max_pfn; pfn += P2M_PER_PAGE) {
		unsigned topidx = p2m_top_index(pfn);
		unsigned mididx = p2m_mid_index(pfn);

		if (p2m_top[topidx] == p2m_mid_missing) {
			unsigned long **mid = extend_brk(PAGE_SIZE, PAGE_SIZE);
391
			p2m_mid_init(mid, p2m_missing);
392 393 394 395

			p2m_top[topidx] = mid;
		}

396 397 398 399 400 401 402
		/*
		 * As long as the mfn_list has enough entries to completely
		 * fill a p2m page, pointing into the array is ok. But if
		 * not the entries beyond the last pfn will be undefined.
		 */
		if (unlikely(pfn + P2M_PER_PAGE > max_pfn)) {
			unsigned long p2midx;
403 404 405 406 407 408

			p2midx = max_pfn % P2M_PER_PAGE;
			for ( ; p2midx < P2M_PER_PAGE; p2midx++)
				mfn_list[pfn + p2midx] = INVALID_P2M_ENTRY;
		}
		p2m_top[topidx][mididx] = &mfn_list[pfn];
409
	}
410 411

	m2p_override_init();
412
}
413 414 415 416 417 418
#ifdef CONFIG_X86_64
unsigned long __init xen_revector_p2m_tree(void)
{
	unsigned long va_start;
	unsigned long va_end;
	unsigned long pfn;
419
	unsigned long pfn_free = 0;
420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447
	unsigned long *mfn_list = NULL;
	unsigned long size;

	va_start = xen_start_info->mfn_list;
	/*We copy in increments of P2M_PER_PAGE * sizeof(unsigned long),
	 * so make sure it is rounded up to that */
	size = PAGE_ALIGN(xen_start_info->nr_pages * sizeof(unsigned long));
	va_end = va_start + size;

	/* If we were revectored already, don't do it again. */
	if (va_start <= __START_KERNEL_map && va_start >= __PAGE_OFFSET)
		return 0;

	mfn_list = alloc_bootmem_align(size, PAGE_SIZE);
	if (!mfn_list) {
		pr_warn("Could not allocate space for a new P2M tree!\n");
		return xen_start_info->mfn_list;
	}
	/* Fill it out with INVALID_P2M_ENTRY value */
	memset(mfn_list, 0xFF, size);

	for (pfn = 0; pfn < ALIGN(MAX_DOMAIN_PAGES, P2M_PER_PAGE); pfn += P2M_PER_PAGE) {
		unsigned topidx = p2m_top_index(pfn);
		unsigned mididx;
		unsigned long *mid_p;

		if (!p2m_top[topidx])
			continue;
448

449 450 451 452 453 454 455 456 457
		if (p2m_top[topidx] == p2m_mid_missing)
			continue;

		mididx = p2m_mid_index(pfn);
		mid_p = p2m_top[topidx][mididx];
		if (!mid_p)
			continue;
		if ((mid_p == p2m_missing) || (mid_p == p2m_identity))
			continue;
458

459 460 461 462 463 464 465
		if ((unsigned long)mid_p == INVALID_P2M_ENTRY)
			continue;

		/* The old va. Rebase it on mfn_list */
		if (mid_p >= (unsigned long *)va_start && mid_p <= (unsigned long *)va_end) {
			unsigned long *new;

466 467 468 469 470 471
			if (pfn_free  > (size / sizeof(unsigned long))) {
				WARN(1, "Only allocated for %ld pages, but we want %ld!\n",
				     size / sizeof(unsigned long), pfn_free);
				return 0;
			}
			new = &mfn_list[pfn_free];
472 473

			copy_page(new, mid_p);
474 475 476
			p2m_top[topidx][mididx] = &mfn_list[pfn_free];

			pfn_free += P2M_PER_PAGE;
477 478 479 480 481 482 483 484 485 486 487 488 489

		}
		/* This should be the leafs allocated for identity from _brk. */
	}
	return (unsigned long)mfn_list;

}
#else
unsigned long __init xen_revector_p2m_tree(void)
{
	return 0;
}
#endif
490 491 492 493 494
unsigned long get_phys_to_machine(unsigned long pfn)
{
	unsigned topidx, mididx, idx;

	if (unlikely(pfn >= MAX_P2M_PFN))
495
		return IDENTITY_FRAME(pfn);
496 497 498 499 500

	topidx = p2m_top_index(pfn);
	mididx = p2m_mid_index(pfn);
	idx = p2m_index(pfn);

501 502 503 504 505 506 507 508
	/*
	 * The INVALID_P2M_ENTRY is filled in both p2m_*identity
	 * and in p2m_*missing, so returning the INVALID_P2M_ENTRY
	 * would be wrong.
	 */
	if (p2m_top[topidx][mididx] == p2m_identity)
		return IDENTITY_FRAME(pfn);

509 510 511 512 513 514 515 516 517 518 519 520 521 522
	return p2m_top[topidx][mididx][idx];
}
EXPORT_SYMBOL_GPL(get_phys_to_machine);

static void *alloc_p2m_page(void)
{
	return (void *)__get_free_page(GFP_KERNEL | __GFP_REPEAT);
}

static void free_p2m_page(void *p)
{
	free_page((unsigned long)p);
}

523
/*
524 525 526 527 528 529 530 531 532 533 534
 * Fully allocate the p2m structure for a given pfn.  We need to check
 * that both the top and mid levels are allocated, and make sure the
 * parallel mfn tree is kept in sync.  We may race with other cpus, so
 * the new pages are installed with cmpxchg; if we lose the race then
 * simply free the page we allocated and use the one that's there.
 */
static bool alloc_p2m(unsigned long pfn)
{
	unsigned topidx, mididx;
	unsigned long ***top_p, **mid;
	unsigned long *top_mfn_p, *mid_mfn;
535
	unsigned long *p2m_orig;
536 537 538 539 540

	topidx = p2m_top_index(pfn);
	mididx = p2m_mid_index(pfn);

	top_p = &p2m_top[topidx];
541
	mid = ACCESS_ONCE(*top_p);
542 543 544 545 546 547 548

	if (mid == p2m_mid_missing) {
		/* Mid level is missing, allocate a new one */
		mid = alloc_p2m_page();
		if (!mid)
			return false;

549
		p2m_mid_init(mid, p2m_missing);
550 551 552 553 554 555

		if (cmpxchg(top_p, p2m_mid_missing, mid) != p2m_mid_missing)
			free_p2m_page(mid);
	}

	top_mfn_p = &p2m_top_mfn[topidx];
556
	mid_mfn = ACCESS_ONCE(p2m_top_mfn_p[topidx]);
557 558 559 560 561 562 563

	BUG_ON(virt_to_mfn(mid_mfn) != *top_mfn_p);

	if (mid_mfn == p2m_mid_missing_mfn) {
		/* Separately check the mid mfn level */
		unsigned long missing_mfn;
		unsigned long mid_mfn_mfn;
564
		unsigned long old_mfn;
565 566 567 568 569

		mid_mfn = alloc_p2m_page();
		if (!mid_mfn)
			return false;

570
		p2m_mid_mfn_init(mid_mfn, p2m_missing);
571 572 573

		missing_mfn = virt_to_mfn(p2m_mid_missing_mfn);
		mid_mfn_mfn = virt_to_mfn(mid_mfn);
574 575
		old_mfn = cmpxchg(top_mfn_p, missing_mfn, mid_mfn_mfn);
		if (old_mfn != missing_mfn) {
576
			free_p2m_page(mid_mfn);
577 578
			mid_mfn = mfn_to_virt(old_mfn);
		} else {
579
			p2m_top_mfn_p[topidx] = mid_mfn;
580
		}
581 582
	}

583 584
	p2m_orig = ACCESS_ONCE(p2m_top[topidx][mididx]);
	if (p2m_orig == p2m_identity || p2m_orig == p2m_missing) {
585 586 587 588 589 590 591 592 593
		/* p2m leaf page is missing */
		unsigned long *p2m;

		p2m = alloc_p2m_page();
		if (!p2m)
			return false;

		p2m_init(p2m);

594
		if (cmpxchg(&mid[mididx], p2m_orig, p2m) != p2m_orig)
595 596 597 598 599 600 601 602
			free_p2m_page(p2m);
		else
			mid_mfn[mididx] = virt_to_mfn(p2m);
	}

	return true;
}

603
static bool __init early_alloc_p2m(unsigned long pfn, bool check_boundary)
604 605
{
	unsigned topidx, mididx, idx;
606
	unsigned long *p2m;
607 608 609 610 611 612

	topidx = p2m_top_index(pfn);
	mididx = p2m_mid_index(pfn);
	idx = p2m_index(pfn);

	/* Pfff.. No boundary cross-over, lets get out. */
613
	if (!idx && check_boundary)
614 615 616 617 618 619 620 621 622 623 624 625 626
		return false;

	WARN(p2m_top[topidx][mididx] == p2m_identity,
		"P2M[%d][%d] == IDENTITY, should be MISSING (or alloced)!\n",
		topidx, mididx);

	/*
	 * Could be done by xen_build_dynamic_phys_to_machine..
	 */
	if (p2m_top[topidx][mididx] != p2m_missing)
		return false;

	/* Boundary cross-over for the edges: */
627
	p2m = extend_brk(PAGE_SIZE, PAGE_SIZE);
628

629
	p2m_init(p2m);
630

631
	p2m_top[topidx][mididx] = p2m;
632

633
	return true;
634
}
635

636
static bool __init early_alloc_p2m_middle(unsigned long pfn)
637 638 639 640 641 642 643 644
{
	unsigned topidx = p2m_top_index(pfn);
	unsigned long **mid;

	mid = p2m_top[topidx];
	if (mid == p2m_mid_missing) {
		mid = extend_brk(PAGE_SIZE, PAGE_SIZE);

645
		p2m_mid_init(mid, p2m_missing);
646 647 648 649 650

		p2m_top[topidx] = mid;
	}
	return true;
}
651 652 653 654 655 656 657

/*
 * Skim over the P2M tree looking at pages that are either filled with
 * INVALID_P2M_ENTRY or with 1:1 PFNs. If found, re-use that page and
 * replace the P2M leaf with a p2m_missing or p2m_identity.
 * Stick the old page in the new P2M tree location.
 */
658
static bool __init early_can_reuse_p2m_middle(unsigned long set_pfn)
659 660 661 662 663 664 665 666 667 668 669 670 671
{
	unsigned topidx;
	unsigned mididx;
	unsigned ident_pfns;
	unsigned inv_pfns;
	unsigned long *p2m;
	unsigned idx;
	unsigned long pfn;

	/* We only look when this entails a P2M middle layer */
	if (p2m_index(set_pfn))
		return false;

672
	for (pfn = 0; pfn < MAX_DOMAIN_PAGES; pfn += P2M_PER_PAGE) {
673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716
		topidx = p2m_top_index(pfn);

		if (!p2m_top[topidx])
			continue;

		if (p2m_top[topidx] == p2m_mid_missing)
			continue;

		mididx = p2m_mid_index(pfn);
		p2m = p2m_top[topidx][mididx];
		if (!p2m)
			continue;

		if ((p2m == p2m_missing) || (p2m == p2m_identity))
			continue;

		if ((unsigned long)p2m == INVALID_P2M_ENTRY)
			continue;

		ident_pfns = 0;
		inv_pfns = 0;
		for (idx = 0; idx < P2M_PER_PAGE; idx++) {
			/* IDENTITY_PFNs are 1:1 */
			if (p2m[idx] == IDENTITY_FRAME(pfn + idx))
				ident_pfns++;
			else if (p2m[idx] == INVALID_P2M_ENTRY)
				inv_pfns++;
			else
				break;
		}
		if ((ident_pfns == P2M_PER_PAGE) || (inv_pfns == P2M_PER_PAGE))
			goto found;
	}
	return false;
found:
	/* Found one, replace old with p2m_identity or p2m_missing */
	p2m_top[topidx][mididx] = (ident_pfns ? p2m_identity : p2m_missing);

	/* Reset where we want to stick the old page in. */
	topidx = p2m_top_index(set_pfn);
	mididx = p2m_mid_index(set_pfn);

	/* This shouldn't happen */
	if (WARN_ON(p2m_top[topidx] == p2m_mid_missing))
717
		early_alloc_p2m_middle(set_pfn);
718 719 720 721 722 723 724 725 726

	if (WARN_ON(p2m_top[topidx][mididx] != p2m_missing))
		return false;

	p2m_init(p2m);
	p2m_top[topidx][mididx] = p2m;

	return true;
}
727 728 729
bool __init early_set_phys_to_machine(unsigned long pfn, unsigned long mfn)
{
	if (unlikely(!__set_phys_to_machine(pfn, mfn)))  {
730
		if (!early_alloc_p2m_middle(pfn))
731 732
			return false;

733
		if (early_can_reuse_p2m_middle(pfn))
734 735
			return __set_phys_to_machine(pfn, mfn);

736
		if (!early_alloc_p2m(pfn, false /* boundary crossover OK!*/))
737 738 739 740 741 742 743 744
			return false;

		if (!__set_phys_to_machine(pfn, mfn))
			return false;
	}

	return true;
}
745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762

static void __init early_split_p2m(unsigned long pfn)
{
	unsigned long mididx, idx;

	mididx = p2m_mid_index(pfn);
	idx = p2m_index(pfn);

	/*
	 * Allocate new middle and leaf pages if this pfn lies in the
	 * middle of one.
	 */
	if (mididx || idx)
		early_alloc_p2m_middle(pfn);
	if (idx)
		early_alloc_p2m(pfn, false);
}

R
Randy Dunlap 已提交
763
unsigned long __init set_phys_range_identity(unsigned long pfn_s,
764 765 766 767
				      unsigned long pfn_e)
{
	unsigned long pfn;

768
	if (unlikely(pfn_s >= MAX_P2M_PFN))
769 770 771 772 773 774 775 776
		return 0;

	if (unlikely(xen_feature(XENFEAT_auto_translated_physmap)))
		return pfn_e - pfn_s;

	if (pfn_s > pfn_e)
		return 0;

777 778 779
	if (pfn_e > MAX_P2M_PFN)
		pfn_e = MAX_P2M_PFN;

780 781
	early_split_p2m(pfn_s);
	early_split_p2m(pfn_e);
782

783 784 785
	for (pfn = pfn_s; pfn < pfn_e;) {
		unsigned topidx = p2m_top_index(pfn);
		unsigned mididx = p2m_mid_index(pfn);
786 787 788

		if (!__set_phys_to_machine(pfn, IDENTITY_FRAME(pfn)))
			break;
789 790 791 792 793 794 795 796 797 798 799 800
		pfn++;

		/*
		 * If the PFN was set to a middle or leaf identity
		 * page the remainder must also be identity, so skip
		 * ahead to the next middle or leaf entry.
		 */
		if (p2m_top[topidx] == p2m_mid_identity)
			pfn = ALIGN(pfn, P2M_MID_PER_PAGE * P2M_PER_PAGE);
		else if (p2m_top[topidx][mididx] == p2m_identity)
			pfn = ALIGN(pfn, P2M_PER_PAGE);
	}
801

802
	WARN((pfn - pfn_s) != (pfn_e - pfn_s),
803
		"Identity mapping failed. We are %ld short of 1-1 mappings!\n",
804
		(pfn_e - pfn_s) - (pfn - pfn_s));
805 806 807 808

	return pfn - pfn_s;
}

809 810 811 812 813
/* Try to install p2m mapping; fail if intermediate bits missing */
bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn)
{
	unsigned topidx, mididx, idx;

814 815
	/* don't track P2M changes in autotranslate guests */
	if (unlikely(xen_feature(XENFEAT_auto_translated_physmap)))
816
		return true;
817

818 819 820 821 822 823 824 825 826
	if (unlikely(pfn >= MAX_P2M_PFN)) {
		BUG_ON(mfn != INVALID_P2M_ENTRY);
		return true;
	}

	topidx = p2m_top_index(pfn);
	mididx = p2m_mid_index(pfn);
	idx = p2m_index(pfn);

827 828
	/* For sparse holes were the p2m leaf has real PFN along with
	 * PCI holes, stick in the PFN as the MFN value.
829 830 831 832 833
	 *
	 * set_phys_range_identity() will have allocated new middle
	 * and leaf pages as required so an existing p2m_mid_missing
	 * or p2m_missing mean that whole range will be identity so
	 * these can be switched to p2m_mid_identity or p2m_identity.
834 835
	 */
	if (mfn != INVALID_P2M_ENTRY && (mfn & IDENTITY_FRAME_BIT)) {
836 837 838 839 840 841 842 843 844
		if (p2m_top[topidx] == p2m_mid_identity)
			return true;

		if (p2m_top[topidx] == p2m_mid_missing) {
			WARN_ON(cmpxchg(&p2m_top[topidx], p2m_mid_missing,
					p2m_mid_identity) != p2m_mid_missing);
			return true;
		}

845 846 847 848 849
		if (p2m_top[topidx][mididx] == p2m_identity)
			return true;

		/* Swap over from MISSING to IDENTITY if needed. */
		if (p2m_top[topidx][mididx] == p2m_missing) {
850 851
			WARN_ON(cmpxchg(&p2m_top[topidx][mididx], p2m_missing,
				p2m_identity) != p2m_missing);
852 853 854 855
			return true;
		}
	}

856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875
	if (p2m_top[topidx][mididx] == p2m_missing)
		return mfn == INVALID_P2M_ENTRY;

	p2m_top[topidx][mididx][idx] = mfn;

	return true;
}

bool set_phys_to_machine(unsigned long pfn, unsigned long mfn)
{
	if (unlikely(!__set_phys_to_machine(pfn, mfn)))  {
		if (!alloc_p2m(pfn))
			return false;

		if (!__set_phys_to_machine(pfn, mfn))
			return false;
	}

	return true;
}
876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898

#define M2P_OVERRIDE_HASH_SHIFT	10
#define M2P_OVERRIDE_HASH	(1 << M2P_OVERRIDE_HASH_SHIFT)

static RESERVE_BRK_ARRAY(struct list_head, m2p_overrides, M2P_OVERRIDE_HASH);
static DEFINE_SPINLOCK(m2p_override_lock);

static void __init m2p_override_init(void)
{
	unsigned i;

	m2p_overrides = extend_brk(sizeof(*m2p_overrides) * M2P_OVERRIDE_HASH,
				   sizeof(unsigned long));

	for (i = 0; i < M2P_OVERRIDE_HASH; i++)
		INIT_LIST_HEAD(&m2p_overrides[i]);
}

static unsigned long mfn_hash(unsigned long mfn)
{
	return hash_long(mfn, M2P_OVERRIDE_HASH_SHIFT);
}

899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957
int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops,
			    struct gnttab_map_grant_ref *kmap_ops,
			    struct page **pages, unsigned int count)
{
	int i, ret = 0;
	bool lazy = false;
	pte_t *pte;

	if (xen_feature(XENFEAT_auto_translated_physmap))
		return 0;

	if (kmap_ops &&
	    !in_interrupt() &&
	    paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) {
		arch_enter_lazy_mmu_mode();
		lazy = true;
	}

	for (i = 0; i < count; i++) {
		unsigned long mfn, pfn;

		/* Do not add to override if the map failed. */
		if (map_ops[i].status)
			continue;

		if (map_ops[i].flags & GNTMAP_contains_pte) {
			pte = (pte_t *) (mfn_to_virt(PFN_DOWN(map_ops[i].host_addr)) +
				(map_ops[i].host_addr & ~PAGE_MASK));
			mfn = pte_mfn(*pte);
		} else {
			mfn = PFN_DOWN(map_ops[i].dev_bus_addr);
		}
		pfn = page_to_pfn(pages[i]);

		WARN_ON(PagePrivate(pages[i]));
		SetPagePrivate(pages[i]);
		set_page_private(pages[i], mfn);
		pages[i]->index = pfn_to_mfn(pfn);

		if (unlikely(!set_phys_to_machine(pfn, FOREIGN_FRAME(mfn)))) {
			ret = -ENOMEM;
			goto out;
		}

		if (kmap_ops) {
			ret = m2p_add_override(mfn, pages[i], &kmap_ops[i]);
			if (ret)
				goto out;
		}
	}

out:
	if (lazy)
		arch_leave_lazy_mmu_mode();

	return ret;
}
EXPORT_SYMBOL_GPL(set_foreign_p2m_mapping);

958
/* Add an MFN override for a particular page */
959 960
int m2p_add_override(unsigned long mfn, struct page *page,
		struct gnttab_map_grant_ref *kmap_op)
961 962
{
	unsigned long flags;
963
	unsigned long pfn;
964
	unsigned long uninitialized_var(address);
965 966 967 968 969 970 971 972 973 974 975
	unsigned level;
	pte_t *ptep = NULL;

	pfn = page_to_pfn(page);
	if (!PageHighMem(page)) {
		address = (unsigned long)__va(pfn << PAGE_SHIFT);
		ptep = lookup_address(address, &level);
		if (WARN(ptep == NULL || level != PG_LEVEL_4K,
					"m2p_add_override: pfn %lx not mapped", pfn))
			return -EINVAL;
	}
976

977 978 979 980 981 982 983 984 985 986 987
	if (kmap_op != NULL) {
		if (!PageHighMem(page)) {
			struct multicall_space mcs =
				xen_mc_entry(sizeof(*kmap_op));

			MULTI_grant_table_op(mcs.mc,
					GNTTABOP_map_grant_ref, kmap_op, 1);

			xen_mc_issue(PARAVIRT_LAZY_MMU);
		}
	}
988 989 990
	spin_lock_irqsave(&m2p_override_lock, flags);
	list_add(&page->lru,  &m2p_overrides[mfn_hash(mfn)]);
	spin_unlock_irqrestore(&m2p_override_lock, flags);
991

992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005
	/* p2m(m2p(mfn)) == mfn: the mfn is already present somewhere in
	 * this domain. Set the FOREIGN_FRAME_BIT in the p2m for the other
	 * pfn so that the following mfn_to_pfn(mfn) calls will return the
	 * pfn from the m2p_override (the backend pfn) instead.
	 * We need to do this because the pages shared by the frontend
	 * (xen-blkfront) can be already locked (lock_page, called by
	 * do_read_cache_page); when the userspace backend tries to use them
	 * with direct_IO, mfn_to_pfn returns the pfn of the frontend, so
	 * do_blockdev_direct_IO is going to try to lock the same pages
	 * again resulting in a deadlock.
	 * As a side effect get_user_pages_fast might not be safe on the
	 * frontend pages while they are being shared with the backend,
	 * because mfn_to_pfn (that ends up being called by GUPF) will
	 * return the backend pfn rather than the frontend pfn. */
1006 1007
	pfn = mfn_to_pfn_no_overrides(mfn);
	if (get_phys_to_machine(pfn) == mfn)
1008 1009
		set_phys_to_machine(pfn, FOREIGN_FRAME(mfn));

1010
	return 0;
1011
}
1012
EXPORT_SYMBOL_GPL(m2p_add_override);
1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057

int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops,
			      struct gnttab_map_grant_ref *kmap_ops,
			      struct page **pages, unsigned int count)
{
	int i, ret = 0;
	bool lazy = false;

	if (xen_feature(XENFEAT_auto_translated_physmap))
		return 0;

	if (kmap_ops &&
	    !in_interrupt() &&
	    paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) {
		arch_enter_lazy_mmu_mode();
		lazy = true;
	}

	for (i = 0; i < count; i++) {
		unsigned long mfn = get_phys_to_machine(page_to_pfn(pages[i]));
		unsigned long pfn = page_to_pfn(pages[i]);

		if (mfn == INVALID_P2M_ENTRY || !(mfn & FOREIGN_FRAME_BIT)) {
			ret = -EINVAL;
			goto out;
		}

		set_page_private(pages[i], INVALID_P2M_ENTRY);
		WARN_ON(!PagePrivate(pages[i]));
		ClearPagePrivate(pages[i]);
		set_phys_to_machine(pfn, pages[i]->index);

		if (kmap_ops)
			ret = m2p_remove_override(pages[i], &kmap_ops[i], mfn);
		if (ret)
			goto out;
	}

out:
	if (lazy)
		arch_leave_lazy_mmu_mode();
	return ret;
}
EXPORT_SYMBOL_GPL(clear_foreign_p2m_mapping);

1058
int m2p_remove_override(struct page *page,
1059 1060
			struct gnttab_map_grant_ref *kmap_op,
			unsigned long mfn)
1061 1062
{
	unsigned long flags;
1063
	unsigned long pfn;
1064
	unsigned long uninitialized_var(address);
1065 1066
	unsigned level;
	pte_t *ptep = NULL;
1067 1068

	pfn = page_to_pfn(page);
1069 1070 1071 1072 1073 1074 1075 1076 1077

	if (!PageHighMem(page)) {
		address = (unsigned long)__va(pfn << PAGE_SHIFT);
		ptep = lookup_address(address, &level);

		if (WARN(ptep == NULL || level != PG_LEVEL_4K,
					"m2p_remove_override: pfn %lx not mapped", pfn))
			return -EINVAL;
	}
1078

1079 1080 1081
	spin_lock_irqsave(&m2p_override_lock, flags);
	list_del(&page->lru);
	spin_unlock_irqrestore(&m2p_override_lock, flags);
1082

1083
	if (kmap_op != NULL) {
1084 1085
		if (!PageHighMem(page)) {
			struct multicall_space mcs;
1086 1087 1088 1089
			struct gnttab_unmap_and_replace *unmap_op;
			struct page *scratch_page = get_balloon_scratch_page();
			unsigned long scratch_page_address = (unsigned long)
				__va(page_to_pfn(scratch_page) << PAGE_SHIFT);
1090 1091 1092 1093 1094 1095 1096 1097

			/*
			 * It might be that we queued all the m2p grant table
			 * hypercalls in a multicall, then m2p_remove_override
			 * get called before the multicall has actually been
			 * issued. In this case handle is going to -1 because
			 * it hasn't been modified yet.
			 */
1098
			if (kmap_op->handle == -1)
1099 1100
				xen_mc_flush();
			/*
1101
			 * Now if kmap_op->handle is negative it means that the
1102 1103
			 * hypercall actually returned an error.
			 */
1104
			if (kmap_op->handle == GNTST_general_error) {
1105 1106 1107
				printk(KERN_WARNING "m2p_remove_override: "
						"pfn %lx mfn %lx, failed to modify kernel mappings",
						pfn, mfn);
1108
				put_balloon_scratch_page();
1109 1110 1111
				return -1;
			}

1112 1113 1114
			xen_mc_batch();

			mcs = __xen_mc_entry(
1115
					sizeof(struct gnttab_unmap_and_replace));
1116
			unmap_op = mcs.args;
1117
			unmap_op->host_addr = kmap_op->host_addr;
1118
			unmap_op->new_addr = scratch_page_address;
1119
			unmap_op->handle = kmap_op->handle;
1120 1121

			MULTI_grant_table_op(mcs.mc,
1122
					GNTTABOP_unmap_and_replace, unmap_op, 1);
1123

1124 1125
			mcs = __xen_mc_entry(0);
			MULTI_update_va_mapping(mcs.mc, scratch_page_address,
1126
					pfn_pte(page_to_pfn(scratch_page),
1127
					PAGE_KERNEL_RO), 0);
1128

1129 1130
			xen_mc_issue(PARAVIRT_LAZY_MMU);

1131
			kmap_op->host_addr = 0;
1132
			put_balloon_scratch_page();
1133
		}
1134
	}
1135

1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146
	/* p2m(m2p(mfn)) == FOREIGN_FRAME(mfn): the mfn is already present
	 * somewhere in this domain, even before being added to the
	 * m2p_override (see comment above in m2p_add_override).
	 * If there are no other entries in the m2p_override corresponding
	 * to this mfn, then remove the FOREIGN_FRAME_BIT from the p2m for
	 * the original pfn (the one shared by the frontend): the backend
	 * cannot do any IO on this page anymore because it has been
	 * unshared. Removing the FOREIGN_FRAME_BIT from the p2m entry of
	 * the original pfn causes mfn_to_pfn(mfn) to return the frontend
	 * pfn again. */
	mfn &= ~FOREIGN_FRAME_BIT;
1147 1148
	pfn = mfn_to_pfn_no_overrides(mfn);
	if (get_phys_to_machine(pfn) == FOREIGN_FRAME(mfn) &&
1149 1150 1151
			m2p_find_override(mfn) == NULL)
		set_phys_to_machine(pfn, mfn);

1152
	return 0;
1153
}
1154
EXPORT_SYMBOL_GPL(m2p_remove_override);
1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166

struct page *m2p_find_override(unsigned long mfn)
{
	unsigned long flags;
	struct list_head *bucket = &m2p_overrides[mfn_hash(mfn)];
	struct page *p, *ret;

	ret = NULL;

	spin_lock_irqsave(&m2p_override_lock, flags);

	list_for_each_entry(p, bucket, lru) {
1167
		if (page_private(p) == mfn) {
1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187
			ret = p;
			break;
		}
	}

	spin_unlock_irqrestore(&m2p_override_lock, flags);

	return ret;
}

unsigned long m2p_find_override_pfn(unsigned long mfn, unsigned long pfn)
{
	struct page *p = m2p_find_override(mfn);
	unsigned long ret = pfn;

	if (p)
		ret = page_to_pfn(p);

	return ret;
}
1188
EXPORT_SYMBOL_GPL(m2p_find_override_pfn);
1189 1190

#ifdef CONFIG_XEN_DEBUG_FS
1191 1192 1193
#include <linux/debugfs.h>
#include "debugfs.h"
static int p2m_dump_show(struct seq_file *m, void *v)
1194 1195
{
	static const char * const level_name[] = { "top", "middle",
1196
						"entry", "abnormal", "error"};
1197 1198 1199 1200
#define TYPE_IDENTITY 0
#define TYPE_MISSING 1
#define TYPE_PFN 2
#define TYPE_UNKNOWN 3
1201 1202 1203 1204 1205
	static const char * const type_name[] = {
				[TYPE_IDENTITY] = "identity",
				[TYPE_MISSING] = "missing",
				[TYPE_PFN] = "pfn",
				[TYPE_UNKNOWN] = "abnormal"};
1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268
	unsigned long pfn, prev_pfn_type = 0, prev_pfn_level = 0;
	unsigned int uninitialized_var(prev_level);
	unsigned int uninitialized_var(prev_type);

	if (!p2m_top)
		return 0;

	for (pfn = 0; pfn < MAX_DOMAIN_PAGES; pfn++) {
		unsigned topidx = p2m_top_index(pfn);
		unsigned mididx = p2m_mid_index(pfn);
		unsigned idx = p2m_index(pfn);
		unsigned lvl, type;

		lvl = 4;
		type = TYPE_UNKNOWN;
		if (p2m_top[topidx] == p2m_mid_missing) {
			lvl = 0; type = TYPE_MISSING;
		} else if (p2m_top[topidx] == NULL) {
			lvl = 0; type = TYPE_UNKNOWN;
		} else if (p2m_top[topidx][mididx] == NULL) {
			lvl = 1; type = TYPE_UNKNOWN;
		} else if (p2m_top[topidx][mididx] == p2m_identity) {
			lvl = 1; type = TYPE_IDENTITY;
		} else if (p2m_top[topidx][mididx] == p2m_missing) {
			lvl = 1; type = TYPE_MISSING;
		} else if (p2m_top[topidx][mididx][idx] == 0) {
			lvl = 2; type = TYPE_UNKNOWN;
		} else if (p2m_top[topidx][mididx][idx] == IDENTITY_FRAME(pfn)) {
			lvl = 2; type = TYPE_IDENTITY;
		} else if (p2m_top[topidx][mididx][idx] == INVALID_P2M_ENTRY) {
			lvl = 2; type = TYPE_MISSING;
		} else if (p2m_top[topidx][mididx][idx] == pfn) {
			lvl = 2; type = TYPE_PFN;
		} else if (p2m_top[topidx][mididx][idx] != pfn) {
			lvl = 2; type = TYPE_PFN;
		}
		if (pfn == 0) {
			prev_level = lvl;
			prev_type = type;
		}
		if (pfn == MAX_DOMAIN_PAGES-1) {
			lvl = 3;
			type = TYPE_UNKNOWN;
		}
		if (prev_type != type) {
			seq_printf(m, " [0x%lx->0x%lx] %s\n",
				prev_pfn_type, pfn, type_name[prev_type]);
			prev_pfn_type = pfn;
			prev_type = type;
		}
		if (prev_level != lvl) {
			seq_printf(m, " [0x%lx->0x%lx] level %s\n",
				prev_pfn_level, pfn, level_name[prev_level]);
			prev_pfn_level = pfn;
			prev_level = lvl;
		}
	}
	return 0;
#undef TYPE_IDENTITY
#undef TYPE_MISSING
#undef TYPE_PFN
#undef TYPE_UNKNOWN
}
1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297

static int p2m_dump_open(struct inode *inode, struct file *filp)
{
	return single_open(filp, p2m_dump_show, NULL);
}

static const struct file_operations p2m_dump_fops = {
	.open		= p2m_dump_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= single_release,
};

static struct dentry *d_mmu_debug;

static int __init xen_p2m_debugfs(void)
{
	struct dentry *d_xen = xen_init_debugfs();

	if (d_xen == NULL)
		return -ENOMEM;

	d_mmu_debug = debugfs_create_dir("mmu", d_xen);

	debugfs_create_file("p2m", 0600, d_mmu_debug, NULL, &p2m_dump_fops);
	return 0;
}
fs_initcall(xen_p2m_debugfs);
#endif /* CONFIG_XEN_DEBUG_FS */