compaction.c 72.0 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
2 3 4 5 6 7 8 9 10
/*
 * linux/mm/compaction.c
 *
 * Memory compaction for the reduction of external fragmentation. Note that
 * this heavily depends upon page migration to do all the real heavy
 * lifting
 *
 * Copyright IBM Corp. 2007-2010 Mel Gorman <mel@csn.ul.ie>
 */
11
#include <linux/cpu.h>
12 13 14 15
#include <linux/swap.h>
#include <linux/migrate.h>
#include <linux/compaction.h>
#include <linux/mm_inline.h>
16
#include <linux/sched/signal.h>
17
#include <linux/backing-dev.h>
18
#include <linux/sysctl.h>
19
#include <linux/sysfs.h>
20
#include <linux/page-isolation.h>
21
#include <linux/kasan.h>
22 23
#include <linux/kthread.h>
#include <linux/freezer.h>
24
#include <linux/page_owner.h>
25
#include <linux/psi.h>
26 27
#include "internal.h"

28 29 30 31 32 33 34 35 36 37 38 39 40 41 42
#ifdef CONFIG_COMPACTION
static inline void count_compact_event(enum vm_event_item item)
{
	count_vm_event(item);
}

static inline void count_compact_events(enum vm_event_item item, long delta)
{
	count_vm_events(item, delta);
}
#else
#define count_compact_event(item) do { } while (0)
#define count_compact_events(item, delta) do { } while (0)
#endif

43 44
#if defined CONFIG_COMPACTION || defined CONFIG_CMA

45 46 47
#define CREATE_TRACE_POINTS
#include <trace/events/compaction.h>

48 49 50 51 52
#define block_start_pfn(pfn, order)	round_down(pfn, 1UL << (order))
#define block_end_pfn(pfn, order)	ALIGN((pfn) + 1, 1UL << (order))
#define pageblock_start_pfn(pfn)	block_start_pfn(pfn, pageblock_order)
#define pageblock_end_pfn(pfn)		block_end_pfn(pfn, pageblock_order)

53 54 55
static unsigned long release_freepages(struct list_head *freelist)
{
	struct page *page, *next;
56
	unsigned long high_pfn = 0;
57 58

	list_for_each_entry_safe(page, next, freelist, lru) {
59
		unsigned long pfn = page_to_pfn(page);
60 61
		list_del(&page->lru);
		__free_page(page);
62 63
		if (pfn > high_pfn)
			high_pfn = pfn;
64 65
	}

66
	return high_pfn;
67 68
}

69
static void split_map_pages(struct list_head *list)
70
{
71 72 73 74 75 76 77 78 79 80
	unsigned int i, order, nr_pages;
	struct page *page, *next;
	LIST_HEAD(tmp_list);

	list_for_each_entry_safe(page, next, list, lru) {
		list_del(&page->lru);

		order = page_private(page);
		nr_pages = 1 << order;

81
		post_alloc_hook(page, order, __GFP_MOVABLE);
82 83
		if (order)
			split_page(page, order);
84

85 86 87 88
		for (i = 0; i < nr_pages; i++) {
			list_add(&page->lru, &tmp_list);
			page++;
		}
89
	}
90 91

	list_splice(&tmp_list, list);
92 93
}

94
#ifdef CONFIG_COMPACTION
95

96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133
int PageMovable(struct page *page)
{
	struct address_space *mapping;

	VM_BUG_ON_PAGE(!PageLocked(page), page);
	if (!__PageMovable(page))
		return 0;

	mapping = page_mapping(page);
	if (mapping && mapping->a_ops && mapping->a_ops->isolate_page)
		return 1;

	return 0;
}
EXPORT_SYMBOL(PageMovable);

void __SetPageMovable(struct page *page, struct address_space *mapping)
{
	VM_BUG_ON_PAGE(!PageLocked(page), page);
	VM_BUG_ON_PAGE((unsigned long)mapping & PAGE_MAPPING_MOVABLE, page);
	page->mapping = (void *)((unsigned long)mapping | PAGE_MAPPING_MOVABLE);
}
EXPORT_SYMBOL(__SetPageMovable);

void __ClearPageMovable(struct page *page)
{
	VM_BUG_ON_PAGE(!PageLocked(page), page);
	VM_BUG_ON_PAGE(!PageMovable(page), page);
	/*
	 * Clear registered address_space val with keeping PAGE_MAPPING_MOVABLE
	 * flag so that VM can catch up released page by driver after isolation.
	 * With it, VM migration doesn't try to put it back.
	 */
	page->mapping = (void *)((unsigned long)page->mapping &
				PAGE_MAPPING_MOVABLE);
}
EXPORT_SYMBOL(__ClearPageMovable);

134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203
/* Do not skip compaction more than 64 times */
#define COMPACT_MAX_DEFER_SHIFT 6

/*
 * Compaction is deferred when compaction fails to result in a page
 * allocation success. 1 << compact_defer_limit compactions are skipped up
 * to a limit of 1 << COMPACT_MAX_DEFER_SHIFT
 */
void defer_compaction(struct zone *zone, int order)
{
	zone->compact_considered = 0;
	zone->compact_defer_shift++;

	if (order < zone->compact_order_failed)
		zone->compact_order_failed = order;

	if (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT)
		zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT;

	trace_mm_compaction_defer_compaction(zone, order);
}

/* Returns true if compaction should be skipped this time */
bool compaction_deferred(struct zone *zone, int order)
{
	unsigned long defer_limit = 1UL << zone->compact_defer_shift;

	if (order < zone->compact_order_failed)
		return false;

	/* Avoid possible overflow */
	if (++zone->compact_considered > defer_limit)
		zone->compact_considered = defer_limit;

	if (zone->compact_considered >= defer_limit)
		return false;

	trace_mm_compaction_deferred(zone, order);

	return true;
}

/*
 * Update defer tracking counters after successful compaction of given order,
 * which means an allocation either succeeded (alloc_success == true) or is
 * expected to succeed.
 */
void compaction_defer_reset(struct zone *zone, int order,
		bool alloc_success)
{
	if (alloc_success) {
		zone->compact_considered = 0;
		zone->compact_defer_shift = 0;
	}
	if (order >= zone->compact_order_failed)
		zone->compact_order_failed = order + 1;

	trace_mm_compaction_defer_reset(zone, order);
}

/* Returns true if restarting compaction after many failures */
bool compaction_restarting(struct zone *zone, int order)
{
	if (order < zone->compact_order_failed)
		return false;

	return zone->compact_defer_shift == COMPACT_MAX_DEFER_SHIFT &&
		zone->compact_considered >= 1UL << zone->compact_defer_shift;
}

204 205 206 207 208 209 210 211 212 213
/* Returns true if the pageblock should be scanned for pages to isolate. */
static inline bool isolation_suitable(struct compact_control *cc,
					struct page *page)
{
	if (cc->ignore_skip_hint)
		return true;

	return !get_pageblock_skip(page);
}

214 215 216 217
static void reset_cached_positions(struct zone *zone)
{
	zone->compact_cached_migrate_pfn[0] = zone->zone_start_pfn;
	zone->compact_cached_migrate_pfn[1] = zone->zone_start_pfn;
218
	zone->compact_cached_free_pfn =
219
				pageblock_start_pfn(zone_end_pfn(zone) - 1);
220 221
}

222
/*
223 224 225
 * Compound pages of >= pageblock_order should consistenly be skipped until
 * released. It is always pointless to compact pages of such order (if they are
 * migratable), and the pageblocks they occupy cannot contain any free pages.
226
 */
227
static bool pageblock_skip_persistent(struct page *page)
228
{
229
	if (!PageCompound(page))
230
		return false;
231 232 233 234 235 236 237

	page = compound_head(page);

	if (compound_order(page) >= pageblock_order)
		return true;

	return false;
238 239
}

240 241 242 243 244
/*
 * This function is called to clear all cached information on pageblocks that
 * should be skipped for page isolation when the migrate and free page scanner
 * meet.
 */
245
static void __reset_isolation_suitable(struct zone *zone)
246 247
{
	unsigned long start_pfn = zone->zone_start_pfn;
248
	unsigned long end_pfn = zone_end_pfn(zone);
249 250
	unsigned long pfn;

251
	zone->compact_blockskip_flush = false;
252 253 254 255 256 257 258

	/* Walk the zone and mark every pageblock as suitable for isolation */
	for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {
		struct page *page;

		cond_resched();

259 260
		page = pfn_to_online_page(pfn);
		if (!page)
261 262 263
			continue;
		if (zone != page_zone(page))
			continue;
264
		if (pageblock_skip_persistent(page))
265
			continue;
266 267 268

		clear_pageblock_skip(page);
	}
269 270

	reset_cached_positions(zone);
271 272
}

273 274 275 276 277 278 279 280 281 282 283 284 285 286 287
void reset_isolation_suitable(pg_data_t *pgdat)
{
	int zoneid;

	for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {
		struct zone *zone = &pgdat->node_zones[zoneid];
		if (!populated_zone(zone))
			continue;

		/* Only flush if a full compaction finished recently */
		if (zone->compact_blockskip_flush)
			__reset_isolation_suitable(zone);
	}
}

288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327
/*
 * Sets the pageblock skip bit if it was clear. Note that this is a hint as
 * locks are not required for read/writers. Returns true if it was already set.
 */
static bool test_and_set_skip(struct compact_control *cc, struct page *page,
							unsigned long pfn)
{
	bool skip;

	/* Do no update if skip hint is being ignored */
	if (cc->ignore_skip_hint)
		return false;

	if (!IS_ALIGNED(pfn, pageblock_nr_pages))
		return false;

	skip = get_pageblock_skip(page);
	if (!skip && !cc->no_set_skip_hint)
		set_pageblock_skip(page);

	return skip;
}

static void update_cached_migrate(struct compact_control *cc, unsigned long pfn)
{
	struct zone *zone = cc->zone;

	pfn = pageblock_end_pfn(pfn);

	/* Set for isolation rather than compaction */
	if (cc->no_set_skip_hint)
		return;

	if (pfn > zone->compact_cached_migrate_pfn[0])
		zone->compact_cached_migrate_pfn[0] = pfn;
	if (cc->mode != MIGRATE_ASYNC &&
	    pfn > zone->compact_cached_migrate_pfn[1])
		zone->compact_cached_migrate_pfn[1] = pfn;
}

328 329
/*
 * If no pages were isolated then mark this pageblock to be skipped in the
330
 * future. The information is later cleared by __reset_isolation_suitable().
331
 */
332
static void update_pageblock_skip(struct compact_control *cc,
333
			struct page *page, unsigned long pfn)
334
{
335
	struct zone *zone = cc->zone;
336

337
	if (cc->no_set_skip_hint)
338 339
		return;

340 341 342
	if (!page)
		return;

343
	set_pageblock_skip(page);
344

345
	/* Update where async and sync compaction should restart */
346 347
	if (pfn < zone->compact_cached_free_pfn)
		zone->compact_cached_free_pfn = pfn;
348 349 350 351 352 353 354 355
}
#else
static inline bool isolation_suitable(struct compact_control *cc,
					struct page *page)
{
	return true;
}

356
static inline bool pageblock_skip_persistent(struct page *page)
357 358 359 360 361
{
	return false;
}

static inline void update_pageblock_skip(struct compact_control *cc,
362
			struct page *page, unsigned long pfn)
363 364
{
}
365 366 367 368 369 370 371 372 373 374

static void update_cached_migrate(struct compact_control *cc, unsigned long pfn)
{
}

static bool test_and_set_skip(struct compact_control *cc, struct page *page,
							unsigned long pfn)
{
	return false;
}
375 376
#endif /* CONFIG_COMPACTION */

377 378
/*
 * Compaction requires the taking of some coarse locks that are potentially
379 380 381 382
 * very heavily contended. For async compaction, trylock and record if the
 * lock is contended. The lock will still be acquired but compaction will
 * abort when the current block is finished regardless of success rate.
 * Sync compaction acquires the lock.
383
 *
384
 * Always returns true which makes it easier to track lock state in callers.
385
 */
386
static bool compact_lock_irqsave(spinlock_t *lock, unsigned long *flags,
387
						struct compact_control *cc)
388
{
389 390 391 392 393 394
	/* Track if the lock is contended in async mode */
	if (cc->mode == MIGRATE_ASYNC && !cc->contended) {
		if (spin_trylock_irqsave(lock, *flags))
			return true;

		cc->contended = true;
395
	}
396

397
	spin_lock_irqsave(lock, *flags);
398
	return true;
399 400
}

401 402
/*
 * Compaction requires the taking of some coarse locks that are potentially
403 404 405 406 407 408 409
 * very heavily contended. The lock should be periodically unlocked to avoid
 * having disabled IRQs for a long time, even when there is nobody waiting on
 * the lock. It might also be that allowing the IRQs will result in
 * need_resched() becoming true. If scheduling is needed, async compaction
 * aborts. Sync compaction schedules.
 * Either compaction type will also abort if a fatal signal is pending.
 * In either case if the lock was locked, it is dropped and not regained.
410
 *
411 412 413 414
 * Returns true if compaction should abort due to fatal signal pending, or
 *		async compaction due to need_resched()
 * Returns false when compaction can continue (sync compaction might have
 *		scheduled)
415
 */
416 417
static bool compact_unlock_should_abort(spinlock_t *lock,
		unsigned long flags, bool *locked, struct compact_control *cc)
418
{
419 420 421 422
	if (*locked) {
		spin_unlock_irqrestore(lock, flags);
		*locked = false;
	}
423

424
	if (fatal_signal_pending(current)) {
425
		cc->contended = true;
426 427
		return true;
	}
428

429
	cond_resched();
430 431 432 433

	return false;
}

434
/*
435 436 437
 * Isolate free pages onto a private freelist. If @strict is true, will abort
 * returning 0 on any invalid PFNs or non-free pages inside of the pageblock
 * (even though it may still end up isolating some pages).
438
 */
439
static unsigned long isolate_freepages_block(struct compact_control *cc,
440
				unsigned long *start_pfn,
441 442
				unsigned long end_pfn,
				struct list_head *freelist,
443
				unsigned int stride,
444
				bool strict)
445
{
446
	int nr_scanned = 0, total_isolated = 0;
447
	struct page *cursor;
448
	unsigned long flags = 0;
449
	bool locked = false;
450
	unsigned long blockpfn = *start_pfn;
451
	unsigned int order;
452

453 454 455 456
	/* Strict mode is for isolation, speed is secondary */
	if (strict)
		stride = 1;

457 458
	cursor = pfn_to_page(blockpfn);

459
	/* Isolate free pages. */
460
	for (; blockpfn < end_pfn; blockpfn += stride, cursor += stride) {
461
		int isolated;
462 463
		struct page *page = cursor;

464 465 466 467 468 469 470 471 472 473
		/*
		 * Periodically drop the lock (if held) regardless of its
		 * contention, to give chance to IRQs. Abort if fatal signal
		 * pending or async compaction detects need_resched()
		 */
		if (!(blockpfn % SWAP_CLUSTER_MAX)
		    && compact_unlock_should_abort(&cc->zone->lock, flags,
								&locked, cc))
			break;

474
		nr_scanned++;
475
		if (!pfn_valid_within(blockpfn))
476 477
			goto isolate_fail;

478 479 480 481 482 483 484
		/*
		 * For compound pages such as THP and hugetlbfs, we can save
		 * potentially a lot of iterations if we skip them at once.
		 * The check is racy, but we can consider only valid values
		 * and the only danger is skipping too much.
		 */
		if (PageCompound(page)) {
485 486
			const unsigned int order = compound_order(page);

487
			if (likely(order < MAX_ORDER)) {
488 489
				blockpfn += (1UL << order) - 1;
				cursor += (1UL << order) - 1;
490 491 492 493
			}
			goto isolate_fail;
		}

494
		if (!PageBuddy(page))
495
			goto isolate_fail;
496 497

		/*
498 499 500 501 502
		 * If we already hold the lock, we can skip some rechecking.
		 * Note that if we hold the lock now, checked_pageblock was
		 * already set in some previous iteration (or strict is true),
		 * so it is correct to skip the suitable migration target
		 * recheck as well.
503
		 */
504
		if (!locked) {
505
			locked = compact_lock_irqsave(&cc->zone->lock,
506
								&flags, cc);
507

508 509 510 511
			/* Recheck this is a buddy page under lock */
			if (!PageBuddy(page))
				goto isolate_fail;
		}
512

513 514 515
		/* Found a free page, will break it into order-0 pages */
		order = page_order(page);
		isolated = __isolate_free_page(page, order);
516 517
		if (!isolated)
			break;
518
		set_page_private(page, order);
519

520
		total_isolated += isolated;
521
		cc->nr_freepages += isolated;
522 523
		list_add_tail(&page->lru, freelist);

524 525 526
		if (!strict && cc->nr_migratepages <= cc->nr_freepages) {
			blockpfn += isolated;
			break;
527
		}
528 529 530 531
		/* Advance to the end of split page */
		blockpfn += isolated - 1;
		cursor += isolated - 1;
		continue;
532 533 534 535 536 537 538

isolate_fail:
		if (strict)
			break;
		else
			continue;

539 540
	}

541 542 543
	if (locked)
		spin_unlock_irqrestore(&cc->zone->lock, flags);

544 545 546 547 548 549 550
	/*
	 * There is a tiny chance that we have read bogus compound_order(),
	 * so be careful to not go outside of the pageblock.
	 */
	if (unlikely(blockpfn > end_pfn))
		blockpfn = end_pfn;

551 552 553
	trace_mm_compaction_isolate_freepages(*start_pfn, blockpfn,
					nr_scanned, total_isolated);

554 555 556
	/* Record how far we have got within the block */
	*start_pfn = blockpfn;

557 558 559 560 561
	/*
	 * If strict isolation is requested by CMA then check that all the
	 * pages requested were isolated. If there were any failures, 0 is
	 * returned and CMA will fail.
	 */
562
	if (strict && blockpfn < end_pfn)
563 564
		total_isolated = 0;

565
	cc->total_free_scanned += nr_scanned;
566
	if (total_isolated)
567
		count_compact_events(COMPACTISOLATED, total_isolated);
568 569 570
	return total_isolated;
}

571 572
/**
 * isolate_freepages_range() - isolate free pages.
573
 * @cc:        Compaction control structure.
574 575 576 577 578 579 580 581 582 583 584
 * @start_pfn: The first PFN to start isolating.
 * @end_pfn:   The one-past-last PFN.
 *
 * Non-free pages, invalid PFNs, or zone boundaries within the
 * [start_pfn, end_pfn) range are considered errors, cause function to
 * undo its actions and return zero.
 *
 * Otherwise, function returns one-past-the-last PFN of isolated page
 * (which may be greater then end_pfn if end fell in a middle of
 * a free page).
 */
585
unsigned long
586 587
isolate_freepages_range(struct compact_control *cc,
			unsigned long start_pfn, unsigned long end_pfn)
588
{
589
	unsigned long isolated, pfn, block_start_pfn, block_end_pfn;
590 591
	LIST_HEAD(freelist);

592
	pfn = start_pfn;
593
	block_start_pfn = pageblock_start_pfn(pfn);
594 595
	if (block_start_pfn < cc->zone->zone_start_pfn)
		block_start_pfn = cc->zone->zone_start_pfn;
596
	block_end_pfn = pageblock_end_pfn(pfn);
597 598

	for (; pfn < end_pfn; pfn += isolated,
599
				block_start_pfn = block_end_pfn,
600
				block_end_pfn += pageblock_nr_pages) {
601 602
		/* Protect pfn from changing by isolate_freepages_block */
		unsigned long isolate_start_pfn = pfn;
603 604 605

		block_end_pfn = min(block_end_pfn, end_pfn);

606 607 608 609 610 611
		/*
		 * pfn could pass the block_end_pfn if isolated freepage
		 * is more than pageblock order. In this case, we adjust
		 * scanning range to right one.
		 */
		if (pfn >= block_end_pfn) {
612 613
			block_start_pfn = pageblock_start_pfn(pfn);
			block_end_pfn = pageblock_end_pfn(pfn);
614 615 616
			block_end_pfn = min(block_end_pfn, end_pfn);
		}

617 618
		if (!pageblock_pfn_to_page(block_start_pfn,
					block_end_pfn, cc->zone))
619 620
			break;

621
		isolated = isolate_freepages_block(cc, &isolate_start_pfn,
622
					block_end_pfn, &freelist, 0, true);
623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638

		/*
		 * In strict mode, isolate_freepages_block() returns 0 if
		 * there are any holes in the block (ie. invalid PFNs or
		 * non-free pages).
		 */
		if (!isolated)
			break;

		/*
		 * If we managed to isolate pages, it is always (1 << n) *
		 * pageblock_nr_pages for some non-negative n.  (Max order
		 * page may span two pageblocks).
		 */
	}

639
	/* __isolate_free_page() does not map the pages */
640
	split_map_pages(&freelist);
641 642 643 644 645 646 647 648 649 650 651

	if (pfn < end_pfn) {
		/* Loop terminated early, cleanup. */
		release_freepages(&freelist);
		return 0;
	}

	/* We don't use freelists for anything. */
	return pfn;
}

652 653 654
/* Similar to reclaim, but different enough that they don't share logic */
static bool too_many_isolated(struct zone *zone)
{
655
	unsigned long active, inactive, isolated;
656

M
Mel Gorman 已提交
657 658 659 660 661 662
	inactive = node_page_state(zone->zone_pgdat, NR_INACTIVE_FILE) +
			node_page_state(zone->zone_pgdat, NR_INACTIVE_ANON);
	active = node_page_state(zone->zone_pgdat, NR_ACTIVE_FILE) +
			node_page_state(zone->zone_pgdat, NR_ACTIVE_ANON);
	isolated = node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE) +
			node_page_state(zone->zone_pgdat, NR_ISOLATED_ANON);
663

664
	return isolated > (inactive + active) / 2;
665 666
}

667
/**
668 669
 * isolate_migratepages_block() - isolate all migrate-able pages within
 *				  a single pageblock
670
 * @cc:		Compaction control structure.
671 672 673
 * @low_pfn:	The first PFN to isolate
 * @end_pfn:	The one-past-the-last PFN to isolate, within same pageblock
 * @isolate_mode: Isolation mode to be used.
674 675
 *
 * Isolate all pages that can be migrated from the range specified by
676 677 678 679
 * [low_pfn, end_pfn). The range is expected to be within same pageblock.
 * Returns zero if there is a fatal signal pending, otherwise PFN of the
 * first page that was not scanned (which may be both less, equal to or more
 * than end_pfn).
680
 *
681 682 683
 * The pages are isolated on cc->migratepages list (not required to be empty),
 * and cc->nr_migratepages is updated accordingly. The cc->migrate_pfn field
 * is neither read nor updated.
684
 */
685 686 687
static unsigned long
isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
			unsigned long end_pfn, isolate_mode_t isolate_mode)
688
{
689
	struct zone *zone = cc->zone;
690
	unsigned long nr_scanned = 0, nr_isolated = 0;
691
	struct lruvec *lruvec;
692
	unsigned long flags = 0;
693
	bool locked = false;
694
	struct page *page = NULL, *valid_page = NULL;
695
	unsigned long start_pfn = low_pfn;
696 697
	bool skip_on_failure = false;
	unsigned long next_skip_pfn = 0;
698
	bool skip_updated = false;
699 700 701 702 703 704 705

	/*
	 * Ensure that there are not too many pages isolated from the LRU
	 * list by either parallel reclaimers or compaction. If there are,
	 * delay for some time until fewer pages are isolated
	 */
	while (unlikely(too_many_isolated(zone))) {
706
		/* async migration should just abort */
707
		if (cc->mode == MIGRATE_ASYNC)
708
			return 0;
709

710 711 712
		congestion_wait(BLK_RW_ASYNC, HZ/10);

		if (fatal_signal_pending(current))
713
			return 0;
714 715
	}

716
	cond_resched();
717

718 719 720 721 722
	if (cc->direct_compaction && (cc->mode == MIGRATE_ASYNC)) {
		skip_on_failure = true;
		next_skip_pfn = block_end_pfn(low_pfn, cc->order);
	}

723 724
	/* Time to isolate some pages for migration */
	for (; low_pfn < end_pfn; low_pfn++) {
725

726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747
		if (skip_on_failure && low_pfn >= next_skip_pfn) {
			/*
			 * We have isolated all migration candidates in the
			 * previous order-aligned block, and did not skip it due
			 * to failure. We should migrate the pages now and
			 * hopefully succeed compaction.
			 */
			if (nr_isolated)
				break;

			/*
			 * We failed to isolate in the previous order-aligned
			 * block. Set the new boundary to the end of the
			 * current block. Note we can't simply increase
			 * next_skip_pfn by 1 << order, as low_pfn might have
			 * been incremented by a higher number due to skipping
			 * a compound or a high-order buddy page in the
			 * previous loop iteration.
			 */
			next_skip_pfn = block_end_pfn(low_pfn, cc->order);
		}

748 749 750 751 752 753
		/*
		 * Periodically drop the lock (if held) regardless of its
		 * contention, to give chance to IRQs. Abort async compaction
		 * if contended.
		 */
		if (!(low_pfn % SWAP_CLUSTER_MAX)
754
		    && compact_unlock_should_abort(zone_lru_lock(zone), flags,
755 756
								&locked, cc))
			break;
757

758
		if (!pfn_valid_within(low_pfn))
759
			goto isolate_fail;
760
		nr_scanned++;
761 762

		page = pfn_to_page(low_pfn);
763

764 765 766 767 768 769 770 771 772 773 774
		/*
		 * Check if the pageblock has already been marked skipped.
		 * Only the aligned PFN is checked as the caller isolates
		 * COMPACT_CLUSTER_MAX at a time so the second call must
		 * not falsely conclude that the block should be skipped.
		 */
		if (!valid_page && IS_ALIGNED(low_pfn, pageblock_nr_pages)) {
			if (!cc->ignore_skip_hint && get_pageblock_skip(page)) {
				low_pfn = end_pfn;
				goto isolate_abort;
			}
775
			valid_page = page;
776
		}
777

778
		/*
779 780 781 782
		 * Skip if free. We read page order here without zone lock
		 * which is generally unsafe, but the race window is small and
		 * the worst thing that can happen is that we skip some
		 * potential isolation targets.
783
		 */
784 785 786 787 788 789 790 791 792 793
		if (PageBuddy(page)) {
			unsigned long freepage_order = page_order_unsafe(page);

			/*
			 * Without lock, we cannot be sure that what we got is
			 * a valid page order. Consider only values in the
			 * valid order range to prevent low_pfn overflow.
			 */
			if (freepage_order > 0 && freepage_order < MAX_ORDER)
				low_pfn += (1UL << freepage_order) - 1;
794
			continue;
795
		}
796

797
		/*
798 799 800 801 802
		 * Regardless of being on LRU, compound pages such as THP and
		 * hugetlbfs are not to be compacted. We can potentially save
		 * a lot of iterations if we skip them at once. The check is
		 * racy, but we can consider only valid values and the only
		 * danger is skipping too much.
803
		 */
804
		if (PageCompound(page)) {
805
			const unsigned int order = compound_order(page);
806

807
			if (likely(order < MAX_ORDER))
808
				low_pfn += (1UL << order) - 1;
809
			goto isolate_fail;
810 811
		}

812 813 814 815 816 817 818 819 820 821 822 823 824
		/*
		 * Check may be lockless but that's ok as we recheck later.
		 * It's possible to migrate LRU and non-lru movable pages.
		 * Skip any other type of page
		 */
		if (!PageLRU(page)) {
			/*
			 * __PageMovable can return false positive so we need
			 * to verify it under page_lock.
			 */
			if (unlikely(__PageMovable(page)) &&
					!PageIsolated(page)) {
				if (locked) {
825
					spin_unlock_irqrestore(zone_lru_lock(zone),
826 827 828 829
									flags);
					locked = false;
				}

830
				if (!isolate_movable_page(page, isolate_mode))
831 832 833
					goto isolate_success;
			}

834
			goto isolate_fail;
835
		}
836

837 838 839 840 841 842 843
		/*
		 * Migration will fail if an anonymous page is pinned in memory,
		 * so avoid taking lru_lock and isolating it unnecessarily in an
		 * admittedly racy check.
		 */
		if (!page_mapping(page) &&
		    page_count(page) > page_mapcount(page))
844
			goto isolate_fail;
845

846 847 848 849 850 851 852
		/*
		 * Only allow to migrate anonymous pages in GFP_NOFS context
		 * because those do not depend on fs locks.
		 */
		if (!(cc->gfp_mask & __GFP_FS) && page_mapping(page))
			goto isolate_fail;

853 854
		/* If we already hold the lock, we can skip some rechecking */
		if (!locked) {
855
			locked = compact_lock_irqsave(zone_lru_lock(zone),
856
								&flags, cc);
857 858 859 860 861 862 863

			/* Try get exclusive access under lock */
			if (!skip_updated) {
				skip_updated = true;
				if (test_and_set_skip(cc, page, low_pfn))
					goto isolate_abort;
			}
864

865
			/* Recheck PageLRU and PageCompound under lock */
866
			if (!PageLRU(page))
867
				goto isolate_fail;
868 869 870 871 872 873 874

			/*
			 * Page become compound since the non-locked check,
			 * and it's on LRU. It can only be a THP so the order
			 * is safe to read and it's 0 for tail pages.
			 */
			if (unlikely(PageCompound(page))) {
875
				low_pfn += (1UL << compound_order(page)) - 1;
876
				goto isolate_fail;
877
			}
878 879
		}

M
Mel Gorman 已提交
880
		lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);
881

882
		/* Try isolate the page */
883
		if (__isolate_lru_page(page, isolate_mode) != 0)
884
			goto isolate_fail;
885

886
		VM_BUG_ON_PAGE(PageCompound(page), page);
887

888
		/* Successfully isolated */
889
		del_page_from_lru_list(page, lruvec, page_lru(page));
890 891
		inc_node_page_state(page,
				NR_ISOLATED_ANON + page_is_file_cache(page));
892 893

isolate_success:
894
		list_add(&page->lru, &cc->migratepages);
895
		cc->nr_migratepages++;
896
		nr_isolated++;
897

898 899
		/*
		 * Avoid isolating too much unless this block is being
900 901 902
		 * rescanned (e.g. dirty/writeback pages, parallel allocation)
		 * or a lock is contended. For contention, isolate quickly to
		 * potentially remove one source of contention.
903
		 */
904 905
		if (cc->nr_migratepages == COMPACT_CLUSTER_MAX &&
		    !cc->rescan && !cc->contended) {
906
			++low_pfn;
907
			break;
908
		}
909 910 911 912 913 914 915 916 917 918 919 920 921

		continue;
isolate_fail:
		if (!skip_on_failure)
			continue;

		/*
		 * We have isolated some pages, but then failed. Release them
		 * instead of migrating, as we cannot form the cc->order buddy
		 * page anyway.
		 */
		if (nr_isolated) {
			if (locked) {
922
				spin_unlock_irqrestore(zone_lru_lock(zone), flags);
923 924 925 926 927 928 929 930 931 932 933 934 935 936 937
				locked = false;
			}
			putback_movable_pages(&cc->migratepages);
			cc->nr_migratepages = 0;
			nr_isolated = 0;
		}

		if (low_pfn < next_skip_pfn) {
			low_pfn = next_skip_pfn - 1;
			/*
			 * The check near the loop beginning would have updated
			 * next_skip_pfn too, but this is a bit simpler.
			 */
			next_skip_pfn += 1UL << cc->order;
		}
938 939
	}

940 941 942 943 944 945 946
	/*
	 * The PageBuddy() check could have potentially brought us outside
	 * the range to be scanned.
	 */
	if (unlikely(low_pfn > end_pfn))
		low_pfn = end_pfn;

947
isolate_abort:
948
	if (locked)
949
		spin_unlock_irqrestore(zone_lru_lock(zone), flags);
950

951
	/*
952 953 954 955 956 957
	 * Updated the cached scanner pfn once the pageblock has been scanned
	 * Pages will either be migrated in which case there is no point
	 * scanning in the near future or migration failed in which case the
	 * failure reason may persist. The block is marked for skipping if
	 * there were no pages isolated in the block or if the block is
	 * rescanned twice in a row.
958
	 */
959
	if (low_pfn == end_pfn && (!nr_isolated || cc->rescan)) {
960 961 962 963
		if (valid_page && !skip_updated)
			set_pageblock_skip(valid_page);
		update_cached_migrate(cc, low_pfn);
	}
964

965 966
	trace_mm_compaction_isolate_migratepages(start_pfn, low_pfn,
						nr_scanned, nr_isolated);
967

968
	cc->total_migrate_scanned += nr_scanned;
969
	if (nr_isolated)
970
		count_compact_events(COMPACTISOLATED, nr_isolated);
971

972 973 974
	return low_pfn;
}

975 976 977 978 979 980 981 982 983 984 985 986 987 988
/**
 * isolate_migratepages_range() - isolate migrate-able pages in a PFN range
 * @cc:        Compaction control structure.
 * @start_pfn: The first PFN to start isolating.
 * @end_pfn:   The one-past-last PFN.
 *
 * Returns zero if isolation fails fatally due to e.g. pending signal.
 * Otherwise, function returns one-past-the-last PFN of isolated page
 * (which may be greater than end_pfn if end fell in a middle of a THP page).
 */
unsigned long
isolate_migratepages_range(struct compact_control *cc, unsigned long start_pfn,
							unsigned long end_pfn)
{
989
	unsigned long pfn, block_start_pfn, block_end_pfn;
990 991 992

	/* Scan block by block. First and last block may be incomplete */
	pfn = start_pfn;
993
	block_start_pfn = pageblock_start_pfn(pfn);
994 995
	if (block_start_pfn < cc->zone->zone_start_pfn)
		block_start_pfn = cc->zone->zone_start_pfn;
996
	block_end_pfn = pageblock_end_pfn(pfn);
997 998

	for (; pfn < end_pfn; pfn = block_end_pfn,
999
				block_start_pfn = block_end_pfn,
1000 1001 1002 1003
				block_end_pfn += pageblock_nr_pages) {

		block_end_pfn = min(block_end_pfn, end_pfn);

1004 1005
		if (!pageblock_pfn_to_page(block_start_pfn,
					block_end_pfn, cc->zone))
1006 1007 1008 1009 1010
			continue;

		pfn = isolate_migratepages_block(cc, pfn, block_end_pfn,
							ISOLATE_UNEVICTABLE);

1011
		if (!pfn)
1012
			break;
1013 1014 1015

		if (cc->nr_migratepages == COMPACT_CLUSTER_MAX)
			break;
1016 1017 1018 1019 1020
	}

	return pfn;
}

1021 1022
#endif /* CONFIG_COMPACTION || CONFIG_CMA */
#ifdef CONFIG_COMPACTION
1023

1024 1025 1026
static bool suitable_migration_source(struct compact_control *cc,
							struct page *page)
{
1027 1028
	int block_mt;

1029 1030 1031
	if (pageblock_skip_persistent(page))
		return false;

1032
	if ((cc->mode != MIGRATE_ASYNC) || !cc->direct_compaction)
1033 1034
		return true;

1035 1036 1037 1038 1039 1040
	block_mt = get_pageblock_migratetype(page);

	if (cc->migratetype == MIGRATE_MOVABLE)
		return is_migrate_movable(block_mt);
	else
		return block_mt == cc->migratetype;
1041 1042
}

1043
/* Returns true if the page is within a block suitable for migration to */
1044 1045
static bool suitable_migration_target(struct compact_control *cc,
							struct page *page)
1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057
{
	/* If the page is a large free page, then disallow migration */
	if (PageBuddy(page)) {
		/*
		 * We are checking page_order without zone->lock taken. But
		 * the only small danger is that we skip a potentially suitable
		 * pageblock, so it's not worth to check order for valid range.
		 */
		if (page_order_unsafe(page) >= pageblock_order)
			return false;
	}

1058 1059 1060
	if (cc->ignore_block_suitable)
		return true;

1061
	/* If the block is MIGRATE_MOVABLE or MIGRATE_CMA, allow migration */
1062
	if (is_migrate_movable(get_pageblock_migratetype(page)))
1063 1064 1065 1066 1067 1068
		return true;

	/* Otherwise skip the block */
	return false;
}

1069 1070 1071 1072 1073 1074
static inline unsigned int
freelist_scan_limit(struct compact_control *cc)
{
	return (COMPACT_CLUSTER_MAX >> cc->fast_search_fail) + 1;
}

1075 1076 1077 1078 1079 1080 1081 1082 1083 1084
/*
 * Test whether the free scanner has reached the same or lower pageblock than
 * the migration scanner, and compaction should thus terminate.
 */
static inline bool compact_scanners_met(struct compact_control *cc)
{
	return (cc->free_pfn >> pageblock_order)
		<= (cc->migrate_pfn >> pageblock_order);
}

1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107
/*
 * Used when scanning for a suitable migration target which scans freelists
 * in reverse. Reorders the list such as the unscanned pages are scanned
 * first on the next iteration of the free scanner
 */
static void
move_freelist_head(struct list_head *freelist, struct page *freepage)
{
	LIST_HEAD(sublist);

	if (!list_is_last(freelist, &freepage->lru)) {
		list_cut_before(&sublist, freelist, &freepage->lru);
		if (!list_empty(&sublist))
			list_splice_tail(&sublist, freelist);
	}
}

/*
 * Similar to move_freelist_head except used by the migration scanner
 * when scanning forward. It's possible for these list operations to
 * move against each other if they search the free list exactly in
 * lockstep.
 */
1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119
static void
move_freelist_tail(struct list_head *freelist, struct page *freepage)
{
	LIST_HEAD(sublist);

	if (!list_is_first(freelist, &freepage->lru)) {
		list_cut_position(&sublist, freelist, &freepage->lru);
		if (!list_empty(&sublist))
			list_splice_tail(&sublist, freelist);
	}
}

1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139
static void
fast_isolate_around(struct compact_control *cc, unsigned long pfn, unsigned long nr_isolated)
{
	unsigned long start_pfn, end_pfn;
	struct page *page = pfn_to_page(pfn);

	/* Do not search around if there are enough pages already */
	if (cc->nr_freepages >= cc->nr_migratepages)
		return;

	/* Minimise scanning during async compaction */
	if (cc->direct_compaction && cc->mode == MIGRATE_ASYNC)
		return;

	/* Pageblock boundaries */
	start_pfn = pageblock_start_pfn(pfn);
	end_pfn = min(start_pfn + pageblock_nr_pages, zone_end_pfn(cc->zone));

	/* Scan before */
	if (start_pfn != pfn) {
1140
		isolate_freepages_block(cc, &start_pfn, pfn, &cc->freepages, 1, false);
1141 1142 1143 1144 1145 1146 1147
		if (cc->nr_freepages >= cc->nr_migratepages)
			return;
	}

	/* Scan after */
	start_pfn = pfn + nr_isolated;
	if (start_pfn != end_pfn)
1148
		isolate_freepages_block(cc, &start_pfn, end_pfn, &cc->freepages, 1, false);
1149 1150 1151 1152 1153 1154

	/* Skip this pageblock in the future as it's full or nearly full */
	if (cc->nr_freepages < cc->nr_migratepages)
		set_pageblock_skip(page);
}

1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172
/* Search orders in round-robin fashion */
static int next_search_order(struct compact_control *cc, int order)
{
	order--;
	if (order < 0)
		order = cc->order - 1;

	/* Search wrapped around? */
	if (order == cc->search_order) {
		cc->search_order--;
		if (cc->search_order < 0)
			cc->search_order = cc->order - 1;
		return -1;
	}

	return order;
}

1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208
static unsigned long
fast_isolate_freepages(struct compact_control *cc)
{
	unsigned int limit = min(1U, freelist_scan_limit(cc) >> 1);
	unsigned int nr_scanned = 0;
	unsigned long low_pfn, min_pfn, high_pfn = 0, highest = 0;
	unsigned long nr_isolated = 0;
	unsigned long distance;
	struct page *page = NULL;
	bool scan_start = false;
	int order;

	/* Full compaction passes in a negative order */
	if (cc->order <= 0)
		return cc->free_pfn;

	/*
	 * If starting the scan, use a deeper search and use the highest
	 * PFN found if a suitable one is not found.
	 */
	if (cc->free_pfn == pageblock_start_pfn(zone_end_pfn(cc->zone) - 1)) {
		limit = pageblock_nr_pages >> 1;
		scan_start = true;
	}

	/*
	 * Preferred point is in the top quarter of the scan space but take
	 * a pfn from the top half if the search is problematic.
	 */
	distance = (cc->free_pfn - cc->migrate_pfn);
	low_pfn = pageblock_start_pfn(cc->free_pfn - (distance >> 2));
	min_pfn = pageblock_start_pfn(cc->free_pfn - (distance >> 1));

	if (WARN_ON_ONCE(min_pfn > low_pfn))
		low_pfn = min_pfn;

1209 1210 1211 1212 1213 1214 1215 1216 1217
	/*
	 * Search starts from the last successful isolation order or the next
	 * order to search after a previous failure
	 */
	cc->search_order = min_t(unsigned int, cc->order - 1, cc->search_order);

	for (order = cc->search_order;
	     !page && order >= 0;
	     order = next_search_order(cc, order)) {
1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240
		struct free_area *area = &cc->zone->free_area[order];
		struct list_head *freelist;
		struct page *freepage;
		unsigned long flags;
		unsigned int order_scanned = 0;

		if (!area->nr_free)
			continue;

		spin_lock_irqsave(&cc->zone->lock, flags);
		freelist = &area->free_list[MIGRATE_MOVABLE];
		list_for_each_entry_reverse(freepage, freelist, lru) {
			unsigned long pfn;

			order_scanned++;
			nr_scanned++;
			pfn = page_to_pfn(freepage);

			if (pfn >= highest)
				highest = pageblock_start_pfn(pfn);

			if (pfn >= low_pfn) {
				cc->fast_search_fail = 0;
1241
				cc->search_order = order;
1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312
				page = freepage;
				break;
			}

			if (pfn >= min_pfn && pfn > high_pfn) {
				high_pfn = pfn;

				/* Shorten the scan if a candidate is found */
				limit >>= 1;
			}

			if (order_scanned >= limit)
				break;
		}

		/* Use a minimum pfn if a preferred one was not found */
		if (!page && high_pfn) {
			page = pfn_to_page(high_pfn);

			/* Update freepage for the list reorder below */
			freepage = page;
		}

		/* Reorder to so a future search skips recent pages */
		move_freelist_head(freelist, freepage);

		/* Isolate the page if available */
		if (page) {
			if (__isolate_free_page(page, order)) {
				set_page_private(page, order);
				nr_isolated = 1 << order;
				cc->nr_freepages += nr_isolated;
				list_add_tail(&page->lru, &cc->freepages);
				count_compact_events(COMPACTISOLATED, nr_isolated);
			} else {
				/* If isolation fails, abort the search */
				order = -1;
				page = NULL;
			}
		}

		spin_unlock_irqrestore(&cc->zone->lock, flags);

		/*
		 * Smaller scan on next order so the total scan ig related
		 * to freelist_scan_limit.
		 */
		if (order_scanned >= limit)
			limit = min(1U, limit >> 1);
	}

	if (!page) {
		cc->fast_search_fail++;
		if (scan_start) {
			/*
			 * Use the highest PFN found above min. If one was
			 * not found, be pessemistic for direct compaction
			 * and use the min mark.
			 */
			if (highest) {
				page = pfn_to_page(highest);
				cc->free_pfn = highest;
			} else {
				if (cc->direct_compaction) {
					page = pfn_to_page(min_pfn);
					cc->free_pfn = min_pfn;
				}
			}
		}
	}

1313 1314
	if (highest && highest >= cc->zone->compact_cached_free_pfn) {
		highest -= pageblock_nr_pages;
1315
		cc->zone->compact_cached_free_pfn = highest;
1316
	}
1317 1318 1319 1320 1321 1322 1323 1324 1325 1326

	cc->total_free_scanned += nr_scanned;
	if (!page)
		return cc->free_pfn;

	low_pfn = page_to_pfn(page);
	fast_isolate_around(cc, low_pfn, nr_isolated);
	return low_pfn;
}

1327
/*
1328 1329
 * Based on information in the current compact_control, find blocks
 * suitable for isolating free pages from and then isolate them.
1330
 */
1331
static void isolate_freepages(struct compact_control *cc)
1332
{
1333
	struct zone *zone = cc->zone;
1334
	struct page *page;
1335
	unsigned long block_start_pfn;	/* start of current pageblock */
1336
	unsigned long isolate_start_pfn; /* exact pfn we start at */
1337 1338
	unsigned long block_end_pfn;	/* end of current pageblock */
	unsigned long low_pfn;	     /* lowest pfn scanner is able to scan */
1339
	struct list_head *freelist = &cc->freepages;
1340
	unsigned int stride;
1341

1342 1343 1344 1345 1346
	/* Try a small search of the free lists for a candidate */
	isolate_start_pfn = fast_isolate_freepages(cc);
	if (cc->nr_freepages)
		goto splitmap;

1347 1348
	/*
	 * Initialise the free scanner. The starting point is where we last
1349
	 * successfully isolated from, zone-cached value, or the end of the
1350 1351
	 * zone when isolating for the first time. For looping we also need
	 * this pfn aligned down to the pageblock boundary, because we do
1352 1353 1354
	 * block_start_pfn -= pageblock_nr_pages in the for loop.
	 * For ending point, take care when isolating in last pageblock of a
	 * a zone which ends in the middle of a pageblock.
1355 1356
	 * The low boundary is the end of the pageblock the migration scanner
	 * is using.
1357
	 */
1358
	isolate_start_pfn = cc->free_pfn;
1359
	block_start_pfn = pageblock_start_pfn(isolate_start_pfn);
1360 1361
	block_end_pfn = min(block_start_pfn + pageblock_nr_pages,
						zone_end_pfn(zone));
1362
	low_pfn = pageblock_end_pfn(cc->migrate_pfn);
1363
	stride = cc->mode == MIGRATE_ASYNC ? COMPACT_CLUSTER_MAX : 1;
1364

1365 1366 1367 1368 1369
	/*
	 * Isolate free pages until enough are available to migrate the
	 * pages on cc->migratepages. We stop searching if the migrate
	 * and free page scanners meet or enough free pages are isolated.
	 */
1370
	for (; block_start_pfn >= low_pfn;
1371
				block_end_pfn = block_start_pfn,
1372 1373
				block_start_pfn -= pageblock_nr_pages,
				isolate_start_pfn = block_start_pfn) {
1374 1375
		unsigned long nr_isolated;

1376 1377
		/*
		 * This can iterate a massively long zone without finding any
1378
		 * suitable migration targets, so periodically check resched.
1379
		 */
1380
		if (!(block_start_pfn % (SWAP_CLUSTER_MAX * pageblock_nr_pages)))
1381
			cond_resched();
1382

1383 1384 1385
		page = pageblock_pfn_to_page(block_start_pfn, block_end_pfn,
									zone);
		if (!page)
1386 1387 1388
			continue;

		/* Check the block is suitable for migration */
1389
		if (!suitable_migration_target(cc, page))
1390
			continue;
1391

1392 1393 1394 1395
		/* If isolation recently failed, do not retry */
		if (!isolation_suitable(cc, page))
			continue;

1396
		/* Found a block suitable for isolating free pages from. */
1397 1398
		nr_isolated = isolate_freepages_block(cc, &isolate_start_pfn,
					block_end_pfn, freelist, stride, false);
1399

1400 1401 1402 1403
		/* Update the skip hint if the full pageblock was scanned */
		if (isolate_start_pfn == block_end_pfn)
			update_pageblock_skip(cc, page, block_start_pfn);

1404 1405
		/* Are enough freepages isolated? */
		if (cc->nr_freepages >= cc->nr_migratepages) {
1406 1407 1408 1409 1410
			if (isolate_start_pfn >= block_end_pfn) {
				/*
				 * Restart at previous pageblock if more
				 * freepages can be isolated next time.
				 */
1411 1412
				isolate_start_pfn =
					block_start_pfn - pageblock_nr_pages;
1413
			}
1414
			break;
1415
		} else if (isolate_start_pfn < block_end_pfn) {
1416
			/*
1417 1418
			 * If isolation failed early, do not continue
			 * needlessly.
1419
			 */
1420
			break;
1421
		}
1422 1423 1424 1425 1426 1427 1428

		/* Adjust stride depending on isolation */
		if (nr_isolated) {
			stride = 1;
			continue;
		}
		stride = min_t(unsigned int, COMPACT_CLUSTER_MAX, stride << 1);
1429 1430
	}

1431
	/*
1432 1433 1434 1435
	 * Record where the free scanner will restart next time. Either we
	 * broke from the loop and set isolate_start_pfn based on the last
	 * call to isolate_freepages_block(), or we met the migration scanner
	 * and the loop terminated due to isolate_start_pfn < low_pfn
1436
	 */
1437
	cc->free_pfn = isolate_start_pfn;
1438 1439 1440 1441

splitmap:
	/* __isolate_free_page() does not map the pages */
	split_map_pages(freelist);
1442 1443 1444 1445 1446 1447 1448
}

/*
 * This is a migrate-callback that "allocates" freepages by taking pages
 * from the isolated freelists in the block we are migrating to.
 */
static struct page *compaction_alloc(struct page *migratepage,
1449
					unsigned long data)
1450 1451 1452 1453 1454
{
	struct compact_control *cc = (struct compact_control *)data;
	struct page *freepage;

	if (list_empty(&cc->freepages)) {
1455
		isolate_freepages(cc);
1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468

		if (list_empty(&cc->freepages))
			return NULL;
	}

	freepage = list_entry(cc->freepages.next, struct page, lru);
	list_del(&freepage->lru);
	cc->nr_freepages--;

	return freepage;
}

/*
1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480
 * This is a migrate-callback that "frees" freepages back to the isolated
 * freelist.  All pages on the freelist are from the same zone, so there is no
 * special handling needed for NUMA.
 */
static void compaction_free(struct page *page, unsigned long data)
{
	struct compact_control *cc = (struct compact_control *)data;

	list_add(&page->lru, &cc->freepages);
	cc->nr_freepages++;
}

1481 1482 1483 1484 1485 1486 1487
/* possible outcome of isolate_migratepages */
typedef enum {
	ISOLATE_ABORT,		/* Abort compaction now */
	ISOLATE_NONE,		/* No pages isolated, continue scanning */
	ISOLATE_SUCCESS,	/* Pages isolated, migrate */
} isolate_migrate_t;

1488 1489 1490 1491 1492 1493
/*
 * Allow userspace to control policy on scanning the unevictable LRU for
 * compactable pages.
 */
int sysctl_compact_unevictable_allowed __read_mostly = 1;

1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606
static inline void
update_fast_start_pfn(struct compact_control *cc, unsigned long pfn)
{
	if (cc->fast_start_pfn == ULONG_MAX)
		return;

	if (!cc->fast_start_pfn)
		cc->fast_start_pfn = pfn;

	cc->fast_start_pfn = min(cc->fast_start_pfn, pfn);
}

static inline unsigned long
reinit_migrate_pfn(struct compact_control *cc)
{
	if (!cc->fast_start_pfn || cc->fast_start_pfn == ULONG_MAX)
		return cc->migrate_pfn;

	cc->migrate_pfn = cc->fast_start_pfn;
	cc->fast_start_pfn = ULONG_MAX;

	return cc->migrate_pfn;
}

/*
 * Briefly search the free lists for a migration source that already has
 * some free pages to reduce the number of pages that need migration
 * before a pageblock is free.
 */
static unsigned long fast_find_migrateblock(struct compact_control *cc)
{
	unsigned int limit = freelist_scan_limit(cc);
	unsigned int nr_scanned = 0;
	unsigned long distance;
	unsigned long pfn = cc->migrate_pfn;
	unsigned long high_pfn;
	int order;

	/* Skip hints are relied on to avoid repeats on the fast search */
	if (cc->ignore_skip_hint)
		return pfn;

	/*
	 * If the migrate_pfn is not at the start of a zone or the start
	 * of a pageblock then assume this is a continuation of a previous
	 * scan restarted due to COMPACT_CLUSTER_MAX.
	 */
	if (pfn != cc->zone->zone_start_pfn && pfn != pageblock_start_pfn(pfn))
		return pfn;

	/*
	 * For smaller orders, just linearly scan as the number of pages
	 * to migrate should be relatively small and does not necessarily
	 * justify freeing up a large block for a small allocation.
	 */
	if (cc->order <= PAGE_ALLOC_COSTLY_ORDER)
		return pfn;

	/*
	 * Only allow kcompactd and direct requests for movable pages to
	 * quickly clear out a MOVABLE pageblock for allocation. This
	 * reduces the risk that a large movable pageblock is freed for
	 * an unmovable/reclaimable small allocation.
	 */
	if (cc->direct_compaction && cc->migratetype != MIGRATE_MOVABLE)
		return pfn;

	/*
	 * When starting the migration scanner, pick any pageblock within the
	 * first half of the search space. Otherwise try and pick a pageblock
	 * within the first eighth to reduce the chances that a migration
	 * target later becomes a source.
	 */
	distance = (cc->free_pfn - cc->migrate_pfn) >> 1;
	if (cc->migrate_pfn != cc->zone->zone_start_pfn)
		distance >>= 2;
	high_pfn = pageblock_start_pfn(cc->migrate_pfn + distance);

	for (order = cc->order - 1;
	     order >= PAGE_ALLOC_COSTLY_ORDER && pfn == cc->migrate_pfn && nr_scanned < limit;
	     order--) {
		struct free_area *area = &cc->zone->free_area[order];
		struct list_head *freelist;
		unsigned long flags;
		struct page *freepage;

		if (!area->nr_free)
			continue;

		spin_lock_irqsave(&cc->zone->lock, flags);
		freelist = &area->free_list[MIGRATE_MOVABLE];
		list_for_each_entry(freepage, freelist, lru) {
			unsigned long free_pfn;

			nr_scanned++;
			free_pfn = page_to_pfn(freepage);
			if (free_pfn < high_pfn) {
				/*
				 * Avoid if skipped recently. Ideally it would
				 * move to the tail but even safe iteration of
				 * the list assumes an entry is deleted, not
				 * reordered.
				 */
				if (get_pageblock_skip(freepage)) {
					if (list_is_last(freelist, &freepage->lru))
						break;

					continue;
				}

				/* Reorder to so a future search skips recent pages */
				move_freelist_tail(freelist, freepage);

1607
				update_fast_start_pfn(cc, free_pfn);
1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634
				pfn = pageblock_start_pfn(free_pfn);
				cc->fast_search_fail = 0;
				set_pageblock_skip(freepage);
				break;
			}

			if (nr_scanned >= limit) {
				cc->fast_search_fail++;
				move_freelist_tail(freelist, freepage);
				break;
			}
		}
		spin_unlock_irqrestore(&cc->zone->lock, flags);
	}

	cc->total_migrate_scanned += nr_scanned;

	/*
	 * If fast scanning failed then use a cached entry for a page block
	 * that had free pages as the basis for starting a linear scan.
	 */
	if (pfn == cc->migrate_pfn)
		pfn = reinit_migrate_pfn(cc);

	return pfn;
}

1635
/*
1636 1637 1638
 * Isolate all pages that can be migrated from the first suitable block,
 * starting at the block pointed to by the migrate scanner pfn within
 * compact_control.
1639 1640 1641 1642
 */
static isolate_migrate_t isolate_migratepages(struct zone *zone,
					struct compact_control *cc)
{
1643 1644 1645
	unsigned long block_start_pfn;
	unsigned long block_end_pfn;
	unsigned long low_pfn;
1646 1647
	struct page *page;
	const isolate_mode_t isolate_mode =
1648
		(sysctl_compact_unevictable_allowed ? ISOLATE_UNEVICTABLE : 0) |
1649
		(cc->mode != MIGRATE_SYNC ? ISOLATE_ASYNC_MIGRATE : 0);
1650
	bool fast_find_block;
1651

1652 1653
	/*
	 * Start at where we last stopped, or beginning of the zone as
1654 1655
	 * initialized by compact_zone(). The first failure will use
	 * the lowest PFN as the starting point for linear scanning.
1656
	 */
1657
	low_pfn = fast_find_migrateblock(cc);
1658
	block_start_pfn = pageblock_start_pfn(low_pfn);
1659 1660
	if (block_start_pfn < zone->zone_start_pfn)
		block_start_pfn = zone->zone_start_pfn;
1661

1662 1663 1664 1665 1666 1667 1668
	/*
	 * fast_find_migrateblock marks a pageblock skipped so to avoid
	 * the isolation_suitable check below, check whether the fast
	 * search was successful.
	 */
	fast_find_block = low_pfn != cc->migrate_pfn && !cc->fast_search_fail;

1669
	/* Only scan within a pageblock boundary */
1670
	block_end_pfn = pageblock_end_pfn(low_pfn);
1671

1672 1673 1674 1675
	/*
	 * Iterate over whole pageblocks until we find the first suitable.
	 * Do not cross the free scanner.
	 */
1676
	for (; block_end_pfn <= cc->free_pfn;
1677
			fast_find_block = false,
1678 1679 1680
			low_pfn = block_end_pfn,
			block_start_pfn = block_end_pfn,
			block_end_pfn += pageblock_nr_pages) {
1681

1682 1683 1684
		/*
		 * This can potentially iterate a massively long zone with
		 * many pageblocks unsuitable, so periodically check if we
1685
		 * need to schedule.
1686
		 */
1687
		if (!(low_pfn % (SWAP_CLUSTER_MAX * pageblock_nr_pages)))
1688
			cond_resched();
1689

1690 1691
		page = pageblock_pfn_to_page(block_start_pfn, block_end_pfn,
									zone);
1692
		if (!page)
1693 1694
			continue;

1695 1696 1697 1698 1699 1700 1701 1702 1703
		/*
		 * If isolation recently failed, do not retry. Only check the
		 * pageblock once. COMPACT_CLUSTER_MAX causes a pageblock
		 * to be visited multiple times. Assume skip was checked
		 * before making it "skip" so other compaction instances do
		 * not scan the same block.
		 */
		if (IS_ALIGNED(low_pfn, pageblock_nr_pages) &&
		    !fast_find_block && !isolation_suitable(cc, page))
1704 1705 1706
			continue;

		/*
1707 1708 1709 1710 1711 1712
		 * For async compaction, also only scan in MOVABLE blocks
		 * without huge pages. Async compaction is optimistic to see
		 * if the minimum amount of work satisfies the allocation.
		 * The cached PFN is updated as it's possible that all
		 * remaining blocks between source and target are unsuitable
		 * and the compaction scanners fail to meet.
1713
		 */
1714 1715
		if (!suitable_migration_source(cc, page)) {
			update_cached_migrate(cc, block_end_pfn);
1716
			continue;
1717
		}
1718 1719

		/* Perform the isolation */
1720 1721
		low_pfn = isolate_migratepages_block(cc, low_pfn,
						block_end_pfn, isolate_mode);
1722

1723
		if (!low_pfn)
1724 1725 1726 1727 1728 1729 1730 1731 1732 1733
			return ISOLATE_ABORT;

		/*
		 * Either we isolated something and proceed with migration. Or
		 * we failed and compact_zone should decide if we should
		 * continue or not.
		 */
		break;
	}

1734 1735
	/* Record where migration scanner will be restarted. */
	cc->migrate_pfn = low_pfn;
1736

1737
	return cc->nr_migratepages ? ISOLATE_SUCCESS : ISOLATE_NONE;
1738 1739
}

1740 1741 1742 1743 1744 1745 1746 1747 1748
/*
 * order == -1 is expected when compacting via
 * /proc/sys/vm/compact_memory
 */
static inline bool is_via_compact_memory(int order)
{
	return order == -1;
}

1749
static enum compact_result __compact_finished(struct compact_control *cc)
1750
{
1751
	unsigned int order;
1752
	const int migratetype = cc->migratetype;
1753
	int ret;
1754

1755
	/* Compaction run completes if the migrate and free scanner meet */
1756
	if (compact_scanners_met(cc)) {
1757
		/* Let the next compaction start anew. */
1758
		reset_cached_positions(cc->zone);
1759

1760 1761
		/*
		 * Mark that the PG_migrate_skip information should be cleared
1762
		 * by kswapd when it goes to sleep. kcompactd does not set the
1763 1764 1765
		 * flag itself as the decision to be clear should be directly
		 * based on an allocation request.
		 */
1766
		if (cc->direct_compaction)
1767
			cc->zone->compact_blockskip_flush = true;
1768

1769 1770 1771 1772
		if (cc->whole_zone)
			return COMPACT_COMPLETE;
		else
			return COMPACT_PARTIAL_SKIPPED;
1773
	}
1774

1775
	if (is_via_compact_memory(cc->order))
1776 1777
		return COMPACT_CONTINUE;

1778 1779 1780 1781 1782 1783 1784 1785
	/*
	 * Always finish scanning a pageblock to reduce the possibility of
	 * fallbacks in the future. This is particularly important when
	 * migration source is unmovable/reclaimable but it's not worth
	 * special casing.
	 */
	if (!IS_ALIGNED(cc->migrate_pfn, pageblock_nr_pages))
		return COMPACT_CONTINUE;
1786

1787
	/* Direct compactor: Is a suitable page free? */
1788
	ret = COMPACT_NO_SUITABLE_PAGE;
1789
	for (order = cc->order; order < MAX_ORDER; order++) {
1790
		struct free_area *area = &cc->zone->free_area[order];
1791
		bool can_steal;
1792 1793

		/* Job done if page is free of the right migratetype */
1794
		if (!free_area_empty(area, migratetype))
1795
			return COMPACT_SUCCESS;
1796

1797 1798 1799
#ifdef CONFIG_CMA
		/* MIGRATE_MOVABLE can fallback on MIGRATE_CMA */
		if (migratetype == MIGRATE_MOVABLE &&
1800
			!free_area_empty(area, MIGRATE_CMA))
1801
			return COMPACT_SUCCESS;
1802 1803 1804 1805 1806 1807
#endif
		/*
		 * Job done if allocation would steal freepages from
		 * other migratetype buddy lists.
		 */
		if (find_suitable_fallback(area, order, migratetype,
1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827
						true, &can_steal) != -1) {

			/* movable pages are OK in any pageblock */
			if (migratetype == MIGRATE_MOVABLE)
				return COMPACT_SUCCESS;

			/*
			 * We are stealing for a non-movable allocation. Make
			 * sure we finish compacting the current pageblock
			 * first so it is as free as possible and we won't
			 * have to steal another one soon. This only applies
			 * to sync compaction, as async compaction operates
			 * on pageblocks of the same migratetype.
			 */
			if (cc->mode == MIGRATE_ASYNC ||
					IS_ALIGNED(cc->migrate_pfn,
							pageblock_nr_pages)) {
				return COMPACT_SUCCESS;
			}

1828 1829
			ret = COMPACT_CONTINUE;
			break;
1830
		}
1831 1832
	}

1833 1834 1835 1836
	if (cc->contended || fatal_signal_pending(current))
		ret = COMPACT_CONTENDED;

	return ret;
1837 1838
}

1839
static enum compact_result compact_finished(struct compact_control *cc)
1840 1841 1842
{
	int ret;

1843 1844
	ret = __compact_finished(cc);
	trace_mm_compaction_finished(cc->zone, cc->order, ret);
1845 1846 1847 1848
	if (ret == COMPACT_NO_SUITABLE_PAGE)
		ret = COMPACT_CONTINUE;

	return ret;
1849 1850
}

1851 1852 1853 1854
/*
 * compaction_suitable: Is this suitable to run compaction on this zone now?
 * Returns
 *   COMPACT_SKIPPED  - If there are too few free pages for compaction
1855
 *   COMPACT_SUCCESS  - If the allocation would succeed without compaction
1856 1857
 *   COMPACT_CONTINUE - If compaction should run now
 */
1858
static enum compact_result __compaction_suitable(struct zone *zone, int order,
1859
					unsigned int alloc_flags,
1860 1861
					int classzone_idx,
					unsigned long wmark_target)
1862 1863 1864
{
	unsigned long watermark;

1865
	if (is_via_compact_memory(order))
1866 1867
		return COMPACT_CONTINUE;

1868
	watermark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);
1869 1870 1871 1872 1873 1874
	/*
	 * If watermarks for high-order allocation are already met, there
	 * should be no need for compaction at all.
	 */
	if (zone_watermark_ok(zone, order, watermark, classzone_idx,
								alloc_flags))
1875
		return COMPACT_SUCCESS;
1876

1877
	/*
1878
	 * Watermarks for order-0 must be met for compaction to be able to
1879 1880 1881 1882 1883 1884 1885
	 * isolate free pages for migration targets. This means that the
	 * watermark and alloc_flags have to match, or be more pessimistic than
	 * the check in __isolate_free_page(). We don't use the direct
	 * compactor's alloc_flags, as they are not relevant for freepage
	 * isolation. We however do use the direct compactor's classzone_idx to
	 * skip over zones where lowmem reserves would prevent allocation even
	 * if compaction succeeds.
1886 1887
	 * For costly orders, we require low watermark instead of min for
	 * compaction to proceed to increase its chances.
1888 1889
	 * ALLOC_CMA is used, as pages in CMA pageblocks are considered
	 * suitable migration targets
1890
	 */
1891 1892 1893
	watermark = (order > PAGE_ALLOC_COSTLY_ORDER) ?
				low_wmark_pages(zone) : min_wmark_pages(zone);
	watermark += compact_gap(order);
1894
	if (!__zone_watermark_ok(zone, 0, watermark, classzone_idx,
1895
						ALLOC_CMA, wmark_target))
1896 1897
		return COMPACT_SKIPPED;

1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909
	return COMPACT_CONTINUE;
}

enum compact_result compaction_suitable(struct zone *zone, int order,
					unsigned int alloc_flags,
					int classzone_idx)
{
	enum compact_result ret;
	int fragindex;

	ret = __compaction_suitable(zone, order, alloc_flags, classzone_idx,
				    zone_page_state(zone, NR_FREE_PAGES));
1910 1911 1912 1913
	/*
	 * fragmentation index determines if allocation failures are due to
	 * low memory or external fragmentation
	 *
1914 1915
	 * index of -1000 would imply allocations might succeed depending on
	 * watermarks, but we already failed the high-order watermark check
1916 1917 1918
	 * index towards 0 implies failure is due to lack of memory
	 * index towards 1000 implies failure is due to fragmentation
	 *
1919 1920 1921 1922 1923 1924
	 * Only compact if a failure would be due to fragmentation. Also
	 * ignore fragindex for non-costly orders where the alternative to
	 * a successful reclaim/compaction is OOM. Fragindex and the
	 * vm.extfrag_threshold sysctl is meant as a heuristic to prevent
	 * excessive compaction for costly orders, but it should not be at the
	 * expense of system stability.
1925
	 */
1926
	if (ret == COMPACT_CONTINUE && (order > PAGE_ALLOC_COSTLY_ORDER)) {
1927 1928 1929 1930
		fragindex = fragmentation_index(zone, order);
		if (fragindex >= 0 && fragindex <= sysctl_extfrag_threshold)
			ret = COMPACT_NOT_SUITABLE_ZONE;
	}
1931 1932 1933 1934 1935 1936 1937 1938

	trace_mm_compaction_suitable(zone, order, ret);
	if (ret == COMPACT_NOT_SUITABLE_ZONE)
		ret = COMPACT_SKIPPED;

	return ret;
}

1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959
bool compaction_zonelist_suitable(struct alloc_context *ac, int order,
		int alloc_flags)
{
	struct zone *zone;
	struct zoneref *z;

	/*
	 * Make sure at least one zone would pass __compaction_suitable if we continue
	 * retrying the reclaim.
	 */
	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
					ac->nodemask) {
		unsigned long available;
		enum compact_result compact_result;

		/*
		 * Do not consider all the reclaimable memory because we do not
		 * want to trash just for a single high order allocation which
		 * is even not guaranteed to appear even if __compaction_suitable
		 * is happy about the watermark check.
		 */
1960
		available = zone_reclaimable_pages(zone) / order;
1961 1962 1963
		available += zone_page_state_snapshot(zone, NR_FREE_PAGES);
		compact_result = __compaction_suitable(zone, order, alloc_flags,
				ac_classzone_idx(ac), available);
1964
		if (compact_result != COMPACT_SKIPPED)
1965 1966 1967 1968 1969 1970
			return true;
	}

	return false;
}

1971
static enum compact_result compact_zone(struct compact_control *cc)
1972
{
1973
	enum compact_result ret;
1974 1975
	unsigned long start_pfn = cc->zone->zone_start_pfn;
	unsigned long end_pfn = zone_end_pfn(cc->zone);
1976
	unsigned long last_migrated_pfn;
1977
	const bool sync = cc->mode != MIGRATE_ASYNC;
1978
	bool update_cached;
1979

1980
	cc->migratetype = gfpflags_to_migratetype(cc->gfp_mask);
1981
	ret = compaction_suitable(cc->zone, cc->order, cc->alloc_flags,
1982
							cc->classzone_idx);
1983
	/* Compaction is likely to fail */
1984
	if (ret == COMPACT_SUCCESS || ret == COMPACT_SKIPPED)
1985
		return ret;
1986 1987 1988

	/* huh, compaction_suitable is returning something unexpected */
	VM_BUG_ON(ret != COMPACT_CONTINUE);
1989

1990 1991
	/*
	 * Clear pageblock skip if there were failures recently and compaction
1992
	 * is about to be retried after being deferred.
1993
	 */
1994 1995
	if (compaction_restarting(cc->zone, cc->order))
		__reset_isolation_suitable(cc->zone);
1996

1997 1998
	/*
	 * Setup to move all movable pages to the end of the zone. Used cached
1999 2000 2001
	 * information on where the scanners should start (unless we explicitly
	 * want to compact the whole zone), but check that it is initialised
	 * by ensuring the values are within zone boundaries.
2002
	 */
2003
	cc->fast_start_pfn = 0;
2004
	if (cc->whole_zone) {
2005
		cc->migrate_pfn = start_pfn;
2006 2007
		cc->free_pfn = pageblock_start_pfn(end_pfn - 1);
	} else {
2008 2009
		cc->migrate_pfn = cc->zone->compact_cached_migrate_pfn[sync];
		cc->free_pfn = cc->zone->compact_cached_free_pfn;
2010 2011
		if (cc->free_pfn < start_pfn || cc->free_pfn >= end_pfn) {
			cc->free_pfn = pageblock_start_pfn(end_pfn - 1);
2012
			cc->zone->compact_cached_free_pfn = cc->free_pfn;
2013 2014 2015
		}
		if (cc->migrate_pfn < start_pfn || cc->migrate_pfn >= end_pfn) {
			cc->migrate_pfn = start_pfn;
2016 2017
			cc->zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn;
			cc->zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn;
2018
		}
2019

2020 2021 2022
		if (cc->migrate_pfn == start_pfn)
			cc->whole_zone = true;
	}
2023

2024
	last_migrated_pfn = 0;
2025

2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036
	/*
	 * Migrate has separate cached PFNs for ASYNC and SYNC* migration on
	 * the basis that some migrations will fail in ASYNC mode. However,
	 * if the cached PFNs match and pageblocks are skipped due to having
	 * no isolation candidates, then the sync state does not matter.
	 * Until a pageblock with isolation candidates is found, keep the
	 * cached PFNs in sync to avoid revisiting the same blocks.
	 */
	update_cached = !sync &&
		cc->zone->compact_cached_migrate_pfn[0] == cc->zone->compact_cached_migrate_pfn[1];

2037 2038
	trace_mm_compaction_begin(start_pfn, cc->migrate_pfn,
				cc->free_pfn, end_pfn, sync);
2039

2040 2041
	migrate_prep_local();

2042
	while ((ret = compact_finished(cc)) == COMPACT_CONTINUE) {
2043
		int err;
2044
		unsigned long start_pfn = cc->migrate_pfn;
2045

2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059
		/*
		 * Avoid multiple rescans which can happen if a page cannot be
		 * isolated (dirty/writeback in async mode) or if the migrated
		 * pages are being allocated before the pageblock is cleared.
		 * The first rescan will capture the entire pageblock for
		 * migration. If it fails, it'll be marked skip and scanning
		 * will proceed as normal.
		 */
		cc->rescan = false;
		if (pageblock_start_pfn(last_migrated_pfn) ==
		    pageblock_start_pfn(start_pfn)) {
			cc->rescan = true;
		}

2060
		switch (isolate_migratepages(cc->zone, cc)) {
2061
		case ISOLATE_ABORT:
2062
			ret = COMPACT_CONTENDED;
2063
			putback_movable_pages(&cc->migratepages);
2064
			cc->nr_migratepages = 0;
2065
			last_migrated_pfn = 0;
2066 2067
			goto out;
		case ISOLATE_NONE:
2068 2069 2070 2071 2072
			if (update_cached) {
				cc->zone->compact_cached_migrate_pfn[1] =
					cc->zone->compact_cached_migrate_pfn[0];
			}

2073 2074 2075 2076 2077 2078
			/*
			 * We haven't isolated and migrated anything, but
			 * there might still be unflushed migrations from
			 * previous cc->order aligned block.
			 */
			goto check_drain;
2079
		case ISOLATE_SUCCESS:
2080
			update_cached = false;
2081
			last_migrated_pfn = start_pfn;
2082 2083
			;
		}
2084

2085
		err = migrate_pages(&cc->migratepages, compaction_alloc,
2086
				compaction_free, (unsigned long)cc, cc->mode,
2087
				MR_COMPACTION);
2088

2089 2090
		trace_mm_compaction_migratepages(cc->nr_migratepages, err,
							&cc->migratepages);
2091

2092 2093
		/* All pages were either migrated or will be released */
		cc->nr_migratepages = 0;
2094
		if (err) {
2095
			putback_movable_pages(&cc->migratepages);
2096 2097 2098 2099
			/*
			 * migrate_pages() may return -ENOMEM when scanners meet
			 * and we want compact_finished() to detect it
			 */
2100
			if (err == -ENOMEM && !compact_scanners_met(cc)) {
2101
				ret = COMPACT_CONTENDED;
2102 2103
				goto out;
			}
2104 2105 2106 2107 2108 2109 2110 2111 2112
			/*
			 * We failed to migrate at least one page in the current
			 * order-aligned block, so skip the rest of it.
			 */
			if (cc->direct_compaction &&
						(cc->mode == MIGRATE_ASYNC)) {
				cc->migrate_pfn = block_end_pfn(
						cc->migrate_pfn - 1, cc->order);
				/* Draining pcplists is useless in this case */
2113
				last_migrated_pfn = 0;
2114
			}
2115
		}
2116 2117 2118 2119 2120 2121 2122 2123 2124

check_drain:
		/*
		 * Has the migration scanner moved away from the previous
		 * cc->order aligned block where we migrated from? If yes,
		 * flush the pages that were freed, so that they can merge and
		 * compact_finished() can detect immediately if allocation
		 * would succeed.
		 */
2125
		if (cc->order > 0 && last_migrated_pfn) {
2126 2127
			int cpu;
			unsigned long current_block_start =
2128
				block_start_pfn(cc->migrate_pfn, cc->order);
2129

2130
			if (last_migrated_pfn < current_block_start) {
2131 2132
				cpu = get_cpu();
				lru_add_drain_cpu(cpu);
2133
				drain_local_pages(cc->zone);
2134 2135
				put_cpu();
				/* No more flushing until we migrate again */
2136
				last_migrated_pfn = 0;
2137 2138 2139
			}
		}

2140 2141
	}

2142
out:
2143 2144 2145 2146 2147 2148 2149 2150 2151 2152
	/*
	 * Release free pages and update where the free scanner should restart,
	 * so we don't leave any returned pages behind in the next attempt.
	 */
	if (cc->nr_freepages > 0) {
		unsigned long free_pfn = release_freepages(&cc->freepages);

		cc->nr_freepages = 0;
		VM_BUG_ON(free_pfn == 0);
		/* The cached pfn is always the first in a pageblock */
2153
		free_pfn = pageblock_start_pfn(free_pfn);
2154 2155 2156 2157
		/*
		 * Only go back, not forward. The cached pfn might have been
		 * already reset to zone end in compact_finished()
		 */
2158 2159
		if (free_pfn > cc->zone->compact_cached_free_pfn)
			cc->zone->compact_cached_free_pfn = free_pfn;
2160
	}
2161

2162 2163 2164
	count_compact_events(COMPACTMIGRATE_SCANNED, cc->total_migrate_scanned);
	count_compact_events(COMPACTFREE_SCANNED, cc->total_free_scanned);

2165 2166
	trace_mm_compaction_end(start_pfn, cc->migrate_pfn,
				cc->free_pfn, end_pfn, sync, ret);
2167

2168 2169
	return ret;
}
2170

2171
static enum compact_result compact_zone_order(struct zone *zone, int order,
2172
		gfp_t gfp_mask, enum compact_priority prio,
2173
		unsigned int alloc_flags, int classzone_idx)
2174
{
2175
	enum compact_result ret;
2176
	struct compact_control cc = {
2177 2178 2179 2180
		.nr_freepages = 0,
		.nr_migratepages = 0,
		.total_migrate_scanned = 0,
		.total_free_scanned = 0,
2181
		.order = order,
2182
		.search_order = order,
2183
		.gfp_mask = gfp_mask,
2184
		.zone = zone,
2185 2186
		.mode = (prio == COMPACT_PRIO_ASYNC) ?
					MIGRATE_ASYNC :	MIGRATE_SYNC_LIGHT,
2187 2188
		.alloc_flags = alloc_flags,
		.classzone_idx = classzone_idx,
2189
		.direct_compaction = true,
2190
		.whole_zone = (prio == MIN_COMPACT_PRIORITY),
2191 2192
		.ignore_skip_hint = (prio == MIN_COMPACT_PRIORITY),
		.ignore_block_suitable = (prio == MIN_COMPACT_PRIORITY)
2193
	};
2194 2195
	INIT_LIST_HEAD(&cc.freepages);
	INIT_LIST_HEAD(&cc.migratepages);
2196

2197
	ret = compact_zone(&cc);
2198 2199 2200 2201 2202

	VM_BUG_ON(!list_empty(&cc.freepages));
	VM_BUG_ON(!list_empty(&cc.migratepages));

	return ret;
2203 2204
}

2205 2206
int sysctl_extfrag_threshold = 500;

2207 2208 2209
/**
 * try_to_compact_pages - Direct compact to satisfy a high-order allocation
 * @gfp_mask: The GFP mask of the current allocation
2210 2211 2212
 * @order: The order of the current allocation
 * @alloc_flags: The allocation flags of the current allocation
 * @ac: The context of current allocation
2213
 * @prio: Determines how hard direct compaction should try to succeed
2214 2215 2216
 *
 * This is the main entry point for direct page compaction.
 */
2217
enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order,
2218
		unsigned int alloc_flags, const struct alloc_context *ac,
2219
		enum compact_priority prio)
2220 2221 2222 2223
{
	int may_perform_io = gfp_mask & __GFP_IO;
	struct zoneref *z;
	struct zone *zone;
2224
	enum compact_result rc = COMPACT_SKIPPED;
2225

2226 2227 2228 2229 2230
	/*
	 * Check if the GFP flags allow compaction - GFP_NOIO is really
	 * tricky context because the migration might require IO
	 */
	if (!may_perform_io)
2231
		return COMPACT_SKIPPED;
2232

2233
	trace_mm_compaction_try_to_compact_pages(order, gfp_mask, prio);
2234

2235
	/* Compact each zone in the list */
2236 2237
	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
								ac->nodemask) {
2238
		enum compact_result status;
2239

2240 2241
		if (prio > MIN_COMPACT_PRIORITY
					&& compaction_deferred(zone, order)) {
2242
			rc = max_t(enum compact_result, COMPACT_DEFERRED, rc);
2243
			continue;
2244
		}
2245

2246
		status = compact_zone_order(zone, order, gfp_mask, prio,
2247
					alloc_flags, ac_classzone_idx(ac));
2248 2249
		rc = max(status, rc);

2250 2251
		/* The allocation should succeed, stop compacting */
		if (status == COMPACT_SUCCESS) {
2252 2253 2254 2255 2256 2257 2258
			/*
			 * We think the allocation will succeed in this zone,
			 * but it is not certain, hence the false. The caller
			 * will repeat this with true if allocation indeed
			 * succeeds in this zone.
			 */
			compaction_defer_reset(zone, order, false);
2259

2260
			break;
2261 2262
		}

2263
		if (prio != COMPACT_PRIO_ASYNC && (status == COMPACT_COMPLETE ||
2264
					status == COMPACT_PARTIAL_SKIPPED))
2265 2266 2267 2268 2269 2270
			/*
			 * We think that allocation won't succeed in this zone
			 * so we defer compaction there. If it ends up
			 * succeeding after all, it will be reset.
			 */
			defer_compaction(zone, order);
2271 2272 2273 2274

		/*
		 * We might have stopped compacting due to need_resched() in
		 * async compaction, or due to a fatal signal detected. In that
2275
		 * case do not try further zones
2276
		 */
2277 2278 2279
		if ((prio == COMPACT_PRIO_ASYNC && need_resched())
					|| fatal_signal_pending(current))
			break;
2280 2281 2282 2283 2284 2285
	}

	return rc;
}


2286
/* Compact all zones within a node */
2287
static void compact_node(int nid)
2288
{
2289
	pg_data_t *pgdat = NODE_DATA(nid);
2290 2291
	int zoneid;
	struct zone *zone;
2292 2293
	struct compact_control cc = {
		.order = -1,
2294 2295
		.total_migrate_scanned = 0,
		.total_free_scanned = 0,
2296 2297 2298
		.mode = MIGRATE_SYNC,
		.ignore_skip_hint = true,
		.whole_zone = true,
2299
		.gfp_mask = GFP_KERNEL,
2300 2301
	};

2302 2303 2304 2305 2306 2307 2308

	for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {

		zone = &pgdat->node_zones[zoneid];
		if (!populated_zone(zone))
			continue;

2309 2310
		cc.nr_freepages = 0;
		cc.nr_migratepages = 0;
2311
		cc.zone = zone;
2312 2313
		INIT_LIST_HEAD(&cc.freepages);
		INIT_LIST_HEAD(&cc.migratepages);
2314

2315
		compact_zone(&cc);
2316

2317 2318
		VM_BUG_ON(!list_empty(&cc.freepages));
		VM_BUG_ON(!list_empty(&cc.migratepages));
2319 2320 2321 2322
	}
}

/* Compact all nodes in the system */
2323
static void compact_nodes(void)
2324 2325 2326
{
	int nid;

2327 2328 2329
	/* Flush pending updates to the LRU lists */
	lru_add_drain_all();

2330 2331 2332 2333 2334 2335 2336
	for_each_online_node(nid)
		compact_node(nid);
}

/* The written value is actually unused, all memory is compacted */
int sysctl_compact_memory;

2337 2338 2339 2340
/*
 * This is the entry point for compacting all nodes via
 * /proc/sys/vm/compact_memory
 */
2341 2342 2343 2344
int sysctl_compaction_handler(struct ctl_table *table, int write,
			void __user *buffer, size_t *length, loff_t *ppos)
{
	if (write)
2345
		compact_nodes();
2346 2347 2348

	return 0;
}
2349

2350 2351 2352 2353 2354 2355 2356 2357
int sysctl_extfrag_handler(struct ctl_table *table, int write,
			void __user *buffer, size_t *length, loff_t *ppos)
{
	proc_dointvec_minmax(table, write, buffer, length, ppos);

	return 0;
}

2358
#if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA)
2359
static ssize_t sysfs_compact_node(struct device *dev,
2360
			struct device_attribute *attr,
2361 2362
			const char *buf, size_t count)
{
2363 2364 2365 2366 2367 2368 2369 2370
	int nid = dev->id;

	if (nid >= 0 && nid < nr_node_ids && node_online(nid)) {
		/* Flush pending updates to the LRU lists */
		lru_add_drain_all();

		compact_node(nid);
	}
2371 2372 2373

	return count;
}
2374
static DEVICE_ATTR(compact, 0200, NULL, sysfs_compact_node);
2375 2376 2377

int compaction_register_node(struct node *node)
{
2378
	return device_create_file(&node->dev, &dev_attr_compact);
2379 2380 2381 2382
}

void compaction_unregister_node(struct node *node)
{
2383
	return device_remove_file(&node->dev, &dev_attr_compact);
2384 2385
}
#endif /* CONFIG_SYSFS && CONFIG_NUMA */
2386

2387 2388
static inline bool kcompactd_work_requested(pg_data_t *pgdat)
{
2389
	return pgdat->kcompactd_max_order > 0 || kthread_should_stop();
2390 2391 2392 2393 2394 2395 2396 2397
}

static bool kcompactd_node_suitable(pg_data_t *pgdat)
{
	int zoneid;
	struct zone *zone;
	enum zone_type classzone_idx = pgdat->kcompactd_classzone_idx;

2398
	for (zoneid = 0; zoneid <= classzone_idx; zoneid++) {
2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421
		zone = &pgdat->node_zones[zoneid];

		if (!populated_zone(zone))
			continue;

		if (compaction_suitable(zone, pgdat->kcompactd_max_order, 0,
					classzone_idx) == COMPACT_CONTINUE)
			return true;
	}

	return false;
}

static void kcompactd_do_work(pg_data_t *pgdat)
{
	/*
	 * With no special task, compact all zones so that a page of requested
	 * order is allocatable.
	 */
	int zoneid;
	struct zone *zone;
	struct compact_control cc = {
		.order = pgdat->kcompactd_max_order,
2422
		.search_order = pgdat->kcompactd_max_order,
2423 2424
		.total_migrate_scanned = 0,
		.total_free_scanned = 0,
2425 2426
		.classzone_idx = pgdat->kcompactd_classzone_idx,
		.mode = MIGRATE_SYNC_LIGHT,
2427
		.ignore_skip_hint = false,
2428
		.gfp_mask = GFP_KERNEL,
2429 2430 2431
	};
	trace_mm_compaction_kcompactd_wake(pgdat->node_id, cc.order,
							cc.classzone_idx);
2432
	count_compact_event(KCOMPACTD_WAKE);
2433

2434
	for (zoneid = 0; zoneid <= cc.classzone_idx; zoneid++) {
2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447
		int status;

		zone = &pgdat->node_zones[zoneid];
		if (!populated_zone(zone))
			continue;

		if (compaction_deferred(zone, cc.order))
			continue;

		if (compaction_suitable(zone, cc.order, 0, zoneid) !=
							COMPACT_CONTINUE)
			continue;

2448 2449 2450 2451 2452 2453 2454 2455
		cc.nr_freepages = 0;
		cc.nr_migratepages = 0;
		cc.total_migrate_scanned = 0;
		cc.total_free_scanned = 0;
		cc.zone = zone;
		INIT_LIST_HEAD(&cc.freepages);
		INIT_LIST_HEAD(&cc.migratepages);

2456 2457
		if (kthread_should_stop())
			return;
2458
		status = compact_zone(&cc);
2459

2460
		if (status == COMPACT_SUCCESS) {
2461
			compaction_defer_reset(zone, cc.order, false);
2462
		} else if (status == COMPACT_PARTIAL_SKIPPED || status == COMPACT_COMPLETE) {
2463 2464 2465 2466 2467 2468 2469 2470
			/*
			 * Buddy pages may become stranded on pcps that could
			 * otherwise coalesce on the zone's free area for
			 * order >= cc.order.  This is ratelimited by the
			 * upcoming deferral.
			 */
			drain_all_pages(zone);

2471 2472 2473 2474 2475 2476 2477
			/*
			 * We use sync migration mode here, so we defer like
			 * sync direct compaction does.
			 */
			defer_compaction(zone, cc.order);
		}

2478 2479 2480 2481 2482
		count_compact_events(KCOMPACTD_MIGRATE_SCANNED,
				     cc.total_migrate_scanned);
		count_compact_events(KCOMPACTD_FREE_SCANNED,
				     cc.total_free_scanned);

2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508
		VM_BUG_ON(!list_empty(&cc.freepages));
		VM_BUG_ON(!list_empty(&cc.migratepages));
	}

	/*
	 * Regardless of success, we are done until woken up next. But remember
	 * the requested order/classzone_idx in case it was higher/tighter than
	 * our current ones
	 */
	if (pgdat->kcompactd_max_order <= cc.order)
		pgdat->kcompactd_max_order = 0;
	if (pgdat->kcompactd_classzone_idx >= cc.classzone_idx)
		pgdat->kcompactd_classzone_idx = pgdat->nr_zones - 1;
}

void wakeup_kcompactd(pg_data_t *pgdat, int order, int classzone_idx)
{
	if (!order)
		return;

	if (pgdat->kcompactd_max_order < order)
		pgdat->kcompactd_max_order = order;

	if (pgdat->kcompactd_classzone_idx > classzone_idx)
		pgdat->kcompactd_classzone_idx = classzone_idx;

2509 2510 2511 2512 2513
	/*
	 * Pairs with implicit barrier in wait_event_freezable()
	 * such that wakeups are not missed.
	 */
	if (!wq_has_sleeper(&pgdat->kcompactd_wait))
2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543
		return;

	if (!kcompactd_node_suitable(pgdat))
		return;

	trace_mm_compaction_wakeup_kcompactd(pgdat->node_id, order,
							classzone_idx);
	wake_up_interruptible(&pgdat->kcompactd_wait);
}

/*
 * The background compaction daemon, started as a kernel thread
 * from the init process.
 */
static int kcompactd(void *p)
{
	pg_data_t *pgdat = (pg_data_t*)p;
	struct task_struct *tsk = current;

	const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);

	if (!cpumask_empty(cpumask))
		set_cpus_allowed_ptr(tsk, cpumask);

	set_freezable();

	pgdat->kcompactd_max_order = 0;
	pgdat->kcompactd_classzone_idx = pgdat->nr_zones - 1;

	while (!kthread_should_stop()) {
2544 2545
		unsigned long pflags;

2546 2547 2548 2549
		trace_mm_compaction_kcompactd_sleep(pgdat->node_id);
		wait_event_freezable(pgdat->kcompactd_wait,
				kcompactd_work_requested(pgdat));

2550
		psi_memstall_enter(&pflags);
2551
		kcompactd_do_work(pgdat);
2552
		psi_memstall_leave(&pflags);
2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598
	}

	return 0;
}

/*
 * This kcompactd start function will be called by init and node-hot-add.
 * On node-hot-add, kcompactd will moved to proper cpus if cpus are hot-added.
 */
int kcompactd_run(int nid)
{
	pg_data_t *pgdat = NODE_DATA(nid);
	int ret = 0;

	if (pgdat->kcompactd)
		return 0;

	pgdat->kcompactd = kthread_run(kcompactd, pgdat, "kcompactd%d", nid);
	if (IS_ERR(pgdat->kcompactd)) {
		pr_err("Failed to start kcompactd on node %d\n", nid);
		ret = PTR_ERR(pgdat->kcompactd);
		pgdat->kcompactd = NULL;
	}
	return ret;
}

/*
 * Called by memory hotplug when all memory in a node is offlined. Caller must
 * hold mem_hotplug_begin/end().
 */
void kcompactd_stop(int nid)
{
	struct task_struct *kcompactd = NODE_DATA(nid)->kcompactd;

	if (kcompactd) {
		kthread_stop(kcompactd);
		NODE_DATA(nid)->kcompactd = NULL;
	}
}

/*
 * It's optimal to keep kcompactd on the same CPUs as their memory, but
 * not required for correctness. So if the last cpu in a node goes
 * away, we get changed to run anywhere: as the first one comes back,
 * restore their cpu bindings.
 */
2599
static int kcompactd_cpu_online(unsigned int cpu)
2600 2601 2602
{
	int nid;

2603 2604 2605
	for_each_node_state(nid, N_MEMORY) {
		pg_data_t *pgdat = NODE_DATA(nid);
		const struct cpumask *mask;
2606

2607
		mask = cpumask_of_node(pgdat->node_id);
2608

2609 2610 2611
		if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids)
			/* One of our CPUs online: restore mask */
			set_cpus_allowed_ptr(pgdat->kcompactd, mask);
2612
	}
2613
	return 0;
2614 2615 2616 2617 2618
}

static int __init kcompactd_init(void)
{
	int nid;
2619 2620 2621 2622 2623 2624 2625 2626 2627
	int ret;

	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
					"mm/compaction:online",
					kcompactd_cpu_online, NULL);
	if (ret < 0) {
		pr_err("kcompactd: failed to register hotplug callbacks.\n");
		return ret;
	}
2628 2629 2630 2631 2632 2633 2634

	for_each_node_state(nid, N_MEMORY)
		kcompactd_run(nid);
	return 0;
}
subsys_initcall(kcompactd_init)

2635
#endif /* CONFIG_COMPACTION */