swapfile.c 95.0 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-only
L
Linus Torvalds 已提交
2 3 4 5 6 7 8 9
/*
 *  linux/mm/swapfile.c
 *
 *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
 *  Swap reorganised 29.12.95, Stephen Tweedie
 */

#include <linux/mm.h>
10
#include <linux/sched/mm.h>
11
#include <linux/sched/task.h>
L
Linus Torvalds 已提交
12 13 14 15 16 17 18 19
#include <linux/hugetlb.h>
#include <linux/mman.h>
#include <linux/slab.h>
#include <linux/kernel_stat.h>
#include <linux/swap.h>
#include <linux/vmalloc.h>
#include <linux/pagemap.h>
#include <linux/namei.h>
20
#include <linux/shmem_fs.h>
L
Linus Torvalds 已提交
21
#include <linux/blkdev.h>
22
#include <linux/random.h>
L
Linus Torvalds 已提交
23 24 25 26
#include <linux/writeback.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/init.h>
27
#include <linux/ksm.h>
L
Linus Torvalds 已提交
28 29 30
#include <linux/rmap.h>
#include <linux/security.h>
#include <linux/backing-dev.h>
31
#include <linux/mutex.h>
32
#include <linux/capability.h>
L
Linus Torvalds 已提交
33
#include <linux/syscalls.h>
34
#include <linux/memcontrol.h>
35
#include <linux/poll.h>
36
#include <linux/oom.h>
37 38
#include <linux/frontswap.h>
#include <linux/swapfile.h>
39
#include <linux/export.h>
40
#include <linux/swap_slots.h>
41
#include <linux/sort.h>
L
Linus Torvalds 已提交
42 43 44 45

#include <asm/pgtable.h>
#include <asm/tlbflush.h>
#include <linux/swapops.h>
46
#include <linux/swap_cgroup.h>
L
Linus Torvalds 已提交
47

48 49 50
static bool swap_count_continued(struct swap_info_struct *, pgoff_t,
				 unsigned char);
static void free_swap_count_continuations(struct swap_info_struct *);
51
static sector_t map_swap_entry(swp_entry_t, struct block_device**);
52

53
DEFINE_SPINLOCK(swap_lock);
54
static unsigned int nr_swapfiles;
55
atomic_long_t nr_swap_pages;
C
Chris Wilson 已提交
56 57 58 59 60 61
/*
 * Some modules use swappable objects and may try to swap them out under
 * memory pressure (via the shrinker). Before doing so, they may wish to
 * check to see if any swap space is available.
 */
EXPORT_SYMBOL_GPL(nr_swap_pages);
62
/* protected with swap_lock. reading in vm_swap_full() doesn't need lock */
L
Linus Torvalds 已提交
63
long total_swap_pages;
64
static int least_priority = -1;
L
Linus Torvalds 已提交
65 66 67 68 69 70

static const char Bad_file[] = "Bad swap file entry ";
static const char Unused_file[] = "Unused swap file entry ";
static const char Bad_offset[] = "Bad swap offset entry ";
static const char Unused_offset[] = "Unused swap offset entry ";

71 72 73 74
/*
 * all active swap_info_structs
 * protected with swap_lock, and ordered by priority.
 */
75 76 77 78 79 80 81 82 83 84 85 86 87 88
PLIST_HEAD(swap_active_head);

/*
 * all available (active, not full) swap_info_structs
 * protected with swap_avail_lock, ordered by priority.
 * This is used by get_swap_page() instead of swap_active_head
 * because swap_active_head includes all swap_info_structs,
 * but get_swap_page() doesn't need to look at full ones.
 * This uses its own lock instead of swap_lock because when a
 * swap_info_struct changes between not-full/full, it needs to
 * add/remove itself to/from this list, but the swap_info_struct->lock
 * is held and the locking order requires swap_lock to be taken
 * before any swap_info_struct->lock.
 */
89
static struct plist_head *swap_avail_heads;
90
static DEFINE_SPINLOCK(swap_avail_lock);
L
Linus Torvalds 已提交
91

92
struct swap_info_struct *swap_info[MAX_SWAPFILES];
L
Linus Torvalds 已提交
93

94
static DEFINE_MUTEX(swapon_mutex);
L
Linus Torvalds 已提交
95

96 97 98 99
static DECLARE_WAIT_QUEUE_HEAD(proc_poll_wait);
/* Activity counter to indicate that a swapon or swapoff has occurred */
static atomic_t proc_poll_event = ATOMIC_INIT(0);

100 101
atomic_t nr_rotate_swap = ATOMIC_INIT(0);

102 103 104 105 106 107 108 109 110
static struct swap_info_struct *swap_type_to_swap_info(int type)
{
	if (type >= READ_ONCE(nr_swapfiles))
		return NULL;

	smp_rmb();	/* Pairs with smp_wmb in alloc_swap_info. */
	return READ_ONCE(swap_info[type]);
}

111
static inline unsigned char swap_count(unsigned char ent)
112
{
113
	return ent & ~SWAP_HAS_CACHE;	/* may include COUNT_CONTINUED flag */
114 115
}

116 117 118 119 120 121 122 123 124 125
/* Reclaim the swap entry anyway if possible */
#define TTRS_ANYWAY		0x1
/*
 * Reclaim the swap entry if there are no more mappings of the
 * corresponding page
 */
#define TTRS_UNMAPPED		0x2
/* Reclaim the swap entry if swap is getting full*/
#define TTRS_FULL		0x4

126
/* returns 1 if swap entry is freed */
127 128
static int __try_to_reclaim_swap(struct swap_info_struct *si,
				 unsigned long offset, unsigned long flags)
129
{
130
	swp_entry_t entry = swp_entry(si->type, offset);
131 132 133
	struct page *page;
	int ret = 0;

134
	page = find_get_page(swap_address_space(entry), offset);
135 136 137
	if (!page)
		return 0;
	/*
138 139 140
	 * When this function is called from scan_swap_map_slots() and it's
	 * called by vmscan.c at reclaiming pages. So, we hold a lock on a page,
	 * here. We have to use trylock for avoiding deadlock. This is a special
141 142 143 144
	 * case and you should use try_to_free_swap() with explicit lock_page()
	 * in usual operations.
	 */
	if (trylock_page(page)) {
145 146 147 148
		if ((flags & TTRS_ANYWAY) ||
		    ((flags & TTRS_UNMAPPED) && !page_mapped(page)) ||
		    ((flags & TTRS_FULL) && mem_cgroup_swap_full(page)))
			ret = try_to_free_swap(page);
149 150
		unlock_page(page);
	}
151
	put_page(page);
152 153
	return ret;
}
154

155 156 157 158 159 160 161 162 163 164 165 166
static inline struct swap_extent *first_se(struct swap_info_struct *sis)
{
	struct rb_node *rb = rb_first(&sis->swap_extent_root);
	return rb_entry(rb, struct swap_extent, rb_node);
}

static inline struct swap_extent *next_se(struct swap_extent *se)
{
	struct rb_node *rb = rb_next(&se->rb_node);
	return rb ? rb_entry(rb, struct swap_extent, rb_node) : NULL;
}

167 168 169 170 171 172 173
/*
 * swapon tell device that all the old swap contents can be discarded,
 * to allow the swap device to optimize its wear-levelling.
 */
static int discard_swap(struct swap_info_struct *si)
{
	struct swap_extent *se;
174 175
	sector_t start_block;
	sector_t nr_blocks;
176 177
	int err = 0;

178
	/* Do not discard the swap header page! */
179
	se = first_se(si);
180 181 182 183
	start_block = (se->start_block + 1) << (PAGE_SHIFT - 9);
	nr_blocks = ((sector_t)se->nr_pages - 1) << (PAGE_SHIFT - 9);
	if (nr_blocks) {
		err = blkdev_issue_discard(si->bdev, start_block,
184
				nr_blocks, GFP_KERNEL, 0);
185 186 187 188
		if (err)
			return err;
		cond_resched();
	}
189

190
	for (se = next_se(se); se; se = next_se(se)) {
191 192
		start_block = se->start_block << (PAGE_SHIFT - 9);
		nr_blocks = (sector_t)se->nr_pages << (PAGE_SHIFT - 9);
193 194

		err = blkdev_issue_discard(si->bdev, start_block,
195
				nr_blocks, GFP_KERNEL, 0);
196 197 198 199 200 201 202 203
		if (err)
			break;

		cond_resched();
	}
	return err;		/* That will often be -EOPNOTSUPP */
}

204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223
static struct swap_extent *
offset_to_swap_extent(struct swap_info_struct *sis, unsigned long offset)
{
	struct swap_extent *se;
	struct rb_node *rb;

	rb = sis->swap_extent_root.rb_node;
	while (rb) {
		se = rb_entry(rb, struct swap_extent, rb_node);
		if (offset < se->start_page)
			rb = rb->rb_left;
		else if (offset >= se->start_page + se->nr_pages)
			rb = rb->rb_right;
		else
			return se;
	}
	/* It *must* be present */
	BUG();
}

224 225 226 227 228 229 230
/*
 * swap allocation tell device that a cluster of swap can now be discarded,
 * to allow the swap device to optimize its wear-levelling.
 */
static void discard_swap_cluster(struct swap_info_struct *si,
				 pgoff_t start_page, pgoff_t nr_pages)
{
231
	struct swap_extent *se = offset_to_swap_extent(si, start_page);
232 233

	while (nr_pages) {
234 235 236 237 238 239 240 241 242 243 244 245 246 247
		pgoff_t offset = start_page - se->start_page;
		sector_t start_block = se->start_block + offset;
		sector_t nr_blocks = se->nr_pages - offset;

		if (nr_blocks > nr_pages)
			nr_blocks = nr_pages;
		start_page += nr_blocks;
		nr_pages -= nr_blocks;

		start_block <<= PAGE_SHIFT - 9;
		nr_blocks <<= PAGE_SHIFT - 9;
		if (blkdev_issue_discard(si->bdev, start_block,
					nr_blocks, GFP_NOIO, 0))
			break;
248

249
		se = next_se(se);
250 251 252
	}
}

253 254
#ifdef CONFIG_THP_SWAP
#define SWAPFILE_CLUSTER	HPAGE_PMD_NR
255 256

#define swap_entry_size(size)	(size)
257
#else
258
#define SWAPFILE_CLUSTER	256
259 260 261 262 263 264

/*
 * Define swap_entry_size() as constant to let compiler to optimize
 * out some code if !CONFIG_THP_SWAP
 */
#define swap_entry_size(size)	1
265
#endif
266 267
#define LATENCY_LIMIT		256

268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325
static inline void cluster_set_flag(struct swap_cluster_info *info,
	unsigned int flag)
{
	info->flags = flag;
}

static inline unsigned int cluster_count(struct swap_cluster_info *info)
{
	return info->data;
}

static inline void cluster_set_count(struct swap_cluster_info *info,
				     unsigned int c)
{
	info->data = c;
}

static inline void cluster_set_count_flag(struct swap_cluster_info *info,
					 unsigned int c, unsigned int f)
{
	info->flags = f;
	info->data = c;
}

static inline unsigned int cluster_next(struct swap_cluster_info *info)
{
	return info->data;
}

static inline void cluster_set_next(struct swap_cluster_info *info,
				    unsigned int n)
{
	info->data = n;
}

static inline void cluster_set_next_flag(struct swap_cluster_info *info,
					 unsigned int n, unsigned int f)
{
	info->flags = f;
	info->data = n;
}

static inline bool cluster_is_free(struct swap_cluster_info *info)
{
	return info->flags & CLUSTER_FLAG_FREE;
}

static inline bool cluster_is_null(struct swap_cluster_info *info)
{
	return info->flags & CLUSTER_FLAG_NEXT_NULL;
}

static inline void cluster_set_null(struct swap_cluster_info *info)
{
	info->flags = CLUSTER_FLAG_NEXT_NULL;
	info->data = 0;
}

326 327
static inline bool cluster_is_huge(struct swap_cluster_info *info)
{
328 329 330
	if (IS_ENABLED(CONFIG_THP_SWAP))
		return info->flags & CLUSTER_FLAG_HUGE;
	return false;
331 332 333 334 335 336 337
}

static inline void cluster_clear_huge(struct swap_cluster_info *info)
{
	info->flags &= ~CLUSTER_FLAG_HUGE;
}

H
Huang, Ying 已提交
338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356
static inline struct swap_cluster_info *lock_cluster(struct swap_info_struct *si,
						     unsigned long offset)
{
	struct swap_cluster_info *ci;

	ci = si->cluster_info;
	if (ci) {
		ci += offset / SWAPFILE_CLUSTER;
		spin_lock(&ci->lock);
	}
	return ci;
}

static inline void unlock_cluster(struct swap_cluster_info *ci)
{
	if (ci)
		spin_unlock(&ci->lock);
}

357 358 359 360
/*
 * Determine the locking method in use for this device.  Return
 * swap_cluster_info if SSD-style cluster-based locking is in place.
 */
H
Huang, Ying 已提交
361
static inline struct swap_cluster_info *lock_cluster_or_swap_info(
362
		struct swap_info_struct *si, unsigned long offset)
H
Huang, Ying 已提交
363 364 365
{
	struct swap_cluster_info *ci;

366
	/* Try to use fine-grained SSD-style locking if available: */
H
Huang, Ying 已提交
367
	ci = lock_cluster(si, offset);
368
	/* Otherwise, fall back to traditional, coarse locking: */
H
Huang, Ying 已提交
369 370 371 372 373 374 375 376 377 378 379 380 381 382 383
	if (!ci)
		spin_lock(&si->lock);

	return ci;
}

static inline void unlock_cluster_or_swap_info(struct swap_info_struct *si,
					       struct swap_cluster_info *ci)
{
	if (ci)
		unlock_cluster(ci);
	else
		spin_unlock(&si->lock);
}

384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407
static inline bool cluster_list_empty(struct swap_cluster_list *list)
{
	return cluster_is_null(&list->head);
}

static inline unsigned int cluster_list_first(struct swap_cluster_list *list)
{
	return cluster_next(&list->head);
}

static void cluster_list_init(struct swap_cluster_list *list)
{
	cluster_set_null(&list->head);
	cluster_set_null(&list->tail);
}

static void cluster_list_add_tail(struct swap_cluster_list *list,
				  struct swap_cluster_info *ci,
				  unsigned int idx)
{
	if (cluster_list_empty(list)) {
		cluster_set_next_flag(&list->head, idx, 0);
		cluster_set_next_flag(&list->tail, idx, 0);
	} else {
H
Huang, Ying 已提交
408
		struct swap_cluster_info *ci_tail;
409 410
		unsigned int tail = cluster_next(&list->tail);

H
Huang, Ying 已提交
411 412 413 414 415 416 417
		/*
		 * Nested cluster lock, but both cluster locks are
		 * only acquired when we held swap_info_struct->lock
		 */
		ci_tail = ci + tail;
		spin_lock_nested(&ci_tail->lock, SINGLE_DEPTH_NESTING);
		cluster_set_next(ci_tail, idx);
418
		spin_unlock(&ci_tail->lock);
419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438
		cluster_set_next_flag(&list->tail, idx, 0);
	}
}

static unsigned int cluster_list_del_first(struct swap_cluster_list *list,
					   struct swap_cluster_info *ci)
{
	unsigned int idx;

	idx = cluster_next(&list->head);
	if (cluster_next(&list->tail) == idx) {
		cluster_set_null(&list->head);
		cluster_set_null(&list->tail);
	} else
		cluster_set_next_flag(&list->head,
				      cluster_next(&ci[idx]), 0);

	return idx;
}

439 440 441 442 443 444 445 446 447 448 449 450 451
/* Add a cluster to discard list and schedule it to do discard */
static void swap_cluster_schedule_discard(struct swap_info_struct *si,
		unsigned int idx)
{
	/*
	 * If scan_swap_map() can't find a free cluster, it will check
	 * si->swap_map directly. To make sure the discarding cluster isn't
	 * taken by scan_swap_map(), mark the swap entries bad (occupied). It
	 * will be cleared after discard
	 */
	memset(si->swap_map + idx * SWAPFILE_CLUSTER,
			SWAP_MAP_BAD, SWAPFILE_CLUSTER);

452
	cluster_list_add_tail(&si->discard_clusters, si->cluster_info, idx);
453 454 455 456

	schedule_work(&si->discard_work);
}

457 458 459 460 461 462 463 464
static void __free_cluster(struct swap_info_struct *si, unsigned long idx)
{
	struct swap_cluster_info *ci = si->cluster_info;

	cluster_set_flag(ci + idx, CLUSTER_FLAG_FREE);
	cluster_list_add_tail(&si->free_clusters, ci, idx);
}

465 466 467 468 469 470
/*
 * Doing discard actually. After a cluster discard is finished, the cluster
 * will be added to free cluster list. caller should hold si->lock.
*/
static void swap_do_scheduled_discard(struct swap_info_struct *si)
{
H
Huang, Ying 已提交
471
	struct swap_cluster_info *info, *ci;
472 473 474 475
	unsigned int idx;

	info = si->cluster_info;

476 477
	while (!cluster_list_empty(&si->discard_clusters)) {
		idx = cluster_list_del_first(&si->discard_clusters, info);
478 479 480 481 482 483
		spin_unlock(&si->lock);

		discard_swap_cluster(si, idx * SWAPFILE_CLUSTER,
				SWAPFILE_CLUSTER);

		spin_lock(&si->lock);
H
Huang, Ying 已提交
484
		ci = lock_cluster(si, idx * SWAPFILE_CLUSTER);
485
		__free_cluster(si, idx);
486 487
		memset(si->swap_map + idx * SWAPFILE_CLUSTER,
				0, SWAPFILE_CLUSTER);
H
Huang, Ying 已提交
488
		unlock_cluster(ci);
489 490 491 492 493 494 495 496 497 498 499 500 501 502
	}
}

static void swap_discard_work(struct work_struct *work)
{
	struct swap_info_struct *si;

	si = container_of(work, struct swap_info_struct, discard_work);

	spin_lock(&si->lock);
	swap_do_scheduled_discard(si);
	spin_unlock(&si->lock);
}

503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530
static void alloc_cluster(struct swap_info_struct *si, unsigned long idx)
{
	struct swap_cluster_info *ci = si->cluster_info;

	VM_BUG_ON(cluster_list_first(&si->free_clusters) != idx);
	cluster_list_del_first(&si->free_clusters, ci);
	cluster_set_count_flag(ci + idx, 0, 0);
}

static void free_cluster(struct swap_info_struct *si, unsigned long idx)
{
	struct swap_cluster_info *ci = si->cluster_info + idx;

	VM_BUG_ON(cluster_count(ci) != 0);
	/*
	 * If the swap is discardable, prepare discard the cluster
	 * instead of free it immediately. The cluster will be freed
	 * after discard.
	 */
	if ((si->flags & (SWP_WRITEOK | SWP_PAGE_DISCARD)) ==
	    (SWP_WRITEOK | SWP_PAGE_DISCARD)) {
		swap_cluster_schedule_discard(si, idx);
		return;
	}

	__free_cluster(si, idx);
}

531 532 533 534 535 536 537 538 539 540 541
/*
 * The cluster corresponding to page_nr will be used. The cluster will be
 * removed from free cluster list and its usage counter will be increased.
 */
static void inc_cluster_info_page(struct swap_info_struct *p,
	struct swap_cluster_info *cluster_info, unsigned long page_nr)
{
	unsigned long idx = page_nr / SWAPFILE_CLUSTER;

	if (!cluster_info)
		return;
542 543
	if (cluster_is_free(&cluster_info[idx]))
		alloc_cluster(p, idx);
544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566

	VM_BUG_ON(cluster_count(&cluster_info[idx]) >= SWAPFILE_CLUSTER);
	cluster_set_count(&cluster_info[idx],
		cluster_count(&cluster_info[idx]) + 1);
}

/*
 * The cluster corresponding to page_nr decreases one usage. If the usage
 * counter becomes 0, which means no page in the cluster is in using, we can
 * optionally discard the cluster and add it to free cluster list.
 */
static void dec_cluster_info_page(struct swap_info_struct *p,
	struct swap_cluster_info *cluster_info, unsigned long page_nr)
{
	unsigned long idx = page_nr / SWAPFILE_CLUSTER;

	if (!cluster_info)
		return;

	VM_BUG_ON(cluster_count(&cluster_info[idx]) == 0);
	cluster_set_count(&cluster_info[idx],
		cluster_count(&cluster_info[idx]) - 1);

567 568
	if (cluster_count(&cluster_info[idx]) == 0)
		free_cluster(p, idx);
569 570 571 572 573 574
}

/*
 * It's possible scan_swap_map() uses a free cluster in the middle of free
 * cluster list. Avoiding such abuse to avoid list corruption.
 */
575 576
static bool
scan_swap_map_ssd_cluster_conflict(struct swap_info_struct *si,
577 578
	unsigned long offset)
{
579 580 581
	struct percpu_cluster *percpu_cluster;
	bool conflict;

582
	offset /= SWAPFILE_CLUSTER;
583 584
	conflict = !cluster_list_empty(&si->free_clusters) &&
		offset != cluster_list_first(&si->free_clusters) &&
585
		cluster_is_free(&si->cluster_info[offset]);
586 587 588 589 590 591 592 593 594 595 596 597 598

	if (!conflict)
		return false;

	percpu_cluster = this_cpu_ptr(si->percpu_cluster);
	cluster_set_null(&percpu_cluster->index);
	return true;
}

/*
 * Try to get a swap entry from current cpu's swap entry pool (a cluster). This
 * might involve allocating a new cluster for current CPU too.
 */
599
static bool scan_swap_map_try_ssd_cluster(struct swap_info_struct *si,
600 601 602
	unsigned long *offset, unsigned long *scan_base)
{
	struct percpu_cluster *cluster;
H
Huang, Ying 已提交
603
	struct swap_cluster_info *ci;
604
	bool found_free;
H
Huang, Ying 已提交
605
	unsigned long tmp, max;
606 607 608 609

new_cluster:
	cluster = this_cpu_ptr(si->percpu_cluster);
	if (cluster_is_null(&cluster->index)) {
610 611
		if (!cluster_list_empty(&si->free_clusters)) {
			cluster->index = si->free_clusters.head;
612 613
			cluster->next = cluster_next(&cluster->index) *
					SWAPFILE_CLUSTER;
614
		} else if (!cluster_list_empty(&si->discard_clusters)) {
615 616 617 618 619 620 621 622
			/*
			 * we don't have free cluster but have some clusters in
			 * discarding, do discard now and reclaim them
			 */
			swap_do_scheduled_discard(si);
			*scan_base = *offset = si->cluster_next;
			goto new_cluster;
		} else
623
			return false;
624 625 626 627 628 629 630 631 632
	}

	found_free = false;

	/*
	 * Other CPUs can use our cluster if they can't find a free cluster,
	 * check if there is still free entry in the cluster
	 */
	tmp = cluster->next;
H
Huang, Ying 已提交
633 634 635 636 637 638 639 640
	max = min_t(unsigned long, si->max,
		    (cluster_next(&cluster->index) + 1) * SWAPFILE_CLUSTER);
	if (tmp >= max) {
		cluster_set_null(&cluster->index);
		goto new_cluster;
	}
	ci = lock_cluster(si, tmp);
	while (tmp < max) {
641 642 643 644 645 646
		if (!si->swap_map[tmp]) {
			found_free = true;
			break;
		}
		tmp++;
	}
H
Huang, Ying 已提交
647
	unlock_cluster(ci);
648 649 650 651 652 653 654
	if (!found_free) {
		cluster_set_null(&cluster->index);
		goto new_cluster;
	}
	cluster->next = tmp + 1;
	*offset = tmp;
	*scan_base = tmp;
655
	return found_free;
656 657
}

658 659 660 661 662 663 664 665 666 667 668 669 670 671 672
static void __del_from_avail_list(struct swap_info_struct *p)
{
	int nid;

	for_each_node(nid)
		plist_del(&p->avail_lists[nid], &swap_avail_heads[nid]);
}

static void del_from_avail_list(struct swap_info_struct *p)
{
	spin_lock(&swap_avail_lock);
	__del_from_avail_list(p);
	spin_unlock(&swap_avail_lock);
}

673 674 675 676 677 678 679 680 681 682 683 684 685
static void swap_range_alloc(struct swap_info_struct *si, unsigned long offset,
			     unsigned int nr_entries)
{
	unsigned int end = offset + nr_entries - 1;

	if (offset == si->lowest_bit)
		si->lowest_bit += nr_entries;
	if (end == si->highest_bit)
		si->highest_bit -= nr_entries;
	si->inuse_pages += nr_entries;
	if (si->inuse_pages == si->pages) {
		si->lowest_bit = si->max;
		si->highest_bit = 0;
686
		del_from_avail_list(si);
687 688 689
	}
}

690 691 692 693 694 695 696 697 698 699 700 701
static void add_to_avail_list(struct swap_info_struct *p)
{
	int nid;

	spin_lock(&swap_avail_lock);
	for_each_node(nid) {
		WARN_ON(!plist_node_empty(&p->avail_lists[nid]));
		plist_add(&p->avail_lists[nid], &swap_avail_heads[nid]);
	}
	spin_unlock(&swap_avail_lock);
}

702 703 704 705 706 707 708 709 710 711 712 713
static void swap_range_free(struct swap_info_struct *si, unsigned long offset,
			    unsigned int nr_entries)
{
	unsigned long end = offset + nr_entries - 1;
	void (*swap_slot_free_notify)(struct block_device *, unsigned long);

	if (offset < si->lowest_bit)
		si->lowest_bit = offset;
	if (end > si->highest_bit) {
		bool was_full = !si->highest_bit;

		si->highest_bit = end;
714 715
		if (was_full && (si->flags & SWP_WRITEOK))
			add_to_avail_list(si);
716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731
	}
	atomic_long_add(nr_entries, &nr_swap_pages);
	si->inuse_pages -= nr_entries;
	if (si->flags & SWP_BLKDEV)
		swap_slot_free_notify =
			si->bdev->bd_disk->fops->swap_slot_free_notify;
	else
		swap_slot_free_notify = NULL;
	while (offset <= end) {
		frontswap_invalidate_page(si->type, offset);
		if (swap_slot_free_notify)
			swap_slot_free_notify(si->bdev, offset);
		offset++;
	}
}

732 733 734
static int scan_swap_map_slots(struct swap_info_struct *si,
			       unsigned char usage, int nr,
			       swp_entry_t slots[])
L
Linus Torvalds 已提交
735
{
H
Huang, Ying 已提交
736
	struct swap_cluster_info *ci;
737
	unsigned long offset;
738
	unsigned long scan_base;
739
	unsigned long last_in_cluster = 0;
740
	int latency_ration = LATENCY_LIMIT;
741 742 743 744
	int n_ret = 0;

	if (nr > SWAP_BATCH)
		nr = SWAP_BATCH;
745

746
	/*
747 748 749 750 751 752 753
	 * We try to cluster swap pages by allocating them sequentially
	 * in swap.  Once we've allocated SWAPFILE_CLUSTER pages this
	 * way, however, we resort to first-free allocation, starting
	 * a new cluster.  This prevents us from scattering swap pages
	 * all over the entire swap partition, so that we reduce
	 * overall disk seek times between swap pages.  -- sct
	 * But we do now try to find an empty cluster.  -Andrea
754
	 * And we let swap pages go all over an SSD partition.  Hugh
755 756
	 */

757
	si->flags += SWP_SCANNING;
758
	scan_base = offset = si->cluster_next;
759

760 761
	/* SSD algorithm */
	if (si->cluster_info) {
762 763 764 765
		if (scan_swap_map_try_ssd_cluster(si, &offset, &scan_base))
			goto checks;
		else
			goto scan;
766 767
	}

768 769 770 771 772
	if (unlikely(!si->cluster_nr--)) {
		if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER) {
			si->cluster_nr = SWAPFILE_CLUSTER - 1;
			goto checks;
		}
773

774
		spin_unlock(&si->lock);
775

776 777 778
		/*
		 * If seek is expensive, start searching for new cluster from
		 * start of partition, to minimize the span of allocated swap.
779 780
		 * If seek is cheap, that is the SWP_SOLIDSTATE si->cluster_info
		 * case, just handled by scan_swap_map_try_ssd_cluster() above.
781
		 */
782
		scan_base = offset = si->lowest_bit;
783 784 785 786
		last_in_cluster = offset + SWAPFILE_CLUSTER - 1;

		/* Locate the first empty (unaligned) cluster */
		for (; last_in_cluster <= si->highest_bit; offset++) {
L
Linus Torvalds 已提交
787
			if (si->swap_map[offset])
788 789
				last_in_cluster = offset + SWAPFILE_CLUSTER;
			else if (offset == last_in_cluster) {
790
				spin_lock(&si->lock);
791 792 793
				offset -= SWAPFILE_CLUSTER - 1;
				si->cluster_next = offset;
				si->cluster_nr = SWAPFILE_CLUSTER - 1;
794 795 796 797 798 799 800 801 802
				goto checks;
			}
			if (unlikely(--latency_ration < 0)) {
				cond_resched();
				latency_ration = LATENCY_LIMIT;
			}
		}

		offset = scan_base;
803
		spin_lock(&si->lock);
804
		si->cluster_nr = SWAPFILE_CLUSTER - 1;
L
Linus Torvalds 已提交
805
	}
806

807
checks:
808
	if (si->cluster_info) {
809 810 811 812 813 814 815 816
		while (scan_swap_map_ssd_cluster_conflict(si, offset)) {
		/* take a break if we already got some slots */
			if (n_ret)
				goto done;
			if (!scan_swap_map_try_ssd_cluster(si, &offset,
							&scan_base))
				goto scan;
		}
817
	}
818
	if (!(si->flags & SWP_WRITEOK))
819
		goto no_page;
820 821
	if (!si->highest_bit)
		goto no_page;
822
	if (offset > si->highest_bit)
823
		scan_base = offset = si->lowest_bit;
824

H
Huang, Ying 已提交
825
	ci = lock_cluster(si, offset);
826 827
	/* reuse swap entry of cache-only swap if not busy. */
	if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) {
828
		int swap_was_freed;
H
Huang, Ying 已提交
829
		unlock_cluster(ci);
830
		spin_unlock(&si->lock);
831
		swap_was_freed = __try_to_reclaim_swap(si, offset, TTRS_ANYWAY);
832
		spin_lock(&si->lock);
833 834 835 836 837 838
		/* entry was freed successfully, try to use this again */
		if (swap_was_freed)
			goto checks;
		goto scan; /* check next one */
	}

H
Huang, Ying 已提交
839 840
	if (si->swap_map[offset]) {
		unlock_cluster(ci);
841 842 843 844
		if (!n_ret)
			goto scan;
		else
			goto done;
H
Huang, Ying 已提交
845
	}
846 847 848
	si->swap_map[offset] = usage;
	inc_cluster_info_page(si, si->cluster_info, offset);
	unlock_cluster(ci);
849

850
	swap_range_alloc(si, offset, 1);
851
	si->cluster_next = offset + 1;
852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884
	slots[n_ret++] = swp_entry(si->type, offset);

	/* got enough slots or reach max slots? */
	if ((n_ret == nr) || (offset >= si->highest_bit))
		goto done;

	/* search for next available slot */

	/* time to take a break? */
	if (unlikely(--latency_ration < 0)) {
		if (n_ret)
			goto done;
		spin_unlock(&si->lock);
		cond_resched();
		spin_lock(&si->lock);
		latency_ration = LATENCY_LIMIT;
	}

	/* try to get more slots in cluster */
	if (si->cluster_info) {
		if (scan_swap_map_try_ssd_cluster(si, &offset, &scan_base))
			goto checks;
		else
			goto done;
	}
	/* non-ssd case */
	++offset;

	/* non-ssd case, still more slots in cluster? */
	if (si->cluster_nr && !si->swap_map[offset]) {
		--si->cluster_nr;
		goto checks;
	}
885

886 887 888
done:
	si->flags -= SWP_SCANNING;
	return n_ret;
889

890
scan:
891
	spin_unlock(&si->lock);
892
	while (++offset <= si->highest_bit) {
893
		if (!si->swap_map[offset]) {
894
			spin_lock(&si->lock);
895 896
			goto checks;
		}
897
		if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) {
898
			spin_lock(&si->lock);
899 900
			goto checks;
		}
901 902 903 904
		if (unlikely(--latency_ration < 0)) {
			cond_resched();
			latency_ration = LATENCY_LIMIT;
		}
905
	}
906
	offset = si->lowest_bit;
907
	while (offset < scan_base) {
908
		if (!si->swap_map[offset]) {
909
			spin_lock(&si->lock);
910 911
			goto checks;
		}
912
		if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) {
913
			spin_lock(&si->lock);
914 915
			goto checks;
		}
916 917 918 919
		if (unlikely(--latency_ration < 0)) {
			cond_resched();
			latency_ration = LATENCY_LIMIT;
		}
920
		offset++;
921
	}
922
	spin_lock(&si->lock);
923 924

no_page:
925
	si->flags -= SWP_SCANNING;
926
	return n_ret;
L
Linus Torvalds 已提交
927 928
}

929 930 931 932 933 934 935
static int swap_alloc_cluster(struct swap_info_struct *si, swp_entry_t *slot)
{
	unsigned long idx;
	struct swap_cluster_info *ci;
	unsigned long offset, i;
	unsigned char *map;

936 937 938 939 940 941 942 943 944
	/*
	 * Should not even be attempting cluster allocations when huge
	 * page swap is disabled.  Warn and fail the allocation.
	 */
	if (!IS_ENABLED(CONFIG_THP_SWAP)) {
		VM_WARN_ON_ONCE(1);
		return 0;
	}

945 946 947 948 949 950 951
	if (cluster_list_empty(&si->free_clusters))
		return 0;

	idx = cluster_list_first(&si->free_clusters);
	offset = idx * SWAPFILE_CLUSTER;
	ci = lock_cluster(si, offset);
	alloc_cluster(si, idx);
952
	cluster_set_count_flag(ci, SWAPFILE_CLUSTER, CLUSTER_FLAG_HUGE);
953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969

	map = si->swap_map + offset;
	for (i = 0; i < SWAPFILE_CLUSTER; i++)
		map[i] = SWAP_HAS_CACHE;
	unlock_cluster(ci);
	swap_range_alloc(si, offset, SWAPFILE_CLUSTER);
	*slot = swp_entry(si->type, offset);

	return 1;
}

static void swap_free_cluster(struct swap_info_struct *si, unsigned long idx)
{
	unsigned long offset = idx * SWAPFILE_CLUSTER;
	struct swap_cluster_info *ci;

	ci = lock_cluster(si, offset);
970
	memset(si->swap_map + offset, 0, SWAPFILE_CLUSTER);
971 972 973 974 975 976
	cluster_set_count_flag(ci, 0, 0);
	free_cluster(si, idx);
	unlock_cluster(ci);
	swap_range_free(si, offset, SWAPFILE_CLUSTER);
}

977 978 979 980 981 982 983 984 985 986 987 988 989 990 991
static unsigned long scan_swap_map(struct swap_info_struct *si,
				   unsigned char usage)
{
	swp_entry_t entry;
	int n_ret;

	n_ret = scan_swap_map_slots(si, usage, 1, &entry);

	if (n_ret)
		return swp_offset(entry);
	else
		return 0;

}

992
int get_swap_pages(int n_goal, swp_entry_t swp_entries[], int entry_size)
L
Linus Torvalds 已提交
993
{
994
	unsigned long size = swap_entry_size(entry_size);
995
	struct swap_info_struct *si, *next;
996 997
	long avail_pgs;
	int n_ret = 0;
998
	int node;
L
Linus Torvalds 已提交
999

1000
	/* Only single cluster request supported */
1001
	WARN_ON_ONCE(n_goal > 1 && size == SWAPFILE_CLUSTER);
1002

1003
	avail_pgs = atomic_long_read(&nr_swap_pages) / size;
1004
	if (avail_pgs <= 0)
1005
		goto noswap;
1006 1007 1008 1009 1010 1011 1012

	if (n_goal > SWAP_BATCH)
		n_goal = SWAP_BATCH;

	if (n_goal > avail_pgs)
		n_goal = avail_pgs;

1013
	atomic_long_sub(n_goal * size, &nr_swap_pages);
1014

1015 1016 1017
	spin_lock(&swap_avail_lock);

start_over:
1018 1019
	node = numa_node_id();
	plist_for_each_entry_safe(si, next, &swap_avail_heads[node], avail_lists[node]) {
1020
		/* requeue si to after same-priority siblings */
1021
		plist_requeue(&si->avail_lists[node], &swap_avail_heads[node]);
1022
		spin_unlock(&swap_avail_lock);
1023
		spin_lock(&si->lock);
1024
		if (!si->highest_bit || !(si->flags & SWP_WRITEOK)) {
1025
			spin_lock(&swap_avail_lock);
1026
			if (plist_node_empty(&si->avail_lists[node])) {
1027 1028 1029 1030 1031 1032 1033 1034 1035
				spin_unlock(&si->lock);
				goto nextsi;
			}
			WARN(!si->highest_bit,
			     "swap_info %d in list but !highest_bit\n",
			     si->type);
			WARN(!(si->flags & SWP_WRITEOK),
			     "swap_info %d in list but !SWP_WRITEOK\n",
			     si->type);
1036
			__del_from_avail_list(si);
1037
			spin_unlock(&si->lock);
1038
			goto nextsi;
1039
		}
1040
		if (size == SWAPFILE_CLUSTER) {
1041
			if (!(si->flags & SWP_FS))
1042 1043
				n_ret = swap_alloc_cluster(si, swp_entries);
		} else
1044 1045
			n_ret = scan_swap_map_slots(si, SWAP_HAS_CACHE,
						    n_goal, swp_entries);
1046
		spin_unlock(&si->lock);
1047
		if (n_ret || size == SWAPFILE_CLUSTER)
1048
			goto check_out;
1049
		pr_debug("scan_swap_map of si %d failed to find offset\n",
1050 1051
			si->type);

1052 1053
		spin_lock(&swap_avail_lock);
nextsi:
1054 1055 1056 1057
		/*
		 * if we got here, it's likely that si was almost full before,
		 * and since scan_swap_map() can drop the si->lock, multiple
		 * callers probably all tried to get a page from the same si
1058 1059 1060 1061
		 * and it filled up before we could get one; or, the si filled
		 * up between us dropping swap_avail_lock and taking si->lock.
		 * Since we dropped the swap_avail_lock, the swap_avail_head
		 * list may have been modified; so if next is still in the
1062 1063
		 * swap_avail_head list then try it, otherwise start over
		 * if we have not gotten any slots.
1064
		 */
1065
		if (plist_node_empty(&next->avail_lists[node]))
1066
			goto start_over;
L
Linus Torvalds 已提交
1067
	}
1068

1069 1070
	spin_unlock(&swap_avail_lock);

1071 1072
check_out:
	if (n_ret < n_goal)
1073
		atomic_long_add((long)(n_goal - n_ret) * size,
1074
				&nr_swap_pages);
1075
noswap:
1076 1077 1078
	return n_ret;
}

1079
/* The only caller of this function is now suspend routine */
1080 1081
swp_entry_t get_swap_page_of_type(int type)
{
1082
	struct swap_info_struct *si = swap_type_to_swap_info(type);
1083 1084
	pgoff_t offset;

1085 1086 1087
	if (!si)
		goto fail;

1088
	spin_lock(&si->lock);
1089
	if (si->flags & SWP_WRITEOK) {
1090
		atomic_long_dec(&nr_swap_pages);
1091 1092 1093
		/* This is called for allocating swap entry, not cache */
		offset = scan_swap_map(si, 1);
		if (offset) {
1094
			spin_unlock(&si->lock);
1095 1096
			return swp_entry(type, offset);
		}
1097
		atomic_long_inc(&nr_swap_pages);
1098
	}
1099
	spin_unlock(&si->lock);
1100
fail:
1101 1102 1103
	return (swp_entry_t) {0};
}

1104
static struct swap_info_struct *__swap_info_get(swp_entry_t entry)
L
Linus Torvalds 已提交
1105
{
1106
	struct swap_info_struct *p;
1107
	unsigned long offset;
L
Linus Torvalds 已提交
1108 1109 1110

	if (!entry.val)
		goto out;
1111
	p = swp_swap_info(entry);
1112
	if (!p)
L
Linus Torvalds 已提交
1113 1114 1115 1116 1117 1118 1119 1120 1121
		goto bad_nofile;
	if (!(p->flags & SWP_USED))
		goto bad_device;
	offset = swp_offset(entry);
	if (offset >= p->max)
		goto bad_offset;
	return p;

bad_offset:
1122
	pr_err("swap_info_get: %s%08lx\n", Bad_offset, entry.val);
L
Linus Torvalds 已提交
1123 1124
	goto out;
bad_device:
1125
	pr_err("swap_info_get: %s%08lx\n", Unused_file, entry.val);
L
Linus Torvalds 已提交
1126 1127
	goto out;
bad_nofile:
1128
	pr_err("swap_info_get: %s%08lx\n", Bad_file, entry.val);
L
Linus Torvalds 已提交
1129 1130
out:
	return NULL;
1131
}
L
Linus Torvalds 已提交
1132

1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150
static struct swap_info_struct *_swap_info_get(swp_entry_t entry)
{
	struct swap_info_struct *p;

	p = __swap_info_get(entry);
	if (!p)
		goto out;
	if (!p->swap_map[swp_offset(entry)])
		goto bad_free;
	return p;

bad_free:
	pr_err("swap_info_get: %s%08lx\n", Unused_offset, entry.val);
	goto out;
out:
	return NULL;
}

H
Huang, Ying 已提交
1151 1152 1153 1154 1155 1156 1157 1158 1159 1160
static struct swap_info_struct *swap_info_get(swp_entry_t entry)
{
	struct swap_info_struct *p;

	p = _swap_info_get(entry);
	if (p)
		spin_lock(&p->lock);
	return p;
}

1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176
static struct swap_info_struct *swap_info_get_cont(swp_entry_t entry,
					struct swap_info_struct *q)
{
	struct swap_info_struct *p;

	p = _swap_info_get(entry);

	if (p != q) {
		if (q != NULL)
			spin_unlock(&q->lock);
		if (p != NULL)
			spin_lock(&p->lock);
	}
	return p;
}

1177 1178 1179
static unsigned char __swap_entry_free_locked(struct swap_info_struct *p,
					      unsigned long offset,
					      unsigned char usage)
L
Linus Torvalds 已提交
1180
{
1181 1182
	unsigned char count;
	unsigned char has_cache;
H
Huang, Ying 已提交
1183

1184
	count = p->swap_map[offset];
H
Huang, Ying 已提交
1185

1186 1187
	has_cache = count & SWAP_HAS_CACHE;
	count &= ~SWAP_HAS_CACHE;
1188

1189
	if (usage == SWAP_HAS_CACHE) {
1190
		VM_BUG_ON(!has_cache);
1191
		has_cache = 0;
1192 1193 1194 1195 1196 1197
	} else if (count == SWAP_MAP_SHMEM) {
		/*
		 * Or we could insist on shmem.c using a special
		 * swap_shmem_free() and free_shmem_swap_and_cache()...
		 */
		count = 0;
1198 1199 1200 1201 1202 1203 1204 1205 1206
	} else if ((count & ~COUNT_CONTINUED) <= SWAP_MAP_MAX) {
		if (count == COUNT_CONTINUED) {
			if (swap_count_continued(p, offset, count))
				count = SWAP_MAP_MAX | COUNT_CONTINUED;
			else
				count = SWAP_MAP_MAX;
		} else
			count--;
	}
1207 1208

	usage = count | has_cache;
1209 1210
	p->swap_map[offset] = usage ? : SWAP_HAS_CACHE;

1211 1212 1213
	return usage;
}

1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276
/*
 * Check whether swap entry is valid in the swap device.  If so,
 * return pointer to swap_info_struct, and keep the swap entry valid
 * via preventing the swap device from being swapoff, until
 * put_swap_device() is called.  Otherwise return NULL.
 *
 * The entirety of the RCU read critical section must come before the
 * return from or after the call to synchronize_rcu() in
 * enable_swap_info() or swapoff().  So if "si->flags & SWP_VALID" is
 * true, the si->map, si->cluster_info, etc. must be valid in the
 * critical section.
 *
 * Notice that swapoff or swapoff+swapon can still happen before the
 * rcu_read_lock() in get_swap_device() or after the rcu_read_unlock()
 * in put_swap_device() if there isn't any other way to prevent
 * swapoff, such as page lock, page table lock, etc.  The caller must
 * be prepared for that.  For example, the following situation is
 * possible.
 *
 *   CPU1				CPU2
 *   do_swap_page()
 *     ...				swapoff+swapon
 *     __read_swap_cache_async()
 *       swapcache_prepare()
 *         __swap_duplicate()
 *           // check swap_map
 *     // verify PTE not changed
 *
 * In __swap_duplicate(), the swap_map need to be checked before
 * changing partly because the specified swap entry may be for another
 * swap device which has been swapoff.  And in do_swap_page(), after
 * the page is read from the swap device, the PTE is verified not
 * changed with the page table locked to check whether the swap device
 * has been swapoff or swapoff+swapon.
 */
struct swap_info_struct *get_swap_device(swp_entry_t entry)
{
	struct swap_info_struct *si;
	unsigned long offset;

	if (!entry.val)
		goto out;
	si = swp_swap_info(entry);
	if (!si)
		goto bad_nofile;

	rcu_read_lock();
	if (!(si->flags & SWP_VALID))
		goto unlock_out;
	offset = swp_offset(entry);
	if (offset >= si->max)
		goto unlock_out;

	return si;
bad_nofile:
	pr_err("%s: %s%08lx\n", __func__, Bad_file, entry.val);
out:
	return NULL;
unlock_out:
	rcu_read_unlock();
	return NULL;
}

1277 1278 1279 1280 1281 1282 1283 1284
static unsigned char __swap_entry_free(struct swap_info_struct *p,
				       swp_entry_t entry, unsigned char usage)
{
	struct swap_cluster_info *ci;
	unsigned long offset = swp_offset(entry);

	ci = lock_cluster_or_swap_info(p, offset);
	usage = __swap_entry_free_locked(p, offset, usage);
1285
	unlock_cluster_or_swap_info(p, ci);
1286 1287
	if (!usage)
		free_swap_slot(entry);
1288 1289 1290

	return usage;
}
1291

1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302
static void swap_entry_free(struct swap_info_struct *p, swp_entry_t entry)
{
	struct swap_cluster_info *ci;
	unsigned long offset = swp_offset(entry);
	unsigned char count;

	ci = lock_cluster(p, offset);
	count = p->swap_map[offset];
	VM_BUG_ON(count != SWAP_HAS_CACHE);
	p->swap_map[offset] = 0;
	dec_cluster_info_page(p, p->cluster_info, offset);
H
Huang, Ying 已提交
1303 1304
	unlock_cluster(ci);

1305 1306
	mem_cgroup_uncharge_swap(entry, 1);
	swap_range_free(p, offset, 1);
L
Linus Torvalds 已提交
1307 1308 1309
}

/*
1310
 * Caller has made sure that the swap device corresponding to entry
L
Linus Torvalds 已提交
1311 1312 1313 1314
 * is still around or has not been recycled.
 */
void swap_free(swp_entry_t entry)
{
1315
	struct swap_info_struct *p;
L
Linus Torvalds 已提交
1316

H
Huang, Ying 已提交
1317
	p = _swap_info_get(entry);
1318 1319
	if (p)
		__swap_entry_free(p, entry, 1);
L
Linus Torvalds 已提交
1320 1321
}

1322 1323 1324
/*
 * Called after dropping swapcache to decrease refcnt to swap entries.
 */
1325
void put_swap_page(struct page *page, swp_entry_t entry)
1326 1327 1328 1329 1330 1331
{
	unsigned long offset = swp_offset(entry);
	unsigned long idx = offset / SWAPFILE_CLUSTER;
	struct swap_cluster_info *ci;
	struct swap_info_struct *si;
	unsigned char *map;
1332 1333
	unsigned int i, free_entries = 0;
	unsigned char val;
1334
	int size = swap_entry_size(hpage_nr_pages(page));
1335

1336
	si = _swap_info_get(entry);
1337 1338 1339
	if (!si)
		return;

1340
	ci = lock_cluster_or_swap_info(si, offset);
1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351
	if (size == SWAPFILE_CLUSTER) {
		VM_BUG_ON(!cluster_is_huge(ci));
		map = si->swap_map + offset;
		for (i = 0; i < SWAPFILE_CLUSTER; i++) {
			val = map[i];
			VM_BUG_ON(!(val & SWAP_HAS_CACHE));
			if (val == SWAP_HAS_CACHE)
				free_entries++;
		}
		cluster_clear_huge(ci);
		if (free_entries == SWAPFILE_CLUSTER) {
1352
			unlock_cluster_or_swap_info(si, ci);
1353 1354 1355 1356 1357 1358 1359
			spin_lock(&si->lock);
			mem_cgroup_uncharge_swap(entry, SWAPFILE_CLUSTER);
			swap_free_cluster(si, idx);
			spin_unlock(&si->lock);
			return;
		}
	}
1360 1361 1362 1363 1364 1365 1366
	for (i = 0; i < size; i++, entry.val++) {
		if (!__swap_entry_free_locked(si, offset + i, SWAP_HAS_CACHE)) {
			unlock_cluster_or_swap_info(si, ci);
			free_swap_slot(entry);
			if (i == size - 1)
				return;
			lock_cluster_or_swap_info(si, offset);
1367 1368
		}
	}
1369
	unlock_cluster_or_swap_info(si, ci);
1370
}
1371

1372
#ifdef CONFIG_THP_SWAP
1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386
int split_swap_cluster(swp_entry_t entry)
{
	struct swap_info_struct *si;
	struct swap_cluster_info *ci;
	unsigned long offset = swp_offset(entry);

	si = _swap_info_get(entry);
	if (!si)
		return -EBUSY;
	ci = lock_cluster(si, offset);
	cluster_clear_huge(ci);
	unlock_cluster(ci);
	return 0;
}
1387
#endif
1388

1389 1390 1391 1392 1393 1394 1395
static int swp_entry_cmp(const void *ent1, const void *ent2)
{
	const swp_entry_t *e1 = ent1, *e2 = ent2;

	return (int)swp_type(*e1) - (int)swp_type(*e2);
}

1396 1397 1398 1399 1400 1401 1402 1403 1404 1405
void swapcache_free_entries(swp_entry_t *entries, int n)
{
	struct swap_info_struct *p, *prev;
	int i;

	if (n <= 0)
		return;

	prev = NULL;
	p = NULL;
1406 1407 1408 1409 1410 1411 1412 1413

	/*
	 * Sort swap entries by swap device, so each lock is only taken once.
	 * nr_swapfiles isn't absolutely correct, but the overhead of sort() is
	 * so low that it isn't necessary to optimize further.
	 */
	if (nr_swapfiles > 1)
		sort(entries, n, sizeof(entries[0]), swp_entry_cmp, NULL);
1414 1415 1416 1417 1418 1419
	for (i = 0; i < n; ++i) {
		p = swap_info_get_cont(entries[i], prev);
		if (p)
			swap_entry_free(p, entries[i]);
		prev = p;
	}
H
Huang, Ying 已提交
1420
	if (p)
1421
		spin_unlock(&p->lock);
1422 1423
}

L
Linus Torvalds 已提交
1424
/*
1425
 * How many references to page are currently swapped out?
1426 1427
 * This does not give an exact answer when swap count is continued,
 * but does include the high COUNT_CONTINUED flag to allow for that.
L
Linus Torvalds 已提交
1428
 */
1429
int page_swapcount(struct page *page)
L
Linus Torvalds 已提交
1430
{
1431 1432
	int count = 0;
	struct swap_info_struct *p;
H
Huang, Ying 已提交
1433
	struct swap_cluster_info *ci;
L
Linus Torvalds 已提交
1434
	swp_entry_t entry;
H
Huang, Ying 已提交
1435
	unsigned long offset;
L
Linus Torvalds 已提交
1436

1437
	entry.val = page_private(page);
H
Huang, Ying 已提交
1438
	p = _swap_info_get(entry);
L
Linus Torvalds 已提交
1439
	if (p) {
H
Huang, Ying 已提交
1440 1441 1442 1443
		offset = swp_offset(entry);
		ci = lock_cluster_or_swap_info(p, offset);
		count = swap_count(p->swap_map[offset]);
		unlock_cluster_or_swap_info(p, ci);
L
Linus Torvalds 已提交
1444
	}
1445
	return count;
L
Linus Torvalds 已提交
1446 1447
}

1448
int __swap_count(swp_entry_t entry)
1449
{
1450
	struct swap_info_struct *si;
1451
	pgoff_t offset = swp_offset(entry);
1452
	int count = 0;
1453

1454 1455 1456 1457 1458 1459
	si = get_swap_device(entry);
	if (si) {
		count = swap_count(si->swap_map[offset]);
		put_swap_device(si);
	}
	return count;
1460 1461
}

1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473
static int swap_swapcount(struct swap_info_struct *si, swp_entry_t entry)
{
	int count = 0;
	pgoff_t offset = swp_offset(entry);
	struct swap_cluster_info *ci;

	ci = lock_cluster_or_swap_info(si, offset);
	count = swap_count(si->swap_map[offset]);
	unlock_cluster_or_swap_info(si, ci);
	return count;
}

1474 1475 1476 1477 1478 1479 1480 1481 1482 1483
/*
 * How many references to @entry are currently swapped out?
 * This does not give an exact answer when swap count is continued,
 * but does include the high COUNT_CONTINUED flag to allow for that.
 */
int __swp_swapcount(swp_entry_t entry)
{
	int count = 0;
	struct swap_info_struct *si;

1484 1485
	si = get_swap_device(entry);
	if (si) {
1486
		count = swap_swapcount(si, entry);
1487 1488
		put_swap_device(si);
	}
1489 1490 1491
	return count;
}

1492 1493 1494 1495 1496 1497 1498 1499
/*
 * How many references to @entry are currently swapped out?
 * This considers COUNT_CONTINUED so it returns exact answer.
 */
int swp_swapcount(swp_entry_t entry)
{
	int count, tmp_count, n;
	struct swap_info_struct *p;
H
Huang, Ying 已提交
1500
	struct swap_cluster_info *ci;
1501 1502 1503 1504
	struct page *page;
	pgoff_t offset;
	unsigned char *map;

H
Huang, Ying 已提交
1505
	p = _swap_info_get(entry);
1506 1507 1508
	if (!p)
		return 0;

H
Huang, Ying 已提交
1509 1510 1511 1512 1513
	offset = swp_offset(entry);

	ci = lock_cluster_or_swap_info(p, offset);

	count = swap_count(p->swap_map[offset]);
1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524
	if (!(count & COUNT_CONTINUED))
		goto out;

	count &= ~COUNT_CONTINUED;
	n = SWAP_MAP_MAX + 1;

	page = vmalloc_to_page(p->swap_map + offset);
	offset &= ~PAGE_MASK;
	VM_BUG_ON(page_private(page) != SWP_CONTINUED);

	do {
1525
		page = list_next_entry(page, lru);
1526 1527 1528 1529 1530 1531 1532 1533
		map = kmap_atomic(page);
		tmp_count = map[offset];
		kunmap_atomic(map);

		count += (tmp_count & ~COUNT_CONTINUED) * n;
		n *= (SWAP_CONT_MAX + 1);
	} while (tmp_count & COUNT_CONTINUED);
out:
H
Huang, Ying 已提交
1534
	unlock_cluster_or_swap_info(p, ci);
1535 1536 1537
	return count;
}

1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549
static bool swap_page_trans_huge_swapped(struct swap_info_struct *si,
					 swp_entry_t entry)
{
	struct swap_cluster_info *ci;
	unsigned char *map = si->swap_map;
	unsigned long roffset = swp_offset(entry);
	unsigned long offset = round_down(roffset, SWAPFILE_CLUSTER);
	int i;
	bool ret = false;

	ci = lock_cluster_or_swap_info(si, offset);
	if (!ci || !cluster_is_huge(ci)) {
1550
		if (swap_count(map[roffset]))
1551 1552 1553 1554
			ret = true;
		goto unlock_out;
	}
	for (i = 0; i < SWAPFILE_CLUSTER; i++) {
1555
		if (swap_count(map[offset + i])) {
1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569
			ret = true;
			break;
		}
	}
unlock_out:
	unlock_cluster_or_swap_info(si, ci);
	return ret;
}

static bool page_swapped(struct page *page)
{
	swp_entry_t entry;
	struct swap_info_struct *si;

1570
	if (!IS_ENABLED(CONFIG_THP_SWAP) || likely(!PageTransCompound(page)))
1571 1572 1573 1574 1575 1576 1577 1578 1579
		return page_swapcount(page) != 0;

	page = compound_head(page);
	entry.val = page_private(page);
	si = _swap_info_get(entry);
	if (si)
		return swap_page_trans_huge_swapped(si, entry);
	return false;
}
1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593

static int page_trans_huge_map_swapcount(struct page *page, int *total_mapcount,
					 int *total_swapcount)
{
	int i, map_swapcount, _total_mapcount, _total_swapcount;
	unsigned long offset = 0;
	struct swap_info_struct *si;
	struct swap_cluster_info *ci = NULL;
	unsigned char *map = NULL;
	int mapcount, swapcount = 0;

	/* hugetlbfs shouldn't call it */
	VM_BUG_ON_PAGE(PageHuge(page), page);

1594 1595
	if (!IS_ENABLED(CONFIG_THP_SWAP) || likely(!PageTransCompound(page))) {
		mapcount = page_trans_huge_mapcount(page, total_mapcount);
1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641
		if (PageSwapCache(page))
			swapcount = page_swapcount(page);
		if (total_swapcount)
			*total_swapcount = swapcount;
		return mapcount + swapcount;
	}

	page = compound_head(page);

	_total_mapcount = _total_swapcount = map_swapcount = 0;
	if (PageSwapCache(page)) {
		swp_entry_t entry;

		entry.val = page_private(page);
		si = _swap_info_get(entry);
		if (si) {
			map = si->swap_map;
			offset = swp_offset(entry);
		}
	}
	if (map)
		ci = lock_cluster(si, offset);
	for (i = 0; i < HPAGE_PMD_NR; i++) {
		mapcount = atomic_read(&page[i]._mapcount) + 1;
		_total_mapcount += mapcount;
		if (map) {
			swapcount = swap_count(map[offset + i]);
			_total_swapcount += swapcount;
		}
		map_swapcount = max(map_swapcount, mapcount + swapcount);
	}
	unlock_cluster(ci);
	if (PageDoubleMap(page)) {
		map_swapcount -= 1;
		_total_mapcount -= HPAGE_PMD_NR;
	}
	mapcount = compound_mapcount(page);
	map_swapcount += mapcount;
	_total_mapcount += mapcount;
	if (total_mapcount)
		*total_mapcount = _total_mapcount;
	if (total_swapcount)
		*total_swapcount = _total_swapcount;

	return map_swapcount;
}
1642

L
Linus Torvalds 已提交
1643
/*
1644 1645 1646 1647
 * We can write to an anon page without COW if there are no other references
 * to it.  And as a side-effect, free up its swap: because the old content
 * on disk will never be read, and seeking back there to write new content
 * later would only waste time away from clustering.
1648
 *
1649
 * NOTE: total_map_swapcount should not be relied upon by the caller if
1650 1651
 * reuse_swap_page() returns false, but it may be always overwritten
 * (see the other implementation for CONFIG_SWAP=n).
L
Linus Torvalds 已提交
1652
 */
1653
bool reuse_swap_page(struct page *page, int *total_map_swapcount)
L
Linus Torvalds 已提交
1654
{
1655
	int count, total_mapcount, total_swapcount;
1656

1657
	VM_BUG_ON_PAGE(!PageLocked(page), page);
1658
	if (unlikely(PageKsm(page)))
1659
		return false;
1660 1661 1662 1663 1664 1665 1666 1667
	count = page_trans_huge_map_swapcount(page, &total_mapcount,
					      &total_swapcount);
	if (total_map_swapcount)
		*total_map_swapcount = total_mapcount + total_swapcount;
	if (count == 1 && PageSwapCache(page) &&
	    (likely(!PageTransCompound(page)) ||
	     /* The remaining swap count will be freed soon */
	     total_swapcount == page_swapcount(page))) {
1668
		if (!PageWriteback(page)) {
1669
			page = compound_head(page);
1670 1671
			delete_from_swap_cache(page);
			SetPageDirty(page);
1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682
		} else {
			swp_entry_t entry;
			struct swap_info_struct *p;

			entry.val = page_private(page);
			p = swap_info_get(entry);
			if (p->flags & SWP_STABLE_WRITES) {
				spin_unlock(&p->lock);
				return false;
			}
			spin_unlock(&p->lock);
1683 1684
		}
	}
1685

1686
	return count <= 1;
L
Linus Torvalds 已提交
1687 1688 1689
}

/*
1690 1691
 * If swap is getting full, or if there are no more mappings of this page,
 * then try_to_free_swap is called to free its swap space.
L
Linus Torvalds 已提交
1692
 */
1693
int try_to_free_swap(struct page *page)
L
Linus Torvalds 已提交
1694
{
1695
	VM_BUG_ON_PAGE(!PageLocked(page), page);
L
Linus Torvalds 已提交
1696 1697 1698 1699 1700

	if (!PageSwapCache(page))
		return 0;
	if (PageWriteback(page))
		return 0;
1701
	if (page_swapped(page))
L
Linus Torvalds 已提交
1702 1703
		return 0;

1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715
	/*
	 * Once hibernation has begun to create its image of memory,
	 * there's a danger that one of the calls to try_to_free_swap()
	 * - most probably a call from __try_to_reclaim_swap() while
	 * hibernation is allocating its own swap pages for the image,
	 * but conceivably even a call from memory reclaim - will free
	 * the swap from a page which has already been recorded in the
	 * image as a clean swapcache page, and then reuse its swap for
	 * another page of the image.  On waking from hibernation, the
	 * original page might be freed under memory pressure, then
	 * later read back in from swap, now with the wrong data.
	 *
1716
	 * Hibernation suspends storage while it is writing the image
1717
	 * to disk so check that here.
1718
	 */
1719
	if (pm_suspended_storage())
1720 1721
		return 0;

1722
	page = compound_head(page);
1723 1724 1725
	delete_from_swap_cache(page);
	SetPageDirty(page);
	return 1;
1726 1727
}

L
Linus Torvalds 已提交
1728 1729 1730 1731
/*
 * Free the swap entry like above, but also try to
 * free the page cache entry if it is the last user.
 */
1732
int free_swap_and_cache(swp_entry_t entry)
L
Linus Torvalds 已提交
1733
{
1734
	struct swap_info_struct *p;
1735
	unsigned char count;
L
Linus Torvalds 已提交
1736

1737
	if (non_swap_entry(entry))
1738
		return 1;
1739

1740
	p = _swap_info_get(entry);
L
Linus Torvalds 已提交
1741
	if (p) {
1742
		count = __swap_entry_free(p, entry, 1);
1743
		if (count == SWAP_HAS_CACHE &&
1744 1745 1746
		    !swap_page_trans_huge_swapped(p, entry))
			__try_to_reclaim_swap(p, swp_offset(entry),
					      TTRS_UNMAPPED | TTRS_FULL);
L
Linus Torvalds 已提交
1747
	}
1748
	return p != NULL;
L
Linus Torvalds 已提交
1749 1750
}

1751
#ifdef CONFIG_HIBERNATION
1752
/*
1753
 * Find the swap type that corresponds to given device (if any).
1754
 *
1755 1756 1757 1758
 * @offset - number of the PAGE_SIZE-sized block of the device, starting
 * from 0, in which the swap header is expected to be located.
 *
 * This is needed for the suspend to disk (aka swsusp).
1759
 */
1760
int swap_type_of(dev_t device, sector_t offset, struct block_device **bdev_p)
1761
{
1762
	struct block_device *bdev = NULL;
1763
	int type;
1764

1765 1766 1767
	if (device)
		bdev = bdget(device);

1768
	spin_lock(&swap_lock);
1769 1770
	for (type = 0; type < nr_swapfiles; type++) {
		struct swap_info_struct *sis = swap_info[type];
1771

1772
		if (!(sis->flags & SWP_WRITEOK))
1773
			continue;
1774

1775
		if (!bdev) {
1776
			if (bdev_p)
1777
				*bdev_p = bdgrab(sis->bdev);
1778

1779
			spin_unlock(&swap_lock);
1780
			return type;
1781
		}
1782
		if (bdev == sis->bdev) {
1783
			struct swap_extent *se = first_se(sis);
1784 1785

			if (se->start_block == offset) {
1786
				if (bdev_p)
1787
					*bdev_p = bdgrab(sis->bdev);
1788

1789 1790
				spin_unlock(&swap_lock);
				bdput(bdev);
1791
				return type;
1792
			}
1793 1794 1795
		}
	}
	spin_unlock(&swap_lock);
1796 1797 1798
	if (bdev)
		bdput(bdev);

1799 1800 1801
	return -ENODEV;
}

1802 1803 1804 1805 1806 1807 1808
/*
 * Get the (PAGE_SIZE) block corresponding to given offset on the swapdev
 * corresponding to given index in swap_info (swap type).
 */
sector_t swapdev_block(int type, pgoff_t offset)
{
	struct block_device *bdev;
1809
	struct swap_info_struct *si = swap_type_to_swap_info(type);
1810

1811
	if (!si || !(si->flags & SWP_WRITEOK))
1812
		return 0;
1813
	return map_swap_entry(swp_entry(type, offset), &bdev);
1814 1815
}

1816 1817 1818 1819 1820 1821 1822 1823 1824 1825
/*
 * Return either the total number of swap pages of given type, or the number
 * of free pages of that type (depending on @free)
 *
 * This is needed for software suspend
 */
unsigned int count_swap_pages(int type, int free)
{
	unsigned int n = 0;

1826 1827 1828 1829
	spin_lock(&swap_lock);
	if ((unsigned int)type < nr_swapfiles) {
		struct swap_info_struct *sis = swap_info[type];

1830
		spin_lock(&sis->lock);
1831 1832
		if (sis->flags & SWP_WRITEOK) {
			n = sis->pages;
1833
			if (free)
1834
				n -= sis->inuse_pages;
1835
		}
1836
		spin_unlock(&sis->lock);
1837
	}
1838
	spin_unlock(&swap_lock);
1839 1840
	return n;
}
1841
#endif /* CONFIG_HIBERNATION */
1842

1843
static inline int pte_same_as_swp(pte_t pte, pte_t swp_pte)
1844
{
1845
	return pte_same(pte_swp_clear_soft_dirty(pte), swp_pte);
1846 1847
}

L
Linus Torvalds 已提交
1848
/*
1849 1850 1851
 * No need to decide whether this PTE shares the swap entry with others,
 * just let do_wp_page work it out if a write is requested later - to
 * force COW, vm_page_prot omits write permission from any private vma.
L
Linus Torvalds 已提交
1852
 */
1853
static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd,
L
Linus Torvalds 已提交
1854 1855
		unsigned long addr, swp_entry_t entry, struct page *page)
{
1856
	struct page *swapcache;
1857
	struct mem_cgroup *memcg;
1858 1859 1860 1861
	spinlock_t *ptl;
	pte_t *pte;
	int ret = 1;

1862 1863 1864 1865 1866
	swapcache = page;
	page = ksm_might_need_to_copy(page, vma, addr);
	if (unlikely(!page))
		return -ENOMEM;

1867 1868
	if (mem_cgroup_try_charge(page, vma->vm_mm, GFP_KERNEL,
				&memcg, false)) {
1869
		ret = -ENOMEM;
1870 1871
		goto out_nolock;
	}
1872 1873

	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
1874
	if (unlikely(!pte_same_as_swp(*pte, swp_entry_to_pte(entry)))) {
1875
		mem_cgroup_cancel_charge(page, memcg, false);
1876 1877 1878
		ret = 0;
		goto out;
	}
1879

K
KAMEZAWA Hiroyuki 已提交
1880
	dec_mm_counter(vma->vm_mm, MM_SWAPENTS);
1881
	inc_mm_counter(vma->vm_mm, MM_ANONPAGES);
L
Linus Torvalds 已提交
1882 1883 1884
	get_page(page);
	set_pte_at(vma->vm_mm, addr, pte,
		   pte_mkold(mk_pte(page, vma->vm_page_prot)));
1885
	if (page == swapcache) {
1886
		page_add_anon_rmap(page, vma, addr, false);
1887
		mem_cgroup_commit_charge(page, memcg, true, false);
1888
	} else { /* ksm created a completely new copy */
1889
		page_add_new_anon_rmap(page, vma, addr, false);
1890
		mem_cgroup_commit_charge(page, memcg, false, false);
1891 1892
		lru_cache_add_active_or_unevictable(page, vma);
	}
L
Linus Torvalds 已提交
1893 1894 1895 1896 1897 1898
	swap_free(entry);
	/*
	 * Move the page to the active list so it is not
	 * immediately swapped out again after swapon.
	 */
	activate_page(page);
1899 1900
out:
	pte_unmap_unlock(pte, ptl);
1901
out_nolock:
1902 1903 1904 1905
	if (page != swapcache) {
		unlock_page(page);
		put_page(page);
	}
1906
	return ret;
L
Linus Torvalds 已提交
1907 1908 1909
}

static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
1910 1911 1912
			unsigned long addr, unsigned long end,
			unsigned int type, bool frontswap,
			unsigned long *fs_pages_to_unuse)
L
Linus Torvalds 已提交
1913
{
1914 1915
	struct page *page;
	swp_entry_t entry;
1916
	pte_t *pte;
1917 1918
	struct swap_info_struct *si;
	unsigned long offset;
1919
	int ret = 0;
1920
	volatile unsigned char *swap_map;
L
Linus Torvalds 已提交
1921

1922
	si = swap_info[type];
1923
	pte = pte_offset_map(pmd, addr);
L
Linus Torvalds 已提交
1924
	do {
1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939
		struct vm_fault vmf;

		if (!is_swap_pte(*pte))
			continue;

		entry = pte_to_swp_entry(*pte);
		if (swp_type(entry) != type)
			continue;

		offset = swp_offset(entry);
		if (frontswap && !frontswap_test(si, offset))
			continue;

		pte_unmap(pte);
		swap_map = &si->swap_map[offset];
1940 1941 1942 1943 1944 1945 1946 1947
		page = lookup_swap_cache(entry, vma, addr);
		if (!page) {
			vmf.vma = vma;
			vmf.address = addr;
			vmf.pmd = pmd;
			page = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE,
						&vmf);
		}
1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969
		if (!page) {
			if (*swap_map == 0 || *swap_map == SWAP_MAP_BAD)
				goto try_next;
			return -ENOMEM;
		}

		lock_page(page);
		wait_on_page_writeback(page);
		ret = unuse_pte(vma, pmd, addr, entry, page);
		if (ret < 0) {
			unlock_page(page);
			put_page(page);
			goto out;
		}

		try_to_free_swap(page);
		unlock_page(page);
		put_page(page);

		if (*fs_pages_to_unuse && !--(*fs_pages_to_unuse)) {
			ret = FRONTSWAP_PAGES_UNUSED;
			goto out;
L
Linus Torvalds 已提交
1970
		}
1971 1972
try_next:
		pte = pte_offset_map(pmd, addr);
L
Linus Torvalds 已提交
1973
	} while (pte++, addr += PAGE_SIZE, addr != end);
1974
	pte_unmap(pte - 1);
1975 1976

	ret = 0;
1977
out:
1978
	return ret;
L
Linus Torvalds 已提交
1979 1980 1981 1982
}

static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud,
				unsigned long addr, unsigned long end,
1983 1984
				unsigned int type, bool frontswap,
				unsigned long *fs_pages_to_unuse)
L
Linus Torvalds 已提交
1985 1986 1987
{
	pmd_t *pmd;
	unsigned long next;
1988
	int ret;
L
Linus Torvalds 已提交
1989 1990 1991

	pmd = pmd_offset(pud, addr);
	do {
1992
		cond_resched();
L
Linus Torvalds 已提交
1993
		next = pmd_addr_end(addr, end);
1994
		if (pmd_none_or_trans_huge_or_clear_bad(pmd))
L
Linus Torvalds 已提交
1995
			continue;
1996 1997
		ret = unuse_pte_range(vma, pmd, addr, next, type,
				      frontswap, fs_pages_to_unuse);
1998 1999
		if (ret)
			return ret;
L
Linus Torvalds 已提交
2000 2001 2002 2003
	} while (pmd++, addr = next, addr != end);
	return 0;
}

2004
static inline int unuse_pud_range(struct vm_area_struct *vma, p4d_t *p4d,
L
Linus Torvalds 已提交
2005
				unsigned long addr, unsigned long end,
2006 2007
				unsigned int type, bool frontswap,
				unsigned long *fs_pages_to_unuse)
L
Linus Torvalds 已提交
2008 2009 2010
{
	pud_t *pud;
	unsigned long next;
2011
	int ret;
L
Linus Torvalds 已提交
2012

2013
	pud = pud_offset(p4d, addr);
L
Linus Torvalds 已提交
2014 2015 2016 2017
	do {
		next = pud_addr_end(addr, end);
		if (pud_none_or_clear_bad(pud))
			continue;
2018 2019
		ret = unuse_pmd_range(vma, pud, addr, next, type,
				      frontswap, fs_pages_to_unuse);
2020 2021
		if (ret)
			return ret;
L
Linus Torvalds 已提交
2022 2023 2024 2025
	} while (pud++, addr = next, addr != end);
	return 0;
}

2026 2027
static inline int unuse_p4d_range(struct vm_area_struct *vma, pgd_t *pgd,
				unsigned long addr, unsigned long end,
2028 2029
				unsigned int type, bool frontswap,
				unsigned long *fs_pages_to_unuse)
2030 2031 2032 2033 2034 2035 2036 2037 2038 2039
{
	p4d_t *p4d;
	unsigned long next;
	int ret;

	p4d = p4d_offset(pgd, addr);
	do {
		next = p4d_addr_end(addr, end);
		if (p4d_none_or_clear_bad(p4d))
			continue;
2040 2041
		ret = unuse_pud_range(vma, p4d, addr, next, type,
				      frontswap, fs_pages_to_unuse);
2042 2043 2044 2045 2046 2047
		if (ret)
			return ret;
	} while (p4d++, addr = next, addr != end);
	return 0;
}

2048 2049
static int unuse_vma(struct vm_area_struct *vma, unsigned int type,
		     bool frontswap, unsigned long *fs_pages_to_unuse)
L
Linus Torvalds 已提交
2050 2051 2052
{
	pgd_t *pgd;
	unsigned long addr, end, next;
2053
	int ret;
L
Linus Torvalds 已提交
2054

2055 2056
	addr = vma->vm_start;
	end = vma->vm_end;
L
Linus Torvalds 已提交
2057 2058 2059 2060 2061 2062

	pgd = pgd_offset(vma->vm_mm, addr);
	do {
		next = pgd_addr_end(addr, end);
		if (pgd_none_or_clear_bad(pgd))
			continue;
2063 2064
		ret = unuse_p4d_range(vma, pgd, addr, next, type,
				      frontswap, fs_pages_to_unuse);
2065 2066
		if (ret)
			return ret;
L
Linus Torvalds 已提交
2067 2068 2069 2070
	} while (pgd++, addr = next, addr != end);
	return 0;
}

2071 2072
static int unuse_mm(struct mm_struct *mm, unsigned int type,
		    bool frontswap, unsigned long *fs_pages_to_unuse)
L
Linus Torvalds 已提交
2073 2074
{
	struct vm_area_struct *vma;
2075
	int ret = 0;
L
Linus Torvalds 已提交
2076

2077
	down_read(&mm->mmap_sem);
L
Linus Torvalds 已提交
2078
	for (vma = mm->mmap; vma; vma = vma->vm_next) {
2079 2080 2081 2082 2083 2084
		if (vma->anon_vma) {
			ret = unuse_vma(vma, type, frontswap,
					fs_pages_to_unuse);
			if (ret)
				break;
		}
2085
		cond_resched();
L
Linus Torvalds 已提交
2086 2087
	}
	up_read(&mm->mmap_sem);
2088
	return ret;
L
Linus Torvalds 已提交
2089 2090 2091
}

/*
2092
 * Scan swap_map (or frontswap_map if frontswap parameter is true)
2093 2094
 * from current position to next entry still in use. Return 0
 * if there are no inuse entries after prev till end of the map.
L
Linus Torvalds 已提交
2095
 */
2096
static unsigned int find_next_to_unuse(struct swap_info_struct *si,
2097
					unsigned int prev, bool frontswap)
L
Linus Torvalds 已提交
2098
{
2099
	unsigned int i;
2100
	unsigned char count;
L
Linus Torvalds 已提交
2101 2102

	/*
2103
	 * No need for swap_lock here: we're just looking
L
Linus Torvalds 已提交
2104 2105
	 * for whether an entry is in use, not modifying it; false
	 * hits are okay, and sys_swapoff() has already prevented new
2106
	 * allocations from this area (while holding swap_lock).
L
Linus Torvalds 已提交
2107
	 */
2108
	for (i = prev + 1; i < si->max; i++) {
2109
		count = READ_ONCE(si->swap_map[i]);
2110
		if (count && swap_count(count) != SWAP_MAP_BAD)
2111 2112 2113 2114
			if (!frontswap || frontswap_test(si, i))
				break;
		if ((i % LATENCY_LIMIT) == 0)
			cond_resched();
L
Linus Torvalds 已提交
2115
	}
2116 2117 2118 2119

	if (i == si->max)
		i = 0;

L
Linus Torvalds 已提交
2120 2121 2122 2123
	return i;
}

/*
2124
 * If the boolean frontswap is true, only unuse pages_to_unuse pages;
2125
 * pages_to_unuse==0 means all pages; ignored if frontswap is false
L
Linus Torvalds 已提交
2126
 */
2127 2128
int try_to_unuse(unsigned int type, bool frontswap,
		 unsigned long pages_to_unuse)
L
Linus Torvalds 已提交
2129
{
2130 2131 2132 2133
	struct mm_struct *prev_mm;
	struct mm_struct *mm;
	struct list_head *p;
	int retval = 0;
2134
	struct swap_info_struct *si = swap_info[type];
L
Linus Torvalds 已提交
2135 2136
	struct page *page;
	swp_entry_t entry;
2137
	unsigned int i;
L
Linus Torvalds 已提交
2138

2139
	if (!READ_ONCE(si->inuse_pages))
2140
		return 0;
L
Linus Torvalds 已提交
2141

2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154
	if (!frontswap)
		pages_to_unuse = 0;

retry:
	retval = shmem_unuse(type, frontswap, &pages_to_unuse);
	if (retval)
		goto out;

	prev_mm = &init_mm;
	mmget(prev_mm);

	spin_lock(&mmlist_lock);
	p = &init_mm.mmlist;
2155
	while (READ_ONCE(si->inuse_pages) &&
2156 2157
	       !signal_pending(current) &&
	       (p = p->next) != &init_mm.mmlist) {
L
Linus Torvalds 已提交
2158

2159 2160 2161 2162 2163 2164 2165
		mm = list_entry(p, struct mm_struct, mmlist);
		if (!mmget_not_zero(mm))
			continue;
		spin_unlock(&mmlist_lock);
		mmput(prev_mm);
		prev_mm = mm;
		retval = unuse_mm(mm, type, frontswap, &pages_to_unuse);
L
Linus Torvalds 已提交
2166

2167 2168 2169
		if (retval) {
			mmput(prev_mm);
			goto out;
L
Linus Torvalds 已提交
2170 2171 2172
		}

		/*
2173 2174
		 * Make sure that we aren't completely killing
		 * interactive performance.
L
Linus Torvalds 已提交
2175
		 */
2176 2177 2178 2179
		cond_resched();
		spin_lock(&mmlist_lock);
	}
	spin_unlock(&mmlist_lock);
L
Linus Torvalds 已提交
2180

2181
	mmput(prev_mm);
L
Linus Torvalds 已提交
2182

2183
	i = 0;
2184
	while (READ_ONCE(si->inuse_pages) &&
2185 2186
	       !signal_pending(current) &&
	       (i = find_next_to_unuse(si, i, frontswap)) != 0) {
L
Linus Torvalds 已提交
2187

2188 2189 2190 2191
		entry = swp_entry(type, i);
		page = find_get_page(swap_address_space(entry), i);
		if (!page)
			continue;
2192 2193 2194

		/*
		 * It is conceivable that a racing task removed this page from
2195 2196 2197
		 * swap cache just before we acquired the page lock. The page
		 * might even be back in swap cache on another swap area. But
		 * that is okay, try_to_free_swap() only removes stale pages.
L
Linus Torvalds 已提交
2198
		 */
2199 2200 2201
		lock_page(page);
		wait_on_page_writeback(page);
		try_to_free_swap(page);
L
Linus Torvalds 已提交
2202
		unlock_page(page);
2203
		put_page(page);
L
Linus Torvalds 已提交
2204 2205

		/*
2206 2207 2208
		 * For frontswap, we just need to unuse pages_to_unuse, if
		 * it was specified. Need not check frontswap again here as
		 * we already zeroed out pages_to_unuse if not frontswap.
L
Linus Torvalds 已提交
2209
		 */
2210 2211
		if (pages_to_unuse && --pages_to_unuse == 0)
			goto out;
L
Linus Torvalds 已提交
2212 2213
	}

2214 2215 2216 2217 2218
	/*
	 * Lets check again to see if there are still swap entries in the map.
	 * If yes, we would need to do retry the unuse logic again.
	 * Under global memory pressure, swap entries can be reinserted back
	 * into process space after the mmlist loop above passes over them.
2219
	 *
2220 2221 2222 2223 2224
	 * Limit the number of retries? No: when mmget_not_zero() above fails,
	 * that mm is likely to be freeing swap from exit_mmap(), which proceeds
	 * at its own independent pace; and even shmem_writepage() could have
	 * been preempted after get_swap_page(), temporarily hiding that swap.
	 * It's easy and robust (though cpu-intensive) just to keep retrying.
2225
	 */
2226
	if (READ_ONCE(si->inuse_pages)) {
2227 2228 2229 2230
		if (!signal_pending(current))
			goto retry;
		retval = -EINTR;
	}
2231 2232
out:
	return (retval == FRONTSWAP_PAGES_UNUSED) ? 0 : retval;
L
Linus Torvalds 已提交
2233 2234 2235
}

/*
2236 2237 2238
 * After a successful try_to_unuse, if no swap is now in use, we know
 * we can empty the mmlist.  swap_lock must be held on entry and exit.
 * Note that mmlist_lock nests inside swap_lock, and an mm must be
L
Linus Torvalds 已提交
2239 2240 2241 2242 2243
 * added to the mmlist just after page_duplicate - before would be racy.
 */
static void drain_mmlist(void)
{
	struct list_head *p, *next;
2244
	unsigned int type;
L
Linus Torvalds 已提交
2245

2246 2247
	for (type = 0; type < nr_swapfiles; type++)
		if (swap_info[type]->inuse_pages)
L
Linus Torvalds 已提交
2248 2249 2250 2251 2252 2253 2254 2255 2256
			return;
	spin_lock(&mmlist_lock);
	list_for_each_safe(p, next, &init_mm.mmlist)
		list_del_init(p);
	spin_unlock(&mmlist_lock);
}

/*
 * Use this swapdev's extent info to locate the (PAGE_SIZE) block which
2257 2258 2259
 * corresponds to page offset for the specified swap entry.
 * Note that the type of this function is sector_t, but it returns page offset
 * into the bdev, not sector offset.
L
Linus Torvalds 已提交
2260
 */
2261
static sector_t map_swap_entry(swp_entry_t entry, struct block_device **bdev)
L
Linus Torvalds 已提交
2262
{
2263 2264 2265 2266
	struct swap_info_struct *sis;
	struct swap_extent *se;
	pgoff_t offset;

2267
	sis = swp_swap_info(entry);
2268 2269 2270
	*bdev = sis->bdev;

	offset = swp_offset(entry);
2271 2272
	se = offset_to_swap_extent(sis, offset);
	return se->start_block + (offset - se->start_page);
L
Linus Torvalds 已提交
2273 2274
}

2275 2276 2277 2278 2279 2280 2281 2282 2283 2284
/*
 * Returns the page offset into bdev for the specified page's swap entry.
 */
sector_t map_swap_page(struct page *page, struct block_device **bdev)
{
	swp_entry_t entry;
	entry.val = page_private(page);
	return map_swap_entry(entry, bdev);
}

L
Linus Torvalds 已提交
2285 2286 2287 2288 2289
/*
 * Free all of a swapdev's extent information
 */
static void destroy_swap_extents(struct swap_info_struct *sis)
{
2290 2291 2292
	while (!RB_EMPTY_ROOT(&sis->swap_extent_root)) {
		struct rb_node *rb = sis->swap_extent_root.rb_node;
		struct swap_extent *se = rb_entry(rb, struct swap_extent, rb_node);
L
Linus Torvalds 已提交
2293

2294
		rb_erase(rb, &sis->swap_extent_root);
L
Linus Torvalds 已提交
2295 2296
		kfree(se);
	}
2297

2298
	if (sis->flags & SWP_ACTIVATED) {
2299 2300 2301
		struct file *swap_file = sis->swap_file;
		struct address_space *mapping = swap_file->f_mapping;

2302 2303 2304
		sis->flags &= ~SWP_ACTIVATED;
		if (mapping->a_ops->swap_deactivate)
			mapping->a_ops->swap_deactivate(swap_file);
2305
	}
L
Linus Torvalds 已提交
2306 2307 2308 2309
}

/*
 * Add a block range (and the corresponding page range) into this swapdev's
2310
 * extent tree.
L
Linus Torvalds 已提交
2311
 *
2312
 * This function rather assumes that it is called in ascending page order.
L
Linus Torvalds 已提交
2313
 */
2314
int
L
Linus Torvalds 已提交
2315 2316 2317
add_swap_extent(struct swap_info_struct *sis, unsigned long start_page,
		unsigned long nr_pages, sector_t start_block)
{
2318
	struct rb_node **link = &sis->swap_extent_root.rb_node, *parent = NULL;
L
Linus Torvalds 已提交
2319 2320
	struct swap_extent *se;
	struct swap_extent *new_se;
2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332

	/*
	 * place the new node at the right most since the
	 * function is called in ascending page order.
	 */
	while (*link) {
		parent = *link;
		link = &parent->rb_right;
	}

	if (parent) {
		se = rb_entry(parent, struct swap_extent, rb_node);
2333 2334
		BUG_ON(se->start_page + se->nr_pages != start_page);
		if (se->start_block + se->nr_pages == start_block) {
L
Linus Torvalds 已提交
2335 2336 2337 2338 2339 2340
			/* Merge it */
			se->nr_pages += nr_pages;
			return 0;
		}
	}

2341
	/* No merge, insert a new extent. */
L
Linus Torvalds 已提交
2342 2343 2344 2345 2346 2347 2348
	new_se = kmalloc(sizeof(*se), GFP_KERNEL);
	if (new_se == NULL)
		return -ENOMEM;
	new_se->start_page = start_page;
	new_se->nr_pages = nr_pages;
	new_se->start_block = start_block;

2349 2350
	rb_link_node(&new_se->rb_node, parent, link);
	rb_insert_color(&new_se->rb_node, &sis->swap_extent_root);
2351
	return 1;
L
Linus Torvalds 已提交
2352
}
2353
EXPORT_SYMBOL_GPL(add_swap_extent);
L
Linus Torvalds 已提交
2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374

/*
 * A `swap extent' is a simple thing which maps a contiguous range of pages
 * onto a contiguous range of disk blocks.  An ordered list of swap extents
 * is built at swapon time and is then used at swap_writepage/swap_readpage
 * time for locating where on disk a page belongs.
 *
 * If the swapfile is an S_ISBLK block device, a single extent is installed.
 * This is done so that the main operating code can treat S_ISBLK and S_ISREG
 * swap files identically.
 *
 * Whether the swapdev is an S_ISREG file or an S_ISBLK blockdev, the swap
 * extent list operates in PAGE_SIZE disk blocks.  Both S_ISREG and S_ISBLK
 * swapfiles are handled *identically* after swapon time.
 *
 * For S_ISREG swapfiles, setup_swap_extents() will walk all the file's blocks
 * and will parse them into an ordered extent list, in PAGE_SIZE chunks.  If
 * some stray blocks are found which do not fall within the PAGE_SIZE alignment
 * requirements, they are simply tossed out - we will never use those blocks
 * for swapping.
 *
2375 2376
 * For all swap devices we set S_SWAPFILE across the life of the swapon.  This
 * prevents users from writing to the swap device, which will corrupt memory.
L
Linus Torvalds 已提交
2377 2378 2379 2380 2381 2382 2383 2384
 *
 * The amount of disk space which a single swap extent represents varies.
 * Typically it is in the 1-4 megabyte range.  So we can have hundreds of
 * extents in the list.  To avoid much list walking, we cache the previous
 * search location in `curr_swap_extent', and start new searches from there.
 * This is extremely effective.  The average number of iterations in
 * map_swap_page() has been measured at about 0.3 per page.  - akpm.
 */
2385
static int setup_swap_extents(struct swap_info_struct *sis, sector_t *span)
L
Linus Torvalds 已提交
2386
{
2387 2388 2389
	struct file *swap_file = sis->swap_file;
	struct address_space *mapping = swap_file->f_mapping;
	struct inode *inode = mapping->host;
L
Linus Torvalds 已提交
2390 2391 2392 2393
	int ret;

	if (S_ISBLK(inode->i_mode)) {
		ret = add_swap_extent(sis, 0, sis->max, 0);
2394
		*span = sis->pages;
2395
		return ret;
L
Linus Torvalds 已提交
2396 2397
	}

2398
	if (mapping->a_ops->swap_activate) {
2399
		ret = mapping->a_ops->swap_activate(sis, swap_file, span);
2400 2401
		if (ret >= 0)
			sis->flags |= SWP_ACTIVATED;
2402
		if (!ret) {
2403
			sis->flags |= SWP_FS;
2404 2405 2406
			ret = add_swap_extent(sis, 0, sis->max, 0);
			*span = sis->pages;
		}
2407
		return ret;
2408 2409
	}

2410
	return generic_swapfile_activate(sis, swap_file, span);
L
Linus Torvalds 已提交
2411 2412
}

2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424
static int swap_node(struct swap_info_struct *p)
{
	struct block_device *bdev;

	if (p->bdev)
		bdev = p->bdev;
	else
		bdev = p->swap_file->f_inode->i_sb->s_bdev;

	return bdev ? bdev->bd_disk->node_id : NUMA_NO_NODE;
}

2425 2426 2427
static void setup_swap_info(struct swap_info_struct *p, int prio,
			    unsigned char *swap_map,
			    struct swap_cluster_info *cluster_info)
2428
{
2429 2430
	int i;

2431 2432 2433 2434
	if (prio >= 0)
		p->prio = prio;
	else
		p->prio = --least_priority;
2435 2436 2437 2438 2439
	/*
	 * the plist prio is negated because plist ordering is
	 * low-to-high, while swap ordering is high-to-low
	 */
	p->list.prio = -p->prio;
2440 2441 2442 2443 2444 2445 2446 2447 2448 2449
	for_each_node(i) {
		if (p->prio >= 0)
			p->avail_lists[i].prio = -p->prio;
		else {
			if (swap_node(p) == i)
				p->avail_lists[i].prio = 1;
			else
				p->avail_lists[i].prio = -p->prio;
		}
	}
2450
	p->swap_map = swap_map;
2451
	p->cluster_info = cluster_info;
2452 2453 2454 2455 2456
}

static void _enable_swap_info(struct swap_info_struct *p)
{
	p->flags |= SWP_WRITEOK | SWP_VALID;
2457
	atomic_long_add(p->pages, &nr_swap_pages);
2458 2459
	total_swap_pages += p->pages;

2460 2461
	assert_spin_locked(&swap_lock);
	/*
2462 2463 2464 2465 2466 2467 2468 2469
	 * both lists are plists, and thus priority ordered.
	 * swap_active_head needs to be priority ordered for swapoff(),
	 * which on removal of any swap_info_struct with an auto-assigned
	 * (i.e. negative) priority increments the auto-assigned priority
	 * of any lower-priority swap_info_structs.
	 * swap_avail_head needs to be priority ordered for get_swap_page(),
	 * which allocates swap pages from the highest available priority
	 * swap_info_struct.
2470
	 */
2471
	plist_add(&p->list, &swap_active_head);
2472
	add_to_avail_list(p);
2473 2474 2475 2476
}

static void enable_swap_info(struct swap_info_struct *p, int prio,
				unsigned char *swap_map,
2477
				struct swap_cluster_info *cluster_info,
2478 2479
				unsigned long *frontswap_map)
{
2480
	frontswap_init(p->type, frontswap_map);
2481
	spin_lock(&swap_lock);
2482
	spin_lock(&p->lock);
2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493
	setup_swap_info(p, prio, swap_map, cluster_info);
	spin_unlock(&p->lock);
	spin_unlock(&swap_lock);
	/*
	 * Guarantee swap_map, cluster_info, etc. fields are valid
	 * between get/put_swap_device() if SWP_VALID bit is set
	 */
	synchronize_rcu();
	spin_lock(&swap_lock);
	spin_lock(&p->lock);
	_enable_swap_info(p);
2494
	spin_unlock(&p->lock);
2495 2496 2497 2498 2499 2500
	spin_unlock(&swap_lock);
}

static void reinsert_swap_info(struct swap_info_struct *p)
{
	spin_lock(&swap_lock);
2501
	spin_lock(&p->lock);
2502 2503
	setup_swap_info(p, p->prio, p->swap_map, p->cluster_info);
	_enable_swap_info(p);
2504
	spin_unlock(&p->lock);
2505 2506 2507
	spin_unlock(&swap_lock);
}

2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518
bool has_usable_swap(void)
{
	bool ret = true;

	spin_lock(&swap_lock);
	if (plist_head_empty(&swap_active_head))
		ret = false;
	spin_unlock(&swap_lock);
	return ret;
}

2519
SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
L
Linus Torvalds 已提交
2520
{
2521
	struct swap_info_struct *p = NULL;
2522
	unsigned char *swap_map;
2523
	struct swap_cluster_info *cluster_info;
2524
	unsigned long *frontswap_map;
L
Linus Torvalds 已提交
2525 2526 2527
	struct file *swap_file, *victim;
	struct address_space *mapping;
	struct inode *inode;
2528
	struct filename *pathname;
2529
	int err, found = 0;
2530
	unsigned int old_block_size;
2531

L
Linus Torvalds 已提交
2532 2533 2534
	if (!capable(CAP_SYS_ADMIN))
		return -EPERM;

2535 2536
	BUG_ON(!current->mm);

L
Linus Torvalds 已提交
2537 2538
	pathname = getname(specialfile);
	if (IS_ERR(pathname))
2539
		return PTR_ERR(pathname);
L
Linus Torvalds 已提交
2540

2541
	victim = file_open_name(pathname, O_RDWR|O_LARGEFILE, 0);
L
Linus Torvalds 已提交
2542 2543 2544 2545 2546
	err = PTR_ERR(victim);
	if (IS_ERR(victim))
		goto out;

	mapping = victim->f_mapping;
2547
	spin_lock(&swap_lock);
2548
	plist_for_each_entry(p, &swap_active_head, list) {
2549
		if (p->flags & SWP_WRITEOK) {
2550 2551
			if (p->swap_file->f_mapping == mapping) {
				found = 1;
L
Linus Torvalds 已提交
2552
				break;
2553
			}
L
Linus Torvalds 已提交
2554 2555
		}
	}
2556
	if (!found) {
L
Linus Torvalds 已提交
2557
		err = -EINVAL;
2558
		spin_unlock(&swap_lock);
L
Linus Torvalds 已提交
2559 2560
		goto out_dput;
	}
2561
	if (!security_vm_enough_memory_mm(current->mm, p->pages))
L
Linus Torvalds 已提交
2562 2563 2564
		vm_unacct_memory(p->pages);
	else {
		err = -ENOMEM;
2565
		spin_unlock(&swap_lock);
L
Linus Torvalds 已提交
2566 2567
		goto out_dput;
	}
2568
	del_from_avail_list(p);
2569
	spin_lock(&p->lock);
2570
	if (p->prio < 0) {
2571
		struct swap_info_struct *si = p;
2572
		int nid;
2573

2574
		plist_for_each_entry_continue(si, &swap_active_head, list) {
2575
			si->prio++;
2576
			si->list.prio--;
2577 2578 2579 2580
			for_each_node(nid) {
				if (si->avail_lists[nid].prio != 1)
					si->avail_lists[nid].prio--;
			}
2581
		}
2582 2583
		least_priority++;
	}
2584
	plist_del(&p->list, &swap_active_head);
2585
	atomic_long_sub(p->pages, &nr_swap_pages);
L
Linus Torvalds 已提交
2586 2587
	total_swap_pages -= p->pages;
	p->flags &= ~SWP_WRITEOK;
2588
	spin_unlock(&p->lock);
2589
	spin_unlock(&swap_lock);
2590

2591 2592
	disable_swap_slots_cache_lock();

2593
	set_current_oom_origin();
2594
	err = try_to_unuse(p->type, false, 0); /* force unuse all pages */
2595
	clear_current_oom_origin();
L
Linus Torvalds 已提交
2596 2597 2598

	if (err) {
		/* re-insert swap space back into swap_list */
2599
		reinsert_swap_info(p);
2600
		reenable_swap_slots_cache_unlock();
L
Linus Torvalds 已提交
2601 2602
		goto out_dput;
	}
2603

2604 2605
	reenable_swap_slots_cache_unlock();

2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616
	spin_lock(&swap_lock);
	spin_lock(&p->lock);
	p->flags &= ~SWP_VALID;		/* mark swap device as invalid */
	spin_unlock(&p->lock);
	spin_unlock(&swap_lock);
	/*
	 * wait for swap operations protected by get/put_swap_device()
	 * to complete
	 */
	synchronize_rcu();

2617 2618
	flush_work(&p->discard_work);

2619
	destroy_swap_extents(p);
2620 2621 2622
	if (p->flags & SWP_CONTINUED)
		free_swap_count_continuations(p);

2623 2624 2625
	if (!p->bdev || !blk_queue_nonrot(bdev_get_queue(p->bdev)))
		atomic_dec(&nr_rotate_swap);

2626
	mutex_lock(&swapon_mutex);
2627
	spin_lock(&swap_lock);
2628
	spin_lock(&p->lock);
2629 2630
	drain_mmlist();

2631 2632 2633
	/* wait for anyone still in scan_swap_map */
	p->highest_bit = 0;		/* cuts scans short */
	while (p->flags >= SWP_SCANNING) {
2634
		spin_unlock(&p->lock);
2635
		spin_unlock(&swap_lock);
2636
		schedule_timeout_uninterruptible(1);
2637
		spin_lock(&swap_lock);
2638
		spin_lock(&p->lock);
2639 2640
	}

L
Linus Torvalds 已提交
2641
	swap_file = p->swap_file;
2642
	old_block_size = p->old_block_size;
L
Linus Torvalds 已提交
2643 2644 2645 2646
	p->swap_file = NULL;
	p->max = 0;
	swap_map = p->swap_map;
	p->swap_map = NULL;
2647 2648
	cluster_info = p->cluster_info;
	p->cluster_info = NULL;
2649
	frontswap_map = frontswap_map_get(p);
2650
	spin_unlock(&p->lock);
2651
	spin_unlock(&swap_lock);
2652
	frontswap_invalidate_area(p->type);
2653
	frontswap_map_set(p, NULL);
2654
	mutex_unlock(&swapon_mutex);
2655 2656
	free_percpu(p->percpu_cluster);
	p->percpu_cluster = NULL;
L
Linus Torvalds 已提交
2657
	vfree(swap_map);
2658 2659
	kvfree(cluster_info);
	kvfree(frontswap_map);
2660
	/* Destroy swap account information */
2661
	swap_cgroup_swapoff(p->type);
2662
	exit_swap_address_space(p->type);
2663

L
Linus Torvalds 已提交
2664 2665 2666
	inode = mapping->host;
	if (S_ISBLK(inode->i_mode)) {
		struct block_device *bdev = I_BDEV(inode);
2667

2668
		set_blocksize(bdev, old_block_size);
2669
		blkdev_put(bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
L
Linus Torvalds 已提交
2670
	}
2671 2672 2673 2674

	inode_lock(inode);
	inode->i_flags &= ~S_SWAPFILE;
	inode_unlock(inode);
L
Linus Torvalds 已提交
2675
	filp_close(swap_file, NULL);
2676 2677 2678 2679 2680 2681 2682 2683 2684 2685

	/*
	 * Clear the SWP_USED flag after all resources are freed so that swapon
	 * can reuse this swap_info in alloc_swap_info() safely.  It is ok to
	 * not hold p->lock after we cleared its SWP_WRITEOK.
	 */
	spin_lock(&swap_lock);
	p->flags = 0;
	spin_unlock(&swap_lock);

L
Linus Torvalds 已提交
2686
	err = 0;
2687 2688
	atomic_inc(&proc_poll_event);
	wake_up_interruptible(&proc_poll_wait);
L
Linus Torvalds 已提交
2689 2690 2691 2692

out_dput:
	filp_close(victim, NULL);
out:
2693
	putname(pathname);
L
Linus Torvalds 已提交
2694 2695 2696 2697
	return err;
}

#ifdef CONFIG_PROC_FS
2698
static __poll_t swaps_poll(struct file *file, poll_table *wait)
2699
{
2700
	struct seq_file *seq = file->private_data;
2701 2702 2703

	poll_wait(file, &proc_poll_wait, wait);

2704 2705
	if (seq->poll_event != atomic_read(&proc_poll_event)) {
		seq->poll_event = atomic_read(&proc_poll_event);
2706
		return EPOLLIN | EPOLLRDNORM | EPOLLERR | EPOLLPRI;
2707 2708
	}

2709
	return EPOLLIN | EPOLLRDNORM;
2710 2711
}

L
Linus Torvalds 已提交
2712 2713 2714
/* iterator */
static void *swap_start(struct seq_file *swap, loff_t *pos)
{
2715 2716
	struct swap_info_struct *si;
	int type;
L
Linus Torvalds 已提交
2717 2718
	loff_t l = *pos;

2719
	mutex_lock(&swapon_mutex);
L
Linus Torvalds 已提交
2720

2721 2722 2723
	if (!l)
		return SEQ_START_TOKEN;

2724
	for (type = 0; (si = swap_type_to_swap_info(type)); type++) {
2725
		if (!(si->flags & SWP_USED) || !si->swap_map)
L
Linus Torvalds 已提交
2726
			continue;
2727
		if (!--l)
2728
			return si;
L
Linus Torvalds 已提交
2729 2730 2731 2732 2733 2734 2735
	}

	return NULL;
}

static void *swap_next(struct seq_file *swap, void *v, loff_t *pos)
{
2736 2737
	struct swap_info_struct *si = v;
	int type;
L
Linus Torvalds 已提交
2738

2739
	if (v == SEQ_START_TOKEN)
2740 2741 2742
		type = 0;
	else
		type = si->type + 1;
2743

2744
	++(*pos);
2745
	for (; (si = swap_type_to_swap_info(type)); type++) {
2746
		if (!(si->flags & SWP_USED) || !si->swap_map)
L
Linus Torvalds 已提交
2747
			continue;
2748
		return si;
L
Linus Torvalds 已提交
2749 2750 2751 2752 2753 2754 2755
	}

	return NULL;
}

static void swap_stop(struct seq_file *swap, void *v)
{
2756
	mutex_unlock(&swapon_mutex);
L
Linus Torvalds 已提交
2757 2758 2759 2760
}

static int swap_show(struct seq_file *swap, void *v)
{
2761
	struct swap_info_struct *si = v;
L
Linus Torvalds 已提交
2762 2763 2764
	struct file *file;
	int len;

2765
	if (si == SEQ_START_TOKEN) {
2766 2767 2768
		seq_puts(swap,"Filename\t\t\t\tType\t\tSize\tUsed\tPriority\n");
		return 0;
	}
L
Linus Torvalds 已提交
2769

2770
	file = si->swap_file;
2771
	len = seq_file_path(swap, file, " \t\n\\");
2772
	seq_printf(swap, "%*s%s\t%u\t%u\t%d\n",
2773
			len < 40 ? 40 - len : 1, " ",
A
Al Viro 已提交
2774
			S_ISBLK(file_inode(file)->i_mode) ?
L
Linus Torvalds 已提交
2775
				"partition" : "file\t",
2776 2777 2778
			si->pages << (PAGE_SHIFT - 10),
			si->inuse_pages << (PAGE_SHIFT - 10),
			si->prio);
L
Linus Torvalds 已提交
2779 2780 2781
	return 0;
}

2782
static const struct seq_operations swaps_op = {
L
Linus Torvalds 已提交
2783 2784 2785 2786 2787 2788 2789 2790
	.start =	swap_start,
	.next =		swap_next,
	.stop =		swap_stop,
	.show =		swap_show
};

static int swaps_open(struct inode *inode, struct file *file)
{
2791
	struct seq_file *seq;
2792 2793 2794
	int ret;

	ret = seq_open(file, &swaps_op);
2795
	if (ret)
2796 2797
		return ret;

2798 2799 2800
	seq = file->private_data;
	seq->poll_event = atomic_read(&proc_poll_event);
	return 0;
L
Linus Torvalds 已提交
2801 2802
}

2803
static const struct proc_ops swaps_proc_ops = {
2804
	.proc_flags	= PROC_ENTRY_PERMANENT,
2805 2806 2807 2808 2809
	.proc_open	= swaps_open,
	.proc_read	= seq_read,
	.proc_lseek	= seq_lseek,
	.proc_release	= seq_release,
	.proc_poll	= swaps_poll,
L
Linus Torvalds 已提交
2810 2811 2812 2813
};

static int __init procswaps_init(void)
{
2814
	proc_create("swaps", 0, NULL, &swaps_proc_ops);
L
Linus Torvalds 已提交
2815 2816 2817 2818 2819
	return 0;
}
__initcall(procswaps_init);
#endif /* CONFIG_PROC_FS */

2820 2821 2822 2823 2824 2825 2826 2827 2828
#ifdef MAX_SWAPFILES_CHECK
static int __init max_swapfiles_check(void)
{
	MAX_SWAPFILES_CHECK();
	return 0;
}
late_initcall(max_swapfiles_check);
#endif

2829
static struct swap_info_struct *alloc_swap_info(void)
L
Linus Torvalds 已提交
2830
{
2831
	struct swap_info_struct *p;
L
Linus Torvalds 已提交
2832
	unsigned int type;
2833
	int i;
2834

2835
	p = kvzalloc(struct_size(p, avail_lists, nr_node_ids), GFP_KERNEL);
2836
	if (!p)
2837
		return ERR_PTR(-ENOMEM);
2838

2839
	spin_lock(&swap_lock);
2840 2841
	for (type = 0; type < nr_swapfiles; type++) {
		if (!(swap_info[type]->flags & SWP_USED))
L
Linus Torvalds 已提交
2842
			break;
2843
	}
2844
	if (type >= MAX_SWAPFILES) {
2845
		spin_unlock(&swap_lock);
2846
		kvfree(p);
2847
		return ERR_PTR(-EPERM);
L
Linus Torvalds 已提交
2848
	}
2849 2850
	if (type >= nr_swapfiles) {
		p->type = type;
2851
		WRITE_ONCE(swap_info[type], p);
2852 2853 2854 2855 2856 2857
		/*
		 * Write swap_info[type] before nr_swapfiles, in case a
		 * racing procfs swap_start() or swap_next() is reading them.
		 * (We never shrink nr_swapfiles, we never free this entry.)
		 */
		smp_wmb();
2858
		WRITE_ONCE(nr_swapfiles, nr_swapfiles + 1);
2859
	} else {
2860
		kvfree(p);
2861 2862 2863 2864 2865 2866
		p = swap_info[type];
		/*
		 * Do not memset this entry: a racing procfs swap_next()
		 * would be relying on p->type to remain valid.
		 */
	}
2867
	p->swap_extent_root = RB_ROOT;
2868
	plist_node_init(&p->list, 0);
2869 2870
	for_each_node(i)
		plist_node_init(&p->avail_lists[i], 0);
L
Linus Torvalds 已提交
2871
	p->flags = SWP_USED;
2872
	spin_unlock(&swap_lock);
2873
	spin_lock_init(&p->lock);
2874
	spin_lock_init(&p->cont_lock);
2875

2876 2877 2878
	return p;
}

2879 2880 2881 2882 2883 2884 2885
static int claim_swapfile(struct swap_info_struct *p, struct inode *inode)
{
	int error;

	if (S_ISBLK(inode->i_mode)) {
		p->bdev = bdgrab(I_BDEV(inode));
		error = blkdev_get(p->bdev,
2886
				   FMODE_READ | FMODE_WRITE | FMODE_EXCL, p);
2887 2888
		if (error < 0) {
			p->bdev = NULL;
2889
			return error;
2890 2891 2892 2893
		}
		p->old_block_size = block_size(p->bdev);
		error = set_blocksize(p->bdev, PAGE_SIZE);
		if (error < 0)
2894
			return error;
2895 2896 2897 2898 2899 2900 2901
		/*
		 * Zoned block devices contain zones that have a sequential
		 * write only restriction.  Hence zoned block devices are not
		 * suitable for swapping.  Disallow them here.
		 */
		if (blk_queue_is_zoned(p->bdev->bd_queue))
			return -EINVAL;
2902 2903 2904
		p->flags |= SWP_BLKDEV;
	} else if (S_ISREG(inode->i_mode)) {
		p->bdev = inode->i_sb->s_bdev;
2905 2906
	}

2907 2908 2909
	return 0;
}

2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938

/*
 * Find out how many pages are allowed for a single swap device. There
 * are two limiting factors:
 * 1) the number of bits for the swap offset in the swp_entry_t type, and
 * 2) the number of bits in the swap pte, as defined by the different
 * architectures.
 *
 * In order to find the largest possible bit mask, a swap entry with
 * swap type 0 and swap offset ~0UL is created, encoded to a swap pte,
 * decoded to a swp_entry_t again, and finally the swap offset is
 * extracted.
 *
 * This will mask all the bits from the initial ~0UL mask that can't
 * be encoded in either the swp_entry_t or the architecture definition
 * of a swap pte.
 */
unsigned long generic_max_swapfile_size(void)
{
	return swp_offset(pte_to_swp_entry(
			swp_entry_to_pte(swp_entry(0, ~0UL)))) + 1;
}

/* Can be overridden by an architecture for additional checks. */
__weak unsigned long max_swapfile_size(void)
{
	return generic_max_swapfile_size();
}

2939 2940 2941 2942 2943 2944 2945
static unsigned long read_swap_header(struct swap_info_struct *p,
					union swap_header *swap_header,
					struct inode *inode)
{
	int i;
	unsigned long maxpages;
	unsigned long swapfilepages;
2946
	unsigned long last_page;
2947 2948

	if (memcmp("SWAPSPACE2", swap_header->magic.magic, 10)) {
2949
		pr_err("Unable to find swap-space signature\n");
2950
		return 0;
2951 2952 2953 2954 2955 2956 2957
	}

	/* swap partition endianess hack... */
	if (swab32(swap_header->info.version) == 1) {
		swab32s(&swap_header->info.version);
		swab32s(&swap_header->info.last_page);
		swab32s(&swap_header->info.nr_badpages);
2958 2959
		if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES)
			return 0;
2960 2961 2962 2963 2964
		for (i = 0; i < swap_header->info.nr_badpages; i++)
			swab32s(&swap_header->info.badpages[i]);
	}
	/* Check the swap header's sub-version */
	if (swap_header->info.version != 1) {
2965 2966
		pr_warn("Unable to handle swap header version %d\n",
			swap_header->info.version);
2967
		return 0;
2968 2969 2970 2971 2972 2973
	}

	p->lowest_bit  = 1;
	p->cluster_next = 1;
	p->cluster_nr = 0;

2974
	maxpages = max_swapfile_size();
2975
	last_page = swap_header->info.last_page;
2976 2977 2978 2979
	if (!last_page) {
		pr_warn("Empty swap-file\n");
		return 0;
	}
2980
	if (last_page > maxpages) {
2981
		pr_warn("Truncating oversized swap area, only using %luk out of %luk\n",
2982 2983 2984 2985 2986
			maxpages << (PAGE_SHIFT - 10),
			last_page << (PAGE_SHIFT - 10));
	}
	if (maxpages > last_page) {
		maxpages = last_page + 1;
2987 2988 2989 2990 2991 2992 2993
		/* p->max is an unsigned int: don't overflow it */
		if ((unsigned int)maxpages == 0)
			maxpages = UINT_MAX;
	}
	p->highest_bit = maxpages - 1;

	if (!maxpages)
2994
		return 0;
2995 2996
	swapfilepages = i_size_read(inode) >> PAGE_SHIFT;
	if (swapfilepages && maxpages > swapfilepages) {
2997
		pr_warn("Swap area shorter than signature indicates\n");
2998
		return 0;
2999 3000
	}
	if (swap_header->info.nr_badpages && S_ISREG(inode->i_mode))
3001
		return 0;
3002
	if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES)
3003
		return 0;
3004 3005 3006 3007

	return maxpages;
}

3008
#define SWAP_CLUSTER_INFO_COLS						\
H
Huang, Ying 已提交
3009
	DIV_ROUND_UP(L1_CACHE_BYTES, sizeof(struct swap_cluster_info))
3010 3011 3012 3013
#define SWAP_CLUSTER_SPACE_COLS						\
	DIV_ROUND_UP(SWAP_ADDRESS_SPACE_PAGES, SWAPFILE_CLUSTER)
#define SWAP_CLUSTER_COLS						\
	max_t(unsigned int, SWAP_CLUSTER_INFO_COLS, SWAP_CLUSTER_SPACE_COLS)
H
Huang, Ying 已提交
3014

3015 3016 3017
static int setup_swap_map_and_extents(struct swap_info_struct *p,
					union swap_header *swap_header,
					unsigned char *swap_map,
3018
					struct swap_cluster_info *cluster_info,
3019 3020 3021
					unsigned long maxpages,
					sector_t *span)
{
H
Huang, Ying 已提交
3022
	unsigned int j, k;
3023 3024
	unsigned int nr_good_pages;
	int nr_extents;
3025
	unsigned long nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
H
Huang, Ying 已提交
3026 3027
	unsigned long col = p->cluster_next / SWAPFILE_CLUSTER % SWAP_CLUSTER_COLS;
	unsigned long i, idx;
3028 3029 3030

	nr_good_pages = maxpages - 1;	/* omit header page */

3031 3032
	cluster_list_init(&p->free_clusters);
	cluster_list_init(&p->discard_clusters);
3033

3034 3035
	for (i = 0; i < swap_header->info.nr_badpages; i++) {
		unsigned int page_nr = swap_header->info.badpages[i];
3036 3037
		if (page_nr == 0 || page_nr > swap_header->info.last_page)
			return -EINVAL;
3038 3039 3040
		if (page_nr < maxpages) {
			swap_map[page_nr] = SWAP_MAP_BAD;
			nr_good_pages--;
3041 3042 3043 3044 3045
			/*
			 * Haven't marked the cluster free yet, no list
			 * operation involved
			 */
			inc_cluster_info_page(p, cluster_info, page_nr);
3046 3047 3048
		}
	}

3049 3050 3051 3052
	/* Haven't marked the cluster free yet, no list operation involved */
	for (i = maxpages; i < round_up(maxpages, SWAPFILE_CLUSTER); i++)
		inc_cluster_info_page(p, cluster_info, i);

3053 3054
	if (nr_good_pages) {
		swap_map[0] = SWAP_MAP_BAD;
3055 3056 3057 3058 3059
		/*
		 * Not mark the cluster free yet, no list
		 * operation involved
		 */
		inc_cluster_info_page(p, cluster_info, 0);
3060 3061 3062
		p->max = maxpages;
		p->pages = nr_good_pages;
		nr_extents = setup_swap_extents(p, span);
3063 3064
		if (nr_extents < 0)
			return nr_extents;
3065 3066 3067
		nr_good_pages = p->pages;
	}
	if (!nr_good_pages) {
3068
		pr_warn("Empty swap-file\n");
3069
		return -EINVAL;
3070 3071
	}

3072 3073 3074
	if (!cluster_info)
		return nr_extents;

H
Huang, Ying 已提交
3075

3076 3077 3078 3079
	/*
	 * Reduce false cache line sharing between cluster_info and
	 * sharing same address space.
	 */
H
Huang, Ying 已提交
3080 3081 3082 3083 3084 3085 3086 3087
	for (k = 0; k < SWAP_CLUSTER_COLS; k++) {
		j = (k + col) % SWAP_CLUSTER_COLS;
		for (i = 0; i < DIV_ROUND_UP(nr_clusters, SWAP_CLUSTER_COLS); i++) {
			idx = i * SWAP_CLUSTER_COLS + j;
			if (idx >= nr_clusters)
				continue;
			if (cluster_count(&cluster_info[idx]))
				continue;
3088
			cluster_set_flag(&cluster_info[idx], CLUSTER_FLAG_FREE);
3089 3090
			cluster_list_add_tail(&p->free_clusters, cluster_info,
					      idx);
3091 3092
		}
	}
3093 3094 3095
	return nr_extents;
}

3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109
/*
 * Helper to sys_swapon determining if a given swap
 * backing device queue supports DISCARD operations.
 */
static bool swap_discardable(struct swap_info_struct *si)
{
	struct request_queue *q = bdev_get_queue(si->bdev);

	if (!q || !blk_queue_discard(q))
		return false;

	return true;
}

3110 3111 3112
SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
{
	struct swap_info_struct *p;
3113
	struct filename *name;
3114 3115
	struct file *swap_file = NULL;
	struct address_space *mapping;
3116
	int prio;
3117 3118
	int error;
	union swap_header *swap_header;
3119
	int nr_extents;
3120 3121 3122
	sector_t span;
	unsigned long maxpages;
	unsigned char *swap_map = NULL;
3123
	struct swap_cluster_info *cluster_info = NULL;
3124
	unsigned long *frontswap_map = NULL;
3125 3126
	struct page *page = NULL;
	struct inode *inode = NULL;
3127
	bool inced_nr_rotate_swap = false;
3128

3129 3130 3131
	if (swap_flags & ~SWAP_FLAGS_VALID)
		return -EINVAL;

3132 3133 3134
	if (!capable(CAP_SYS_ADMIN))
		return -EPERM;

3135 3136 3137
	if (!swap_avail_heads)
		return -ENOMEM;

3138
	p = alloc_swap_info();
3139 3140
	if (IS_ERR(p))
		return PTR_ERR(p);
3141

3142 3143
	INIT_WORK(&p->discard_work, swap_discard_work);

L
Linus Torvalds 已提交
3144 3145
	name = getname(specialfile);
	if (IS_ERR(name)) {
3146
		error = PTR_ERR(name);
L
Linus Torvalds 已提交
3147
		name = NULL;
3148
		goto bad_swap;
L
Linus Torvalds 已提交
3149
	}
3150
	swap_file = file_open_name(name, O_RDWR|O_LARGEFILE, 0);
L
Linus Torvalds 已提交
3151
	if (IS_ERR(swap_file)) {
3152
		error = PTR_ERR(swap_file);
L
Linus Torvalds 已提交
3153
		swap_file = NULL;
3154
		goto bad_swap;
L
Linus Torvalds 已提交
3155 3156 3157 3158
	}

	p->swap_file = swap_file;
	mapping = swap_file->f_mapping;
3159
	inode = mapping->host;
3160

3161 3162
	error = claim_swapfile(p, inode);
	if (unlikely(error))
L
Linus Torvalds 已提交
3163 3164
		goto bad_swap;

3165 3166 3167 3168 3169 3170
	inode_lock(inode);
	if (IS_SWAPFILE(inode)) {
		error = -EBUSY;
		goto bad_swap_unlock_inode;
	}

L
Linus Torvalds 已提交
3171 3172 3173 3174 3175
	/*
	 * Read the swap header.
	 */
	if (!mapping->a_ops->readpage) {
		error = -EINVAL;
3176
		goto bad_swap_unlock_inode;
L
Linus Torvalds 已提交
3177
	}
3178
	page = read_mapping_page(mapping, 0, swap_file);
L
Linus Torvalds 已提交
3179 3180
	if (IS_ERR(page)) {
		error = PTR_ERR(page);
3181
		goto bad_swap_unlock_inode;
L
Linus Torvalds 已提交
3182
	}
3183
	swap_header = kmap(page);
L
Linus Torvalds 已提交
3184

3185 3186
	maxpages = read_swap_header(p, swap_header, inode);
	if (unlikely(!maxpages)) {
L
Linus Torvalds 已提交
3187
		error = -EINVAL;
3188
		goto bad_swap_unlock_inode;
L
Linus Torvalds 已提交
3189
	}
3190

3191
	/* OK, set up the swap map and apply the bad block list */
3192
	swap_map = vzalloc(maxpages);
3193 3194
	if (!swap_map) {
		error = -ENOMEM;
3195
		goto bad_swap_unlock_inode;
3196
	}
3197 3198 3199 3200

	if (bdi_cap_stable_pages_required(inode_to_bdi(inode)))
		p->flags |= SWP_STABLE_WRITES;

3201 3202 3203
	if (bdi_cap_synchronous_io(inode_to_bdi(inode)))
		p->flags |= SWP_SYNCHRONOUS_IO;

3204
	if (p->bdev && blk_queue_nonrot(bdev_get_queue(p->bdev))) {
3205
		int cpu;
H
Huang, Ying 已提交
3206
		unsigned long ci, nr_cluster;
3207

3208 3209 3210 3211 3212 3213
		p->flags |= SWP_SOLIDSTATE;
		/*
		 * select a random position to start with to help wear leveling
		 * SSD
		 */
		p->cluster_next = 1 + (prandom_u32() % p->highest_bit);
H
Huang, Ying 已提交
3214
		nr_cluster = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
3215

3216
		cluster_info = kvcalloc(nr_cluster, sizeof(*cluster_info),
3217
					GFP_KERNEL);
3218 3219
		if (!cluster_info) {
			error = -ENOMEM;
3220
			goto bad_swap_unlock_inode;
3221
		}
H
Huang, Ying 已提交
3222 3223 3224 3225

		for (ci = 0; ci < nr_cluster; ci++)
			spin_lock_init(&((cluster_info + ci)->lock));

3226 3227 3228
		p->percpu_cluster = alloc_percpu(struct percpu_cluster);
		if (!p->percpu_cluster) {
			error = -ENOMEM;
3229
			goto bad_swap_unlock_inode;
3230
		}
3231
		for_each_possible_cpu(cpu) {
3232
			struct percpu_cluster *cluster;
3233
			cluster = per_cpu_ptr(p->percpu_cluster, cpu);
3234 3235
			cluster_set_null(&cluster->index);
		}
3236
	} else {
3237
		atomic_inc(&nr_rotate_swap);
3238 3239
		inced_nr_rotate_swap = true;
	}
L
Linus Torvalds 已提交
3240

3241 3242
	error = swap_cgroup_swapon(p->type, maxpages);
	if (error)
3243
		goto bad_swap_unlock_inode;
3244

3245
	nr_extents = setup_swap_map_and_extents(p, swap_header, swap_map,
3246
		cluster_info, maxpages, &span);
3247 3248
	if (unlikely(nr_extents < 0)) {
		error = nr_extents;
3249
		goto bad_swap_unlock_inode;
L
Linus Torvalds 已提交
3250
	}
3251
	/* frontswap enabled? set up bit-per-page map for frontswap */
3252
	if (IS_ENABLED(CONFIG_FRONTSWAP))
3253 3254
		frontswap_map = kvcalloc(BITS_TO_LONGS(maxpages),
					 sizeof(long),
3255
					 GFP_KERNEL);
L
Linus Torvalds 已提交
3256

3257 3258 3259 3260 3261 3262 3263 3264 3265
	if (p->bdev &&(swap_flags & SWAP_FLAG_DISCARD) && swap_discardable(p)) {
		/*
		 * When discard is enabled for swap with no particular
		 * policy flagged, we set all swap discard flags here in
		 * order to sustain backward compatibility with older
		 * swapon(8) releases.
		 */
		p->flags |= (SWP_DISCARDABLE | SWP_AREA_DISCARD |
			     SWP_PAGE_DISCARD);
3266

3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283
		/*
		 * By flagging sys_swapon, a sysadmin can tell us to
		 * either do single-time area discards only, or to just
		 * perform discards for released swap page-clusters.
		 * Now it's time to adjust the p->flags accordingly.
		 */
		if (swap_flags & SWAP_FLAG_DISCARD_ONCE)
			p->flags &= ~SWP_PAGE_DISCARD;
		else if (swap_flags & SWAP_FLAG_DISCARD_PAGES)
			p->flags &= ~SWP_AREA_DISCARD;

		/* issue a swapon-time discard if it's still required */
		if (p->flags & SWP_AREA_DISCARD) {
			int err = discard_swap(p);
			if (unlikely(err))
				pr_err("swapon: discard_swap(%p): %d\n",
					p, err);
3284
		}
3285
	}
3286

3287 3288
	error = init_swap_address_space(p->type, maxpages);
	if (error)
3289
		goto bad_swap_unlock_inode;
3290

3291 3292 3293 3294 3295 3296 3297 3298
	/*
	 * Flush any pending IO and dirty mappings before we start using this
	 * swap device.
	 */
	inode->i_flags |= S_SWAPFILE;
	error = inode_drain_writes(inode);
	if (error) {
		inode->i_flags &= ~S_SWAPFILE;
3299
		goto bad_swap_unlock_inode;
3300 3301
	}

3302
	mutex_lock(&swapon_mutex);
3303
	prio = -1;
3304
	if (swap_flags & SWAP_FLAG_PREFER)
3305
		prio =
3306
		  (swap_flags & SWAP_FLAG_PRIO_MASK) >> SWAP_FLAG_PRIO_SHIFT;
3307
	enable_swap_info(p, prio, swap_map, cluster_info, frontswap_map);
3308

3309
	pr_info("Adding %uk swap on %s.  Priority:%d extents:%d across:%lluk %s%s%s%s%s\n",
3310
		p->pages<<(PAGE_SHIFT-10), name->name, p->prio,
3311 3312
		nr_extents, (unsigned long long)span<<(PAGE_SHIFT-10),
		(p->flags & SWP_SOLIDSTATE) ? "SS" : "",
3313
		(p->flags & SWP_DISCARDABLE) ? "D" : "",
3314 3315
		(p->flags & SWP_AREA_DISCARD) ? "s" : "",
		(p->flags & SWP_PAGE_DISCARD) ? "c" : "",
3316
		(frontswap_map) ? "FS" : "");
3317

3318
	mutex_unlock(&swapon_mutex);
3319 3320 3321
	atomic_inc(&proc_poll_event);
	wake_up_interruptible(&proc_poll_wait);

L
Linus Torvalds 已提交
3322 3323
	error = 0;
	goto out;
3324 3325
bad_swap_unlock_inode:
	inode_unlock(inode);
L
Linus Torvalds 已提交
3326
bad_swap:
3327 3328
	free_percpu(p->percpu_cluster);
	p->percpu_cluster = NULL;
3329
	if (inode && S_ISBLK(inode->i_mode) && p->bdev) {
3330 3331
		set_blocksize(p->bdev, p->old_block_size);
		blkdev_put(p->bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
L
Linus Torvalds 已提交
3332
	}
3333
	inode = NULL;
3334
	destroy_swap_extents(p);
3335
	swap_cgroup_swapoff(p->type);
3336
	spin_lock(&swap_lock);
L
Linus Torvalds 已提交
3337 3338
	p->swap_file = NULL;
	p->flags = 0;
3339
	spin_unlock(&swap_lock);
L
Linus Torvalds 已提交
3340
	vfree(swap_map);
3341
	kvfree(cluster_info);
3342
	kvfree(frontswap_map);
3343 3344
	if (inced_nr_rotate_swap)
		atomic_dec(&nr_rotate_swap);
3345
	if (swap_file)
L
Linus Torvalds 已提交
3346 3347 3348 3349
		filp_close(swap_file, NULL);
out:
	if (page && !IS_ERR(page)) {
		kunmap(page);
3350
		put_page(page);
L
Linus Torvalds 已提交
3351 3352 3353
	}
	if (name)
		putname(name);
3354
	if (inode)
A
Al Viro 已提交
3355
		inode_unlock(inode);
3356 3357
	if (!error)
		enable_swap_slots_cache();
L
Linus Torvalds 已提交
3358 3359 3360 3361 3362
	return error;
}

void si_swapinfo(struct sysinfo *val)
{
3363
	unsigned int type;
L
Linus Torvalds 已提交
3364 3365
	unsigned long nr_to_be_unused = 0;

3366
	spin_lock(&swap_lock);
3367 3368 3369 3370 3371
	for (type = 0; type < nr_swapfiles; type++) {
		struct swap_info_struct *si = swap_info[type];

		if ((si->flags & SWP_USED) && !(si->flags & SWP_WRITEOK))
			nr_to_be_unused += si->inuse_pages;
L
Linus Torvalds 已提交
3372
	}
3373
	val->freeswap = atomic_long_read(&nr_swap_pages) + nr_to_be_unused;
L
Linus Torvalds 已提交
3374
	val->totalswap = total_swap_pages + nr_to_be_unused;
3375
	spin_unlock(&swap_lock);
L
Linus Torvalds 已提交
3376 3377 3378 3379 3380
}

/*
 * Verify that a swap entry is valid and increment its swap map count.
 *
3381 3382 3383 3384 3385 3386
 * Returns error code in following case.
 * - success -> 0
 * - swp_entry is invalid -> EINVAL
 * - swp_entry is migration entry -> EINVAL
 * - swap-cache reference is requested but there is already one. -> EEXIST
 * - swap-cache reference is requested but the entry is not used. -> ENOENT
3387
 * - swap-mapped reference requested but needs continued swap count. -> ENOMEM
L
Linus Torvalds 已提交
3388
 */
3389
static int __swap_duplicate(swp_entry_t entry, unsigned char usage)
L
Linus Torvalds 已提交
3390
{
3391
	struct swap_info_struct *p;
H
Huang, Ying 已提交
3392
	struct swap_cluster_info *ci;
3393
	unsigned long offset;
3394 3395
	unsigned char count;
	unsigned char has_cache;
3396
	int err = -EINVAL;
L
Linus Torvalds 已提交
3397

3398
	p = get_swap_device(entry);
3399
	if (!p)
H
Huang, Ying 已提交
3400 3401
		goto out;

3402
	offset = swp_offset(entry);
H
Huang, Ying 已提交
3403
	ci = lock_cluster_or_swap_info(p, offset);
3404

3405
	count = p->swap_map[offset];
3406 3407 3408 3409 3410 3411 3412 3413 3414 3415

	/*
	 * swapin_readahead() doesn't check if a swap entry is valid, so the
	 * swap entry could be SWAP_MAP_BAD. Check here with lock held.
	 */
	if (unlikely(swap_count(count) == SWAP_MAP_BAD)) {
		err = -ENOENT;
		goto unlock_out;
	}

3416 3417 3418
	has_cache = count & SWAP_HAS_CACHE;
	count &= ~SWAP_HAS_CACHE;
	err = 0;
3419

3420
	if (usage == SWAP_HAS_CACHE) {
3421 3422

		/* set SWAP_HAS_CACHE if there is no cache and entry is used */
3423 3424 3425 3426 3427 3428
		if (!has_cache && count)
			has_cache = SWAP_HAS_CACHE;
		else if (has_cache)		/* someone else added cache */
			err = -EEXIST;
		else				/* no users remaining */
			err = -ENOENT;
3429 3430

	} else if (count || has_cache) {
3431

3432 3433 3434
		if ((count & ~COUNT_CONTINUED) < SWAP_MAP_MAX)
			count += usage;
		else if ((count & ~COUNT_CONTINUED) > SWAP_MAP_MAX)
3435
			err = -EINVAL;
3436 3437 3438 3439
		else if (swap_count_continued(p, offset, count))
			count = COUNT_CONTINUED;
		else
			err = -ENOMEM;
3440
	} else
3441 3442 3443 3444
		err = -ENOENT;			/* unused swap entry */

	p->swap_map[offset] = count | has_cache;

3445
unlock_out:
H
Huang, Ying 已提交
3446
	unlock_cluster_or_swap_info(p, ci);
L
Linus Torvalds 已提交
3447
out:
3448 3449
	if (p)
		put_swap_device(p);
3450
	return err;
L
Linus Torvalds 已提交
3451
}
3452

3453 3454 3455 3456 3457 3458 3459 3460 3461
/*
 * Help swapoff by noting that swap entry belongs to shmem/tmpfs
 * (in which case its reference count is never incremented).
 */
void swap_shmem_alloc(swp_entry_t entry)
{
	__swap_duplicate(entry, SWAP_MAP_SHMEM);
}

3462
/*
3463 3464 3465 3466 3467
 * Increase reference count of swap entry by 1.
 * Returns 0 for success, or -ENOMEM if a swap_count_continuation is required
 * but could not be atomically allocated.  Returns 0, just as if it succeeded,
 * if __swap_duplicate() fails for another reason (-EINVAL or -ENOENT), which
 * might occur if a page table entry has got corrupted.
3468
 */
3469
int swap_duplicate(swp_entry_t entry)
3470
{
3471 3472 3473 3474 3475
	int err = 0;

	while (!err && __swap_duplicate(entry, 1) == -ENOMEM)
		err = add_swap_count_continuation(entry, GFP_ATOMIC);
	return err;
3476
}
L
Linus Torvalds 已提交
3477

3478
/*
3479 3480
 * @entry: swap entry for which we allocate swap cache.
 *
3481
 * Called when allocating swap cache for existing swap entry,
3482
 * This can return error codes. Returns 0 at success.
3483
 * -EEXIST means there is a swap cache.
3484
 * Note: return code is different from swap_duplicate().
3485 3486 3487
 */
int swapcache_prepare(swp_entry_t entry)
{
3488
	return __swap_duplicate(entry, SWAP_HAS_CACHE);
3489 3490
}

3491 3492
struct swap_info_struct *swp_swap_info(swp_entry_t entry)
{
3493
	return swap_type_to_swap_info(swp_type(entry));
3494 3495
}

3496 3497
struct swap_info_struct *page_swap_info(struct page *page)
{
3498 3499
	swp_entry_t entry = { .val = page_private(page) };
	return swp_swap_info(entry);
3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517
}

/*
 * out-of-line __page_file_ methods to avoid include hell.
 */
struct address_space *__page_file_mapping(struct page *page)
{
	return page_swap_info(page)->swap_file->f_mapping;
}
EXPORT_SYMBOL_GPL(__page_file_mapping);

pgoff_t __page_file_index(struct page *page)
{
	swp_entry_t swap = { .val = page_private(page) };
	return swp_offset(swap);
}
EXPORT_SYMBOL_GPL(__page_file_index);

3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535
/*
 * add_swap_count_continuation - called when a swap count is duplicated
 * beyond SWAP_MAP_MAX, it allocates a new page and links that to the entry's
 * page of the original vmalloc'ed swap_map, to hold the continuation count
 * (for that entry and for its neighbouring PAGE_SIZE swap entries).  Called
 * again when count is duplicated beyond SWAP_MAP_MAX * SWAP_CONT_MAX, etc.
 *
 * These continuation pages are seldom referenced: the common paths all work
 * on the original swap_map, only referring to a continuation page when the
 * low "digit" of a count is incremented or decremented through SWAP_MAP_MAX.
 *
 * add_swap_count_continuation(, GFP_ATOMIC) can be called while holding
 * page table locks; if it fails, add_swap_count_continuation(, GFP_KERNEL)
 * can be called after dropping locks.
 */
int add_swap_count_continuation(swp_entry_t entry, gfp_t gfp_mask)
{
	struct swap_info_struct *si;
H
Huang, Ying 已提交
3536
	struct swap_cluster_info *ci;
3537 3538 3539 3540 3541
	struct page *head;
	struct page *page;
	struct page *list_page;
	pgoff_t offset;
	unsigned char count;
3542
	int ret = 0;
3543 3544 3545 3546 3547 3548 3549

	/*
	 * When debugging, it's easier to use __GFP_ZERO here; but it's better
	 * for latency not to zero a page while GFP_ATOMIC and holding locks.
	 */
	page = alloc_page(gfp_mask | __GFP_HIGHMEM);

3550
	si = get_swap_device(entry);
3551 3552 3553
	if (!si) {
		/*
		 * An acceptable race has occurred since the failing
3554
		 * __swap_duplicate(): the swap device may be swapoff
3555 3556 3557
		 */
		goto outer;
	}
3558
	spin_lock(&si->lock);
3559 3560

	offset = swp_offset(entry);
H
Huang, Ying 已提交
3561 3562 3563

	ci = lock_cluster(si, offset);

3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575
	count = si->swap_map[offset] & ~SWAP_HAS_CACHE;

	if ((count & ~COUNT_CONTINUED) != SWAP_MAP_MAX) {
		/*
		 * The higher the swap count, the more likely it is that tasks
		 * will race to add swap count continuation: we need to avoid
		 * over-provisioning.
		 */
		goto out;
	}

	if (!page) {
3576 3577
		ret = -ENOMEM;
		goto out;
3578 3579 3580 3581
	}

	/*
	 * We are fortunate that although vmalloc_to_page uses pte_offset_map,
3582 3583
	 * no architecture is using highmem pages for kernel page tables: so it
	 * will not corrupt the GFP_ATOMIC caller's atomic page table kmaps.
3584 3585 3586 3587
	 */
	head = vmalloc_to_page(si->swap_map + offset);
	offset &= ~PAGE_MASK;

3588
	spin_lock(&si->cont_lock);
3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607
	/*
	 * Page allocation does not initialize the page's lru field,
	 * but it does always reset its private field.
	 */
	if (!page_private(head)) {
		BUG_ON(count & COUNT_CONTINUED);
		INIT_LIST_HEAD(&head->lru);
		set_page_private(head, SWP_CONTINUED);
		si->flags |= SWP_CONTINUED;
	}

	list_for_each_entry(list_page, &head->lru, lru) {
		unsigned char *map;

		/*
		 * If the previous map said no continuation, but we've found
		 * a continuation page, free our allocation and use this one.
		 */
		if (!(count & COUNT_CONTINUED))
3608
			goto out_unlock_cont;
3609

3610
		map = kmap_atomic(list_page) + offset;
3611
		count = *map;
3612
		kunmap_atomic(map);
3613 3614 3615 3616 3617 3618

		/*
		 * If this continuation count now has some space in it,
		 * free our allocation and use this one.
		 */
		if ((count & ~COUNT_CONTINUED) != SWAP_CONT_MAX)
3619
			goto out_unlock_cont;
3620 3621 3622 3623
	}

	list_add_tail(&page->lru, &head->lru);
	page = NULL;			/* now it's attached, don't free it */
3624 3625
out_unlock_cont:
	spin_unlock(&si->cont_lock);
3626
out:
H
Huang, Ying 已提交
3627
	unlock_cluster(ci);
3628
	spin_unlock(&si->lock);
3629
	put_swap_device(si);
3630 3631 3632
outer:
	if (page)
		__free_page(page);
3633
	return ret;
3634 3635 3636 3637 3638 3639 3640 3641
}

/*
 * swap_count_continued - when the original swap_map count is incremented
 * from SWAP_MAP_MAX, check if there is already a continuation page to carry
 * into, carry if so, or else fail until a new continuation page is allocated;
 * when the original swap_map count is decremented from 0 with continuation,
 * borrow from the continuation and report whether it still holds more.
H
Huang, Ying 已提交
3642 3643
 * Called while __swap_duplicate() or swap_entry_free() holds swap or cluster
 * lock.
3644 3645 3646 3647 3648 3649 3650
 */
static bool swap_count_continued(struct swap_info_struct *si,
				 pgoff_t offset, unsigned char count)
{
	struct page *head;
	struct page *page;
	unsigned char *map;
3651
	bool ret;
3652 3653 3654 3655 3656 3657 3658

	head = vmalloc_to_page(si->swap_map + offset);
	if (page_private(head) != SWP_CONTINUED) {
		BUG_ON(count & COUNT_CONTINUED);
		return false;		/* need to add count continuation */
	}

3659
	spin_lock(&si->cont_lock);
3660
	offset &= ~PAGE_MASK;
3661
	page = list_next_entry(head, lru);
3662
	map = kmap_atomic(page) + offset;
3663 3664 3665 3666 3667 3668 3669 3670 3671

	if (count == SWAP_MAP_MAX)	/* initial increment from swap_map */
		goto init_map;		/* jump over SWAP_CONT_MAX checks */

	if (count == (SWAP_MAP_MAX | COUNT_CONTINUED)) { /* incrementing */
		/*
		 * Think of how you add 1 to 999
		 */
		while (*map == (SWAP_CONT_MAX | COUNT_CONTINUED)) {
3672
			kunmap_atomic(map);
3673
			page = list_next_entry(page, lru);
3674
			BUG_ON(page == head);
3675
			map = kmap_atomic(page) + offset;
3676 3677
		}
		if (*map == SWAP_CONT_MAX) {
3678
			kunmap_atomic(map);
3679
			page = list_next_entry(page, lru);
3680 3681 3682 3683
			if (page == head) {
				ret = false;	/* add count continuation */
				goto out;
			}
3684
			map = kmap_atomic(page) + offset;
3685 3686 3687
init_map:		*map = 0;		/* we didn't zero the page */
		}
		*map += 1;
3688
		kunmap_atomic(map);
3689
		while ((page = list_prev_entry(page, lru)) != head) {
3690
			map = kmap_atomic(page) + offset;
3691
			*map = COUNT_CONTINUED;
3692
			kunmap_atomic(map);
3693
		}
3694
		ret = true;			/* incremented */
3695 3696 3697 3698 3699 3700 3701

	} else {				/* decrementing */
		/*
		 * Think of how you subtract 1 from 1000
		 */
		BUG_ON(count != COUNT_CONTINUED);
		while (*map == COUNT_CONTINUED) {
3702
			kunmap_atomic(map);
3703
			page = list_next_entry(page, lru);
3704
			BUG_ON(page == head);
3705
			map = kmap_atomic(page) + offset;
3706 3707 3708 3709 3710
		}
		BUG_ON(*map == 0);
		*map -= 1;
		if (*map == 0)
			count = 0;
3711
		kunmap_atomic(map);
3712
		while ((page = list_prev_entry(page, lru)) != head) {
3713
			map = kmap_atomic(page) + offset;
3714 3715
			*map = SWAP_CONT_MAX | count;
			count = COUNT_CONTINUED;
3716
			kunmap_atomic(map);
3717
		}
3718
		ret = count == COUNT_CONTINUED;
3719
	}
3720 3721 3722
out:
	spin_unlock(&si->cont_lock);
	return ret;
3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736
}

/*
 * free_swap_count_continuations - swapoff free all the continuation pages
 * appended to the swap_map, after swap_map is quiesced, before vfree'ing it.
 */
static void free_swap_count_continuations(struct swap_info_struct *si)
{
	pgoff_t offset;

	for (offset = 0; offset < si->max; offset += PAGE_SIZE) {
		struct page *head;
		head = vmalloc_to_page(si->swap_map + offset);
		if (page_private(head)) {
3737 3738 3739 3740
			struct page *page, *next;

			list_for_each_entry_safe(page, next, &head->lru, lru) {
				list_del(&page->lru);
3741 3742 3743 3744 3745
				__free_page(page);
			}
		}
	}
}
3746

3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777
#if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)
void mem_cgroup_throttle_swaprate(struct mem_cgroup *memcg, int node,
				  gfp_t gfp_mask)
{
	struct swap_info_struct *si, *next;
	if (!(gfp_mask & __GFP_IO) || !memcg)
		return;

	if (!blk_cgroup_congested())
		return;

	/*
	 * We've already scheduled a throttle, avoid taking the global swap
	 * lock.
	 */
	if (current->throttle_queue)
		return;

	spin_lock(&swap_avail_lock);
	plist_for_each_entry_safe(si, next, &swap_avail_heads[node],
				  avail_lists[node]) {
		if (si->bdev) {
			blkcg_schedule_throttle(bdev_get_queue(si->bdev),
						true);
			break;
		}
	}
	spin_unlock(&swap_avail_lock);
}
#endif

3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794
static int __init swapfile_init(void)
{
	int nid;

	swap_avail_heads = kmalloc_array(nr_node_ids, sizeof(struct plist_head),
					 GFP_KERNEL);
	if (!swap_avail_heads) {
		pr_emerg("Not enough memory for swap heads, swap is disabled\n");
		return -ENOMEM;
	}

	for_each_node(nid)
		plist_head_init(&swap_avail_heads[nid]);

	return 0;
}
subsys_initcall(swapfile_init);
新手
引导
客服 返回
顶部