virtio_balloon.c 19.4 KB
Newer Older
S
Sasha Levin 已提交
1 2
/*
 * Virtio balloon implementation, inspired by Dor Laor and Marcelo
R
Rusty Russell 已提交
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
 * Tosatti's implementations.
 *
 *  Copyright 2008 Rusty Russell IBM Corporation
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */
S
Sasha Levin 已提交
21

R
Rusty Russell 已提交
22 23 24
#include <linux/virtio.h>
#include <linux/virtio_balloon.h>
#include <linux/swap.h>
25
#include <linux/workqueue.h>
26
#include <linux/delay.h>
27
#include <linux/slab.h>
28
#include <linux/module.h>
29
#include <linux/balloon_compaction.h>
30
#include <linux/oom.h>
31
#include <linux/wait.h>
32
#include <linux/mm.h>
33
#include <linux/mount.h>
34
#include <linux/magic.h>
R
Rusty Russell 已提交
35

36 37 38 39 40
/*
 * Balloon device works in 4K page units.  So each page is pointed to by
 * multiple balloon pages.  All memory counters in this driver are in balloon
 * page units.
 */
41 42
#define VIRTIO_BALLOON_PAGES_PER_PAGE (unsigned)(PAGE_SIZE >> VIRTIO_BALLOON_PFN_SHIFT)
#define VIRTIO_BALLOON_ARRAY_PFNS_MAX 256
43 44 45 46 47 48
#define OOM_VBALLOON_DEFAULT_PAGES 256
#define VIRTBALLOON_OOM_NOTIFY_PRIORITY 80

static int oom_pages = OOM_VBALLOON_DEFAULT_PAGES;
module_param(oom_pages, int, S_IRUSR | S_IWUSR);
MODULE_PARM_DESC(oom_pages, "pages to free on OOM");
49

50 51 52 53
#ifdef CONFIG_BALLOON_COMPACTION
static struct vfsmount *balloon_mnt;
#endif

54
struct virtio_balloon {
R
Rusty Russell 已提交
55
	struct virtio_device *vdev;
56
	struct virtqueue *inflate_vq, *deflate_vq, *stats_vq;
R
Rusty Russell 已提交
57

58
	/* The balloon servicing is delegated to a freezable workqueue. */
59 60
	struct work_struct update_balloon_stats_work;
	struct work_struct update_balloon_size_work;
R
Rusty Russell 已提交
61

62 63 64
	/* Prevent updating balloon when it is being canceled. */
	spinlock_t stop_update_lock;
	bool stop_update;
R
Rusty Russell 已提交
65 66

	/* Waiting for host to ack the pages we released. */
67
	wait_queue_head_t acked;
R
Rusty Russell 已提交
68

69
	/* Number of balloon pages we've told the Host we're not using. */
R
Rusty Russell 已提交
70
	unsigned int num_pages;
71
	/*
72 73
	 * The pages we've told the Host we're not using are enqueued
	 * at vb_dev_info->pages list.
74 75 76
	 * Each page on this list adds VIRTIO_BALLOON_PAGES_PER_PAGE
	 * to num_pages above.
	 */
77
	struct balloon_dev_info vb_dev_info;
78 79 80

	/* Synchronize access/update to this struct virtio_balloon elements */
	struct mutex balloon_lock;
R
Rusty Russell 已提交
81 82 83

	/* The array of pfns we tell the Host about. */
	unsigned int num_pfns;
84
	__virtio32 pfns[VIRTIO_BALLOON_ARRAY_PFNS_MAX];
85 86 87

	/* Memory statistics */
	struct virtio_balloon_stat stats[VIRTIO_BALLOON_S_NR];
88 89 90

	/* To register callback in oom notifier call chain */
	struct notifier_block nb;
R
Rusty Russell 已提交
91 92 93 94 95 96 97
};

static struct virtio_device_id id_table[] = {
	{ VIRTIO_ID_BALLOON, VIRTIO_DEV_ANY_ID },
	{ 0 },
};

98 99 100 101 102 103
static u32 page_to_balloon_pfn(struct page *page)
{
	unsigned long pfn = page_to_pfn(page);

	BUILD_BUG_ON(PAGE_SHIFT < VIRTIO_BALLOON_PFN_SHIFT);
	/* Convert pfn from Linux page size to balloon page size. */
104 105 106
	return pfn * VIRTIO_BALLOON_PAGES_PER_PAGE;
}

R
Rusty Russell 已提交
107 108
static void balloon_ack(struct virtqueue *vq)
{
109
	struct virtio_balloon *vb = vq->vdev->priv;
R
Rusty Russell 已提交
110

111
	wake_up(&vb->acked);
R
Rusty Russell 已提交
112 113 114 115 116
}

static void tell_host(struct virtio_balloon *vb, struct virtqueue *vq)
{
	struct scatterlist sg;
117
	unsigned int len;
R
Rusty Russell 已提交
118 119 120 121

	sg_init_one(&sg, vb->pfns, sizeof(vb->pfns[0]) * vb->num_pfns);

	/* We should always be able to add one buffer to an empty queue. */
122
	virtqueue_add_outbuf(vq, &sg, 1, vb, GFP_KERNEL);
123
	virtqueue_kick(vq);
R
Rusty Russell 已提交
124 125

	/* When host has read buffer, this completes via balloon_ack */
126
	wait_event(vb->acked, virtqueue_get_buf(vq, &len));
127

R
Rusty Russell 已提交
128 129
}

130 131
static void set_page_pfns(struct virtio_balloon *vb,
			  __virtio32 pfns[], struct page *page)
132 133 134
{
	unsigned int i;

W
Wei Wang 已提交
135 136 137 138
	/*
	 * Set balloon pfns pointing at this page.
	 * Note that the first pfn points at start of the page.
	 */
139
	for (i = 0; i < VIRTIO_BALLOON_PAGES_PER_PAGE; i++)
140 141
		pfns[i] = cpu_to_virtio32(vb->vdev,
					  page_to_balloon_pfn(page) + i);
142 143
}

144
static unsigned fill_balloon(struct virtio_balloon *vb, size_t num)
R
Rusty Russell 已提交
145
{
146
	unsigned num_allocated_pages;
147 148 149
	unsigned num_pfns;
	struct page *page;
	LIST_HEAD(pages);
150

R
Rusty Russell 已提交
151 152 153
	/* We can only do one array worth at a time. */
	num = min(num, ARRAY_SIZE(vb->pfns));

154 155 156
	for (num_pfns = 0; num_pfns < num;
	     num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) {
		struct page *page = balloon_page_alloc();
157

R
Rusty Russell 已提交
158
		if (!page) {
159
			dev_info_ratelimited(&vb->vdev->dev,
160 161
					     "Out of puff! Can't get %u pages\n",
					     VIRTIO_BALLOON_PAGES_PER_PAGE);
R
Rusty Russell 已提交
162 163 164 165
			/* Sleep for at least 1/5 of a second before retry. */
			msleep(200);
			break;
		}
166 167 168 169 170 171 172 173 174 175 176

		balloon_page_push(&pages, page);
	}

	mutex_lock(&vb->balloon_lock);

	vb->num_pfns = 0;

	while ((page = balloon_page_pop(&pages))) {
		balloon_page_enqueue(&vb->vb_dev_info, page);

177
		set_page_pfns(vb, vb->pfns + vb->num_pfns, page);
178
		vb->num_pages += VIRTIO_BALLOON_PAGES_PER_PAGE;
179 180 181
		if (!virtio_has_feature(vb->vdev,
					VIRTIO_BALLOON_F_DEFLATE_ON_OOM))
			adjust_managed_page_count(page, -1);
182
		vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE;
R
Rusty Russell 已提交
183 184
	}

185
	num_allocated_pages = vb->num_pfns;
186 187 188 189
	/* Did we get any? */
	if (vb->num_pfns != 0)
		tell_host(vb, vb->inflate_vq);
	mutex_unlock(&vb->balloon_lock);
190 191

	return num_allocated_pages;
R
Rusty Russell 已提交
192 193
}

194 195
static void release_pages_balloon(struct virtio_balloon *vb,
				 struct list_head *pages)
R
Rusty Russell 已提交
196
{
197
	struct page *page, *next;
R
Rusty Russell 已提交
198

199
	list_for_each_entry_safe(page, next, pages, lru) {
200 201 202
		if (!virtio_has_feature(vb->vdev,
					VIRTIO_BALLOON_F_DEFLATE_ON_OOM))
			adjust_managed_page_count(page, 1);
203
		list_del(&page->lru);
204
		put_page(page); /* balloon reference */
R
Rusty Russell 已提交
205 206 207
	}
}

208
static unsigned leak_balloon(struct virtio_balloon *vb, size_t num)
R
Rusty Russell 已提交
209
{
210
	unsigned num_freed_pages;
R
Rusty Russell 已提交
211
	struct page *page;
212
	struct balloon_dev_info *vb_dev_info = &vb->vb_dev_info;
213
	LIST_HEAD(pages);
R
Rusty Russell 已提交
214 215 216 217

	/* We can only do one array worth at a time. */
	num = min(num, ARRAY_SIZE(vb->pfns));

218
	mutex_lock(&vb->balloon_lock);
219 220
	/* We can't release more pages than taken */
	num = min(num, (size_t)vb->num_pages);
221 222
	for (vb->num_pfns = 0; vb->num_pfns < num;
	     vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) {
223 224 225
		page = balloon_page_dequeue(vb_dev_info);
		if (!page)
			break;
226
		set_page_pfns(vb, vb->pfns + vb->num_pfns, page);
227
		list_add(&page->lru, &pages);
228
		vb->num_pages -= VIRTIO_BALLOON_PAGES_PER_PAGE;
R
Rusty Russell 已提交
229 230
	}

231
	num_freed_pages = vb->num_pfns;
232 233 234 235 236
	/*
	 * Note that if
	 * virtio_has_feature(vdev, VIRTIO_BALLOON_F_MUST_TELL_HOST);
	 * is true, we *have* to do it in this order
	 */
237 238
	if (vb->num_pfns != 0)
		tell_host(vb, vb->deflate_vq);
239
	release_pages_balloon(vb, &pages);
240
	mutex_unlock(&vb->balloon_lock);
241
	return num_freed_pages;
R
Rusty Russell 已提交
242 243
}

244 245 246 247
static inline void update_stat(struct virtio_balloon *vb, int idx,
			       u16 tag, u64 val)
{
	BUG_ON(idx >= VIRTIO_BALLOON_S_NR);
248 249
	vb->stats[idx].tag = cpu_to_virtio16(vb->vdev, tag);
	vb->stats[idx].val = cpu_to_virtio64(vb->vdev, val);
250 251 252 253
}

#define pages_to_bytes(x) ((u64)(x) << PAGE_SHIFT)

254
static unsigned int update_balloon_stats(struct virtio_balloon *vb)
255 256 257
{
	unsigned long events[NR_VM_EVENT_ITEMS];
	struct sysinfo i;
258
	unsigned int idx = 0;
259
	long available;
260 261 262 263

	all_vm_events(events);
	si_meminfo(&i);

264 265
	available = si_mem_available();

266
#ifdef CONFIG_VM_EVENT_COUNTERS
267 268 269 270 271 272
	update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_IN,
				pages_to_bytes(events[PSWPIN]));
	update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_OUT,
				pages_to_bytes(events[PSWPOUT]));
	update_stat(vb, idx++, VIRTIO_BALLOON_S_MAJFLT, events[PGMAJFAULT]);
	update_stat(vb, idx++, VIRTIO_BALLOON_S_MINFLT, events[PGFAULT]);
273
#endif
274 275 276 277
	update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMFREE,
				pages_to_bytes(i.freeram));
	update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMTOT,
				pages_to_bytes(i.totalram));
278 279
	update_stat(vb, idx++, VIRTIO_BALLOON_S_AVAIL,
				pages_to_bytes(available));
280 281

	return idx;
282 283 284 285 286 287 288
}

/*
 * While most virtqueues communicate guest-initiated requests to the hypervisor,
 * the stats queue operates in reverse.  The driver initializes the virtqueue
 * with a single buffer.  From that point forward, all conversations consist of
 * a hypervisor request (a call to this function) which directs us to refill
289
 * the virtqueue with a fresh stats buffer.  Since stats collection can sleep,
290 291
 * we delegate the job to a freezable workqueue that will do the actual work via
 * stats_handle_request().
292
 */
293
static void stats_request(struct virtqueue *vq)
294
{
295
	struct virtio_balloon *vb = vq->vdev->priv;
296

297 298
	spin_lock(&vb->stop_update_lock);
	if (!vb->stop_update)
299
		queue_work(system_freezable_wq, &vb->update_balloon_stats_work);
300
	spin_unlock(&vb->stop_update_lock);
301 302 303 304 305 306
}

static void stats_handle_request(struct virtio_balloon *vb)
{
	struct virtqueue *vq;
	struct scatterlist sg;
307
	unsigned int len, num_stats;
308

309
	num_stats = update_balloon_stats(vb);
310

311
	vq = vb->stats_vq;
312 313
	if (!virtqueue_get_buf(vq, &len))
		return;
314
	sg_init_one(&sg, vb->stats, sizeof(vb->stats[0]) * num_stats);
315
	virtqueue_add_outbuf(vq, &sg, 1, vb, GFP_KERNEL);
316
	virtqueue_kick(vq);
317 318
}

R
Rusty Russell 已提交
319 320 321
static void virtballoon_changed(struct virtio_device *vdev)
{
	struct virtio_balloon *vb = vdev->priv;
322
	unsigned long flags;
R
Rusty Russell 已提交
323

324 325
	spin_lock_irqsave(&vb->stop_update_lock, flags);
	if (!vb->stop_update)
326
		queue_work(system_freezable_wq, &vb->update_balloon_size_work);
327
	spin_unlock_irqrestore(&vb->stop_update_lock, flags);
R
Rusty Russell 已提交
328 329
}

330
static inline s64 towards_target(struct virtio_balloon *vb)
R
Rusty Russell 已提交
331
{
D
David Gibson 已提交
332
	s64 target;
333
	u32 num_pages;
D
David Gibson 已提交
334

335 336
	virtio_cread(vb->vdev, struct virtio_balloon_config, num_pages,
		     &num_pages);
337

338 339 340 341 342
	/* Legacy balloon config space is LE, unlike all other devices. */
	if (!virtio_has_feature(vb->vdev, VIRTIO_F_VERSION_1))
		num_pages = le32_to_cpu((__force __le32)num_pages);

	target = num_pages;
D
David Gibson 已提交
343
	return target - vb->num_pages;
R
Rusty Russell 已提交
344 345 346 347
}

static void update_balloon_size(struct virtio_balloon *vb)
{
348 349 350 351 352
	u32 actual = vb->num_pages;

	/* Legacy balloon config space is LE, unlike all other devices. */
	if (!virtio_has_feature(vb->vdev, VIRTIO_F_VERSION_1))
		actual = (__force u32)cpu_to_le32(actual);
R
Rusty Russell 已提交
353

354
	virtio_cwrite(vb->vdev, struct virtio_balloon_config, actual,
355
		      &actual);
R
Rusty Russell 已提交
356 357
}

358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389
/*
 * virtballoon_oom_notify - release pages when system is under severe
 *			    memory pressure (called from out_of_memory())
 * @self : notifier block struct
 * @dummy: not used
 * @parm : returned - number of freed pages
 *
 * The balancing of memory by use of the virtio balloon should not cause
 * the termination of processes while there are pages in the balloon.
 * If virtio balloon manages to release some memory, it will make the
 * system return and retry the allocation that forced the OOM killer
 * to run.
 */
static int virtballoon_oom_notify(struct notifier_block *self,
				  unsigned long dummy, void *parm)
{
	struct virtio_balloon *vb;
	unsigned long *freed;
	unsigned num_freed_pages;

	vb = container_of(self, struct virtio_balloon, nb);
	if (!virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM))
		return NOTIFY_OK;

	freed = parm;
	num_freed_pages = leak_balloon(vb, oom_pages);
	update_balloon_size(vb);
	*freed += num_freed_pages;

	return NOTIFY_OK;
}

390
static void update_balloon_stats_func(struct work_struct *work)
R
Rusty Russell 已提交
391
{
392
	struct virtio_balloon *vb;
393

394 395 396 397
	vb = container_of(work, struct virtio_balloon,
			  update_balloon_stats_work);
	stats_handle_request(vb);
}
398

399
static void update_balloon_size_func(struct work_struct *work)
R
Rusty Russell 已提交
400
{
401 402
	struct virtio_balloon *vb;
	s64 diff;
403

404 405
	vb = container_of(work, struct virtio_balloon,
			  update_balloon_size_work);
406
	diff = towards_target(vb);
407

408 409 410 411 412 413 414 415
	if (diff > 0)
		diff -= fill_balloon(vb, diff);
	else if (diff < 0)
		diff += leak_balloon(vb, -diff);
	update_balloon_size(vb);

	if (diff)
		queue_work(system_freezable_wq, work);
R
Rusty Russell 已提交
416 417
}

418
static int init_vqs(struct virtio_balloon *vb)
R
Rusty Russell 已提交
419
{
420
	struct virtqueue *vqs[3];
421
	vq_callback_t *callbacks[] = { balloon_ack, balloon_ack, stats_request };
422
	static const char * const names[] = { "inflate", "deflate", "stats" };
423
	int err, nvqs;
R
Rusty Russell 已提交
424

425 426 427 428
	/*
	 * We expect two virtqueues: inflate and deflate, and
	 * optionally stat.
	 */
429
	nvqs = virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ) ? 3 : 2;
M
Michael S. Tsirkin 已提交
430
	err = virtio_find_vqs(vb->vdev, nvqs, vqs, callbacks, names, NULL);
431
	if (err)
432
		return err;
R
Rusty Russell 已提交
433

434 435
	vb->inflate_vq = vqs[0];
	vb->deflate_vq = vqs[1];
436 437
	if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) {
		struct scatterlist sg;
438
		unsigned int num_stats;
439 440 441 442
		vb->stats_vq = vqs[2];

		/*
		 * Prime this virtqueue with one buffer so the hypervisor can
443
		 * use it to signal us later (it can't be broken yet!).
444
		 */
445
		num_stats = update_balloon_stats(vb);
446

447
		sg_init_one(&sg, vb->stats, sizeof(vb->stats[0]) * num_stats);
448
		if (virtqueue_add_outbuf(vb->stats_vq, &sg, 1, vb, GFP_KERNEL)
449
		    < 0)
450
			BUG();
451
		virtqueue_kick(vb->stats_vq);
452
	}
453 454 455
	return 0;
}

456 457 458 459
#ifdef CONFIG_BALLOON_COMPACTION
/*
 * virtballoon_migratepage - perform the balloon page migration on behalf of
 *			     a compation thread.     (called under page lock)
460
 * @vb_dev_info: the balloon device
461 462 463 464 465 466 467 468 469 470 471 472 473 474
 * @newpage: page that will replace the isolated page after migration finishes.
 * @page   : the isolated (old) page that is about to be migrated to newpage.
 * @mode   : compaction mode -- not used for balloon page migration.
 *
 * After a ballooned page gets isolated by compaction procedures, this is the
 * function that performs the page migration on behalf of a compaction thread
 * The page migration for virtio balloon is done in a simple swap fashion which
 * follows these two macro steps:
 *  1) insert newpage into vb->pages list and update the host about it;
 *  2) update the host about the old page removed from vb->pages list;
 *
 * This function preforms the balloon page migration task.
 * Called through balloon_mapping->a_ops->migratepage
 */
475
static int virtballoon_migratepage(struct balloon_dev_info *vb_dev_info,
476 477
		struct page *newpage, struct page *page, enum migrate_mode mode)
{
478 479
	struct virtio_balloon *vb = container_of(vb_dev_info,
			struct virtio_balloon, vb_dev_info);
480 481 482 483 484 485 486 487 488 489 490 491 492
	unsigned long flags;

	/*
	 * In order to avoid lock contention while migrating pages concurrently
	 * to leak_balloon() or fill_balloon() we just give up the balloon_lock
	 * this turn, as it is easier to retry the page migration later.
	 * This also prevents fill_balloon() getting stuck into a mutex
	 * recursion in the case it ends up triggering memory compaction
	 * while it is attempting to inflate the ballon.
	 */
	if (!mutex_trylock(&vb->balloon_lock))
		return -EAGAIN;

493 494
	get_page(newpage); /* balloon reference */

495 496
	/* balloon's page migration 1st step  -- inflate "newpage" */
	spin_lock_irqsave(&vb_dev_info->pages_lock, flags);
497
	balloon_page_insert(vb_dev_info, newpage);
498
	vb_dev_info->isolated_pages--;
499
	__count_vm_event(BALLOON_MIGRATE);
500 501
	spin_unlock_irqrestore(&vb_dev_info->pages_lock, flags);
	vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE;
502
	set_page_pfns(vb, vb->pfns, newpage);
503 504
	tell_host(vb, vb->inflate_vq);

505
	/* balloon's page migration 2nd step -- deflate "page" */
506 507
	balloon_page_delete(page);
	vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE;
508
	set_page_pfns(vb, vb->pfns, page);
509 510 511 512
	tell_host(vb, vb->deflate_vq);

	mutex_unlock(&vb->balloon_lock);

513 514
	put_page(page); /* balloon reference */

515
	return MIGRATEPAGE_SUCCESS;
516
}
517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534

static struct dentry *balloon_mount(struct file_system_type *fs_type,
		int flags, const char *dev_name, void *data)
{
	static const struct dentry_operations ops = {
		.d_dname = simple_dname,
	};

	return mount_pseudo(fs_type, "balloon-kvm:", NULL, &ops,
				BALLOON_KVM_MAGIC);
}

static struct file_system_type balloon_fs = {
	.name           = "balloon-kvm",
	.mount          = balloon_mount,
	.kill_sb        = kill_anon_super,
};

535 536
#endif /* CONFIG_BALLOON_COMPACTION */

537 538 539 540 541
static int virtballoon_probe(struct virtio_device *vdev)
{
	struct virtio_balloon *vb;
	int err;

542 543 544 545 546 547
	if (!vdev->config->get) {
		dev_err(&vdev->dev, "%s failure: config access disabled\n",
			__func__);
		return -EINVAL;
	}

548 549 550 551 552 553
	vdev->priv = vb = kmalloc(sizeof(*vb), GFP_KERNEL);
	if (!vb) {
		err = -ENOMEM;
		goto out;
	}

554 555
	INIT_WORK(&vb->update_balloon_stats_work, update_balloon_stats_func);
	INIT_WORK(&vb->update_balloon_size_work, update_balloon_size_func);
556 557
	spin_lock_init(&vb->stop_update_lock);
	vb->stop_update = false;
558
	vb->num_pages = 0;
559
	mutex_init(&vb->balloon_lock);
560
	init_waitqueue_head(&vb->acked);
561 562
	vb->vdev = vdev;

563
	balloon_devinfo_init(&vb->vb_dev_info);
564

565 566
	err = init_vqs(vb);
	if (err)
567
		goto out_free_vb;
R
Rusty Russell 已提交
568

569 570 571 572
	vb->nb.notifier_call = virtballoon_oom_notify;
	vb->nb.priority = VIRTBALLOON_OOM_NOTIFY_PRIORITY;
	err = register_oom_notifier(&vb->nb);
	if (err < 0)
573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593
		goto out_del_vqs;

#ifdef CONFIG_BALLOON_COMPACTION
	balloon_mnt = kern_mount(&balloon_fs);
	if (IS_ERR(balloon_mnt)) {
		err = PTR_ERR(balloon_mnt);
		unregister_oom_notifier(&vb->nb);
		goto out_del_vqs;
	}

	vb->vb_dev_info.migratepage = virtballoon_migratepage;
	vb->vb_dev_info.inode = alloc_anon_inode(balloon_mnt->mnt_sb);
	if (IS_ERR(vb->vb_dev_info.inode)) {
		err = PTR_ERR(vb->vb_dev_info.inode);
		kern_unmount(balloon_mnt);
		unregister_oom_notifier(&vb->nb);
		vb->vb_dev_info.inode = NULL;
		goto out_del_vqs;
	}
	vb->vb_dev_info.inode->i_mapping->a_ops = &balloon_aops;
#endif
594

595 596
	virtio_device_ready(vdev);

597 598
	if (towards_target(vb))
		virtballoon_changed(vdev);
R
Rusty Russell 已提交
599 600
	return 0;

601
out_del_vqs:
602
	vdev->config->del_vqs(vdev);
R
Rusty Russell 已提交
603 604 605 606 607 608
out_free_vb:
	kfree(vb);
out:
	return err;
}

609
static void remove_common(struct virtio_balloon *vb)
R
Rusty Russell 已提交
610 611 612 613
{
	/* There might be pages left in the balloon: free them. */
	while (vb->num_pages)
		leak_balloon(vb, vb->num_pages);
614
	update_balloon_size(vb);
R
Rusty Russell 已提交
615 616

	/* Now we reset the device so we can clean up the queues. */
617
	vb->vdev->config->reset(vb->vdev);
R
Rusty Russell 已提交
618

619 620 621
	vb->vdev->config->del_vqs(vb->vdev);
}

622
static void virtballoon_remove(struct virtio_device *vdev)
623 624 625
{
	struct virtio_balloon *vb = vdev->priv;

626
	unregister_oom_notifier(&vb->nb);
627 628 629 630

	spin_lock_irq(&vb->stop_update_lock);
	vb->stop_update = true;
	spin_unlock_irq(&vb->stop_update_lock);
631 632
	cancel_work_sync(&vb->update_balloon_size_work);
	cancel_work_sync(&vb->update_balloon_stats_work);
633

634
	remove_common(vb);
635
#ifdef CONFIG_BALLOON_COMPACTION
636 637
	if (vb->vb_dev_info.inode)
		iput(vb->vb_dev_info.inode);
638 639 640

	kern_unmount(balloon_mnt);
#endif
R
Rusty Russell 已提交
641 642 643
	kfree(vb);
}

644
#ifdef CONFIG_PM_SLEEP
645 646
static int virtballoon_freeze(struct virtio_device *vdev)
{
647 648
	struct virtio_balloon *vb = vdev->priv;

649
	/*
650
	 * The workqueue is already frozen by the PM core before this
651 652
	 * function is called.
	 */
653
	remove_common(vb);
654 655 656
	return 0;
}

657
static int virtballoon_restore(struct virtio_device *vdev)
658 659 660 661 662 663 664 665
{
	struct virtio_balloon *vb = vdev->priv;
	int ret;

	ret = init_vqs(vdev->priv);
	if (ret)
		return ret;

666 667
	virtio_device_ready(vdev);

668 669
	if (towards_target(vb))
		virtballoon_changed(vdev);
670 671 672
	update_balloon_size(vb);
	return 0;
}
673 674
#endif

675 676 677 678 679 680
static int virtballoon_validate(struct virtio_device *vdev)
{
	__virtio_clear_bit(vdev, VIRTIO_F_IOMMU_PLATFORM);
	return 0;
}

681 682 683
static unsigned int features[] = {
	VIRTIO_BALLOON_F_MUST_TELL_HOST,
	VIRTIO_BALLOON_F_STATS_VQ,
684
	VIRTIO_BALLOON_F_DEFLATE_ON_OOM,
685
};
686

687
static struct virtio_driver virtio_balloon_driver = {
688 689
	.feature_table = features,
	.feature_table_size = ARRAY_SIZE(features),
R
Rusty Russell 已提交
690 691 692
	.driver.name =	KBUILD_MODNAME,
	.driver.owner =	THIS_MODULE,
	.id_table =	id_table,
693
	.validate =	virtballoon_validate,
R
Rusty Russell 已提交
694
	.probe =	virtballoon_probe,
695
	.remove =	virtballoon_remove,
R
Rusty Russell 已提交
696
	.config_changed = virtballoon_changed,
697
#ifdef CONFIG_PM_SLEEP
698 699 700
	.freeze	=	virtballoon_freeze,
	.restore =	virtballoon_restore,
#endif
R
Rusty Russell 已提交
701 702
};

703
module_virtio_driver(virtio_balloon_driver);
R
Rusty Russell 已提交
704 705 706
MODULE_DEVICE_TABLE(virtio, id_table);
MODULE_DESCRIPTION("Virtio balloon driver");
MODULE_LICENSE("GPL");