virtio_balloon.c 19.2 KB
Newer Older
S
Sasha Levin 已提交
1 2
/*
 * Virtio balloon implementation, inspired by Dor Laor and Marcelo
R
Rusty Russell 已提交
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
 * Tosatti's implementations.
 *
 *  Copyright 2008 Rusty Russell IBM Corporation
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */
S
Sasha Levin 已提交
21

R
Rusty Russell 已提交
22 23 24
#include <linux/virtio.h>
#include <linux/virtio_balloon.h>
#include <linux/swap.h>
25
#include <linux/workqueue.h>
26
#include <linux/delay.h>
27
#include <linux/slab.h>
28
#include <linux/module.h>
29
#include <linux/balloon_compaction.h>
30
#include <linux/oom.h>
31
#include <linux/wait.h>
32
#include <linux/mm.h>
33
#include <linux/mount.h>
34
#include <linux/magic.h>
R
Rusty Russell 已提交
35

36 37 38 39 40
/*
 * Balloon device works in 4K page units.  So each page is pointed to by
 * multiple balloon pages.  All memory counters in this driver are in balloon
 * page units.
 */
41 42
#define VIRTIO_BALLOON_PAGES_PER_PAGE (unsigned)(PAGE_SIZE >> VIRTIO_BALLOON_PFN_SHIFT)
#define VIRTIO_BALLOON_ARRAY_PFNS_MAX 256
43 44 45 46 47 48
#define OOM_VBALLOON_DEFAULT_PAGES 256
#define VIRTBALLOON_OOM_NOTIFY_PRIORITY 80

static int oom_pages = OOM_VBALLOON_DEFAULT_PAGES;
module_param(oom_pages, int, S_IRUSR | S_IWUSR);
MODULE_PARM_DESC(oom_pages, "pages to free on OOM");
49

50 51 52 53
#ifdef CONFIG_BALLOON_COMPACTION
static struct vfsmount *balloon_mnt;
#endif

54
struct virtio_balloon {
R
Rusty Russell 已提交
55
	struct virtio_device *vdev;
56
	struct virtqueue *inflate_vq, *deflate_vq, *stats_vq;
R
Rusty Russell 已提交
57

58
	/* The balloon servicing is delegated to a freezable workqueue. */
59 60
	struct work_struct update_balloon_stats_work;
	struct work_struct update_balloon_size_work;
R
Rusty Russell 已提交
61

62 63 64
	/* Prevent updating balloon when it is being canceled. */
	spinlock_t stop_update_lock;
	bool stop_update;
R
Rusty Russell 已提交
65 66

	/* Waiting for host to ack the pages we released. */
67
	wait_queue_head_t acked;
R
Rusty Russell 已提交
68

69
	/* Number of balloon pages we've told the Host we're not using. */
R
Rusty Russell 已提交
70
	unsigned int num_pages;
71
	/*
72 73
	 * The pages we've told the Host we're not using are enqueued
	 * at vb_dev_info->pages list.
74 75 76
	 * Each page on this list adds VIRTIO_BALLOON_PAGES_PER_PAGE
	 * to num_pages above.
	 */
77
	struct balloon_dev_info vb_dev_info;
78 79 80

	/* Synchronize access/update to this struct virtio_balloon elements */
	struct mutex balloon_lock;
R
Rusty Russell 已提交
81 82 83

	/* The array of pfns we tell the Host about. */
	unsigned int num_pfns;
84
	__virtio32 pfns[VIRTIO_BALLOON_ARRAY_PFNS_MAX];
85 86 87

	/* Memory statistics */
	struct virtio_balloon_stat stats[VIRTIO_BALLOON_S_NR];
88 89 90

	/* To register callback in oom notifier call chain */
	struct notifier_block nb;
R
Rusty Russell 已提交
91 92 93 94 95 96 97
};

static struct virtio_device_id id_table[] = {
	{ VIRTIO_ID_BALLOON, VIRTIO_DEV_ANY_ID },
	{ 0 },
};

98 99 100 101 102 103
static u32 page_to_balloon_pfn(struct page *page)
{
	unsigned long pfn = page_to_pfn(page);

	BUILD_BUG_ON(PAGE_SHIFT < VIRTIO_BALLOON_PFN_SHIFT);
	/* Convert pfn from Linux page size to balloon page size. */
104 105 106 107 108 109 110
	return pfn * VIRTIO_BALLOON_PAGES_PER_PAGE;
}

static struct page *balloon_pfn_to_page(u32 pfn)
{
	BUG_ON(pfn % VIRTIO_BALLOON_PAGES_PER_PAGE);
	return pfn_to_page(pfn / VIRTIO_BALLOON_PAGES_PER_PAGE);
111 112
}

R
Rusty Russell 已提交
113 114
static void balloon_ack(struct virtqueue *vq)
{
115
	struct virtio_balloon *vb = vq->vdev->priv;
R
Rusty Russell 已提交
116

117
	wake_up(&vb->acked);
R
Rusty Russell 已提交
118 119 120 121 122
}

static void tell_host(struct virtio_balloon *vb, struct virtqueue *vq)
{
	struct scatterlist sg;
123
	unsigned int len;
R
Rusty Russell 已提交
124 125 126 127

	sg_init_one(&sg, vb->pfns, sizeof(vb->pfns[0]) * vb->num_pfns);

	/* We should always be able to add one buffer to an empty queue. */
128
	virtqueue_add_outbuf(vq, &sg, 1, vb, GFP_KERNEL);
129
	virtqueue_kick(vq);
R
Rusty Russell 已提交
130 131

	/* When host has read buffer, this completes via balloon_ack */
132
	wait_event(vb->acked, virtqueue_get_buf(vq, &len));
133

R
Rusty Russell 已提交
134 135
}

136 137
static void set_page_pfns(struct virtio_balloon *vb,
			  __virtio32 pfns[], struct page *page)
138 139 140 141 142 143
{
	unsigned int i;

	/* Set balloon pfns pointing at this page.
	 * Note that the first pfn points at start of the page. */
	for (i = 0; i < VIRTIO_BALLOON_PAGES_PER_PAGE; i++)
144 145
		pfns[i] = cpu_to_virtio32(vb->vdev,
					  page_to_balloon_pfn(page) + i);
146 147
}

148
static unsigned fill_balloon(struct virtio_balloon *vb, size_t num)
R
Rusty Russell 已提交
149
{
150
	struct balloon_dev_info *vb_dev_info = &vb->vb_dev_info;
151
	unsigned num_allocated_pages;
152

R
Rusty Russell 已提交
153 154 155
	/* We can only do one array worth at a time. */
	num = min(num, ARRAY_SIZE(vb->pfns));

156
	mutex_lock(&vb->balloon_lock);
157 158
	for (vb->num_pfns = 0; vb->num_pfns < num;
	     vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) {
159 160
		struct page *page = balloon_page_enqueue(vb_dev_info);

R
Rusty Russell 已提交
161
		if (!page) {
162
			dev_info_ratelimited(&vb->vdev->dev,
163 164
					     "Out of puff! Can't get %u pages\n",
					     VIRTIO_BALLOON_PAGES_PER_PAGE);
R
Rusty Russell 已提交
165 166 167 168
			/* Sleep for at least 1/5 of a second before retry. */
			msleep(200);
			break;
		}
169
		set_page_pfns(vb, vb->pfns + vb->num_pfns, page);
170
		vb->num_pages += VIRTIO_BALLOON_PAGES_PER_PAGE;
171 172 173
		if (!virtio_has_feature(vb->vdev,
					VIRTIO_BALLOON_F_DEFLATE_ON_OOM))
			adjust_managed_page_count(page, -1);
R
Rusty Russell 已提交
174 175
	}

176
	num_allocated_pages = vb->num_pfns;
177 178 179 180
	/* Did we get any? */
	if (vb->num_pfns != 0)
		tell_host(vb, vb->inflate_vq);
	mutex_unlock(&vb->balloon_lock);
181 182

	return num_allocated_pages;
R
Rusty Russell 已提交
183 184
}

185
static void release_pages_balloon(struct virtio_balloon *vb)
R
Rusty Russell 已提交
186 187
{
	unsigned int i;
188
	struct page *page;
R
Rusty Russell 已提交
189

190
	/* Find pfns pointing at start of each page, get pages and free them. */
191
	for (i = 0; i < vb->num_pfns; i += VIRTIO_BALLOON_PAGES_PER_PAGE) {
192 193
		page = balloon_pfn_to_page(virtio32_to_cpu(vb->vdev,
							   vb->pfns[i]));
194 195 196
		if (!virtio_has_feature(vb->vdev,
					VIRTIO_BALLOON_F_DEFLATE_ON_OOM))
			adjust_managed_page_count(page, 1);
197
		put_page(page); /* balloon reference */
R
Rusty Russell 已提交
198 199 200
	}
}

201
static unsigned leak_balloon(struct virtio_balloon *vb, size_t num)
R
Rusty Russell 已提交
202
{
203
	unsigned num_freed_pages;
R
Rusty Russell 已提交
204
	struct page *page;
205
	struct balloon_dev_info *vb_dev_info = &vb->vb_dev_info;
R
Rusty Russell 已提交
206 207 208 209

	/* We can only do one array worth at a time. */
	num = min(num, ARRAY_SIZE(vb->pfns));

210
	mutex_lock(&vb->balloon_lock);
211 212
	/* We can't release more pages than taken */
	num = min(num, (size_t)vb->num_pages);
213 214
	for (vb->num_pfns = 0; vb->num_pfns < num;
	     vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) {
215 216 217
		page = balloon_page_dequeue(vb_dev_info);
		if (!page)
			break;
218
		set_page_pfns(vb, vb->pfns + vb->num_pfns, page);
219
		vb->num_pages -= VIRTIO_BALLOON_PAGES_PER_PAGE;
R
Rusty Russell 已提交
220 221
	}

222
	num_freed_pages = vb->num_pfns;
223 224 225 226 227
	/*
	 * Note that if
	 * virtio_has_feature(vdev, VIRTIO_BALLOON_F_MUST_TELL_HOST);
	 * is true, we *have* to do it in this order
	 */
228 229
	if (vb->num_pfns != 0)
		tell_host(vb, vb->deflate_vq);
230
	release_pages_balloon(vb);
231
	mutex_unlock(&vb->balloon_lock);
232
	return num_freed_pages;
R
Rusty Russell 已提交
233 234
}

235 236 237 238
static inline void update_stat(struct virtio_balloon *vb, int idx,
			       u16 tag, u64 val)
{
	BUG_ON(idx >= VIRTIO_BALLOON_S_NR);
239 240
	vb->stats[idx].tag = cpu_to_virtio16(vb->vdev, tag);
	vb->stats[idx].val = cpu_to_virtio64(vb->vdev, val);
241 242 243 244
}

#define pages_to_bytes(x) ((u64)(x) << PAGE_SHIFT)

245
static unsigned int update_balloon_stats(struct virtio_balloon *vb)
246 247 248
{
	unsigned long events[NR_VM_EVENT_ITEMS];
	struct sysinfo i;
249
	unsigned int idx = 0;
250
	long available;
251 252 253 254

	all_vm_events(events);
	si_meminfo(&i);

255 256
	available = si_mem_available();

257 258 259 260 261 262 263 264 265 266
	update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_IN,
				pages_to_bytes(events[PSWPIN]));
	update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_OUT,
				pages_to_bytes(events[PSWPOUT]));
	update_stat(vb, idx++, VIRTIO_BALLOON_S_MAJFLT, events[PGMAJFAULT]);
	update_stat(vb, idx++, VIRTIO_BALLOON_S_MINFLT, events[PGFAULT]);
	update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMFREE,
				pages_to_bytes(i.freeram));
	update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMTOT,
				pages_to_bytes(i.totalram));
267 268
	update_stat(vb, idx++, VIRTIO_BALLOON_S_AVAIL,
				pages_to_bytes(available));
269 270

	return idx;
271 272 273 274 275 276 277
}

/*
 * While most virtqueues communicate guest-initiated requests to the hypervisor,
 * the stats queue operates in reverse.  The driver initializes the virtqueue
 * with a single buffer.  From that point forward, all conversations consist of
 * a hypervisor request (a call to this function) which directs us to refill
278
 * the virtqueue with a fresh stats buffer.  Since stats collection can sleep,
279 280
 * we delegate the job to a freezable workqueue that will do the actual work via
 * stats_handle_request().
281
 */
282
static void stats_request(struct virtqueue *vq)
283
{
284
	struct virtio_balloon *vb = vq->vdev->priv;
285

286 287
	spin_lock(&vb->stop_update_lock);
	if (!vb->stop_update)
288
		queue_work(system_freezable_wq, &vb->update_balloon_stats_work);
289
	spin_unlock(&vb->stop_update_lock);
290 291 292 293 294 295
}

static void stats_handle_request(struct virtio_balloon *vb)
{
	struct virtqueue *vq;
	struct scatterlist sg;
296
	unsigned int len, num_stats;
297

298
	num_stats = update_balloon_stats(vb);
299

300
	vq = vb->stats_vq;
301 302
	if (!virtqueue_get_buf(vq, &len))
		return;
303
	sg_init_one(&sg, vb->stats, sizeof(vb->stats[0]) * num_stats);
304
	virtqueue_add_outbuf(vq, &sg, 1, vb, GFP_KERNEL);
305
	virtqueue_kick(vq);
306 307
}

R
Rusty Russell 已提交
308 309 310
static void virtballoon_changed(struct virtio_device *vdev)
{
	struct virtio_balloon *vb = vdev->priv;
311
	unsigned long flags;
R
Rusty Russell 已提交
312

313 314
	spin_lock_irqsave(&vb->stop_update_lock, flags);
	if (!vb->stop_update)
315
		queue_work(system_freezable_wq, &vb->update_balloon_size_work);
316
	spin_unlock_irqrestore(&vb->stop_update_lock, flags);
R
Rusty Russell 已提交
317 318
}

319
static inline s64 towards_target(struct virtio_balloon *vb)
R
Rusty Russell 已提交
320
{
D
David Gibson 已提交
321
	s64 target;
322
	u32 num_pages;
D
David Gibson 已提交
323

324 325
	virtio_cread(vb->vdev, struct virtio_balloon_config, num_pages,
		     &num_pages);
326

327 328 329 330 331
	/* Legacy balloon config space is LE, unlike all other devices. */
	if (!virtio_has_feature(vb->vdev, VIRTIO_F_VERSION_1))
		num_pages = le32_to_cpu((__force __le32)num_pages);

	target = num_pages;
D
David Gibson 已提交
332
	return target - vb->num_pages;
R
Rusty Russell 已提交
333 334 335 336
}

static void update_balloon_size(struct virtio_balloon *vb)
{
337 338 339 340 341
	u32 actual = vb->num_pages;

	/* Legacy balloon config space is LE, unlike all other devices. */
	if (!virtio_has_feature(vb->vdev, VIRTIO_F_VERSION_1))
		actual = (__force u32)cpu_to_le32(actual);
R
Rusty Russell 已提交
342

343
	virtio_cwrite(vb->vdev, struct virtio_balloon_config, actual,
344
		      &actual);
R
Rusty Russell 已提交
345 346
}

347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378
/*
 * virtballoon_oom_notify - release pages when system is under severe
 *			    memory pressure (called from out_of_memory())
 * @self : notifier block struct
 * @dummy: not used
 * @parm : returned - number of freed pages
 *
 * The balancing of memory by use of the virtio balloon should not cause
 * the termination of processes while there are pages in the balloon.
 * If virtio balloon manages to release some memory, it will make the
 * system return and retry the allocation that forced the OOM killer
 * to run.
 */
static int virtballoon_oom_notify(struct notifier_block *self,
				  unsigned long dummy, void *parm)
{
	struct virtio_balloon *vb;
	unsigned long *freed;
	unsigned num_freed_pages;

	vb = container_of(self, struct virtio_balloon, nb);
	if (!virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM))
		return NOTIFY_OK;

	freed = parm;
	num_freed_pages = leak_balloon(vb, oom_pages);
	update_balloon_size(vb);
	*freed += num_freed_pages;

	return NOTIFY_OK;
}

379
static void update_balloon_stats_func(struct work_struct *work)
R
Rusty Russell 已提交
380
{
381
	struct virtio_balloon *vb;
382

383 384 385 386
	vb = container_of(work, struct virtio_balloon,
			  update_balloon_stats_work);
	stats_handle_request(vb);
}
387

388
static void update_balloon_size_func(struct work_struct *work)
R
Rusty Russell 已提交
389
{
390 391
	struct virtio_balloon *vb;
	s64 diff;
392

393 394
	vb = container_of(work, struct virtio_balloon,
			  update_balloon_size_work);
395
	diff = towards_target(vb);
396

397 398 399 400 401 402 403 404
	if (diff > 0)
		diff -= fill_balloon(vb, diff);
	else if (diff < 0)
		diff += leak_balloon(vb, -diff);
	update_balloon_size(vb);

	if (diff)
		queue_work(system_freezable_wq, work);
R
Rusty Russell 已提交
405 406
}

407
static int init_vqs(struct virtio_balloon *vb)
R
Rusty Russell 已提交
408
{
409
	struct virtqueue *vqs[3];
410
	vq_callback_t *callbacks[] = { balloon_ack, balloon_ack, stats_request };
411
	static const char * const names[] = { "inflate", "deflate", "stats" };
412
	int err, nvqs;
R
Rusty Russell 已提交
413

414 415 416 417
	/*
	 * We expect two virtqueues: inflate and deflate, and
	 * optionally stat.
	 */
418
	nvqs = virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ) ? 3 : 2;
419 420
	err = vb->vdev->config->find_vqs(vb->vdev, nvqs, vqs, callbacks, names,
			NULL);
421
	if (err)
422
		return err;
R
Rusty Russell 已提交
423

424 425
	vb->inflate_vq = vqs[0];
	vb->deflate_vq = vqs[1];
426 427
	if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) {
		struct scatterlist sg;
428
		unsigned int num_stats;
429 430 431 432
		vb->stats_vq = vqs[2];

		/*
		 * Prime this virtqueue with one buffer so the hypervisor can
433
		 * use it to signal us later (it can't be broken yet!).
434
		 */
435
		num_stats = update_balloon_stats(vb);
436

437
		sg_init_one(&sg, vb->stats, sizeof(vb->stats[0]) * num_stats);
438
		if (virtqueue_add_outbuf(vb->stats_vq, &sg, 1, vb, GFP_KERNEL)
439
		    < 0)
440
			BUG();
441
		virtqueue_kick(vb->stats_vq);
442
	}
443 444 445
	return 0;
}

446 447 448 449
#ifdef CONFIG_BALLOON_COMPACTION
/*
 * virtballoon_migratepage - perform the balloon page migration on behalf of
 *			     a compation thread.     (called under page lock)
450
 * @vb_dev_info: the balloon device
451 452 453 454 455 456 457 458 459 460 461 462 463 464
 * @newpage: page that will replace the isolated page after migration finishes.
 * @page   : the isolated (old) page that is about to be migrated to newpage.
 * @mode   : compaction mode -- not used for balloon page migration.
 *
 * After a ballooned page gets isolated by compaction procedures, this is the
 * function that performs the page migration on behalf of a compaction thread
 * The page migration for virtio balloon is done in a simple swap fashion which
 * follows these two macro steps:
 *  1) insert newpage into vb->pages list and update the host about it;
 *  2) update the host about the old page removed from vb->pages list;
 *
 * This function preforms the balloon page migration task.
 * Called through balloon_mapping->a_ops->migratepage
 */
465
static int virtballoon_migratepage(struct balloon_dev_info *vb_dev_info,
466 467
		struct page *newpage, struct page *page, enum migrate_mode mode)
{
468 469
	struct virtio_balloon *vb = container_of(vb_dev_info,
			struct virtio_balloon, vb_dev_info);
470 471 472 473 474 475 476 477 478 479 480 481 482
	unsigned long flags;

	/*
	 * In order to avoid lock contention while migrating pages concurrently
	 * to leak_balloon() or fill_balloon() we just give up the balloon_lock
	 * this turn, as it is easier to retry the page migration later.
	 * This also prevents fill_balloon() getting stuck into a mutex
	 * recursion in the case it ends up triggering memory compaction
	 * while it is attempting to inflate the ballon.
	 */
	if (!mutex_trylock(&vb->balloon_lock))
		return -EAGAIN;

483 484
	get_page(newpage); /* balloon reference */

485 486
	/* balloon's page migration 1st step  -- inflate "newpage" */
	spin_lock_irqsave(&vb_dev_info->pages_lock, flags);
487
	balloon_page_insert(vb_dev_info, newpage);
488
	vb_dev_info->isolated_pages--;
489
	__count_vm_event(BALLOON_MIGRATE);
490 491
	spin_unlock_irqrestore(&vb_dev_info->pages_lock, flags);
	vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE;
492
	set_page_pfns(vb, vb->pfns, newpage);
493 494
	tell_host(vb, vb->inflate_vq);

495
	/* balloon's page migration 2nd step -- deflate "page" */
496 497
	balloon_page_delete(page);
	vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE;
498
	set_page_pfns(vb, vb->pfns, page);
499 500 501 502
	tell_host(vb, vb->deflate_vq);

	mutex_unlock(&vb->balloon_lock);

503 504
	put_page(page); /* balloon reference */

505
	return MIGRATEPAGE_SUCCESS;
506
}
507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524

static struct dentry *balloon_mount(struct file_system_type *fs_type,
		int flags, const char *dev_name, void *data)
{
	static const struct dentry_operations ops = {
		.d_dname = simple_dname,
	};

	return mount_pseudo(fs_type, "balloon-kvm:", NULL, &ops,
				BALLOON_KVM_MAGIC);
}

static struct file_system_type balloon_fs = {
	.name           = "balloon-kvm",
	.mount          = balloon_mount,
	.kill_sb        = kill_anon_super,
};

525 526
#endif /* CONFIG_BALLOON_COMPACTION */

527 528 529 530 531
static int virtballoon_probe(struct virtio_device *vdev)
{
	struct virtio_balloon *vb;
	int err;

532 533 534 535 536 537
	if (!vdev->config->get) {
		dev_err(&vdev->dev, "%s failure: config access disabled\n",
			__func__);
		return -EINVAL;
	}

538 539 540 541 542 543
	vdev->priv = vb = kmalloc(sizeof(*vb), GFP_KERNEL);
	if (!vb) {
		err = -ENOMEM;
		goto out;
	}

544 545
	INIT_WORK(&vb->update_balloon_stats_work, update_balloon_stats_func);
	INIT_WORK(&vb->update_balloon_size_work, update_balloon_size_func);
546 547
	spin_lock_init(&vb->stop_update_lock);
	vb->stop_update = false;
548
	vb->num_pages = 0;
549
	mutex_init(&vb->balloon_lock);
550
	init_waitqueue_head(&vb->acked);
551 552
	vb->vdev = vdev;

553
	balloon_devinfo_init(&vb->vb_dev_info);
554

555 556
	err = init_vqs(vb);
	if (err)
557
		goto out_free_vb;
R
Rusty Russell 已提交
558

559 560 561 562
	vb->nb.notifier_call = virtballoon_oom_notify;
	vb->nb.priority = VIRTBALLOON_OOM_NOTIFY_PRIORITY;
	err = register_oom_notifier(&vb->nb);
	if (err < 0)
563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583
		goto out_del_vqs;

#ifdef CONFIG_BALLOON_COMPACTION
	balloon_mnt = kern_mount(&balloon_fs);
	if (IS_ERR(balloon_mnt)) {
		err = PTR_ERR(balloon_mnt);
		unregister_oom_notifier(&vb->nb);
		goto out_del_vqs;
	}

	vb->vb_dev_info.migratepage = virtballoon_migratepage;
	vb->vb_dev_info.inode = alloc_anon_inode(balloon_mnt->mnt_sb);
	if (IS_ERR(vb->vb_dev_info.inode)) {
		err = PTR_ERR(vb->vb_dev_info.inode);
		kern_unmount(balloon_mnt);
		unregister_oom_notifier(&vb->nb);
		vb->vb_dev_info.inode = NULL;
		goto out_del_vqs;
	}
	vb->vb_dev_info.inode->i_mapping->a_ops = &balloon_aops;
#endif
584

585 586
	virtio_device_ready(vdev);

587 588
	if (towards_target(vb))
		virtballoon_changed(vdev);
R
Rusty Russell 已提交
589 590
	return 0;

591
out_del_vqs:
592
	vdev->config->del_vqs(vdev);
R
Rusty Russell 已提交
593 594 595 596 597 598
out_free_vb:
	kfree(vb);
out:
	return err;
}

599
static void remove_common(struct virtio_balloon *vb)
R
Rusty Russell 已提交
600 601 602 603
{
	/* There might be pages left in the balloon: free them. */
	while (vb->num_pages)
		leak_balloon(vb, vb->num_pages);
604
	update_balloon_size(vb);
R
Rusty Russell 已提交
605 606

	/* Now we reset the device so we can clean up the queues. */
607
	vb->vdev->config->reset(vb->vdev);
R
Rusty Russell 已提交
608

609 610 611
	vb->vdev->config->del_vqs(vb->vdev);
}

612
static void virtballoon_remove(struct virtio_device *vdev)
613 614 615
{
	struct virtio_balloon *vb = vdev->priv;

616
	unregister_oom_notifier(&vb->nb);
617 618 619 620

	spin_lock_irq(&vb->stop_update_lock);
	vb->stop_update = true;
	spin_unlock_irq(&vb->stop_update_lock);
621 622
	cancel_work_sync(&vb->update_balloon_size_work);
	cancel_work_sync(&vb->update_balloon_stats_work);
623

624
	remove_common(vb);
625
#ifdef CONFIG_BALLOON_COMPACTION
626 627
	if (vb->vb_dev_info.inode)
		iput(vb->vb_dev_info.inode);
628 629 630

	kern_unmount(balloon_mnt);
#endif
R
Rusty Russell 已提交
631 632 633
	kfree(vb);
}

634
#ifdef CONFIG_PM_SLEEP
635 636
static int virtballoon_freeze(struct virtio_device *vdev)
{
637 638
	struct virtio_balloon *vb = vdev->priv;

639
	/*
640
	 * The workqueue is already frozen by the PM core before this
641 642
	 * function is called.
	 */
643
	remove_common(vb);
644 645 646
	return 0;
}

647
static int virtballoon_restore(struct virtio_device *vdev)
648 649 650 651 652 653 654 655
{
	struct virtio_balloon *vb = vdev->priv;
	int ret;

	ret = init_vqs(vdev->priv);
	if (ret)
		return ret;

656 657
	virtio_device_ready(vdev);

658 659
	if (towards_target(vb))
		virtballoon_changed(vdev);
660 661 662
	update_balloon_size(vb);
	return 0;
}
663 664
#endif

665 666 667
static unsigned int features[] = {
	VIRTIO_BALLOON_F_MUST_TELL_HOST,
	VIRTIO_BALLOON_F_STATS_VQ,
668
	VIRTIO_BALLOON_F_DEFLATE_ON_OOM,
669
};
670

671
static struct virtio_driver virtio_balloon_driver = {
672 673
	.feature_table = features,
	.feature_table_size = ARRAY_SIZE(features),
R
Rusty Russell 已提交
674 675 676 677
	.driver.name =	KBUILD_MODNAME,
	.driver.owner =	THIS_MODULE,
	.id_table =	id_table,
	.probe =	virtballoon_probe,
678
	.remove =	virtballoon_remove,
R
Rusty Russell 已提交
679
	.config_changed = virtballoon_changed,
680
#ifdef CONFIG_PM_SLEEP
681 682 683
	.freeze	=	virtballoon_freeze,
	.restore =	virtballoon_restore,
#endif
R
Rusty Russell 已提交
684 685
};

686
module_virtio_driver(virtio_balloon_driver);
R
Rusty Russell 已提交
687 688 689
MODULE_DEVICE_TABLE(virtio, id_table);
MODULE_DESCRIPTION("Virtio balloon driver");
MODULE_LICENSE("GPL");