virtio_balloon.c 19.0 KB
Newer Older
S
Sasha Levin 已提交
1 2
/*
 * Virtio balloon implementation, inspired by Dor Laor and Marcelo
R
Rusty Russell 已提交
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
 * Tosatti's implementations.
 *
 *  Copyright 2008 Rusty Russell IBM Corporation
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */
S
Sasha Levin 已提交
21

R
Rusty Russell 已提交
22 23 24
#include <linux/virtio.h>
#include <linux/virtio_balloon.h>
#include <linux/swap.h>
25
#include <linux/workqueue.h>
26
#include <linux/delay.h>
27
#include <linux/slab.h>
28
#include <linux/module.h>
29
#include <linux/balloon_compaction.h>
30
#include <linux/oom.h>
31
#include <linux/wait.h>
32
#include <linux/mm.h>
33
#include <linux/mount.h>
R
Rusty Russell 已提交
34

35 36 37 38 39
/*
 * Balloon device works in 4K page units.  So each page is pointed to by
 * multiple balloon pages.  All memory counters in this driver are in balloon
 * page units.
 */
40 41
#define VIRTIO_BALLOON_PAGES_PER_PAGE (unsigned)(PAGE_SIZE >> VIRTIO_BALLOON_PFN_SHIFT)
#define VIRTIO_BALLOON_ARRAY_PFNS_MAX 256
42 43 44 45 46 47
#define OOM_VBALLOON_DEFAULT_PAGES 256
#define VIRTBALLOON_OOM_NOTIFY_PRIORITY 80

static int oom_pages = OOM_VBALLOON_DEFAULT_PAGES;
module_param(oom_pages, int, S_IRUSR | S_IWUSR);
MODULE_PARM_DESC(oom_pages, "pages to free on OOM");
48

49 50 51 52
#ifdef CONFIG_BALLOON_COMPACTION
static struct vfsmount *balloon_mnt;
#endif

53
struct virtio_balloon {
R
Rusty Russell 已提交
54
	struct virtio_device *vdev;
55
	struct virtqueue *inflate_vq, *deflate_vq, *stats_vq;
R
Rusty Russell 已提交
56

57
	/* The balloon servicing is delegated to a freezable workqueue. */
58 59
	struct work_struct update_balloon_stats_work;
	struct work_struct update_balloon_size_work;
R
Rusty Russell 已提交
60

61 62 63
	/* Prevent updating balloon when it is being canceled. */
	spinlock_t stop_update_lock;
	bool stop_update;
R
Rusty Russell 已提交
64 65

	/* Waiting for host to ack the pages we released. */
66
	wait_queue_head_t acked;
R
Rusty Russell 已提交
67

68
	/* Number of balloon pages we've told the Host we're not using. */
R
Rusty Russell 已提交
69
	unsigned int num_pages;
70
	/*
71 72
	 * The pages we've told the Host we're not using are enqueued
	 * at vb_dev_info->pages list.
73 74 75
	 * Each page on this list adds VIRTIO_BALLOON_PAGES_PER_PAGE
	 * to num_pages above.
	 */
76
	struct balloon_dev_info vb_dev_info;
77 78 79

	/* Synchronize access/update to this struct virtio_balloon elements */
	struct mutex balloon_lock;
R
Rusty Russell 已提交
80 81 82

	/* The array of pfns we tell the Host about. */
	unsigned int num_pfns;
83
	__virtio32 pfns[VIRTIO_BALLOON_ARRAY_PFNS_MAX];
84 85 86

	/* Memory statistics */
	struct virtio_balloon_stat stats[VIRTIO_BALLOON_S_NR];
87 88 89

	/* To register callback in oom notifier call chain */
	struct notifier_block nb;
R
Rusty Russell 已提交
90 91 92 93 94 95 96
};

static struct virtio_device_id id_table[] = {
	{ VIRTIO_ID_BALLOON, VIRTIO_DEV_ANY_ID },
	{ 0 },
};

97 98 99 100 101 102
static u32 page_to_balloon_pfn(struct page *page)
{
	unsigned long pfn = page_to_pfn(page);

	BUILD_BUG_ON(PAGE_SHIFT < VIRTIO_BALLOON_PFN_SHIFT);
	/* Convert pfn from Linux page size to balloon page size. */
103 104 105 106 107 108 109
	return pfn * VIRTIO_BALLOON_PAGES_PER_PAGE;
}

static struct page *balloon_pfn_to_page(u32 pfn)
{
	BUG_ON(pfn % VIRTIO_BALLOON_PAGES_PER_PAGE);
	return pfn_to_page(pfn / VIRTIO_BALLOON_PAGES_PER_PAGE);
110 111
}

R
Rusty Russell 已提交
112 113
static void balloon_ack(struct virtqueue *vq)
{
114
	struct virtio_balloon *vb = vq->vdev->priv;
R
Rusty Russell 已提交
115

116
	wake_up(&vb->acked);
R
Rusty Russell 已提交
117 118 119 120 121
}

static void tell_host(struct virtio_balloon *vb, struct virtqueue *vq)
{
	struct scatterlist sg;
122
	unsigned int len;
R
Rusty Russell 已提交
123 124 125 126

	sg_init_one(&sg, vb->pfns, sizeof(vb->pfns[0]) * vb->num_pfns);

	/* We should always be able to add one buffer to an empty queue. */
127
	virtqueue_add_outbuf(vq, &sg, 1, vb, GFP_KERNEL);
128
	virtqueue_kick(vq);
R
Rusty Russell 已提交
129 130

	/* When host has read buffer, this completes via balloon_ack */
131
	wait_event(vb->acked, virtqueue_get_buf(vq, &len));
132

R
Rusty Russell 已提交
133 134
}

135 136
static void set_page_pfns(struct virtio_balloon *vb,
			  __virtio32 pfns[], struct page *page)
137 138 139 140 141 142
{
	unsigned int i;

	/* Set balloon pfns pointing at this page.
	 * Note that the first pfn points at start of the page. */
	for (i = 0; i < VIRTIO_BALLOON_PAGES_PER_PAGE; i++)
143 144
		pfns[i] = cpu_to_virtio32(vb->vdev,
					  page_to_balloon_pfn(page) + i);
145 146
}

147
static unsigned fill_balloon(struct virtio_balloon *vb, size_t num)
R
Rusty Russell 已提交
148
{
149
	struct balloon_dev_info *vb_dev_info = &vb->vb_dev_info;
150
	unsigned num_allocated_pages;
151

R
Rusty Russell 已提交
152 153 154
	/* We can only do one array worth at a time. */
	num = min(num, ARRAY_SIZE(vb->pfns));

155
	mutex_lock(&vb->balloon_lock);
156 157
	for (vb->num_pfns = 0; vb->num_pfns < num;
	     vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) {
158 159
		struct page *page = balloon_page_enqueue(vb_dev_info);

R
Rusty Russell 已提交
160
		if (!page) {
161
			dev_info_ratelimited(&vb->vdev->dev,
162 163
					     "Out of puff! Can't get %u pages\n",
					     VIRTIO_BALLOON_PAGES_PER_PAGE);
R
Rusty Russell 已提交
164 165 166 167
			/* Sleep for at least 1/5 of a second before retry. */
			msleep(200);
			break;
		}
168
		set_page_pfns(vb, vb->pfns + vb->num_pfns, page);
169
		vb->num_pages += VIRTIO_BALLOON_PAGES_PER_PAGE;
170 171 172
		if (!virtio_has_feature(vb->vdev,
					VIRTIO_BALLOON_F_DEFLATE_ON_OOM))
			adjust_managed_page_count(page, -1);
R
Rusty Russell 已提交
173 174
	}

175
	num_allocated_pages = vb->num_pfns;
176 177 178 179
	/* Did we get any? */
	if (vb->num_pfns != 0)
		tell_host(vb, vb->inflate_vq);
	mutex_unlock(&vb->balloon_lock);
180 181

	return num_allocated_pages;
R
Rusty Russell 已提交
182 183
}

184
static void release_pages_balloon(struct virtio_balloon *vb)
R
Rusty Russell 已提交
185 186
{
	unsigned int i;
187
	struct page *page;
R
Rusty Russell 已提交
188

189
	/* Find pfns pointing at start of each page, get pages and free them. */
190
	for (i = 0; i < vb->num_pfns; i += VIRTIO_BALLOON_PAGES_PER_PAGE) {
191 192
		page = balloon_pfn_to_page(virtio32_to_cpu(vb->vdev,
							   vb->pfns[i]));
193 194 195
		if (!virtio_has_feature(vb->vdev,
					VIRTIO_BALLOON_F_DEFLATE_ON_OOM))
			adjust_managed_page_count(page, 1);
196
		put_page(page); /* balloon reference */
R
Rusty Russell 已提交
197 198 199
	}
}

200
static unsigned leak_balloon(struct virtio_balloon *vb, size_t num)
R
Rusty Russell 已提交
201
{
202
	unsigned num_freed_pages;
R
Rusty Russell 已提交
203
	struct page *page;
204
	struct balloon_dev_info *vb_dev_info = &vb->vb_dev_info;
R
Rusty Russell 已提交
205 206 207 208

	/* We can only do one array worth at a time. */
	num = min(num, ARRAY_SIZE(vb->pfns));

209
	mutex_lock(&vb->balloon_lock);
210 211
	/* We can't release more pages than taken */
	num = min(num, (size_t)vb->num_pages);
212 213
	for (vb->num_pfns = 0; vb->num_pfns < num;
	     vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) {
214 215 216
		page = balloon_page_dequeue(vb_dev_info);
		if (!page)
			break;
217
		set_page_pfns(vb, vb->pfns + vb->num_pfns, page);
218
		vb->num_pages -= VIRTIO_BALLOON_PAGES_PER_PAGE;
R
Rusty Russell 已提交
219 220
	}

221
	num_freed_pages = vb->num_pfns;
222 223 224 225 226
	/*
	 * Note that if
	 * virtio_has_feature(vdev, VIRTIO_BALLOON_F_MUST_TELL_HOST);
	 * is true, we *have* to do it in this order
	 */
227 228
	if (vb->num_pfns != 0)
		tell_host(vb, vb->deflate_vq);
229
	release_pages_balloon(vb);
230
	mutex_unlock(&vb->balloon_lock);
231
	return num_freed_pages;
R
Rusty Russell 已提交
232 233
}

234 235 236 237
static inline void update_stat(struct virtio_balloon *vb, int idx,
			       u16 tag, u64 val)
{
	BUG_ON(idx >= VIRTIO_BALLOON_S_NR);
238 239
	vb->stats[idx].tag = cpu_to_virtio16(vb->vdev, tag);
	vb->stats[idx].val = cpu_to_virtio64(vb->vdev, val);
240 241 242 243 244 245 246 247 248
}

#define pages_to_bytes(x) ((u64)(x) << PAGE_SHIFT)

static void update_balloon_stats(struct virtio_balloon *vb)
{
	unsigned long events[NR_VM_EVENT_ITEMS];
	struct sysinfo i;
	int idx = 0;
249
	long available;
250 251 252 253

	all_vm_events(events);
	si_meminfo(&i);

254 255
	available = si_mem_available();

256 257 258 259 260 261 262 263 264 265
	update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_IN,
				pages_to_bytes(events[PSWPIN]));
	update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_OUT,
				pages_to_bytes(events[PSWPOUT]));
	update_stat(vb, idx++, VIRTIO_BALLOON_S_MAJFLT, events[PGMAJFAULT]);
	update_stat(vb, idx++, VIRTIO_BALLOON_S_MINFLT, events[PGFAULT]);
	update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMFREE,
				pages_to_bytes(i.freeram));
	update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMTOT,
				pages_to_bytes(i.totalram));
266 267
	update_stat(vb, idx++, VIRTIO_BALLOON_S_AVAIL,
				pages_to_bytes(available));
268 269 270 271 272 273 274
}

/*
 * While most virtqueues communicate guest-initiated requests to the hypervisor,
 * the stats queue operates in reverse.  The driver initializes the virtqueue
 * with a single buffer.  From that point forward, all conversations consist of
 * a hypervisor request (a call to this function) which directs us to refill
275
 * the virtqueue with a fresh stats buffer.  Since stats collection can sleep,
276 277
 * we delegate the job to a freezable workqueue that will do the actual work via
 * stats_handle_request().
278
 */
279
static void stats_request(struct virtqueue *vq)
280
{
281
	struct virtio_balloon *vb = vq->vdev->priv;
282

283 284
	spin_lock(&vb->stop_update_lock);
	if (!vb->stop_update)
285
		queue_work(system_freezable_wq, &vb->update_balloon_stats_work);
286
	spin_unlock(&vb->stop_update_lock);
287 288 289 290 291 292
}

static void stats_handle_request(struct virtio_balloon *vb)
{
	struct virtqueue *vq;
	struct scatterlist sg;
293
	unsigned int len;
294 295 296

	update_balloon_stats(vb);

297
	vq = vb->stats_vq;
298 299
	if (!virtqueue_get_buf(vq, &len))
		return;
300
	sg_init_one(&sg, vb->stats, sizeof(vb->stats));
301
	virtqueue_add_outbuf(vq, &sg, 1, vb, GFP_KERNEL);
302
	virtqueue_kick(vq);
303 304
}

R
Rusty Russell 已提交
305 306 307
static void virtballoon_changed(struct virtio_device *vdev)
{
	struct virtio_balloon *vb = vdev->priv;
308
	unsigned long flags;
R
Rusty Russell 已提交
309

310 311
	spin_lock_irqsave(&vb->stop_update_lock, flags);
	if (!vb->stop_update)
312
		queue_work(system_freezable_wq, &vb->update_balloon_size_work);
313
	spin_unlock_irqrestore(&vb->stop_update_lock, flags);
R
Rusty Russell 已提交
314 315
}

316
static inline s64 towards_target(struct virtio_balloon *vb)
R
Rusty Russell 已提交
317
{
D
David Gibson 已提交
318
	s64 target;
319
	u32 num_pages;
D
David Gibson 已提交
320

321 322
	virtio_cread(vb->vdev, struct virtio_balloon_config, num_pages,
		     &num_pages);
323

324 325 326 327 328
	/* Legacy balloon config space is LE, unlike all other devices. */
	if (!virtio_has_feature(vb->vdev, VIRTIO_F_VERSION_1))
		num_pages = le32_to_cpu((__force __le32)num_pages);

	target = num_pages;
D
David Gibson 已提交
329
	return target - vb->num_pages;
R
Rusty Russell 已提交
330 331 332 333
}

static void update_balloon_size(struct virtio_balloon *vb)
{
334 335 336 337 338
	u32 actual = vb->num_pages;

	/* Legacy balloon config space is LE, unlike all other devices. */
	if (!virtio_has_feature(vb->vdev, VIRTIO_F_VERSION_1))
		actual = (__force u32)cpu_to_le32(actual);
R
Rusty Russell 已提交
339

340
	virtio_cwrite(vb->vdev, struct virtio_balloon_config, actual,
341
		      &actual);
R
Rusty Russell 已提交
342 343
}

344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375
/*
 * virtballoon_oom_notify - release pages when system is under severe
 *			    memory pressure (called from out_of_memory())
 * @self : notifier block struct
 * @dummy: not used
 * @parm : returned - number of freed pages
 *
 * The balancing of memory by use of the virtio balloon should not cause
 * the termination of processes while there are pages in the balloon.
 * If virtio balloon manages to release some memory, it will make the
 * system return and retry the allocation that forced the OOM killer
 * to run.
 */
static int virtballoon_oom_notify(struct notifier_block *self,
				  unsigned long dummy, void *parm)
{
	struct virtio_balloon *vb;
	unsigned long *freed;
	unsigned num_freed_pages;

	vb = container_of(self, struct virtio_balloon, nb);
	if (!virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM))
		return NOTIFY_OK;

	freed = parm;
	num_freed_pages = leak_balloon(vb, oom_pages);
	update_balloon_size(vb);
	*freed += num_freed_pages;

	return NOTIFY_OK;
}

376
static void update_balloon_stats_func(struct work_struct *work)
R
Rusty Russell 已提交
377
{
378
	struct virtio_balloon *vb;
379

380 381 382 383
	vb = container_of(work, struct virtio_balloon,
			  update_balloon_stats_work);
	stats_handle_request(vb);
}
384

385
static void update_balloon_size_func(struct work_struct *work)
R
Rusty Russell 已提交
386
{
387 388
	struct virtio_balloon *vb;
	s64 diff;
389

390 391
	vb = container_of(work, struct virtio_balloon,
			  update_balloon_size_work);
392
	diff = towards_target(vb);
393

394 395 396 397 398 399 400 401
	if (diff > 0)
		diff -= fill_balloon(vb, diff);
	else if (diff < 0)
		diff += leak_balloon(vb, -diff);
	update_balloon_size(vb);

	if (diff)
		queue_work(system_freezable_wq, work);
R
Rusty Russell 已提交
402 403
}

404
static int init_vqs(struct virtio_balloon *vb)
R
Rusty Russell 已提交
405
{
406
	struct virtqueue *vqs[3];
407
	vq_callback_t *callbacks[] = { balloon_ack, balloon_ack, stats_request };
408
	static const char * const names[] = { "inflate", "deflate", "stats" };
409
	int err, nvqs;
R
Rusty Russell 已提交
410

411 412 413 414
	/*
	 * We expect two virtqueues: inflate and deflate, and
	 * optionally stat.
	 */
415
	nvqs = virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ) ? 3 : 2;
416
	err = vb->vdev->config->find_vqs(vb->vdev, nvqs, vqs, callbacks, names);
417
	if (err)
418
		return err;
R
Rusty Russell 已提交
419

420 421
	vb->inflate_vq = vqs[0];
	vb->deflate_vq = vqs[1];
422 423 424 425 426 427
	if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) {
		struct scatterlist sg;
		vb->stats_vq = vqs[2];

		/*
		 * Prime this virtqueue with one buffer so the hypervisor can
428
		 * use it to signal us later (it can't be broken yet!).
429 430
		 */
		sg_init_one(&sg, vb->stats, sizeof vb->stats);
431
		if (virtqueue_add_outbuf(vb->stats_vq, &sg, 1, vb, GFP_KERNEL)
432
		    < 0)
433
			BUG();
434
		virtqueue_kick(vb->stats_vq);
435
	}
436 437 438
	return 0;
}

439 440 441 442
#ifdef CONFIG_BALLOON_COMPACTION
/*
 * virtballoon_migratepage - perform the balloon page migration on behalf of
 *			     a compation thread.     (called under page lock)
443
 * @vb_dev_info: the balloon device
444 445 446 447 448 449 450 451 452 453 454 455 456 457
 * @newpage: page that will replace the isolated page after migration finishes.
 * @page   : the isolated (old) page that is about to be migrated to newpage.
 * @mode   : compaction mode -- not used for balloon page migration.
 *
 * After a ballooned page gets isolated by compaction procedures, this is the
 * function that performs the page migration on behalf of a compaction thread
 * The page migration for virtio balloon is done in a simple swap fashion which
 * follows these two macro steps:
 *  1) insert newpage into vb->pages list and update the host about it;
 *  2) update the host about the old page removed from vb->pages list;
 *
 * This function preforms the balloon page migration task.
 * Called through balloon_mapping->a_ops->migratepage
 */
458
static int virtballoon_migratepage(struct balloon_dev_info *vb_dev_info,
459 460
		struct page *newpage, struct page *page, enum migrate_mode mode)
{
461 462
	struct virtio_balloon *vb = container_of(vb_dev_info,
			struct virtio_balloon, vb_dev_info);
463 464 465 466 467 468 469 470 471 472 473 474 475
	unsigned long flags;

	/*
	 * In order to avoid lock contention while migrating pages concurrently
	 * to leak_balloon() or fill_balloon() we just give up the balloon_lock
	 * this turn, as it is easier to retry the page migration later.
	 * This also prevents fill_balloon() getting stuck into a mutex
	 * recursion in the case it ends up triggering memory compaction
	 * while it is attempting to inflate the ballon.
	 */
	if (!mutex_trylock(&vb->balloon_lock))
		return -EAGAIN;

476 477
	get_page(newpage); /* balloon reference */

478 479
	/* balloon's page migration 1st step  -- inflate "newpage" */
	spin_lock_irqsave(&vb_dev_info->pages_lock, flags);
480
	balloon_page_insert(vb_dev_info, newpage);
481
	vb_dev_info->isolated_pages--;
482
	__count_vm_event(BALLOON_MIGRATE);
483 484
	spin_unlock_irqrestore(&vb_dev_info->pages_lock, flags);
	vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE;
485
	set_page_pfns(vb, vb->pfns, newpage);
486 487
	tell_host(vb, vb->inflate_vq);

488
	/* balloon's page migration 2nd step -- deflate "page" */
489 490
	balloon_page_delete(page);
	vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE;
491
	set_page_pfns(vb, vb->pfns, page);
492 493 494 495
	tell_host(vb, vb->deflate_vq);

	mutex_unlock(&vb->balloon_lock);

496 497
	put_page(page); /* balloon reference */

498
	return MIGRATEPAGE_SUCCESS;
499
}
500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517

static struct dentry *balloon_mount(struct file_system_type *fs_type,
		int flags, const char *dev_name, void *data)
{
	static const struct dentry_operations ops = {
		.d_dname = simple_dname,
	};

	return mount_pseudo(fs_type, "balloon-kvm:", NULL, &ops,
				BALLOON_KVM_MAGIC);
}

static struct file_system_type balloon_fs = {
	.name           = "balloon-kvm",
	.mount          = balloon_mount,
	.kill_sb        = kill_anon_super,
};

518 519
#endif /* CONFIG_BALLOON_COMPACTION */

520 521 522 523 524
static int virtballoon_probe(struct virtio_device *vdev)
{
	struct virtio_balloon *vb;
	int err;

525 526 527 528 529 530
	if (!vdev->config->get) {
		dev_err(&vdev->dev, "%s failure: config access disabled\n",
			__func__);
		return -EINVAL;
	}

531 532 533 534 535 536
	vdev->priv = vb = kmalloc(sizeof(*vb), GFP_KERNEL);
	if (!vb) {
		err = -ENOMEM;
		goto out;
	}

537 538
	INIT_WORK(&vb->update_balloon_stats_work, update_balloon_stats_func);
	INIT_WORK(&vb->update_balloon_size_work, update_balloon_size_func);
539 540
	spin_lock_init(&vb->stop_update_lock);
	vb->stop_update = false;
541
	vb->num_pages = 0;
542
	mutex_init(&vb->balloon_lock);
543
	init_waitqueue_head(&vb->acked);
544 545
	vb->vdev = vdev;

546
	balloon_devinfo_init(&vb->vb_dev_info);
547

548 549
	err = init_vqs(vb);
	if (err)
550
		goto out_free_vb;
R
Rusty Russell 已提交
551

552 553 554 555
	vb->nb.notifier_call = virtballoon_oom_notify;
	vb->nb.priority = VIRTBALLOON_OOM_NOTIFY_PRIORITY;
	err = register_oom_notifier(&vb->nb);
	if (err < 0)
556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576
		goto out_del_vqs;

#ifdef CONFIG_BALLOON_COMPACTION
	balloon_mnt = kern_mount(&balloon_fs);
	if (IS_ERR(balloon_mnt)) {
		err = PTR_ERR(balloon_mnt);
		unregister_oom_notifier(&vb->nb);
		goto out_del_vqs;
	}

	vb->vb_dev_info.migratepage = virtballoon_migratepage;
	vb->vb_dev_info.inode = alloc_anon_inode(balloon_mnt->mnt_sb);
	if (IS_ERR(vb->vb_dev_info.inode)) {
		err = PTR_ERR(vb->vb_dev_info.inode);
		kern_unmount(balloon_mnt);
		unregister_oom_notifier(&vb->nb);
		vb->vb_dev_info.inode = NULL;
		goto out_del_vqs;
	}
	vb->vb_dev_info.inode->i_mapping->a_ops = &balloon_aops;
#endif
577

578 579
	virtio_device_ready(vdev);

580 581
	if (towards_target(vb))
		virtballoon_changed(vdev);
R
Rusty Russell 已提交
582 583
	return 0;

584
out_del_vqs:
585
	vdev->config->del_vqs(vdev);
R
Rusty Russell 已提交
586 587 588 589 590 591
out_free_vb:
	kfree(vb);
out:
	return err;
}

592
static void remove_common(struct virtio_balloon *vb)
R
Rusty Russell 已提交
593 594 595 596
{
	/* There might be pages left in the balloon: free them. */
	while (vb->num_pages)
		leak_balloon(vb, vb->num_pages);
597
	update_balloon_size(vb);
R
Rusty Russell 已提交
598 599

	/* Now we reset the device so we can clean up the queues. */
600
	vb->vdev->config->reset(vb->vdev);
R
Rusty Russell 已提交
601

602 603 604
	vb->vdev->config->del_vqs(vb->vdev);
}

605
static void virtballoon_remove(struct virtio_device *vdev)
606 607 608
{
	struct virtio_balloon *vb = vdev->priv;

609
	unregister_oom_notifier(&vb->nb);
610 611 612 613

	spin_lock_irq(&vb->stop_update_lock);
	vb->stop_update = true;
	spin_unlock_irq(&vb->stop_update_lock);
614 615
	cancel_work_sync(&vb->update_balloon_size_work);
	cancel_work_sync(&vb->update_balloon_stats_work);
616

617
	remove_common(vb);
618 619
	if (vb->vb_dev_info.inode)
		iput(vb->vb_dev_info.inode);
R
Rusty Russell 已提交
620 621 622
	kfree(vb);
}

623
#ifdef CONFIG_PM_SLEEP
624 625
static int virtballoon_freeze(struct virtio_device *vdev)
{
626 627
	struct virtio_balloon *vb = vdev->priv;

628
	/*
629
	 * The workqueue is already frozen by the PM core before this
630 631
	 * function is called.
	 */
632
	remove_common(vb);
633 634 635
	return 0;
}

636
static int virtballoon_restore(struct virtio_device *vdev)
637 638 639 640 641 642 643 644
{
	struct virtio_balloon *vb = vdev->priv;
	int ret;

	ret = init_vqs(vdev->priv);
	if (ret)
		return ret;

645 646
	virtio_device_ready(vdev);

647 648
	if (towards_target(vb))
		virtballoon_changed(vdev);
649 650 651
	update_balloon_size(vb);
	return 0;
}
652 653
#endif

654 655 656
static unsigned int features[] = {
	VIRTIO_BALLOON_F_MUST_TELL_HOST,
	VIRTIO_BALLOON_F_STATS_VQ,
657
	VIRTIO_BALLOON_F_DEFLATE_ON_OOM,
658
};
659

660
static struct virtio_driver virtio_balloon_driver = {
661 662
	.feature_table = features,
	.feature_table_size = ARRAY_SIZE(features),
R
Rusty Russell 已提交
663 664 665 666
	.driver.name =	KBUILD_MODNAME,
	.driver.owner =	THIS_MODULE,
	.id_table =	id_table,
	.probe =	virtballoon_probe,
667
	.remove =	virtballoon_remove,
R
Rusty Russell 已提交
668
	.config_changed = virtballoon_changed,
669
#ifdef CONFIG_PM_SLEEP
670 671 672
	.freeze	=	virtballoon_freeze,
	.restore =	virtballoon_restore,
#endif
R
Rusty Russell 已提交
673 674
};

675
module_virtio_driver(virtio_balloon_driver);
R
Rusty Russell 已提交
676 677 678
MODULE_DEVICE_TABLE(virtio, id_table);
MODULE_DESCRIPTION("Virtio balloon driver");
MODULE_LICENSE("GPL");