virtio_balloon.c 11.8 KB
Newer Older
S
Sasha Levin 已提交
1 2
/*
 * Virtio balloon implementation, inspired by Dor Laor and Marcelo
R
Rusty Russell 已提交
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
 * Tosatti's implementations.
 *
 *  Copyright 2008 Rusty Russell IBM Corporation
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */
S
Sasha Levin 已提交
21

R
Rusty Russell 已提交
22 23 24 25 26
#include <linux/virtio.h>
#include <linux/virtio_balloon.h>
#include <linux/swap.h>
#include <linux/kthread.h>
#include <linux/freezer.h>
27
#include <linux/delay.h>
28
#include <linux/slab.h>
29
#include <linux/module.h>
R
Rusty Russell 已提交
30

31 32 33 34 35 36 37
/*
 * Balloon device works in 4K page units.  So each page is pointed to by
 * multiple balloon pages.  All memory counters in this driver are in balloon
 * page units.
 */
#define VIRTIO_BALLOON_PAGES_PER_PAGE (PAGE_SIZE >> VIRTIO_BALLOON_PFN_SHIFT)

R
Rusty Russell 已提交
38 39 40
struct virtio_balloon
{
	struct virtio_device *vdev;
41
	struct virtqueue *inflate_vq, *deflate_vq, *stats_vq;
R
Rusty Russell 已提交
42 43 44 45 46 47 48 49 50 51

	/* Where the ballooning thread waits for config to change. */
	wait_queue_head_t config_change;

	/* The thread servicing the balloon. */
	struct task_struct *thread;

	/* Waiting for host to ack the pages we released. */
	struct completion acked;

52
	/* Number of balloon pages we've told the Host we're not using. */
R
Rusty Russell 已提交
53
	unsigned int num_pages;
54 55 56 57 58
	/*
	 * The pages we've told the Host we're not using.
	 * Each page on this list adds VIRTIO_BALLOON_PAGES_PER_PAGE
	 * to num_pages above.
	 */
R
Rusty Russell 已提交
59 60 61 62 63
	struct list_head pages;

	/* The array of pfns we tell the Host about. */
	unsigned int num_pfns;
	u32 pfns[256];
64 65

	/* Memory statistics */
66
	int need_stats_update;
67
	struct virtio_balloon_stat stats[VIRTIO_BALLOON_S_NR];
R
Rusty Russell 已提交
68 69 70 71 72 73 74
};

static struct virtio_device_id id_table[] = {
	{ VIRTIO_ID_BALLOON, VIRTIO_DEV_ANY_ID },
	{ 0 },
};

75 76 77 78 79 80
static u32 page_to_balloon_pfn(struct page *page)
{
	unsigned long pfn = page_to_pfn(page);

	BUILD_BUG_ON(PAGE_SHIFT < VIRTIO_BALLOON_PFN_SHIFT);
	/* Convert pfn from Linux page size to balloon page size. */
81 82 83 84 85 86 87
	return pfn * VIRTIO_BALLOON_PAGES_PER_PAGE;
}

static struct page *balloon_pfn_to_page(u32 pfn)
{
	BUG_ON(pfn % VIRTIO_BALLOON_PAGES_PER_PAGE);
	return pfn_to_page(pfn / VIRTIO_BALLOON_PAGES_PER_PAGE);
88 89
}

R
Rusty Russell 已提交
90 91 92 93 94
static void balloon_ack(struct virtqueue *vq)
{
	struct virtio_balloon *vb;
	unsigned int len;

95
	vb = virtqueue_get_buf(vq, &len);
R
Rusty Russell 已提交
96 97 98 99 100 101 102 103 104 105 106 107 108
	if (vb)
		complete(&vb->acked);
}

static void tell_host(struct virtio_balloon *vb, struct virtqueue *vq)
{
	struct scatterlist sg;

	sg_init_one(&sg, vb->pfns, sizeof(vb->pfns[0]) * vb->num_pfns);

	init_completion(&vb->acked);

	/* We should always be able to add one buffer to an empty queue. */
109
	if (virtqueue_add_buf(vq, &sg, 1, 0, vb, GFP_KERNEL) < 0)
R
Rusty Russell 已提交
110
		BUG();
111
	virtqueue_kick(vq);
R
Rusty Russell 已提交
112 113 114 115 116

	/* When host has read buffer, this completes via balloon_ack */
	wait_for_completion(&vb->acked);
}

117 118 119 120 121 122 123 124 125 126
static void set_page_pfns(u32 pfns[], struct page *page)
{
	unsigned int i;

	/* Set balloon pfns pointing at this page.
	 * Note that the first pfn points at start of the page. */
	for (i = 0; i < VIRTIO_BALLOON_PAGES_PER_PAGE; i++)
		pfns[i] = page_to_balloon_pfn(page) + i;
}

R
Rusty Russell 已提交
127 128 129 130 131
static void fill_balloon(struct virtio_balloon *vb, size_t num)
{
	/* We can only do one array worth at a time. */
	num = min(num, ARRAY_SIZE(vb->pfns));

132 133
	for (vb->num_pfns = 0; vb->num_pfns < num;
	     vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) {
134 135
		struct page *page = alloc_page(GFP_HIGHUSER | __GFP_NORETRY |
					__GFP_NOMEMALLOC | __GFP_NOWARN);
R
Rusty Russell 已提交
136 137 138 139 140 141 142 143 144
		if (!page) {
			if (printk_ratelimit())
				dev_printk(KERN_INFO, &vb->vdev->dev,
					   "Out of puff! Can't get %zu pages\n",
					   num);
			/* Sleep for at least 1/5 of a second before retry. */
			msleep(200);
			break;
		}
145 146
		set_page_pfns(vb->pfns + vb->num_pfns, page);
		vb->num_pages += VIRTIO_BALLOON_PAGES_PER_PAGE;
R
Rusty Russell 已提交
147 148 149 150 151 152 153 154 155 156 157 158 159 160 161
		totalram_pages--;
		list_add(&page->lru, &vb->pages);
	}

	/* Didn't get any?  Oh well. */
	if (vb->num_pfns == 0)
		return;

	tell_host(vb, vb->inflate_vq);
}

static void release_pages_by_pfn(const u32 pfns[], unsigned int num)
{
	unsigned int i;

162 163 164
	/* Find pfns pointing at start of each page, get pages and free them. */
	for (i = 0; i < num; i += VIRTIO_BALLOON_PAGES_PER_PAGE) {
		__free_page(balloon_pfn_to_page(pfns[i]));
R
Rusty Russell 已提交
165 166 167 168 169 170 171 172 173 174 175
		totalram_pages++;
	}
}

static void leak_balloon(struct virtio_balloon *vb, size_t num)
{
	struct page *page;

	/* We can only do one array worth at a time. */
	num = min(num, ARRAY_SIZE(vb->pfns));

176 177
	for (vb->num_pfns = 0; vb->num_pfns < num;
	     vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) {
R
Rusty Russell 已提交
178 179
		page = list_first_entry(&vb->pages, struct page, lru);
		list_del(&page->lru);
180 181
		set_page_pfns(vb->pfns + vb->num_pfns, page);
		vb->num_pages -= VIRTIO_BALLOON_PAGES_PER_PAGE;
R
Rusty Russell 已提交
182 183
	}

184 185 186 187 188 189 190
	/*
	 * Note that if
	 * virtio_has_feature(vdev, VIRTIO_BALLOON_F_MUST_TELL_HOST);
	 * is true, we *have* to do it in this order
	 */
	tell_host(vb, vb->deflate_vq);
	release_pages_by_pfn(vb->pfns, vb->num_pfns);
R
Rusty Russell 已提交
191 192
}

193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228
static inline void update_stat(struct virtio_balloon *vb, int idx,
			       u16 tag, u64 val)
{
	BUG_ON(idx >= VIRTIO_BALLOON_S_NR);
	vb->stats[idx].tag = tag;
	vb->stats[idx].val = val;
}

#define pages_to_bytes(x) ((u64)(x) << PAGE_SHIFT)

static void update_balloon_stats(struct virtio_balloon *vb)
{
	unsigned long events[NR_VM_EVENT_ITEMS];
	struct sysinfo i;
	int idx = 0;

	all_vm_events(events);
	si_meminfo(&i);

	update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_IN,
				pages_to_bytes(events[PSWPIN]));
	update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_OUT,
				pages_to_bytes(events[PSWPOUT]));
	update_stat(vb, idx++, VIRTIO_BALLOON_S_MAJFLT, events[PGMAJFAULT]);
	update_stat(vb, idx++, VIRTIO_BALLOON_S_MINFLT, events[PGFAULT]);
	update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMFREE,
				pages_to_bytes(i.freeram));
	update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMTOT,
				pages_to_bytes(i.totalram));
}

/*
 * While most virtqueues communicate guest-initiated requests to the hypervisor,
 * the stats queue operates in reverse.  The driver initializes the virtqueue
 * with a single buffer.  From that point forward, all conversations consist of
 * a hypervisor request (a call to this function) which directs us to refill
229 230
 * the virtqueue with a fresh stats buffer.  Since stats collection can sleep,
 * we notify our kthread which does the actual work via stats_handle_request().
231
 */
232
static void stats_request(struct virtqueue *vq)
233 234 235 236
{
	struct virtio_balloon *vb;
	unsigned int len;

237
	vb = virtqueue_get_buf(vq, &len);
238 239
	if (!vb)
		return;
240 241 242 243 244 245 246 247
	vb->need_stats_update = 1;
	wake_up(&vb->config_change);
}

static void stats_handle_request(struct virtio_balloon *vb)
{
	struct virtqueue *vq;
	struct scatterlist sg;
248

249
	vb->need_stats_update = 0;
250 251
	update_balloon_stats(vb);

252
	vq = vb->stats_vq;
253
	sg_init_one(&sg, vb->stats, sizeof(vb->stats));
254
	if (virtqueue_add_buf(vq, &sg, 1, 0, vb, GFP_KERNEL) < 0)
255
		BUG();
256
	virtqueue_kick(vq);
257 258
}

R
Rusty Russell 已提交
259 260 261 262 263 264 265
static void virtballoon_changed(struct virtio_device *vdev)
{
	struct virtio_balloon *vb = vdev->priv;

	wake_up(&vb->config_change);
}

266
static inline s64 towards_target(struct virtio_balloon *vb)
R
Rusty Russell 已提交
267
{
D
David Gibson 已提交
268 269 270
	__le32 v;
	s64 target;

271 272 273
	vb->vdev->config->get(vb->vdev,
			      offsetof(struct virtio_balloon_config, num_pages),
			      &v, sizeof(v));
D
David Gibson 已提交
274 275
	target = le32_to_cpu(v);
	return target - vb->num_pages;
R
Rusty Russell 已提交
276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292
}

static void update_balloon_size(struct virtio_balloon *vb)
{
	__le32 actual = cpu_to_le32(vb->num_pages);

	vb->vdev->config->set(vb->vdev,
			      offsetof(struct virtio_balloon_config, actual),
			      &actual, sizeof(actual));
}

static int balloon(void *_vballoon)
{
	struct virtio_balloon *vb = _vballoon;

	set_freezable();
	while (!kthread_should_stop()) {
293
		s64 diff;
R
Rusty Russell 已提交
294 295 296 297

		try_to_freeze();
		wait_event_interruptible(vb->config_change,
					 (diff = towards_target(vb)) != 0
298
					 || vb->need_stats_update
299 300
					 || kthread_should_stop()
					 || freezing(current));
301 302
		if (vb->need_stats_update)
			stats_handle_request(vb);
R
Rusty Russell 已提交
303 304 305 306 307 308 309 310 311
		if (diff > 0)
			fill_balloon(vb, diff);
		else if (diff < 0)
			leak_balloon(vb, -diff);
		update_balloon_size(vb);
	}
	return 0;
}

312
static int init_vqs(struct virtio_balloon *vb)
R
Rusty Russell 已提交
313
{
314
	struct virtqueue *vqs[3];
315
	vq_callback_t *callbacks[] = { balloon_ack, balloon_ack, stats_request };
316 317
	const char *names[] = { "inflate", "deflate", "stats" };
	int err, nvqs;
R
Rusty Russell 已提交
318

319 320 321 322
	/*
	 * We expect two virtqueues: inflate and deflate, and
	 * optionally stat.
	 */
323
	nvqs = virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ) ? 3 : 2;
324
	err = vb->vdev->config->find_vqs(vb->vdev, nvqs, vqs, callbacks, names);
325
	if (err)
326
		return err;
R
Rusty Russell 已提交
327

328 329
	vb->inflate_vq = vqs[0];
	vb->deflate_vq = vqs[1];
330 331 332 333 334 335 336 337 338
	if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) {
		struct scatterlist sg;
		vb->stats_vq = vqs[2];

		/*
		 * Prime this virtqueue with one buffer so the hypervisor can
		 * use it to signal us later.
		 */
		sg_init_one(&sg, vb->stats, sizeof vb->stats);
339 340
		if (virtqueue_add_buf(vb->stats_vq, &sg, 1, 0, vb, GFP_KERNEL)
		    < 0)
341
			BUG();
342
		virtqueue_kick(vb->stats_vq);
343
	}
344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366
	return 0;
}

static int virtballoon_probe(struct virtio_device *vdev)
{
	struct virtio_balloon *vb;
	int err;

	vdev->priv = vb = kmalloc(sizeof(*vb), GFP_KERNEL);
	if (!vb) {
		err = -ENOMEM;
		goto out;
	}

	INIT_LIST_HEAD(&vb->pages);
	vb->num_pages = 0;
	init_waitqueue_head(&vb->config_change);
	vb->vdev = vdev;
	vb->need_stats_update = 0;

	err = init_vqs(vb);
	if (err)
		goto out_free_vb;
R
Rusty Russell 已提交
367 368 369 370

	vb->thread = kthread_run(balloon, vb, "vballoon");
	if (IS_ERR(vb->thread)) {
		err = PTR_ERR(vb->thread);
371
		goto out_del_vqs;
R
Rusty Russell 已提交
372 373 374 375
	}

	return 0;

376 377
out_del_vqs:
	vdev->config->del_vqs(vdev);
R
Rusty Russell 已提交
378 379 380 381 382 383
out_free_vb:
	kfree(vb);
out:
	return err;
}

384
static void __devexit virtballoon_remove(struct virtio_device *vdev)
R
Rusty Russell 已提交
385 386 387 388 389 390 391 392 393 394 395 396
{
	struct virtio_balloon *vb = vdev->priv;

	kthread_stop(vb->thread);

	/* There might be pages left in the balloon: free them. */
	while (vb->num_pages)
		leak_balloon(vb, vb->num_pages);

	/* Now we reset the device so we can clean up the queues. */
	vdev->config->reset(vdev);

397
	vdev->config->del_vqs(vdev);
R
Rusty Russell 已提交
398 399 400
	kfree(vb);
}

401 402 403
#ifdef CONFIG_PM
static int virtballoon_freeze(struct virtio_device *vdev)
{
404 405
	struct virtio_balloon *vb = vdev->priv;

406 407 408 409 410
	/*
	 * The kthread is already frozen by the PM core before this
	 * function is called.
	 */

411 412 413 414
	while (vb->num_pages)
		leak_balloon(vb, vb->num_pages);
	update_balloon_size(vb);

415 416 417 418 419 420
	/* Ensure we don't get any more requests from the host */
	vdev->config->reset(vdev);
	vdev->config->del_vqs(vdev);
	return 0;
}

421 422 423 424 425 426 427 428 429 430 431 432 433 434
static int restore_common(struct virtio_device *vdev)
{
	struct virtio_balloon *vb = vdev->priv;
	int ret;

	ret = init_vqs(vdev->priv);
	if (ret)
		return ret;

	fill_balloon(vb, towards_target(vb));
	update_balloon_size(vb);
	return 0;
}

435 436
static int virtballoon_restore(struct virtio_device *vdev)
{
437
	return restore_common(vdev);
438 439 440
}
#endif

441 442 443 444
static unsigned int features[] = {
	VIRTIO_BALLOON_F_MUST_TELL_HOST,
	VIRTIO_BALLOON_F_STATS_VQ,
};
445

446
static struct virtio_driver virtio_balloon_driver = {
447 448
	.feature_table = features,
	.feature_table_size = ARRAY_SIZE(features),
R
Rusty Russell 已提交
449 450 451 452 453 454
	.driver.name =	KBUILD_MODNAME,
	.driver.owner =	THIS_MODULE,
	.id_table =	id_table,
	.probe =	virtballoon_probe,
	.remove =	__devexit_p(virtballoon_remove),
	.config_changed = virtballoon_changed,
455 456 457 458
#ifdef CONFIG_PM
	.freeze	=	virtballoon_freeze,
	.restore =	virtballoon_restore,
#endif
R
Rusty Russell 已提交
459 460 461 462
};

static int __init init(void)
{
463
	return register_virtio_driver(&virtio_balloon_driver);
R
Rusty Russell 已提交
464 465 466 467
}

static void __exit fini(void)
{
468
	unregister_virtio_driver(&virtio_balloon_driver);
R
Rusty Russell 已提交
469 470 471 472 473 474 475
}
module_init(init);
module_exit(fini);

MODULE_DEVICE_TABLE(virtio, id_table);
MODULE_DESCRIPTION("Virtio balloon driver");
MODULE_LICENSE("GPL");