virtio_balloon.c 11.6 KB
Newer Older
S
Sasha Levin 已提交
1 2
/*
 * Virtio balloon implementation, inspired by Dor Laor and Marcelo
R
Rusty Russell 已提交
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
 * Tosatti's implementations.
 *
 *  Copyright 2008 Rusty Russell IBM Corporation
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */
S
Sasha Levin 已提交
21

R
Rusty Russell 已提交
22 23 24 25 26
#include <linux/virtio.h>
#include <linux/virtio_balloon.h>
#include <linux/swap.h>
#include <linux/kthread.h>
#include <linux/freezer.h>
27
#include <linux/delay.h>
28
#include <linux/slab.h>
29
#include <linux/module.h>
R
Rusty Russell 已提交
30

31 32 33 34 35 36 37
/*
 * Balloon device works in 4K page units.  So each page is pointed to by
 * multiple balloon pages.  All memory counters in this driver are in balloon
 * page units.
 */
#define VIRTIO_BALLOON_PAGES_PER_PAGE (PAGE_SIZE >> VIRTIO_BALLOON_PFN_SHIFT)

R
Rusty Russell 已提交
38 39 40
struct virtio_balloon
{
	struct virtio_device *vdev;
41
	struct virtqueue *inflate_vq, *deflate_vq, *stats_vq;
R
Rusty Russell 已提交
42 43 44 45 46 47 48 49

	/* Where the ballooning thread waits for config to change. */
	wait_queue_head_t config_change;

	/* The thread servicing the balloon. */
	struct task_struct *thread;

	/* Waiting for host to ack the pages we released. */
50
	wait_queue_head_t acked;
R
Rusty Russell 已提交
51

52
	/* Number of balloon pages we've told the Host we're not using. */
R
Rusty Russell 已提交
53
	unsigned int num_pages;
54 55 56 57 58
	/*
	 * The pages we've told the Host we're not using.
	 * Each page on this list adds VIRTIO_BALLOON_PAGES_PER_PAGE
	 * to num_pages above.
	 */
R
Rusty Russell 已提交
59 60 61 62 63
	struct list_head pages;

	/* The array of pfns we tell the Host about. */
	unsigned int num_pfns;
	u32 pfns[256];
64 65

	/* Memory statistics */
66
	int need_stats_update;
67
	struct virtio_balloon_stat stats[VIRTIO_BALLOON_S_NR];
R
Rusty Russell 已提交
68 69 70 71 72 73 74
};

static struct virtio_device_id id_table[] = {
	{ VIRTIO_ID_BALLOON, VIRTIO_DEV_ANY_ID },
	{ 0 },
};

75 76 77 78 79 80
static u32 page_to_balloon_pfn(struct page *page)
{
	unsigned long pfn = page_to_pfn(page);

	BUILD_BUG_ON(PAGE_SHIFT < VIRTIO_BALLOON_PFN_SHIFT);
	/* Convert pfn from Linux page size to balloon page size. */
81 82 83 84 85 86 87
	return pfn * VIRTIO_BALLOON_PAGES_PER_PAGE;
}

static struct page *balloon_pfn_to_page(u32 pfn)
{
	BUG_ON(pfn % VIRTIO_BALLOON_PAGES_PER_PAGE);
	return pfn_to_page(pfn / VIRTIO_BALLOON_PAGES_PER_PAGE);
88 89
}

R
Rusty Russell 已提交
90 91
static void balloon_ack(struct virtqueue *vq)
{
92
	struct virtio_balloon *vb = vq->vdev->priv;
R
Rusty Russell 已提交
93

94
	wake_up(&vb->acked);
R
Rusty Russell 已提交
95 96 97 98 99
}

static void tell_host(struct virtio_balloon *vb, struct virtqueue *vq)
{
	struct scatterlist sg;
100
	unsigned int len;
R
Rusty Russell 已提交
101 102 103 104

	sg_init_one(&sg, vb->pfns, sizeof(vb->pfns[0]) * vb->num_pfns);

	/* We should always be able to add one buffer to an empty queue. */
105
	if (virtqueue_add_buf(vq, &sg, 1, 0, vb, GFP_KERNEL) < 0)
R
Rusty Russell 已提交
106
		BUG();
107
	virtqueue_kick(vq);
R
Rusty Russell 已提交
108 109

	/* When host has read buffer, this completes via balloon_ack */
110
	wait_event(vb->acked, virtqueue_get_buf(vq, &len));
R
Rusty Russell 已提交
111 112
}

113 114 115 116 117 118 119 120 121 122
static void set_page_pfns(u32 pfns[], struct page *page)
{
	unsigned int i;

	/* Set balloon pfns pointing at this page.
	 * Note that the first pfn points at start of the page. */
	for (i = 0; i < VIRTIO_BALLOON_PAGES_PER_PAGE; i++)
		pfns[i] = page_to_balloon_pfn(page) + i;
}

R
Rusty Russell 已提交
123 124 125 126 127
static void fill_balloon(struct virtio_balloon *vb, size_t num)
{
	/* We can only do one array worth at a time. */
	num = min(num, ARRAY_SIZE(vb->pfns));

128 129
	for (vb->num_pfns = 0; vb->num_pfns < num;
	     vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) {
130 131
		struct page *page = alloc_page(GFP_HIGHUSER | __GFP_NORETRY |
					__GFP_NOMEMALLOC | __GFP_NOWARN);
R
Rusty Russell 已提交
132 133 134 135 136 137 138 139 140
		if (!page) {
			if (printk_ratelimit())
				dev_printk(KERN_INFO, &vb->vdev->dev,
					   "Out of puff! Can't get %zu pages\n",
					   num);
			/* Sleep for at least 1/5 of a second before retry. */
			msleep(200);
			break;
		}
141 142
		set_page_pfns(vb->pfns + vb->num_pfns, page);
		vb->num_pages += VIRTIO_BALLOON_PAGES_PER_PAGE;
R
Rusty Russell 已提交
143 144 145 146 147 148 149 150 151 152 153 154 155 156 157
		totalram_pages--;
		list_add(&page->lru, &vb->pages);
	}

	/* Didn't get any?  Oh well. */
	if (vb->num_pfns == 0)
		return;

	tell_host(vb, vb->inflate_vq);
}

static void release_pages_by_pfn(const u32 pfns[], unsigned int num)
{
	unsigned int i;

158 159 160
	/* Find pfns pointing at start of each page, get pages and free them. */
	for (i = 0; i < num; i += VIRTIO_BALLOON_PAGES_PER_PAGE) {
		__free_page(balloon_pfn_to_page(pfns[i]));
R
Rusty Russell 已提交
161 162 163 164 165 166 167 168 169 170 171
		totalram_pages++;
	}
}

static void leak_balloon(struct virtio_balloon *vb, size_t num)
{
	struct page *page;

	/* We can only do one array worth at a time. */
	num = min(num, ARRAY_SIZE(vb->pfns));

172 173
	for (vb->num_pfns = 0; vb->num_pfns < num;
	     vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) {
R
Rusty Russell 已提交
174 175
		page = list_first_entry(&vb->pages, struct page, lru);
		list_del(&page->lru);
176 177
		set_page_pfns(vb->pfns + vb->num_pfns, page);
		vb->num_pages -= VIRTIO_BALLOON_PAGES_PER_PAGE;
R
Rusty Russell 已提交
178 179
	}

180 181 182 183 184 185 186
	/*
	 * Note that if
	 * virtio_has_feature(vdev, VIRTIO_BALLOON_F_MUST_TELL_HOST);
	 * is true, we *have* to do it in this order
	 */
	tell_host(vb, vb->deflate_vq);
	release_pages_by_pfn(vb->pfns, vb->num_pfns);
R
Rusty Russell 已提交
187 188
}

189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224
static inline void update_stat(struct virtio_balloon *vb, int idx,
			       u16 tag, u64 val)
{
	BUG_ON(idx >= VIRTIO_BALLOON_S_NR);
	vb->stats[idx].tag = tag;
	vb->stats[idx].val = val;
}

#define pages_to_bytes(x) ((u64)(x) << PAGE_SHIFT)

static void update_balloon_stats(struct virtio_balloon *vb)
{
	unsigned long events[NR_VM_EVENT_ITEMS];
	struct sysinfo i;
	int idx = 0;

	all_vm_events(events);
	si_meminfo(&i);

	update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_IN,
				pages_to_bytes(events[PSWPIN]));
	update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_OUT,
				pages_to_bytes(events[PSWPOUT]));
	update_stat(vb, idx++, VIRTIO_BALLOON_S_MAJFLT, events[PGMAJFAULT]);
	update_stat(vb, idx++, VIRTIO_BALLOON_S_MINFLT, events[PGFAULT]);
	update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMFREE,
				pages_to_bytes(i.freeram));
	update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMTOT,
				pages_to_bytes(i.totalram));
}

/*
 * While most virtqueues communicate guest-initiated requests to the hypervisor,
 * the stats queue operates in reverse.  The driver initializes the virtqueue
 * with a single buffer.  From that point forward, all conversations consist of
 * a hypervisor request (a call to this function) which directs us to refill
225 226
 * the virtqueue with a fresh stats buffer.  Since stats collection can sleep,
 * we notify our kthread which does the actual work via stats_handle_request().
227
 */
228
static void stats_request(struct virtqueue *vq)
229
{
230
	struct virtio_balloon *vb = vq->vdev->priv;
231

232 233 234 235 236 237 238 239
	vb->need_stats_update = 1;
	wake_up(&vb->config_change);
}

static void stats_handle_request(struct virtio_balloon *vb)
{
	struct virtqueue *vq;
	struct scatterlist sg;
240
	unsigned int len;
241

242
	vb->need_stats_update = 0;
243 244
	update_balloon_stats(vb);

245
	vq = vb->stats_vq;
246 247
	if (!virtqueue_get_buf(vq, &len))
		return;
248
	sg_init_one(&sg, vb->stats, sizeof(vb->stats));
249
	if (virtqueue_add_buf(vq, &sg, 1, 0, vb, GFP_KERNEL) < 0)
250
		BUG();
251
	virtqueue_kick(vq);
252 253
}

R
Rusty Russell 已提交
254 255 256 257 258 259 260
static void virtballoon_changed(struct virtio_device *vdev)
{
	struct virtio_balloon *vb = vdev->priv;

	wake_up(&vb->config_change);
}

261
static inline s64 towards_target(struct virtio_balloon *vb)
R
Rusty Russell 已提交
262
{
D
David Gibson 已提交
263 264 265
	__le32 v;
	s64 target;

266 267 268
	vb->vdev->config->get(vb->vdev,
			      offsetof(struct virtio_balloon_config, num_pages),
			      &v, sizeof(v));
D
David Gibson 已提交
269 270
	target = le32_to_cpu(v);
	return target - vb->num_pages;
R
Rusty Russell 已提交
271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287
}

static void update_balloon_size(struct virtio_balloon *vb)
{
	__le32 actual = cpu_to_le32(vb->num_pages);

	vb->vdev->config->set(vb->vdev,
			      offsetof(struct virtio_balloon_config, actual),
			      &actual, sizeof(actual));
}

static int balloon(void *_vballoon)
{
	struct virtio_balloon *vb = _vballoon;

	set_freezable();
	while (!kthread_should_stop()) {
288
		s64 diff;
R
Rusty Russell 已提交
289 290 291 292

		try_to_freeze();
		wait_event_interruptible(vb->config_change,
					 (diff = towards_target(vb)) != 0
293
					 || vb->need_stats_update
294 295
					 || kthread_should_stop()
					 || freezing(current));
296 297
		if (vb->need_stats_update)
			stats_handle_request(vb);
R
Rusty Russell 已提交
298 299 300 301 302 303 304 305 306
		if (diff > 0)
			fill_balloon(vb, diff);
		else if (diff < 0)
			leak_balloon(vb, -diff);
		update_balloon_size(vb);
	}
	return 0;
}

307
static int init_vqs(struct virtio_balloon *vb)
R
Rusty Russell 已提交
308
{
309
	struct virtqueue *vqs[3];
310
	vq_callback_t *callbacks[] = { balloon_ack, balloon_ack, stats_request };
311 312
	const char *names[] = { "inflate", "deflate", "stats" };
	int err, nvqs;
R
Rusty Russell 已提交
313

314 315 316 317
	/*
	 * We expect two virtqueues: inflate and deflate, and
	 * optionally stat.
	 */
318
	nvqs = virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ) ? 3 : 2;
319
	err = vb->vdev->config->find_vqs(vb->vdev, nvqs, vqs, callbacks, names);
320
	if (err)
321
		return err;
R
Rusty Russell 已提交
322

323 324
	vb->inflate_vq = vqs[0];
	vb->deflate_vq = vqs[1];
325 326 327 328 329 330 331 332 333
	if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) {
		struct scatterlist sg;
		vb->stats_vq = vqs[2];

		/*
		 * Prime this virtqueue with one buffer so the hypervisor can
		 * use it to signal us later.
		 */
		sg_init_one(&sg, vb->stats, sizeof vb->stats);
334 335
		if (virtqueue_add_buf(vb->stats_vq, &sg, 1, 0, vb, GFP_KERNEL)
		    < 0)
336
			BUG();
337
		virtqueue_kick(vb->stats_vq);
338
	}
339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355
	return 0;
}

static int virtballoon_probe(struct virtio_device *vdev)
{
	struct virtio_balloon *vb;
	int err;

	vdev->priv = vb = kmalloc(sizeof(*vb), GFP_KERNEL);
	if (!vb) {
		err = -ENOMEM;
		goto out;
	}

	INIT_LIST_HEAD(&vb->pages);
	vb->num_pages = 0;
	init_waitqueue_head(&vb->config_change);
356
	init_waitqueue_head(&vb->acked);
357 358 359 360 361 362
	vb->vdev = vdev;
	vb->need_stats_update = 0;

	err = init_vqs(vb);
	if (err)
		goto out_free_vb;
R
Rusty Russell 已提交
363 364 365 366

	vb->thread = kthread_run(balloon, vb, "vballoon");
	if (IS_ERR(vb->thread)) {
		err = PTR_ERR(vb->thread);
367
		goto out_del_vqs;
R
Rusty Russell 已提交
368 369 370 371
	}

	return 0;

372 373
out_del_vqs:
	vdev->config->del_vqs(vdev);
R
Rusty Russell 已提交
374 375 376 377 378 379
out_free_vb:
	kfree(vb);
out:
	return err;
}

380
static void remove_common(struct virtio_balloon *vb)
R
Rusty Russell 已提交
381 382 383 384
{
	/* There might be pages left in the balloon: free them. */
	while (vb->num_pages)
		leak_balloon(vb, vb->num_pages);
385
	update_balloon_size(vb);
R
Rusty Russell 已提交
386 387

	/* Now we reset the device so we can clean up the queues. */
388
	vb->vdev->config->reset(vb->vdev);
R
Rusty Russell 已提交
389

390 391 392 393 394 395 396 397 398
	vb->vdev->config->del_vqs(vb->vdev);
}

static void __devexit virtballoon_remove(struct virtio_device *vdev)
{
	struct virtio_balloon *vb = vdev->priv;

	kthread_stop(vb->thread);
	remove_common(vb);
R
Rusty Russell 已提交
399 400 401
	kfree(vb);
}

402 403 404
#ifdef CONFIG_PM
static int virtballoon_freeze(struct virtio_device *vdev)
{
405 406
	struct virtio_balloon *vb = vdev->priv;

407 408 409 410 411
	/*
	 * The kthread is already frozen by the PM core before this
	 * function is called.
	 */

412
	remove_common(vb);
413 414 415
	return 0;
}

416
static int virtballoon_restore(struct virtio_device *vdev)
417 418 419 420 421 422 423 424 425 426 427 428
{
	struct virtio_balloon *vb = vdev->priv;
	int ret;

	ret = init_vqs(vdev->priv);
	if (ret)
		return ret;

	fill_balloon(vb, towards_target(vb));
	update_balloon_size(vb);
	return 0;
}
429 430
#endif

431 432 433 434
static unsigned int features[] = {
	VIRTIO_BALLOON_F_MUST_TELL_HOST,
	VIRTIO_BALLOON_F_STATS_VQ,
};
435

436
static struct virtio_driver virtio_balloon_driver = {
437 438
	.feature_table = features,
	.feature_table_size = ARRAY_SIZE(features),
R
Rusty Russell 已提交
439 440 441 442 443 444
	.driver.name =	KBUILD_MODNAME,
	.driver.owner =	THIS_MODULE,
	.id_table =	id_table,
	.probe =	virtballoon_probe,
	.remove =	__devexit_p(virtballoon_remove),
	.config_changed = virtballoon_changed,
445 446 447 448
#ifdef CONFIG_PM
	.freeze	=	virtballoon_freeze,
	.restore =	virtballoon_restore,
#endif
R
Rusty Russell 已提交
449 450 451 452
};

static int __init init(void)
{
453
	return register_virtio_driver(&virtio_balloon_driver);
R
Rusty Russell 已提交
454 455 456 457
}

static void __exit fini(void)
{
458
	unregister_virtio_driver(&virtio_balloon_driver);
R
Rusty Russell 已提交
459 460 461 462 463 464 465
}
module_init(init);
module_exit(fini);

MODULE_DEVICE_TABLE(virtio, id_table);
MODULE_DESCRIPTION("Virtio balloon driver");
MODULE_LICENSE("GPL");