xen-blkfront.c 71.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39
/*
 * blkfront.c
 *
 * XenLinux virtual block device driver.
 *
 * Copyright (c) 2003-2004, Keir Fraser & Steve Hand
 * Modifications by Mark A. Williamson are (c) Intel Research Cambridge
 * Copyright (c) 2004, Christian Limpach
 * Copyright (c) 2004, Andrew Warfield
 * Copyright (c) 2005, Christopher Clark
 * Copyright (c) 2005, XenSource Ltd
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License version 2
 * as published by the Free Software Foundation; or, when distributed
 * separately from the Linux kernel or incorporated into other
 * software packages, subject to the following license:
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this source file (the "Software"), to deal in the Software without
 * restriction, including without limitation the rights to use, copy, modify,
 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
 * and to permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include <linux/interrupt.h>
#include <linux/blkdev.h>
B
Bob Liu 已提交
40
#include <linux/blk-mq.h>
41
#include <linux/hdreg.h>
42
#include <linux/cdrom.h>
43
#include <linux/module.h>
44
#include <linux/slab.h>
45
#include <linux/mutex.h>
46
#include <linux/scatterlist.h>
47
#include <linux/bitmap.h>
48
#include <linux/list.h>
49

50
#include <xen/xen.h>
51 52 53 54
#include <xen/xenbus.h>
#include <xen/grant_table.h>
#include <xen/events.h>
#include <xen/page.h>
55
#include <xen/platform_pci.h>
56 57 58

#include <xen/interface/grant_table.h>
#include <xen/interface/io/blkif.h>
59
#include <xen/interface/io/protocols.h>
60 61 62

#include <asm/xen/hypervisor.h>

63 64 65 66 67 68 69 70 71 72 73 74 75 76
/*
 * The minimal size of segment supported by the block framework is PAGE_SIZE.
 * When Linux is using a different page size than Xen, it may not be possible
 * to put all the data in a single segment.
 * This can happen when the backend doesn't support indirect descriptor and
 * therefore the maximum amount of data that a request can carry is
 * BLKIF_MAX_SEGMENTS_PER_REQUEST * XEN_PAGE_SIZE = 44KB
 *
 * Note that we only support one extra request. So the Linux page size
 * should be <= ( 2 * BLKIF_MAX_SEGMENTS_PER_REQUEST * XEN_PAGE_SIZE) =
 * 88KB.
 */
#define HAS_EXTRA_REQ (BLKIF_MAX_SEGMENTS_PER_REQUEST < XEN_PFN_PER_PAGE)

77 78 79 80 81 82
enum blkif_state {
	BLKIF_STATE_DISCONNECTED,
	BLKIF_STATE_CONNECTED,
	BLKIF_STATE_SUSPENDED,
};

83 84
struct grant {
	grant_ref_t gref;
85
	struct page *page;
86
	struct list_head node;
87 88
};

89 90 91 92 93 94 95
enum blk_req_status {
	REQ_WAITING,
	REQ_DONE,
	REQ_ERROR,
	REQ_EOPNOTSUPP,
};

96 97
struct blk_shadow {
	struct blkif_request req;
98
	struct request *request;
99 100
	struct grant **grants_used;
	struct grant **indirect_grants;
101
	struct scatterlist *sg;
102
	unsigned int num_sg;
103 104 105 106 107 108 109 110
	enum blk_req_status status;

	#define NO_ASSOCIATED_ID ~0UL
	/*
	 * Id of the sibling if we ever need 2 requests when handling a
	 * block I/O request
	 */
	unsigned long associated_id;
111 112 113 114 115
};

struct split_bio {
	struct bio *bio;
	atomic_t pending;
116 117
};

118
static DEFINE_MUTEX(blkfront_mutex);
119
static const struct block_device_operations xlvbd_block_fops;
120

121 122 123 124 125 126 127
/*
 * Maximum number of segments in indirect requests, the actual value used by
 * the frontend driver is the minimum of this value and the value provided
 * by the backend driver.
 */

static unsigned int xen_blkif_max_segments = 32;
128 129
module_param_named(max, xen_blkif_max_segments, int, S_IRUGO);
MODULE_PARM_DESC(max, "Maximum amount of segments in indirect requests (default is 32)");
130

131 132 133 134
static unsigned int xen_blkif_max_queues = 4;
module_param_named(max_queues, xen_blkif_max_queues, uint, S_IRUGO);
MODULE_PARM_DESC(max_queues, "Maximum number of hardware queues/rings used per virtual disk");

B
Bob Liu 已提交
135 136 137 138 139 140 141 142
/*
 * Maximum order of pages to be used for the shared ring between front and
 * backend, 4KB page granularity is used.
 */
static unsigned int xen_blkif_max_ring_order;
module_param_named(max_ring_page_order, xen_blkif_max_ring_order, int, S_IRUGO);
MODULE_PARM_DESC(max_ring_page_order, "Maximum order of pages to be used for the shared ring");

143 144 145 146
#define BLK_RING_SIZE(info)	\
	__CONST_RING_SIZE(blkif, XEN_PAGE_SIZE * (info)->nr_ring_pages)

#define BLK_MAX_RING_SIZE	\
147
	__CONST_RING_SIZE(blkif, XEN_PAGE_SIZE * XENBUS_MAX_RING_GRANTS)
148

B
Bob Liu 已提交
149
/*
150 151
 * ring-ref%u i=(-1UL) would take 11 characters + 'ring-ref' is 8, so 19
 * characters are enough. Define to 20 to keep consistent with backend.
B
Bob Liu 已提交
152 153
 */
#define RINGREF_NAME_LEN (20)
154 155 156 157
/*
 * queue-%u would take 7 + 10(UINT_MAX) = 17 characters.
 */
#define QUEUE_NAME_LEN (17)
158

159 160 161 162 163 164
/*
 *  Per-ring info.
 *  Every blkfront device can associate with one or more blkfront_ring_info,
 *  depending on how many hardware queues/rings to be used.
 */
struct blkfront_ring_info {
B
Bob Liu 已提交
165 166
	/* Lock to protect data in every ring buffer. */
	spinlock_t ring_lock;
167 168 169 170 171 172 173
	struct blkif_front_ring ring;
	unsigned int ring_ref[XENBUS_MAX_RING_GRANTS];
	unsigned int evtchn, irq;
	struct work_struct work;
	struct gnttab_free_callback callback;
	struct blk_shadow shadow[BLK_MAX_RING_SIZE];
	struct list_head indirect_pages;
174 175
	struct list_head grants;
	unsigned int persistent_gnts_c;
176 177 178 179
	unsigned long shadow_free;
	struct blkfront_info *dev_info;
};

180 181 182 183 184 185 186
/*
 * We have one of these per vbd, whether ide, scsi or 'other'.  They
 * hang in private_data off the gendisk structure. We may end up
 * putting all kinds of interesting stuff here :-)
 */
struct blkfront_info
{
187
	struct mutex mutex;
188 189 190 191 192
	struct xenbus_device *xbdev;
	struct gendisk *gd;
	int vdevice;
	blkif_vdev_t handle;
	enum blkif_state connected;
193
	/* Number of pages per ring buffer. */
B
Bob Liu 已提交
194
	unsigned int nr_ring_pages;
195
	struct request_queue *rq;
196
	unsigned int feature_flush;
197 198
	unsigned int feature_discard:1;
	unsigned int feature_secdiscard:1;
199 200
	unsigned int discard_granularity;
	unsigned int discard_alignment;
201
	unsigned int feature_persistent:1;
202
	/* Number of 4KB segments handled */
203
	unsigned int max_indirect_segments;
204
	int is_ready;
B
Bob Liu 已提交
205
	struct blk_mq_tag_set tag_set;
206 207
	struct blkfront_ring_info *rinfo;
	unsigned int nr_rings;
208 209
};

210 211 212 213
static unsigned int nr_minors;
static unsigned long *minors;
static DEFINE_SPINLOCK(minor_lock);

214 215 216
#define GRANT_INVALID_REF	0

#define PARTS_PER_DISK		16
217
#define PARTS_PER_EXT_DISK      256
218 219 220 221

#define BLKIF_MAJOR(dev) ((dev)>>8)
#define BLKIF_MINOR(dev) ((dev) & 0xff)

222 223 224 225
#define EXT_SHIFT 28
#define EXTENDED (1<<EXT_SHIFT)
#define VDEV_IS_EXTENDED(dev) ((dev)&(EXTENDED))
#define BLKIF_MINOR_EXT(dev) ((dev)&(~EXTENDED))
226 227
#define EMULATED_HD_DISK_MINOR_OFFSET (0)
#define EMULATED_HD_DISK_NAME_OFFSET (EMULATED_HD_DISK_MINOR_OFFSET / 256)
228 229
#define EMULATED_SD_DISK_MINOR_OFFSET (0)
#define EMULATED_SD_DISK_NAME_OFFSET (EMULATED_SD_DISK_MINOR_OFFSET / 256)
230

231
#define DEV_NAME	"xvd"	/* name in /dev */
232

233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249
/*
 * Grants are always the same size as a Xen page (i.e 4KB).
 * A physical segment is always the same size as a Linux page.
 * Number of grants per physical segment
 */
#define GRANTS_PER_PSEG	(PAGE_SIZE / XEN_PAGE_SIZE)

#define GRANTS_PER_INDIRECT_FRAME \
	(XEN_PAGE_SIZE / sizeof(struct blkif_request_segment))

#define PSEGS_PER_INDIRECT_FRAME	\
	(GRANTS_INDIRECT_FRAME / GRANTS_PSEGS)

#define INDIRECT_GREFS(_grants)		\
	DIV_ROUND_UP(_grants, GRANTS_PER_INDIRECT_FRAME)

#define GREFS(_psegs)	((_psegs) * GRANTS_PER_PSEG)
250

251
static int blkfront_setup_indirect(struct blkfront_ring_info *rinfo);
252
static void blkfront_gather_backend_features(struct blkfront_info *info);
253

254
static int get_id_from_freelist(struct blkfront_ring_info *rinfo)
255
{
256 257 258 259 260
	unsigned long free = rinfo->shadow_free;

	BUG_ON(free >= BLK_RING_SIZE(rinfo->dev_info));
	rinfo->shadow_free = rinfo->shadow[free].req.u.rw.id;
	rinfo->shadow[free].req.u.rw.id = 0x0fffffee; /* debug */
261 262 263
	return free;
}

264
static int add_id_to_freelist(struct blkfront_ring_info *rinfo,
265
			      unsigned long id)
266
{
267
	if (rinfo->shadow[id].req.u.rw.id != id)
268
		return -EINVAL;
269
	if (rinfo->shadow[id].request == NULL)
270
		return -EINVAL;
271 272 273
	rinfo->shadow[id].req.u.rw.id  = rinfo->shadow_free;
	rinfo->shadow[id].request = NULL;
	rinfo->shadow_free = id;
274
	return 0;
275 276
}

277
static int fill_grant_buffer(struct blkfront_ring_info *rinfo, int num)
278
{
279
	struct blkfront_info *info = rinfo->dev_info;
280 281 282 283
	struct page *granted_page;
	struct grant *gnt_list_entry, *n;
	int i = 0;

284
	while (i < num) {
285 286 287 288
		gnt_list_entry = kzalloc(sizeof(struct grant), GFP_NOIO);
		if (!gnt_list_entry)
			goto out_of_memory;

289 290 291 292 293 294
		if (info->feature_persistent) {
			granted_page = alloc_page(GFP_NOIO);
			if (!granted_page) {
				kfree(gnt_list_entry);
				goto out_of_memory;
			}
295
			gnt_list_entry->page = granted_page;
296 297 298
		}

		gnt_list_entry->gref = GRANT_INVALID_REF;
299
		list_add(&gnt_list_entry->node, &rinfo->grants);
300 301 302 303 304 305 306
		i++;
	}

	return 0;

out_of_memory:
	list_for_each_entry_safe(gnt_list_entry, n,
307
	                         &rinfo->grants, node) {
308
		list_del(&gnt_list_entry->node);
309
		if (info->feature_persistent)
310
			__free_page(gnt_list_entry->page);
311 312 313 314 315 316 317
		kfree(gnt_list_entry);
		i--;
	}
	BUG_ON(i != 0);
	return -ENOMEM;
}

318
static struct grant *get_free_grant(struct blkfront_ring_info *rinfo)
319 320 321
{
	struct grant *gnt_list_entry;

322 323
	BUG_ON(list_empty(&rinfo->grants));
	gnt_list_entry = list_first_entry(&rinfo->grants, struct grant,
324
					  node);
325 326
	list_del(&gnt_list_entry->node);

327
	if (gnt_list_entry->gref != GRANT_INVALID_REF)
328
		rinfo->persistent_gnts_c--;
329 330 331 332 333 334 335 336 337 338 339 340 341 342 343

	return gnt_list_entry;
}

static inline void grant_foreign_access(const struct grant *gnt_list_entry,
					const struct blkfront_info *info)
{
	gnttab_page_grant_foreign_access_ref_one(gnt_list_entry->gref,
						 info->xbdev->otherend_id,
						 gnt_list_entry->page,
						 0);
}

static struct grant *get_grant(grant_ref_t *gref_head,
			       unsigned long gfn,
344
			       struct blkfront_ring_info *rinfo)
345
{
346 347
	struct grant *gnt_list_entry = get_free_grant(rinfo);
	struct blkfront_info *info = rinfo->dev_info;
348 349

	if (gnt_list_entry->gref != GRANT_INVALID_REF)
350
		return gnt_list_entry;
351 352 353 354 355 356 357 358 359 360 361

	/* Assign a gref to this page */
	gnt_list_entry->gref = gnttab_claim_grant_reference(gref_head);
	BUG_ON(gnt_list_entry->gref == -ENOSPC);
	if (info->feature_persistent)
		grant_foreign_access(gnt_list_entry, info);
	else {
		/* Grant access to the GFN passed by the caller */
		gnttab_grant_foreign_access_ref(gnt_list_entry->gref,
						info->xbdev->otherend_id,
						gfn, 0);
362 363
	}

364 365 366 367
	return gnt_list_entry;
}

static struct grant *get_indirect_grant(grant_ref_t *gref_head,
368
					struct blkfront_ring_info *rinfo)
369
{
370 371
	struct grant *gnt_list_entry = get_free_grant(rinfo);
	struct blkfront_info *info = rinfo->dev_info;
372 373 374 375

	if (gnt_list_entry->gref != GRANT_INVALID_REF)
		return gnt_list_entry;

376 377 378
	/* Assign a gref to this page */
	gnt_list_entry->gref = gnttab_claim_grant_reference(gref_head);
	BUG_ON(gnt_list_entry->gref == -ENOSPC);
379
	if (!info->feature_persistent) {
380 381 382
		struct page *indirect_page;

		/* Fetch a pre-allocated page to use for indirect grefs */
383 384
		BUG_ON(list_empty(&rinfo->indirect_pages));
		indirect_page = list_first_entry(&rinfo->indirect_pages,
385 386 387
						 struct page, lru);
		list_del(&indirect_page->lru);
		gnt_list_entry->page = indirect_page;
388
	}
389 390
	grant_foreign_access(gnt_list_entry, info);

391 392 393
	return gnt_list_entry;
}

394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410
static const char *op_name(int op)
{
	static const char *const names[] = {
		[BLKIF_OP_READ] = "read",
		[BLKIF_OP_WRITE] = "write",
		[BLKIF_OP_WRITE_BARRIER] = "barrier",
		[BLKIF_OP_FLUSH_DISKCACHE] = "flush",
		[BLKIF_OP_DISCARD] = "discard" };

	if (op < 0 || op >= ARRAY_SIZE(names))
		return "unknown";

	if (!names[op])
		return "reserved";

	return names[op];
}
411 412 413 414 415 416 417 418
static int xlbd_reserve_minors(unsigned int minor, unsigned int nr)
{
	unsigned int end = minor + nr;
	int rc;

	if (end > nr_minors) {
		unsigned long *bitmap, *old;

419
		bitmap = kcalloc(BITS_TO_LONGS(end), sizeof(*bitmap),
420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438
				 GFP_KERNEL);
		if (bitmap == NULL)
			return -ENOMEM;

		spin_lock(&minor_lock);
		if (end > nr_minors) {
			old = minors;
			memcpy(bitmap, minors,
			       BITS_TO_LONGS(nr_minors) * sizeof(*bitmap));
			minors = bitmap;
			nr_minors = BITS_TO_LONGS(end) * BITS_PER_LONG;
		} else
			old = bitmap;
		spin_unlock(&minor_lock);
		kfree(old);
	}

	spin_lock(&minor_lock);
	if (find_next_bit(minors, end, minor) >= end) {
439
		bitmap_set(minors, minor, nr);
440 441 442 443 444 445 446 447 448 449 450 451 452 453
		rc = 0;
	} else
		rc = -EBUSY;
	spin_unlock(&minor_lock);

	return rc;
}

static void xlbd_release_minors(unsigned int minor, unsigned int nr)
{
	unsigned int end = minor + nr;

	BUG_ON(end > nr_minors);
	spin_lock(&minor_lock);
454
	bitmap_clear(minors,  minor, nr);
455 456 457
	spin_unlock(&minor_lock);
}

458 459
static void blkif_restart_queue_callback(void *arg)
{
460 461
	struct blkfront_ring_info *rinfo = (struct blkfront_ring_info *)arg;
	schedule_work(&rinfo->work);
462 463
}

H
Harvey Harrison 已提交
464
static int blkif_getgeo(struct block_device *bd, struct hd_geometry *hg)
465 466 467 468 469 470 471 472 473 474 475 476 477 478 479
{
	/* We don't have real geometry info, but let's at least return
	   values consistent with the size of the device */
	sector_t nsect = get_capacity(bd->bd_disk);
	sector_t cylinders = nsect;

	hg->heads = 0xff;
	hg->sectors = 0x3f;
	sector_div(cylinders, hg->heads * hg->sectors);
	hg->cylinders = cylinders;
	if ((sector_t)(hg->cylinders + 1) * hg->heads * hg->sectors < nsect)
		hg->cylinders = 0xffff;
	return 0;
}

A
Al Viro 已提交
480
static int blkif_ioctl(struct block_device *bdev, fmode_t mode,
481
		       unsigned command, unsigned long argument)
482
{
A
Al Viro 已提交
483
	struct blkfront_info *info = bdev->bd_disk->private_data;
484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512
	int i;

	dev_dbg(&info->xbdev->dev, "command: 0x%x, argument: 0x%lx\n",
		command, (long)argument);

	switch (command) {
	case CDROMMULTISESSION:
		dev_dbg(&info->xbdev->dev, "FIXME: support multisession CDs later\n");
		for (i = 0; i < sizeof(struct cdrom_multisession); i++)
			if (put_user(0, (char __user *)(argument + i)))
				return -EFAULT;
		return 0;

	case CDROM_GET_CAPABILITY: {
		struct gendisk *gd = info->gd;
		if (gd->flags & GENHD_FL_CD)
			return 0;
		return -EINVAL;
	}

	default:
		/*printk(KERN_ALERT "ioctl %08x not supported by Xen blkdev\n",
		  command);*/
		return -EINVAL; /* same return as native Linux */
	}

	return 0;
}

513 514 515 516 517 518 519 520 521 522 523
static unsigned long blkif_ring_get_request(struct blkfront_ring_info *rinfo,
					    struct request *req,
					    struct blkif_request **ring_req)
{
	unsigned long id;

	*ring_req = RING_GET_REQUEST(&rinfo->ring, rinfo->ring.req_prod_pvt);
	rinfo->ring.req_prod_pvt++;

	id = get_id_from_freelist(rinfo);
	rinfo->shadow[id].request = req;
524 525
	rinfo->shadow[id].status = REQ_WAITING;
	rinfo->shadow[id].associated_id = NO_ASSOCIATED_ID;
526 527 528 529 530 531

	(*ring_req)->u.rw.id = id;

	return id;
}

532
static int blkif_queue_discard_req(struct request *req, struct blkfront_ring_info *rinfo)
533
{
534
	struct blkfront_info *info = rinfo->dev_info;
535 536
	struct blkif_request *ring_req;
	unsigned long id;
537 538

	/* Fill out a communications ring structure. */
539
	id = blkif_ring_get_request(rinfo, req, &ring_req);
540 541 542 543 544 545 546 547 548 549 550

	ring_req->operation = BLKIF_OP_DISCARD;
	ring_req->u.discard.nr_sectors = blk_rq_sectors(req);
	ring_req->u.discard.id = id;
	ring_req->u.discard.sector_number = (blkif_sector_t)blk_rq_pos(req);
	if ((req->cmd_flags & REQ_SECURE) && info->feature_secdiscard)
		ring_req->u.discard.flag = BLKIF_DISCARD_SECURE;
	else
		ring_req->u.discard.flag = 0;

	/* Keep a private copy so we can reissue requests when recovering. */
551
	rinfo->shadow[id].req = *ring_req;
552 553 554 555

	return 0;
}

556 557 558
struct setup_rw_req {
	unsigned int grant_idx;
	struct blkif_request_segment *segments;
559
	struct blkfront_ring_info *rinfo;
560 561 562 563 564 565 566
	struct blkif_request *ring_req;
	grant_ref_t gref_head;
	unsigned int id;
	/* Only used when persistent grant is used and it's a read request */
	bool need_copy;
	unsigned int bvec_off;
	char *bvec_data;
567 568 569

	bool require_extra_req;
	struct blkif_request *extra_ring_req;
570 571 572 573 574 575 576 577
};

static void blkif_setup_rw_req_grant(unsigned long gfn, unsigned int offset,
				     unsigned int len, void *data)
{
	struct setup_rw_req *setup = data;
	int n, ref;
	struct grant *gnt_list_entry;
578
	unsigned int fsect, lsect;
579 580 581
	/* Convenient aliases */
	unsigned int grant_idx = setup->grant_idx;
	struct blkif_request *ring_req = setup->ring_req;
582
	struct blkfront_ring_info *rinfo = setup->rinfo;
583 584 585 586 587 588
	/*
	 * We always use the shadow of the first request to store the list
	 * of grant associated to the block I/O request. This made the
	 * completion more easy to handle even if the block I/O request is
	 * split.
	 */
589
	struct blk_shadow *shadow = &rinfo->shadow[setup->id];
590

591 592 593 594 595 596 597 598 599 600
	if (unlikely(setup->require_extra_req &&
		     grant_idx >= BLKIF_MAX_SEGMENTS_PER_REQUEST)) {
		/*
		 * We are using the second request, setup grant_idx
		 * to be the index of the segment array.
		 */
		grant_idx -= BLKIF_MAX_SEGMENTS_PER_REQUEST;
		ring_req = setup->extra_ring_req;
	}

601 602 603 604 605 606
	if ((ring_req->operation == BLKIF_OP_INDIRECT) &&
	    (grant_idx % GRANTS_PER_INDIRECT_FRAME == 0)) {
		if (setup->segments)
			kunmap_atomic(setup->segments);

		n = grant_idx / GRANTS_PER_INDIRECT_FRAME;
607
		gnt_list_entry = get_indirect_grant(&setup->gref_head, rinfo);
608 609 610 611 612
		shadow->indirect_grants[n] = gnt_list_entry;
		setup->segments = kmap_atomic(gnt_list_entry->page);
		ring_req->u.indirect.indirect_grefs[n] = gnt_list_entry->gref;
	}

613
	gnt_list_entry = get_grant(&setup->gref_head, gfn, rinfo);
614
	ref = gnt_list_entry->gref;
615 616 617 618 619
	/*
	 * All the grants are stored in the shadow of the first
	 * request. Therefore we have to use the global index.
	 */
	shadow->grants_used[setup->grant_idx] = gnt_list_entry;
620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660

	if (setup->need_copy) {
		void *shared_data;

		shared_data = kmap_atomic(gnt_list_entry->page);
		/*
		 * this does not wipe data stored outside the
		 * range sg->offset..sg->offset+sg->length.
		 * Therefore, blkback *could* see data from
		 * previous requests. This is OK as long as
		 * persistent grants are shared with just one
		 * domain. It may need refactoring if this
		 * changes
		 */
		memcpy(shared_data + offset,
		       setup->bvec_data + setup->bvec_off,
		       len);

		kunmap_atomic(shared_data);
		setup->bvec_off += len;
	}

	fsect = offset >> 9;
	lsect = fsect + (len >> 9) - 1;
	if (ring_req->operation != BLKIF_OP_INDIRECT) {
		ring_req->u.rw.seg[grant_idx] =
			(struct blkif_request_segment) {
				.gref       = ref,
				.first_sect = fsect,
				.last_sect  = lsect };
	} else {
		setup->segments[grant_idx % GRANTS_PER_INDIRECT_FRAME] =
			(struct blkif_request_segment) {
				.gref       = ref,
				.first_sect = fsect,
				.last_sect  = lsect };
	}

	(setup->grant_idx)++;
}

661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679
static void blkif_setup_extra_req(struct blkif_request *first,
				  struct blkif_request *second)
{
	uint16_t nr_segments = first->u.rw.nr_segments;

	/*
	 * The second request is only present when the first request uses
	 * all its segments. It's always the continuity of the first one.
	 */
	first->u.rw.nr_segments = BLKIF_MAX_SEGMENTS_PER_REQUEST;

	second->u.rw.nr_segments = nr_segments - BLKIF_MAX_SEGMENTS_PER_REQUEST;
	second->u.rw.sector_number = first->u.rw.sector_number +
		(BLKIF_MAX_SEGMENTS_PER_REQUEST * XEN_PAGE_SIZE) / 512;

	second->u.rw.handle = first->u.rw.handle;
	second->operation = first->operation;
}

680
static int blkif_queue_rw_req(struct request *req, struct blkfront_ring_info *rinfo)
681
{
682
	struct blkfront_info *info = rinfo->dev_info;
683 684 685
	struct blkif_request *ring_req, *extra_ring_req = NULL;
	unsigned long id, extra_id = NO_ASSOCIATED_ID;
	bool require_extra_req = false;
686 687 688 689
	int i;
	struct setup_rw_req setup = {
		.grant_idx = 0,
		.segments = NULL,
690
		.rinfo = rinfo,
691 692
		.need_copy = rq_data_dir(req) && info->feature_persistent,
	};
693 694 695 696 697 698

	/*
	 * Used to store if we are able to queue the request by just using
	 * existing persistent grants, or if we have to get new grants,
	 * as there are not sufficiently many free.
	 */
699
	struct scatterlist *sg;
700
	int num_sg, max_grefs, num_grant;
701

702
	max_grefs = req->nr_phys_segments * GRANTS_PER_PSEG;
703 704 705 706 707
	if (max_grefs > BLKIF_MAX_SEGMENTS_PER_REQUEST)
		/*
		 * If we are using indirect segments we need to account
		 * for the indirect grefs used in the request.
		 */
708
		max_grefs += INDIRECT_GREFS(max_grefs);
709

710 711 712 713 714 715
	/*
	 * We have to reserve 'max_grefs' grants because persistent
	 * grants are shared by all rings.
	 */
	if (max_grefs > 0)
		if (gnttab_alloc_grant_references(max_grefs, &setup.gref_head) < 0) {
716
			gnttab_request_free_callback(
717
				&rinfo->callback,
718
				blkif_restart_queue_callback,
719
				rinfo,
720
				max_grefs);
721 722
			return 1;
		}
723 724

	/* Fill out a communications ring structure. */
725
	id = blkif_ring_get_request(rinfo, req, &ring_req);
726

727
	num_sg = blk_rq_map_sg(req->q, req, rinfo->shadow[id].sg);
728 729
	num_grant = 0;
	/* Calculate the number of grant used */
730
	for_each_sg(rinfo->shadow[id].sg, sg, num_sg, i)
731 732
	       num_grant += gnttab_count_grant(sg->offset, sg->length);

733 734 735 736
	require_extra_req = info->max_indirect_segments == 0 &&
		num_grant > BLKIF_MAX_SEGMENTS_PER_REQUEST;
	BUG_ON(!HAS_EXTRA_REQ && require_extra_req);

737
	rinfo->shadow[id].num_sg = num_sg;
738 739
	if (num_grant > BLKIF_MAX_SEGMENTS_PER_REQUEST &&
	    likely(!require_extra_req)) {
740 741 742 743 744 745 746 747 748 749
		/*
		 * The indirect operation can only be a BLKIF_OP_READ or
		 * BLKIF_OP_WRITE
		 */
		BUG_ON(req->cmd_flags & (REQ_FLUSH | REQ_FUA));
		ring_req->operation = BLKIF_OP_INDIRECT;
		ring_req->u.indirect.indirect_op = rq_data_dir(req) ?
			BLKIF_OP_WRITE : BLKIF_OP_READ;
		ring_req->u.indirect.sector_number = (blkif_sector_t)blk_rq_pos(req);
		ring_req->u.indirect.handle = info->handle;
750
		ring_req->u.indirect.nr_segments = num_grant;
751
	} else {
752 753 754 755 756
		ring_req->u.rw.sector_number = (blkif_sector_t)blk_rq_pos(req);
		ring_req->u.rw.handle = info->handle;
		ring_req->operation = rq_data_dir(req) ?
			BLKIF_OP_WRITE : BLKIF_OP_READ;
		if (req->cmd_flags & (REQ_FLUSH | REQ_FUA)) {
757
			/*
758 759 760 761 762
			 * Ideally we can do an unordered flush-to-disk.
			 * In case the backend onlysupports barriers, use that.
			 * A barrier request a superset of FUA, so we can
			 * implement it the same way.  (It's also a FLUSH+FUA,
			 * since it is guaranteed ordered WRT previous writes.)
763
			 */
764 765 766 767 768 769 770 771 772 773 774 775
			switch (info->feature_flush &
				((REQ_FLUSH|REQ_FUA))) {
			case REQ_FLUSH|REQ_FUA:
				ring_req->operation =
					BLKIF_OP_WRITE_BARRIER;
				break;
			case REQ_FLUSH:
				ring_req->operation =
					BLKIF_OP_FLUSH_DISKCACHE;
				break;
			default:
				ring_req->operation = 0;
776 777
			}
		}
778
		ring_req->u.rw.nr_segments = num_grant;
779 780 781 782 783 784 785 786 787 788 789 790 791 792 793
		if (unlikely(require_extra_req)) {
			extra_id = blkif_ring_get_request(rinfo, req,
							  &extra_ring_req);
			/*
			 * Only the first request contains the scatter-gather
			 * list.
			 */
			rinfo->shadow[extra_id].num_sg = 0;

			blkif_setup_extra_req(ring_req, extra_ring_req);

			/* Link the 2 requests together */
			rinfo->shadow[extra_id].associated_id = id;
			rinfo->shadow[id].associated_id = extra_id;
		}
794
	}
795

796 797
	setup.ring_req = ring_req;
	setup.id = id;
798 799 800 801 802

	setup.require_extra_req = require_extra_req;
	if (unlikely(require_extra_req))
		setup.extra_ring_req = extra_ring_req;

803
	for_each_sg(rinfo->shadow[id].sg, sg, num_sg, i) {
804
		BUG_ON(sg->offset + sg->length > PAGE_SIZE);
805

806 807 808 809
		if (setup.need_copy) {
			setup.bvec_off = sg->offset;
			setup.bvec_data = kmap_atomic(sg_page(sg));
		}
810

811 812 813 814 815
		gnttab_foreach_grant_in_range(sg_page(sg),
					      sg->offset,
					      sg->length,
					      blkif_setup_rw_req_grant,
					      &setup);
816

817 818
		if (setup.need_copy)
			kunmap_atomic(setup.bvec_data);
819
	}
820 821
	if (setup.segments)
		kunmap_atomic(setup.segments);
822 823

	/* Keep a private copy so we can reissue requests when recovering. */
824
	rinfo->shadow[id].req = *ring_req;
825 826
	if (unlikely(require_extra_req))
		rinfo->shadow[extra_id].req = *extra_ring_req;
827

828
	if (max_grefs > 0)
829
		gnttab_free_grant_references(setup.gref_head);
830 831 832 833

	return 0;
}

834 835 836 837 838 839
/*
 * Generate a Xen blkfront IO request from a blk layer request.  Reads
 * and writes are handled as expected.
 *
 * @req: a request struct
 */
840
static int blkif_queue_request(struct request *req, struct blkfront_ring_info *rinfo)
841
{
842
	if (unlikely(rinfo->dev_info->connected != BLKIF_STATE_CONNECTED))
843 844 845
		return 1;

	if (unlikely(req->cmd_flags & (REQ_DISCARD | REQ_SECURE)))
846
		return blkif_queue_discard_req(req, rinfo);
847
	else
848
		return blkif_queue_rw_req(req, rinfo);
849
}
850

851
static inline void flush_requests(struct blkfront_ring_info *rinfo)
852 853 854
{
	int notify;

855
	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&rinfo->ring, notify);
856 857

	if (notify)
858
		notify_remote_via_irq(rinfo->irq);
859 860
}

861 862
static inline bool blkif_request_flush_invalid(struct request *req,
					       struct blkfront_info *info)
863 864
{
	return ((req->cmd_type != REQ_TYPE_FS) ||
865 866 867 868
		((req->cmd_flags & REQ_FLUSH) &&
		 !(info->feature_flush & REQ_FLUSH)) ||
		((req->cmd_flags & REQ_FUA) &&
		 !(info->feature_flush & REQ_FUA)));
869 870
}

B
Bob Liu 已提交
871
static int blkif_queue_rq(struct blk_mq_hw_ctx *hctx,
872
			  const struct blk_mq_queue_data *qd)
873
{
B
Bob Liu 已提交
874
	unsigned long flags;
875
	struct blkfront_ring_info *rinfo = (struct blkfront_ring_info *)hctx->driver_data;
876

B
Bob Liu 已提交
877
	blk_mq_start_request(qd->rq);
B
Bob Liu 已提交
878
	spin_lock_irqsave(&rinfo->ring_lock, flags);
879
	if (RING_FULL(&rinfo->ring))
B
Bob Liu 已提交
880
		goto out_busy;
881

882
	if (blkif_request_flush_invalid(qd->rq, rinfo->dev_info))
B
Bob Liu 已提交
883
		goto out_err;
884

885
	if (blkif_queue_request(qd->rq, rinfo))
B
Bob Liu 已提交
886
		goto out_busy;
887

888
	flush_requests(rinfo);
B
Bob Liu 已提交
889
	spin_unlock_irqrestore(&rinfo->ring_lock, flags);
B
Bob Liu 已提交
890
	return BLK_MQ_RQ_QUEUE_OK;
891

B
Bob Liu 已提交
892
out_err:
B
Bob Liu 已提交
893
	spin_unlock_irqrestore(&rinfo->ring_lock, flags);
B
Bob Liu 已提交
894
	return BLK_MQ_RQ_QUEUE_ERROR;
895

B
Bob Liu 已提交
896
out_busy:
B
Bob Liu 已提交
897
	spin_unlock_irqrestore(&rinfo->ring_lock, flags);
B
Bob Liu 已提交
898 899
	blk_mq_stop_hw_queue(hctx);
	return BLK_MQ_RQ_QUEUE_BUSY;
900 901
}

902 903 904 905 906
static int blk_mq_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
			    unsigned int index)
{
	struct blkfront_info *info = (struct blkfront_info *)data;

907 908
	BUG_ON(info->nr_rings <= index);
	hctx->driver_data = &info->rinfo[index];
909 910 911
	return 0;
}

B
Bob Liu 已提交
912 913 914
static struct blk_mq_ops blkfront_mq_ops = {
	.queue_rq = blkif_queue_rq,
	.map_queue = blk_mq_map_queue,
915
	.init_hctx = blk_mq_init_hctx,
B
Bob Liu 已提交
916 917
};

918
static int xlvbd_init_blk_queue(struct gendisk *gd, u16 sector_size,
919
				unsigned int physical_sector_size,
920
				unsigned int segments)
921
{
922
	struct request_queue *rq;
923
	struct blkfront_info *info = gd->private_data;
924

B
Bob Liu 已提交
925 926
	memset(&info->tag_set, 0, sizeof(info->tag_set));
	info->tag_set.ops = &blkfront_mq_ops;
927
	info->tag_set.nr_hw_queues = info->nr_rings;
928 929 930 931 932 933 934 935 936 937
	if (HAS_EXTRA_REQ && info->max_indirect_segments == 0) {
		/*
		 * When indirect descriptior is not supported, the I/O request
		 * will be split between multiple request in the ring.
		 * To avoid problems when sending the request, divide by
		 * 2 the depth of the queue.
		 */
		info->tag_set.queue_depth =  BLK_RING_SIZE(info) / 2;
	} else
		info->tag_set.queue_depth = BLK_RING_SIZE(info);
B
Bob Liu 已提交
938 939 940 941 942 943
	info->tag_set.numa_node = NUMA_NO_NODE;
	info->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE;
	info->tag_set.cmd_size = 0;
	info->tag_set.driver_data = info;

	if (blk_mq_alloc_tag_set(&info->tag_set))
944
		return -EINVAL;
B
Bob Liu 已提交
945 946 947
	rq = blk_mq_init_queue(&info->tag_set);
	if (IS_ERR(rq)) {
		blk_mq_free_tag_set(&info->tag_set);
948
		return PTR_ERR(rq);
B
Bob Liu 已提交
949
	}
950

951
	queue_flag_set_unlocked(QUEUE_FLAG_VIRT, rq);
952

953 954 955 956 957
	if (info->feature_discard) {
		queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, rq);
		blk_queue_max_discard_sectors(rq, get_capacity(gd));
		rq->limits.discard_granularity = info->discard_granularity;
		rq->limits.discard_alignment = info->discard_alignment;
958 959
		if (info->feature_secdiscard)
			queue_flag_set_unlocked(QUEUE_FLAG_SECDISCARD, rq);
960 961
	}

962
	/* Hard sector size and max sectors impersonate the equiv. hardware. */
963
	blk_queue_logical_block_size(rq, sector_size);
964
	blk_queue_physical_block_size(rq, physical_sector_size);
965
	blk_queue_max_hw_sectors(rq, (segments * XEN_PAGE_SIZE) / 512);
966 967 968 969 970 971

	/* Each segment in a request is up to an aligned page in size. */
	blk_queue_segment_boundary(rq, PAGE_SIZE - 1);
	blk_queue_max_segment_size(rq, PAGE_SIZE);

	/* Ensure a merged request will fit in a single I/O ring slot. */
972
	blk_queue_max_segments(rq, segments / GRANTS_PER_PSEG);
973 974 975 976

	/* Make sure buffer addresses are sector-aligned. */
	blk_queue_dma_alignment(rq, 511);

977 978 979
	/* Make sure we don't use bounce buffers. */
	blk_queue_bounce_limit(rq, BLK_BOUNCE_ANY);

980 981 982 983 984
	gd->queue = rq;

	return 0;
}

985 986 987 988 989 990 991 992 993 994 995
static const char *flush_info(unsigned int feature_flush)
{
	switch (feature_flush & ((REQ_FLUSH | REQ_FUA))) {
	case REQ_FLUSH|REQ_FUA:
		return "barrier: enabled;";
	case REQ_FLUSH:
		return "flush diskcache: enabled;";
	default:
		return "barrier or flush: disabled;";
	}
}
996

997
static void xlvbd_flush(struct blkfront_info *info)
998
{
999
	blk_queue_flush(info->rq, info->feature_flush);
1000 1001 1002 1003 1004
	pr_info("blkfront: %s: %s %s %s %s %s\n",
		info->gd->disk_name, flush_info(info->feature_flush),
		"persistent grants:", info->feature_persistent ?
		"enabled;" : "disabled;", "indirect descriptors:",
		info->max_indirect_segments ? "enabled;" : "disabled;");
1005 1006
}

1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065
static int xen_translate_vdev(int vdevice, int *minor, unsigned int *offset)
{
	int major;
	major = BLKIF_MAJOR(vdevice);
	*minor = BLKIF_MINOR(vdevice);
	switch (major) {
		case XEN_IDE0_MAJOR:
			*offset = (*minor / 64) + EMULATED_HD_DISK_NAME_OFFSET;
			*minor = ((*minor / 64) * PARTS_PER_DISK) +
				EMULATED_HD_DISK_MINOR_OFFSET;
			break;
		case XEN_IDE1_MAJOR:
			*offset = (*minor / 64) + 2 + EMULATED_HD_DISK_NAME_OFFSET;
			*minor = (((*minor / 64) + 2) * PARTS_PER_DISK) +
				EMULATED_HD_DISK_MINOR_OFFSET;
			break;
		case XEN_SCSI_DISK0_MAJOR:
			*offset = (*minor / PARTS_PER_DISK) + EMULATED_SD_DISK_NAME_OFFSET;
			*minor = *minor + EMULATED_SD_DISK_MINOR_OFFSET;
			break;
		case XEN_SCSI_DISK1_MAJOR:
		case XEN_SCSI_DISK2_MAJOR:
		case XEN_SCSI_DISK3_MAJOR:
		case XEN_SCSI_DISK4_MAJOR:
		case XEN_SCSI_DISK5_MAJOR:
		case XEN_SCSI_DISK6_MAJOR:
		case XEN_SCSI_DISK7_MAJOR:
			*offset = (*minor / PARTS_PER_DISK) + 
				((major - XEN_SCSI_DISK1_MAJOR + 1) * 16) +
				EMULATED_SD_DISK_NAME_OFFSET;
			*minor = *minor +
				((major - XEN_SCSI_DISK1_MAJOR + 1) * 16 * PARTS_PER_DISK) +
				EMULATED_SD_DISK_MINOR_OFFSET;
			break;
		case XEN_SCSI_DISK8_MAJOR:
		case XEN_SCSI_DISK9_MAJOR:
		case XEN_SCSI_DISK10_MAJOR:
		case XEN_SCSI_DISK11_MAJOR:
		case XEN_SCSI_DISK12_MAJOR:
		case XEN_SCSI_DISK13_MAJOR:
		case XEN_SCSI_DISK14_MAJOR:
		case XEN_SCSI_DISK15_MAJOR:
			*offset = (*minor / PARTS_PER_DISK) + 
				((major - XEN_SCSI_DISK8_MAJOR + 8) * 16) +
				EMULATED_SD_DISK_NAME_OFFSET;
			*minor = *minor +
				((major - XEN_SCSI_DISK8_MAJOR + 8) * 16 * PARTS_PER_DISK) +
				EMULATED_SD_DISK_MINOR_OFFSET;
			break;
		case XENVBD_MAJOR:
			*offset = *minor / PARTS_PER_DISK;
			break;
		default:
			printk(KERN_WARNING "blkfront: your disk configuration is "
					"incorrect, please use an xvd device instead\n");
			return -ENODEV;
	}
	return 0;
}
1066

1067 1068 1069 1070 1071 1072 1073 1074
static char *encode_disk_name(char *ptr, unsigned int n)
{
	if (n >= 26)
		ptr = encode_disk_name(ptr, n / 26 - 1);
	*ptr = 'a' + n % 26;
	return ptr + 1;
}

1075 1076
static int xlvbd_alloc_gendisk(blkif_sector_t capacity,
			       struct blkfront_info *info,
1077 1078
			       u16 vdisk_info, u16 sector_size,
			       unsigned int physical_sector_size)
1079 1080 1081
{
	struct gendisk *gd;
	int nr_minors = 1;
1082
	int err;
1083 1084 1085
	unsigned int offset;
	int minor;
	int nr_parts;
1086
	char *ptr;
1087 1088 1089 1090

	BUG_ON(info->gd != NULL);
	BUG_ON(info->rq != NULL);

1091 1092 1093 1094 1095 1096 1097
	if ((info->vdevice>>EXT_SHIFT) > 1) {
		/* this is above the extended range; something is wrong */
		printk(KERN_WARNING "blkfront: vdevice 0x%x is above the extended range; ignoring\n", info->vdevice);
		return -ENODEV;
	}

	if (!VDEV_IS_EXTENDED(info->vdevice)) {
1098 1099 1100 1101
		err = xen_translate_vdev(info->vdevice, &minor, &offset);
		if (err)
			return err;		
 		nr_parts = PARTS_PER_DISK;
1102 1103 1104
	} else {
		minor = BLKIF_MINOR_EXT(info->vdevice);
		nr_parts = PARTS_PER_EXT_DISK;
1105
		offset = minor / nr_parts;
1106
		if (xen_hvm_domain() && offset < EMULATED_HD_DISK_NAME_OFFSET + 4)
1107 1108 1109
			printk(KERN_WARNING "blkfront: vdevice 0x%x might conflict with "
					"emulated IDE disks,\n\t choose an xvd device name"
					"from xvde on\n", info->vdevice);
1110
	}
1111 1112 1113 1114 1115
	if (minor >> MINORBITS) {
		pr_warn("blkfront: %#x's minor (%#x) out of range; ignoring\n",
			info->vdevice, minor);
		return -ENODEV;
	}
1116 1117 1118

	if ((minor % nr_parts) == 0)
		nr_minors = nr_parts;
1119

1120 1121 1122 1123 1124
	err = xlbd_reserve_minors(minor, nr_minors);
	if (err)
		goto out;
	err = -ENODEV;

1125 1126
	gd = alloc_disk(nr_minors);
	if (gd == NULL)
1127
		goto release;
1128

1129 1130 1131 1132 1133 1134 1135 1136
	strcpy(gd->disk_name, DEV_NAME);
	ptr = encode_disk_name(gd->disk_name + sizeof(DEV_NAME) - 1, offset);
	BUG_ON(ptr >= gd->disk_name + DISK_NAME_LEN);
	if (nr_minors > 1)
		*ptr = 0;
	else
		snprintf(ptr, gd->disk_name + DISK_NAME_LEN - ptr,
			 "%d", minor & (nr_parts - 1));
1137 1138 1139 1140 1141 1142 1143 1144

	gd->major = XENVBD_MAJOR;
	gd->first_minor = minor;
	gd->fops = &xlvbd_block_fops;
	gd->private_data = info;
	gd->driverfs_dev = &(info->xbdev->dev);
	set_capacity(gd, capacity);

1145
	if (xlvbd_init_blk_queue(gd, sector_size, physical_sector_size,
1146 1147
				 info->max_indirect_segments ? :
				 BLKIF_MAX_SEGMENTS_PER_REQUEST)) {
1148
		del_gendisk(gd);
1149
		goto release;
1150 1151 1152 1153 1154
	}

	info->rq = gd->queue;
	info->gd = gd;

1155
	xlvbd_flush(info);
1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167

	if (vdisk_info & VDISK_READONLY)
		set_disk_ro(gd, 1);

	if (vdisk_info & VDISK_REMOVABLE)
		gd->flags |= GENHD_FL_REMOVABLE;

	if (vdisk_info & VDISK_CDROM)
		gd->flags |= GENHD_FL_CD;

	return 0;

1168 1169
 release:
	xlbd_release_minors(minor, nr_minors);
1170 1171 1172 1173
 out:
	return err;
}

D
Daniel Stodden 已提交
1174 1175
static void xlvbd_release_gendisk(struct blkfront_info *info)
{
1176
	unsigned int minor, nr_minors, i;
D
Daniel Stodden 已提交
1177 1178 1179 1180 1181

	if (info->rq == NULL)
		return;

	/* No more blkif_request(). */
B
Bob Liu 已提交
1182
	blk_mq_stop_hw_queues(info->rq);
D
Daniel Stodden 已提交
1183

1184 1185
	for (i = 0; i < info->nr_rings; i++) {
		struct blkfront_ring_info *rinfo = &info->rinfo[i];
D
Daniel Stodden 已提交
1186

1187 1188 1189 1190 1191 1192
		/* No more gnttab callback work. */
		gnttab_cancel_free_callback(&rinfo->callback);

		/* Flush gnttab callback work. Must be done with no locks held. */
		flush_work(&rinfo->work);
	}
D
Daniel Stodden 已提交
1193 1194 1195 1196 1197 1198 1199 1200

	del_gendisk(info->gd);

	minor = info->gd->first_minor;
	nr_minors = info->gd->minors;
	xlbd_release_minors(minor, nr_minors);

	blk_cleanup_queue(info->rq);
B
Bob Liu 已提交
1201
	blk_mq_free_tag_set(&info->tag_set);
D
Daniel Stodden 已提交
1202 1203 1204 1205 1206 1207
	info->rq = NULL;

	put_disk(info->gd);
	info->gd = NULL;
}

B
Bob Liu 已提交
1208 1209
/* Already hold rinfo->ring_lock. */
static inline void kick_pending_request_queues_locked(struct blkfront_ring_info *rinfo)
1210
{
1211 1212
	if (!RING_FULL(&rinfo->ring))
		blk_mq_start_stopped_hw_queues(rinfo->dev_info->rq, true);
1213 1214
}

B
Bob Liu 已提交
1215 1216 1217 1218 1219 1220 1221 1222 1223
static void kick_pending_request_queues(struct blkfront_ring_info *rinfo)
{
	unsigned long flags;

	spin_lock_irqsave(&rinfo->ring_lock, flags);
	kick_pending_request_queues_locked(rinfo);
	spin_unlock_irqrestore(&rinfo->ring_lock, flags);
}

1224 1225
static void blkif_restart_queue(struct work_struct *work)
{
1226
	struct blkfront_ring_info *rinfo = container_of(work, struct blkfront_ring_info, work);
1227

1228 1229
	if (rinfo->dev_info->connected == BLKIF_STATE_CONNECTED)
		kick_pending_request_queues(rinfo);
1230 1231
}

1232
static void blkif_free_ring(struct blkfront_ring_info *rinfo)
1233
{
1234
	struct grant *persistent_gnt, *n;
1235
	struct blkfront_info *info = rinfo->dev_info;
1236
	int i, j, segs;
1237

1238 1239 1240 1241
	/*
	 * Remove indirect pages, this only happens when using indirect
	 * descriptors but not persistent grants
	 */
1242
	if (!list_empty(&rinfo->indirect_pages)) {
1243 1244 1245
		struct page *indirect_page, *n;

		BUG_ON(info->feature_persistent);
1246
		list_for_each_entry_safe(indirect_page, n, &rinfo->indirect_pages, lru) {
1247 1248 1249 1250 1251
			list_del(&indirect_page->lru);
			__free_page(indirect_page);
		}
	}

1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268
	/* Remove all persistent grants. */
	if (!list_empty(&rinfo->grants)) {
		list_for_each_entry_safe(persistent_gnt, n,
					 &rinfo->grants, node) {
			list_del(&persistent_gnt->node);
			if (persistent_gnt->gref != GRANT_INVALID_REF) {
				gnttab_end_foreign_access(persistent_gnt->gref,
							  0, 0UL);
				rinfo->persistent_gnts_c--;
			}
			if (info->feature_persistent)
				__free_page(persistent_gnt->page);
			kfree(persistent_gnt);
		}
	}
	BUG_ON(rinfo->persistent_gnts_c != 0);

B
Bob Liu 已提交
1269
	for (i = 0; i < BLK_RING_SIZE(info); i++) {
1270 1271 1272 1273
		/*
		 * Clear persistent grants present in requests already
		 * on the shared ring
		 */
1274
		if (!rinfo->shadow[i].request)
1275 1276
			goto free_shadow;

1277 1278 1279
		segs = rinfo->shadow[i].req.operation == BLKIF_OP_INDIRECT ?
		       rinfo->shadow[i].req.u.indirect.nr_segments :
		       rinfo->shadow[i].req.u.rw.nr_segments;
1280
		for (j = 0; j < segs; j++) {
1281
			persistent_gnt = rinfo->shadow[i].grants_used[j];
1282
			gnttab_end_foreign_access(persistent_gnt->gref, 0, 0UL);
1283
			if (info->feature_persistent)
1284
				__free_page(persistent_gnt->page);
1285 1286 1287
			kfree(persistent_gnt);
		}

1288
		if (rinfo->shadow[i].req.operation != BLKIF_OP_INDIRECT)
1289 1290 1291 1292 1293 1294 1295
			/*
			 * If this is not an indirect operation don't try to
			 * free indirect segments
			 */
			goto free_shadow;

		for (j = 0; j < INDIRECT_GREFS(segs); j++) {
1296
			persistent_gnt = rinfo->shadow[i].indirect_grants[j];
1297
			gnttab_end_foreign_access(persistent_gnt->gref, 0, 0UL);
1298
			__free_page(persistent_gnt->page);
1299 1300 1301 1302
			kfree(persistent_gnt);
		}

free_shadow:
1303 1304 1305 1306 1307 1308
		kfree(rinfo->shadow[i].grants_used);
		rinfo->shadow[i].grants_used = NULL;
		kfree(rinfo->shadow[i].indirect_grants);
		rinfo->shadow[i].indirect_grants = NULL;
		kfree(rinfo->shadow[i].sg);
		rinfo->shadow[i].sg = NULL;
1309 1310
	}

1311
	/* No more gnttab callback work. */
1312
	gnttab_cancel_free_callback(&rinfo->callback);
1313 1314

	/* Flush gnttab callback work. Must be done with no locks held. */
1315
	flush_work(&rinfo->work);
1316 1317

	/* Free resources associated with old device channel. */
B
Bob Liu 已提交
1318
	for (i = 0; i < info->nr_ring_pages; i++) {
1319 1320 1321
		if (rinfo->ring_ref[i] != GRANT_INVALID_REF) {
			gnttab_end_foreign_access(rinfo->ring_ref[i], 0, 0);
			rinfo->ring_ref[i] = GRANT_INVALID_REF;
B
Bob Liu 已提交
1322
		}
1323
	}
1324 1325
	free_pages((unsigned long)rinfo->ring.sring, get_order(info->nr_ring_pages * PAGE_SIZE));
	rinfo->ring.sring = NULL;
B
Bob Liu 已提交
1326

1327 1328 1329
	if (rinfo->irq)
		unbind_from_irqhandler(rinfo->irq, rinfo);
	rinfo->evtchn = rinfo->irq = 0;
1330
}
1331

1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348
static void blkif_free(struct blkfront_info *info, int suspend)
{
	unsigned int i;

	/* Prevent new requests being issued until we fix things up. */
	info->connected = suspend ?
		BLKIF_STATE_SUSPENDED : BLKIF_STATE_DISCONNECTED;
	/* No more blkif_request(). */
	if (info->rq)
		blk_mq_stop_hw_queues(info->rq);

	for (i = 0; i < info->nr_rings; i++)
		blkif_free_ring(&info->rinfo[i]);

	kfree(info->rinfo);
	info->rinfo = NULL;
	info->nr_rings = 0;
1349 1350
}

1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376
struct copy_from_grant {
	const struct blk_shadow *s;
	unsigned int grant_idx;
	unsigned int bvec_offset;
	char *bvec_data;
};

static void blkif_copy_from_grant(unsigned long gfn, unsigned int offset,
				  unsigned int len, void *data)
{
	struct copy_from_grant *info = data;
	char *shared_data;
	/* Convenient aliases */
	const struct blk_shadow *s = info->s;

	shared_data = kmap_atomic(s->grants_used[info->grant_idx]->page);

	memcpy(info->bvec_data + info->bvec_offset,
	       shared_data + offset, len);

	info->bvec_offset += len;
	info->grant_idx++;

	kunmap_atomic(shared_data);
}

1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409
static enum blk_req_status blkif_rsp_to_req_status(int rsp)
{
	switch (rsp)
	{
	case BLKIF_RSP_OKAY:
		return REQ_DONE;
	case BLKIF_RSP_EOPNOTSUPP:
		return REQ_EOPNOTSUPP;
	case BLKIF_RSP_ERROR:
		/* Fallthrough. */
	default:
		return REQ_ERROR;
	}
}

/*
 * Get the final status of the block request based on two ring response
 */
static int blkif_get_final_status(enum blk_req_status s1,
				  enum blk_req_status s2)
{
	BUG_ON(s1 == REQ_WAITING);
	BUG_ON(s2 == REQ_WAITING);

	if (s1 == REQ_ERROR || s2 == REQ_ERROR)
		return BLKIF_RSP_ERROR;
	else if (s1 == REQ_EOPNOTSUPP || s2 == REQ_EOPNOTSUPP)
		return BLKIF_RSP_EOPNOTSUPP;
	return BLKIF_RSP_OKAY;
}

static bool blkif_completion(unsigned long *id,
			     struct blkfront_ring_info *rinfo,
1410
			     struct blkif_response *bret)
1411
{
1412
	int i = 0;
1413
	struct scatterlist *sg;
1414
	int num_sg, num_grant;
1415
	struct blkfront_info *info = rinfo->dev_info;
1416
	struct blk_shadow *s = &rinfo->shadow[*id];
1417 1418 1419
	struct copy_from_grant data = {
		.grant_idx = 0,
	};
1420

1421
	num_grant = s->req.operation == BLKIF_OP_INDIRECT ?
1422
		s->req.u.indirect.nr_segments : s->req.u.rw.nr_segments;
1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463

	/* The I/O request may be split in two. */
	if (unlikely(s->associated_id != NO_ASSOCIATED_ID)) {
		struct blk_shadow *s2 = &rinfo->shadow[s->associated_id];

		/* Keep the status of the current response in shadow. */
		s->status = blkif_rsp_to_req_status(bret->status);

		/* Wait the second response if not yet here. */
		if (s2->status == REQ_WAITING)
			return 0;

		bret->status = blkif_get_final_status(s->status,
						      s2->status);

		/*
		 * All the grants is stored in the first shadow in order
		 * to make the completion code simpler.
		 */
		num_grant += s2->req.u.rw.nr_segments;

		/*
		 * The two responses may not come in order. Only the
		 * first request will store the scatter-gather list.
		 */
		if (s2->num_sg != 0) {
			/* Update "id" with the ID of the first response. */
			*id = s->associated_id;
			s = s2;
		}

		/*
		 * We don't need anymore the second request, so recycling
		 * it now.
		 */
		if (add_id_to_freelist(rinfo, s->associated_id))
			WARN(1, "%s: can't recycle the second part (id = %ld) of the request\n",
			     info->gd->disk_name, s->associated_id);
	}

	data.s = s;
1464
	num_sg = s->num_sg;
1465

1466
	if (bret->operation == BLKIF_OP_READ && info->feature_persistent) {
1467
		for_each_sg(s->sg, sg, num_sg, i) {
1468
			BUG_ON(sg->offset + sg->length > PAGE_SIZE);
1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479

			data.bvec_offset = sg->offset;
			data.bvec_data = kmap_atomic(sg_page(sg));

			gnttab_foreach_grant_in_range(sg_page(sg),
						      sg->offset,
						      sg->length,
						      blkif_copy_from_grant,
						      &data);

			kunmap_atomic(data.bvec_data);
1480 1481 1482
		}
	}
	/* Add the persistent grant into the list of free grants */
1483
	for (i = 0; i < num_grant; i++) {
1484 1485 1486 1487 1488 1489 1490
		if (gnttab_query_foreign_access(s->grants_used[i]->gref)) {
			/*
			 * If the grant is still mapped by the backend (the
			 * backend has chosen to make this grant persistent)
			 * we add it at the head of the list, so it will be
			 * reused first.
			 */
1491 1492 1493
			if (!info->feature_persistent)
				pr_alert_ratelimited("backed has not unmapped grant: %u\n",
						     s->grants_used[i]->gref);
1494 1495
			list_add(&s->grants_used[i]->node, &rinfo->grants);
			rinfo->persistent_gnts_c++;
1496 1497 1498 1499 1500 1501 1502 1503 1504
		} else {
			/*
			 * If the grant is not mapped by the backend we end the
			 * foreign access and add it to the tail of the list,
			 * so it will not be picked again unless we run out of
			 * persistent grants.
			 */
			gnttab_end_foreign_access(s->grants_used[i]->gref, 0, 0UL);
			s->grants_used[i]->gref = GRANT_INVALID_REF;
1505
			list_add_tail(&s->grants_used[i]->node, &rinfo->grants);
1506
		}
1507
	}
1508
	if (s->req.operation == BLKIF_OP_INDIRECT) {
1509
		for (i = 0; i < INDIRECT_GREFS(num_grant); i++) {
1510
			if (gnttab_query_foreign_access(s->indirect_grants[i]->gref)) {
1511 1512 1513
				if (!info->feature_persistent)
					pr_alert_ratelimited("backed has not unmapped grant: %u\n",
							     s->indirect_grants[i]->gref);
1514 1515
				list_add(&s->indirect_grants[i]->node, &rinfo->grants);
				rinfo->persistent_gnts_c++;
1516
			} else {
1517 1518
				struct page *indirect_page;

1519
				gnttab_end_foreign_access(s->indirect_grants[i]->gref, 0, 0UL);
1520 1521 1522 1523
				/*
				 * Add the used indirect page back to the list of
				 * available pages for indirect grefs.
				 */
1524
				if (!info->feature_persistent) {
1525
					indirect_page = s->indirect_grants[i]->page;
1526
					list_add(&indirect_page->lru, &rinfo->indirect_pages);
1527
				}
1528
				s->indirect_grants[i]->gref = GRANT_INVALID_REF;
1529
				list_add_tail(&s->indirect_grants[i]->node, &rinfo->grants);
1530
			}
1531 1532
		}
	}
1533 1534

	return 1;
1535 1536 1537 1538 1539 1540 1541 1542
}

static irqreturn_t blkif_interrupt(int irq, void *dev_id)
{
	struct request *req;
	struct blkif_response *bret;
	RING_IDX i, rp;
	unsigned long flags;
1543 1544
	struct blkfront_ring_info *rinfo = (struct blkfront_ring_info *)dev_id;
	struct blkfront_info *info = rinfo->dev_info;
1545
	int error;
1546

B
Bob Liu 已提交
1547
	if (unlikely(info->connected != BLKIF_STATE_CONNECTED))
1548 1549
		return IRQ_HANDLED;

B
Bob Liu 已提交
1550
	spin_lock_irqsave(&rinfo->ring_lock, flags);
1551
 again:
1552
	rp = rinfo->ring.sring->rsp_prod;
1553 1554
	rmb(); /* Ensure we see queued responses up to 'rp'. */

1555
	for (i = rinfo->ring.rsp_cons; i != rp; i++) {
1556 1557
		unsigned long id;

1558
		bret = RING_GET_RESPONSE(&rinfo->ring, i);
1559
		id   = bret->id;
1560 1561 1562 1563 1564
		/*
		 * The backend has messed up and given us an id that we would
		 * never have given to it (we stamp it up to BLK_RING_SIZE -
		 * look in get_id_from_freelist.
		 */
B
Bob Liu 已提交
1565
		if (id >= BLK_RING_SIZE(info)) {
1566 1567 1568 1569 1570 1571
			WARN(1, "%s: response to %s has incorrect id (%ld)\n",
			     info->gd->disk_name, op_name(bret->operation), id);
			/* We can't safely get the 'struct request' as
			 * the id is busted. */
			continue;
		}
1572
		req  = rinfo->shadow[id].request;
1573

1574 1575 1576 1577 1578 1579 1580 1581
		if (bret->operation != BLKIF_OP_DISCARD) {
			/*
			 * We may need to wait for an extra response if the
			 * I/O request is split in 2
			 */
			if (!blkif_completion(&id, rinfo, bret))
				continue;
		}
1582

1583
		if (add_id_to_freelist(rinfo, id)) {
1584 1585 1586 1587
			WARN(1, "%s: response to %s (id %ld) couldn't be recycled!\n",
			     info->gd->disk_name, op_name(bret->operation), id);
			continue;
		}
1588

1589
		error = (bret->status == BLKIF_RSP_OKAY) ? 0 : -EIO;
1590
		switch (bret->operation) {
1591 1592 1593
		case BLKIF_OP_DISCARD:
			if (unlikely(bret->status == BLKIF_RSP_EOPNOTSUPP)) {
				struct request_queue *rq = info->rq;
1594 1595
				printk(KERN_WARNING "blkfront: %s: %s op failed\n",
					   info->gd->disk_name, op_name(bret->operation));
1596
				error = -EOPNOTSUPP;
1597
				info->feature_discard = 0;
1598
				info->feature_secdiscard = 0;
1599
				queue_flag_clear(QUEUE_FLAG_DISCARD, rq);
1600
				queue_flag_clear(QUEUE_FLAG_SECDISCARD, rq);
1601
			}
1602
			blk_mq_complete_request(req, error);
1603
			break;
1604
		case BLKIF_OP_FLUSH_DISKCACHE:
1605 1606
		case BLKIF_OP_WRITE_BARRIER:
			if (unlikely(bret->status == BLKIF_RSP_EOPNOTSUPP)) {
1607 1608
				printk(KERN_WARNING "blkfront: %s: %s op failed\n",
				       info->gd->disk_name, op_name(bret->operation));
1609
				error = -EOPNOTSUPP;
1610 1611
			}
			if (unlikely(bret->status == BLKIF_RSP_ERROR &&
1612
				     rinfo->shadow[id].req.u.rw.nr_segments == 0)) {
1613 1614
				printk(KERN_WARNING "blkfront: %s: empty %s op failed\n",
				       info->gd->disk_name, op_name(bret->operation));
1615
				error = -EOPNOTSUPP;
1616
			}
1617 1618 1619
			if (unlikely(error)) {
				if (error == -EOPNOTSUPP)
					error = 0;
1620 1621
				info->feature_flush = 0;
				xlvbd_flush(info);
1622 1623 1624 1625 1626 1627 1628 1629
			}
			/* fall through */
		case BLKIF_OP_READ:
		case BLKIF_OP_WRITE:
			if (unlikely(bret->status != BLKIF_RSP_OKAY))
				dev_dbg(&info->xbdev->dev, "Bad return from blkdev data "
					"request: %x\n", bret->status);

1630
			blk_mq_complete_request(req, error);
1631 1632 1633 1634 1635 1636
			break;
		default:
			BUG();
		}
	}

1637
	rinfo->ring.rsp_cons = i;
1638

1639
	if (i != rinfo->ring.req_prod_pvt) {
1640
		int more_to_do;
1641
		RING_FINAL_CHECK_FOR_RESPONSES(&rinfo->ring, more_to_do);
1642 1643 1644
		if (more_to_do)
			goto again;
	} else
1645
		rinfo->ring.sring->rsp_event = i + 1;
1646

B
Bob Liu 已提交
1647
	kick_pending_request_queues_locked(rinfo);
1648

B
Bob Liu 已提交
1649
	spin_unlock_irqrestore(&rinfo->ring_lock, flags);
1650 1651 1652 1653 1654 1655

	return IRQ_HANDLED;
}


static int setup_blkring(struct xenbus_device *dev,
1656
			 struct blkfront_ring_info *rinfo)
1657 1658
{
	struct blkif_sring *sring;
B
Bob Liu 已提交
1659
	int err, i;
1660
	struct blkfront_info *info = rinfo->dev_info;
1661
	unsigned long ring_size = info->nr_ring_pages * XEN_PAGE_SIZE;
1662
	grant_ref_t gref[XENBUS_MAX_RING_GRANTS];
1663

B
Bob Liu 已提交
1664
	for (i = 0; i < info->nr_ring_pages; i++)
1665
		rinfo->ring_ref[i] = GRANT_INVALID_REF;
1666

B
Bob Liu 已提交
1667 1668
	sring = (struct blkif_sring *)__get_free_pages(GFP_NOIO | __GFP_HIGH,
						       get_order(ring_size));
1669 1670 1671 1672 1673
	if (!sring) {
		xenbus_dev_fatal(dev, -ENOMEM, "allocating shared ring");
		return -ENOMEM;
	}
	SHARED_RING_INIT(sring);
1674
	FRONT_RING_INIT(&rinfo->ring, sring, ring_size);
1675

1676
	err = xenbus_grant_ring(dev, rinfo->ring.sring, info->nr_ring_pages, gref);
1677
	if (err < 0) {
B
Bob Liu 已提交
1678
		free_pages((unsigned long)sring, get_order(ring_size));
1679
		rinfo->ring.sring = NULL;
1680 1681
		goto fail;
	}
B
Bob Liu 已提交
1682
	for (i = 0; i < info->nr_ring_pages; i++)
1683
		rinfo->ring_ref[i] = gref[i];
1684

1685
	err = xenbus_alloc_evtchn(dev, &rinfo->evtchn);
1686 1687 1688
	if (err)
		goto fail;

1689 1690
	err = bind_evtchn_to_irqhandler(rinfo->evtchn, blkif_interrupt, 0,
					"blkif", rinfo);
1691 1692 1693 1694 1695
	if (err <= 0) {
		xenbus_dev_fatal(dev, err,
				 "bind_evtchn_to_irqhandler failed");
		goto fail;
	}
1696
	rinfo->irq = err;
1697 1698 1699 1700 1701 1702 1703

	return 0;
fail:
	blkif_free(info, 0);
	return err;
}

1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750
/*
 * Write out per-ring/queue nodes including ring-ref and event-channel, and each
 * ring buffer may have multi pages depending on ->nr_ring_pages.
 */
static int write_per_ring_nodes(struct xenbus_transaction xbt,
				struct blkfront_ring_info *rinfo, const char *dir)
{
	int err;
	unsigned int i;
	const char *message = NULL;
	struct blkfront_info *info = rinfo->dev_info;

	if (info->nr_ring_pages == 1) {
		err = xenbus_printf(xbt, dir, "ring-ref", "%u", rinfo->ring_ref[0]);
		if (err) {
			message = "writing ring-ref";
			goto abort_transaction;
		}
	} else {
		for (i = 0; i < info->nr_ring_pages; i++) {
			char ring_ref_name[RINGREF_NAME_LEN];

			snprintf(ring_ref_name, RINGREF_NAME_LEN, "ring-ref%u", i);
			err = xenbus_printf(xbt, dir, ring_ref_name,
					    "%u", rinfo->ring_ref[i]);
			if (err) {
				message = "writing ring-ref";
				goto abort_transaction;
			}
		}
	}

	err = xenbus_printf(xbt, dir, "event-channel", "%u", rinfo->evtchn);
	if (err) {
		message = "writing event-channel";
		goto abort_transaction;
	}

	return 0;

abort_transaction:
	xenbus_transaction_end(xbt, 1);
	if (message)
		xenbus_dev_fatal(info->xbdev, err, "%s", message);

	return err;
}
1751 1752

/* Common code used when first setting up, and when resuming. */
1753
static int talk_to_blkback(struct xenbus_device *dev,
1754 1755 1756 1757
			   struct blkfront_info *info)
{
	const char *message = NULL;
	struct xenbus_transaction xbt;
1758 1759
	int err;
	unsigned int i, max_page_order = 0;
B
Bob Liu 已提交
1760 1761 1762 1763 1764 1765 1766 1767 1768 1769
	unsigned int ring_page_order = 0;

	err = xenbus_scanf(XBT_NIL, info->xbdev->otherend,
			   "max-ring-page-order", "%u", &max_page_order);
	if (err != 1)
		info->nr_ring_pages = 1;
	else {
		ring_page_order = min(xen_blkif_max_ring_order, max_page_order);
		info->nr_ring_pages = 1 << ring_page_order;
	}
1770

1771
	for (i = 0; i < info->nr_rings; i++) {
1772 1773
		struct blkfront_ring_info *rinfo = &info->rinfo[i];

1774 1775 1776 1777 1778
		/* Create shared ring, alloc event channel. */
		err = setup_blkring(dev, rinfo);
		if (err)
			goto destroy_blkring;
	}
1779 1780 1781 1782 1783 1784 1785 1786

again:
	err = xenbus_transaction_start(&xbt);
	if (err) {
		xenbus_dev_fatal(dev, err, "starting transaction");
		goto destroy_blkring;
	}

1787 1788 1789 1790 1791 1792 1793 1794
	if (info->nr_ring_pages > 1) {
		err = xenbus_printf(xbt, dev->nodename, "ring-page-order", "%u",
				    ring_page_order);
		if (err) {
			message = "writing ring-page-order";
			goto abort_transaction;
		}
	}
1795

1796 1797 1798 1799 1800 1801 1802 1803
	/* We already got the number of queues/rings in _probe */
	if (info->nr_rings == 1) {
		err = write_per_ring_nodes(xbt, &info->rinfo[0], dev->nodename);
		if (err)
			goto destroy_blkring;
	} else {
		char *path;
		size_t pathsize;
1804

1805 1806
		err = xenbus_printf(xbt, dev->nodename, "multi-queue-num-queues", "%u",
				    info->nr_rings);
1807
		if (err) {
1808
			message = "writing multi-queue-num-queues";
1809 1810
			goto abort_transaction;
		}
1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829

		pathsize = strlen(dev->nodename) + QUEUE_NAME_LEN;
		path = kmalloc(pathsize, GFP_KERNEL);
		if (!path) {
			err = -ENOMEM;
			message = "ENOMEM while writing ring references";
			goto abort_transaction;
		}

		for (i = 0; i < info->nr_rings; i++) {
			memset(path, 0, pathsize);
			snprintf(path, pathsize, "%s/queue-%u", dev->nodename, i);
			err = write_per_ring_nodes(xbt, &info->rinfo[i], path);
			if (err) {
				kfree(path);
				goto destroy_blkring;
			}
		}
		kfree(path);
1830
	}
1831 1832 1833 1834 1835 1836
	err = xenbus_printf(xbt, dev->nodename, "protocol", "%s",
			    XEN_IO_PROTO_ABI_NATIVE);
	if (err) {
		message = "writing protocol";
		goto abort_transaction;
	}
1837
	err = xenbus_printf(xbt, dev->nodename,
1838
			    "feature-persistent", "%u", 1);
1839 1840 1841
	if (err)
		dev_warn(&dev->dev,
			 "writing persistent grants feature to xenbus");
1842 1843 1844 1845 1846 1847 1848 1849 1850

	err = xenbus_transaction_end(xbt, 0);
	if (err) {
		if (err == -EAGAIN)
			goto again;
		xenbus_dev_fatal(dev, err, "completing transaction");
		goto destroy_blkring;
	}

1851 1852
	for (i = 0; i < info->nr_rings; i++) {
		unsigned int j;
1853
		struct blkfront_ring_info *rinfo = &info->rinfo[i];
1854 1855 1856 1857 1858

		for (j = 0; j < BLK_RING_SIZE(info); j++)
			rinfo->shadow[j].req.u.rw.id = j + 1;
		rinfo->shadow[BLK_RING_SIZE(info)-1].req.u.rw.id = 0x0fffffff;
	}
1859 1860 1861 1862 1863 1864 1865 1866 1867 1868
	xenbus_switch_state(dev, XenbusStateInitialised);

	return 0;

 abort_transaction:
	xenbus_transaction_end(xbt, 1);
	if (message)
		xenbus_dev_fatal(dev, err, "%s", message);
 destroy_blkring:
	blkif_free(info, 0);
1869

1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881
	return err;
}

/**
 * Entry point to this code when a new device is created.  Allocate the basic
 * structures and the ring buffer for communication with the backend, and
 * inform the backend of the appropriate details for those.  Switch to
 * Initialised state.
 */
static int blkfront_probe(struct xenbus_device *dev,
			  const struct xenbus_device_id *id)
{
B
Bob Liu 已提交
1882
	int err, vdevice;
1883
	unsigned int r_index;
1884
	struct blkfront_info *info;
1885
	unsigned int backend_max_queues = 0;
1886 1887 1888 1889 1890

	/* FIXME: Use dynamic device id if this is not set. */
	err = xenbus_scanf(XBT_NIL, dev->nodename,
			   "virtual-device", "%i", &vdevice);
	if (err != 1) {
1891 1892 1893 1894 1895 1896 1897
		/* go looking in the extended area instead */
		err = xenbus_scanf(XBT_NIL, dev->nodename, "virtual-device-ext",
				   "%i", &vdevice);
		if (err != 1) {
			xenbus_dev_fatal(dev, err, "reading virtual-device");
			return err;
		}
1898 1899
	}

1900 1901 1902 1903
	if (xen_hvm_domain()) {
		char *type;
		int len;
		/* no unplug has been done: do not hook devices != xen vbds */
1904
		if (xen_has_pv_and_legacy_disk_devices()) {
1905 1906 1907 1908 1909 1910 1911 1912 1913 1914
			int major;

			if (!VDEV_IS_EXTENDED(vdevice))
				major = BLKIF_MAJOR(vdevice);
			else
				major = XENVBD_MAJOR;

			if (major != XENVBD_MAJOR) {
				printk(KERN_INFO
						"%s: HVM does not support vbd %d as xen block device\n",
1915
						__func__, vdevice);
1916 1917 1918 1919 1920 1921 1922 1923 1924
				return -ENODEV;
			}
		}
		/* do not create a PV cdrom device if we are an HVM guest */
		type = xenbus_read(XBT_NIL, dev->nodename, "device-type", &len);
		if (IS_ERR(type))
			return -ENODEV;
		if (strncmp(type, "cdrom", 5) == 0) {
			kfree(type);
1925 1926
			return -ENODEV;
		}
1927
		kfree(type);
1928
	}
1929 1930 1931 1932 1933 1934
	info = kzalloc(sizeof(*info), GFP_KERNEL);
	if (!info) {
		xenbus_dev_fatal(dev, -ENOMEM, "allocating info structure");
		return -ENOMEM;
	}

1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946
	info->xbdev = dev;
	/* Check if backend supports multiple queues. */
	err = xenbus_scanf(XBT_NIL, info->xbdev->otherend,
			   "multi-queue-max-queues", "%u", &backend_max_queues);
	if (err < 0)
		backend_max_queues = 1;

	info->nr_rings = min(backend_max_queues, xen_blkif_max_queues);
	/* We need at least one ring. */
	if (!info->nr_rings)
		info->nr_rings = 1;

1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958
	info->rinfo = kzalloc(sizeof(struct blkfront_ring_info) * info->nr_rings, GFP_KERNEL);
	if (!info->rinfo) {
		xenbus_dev_fatal(dev, -ENOMEM, "allocating ring_info structure");
		kfree(info);
		return -ENOMEM;
	}

	for (r_index = 0; r_index < info->nr_rings; r_index++) {
		struct blkfront_ring_info *rinfo;

		rinfo = &info->rinfo[r_index];
		INIT_LIST_HEAD(&rinfo->indirect_pages);
1959
		INIT_LIST_HEAD(&rinfo->grants);
1960 1961
		rinfo->dev_info = info;
		INIT_WORK(&rinfo->work, blkif_restart_queue);
B
Bob Liu 已提交
1962
		spin_lock_init(&rinfo->ring_lock);
1963
	}
1964

1965
	mutex_init(&info->mutex);
1966 1967 1968 1969 1970
	info->vdevice = vdevice;
	info->connected = BLKIF_STATE_DISCONNECTED;

	/* Front end dir is a number, which is used as the id. */
	info->handle = simple_strtoul(strrchr(dev->nodename, '/')+1, NULL, 0);
1971
	dev_set_drvdata(&dev->dev, info);
1972 1973 1974 1975

	return 0;
}

1976
static void split_bio_end(struct bio *bio)
1977 1978 1979 1980 1981
{
	struct split_bio *split_bio = bio->bi_private;

	if (atomic_dec_and_test(&split_bio->pending)) {
		split_bio->bio->bi_phys_segments = 0;
1982 1983
		split_bio->bio->bi_error = bio->bi_error;
		bio_endio(split_bio->bio);
1984 1985 1986 1987
		kfree(split_bio);
	}
	bio_put(bio);
}
1988 1989 1990

static int blkif_recover(struct blkfront_info *info)
{
1991
	unsigned int i, r_index;
1992
	struct request *req, *n;
1993
	struct blk_shadow *copy;
1994 1995 1996 1997 1998 1999 2000 2001
	int rc;
	struct bio *bio, *cloned_bio;
	struct bio_list bio_list, merge_bio;
	unsigned int segs, offset;
	int pending, size;
	struct split_bio *split_bio;
	struct list_head requests;

2002
	blkfront_gather_backend_features(info);
2003 2004 2005 2006
	segs = info->max_indirect_segments ? : BLKIF_MAX_SEGMENTS_PER_REQUEST;
	blk_queue_max_segments(info->rq, segs);
	bio_list_init(&bio_list);
	INIT_LIST_HEAD(&requests);
2007

2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035
	for (r_index = 0; r_index < info->nr_rings; r_index++) {
		struct blkfront_ring_info *rinfo;

		rinfo = &info->rinfo[r_index];
		/* Stage 1: Make a safe copy of the shadow state. */
		copy = kmemdup(rinfo->shadow, sizeof(rinfo->shadow),
			       GFP_NOIO | __GFP_REPEAT | __GFP_HIGH);
		if (!copy)
			return -ENOMEM;

		/* Stage 2: Set up free list. */
		memset(&rinfo->shadow, 0, sizeof(rinfo->shadow));
		for (i = 0; i < BLK_RING_SIZE(info); i++)
			rinfo->shadow[i].req.u.rw.id = i+1;
		rinfo->shadow_free = rinfo->ring.req_prod_pvt;
		rinfo->shadow[BLK_RING_SIZE(info)-1].req.u.rw.id = 0x0fffffff;

		rc = blkfront_setup_indirect(rinfo);
		if (rc) {
			kfree(copy);
			return rc;
		}

		for (i = 0; i < BLK_RING_SIZE(info); i++) {
			/* Not in use? */
			if (!copy[i].request)
				continue;

2036
			/*
2037
			 * Get the bios in the request so we can re-queue them.
2038
			 */
2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052
			if (copy[i].request->cmd_flags &
			    (REQ_FLUSH | REQ_FUA | REQ_DISCARD | REQ_SECURE)) {
				/*
				 * Flush operations don't contain bios, so
				 * we need to requeue the whole request
				 */
				list_add(&copy[i].request->queuelist, &requests);
				continue;
			}
			merge_bio.head = copy[i].request->bio;
			merge_bio.tail = copy[i].request->biotail;
			bio_list_merge(&bio_list, &merge_bio);
			copy[i].request->bio = NULL;
			blk_end_request_all(copy[i].request, 0);
2053
		}
2054

2055 2056
		kfree(copy);
	}
2057 2058 2059 2060 2061
	xenbus_switch_state(info->xbdev, XenbusStateConnected);

	/* Now safe for us to use the shared ring */
	info->connected = BLKIF_STATE_CONNECTED;

2062 2063 2064 2065 2066 2067 2068
	for (r_index = 0; r_index < info->nr_rings; r_index++) {
		struct blkfront_ring_info *rinfo;

		rinfo = &info->rinfo[r_index];
		/* Kick any other new requests queued since we resumed */
		kick_pending_request_queues(rinfo);
	}
2069

2070 2071 2072 2073
	list_for_each_entry_safe(req, n, &requests, queuelist) {
		/* Requeue pending requests (flush or discard) */
		list_del_init(&req->queuelist);
		BUG_ON(req->nr_phys_segments > segs);
B
Bob Liu 已提交
2074
		blk_mq_requeue_request(req);
2075
	}
B
Bob Liu 已提交
2076
	blk_mq_kick_requeue_list(info->rq);
2077

2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090
	while ((bio = bio_list_pop(&bio_list)) != NULL) {
		/* Traverse the list of pending bios and re-queue them */
		if (bio_segments(bio) > segs) {
			/*
			 * This bio has more segments than what we can
			 * handle, we have to split it.
			 */
			pending = (bio_segments(bio) + segs - 1) / segs;
			split_bio = kzalloc(sizeof(*split_bio), GFP_NOIO);
			BUG_ON(split_bio == NULL);
			atomic_set(&split_bio->pending, pending);
			split_bio->bio = bio;
			for (i = 0; i < pending; i++) {
2091 2092
				offset = (i * segs * XEN_PAGE_SIZE) >> 9;
				size = min((unsigned int)(segs * XEN_PAGE_SIZE) >> 9,
2093
					   (unsigned int)bio_sectors(bio) - offset);
2094 2095
				cloned_bio = bio_clone(bio, GFP_NOIO);
				BUG_ON(cloned_bio == NULL);
2096
				bio_trim(cloned_bio, offset, size);
2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110
				cloned_bio->bi_private = split_bio;
				cloned_bio->bi_end_io = split_bio_end;
				submit_bio(cloned_bio->bi_rw, cloned_bio);
			}
			/*
			 * Now we have to wait for all those smaller bios to
			 * end, so we can also end the "parent" bio.
			 */
			continue;
		}
		/* We don't need to split this bio */
		submit_bio(bio->bi_rw, bio);
	}

2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121
	return 0;
}

/**
 * We are reconnecting to the backend, due to a suspend/resume, or a backend
 * driver restart.  We tear down our blkif structure and recreate it, but
 * leave the device-layer structures intact so that this is transparent to the
 * rest of the kernel.
 */
static int blkfront_resume(struct xenbus_device *dev)
{
2122
	struct blkfront_info *info = dev_get_drvdata(&dev->dev);
2123 2124 2125 2126 2127 2128
	int err;

	dev_dbg(&dev->dev, "blkfront_resume: %s\n", dev->nodename);

	blkif_free(info, info->connected == BLKIF_STATE_CONNECTED);

2129
	err = talk_to_blkback(dev, info);
2130 2131 2132 2133 2134 2135

	/*
	 * We have to wait for the backend to switch to
	 * connected state, since we want to read which
	 * features it supports.
	 */
2136 2137 2138 2139

	return err;
}

2140
static void blkfront_closing(struct blkfront_info *info)
2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163
{
	struct xenbus_device *xbdev = info->xbdev;
	struct block_device *bdev = NULL;

	mutex_lock(&info->mutex);

	if (xbdev->state == XenbusStateClosing) {
		mutex_unlock(&info->mutex);
		return;
	}

	if (info->gd)
		bdev = bdget_disk(info->gd, 0);

	mutex_unlock(&info->mutex);

	if (!bdev) {
		xenbus_frontend_closed(xbdev);
		return;
	}

	mutex_lock(&bdev->bd_mutex);

2164
	if (bdev->bd_openers) {
2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175
		xenbus_dev_error(xbdev, -EBUSY,
				 "Device in use; refusing to close");
		xenbus_switch_state(xbdev, XenbusStateClosing);
	} else {
		xlvbd_release_gendisk(info);
		xenbus_frontend_closed(xbdev);
	}

	mutex_unlock(&bdev->bd_mutex);
	bdput(bdev);
}
2176

2177 2178 2179 2180 2181
static void blkfront_setup_discard(struct blkfront_info *info)
{
	int err;
	unsigned int discard_granularity;
	unsigned int discard_alignment;
2182
	unsigned int discard_secure;
2183

2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197
	info->feature_discard = 1;
	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
		"discard-granularity", "%u", &discard_granularity,
		"discard-alignment", "%u", &discard_alignment,
		NULL);
	if (!err) {
		info->discard_granularity = discard_granularity;
		info->discard_alignment = discard_alignment;
	}
	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
		    "discard-secure", "%d", &discard_secure,
		    NULL);
	if (!err)
		info->feature_secdiscard = !!discard_secure;
2198 2199
}

2200
static int blkfront_setup_indirect(struct blkfront_ring_info *rinfo)
2201
{
2202
	unsigned int psegs, grants;
2203
	int err, i;
2204
	struct blkfront_info *info = rinfo->dev_info;
2205

2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217
	if (info->max_indirect_segments == 0) {
		if (!HAS_EXTRA_REQ)
			grants = BLKIF_MAX_SEGMENTS_PER_REQUEST;
		else {
			/*
			 * When an extra req is required, the maximum
			 * grants supported is related to the size of the
			 * Linux block segment.
			 */
			grants = GRANTS_PER_PSEG;
		}
	}
2218
	else
2219 2220
		grants = info->max_indirect_segments;
	psegs = grants / GRANTS_PER_PSEG;
2221

2222
	err = fill_grant_buffer(rinfo,
2223
				(grants + INDIRECT_GREFS(grants)) * BLK_RING_SIZE(info));
2224 2225 2226
	if (err)
		goto out_of_memory;

2227 2228 2229 2230 2231 2232
	if (!info->feature_persistent && info->max_indirect_segments) {
		/*
		 * We are using indirect descriptors but not persistent
		 * grants, we need to allocate a set of pages that can be
		 * used for mapping indirect grefs
		 */
2233
		int num = INDIRECT_GREFS(grants) * BLK_RING_SIZE(info);
2234

2235
		BUG_ON(!list_empty(&rinfo->indirect_pages));
2236 2237 2238 2239
		for (i = 0; i < num; i++) {
			struct page *indirect_page = alloc_page(GFP_NOIO);
			if (!indirect_page)
				goto out_of_memory;
2240
			list_add(&indirect_page->lru, &rinfo->indirect_pages);
2241 2242 2243
		}
	}

B
Bob Liu 已提交
2244
	for (i = 0; i < BLK_RING_SIZE(info); i++) {
2245 2246
		rinfo->shadow[i].grants_used = kzalloc(
			sizeof(rinfo->shadow[i].grants_used[0]) * grants,
2247
			GFP_NOIO);
2248
		rinfo->shadow[i].sg = kzalloc(sizeof(rinfo->shadow[i].sg[0]) * psegs, GFP_NOIO);
2249
		if (info->max_indirect_segments)
2250 2251
			rinfo->shadow[i].indirect_grants = kzalloc(
				sizeof(rinfo->shadow[i].indirect_grants[0]) *
2252
				INDIRECT_GREFS(grants),
2253
				GFP_NOIO);
2254 2255
		if ((rinfo->shadow[i].grants_used == NULL) ||
			(rinfo->shadow[i].sg == NULL) ||
2256
		     (info->max_indirect_segments &&
2257
		     (rinfo->shadow[i].indirect_grants == NULL)))
2258
			goto out_of_memory;
2259
		sg_init_table(rinfo->shadow[i].sg, psegs);
2260 2261 2262 2263 2264 2265
	}


	return 0;

out_of_memory:
B
Bob Liu 已提交
2266
	for (i = 0; i < BLK_RING_SIZE(info); i++) {
2267 2268 2269 2270 2271 2272
		kfree(rinfo->shadow[i].grants_used);
		rinfo->shadow[i].grants_used = NULL;
		kfree(rinfo->shadow[i].sg);
		rinfo->shadow[i].sg = NULL;
		kfree(rinfo->shadow[i].indirect_grants);
		rinfo->shadow[i].indirect_grants = NULL;
2273
	}
2274
	if (!list_empty(&rinfo->indirect_pages)) {
2275
		struct page *indirect_page, *n;
2276
		list_for_each_entry_safe(indirect_page, n, &rinfo->indirect_pages, lru) {
2277 2278 2279 2280
			list_del(&indirect_page->lru);
			__free_page(indirect_page);
		}
	}
2281 2282 2283
	return -ENOMEM;
}

2284 2285 2286
/*
 * Gather all backend feature-*
 */
2287
static void blkfront_gather_backend_features(struct blkfront_info *info)
2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343
{
	int err;
	int barrier, flush, discard, persistent;
	unsigned int indirect_segments;

	info->feature_flush = 0;

	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
			"feature-barrier", "%d", &barrier,
			NULL);

	/*
	 * If there's no "feature-barrier" defined, then it means
	 * we're dealing with a very old backend which writes
	 * synchronously; nothing to do.
	 *
	 * If there are barriers, then we use flush.
	 */
	if (!err && barrier)
		info->feature_flush = REQ_FLUSH | REQ_FUA;
	/*
	 * And if there is "feature-flush-cache" use that above
	 * barriers.
	 */
	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
			"feature-flush-cache", "%d", &flush,
			NULL);

	if (!err && flush)
		info->feature_flush = REQ_FLUSH;

	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
			"feature-discard", "%d", &discard,
			NULL);

	if (!err && discard)
		blkfront_setup_discard(info);

	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
			"feature-persistent", "%u", &persistent,
			NULL);
	if (err)
		info->feature_persistent = 0;
	else
		info->feature_persistent = persistent;

	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
			    "feature-max-indirect-segments", "%u", &indirect_segments,
			    NULL);
	if (err)
		info->max_indirect_segments = 0;
	else
		info->max_indirect_segments = min(indirect_segments,
						  xen_blkif_max_segments);
}

2344 2345 2346 2347 2348 2349 2350 2351
/*
 * Invoked when the backend is finally 'ready' (and has told produced
 * the details about the physical device - #sectors, size, etc).
 */
static void blkfront_connect(struct blkfront_info *info)
{
	unsigned long long sectors;
	unsigned long sector_size;
2352
	unsigned int physical_sector_size;
2353
	unsigned int binfo;
2354
	int err, i;
2355

2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368
	switch (info->connected) {
	case BLKIF_STATE_CONNECTED:
		/*
		 * Potentially, the back-end may be signalling
		 * a capacity change; update the capacity.
		 */
		err = xenbus_scanf(XBT_NIL, info->xbdev->otherend,
				   "sectors", "%Lu", &sectors);
		if (XENBUS_EXIST_ERR(err))
			return;
		printk(KERN_INFO "Setting capacity to %Lu\n",
		       sectors);
		set_capacity(info->gd, sectors);
2369
		revalidate_disk(info->gd);
2370

2371
		return;
2372
	case BLKIF_STATE_SUSPENDED:
2373 2374 2375 2376 2377 2378 2379
		/*
		 * If we are recovering from suspension, we need to wait
		 * for the backend to announce it's features before
		 * reconnecting, at least we need to know if the backend
		 * supports indirect descriptors, and how many.
		 */
		blkif_recover(info);
2380 2381
		return;

2382 2383
	default:
		break;
2384
	}
2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400

	dev_dbg(&info->xbdev->dev, "%s:%s.\n",
		__func__, info->xbdev->otherend);

	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
			    "sectors", "%llu", &sectors,
			    "info", "%u", &binfo,
			    "sector-size", "%lu", &sector_size,
			    NULL);
	if (err) {
		xenbus_dev_fatal(info->xbdev, err,
				 "reading backend fields at %s",
				 info->xbdev->otherend);
		return;
	}

2401 2402 2403 2404 2405 2406 2407 2408 2409 2410
	/*
	 * physcial-sector-size is a newer field, so old backends may not
	 * provide this. Assume physical sector size to be the same as
	 * sector_size in that case.
	 */
	err = xenbus_scanf(XBT_NIL, info->xbdev->otherend,
			   "physical-sector-size", "%u", &physical_sector_size);
	if (err != 1)
		physical_sector_size = sector_size;

2411 2412 2413 2414 2415 2416 2417 2418 2419
	blkfront_gather_backend_features(info);
	for (i = 0; i < info->nr_rings; i++) {
		err = blkfront_setup_indirect(&info->rinfo[i]);
		if (err) {
			xenbus_dev_fatal(info->xbdev, err, "setup_indirect at %s",
					 info->xbdev->otherend);
			blkif_free(info, 0);
			break;
		}
2420 2421
	}

2422 2423
	err = xlvbd_alloc_gendisk(sectors, info, binfo, sector_size,
				  physical_sector_size);
2424 2425 2426 2427 2428 2429 2430 2431 2432 2433
	if (err) {
		xenbus_dev_fatal(info->xbdev, err, "xlvbd_add at %s",
				 info->xbdev->otherend);
		return;
	}

	xenbus_switch_state(info->xbdev, XenbusStateConnected);

	/* Kick pending requests. */
	info->connected = BLKIF_STATE_CONNECTED;
2434 2435
	for (i = 0; i < info->nr_rings; i++)
		kick_pending_request_queues(&info->rinfo[i]);
2436 2437

	add_disk(info->gd);
2438 2439

	info->is_ready = 1;
2440 2441 2442 2443 2444
}

/**
 * Callback received when the backend's state changes.
 */
2445
static void blkback_changed(struct xenbus_device *dev,
2446 2447
			    enum xenbus_state backend_state)
{
2448
	struct blkfront_info *info = dev_get_drvdata(&dev->dev);
2449

2450
	dev_dbg(&dev->dev, "blkfront:blkback_changed to state %d.\n", backend_state);
2451 2452 2453

	switch (backend_state) {
	case XenbusStateInitWait:
2454 2455
		if (dev->state != XenbusStateInitialising)
			break;
2456 2457 2458 2459 2460 2461
		if (talk_to_blkback(dev, info)) {
			kfree(info);
			dev_set_drvdata(&dev->dev, NULL);
			break;
		}
	case XenbusStateInitialising:
2462
	case XenbusStateInitialised:
2463 2464
	case XenbusStateReconfiguring:
	case XenbusStateReconfigured:
2465 2466 2467 2468 2469 2470 2471
	case XenbusStateUnknown:
		break;

	case XenbusStateConnected:
		blkfront_connect(info);
		break;

2472 2473 2474 2475
	case XenbusStateClosed:
		if (dev->state == XenbusStateClosed)
			break;
		/* Missed the backend's Closing state -- fallthrough */
2476
	case XenbusStateClosing:
2477 2478
		if (info)
			blkfront_closing(info);
2479 2480 2481 2482
		break;
	}
}

2483
static int blkfront_remove(struct xenbus_device *xbdev)
2484
{
2485 2486 2487
	struct blkfront_info *info = dev_get_drvdata(&xbdev->dev);
	struct block_device *bdev = NULL;
	struct gendisk *disk;
2488

2489
	dev_dbg(&xbdev->dev, "%s removed", xbdev->nodename);
2490 2491 2492

	blkif_free(info, 0);

2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515
	mutex_lock(&info->mutex);

	disk = info->gd;
	if (disk)
		bdev = bdget_disk(disk, 0);

	info->xbdev = NULL;
	mutex_unlock(&info->mutex);

	if (!bdev) {
		kfree(info);
		return 0;
	}

	/*
	 * The xbdev was removed before we reached the Closed
	 * state. See if it's safe to remove the disk. If the bdev
	 * isn't closed yet, we let release take care of it.
	 */

	mutex_lock(&bdev->bd_mutex);
	info = disk->private_data;

2516 2517 2518 2519
	dev_warn(disk_to_dev(disk),
		 "%s was hot-unplugged, %d stale handles\n",
		 xbdev->nodename, bdev->bd_openers);

2520
	if (info && !bdev->bd_openers) {
2521 2522
		xlvbd_release_gendisk(info);
		disk->private_data = NULL;
2523
		kfree(info);
2524 2525 2526 2527
	}

	mutex_unlock(&bdev->bd_mutex);
	bdput(bdev);
2528 2529 2530 2531

	return 0;
}

2532 2533
static int blkfront_is_ready(struct xenbus_device *dev)
{
2534
	struct blkfront_info *info = dev_get_drvdata(&dev->dev);
2535

2536
	return info->is_ready && info->xbdev;
2537 2538
}

A
Al Viro 已提交
2539
static int blkif_open(struct block_device *bdev, fmode_t mode)
2540
{
2541 2542 2543
	struct gendisk *disk = bdev->bd_disk;
	struct blkfront_info *info;
	int err = 0;
2544

2545
	mutex_lock(&blkfront_mutex);
2546

2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562
	info = disk->private_data;
	if (!info) {
		/* xbdev gone */
		err = -ERESTARTSYS;
		goto out;
	}

	mutex_lock(&info->mutex);

	if (!info->gd)
		/* xbdev is closed */
		err = -ERESTARTSYS;

	mutex_unlock(&info->mutex);

out:
2563
	mutex_unlock(&blkfront_mutex);
2564
	return err;
2565 2566
}

2567
static void blkif_release(struct gendisk *disk, fmode_t mode)
2568
{
A
Al Viro 已提交
2569
	struct blkfront_info *info = disk->private_data;
2570 2571 2572
	struct block_device *bdev;
	struct xenbus_device *xbdev;

2573
	mutex_lock(&blkfront_mutex);
2574 2575 2576

	bdev = bdget_disk(disk, 0);

2577 2578 2579 2580
	if (!bdev) {
		WARN(1, "Block device %s yanked out from us!\n", disk->disk_name);
		goto out_mutex;
	}
2581 2582 2583
	if (bdev->bd_openers)
		goto out;

2584 2585 2586 2587 2588 2589 2590 2591 2592 2593
	/*
	 * Check if we have been instructed to close. We will have
	 * deferred this request, because the bdev was still open.
	 */

	mutex_lock(&info->mutex);
	xbdev = info->xbdev;

	if (xbdev && xbdev->state == XenbusStateClosing) {
		/* pending switch to state closed */
2594
		dev_info(disk_to_dev(bdev->bd_disk), "releasing disk\n");
2595 2596 2597 2598 2599 2600 2601 2602
		xlvbd_release_gendisk(info);
		xenbus_frontend_closed(info->xbdev);
 	}

	mutex_unlock(&info->mutex);

	if (!xbdev) {
		/* sudden device removal */
2603
		dev_info(disk_to_dev(bdev->bd_disk), "releasing disk\n");
2604 2605 2606
		xlvbd_release_gendisk(info);
		disk->private_data = NULL;
		kfree(info);
2607
	}
2608

J
Jens Axboe 已提交
2609
out:
2610
	bdput(bdev);
2611
out_mutex:
2612
	mutex_unlock(&blkfront_mutex);
2613 2614
}

2615
static const struct block_device_operations xlvbd_block_fops =
2616 2617
{
	.owner = THIS_MODULE,
A
Al Viro 已提交
2618 2619
	.open = blkif_open,
	.release = blkif_release,
2620
	.getgeo = blkif_getgeo,
2621
	.ioctl = blkif_ioctl,
2622 2623 2624
};


2625
static const struct xenbus_device_id blkfront_ids[] = {
2626 2627 2628 2629
	{ "vbd" },
	{ "" }
};

2630 2631
static struct xenbus_driver blkfront_driver = {
	.ids  = blkfront_ids,
2632 2633 2634
	.probe = blkfront_probe,
	.remove = blkfront_remove,
	.resume = blkfront_resume,
2635
	.otherend_changed = blkback_changed,
2636
	.is_ready = blkfront_is_ready,
2637
};
2638 2639 2640

static int __init xlblk_init(void)
{
2641
	int ret;
2642
	int nr_cpus = num_online_cpus();
2643

2644
	if (!xen_domain())
2645 2646
		return -ENODEV;

2647
	if (xen_blkif_max_ring_order > XENBUS_MAX_RING_GRANT_ORDER) {
B
Bob Liu 已提交
2648
		pr_info("Invalid max_ring_order (%d), will use default max: %d.\n",
2649
			xen_blkif_max_ring_order, XENBUS_MAX_RING_GRANT_ORDER);
2650
		xen_blkif_max_ring_order = XENBUS_MAX_RING_GRANT_ORDER;
B
Bob Liu 已提交
2651 2652
	}

2653 2654 2655 2656 2657 2658
	if (xen_blkif_max_queues > nr_cpus) {
		pr_info("Invalid max_queues (%d), will use default max: %d.\n",
			xen_blkif_max_queues, nr_cpus);
		xen_blkif_max_queues = nr_cpus;
	}

2659
	if (!xen_has_pv_disk_devices())
2660 2661
		return -ENODEV;

2662 2663 2664 2665 2666 2667
	if (register_blkdev(XENVBD_MAJOR, DEV_NAME)) {
		printk(KERN_WARNING "xen_blk: can't get major %d with name %s\n",
		       XENVBD_MAJOR, DEV_NAME);
		return -ENODEV;
	}

2668
	ret = xenbus_register_frontend(&blkfront_driver);
2669 2670 2671 2672 2673 2674
	if (ret) {
		unregister_blkdev(XENVBD_MAJOR, DEV_NAME);
		return ret;
	}

	return 0;
2675 2676 2677 2678
}
module_init(xlblk_init);


2679
static void __exit xlblk_exit(void)
2680
{
2681 2682 2683
	xenbus_unregister_driver(&blkfront_driver);
	unregister_blkdev(XENVBD_MAJOR, DEV_NAME);
	kfree(minors);
2684 2685 2686 2687 2688 2689
}
module_exit(xlblk_exit);

MODULE_DESCRIPTION("Xen virtual block device frontend");
MODULE_LICENSE("GPL");
MODULE_ALIAS_BLOCKDEV_MAJOR(XENVBD_MAJOR);
2690
MODULE_ALIAS("xen:vbd");
2691
MODULE_ALIAS("xenblk");