xen-blkfront.c 72.0 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39
/*
 * blkfront.c
 *
 * XenLinux virtual block device driver.
 *
 * Copyright (c) 2003-2004, Keir Fraser & Steve Hand
 * Modifications by Mark A. Williamson are (c) Intel Research Cambridge
 * Copyright (c) 2004, Christian Limpach
 * Copyright (c) 2004, Andrew Warfield
 * Copyright (c) 2005, Christopher Clark
 * Copyright (c) 2005, XenSource Ltd
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License version 2
 * as published by the Free Software Foundation; or, when distributed
 * separately from the Linux kernel or incorporated into other
 * software packages, subject to the following license:
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this source file (the "Software"), to deal in the Software without
 * restriction, including without limitation the rights to use, copy, modify,
 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
 * and to permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include <linux/interrupt.h>
#include <linux/blkdev.h>
B
Bob Liu 已提交
40
#include <linux/blk-mq.h>
41
#include <linux/hdreg.h>
42
#include <linux/cdrom.h>
43
#include <linux/module.h>
44
#include <linux/slab.h>
45
#include <linux/mutex.h>
46
#include <linux/scatterlist.h>
47
#include <linux/bitmap.h>
48
#include <linux/list.h>
49

50
#include <xen/xen.h>
51 52 53 54
#include <xen/xenbus.h>
#include <xen/grant_table.h>
#include <xen/events.h>
#include <xen/page.h>
55
#include <xen/platform_pci.h>
56 57 58

#include <xen/interface/grant_table.h>
#include <xen/interface/io/blkif.h>
59
#include <xen/interface/io/protocols.h>
60 61 62

#include <asm/xen/hypervisor.h>

63 64 65 66 67 68 69 70 71 72 73 74 75 76
/*
 * The minimal size of segment supported by the block framework is PAGE_SIZE.
 * When Linux is using a different page size than Xen, it may not be possible
 * to put all the data in a single segment.
 * This can happen when the backend doesn't support indirect descriptor and
 * therefore the maximum amount of data that a request can carry is
 * BLKIF_MAX_SEGMENTS_PER_REQUEST * XEN_PAGE_SIZE = 44KB
 *
 * Note that we only support one extra request. So the Linux page size
 * should be <= ( 2 * BLKIF_MAX_SEGMENTS_PER_REQUEST * XEN_PAGE_SIZE) =
 * 88KB.
 */
#define HAS_EXTRA_REQ (BLKIF_MAX_SEGMENTS_PER_REQUEST < XEN_PFN_PER_PAGE)

77 78 79 80 81 82
enum blkif_state {
	BLKIF_STATE_DISCONNECTED,
	BLKIF_STATE_CONNECTED,
	BLKIF_STATE_SUSPENDED,
};

83 84
struct grant {
	grant_ref_t gref;
85
	struct page *page;
86
	struct list_head node;
87 88
};

89 90 91 92 93 94 95
enum blk_req_status {
	REQ_WAITING,
	REQ_DONE,
	REQ_ERROR,
	REQ_EOPNOTSUPP,
};

96 97
struct blk_shadow {
	struct blkif_request req;
98
	struct request *request;
99 100
	struct grant **grants_used;
	struct grant **indirect_grants;
101
	struct scatterlist *sg;
102
	unsigned int num_sg;
103 104 105 106 107 108 109 110
	enum blk_req_status status;

	#define NO_ASSOCIATED_ID ~0UL
	/*
	 * Id of the sibling if we ever need 2 requests when handling a
	 * block I/O request
	 */
	unsigned long associated_id;
111 112 113 114 115
};

struct split_bio {
	struct bio *bio;
	atomic_t pending;
116 117
};

118
static DEFINE_MUTEX(blkfront_mutex);
119
static const struct block_device_operations xlvbd_block_fops;
120

121 122 123 124 125 126 127
/*
 * Maximum number of segments in indirect requests, the actual value used by
 * the frontend driver is the minimum of this value and the value provided
 * by the backend driver.
 */

static unsigned int xen_blkif_max_segments = 32;
128 129 130 131
module_param_named(max_indirect_segments, xen_blkif_max_segments, uint,
		   S_IRUGO);
MODULE_PARM_DESC(max_indirect_segments,
		 "Maximum amount of segments in indirect requests (default is 32)");
132

133 134 135 136
static unsigned int xen_blkif_max_queues = 4;
module_param_named(max_queues, xen_blkif_max_queues, uint, S_IRUGO);
MODULE_PARM_DESC(max_queues, "Maximum number of hardware queues/rings used per virtual disk");

B
Bob Liu 已提交
137 138 139 140 141 142 143 144
/*
 * Maximum order of pages to be used for the shared ring between front and
 * backend, 4KB page granularity is used.
 */
static unsigned int xen_blkif_max_ring_order;
module_param_named(max_ring_page_order, xen_blkif_max_ring_order, int, S_IRUGO);
MODULE_PARM_DESC(max_ring_page_order, "Maximum order of pages to be used for the shared ring");

145 146 147 148
#define BLK_RING_SIZE(info)	\
	__CONST_RING_SIZE(blkif, XEN_PAGE_SIZE * (info)->nr_ring_pages)

#define BLK_MAX_RING_SIZE	\
149
	__CONST_RING_SIZE(blkif, XEN_PAGE_SIZE * XENBUS_MAX_RING_GRANTS)
150

B
Bob Liu 已提交
151
/*
152 153
 * ring-ref%u i=(-1UL) would take 11 characters + 'ring-ref' is 8, so 19
 * characters are enough. Define to 20 to keep consistent with backend.
B
Bob Liu 已提交
154 155
 */
#define RINGREF_NAME_LEN (20)
156 157 158 159
/*
 * queue-%u would take 7 + 10(UINT_MAX) = 17 characters.
 */
#define QUEUE_NAME_LEN (17)
160

161 162 163 164 165 166
/*
 *  Per-ring info.
 *  Every blkfront device can associate with one or more blkfront_ring_info,
 *  depending on how many hardware queues/rings to be used.
 */
struct blkfront_ring_info {
B
Bob Liu 已提交
167 168
	/* Lock to protect data in every ring buffer. */
	spinlock_t ring_lock;
169 170 171 172 173 174 175
	struct blkif_front_ring ring;
	unsigned int ring_ref[XENBUS_MAX_RING_GRANTS];
	unsigned int evtchn, irq;
	struct work_struct work;
	struct gnttab_free_callback callback;
	struct blk_shadow shadow[BLK_MAX_RING_SIZE];
	struct list_head indirect_pages;
176 177
	struct list_head grants;
	unsigned int persistent_gnts_c;
178 179 180 181
	unsigned long shadow_free;
	struct blkfront_info *dev_info;
};

182 183 184 185 186 187 188
/*
 * We have one of these per vbd, whether ide, scsi or 'other'.  They
 * hang in private_data off the gendisk structure. We may end up
 * putting all kinds of interesting stuff here :-)
 */
struct blkfront_info
{
189
	struct mutex mutex;
190 191 192 193 194
	struct xenbus_device *xbdev;
	struct gendisk *gd;
	int vdevice;
	blkif_vdev_t handle;
	enum blkif_state connected;
195
	/* Number of pages per ring buffer. */
B
Bob Liu 已提交
196
	unsigned int nr_ring_pages;
197
	struct request_queue *rq;
198
	unsigned int feature_flush;
199
	unsigned int feature_fua;
200 201
	unsigned int feature_discard:1;
	unsigned int feature_secdiscard:1;
202 203
	unsigned int discard_granularity;
	unsigned int discard_alignment;
204
	unsigned int feature_persistent:1;
205
	/* Number of 4KB segments handled */
206
	unsigned int max_indirect_segments;
207
	int is_ready;
B
Bob Liu 已提交
208
	struct blk_mq_tag_set tag_set;
209 210
	struct blkfront_ring_info *rinfo;
	unsigned int nr_rings;
211 212
};

213 214 215 216
static unsigned int nr_minors;
static unsigned long *minors;
static DEFINE_SPINLOCK(minor_lock);

217 218 219
#define GRANT_INVALID_REF	0

#define PARTS_PER_DISK		16
220
#define PARTS_PER_EXT_DISK      256
221 222 223 224

#define BLKIF_MAJOR(dev) ((dev)>>8)
#define BLKIF_MINOR(dev) ((dev) & 0xff)

225 226 227 228
#define EXT_SHIFT 28
#define EXTENDED (1<<EXT_SHIFT)
#define VDEV_IS_EXTENDED(dev) ((dev)&(EXTENDED))
#define BLKIF_MINOR_EXT(dev) ((dev)&(~EXTENDED))
229 230
#define EMULATED_HD_DISK_MINOR_OFFSET (0)
#define EMULATED_HD_DISK_NAME_OFFSET (EMULATED_HD_DISK_MINOR_OFFSET / 256)
231 232
#define EMULATED_SD_DISK_MINOR_OFFSET (0)
#define EMULATED_SD_DISK_NAME_OFFSET (EMULATED_SD_DISK_MINOR_OFFSET / 256)
233

234
#define DEV_NAME	"xvd"	/* name in /dev */
235

236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252
/*
 * Grants are always the same size as a Xen page (i.e 4KB).
 * A physical segment is always the same size as a Linux page.
 * Number of grants per physical segment
 */
#define GRANTS_PER_PSEG	(PAGE_SIZE / XEN_PAGE_SIZE)

#define GRANTS_PER_INDIRECT_FRAME \
	(XEN_PAGE_SIZE / sizeof(struct blkif_request_segment))

#define PSEGS_PER_INDIRECT_FRAME	\
	(GRANTS_INDIRECT_FRAME / GRANTS_PSEGS)

#define INDIRECT_GREFS(_grants)		\
	DIV_ROUND_UP(_grants, GRANTS_PER_INDIRECT_FRAME)

#define GREFS(_psegs)	((_psegs) * GRANTS_PER_PSEG)
253

254
static int blkfront_setup_indirect(struct blkfront_ring_info *rinfo);
255
static void blkfront_gather_backend_features(struct blkfront_info *info);
256

257
static int get_id_from_freelist(struct blkfront_ring_info *rinfo)
258
{
259 260 261 262 263
	unsigned long free = rinfo->shadow_free;

	BUG_ON(free >= BLK_RING_SIZE(rinfo->dev_info));
	rinfo->shadow_free = rinfo->shadow[free].req.u.rw.id;
	rinfo->shadow[free].req.u.rw.id = 0x0fffffee; /* debug */
264 265 266
	return free;
}

267
static int add_id_to_freelist(struct blkfront_ring_info *rinfo,
268
			      unsigned long id)
269
{
270
	if (rinfo->shadow[id].req.u.rw.id != id)
271
		return -EINVAL;
272
	if (rinfo->shadow[id].request == NULL)
273
		return -EINVAL;
274 275 276
	rinfo->shadow[id].req.u.rw.id  = rinfo->shadow_free;
	rinfo->shadow[id].request = NULL;
	rinfo->shadow_free = id;
277
	return 0;
278 279
}

280
static int fill_grant_buffer(struct blkfront_ring_info *rinfo, int num)
281
{
282
	struct blkfront_info *info = rinfo->dev_info;
283 284 285 286
	struct page *granted_page;
	struct grant *gnt_list_entry, *n;
	int i = 0;

287
	while (i < num) {
288 289 290 291
		gnt_list_entry = kzalloc(sizeof(struct grant), GFP_NOIO);
		if (!gnt_list_entry)
			goto out_of_memory;

292 293 294 295 296 297
		if (info->feature_persistent) {
			granted_page = alloc_page(GFP_NOIO);
			if (!granted_page) {
				kfree(gnt_list_entry);
				goto out_of_memory;
			}
298
			gnt_list_entry->page = granted_page;
299 300 301
		}

		gnt_list_entry->gref = GRANT_INVALID_REF;
302
		list_add(&gnt_list_entry->node, &rinfo->grants);
303 304 305 306 307 308 309
		i++;
	}

	return 0;

out_of_memory:
	list_for_each_entry_safe(gnt_list_entry, n,
310
	                         &rinfo->grants, node) {
311
		list_del(&gnt_list_entry->node);
312
		if (info->feature_persistent)
313
			__free_page(gnt_list_entry->page);
314 315 316 317 318 319 320
		kfree(gnt_list_entry);
		i--;
	}
	BUG_ON(i != 0);
	return -ENOMEM;
}

321
static struct grant *get_free_grant(struct blkfront_ring_info *rinfo)
322 323 324
{
	struct grant *gnt_list_entry;

325 326
	BUG_ON(list_empty(&rinfo->grants));
	gnt_list_entry = list_first_entry(&rinfo->grants, struct grant,
327
					  node);
328 329
	list_del(&gnt_list_entry->node);

330
	if (gnt_list_entry->gref != GRANT_INVALID_REF)
331
		rinfo->persistent_gnts_c--;
332 333 334 335 336 337 338 339 340 341 342 343 344 345 346

	return gnt_list_entry;
}

static inline void grant_foreign_access(const struct grant *gnt_list_entry,
					const struct blkfront_info *info)
{
	gnttab_page_grant_foreign_access_ref_one(gnt_list_entry->gref,
						 info->xbdev->otherend_id,
						 gnt_list_entry->page,
						 0);
}

static struct grant *get_grant(grant_ref_t *gref_head,
			       unsigned long gfn,
347
			       struct blkfront_ring_info *rinfo)
348
{
349 350
	struct grant *gnt_list_entry = get_free_grant(rinfo);
	struct blkfront_info *info = rinfo->dev_info;
351 352

	if (gnt_list_entry->gref != GRANT_INVALID_REF)
353
		return gnt_list_entry;
354 355 356 357 358 359 360 361 362 363 364

	/* Assign a gref to this page */
	gnt_list_entry->gref = gnttab_claim_grant_reference(gref_head);
	BUG_ON(gnt_list_entry->gref == -ENOSPC);
	if (info->feature_persistent)
		grant_foreign_access(gnt_list_entry, info);
	else {
		/* Grant access to the GFN passed by the caller */
		gnttab_grant_foreign_access_ref(gnt_list_entry->gref,
						info->xbdev->otherend_id,
						gfn, 0);
365 366
	}

367 368 369 370
	return gnt_list_entry;
}

static struct grant *get_indirect_grant(grant_ref_t *gref_head,
371
					struct blkfront_ring_info *rinfo)
372
{
373 374
	struct grant *gnt_list_entry = get_free_grant(rinfo);
	struct blkfront_info *info = rinfo->dev_info;
375 376 377 378

	if (gnt_list_entry->gref != GRANT_INVALID_REF)
		return gnt_list_entry;

379 380 381
	/* Assign a gref to this page */
	gnt_list_entry->gref = gnttab_claim_grant_reference(gref_head);
	BUG_ON(gnt_list_entry->gref == -ENOSPC);
382
	if (!info->feature_persistent) {
383 384 385
		struct page *indirect_page;

		/* Fetch a pre-allocated page to use for indirect grefs */
386 387
		BUG_ON(list_empty(&rinfo->indirect_pages));
		indirect_page = list_first_entry(&rinfo->indirect_pages,
388 389 390
						 struct page, lru);
		list_del(&indirect_page->lru);
		gnt_list_entry->page = indirect_page;
391
	}
392 393
	grant_foreign_access(gnt_list_entry, info);

394 395 396
	return gnt_list_entry;
}

397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413
static const char *op_name(int op)
{
	static const char *const names[] = {
		[BLKIF_OP_READ] = "read",
		[BLKIF_OP_WRITE] = "write",
		[BLKIF_OP_WRITE_BARRIER] = "barrier",
		[BLKIF_OP_FLUSH_DISKCACHE] = "flush",
		[BLKIF_OP_DISCARD] = "discard" };

	if (op < 0 || op >= ARRAY_SIZE(names))
		return "unknown";

	if (!names[op])
		return "reserved";

	return names[op];
}
414 415 416 417 418 419 420 421
static int xlbd_reserve_minors(unsigned int minor, unsigned int nr)
{
	unsigned int end = minor + nr;
	int rc;

	if (end > nr_minors) {
		unsigned long *bitmap, *old;

422
		bitmap = kcalloc(BITS_TO_LONGS(end), sizeof(*bitmap),
423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441
				 GFP_KERNEL);
		if (bitmap == NULL)
			return -ENOMEM;

		spin_lock(&minor_lock);
		if (end > nr_minors) {
			old = minors;
			memcpy(bitmap, minors,
			       BITS_TO_LONGS(nr_minors) * sizeof(*bitmap));
			minors = bitmap;
			nr_minors = BITS_TO_LONGS(end) * BITS_PER_LONG;
		} else
			old = bitmap;
		spin_unlock(&minor_lock);
		kfree(old);
	}

	spin_lock(&minor_lock);
	if (find_next_bit(minors, end, minor) >= end) {
442
		bitmap_set(minors, minor, nr);
443 444 445 446 447 448 449 450 451 452 453 454 455 456
		rc = 0;
	} else
		rc = -EBUSY;
	spin_unlock(&minor_lock);

	return rc;
}

static void xlbd_release_minors(unsigned int minor, unsigned int nr)
{
	unsigned int end = minor + nr;

	BUG_ON(end > nr_minors);
	spin_lock(&minor_lock);
457
	bitmap_clear(minors,  minor, nr);
458 459 460
	spin_unlock(&minor_lock);
}

461 462
static void blkif_restart_queue_callback(void *arg)
{
463 464
	struct blkfront_ring_info *rinfo = (struct blkfront_ring_info *)arg;
	schedule_work(&rinfo->work);
465 466
}

H
Harvey Harrison 已提交
467
static int blkif_getgeo(struct block_device *bd, struct hd_geometry *hg)
468 469 470 471 472 473 474 475 476 477 478 479 480 481 482
{
	/* We don't have real geometry info, but let's at least return
	   values consistent with the size of the device */
	sector_t nsect = get_capacity(bd->bd_disk);
	sector_t cylinders = nsect;

	hg->heads = 0xff;
	hg->sectors = 0x3f;
	sector_div(cylinders, hg->heads * hg->sectors);
	hg->cylinders = cylinders;
	if ((sector_t)(hg->cylinders + 1) * hg->heads * hg->sectors < nsect)
		hg->cylinders = 0xffff;
	return 0;
}

A
Al Viro 已提交
483
static int blkif_ioctl(struct block_device *bdev, fmode_t mode,
484
		       unsigned command, unsigned long argument)
485
{
A
Al Viro 已提交
486
	struct blkfront_info *info = bdev->bd_disk->private_data;
487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515
	int i;

	dev_dbg(&info->xbdev->dev, "command: 0x%x, argument: 0x%lx\n",
		command, (long)argument);

	switch (command) {
	case CDROMMULTISESSION:
		dev_dbg(&info->xbdev->dev, "FIXME: support multisession CDs later\n");
		for (i = 0; i < sizeof(struct cdrom_multisession); i++)
			if (put_user(0, (char __user *)(argument + i)))
				return -EFAULT;
		return 0;

	case CDROM_GET_CAPABILITY: {
		struct gendisk *gd = info->gd;
		if (gd->flags & GENHD_FL_CD)
			return 0;
		return -EINVAL;
	}

	default:
		/*printk(KERN_ALERT "ioctl %08x not supported by Xen blkdev\n",
		  command);*/
		return -EINVAL; /* same return as native Linux */
	}

	return 0;
}

516 517 518 519 520 521 522 523 524 525 526
static unsigned long blkif_ring_get_request(struct blkfront_ring_info *rinfo,
					    struct request *req,
					    struct blkif_request **ring_req)
{
	unsigned long id;

	*ring_req = RING_GET_REQUEST(&rinfo->ring, rinfo->ring.req_prod_pvt);
	rinfo->ring.req_prod_pvt++;

	id = get_id_from_freelist(rinfo);
	rinfo->shadow[id].request = req;
527 528
	rinfo->shadow[id].status = REQ_WAITING;
	rinfo->shadow[id].associated_id = NO_ASSOCIATED_ID;
529 530 531 532 533 534

	(*ring_req)->u.rw.id = id;

	return id;
}

535
static int blkif_queue_discard_req(struct request *req, struct blkfront_ring_info *rinfo)
536
{
537
	struct blkfront_info *info = rinfo->dev_info;
538 539
	struct blkif_request *ring_req;
	unsigned long id;
540 541

	/* Fill out a communications ring structure. */
542
	id = blkif_ring_get_request(rinfo, req, &ring_req);
543 544 545 546 547

	ring_req->operation = BLKIF_OP_DISCARD;
	ring_req->u.discard.nr_sectors = blk_rq_sectors(req);
	ring_req->u.discard.id = id;
	ring_req->u.discard.sector_number = (blkif_sector_t)blk_rq_pos(req);
548
	if (req_op(req) == REQ_OP_SECURE_ERASE && info->feature_secdiscard)
549 550 551 552 553
		ring_req->u.discard.flag = BLKIF_DISCARD_SECURE;
	else
		ring_req->u.discard.flag = 0;

	/* Keep a private copy so we can reissue requests when recovering. */
554
	rinfo->shadow[id].req = *ring_req;
555 556 557 558

	return 0;
}

559 560 561
struct setup_rw_req {
	unsigned int grant_idx;
	struct blkif_request_segment *segments;
562
	struct blkfront_ring_info *rinfo;
563 564 565 566 567 568 569
	struct blkif_request *ring_req;
	grant_ref_t gref_head;
	unsigned int id;
	/* Only used when persistent grant is used and it's a read request */
	bool need_copy;
	unsigned int bvec_off;
	char *bvec_data;
570 571 572

	bool require_extra_req;
	struct blkif_request *extra_ring_req;
573 574 575 576 577 578 579 580
};

static void blkif_setup_rw_req_grant(unsigned long gfn, unsigned int offset,
				     unsigned int len, void *data)
{
	struct setup_rw_req *setup = data;
	int n, ref;
	struct grant *gnt_list_entry;
581
	unsigned int fsect, lsect;
582 583 584
	/* Convenient aliases */
	unsigned int grant_idx = setup->grant_idx;
	struct blkif_request *ring_req = setup->ring_req;
585
	struct blkfront_ring_info *rinfo = setup->rinfo;
586 587 588 589 590 591
	/*
	 * We always use the shadow of the first request to store the list
	 * of grant associated to the block I/O request. This made the
	 * completion more easy to handle even if the block I/O request is
	 * split.
	 */
592
	struct blk_shadow *shadow = &rinfo->shadow[setup->id];
593

594 595 596 597 598 599 600 601 602 603
	if (unlikely(setup->require_extra_req &&
		     grant_idx >= BLKIF_MAX_SEGMENTS_PER_REQUEST)) {
		/*
		 * We are using the second request, setup grant_idx
		 * to be the index of the segment array.
		 */
		grant_idx -= BLKIF_MAX_SEGMENTS_PER_REQUEST;
		ring_req = setup->extra_ring_req;
	}

604 605 606 607 608 609
	if ((ring_req->operation == BLKIF_OP_INDIRECT) &&
	    (grant_idx % GRANTS_PER_INDIRECT_FRAME == 0)) {
		if (setup->segments)
			kunmap_atomic(setup->segments);

		n = grant_idx / GRANTS_PER_INDIRECT_FRAME;
610
		gnt_list_entry = get_indirect_grant(&setup->gref_head, rinfo);
611 612 613 614 615
		shadow->indirect_grants[n] = gnt_list_entry;
		setup->segments = kmap_atomic(gnt_list_entry->page);
		ring_req->u.indirect.indirect_grefs[n] = gnt_list_entry->gref;
	}

616
	gnt_list_entry = get_grant(&setup->gref_head, gfn, rinfo);
617
	ref = gnt_list_entry->gref;
618 619 620 621 622
	/*
	 * All the grants are stored in the shadow of the first
	 * request. Therefore we have to use the global index.
	 */
	shadow->grants_used[setup->grant_idx] = gnt_list_entry;
623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663

	if (setup->need_copy) {
		void *shared_data;

		shared_data = kmap_atomic(gnt_list_entry->page);
		/*
		 * this does not wipe data stored outside the
		 * range sg->offset..sg->offset+sg->length.
		 * Therefore, blkback *could* see data from
		 * previous requests. This is OK as long as
		 * persistent grants are shared with just one
		 * domain. It may need refactoring if this
		 * changes
		 */
		memcpy(shared_data + offset,
		       setup->bvec_data + setup->bvec_off,
		       len);

		kunmap_atomic(shared_data);
		setup->bvec_off += len;
	}

	fsect = offset >> 9;
	lsect = fsect + (len >> 9) - 1;
	if (ring_req->operation != BLKIF_OP_INDIRECT) {
		ring_req->u.rw.seg[grant_idx] =
			(struct blkif_request_segment) {
				.gref       = ref,
				.first_sect = fsect,
				.last_sect  = lsect };
	} else {
		setup->segments[grant_idx % GRANTS_PER_INDIRECT_FRAME] =
			(struct blkif_request_segment) {
				.gref       = ref,
				.first_sect = fsect,
				.last_sect  = lsect };
	}

	(setup->grant_idx)++;
}

664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682
static void blkif_setup_extra_req(struct blkif_request *first,
				  struct blkif_request *second)
{
	uint16_t nr_segments = first->u.rw.nr_segments;

	/*
	 * The second request is only present when the first request uses
	 * all its segments. It's always the continuity of the first one.
	 */
	first->u.rw.nr_segments = BLKIF_MAX_SEGMENTS_PER_REQUEST;

	second->u.rw.nr_segments = nr_segments - BLKIF_MAX_SEGMENTS_PER_REQUEST;
	second->u.rw.sector_number = first->u.rw.sector_number +
		(BLKIF_MAX_SEGMENTS_PER_REQUEST * XEN_PAGE_SIZE) / 512;

	second->u.rw.handle = first->u.rw.handle;
	second->operation = first->operation;
}

683
static int blkif_queue_rw_req(struct request *req, struct blkfront_ring_info *rinfo)
684
{
685
	struct blkfront_info *info = rinfo->dev_info;
686 687 688
	struct blkif_request *ring_req, *extra_ring_req = NULL;
	unsigned long id, extra_id = NO_ASSOCIATED_ID;
	bool require_extra_req = false;
689 690 691 692
	int i;
	struct setup_rw_req setup = {
		.grant_idx = 0,
		.segments = NULL,
693
		.rinfo = rinfo,
694 695
		.need_copy = rq_data_dir(req) && info->feature_persistent,
	};
696 697 698 699 700 701

	/*
	 * Used to store if we are able to queue the request by just using
	 * existing persistent grants, or if we have to get new grants,
	 * as there are not sufficiently many free.
	 */
702
	struct scatterlist *sg;
703
	int num_sg, max_grefs, num_grant;
704

705
	max_grefs = req->nr_phys_segments * GRANTS_PER_PSEG;
706 707 708 709 710
	if (max_grefs > BLKIF_MAX_SEGMENTS_PER_REQUEST)
		/*
		 * If we are using indirect segments we need to account
		 * for the indirect grefs used in the request.
		 */
711
		max_grefs += INDIRECT_GREFS(max_grefs);
712

713 714 715 716 717 718
	/*
	 * We have to reserve 'max_grefs' grants because persistent
	 * grants are shared by all rings.
	 */
	if (max_grefs > 0)
		if (gnttab_alloc_grant_references(max_grefs, &setup.gref_head) < 0) {
719
			gnttab_request_free_callback(
720
				&rinfo->callback,
721
				blkif_restart_queue_callback,
722
				rinfo,
723
				max_grefs);
724 725
			return 1;
		}
726 727

	/* Fill out a communications ring structure. */
728
	id = blkif_ring_get_request(rinfo, req, &ring_req);
729

730
	num_sg = blk_rq_map_sg(req->q, req, rinfo->shadow[id].sg);
731 732
	num_grant = 0;
	/* Calculate the number of grant used */
733
	for_each_sg(rinfo->shadow[id].sg, sg, num_sg, i)
734 735
	       num_grant += gnttab_count_grant(sg->offset, sg->length);

736 737 738 739
	require_extra_req = info->max_indirect_segments == 0 &&
		num_grant > BLKIF_MAX_SEGMENTS_PER_REQUEST;
	BUG_ON(!HAS_EXTRA_REQ && require_extra_req);

740
	rinfo->shadow[id].num_sg = num_sg;
741 742
	if (num_grant > BLKIF_MAX_SEGMENTS_PER_REQUEST &&
	    likely(!require_extra_req)) {
743 744 745 746
		/*
		 * The indirect operation can only be a BLKIF_OP_READ or
		 * BLKIF_OP_WRITE
		 */
747
		BUG_ON(req_op(req) == REQ_OP_FLUSH || req->cmd_flags & REQ_FUA);
748 749 750 751 752
		ring_req->operation = BLKIF_OP_INDIRECT;
		ring_req->u.indirect.indirect_op = rq_data_dir(req) ?
			BLKIF_OP_WRITE : BLKIF_OP_READ;
		ring_req->u.indirect.sector_number = (blkif_sector_t)blk_rq_pos(req);
		ring_req->u.indirect.handle = info->handle;
753
		ring_req->u.indirect.nr_segments = num_grant;
754
	} else {
755 756 757 758
		ring_req->u.rw.sector_number = (blkif_sector_t)blk_rq_pos(req);
		ring_req->u.rw.handle = info->handle;
		ring_req->operation = rq_data_dir(req) ?
			BLKIF_OP_WRITE : BLKIF_OP_READ;
759
		if (req_op(req) == REQ_OP_FLUSH || req->cmd_flags & REQ_FUA) {
760
			/*
761 762 763 764 765
			 * Ideally we can do an unordered flush-to-disk.
			 * In case the backend onlysupports barriers, use that.
			 * A barrier request a superset of FUA, so we can
			 * implement it the same way.  (It's also a FLUSH+FUA,
			 * since it is guaranteed ordered WRT previous writes.)
766
			 */
767
			if (info->feature_flush && info->feature_fua)
768 769
				ring_req->operation =
					BLKIF_OP_WRITE_BARRIER;
770
			else if (info->feature_flush)
771 772
				ring_req->operation =
					BLKIF_OP_FLUSH_DISKCACHE;
773
			else
774
				ring_req->operation = 0;
775
		}
776
		ring_req->u.rw.nr_segments = num_grant;
777 778 779 780 781 782 783 784 785 786 787 788 789 790 791
		if (unlikely(require_extra_req)) {
			extra_id = blkif_ring_get_request(rinfo, req,
							  &extra_ring_req);
			/*
			 * Only the first request contains the scatter-gather
			 * list.
			 */
			rinfo->shadow[extra_id].num_sg = 0;

			blkif_setup_extra_req(ring_req, extra_ring_req);

			/* Link the 2 requests together */
			rinfo->shadow[extra_id].associated_id = id;
			rinfo->shadow[id].associated_id = extra_id;
		}
792
	}
793

794 795
	setup.ring_req = ring_req;
	setup.id = id;
796 797 798 799 800

	setup.require_extra_req = require_extra_req;
	if (unlikely(require_extra_req))
		setup.extra_ring_req = extra_ring_req;

801
	for_each_sg(rinfo->shadow[id].sg, sg, num_sg, i) {
802
		BUG_ON(sg->offset + sg->length > PAGE_SIZE);
803

804 805 806 807
		if (setup.need_copy) {
			setup.bvec_off = sg->offset;
			setup.bvec_data = kmap_atomic(sg_page(sg));
		}
808

809 810 811 812 813
		gnttab_foreach_grant_in_range(sg_page(sg),
					      sg->offset,
					      sg->length,
					      blkif_setup_rw_req_grant,
					      &setup);
814

815 816
		if (setup.need_copy)
			kunmap_atomic(setup.bvec_data);
817
	}
818 819
	if (setup.segments)
		kunmap_atomic(setup.segments);
820 821

	/* Keep a private copy so we can reissue requests when recovering. */
822
	rinfo->shadow[id].req = *ring_req;
823 824
	if (unlikely(require_extra_req))
		rinfo->shadow[extra_id].req = *extra_ring_req;
825

826
	if (max_grefs > 0)
827
		gnttab_free_grant_references(setup.gref_head);
828 829 830 831

	return 0;
}

832 833 834 835 836 837
/*
 * Generate a Xen blkfront IO request from a blk layer request.  Reads
 * and writes are handled as expected.
 *
 * @req: a request struct
 */
838
static int blkif_queue_request(struct request *req, struct blkfront_ring_info *rinfo)
839
{
840
	if (unlikely(rinfo->dev_info->connected != BLKIF_STATE_CONNECTED))
841 842
		return 1;

M
Mike Christie 已提交
843
	if (unlikely(req_op(req) == REQ_OP_DISCARD ||
844
		     req_op(req) == REQ_OP_SECURE_ERASE))
845
		return blkif_queue_discard_req(req, rinfo);
846
	else
847
		return blkif_queue_rw_req(req, rinfo);
848
}
849

850
static inline void flush_requests(struct blkfront_ring_info *rinfo)
851 852 853
{
	int notify;

854
	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&rinfo->ring, notify);
855 856

	if (notify)
857
		notify_remote_via_irq(rinfo->irq);
858 859
}

860 861
static inline bool blkif_request_flush_invalid(struct request *req,
					       struct blkfront_info *info)
862 863
{
	return ((req->cmd_type != REQ_TYPE_FS) ||
864
		((req_op(req) == REQ_OP_FLUSH) &&
865
		 !info->feature_flush) ||
866
		((req->cmd_flags & REQ_FUA) &&
867
		 !info->feature_fua));
868 869
}

B
Bob Liu 已提交
870
static int blkif_queue_rq(struct blk_mq_hw_ctx *hctx,
871
			  const struct blk_mq_queue_data *qd)
872
{
B
Bob Liu 已提交
873
	unsigned long flags;
874
	struct blkfront_ring_info *rinfo = (struct blkfront_ring_info *)hctx->driver_data;
875

B
Bob Liu 已提交
876
	blk_mq_start_request(qd->rq);
B
Bob Liu 已提交
877
	spin_lock_irqsave(&rinfo->ring_lock, flags);
878
	if (RING_FULL(&rinfo->ring))
B
Bob Liu 已提交
879
		goto out_busy;
880

881
	if (blkif_request_flush_invalid(qd->rq, rinfo->dev_info))
B
Bob Liu 已提交
882
		goto out_err;
883

884
	if (blkif_queue_request(qd->rq, rinfo))
B
Bob Liu 已提交
885
		goto out_busy;
886

887
	flush_requests(rinfo);
B
Bob Liu 已提交
888
	spin_unlock_irqrestore(&rinfo->ring_lock, flags);
B
Bob Liu 已提交
889
	return BLK_MQ_RQ_QUEUE_OK;
890

B
Bob Liu 已提交
891
out_err:
B
Bob Liu 已提交
892
	spin_unlock_irqrestore(&rinfo->ring_lock, flags);
B
Bob Liu 已提交
893
	return BLK_MQ_RQ_QUEUE_ERROR;
894

B
Bob Liu 已提交
895
out_busy:
B
Bob Liu 已提交
896
	spin_unlock_irqrestore(&rinfo->ring_lock, flags);
B
Bob Liu 已提交
897 898
	blk_mq_stop_hw_queue(hctx);
	return BLK_MQ_RQ_QUEUE_BUSY;
899 900
}

901 902 903 904 905
static int blk_mq_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
			    unsigned int index)
{
	struct blkfront_info *info = (struct blkfront_info *)data;

906 907
	BUG_ON(info->nr_rings <= index);
	hctx->driver_data = &info->rinfo[index];
908 909 910
	return 0;
}

B
Bob Liu 已提交
911 912 913
static struct blk_mq_ops blkfront_mq_ops = {
	.queue_rq = blkif_queue_rq,
	.map_queue = blk_mq_map_queue,
914
	.init_hctx = blk_mq_init_hctx,
B
Bob Liu 已提交
915 916
};

917
static int xlvbd_init_blk_queue(struct gendisk *gd, u16 sector_size,
918
				unsigned int physical_sector_size,
919
				unsigned int segments)
920
{
921
	struct request_queue *rq;
922
	struct blkfront_info *info = gd->private_data;
923

B
Bob Liu 已提交
924 925
	memset(&info->tag_set, 0, sizeof(info->tag_set));
	info->tag_set.ops = &blkfront_mq_ops;
926
	info->tag_set.nr_hw_queues = info->nr_rings;
927 928 929 930 931 932 933 934 935 936
	if (HAS_EXTRA_REQ && info->max_indirect_segments == 0) {
		/*
		 * When indirect descriptior is not supported, the I/O request
		 * will be split between multiple request in the ring.
		 * To avoid problems when sending the request, divide by
		 * 2 the depth of the queue.
		 */
		info->tag_set.queue_depth =  BLK_RING_SIZE(info) / 2;
	} else
		info->tag_set.queue_depth = BLK_RING_SIZE(info);
B
Bob Liu 已提交
937 938 939 940 941 942
	info->tag_set.numa_node = NUMA_NO_NODE;
	info->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE;
	info->tag_set.cmd_size = 0;
	info->tag_set.driver_data = info;

	if (blk_mq_alloc_tag_set(&info->tag_set))
943
		return -EINVAL;
B
Bob Liu 已提交
944 945 946
	rq = blk_mq_init_queue(&info->tag_set);
	if (IS_ERR(rq)) {
		blk_mq_free_tag_set(&info->tag_set);
947
		return PTR_ERR(rq);
B
Bob Liu 已提交
948
	}
949

950
	queue_flag_set_unlocked(QUEUE_FLAG_VIRT, rq);
951

952 953 954 955 956
	if (info->feature_discard) {
		queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, rq);
		blk_queue_max_discard_sectors(rq, get_capacity(gd));
		rq->limits.discard_granularity = info->discard_granularity;
		rq->limits.discard_alignment = info->discard_alignment;
957
		if (info->feature_secdiscard)
958
			queue_flag_set_unlocked(QUEUE_FLAG_SECERASE, rq);
959 960
	}

961
	/* Hard sector size and max sectors impersonate the equiv. hardware. */
962
	blk_queue_logical_block_size(rq, sector_size);
963
	blk_queue_physical_block_size(rq, physical_sector_size);
964
	blk_queue_max_hw_sectors(rq, (segments * XEN_PAGE_SIZE) / 512);
965 966 967 968 969 970

	/* Each segment in a request is up to an aligned page in size. */
	blk_queue_segment_boundary(rq, PAGE_SIZE - 1);
	blk_queue_max_segment_size(rq, PAGE_SIZE);

	/* Ensure a merged request will fit in a single I/O ring slot. */
971
	blk_queue_max_segments(rq, segments / GRANTS_PER_PSEG);
972 973 974 975

	/* Make sure buffer addresses are sector-aligned. */
	blk_queue_dma_alignment(rq, 511);

976 977 978
	/* Make sure we don't use bounce buffers. */
	blk_queue_bounce_limit(rq, BLK_BOUNCE_ANY);

979 980 981 982 983
	gd->queue = rq;

	return 0;
}

984
static const char *flush_info(struct blkfront_info *info)
985
{
986
	if (info->feature_flush && info->feature_fua)
987
		return "barrier: enabled;";
988
	else if (info->feature_flush)
989
		return "flush diskcache: enabled;";
990
	else
991 992
		return "barrier or flush: disabled;";
}
993

994
static void xlvbd_flush(struct blkfront_info *info)
995
{
996 997
	blk_queue_write_cache(info->rq, info->feature_flush ? true : false,
			      info->feature_fua ? true : false);
998
	pr_info("blkfront: %s: %s %s %s %s %s\n",
999
		info->gd->disk_name, flush_info(info),
1000 1001 1002
		"persistent grants:", info->feature_persistent ?
		"enabled;" : "disabled;", "indirect descriptors:",
		info->max_indirect_segments ? "enabled;" : "disabled;");
1003 1004
}

1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063
static int xen_translate_vdev(int vdevice, int *minor, unsigned int *offset)
{
	int major;
	major = BLKIF_MAJOR(vdevice);
	*minor = BLKIF_MINOR(vdevice);
	switch (major) {
		case XEN_IDE0_MAJOR:
			*offset = (*minor / 64) + EMULATED_HD_DISK_NAME_OFFSET;
			*minor = ((*minor / 64) * PARTS_PER_DISK) +
				EMULATED_HD_DISK_MINOR_OFFSET;
			break;
		case XEN_IDE1_MAJOR:
			*offset = (*minor / 64) + 2 + EMULATED_HD_DISK_NAME_OFFSET;
			*minor = (((*minor / 64) + 2) * PARTS_PER_DISK) +
				EMULATED_HD_DISK_MINOR_OFFSET;
			break;
		case XEN_SCSI_DISK0_MAJOR:
			*offset = (*minor / PARTS_PER_DISK) + EMULATED_SD_DISK_NAME_OFFSET;
			*minor = *minor + EMULATED_SD_DISK_MINOR_OFFSET;
			break;
		case XEN_SCSI_DISK1_MAJOR:
		case XEN_SCSI_DISK2_MAJOR:
		case XEN_SCSI_DISK3_MAJOR:
		case XEN_SCSI_DISK4_MAJOR:
		case XEN_SCSI_DISK5_MAJOR:
		case XEN_SCSI_DISK6_MAJOR:
		case XEN_SCSI_DISK7_MAJOR:
			*offset = (*minor / PARTS_PER_DISK) + 
				((major - XEN_SCSI_DISK1_MAJOR + 1) * 16) +
				EMULATED_SD_DISK_NAME_OFFSET;
			*minor = *minor +
				((major - XEN_SCSI_DISK1_MAJOR + 1) * 16 * PARTS_PER_DISK) +
				EMULATED_SD_DISK_MINOR_OFFSET;
			break;
		case XEN_SCSI_DISK8_MAJOR:
		case XEN_SCSI_DISK9_MAJOR:
		case XEN_SCSI_DISK10_MAJOR:
		case XEN_SCSI_DISK11_MAJOR:
		case XEN_SCSI_DISK12_MAJOR:
		case XEN_SCSI_DISK13_MAJOR:
		case XEN_SCSI_DISK14_MAJOR:
		case XEN_SCSI_DISK15_MAJOR:
			*offset = (*minor / PARTS_PER_DISK) + 
				((major - XEN_SCSI_DISK8_MAJOR + 8) * 16) +
				EMULATED_SD_DISK_NAME_OFFSET;
			*minor = *minor +
				((major - XEN_SCSI_DISK8_MAJOR + 8) * 16 * PARTS_PER_DISK) +
				EMULATED_SD_DISK_MINOR_OFFSET;
			break;
		case XENVBD_MAJOR:
			*offset = *minor / PARTS_PER_DISK;
			break;
		default:
			printk(KERN_WARNING "blkfront: your disk configuration is "
					"incorrect, please use an xvd device instead\n");
			return -ENODEV;
	}
	return 0;
}
1064

1065 1066 1067 1068 1069 1070 1071 1072
static char *encode_disk_name(char *ptr, unsigned int n)
{
	if (n >= 26)
		ptr = encode_disk_name(ptr, n / 26 - 1);
	*ptr = 'a' + n % 26;
	return ptr + 1;
}

1073 1074
static int xlvbd_alloc_gendisk(blkif_sector_t capacity,
			       struct blkfront_info *info,
1075 1076
			       u16 vdisk_info, u16 sector_size,
			       unsigned int physical_sector_size)
1077 1078 1079
{
	struct gendisk *gd;
	int nr_minors = 1;
1080
	int err;
1081 1082 1083
	unsigned int offset;
	int minor;
	int nr_parts;
1084
	char *ptr;
1085 1086 1087 1088

	BUG_ON(info->gd != NULL);
	BUG_ON(info->rq != NULL);

1089 1090 1091 1092 1093 1094 1095
	if ((info->vdevice>>EXT_SHIFT) > 1) {
		/* this is above the extended range; something is wrong */
		printk(KERN_WARNING "blkfront: vdevice 0x%x is above the extended range; ignoring\n", info->vdevice);
		return -ENODEV;
	}

	if (!VDEV_IS_EXTENDED(info->vdevice)) {
1096 1097 1098 1099
		err = xen_translate_vdev(info->vdevice, &minor, &offset);
		if (err)
			return err;		
 		nr_parts = PARTS_PER_DISK;
1100 1101 1102
	} else {
		minor = BLKIF_MINOR_EXT(info->vdevice);
		nr_parts = PARTS_PER_EXT_DISK;
1103
		offset = minor / nr_parts;
1104
		if (xen_hvm_domain() && offset < EMULATED_HD_DISK_NAME_OFFSET + 4)
1105 1106 1107
			printk(KERN_WARNING "blkfront: vdevice 0x%x might conflict with "
					"emulated IDE disks,\n\t choose an xvd device name"
					"from xvde on\n", info->vdevice);
1108
	}
1109 1110 1111 1112 1113
	if (minor >> MINORBITS) {
		pr_warn("blkfront: %#x's minor (%#x) out of range; ignoring\n",
			info->vdevice, minor);
		return -ENODEV;
	}
1114 1115 1116

	if ((minor % nr_parts) == 0)
		nr_minors = nr_parts;
1117

1118 1119 1120 1121 1122
	err = xlbd_reserve_minors(minor, nr_minors);
	if (err)
		goto out;
	err = -ENODEV;

1123 1124
	gd = alloc_disk(nr_minors);
	if (gd == NULL)
1125
		goto release;
1126

1127 1128 1129 1130 1131 1132 1133 1134
	strcpy(gd->disk_name, DEV_NAME);
	ptr = encode_disk_name(gd->disk_name + sizeof(DEV_NAME) - 1, offset);
	BUG_ON(ptr >= gd->disk_name + DISK_NAME_LEN);
	if (nr_minors > 1)
		*ptr = 0;
	else
		snprintf(ptr, gd->disk_name + DISK_NAME_LEN - ptr,
			 "%d", minor & (nr_parts - 1));
1135 1136 1137 1138 1139 1140 1141 1142

	gd->major = XENVBD_MAJOR;
	gd->first_minor = minor;
	gd->fops = &xlvbd_block_fops;
	gd->private_data = info;
	gd->driverfs_dev = &(info->xbdev->dev);
	set_capacity(gd, capacity);

1143
	if (xlvbd_init_blk_queue(gd, sector_size, physical_sector_size,
1144 1145
				 info->max_indirect_segments ? :
				 BLKIF_MAX_SEGMENTS_PER_REQUEST)) {
1146
		del_gendisk(gd);
1147
		goto release;
1148 1149 1150 1151 1152
	}

	info->rq = gd->queue;
	info->gd = gd;

1153
	xlvbd_flush(info);
1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165

	if (vdisk_info & VDISK_READONLY)
		set_disk_ro(gd, 1);

	if (vdisk_info & VDISK_REMOVABLE)
		gd->flags |= GENHD_FL_REMOVABLE;

	if (vdisk_info & VDISK_CDROM)
		gd->flags |= GENHD_FL_CD;

	return 0;

1166 1167
 release:
	xlbd_release_minors(minor, nr_minors);
1168 1169 1170 1171
 out:
	return err;
}

D
Daniel Stodden 已提交
1172 1173
static void xlvbd_release_gendisk(struct blkfront_info *info)
{
1174
	unsigned int minor, nr_minors, i;
D
Daniel Stodden 已提交
1175 1176 1177 1178 1179

	if (info->rq == NULL)
		return;

	/* No more blkif_request(). */
B
Bob Liu 已提交
1180
	blk_mq_stop_hw_queues(info->rq);
D
Daniel Stodden 已提交
1181

1182 1183
	for (i = 0; i < info->nr_rings; i++) {
		struct blkfront_ring_info *rinfo = &info->rinfo[i];
D
Daniel Stodden 已提交
1184

1185 1186 1187 1188 1189 1190
		/* No more gnttab callback work. */
		gnttab_cancel_free_callback(&rinfo->callback);

		/* Flush gnttab callback work. Must be done with no locks held. */
		flush_work(&rinfo->work);
	}
D
Daniel Stodden 已提交
1191 1192 1193 1194 1195 1196 1197 1198

	del_gendisk(info->gd);

	minor = info->gd->first_minor;
	nr_minors = info->gd->minors;
	xlbd_release_minors(minor, nr_minors);

	blk_cleanup_queue(info->rq);
B
Bob Liu 已提交
1199
	blk_mq_free_tag_set(&info->tag_set);
D
Daniel Stodden 已提交
1200 1201 1202 1203 1204 1205
	info->rq = NULL;

	put_disk(info->gd);
	info->gd = NULL;
}

B
Bob Liu 已提交
1206 1207
/* Already hold rinfo->ring_lock. */
static inline void kick_pending_request_queues_locked(struct blkfront_ring_info *rinfo)
1208
{
1209 1210
	if (!RING_FULL(&rinfo->ring))
		blk_mq_start_stopped_hw_queues(rinfo->dev_info->rq, true);
1211 1212
}

B
Bob Liu 已提交
1213 1214 1215 1216 1217 1218 1219 1220 1221
static void kick_pending_request_queues(struct blkfront_ring_info *rinfo)
{
	unsigned long flags;

	spin_lock_irqsave(&rinfo->ring_lock, flags);
	kick_pending_request_queues_locked(rinfo);
	spin_unlock_irqrestore(&rinfo->ring_lock, flags);
}

1222 1223
static void blkif_restart_queue(struct work_struct *work)
{
1224
	struct blkfront_ring_info *rinfo = container_of(work, struct blkfront_ring_info, work);
1225

1226 1227
	if (rinfo->dev_info->connected == BLKIF_STATE_CONNECTED)
		kick_pending_request_queues(rinfo);
1228 1229
}

1230
static void blkif_free_ring(struct blkfront_ring_info *rinfo)
1231
{
1232
	struct grant *persistent_gnt, *n;
1233
	struct blkfront_info *info = rinfo->dev_info;
1234
	int i, j, segs;
1235

1236 1237 1238 1239
	/*
	 * Remove indirect pages, this only happens when using indirect
	 * descriptors but not persistent grants
	 */
1240
	if (!list_empty(&rinfo->indirect_pages)) {
1241 1242 1243
		struct page *indirect_page, *n;

		BUG_ON(info->feature_persistent);
1244
		list_for_each_entry_safe(indirect_page, n, &rinfo->indirect_pages, lru) {
1245 1246 1247 1248 1249
			list_del(&indirect_page->lru);
			__free_page(indirect_page);
		}
	}

1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266
	/* Remove all persistent grants. */
	if (!list_empty(&rinfo->grants)) {
		list_for_each_entry_safe(persistent_gnt, n,
					 &rinfo->grants, node) {
			list_del(&persistent_gnt->node);
			if (persistent_gnt->gref != GRANT_INVALID_REF) {
				gnttab_end_foreign_access(persistent_gnt->gref,
							  0, 0UL);
				rinfo->persistent_gnts_c--;
			}
			if (info->feature_persistent)
				__free_page(persistent_gnt->page);
			kfree(persistent_gnt);
		}
	}
	BUG_ON(rinfo->persistent_gnts_c != 0);

B
Bob Liu 已提交
1267
	for (i = 0; i < BLK_RING_SIZE(info); i++) {
1268 1269 1270 1271
		/*
		 * Clear persistent grants present in requests already
		 * on the shared ring
		 */
1272
		if (!rinfo->shadow[i].request)
1273 1274
			goto free_shadow;

1275 1276 1277
		segs = rinfo->shadow[i].req.operation == BLKIF_OP_INDIRECT ?
		       rinfo->shadow[i].req.u.indirect.nr_segments :
		       rinfo->shadow[i].req.u.rw.nr_segments;
1278
		for (j = 0; j < segs; j++) {
1279
			persistent_gnt = rinfo->shadow[i].grants_used[j];
1280
			gnttab_end_foreign_access(persistent_gnt->gref, 0, 0UL);
1281
			if (info->feature_persistent)
1282
				__free_page(persistent_gnt->page);
1283 1284 1285
			kfree(persistent_gnt);
		}

1286
		if (rinfo->shadow[i].req.operation != BLKIF_OP_INDIRECT)
1287 1288 1289 1290 1291 1292 1293
			/*
			 * If this is not an indirect operation don't try to
			 * free indirect segments
			 */
			goto free_shadow;

		for (j = 0; j < INDIRECT_GREFS(segs); j++) {
1294
			persistent_gnt = rinfo->shadow[i].indirect_grants[j];
1295
			gnttab_end_foreign_access(persistent_gnt->gref, 0, 0UL);
1296
			__free_page(persistent_gnt->page);
1297 1298 1299 1300
			kfree(persistent_gnt);
		}

free_shadow:
1301 1302 1303 1304 1305 1306
		kfree(rinfo->shadow[i].grants_used);
		rinfo->shadow[i].grants_used = NULL;
		kfree(rinfo->shadow[i].indirect_grants);
		rinfo->shadow[i].indirect_grants = NULL;
		kfree(rinfo->shadow[i].sg);
		rinfo->shadow[i].sg = NULL;
1307 1308
	}

1309
	/* No more gnttab callback work. */
1310
	gnttab_cancel_free_callback(&rinfo->callback);
1311 1312

	/* Flush gnttab callback work. Must be done with no locks held. */
1313
	flush_work(&rinfo->work);
1314 1315

	/* Free resources associated with old device channel. */
B
Bob Liu 已提交
1316
	for (i = 0; i < info->nr_ring_pages; i++) {
1317 1318 1319
		if (rinfo->ring_ref[i] != GRANT_INVALID_REF) {
			gnttab_end_foreign_access(rinfo->ring_ref[i], 0, 0);
			rinfo->ring_ref[i] = GRANT_INVALID_REF;
B
Bob Liu 已提交
1320
		}
1321
	}
1322 1323
	free_pages((unsigned long)rinfo->ring.sring, get_order(info->nr_ring_pages * PAGE_SIZE));
	rinfo->ring.sring = NULL;
B
Bob Liu 已提交
1324

1325 1326 1327
	if (rinfo->irq)
		unbind_from_irqhandler(rinfo->irq, rinfo);
	rinfo->evtchn = rinfo->irq = 0;
1328
}
1329

1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346
static void blkif_free(struct blkfront_info *info, int suspend)
{
	unsigned int i;

	/* Prevent new requests being issued until we fix things up. */
	info->connected = suspend ?
		BLKIF_STATE_SUSPENDED : BLKIF_STATE_DISCONNECTED;
	/* No more blkif_request(). */
	if (info->rq)
		blk_mq_stop_hw_queues(info->rq);

	for (i = 0; i < info->nr_rings; i++)
		blkif_free_ring(&info->rinfo[i]);

	kfree(info->rinfo);
	info->rinfo = NULL;
	info->nr_rings = 0;
1347 1348
}

1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374
struct copy_from_grant {
	const struct blk_shadow *s;
	unsigned int grant_idx;
	unsigned int bvec_offset;
	char *bvec_data;
};

static void blkif_copy_from_grant(unsigned long gfn, unsigned int offset,
				  unsigned int len, void *data)
{
	struct copy_from_grant *info = data;
	char *shared_data;
	/* Convenient aliases */
	const struct blk_shadow *s = info->s;

	shared_data = kmap_atomic(s->grants_used[info->grant_idx]->page);

	memcpy(info->bvec_data + info->bvec_offset,
	       shared_data + offset, len);

	info->bvec_offset += len;
	info->grant_idx++;

	kunmap_atomic(shared_data);
}

1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407
static enum blk_req_status blkif_rsp_to_req_status(int rsp)
{
	switch (rsp)
	{
	case BLKIF_RSP_OKAY:
		return REQ_DONE;
	case BLKIF_RSP_EOPNOTSUPP:
		return REQ_EOPNOTSUPP;
	case BLKIF_RSP_ERROR:
		/* Fallthrough. */
	default:
		return REQ_ERROR;
	}
}

/*
 * Get the final status of the block request based on two ring response
 */
static int blkif_get_final_status(enum blk_req_status s1,
				  enum blk_req_status s2)
{
	BUG_ON(s1 == REQ_WAITING);
	BUG_ON(s2 == REQ_WAITING);

	if (s1 == REQ_ERROR || s2 == REQ_ERROR)
		return BLKIF_RSP_ERROR;
	else if (s1 == REQ_EOPNOTSUPP || s2 == REQ_EOPNOTSUPP)
		return BLKIF_RSP_EOPNOTSUPP;
	return BLKIF_RSP_OKAY;
}

static bool blkif_completion(unsigned long *id,
			     struct blkfront_ring_info *rinfo,
1408
			     struct blkif_response *bret)
1409
{
1410
	int i = 0;
1411
	struct scatterlist *sg;
1412
	int num_sg, num_grant;
1413
	struct blkfront_info *info = rinfo->dev_info;
1414
	struct blk_shadow *s = &rinfo->shadow[*id];
1415 1416 1417
	struct copy_from_grant data = {
		.grant_idx = 0,
	};
1418

1419
	num_grant = s->req.operation == BLKIF_OP_INDIRECT ?
1420
		s->req.u.indirect.nr_segments : s->req.u.rw.nr_segments;
1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461

	/* The I/O request may be split in two. */
	if (unlikely(s->associated_id != NO_ASSOCIATED_ID)) {
		struct blk_shadow *s2 = &rinfo->shadow[s->associated_id];

		/* Keep the status of the current response in shadow. */
		s->status = blkif_rsp_to_req_status(bret->status);

		/* Wait the second response if not yet here. */
		if (s2->status == REQ_WAITING)
			return 0;

		bret->status = blkif_get_final_status(s->status,
						      s2->status);

		/*
		 * All the grants is stored in the first shadow in order
		 * to make the completion code simpler.
		 */
		num_grant += s2->req.u.rw.nr_segments;

		/*
		 * The two responses may not come in order. Only the
		 * first request will store the scatter-gather list.
		 */
		if (s2->num_sg != 0) {
			/* Update "id" with the ID of the first response. */
			*id = s->associated_id;
			s = s2;
		}

		/*
		 * We don't need anymore the second request, so recycling
		 * it now.
		 */
		if (add_id_to_freelist(rinfo, s->associated_id))
			WARN(1, "%s: can't recycle the second part (id = %ld) of the request\n",
			     info->gd->disk_name, s->associated_id);
	}

	data.s = s;
1462
	num_sg = s->num_sg;
1463

1464
	if (bret->operation == BLKIF_OP_READ && info->feature_persistent) {
1465
		for_each_sg(s->sg, sg, num_sg, i) {
1466
			BUG_ON(sg->offset + sg->length > PAGE_SIZE);
1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477

			data.bvec_offset = sg->offset;
			data.bvec_data = kmap_atomic(sg_page(sg));

			gnttab_foreach_grant_in_range(sg_page(sg),
						      sg->offset,
						      sg->length,
						      blkif_copy_from_grant,
						      &data);

			kunmap_atomic(data.bvec_data);
1478 1479 1480
		}
	}
	/* Add the persistent grant into the list of free grants */
1481
	for (i = 0; i < num_grant; i++) {
1482 1483 1484 1485 1486 1487 1488
		if (gnttab_query_foreign_access(s->grants_used[i]->gref)) {
			/*
			 * If the grant is still mapped by the backend (the
			 * backend has chosen to make this grant persistent)
			 * we add it at the head of the list, so it will be
			 * reused first.
			 */
1489 1490 1491
			if (!info->feature_persistent)
				pr_alert_ratelimited("backed has not unmapped grant: %u\n",
						     s->grants_used[i]->gref);
1492 1493
			list_add(&s->grants_used[i]->node, &rinfo->grants);
			rinfo->persistent_gnts_c++;
1494 1495 1496 1497 1498 1499 1500 1501 1502
		} else {
			/*
			 * If the grant is not mapped by the backend we end the
			 * foreign access and add it to the tail of the list,
			 * so it will not be picked again unless we run out of
			 * persistent grants.
			 */
			gnttab_end_foreign_access(s->grants_used[i]->gref, 0, 0UL);
			s->grants_used[i]->gref = GRANT_INVALID_REF;
1503
			list_add_tail(&s->grants_used[i]->node, &rinfo->grants);
1504
		}
1505
	}
1506
	if (s->req.operation == BLKIF_OP_INDIRECT) {
1507
		for (i = 0; i < INDIRECT_GREFS(num_grant); i++) {
1508
			if (gnttab_query_foreign_access(s->indirect_grants[i]->gref)) {
1509 1510 1511
				if (!info->feature_persistent)
					pr_alert_ratelimited("backed has not unmapped grant: %u\n",
							     s->indirect_grants[i]->gref);
1512 1513
				list_add(&s->indirect_grants[i]->node, &rinfo->grants);
				rinfo->persistent_gnts_c++;
1514
			} else {
1515 1516
				struct page *indirect_page;

1517
				gnttab_end_foreign_access(s->indirect_grants[i]->gref, 0, 0UL);
1518 1519 1520 1521
				/*
				 * Add the used indirect page back to the list of
				 * available pages for indirect grefs.
				 */
1522
				if (!info->feature_persistent) {
1523
					indirect_page = s->indirect_grants[i]->page;
1524
					list_add(&indirect_page->lru, &rinfo->indirect_pages);
1525
				}
1526
				s->indirect_grants[i]->gref = GRANT_INVALID_REF;
1527
				list_add_tail(&s->indirect_grants[i]->node, &rinfo->grants);
1528
			}
1529 1530
		}
	}
1531 1532

	return 1;
1533 1534 1535 1536 1537 1538 1539 1540
}

static irqreturn_t blkif_interrupt(int irq, void *dev_id)
{
	struct request *req;
	struct blkif_response *bret;
	RING_IDX i, rp;
	unsigned long flags;
1541 1542
	struct blkfront_ring_info *rinfo = (struct blkfront_ring_info *)dev_id;
	struct blkfront_info *info = rinfo->dev_info;
1543
	int error;
1544

B
Bob Liu 已提交
1545
	if (unlikely(info->connected != BLKIF_STATE_CONNECTED))
1546 1547
		return IRQ_HANDLED;

B
Bob Liu 已提交
1548
	spin_lock_irqsave(&rinfo->ring_lock, flags);
1549
 again:
1550
	rp = rinfo->ring.sring->rsp_prod;
1551 1552
	rmb(); /* Ensure we see queued responses up to 'rp'. */

1553
	for (i = rinfo->ring.rsp_cons; i != rp; i++) {
1554 1555
		unsigned long id;

1556
		bret = RING_GET_RESPONSE(&rinfo->ring, i);
1557
		id   = bret->id;
1558 1559 1560 1561 1562
		/*
		 * The backend has messed up and given us an id that we would
		 * never have given to it (we stamp it up to BLK_RING_SIZE -
		 * look in get_id_from_freelist.
		 */
B
Bob Liu 已提交
1563
		if (id >= BLK_RING_SIZE(info)) {
1564 1565 1566 1567 1568 1569
			WARN(1, "%s: response to %s has incorrect id (%ld)\n",
			     info->gd->disk_name, op_name(bret->operation), id);
			/* We can't safely get the 'struct request' as
			 * the id is busted. */
			continue;
		}
1570
		req  = rinfo->shadow[id].request;
1571

1572 1573 1574 1575 1576 1577 1578 1579
		if (bret->operation != BLKIF_OP_DISCARD) {
			/*
			 * We may need to wait for an extra response if the
			 * I/O request is split in 2
			 */
			if (!blkif_completion(&id, rinfo, bret))
				continue;
		}
1580

1581
		if (add_id_to_freelist(rinfo, id)) {
1582 1583 1584 1585
			WARN(1, "%s: response to %s (id %ld) couldn't be recycled!\n",
			     info->gd->disk_name, op_name(bret->operation), id);
			continue;
		}
1586

1587
		error = (bret->status == BLKIF_RSP_OKAY) ? 0 : -EIO;
1588
		switch (bret->operation) {
1589 1590 1591
		case BLKIF_OP_DISCARD:
			if (unlikely(bret->status == BLKIF_RSP_EOPNOTSUPP)) {
				struct request_queue *rq = info->rq;
1592 1593
				printk(KERN_WARNING "blkfront: %s: %s op failed\n",
					   info->gd->disk_name, op_name(bret->operation));
1594
				error = -EOPNOTSUPP;
1595
				info->feature_discard = 0;
1596
				info->feature_secdiscard = 0;
1597
				queue_flag_clear(QUEUE_FLAG_DISCARD, rq);
1598
				queue_flag_clear(QUEUE_FLAG_SECERASE, rq);
1599
			}
1600
			blk_mq_complete_request(req, error);
1601
			break;
1602
		case BLKIF_OP_FLUSH_DISKCACHE:
1603 1604
		case BLKIF_OP_WRITE_BARRIER:
			if (unlikely(bret->status == BLKIF_RSP_EOPNOTSUPP)) {
1605 1606
				printk(KERN_WARNING "blkfront: %s: %s op failed\n",
				       info->gd->disk_name, op_name(bret->operation));
1607
				error = -EOPNOTSUPP;
1608 1609
			}
			if (unlikely(bret->status == BLKIF_RSP_ERROR &&
1610
				     rinfo->shadow[id].req.u.rw.nr_segments == 0)) {
1611 1612
				printk(KERN_WARNING "blkfront: %s: empty %s op failed\n",
				       info->gd->disk_name, op_name(bret->operation));
1613
				error = -EOPNOTSUPP;
1614
			}
1615 1616 1617
			if (unlikely(error)) {
				if (error == -EOPNOTSUPP)
					error = 0;
1618
				info->feature_fua = 0;
1619 1620
				info->feature_flush = 0;
				xlvbd_flush(info);
1621 1622 1623 1624 1625 1626 1627 1628
			}
			/* fall through */
		case BLKIF_OP_READ:
		case BLKIF_OP_WRITE:
			if (unlikely(bret->status != BLKIF_RSP_OKAY))
				dev_dbg(&info->xbdev->dev, "Bad return from blkdev data "
					"request: %x\n", bret->status);

1629
			blk_mq_complete_request(req, error);
1630 1631 1632 1633 1634 1635
			break;
		default:
			BUG();
		}
	}

1636
	rinfo->ring.rsp_cons = i;
1637

1638
	if (i != rinfo->ring.req_prod_pvt) {
1639
		int more_to_do;
1640
		RING_FINAL_CHECK_FOR_RESPONSES(&rinfo->ring, more_to_do);
1641 1642 1643
		if (more_to_do)
			goto again;
	} else
1644
		rinfo->ring.sring->rsp_event = i + 1;
1645

B
Bob Liu 已提交
1646
	kick_pending_request_queues_locked(rinfo);
1647

B
Bob Liu 已提交
1648
	spin_unlock_irqrestore(&rinfo->ring_lock, flags);
1649 1650 1651 1652 1653 1654

	return IRQ_HANDLED;
}


static int setup_blkring(struct xenbus_device *dev,
1655
			 struct blkfront_ring_info *rinfo)
1656 1657
{
	struct blkif_sring *sring;
B
Bob Liu 已提交
1658
	int err, i;
1659
	struct blkfront_info *info = rinfo->dev_info;
1660
	unsigned long ring_size = info->nr_ring_pages * XEN_PAGE_SIZE;
1661
	grant_ref_t gref[XENBUS_MAX_RING_GRANTS];
1662

B
Bob Liu 已提交
1663
	for (i = 0; i < info->nr_ring_pages; i++)
1664
		rinfo->ring_ref[i] = GRANT_INVALID_REF;
1665

B
Bob Liu 已提交
1666 1667
	sring = (struct blkif_sring *)__get_free_pages(GFP_NOIO | __GFP_HIGH,
						       get_order(ring_size));
1668 1669 1670 1671 1672
	if (!sring) {
		xenbus_dev_fatal(dev, -ENOMEM, "allocating shared ring");
		return -ENOMEM;
	}
	SHARED_RING_INIT(sring);
1673
	FRONT_RING_INIT(&rinfo->ring, sring, ring_size);
1674

1675
	err = xenbus_grant_ring(dev, rinfo->ring.sring, info->nr_ring_pages, gref);
1676
	if (err < 0) {
B
Bob Liu 已提交
1677
		free_pages((unsigned long)sring, get_order(ring_size));
1678
		rinfo->ring.sring = NULL;
1679 1680
		goto fail;
	}
B
Bob Liu 已提交
1681
	for (i = 0; i < info->nr_ring_pages; i++)
1682
		rinfo->ring_ref[i] = gref[i];
1683

1684
	err = xenbus_alloc_evtchn(dev, &rinfo->evtchn);
1685 1686 1687
	if (err)
		goto fail;

1688 1689
	err = bind_evtchn_to_irqhandler(rinfo->evtchn, blkif_interrupt, 0,
					"blkif", rinfo);
1690 1691 1692 1693 1694
	if (err <= 0) {
		xenbus_dev_fatal(dev, err,
				 "bind_evtchn_to_irqhandler failed");
		goto fail;
	}
1695
	rinfo->irq = err;
1696 1697 1698 1699 1700 1701 1702

	return 0;
fail:
	blkif_free(info, 0);
	return err;
}

1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749
/*
 * Write out per-ring/queue nodes including ring-ref and event-channel, and each
 * ring buffer may have multi pages depending on ->nr_ring_pages.
 */
static int write_per_ring_nodes(struct xenbus_transaction xbt,
				struct blkfront_ring_info *rinfo, const char *dir)
{
	int err;
	unsigned int i;
	const char *message = NULL;
	struct blkfront_info *info = rinfo->dev_info;

	if (info->nr_ring_pages == 1) {
		err = xenbus_printf(xbt, dir, "ring-ref", "%u", rinfo->ring_ref[0]);
		if (err) {
			message = "writing ring-ref";
			goto abort_transaction;
		}
	} else {
		for (i = 0; i < info->nr_ring_pages; i++) {
			char ring_ref_name[RINGREF_NAME_LEN];

			snprintf(ring_ref_name, RINGREF_NAME_LEN, "ring-ref%u", i);
			err = xenbus_printf(xbt, dir, ring_ref_name,
					    "%u", rinfo->ring_ref[i]);
			if (err) {
				message = "writing ring-ref";
				goto abort_transaction;
			}
		}
	}

	err = xenbus_printf(xbt, dir, "event-channel", "%u", rinfo->evtchn);
	if (err) {
		message = "writing event-channel";
		goto abort_transaction;
	}

	return 0;

abort_transaction:
	xenbus_transaction_end(xbt, 1);
	if (message)
		xenbus_dev_fatal(info->xbdev, err, "%s", message);

	return err;
}
1750 1751

/* Common code used when first setting up, and when resuming. */
1752
static int talk_to_blkback(struct xenbus_device *dev,
1753 1754 1755 1756
			   struct blkfront_info *info)
{
	const char *message = NULL;
	struct xenbus_transaction xbt;
1757 1758
	int err;
	unsigned int i, max_page_order = 0;
B
Bob Liu 已提交
1759 1760 1761 1762 1763 1764 1765 1766 1767 1768
	unsigned int ring_page_order = 0;

	err = xenbus_scanf(XBT_NIL, info->xbdev->otherend,
			   "max-ring-page-order", "%u", &max_page_order);
	if (err != 1)
		info->nr_ring_pages = 1;
	else {
		ring_page_order = min(xen_blkif_max_ring_order, max_page_order);
		info->nr_ring_pages = 1 << ring_page_order;
	}
1769

1770
	for (i = 0; i < info->nr_rings; i++) {
1771 1772
		struct blkfront_ring_info *rinfo = &info->rinfo[i];

1773 1774 1775 1776 1777
		/* Create shared ring, alloc event channel. */
		err = setup_blkring(dev, rinfo);
		if (err)
			goto destroy_blkring;
	}
1778 1779 1780 1781 1782 1783 1784 1785

again:
	err = xenbus_transaction_start(&xbt);
	if (err) {
		xenbus_dev_fatal(dev, err, "starting transaction");
		goto destroy_blkring;
	}

1786 1787 1788 1789 1790 1791 1792 1793
	if (info->nr_ring_pages > 1) {
		err = xenbus_printf(xbt, dev->nodename, "ring-page-order", "%u",
				    ring_page_order);
		if (err) {
			message = "writing ring-page-order";
			goto abort_transaction;
		}
	}
1794

1795 1796 1797 1798 1799 1800 1801 1802
	/* We already got the number of queues/rings in _probe */
	if (info->nr_rings == 1) {
		err = write_per_ring_nodes(xbt, &info->rinfo[0], dev->nodename);
		if (err)
			goto destroy_blkring;
	} else {
		char *path;
		size_t pathsize;
1803

1804 1805
		err = xenbus_printf(xbt, dev->nodename, "multi-queue-num-queues", "%u",
				    info->nr_rings);
1806
		if (err) {
1807
			message = "writing multi-queue-num-queues";
1808 1809
			goto abort_transaction;
		}
1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828

		pathsize = strlen(dev->nodename) + QUEUE_NAME_LEN;
		path = kmalloc(pathsize, GFP_KERNEL);
		if (!path) {
			err = -ENOMEM;
			message = "ENOMEM while writing ring references";
			goto abort_transaction;
		}

		for (i = 0; i < info->nr_rings; i++) {
			memset(path, 0, pathsize);
			snprintf(path, pathsize, "%s/queue-%u", dev->nodename, i);
			err = write_per_ring_nodes(xbt, &info->rinfo[i], path);
			if (err) {
				kfree(path);
				goto destroy_blkring;
			}
		}
		kfree(path);
1829
	}
1830 1831 1832 1833 1834 1835
	err = xenbus_printf(xbt, dev->nodename, "protocol", "%s",
			    XEN_IO_PROTO_ABI_NATIVE);
	if (err) {
		message = "writing protocol";
		goto abort_transaction;
	}
1836
	err = xenbus_printf(xbt, dev->nodename,
1837
			    "feature-persistent", "%u", 1);
1838 1839 1840
	if (err)
		dev_warn(&dev->dev,
			 "writing persistent grants feature to xenbus");
1841 1842 1843 1844 1845 1846 1847 1848 1849

	err = xenbus_transaction_end(xbt, 0);
	if (err) {
		if (err == -EAGAIN)
			goto again;
		xenbus_dev_fatal(dev, err, "completing transaction");
		goto destroy_blkring;
	}

1850 1851
	for (i = 0; i < info->nr_rings; i++) {
		unsigned int j;
1852
		struct blkfront_ring_info *rinfo = &info->rinfo[i];
1853 1854 1855 1856 1857

		for (j = 0; j < BLK_RING_SIZE(info); j++)
			rinfo->shadow[j].req.u.rw.id = j + 1;
		rinfo->shadow[BLK_RING_SIZE(info)-1].req.u.rw.id = 0x0fffffff;
	}
1858 1859 1860 1861 1862 1863 1864 1865 1866 1867
	xenbus_switch_state(dev, XenbusStateInitialised);

	return 0;

 abort_transaction:
	xenbus_transaction_end(xbt, 1);
	if (message)
		xenbus_dev_fatal(dev, err, "%s", message);
 destroy_blkring:
	blkif_free(info, 0);
1868

1869 1870 1871
	kfree(info);
	dev_set_drvdata(&dev->dev, NULL);

1872 1873 1874
	return err;
}

1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911
static int negotiate_mq(struct blkfront_info *info)
{
	unsigned int backend_max_queues = 0;
	int err;
	unsigned int i;

	BUG_ON(info->nr_rings);

	/* Check if backend supports multiple queues. */
	err = xenbus_scanf(XBT_NIL, info->xbdev->otherend,
			   "multi-queue-max-queues", "%u", &backend_max_queues);
	if (err < 0)
		backend_max_queues = 1;

	info->nr_rings = min(backend_max_queues, xen_blkif_max_queues);
	/* We need at least one ring. */
	if (!info->nr_rings)
		info->nr_rings = 1;

	info->rinfo = kzalloc(sizeof(struct blkfront_ring_info) * info->nr_rings, GFP_KERNEL);
	if (!info->rinfo) {
		xenbus_dev_fatal(info->xbdev, -ENOMEM, "allocating ring_info structure");
		return -ENOMEM;
	}

	for (i = 0; i < info->nr_rings; i++) {
		struct blkfront_ring_info *rinfo;

		rinfo = &info->rinfo[i];
		INIT_LIST_HEAD(&rinfo->indirect_pages);
		INIT_LIST_HEAD(&rinfo->grants);
		rinfo->dev_info = info;
		INIT_WORK(&rinfo->work, blkif_restart_queue);
		spin_lock_init(&rinfo->ring_lock);
	}
	return 0;
}
1912 1913 1914 1915 1916 1917 1918 1919 1920
/**
 * Entry point to this code when a new device is created.  Allocate the basic
 * structures and the ring buffer for communication with the backend, and
 * inform the backend of the appropriate details for those.  Switch to
 * Initialised state.
 */
static int blkfront_probe(struct xenbus_device *dev,
			  const struct xenbus_device_id *id)
{
B
Bob Liu 已提交
1921
	int err, vdevice;
1922 1923 1924 1925 1926 1927
	struct blkfront_info *info;

	/* FIXME: Use dynamic device id if this is not set. */
	err = xenbus_scanf(XBT_NIL, dev->nodename,
			   "virtual-device", "%i", &vdevice);
	if (err != 1) {
1928 1929 1930 1931 1932 1933 1934
		/* go looking in the extended area instead */
		err = xenbus_scanf(XBT_NIL, dev->nodename, "virtual-device-ext",
				   "%i", &vdevice);
		if (err != 1) {
			xenbus_dev_fatal(dev, err, "reading virtual-device");
			return err;
		}
1935 1936
	}

1937 1938 1939 1940
	if (xen_hvm_domain()) {
		char *type;
		int len;
		/* no unplug has been done: do not hook devices != xen vbds */
1941
		if (xen_has_pv_and_legacy_disk_devices()) {
1942 1943 1944 1945 1946 1947 1948 1949 1950 1951
			int major;

			if (!VDEV_IS_EXTENDED(vdevice))
				major = BLKIF_MAJOR(vdevice);
			else
				major = XENVBD_MAJOR;

			if (major != XENVBD_MAJOR) {
				printk(KERN_INFO
						"%s: HVM does not support vbd %d as xen block device\n",
1952
						__func__, vdevice);
1953 1954 1955 1956 1957 1958 1959 1960 1961
				return -ENODEV;
			}
		}
		/* do not create a PV cdrom device if we are an HVM guest */
		type = xenbus_read(XBT_NIL, dev->nodename, "device-type", &len);
		if (IS_ERR(type))
			return -ENODEV;
		if (strncmp(type, "cdrom", 5) == 0) {
			kfree(type);
1962 1963
			return -ENODEV;
		}
1964
		kfree(type);
1965
	}
1966 1967 1968 1969 1970 1971
	info = kzalloc(sizeof(*info), GFP_KERNEL);
	if (!info) {
		xenbus_dev_fatal(dev, -ENOMEM, "allocating info structure");
		return -ENOMEM;
	}

1972
	info->xbdev = dev;
1973 1974
	err = negotiate_mq(info);
	if (err) {
1975
		kfree(info);
1976
		return err;
1977
	}
1978

1979
	mutex_init(&info->mutex);
1980 1981 1982 1983 1984
	info->vdevice = vdevice;
	info->connected = BLKIF_STATE_DISCONNECTED;

	/* Front end dir is a number, which is used as the id. */
	info->handle = simple_strtoul(strrchr(dev->nodename, '/')+1, NULL, 0);
1985
	dev_set_drvdata(&dev->dev, info);
1986 1987 1988 1989

	return 0;
}

1990
static void split_bio_end(struct bio *bio)
1991 1992 1993 1994 1995
{
	struct split_bio *split_bio = bio->bi_private;

	if (atomic_dec_and_test(&split_bio->pending)) {
		split_bio->bio->bi_phys_segments = 0;
1996 1997
		split_bio->bio->bi_error = bio->bi_error;
		bio_endio(split_bio->bio);
1998 1999 2000 2001
		kfree(split_bio);
	}
	bio_put(bio);
}
2002 2003 2004

static int blkif_recover(struct blkfront_info *info)
{
2005
	unsigned int i, r_index;
2006
	struct request *req, *n;
2007
	struct blk_shadow *copy;
2008 2009 2010 2011 2012 2013 2014 2015
	int rc;
	struct bio *bio, *cloned_bio;
	struct bio_list bio_list, merge_bio;
	unsigned int segs, offset;
	int pending, size;
	struct split_bio *split_bio;
	struct list_head requests;

2016
	blkfront_gather_backend_features(info);
2017 2018 2019 2020
	segs = info->max_indirect_segments ? : BLKIF_MAX_SEGMENTS_PER_REQUEST;
	blk_queue_max_segments(info->rq, segs);
	bio_list_init(&bio_list);
	INIT_LIST_HEAD(&requests);
2021

2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049
	for (r_index = 0; r_index < info->nr_rings; r_index++) {
		struct blkfront_ring_info *rinfo;

		rinfo = &info->rinfo[r_index];
		/* Stage 1: Make a safe copy of the shadow state. */
		copy = kmemdup(rinfo->shadow, sizeof(rinfo->shadow),
			       GFP_NOIO | __GFP_REPEAT | __GFP_HIGH);
		if (!copy)
			return -ENOMEM;

		/* Stage 2: Set up free list. */
		memset(&rinfo->shadow, 0, sizeof(rinfo->shadow));
		for (i = 0; i < BLK_RING_SIZE(info); i++)
			rinfo->shadow[i].req.u.rw.id = i+1;
		rinfo->shadow_free = rinfo->ring.req_prod_pvt;
		rinfo->shadow[BLK_RING_SIZE(info)-1].req.u.rw.id = 0x0fffffff;

		rc = blkfront_setup_indirect(rinfo);
		if (rc) {
			kfree(copy);
			return rc;
		}

		for (i = 0; i < BLK_RING_SIZE(info); i++) {
			/* Not in use? */
			if (!copy[i].request)
				continue;

2050
			/*
2051
			 * Get the bios in the request so we can re-queue them.
2052
			 */
2053
			if (req_op(copy[i].request) == REQ_OP_FLUSH ||
M
Mike Christie 已提交
2054
			    req_op(copy[i].request) == REQ_OP_DISCARD ||
2055 2056
			    req_op(copy[i].request) == REQ_OP_SECURE_ERASE ||
			    copy[i].request->cmd_flags & REQ_FUA) {
2057 2058 2059
				/*
				 * Flush operations don't contain bios, so
				 * we need to requeue the whole request
2060 2061 2062
				 *
				 * XXX: but this doesn't make any sense for a
				 * write with the FUA flag set..
2063 2064 2065 2066 2067 2068 2069 2070 2071
				 */
				list_add(&copy[i].request->queuelist, &requests);
				continue;
			}
			merge_bio.head = copy[i].request->bio;
			merge_bio.tail = copy[i].request->biotail;
			bio_list_merge(&bio_list, &merge_bio);
			copy[i].request->bio = NULL;
			blk_end_request_all(copy[i].request, 0);
2072
		}
2073

2074 2075
		kfree(copy);
	}
2076 2077 2078 2079 2080
	xenbus_switch_state(info->xbdev, XenbusStateConnected);

	/* Now safe for us to use the shared ring */
	info->connected = BLKIF_STATE_CONNECTED;

2081 2082 2083 2084 2085 2086 2087
	for (r_index = 0; r_index < info->nr_rings; r_index++) {
		struct blkfront_ring_info *rinfo;

		rinfo = &info->rinfo[r_index];
		/* Kick any other new requests queued since we resumed */
		kick_pending_request_queues(rinfo);
	}
2088

2089 2090 2091 2092
	list_for_each_entry_safe(req, n, &requests, queuelist) {
		/* Requeue pending requests (flush or discard) */
		list_del_init(&req->queuelist);
		BUG_ON(req->nr_phys_segments > segs);
B
Bob Liu 已提交
2093
		blk_mq_requeue_request(req);
2094
	}
B
Bob Liu 已提交
2095
	blk_mq_kick_requeue_list(info->rq);
2096

2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109
	while ((bio = bio_list_pop(&bio_list)) != NULL) {
		/* Traverse the list of pending bios and re-queue them */
		if (bio_segments(bio) > segs) {
			/*
			 * This bio has more segments than what we can
			 * handle, we have to split it.
			 */
			pending = (bio_segments(bio) + segs - 1) / segs;
			split_bio = kzalloc(sizeof(*split_bio), GFP_NOIO);
			BUG_ON(split_bio == NULL);
			atomic_set(&split_bio->pending, pending);
			split_bio->bio = bio;
			for (i = 0; i < pending; i++) {
2110 2111
				offset = (i * segs * XEN_PAGE_SIZE) >> 9;
				size = min((unsigned int)(segs * XEN_PAGE_SIZE) >> 9,
2112
					   (unsigned int)bio_sectors(bio) - offset);
2113 2114
				cloned_bio = bio_clone(bio, GFP_NOIO);
				BUG_ON(cloned_bio == NULL);
2115
				bio_trim(cloned_bio, offset, size);
2116 2117
				cloned_bio->bi_private = split_bio;
				cloned_bio->bi_end_io = split_bio_end;
2118
				submit_bio(cloned_bio);
2119 2120 2121 2122 2123 2124 2125 2126
			}
			/*
			 * Now we have to wait for all those smaller bios to
			 * end, so we can also end the "parent" bio.
			 */
			continue;
		}
		/* We don't need to split this bio */
2127
		submit_bio(bio);
2128 2129
	}

2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140
	return 0;
}

/**
 * We are reconnecting to the backend, due to a suspend/resume, or a backend
 * driver restart.  We tear down our blkif structure and recreate it, but
 * leave the device-layer structures intact so that this is transparent to the
 * rest of the kernel.
 */
static int blkfront_resume(struct xenbus_device *dev)
{
2141
	struct blkfront_info *info = dev_get_drvdata(&dev->dev);
2142
	int err = 0;
2143 2144 2145 2146 2147

	dev_dbg(&dev->dev, "blkfront_resume: %s\n", dev->nodename);

	blkif_free(info, info->connected == BLKIF_STATE_CONNECTED);

2148 2149 2150 2151
	err = negotiate_mq(info);
	if (err)
		return err;

2152
	err = talk_to_blkback(dev, info);
2153 2154 2155 2156 2157 2158

	/*
	 * We have to wait for the backend to switch to
	 * connected state, since we want to read which
	 * features it supports.
	 */
2159 2160 2161 2162

	return err;
}

2163
static void blkfront_closing(struct blkfront_info *info)
2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186
{
	struct xenbus_device *xbdev = info->xbdev;
	struct block_device *bdev = NULL;

	mutex_lock(&info->mutex);

	if (xbdev->state == XenbusStateClosing) {
		mutex_unlock(&info->mutex);
		return;
	}

	if (info->gd)
		bdev = bdget_disk(info->gd, 0);

	mutex_unlock(&info->mutex);

	if (!bdev) {
		xenbus_frontend_closed(xbdev);
		return;
	}

	mutex_lock(&bdev->bd_mutex);

2187
	if (bdev->bd_openers) {
2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198
		xenbus_dev_error(xbdev, -EBUSY,
				 "Device in use; refusing to close");
		xenbus_switch_state(xbdev, XenbusStateClosing);
	} else {
		xlvbd_release_gendisk(info);
		xenbus_frontend_closed(xbdev);
	}

	mutex_unlock(&bdev->bd_mutex);
	bdput(bdev);
}
2199

2200 2201 2202 2203 2204
static void blkfront_setup_discard(struct blkfront_info *info)
{
	int err;
	unsigned int discard_granularity;
	unsigned int discard_alignment;
2205
	unsigned int discard_secure;
2206

2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220
	info->feature_discard = 1;
	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
		"discard-granularity", "%u", &discard_granularity,
		"discard-alignment", "%u", &discard_alignment,
		NULL);
	if (!err) {
		info->discard_granularity = discard_granularity;
		info->discard_alignment = discard_alignment;
	}
	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
		    "discard-secure", "%d", &discard_secure,
		    NULL);
	if (!err)
		info->feature_secdiscard = !!discard_secure;
2221 2222
}

2223
static int blkfront_setup_indirect(struct blkfront_ring_info *rinfo)
2224
{
2225
	unsigned int psegs, grants;
2226
	int err, i;
2227
	struct blkfront_info *info = rinfo->dev_info;
2228

2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240
	if (info->max_indirect_segments == 0) {
		if (!HAS_EXTRA_REQ)
			grants = BLKIF_MAX_SEGMENTS_PER_REQUEST;
		else {
			/*
			 * When an extra req is required, the maximum
			 * grants supported is related to the size of the
			 * Linux block segment.
			 */
			grants = GRANTS_PER_PSEG;
		}
	}
2241
	else
2242 2243
		grants = info->max_indirect_segments;
	psegs = grants / GRANTS_PER_PSEG;
2244

2245
	err = fill_grant_buffer(rinfo,
2246
				(grants + INDIRECT_GREFS(grants)) * BLK_RING_SIZE(info));
2247 2248 2249
	if (err)
		goto out_of_memory;

2250 2251 2252 2253 2254 2255
	if (!info->feature_persistent && info->max_indirect_segments) {
		/*
		 * We are using indirect descriptors but not persistent
		 * grants, we need to allocate a set of pages that can be
		 * used for mapping indirect grefs
		 */
2256
		int num = INDIRECT_GREFS(grants) * BLK_RING_SIZE(info);
2257

2258
		BUG_ON(!list_empty(&rinfo->indirect_pages));
2259 2260 2261 2262
		for (i = 0; i < num; i++) {
			struct page *indirect_page = alloc_page(GFP_NOIO);
			if (!indirect_page)
				goto out_of_memory;
2263
			list_add(&indirect_page->lru, &rinfo->indirect_pages);
2264 2265 2266
		}
	}

B
Bob Liu 已提交
2267
	for (i = 0; i < BLK_RING_SIZE(info); i++) {
2268 2269
		rinfo->shadow[i].grants_used = kzalloc(
			sizeof(rinfo->shadow[i].grants_used[0]) * grants,
2270
			GFP_NOIO);
2271
		rinfo->shadow[i].sg = kzalloc(sizeof(rinfo->shadow[i].sg[0]) * psegs, GFP_NOIO);
2272
		if (info->max_indirect_segments)
2273 2274
			rinfo->shadow[i].indirect_grants = kzalloc(
				sizeof(rinfo->shadow[i].indirect_grants[0]) *
2275
				INDIRECT_GREFS(grants),
2276
				GFP_NOIO);
2277 2278
		if ((rinfo->shadow[i].grants_used == NULL) ||
			(rinfo->shadow[i].sg == NULL) ||
2279
		     (info->max_indirect_segments &&
2280
		     (rinfo->shadow[i].indirect_grants == NULL)))
2281
			goto out_of_memory;
2282
		sg_init_table(rinfo->shadow[i].sg, psegs);
2283 2284 2285 2286 2287 2288
	}


	return 0;

out_of_memory:
B
Bob Liu 已提交
2289
	for (i = 0; i < BLK_RING_SIZE(info); i++) {
2290 2291 2292 2293 2294 2295
		kfree(rinfo->shadow[i].grants_used);
		rinfo->shadow[i].grants_used = NULL;
		kfree(rinfo->shadow[i].sg);
		rinfo->shadow[i].sg = NULL;
		kfree(rinfo->shadow[i].indirect_grants);
		rinfo->shadow[i].indirect_grants = NULL;
2296
	}
2297
	if (!list_empty(&rinfo->indirect_pages)) {
2298
		struct page *indirect_page, *n;
2299
		list_for_each_entry_safe(indirect_page, n, &rinfo->indirect_pages, lru) {
2300 2301 2302 2303
			list_del(&indirect_page->lru);
			__free_page(indirect_page);
		}
	}
2304 2305 2306
	return -ENOMEM;
}

2307 2308 2309
/*
 * Gather all backend feature-*
 */
2310
static void blkfront_gather_backend_features(struct blkfront_info *info)
2311 2312 2313 2314 2315 2316
{
	int err;
	int barrier, flush, discard, persistent;
	unsigned int indirect_segments;

	info->feature_flush = 0;
2317
	info->feature_fua = 0;
2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329

	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
			"feature-barrier", "%d", &barrier,
			NULL);

	/*
	 * If there's no "feature-barrier" defined, then it means
	 * we're dealing with a very old backend which writes
	 * synchronously; nothing to do.
	 *
	 * If there are barriers, then we use flush.
	 */
2330 2331 2332 2333 2334
	if (!err && barrier) {
		info->feature_flush = 1;
		info->feature_fua = 1;
	}

2335 2336 2337 2338 2339 2340 2341 2342
	/*
	 * And if there is "feature-flush-cache" use that above
	 * barriers.
	 */
	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
			"feature-flush-cache", "%d", &flush,
			NULL);

2343 2344 2345 2346
	if (!err && flush) {
		info->feature_flush = 1;
		info->feature_fua = 0;
	}
2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372

	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
			"feature-discard", "%d", &discard,
			NULL);

	if (!err && discard)
		blkfront_setup_discard(info);

	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
			"feature-persistent", "%u", &persistent,
			NULL);
	if (err)
		info->feature_persistent = 0;
	else
		info->feature_persistent = persistent;

	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
			    "feature-max-indirect-segments", "%u", &indirect_segments,
			    NULL);
	if (err)
		info->max_indirect_segments = 0;
	else
		info->max_indirect_segments = min(indirect_segments,
						  xen_blkif_max_segments);
}

2373 2374 2375 2376 2377 2378 2379 2380
/*
 * Invoked when the backend is finally 'ready' (and has told produced
 * the details about the physical device - #sectors, size, etc).
 */
static void blkfront_connect(struct blkfront_info *info)
{
	unsigned long long sectors;
	unsigned long sector_size;
2381
	unsigned int physical_sector_size;
2382
	unsigned int binfo;
2383
	int err, i;
2384

2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397
	switch (info->connected) {
	case BLKIF_STATE_CONNECTED:
		/*
		 * Potentially, the back-end may be signalling
		 * a capacity change; update the capacity.
		 */
		err = xenbus_scanf(XBT_NIL, info->xbdev->otherend,
				   "sectors", "%Lu", &sectors);
		if (XENBUS_EXIST_ERR(err))
			return;
		printk(KERN_INFO "Setting capacity to %Lu\n",
		       sectors);
		set_capacity(info->gd, sectors);
2398
		revalidate_disk(info->gd);
2399

2400
		return;
2401
	case BLKIF_STATE_SUSPENDED:
2402 2403 2404 2405 2406 2407 2408
		/*
		 * If we are recovering from suspension, we need to wait
		 * for the backend to announce it's features before
		 * reconnecting, at least we need to know if the backend
		 * supports indirect descriptors, and how many.
		 */
		blkif_recover(info);
2409 2410
		return;

2411 2412
	default:
		break;
2413
	}
2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429

	dev_dbg(&info->xbdev->dev, "%s:%s.\n",
		__func__, info->xbdev->otherend);

	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
			    "sectors", "%llu", &sectors,
			    "info", "%u", &binfo,
			    "sector-size", "%lu", &sector_size,
			    NULL);
	if (err) {
		xenbus_dev_fatal(info->xbdev, err,
				 "reading backend fields at %s",
				 info->xbdev->otherend);
		return;
	}

2430 2431 2432 2433 2434 2435 2436 2437 2438 2439
	/*
	 * physcial-sector-size is a newer field, so old backends may not
	 * provide this. Assume physical sector size to be the same as
	 * sector_size in that case.
	 */
	err = xenbus_scanf(XBT_NIL, info->xbdev->otherend,
			   "physical-sector-size", "%u", &physical_sector_size);
	if (err != 1)
		physical_sector_size = sector_size;

2440 2441 2442 2443 2444 2445 2446 2447 2448
	blkfront_gather_backend_features(info);
	for (i = 0; i < info->nr_rings; i++) {
		err = blkfront_setup_indirect(&info->rinfo[i]);
		if (err) {
			xenbus_dev_fatal(info->xbdev, err, "setup_indirect at %s",
					 info->xbdev->otherend);
			blkif_free(info, 0);
			break;
		}
2449 2450
	}

2451 2452
	err = xlvbd_alloc_gendisk(sectors, info, binfo, sector_size,
				  physical_sector_size);
2453 2454 2455 2456 2457 2458 2459 2460 2461 2462
	if (err) {
		xenbus_dev_fatal(info->xbdev, err, "xlvbd_add at %s",
				 info->xbdev->otherend);
		return;
	}

	xenbus_switch_state(info->xbdev, XenbusStateConnected);

	/* Kick pending requests. */
	info->connected = BLKIF_STATE_CONNECTED;
2463 2464
	for (i = 0; i < info->nr_rings; i++)
		kick_pending_request_queues(&info->rinfo[i]);
2465 2466

	add_disk(info->gd);
2467 2468

	info->is_ready = 1;
2469 2470 2471 2472 2473
}

/**
 * Callback received when the backend's state changes.
 */
2474
static void blkback_changed(struct xenbus_device *dev,
2475 2476
			    enum xenbus_state backend_state)
{
2477
	struct blkfront_info *info = dev_get_drvdata(&dev->dev);
2478

2479
	dev_dbg(&dev->dev, "blkfront:blkback_changed to state %d.\n", backend_state);
2480 2481 2482

	switch (backend_state) {
	case XenbusStateInitWait:
2483 2484
		if (dev->state != XenbusStateInitialising)
			break;
2485
		if (talk_to_blkback(dev, info))
2486 2487
			break;
	case XenbusStateInitialising:
2488
	case XenbusStateInitialised:
2489 2490
	case XenbusStateReconfiguring:
	case XenbusStateReconfigured:
2491 2492 2493 2494
	case XenbusStateUnknown:
		break;

	case XenbusStateConnected:
2495 2496 2497 2498
		if (dev->state != XenbusStateInitialised) {
			if (talk_to_blkback(dev, info))
				break;
		}
2499 2500 2501
		blkfront_connect(info);
		break;

2502 2503 2504 2505
	case XenbusStateClosed:
		if (dev->state == XenbusStateClosed)
			break;
		/* Missed the backend's Closing state -- fallthrough */
2506
	case XenbusStateClosing:
2507 2508
		if (info)
			blkfront_closing(info);
2509 2510 2511 2512
		break;
	}
}

2513
static int blkfront_remove(struct xenbus_device *xbdev)
2514
{
2515 2516 2517
	struct blkfront_info *info = dev_get_drvdata(&xbdev->dev);
	struct block_device *bdev = NULL;
	struct gendisk *disk;
2518

2519
	dev_dbg(&xbdev->dev, "%s removed", xbdev->nodename);
2520 2521 2522

	blkif_free(info, 0);

2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545
	mutex_lock(&info->mutex);

	disk = info->gd;
	if (disk)
		bdev = bdget_disk(disk, 0);

	info->xbdev = NULL;
	mutex_unlock(&info->mutex);

	if (!bdev) {
		kfree(info);
		return 0;
	}

	/*
	 * The xbdev was removed before we reached the Closed
	 * state. See if it's safe to remove the disk. If the bdev
	 * isn't closed yet, we let release take care of it.
	 */

	mutex_lock(&bdev->bd_mutex);
	info = disk->private_data;

2546 2547 2548 2549
	dev_warn(disk_to_dev(disk),
		 "%s was hot-unplugged, %d stale handles\n",
		 xbdev->nodename, bdev->bd_openers);

2550
	if (info && !bdev->bd_openers) {
2551 2552
		xlvbd_release_gendisk(info);
		disk->private_data = NULL;
2553
		kfree(info);
2554 2555 2556 2557
	}

	mutex_unlock(&bdev->bd_mutex);
	bdput(bdev);
2558 2559 2560 2561

	return 0;
}

2562 2563
static int blkfront_is_ready(struct xenbus_device *dev)
{
2564
	struct blkfront_info *info = dev_get_drvdata(&dev->dev);
2565

2566
	return info->is_ready && info->xbdev;
2567 2568
}

A
Al Viro 已提交
2569
static int blkif_open(struct block_device *bdev, fmode_t mode)
2570
{
2571 2572 2573
	struct gendisk *disk = bdev->bd_disk;
	struct blkfront_info *info;
	int err = 0;
2574

2575
	mutex_lock(&blkfront_mutex);
2576

2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592
	info = disk->private_data;
	if (!info) {
		/* xbdev gone */
		err = -ERESTARTSYS;
		goto out;
	}

	mutex_lock(&info->mutex);

	if (!info->gd)
		/* xbdev is closed */
		err = -ERESTARTSYS;

	mutex_unlock(&info->mutex);

out:
2593
	mutex_unlock(&blkfront_mutex);
2594
	return err;
2595 2596
}

2597
static void blkif_release(struct gendisk *disk, fmode_t mode)
2598
{
A
Al Viro 已提交
2599
	struct blkfront_info *info = disk->private_data;
2600 2601 2602
	struct block_device *bdev;
	struct xenbus_device *xbdev;

2603
	mutex_lock(&blkfront_mutex);
2604 2605 2606

	bdev = bdget_disk(disk, 0);

2607 2608 2609 2610
	if (!bdev) {
		WARN(1, "Block device %s yanked out from us!\n", disk->disk_name);
		goto out_mutex;
	}
2611 2612 2613
	if (bdev->bd_openers)
		goto out;

2614 2615 2616 2617 2618 2619 2620 2621 2622 2623
	/*
	 * Check if we have been instructed to close. We will have
	 * deferred this request, because the bdev was still open.
	 */

	mutex_lock(&info->mutex);
	xbdev = info->xbdev;

	if (xbdev && xbdev->state == XenbusStateClosing) {
		/* pending switch to state closed */
2624
		dev_info(disk_to_dev(bdev->bd_disk), "releasing disk\n");
2625 2626 2627 2628 2629 2630 2631 2632
		xlvbd_release_gendisk(info);
		xenbus_frontend_closed(info->xbdev);
 	}

	mutex_unlock(&info->mutex);

	if (!xbdev) {
		/* sudden device removal */
2633
		dev_info(disk_to_dev(bdev->bd_disk), "releasing disk\n");
2634 2635 2636
		xlvbd_release_gendisk(info);
		disk->private_data = NULL;
		kfree(info);
2637
	}
2638

J
Jens Axboe 已提交
2639
out:
2640
	bdput(bdev);
2641
out_mutex:
2642
	mutex_unlock(&blkfront_mutex);
2643 2644
}

2645
static const struct block_device_operations xlvbd_block_fops =
2646 2647
{
	.owner = THIS_MODULE,
A
Al Viro 已提交
2648 2649
	.open = blkif_open,
	.release = blkif_release,
2650
	.getgeo = blkif_getgeo,
2651
	.ioctl = blkif_ioctl,
2652 2653 2654
};


2655
static const struct xenbus_device_id blkfront_ids[] = {
2656 2657 2658 2659
	{ "vbd" },
	{ "" }
};

2660 2661
static struct xenbus_driver blkfront_driver = {
	.ids  = blkfront_ids,
2662 2663 2664
	.probe = blkfront_probe,
	.remove = blkfront_remove,
	.resume = blkfront_resume,
2665
	.otherend_changed = blkback_changed,
2666
	.is_ready = blkfront_is_ready,
2667
};
2668 2669 2670

static int __init xlblk_init(void)
{
2671
	int ret;
2672
	int nr_cpus = num_online_cpus();
2673

2674
	if (!xen_domain())
2675 2676
		return -ENODEV;

2677
	if (xen_blkif_max_ring_order > XENBUS_MAX_RING_GRANT_ORDER) {
B
Bob Liu 已提交
2678
		pr_info("Invalid max_ring_order (%d), will use default max: %d.\n",
2679
			xen_blkif_max_ring_order, XENBUS_MAX_RING_GRANT_ORDER);
2680
		xen_blkif_max_ring_order = XENBUS_MAX_RING_GRANT_ORDER;
B
Bob Liu 已提交
2681 2682
	}

2683 2684 2685 2686 2687 2688
	if (xen_blkif_max_queues > nr_cpus) {
		pr_info("Invalid max_queues (%d), will use default max: %d.\n",
			xen_blkif_max_queues, nr_cpus);
		xen_blkif_max_queues = nr_cpus;
	}

2689
	if (!xen_has_pv_disk_devices())
2690 2691
		return -ENODEV;

2692 2693 2694 2695 2696 2697
	if (register_blkdev(XENVBD_MAJOR, DEV_NAME)) {
		printk(KERN_WARNING "xen_blk: can't get major %d with name %s\n",
		       XENVBD_MAJOR, DEV_NAME);
		return -ENODEV;
	}

2698
	ret = xenbus_register_frontend(&blkfront_driver);
2699 2700 2701 2702 2703 2704
	if (ret) {
		unregister_blkdev(XENVBD_MAJOR, DEV_NAME);
		return ret;
	}

	return 0;
2705 2706 2707 2708
}
module_init(xlblk_init);


2709
static void __exit xlblk_exit(void)
2710
{
2711 2712 2713
	xenbus_unregister_driver(&blkfront_driver);
	unregister_blkdev(XENVBD_MAJOR, DEV_NAME);
	kfree(minors);
2714 2715 2716 2717 2718 2719
}
module_exit(xlblk_exit);

MODULE_DESCRIPTION("Xen virtual block device frontend");
MODULE_LICENSE("GPL");
MODULE_ALIAS_BLOCKDEV_MAJOR(XENVBD_MAJOR);
2720
MODULE_ALIAS("xen:vbd");
2721
MODULE_ALIAS("xenblk");