virtio_scsi.c 19.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
/*
 * Virtio SCSI HBA driver
 *
 * Copyright IBM Corp. 2010
 * Copyright Red Hat, Inc. 2011
 *
 * Authors:
 *  Stefan Hajnoczi   <stefanha@linux.vnet.ibm.com>
 *  Paolo Bonzini   <pbonzini@redhat.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2 or later.
 * See the COPYING file in the top-level directory.
 *
 */

16 17
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

18 19 20 21 22 23 24 25 26 27 28 29
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/mempool.h>
#include <linux/virtio.h>
#include <linux/virtio_ids.h>
#include <linux/virtio_config.h>
#include <linux/virtio_scsi.h>
#include <scsi/scsi_host.h>
#include <scsi/scsi_device.h>
#include <scsi/scsi_cmnd.h>

#define VIRTIO_SCSI_MEMPOOL_SZ 64
30
#define VIRTIO_SCSI_EVENT_LEN 8
31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48

/* Command queue element */
struct virtio_scsi_cmd {
	struct scsi_cmnd *sc;
	struct completion *comp;
	union {
		struct virtio_scsi_cmd_req       cmd;
		struct virtio_scsi_ctrl_tmf_req  tmf;
		struct virtio_scsi_ctrl_an_req   an;
	} req;
	union {
		struct virtio_scsi_cmd_resp      cmd;
		struct virtio_scsi_ctrl_tmf_resp tmf;
		struct virtio_scsi_ctrl_an_resp  an;
		struct virtio_scsi_event         evt;
	} resp;
} ____cacheline_aligned_in_smp;

49 50 51 52 53 54
struct virtio_scsi_event_node {
	struct virtio_scsi *vscsi;
	struct virtio_scsi_event event;
	struct work_struct work;
};

55 56 57 58 59 60 61
struct virtio_scsi_vq {
	/* Protects vq */
	spinlock_t vq_lock;

	struct virtqueue *vq;
};

62 63
/* Per-target queue state */
struct virtio_scsi_target_state {
64
	/* Never held at the same time as vq_lock.  */
65 66 67
	spinlock_t tgt_lock;
};

68 69 70
/* Driver instance state */
struct virtio_scsi {
	struct virtio_device *vdev;
71

72 73 74
	struct virtio_scsi_vq ctrl_vq;
	struct virtio_scsi_vq event_vq;
	struct virtio_scsi_vq req_vq;
75

76 77 78
	/* Get some buffers ready for event vq */
	struct virtio_scsi_event_node event_list[VIRTIO_SCSI_EVENT_LEN];

79
	struct virtio_scsi_target_state *tgt[];
80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183
};

static struct kmem_cache *virtscsi_cmd_cache;
static mempool_t *virtscsi_cmd_pool;

static inline struct Scsi_Host *virtio_scsi_host(struct virtio_device *vdev)
{
	return vdev->priv;
}

static void virtscsi_compute_resid(struct scsi_cmnd *sc, u32 resid)
{
	if (!resid)
		return;

	if (!scsi_bidi_cmnd(sc)) {
		scsi_set_resid(sc, resid);
		return;
	}

	scsi_in(sc)->resid = min(resid, scsi_in(sc)->length);
	scsi_out(sc)->resid = resid - scsi_in(sc)->resid;
}

/**
 * virtscsi_complete_cmd - finish a scsi_cmd and invoke scsi_done
 *
 * Called with vq_lock held.
 */
static void virtscsi_complete_cmd(void *buf)
{
	struct virtio_scsi_cmd *cmd = buf;
	struct scsi_cmnd *sc = cmd->sc;
	struct virtio_scsi_cmd_resp *resp = &cmd->resp.cmd;

	dev_dbg(&sc->device->sdev_gendev,
		"cmd %p response %u status %#02x sense_len %u\n",
		sc, resp->response, resp->status, resp->sense_len);

	sc->result = resp->status;
	virtscsi_compute_resid(sc, resp->resid);
	switch (resp->response) {
	case VIRTIO_SCSI_S_OK:
		set_host_byte(sc, DID_OK);
		break;
	case VIRTIO_SCSI_S_OVERRUN:
		set_host_byte(sc, DID_ERROR);
		break;
	case VIRTIO_SCSI_S_ABORTED:
		set_host_byte(sc, DID_ABORT);
		break;
	case VIRTIO_SCSI_S_BAD_TARGET:
		set_host_byte(sc, DID_BAD_TARGET);
		break;
	case VIRTIO_SCSI_S_RESET:
		set_host_byte(sc, DID_RESET);
		break;
	case VIRTIO_SCSI_S_BUSY:
		set_host_byte(sc, DID_BUS_BUSY);
		break;
	case VIRTIO_SCSI_S_TRANSPORT_FAILURE:
		set_host_byte(sc, DID_TRANSPORT_DISRUPTED);
		break;
	case VIRTIO_SCSI_S_TARGET_FAILURE:
		set_host_byte(sc, DID_TARGET_FAILURE);
		break;
	case VIRTIO_SCSI_S_NEXUS_FAILURE:
		set_host_byte(sc, DID_NEXUS_FAILURE);
		break;
	default:
		scmd_printk(KERN_WARNING, sc, "Unknown response %d",
			    resp->response);
		/* fall through */
	case VIRTIO_SCSI_S_FAILURE:
		set_host_byte(sc, DID_ERROR);
		break;
	}

	WARN_ON(resp->sense_len > VIRTIO_SCSI_SENSE_SIZE);
	if (sc->sense_buffer) {
		memcpy(sc->sense_buffer, resp->sense,
		       min_t(u32, resp->sense_len, VIRTIO_SCSI_SENSE_SIZE));
		if (resp->sense_len)
			set_driver_byte(sc, DRIVER_SENSE);
	}

	mempool_free(cmd, virtscsi_cmd_pool);
	sc->scsi_done(sc);
}

static void virtscsi_vq_done(struct virtqueue *vq, void (*fn)(void *buf))
{
	void *buf;
	unsigned int len;

	do {
		virtqueue_disable_cb(vq);
		while ((buf = virtqueue_get_buf(vq, &len)) != NULL)
			fn(buf);
	} while (!virtqueue_enable_cb(vq));
}

static void virtscsi_req_done(struct virtqueue *vq)
{
184 185 186 187 188
	struct Scsi_Host *sh = virtio_scsi_host(vq->vdev);
	struct virtio_scsi *vscsi = shost_priv(sh);
	unsigned long flags;

	spin_lock_irqsave(&vscsi->req_vq.vq_lock, flags);
189
	virtscsi_vq_done(vq, virtscsi_complete_cmd);
190
	spin_unlock_irqrestore(&vscsi->req_vq.vq_lock, flags);
191 192 193 194 195 196 197 198
};

static void virtscsi_complete_free(void *buf)
{
	struct virtio_scsi_cmd *cmd = buf;

	if (cmd->comp)
		complete_all(cmd->comp);
199 200
	else
		mempool_free(cmd, virtscsi_cmd_pool);
201 202 203 204
}

static void virtscsi_ctrl_done(struct virtqueue *vq)
{
205 206 207 208 209
	struct Scsi_Host *sh = virtio_scsi_host(vq->vdev);
	struct virtio_scsi *vscsi = shost_priv(sh);
	unsigned long flags;

	spin_lock_irqsave(&vscsi->ctrl_vq.vq_lock, flags);
210
	virtscsi_vq_done(vq, virtscsi_complete_free);
211
	spin_unlock_irqrestore(&vscsi->ctrl_vq.vq_lock, flags);
212 213
};

214 215 216
static int virtscsi_kick_event(struct virtio_scsi *vscsi,
			       struct virtio_scsi_event_node *event_node)
{
217
	int err;
218 219 220
	struct scatterlist sg;
	unsigned long flags;

221
	sg_init_one(&sg, &event_node->event, sizeof(struct virtio_scsi_event));
222 223 224

	spin_lock_irqsave(&vscsi->event_vq.vq_lock, flags);

225 226
	err = virtqueue_add_inbuf(vscsi->event_vq.vq, &sg, 1, event_node,
				  GFP_ATOMIC);
227
	if (!err)
228 229 230 231
		virtqueue_kick(vscsi->event_vq.vq);

	spin_unlock_irqrestore(&vscsi->event_vq.vq_lock, flags);

232
	return err;
233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281
}

static int virtscsi_kick_event_all(struct virtio_scsi *vscsi)
{
	int i;

	for (i = 0; i < VIRTIO_SCSI_EVENT_LEN; i++) {
		vscsi->event_list[i].vscsi = vscsi;
		virtscsi_kick_event(vscsi, &vscsi->event_list[i]);
	}

	return 0;
}

static void virtscsi_cancel_event_work(struct virtio_scsi *vscsi)
{
	int i;

	for (i = 0; i < VIRTIO_SCSI_EVENT_LEN; i++)
		cancel_work_sync(&vscsi->event_list[i].work);
}

static void virtscsi_handle_transport_reset(struct virtio_scsi *vscsi,
						struct virtio_scsi_event *event)
{
	struct scsi_device *sdev;
	struct Scsi_Host *shost = virtio_scsi_host(vscsi->vdev);
	unsigned int target = event->lun[1];
	unsigned int lun = (event->lun[2] << 8) | event->lun[3];

	switch (event->reason) {
	case VIRTIO_SCSI_EVT_RESET_RESCAN:
		scsi_add_device(shost, 0, target, lun);
		break;
	case VIRTIO_SCSI_EVT_RESET_REMOVED:
		sdev = scsi_device_lookup(shost, 0, target, lun);
		if (sdev) {
			scsi_remove_device(sdev);
			scsi_device_put(sdev);
		} else {
			pr_err("SCSI device %d 0 %d %d not found\n",
				shost->host_no, target, lun);
		}
		break;
	default:
		pr_info("Unsupport virtio scsi event reason %x\n", event->reason);
	}
}

282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306
static void virtscsi_handle_param_change(struct virtio_scsi *vscsi,
					 struct virtio_scsi_event *event)
{
	struct scsi_device *sdev;
	struct Scsi_Host *shost = virtio_scsi_host(vscsi->vdev);
	unsigned int target = event->lun[1];
	unsigned int lun = (event->lun[2] << 8) | event->lun[3];
	u8 asc = event->reason & 255;
	u8 ascq = event->reason >> 8;

	sdev = scsi_device_lookup(shost, 0, target, lun);
	if (!sdev) {
		pr_err("SCSI device %d 0 %d %d not found\n",
			shost->host_no, target, lun);
		return;
	}

	/* Handle "Parameters changed", "Mode parameters changed", and
	   "Capacity data has changed".  */
	if (asc == 0x2a && (ascq == 0x00 || ascq == 0x01 || ascq == 0x09))
		scsi_rescan_device(&sdev->sdev_gendev);

	scsi_device_put(sdev);
}

307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324
static void virtscsi_handle_event(struct work_struct *work)
{
	struct virtio_scsi_event_node *event_node =
		container_of(work, struct virtio_scsi_event_node, work);
	struct virtio_scsi *vscsi = event_node->vscsi;
	struct virtio_scsi_event *event = &event_node->event;

	if (event->event & VIRTIO_SCSI_T_EVENTS_MISSED) {
		event->event &= ~VIRTIO_SCSI_T_EVENTS_MISSED;
		scsi_scan_host(virtio_scsi_host(vscsi->vdev));
	}

	switch (event->event) {
	case VIRTIO_SCSI_T_NO_EVENT:
		break;
	case VIRTIO_SCSI_T_TRANSPORT_RESET:
		virtscsi_handle_transport_reset(vscsi, event);
		break;
325 326 327
	case VIRTIO_SCSI_T_PARAM_CHANGE:
		virtscsi_handle_param_change(vscsi, event);
		break;
328 329 330 331 332 333 334 335 336 337 338 339 340 341
	default:
		pr_err("Unsupport virtio scsi event %x\n", event->event);
	}
	virtscsi_kick_event(vscsi, event_node);
}

static void virtscsi_complete_event(void *buf)
{
	struct virtio_scsi_event_node *event_node = buf;

	INIT_WORK(&event_node->work, virtscsi_handle_event);
	schedule_work(&event_node->work);
}

342 343
static void virtscsi_event_done(struct virtqueue *vq)
{
344 345 346 347 348
	struct Scsi_Host *sh = virtio_scsi_host(vq->vdev);
	struct virtio_scsi *vscsi = shost_priv(sh);
	unsigned long flags;

	spin_lock_irqsave(&vscsi->event_vq.vq_lock, flags);
349
	virtscsi_vq_done(vq, virtscsi_complete_event);
350
	spin_unlock_irqrestore(&vscsi->event_vq.vq_lock, flags);
351 352 353
};

/**
354 355
 * virtscsi_add_cmd - add a virtio_scsi_cmd to a virtqueue
 * @vq		: the struct virtqueue we're talking about
356 357 358
 * @cmd		: command structure
 * @req_size	: size of the request buffer
 * @resp_size	: size of the response buffer
359
 * @gfp	: flags to use for memory allocations
360
 */
361 362 363
static int virtscsi_add_cmd(struct virtqueue *vq,
			    struct virtio_scsi_cmd *cmd,
			    size_t req_size, size_t resp_size, gfp_t gfp)
364 365
{
	struct scsi_cmnd *sc = cmd->sc;
366 367 368 369 370 371 372 373 374 375 376 377
	struct scatterlist *sgs[4], req, resp;
	struct sg_table *out, *in;
	unsigned out_num = 0, in_num = 0;

	out = in = NULL;

	if (sc && sc->sc_data_direction != DMA_NONE) {
		if (sc->sc_data_direction != DMA_FROM_DEVICE)
			out = &scsi_out(sc)->table;
		if (sc->sc_data_direction != DMA_TO_DEVICE)
			in = &scsi_in(sc)->table;
	}
378 379

	/* Request header.  */
380 381
	sg_init_one(&req, &cmd->req, req_size);
	sgs[out_num++] = &req;
382 383

	/* Data-out buffer.  */
384 385
	if (out)
		sgs[out_num++] = out->sgl;
386 387

	/* Response header.  */
388 389
	sg_init_one(&resp, &cmd->resp, resp_size);
	sgs[out_num + in_num++] = &resp;
390 391

	/* Data-in buffer */
392 393
	if (in)
		sgs[out_num + in_num++] = in->sgl;
394

395
	return virtqueue_add_sgs(vq, sgs, out_num, in_num, cmd, gfp);
396 397
}

398
static int virtscsi_kick_cmd(struct virtio_scsi_vq *vq,
399 400 401 402
			     struct virtio_scsi_cmd *cmd,
			     size_t req_size, size_t resp_size, gfp_t gfp)
{
	unsigned long flags;
403 404
	int err;
	bool needs_kick = false;
405

406 407
	spin_lock_irqsave(&vq->vq_lock, flags);
	err = virtscsi_add_cmd(vq->vq, cmd, req_size, resp_size, gfp);
408 409
	if (!err)
		needs_kick = virtqueue_kick_prepare(vq->vq);
410

411
	spin_unlock_irqrestore(&vq->vq_lock, flags);
412

413
	if (needs_kick)
414
		virtqueue_notify(vq->vq);
415
	return err;
416 417 418 419 420 421 422 423
}

static int virtscsi_queuecommand(struct Scsi_Host *sh, struct scsi_cmnd *sc)
{
	struct virtio_scsi *vscsi = shost_priv(sh);
	struct virtio_scsi_cmd *cmd;
	int ret;

424 425 426 427 428 429
	struct Scsi_Host *shost = virtio_scsi_host(vscsi->vdev);
	BUG_ON(scsi_sg_count(sc) > shost->sg_tablesize);

	/* TODO: check feature bit and fail if unsupported?  */
	BUG_ON(sc->sc_data_direction == DMA_BIDIRECTIONAL);

430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453
	dev_dbg(&sc->device->sdev_gendev,
		"cmd %p CDB: %#02x\n", sc, sc->cmnd[0]);

	ret = SCSI_MLQUEUE_HOST_BUSY;
	cmd = mempool_alloc(virtscsi_cmd_pool, GFP_ATOMIC);
	if (!cmd)
		goto out;

	memset(cmd, 0, sizeof(*cmd));
	cmd->sc = sc;
	cmd->req.cmd = (struct virtio_scsi_cmd_req){
		.lun[0] = 1,
		.lun[1] = sc->device->id,
		.lun[2] = (sc->device->lun >> 8) | 0x40,
		.lun[3] = sc->device->lun & 0xff,
		.tag = (unsigned long)sc,
		.task_attr = VIRTIO_SCSI_S_SIMPLE,
		.prio = 0,
		.crn = 0,
	};

	BUG_ON(sc->cmd_len > VIRTIO_SCSI_CDB_SIZE);
	memcpy(cmd->req.cmd.cdb, sc->cmnd, sc->cmd_len);

454
	if (virtscsi_kick_cmd(&vscsi->req_vq, cmd,
455
			      sizeof cmd->req.cmd, sizeof cmd->resp.cmd,
456
			      GFP_ATOMIC) == 0)
457
		ret = 0;
458 459
	else
		mempool_free(cmd, virtscsi_cmd_pool);
460 461 462 463 464 465 466 467

out:
	return ret;
}

static int virtscsi_tmf(struct virtio_scsi *vscsi, struct virtio_scsi_cmd *cmd)
{
	DECLARE_COMPLETION_ONSTACK(comp);
468
	int ret = FAILED;
469 470

	cmd->comp = &comp;
471
	if (virtscsi_kick_cmd(&vscsi->ctrl_vq, cmd,
472 473 474
			      sizeof cmd->req.tmf, sizeof cmd->resp.tmf,
			      GFP_NOIO) < 0)
		goto out;
475 476

	wait_for_completion(&comp);
477 478 479
	if (cmd->resp.tmf.response == VIRTIO_SCSI_S_OK ||
	    cmd->resp.tmf.response == VIRTIO_SCSI_S_FUNCTION_SUCCEEDED)
		ret = SUCCESS;
480

481 482 483
out:
	mempool_free(cmd, virtscsi_cmd_pool);
	return ret;
484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563
}

static int virtscsi_device_reset(struct scsi_cmnd *sc)
{
	struct virtio_scsi *vscsi = shost_priv(sc->device->host);
	struct virtio_scsi_cmd *cmd;

	sdev_printk(KERN_INFO, sc->device, "device reset\n");
	cmd = mempool_alloc(virtscsi_cmd_pool, GFP_NOIO);
	if (!cmd)
		return FAILED;

	memset(cmd, 0, sizeof(*cmd));
	cmd->sc = sc;
	cmd->req.tmf = (struct virtio_scsi_ctrl_tmf_req){
		.type = VIRTIO_SCSI_T_TMF,
		.subtype = VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET,
		.lun[0] = 1,
		.lun[1] = sc->device->id,
		.lun[2] = (sc->device->lun >> 8) | 0x40,
		.lun[3] = sc->device->lun & 0xff,
	};
	return virtscsi_tmf(vscsi, cmd);
}

static int virtscsi_abort(struct scsi_cmnd *sc)
{
	struct virtio_scsi *vscsi = shost_priv(sc->device->host);
	struct virtio_scsi_cmd *cmd;

	scmd_printk(KERN_INFO, sc, "abort\n");
	cmd = mempool_alloc(virtscsi_cmd_pool, GFP_NOIO);
	if (!cmd)
		return FAILED;

	memset(cmd, 0, sizeof(*cmd));
	cmd->sc = sc;
	cmd->req.tmf = (struct virtio_scsi_ctrl_tmf_req){
		.type = VIRTIO_SCSI_T_TMF,
		.subtype = VIRTIO_SCSI_T_TMF_ABORT_TASK,
		.lun[0] = 1,
		.lun[1] = sc->device->id,
		.lun[2] = (sc->device->lun >> 8) | 0x40,
		.lun[3] = sc->device->lun & 0xff,
		.tag = (unsigned long)sc,
	};
	return virtscsi_tmf(vscsi, cmd);
}

static struct scsi_host_template virtscsi_host_template = {
	.module = THIS_MODULE,
	.name = "Virtio SCSI HBA",
	.proc_name = "virtio_scsi",
	.queuecommand = virtscsi_queuecommand,
	.this_id = -1,
	.eh_abort_handler = virtscsi_abort,
	.eh_device_reset_handler = virtscsi_device_reset,

	.can_queue = 1024,
	.dma_boundary = UINT_MAX,
	.use_clustering = ENABLE_CLUSTERING,
};

#define virtscsi_config_get(vdev, fld) \
	({ \
		typeof(((struct virtio_scsi_config *)0)->fld) __val; \
		vdev->config->get(vdev, \
				  offsetof(struct virtio_scsi_config, fld), \
				  &__val, sizeof(__val)); \
		__val; \
	})

#define virtscsi_config_set(vdev, fld, val) \
	(void)({ \
		typeof(((struct virtio_scsi_config *)0)->fld) __val = (val); \
		vdev->config->set(vdev, \
				  offsetof(struct virtio_scsi_config, fld), \
				  &__val, sizeof(__val)); \
	})

564 565 566 567 568 569 570
static void virtscsi_init_vq(struct virtio_scsi_vq *virtscsi_vq,
			     struct virtqueue *vq)
{
	spin_lock_init(&virtscsi_vq->vq_lock);
	virtscsi_vq->vq = vq;
}

571
static struct virtio_scsi_target_state *virtscsi_alloc_tgt(
572
	struct virtio_device *vdev)
573 574 575 576
{
	struct virtio_scsi_target_state *tgt;
	gfp_t gfp_mask = GFP_KERNEL;

577
	tgt = kmalloc(sizeof(*tgt), gfp_mask);
578 579 580 581 582 583 584
	if (!tgt)
		return NULL;

	spin_lock_init(&tgt->tgt_lock);
	return tgt;
}

585 586 587 588 589 590 591
static void virtscsi_scan(struct virtio_device *vdev)
{
	struct Scsi_Host *shost = (struct Scsi_Host *)vdev->priv;

	scsi_scan_host(shost);
}

592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609
static void virtscsi_remove_vqs(struct virtio_device *vdev)
{
	struct Scsi_Host *sh = virtio_scsi_host(vdev);
	struct virtio_scsi *vscsi = shost_priv(sh);
	u32 i, num_targets;

	/* Stop all the virtqueues. */
	vdev->config->reset(vdev);

	num_targets = sh->max_id;
	for (i = 0; i < num_targets; i++) {
		kfree(vscsi->tgt[i]);
		vscsi->tgt[i] = NULL;
	}

	vdev->config->del_vqs(vdev);
}

610
static int virtscsi_init(struct virtio_device *vdev,
611
			 struct virtio_scsi *vscsi, int num_targets)
612 613 614
{
	int err;
	struct virtqueue *vqs[3];
615
	u32 i;
616

617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632
	vq_callback_t *callbacks[] = {
		virtscsi_ctrl_done,
		virtscsi_event_done,
		virtscsi_req_done
	};
	const char *names[] = {
		"control",
		"event",
		"request"
	};

	/* Discover virtqueues and write information to configuration.  */
	err = vdev->config->find_vqs(vdev, 3, vqs, callbacks, names);
	if (err)
		return err;

633 634 635
	virtscsi_init_vq(&vscsi->ctrl_vq, vqs[0]);
	virtscsi_init_vq(&vscsi->event_vq, vqs[1]);
	virtscsi_init_vq(&vscsi->req_vq, vqs[2]);
636 637 638

	virtscsi_config_set(vdev, cdb_size, VIRTIO_SCSI_CDB_SIZE);
	virtscsi_config_set(vdev, sense_size, VIRTIO_SCSI_SENSE_SIZE);
639

640 641 642
	if (virtio_has_feature(vdev, VIRTIO_SCSI_F_HOTPLUG))
		virtscsi_kick_event_all(vscsi);

643
	for (i = 0; i < num_targets; i++) {
644
		vscsi->tgt[i] = virtscsi_alloc_tgt(vdev);
645 646 647 648 649 650 651 652 653 654 655
		if (!vscsi->tgt[i]) {
			err = -ENOMEM;
			goto out;
		}
	}
	err = 0;

out:
	if (err)
		virtscsi_remove_vqs(vdev);
	return err;
656 657
}

658
static int virtscsi_probe(struct virtio_device *vdev)
659 660 661 662
{
	struct Scsi_Host *shost;
	struct virtio_scsi *vscsi;
	int err;
663
	u32 sg_elems, num_targets;
664 665 666
	u32 cmd_per_lun;

	/* Allocate memory and link the structs together.  */
667
	num_targets = virtscsi_config_get(vdev, max_target) + 1;
668
	shost = scsi_host_alloc(&virtscsi_host_template,
669 670
		sizeof(*vscsi)
		+ num_targets * sizeof(struct virtio_scsi_target_state));
671 672 673 674

	if (!shost)
		return -ENOMEM;

675
	sg_elems = virtscsi_config_get(vdev, seg_max) ?: 1;
676 677 678 679 680
	shost->sg_tablesize = sg_elems;
	vscsi = shost_priv(shost);
	vscsi->vdev = vdev;
	vdev->priv = shost;

681
	err = virtscsi_init(vdev, vscsi, num_targets);
682 683 684 685 686 687
	if (err)
		goto virtscsi_init_failed;

	cmd_per_lun = virtscsi_config_get(vdev, cmd_per_lun) ?: 1;
	shost->cmd_per_lun = min_t(u32, cmd_per_lun, shost->can_queue);
	shost->max_sectors = virtscsi_config_get(vdev, max_sectors) ?: 0xFFFF;
688 689 690 691 692

	/* LUNs > 256 are reported with format 1, so they go in the range
	 * 16640-32767.
	 */
	shost->max_lun = virtscsi_config_get(vdev, max_lun) + 1 + 0x4000;
693
	shost->max_id = num_targets;
694 695 696 697 698
	shost->max_channel = 0;
	shost->max_cmd_len = VIRTIO_SCSI_CDB_SIZE;
	err = scsi_add_host(shost, &vdev->dev);
	if (err)
		goto scsi_add_host_failed;
699 700 701 702
	/*
	 * scsi_scan_host() happens in virtscsi_scan() via virtio_driver->scan()
	 * after VIRTIO_CONFIG_S_DRIVER_OK has been set..
	 */
703 704 705 706 707 708 709 710 711
	return 0;

scsi_add_host_failed:
	vdev->config->del_vqs(vdev);
virtscsi_init_failed:
	scsi_host_put(shost);
	return err;
}

712
static void virtscsi_remove(struct virtio_device *vdev)
713 714
{
	struct Scsi_Host *shost = virtio_scsi_host(vdev);
715 716 717 718
	struct virtio_scsi *vscsi = shost_priv(shost);

	if (virtio_has_feature(vdev, VIRTIO_SCSI_F_HOTPLUG))
		virtscsi_cancel_event_work(vscsi);
719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737

	scsi_remove_host(shost);

	virtscsi_remove_vqs(vdev);
	scsi_host_put(shost);
}

#ifdef CONFIG_PM
static int virtscsi_freeze(struct virtio_device *vdev)
{
	virtscsi_remove_vqs(vdev);
	return 0;
}

static int virtscsi_restore(struct virtio_device *vdev)
{
	struct Scsi_Host *sh = virtio_scsi_host(vdev);
	struct virtio_scsi *vscsi = shost_priv(sh);

738
	return virtscsi_init(vdev, vscsi, sh->max_id);
739 740 741 742 743 744 745 746
}
#endif

static struct virtio_device_id id_table[] = {
	{ VIRTIO_ID_SCSI, VIRTIO_DEV_ANY_ID },
	{ 0 },
};

747
static unsigned int features[] = {
748 749
	VIRTIO_SCSI_F_HOTPLUG,
	VIRTIO_SCSI_F_CHANGE,
750 751
};

752
static struct virtio_driver virtio_scsi_driver = {
753 754
	.feature_table = features,
	.feature_table_size = ARRAY_SIZE(features),
755 756 757 758
	.driver.name = KBUILD_MODNAME,
	.driver.owner = THIS_MODULE,
	.id_table = id_table,
	.probe = virtscsi_probe,
759
	.scan = virtscsi_scan,
760 761 762 763
#ifdef CONFIG_PM
	.freeze = virtscsi_freeze,
	.restore = virtscsi_restore,
#endif
764
	.remove = virtscsi_remove,
765 766 767 768 769 770 771 772
};

static int __init init(void)
{
	int ret = -ENOMEM;

	virtscsi_cmd_cache = KMEM_CACHE(virtio_scsi_cmd, 0);
	if (!virtscsi_cmd_cache) {
773
		pr_err("kmem_cache_create() for virtscsi_cmd_cache failed\n");
774 775 776 777 778 779 780 781
		goto error;
	}


	virtscsi_cmd_pool =
		mempool_create_slab_pool(VIRTIO_SCSI_MEMPOOL_SZ,
					 virtscsi_cmd_cache);
	if (!virtscsi_cmd_pool) {
782
		pr_err("mempool_create() for virtscsi_cmd_pool failed\n");
783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814
		goto error;
	}
	ret = register_virtio_driver(&virtio_scsi_driver);
	if (ret < 0)
		goto error;

	return 0;

error:
	if (virtscsi_cmd_pool) {
		mempool_destroy(virtscsi_cmd_pool);
		virtscsi_cmd_pool = NULL;
	}
	if (virtscsi_cmd_cache) {
		kmem_cache_destroy(virtscsi_cmd_cache);
		virtscsi_cmd_cache = NULL;
	}
	return ret;
}

static void __exit fini(void)
{
	unregister_virtio_driver(&virtio_scsi_driver);
	mempool_destroy(virtscsi_cmd_pool);
	kmem_cache_destroy(virtscsi_cmd_cache);
}
module_init(init);
module_exit(fini);

MODULE_DEVICE_TABLE(virtio, id_table);
MODULE_DESCRIPTION("Virtio SCSI HBA driver");
MODULE_LICENSE("GPL");