ib_srp.c 93.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32
/*
 * Copyright (c) 2005 Cisco Systems.  All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

33
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
34

35 36 37 38 39 40 41
#include <linux/module.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/err.h>
#include <linux/string.h>
#include <linux/parser.h>
#include <linux/random.h>
42
#include <linux/jiffies.h>
43
#include <rdma/ib_cache.h>
44

A
Arun Sharma 已提交
45
#include <linux/atomic.h>
46 47 48 49

#include <scsi/scsi.h>
#include <scsi/scsi_device.h>
#include <scsi/scsi_dbg.h>
50
#include <scsi/scsi_tcq.h>
51
#include <scsi/srp.h>
52
#include <scsi/scsi_transport_srp.h>
53 54 55 56 57

#include "ib_srp.h"

#define DRV_NAME	"ib_srp"
#define PFX		DRV_NAME ": "
58 59
#define DRV_VERSION	"1.0"
#define DRV_RELDATE	"July 1, 2013"
60 61

MODULE_AUTHOR("Roland Dreier");
62
MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator");
63
MODULE_LICENSE("Dual BSD/GPL");
64 65
MODULE_VERSION(DRV_VERSION);
MODULE_INFO(release_date, DRV_RELDATE);
66

67 68
static unsigned int srp_sg_tablesize;
static unsigned int cmd_sg_entries;
69 70
static unsigned int indirect_sg_entries;
static bool allow_ext_sg;
71
static bool prefer_fr;
72
static bool register_always;
73
static int topspin_workarounds = 1;
74

75 76
module_param(srp_sg_tablesize, uint, 0444);
MODULE_PARM_DESC(srp_sg_tablesize, "Deprecated name for cmd_sg_entries");
77

78 79 80
module_param(cmd_sg_entries, uint, 0444);
MODULE_PARM_DESC(cmd_sg_entries,
		 "Default number of gather/scatter entries in the SRP command (default is 12, max 255)");
81

82 83 84 85 86 87 88 89
module_param(indirect_sg_entries, uint, 0444);
MODULE_PARM_DESC(indirect_sg_entries,
		 "Default max number of gather/scatter entries (default is 12, max is " __stringify(SCSI_MAX_SG_CHAIN_SEGMENTS) ")");

module_param(allow_ext_sg, bool, 0444);
MODULE_PARM_DESC(allow_ext_sg,
		  "Default behavior when there are more than cmd_sg_entries S/G entries after mapping; fails the request when false (default false)");

90 91 92 93
module_param(topspin_workarounds, int, 0444);
MODULE_PARM_DESC(topspin_workarounds,
		 "Enable workarounds for Topspin/Cisco SRP target bugs if != 0");

94 95 96 97
module_param(prefer_fr, bool, 0444);
MODULE_PARM_DESC(prefer_fr,
"Whether to use fast registration if both FMR and fast registration are supported");

98 99 100 101
module_param(register_always, bool, 0444);
MODULE_PARM_DESC(register_always,
		 "Use memory registration even for contiguous memory regions");

102 103
static struct kernel_param_ops srp_tmo_ops;

104 105 106 107 108
static int srp_reconnect_delay = 10;
module_param_cb(reconnect_delay, &srp_tmo_ops, &srp_reconnect_delay,
		S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(reconnect_delay, "Time between successive reconnect attempts");

109 110 111 112 113 114 115 116
static int srp_fast_io_fail_tmo = 15;
module_param_cb(fast_io_fail_tmo, &srp_tmo_ops, &srp_fast_io_fail_tmo,
		S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(fast_io_fail_tmo,
		 "Number of seconds between the observation of a transport"
		 " layer error and failing all I/O. \"off\" means that this"
		 " functionality is disabled.");

117
static int srp_dev_loss_tmo = 600;
118 119 120 121 122 123 124 125 126 127
module_param_cb(dev_loss_tmo, &srp_tmo_ops, &srp_dev_loss_tmo,
		S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(dev_loss_tmo,
		 "Maximum number of seconds that the SRP transport should"
		 " insulate transport layer errors. After this time has been"
		 " exceeded the SCSI host is removed. Should be"
		 " between 1 and " __stringify(SCSI_DEVICE_BLOCK_MAX_TIMEOUT)
		 " if fast_io_fail_tmo has not been set. \"off\" means that"
		 " this functionality is disabled.");

B
Bart Van Assche 已提交
128 129 130 131 132
static unsigned ch_count;
module_param(ch_count, uint, 0444);
MODULE_PARM_DESC(ch_count,
		 "Number of RDMA channels to use for communication with an SRP target. Using more than one channel improves performance if the HCA supports multiple completion vectors. The default value is the minimum of four times the number of online CPU sockets and the number of completion vectors supported by the HCA.");

133 134
static void srp_add_one(struct ib_device *device);
static void srp_remove_one(struct ib_device *device);
135 136
static void srp_recv_completion(struct ib_cq *cq, void *ch_ptr);
static void srp_send_completion(struct ib_cq *cq, void *ch_ptr);
137 138
static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event);

139
static struct scsi_transport_template *ib_srp_transport_template;
140
static struct workqueue_struct *srp_remove_wq;
141

142 143 144 145 146 147
static struct ib_client srp_client = {
	.name   = "srp",
	.add    = srp_add_one,
	.remove = srp_remove_one
};

148 149
static struct ib_sa_client srp_sa_client;

150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170
static int srp_tmo_get(char *buffer, const struct kernel_param *kp)
{
	int tmo = *(int *)kp->arg;

	if (tmo >= 0)
		return sprintf(buffer, "%d", tmo);
	else
		return sprintf(buffer, "off");
}

static int srp_tmo_set(const char *val, const struct kernel_param *kp)
{
	int tmo, res;

	if (strncmp(val, "off", 3) != 0) {
		res = kstrtoint(val, 0, &tmo);
		if (res)
			goto out;
	} else {
		tmo = -1;
	}
171 172 173 174 175
	if (kp->arg == &srp_reconnect_delay)
		res = srp_tmo_valid(tmo, srp_fast_io_fail_tmo,
				    srp_dev_loss_tmo);
	else if (kp->arg == &srp_fast_io_fail_tmo)
		res = srp_tmo_valid(srp_reconnect_delay, tmo, srp_dev_loss_tmo);
176
	else
177 178
		res = srp_tmo_valid(srp_reconnect_delay, srp_fast_io_fail_tmo,
				    tmo);
179 180 181 182 183 184 185 186 187 188 189 190 191
	if (res)
		goto out;
	*(int *)kp->arg = tmo;

out:
	return res;
}

static struct kernel_param_ops srp_tmo_ops = {
	.get = srp_tmo_get,
	.set = srp_tmo_set,
};

192 193 194 195 196 197 198 199 200 201
static inline struct srp_target_port *host_to_target(struct Scsi_Host *host)
{
	return (struct srp_target_port *) host->hostdata;
}

static const char *srp_target_info(struct Scsi_Host *host)
{
	return host_to_target(host)->target_name;
}

202 203 204
static int srp_target_is_topspin(struct srp_target_port *target)
{
	static const u8 topspin_oui[3] = { 0x00, 0x05, 0xad };
205
	static const u8 cisco_oui[3]   = { 0x00, 0x1b, 0x0d };
206 207

	return topspin_workarounds &&
208 209
		(!memcmp(&target->ioc_guid, topspin_oui, sizeof topspin_oui) ||
		 !memcmp(&target->ioc_guid, cisco_oui, sizeof cisco_oui));
210 211
}

212 213 214 215 216 217 218 219 220 221 222 223 224 225
static struct srp_iu *srp_alloc_iu(struct srp_host *host, size_t size,
				   gfp_t gfp_mask,
				   enum dma_data_direction direction)
{
	struct srp_iu *iu;

	iu = kmalloc(sizeof *iu, gfp_mask);
	if (!iu)
		goto out;

	iu->buf = kzalloc(size, gfp_mask);
	if (!iu->buf)
		goto out_free_iu;

226 227 228
	iu->dma = ib_dma_map_single(host->srp_dev->dev, iu->buf, size,
				    direction);
	if (ib_dma_mapping_error(host->srp_dev->dev, iu->dma))
229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248
		goto out_free_buf;

	iu->size      = size;
	iu->direction = direction;

	return iu;

out_free_buf:
	kfree(iu->buf);
out_free_iu:
	kfree(iu);
out:
	return NULL;
}

static void srp_free_iu(struct srp_host *host, struct srp_iu *iu)
{
	if (!iu)
		return;

249 250
	ib_dma_unmap_single(host->srp_dev->dev, iu->dma, iu->size,
			    iu->direction);
251 252 253 254 255 256
	kfree(iu->buf);
	kfree(iu);
}

static void srp_qp_event(struct ib_event *event, void *context)
{
257
	pr_debug("QP event %d\n", event->event);
258 259 260 261 262 263 264 265 266 267 268 269
}

static int srp_init_qp(struct srp_target_port *target,
		       struct ib_qp *qp)
{
	struct ib_qp_attr *attr;
	int ret;

	attr = kmalloc(sizeof *attr, GFP_KERNEL);
	if (!attr)
		return -ENOMEM;

270 271 272 273
	ret = ib_find_cached_pkey(target->srp_host->srp_dev->dev,
				  target->srp_host->port,
				  be16_to_cpu(target->pkey),
				  &attr->pkey_index);
274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292
	if (ret)
		goto out;

	attr->qp_state        = IB_QPS_INIT;
	attr->qp_access_flags = (IB_ACCESS_REMOTE_READ |
				    IB_ACCESS_REMOTE_WRITE);
	attr->port_num        = target->srp_host->port;

	ret = ib_modify_qp(qp, attr,
			   IB_QP_STATE		|
			   IB_QP_PKEY_INDEX	|
			   IB_QP_ACCESS_FLAGS	|
			   IB_QP_PORT);

out:
	kfree(attr);
	return ret;
}

293
static int srp_new_cm_id(struct srp_rdma_ch *ch)
D
David Dillow 已提交
294
{
295
	struct srp_target_port *target = ch->target;
D
David Dillow 已提交
296 297
	struct ib_cm_id *new_cm_id;

298
	new_cm_id = ib_create_cm_id(target->srp_host->srp_dev->dev,
299
				    srp_cm_handler, ch);
D
David Dillow 已提交
300 301 302
	if (IS_ERR(new_cm_id))
		return PTR_ERR(new_cm_id);

303 304 305 306 307 308 309
	if (ch->cm_id)
		ib_destroy_cm_id(ch->cm_id);
	ch->cm_id = new_cm_id;
	ch->path.sgid = target->sgid;
	ch->path.dgid = target->orig_dgid;
	ch->path.pkey = target->pkey;
	ch->path.service_id = target->service_id;
D
David Dillow 已提交
310 311 312 313

	return 0;
}

314 315 316 317 318 319 320 321 322
static struct ib_fmr_pool *srp_alloc_fmr_pool(struct srp_target_port *target)
{
	struct srp_device *dev = target->srp_host->srp_dev;
	struct ib_fmr_pool_param fmr_param;

	memset(&fmr_param, 0, sizeof(fmr_param));
	fmr_param.pool_size	    = target->scsi_host->can_queue;
	fmr_param.dirty_watermark   = fmr_param.pool_size / 4;
	fmr_param.cache		    = 1;
323 324
	fmr_param.max_pages_per_fmr = dev->max_pages_per_mr;
	fmr_param.page_shift	    = ilog2(dev->mr_page_size);
325 326 327 328 329 330 331
	fmr_param.access	    = (IB_ACCESS_LOCAL_WRITE |
				       IB_ACCESS_REMOTE_WRITE |
				       IB_ACCESS_REMOTE_READ);

	return ib_create_fmr_pool(dev->pd, &fmr_param);
}

332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457
/**
 * srp_destroy_fr_pool() - free the resources owned by a pool
 * @pool: Fast registration pool to be destroyed.
 */
static void srp_destroy_fr_pool(struct srp_fr_pool *pool)
{
	int i;
	struct srp_fr_desc *d;

	if (!pool)
		return;

	for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
		if (d->frpl)
			ib_free_fast_reg_page_list(d->frpl);
		if (d->mr)
			ib_dereg_mr(d->mr);
	}
	kfree(pool);
}

/**
 * srp_create_fr_pool() - allocate and initialize a pool for fast registration
 * @device:            IB device to allocate fast registration descriptors for.
 * @pd:                Protection domain associated with the FR descriptors.
 * @pool_size:         Number of descriptors to allocate.
 * @max_page_list_len: Maximum fast registration work request page list length.
 */
static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device,
					      struct ib_pd *pd, int pool_size,
					      int max_page_list_len)
{
	struct srp_fr_pool *pool;
	struct srp_fr_desc *d;
	struct ib_mr *mr;
	struct ib_fast_reg_page_list *frpl;
	int i, ret = -EINVAL;

	if (pool_size <= 0)
		goto err;
	ret = -ENOMEM;
	pool = kzalloc(sizeof(struct srp_fr_pool) +
		       pool_size * sizeof(struct srp_fr_desc), GFP_KERNEL);
	if (!pool)
		goto err;
	pool->size = pool_size;
	pool->max_page_list_len = max_page_list_len;
	spin_lock_init(&pool->lock);
	INIT_LIST_HEAD(&pool->free_list);

	for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
		mr = ib_alloc_fast_reg_mr(pd, max_page_list_len);
		if (IS_ERR(mr)) {
			ret = PTR_ERR(mr);
			goto destroy_pool;
		}
		d->mr = mr;
		frpl = ib_alloc_fast_reg_page_list(device, max_page_list_len);
		if (IS_ERR(frpl)) {
			ret = PTR_ERR(frpl);
			goto destroy_pool;
		}
		d->frpl = frpl;
		list_add_tail(&d->entry, &pool->free_list);
	}

out:
	return pool;

destroy_pool:
	srp_destroy_fr_pool(pool);

err:
	pool = ERR_PTR(ret);
	goto out;
}

/**
 * srp_fr_pool_get() - obtain a descriptor suitable for fast registration
 * @pool: Pool to obtain descriptor from.
 */
static struct srp_fr_desc *srp_fr_pool_get(struct srp_fr_pool *pool)
{
	struct srp_fr_desc *d = NULL;
	unsigned long flags;

	spin_lock_irqsave(&pool->lock, flags);
	if (!list_empty(&pool->free_list)) {
		d = list_first_entry(&pool->free_list, typeof(*d), entry);
		list_del(&d->entry);
	}
	spin_unlock_irqrestore(&pool->lock, flags);

	return d;
}

/**
 * srp_fr_pool_put() - put an FR descriptor back in the free list
 * @pool: Pool the descriptor was allocated from.
 * @desc: Pointer to an array of fast registration descriptor pointers.
 * @n:    Number of descriptors to put back.
 *
 * Note: The caller must already have queued an invalidation request for
 * desc->mr->rkey before calling this function.
 */
static void srp_fr_pool_put(struct srp_fr_pool *pool, struct srp_fr_desc **desc,
			    int n)
{
	unsigned long flags;
	int i;

	spin_lock_irqsave(&pool->lock, flags);
	for (i = 0; i < n; i++)
		list_add(&desc[i]->entry, &pool->free_list);
	spin_unlock_irqrestore(&pool->lock, flags);
}

static struct srp_fr_pool *srp_alloc_fr_pool(struct srp_target_port *target)
{
	struct srp_device *dev = target->srp_host->srp_dev;

	return srp_create_fr_pool(dev->dev, dev->pd,
				  target->scsi_host->can_queue,
				  dev->max_pages_per_mr);
}

458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474
/**
 * srp_destroy_qp() - destroy an RDMA queue pair
 * @ch: SRP RDMA channel.
 *
 * Change a queue pair into the error state and wait until all receive
 * completions have been processed before destroying it. This avoids that
 * the receive completion handler can access the queue pair while it is
 * being destroyed.
 */
static void srp_destroy_qp(struct srp_rdma_ch *ch)
{
	static struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
	static struct ib_recv_wr wr = { .wr_id = SRP_LAST_WR_ID };
	struct ib_recv_wr *bad_wr;
	int ret;

	/* Destroying a QP and reusing ch->done is only safe if not connected */
475
	WARN_ON_ONCE(ch->connected);
476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491

	ret = ib_modify_qp(ch->qp, &attr, IB_QP_STATE);
	WARN_ONCE(ret, "ib_cm_init_qp_attr() returned %d\n", ret);
	if (ret)
		goto out;

	init_completion(&ch->done);
	ret = ib_post_recv(ch->qp, &wr, &bad_wr);
	WARN_ONCE(ret, "ib_post_recv() returned %d\n", ret);
	if (ret == 0)
		wait_for_completion(&ch->done);

out:
	ib_destroy_qp(ch->qp);
}

492
static int srp_create_ch_ib(struct srp_rdma_ch *ch)
493
{
494
	struct srp_target_port *target = ch->target;
495
	struct srp_device *dev = target->srp_host->srp_dev;
496
	struct ib_qp_init_attr *init_attr;
497 498
	struct ib_cq *recv_cq, *send_cq;
	struct ib_qp *qp;
499
	struct ib_fmr_pool *fmr_pool = NULL;
500 501
	struct srp_fr_pool *fr_pool = NULL;
	const int m = 1 + dev->use_fast_reg;
502 503 504 505 506 507
	int ret;

	init_attr = kzalloc(sizeof *init_attr, GFP_KERNEL);
	if (!init_attr)
		return -ENOMEM;

508
	/* + 1 for SRP_LAST_WR_ID */
509
	recv_cq = ib_create_cq(dev->dev, srp_recv_completion, NULL, ch,
510
			       target->queue_size + 1, ch->comp_vector);
511 512
	if (IS_ERR(recv_cq)) {
		ret = PTR_ERR(recv_cq);
513
		goto err;
514 515
	}

516 517
	send_cq = ib_create_cq(dev->dev, srp_send_completion, NULL, ch,
			       m * target->queue_size, ch->comp_vector);
518 519
	if (IS_ERR(send_cq)) {
		ret = PTR_ERR(send_cq);
520
		goto err_recv_cq;
521 522
	}

523
	ib_req_notify_cq(recv_cq, IB_CQ_NEXT_COMP);
524 525

	init_attr->event_handler       = srp_qp_event;
526
	init_attr->cap.max_send_wr     = m * target->queue_size;
527
	init_attr->cap.max_recv_wr     = target->queue_size + 1;
528 529
	init_attr->cap.max_recv_sge    = 1;
	init_attr->cap.max_send_sge    = 1;
530
	init_attr->sq_sig_type         = IB_SIGNAL_REQ_WR;
531
	init_attr->qp_type             = IB_QPT_RC;
532 533
	init_attr->send_cq             = send_cq;
	init_attr->recv_cq             = recv_cq;
534

535
	qp = ib_create_qp(dev->pd, init_attr);
536 537
	if (IS_ERR(qp)) {
		ret = PTR_ERR(qp);
538
		goto err_send_cq;
539 540
	}

541
	ret = srp_init_qp(target, qp);
542 543
	if (ret)
		goto err_qp;
544

545 546 547 548 549 550 551 552
	if (dev->use_fast_reg && dev->has_fr) {
		fr_pool = srp_alloc_fr_pool(target);
		if (IS_ERR(fr_pool)) {
			ret = PTR_ERR(fr_pool);
			shost_printk(KERN_WARNING, target->scsi_host, PFX
				     "FR pool allocation failed (%d)\n", ret);
			goto err_qp;
		}
553 554 555
		if (ch->fr_pool)
			srp_destroy_fr_pool(ch->fr_pool);
		ch->fr_pool = fr_pool;
556
	} else if (!dev->use_fast_reg && dev->has_fmr) {
557 558 559 560 561 562 563
		fmr_pool = srp_alloc_fmr_pool(target);
		if (IS_ERR(fmr_pool)) {
			ret = PTR_ERR(fmr_pool);
			shost_printk(KERN_WARNING, target->scsi_host, PFX
				     "FMR pool allocation failed (%d)\n", ret);
			goto err_qp;
		}
564 565 566
		if (ch->fmr_pool)
			ib_destroy_fmr_pool(ch->fmr_pool);
		ch->fmr_pool = fmr_pool;
567 568
	}

569
	if (ch->qp)
570
		srp_destroy_qp(ch);
571 572 573 574
	if (ch->recv_cq)
		ib_destroy_cq(ch->recv_cq);
	if (ch->send_cq)
		ib_destroy_cq(ch->send_cq);
575

576 577 578
	ch->qp = qp;
	ch->recv_cq = recv_cq;
	ch->send_cq = send_cq;
579

580 581 582 583
	kfree(init_attr);
	return 0;

err_qp:
584
	ib_destroy_qp(qp);
585 586

err_send_cq:
587
	ib_destroy_cq(send_cq);
588 589

err_recv_cq:
590
	ib_destroy_cq(recv_cq);
591 592

err:
593 594 595 596
	kfree(init_attr);
	return ret;
}

597 598
/*
 * Note: this function may be called without srp_alloc_iu_bufs() having been
599
 * invoked. Hence the ch->[rt]x_ring checks.
600
 */
601 602
static void srp_free_ch_ib(struct srp_target_port *target,
			   struct srp_rdma_ch *ch)
603
{
604
	struct srp_device *dev = target->srp_host->srp_dev;
605 606
	int i;

B
Bart Van Assche 已提交
607 608 609
	if (!ch->target)
		return;

610 611 612
	if (ch->cm_id) {
		ib_destroy_cm_id(ch->cm_id);
		ch->cm_id = NULL;
613 614
	}

B
Bart Van Assche 已提交
615 616 617 618
	/* If srp_new_cm_id() succeeded but srp_create_ch_ib() not, return. */
	if (!ch->qp)
		return;

619
	if (dev->use_fast_reg) {
620 621
		if (ch->fr_pool)
			srp_destroy_fr_pool(ch->fr_pool);
622
	} else {
623 624
		if (ch->fmr_pool)
			ib_destroy_fmr_pool(ch->fmr_pool);
625
	}
626
	srp_destroy_qp(ch);
627 628
	ib_destroy_cq(ch->send_cq);
	ib_destroy_cq(ch->recv_cq);
629

B
Bart Van Assche 已提交
630 631 632 633 634 635 636 637
	/*
	 * Avoid that the SCSI error handler tries to use this channel after
	 * it has been freed. The SCSI error handler can namely continue
	 * trying to perform recovery actions after scsi_remove_host()
	 * returned.
	 */
	ch->target = NULL;

638 639
	ch->qp = NULL;
	ch->send_cq = ch->recv_cq = NULL;
640

641
	if (ch->rx_ring) {
642
		for (i = 0; i < target->queue_size; ++i)
643 644 645
			srp_free_iu(target->srp_host, ch->rx_ring[i]);
		kfree(ch->rx_ring);
		ch->rx_ring = NULL;
646
	}
647
	if (ch->tx_ring) {
648
		for (i = 0; i < target->queue_size; ++i)
649 650 651
			srp_free_iu(target->srp_host, ch->tx_ring[i]);
		kfree(ch->tx_ring);
		ch->tx_ring = NULL;
652
	}
653 654 655 656
}

static void srp_path_rec_completion(int status,
				    struct ib_sa_path_rec *pathrec,
657
				    void *ch_ptr)
658
{
659 660
	struct srp_rdma_ch *ch = ch_ptr;
	struct srp_target_port *target = ch->target;
661

662
	ch->status = status;
663
	if (status)
664 665
		shost_printk(KERN_ERR, target->scsi_host,
			     PFX "Got failed path rec status %d\n", status);
666
	else
667 668
		ch->path = *pathrec;
	complete(&ch->done);
669 670
}

671
static int srp_lookup_path(struct srp_rdma_ch *ch)
672
{
673
	struct srp_target_port *target = ch->target;
674 675
	int ret;

676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696
	ch->path.numb_path = 1;

	init_completion(&ch->done);

	ch->path_query_id = ib_sa_path_rec_get(&srp_sa_client,
					       target->srp_host->srp_dev->dev,
					       target->srp_host->port,
					       &ch->path,
					       IB_SA_PATH_REC_SERVICE_ID |
					       IB_SA_PATH_REC_DGID	 |
					       IB_SA_PATH_REC_SGID	 |
					       IB_SA_PATH_REC_NUMB_PATH	 |
					       IB_SA_PATH_REC_PKEY,
					       SRP_PATH_REC_TIMEOUT_MS,
					       GFP_KERNEL,
					       srp_path_rec_completion,
					       ch, &ch->path_query);
	if (ch->path_query_id < 0)
		return ch->path_query_id;

	ret = wait_for_completion_interruptible(&ch->done);
697 698
	if (ret < 0)
		return ret;
699

700
	if (ch->status < 0)
701 702
		shost_printk(KERN_WARNING, target->scsi_host,
			     PFX "Path record query failed\n");
703

704
	return ch->status;
705 706
}

B
Bart Van Assche 已提交
707
static int srp_send_req(struct srp_rdma_ch *ch, bool multich)
708
{
709
	struct srp_target_port *target = ch->target;
710 711 712 713 714 715 716 717 718 719
	struct {
		struct ib_cm_req_param param;
		struct srp_login_req   priv;
	} *req = NULL;
	int status;

	req = kzalloc(sizeof *req, GFP_KERNEL);
	if (!req)
		return -ENOMEM;

720
	req->param.primary_path		      = &ch->path;
721 722
	req->param.alternate_path 	      = NULL;
	req->param.service_id 		      = target->service_id;
723 724
	req->param.qp_num		      = ch->qp->qp_num;
	req->param.qp_type		      = ch->qp->qp_type;
725 726 727 728 729 730 731 732 733 734 735 736 737 738
	req->param.private_data 	      = &req->priv;
	req->param.private_data_len 	      = sizeof req->priv;
	req->param.flow_control 	      = 1;

	get_random_bytes(&req->param.starting_psn, 4);
	req->param.starting_psn 	     &= 0xffffff;

	/*
	 * Pick some arbitrary defaults here; we could make these
	 * module parameters if anyone cared about setting them.
	 */
	req->param.responder_resources	      = 4;
	req->param.remote_cm_response_timeout = 20;
	req->param.local_cm_response_timeout  = 20;
739
	req->param.retry_count                = target->tl_retry_count;
740 741 742 743 744
	req->param.rnr_retry_count 	      = 7;
	req->param.max_cm_retries 	      = 15;

	req->priv.opcode     	= SRP_LOGIN_REQ;
	req->priv.tag        	= 0;
745
	req->priv.req_it_iu_len = cpu_to_be32(target->max_iu_len);
746 747
	req->priv.req_buf_fmt 	= cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
					      SRP_BUF_FORMAT_INDIRECT);
B
Bart Van Assche 已提交
748 749
	req->priv.req_flags	= (multich ? SRP_MULTICHAN_MULTI :
				   SRP_MULTICHAN_SINGLE);
750
	/*
R
Roland Dreier 已提交
751
	 * In the published SRP specification (draft rev. 16a), the
752 753 754 755 756 757 758 759 760
	 * port identifier format is 8 bytes of ID extension followed
	 * by 8 bytes of GUID.  Older drafts put the two halves in the
	 * opposite order, so that the GUID comes first.
	 *
	 * Targets conforming to these obsolete drafts can be
	 * recognized by the I/O Class they report.
	 */
	if (target->io_class == SRP_REV10_IB_IO_CLASS) {
		memcpy(req->priv.initiator_port_id,
761
		       &target->sgid.global.interface_id, 8);
762
		memcpy(req->priv.initiator_port_id + 8,
763
		       &target->initiator_ext, 8);
764 765 766 767
		memcpy(req->priv.target_port_id,     &target->ioc_guid, 8);
		memcpy(req->priv.target_port_id + 8, &target->id_ext, 8);
	} else {
		memcpy(req->priv.initiator_port_id,
768 769
		       &target->initiator_ext, 8);
		memcpy(req->priv.initiator_port_id + 8,
770
		       &target->sgid.global.interface_id, 8);
771 772 773 774
		memcpy(req->priv.target_port_id,     &target->id_ext, 8);
		memcpy(req->priv.target_port_id + 8, &target->ioc_guid, 8);
	}

775 776
	/*
	 * Topspin/Cisco SRP targets will reject our login unless we
777 778
	 * zero out the first 8 bytes of our initiator port ID and set
	 * the second 8 bytes to the local node GUID.
779
	 */
780
	if (srp_target_is_topspin(target)) {
781 782 783
		shost_printk(KERN_DEBUG, target->scsi_host,
			     PFX "Topspin/Cisco initiator port ID workaround "
			     "activated for target GUID %016llx\n",
784
			     be64_to_cpu(target->ioc_guid));
785
		memset(req->priv.initiator_port_id, 0, 8);
786
		memcpy(req->priv.initiator_port_id + 8,
787
		       &target->srp_host->srp_dev->dev->node_guid, 8);
788 789
	}

790
	status = ib_send_cm_req(ch->cm_id, &req->param);
791 792 793 794 795 796

	kfree(req);

	return status;
}

797 798 799 800 801 802 803 804 805 806 807 808
static bool srp_queue_remove_work(struct srp_target_port *target)
{
	bool changed = false;

	spin_lock_irq(&target->lock);
	if (target->state != SRP_TARGET_REMOVED) {
		target->state = SRP_TARGET_REMOVED;
		changed = true;
	}
	spin_unlock_irq(&target->lock);

	if (changed)
809
		queue_work(srp_remove_wq, &target->remove_work);
810 811 812 813

	return changed;
}

814 815
static void srp_disconnect_target(struct srp_target_port *target)
{
B
Bart Van Assche 已提交
816 817
	struct srp_rdma_ch *ch;
	int i;
818

819
	/* XXX should send SRP_I_LOGOUT request */
820

821 822 823 824 825 826
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
		ch->connected = false;
		if (ch->cm_id && ib_send_cm_dreq(ch->cm_id, NULL, 0)) {
			shost_printk(KERN_DEBUG, target->scsi_host,
				     PFX "Sending CM DREQ failed\n");
827
		}
828
	}
829 830
}

831 832
static void srp_free_req_data(struct srp_target_port *target,
			      struct srp_rdma_ch *ch)
833
{
834 835
	struct srp_device *dev = target->srp_host->srp_dev;
	struct ib_device *ibdev = dev->dev;
836 837 838
	struct srp_request *req;
	int i;

839
	if (!ch->req_ring)
840 841 842
		return;

	for (i = 0; i < target->req_ring_size; ++i) {
843
		req = &ch->req_ring[i];
844 845 846 847
		if (dev->use_fast_reg)
			kfree(req->fr_list);
		else
			kfree(req->fmr_list);
848
		kfree(req->map_page);
849 850 851 852 853 854
		if (req->indirect_dma_addr) {
			ib_dma_unmap_single(ibdev, req->indirect_dma_addr,
					    target->indirect_size,
					    DMA_TO_DEVICE);
		}
		kfree(req->indirect_desc);
855
	}
856

857 858
	kfree(ch->req_ring);
	ch->req_ring = NULL;
859 860
}

861
static int srp_alloc_req_data(struct srp_rdma_ch *ch)
862
{
863
	struct srp_target_port *target = ch->target;
864 865 866
	struct srp_device *srp_dev = target->srp_host->srp_dev;
	struct ib_device *ibdev = srp_dev->dev;
	struct srp_request *req;
867
	void *mr_list;
868 869 870
	dma_addr_t dma_addr;
	int i, ret = -ENOMEM;

871 872 873
	ch->req_ring = kcalloc(target->req_ring_size, sizeof(*ch->req_ring),
			       GFP_KERNEL);
	if (!ch->req_ring)
874 875 876
		goto out;

	for (i = 0; i < target->req_ring_size; ++i) {
877
		req = &ch->req_ring[i];
878 879 880 881 882 883 884 885
		mr_list = kmalloc(target->cmd_sg_cnt * sizeof(void *),
				  GFP_KERNEL);
		if (!mr_list)
			goto out;
		if (srp_dev->use_fast_reg)
			req->fr_list = mr_list;
		else
			req->fmr_list = mr_list;
886
		req->map_page = kmalloc(srp_dev->max_pages_per_mr *
887
					sizeof(void *), GFP_KERNEL);
888 889
		if (!req->map_page)
			goto out;
890
		req->indirect_desc = kmalloc(target->indirect_size, GFP_KERNEL);
891
		if (!req->indirect_desc)
892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907
			goto out;

		dma_addr = ib_dma_map_single(ibdev, req->indirect_desc,
					     target->indirect_size,
					     DMA_TO_DEVICE);
		if (ib_dma_mapping_error(ibdev, dma_addr))
			goto out;

		req->indirect_dma_addr = dma_addr;
	}
	ret = 0;

out:
	return ret;
}

908 909 910 911 912 913 914 915 916 917 918 919 920 921 922
/**
 * srp_del_scsi_host_attr() - Remove attributes defined in the host template.
 * @shost: SCSI host whose attributes to remove from sysfs.
 *
 * Note: Any attributes defined in the host template and that did not exist
 * before invocation of this function will be ignored.
 */
static void srp_del_scsi_host_attr(struct Scsi_Host *shost)
{
	struct device_attribute **attr;

	for (attr = shost->hostt->shost_attrs; attr && *attr; ++attr)
		device_remove_file(&shost->shost_dev, *attr);
}

923 924
static void srp_remove_target(struct srp_target_port *target)
{
B
Bart Van Assche 已提交
925 926
	struct srp_rdma_ch *ch;
	int i;
927

928 929
	WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);

930
	srp_del_scsi_host_attr(target->scsi_host);
931
	srp_rport_get(target->rport);
932 933
	srp_remove_host(target->scsi_host);
	scsi_remove_host(target->scsi_host);
934
	srp_stop_rport_timers(target->rport);
935
	srp_disconnect_target(target);
B
Bart Van Assche 已提交
936 937 938 939
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
		srp_free_ch_ib(target, ch);
	}
940
	cancel_work_sync(&target->tl_err_work);
941
	srp_rport_put(target->rport);
B
Bart Van Assche 已提交
942 943 944 945 946 947
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
		srp_free_req_data(target, ch);
	}
	kfree(target->ch);
	target->ch = NULL;
948 949 950 951 952

	spin_lock(&target->srp_host->target_lock);
	list_del(&target->list);
	spin_unlock(&target->srp_host->target_lock);

953 954 955
	scsi_host_put(target->scsi_host);
}

D
David Howells 已提交
956
static void srp_remove_work(struct work_struct *work)
957
{
D
David Howells 已提交
958
	struct srp_target_port *target =
959
		container_of(work, struct srp_target_port, remove_work);
960

961
	WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
962

963
	srp_remove_target(target);
964 965
}

966 967 968 969 970 971 972
static void srp_rport_delete(struct srp_rport *rport)
{
	struct srp_target_port *target = rport->lld_data;

	srp_queue_remove_work(target);
}

973 974 975 976 977 978 979 980 981 982 983 984 985 986
/**
 * srp_connected_ch() - number of connected channels
 * @target: SRP target port.
 */
static int srp_connected_ch(struct srp_target_port *target)
{
	int i, c = 0;

	for (i = 0; i < target->ch_count; i++)
		c += target->ch[i].connected;

	return c;
}

B
Bart Van Assche 已提交
987
static int srp_connect_ch(struct srp_rdma_ch *ch, bool multich)
988
{
989
	struct srp_target_port *target = ch->target;
990 991
	int ret;

992
	WARN_ON_ONCE(!multich && srp_connected_ch(target) > 0);
993

994
	ret = srp_lookup_path(ch);
995 996 997 998
	if (ret)
		return ret;

	while (1) {
999
		init_completion(&ch->done);
B
Bart Van Assche 已提交
1000
		ret = srp_send_req(ch, multich);
1001 1002
		if (ret)
			return ret;
1003
		ret = wait_for_completion_interruptible(&ch->done);
1004 1005
		if (ret < 0)
			return ret;
1006 1007 1008 1009 1010 1011 1012

		/*
		 * The CM event handling code will set status to
		 * SRP_PORT_REDIRECT if we get a port redirect REJ
		 * back, or SRP_DLID_REDIRECT if we get a lid/qp
		 * redirect REJ back.
		 */
1013
		switch (ch->status) {
1014
		case 0:
1015
			ch->connected = true;
1016 1017 1018
			return 0;

		case SRP_PORT_REDIRECT:
1019
			ret = srp_lookup_path(ch);
1020 1021 1022 1023 1024 1025 1026
			if (ret)
				return ret;
			break;

		case SRP_DLID_REDIRECT:
			break;

D
David Dillow 已提交
1027 1028
		case SRP_STALE_CONN:
			shost_printk(KERN_ERR, target->scsi_host, PFX
1029
				     "giving up on stale connection\n");
1030 1031
			ch->status = -ECONNRESET;
			return ch->status;
D
David Dillow 已提交
1032

1033
		default:
1034
			return ch->status;
1035 1036 1037 1038
		}
	}
}

1039
static int srp_inv_rkey(struct srp_rdma_ch *ch, u32 rkey)
1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050
{
	struct ib_send_wr *bad_wr;
	struct ib_send_wr wr = {
		.opcode		    = IB_WR_LOCAL_INV,
		.wr_id		    = LOCAL_INV_WR_ID_MASK,
		.next		    = NULL,
		.num_sge	    = 0,
		.send_flags	    = 0,
		.ex.invalidate_rkey = rkey,
	};

1051
	return ib_post_send(ch->qp, &wr, &bad_wr);
1052 1053
}

1054
static void srp_unmap_data(struct scsi_cmnd *scmnd,
1055
			   struct srp_rdma_ch *ch,
1056 1057
			   struct srp_request *req)
{
1058
	struct srp_target_port *target = ch->target;
1059 1060 1061
	struct srp_device *dev = target->srp_host->srp_dev;
	struct ib_device *ibdev = dev->dev;
	int i, res;
1062

1063
	if (!scsi_sglist(scmnd) ||
1064 1065 1066 1067
	    (scmnd->sc_data_direction != DMA_TO_DEVICE &&
	     scmnd->sc_data_direction != DMA_FROM_DEVICE))
		return;

1068 1069 1070 1071
	if (dev->use_fast_reg) {
		struct srp_fr_desc **pfr;

		for (i = req->nmdesc, pfr = req->fr_list; i > 0; i--, pfr++) {
1072
			res = srp_inv_rkey(ch, (*pfr)->mr->rkey);
1073 1074 1075 1076 1077 1078 1079 1080 1081
			if (res < 0) {
				shost_printk(KERN_ERR, target->scsi_host, PFX
				  "Queueing INV WR for rkey %#x failed (%d)\n",
				  (*pfr)->mr->rkey, res);
				queue_work(system_long_wq,
					   &target->tl_err_work);
			}
		}
		if (req->nmdesc)
1082
			srp_fr_pool_put(ch->fr_pool, req->fr_list,
1083 1084 1085 1086 1087 1088 1089
					req->nmdesc);
	} else {
		struct ib_pool_fmr **pfmr;

		for (i = req->nmdesc, pfmr = req->fmr_list; i > 0; i--, pfmr++)
			ib_fmr_pool_unmap(*pfmr);
	}
1090

1091 1092
	ib_dma_unmap_sg(ibdev, scsi_sglist(scmnd), scsi_sg_count(scmnd),
			scmnd->sc_data_direction);
1093 1094
}

B
Bart Van Assche 已提交
1095 1096
/**
 * srp_claim_req - Take ownership of the scmnd associated with a request.
1097
 * @ch: SRP RDMA channel.
B
Bart Van Assche 已提交
1098
 * @req: SRP request.
1099
 * @sdev: If not NULL, only take ownership for this SCSI device.
B
Bart Van Assche 已提交
1100 1101 1102 1103 1104 1105
 * @scmnd: If NULL, take ownership of @req->scmnd. If not NULL, only take
 *         ownership of @req->scmnd if it equals @scmnd.
 *
 * Return value:
 * Either NULL or a pointer to the SCSI command the caller became owner of.
 */
1106
static struct scsi_cmnd *srp_claim_req(struct srp_rdma_ch *ch,
B
Bart Van Assche 已提交
1107
				       struct srp_request *req,
1108
				       struct scsi_device *sdev,
B
Bart Van Assche 已提交
1109 1110 1111 1112
				       struct scsi_cmnd *scmnd)
{
	unsigned long flags;

1113
	spin_lock_irqsave(&ch->lock, flags);
1114 1115 1116
	if (req->scmnd &&
	    (!sdev || req->scmnd->device == sdev) &&
	    (!scmnd || req->scmnd == scmnd)) {
B
Bart Van Assche 已提交
1117 1118 1119 1120 1121
		scmnd = req->scmnd;
		req->scmnd = NULL;
	} else {
		scmnd = NULL;
	}
1122
	spin_unlock_irqrestore(&ch->lock, flags);
B
Bart Van Assche 已提交
1123 1124 1125 1126 1127 1128

	return scmnd;
}

/**
 * srp_free_req() - Unmap data and add request to the free request list.
1129
 * @ch:     SRP RDMA channel.
1130 1131 1132
 * @req:    Request to be freed.
 * @scmnd:  SCSI command associated with @req.
 * @req_lim_delta: Amount to be added to @target->req_lim.
B
Bart Van Assche 已提交
1133
 */
1134 1135
static void srp_free_req(struct srp_rdma_ch *ch, struct srp_request *req,
			 struct scsi_cmnd *scmnd, s32 req_lim_delta)
1136
{
1137 1138
	unsigned long flags;

1139
	srp_unmap_data(scmnd, ch, req);
B
Bart Van Assche 已提交
1140

1141 1142 1143
	spin_lock_irqsave(&ch->lock, flags);
	ch->req_lim += req_lim_delta;
	spin_unlock_irqrestore(&ch->lock, flags);
1144 1145
}

1146 1147
static void srp_finish_req(struct srp_rdma_ch *ch, struct srp_request *req,
			   struct scsi_device *sdev, int result)
1148
{
1149
	struct scsi_cmnd *scmnd = srp_claim_req(ch, req, sdev, NULL);
B
Bart Van Assche 已提交
1150 1151

	if (scmnd) {
1152
		srp_free_req(ch, req, scmnd, 0);
1153
		scmnd->result = result;
B
Bart Van Assche 已提交
1154 1155
		scmnd->scsi_done(scmnd);
	}
1156 1157
}

1158
static void srp_terminate_io(struct srp_rport *rport)
1159
{
1160
	struct srp_target_port *target = rport->lld_data;
B
Bart Van Assche 已提交
1161
	struct srp_rdma_ch *ch;
1162 1163
	struct Scsi_Host *shost = target->scsi_host;
	struct scsi_device *sdev;
B
Bart Van Assche 已提交
1164
	int i, j;
1165

1166 1167 1168 1169 1170 1171 1172
	/*
	 * Invoking srp_terminate_io() while srp_queuecommand() is running
	 * is not safe. Hence the warning statement below.
	 */
	shost_for_each_device(sdev, shost)
		WARN_ON_ONCE(sdev->request_queue->request_fn_active);

B
Bart Van Assche 已提交
1173 1174
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
1175

B
Bart Van Assche 已提交
1176 1177 1178 1179 1180 1181
		for (j = 0; j < target->req_ring_size; ++j) {
			struct srp_request *req = &ch->req_ring[j];

			srp_finish_req(ch, req, NULL,
				       DID_TRANSPORT_FAILFAST << 16);
		}
1182 1183
	}
}
1184

1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196
/*
 * It is up to the caller to ensure that srp_rport_reconnect() calls are
 * serialized and that no concurrent srp_queuecommand(), srp_abort(),
 * srp_reset_device() or srp_reset_host() calls will occur while this function
 * is in progress. One way to realize that is not to call this function
 * directly but to call srp_reconnect_rport() instead since that last function
 * serializes calls of this function via rport->mutex and also blocks
 * srp_queuecommand() calls before invoking this function.
 */
static int srp_rport_reconnect(struct srp_rport *rport)
{
	struct srp_target_port *target = rport->lld_data;
B
Bart Van Assche 已提交
1197 1198 1199
	struct srp_rdma_ch *ch;
	int i, j, ret = 0;
	bool multich = false;
1200

1201
	srp_disconnect_target(target);
1202 1203 1204 1205

	if (target->state == SRP_TARGET_SCANNING)
		return -ENODEV;

1206
	/*
1207 1208 1209
	 * Now get a new local CM ID so that we avoid confusing the target in
	 * case things are really fouled up. Doing so also ensures that all CM
	 * callbacks will have finished before a new QP is allocated.
1210
	 */
B
Bart Van Assche 已提交
1211 1212 1213 1214 1215
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
		if (!ch->target)
			break;
		ret += srp_new_cm_id(ch);
1216
	}
B
Bart Van Assche 已提交
1217 1218 1219 1220 1221 1222
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
		if (!ch->target)
			break;
		for (j = 0; j < target->req_ring_size; ++j) {
			struct srp_request *req = &ch->req_ring[j];
1223

B
Bart Van Assche 已提交
1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236
			srp_finish_req(ch, req, NULL, DID_RESET << 16);
		}
	}
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
		if (!ch->target)
			break;
		/*
		 * Whether or not creating a new CM ID succeeded, create a new
		 * QP. This guarantees that all completion callback function
		 * invocations have finished before request resetting starts.
		 */
		ret += srp_create_ch_ib(ch);
1237

B
Bart Van Assche 已提交
1238 1239 1240 1241
		INIT_LIST_HEAD(&ch->free_tx);
		for (j = 0; j < target->queue_size; ++j)
			list_add(&ch->tx_ring[j]->list, &ch->free_tx);
	}
1242 1243 1244

	target->qp_in_error = false;

B
Bart Van Assche 已提交
1245 1246
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
1247
		if (ret || !ch->target)
B
Bart Van Assche 已提交
1248 1249 1250 1251
			break;
		ret = srp_connect_ch(ch, multich);
		multich = true;
	}
1252

1253 1254 1255
	if (ret == 0)
		shost_printk(KERN_INFO, target->scsi_host,
			     PFX "reconnect succeeded\n");
1256 1257 1258 1259

	return ret;
}

1260 1261
static void srp_map_desc(struct srp_map_state *state, dma_addr_t dma_addr,
			 unsigned int dma_len, u32 rkey)
1262
{
1263
	struct srp_direct_buf *desc = state->desc;
1264

1265 1266 1267
	desc->va = cpu_to_be64(dma_addr);
	desc->key = cpu_to_be32(rkey);
	desc->len = cpu_to_be32(dma_len);
1268

1269 1270 1271 1272
	state->total_len += dma_len;
	state->desc++;
	state->ndesc++;
}
1273

1274
static int srp_map_finish_fmr(struct srp_map_state *state,
1275
			      struct srp_rdma_ch *ch)
1276 1277 1278
{
	struct ib_pool_fmr *fmr;
	u64 io_addr = 0;
1279

1280
	fmr = ib_fmr_pool_map_phys(ch->fmr_pool, state->pages,
1281 1282 1283
				   state->npages, io_addr);
	if (IS_ERR(fmr))
		return PTR_ERR(fmr);
1284

1285
	*state->next_fmr++ = fmr;
1286
	state->nmdesc++;
1287

1288
	srp_map_desc(state, 0, state->dma_len, fmr->fmr->rkey);
1289

1290 1291 1292
	return 0;
}

1293
static int srp_map_finish_fr(struct srp_map_state *state,
1294
			     struct srp_rdma_ch *ch)
1295
{
1296
	struct srp_target_port *target = ch->target;
1297 1298 1299 1300 1301 1302
	struct srp_device *dev = target->srp_host->srp_dev;
	struct ib_send_wr *bad_wr;
	struct ib_send_wr wr;
	struct srp_fr_desc *desc;
	u32 rkey;

1303
	desc = srp_fr_pool_get(ch->fr_pool);
1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331
	if (!desc)
		return -ENOMEM;

	rkey = ib_inc_rkey(desc->mr->rkey);
	ib_update_fast_reg_key(desc->mr, rkey);

	memcpy(desc->frpl->page_list, state->pages,
	       sizeof(state->pages[0]) * state->npages);

	memset(&wr, 0, sizeof(wr));
	wr.opcode = IB_WR_FAST_REG_MR;
	wr.wr_id = FAST_REG_WR_ID_MASK;
	wr.wr.fast_reg.iova_start = state->base_dma_addr;
	wr.wr.fast_reg.page_list = desc->frpl;
	wr.wr.fast_reg.page_list_len = state->npages;
	wr.wr.fast_reg.page_shift = ilog2(dev->mr_page_size);
	wr.wr.fast_reg.length = state->dma_len;
	wr.wr.fast_reg.access_flags = (IB_ACCESS_LOCAL_WRITE |
				       IB_ACCESS_REMOTE_READ |
				       IB_ACCESS_REMOTE_WRITE);
	wr.wr.fast_reg.rkey = desc->mr->lkey;

	*state->next_fr++ = desc;
	state->nmdesc++;

	srp_map_desc(state, state->base_dma_addr, state->dma_len,
		     desc->mr->rkey);

1332
	return ib_post_send(ch->qp, &wr, &bad_wr);
1333 1334
}

1335
static int srp_finish_mapping(struct srp_map_state *state,
1336
			      struct srp_rdma_ch *ch)
1337
{
1338
	struct srp_target_port *target = ch->target;
1339 1340 1341 1342 1343
	int ret = 0;

	if (state->npages == 0)
		return 0;

1344
	if (state->npages == 1 && !register_always)
1345
		srp_map_desc(state, state->base_dma_addr, state->dma_len,
1346 1347
			     target->rkey);
	else
1348
		ret = target->srp_host->srp_dev->use_fast_reg ?
1349 1350
			srp_map_finish_fr(state, ch) :
			srp_map_finish_fmr(state, ch);
1351 1352 1353

	if (ret == 0) {
		state->npages = 0;
1354
		state->dma_len = 0;
1355 1356 1357 1358 1359
	}

	return ret;
}

1360 1361 1362 1363 1364 1365 1366 1367
static void srp_map_update_start(struct srp_map_state *state,
				 struct scatterlist *sg, int sg_index,
				 dma_addr_t dma_addr)
{
	state->unmapped_sg = sg;
	state->unmapped_index = sg_index;
	state->unmapped_addr = dma_addr;
}
1368

1369
static int srp_map_sg_entry(struct srp_map_state *state,
1370
			    struct srp_rdma_ch *ch,
1371
			    struct scatterlist *sg, int sg_index,
1372
			    bool use_mr)
1373
{
1374
	struct srp_target_port *target = ch->target;
1375 1376 1377 1378 1379 1380 1381 1382 1383 1384
	struct srp_device *dev = target->srp_host->srp_dev;
	struct ib_device *ibdev = dev->dev;
	dma_addr_t dma_addr = ib_sg_dma_address(ibdev, sg);
	unsigned int dma_len = ib_sg_dma_len(ibdev, sg);
	unsigned int len;
	int ret;

	if (!dma_len)
		return 0;

1385 1386 1387 1388
	if (!use_mr) {
		/*
		 * Once we're in direct map mode for a request, we don't
		 * go back to FMR or FR mode, so no need to update anything
1389 1390 1391 1392
		 * other than the descriptor.
		 */
		srp_map_desc(state, dma_addr, dma_len, target->rkey);
		return 0;
1393
	}
1394

1395 1396 1397 1398 1399
	/*
	 * Since not all RDMA HW drivers support non-zero page offsets for
	 * FMR, if we start at an offset into a page, don't merge into the
	 * current FMR mapping. Finish it out, and use the kernel's MR for
	 * this sg entry.
1400
	 */
1401 1402
	if ((!dev->use_fast_reg && dma_addr & ~dev->mr_page_mask) ||
	    dma_len > dev->mr_max_size) {
1403
		ret = srp_finish_mapping(state, ch);
1404 1405 1406 1407 1408 1409
		if (ret)
			return ret;

		srp_map_desc(state, dma_addr, dma_len, target->rkey);
		srp_map_update_start(state, NULL, 0, 0);
		return 0;
1410 1411
	}

1412 1413 1414 1415 1416
	/*
	 * If this is the first sg that will be mapped via FMR or via FR, save
	 * our position. We need to know the first unmapped entry, its index,
	 * and the first unmapped address within that entry to be able to
	 * restart mapping after an error.
1417 1418 1419
	 */
	if (!state->unmapped_sg)
		srp_map_update_start(state, sg, sg_index, dma_addr);
1420

1421
	while (dma_len) {
1422 1423
		unsigned offset = dma_addr & ~dev->mr_page_mask;
		if (state->npages == dev->max_pages_per_mr || offset != 0) {
1424
			ret = srp_finish_mapping(state, ch);
1425 1426
			if (ret)
				return ret;
1427

1428 1429 1430
			srp_map_update_start(state, sg, sg_index, dma_addr);
		}

1431
		len = min_t(unsigned int, dma_len, dev->mr_page_size - offset);
1432

1433 1434
		if (!state->npages)
			state->base_dma_addr = dma_addr;
1435
		state->pages[state->npages++] = dma_addr & dev->mr_page_mask;
1436
		state->dma_len += len;
1437 1438 1439 1440
		dma_addr += len;
		dma_len -= len;
	}

1441 1442
	/*
	 * If the last entry of the MR wasn't a full page, then we need to
1443 1444 1445 1446
	 * close it out and start a new one -- we can only merge at page
	 * boundries.
	 */
	ret = 0;
1447
	if (len != dev->mr_page_size) {
1448
		ret = srp_finish_mapping(state, ch);
1449 1450 1451
		if (!ret)
			srp_map_update_start(state, NULL, 0, 0);
	}
1452 1453 1454
	return ret;
}

1455 1456 1457
static int srp_map_sg(struct srp_map_state *state, struct srp_rdma_ch *ch,
		      struct srp_request *req, struct scatterlist *scat,
		      int count)
1458
{
1459
	struct srp_target_port *target = ch->target;
1460 1461 1462
	struct srp_device *dev = target->srp_host->srp_dev;
	struct ib_device *ibdev = dev->dev;
	struct scatterlist *sg;
1463 1464
	int i;
	bool use_mr;
1465 1466 1467

	state->desc	= req->indirect_desc;
	state->pages	= req->map_page;
1468 1469
	if (dev->use_fast_reg) {
		state->next_fr = req->fr_list;
1470
		use_mr = !!ch->fr_pool;
1471 1472
	} else {
		state->next_fmr = req->fmr_list;
1473
		use_mr = !!ch->fmr_pool;
1474
	}
1475 1476

	for_each_sg(scat, sg, count, i) {
1477
		if (srp_map_sg_entry(state, ch, sg, i, use_mr)) {
1478 1479 1480 1481
			/*
			 * Memory registration failed, so backtrack to the
			 * first unmapped entry and continue on without using
			 * memory registration.
1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493
			 */
			dma_addr_t dma_addr;
			unsigned int dma_len;

backtrack:
			sg = state->unmapped_sg;
			i = state->unmapped_index;

			dma_addr = ib_sg_dma_address(ibdev, sg);
			dma_len = ib_sg_dma_len(ibdev, sg);
			dma_len -= (state->unmapped_addr - dma_addr);
			dma_addr = state->unmapped_addr;
1494
			use_mr = false;
1495 1496 1497 1498
			srp_map_desc(state, dma_addr, dma_len, target->rkey);
		}
	}

1499
	if (use_mr && srp_finish_mapping(state, ch))
1500 1501
		goto backtrack;

1502
	req->nmdesc = state->nmdesc;
1503 1504

	return 0;
1505 1506
}

1507
static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch,
1508 1509
			struct srp_request *req)
{
1510
	struct srp_target_port *target = ch->target;
1511
	struct scatterlist *scat;
1512
	struct srp_cmd *cmd = req->cmd->buf;
1513
	int len, nents, count;
1514 1515
	struct srp_device *dev;
	struct ib_device *ibdev;
1516 1517 1518 1519
	struct srp_map_state state;
	struct srp_indirect_buf *indirect_hdr;
	u32 table_len;
	u8 fmt;
1520

1521
	if (!scsi_sglist(scmnd) || scmnd->sc_data_direction == DMA_NONE)
1522 1523 1524 1525
		return sizeof (struct srp_cmd);

	if (scmnd->sc_data_direction != DMA_FROM_DEVICE &&
	    scmnd->sc_data_direction != DMA_TO_DEVICE) {
1526 1527 1528
		shost_printk(KERN_WARNING, target->scsi_host,
			     PFX "Unhandled data direction %d\n",
			     scmnd->sc_data_direction);
1529 1530 1531
		return -EINVAL;
	}

1532 1533
	nents = scsi_sg_count(scmnd);
	scat  = scsi_sglist(scmnd);
1534

1535
	dev = target->srp_host->srp_dev;
1536 1537 1538
	ibdev = dev->dev;

	count = ib_dma_map_sg(ibdev, scat, nents, scmnd->sc_data_direction);
1539 1540
	if (unlikely(count == 0))
		return -EIO;
1541 1542 1543

	fmt = SRP_DATA_DESC_DIRECT;
	len = sizeof (struct srp_cmd) +	sizeof (struct srp_direct_buf);
1544

1545
	if (count == 1 && !register_always) {
1546 1547 1548 1549 1550 1551
		/*
		 * The midlayer only generated a single gather/scatter
		 * entry, or DMA mapping coalesced everything to a
		 * single entry.  So a direct descriptor along with
		 * the DMA MR suffices.
		 */
1552
		struct srp_direct_buf *buf = (void *) cmd->add_data;
1553

1554
		buf->va  = cpu_to_be64(ib_sg_dma_address(ibdev, scat));
1555
		buf->key = cpu_to_be32(target->rkey);
1556
		buf->len = cpu_to_be32(ib_sg_dma_len(ibdev, scat));
1557

1558
		req->nmdesc = 0;
1559 1560 1561
		goto map_complete;
	}

1562 1563 1564
	/*
	 * We have more than one scatter/gather entry, so build our indirect
	 * descriptor table, trying to merge as many entries as we can.
1565 1566 1567
	 */
	indirect_hdr = (void *) cmd->add_data;

1568 1569 1570
	ib_dma_sync_single_for_cpu(ibdev, req->indirect_dma_addr,
				   target->indirect_size, DMA_TO_DEVICE);

1571
	memset(&state, 0, sizeof(state));
1572
	srp_map_sg(&state, ch, req, scat, count);
1573

1574 1575 1576 1577 1578
	/* We've mapped the request, now pull as much of the indirect
	 * descriptor table as we can into the command buffer. If this
	 * target is not using an external indirect table, we are
	 * guaranteed to fit into the command, as the SCSI layer won't
	 * give us more S/G entries than we allow.
1579 1580
	 */
	if (state.ndesc == 1) {
1581 1582
		/*
		 * Memory registration collapsed the sg-list into one entry,
1583 1584 1585
		 * so use a direct descriptor.
		 */
		struct srp_direct_buf *buf = (void *) cmd->add_data;
1586

1587
		*buf = req->indirect_desc[0];
1588
		goto map_complete;
1589 1590
	}

1591 1592 1593 1594 1595 1596 1597 1598
	if (unlikely(target->cmd_sg_cnt < state.ndesc &&
						!target->allow_ext_sg)) {
		shost_printk(KERN_ERR, target->scsi_host,
			     "Could not fit S/G list into SRP_CMD\n");
		return -EIO;
	}

	count = min(state.ndesc, target->cmd_sg_cnt);
1599 1600 1601 1602
	table_len = state.ndesc * sizeof (struct srp_direct_buf);

	fmt = SRP_DATA_DESC_INDIRECT;
	len = sizeof(struct srp_cmd) + sizeof (struct srp_indirect_buf);
1603
	len += count * sizeof (struct srp_direct_buf);
1604

1605 1606
	memcpy(indirect_hdr->desc_list, req->indirect_desc,
	       count * sizeof (struct srp_direct_buf));
1607

1608
	indirect_hdr->table_desc.va = cpu_to_be64(req->indirect_dma_addr);
1609 1610 1611 1612 1613
	indirect_hdr->table_desc.key = cpu_to_be32(target->rkey);
	indirect_hdr->table_desc.len = cpu_to_be32(table_len);
	indirect_hdr->len = cpu_to_be32(state.total_len);

	if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1614
		cmd->data_out_desc_cnt = count;
1615
	else
1616 1617 1618 1619
		cmd->data_in_desc_cnt = count;

	ib_dma_sync_single_for_device(ibdev, req->indirect_dma_addr, table_len,
				      DMA_TO_DEVICE);
1620 1621

map_complete:
1622 1623 1624 1625 1626 1627 1628 1629
	if (scmnd->sc_data_direction == DMA_TO_DEVICE)
		cmd->buf_fmt = fmt << 4;
	else
		cmd->buf_fmt = fmt;

	return len;
}

1630 1631 1632
/*
 * Return an IU and possible credit to the free pool
 */
1633
static void srp_put_tx_iu(struct srp_rdma_ch *ch, struct srp_iu *iu,
1634 1635 1636 1637
			  enum srp_iu_type iu_type)
{
	unsigned long flags;

1638 1639
	spin_lock_irqsave(&ch->lock, flags);
	list_add(&iu->list, &ch->free_tx);
1640
	if (iu_type != SRP_IU_RSP)
1641 1642
		++ch->req_lim;
	spin_unlock_irqrestore(&ch->lock, flags);
1643 1644
}

1645
/*
1646
 * Must be called with ch->lock held to protect req_lim and free_tx.
1647
 * If IU is not sent, it must be returned using srp_put_tx_iu().
1648 1649 1650 1651 1652 1653 1654 1655 1656 1657
 *
 * Note:
 * An upper limit for the number of allocated information units for each
 * request type is:
 * - SRP_IU_CMD: SRP_CMD_SQ_SIZE, since the SCSI mid-layer never queues
 *   more than Scsi_Host.can_queue requests.
 * - SRP_IU_TSK_MGMT: SRP_TSK_MGMT_SQ_SIZE.
 * - SRP_IU_RSP: 1, since a conforming SRP target never sends more than
 *   one unanswered SRP request to an initiator.
 */
1658
static struct srp_iu *__srp_get_tx_iu(struct srp_rdma_ch *ch,
1659 1660
				      enum srp_iu_type iu_type)
{
1661
	struct srp_target_port *target = ch->target;
1662 1663 1664
	s32 rsv = (iu_type == SRP_IU_TSK_MGMT) ? 0 : SRP_TSK_MGMT_SQ_SIZE;
	struct srp_iu *iu;

1665
	srp_send_completion(ch->send_cq, ch);
1666

1667
	if (list_empty(&ch->free_tx))
1668 1669 1670
		return NULL;

	/* Initiator responses to target requests do not consume credits */
1671
	if (iu_type != SRP_IU_RSP) {
1672
		if (ch->req_lim <= rsv) {
1673 1674 1675 1676
			++target->zero_req_lim;
			return NULL;
		}

1677
		--ch->req_lim;
1678 1679
	}

1680
	iu = list_first_entry(&ch->free_tx, struct srp_iu, list);
1681
	list_del(&iu->list);
1682 1683 1684
	return iu;
}

1685
static int srp_post_send(struct srp_rdma_ch *ch, struct srp_iu *iu, int len)
1686
{
1687
	struct srp_target_port *target = ch->target;
1688 1689 1690 1691 1692
	struct ib_sge list;
	struct ib_send_wr wr, *bad_wr;

	list.addr   = iu->dma;
	list.length = len;
1693
	list.lkey   = target->lkey;
1694 1695

	wr.next       = NULL;
1696
	wr.wr_id      = (uintptr_t) iu;
1697 1698 1699 1700 1701
	wr.sg_list    = &list;
	wr.num_sge    = 1;
	wr.opcode     = IB_WR_SEND;
	wr.send_flags = IB_SEND_SIGNALED;

1702
	return ib_post_send(ch->qp, &wr, &bad_wr);
1703 1704
}

1705
static int srp_post_recv(struct srp_rdma_ch *ch, struct srp_iu *iu)
1706
{
1707
	struct srp_target_port *target = ch->target;
1708
	struct ib_recv_wr wr, *bad_wr;
1709
	struct ib_sge list;
1710 1711 1712

	list.addr   = iu->dma;
	list.length = iu->size;
1713
	list.lkey   = target->lkey;
1714 1715

	wr.next     = NULL;
1716
	wr.wr_id    = (uintptr_t) iu;
1717 1718 1719
	wr.sg_list  = &list;
	wr.num_sge  = 1;

1720
	return ib_post_recv(ch->qp, &wr, &bad_wr);
1721 1722
}

1723
static void srp_process_rsp(struct srp_rdma_ch *ch, struct srp_rsp *rsp)
1724
{
1725
	struct srp_target_port *target = ch->target;
1726 1727 1728 1729 1730
	struct srp_request *req;
	struct scsi_cmnd *scmnd;
	unsigned long flags;

	if (unlikely(rsp->tag & SRP_TAG_TSK_MGMT)) {
1731 1732 1733
		spin_lock_irqsave(&ch->lock, flags);
		ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
		spin_unlock_irqrestore(&ch->lock, flags);
1734

1735
		ch->tsk_mgmt_status = -1;
1736
		if (be32_to_cpu(rsp->resp_data_len) >= 4)
1737 1738
			ch->tsk_mgmt_status = rsp->data[3];
		complete(&ch->tsk_mgmt_done);
1739
	} else {
B
Bart Van Assche 已提交
1740 1741 1742 1743 1744
		scmnd = scsi_host_find_tag(target->scsi_host, rsp->tag);
		if (scmnd) {
			req = (void *)scmnd->host_scribble;
			scmnd = srp_claim_req(ch, req, NULL, scmnd);
		}
B
Bart Van Assche 已提交
1745
		if (!scmnd) {
1746
			shost_printk(KERN_ERR, target->scsi_host,
B
Bart Van Assche 已提交
1747 1748
				     "Null scmnd for RSP w/tag %#016llx received on ch %td / QP %#x\n",
				     rsp->tag, ch - target->ch, ch->qp->qp_num);
B
Bart Van Assche 已提交
1749

1750 1751 1752
			spin_lock_irqsave(&ch->lock, flags);
			ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
			spin_unlock_irqrestore(&ch->lock, flags);
B
Bart Van Assche 已提交
1753 1754 1755

			return;
		}
1756 1757 1758 1759 1760 1761 1762 1763 1764
		scmnd->result = rsp->status;

		if (rsp->flags & SRP_RSP_FLAG_SNSVALID) {
			memcpy(scmnd->sense_buffer, rsp->data +
			       be32_to_cpu(rsp->resp_data_len),
			       min_t(int, be32_to_cpu(rsp->sense_data_len),
				     SCSI_SENSE_BUFFERSIZE));
		}

B
Bart Van Assche 已提交
1765
		if (unlikely(rsp->flags & SRP_RSP_FLAG_DIUNDER))
1766
			scsi_set_resid(scmnd, be32_to_cpu(rsp->data_in_res_cnt));
B
Bart Van Assche 已提交
1767 1768 1769 1770 1771 1772
		else if (unlikely(rsp->flags & SRP_RSP_FLAG_DIOVER))
			scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_in_res_cnt));
		else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOUNDER))
			scsi_set_resid(scmnd, be32_to_cpu(rsp->data_out_res_cnt));
		else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOOVER))
			scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_out_res_cnt));
1773

1774
		srp_free_req(ch, req, scmnd,
B
Bart Van Assche 已提交
1775 1776
			     be32_to_cpu(rsp->req_lim_delta));

1777 1778
		scmnd->host_scribble = NULL;
		scmnd->scsi_done(scmnd);
1779 1780 1781
	}
}

1782
static int srp_response_common(struct srp_rdma_ch *ch, s32 req_delta,
1783 1784
			       void *rsp, int len)
{
1785
	struct srp_target_port *target = ch->target;
1786
	struct ib_device *dev = target->srp_host->srp_dev->dev;
1787 1788
	unsigned long flags;
	struct srp_iu *iu;
1789
	int err;
1790

1791 1792 1793 1794
	spin_lock_irqsave(&ch->lock, flags);
	ch->req_lim += req_delta;
	iu = __srp_get_tx_iu(ch, SRP_IU_RSP);
	spin_unlock_irqrestore(&ch->lock, flags);
1795

1796 1797 1798
	if (!iu) {
		shost_printk(KERN_ERR, target->scsi_host, PFX
			     "no IU available to send response\n");
1799
		return 1;
1800 1801 1802 1803 1804 1805
	}

	ib_dma_sync_single_for_cpu(dev, iu->dma, len, DMA_TO_DEVICE);
	memcpy(iu->buf, rsp, len);
	ib_dma_sync_single_for_device(dev, iu->dma, len, DMA_TO_DEVICE);

1806
	err = srp_post_send(ch, iu, len);
1807
	if (err) {
1808 1809
		shost_printk(KERN_ERR, target->scsi_host, PFX
			     "unable to post response: %d\n", err);
1810
		srp_put_tx_iu(ch, iu, SRP_IU_RSP);
1811
	}
1812 1813 1814 1815

	return err;
}

1816
static void srp_process_cred_req(struct srp_rdma_ch *ch,
1817 1818 1819 1820 1821 1822 1823 1824
				 struct srp_cred_req *req)
{
	struct srp_cred_rsp rsp = {
		.opcode = SRP_CRED_RSP,
		.tag = req->tag,
	};
	s32 delta = be32_to_cpu(req->req_lim_delta);

1825 1826
	if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
		shost_printk(KERN_ERR, ch->target->scsi_host, PFX
1827 1828 1829
			     "problems processing SRP_CRED_REQ\n");
}

1830
static void srp_process_aer_req(struct srp_rdma_ch *ch,
1831 1832
				struct srp_aer_req *req)
{
1833
	struct srp_target_port *target = ch->target;
1834 1835 1836 1837 1838 1839 1840 1841 1842
	struct srp_aer_rsp rsp = {
		.opcode = SRP_AER_RSP,
		.tag = req->tag,
	};
	s32 delta = be32_to_cpu(req->req_lim_delta);

	shost_printk(KERN_ERR, target->scsi_host, PFX
		     "ignoring AER for LUN %llu\n", be64_to_cpu(req->lun));

1843
	if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
1844 1845 1846 1847
		shost_printk(KERN_ERR, target->scsi_host, PFX
			     "problems processing SRP_AER_REQ\n");
}

1848
static void srp_handle_recv(struct srp_rdma_ch *ch, struct ib_wc *wc)
1849
{
1850
	struct srp_target_port *target = ch->target;
1851
	struct ib_device *dev = target->srp_host->srp_dev->dev;
1852
	struct srp_iu *iu = (struct srp_iu *) (uintptr_t) wc->wr_id;
1853
	int res;
1854 1855
	u8 opcode;

1856
	ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_ti_iu_len,
1857
				   DMA_FROM_DEVICE);
1858 1859 1860 1861

	opcode = *(u8 *) iu->buf;

	if (0) {
1862 1863
		shost_printk(KERN_ERR, target->scsi_host,
			     PFX "recv completion, opcode 0x%02x\n", opcode);
B
Bart Van Assche 已提交
1864 1865
		print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 8, 1,
			       iu->buf, wc->byte_len, true);
1866 1867 1868 1869
	}

	switch (opcode) {
	case SRP_RSP:
1870
		srp_process_rsp(ch, iu->buf);
1871 1872
		break;

1873
	case SRP_CRED_REQ:
1874
		srp_process_cred_req(ch, iu->buf);
1875 1876 1877
		break;

	case SRP_AER_REQ:
1878
		srp_process_aer_req(ch, iu->buf);
1879 1880
		break;

1881 1882
	case SRP_T_LOGOUT:
		/* XXX Handle target logout */
1883 1884
		shost_printk(KERN_WARNING, target->scsi_host,
			     PFX "Got target logout request\n");
1885 1886 1887
		break;

	default:
1888 1889
		shost_printk(KERN_WARNING, target->scsi_host,
			     PFX "Unhandled SRP opcode 0x%02x\n", opcode);
1890 1891 1892
		break;
	}

1893
	ib_dma_sync_single_for_device(dev, iu->dma, ch->max_ti_iu_len,
1894
				      DMA_FROM_DEVICE);
1895

1896
	res = srp_post_recv(ch, iu);
1897 1898 1899
	if (res != 0)
		shost_printk(KERN_ERR, target->scsi_host,
			     PFX "Recv failed with error code %d\n", res);
1900 1901
}

1902 1903
/**
 * srp_tl_err_work() - handle a transport layer error
1904
 * @work: Work structure embedded in an SRP target port.
1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917
 *
 * Note: This function may get invoked before the rport has been created,
 * hence the target->rport test.
 */
static void srp_tl_err_work(struct work_struct *work)
{
	struct srp_target_port *target;

	target = container_of(work, struct srp_target_port, tl_err_work);
	if (target->rport)
		srp_start_tl_fail_timers(target->rport);
}

1918
static void srp_handle_qp_err(u64 wr_id, enum ib_wc_status wc_status,
1919
			      bool send_err, struct srp_rdma_ch *ch)
1920
{
1921 1922 1923 1924 1925 1926 1927
	struct srp_target_port *target = ch->target;

	if (wr_id == SRP_LAST_WR_ID) {
		complete(&ch->done);
		return;
	}

1928
	if (ch->connected && !target->qp_in_error) {
1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942
		if (wr_id & LOCAL_INV_WR_ID_MASK) {
			shost_printk(KERN_ERR, target->scsi_host, PFX
				     "LOCAL_INV failed with status %d\n",
				     wc_status);
		} else if (wr_id & FAST_REG_WR_ID_MASK) {
			shost_printk(KERN_ERR, target->scsi_host, PFX
				     "FAST_REG_MR failed status %d\n",
				     wc_status);
		} else {
			shost_printk(KERN_ERR, target->scsi_host,
				     PFX "failed %s status %d for iu %p\n",
				     send_err ? "send" : "receive",
				     wc_status, (void *)(uintptr_t)wr_id);
		}
1943
		queue_work(system_long_wq, &target->tl_err_work);
1944
	}
1945 1946 1947
	target->qp_in_error = true;
}

1948
static void srp_recv_completion(struct ib_cq *cq, void *ch_ptr)
1949
{
1950
	struct srp_rdma_ch *ch = ch_ptr;
1951 1952 1953 1954
	struct ib_wc wc;

	ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
	while (ib_poll_cq(cq, 1, &wc) > 0) {
1955
		if (likely(wc.status == IB_WC_SUCCESS)) {
1956
			srp_handle_recv(ch, &wc);
1957
		} else {
1958
			srp_handle_qp_err(wc.wr_id, wc.status, false, ch);
1959
		}
1960 1961 1962
	}
}

1963
static void srp_send_completion(struct ib_cq *cq, void *ch_ptr)
1964
{
1965
	struct srp_rdma_ch *ch = ch_ptr;
1966
	struct ib_wc wc;
1967
	struct srp_iu *iu;
1968 1969

	while (ib_poll_cq(cq, 1, &wc) > 0) {
1970 1971
		if (likely(wc.status == IB_WC_SUCCESS)) {
			iu = (struct srp_iu *) (uintptr_t) wc.wr_id;
1972
			list_add(&iu->list, &ch->free_tx);
1973
		} else {
1974
			srp_handle_qp_err(wc.wr_id, wc.status, true, ch);
1975
		}
1976 1977 1978
	}
}

1979
static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
1980
{
1981
	struct srp_target_port *target = host_to_target(shost);
1982
	struct srp_rport *rport = target->rport;
1983
	struct srp_rdma_ch *ch;
1984 1985 1986
	struct srp_request *req;
	struct srp_iu *iu;
	struct srp_cmd *cmd;
1987
	struct ib_device *dev;
1988
	unsigned long flags;
B
Bart Van Assche 已提交
1989 1990
	u32 tag;
	u16 idx;
1991
	int len, ret;
1992 1993 1994 1995 1996 1997 1998 1999 2000 2001
	const bool in_scsi_eh = !in_interrupt() && current == shost->ehandler;

	/*
	 * The SCSI EH thread is the only context from which srp_queuecommand()
	 * can get invoked for blocked devices (SDEV_BLOCK /
	 * SDEV_CREATED_BLOCK). Avoid racing with srp_reconnect_rport() by
	 * locking the rport mutex if invoked from inside the SCSI EH.
	 */
	if (in_scsi_eh)
		mutex_lock(&rport->mutex);
2002

2003 2004 2005
	scmnd->result = srp_chkready(target->rport);
	if (unlikely(scmnd->result))
		goto err;
2006

B
Bart Van Assche 已提交
2007 2008
	WARN_ON_ONCE(scmnd->request->tag < 0);
	tag = blk_mq_unique_tag(scmnd->request);
B
Bart Van Assche 已提交
2009
	ch = &target->ch[blk_mq_unique_tag_to_hwq(tag)];
B
Bart Van Assche 已提交
2010 2011 2012 2013
	idx = blk_mq_unique_tag_to_tag(tag);
	WARN_ONCE(idx >= target->req_ring_size, "%s: tag %#x: idx %d >= %d\n",
		  dev_name(&shost->shost_gendev), tag, idx,
		  target->req_ring_size);
2014 2015 2016 2017

	spin_lock_irqsave(&ch->lock, flags);
	iu = __srp_get_tx_iu(ch, SRP_IU_CMD);
	spin_unlock_irqrestore(&ch->lock, flags);
2018

B
Bart Van Assche 已提交
2019 2020 2021 2022
	if (!iu)
		goto err;

	req = &ch->req_ring[idx];
2023
	dev = target->srp_host->srp_dev->dev;
2024
	ib_dma_sync_single_for_cpu(dev, iu->dma, target->max_iu_len,
2025
				   DMA_TO_DEVICE);
2026

2027
	scmnd->host_scribble = (void *) req;
2028 2029 2030 2031 2032 2033

	cmd = iu->buf;
	memset(cmd, 0, sizeof *cmd);

	cmd->opcode = SRP_CMD;
	cmd->lun    = cpu_to_be64((u64) scmnd->device->lun << 48);
B
Bart Van Assche 已提交
2034
	cmd->tag    = tag;
2035 2036 2037 2038 2039
	memcpy(cmd->cdb, scmnd->cmnd, scmnd->cmd_len);

	req->scmnd    = scmnd;
	req->cmd      = iu;

2040
	len = srp_map_data(scmnd, ch, req);
2041
	if (len < 0) {
2042
		shost_printk(KERN_ERR, target->scsi_host,
2043 2044 2045 2046
			     PFX "Failed to map data (%d)\n", len);
		/*
		 * If we ran out of memory descriptors (-ENOMEM) because an
		 * application is queuing many requests with more than
2047
		 * max_pages_per_mr sg-list elements, tell the SCSI mid-layer
2048 2049 2050 2051
		 * to reduce queue depth temporarily.
		 */
		scmnd->result = len == -ENOMEM ?
			DID_OK << 16 | QUEUE_FULL << 1 : DID_ERROR << 16;
2052
		goto err_iu;
2053 2054
	}

2055
	ib_dma_sync_single_for_device(dev, iu->dma, target->max_iu_len,
2056
				      DMA_TO_DEVICE);
2057

2058
	if (srp_post_send(ch, iu, len)) {
2059
		shost_printk(KERN_ERR, target->scsi_host, PFX "Send failed\n");
2060 2061 2062
		goto err_unmap;
	}

2063 2064
	ret = 0;

2065 2066 2067 2068
unlock_rport:
	if (in_scsi_eh)
		mutex_unlock(&rport->mutex);

2069
	return ret;
2070 2071

err_unmap:
2072
	srp_unmap_data(scmnd, ch, req);
2073

2074
err_iu:
2075
	srp_put_tx_iu(ch, iu, SRP_IU_CMD);
2076

2077 2078 2079 2080 2081 2082
	/*
	 * Avoid that the loops that iterate over the request ring can
	 * encounter a dangling SCSI command pointer.
	 */
	req->scmnd = NULL;

2083 2084 2085 2086 2087 2088 2089
err:
	if (scmnd->result) {
		scmnd->scsi_done(scmnd);
		ret = 0;
	} else {
		ret = SCSI_MLQUEUE_HOST_BUSY;
	}
2090

2091
	goto unlock_rport;
2092 2093
}

2094 2095
/*
 * Note: the resources allocated in this function are freed in
2096
 * srp_free_ch_ib().
2097
 */
2098
static int srp_alloc_iu_bufs(struct srp_rdma_ch *ch)
2099
{
2100
	struct srp_target_port *target = ch->target;
2101 2102
	int i;

2103 2104 2105
	ch->rx_ring = kcalloc(target->queue_size, sizeof(*ch->rx_ring),
			      GFP_KERNEL);
	if (!ch->rx_ring)
2106
		goto err_no_ring;
2107 2108 2109
	ch->tx_ring = kcalloc(target->queue_size, sizeof(*ch->tx_ring),
			      GFP_KERNEL);
	if (!ch->tx_ring)
2110 2111 2112
		goto err_no_ring;

	for (i = 0; i < target->queue_size; ++i) {
2113 2114 2115 2116
		ch->rx_ring[i] = srp_alloc_iu(target->srp_host,
					      ch->max_ti_iu_len,
					      GFP_KERNEL, DMA_FROM_DEVICE);
		if (!ch->rx_ring[i])
2117 2118 2119
			goto err;
	}

2120
	for (i = 0; i < target->queue_size; ++i) {
2121 2122 2123 2124
		ch->tx_ring[i] = srp_alloc_iu(target->srp_host,
					      target->max_iu_len,
					      GFP_KERNEL, DMA_TO_DEVICE);
		if (!ch->tx_ring[i])
2125
			goto err;
2126

2127
		list_add(&ch->tx_ring[i]->list, &ch->free_tx);
2128 2129 2130 2131 2132
	}

	return 0;

err:
2133
	for (i = 0; i < target->queue_size; ++i) {
2134 2135
		srp_free_iu(target->srp_host, ch->rx_ring[i]);
		srp_free_iu(target->srp_host, ch->tx_ring[i]);
2136 2137
	}

2138 2139

err_no_ring:
2140 2141 2142 2143
	kfree(ch->tx_ring);
	ch->tx_ring = NULL;
	kfree(ch->rx_ring);
	ch->rx_ring = NULL;
2144

2145 2146 2147
	return -ENOMEM;
}

2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174
static uint32_t srp_compute_rq_tmo(struct ib_qp_attr *qp_attr, int attr_mask)
{
	uint64_t T_tr_ns, max_compl_time_ms;
	uint32_t rq_tmo_jiffies;

	/*
	 * According to section 11.2.4.2 in the IBTA spec (Modify Queue Pair,
	 * table 91), both the QP timeout and the retry count have to be set
	 * for RC QP's during the RTR to RTS transition.
	 */
	WARN_ON_ONCE((attr_mask & (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)) !=
		     (IB_QP_TIMEOUT | IB_QP_RETRY_CNT));

	/*
	 * Set target->rq_tmo_jiffies to one second more than the largest time
	 * it can take before an error completion is generated. See also
	 * C9-140..142 in the IBTA spec for more information about how to
	 * convert the QP Local ACK Timeout value to nanoseconds.
	 */
	T_tr_ns = 4096 * (1ULL << qp_attr->timeout);
	max_compl_time_ms = qp_attr->retry_cnt * 4 * T_tr_ns;
	do_div(max_compl_time_ms, NSEC_PER_MSEC);
	rq_tmo_jiffies = msecs_to_jiffies(max_compl_time_ms + 1000);

	return rq_tmo_jiffies;
}

2175 2176
static void srp_cm_rep_handler(struct ib_cm_id *cm_id,
			       struct srp_login_rsp *lrsp,
2177
			       struct srp_rdma_ch *ch)
2178
{
2179
	struct srp_target_port *target = ch->target;
2180 2181 2182 2183 2184 2185
	struct ib_qp_attr *qp_attr = NULL;
	int attr_mask = 0;
	int ret;
	int i;

	if (lrsp->opcode == SRP_LOGIN_RSP) {
2186 2187
		ch->max_ti_iu_len = be32_to_cpu(lrsp->max_ti_iu_len);
		ch->req_lim       = be32_to_cpu(lrsp->req_lim_delta);
2188 2189 2190 2191 2192 2193

		/*
		 * Reserve credits for task management so we don't
		 * bounce requests back to the SCSI mid-layer.
		 */
		target->scsi_host->can_queue
2194
			= min(ch->req_lim - SRP_TSK_MGMT_SQ_SIZE,
2195
			      target->scsi_host->can_queue);
2196 2197 2198
		target->scsi_host->cmd_per_lun
			= min_t(int, target->scsi_host->can_queue,
				target->scsi_host->cmd_per_lun);
2199 2200 2201 2202 2203 2204 2205
	} else {
		shost_printk(KERN_WARNING, target->scsi_host,
			     PFX "Unhandled RSP opcode %#x\n", lrsp->opcode);
		ret = -ECONNRESET;
		goto error;
	}

2206 2207
	if (!ch->rx_ring) {
		ret = srp_alloc_iu_bufs(ch);
2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221
		if (ret)
			goto error;
	}

	ret = -ENOMEM;
	qp_attr = kmalloc(sizeof *qp_attr, GFP_KERNEL);
	if (!qp_attr)
		goto error;

	qp_attr->qp_state = IB_QPS_RTR;
	ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
	if (ret)
		goto error_free;

2222
	ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
2223 2224 2225
	if (ret)
		goto error_free;

2226
	for (i = 0; i < target->queue_size; i++) {
2227 2228 2229
		struct srp_iu *iu = ch->rx_ring[i];

		ret = srp_post_recv(ch, iu);
2230 2231 2232 2233 2234 2235 2236 2237 2238
		if (ret)
			goto error_free;
	}

	qp_attr->qp_state = IB_QPS_RTS;
	ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
	if (ret)
		goto error_free;

2239 2240
	target->rq_tmo_jiffies = srp_compute_rq_tmo(qp_attr, attr_mask);

2241
	ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
2242 2243 2244 2245 2246 2247 2248 2249 2250
	if (ret)
		goto error_free;

	ret = ib_send_cm_rtu(cm_id, NULL, 0);

error_free:
	kfree(qp_attr);

error:
2251
	ch->status = ret;
2252 2253
}

2254 2255
static void srp_cm_rej_handler(struct ib_cm_id *cm_id,
			       struct ib_cm_event *event,
2256
			       struct srp_rdma_ch *ch)
2257
{
2258
	struct srp_target_port *target = ch->target;
2259
	struct Scsi_Host *shost = target->scsi_host;
2260 2261 2262 2263 2264 2265
	struct ib_class_port_info *cpi;
	int opcode;

	switch (event->param.rej_rcvd.reason) {
	case IB_CM_REJ_PORT_CM_REDIRECT:
		cpi = event->param.rej_rcvd.ari;
2266 2267
		ch->path.dlid = cpi->redirect_lid;
		ch->path.pkey = cpi->redirect_pkey;
2268
		cm_id->remote_cm_qpn = be32_to_cpu(cpi->redirect_qp) & 0x00ffffff;
2269
		memcpy(ch->path.dgid.raw, cpi->redirect_gid, 16);
2270

2271
		ch->status = ch->path.dlid ?
2272 2273 2274 2275
			SRP_DLID_REDIRECT : SRP_PORT_REDIRECT;
		break;

	case IB_CM_REJ_PORT_REDIRECT:
2276
		if (srp_target_is_topspin(target)) {
2277 2278 2279 2280 2281
			/*
			 * Topspin/Cisco SRP gateways incorrectly send
			 * reject reason code 25 when they mean 24
			 * (port redirect).
			 */
2282
			memcpy(ch->path.dgid.raw,
2283 2284
			       event->param.rej_rcvd.ari, 16);

2285 2286
			shost_printk(KERN_DEBUG, shost,
				     PFX "Topspin/Cisco redirect to target port GID %016llx%016llx\n",
2287 2288
				     be64_to_cpu(ch->path.dgid.global.subnet_prefix),
				     be64_to_cpu(ch->path.dgid.global.interface_id));
2289

2290
			ch->status = SRP_PORT_REDIRECT;
2291
		} else {
2292 2293
			shost_printk(KERN_WARNING, shost,
				     "  REJ reason: IB_CM_REJ_PORT_REDIRECT\n");
2294
			ch->status = -ECONNRESET;
2295 2296 2297 2298
		}
		break;

	case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID:
2299 2300
		shost_printk(KERN_WARNING, shost,
			    "  REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n");
2301
		ch->status = -ECONNRESET;
2302 2303 2304 2305 2306 2307 2308 2309 2310
		break;

	case IB_CM_REJ_CONSUMER_DEFINED:
		opcode = *(u8 *) event->private_data;
		if (opcode == SRP_LOGIN_REJ) {
			struct srp_login_rej *rej = event->private_data;
			u32 reason = be32_to_cpu(rej->reason);

			if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE)
2311 2312
				shost_printk(KERN_WARNING, shost,
					     PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n");
2313
			else
B
Bart Van Assche 已提交
2314 2315
				shost_printk(KERN_WARNING, shost, PFX
					     "SRP LOGIN from %pI6 to %pI6 REJECTED, reason 0x%08x\n",
2316 2317
					     target->sgid.raw,
					     target->orig_dgid.raw, reason);
2318
		} else
2319 2320 2321
			shost_printk(KERN_WARNING, shost,
				     "  REJ reason: IB_CM_REJ_CONSUMER_DEFINED,"
				     " opcode 0x%02x\n", opcode);
2322
		ch->status = -ECONNRESET;
2323 2324
		break;

D
David Dillow 已提交
2325 2326
	case IB_CM_REJ_STALE_CONN:
		shost_printk(KERN_WARNING, shost, "  REJ reason: stale connection\n");
2327
		ch->status = SRP_STALE_CONN;
D
David Dillow 已提交
2328 2329
		break;

2330
	default:
2331 2332
		shost_printk(KERN_WARNING, shost, "  REJ reason 0x%x\n",
			     event->param.rej_rcvd.reason);
2333
		ch->status = -ECONNRESET;
2334 2335 2336 2337 2338
	}
}

static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
{
2339 2340
	struct srp_rdma_ch *ch = cm_id->context;
	struct srp_target_port *target = ch->target;
2341 2342 2343 2344
	int comp = 0;

	switch (event->event) {
	case IB_CM_REQ_ERROR:
2345 2346
		shost_printk(KERN_DEBUG, target->scsi_host,
			     PFX "Sending CM REQ failed\n");
2347
		comp = 1;
2348
		ch->status = -ECONNRESET;
2349 2350 2351 2352
		break;

	case IB_CM_REP_RECEIVED:
		comp = 1;
2353
		srp_cm_rep_handler(cm_id, event->private_data, ch);
2354 2355 2356
		break;

	case IB_CM_REJ_RECEIVED:
2357
		shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n");
2358 2359
		comp = 1;

2360
		srp_cm_rej_handler(cm_id, event, ch);
2361 2362
		break;

2363
	case IB_CM_DREQ_RECEIVED:
2364 2365
		shost_printk(KERN_WARNING, target->scsi_host,
			     PFX "DREQ received - connection closed\n");
2366
		ch->connected = false;
2367
		if (ib_send_cm_drep(cm_id, NULL, 0))
2368 2369
			shost_printk(KERN_ERR, target->scsi_host,
				     PFX "Sending CM DREP failed\n");
2370
		queue_work(system_long_wq, &target->tl_err_work);
2371 2372 2373
		break;

	case IB_CM_TIMEWAIT_EXIT:
2374 2375
		shost_printk(KERN_ERR, target->scsi_host,
			     PFX "connection closed\n");
2376
		comp = 1;
2377

2378
		ch->status = 0;
2379 2380
		break;

2381 2382 2383 2384 2385
	case IB_CM_MRA_RECEIVED:
	case IB_CM_DREQ_ERROR:
	case IB_CM_DREP_RECEIVED:
		break;

2386
	default:
2387 2388
		shost_printk(KERN_WARNING, target->scsi_host,
			     PFX "Unhandled CM event %d\n", event->event);
2389 2390 2391 2392
		break;
	}

	if (comp)
2393
		complete(&ch->done);
2394 2395 2396 2397

	return 0;
}

2398 2399 2400 2401 2402 2403 2404 2405
/**
 * srp_change_queue_depth - setting device queue depth
 * @sdev: scsi device struct
 * @qdepth: requested queue depth
 *
 * Returns queue depth.
 */
static int
2406
srp_change_queue_depth(struct scsi_device *sdev, int qdepth)
2407
{
2408
	if (!sdev->tagged_supported)
2409
		qdepth = 1;
2410
	return scsi_change_queue_depth(sdev, qdepth);
2411 2412
}

2413 2414
static int srp_send_tsk_mgmt(struct srp_rdma_ch *ch, u64 req_tag,
			     unsigned int lun, u8 func)
2415
{
2416
	struct srp_target_port *target = ch->target;
2417
	struct srp_rport *rport = target->rport;
2418
	struct ib_device *dev = target->srp_host->srp_dev->dev;
2419 2420 2421
	struct srp_iu *iu;
	struct srp_tsk_mgmt *tsk_mgmt;

2422
	if (!ch->connected || target->qp_in_error)
2423 2424
		return -1;

2425
	init_completion(&ch->tsk_mgmt_done);
2426

2427
	/*
2428
	 * Lock the rport mutex to avoid that srp_create_ch_ib() is
2429 2430 2431
	 * invoked while a task management function is being sent.
	 */
	mutex_lock(&rport->mutex);
2432 2433 2434
	spin_lock_irq(&ch->lock);
	iu = __srp_get_tx_iu(ch, SRP_IU_TSK_MGMT);
	spin_unlock_irq(&ch->lock);
2435

2436 2437 2438
	if (!iu) {
		mutex_unlock(&rport->mutex);

2439
		return -1;
2440
	}
2441

2442 2443
	ib_dma_sync_single_for_cpu(dev, iu->dma, sizeof *tsk_mgmt,
				   DMA_TO_DEVICE);
2444 2445 2446 2447
	tsk_mgmt = iu->buf;
	memset(tsk_mgmt, 0, sizeof *tsk_mgmt);

	tsk_mgmt->opcode 	= SRP_TSK_MGMT;
2448 2449
	tsk_mgmt->lun		= cpu_to_be64((u64) lun << 48);
	tsk_mgmt->tag		= req_tag | SRP_TAG_TSK_MGMT;
2450
	tsk_mgmt->tsk_mgmt_func = func;
2451
	tsk_mgmt->task_tag	= req_tag;
2452

2453 2454
	ib_dma_sync_single_for_device(dev, iu->dma, sizeof *tsk_mgmt,
				      DMA_TO_DEVICE);
2455 2456
	if (srp_post_send(ch, iu, sizeof(*tsk_mgmt))) {
		srp_put_tx_iu(ch, iu, SRP_IU_TSK_MGMT);
2457 2458
		mutex_unlock(&rport->mutex);

2459 2460
		return -1;
	}
2461
	mutex_unlock(&rport->mutex);
2462

2463
	if (!wait_for_completion_timeout(&ch->tsk_mgmt_done,
2464
					 msecs_to_jiffies(SRP_ABORT_TIMEOUT_MS)))
2465
		return -1;
2466

2467 2468 2469
	return 0;
}

2470 2471
static int srp_abort(struct scsi_cmnd *scmnd)
{
2472
	struct srp_target_port *target = host_to_target(scmnd->device->host);
2473
	struct srp_request *req = (struct srp_request *) scmnd->host_scribble;
B
Bart Van Assche 已提交
2474
	u32 tag;
B
Bart Van Assche 已提交
2475
	u16 ch_idx;
2476
	struct srp_rdma_ch *ch;
2477
	int ret;
2478

2479
	shost_printk(KERN_ERR, target->scsi_host, "SRP abort called\n");
2480

B
Bart Van Assche 已提交
2481
	if (!req)
2482
		return SUCCESS;
B
Bart Van Assche 已提交
2483
	tag = blk_mq_unique_tag(scmnd->request);
B
Bart Van Assche 已提交
2484 2485 2486 2487 2488 2489 2490 2491
	ch_idx = blk_mq_unique_tag_to_hwq(tag);
	if (WARN_ON_ONCE(ch_idx >= target->ch_count))
		return SUCCESS;
	ch = &target->ch[ch_idx];
	if (!srp_claim_req(ch, req, NULL, scmnd))
		return SUCCESS;
	shost_printk(KERN_ERR, target->scsi_host,
		     "Sending SRP abort for tag %#x\n", tag);
B
Bart Van Assche 已提交
2492
	if (srp_send_tsk_mgmt(ch, tag, scmnd->device->lun,
2493
			      SRP_TSK_ABORT_TASK) == 0)
2494
		ret = SUCCESS;
2495
	else if (target->rport->state == SRP_RPORT_LOST)
2496
		ret = FAST_IO_FAIL;
2497 2498
	else
		ret = FAILED;
2499
	srp_free_req(ch, req, scmnd, 0);
B
Bart Van Assche 已提交
2500
	scmnd->result = DID_ABORT << 16;
2501
	scmnd->scsi_done(scmnd);
2502

2503
	return ret;
2504 2505 2506 2507
}

static int srp_reset_device(struct scsi_cmnd *scmnd)
{
2508
	struct srp_target_port *target = host_to_target(scmnd->device->host);
B
Bart Van Assche 已提交
2509
	struct srp_rdma_ch *ch;
2510
	int i;
2511

2512
	shost_printk(KERN_ERR, target->scsi_host, "SRP reset_device called\n");
2513

B
Bart Van Assche 已提交
2514
	ch = &target->ch[0];
2515
	if (srp_send_tsk_mgmt(ch, SRP_TAG_NO_REQ, scmnd->device->lun,
2516
			      SRP_TSK_LUN_RESET))
2517
		return FAILED;
2518
	if (ch->tsk_mgmt_status)
2519 2520
		return FAILED;

B
Bart Van Assche 已提交
2521 2522 2523 2524
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
		for (i = 0; i < target->req_ring_size; ++i) {
			struct srp_request *req = &ch->req_ring[i];
2525

B
Bart Van Assche 已提交
2526 2527
			srp_finish_req(ch, req, scmnd->device, DID_RESET << 16);
		}
2528
	}
2529 2530

	return SUCCESS;
2531 2532 2533 2534 2535 2536
}

static int srp_reset_host(struct scsi_cmnd *scmnd)
{
	struct srp_target_port *target = host_to_target(scmnd->device->host);

2537
	shost_printk(KERN_ERR, target->scsi_host, PFX "SRP reset_host called\n");
2538

2539
	return srp_reconnect_rport(target->rport) == 0 ? SUCCESS : FAILED;
2540 2541
}

2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556
static int srp_slave_configure(struct scsi_device *sdev)
{
	struct Scsi_Host *shost = sdev->host;
	struct srp_target_port *target = host_to_target(shost);
	struct request_queue *q = sdev->request_queue;
	unsigned long timeout;

	if (sdev->type == TYPE_DISK) {
		timeout = max_t(unsigned, 30 * HZ, target->rq_tmo_jiffies);
		blk_queue_rq_timeout(q, timeout);
	}

	return 0;
}

2557 2558
static ssize_t show_id_ext(struct device *dev, struct device_attribute *attr,
			   char *buf)
2559
{
2560
	struct srp_target_port *target = host_to_target(class_to_shost(dev));
2561

2562
	return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->id_ext));
2563 2564
}

2565 2566
static ssize_t show_ioc_guid(struct device *dev, struct device_attribute *attr,
			     char *buf)
2567
{
2568
	struct srp_target_port *target = host_to_target(class_to_shost(dev));
2569

2570
	return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->ioc_guid));
2571 2572
}

2573 2574
static ssize_t show_service_id(struct device *dev,
			       struct device_attribute *attr, char *buf)
2575
{
2576
	struct srp_target_port *target = host_to_target(class_to_shost(dev));
2577

2578
	return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->service_id));
2579 2580
}

2581 2582
static ssize_t show_pkey(struct device *dev, struct device_attribute *attr,
			 char *buf)
2583
{
2584
	struct srp_target_port *target = host_to_target(class_to_shost(dev));
2585

2586
	return sprintf(buf, "0x%04x\n", be16_to_cpu(target->pkey));
2587 2588
}

B
Bart Van Assche 已提交
2589 2590 2591 2592 2593
static ssize_t show_sgid(struct device *dev, struct device_attribute *attr,
			 char *buf)
{
	struct srp_target_port *target = host_to_target(class_to_shost(dev));

2594
	return sprintf(buf, "%pI6\n", target->sgid.raw);
B
Bart Van Assche 已提交
2595 2596
}

2597 2598
static ssize_t show_dgid(struct device *dev, struct device_attribute *attr,
			 char *buf)
2599
{
2600
	struct srp_target_port *target = host_to_target(class_to_shost(dev));
B
Bart Van Assche 已提交
2601
	struct srp_rdma_ch *ch = &target->ch[0];
2602

2603
	return sprintf(buf, "%pI6\n", ch->path.dgid.raw);
2604 2605
}

2606 2607
static ssize_t show_orig_dgid(struct device *dev,
			      struct device_attribute *attr, char *buf)
2608
{
2609
	struct srp_target_port *target = host_to_target(class_to_shost(dev));
2610

2611
	return sprintf(buf, "%pI6\n", target->orig_dgid.raw);
2612 2613
}

2614 2615 2616 2617
static ssize_t show_req_lim(struct device *dev,
			    struct device_attribute *attr, char *buf)
{
	struct srp_target_port *target = host_to_target(class_to_shost(dev));
B
Bart Van Assche 已提交
2618 2619
	struct srp_rdma_ch *ch;
	int i, req_lim = INT_MAX;
2620

B
Bart Van Assche 已提交
2621 2622 2623 2624 2625
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
		req_lim = min(req_lim, ch->req_lim);
	}
	return sprintf(buf, "%d\n", req_lim);
2626 2627
}

2628 2629
static ssize_t show_zero_req_lim(struct device *dev,
				 struct device_attribute *attr, char *buf)
2630
{
2631
	struct srp_target_port *target = host_to_target(class_to_shost(dev));
2632 2633 2634 2635

	return sprintf(buf, "%d\n", target->zero_req_lim);
}

2636 2637
static ssize_t show_local_ib_port(struct device *dev,
				  struct device_attribute *attr, char *buf)
2638
{
2639
	struct srp_target_port *target = host_to_target(class_to_shost(dev));
2640 2641 2642 2643

	return sprintf(buf, "%d\n", target->srp_host->port);
}

2644 2645
static ssize_t show_local_ib_device(struct device *dev,
				    struct device_attribute *attr, char *buf)
2646
{
2647
	struct srp_target_port *target = host_to_target(class_to_shost(dev));
2648

2649
	return sprintf(buf, "%s\n", target->srp_host->srp_dev->dev->name);
2650 2651
}

B
Bart Van Assche 已提交
2652 2653 2654 2655 2656 2657 2658 2659
static ssize_t show_ch_count(struct device *dev, struct device_attribute *attr,
			     char *buf)
{
	struct srp_target_port *target = host_to_target(class_to_shost(dev));

	return sprintf(buf, "%d\n", target->ch_count);
}

2660 2661 2662 2663 2664 2665 2666 2667
static ssize_t show_comp_vector(struct device *dev,
				struct device_attribute *attr, char *buf)
{
	struct srp_target_port *target = host_to_target(class_to_shost(dev));

	return sprintf(buf, "%d\n", target->comp_vector);
}

2668 2669 2670 2671 2672 2673 2674 2675
static ssize_t show_tl_retry_count(struct device *dev,
				   struct device_attribute *attr, char *buf)
{
	struct srp_target_port *target = host_to_target(class_to_shost(dev));

	return sprintf(buf, "%d\n", target->tl_retry_count);
}

2676 2677 2678 2679 2680 2681 2682 2683
static ssize_t show_cmd_sg_entries(struct device *dev,
				   struct device_attribute *attr, char *buf)
{
	struct srp_target_port *target = host_to_target(class_to_shost(dev));

	return sprintf(buf, "%u\n", target->cmd_sg_cnt);
}

2684 2685 2686 2687 2688 2689 2690 2691
static ssize_t show_allow_ext_sg(struct device *dev,
				 struct device_attribute *attr, char *buf)
{
	struct srp_target_port *target = host_to_target(class_to_shost(dev));

	return sprintf(buf, "%s\n", target->allow_ext_sg ? "true" : "false");
}

2692 2693 2694 2695
static DEVICE_ATTR(id_ext,	    S_IRUGO, show_id_ext,	   NULL);
static DEVICE_ATTR(ioc_guid,	    S_IRUGO, show_ioc_guid,	   NULL);
static DEVICE_ATTR(service_id,	    S_IRUGO, show_service_id,	   NULL);
static DEVICE_ATTR(pkey,	    S_IRUGO, show_pkey,		   NULL);
B
Bart Van Assche 已提交
2696
static DEVICE_ATTR(sgid,	    S_IRUGO, show_sgid,		   NULL);
2697 2698
static DEVICE_ATTR(dgid,	    S_IRUGO, show_dgid,		   NULL);
static DEVICE_ATTR(orig_dgid,	    S_IRUGO, show_orig_dgid,	   NULL);
2699
static DEVICE_ATTR(req_lim,         S_IRUGO, show_req_lim,         NULL);
2700 2701 2702
static DEVICE_ATTR(zero_req_lim,    S_IRUGO, show_zero_req_lim,	   NULL);
static DEVICE_ATTR(local_ib_port,   S_IRUGO, show_local_ib_port,   NULL);
static DEVICE_ATTR(local_ib_device, S_IRUGO, show_local_ib_device, NULL);
B
Bart Van Assche 已提交
2703
static DEVICE_ATTR(ch_count,        S_IRUGO, show_ch_count,        NULL);
2704
static DEVICE_ATTR(comp_vector,     S_IRUGO, show_comp_vector,     NULL);
2705
static DEVICE_ATTR(tl_retry_count,  S_IRUGO, show_tl_retry_count,  NULL);
2706
static DEVICE_ATTR(cmd_sg_entries,  S_IRUGO, show_cmd_sg_entries,  NULL);
2707
static DEVICE_ATTR(allow_ext_sg,    S_IRUGO, show_allow_ext_sg,    NULL);
2708 2709 2710 2711 2712 2713

static struct device_attribute *srp_host_attrs[] = {
	&dev_attr_id_ext,
	&dev_attr_ioc_guid,
	&dev_attr_service_id,
	&dev_attr_pkey,
B
Bart Van Assche 已提交
2714
	&dev_attr_sgid,
2715 2716
	&dev_attr_dgid,
	&dev_attr_orig_dgid,
2717
	&dev_attr_req_lim,
2718 2719 2720
	&dev_attr_zero_req_lim,
	&dev_attr_local_ib_port,
	&dev_attr_local_ib_device,
B
Bart Van Assche 已提交
2721
	&dev_attr_ch_count,
2722
	&dev_attr_comp_vector,
2723
	&dev_attr_tl_retry_count,
2724
	&dev_attr_cmd_sg_entries,
2725
	&dev_attr_allow_ext_sg,
2726 2727 2728
	NULL
};

2729 2730
static struct scsi_host_template srp_template = {
	.module				= THIS_MODULE,
R
Roland Dreier 已提交
2731 2732
	.name				= "InfiniBand SRP initiator",
	.proc_name			= DRV_NAME,
2733
	.slave_configure		= srp_slave_configure,
2734 2735
	.info				= srp_target_info,
	.queuecommand			= srp_queuecommand,
2736
	.change_queue_depth             = srp_change_queue_depth,
2737 2738 2739
	.eh_abort_handler		= srp_abort,
	.eh_device_reset_handler	= srp_reset_device,
	.eh_host_reset_handler		= srp_reset_host,
B
Bart Van Assche 已提交
2740
	.skip_settle_delay		= true,
2741
	.sg_tablesize			= SRP_DEF_SG_TABLESIZE,
2742
	.can_queue			= SRP_DEFAULT_CMD_SQ_SIZE,
2743
	.this_id			= -1,
2744
	.cmd_per_lun			= SRP_DEFAULT_CMD_SQ_SIZE,
2745
	.use_clustering			= ENABLE_CLUSTERING,
B
Bart Van Assche 已提交
2746 2747
	.shost_attrs			= srp_host_attrs,
	.use_blk_tags			= 1,
2748
	.track_queue_depth		= 1,
2749 2750
};

2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761
static int srp_sdev_count(struct Scsi_Host *host)
{
	struct scsi_device *sdev;
	int c = 0;

	shost_for_each_device(sdev, host)
		c++;

	return c;
}

2762 2763
static int srp_add_target(struct srp_host *host, struct srp_target_port *target)
{
2764 2765 2766
	struct srp_rport_identifiers ids;
	struct srp_rport *rport;

2767
	target->state = SRP_TARGET_SCANNING;
2768
	sprintf(target->target_name, "SRP.T10:%016llX",
2769
		be64_to_cpu(target->id_ext));
2770

2771
	if (scsi_add_host(target->scsi_host, host->srp_dev->dev->dma_device))
2772 2773
		return -ENODEV;

2774 2775
	memcpy(ids.port_id, &target->id_ext, 8);
	memcpy(ids.port_id + 8, &target->ioc_guid, 8);
2776
	ids.roles = SRP_RPORT_ROLE_TARGET;
2777 2778 2779 2780 2781 2782
	rport = srp_rport_add(target->scsi_host, &ids);
	if (IS_ERR(rport)) {
		scsi_remove_host(target->scsi_host);
		return PTR_ERR(rport);
	}

2783
	rport->lld_data = target;
2784
	target->rport = rport;
2785

2786
	spin_lock(&host->target_lock);
2787
	list_add_tail(&target->list, &host->target_list);
2788
	spin_unlock(&host->target_lock);
2789 2790

	scsi_scan_target(&target->scsi_host->shost_gendev,
2791
			 0, target->scsi_id, SCAN_WILD_CARD, 0);
2792

2793 2794
	if (srp_connected_ch(target) < target->ch_count ||
	    target->qp_in_error) {
2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810
		shost_printk(KERN_INFO, target->scsi_host,
			     PFX "SCSI scan failed - removing SCSI host\n");
		srp_queue_remove_work(target);
		goto out;
	}

	pr_debug(PFX "%s: SCSI scan succeeded - detected %d LUNs\n",
		 dev_name(&target->scsi_host->shost_gendev),
		 srp_sdev_count(target->scsi_host));

	spin_lock_irq(&target->lock);
	if (target->state == SRP_TARGET_SCANNING)
		target->state = SRP_TARGET_LIVE;
	spin_unlock_irq(&target->lock);

out:
2811 2812 2813
	return 0;
}

2814
static void srp_release_dev(struct device *dev)
2815 2816
{
	struct srp_host *host =
2817
		container_of(dev, struct srp_host, dev);
2818 2819 2820 2821 2822 2823

	complete(&host->released);
}

static struct class srp_class = {
	.name    = "infiniband_srp",
2824
	.dev_release = srp_release_dev
2825 2826
};

2827 2828
/**
 * srp_conn_unique() - check whether the connection to a target is unique
2829 2830
 * @host:   SRP host.
 * @target: SRP target port.
2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858
 */
static bool srp_conn_unique(struct srp_host *host,
			    struct srp_target_port *target)
{
	struct srp_target_port *t;
	bool ret = false;

	if (target->state == SRP_TARGET_REMOVED)
		goto out;

	ret = true;

	spin_lock(&host->target_lock);
	list_for_each_entry(t, &host->target_list, list) {
		if (t != target &&
		    target->id_ext == t->id_ext &&
		    target->ioc_guid == t->ioc_guid &&
		    target->initiator_ext == t->initiator_ext) {
			ret = false;
			break;
		}
	}
	spin_unlock(&host->target_lock);

out:
	return ret;
}

2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874
/*
 * Target ports are added by writing
 *
 *     id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,dgid=<dest GID>,
 *     pkey=<P_Key>,service_id=<service ID>
 *
 * to the add_target sysfs attribute.
 */
enum {
	SRP_OPT_ERR		= 0,
	SRP_OPT_ID_EXT		= 1 << 0,
	SRP_OPT_IOC_GUID	= 1 << 1,
	SRP_OPT_DGID		= 1 << 2,
	SRP_OPT_PKEY		= 1 << 3,
	SRP_OPT_SERVICE_ID	= 1 << 4,
	SRP_OPT_MAX_SECT	= 1 << 5,
2875
	SRP_OPT_MAX_CMD_PER_LUN	= 1 << 6,
2876
	SRP_OPT_IO_CLASS	= 1 << 7,
2877
	SRP_OPT_INITIATOR_EXT	= 1 << 8,
2878
	SRP_OPT_CMD_SG_ENTRIES	= 1 << 9,
2879 2880
	SRP_OPT_ALLOW_EXT_SG	= 1 << 10,
	SRP_OPT_SG_TABLESIZE	= 1 << 11,
2881
	SRP_OPT_COMP_VECTOR	= 1 << 12,
2882
	SRP_OPT_TL_RETRY_COUNT	= 1 << 13,
2883
	SRP_OPT_QUEUE_SIZE	= 1 << 14,
2884 2885 2886 2887 2888 2889 2890
	SRP_OPT_ALL		= (SRP_OPT_ID_EXT	|
				   SRP_OPT_IOC_GUID	|
				   SRP_OPT_DGID		|
				   SRP_OPT_PKEY		|
				   SRP_OPT_SERVICE_ID),
};

2891
static const match_table_t srp_opt_tokens = {
2892 2893 2894 2895 2896 2897 2898
	{ SRP_OPT_ID_EXT,		"id_ext=%s" 		},
	{ SRP_OPT_IOC_GUID,		"ioc_guid=%s" 		},
	{ SRP_OPT_DGID,			"dgid=%s" 		},
	{ SRP_OPT_PKEY,			"pkey=%x" 		},
	{ SRP_OPT_SERVICE_ID,		"service_id=%s"		},
	{ SRP_OPT_MAX_SECT,		"max_sect=%d" 		},
	{ SRP_OPT_MAX_CMD_PER_LUN,	"max_cmd_per_lun=%d" 	},
2899
	{ SRP_OPT_IO_CLASS,		"io_class=%x"		},
2900
	{ SRP_OPT_INITIATOR_EXT,	"initiator_ext=%s"	},
2901
	{ SRP_OPT_CMD_SG_ENTRIES,	"cmd_sg_entries=%u"	},
2902 2903
	{ SRP_OPT_ALLOW_EXT_SG,		"allow_ext_sg=%u"	},
	{ SRP_OPT_SG_TABLESIZE,		"sg_tablesize=%u"	},
2904
	{ SRP_OPT_COMP_VECTOR,		"comp_vector=%u"	},
2905
	{ SRP_OPT_TL_RETRY_COUNT,	"tl_retry_count=%u"	},
2906
	{ SRP_OPT_QUEUE_SIZE,		"queue_size=%d"		},
2907
	{ SRP_OPT_ERR,			NULL 			}
2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925
};

static int srp_parse_options(const char *buf, struct srp_target_port *target)
{
	char *options, *sep_opt;
	char *p;
	char dgid[3];
	substring_t args[MAX_OPT_ARGS];
	int opt_mask = 0;
	int token;
	int ret = -EINVAL;
	int i;

	options = kstrdup(buf, GFP_KERNEL);
	if (!options)
		return -ENOMEM;

	sep_opt = options;
2926
	while ((p = strsep(&sep_opt, ",\n")) != NULL) {
2927 2928 2929 2930 2931 2932 2933 2934 2935
		if (!*p)
			continue;

		token = match_token(p, srp_opt_tokens, args);
		opt_mask |= token;

		switch (token) {
		case SRP_OPT_ID_EXT:
			p = match_strdup(args);
2936 2937 2938 2939
			if (!p) {
				ret = -ENOMEM;
				goto out;
			}
2940 2941 2942 2943 2944 2945
			target->id_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
			kfree(p);
			break;

		case SRP_OPT_IOC_GUID:
			p = match_strdup(args);
2946 2947 2948 2949
			if (!p) {
				ret = -ENOMEM;
				goto out;
			}
2950 2951 2952 2953 2954 2955
			target->ioc_guid = cpu_to_be64(simple_strtoull(p, NULL, 16));
			kfree(p);
			break;

		case SRP_OPT_DGID:
			p = match_strdup(args);
2956 2957 2958 2959
			if (!p) {
				ret = -ENOMEM;
				goto out;
			}
2960
			if (strlen(p) != 32) {
2961
				pr_warn("bad dest GID parameter '%s'\n", p);
2962
				kfree(p);
2963 2964 2965 2966
				goto out;
			}

			for (i = 0; i < 16; ++i) {
2967 2968 2969 2970 2971 2972 2973
				strlcpy(dgid, p + i * 2, sizeof(dgid));
				if (sscanf(dgid, "%hhx",
					   &target->orig_dgid.raw[i]) < 1) {
					ret = -EINVAL;
					kfree(p);
					goto out;
				}
2974
			}
2975
			kfree(p);
2976 2977 2978 2979
			break;

		case SRP_OPT_PKEY:
			if (match_hex(args, &token)) {
2980
				pr_warn("bad P_Key parameter '%s'\n", p);
2981 2982
				goto out;
			}
2983
			target->pkey = cpu_to_be16(token);
2984 2985 2986 2987
			break;

		case SRP_OPT_SERVICE_ID:
			p = match_strdup(args);
2988 2989 2990 2991
			if (!p) {
				ret = -ENOMEM;
				goto out;
			}
2992 2993 2994 2995 2996 2997
			target->service_id = cpu_to_be64(simple_strtoull(p, NULL, 16));
			kfree(p);
			break;

		case SRP_OPT_MAX_SECT:
			if (match_int(args, &token)) {
2998
				pr_warn("bad max sect parameter '%s'\n", p);
2999 3000 3001 3002 3003
				goto out;
			}
			target->scsi_host->max_sectors = token;
			break;

3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015
		case SRP_OPT_QUEUE_SIZE:
			if (match_int(args, &token) || token < 1) {
				pr_warn("bad queue_size parameter '%s'\n", p);
				goto out;
			}
			target->scsi_host->can_queue = token;
			target->queue_size = token + SRP_RSP_SQ_SIZE +
					     SRP_TSK_MGMT_SQ_SIZE;
			if (!(opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
				target->scsi_host->cmd_per_lun = token;
			break;

3016
		case SRP_OPT_MAX_CMD_PER_LUN:
3017
			if (match_int(args, &token) || token < 1) {
3018 3019
				pr_warn("bad max cmd_per_lun parameter '%s'\n",
					p);
3020 3021
				goto out;
			}
3022
			target->scsi_host->cmd_per_lun = token;
3023 3024
			break;

3025 3026
		case SRP_OPT_IO_CLASS:
			if (match_hex(args, &token)) {
3027
				pr_warn("bad IO class parameter '%s'\n", p);
3028 3029 3030 3031
				goto out;
			}
			if (token != SRP_REV10_IB_IO_CLASS &&
			    token != SRP_REV16A_IB_IO_CLASS) {
3032 3033 3034
				pr_warn("unknown IO class parameter value %x specified (use %x or %x).\n",
					token, SRP_REV10_IB_IO_CLASS,
					SRP_REV16A_IB_IO_CLASS);
3035 3036 3037 3038 3039
				goto out;
			}
			target->io_class = token;
			break;

3040 3041
		case SRP_OPT_INITIATOR_EXT:
			p = match_strdup(args);
3042 3043 3044 3045
			if (!p) {
				ret = -ENOMEM;
				goto out;
			}
3046 3047 3048 3049
			target->initiator_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
			kfree(p);
			break;

3050 3051
		case SRP_OPT_CMD_SG_ENTRIES:
			if (match_int(args, &token) || token < 1 || token > 255) {
3052 3053
				pr_warn("bad max cmd_sg_entries parameter '%s'\n",
					p);
3054 3055 3056 3057 3058
				goto out;
			}
			target->cmd_sg_cnt = token;
			break;

3059 3060
		case SRP_OPT_ALLOW_EXT_SG:
			if (match_int(args, &token)) {
3061
				pr_warn("bad allow_ext_sg parameter '%s'\n", p);
3062 3063 3064 3065 3066 3067 3068 3069
				goto out;
			}
			target->allow_ext_sg = !!token;
			break;

		case SRP_OPT_SG_TABLESIZE:
			if (match_int(args, &token) || token < 1 ||
					token > SCSI_MAX_SG_CHAIN_SEGMENTS) {
3070 3071
				pr_warn("bad max sg_tablesize parameter '%s'\n",
					p);
3072 3073 3074 3075 3076
				goto out;
			}
			target->sg_tablesize = token;
			break;

3077 3078 3079 3080 3081 3082 3083 3084
		case SRP_OPT_COMP_VECTOR:
			if (match_int(args, &token) || token < 0) {
				pr_warn("bad comp_vector parameter '%s'\n", p);
				goto out;
			}
			target->comp_vector = token;
			break;

3085 3086 3087 3088 3089 3090 3091 3092 3093
		case SRP_OPT_TL_RETRY_COUNT:
			if (match_int(args, &token) || token < 2 || token > 7) {
				pr_warn("bad tl_retry_count parameter '%s' (must be a number between 2 and 7)\n",
					p);
				goto out;
			}
			target->tl_retry_count = token;
			break;

3094
		default:
3095 3096
			pr_warn("unknown parameter or missing value '%s' in target creation request\n",
				p);
3097 3098 3099 3100 3101 3102 3103 3104 3105 3106
			goto out;
		}
	}

	if ((opt_mask & SRP_OPT_ALL) == SRP_OPT_ALL)
		ret = 0;
	else
		for (i = 0; i < ARRAY_SIZE(srp_opt_tokens); ++i)
			if ((srp_opt_tokens[i].token & SRP_OPT_ALL) &&
			    !(srp_opt_tokens[i].token & opt_mask))
3107 3108
				pr_warn("target creation request is missing parameter '%s'\n",
					srp_opt_tokens[i].pattern);
3109

3110 3111 3112 3113 3114 3115
	if (target->scsi_host->cmd_per_lun > target->scsi_host->can_queue
	    && (opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
		pr_warn("cmd_per_lun = %d > queue_size = %d\n",
			target->scsi_host->cmd_per_lun,
			target->scsi_host->can_queue);

3116 3117 3118 3119 3120
out:
	kfree(options);
	return ret;
}

3121 3122
static ssize_t srp_create_target(struct device *dev,
				 struct device_attribute *attr,
3123 3124 3125
				 const char *buf, size_t count)
{
	struct srp_host *host =
3126
		container_of(dev, struct srp_host, dev);
3127 3128
	struct Scsi_Host *target_host;
	struct srp_target_port *target;
3129
	struct srp_rdma_ch *ch;
3130 3131
	struct srp_device *srp_dev = host->srp_dev;
	struct ib_device *ibdev = srp_dev->dev;
B
Bart Van Assche 已提交
3132 3133
	int ret, node_idx, node, cpu, i;
	bool multich = false;
3134 3135 3136 3137 3138 3139

	target_host = scsi_host_alloc(&srp_template,
				      sizeof (struct srp_target_port));
	if (!target_host)
		return -ENOMEM;

3140
	target_host->transportt  = ib_srp_transport_template;
3141 3142
	target_host->max_channel = 0;
	target_host->max_id      = 1;
A
Arne Redlich 已提交
3143 3144
	target_host->max_lun     = SRP_MAX_LUN;
	target_host->max_cmd_len = sizeof ((struct srp_cmd *) (void *) 0L)->cdb;
R
Roland Dreier 已提交
3145

3146 3147
	target = host_to_target(target_host);

3148 3149 3150 3151 3152 3153
	target->io_class	= SRP_REV16A_IB_IO_CLASS;
	target->scsi_host	= target_host;
	target->srp_host	= host;
	target->lkey		= host->srp_dev->mr->lkey;
	target->rkey		= host->srp_dev->mr->rkey;
	target->cmd_sg_cnt	= cmd_sg_entries;
3154 3155
	target->sg_tablesize	= indirect_sg_entries ? : cmd_sg_entries;
	target->allow_ext_sg	= allow_ext_sg;
3156
	target->tl_retry_count	= 7;
3157
	target->queue_size	= SRP_DEFAULT_QUEUE_SIZE;
3158

3159 3160 3161 3162 3163 3164
	/*
	 * Avoid that the SCSI host can be removed by srp_remove_target()
	 * before this function returns.
	 */
	scsi_host_get(target->scsi_host);

3165 3166
	mutex_lock(&host->add_target_mutex);

3167 3168
	ret = srp_parse_options(buf, target);
	if (ret)
3169
		goto out;
3170

B
Bart Van Assche 已提交
3171 3172
	ret = scsi_init_shared_tag_map(target_host, target_host->can_queue);
	if (ret)
3173
		goto out;
B
Bart Van Assche 已提交
3174

3175 3176
	target->req_ring_size = target->queue_size - SRP_TSK_MGMT_SQ_SIZE;

3177 3178 3179 3180 3181 3182 3183
	if (!srp_conn_unique(target->srp_host, target)) {
		shost_printk(KERN_INFO, target->scsi_host,
			     PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;initiator_ext=%016llx\n",
			     be64_to_cpu(target->id_ext),
			     be64_to_cpu(target->ioc_guid),
			     be64_to_cpu(target->initiator_ext));
		ret = -EEXIST;
3184
		goto out;
3185 3186
	}

3187
	if (!srp_dev->has_fmr && !srp_dev->has_fr && !target->allow_ext_sg &&
3188
	    target->cmd_sg_cnt < target->sg_tablesize) {
3189
		pr_warn("No MR pool and no external indirect descriptors, limiting sg_tablesize to cmd_sg_cnt\n");
3190 3191 3192 3193 3194 3195
		target->sg_tablesize = target->cmd_sg_cnt;
	}

	target_host->sg_tablesize = target->sg_tablesize;
	target->indirect_size = target->sg_tablesize *
				sizeof (struct srp_direct_buf);
3196 3197 3198 3199
	target->max_iu_len = sizeof (struct srp_cmd) +
			     sizeof (struct srp_indirect_buf) +
			     target->cmd_sg_cnt * sizeof (struct srp_direct_buf);

3200
	INIT_WORK(&target->tl_err_work, srp_tl_err_work);
3201
	INIT_WORK(&target->remove_work, srp_remove_work);
3202
	spin_lock_init(&target->lock);
3203
	ret = ib_query_gid(ibdev, host->port, 0, &target->sgid);
3204
	if (ret)
3205
		goto out;
3206

B
Bart Van Assche 已提交
3207 3208 3209 3210 3211 3212 3213 3214 3215
	ret = -ENOMEM;
	target->ch_count = max_t(unsigned, num_online_nodes(),
				 min(ch_count ? :
				     min(4 * num_online_nodes(),
					 ibdev->num_comp_vectors),
				     num_online_cpus()));
	target->ch = kcalloc(target->ch_count, sizeof(*target->ch),
			     GFP_KERNEL);
	if (!target->ch)
3216
		goto out;
3217

B
Bart Van Assche 已提交
3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245
	node_idx = 0;
	for_each_online_node(node) {
		const int ch_start = (node_idx * target->ch_count /
				      num_online_nodes());
		const int ch_end = ((node_idx + 1) * target->ch_count /
				    num_online_nodes());
		const int cv_start = (node_idx * ibdev->num_comp_vectors /
				      num_online_nodes() + target->comp_vector)
				     % ibdev->num_comp_vectors;
		const int cv_end = ((node_idx + 1) * ibdev->num_comp_vectors /
				    num_online_nodes() + target->comp_vector)
				   % ibdev->num_comp_vectors;
		int cpu_idx = 0;

		for_each_online_cpu(cpu) {
			if (cpu_to_node(cpu) != node)
				continue;
			if (ch_start + cpu_idx >= ch_end)
				continue;
			ch = &target->ch[ch_start + cpu_idx];
			ch->target = target;
			ch->comp_vector = cv_start == cv_end ? cv_start :
				cv_start + cpu_idx % (cv_end - cv_start);
			spin_lock_init(&ch->lock);
			INIT_LIST_HEAD(&ch->free_tx);
			ret = srp_new_cm_id(ch);
			if (ret)
				goto err_disconnect;
3246

B
Bart Van Assche 已提交
3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274
			ret = srp_create_ch_ib(ch);
			if (ret)
				goto err_disconnect;

			ret = srp_alloc_req_data(ch);
			if (ret)
				goto err_disconnect;

			ret = srp_connect_ch(ch, multich);
			if (ret) {
				shost_printk(KERN_ERR, target->scsi_host,
					     PFX "Connection %d/%d failed\n",
					     ch_start + cpu_idx,
					     target->ch_count);
				if (node_idx == 0 && cpu_idx == 0) {
					goto err_disconnect;
				} else {
					srp_free_ch_ib(target, ch);
					srp_free_req_data(target, ch);
					target->ch_count = ch - target->ch;
					break;
				}
			}

			multich = true;
			cpu_idx++;
		}
		node_idx++;
3275 3276
	}

B
Bart Van Assche 已提交
3277 3278
	target->scsi_host->nr_hw_queues = target->ch_count;

3279 3280 3281 3282
	ret = srp_add_target(host, target);
	if (ret)
		goto err_disconnect;

3283 3284 3285 3286 3287
	if (target->state != SRP_TARGET_REMOVED) {
		shost_printk(KERN_DEBUG, target->scsi_host, PFX
			     "new target: id_ext %016llx ioc_guid %016llx pkey %04x service_id %016llx sgid %pI6 dgid %pI6\n",
			     be64_to_cpu(target->id_ext),
			     be64_to_cpu(target->ioc_guid),
3288
			     be16_to_cpu(target->pkey),
3289
			     be64_to_cpu(target->service_id),
3290
			     target->sgid.raw, target->orig_dgid.raw);
3291
	}
B
Bart Van Assche 已提交
3292

3293 3294 3295 3296
	ret = count;

out:
	mutex_unlock(&host->add_target_mutex);
3297 3298 3299

	scsi_host_put(target->scsi_host);

3300
	return ret;
3301 3302 3303 3304

err_disconnect:
	srp_disconnect_target(target);

B
Bart Van Assche 已提交
3305 3306 3307 3308 3309
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
		srp_free_ch_ib(target, ch);
		srp_free_req_data(target, ch);
	}
3310

B
Bart Van Assche 已提交
3311
	kfree(target->ch);
3312
	goto out;
3313 3314
}

3315
static DEVICE_ATTR(add_target, S_IWUSR, NULL, srp_create_target);
3316

3317 3318
static ssize_t show_ibdev(struct device *dev, struct device_attribute *attr,
			  char *buf)
3319
{
3320
	struct srp_host *host = container_of(dev, struct srp_host, dev);
3321

3322
	return sprintf(buf, "%s\n", host->srp_dev->dev->name);
3323 3324
}

3325
static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);
3326

3327 3328
static ssize_t show_port(struct device *dev, struct device_attribute *attr,
			 char *buf)
3329
{
3330
	struct srp_host *host = container_of(dev, struct srp_host, dev);
3331 3332 3333 3334

	return sprintf(buf, "%d\n", host->port);
}

3335
static DEVICE_ATTR(port, S_IRUGO, show_port, NULL);
3336

3337
static struct srp_host *srp_add_port(struct srp_device *device, u8 port)
3338 3339 3340 3341 3342 3343 3344 3345
{
	struct srp_host *host;

	host = kzalloc(sizeof *host, GFP_KERNEL);
	if (!host)
		return NULL;

	INIT_LIST_HEAD(&host->target_list);
3346
	spin_lock_init(&host->target_lock);
3347
	init_completion(&host->released);
3348
	mutex_init(&host->add_target_mutex);
3349
	host->srp_dev = device;
3350 3351
	host->port = port;

3352 3353
	host->dev.class = &srp_class;
	host->dev.parent = device->dev->dma_device;
3354
	dev_set_name(&host->dev, "srp-%s-%d", device->dev->name, port);
3355

3356
	if (device_register(&host->dev))
3357
		goto free_host;
3358
	if (device_create_file(&host->dev, &dev_attr_add_target))
3359
		goto err_class;
3360
	if (device_create_file(&host->dev, &dev_attr_ibdev))
3361
		goto err_class;
3362
	if (device_create_file(&host->dev, &dev_attr_port))
3363 3364 3365 3366 3367
		goto err_class;

	return host;

err_class:
3368
	device_unregister(&host->dev);
3369

3370
free_host:
3371 3372 3373 3374 3375 3376 3377
	kfree(host);

	return NULL;
}

static void srp_add_one(struct ib_device *device)
{
3378 3379
	struct srp_device *srp_dev;
	struct ib_device_attr *dev_attr;
3380
	struct srp_host *host;
3381 3382
	int mr_page_shift, s, e, p;
	u64 max_pages_per_mr;
3383

3384 3385
	dev_attr = kmalloc(sizeof *dev_attr, GFP_KERNEL);
	if (!dev_attr)
3386
		return;
3387

3388
	if (ib_query_device(device, dev_attr)) {
3389
		pr_warn("Query device failed for %s\n", device->name);
3390 3391 3392 3393 3394 3395 3396
		goto free_attr;
	}

	srp_dev = kmalloc(sizeof *srp_dev, GFP_KERNEL);
	if (!srp_dev)
		goto free_attr;

3397 3398
	srp_dev->has_fmr = (device->alloc_fmr && device->dealloc_fmr &&
			    device->map_phys_fmr && device->unmap_fmr);
3399 3400 3401 3402 3403 3404 3405
	srp_dev->has_fr = (dev_attr->device_cap_flags &
			   IB_DEVICE_MEM_MGT_EXTENSIONS);
	if (!srp_dev->has_fmr && !srp_dev->has_fr)
		dev_warn(&device->dev, "neither FMR nor FR is supported\n");

	srp_dev->use_fast_reg = (srp_dev->has_fr &&
				 (!srp_dev->has_fmr || prefer_fr));
3406

3407 3408
	/*
	 * Use the smallest page size supported by the HCA, down to a
3409 3410
	 * minimum of 4096 bytes. We're unlikely to build large sglists
	 * out of smaller entries.
3411
	 */
3412 3413 3414 3415 3416 3417 3418
	mr_page_shift		= max(12, ffs(dev_attr->page_size_cap) - 1);
	srp_dev->mr_page_size	= 1 << mr_page_shift;
	srp_dev->mr_page_mask	= ~((u64) srp_dev->mr_page_size - 1);
	max_pages_per_mr	= dev_attr->max_mr_size;
	do_div(max_pages_per_mr, srp_dev->mr_page_size);
	srp_dev->max_pages_per_mr = min_t(u64, SRP_MAX_PAGES_PER_MR,
					  max_pages_per_mr);
3419 3420 3421 3422 3423
	if (srp_dev->use_fast_reg) {
		srp_dev->max_pages_per_mr =
			min_t(u32, srp_dev->max_pages_per_mr,
			      dev_attr->max_fast_reg_page_list_len);
	}
3424 3425
	srp_dev->mr_max_size	= srp_dev->mr_page_size *
				   srp_dev->max_pages_per_mr;
3426
	pr_debug("%s: mr_page_shift = %d, dev_attr->max_mr_size = %#llx, dev_attr->max_fast_reg_page_list_len = %u, max_pages_per_mr = %d, mr_max_size = %#x\n",
3427
		 device->name, mr_page_shift, dev_attr->max_mr_size,
3428
		 dev_attr->max_fast_reg_page_list_len,
3429
		 srp_dev->max_pages_per_mr, srp_dev->mr_max_size);
3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444

	INIT_LIST_HEAD(&srp_dev->dev_list);

	srp_dev->dev = device;
	srp_dev->pd  = ib_alloc_pd(device);
	if (IS_ERR(srp_dev->pd))
		goto free_dev;

	srp_dev->mr = ib_get_dma_mr(srp_dev->pd,
				    IB_ACCESS_LOCAL_WRITE |
				    IB_ACCESS_REMOTE_READ |
				    IB_ACCESS_REMOTE_WRITE);
	if (IS_ERR(srp_dev->mr))
		goto err_pd;

T
Tom Tucker 已提交
3445
	if (device->node_type == RDMA_NODE_IB_SWITCH) {
3446 3447 3448 3449 3450 3451 3452 3453
		s = 0;
		e = 0;
	} else {
		s = 1;
		e = device->phys_port_cnt;
	}

	for (p = s; p <= e; ++p) {
3454
		host = srp_add_port(srp_dev, p);
3455
		if (host)
3456
			list_add_tail(&host->list, &srp_dev->dev_list);
3457 3458
	}

3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470
	ib_set_client_data(device, &srp_client, srp_dev);

	goto free_attr;

err_pd:
	ib_dealloc_pd(srp_dev->pd);

free_dev:
	kfree(srp_dev);

free_attr:
	kfree(dev_attr);
3471 3472 3473 3474
}

static void srp_remove_one(struct ib_device *device)
{
3475
	struct srp_device *srp_dev;
3476
	struct srp_host *host, *tmp_host;
3477
	struct srp_target_port *target;
3478

3479
	srp_dev = ib_get_client_data(device, &srp_client);
3480 3481
	if (!srp_dev)
		return;
3482

3483
	list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) {
3484
		device_unregister(&host->dev);
3485 3486 3487 3488 3489 3490 3491
		/*
		 * Wait for the sysfs entry to go away, so that no new
		 * target ports can be created.
		 */
		wait_for_completion(&host->released);

		/*
3492
		 * Remove all target ports.
3493
		 */
3494
		spin_lock(&host->target_lock);
3495 3496
		list_for_each_entry(target, &host->target_list, list)
			srp_queue_remove_work(target);
3497
		spin_unlock(&host->target_lock);
3498 3499

		/*
3500
		 * Wait for tl_err and target port removal tasks.
3501
		 */
3502
		flush_workqueue(system_long_wq);
3503
		flush_workqueue(srp_remove_wq);
3504 3505 3506 3507

		kfree(host);
	}

3508 3509 3510 3511
	ib_dereg_mr(srp_dev->mr);
	ib_dealloc_pd(srp_dev->pd);

	kfree(srp_dev);
3512 3513
}

3514
static struct srp_function_template ib_srp_transport_functions = {
3515 3516
	.has_rport_state	 = true,
	.reset_timer_if_blocked	 = true,
3517
	.reconnect_delay	 = &srp_reconnect_delay,
3518 3519 3520
	.fast_io_fail_tmo	 = &srp_fast_io_fail_tmo,
	.dev_loss_tmo		 = &srp_dev_loss_tmo,
	.reconnect		 = srp_rport_reconnect,
3521
	.rport_delete		 = srp_rport_delete,
3522
	.terminate_rport_io	 = srp_terminate_io,
3523 3524
};

3525 3526 3527 3528
static int __init srp_init_module(void)
{
	int ret;

3529
	BUILD_BUG_ON(FIELD_SIZEOF(struct ib_wc, wr_id) < sizeof(void *));
3530

3531
	if (srp_sg_tablesize) {
3532
		pr_warn("srp_sg_tablesize is deprecated, please use cmd_sg_entries\n");
3533 3534 3535 3536 3537 3538 3539 3540
		if (!cmd_sg_entries)
			cmd_sg_entries = srp_sg_tablesize;
	}

	if (!cmd_sg_entries)
		cmd_sg_entries = SRP_DEF_SG_TABLESIZE;

	if (cmd_sg_entries > 255) {
3541
		pr_warn("Clamping cmd_sg_entries to 255\n");
3542
		cmd_sg_entries = 255;
3543 3544
	}

3545 3546 3547
	if (!indirect_sg_entries)
		indirect_sg_entries = cmd_sg_entries;
	else if (indirect_sg_entries < cmd_sg_entries) {
3548 3549
		pr_warn("Bumping up indirect_sg_entries to match cmd_sg_entries (%u)\n",
			cmd_sg_entries);
3550 3551 3552
		indirect_sg_entries = cmd_sg_entries;
	}

3553
	srp_remove_wq = create_workqueue("srp_remove");
3554 3555
	if (!srp_remove_wq) {
		ret = -ENOMEM;
3556 3557 3558 3559
		goto out;
	}

	ret = -ENOMEM;
3560 3561 3562
	ib_srp_transport_template =
		srp_attach_transport(&ib_srp_transport_functions);
	if (!ib_srp_transport_template)
3563
		goto destroy_wq;
3564

3565 3566
	ret = class_register(&srp_class);
	if (ret) {
3567
		pr_err("couldn't register class infiniband_srp\n");
3568
		goto release_tr;
3569 3570
	}

3571 3572
	ib_sa_register_client(&srp_sa_client);

3573 3574
	ret = ib_register_client(&srp_client);
	if (ret) {
3575
		pr_err("couldn't register IB client\n");
3576
		goto unreg_sa;
3577 3578
	}

3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591
out:
	return ret;

unreg_sa:
	ib_sa_unregister_client(&srp_sa_client);
	class_unregister(&srp_class);

release_tr:
	srp_release_transport(ib_srp_transport_template);

destroy_wq:
	destroy_workqueue(srp_remove_wq);
	goto out;
3592 3593 3594 3595 3596
}

static void __exit srp_cleanup_module(void)
{
	ib_unregister_client(&srp_client);
3597
	ib_sa_unregister_client(&srp_sa_client);
3598
	class_unregister(&srp_class);
3599
	srp_release_transport(ib_srp_transport_template);
3600
	destroy_workqueue(srp_remove_wq);
3601 3602 3603 3604
}

module_init(srp_init_module);
module_exit(srp_cleanup_module);