ib_srp.c 95.0 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32
/*
 * Copyright (c) 2005 Cisco Systems.  All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

33
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
34

35 36 37 38 39 40 41
#include <linux/module.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/err.h>
#include <linux/string.h>
#include <linux/parser.h>
#include <linux/random.h>
42
#include <linux/jiffies.h>
43
#include <rdma/ib_cache.h>
44

A
Arun Sharma 已提交
45
#include <linux/atomic.h>
46 47 48 49

#include <scsi/scsi.h>
#include <scsi/scsi_device.h>
#include <scsi/scsi_dbg.h>
50
#include <scsi/scsi_tcq.h>
51
#include <scsi/srp.h>
52
#include <scsi/scsi_transport_srp.h>
53 54 55 56 57

#include "ib_srp.h"

#define DRV_NAME	"ib_srp"
#define PFX		DRV_NAME ": "
58 59
#define DRV_VERSION	"2.0"
#define DRV_RELDATE	"July 26, 2015"
60 61

MODULE_AUTHOR("Roland Dreier");
62
MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator");
63
MODULE_LICENSE("Dual BSD/GPL");
64 65
MODULE_VERSION(DRV_VERSION);
MODULE_INFO(release_date, DRV_RELDATE);
66

67 68
static unsigned int srp_sg_tablesize;
static unsigned int cmd_sg_entries;
69 70
static unsigned int indirect_sg_entries;
static bool allow_ext_sg;
71 72
static bool prefer_fr = true;
static bool register_always = true;
73
static int topspin_workarounds = 1;
74

75 76
module_param(srp_sg_tablesize, uint, 0444);
MODULE_PARM_DESC(srp_sg_tablesize, "Deprecated name for cmd_sg_entries");
77

78 79 80
module_param(cmd_sg_entries, uint, 0444);
MODULE_PARM_DESC(cmd_sg_entries,
		 "Default number of gather/scatter entries in the SRP command (default is 12, max 255)");
81

82 83 84 85 86 87 88 89
module_param(indirect_sg_entries, uint, 0444);
MODULE_PARM_DESC(indirect_sg_entries,
		 "Default max number of gather/scatter entries (default is 12, max is " __stringify(SCSI_MAX_SG_CHAIN_SEGMENTS) ")");

module_param(allow_ext_sg, bool, 0444);
MODULE_PARM_DESC(allow_ext_sg,
		  "Default behavior when there are more than cmd_sg_entries S/G entries after mapping; fails the request when false (default false)");

90 91 92 93
module_param(topspin_workarounds, int, 0444);
MODULE_PARM_DESC(topspin_workarounds,
		 "Enable workarounds for Topspin/Cisco SRP target bugs if != 0");

94 95 96 97
module_param(prefer_fr, bool, 0444);
MODULE_PARM_DESC(prefer_fr,
"Whether to use fast registration if both FMR and fast registration are supported");

98 99 100 101
module_param(register_always, bool, 0444);
MODULE_PARM_DESC(register_always,
		 "Use memory registration even for contiguous memory regions");

102
static const struct kernel_param_ops srp_tmo_ops;
103

104 105 106 107 108
static int srp_reconnect_delay = 10;
module_param_cb(reconnect_delay, &srp_tmo_ops, &srp_reconnect_delay,
		S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(reconnect_delay, "Time between successive reconnect attempts");

109 110 111 112 113 114 115 116
static int srp_fast_io_fail_tmo = 15;
module_param_cb(fast_io_fail_tmo, &srp_tmo_ops, &srp_fast_io_fail_tmo,
		S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(fast_io_fail_tmo,
		 "Number of seconds between the observation of a transport"
		 " layer error and failing all I/O. \"off\" means that this"
		 " functionality is disabled.");

117
static int srp_dev_loss_tmo = 600;
118 119 120 121 122 123 124 125 126 127
module_param_cb(dev_loss_tmo, &srp_tmo_ops, &srp_dev_loss_tmo,
		S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(dev_loss_tmo,
		 "Maximum number of seconds that the SRP transport should"
		 " insulate transport layer errors. After this time has been"
		 " exceeded the SCSI host is removed. Should be"
		 " between 1 and " __stringify(SCSI_DEVICE_BLOCK_MAX_TIMEOUT)
		 " if fast_io_fail_tmo has not been set. \"off\" means that"
		 " this functionality is disabled.");

B
Bart Van Assche 已提交
128 129 130 131 132
static unsigned ch_count;
module_param(ch_count, uint, 0444);
MODULE_PARM_DESC(ch_count,
		 "Number of RDMA channels to use for communication with an SRP target. Using more than one channel improves performance if the HCA supports multiple completion vectors. The default value is the minimum of four times the number of online CPU sockets and the number of completion vectors supported by the HCA.");

133
static void srp_add_one(struct ib_device *device);
134
static void srp_remove_one(struct ib_device *device, void *client_data);
135 136
static void srp_recv_completion(struct ib_cq *cq, void *ch_ptr);
static void srp_send_completion(struct ib_cq *cq, void *ch_ptr);
137 138
static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event);

139
static struct scsi_transport_template *ib_srp_transport_template;
140
static struct workqueue_struct *srp_remove_wq;
141

142 143 144 145 146 147
static struct ib_client srp_client = {
	.name   = "srp",
	.add    = srp_add_one,
	.remove = srp_remove_one
};

148 149
static struct ib_sa_client srp_sa_client;

150 151 152 153 154 155 156 157 158 159 160 161 162 163
static int srp_tmo_get(char *buffer, const struct kernel_param *kp)
{
	int tmo = *(int *)kp->arg;

	if (tmo >= 0)
		return sprintf(buffer, "%d", tmo);
	else
		return sprintf(buffer, "off");
}

static int srp_tmo_set(const char *val, const struct kernel_param *kp)
{
	int tmo, res;

164 165 166 167
	res = srp_parse_tmo(&tmo, val);
	if (res)
		goto out;

168 169 170 171 172
	if (kp->arg == &srp_reconnect_delay)
		res = srp_tmo_valid(tmo, srp_fast_io_fail_tmo,
				    srp_dev_loss_tmo);
	else if (kp->arg == &srp_fast_io_fail_tmo)
		res = srp_tmo_valid(srp_reconnect_delay, tmo, srp_dev_loss_tmo);
173
	else
174 175
		res = srp_tmo_valid(srp_reconnect_delay, srp_fast_io_fail_tmo,
				    tmo);
176 177 178 179 180 181 182 183
	if (res)
		goto out;
	*(int *)kp->arg = tmo;

out:
	return res;
}

184
static const struct kernel_param_ops srp_tmo_ops = {
185 186 187 188
	.get = srp_tmo_get,
	.set = srp_tmo_set,
};

189 190 191 192 193 194 195 196 197 198
static inline struct srp_target_port *host_to_target(struct Scsi_Host *host)
{
	return (struct srp_target_port *) host->hostdata;
}

static const char *srp_target_info(struct Scsi_Host *host)
{
	return host_to_target(host)->target_name;
}

199 200 201
static int srp_target_is_topspin(struct srp_target_port *target)
{
	static const u8 topspin_oui[3] = { 0x00, 0x05, 0xad };
202
	static const u8 cisco_oui[3]   = { 0x00, 0x1b, 0x0d };
203 204

	return topspin_workarounds &&
205 206
		(!memcmp(&target->ioc_guid, topspin_oui, sizeof topspin_oui) ||
		 !memcmp(&target->ioc_guid, cisco_oui, sizeof cisco_oui));
207 208
}

209 210 211 212 213 214 215 216 217 218 219 220 221 222
static struct srp_iu *srp_alloc_iu(struct srp_host *host, size_t size,
				   gfp_t gfp_mask,
				   enum dma_data_direction direction)
{
	struct srp_iu *iu;

	iu = kmalloc(sizeof *iu, gfp_mask);
	if (!iu)
		goto out;

	iu->buf = kzalloc(size, gfp_mask);
	if (!iu->buf)
		goto out_free_iu;

223 224 225
	iu->dma = ib_dma_map_single(host->srp_dev->dev, iu->buf, size,
				    direction);
	if (ib_dma_mapping_error(host->srp_dev->dev, iu->dma))
226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245
		goto out_free_buf;

	iu->size      = size;
	iu->direction = direction;

	return iu;

out_free_buf:
	kfree(iu->buf);
out_free_iu:
	kfree(iu);
out:
	return NULL;
}

static void srp_free_iu(struct srp_host *host, struct srp_iu *iu)
{
	if (!iu)
		return;

246 247
	ib_dma_unmap_single(host->srp_dev->dev, iu->dma, iu->size,
			    iu->direction);
248 249 250 251 252 253
	kfree(iu->buf);
	kfree(iu);
}

static void srp_qp_event(struct ib_event *event, void *context)
{
254 255
	pr_debug("QP event %s (%d)\n",
		 ib_event_msg(event->event), event->event);
256 257 258 259 260 261 262 263 264 265 266 267
}

static int srp_init_qp(struct srp_target_port *target,
		       struct ib_qp *qp)
{
	struct ib_qp_attr *attr;
	int ret;

	attr = kmalloc(sizeof *attr, GFP_KERNEL);
	if (!attr)
		return -ENOMEM;

268 269 270 271
	ret = ib_find_cached_pkey(target->srp_host->srp_dev->dev,
				  target->srp_host->port,
				  be16_to_cpu(target->pkey),
				  &attr->pkey_index);
272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290
	if (ret)
		goto out;

	attr->qp_state        = IB_QPS_INIT;
	attr->qp_access_flags = (IB_ACCESS_REMOTE_READ |
				    IB_ACCESS_REMOTE_WRITE);
	attr->port_num        = target->srp_host->port;

	ret = ib_modify_qp(qp, attr,
			   IB_QP_STATE		|
			   IB_QP_PKEY_INDEX	|
			   IB_QP_ACCESS_FLAGS	|
			   IB_QP_PORT);

out:
	kfree(attr);
	return ret;
}

291
static int srp_new_cm_id(struct srp_rdma_ch *ch)
D
David Dillow 已提交
292
{
293
	struct srp_target_port *target = ch->target;
D
David Dillow 已提交
294 295
	struct ib_cm_id *new_cm_id;

296
	new_cm_id = ib_create_cm_id(target->srp_host->srp_dev->dev,
297
				    srp_cm_handler, ch);
D
David Dillow 已提交
298 299 300
	if (IS_ERR(new_cm_id))
		return PTR_ERR(new_cm_id);

301 302 303 304 305 306 307
	if (ch->cm_id)
		ib_destroy_cm_id(ch->cm_id);
	ch->cm_id = new_cm_id;
	ch->path.sgid = target->sgid;
	ch->path.dgid = target->orig_dgid;
	ch->path.pkey = target->pkey;
	ch->path.service_id = target->service_id;
D
David Dillow 已提交
308 309 310 311

	return 0;
}

312 313 314 315 316 317 318 319 320
static struct ib_fmr_pool *srp_alloc_fmr_pool(struct srp_target_port *target)
{
	struct srp_device *dev = target->srp_host->srp_dev;
	struct ib_fmr_pool_param fmr_param;

	memset(&fmr_param, 0, sizeof(fmr_param));
	fmr_param.pool_size	    = target->scsi_host->can_queue;
	fmr_param.dirty_watermark   = fmr_param.pool_size / 4;
	fmr_param.cache		    = 1;
321 322
	fmr_param.max_pages_per_fmr = dev->max_pages_per_mr;
	fmr_param.page_shift	    = ilog2(dev->mr_page_size);
323 324 325 326 327 328 329
	fmr_param.access	    = (IB_ACCESS_LOCAL_WRITE |
				       IB_ACCESS_REMOTE_WRITE |
				       IB_ACCESS_REMOTE_READ);

	return ib_create_fmr_pool(dev->pd, &fmr_param);
}

330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377
/**
 * srp_destroy_fr_pool() - free the resources owned by a pool
 * @pool: Fast registration pool to be destroyed.
 */
static void srp_destroy_fr_pool(struct srp_fr_pool *pool)
{
	int i;
	struct srp_fr_desc *d;

	if (!pool)
		return;

	for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
		if (d->mr)
			ib_dereg_mr(d->mr);
	}
	kfree(pool);
}

/**
 * srp_create_fr_pool() - allocate and initialize a pool for fast registration
 * @device:            IB device to allocate fast registration descriptors for.
 * @pd:                Protection domain associated with the FR descriptors.
 * @pool_size:         Number of descriptors to allocate.
 * @max_page_list_len: Maximum fast registration work request page list length.
 */
static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device,
					      struct ib_pd *pd, int pool_size,
					      int max_page_list_len)
{
	struct srp_fr_pool *pool;
	struct srp_fr_desc *d;
	struct ib_mr *mr;
	int i, ret = -EINVAL;

	if (pool_size <= 0)
		goto err;
	ret = -ENOMEM;
	pool = kzalloc(sizeof(struct srp_fr_pool) +
		       pool_size * sizeof(struct srp_fr_desc), GFP_KERNEL);
	if (!pool)
		goto err;
	pool->size = pool_size;
	pool->max_page_list_len = max_page_list_len;
	spin_lock_init(&pool->lock);
	INIT_LIST_HEAD(&pool->free_list);

	for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
S
Sagi Grimberg 已提交
378 379
		mr = ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG,
				 max_page_list_len);
380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447
		if (IS_ERR(mr)) {
			ret = PTR_ERR(mr);
			goto destroy_pool;
		}
		d->mr = mr;
		list_add_tail(&d->entry, &pool->free_list);
	}

out:
	return pool;

destroy_pool:
	srp_destroy_fr_pool(pool);

err:
	pool = ERR_PTR(ret);
	goto out;
}

/**
 * srp_fr_pool_get() - obtain a descriptor suitable for fast registration
 * @pool: Pool to obtain descriptor from.
 */
static struct srp_fr_desc *srp_fr_pool_get(struct srp_fr_pool *pool)
{
	struct srp_fr_desc *d = NULL;
	unsigned long flags;

	spin_lock_irqsave(&pool->lock, flags);
	if (!list_empty(&pool->free_list)) {
		d = list_first_entry(&pool->free_list, typeof(*d), entry);
		list_del(&d->entry);
	}
	spin_unlock_irqrestore(&pool->lock, flags);

	return d;
}

/**
 * srp_fr_pool_put() - put an FR descriptor back in the free list
 * @pool: Pool the descriptor was allocated from.
 * @desc: Pointer to an array of fast registration descriptor pointers.
 * @n:    Number of descriptors to put back.
 *
 * Note: The caller must already have queued an invalidation request for
 * desc->mr->rkey before calling this function.
 */
static void srp_fr_pool_put(struct srp_fr_pool *pool, struct srp_fr_desc **desc,
			    int n)
{
	unsigned long flags;
	int i;

	spin_lock_irqsave(&pool->lock, flags);
	for (i = 0; i < n; i++)
		list_add(&desc[i]->entry, &pool->free_list);
	spin_unlock_irqrestore(&pool->lock, flags);
}

static struct srp_fr_pool *srp_alloc_fr_pool(struct srp_target_port *target)
{
	struct srp_device *dev = target->srp_host->srp_dev;

	return srp_create_fr_pool(dev->dev, dev->pd,
				  target->scsi_host->can_queue,
				  dev->max_pages_per_mr);
}

448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464
/**
 * srp_destroy_qp() - destroy an RDMA queue pair
 * @ch: SRP RDMA channel.
 *
 * Change a queue pair into the error state and wait until all receive
 * completions have been processed before destroying it. This avoids that
 * the receive completion handler can access the queue pair while it is
 * being destroyed.
 */
static void srp_destroy_qp(struct srp_rdma_ch *ch)
{
	static struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
	static struct ib_recv_wr wr = { .wr_id = SRP_LAST_WR_ID };
	struct ib_recv_wr *bad_wr;
	int ret;

	/* Destroying a QP and reusing ch->done is only safe if not connected */
465
	WARN_ON_ONCE(ch->connected);
466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481

	ret = ib_modify_qp(ch->qp, &attr, IB_QP_STATE);
	WARN_ONCE(ret, "ib_cm_init_qp_attr() returned %d\n", ret);
	if (ret)
		goto out;

	init_completion(&ch->done);
	ret = ib_post_recv(ch->qp, &wr, &bad_wr);
	WARN_ONCE(ret, "ib_post_recv() returned %d\n", ret);
	if (ret == 0)
		wait_for_completion(&ch->done);

out:
	ib_destroy_qp(ch->qp);
}

482
static int srp_create_ch_ib(struct srp_rdma_ch *ch)
483
{
484
	struct srp_target_port *target = ch->target;
485
	struct srp_device *dev = target->srp_host->srp_dev;
486
	struct ib_qp_init_attr *init_attr;
487 488
	struct ib_cq *recv_cq, *send_cq;
	struct ib_qp *qp;
489
	struct ib_fmr_pool *fmr_pool = NULL;
490
	struct srp_fr_pool *fr_pool = NULL;
491
	const int m = dev->use_fast_reg ? 3 : 1;
492
	struct ib_cq_init_attr cq_attr = {};
493 494 495 496 497 498
	int ret;

	init_attr = kzalloc(sizeof *init_attr, GFP_KERNEL);
	if (!init_attr)
		return -ENOMEM;

499
	/* + 1 for SRP_LAST_WR_ID */
500 501
	cq_attr.cqe = target->queue_size + 1;
	cq_attr.comp_vector = ch->comp_vector;
502
	recv_cq = ib_create_cq(dev->dev, srp_recv_completion, NULL, ch,
503
			       &cq_attr);
504 505
	if (IS_ERR(recv_cq)) {
		ret = PTR_ERR(recv_cq);
506
		goto err;
507 508
	}

509 510
	cq_attr.cqe = m * target->queue_size;
	cq_attr.comp_vector = ch->comp_vector;
511
	send_cq = ib_create_cq(dev->dev, srp_send_completion, NULL, ch,
512
			       &cq_attr);
513 514
	if (IS_ERR(send_cq)) {
		ret = PTR_ERR(send_cq);
515
		goto err_recv_cq;
516 517
	}

518
	ib_req_notify_cq(recv_cq, IB_CQ_NEXT_COMP);
519 520

	init_attr->event_handler       = srp_qp_event;
521
	init_attr->cap.max_send_wr     = m * target->queue_size;
522
	init_attr->cap.max_recv_wr     = target->queue_size + 1;
523 524
	init_attr->cap.max_recv_sge    = 1;
	init_attr->cap.max_send_sge    = 1;
525
	init_attr->sq_sig_type         = IB_SIGNAL_REQ_WR;
526
	init_attr->qp_type             = IB_QPT_RC;
527 528
	init_attr->send_cq             = send_cq;
	init_attr->recv_cq             = recv_cq;
529

530
	qp = ib_create_qp(dev->pd, init_attr);
531 532
	if (IS_ERR(qp)) {
		ret = PTR_ERR(qp);
533
		goto err_send_cq;
534 535
	}

536
	ret = srp_init_qp(target, qp);
537 538
	if (ret)
		goto err_qp;
539

540
	if (dev->use_fast_reg) {
541 542 543 544 545 546 547
		fr_pool = srp_alloc_fr_pool(target);
		if (IS_ERR(fr_pool)) {
			ret = PTR_ERR(fr_pool);
			shost_printk(KERN_WARNING, target->scsi_host, PFX
				     "FR pool allocation failed (%d)\n", ret);
			goto err_qp;
		}
548
	} else if (dev->use_fmr) {
549 550 551 552 553 554 555 556 557
		fmr_pool = srp_alloc_fmr_pool(target);
		if (IS_ERR(fmr_pool)) {
			ret = PTR_ERR(fmr_pool);
			shost_printk(KERN_WARNING, target->scsi_host, PFX
				     "FMR pool allocation failed (%d)\n", ret);
			goto err_qp;
		}
	}

558
	if (ch->qp)
559
		srp_destroy_qp(ch);
560 561 562 563
	if (ch->recv_cq)
		ib_destroy_cq(ch->recv_cq);
	if (ch->send_cq)
		ib_destroy_cq(ch->send_cq);
564

565 566 567
	ch->qp = qp;
	ch->recv_cq = recv_cq;
	ch->send_cq = send_cq;
568

569 570 571 572 573 574 575 576 577 578
	if (dev->use_fast_reg) {
		if (ch->fr_pool)
			srp_destroy_fr_pool(ch->fr_pool);
		ch->fr_pool = fr_pool;
	} else if (dev->use_fmr) {
		if (ch->fmr_pool)
			ib_destroy_fmr_pool(ch->fmr_pool);
		ch->fmr_pool = fmr_pool;
	}

579 580 581 582
	kfree(init_attr);
	return 0;

err_qp:
583
	ib_destroy_qp(qp);
584 585

err_send_cq:
586
	ib_destroy_cq(send_cq);
587 588

err_recv_cq:
589
	ib_destroy_cq(recv_cq);
590 591

err:
592 593 594 595
	kfree(init_attr);
	return ret;
}

596 597
/*
 * Note: this function may be called without srp_alloc_iu_bufs() having been
598
 * invoked. Hence the ch->[rt]x_ring checks.
599
 */
600 601
static void srp_free_ch_ib(struct srp_target_port *target,
			   struct srp_rdma_ch *ch)
602
{
603
	struct srp_device *dev = target->srp_host->srp_dev;
604 605
	int i;

B
Bart Van Assche 已提交
606 607 608
	if (!ch->target)
		return;

609 610 611
	if (ch->cm_id) {
		ib_destroy_cm_id(ch->cm_id);
		ch->cm_id = NULL;
612 613
	}

B
Bart Van Assche 已提交
614 615 616 617
	/* If srp_new_cm_id() succeeded but srp_create_ch_ib() not, return. */
	if (!ch->qp)
		return;

618
	if (dev->use_fast_reg) {
619 620
		if (ch->fr_pool)
			srp_destroy_fr_pool(ch->fr_pool);
621
	} else if (dev->use_fmr) {
622 623
		if (ch->fmr_pool)
			ib_destroy_fmr_pool(ch->fmr_pool);
624
	}
625
	srp_destroy_qp(ch);
626 627
	ib_destroy_cq(ch->send_cq);
	ib_destroy_cq(ch->recv_cq);
628

B
Bart Van Assche 已提交
629 630 631 632 633 634 635 636
	/*
	 * Avoid that the SCSI error handler tries to use this channel after
	 * it has been freed. The SCSI error handler can namely continue
	 * trying to perform recovery actions after scsi_remove_host()
	 * returned.
	 */
	ch->target = NULL;

637 638
	ch->qp = NULL;
	ch->send_cq = ch->recv_cq = NULL;
639

640
	if (ch->rx_ring) {
641
		for (i = 0; i < target->queue_size; ++i)
642 643 644
			srp_free_iu(target->srp_host, ch->rx_ring[i]);
		kfree(ch->rx_ring);
		ch->rx_ring = NULL;
645
	}
646
	if (ch->tx_ring) {
647
		for (i = 0; i < target->queue_size; ++i)
648 649 650
			srp_free_iu(target->srp_host, ch->tx_ring[i]);
		kfree(ch->tx_ring);
		ch->tx_ring = NULL;
651
	}
652 653 654 655
}

static void srp_path_rec_completion(int status,
				    struct ib_sa_path_rec *pathrec,
656
				    void *ch_ptr)
657
{
658 659
	struct srp_rdma_ch *ch = ch_ptr;
	struct srp_target_port *target = ch->target;
660

661
	ch->status = status;
662
	if (status)
663 664
		shost_printk(KERN_ERR, target->scsi_host,
			     PFX "Got failed path rec status %d\n", status);
665
	else
666 667
		ch->path = *pathrec;
	complete(&ch->done);
668 669
}

670
static int srp_lookup_path(struct srp_rdma_ch *ch)
671
{
672
	struct srp_target_port *target = ch->target;
673 674
	int ret;

675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695
	ch->path.numb_path = 1;

	init_completion(&ch->done);

	ch->path_query_id = ib_sa_path_rec_get(&srp_sa_client,
					       target->srp_host->srp_dev->dev,
					       target->srp_host->port,
					       &ch->path,
					       IB_SA_PATH_REC_SERVICE_ID |
					       IB_SA_PATH_REC_DGID	 |
					       IB_SA_PATH_REC_SGID	 |
					       IB_SA_PATH_REC_NUMB_PATH	 |
					       IB_SA_PATH_REC_PKEY,
					       SRP_PATH_REC_TIMEOUT_MS,
					       GFP_KERNEL,
					       srp_path_rec_completion,
					       ch, &ch->path_query);
	if (ch->path_query_id < 0)
		return ch->path_query_id;

	ret = wait_for_completion_interruptible(&ch->done);
696 697
	if (ret < 0)
		return ret;
698

699
	if (ch->status < 0)
700 701
		shost_printk(KERN_WARNING, target->scsi_host,
			     PFX "Path record query failed\n");
702

703
	return ch->status;
704 705
}

B
Bart Van Assche 已提交
706
static int srp_send_req(struct srp_rdma_ch *ch, bool multich)
707
{
708
	struct srp_target_port *target = ch->target;
709 710 711 712 713 714 715 716 717 718
	struct {
		struct ib_cm_req_param param;
		struct srp_login_req   priv;
	} *req = NULL;
	int status;

	req = kzalloc(sizeof *req, GFP_KERNEL);
	if (!req)
		return -ENOMEM;

719
	req->param.primary_path		      = &ch->path;
720 721
	req->param.alternate_path 	      = NULL;
	req->param.service_id 		      = target->service_id;
722 723
	req->param.qp_num		      = ch->qp->qp_num;
	req->param.qp_type		      = ch->qp->qp_type;
724 725 726 727 728 729 730 731 732 733 734 735 736 737
	req->param.private_data 	      = &req->priv;
	req->param.private_data_len 	      = sizeof req->priv;
	req->param.flow_control 	      = 1;

	get_random_bytes(&req->param.starting_psn, 4);
	req->param.starting_psn 	     &= 0xffffff;

	/*
	 * Pick some arbitrary defaults here; we could make these
	 * module parameters if anyone cared about setting them.
	 */
	req->param.responder_resources	      = 4;
	req->param.remote_cm_response_timeout = 20;
	req->param.local_cm_response_timeout  = 20;
738
	req->param.retry_count                = target->tl_retry_count;
739 740 741 742 743
	req->param.rnr_retry_count 	      = 7;
	req->param.max_cm_retries 	      = 15;

	req->priv.opcode     	= SRP_LOGIN_REQ;
	req->priv.tag        	= 0;
744
	req->priv.req_it_iu_len = cpu_to_be32(target->max_iu_len);
745 746
	req->priv.req_buf_fmt 	= cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
					      SRP_BUF_FORMAT_INDIRECT);
B
Bart Van Assche 已提交
747 748
	req->priv.req_flags	= (multich ? SRP_MULTICHAN_MULTI :
				   SRP_MULTICHAN_SINGLE);
749
	/*
R
Roland Dreier 已提交
750
	 * In the published SRP specification (draft rev. 16a), the
751 752 753 754 755 756 757 758 759
	 * port identifier format is 8 bytes of ID extension followed
	 * by 8 bytes of GUID.  Older drafts put the two halves in the
	 * opposite order, so that the GUID comes first.
	 *
	 * Targets conforming to these obsolete drafts can be
	 * recognized by the I/O Class they report.
	 */
	if (target->io_class == SRP_REV10_IB_IO_CLASS) {
		memcpy(req->priv.initiator_port_id,
760
		       &target->sgid.global.interface_id, 8);
761
		memcpy(req->priv.initiator_port_id + 8,
762
		       &target->initiator_ext, 8);
763 764 765 766
		memcpy(req->priv.target_port_id,     &target->ioc_guid, 8);
		memcpy(req->priv.target_port_id + 8, &target->id_ext, 8);
	} else {
		memcpy(req->priv.initiator_port_id,
767 768
		       &target->initiator_ext, 8);
		memcpy(req->priv.initiator_port_id + 8,
769
		       &target->sgid.global.interface_id, 8);
770 771 772 773
		memcpy(req->priv.target_port_id,     &target->id_ext, 8);
		memcpy(req->priv.target_port_id + 8, &target->ioc_guid, 8);
	}

774 775
	/*
	 * Topspin/Cisco SRP targets will reject our login unless we
776 777
	 * zero out the first 8 bytes of our initiator port ID and set
	 * the second 8 bytes to the local node GUID.
778
	 */
779
	if (srp_target_is_topspin(target)) {
780 781 782
		shost_printk(KERN_DEBUG, target->scsi_host,
			     PFX "Topspin/Cisco initiator port ID workaround "
			     "activated for target GUID %016llx\n",
783
			     be64_to_cpu(target->ioc_guid));
784
		memset(req->priv.initiator_port_id, 0, 8);
785
		memcpy(req->priv.initiator_port_id + 8,
786
		       &target->srp_host->srp_dev->dev->node_guid, 8);
787 788
	}

789
	status = ib_send_cm_req(ch->cm_id, &req->param);
790 791 792 793 794 795

	kfree(req);

	return status;
}

796 797 798 799 800 801 802 803 804 805 806 807
static bool srp_queue_remove_work(struct srp_target_port *target)
{
	bool changed = false;

	spin_lock_irq(&target->lock);
	if (target->state != SRP_TARGET_REMOVED) {
		target->state = SRP_TARGET_REMOVED;
		changed = true;
	}
	spin_unlock_irq(&target->lock);

	if (changed)
808
		queue_work(srp_remove_wq, &target->remove_work);
809 810 811 812

	return changed;
}

813 814
static void srp_disconnect_target(struct srp_target_port *target)
{
B
Bart Van Assche 已提交
815 816
	struct srp_rdma_ch *ch;
	int i;
817

818
	/* XXX should send SRP_I_LOGOUT request */
819

820 821 822 823 824 825
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
		ch->connected = false;
		if (ch->cm_id && ib_send_cm_dreq(ch->cm_id, NULL, 0)) {
			shost_printk(KERN_DEBUG, target->scsi_host,
				     PFX "Sending CM DREQ failed\n");
826
		}
827
	}
828 829
}

830 831
static void srp_free_req_data(struct srp_target_port *target,
			      struct srp_rdma_ch *ch)
832
{
833 834
	struct srp_device *dev = target->srp_host->srp_dev;
	struct ib_device *ibdev = dev->dev;
835 836 837
	struct srp_request *req;
	int i;

838
	if (!ch->req_ring)
839 840 841
		return;

	for (i = 0; i < target->req_ring_size; ++i) {
842
		req = &ch->req_ring[i];
843
		if (dev->use_fast_reg) {
844
			kfree(req->fr_list);
845
		} else {
846
			kfree(req->fmr_list);
847 848
			kfree(req->map_page);
		}
849 850 851 852 853 854
		if (req->indirect_dma_addr) {
			ib_dma_unmap_single(ibdev, req->indirect_dma_addr,
					    target->indirect_size,
					    DMA_TO_DEVICE);
		}
		kfree(req->indirect_desc);
855
	}
856

857 858
	kfree(ch->req_ring);
	ch->req_ring = NULL;
859 860
}

861
static int srp_alloc_req_data(struct srp_rdma_ch *ch)
862
{
863
	struct srp_target_port *target = ch->target;
864 865 866
	struct srp_device *srp_dev = target->srp_host->srp_dev;
	struct ib_device *ibdev = srp_dev->dev;
	struct srp_request *req;
867
	void *mr_list;
868 869 870
	dma_addr_t dma_addr;
	int i, ret = -ENOMEM;

871 872 873
	ch->req_ring = kcalloc(target->req_ring_size, sizeof(*ch->req_ring),
			       GFP_KERNEL);
	if (!ch->req_ring)
874 875 876
		goto out;

	for (i = 0; i < target->req_ring_size; ++i) {
877
		req = &ch->req_ring[i];
878 879 880 881
		mr_list = kmalloc(target->cmd_sg_cnt * sizeof(void *),
				  GFP_KERNEL);
		if (!mr_list)
			goto out;
882
		if (srp_dev->use_fast_reg) {
883
			req->fr_list = mr_list;
884
		} else {
885
			req->fmr_list = mr_list;
886 887 888 889 890
			req->map_page = kmalloc(srp_dev->max_pages_per_mr *
						sizeof(void *), GFP_KERNEL);
			if (!req->map_page)
				goto out;
		}
891
		req->indirect_desc = kmalloc(target->indirect_size, GFP_KERNEL);
892
		if (!req->indirect_desc)
893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908
			goto out;

		dma_addr = ib_dma_map_single(ibdev, req->indirect_desc,
					     target->indirect_size,
					     DMA_TO_DEVICE);
		if (ib_dma_mapping_error(ibdev, dma_addr))
			goto out;

		req->indirect_dma_addr = dma_addr;
	}
	ret = 0;

out:
	return ret;
}

909 910 911 912 913 914 915 916 917 918 919 920 921 922 923
/**
 * srp_del_scsi_host_attr() - Remove attributes defined in the host template.
 * @shost: SCSI host whose attributes to remove from sysfs.
 *
 * Note: Any attributes defined in the host template and that did not exist
 * before invocation of this function will be ignored.
 */
static void srp_del_scsi_host_attr(struct Scsi_Host *shost)
{
	struct device_attribute **attr;

	for (attr = shost->hostt->shost_attrs; attr && *attr; ++attr)
		device_remove_file(&shost->shost_dev, *attr);
}

924 925
static void srp_remove_target(struct srp_target_port *target)
{
B
Bart Van Assche 已提交
926 927
	struct srp_rdma_ch *ch;
	int i;
928

929 930
	WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);

931
	srp_del_scsi_host_attr(target->scsi_host);
932
	srp_rport_get(target->rport);
933 934
	srp_remove_host(target->scsi_host);
	scsi_remove_host(target->scsi_host);
935
	srp_stop_rport_timers(target->rport);
936
	srp_disconnect_target(target);
B
Bart Van Assche 已提交
937 938 939 940
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
		srp_free_ch_ib(target, ch);
	}
941
	cancel_work_sync(&target->tl_err_work);
942
	srp_rport_put(target->rport);
B
Bart Van Assche 已提交
943 944 945 946 947 948
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
		srp_free_req_data(target, ch);
	}
	kfree(target->ch);
	target->ch = NULL;
949 950 951 952 953

	spin_lock(&target->srp_host->target_lock);
	list_del(&target->list);
	spin_unlock(&target->srp_host->target_lock);

954 955 956
	scsi_host_put(target->scsi_host);
}

D
David Howells 已提交
957
static void srp_remove_work(struct work_struct *work)
958
{
D
David Howells 已提交
959
	struct srp_target_port *target =
960
		container_of(work, struct srp_target_port, remove_work);
961

962
	WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
963

964
	srp_remove_target(target);
965 966
}

967 968 969 970 971 972 973
static void srp_rport_delete(struct srp_rport *rport)
{
	struct srp_target_port *target = rport->lld_data;

	srp_queue_remove_work(target);
}

974 975 976 977 978 979 980 981 982 983 984 985 986 987
/**
 * srp_connected_ch() - number of connected channels
 * @target: SRP target port.
 */
static int srp_connected_ch(struct srp_target_port *target)
{
	int i, c = 0;

	for (i = 0; i < target->ch_count; i++)
		c += target->ch[i].connected;

	return c;
}

B
Bart Van Assche 已提交
988
static int srp_connect_ch(struct srp_rdma_ch *ch, bool multich)
989
{
990
	struct srp_target_port *target = ch->target;
991 992
	int ret;

993
	WARN_ON_ONCE(!multich && srp_connected_ch(target) > 0);
994

995
	ret = srp_lookup_path(ch);
996
	if (ret)
B
Bart Van Assche 已提交
997
		goto out;
998 999

	while (1) {
1000
		init_completion(&ch->done);
B
Bart Van Assche 已提交
1001
		ret = srp_send_req(ch, multich);
1002
		if (ret)
B
Bart Van Assche 已提交
1003
			goto out;
1004
		ret = wait_for_completion_interruptible(&ch->done);
1005
		if (ret < 0)
B
Bart Van Assche 已提交
1006
			goto out;
1007 1008 1009 1010 1011 1012 1013

		/*
		 * The CM event handling code will set status to
		 * SRP_PORT_REDIRECT if we get a port redirect REJ
		 * back, or SRP_DLID_REDIRECT if we get a lid/qp
		 * redirect REJ back.
		 */
B
Bart Van Assche 已提交
1014 1015
		ret = ch->status;
		switch (ret) {
1016
		case 0:
1017
			ch->connected = true;
B
Bart Van Assche 已提交
1018
			goto out;
1019 1020

		case SRP_PORT_REDIRECT:
1021
			ret = srp_lookup_path(ch);
1022
			if (ret)
B
Bart Van Assche 已提交
1023
				goto out;
1024 1025 1026 1027 1028
			break;

		case SRP_DLID_REDIRECT:
			break;

D
David Dillow 已提交
1029 1030
		case SRP_STALE_CONN:
			shost_printk(KERN_ERR, target->scsi_host, PFX
1031
				     "giving up on stale connection\n");
B
Bart Van Assche 已提交
1032 1033
			ret = -ECONNRESET;
			goto out;
D
David Dillow 已提交
1034

1035
		default:
B
Bart Van Assche 已提交
1036
			goto out;
1037 1038
		}
	}
B
Bart Van Assche 已提交
1039 1040 1041

out:
	return ret <= 0 ? ret : -ENODEV;
1042 1043
}

1044
static int srp_inv_rkey(struct srp_rdma_ch *ch, u32 rkey)
1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055
{
	struct ib_send_wr *bad_wr;
	struct ib_send_wr wr = {
		.opcode		    = IB_WR_LOCAL_INV,
		.wr_id		    = LOCAL_INV_WR_ID_MASK,
		.next		    = NULL,
		.num_sge	    = 0,
		.send_flags	    = 0,
		.ex.invalidate_rkey = rkey,
	};

1056
	return ib_post_send(ch->qp, &wr, &bad_wr);
1057 1058
}

1059
static void srp_unmap_data(struct scsi_cmnd *scmnd,
1060
			   struct srp_rdma_ch *ch,
1061 1062
			   struct srp_request *req)
{
1063
	struct srp_target_port *target = ch->target;
1064 1065 1066
	struct srp_device *dev = target->srp_host->srp_dev;
	struct ib_device *ibdev = dev->dev;
	int i, res;
1067

1068
	if (!scsi_sglist(scmnd) ||
1069 1070 1071 1072
	    (scmnd->sc_data_direction != DMA_TO_DEVICE &&
	     scmnd->sc_data_direction != DMA_FROM_DEVICE))
		return;

1073 1074 1075 1076
	if (dev->use_fast_reg) {
		struct srp_fr_desc **pfr;

		for (i = req->nmdesc, pfr = req->fr_list; i > 0; i--, pfr++) {
1077
			res = srp_inv_rkey(ch, (*pfr)->mr->rkey);
1078 1079 1080 1081 1082 1083 1084 1085 1086
			if (res < 0) {
				shost_printk(KERN_ERR, target->scsi_host, PFX
				  "Queueing INV WR for rkey %#x failed (%d)\n",
				  (*pfr)->mr->rkey, res);
				queue_work(system_long_wq,
					   &target->tl_err_work);
			}
		}
		if (req->nmdesc)
1087
			srp_fr_pool_put(ch->fr_pool, req->fr_list,
1088
					req->nmdesc);
1089
	} else if (dev->use_fmr) {
1090 1091 1092 1093 1094
		struct ib_pool_fmr **pfmr;

		for (i = req->nmdesc, pfmr = req->fmr_list; i > 0; i--, pfmr++)
			ib_fmr_pool_unmap(*pfmr);
	}
1095

1096 1097
	ib_dma_unmap_sg(ibdev, scsi_sglist(scmnd), scsi_sg_count(scmnd),
			scmnd->sc_data_direction);
1098 1099
}

B
Bart Van Assche 已提交
1100 1101
/**
 * srp_claim_req - Take ownership of the scmnd associated with a request.
1102
 * @ch: SRP RDMA channel.
B
Bart Van Assche 已提交
1103
 * @req: SRP request.
1104
 * @sdev: If not NULL, only take ownership for this SCSI device.
B
Bart Van Assche 已提交
1105 1106 1107 1108 1109 1110
 * @scmnd: If NULL, take ownership of @req->scmnd. If not NULL, only take
 *         ownership of @req->scmnd if it equals @scmnd.
 *
 * Return value:
 * Either NULL or a pointer to the SCSI command the caller became owner of.
 */
1111
static struct scsi_cmnd *srp_claim_req(struct srp_rdma_ch *ch,
B
Bart Van Assche 已提交
1112
				       struct srp_request *req,
1113
				       struct scsi_device *sdev,
B
Bart Van Assche 已提交
1114 1115 1116 1117
				       struct scsi_cmnd *scmnd)
{
	unsigned long flags;

1118
	spin_lock_irqsave(&ch->lock, flags);
1119 1120 1121
	if (req->scmnd &&
	    (!sdev || req->scmnd->device == sdev) &&
	    (!scmnd || req->scmnd == scmnd)) {
B
Bart Van Assche 已提交
1122 1123 1124 1125 1126
		scmnd = req->scmnd;
		req->scmnd = NULL;
	} else {
		scmnd = NULL;
	}
1127
	spin_unlock_irqrestore(&ch->lock, flags);
B
Bart Van Assche 已提交
1128 1129 1130 1131 1132 1133

	return scmnd;
}

/**
 * srp_free_req() - Unmap data and add request to the free request list.
1134
 * @ch:     SRP RDMA channel.
1135 1136 1137
 * @req:    Request to be freed.
 * @scmnd:  SCSI command associated with @req.
 * @req_lim_delta: Amount to be added to @target->req_lim.
B
Bart Van Assche 已提交
1138
 */
1139 1140
static void srp_free_req(struct srp_rdma_ch *ch, struct srp_request *req,
			 struct scsi_cmnd *scmnd, s32 req_lim_delta)
1141
{
1142 1143
	unsigned long flags;

1144
	srp_unmap_data(scmnd, ch, req);
B
Bart Van Assche 已提交
1145

1146 1147 1148
	spin_lock_irqsave(&ch->lock, flags);
	ch->req_lim += req_lim_delta;
	spin_unlock_irqrestore(&ch->lock, flags);
1149 1150
}

1151 1152
static void srp_finish_req(struct srp_rdma_ch *ch, struct srp_request *req,
			   struct scsi_device *sdev, int result)
1153
{
1154
	struct scsi_cmnd *scmnd = srp_claim_req(ch, req, sdev, NULL);
B
Bart Van Assche 已提交
1155 1156

	if (scmnd) {
1157
		srp_free_req(ch, req, scmnd, 0);
1158
		scmnd->result = result;
B
Bart Van Assche 已提交
1159 1160
		scmnd->scsi_done(scmnd);
	}
1161 1162
}

1163
static void srp_terminate_io(struct srp_rport *rport)
1164
{
1165
	struct srp_target_port *target = rport->lld_data;
B
Bart Van Assche 已提交
1166
	struct srp_rdma_ch *ch;
1167 1168
	struct Scsi_Host *shost = target->scsi_host;
	struct scsi_device *sdev;
B
Bart Van Assche 已提交
1169
	int i, j;
1170

1171 1172 1173 1174 1175 1176 1177
	/*
	 * Invoking srp_terminate_io() while srp_queuecommand() is running
	 * is not safe. Hence the warning statement below.
	 */
	shost_for_each_device(sdev, shost)
		WARN_ON_ONCE(sdev->request_queue->request_fn_active);

B
Bart Van Assche 已提交
1178 1179
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
1180

B
Bart Van Assche 已提交
1181 1182 1183 1184 1185 1186
		for (j = 0; j < target->req_ring_size; ++j) {
			struct srp_request *req = &ch->req_ring[j];

			srp_finish_req(ch, req, NULL,
				       DID_TRANSPORT_FAILFAST << 16);
		}
1187 1188
	}
}
1189

1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201
/*
 * It is up to the caller to ensure that srp_rport_reconnect() calls are
 * serialized and that no concurrent srp_queuecommand(), srp_abort(),
 * srp_reset_device() or srp_reset_host() calls will occur while this function
 * is in progress. One way to realize that is not to call this function
 * directly but to call srp_reconnect_rport() instead since that last function
 * serializes calls of this function via rport->mutex and also blocks
 * srp_queuecommand() calls before invoking this function.
 */
static int srp_rport_reconnect(struct srp_rport *rport)
{
	struct srp_target_port *target = rport->lld_data;
B
Bart Van Assche 已提交
1202 1203 1204
	struct srp_rdma_ch *ch;
	int i, j, ret = 0;
	bool multich = false;
1205

1206
	srp_disconnect_target(target);
1207 1208 1209 1210

	if (target->state == SRP_TARGET_SCANNING)
		return -ENODEV;

1211
	/*
1212 1213 1214
	 * Now get a new local CM ID so that we avoid confusing the target in
	 * case things are really fouled up. Doing so also ensures that all CM
	 * callbacks will have finished before a new QP is allocated.
1215
	 */
B
Bart Van Assche 已提交
1216 1217 1218
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
		ret += srp_new_cm_id(ch);
1219
	}
B
Bart Van Assche 已提交
1220 1221 1222 1223
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
		for (j = 0; j < target->req_ring_size; ++j) {
			struct srp_request *req = &ch->req_ring[j];
1224

B
Bart Van Assche 已提交
1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235
			srp_finish_req(ch, req, NULL, DID_RESET << 16);
		}
	}
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
		/*
		 * Whether or not creating a new CM ID succeeded, create a new
		 * QP. This guarantees that all completion callback function
		 * invocations have finished before request resetting starts.
		 */
		ret += srp_create_ch_ib(ch);
1236

B
Bart Van Assche 已提交
1237 1238 1239 1240
		INIT_LIST_HEAD(&ch->free_tx);
		for (j = 0; j < target->queue_size; ++j)
			list_add(&ch->tx_ring[j]->list, &ch->free_tx);
	}
1241 1242 1243

	target->qp_in_error = false;

B
Bart Van Assche 已提交
1244 1245
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
1246
		if (ret)
B
Bart Van Assche 已提交
1247 1248 1249 1250
			break;
		ret = srp_connect_ch(ch, multich);
		multich = true;
	}
1251

1252 1253 1254
	if (ret == 0)
		shost_printk(KERN_INFO, target->scsi_host,
			     PFX "reconnect succeeded\n");
1255 1256 1257 1258

	return ret;
}

1259 1260
static void srp_map_desc(struct srp_map_state *state, dma_addr_t dma_addr,
			 unsigned int dma_len, u32 rkey)
1261
{
1262
	struct srp_direct_buf *desc = state->desc;
1263

1264 1265
	WARN_ON_ONCE(!dma_len);

1266 1267 1268
	desc->va = cpu_to_be64(dma_addr);
	desc->key = cpu_to_be32(rkey);
	desc->len = cpu_to_be32(dma_len);
1269

1270 1271 1272 1273
	state->total_len += dma_len;
	state->desc++;
	state->ndesc++;
}
1274

1275
static int srp_map_finish_fmr(struct srp_map_state *state,
1276
			      struct srp_rdma_ch *ch)
1277
{
1278 1279
	struct srp_target_port *target = ch->target;
	struct srp_device *dev = target->srp_host->srp_dev;
1280 1281
	struct ib_pool_fmr *fmr;
	u64 io_addr = 0;
1282

1283 1284 1285
	if (state->fmr.next >= state->fmr.end)
		return -ENOMEM;

S
Sagi Grimberg 已提交
1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296
	WARN_ON_ONCE(!dev->use_fmr);

	if (state->npages == 0)
		return 0;

	if (state->npages == 1 && target->global_mr) {
		srp_map_desc(state, state->base_dma_addr, state->dma_len,
			     target->global_mr->rkey);
		goto reset_state;
	}

1297
	fmr = ib_fmr_pool_map_phys(ch->fmr_pool, state->pages,
1298 1299 1300
				   state->npages, io_addr);
	if (IS_ERR(fmr))
		return PTR_ERR(fmr);
1301

1302
	*state->fmr.next++ = fmr;
1303
	state->nmdesc++;
1304

1305 1306
	srp_map_desc(state, state->base_dma_addr & ~dev->mr_page_mask,
		     state->dma_len, fmr->fmr->rkey);
1307

S
Sagi Grimberg 已提交
1308 1309 1310 1311
reset_state:
	state->npages = 0;
	state->dma_len = 0;

1312 1313 1314
	return 0;
}

1315
static int srp_map_finish_fr(struct srp_map_state *state,
1316
			     struct srp_rdma_ch *ch)
1317
{
1318
	struct srp_target_port *target = ch->target;
1319 1320
	struct srp_device *dev = target->srp_host->srp_dev;
	struct ib_send_wr *bad_wr;
1321
	struct ib_reg_wr wr;
1322 1323
	struct srp_fr_desc *desc;
	u32 rkey;
1324
	int n, err;
1325

1326 1327 1328
	if (state->fr.next >= state->fr.end)
		return -ENOMEM;

S
Sagi Grimberg 已提交
1329 1330
	WARN_ON_ONCE(!dev->use_fast_reg);

1331
	if (state->sg_nents == 0)
S
Sagi Grimberg 已提交
1332 1333
		return 0;

1334 1335 1336
	if (state->sg_nents == 1 && target->global_mr) {
		srp_map_desc(state, sg_dma_address(state->sg),
			     sg_dma_len(state->sg),
S
Sagi Grimberg 已提交
1337
			     target->global_mr->rkey);
1338
		return 1;
S
Sagi Grimberg 已提交
1339 1340
	}

1341
	desc = srp_fr_pool_get(ch->fr_pool);
1342 1343 1344 1345 1346 1347
	if (!desc)
		return -ENOMEM;

	rkey = ib_inc_rkey(desc->mr->rkey);
	ib_update_fast_reg_key(desc->mr, rkey);

1348 1349 1350 1351
	n = ib_map_mr_sg(desc->mr, state->sg, state->sg_nents,
			 dev->mr_page_size);
	if (unlikely(n < 0))
		return n;
1352

1353 1354
	wr.wr.next = NULL;
	wr.wr.opcode = IB_WR_REG_MR;
C
Christoph Hellwig 已提交
1355
	wr.wr.wr_id = FAST_REG_WR_ID_MASK;
1356 1357 1358 1359 1360 1361 1362
	wr.wr.num_sge = 0;
	wr.wr.send_flags = 0;
	wr.mr = desc->mr;
	wr.key = desc->mr->rkey;
	wr.access = (IB_ACCESS_LOCAL_WRITE |
		     IB_ACCESS_REMOTE_READ |
		     IB_ACCESS_REMOTE_WRITE);
1363

1364
	*state->fr.next++ = desc;
1365 1366
	state->nmdesc++;

1367 1368
	srp_map_desc(state, desc->mr->iova,
		     desc->mr->length, desc->mr->rkey);
1369

S
Sagi Grimberg 已提交
1370
	err = ib_post_send(ch->qp, &wr.wr, &bad_wr);
1371
	if (unlikely(err))
S
Sagi Grimberg 已提交
1372 1373
		return err;

1374
	return n;
1375 1376
}

1377
static int srp_map_sg_entry(struct srp_map_state *state,
1378
			    struct srp_rdma_ch *ch,
1379
			    struct scatterlist *sg, int sg_index)
1380
{
1381
	struct srp_target_port *target = ch->target;
1382 1383 1384 1385
	struct srp_device *dev = target->srp_host->srp_dev;
	struct ib_device *ibdev = dev->dev;
	dma_addr_t dma_addr = ib_sg_dma_address(ibdev, sg);
	unsigned int dma_len = ib_sg_dma_len(ibdev, sg);
1386
	unsigned int len = 0;
1387 1388
	int ret;

1389
	WARN_ON_ONCE(!dma_len);
1390

1391
	while (dma_len) {
1392 1393
		unsigned offset = dma_addr & ~dev->mr_page_mask;
		if (state->npages == dev->max_pages_per_mr || offset != 0) {
1394
			ret = srp_map_finish_fmr(state, ch);
1395 1396 1397 1398
			if (ret)
				return ret;
		}

1399
		len = min_t(unsigned int, dma_len, dev->mr_page_size - offset);
1400

1401 1402
		if (!state->npages)
			state->base_dma_addr = dma_addr;
1403
		state->pages[state->npages++] = dma_addr & dev->mr_page_mask;
1404
		state->dma_len += len;
1405 1406 1407 1408
		dma_addr += len;
		dma_len -= len;
	}

1409 1410
	/*
	 * If the last entry of the MR wasn't a full page, then we need to
1411 1412 1413 1414
	 * close it out and start a new one -- we can only merge at page
	 * boundries.
	 */
	ret = 0;
1415
	if (len != dev->mr_page_size)
1416
		ret = srp_map_finish_fmr(state, ch);
1417 1418 1419
	return ret;
}

S
Sagi Grimberg 已提交
1420 1421 1422
static int srp_map_sg_fmr(struct srp_map_state *state, struct srp_rdma_ch *ch,
			  struct srp_request *req, struct scatterlist *scat,
			  int count)
1423 1424
{
	struct scatterlist *sg;
1425
	int i, ret;
1426

S
Sagi Grimberg 已提交
1427 1428 1429 1430 1431 1432 1433 1434 1435
	state->desc = req->indirect_desc;
	state->pages = req->map_page;
	state->fmr.next = req->fmr_list;
	state->fmr.end = req->fmr_list + ch->target->cmd_sg_cnt;

	for_each_sg(scat, sg, count, i) {
		ret = srp_map_sg_entry(state, ch, sg, i);
		if (ret)
			return ret;
1436
	}
1437

1438
	ret = srp_map_finish_fmr(state, ch);
S
Sagi Grimberg 已提交
1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451
	if (ret)
		return ret;

	req->nmdesc = state->nmdesc;

	return 0;
}

static int srp_map_sg_fr(struct srp_map_state *state, struct srp_rdma_ch *ch,
			 struct srp_request *req, struct scatterlist *scat,
			 int count)
{
	state->desc = req->indirect_desc;
1452 1453 1454 1455
	state->fr.next = req->fr_list;
	state->fr.end = req->fr_list + ch->target->cmd_sg_cnt;
	state->sg = scat;
	state->sg_nents = scsi_sg_count(req->scmnd);
S
Sagi Grimberg 已提交
1456

1457 1458
	while (state->sg_nents) {
		int i, n;
S
Sagi Grimberg 已提交
1459

1460 1461 1462 1463 1464 1465 1466 1467
		n = srp_map_finish_fr(state, ch);
		if (unlikely(n < 0))
			return n;

		state->sg_nents -= n;
		for (i = 0; i < n; i++)
			state->sg = sg_next(state->sg);
	}
S
Sagi Grimberg 已提交
1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487

	req->nmdesc = state->nmdesc;

	return 0;
}

static int srp_map_sg_dma(struct srp_map_state *state, struct srp_rdma_ch *ch,
			  struct srp_request *req, struct scatterlist *scat,
			  int count)
{
	struct srp_target_port *target = ch->target;
	struct srp_device *dev = target->srp_host->srp_dev;
	struct scatterlist *sg;
	int i;

	state->desc = req->indirect_desc;
	for_each_sg(scat, sg, count, i) {
		srp_map_desc(state, ib_sg_dma_address(dev->dev, sg),
			     ib_sg_dma_len(dev->dev, sg),
			     target->global_mr->rkey);
1488
	}
1489

1490
	req->nmdesc = state->nmdesc;
1491

S
Sagi Grimberg 已提交
1492
	return 0;
1493 1494
}

1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510
/*
 * Register the indirect data buffer descriptor with the HCA.
 *
 * Note: since the indirect data buffer descriptor has been allocated with
 * kmalloc() it is guaranteed that this buffer is a physically contiguous
 * memory buffer.
 */
static int srp_map_idb(struct srp_rdma_ch *ch, struct srp_request *req,
		       void **next_mr, void **end_mr, u32 idb_len,
		       __be32 *idb_rkey)
{
	struct srp_target_port *target = ch->target;
	struct srp_device *dev = target->srp_host->srp_dev;
	struct srp_map_state state;
	struct srp_direct_buf idb_desc;
	u64 idb_pages[1];
1511
	struct scatterlist idb_sg[1];
1512 1513 1514 1515 1516 1517 1518 1519 1520
	int ret;

	memset(&state, 0, sizeof(state));
	memset(&idb_desc, 0, sizeof(idb_desc));
	state.gen.next = next_mr;
	state.gen.end = end_mr;
	state.desc = &idb_desc;
	state.base_dma_addr = req->indirect_dma_addr;
	state.dma_len = idb_len;
1521 1522 1523 1524 1525 1526

	if (dev->use_fast_reg) {
		state.sg = idb_sg;
		state.sg_nents = 1;
		sg_set_buf(idb_sg, req->indirect_desc, idb_len);
		idb_sg->dma_address = req->indirect_dma_addr; /* hack! */
1527 1528 1529
#ifdef CONFIG_NEED_SG_DMA_LENGTH
		idb_sg->dma_length = idb_sg->length;	      /* hack^2 */
#endif
1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543
		ret = srp_map_finish_fr(&state, ch);
		if (ret < 0)
			return ret;
	} else if (dev->use_fmr) {
		state.pages = idb_pages;
		state.pages[0] = (req->indirect_dma_addr &
				  dev->mr_page_mask);
		state.npages = 1;
		ret = srp_map_finish_fmr(&state, ch);
		if (ret < 0)
			return ret;
	} else {
		return -EINVAL;
	}
1544 1545 1546

	*idb_rkey = idb_desc.key;

1547
	return 0;
1548 1549
}

1550
static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch,
1551 1552
			struct srp_request *req)
{
1553
	struct srp_target_port *target = ch->target;
1554
	struct scatterlist *scat;
1555
	struct srp_cmd *cmd = req->cmd->buf;
1556
	int len, nents, count, ret;
1557 1558
	struct srp_device *dev;
	struct ib_device *ibdev;
1559 1560
	struct srp_map_state state;
	struct srp_indirect_buf *indirect_hdr;
1561 1562
	u32 idb_len, table_len;
	__be32 idb_rkey;
1563
	u8 fmt;
1564

1565
	if (!scsi_sglist(scmnd) || scmnd->sc_data_direction == DMA_NONE)
1566 1567 1568 1569
		return sizeof (struct srp_cmd);

	if (scmnd->sc_data_direction != DMA_FROM_DEVICE &&
	    scmnd->sc_data_direction != DMA_TO_DEVICE) {
1570 1571 1572
		shost_printk(KERN_WARNING, target->scsi_host,
			     PFX "Unhandled data direction %d\n",
			     scmnd->sc_data_direction);
1573 1574 1575
		return -EINVAL;
	}

1576 1577
	nents = scsi_sg_count(scmnd);
	scat  = scsi_sglist(scmnd);
1578

1579
	dev = target->srp_host->srp_dev;
1580 1581 1582
	ibdev = dev->dev;

	count = ib_dma_map_sg(ibdev, scat, nents, scmnd->sc_data_direction);
1583 1584
	if (unlikely(count == 0))
		return -EIO;
1585 1586 1587

	fmt = SRP_DATA_DESC_DIRECT;
	len = sizeof (struct srp_cmd) +	sizeof (struct srp_direct_buf);
1588

1589
	if (count == 1 && target->global_mr) {
1590 1591 1592 1593 1594 1595
		/*
		 * The midlayer only generated a single gather/scatter
		 * entry, or DMA mapping coalesced everything to a
		 * single entry.  So a direct descriptor along with
		 * the DMA MR suffices.
		 */
1596
		struct srp_direct_buf *buf = (void *) cmd->add_data;
1597

1598
		buf->va  = cpu_to_be64(ib_sg_dma_address(ibdev, scat));
1599
		buf->key = cpu_to_be32(target->global_mr->rkey);
1600
		buf->len = cpu_to_be32(ib_sg_dma_len(ibdev, scat));
1601

1602
		req->nmdesc = 0;
1603 1604 1605
		goto map_complete;
	}

1606 1607 1608
	/*
	 * We have more than one scatter/gather entry, so build our indirect
	 * descriptor table, trying to merge as many entries as we can.
1609 1610 1611
	 */
	indirect_hdr = (void *) cmd->add_data;

1612 1613 1614
	ib_dma_sync_single_for_cpu(ibdev, req->indirect_dma_addr,
				   target->indirect_size, DMA_TO_DEVICE);

1615
	memset(&state, 0, sizeof(state));
S
Sagi Grimberg 已提交
1616 1617 1618 1619 1620 1621
	if (dev->use_fast_reg)
		srp_map_sg_fr(&state, ch, req, scat, count);
	else if (dev->use_fmr)
		srp_map_sg_fmr(&state, ch, req, scat, count);
	else
		srp_map_sg_dma(&state, ch, req, scat, count);
1622

1623 1624 1625 1626 1627
	/* We've mapped the request, now pull as much of the indirect
	 * descriptor table as we can into the command buffer. If this
	 * target is not using an external indirect table, we are
	 * guaranteed to fit into the command, as the SCSI layer won't
	 * give us more S/G entries than we allow.
1628 1629
	 */
	if (state.ndesc == 1) {
1630 1631
		/*
		 * Memory registration collapsed the sg-list into one entry,
1632 1633 1634
		 * so use a direct descriptor.
		 */
		struct srp_direct_buf *buf = (void *) cmd->add_data;
1635

1636
		*buf = req->indirect_desc[0];
1637
		goto map_complete;
1638 1639
	}

1640 1641 1642 1643 1644 1645 1646 1647
	if (unlikely(target->cmd_sg_cnt < state.ndesc &&
						!target->allow_ext_sg)) {
		shost_printk(KERN_ERR, target->scsi_host,
			     "Could not fit S/G list into SRP_CMD\n");
		return -EIO;
	}

	count = min(state.ndesc, target->cmd_sg_cnt);
1648
	table_len = state.ndesc * sizeof (struct srp_direct_buf);
1649
	idb_len = sizeof(struct srp_indirect_buf) + table_len;
1650 1651 1652

	fmt = SRP_DATA_DESC_INDIRECT;
	len = sizeof(struct srp_cmd) + sizeof (struct srp_indirect_buf);
1653
	len += count * sizeof (struct srp_direct_buf);
1654

1655 1656
	memcpy(indirect_hdr->desc_list, req->indirect_desc,
	       count * sizeof (struct srp_direct_buf));
1657

1658
	if (!target->global_mr) {
1659 1660 1661 1662 1663 1664
		ret = srp_map_idb(ch, req, state.gen.next, state.gen.end,
				  idb_len, &idb_rkey);
		if (ret < 0)
			return ret;
		req->nmdesc++;
	} else {
1665
		idb_rkey = cpu_to_be32(target->global_mr->rkey);
1666 1667
	}

1668
	indirect_hdr->table_desc.va = cpu_to_be64(req->indirect_dma_addr);
1669
	indirect_hdr->table_desc.key = idb_rkey;
1670 1671 1672 1673
	indirect_hdr->table_desc.len = cpu_to_be32(table_len);
	indirect_hdr->len = cpu_to_be32(state.total_len);

	if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1674
		cmd->data_out_desc_cnt = count;
1675
	else
1676 1677 1678 1679
		cmd->data_in_desc_cnt = count;

	ib_dma_sync_single_for_device(ibdev, req->indirect_dma_addr, table_len,
				      DMA_TO_DEVICE);
1680 1681

map_complete:
1682 1683 1684 1685 1686 1687 1688 1689
	if (scmnd->sc_data_direction == DMA_TO_DEVICE)
		cmd->buf_fmt = fmt << 4;
	else
		cmd->buf_fmt = fmt;

	return len;
}

1690 1691 1692
/*
 * Return an IU and possible credit to the free pool
 */
1693
static void srp_put_tx_iu(struct srp_rdma_ch *ch, struct srp_iu *iu,
1694 1695 1696 1697
			  enum srp_iu_type iu_type)
{
	unsigned long flags;

1698 1699
	spin_lock_irqsave(&ch->lock, flags);
	list_add(&iu->list, &ch->free_tx);
1700
	if (iu_type != SRP_IU_RSP)
1701 1702
		++ch->req_lim;
	spin_unlock_irqrestore(&ch->lock, flags);
1703 1704
}

1705
/*
1706
 * Must be called with ch->lock held to protect req_lim and free_tx.
1707
 * If IU is not sent, it must be returned using srp_put_tx_iu().
1708 1709 1710 1711 1712 1713 1714 1715 1716 1717
 *
 * Note:
 * An upper limit for the number of allocated information units for each
 * request type is:
 * - SRP_IU_CMD: SRP_CMD_SQ_SIZE, since the SCSI mid-layer never queues
 *   more than Scsi_Host.can_queue requests.
 * - SRP_IU_TSK_MGMT: SRP_TSK_MGMT_SQ_SIZE.
 * - SRP_IU_RSP: 1, since a conforming SRP target never sends more than
 *   one unanswered SRP request to an initiator.
 */
1718
static struct srp_iu *__srp_get_tx_iu(struct srp_rdma_ch *ch,
1719 1720
				      enum srp_iu_type iu_type)
{
1721
	struct srp_target_port *target = ch->target;
1722 1723 1724
	s32 rsv = (iu_type == SRP_IU_TSK_MGMT) ? 0 : SRP_TSK_MGMT_SQ_SIZE;
	struct srp_iu *iu;

1725
	srp_send_completion(ch->send_cq, ch);
1726

1727
	if (list_empty(&ch->free_tx))
1728 1729 1730
		return NULL;

	/* Initiator responses to target requests do not consume credits */
1731
	if (iu_type != SRP_IU_RSP) {
1732
		if (ch->req_lim <= rsv) {
1733 1734 1735 1736
			++target->zero_req_lim;
			return NULL;
		}

1737
		--ch->req_lim;
1738 1739
	}

1740
	iu = list_first_entry(&ch->free_tx, struct srp_iu, list);
1741
	list_del(&iu->list);
1742 1743 1744
	return iu;
}

1745
static int srp_post_send(struct srp_rdma_ch *ch, struct srp_iu *iu, int len)
1746
{
1747
	struct srp_target_port *target = ch->target;
1748 1749 1750 1751 1752
	struct ib_sge list;
	struct ib_send_wr wr, *bad_wr;

	list.addr   = iu->dma;
	list.length = len;
1753
	list.lkey   = target->lkey;
1754 1755

	wr.next       = NULL;
1756
	wr.wr_id      = (uintptr_t) iu;
1757 1758 1759 1760 1761
	wr.sg_list    = &list;
	wr.num_sge    = 1;
	wr.opcode     = IB_WR_SEND;
	wr.send_flags = IB_SEND_SIGNALED;

1762
	return ib_post_send(ch->qp, &wr, &bad_wr);
1763 1764
}

1765
static int srp_post_recv(struct srp_rdma_ch *ch, struct srp_iu *iu)
1766
{
1767
	struct srp_target_port *target = ch->target;
1768
	struct ib_recv_wr wr, *bad_wr;
1769
	struct ib_sge list;
1770 1771 1772

	list.addr   = iu->dma;
	list.length = iu->size;
1773
	list.lkey   = target->lkey;
1774 1775

	wr.next     = NULL;
1776
	wr.wr_id    = (uintptr_t) iu;
1777 1778 1779
	wr.sg_list  = &list;
	wr.num_sge  = 1;

1780
	return ib_post_recv(ch->qp, &wr, &bad_wr);
1781 1782
}

1783
static void srp_process_rsp(struct srp_rdma_ch *ch, struct srp_rsp *rsp)
1784
{
1785
	struct srp_target_port *target = ch->target;
1786 1787 1788 1789 1790
	struct srp_request *req;
	struct scsi_cmnd *scmnd;
	unsigned long flags;

	if (unlikely(rsp->tag & SRP_TAG_TSK_MGMT)) {
1791 1792 1793
		spin_lock_irqsave(&ch->lock, flags);
		ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
		spin_unlock_irqrestore(&ch->lock, flags);
1794

1795
		ch->tsk_mgmt_status = -1;
1796
		if (be32_to_cpu(rsp->resp_data_len) >= 4)
1797 1798
			ch->tsk_mgmt_status = rsp->data[3];
		complete(&ch->tsk_mgmt_done);
1799
	} else {
B
Bart Van Assche 已提交
1800 1801 1802 1803 1804
		scmnd = scsi_host_find_tag(target->scsi_host, rsp->tag);
		if (scmnd) {
			req = (void *)scmnd->host_scribble;
			scmnd = srp_claim_req(ch, req, NULL, scmnd);
		}
B
Bart Van Assche 已提交
1805
		if (!scmnd) {
1806
			shost_printk(KERN_ERR, target->scsi_host,
B
Bart Van Assche 已提交
1807 1808
				     "Null scmnd for RSP w/tag %#016llx received on ch %td / QP %#x\n",
				     rsp->tag, ch - target->ch, ch->qp->qp_num);
B
Bart Van Assche 已提交
1809

1810 1811 1812
			spin_lock_irqsave(&ch->lock, flags);
			ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
			spin_unlock_irqrestore(&ch->lock, flags);
B
Bart Van Assche 已提交
1813 1814 1815

			return;
		}
1816 1817 1818 1819 1820 1821 1822 1823 1824
		scmnd->result = rsp->status;

		if (rsp->flags & SRP_RSP_FLAG_SNSVALID) {
			memcpy(scmnd->sense_buffer, rsp->data +
			       be32_to_cpu(rsp->resp_data_len),
			       min_t(int, be32_to_cpu(rsp->sense_data_len),
				     SCSI_SENSE_BUFFERSIZE));
		}

B
Bart Van Assche 已提交
1825
		if (unlikely(rsp->flags & SRP_RSP_FLAG_DIUNDER))
1826
			scsi_set_resid(scmnd, be32_to_cpu(rsp->data_in_res_cnt));
B
Bart Van Assche 已提交
1827 1828 1829 1830 1831 1832
		else if (unlikely(rsp->flags & SRP_RSP_FLAG_DIOVER))
			scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_in_res_cnt));
		else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOUNDER))
			scsi_set_resid(scmnd, be32_to_cpu(rsp->data_out_res_cnt));
		else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOOVER))
			scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_out_res_cnt));
1833

1834
		srp_free_req(ch, req, scmnd,
B
Bart Van Assche 已提交
1835 1836
			     be32_to_cpu(rsp->req_lim_delta));

1837 1838
		scmnd->host_scribble = NULL;
		scmnd->scsi_done(scmnd);
1839 1840 1841
	}
}

1842
static int srp_response_common(struct srp_rdma_ch *ch, s32 req_delta,
1843 1844
			       void *rsp, int len)
{
1845
	struct srp_target_port *target = ch->target;
1846
	struct ib_device *dev = target->srp_host->srp_dev->dev;
1847 1848
	unsigned long flags;
	struct srp_iu *iu;
1849
	int err;
1850

1851 1852 1853 1854
	spin_lock_irqsave(&ch->lock, flags);
	ch->req_lim += req_delta;
	iu = __srp_get_tx_iu(ch, SRP_IU_RSP);
	spin_unlock_irqrestore(&ch->lock, flags);
1855

1856 1857 1858
	if (!iu) {
		shost_printk(KERN_ERR, target->scsi_host, PFX
			     "no IU available to send response\n");
1859
		return 1;
1860 1861 1862 1863 1864 1865
	}

	ib_dma_sync_single_for_cpu(dev, iu->dma, len, DMA_TO_DEVICE);
	memcpy(iu->buf, rsp, len);
	ib_dma_sync_single_for_device(dev, iu->dma, len, DMA_TO_DEVICE);

1866
	err = srp_post_send(ch, iu, len);
1867
	if (err) {
1868 1869
		shost_printk(KERN_ERR, target->scsi_host, PFX
			     "unable to post response: %d\n", err);
1870
		srp_put_tx_iu(ch, iu, SRP_IU_RSP);
1871
	}
1872 1873 1874 1875

	return err;
}

1876
static void srp_process_cred_req(struct srp_rdma_ch *ch,
1877 1878 1879 1880 1881 1882 1883 1884
				 struct srp_cred_req *req)
{
	struct srp_cred_rsp rsp = {
		.opcode = SRP_CRED_RSP,
		.tag = req->tag,
	};
	s32 delta = be32_to_cpu(req->req_lim_delta);

1885 1886
	if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
		shost_printk(KERN_ERR, ch->target->scsi_host, PFX
1887 1888 1889
			     "problems processing SRP_CRED_REQ\n");
}

1890
static void srp_process_aer_req(struct srp_rdma_ch *ch,
1891 1892
				struct srp_aer_req *req)
{
1893
	struct srp_target_port *target = ch->target;
1894 1895 1896 1897 1898 1899 1900
	struct srp_aer_rsp rsp = {
		.opcode = SRP_AER_RSP,
		.tag = req->tag,
	};
	s32 delta = be32_to_cpu(req->req_lim_delta);

	shost_printk(KERN_ERR, target->scsi_host, PFX
B
Bart Van Assche 已提交
1901
		     "ignoring AER for LUN %llu\n", scsilun_to_int(&req->lun));
1902

1903
	if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
1904 1905 1906 1907
		shost_printk(KERN_ERR, target->scsi_host, PFX
			     "problems processing SRP_AER_REQ\n");
}

1908
static void srp_handle_recv(struct srp_rdma_ch *ch, struct ib_wc *wc)
1909
{
1910
	struct srp_target_port *target = ch->target;
1911
	struct ib_device *dev = target->srp_host->srp_dev->dev;
1912
	struct srp_iu *iu = (struct srp_iu *) (uintptr_t) wc->wr_id;
1913
	int res;
1914 1915
	u8 opcode;

1916
	ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_ti_iu_len,
1917
				   DMA_FROM_DEVICE);
1918 1919 1920 1921

	opcode = *(u8 *) iu->buf;

	if (0) {
1922 1923
		shost_printk(KERN_ERR, target->scsi_host,
			     PFX "recv completion, opcode 0x%02x\n", opcode);
B
Bart Van Assche 已提交
1924 1925
		print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 8, 1,
			       iu->buf, wc->byte_len, true);
1926 1927 1928 1929
	}

	switch (opcode) {
	case SRP_RSP:
1930
		srp_process_rsp(ch, iu->buf);
1931 1932
		break;

1933
	case SRP_CRED_REQ:
1934
		srp_process_cred_req(ch, iu->buf);
1935 1936 1937
		break;

	case SRP_AER_REQ:
1938
		srp_process_aer_req(ch, iu->buf);
1939 1940
		break;

1941 1942
	case SRP_T_LOGOUT:
		/* XXX Handle target logout */
1943 1944
		shost_printk(KERN_WARNING, target->scsi_host,
			     PFX "Got target logout request\n");
1945 1946 1947
		break;

	default:
1948 1949
		shost_printk(KERN_WARNING, target->scsi_host,
			     PFX "Unhandled SRP opcode 0x%02x\n", opcode);
1950 1951 1952
		break;
	}

1953
	ib_dma_sync_single_for_device(dev, iu->dma, ch->max_ti_iu_len,
1954
				      DMA_FROM_DEVICE);
1955

1956
	res = srp_post_recv(ch, iu);
1957 1958 1959
	if (res != 0)
		shost_printk(KERN_ERR, target->scsi_host,
			     PFX "Recv failed with error code %d\n", res);
1960 1961
}

1962 1963
/**
 * srp_tl_err_work() - handle a transport layer error
1964
 * @work: Work structure embedded in an SRP target port.
1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977
 *
 * Note: This function may get invoked before the rport has been created,
 * hence the target->rport test.
 */
static void srp_tl_err_work(struct work_struct *work)
{
	struct srp_target_port *target;

	target = container_of(work, struct srp_target_port, tl_err_work);
	if (target->rport)
		srp_start_tl_fail_timers(target->rport);
}

1978
static void srp_handle_qp_err(u64 wr_id, enum ib_wc_status wc_status,
1979
			      bool send_err, struct srp_rdma_ch *ch)
1980
{
1981 1982 1983 1984 1985 1986 1987
	struct srp_target_port *target = ch->target;

	if (wr_id == SRP_LAST_WR_ID) {
		complete(&ch->done);
		return;
	}

1988
	if (ch->connected && !target->qp_in_error) {
1989 1990
		if (wr_id & LOCAL_INV_WR_ID_MASK) {
			shost_printk(KERN_ERR, target->scsi_host, PFX
1991 1992
				     "LOCAL_INV failed with status %s (%d)\n",
				     ib_wc_status_msg(wc_status), wc_status);
1993 1994
		} else if (wr_id & FAST_REG_WR_ID_MASK) {
			shost_printk(KERN_ERR, target->scsi_host, PFX
1995 1996
				     "FAST_REG_MR failed status %s (%d)\n",
				     ib_wc_status_msg(wc_status), wc_status);
1997 1998
		} else {
			shost_printk(KERN_ERR, target->scsi_host,
1999
				     PFX "failed %s status %s (%d) for iu %p\n",
2000
				     send_err ? "send" : "receive",
2001 2002
				     ib_wc_status_msg(wc_status), wc_status,
				     (void *)(uintptr_t)wr_id);
2003
		}
2004
		queue_work(system_long_wq, &target->tl_err_work);
2005
	}
2006 2007 2008
	target->qp_in_error = true;
}

2009
static void srp_recv_completion(struct ib_cq *cq, void *ch_ptr)
2010
{
2011
	struct srp_rdma_ch *ch = ch_ptr;
2012 2013 2014 2015
	struct ib_wc wc;

	ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
	while (ib_poll_cq(cq, 1, &wc) > 0) {
2016
		if (likely(wc.status == IB_WC_SUCCESS)) {
2017
			srp_handle_recv(ch, &wc);
2018
		} else {
2019
			srp_handle_qp_err(wc.wr_id, wc.status, false, ch);
2020
		}
2021 2022 2023
	}
}

2024
static void srp_send_completion(struct ib_cq *cq, void *ch_ptr)
2025
{
2026
	struct srp_rdma_ch *ch = ch_ptr;
2027
	struct ib_wc wc;
2028
	struct srp_iu *iu;
2029 2030

	while (ib_poll_cq(cq, 1, &wc) > 0) {
2031 2032
		if (likely(wc.status == IB_WC_SUCCESS)) {
			iu = (struct srp_iu *) (uintptr_t) wc.wr_id;
2033
			list_add(&iu->list, &ch->free_tx);
2034
		} else {
2035
			srp_handle_qp_err(wc.wr_id, wc.status, true, ch);
2036
		}
2037 2038 2039
	}
}

2040
static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
2041
{
2042
	struct srp_target_port *target = host_to_target(shost);
2043
	struct srp_rport *rport = target->rport;
2044
	struct srp_rdma_ch *ch;
2045 2046 2047
	struct srp_request *req;
	struct srp_iu *iu;
	struct srp_cmd *cmd;
2048
	struct ib_device *dev;
2049
	unsigned long flags;
B
Bart Van Assche 已提交
2050 2051
	u32 tag;
	u16 idx;
2052
	int len, ret;
2053 2054 2055 2056 2057 2058 2059 2060 2061 2062
	const bool in_scsi_eh = !in_interrupt() && current == shost->ehandler;

	/*
	 * The SCSI EH thread is the only context from which srp_queuecommand()
	 * can get invoked for blocked devices (SDEV_BLOCK /
	 * SDEV_CREATED_BLOCK). Avoid racing with srp_reconnect_rport() by
	 * locking the rport mutex if invoked from inside the SCSI EH.
	 */
	if (in_scsi_eh)
		mutex_lock(&rport->mutex);
2063

2064 2065 2066
	scmnd->result = srp_chkready(target->rport);
	if (unlikely(scmnd->result))
		goto err;
2067

B
Bart Van Assche 已提交
2068 2069
	WARN_ON_ONCE(scmnd->request->tag < 0);
	tag = blk_mq_unique_tag(scmnd->request);
B
Bart Van Assche 已提交
2070
	ch = &target->ch[blk_mq_unique_tag_to_hwq(tag)];
B
Bart Van Assche 已提交
2071 2072 2073 2074
	idx = blk_mq_unique_tag_to_tag(tag);
	WARN_ONCE(idx >= target->req_ring_size, "%s: tag %#x: idx %d >= %d\n",
		  dev_name(&shost->shost_gendev), tag, idx,
		  target->req_ring_size);
2075 2076 2077 2078

	spin_lock_irqsave(&ch->lock, flags);
	iu = __srp_get_tx_iu(ch, SRP_IU_CMD);
	spin_unlock_irqrestore(&ch->lock, flags);
2079

B
Bart Van Assche 已提交
2080 2081 2082 2083
	if (!iu)
		goto err;

	req = &ch->req_ring[idx];
2084
	dev = target->srp_host->srp_dev->dev;
2085
	ib_dma_sync_single_for_cpu(dev, iu->dma, target->max_iu_len,
2086
				   DMA_TO_DEVICE);
2087

2088
	scmnd->host_scribble = (void *) req;
2089 2090 2091 2092 2093

	cmd = iu->buf;
	memset(cmd, 0, sizeof *cmd);

	cmd->opcode = SRP_CMD;
B
Bart Van Assche 已提交
2094
	int_to_scsilun(scmnd->device->lun, &cmd->lun);
B
Bart Van Assche 已提交
2095
	cmd->tag    = tag;
2096 2097 2098 2099 2100
	memcpy(cmd->cdb, scmnd->cmnd, scmnd->cmd_len);

	req->scmnd    = scmnd;
	req->cmd      = iu;

2101
	len = srp_map_data(scmnd, ch, req);
2102
	if (len < 0) {
2103
		shost_printk(KERN_ERR, target->scsi_host,
2104 2105 2106 2107
			     PFX "Failed to map data (%d)\n", len);
		/*
		 * If we ran out of memory descriptors (-ENOMEM) because an
		 * application is queuing many requests with more than
2108
		 * max_pages_per_mr sg-list elements, tell the SCSI mid-layer
2109 2110 2111 2112
		 * to reduce queue depth temporarily.
		 */
		scmnd->result = len == -ENOMEM ?
			DID_OK << 16 | QUEUE_FULL << 1 : DID_ERROR << 16;
2113
		goto err_iu;
2114 2115
	}

2116
	ib_dma_sync_single_for_device(dev, iu->dma, target->max_iu_len,
2117
				      DMA_TO_DEVICE);
2118

2119
	if (srp_post_send(ch, iu, len)) {
2120
		shost_printk(KERN_ERR, target->scsi_host, PFX "Send failed\n");
2121 2122 2123
		goto err_unmap;
	}

2124 2125
	ret = 0;

2126 2127 2128 2129
unlock_rport:
	if (in_scsi_eh)
		mutex_unlock(&rport->mutex);

2130
	return ret;
2131 2132

err_unmap:
2133
	srp_unmap_data(scmnd, ch, req);
2134

2135
err_iu:
2136
	srp_put_tx_iu(ch, iu, SRP_IU_CMD);
2137

2138 2139 2140 2141 2142 2143
	/*
	 * Avoid that the loops that iterate over the request ring can
	 * encounter a dangling SCSI command pointer.
	 */
	req->scmnd = NULL;

2144 2145 2146 2147 2148 2149 2150
err:
	if (scmnd->result) {
		scmnd->scsi_done(scmnd);
		ret = 0;
	} else {
		ret = SCSI_MLQUEUE_HOST_BUSY;
	}
2151

2152
	goto unlock_rport;
2153 2154
}

2155 2156
/*
 * Note: the resources allocated in this function are freed in
2157
 * srp_free_ch_ib().
2158
 */
2159
static int srp_alloc_iu_bufs(struct srp_rdma_ch *ch)
2160
{
2161
	struct srp_target_port *target = ch->target;
2162 2163
	int i;

2164 2165 2166
	ch->rx_ring = kcalloc(target->queue_size, sizeof(*ch->rx_ring),
			      GFP_KERNEL);
	if (!ch->rx_ring)
2167
		goto err_no_ring;
2168 2169 2170
	ch->tx_ring = kcalloc(target->queue_size, sizeof(*ch->tx_ring),
			      GFP_KERNEL);
	if (!ch->tx_ring)
2171 2172 2173
		goto err_no_ring;

	for (i = 0; i < target->queue_size; ++i) {
2174 2175 2176 2177
		ch->rx_ring[i] = srp_alloc_iu(target->srp_host,
					      ch->max_ti_iu_len,
					      GFP_KERNEL, DMA_FROM_DEVICE);
		if (!ch->rx_ring[i])
2178 2179 2180
			goto err;
	}

2181
	for (i = 0; i < target->queue_size; ++i) {
2182 2183 2184 2185
		ch->tx_ring[i] = srp_alloc_iu(target->srp_host,
					      target->max_iu_len,
					      GFP_KERNEL, DMA_TO_DEVICE);
		if (!ch->tx_ring[i])
2186
			goto err;
2187

2188
		list_add(&ch->tx_ring[i]->list, &ch->free_tx);
2189 2190 2191 2192 2193
	}

	return 0;

err:
2194
	for (i = 0; i < target->queue_size; ++i) {
2195 2196
		srp_free_iu(target->srp_host, ch->rx_ring[i]);
		srp_free_iu(target->srp_host, ch->tx_ring[i]);
2197 2198
	}

2199 2200

err_no_ring:
2201 2202 2203 2204
	kfree(ch->tx_ring);
	ch->tx_ring = NULL;
	kfree(ch->rx_ring);
	ch->rx_ring = NULL;
2205

2206 2207 2208
	return -ENOMEM;
}

2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235
static uint32_t srp_compute_rq_tmo(struct ib_qp_attr *qp_attr, int attr_mask)
{
	uint64_t T_tr_ns, max_compl_time_ms;
	uint32_t rq_tmo_jiffies;

	/*
	 * According to section 11.2.4.2 in the IBTA spec (Modify Queue Pair,
	 * table 91), both the QP timeout and the retry count have to be set
	 * for RC QP's during the RTR to RTS transition.
	 */
	WARN_ON_ONCE((attr_mask & (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)) !=
		     (IB_QP_TIMEOUT | IB_QP_RETRY_CNT));

	/*
	 * Set target->rq_tmo_jiffies to one second more than the largest time
	 * it can take before an error completion is generated. See also
	 * C9-140..142 in the IBTA spec for more information about how to
	 * convert the QP Local ACK Timeout value to nanoseconds.
	 */
	T_tr_ns = 4096 * (1ULL << qp_attr->timeout);
	max_compl_time_ms = qp_attr->retry_cnt * 4 * T_tr_ns;
	do_div(max_compl_time_ms, NSEC_PER_MSEC);
	rq_tmo_jiffies = msecs_to_jiffies(max_compl_time_ms + 1000);

	return rq_tmo_jiffies;
}

2236
static void srp_cm_rep_handler(struct ib_cm_id *cm_id,
2237
			       const struct srp_login_rsp *lrsp,
2238
			       struct srp_rdma_ch *ch)
2239
{
2240
	struct srp_target_port *target = ch->target;
2241 2242 2243 2244 2245 2246
	struct ib_qp_attr *qp_attr = NULL;
	int attr_mask = 0;
	int ret;
	int i;

	if (lrsp->opcode == SRP_LOGIN_RSP) {
2247 2248
		ch->max_ti_iu_len = be32_to_cpu(lrsp->max_ti_iu_len);
		ch->req_lim       = be32_to_cpu(lrsp->req_lim_delta);
2249 2250 2251 2252 2253 2254

		/*
		 * Reserve credits for task management so we don't
		 * bounce requests back to the SCSI mid-layer.
		 */
		target->scsi_host->can_queue
2255
			= min(ch->req_lim - SRP_TSK_MGMT_SQ_SIZE,
2256
			      target->scsi_host->can_queue);
2257 2258 2259
		target->scsi_host->cmd_per_lun
			= min_t(int, target->scsi_host->can_queue,
				target->scsi_host->cmd_per_lun);
2260 2261 2262 2263 2264 2265 2266
	} else {
		shost_printk(KERN_WARNING, target->scsi_host,
			     PFX "Unhandled RSP opcode %#x\n", lrsp->opcode);
		ret = -ECONNRESET;
		goto error;
	}

2267 2268
	if (!ch->rx_ring) {
		ret = srp_alloc_iu_bufs(ch);
2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282
		if (ret)
			goto error;
	}

	ret = -ENOMEM;
	qp_attr = kmalloc(sizeof *qp_attr, GFP_KERNEL);
	if (!qp_attr)
		goto error;

	qp_attr->qp_state = IB_QPS_RTR;
	ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
	if (ret)
		goto error_free;

2283
	ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
2284 2285 2286
	if (ret)
		goto error_free;

2287
	for (i = 0; i < target->queue_size; i++) {
2288 2289 2290
		struct srp_iu *iu = ch->rx_ring[i];

		ret = srp_post_recv(ch, iu);
2291 2292 2293 2294 2295 2296 2297 2298 2299
		if (ret)
			goto error_free;
	}

	qp_attr->qp_state = IB_QPS_RTS;
	ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
	if (ret)
		goto error_free;

2300 2301
	target->rq_tmo_jiffies = srp_compute_rq_tmo(qp_attr, attr_mask);

2302
	ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
2303 2304 2305 2306 2307 2308 2309 2310 2311
	if (ret)
		goto error_free;

	ret = ib_send_cm_rtu(cm_id, NULL, 0);

error_free:
	kfree(qp_attr);

error:
2312
	ch->status = ret;
2313 2314
}

2315 2316
static void srp_cm_rej_handler(struct ib_cm_id *cm_id,
			       struct ib_cm_event *event,
2317
			       struct srp_rdma_ch *ch)
2318
{
2319
	struct srp_target_port *target = ch->target;
2320
	struct Scsi_Host *shost = target->scsi_host;
2321 2322 2323 2324 2325 2326
	struct ib_class_port_info *cpi;
	int opcode;

	switch (event->param.rej_rcvd.reason) {
	case IB_CM_REJ_PORT_CM_REDIRECT:
		cpi = event->param.rej_rcvd.ari;
2327 2328
		ch->path.dlid = cpi->redirect_lid;
		ch->path.pkey = cpi->redirect_pkey;
2329
		cm_id->remote_cm_qpn = be32_to_cpu(cpi->redirect_qp) & 0x00ffffff;
2330
		memcpy(ch->path.dgid.raw, cpi->redirect_gid, 16);
2331

2332
		ch->status = ch->path.dlid ?
2333 2334 2335 2336
			SRP_DLID_REDIRECT : SRP_PORT_REDIRECT;
		break;

	case IB_CM_REJ_PORT_REDIRECT:
2337
		if (srp_target_is_topspin(target)) {
2338 2339 2340 2341 2342
			/*
			 * Topspin/Cisco SRP gateways incorrectly send
			 * reject reason code 25 when they mean 24
			 * (port redirect).
			 */
2343
			memcpy(ch->path.dgid.raw,
2344 2345
			       event->param.rej_rcvd.ari, 16);

2346 2347
			shost_printk(KERN_DEBUG, shost,
				     PFX "Topspin/Cisco redirect to target port GID %016llx%016llx\n",
2348 2349
				     be64_to_cpu(ch->path.dgid.global.subnet_prefix),
				     be64_to_cpu(ch->path.dgid.global.interface_id));
2350

2351
			ch->status = SRP_PORT_REDIRECT;
2352
		} else {
2353 2354
			shost_printk(KERN_WARNING, shost,
				     "  REJ reason: IB_CM_REJ_PORT_REDIRECT\n");
2355
			ch->status = -ECONNRESET;
2356 2357 2358 2359
		}
		break;

	case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID:
2360 2361
		shost_printk(KERN_WARNING, shost,
			    "  REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n");
2362
		ch->status = -ECONNRESET;
2363 2364 2365 2366 2367 2368 2369 2370 2371
		break;

	case IB_CM_REJ_CONSUMER_DEFINED:
		opcode = *(u8 *) event->private_data;
		if (opcode == SRP_LOGIN_REJ) {
			struct srp_login_rej *rej = event->private_data;
			u32 reason = be32_to_cpu(rej->reason);

			if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE)
2372 2373
				shost_printk(KERN_WARNING, shost,
					     PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n");
2374
			else
B
Bart Van Assche 已提交
2375 2376
				shost_printk(KERN_WARNING, shost, PFX
					     "SRP LOGIN from %pI6 to %pI6 REJECTED, reason 0x%08x\n",
2377 2378
					     target->sgid.raw,
					     target->orig_dgid.raw, reason);
2379
		} else
2380 2381 2382
			shost_printk(KERN_WARNING, shost,
				     "  REJ reason: IB_CM_REJ_CONSUMER_DEFINED,"
				     " opcode 0x%02x\n", opcode);
2383
		ch->status = -ECONNRESET;
2384 2385
		break;

D
David Dillow 已提交
2386 2387
	case IB_CM_REJ_STALE_CONN:
		shost_printk(KERN_WARNING, shost, "  REJ reason: stale connection\n");
2388
		ch->status = SRP_STALE_CONN;
D
David Dillow 已提交
2389 2390
		break;

2391
	default:
2392 2393
		shost_printk(KERN_WARNING, shost, "  REJ reason 0x%x\n",
			     event->param.rej_rcvd.reason);
2394
		ch->status = -ECONNRESET;
2395 2396 2397 2398 2399
	}
}

static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
{
2400 2401
	struct srp_rdma_ch *ch = cm_id->context;
	struct srp_target_port *target = ch->target;
2402 2403 2404 2405
	int comp = 0;

	switch (event->event) {
	case IB_CM_REQ_ERROR:
2406 2407
		shost_printk(KERN_DEBUG, target->scsi_host,
			     PFX "Sending CM REQ failed\n");
2408
		comp = 1;
2409
		ch->status = -ECONNRESET;
2410 2411 2412 2413
		break;

	case IB_CM_REP_RECEIVED:
		comp = 1;
2414
		srp_cm_rep_handler(cm_id, event->private_data, ch);
2415 2416 2417
		break;

	case IB_CM_REJ_RECEIVED:
2418
		shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n");
2419 2420
		comp = 1;

2421
		srp_cm_rej_handler(cm_id, event, ch);
2422 2423
		break;

2424
	case IB_CM_DREQ_RECEIVED:
2425 2426
		shost_printk(KERN_WARNING, target->scsi_host,
			     PFX "DREQ received - connection closed\n");
2427
		ch->connected = false;
2428
		if (ib_send_cm_drep(cm_id, NULL, 0))
2429 2430
			shost_printk(KERN_ERR, target->scsi_host,
				     PFX "Sending CM DREP failed\n");
2431
		queue_work(system_long_wq, &target->tl_err_work);
2432 2433 2434
		break;

	case IB_CM_TIMEWAIT_EXIT:
2435 2436
		shost_printk(KERN_ERR, target->scsi_host,
			     PFX "connection closed\n");
2437
		comp = 1;
2438

2439
		ch->status = 0;
2440 2441
		break;

2442 2443 2444 2445 2446
	case IB_CM_MRA_RECEIVED:
	case IB_CM_DREQ_ERROR:
	case IB_CM_DREP_RECEIVED:
		break;

2447
	default:
2448 2449
		shost_printk(KERN_WARNING, target->scsi_host,
			     PFX "Unhandled CM event %d\n", event->event);
2450 2451 2452 2453
		break;
	}

	if (comp)
2454
		complete(&ch->done);
2455 2456 2457 2458

	return 0;
}

2459 2460 2461 2462 2463 2464 2465 2466
/**
 * srp_change_queue_depth - setting device queue depth
 * @sdev: scsi device struct
 * @qdepth: requested queue depth
 *
 * Returns queue depth.
 */
static int
2467
srp_change_queue_depth(struct scsi_device *sdev, int qdepth)
2468
{
2469
	if (!sdev->tagged_supported)
2470
		qdepth = 1;
2471
	return scsi_change_queue_depth(sdev, qdepth);
2472 2473
}

B
Bart Van Assche 已提交
2474 2475
static int srp_send_tsk_mgmt(struct srp_rdma_ch *ch, u64 req_tag, u64 lun,
			     u8 func)
2476
{
2477
	struct srp_target_port *target = ch->target;
2478
	struct srp_rport *rport = target->rport;
2479
	struct ib_device *dev = target->srp_host->srp_dev->dev;
2480 2481 2482
	struct srp_iu *iu;
	struct srp_tsk_mgmt *tsk_mgmt;

2483
	if (!ch->connected || target->qp_in_error)
2484 2485
		return -1;

2486
	init_completion(&ch->tsk_mgmt_done);
2487

2488
	/*
2489
	 * Lock the rport mutex to avoid that srp_create_ch_ib() is
2490 2491 2492
	 * invoked while a task management function is being sent.
	 */
	mutex_lock(&rport->mutex);
2493 2494 2495
	spin_lock_irq(&ch->lock);
	iu = __srp_get_tx_iu(ch, SRP_IU_TSK_MGMT);
	spin_unlock_irq(&ch->lock);
2496

2497 2498 2499
	if (!iu) {
		mutex_unlock(&rport->mutex);

2500
		return -1;
2501
	}
2502

2503 2504
	ib_dma_sync_single_for_cpu(dev, iu->dma, sizeof *tsk_mgmt,
				   DMA_TO_DEVICE);
2505 2506 2507 2508
	tsk_mgmt = iu->buf;
	memset(tsk_mgmt, 0, sizeof *tsk_mgmt);

	tsk_mgmt->opcode 	= SRP_TSK_MGMT;
B
Bart Van Assche 已提交
2509
	int_to_scsilun(lun, &tsk_mgmt->lun);
2510
	tsk_mgmt->tag		= req_tag | SRP_TAG_TSK_MGMT;
2511
	tsk_mgmt->tsk_mgmt_func = func;
2512
	tsk_mgmt->task_tag	= req_tag;
2513

2514 2515
	ib_dma_sync_single_for_device(dev, iu->dma, sizeof *tsk_mgmt,
				      DMA_TO_DEVICE);
2516 2517
	if (srp_post_send(ch, iu, sizeof(*tsk_mgmt))) {
		srp_put_tx_iu(ch, iu, SRP_IU_TSK_MGMT);
2518 2519
		mutex_unlock(&rport->mutex);

2520 2521
		return -1;
	}
2522
	mutex_unlock(&rport->mutex);
2523

2524
	if (!wait_for_completion_timeout(&ch->tsk_mgmt_done,
2525
					 msecs_to_jiffies(SRP_ABORT_TIMEOUT_MS)))
2526
		return -1;
2527

2528 2529 2530
	return 0;
}

2531 2532
static int srp_abort(struct scsi_cmnd *scmnd)
{
2533
	struct srp_target_port *target = host_to_target(scmnd->device->host);
2534
	struct srp_request *req = (struct srp_request *) scmnd->host_scribble;
B
Bart Van Assche 已提交
2535
	u32 tag;
B
Bart Van Assche 已提交
2536
	u16 ch_idx;
2537
	struct srp_rdma_ch *ch;
2538
	int ret;
2539

2540
	shost_printk(KERN_ERR, target->scsi_host, "SRP abort called\n");
2541

B
Bart Van Assche 已提交
2542
	if (!req)
2543
		return SUCCESS;
B
Bart Van Assche 已提交
2544
	tag = blk_mq_unique_tag(scmnd->request);
B
Bart Van Assche 已提交
2545 2546 2547 2548 2549 2550 2551 2552
	ch_idx = blk_mq_unique_tag_to_hwq(tag);
	if (WARN_ON_ONCE(ch_idx >= target->ch_count))
		return SUCCESS;
	ch = &target->ch[ch_idx];
	if (!srp_claim_req(ch, req, NULL, scmnd))
		return SUCCESS;
	shost_printk(KERN_ERR, target->scsi_host,
		     "Sending SRP abort for tag %#x\n", tag);
B
Bart Van Assche 已提交
2553
	if (srp_send_tsk_mgmt(ch, tag, scmnd->device->lun,
2554
			      SRP_TSK_ABORT_TASK) == 0)
2555
		ret = SUCCESS;
2556
	else if (target->rport->state == SRP_RPORT_LOST)
2557
		ret = FAST_IO_FAIL;
2558 2559
	else
		ret = FAILED;
2560
	srp_free_req(ch, req, scmnd, 0);
B
Bart Van Assche 已提交
2561
	scmnd->result = DID_ABORT << 16;
2562
	scmnd->scsi_done(scmnd);
2563

2564
	return ret;
2565 2566 2567 2568
}

static int srp_reset_device(struct scsi_cmnd *scmnd)
{
2569
	struct srp_target_port *target = host_to_target(scmnd->device->host);
B
Bart Van Assche 已提交
2570
	struct srp_rdma_ch *ch;
2571
	int i;
2572

2573
	shost_printk(KERN_ERR, target->scsi_host, "SRP reset_device called\n");
2574

B
Bart Van Assche 已提交
2575
	ch = &target->ch[0];
2576
	if (srp_send_tsk_mgmt(ch, SRP_TAG_NO_REQ, scmnd->device->lun,
2577
			      SRP_TSK_LUN_RESET))
2578
		return FAILED;
2579
	if (ch->tsk_mgmt_status)
2580 2581
		return FAILED;

B
Bart Van Assche 已提交
2582 2583 2584 2585
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
		for (i = 0; i < target->req_ring_size; ++i) {
			struct srp_request *req = &ch->req_ring[i];
2586

B
Bart Van Assche 已提交
2587 2588
			srp_finish_req(ch, req, scmnd->device, DID_RESET << 16);
		}
2589
	}
2590 2591

	return SUCCESS;
2592 2593 2594 2595 2596 2597
}

static int srp_reset_host(struct scsi_cmnd *scmnd)
{
	struct srp_target_port *target = host_to_target(scmnd->device->host);

2598
	shost_printk(KERN_ERR, target->scsi_host, PFX "SRP reset_host called\n");
2599

2600
	return srp_reconnect_rport(target->rport) == 0 ? SUCCESS : FAILED;
2601 2602
}

2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617
static int srp_slave_configure(struct scsi_device *sdev)
{
	struct Scsi_Host *shost = sdev->host;
	struct srp_target_port *target = host_to_target(shost);
	struct request_queue *q = sdev->request_queue;
	unsigned long timeout;

	if (sdev->type == TYPE_DISK) {
		timeout = max_t(unsigned, 30 * HZ, target->rq_tmo_jiffies);
		blk_queue_rq_timeout(q, timeout);
	}

	return 0;
}

2618 2619
static ssize_t show_id_ext(struct device *dev, struct device_attribute *attr,
			   char *buf)
2620
{
2621
	struct srp_target_port *target = host_to_target(class_to_shost(dev));
2622

2623
	return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->id_ext));
2624 2625
}

2626 2627
static ssize_t show_ioc_guid(struct device *dev, struct device_attribute *attr,
			     char *buf)
2628
{
2629
	struct srp_target_port *target = host_to_target(class_to_shost(dev));
2630

2631
	return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->ioc_guid));
2632 2633
}

2634 2635
static ssize_t show_service_id(struct device *dev,
			       struct device_attribute *attr, char *buf)
2636
{
2637
	struct srp_target_port *target = host_to_target(class_to_shost(dev));
2638

2639
	return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->service_id));
2640 2641
}

2642 2643
static ssize_t show_pkey(struct device *dev, struct device_attribute *attr,
			 char *buf)
2644
{
2645
	struct srp_target_port *target = host_to_target(class_to_shost(dev));
2646

2647
	return sprintf(buf, "0x%04x\n", be16_to_cpu(target->pkey));
2648 2649
}

B
Bart Van Assche 已提交
2650 2651 2652 2653 2654
static ssize_t show_sgid(struct device *dev, struct device_attribute *attr,
			 char *buf)
{
	struct srp_target_port *target = host_to_target(class_to_shost(dev));

2655
	return sprintf(buf, "%pI6\n", target->sgid.raw);
B
Bart Van Assche 已提交
2656 2657
}

2658 2659
static ssize_t show_dgid(struct device *dev, struct device_attribute *attr,
			 char *buf)
2660
{
2661
	struct srp_target_port *target = host_to_target(class_to_shost(dev));
B
Bart Van Assche 已提交
2662
	struct srp_rdma_ch *ch = &target->ch[0];
2663

2664
	return sprintf(buf, "%pI6\n", ch->path.dgid.raw);
2665 2666
}

2667 2668
static ssize_t show_orig_dgid(struct device *dev,
			      struct device_attribute *attr, char *buf)
2669
{
2670
	struct srp_target_port *target = host_to_target(class_to_shost(dev));
2671

2672
	return sprintf(buf, "%pI6\n", target->orig_dgid.raw);
2673 2674
}

2675 2676 2677 2678
static ssize_t show_req_lim(struct device *dev,
			    struct device_attribute *attr, char *buf)
{
	struct srp_target_port *target = host_to_target(class_to_shost(dev));
B
Bart Van Assche 已提交
2679 2680
	struct srp_rdma_ch *ch;
	int i, req_lim = INT_MAX;
2681

B
Bart Van Assche 已提交
2682 2683 2684 2685 2686
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
		req_lim = min(req_lim, ch->req_lim);
	}
	return sprintf(buf, "%d\n", req_lim);
2687 2688
}

2689 2690
static ssize_t show_zero_req_lim(struct device *dev,
				 struct device_attribute *attr, char *buf)
2691
{
2692
	struct srp_target_port *target = host_to_target(class_to_shost(dev));
2693 2694 2695 2696

	return sprintf(buf, "%d\n", target->zero_req_lim);
}

2697 2698
static ssize_t show_local_ib_port(struct device *dev,
				  struct device_attribute *attr, char *buf)
2699
{
2700
	struct srp_target_port *target = host_to_target(class_to_shost(dev));
2701 2702 2703 2704

	return sprintf(buf, "%d\n", target->srp_host->port);
}

2705 2706
static ssize_t show_local_ib_device(struct device *dev,
				    struct device_attribute *attr, char *buf)
2707
{
2708
	struct srp_target_port *target = host_to_target(class_to_shost(dev));
2709

2710
	return sprintf(buf, "%s\n", target->srp_host->srp_dev->dev->name);
2711 2712
}

B
Bart Van Assche 已提交
2713 2714 2715 2716 2717 2718 2719 2720
static ssize_t show_ch_count(struct device *dev, struct device_attribute *attr,
			     char *buf)
{
	struct srp_target_port *target = host_to_target(class_to_shost(dev));

	return sprintf(buf, "%d\n", target->ch_count);
}

2721 2722 2723 2724 2725 2726 2727 2728
static ssize_t show_comp_vector(struct device *dev,
				struct device_attribute *attr, char *buf)
{
	struct srp_target_port *target = host_to_target(class_to_shost(dev));

	return sprintf(buf, "%d\n", target->comp_vector);
}

2729 2730 2731 2732 2733 2734 2735 2736
static ssize_t show_tl_retry_count(struct device *dev,
				   struct device_attribute *attr, char *buf)
{
	struct srp_target_port *target = host_to_target(class_to_shost(dev));

	return sprintf(buf, "%d\n", target->tl_retry_count);
}

2737 2738 2739 2740 2741 2742 2743 2744
static ssize_t show_cmd_sg_entries(struct device *dev,
				   struct device_attribute *attr, char *buf)
{
	struct srp_target_port *target = host_to_target(class_to_shost(dev));

	return sprintf(buf, "%u\n", target->cmd_sg_cnt);
}

2745 2746 2747 2748 2749 2750 2751 2752
static ssize_t show_allow_ext_sg(struct device *dev,
				 struct device_attribute *attr, char *buf)
{
	struct srp_target_port *target = host_to_target(class_to_shost(dev));

	return sprintf(buf, "%s\n", target->allow_ext_sg ? "true" : "false");
}

2753 2754 2755 2756
static DEVICE_ATTR(id_ext,	    S_IRUGO, show_id_ext,	   NULL);
static DEVICE_ATTR(ioc_guid,	    S_IRUGO, show_ioc_guid,	   NULL);
static DEVICE_ATTR(service_id,	    S_IRUGO, show_service_id,	   NULL);
static DEVICE_ATTR(pkey,	    S_IRUGO, show_pkey,		   NULL);
B
Bart Van Assche 已提交
2757
static DEVICE_ATTR(sgid,	    S_IRUGO, show_sgid,		   NULL);
2758 2759
static DEVICE_ATTR(dgid,	    S_IRUGO, show_dgid,		   NULL);
static DEVICE_ATTR(orig_dgid,	    S_IRUGO, show_orig_dgid,	   NULL);
2760
static DEVICE_ATTR(req_lim,         S_IRUGO, show_req_lim,         NULL);
2761 2762 2763
static DEVICE_ATTR(zero_req_lim,    S_IRUGO, show_zero_req_lim,	   NULL);
static DEVICE_ATTR(local_ib_port,   S_IRUGO, show_local_ib_port,   NULL);
static DEVICE_ATTR(local_ib_device, S_IRUGO, show_local_ib_device, NULL);
B
Bart Van Assche 已提交
2764
static DEVICE_ATTR(ch_count,        S_IRUGO, show_ch_count,        NULL);
2765
static DEVICE_ATTR(comp_vector,     S_IRUGO, show_comp_vector,     NULL);
2766
static DEVICE_ATTR(tl_retry_count,  S_IRUGO, show_tl_retry_count,  NULL);
2767
static DEVICE_ATTR(cmd_sg_entries,  S_IRUGO, show_cmd_sg_entries,  NULL);
2768
static DEVICE_ATTR(allow_ext_sg,    S_IRUGO, show_allow_ext_sg,    NULL);
2769 2770 2771 2772 2773 2774

static struct device_attribute *srp_host_attrs[] = {
	&dev_attr_id_ext,
	&dev_attr_ioc_guid,
	&dev_attr_service_id,
	&dev_attr_pkey,
B
Bart Van Assche 已提交
2775
	&dev_attr_sgid,
2776 2777
	&dev_attr_dgid,
	&dev_attr_orig_dgid,
2778
	&dev_attr_req_lim,
2779 2780 2781
	&dev_attr_zero_req_lim,
	&dev_attr_local_ib_port,
	&dev_attr_local_ib_device,
B
Bart Van Assche 已提交
2782
	&dev_attr_ch_count,
2783
	&dev_attr_comp_vector,
2784
	&dev_attr_tl_retry_count,
2785
	&dev_attr_cmd_sg_entries,
2786
	&dev_attr_allow_ext_sg,
2787 2788 2789
	NULL
};

2790 2791
static struct scsi_host_template srp_template = {
	.module				= THIS_MODULE,
R
Roland Dreier 已提交
2792 2793
	.name				= "InfiniBand SRP initiator",
	.proc_name			= DRV_NAME,
2794
	.slave_configure		= srp_slave_configure,
2795 2796
	.info				= srp_target_info,
	.queuecommand			= srp_queuecommand,
2797
	.change_queue_depth             = srp_change_queue_depth,
2798 2799 2800
	.eh_abort_handler		= srp_abort,
	.eh_device_reset_handler	= srp_reset_device,
	.eh_host_reset_handler		= srp_reset_host,
B
Bart Van Assche 已提交
2801
	.skip_settle_delay		= true,
2802
	.sg_tablesize			= SRP_DEF_SG_TABLESIZE,
2803
	.can_queue			= SRP_DEFAULT_CMD_SQ_SIZE,
2804
	.this_id			= -1,
2805
	.cmd_per_lun			= SRP_DEFAULT_CMD_SQ_SIZE,
2806
	.use_clustering			= ENABLE_CLUSTERING,
B
Bart Van Assche 已提交
2807
	.shost_attrs			= srp_host_attrs,
2808
	.track_queue_depth		= 1,
2809 2810
};

2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821
static int srp_sdev_count(struct Scsi_Host *host)
{
	struct scsi_device *sdev;
	int c = 0;

	shost_for_each_device(sdev, host)
		c++;

	return c;
}

2822 2823 2824 2825 2826 2827 2828
/*
 * Return values:
 * < 0 upon failure. Caller is responsible for SRP target port cleanup.
 * 0 and target->state == SRP_TARGET_REMOVED if asynchronous target port
 *    removal has been scheduled.
 * 0 and target->state != SRP_TARGET_REMOVED upon success.
 */
2829 2830
static int srp_add_target(struct srp_host *host, struct srp_target_port *target)
{
2831 2832 2833
	struct srp_rport_identifiers ids;
	struct srp_rport *rport;

2834
	target->state = SRP_TARGET_SCANNING;
2835
	sprintf(target->target_name, "SRP.T10:%016llX",
2836
		be64_to_cpu(target->id_ext));
2837

2838
	if (scsi_add_host(target->scsi_host, host->srp_dev->dev->dma_device))
2839 2840
		return -ENODEV;

2841 2842
	memcpy(ids.port_id, &target->id_ext, 8);
	memcpy(ids.port_id + 8, &target->ioc_guid, 8);
2843
	ids.roles = SRP_RPORT_ROLE_TARGET;
2844 2845 2846 2847 2848 2849
	rport = srp_rport_add(target->scsi_host, &ids);
	if (IS_ERR(rport)) {
		scsi_remove_host(target->scsi_host);
		return PTR_ERR(rport);
	}

2850
	rport->lld_data = target;
2851
	target->rport = rport;
2852

2853
	spin_lock(&host->target_lock);
2854
	list_add_tail(&target->list, &host->target_list);
2855
	spin_unlock(&host->target_lock);
2856 2857

	scsi_scan_target(&target->scsi_host->shost_gendev,
2858
			 0, target->scsi_id, SCAN_WILD_CARD, 0);
2859

2860 2861
	if (srp_connected_ch(target) < target->ch_count ||
	    target->qp_in_error) {
2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877
		shost_printk(KERN_INFO, target->scsi_host,
			     PFX "SCSI scan failed - removing SCSI host\n");
		srp_queue_remove_work(target);
		goto out;
	}

	pr_debug(PFX "%s: SCSI scan succeeded - detected %d LUNs\n",
		 dev_name(&target->scsi_host->shost_gendev),
		 srp_sdev_count(target->scsi_host));

	spin_lock_irq(&target->lock);
	if (target->state == SRP_TARGET_SCANNING)
		target->state = SRP_TARGET_LIVE;
	spin_unlock_irq(&target->lock);

out:
2878 2879 2880
	return 0;
}

2881
static void srp_release_dev(struct device *dev)
2882 2883
{
	struct srp_host *host =
2884
		container_of(dev, struct srp_host, dev);
2885 2886 2887 2888 2889 2890

	complete(&host->released);
}

static struct class srp_class = {
	.name    = "infiniband_srp",
2891
	.dev_release = srp_release_dev
2892 2893
};

2894 2895
/**
 * srp_conn_unique() - check whether the connection to a target is unique
2896 2897
 * @host:   SRP host.
 * @target: SRP target port.
2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925
 */
static bool srp_conn_unique(struct srp_host *host,
			    struct srp_target_port *target)
{
	struct srp_target_port *t;
	bool ret = false;

	if (target->state == SRP_TARGET_REMOVED)
		goto out;

	ret = true;

	spin_lock(&host->target_lock);
	list_for_each_entry(t, &host->target_list, list) {
		if (t != target &&
		    target->id_ext == t->id_ext &&
		    target->ioc_guid == t->ioc_guid &&
		    target->initiator_ext == t->initiator_ext) {
			ret = false;
			break;
		}
	}
	spin_unlock(&host->target_lock);

out:
	return ret;
}

2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941
/*
 * Target ports are added by writing
 *
 *     id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,dgid=<dest GID>,
 *     pkey=<P_Key>,service_id=<service ID>
 *
 * to the add_target sysfs attribute.
 */
enum {
	SRP_OPT_ERR		= 0,
	SRP_OPT_ID_EXT		= 1 << 0,
	SRP_OPT_IOC_GUID	= 1 << 1,
	SRP_OPT_DGID		= 1 << 2,
	SRP_OPT_PKEY		= 1 << 3,
	SRP_OPT_SERVICE_ID	= 1 << 4,
	SRP_OPT_MAX_SECT	= 1 << 5,
2942
	SRP_OPT_MAX_CMD_PER_LUN	= 1 << 6,
2943
	SRP_OPT_IO_CLASS	= 1 << 7,
2944
	SRP_OPT_INITIATOR_EXT	= 1 << 8,
2945
	SRP_OPT_CMD_SG_ENTRIES	= 1 << 9,
2946 2947
	SRP_OPT_ALLOW_EXT_SG	= 1 << 10,
	SRP_OPT_SG_TABLESIZE	= 1 << 11,
2948
	SRP_OPT_COMP_VECTOR	= 1 << 12,
2949
	SRP_OPT_TL_RETRY_COUNT	= 1 << 13,
2950
	SRP_OPT_QUEUE_SIZE	= 1 << 14,
2951 2952 2953 2954 2955 2956 2957
	SRP_OPT_ALL		= (SRP_OPT_ID_EXT	|
				   SRP_OPT_IOC_GUID	|
				   SRP_OPT_DGID		|
				   SRP_OPT_PKEY		|
				   SRP_OPT_SERVICE_ID),
};

2958
static const match_table_t srp_opt_tokens = {
2959 2960 2961 2962 2963 2964 2965
	{ SRP_OPT_ID_EXT,		"id_ext=%s" 		},
	{ SRP_OPT_IOC_GUID,		"ioc_guid=%s" 		},
	{ SRP_OPT_DGID,			"dgid=%s" 		},
	{ SRP_OPT_PKEY,			"pkey=%x" 		},
	{ SRP_OPT_SERVICE_ID,		"service_id=%s"		},
	{ SRP_OPT_MAX_SECT,		"max_sect=%d" 		},
	{ SRP_OPT_MAX_CMD_PER_LUN,	"max_cmd_per_lun=%d" 	},
2966
	{ SRP_OPT_IO_CLASS,		"io_class=%x"		},
2967
	{ SRP_OPT_INITIATOR_EXT,	"initiator_ext=%s"	},
2968
	{ SRP_OPT_CMD_SG_ENTRIES,	"cmd_sg_entries=%u"	},
2969 2970
	{ SRP_OPT_ALLOW_EXT_SG,		"allow_ext_sg=%u"	},
	{ SRP_OPT_SG_TABLESIZE,		"sg_tablesize=%u"	},
2971
	{ SRP_OPT_COMP_VECTOR,		"comp_vector=%u"	},
2972
	{ SRP_OPT_TL_RETRY_COUNT,	"tl_retry_count=%u"	},
2973
	{ SRP_OPT_QUEUE_SIZE,		"queue_size=%d"		},
2974
	{ SRP_OPT_ERR,			NULL 			}
2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992
};

static int srp_parse_options(const char *buf, struct srp_target_port *target)
{
	char *options, *sep_opt;
	char *p;
	char dgid[3];
	substring_t args[MAX_OPT_ARGS];
	int opt_mask = 0;
	int token;
	int ret = -EINVAL;
	int i;

	options = kstrdup(buf, GFP_KERNEL);
	if (!options)
		return -ENOMEM;

	sep_opt = options;
2993
	while ((p = strsep(&sep_opt, ",\n")) != NULL) {
2994 2995 2996 2997 2998 2999 3000 3001 3002
		if (!*p)
			continue;

		token = match_token(p, srp_opt_tokens, args);
		opt_mask |= token;

		switch (token) {
		case SRP_OPT_ID_EXT:
			p = match_strdup(args);
3003 3004 3005 3006
			if (!p) {
				ret = -ENOMEM;
				goto out;
			}
3007 3008 3009 3010 3011 3012
			target->id_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
			kfree(p);
			break;

		case SRP_OPT_IOC_GUID:
			p = match_strdup(args);
3013 3014 3015 3016
			if (!p) {
				ret = -ENOMEM;
				goto out;
			}
3017 3018 3019 3020 3021 3022
			target->ioc_guid = cpu_to_be64(simple_strtoull(p, NULL, 16));
			kfree(p);
			break;

		case SRP_OPT_DGID:
			p = match_strdup(args);
3023 3024 3025 3026
			if (!p) {
				ret = -ENOMEM;
				goto out;
			}
3027
			if (strlen(p) != 32) {
3028
				pr_warn("bad dest GID parameter '%s'\n", p);
3029
				kfree(p);
3030 3031 3032 3033
				goto out;
			}

			for (i = 0; i < 16; ++i) {
3034 3035 3036 3037 3038 3039 3040
				strlcpy(dgid, p + i * 2, sizeof(dgid));
				if (sscanf(dgid, "%hhx",
					   &target->orig_dgid.raw[i]) < 1) {
					ret = -EINVAL;
					kfree(p);
					goto out;
				}
3041
			}
3042
			kfree(p);
3043 3044 3045 3046
			break;

		case SRP_OPT_PKEY:
			if (match_hex(args, &token)) {
3047
				pr_warn("bad P_Key parameter '%s'\n", p);
3048 3049
				goto out;
			}
3050
			target->pkey = cpu_to_be16(token);
3051 3052 3053 3054
			break;

		case SRP_OPT_SERVICE_ID:
			p = match_strdup(args);
3055 3056 3057 3058
			if (!p) {
				ret = -ENOMEM;
				goto out;
			}
3059 3060 3061 3062 3063 3064
			target->service_id = cpu_to_be64(simple_strtoull(p, NULL, 16));
			kfree(p);
			break;

		case SRP_OPT_MAX_SECT:
			if (match_int(args, &token)) {
3065
				pr_warn("bad max sect parameter '%s'\n", p);
3066 3067 3068 3069 3070
				goto out;
			}
			target->scsi_host->max_sectors = token;
			break;

3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082
		case SRP_OPT_QUEUE_SIZE:
			if (match_int(args, &token) || token < 1) {
				pr_warn("bad queue_size parameter '%s'\n", p);
				goto out;
			}
			target->scsi_host->can_queue = token;
			target->queue_size = token + SRP_RSP_SQ_SIZE +
					     SRP_TSK_MGMT_SQ_SIZE;
			if (!(opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
				target->scsi_host->cmd_per_lun = token;
			break;

3083
		case SRP_OPT_MAX_CMD_PER_LUN:
3084
			if (match_int(args, &token) || token < 1) {
3085 3086
				pr_warn("bad max cmd_per_lun parameter '%s'\n",
					p);
3087 3088
				goto out;
			}
3089
			target->scsi_host->cmd_per_lun = token;
3090 3091
			break;

3092 3093
		case SRP_OPT_IO_CLASS:
			if (match_hex(args, &token)) {
3094
				pr_warn("bad IO class parameter '%s'\n", p);
3095 3096 3097 3098
				goto out;
			}
			if (token != SRP_REV10_IB_IO_CLASS &&
			    token != SRP_REV16A_IB_IO_CLASS) {
3099 3100 3101
				pr_warn("unknown IO class parameter value %x specified (use %x or %x).\n",
					token, SRP_REV10_IB_IO_CLASS,
					SRP_REV16A_IB_IO_CLASS);
3102 3103 3104 3105 3106
				goto out;
			}
			target->io_class = token;
			break;

3107 3108
		case SRP_OPT_INITIATOR_EXT:
			p = match_strdup(args);
3109 3110 3111 3112
			if (!p) {
				ret = -ENOMEM;
				goto out;
			}
3113 3114 3115 3116
			target->initiator_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
			kfree(p);
			break;

3117 3118
		case SRP_OPT_CMD_SG_ENTRIES:
			if (match_int(args, &token) || token < 1 || token > 255) {
3119 3120
				pr_warn("bad max cmd_sg_entries parameter '%s'\n",
					p);
3121 3122 3123 3124 3125
				goto out;
			}
			target->cmd_sg_cnt = token;
			break;

3126 3127
		case SRP_OPT_ALLOW_EXT_SG:
			if (match_int(args, &token)) {
3128
				pr_warn("bad allow_ext_sg parameter '%s'\n", p);
3129 3130 3131 3132 3133 3134 3135 3136
				goto out;
			}
			target->allow_ext_sg = !!token;
			break;

		case SRP_OPT_SG_TABLESIZE:
			if (match_int(args, &token) || token < 1 ||
					token > SCSI_MAX_SG_CHAIN_SEGMENTS) {
3137 3138
				pr_warn("bad max sg_tablesize parameter '%s'\n",
					p);
3139 3140 3141 3142 3143
				goto out;
			}
			target->sg_tablesize = token;
			break;

3144 3145 3146 3147 3148 3149 3150 3151
		case SRP_OPT_COMP_VECTOR:
			if (match_int(args, &token) || token < 0) {
				pr_warn("bad comp_vector parameter '%s'\n", p);
				goto out;
			}
			target->comp_vector = token;
			break;

3152 3153 3154 3155 3156 3157 3158 3159 3160
		case SRP_OPT_TL_RETRY_COUNT:
			if (match_int(args, &token) || token < 2 || token > 7) {
				pr_warn("bad tl_retry_count parameter '%s' (must be a number between 2 and 7)\n",
					p);
				goto out;
			}
			target->tl_retry_count = token;
			break;

3161
		default:
3162 3163
			pr_warn("unknown parameter or missing value '%s' in target creation request\n",
				p);
3164 3165 3166 3167 3168 3169 3170 3171 3172 3173
			goto out;
		}
	}

	if ((opt_mask & SRP_OPT_ALL) == SRP_OPT_ALL)
		ret = 0;
	else
		for (i = 0; i < ARRAY_SIZE(srp_opt_tokens); ++i)
			if ((srp_opt_tokens[i].token & SRP_OPT_ALL) &&
			    !(srp_opt_tokens[i].token & opt_mask))
3174 3175
				pr_warn("target creation request is missing parameter '%s'\n",
					srp_opt_tokens[i].pattern);
3176

3177 3178 3179 3180 3181 3182
	if (target->scsi_host->cmd_per_lun > target->scsi_host->can_queue
	    && (opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
		pr_warn("cmd_per_lun = %d > queue_size = %d\n",
			target->scsi_host->cmd_per_lun,
			target->scsi_host->can_queue);

3183 3184 3185 3186 3187
out:
	kfree(options);
	return ret;
}

3188 3189
static ssize_t srp_create_target(struct device *dev,
				 struct device_attribute *attr,
3190 3191 3192
				 const char *buf, size_t count)
{
	struct srp_host *host =
3193
		container_of(dev, struct srp_host, dev);
3194 3195
	struct Scsi_Host *target_host;
	struct srp_target_port *target;
3196
	struct srp_rdma_ch *ch;
3197 3198
	struct srp_device *srp_dev = host->srp_dev;
	struct ib_device *ibdev = srp_dev->dev;
B
Bart Van Assche 已提交
3199 3200
	int ret, node_idx, node, cpu, i;
	bool multich = false;
3201 3202 3203 3204 3205 3206

	target_host = scsi_host_alloc(&srp_template,
				      sizeof (struct srp_target_port));
	if (!target_host)
		return -ENOMEM;

3207
	target_host->transportt  = ib_srp_transport_template;
3208 3209
	target_host->max_channel = 0;
	target_host->max_id      = 1;
B
Bart Van Assche 已提交
3210
	target_host->max_lun     = -1LL;
A
Arne Redlich 已提交
3211
	target_host->max_cmd_len = sizeof ((struct srp_cmd *) (void *) 0L)->cdb;
R
Roland Dreier 已提交
3212

3213 3214
	target = host_to_target(target_host);

3215 3216 3217
	target->io_class	= SRP_REV16A_IB_IO_CLASS;
	target->scsi_host	= target_host;
	target->srp_host	= host;
J
Jason Gunthorpe 已提交
3218
	target->lkey		= host->srp_dev->pd->local_dma_lkey;
3219
	target->global_mr	= host->srp_dev->global_mr;
3220
	target->cmd_sg_cnt	= cmd_sg_entries;
3221 3222
	target->sg_tablesize	= indirect_sg_entries ? : cmd_sg_entries;
	target->allow_ext_sg	= allow_ext_sg;
3223
	target->tl_retry_count	= 7;
3224
	target->queue_size	= SRP_DEFAULT_QUEUE_SIZE;
3225

3226 3227 3228 3229 3230 3231
	/*
	 * Avoid that the SCSI host can be removed by srp_remove_target()
	 * before this function returns.
	 */
	scsi_host_get(target->scsi_host);

3232 3233
	mutex_lock(&host->add_target_mutex);

3234 3235
	ret = srp_parse_options(buf, target);
	if (ret)
3236
		goto out;
3237

3238 3239
	target->req_ring_size = target->queue_size - SRP_TSK_MGMT_SQ_SIZE;

3240 3241 3242 3243 3244 3245 3246
	if (!srp_conn_unique(target->srp_host, target)) {
		shost_printk(KERN_INFO, target->scsi_host,
			     PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;initiator_ext=%016llx\n",
			     be64_to_cpu(target->id_ext),
			     be64_to_cpu(target->ioc_guid),
			     be64_to_cpu(target->initiator_ext));
		ret = -EEXIST;
3247
		goto out;
3248 3249
	}

3250
	if (!srp_dev->has_fmr && !srp_dev->has_fr && !target->allow_ext_sg &&
3251
	    target->cmd_sg_cnt < target->sg_tablesize) {
3252
		pr_warn("No MR pool and no external indirect descriptors, limiting sg_tablesize to cmd_sg_cnt\n");
3253 3254 3255 3256 3257 3258
		target->sg_tablesize = target->cmd_sg_cnt;
	}

	target_host->sg_tablesize = target->sg_tablesize;
	target->indirect_size = target->sg_tablesize *
				sizeof (struct srp_direct_buf);
3259 3260 3261 3262
	target->max_iu_len = sizeof (struct srp_cmd) +
			     sizeof (struct srp_indirect_buf) +
			     target->cmd_sg_cnt * sizeof (struct srp_direct_buf);

3263
	INIT_WORK(&target->tl_err_work, srp_tl_err_work);
3264
	INIT_WORK(&target->remove_work, srp_remove_work);
3265
	spin_lock_init(&target->lock);
3266
	ret = ib_query_gid(ibdev, host->port, 0, &target->sgid, NULL);
3267
	if (ret)
3268
		goto out;
3269

B
Bart Van Assche 已提交
3270 3271 3272 3273 3274 3275 3276 3277 3278
	ret = -ENOMEM;
	target->ch_count = max_t(unsigned, num_online_nodes(),
				 min(ch_count ? :
				     min(4 * num_online_nodes(),
					 ibdev->num_comp_vectors),
				     num_online_cpus()));
	target->ch = kcalloc(target->ch_count, sizeof(*target->ch),
			     GFP_KERNEL);
	if (!target->ch)
3279
		goto out;
3280

B
Bart Van Assche 已提交
3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308
	node_idx = 0;
	for_each_online_node(node) {
		const int ch_start = (node_idx * target->ch_count /
				      num_online_nodes());
		const int ch_end = ((node_idx + 1) * target->ch_count /
				    num_online_nodes());
		const int cv_start = (node_idx * ibdev->num_comp_vectors /
				      num_online_nodes() + target->comp_vector)
				     % ibdev->num_comp_vectors;
		const int cv_end = ((node_idx + 1) * ibdev->num_comp_vectors /
				    num_online_nodes() + target->comp_vector)
				   % ibdev->num_comp_vectors;
		int cpu_idx = 0;

		for_each_online_cpu(cpu) {
			if (cpu_to_node(cpu) != node)
				continue;
			if (ch_start + cpu_idx >= ch_end)
				continue;
			ch = &target->ch[ch_start + cpu_idx];
			ch->target = target;
			ch->comp_vector = cv_start == cv_end ? cv_start :
				cv_start + cpu_idx % (cv_end - cv_start);
			spin_lock_init(&ch->lock);
			INIT_LIST_HEAD(&ch->free_tx);
			ret = srp_new_cm_id(ch);
			if (ret)
				goto err_disconnect;
3309

B
Bart Van Assche 已提交
3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329
			ret = srp_create_ch_ib(ch);
			if (ret)
				goto err_disconnect;

			ret = srp_alloc_req_data(ch);
			if (ret)
				goto err_disconnect;

			ret = srp_connect_ch(ch, multich);
			if (ret) {
				shost_printk(KERN_ERR, target->scsi_host,
					     PFX "Connection %d/%d failed\n",
					     ch_start + cpu_idx,
					     target->ch_count);
				if (node_idx == 0 && cpu_idx == 0) {
					goto err_disconnect;
				} else {
					srp_free_ch_ib(target, ch);
					srp_free_req_data(target, ch);
					target->ch_count = ch - target->ch;
3330
					goto connected;
B
Bart Van Assche 已提交
3331 3332 3333 3334 3335 3336 3337
				}
			}

			multich = true;
			cpu_idx++;
		}
		node_idx++;
3338 3339
	}

3340
connected:
B
Bart Van Assche 已提交
3341 3342
	target->scsi_host->nr_hw_queues = target->ch_count;

3343 3344 3345 3346
	ret = srp_add_target(host, target);
	if (ret)
		goto err_disconnect;

3347 3348 3349 3350 3351
	if (target->state != SRP_TARGET_REMOVED) {
		shost_printk(KERN_DEBUG, target->scsi_host, PFX
			     "new target: id_ext %016llx ioc_guid %016llx pkey %04x service_id %016llx sgid %pI6 dgid %pI6\n",
			     be64_to_cpu(target->id_ext),
			     be64_to_cpu(target->ioc_guid),
3352
			     be16_to_cpu(target->pkey),
3353
			     be64_to_cpu(target->service_id),
3354
			     target->sgid.raw, target->orig_dgid.raw);
3355
	}
B
Bart Van Assche 已提交
3356

3357 3358 3359 3360
	ret = count;

out:
	mutex_unlock(&host->add_target_mutex);
3361 3362

	scsi_host_put(target->scsi_host);
3363 3364
	if (ret < 0)
		scsi_host_put(target->scsi_host);
3365

3366
	return ret;
3367 3368 3369 3370

err_disconnect:
	srp_disconnect_target(target);

B
Bart Van Assche 已提交
3371 3372 3373 3374 3375
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
		srp_free_ch_ib(target, ch);
		srp_free_req_data(target, ch);
	}
3376

B
Bart Van Assche 已提交
3377
	kfree(target->ch);
3378
	goto out;
3379 3380
}

3381
static DEVICE_ATTR(add_target, S_IWUSR, NULL, srp_create_target);
3382

3383 3384
static ssize_t show_ibdev(struct device *dev, struct device_attribute *attr,
			  char *buf)
3385
{
3386
	struct srp_host *host = container_of(dev, struct srp_host, dev);
3387

3388
	return sprintf(buf, "%s\n", host->srp_dev->dev->name);
3389 3390
}

3391
static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);
3392

3393 3394
static ssize_t show_port(struct device *dev, struct device_attribute *attr,
			 char *buf)
3395
{
3396
	struct srp_host *host = container_of(dev, struct srp_host, dev);
3397 3398 3399 3400

	return sprintf(buf, "%d\n", host->port);
}

3401
static DEVICE_ATTR(port, S_IRUGO, show_port, NULL);
3402

3403
static struct srp_host *srp_add_port(struct srp_device *device, u8 port)
3404 3405 3406 3407 3408 3409 3410 3411
{
	struct srp_host *host;

	host = kzalloc(sizeof *host, GFP_KERNEL);
	if (!host)
		return NULL;

	INIT_LIST_HEAD(&host->target_list);
3412
	spin_lock_init(&host->target_lock);
3413
	init_completion(&host->released);
3414
	mutex_init(&host->add_target_mutex);
3415
	host->srp_dev = device;
3416 3417
	host->port = port;

3418 3419
	host->dev.class = &srp_class;
	host->dev.parent = device->dev->dma_device;
3420
	dev_set_name(&host->dev, "srp-%s-%d", device->dev->name, port);
3421

3422
	if (device_register(&host->dev))
3423
		goto free_host;
3424
	if (device_create_file(&host->dev, &dev_attr_add_target))
3425
		goto err_class;
3426
	if (device_create_file(&host->dev, &dev_attr_ibdev))
3427
		goto err_class;
3428
	if (device_create_file(&host->dev, &dev_attr_port))
3429 3430 3431 3432 3433
		goto err_class;

	return host;

err_class:
3434
	device_unregister(&host->dev);
3435

3436
free_host:
3437 3438 3439 3440 3441 3442 3443
	kfree(host);

	return NULL;
}

static void srp_add_one(struct ib_device *device)
{
3444 3445
	struct srp_device *srp_dev;
	struct ib_device_attr *dev_attr;
3446
	struct srp_host *host;
3447
	int mr_page_shift, p;
3448
	u64 max_pages_per_mr;
3449

3450 3451
	dev_attr = kmalloc(sizeof *dev_attr, GFP_KERNEL);
	if (!dev_attr)
3452
		return;
3453

3454
	if (ib_query_device(device, dev_attr)) {
3455
		pr_warn("Query device failed for %s\n", device->name);
3456 3457 3458 3459 3460 3461 3462
		goto free_attr;
	}

	srp_dev = kmalloc(sizeof *srp_dev, GFP_KERNEL);
	if (!srp_dev)
		goto free_attr;

3463 3464
	srp_dev->has_fmr = (device->alloc_fmr && device->dealloc_fmr &&
			    device->map_phys_fmr && device->unmap_fmr);
3465 3466 3467 3468 3469 3470 3471
	srp_dev->has_fr = (dev_attr->device_cap_flags &
			   IB_DEVICE_MEM_MGT_EXTENSIONS);
	if (!srp_dev->has_fmr && !srp_dev->has_fr)
		dev_warn(&device->dev, "neither FMR nor FR is supported\n");

	srp_dev->use_fast_reg = (srp_dev->has_fr &&
				 (!srp_dev->has_fmr || prefer_fr));
3472
	srp_dev->use_fmr = !srp_dev->use_fast_reg && srp_dev->has_fmr;
3473

3474 3475
	/*
	 * Use the smallest page size supported by the HCA, down to a
3476 3477
	 * minimum of 4096 bytes. We're unlikely to build large sglists
	 * out of smaller entries.
3478
	 */
3479 3480 3481 3482 3483 3484 3485
	mr_page_shift		= max(12, ffs(dev_attr->page_size_cap) - 1);
	srp_dev->mr_page_size	= 1 << mr_page_shift;
	srp_dev->mr_page_mask	= ~((u64) srp_dev->mr_page_size - 1);
	max_pages_per_mr	= dev_attr->max_mr_size;
	do_div(max_pages_per_mr, srp_dev->mr_page_size);
	srp_dev->max_pages_per_mr = min_t(u64, SRP_MAX_PAGES_PER_MR,
					  max_pages_per_mr);
3486 3487 3488 3489 3490
	if (srp_dev->use_fast_reg) {
		srp_dev->max_pages_per_mr =
			min_t(u32, srp_dev->max_pages_per_mr,
			      dev_attr->max_fast_reg_page_list_len);
	}
3491 3492
	srp_dev->mr_max_size	= srp_dev->mr_page_size *
				   srp_dev->max_pages_per_mr;
3493
	pr_debug("%s: mr_page_shift = %d, dev_attr->max_mr_size = %#llx, dev_attr->max_fast_reg_page_list_len = %u, max_pages_per_mr = %d, mr_max_size = %#x\n",
3494
		 device->name, mr_page_shift, dev_attr->max_mr_size,
3495
		 dev_attr->max_fast_reg_page_list_len,
3496
		 srp_dev->max_pages_per_mr, srp_dev->mr_max_size);
3497 3498 3499 3500 3501 3502 3503 3504

	INIT_LIST_HEAD(&srp_dev->dev_list);

	srp_dev->dev = device;
	srp_dev->pd  = ib_alloc_pd(device);
	if (IS_ERR(srp_dev->pd))
		goto free_dev;

3505 3506 3507 3508 3509 3510 3511 3512 3513 3514
	if (!register_always || (!srp_dev->has_fmr && !srp_dev->has_fr)) {
		srp_dev->global_mr = ib_get_dma_mr(srp_dev->pd,
						   IB_ACCESS_LOCAL_WRITE |
						   IB_ACCESS_REMOTE_READ |
						   IB_ACCESS_REMOTE_WRITE);
		if (IS_ERR(srp_dev->global_mr))
			goto err_pd;
	} else {
		srp_dev->global_mr = NULL;
	}
3515

3516
	for (p = rdma_start_port(device); p <= rdma_end_port(device); ++p) {
3517
		host = srp_add_port(srp_dev, p);
3518
		if (host)
3519
			list_add_tail(&host->list, &srp_dev->dev_list);
3520 3521
	}

3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533
	ib_set_client_data(device, &srp_client, srp_dev);

	goto free_attr;

err_pd:
	ib_dealloc_pd(srp_dev->pd);

free_dev:
	kfree(srp_dev);

free_attr:
	kfree(dev_attr);
3534 3535
}

3536
static void srp_remove_one(struct ib_device *device, void *client_data)
3537
{
3538
	struct srp_device *srp_dev;
3539
	struct srp_host *host, *tmp_host;
3540
	struct srp_target_port *target;
3541

3542
	srp_dev = client_data;
3543 3544
	if (!srp_dev)
		return;
3545

3546
	list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) {
3547
		device_unregister(&host->dev);
3548 3549 3550 3551 3552 3553 3554
		/*
		 * Wait for the sysfs entry to go away, so that no new
		 * target ports can be created.
		 */
		wait_for_completion(&host->released);

		/*
3555
		 * Remove all target ports.
3556
		 */
3557
		spin_lock(&host->target_lock);
3558 3559
		list_for_each_entry(target, &host->target_list, list)
			srp_queue_remove_work(target);
3560
		spin_unlock(&host->target_lock);
3561 3562

		/*
3563
		 * Wait for tl_err and target port removal tasks.
3564
		 */
3565
		flush_workqueue(system_long_wq);
3566
		flush_workqueue(srp_remove_wq);
3567 3568 3569 3570

		kfree(host);
	}

3571 3572
	if (srp_dev->global_mr)
		ib_dereg_mr(srp_dev->global_mr);
3573 3574 3575
	ib_dealloc_pd(srp_dev->pd);

	kfree(srp_dev);
3576 3577
}

3578
static struct srp_function_template ib_srp_transport_functions = {
3579 3580
	.has_rport_state	 = true,
	.reset_timer_if_blocked	 = true,
3581
	.reconnect_delay	 = &srp_reconnect_delay,
3582 3583 3584
	.fast_io_fail_tmo	 = &srp_fast_io_fail_tmo,
	.dev_loss_tmo		 = &srp_dev_loss_tmo,
	.reconnect		 = srp_rport_reconnect,
3585
	.rport_delete		 = srp_rport_delete,
3586
	.terminate_rport_io	 = srp_terminate_io,
3587 3588
};

3589 3590 3591 3592
static int __init srp_init_module(void)
{
	int ret;

3593
	BUILD_BUG_ON(FIELD_SIZEOF(struct ib_wc, wr_id) < sizeof(void *));
3594

3595
	if (srp_sg_tablesize) {
3596
		pr_warn("srp_sg_tablesize is deprecated, please use cmd_sg_entries\n");
3597 3598 3599 3600 3601 3602 3603 3604
		if (!cmd_sg_entries)
			cmd_sg_entries = srp_sg_tablesize;
	}

	if (!cmd_sg_entries)
		cmd_sg_entries = SRP_DEF_SG_TABLESIZE;

	if (cmd_sg_entries > 255) {
3605
		pr_warn("Clamping cmd_sg_entries to 255\n");
3606
		cmd_sg_entries = 255;
3607 3608
	}

3609 3610 3611
	if (!indirect_sg_entries)
		indirect_sg_entries = cmd_sg_entries;
	else if (indirect_sg_entries < cmd_sg_entries) {
3612 3613
		pr_warn("Bumping up indirect_sg_entries to match cmd_sg_entries (%u)\n",
			cmd_sg_entries);
3614 3615 3616
		indirect_sg_entries = cmd_sg_entries;
	}

3617
	srp_remove_wq = create_workqueue("srp_remove");
3618 3619
	if (!srp_remove_wq) {
		ret = -ENOMEM;
3620 3621 3622 3623
		goto out;
	}

	ret = -ENOMEM;
3624 3625 3626
	ib_srp_transport_template =
		srp_attach_transport(&ib_srp_transport_functions);
	if (!ib_srp_transport_template)
3627
		goto destroy_wq;
3628

3629 3630
	ret = class_register(&srp_class);
	if (ret) {
3631
		pr_err("couldn't register class infiniband_srp\n");
3632
		goto release_tr;
3633 3634
	}

3635 3636
	ib_sa_register_client(&srp_sa_client);

3637 3638
	ret = ib_register_client(&srp_client);
	if (ret) {
3639
		pr_err("couldn't register IB client\n");
3640
		goto unreg_sa;
3641 3642
	}

3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655
out:
	return ret;

unreg_sa:
	ib_sa_unregister_client(&srp_sa_client);
	class_unregister(&srp_class);

release_tr:
	srp_release_transport(ib_srp_transport_template);

destroy_wq:
	destroy_workqueue(srp_remove_wq);
	goto out;
3656 3657 3658 3659 3660
}

static void __exit srp_cleanup_module(void)
{
	ib_unregister_client(&srp_client);
3661
	ib_sa_unregister_client(&srp_sa_client);
3662
	class_unregister(&srp_class);
3663
	srp_release_transport(ib_srp_transport_template);
3664
	destroy_workqueue(srp_remove_wq);
3665 3666 3667 3668
}

module_init(srp_init_module);
module_exit(srp_cleanup_module);