ib_srp.c 110.9 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32
/*
 * Copyright (c) 2005 Cisco Systems.  All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

33
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
34

35 36 37 38 39 40 41
#include <linux/module.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/err.h>
#include <linux/string.h>
#include <linux/parser.h>
#include <linux/random.h>
42
#include <linux/jiffies.h>
43
#include <linux/lockdep.h>
B
Bart Van Assche 已提交
44
#include <linux/inet.h>
45
#include <rdma/ib_cache.h>
46

A
Arun Sharma 已提交
47
#include <linux/atomic.h>
48 49 50 51

#include <scsi/scsi.h>
#include <scsi/scsi_device.h>
#include <scsi/scsi_dbg.h>
52
#include <scsi/scsi_tcq.h>
53
#include <scsi/srp.h>
54
#include <scsi/scsi_transport_srp.h>
55 56 57 58 59

#include "ib_srp.h"

#define DRV_NAME	"ib_srp"
#define PFX		DRV_NAME ": "
60 61
#define DRV_VERSION	"2.0"
#define DRV_RELDATE	"July 26, 2015"
62 63

MODULE_AUTHOR("Roland Dreier");
64
MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator");
65
MODULE_LICENSE("Dual BSD/GPL");
66
MODULE_INFO(release_date, DRV_RELDATE);
67

68 69 70 71 72
#if !defined(CONFIG_DYNAMIC_DEBUG)
#define DEFINE_DYNAMIC_DEBUG_METADATA(name, fmt)
#define DYNAMIC_DEBUG_BRANCH(descriptor) false
#endif

73 74
static unsigned int srp_sg_tablesize;
static unsigned int cmd_sg_entries;
75 76
static unsigned int indirect_sg_entries;
static bool allow_ext_sg;
77 78
static bool prefer_fr = true;
static bool register_always = true;
79
static bool never_register;
80
static int topspin_workarounds = 1;
81

82 83
module_param(srp_sg_tablesize, uint, 0444);
MODULE_PARM_DESC(srp_sg_tablesize, "Deprecated name for cmd_sg_entries");
84

85 86 87
module_param(cmd_sg_entries, uint, 0444);
MODULE_PARM_DESC(cmd_sg_entries,
		 "Default number of gather/scatter entries in the SRP command (default is 12, max 255)");
88

89 90
module_param(indirect_sg_entries, uint, 0444);
MODULE_PARM_DESC(indirect_sg_entries,
91
		 "Default max number of gather/scatter entries (default is 12, max is " __stringify(SG_MAX_SEGMENTS) ")");
92 93 94 95 96

module_param(allow_ext_sg, bool, 0444);
MODULE_PARM_DESC(allow_ext_sg,
		  "Default behavior when there are more than cmd_sg_entries S/G entries after mapping; fails the request when false (default false)");

97 98 99 100
module_param(topspin_workarounds, int, 0444);
MODULE_PARM_DESC(topspin_workarounds,
		 "Enable workarounds for Topspin/Cisco SRP target bugs if != 0");

101 102 103 104
module_param(prefer_fr, bool, 0444);
MODULE_PARM_DESC(prefer_fr,
"Whether to use fast registration if both FMR and fast registration are supported");

105 106 107 108
module_param(register_always, bool, 0444);
MODULE_PARM_DESC(register_always,
		 "Use memory registration even for contiguous memory regions");

109 110 111
module_param(never_register, bool, 0444);
MODULE_PARM_DESC(never_register, "Never register memory");

112
static const struct kernel_param_ops srp_tmo_ops;
113

114 115 116 117 118
static int srp_reconnect_delay = 10;
module_param_cb(reconnect_delay, &srp_tmo_ops, &srp_reconnect_delay,
		S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(reconnect_delay, "Time between successive reconnect attempts");

119 120 121 122 123 124 125 126
static int srp_fast_io_fail_tmo = 15;
module_param_cb(fast_io_fail_tmo, &srp_tmo_ops, &srp_fast_io_fail_tmo,
		S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(fast_io_fail_tmo,
		 "Number of seconds between the observation of a transport"
		 " layer error and failing all I/O. \"off\" means that this"
		 " functionality is disabled.");

127
static int srp_dev_loss_tmo = 600;
128 129 130 131 132 133 134 135 136 137
module_param_cb(dev_loss_tmo, &srp_tmo_ops, &srp_dev_loss_tmo,
		S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(dev_loss_tmo,
		 "Maximum number of seconds that the SRP transport should"
		 " insulate transport layer errors. After this time has been"
		 " exceeded the SCSI host is removed. Should be"
		 " between 1 and " __stringify(SCSI_DEVICE_BLOCK_MAX_TIMEOUT)
		 " if fast_io_fail_tmo has not been set. \"off\" means that"
		 " this functionality is disabled.");

B
Bart Van Assche 已提交
138 139 140 141 142
static unsigned ch_count;
module_param(ch_count, uint, 0444);
MODULE_PARM_DESC(ch_count,
		 "Number of RDMA channels to use for communication with an SRP target. Using more than one channel improves performance if the HCA supports multiple completion vectors. The default value is the minimum of four times the number of online CPU sockets and the number of completion vectors supported by the HCA.");

143
static void srp_add_one(struct ib_device *device);
144
static void srp_remove_one(struct ib_device *device, void *client_data);
C
Christoph Hellwig 已提交
145 146 147
static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc);
static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc,
		const char *opname);
B
Bart Van Assche 已提交
148 149 150
static int srp_ib_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event);
static int srp_rdma_cm_handler(struct rdma_cm_id *cm_id,
			       struct rdma_cm_event *event);
151

152
static struct scsi_transport_template *ib_srp_transport_template;
153
static struct workqueue_struct *srp_remove_wq;
154

155 156 157 158 159 160
static struct ib_client srp_client = {
	.name   = "srp",
	.add    = srp_add_one,
	.remove = srp_remove_one
};

161 162
static struct ib_sa_client srp_sa_client;

163 164 165 166 167 168 169 170 171 172 173 174 175 176
static int srp_tmo_get(char *buffer, const struct kernel_param *kp)
{
	int tmo = *(int *)kp->arg;

	if (tmo >= 0)
		return sprintf(buffer, "%d", tmo);
	else
		return sprintf(buffer, "off");
}

static int srp_tmo_set(const char *val, const struct kernel_param *kp)
{
	int tmo, res;

177 178 179 180
	res = srp_parse_tmo(&tmo, val);
	if (res)
		goto out;

181 182 183 184 185
	if (kp->arg == &srp_reconnect_delay)
		res = srp_tmo_valid(tmo, srp_fast_io_fail_tmo,
				    srp_dev_loss_tmo);
	else if (kp->arg == &srp_fast_io_fail_tmo)
		res = srp_tmo_valid(srp_reconnect_delay, tmo, srp_dev_loss_tmo);
186
	else
187 188
		res = srp_tmo_valid(srp_reconnect_delay, srp_fast_io_fail_tmo,
				    tmo);
189 190 191 192 193 194 195 196
	if (res)
		goto out;
	*(int *)kp->arg = tmo;

out:
	return res;
}

197
static const struct kernel_param_ops srp_tmo_ops = {
198 199 200 201
	.get = srp_tmo_get,
	.set = srp_tmo_set,
};

202 203 204 205 206 207 208 209 210 211
static inline struct srp_target_port *host_to_target(struct Scsi_Host *host)
{
	return (struct srp_target_port *) host->hostdata;
}

static const char *srp_target_info(struct Scsi_Host *host)
{
	return host_to_target(host)->target_name;
}

212 213 214
static int srp_target_is_topspin(struct srp_target_port *target)
{
	static const u8 topspin_oui[3] = { 0x00, 0x05, 0xad };
215
	static const u8 cisco_oui[3]   = { 0x00, 0x1b, 0x0d };
216 217

	return topspin_workarounds &&
218 219
		(!memcmp(&target->ioc_guid, topspin_oui, sizeof topspin_oui) ||
		 !memcmp(&target->ioc_guid, cisco_oui, sizeof cisco_oui));
220 221
}

222 223 224 225 226 227 228 229 230 231 232 233 234 235
static struct srp_iu *srp_alloc_iu(struct srp_host *host, size_t size,
				   gfp_t gfp_mask,
				   enum dma_data_direction direction)
{
	struct srp_iu *iu;

	iu = kmalloc(sizeof *iu, gfp_mask);
	if (!iu)
		goto out;

	iu->buf = kzalloc(size, gfp_mask);
	if (!iu->buf)
		goto out_free_iu;

236 237 238
	iu->dma = ib_dma_map_single(host->srp_dev->dev, iu->buf, size,
				    direction);
	if (ib_dma_mapping_error(host->srp_dev->dev, iu->dma))
239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258
		goto out_free_buf;

	iu->size      = size;
	iu->direction = direction;

	return iu;

out_free_buf:
	kfree(iu->buf);
out_free_iu:
	kfree(iu);
out:
	return NULL;
}

static void srp_free_iu(struct srp_host *host, struct srp_iu *iu)
{
	if (!iu)
		return;

259 260
	ib_dma_unmap_single(host->srp_dev->dev, iu->dma, iu->size,
			    iu->direction);
261 262 263 264 265 266
	kfree(iu->buf);
	kfree(iu);
}

static void srp_qp_event(struct ib_event *event, void *context)
{
267 268
	pr_debug("QP event %s (%d)\n",
		 ib_event_msg(event->event), event->event);
269 270
}

B
Bart Van Assche 已提交
271 272
static int srp_init_ib_qp(struct srp_target_port *target,
			  struct ib_qp *qp)
273 274 275 276 277 278 279 280
{
	struct ib_qp_attr *attr;
	int ret;

	attr = kmalloc(sizeof *attr, GFP_KERNEL);
	if (!attr)
		return -ENOMEM;

281 282
	ret = ib_find_cached_pkey(target->srp_host->srp_dev->dev,
				  target->srp_host->port,
B
Bart Van Assche 已提交
283
				  be16_to_cpu(target->ib_cm.pkey),
284
				  &attr->pkey_index);
285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303
	if (ret)
		goto out;

	attr->qp_state        = IB_QPS_INIT;
	attr->qp_access_flags = (IB_ACCESS_REMOTE_READ |
				    IB_ACCESS_REMOTE_WRITE);
	attr->port_num        = target->srp_host->port;

	ret = ib_modify_qp(qp, attr,
			   IB_QP_STATE		|
			   IB_QP_PKEY_INDEX	|
			   IB_QP_ACCESS_FLAGS	|
			   IB_QP_PORT);

out:
	kfree(attr);
	return ret;
}

B
Bart Van Assche 已提交
304
static int srp_new_ib_cm_id(struct srp_rdma_ch *ch)
D
David Dillow 已提交
305
{
306
	struct srp_target_port *target = ch->target;
D
David Dillow 已提交
307 308
	struct ib_cm_id *new_cm_id;

309
	new_cm_id = ib_create_cm_id(target->srp_host->srp_dev->dev,
B
Bart Van Assche 已提交
310
				    srp_ib_cm_handler, ch);
D
David Dillow 已提交
311 312 313
	if (IS_ERR(new_cm_id))
		return PTR_ERR(new_cm_id);

B
Bart Van Assche 已提交
314 315 316
	if (ch->ib_cm.cm_id)
		ib_destroy_cm_id(ch->ib_cm.cm_id);
	ch->ib_cm.cm_id = new_cm_id;
317 318
	if (rdma_cap_opa_ah(target->srp_host->srp_dev->dev,
			    target->srp_host->port))
B
Bart Van Assche 已提交
319
		ch->ib_cm.path.rec_type = SA_PATH_REC_TYPE_OPA;
320
	else
B
Bart Van Assche 已提交
321 322 323 324 325
		ch->ib_cm.path.rec_type = SA_PATH_REC_TYPE_IB;
	ch->ib_cm.path.sgid = target->sgid;
	ch->ib_cm.path.dgid = target->ib_cm.orig_dgid;
	ch->ib_cm.path.pkey = target->ib_cm.pkey;
	ch->ib_cm.path.service_id = target->ib_cm.service_id;
D
David Dillow 已提交
326 327 328 329

	return 0;
}

B
Bart Van Assche 已提交
330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349
static int srp_new_rdma_cm_id(struct srp_rdma_ch *ch)
{
	struct srp_target_port *target = ch->target;
	struct rdma_cm_id *new_cm_id;
	int ret;

	new_cm_id = rdma_create_id(target->net, srp_rdma_cm_handler, ch,
				   RDMA_PS_TCP, IB_QPT_RC);
	if (IS_ERR(new_cm_id)) {
		ret = PTR_ERR(new_cm_id);
		new_cm_id = NULL;
		goto out;
	}

	init_completion(&ch->done);
	ret = rdma_resolve_addr(new_cm_id, target->rdma_cm.src_specified ?
				(struct sockaddr *)&target->rdma_cm.src : NULL,
				(struct sockaddr *)&target->rdma_cm.dst,
				SRP_PATH_REC_TIMEOUT_MS);
	if (ret) {
350 351
		pr_err("No route available from %pIS to %pIS (%d)\n",
		       &target->rdma_cm.src, &target->rdma_cm.dst, ret);
B
Bart Van Assche 已提交
352 353 354 355 356 357 358 359
		goto out;
	}
	ret = wait_for_completion_interruptible(&ch->done);
	if (ret < 0)
		goto out;

	ret = ch->status;
	if (ret) {
360 361
		pr_err("Resolving address %pIS failed (%d)\n",
		       &target->rdma_cm.dst, ret);
B
Bart Van Assche 已提交
362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381
		goto out;
	}

	swap(ch->rdma_cm.cm_id, new_cm_id);

out:
	if (new_cm_id)
		rdma_destroy_id(new_cm_id);

	return ret;
}

static int srp_new_cm_id(struct srp_rdma_ch *ch)
{
	struct srp_target_port *target = ch->target;

	return target->using_rdma_cm ? srp_new_rdma_cm_id(ch) :
		srp_new_ib_cm_id(ch);
}

382 383 384 385 386 387
static struct ib_fmr_pool *srp_alloc_fmr_pool(struct srp_target_port *target)
{
	struct srp_device *dev = target->srp_host->srp_dev;
	struct ib_fmr_pool_param fmr_param;

	memset(&fmr_param, 0, sizeof(fmr_param));
388
	fmr_param.pool_size	    = target->mr_pool_size;
389 390
	fmr_param.dirty_watermark   = fmr_param.pool_size / 4;
	fmr_param.cache		    = 1;
391 392
	fmr_param.max_pages_per_fmr = dev->max_pages_per_mr;
	fmr_param.page_shift	    = ilog2(dev->mr_page_size);
393 394 395 396 397 398 399
	fmr_param.access	    = (IB_ACCESS_LOCAL_WRITE |
				       IB_ACCESS_REMOTE_WRITE |
				       IB_ACCESS_REMOTE_READ);

	return ib_create_fmr_pool(dev->pd, &fmr_param);
}

400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433
/**
 * srp_destroy_fr_pool() - free the resources owned by a pool
 * @pool: Fast registration pool to be destroyed.
 */
static void srp_destroy_fr_pool(struct srp_fr_pool *pool)
{
	int i;
	struct srp_fr_desc *d;

	if (!pool)
		return;

	for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
		if (d->mr)
			ib_dereg_mr(d->mr);
	}
	kfree(pool);
}

/**
 * srp_create_fr_pool() - allocate and initialize a pool for fast registration
 * @device:            IB device to allocate fast registration descriptors for.
 * @pd:                Protection domain associated with the FR descriptors.
 * @pool_size:         Number of descriptors to allocate.
 * @max_page_list_len: Maximum fast registration work request page list length.
 */
static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device,
					      struct ib_pd *pd, int pool_size,
					      int max_page_list_len)
{
	struct srp_fr_pool *pool;
	struct srp_fr_desc *d;
	struct ib_mr *mr;
	int i, ret = -EINVAL;
434
	enum ib_mr_type mr_type;
435 436 437 438 439 440 441 442 443 444 445 446 447

	if (pool_size <= 0)
		goto err;
	ret = -ENOMEM;
	pool = kzalloc(sizeof(struct srp_fr_pool) +
		       pool_size * sizeof(struct srp_fr_desc), GFP_KERNEL);
	if (!pool)
		goto err;
	pool->size = pool_size;
	pool->max_page_list_len = max_page_list_len;
	spin_lock_init(&pool->lock);
	INIT_LIST_HEAD(&pool->free_list);

448 449 450 451 452
	if (device->attrs.device_cap_flags & IB_DEVICE_SG_GAPS_REG)
		mr_type = IB_MR_TYPE_SG_GAPS;
	else
		mr_type = IB_MR_TYPE_MEM_REG;

453
	for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
454
		mr = ib_alloc_mr(pd, mr_type, max_page_list_len);
455 456
		if (IS_ERR(mr)) {
			ret = PTR_ERR(mr);
457 458 459
			if (ret == -ENOMEM)
				pr_info("%s: ib_alloc_mr() failed. Try to reduce max_cmd_per_lun, max_sect or ch_count\n",
					dev_name(&device->dev));
460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520
			goto destroy_pool;
		}
		d->mr = mr;
		list_add_tail(&d->entry, &pool->free_list);
	}

out:
	return pool;

destroy_pool:
	srp_destroy_fr_pool(pool);

err:
	pool = ERR_PTR(ret);
	goto out;
}

/**
 * srp_fr_pool_get() - obtain a descriptor suitable for fast registration
 * @pool: Pool to obtain descriptor from.
 */
static struct srp_fr_desc *srp_fr_pool_get(struct srp_fr_pool *pool)
{
	struct srp_fr_desc *d = NULL;
	unsigned long flags;

	spin_lock_irqsave(&pool->lock, flags);
	if (!list_empty(&pool->free_list)) {
		d = list_first_entry(&pool->free_list, typeof(*d), entry);
		list_del(&d->entry);
	}
	spin_unlock_irqrestore(&pool->lock, flags);

	return d;
}

/**
 * srp_fr_pool_put() - put an FR descriptor back in the free list
 * @pool: Pool the descriptor was allocated from.
 * @desc: Pointer to an array of fast registration descriptor pointers.
 * @n:    Number of descriptors to put back.
 *
 * Note: The caller must already have queued an invalidation request for
 * desc->mr->rkey before calling this function.
 */
static void srp_fr_pool_put(struct srp_fr_pool *pool, struct srp_fr_desc **desc,
			    int n)
{
	unsigned long flags;
	int i;

	spin_lock_irqsave(&pool->lock, flags);
	for (i = 0; i < n; i++)
		list_add(&desc[i]->entry, &pool->free_list);
	spin_unlock_irqrestore(&pool->lock, flags);
}

static struct srp_fr_pool *srp_alloc_fr_pool(struct srp_target_port *target)
{
	struct srp_device *dev = target->srp_host->srp_dev;

521
	return srp_create_fr_pool(dev->dev, dev->pd, target->mr_pool_size,
522 523 524
				  dev->max_pages_per_mr);
}

525 526
/**
 * srp_destroy_qp() - destroy an RDMA queue pair
527
 * @ch: SRP RDMA channel.
528
 *
S
Steve Wise 已提交
529 530
 * Drain the qp before destroying it.  This avoids that the receive
 * completion handler can access the queue pair while it is
531 532
 * being destroyed.
 */
533
static void srp_destroy_qp(struct srp_rdma_ch *ch)
534
{
535 536 537 538
	spin_lock_irq(&ch->lock);
	ib_process_cq_direct(ch->send_cq, -1);
	spin_unlock_irq(&ch->lock);

539 540
	ib_drain_qp(ch->qp);
	ib_destroy_qp(ch->qp);
541 542
}

543
static int srp_create_ch_ib(struct srp_rdma_ch *ch)
544
{
545
	struct srp_target_port *target = ch->target;
546
	struct srp_device *dev = target->srp_host->srp_dev;
547
	struct ib_qp_init_attr *init_attr;
548 549
	struct ib_cq *recv_cq, *send_cq;
	struct ib_qp *qp;
550
	struct ib_fmr_pool *fmr_pool = NULL;
551
	struct srp_fr_pool *fr_pool = NULL;
552
	const int m = 1 + dev->use_fast_reg * target->mr_per_cmd * 2;
553 554 555 556 557 558
	int ret;

	init_attr = kzalloc(sizeof *init_attr, GFP_KERNEL);
	if (!init_attr)
		return -ENOMEM;

S
Steve Wise 已提交
559
	/* queue_size + 1 for ib_drain_rq() */
C
Christoph Hellwig 已提交
560 561
	recv_cq = ib_alloc_cq(dev->dev, ch, target->queue_size + 1,
				ch->comp_vector, IB_POLL_SOFTIRQ);
562 563
	if (IS_ERR(recv_cq)) {
		ret = PTR_ERR(recv_cq);
564
		goto err;
565 566
	}

C
Christoph Hellwig 已提交
567 568
	send_cq = ib_alloc_cq(dev->dev, ch, m * target->queue_size,
				ch->comp_vector, IB_POLL_DIRECT);
569 570
	if (IS_ERR(send_cq)) {
		ret = PTR_ERR(send_cq);
571
		goto err_recv_cq;
572 573
	}

574
	init_attr->event_handler       = srp_qp_event;
575
	init_attr->cap.max_send_wr     = m * target->queue_size;
576
	init_attr->cap.max_recv_wr     = target->queue_size + 1;
577 578
	init_attr->cap.max_recv_sge    = 1;
	init_attr->cap.max_send_sge    = 1;
579
	init_attr->sq_sig_type         = IB_SIGNAL_REQ_WR;
580
	init_attr->qp_type             = IB_QPT_RC;
581 582
	init_attr->send_cq             = send_cq;
	init_attr->recv_cq             = recv_cq;
583

B
Bart Van Assche 已提交
584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599
	if (target->using_rdma_cm) {
		ret = rdma_create_qp(ch->rdma_cm.cm_id, dev->pd, init_attr);
		qp = ch->rdma_cm.cm_id->qp;
	} else {
		qp = ib_create_qp(dev->pd, init_attr);
		if (!IS_ERR(qp)) {
			ret = srp_init_ib_qp(target, qp);
			if (ret)
				ib_destroy_qp(qp);
		} else {
			ret = PTR_ERR(qp);
		}
	}
	if (ret) {
		pr_err("QP creation failed for dev %s: %d\n",
		       dev_name(&dev->dev->dev), ret);
600
		goto err_send_cq;
601 602
	}

603
	if (dev->use_fast_reg) {
604 605 606 607 608 609 610
		fr_pool = srp_alloc_fr_pool(target);
		if (IS_ERR(fr_pool)) {
			ret = PTR_ERR(fr_pool);
			shost_printk(KERN_WARNING, target->scsi_host, PFX
				     "FR pool allocation failed (%d)\n", ret);
			goto err_qp;
		}
611
	} else if (dev->use_fmr) {
612 613 614 615 616 617 618 619 620
		fmr_pool = srp_alloc_fmr_pool(target);
		if (IS_ERR(fmr_pool)) {
			ret = PTR_ERR(fmr_pool);
			shost_printk(KERN_WARNING, target->scsi_host, PFX
				     "FMR pool allocation failed (%d)\n", ret);
			goto err_qp;
		}
	}

621
	if (ch->qp)
622
		srp_destroy_qp(ch);
623
	if (ch->recv_cq)
C
Christoph Hellwig 已提交
624
		ib_free_cq(ch->recv_cq);
625
	if (ch->send_cq)
C
Christoph Hellwig 已提交
626
		ib_free_cq(ch->send_cq);
627

628 629 630
	ch->qp = qp;
	ch->recv_cq = recv_cq;
	ch->send_cq = send_cq;
631

632 633 634 635 636 637 638 639 640 641
	if (dev->use_fast_reg) {
		if (ch->fr_pool)
			srp_destroy_fr_pool(ch->fr_pool);
		ch->fr_pool = fr_pool;
	} else if (dev->use_fmr) {
		if (ch->fmr_pool)
			ib_destroy_fmr_pool(ch->fmr_pool);
		ch->fmr_pool = fmr_pool;
	}

642 643 644 645
	kfree(init_attr);
	return 0;

err_qp:
B
Bart Van Assche 已提交
646 647 648 649
	if (target->using_rdma_cm)
		rdma_destroy_qp(ch->rdma_cm.cm_id);
	else
		ib_destroy_qp(qp);
650 651

err_send_cq:
C
Christoph Hellwig 已提交
652
	ib_free_cq(send_cq);
653 654

err_recv_cq:
C
Christoph Hellwig 已提交
655
	ib_free_cq(recv_cq);
656 657

err:
658 659 660 661
	kfree(init_attr);
	return ret;
}

662 663
/*
 * Note: this function may be called without srp_alloc_iu_bufs() having been
664
 * invoked. Hence the ch->[rt]x_ring checks.
665
 */
666 667
static void srp_free_ch_ib(struct srp_target_port *target,
			   struct srp_rdma_ch *ch)
668
{
669
	struct srp_device *dev = target->srp_host->srp_dev;
670 671
	int i;

B
Bart Van Assche 已提交
672 673 674
	if (!ch->target)
		return;

B
Bart Van Assche 已提交
675 676 677 678 679 680 681 682 683 684
	if (target->using_rdma_cm) {
		if (ch->rdma_cm.cm_id) {
			rdma_destroy_id(ch->rdma_cm.cm_id);
			ch->rdma_cm.cm_id = NULL;
		}
	} else {
		if (ch->ib_cm.cm_id) {
			ib_destroy_cm_id(ch->ib_cm.cm_id);
			ch->ib_cm.cm_id = NULL;
		}
685 686
	}

B
Bart Van Assche 已提交
687 688 689 690
	/* If srp_new_cm_id() succeeded but srp_create_ch_ib() not, return. */
	if (!ch->qp)
		return;

691
	if (dev->use_fast_reg) {
692 693
		if (ch->fr_pool)
			srp_destroy_fr_pool(ch->fr_pool);
694
	} else if (dev->use_fmr) {
695 696
		if (ch->fmr_pool)
			ib_destroy_fmr_pool(ch->fmr_pool);
697
	}
C
Christoph Hellwig 已提交
698

699
	srp_destroy_qp(ch);
C
Christoph Hellwig 已提交
700 701
	ib_free_cq(ch->send_cq);
	ib_free_cq(ch->recv_cq);
702

B
Bart Van Assche 已提交
703 704 705 706 707 708 709 710
	/*
	 * Avoid that the SCSI error handler tries to use this channel after
	 * it has been freed. The SCSI error handler can namely continue
	 * trying to perform recovery actions after scsi_remove_host()
	 * returned.
	 */
	ch->target = NULL;

711 712
	ch->qp = NULL;
	ch->send_cq = ch->recv_cq = NULL;
713

714
	if (ch->rx_ring) {
715
		for (i = 0; i < target->queue_size; ++i)
716 717 718
			srp_free_iu(target->srp_host, ch->rx_ring[i]);
		kfree(ch->rx_ring);
		ch->rx_ring = NULL;
719
	}
720
	if (ch->tx_ring) {
721
		for (i = 0; i < target->queue_size; ++i)
722 723 724
			srp_free_iu(target->srp_host, ch->tx_ring[i]);
		kfree(ch->tx_ring);
		ch->tx_ring = NULL;
725
	}
726 727 728
}

static void srp_path_rec_completion(int status,
729
				    struct sa_path_rec *pathrec,
730
				    void *ch_ptr)
731
{
732 733
	struct srp_rdma_ch *ch = ch_ptr;
	struct srp_target_port *target = ch->target;
734

735
	ch->status = status;
736
	if (status)
737 738
		shost_printk(KERN_ERR, target->scsi_host,
			     PFX "Got failed path rec status %d\n", status);
739
	else
B
Bart Van Assche 已提交
740
		ch->ib_cm.path = *pathrec;
741
	complete(&ch->done);
742 743
}

B
Bart Van Assche 已提交
744
static int srp_ib_lookup_path(struct srp_rdma_ch *ch)
745
{
746
	struct srp_target_port *target = ch->target;
747
	int ret;
748

B
Bart Van Assche 已提交
749
	ch->ib_cm.path.numb_path = 1;
750 751 752

	init_completion(&ch->done);

B
Bart Van Assche 已提交
753
	ch->ib_cm.path_query_id = ib_sa_path_rec_get(&srp_sa_client,
754 755
					       target->srp_host->srp_dev->dev,
					       target->srp_host->port,
B
Bart Van Assche 已提交
756
					       &ch->ib_cm.path,
757 758 759 760 761 762 763 764
					       IB_SA_PATH_REC_SERVICE_ID |
					       IB_SA_PATH_REC_DGID	 |
					       IB_SA_PATH_REC_SGID	 |
					       IB_SA_PATH_REC_NUMB_PATH	 |
					       IB_SA_PATH_REC_PKEY,
					       SRP_PATH_REC_TIMEOUT_MS,
					       GFP_KERNEL,
					       srp_path_rec_completion,
B
Bart Van Assche 已提交
765
					       ch, &ch->ib_cm.path_query);
766 767
	if (ch->ib_cm.path_query_id < 0)
		return ch->ib_cm.path_query_id;
768 769

	ret = wait_for_completion_interruptible(&ch->done);
770
	if (ret < 0)
771
		return ret;
772

773
	if (ch->status < 0)
774
		shost_printk(KERN_WARNING, target->scsi_host,
775
			     PFX "Path record query failed: sgid %pI6, dgid %pI6, pkey %#04x, service_id %#16llx\n",
B
Bart Van Assche 已提交
776 777 778
			     ch->ib_cm.path.sgid.raw, ch->ib_cm.path.dgid.raw,
			     be16_to_cpu(target->ib_cm.pkey),
			     be64_to_cpu(target->ib_cm.service_id));
779

780
	return ch->status;
781 782
}

B
Bart Van Assche 已提交
783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810
static int srp_rdma_lookup_path(struct srp_rdma_ch *ch)
{
	struct srp_target_port *target = ch->target;
	int ret;

	init_completion(&ch->done);

	ret = rdma_resolve_route(ch->rdma_cm.cm_id, SRP_PATH_REC_TIMEOUT_MS);
	if (ret)
		return ret;

	wait_for_completion_interruptible(&ch->done);

	if (ch->status != 0)
		shost_printk(KERN_WARNING, target->scsi_host,
			     PFX "Path resolution failed\n");

	return ch->status;
}

static int srp_lookup_path(struct srp_rdma_ch *ch)
{
	struct srp_target_port *target = ch->target;

	return target->using_rdma_cm ? srp_rdma_lookup_path(ch) :
		srp_ib_lookup_path(ch);
}

811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827
static u8 srp_get_subnet_timeout(struct srp_host *host)
{
	struct ib_port_attr attr;
	int ret;
	u8 subnet_timeout = 18;

	ret = ib_query_port(host->srp_dev->dev, host->port, &attr);
	if (ret == 0)
		subnet_timeout = attr.subnet_timeout;

	if (unlikely(subnet_timeout < 15))
		pr_warn("%s: subnet timeout %d may cause SRP login to fail.\n",
			dev_name(&host->srp_dev->dev->dev), subnet_timeout);

	return subnet_timeout;
}

B
Bart Van Assche 已提交
828
static int srp_send_req(struct srp_rdma_ch *ch, bool multich)
829
{
830
	struct srp_target_port *target = ch->target;
831
	struct {
B
Bart Van Assche 已提交
832 833 834 835
		struct rdma_conn_param	  rdma_param;
		struct srp_login_req_rdma rdma_req;
		struct ib_cm_req_param	  ib_param;
		struct srp_login_req	  ib_req;
836
	} *req = NULL;
837
	char *ipi, *tpi;
838 839 840 841 842 843
	int status;

	req = kzalloc(sizeof *req, GFP_KERNEL);
	if (!req)
		return -ENOMEM;

B
Bart Van Assche 已提交
844 845
	req->ib_param.flow_control = 1;
	req->ib_param.retry_count = target->tl_retry_count;
846 847 848 849 850

	/*
	 * Pick some arbitrary defaults here; we could make these
	 * module parameters if anyone cared about setting them.
	 */
B
Bart Van Assche 已提交
851 852 853 854 855 856 857 858
	req->ib_param.responder_resources = 4;
	req->ib_param.rnr_retry_count = 7;
	req->ib_param.max_cm_retries = 15;

	req->ib_req.opcode = SRP_LOGIN_REQ;
	req->ib_req.tag = 0;
	req->ib_req.req_it_iu_len = cpu_to_be32(target->max_iu_len);
	req->ib_req.req_buf_fmt	= cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
859
					      SRP_BUF_FORMAT_INDIRECT);
B
Bart Van Assche 已提交
860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881
	req->ib_req.req_flags = (multich ? SRP_MULTICHAN_MULTI :
				 SRP_MULTICHAN_SINGLE);

	if (target->using_rdma_cm) {
		req->rdma_param.flow_control = req->ib_param.flow_control;
		req->rdma_param.responder_resources =
			req->ib_param.responder_resources;
		req->rdma_param.initiator_depth = req->ib_param.initiator_depth;
		req->rdma_param.retry_count = req->ib_param.retry_count;
		req->rdma_param.rnr_retry_count = req->ib_param.rnr_retry_count;
		req->rdma_param.private_data = &req->rdma_req;
		req->rdma_param.private_data_len = sizeof(req->rdma_req);

		req->rdma_req.opcode = req->ib_req.opcode;
		req->rdma_req.tag = req->ib_req.tag;
		req->rdma_req.req_it_iu_len = req->ib_req.req_it_iu_len;
		req->rdma_req.req_buf_fmt = req->ib_req.req_buf_fmt;
		req->rdma_req.req_flags	= req->ib_req.req_flags;

		ipi = req->rdma_req.initiator_port_id;
		tpi = req->rdma_req.target_port_id;
	} else {
882 883 884 885
		u8 subnet_timeout;

		subnet_timeout = srp_get_subnet_timeout(target->srp_host);

B
Bart Van Assche 已提交
886 887 888 889 890 891 892 893 894 895 896
		req->ib_param.primary_path = &ch->ib_cm.path;
		req->ib_param.alternate_path = NULL;
		req->ib_param.service_id = target->ib_cm.service_id;
		get_random_bytes(&req->ib_param.starting_psn, 4);
		req->ib_param.starting_psn &= 0xffffff;
		req->ib_param.qp_num = ch->qp->qp_num;
		req->ib_param.qp_type = ch->qp->qp_type;
		req->ib_param.local_cm_response_timeout = subnet_timeout + 2;
		req->ib_param.remote_cm_response_timeout = subnet_timeout + 2;
		req->ib_param.private_data = &req->ib_req;
		req->ib_param.private_data_len = sizeof(req->ib_req);
897

B
Bart Van Assche 已提交
898 899
		ipi = req->ib_req.initiator_port_id;
		tpi = req->ib_req.target_port_id;
900 901
	}

902
	/*
R
Roland Dreier 已提交
903
	 * In the published SRP specification (draft rev. 16a), the
904 905 906 907 908 909 910 911
	 * port identifier format is 8 bytes of ID extension followed
	 * by 8 bytes of GUID.  Older drafts put the two halves in the
	 * opposite order, so that the GUID comes first.
	 *
	 * Targets conforming to these obsolete drafts can be
	 * recognized by the I/O Class they report.
	 */
	if (target->io_class == SRP_REV10_IB_IO_CLASS) {
912 913 914 915
		memcpy(ipi,     &target->sgid.global.interface_id, 8);
		memcpy(ipi + 8, &target->initiator_ext, 8);
		memcpy(tpi,     &target->ioc_guid, 8);
		memcpy(tpi + 8, &target->id_ext, 8);
916
	} else {
917 918 919 920
		memcpy(ipi,     &target->initiator_ext, 8);
		memcpy(ipi + 8, &target->sgid.global.interface_id, 8);
		memcpy(tpi,     &target->id_ext, 8);
		memcpy(tpi + 8, &target->ioc_guid, 8);
921 922
	}

923 924
	/*
	 * Topspin/Cisco SRP targets will reject our login unless we
925 926
	 * zero out the first 8 bytes of our initiator port ID and set
	 * the second 8 bytes to the local node GUID.
927
	 */
928
	if (srp_target_is_topspin(target)) {
929 930 931
		shost_printk(KERN_DEBUG, target->scsi_host,
			     PFX "Topspin/Cisco initiator port ID workaround "
			     "activated for target GUID %016llx\n",
932
			     be64_to_cpu(target->ioc_guid));
933 934
		memset(ipi, 0, 8);
		memcpy(ipi + 8, &target->srp_host->srp_dev->dev->node_guid, 8);
935 936
	}

B
Bart Van Assche 已提交
937 938 939 940
	if (target->using_rdma_cm)
		status = rdma_connect(ch->rdma_cm.cm_id, &req->rdma_param);
	else
		status = ib_send_cm_req(ch->ib_cm.cm_id, &req->ib_param);
941 942 943 944 945 946

	kfree(req);

	return status;
}

947 948 949 950 951 952 953 954 955 956 957 958
static bool srp_queue_remove_work(struct srp_target_port *target)
{
	bool changed = false;

	spin_lock_irq(&target->lock);
	if (target->state != SRP_TARGET_REMOVED) {
		target->state = SRP_TARGET_REMOVED;
		changed = true;
	}
	spin_unlock_irq(&target->lock);

	if (changed)
959
		queue_work(srp_remove_wq, &target->remove_work);
960 961 962 963

	return changed;
}

964 965
static void srp_disconnect_target(struct srp_target_port *target)
{
B
Bart Van Assche 已提交
966
	struct srp_rdma_ch *ch;
B
Bart Van Assche 已提交
967
	int i, ret;
968

969
	/* XXX should send SRP_I_LOGOUT request */
970

971 972 973
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
		ch->connected = false;
B
Bart Van Assche 已提交
974 975 976 977 978 979 980 981 982 983
		ret = 0;
		if (target->using_rdma_cm) {
			if (ch->rdma_cm.cm_id)
				rdma_disconnect(ch->rdma_cm.cm_id);
		} else {
			if (ch->ib_cm.cm_id)
				ret = ib_send_cm_dreq(ch->ib_cm.cm_id,
						      NULL, 0);
		}
		if (ret < 0) {
984 985
			shost_printk(KERN_DEBUG, target->scsi_host,
				     PFX "Sending CM DREQ failed\n");
986
		}
987
	}
988 989
}

990 991
static void srp_free_req_data(struct srp_target_port *target,
			      struct srp_rdma_ch *ch)
992
{
993 994
	struct srp_device *dev = target->srp_host->srp_dev;
	struct ib_device *ibdev = dev->dev;
995 996 997
	struct srp_request *req;
	int i;

998
	if (!ch->req_ring)
999 1000 1001
		return;

	for (i = 0; i < target->req_ring_size; ++i) {
1002
		req = &ch->req_ring[i];
1003
		if (dev->use_fast_reg) {
1004
			kfree(req->fr_list);
1005
		} else {
1006
			kfree(req->fmr_list);
1007 1008
			kfree(req->map_page);
		}
1009 1010 1011 1012 1013 1014
		if (req->indirect_dma_addr) {
			ib_dma_unmap_single(ibdev, req->indirect_dma_addr,
					    target->indirect_size,
					    DMA_TO_DEVICE);
		}
		kfree(req->indirect_desc);
1015
	}
1016

1017 1018
	kfree(ch->req_ring);
	ch->req_ring = NULL;
1019 1020
}

1021
static int srp_alloc_req_data(struct srp_rdma_ch *ch)
1022
{
1023
	struct srp_target_port *target = ch->target;
1024 1025 1026
	struct srp_device *srp_dev = target->srp_host->srp_dev;
	struct ib_device *ibdev = srp_dev->dev;
	struct srp_request *req;
1027
	void *mr_list;
1028 1029 1030
	dma_addr_t dma_addr;
	int i, ret = -ENOMEM;

1031 1032 1033
	ch->req_ring = kcalloc(target->req_ring_size, sizeof(*ch->req_ring),
			       GFP_KERNEL);
	if (!ch->req_ring)
1034 1035 1036
		goto out;

	for (i = 0; i < target->req_ring_size; ++i) {
1037
		req = &ch->req_ring[i];
1038
		mr_list = kmalloc(target->mr_per_cmd * sizeof(void *),
1039 1040 1041
				  GFP_KERNEL);
		if (!mr_list)
			goto out;
1042
		if (srp_dev->use_fast_reg) {
1043
			req->fr_list = mr_list;
1044
		} else {
1045
			req->fmr_list = mr_list;
1046 1047 1048 1049 1050
			req->map_page = kmalloc(srp_dev->max_pages_per_mr *
						sizeof(void *), GFP_KERNEL);
			if (!req->map_page)
				goto out;
		}
1051
		req->indirect_desc = kmalloc(target->indirect_size, GFP_KERNEL);
1052
		if (!req->indirect_desc)
1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068
			goto out;

		dma_addr = ib_dma_map_single(ibdev, req->indirect_desc,
					     target->indirect_size,
					     DMA_TO_DEVICE);
		if (ib_dma_mapping_error(ibdev, dma_addr))
			goto out;

		req->indirect_dma_addr = dma_addr;
	}
	ret = 0;

out:
	return ret;
}

1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083
/**
 * srp_del_scsi_host_attr() - Remove attributes defined in the host template.
 * @shost: SCSI host whose attributes to remove from sysfs.
 *
 * Note: Any attributes defined in the host template and that did not exist
 * before invocation of this function will be ignored.
 */
static void srp_del_scsi_host_attr(struct Scsi_Host *shost)
{
	struct device_attribute **attr;

	for (attr = shost->hostt->shost_attrs; attr && *attr; ++attr)
		device_remove_file(&shost->shost_dev, *attr);
}

1084 1085
static void srp_remove_target(struct srp_target_port *target)
{
B
Bart Van Assche 已提交
1086 1087
	struct srp_rdma_ch *ch;
	int i;
1088

1089 1090
	WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);

1091
	srp_del_scsi_host_attr(target->scsi_host);
1092
	srp_rport_get(target->rport);
1093 1094
	srp_remove_host(target->scsi_host);
	scsi_remove_host(target->scsi_host);
1095
	srp_stop_rport_timers(target->rport);
1096
	srp_disconnect_target(target);
B
Bart Van Assche 已提交
1097
	kobj_ns_drop(KOBJ_NS_TYPE_NET, target->net);
B
Bart Van Assche 已提交
1098 1099 1100 1101
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
		srp_free_ch_ib(target, ch);
	}
1102
	cancel_work_sync(&target->tl_err_work);
1103
	srp_rport_put(target->rport);
B
Bart Van Assche 已提交
1104 1105 1106 1107 1108 1109
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
		srp_free_req_data(target, ch);
	}
	kfree(target->ch);
	target->ch = NULL;
1110 1111 1112 1113 1114

	spin_lock(&target->srp_host->target_lock);
	list_del(&target->list);
	spin_unlock(&target->srp_host->target_lock);

1115 1116 1117
	scsi_host_put(target->scsi_host);
}

D
David Howells 已提交
1118
static void srp_remove_work(struct work_struct *work)
1119
{
D
David Howells 已提交
1120
	struct srp_target_port *target =
1121
		container_of(work, struct srp_target_port, remove_work);
1122

1123
	WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
1124

1125
	srp_remove_target(target);
1126 1127
}

1128 1129 1130 1131 1132 1133 1134
static void srp_rport_delete(struct srp_rport *rport)
{
	struct srp_target_port *target = rport->lld_data;

	srp_queue_remove_work(target);
}

1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148
/**
 * srp_connected_ch() - number of connected channels
 * @target: SRP target port.
 */
static int srp_connected_ch(struct srp_target_port *target)
{
	int i, c = 0;

	for (i = 0; i < target->ch_count; i++)
		c += target->ch[i].connected;

	return c;
}

B
Bart Van Assche 已提交
1149
static int srp_connect_ch(struct srp_rdma_ch *ch, bool multich)
1150
{
1151
	struct srp_target_port *target = ch->target;
1152 1153
	int ret;

1154
	WARN_ON_ONCE(!multich && srp_connected_ch(target) > 0);
1155

1156
	ret = srp_lookup_path(ch);
1157
	if (ret)
B
Bart Van Assche 已提交
1158
		goto out;
1159 1160

	while (1) {
1161
		init_completion(&ch->done);
B
Bart Van Assche 已提交
1162
		ret = srp_send_req(ch, multich);
1163
		if (ret)
B
Bart Van Assche 已提交
1164
			goto out;
1165
		ret = wait_for_completion_interruptible(&ch->done);
1166
		if (ret < 0)
B
Bart Van Assche 已提交
1167
			goto out;
1168 1169 1170 1171 1172 1173 1174

		/*
		 * The CM event handling code will set status to
		 * SRP_PORT_REDIRECT if we get a port redirect REJ
		 * back, or SRP_DLID_REDIRECT if we get a lid/qp
		 * redirect REJ back.
		 */
B
Bart Van Assche 已提交
1175 1176
		ret = ch->status;
		switch (ret) {
1177
		case 0:
1178
			ch->connected = true;
B
Bart Van Assche 已提交
1179
			goto out;
1180 1181

		case SRP_PORT_REDIRECT:
1182
			ret = srp_lookup_path(ch);
1183
			if (ret)
B
Bart Van Assche 已提交
1184
				goto out;
1185 1186 1187 1188 1189
			break;

		case SRP_DLID_REDIRECT:
			break;

D
David Dillow 已提交
1190 1191
		case SRP_STALE_CONN:
			shost_printk(KERN_ERR, target->scsi_host, PFX
1192
				     "giving up on stale connection\n");
B
Bart Van Assche 已提交
1193 1194
			ret = -ECONNRESET;
			goto out;
D
David Dillow 已提交
1195

1196
		default:
B
Bart Van Assche 已提交
1197
			goto out;
1198 1199
		}
	}
B
Bart Van Assche 已提交
1200 1201 1202

out:
	return ret <= 0 ? ret : -ENODEV;
1203 1204
}

C
Christoph Hellwig 已提交
1205 1206 1207 1208 1209 1210 1211
static void srp_inv_rkey_err_done(struct ib_cq *cq, struct ib_wc *wc)
{
	srp_handle_qp_err(cq, wc, "INV RKEY");
}

static int srp_inv_rkey(struct srp_request *req, struct srp_rdma_ch *ch,
		u32 rkey)
1212 1213 1214 1215 1216 1217 1218 1219 1220 1221
{
	struct ib_send_wr *bad_wr;
	struct ib_send_wr wr = {
		.opcode		    = IB_WR_LOCAL_INV,
		.next		    = NULL,
		.num_sge	    = 0,
		.send_flags	    = 0,
		.ex.invalidate_rkey = rkey,
	};

C
Christoph Hellwig 已提交
1222 1223
	wr.wr_cqe = &req->reg_cqe;
	req->reg_cqe.done = srp_inv_rkey_err_done;
1224
	return ib_post_send(ch->qp, &wr, &bad_wr);
1225 1226
}

1227
static void srp_unmap_data(struct scsi_cmnd *scmnd,
1228
			   struct srp_rdma_ch *ch,
1229 1230
			   struct srp_request *req)
{
1231
	struct srp_target_port *target = ch->target;
1232 1233 1234
	struct srp_device *dev = target->srp_host->srp_dev;
	struct ib_device *ibdev = dev->dev;
	int i, res;
1235

1236
	if (!scsi_sglist(scmnd) ||
1237 1238 1239 1240
	    (scmnd->sc_data_direction != DMA_TO_DEVICE &&
	     scmnd->sc_data_direction != DMA_FROM_DEVICE))
		return;

1241 1242 1243 1244
	if (dev->use_fast_reg) {
		struct srp_fr_desc **pfr;

		for (i = req->nmdesc, pfr = req->fr_list; i > 0; i--, pfr++) {
C
Christoph Hellwig 已提交
1245
			res = srp_inv_rkey(req, ch, (*pfr)->mr->rkey);
1246 1247 1248 1249 1250 1251 1252 1253 1254
			if (res < 0) {
				shost_printk(KERN_ERR, target->scsi_host, PFX
				  "Queueing INV WR for rkey %#x failed (%d)\n",
				  (*pfr)->mr->rkey, res);
				queue_work(system_long_wq,
					   &target->tl_err_work);
			}
		}
		if (req->nmdesc)
1255
			srp_fr_pool_put(ch->fr_pool, req->fr_list,
1256
					req->nmdesc);
1257
	} else if (dev->use_fmr) {
1258 1259 1260 1261 1262
		struct ib_pool_fmr **pfmr;

		for (i = req->nmdesc, pfmr = req->fmr_list; i > 0; i--, pfmr++)
			ib_fmr_pool_unmap(*pfmr);
	}
1263

1264 1265
	ib_dma_unmap_sg(ibdev, scsi_sglist(scmnd), scsi_sg_count(scmnd),
			scmnd->sc_data_direction);
1266 1267
}

B
Bart Van Assche 已提交
1268 1269
/**
 * srp_claim_req - Take ownership of the scmnd associated with a request.
1270
 * @ch: SRP RDMA channel.
B
Bart Van Assche 已提交
1271
 * @req: SRP request.
1272
 * @sdev: If not NULL, only take ownership for this SCSI device.
B
Bart Van Assche 已提交
1273 1274 1275 1276 1277 1278
 * @scmnd: If NULL, take ownership of @req->scmnd. If not NULL, only take
 *         ownership of @req->scmnd if it equals @scmnd.
 *
 * Return value:
 * Either NULL or a pointer to the SCSI command the caller became owner of.
 */
1279
static struct scsi_cmnd *srp_claim_req(struct srp_rdma_ch *ch,
B
Bart Van Assche 已提交
1280
				       struct srp_request *req,
1281
				       struct scsi_device *sdev,
B
Bart Van Assche 已提交
1282 1283 1284 1285
				       struct scsi_cmnd *scmnd)
{
	unsigned long flags;

1286
	spin_lock_irqsave(&ch->lock, flags);
1287 1288 1289
	if (req->scmnd &&
	    (!sdev || req->scmnd->device == sdev) &&
	    (!scmnd || req->scmnd == scmnd)) {
B
Bart Van Assche 已提交
1290 1291 1292 1293 1294
		scmnd = req->scmnd;
		req->scmnd = NULL;
	} else {
		scmnd = NULL;
	}
1295
	spin_unlock_irqrestore(&ch->lock, flags);
B
Bart Van Assche 已提交
1296 1297 1298 1299 1300

	return scmnd;
}

/**
B
Bart Van Assche 已提交
1301
 * srp_free_req() - Unmap data and adjust ch->req_lim.
1302
 * @ch:     SRP RDMA channel.
1303 1304 1305
 * @req:    Request to be freed.
 * @scmnd:  SCSI command associated with @req.
 * @req_lim_delta: Amount to be added to @target->req_lim.
B
Bart Van Assche 已提交
1306
 */
1307 1308
static void srp_free_req(struct srp_rdma_ch *ch, struct srp_request *req,
			 struct scsi_cmnd *scmnd, s32 req_lim_delta)
1309
{
1310 1311
	unsigned long flags;

1312
	srp_unmap_data(scmnd, ch, req);
B
Bart Van Assche 已提交
1313

1314 1315 1316
	spin_lock_irqsave(&ch->lock, flags);
	ch->req_lim += req_lim_delta;
	spin_unlock_irqrestore(&ch->lock, flags);
1317 1318
}

1319 1320
static void srp_finish_req(struct srp_rdma_ch *ch, struct srp_request *req,
			   struct scsi_device *sdev, int result)
1321
{
1322
	struct scsi_cmnd *scmnd = srp_claim_req(ch, req, sdev, NULL);
B
Bart Van Assche 已提交
1323 1324

	if (scmnd) {
1325
		srp_free_req(ch, req, scmnd, 0);
1326
		scmnd->result = result;
B
Bart Van Assche 已提交
1327 1328
		scmnd->scsi_done(scmnd);
	}
1329 1330
}

1331
static void srp_terminate_io(struct srp_rport *rport)
1332
{
1333
	struct srp_target_port *target = rport->lld_data;
B
Bart Van Assche 已提交
1334
	struct srp_rdma_ch *ch;
1335 1336
	struct Scsi_Host *shost = target->scsi_host;
	struct scsi_device *sdev;
B
Bart Van Assche 已提交
1337
	int i, j;
1338

1339 1340 1341 1342 1343 1344 1345
	/*
	 * Invoking srp_terminate_io() while srp_queuecommand() is running
	 * is not safe. Hence the warning statement below.
	 */
	shost_for_each_device(sdev, shost)
		WARN_ON_ONCE(sdev->request_queue->request_fn_active);

B
Bart Van Assche 已提交
1346 1347
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
1348

B
Bart Van Assche 已提交
1349 1350 1351 1352 1353 1354
		for (j = 0; j < target->req_ring_size; ++j) {
			struct srp_request *req = &ch->req_ring[j];

			srp_finish_req(ch, req, NULL,
				       DID_TRANSPORT_FAILFAST << 16);
		}
1355 1356
	}
}
1357

1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369
/*
 * It is up to the caller to ensure that srp_rport_reconnect() calls are
 * serialized and that no concurrent srp_queuecommand(), srp_abort(),
 * srp_reset_device() or srp_reset_host() calls will occur while this function
 * is in progress. One way to realize that is not to call this function
 * directly but to call srp_reconnect_rport() instead since that last function
 * serializes calls of this function via rport->mutex and also blocks
 * srp_queuecommand() calls before invoking this function.
 */
static int srp_rport_reconnect(struct srp_rport *rport)
{
	struct srp_target_port *target = rport->lld_data;
B
Bart Van Assche 已提交
1370 1371 1372
	struct srp_rdma_ch *ch;
	int i, j, ret = 0;
	bool multich = false;
1373

1374
	srp_disconnect_target(target);
1375 1376 1377 1378

	if (target->state == SRP_TARGET_SCANNING)
		return -ENODEV;

1379
	/*
1380 1381 1382
	 * Now get a new local CM ID so that we avoid confusing the target in
	 * case things are really fouled up. Doing so also ensures that all CM
	 * callbacks will have finished before a new QP is allocated.
1383
	 */
B
Bart Van Assche 已提交
1384 1385 1386
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
		ret += srp_new_cm_id(ch);
1387
	}
B
Bart Van Assche 已提交
1388 1389 1390 1391
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
		for (j = 0; j < target->req_ring_size; ++j) {
			struct srp_request *req = &ch->req_ring[j];
1392

B
Bart Van Assche 已提交
1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403
			srp_finish_req(ch, req, NULL, DID_RESET << 16);
		}
	}
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
		/*
		 * Whether or not creating a new CM ID succeeded, create a new
		 * QP. This guarantees that all completion callback function
		 * invocations have finished before request resetting starts.
		 */
		ret += srp_create_ch_ib(ch);
1404

B
Bart Van Assche 已提交
1405 1406 1407 1408
		INIT_LIST_HEAD(&ch->free_tx);
		for (j = 0; j < target->queue_size; ++j)
			list_add(&ch->tx_ring[j]->list, &ch->free_tx);
	}
1409 1410 1411

	target->qp_in_error = false;

B
Bart Van Assche 已提交
1412 1413
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
1414
		if (ret)
B
Bart Van Assche 已提交
1415 1416 1417 1418
			break;
		ret = srp_connect_ch(ch, multich);
		multich = true;
	}
1419

1420 1421 1422
	if (ret == 0)
		shost_printk(KERN_INFO, target->scsi_host,
			     PFX "reconnect succeeded\n");
1423 1424 1425 1426

	return ret;
}

1427 1428
static void srp_map_desc(struct srp_map_state *state, dma_addr_t dma_addr,
			 unsigned int dma_len, u32 rkey)
1429
{
1430
	struct srp_direct_buf *desc = state->desc;
1431

1432 1433
	WARN_ON_ONCE(!dma_len);

1434 1435 1436
	desc->va = cpu_to_be64(dma_addr);
	desc->key = cpu_to_be32(rkey);
	desc->len = cpu_to_be32(dma_len);
1437

1438 1439 1440 1441
	state->total_len += dma_len;
	state->desc++;
	state->ndesc++;
}
1442

1443
static int srp_map_finish_fmr(struct srp_map_state *state,
1444
			      struct srp_rdma_ch *ch)
1445
{
1446 1447
	struct srp_target_port *target = ch->target;
	struct srp_device *dev = target->srp_host->srp_dev;
1448 1449
	struct ib_pool_fmr *fmr;
	u64 io_addr = 0;
1450

1451 1452 1453 1454
	if (state->fmr.next >= state->fmr.end) {
		shost_printk(KERN_ERR, ch->target->scsi_host,
			     PFX "Out of MRs (mr_per_cmd = %d)\n",
			     ch->target->mr_per_cmd);
1455
		return -ENOMEM;
1456
	}
1457

S
Sagi Grimberg 已提交
1458 1459 1460 1461 1462
	WARN_ON_ONCE(!dev->use_fmr);

	if (state->npages == 0)
		return 0;

B
Bart Van Assche 已提交
1463
	if (state->npages == 1 && target->global_rkey) {
S
Sagi Grimberg 已提交
1464
		srp_map_desc(state, state->base_dma_addr, state->dma_len,
B
Bart Van Assche 已提交
1465
			     target->global_rkey);
S
Sagi Grimberg 已提交
1466 1467 1468
		goto reset_state;
	}

1469
	fmr = ib_fmr_pool_map_phys(ch->fmr_pool, state->pages,
1470 1471 1472
				   state->npages, io_addr);
	if (IS_ERR(fmr))
		return PTR_ERR(fmr);
1473

1474
	*state->fmr.next++ = fmr;
1475
	state->nmdesc++;
1476

1477 1478
	srp_map_desc(state, state->base_dma_addr & ~dev->mr_page_mask,
		     state->dma_len, fmr->fmr->rkey);
1479

S
Sagi Grimberg 已提交
1480 1481 1482 1483
reset_state:
	state->npages = 0;
	state->dma_len = 0;

1484 1485 1486
	return 0;
}

C
Christoph Hellwig 已提交
1487 1488 1489 1490 1491
static void srp_reg_mr_err_done(struct ib_cq *cq, struct ib_wc *wc)
{
	srp_handle_qp_err(cq, wc, "FAST REG");
}

1492 1493 1494 1495 1496 1497
/*
 * Map up to sg_nents elements of state->sg where *sg_offset_p is the offset
 * where to start in the first element. If sg_offset_p != NULL then
 * *sg_offset_p is updated to the offset in state->sg[retval] of the first
 * byte that has not yet been mapped.
 */
1498
static int srp_map_finish_fr(struct srp_map_state *state,
C
Christoph Hellwig 已提交
1499
			     struct srp_request *req,
1500 1501
			     struct srp_rdma_ch *ch, int sg_nents,
			     unsigned int *sg_offset_p)
1502
{
1503
	struct srp_target_port *target = ch->target;
1504 1505
	struct srp_device *dev = target->srp_host->srp_dev;
	struct ib_send_wr *bad_wr;
1506
	struct ib_reg_wr wr;
1507 1508
	struct srp_fr_desc *desc;
	u32 rkey;
1509
	int n, err;
1510

1511 1512 1513 1514
	if (state->fr.next >= state->fr.end) {
		shost_printk(KERN_ERR, ch->target->scsi_host,
			     PFX "Out of MRs (mr_per_cmd = %d)\n",
			     ch->target->mr_per_cmd);
1515
		return -ENOMEM;
1516
	}
1517

S
Sagi Grimberg 已提交
1518 1519
	WARN_ON_ONCE(!dev->use_fast_reg);

B
Bart Van Assche 已提交
1520
	if (sg_nents == 1 && target->global_rkey) {
1521 1522 1523 1524
		unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0;

		srp_map_desc(state, sg_dma_address(state->sg) + sg_offset,
			     sg_dma_len(state->sg) - sg_offset,
B
Bart Van Assche 已提交
1525
			     target->global_rkey);
1526 1527
		if (sg_offset_p)
			*sg_offset_p = 0;
1528
		return 1;
S
Sagi Grimberg 已提交
1529 1530
	}

1531
	desc = srp_fr_pool_get(ch->fr_pool);
1532 1533 1534 1535 1536 1537
	if (!desc)
		return -ENOMEM;

	rkey = ib_inc_rkey(desc->mr->rkey);
	ib_update_fast_reg_key(desc->mr, rkey);

1538 1539
	n = ib_map_mr_sg(desc->mr, state->sg, sg_nents, sg_offset_p,
			 dev->mr_page_size);
1540 1541
	if (unlikely(n < 0)) {
		srp_fr_pool_put(ch->fr_pool, &desc, 1);
1542
		pr_debug("%s: ib_map_mr_sg(%d, %d) returned %d.\n",
1543
			 dev_name(&req->scmnd->device->sdev_gendev), sg_nents,
1544
			 sg_offset_p ? *sg_offset_p : -1, n);
1545
		return n;
1546
	}
1547

1548
	WARN_ON_ONCE(desc->mr->length == 0);
1549

C
Christoph Hellwig 已提交
1550 1551
	req->reg_cqe.done = srp_reg_mr_err_done;

1552 1553
	wr.wr.next = NULL;
	wr.wr.opcode = IB_WR_REG_MR;
C
Christoph Hellwig 已提交
1554
	wr.wr.wr_cqe = &req->reg_cqe;
1555 1556 1557 1558 1559 1560 1561
	wr.wr.num_sge = 0;
	wr.wr.send_flags = 0;
	wr.mr = desc->mr;
	wr.key = desc->mr->rkey;
	wr.access = (IB_ACCESS_LOCAL_WRITE |
		     IB_ACCESS_REMOTE_READ |
		     IB_ACCESS_REMOTE_WRITE);
1562

1563
	*state->fr.next++ = desc;
1564 1565
	state->nmdesc++;

1566 1567
	srp_map_desc(state, desc->mr->iova,
		     desc->mr->length, desc->mr->rkey);
1568

S
Sagi Grimberg 已提交
1569
	err = ib_post_send(ch->qp, &wr.wr, &bad_wr);
1570 1571
	if (unlikely(err)) {
		WARN_ON_ONCE(err == -ENOMEM);
S
Sagi Grimberg 已提交
1572
		return err;
1573
	}
S
Sagi Grimberg 已提交
1574

1575
	return n;
1576 1577
}

1578
static int srp_map_sg_entry(struct srp_map_state *state,
1579
			    struct srp_rdma_ch *ch,
1580
			    struct scatterlist *sg)
1581
{
1582
	struct srp_target_port *target = ch->target;
1583 1584 1585 1586
	struct srp_device *dev = target->srp_host->srp_dev;
	struct ib_device *ibdev = dev->dev;
	dma_addr_t dma_addr = ib_sg_dma_address(ibdev, sg);
	unsigned int dma_len = ib_sg_dma_len(ibdev, sg);
1587
	unsigned int len = 0;
1588 1589
	int ret;

1590
	WARN_ON_ONCE(!dma_len);
1591

1592
	while (dma_len) {
1593
		unsigned offset = dma_addr & ~dev->mr_page_mask;
1594 1595 1596

		if (state->npages == dev->max_pages_per_mr ||
		    (state->npages > 0 && offset != 0)) {
1597
			ret = srp_map_finish_fmr(state, ch);
1598 1599 1600 1601
			if (ret)
				return ret;
		}

1602
		len = min_t(unsigned int, dma_len, dev->mr_page_size - offset);
1603

1604 1605
		if (!state->npages)
			state->base_dma_addr = dma_addr;
1606
		state->pages[state->npages++] = dma_addr & dev->mr_page_mask;
1607
		state->dma_len += len;
1608 1609 1610 1611
		dma_addr += len;
		dma_len -= len;
	}

1612
	/*
1613
	 * If the end of the MR is not on a page boundary then we need to
1614
	 * close it out and start a new one -- we can only merge at page
1615
	 * boundaries.
1616 1617
	 */
	ret = 0;
1618
	if ((dma_addr & ~dev->mr_page_mask) != 0)
1619
		ret = srp_map_finish_fmr(state, ch);
1620 1621 1622
	return ret;
}

S
Sagi Grimberg 已提交
1623 1624 1625
static int srp_map_sg_fmr(struct srp_map_state *state, struct srp_rdma_ch *ch,
			  struct srp_request *req, struct scatterlist *scat,
			  int count)
1626 1627
{
	struct scatterlist *sg;
1628
	int i, ret;
1629

S
Sagi Grimberg 已提交
1630 1631
	state->pages = req->map_page;
	state->fmr.next = req->fmr_list;
1632
	state->fmr.end = req->fmr_list + ch->target->mr_per_cmd;
S
Sagi Grimberg 已提交
1633 1634

	for_each_sg(scat, sg, count, i) {
1635
		ret = srp_map_sg_entry(state, ch, sg);
S
Sagi Grimberg 已提交
1636 1637
		if (ret)
			return ret;
1638
	}
1639

1640
	ret = srp_map_finish_fmr(state, ch);
S
Sagi Grimberg 已提交
1641 1642 1643 1644 1645 1646 1647 1648 1649 1650
	if (ret)
		return ret;

	return 0;
}

static int srp_map_sg_fr(struct srp_map_state *state, struct srp_rdma_ch *ch,
			 struct srp_request *req, struct scatterlist *scat,
			 int count)
{
1651 1652
	unsigned int sg_offset = 0;

1653
	state->fr.next = req->fr_list;
1654
	state->fr.end = req->fr_list + ch->target->mr_per_cmd;
1655
	state->sg = scat;
S
Sagi Grimberg 已提交
1656

1657 1658 1659
	if (count == 0)
		return 0;

B
Bart Van Assche 已提交
1660
	while (count) {
1661
		int i, n;
S
Sagi Grimberg 已提交
1662

1663
		n = srp_map_finish_fr(state, req, ch, count, &sg_offset);
1664 1665 1666
		if (unlikely(n < 0))
			return n;

B
Bart Van Assche 已提交
1667
		count -= n;
1668 1669 1670
		for (i = 0; i < n; i++)
			state->sg = sg_next(state->sg);
	}
S
Sagi Grimberg 已提交
1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686

	return 0;
}

static int srp_map_sg_dma(struct srp_map_state *state, struct srp_rdma_ch *ch,
			  struct srp_request *req, struct scatterlist *scat,
			  int count)
{
	struct srp_target_port *target = ch->target;
	struct srp_device *dev = target->srp_host->srp_dev;
	struct scatterlist *sg;
	int i;

	for_each_sg(scat, sg, count, i) {
		srp_map_desc(state, ib_sg_dma_address(dev->dev, sg),
			     ib_sg_dma_len(dev->dev, sg),
B
Bart Van Assche 已提交
1687
			     target->global_rkey);
1688
	}
1689

S
Sagi Grimberg 已提交
1690
	return 0;
1691 1692
}

1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708
/*
 * Register the indirect data buffer descriptor with the HCA.
 *
 * Note: since the indirect data buffer descriptor has been allocated with
 * kmalloc() it is guaranteed that this buffer is a physically contiguous
 * memory buffer.
 */
static int srp_map_idb(struct srp_rdma_ch *ch, struct srp_request *req,
		       void **next_mr, void **end_mr, u32 idb_len,
		       __be32 *idb_rkey)
{
	struct srp_target_port *target = ch->target;
	struct srp_device *dev = target->srp_host->srp_dev;
	struct srp_map_state state;
	struct srp_direct_buf idb_desc;
	u64 idb_pages[1];
1709
	struct scatterlist idb_sg[1];
1710 1711 1712 1713 1714 1715 1716 1717 1718
	int ret;

	memset(&state, 0, sizeof(state));
	memset(&idb_desc, 0, sizeof(idb_desc));
	state.gen.next = next_mr;
	state.gen.end = end_mr;
	state.desc = &idb_desc;
	state.base_dma_addr = req->indirect_dma_addr;
	state.dma_len = idb_len;
1719 1720 1721

	if (dev->use_fast_reg) {
		state.sg = idb_sg;
1722
		sg_init_one(idb_sg, req->indirect_desc, idb_len);
1723
		idb_sg->dma_address = req->indirect_dma_addr; /* hack! */
1724 1725 1726
#ifdef CONFIG_NEED_SG_DMA_LENGTH
		idb_sg->dma_length = idb_sg->length;	      /* hack^2 */
#endif
1727
		ret = srp_map_finish_fr(&state, req, ch, 1, NULL);
1728 1729
		if (ret < 0)
			return ret;
1730
		WARN_ON_ONCE(ret < 1);
1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741
	} else if (dev->use_fmr) {
		state.pages = idb_pages;
		state.pages[0] = (req->indirect_dma_addr &
				  dev->mr_page_mask);
		state.npages = 1;
		ret = srp_map_finish_fmr(&state, ch);
		if (ret < 0)
			return ret;
	} else {
		return -EINVAL;
	}
1742 1743 1744

	*idb_rkey = idb_desc.key;

1745
	return 0;
1746 1747
}

1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771
static void srp_check_mapping(struct srp_map_state *state,
			      struct srp_rdma_ch *ch, struct srp_request *req,
			      struct scatterlist *scat, int count)
{
	struct srp_device *dev = ch->target->srp_host->srp_dev;
	struct srp_fr_desc **pfr;
	u64 desc_len = 0, mr_len = 0;
	int i;

	for (i = 0; i < state->ndesc; i++)
		desc_len += be32_to_cpu(req->indirect_desc[i].len);
	if (dev->use_fast_reg)
		for (i = 0, pfr = req->fr_list; i < state->nmdesc; i++, pfr++)
			mr_len += (*pfr)->mr->length;
	else if (dev->use_fmr)
		for (i = 0; i < state->nmdesc; i++)
			mr_len += be32_to_cpu(req->indirect_desc[i].len);
	if (desc_len != scsi_bufflen(req->scmnd) ||
	    mr_len > scsi_bufflen(req->scmnd))
		pr_err("Inconsistent: scsi len %d <> desc len %lld <> mr len %lld; ndesc %d; nmdesc = %d\n",
		       scsi_bufflen(req->scmnd), desc_len, mr_len,
		       state->ndesc, state->nmdesc);
}

1772 1773 1774 1775 1776 1777 1778 1779 1780
/**
 * srp_map_data() - map SCSI data buffer onto an SRP request
 * @scmnd: SCSI command to map
 * @ch: SRP RDMA channel
 * @req: SRP request
 *
 * Returns the length in bytes of the SRP_CMD IU or a negative value if
 * mapping failed.
 */
1781
static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch,
1782 1783
			struct srp_request *req)
{
1784
	struct srp_target_port *target = ch->target;
1785
	struct scatterlist *scat;
1786
	struct srp_cmd *cmd = req->cmd->buf;
1787
	int len, nents, count, ret;
1788 1789
	struct srp_device *dev;
	struct ib_device *ibdev;
1790 1791
	struct srp_map_state state;
	struct srp_indirect_buf *indirect_hdr;
1792 1793
	u32 idb_len, table_len;
	__be32 idb_rkey;
1794
	u8 fmt;
1795

1796
	if (!scsi_sglist(scmnd) || scmnd->sc_data_direction == DMA_NONE)
1797 1798 1799 1800
		return sizeof (struct srp_cmd);

	if (scmnd->sc_data_direction != DMA_FROM_DEVICE &&
	    scmnd->sc_data_direction != DMA_TO_DEVICE) {
1801 1802 1803
		shost_printk(KERN_WARNING, target->scsi_host,
			     PFX "Unhandled data direction %d\n",
			     scmnd->sc_data_direction);
1804 1805 1806
		return -EINVAL;
	}

1807 1808
	nents = scsi_sg_count(scmnd);
	scat  = scsi_sglist(scmnd);
1809

1810
	dev = target->srp_host->srp_dev;
1811 1812 1813
	ibdev = dev->dev;

	count = ib_dma_map_sg(ibdev, scat, nents, scmnd->sc_data_direction);
1814 1815
	if (unlikely(count == 0))
		return -EIO;
1816 1817 1818

	fmt = SRP_DATA_DESC_DIRECT;
	len = sizeof (struct srp_cmd) +	sizeof (struct srp_direct_buf);
1819

B
Bart Van Assche 已提交
1820
	if (count == 1 && target->global_rkey) {
1821 1822 1823 1824 1825 1826
		/*
		 * The midlayer only generated a single gather/scatter
		 * entry, or DMA mapping coalesced everything to a
		 * single entry.  So a direct descriptor along with
		 * the DMA MR suffices.
		 */
1827
		struct srp_direct_buf *buf = (void *) cmd->add_data;
1828

1829
		buf->va  = cpu_to_be64(ib_sg_dma_address(ibdev, scat));
B
Bart Van Assche 已提交
1830
		buf->key = cpu_to_be32(target->global_rkey);
1831
		buf->len = cpu_to_be32(ib_sg_dma_len(ibdev, scat));
1832

1833
		req->nmdesc = 0;
1834 1835 1836
		goto map_complete;
	}

1837 1838 1839
	/*
	 * We have more than one scatter/gather entry, so build our indirect
	 * descriptor table, trying to merge as many entries as we can.
1840 1841 1842
	 */
	indirect_hdr = (void *) cmd->add_data;

1843 1844 1845
	ib_dma_sync_single_for_cpu(ibdev, req->indirect_dma_addr,
				   target->indirect_size, DMA_TO_DEVICE);

1846
	memset(&state, 0, sizeof(state));
B
Bart Van Assche 已提交
1847
	state.desc = req->indirect_desc;
S
Sagi Grimberg 已提交
1848
	if (dev->use_fast_reg)
1849
		ret = srp_map_sg_fr(&state, ch, req, scat, count);
S
Sagi Grimberg 已提交
1850
	else if (dev->use_fmr)
1851
		ret = srp_map_sg_fmr(&state, ch, req, scat, count);
S
Sagi Grimberg 已提交
1852
	else
1853 1854 1855 1856
		ret = srp_map_sg_dma(&state, ch, req, scat, count);
	req->nmdesc = state.nmdesc;
	if (ret < 0)
		goto unmap;
1857

1858 1859 1860
	{
		DEFINE_DYNAMIC_DEBUG_METADATA(ddm,
			"Memory mapping consistency check");
1861
		if (DYNAMIC_DEBUG_BRANCH(ddm))
1862 1863
			srp_check_mapping(&state, ch, req, scat, count);
	}
1864

1865 1866 1867 1868 1869
	/* We've mapped the request, now pull as much of the indirect
	 * descriptor table as we can into the command buffer. If this
	 * target is not using an external indirect table, we are
	 * guaranteed to fit into the command, as the SCSI layer won't
	 * give us more S/G entries than we allow.
1870 1871
	 */
	if (state.ndesc == 1) {
1872 1873
		/*
		 * Memory registration collapsed the sg-list into one entry,
1874 1875 1876
		 * so use a direct descriptor.
		 */
		struct srp_direct_buf *buf = (void *) cmd->add_data;
1877

1878
		*buf = req->indirect_desc[0];
1879
		goto map_complete;
1880 1881
	}

1882 1883 1884 1885
	if (unlikely(target->cmd_sg_cnt < state.ndesc &&
						!target->allow_ext_sg)) {
		shost_printk(KERN_ERR, target->scsi_host,
			     "Could not fit S/G list into SRP_CMD\n");
1886 1887
		ret = -EIO;
		goto unmap;
1888 1889 1890
	}

	count = min(state.ndesc, target->cmd_sg_cnt);
1891
	table_len = state.ndesc * sizeof (struct srp_direct_buf);
1892
	idb_len = sizeof(struct srp_indirect_buf) + table_len;
1893 1894 1895

	fmt = SRP_DATA_DESC_INDIRECT;
	len = sizeof(struct srp_cmd) + sizeof (struct srp_indirect_buf);
1896
	len += count * sizeof (struct srp_direct_buf);
1897

1898 1899
	memcpy(indirect_hdr->desc_list, req->indirect_desc,
	       count * sizeof (struct srp_direct_buf));
1900

B
Bart Van Assche 已提交
1901
	if (!target->global_rkey) {
1902 1903 1904
		ret = srp_map_idb(ch, req, state.gen.next, state.gen.end,
				  idb_len, &idb_rkey);
		if (ret < 0)
1905
			goto unmap;
1906 1907
		req->nmdesc++;
	} else {
B
Bart Van Assche 已提交
1908
		idb_rkey = cpu_to_be32(target->global_rkey);
1909 1910
	}

1911
	indirect_hdr->table_desc.va = cpu_to_be64(req->indirect_dma_addr);
1912
	indirect_hdr->table_desc.key = idb_rkey;
1913 1914 1915 1916
	indirect_hdr->table_desc.len = cpu_to_be32(table_len);
	indirect_hdr->len = cpu_to_be32(state.total_len);

	if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1917
		cmd->data_out_desc_cnt = count;
1918
	else
1919 1920 1921 1922
		cmd->data_in_desc_cnt = count;

	ib_dma_sync_single_for_device(ibdev, req->indirect_dma_addr, table_len,
				      DMA_TO_DEVICE);
1923 1924

map_complete:
1925 1926 1927 1928 1929 1930
	if (scmnd->sc_data_direction == DMA_TO_DEVICE)
		cmd->buf_fmt = fmt << 4;
	else
		cmd->buf_fmt = fmt;

	return len;
1931 1932 1933

unmap:
	srp_unmap_data(scmnd, ch, req);
1934 1935
	if (ret == -ENOMEM && req->nmdesc >= target->mr_pool_size)
		ret = -E2BIG;
1936
	return ret;
1937 1938
}

1939 1940 1941
/*
 * Return an IU and possible credit to the free pool
 */
1942
static void srp_put_tx_iu(struct srp_rdma_ch *ch, struct srp_iu *iu,
1943 1944 1945 1946
			  enum srp_iu_type iu_type)
{
	unsigned long flags;

1947 1948
	spin_lock_irqsave(&ch->lock, flags);
	list_add(&iu->list, &ch->free_tx);
1949
	if (iu_type != SRP_IU_RSP)
1950 1951
		++ch->req_lim;
	spin_unlock_irqrestore(&ch->lock, flags);
1952 1953
}

1954
/*
1955
 * Must be called with ch->lock held to protect req_lim and free_tx.
1956
 * If IU is not sent, it must be returned using srp_put_tx_iu().
1957 1958 1959 1960 1961 1962 1963 1964 1965 1966
 *
 * Note:
 * An upper limit for the number of allocated information units for each
 * request type is:
 * - SRP_IU_CMD: SRP_CMD_SQ_SIZE, since the SCSI mid-layer never queues
 *   more than Scsi_Host.can_queue requests.
 * - SRP_IU_TSK_MGMT: SRP_TSK_MGMT_SQ_SIZE.
 * - SRP_IU_RSP: 1, since a conforming SRP target never sends more than
 *   one unanswered SRP request to an initiator.
 */
1967
static struct srp_iu *__srp_get_tx_iu(struct srp_rdma_ch *ch,
1968 1969
				      enum srp_iu_type iu_type)
{
1970
	struct srp_target_port *target = ch->target;
1971 1972 1973
	s32 rsv = (iu_type == SRP_IU_TSK_MGMT) ? 0 : SRP_TSK_MGMT_SQ_SIZE;
	struct srp_iu *iu;

1974 1975
	lockdep_assert_held(&ch->lock);

C
Christoph Hellwig 已提交
1976
	ib_process_cq_direct(ch->send_cq, -1);
1977

1978
	if (list_empty(&ch->free_tx))
1979 1980 1981
		return NULL;

	/* Initiator responses to target requests do not consume credits */
1982
	if (iu_type != SRP_IU_RSP) {
1983
		if (ch->req_lim <= rsv) {
1984 1985 1986 1987
			++target->zero_req_lim;
			return NULL;
		}

1988
		--ch->req_lim;
1989 1990
	}

1991
	iu = list_first_entry(&ch->free_tx, struct srp_iu, list);
1992
	list_del(&iu->list);
1993 1994 1995
	return iu;
}

1996 1997 1998 1999 2000
/*
 * Note: if this function is called from inside ib_drain_sq() then it will
 * be called without ch->lock being held. If ib_drain_sq() dequeues a WQE
 * with status IB_WC_SUCCESS then that's a bug.
 */
C
Christoph Hellwig 已提交
2001 2002 2003 2004 2005 2006 2007 2008 2009 2010
static void srp_send_done(struct ib_cq *cq, struct ib_wc *wc)
{
	struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe);
	struct srp_rdma_ch *ch = cq->cq_context;

	if (unlikely(wc->status != IB_WC_SUCCESS)) {
		srp_handle_qp_err(cq, wc, "SEND");
		return;
	}

2011 2012
	lockdep_assert_held(&ch->lock);

C
Christoph Hellwig 已提交
2013 2014 2015
	list_add(&iu->list, &ch->free_tx);
}

2016
static int srp_post_send(struct srp_rdma_ch *ch, struct srp_iu *iu, int len)
2017
{
2018
	struct srp_target_port *target = ch->target;
2019 2020 2021 2022 2023
	struct ib_sge list;
	struct ib_send_wr wr, *bad_wr;

	list.addr   = iu->dma;
	list.length = len;
2024
	list.lkey   = target->lkey;
2025

C
Christoph Hellwig 已提交
2026 2027
	iu->cqe.done = srp_send_done;

2028
	wr.next       = NULL;
C
Christoph Hellwig 已提交
2029
	wr.wr_cqe     = &iu->cqe;
2030 2031 2032 2033 2034
	wr.sg_list    = &list;
	wr.num_sge    = 1;
	wr.opcode     = IB_WR_SEND;
	wr.send_flags = IB_SEND_SIGNALED;

2035
	return ib_post_send(ch->qp, &wr, &bad_wr);
2036 2037
}

2038
static int srp_post_recv(struct srp_rdma_ch *ch, struct srp_iu *iu)
2039
{
2040
	struct srp_target_port *target = ch->target;
2041
	struct ib_recv_wr wr, *bad_wr;
2042
	struct ib_sge list;
2043 2044 2045

	list.addr   = iu->dma;
	list.length = iu->size;
2046
	list.lkey   = target->lkey;
2047

C
Christoph Hellwig 已提交
2048 2049
	iu->cqe.done = srp_recv_done;

2050
	wr.next     = NULL;
C
Christoph Hellwig 已提交
2051
	wr.wr_cqe   = &iu->cqe;
2052 2053 2054
	wr.sg_list  = &list;
	wr.num_sge  = 1;

2055
	return ib_post_recv(ch->qp, &wr, &bad_wr);
2056 2057
}

2058
static void srp_process_rsp(struct srp_rdma_ch *ch, struct srp_rsp *rsp)
2059
{
2060
	struct srp_target_port *target = ch->target;
2061 2062 2063 2064 2065
	struct srp_request *req;
	struct scsi_cmnd *scmnd;
	unsigned long flags;

	if (unlikely(rsp->tag & SRP_TAG_TSK_MGMT)) {
2066 2067
		spin_lock_irqsave(&ch->lock, flags);
		ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
2068 2069 2070 2071 2072 2073 2074 2075 2076 2077
		if (rsp->tag == ch->tsk_mgmt_tag) {
			ch->tsk_mgmt_status = -1;
			if (be32_to_cpu(rsp->resp_data_len) >= 4)
				ch->tsk_mgmt_status = rsp->data[3];
			complete(&ch->tsk_mgmt_done);
		} else {
			shost_printk(KERN_ERR, target->scsi_host,
				     "Received tsk mgmt response too late for tag %#llx\n",
				     rsp->tag);
		}
2078
		spin_unlock_irqrestore(&ch->lock, flags);
2079
	} else {
B
Bart Van Assche 已提交
2080
		scmnd = scsi_host_find_tag(target->scsi_host, rsp->tag);
2081
		if (scmnd && scmnd->host_scribble) {
B
Bart Van Assche 已提交
2082 2083
			req = (void *)scmnd->host_scribble;
			scmnd = srp_claim_req(ch, req, NULL, scmnd);
2084 2085
		} else {
			scmnd = NULL;
B
Bart Van Assche 已提交
2086
		}
B
Bart Van Assche 已提交
2087
		if (!scmnd) {
2088
			shost_printk(KERN_ERR, target->scsi_host,
B
Bart Van Assche 已提交
2089 2090
				     "Null scmnd for RSP w/tag %#016llx received on ch %td / QP %#x\n",
				     rsp->tag, ch - target->ch, ch->qp->qp_num);
B
Bart Van Assche 已提交
2091

2092 2093 2094
			spin_lock_irqsave(&ch->lock, flags);
			ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
			spin_unlock_irqrestore(&ch->lock, flags);
B
Bart Van Assche 已提交
2095 2096 2097

			return;
		}
2098 2099 2100 2101 2102 2103 2104 2105 2106
		scmnd->result = rsp->status;

		if (rsp->flags & SRP_RSP_FLAG_SNSVALID) {
			memcpy(scmnd->sense_buffer, rsp->data +
			       be32_to_cpu(rsp->resp_data_len),
			       min_t(int, be32_to_cpu(rsp->sense_data_len),
				     SCSI_SENSE_BUFFERSIZE));
		}

B
Bart Van Assche 已提交
2107
		if (unlikely(rsp->flags & SRP_RSP_FLAG_DIUNDER))
2108
			scsi_set_resid(scmnd, be32_to_cpu(rsp->data_in_res_cnt));
B
Bart Van Assche 已提交
2109 2110 2111 2112 2113 2114
		else if (unlikely(rsp->flags & SRP_RSP_FLAG_DIOVER))
			scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_in_res_cnt));
		else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOUNDER))
			scsi_set_resid(scmnd, be32_to_cpu(rsp->data_out_res_cnt));
		else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOOVER))
			scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_out_res_cnt));
2115

2116
		srp_free_req(ch, req, scmnd,
B
Bart Van Assche 已提交
2117 2118
			     be32_to_cpu(rsp->req_lim_delta));

2119 2120
		scmnd->host_scribble = NULL;
		scmnd->scsi_done(scmnd);
2121 2122 2123
	}
}

2124
static int srp_response_common(struct srp_rdma_ch *ch, s32 req_delta,
2125 2126
			       void *rsp, int len)
{
2127
	struct srp_target_port *target = ch->target;
2128
	struct ib_device *dev = target->srp_host->srp_dev->dev;
2129 2130
	unsigned long flags;
	struct srp_iu *iu;
2131
	int err;
2132

2133 2134 2135 2136
	spin_lock_irqsave(&ch->lock, flags);
	ch->req_lim += req_delta;
	iu = __srp_get_tx_iu(ch, SRP_IU_RSP);
	spin_unlock_irqrestore(&ch->lock, flags);
2137

2138 2139 2140
	if (!iu) {
		shost_printk(KERN_ERR, target->scsi_host, PFX
			     "no IU available to send response\n");
2141
		return 1;
2142 2143 2144 2145 2146 2147
	}

	ib_dma_sync_single_for_cpu(dev, iu->dma, len, DMA_TO_DEVICE);
	memcpy(iu->buf, rsp, len);
	ib_dma_sync_single_for_device(dev, iu->dma, len, DMA_TO_DEVICE);

2148
	err = srp_post_send(ch, iu, len);
2149
	if (err) {
2150 2151
		shost_printk(KERN_ERR, target->scsi_host, PFX
			     "unable to post response: %d\n", err);
2152
		srp_put_tx_iu(ch, iu, SRP_IU_RSP);
2153
	}
2154 2155 2156 2157

	return err;
}

2158
static void srp_process_cred_req(struct srp_rdma_ch *ch,
2159 2160 2161 2162 2163 2164 2165 2166
				 struct srp_cred_req *req)
{
	struct srp_cred_rsp rsp = {
		.opcode = SRP_CRED_RSP,
		.tag = req->tag,
	};
	s32 delta = be32_to_cpu(req->req_lim_delta);

2167 2168
	if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
		shost_printk(KERN_ERR, ch->target->scsi_host, PFX
2169 2170 2171
			     "problems processing SRP_CRED_REQ\n");
}

2172
static void srp_process_aer_req(struct srp_rdma_ch *ch,
2173 2174
				struct srp_aer_req *req)
{
2175
	struct srp_target_port *target = ch->target;
2176 2177 2178 2179 2180 2181 2182
	struct srp_aer_rsp rsp = {
		.opcode = SRP_AER_RSP,
		.tag = req->tag,
	};
	s32 delta = be32_to_cpu(req->req_lim_delta);

	shost_printk(KERN_ERR, target->scsi_host, PFX
B
Bart Van Assche 已提交
2183
		     "ignoring AER for LUN %llu\n", scsilun_to_int(&req->lun));
2184

2185
	if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
2186 2187 2188 2189
		shost_printk(KERN_ERR, target->scsi_host, PFX
			     "problems processing SRP_AER_REQ\n");
}

C
Christoph Hellwig 已提交
2190
static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc)
2191
{
C
Christoph Hellwig 已提交
2192 2193
	struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe);
	struct srp_rdma_ch *ch = cq->cq_context;
2194
	struct srp_target_port *target = ch->target;
2195
	struct ib_device *dev = target->srp_host->srp_dev->dev;
2196
	int res;
2197 2198
	u8 opcode;

C
Christoph Hellwig 已提交
2199 2200 2201 2202 2203
	if (unlikely(wc->status != IB_WC_SUCCESS)) {
		srp_handle_qp_err(cq, wc, "RECV");
		return;
	}

2204
	ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_ti_iu_len,
2205
				   DMA_FROM_DEVICE);
2206 2207 2208 2209

	opcode = *(u8 *) iu->buf;

	if (0) {
2210 2211
		shost_printk(KERN_ERR, target->scsi_host,
			     PFX "recv completion, opcode 0x%02x\n", opcode);
B
Bart Van Assche 已提交
2212 2213
		print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 8, 1,
			       iu->buf, wc->byte_len, true);
2214 2215 2216 2217
	}

	switch (opcode) {
	case SRP_RSP:
2218
		srp_process_rsp(ch, iu->buf);
2219 2220
		break;

2221
	case SRP_CRED_REQ:
2222
		srp_process_cred_req(ch, iu->buf);
2223 2224 2225
		break;

	case SRP_AER_REQ:
2226
		srp_process_aer_req(ch, iu->buf);
2227 2228
		break;

2229 2230
	case SRP_T_LOGOUT:
		/* XXX Handle target logout */
2231 2232
		shost_printk(KERN_WARNING, target->scsi_host,
			     PFX "Got target logout request\n");
2233 2234 2235
		break;

	default:
2236 2237
		shost_printk(KERN_WARNING, target->scsi_host,
			     PFX "Unhandled SRP opcode 0x%02x\n", opcode);
2238 2239 2240
		break;
	}

2241
	ib_dma_sync_single_for_device(dev, iu->dma, ch->max_ti_iu_len,
2242
				      DMA_FROM_DEVICE);
2243

2244
	res = srp_post_recv(ch, iu);
2245 2246 2247
	if (res != 0)
		shost_printk(KERN_ERR, target->scsi_host,
			     PFX "Recv failed with error code %d\n", res);
2248 2249
}

2250 2251
/**
 * srp_tl_err_work() - handle a transport layer error
2252
 * @work: Work structure embedded in an SRP target port.
2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265
 *
 * Note: This function may get invoked before the rport has been created,
 * hence the target->rport test.
 */
static void srp_tl_err_work(struct work_struct *work)
{
	struct srp_target_port *target;

	target = container_of(work, struct srp_target_port, tl_err_work);
	if (target->rport)
		srp_start_tl_fail_timers(target->rport);
}

C
Christoph Hellwig 已提交
2266 2267
static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc,
		const char *opname)
2268
{
C
Christoph Hellwig 已提交
2269
	struct srp_rdma_ch *ch = cq->cq_context;
2270 2271
	struct srp_target_port *target = ch->target;

2272
	if (ch->connected && !target->qp_in_error) {
C
Christoph Hellwig 已提交
2273 2274 2275 2276
		shost_printk(KERN_ERR, target->scsi_host,
			     PFX "failed %s status %s (%d) for CQE %p\n",
			     opname, ib_wc_status_msg(wc->status), wc->status,
			     wc->wr_cqe);
2277
		queue_work(system_long_wq, &target->tl_err_work);
2278
	}
2279 2280 2281
	target->qp_in_error = true;
}

2282
static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
2283
{
2284
	struct srp_target_port *target = host_to_target(shost);
2285
	struct srp_rport *rport = target->rport;
2286
	struct srp_rdma_ch *ch;
2287 2288 2289
	struct srp_request *req;
	struct srp_iu *iu;
	struct srp_cmd *cmd;
2290
	struct ib_device *dev;
2291
	unsigned long flags;
B
Bart Van Assche 已提交
2292 2293
	u32 tag;
	u16 idx;
2294
	int len, ret;
2295 2296 2297 2298 2299 2300 2301 2302 2303 2304
	const bool in_scsi_eh = !in_interrupt() && current == shost->ehandler;

	/*
	 * The SCSI EH thread is the only context from which srp_queuecommand()
	 * can get invoked for blocked devices (SDEV_BLOCK /
	 * SDEV_CREATED_BLOCK). Avoid racing with srp_reconnect_rport() by
	 * locking the rport mutex if invoked from inside the SCSI EH.
	 */
	if (in_scsi_eh)
		mutex_lock(&rport->mutex);
2305

2306 2307 2308
	scmnd->result = srp_chkready(target->rport);
	if (unlikely(scmnd->result))
		goto err;
2309

B
Bart Van Assche 已提交
2310 2311
	WARN_ON_ONCE(scmnd->request->tag < 0);
	tag = blk_mq_unique_tag(scmnd->request);
B
Bart Van Assche 已提交
2312
	ch = &target->ch[blk_mq_unique_tag_to_hwq(tag)];
B
Bart Van Assche 已提交
2313 2314 2315 2316
	idx = blk_mq_unique_tag_to_tag(tag);
	WARN_ONCE(idx >= target->req_ring_size, "%s: tag %#x: idx %d >= %d\n",
		  dev_name(&shost->shost_gendev), tag, idx,
		  target->req_ring_size);
2317 2318 2319 2320

	spin_lock_irqsave(&ch->lock, flags);
	iu = __srp_get_tx_iu(ch, SRP_IU_CMD);
	spin_unlock_irqrestore(&ch->lock, flags);
2321

B
Bart Van Assche 已提交
2322 2323 2324 2325
	if (!iu)
		goto err;

	req = &ch->req_ring[idx];
2326
	dev = target->srp_host->srp_dev->dev;
2327
	ib_dma_sync_single_for_cpu(dev, iu->dma, target->max_iu_len,
2328
				   DMA_TO_DEVICE);
2329

2330
	scmnd->host_scribble = (void *) req;
2331 2332 2333 2334 2335

	cmd = iu->buf;
	memset(cmd, 0, sizeof *cmd);

	cmd->opcode = SRP_CMD;
B
Bart Van Assche 已提交
2336
	int_to_scsilun(scmnd->device->lun, &cmd->lun);
B
Bart Van Assche 已提交
2337
	cmd->tag    = tag;
2338 2339 2340 2341 2342
	memcpy(cmd->cdb, scmnd->cmnd, scmnd->cmd_len);

	req->scmnd    = scmnd;
	req->cmd      = iu;

2343
	len = srp_map_data(scmnd, ch, req);
2344
	if (len < 0) {
2345
		shost_printk(KERN_ERR, target->scsi_host,
2346 2347 2348 2349
			     PFX "Failed to map data (%d)\n", len);
		/*
		 * If we ran out of memory descriptors (-ENOMEM) because an
		 * application is queuing many requests with more than
2350
		 * max_pages_per_mr sg-list elements, tell the SCSI mid-layer
2351 2352 2353 2354
		 * to reduce queue depth temporarily.
		 */
		scmnd->result = len == -ENOMEM ?
			DID_OK << 16 | QUEUE_FULL << 1 : DID_ERROR << 16;
2355
		goto err_iu;
2356 2357
	}

2358
	ib_dma_sync_single_for_device(dev, iu->dma, target->max_iu_len,
2359
				      DMA_TO_DEVICE);
2360

2361
	if (srp_post_send(ch, iu, len)) {
2362
		shost_printk(KERN_ERR, target->scsi_host, PFX "Send failed\n");
2363 2364 2365
		goto err_unmap;
	}

2366 2367
	ret = 0;

2368 2369 2370 2371
unlock_rport:
	if (in_scsi_eh)
		mutex_unlock(&rport->mutex);

2372
	return ret;
2373 2374

err_unmap:
2375
	srp_unmap_data(scmnd, ch, req);
2376

2377
err_iu:
2378
	srp_put_tx_iu(ch, iu, SRP_IU_CMD);
2379

2380 2381 2382 2383 2384 2385
	/*
	 * Avoid that the loops that iterate over the request ring can
	 * encounter a dangling SCSI command pointer.
	 */
	req->scmnd = NULL;

2386 2387 2388 2389 2390 2391 2392
err:
	if (scmnd->result) {
		scmnd->scsi_done(scmnd);
		ret = 0;
	} else {
		ret = SCSI_MLQUEUE_HOST_BUSY;
	}
2393

2394
	goto unlock_rport;
2395 2396
}

2397 2398
/*
 * Note: the resources allocated in this function are freed in
2399
 * srp_free_ch_ib().
2400
 */
2401
static int srp_alloc_iu_bufs(struct srp_rdma_ch *ch)
2402
{
2403
	struct srp_target_port *target = ch->target;
2404 2405
	int i;

2406 2407 2408
	ch->rx_ring = kcalloc(target->queue_size, sizeof(*ch->rx_ring),
			      GFP_KERNEL);
	if (!ch->rx_ring)
2409
		goto err_no_ring;
2410 2411 2412
	ch->tx_ring = kcalloc(target->queue_size, sizeof(*ch->tx_ring),
			      GFP_KERNEL);
	if (!ch->tx_ring)
2413 2414 2415
		goto err_no_ring;

	for (i = 0; i < target->queue_size; ++i) {
2416 2417 2418 2419
		ch->rx_ring[i] = srp_alloc_iu(target->srp_host,
					      ch->max_ti_iu_len,
					      GFP_KERNEL, DMA_FROM_DEVICE);
		if (!ch->rx_ring[i])
2420 2421 2422
			goto err;
	}

2423
	for (i = 0; i < target->queue_size; ++i) {
2424 2425 2426 2427
		ch->tx_ring[i] = srp_alloc_iu(target->srp_host,
					      target->max_iu_len,
					      GFP_KERNEL, DMA_TO_DEVICE);
		if (!ch->tx_ring[i])
2428
			goto err;
2429

2430
		list_add(&ch->tx_ring[i]->list, &ch->free_tx);
2431 2432 2433 2434 2435
	}

	return 0;

err:
2436
	for (i = 0; i < target->queue_size; ++i) {
2437 2438
		srp_free_iu(target->srp_host, ch->rx_ring[i]);
		srp_free_iu(target->srp_host, ch->tx_ring[i]);
2439 2440
	}

2441 2442

err_no_ring:
2443 2444 2445 2446
	kfree(ch->tx_ring);
	ch->tx_ring = NULL;
	kfree(ch->rx_ring);
	ch->rx_ring = NULL;
2447

2448 2449 2450
	return -ENOMEM;
}

2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477
static uint32_t srp_compute_rq_tmo(struct ib_qp_attr *qp_attr, int attr_mask)
{
	uint64_t T_tr_ns, max_compl_time_ms;
	uint32_t rq_tmo_jiffies;

	/*
	 * According to section 11.2.4.2 in the IBTA spec (Modify Queue Pair,
	 * table 91), both the QP timeout and the retry count have to be set
	 * for RC QP's during the RTR to RTS transition.
	 */
	WARN_ON_ONCE((attr_mask & (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)) !=
		     (IB_QP_TIMEOUT | IB_QP_RETRY_CNT));

	/*
	 * Set target->rq_tmo_jiffies to one second more than the largest time
	 * it can take before an error completion is generated. See also
	 * C9-140..142 in the IBTA spec for more information about how to
	 * convert the QP Local ACK Timeout value to nanoseconds.
	 */
	T_tr_ns = 4096 * (1ULL << qp_attr->timeout);
	max_compl_time_ms = qp_attr->retry_cnt * 4 * T_tr_ns;
	do_div(max_compl_time_ms, NSEC_PER_MSEC);
	rq_tmo_jiffies = msecs_to_jiffies(max_compl_time_ms + 1000);

	return rq_tmo_jiffies;
}

2478
static void srp_cm_rep_handler(struct ib_cm_id *cm_id,
2479
			       const struct srp_login_rsp *lrsp,
2480
			       struct srp_rdma_ch *ch)
2481
{
2482
	struct srp_target_port *target = ch->target;
2483 2484
	struct ib_qp_attr *qp_attr = NULL;
	int attr_mask = 0;
B
Bart Van Assche 已提交
2485
	int ret = 0;
2486 2487 2488
	int i;

	if (lrsp->opcode == SRP_LOGIN_RSP) {
2489 2490
		ch->max_ti_iu_len = be32_to_cpu(lrsp->max_ti_iu_len);
		ch->req_lim       = be32_to_cpu(lrsp->req_lim_delta);
2491 2492 2493 2494 2495 2496

		/*
		 * Reserve credits for task management so we don't
		 * bounce requests back to the SCSI mid-layer.
		 */
		target->scsi_host->can_queue
2497
			= min(ch->req_lim - SRP_TSK_MGMT_SQ_SIZE,
2498
			      target->scsi_host->can_queue);
2499 2500 2501
		target->scsi_host->cmd_per_lun
			= min_t(int, target->scsi_host->can_queue,
				target->scsi_host->cmd_per_lun);
2502 2503 2504 2505 2506 2507 2508
	} else {
		shost_printk(KERN_WARNING, target->scsi_host,
			     PFX "Unhandled RSP opcode %#x\n", lrsp->opcode);
		ret = -ECONNRESET;
		goto error;
	}

2509 2510
	if (!ch->rx_ring) {
		ret = srp_alloc_iu_bufs(ch);
2511 2512 2513 2514
		if (ret)
			goto error;
	}

2515
	for (i = 0; i < target->queue_size; i++) {
2516 2517 2518
		struct srp_iu *iu = ch->rx_ring[i];

		ret = srp_post_recv(ch, iu);
2519
		if (ret)
B
Bart Van Assche 已提交
2520
			goto error;
2521 2522
	}

B
Bart Van Assche 已提交
2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536
	if (!target->using_rdma_cm) {
		ret = -ENOMEM;
		qp_attr = kmalloc(sizeof(*qp_attr), GFP_KERNEL);
		if (!qp_attr)
			goto error;

		qp_attr->qp_state = IB_QPS_RTR;
		ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
		if (ret)
			goto error_free;

		ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
		if (ret)
			goto error_free;
2537

B
Bart Van Assche 已提交
2538 2539 2540 2541
		qp_attr->qp_state = IB_QPS_RTS;
		ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
		if (ret)
			goto error_free;
2542

B
Bart Van Assche 已提交
2543
		target->rq_tmo_jiffies = srp_compute_rq_tmo(qp_attr, attr_mask);
2544

B
Bart Van Assche 已提交
2545 2546 2547 2548 2549 2550
		ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
		if (ret)
			goto error_free;

		ret = ib_send_cm_rtu(cm_id, NULL, 0);
	}
2551 2552 2553 2554 2555

error_free:
	kfree(qp_attr);

error:
2556
	ch->status = ret;
2557 2558
}

B
Bart Van Assche 已提交
2559 2560 2561
static void srp_ib_cm_rej_handler(struct ib_cm_id *cm_id,
				  struct ib_cm_event *event,
				  struct srp_rdma_ch *ch)
2562
{
2563
	struct srp_target_port *target = ch->target;
2564
	struct Scsi_Host *shost = target->scsi_host;
2565 2566
	struct ib_class_port_info *cpi;
	int opcode;
B
Bart Van Assche 已提交
2567
	u16 dlid;
2568 2569 2570 2571

	switch (event->param.rej_rcvd.reason) {
	case IB_CM_REJ_PORT_CM_REDIRECT:
		cpi = event->param.rej_rcvd.ari;
B
Bart Van Assche 已提交
2572 2573 2574
		dlid = be16_to_cpu(cpi->redirect_lid);
		sa_path_set_dlid(&ch->ib_cm.path, dlid);
		ch->ib_cm.path.pkey = cpi->redirect_pkey;
2575
		cm_id->remote_cm_qpn = be32_to_cpu(cpi->redirect_qp) & 0x00ffffff;
B
Bart Van Assche 已提交
2576
		memcpy(ch->ib_cm.path.dgid.raw, cpi->redirect_gid, 16);
2577

B
Bart Van Assche 已提交
2578
		ch->status = dlid ? SRP_DLID_REDIRECT : SRP_PORT_REDIRECT;
2579 2580 2581
		break;

	case IB_CM_REJ_PORT_REDIRECT:
2582
		if (srp_target_is_topspin(target)) {
B
Bart Van Assche 已提交
2583 2584
			union ib_gid *dgid = &ch->ib_cm.path.dgid;

2585 2586 2587 2588 2589
			/*
			 * Topspin/Cisco SRP gateways incorrectly send
			 * reject reason code 25 when they mean 24
			 * (port redirect).
			 */
B
Bart Van Assche 已提交
2590
			memcpy(dgid->raw, event->param.rej_rcvd.ari, 16);
2591

2592 2593
			shost_printk(KERN_DEBUG, shost,
				     PFX "Topspin/Cisco redirect to target port GID %016llx%016llx\n",
B
Bart Van Assche 已提交
2594 2595
				     be64_to_cpu(dgid->global.subnet_prefix),
				     be64_to_cpu(dgid->global.interface_id));
2596

2597
			ch->status = SRP_PORT_REDIRECT;
2598
		} else {
2599 2600
			shost_printk(KERN_WARNING, shost,
				     "  REJ reason: IB_CM_REJ_PORT_REDIRECT\n");
2601
			ch->status = -ECONNRESET;
2602 2603 2604 2605
		}
		break;

	case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID:
2606 2607
		shost_printk(KERN_WARNING, shost,
			    "  REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n");
2608
		ch->status = -ECONNRESET;
2609 2610 2611 2612 2613 2614 2615 2616 2617
		break;

	case IB_CM_REJ_CONSUMER_DEFINED:
		opcode = *(u8 *) event->private_data;
		if (opcode == SRP_LOGIN_REJ) {
			struct srp_login_rej *rej = event->private_data;
			u32 reason = be32_to_cpu(rej->reason);

			if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE)
2618 2619
				shost_printk(KERN_WARNING, shost,
					     PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n");
2620
			else
B
Bart Van Assche 已提交
2621 2622
				shost_printk(KERN_WARNING, shost, PFX
					     "SRP LOGIN from %pI6 to %pI6 REJECTED, reason 0x%08x\n",
2623
					     target->sgid.raw,
B
Bart Van Assche 已提交
2624 2625
					     target->ib_cm.orig_dgid.raw,
					     reason);
2626
		} else
2627 2628 2629
			shost_printk(KERN_WARNING, shost,
				     "  REJ reason: IB_CM_REJ_CONSUMER_DEFINED,"
				     " opcode 0x%02x\n", opcode);
2630
		ch->status = -ECONNRESET;
2631 2632
		break;

D
David Dillow 已提交
2633 2634
	case IB_CM_REJ_STALE_CONN:
		shost_printk(KERN_WARNING, shost, "  REJ reason: stale connection\n");
2635
		ch->status = SRP_STALE_CONN;
D
David Dillow 已提交
2636 2637
		break;

2638
	default:
2639 2640
		shost_printk(KERN_WARNING, shost, "  REJ reason 0x%x\n",
			     event->param.rej_rcvd.reason);
2641
		ch->status = -ECONNRESET;
2642 2643 2644
	}
}

B
Bart Van Assche 已提交
2645
static int srp_ib_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
2646
{
2647 2648
	struct srp_rdma_ch *ch = cm_id->context;
	struct srp_target_port *target = ch->target;
2649 2650 2651 2652
	int comp = 0;

	switch (event->event) {
	case IB_CM_REQ_ERROR:
2653 2654
		shost_printk(KERN_DEBUG, target->scsi_host,
			     PFX "Sending CM REQ failed\n");
2655
		comp = 1;
2656
		ch->status = -ECONNRESET;
2657 2658 2659 2660
		break;

	case IB_CM_REP_RECEIVED:
		comp = 1;
2661
		srp_cm_rep_handler(cm_id, event->private_data, ch);
2662 2663 2664
		break;

	case IB_CM_REJ_RECEIVED:
2665
		shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n");
2666 2667
		comp = 1;

B
Bart Van Assche 已提交
2668
		srp_ib_cm_rej_handler(cm_id, event, ch);
2669 2670
		break;

2671
	case IB_CM_DREQ_RECEIVED:
2672 2673
		shost_printk(KERN_WARNING, target->scsi_host,
			     PFX "DREQ received - connection closed\n");
2674
		ch->connected = false;
2675
		if (ib_send_cm_drep(cm_id, NULL, 0))
2676 2677
			shost_printk(KERN_ERR, target->scsi_host,
				     PFX "Sending CM DREP failed\n");
2678
		queue_work(system_long_wq, &target->tl_err_work);
2679 2680 2681
		break;

	case IB_CM_TIMEWAIT_EXIT:
2682 2683
		shost_printk(KERN_ERR, target->scsi_host,
			     PFX "connection closed\n");
2684
		comp = 1;
2685

2686
		ch->status = 0;
2687 2688
		break;

2689 2690 2691 2692 2693
	case IB_CM_MRA_RECEIVED:
	case IB_CM_DREQ_ERROR:
	case IB_CM_DREP_RECEIVED:
		break;

2694
	default:
2695 2696
		shost_printk(KERN_WARNING, target->scsi_host,
			     PFX "Unhandled CM event %d\n", event->event);
2697 2698 2699 2700
		break;
	}

	if (comp)
2701
		complete(&ch->done);
2702 2703 2704 2705

	return 0;
}

B
Bart Van Assche 已提交
2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834
static void srp_rdma_cm_rej_handler(struct srp_rdma_ch *ch,
				    struct rdma_cm_event *event)
{
	struct srp_target_port *target = ch->target;
	struct Scsi_Host *shost = target->scsi_host;
	int opcode;

	switch (event->status) {
	case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID:
		shost_printk(KERN_WARNING, shost,
			    "  REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n");
		ch->status = -ECONNRESET;
		break;

	case IB_CM_REJ_CONSUMER_DEFINED:
		opcode = *(u8 *) event->param.conn.private_data;
		if (opcode == SRP_LOGIN_REJ) {
			struct srp_login_rej *rej =
				(struct srp_login_rej *)
				event->param.conn.private_data;
			u32 reason = be32_to_cpu(rej->reason);

			if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE)
				shost_printk(KERN_WARNING, shost,
					     PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n");
			else
				shost_printk(KERN_WARNING, shost,
					    PFX "SRP LOGIN REJECTED, reason 0x%08x\n", reason);
		} else {
			shost_printk(KERN_WARNING, shost,
				     "  REJ reason: IB_CM_REJ_CONSUMER_DEFINED, opcode 0x%02x\n",
				     opcode);
		}
		ch->status = -ECONNRESET;
		break;

	case IB_CM_REJ_STALE_CONN:
		shost_printk(KERN_WARNING, shost,
			     "  REJ reason: stale connection\n");
		ch->status = SRP_STALE_CONN;
		break;

	default:
		shost_printk(KERN_WARNING, shost, "  REJ reason 0x%x\n",
			     event->status);
		ch->status = -ECONNRESET;
		break;
	}
}

static int srp_rdma_cm_handler(struct rdma_cm_id *cm_id,
			       struct rdma_cm_event *event)
{
	struct srp_rdma_ch *ch = cm_id->context;
	struct srp_target_port *target = ch->target;
	int comp = 0;

	switch (event->event) {
	case RDMA_CM_EVENT_ADDR_RESOLVED:
		ch->status = 0;
		comp = 1;
		break;

	case RDMA_CM_EVENT_ADDR_ERROR:
		ch->status = -ENXIO;
		comp = 1;
		break;

	case RDMA_CM_EVENT_ROUTE_RESOLVED:
		ch->status = 0;
		comp = 1;
		break;

	case RDMA_CM_EVENT_ROUTE_ERROR:
	case RDMA_CM_EVENT_UNREACHABLE:
		ch->status = -EHOSTUNREACH;
		comp = 1;
		break;

	case RDMA_CM_EVENT_CONNECT_ERROR:
		shost_printk(KERN_DEBUG, target->scsi_host,
			     PFX "Sending CM REQ failed\n");
		comp = 1;
		ch->status = -ECONNRESET;
		break;

	case RDMA_CM_EVENT_ESTABLISHED:
		comp = 1;
		srp_cm_rep_handler(NULL, event->param.conn.private_data, ch);
		break;

	case RDMA_CM_EVENT_REJECTED:
		shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n");
		comp = 1;

		srp_rdma_cm_rej_handler(ch, event);
		break;

	case RDMA_CM_EVENT_DISCONNECTED:
		if (ch->connected) {
			shost_printk(KERN_WARNING, target->scsi_host,
				     PFX "received DREQ\n");
			rdma_disconnect(ch->rdma_cm.cm_id);
			comp = 1;
			ch->status = 0;
			queue_work(system_long_wq, &target->tl_err_work);
		}
		break;

	case RDMA_CM_EVENT_TIMEWAIT_EXIT:
		shost_printk(KERN_ERR, target->scsi_host,
			     PFX "connection closed\n");

		comp = 1;
		ch->status = 0;
		break;

	default:
		shost_printk(KERN_WARNING, target->scsi_host,
			     PFX "Unhandled CM event %d\n", event->event);
		break;
	}

	if (comp)
		complete(&ch->done);

	return 0;
}

2835 2836 2837 2838 2839 2840 2841 2842
/**
 * srp_change_queue_depth - setting device queue depth
 * @sdev: scsi device struct
 * @qdepth: requested queue depth
 *
 * Returns queue depth.
 */
static int
2843
srp_change_queue_depth(struct scsi_device *sdev, int qdepth)
2844
{
2845
	if (!sdev->tagged_supported)
2846
		qdepth = 1;
2847
	return scsi_change_queue_depth(sdev, qdepth);
2848 2849
}

B
Bart Van Assche 已提交
2850
static int srp_send_tsk_mgmt(struct srp_rdma_ch *ch, u64 req_tag, u64 lun,
2851
			     u8 func, u8 *status)
2852
{
2853
	struct srp_target_port *target = ch->target;
2854
	struct srp_rport *rport = target->rport;
2855
	struct ib_device *dev = target->srp_host->srp_dev->dev;
2856 2857
	struct srp_iu *iu;
	struct srp_tsk_mgmt *tsk_mgmt;
2858
	int res;
2859

2860
	if (!ch->connected || target->qp_in_error)
2861 2862
		return -1;

2863
	/*
2864
	 * Lock the rport mutex to avoid that srp_create_ch_ib() is
2865 2866 2867
	 * invoked while a task management function is being sent.
	 */
	mutex_lock(&rport->mutex);
2868 2869 2870
	spin_lock_irq(&ch->lock);
	iu = __srp_get_tx_iu(ch, SRP_IU_TSK_MGMT);
	spin_unlock_irq(&ch->lock);
2871

2872 2873 2874
	if (!iu) {
		mutex_unlock(&rport->mutex);

2875
		return -1;
2876
	}
2877

2878 2879
	ib_dma_sync_single_for_cpu(dev, iu->dma, sizeof *tsk_mgmt,
				   DMA_TO_DEVICE);
2880 2881 2882 2883
	tsk_mgmt = iu->buf;
	memset(tsk_mgmt, 0, sizeof *tsk_mgmt);

	tsk_mgmt->opcode 	= SRP_TSK_MGMT;
B
Bart Van Assche 已提交
2884
	int_to_scsilun(lun, &tsk_mgmt->lun);
2885
	tsk_mgmt->tsk_mgmt_func = func;
2886
	tsk_mgmt->task_tag	= req_tag;
2887

2888 2889 2890 2891 2892 2893 2894
	spin_lock_irq(&ch->lock);
	ch->tsk_mgmt_tag = (ch->tsk_mgmt_tag + 1) | SRP_TAG_TSK_MGMT;
	tsk_mgmt->tag = ch->tsk_mgmt_tag;
	spin_unlock_irq(&ch->lock);

	init_completion(&ch->tsk_mgmt_done);

2895 2896
	ib_dma_sync_single_for_device(dev, iu->dma, sizeof *tsk_mgmt,
				      DMA_TO_DEVICE);
2897 2898
	if (srp_post_send(ch, iu, sizeof(*tsk_mgmt))) {
		srp_put_tx_iu(ch, iu, SRP_IU_TSK_MGMT);
2899 2900
		mutex_unlock(&rport->mutex);

2901 2902
		return -1;
	}
2903 2904 2905 2906
	res = wait_for_completion_timeout(&ch->tsk_mgmt_done,
					msecs_to_jiffies(SRP_ABORT_TIMEOUT_MS));
	if (res > 0 && status)
		*status = ch->tsk_mgmt_status;
2907
	mutex_unlock(&rport->mutex);
2908

2909
	WARN_ON_ONCE(res < 0);
2910

2911
	return res > 0 ? 0 : -1;
2912 2913
}

2914 2915
static int srp_abort(struct scsi_cmnd *scmnd)
{
2916
	struct srp_target_port *target = host_to_target(scmnd->device->host);
2917
	struct srp_request *req = (struct srp_request *) scmnd->host_scribble;
B
Bart Van Assche 已提交
2918
	u32 tag;
B
Bart Van Assche 已提交
2919
	u16 ch_idx;
2920
	struct srp_rdma_ch *ch;
2921
	int ret;
2922

2923
	shost_printk(KERN_ERR, target->scsi_host, "SRP abort called\n");
2924

B
Bart Van Assche 已提交
2925
	if (!req)
2926
		return SUCCESS;
B
Bart Van Assche 已提交
2927
	tag = blk_mq_unique_tag(scmnd->request);
B
Bart Van Assche 已提交
2928 2929 2930 2931 2932 2933 2934 2935
	ch_idx = blk_mq_unique_tag_to_hwq(tag);
	if (WARN_ON_ONCE(ch_idx >= target->ch_count))
		return SUCCESS;
	ch = &target->ch[ch_idx];
	if (!srp_claim_req(ch, req, NULL, scmnd))
		return SUCCESS;
	shost_printk(KERN_ERR, target->scsi_host,
		     "Sending SRP abort for tag %#x\n", tag);
B
Bart Van Assche 已提交
2936
	if (srp_send_tsk_mgmt(ch, tag, scmnd->device->lun,
2937
			      SRP_TSK_ABORT_TASK, NULL) == 0)
2938
		ret = SUCCESS;
2939
	else if (target->rport->state == SRP_RPORT_LOST)
2940
		ret = FAST_IO_FAIL;
2941 2942
	else
		ret = FAILED;
B
Bart Van Assche 已提交
2943 2944 2945 2946 2947
	if (ret == SUCCESS) {
		srp_free_req(ch, req, scmnd, 0);
		scmnd->result = DID_ABORT << 16;
		scmnd->scsi_done(scmnd);
	}
2948

2949
	return ret;
2950 2951 2952 2953
}

static int srp_reset_device(struct scsi_cmnd *scmnd)
{
2954
	struct srp_target_port *target = host_to_target(scmnd->device->host);
B
Bart Van Assche 已提交
2955
	struct srp_rdma_ch *ch;
2956
	int i;
2957
	u8 status;
2958

2959
	shost_printk(KERN_ERR, target->scsi_host, "SRP reset_device called\n");
2960

B
Bart Van Assche 已提交
2961
	ch = &target->ch[0];
2962
	if (srp_send_tsk_mgmt(ch, SRP_TAG_NO_REQ, scmnd->device->lun,
2963
			      SRP_TSK_LUN_RESET, &status))
2964
		return FAILED;
2965
	if (status)
2966 2967
		return FAILED;

B
Bart Van Assche 已提交
2968 2969 2970 2971
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
		for (i = 0; i < target->req_ring_size; ++i) {
			struct srp_request *req = &ch->req_ring[i];
2972

B
Bart Van Assche 已提交
2973 2974
			srp_finish_req(ch, req, scmnd->device, DID_RESET << 16);
		}
2975
	}
2976 2977

	return SUCCESS;
2978 2979 2980 2981 2982 2983
}

static int srp_reset_host(struct scsi_cmnd *scmnd)
{
	struct srp_target_port *target = host_to_target(scmnd->device->host);

2984
	shost_printk(KERN_ERR, target->scsi_host, PFX "SRP reset_host called\n");
2985

2986
	return srp_reconnect_rport(target->rport) == 0 ? SUCCESS : FAILED;
2987 2988
}

2989 2990 2991 2992 2993 2994 2995 2996 2997 2998
static int srp_target_alloc(struct scsi_target *starget)
{
	struct Scsi_Host *shost = dev_to_shost(starget->dev.parent);
	struct srp_target_port *target = host_to_target(shost);

	if (target->target_can_queue)
		starget->can_queue = target->target_can_queue;
	return 0;
}

2999 3000 3001 3002 3003
static int srp_slave_alloc(struct scsi_device *sdev)
{
	struct Scsi_Host *shost = sdev->host;
	struct srp_target_port *target = host_to_target(shost);
	struct srp_device *srp_dev = target->srp_host->srp_dev;
3004
	struct ib_device *ibdev = srp_dev->dev;
3005

3006
	if (!(ibdev->attrs.device_cap_flags & IB_DEVICE_SG_GAPS_REG))
3007 3008 3009 3010 3011 3012
		blk_queue_virt_boundary(sdev->request_queue,
					~srp_dev->mr_page_mask);

	return 0;
}

3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027
static int srp_slave_configure(struct scsi_device *sdev)
{
	struct Scsi_Host *shost = sdev->host;
	struct srp_target_port *target = host_to_target(shost);
	struct request_queue *q = sdev->request_queue;
	unsigned long timeout;

	if (sdev->type == TYPE_DISK) {
		timeout = max_t(unsigned, 30 * HZ, target->rq_tmo_jiffies);
		blk_queue_rq_timeout(q, timeout);
	}

	return 0;
}

3028 3029
static ssize_t show_id_ext(struct device *dev, struct device_attribute *attr,
			   char *buf)
3030
{
3031
	struct srp_target_port *target = host_to_target(class_to_shost(dev));
3032

3033
	return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->id_ext));
3034 3035
}

3036 3037
static ssize_t show_ioc_guid(struct device *dev, struct device_attribute *attr,
			     char *buf)
3038
{
3039
	struct srp_target_port *target = host_to_target(class_to_shost(dev));
3040

3041
	return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->ioc_guid));
3042 3043
}

3044 3045
static ssize_t show_service_id(struct device *dev,
			       struct device_attribute *attr, char *buf)
3046
{
3047
	struct srp_target_port *target = host_to_target(class_to_shost(dev));
3048

B
Bart Van Assche 已提交
3049 3050 3051 3052
	if (target->using_rdma_cm)
		return -ENOENT;
	return sprintf(buf, "0x%016llx\n",
		       be64_to_cpu(target->ib_cm.service_id));
3053 3054
}

3055 3056
static ssize_t show_pkey(struct device *dev, struct device_attribute *attr,
			 char *buf)
3057
{
3058
	struct srp_target_port *target = host_to_target(class_to_shost(dev));
3059

B
Bart Van Assche 已提交
3060 3061 3062
	if (target->using_rdma_cm)
		return -ENOENT;
	return sprintf(buf, "0x%04x\n", be16_to_cpu(target->ib_cm.pkey));
3063 3064
}

B
Bart Van Assche 已提交
3065 3066 3067 3068 3069
static ssize_t show_sgid(struct device *dev, struct device_attribute *attr,
			 char *buf)
{
	struct srp_target_port *target = host_to_target(class_to_shost(dev));

3070
	return sprintf(buf, "%pI6\n", target->sgid.raw);
B
Bart Van Assche 已提交
3071 3072
}

3073 3074
static ssize_t show_dgid(struct device *dev, struct device_attribute *attr,
			 char *buf)
3075
{
3076
	struct srp_target_port *target = host_to_target(class_to_shost(dev));
B
Bart Van Assche 已提交
3077
	struct srp_rdma_ch *ch = &target->ch[0];
3078

B
Bart Van Assche 已提交
3079 3080 3081
	if (target->using_rdma_cm)
		return -ENOENT;
	return sprintf(buf, "%pI6\n", ch->ib_cm.path.dgid.raw);
3082 3083
}

3084 3085
static ssize_t show_orig_dgid(struct device *dev,
			      struct device_attribute *attr, char *buf)
3086
{
3087
	struct srp_target_port *target = host_to_target(class_to_shost(dev));
3088

B
Bart Van Assche 已提交
3089 3090 3091
	if (target->using_rdma_cm)
		return -ENOENT;
	return sprintf(buf, "%pI6\n", target->ib_cm.orig_dgid.raw);
3092 3093
}

3094 3095 3096 3097
static ssize_t show_req_lim(struct device *dev,
			    struct device_attribute *attr, char *buf)
{
	struct srp_target_port *target = host_to_target(class_to_shost(dev));
B
Bart Van Assche 已提交
3098 3099
	struct srp_rdma_ch *ch;
	int i, req_lim = INT_MAX;
3100

B
Bart Van Assche 已提交
3101 3102 3103 3104 3105
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
		req_lim = min(req_lim, ch->req_lim);
	}
	return sprintf(buf, "%d\n", req_lim);
3106 3107
}

3108 3109
static ssize_t show_zero_req_lim(struct device *dev,
				 struct device_attribute *attr, char *buf)
3110
{
3111
	struct srp_target_port *target = host_to_target(class_to_shost(dev));
3112 3113 3114 3115

	return sprintf(buf, "%d\n", target->zero_req_lim);
}

3116 3117
static ssize_t show_local_ib_port(struct device *dev,
				  struct device_attribute *attr, char *buf)
3118
{
3119
	struct srp_target_port *target = host_to_target(class_to_shost(dev));
3120 3121 3122 3123

	return sprintf(buf, "%d\n", target->srp_host->port);
}

3124 3125
static ssize_t show_local_ib_device(struct device *dev,
				    struct device_attribute *attr, char *buf)
3126
{
3127
	struct srp_target_port *target = host_to_target(class_to_shost(dev));
3128

3129
	return sprintf(buf, "%s\n", target->srp_host->srp_dev->dev->name);
3130 3131
}

B
Bart Van Assche 已提交
3132 3133 3134 3135 3136 3137 3138 3139
static ssize_t show_ch_count(struct device *dev, struct device_attribute *attr,
			     char *buf)
{
	struct srp_target_port *target = host_to_target(class_to_shost(dev));

	return sprintf(buf, "%d\n", target->ch_count);
}

3140 3141 3142 3143 3144 3145 3146 3147
static ssize_t show_comp_vector(struct device *dev,
				struct device_attribute *attr, char *buf)
{
	struct srp_target_port *target = host_to_target(class_to_shost(dev));

	return sprintf(buf, "%d\n", target->comp_vector);
}

3148 3149 3150 3151 3152 3153 3154 3155
static ssize_t show_tl_retry_count(struct device *dev,
				   struct device_attribute *attr, char *buf)
{
	struct srp_target_port *target = host_to_target(class_to_shost(dev));

	return sprintf(buf, "%d\n", target->tl_retry_count);
}

3156 3157 3158 3159 3160 3161 3162 3163
static ssize_t show_cmd_sg_entries(struct device *dev,
				   struct device_attribute *attr, char *buf)
{
	struct srp_target_port *target = host_to_target(class_to_shost(dev));

	return sprintf(buf, "%u\n", target->cmd_sg_cnt);
}

3164 3165 3166 3167 3168 3169 3170 3171
static ssize_t show_allow_ext_sg(struct device *dev,
				 struct device_attribute *attr, char *buf)
{
	struct srp_target_port *target = host_to_target(class_to_shost(dev));

	return sprintf(buf, "%s\n", target->allow_ext_sg ? "true" : "false");
}

3172 3173 3174 3175
static DEVICE_ATTR(id_ext,	    S_IRUGO, show_id_ext,	   NULL);
static DEVICE_ATTR(ioc_guid,	    S_IRUGO, show_ioc_guid,	   NULL);
static DEVICE_ATTR(service_id,	    S_IRUGO, show_service_id,	   NULL);
static DEVICE_ATTR(pkey,	    S_IRUGO, show_pkey,		   NULL);
B
Bart Van Assche 已提交
3176
static DEVICE_ATTR(sgid,	    S_IRUGO, show_sgid,		   NULL);
3177 3178
static DEVICE_ATTR(dgid,	    S_IRUGO, show_dgid,		   NULL);
static DEVICE_ATTR(orig_dgid,	    S_IRUGO, show_orig_dgid,	   NULL);
3179
static DEVICE_ATTR(req_lim,         S_IRUGO, show_req_lim,         NULL);
3180 3181 3182
static DEVICE_ATTR(zero_req_lim,    S_IRUGO, show_zero_req_lim,	   NULL);
static DEVICE_ATTR(local_ib_port,   S_IRUGO, show_local_ib_port,   NULL);
static DEVICE_ATTR(local_ib_device, S_IRUGO, show_local_ib_device, NULL);
B
Bart Van Assche 已提交
3183
static DEVICE_ATTR(ch_count,        S_IRUGO, show_ch_count,        NULL);
3184
static DEVICE_ATTR(comp_vector,     S_IRUGO, show_comp_vector,     NULL);
3185
static DEVICE_ATTR(tl_retry_count,  S_IRUGO, show_tl_retry_count,  NULL);
3186
static DEVICE_ATTR(cmd_sg_entries,  S_IRUGO, show_cmd_sg_entries,  NULL);
3187
static DEVICE_ATTR(allow_ext_sg,    S_IRUGO, show_allow_ext_sg,    NULL);
3188 3189 3190 3191 3192 3193

static struct device_attribute *srp_host_attrs[] = {
	&dev_attr_id_ext,
	&dev_attr_ioc_guid,
	&dev_attr_service_id,
	&dev_attr_pkey,
B
Bart Van Assche 已提交
3194
	&dev_attr_sgid,
3195 3196
	&dev_attr_dgid,
	&dev_attr_orig_dgid,
3197
	&dev_attr_req_lim,
3198 3199 3200
	&dev_attr_zero_req_lim,
	&dev_attr_local_ib_port,
	&dev_attr_local_ib_device,
B
Bart Van Assche 已提交
3201
	&dev_attr_ch_count,
3202
	&dev_attr_comp_vector,
3203
	&dev_attr_tl_retry_count,
3204
	&dev_attr_cmd_sg_entries,
3205
	&dev_attr_allow_ext_sg,
3206 3207 3208
	NULL
};

3209 3210
static struct scsi_host_template srp_template = {
	.module				= THIS_MODULE,
R
Roland Dreier 已提交
3211 3212
	.name				= "InfiniBand SRP initiator",
	.proc_name			= DRV_NAME,
3213
	.target_alloc			= srp_target_alloc,
3214
	.slave_alloc			= srp_slave_alloc,
3215
	.slave_configure		= srp_slave_configure,
3216 3217
	.info				= srp_target_info,
	.queuecommand			= srp_queuecommand,
3218
	.change_queue_depth             = srp_change_queue_depth,
3219
	.eh_timed_out			= srp_timed_out,
3220 3221 3222
	.eh_abort_handler		= srp_abort,
	.eh_device_reset_handler	= srp_reset_device,
	.eh_host_reset_handler		= srp_reset_host,
B
Bart Van Assche 已提交
3223
	.skip_settle_delay		= true,
3224
	.sg_tablesize			= SRP_DEF_SG_TABLESIZE,
3225
	.can_queue			= SRP_DEFAULT_CMD_SQ_SIZE,
3226
	.this_id			= -1,
3227
	.cmd_per_lun			= SRP_DEFAULT_CMD_SQ_SIZE,
3228
	.use_clustering			= ENABLE_CLUSTERING,
B
Bart Van Assche 已提交
3229
	.shost_attrs			= srp_host_attrs,
3230
	.track_queue_depth		= 1,
3231 3232
};

3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243
static int srp_sdev_count(struct Scsi_Host *host)
{
	struct scsi_device *sdev;
	int c = 0;

	shost_for_each_device(sdev, host)
		c++;

	return c;
}

3244 3245 3246 3247 3248 3249 3250
/*
 * Return values:
 * < 0 upon failure. Caller is responsible for SRP target port cleanup.
 * 0 and target->state == SRP_TARGET_REMOVED if asynchronous target port
 *    removal has been scheduled.
 * 0 and target->state != SRP_TARGET_REMOVED upon success.
 */
3251 3252
static int srp_add_target(struct srp_host *host, struct srp_target_port *target)
{
3253 3254 3255
	struct srp_rport_identifiers ids;
	struct srp_rport *rport;

3256
	target->state = SRP_TARGET_SCANNING;
3257
	sprintf(target->target_name, "SRP.T10:%016llX",
3258
		be64_to_cpu(target->id_ext));
3259

3260
	if (scsi_add_host(target->scsi_host, host->srp_dev->dev->dev.parent))
3261 3262
		return -ENODEV;

3263 3264
	memcpy(ids.port_id, &target->id_ext, 8);
	memcpy(ids.port_id + 8, &target->ioc_guid, 8);
3265
	ids.roles = SRP_RPORT_ROLE_TARGET;
3266 3267 3268 3269 3270 3271
	rport = srp_rport_add(target->scsi_host, &ids);
	if (IS_ERR(rport)) {
		scsi_remove_host(target->scsi_host);
		return PTR_ERR(rport);
	}

3272
	rport->lld_data = target;
3273
	target->rport = rport;
3274

3275
	spin_lock(&host->target_lock);
3276
	list_add_tail(&target->list, &host->target_list);
3277
	spin_unlock(&host->target_lock);
3278 3279

	scsi_scan_target(&target->scsi_host->shost_gendev,
3280
			 0, target->scsi_id, SCAN_WILD_CARD, SCSI_SCAN_INITIAL);
3281

3282 3283
	if (srp_connected_ch(target) < target->ch_count ||
	    target->qp_in_error) {
3284 3285 3286 3287 3288 3289
		shost_printk(KERN_INFO, target->scsi_host,
			     PFX "SCSI scan failed - removing SCSI host\n");
		srp_queue_remove_work(target);
		goto out;
	}

3290
	pr_debug("%s: SCSI scan succeeded - detected %d LUNs\n",
3291 3292 3293 3294 3295 3296 3297 3298 3299
		 dev_name(&target->scsi_host->shost_gendev),
		 srp_sdev_count(target->scsi_host));

	spin_lock_irq(&target->lock);
	if (target->state == SRP_TARGET_SCANNING)
		target->state = SRP_TARGET_LIVE;
	spin_unlock_irq(&target->lock);

out:
3300 3301 3302
	return 0;
}

3303
static void srp_release_dev(struct device *dev)
3304 3305
{
	struct srp_host *host =
3306
		container_of(dev, struct srp_host, dev);
3307 3308 3309 3310 3311 3312

	complete(&host->released);
}

static struct class srp_class = {
	.name    = "infiniband_srp",
3313
	.dev_release = srp_release_dev
3314 3315
};

3316 3317
/**
 * srp_conn_unique() - check whether the connection to a target is unique
3318 3319
 * @host:   SRP host.
 * @target: SRP target port.
3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336
 */
static bool srp_conn_unique(struct srp_host *host,
			    struct srp_target_port *target)
{
	struct srp_target_port *t;
	bool ret = false;

	if (target->state == SRP_TARGET_REMOVED)
		goto out;

	ret = true;

	spin_lock(&host->target_lock);
	list_for_each_entry(t, &host->target_list, list) {
		if (t != target &&
		    target->id_ext == t->id_ext &&
		    target->ioc_guid == t->ioc_guid &&
B
Bart Van Assche 已提交
3337 3338 3339
		    (!target->using_rdma_cm ||
		     memcmp(&target->rdma_cm.dst, &t->rdma_cm.dst,
			    sizeof(target->rdma_cm.dst)) == 0) &&
3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350
		    target->initiator_ext == t->initiator_ext) {
			ret = false;
			break;
		}
	}
	spin_unlock(&host->target_lock);

out:
	return ret;
}

3351 3352 3353 3354 3355
/*
 * Target ports are added by writing
 *
 *     id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,dgid=<dest GID>,
 *     pkey=<P_Key>,service_id=<service ID>
B
Bart Van Assche 已提交
3356 3357 3358
 * or
 *     id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,
 *     [src=<IPv4 address>,]dest=<IPv4 address>:<port number>
3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369
 *
 * to the add_target sysfs attribute.
 */
enum {
	SRP_OPT_ERR		= 0,
	SRP_OPT_ID_EXT		= 1 << 0,
	SRP_OPT_IOC_GUID	= 1 << 1,
	SRP_OPT_DGID		= 1 << 2,
	SRP_OPT_PKEY		= 1 << 3,
	SRP_OPT_SERVICE_ID	= 1 << 4,
	SRP_OPT_MAX_SECT	= 1 << 5,
3370
	SRP_OPT_MAX_CMD_PER_LUN	= 1 << 6,
3371
	SRP_OPT_IO_CLASS	= 1 << 7,
3372
	SRP_OPT_INITIATOR_EXT	= 1 << 8,
3373
	SRP_OPT_CMD_SG_ENTRIES	= 1 << 9,
3374 3375
	SRP_OPT_ALLOW_EXT_SG	= 1 << 10,
	SRP_OPT_SG_TABLESIZE	= 1 << 11,
3376
	SRP_OPT_COMP_VECTOR	= 1 << 12,
3377
	SRP_OPT_TL_RETRY_COUNT	= 1 << 13,
3378
	SRP_OPT_QUEUE_SIZE	= 1 << 14,
B
Bart Van Assche 已提交
3379 3380
	SRP_OPT_IP_SRC		= 1 << 15,
	SRP_OPT_IP_DEST		= 1 << 16,
3381
	SRP_OPT_TARGET_CAN_QUEUE= 1 << 17,
B
Bart Van Assche 已提交
3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392
};

static unsigned int srp_opt_mandatory[] = {
	SRP_OPT_ID_EXT		|
	SRP_OPT_IOC_GUID	|
	SRP_OPT_DGID		|
	SRP_OPT_PKEY		|
	SRP_OPT_SERVICE_ID,
	SRP_OPT_ID_EXT		|
	SRP_OPT_IOC_GUID	|
	SRP_OPT_IP_DEST,
3393 3394
};

3395
static const match_table_t srp_opt_tokens = {
3396 3397 3398 3399 3400 3401 3402
	{ SRP_OPT_ID_EXT,		"id_ext=%s" 		},
	{ SRP_OPT_IOC_GUID,		"ioc_guid=%s" 		},
	{ SRP_OPT_DGID,			"dgid=%s" 		},
	{ SRP_OPT_PKEY,			"pkey=%x" 		},
	{ SRP_OPT_SERVICE_ID,		"service_id=%s"		},
	{ SRP_OPT_MAX_SECT,		"max_sect=%d" 		},
	{ SRP_OPT_MAX_CMD_PER_LUN,	"max_cmd_per_lun=%d" 	},
3403
	{ SRP_OPT_TARGET_CAN_QUEUE,	"target_can_queue=%d"	},
3404
	{ SRP_OPT_IO_CLASS,		"io_class=%x"		},
3405
	{ SRP_OPT_INITIATOR_EXT,	"initiator_ext=%s"	},
3406
	{ SRP_OPT_CMD_SG_ENTRIES,	"cmd_sg_entries=%u"	},
3407 3408
	{ SRP_OPT_ALLOW_EXT_SG,		"allow_ext_sg=%u"	},
	{ SRP_OPT_SG_TABLESIZE,		"sg_tablesize=%u"	},
3409
	{ SRP_OPT_COMP_VECTOR,		"comp_vector=%u"	},
3410
	{ SRP_OPT_TL_RETRY_COUNT,	"tl_retry_count=%u"	},
3411
	{ SRP_OPT_QUEUE_SIZE,		"queue_size=%d"		},
B
Bart Van Assche 已提交
3412 3413
	{ SRP_OPT_IP_SRC,		"src=%s"		},
	{ SRP_OPT_IP_DEST,		"dest=%s"		},
3414
	{ SRP_OPT_ERR,			NULL 			}
3415 3416
};

3417 3418 3419 3420 3421 3422 3423
/**
 * srp_parse_in - parse an IP address and port number combination
 *
 * Parse the following address formats:
 * - IPv4: <ip_address>:<port>, e.g. 1.2.3.4:5.
 * - IPv6: \[<ipv6_address>\]:<port>, e.g. [1::2:3%4]:5.
 */
B
Bart Van Assche 已提交
3424 3425 3426
static int srp_parse_in(struct net *net, struct sockaddr_storage *sa,
			const char *addr_port_str)
{
3427 3428
	char *addr_end, *addr = kstrdup(addr_port_str, GFP_KERNEL);
	char *port_str;
B
Bart Van Assche 已提交
3429 3430 3431 3432
	int ret;

	if (!addr)
		return -ENOMEM;
3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445
	port_str = strrchr(addr, ':');
	if (!port_str)
		return -EINVAL;
	*port_str++ = '\0';
	ret = inet_pton_with_scope(net, AF_INET, addr, port_str, sa);
	if (ret && addr[0]) {
		addr_end = addr + strlen(addr) - 1;
		if (addr[0] == '[' && *addr_end == ']') {
			*addr_end = '\0';
			ret = inet_pton_with_scope(net, AF_INET6, addr + 1,
						   port_str, sa);
		}
	}
B
Bart Van Assche 已提交
3446
	kfree(addr);
3447
	pr_debug("%s -> %pISpfsc\n", addr_port_str, sa);
B
Bart Van Assche 已提交
3448 3449 3450 3451 3452
	return ret;
}

static int srp_parse_options(struct net *net, const char *buf,
			     struct srp_target_port *target)
3453 3454 3455 3456
{
	char *options, *sep_opt;
	char *p;
	substring_t args[MAX_OPT_ARGS];
3457
	unsigned long long ull;
3458 3459 3460 3461 3462 3463 3464 3465 3466 3467
	int opt_mask = 0;
	int token;
	int ret = -EINVAL;
	int i;

	options = kstrdup(buf, GFP_KERNEL);
	if (!options)
		return -ENOMEM;

	sep_opt = options;
3468
	while ((p = strsep(&sep_opt, ",\n")) != NULL) {
3469 3470 3471 3472 3473 3474 3475 3476 3477
		if (!*p)
			continue;

		token = match_token(p, srp_opt_tokens, args);
		opt_mask |= token;

		switch (token) {
		case SRP_OPT_ID_EXT:
			p = match_strdup(args);
3478 3479 3480 3481
			if (!p) {
				ret = -ENOMEM;
				goto out;
			}
3482 3483 3484 3485 3486 3487 3488
			ret = kstrtoull(p, 16, &ull);
			if (ret) {
				pr_warn("invalid id_ext parameter '%s'\n", p);
				kfree(p);
				goto out;
			}
			target->id_ext = cpu_to_be64(ull);
3489 3490 3491 3492 3493
			kfree(p);
			break;

		case SRP_OPT_IOC_GUID:
			p = match_strdup(args);
3494 3495 3496 3497
			if (!p) {
				ret = -ENOMEM;
				goto out;
			}
3498 3499 3500 3501 3502 3503 3504
			ret = kstrtoull(p, 16, &ull);
			if (ret) {
				pr_warn("invalid ioc_guid parameter '%s'\n", p);
				kfree(p);
				goto out;
			}
			target->ioc_guid = cpu_to_be64(ull);
3505 3506 3507 3508 3509
			kfree(p);
			break;

		case SRP_OPT_DGID:
			p = match_strdup(args);
3510 3511 3512 3513
			if (!p) {
				ret = -ENOMEM;
				goto out;
			}
3514
			if (strlen(p) != 32) {
3515
				pr_warn("bad dest GID parameter '%s'\n", p);
3516
				kfree(p);
3517 3518 3519
				goto out;
			}

B
Bart Van Assche 已提交
3520
			ret = hex2bin(target->ib_cm.orig_dgid.raw, p, 16);
3521
			kfree(p);
3522 3523
			if (ret < 0)
				goto out;
3524 3525 3526 3527
			break;

		case SRP_OPT_PKEY:
			if (match_hex(args, &token)) {
3528
				pr_warn("bad P_Key parameter '%s'\n", p);
3529 3530
				goto out;
			}
B
Bart Van Assche 已提交
3531
			target->ib_cm.pkey = cpu_to_be16(token);
3532 3533 3534 3535
			break;

		case SRP_OPT_SERVICE_ID:
			p = match_strdup(args);
3536 3537 3538 3539
			if (!p) {
				ret = -ENOMEM;
				goto out;
			}
3540 3541 3542 3543 3544 3545
			ret = kstrtoull(p, 16, &ull);
			if (ret) {
				pr_warn("bad service_id parameter '%s'\n", p);
				kfree(p);
				goto out;
			}
B
Bart Van Assche 已提交
3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578
			target->ib_cm.service_id = cpu_to_be64(ull);
			kfree(p);
			break;

		case SRP_OPT_IP_SRC:
			p = match_strdup(args);
			if (!p) {
				ret = -ENOMEM;
				goto out;
			}
			ret = srp_parse_in(net, &target->rdma_cm.src.ss, p);
			if (ret < 0) {
				pr_warn("bad source parameter '%s'\n", p);
				kfree(p);
				goto out;
			}
			target->rdma_cm.src_specified = true;
			kfree(p);
			break;

		case SRP_OPT_IP_DEST:
			p = match_strdup(args);
			if (!p) {
				ret = -ENOMEM;
				goto out;
			}
			ret = srp_parse_in(net, &target->rdma_cm.dst.ss, p);
			if (ret < 0) {
				pr_warn("bad dest parameter '%s'\n", p);
				kfree(p);
				goto out;
			}
			target->using_rdma_cm = true;
3579 3580 3581 3582 3583
			kfree(p);
			break;

		case SRP_OPT_MAX_SECT:
			if (match_int(args, &token)) {
3584
				pr_warn("bad max sect parameter '%s'\n", p);
3585 3586 3587 3588 3589
				goto out;
			}
			target->scsi_host->max_sectors = token;
			break;

3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601
		case SRP_OPT_QUEUE_SIZE:
			if (match_int(args, &token) || token < 1) {
				pr_warn("bad queue_size parameter '%s'\n", p);
				goto out;
			}
			target->scsi_host->can_queue = token;
			target->queue_size = token + SRP_RSP_SQ_SIZE +
					     SRP_TSK_MGMT_SQ_SIZE;
			if (!(opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
				target->scsi_host->cmd_per_lun = token;
			break;

3602
		case SRP_OPT_MAX_CMD_PER_LUN:
3603
			if (match_int(args, &token) || token < 1) {
3604 3605
				pr_warn("bad max cmd_per_lun parameter '%s'\n",
					p);
3606 3607
				goto out;
			}
3608
			target->scsi_host->cmd_per_lun = token;
3609 3610
			break;

3611 3612 3613 3614 3615 3616 3617 3618 3619
		case SRP_OPT_TARGET_CAN_QUEUE:
			if (match_int(args, &token) || token < 1) {
				pr_warn("bad max target_can_queue parameter '%s'\n",
					p);
				goto out;
			}
			target->target_can_queue = token;
			break;

3620 3621
		case SRP_OPT_IO_CLASS:
			if (match_hex(args, &token)) {
3622
				pr_warn("bad IO class parameter '%s'\n", p);
3623 3624 3625 3626
				goto out;
			}
			if (token != SRP_REV10_IB_IO_CLASS &&
			    token != SRP_REV16A_IB_IO_CLASS) {
3627 3628 3629
				pr_warn("unknown IO class parameter value %x specified (use %x or %x).\n",
					token, SRP_REV10_IB_IO_CLASS,
					SRP_REV16A_IB_IO_CLASS);
3630 3631 3632 3633 3634
				goto out;
			}
			target->io_class = token;
			break;

3635 3636
		case SRP_OPT_INITIATOR_EXT:
			p = match_strdup(args);
3637 3638 3639 3640
			if (!p) {
				ret = -ENOMEM;
				goto out;
			}
3641 3642 3643 3644 3645 3646 3647
			ret = kstrtoull(p, 16, &ull);
			if (ret) {
				pr_warn("bad initiator_ext value '%s'\n", p);
				kfree(p);
				goto out;
			}
			target->initiator_ext = cpu_to_be64(ull);
3648 3649 3650
			kfree(p);
			break;

3651 3652
		case SRP_OPT_CMD_SG_ENTRIES:
			if (match_int(args, &token) || token < 1 || token > 255) {
3653 3654
				pr_warn("bad max cmd_sg_entries parameter '%s'\n",
					p);
3655 3656 3657 3658 3659
				goto out;
			}
			target->cmd_sg_cnt = token;
			break;

3660 3661
		case SRP_OPT_ALLOW_EXT_SG:
			if (match_int(args, &token)) {
3662
				pr_warn("bad allow_ext_sg parameter '%s'\n", p);
3663 3664 3665 3666 3667 3668 3669
				goto out;
			}
			target->allow_ext_sg = !!token;
			break;

		case SRP_OPT_SG_TABLESIZE:
			if (match_int(args, &token) || token < 1 ||
3670
					token > SG_MAX_SEGMENTS) {
3671 3672
				pr_warn("bad max sg_tablesize parameter '%s'\n",
					p);
3673 3674 3675 3676 3677
				goto out;
			}
			target->sg_tablesize = token;
			break;

3678 3679 3680 3681 3682 3683 3684 3685
		case SRP_OPT_COMP_VECTOR:
			if (match_int(args, &token) || token < 0) {
				pr_warn("bad comp_vector parameter '%s'\n", p);
				goto out;
			}
			target->comp_vector = token;
			break;

3686 3687 3688 3689 3690 3691 3692 3693 3694
		case SRP_OPT_TL_RETRY_COUNT:
			if (match_int(args, &token) || token < 2 || token > 7) {
				pr_warn("bad tl_retry_count parameter '%s' (must be a number between 2 and 7)\n",
					p);
				goto out;
			}
			target->tl_retry_count = token;
			break;

3695
		default:
3696 3697
			pr_warn("unknown parameter or missing value '%s' in target creation request\n",
				p);
3698 3699 3700 3701
			goto out;
		}
	}

B
Bart Van Assche 已提交
3702 3703 3704 3705 3706 3707 3708 3709
	for (i = 0; i < ARRAY_SIZE(srp_opt_mandatory); i++) {
		if ((opt_mask & srp_opt_mandatory[i]) == srp_opt_mandatory[i]) {
			ret = 0;
			break;
		}
	}
	if (ret)
		pr_warn("target creation request is missing one or more parameters\n");
3710

3711 3712 3713 3714 3715 3716
	if (target->scsi_host->cmd_per_lun > target->scsi_host->can_queue
	    && (opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
		pr_warn("cmd_per_lun = %d > queue_size = %d\n",
			target->scsi_host->cmd_per_lun,
			target->scsi_host->can_queue);

3717 3718 3719 3720 3721
out:
	kfree(options);
	return ret;
}

3722 3723
static ssize_t srp_create_target(struct device *dev,
				 struct device_attribute *attr,
3724 3725 3726
				 const char *buf, size_t count)
{
	struct srp_host *host =
3727
		container_of(dev, struct srp_host, dev);
3728 3729
	struct Scsi_Host *target_host;
	struct srp_target_port *target;
3730
	struct srp_rdma_ch *ch;
3731 3732
	struct srp_device *srp_dev = host->srp_dev;
	struct ib_device *ibdev = srp_dev->dev;
B
Bart Van Assche 已提交
3733
	int ret, node_idx, node, cpu, i;
3734
	unsigned int max_sectors_per_mr, mr_per_cmd = 0;
B
Bart Van Assche 已提交
3735
	bool multich = false;
3736 3737 3738 3739 3740 3741

	target_host = scsi_host_alloc(&srp_template,
				      sizeof (struct srp_target_port));
	if (!target_host)
		return -ENOMEM;

3742
	target_host->transportt  = ib_srp_transport_template;
3743 3744
	target_host->max_channel = 0;
	target_host->max_id      = 1;
B
Bart Van Assche 已提交
3745
	target_host->max_lun     = -1LL;
A
Arne Redlich 已提交
3746
	target_host->max_cmd_len = sizeof ((struct srp_cmd *) (void *) 0L)->cdb;
R
Roland Dreier 已提交
3747

3748 3749
	target = host_to_target(target_host);

B
Bart Van Assche 已提交
3750
	target->net		= kobj_ns_grab_current(KOBJ_NS_TYPE_NET);
3751 3752 3753
	target->io_class	= SRP_REV16A_IB_IO_CLASS;
	target->scsi_host	= target_host;
	target->srp_host	= host;
J
Jason Gunthorpe 已提交
3754
	target->lkey		= host->srp_dev->pd->local_dma_lkey;
B
Bart Van Assche 已提交
3755
	target->global_rkey	= host->srp_dev->global_rkey;
3756
	target->cmd_sg_cnt	= cmd_sg_entries;
3757 3758
	target->sg_tablesize	= indirect_sg_entries ? : cmd_sg_entries;
	target->allow_ext_sg	= allow_ext_sg;
3759
	target->tl_retry_count	= 7;
3760
	target->queue_size	= SRP_DEFAULT_QUEUE_SIZE;
3761

3762 3763 3764 3765 3766 3767
	/*
	 * Avoid that the SCSI host can be removed by srp_remove_target()
	 * before this function returns.
	 */
	scsi_host_get(target->scsi_host);

3768 3769 3770
	ret = mutex_lock_interruptible(&host->add_target_mutex);
	if (ret < 0)
		goto put;
3771

B
Bart Van Assche 已提交
3772
	ret = srp_parse_options(target->net, buf, target);
3773
	if (ret)
3774
		goto out;
3775

3776 3777
	target->req_ring_size = target->queue_size - SRP_TSK_MGMT_SQ_SIZE;

3778
	if (!srp_conn_unique(target->srp_host, target)) {
B
Bart Van Assche 已提交
3779 3780
		if (target->using_rdma_cm) {
			shost_printk(KERN_INFO, target->scsi_host,
3781
				     PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;dest=%pIS\n",
B
Bart Van Assche 已提交
3782 3783
				     be64_to_cpu(target->id_ext),
				     be64_to_cpu(target->ioc_guid),
3784
				     &target->rdma_cm.dst);
B
Bart Van Assche 已提交
3785 3786 3787 3788 3789 3790 3791
		} else {
			shost_printk(KERN_INFO, target->scsi_host,
				     PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;initiator_ext=%016llx\n",
				     be64_to_cpu(target->id_ext),
				     be64_to_cpu(target->ioc_guid),
				     be64_to_cpu(target->initiator_ext));
		}
3792
		ret = -EEXIST;
3793
		goto out;
3794 3795
	}

3796
	if (!srp_dev->has_fmr && !srp_dev->has_fr && !target->allow_ext_sg &&
3797
	    target->cmd_sg_cnt < target->sg_tablesize) {
3798
		pr_warn("No MR pool and no external indirect descriptors, limiting sg_tablesize to cmd_sg_cnt\n");
3799 3800 3801
		target->sg_tablesize = target->cmd_sg_cnt;
	}

3802
	if (srp_dev->use_fast_reg || srp_dev->use_fmr) {
3803 3804 3805
		bool gaps_reg = (ibdev->attrs.device_cap_flags &
				 IB_DEVICE_SG_GAPS_REG);

3806 3807
		max_sectors_per_mr = srp_dev->max_pages_per_mr <<
				  (ilog2(srp_dev->mr_page_size) - 9);
3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830
		if (!gaps_reg) {
			/*
			 * FR and FMR can only map one HCA page per entry. If
			 * the start address is not aligned on a HCA page
			 * boundary two entries will be used for the head and
			 * the tail although these two entries combined
			 * contain at most one HCA page of data. Hence the "+
			 * 1" in the calculation below.
			 *
			 * The indirect data buffer descriptor is contiguous
			 * so the memory for that buffer will only be
			 * registered if register_always is true. Hence add
			 * one to mr_per_cmd if register_always has been set.
			 */
			mr_per_cmd = register_always +
				(target->scsi_host->max_sectors + 1 +
				 max_sectors_per_mr - 1) / max_sectors_per_mr;
		} else {
			mr_per_cmd = register_always +
				(target->sg_tablesize +
				 srp_dev->max_pages_per_mr - 1) /
				srp_dev->max_pages_per_mr;
		}
3831
		pr_debug("max_sectors = %u; max_pages_per_mr = %u; mr_page_size = %u; max_sectors_per_mr = %u; mr_per_cmd = %u\n",
3832
			 target->scsi_host->max_sectors, srp_dev->max_pages_per_mr, srp_dev->mr_page_size,
3833 3834 3835
			 max_sectors_per_mr, mr_per_cmd);
	}

3836
	target_host->sg_tablesize = target->sg_tablesize;
3837 3838
	target->mr_pool_size = target->scsi_host->can_queue * mr_per_cmd;
	target->mr_per_cmd = mr_per_cmd;
3839 3840
	target->indirect_size = target->sg_tablesize *
				sizeof (struct srp_direct_buf);
3841 3842 3843 3844
	target->max_iu_len = sizeof (struct srp_cmd) +
			     sizeof (struct srp_indirect_buf) +
			     target->cmd_sg_cnt * sizeof (struct srp_direct_buf);

3845
	INIT_WORK(&target->tl_err_work, srp_tl_err_work);
3846
	INIT_WORK(&target->remove_work, srp_remove_work);
3847
	spin_lock_init(&target->lock);
3848
	ret = ib_query_gid(ibdev, host->port, 0, &target->sgid, NULL);
3849
	if (ret)
3850
		goto out;
3851

B
Bart Van Assche 已提交
3852 3853 3854 3855 3856 3857 3858 3859 3860
	ret = -ENOMEM;
	target->ch_count = max_t(unsigned, num_online_nodes(),
				 min(ch_count ? :
				     min(4 * num_online_nodes(),
					 ibdev->num_comp_vectors),
				     num_online_cpus()));
	target->ch = kcalloc(target->ch_count, sizeof(*target->ch),
			     GFP_KERNEL);
	if (!target->ch)
3861
		goto out;
3862

B
Bart Van Assche 已提交
3863 3864 3865 3866 3867 3868
	node_idx = 0;
	for_each_online_node(node) {
		const int ch_start = (node_idx * target->ch_count /
				      num_online_nodes());
		const int ch_end = ((node_idx + 1) * target->ch_count /
				    num_online_nodes());
3869 3870 3871 3872
		const int cv_start = node_idx * ibdev->num_comp_vectors /
				     num_online_nodes();
		const int cv_end = (node_idx + 1) * ibdev->num_comp_vectors /
				   num_online_nodes();
B
Bart Van Assche 已提交
3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888
		int cpu_idx = 0;

		for_each_online_cpu(cpu) {
			if (cpu_to_node(cpu) != node)
				continue;
			if (ch_start + cpu_idx >= ch_end)
				continue;
			ch = &target->ch[ch_start + cpu_idx];
			ch->target = target;
			ch->comp_vector = cv_start == cv_end ? cv_start :
				cv_start + cpu_idx % (cv_end - cv_start);
			spin_lock_init(&ch->lock);
			INIT_LIST_HEAD(&ch->free_tx);
			ret = srp_new_cm_id(ch);
			if (ret)
				goto err_disconnect;
3889

B
Bart Van Assche 已提交
3890 3891 3892 3893 3894 3895 3896 3897 3898 3899
			ret = srp_create_ch_ib(ch);
			if (ret)
				goto err_disconnect;

			ret = srp_alloc_req_data(ch);
			if (ret)
				goto err_disconnect;

			ret = srp_connect_ch(ch, multich);
			if (ret) {
B
Bart Van Assche 已提交
3900 3901 3902
				char dst[64];

				if (target->using_rdma_cm)
3903 3904
					snprintf(dst, sizeof(dst), "%pIS",
						 &target->rdma_cm.dst);
B
Bart Van Assche 已提交
3905 3906 3907
				else
					snprintf(dst, sizeof(dst), "%pI6",
						 target->ib_cm.orig_dgid.raw);
B
Bart Van Assche 已提交
3908
				shost_printk(KERN_ERR, target->scsi_host,
B
Bart Van Assche 已提交
3909
					     PFX "Connection %d/%d to %s failed\n",
B
Bart Van Assche 已提交
3910
					     ch_start + cpu_idx,
B
Bart Van Assche 已提交
3911
					     target->ch_count, dst);
B
Bart Van Assche 已提交
3912
				if (node_idx == 0 && cpu_idx == 0) {
B
Bart Van Assche 已提交
3913
					goto free_ch;
B
Bart Van Assche 已提交
3914 3915 3916 3917
				} else {
					srp_free_ch_ib(target, ch);
					srp_free_req_data(target, ch);
					target->ch_count = ch - target->ch;
3918
					goto connected;
B
Bart Van Assche 已提交
3919 3920 3921 3922 3923 3924 3925
				}
			}

			multich = true;
			cpu_idx++;
		}
		node_idx++;
3926 3927
	}

3928
connected:
B
Bart Van Assche 已提交
3929 3930
	target->scsi_host->nr_hw_queues = target->ch_count;

3931 3932 3933 3934
	ret = srp_add_target(host, target);
	if (ret)
		goto err_disconnect;

3935
	if (target->state != SRP_TARGET_REMOVED) {
B
Bart Van Assche 已提交
3936 3937
		if (target->using_rdma_cm) {
			shost_printk(KERN_DEBUG, target->scsi_host, PFX
3938
				     "new target: id_ext %016llx ioc_guid %016llx sgid %pI6 dest %pIS\n",
B
Bart Van Assche 已提交
3939 3940
				     be64_to_cpu(target->id_ext),
				     be64_to_cpu(target->ioc_guid),
3941
				     target->sgid.raw, &target->rdma_cm.dst);
B
Bart Van Assche 已提交
3942 3943 3944 3945 3946 3947 3948 3949 3950 3951
		} else {
			shost_printk(KERN_DEBUG, target->scsi_host, PFX
				     "new target: id_ext %016llx ioc_guid %016llx pkey %04x service_id %016llx sgid %pI6 dgid %pI6\n",
				     be64_to_cpu(target->id_ext),
				     be64_to_cpu(target->ioc_guid),
				     be16_to_cpu(target->ib_cm.pkey),
				     be64_to_cpu(target->ib_cm.service_id),
				     target->sgid.raw,
				     target->ib_cm.orig_dgid.raw);
		}
3952
	}
B
Bart Van Assche 已提交
3953

3954 3955 3956 3957
	ret = count;

out:
	mutex_unlock(&host->add_target_mutex);
3958

3959
put:
3960
	scsi_host_put(target->scsi_host);
B
Bart Van Assche 已提交
3961 3962 3963 3964 3965 3966 3967 3968
	if (ret < 0) {
		/*
		 * If a call to srp_remove_target() has not been scheduled,
		 * drop the network namespace reference now that was obtained
		 * earlier in this function.
		 */
		if (target->state != SRP_TARGET_REMOVED)
			kobj_ns_drop(KOBJ_NS_TYPE_NET, target->net);
3969
		scsi_host_put(target->scsi_host);
B
Bart Van Assche 已提交
3970
	}
3971

3972
	return ret;
3973 3974 3975 3976

err_disconnect:
	srp_disconnect_target(target);

B
Bart Van Assche 已提交
3977
free_ch:
B
Bart Van Assche 已提交
3978 3979 3980 3981 3982
	for (i = 0; i < target->ch_count; i++) {
		ch = &target->ch[i];
		srp_free_ch_ib(target, ch);
		srp_free_req_data(target, ch);
	}
3983

B
Bart Van Assche 已提交
3984
	kfree(target->ch);
3985
	goto out;
3986 3987
}

3988
static DEVICE_ATTR(add_target, S_IWUSR, NULL, srp_create_target);
3989

3990 3991
static ssize_t show_ibdev(struct device *dev, struct device_attribute *attr,
			  char *buf)
3992
{
3993
	struct srp_host *host = container_of(dev, struct srp_host, dev);
3994

3995
	return sprintf(buf, "%s\n", host->srp_dev->dev->name);
3996 3997
}

3998
static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);
3999

4000 4001
static ssize_t show_port(struct device *dev, struct device_attribute *attr,
			 char *buf)
4002
{
4003
	struct srp_host *host = container_of(dev, struct srp_host, dev);
4004 4005 4006 4007

	return sprintf(buf, "%d\n", host->port);
}

4008
static DEVICE_ATTR(port, S_IRUGO, show_port, NULL);
4009

4010
static struct srp_host *srp_add_port(struct srp_device *device, u8 port)
4011 4012 4013 4014 4015 4016 4017 4018
{
	struct srp_host *host;

	host = kzalloc(sizeof *host, GFP_KERNEL);
	if (!host)
		return NULL;

	INIT_LIST_HEAD(&host->target_list);
4019
	spin_lock_init(&host->target_lock);
4020
	init_completion(&host->released);
4021
	mutex_init(&host->add_target_mutex);
4022
	host->srp_dev = device;
4023 4024
	host->port = port;

4025
	host->dev.class = &srp_class;
4026
	host->dev.parent = device->dev->dev.parent;
4027
	dev_set_name(&host->dev, "srp-%s-%d", device->dev->name, port);
4028

4029
	if (device_register(&host->dev))
4030
		goto free_host;
4031
	if (device_create_file(&host->dev, &dev_attr_add_target))
4032
		goto err_class;
4033
	if (device_create_file(&host->dev, &dev_attr_ibdev))
4034
		goto err_class;
4035
	if (device_create_file(&host->dev, &dev_attr_port))
4036 4037 4038 4039 4040
		goto err_class;

	return host;

err_class:
4041
	device_unregister(&host->dev);
4042

4043
free_host:
4044 4045 4046 4047 4048 4049 4050
	kfree(host);

	return NULL;
}

static void srp_add_one(struct ib_device *device)
{
4051
	struct srp_device *srp_dev;
4052
	struct ib_device_attr *attr = &device->attrs;
4053
	struct srp_host *host;
4054
	int mr_page_shift, p;
4055
	u64 max_pages_per_mr;
4056
	unsigned int flags = 0;
4057

4058
	srp_dev = kzalloc(sizeof(*srp_dev), GFP_KERNEL);
4059
	if (!srp_dev)
4060
		return;
4061 4062 4063

	/*
	 * Use the smallest page size supported by the HCA, down to a
4064 4065
	 * minimum of 4096 bytes. We're unlikely to build large sglists
	 * out of smaller entries.
4066
	 */
4067
	mr_page_shift		= max(12, ffs(attr->page_size_cap) - 1);
4068 4069
	srp_dev->mr_page_size	= 1 << mr_page_shift;
	srp_dev->mr_page_mask	= ~((u64) srp_dev->mr_page_size - 1);
4070
	max_pages_per_mr	= attr->max_mr_size;
4071
	do_div(max_pages_per_mr, srp_dev->mr_page_size);
4072
	pr_debug("%s: %llu / %u = %llu <> %u\n", __func__,
4073
		 attr->max_mr_size, srp_dev->mr_page_size,
4074
		 max_pages_per_mr, SRP_MAX_PAGES_PER_MR);
4075 4076
	srp_dev->max_pages_per_mr = min_t(u64, SRP_MAX_PAGES_PER_MR,
					  max_pages_per_mr);
4077 4078 4079

	srp_dev->has_fmr = (device->alloc_fmr && device->dealloc_fmr &&
			    device->map_phys_fmr && device->unmap_fmr);
4080
	srp_dev->has_fr = (attr->device_cap_flags &
4081
			   IB_DEVICE_MEM_MGT_EXTENSIONS);
4082
	if (!never_register && !srp_dev->has_fmr && !srp_dev->has_fr) {
4083
		dev_warn(&device->dev, "neither FMR nor FR is supported\n");
4084
	} else if (!never_register &&
4085
		   attr->max_mr_size >= 2 * srp_dev->mr_page_size) {
4086 4087 4088 4089
		srp_dev->use_fast_reg = (srp_dev->has_fr &&
					 (!srp_dev->has_fmr || prefer_fr));
		srp_dev->use_fmr = !srp_dev->use_fast_reg && srp_dev->has_fmr;
	}
4090

4091 4092 4093 4094
	if (never_register || !register_always ||
	    (!srp_dev->has_fmr && !srp_dev->has_fr))
		flags |= IB_PD_UNSAFE_GLOBAL_RKEY;

4095 4096 4097
	if (srp_dev->use_fast_reg) {
		srp_dev->max_pages_per_mr =
			min_t(u32, srp_dev->max_pages_per_mr,
4098
			      attr->max_fast_reg_page_list_len);
4099
	}
4100 4101
	srp_dev->mr_max_size	= srp_dev->mr_page_size *
				   srp_dev->max_pages_per_mr;
4102
	pr_debug("%s: mr_page_shift = %d, device->max_mr_size = %#llx, device->max_fast_reg_page_list_len = %u, max_pages_per_mr = %d, mr_max_size = %#x\n",
4103 4104
		 device->name, mr_page_shift, attr->max_mr_size,
		 attr->max_fast_reg_page_list_len,
4105
		 srp_dev->max_pages_per_mr, srp_dev->mr_max_size);
4106 4107 4108 4109

	INIT_LIST_HEAD(&srp_dev->dev_list);

	srp_dev->dev = device;
4110
	srp_dev->pd  = ib_alloc_pd(device, flags);
4111 4112 4113
	if (IS_ERR(srp_dev->pd))
		goto free_dev;

B
Bart Van Assche 已提交
4114 4115 4116 4117
	if (flags & IB_PD_UNSAFE_GLOBAL_RKEY) {
		srp_dev->global_rkey = srp_dev->pd->unsafe_global_rkey;
		WARN_ON_ONCE(srp_dev->global_rkey == 0);
	}
4118

4119
	for (p = rdma_start_port(device); p <= rdma_end_port(device); ++p) {
4120
		host = srp_add_port(srp_dev, p);
4121
		if (host)
4122
			list_add_tail(&host->list, &srp_dev->dev_list);
4123 4124
	}

4125
	ib_set_client_data(device, &srp_client, srp_dev);
4126
	return;
4127 4128 4129

free_dev:
	kfree(srp_dev);
4130 4131
}

4132
static void srp_remove_one(struct ib_device *device, void *client_data)
4133
{
4134
	struct srp_device *srp_dev;
4135
	struct srp_host *host, *tmp_host;
4136
	struct srp_target_port *target;
4137

4138
	srp_dev = client_data;
4139 4140
	if (!srp_dev)
		return;
4141

4142
	list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) {
4143
		device_unregister(&host->dev);
4144 4145 4146 4147 4148 4149 4150
		/*
		 * Wait for the sysfs entry to go away, so that no new
		 * target ports can be created.
		 */
		wait_for_completion(&host->released);

		/*
4151
		 * Remove all target ports.
4152
		 */
4153
		spin_lock(&host->target_lock);
4154 4155
		list_for_each_entry(target, &host->target_list, list)
			srp_queue_remove_work(target);
4156
		spin_unlock(&host->target_lock);
4157 4158

		/*
4159
		 * Wait for tl_err and target port removal tasks.
4160
		 */
4161
		flush_workqueue(system_long_wq);
4162
		flush_workqueue(srp_remove_wq);
4163 4164 4165 4166

		kfree(host);
	}

4167 4168 4169
	ib_dealloc_pd(srp_dev->pd);

	kfree(srp_dev);
4170 4171
}

4172
static struct srp_function_template ib_srp_transport_functions = {
4173 4174
	.has_rport_state	 = true,
	.reset_timer_if_blocked	 = true,
4175
	.reconnect_delay	 = &srp_reconnect_delay,
4176 4177 4178
	.fast_io_fail_tmo	 = &srp_fast_io_fail_tmo,
	.dev_loss_tmo		 = &srp_dev_loss_tmo,
	.reconnect		 = srp_rport_reconnect,
4179
	.rport_delete		 = srp_rport_delete,
4180
	.terminate_rport_io	 = srp_terminate_io,
4181 4182
};

4183 4184 4185 4186
static int __init srp_init_module(void)
{
	int ret;

4187
	if (srp_sg_tablesize) {
4188
		pr_warn("srp_sg_tablesize is deprecated, please use cmd_sg_entries\n");
4189 4190 4191 4192 4193 4194 4195 4196
		if (!cmd_sg_entries)
			cmd_sg_entries = srp_sg_tablesize;
	}

	if (!cmd_sg_entries)
		cmd_sg_entries = SRP_DEF_SG_TABLESIZE;

	if (cmd_sg_entries > 255) {
4197
		pr_warn("Clamping cmd_sg_entries to 255\n");
4198
		cmd_sg_entries = 255;
4199 4200
	}

4201 4202 4203
	if (!indirect_sg_entries)
		indirect_sg_entries = cmd_sg_entries;
	else if (indirect_sg_entries < cmd_sg_entries) {
4204 4205
		pr_warn("Bumping up indirect_sg_entries to match cmd_sg_entries (%u)\n",
			cmd_sg_entries);
4206 4207 4208
		indirect_sg_entries = cmd_sg_entries;
	}

4209 4210 4211 4212 4213 4214
	if (indirect_sg_entries > SG_MAX_SEGMENTS) {
		pr_warn("Clamping indirect_sg_entries to %u\n",
			SG_MAX_SEGMENTS);
		indirect_sg_entries = SG_MAX_SEGMENTS;
	}

4215
	srp_remove_wq = create_workqueue("srp_remove");
4216 4217
	if (!srp_remove_wq) {
		ret = -ENOMEM;
4218 4219 4220 4221
		goto out;
	}

	ret = -ENOMEM;
4222 4223 4224
	ib_srp_transport_template =
		srp_attach_transport(&ib_srp_transport_functions);
	if (!ib_srp_transport_template)
4225
		goto destroy_wq;
4226

4227 4228
	ret = class_register(&srp_class);
	if (ret) {
4229
		pr_err("couldn't register class infiniband_srp\n");
4230
		goto release_tr;
4231 4232
	}

4233 4234
	ib_sa_register_client(&srp_sa_client);

4235 4236
	ret = ib_register_client(&srp_client);
	if (ret) {
4237
		pr_err("couldn't register IB client\n");
4238
		goto unreg_sa;
4239 4240
	}

4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253
out:
	return ret;

unreg_sa:
	ib_sa_unregister_client(&srp_sa_client);
	class_unregister(&srp_class);

release_tr:
	srp_release_transport(ib_srp_transport_template);

destroy_wq:
	destroy_workqueue(srp_remove_wq);
	goto out;
4254 4255 4256 4257 4258
}

static void __exit srp_cleanup_module(void)
{
	ib_unregister_client(&srp_client);
4259
	ib_sa_unregister_client(&srp_sa_client);
4260
	class_unregister(&srp_class);
4261
	srp_release_transport(ib_srp_transport_template);
4262
	destroy_workqueue(srp_remove_wq);
4263 4264 4265 4266
}

module_init(srp_init_module);
module_exit(srp_cleanup_module);