cma.c 91.1 KB
Newer Older
1 2 3 4 5 6
/*
 * Copyright (c) 2005 Voltaire Inc.  All rights reserved.
 * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved.
 * Copyright (c) 1999-2005, Mellanox Technologies, Inc. All rights reserved.
 * Copyright (c) 2005-2006 Intel Corporation.  All rights reserved.
 *
S
Sean Hefty 已提交
7 8 9 10 11
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
12
 *
S
Sean Hefty 已提交
13 14 15
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
16
 *
S
Sean Hefty 已提交
17 18 19
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
20
 *
S
Sean Hefty 已提交
21 22 23 24
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
25
 *
S
Sean Hefty 已提交
26 27 28 29 30 31 32 33
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
34 35 36 37 38 39 40 41
 */

#include <linux/completion.h>
#include <linux/in.h>
#include <linux/in6.h>
#include <linux/mutex.h>
#include <linux/random.h>
#include <linux/idr.h>
T
Tom Tucker 已提交
42
#include <linux/inetdevice.h>
43
#include <linux/slab.h>
44
#include <linux/module.h>
45
#include <net/route.h>
46 47

#include <net/tcp.h>
A
Aleksey Senin 已提交
48
#include <net/ipv6.h>
49 50 51

#include <rdma/rdma_cm.h>
#include <rdma/rdma_cm_ib.h>
52
#include <rdma/rdma_netlink.h>
53
#include <rdma/ib.h>
54 55 56
#include <rdma/ib_cache.h>
#include <rdma/ib_cm.h>
#include <rdma/ib_sa.h>
T
Tom Tucker 已提交
57
#include <rdma/iw_cm.h>
58 59 60 61 62 63

MODULE_AUTHOR("Sean Hefty");
MODULE_DESCRIPTION("Generic RDMA CM Agent");
MODULE_LICENSE("Dual BSD/GPL");

#define CMA_CM_RESPONSE_TIMEOUT 20
64
#define CMA_MAX_CM_RETRIES 15
65
#define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24)
66
#define CMA_IBOE_PACKET_LIFETIME 18
67 68 69 70 71 72 73 74 75 76

static void cma_add_one(struct ib_device *device);
static void cma_remove_one(struct ib_device *device);

static struct ib_client cma_client = {
	.name   = "cma",
	.add    = cma_add_one,
	.remove = cma_remove_one
};

77
static struct ib_sa_client sa_client;
78
static struct rdma_addr_client addr_client;
79 80 81 82 83 84
static LIST_HEAD(dev_list);
static LIST_HEAD(listen_any_list);
static DEFINE_MUTEX(lock);
static struct workqueue_struct *cma_wq;
static DEFINE_IDR(sdp_ps);
static DEFINE_IDR(tcp_ps);
S
Sean Hefty 已提交
85
static DEFINE_IDR(udp_ps);
86
static DEFINE_IDR(ipoib_ps);
87
static DEFINE_IDR(ib_ps);
88 89 90 91 92 93 94 95 96 97 98 99 100 101 102

struct cma_device {
	struct list_head	list;
	struct ib_device	*device;
	struct completion	comp;
	atomic_t		refcount;
	struct list_head	id_list;
};

struct rdma_bind_list {
	struct idr		*ps;
	struct hlist_head	owners;
	unsigned short		port;
};

103 104 105 106
enum {
	CMA_OPTION_AFONLY,
};

107 108 109 110 111 112 113 114 115 116 117
/*
 * Device removal can occur at anytime, so we need extra handling to
 * serialize notifying the user of device removal with other callbacks.
 * We do this by disabling removal notification while a callback is in process,
 * and reporting it after the callback completes.
 */
struct rdma_id_private {
	struct rdma_cm_id	id;

	struct rdma_bind_list	*bind_list;
	struct hlist_node	node;
118 119
	struct list_head	list; /* listen_any_list or cma_device.list */
	struct list_head	listen_list; /* per device listens */
120
	struct cma_device	*cma_dev;
121
	struct list_head	mc_list;
122

123
	int			internal_id;
124
	enum rdma_cm_state	state;
125
	spinlock_t		lock;
126 127
	struct mutex		qp_mutex;

128 129
	struct completion	comp;
	atomic_t		refcount;
130
	struct mutex		handler_mutex;
131 132 133 134 135 136 137

	int			backlog;
	int			timeout_ms;
	struct ib_sa_query	*query;
	int			query_id;
	union {
		struct ib_cm_id	*ib;
T
Tom Tucker 已提交
138
		struct iw_cm_id	*iw;
139 140 141
	} cm_id;

	u32			seq_num;
142
	u32			qkey;
143
	u32			qp_num;
N
Nir Muchtar 已提交
144
	pid_t			owner;
145
	u32			options;
146
	u8			srq;
147
	u8			tos;
148
	u8			reuseaddr;
149
	u8			afonly;
150 151
};

152 153 154 155 156 157 158
struct cma_multicast {
	struct rdma_id_private *id_priv;
	union {
		struct ib_sa_multicast *ib;
	} multicast;
	struct list_head	list;
	void			*context;
159
	struct sockaddr_storage	addr;
160
	struct kref		mcref;
161 162
};

163 164 165
struct cma_work {
	struct work_struct	work;
	struct rdma_id_private	*id;
166 167
	enum rdma_cm_state	old_state;
	enum rdma_cm_state	new_state;
168 169 170
	struct rdma_cm_event	event;
};

171 172 173 174 175 176
struct cma_ndev_work {
	struct work_struct	work;
	struct rdma_id_private	*id;
	struct rdma_cm_event	event;
};

177 178 179 180 181 182
struct iboe_mcast_work {
	struct work_struct	 work;
	struct rdma_id_private	*id;
	struct cma_multicast	*mc;
};

183 184 185
union cma_ip_addr {
	struct in6_addr ip6;
	struct {
186 187
		__be32 pad[3];
		__be32 addr;
188 189 190 191 192 193
	} ip4;
};

struct cma_hdr {
	u8 cma_version;
	u8 ip_version;	/* IP version: 7:4 */
194
	__be16 port;
195 196 197 198 199 200 201 202 203
	union cma_ip_addr src_addr;
	union cma_ip_addr dst_addr;
};

struct sdp_hh {
	u8 bsdh[16];
	u8 sdp_version; /* Major version: 7:4 */
	u8 ip_version;	/* IP version: 7:4 */
	u8 sdp_specific1[10];
204 205
	__be16 port;
	__be16 sdp_specific2;
206 207 208 209 210 211 212 213 214 215 216 217
	union cma_ip_addr src_addr;
	union cma_ip_addr dst_addr;
};

struct sdp_hah {
	u8 bsdh[16];
	u8 sdp_version;
};

#define CMA_VERSION 0x00
#define SDP_MAJ_VERSION 0x2

218
static int cma_comp(struct rdma_id_private *id_priv, enum rdma_cm_state comp)
219 220 221 222 223 224 225 226 227 228 229
{
	unsigned long flags;
	int ret;

	spin_lock_irqsave(&id_priv->lock, flags);
	ret = (id_priv->state == comp);
	spin_unlock_irqrestore(&id_priv->lock, flags);
	return ret;
}

static int cma_comp_exch(struct rdma_id_private *id_priv,
230
			 enum rdma_cm_state comp, enum rdma_cm_state exch)
231 232 233 234 235 236 237 238 239 240 241
{
	unsigned long flags;
	int ret;

	spin_lock_irqsave(&id_priv->lock, flags);
	if ((ret = (id_priv->state == comp)))
		id_priv->state = exch;
	spin_unlock_irqrestore(&id_priv->lock, flags);
	return ret;
}

242 243
static enum rdma_cm_state cma_exch(struct rdma_id_private *id_priv,
				   enum rdma_cm_state exch)
244 245
{
	unsigned long flags;
246
	enum rdma_cm_state old;
247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285

	spin_lock_irqsave(&id_priv->lock, flags);
	old = id_priv->state;
	id_priv->state = exch;
	spin_unlock_irqrestore(&id_priv->lock, flags);
	return old;
}

static inline u8 cma_get_ip_ver(struct cma_hdr *hdr)
{
	return hdr->ip_version >> 4;
}

static inline void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver)
{
	hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF);
}

static inline u8 sdp_get_majv(u8 sdp_version)
{
	return sdp_version >> 4;
}

static inline u8 sdp_get_ip_ver(struct sdp_hh *hh)
{
	return hh->ip_version >> 4;
}

static inline void sdp_set_ip_ver(struct sdp_hh *hh, u8 ip_ver)
{
	hh->ip_version = (ip_ver << 4) | (hh->ip_version & 0xF);
}

static void cma_attach_to_dev(struct rdma_id_private *id_priv,
			      struct cma_device *cma_dev)
{
	atomic_inc(&cma_dev->refcount);
	id_priv->cma_dev = cma_dev;
	id_priv->id.device = cma_dev->device;
286 287
	id_priv->id.route.addr.dev_addr.transport =
		rdma_node_get_transport(cma_dev->device->node_type);
288 289 290 291 292 293 294 295 296
	list_add_tail(&id_priv->list, &cma_dev->id_list);
}

static inline void cma_deref_dev(struct cma_device *cma_dev)
{
	if (atomic_dec_and_test(&cma_dev->refcount))
		complete(&cma_dev->comp);
}

297 298 299 300 301 302 303 304
static inline void release_mc(struct kref *kref)
{
	struct cma_multicast *mc = container_of(kref, struct cma_multicast, mcref);

	kfree(mc->multicast.ib);
	kfree(mc);
}

305
static void cma_release_dev(struct rdma_id_private *id_priv)
306
{
307
	mutex_lock(&lock);
308 309 310
	list_del(&id_priv->list);
	cma_deref_dev(id_priv->cma_dev);
	id_priv->cma_dev = NULL;
311
	mutex_unlock(&lock);
312 313
}

314 315 316 317 318 319 320 321 322 323 324 325 326 327 328
static inline struct sockaddr *cma_src_addr(struct rdma_id_private *id_priv)
{
	return (struct sockaddr *) &id_priv->id.route.addr.src_addr;
}

static inline struct sockaddr *cma_dst_addr(struct rdma_id_private *id_priv)
{
	return (struct sockaddr *) &id_priv->id.route.addr.dst_addr;
}

static inline unsigned short cma_family(struct rdma_id_private *id_priv)
{
	return id_priv->id.route.addr.src_addr.ss_family;
}

329
static int cma_set_qkey(struct rdma_id_private *id_priv)
330 331 332 333
{
	struct ib_sa_mcmember_rec rec;
	int ret = 0;

334 335 336 337
	if (id_priv->qkey)
		return 0;

	switch (id_priv->id.ps) {
338
	case RDMA_PS_UDP:
339
		id_priv->qkey = RDMA_UDP_QKEY;
340 341
		break;
	case RDMA_PS_IPOIB:
342 343 344 345 346 347
		ib_addr_get_mgid(&id_priv->id.route.addr.dev_addr, &rec.mgid);
		ret = ib_sa_get_mcmember_rec(id_priv->id.device,
					     id_priv->id.port_num, &rec.mgid,
					     &rec);
		if (!ret)
			id_priv->qkey = be32_to_cpu(rec.qkey);
348 349 350 351 352 353 354
		break;
	default:
		break;
	}
	return ret;
}

355 356 357 358 359 360 361 362 363
static int find_gid_port(struct ib_device *device, union ib_gid *gid, u8 port_num)
{
	int i;
	int err;
	struct ib_port_attr props;
	union ib_gid tmp;

	err = ib_query_port(device, port_num, &props);
	if (err)
364
		return err;
365 366 367 368

	for (i = 0; i < props.gid_tbl_len; ++i) {
		err = ib_query_gid(device, port_num, i, &tmp);
		if (err)
369
			return err;
370 371 372 373
		if (!memcmp(&tmp, gid, sizeof tmp))
			return 0;
	}

374
	return -EADDRNOTAVAIL;
375 376
}

377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397
static void cma_translate_ib(struct sockaddr_ib *sib, struct rdma_dev_addr *dev_addr)
{
	dev_addr->dev_type = ARPHRD_INFINIBAND;
	rdma_addr_set_sgid(dev_addr, (union ib_gid *) &sib->sib_addr);
	ib_addr_set_pkey(dev_addr, ntohs(sib->sib_pkey));
}

static int cma_translate_addr(struct sockaddr *addr, struct rdma_dev_addr *dev_addr)
{
	int ret;

	if (addr->sa_family != AF_IB) {
		ret = rdma_translate_ip(addr, dev_addr);
	} else {
		cma_translate_ib((struct sockaddr_ib *) addr, dev_addr);
		ret = 0;
	}

	return ret;
}

T
Tom Tucker 已提交
398
static int cma_acquire_dev(struct rdma_id_private *id_priv)
399
{
400
	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
401
	struct cma_device *cma_dev;
402
	union ib_gid gid, iboe_gid;
403
	int ret = -ENODEV;
404 405 406
	u8 port;
	enum rdma_link_layer dev_ll = dev_addr->dev_type == ARPHRD_INFINIBAND ?
		IB_LINK_LAYER_INFINIBAND : IB_LINK_LAYER_ETHERNET;
407

408 409 410 411
	if (dev_ll != IB_LINK_LAYER_INFINIBAND &&
	    id_priv->id.ps == RDMA_PS_IPOIB)
		return -EINVAL;

412
	mutex_lock(&lock);
413 414 415
	iboe_addr_get_sgid(dev_addr, &iboe_gid);
	memcpy(&gid, dev_addr->src_dev_addr +
	       rdma_addr_gid_offset(dev_addr), sizeof gid);
416
	list_for_each_entry(cma_dev, &dev_list, list) {
417 418 419 420 421 422 423 424 425 426 427
		for (port = 1; port <= cma_dev->device->phys_port_cnt; ++port) {
			if (rdma_port_get_link_layer(cma_dev->device, port) == dev_ll) {
				if (rdma_node_get_transport(cma_dev->device->node_type) == RDMA_TRANSPORT_IB &&
				    rdma_port_get_link_layer(cma_dev->device, port) == IB_LINK_LAYER_ETHERNET)
					ret = find_gid_port(cma_dev->device, &iboe_gid, port);
				else
					ret = find_gid_port(cma_dev->device, &gid, port);

				if (!ret) {
					id_priv->id.port_num = port;
					goto out;
428
				}
429
			}
430 431
		}
	}
432 433 434 435 436

out:
	if (!ret)
		cma_attach_to_dev(id_priv, cma_dev);

437
	mutex_unlock(&lock);
438 439 440 441 442 443 444 445 446
	return ret;
}

static void cma_deref_id(struct rdma_id_private *id_priv)
{
	if (atomic_dec_and_test(&id_priv->refcount))
		complete(&id_priv->comp);
}

447
static int cma_disable_callback(struct rdma_id_private *id_priv,
448
				enum rdma_cm_state state)
449
{
450 451 452 453 454 455
	mutex_lock(&id_priv->handler_mutex);
	if (id_priv->state != state) {
		mutex_unlock(&id_priv->handler_mutex);
		return -EINVAL;
	}
	return 0;
456 457 458
}

struct rdma_cm_id *rdma_create_id(rdma_cm_event_handler event_handler,
459 460
				  void *context, enum rdma_port_space ps,
				  enum ib_qp_type qp_type)
461 462 463 464 465 466 467
{
	struct rdma_id_private *id_priv;

	id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL);
	if (!id_priv)
		return ERR_PTR(-ENOMEM);

N
Nir Muchtar 已提交
468
	id_priv->owner = task_pid_nr(current);
469
	id_priv->state = RDMA_CM_IDLE;
470 471 472
	id_priv->id.context = context;
	id_priv->id.event_handler = event_handler;
	id_priv->id.ps = ps;
473
	id_priv->id.qp_type = qp_type;
474
	spin_lock_init(&id_priv->lock);
475
	mutex_init(&id_priv->qp_mutex);
476 477
	init_completion(&id_priv->comp);
	atomic_set(&id_priv->refcount, 1);
478
	mutex_init(&id_priv->handler_mutex);
479
	INIT_LIST_HEAD(&id_priv->listen_list);
480
	INIT_LIST_HEAD(&id_priv->mc_list);
481 482 483 484 485 486
	get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num);

	return &id_priv->id;
}
EXPORT_SYMBOL(rdma_create_id);

487
static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
488 489
{
	struct ib_qp_attr qp_attr;
490
	int qp_attr_mask, ret;
491

492 493
	qp_attr.qp_state = IB_QPS_INIT;
	ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
494 495 496
	if (ret)
		return ret;

497 498 499 500 501 502 503 504 505 506 507 508 509 510
	ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask);
	if (ret)
		return ret;

	qp_attr.qp_state = IB_QPS_RTR;
	ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE);
	if (ret)
		return ret;

	qp_attr.qp_state = IB_QPS_RTS;
	qp_attr.sq_psn = 0;
	ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN);

	return ret;
511 512
}

513
static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
T
Tom Tucker 已提交
514 515
{
	struct ib_qp_attr qp_attr;
516
	int qp_attr_mask, ret;
T
Tom Tucker 已提交
517 518

	qp_attr.qp_state = IB_QPS_INIT;
519 520 521
	ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
	if (ret)
		return ret;
T
Tom Tucker 已提交
522

523
	return ib_modify_qp(qp, &qp_attr, qp_attr_mask);
T
Tom Tucker 已提交
524 525
}

526 527 528 529 530 531 532 533 534 535 536 537 538 539 540
int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd,
		   struct ib_qp_init_attr *qp_init_attr)
{
	struct rdma_id_private *id_priv;
	struct ib_qp *qp;
	int ret;

	id_priv = container_of(id, struct rdma_id_private, id);
	if (id->device != pd->device)
		return -EINVAL;

	qp = ib_create_qp(pd, qp_init_attr);
	if (IS_ERR(qp))
		return PTR_ERR(qp);

541
	if (id->qp_type == IB_QPT_UD)
542 543 544
		ret = cma_init_ud_qp(id_priv, qp);
	else
		ret = cma_init_conn_qp(id_priv, qp);
545 546 547 548 549 550 551 552 553 554 555 556 557 558 559
	if (ret)
		goto err;

	id->qp = qp;
	id_priv->qp_num = qp->qp_num;
	id_priv->srq = (qp->srq != NULL);
	return 0;
err:
	ib_destroy_qp(qp);
	return ret;
}
EXPORT_SYMBOL(rdma_create_qp);

void rdma_destroy_qp(struct rdma_cm_id *id)
{
560 561 562 563 564 565 566
	struct rdma_id_private *id_priv;

	id_priv = container_of(id, struct rdma_id_private, id);
	mutex_lock(&id_priv->qp_mutex);
	ib_destroy_qp(id_priv->id.qp);
	id_priv->id.qp = NULL;
	mutex_unlock(&id_priv->qp_mutex);
567 568 569
}
EXPORT_SYMBOL(rdma_destroy_qp);

570 571
static int cma_modify_qp_rtr(struct rdma_id_private *id_priv,
			     struct rdma_conn_param *conn_param)
572 573 574 575
{
	struct ib_qp_attr qp_attr;
	int qp_attr_mask, ret;

576 577 578 579 580
	mutex_lock(&id_priv->qp_mutex);
	if (!id_priv->id.qp) {
		ret = 0;
		goto out;
	}
581 582 583

	/* Need to update QP attributes from default values. */
	qp_attr.qp_state = IB_QPS_INIT;
584
	ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
585
	if (ret)
586
		goto out;
587

588
	ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
589
	if (ret)
590
		goto out;
591 592

	qp_attr.qp_state = IB_QPS_RTR;
593
	ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
594
	if (ret)
595
		goto out;
596

597 598
	if (conn_param)
		qp_attr.max_dest_rd_atomic = conn_param->responder_resources;
599 600 601 602
	ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
out:
	mutex_unlock(&id_priv->qp_mutex);
	return ret;
603 604
}

605 606
static int cma_modify_qp_rts(struct rdma_id_private *id_priv,
			     struct rdma_conn_param *conn_param)
607 608 609 610
{
	struct ib_qp_attr qp_attr;
	int qp_attr_mask, ret;

611 612 613 614 615
	mutex_lock(&id_priv->qp_mutex);
	if (!id_priv->id.qp) {
		ret = 0;
		goto out;
	}
616 617

	qp_attr.qp_state = IB_QPS_RTS;
618
	ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
619
	if (ret)
620
		goto out;
621

622 623
	if (conn_param)
		qp_attr.max_rd_atomic = conn_param->initiator_depth;
624 625 626 627
	ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
out:
	mutex_unlock(&id_priv->qp_mutex);
	return ret;
628 629
}

630
static int cma_modify_qp_err(struct rdma_id_private *id_priv)
631 632
{
	struct ib_qp_attr qp_attr;
633
	int ret;
634

635 636 637 638 639
	mutex_lock(&id_priv->qp_mutex);
	if (!id_priv->id.qp) {
		ret = 0;
		goto out;
	}
640 641

	qp_attr.qp_state = IB_QPS_ERR;
642 643 644 645
	ret = ib_modify_qp(id_priv->id.qp, &qp_attr, IB_QP_STATE);
out:
	mutex_unlock(&id_priv->qp_mutex);
	return ret;
646 647
}

648 649 650 651 652
static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv,
			       struct ib_qp_attr *qp_attr, int *qp_attr_mask)
{
	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
	int ret;
653 654 655 656 657 658 659
	u16 pkey;

	if (rdma_port_get_link_layer(id_priv->id.device, id_priv->id.port_num) ==
	    IB_LINK_LAYER_INFINIBAND)
		pkey = ib_addr_get_pkey(dev_addr);
	else
		pkey = 0xffff;
660 661

	ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num,
662
				  pkey, &qp_attr->pkey_index);
663 664 665 666 667 668
	if (ret)
		return ret;

	qp_attr->port_num = id_priv->id.port_num;
	*qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT;

669
	if (id_priv->id.qp_type == IB_QPT_UD) {
670 671 672 673
		ret = cma_set_qkey(id_priv);
		if (ret)
			return ret;

674 675 676 677 678 679 680 681 682
		qp_attr->qkey = id_priv->qkey;
		*qp_attr_mask |= IB_QP_QKEY;
	} else {
		qp_attr->qp_access_flags = 0;
		*qp_attr_mask |= IB_QP_ACCESS_FLAGS;
	}
	return 0;
}

683 684 685 686
int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,
		       int *qp_attr_mask)
{
	struct rdma_id_private *id_priv;
687
	int ret = 0;
688 689

	id_priv = container_of(id, struct rdma_id_private, id);
T
Tom Tucker 已提交
690 691
	switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
	case RDMA_TRANSPORT_IB:
692
		if (!id_priv->cm_id.ib || (id_priv->id.qp_type == IB_QPT_UD))
693 694 695 696
			ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask);
		else
			ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr,
						 qp_attr_mask);
697 698 699
		if (qp_attr->qp_state == IB_QPS_RTR)
			qp_attr->rq_psn = id_priv->seq_num;
		break;
T
Tom Tucker 已提交
700
	case RDMA_TRANSPORT_IWARP:
701
		if (!id_priv->cm_id.iw) {
702
			qp_attr->qp_access_flags = 0;
703 704 705 706
			*qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS;
		} else
			ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr,
						 qp_attr_mask);
T
Tom Tucker 已提交
707
		break;
708 709 710 711 712 713 714 715 716 717 718
	default:
		ret = -ENOSYS;
		break;
	}

	return ret;
}
EXPORT_SYMBOL(rdma_init_qp_attr);

static inline int cma_zero_addr(struct sockaddr *addr)
{
719 720 721 722 723 724 725 726 727
	switch (addr->sa_family) {
	case AF_INET:
		return ipv4_is_zeronet(((struct sockaddr_in *)addr)->sin_addr.s_addr);
	case AF_INET6:
		return ipv6_addr_any(&((struct sockaddr_in6 *) addr)->sin6_addr);
	case AF_IB:
		return ib_addr_any(&((struct sockaddr_ib *) addr)->sib_addr);
	default:
		return 0;
728 729 730 731 732
	}
}

static inline int cma_loopback_addr(struct sockaddr *addr)
{
733 734 735 736 737 738 739 740 741 742
	switch (addr->sa_family) {
	case AF_INET:
		return ipv4_is_loopback(((struct sockaddr_in *) addr)->sin_addr.s_addr);
	case AF_INET6:
		return ipv6_addr_loopback(&((struct sockaddr_in6 *) addr)->sin6_addr);
	case AF_IB:
		return ib_addr_loopback(&((struct sockaddr_ib *) addr)->sib_addr);
	default:
		return 0;
	}
743 744 745 746 747 748 749
}

static inline int cma_any_addr(struct sockaddr *addr)
{
	return cma_zero_addr(addr) || cma_loopback_addr(addr);
}

750 751 752 753 754 755 756 757 758
static int cma_addr_cmp(struct sockaddr *src, struct sockaddr *dst)
{
	if (src->sa_family != dst->sa_family)
		return -1;

	switch (src->sa_family) {
	case AF_INET:
		return ((struct sockaddr_in *) src)->sin_addr.s_addr !=
		       ((struct sockaddr_in *) dst)->sin_addr.s_addr;
759
	case AF_INET6:
760 761
		return ipv6_addr_cmp(&((struct sockaddr_in6 *) src)->sin6_addr,
				     &((struct sockaddr_in6 *) dst)->sin6_addr);
762 763 764
	default:
		return ib_addr_cmp(&((struct sockaddr_ib *) src)->sib_addr,
				   &((struct sockaddr_ib *) dst)->sib_addr);
765 766 767
	}
}

768
static __be16 cma_port(struct sockaddr *addr)
S
Sean Hefty 已提交
769
{
770 771 772 773
	struct sockaddr_ib *sib;

	switch (addr->sa_family) {
	case AF_INET:
S
Sean Hefty 已提交
774
		return ((struct sockaddr_in *) addr)->sin_port;
775
	case AF_INET6:
S
Sean Hefty 已提交
776
		return ((struct sockaddr_in6 *) addr)->sin6_port;
777 778 779 780 781 782 783
	case AF_IB:
		sib = (struct sockaddr_ib *) addr;
		return htons((u16) (be64_to_cpu(sib->sib_sid) &
				    be64_to_cpu(sib->sib_sid_mask)));
	default:
		return 0;
	}
S
Sean Hefty 已提交
784 785
}

786 787
static inline int cma_any_port(struct sockaddr *addr)
{
S
Sean Hefty 已提交
788
	return !cma_port(addr);
789 790 791
}

static int cma_get_net_info(void *hdr, enum rdma_port_space ps,
792
			    u8 *ip_ver, __be16 *port,
793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823
			    union cma_ip_addr **src, union cma_ip_addr **dst)
{
	switch (ps) {
	case RDMA_PS_SDP:
		if (sdp_get_majv(((struct sdp_hh *) hdr)->sdp_version) !=
		    SDP_MAJ_VERSION)
			return -EINVAL;

		*ip_ver	= sdp_get_ip_ver(hdr);
		*port	= ((struct sdp_hh *) hdr)->port;
		*src	= &((struct sdp_hh *) hdr)->src_addr;
		*dst	= &((struct sdp_hh *) hdr)->dst_addr;
		break;
	default:
		if (((struct cma_hdr *) hdr)->cma_version != CMA_VERSION)
			return -EINVAL;

		*ip_ver	= cma_get_ip_ver(hdr);
		*port	= ((struct cma_hdr *) hdr)->port;
		*src	= &((struct cma_hdr *) hdr)->src_addr;
		*dst	= &((struct cma_hdr *) hdr)->dst_addr;
		break;
	}

	if (*ip_ver != 4 && *ip_ver != 6)
		return -EINVAL;
	return 0;
}

static void cma_save_net_info(struct rdma_addr *addr,
			      struct rdma_addr *listen_addr,
824
			      u8 ip_ver, __be16 port,
825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871
			      union cma_ip_addr *src, union cma_ip_addr *dst)
{
	struct sockaddr_in *listen4, *ip4;
	struct sockaddr_in6 *listen6, *ip6;

	switch (ip_ver) {
	case 4:
		listen4 = (struct sockaddr_in *) &listen_addr->src_addr;
		ip4 = (struct sockaddr_in *) &addr->src_addr;
		ip4->sin_family = listen4->sin_family;
		ip4->sin_addr.s_addr = dst->ip4.addr;
		ip4->sin_port = listen4->sin_port;

		ip4 = (struct sockaddr_in *) &addr->dst_addr;
		ip4->sin_family = listen4->sin_family;
		ip4->sin_addr.s_addr = src->ip4.addr;
		ip4->sin_port = port;
		break;
	case 6:
		listen6 = (struct sockaddr_in6 *) &listen_addr->src_addr;
		ip6 = (struct sockaddr_in6 *) &addr->src_addr;
		ip6->sin6_family = listen6->sin6_family;
		ip6->sin6_addr = dst->ip6;
		ip6->sin6_port = listen6->sin6_port;

		ip6 = (struct sockaddr_in6 *) &addr->dst_addr;
		ip6->sin6_family = listen6->sin6_family;
		ip6->sin6_addr = src->ip6;
		ip6->sin6_port = port;
		break;
	default:
		break;
	}
}

static inline int cma_user_data_offset(enum rdma_port_space ps)
{
	switch (ps) {
	case RDMA_PS_SDP:
		return 0;
	default:
		return sizeof(struct cma_hdr);
	}
}

static void cma_cancel_route(struct rdma_id_private *id_priv)
{
872 873
	switch (rdma_port_get_link_layer(id_priv->id.device, id_priv->id.port_num)) {
	case IB_LINK_LAYER_INFINIBAND:
874 875 876 877 878 879 880 881 882 883 884 885
		if (id_priv->query)
			ib_sa_cancel_query(id_priv->query_id, id_priv->query);
		break;
	default:
		break;
	}
}

static void cma_cancel_listens(struct rdma_id_private *id_priv)
{
	struct rdma_id_private *dev_id_priv;

886 887 888 889
	/*
	 * Remove from listen_any_list to prevent added devices from spawning
	 * additional listen requests.
	 */
890 891 892 893 894 895
	mutex_lock(&lock);
	list_del(&id_priv->list);

	while (!list_empty(&id_priv->listen_list)) {
		dev_id_priv = list_entry(id_priv->listen_list.next,
					 struct rdma_id_private, listen_list);
896 897 898 899 900 901 902
		/* sync with device removal to avoid duplicate destruction */
		list_del_init(&dev_id_priv->list);
		list_del(&dev_id_priv->listen_list);
		mutex_unlock(&lock);

		rdma_destroy_id(&dev_id_priv->id);
		mutex_lock(&lock);
903 904 905 906 907
	}
	mutex_unlock(&lock);
}

static void cma_cancel_operation(struct rdma_id_private *id_priv,
908
				 enum rdma_cm_state state)
909 910
{
	switch (state) {
911
	case RDMA_CM_ADDR_QUERY:
912 913
		rdma_addr_cancel(&id_priv->id.route.addr.dev_addr);
		break;
914
	case RDMA_CM_ROUTE_QUERY:
915 916
		cma_cancel_route(id_priv);
		break;
917
	case RDMA_CM_LISTEN:
918
		if (cma_any_addr(cma_src_addr(id_priv)) && !id_priv->cma_dev)
919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941
			cma_cancel_listens(id_priv);
		break;
	default:
		break;
	}
}

static void cma_release_port(struct rdma_id_private *id_priv)
{
	struct rdma_bind_list *bind_list = id_priv->bind_list;

	if (!bind_list)
		return;

	mutex_lock(&lock);
	hlist_del(&id_priv->node);
	if (hlist_empty(&bind_list->owners)) {
		idr_remove(bind_list->ps, bind_list->port);
		kfree(bind_list);
	}
	mutex_unlock(&lock);
}

942 943 944 945 946 947 948 949
static void cma_leave_mc_groups(struct rdma_id_private *id_priv)
{
	struct cma_multicast *mc;

	while (!list_empty(&id_priv->mc_list)) {
		mc = container_of(id_priv->mc_list.next,
				  struct cma_multicast, list);
		list_del(&mc->list);
950 951 952 953 954 955 956 957 958 959 960
		switch (rdma_port_get_link_layer(id_priv->cma_dev->device, id_priv->id.port_num)) {
		case IB_LINK_LAYER_INFINIBAND:
			ib_sa_free_multicast(mc->multicast.ib);
			kfree(mc);
			break;
		case IB_LINK_LAYER_ETHERNET:
			kref_put(&mc->mcref, release_mc);
			break;
		default:
			break;
		}
961 962 963
	}
}

964 965 966
void rdma_destroy_id(struct rdma_cm_id *id)
{
	struct rdma_id_private *id_priv;
967
	enum rdma_cm_state state;
968 969

	id_priv = container_of(id, struct rdma_id_private, id);
970
	state = cma_exch(id_priv, RDMA_CM_DESTROYING);
971 972
	cma_cancel_operation(id_priv, state);

973 974 975 976 977 978 979
	/*
	 * Wait for any active callback to finish.  New callbacks will find
	 * the id_priv state set to destroying and abort.
	 */
	mutex_lock(&id_priv->handler_mutex);
	mutex_unlock(&id_priv->handler_mutex);

980
	if (id_priv->cma_dev) {
981
		switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
T
Tom Tucker 已提交
982
		case RDMA_TRANSPORT_IB:
983
			if (id_priv->cm_id.ib)
984 985
				ib_destroy_cm_id(id_priv->cm_id.ib);
			break;
T
Tom Tucker 已提交
986
		case RDMA_TRANSPORT_IWARP:
987
			if (id_priv->cm_id.iw)
T
Tom Tucker 已提交
988 989
				iw_destroy_cm_id(id_priv->cm_id.iw);
			break;
990 991 992
		default:
			break;
		}
993
		cma_leave_mc_groups(id_priv);
994
		cma_release_dev(id_priv);
995 996 997 998 999 1000
	}

	cma_release_port(id_priv);
	cma_deref_id(id_priv);
	wait_for_completion(&id_priv->comp);

1001 1002 1003
	if (id_priv->internal_id)
		cma_deref_id(id_priv->id.context);

1004 1005 1006 1007 1008 1009 1010 1011 1012
	kfree(id_priv->id.route.path_rec);
	kfree(id_priv);
}
EXPORT_SYMBOL(rdma_destroy_id);

static int cma_rep_recv(struct rdma_id_private *id_priv)
{
	int ret;

1013
	ret = cma_modify_qp_rtr(id_priv, NULL);
1014 1015 1016
	if (ret)
		goto reject;

1017
	ret = cma_modify_qp_rts(id_priv, NULL);
1018 1019 1020 1021 1022 1023 1024 1025 1026
	if (ret)
		goto reject;

	ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0);
	if (ret)
		goto reject;

	return 0;
reject:
1027
	cma_modify_qp_err(id_priv);
1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042
	ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED,
		       NULL, 0, NULL, 0);
	return ret;
}

static int cma_verify_rep(struct rdma_id_private *id_priv, void *data)
{
	if (id_priv->id.ps == RDMA_PS_SDP &&
	    sdp_get_majv(((struct sdp_hah *) data)->sdp_version) !=
	    SDP_MAJ_VERSION)
		return -EINVAL;

	return 0;
}

1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056
static void cma_set_rep_event_data(struct rdma_cm_event *event,
				   struct ib_cm_rep_event_param *rep_data,
				   void *private_data)
{
	event->param.conn.private_data = private_data;
	event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE;
	event->param.conn.responder_resources = rep_data->responder_resources;
	event->param.conn.initiator_depth = rep_data->initiator_depth;
	event->param.conn.flow_control = rep_data->flow_control;
	event->param.conn.rnr_retry_count = rep_data->rnr_retry_count;
	event->param.conn.srq = rep_data->srq;
	event->param.conn.qp_num = rep_data->remote_qpn;
}

1057 1058 1059
static int cma_ib_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
{
	struct rdma_id_private *id_priv = cm_id->context;
1060 1061
	struct rdma_cm_event event;
	int ret = 0;
1062

1063
	if ((ib_event->event != IB_CM_TIMEWAIT_EXIT &&
1064
		cma_disable_callback(id_priv, RDMA_CM_CONNECT)) ||
1065
	    (ib_event->event == IB_CM_TIMEWAIT_EXIT &&
1066
		cma_disable_callback(id_priv, RDMA_CM_DISCONNECT)))
1067
		return 0;
1068

1069
	memset(&event, 0, sizeof event);
1070 1071 1072
	switch (ib_event->event) {
	case IB_CM_REQ_ERROR:
	case IB_CM_REP_ERROR:
1073 1074
		event.event = RDMA_CM_EVENT_UNREACHABLE;
		event.status = -ETIMEDOUT;
1075 1076
		break;
	case IB_CM_REP_RECEIVED:
1077 1078 1079
		event.status = cma_verify_rep(id_priv, ib_event->private_data);
		if (event.status)
			event.event = RDMA_CM_EVENT_CONNECT_ERROR;
1080
		else if (id_priv->id.qp && id_priv->id.ps != RDMA_PS_SDP) {
1081 1082 1083
			event.status = cma_rep_recv(id_priv);
			event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR :
						     RDMA_CM_EVENT_ESTABLISHED;
1084
		} else
1085 1086 1087
			event.event = RDMA_CM_EVENT_CONNECT_RESPONSE;
		cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd,
				       ib_event->private_data);
1088 1089
		break;
	case IB_CM_RTU_RECEIVED:
1090 1091
	case IB_CM_USER_ESTABLISHED:
		event.event = RDMA_CM_EVENT_ESTABLISHED;
1092 1093
		break;
	case IB_CM_DREQ_ERROR:
1094
		event.status = -ETIMEDOUT; /* fall through */
1095 1096
	case IB_CM_DREQ_RECEIVED:
	case IB_CM_DREP_RECEIVED:
1097 1098
		if (!cma_comp_exch(id_priv, RDMA_CM_CONNECT,
				   RDMA_CM_DISCONNECT))
1099
			goto out;
1100
		event.event = RDMA_CM_EVENT_DISCONNECTED;
1101 1102
		break;
	case IB_CM_TIMEWAIT_EXIT:
1103 1104
		event.event = RDMA_CM_EVENT_TIMEWAIT_EXIT;
		break;
1105 1106 1107 1108
	case IB_CM_MRA_RECEIVED:
		/* ignore event */
		goto out;
	case IB_CM_REJ_RECEIVED:
1109
		cma_modify_qp_err(id_priv);
1110 1111 1112 1113
		event.status = ib_event->param.rej_rcvd.reason;
		event.event = RDMA_CM_EVENT_REJECTED;
		event.param.conn.private_data = ib_event->private_data;
		event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE;
1114 1115
		break;
	default:
1116
		printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n",
1117 1118 1119 1120
		       ib_event->event);
		goto out;
	}

1121
	ret = id_priv->id.event_handler(&id_priv->id, &event);
1122 1123 1124
	if (ret) {
		/* Destroy the CM ID by returning a non-zero value. */
		id_priv->cm_id.ib = NULL;
1125
		cma_exch(id_priv, RDMA_CM_DESTROYING);
1126
		mutex_unlock(&id_priv->handler_mutex);
1127 1128 1129 1130
		rdma_destroy_id(&id_priv->id);
		return ret;
	}
out:
1131
	mutex_unlock(&id_priv->handler_mutex);
1132 1133 1134
	return ret;
}

S
Sean Hefty 已提交
1135 1136
static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id,
					       struct ib_cm_event *ib_event)
1137 1138 1139 1140 1141
{
	struct rdma_id_private *id_priv;
	struct rdma_cm_id *id;
	struct rdma_route *rt;
	union cma_ip_addr *src, *dst;
1142
	__be16 port;
1143
	u8 ip_ver;
1144
	int ret;
1145

K
Krishna Kumar 已提交
1146 1147
	if (cma_get_net_info(ib_event->private_data, listen_id->ps,
			     &ip_ver, &port, &src, &dst))
1148
		return NULL;
K
Krishna Kumar 已提交
1149

1150
	id = rdma_create_id(listen_id->event_handler, listen_id->context,
1151
			    listen_id->ps, ib_event->param.req_rcvd.qp_type);
1152
	if (IS_ERR(id))
1153
		return NULL;
K
Krishna Kumar 已提交
1154

1155
	id_priv = container_of(id, struct rdma_id_private, id);
K
Krishna Kumar 已提交
1156 1157
	cma_save_net_info(&id->route.addr, &listen_id->route.addr,
			  ip_ver, port, src, dst);
1158 1159 1160

	rt = &id->route;
	rt->num_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1;
K
Krishna Kumar 已提交
1161 1162
	rt->path_rec = kmalloc(sizeof *rt->path_rec * rt->num_paths,
			       GFP_KERNEL);
1163
	if (!rt->path_rec)
1164
		goto err;
1165 1166 1167 1168 1169

	rt->path_rec[0] = *ib_event->param.req_rcvd.primary_path;
	if (rt->num_paths == 2)
		rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path;

1170
	if (cma_any_addr(cma_src_addr(id_priv))) {
S
Sean Hefty 已提交
1171 1172
		rt->addr.dev_addr.dev_type = ARPHRD_INFINIBAND;
		rdma_addr_set_sgid(&rt->addr.dev_addr, &rt->path_rec[0].sgid);
S
Sean Hefty 已提交
1173
		ib_addr_set_pkey(&rt->addr.dev_addr, be16_to_cpu(rt->path_rec[0].pkey));
S
Sean Hefty 已提交
1174
	} else {
1175
		ret = cma_translate_addr(cma_src_addr(id_priv), &rt->addr.dev_addr);
S
Sean Hefty 已提交
1176
		if (ret)
1177
			goto err;
S
Sean Hefty 已提交
1178 1179
	}
	rdma_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid);
1180

1181
	id_priv->state = RDMA_CM_CONNECT;
1182
	return id_priv;
K
Krishna Kumar 已提交
1183 1184

err:
1185
	rdma_destroy_id(id);
1186 1187 1188
	return NULL;
}

S
Sean Hefty 已提交
1189 1190 1191 1192 1193 1194
static struct rdma_id_private *cma_new_udp_id(struct rdma_cm_id *listen_id,
					      struct ib_cm_event *ib_event)
{
	struct rdma_id_private *id_priv;
	struct rdma_cm_id *id;
	union cma_ip_addr *src, *dst;
1195
	__be16 port;
S
Sean Hefty 已提交
1196 1197 1198 1199
	u8 ip_ver;
	int ret;

	id = rdma_create_id(listen_id->event_handler, listen_id->context,
1200
			    listen_id->ps, IB_QPT_UD);
S
Sean Hefty 已提交
1201 1202 1203
	if (IS_ERR(id))
		return NULL;

1204
	id_priv = container_of(id, struct rdma_id_private, id);
S
Sean Hefty 已提交
1205 1206 1207 1208 1209 1210 1211
	if (cma_get_net_info(ib_event->private_data, listen_id->ps,
			     &ip_ver, &port, &src, &dst))
		goto err;

	cma_save_net_info(&id->route.addr, &listen_id->route.addr,
			  ip_ver, port, src, dst);

S
Sean Hefty 已提交
1212
	if (!cma_any_addr((struct sockaddr *) &id->route.addr.src_addr)) {
1213
		ret = cma_translate_addr(cma_src_addr(id_priv), &id->route.addr.dev_addr);
S
Sean Hefty 已提交
1214 1215 1216
		if (ret)
			goto err;
	}
S
Sean Hefty 已提交
1217

1218
	id_priv->state = RDMA_CM_CONNECT;
S
Sean Hefty 已提交
1219 1220 1221 1222 1223 1224
	return id_priv;
err:
	rdma_destroy_id(id);
	return NULL;
}

1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239
static void cma_set_req_event_data(struct rdma_cm_event *event,
				   struct ib_cm_req_event_param *req_data,
				   void *private_data, int offset)
{
	event->param.conn.private_data = private_data + offset;
	event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset;
	event->param.conn.responder_resources = req_data->responder_resources;
	event->param.conn.initiator_depth = req_data->initiator_depth;
	event->param.conn.flow_control = req_data->flow_control;
	event->param.conn.retry_count = req_data->retry_count;
	event->param.conn.rnr_retry_count = req_data->rnr_retry_count;
	event->param.conn.srq = req_data->srq;
	event->param.conn.qp_num = req_data->remote_qpn;
}

1240 1241
static int cma_check_req_qp_type(struct rdma_cm_id *id, struct ib_cm_event *ib_event)
{
1242
	return (((ib_event->event == IB_CM_REQ_RECEIVED) &&
1243 1244 1245 1246 1247 1248
		 (ib_event->param.req_rcvd.qp_type == id->qp_type)) ||
		((ib_event->event == IB_CM_SIDR_REQ_RECEIVED) &&
		 (id->qp_type == IB_QPT_UD)) ||
		(!id->qp_type));
}

1249 1250 1251
static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
{
	struct rdma_id_private *listen_id, *conn_id;
1252
	struct rdma_cm_event event;
1253 1254 1255
	int offset, ret;

	listen_id = cm_id->context;
1256 1257 1258
	if (!cma_check_req_qp_type(&listen_id->id, ib_event))
		return -EINVAL;

1259
	if (cma_disable_callback(listen_id, RDMA_CM_LISTEN))
1260
		return -ECONNABORTED;
1261

S
Sean Hefty 已提交
1262 1263 1264
	memset(&event, 0, sizeof event);
	offset = cma_user_data_offset(listen_id->id.ps);
	event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
1265
	if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED) {
S
Sean Hefty 已提交
1266 1267 1268 1269 1270 1271 1272 1273 1274
		conn_id = cma_new_udp_id(&listen_id->id, ib_event);
		event.param.ud.private_data = ib_event->private_data + offset;
		event.param.ud.private_data_len =
				IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset;
	} else {
		conn_id = cma_new_conn_id(&listen_id->id, ib_event);
		cma_set_req_event_data(&event, &ib_event->param.req_rcvd,
				       ib_event->private_data, offset);
	}
1275 1276
	if (!conn_id) {
		ret = -ENOMEM;
1277
		goto err1;
1278 1279
	}

1280
	mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
T
Tom Tucker 已提交
1281
	ret = cma_acquire_dev(conn_id);
1282
	if (ret)
1283
		goto err2;
1284 1285 1286 1287 1288

	conn_id->cm_id.ib = cm_id;
	cm_id->context = conn_id;
	cm_id->cm_handler = cma_ib_handler;

1289 1290 1291 1292 1293
	/*
	 * Protect against the user destroying conn_id from another thread
	 * until we're done accessing it.
	 */
	atomic_inc(&conn_id->refcount);
1294
	ret = conn_id->id.event_handler(&conn_id->id, &event);
1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307
	if (ret)
		goto err3;

	/*
	 * Acquire mutex to prevent user executing rdma_destroy_id()
	 * while we're accessing the cm_id.
	 */
	mutex_lock(&lock);
	if (cma_comp(conn_id, RDMA_CM_CONNECT) && (conn_id->id.qp_type != IB_QPT_UD))
		ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);
	mutex_unlock(&lock);
	mutex_unlock(&conn_id->handler_mutex);
	mutex_unlock(&listen_id->handler_mutex);
1308
	cma_deref_id(conn_id);
1309
	return 0;
1310

1311 1312
err3:
	cma_deref_id(conn_id);
1313 1314
	/* Destroy the CM ID by returning a non-zero value. */
	conn_id->cm_id.ib = NULL;
1315
err2:
1316
	cma_exch(conn_id, RDMA_CM_DESTROYING);
1317
	mutex_unlock(&conn_id->handler_mutex);
1318
err1:
1319
	mutex_unlock(&listen_id->handler_mutex);
1320 1321
	if (conn_id)
		rdma_destroy_id(&conn_id->id);
1322 1323 1324 1325 1326
	return ret;
}

static __be64 cma_get_service_id(enum rdma_port_space ps, struct sockaddr *addr)
{
S
Sean Hefty 已提交
1327
	return cpu_to_be64(((u64)ps << 16) + be16_to_cpu(cma_port(addr)));
1328 1329 1330 1331 1332 1333 1334
}

static void cma_set_compare_data(enum rdma_port_space ps, struct sockaddr *addr,
				 struct ib_cm_compare_data *compare)
{
	struct cma_hdr *cma_data, *cma_mask;
	struct sdp_hh *sdp_data, *sdp_mask;
1335
	__be32 ip4_addr;
1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350
	struct in6_addr ip6_addr;

	memset(compare, 0, sizeof *compare);
	cma_data = (void *) compare->data;
	cma_mask = (void *) compare->mask;
	sdp_data = (void *) compare->data;
	sdp_mask = (void *) compare->mask;

	switch (addr->sa_family) {
	case AF_INET:
		ip4_addr = ((struct sockaddr_in *) addr)->sin_addr.s_addr;
		if (ps == RDMA_PS_SDP) {
			sdp_set_ip_ver(sdp_data, 4);
			sdp_set_ip_ver(sdp_mask, 0xF);
			sdp_data->dst_addr.ip4.addr = ip4_addr;
1351
			sdp_mask->dst_addr.ip4.addr = htonl(~0);
1352 1353 1354
		} else {
			cma_set_ip_ver(cma_data, 4);
			cma_set_ip_ver(cma_mask, 0xF);
1355 1356 1357 1358
			if (!cma_any_addr(addr)) {
				cma_data->dst_addr.ip4.addr = ip4_addr;
				cma_mask->dst_addr.ip4.addr = htonl(~0);
			}
1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371
		}
		break;
	case AF_INET6:
		ip6_addr = ((struct sockaddr_in6 *) addr)->sin6_addr;
		if (ps == RDMA_PS_SDP) {
			sdp_set_ip_ver(sdp_data, 6);
			sdp_set_ip_ver(sdp_mask, 0xF);
			sdp_data->dst_addr.ip6 = ip6_addr;
			memset(&sdp_mask->dst_addr.ip6, 0xFF,
			       sizeof sdp_mask->dst_addr.ip6);
		} else {
			cma_set_ip_ver(cma_data, 6);
			cma_set_ip_ver(cma_mask, 0xF);
1372 1373 1374 1375 1376
			if (!cma_any_addr(addr)) {
				cma_data->dst_addr.ip6 = ip6_addr;
				memset(&cma_mask->dst_addr.ip6, 0xFF,
				       sizeof cma_mask->dst_addr.ip6);
			}
1377 1378 1379 1380 1381 1382 1383
		}
		break;
	default:
		break;
	}
}

T
Tom Tucker 已提交
1384 1385 1386
static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event)
{
	struct rdma_id_private *id_priv = iw_id->context;
1387
	struct rdma_cm_event event;
T
Tom Tucker 已提交
1388 1389 1390
	struct sockaddr_in *sin;
	int ret = 0;

1391
	if (cma_disable_callback(id_priv, RDMA_CM_CONNECT))
1392
		return 0;
T
Tom Tucker 已提交
1393

1394
	memset(&event, 0, sizeof event);
T
Tom Tucker 已提交
1395 1396
	switch (iw_event->event) {
	case IW_CM_EVENT_CLOSE:
1397
		event.event = RDMA_CM_EVENT_DISCONNECTED;
T
Tom Tucker 已提交
1398 1399
		break;
	case IW_CM_EVENT_CONNECT_REPLY:
1400
		sin = (struct sockaddr_in *) cma_src_addr(id_priv);
T
Tom Tucker 已提交
1401
		*sin = iw_event->local_addr;
1402
		sin = (struct sockaddr_in *) cma_dst_addr(id_priv);
T
Tom Tucker 已提交
1403
		*sin = iw_event->remote_addr;
1404 1405
		switch (iw_event->status) {
		case 0:
1406
			event.event = RDMA_CM_EVENT_ESTABLISHED;
1407 1408
			event.param.conn.initiator_depth = iw_event->ird;
			event.param.conn.responder_resources = iw_event->ord;
1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420
			break;
		case -ECONNRESET:
		case -ECONNREFUSED:
			event.event = RDMA_CM_EVENT_REJECTED;
			break;
		case -ETIMEDOUT:
			event.event = RDMA_CM_EVENT_UNREACHABLE;
			break;
		default:
			event.event = RDMA_CM_EVENT_CONNECT_ERROR;
			break;
		}
T
Tom Tucker 已提交
1421 1422
		break;
	case IW_CM_EVENT_ESTABLISHED:
1423
		event.event = RDMA_CM_EVENT_ESTABLISHED;
1424 1425
		event.param.conn.initiator_depth = iw_event->ird;
		event.param.conn.responder_resources = iw_event->ord;
T
Tom Tucker 已提交
1426 1427 1428 1429 1430
		break;
	default:
		BUG_ON(1);
	}

1431 1432 1433 1434
	event.status = iw_event->status;
	event.param.conn.private_data = iw_event->private_data;
	event.param.conn.private_data_len = iw_event->private_data_len;
	ret = id_priv->id.event_handler(&id_priv->id, &event);
T
Tom Tucker 已提交
1435 1436 1437
	if (ret) {
		/* Destroy the CM ID by returning a non-zero value. */
		id_priv->cm_id.iw = NULL;
1438
		cma_exch(id_priv, RDMA_CM_DESTROYING);
1439
		mutex_unlock(&id_priv->handler_mutex);
T
Tom Tucker 已提交
1440 1441 1442 1443
		rdma_destroy_id(&id_priv->id);
		return ret;
	}

1444
	mutex_unlock(&id_priv->handler_mutex);
T
Tom Tucker 已提交
1445 1446 1447 1448 1449 1450 1451 1452 1453 1454
	return ret;
}

static int iw_conn_req_handler(struct iw_cm_id *cm_id,
			       struct iw_cm_event *iw_event)
{
	struct rdma_cm_id *new_cm_id;
	struct rdma_id_private *listen_id, *conn_id;
	struct sockaddr_in *sin;
	struct net_device *dev = NULL;
1455
	struct rdma_cm_event event;
T
Tom Tucker 已提交
1456
	int ret;
1457
	struct ib_device_attr attr;
T
Tom Tucker 已提交
1458 1459

	listen_id = cm_id->context;
1460
	if (cma_disable_callback(listen_id, RDMA_CM_LISTEN))
1461
		return -ECONNABORTED;
T
Tom Tucker 已提交
1462 1463 1464 1465

	/* Create a new RDMA id for the new IW CM ID */
	new_cm_id = rdma_create_id(listen_id->id.event_handler,
				   listen_id->id.context,
1466
				   RDMA_PS_TCP, IB_QPT_RC);
1467
	if (IS_ERR(new_cm_id)) {
T
Tom Tucker 已提交
1468 1469 1470 1471
		ret = -ENOMEM;
		goto out;
	}
	conn_id = container_of(new_cm_id, struct rdma_id_private, id);
1472
	mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
1473
	conn_id->state = RDMA_CM_CONNECT;
T
Tom Tucker 已提交
1474

1475
	dev = ip_dev_find(&init_net, iw_event->local_addr.sin_addr.s_addr);
T
Tom Tucker 已提交
1476 1477
	if (!dev) {
		ret = -EADDRNOTAVAIL;
1478
		mutex_unlock(&conn_id->handler_mutex);
T
Tom Tucker 已提交
1479 1480 1481 1482 1483
		rdma_destroy_id(new_cm_id);
		goto out;
	}
	ret = rdma_copy_addr(&conn_id->id.route.addr.dev_addr, dev, NULL);
	if (ret) {
1484
		mutex_unlock(&conn_id->handler_mutex);
T
Tom Tucker 已提交
1485 1486 1487 1488 1489 1490
		rdma_destroy_id(new_cm_id);
		goto out;
	}

	ret = cma_acquire_dev(conn_id);
	if (ret) {
1491
		mutex_unlock(&conn_id->handler_mutex);
T
Tom Tucker 已提交
1492 1493 1494 1495 1496 1497 1498 1499
		rdma_destroy_id(new_cm_id);
		goto out;
	}

	conn_id->cm_id.iw = cm_id;
	cm_id->context = conn_id;
	cm_id->cm_handler = cma_iw_handler;

1500
	sin = (struct sockaddr_in *) cma_src_addr(conn_id);
T
Tom Tucker 已提交
1501
	*sin = iw_event->local_addr;
1502
	sin = (struct sockaddr_in *) cma_dst_addr(conn_id);
T
Tom Tucker 已提交
1503 1504
	*sin = iw_event->remote_addr;

1505 1506
	ret = ib_query_device(conn_id->id.device, &attr);
	if (ret) {
1507
		mutex_unlock(&conn_id->handler_mutex);
1508 1509 1510 1511
		rdma_destroy_id(new_cm_id);
		goto out;
	}

1512 1513 1514 1515
	memset(&event, 0, sizeof event);
	event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
	event.param.conn.private_data = iw_event->private_data;
	event.param.conn.private_data_len = iw_event->private_data_len;
1516 1517
	event.param.conn.initiator_depth = iw_event->ird;
	event.param.conn.responder_resources = iw_event->ord;
1518 1519 1520 1521 1522 1523

	/*
	 * Protect against the user destroying conn_id from another thread
	 * until we're done accessing it.
	 */
	atomic_inc(&conn_id->refcount);
1524
	ret = conn_id->id.event_handler(&conn_id->id, &event);
T
Tom Tucker 已提交
1525 1526 1527
	if (ret) {
		/* User wants to destroy the CM ID */
		conn_id->cm_id.iw = NULL;
1528
		cma_exch(conn_id, RDMA_CM_DESTROYING);
1529
		mutex_unlock(&conn_id->handler_mutex);
1530
		cma_deref_id(conn_id);
T
Tom Tucker 已提交
1531
		rdma_destroy_id(&conn_id->id);
1532
		goto out;
T
Tom Tucker 已提交
1533 1534
	}

1535
	mutex_unlock(&conn_id->handler_mutex);
1536
	cma_deref_id(conn_id);
1537

T
Tom Tucker 已提交
1538 1539 1540
out:
	if (dev)
		dev_put(dev);
1541
	mutex_unlock(&listen_id->handler_mutex);
T
Tom Tucker 已提交
1542 1543 1544
	return ret;
}

1545 1546 1547 1548
static int cma_ib_listen(struct rdma_id_private *id_priv)
{
	struct ib_cm_compare_data compare_data;
	struct sockaddr *addr;
1549
	struct ib_cm_id	*id;
1550 1551 1552
	__be64 svc_id;
	int ret;

1553 1554 1555 1556 1557
	id = ib_create_cm_id(id_priv->id.device, cma_req_handler, id_priv);
	if (IS_ERR(id))
		return PTR_ERR(id);

	id_priv->cm_id.ib = id;
1558

1559
	addr = cma_src_addr(id_priv);
1560
	svc_id = cma_get_service_id(id_priv->id.ps, addr);
1561
	if (cma_any_addr(addr) && !id_priv->afonly)
1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575
		ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, NULL);
	else {
		cma_set_compare_data(id_priv->id.ps, addr, &compare_data);
		ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, &compare_data);
	}

	if (ret) {
		ib_destroy_cm_id(id_priv->cm_id.ib);
		id_priv->cm_id.ib = NULL;
	}

	return ret;
}

T
Tom Tucker 已提交
1576 1577 1578 1579
static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog)
{
	int ret;
	struct sockaddr_in *sin;
1580 1581 1582 1583 1584 1585 1586
	struct iw_cm_id	*id;

	id = iw_create_cm_id(id_priv->id.device,
			     iw_conn_req_handler,
			     id_priv);
	if (IS_ERR(id))
		return PTR_ERR(id);
T
Tom Tucker 已提交
1587

1588
	id_priv->cm_id.iw = id;
T
Tom Tucker 已提交
1589

1590
	sin = (struct sockaddr_in *) cma_src_addr(id_priv);
T
Tom Tucker 已提交
1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602
	id_priv->cm_id.iw->local_addr = *sin;

	ret = iw_cm_listen(id_priv->cm_id.iw, backlog);

	if (ret) {
		iw_destroy_cm_id(id_priv->cm_id.iw);
		id_priv->cm_id.iw = NULL;
	}

	return ret;
}

1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619
static int cma_listen_handler(struct rdma_cm_id *id,
			      struct rdma_cm_event *event)
{
	struct rdma_id_private *id_priv = id->context;

	id->context = id_priv->id.context;
	id->event_handler = id_priv->id.event_handler;
	return id_priv->id.event_handler(id, event);
}

static void cma_listen_on_dev(struct rdma_id_private *id_priv,
			      struct cma_device *cma_dev)
{
	struct rdma_id_private *dev_id_priv;
	struct rdma_cm_id *id;
	int ret;

1620 1621
	id = rdma_create_id(cma_listen_handler, id_priv, id_priv->id.ps,
			    id_priv->id.qp_type);
1622 1623 1624 1625 1626
	if (IS_ERR(id))
		return;

	dev_id_priv = container_of(id, struct rdma_id_private, id);

1627
	dev_id_priv->state = RDMA_CM_ADDR_BOUND;
1628 1629
	memcpy(cma_src_addr(dev_id_priv), cma_src_addr(id_priv),
	       rdma_addr_size(cma_src_addr(id_priv)));
1630 1631 1632

	cma_attach_to_dev(dev_id_priv, cma_dev);
	list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list);
1633 1634
	atomic_inc(&id_priv->refcount);
	dev_id_priv->internal_id = 1;
1635
	dev_id_priv->afonly = id_priv->afonly;
1636 1637 1638

	ret = rdma_listen(id, id_priv->backlog);
	if (ret)
1639
		printk(KERN_WARNING "RDMA CMA: cma_listen_on_dev, error %d, "
1640
		       "listening on device %s\n", ret, cma_dev->device->name);
1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653
}

static void cma_listen_on_all(struct rdma_id_private *id_priv)
{
	struct cma_device *cma_dev;

	mutex_lock(&lock);
	list_add_tail(&id_priv->list, &listen_any_list);
	list_for_each_entry(cma_dev, &dev_list, list)
		cma_listen_on_dev(id_priv, cma_dev);
	mutex_unlock(&lock);
}

1654 1655 1656 1657 1658 1659 1660 1661 1662
void rdma_set_service_type(struct rdma_cm_id *id, int tos)
{
	struct rdma_id_private *id_priv;

	id_priv = container_of(id, struct rdma_id_private, id);
	id_priv->tos = (u8) tos;
}
EXPORT_SYMBOL(rdma_set_service_type);

1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674
static void cma_query_handler(int status, struct ib_sa_path_rec *path_rec,
			      void *context)
{
	struct cma_work *work = context;
	struct rdma_route *route;

	route = &work->id->id.route;

	if (!status) {
		route->num_paths = 1;
		*route->path_rec = *path_rec;
	} else {
1675 1676
		work->old_state = RDMA_CM_ROUTE_QUERY;
		work->new_state = RDMA_CM_ADDR_RESOLVED;
1677
		work->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
1678
		work->event.status = status;
1679 1680 1681 1682 1683 1684 1685 1686
	}

	queue_work(cma_wq, &work->work);
}

static int cma_query_ib_route(struct rdma_id_private *id_priv, int timeout_ms,
			      struct cma_work *work)
{
1687
	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
1688
	struct ib_sa_path_rec path_rec;
1689 1690
	ib_sa_comp_mask comp_mask;
	struct sockaddr_in6 *sin6;
1691 1692

	memset(&path_rec, 0, sizeof path_rec);
1693 1694 1695
	rdma_addr_get_sgid(dev_addr, &path_rec.sgid);
	rdma_addr_get_dgid(dev_addr, &path_rec.dgid);
	path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
1696
	path_rec.numb_path = 1;
1697
	path_rec.reversible = 1;
1698
	path_rec.service_id = cma_get_service_id(id_priv->id.ps, cma_dst_addr(id_priv));
1699 1700 1701 1702 1703

	comp_mask = IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID |
		    IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH |
		    IB_SA_PATH_REC_REVERSIBLE | IB_SA_PATH_REC_SERVICE_ID;

1704
	if (cma_family(id_priv) == AF_INET) {
1705 1706 1707
		path_rec.qos_class = cpu_to_be16((u16) id_priv->tos);
		comp_mask |= IB_SA_PATH_REC_QOS_CLASS;
	} else {
1708
		sin6 = (struct sockaddr_in6 *) cma_src_addr(id_priv);
1709 1710 1711
		path_rec.traffic_class = (u8) (be32_to_cpu(sin6->sin6_flowinfo) >> 20);
		comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS;
	}
1712

1713
	id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device,
1714 1715 1716 1717
					       id_priv->id.port_num, &path_rec,
					       comp_mask, timeout_ms,
					       GFP_KERNEL, cma_query_handler,
					       work, &id_priv->query);
1718 1719 1720 1721

	return (id_priv->query_id < 0) ? id_priv->query_id : 0;
}

D
David Howells 已提交
1722
static void cma_work_handler(struct work_struct *_work)
1723
{
D
David Howells 已提交
1724
	struct cma_work *work = container_of(_work, struct cma_work, work);
1725 1726 1727
	struct rdma_id_private *id_priv = work->id;
	int destroy = 0;

1728
	mutex_lock(&id_priv->handler_mutex);
1729 1730 1731 1732
	if (!cma_comp_exch(id_priv, work->old_state, work->new_state))
		goto out;

	if (id_priv->id.event_handler(&id_priv->id, &work->event)) {
1733
		cma_exch(id_priv, RDMA_CM_DESTROYING);
1734 1735 1736
		destroy = 1;
	}
out:
1737
	mutex_unlock(&id_priv->handler_mutex);
1738 1739 1740 1741 1742 1743
	cma_deref_id(id_priv);
	if (destroy)
		rdma_destroy_id(&id_priv->id);
	kfree(work);
}

1744 1745 1746 1747 1748 1749 1750
static void cma_ndev_work_handler(struct work_struct *_work)
{
	struct cma_ndev_work *work = container_of(_work, struct cma_ndev_work, work);
	struct rdma_id_private *id_priv = work->id;
	int destroy = 0;

	mutex_lock(&id_priv->handler_mutex);
1751 1752
	if (id_priv->state == RDMA_CM_DESTROYING ||
	    id_priv->state == RDMA_CM_DEVICE_REMOVAL)
1753 1754 1755
		goto out;

	if (id_priv->id.event_handler(&id_priv->id, &work->event)) {
1756
		cma_exch(id_priv, RDMA_CM_DESTROYING);
1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767
		destroy = 1;
	}

out:
	mutex_unlock(&id_priv->handler_mutex);
	cma_deref_id(id_priv);
	if (destroy)
		rdma_destroy_id(&id_priv->id);
	kfree(work);
}

1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778
static int cma_resolve_ib_route(struct rdma_id_private *id_priv, int timeout_ms)
{
	struct rdma_route *route = &id_priv->id.route;
	struct cma_work *work;
	int ret;

	work = kzalloc(sizeof *work, GFP_KERNEL);
	if (!work)
		return -ENOMEM;

	work->id = id_priv;
D
David Howells 已提交
1779
	INIT_WORK(&work->work, cma_work_handler);
1780 1781
	work->old_state = RDMA_CM_ROUTE_QUERY;
	work->new_state = RDMA_CM_ROUTE_RESOLVED;
1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809
	work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;

	route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL);
	if (!route->path_rec) {
		ret = -ENOMEM;
		goto err1;
	}

	ret = cma_query_ib_route(id_priv, timeout_ms, work);
	if (ret)
		goto err2;

	return 0;
err2:
	kfree(route->path_rec);
	route->path_rec = NULL;
err1:
	kfree(work);
	return ret;
}

int rdma_set_ib_paths(struct rdma_cm_id *id,
		      struct ib_sa_path_rec *path_rec, int num_paths)
{
	struct rdma_id_private *id_priv;
	int ret;

	id_priv = container_of(id, struct rdma_id_private, id);
1810 1811
	if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
			   RDMA_CM_ROUTE_RESOLVED))
1812 1813
		return -EINVAL;

1814 1815
	id->route.path_rec = kmemdup(path_rec, sizeof *path_rec * num_paths,
				     GFP_KERNEL);
1816 1817 1818 1819 1820
	if (!id->route.path_rec) {
		ret = -ENOMEM;
		goto err;
	}

1821
	id->route.num_paths = num_paths;
1822 1823
	return 0;
err:
1824
	cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_ADDR_RESOLVED);
1825 1826 1827 1828
	return ret;
}
EXPORT_SYMBOL(rdma_set_ib_paths);

T
Tom Tucker 已提交
1829 1830 1831 1832 1833 1834 1835 1836 1837
static int cma_resolve_iw_route(struct rdma_id_private *id_priv, int timeout_ms)
{
	struct cma_work *work;

	work = kzalloc(sizeof *work, GFP_KERNEL);
	if (!work)
		return -ENOMEM;

	work->id = id_priv;
D
David Howells 已提交
1838
	INIT_WORK(&work->work, cma_work_handler);
1839 1840
	work->old_state = RDMA_CM_ROUTE_QUERY;
	work->new_state = RDMA_CM_ROUTE_RESOLVED;
T
Tom Tucker 已提交
1841 1842 1843 1844 1845
	work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
	queue_work(cma_wq, &work->work);
	return 0;
}

1846 1847 1848 1849 1850 1851 1852
static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)
{
	struct rdma_route *route = &id_priv->id.route;
	struct rdma_addr *addr = &route->addr;
	struct cma_work *work;
	int ret;
	struct net_device *ndev = NULL;
E
Eli Cohen 已提交
1853
	u16 vid;
1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876

	work = kzalloc(sizeof *work, GFP_KERNEL);
	if (!work)
		return -ENOMEM;

	work->id = id_priv;
	INIT_WORK(&work->work, cma_work_handler);

	route->path_rec = kzalloc(sizeof *route->path_rec, GFP_KERNEL);
	if (!route->path_rec) {
		ret = -ENOMEM;
		goto err1;
	}

	route->num_paths = 1;

	if (addr->dev_addr.bound_dev_if)
		ndev = dev_get_by_index(&init_net, addr->dev_addr.bound_dev_if);
	if (!ndev) {
		ret = -ENODEV;
		goto err2;
	}

E
Eli Cohen 已提交
1877 1878 1879 1880 1881 1882 1883 1884 1885
	vid = rdma_vlan_dev_vlan_id(ndev);

	iboe_mac_vlan_to_ll(&route->path_rec->sgid, addr->dev_addr.src_dev_addr, vid);
	iboe_mac_vlan_to_ll(&route->path_rec->dgid, addr->dev_addr.dst_dev_addr, vid);

	route->path_rec->hop_limit = 1;
	route->path_rec->reversible = 1;
	route->path_rec->pkey = cpu_to_be16(0xffff);
	route->path_rec->mtu_selector = IB_SA_EQ;
1886 1887 1888 1889
	route->path_rec->sl = netdev_get_prio_tc_map(
			ndev->priv_flags & IFF_802_1Q_VLAN ?
				vlan_dev_real_dev(ndev) : ndev,
			rt_tos2priority(id_priv->tos));
E
Eli Cohen 已提交
1890

1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901
	route->path_rec->mtu = iboe_get_mtu(ndev->mtu);
	route->path_rec->rate_selector = IB_SA_EQ;
	route->path_rec->rate = iboe_get_rate(ndev);
	dev_put(ndev);
	route->path_rec->packet_life_time_selector = IB_SA_EQ;
	route->path_rec->packet_life_time = CMA_IBOE_PACKET_LIFETIME;
	if (!route->path_rec->mtu) {
		ret = -EINVAL;
		goto err2;
	}

1902 1903
	work->old_state = RDMA_CM_ROUTE_QUERY;
	work->new_state = RDMA_CM_ROUTE_RESOLVED;
1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918
	work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
	work->event.status = 0;

	queue_work(cma_wq, &work->work);

	return 0;

err2:
	kfree(route->path_rec);
	route->path_rec = NULL;
err1:
	kfree(work);
	return ret;
}

1919 1920 1921 1922 1923 1924
int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
{
	struct rdma_id_private *id_priv;
	int ret;

	id_priv = container_of(id, struct rdma_id_private, id);
1925
	if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED, RDMA_CM_ROUTE_QUERY))
1926 1927 1928
		return -EINVAL;

	atomic_inc(&id_priv->refcount);
T
Tom Tucker 已提交
1929 1930
	switch (rdma_node_get_transport(id->device->node_type)) {
	case RDMA_TRANSPORT_IB:
1931 1932 1933 1934 1935 1936 1937 1938 1939 1940
		switch (rdma_port_get_link_layer(id->device, id->port_num)) {
		case IB_LINK_LAYER_INFINIBAND:
			ret = cma_resolve_ib_route(id_priv, timeout_ms);
			break;
		case IB_LINK_LAYER_ETHERNET:
			ret = cma_resolve_iboe_route(id_priv);
			break;
		default:
			ret = -ENOSYS;
		}
1941
		break;
T
Tom Tucker 已提交
1942 1943 1944
	case RDMA_TRANSPORT_IWARP:
		ret = cma_resolve_iw_route(id_priv, timeout_ms);
		break;
1945 1946 1947 1948 1949 1950 1951 1952 1953
	default:
		ret = -ENOSYS;
		break;
	}
	if (ret)
		goto err;

	return 0;
err:
1954
	cma_comp_exch(id_priv, RDMA_CM_ROUTE_QUERY, RDMA_CM_ADDR_RESOLVED);
1955 1956 1957 1958 1959
	cma_deref_id(id_priv);
	return ret;
}
EXPORT_SYMBOL(rdma_resolve_route);

1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976
static void cma_set_loopback(struct sockaddr *addr)
{
	switch (addr->sa_family) {
	case AF_INET:
		((struct sockaddr_in *) addr)->sin_addr.s_addr = htonl(INADDR_LOOPBACK);
		break;
	case AF_INET6:
		ipv6_addr_set(&((struct sockaddr_in6 *) addr)->sin6_addr,
			      0, 0, 0, htonl(1));
		break;
	default:
		ib_addr_set(&((struct sockaddr_ib *) addr)->sib_addr,
			    0, 0, 0, htonl(1));
		break;
	}
}

1977 1978
static int cma_bind_loopback(struct rdma_id_private *id_priv)
{
1979
	struct cma_device *cma_dev, *cur_dev;
1980
	struct ib_port_attr port_attr;
1981
	union ib_gid gid;
1982 1983 1984 1985
	u16 pkey;
	int ret;
	u8 p;

1986
	cma_dev = NULL;
1987
	mutex_lock(&lock);
1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005
	list_for_each_entry(cur_dev, &dev_list, list) {
		if (cma_family(id_priv) == AF_IB &&
		    rdma_node_get_transport(cur_dev->device->node_type) != RDMA_TRANSPORT_IB)
			continue;

		if (!cma_dev)
			cma_dev = cur_dev;

		for (p = 1; p <= cur_dev->device->phys_port_cnt; ++p) {
			if (!ib_query_port(cur_dev->device, p, &port_attr) &&
			    port_attr.state == IB_PORT_ACTIVE) {
				cma_dev = cur_dev;
				goto port_found;
			}
		}
	}

	if (!cma_dev) {
2006 2007 2008
		ret = -ENODEV;
		goto out;
	}
2009

2010
	p = 1;
2011 2012

port_found:
2013
	ret = ib_get_cached_gid(cma_dev->device, p, 0, &gid);
2014 2015 2016 2017 2018 2019 2020
	if (ret)
		goto out;

	ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey);
	if (ret)
		goto out;

S
Sean Hefty 已提交
2021
	id_priv->id.route.addr.dev_addr.dev_type =
2022
		(rdma_port_get_link_layer(cma_dev->device, p) == IB_LINK_LAYER_INFINIBAND) ?
S
Sean Hefty 已提交
2023 2024 2025
		ARPHRD_INFINIBAND : ARPHRD_ETHER;

	rdma_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid);
2026 2027 2028
	ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey);
	id_priv->id.port_num = p;
	cma_attach_to_dev(id_priv, cma_dev);
2029
	cma_set_loopback(cma_src_addr(id_priv));
2030 2031 2032 2033 2034 2035 2036 2037 2038
out:
	mutex_unlock(&lock);
	return ret;
}

static void addr_handler(int status, struct sockaddr *src_addr,
			 struct rdma_dev_addr *dev_addr, void *context)
{
	struct rdma_id_private *id_priv = context;
2039
	struct rdma_cm_event event;
2040

2041
	memset(&event, 0, sizeof event);
2042
	mutex_lock(&id_priv->handler_mutex);
2043 2044
	if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY,
			   RDMA_CM_ADDR_RESOLVED))
2045 2046 2047
		goto out;

	if (!status && !id_priv->cma_dev)
2048 2049 2050
		status = cma_acquire_dev(id_priv);

	if (status) {
2051 2052
		if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
				   RDMA_CM_ADDR_BOUND))
2053
			goto out;
2054 2055
		event.event = RDMA_CM_EVENT_ADDR_ERROR;
		event.status = status;
2056
	} else {
2057
		memcpy(cma_src_addr(id_priv), src_addr, rdma_addr_size(src_addr));
2058
		event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
2059 2060
	}

2061
	if (id_priv->id.event_handler(&id_priv->id, &event)) {
2062
		cma_exch(id_priv, RDMA_CM_DESTROYING);
2063
		mutex_unlock(&id_priv->handler_mutex);
2064 2065 2066 2067 2068
		cma_deref_id(id_priv);
		rdma_destroy_id(&id_priv->id);
		return;
	}
out:
2069
	mutex_unlock(&id_priv->handler_mutex);
2070 2071 2072 2073 2074 2075
	cma_deref_id(id_priv);
}

static int cma_resolve_loopback(struct rdma_id_private *id_priv)
{
	struct cma_work *work;
2076
	union ib_gid gid;
2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088
	int ret;

	work = kzalloc(sizeof *work, GFP_KERNEL);
	if (!work)
		return -ENOMEM;

	if (!id_priv->cma_dev) {
		ret = cma_bind_loopback(id_priv);
		if (ret)
			goto err;
	}

S
Sean Hefty 已提交
2089 2090
	rdma_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid);
	rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid);
2091 2092

	work->id = id_priv;
D
David Howells 已提交
2093
	INIT_WORK(&work->work, cma_work_handler);
2094 2095
	work->old_state = RDMA_CM_ADDR_QUERY;
	work->new_state = RDMA_CM_ADDR_RESOLVED;
2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106
	work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
	queue_work(cma_wq, &work->work);
	return 0;
err:
	kfree(work);
	return ret;
}

static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
			 struct sockaddr *dst_addr)
{
S
Sean Hefty 已提交
2107 2108 2109 2110 2111 2112 2113 2114
	if (!src_addr || !src_addr->sa_family) {
		src_addr = (struct sockaddr *) &id->route.addr.src_addr;
		if ((src_addr->sa_family = dst_addr->sa_family) == AF_INET6) {
			((struct sockaddr_in6 *) src_addr)->sin6_scope_id =
				((struct sockaddr_in6 *) dst_addr)->sin6_scope_id;
		}
	}
	return rdma_bind_addr(id, src_addr);
2115 2116 2117 2118 2119 2120 2121 2122 2123
}

int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
		      struct sockaddr *dst_addr, int timeout_ms)
{
	struct rdma_id_private *id_priv;
	int ret;

	id_priv = container_of(id, struct rdma_id_private, id);
2124
	if (id_priv->state == RDMA_CM_IDLE) {
2125 2126 2127 2128 2129
		ret = cma_bind_addr(id, src_addr, dst_addr);
		if (ret)
			return ret;
	}

2130 2131 2132
	if (cma_family(id_priv) != dst_addr->sa_family)
		return -EINVAL;

2133
	if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY))
2134 2135 2136
		return -EINVAL;

	atomic_inc(&id_priv->refcount);
2137
	memcpy(cma_dst_addr(id_priv), dst_addr, rdma_addr_size(dst_addr));
2138 2139 2140
	if (cma_any_addr(dst_addr))
		ret = cma_resolve_loopback(id_priv);
	else
2141
		ret = rdma_resolve_ip(&addr_client, cma_src_addr(id_priv),
2142
				      dst_addr, &id->route.addr.dev_addr,
2143 2144 2145 2146 2147 2148
				      timeout_ms, addr_handler, id_priv);
	if (ret)
		goto err;

	return 0;
err:
2149
	cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);
2150 2151 2152 2153 2154
	cma_deref_id(id_priv);
	return ret;
}
EXPORT_SYMBOL(rdma_resolve_addr);

2155 2156 2157 2158 2159 2160 2161 2162
int rdma_set_reuseaddr(struct rdma_cm_id *id, int reuse)
{
	struct rdma_id_private *id_priv;
	unsigned long flags;
	int ret;

	id_priv = container_of(id, struct rdma_id_private, id);
	spin_lock_irqsave(&id_priv->lock, flags);
2163
	if (reuse || id_priv->state == RDMA_CM_IDLE) {
2164 2165 2166 2167 2168 2169 2170 2171 2172 2173
		id_priv->reuseaddr = reuse;
		ret = 0;
	} else {
		ret = -EINVAL;
	}
	spin_unlock_irqrestore(&id_priv->lock, flags);
	return ret;
}
EXPORT_SYMBOL(rdma_set_reuseaddr);

2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193
int rdma_set_afonly(struct rdma_cm_id *id, int afonly)
{
	struct rdma_id_private *id_priv;
	unsigned long flags;
	int ret;

	id_priv = container_of(id, struct rdma_id_private, id);
	spin_lock_irqsave(&id_priv->lock, flags);
	if (id_priv->state == RDMA_CM_IDLE || id_priv->state == RDMA_CM_ADDR_BOUND) {
		id_priv->options |= (1 << CMA_OPTION_AFONLY);
		id_priv->afonly = afonly;
		ret = 0;
	} else {
		ret = -EINVAL;
	}
	spin_unlock_irqrestore(&id_priv->lock, flags);
	return ret;
}
EXPORT_SYMBOL(rdma_set_afonly);

2194 2195 2196
static void cma_bind_port(struct rdma_bind_list *bind_list,
			  struct rdma_id_private *id_priv)
{
2197 2198 2199 2200
	struct sockaddr *addr;
	struct sockaddr_ib *sib;
	u64 sid, mask;
	__be16 port;
2201

2202
	addr = cma_src_addr(id_priv);
2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219
	port = htons(bind_list->port);

	switch (addr->sa_family) {
	case AF_INET:
		((struct sockaddr_in *) addr)->sin_port = port;
		break;
	case AF_INET6:
		((struct sockaddr_in6 *) addr)->sin6_port = port;
		break;
	case AF_IB:
		sib = (struct sockaddr_ib *) addr;
		sid = be64_to_cpu(sib->sib_sid);
		mask = be64_to_cpu(sib->sib_sid_mask);
		sib->sib_sid = cpu_to_be64((sid & mask) | (u64) ntohs(port));
		sib->sib_sid_mask = cpu_to_be64(~0ULL);
		break;
	}
2220 2221 2222 2223 2224 2225 2226 2227
	id_priv->bind_list = bind_list;
	hlist_add_head(&id_priv->node, &bind_list->owners);
}

static int cma_alloc_port(struct idr *ps, struct rdma_id_private *id_priv,
			  unsigned short snum)
{
	struct rdma_bind_list *bind_list;
T
Tejun Heo 已提交
2228
	int ret;
2229

2230
	bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL);
2231 2232 2233
	if (!bind_list)
		return -ENOMEM;

T
Tejun Heo 已提交
2234 2235 2236
	ret = idr_alloc(ps, bind_list, snum, snum + 1, GFP_KERNEL);
	if (ret < 0)
		goto err;
2237 2238

	bind_list->ps = ps;
T
Tejun Heo 已提交
2239
	bind_list->port = (unsigned short)ret;
2240 2241
	cma_bind_port(bind_list, id_priv);
	return 0;
T
Tejun Heo 已提交
2242
err:
2243
	kfree(bind_list);
T
Tejun Heo 已提交
2244
	return ret == -ENOSPC ? -EADDRNOTAVAIL : ret;
2245
}
2246

2247 2248
static int cma_alloc_any_port(struct idr *ps, struct rdma_id_private *id_priv)
{
2249 2250 2251
	static unsigned int last_used_port;
	int low, high, remaining;
	unsigned int rover;
2252

2253
	inet_get_local_port_range(&low, &high);
2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267
	remaining = (high - low) + 1;
	rover = net_random() % remaining + low;
retry:
	if (last_used_port != rover &&
	    !idr_find(ps, (unsigned short) rover)) {
		int ret = cma_alloc_port(ps, id_priv, rover);
		/*
		 * Remember previously used port number in order to avoid
		 * re-using same port immediately after it is closed.
		 */
		if (!ret)
			last_used_port = rover;
		if (ret != -EADDRNOTAVAIL)
			return ret;
2268
	}
2269 2270 2271 2272 2273 2274 2275
	if (--remaining) {
		rover++;
		if ((rover < low) || (rover > high))
			rover = low;
		goto retry;
	}
	return -EADDRNOTAVAIL;
2276 2277
}

2278 2279 2280 2281 2282 2283 2284 2285
/*
 * Check that the requested port is available.  This is called when trying to
 * bind to a specific port, or when trying to listen on a bound port.  In
 * the latter case, the provided id_priv may already be on the bind_list, but
 * we still need to check that it's okay to start listening.
 */
static int cma_check_port(struct rdma_bind_list *bind_list,
			  struct rdma_id_private *id_priv, uint8_t reuseaddr)
2286 2287
{
	struct rdma_id_private *cur_id;
2288
	struct sockaddr *addr, *cur_addr;
2289

2290
	addr = cma_src_addr(id_priv);
2291
	hlist_for_each_entry(cur_id, &bind_list->owners, node) {
2292 2293
		if (id_priv == cur_id)
			continue;
R
Roland Dreier 已提交
2294

2295 2296 2297
		if ((cur_id->state != RDMA_CM_LISTEN) && reuseaddr &&
		    cur_id->reuseaddr)
			continue;
2298

2299
		cur_addr = cma_src_addr(cur_id);
2300 2301 2302 2303 2304 2305 2306 2307 2308
		if (id_priv->afonly && cur_id->afonly &&
		    (addr->sa_family != cur_addr->sa_family))
			continue;

		if (cma_any_addr(addr) || cma_any_addr(cur_addr))
			return -EADDRNOTAVAIL;

		if (!cma_addr_cmp(addr, cur_addr))
			return -EADDRINUSE;
2309
	}
2310 2311 2312
	return 0;
}

2313 2314 2315 2316 2317 2318
static int cma_use_port(struct idr *ps, struct rdma_id_private *id_priv)
{
	struct rdma_bind_list *bind_list;
	unsigned short snum;
	int ret;

2319
	snum = ntohs(cma_port(cma_src_addr(id_priv)));
2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345
	if (snum < PROT_SOCK && !capable(CAP_NET_BIND_SERVICE))
		return -EACCES;

	bind_list = idr_find(ps, snum);
	if (!bind_list) {
		ret = cma_alloc_port(ps, id_priv, snum);
	} else {
		ret = cma_check_port(bind_list, id_priv, id_priv->reuseaddr);
		if (!ret)
			cma_bind_port(bind_list, id_priv);
	}
	return ret;
}

static int cma_bind_listen(struct rdma_id_private *id_priv)
{
	struct rdma_bind_list *bind_list = id_priv->bind_list;
	int ret = 0;

	mutex_lock(&lock);
	if (bind_list->owners.first->next)
		ret = cma_check_port(bind_list, id_priv, 0);
	mutex_unlock(&lock);
	return ret;
}

2346
static struct idr *cma_select_inet_ps(struct rdma_id_private *id_priv)
2347 2348 2349
{
	switch (id_priv->id.ps) {
	case RDMA_PS_SDP:
2350
		return &sdp_ps;
2351
	case RDMA_PS_TCP:
2352
		return &tcp_ps;
S
Sean Hefty 已提交
2353
	case RDMA_PS_UDP:
2354
		return &udp_ps;
2355
	case RDMA_PS_IPOIB:
2356
		return &ipoib_ps;
2357
	case RDMA_PS_IB:
2358
		return &ib_ps;
2359
	default:
2360 2361 2362 2363 2364 2365 2366 2367 2368 2369
		return NULL;
	}
}

static struct idr *cma_select_ib_ps(struct rdma_id_private *id_priv)
{
	struct idr *ps = NULL;
	struct sockaddr_ib *sib;
	u64 sid_ps, mask, sid;

2370
	sib = (struct sockaddr_ib *) cma_src_addr(id_priv);
2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384
	mask = be64_to_cpu(sib->sib_sid_mask) & RDMA_IB_IP_PS_MASK;
	sid = be64_to_cpu(sib->sib_sid) & mask;

	if ((id_priv->id.ps == RDMA_PS_IB) && (sid == (RDMA_IB_IP_PS_IB & mask))) {
		sid_ps = RDMA_IB_IP_PS_IB;
		ps = &ib_ps;
	} else if (((id_priv->id.ps == RDMA_PS_IB) || (id_priv->id.ps == RDMA_PS_TCP)) &&
		   (sid == (RDMA_IB_IP_PS_TCP & mask))) {
		sid_ps = RDMA_IB_IP_PS_TCP;
		ps = &tcp_ps;
	} else if (((id_priv->id.ps == RDMA_PS_IB) || (id_priv->id.ps == RDMA_PS_UDP)) &&
		   (sid == (RDMA_IB_IP_PS_UDP & mask))) {
		sid_ps = RDMA_IB_IP_PS_UDP;
		ps = &udp_ps;
2385 2386
	}

2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399
	if (ps) {
		sib->sib_sid = cpu_to_be64(sid_ps | ntohs(cma_port((struct sockaddr *) sib)));
		sib->sib_sid_mask = cpu_to_be64(RDMA_IB_IP_PS_MASK |
						be64_to_cpu(sib->sib_sid_mask));
	}
	return ps;
}

static int cma_get_port(struct rdma_id_private *id_priv)
{
	struct idr *ps;
	int ret;

2400
	if (cma_family(id_priv) != AF_IB)
2401 2402 2403 2404 2405 2406
		ps = cma_select_inet_ps(id_priv);
	else
		ps = cma_select_ib_ps(id_priv);
	if (!ps)
		return -EPROTONOSUPPORT;

2407
	mutex_lock(&lock);
2408
	if (cma_any_port(cma_src_addr(id_priv)))
2409
		ret = cma_alloc_any_port(ps, id_priv);
2410 2411 2412 2413 2414 2415 2416
	else
		ret = cma_use_port(ps, id_priv);
	mutex_unlock(&lock);

	return ret;
}

S
Sean Hefty 已提交
2417 2418 2419
static int cma_check_linklocal(struct rdma_dev_addr *dev_addr,
			       struct sockaddr *addr)
{
R
Roland Dreier 已提交
2420
#if IS_ENABLED(CONFIG_IPV6)
S
Sean Hefty 已提交
2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435
	struct sockaddr_in6 *sin6;

	if (addr->sa_family != AF_INET6)
		return 0;

	sin6 = (struct sockaddr_in6 *) addr;
	if ((ipv6_addr_type(&sin6->sin6_addr) & IPV6_ADDR_LINKLOCAL) &&
	    !sin6->sin6_scope_id)
			return -EINVAL;

	dev_addr->bound_dev_if = sin6->sin6_scope_id;
#endif
	return 0;
}

2436 2437 2438 2439 2440 2441
int rdma_listen(struct rdma_cm_id *id, int backlog)
{
	struct rdma_id_private *id_priv;
	int ret;

	id_priv = container_of(id, struct rdma_id_private, id);
2442
	if (id_priv->state == RDMA_CM_IDLE) {
2443 2444
		id->route.addr.src_addr.ss_family = AF_INET;
		ret = rdma_bind_addr(id, cma_src_addr(id_priv));
2445 2446 2447 2448
		if (ret)
			return ret;
	}

2449
	if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_LISTEN))
2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480
		return -EINVAL;

	if (id_priv->reuseaddr) {
		ret = cma_bind_listen(id_priv);
		if (ret)
			goto err;
	}

	id_priv->backlog = backlog;
	if (id->device) {
		switch (rdma_node_get_transport(id->device->node_type)) {
		case RDMA_TRANSPORT_IB:
			ret = cma_ib_listen(id_priv);
			if (ret)
				goto err;
			break;
		case RDMA_TRANSPORT_IWARP:
			ret = cma_iw_listen(id_priv, backlog);
			if (ret)
				goto err;
			break;
		default:
			ret = -ENOSYS;
			goto err;
		}
	} else
		cma_listen_on_all(id_priv);

	return 0;
err:
	id_priv->backlog = 0;
2481
	cma_comp_exch(id_priv, RDMA_CM_LISTEN, RDMA_CM_ADDR_BOUND);
2482 2483 2484 2485
	return ret;
}
EXPORT_SYMBOL(rdma_listen);

2486 2487 2488 2489 2490
int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
{
	struct rdma_id_private *id_priv;
	int ret;

2491 2492
	if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6 &&
	    addr->sa_family != AF_IB)
2493 2494 2495
		return -EAFNOSUPPORT;

	id_priv = container_of(id, struct rdma_id_private, id);
2496
	if (!cma_comp_exch(id_priv, RDMA_CM_IDLE, RDMA_CM_ADDR_BOUND))
2497 2498
		return -EINVAL;

S
Sean Hefty 已提交
2499 2500 2501 2502
	ret = cma_check_linklocal(&id->route.addr.dev_addr, addr);
	if (ret)
		goto err1;

2503
	if (!cma_any_addr(addr)) {
2504
		ret = cma_translate_addr(addr, &id->route.addr.dev_addr);
2505
		if (ret)
2506 2507 2508 2509 2510
			goto err1;

		ret = cma_acquire_dev(id_priv);
		if (ret)
			goto err1;
2511 2512
	}

2513
	memcpy(cma_src_addr(id_priv), addr, rdma_addr_size(addr));
2514 2515 2516
	if (!(id_priv->options & (1 << CMA_OPTION_AFONLY))) {
		if (addr->sa_family == AF_INET)
			id_priv->afonly = 1;
2517
#if IS_ENABLED(CONFIG_IPV6)
2518 2519
		else if (addr->sa_family == AF_INET6)
			id_priv->afonly = init_net.ipv6.sysctl.bindv6only;
2520
#endif
2521
	}
2522 2523
	ret = cma_get_port(id_priv);
	if (ret)
2524
		goto err2;
2525 2526

	return 0;
2527
err2:
2528 2529
	if (id_priv->cma_dev)
		cma_release_dev(id_priv);
2530
err1:
2531
	cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_IDLE);
2532 2533 2534 2535
	return ret;
}
EXPORT_SYMBOL(rdma_bind_addr);

2536
static int cma_format_hdr(void *hdr, struct rdma_id_private *id_priv)
2537 2538 2539 2540
{
	struct cma_hdr *cma_hdr;
	struct sdp_hh *sdp_hdr;

2541
	if (cma_family(id_priv) == AF_INET) {
A
Aleksey Senin 已提交
2542 2543
		struct sockaddr_in *src4, *dst4;

2544 2545
		src4 = (struct sockaddr_in *) cma_src_addr(id_priv);
		dst4 = (struct sockaddr_in *) cma_dst_addr(id_priv);
A
Aleksey Senin 已提交
2546

2547
		switch (id_priv->id.ps) {
A
Aleksey Senin 已提交
2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568
		case RDMA_PS_SDP:
			sdp_hdr = hdr;
			if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION)
				return -EINVAL;
			sdp_set_ip_ver(sdp_hdr, 4);
			sdp_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
			sdp_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
			sdp_hdr->port = src4->sin_port;
			break;
		default:
			cma_hdr = hdr;
			cma_hdr->cma_version = CMA_VERSION;
			cma_set_ip_ver(cma_hdr, 4);
			cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
			cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
			cma_hdr->port = src4->sin_port;
			break;
		}
	} else {
		struct sockaddr_in6 *src6, *dst6;

2569 2570
		src6 = (struct sockaddr_in6 *) cma_src_addr(id_priv);
		dst6 = (struct sockaddr_in6 *) cma_dst_addr(id_priv);
A
Aleksey Senin 已提交
2571

2572
		switch (id_priv->id.ps) {
A
Aleksey Senin 已提交
2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590
		case RDMA_PS_SDP:
			sdp_hdr = hdr;
			if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION)
				return -EINVAL;
			sdp_set_ip_ver(sdp_hdr, 6);
			sdp_hdr->src_addr.ip6 = src6->sin6_addr;
			sdp_hdr->dst_addr.ip6 = dst6->sin6_addr;
			sdp_hdr->port = src6->sin6_port;
			break;
		default:
			cma_hdr = hdr;
			cma_hdr->cma_version = CMA_VERSION;
			cma_set_ip_ver(cma_hdr, 6);
			cma_hdr->src_addr.ip6 = src6->sin6_addr;
			cma_hdr->dst_addr.ip6 = dst6->sin6_addr;
			cma_hdr->port = src6->sin6_port;
			break;
		}
2591 2592 2593 2594
	}
	return 0;
}

S
Sean Hefty 已提交
2595 2596 2597 2598 2599 2600 2601 2602
static int cma_sidr_rep_handler(struct ib_cm_id *cm_id,
				struct ib_cm_event *ib_event)
{
	struct rdma_id_private *id_priv = cm_id->context;
	struct rdma_cm_event event;
	struct ib_cm_sidr_rep_event_param *rep = &ib_event->param.sidr_rep_rcvd;
	int ret = 0;

2603
	if (cma_disable_callback(id_priv, RDMA_CM_CONNECT))
2604
		return 0;
S
Sean Hefty 已提交
2605

2606
	memset(&event, 0, sizeof event);
S
Sean Hefty 已提交
2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619
	switch (ib_event->event) {
	case IB_CM_SIDR_REQ_ERROR:
		event.event = RDMA_CM_EVENT_UNREACHABLE;
		event.status = -ETIMEDOUT;
		break;
	case IB_CM_SIDR_REP_RECEIVED:
		event.param.ud.private_data = ib_event->private_data;
		event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE;
		if (rep->status != IB_SIDR_SUCCESS) {
			event.event = RDMA_CM_EVENT_UNREACHABLE;
			event.status = ib_event->param.sidr_rep_rcvd.status;
			break;
		}
2620 2621 2622 2623 2624 2625
		ret = cma_set_qkey(id_priv);
		if (ret) {
			event.event = RDMA_CM_EVENT_ADDR_ERROR;
			event.status = -EINVAL;
			break;
		}
2626
		if (id_priv->qkey != rep->qkey) {
S
Sean Hefty 已提交
2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639
			event.event = RDMA_CM_EVENT_UNREACHABLE;
			event.status = -EINVAL;
			break;
		}
		ib_init_ah_from_path(id_priv->id.device, id_priv->id.port_num,
				     id_priv->id.route.path_rec,
				     &event.param.ud.ah_attr);
		event.param.ud.qp_num = rep->qpn;
		event.param.ud.qkey = rep->qkey;
		event.event = RDMA_CM_EVENT_ESTABLISHED;
		event.status = 0;
		break;
	default:
2640
		printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n",
S
Sean Hefty 已提交
2641 2642 2643 2644 2645 2646 2647 2648
		       ib_event->event);
		goto out;
	}

	ret = id_priv->id.event_handler(&id_priv->id, &event);
	if (ret) {
		/* Destroy the CM ID by returning a non-zero value. */
		id_priv->cm_id.ib = NULL;
2649
		cma_exch(id_priv, RDMA_CM_DESTROYING);
2650
		mutex_unlock(&id_priv->handler_mutex);
S
Sean Hefty 已提交
2651 2652 2653 2654
		rdma_destroy_id(&id_priv->id);
		return ret;
	}
out:
2655
	mutex_unlock(&id_priv->handler_mutex);
S
Sean Hefty 已提交
2656 2657 2658 2659 2660 2661 2662
	return ret;
}

static int cma_resolve_ib_udp(struct rdma_id_private *id_priv,
			      struct rdma_conn_param *conn_param)
{
	struct ib_cm_sidr_req_param req;
2663
	struct ib_cm_id	*id;
S
Sean Hefty 已提交
2664 2665 2666 2667
	int ret;

	req.private_data_len = sizeof(struct cma_hdr) +
			       conn_param->private_data_len;
S
Sean Hefty 已提交
2668 2669 2670
	if (req.private_data_len < conn_param->private_data_len)
		return -EINVAL;

S
Sean Hefty 已提交
2671 2672 2673 2674 2675 2676 2677 2678
	req.private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
	if (!req.private_data)
		return -ENOMEM;

	if (conn_param->private_data && conn_param->private_data_len)
		memcpy((void *) req.private_data + sizeof(struct cma_hdr),
		       conn_param->private_data, conn_param->private_data_len);

2679
	ret = cma_format_hdr((void *) req.private_data, id_priv);
S
Sean Hefty 已提交
2680 2681 2682
	if (ret)
		goto out;

2683 2684 2685 2686
	id = ib_create_cm_id(id_priv->id.device, cma_sidr_rep_handler,
			     id_priv);
	if (IS_ERR(id)) {
		ret = PTR_ERR(id);
S
Sean Hefty 已提交
2687 2688
		goto out;
	}
2689
	id_priv->cm_id.ib = id;
S
Sean Hefty 已提交
2690

2691 2692
	req.path = id_priv->id.route.path_rec;
	req.service_id = cma_get_service_id(id_priv->id.ps, cma_dst_addr(id_priv));
S
Sean Hefty 已提交
2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705
	req.timeout_ms = 1 << (CMA_CM_RESPONSE_TIMEOUT - 8);
	req.max_cm_retries = CMA_MAX_CM_RETRIES;

	ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req);
	if (ret) {
		ib_destroy_cm_id(id_priv->cm_id.ib);
		id_priv->cm_id.ib = NULL;
	}
out:
	kfree(req.private_data);
	return ret;
}

2706 2707 2708 2709 2710 2711
static int cma_connect_ib(struct rdma_id_private *id_priv,
			  struct rdma_conn_param *conn_param)
{
	struct ib_cm_req_param req;
	struct rdma_route *route;
	void *private_data;
2712
	struct ib_cm_id	*id;
2713 2714 2715 2716 2717
	int offset, ret;

	memset(&req, 0, sizeof req);
	offset = cma_user_data_offset(id_priv->id.ps);
	req.private_data_len = offset + conn_param->private_data_len;
S
Sean Hefty 已提交
2718 2719 2720
	if (req.private_data_len < conn_param->private_data_len)
		return -EINVAL;

2721 2722 2723 2724 2725 2726 2727 2728
	private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
	if (!private_data)
		return -ENOMEM;

	if (conn_param->private_data && conn_param->private_data_len)
		memcpy(private_data + offset, conn_param->private_data,
		       conn_param->private_data_len);

2729 2730 2731
	id = ib_create_cm_id(id_priv->id.device, cma_ib_handler, id_priv);
	if (IS_ERR(id)) {
		ret = PTR_ERR(id);
2732 2733
		goto out;
	}
2734
	id_priv->cm_id.ib = id;
2735 2736

	route = &id_priv->id.route;
2737
	ret = cma_format_hdr(private_data, id_priv);
2738 2739 2740 2741 2742 2743 2744 2745
	if (ret)
		goto out;
	req.private_data = private_data;

	req.primary_path = &route->path_rec[0];
	if (route->num_paths == 2)
		req.alternate_path = &route->path_rec[1];

2746
	req.service_id = cma_get_service_id(id_priv->id.ps, cma_dst_addr(id_priv));
2747
	req.qp_num = id_priv->qp_num;
S
Sean Hefty 已提交
2748
	req.qp_type = id_priv->id.qp_type;
2749 2750 2751 2752
	req.starting_psn = id_priv->seq_num;
	req.responder_resources = conn_param->responder_resources;
	req.initiator_depth = conn_param->initiator_depth;
	req.flow_control = conn_param->flow_control;
2753 2754
	req.retry_count = min_t(u8, 7, conn_param->retry_count);
	req.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count);
2755 2756 2757 2758 2759 2760 2761
	req.remote_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
	req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
	req.max_cm_retries = CMA_MAX_CM_RETRIES;
	req.srq = id_priv->srq ? 1 : 0;

	ret = ib_send_cm_req(id_priv->cm_id.ib, &req);
out:
2762 2763
	if (ret && !IS_ERR(id)) {
		ib_destroy_cm_id(id);
2764 2765 2766
		id_priv->cm_id.ib = NULL;
	}

2767 2768 2769 2770
	kfree(private_data);
	return ret;
}

T
Tom Tucker 已提交
2771 2772 2773 2774 2775 2776 2777 2778 2779
static int cma_connect_iw(struct rdma_id_private *id_priv,
			  struct rdma_conn_param *conn_param)
{
	struct iw_cm_id *cm_id;
	struct sockaddr_in* sin;
	int ret;
	struct iw_cm_conn_param iw_param;

	cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv);
2780 2781
	if (IS_ERR(cm_id))
		return PTR_ERR(cm_id);
T
Tom Tucker 已提交
2782 2783 2784

	id_priv->cm_id.iw = cm_id;

2785
	sin = (struct sockaddr_in *) cma_src_addr(id_priv);
T
Tom Tucker 已提交
2786 2787
	cm_id->local_addr = *sin;

2788
	sin = (struct sockaddr_in *) cma_dst_addr(id_priv);
T
Tom Tucker 已提交
2789 2790
	cm_id->remote_addr = *sin;

2791
	ret = cma_modify_qp_rtr(id_priv, conn_param);
2792 2793
	if (ret)
		goto out;
T
Tom Tucker 已提交
2794

2795 2796 2797 2798 2799 2800 2801 2802
	if (conn_param) {
		iw_param.ord = conn_param->initiator_depth;
		iw_param.ird = conn_param->responder_resources;
		iw_param.private_data = conn_param->private_data;
		iw_param.private_data_len = conn_param->private_data_len;
		iw_param.qpn = id_priv->id.qp ? id_priv->qp_num : conn_param->qp_num;
	} else {
		memset(&iw_param, 0, sizeof iw_param);
T
Tom Tucker 已提交
2803
		iw_param.qpn = id_priv->qp_num;
2804
	}
T
Tom Tucker 已提交
2805 2806
	ret = iw_cm_connect(cm_id, &iw_param);
out:
2807
	if (ret) {
2808 2809 2810
		iw_destroy_cm_id(cm_id);
		id_priv->cm_id.iw = NULL;
	}
T
Tom Tucker 已提交
2811 2812 2813
	return ret;
}

2814 2815 2816 2817 2818 2819
int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
{
	struct rdma_id_private *id_priv;
	int ret;

	id_priv = container_of(id, struct rdma_id_private, id);
2820
	if (!cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_CONNECT))
2821 2822 2823 2824 2825 2826 2827
		return -EINVAL;

	if (!id->qp) {
		id_priv->qp_num = conn_param->qp_num;
		id_priv->srq = conn_param->srq;
	}

T
Tom Tucker 已提交
2828 2829
	switch (rdma_node_get_transport(id->device->node_type)) {
	case RDMA_TRANSPORT_IB:
2830
		if (id->qp_type == IB_QPT_UD)
S
Sean Hefty 已提交
2831 2832 2833
			ret = cma_resolve_ib_udp(id_priv, conn_param);
		else
			ret = cma_connect_ib(id_priv, conn_param);
2834
		break;
T
Tom Tucker 已提交
2835 2836 2837
	case RDMA_TRANSPORT_IWARP:
		ret = cma_connect_iw(id_priv, conn_param);
		break;
2838 2839 2840 2841 2842 2843 2844 2845 2846
	default:
		ret = -ENOSYS;
		break;
	}
	if (ret)
		goto err;

	return 0;
err:
2847
	cma_comp_exch(id_priv, RDMA_CM_CONNECT, RDMA_CM_ROUTE_RESOLVED);
2848 2849 2850 2851 2852 2853 2854 2855
	return ret;
}
EXPORT_SYMBOL(rdma_connect);

static int cma_accept_ib(struct rdma_id_private *id_priv,
			 struct rdma_conn_param *conn_param)
{
	struct ib_cm_rep_param rep;
2856
	int ret;
2857

2858 2859 2860
	ret = cma_modify_qp_rtr(id_priv, conn_param);
	if (ret)
		goto out;
2861

2862 2863 2864
	ret = cma_modify_qp_rts(id_priv, conn_param);
	if (ret)
		goto out;
2865 2866 2867 2868 2869 2870 2871 2872 2873 2874

	memset(&rep, 0, sizeof rep);
	rep.qp_num = id_priv->qp_num;
	rep.starting_psn = id_priv->seq_num;
	rep.private_data = conn_param->private_data;
	rep.private_data_len = conn_param->private_data_len;
	rep.responder_resources = conn_param->responder_resources;
	rep.initiator_depth = conn_param->initiator_depth;
	rep.failover_accepted = 0;
	rep.flow_control = conn_param->flow_control;
2875
	rep.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count);
2876 2877
	rep.srq = id_priv->srq ? 1 : 0;

2878 2879 2880
	ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep);
out:
	return ret;
2881 2882
}

T
Tom Tucker 已提交
2883 2884 2885 2886 2887 2888
static int cma_accept_iw(struct rdma_id_private *id_priv,
		  struct rdma_conn_param *conn_param)
{
	struct iw_cm_conn_param iw_param;
	int ret;

2889
	ret = cma_modify_qp_rtr(id_priv, conn_param);
T
Tom Tucker 已提交
2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904
	if (ret)
		return ret;

	iw_param.ord = conn_param->initiator_depth;
	iw_param.ird = conn_param->responder_resources;
	iw_param.private_data = conn_param->private_data;
	iw_param.private_data_len = conn_param->private_data_len;
	if (id_priv->id.qp) {
		iw_param.qpn = id_priv->qp_num;
	} else
		iw_param.qpn = conn_param->qp_num;

	return iw_cm_accept(id_priv->cm_id.iw, &iw_param);
}

S
Sean Hefty 已提交
2905 2906 2907 2908 2909
static int cma_send_sidr_rep(struct rdma_id_private *id_priv,
			     enum ib_cm_sidr_status status,
			     const void *private_data, int private_data_len)
{
	struct ib_cm_sidr_rep_param rep;
2910
	int ret;
S
Sean Hefty 已提交
2911 2912 2913 2914

	memset(&rep, 0, sizeof rep);
	rep.status = status;
	if (status == IB_SIDR_SUCCESS) {
2915 2916 2917
		ret = cma_set_qkey(id_priv);
		if (ret)
			return ret;
S
Sean Hefty 已提交
2918
		rep.qp_num = id_priv->qp_num;
2919
		rep.qkey = id_priv->qkey;
S
Sean Hefty 已提交
2920 2921 2922 2923 2924 2925 2926
	}
	rep.private_data = private_data;
	rep.private_data_len = private_data_len;

	return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep);
}

2927 2928 2929 2930 2931 2932
int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
{
	struct rdma_id_private *id_priv;
	int ret;

	id_priv = container_of(id, struct rdma_id_private, id);
N
Nir Muchtar 已提交
2933 2934 2935

	id_priv->owner = task_pid_nr(current);

2936
	if (!cma_comp(id_priv, RDMA_CM_CONNECT))
2937 2938 2939 2940 2941 2942 2943
		return -EINVAL;

	if (!id->qp && conn_param) {
		id_priv->qp_num = conn_param->qp_num;
		id_priv->srq = conn_param->srq;
	}

T
Tom Tucker 已提交
2944 2945
	switch (rdma_node_get_transport(id->device->node_type)) {
	case RDMA_TRANSPORT_IB:
2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959
		if (id->qp_type == IB_QPT_UD) {
			if (conn_param)
				ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
							conn_param->private_data,
							conn_param->private_data_len);
			else
				ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
							NULL, 0);
		} else {
			if (conn_param)
				ret = cma_accept_ib(id_priv, conn_param);
			else
				ret = cma_rep_recv(id_priv);
		}
2960
		break;
T
Tom Tucker 已提交
2961 2962 2963
	case RDMA_TRANSPORT_IWARP:
		ret = cma_accept_iw(id_priv, conn_param);
		break;
2964 2965 2966 2967 2968 2969 2970 2971 2972 2973
	default:
		ret = -ENOSYS;
		break;
	}

	if (ret)
		goto reject;

	return 0;
reject:
2974
	cma_modify_qp_err(id_priv);
2975 2976 2977 2978 2979
	rdma_reject(id, NULL, 0);
	return ret;
}
EXPORT_SYMBOL(rdma_accept);

2980 2981 2982 2983 2984 2985
int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event)
{
	struct rdma_id_private *id_priv;
	int ret;

	id_priv = container_of(id, struct rdma_id_private, id);
2986
	if (!id_priv->cm_id.ib)
2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000
		return -EINVAL;

	switch (id->device->node_type) {
	case RDMA_NODE_IB_CA:
		ret = ib_cm_notify(id_priv->cm_id.ib, event);
		break;
	default:
		ret = 0;
		break;
	}
	return ret;
}
EXPORT_SYMBOL(rdma_notify);

3001 3002 3003 3004 3005 3006 3007
int rdma_reject(struct rdma_cm_id *id, const void *private_data,
		u8 private_data_len)
{
	struct rdma_id_private *id_priv;
	int ret;

	id_priv = container_of(id, struct rdma_id_private, id);
3008
	if (!id_priv->cm_id.ib)
3009 3010
		return -EINVAL;

T
Tom Tucker 已提交
3011 3012
	switch (rdma_node_get_transport(id->device->node_type)) {
	case RDMA_TRANSPORT_IB:
3013
		if (id->qp_type == IB_QPT_UD)
S
Sean Hefty 已提交
3014 3015 3016 3017 3018 3019
			ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT,
						private_data, private_data_len);
		else
			ret = ib_send_cm_rej(id_priv->cm_id.ib,
					     IB_CM_REJ_CONSUMER_DEFINED, NULL,
					     0, private_data, private_data_len);
3020
		break;
T
Tom Tucker 已提交
3021 3022 3023 3024
	case RDMA_TRANSPORT_IWARP:
		ret = iw_cm_reject(id_priv->cm_id.iw,
				   private_data, private_data_len);
		break;
3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038
	default:
		ret = -ENOSYS;
		break;
	}
	return ret;
}
EXPORT_SYMBOL(rdma_reject);

int rdma_disconnect(struct rdma_cm_id *id)
{
	struct rdma_id_private *id_priv;
	int ret;

	id_priv = container_of(id, struct rdma_id_private, id);
3039
	if (!id_priv->cm_id.ib)
3040 3041
		return -EINVAL;

T
Tom Tucker 已提交
3042 3043
	switch (rdma_node_get_transport(id->device->node_type)) {
	case RDMA_TRANSPORT_IB:
3044
		ret = cma_modify_qp_err(id_priv);
T
Tom Tucker 已提交
3045 3046
		if (ret)
			goto out;
3047 3048 3049 3050
		/* Initiate or respond to a disconnect. */
		if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0))
			ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0);
		break;
T
Tom Tucker 已提交
3051 3052 3053
	case RDMA_TRANSPORT_IWARP:
		ret = iw_cm_disconnect(id_priv->cm_id.iw, 0);
		break;
3054
	default:
T
Tom Tucker 已提交
3055
		ret = -EINVAL;
3056 3057 3058 3059 3060 3061 3062
		break;
	}
out:
	return ret;
}
EXPORT_SYMBOL(rdma_disconnect);

3063 3064 3065 3066 3067 3068 3069 3070
static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast)
{
	struct rdma_id_private *id_priv;
	struct cma_multicast *mc = multicast->context;
	struct rdma_cm_event event;
	int ret;

	id_priv = mc->id_priv;
3071 3072
	if (cma_disable_callback(id_priv, RDMA_CM_ADDR_BOUND) &&
	    cma_disable_callback(id_priv, RDMA_CM_ADDR_RESOLVED))
3073
		return 0;
3074

3075
	mutex_lock(&id_priv->qp_mutex);
3076 3077
	if (!status && id_priv->id.qp)
		status = ib_attach_mcast(id_priv->id.qp, &multicast->rec.mgid,
S
Sean Hefty 已提交
3078
					 be16_to_cpu(multicast->rec.mlid));
3079
	mutex_unlock(&id_priv->qp_mutex);
3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095

	memset(&event, 0, sizeof event);
	event.status = status;
	event.param.ud.private_data = mc->context;
	if (!status) {
		event.event = RDMA_CM_EVENT_MULTICAST_JOIN;
		ib_init_ah_from_mcmember(id_priv->id.device,
					 id_priv->id.port_num, &multicast->rec,
					 &event.param.ud.ah_attr);
		event.param.ud.qp_num = 0xFFFFFF;
		event.param.ud.qkey = be32_to_cpu(multicast->rec.qkey);
	} else
		event.event = RDMA_CM_EVENT_MULTICAST_ERROR;

	ret = id_priv->id.event_handler(&id_priv->id, &event);
	if (ret) {
3096
		cma_exch(id_priv, RDMA_CM_DESTROYING);
3097
		mutex_unlock(&id_priv->handler_mutex);
3098 3099 3100
		rdma_destroy_id(&id_priv->id);
		return 0;
	}
3101

3102
	mutex_unlock(&id_priv->handler_mutex);
3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116
	return 0;
}

static void cma_set_mgid(struct rdma_id_private *id_priv,
			 struct sockaddr *addr, union ib_gid *mgid)
{
	unsigned char mc_map[MAX_ADDR_LEN];
	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
	struct sockaddr_in *sin = (struct sockaddr_in *) addr;
	struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr;

	if (cma_any_addr(addr)) {
		memset(mgid, 0, sizeof *mgid);
	} else if ((addr->sa_family == AF_INET6) &&
3117
		   ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFFF0FFFF) ==
3118 3119 3120
								 0xFF10A01B)) {
		/* IPv6 address is an SA assigned MGID. */
		memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
3121 3122 3123 3124 3125
	} else if ((addr->sa_family == AF_INET6)) {
		ipv6_ib_mc_map(&sin6->sin6_addr, dev_addr->broadcast, mc_map);
		if (id_priv->id.ps == RDMA_PS_UDP)
			mc_map[7] = 0x01;	/* Use RDMA CM signature */
		*mgid = *(union ib_gid *) (mc_map + 4);
3126
	} else {
3127
		ip_ib_mc_map(sin->sin_addr.s_addr, dev_addr->broadcast, mc_map);
3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147
		if (id_priv->id.ps == RDMA_PS_UDP)
			mc_map[7] = 0x01;	/* Use RDMA CM signature */
		*mgid = *(union ib_gid *) (mc_map + 4);
	}
}

static int cma_join_ib_multicast(struct rdma_id_private *id_priv,
				 struct cma_multicast *mc)
{
	struct ib_sa_mcmember_rec rec;
	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
	ib_sa_comp_mask comp_mask;
	int ret;

	ib_addr_get_mgid(dev_addr, &rec.mgid);
	ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num,
				     &rec.mgid, &rec);
	if (ret)
		return ret;

3148
	cma_set_mgid(id_priv, (struct sockaddr *) &mc->addr, &rec.mgid);
3149 3150
	if (id_priv->id.ps == RDMA_PS_UDP)
		rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
S
Sean Hefty 已提交
3151
	rdma_addr_get_sgid(dev_addr, &rec.port_gid);
3152 3153 3154 3155 3156 3157 3158 3159 3160
	rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
	rec.join_state = 1;

	comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
		    IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE |
		    IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL |
		    IB_SA_MCMEMBER_REC_FLOW_LABEL |
		    IB_SA_MCMEMBER_REC_TRAFFIC_CLASS;

3161 3162
	if (id_priv->id.ps == RDMA_PS_IPOIB)
		comp_mask |= IB_SA_MCMEMBER_REC_RATE |
3163 3164 3165 3166
			     IB_SA_MCMEMBER_REC_RATE_SELECTOR |
			     IB_SA_MCMEMBER_REC_MTU_SELECTOR |
			     IB_SA_MCMEMBER_REC_MTU |
			     IB_SA_MCMEMBER_REC_HOP_LIMIT;
3167

3168 3169 3170 3171
	mc->multicast.ib = ib_sa_join_multicast(&sa_client, id_priv->id.device,
						id_priv->id.port_num, &rec,
						comp_mask, GFP_KERNEL,
						cma_ib_mc_handler, mc);
3172
	return PTR_RET(mc->multicast.ib);
3173 3174
}

3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270
static void iboe_mcast_work_handler(struct work_struct *work)
{
	struct iboe_mcast_work *mw = container_of(work, struct iboe_mcast_work, work);
	struct cma_multicast *mc = mw->mc;
	struct ib_sa_multicast *m = mc->multicast.ib;

	mc->multicast.ib->context = mc;
	cma_ib_mc_handler(0, m);
	kref_put(&mc->mcref, release_mc);
	kfree(mw);
}

static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid)
{
	struct sockaddr_in *sin = (struct sockaddr_in *)addr;
	struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr;

	if (cma_any_addr(addr)) {
		memset(mgid, 0, sizeof *mgid);
	} else if (addr->sa_family == AF_INET6) {
		memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
	} else {
		mgid->raw[0] = 0xff;
		mgid->raw[1] = 0x0e;
		mgid->raw[2] = 0;
		mgid->raw[3] = 0;
		mgid->raw[4] = 0;
		mgid->raw[5] = 0;
		mgid->raw[6] = 0;
		mgid->raw[7] = 0;
		mgid->raw[8] = 0;
		mgid->raw[9] = 0;
		mgid->raw[10] = 0xff;
		mgid->raw[11] = 0xff;
		*(__be32 *)(&mgid->raw[12]) = sin->sin_addr.s_addr;
	}
}

static int cma_iboe_join_multicast(struct rdma_id_private *id_priv,
				   struct cma_multicast *mc)
{
	struct iboe_mcast_work *work;
	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
	int err;
	struct sockaddr *addr = (struct sockaddr *)&mc->addr;
	struct net_device *ndev = NULL;

	if (cma_zero_addr((struct sockaddr *)&mc->addr))
		return -EINVAL;

	work = kzalloc(sizeof *work, GFP_KERNEL);
	if (!work)
		return -ENOMEM;

	mc->multicast.ib = kzalloc(sizeof(struct ib_sa_multicast), GFP_KERNEL);
	if (!mc->multicast.ib) {
		err = -ENOMEM;
		goto out1;
	}

	cma_iboe_set_mgid(addr, &mc->multicast.ib->rec.mgid);

	mc->multicast.ib->rec.pkey = cpu_to_be16(0xffff);
	if (id_priv->id.ps == RDMA_PS_UDP)
		mc->multicast.ib->rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);

	if (dev_addr->bound_dev_if)
		ndev = dev_get_by_index(&init_net, dev_addr->bound_dev_if);
	if (!ndev) {
		err = -ENODEV;
		goto out2;
	}
	mc->multicast.ib->rec.rate = iboe_get_rate(ndev);
	mc->multicast.ib->rec.hop_limit = 1;
	mc->multicast.ib->rec.mtu = iboe_get_mtu(ndev->mtu);
	dev_put(ndev);
	if (!mc->multicast.ib->rec.mtu) {
		err = -EINVAL;
		goto out2;
	}
	iboe_addr_get_sgid(dev_addr, &mc->multicast.ib->rec.port_gid);
	work->id = id_priv;
	work->mc = mc;
	INIT_WORK(&work->work, iboe_mcast_work_handler);
	kref_get(&mc->mcref);
	queue_work(cma_wq, &work->work);

	return 0;

out2:
	kfree(mc->multicast.ib);
out1:
	kfree(work);
	return err;
}

3271 3272 3273 3274 3275 3276 3277 3278
int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
			void *context)
{
	struct rdma_id_private *id_priv;
	struct cma_multicast *mc;
	int ret;

	id_priv = container_of(id, struct rdma_id_private, id);
3279 3280
	if (!cma_comp(id_priv, RDMA_CM_ADDR_BOUND) &&
	    !cma_comp(id_priv, RDMA_CM_ADDR_RESOLVED))
3281 3282 3283 3284 3285 3286
		return -EINVAL;

	mc = kmalloc(sizeof *mc, GFP_KERNEL);
	if (!mc)
		return -ENOMEM;

3287
	memcpy(&mc->addr, addr, rdma_addr_size(addr));
3288 3289 3290 3291 3292 3293 3294 3295 3296
	mc->context = context;
	mc->id_priv = id_priv;

	spin_lock(&id_priv->lock);
	list_add(&mc->list, &id_priv->mc_list);
	spin_unlock(&id_priv->lock);

	switch (rdma_node_get_transport(id->device->node_type)) {
	case RDMA_TRANSPORT_IB:
3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307
		switch (rdma_port_get_link_layer(id->device, id->port_num)) {
		case IB_LINK_LAYER_INFINIBAND:
			ret = cma_join_ib_multicast(id_priv, mc);
			break;
		case IB_LINK_LAYER_ETHERNET:
			kref_init(&mc->mcref);
			ret = cma_iboe_join_multicast(id_priv, mc);
			break;
		default:
			ret = -EINVAL;
		}
3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331
		break;
	default:
		ret = -ENOSYS;
		break;
	}

	if (ret) {
		spin_lock_irq(&id_priv->lock);
		list_del(&mc->list);
		spin_unlock_irq(&id_priv->lock);
		kfree(mc);
	}
	return ret;
}
EXPORT_SYMBOL(rdma_join_multicast);

void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
{
	struct rdma_id_private *id_priv;
	struct cma_multicast *mc;

	id_priv = container_of(id, struct rdma_id_private, id);
	spin_lock_irq(&id_priv->lock);
	list_for_each_entry(mc, &id_priv->mc_list, list) {
3332
		if (!memcmp(&mc->addr, addr, rdma_addr_size(addr))) {
3333 3334 3335 3336 3337 3338
			list_del(&mc->list);
			spin_unlock_irq(&id_priv->lock);

			if (id->qp)
				ib_detach_mcast(id->qp,
						&mc->multicast.ib->rec.mgid,
S
Sean Hefty 已提交
3339
						be16_to_cpu(mc->multicast.ib->rec.mlid));
3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352
			if (rdma_node_get_transport(id_priv->cma_dev->device->node_type) == RDMA_TRANSPORT_IB) {
				switch (rdma_port_get_link_layer(id->device, id->port_num)) {
				case IB_LINK_LAYER_INFINIBAND:
					ib_sa_free_multicast(mc->multicast.ib);
					kfree(mc);
					break;
				case IB_LINK_LAYER_ETHERNET:
					kref_put(&mc->mcref, release_mc);
					break;
				default:
					break;
				}
			}
3353 3354 3355 3356 3357 3358 3359
			return;
		}
	}
	spin_unlock_irq(&id_priv->lock);
}
EXPORT_SYMBOL(rdma_leave_multicast);

3360 3361 3362 3363 3364 3365 3366
static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private *id_priv)
{
	struct rdma_dev_addr *dev_addr;
	struct cma_ndev_work *work;

	dev_addr = &id_priv->id.route.addr.dev_addr;

3367
	if ((dev_addr->bound_dev_if == ndev->ifindex) &&
3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385
	    memcmp(dev_addr->src_dev_addr, ndev->dev_addr, ndev->addr_len)) {
		printk(KERN_INFO "RDMA CM addr change for ndev %s used by id %p\n",
		       ndev->name, &id_priv->id);
		work = kzalloc(sizeof *work, GFP_KERNEL);
		if (!work)
			return -ENOMEM;

		INIT_WORK(&work->work, cma_ndev_work_handler);
		work->id = id_priv;
		work->event.event = RDMA_CM_EVENT_ADDR_CHANGE;
		atomic_inc(&id_priv->refcount);
		queue_work(cma_wq, &work->work);
	}

	return 0;
}

static int cma_netdev_callback(struct notifier_block *self, unsigned long event,
3386
			       void *ptr)
3387
{
3388
	struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418
	struct cma_device *cma_dev;
	struct rdma_id_private *id_priv;
	int ret = NOTIFY_DONE;

	if (dev_net(ndev) != &init_net)
		return NOTIFY_DONE;

	if (event != NETDEV_BONDING_FAILOVER)
		return NOTIFY_DONE;

	if (!(ndev->flags & IFF_MASTER) || !(ndev->priv_flags & IFF_BONDING))
		return NOTIFY_DONE;

	mutex_lock(&lock);
	list_for_each_entry(cma_dev, &dev_list, list)
		list_for_each_entry(id_priv, &cma_dev->id_list, list) {
			ret = cma_netdev_change(ndev, id_priv);
			if (ret)
				goto out;
		}

out:
	mutex_unlock(&lock);
	return ret;
}

static struct notifier_block cma_nb = {
	.notifier_call = cma_netdev_callback
};

3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443
static void cma_add_one(struct ib_device *device)
{
	struct cma_device *cma_dev;
	struct rdma_id_private *id_priv;

	cma_dev = kmalloc(sizeof *cma_dev, GFP_KERNEL);
	if (!cma_dev)
		return;

	cma_dev->device = device;

	init_completion(&cma_dev->comp);
	atomic_set(&cma_dev->refcount, 1);
	INIT_LIST_HEAD(&cma_dev->id_list);
	ib_set_client_data(device, &cma_client, cma_dev);

	mutex_lock(&lock);
	list_add_tail(&cma_dev->list, &dev_list);
	list_for_each_entry(id_priv, &listen_any_list, list)
		cma_listen_on_dev(id_priv, cma_dev);
	mutex_unlock(&lock);
}

static int cma_remove_id_dev(struct rdma_id_private *id_priv)
{
3444
	struct rdma_cm_event event;
3445
	enum rdma_cm_state state;
3446
	int ret = 0;
3447 3448

	/* Record that we want to remove the device */
3449 3450
	state = cma_exch(id_priv, RDMA_CM_DEVICE_REMOVAL);
	if (state == RDMA_CM_DESTROYING)
3451 3452 3453
		return 0;

	cma_cancel_operation(id_priv, state);
3454
	mutex_lock(&id_priv->handler_mutex);
3455 3456

	/* Check for destruction from another callback. */
3457
	if (!cma_comp(id_priv, RDMA_CM_DEVICE_REMOVAL))
3458
		goto out;
3459

3460 3461
	memset(&event, 0, sizeof event);
	event.event = RDMA_CM_EVENT_DEVICE_REMOVAL;
3462 3463 3464 3465
	ret = id_priv->id.event_handler(&id_priv->id, &event);
out:
	mutex_unlock(&id_priv->handler_mutex);
	return ret;
3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477
}

static void cma_process_remove(struct cma_device *cma_dev)
{
	struct rdma_id_private *id_priv;
	int ret;

	mutex_lock(&lock);
	while (!list_empty(&cma_dev->id_list)) {
		id_priv = list_entry(cma_dev->id_list.next,
				     struct rdma_id_private, list);

3478
		list_del(&id_priv->listen_list);
3479
		list_del_init(&id_priv->list);
3480 3481 3482
		atomic_inc(&id_priv->refcount);
		mutex_unlock(&lock);

3483
		ret = id_priv->internal_id ? 1 : cma_remove_id_dev(id_priv);
3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511
		cma_deref_id(id_priv);
		if (ret)
			rdma_destroy_id(&id_priv->id);

		mutex_lock(&lock);
	}
	mutex_unlock(&lock);

	cma_deref_dev(cma_dev);
	wait_for_completion(&cma_dev->comp);
}

static void cma_remove_one(struct ib_device *device)
{
	struct cma_device *cma_dev;

	cma_dev = ib_get_client_data(device, &cma_client);
	if (!cma_dev)
		return;

	mutex_lock(&lock);
	list_del(&cma_dev->list);
	mutex_unlock(&lock);

	cma_process_remove(cma_dev);
	kfree(cma_dev);
}

3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552
static int cma_get_id_stats(struct sk_buff *skb, struct netlink_callback *cb)
{
	struct nlmsghdr *nlh;
	struct rdma_cm_id_stats *id_stats;
	struct rdma_id_private *id_priv;
	struct rdma_cm_id *id = NULL;
	struct cma_device *cma_dev;
	int i_dev = 0, i_id = 0;

	/*
	 * We export all of the IDs as a sequence of messages.  Each
	 * ID gets its own netlink message.
	 */
	mutex_lock(&lock);

	list_for_each_entry(cma_dev, &dev_list, list) {
		if (i_dev < cb->args[0]) {
			i_dev++;
			continue;
		}

		i_id = 0;
		list_for_each_entry(id_priv, &cma_dev->id_list, list) {
			if (i_id < cb->args[1]) {
				i_id++;
				continue;
			}

			id_stats = ibnl_put_msg(skb, &nlh, cb->nlh->nlmsg_seq,
						sizeof *id_stats, RDMA_NL_RDMA_CM,
						RDMA_NL_RDMA_CM_ID_STATS);
			if (!id_stats)
				goto out;

			memset(id_stats, 0, sizeof *id_stats);
			id = &id_priv->id;
			id_stats->node_type = id->route.addr.dev_addr.dev_type;
			id_stats->port_num = id->port_num;
			id_stats->bound_dev_if =
				id->route.addr.dev_addr.bound_dev_if;

3553
			if (cma_family(id_priv) == AF_INET) {
3554 3555
				if (ibnl_put_attr(skb, nlh,
						  sizeof(struct sockaddr_in),
3556
						  cma_src_addr(id_priv),
3557 3558 3559 3560 3561
						  RDMA_NL_RDMA_CM_ATTR_SRC_ADDR)) {
					goto out;
				}
				if (ibnl_put_attr(skb, nlh,
						  sizeof(struct sockaddr_in),
3562
						  cma_dst_addr(id_priv),
3563 3564 3565
						  RDMA_NL_RDMA_CM_ATTR_DST_ADDR)) {
					goto out;
				}
3566
			} else if (cma_family(id_priv) == AF_INET6) {
3567 3568
				if (ibnl_put_attr(skb, nlh,
						  sizeof(struct sockaddr_in6),
3569
						  cma_src_addr(id_priv),
3570 3571 3572 3573 3574
						  RDMA_NL_RDMA_CM_ATTR_SRC_ADDR)) {
					goto out;
				}
				if (ibnl_put_attr(skb, nlh,
						  sizeof(struct sockaddr_in6),
3575
						  cma_dst_addr(id_priv),
3576 3577 3578 3579 3580
						  RDMA_NL_RDMA_CM_ATTR_DST_ADDR)) {
					goto out;
				}
			}

N
Nir Muchtar 已提交
3581
			id_stats->pid		= id_priv->owner;
3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602
			id_stats->port_space	= id->ps;
			id_stats->cm_state	= id_priv->state;
			id_stats->qp_num	= id_priv->qp_num;
			id_stats->qp_type	= id->qp_type;

			i_id++;
		}

		cb->args[1] = 0;
		i_dev++;
	}

out:
	mutex_unlock(&lock);
	cb->args[0] = i_dev;
	cb->args[1] = i_id;

	return skb->len;
}

static const struct ibnl_client_cbs cma_cb_table[] = {
3603 3604
	[RDMA_NL_RDMA_CM_ID_STATS] = { .dump = cma_get_id_stats,
				       .module = THIS_MODULE },
3605 3606
};

3607
static int __init cma_init(void)
3608
{
3609
	int ret;
3610

3611
	cma_wq = create_singlethread_workqueue("rdma_cm");
3612 3613 3614
	if (!cma_wq)
		return -ENOMEM;

3615
	ib_sa_register_client(&sa_client);
3616
	rdma_addr_register_client(&addr_client);
3617
	register_netdevice_notifier(&cma_nb);
3618

3619 3620 3621
	ret = ib_register_client(&cma_client);
	if (ret)
		goto err;
3622 3623 3624 3625

	if (ibnl_add_client(RDMA_NL_RDMA_CM, RDMA_NL_RDMA_CM_NUM_OPS, cma_cb_table))
		printk(KERN_WARNING "RDMA CMA: failed to add netlink callback\n");

3626 3627 3628
	return 0;

err:
3629
	unregister_netdevice_notifier(&cma_nb);
3630
	rdma_addr_unregister_client(&addr_client);
3631
	ib_sa_unregister_client(&sa_client);
3632 3633 3634 3635
	destroy_workqueue(cma_wq);
	return ret;
}

3636
static void __exit cma_cleanup(void)
3637
{
3638
	ibnl_remove_client(RDMA_NL_RDMA_CM);
3639
	ib_unregister_client(&cma_client);
3640
	unregister_netdevice_notifier(&cma_nb);
3641
	rdma_addr_unregister_client(&addr_client);
3642
	ib_sa_unregister_client(&sa_client);
3643 3644 3645
	destroy_workqueue(cma_wq);
	idr_destroy(&sdp_ps);
	idr_destroy(&tcp_ps);
S
Sean Hefty 已提交
3646
	idr_destroy(&udp_ps);
3647
	idr_destroy(&ipoib_ps);
3648
	idr_destroy(&ib_ps);
3649 3650 3651 3652
}

module_init(cma_init);
module_exit(cma_cleanup);