af_smc.c 42.9 KB
Newer Older
U
Ursula Braun 已提交
1 2 3 4 5 6 7 8
/*
 *  Shared Memory Communications over RDMA (SMC-R) and RoCE
 *
 *  AF_SMC protocol family socket handler keeping the AF_INET sock address type
 *  applies to SOCK_STREAM sockets only
 *  offers an alternative communication option for TCP-protocol sockets
 *  applicable with RoCE-cards only
 *
9 10 11
 *  Initial restrictions:
 *    - support for alternate links postponed
 *
12
 *  Copyright IBM Corp. 2016, 2018
U
Ursula Braun 已提交
13 14 15 16 17 18 19 20 21 22
 *
 *  Author(s):  Ursula Braun <ubraun@linux.vnet.ibm.com>
 *              based on prototype from Frank Blaschka
 */

#define KMSG_COMPONENT "smc"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

#include <linux/module.h>
#include <linux/socket.h>
23
#include <linux/workqueue.h>
24
#include <linux/in.h>
25 26
#include <linux/sched/signal.h>

U
Ursula Braun 已提交
27
#include <net/sock.h>
28
#include <net/tcp.h>
29
#include <net/smc.h>
30
#include <asm/ioctls.h>
U
Ursula Braun 已提交
31 32

#include "smc.h"
33
#include "smc_clc.h"
U
Ursula Braun 已提交
34
#include "smc_llc.h"
35
#include "smc_cdc.h"
36
#include "smc_core.h"
37
#include "smc_ib.h"
38
#include "smc_pnet.h"
U
Ursula Braun 已提交
39
#include "smc_tx.h"
U
Ursula Braun 已提交
40
#include "smc_rx.h"
41
#include "smc_close.h"
U
Ursula Braun 已提交
42

43 44 45 46
static DEFINE_MUTEX(smc_create_lgr_pending);	/* serialize link group
						 * creation
						 */

47 48
static void smc_tcp_listen_work(struct work_struct *);

U
Ursula Braun 已提交
49 50 51 52 53 54 55
static void smc_set_keepalive(struct sock *sk, int val)
{
	struct smc_sock *smc = smc_sk(sk);

	smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val);
}

56 57 58 59
static struct smc_hashinfo smc_v4_hashinfo = {
	.lock = __RW_LOCK_UNLOCKED(smc_v4_hashinfo.lock),
};

60 61 62 63
static struct smc_hashinfo smc_v6_hashinfo = {
	.lock = __RW_LOCK_UNLOCKED(smc_v6_hashinfo.lock),
};

64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91
int smc_hash_sk(struct sock *sk)
{
	struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
	struct hlist_head *head;

	head = &h->ht;

	write_lock_bh(&h->lock);
	sk_add_node(sk, head);
	sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
	write_unlock_bh(&h->lock);

	return 0;
}
EXPORT_SYMBOL_GPL(smc_hash_sk);

void smc_unhash_sk(struct sock *sk)
{
	struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;

	write_lock_bh(&h->lock);
	if (sk_del_node_init(sk))
		sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
	write_unlock_bh(&h->lock);
}
EXPORT_SYMBOL_GPL(smc_unhash_sk);

struct proto smc_proto = {
U
Ursula Braun 已提交
92 93 94
	.name		= "SMC",
	.owner		= THIS_MODULE,
	.keepalive	= smc_set_keepalive,
95 96
	.hash		= smc_hash_sk,
	.unhash		= smc_unhash_sk,
U
Ursula Braun 已提交
97
	.obj_size	= sizeof(struct smc_sock),
98
	.h.smc_hash	= &smc_v4_hashinfo,
99
	.slab_flags	= SLAB_TYPESAFE_BY_RCU,
U
Ursula Braun 已提交
100
};
101
EXPORT_SYMBOL_GPL(smc_proto);
U
Ursula Braun 已提交
102

103 104 105 106 107 108 109 110 111 112 113 114
struct proto smc_proto6 = {
	.name		= "SMC6",
	.owner		= THIS_MODULE,
	.keepalive	= smc_set_keepalive,
	.hash		= smc_hash_sk,
	.unhash		= smc_unhash_sk,
	.obj_size	= sizeof(struct smc_sock),
	.h.smc_hash	= &smc_v6_hashinfo,
	.slab_flags	= SLAB_TYPESAFE_BY_RCU,
};
EXPORT_SYMBOL_GPL(smc_proto6);

U
Ursula Braun 已提交
115 116 117 118
static int smc_release(struct socket *sock)
{
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
119
	int rc = 0;
U
Ursula Braun 已提交
120 121 122 123 124

	if (!sk)
		goto out;

	smc = smc_sk(sk);
125 126 127 128 129 130 131
	if (sk->sk_state == SMC_LISTEN)
		/* smc_close_non_accepted() is called and acquires
		 * sock lock for child sockets again
		 */
		lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
	else
		lock_sock(sk);
U
Ursula Braun 已提交
132

133
	if (!smc->use_fallback) {
134 135 136 137
		rc = smc_close_active(smc);
		sock_set_flag(sk, SOCK_DEAD);
		sk->sk_shutdown |= SHUTDOWN_MASK;
	}
U
Ursula Braun 已提交
138 139 140 141
	if (smc->clcsock) {
		sock_release(smc->clcsock);
		smc->clcsock = NULL;
	}
142 143 144 145 146
	if (smc->use_fallback) {
		sock_put(sk); /* passive closing */
		sk->sk_state = SMC_CLOSED;
		sk->sk_state_change(sk);
	}
U
Ursula Braun 已提交
147 148 149 150

	/* detach socket */
	sock_orphan(sk);
	sock->sk = NULL;
151
	if (!smc->use_fallback && sk->sk_state == SMC_CLOSED)
152
		smc_conn_free(&smc->conn);
U
Ursula Braun 已提交
153 154
	release_sock(sk);

155 156
	sk->sk_prot->unhash(sk);
	sock_put(sk); /* final sock_put */
U
Ursula Braun 已提交
157
out:
158
	return rc;
U
Ursula Braun 已提交
159 160 161 162 163 164 165 166 167 168 169 170
}

static void smc_destruct(struct sock *sk)
{
	if (sk->sk_state != SMC_CLOSED)
		return;
	if (!sock_flag(sk, SOCK_DEAD))
		return;

	sk_refcnt_debug_dec(sk);
}

171 172
static struct sock *smc_sock_alloc(struct net *net, struct socket *sock,
				   int protocol)
U
Ursula Braun 已提交
173 174
{
	struct smc_sock *smc;
175
	struct proto *prot;
U
Ursula Braun 已提交
176 177
	struct sock *sk;

178 179
	prot = (protocol == SMCPROTO_SMC6) ? &smc_proto6 : &smc_proto;
	sk = sk_alloc(net, PF_SMC, GFP_KERNEL, prot, 0);
U
Ursula Braun 已提交
180 181 182 183 184 185
	if (!sk)
		return NULL;

	sock_init_data(sock, sk); /* sets sk_refcnt to 1 */
	sk->sk_state = SMC_INIT;
	sk->sk_destruct = smc_destruct;
186
	sk->sk_protocol = protocol;
U
Ursula Braun 已提交
187
	smc = smc_sk(sk);
188
	INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
189
	INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work);
190 191
	INIT_LIST_HEAD(&smc->accept_q);
	spin_lock_init(&smc->accept_q_lock);
192
	spin_lock_init(&smc->conn.send_lock);
193
	sk->sk_prot->hash(sk);
194
	sk_refcnt_debug_inc(sk);
U
Ursula Braun 已提交
195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214

	return sk;
}

static int smc_bind(struct socket *sock, struct sockaddr *uaddr,
		    int addr_len)
{
	struct sockaddr_in *addr = (struct sockaddr_in *)uaddr;
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
	int rc;

	smc = smc_sk(sk);

	/* replicate tests from inet_bind(), to be safe wrt. future changes */
	rc = -EINVAL;
	if (addr_len < sizeof(struct sockaddr_in))
		goto out;

	rc = -EAFNOSUPPORT;
215 216 217 218
	if (addr->sin_family != AF_INET &&
	    addr->sin_family != AF_INET6 &&
	    addr->sin_family != AF_UNSPEC)
		goto out;
U
Ursula Braun 已提交
219
	/* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */
220 221
	if (addr->sin_family == AF_UNSPEC &&
	    addr->sin_addr.s_addr != htonl(INADDR_ANY))
U
Ursula Braun 已提交
222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290
		goto out;

	lock_sock(sk);

	/* Check if socket is already active */
	rc = -EINVAL;
	if (sk->sk_state != SMC_INIT)
		goto out_rel;

	smc->clcsock->sk->sk_reuse = sk->sk_reuse;
	rc = kernel_bind(smc->clcsock, uaddr, addr_len);

out_rel:
	release_sock(sk);
out:
	return rc;
}

static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk,
				   unsigned long mask)
{
	/* options we don't get control via setsockopt for */
	nsk->sk_type = osk->sk_type;
	nsk->sk_sndbuf = osk->sk_sndbuf;
	nsk->sk_rcvbuf = osk->sk_rcvbuf;
	nsk->sk_sndtimeo = osk->sk_sndtimeo;
	nsk->sk_rcvtimeo = osk->sk_rcvtimeo;
	nsk->sk_mark = osk->sk_mark;
	nsk->sk_priority = osk->sk_priority;
	nsk->sk_rcvlowat = osk->sk_rcvlowat;
	nsk->sk_bound_dev_if = osk->sk_bound_dev_if;
	nsk->sk_err = osk->sk_err;

	nsk->sk_flags &= ~mask;
	nsk->sk_flags |= osk->sk_flags & mask;
}

#define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \
			     (1UL << SOCK_KEEPOPEN) | \
			     (1UL << SOCK_LINGER) | \
			     (1UL << SOCK_BROADCAST) | \
			     (1UL << SOCK_TIMESTAMP) | \
			     (1UL << SOCK_DBG) | \
			     (1UL << SOCK_RCVTSTAMP) | \
			     (1UL << SOCK_RCVTSTAMPNS) | \
			     (1UL << SOCK_LOCALROUTE) | \
			     (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
			     (1UL << SOCK_RXQ_OVFL) | \
			     (1UL << SOCK_WIFI_STATUS) | \
			     (1UL << SOCK_NOFCS) | \
			     (1UL << SOCK_FILTER_LOCKED))
/* copy only relevant settings and flags of SOL_SOCKET level from smc to
 * clc socket (since smc is not called for these options from net/core)
 */
static void smc_copy_sock_settings_to_clc(struct smc_sock *smc)
{
	smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC);
}

#define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \
			     (1UL << SOCK_KEEPOPEN) | \
			     (1UL << SOCK_LINGER) | \
			     (1UL << SOCK_DBG))
/* copy only settings and flags relevant for smc from clc to smc socket */
static void smc_copy_sock_settings_to_smc(struct smc_sock *smc)
{
	smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC);
}

291 292 293
/* register a new rmb, optionally send confirm_rkey msg to register with peer */
static int smc_reg_rmb(struct smc_link *link, struct smc_buf_desc *rmb_desc,
		       bool conf_rkey)
K
Karsten Graul 已提交
294 295
{
	/* register memory region for new rmb */
296 297
	if (smc_wr_reg_send(link, rmb_desc->mr_rx[SMC_SINGLE_LINK])) {
		rmb_desc->regerr = 1;
K
Karsten Graul 已提交
298
		return -EFAULT;
299
	}
300 301 302 303 304 305 306
	if (!conf_rkey)
		return 0;
	/* exchange confirm_rkey msg with peer */
	if (smc_llc_do_confirm_rkey(link, rmb_desc)) {
		rmb_desc->regerr = 1;
		return -EFAULT;
	}
K
Karsten Graul 已提交
307 308 309
	return 0;
}

310
static int smc_clnt_conf_first_link(struct smc_sock *smc)
U
Ursula Braun 已提交
311
{
312
	struct net *net = sock_net(smc->clcsock->sk);
U
Ursula Braun 已提交
313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330
	struct smc_link_group *lgr = smc->conn.lgr;
	struct smc_link *link;
	int rest;
	int rc;

	link = &lgr->lnk[SMC_SINGLE_LINK];
	/* receive CONFIRM LINK request from server over RoCE fabric */
	rest = wait_for_completion_interruptible_timeout(
		&link->llc_confirm,
		SMC_LLC_WAIT_FIRST_TIME);
	if (rest <= 0) {
		struct smc_clc_msg_decline dclc;

		rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
				      SMC_CLC_DECLINE);
		return rc;
	}

331 332 333
	if (link->llc_confirm_rc)
		return SMC_CLC_DECL_RMBE_EC;

U
Ursula Braun 已提交
334 335 336 337 338
	rc = smc_ib_modify_qp_rts(link);
	if (rc)
		return SMC_CLC_DECL_INTERR;

	smc_wr_remember_qp_attr(link);
339

340
	if (smc_reg_rmb(link, smc->conn.rmb_desc, false))
341 342
		return SMC_CLC_DECL_INTERR;

U
Ursula Braun 已提交
343 344 345
	/* send CONFIRM LINK response over RoCE fabric */
	rc = smc_llc_send_confirm_link(link,
				       link->smcibdev->mac[link->ibport - 1],
346 347
				       &link->smcibdev->gid[link->ibport - 1],
				       SMC_LLC_RESP);
U
Ursula Braun 已提交
348 349 350
	if (rc < 0)
		return SMC_CLC_DECL_TCL;

351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369
	/* receive ADD LINK request from server over RoCE fabric */
	rest = wait_for_completion_interruptible_timeout(&link->llc_add,
							 SMC_LLC_WAIT_TIME);
	if (rest <= 0) {
		struct smc_clc_msg_decline dclc;

		rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
				      SMC_CLC_DECLINE);
		return rc;
	}

	/* send add link reject message, only one link supported for now */
	rc = smc_llc_send_add_link(link,
				   link->smcibdev->mac[link->ibport - 1],
				   &link->smcibdev->gid[link->ibport - 1],
				   SMC_LLC_RESP);
	if (rc < 0)
		return SMC_CLC_DECL_TCL;

370
	smc_llc_link_active(link, net->ipv4.sysctl_tcp_keepalive_time);
371

372
	return 0;
U
Ursula Braun 已提交
373 374
}

375 376 377
static void smc_conn_save_peer_info(struct smc_sock *smc,
				    struct smc_clc_msg_accept_confirm *clc)
{
378 379
	int bufsize = smc_uncompress_bufsize(clc->rmbe_size);

380
	smc->conn.peer_rmbe_idx = clc->rmbe_idx;
381
	smc->conn.local_tx_ctrl.token = ntohl(clc->rmbe_alert_token);
382
	smc->conn.peer_rmbe_size = bufsize;
U
Ursula Braun 已提交
383
	atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
384
	smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1);
385 386 387 388 389 390 391 392 393 394 395 396
}

static void smc_link_save_peer_info(struct smc_link *link,
				    struct smc_clc_msg_accept_confirm *clc)
{
	link->peer_qpn = ntoh24(clc->qpn);
	memcpy(link->peer_gid, clc->lcl.gid, SMC_GID_SIZE);
	memcpy(link->peer_mac, clc->lcl.mac, sizeof(link->peer_mac));
	link->peer_psn = ntoh24(clc->psn);
	link->peer_mtu = clc->qp_mtu;
}

397 398
/* fall back during connect */
static int smc_connect_fallback(struct smc_sock *smc)
399
{
400 401 402 403 404 405
	smc->use_fallback = true;
	smc_copy_sock_settings_to_clc(smc);
	if (smc->sk.sk_state == SMC_INIT)
		smc->sk.sk_state = SMC_ACTIVE;
	return 0;
}
406

407 408 409 410
/* decline and fall back during connect */
static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code)
{
	int rc;
411

412 413 414 415 416 417
	if (reason_code < 0) /* error, fallback is not possible */
		return reason_code;
	if (reason_code != SMC_CLC_DECL_REPLY) {
		rc = smc_clc_send_decline(smc, reason_code);
		if (rc < 0)
			return rc;
U
Ursula Braun 已提交
418
	}
419 420
	return smc_connect_fallback(smc);
}
U
Ursula Braun 已提交
421

422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440
/* abort connecting */
static int smc_connect_abort(struct smc_sock *smc, int reason_code,
			     int local_contact)
{
	if (local_contact == SMC_FIRST_CONTACT)
		smc_lgr_forget(smc->conn.lgr);
	mutex_unlock(&smc_create_lgr_pending);
	smc_conn_free(&smc->conn);
	if (reason_code < 0 && smc->sk.sk_state == SMC_INIT)
		sock_put(&smc->sk); /* passive closing */
	return reason_code;
}

/* check if there is a rdma device available for this connection. */
/* called for connect and listen */
static int smc_check_rdma(struct smc_sock *smc, struct smc_ib_device **ibdev,
			  u8 *ibport)
{
	int reason_code = 0;
441 442 443 444 445

	/* PNET table look up: search active ib_device and port
	 * within same PNETID that also contains the ethernet device
	 * used for the internal TCP socket
	 */
446 447
	smc_pnet_find_roce_resource(smc->clcsock->sk, ibdev, ibport);
	if (!(*ibdev))
448
		reason_code = SMC_CLC_DECL_CNFERR; /* configuration error */
449 450 451 452 453 454 455 456 457 458

	return reason_code;
}

/* CLC handshake during connect */
static int smc_connect_clc(struct smc_sock *smc,
			   struct smc_clc_msg_accept_confirm *aclc,
			   struct smc_ib_device *ibdev, u8 ibport)
{
	int rc = 0;
459 460

	/* do inband token exchange */
461 462 463
	rc = smc_clc_send_proposal(smc, ibdev, ibport);
	if (rc)
		return rc;
464
	/* receive SMC Accept CLC message */
465 466 467 468 469 470 471 472 473 474 475
	return smc_clc_wait_msg(smc, aclc, sizeof(*aclc), SMC_CLC_ACCEPT);
}

/* setup for RDMA connection of client */
static int smc_connect_rdma(struct smc_sock *smc,
			    struct smc_clc_msg_accept_confirm *aclc,
			    struct smc_ib_device *ibdev, u8 ibport)
{
	int local_contact = SMC_FIRST_CONTACT;
	struct smc_link *link;
	int reason_code = 0;
476

477
	mutex_lock(&smc_create_lgr_pending);
478 479
	local_contact = smc_conn_create(smc, ibdev, ibport, &aclc->lcl,
					aclc->hdr.flag);
480
	if (local_contact < 0) {
481
		if (local_contact == -ENOMEM)
482
			reason_code = SMC_CLC_DECL_MEM;/* insufficient memory*/
483
		else if (local_contact == -ENOLINK)
484
			reason_code = SMC_CLC_DECL_SYNCERR; /* synchr. error */
485 486
		else
			reason_code = SMC_CLC_DECL_INTERR; /* other error */
487
		return smc_connect_abort(smc, reason_code, 0);
488 489
	}
	link = &smc->conn.lgr->lnk[SMC_SINGLE_LINK];
490

491
	smc_conn_save_peer_info(smc, aclc);
U
Ursula Braun 已提交
492

493
	/* create send buffer and rmb */
494 495
	if (smc_buf_create(smc))
		return smc_connect_abort(smc, SMC_CLC_DECL_MEM, local_contact);
U
Ursula Braun 已提交
496

497
	if (local_contact == SMC_FIRST_CONTACT)
498
		smc_link_save_peer_info(link, aclc);
499

500 501 502
	if (smc_rmb_rtoken_handling(&smc->conn, aclc))
		return smc_connect_abort(smc, SMC_CLC_DECL_INTERR,
					 local_contact);
503

504 505 506
	smc_close_init(smc);
	smc_rx_init(smc);

507
	if (local_contact == SMC_FIRST_CONTACT) {
508 509 510
		if (smc_ib_ready_link(link))
			return smc_connect_abort(smc, SMC_CLC_DECL_INTERR,
						 local_contact);
511
	} else {
512 513 514 515
		if (!smc->conn.rmb_desc->reused &&
		    smc_reg_rmb(link, smc->conn.rmb_desc, true))
			return smc_connect_abort(smc, SMC_CLC_DECL_INTERR,
						 local_contact);
516
	}
517
	smc_rmb_sync_sg_for_device(&smc->conn);
518

519 520 521 522 523
	reason_code = smc_clc_send_confirm(smc);
	if (reason_code)
		return smc_connect_abort(smc, reason_code, local_contact);

	smc_tx_init(smc);
524

U
Ursula Braun 已提交
525 526
	if (local_contact == SMC_FIRST_CONTACT) {
		/* QP confirmation over RoCE fabric */
527
		reason_code = smc_clnt_conf_first_link(smc);
528 529 530
		if (reason_code)
			return smc_connect_abort(smc, reason_code,
						 local_contact);
U
Ursula Braun 已提交
531
	}
532
	mutex_unlock(&smc_create_lgr_pending);
U
Ursula Braun 已提交
533

534
	smc_copy_sock_settings_to_clc(smc);
535 536
	if (smc->sk.sk_state == SMC_INIT)
		smc->sk.sk_state = SMC_ACTIVE;
537

538 539
	return 0;
}
540

541 542 543 544 545 546 547
/* perform steps before actually connecting */
static int __smc_connect(struct smc_sock *smc)
{
	struct smc_clc_msg_accept_confirm aclc;
	struct smc_ib_device *ibdev;
	int rc = 0;
	u8 ibport;
548

549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576
	sock_hold(&smc->sk); /* sock put in passive closing */

	if (smc->use_fallback)
		return smc_connect_fallback(smc);

	/* if peer has not signalled SMC-capability, fall back */
	if (!tcp_sk(smc->clcsock->sk)->syn_smc)
		return smc_connect_fallback(smc);

	/* IPSec connections opt out of SMC-R optimizations */
	if (using_ipsec(smc))
		return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC);

	/* check if a RDMA device is available; if not, fall back */
	if (smc_check_rdma(smc, &ibdev, &ibport))
		return smc_connect_decline_fallback(smc, SMC_CLC_DECL_CNFERR);

	/* perform CLC handshake */
	rc = smc_connect_clc(smc, &aclc, ibdev, ibport);
	if (rc)
		return smc_connect_decline_fallback(smc, rc);

	/* connect using rdma */
	rc = smc_connect_rdma(smc, &aclc, ibdev, ibport);
	if (rc)
		return smc_connect_decline_fallback(smc, rc);

	return 0;
577 578
}

U
Ursula Braun 已提交
579 580 581 582 583 584 585 586 587 588 589 590
static int smc_connect(struct socket *sock, struct sockaddr *addr,
		       int alen, int flags)
{
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
	int rc = -EINVAL;

	smc = smc_sk(sk);

	/* separate smc parameter checking to be safe */
	if (alen < sizeof(addr->sa_family))
		goto out_err;
591
	if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
U
Ursula Braun 已提交
592 593 594 595 596 597 598 599 600 601 602 603 604 605 606
		goto out_err;

	lock_sock(sk);
	switch (sk->sk_state) {
	default:
		goto out;
	case SMC_ACTIVE:
		rc = -EISCONN;
		goto out;
	case SMC_INIT:
		rc = 0;
		break;
	}

	smc_copy_sock_settings_to_clc(smc);
U
Ursula Braun 已提交
607
	tcp_sk(smc->clcsock->sk)->syn_smc = 1;
U
Ursula Braun 已提交
608 609 610 611
	rc = kernel_connect(smc->clcsock, addr, alen, flags);
	if (rc)
		goto out;

612
	rc = __smc_connect(smc);
613 614 615 616
	if (rc < 0)
		goto out;
	else
		rc = 0; /* success cases including fallback */
U
Ursula Braun 已提交
617 618 619 620 621 622 623 624 625

out:
	release_sock(sk);
out_err:
	return rc;
}

static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc)
{
626 627
	struct socket *new_clcsock = NULL;
	struct sock *lsk = &lsmc->sk;
U
Ursula Braun 已提交
628 629 630
	struct sock *new_sk;
	int rc;

631
	release_sock(lsk);
632
	new_sk = smc_sock_alloc(sock_net(lsk), NULL, lsk->sk_protocol);
U
Ursula Braun 已提交
633 634
	if (!new_sk) {
		rc = -ENOMEM;
635
		lsk->sk_err = ENOMEM;
U
Ursula Braun 已提交
636
		*new_smc = NULL;
637
		lock_sock(lsk);
U
Ursula Braun 已提交
638 639 640 641 642
		goto out;
	}
	*new_smc = smc_sk(new_sk);

	rc = kernel_accept(lsmc->clcsock, &new_clcsock, 0);
643
	lock_sock(lsk);
644
	if  (rc < 0)
645
		lsk->sk_err = -rc;
646
	if (rc < 0 || lsk->sk_state == SMC_CLOSED) {
647 648 649 650
		if (new_clcsock)
			sock_release(new_clcsock);
		new_sk->sk_state = SMC_CLOSED;
		sock_set_flag(new_sk, SOCK_DEAD);
651
		new_sk->sk_prot->unhash(new_sk);
652
		sock_put(new_sk); /* final */
U
Ursula Braun 已提交
653 654 655 656 657 658 659 660 661
		*new_smc = NULL;
		goto out;
	}

	(*new_smc)->clcsock = new_clcsock;
out:
	return rc;
}

662 663 664 665 666 667 668
/* add a just created sock to the accept queue of the listen sock as
 * candidate for a following socket accept call from user space
 */
static void smc_accept_enqueue(struct sock *parent, struct sock *sk)
{
	struct smc_sock *par = smc_sk(parent);

669
	sock_hold(sk); /* sock_put in smc_accept_unlink () */
670 671 672 673 674 675 676 677 678 679 680 681 682 683 684
	spin_lock(&par->accept_q_lock);
	list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q);
	spin_unlock(&par->accept_q_lock);
	sk_acceptq_added(parent);
}

/* remove a socket from the accept queue of its parental listening socket */
static void smc_accept_unlink(struct sock *sk)
{
	struct smc_sock *par = smc_sk(sk)->listen_smc;

	spin_lock(&par->accept_q_lock);
	list_del_init(&smc_sk(sk)->accept_q);
	spin_unlock(&par->accept_q_lock);
	sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk);
685
	sock_put(sk); /* sock_hold in smc_accept_enqueue */
686 687 688 689 690
}

/* remove a sock from the accept queue to bind it to a new socket created
 * for a socket accept call from user space
 */
691 692
struct sock *smc_accept_dequeue(struct sock *parent,
				struct socket *new_sock)
693 694 695 696 697 698 699 700 701
{
	struct smc_sock *isk, *n;
	struct sock *new_sk;

	list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) {
		new_sk = (struct sock *)isk;

		smc_accept_unlink(new_sk);
		if (new_sk->sk_state == SMC_CLOSED) {
702 703 704 705
			if (isk->clcsock) {
				sock_release(isk->clcsock);
				isk->clcsock = NULL;
			}
706
			new_sk->sk_prot->unhash(new_sk);
707
			sock_put(new_sk); /* final */
708 709 710 711 712 713 714 715 716 717
			continue;
		}
		if (new_sock)
			sock_graft(new_sk, new_sock);
		return new_sk;
	}
	return NULL;
}

/* clean up for a created but never accepted sock */
718
void smc_close_non_accepted(struct sock *sk)
719 720 721
{
	struct smc_sock *smc = smc_sk(sk);

722 723 724 725
	lock_sock(sk);
	if (!sk->sk_lingertime)
		/* wait for peer closing */
		sk->sk_lingertime = SMC_MAX_STREAM_WAIT_TIMEOUT;
726
	if (!smc->use_fallback) {
727
		smc_close_active(smc);
728 729 730
		sock_set_flag(sk, SOCK_DEAD);
		sk->sk_shutdown |= SHUTDOWN_MASK;
	}
731 732 733 734 735 736 737
	if (smc->clcsock) {
		struct socket *tcp;

		tcp = smc->clcsock;
		smc->clcsock = NULL;
		sock_release(tcp);
	}
738
	if (smc->use_fallback) {
739 740 741 742 743
		sock_put(sk); /* passive closing */
		sk->sk_state = SMC_CLOSED;
	} else {
		if (sk->sk_state == SMC_CLOSED)
			smc_conn_free(&smc->conn);
744 745
	}
	release_sock(sk);
746 747
	sk->sk_prot->unhash(sk);
	sock_put(sk); /* final sock_put */
748 749
}

U
Ursula Braun 已提交
750 751
static int smc_serv_conf_first_link(struct smc_sock *smc)
{
752
	struct net *net = sock_net(smc->clcsock->sk);
U
Ursula Braun 已提交
753 754 755 756 757 758
	struct smc_link_group *lgr = smc->conn.lgr;
	struct smc_link *link;
	int rest;
	int rc;

	link = &lgr->lnk[SMC_SINGLE_LINK];
759

760
	if (smc_reg_rmb(link, smc->conn.rmb_desc, false))
761 762
		return SMC_CLC_DECL_INTERR;

U
Ursula Braun 已提交
763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779
	/* send CONFIRM LINK request to client over the RoCE fabric */
	rc = smc_llc_send_confirm_link(link,
				       link->smcibdev->mac[link->ibport - 1],
				       &link->smcibdev->gid[link->ibport - 1],
				       SMC_LLC_REQ);
	if (rc < 0)
		return SMC_CLC_DECL_TCL;

	/* receive CONFIRM LINK response from client over the RoCE fabric */
	rest = wait_for_completion_interruptible_timeout(
		&link->llc_confirm_resp,
		SMC_LLC_WAIT_FIRST_TIME);
	if (rest <= 0) {
		struct smc_clc_msg_decline dclc;

		rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
				      SMC_CLC_DECLINE);
780
		return rc;
U
Ursula Braun 已提交
781 782
	}

783 784 785
	if (link->llc_confirm_resp_rc)
		return SMC_CLC_DECL_RMBE_EC;

786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804
	/* send ADD LINK request to client over the RoCE fabric */
	rc = smc_llc_send_add_link(link,
				   link->smcibdev->mac[link->ibport - 1],
				   &link->smcibdev->gid[link->ibport - 1],
				   SMC_LLC_REQ);
	if (rc < 0)
		return SMC_CLC_DECL_TCL;

	/* receive ADD LINK response from client over the RoCE fabric */
	rest = wait_for_completion_interruptible_timeout(&link->llc_add_resp,
							 SMC_LLC_WAIT_TIME);
	if (rest <= 0) {
		struct smc_clc_msg_decline dclc;

		rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
				      SMC_CLC_DECLINE);
		return rc;
	}

805
	smc_llc_link_active(link, net->ipv4.sysctl_tcp_keepalive_time);
806

807
	return 0;
U
Ursula Braun 已提交
808 809
}

810 811
/* listen worker: finish */
static void smc_listen_out(struct smc_sock *new_smc)
812 813 814 815
{
	struct smc_sock *lsmc = new_smc->listen_smc;
	struct sock *newsmcsk = &new_smc->sk;

816 817 818 819 820
	lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING);
	if (lsmc->sk.sk_state == SMC_LISTEN) {
		smc_accept_enqueue(&lsmc->sk, newsmcsk);
	} else { /* no longer listening */
		smc_close_non_accepted(newsmcsk);
U
Ursula Braun 已提交
821
	}
822
	release_sock(&lsmc->sk);
U
Ursula Braun 已提交
823

824 825 826 827
	/* Wake up accept */
	lsmc->sk.sk_data_ready(&lsmc->sk);
	sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */
}
828

829 830 831 832
/* listen worker: finish in state connected */
static void smc_listen_out_connected(struct smc_sock *new_smc)
{
	struct sock *newsmcsk = &new_smc->sk;
833

834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871
	sk_refcnt_debug_inc(newsmcsk);
	if (newsmcsk->sk_state == SMC_INIT)
		newsmcsk->sk_state = SMC_ACTIVE;

	smc_listen_out(new_smc);
}

/* listen worker: finish in error state */
static void smc_listen_out_err(struct smc_sock *new_smc)
{
	struct sock *newsmcsk = &new_smc->sk;

	if (newsmcsk->sk_state == SMC_INIT)
		sock_put(&new_smc->sk); /* passive closing */
	newsmcsk->sk_state = SMC_CLOSED;
	smc_conn_free(&new_smc->conn);

	smc_listen_out(new_smc);
}

/* listen worker: decline and fall back if possible */
static void smc_listen_decline(struct smc_sock *new_smc, int reason_code,
			       int local_contact)
{
	/* RDMA setup failed, switch back to TCP */
	if (local_contact == SMC_FIRST_CONTACT)
		smc_lgr_forget(new_smc->conn.lgr);
	if (reason_code < 0) { /* error, no fallback possible */
		smc_listen_out_err(new_smc);
		return;
	}
	smc_conn_free(&new_smc->conn);
	new_smc->use_fallback = true;
	if (reason_code && reason_code != SMC_CLC_DECL_REPLY) {
		if (smc_clc_send_decline(new_smc, reason_code) < 0) {
			smc_listen_out_err(new_smc);
			return;
		}
872
	}
873 874 875 876 877 878 879 880 881
	smc_listen_out_connected(new_smc);
}

/* listen worker: check prefixes */
static int smc_listen_rdma_check(struct smc_sock *new_smc,
				 struct smc_clc_msg_proposal *pclc)
{
	struct smc_clc_msg_proposal_prefix *pclc_prfx;
	struct socket *newclcsock = new_smc->clcsock;
882

883
	pclc_prfx = smc_clc_proposal_get_prefix(pclc);
884 885
	if (smc_clc_prfx_match(newclcsock, pclc_prfx))
		return SMC_CLC_DECL_CNFERR;
886

887 888
	return 0;
}
889

890 891 892 893 894 895
/* listen worker: initialize connection and buffers */
static int smc_listen_rdma_init(struct smc_sock *new_smc,
				struct smc_clc_msg_proposal *pclc,
				struct smc_ib_device *ibdev, u8 ibport,
				int *local_contact)
{
896
	/* allocate connection / link group */
897 898 899 900 901
	*local_contact = smc_conn_create(new_smc, ibdev, ibport, &pclc->lcl, 0);
	if (*local_contact < 0) {
		if (*local_contact == -ENOMEM)
			return SMC_CLC_DECL_MEM;/* insufficient memory*/
		return SMC_CLC_DECL_INTERR; /* other error */
902
	}
903

904
	/* create send buffer and rmb */
905 906
	if (smc_buf_create(new_smc))
		return SMC_CLC_DECL_MEM;
907

908 909 910 911 912 913 914
	return 0;
}

/* listen worker: register buffers */
static int smc_listen_rdma_reg(struct smc_sock *new_smc, int local_contact)
{
	struct smc_link *link = &new_smc->conn.lgr->lnk[SMC_SINGLE_LINK];
915

916
	if (local_contact != SMC_FIRST_CONTACT) {
K
Karsten Graul 已提交
917
		if (!new_smc->conn.rmb_desc->reused) {
918 919
			if (smc_reg_rmb(link, new_smc->conn.rmb_desc, true))
				return SMC_CLC_DECL_INTERR;
920 921
		}
	}
922
	smc_rmb_sync_sg_for_device(&new_smc->conn);
923

924 925 926 927 928 929 930 931 932 933
	return 0;
}

/* listen worker: finish RDMA setup */
static void smc_listen_rdma_finish(struct smc_sock *new_smc,
				   struct smc_clc_msg_accept_confirm *cclc,
				   int local_contact)
{
	struct smc_link *link = &new_smc->conn.lgr->lnk[SMC_SINGLE_LINK];
	int reason_code = 0;
934

935
	if (local_contact == SMC_FIRST_CONTACT)
936
		smc_link_save_peer_info(link, cclc);
937

938
	if (smc_rmb_rtoken_handling(&new_smc->conn, cclc)) {
939
		reason_code = SMC_CLC_DECL_INTERR;
940
		goto decline;
941 942 943
	}

	if (local_contact == SMC_FIRST_CONTACT) {
944
		if (smc_ib_ready_link(link)) {
945
			reason_code = SMC_CLC_DECL_INTERR;
946
			goto decline;
947
		}
U
Ursula Braun 已提交
948 949
		/* QP confirmation over RoCE fabric */
		reason_code = smc_serv_conf_first_link(new_smc);
950 951
		if (reason_code)
			goto decline;
952
	}
953
	return;
954

955
decline:
956
	mutex_unlock(&smc_create_lgr_pending);
957 958
	smc_listen_decline(new_smc, reason_code, local_contact);
}
U
Ursula Braun 已提交
959

960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977
/* setup for RDMA connection of server */
static void smc_listen_work(struct work_struct *work)
{
	struct smc_sock *new_smc = container_of(work, struct smc_sock,
						smc_listen_work);
	struct socket *newclcsock = new_smc->clcsock;
	struct smc_clc_msg_accept_confirm cclc;
	struct smc_clc_msg_proposal *pclc;
	struct smc_ib_device *ibdev;
	u8 buf[SMC_CLC_MAX_LEN];
	int local_contact = 0;
	int reason_code = 0;
	int rc = 0;
	u8 ibport;

	if (new_smc->use_fallback) {
		smc_listen_out_connected(new_smc);
		return;
978 979
	}

980 981 982 983 984 985
	/* check if peer is smc capable */
	if (!tcp_sk(newclcsock->sk)->syn_smc) {
		new_smc->use_fallback = true;
		smc_listen_out_connected(new_smc);
		return;
	}
986

987 988 989 990 991 992 993 994 995
	/* do inband token exchange -
	 * wait for and receive SMC Proposal CLC message
	 */
	pclc = (struct smc_clc_msg_proposal *)&buf;
	reason_code = smc_clc_wait_msg(new_smc, pclc, SMC_CLC_MAX_LEN,
				       SMC_CLC_PROPOSAL);
	if (reason_code) {
		smc_listen_decline(new_smc, reason_code, 0);
		return;
996 997
	}

998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040
	/* IPSec connections opt out of SMC-R optimizations */
	if (using_ipsec(new_smc)) {
		smc_listen_decline(new_smc, SMC_CLC_DECL_IPSEC, 0);
		return;
	}

	mutex_lock(&smc_create_lgr_pending);
	smc_close_init(new_smc);
	smc_rx_init(new_smc);
	smc_tx_init(new_smc);

	/* check if RDMA is available */
	if (smc_check_rdma(new_smc, &ibdev, &ibport) ||
	    smc_listen_rdma_check(new_smc, pclc) ||
	    smc_listen_rdma_init(new_smc, pclc, ibdev, ibport,
				 &local_contact) ||
	    smc_listen_rdma_reg(new_smc, local_contact)) {
		/* SMC not supported, decline */
		mutex_unlock(&smc_create_lgr_pending);
		smc_listen_decline(new_smc, SMC_CLC_DECL_CNFERR, local_contact);
		return;
	}

	/* send SMC Accept CLC message */
	rc = smc_clc_send_accept(new_smc, local_contact);
	if (rc) {
		mutex_unlock(&smc_create_lgr_pending);
		smc_listen_decline(new_smc, rc, local_contact);
		return;
	}

	/* receive SMC Confirm CLC message */
	reason_code = smc_clc_wait_msg(new_smc, &cclc, sizeof(cclc),
				       SMC_CLC_CONFIRM);
	if (reason_code) {
		mutex_unlock(&smc_create_lgr_pending);
		smc_listen_decline(new_smc, reason_code, local_contact);
		return;
	}

	/* finish worker */
	smc_listen_rdma_finish(new_smc, &cclc, local_contact);
	smc_conn_save_peer_info(new_smc, &cclc);
1041
	mutex_unlock(&smc_create_lgr_pending);
1042
	smc_listen_out_connected(new_smc);
1043 1044 1045 1046 1047 1048
}

static void smc_tcp_listen_work(struct work_struct *work)
{
	struct smc_sock *lsmc = container_of(work, struct smc_sock,
					     tcp_listen_work);
1049
	struct sock *lsk = &lsmc->sk;
1050 1051 1052
	struct smc_sock *new_smc;
	int rc = 0;

1053 1054
	lock_sock(lsk);
	while (lsk->sk_state == SMC_LISTEN) {
1055 1056 1057 1058 1059 1060 1061
		rc = smc_clcsock_accept(lsmc, &new_smc);
		if (rc)
			goto out;
		if (!new_smc)
			continue;

		new_smc->listen_smc = lsmc;
1062
		new_smc->use_fallback = lsmc->use_fallback;
1063
		sock_hold(lsk); /* sock_put in smc_listen_work */
1064 1065
		INIT_WORK(&new_smc->smc_listen_work, smc_listen_work);
		smc_copy_sock_settings_to_smc(new_smc);
1066 1067 1068
		sock_hold(&new_smc->sk); /* sock_put in passive closing */
		if (!schedule_work(&new_smc->smc_listen_work))
			sock_put(&new_smc->sk);
1069 1070 1071
	}

out:
1072
	release_sock(lsk);
1073
	sock_put(&lsmc->sk); /* sock_hold in smc_listen */
1074 1075
}

U
Ursula Braun 已提交
1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097
static int smc_listen(struct socket *sock, int backlog)
{
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
	int rc;

	smc = smc_sk(sk);
	lock_sock(sk);

	rc = -EINVAL;
	if ((sk->sk_state != SMC_INIT) && (sk->sk_state != SMC_LISTEN))
		goto out;

	rc = 0;
	if (sk->sk_state == SMC_LISTEN) {
		sk->sk_max_ack_backlog = backlog;
		goto out;
	}
	/* some socket options are handled in core, so we could not apply
	 * them to the clc socket -- copy smc socket options to clc socket
	 */
	smc_copy_sock_settings_to_clc(smc);
1098 1099
	if (!smc->use_fallback)
		tcp_sk(smc->clcsock->sk)->syn_smc = 1;
U
Ursula Braun 已提交
1100 1101 1102 1103 1104 1105 1106

	rc = kernel_listen(smc->clcsock, backlog);
	if (rc)
		goto out;
	sk->sk_max_ack_backlog = backlog;
	sk->sk_ack_backlog = 0;
	sk->sk_state = SMC_LISTEN;
1107
	INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
1108 1109 1110
	sock_hold(sk); /* sock_hold in tcp_listen_worker */
	if (!schedule_work(&smc->tcp_listen_work))
		sock_put(sk);
U
Ursula Braun 已提交
1111 1112 1113 1114 1115 1116 1117

out:
	release_sock(sk);
	return rc;
}

static int smc_accept(struct socket *sock, struct socket *new_sock,
1118
		      int flags, bool kern)
U
Ursula Braun 已提交
1119
{
1120 1121
	struct sock *sk = sock->sk, *nsk;
	DECLARE_WAITQUEUE(wait, current);
U
Ursula Braun 已提交
1122
	struct smc_sock *lsmc;
1123 1124
	long timeo;
	int rc = 0;
U
Ursula Braun 已提交
1125 1126

	lsmc = smc_sk(sk);
1127
	sock_hold(sk); /* sock_put below */
U
Ursula Braun 已提交
1128 1129 1130 1131
	lock_sock(sk);

	if (lsmc->sk.sk_state != SMC_LISTEN) {
		rc = -EINVAL;
1132
		release_sock(sk);
U
Ursula Braun 已提交
1133 1134 1135
		goto out;
	}

1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156
	/* Wait for an incoming connection */
	timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
	add_wait_queue_exclusive(sk_sleep(sk), &wait);
	while (!(nsk = smc_accept_dequeue(sk, new_sock))) {
		set_current_state(TASK_INTERRUPTIBLE);
		if (!timeo) {
			rc = -EAGAIN;
			break;
		}
		release_sock(sk);
		timeo = schedule_timeout(timeo);
		/* wakeup by sk_data_ready in smc_listen_work() */
		sched_annotate_sleep();
		lock_sock(sk);
		if (signal_pending(current)) {
			rc = sock_intr_errno(timeo);
			break;
		}
	}
	set_current_state(TASK_RUNNING);
	remove_wait_queue(sk_sleep(sk), &wait);
U
Ursula Braun 已提交
1157

1158 1159
	if (!rc)
		rc = sock_error(nsk);
1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176
	release_sock(sk);
	if (rc)
		goto out;

	if (lsmc->sockopt_defer_accept && !(flags & O_NONBLOCK)) {
		/* wait till data arrives on the socket */
		timeo = msecs_to_jiffies(lsmc->sockopt_defer_accept *
								MSEC_PER_SEC);
		if (smc_sk(nsk)->use_fallback) {
			struct sock *clcsk = smc_sk(nsk)->clcsock->sk;

			lock_sock(clcsk);
			if (skb_queue_empty(&clcsk->sk_receive_queue))
				sk_wait_data(clcsk, &timeo, NULL);
			release_sock(clcsk);
		} else if (!atomic_read(&smc_sk(nsk)->conn.bytes_to_rcv)) {
			lock_sock(nsk);
1177
			smc_rx_wait(smc_sk(nsk), &timeo, smc_rx_data_available);
1178 1179 1180
			release_sock(nsk);
		}
	}
U
Ursula Braun 已提交
1181 1182

out:
1183
	sock_put(sk); /* sock_hold above */
U
Ursula Braun 已提交
1184 1185 1186 1187
	return rc;
}

static int smc_getname(struct socket *sock, struct sockaddr *addr,
1188
		       int peer)
U
Ursula Braun 已提交
1189 1190 1191
{
	struct smc_sock *smc;

1192 1193
	if (peer && (sock->sk->sk_state != SMC_ACTIVE) &&
	    (sock->sk->sk_state != SMC_APPCLOSEWAIT1))
U
Ursula Braun 已提交
1194 1195 1196 1197
		return -ENOTCONN;

	smc = smc_sk(sock->sk);

1198
	return smc->clcsock->ops->getname(smc->clcsock, addr, peer);
U
Ursula Braun 已提交
1199 1200 1201 1202 1203 1204 1205 1206 1207 1208
}

static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
{
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
	int rc = -EPIPE;

	smc = smc_sk(sk);
	lock_sock(sk);
1209 1210 1211
	if ((sk->sk_state != SMC_ACTIVE) &&
	    (sk->sk_state != SMC_APPCLOSEWAIT1) &&
	    (sk->sk_state != SMC_INIT))
U
Ursula Braun 已提交
1212
		goto out;
1213 1214 1215 1216 1217 1218 1219 1220 1221 1222

	if (msg->msg_flags & MSG_FASTOPEN) {
		if (sk->sk_state == SMC_INIT) {
			smc->use_fallback = true;
		} else {
			rc = -EINVAL;
			goto out;
		}
	}

U
Ursula Braun 已提交
1223 1224 1225
	if (smc->use_fallback)
		rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len);
	else
U
Ursula Braun 已提交
1226
		rc = smc_tx_sendmsg(smc, msg, len);
U
Ursula Braun 已提交
1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240
out:
	release_sock(sk);
	return rc;
}

static int smc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
		       int flags)
{
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
	int rc = -ENOTCONN;

	smc = smc_sk(sk);
	lock_sock(sk);
1241 1242 1243
	if ((sk->sk_state == SMC_INIT) ||
	    (sk->sk_state == SMC_LISTEN) ||
	    (sk->sk_state == SMC_CLOSED))
U
Ursula Braun 已提交
1244 1245
		goto out;

1246 1247 1248 1249 1250
	if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
		rc = 0;
		goto out;
	}

S
Stefan Raspl 已提交
1251
	if (smc->use_fallback) {
U
Ursula Braun 已提交
1252
		rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags);
S
Stefan Raspl 已提交
1253 1254 1255 1256
	} else {
		msg->msg_namelen = 0;
		rc = smc_rx_recvmsg(smc, msg, NULL, len, flags);
	}
1257

U
Ursula Braun 已提交
1258 1259 1260 1261 1262
out:
	release_sock(sk);
	return rc;
}

A
Al Viro 已提交
1263
static __poll_t smc_accept_poll(struct sock *parent)
1264
{
U
Ursula Braun 已提交
1265
	struct smc_sock *isk = smc_sk(parent);
A
Al Viro 已提交
1266
	__poll_t mask = 0;
1267

U
Ursula Braun 已提交
1268 1269
	spin_lock(&isk->accept_q_lock);
	if (!list_empty(&isk->accept_q))
1270
		mask = EPOLLIN | EPOLLRDNORM;
U
Ursula Braun 已提交
1271
	spin_unlock(&isk->accept_q_lock);
1272

U
Ursula Braun 已提交
1273
	return mask;
1274 1275
}

A
Al Viro 已提交
1276
static __poll_t smc_poll(struct file *file, struct socket *sock,
U
Ursula Braun 已提交
1277 1278 1279
			     poll_table *wait)
{
	struct sock *sk = sock->sk;
1280
	__poll_t mask = 0;
U
Ursula Braun 已提交
1281
	struct smc_sock *smc;
1282
	int rc;
U
Ursula Braun 已提交
1283

U
Ursula Braun 已提交
1284
	if (!sk)
1285
		return EPOLLNVAL;
U
Ursula Braun 已提交
1286

U
Ursula Braun 已提交
1287
	smc = smc_sk(sock->sk);
U
Ursula Braun 已提交
1288 1289
	sock_hold(sk);
	lock_sock(sk);
1290 1291
	if ((sk->sk_state == SMC_INIT) || smc->use_fallback) {
		/* delegate to CLC child sock */
U
Ursula Braun 已提交
1292
		release_sock(sk);
U
Ursula Braun 已提交
1293 1294
		mask = smc->clcsock->ops->poll(file, smc->clcsock, wait);
		lock_sock(sk);
1295 1296 1297 1298 1299 1300 1301 1302
		sk->sk_err = smc->clcsock->sk->sk_err;
		if (sk->sk_err) {
			mask |= EPOLLERR;
		} else {
			/* if non-blocking connect finished ... */
			if (sk->sk_state == SMC_INIT &&
			    mask & EPOLLOUT &&
			    smc->clcsock->sk->sk_state != TCP_CLOSE) {
1303
				rc = __smc_connect(smc);
1304
				if (rc < 0)
1305
					mask |= EPOLLERR;
U
Ursula Braun 已提交
1306
				/* success cases including fallback */
1307
				mask |= EPOLLOUT | EPOLLWRNORM;
1308
			}
U
Ursula Braun 已提交
1309 1310
		}
	} else {
U
Ursula Braun 已提交
1311 1312 1313 1314 1315
		if (sk->sk_state != SMC_CLOSED) {
			release_sock(sk);
			sock_poll_wait(file, sk_sleep(sk), wait);
			lock_sock(sk);
		}
1316
		if (sk->sk_err)
1317
			mask |= EPOLLERR;
1318 1319
		if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
		    (sk->sk_state == SMC_CLOSED))
1320
			mask |= EPOLLHUP;
U
Ursula Braun 已提交
1321 1322 1323 1324 1325 1326
		if (sk->sk_state == SMC_LISTEN) {
			/* woken up by sk_data_ready in smc_listen_work() */
			mask = smc_accept_poll(sk);
		} else {
			if (atomic_read(&smc->conn.sndbuf_space) ||
			    sk->sk_shutdown & SEND_SHUTDOWN) {
1327
				mask |= EPOLLOUT | EPOLLWRNORM;
U
Ursula Braun 已提交
1328 1329 1330 1331 1332
			} else {
				sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
				set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
			}
			if (atomic_read(&smc->conn.bytes_to_rcv))
1333
				mask |= EPOLLIN | EPOLLRDNORM;
U
Ursula Braun 已提交
1334
			if (sk->sk_shutdown & RCV_SHUTDOWN)
1335
				mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
U
Ursula Braun 已提交
1336
			if (sk->sk_state == SMC_APPCLOSEWAIT1)
1337
				mask |= EPOLLIN;
U
Ursula Braun 已提交
1338
		}
S
Stefan Raspl 已提交
1339 1340
		if (smc->conn.urg_state == SMC_URG_VALID)
			mask |= EPOLLPRI;
1341

U
Ursula Braun 已提交
1342
	}
U
Ursula Braun 已提交
1343 1344
	release_sock(sk);
	sock_put(sk);
U
Ursula Braun 已提交
1345 1346 1347 1348 1349 1350 1351 1352 1353

	return mask;
}

static int smc_shutdown(struct socket *sock, int how)
{
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
	int rc = -EINVAL;
1354
	int rc1 = 0;
U
Ursula Braun 已提交
1355 1356 1357 1358

	smc = smc_sk(sk);

	if ((how < SHUT_RD) || (how > SHUT_RDWR))
1359
		return rc;
U
Ursula Braun 已提交
1360 1361 1362 1363

	lock_sock(sk);

	rc = -ENOTCONN;
1364 1365 1366 1367 1368 1369 1370
	if ((sk->sk_state != SMC_LISTEN) &&
	    (sk->sk_state != SMC_ACTIVE) &&
	    (sk->sk_state != SMC_PEERCLOSEWAIT1) &&
	    (sk->sk_state != SMC_PEERCLOSEWAIT2) &&
	    (sk->sk_state != SMC_APPCLOSEWAIT1) &&
	    (sk->sk_state != SMC_APPCLOSEWAIT2) &&
	    (sk->sk_state != SMC_APPFINCLOSEWAIT))
U
Ursula Braun 已提交
1371 1372 1373 1374 1375 1376
		goto out;
	if (smc->use_fallback) {
		rc = kernel_sock_shutdown(smc->clcsock, how);
		sk->sk_shutdown = smc->clcsock->sk->sk_shutdown;
		if (sk->sk_shutdown == SHUTDOWN_MASK)
			sk->sk_state = SMC_CLOSED;
1377
		goto out;
U
Ursula Braun 已提交
1378
	}
1379 1380 1381 1382 1383 1384 1385 1386
	switch (how) {
	case SHUT_RDWR:		/* shutdown in both directions */
		rc = smc_close_active(smc);
		break;
	case SHUT_WR:
		rc = smc_close_shutdown_write(smc);
		break;
	case SHUT_RD:
1387 1388
		rc = 0;
		/* nothing more to do because peer is not involved */
1389 1390
		break;
	}
1391 1392
	if (smc->clcsock)
		rc1 = kernel_sock_shutdown(smc->clcsock, how);
1393 1394
	/* map sock_shutdown_cmd constants to sk_shutdown value range */
	sk->sk_shutdown |= how + 1;
U
Ursula Braun 已提交
1395 1396 1397

out:
	release_sock(sk);
1398
	return rc ? rc : rc1;
U
Ursula Braun 已提交
1399 1400 1401 1402 1403 1404 1405
}

static int smc_setsockopt(struct socket *sock, int level, int optname,
			  char __user *optval, unsigned int optlen)
{
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
1406
	int val, rc;
U
Ursula Braun 已提交
1407 1408 1409 1410 1411 1412

	smc = smc_sk(sk);

	/* generic setsockopts reaching us here always apply to the
	 * CLC socket
	 */
1413 1414 1415 1416 1417 1418 1419 1420 1421
	rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname,
					   optval, optlen);
	if (smc->clcsock->sk->sk_err) {
		sk->sk_err = smc->clcsock->sk->sk_err;
		sk->sk_error_report(sk);
	}
	if (rc)
		return rc;

1422 1423 1424 1425
	if (optlen < sizeof(int))
		return rc;
	get_user(val, (int __user *)optval);

1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440
	lock_sock(sk);
	switch (optname) {
	case TCP_ULP:
	case TCP_FASTOPEN:
	case TCP_FASTOPEN_CONNECT:
	case TCP_FASTOPEN_KEY:
	case TCP_FASTOPEN_NO_COOKIE:
		/* option not supported by SMC */
		if (sk->sk_state == SMC_INIT) {
			smc->use_fallback = true;
		} else {
			if (!smc->use_fallback)
				rc = -EINVAL;
		}
		break;
1441 1442
	case TCP_NODELAY:
		if (sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) {
1443
			if (val && !smc->use_fallback)
1444 1445 1446 1447 1448 1449
				mod_delayed_work(system_wq, &smc->conn.tx_work,
						 0);
		}
		break;
	case TCP_CORK:
		if (sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) {
1450
			if (!val && !smc->use_fallback)
1451 1452 1453 1454
				mod_delayed_work(system_wq, &smc->conn.tx_work,
						 0);
		}
		break;
1455 1456 1457
	case TCP_DEFER_ACCEPT:
		smc->sockopt_defer_accept = val;
		break;
1458 1459 1460 1461 1462 1463
	default:
		break;
	}
	release_sock(sk);

	return rc;
U
Ursula Braun 已提交
1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479
}

static int smc_getsockopt(struct socket *sock, int level, int optname,
			  char __user *optval, int __user *optlen)
{
	struct smc_sock *smc;

	smc = smc_sk(sock->sk);
	/* socket options apply to the CLC socket */
	return smc->clcsock->ops->getsockopt(smc->clcsock, level, optname,
					     optval, optlen);
}

static int smc_ioctl(struct socket *sock, unsigned int cmd,
		     unsigned long arg)
{
S
Stefan Raspl 已提交
1480 1481
	union smc_host_cursor cons, urg;
	struct smc_connection *conn;
U
Ursula Braun 已提交
1482
	struct smc_sock *smc;
1483
	int answ;
U
Ursula Braun 已提交
1484 1485

	smc = smc_sk(sock->sk);
S
Stefan Raspl 已提交
1486
	conn = &smc->conn;
1487 1488 1489
	if (smc->use_fallback) {
		if (!smc->clcsock)
			return -EBADF;
U
Ursula Braun 已提交
1490
		return smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg);
1491 1492 1493 1494 1495
	}
	switch (cmd) {
	case SIOCINQ: /* same as FIONREAD */
		if (smc->sk.sk_state == SMC_LISTEN)
			return -EINVAL;
1496 1497 1498 1499 1500
		if (smc->sk.sk_state == SMC_INIT ||
		    smc->sk.sk_state == SMC_CLOSED)
			answ = 0;
		else
			answ = atomic_read(&smc->conn.bytes_to_rcv);
1501 1502 1503 1504 1505
		break;
	case SIOCOUTQ:
		/* output queue size (not send + not acked) */
		if (smc->sk.sk_state == SMC_LISTEN)
			return -EINVAL;
1506 1507 1508 1509 1510
		if (smc->sk.sk_state == SMC_INIT ||
		    smc->sk.sk_state == SMC_CLOSED)
			answ = 0;
		else
			answ = smc->conn.sndbuf_desc->len -
1511 1512 1513 1514 1515 1516
					atomic_read(&smc->conn.sndbuf_space);
		break;
	case SIOCOUTQNSD:
		/* output queue size (not send only) */
		if (smc->sk.sk_state == SMC_LISTEN)
			return -EINVAL;
1517 1518 1519 1520 1521
		if (smc->sk.sk_state == SMC_INIT ||
		    smc->sk.sk_state == SMC_CLOSED)
			answ = 0;
		else
			answ = smc_tx_prepared_sends(&smc->conn);
1522
		break;
S
Stefan Raspl 已提交
1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539
	case SIOCATMARK:
		if (smc->sk.sk_state == SMC_LISTEN)
			return -EINVAL;
		if (smc->sk.sk_state == SMC_INIT ||
		    smc->sk.sk_state == SMC_CLOSED) {
			answ = 0;
		} else {
			smc_curs_write(&cons,
			       smc_curs_read(&conn->local_tx_ctrl.cons, conn),
				       conn);
			smc_curs_write(&urg,
				       smc_curs_read(&conn->urg_curs, conn),
				       conn);
			answ = smc_curs_diff(conn->rmb_desc->len,
					     &cons, &urg) == 1;
		}
		break;
1540 1541 1542 1543 1544
	default:
		return -ENOIOCTLCMD;
	}

	return put_user(answ, (int __user *)arg);
U
Ursula Braun 已提交
1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555
}

static ssize_t smc_sendpage(struct socket *sock, struct page *page,
			    int offset, size_t size, int flags)
{
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
	int rc = -EPIPE;

	smc = smc_sk(sk);
	lock_sock(sk);
S
Stefan Raspl 已提交
1556 1557
	if (sk->sk_state != SMC_ACTIVE) {
		release_sock(sk);
U
Ursula Braun 已提交
1558
		goto out;
S
Stefan Raspl 已提交
1559 1560
	}
	release_sock(sk);
U
Ursula Braun 已提交
1561 1562 1563 1564 1565 1566 1567 1568 1569 1570
	if (smc->use_fallback)
		rc = kernel_sendpage(smc->clcsock, page, offset,
				     size, flags);
	else
		rc = sock_no_sendpage(sock, page, offset, size, flags);

out:
	return rc;
}

S
Stefan Raspl 已提交
1571 1572 1573 1574 1575 1576
/* Map the affected portions of the rmbe into an spd, note the number of bytes
 * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor
 * updates till whenever a respective page has been fully processed.
 * Note that subsequent recv() calls have to wait till all splice() processing
 * completed.
 */
U
Ursula Braun 已提交
1577 1578
static ssize_t smc_splice_read(struct socket *sock, loff_t *ppos,
			       struct pipe_inode_info *pipe, size_t len,
S
Stefan Raspl 已提交
1579
			       unsigned int flags)
U
Ursula Braun 已提交
1580 1581 1582 1583 1584 1585 1586
{
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
	int rc = -ENOTCONN;

	smc = smc_sk(sk);
	lock_sock(sk);
S
Stefan Raspl 已提交
1587 1588 1589 1590 1591 1592 1593 1594

	if (sk->sk_state == SMC_INIT ||
	    sk->sk_state == SMC_LISTEN ||
	    sk->sk_state == SMC_CLOSED)
		goto out;

	if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
		rc = 0;
U
Ursula Braun 已提交
1595
		goto out;
S
Stefan Raspl 已提交
1596 1597
	}

U
Ursula Braun 已提交
1598 1599 1600 1601
	if (smc->use_fallback) {
		rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos,
						    pipe, len, flags);
	} else {
S
Stefan Raspl 已提交
1602 1603 1604 1605 1606 1607 1608 1609 1610
		if (*ppos) {
			rc = -ESPIPE;
			goto out;
		}
		if (flags & SPLICE_F_NONBLOCK)
			flags = MSG_DONTWAIT;
		else
			flags = 0;
		rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags);
U
Ursula Braun 已提交
1611 1612 1613
	}
out:
	release_sock(sk);
S
Stefan Raspl 已提交
1614

U
Ursula Braun 已提交
1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643
	return rc;
}

/* must look like tcp */
static const struct proto_ops smc_sock_ops = {
	.family		= PF_SMC,
	.owner		= THIS_MODULE,
	.release	= smc_release,
	.bind		= smc_bind,
	.connect	= smc_connect,
	.socketpair	= sock_no_socketpair,
	.accept		= smc_accept,
	.getname	= smc_getname,
	.poll		= smc_poll,
	.ioctl		= smc_ioctl,
	.listen		= smc_listen,
	.shutdown	= smc_shutdown,
	.setsockopt	= smc_setsockopt,
	.getsockopt	= smc_getsockopt,
	.sendmsg	= smc_sendmsg,
	.recvmsg	= smc_recvmsg,
	.mmap		= sock_no_mmap,
	.sendpage	= smc_sendpage,
	.splice_read	= smc_splice_read,
};

static int smc_create(struct net *net, struct socket *sock, int protocol,
		      int kern)
{
1644
	int family = (protocol == SMCPROTO_SMC6) ? PF_INET6 : PF_INET;
U
Ursula Braun 已提交
1645 1646 1647 1648 1649 1650 1651 1652 1653
	struct smc_sock *smc;
	struct sock *sk;
	int rc;

	rc = -ESOCKTNOSUPPORT;
	if (sock->type != SOCK_STREAM)
		goto out;

	rc = -EPROTONOSUPPORT;
1654
	if (protocol != SMCPROTO_SMC && protocol != SMCPROTO_SMC6)
U
Ursula Braun 已提交
1655 1656 1657 1658
		goto out;

	rc = -ENOBUFS;
	sock->ops = &smc_sock_ops;
1659
	sk = smc_sock_alloc(net, sock, protocol);
U
Ursula Braun 已提交
1660 1661 1662 1663 1664
	if (!sk)
		goto out;

	/* create internal TCP socket for CLC handshake and fallback */
	smc = smc_sk(sk);
1665
	smc->use_fallback = false; /* assume rdma capability first */
1666 1667
	rc = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP,
			      &smc->clcsock);
1668
	if (rc) {
U
Ursula Braun 已提交
1669
		sk_common_release(sk);
1670 1671
		goto out;
	}
U
Ursula Braun 已提交
1672 1673
	smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE);
	smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE);
U
Ursula Braun 已提交
1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688

out:
	return rc;
}

static const struct net_proto_family smc_sock_family_ops = {
	.family	= PF_SMC,
	.owner	= THIS_MODULE,
	.create	= smc_create,
};

static int __init smc_init(void)
{
	int rc;

1689 1690 1691 1692
	rc = smc_pnet_init();
	if (rc)
		return rc;

U
Ursula Braun 已提交
1693 1694 1695 1696 1697 1698
	rc = smc_llc_init();
	if (rc) {
		pr_err("%s: smc_llc_init fails with %d\n", __func__, rc);
		goto out_pnet;
	}

1699 1700 1701 1702 1703 1704
	rc = smc_cdc_init();
	if (rc) {
		pr_err("%s: smc_cdc_init fails with %d\n", __func__, rc);
		goto out_pnet;
	}

U
Ursula Braun 已提交
1705 1706
	rc = proto_register(&smc_proto, 1);
	if (rc) {
1707
		pr_err("%s: proto_register(v4) fails with %d\n", __func__, rc);
1708
		goto out_pnet;
U
Ursula Braun 已提交
1709 1710
	}

1711 1712 1713 1714 1715 1716
	rc = proto_register(&smc_proto6, 1);
	if (rc) {
		pr_err("%s: proto_register(v6) fails with %d\n", __func__, rc);
		goto out_proto;
	}

U
Ursula Braun 已提交
1717 1718 1719
	rc = sock_register(&smc_sock_family_ops);
	if (rc) {
		pr_err("%s: sock_register fails with %d\n", __func__, rc);
1720
		goto out_proto6;
U
Ursula Braun 已提交
1721
	}
1722
	INIT_HLIST_HEAD(&smc_v4_hashinfo.ht);
1723
	INIT_HLIST_HEAD(&smc_v6_hashinfo.ht);
U
Ursula Braun 已提交
1724

1725 1726 1727 1728 1729 1730
	rc = smc_ib_register_client();
	if (rc) {
		pr_err("%s: ib_register fails with %d\n", __func__, rc);
		goto out_sock;
	}

U
Ursula Braun 已提交
1731
	static_branch_enable(&tcp_have_smc);
U
Ursula Braun 已提交
1732 1733
	return 0;

1734 1735
out_sock:
	sock_unregister(PF_SMC);
1736 1737
out_proto6:
	proto_unregister(&smc_proto6);
U
Ursula Braun 已提交
1738 1739
out_proto:
	proto_unregister(&smc_proto);
1740 1741
out_pnet:
	smc_pnet_exit();
U
Ursula Braun 已提交
1742 1743 1744 1745 1746
	return rc;
}

static void __exit smc_exit(void)
{
1747
	smc_core_exit();
U
Ursula Braun 已提交
1748
	static_branch_disable(&tcp_have_smc);
1749
	smc_ib_unregister_client();
U
Ursula Braun 已提交
1750
	sock_unregister(PF_SMC);
1751
	proto_unregister(&smc_proto6);
U
Ursula Braun 已提交
1752
	proto_unregister(&smc_proto);
1753
	smc_pnet_exit();
U
Ursula Braun 已提交
1754 1755 1756 1757 1758 1759 1760 1761 1762
}

module_init(smc_init);
module_exit(smc_exit);

MODULE_AUTHOR("Ursula Braun <ubraun@linux.vnet.ibm.com>");
MODULE_DESCRIPTION("smc socket address family");
MODULE_LICENSE("GPL");
MODULE_ALIAS_NETPROTO(PF_SMC);