af_smc.c 64.7 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-only
U
Ursula Braun 已提交
2 3 4 5 6 7 8 9
/*
 *  Shared Memory Communications over RDMA (SMC-R) and RoCE
 *
 *  AF_SMC protocol family socket handler keeping the AF_INET sock address type
 *  applies to SOCK_STREAM sockets only
 *  offers an alternative communication option for TCP-protocol sockets
 *  applicable with RoCE-cards only
 *
10 11 12
 *  Initial restrictions:
 *    - support for alternate links postponed
 *
13
 *  Copyright IBM Corp. 2016, 2018
U
Ursula Braun 已提交
14 15 16 17 18 19 20 21 22 23
 *
 *  Author(s):  Ursula Braun <ubraun@linux.vnet.ibm.com>
 *              based on prototype from Frank Blaschka
 */

#define KMSG_COMPONENT "smc"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

#include <linux/module.h>
#include <linux/socket.h>
24
#include <linux/workqueue.h>
25
#include <linux/in.h>
26
#include <linux/sched/signal.h>
27
#include <linux/if_vlan.h>
28
#include <linux/rcupdate_wait.h>
29
#include <linux/ctype.h>
30

U
Ursula Braun 已提交
31
#include <net/sock.h>
32
#include <net/tcp.h>
33
#include <net/smc.h>
34
#include <asm/ioctls.h>
U
Ursula Braun 已提交
35

36 37 38 39
#include <net/net_namespace.h>
#include <net/netns/generic.h>
#include "smc_netns.h"

U
Ursula Braun 已提交
40
#include "smc.h"
41
#include "smc_clc.h"
U
Ursula Braun 已提交
42
#include "smc_llc.h"
43
#include "smc_cdc.h"
44
#include "smc_core.h"
45
#include "smc_ib.h"
46
#include "smc_ism.h"
47
#include "smc_pnet.h"
U
Ursula Braun 已提交
48
#include "smc_tx.h"
U
Ursula Braun 已提交
49
#include "smc_rx.h"
50
#include "smc_close.h"
U
Ursula Braun 已提交
51

52 53 54 55 56
static DEFINE_MUTEX(smc_server_lgr_pending);	/* serialize link group
						 * creation on server
						 */
static DEFINE_MUTEX(smc_client_lgr_pending);	/* serialize link group
						 * creation on client
57 58
						 */

59 60 61
struct workqueue_struct	*smc_hs_wq;	/* wq for handshake work */
struct workqueue_struct	*smc_close_wq;	/* wq for close work */

62
static void smc_tcp_listen_work(struct work_struct *);
63
static void smc_connect_work(struct work_struct *);
64

U
Ursula Braun 已提交
65 66 67 68 69 70 71
static void smc_set_keepalive(struct sock *sk, int val)
{
	struct smc_sock *smc = smc_sk(sk);

	smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val);
}

72 73 74 75
static struct smc_hashinfo smc_v4_hashinfo = {
	.lock = __RW_LOCK_UNLOCKED(smc_v4_hashinfo.lock),
};

76 77 78 79
static struct smc_hashinfo smc_v6_hashinfo = {
	.lock = __RW_LOCK_UNLOCKED(smc_v6_hashinfo.lock),
};

80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107
int smc_hash_sk(struct sock *sk)
{
	struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
	struct hlist_head *head;

	head = &h->ht;

	write_lock_bh(&h->lock);
	sk_add_node(sk, head);
	sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
	write_unlock_bh(&h->lock);

	return 0;
}
EXPORT_SYMBOL_GPL(smc_hash_sk);

void smc_unhash_sk(struct sock *sk)
{
	struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;

	write_lock_bh(&h->lock);
	if (sk_del_node_init(sk))
		sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
	write_unlock_bh(&h->lock);
}
EXPORT_SYMBOL_GPL(smc_unhash_sk);

struct proto smc_proto = {
U
Ursula Braun 已提交
108 109 110
	.name		= "SMC",
	.owner		= THIS_MODULE,
	.keepalive	= smc_set_keepalive,
111 112
	.hash		= smc_hash_sk,
	.unhash		= smc_unhash_sk,
U
Ursula Braun 已提交
113
	.obj_size	= sizeof(struct smc_sock),
114
	.h.smc_hash	= &smc_v4_hashinfo,
115
	.slab_flags	= SLAB_TYPESAFE_BY_RCU,
U
Ursula Braun 已提交
116
};
117
EXPORT_SYMBOL_GPL(smc_proto);
U
Ursula Braun 已提交
118

119 120 121 122 123 124 125 126 127 128 129 130
struct proto smc_proto6 = {
	.name		= "SMC6",
	.owner		= THIS_MODULE,
	.keepalive	= smc_set_keepalive,
	.hash		= smc_hash_sk,
	.unhash		= smc_unhash_sk,
	.obj_size	= sizeof(struct smc_sock),
	.h.smc_hash	= &smc_v6_hashinfo,
	.slab_flags	= SLAB_TYPESAFE_BY_RCU,
};
EXPORT_SYMBOL_GPL(smc_proto6);

131 132
static void smc_restore_fallback_changes(struct smc_sock *smc)
{
133 134 135 136
	if (smc->clcsock->file) { /* non-accepted sockets have no file yet */
		smc->clcsock->file->private_data = smc->sk.sk_socket;
		smc->clcsock->file = NULL;
	}
137 138
}

139
static int __smc_release(struct smc_sock *smc)
U
Ursula Braun 已提交
140
{
141
	struct sock *sk = &smc->sk;
142
	int rc = 0;
U
Ursula Braun 已提交
143

144
	if (!smc->use_fallback) {
145 146 147
		rc = smc_close_active(smc);
		sock_set_flag(sk, SOCK_DEAD);
		sk->sk_shutdown |= SHUTDOWN_MASK;
148 149 150 151
	} else {
		if (sk->sk_state != SMC_LISTEN && sk->sk_state != SMC_INIT)
			sock_put(sk); /* passive closing */
		if (sk->sk_state == SMC_LISTEN) {
152 153 154
			/* wake up clcsock accept */
			rc = kernel_sock_shutdown(smc->clcsock, SHUT_RDWR);
		}
155 156
		sk->sk_state = SMC_CLOSED;
		sk->sk_state_change(sk);
157
		smc_restore_fallback_changes(smc);
158
	}
U
Ursula Braun 已提交
159

160 161 162 163
	sk->sk_prot->unhash(sk);

	if (sk->sk_state == SMC_CLOSED) {
		if (smc->clcsock) {
164 165 166
			release_sock(sk);
			smc_clcsock_release(smc);
			lock_sock(sk);
167 168 169 170 171
		}
		if (!smc->use_fallback)
			smc_conn_free(&smc->conn);
	}

172 173 174 175 176 177 178 179 180 181 182 183
	return rc;
}

static int smc_release(struct socket *sock)
{
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
	int rc = 0;

	if (!sk)
		goto out;

U
Ursula Braun 已提交
184
	sock_hold(sk); /* sock_put below */
185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201
	smc = smc_sk(sk);

	/* cleanup for a dangling non-blocking connect */
	if (smc->connect_nonblock && sk->sk_state == SMC_INIT)
		tcp_abort(smc->clcsock->sk, ECONNABORTED);
	flush_work(&smc->connect_work);

	if (sk->sk_state == SMC_LISTEN)
		/* smc_close_non_accepted() is called and acquires
		 * sock lock for child sockets again
		 */
		lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
	else
		lock_sock(sk);

	rc = __smc_release(smc);

U
Ursula Braun 已提交
202 203 204 205 206
	/* detach socket */
	sock_orphan(sk);
	sock->sk = NULL;
	release_sock(sk);

U
Ursula Braun 已提交
207
	sock_put(sk); /* sock_hold above */
208
	sock_put(sk); /* final sock_put */
U
Ursula Braun 已提交
209
out:
210
	return rc;
U
Ursula Braun 已提交
211 212 213 214 215 216 217 218 219 220 221 222
}

static void smc_destruct(struct sock *sk)
{
	if (sk->sk_state != SMC_CLOSED)
		return;
	if (!sock_flag(sk, SOCK_DEAD))
		return;

	sk_refcnt_debug_dec(sk);
}

223 224
static struct sock *smc_sock_alloc(struct net *net, struct socket *sock,
				   int protocol)
U
Ursula Braun 已提交
225 226
{
	struct smc_sock *smc;
227
	struct proto *prot;
U
Ursula Braun 已提交
228 229
	struct sock *sk;

230 231
	prot = (protocol == SMCPROTO_SMC6) ? &smc_proto6 : &smc_proto;
	sk = sk_alloc(net, PF_SMC, GFP_KERNEL, prot, 0);
U
Ursula Braun 已提交
232 233 234 235 236 237
	if (!sk)
		return NULL;

	sock_init_data(sock, sk); /* sets sk_refcnt to 1 */
	sk->sk_state = SMC_INIT;
	sk->sk_destruct = smc_destruct;
238
	sk->sk_protocol = protocol;
U
Ursula Braun 已提交
239
	smc = smc_sk(sk);
240
	INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
241
	INIT_WORK(&smc->connect_work, smc_connect_work);
242
	INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work);
243 244
	INIT_LIST_HEAD(&smc->accept_q);
	spin_lock_init(&smc->accept_q_lock);
245
	spin_lock_init(&smc->conn.send_lock);
246
	sk->sk_prot->hash(sk);
247
	sk_refcnt_debug_inc(sk);
248
	mutex_init(&smc->clcsock_release_lock);
U
Ursula Braun 已提交
249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268

	return sk;
}

static int smc_bind(struct socket *sock, struct sockaddr *uaddr,
		    int addr_len)
{
	struct sockaddr_in *addr = (struct sockaddr_in *)uaddr;
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
	int rc;

	smc = smc_sk(sk);

	/* replicate tests from inet_bind(), to be safe wrt. future changes */
	rc = -EINVAL;
	if (addr_len < sizeof(struct sockaddr_in))
		goto out;

	rc = -EAFNOSUPPORT;
269 270 271 272
	if (addr->sin_family != AF_INET &&
	    addr->sin_family != AF_INET6 &&
	    addr->sin_family != AF_UNSPEC)
		goto out;
U
Ursula Braun 已提交
273
	/* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */
274 275
	if (addr->sin_family == AF_UNSPEC &&
	    addr->sin_addr.s_addr != htonl(INADDR_ANY))
U
Ursula Braun 已提交
276 277 278 279 280 281
		goto out;

	lock_sock(sk);

	/* Check if socket is already active */
	rc = -EINVAL;
282
	if (sk->sk_state != SMC_INIT || smc->connect_nonblock)
U
Ursula Braun 已提交
283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325
		goto out_rel;

	smc->clcsock->sk->sk_reuse = sk->sk_reuse;
	rc = kernel_bind(smc->clcsock, uaddr, addr_len);

out_rel:
	release_sock(sk);
out:
	return rc;
}

static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk,
				   unsigned long mask)
{
	/* options we don't get control via setsockopt for */
	nsk->sk_type = osk->sk_type;
	nsk->sk_sndbuf = osk->sk_sndbuf;
	nsk->sk_rcvbuf = osk->sk_rcvbuf;
	nsk->sk_sndtimeo = osk->sk_sndtimeo;
	nsk->sk_rcvtimeo = osk->sk_rcvtimeo;
	nsk->sk_mark = osk->sk_mark;
	nsk->sk_priority = osk->sk_priority;
	nsk->sk_rcvlowat = osk->sk_rcvlowat;
	nsk->sk_bound_dev_if = osk->sk_bound_dev_if;
	nsk->sk_err = osk->sk_err;

	nsk->sk_flags &= ~mask;
	nsk->sk_flags |= osk->sk_flags & mask;
}

#define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \
			     (1UL << SOCK_KEEPOPEN) | \
			     (1UL << SOCK_LINGER) | \
			     (1UL << SOCK_BROADCAST) | \
			     (1UL << SOCK_TIMESTAMP) | \
			     (1UL << SOCK_DBG) | \
			     (1UL << SOCK_RCVTSTAMP) | \
			     (1UL << SOCK_RCVTSTAMPNS) | \
			     (1UL << SOCK_LOCALROUTE) | \
			     (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
			     (1UL << SOCK_RXQ_OVFL) | \
			     (1UL << SOCK_WIFI_STATUS) | \
			     (1UL << SOCK_NOFCS) | \
D
Deepa Dinamani 已提交
326 327
			     (1UL << SOCK_FILTER_LOCKED) | \
			     (1UL << SOCK_TSTAMP_NEW))
U
Ursula Braun 已提交
328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345
/* copy only relevant settings and flags of SOL_SOCKET level from smc to
 * clc socket (since smc is not called for these options from net/core)
 */
static void smc_copy_sock_settings_to_clc(struct smc_sock *smc)
{
	smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC);
}

#define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \
			     (1UL << SOCK_KEEPOPEN) | \
			     (1UL << SOCK_LINGER) | \
			     (1UL << SOCK_DBG))
/* copy only settings and flags relevant for smc from clc to smc socket */
static void smc_copy_sock_settings_to_smc(struct smc_sock *smc)
{
	smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC);
}

346
/* register the new rmb on all links */
347
static int smcr_lgr_reg_rmbs(struct smc_link *link,
348 349
			     struct smc_buf_desc *rmb_desc)
{
350 351
	struct smc_link_group *lgr = link->lgr;
	int i, rc = 0;
352

353 354 355 356 357 358 359
	rc = smc_llc_flow_initiate(lgr, SMC_LLC_FLOW_RKEY);
	if (rc)
		return rc;
	/* protect against parallel smc_llc_cli_rkey_exchange() and
	 * parallel smcr_link_reg_rmb()
	 */
	mutex_lock(&lgr->llc_conf_mutex);
360
	for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
361
		if (!smc_link_active(&lgr->lnk[i]))
362
			continue;
363
		rc = smcr_link_reg_rmb(&lgr->lnk[i], rmb_desc);
364
		if (rc)
365
			goto out;
366
	}
367 368 369 370 371 372 373 374 375

	/* exchange confirm_rkey msg with peer */
	rc = smc_llc_do_confirm_rkey(link, rmb_desc);
	if (rc) {
		rc = -EFAULT;
		goto out;
	}
	rmb_desc->is_conf_rkey = true;
out:
376 377
	mutex_unlock(&lgr->llc_conf_mutex);
	smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
378
	return rc;
K
Karsten Graul 已提交
379 380
}

381
static int smcr_clnt_conf_first_link(struct smc_sock *smc)
U
Ursula Braun 已提交
382
{
383
	struct smc_link *link = smc->conn.lnk;
384
	struct smc_llc_qentry *qentry;
U
Ursula Braun 已提交
385 386 387
	int rc;

	/* receive CONFIRM LINK request from server over RoCE fabric */
388 389 390
	qentry = smc_llc_wait(link->lgr, NULL, SMC_LLC_WAIT_TIME,
			      SMC_LLC_CONFIRM_LINK);
	if (!qentry) {
U
Ursula Braun 已提交
391 392 393
		struct smc_clc_msg_decline dclc;

		rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
394
				      SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
395
		return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
U
Ursula Braun 已提交
396
	}
397
	smc_llc_save_peer_uid(qentry);
398 399 400
	rc = smc_llc_eval_conf_link(qentry, SMC_LLC_REQ);
	smc_llc_flow_qentry_del(&link->lgr->llc_flow_lcl);
	if (rc)
401 402
		return SMC_CLC_DECL_RMBE_EC;

U
Ursula Braun 已提交
403 404
	rc = smc_ib_modify_qp_rts(link);
	if (rc)
405
		return SMC_CLC_DECL_ERR_RDYLNK;
U
Ursula Braun 已提交
406 407

	smc_wr_remember_qp_attr(link);
408

409
	if (smcr_link_reg_rmb(link, smc->conn.rmb_desc))
410
		return SMC_CLC_DECL_ERR_REGRMB;
411

412 413 414
	/* confirm_rkey is implicit on 1st contact */
	smc->conn.rmb_desc->is_conf_rkey = true;

U
Ursula Braun 已提交
415
	/* send CONFIRM LINK response over RoCE fabric */
416
	rc = smc_llc_send_confirm_link(link, SMC_LLC_RESP);
U
Ursula Braun 已提交
417
	if (rc < 0)
418
		return SMC_CLC_DECL_TIMEOUT_CL;
U
Ursula Braun 已提交
419

420
	smc_llc_link_active(link);
421
	smcr_lgr_set_type(link->lgr, SMC_LGR_SINGLE);
422 423 424 425 426

	/* optional 2nd link, receive ADD LINK request from server */
	qentry = smc_llc_wait(link->lgr, NULL, SMC_LLC_WAIT_TIME,
			      SMC_LLC_ADD_LINK);
	if (!qentry) {
427 428 429
		struct smc_clc_msg_decline dclc;

		rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
430
				      SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
431 432 433
		if (rc == -EAGAIN)
			rc = 0; /* no DECLINE received, go with one link */
		return rc;
434
	}
435
	smc_llc_flow_qentry_clr(&link->lgr->llc_flow_lcl);
436
	smc_llc_cli_add_link(link, qentry);
437
	return 0;
U
Ursula Braun 已提交
438 439
}

440 441
static void smcr_conn_save_peer_info(struct smc_sock *smc,
				     struct smc_clc_msg_accept_confirm *clc)
442
{
443
	int bufsize = smc_uncompress_bufsize(clc->r0.rmbe_size);
444

445 446
	smc->conn.peer_rmbe_idx = clc->r0.rmbe_idx;
	smc->conn.local_tx_ctrl.token = ntohl(clc->r0.rmbe_alert_token);
447
	smc->conn.peer_rmbe_size = bufsize;
U
Ursula Braun 已提交
448
	atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
449
	smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1);
450 451
}

452 453 454 455 456 457 458 459 460 461
static bool smc_isascii(char *hostname)
{
	int i;

	for (i = 0; i < SMC_MAX_HOSTNAME_LEN; i++)
		if (!isascii(hostname[i]))
			return false;
	return true;
}

462 463 464
static void smcd_conn_save_peer_info(struct smc_sock *smc,
				     struct smc_clc_msg_accept_confirm *clc)
{
465
	int bufsize = smc_uncompress_bufsize(clc->d0.dmbe_size);
466

467 468
	smc->conn.peer_rmbe_idx = clc->d0.dmbe_idx;
	smc->conn.peer_token = clc->d0.token;
469 470 471 472
	/* msg header takes up space in the buffer */
	smc->conn.peer_rmbe_size = bufsize - sizeof(struct smcd_cdc_msg);
	atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
	smc->conn.tx_off = bufsize * smc->conn.peer_rmbe_idx;
473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488
	if (clc->hdr.version > SMC_V1 &&
	    (clc->hdr.typev2 & SMC_FIRST_CONTACT_MASK)) {
		struct smc_clc_msg_accept_confirm_v2 *clc_v2 =
			(struct smc_clc_msg_accept_confirm_v2 *)clc;
		struct smc_clc_first_contact_ext *fce =
			(struct smc_clc_first_contact_ext *)
				(((u8 *)clc_v2) + sizeof(*clc_v2));

		memcpy(smc->conn.lgr->negotiated_eid, clc_v2->eid,
		       SMC_MAX_EID_LEN);
		smc->conn.lgr->peer_os = fce->os_type;
		smc->conn.lgr->peer_smc_release = fce->release;
		if (smc_isascii(fce->hostname))
			memcpy(smc->conn.lgr->peer_hostname, fce->hostname,
			       SMC_MAX_HOSTNAME_LEN);
	}
489 490 491 492 493 494 495 496 497 498 499
}

static void smc_conn_save_peer_info(struct smc_sock *smc,
				    struct smc_clc_msg_accept_confirm *clc)
{
	if (smc->conn.lgr->is_smcd)
		smcd_conn_save_peer_info(smc, clc);
	else
		smcr_conn_save_peer_info(smc, clc);
}

500 501 502
static void smc_link_save_peer_info(struct smc_link *link,
				    struct smc_clc_msg_accept_confirm *clc)
{
503 504 505 506 507
	link->peer_qpn = ntoh24(clc->r0.qpn);
	memcpy(link->peer_gid, clc->r0.lcl.gid, SMC_GID_SIZE);
	memcpy(link->peer_mac, clc->r0.lcl.mac, sizeof(link->peer_mac));
	link->peer_psn = ntoh24(clc->r0.psn);
	link->peer_mtu = clc->r0.qp_mtu;
508 509
}

510 511 512 513 514 515
static void smc_switch_to_fallback(struct smc_sock *smc)
{
	smc->use_fallback = true;
	if (smc->sk.sk_socket && smc->sk.sk_socket->file) {
		smc->clcsock->file = smc->sk.sk_socket->file;
		smc->clcsock->file->private_data = smc->clcsock;
516 517
		smc->clcsock->wq.fasync_list =
			smc->sk.sk_socket->wq.fasync_list;
518 519 520
	}
}

521
/* fall back during connect */
522
static int smc_connect_fallback(struct smc_sock *smc, int reason_code)
523
{
524
	smc_switch_to_fallback(smc);
525
	smc->fallback_rsn = reason_code;
526
	smc_copy_sock_settings_to_clc(smc);
527
	smc->connect_nonblock = 0;
528 529 530 531
	if (smc->sk.sk_state == SMC_INIT)
		smc->sk.sk_state = SMC_ACTIVE;
	return 0;
}
532

533
/* decline and fall back during connect */
534 535
static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code,
					u8 version)
536 537
{
	int rc;
538

539 540 541
	if (reason_code < 0) { /* error, fallback is not possible */
		if (smc->sk.sk_state == SMC_INIT)
			sock_put(&smc->sk); /* passive closing */
542
		return reason_code;
543
	}
544
	if (reason_code != SMC_CLC_DECL_PEERDECL) {
545
		rc = smc_clc_send_decline(smc, reason_code, version);
546 547 548
		if (rc < 0) {
			if (smc->sk.sk_state == SMC_INIT)
				sock_put(&smc->sk); /* passive closing */
549
			return rc;
550
		}
U
Ursula Braun 已提交
551
	}
552
	return smc_connect_fallback(smc, reason_code);
553
}
U
Ursula Braun 已提交
554

555
/* abort connecting */
556
static void smc_connect_abort(struct smc_sock *smc, int local_first)
557
{
558
	if (local_first)
559 560 561
		smc_lgr_cleanup_early(&smc->conn);
	else
		smc_conn_free(&smc->conn);
562 563 564 565
}

/* check if there is a rdma device available for this connection. */
/* called for connect and listen */
566
static int smc_find_rdma_device(struct smc_sock *smc, struct smc_init_info *ini)
567
{
568 569 570 571
	/* PNET table look up: search active ib_device and port
	 * within same PNETID that also contains the ethernet device
	 * used for the internal TCP socket
	 */
572
	smc_pnet_find_roce_resource(smc->clcsock->sk, ini);
573 574
	if (!ini->ib_dev)
		return SMC_CLC_DECL_NOSMCRDEV;
575
	return 0;
576 577
}

578 579
/* check if there is an ISM device available for this connection. */
/* called for connect and listen */
580
static int smc_find_ism_device(struct smc_sock *smc, struct smc_init_info *ini)
581 582
{
	/* Find ISM device with same PNETID as connecting interface  */
583
	smc_pnet_find_ism_resource(smc->clcsock->sk, ini);
584
	if (!ini->ism_dev[0])
585
		return SMC_CLC_DECL_NOSMCDDEV;
586 587
	else
		ini->ism_chid[0] = smc_ism_get_chid(ini->ism_dev[0]);
588 589 590
	return 0;
}

591 592 593 594 595 596 597 598 599 600 601 602
/* is chid unique for the ism devices that are already determined? */
static bool smc_find_ism_v2_is_unique_chid(u16 chid, struct smc_init_info *ini,
					   int cnt)
{
	int i = (!ini->ism_dev[0]) ? 1 : 0;

	for (; i < cnt; i++)
		if (ini->ism_chid[i] == chid)
			return false;
	return true;
}

603 604 605 606 607 608 609 610 611
/* determine possible V2 ISM devices (either without PNETID or with PNETID plus
 * PNETID matching net_device)
 */
static int smc_find_ism_v2_device_clnt(struct smc_sock *smc,
				       struct smc_init_info *ini)
{
	int rc = SMC_CLC_DECL_NOSMCDDEV;
	struct smcd_dev *smcd;
	int i = 1;
612
	u16 chid;
613 614 615 616 617 618 619

	if (smcd_indicated(ini->smc_type_v1))
		rc = 0;		/* already initialized for V1 */
	mutex_lock(&smcd_dev_list.mutex);
	list_for_each_entry(smcd, &smcd_dev_list.list, list) {
		if (smcd->going_away || smcd == ini->ism_dev[0])
			continue;
620 621 622
		chid = smc_ism_get_chid(smcd);
		if (!smc_find_ism_v2_is_unique_chid(chid, ini, i))
			continue;
623 624 625
		if (!smc_pnet_is_pnetid_set(smcd->pnetid) ||
		    smc_pnet_is_ndev_pnetid(sock_net(&smc->sk), smcd->pnetid)) {
			ini->ism_dev[i] = smcd;
626
			ini->ism_chid[i] = chid;
627 628 629 630 631 632 633 634 635 636 637 638 639 640 641
			ini->is_smcd = true;
			rc = 0;
			i++;
			if (i > SMC_MAX_ISM_DEVS)
				break;
		}
	}
	mutex_unlock(&smcd_dev_list.mutex);
	ini->ism_offered_cnt = i - 1;
	if (!ini->ism_dev[0] && !ini->ism_dev[1])
		ini->smcd_version = 0;

	return rc;
}

642 643
/* Check for VLAN ID and register it on ISM device just for CLC handshake */
static int smc_connect_ism_vlan_setup(struct smc_sock *smc,
644
				      struct smc_init_info *ini)
645
{
646
	if (ini->vlan_id && smc_ism_get_vlan(ini->ism_dev[0], ini->vlan_id))
647
		return SMC_CLC_DECL_ISMVLANERR;
648 649 650
	return 0;
}

651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682
static int smc_find_proposal_devices(struct smc_sock *smc,
				     struct smc_init_info *ini)
{
	int rc = 0;

	/* check if there is an ism device available */
	if (ini->smcd_version & SMC_V1) {
		if (smc_find_ism_device(smc, ini) ||
		    smc_connect_ism_vlan_setup(smc, ini)) {
			if (ini->smc_type_v1 == SMC_TYPE_B)
				ini->smc_type_v1 = SMC_TYPE_R;
			else
				ini->smc_type_v1 = SMC_TYPE_N;
		} /* else ISM V1 is supported for this connection */
		if (smc_find_rdma_device(smc, ini)) {
			if (ini->smc_type_v1 == SMC_TYPE_B)
				ini->smc_type_v1 = SMC_TYPE_D;
			else
				ini->smc_type_v1 = SMC_TYPE_N;
		} /* else RDMA is supported for this connection */
	}
	if (smc_ism_v2_capable && smc_find_ism_v2_device_clnt(smc, ini))
		ini->smc_type_v2 = SMC_TYPE_N;

	/* if neither ISM nor RDMA are supported, fallback */
	if (!smcr_indicated(ini->smc_type_v1) &&
	    ini->smc_type_v1 == SMC_TYPE_N && ini->smc_type_v2 == SMC_TYPE_N)
		rc = SMC_CLC_DECL_NOSMCDEV;

	return rc;
}

683 684 685
/* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is
 * used, the VLAN ID will be registered again during the connection setup.
 */
686
static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc,
687
					struct smc_init_info *ini)
688
{
689
	if (!smcd_indicated(ini->smc_type_v1))
690
		return 0;
691
	if (ini->vlan_id && smc_ism_put_vlan(ini->ism_dev[0], ini->vlan_id))
692 693 694 695
		return SMC_CLC_DECL_CNFERR;
	return 0;
}

U
Ursula Braun 已提交
696 697
#define SMC_CLC_MAX_ACCEPT_LEN \
	(sizeof(struct smc_clc_msg_accept_confirm_v2) + \
698
	 sizeof(struct smc_clc_first_contact_ext) + \
U
Ursula Braun 已提交
699 700
	 sizeof(struct smc_clc_msg_trail))

701
/* CLC handshake during connect */
702
static int smc_connect_clc(struct smc_sock *smc,
U
Ursula Braun 已提交
703
			   struct smc_clc_msg_accept_confirm_v2 *aclc2,
704
			   struct smc_init_info *ini)
705 706
{
	int rc = 0;
707 708

	/* do inband token exchange */
709
	rc = smc_clc_send_proposal(smc, ini);
710 711
	if (rc)
		return rc;
712
	/* receive SMC Accept CLC message */
U
Ursula Braun 已提交
713 714
	return smc_clc_wait_msg(smc, aclc2, SMC_CLC_MAX_ACCEPT_LEN,
				SMC_CLC_ACCEPT, CLC_WAIT_TIME);
715 716 717 718 719
}

/* setup for RDMA connection of client */
static int smc_connect_rdma(struct smc_sock *smc,
			    struct smc_clc_msg_accept_confirm *aclc,
720
			    struct smc_init_info *ini)
721
{
722
	int i, reason_code = 0;
723
	struct smc_link *link;
724

725
	ini->is_smcd = false;
726 727
	ini->ib_lcl = &aclc->r0.lcl;
	ini->ib_clcqpn = ntoh24(aclc->r0.qpn);
728
	ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK;
729

730
	mutex_lock(&smc_client_lgr_pending);
731 732
	reason_code = smc_conn_create(smc, ini);
	if (reason_code) {
733 734
		mutex_unlock(&smc_client_lgr_pending);
		return reason_code;
735
	}
736

737
	smc_conn_save_peer_info(smc, aclc);
U
Ursula Braun 已提交
738

739
	if (ini->first_contact_local) {
740 741 742 743 744 745 746
		link = smc->conn.lnk;
	} else {
		/* set link that was assigned by server */
		link = NULL;
		for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
			struct smc_link *l = &smc->conn.lgr->lnk[i];

747 748 749 750 751
			if (l->peer_qpn == ntoh24(aclc->r0.qpn) &&
			    !memcmp(l->peer_gid, &aclc->r0.lcl.gid,
				    SMC_GID_SIZE) &&
			    !memcmp(l->peer_mac, &aclc->r0.lcl.mac,
				    sizeof(l->peer_mac))) {
752 753 754 755
				link = l;
				break;
			}
		}
756 757 758 759
		if (!link) {
			reason_code = SMC_CLC_DECL_NOSRVLINK;
			goto connect_abort;
		}
760 761 762
		smc->conn.lnk = link;
	}

763
	/* create send buffer and rmb */
764 765 766 767
	if (smc_buf_create(smc, false)) {
		reason_code = SMC_CLC_DECL_MEM;
		goto connect_abort;
	}
U
Ursula Braun 已提交
768

769
	if (ini->first_contact_local)
770
		smc_link_save_peer_info(link, aclc);
771

772 773 774 775
	if (smc_rmb_rtoken_handling(&smc->conn, link, aclc)) {
		reason_code = SMC_CLC_DECL_ERR_RTOK;
		goto connect_abort;
	}
776

777 778 779
	smc_close_init(smc);
	smc_rx_init(smc);

780
	if (ini->first_contact_local) {
781 782 783 784
		if (smc_ib_ready_link(link)) {
			reason_code = SMC_CLC_DECL_ERR_RDYLNK;
			goto connect_abort;
		}
785
	} else {
786 787 788 789
		if (smcr_lgr_reg_rmbs(link, smc->conn.rmb_desc)) {
			reason_code = SMC_CLC_DECL_ERR_REGRMB;
			goto connect_abort;
		}
790
	}
791
	smc_rmb_sync_sg_for_device(&smc->conn);
792

U
Ursula Braun 已提交
793 794
	reason_code = smc_clc_send_confirm(smc, ini->first_contact_local,
					   SMC_V1);
795
	if (reason_code)
796
		goto connect_abort;
797 798

	smc_tx_init(smc);
799

800
	if (ini->first_contact_local) {
U
Ursula Braun 已提交
801
		/* QP confirmation over RoCE fabric */
802
		smc_llc_flow_initiate(link->lgr, SMC_LLC_FLOW_ADD_LINK);
803
		reason_code = smcr_clnt_conf_first_link(smc);
804
		smc_llc_flow_stop(link->lgr, &link->lgr->llc_flow_lcl);
805
		if (reason_code)
806
			goto connect_abort;
U
Ursula Braun 已提交
807
	}
808
	mutex_unlock(&smc_client_lgr_pending);
U
Ursula Braun 已提交
809

810
	smc_copy_sock_settings_to_clc(smc);
811
	smc->connect_nonblock = 0;
812 813
	if (smc->sk.sk_state == SMC_INIT)
		smc->sk.sk_state = SMC_ACTIVE;
814

815
	return 0;
816 817 818 819 820 821
connect_abort:
	smc_connect_abort(smc, ini->first_contact_local);
	mutex_unlock(&smc_client_lgr_pending);
	smc->connect_nonblock = 0;

	return reason_code;
822
}
823

U
Ursula Braun 已提交
824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842
/* The server has chosen one of the proposed ISM devices for the communication.
 * Determine from the CHID of the received CLC ACCEPT the ISM device chosen.
 */
static int
smc_v2_determine_accepted_chid(struct smc_clc_msg_accept_confirm_v2 *aclc,
			       struct smc_init_info *ini)
{
	int i;

	for (i = 0; i < ini->ism_offered_cnt + 1; i++) {
		if (ini->ism_chid[i] == ntohs(aclc->chid)) {
			ini->ism_selected = i;
			return 0;
		}
	}

	return -EPROTO;
}

843 844 845
/* setup for ISM connection of client */
static int smc_connect_ism(struct smc_sock *smc,
			   struct smc_clc_msg_accept_confirm *aclc,
846
			   struct smc_init_info *ini)
847 848 849
{
	int rc = 0;

850
	ini->is_smcd = true;
851
	ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK;
852

U
Ursula Braun 已提交
853 854 855 856 857 858 859 860 861 862
	if (aclc->hdr.version == SMC_V2) {
		struct smc_clc_msg_accept_confirm_v2 *aclc_v2 =
			(struct smc_clc_msg_accept_confirm_v2 *)aclc;

		rc = smc_v2_determine_accepted_chid(aclc_v2, ini);
		if (rc)
			return rc;
	}
	ini->ism_peer_gid[ini->ism_selected] = aclc->d0.gid;

863 864
	/* there is only one lgr role for SMC-D; use server lock */
	mutex_lock(&smc_server_lgr_pending);
865 866
	rc = smc_conn_create(smc, ini);
	if (rc) {
867
		mutex_unlock(&smc_server_lgr_pending);
868
		return rc;
869
	}
870 871

	/* Create send and receive buffers */
872
	rc = smc_buf_create(smc, true);
873 874 875 876
	if (rc) {
		rc = (rc == -ENOSPC) ? SMC_CLC_DECL_MAX_DMB : SMC_CLC_DECL_MEM;
		goto connect_abort;
	}
877 878 879 880 881 882

	smc_conn_save_peer_info(smc, aclc);
	smc_close_init(smc);
	smc_rx_init(smc);
	smc_tx_init(smc);

U
Ursula Braun 已提交
883 884
	rc = smc_clc_send_confirm(smc, ini->first_contact_local,
				  aclc->hdr.version);
885
	if (rc)
886
		goto connect_abort;
887
	mutex_unlock(&smc_server_lgr_pending);
888 889

	smc_copy_sock_settings_to_clc(smc);
890
	smc->connect_nonblock = 0;
891 892 893 894
	if (smc->sk.sk_state == SMC_INIT)
		smc->sk.sk_state = SMC_ACTIVE;

	return 0;
895 896 897 898 899 900
connect_abort:
	smc_connect_abort(smc, ini->first_contact_local);
	mutex_unlock(&smc_server_lgr_pending);
	smc->connect_nonblock = 0;

	return rc;
901 902
}

903 904 905 906 907 908 909
/* check if received accept type and version matches a proposed one */
static int smc_connect_check_aclc(struct smc_init_info *ini,
				  struct smc_clc_msg_accept_confirm *aclc)
{
	if ((aclc->hdr.typev1 == SMC_TYPE_R &&
	     !smcr_indicated(ini->smc_type_v1)) ||
	    (aclc->hdr.typev1 == SMC_TYPE_D &&
U
Ursula Braun 已提交
910 911 912 913 914 915
	     ((!smcd_indicated(ini->smc_type_v1) &&
	       !smcd_indicated(ini->smc_type_v2)) ||
	      (aclc->hdr.version == SMC_V1 &&
	       !smcd_indicated(ini->smc_type_v1)) ||
	      (aclc->hdr.version == SMC_V2 &&
	       !smcd_indicated(ini->smc_type_v2)))))
916 917 918 919 920
		return SMC_CLC_DECL_MODEUNSUPP;

	return 0;
}

921 922 923
/* perform steps before actually connecting */
static int __smc_connect(struct smc_sock *smc)
{
924
	u8 version = smc_ism_v2_capable ? SMC_V2 : SMC_V1;
U
Ursula Braun 已提交
925 926
	struct smc_clc_msg_accept_confirm_v2 *aclc2;
	struct smc_clc_msg_accept_confirm *aclc;
927
	struct smc_init_info *ini = NULL;
U
Ursula Braun 已提交
928
	u8 *buf = NULL;
929
	int rc = 0;
930

931
	if (smc->use_fallback)
932
		return smc_connect_fallback(smc, smc->fallback_rsn);
933 934 935

	/* if peer has not signalled SMC-capability, fall back */
	if (!tcp_sk(smc->clcsock->sk)->syn_smc)
936
		return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC);
937

938
	/* IPSec connections opt out of SMC optimizations */
939
	if (using_ipsec(smc))
940 941
		return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC,
						    version);
942

943 944
	ini = kzalloc(sizeof(*ini), GFP_KERNEL);
	if (!ini)
945 946
		return smc_connect_decline_fallback(smc, SMC_CLC_DECL_MEM,
						    version);
947

948 949 950 951 952
	ini->smcd_version = SMC_V1;
	ini->smcd_version |= smc_ism_v2_capable ? SMC_V2 : 0;
	ini->smc_type_v1 = SMC_TYPE_B;
	ini->smc_type_v2 = smc_ism_v2_capable ? SMC_TYPE_D : SMC_TYPE_N;

K
Karsten Graul 已提交
953
	/* get vlan id from IP device */
954
	if (smc_vlan_by_tcpsk(smc->clcsock, ini)) {
955 956 957 958 959 960
		ini->smcd_version &= ~SMC_V1;
		ini->smc_type_v1 = SMC_TYPE_N;
		if (!ini->smcd_version) {
			rc = SMC_CLC_DECL_GETVLANERR;
			goto fallback;
		}
961 962
	}

963 964 965
	rc = smc_find_proposal_devices(smc, ini);
	if (rc)
		goto fallback;
966

U
Ursula Braun 已提交
967 968 969 970 971 972 973 974
	buf = kzalloc(SMC_CLC_MAX_ACCEPT_LEN, GFP_KERNEL);
	if (!buf) {
		rc = SMC_CLC_DECL_MEM;
		goto fallback;
	}
	aclc2 = (struct smc_clc_msg_accept_confirm_v2 *)buf;
	aclc = (struct smc_clc_msg_accept_confirm *)aclc2;

975
	/* perform CLC handshake */
U
Ursula Braun 已提交
976
	rc = smc_connect_clc(smc, aclc2, ini);
977 978 979 980
	if (rc)
		goto vlan_cleanup;

	/* check if smc modes and versions of CLC proposal and accept match */
U
Ursula Braun 已提交
981
	rc = smc_connect_check_aclc(ini, aclc);
982
	version = aclc->hdr.version == SMC_V1 ? SMC_V1 : version;
983 984
	if (rc)
		goto vlan_cleanup;
985

986
	/* depending on previous steps, connect using rdma or ism */
U
Ursula Braun 已提交
987 988 989 990
	if (aclc->hdr.typev1 == SMC_TYPE_R)
		rc = smc_connect_rdma(smc, aclc, ini);
	else if (aclc->hdr.typev1 == SMC_TYPE_D)
		rc = smc_connect_ism(smc, aclc, ini);
991 992
	if (rc)
		goto vlan_cleanup;
993

994
	smc_connect_ism_vlan_cleanup(smc, ini);
U
Ursula Braun 已提交
995
	kfree(buf);
996
	kfree(ini);
997
	return 0;
998 999 1000

vlan_cleanup:
	smc_connect_ism_vlan_cleanup(smc, ini);
U
Ursula Braun 已提交
1001
	kfree(buf);
1002 1003
fallback:
	kfree(ini);
1004
	return smc_connect_decline_fallback(smc, rc, version);
1005 1006
}

1007 1008 1009 1010
static void smc_connect_work(struct work_struct *work)
{
	struct smc_sock *smc = container_of(work, struct smc_sock,
					    connect_work);
1011 1012
	long timeo = smc->sk.sk_sndtimeo;
	int rc = 0;
1013

1014 1015 1016
	if (!timeo)
		timeo = MAX_SCHEDULE_TIMEOUT;
	lock_sock(smc->clcsock->sk);
1017 1018
	if (smc->clcsock->sk->sk_err) {
		smc->sk.sk_err = smc->clcsock->sk->sk_err;
1019 1020 1021 1022 1023 1024 1025
	} else if ((1 << smc->clcsock->sk->sk_state) &
					(TCPF_SYN_SENT | TCP_SYN_RECV)) {
		rc = sk_stream_wait_connect(smc->clcsock->sk, &timeo);
		if ((rc == -EPIPE) &&
		    ((1 << smc->clcsock->sk->sk_state) &
					(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)))
			rc = 0;
1026
	}
1027 1028 1029 1030 1031 1032 1033 1034
	release_sock(smc->clcsock->sk);
	lock_sock(&smc->sk);
	if (rc != 0 || smc->sk.sk_err) {
		smc->sk.sk_state = SMC_CLOSED;
		if (rc == -EPIPE || rc == -EAGAIN)
			smc->sk.sk_err = EPIPE;
		else if (signal_pending(current))
			smc->sk.sk_err = -sock_intr_errno(timeo);
1035
		sock_put(&smc->sk); /* passive closing */
1036 1037 1038 1039 1040 1041 1042 1043
		goto out;
	}

	rc = __smc_connect(smc);
	if (rc < 0)
		smc->sk.sk_err = -rc;

out:
1044 1045 1046 1047 1048 1049 1050 1051
	if (!sock_flag(&smc->sk, SOCK_DEAD)) {
		if (smc->sk.sk_err) {
			smc->sk.sk_state_change(&smc->sk);
		} else { /* allow polling before and after fallback decision */
			smc->clcsock->sk->sk_write_space(smc->clcsock->sk);
			smc->sk.sk_write_space(&smc->sk);
		}
	}
1052 1053 1054
	release_sock(&smc->sk);
}

U
Ursula Braun 已提交
1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066
static int smc_connect(struct socket *sock, struct sockaddr *addr,
		       int alen, int flags)
{
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
	int rc = -EINVAL;

	smc = smc_sk(sk);

	/* separate smc parameter checking to be safe */
	if (alen < sizeof(addr->sa_family))
		goto out_err;
1067
	if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
U
Ursula Braun 已提交
1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082
		goto out_err;

	lock_sock(sk);
	switch (sk->sk_state) {
	default:
		goto out;
	case SMC_ACTIVE:
		rc = -EISCONN;
		goto out;
	case SMC_INIT:
		rc = 0;
		break;
	}

	smc_copy_sock_settings_to_clc(smc);
U
Ursula Braun 已提交
1083
	tcp_sk(smc->clcsock->sk)->syn_smc = 1;
1084 1085 1086 1087 1088 1089 1090
	if (smc->connect_nonblock) {
		rc = -EALREADY;
		goto out;
	}
	rc = kernel_connect(smc->clcsock, addr, alen, flags);
	if (rc && rc != -EINPROGRESS)
		goto out;
1091 1092

	sock_hold(&smc->sk); /* sock put in passive closing */
1093 1094
	if (smc->use_fallback)
		goto out;
1095
	if (flags & O_NONBLOCK) {
1096
		if (queue_work(smc_hs_wq, &smc->connect_work))
1097
			smc->connect_nonblock = 1;
1098 1099 1100 1101 1102 1103 1104 1105
		rc = -EINPROGRESS;
	} else {
		rc = __smc_connect(smc);
		if (rc < 0)
			goto out;
		else
			rc = 0; /* success cases including fallback */
	}
U
Ursula Braun 已提交
1106 1107 1108 1109 1110 1111 1112 1113 1114

out:
	release_sock(sk);
out_err:
	return rc;
}

static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc)
{
1115 1116
	struct socket *new_clcsock = NULL;
	struct sock *lsk = &lsmc->sk;
U
Ursula Braun 已提交
1117
	struct sock *new_sk;
1118
	int rc = -EINVAL;
U
Ursula Braun 已提交
1119

1120
	release_sock(lsk);
1121
	new_sk = smc_sock_alloc(sock_net(lsk), NULL, lsk->sk_protocol);
U
Ursula Braun 已提交
1122 1123
	if (!new_sk) {
		rc = -ENOMEM;
1124
		lsk->sk_err = ENOMEM;
U
Ursula Braun 已提交
1125
		*new_smc = NULL;
1126
		lock_sock(lsk);
U
Ursula Braun 已提交
1127 1128 1129 1130
		goto out;
	}
	*new_smc = smc_sk(new_sk);

1131 1132
	mutex_lock(&lsmc->clcsock_release_lock);
	if (lsmc->clcsock)
1133
		rc = kernel_accept(lsmc->clcsock, &new_clcsock, SOCK_NONBLOCK);
1134
	mutex_unlock(&lsmc->clcsock_release_lock);
1135
	lock_sock(lsk);
1136
	if  (rc < 0 && rc != -EAGAIN)
1137
		lsk->sk_err = -rc;
1138
	if (rc < 0 || lsk->sk_state == SMC_CLOSED) {
1139
		new_sk->sk_prot->unhash(new_sk);
1140 1141 1142 1143
		if (new_clcsock)
			sock_release(new_clcsock);
		new_sk->sk_state = SMC_CLOSED;
		sock_set_flag(new_sk, SOCK_DEAD);
1144
		sock_put(new_sk); /* final */
U
Ursula Braun 已提交
1145 1146 1147 1148
		*new_smc = NULL;
		goto out;
	}

1149 1150 1151 1152
	/* new clcsock has inherited the smc listen-specific sk_data_ready
	 * function; switch it back to the original sk_data_ready function
	 */
	new_clcsock->sk->sk_data_ready = lsmc->clcsk_data_ready;
U
Ursula Braun 已提交
1153 1154 1155 1156 1157
	(*new_smc)->clcsock = new_clcsock;
out:
	return rc;
}

1158 1159 1160 1161 1162 1163 1164
/* add a just created sock to the accept queue of the listen sock as
 * candidate for a following socket accept call from user space
 */
static void smc_accept_enqueue(struct sock *parent, struct sock *sk)
{
	struct smc_sock *par = smc_sk(parent);

1165
	sock_hold(sk); /* sock_put in smc_accept_unlink () */
1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180
	spin_lock(&par->accept_q_lock);
	list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q);
	spin_unlock(&par->accept_q_lock);
	sk_acceptq_added(parent);
}

/* remove a socket from the accept queue of its parental listening socket */
static void smc_accept_unlink(struct sock *sk)
{
	struct smc_sock *par = smc_sk(sk)->listen_smc;

	spin_lock(&par->accept_q_lock);
	list_del_init(&smc_sk(sk)->accept_q);
	spin_unlock(&par->accept_q_lock);
	sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk);
1181
	sock_put(sk); /* sock_hold in smc_accept_enqueue */
1182 1183 1184 1185 1186
}

/* remove a sock from the accept queue to bind it to a new socket created
 * for a socket accept call from user space
 */
1187 1188
struct sock *smc_accept_dequeue(struct sock *parent,
				struct socket *new_sock)
1189 1190 1191 1192 1193 1194 1195 1196 1197
{
	struct smc_sock *isk, *n;
	struct sock *new_sk;

	list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) {
		new_sk = (struct sock *)isk;

		smc_accept_unlink(new_sk);
		if (new_sk->sk_state == SMC_CLOSED) {
1198
			new_sk->sk_prot->unhash(new_sk);
1199 1200 1201 1202
			if (isk->clcsock) {
				sock_release(isk->clcsock);
				isk->clcsock = NULL;
			}
1203
			sock_put(new_sk); /* final */
1204 1205
			continue;
		}
1206
		if (new_sock) {
1207
			sock_graft(new_sk, new_sock);
1208 1209 1210 1211 1212
			if (isk->use_fallback) {
				smc_sk(new_sk)->clcsock->file = new_sock->file;
				isk->clcsock->file->private_data = isk->clcsock;
			}
		}
1213 1214 1215 1216 1217 1218
		return new_sk;
	}
	return NULL;
}

/* clean up for a created but never accepted sock */
1219
void smc_close_non_accepted(struct sock *sk)
1220 1221 1222
{
	struct smc_sock *smc = smc_sk(sk);

U
Ursula Braun 已提交
1223
	sock_hold(sk); /* sock_put below */
1224 1225 1226 1227
	lock_sock(sk);
	if (!sk->sk_lingertime)
		/* wait for peer closing */
		sk->sk_lingertime = SMC_MAX_STREAM_WAIT_TIMEOUT;
1228
	__smc_release(smc);
1229
	release_sock(sk);
U
Ursula Braun 已提交
1230
	sock_put(sk); /* sock_hold above */
1231
	sock_put(sk); /* final sock_put */
1232 1233
}

1234
static int smcr_serv_conf_first_link(struct smc_sock *smc)
U
Ursula Braun 已提交
1235
{
1236
	struct smc_link *link = smc->conn.lnk;
1237
	struct smc_llc_qentry *qentry;
U
Ursula Braun 已提交
1238 1239
	int rc;

1240
	if (smcr_link_reg_rmb(link, smc->conn.rmb_desc))
1241
		return SMC_CLC_DECL_ERR_REGRMB;
1242

U
Ursula Braun 已提交
1243
	/* send CONFIRM LINK request to client over the RoCE fabric */
1244
	rc = smc_llc_send_confirm_link(link, SMC_LLC_REQ);
U
Ursula Braun 已提交
1245
	if (rc < 0)
1246
		return SMC_CLC_DECL_TIMEOUT_CL;
U
Ursula Braun 已提交
1247 1248

	/* receive CONFIRM LINK response from client over the RoCE fabric */
1249 1250 1251
	qentry = smc_llc_wait(link->lgr, link, SMC_LLC_WAIT_TIME,
			      SMC_LLC_CONFIRM_LINK);
	if (!qentry) {
U
Ursula Braun 已提交
1252 1253 1254
		struct smc_clc_msg_decline dclc;

		rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
1255
				      SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
1256
		return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
U
Ursula Braun 已提交
1257
	}
1258
	smc_llc_save_peer_uid(qentry);
1259 1260 1261
	rc = smc_llc_eval_conf_link(qentry, SMC_LLC_RESP);
	smc_llc_flow_qentry_del(&link->lgr->llc_flow_lcl);
	if (rc)
1262 1263
		return SMC_CLC_DECL_RMBE_EC;

1264 1265
	/* confirm_rkey is implicit on 1st contact */
	smc->conn.rmb_desc->is_conf_rkey = true;
1266

1267
	smc_llc_link_active(link);
1268
	smcr_lgr_set_type(link->lgr, SMC_LGR_SINGLE);
1269

1270
	/* initial contact - try to establish second link */
1271
	smc_llc_srv_add_link(link);
1272
	return 0;
U
Ursula Braun 已提交
1273 1274
}

1275 1276
/* listen worker: finish */
static void smc_listen_out(struct smc_sock *new_smc)
1277 1278 1279 1280
{
	struct smc_sock *lsmc = new_smc->listen_smc;
	struct sock *newsmcsk = &new_smc->sk;

1281
	if (lsmc->sk.sk_state == SMC_LISTEN) {
1282
		lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING);
1283
		smc_accept_enqueue(&lsmc->sk, newsmcsk);
1284
		release_sock(&lsmc->sk);
1285 1286
	} else { /* no longer listening */
		smc_close_non_accepted(newsmcsk);
U
Ursula Braun 已提交
1287 1288
	}

1289 1290 1291 1292
	/* Wake up accept */
	lsmc->sk.sk_data_ready(&lsmc->sk);
	sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */
}
1293

1294 1295 1296 1297
/* listen worker: finish in state connected */
static void smc_listen_out_connected(struct smc_sock *new_smc)
{
	struct sock *newsmcsk = &new_smc->sk;
1298

1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319
	sk_refcnt_debug_inc(newsmcsk);
	if (newsmcsk->sk_state == SMC_INIT)
		newsmcsk->sk_state = SMC_ACTIVE;

	smc_listen_out(new_smc);
}

/* listen worker: finish in error state */
static void smc_listen_out_err(struct smc_sock *new_smc)
{
	struct sock *newsmcsk = &new_smc->sk;

	if (newsmcsk->sk_state == SMC_INIT)
		sock_put(&new_smc->sk); /* passive closing */
	newsmcsk->sk_state = SMC_CLOSED;

	smc_listen_out(new_smc);
}

/* listen worker: decline and fall back if possible */
static void smc_listen_decline(struct smc_sock *new_smc, int reason_code,
1320
			       int local_first, u8 version)
1321 1322
{
	/* RDMA setup failed, switch back to TCP */
1323
	if (local_first)
1324 1325 1326
		smc_lgr_cleanup_early(&new_smc->conn);
	else
		smc_conn_free(&new_smc->conn);
1327 1328 1329 1330
	if (reason_code < 0) { /* error, no fallback possible */
		smc_listen_out_err(new_smc);
		return;
	}
1331
	smc_switch_to_fallback(new_smc);
1332 1333
	new_smc->fallback_rsn = reason_code;
	if (reason_code && reason_code != SMC_CLC_DECL_PEERDECL) {
1334
		if (smc_clc_send_decline(new_smc, reason_code, version) < 0) {
1335 1336 1337
			smc_listen_out_err(new_smc);
			return;
		}
1338
	}
1339 1340 1341
	smc_listen_out_connected(new_smc);
}

1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382
/* listen worker: version checking */
static int smc_listen_v2_check(struct smc_sock *new_smc,
			       struct smc_clc_msg_proposal *pclc,
			       struct smc_init_info *ini)
{
	struct smc_clc_smcd_v2_extension *pclc_smcd_v2_ext;
	struct smc_clc_v2_extension *pclc_v2_ext;

	ini->smc_type_v1 = pclc->hdr.typev1;
	ini->smc_type_v2 = pclc->hdr.typev2;
	ini->smcd_version = ini->smc_type_v1 != SMC_TYPE_N ? SMC_V1 : 0;
	if (pclc->hdr.version > SMC_V1)
		ini->smcd_version |=
				ini->smc_type_v2 != SMC_TYPE_N ? SMC_V2 : 0;
	if (!smc_ism_v2_capable) {
		ini->smcd_version &= ~SMC_V2;
		goto out;
	}
	pclc_v2_ext = smc_get_clc_v2_ext(pclc);
	if (!pclc_v2_ext) {
		ini->smcd_version &= ~SMC_V2;
		goto out;
	}
	pclc_smcd_v2_ext = smc_get_clc_smcd_v2_ext(pclc_v2_ext);
	if (!pclc_smcd_v2_ext)
		ini->smcd_version &= ~SMC_V2;

out:
	if (!ini->smcd_version) {
		if (pclc->hdr.typev1 == SMC_TYPE_B ||
		    pclc->hdr.typev2 == SMC_TYPE_B)
			return SMC_CLC_DECL_NOSMCDEV;
		if (pclc->hdr.typev1 == SMC_TYPE_D ||
		    pclc->hdr.typev2 == SMC_TYPE_D)
			return SMC_CLC_DECL_NOSMCDDEV;
		return SMC_CLC_DECL_NOSMCRDEV;
	}

	return 0;
}

1383
/* listen worker: check prefixes */
1384
static int smc_listen_prfx_check(struct smc_sock *new_smc,
1385 1386 1387 1388
				 struct smc_clc_msg_proposal *pclc)
{
	struct smc_clc_msg_proposal_prefix *pclc_prfx;
	struct socket *newclcsock = new_smc->clcsock;
1389

1390 1391
	if (pclc->hdr.typev1 == SMC_TYPE_N)
		return 0;
1392
	pclc_prfx = smc_clc_proposal_get_prefix(pclc);
1393
	if (smc_clc_prfx_match(newclcsock, pclc_prfx))
1394
		return SMC_CLC_DECL_DIFFPREFIX;
1395

1396 1397
	return 0;
}
1398

1399 1400
/* listen worker: initialize connection and buffers */
static int smc_listen_rdma_init(struct smc_sock *new_smc,
1401
				struct smc_init_info *ini)
1402
{
1403 1404
	int rc;

1405
	/* allocate connection / link group */
1406 1407 1408
	rc = smc_conn_create(new_smc, ini);
	if (rc)
		return rc;
1409

1410
	/* create send buffer and rmb */
1411
	if (smc_buf_create(new_smc, false))
1412
		return SMC_CLC_DECL_MEM;
1413

1414 1415 1416
	return 0;
}

1417 1418
/* listen worker: initialize connection and buffers for SMC-D */
static int smc_listen_ism_init(struct smc_sock *new_smc,
1419
			       struct smc_init_info *ini)
1420
{
1421
	int rc;
1422

1423 1424 1425
	rc = smc_conn_create(new_smc, ini);
	if (rc)
		return rc;
1426 1427

	/* Create send and receive buffers */
1428 1429
	rc = smc_buf_create(new_smc, true);
	if (rc) {
1430
		if (ini->first_contact_local)
1431 1432 1433
			smc_lgr_cleanup_early(&new_smc->conn);
		else
			smc_conn_free(&new_smc->conn);
1434 1435
		return (rc == -ENOSPC) ? SMC_CLC_DECL_MAX_DMB :
					 SMC_CLC_DECL_MEM;
1436 1437 1438 1439 1440
	}

	return 0;
}

1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483
static bool smc_is_already_selected(struct smcd_dev *smcd,
				    struct smc_init_info *ini,
				    int matches)
{
	int i;

	for (i = 0; i < matches; i++)
		if (smcd == ini->ism_dev[i])
			return true;

	return false;
}

/* check for ISM devices matching proposed ISM devices */
static void smc_check_ism_v2_match(struct smc_init_info *ini,
				   u16 proposed_chid, u64 proposed_gid,
				   unsigned int *matches)
{
	struct smcd_dev *smcd;

	list_for_each_entry(smcd, &smcd_dev_list.list, list) {
		if (smcd->going_away)
			continue;
		if (smc_is_already_selected(smcd, ini, *matches))
			continue;
		if (smc_ism_get_chid(smcd) == proposed_chid &&
		    !smc_ism_cantalk(proposed_gid, ISM_RESERVED_VLANID, smcd)) {
			ini->ism_peer_gid[*matches] = proposed_gid;
			ini->ism_dev[*matches] = smcd;
			(*matches)++;
			break;
		}
	}
}

static void smc_find_ism_v2_device_serv(struct smc_sock *new_smc,
					struct smc_clc_msg_proposal *pclc,
					struct smc_init_info *ini)
{
	struct smc_clc_smcd_v2_extension *smcd_v2_ext;
	struct smc_clc_v2_extension *smc_v2_ext;
	struct smc_clc_msg_smcd *pclc_smcd;
	unsigned int matches = 0;
1484
	u8 smcd_version;
1485 1486 1487 1488
	u8 *eid = NULL;
	int i;

	if (!(ini->smcd_version & SMC_V2) || !smcd_indicated(ini->smc_type_v2))
1489
		goto not_found;
1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522

	pclc_smcd = smc_get_clc_msg_smcd(pclc);
	smc_v2_ext = smc_get_clc_v2_ext(pclc);
	smcd_v2_ext = smc_get_clc_smcd_v2_ext(smc_v2_ext);
	if (!smcd_v2_ext ||
	    !smc_v2_ext->hdr.flag.seid) /* no system EID support for SMCD */
		goto not_found;

	mutex_lock(&smcd_dev_list.mutex);
	if (pclc_smcd->ism.chid)
		/* check for ISM device matching proposed native ISM device */
		smc_check_ism_v2_match(ini, ntohs(pclc_smcd->ism.chid),
				       ntohll(pclc_smcd->ism.gid), &matches);
	for (i = 1; i <= smc_v2_ext->hdr.ism_gid_cnt; i++) {
		/* check for ISM devices matching proposed non-native ISM
		 * devices
		 */
		smc_check_ism_v2_match(ini,
				       ntohs(smcd_v2_ext->gidchid[i - 1].chid),
				       ntohll(smcd_v2_ext->gidchid[i - 1].gid),
				       &matches);
	}
	mutex_unlock(&smcd_dev_list.mutex);

	if (ini->ism_dev[0]) {
		smc_ism_get_system_eid(ini->ism_dev[0], &eid);
		if (memcmp(eid, smcd_v2_ext->system_eid, SMC_MAX_EID_LEN))
			goto not_found;
	} else {
		goto not_found;
	}

	/* separate - outside the smcd_dev_list.lock */
1523
	smcd_version = ini->smcd_version;
1524 1525 1526 1527 1528 1529 1530 1531 1532
	for (i = 0; i < matches; i++) {
		ini->smcd_version = SMC_V2;
		ini->is_smcd = true;
		ini->ism_selected = i;
		if (smc_listen_ism_init(new_smc, ini))
			/* try next active ISM device */
			continue;
		return; /* matching and usable V2 ISM device found */
	}
1533 1534
	/* no V2 ISM device could be initialized */
	ini->smcd_version = smcd_version;	/* restore original value */
1535 1536 1537 1538 1539 1540 1541 1542 1543 1544

not_found:
	ini->smcd_version &= ~SMC_V2;
	ini->ism_dev[0] = NULL;
	ini->is_smcd = false;
}

static void smc_find_ism_v1_device_serv(struct smc_sock *new_smc,
					struct smc_clc_msg_proposal *pclc,
					struct smc_init_info *ini)
1545 1546 1547
{
	struct smc_clc_msg_smcd *pclc_smcd = smc_get_clc_msg_smcd(pclc);

1548 1549
	/* check if ISM V1 is available */
	if (!(ini->smcd_version & SMC_V1) || !smcd_indicated(ini->smc_type_v1))
1550 1551
		goto not_found;
	ini->is_smcd = true; /* prepare ISM check */
1552
	ini->ism_peer_gid[0] = ntohll(pclc_smcd->ism.gid);
1553 1554
	if (smc_find_ism_device(new_smc, ini))
		goto not_found;
1555
	ini->ism_selected = 0;
1556
	if (!smc_listen_ism_init(new_smc, ini))
1557
		return;		/* V1 ISM device found */
1558 1559

not_found:
1560
	ini->ism_dev[0] = NULL;
1561 1562 1563
	ini->is_smcd = false;
}

1564
/* listen worker: register buffers */
1565
static int smc_listen_rdma_reg(struct smc_sock *new_smc, bool local_first)
1566
{
1567
	struct smc_connection *conn = &new_smc->conn;
1568

1569
	if (!local_first) {
1570
		if (smcr_lgr_reg_rmbs(conn->lnk, conn->rmb_desc))
1571
			return SMC_CLC_DECL_ERR_REGRMB;
1572
	}
1573
	smc_rmb_sync_sg_for_device(&new_smc->conn);
1574

1575 1576 1577
	return 0;
}

1578 1579 1580
static int smc_find_rdma_v1_device_serv(struct smc_sock *new_smc,
					struct smc_clc_msg_proposal *pclc,
					struct smc_init_info *ini)
1581 1582 1583
{
	int rc;

1584
	if (!smcr_indicated(ini->smc_type_v1))
1585 1586 1587 1588 1589 1590 1591
		return SMC_CLC_DECL_NOSMCDEV;

	/* prepare RDMA check */
	ini->ib_lcl = &pclc->lcl;
	rc = smc_find_rdma_device(new_smc, ini);
	if (rc) {
		/* no RDMA device found */
1592
		if (ini->smc_type_v1 == SMC_TYPE_B)
1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607
			/* neither ISM nor RDMA device found */
			rc = SMC_CLC_DECL_NOSMCDEV;
		return rc;
	}
	rc = smc_listen_rdma_init(new_smc, ini);
	if (rc)
		return rc;
	return smc_listen_rdma_reg(new_smc, ini->first_contact_local);
}

/* determine the local device matching to proposal */
static int smc_listen_find_device(struct smc_sock *new_smc,
				  struct smc_clc_msg_proposal *pclc,
				  struct smc_init_info *ini)
{
1608 1609 1610 1611 1612
	int rc;

	/* check for ISM device matching V2 proposed device */
	smc_find_ism_v2_device_serv(new_smc, pclc, ini);
	if (ini->ism_dev[0])
1613
		return 0;
1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631

	if (!(ini->smcd_version & SMC_V1))
		return SMC_CLC_DECL_NOSMCDEV;

	/* check for matching IP prefix and subnet length */
	rc = smc_listen_prfx_check(new_smc, pclc);
	if (rc)
		return rc;

	/* get vlan id from IP device */
	if (smc_vlan_by_tcpsk(new_smc->clcsock, ini))
		return SMC_CLC_DECL_GETVLANERR;

	/* check for ISM device matching V1 proposed device */
	smc_find_ism_v1_device_serv(new_smc, pclc, ini);
	if (ini->ism_dev[0])
		return 0;

1632 1633 1634 1635
	if (pclc->hdr.typev1 == SMC_TYPE_D)
		return SMC_CLC_DECL_NOSMCDDEV; /* skip RDMA and decline */

	/* check if RDMA is available */
1636
	return smc_find_rdma_v1_device_serv(new_smc, pclc, ini);
1637 1638
}

1639
/* listen worker: finish RDMA setup */
1640 1641
static int smc_listen_rdma_finish(struct smc_sock *new_smc,
				  struct smc_clc_msg_accept_confirm *cclc,
1642
				  bool local_first)
1643
{
1644
	struct smc_link *link = new_smc->conn.lnk;
1645
	int reason_code = 0;
1646

1647
	if (local_first)
1648
		smc_link_save_peer_info(link, cclc);
1649

1650 1651
	if (smc_rmb_rtoken_handling(&new_smc->conn, link, cclc))
		return SMC_CLC_DECL_ERR_RTOK;
1652

1653
	if (local_first) {
1654 1655
		if (smc_ib_ready_link(link))
			return SMC_CLC_DECL_ERR_RDYLNK;
U
Ursula Braun 已提交
1656
		/* QP confirmation over RoCE fabric */
1657
		smc_llc_flow_initiate(link->lgr, SMC_LLC_FLOW_ADD_LINK);
1658
		reason_code = smcr_serv_conf_first_link(new_smc);
1659
		smc_llc_flow_stop(link->lgr, &link->lgr->llc_flow_lcl);
1660
	}
1661
	return reason_code;
1662
}
U
Ursula Braun 已提交
1663

1664
/* setup for connection of server */
1665 1666 1667 1668
static void smc_listen_work(struct work_struct *work)
{
	struct smc_sock *new_smc = container_of(work, struct smc_sock,
						smc_listen_work);
1669
	u8 version = smc_ism_v2_capable ? SMC_V2 : SMC_V1;
1670
	struct socket *newclcsock = new_smc->clcsock;
U
Ursula Braun 已提交
1671
	struct smc_clc_msg_accept_confirm *cclc;
1672
	struct smc_clc_msg_proposal_area *buf;
1673
	struct smc_clc_msg_proposal *pclc;
1674
	struct smc_init_info *ini = NULL;
1675 1676
	int rc = 0;

1677 1678 1679
	if (new_smc->listen_smc->sk.sk_state != SMC_LISTEN)
		return smc_listen_out_err(new_smc);

1680 1681 1682
	if (new_smc->use_fallback) {
		smc_listen_out_connected(new_smc);
		return;
1683 1684
	}

1685 1686
	/* check if peer is smc capable */
	if (!tcp_sk(newclcsock->sk)->syn_smc) {
1687
		smc_switch_to_fallback(new_smc);
1688
		new_smc->fallback_rsn = SMC_CLC_DECL_PEERNOSMC;
1689 1690 1691
		smc_listen_out_connected(new_smc);
		return;
	}
1692

1693 1694 1695
	/* do inband token exchange -
	 * wait for and receive SMC Proposal CLC message
	 */
1696 1697 1698 1699 1700 1701 1702
	buf = kzalloc(sizeof(*buf), GFP_KERNEL);
	if (!buf) {
		rc = SMC_CLC_DECL_MEM;
		goto out_decl;
	}
	pclc = (struct smc_clc_msg_proposal *)buf;
	rc = smc_clc_wait_msg(new_smc, pclc, sizeof(*buf),
1703
			      SMC_CLC_PROPOSAL, CLC_WAIT_TIME);
1704 1705
	if (rc)
		goto out_decl;
1706
	version = pclc->hdr.version == SMC_V1 ? SMC_V1 : version;
1707

1708
	/* IPSec connections opt out of SMC optimizations */
1709
	if (using_ipsec(new_smc)) {
1710 1711
		rc = SMC_CLC_DECL_IPSEC;
		goto out_decl;
1712 1713
	}

1714 1715 1716 1717 1718 1719
	ini = kzalloc(sizeof(*ini), GFP_KERNEL);
	if (!ini) {
		rc = SMC_CLC_DECL_MEM;
		goto out_decl;
	}

1720 1721 1722
	/* initial version checking */
	rc = smc_listen_v2_check(new_smc, pclc, ini);
	if (rc)
1723
		goto out_decl;
K
Karsten Graul 已提交
1724

1725
	mutex_lock(&smc_server_lgr_pending);
1726 1727 1728 1729
	smc_close_init(new_smc);
	smc_rx_init(new_smc);
	smc_tx_init(new_smc);

1730
	/* determine ISM or RoCE device used for connection */
1731
	rc = smc_listen_find_device(new_smc, pclc, ini);
1732 1733
	if (rc)
		goto out_unlock;
1734 1735

	/* send SMC Accept CLC message */
U
Ursula Braun 已提交
1736 1737
	rc = smc_clc_send_accept(new_smc, ini->first_contact_local,
				 ini->smcd_version == SMC_V2 ? SMC_V2 : SMC_V1);
1738 1739
	if (rc)
		goto out_unlock;
1740

1741
	/* SMC-D does not need this lock any more */
1742
	if (ini->is_smcd)
1743
		mutex_unlock(&smc_server_lgr_pending);
1744

1745
	/* receive SMC Confirm CLC message */
1746 1747 1748
	memset(buf, 0, sizeof(*buf));
	cclc = (struct smc_clc_msg_accept_confirm *)buf;
	rc = smc_clc_wait_msg(new_smc, cclc, sizeof(*buf),
1749 1750
			      SMC_CLC_CONFIRM, CLC_WAIT_TIME);
	if (rc) {
1751
		if (!ini->is_smcd)
1752 1753
			goto out_unlock;
		goto out_decl;
1754 1755 1756
	}

	/* finish worker */
1757
	if (!ini->is_smcd) {
U
Ursula Braun 已提交
1758
		rc = smc_listen_rdma_finish(new_smc, cclc,
1759
					    ini->first_contact_local);
1760
		if (rc)
1761 1762
			goto out_unlock;
		mutex_unlock(&smc_server_lgr_pending);
1763
	}
U
Ursula Braun 已提交
1764
	smc_conn_save_peer_info(new_smc, cclc);
1765
	smc_listen_out_connected(new_smc);
1766
	goto out_free;
1767 1768 1769 1770

out_unlock:
	mutex_unlock(&smc_server_lgr_pending);
out_decl:
1771 1772
	smc_listen_decline(new_smc, rc, ini ? ini->first_contact_local : 0,
			   version);
1773
out_free:
1774
	kfree(ini);
1775
	kfree(buf);
1776 1777 1778 1779 1780 1781
}

static void smc_tcp_listen_work(struct work_struct *work)
{
	struct smc_sock *lsmc = container_of(work, struct smc_sock,
					     tcp_listen_work);
1782
	struct sock *lsk = &lsmc->sk;
1783 1784 1785
	struct smc_sock *new_smc;
	int rc = 0;

1786 1787
	lock_sock(lsk);
	while (lsk->sk_state == SMC_LISTEN) {
1788
		rc = smc_clcsock_accept(lsmc, &new_smc);
1789
		if (rc) /* clcsock accept queue empty or error */
1790 1791 1792 1793 1794
			goto out;
		if (!new_smc)
			continue;

		new_smc->listen_smc = lsmc;
1795
		new_smc->use_fallback = lsmc->use_fallback;
1796
		new_smc->fallback_rsn = lsmc->fallback_rsn;
1797
		sock_hold(lsk); /* sock_put in smc_listen_work */
1798 1799
		INIT_WORK(&new_smc->smc_listen_work, smc_listen_work);
		smc_copy_sock_settings_to_smc(new_smc);
1800 1801
		new_smc->sk.sk_sndbuf = lsmc->sk.sk_sndbuf;
		new_smc->sk.sk_rcvbuf = lsmc->sk.sk_rcvbuf;
1802
		sock_hold(&new_smc->sk); /* sock_put in passive closing */
1803
		if (!queue_work(smc_hs_wq, &new_smc->smc_listen_work))
1804
			sock_put(&new_smc->sk);
1805 1806 1807
	}

out:
1808
	release_sock(lsk);
1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822
	sock_put(&lsmc->sk); /* sock_hold in smc_clcsock_data_ready() */
}

static void smc_clcsock_data_ready(struct sock *listen_clcsock)
{
	struct smc_sock *lsmc;

	lsmc = (struct smc_sock *)
	       ((uintptr_t)listen_clcsock->sk_user_data & ~SK_USER_DATA_NOCOPY);
	if (!lsmc)
		return;
	lsmc->clcsk_data_ready(listen_clcsock);
	if (lsmc->sk.sk_state == SMC_LISTEN) {
		sock_hold(&lsmc->sk); /* sock_put in smc_tcp_listen_work() */
1823
		if (!queue_work(smc_hs_wq, &lsmc->tcp_listen_work))
1824 1825
			sock_put(&lsmc->sk);
	}
1826 1827
}

U
Ursula Braun 已提交
1828 1829 1830 1831 1832 1833 1834 1835 1836 1837
static int smc_listen(struct socket *sock, int backlog)
{
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
	int rc;

	smc = smc_sk(sk);
	lock_sock(sk);

	rc = -EINVAL;
1838 1839
	if ((sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) ||
	    smc->connect_nonblock)
U
Ursula Braun 已提交
1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850
		goto out;

	rc = 0;
	if (sk->sk_state == SMC_LISTEN) {
		sk->sk_max_ack_backlog = backlog;
		goto out;
	}
	/* some socket options are handled in core, so we could not apply
	 * them to the clc socket -- copy smc socket options to clc socket
	 */
	smc_copy_sock_settings_to_clc(smc);
1851 1852
	if (!smc->use_fallback)
		tcp_sk(smc->clcsock->sk)->syn_smc = 1;
U
Ursula Braun 已提交
1853

1854 1855 1856 1857 1858 1859 1860
	/* save original sk_data_ready function and establish
	 * smc-specific sk_data_ready function
	 */
	smc->clcsk_data_ready = smc->clcsock->sk->sk_data_ready;
	smc->clcsock->sk->sk_data_ready = smc_clcsock_data_ready;
	smc->clcsock->sk->sk_user_data =
		(void *)((uintptr_t)smc | SK_USER_DATA_NOCOPY);
U
Ursula Braun 已提交
1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873
	rc = kernel_listen(smc->clcsock, backlog);
	if (rc)
		goto out;
	sk->sk_max_ack_backlog = backlog;
	sk->sk_ack_backlog = 0;
	sk->sk_state = SMC_LISTEN;

out:
	release_sock(sk);
	return rc;
}

static int smc_accept(struct socket *sock, struct socket *new_sock,
1874
		      int flags, bool kern)
U
Ursula Braun 已提交
1875
{
1876 1877
	struct sock *sk = sock->sk, *nsk;
	DECLARE_WAITQUEUE(wait, current);
U
Ursula Braun 已提交
1878
	struct smc_sock *lsmc;
1879 1880
	long timeo;
	int rc = 0;
U
Ursula Braun 已提交
1881 1882

	lsmc = smc_sk(sk);
1883
	sock_hold(sk); /* sock_put below */
U
Ursula Braun 已提交
1884 1885 1886 1887
	lock_sock(sk);

	if (lsmc->sk.sk_state != SMC_LISTEN) {
		rc = -EINVAL;
1888
		release_sock(sk);
U
Ursula Braun 已提交
1889 1890 1891
		goto out;
	}

1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912
	/* Wait for an incoming connection */
	timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
	add_wait_queue_exclusive(sk_sleep(sk), &wait);
	while (!(nsk = smc_accept_dequeue(sk, new_sock))) {
		set_current_state(TASK_INTERRUPTIBLE);
		if (!timeo) {
			rc = -EAGAIN;
			break;
		}
		release_sock(sk);
		timeo = schedule_timeout(timeo);
		/* wakeup by sk_data_ready in smc_listen_work() */
		sched_annotate_sleep();
		lock_sock(sk);
		if (signal_pending(current)) {
			rc = sock_intr_errno(timeo);
			break;
		}
	}
	set_current_state(TASK_RUNNING);
	remove_wait_queue(sk_sleep(sk), &wait);
U
Ursula Braun 已提交
1913

1914 1915
	if (!rc)
		rc = sock_error(nsk);
1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932
	release_sock(sk);
	if (rc)
		goto out;

	if (lsmc->sockopt_defer_accept && !(flags & O_NONBLOCK)) {
		/* wait till data arrives on the socket */
		timeo = msecs_to_jiffies(lsmc->sockopt_defer_accept *
								MSEC_PER_SEC);
		if (smc_sk(nsk)->use_fallback) {
			struct sock *clcsk = smc_sk(nsk)->clcsock->sk;

			lock_sock(clcsk);
			if (skb_queue_empty(&clcsk->sk_receive_queue))
				sk_wait_data(clcsk, &timeo, NULL);
			release_sock(clcsk);
		} else if (!atomic_read(&smc_sk(nsk)->conn.bytes_to_rcv)) {
			lock_sock(nsk);
1933
			smc_rx_wait(smc_sk(nsk), &timeo, smc_rx_data_available);
1934 1935 1936
			release_sock(nsk);
		}
	}
U
Ursula Braun 已提交
1937 1938

out:
1939
	sock_put(sk); /* sock_hold above */
U
Ursula Braun 已提交
1940 1941 1942 1943
	return rc;
}

static int smc_getname(struct socket *sock, struct sockaddr *addr,
1944
		       int peer)
U
Ursula Braun 已提交
1945 1946 1947
{
	struct smc_sock *smc;

1948 1949
	if (peer && (sock->sk->sk_state != SMC_ACTIVE) &&
	    (sock->sk->sk_state != SMC_APPCLOSEWAIT1))
U
Ursula Braun 已提交
1950 1951 1952 1953
		return -ENOTCONN;

	smc = smc_sk(sock->sk);

1954
	return smc->clcsock->ops->getname(smc->clcsock, addr, peer);
U
Ursula Braun 已提交
1955 1956 1957 1958 1959 1960 1961 1962 1963 1964
}

static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
{
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
	int rc = -EPIPE;

	smc = smc_sk(sk);
	lock_sock(sk);
1965 1966 1967
	if ((sk->sk_state != SMC_ACTIVE) &&
	    (sk->sk_state != SMC_APPCLOSEWAIT1) &&
	    (sk->sk_state != SMC_INIT))
U
Ursula Braun 已提交
1968
		goto out;
1969 1970

	if (msg->msg_flags & MSG_FASTOPEN) {
1971
		if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
1972
			smc_switch_to_fallback(smc);
1973
			smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP;
1974 1975 1976 1977 1978 1979
		} else {
			rc = -EINVAL;
			goto out;
		}
	}

U
Ursula Braun 已提交
1980 1981 1982
	if (smc->use_fallback)
		rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len);
	else
U
Ursula Braun 已提交
1983
		rc = smc_tx_sendmsg(smc, msg, len);
U
Ursula Braun 已提交
1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997
out:
	release_sock(sk);
	return rc;
}

static int smc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
		       int flags)
{
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
	int rc = -ENOTCONN;

	smc = smc_sk(sk);
	lock_sock(sk);
1998 1999 2000 2001 2002
	if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
		/* socket was connected before, no more data to read */
		rc = 0;
		goto out;
	}
2003 2004 2005
	if ((sk->sk_state == SMC_INIT) ||
	    (sk->sk_state == SMC_LISTEN) ||
	    (sk->sk_state == SMC_CLOSED))
U
Ursula Braun 已提交
2006 2007
		goto out;

2008 2009 2010 2011 2012
	if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
		rc = 0;
		goto out;
	}

S
Stefan Raspl 已提交
2013
	if (smc->use_fallback) {
U
Ursula Braun 已提交
2014
		rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags);
S
Stefan Raspl 已提交
2015 2016 2017 2018
	} else {
		msg->msg_namelen = 0;
		rc = smc_rx_recvmsg(smc, msg, NULL, len, flags);
	}
2019

U
Ursula Braun 已提交
2020 2021 2022 2023 2024
out:
	release_sock(sk);
	return rc;
}

A
Al Viro 已提交
2025
static __poll_t smc_accept_poll(struct sock *parent)
2026
{
U
Ursula Braun 已提交
2027
	struct smc_sock *isk = smc_sk(parent);
A
Al Viro 已提交
2028
	__poll_t mask = 0;
2029

U
Ursula Braun 已提交
2030 2031
	spin_lock(&isk->accept_q_lock);
	if (!list_empty(&isk->accept_q))
2032
		mask = EPOLLIN | EPOLLRDNORM;
U
Ursula Braun 已提交
2033
	spin_unlock(&isk->accept_q_lock);
2034

U
Ursula Braun 已提交
2035
	return mask;
2036 2037
}

2038 2039
static __poll_t smc_poll(struct file *file, struct socket *sock,
			     poll_table *wait)
U
Ursula Braun 已提交
2040 2041 2042
{
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
2043
	__poll_t mask = 0;
U
Ursula Braun 已提交
2044

U
Ursula Braun 已提交
2045
	if (!sk)
2046
		return EPOLLNVAL;
U
Ursula Braun 已提交
2047

U
Ursula Braun 已提交
2048
	smc = smc_sk(sock->sk);
2049
	if (smc->use_fallback) {
2050
		/* delegate to CLC child sock */
2051
		mask = smc->clcsock->ops->poll(file, smc->clcsock, wait);
2052
		sk->sk_err = smc->clcsock->sk->sk_err;
U
Ursula Braun 已提交
2053
	} else {
2054
		if (sk->sk_state != SMC_CLOSED)
2055
			sock_poll_wait(file, sock, wait);
2056
		if (sk->sk_err)
2057
			mask |= EPOLLERR;
2058 2059
		if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
		    (sk->sk_state == SMC_CLOSED))
2060
			mask |= EPOLLHUP;
U
Ursula Braun 已提交
2061 2062
		if (sk->sk_state == SMC_LISTEN) {
			/* woken up by sk_data_ready in smc_listen_work() */
2063 2064 2065 2066 2067
			mask |= smc_accept_poll(sk);
		} else if (smc->use_fallback) { /* as result of connect_work()*/
			mask |= smc->clcsock->ops->poll(file, smc->clcsock,
							   wait);
			sk->sk_err = smc->clcsock->sk->sk_err;
U
Ursula Braun 已提交
2068
		} else {
2069 2070
			if ((sk->sk_state != SMC_INIT &&
			     atomic_read(&smc->conn.sndbuf_space)) ||
U
Ursula Braun 已提交
2071
			    sk->sk_shutdown & SEND_SHUTDOWN) {
2072
				mask |= EPOLLOUT | EPOLLWRNORM;
U
Ursula Braun 已提交
2073 2074 2075 2076 2077
			} else {
				sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
				set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
			}
			if (atomic_read(&smc->conn.bytes_to_rcv))
2078
				mask |= EPOLLIN | EPOLLRDNORM;
U
Ursula Braun 已提交
2079
			if (sk->sk_shutdown & RCV_SHUTDOWN)
2080
				mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
U
Ursula Braun 已提交
2081
			if (sk->sk_state == SMC_APPCLOSEWAIT1)
2082
				mask |= EPOLLIN;
2083 2084
			if (smc->conn.urg_state == SMC_URG_VALID)
				mask |= EPOLLPRI;
U
Ursula Braun 已提交
2085
		}
U
Ursula Braun 已提交
2086 2087 2088 2089 2090 2091 2092 2093 2094 2095
	}

	return mask;
}

static int smc_shutdown(struct socket *sock, int how)
{
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
	int rc = -EINVAL;
2096
	int rc1 = 0;
U
Ursula Braun 已提交
2097 2098 2099 2100

	smc = smc_sk(sk);

	if ((how < SHUT_RD) || (how > SHUT_RDWR))
2101
		return rc;
U
Ursula Braun 已提交
2102 2103 2104 2105

	lock_sock(sk);

	rc = -ENOTCONN;
2106
	if ((sk->sk_state != SMC_ACTIVE) &&
2107 2108 2109 2110 2111
	    (sk->sk_state != SMC_PEERCLOSEWAIT1) &&
	    (sk->sk_state != SMC_PEERCLOSEWAIT2) &&
	    (sk->sk_state != SMC_APPCLOSEWAIT1) &&
	    (sk->sk_state != SMC_APPCLOSEWAIT2) &&
	    (sk->sk_state != SMC_APPFINCLOSEWAIT))
U
Ursula Braun 已提交
2112 2113 2114 2115 2116 2117
		goto out;
	if (smc->use_fallback) {
		rc = kernel_sock_shutdown(smc->clcsock, how);
		sk->sk_shutdown = smc->clcsock->sk->sk_shutdown;
		if (sk->sk_shutdown == SHUTDOWN_MASK)
			sk->sk_state = SMC_CLOSED;
2118
		goto out;
U
Ursula Braun 已提交
2119
	}
2120 2121 2122 2123 2124 2125 2126 2127
	switch (how) {
	case SHUT_RDWR:		/* shutdown in both directions */
		rc = smc_close_active(smc);
		break;
	case SHUT_WR:
		rc = smc_close_shutdown_write(smc);
		break;
	case SHUT_RD:
2128 2129
		rc = 0;
		/* nothing more to do because peer is not involved */
2130 2131
		break;
	}
2132 2133
	if (smc->clcsock)
		rc1 = kernel_sock_shutdown(smc->clcsock, how);
2134 2135
	/* map sock_shutdown_cmd constants to sk_shutdown value range */
	sk->sk_shutdown |= how + 1;
U
Ursula Braun 已提交
2136 2137 2138

out:
	release_sock(sk);
2139
	return rc ? rc : rc1;
U
Ursula Braun 已提交
2140 2141 2142
}

static int smc_setsockopt(struct socket *sock, int level, int optname,
2143
			  sockptr_t optval, unsigned int optlen)
U
Ursula Braun 已提交
2144 2145 2146
{
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
2147
	int val, rc;
U
Ursula Braun 已提交
2148 2149 2150 2151 2152 2153

	smc = smc_sk(sk);

	/* generic setsockopts reaching us here always apply to the
	 * CLC socket
	 */
2154 2155 2156 2157 2158
	if (unlikely(!smc->clcsock->ops->setsockopt))
		rc = -EOPNOTSUPP;
	else
		rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname,
						   optval, optlen);
2159 2160 2161 2162 2163
	if (smc->clcsock->sk->sk_err) {
		sk->sk_err = smc->clcsock->sk->sk_err;
		sk->sk_error_report(sk);
	}

2164
	if (optlen < sizeof(int))
2165
		return -EINVAL;
2166
	if (copy_from_sockptr(&val, optval, sizeof(int)))
2167
		return -EFAULT;
2168

2169
	lock_sock(sk);
2170 2171
	if (rc || smc->use_fallback)
		goto out;
2172 2173 2174 2175 2176 2177 2178
	switch (optname) {
	case TCP_ULP:
	case TCP_FASTOPEN:
	case TCP_FASTOPEN_CONNECT:
	case TCP_FASTOPEN_KEY:
	case TCP_FASTOPEN_NO_COOKIE:
		/* option not supported by SMC */
2179
		if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
2180
			smc_switch_to_fallback(smc);
2181
			smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP;
2182
		} else {
2183
			rc = -EINVAL;
2184 2185
		}
		break;
2186
	case TCP_NODELAY:
2187 2188 2189
		if (sk->sk_state != SMC_INIT &&
		    sk->sk_state != SMC_LISTEN &&
		    sk->sk_state != SMC_CLOSED) {
2190
			if (val)
2191 2192
				mod_delayed_work(smc->conn.lgr->tx_wq,
						 &smc->conn.tx_work, 0);
2193 2194 2195
		}
		break;
	case TCP_CORK:
2196 2197 2198
		if (sk->sk_state != SMC_INIT &&
		    sk->sk_state != SMC_LISTEN &&
		    sk->sk_state != SMC_CLOSED) {
2199
			if (!val)
2200 2201
				mod_delayed_work(smc->conn.lgr->tx_wq,
						 &smc->conn.tx_work, 0);
2202 2203
		}
		break;
2204 2205 2206
	case TCP_DEFER_ACCEPT:
		smc->sockopt_defer_accept = val;
		break;
2207 2208 2209
	default:
		break;
	}
2210
out:
2211 2212 2213
	release_sock(sk);

	return rc;
U
Ursula Braun 已提交
2214 2215 2216 2217 2218 2219 2220 2221 2222
}

static int smc_getsockopt(struct socket *sock, int level, int optname,
			  char __user *optval, int __user *optlen)
{
	struct smc_sock *smc;

	smc = smc_sk(sock->sk);
	/* socket options apply to the CLC socket */
2223 2224
	if (unlikely(!smc->clcsock->ops->getsockopt))
		return -EOPNOTSUPP;
U
Ursula Braun 已提交
2225 2226 2227 2228 2229 2230 2231
	return smc->clcsock->ops->getsockopt(smc->clcsock, level, optname,
					     optval, optlen);
}

static int smc_ioctl(struct socket *sock, unsigned int cmd,
		     unsigned long arg)
{
S
Stefan Raspl 已提交
2232 2233
	union smc_host_cursor cons, urg;
	struct smc_connection *conn;
U
Ursula Braun 已提交
2234
	struct smc_sock *smc;
2235
	int answ;
U
Ursula Braun 已提交
2236 2237

	smc = smc_sk(sock->sk);
S
Stefan Raspl 已提交
2238
	conn = &smc->conn;
2239
	lock_sock(&smc->sk);
2240
	if (smc->use_fallback) {
2241 2242
		if (!smc->clcsock) {
			release_sock(&smc->sk);
2243
			return -EBADF;
2244 2245 2246 2247
		}
		answ = smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg);
		release_sock(&smc->sk);
		return answ;
2248 2249 2250
	}
	switch (cmd) {
	case SIOCINQ: /* same as FIONREAD */
2251 2252
		if (smc->sk.sk_state == SMC_LISTEN) {
			release_sock(&smc->sk);
2253
			return -EINVAL;
2254
		}
2255 2256 2257 2258 2259
		if (smc->sk.sk_state == SMC_INIT ||
		    smc->sk.sk_state == SMC_CLOSED)
			answ = 0;
		else
			answ = atomic_read(&smc->conn.bytes_to_rcv);
2260 2261 2262
		break;
	case SIOCOUTQ:
		/* output queue size (not send + not acked) */
2263 2264
		if (smc->sk.sk_state == SMC_LISTEN) {
			release_sock(&smc->sk);
2265
			return -EINVAL;
2266
		}
2267 2268 2269 2270 2271
		if (smc->sk.sk_state == SMC_INIT ||
		    smc->sk.sk_state == SMC_CLOSED)
			answ = 0;
		else
			answ = smc->conn.sndbuf_desc->len -
2272 2273 2274 2275
					atomic_read(&smc->conn.sndbuf_space);
		break;
	case SIOCOUTQNSD:
		/* output queue size (not send only) */
2276 2277
		if (smc->sk.sk_state == SMC_LISTEN) {
			release_sock(&smc->sk);
2278
			return -EINVAL;
2279
		}
2280 2281 2282 2283 2284
		if (smc->sk.sk_state == SMC_INIT ||
		    smc->sk.sk_state == SMC_CLOSED)
			answ = 0;
		else
			answ = smc_tx_prepared_sends(&smc->conn);
2285
		break;
S
Stefan Raspl 已提交
2286
	case SIOCATMARK:
2287 2288
		if (smc->sk.sk_state == SMC_LISTEN) {
			release_sock(&smc->sk);
S
Stefan Raspl 已提交
2289
			return -EINVAL;
2290
		}
S
Stefan Raspl 已提交
2291 2292 2293 2294
		if (smc->sk.sk_state == SMC_INIT ||
		    smc->sk.sk_state == SMC_CLOSED) {
			answ = 0;
		} else {
2295 2296
			smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
			smc_curs_copy(&urg, &conn->urg_curs, conn);
S
Stefan Raspl 已提交
2297 2298 2299 2300
			answ = smc_curs_diff(conn->rmb_desc->len,
					     &cons, &urg) == 1;
		}
		break;
2301
	default:
2302
		release_sock(&smc->sk);
2303 2304
		return -ENOIOCTLCMD;
	}
2305
	release_sock(&smc->sk);
2306 2307

	return put_user(answ, (int __user *)arg);
U
Ursula Braun 已提交
2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318
}

static ssize_t smc_sendpage(struct socket *sock, struct page *page,
			    int offset, size_t size, int flags)
{
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
	int rc = -EPIPE;

	smc = smc_sk(sk);
	lock_sock(sk);
S
Stefan Raspl 已提交
2319 2320
	if (sk->sk_state != SMC_ACTIVE) {
		release_sock(sk);
U
Ursula Braun 已提交
2321
		goto out;
S
Stefan Raspl 已提交
2322 2323
	}
	release_sock(sk);
U
Ursula Braun 已提交
2324 2325 2326 2327 2328 2329 2330 2331 2332 2333
	if (smc->use_fallback)
		rc = kernel_sendpage(smc->clcsock, page, offset,
				     size, flags);
	else
		rc = sock_no_sendpage(sock, page, offset, size, flags);

out:
	return rc;
}

S
Stefan Raspl 已提交
2334 2335 2336 2337 2338 2339
/* Map the affected portions of the rmbe into an spd, note the number of bytes
 * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor
 * updates till whenever a respective page has been fully processed.
 * Note that subsequent recv() calls have to wait till all splice() processing
 * completed.
 */
U
Ursula Braun 已提交
2340 2341
static ssize_t smc_splice_read(struct socket *sock, loff_t *ppos,
			       struct pipe_inode_info *pipe, size_t len,
S
Stefan Raspl 已提交
2342
			       unsigned int flags)
U
Ursula Braun 已提交
2343 2344 2345 2346 2347 2348 2349
{
	struct sock *sk = sock->sk;
	struct smc_sock *smc;
	int rc = -ENOTCONN;

	smc = smc_sk(sk);
	lock_sock(sk);
2350 2351 2352 2353 2354
	if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
		/* socket was connected before, no more data to read */
		rc = 0;
		goto out;
	}
S
Stefan Raspl 已提交
2355 2356 2357 2358 2359 2360 2361
	if (sk->sk_state == SMC_INIT ||
	    sk->sk_state == SMC_LISTEN ||
	    sk->sk_state == SMC_CLOSED)
		goto out;

	if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
		rc = 0;
U
Ursula Braun 已提交
2362
		goto out;
S
Stefan Raspl 已提交
2363 2364
	}

U
Ursula Braun 已提交
2365 2366 2367 2368
	if (smc->use_fallback) {
		rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos,
						    pipe, len, flags);
	} else {
S
Stefan Raspl 已提交
2369 2370 2371 2372 2373 2374 2375 2376 2377
		if (*ppos) {
			rc = -ESPIPE;
			goto out;
		}
		if (flags & SPLICE_F_NONBLOCK)
			flags = MSG_DONTWAIT;
		else
			flags = 0;
		rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags);
U
Ursula Braun 已提交
2378 2379 2380
	}
out:
	release_sock(sk);
S
Stefan Raspl 已提交
2381

U
Ursula Braun 已提交
2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394
	return rc;
}

/* must look like tcp */
static const struct proto_ops smc_sock_ops = {
	.family		= PF_SMC,
	.owner		= THIS_MODULE,
	.release	= smc_release,
	.bind		= smc_bind,
	.connect	= smc_connect,
	.socketpair	= sock_no_socketpair,
	.accept		= smc_accept,
	.getname	= smc_getname,
2395
	.poll		= smc_poll,
U
Ursula Braun 已提交
2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410
	.ioctl		= smc_ioctl,
	.listen		= smc_listen,
	.shutdown	= smc_shutdown,
	.setsockopt	= smc_setsockopt,
	.getsockopt	= smc_getsockopt,
	.sendmsg	= smc_sendmsg,
	.recvmsg	= smc_recvmsg,
	.mmap		= sock_no_mmap,
	.sendpage	= smc_sendpage,
	.splice_read	= smc_splice_read,
};

static int smc_create(struct net *net, struct socket *sock, int protocol,
		      int kern)
{
2411
	int family = (protocol == SMCPROTO_SMC6) ? PF_INET6 : PF_INET;
U
Ursula Braun 已提交
2412 2413 2414 2415 2416 2417 2418 2419 2420
	struct smc_sock *smc;
	struct sock *sk;
	int rc;

	rc = -ESOCKTNOSUPPORT;
	if (sock->type != SOCK_STREAM)
		goto out;

	rc = -EPROTONOSUPPORT;
2421
	if (protocol != SMCPROTO_SMC && protocol != SMCPROTO_SMC6)
U
Ursula Braun 已提交
2422 2423 2424 2425
		goto out;

	rc = -ENOBUFS;
	sock->ops = &smc_sock_ops;
2426
	sk = smc_sock_alloc(net, sock, protocol);
U
Ursula Braun 已提交
2427 2428 2429 2430 2431
	if (!sk)
		goto out;

	/* create internal TCP socket for CLC handshake and fallback */
	smc = smc_sk(sk);
2432
	smc->use_fallback = false; /* assume rdma capability first */
2433
	smc->fallback_rsn = 0;
2434 2435
	rc = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP,
			      &smc->clcsock);
2436
	if (rc) {
U
Ursula Braun 已提交
2437
		sk_common_release(sk);
2438 2439
		goto out;
	}
U
Ursula Braun 已提交
2440 2441
	smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE);
	smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE);
U
Ursula Braun 已提交
2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452

out:
	return rc;
}

static const struct net_proto_family smc_sock_family_ops = {
	.family	= PF_SMC,
	.owner	= THIS_MODULE,
	.create	= smc_create,
};

2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471
unsigned int smc_net_id;

static __net_init int smc_net_init(struct net *net)
{
	return smc_pnet_net_init(net);
}

static void __net_exit smc_net_exit(struct net *net)
{
	smc_pnet_net_exit(net);
}

static struct pernet_operations smc_net_ops = {
	.init = smc_net_init,
	.exit = smc_net_exit,
	.id   = &smc_net_id,
	.size = sizeof(struct smc_net),
};

U
Ursula Braun 已提交
2472 2473 2474 2475
static int __init smc_init(void)
{
	int rc;

2476 2477 2478 2479
	rc = register_pernet_subsys(&smc_net_ops);
	if (rc)
		return rc;

2480
	smc_ism_init();
2481
	smc_clc_init();
2482

2483 2484
	rc = smc_pnet_init();
	if (rc)
Y
YueHaibing 已提交
2485
		goto out_pernet_subsys;
2486

2487 2488 2489 2490 2491 2492 2493 2494 2495
	rc = -ENOMEM;
	smc_hs_wq = alloc_workqueue("smc_hs_wq", 0, 0);
	if (!smc_hs_wq)
		goto out_pnet;

	smc_close_wq = alloc_workqueue("smc_close_wq", 0, 0);
	if (!smc_close_wq)
		goto out_alloc_hs_wq;

2496 2497 2498
	rc = smc_core_init();
	if (rc) {
		pr_err("%s: smc_core_init fails with %d\n", __func__, rc);
2499
		goto out_alloc_wqs;
2500 2501
	}

U
Ursula Braun 已提交
2502 2503 2504
	rc = smc_llc_init();
	if (rc) {
		pr_err("%s: smc_llc_init fails with %d\n", __func__, rc);
2505
		goto out_core;
U
Ursula Braun 已提交
2506 2507
	}

2508 2509 2510
	rc = smc_cdc_init();
	if (rc) {
		pr_err("%s: smc_cdc_init fails with %d\n", __func__, rc);
2511
		goto out_core;
2512 2513
	}

U
Ursula Braun 已提交
2514 2515
	rc = proto_register(&smc_proto, 1);
	if (rc) {
2516
		pr_err("%s: proto_register(v4) fails with %d\n", __func__, rc);
2517
		goto out_core;
U
Ursula Braun 已提交
2518 2519
	}

2520 2521 2522 2523 2524 2525
	rc = proto_register(&smc_proto6, 1);
	if (rc) {
		pr_err("%s: proto_register(v6) fails with %d\n", __func__, rc);
		goto out_proto;
	}

U
Ursula Braun 已提交
2526 2527 2528
	rc = sock_register(&smc_sock_family_ops);
	if (rc) {
		pr_err("%s: sock_register fails with %d\n", __func__, rc);
2529
		goto out_proto6;
U
Ursula Braun 已提交
2530
	}
2531
	INIT_HLIST_HEAD(&smc_v4_hashinfo.ht);
2532
	INIT_HLIST_HEAD(&smc_v6_hashinfo.ht);
U
Ursula Braun 已提交
2533

2534 2535 2536 2537 2538 2539
	rc = smc_ib_register_client();
	if (rc) {
		pr_err("%s: ib_register fails with %d\n", __func__, rc);
		goto out_sock;
	}

U
Ursula Braun 已提交
2540
	static_branch_enable(&tcp_have_smc);
U
Ursula Braun 已提交
2541 2542
	return 0;

2543 2544
out_sock:
	sock_unregister(PF_SMC);
2545 2546
out_proto6:
	proto_unregister(&smc_proto6);
U
Ursula Braun 已提交
2547 2548
out_proto:
	proto_unregister(&smc_proto);
2549 2550
out_core:
	smc_core_exit();
2551 2552 2553 2554
out_alloc_wqs:
	destroy_workqueue(smc_close_wq);
out_alloc_hs_wq:
	destroy_workqueue(smc_hs_wq);
2555 2556
out_pnet:
	smc_pnet_exit();
Y
YueHaibing 已提交
2557 2558 2559
out_pernet_subsys:
	unregister_pernet_subsys(&smc_net_ops);

U
Ursula Braun 已提交
2560 2561 2562 2563 2564
	return rc;
}

static void __exit smc_exit(void)
{
U
Ursula Braun 已提交
2565
	static_branch_disable(&tcp_have_smc);
U
Ursula Braun 已提交
2566
	sock_unregister(PF_SMC);
2567 2568
	smc_core_exit();
	smc_ib_unregister_client();
2569 2570
	destroy_workqueue(smc_close_wq);
	destroy_workqueue(smc_hs_wq);
2571
	proto_unregister(&smc_proto6);
U
Ursula Braun 已提交
2572
	proto_unregister(&smc_proto);
2573
	smc_pnet_exit();
2574
	unregister_pernet_subsys(&smc_net_ops);
2575
	rcu_barrier();
U
Ursula Braun 已提交
2576 2577 2578 2579 2580 2581 2582 2583 2584
}

module_init(smc_init);
module_exit(smc_exit);

MODULE_AUTHOR("Ursula Braun <ubraun@linux.vnet.ibm.com>");
MODULE_DESCRIPTION("smc socket address family");
MODULE_LICENSE("GPL");
MODULE_ALIAS_NETPROTO(PF_SMC);